From 106b4a2a78b2a2f14885604891957b84bba502db Mon Sep 17 00:00:00 2001 From: Adrian Betschart Date: Fri, 11 Sep 2026 17:13:14 +0200 Subject: drm/amd/display: Try RGB before YCbCr 4:4:4 in stream validation amdgpu_dm_create_validate_stream_for_sink() walks encoding_order[] and uses the first encoding that validates. YCbCr 4:4:4 is listed before RGB, so an HDMI sink that advertises 4:4:4 gets YCbCr 4:4:4 whenever the "color format" property is left at AUTO, even though RGB fits the same link. That contradicts the documented AUTO behaviour for HDMI in enum drm_connector_color_format (RGB, falling back to YCbCr 4:2:0 only when the bandwidth is not available or the mode is 4:2:0-only), which the amdgpu implementation of the property also describes. It also leaves the "Broadcast RGB" property without effect on such sinks, since the quantization range it selects only applies to RGB output. Try RGB first. The mask still holds every encoding the sink supports, so a mode that cannot carry RGB falls back exactly as before. For reference, v7.2 picked RGB here unless YCbCr 4:4:4 was forced through debugfs, while earlier kernels picked YCbCr 4:4:4 for any HDMI sink that advertised it. Fixes: 0b0ff65d3ca1 ("drm/amd/display: Refactor stream validation") Suggested-by: Adolfo Rodrigues Assisted-by: Claude Code:claude-fable-5-1 Signed-off-by: Adrian Betschart Reviewed-by: Fangzhi Zuo Tested-by: Adolfo Rodrigues Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher (cherry picked from commit 7de432bc753133764dc40d37f9388da56de251e7) --- drivers/crypto/Kconfig | 834 +++ drivers/crypto/Makefile | 48 + drivers/crypto/allwinner/Kconfig | 115 + drivers/crypto/allwinner/Makefile | 3 + drivers/crypto/allwinner/sun4i-ss/Makefile | 3 + .../crypto/allwinner/sun4i-ss/sun4i-ss-cipher.c | 644 ++ drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c | 522 ++ drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c | 545 ++ drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h | 216 + drivers/crypto/allwinner/sun8i-ce/Makefile | 4 + .../crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c | 512 ++ drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c | 1075 ++++ drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c | 486 ++ drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c | 122 + drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h | 379 ++ drivers/crypto/allwinner/sun8i-ss/Makefile | 3 + .../crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c | 486 ++ drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c | 909 +++ drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c | 705 +++ drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h | 297 + drivers/crypto/amcc/Makefile | 4 + drivers/crypto/amcc/crypto4xx_alg.c | 601 ++ drivers/crypto/amcc/crypto4xx_core.c | 1367 +++++ drivers/crypto/amcc/crypto4xx_core.h | 235 + drivers/crypto/amcc/crypto4xx_reg_def.h | 271 + drivers/crypto/amcc/crypto4xx_sa.h | 309 + drivers/crypto/amcc/crypto4xx_trng.c | 129 + drivers/crypto/amcc/crypto4xx_trng.h | 25 + drivers/crypto/amlogic/Kconfig | 25 + drivers/crypto/amlogic/Makefile | 2 + drivers/crypto/amlogic/amlogic-gxl-cipher.c | 371 ++ drivers/crypto/amlogic/amlogic-gxl-core.c | 332 + drivers/crypto/amlogic/amlogic-gxl.h | 159 + drivers/crypto/aspeed/Kconfig | 57 + drivers/crypto/aspeed/Makefile | 11 + drivers/crypto/aspeed/aspeed-acry.c | 816 +++ drivers/crypto/aspeed/aspeed-hace-crypto.c | 955 +++ drivers/crypto/aspeed/aspeed-hace-hash.c | 825 +++ drivers/crypto/aspeed/aspeed-hace.c | 278 + drivers/crypto/aspeed/aspeed-hace.h | 274 + drivers/crypto/atmel-aes-regs.h | 93 + drivers/crypto/atmel-aes.c | 2459 ++++++++ drivers/crypto/atmel-authenc.h | 53 + drivers/crypto/atmel-ecc.c | 391 ++ drivers/crypto/atmel-i2c.c | 418 ++ drivers/crypto/atmel-i2c.h | 192 + drivers/crypto/atmel-sha-regs.h | 76 + drivers/crypto/atmel-sha.c | 2690 +++++++++ drivers/crypto/atmel-sha204a.c | 258 + drivers/crypto/atmel-tdes-regs.h | 92 + drivers/crypto/atmel-tdes.c | 1060 ++++ drivers/crypto/axis/Makefile | 2 + drivers/crypto/axis/artpec6_crypto.c | 2980 +++++++++ drivers/crypto/bcm/Makefile | 14 + drivers/crypto/bcm/cipher.c | 4709 +++++++++++++++ drivers/crypto/bcm/cipher.h | 469 ++ drivers/crypto/bcm/spu.c | 1209 ++++ drivers/crypto/bcm/spu.h | 276 + drivers/crypto/bcm/spu2.c | 1385 +++++ drivers/crypto/bcm/spu2.h | 216 + drivers/crypto/bcm/spum.h | 163 + drivers/crypto/bcm/util.c | 530 ++ drivers/crypto/bcm/util.h | 113 + drivers/crypto/caam/Kconfig | 188 + drivers/crypto/caam/Makefile | 32 + drivers/crypto/caam/blob_gen.c | 246 + drivers/crypto/caam/caamalg.c | 3988 ++++++++++++ drivers/crypto/caam/caamalg_desc.c | 1725 ++++++ drivers/crypto/caam/caamalg_desc.h | 127 + drivers/crypto/caam/caamalg_qi.c | 2743 +++++++++ drivers/crypto/caam/caamalg_qi2.c | 5565 +++++++++++++++++ drivers/crypto/caam/caamalg_qi2.h | 200 + drivers/crypto/caam/caamhash.c | 2029 +++++++ drivers/crypto/caam/caamhash_desc.c | 145 + drivers/crypto/caam/caamhash_desc.h | 29 + drivers/crypto/caam/caampkc.c | 1230 ++++ drivers/crypto/caam/caampkc.h | 155 + drivers/crypto/caam/caamrng.c | 313 + drivers/crypto/caam/compat.h | 52 + drivers/crypto/caam/ctrl.c | 1172 ++++ drivers/crypto/caam/ctrl.h | 14 + drivers/crypto/caam/debugfs.c | 98 + drivers/crypto/caam/debugfs.h | 29 + drivers/crypto/caam/desc.h | 1694 ++++++ drivers/crypto/caam/desc_constr.h | 612 ++ drivers/crypto/caam/dpseci-debugfs.c | 60 + drivers/crypto/caam/dpseci-debugfs.h | 18 + drivers/crypto/caam/dpseci.c | 444 ++ drivers/crypto/caam/dpseci.h | 335 ++ drivers/crypto/caam/dpseci_cmd.h | 150 + drivers/crypto/caam/error.c | 393 ++ drivers/crypto/caam/error.h | 29 + drivers/crypto/caam/intern.h | 251 + drivers/crypto/caam/jr.c | 835 +++ drivers/crypto/caam/jr.h | 19 + drivers/crypto/caam/key_gen.c | 126 + drivers/crypto/caam/key_gen.h | 48 + drivers/crypto/caam/pdb.h | 599 ++ drivers/crypto/caam/pkc_desc.c | 73 + drivers/crypto/caam/qi.c | 813 +++ drivers/crypto/caam/qi.h | 183 + drivers/crypto/caam/regs.h | 1036 ++++ drivers/crypto/caam/sg_sw_qm.h | 85 + drivers/crypto/caam/sg_sw_qm2.h | 57 + drivers/crypto/caam/sg_sw_sec4.h | 85 + drivers/crypto/cavium/Makefile | 6 + drivers/crypto/cavium/cpt/Kconfig | 18 + drivers/crypto/cavium/cpt/Makefile | 4 + drivers/crypto/cavium/cpt/cpt_common.h | 153 + drivers/crypto/cavium/cpt/cpt_hw_types.h | 655 ++ drivers/crypto/cavium/cpt/cptpf.h | 61 + drivers/crypto/cavium/cpt/cptpf_main.c | 669 ++ drivers/crypto/cavium/cpt/cptpf_mbox.c | 160 + drivers/crypto/cavium/cpt/cptvf.h | 129 + drivers/crypto/cavium/cpt/cptvf_algs.c | 462 ++ drivers/crypto/cavium/cpt/cptvf_algs.h | 117 + drivers/crypto/cavium/cpt/cptvf_main.c | 856 +++ drivers/crypto/cavium/cpt/cptvf_mbox.c | 191 + drivers/crypto/cavium/cpt/cptvf_reqmanager.c | 575 ++ drivers/crypto/cavium/cpt/request_manager.h | 146 + drivers/crypto/cavium/nitrox/Kconfig | 21 + drivers/crypto/cavium/nitrox/Makefile | 15 + drivers/crypto/cavium/nitrox/nitrox_aead.c | 567 ++ drivers/crypto/cavium/nitrox/nitrox_algs.c | 24 + drivers/crypto/cavium/nitrox/nitrox_common.h | 30 + drivers/crypto/cavium/nitrox/nitrox_csr.h | 1439 +++++ drivers/crypto/cavium/nitrox/nitrox_debugfs.c | 70 + drivers/crypto/cavium/nitrox/nitrox_debugfs.h | 20 + drivers/crypto/cavium/nitrox/nitrox_dev.h | 302 + drivers/crypto/cavium/nitrox/nitrox_hal.c | 680 +++ drivers/crypto/cavium/nitrox/nitrox_hal.h | 29 + drivers/crypto/cavium/nitrox/nitrox_isr.c | 458 ++ drivers/crypto/cavium/nitrox/nitrox_isr.h | 21 + drivers/crypto/cavium/nitrox/nitrox_lib.c | 300 + drivers/crypto/cavium/nitrox/nitrox_main.c | 582 ++ drivers/crypto/cavium/nitrox/nitrox_mbx.c | 217 + drivers/crypto/cavium/nitrox/nitrox_mbx.h | 9 + drivers/crypto/cavium/nitrox/nitrox_req.h | 728 +++ drivers/crypto/cavium/nitrox/nitrox_reqmgr.c | 607 ++ drivers/crypto/cavium/nitrox/nitrox_skcipher.c | 531 ++ drivers/crypto/cavium/nitrox/nitrox_sriov.c | 234 + drivers/crypto/ccp/Kconfig | 56 + drivers/crypto/ccp/Makefile | 31 + drivers/crypto/ccp/ccp-crypto-aes-cmac.c | 402 ++ drivers/crypto/ccp/ccp-crypto-aes-galois.c | 258 + drivers/crypto/ccp/ccp-crypto-aes-xts.c | 288 + drivers/crypto/ccp/ccp-crypto-aes.c | 343 ++ drivers/crypto/ccp/ccp-crypto-des3.c | 228 + drivers/crypto/ccp/ccp-crypto-main.c | 426 ++ drivers/crypto/ccp/ccp-crypto-rsa.c | 293 + drivers/crypto/ccp/ccp-crypto-sha.c | 529 ++ drivers/crypto/ccp/ccp-crypto.h | 286 + drivers/crypto/ccp/ccp-debugfs.c | 326 + drivers/crypto/ccp/ccp-dev-v3.c | 597 ++ drivers/crypto/ccp/ccp-dev-v5.c | 1128 ++++ drivers/crypto/ccp/ccp-dev.c | 670 +++ drivers/crypto/ccp/ccp-dev.h | 673 +++ drivers/crypto/ccp/ccp-dmaengine.c | 791 +++ drivers/crypto/ccp/ccp-ops.c | 2525 ++++++++ drivers/crypto/ccp/dbc.c | 259 + drivers/crypto/ccp/dbc.h | 47 + drivers/crypto/ccp/hsti.c | 141 + drivers/crypto/ccp/hsti.h | 17 + drivers/crypto/ccp/platform-access.c | 227 + drivers/crypto/ccp/platform-access.h | 35 + drivers/crypto/ccp/psp-dev.c | 381 ++ drivers/crypto/ccp/psp-dev.h | 137 + drivers/crypto/ccp/sev-dev-tio.c | 864 +++ drivers/crypto/ccp/sev-dev-tio.h | 123 + drivers/crypto/ccp/sev-dev-tsm.c | 399 ++ drivers/crypto/ccp/sev-dev.c | 3168 ++++++++++ drivers/crypto/ccp/sev-dev.h | 89 + drivers/crypto/ccp/sfs.c | 311 + drivers/crypto/ccp/sfs.h | 47 + drivers/crypto/ccp/sp-dev.c | 325 + drivers/crypto/ccp/sp-dev.h | 190 + drivers/crypto/ccp/sp-pci.c | 600 ++ drivers/crypto/ccp/sp-platform.c | 217 + drivers/crypto/ccp/tee-dev.c | 399 ++ drivers/crypto/ccp/tee-dev.h | 116 + drivers/crypto/ccree/Makefile | 8 + drivers/crypto/ccree/cc_aead.c | 2667 ++++++++ drivers/crypto/ccree/cc_aead.h | 108 + drivers/crypto/ccree/cc_buffer_mgr.c | 1397 +++++ drivers/crypto/ccree/cc_buffer_mgr.h | 70 + drivers/crypto/ccree/cc_cipher.c | 1471 +++++ drivers/crypto/ccree/cc_cipher.h | 39 + drivers/crypto/ccree/cc_crypto_ctx.h | 142 + drivers/crypto/ccree/cc_debugfs.c | 107 + drivers/crypto/ccree/cc_debugfs.h | 28 + drivers/crypto/ccree/cc_driver.c | 674 +++ drivers/crypto/ccree/cc_driver.h | 235 + drivers/crypto/ccree/cc_fips.c | 154 + drivers/crypto/ccree/cc_fips.h | 38 + drivers/crypto/ccree/cc_hash.c | 2313 +++++++ drivers/crypto/ccree/cc_hash.h | 106 + drivers/crypto/ccree/cc_host_regs.h | 282 + drivers/crypto/ccree/cc_hw_queue_defs.h | 633 ++ drivers/crypto/ccree/cc_kernel_regs.h | 168 + drivers/crypto/ccree/cc_lli_defs.h | 59 + drivers/crypto/ccree/cc_pm.c | 81 + drivers/crypto/ccree/cc_pm.h | 32 + drivers/crypto/ccree/cc_request_mgr.c | 662 ++ drivers/crypto/ccree/cc_request_mgr.h | 42 + drivers/crypto/ccree/cc_sram_mgr.c | 91 + drivers/crypto/ccree/cc_sram_mgr.h | 52 + drivers/crypto/chelsio/Kconfig | 23 + drivers/crypto/chelsio/Makefile | 5 + drivers/crypto/chelsio/chcr_algo.c | 4401 ++++++++++++++ drivers/crypto/chelsio/chcr_algo.h | 405 ++ drivers/crypto/chelsio/chcr_core.c | 311 + drivers/crypto/chelsio/chcr_core.h | 138 + drivers/crypto/chelsio/chcr_crypto.h | 351 ++ drivers/crypto/gemini/Makefile | 2 + drivers/crypto/gemini/sl3516-ce-cipher.c | 381 ++ drivers/crypto/gemini/sl3516-ce-core.c | 541 ++ drivers/crypto/gemini/sl3516-ce-rng.c | 61 + drivers/crypto/gemini/sl3516-ce.h | 341 ++ drivers/crypto/geode-aes.c | 436 ++ drivers/crypto/geode-aes.h | 58 + drivers/crypto/hifn_795x.c | 2560 ++++++++ drivers/crypto/hisilicon/Kconfig | 77 + drivers/crypto/hisilicon/Makefile | 7 + drivers/crypto/hisilicon/debugfs.c | 1272 ++++ drivers/crypto/hisilicon/hpre/Makefile | 2 + drivers/crypto/hisilicon/hpre/hpre.h | 129 + drivers/crypto/hisilicon/hpre/hpre_crypto.c | 1897 ++++++ drivers/crypto/hisilicon/hpre/hpre_main.c | 1754 ++++++ drivers/crypto/hisilicon/qm.c | 6362 ++++++++++++++++++++ drivers/crypto/hisilicon/qm_common.h | 79 + drivers/crypto/hisilicon/sec/Makefile | 3 + drivers/crypto/hisilicon/sec/sec_algs.c | 1122 ++++ drivers/crypto/hisilicon/sec/sec_drv.c | 1307 ++++ drivers/crypto/hisilicon/sec/sec_drv.h | 428 ++ drivers/crypto/hisilicon/sec2/Makefile | 2 + drivers/crypto/hisilicon/sec2/sec.h | 286 + drivers/crypto/hisilicon/sec2/sec_crypto.c | 2780 +++++++++ drivers/crypto/hisilicon/sec2/sec_crypto.h | 397 ++ drivers/crypto/hisilicon/sec2/sec_main.c | 1575 +++++ drivers/crypto/hisilicon/sgl.c | 290 + drivers/crypto/hisilicon/zip/Makefile | 2 + drivers/crypto/hisilicon/zip/dae_main.c | 277 + drivers/crypto/hisilicon/zip/zip.h | 114 + drivers/crypto/hisilicon/zip/zip_crypto.c | 727 +++ drivers/crypto/hisilicon/zip/zip_main.c | 1680 ++++++ drivers/crypto/img-hash.c | 1082 ++++ drivers/crypto/inside-secure/Makefile | 4 + drivers/crypto/inside-secure/eip93/Kconfig | 20 + drivers/crypto/inside-secure/eip93/Makefile | 5 + drivers/crypto/inside-secure/eip93/eip93-aead.c | 711 +++ drivers/crypto/inside-secure/eip93/eip93-aead.h | 38 + drivers/crypto/inside-secure/eip93/eip93-aes.h | 16 + drivers/crypto/inside-secure/eip93/eip93-cipher.c | 413 ++ drivers/crypto/inside-secure/eip93/eip93-cipher.h | 60 + drivers/crypto/inside-secure/eip93/eip93-common.c | 821 +++ drivers/crypto/inside-secure/eip93/eip93-common.h | 24 + drivers/crypto/inside-secure/eip93/eip93-des.h | 16 + drivers/crypto/inside-secure/eip93/eip93-hash.c | 875 +++ drivers/crypto/inside-secure/eip93/eip93-hash.h | 82 + drivers/crypto/inside-secure/eip93/eip93-main.c | 512 ++ drivers/crypto/inside-secure/eip93/eip93-main.h | 151 + drivers/crypto/inside-secure/eip93/eip93-regs.h | 335 ++ drivers/crypto/inside-secure/safexcel.c | 2044 +++++++ drivers/crypto/inside-secure/safexcel.h | 1002 +++ drivers/crypto/inside-secure/safexcel_cipher.c | 3758 ++++++++++++ drivers/crypto/inside-secure/safexcel_hash.c | 3012 +++++++++ drivers/crypto/inside-secure/safexcel_ring.c | 254 + drivers/crypto/intel/Kconfig | 6 + drivers/crypto/intel/Makefile | 6 + drivers/crypto/intel/iaa/Kconfig | 19 + drivers/crypto/intel/iaa/Makefile | 12 + drivers/crypto/intel/iaa/iaa_crypto.h | 148 + drivers/crypto/intel/iaa/iaa_crypto_comp_fixed.c | 91 + drivers/crypto/intel/iaa/iaa_crypto_main.c | 2035 +++++++ drivers/crypto/intel/iaa/iaa_crypto_stats.c | 280 + drivers/crypto/intel/iaa/iaa_crypto_stats.h | 47 + drivers/crypto/intel/ixp4xx/Kconfig | 14 + drivers/crypto/intel/ixp4xx/Makefile | 2 + drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c | 1609 +++++ drivers/crypto/intel/keembay/Kconfig | 90 + drivers/crypto/intel/keembay/Makefile | 10 + .../crypto/intel/keembay/keembay-ocs-aes-core.c | 1689 ++++++ drivers/crypto/intel/keembay/keembay-ocs-ecc.c | 997 +++ .../crypto/intel/keembay/keembay-ocs-hcu-core.c | 1261 ++++ drivers/crypto/intel/keembay/ocs-aes.c | 1490 +++++ drivers/crypto/intel/keembay/ocs-aes.h | 129 + drivers/crypto/intel/keembay/ocs-hcu.c | 841 +++ drivers/crypto/intel/keembay/ocs-hcu.h | 106 + drivers/crypto/intel/qat/Kconfig | 135 + drivers/crypto/intel/qat/Makefile | 12 + drivers/crypto/intel/qat/qat_420xx/Makefile | 3 + .../crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c | 506 ++ .../crypto/intel/qat/qat_420xx/adf_420xx_hw_data.h | 55 + drivers/crypto/intel/qat/qat_420xx/adf_drv.c | 212 + drivers/crypto/intel/qat/qat_4xxx/Makefile | 3 + .../crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c | 491 ++ .../crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.h | 62 + drivers/crypto/intel/qat/qat_4xxx/adf_drv.c | 216 + drivers/crypto/intel/qat/qat_6xxx/Makefile | 3 + .../crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c | 1085 ++++ .../crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.h | 197 + drivers/crypto/intel/qat/qat_6xxx/adf_drv.c | 259 + drivers/crypto/intel/qat/qat_c3xxx/Makefile | 3 + .../crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 167 + .../crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.h | 35 + drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c | 258 + drivers/crypto/intel/qat/qat_c3xxxvf/Makefile | 3 + .../intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c | 101 + .../intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h | 18 + drivers/crypto/intel/qat/qat_c3xxxvf/adf_drv.c | 227 + drivers/crypto/intel/qat/qat_c62x/Makefile | 3 + .../crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c | 169 + .../crypto/intel/qat/qat_c62x/adf_c62x_hw_data.h | 35 + drivers/crypto/intel/qat/qat_c62x/adf_drv.c | 258 + drivers/crypto/intel/qat/qat_c62xvf/Makefile | 3 + .../intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c | 101 + .../intel/qat/qat_c62xvf/adf_c62xvf_hw_data.h | 18 + drivers/crypto/intel/qat/qat_c62xvf/adf_drv.c | 227 + drivers/crypto/intel/qat/qat_common/Makefile | 72 + .../intel/qat/qat_common/adf_accel_devices.h | 501 ++ .../crypto/intel/qat/qat_common/adf_accel_engine.c | 219 + drivers/crypto/intel/qat/qat_common/adf_admin.c | 719 +++ drivers/crypto/intel/qat/qat_common/adf_admin.h | 35 + drivers/crypto/intel/qat/qat_common/adf_aer.c | 344 ++ drivers/crypto/intel/qat/qat_common/adf_anti_rb.c | 66 + drivers/crypto/intel/qat/qat_common/adf_anti_rb.h | 37 + .../crypto/intel/qat/qat_common/adf_bank_state.c | 238 + .../crypto/intel/qat/qat_common/adf_bank_state.h | 49 + drivers/crypto/intel/qat/qat_common/adf_cfg.c | 372 ++ drivers/crypto/intel/qat/qat_common/adf_cfg.h | 46 + .../crypto/intel/qat/qat_common/adf_cfg_common.h | 46 + .../crypto/intel/qat/qat_common/adf_cfg_services.c | 208 + .../crypto/intel/qat/qat_common/adf_cfg_services.h | 44 + .../crypto/intel/qat/qat_common/adf_cfg_strings.h | 51 + drivers/crypto/intel/qat/qat_common/adf_clock.c | 135 + drivers/crypto/intel/qat/qat_common/adf_clock.h | 14 + .../crypto/intel/qat/qat_common/adf_cnv_dbgfs.c | 299 + .../crypto/intel/qat/qat_common/adf_cnv_dbgfs.h | 11 + .../crypto/intel/qat/qat_common/adf_common_drv.h | 262 + drivers/crypto/intel/qat/qat_common/adf_dbgfs.c | 79 + drivers/crypto/intel/qat/qat_common/adf_dbgfs.h | 29 + drivers/crypto/intel/qat/qat_common/adf_dc.c | 64 + drivers/crypto/intel/qat/qat_common/adf_dc.h | 17 + drivers/crypto/intel/qat/qat_common/adf_dev_mgr.c | 372 ++ .../crypto/intel/qat/qat_common/adf_fw_config.h | 20 + .../crypto/intel/qat/qat_common/adf_fw_counters.c | 265 + .../crypto/intel/qat/qat_common/adf_fw_counters.h | 11 + .../crypto/intel/qat/qat_common/adf_gen2_config.c | 209 + .../crypto/intel/qat/qat_common/adf_gen2_config.h | 10 + .../intel/qat/qat_common/adf_gen2_hw_csr_data.c | 101 + .../intel/qat/qat_common/adf_gen2_hw_csr_data.h | 86 + .../crypto/intel/qat/qat_common/adf_gen2_hw_data.c | 228 + .../crypto/intel/qat/qat_common/adf_gen2_hw_data.h | 93 + .../crypto/intel/qat/qat_common/adf_gen2_pfvf.c | 401 ++ .../crypto/intel/qat/qat_common/adf_gen2_pfvf.h | 30 + .../crypto/intel/qat/qat_common/adf_gen4_config.c | 276 + .../crypto/intel/qat/qat_common/adf_gen4_config.h | 14 + .../intel/qat/qat_common/adf_gen4_hw_csr_data.c | 231 + .../intel/qat/qat_common/adf_gen4_hw_csr_data.h | 188 + .../crypto/intel/qat/qat_common/adf_gen4_hw_data.c | 610 ++ .../crypto/intel/qat/qat_common/adf_gen4_hw_data.h | 182 + .../crypto/intel/qat/qat_common/adf_gen4_pfvf.c | 145 + .../crypto/intel/qat/qat_common/adf_gen4_pfvf.h | 18 + drivers/crypto/intel/qat/qat_common/adf_gen4_pm.c | 164 + drivers/crypto/intel/qat/qat_common/adf_gen4_pm.h | 93 + .../intel/qat/qat_common/adf_gen4_pm_debugfs.c | 199 + drivers/crypto/intel/qat/qat_common/adf_gen4_ras.c | 1517 +++++ drivers/crypto/intel/qat/qat_common/adf_gen4_ras.h | 825 +++ drivers/crypto/intel/qat/qat_common/adf_gen4_tl.c | 154 + drivers/crypto/intel/qat/qat_common/adf_gen4_tl.h | 158 + .../crypto/intel/qat/qat_common/adf_gen4_vf_mig.c | 1011 ++++ .../crypto/intel/qat/qat_common/adf_gen4_vf_mig.h | 10 + drivers/crypto/intel/qat/qat_common/adf_gen6_pm.h | 52 + .../intel/qat/qat_common/adf_gen6_pm_dbgfs.c | 124 + drivers/crypto/intel/qat/qat_common/adf_gen6_ras.c | 818 +++ drivers/crypto/intel/qat/qat_common/adf_gen6_ras.h | 504 ++ .../crypto/intel/qat/qat_common/adf_gen6_shared.c | 50 + .../crypto/intel/qat/qat_common/adf_gen6_shared.h | 16 + drivers/crypto/intel/qat/qat_common/adf_gen6_tl.c | 258 + drivers/crypto/intel/qat/qat_common/adf_gen6_tl.h | 198 + .../crypto/intel/qat/qat_common/adf_heartbeat.c | 345 ++ .../crypto/intel/qat/qat_common/adf_heartbeat.h | 100 + .../intel/qat/qat_common/adf_heartbeat_dbgfs.c | 248 + .../intel/qat/qat_common/adf_heartbeat_dbgfs.h | 12 + .../intel/qat/qat_common/adf_heartbeat_inject.c | 74 + .../crypto/intel/qat/qat_common/adf_hw_arbiter.c | 101 + drivers/crypto/intel/qat/qat_common/adf_init.c | 547 ++ drivers/crypto/intel/qat/qat_common/adf_isr.c | 454 ++ drivers/crypto/intel/qat/qat_common/adf_kpt.c | 56 + drivers/crypto/intel/qat/qat_common/adf_kpt.h | 29 + drivers/crypto/intel/qat/qat_common/adf_module.c | 63 + .../crypto/intel/qat/qat_common/adf_mstate_mgr.c | 319 + .../crypto/intel/qat/qat_common/adf_mstate_mgr.h | 89 + drivers/crypto/intel/qat/qat_common/adf_pfvf_msg.h | 264 + .../crypto/intel/qat/qat_common/adf_pfvf_pf_msg.c | 117 + .../crypto/intel/qat/qat_common/adf_pfvf_pf_msg.h | 39 + .../intel/qat/qat_common/adf_pfvf_pf_proto.c | 360 ++ .../intel/qat/qat_common/adf_pfvf_pf_proto.h | 13 + .../crypto/intel/qat/qat_common/adf_pfvf_utils.c | 65 + .../crypto/intel/qat/qat_common/adf_pfvf_utils.h | 42 + .../crypto/intel/qat/qat_common/adf_pfvf_vf_msg.c | 181 + .../crypto/intel/qat/qat_common/adf_pfvf_vf_msg.h | 24 + .../intel/qat/qat_common/adf_pfvf_vf_proto.c | 374 ++ .../intel/qat/qat_common/adf_pfvf_vf_proto.h | 17 + drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs.c | 48 + drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs.h | 12 + .../intel/qat/qat_common/adf_pm_dbgfs_utils.c | 46 + .../intel/qat/qat_common/adf_pm_dbgfs_utils.h | 36 + drivers/crypto/intel/qat/qat_common/adf_rl.c | 1155 ++++ drivers/crypto/intel/qat/qat_common/adf_rl.h | 176 + drivers/crypto/intel/qat/qat_common/adf_rl_admin.c | 98 + drivers/crypto/intel/qat/qat_common/adf_rl_admin.h | 18 + drivers/crypto/intel/qat/qat_common/adf_sriov.c | 339 ++ drivers/crypto/intel/qat/qat_common/adf_sysfs.c | 353 ++ .../intel/qat/qat_common/adf_sysfs_anti_rb.c | 133 + .../intel/qat/qat_common/adf_sysfs_anti_rb.h | 11 + .../crypto/intel/qat/qat_common/adf_sysfs_kpt.c | 296 + .../crypto/intel/qat/qat_common/adf_sysfs_kpt.h | 10 + .../intel/qat/qat_common/adf_sysfs_ras_counters.c | 117 + .../intel/qat/qat_common/adf_sysfs_ras_counters.h | 27 + drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.c | 464 ++ drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.h | 11 + .../crypto/intel/qat/qat_common/adf_telemetry.c | 327 + .../crypto/intel/qat/qat_common/adf_telemetry.h | 105 + drivers/crypto/intel/qat/qat_common/adf_timer.c | 71 + drivers/crypto/intel/qat/qat_common/adf_timer.h | 21 + .../crypto/intel/qat/qat_common/adf_tl_debugfs.c | 737 +++ .../crypto/intel/qat/qat_common/adf_tl_debugfs.h | 122 + .../crypto/intel/qat/qat_common/adf_transport.c | 575 ++ .../crypto/intel/qat/qat_common/adf_transport.h | 20 + .../qat/qat_common/adf_transport_access_macros.h | 58 + .../intel/qat/qat_common/adf_transport_debug.c | 217 + .../intel/qat/qat_common/adf_transport_internal.h | 73 + drivers/crypto/intel/qat/qat_common/adf_vf_isr.c | 314 + drivers/crypto/intel/qat/qat_common/icp_qat_fw.h | 305 + .../crypto/intel/qat/qat_common/icp_qat_fw_comp.h | 423 ++ .../intel/qat/qat_common/icp_qat_fw_init_admin.h | 223 + .../crypto/intel/qat/qat_common/icp_qat_fw_la.h | 367 ++ .../qat/qat_common/icp_qat_fw_loader_handle.h | 70 + .../crypto/intel/qat/qat_common/icp_qat_fw_pke.h | 68 + drivers/crypto/intel/qat/qat_common/icp_qat_hal.h | 143 + drivers/crypto/intel/qat/qat_common/icp_qat_hw.h | 391 ++ .../intel/qat/qat_common/icp_qat_hw_20_comp.h | 166 + .../intel/qat/qat_common/icp_qat_hw_20_comp_defs.h | 300 + .../intel/qat/qat_common/icp_qat_hw_51_comp.h | 99 + .../intel/qat/qat_common/icp_qat_hw_51_comp_defs.h | 318 + drivers/crypto/intel/qat/qat_common/icp_qat_uclo.h | 598 ++ drivers/crypto/intel/qat/qat_common/qat_algs.c | 1349 +++++ .../crypto/intel/qat/qat_common/qat_algs_send.c | 91 + .../crypto/intel/qat/qat_common/qat_algs_send.h | 25 + .../crypto/intel/qat/qat_common/qat_asym_algs.c | 1361 +++++ drivers/crypto/intel/qat/qat_common/qat_bl.c | 253 + drivers/crypto/intel/qat/qat_common/qat_bl.h | 68 + .../crypto/intel/qat/qat_common/qat_comp_algs.c | 758 +++ drivers/crypto/intel/qat/qat_common/qat_comp_req.h | 122 + .../intel/qat/qat_common/qat_comp_zstd_utils.c | 165 + .../intel/qat/qat_common/qat_comp_zstd_utils.h | 13 + .../crypto/intel/qat/qat_common/qat_compression.c | 317 + .../crypto/intel/qat/qat_common/qat_compression.h | 36 + drivers/crypto/intel/qat/qat_common/qat_crypto.c | 287 + drivers/crypto/intel/qat/qat_common/qat_crypto.h | 68 + drivers/crypto/intel/qat/qat_common/qat_hal.c | 1604 +++++ drivers/crypto/intel/qat/qat_common/qat_mig_dev.c | 130 + drivers/crypto/intel/qat/qat_common/qat_uclo.c | 2291 +++++++ drivers/crypto/intel/qat/qat_dh895xcc/Makefile | 3 + .../intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 269 + .../intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h | 44 + drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c | 258 + drivers/crypto/intel/qat/qat_dh895xccvf/Makefile | 3 + .../qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c | 101 + .../qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h | 18 + drivers/crypto/intel/qat/qat_dh895xccvf/adf_drv.c | 227 + drivers/crypto/marvell/Kconfig | 55 + drivers/crypto/marvell/Makefile | 5 + drivers/crypto/marvell/cesa/Makefile | 3 + drivers/crypto/marvell/cesa/cesa.c | 571 ++ drivers/crypto/marvell/cesa/cesa.h | 920 +++ drivers/crypto/marvell/cesa/cipher.c | 822 +++ drivers/crypto/marvell/cesa/hash.c | 1454 +++++ drivers/crypto/marvell/cesa/tdma.c | 413 ++ drivers/crypto/marvell/octeontx/Makefile | 6 + drivers/crypto/marvell/octeontx/otx_cpt_common.h | 51 + drivers/crypto/marvell/octeontx/otx_cpt_hw_types.h | 824 +++ drivers/crypto/marvell/octeontx/otx_cptpf.h | 34 + drivers/crypto/marvell/octeontx/otx_cptpf_main.c | 301 + drivers/crypto/marvell/octeontx/otx_cptpf_mbox.c | 253 + drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c | 1675 ++++++ drivers/crypto/marvell/octeontx/otx_cptpf_ucode.h | 178 + drivers/crypto/marvell/octeontx/otx_cptvf.h | 104 + drivers/crypto/marvell/octeontx/otx_cptvf_algs.c | 1637 +++++ drivers/crypto/marvell/octeontx/otx_cptvf_algs.h | 189 + drivers/crypto/marvell/octeontx/otx_cptvf_main.c | 977 +++ drivers/crypto/marvell/octeontx/otx_cptvf_mbox.c | 237 + drivers/crypto/marvell/octeontx/otx_cptvf_reqmgr.c | 609 ++ drivers/crypto/marvell/octeontx/otx_cptvf_reqmgr.h | 227 + drivers/crypto/marvell/octeontx2/Makefile | 10 + drivers/crypto/marvell/octeontx2/cn10k_cpt.c | 223 + drivers/crypto/marvell/octeontx2/cn10k_cpt.h | 62 + drivers/crypto/marvell/octeontx2/otx2_cpt_common.h | 213 + .../crypto/marvell/octeontx2/otx2_cpt_devlink.c | 185 + .../crypto/marvell/octeontx2/otx2_cpt_devlink.h | 20 + .../crypto/marvell/octeontx2/otx2_cpt_hw_types.h | 483 ++ .../marvell/octeontx2/otx2_cpt_mbox_common.c | 277 + drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h | 561 ++ drivers/crypto/marvell/octeontx2/otx2_cptlf.c | 498 ++ drivers/crypto/marvell/octeontx2/otx2_cptlf.h | 448 ++ drivers/crypto/marvell/octeontx2/otx2_cptpf.h | 78 + drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c | 883 +++ drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c | 661 ++ .../crypto/marvell/octeontx2/otx2_cptpf_ucode.c | 1797 ++++++ .../crypto/marvell/octeontx2/otx2_cptpf_ucode.h | 172 + drivers/crypto/marvell/octeontx2/otx2_cptvf.h | 35 + drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c | 1700 ++++++ drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.h | 183 + drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c | 482 ++ drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c | 241 + .../crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c | 409 ++ drivers/crypto/mxs-dcp.c | 1258 ++++ drivers/crypto/nx/Kconfig | 51 + drivers/crypto/nx/Makefile | 18 + drivers/crypto/nx/nx-842.c | 526 ++ drivers/crypto/nx/nx-842.h | 195 + drivers/crypto/nx/nx-aes-cbc.c | 129 + drivers/crypto/nx/nx-aes-ccm.c | 559 ++ drivers/crypto/nx/nx-aes-ctr.c | 147 + drivers/crypto/nx/nx-aes-ecb.c | 127 + drivers/crypto/nx/nx-aes-gcm.c | 500 ++ drivers/crypto/nx/nx-aes-xcbc.c | 331 + drivers/crypto/nx/nx-common-powernv.c | 1133 ++++ drivers/crypto/nx/nx-common-pseries.c | 1271 ++++ drivers/crypto/nx/nx-sha256.c | 260 + drivers/crypto/nx/nx-sha512.c | 265 + drivers/crypto/nx/nx.c | 830 +++ drivers/crypto/nx/nx.h | 197 + drivers/crypto/nx/nx_csbcpb.h | 206 + drivers/crypto/nx/nx_debugfs.c | 66 + drivers/crypto/omap-aes-gcm.c | 387 ++ drivers/crypto/omap-aes.c | 1245 ++++ drivers/crypto/omap-aes.h | 219 + drivers/crypto/omap-crypto.c | 225 + drivers/crypto/omap-crypto.h | 34 + drivers/crypto/omap-des.c | 1126 ++++ drivers/crypto/omap-sham.c | 2155 +++++++ drivers/crypto/padlock-aes.c | 539 ++ drivers/crypto/padlock-sha.c | 395 ++ drivers/crypto/qce/Makefile | 9 + drivers/crypto/qce/aead.c | 787 +++ drivers/crypto/qce/aead.h | 56 + drivers/crypto/qce/cipher.h | 56 + drivers/crypto/qce/common.c | 564 ++ drivers/crypto/qce/common.h | 94 + drivers/crypto/qce/core.c | 353 ++ drivers/crypto/qce/core.h | 64 + drivers/crypto/qce/dma.c | 135 + drivers/crypto/qce/dma.h | 47 + drivers/crypto/qce/regs-v5.h | 322 + drivers/crypto/qce/sha.c | 498 ++ drivers/crypto/qce/sha.h | 71 + drivers/crypto/qce/skcipher.c | 434 ++ drivers/crypto/rockchip/Makefile | 5 + drivers/crypto/rockchip/rk3288_crypto.c | 446 ++ drivers/crypto/rockchip/rk3288_crypto.h | 285 + drivers/crypto/rockchip/rk3288_crypto_ahash.c | 474 ++ drivers/crypto/rockchip/rk3288_crypto_skcipher.c | 612 ++ drivers/crypto/s5p-sss.c | 2325 +++++++ drivers/crypto/sa2ul.c | 2474 ++++++++ drivers/crypto/sa2ul.h | 407 ++ drivers/crypto/sahara.c | 1435 +++++ drivers/crypto/starfive/Kconfig | 26 + drivers/crypto/starfive/Makefile | 4 + drivers/crypto/starfive/jh7110-aes.c | 1155 ++++ drivers/crypto/starfive/jh7110-cryp.c | 219 + drivers/crypto/starfive/jh7110-cryp.h | 240 + drivers/crypto/starfive/jh7110-hash.c | 854 +++ drivers/crypto/starfive/jh7110-rsa.c | 592 ++ drivers/crypto/stm32/Kconfig | 25 + drivers/crypto/stm32/Makefile | 3 + drivers/crypto/stm32/stm32-cryp.c | 2776 +++++++++ drivers/crypto/stm32/stm32-hash.c | 2536 ++++++++ drivers/crypto/talitos.c | 3635 +++++++++++ drivers/crypto/talitos.h | 427 ++ drivers/crypto/tegra/Makefile | 9 + drivers/crypto/tegra/tegra-se-aes.c | 2061 +++++++ drivers/crypto/tegra/tegra-se-hash.c | 1158 ++++ drivers/crypto/tegra/tegra-se-key.c | 179 + drivers/crypto/tegra/tegra-se-main.c | 438 ++ drivers/crypto/tegra/tegra-se.h | 589 ++ drivers/crypto/ti/Kconfig | 19 + drivers/crypto/ti/Makefile | 3 + drivers/crypto/ti/dthev2-aes.c | 1375 +++++ drivers/crypto/ti/dthev2-common.c | 235 + drivers/crypto/ti/dthev2-common.h | 134 + drivers/crypto/virtio/Kconfig | 13 + drivers/crypto/virtio/Makefile | 7 + .../crypto/virtio/virtio_crypto_akcipher_algs.c | 569 ++ drivers/crypto/virtio/virtio_crypto_common.h | 153 + drivers/crypto/virtio/virtio_crypto_core.c | 595 ++ drivers/crypto/virtio/virtio_crypto_mgr.c | 296 + .../crypto/virtio/virtio_crypto_skcipher_algs.c | 643 ++ drivers/crypto/xilinx/Makefile | 3 + drivers/crypto/xilinx/zynqmp-aes-gcm.c | 1026 ++++ drivers/crypto/xilinx/zynqmp-sha.c | 258 + 602 files changed, 273700 insertions(+) create mode 100644 drivers/crypto/Kconfig create mode 100644 drivers/crypto/Makefile create mode 100644 drivers/crypto/allwinner/Kconfig create mode 100644 drivers/crypto/allwinner/Makefile create mode 100644 drivers/crypto/allwinner/sun4i-ss/Makefile create mode 100644 drivers/crypto/allwinner/sun4i-ss/sun4i-ss-cipher.c create mode 100644 drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c create mode 100644 drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c create mode 100644 drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h create mode 100644 drivers/crypto/allwinner/sun8i-ce/Makefile create mode 100644 drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c create mode 100644 drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c create mode 100644 drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c create mode 100644 drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c create mode 100644 drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h create mode 100644 drivers/crypto/allwinner/sun8i-ss/Makefile create mode 100644 drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c create mode 100644 drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c create mode 100644 drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c create mode 100644 drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h create mode 100644 drivers/crypto/amcc/Makefile create mode 100644 drivers/crypto/amcc/crypto4xx_alg.c create mode 100644 drivers/crypto/amcc/crypto4xx_core.c create mode 100644 drivers/crypto/amcc/crypto4xx_core.h create mode 100644 drivers/crypto/amcc/crypto4xx_reg_def.h create mode 100644 drivers/crypto/amcc/crypto4xx_sa.h create mode 100644 drivers/crypto/amcc/crypto4xx_trng.c create mode 100644 drivers/crypto/amcc/crypto4xx_trng.h create mode 100644 drivers/crypto/amlogic/Kconfig create mode 100644 drivers/crypto/amlogic/Makefile create mode 100644 drivers/crypto/amlogic/amlogic-gxl-cipher.c create mode 100644 drivers/crypto/amlogic/amlogic-gxl-core.c create mode 100644 drivers/crypto/amlogic/amlogic-gxl.h create mode 100644 drivers/crypto/aspeed/Kconfig create mode 100644 drivers/crypto/aspeed/Makefile create mode 100644 drivers/crypto/aspeed/aspeed-acry.c create mode 100644 drivers/crypto/aspeed/aspeed-hace-crypto.c create mode 100644 drivers/crypto/aspeed/aspeed-hace-hash.c create mode 100644 drivers/crypto/aspeed/aspeed-hace.c create mode 100644 drivers/crypto/aspeed/aspeed-hace.h create mode 100644 drivers/crypto/atmel-aes-regs.h create mode 100644 drivers/crypto/atmel-aes.c create mode 100644 drivers/crypto/atmel-authenc.h create mode 100644 drivers/crypto/atmel-ecc.c create mode 100644 drivers/crypto/atmel-i2c.c create mode 100644 drivers/crypto/atmel-i2c.h create mode 100644 drivers/crypto/atmel-sha-regs.h create mode 100644 drivers/crypto/atmel-sha.c create mode 100644 drivers/crypto/atmel-sha204a.c create mode 100644 drivers/crypto/atmel-tdes-regs.h create mode 100644 drivers/crypto/atmel-tdes.c create mode 100644 drivers/crypto/axis/Makefile create mode 100644 drivers/crypto/axis/artpec6_crypto.c create mode 100644 drivers/crypto/bcm/Makefile create mode 100644 drivers/crypto/bcm/cipher.c create mode 100644 drivers/crypto/bcm/cipher.h create mode 100644 drivers/crypto/bcm/spu.c create mode 100644 drivers/crypto/bcm/spu.h create mode 100644 drivers/crypto/bcm/spu2.c create mode 100644 drivers/crypto/bcm/spu2.h create mode 100644 drivers/crypto/bcm/spum.h create mode 100644 drivers/crypto/bcm/util.c create mode 100644 drivers/crypto/bcm/util.h create mode 100644 drivers/crypto/caam/Kconfig create mode 100644 drivers/crypto/caam/Makefile create mode 100644 drivers/crypto/caam/blob_gen.c create mode 100644 drivers/crypto/caam/caamalg.c create mode 100644 drivers/crypto/caam/caamalg_desc.c create mode 100644 drivers/crypto/caam/caamalg_desc.h create mode 100644 drivers/crypto/caam/caamalg_qi.c create mode 100644 drivers/crypto/caam/caamalg_qi2.c create mode 100644 drivers/crypto/caam/caamalg_qi2.h create mode 100644 drivers/crypto/caam/caamhash.c create mode 100644 drivers/crypto/caam/caamhash_desc.c create mode 100644 drivers/crypto/caam/caamhash_desc.h create mode 100644 drivers/crypto/caam/caampkc.c create mode 100644 drivers/crypto/caam/caampkc.h create mode 100644 drivers/crypto/caam/caamrng.c create mode 100644 drivers/crypto/caam/compat.h create mode 100644 drivers/crypto/caam/ctrl.c create mode 100644 drivers/crypto/caam/ctrl.h create mode 100644 drivers/crypto/caam/debugfs.c create mode 100644 drivers/crypto/caam/debugfs.h create mode 100644 drivers/crypto/caam/desc.h create mode 100644 drivers/crypto/caam/desc_constr.h create mode 100644 drivers/crypto/caam/dpseci-debugfs.c create mode 100644 drivers/crypto/caam/dpseci-debugfs.h create mode 100644 drivers/crypto/caam/dpseci.c create mode 100644 drivers/crypto/caam/dpseci.h create mode 100644 drivers/crypto/caam/dpseci_cmd.h create mode 100644 drivers/crypto/caam/error.c create mode 100644 drivers/crypto/caam/error.h create mode 100644 drivers/crypto/caam/intern.h create mode 100644 drivers/crypto/caam/jr.c create mode 100644 drivers/crypto/caam/jr.h create mode 100644 drivers/crypto/caam/key_gen.c create mode 100644 drivers/crypto/caam/key_gen.h create mode 100644 drivers/crypto/caam/pdb.h create mode 100644 drivers/crypto/caam/pkc_desc.c create mode 100644 drivers/crypto/caam/qi.c create mode 100644 drivers/crypto/caam/qi.h create mode 100644 drivers/crypto/caam/regs.h create mode 100644 drivers/crypto/caam/sg_sw_qm.h create mode 100644 drivers/crypto/caam/sg_sw_qm2.h create mode 100644 drivers/crypto/caam/sg_sw_sec4.h create mode 100644 drivers/crypto/cavium/Makefile create mode 100644 drivers/crypto/cavium/cpt/Kconfig create mode 100644 drivers/crypto/cavium/cpt/Makefile create mode 100644 drivers/crypto/cavium/cpt/cpt_common.h create mode 100644 drivers/crypto/cavium/cpt/cpt_hw_types.h create mode 100644 drivers/crypto/cavium/cpt/cptpf.h create mode 100644 drivers/crypto/cavium/cpt/cptpf_main.c create mode 100644 drivers/crypto/cavium/cpt/cptpf_mbox.c create mode 100644 drivers/crypto/cavium/cpt/cptvf.h create mode 100644 drivers/crypto/cavium/cpt/cptvf_algs.c create mode 100644 drivers/crypto/cavium/cpt/cptvf_algs.h create mode 100644 drivers/crypto/cavium/cpt/cptvf_main.c create mode 100644 drivers/crypto/cavium/cpt/cptvf_mbox.c create mode 100644 drivers/crypto/cavium/cpt/cptvf_reqmanager.c create mode 100644 drivers/crypto/cavium/cpt/request_manager.h create mode 100644 drivers/crypto/cavium/nitrox/Kconfig create mode 100644 drivers/crypto/cavium/nitrox/Makefile create mode 100644 drivers/crypto/cavium/nitrox/nitrox_aead.c create mode 100644 drivers/crypto/cavium/nitrox/nitrox_algs.c create mode 100644 drivers/crypto/cavium/nitrox/nitrox_common.h create mode 100644 drivers/crypto/cavium/nitrox/nitrox_csr.h create mode 100644 drivers/crypto/cavium/nitrox/nitrox_debugfs.c create mode 100644 drivers/crypto/cavium/nitrox/nitrox_debugfs.h create mode 100644 drivers/crypto/cavium/nitrox/nitrox_dev.h create mode 100644 drivers/crypto/cavium/nitrox/nitrox_hal.c create mode 100644 drivers/crypto/cavium/nitrox/nitrox_hal.h create mode 100644 drivers/crypto/cavium/nitrox/nitrox_isr.c create mode 100644 drivers/crypto/cavium/nitrox/nitrox_isr.h create mode 100644 drivers/crypto/cavium/nitrox/nitrox_lib.c create mode 100644 drivers/crypto/cavium/nitrox/nitrox_main.c create mode 100644 drivers/crypto/cavium/nitrox/nitrox_mbx.c create mode 100644 drivers/crypto/cavium/nitrox/nitrox_mbx.h create mode 100644 drivers/crypto/cavium/nitrox/nitrox_req.h create mode 100644 drivers/crypto/cavium/nitrox/nitrox_reqmgr.c create mode 100644 drivers/crypto/cavium/nitrox/nitrox_skcipher.c create mode 100644 drivers/crypto/cavium/nitrox/nitrox_sriov.c create mode 100644 drivers/crypto/ccp/Kconfig create mode 100644 drivers/crypto/ccp/Makefile create mode 100644 drivers/crypto/ccp/ccp-crypto-aes-cmac.c create mode 100644 drivers/crypto/ccp/ccp-crypto-aes-galois.c create mode 100644 drivers/crypto/ccp/ccp-crypto-aes-xts.c create mode 100644 drivers/crypto/ccp/ccp-crypto-aes.c create mode 100644 drivers/crypto/ccp/ccp-crypto-des3.c create mode 100644 drivers/crypto/ccp/ccp-crypto-main.c create mode 100644 drivers/crypto/ccp/ccp-crypto-rsa.c create mode 100644 drivers/crypto/ccp/ccp-crypto-sha.c create mode 100644 drivers/crypto/ccp/ccp-crypto.h create mode 100644 drivers/crypto/ccp/ccp-debugfs.c create mode 100644 drivers/crypto/ccp/ccp-dev-v3.c create mode 100644 drivers/crypto/ccp/ccp-dev-v5.c create mode 100644 drivers/crypto/ccp/ccp-dev.c create mode 100644 drivers/crypto/ccp/ccp-dev.h create mode 100644 drivers/crypto/ccp/ccp-dmaengine.c create mode 100644 drivers/crypto/ccp/ccp-ops.c create mode 100644 drivers/crypto/ccp/dbc.c create mode 100644 drivers/crypto/ccp/dbc.h create mode 100644 drivers/crypto/ccp/hsti.c create mode 100644 drivers/crypto/ccp/hsti.h create mode 100644 drivers/crypto/ccp/platform-access.c create mode 100644 drivers/crypto/ccp/platform-access.h create mode 100644 drivers/crypto/ccp/psp-dev.c create mode 100644 drivers/crypto/ccp/psp-dev.h create mode 100644 drivers/crypto/ccp/sev-dev-tio.c create mode 100644 drivers/crypto/ccp/sev-dev-tio.h create mode 100644 drivers/crypto/ccp/sev-dev-tsm.c create mode 100644 drivers/crypto/ccp/sev-dev.c create mode 100644 drivers/crypto/ccp/sev-dev.h create mode 100644 drivers/crypto/ccp/sfs.c create mode 100644 drivers/crypto/ccp/sfs.h create mode 100644 drivers/crypto/ccp/sp-dev.c create mode 100644 drivers/crypto/ccp/sp-dev.h create mode 100644 drivers/crypto/ccp/sp-pci.c create mode 100644 drivers/crypto/ccp/sp-platform.c create mode 100644 drivers/crypto/ccp/tee-dev.c create mode 100644 drivers/crypto/ccp/tee-dev.h create mode 100644 drivers/crypto/ccree/Makefile create mode 100644 drivers/crypto/ccree/cc_aead.c create mode 100644 drivers/crypto/ccree/cc_aead.h create mode 100644 drivers/crypto/ccree/cc_buffer_mgr.c create mode 100644 drivers/crypto/ccree/cc_buffer_mgr.h create mode 100644 drivers/crypto/ccree/cc_cipher.c create mode 100644 drivers/crypto/ccree/cc_cipher.h create mode 100644 drivers/crypto/ccree/cc_crypto_ctx.h create mode 100644 drivers/crypto/ccree/cc_debugfs.c create mode 100644 drivers/crypto/ccree/cc_debugfs.h create mode 100644 drivers/crypto/ccree/cc_driver.c create mode 100644 drivers/crypto/ccree/cc_driver.h create mode 100644 drivers/crypto/ccree/cc_fips.c create mode 100644 drivers/crypto/ccree/cc_fips.h create mode 100644 drivers/crypto/ccree/cc_hash.c create mode 100644 drivers/crypto/ccree/cc_hash.h create mode 100644 drivers/crypto/ccree/cc_host_regs.h create mode 100644 drivers/crypto/ccree/cc_hw_queue_defs.h create mode 100644 drivers/crypto/ccree/cc_kernel_regs.h create mode 100644 drivers/crypto/ccree/cc_lli_defs.h create mode 100644 drivers/crypto/ccree/cc_pm.c create mode 100644 drivers/crypto/ccree/cc_pm.h create mode 100644 drivers/crypto/ccree/cc_request_mgr.c create mode 100644 drivers/crypto/ccree/cc_request_mgr.h create mode 100644 drivers/crypto/ccree/cc_sram_mgr.c create mode 100644 drivers/crypto/ccree/cc_sram_mgr.h create mode 100644 drivers/crypto/chelsio/Kconfig create mode 100644 drivers/crypto/chelsio/Makefile create mode 100644 drivers/crypto/chelsio/chcr_algo.c create mode 100644 drivers/crypto/chelsio/chcr_algo.h create mode 100644 drivers/crypto/chelsio/chcr_core.c create mode 100644 drivers/crypto/chelsio/chcr_core.h create mode 100644 drivers/crypto/chelsio/chcr_crypto.h create mode 100644 drivers/crypto/gemini/Makefile create mode 100644 drivers/crypto/gemini/sl3516-ce-cipher.c create mode 100644 drivers/crypto/gemini/sl3516-ce-core.c create mode 100644 drivers/crypto/gemini/sl3516-ce-rng.c create mode 100644 drivers/crypto/gemini/sl3516-ce.h create mode 100644 drivers/crypto/geode-aes.c create mode 100644 drivers/crypto/geode-aes.h create mode 100644 drivers/crypto/hifn_795x.c create mode 100644 drivers/crypto/hisilicon/Kconfig create mode 100644 drivers/crypto/hisilicon/Makefile create mode 100644 drivers/crypto/hisilicon/debugfs.c create mode 100644 drivers/crypto/hisilicon/hpre/Makefile create mode 100644 drivers/crypto/hisilicon/hpre/hpre.h create mode 100644 drivers/crypto/hisilicon/hpre/hpre_crypto.c create mode 100644 drivers/crypto/hisilicon/hpre/hpre_main.c create mode 100644 drivers/crypto/hisilicon/qm.c create mode 100644 drivers/crypto/hisilicon/qm_common.h create mode 100644 drivers/crypto/hisilicon/sec/Makefile create mode 100644 drivers/crypto/hisilicon/sec/sec_algs.c create mode 100644 drivers/crypto/hisilicon/sec/sec_drv.c create mode 100644 drivers/crypto/hisilicon/sec/sec_drv.h create mode 100644 drivers/crypto/hisilicon/sec2/Makefile create mode 100644 drivers/crypto/hisilicon/sec2/sec.h create mode 100644 drivers/crypto/hisilicon/sec2/sec_crypto.c create mode 100644 drivers/crypto/hisilicon/sec2/sec_crypto.h create mode 100644 drivers/crypto/hisilicon/sec2/sec_main.c create mode 100644 drivers/crypto/hisilicon/sgl.c create mode 100644 drivers/crypto/hisilicon/zip/Makefile create mode 100644 drivers/crypto/hisilicon/zip/dae_main.c create mode 100644 drivers/crypto/hisilicon/zip/zip.h create mode 100644 drivers/crypto/hisilicon/zip/zip_crypto.c create mode 100644 drivers/crypto/hisilicon/zip/zip_main.c create mode 100644 drivers/crypto/img-hash.c create mode 100644 drivers/crypto/inside-secure/Makefile create mode 100644 drivers/crypto/inside-secure/eip93/Kconfig create mode 100644 drivers/crypto/inside-secure/eip93/Makefile create mode 100644 drivers/crypto/inside-secure/eip93/eip93-aead.c create mode 100644 drivers/crypto/inside-secure/eip93/eip93-aead.h create mode 100644 drivers/crypto/inside-secure/eip93/eip93-aes.h create mode 100644 drivers/crypto/inside-secure/eip93/eip93-cipher.c create mode 100644 drivers/crypto/inside-secure/eip93/eip93-cipher.h create mode 100644 drivers/crypto/inside-secure/eip93/eip93-common.c create mode 100644 drivers/crypto/inside-secure/eip93/eip93-common.h create mode 100644 drivers/crypto/inside-secure/eip93/eip93-des.h create mode 100644 drivers/crypto/inside-secure/eip93/eip93-hash.c create mode 100644 drivers/crypto/inside-secure/eip93/eip93-hash.h create mode 100644 drivers/crypto/inside-secure/eip93/eip93-main.c create mode 100644 drivers/crypto/inside-secure/eip93/eip93-main.h create mode 100644 drivers/crypto/inside-secure/eip93/eip93-regs.h create mode 100644 drivers/crypto/inside-secure/safexcel.c create mode 100644 drivers/crypto/inside-secure/safexcel.h create mode 100644 drivers/crypto/inside-secure/safexcel_cipher.c create mode 100644 drivers/crypto/inside-secure/safexcel_hash.c create mode 100644 drivers/crypto/inside-secure/safexcel_ring.c create mode 100644 drivers/crypto/intel/Kconfig create mode 100644 drivers/crypto/intel/Makefile create mode 100644 drivers/crypto/intel/iaa/Kconfig create mode 100644 drivers/crypto/intel/iaa/Makefile create mode 100644 drivers/crypto/intel/iaa/iaa_crypto.h create mode 100644 drivers/crypto/intel/iaa/iaa_crypto_comp_fixed.c create mode 100644 drivers/crypto/intel/iaa/iaa_crypto_main.c create mode 100644 drivers/crypto/intel/iaa/iaa_crypto_stats.c create mode 100644 drivers/crypto/intel/iaa/iaa_crypto_stats.h create mode 100644 drivers/crypto/intel/ixp4xx/Kconfig create mode 100644 drivers/crypto/intel/ixp4xx/Makefile create mode 100644 drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c create mode 100644 drivers/crypto/intel/keembay/Kconfig create mode 100644 drivers/crypto/intel/keembay/Makefile create mode 100644 drivers/crypto/intel/keembay/keembay-ocs-aes-core.c create mode 100644 drivers/crypto/intel/keembay/keembay-ocs-ecc.c create mode 100644 drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c create mode 100644 drivers/crypto/intel/keembay/ocs-aes.c create mode 100644 drivers/crypto/intel/keembay/ocs-aes.h create mode 100644 drivers/crypto/intel/keembay/ocs-hcu.c create mode 100644 drivers/crypto/intel/keembay/ocs-hcu.h create mode 100644 drivers/crypto/intel/qat/Kconfig create mode 100644 drivers/crypto/intel/qat/Makefile create mode 100644 drivers/crypto/intel/qat/qat_420xx/Makefile create mode 100644 drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c create mode 100644 drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.h create mode 100644 drivers/crypto/intel/qat/qat_420xx/adf_drv.c create mode 100644 drivers/crypto/intel/qat/qat_4xxx/Makefile create mode 100644 drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c create mode 100644 drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.h create mode 100644 drivers/crypto/intel/qat/qat_4xxx/adf_drv.c create mode 100644 drivers/crypto/intel/qat/qat_6xxx/Makefile create mode 100644 drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c create mode 100644 drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.h create mode 100644 drivers/crypto/intel/qat/qat_6xxx/adf_drv.c create mode 100644 drivers/crypto/intel/qat/qat_c3xxx/Makefile create mode 100644 drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c create mode 100644 drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.h create mode 100644 drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c create mode 100644 drivers/crypto/intel/qat/qat_c3xxxvf/Makefile create mode 100644 drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c create mode 100644 drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h create mode 100644 drivers/crypto/intel/qat/qat_c3xxxvf/adf_drv.c create mode 100644 drivers/crypto/intel/qat/qat_c62x/Makefile create mode 100644 drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c create mode 100644 drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.h create mode 100644 drivers/crypto/intel/qat/qat_c62x/adf_drv.c create mode 100644 drivers/crypto/intel/qat/qat_c62xvf/Makefile create mode 100644 drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c create mode 100644 drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.h create mode 100644 drivers/crypto/intel/qat/qat_c62xvf/adf_drv.c create mode 100644 drivers/crypto/intel/qat/qat_common/Makefile create mode 100644 drivers/crypto/intel/qat/qat_common/adf_accel_devices.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_accel_engine.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_admin.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_admin.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_aer.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_anti_rb.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_anti_rb.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_bank_state.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_bank_state.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_cfg.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_cfg.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_cfg_common.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_cfg_services.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_cfg_services.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_cfg_strings.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_clock.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_clock.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_cnv_dbgfs.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_cnv_dbgfs.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_common_drv.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_dbgfs.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_dbgfs.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_dc.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_dc.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_dev_mgr.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_fw_config.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_fw_counters.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_fw_counters.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen2_config.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen2_config.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen2_hw_csr_data.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen2_hw_csr_data.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen2_pfvf.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen2_pfvf.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen4_config.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen4_config.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen4_hw_csr_data.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen4_hw_csr_data.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen4_pm.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen4_pm.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen4_pm_debugfs.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen4_ras.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen4_ras.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen4_tl.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen4_tl.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen4_vf_mig.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen4_vf_mig.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen6_pm.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen6_pm_dbgfs.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen6_ras.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen6_ras.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen6_shared.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen6_shared.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen6_tl.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen6_tl.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_heartbeat.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_heartbeat.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_heartbeat_dbgfs.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_heartbeat_dbgfs.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_heartbeat_inject.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_hw_arbiter.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_init.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_isr.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_kpt.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_kpt.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_module.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_pfvf_msg.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_msg.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_msg.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_pfvf_utils.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_pfvf_utils.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_pfvf_vf_msg.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_pfvf_vf_msg.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_pfvf_vf_proto.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_pfvf_vf_proto.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs_utils.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs_utils.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_rl.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_rl.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_rl_admin.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_rl_admin.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_sriov.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_sysfs.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_sysfs_anti_rb.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_sysfs_anti_rb.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_sysfs_kpt.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_sysfs_kpt.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_sysfs_ras_counters.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_sysfs_ras_counters.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_telemetry.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_telemetry.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_timer.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_timer.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_transport.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_transport.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_transport_access_macros.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_transport_debug.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_transport_internal.h create mode 100644 drivers/crypto/intel/qat/qat_common/adf_vf_isr.c create mode 100644 drivers/crypto/intel/qat/qat_common/icp_qat_fw.h create mode 100644 drivers/crypto/intel/qat/qat_common/icp_qat_fw_comp.h create mode 100644 drivers/crypto/intel/qat/qat_common/icp_qat_fw_init_admin.h create mode 100644 drivers/crypto/intel/qat/qat_common/icp_qat_fw_la.h create mode 100644 drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h create mode 100644 drivers/crypto/intel/qat/qat_common/icp_qat_fw_pke.h create mode 100644 drivers/crypto/intel/qat/qat_common/icp_qat_hal.h create mode 100644 drivers/crypto/intel/qat/qat_common/icp_qat_hw.h create mode 100644 drivers/crypto/intel/qat/qat_common/icp_qat_hw_20_comp.h create mode 100644 drivers/crypto/intel/qat/qat_common/icp_qat_hw_20_comp_defs.h create mode 100644 drivers/crypto/intel/qat/qat_common/icp_qat_hw_51_comp.h create mode 100644 drivers/crypto/intel/qat/qat_common/icp_qat_hw_51_comp_defs.h create mode 100644 drivers/crypto/intel/qat/qat_common/icp_qat_uclo.h create mode 100644 drivers/crypto/intel/qat/qat_common/qat_algs.c create mode 100644 drivers/crypto/intel/qat/qat_common/qat_algs_send.c create mode 100644 drivers/crypto/intel/qat/qat_common/qat_algs_send.h create mode 100644 drivers/crypto/intel/qat/qat_common/qat_asym_algs.c create mode 100644 drivers/crypto/intel/qat/qat_common/qat_bl.c create mode 100644 drivers/crypto/intel/qat/qat_common/qat_bl.h create mode 100644 drivers/crypto/intel/qat/qat_common/qat_comp_algs.c create mode 100644 drivers/crypto/intel/qat/qat_common/qat_comp_req.h create mode 100644 drivers/crypto/intel/qat/qat_common/qat_comp_zstd_utils.c create mode 100644 drivers/crypto/intel/qat/qat_common/qat_comp_zstd_utils.h create mode 100644 drivers/crypto/intel/qat/qat_common/qat_compression.c create mode 100644 drivers/crypto/intel/qat/qat_common/qat_compression.h create mode 100644 drivers/crypto/intel/qat/qat_common/qat_crypto.c create mode 100644 drivers/crypto/intel/qat/qat_common/qat_crypto.h create mode 100644 drivers/crypto/intel/qat/qat_common/qat_hal.c create mode 100644 drivers/crypto/intel/qat/qat_common/qat_mig_dev.c create mode 100644 drivers/crypto/intel/qat/qat_common/qat_uclo.c create mode 100644 drivers/crypto/intel/qat/qat_dh895xcc/Makefile create mode 100644 drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c create mode 100644 drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h create mode 100644 drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c create mode 100644 drivers/crypto/intel/qat/qat_dh895xccvf/Makefile create mode 100644 drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c create mode 100644 drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h create mode 100644 drivers/crypto/intel/qat/qat_dh895xccvf/adf_drv.c create mode 100644 drivers/crypto/marvell/Kconfig create mode 100644 drivers/crypto/marvell/Makefile create mode 100644 drivers/crypto/marvell/cesa/Makefile create mode 100644 drivers/crypto/marvell/cesa/cesa.c create mode 100644 drivers/crypto/marvell/cesa/cesa.h create mode 100644 drivers/crypto/marvell/cesa/cipher.c create mode 100644 drivers/crypto/marvell/cesa/hash.c create mode 100644 drivers/crypto/marvell/cesa/tdma.c create mode 100644 drivers/crypto/marvell/octeontx/Makefile create mode 100644 drivers/crypto/marvell/octeontx/otx_cpt_common.h create mode 100644 drivers/crypto/marvell/octeontx/otx_cpt_hw_types.h create mode 100644 drivers/crypto/marvell/octeontx/otx_cptpf.h create mode 100644 drivers/crypto/marvell/octeontx/otx_cptpf_main.c create mode 100644 drivers/crypto/marvell/octeontx/otx_cptpf_mbox.c create mode 100644 drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c create mode 100644 drivers/crypto/marvell/octeontx/otx_cptpf_ucode.h create mode 100644 drivers/crypto/marvell/octeontx/otx_cptvf.h create mode 100644 drivers/crypto/marvell/octeontx/otx_cptvf_algs.c create mode 100644 drivers/crypto/marvell/octeontx/otx_cptvf_algs.h create mode 100644 drivers/crypto/marvell/octeontx/otx_cptvf_main.c create mode 100644 drivers/crypto/marvell/octeontx/otx_cptvf_mbox.c create mode 100644 drivers/crypto/marvell/octeontx/otx_cptvf_reqmgr.c create mode 100644 drivers/crypto/marvell/octeontx/otx_cptvf_reqmgr.h create mode 100644 drivers/crypto/marvell/octeontx2/Makefile create mode 100644 drivers/crypto/marvell/octeontx2/cn10k_cpt.c create mode 100644 drivers/crypto/marvell/octeontx2/cn10k_cpt.h create mode 100644 drivers/crypto/marvell/octeontx2/otx2_cpt_common.h create mode 100644 drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c create mode 100644 drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.h create mode 100644 drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h create mode 100644 drivers/crypto/marvell/octeontx2/otx2_cpt_mbox_common.c create mode 100644 drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h create mode 100644 drivers/crypto/marvell/octeontx2/otx2_cptlf.c create mode 100644 drivers/crypto/marvell/octeontx2/otx2_cptlf.h create mode 100644 drivers/crypto/marvell/octeontx2/otx2_cptpf.h create mode 100644 drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c create mode 100644 drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c create mode 100644 drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c create mode 100644 drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h create mode 100644 drivers/crypto/marvell/octeontx2/otx2_cptvf.h create mode 100644 drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c create mode 100644 drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.h create mode 100644 drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c create mode 100644 drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c create mode 100644 drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c create mode 100644 drivers/crypto/mxs-dcp.c create mode 100644 drivers/crypto/nx/Kconfig create mode 100644 drivers/crypto/nx/Makefile create mode 100644 drivers/crypto/nx/nx-842.c create mode 100644 drivers/crypto/nx/nx-842.h create mode 100644 drivers/crypto/nx/nx-aes-cbc.c create mode 100644 drivers/crypto/nx/nx-aes-ccm.c create mode 100644 drivers/crypto/nx/nx-aes-ctr.c create mode 100644 drivers/crypto/nx/nx-aes-ecb.c create mode 100644 drivers/crypto/nx/nx-aes-gcm.c create mode 100644 drivers/crypto/nx/nx-aes-xcbc.c create mode 100644 drivers/crypto/nx/nx-common-powernv.c create mode 100644 drivers/crypto/nx/nx-common-pseries.c create mode 100644 drivers/crypto/nx/nx-sha256.c create mode 100644 drivers/crypto/nx/nx-sha512.c create mode 100644 drivers/crypto/nx/nx.c create mode 100644 drivers/crypto/nx/nx.h create mode 100644 drivers/crypto/nx/nx_csbcpb.h create mode 100644 drivers/crypto/nx/nx_debugfs.c create mode 100644 drivers/crypto/omap-aes-gcm.c create mode 100644 drivers/crypto/omap-aes.c create mode 100644 drivers/crypto/omap-aes.h create mode 100644 drivers/crypto/omap-crypto.c create mode 100644 drivers/crypto/omap-crypto.h create mode 100644 drivers/crypto/omap-des.c create mode 100644 drivers/crypto/omap-sham.c create mode 100644 drivers/crypto/padlock-aes.c create mode 100644 drivers/crypto/padlock-sha.c create mode 100644 drivers/crypto/qce/Makefile create mode 100644 drivers/crypto/qce/aead.c create mode 100644 drivers/crypto/qce/aead.h create mode 100644 drivers/crypto/qce/cipher.h create mode 100644 drivers/crypto/qce/common.c create mode 100644 drivers/crypto/qce/common.h create mode 100644 drivers/crypto/qce/core.c create mode 100644 drivers/crypto/qce/core.h create mode 100644 drivers/crypto/qce/dma.c create mode 100644 drivers/crypto/qce/dma.h create mode 100644 drivers/crypto/qce/regs-v5.h create mode 100644 drivers/crypto/qce/sha.c create mode 100644 drivers/crypto/qce/sha.h create mode 100644 drivers/crypto/qce/skcipher.c create mode 100644 drivers/crypto/rockchip/Makefile create mode 100644 drivers/crypto/rockchip/rk3288_crypto.c create mode 100644 drivers/crypto/rockchip/rk3288_crypto.h create mode 100644 drivers/crypto/rockchip/rk3288_crypto_ahash.c create mode 100644 drivers/crypto/rockchip/rk3288_crypto_skcipher.c create mode 100644 drivers/crypto/s5p-sss.c create mode 100644 drivers/crypto/sa2ul.c create mode 100644 drivers/crypto/sa2ul.h create mode 100644 drivers/crypto/sahara.c create mode 100644 drivers/crypto/starfive/Kconfig create mode 100644 drivers/crypto/starfive/Makefile create mode 100644 drivers/crypto/starfive/jh7110-aes.c create mode 100644 drivers/crypto/starfive/jh7110-cryp.c create mode 100644 drivers/crypto/starfive/jh7110-cryp.h create mode 100644 drivers/crypto/starfive/jh7110-hash.c create mode 100644 drivers/crypto/starfive/jh7110-rsa.c create mode 100644 drivers/crypto/stm32/Kconfig create mode 100644 drivers/crypto/stm32/Makefile create mode 100644 drivers/crypto/stm32/stm32-cryp.c create mode 100644 drivers/crypto/stm32/stm32-hash.c create mode 100644 drivers/crypto/talitos.c create mode 100644 drivers/crypto/talitos.h create mode 100644 drivers/crypto/tegra/Makefile create mode 100644 drivers/crypto/tegra/tegra-se-aes.c create mode 100644 drivers/crypto/tegra/tegra-se-hash.c create mode 100644 drivers/crypto/tegra/tegra-se-key.c create mode 100644 drivers/crypto/tegra/tegra-se-main.c create mode 100644 drivers/crypto/tegra/tegra-se.h create mode 100644 drivers/crypto/ti/Kconfig create mode 100644 drivers/crypto/ti/Makefile create mode 100644 drivers/crypto/ti/dthev2-aes.c create mode 100644 drivers/crypto/ti/dthev2-common.c create mode 100644 drivers/crypto/ti/dthev2-common.h create mode 100644 drivers/crypto/virtio/Kconfig create mode 100644 drivers/crypto/virtio/Makefile create mode 100644 drivers/crypto/virtio/virtio_crypto_akcipher_algs.c create mode 100644 drivers/crypto/virtio/virtio_crypto_common.h create mode 100644 drivers/crypto/virtio/virtio_crypto_core.c create mode 100644 drivers/crypto/virtio/virtio_crypto_mgr.c create mode 100644 drivers/crypto/virtio/virtio_crypto_skcipher_algs.c create mode 100644 drivers/crypto/xilinx/Makefile create mode 100644 drivers/crypto/xilinx/zynqmp-aes-gcm.c create mode 100644 drivers/crypto/xilinx/zynqmp-sha.c (limited to 'drivers/crypto') diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig new file mode 100644 index 000000000..0189dfdcb --- /dev/null +++ b/drivers/crypto/Kconfig @@ -0,0 +1,834 @@ +# SPDX-License-Identifier: GPL-2.0-only + +menuconfig CRYPTO_HW + bool "Hardware crypto devices" + default y + help + Say Y here to get to see options for hardware crypto devices and + processors. This option alone does not add any kernel code. + + If you say N, all options in this submenu will be skipped and disabled. + +if CRYPTO_HW + +source "drivers/crypto/allwinner/Kconfig" + +config CRYPTO_DEV_PADLOCK + tristate "Support for VIA PadLock ACE" + depends on X86 && !UML + help + Some VIA processors come with an integrated crypto engine + (so called VIA PadLock ACE, Advanced Cryptography Engine) + that provides instructions for very fast cryptographic + operations with supported algorithms. + + The instructions are used only when the CPU supports them. + Otherwise software encryption is used. + +config CRYPTO_DEV_PADLOCK_AES + tristate "PadLock driver for AES algorithm" + depends on CRYPTO_DEV_PADLOCK + select CRYPTO_SKCIPHER + select CRYPTO_LIB_AES + help + Use VIA PadLock for AES algorithm. + + Available in VIA C3 and newer CPUs. + + If unsure say M. The compiled module will be + called padlock-aes. + +config CRYPTO_DEV_PADLOCK_SHA + tristate "PadLock driver for SHA1 and SHA256 algorithms" + depends on CRYPTO_DEV_PADLOCK + select CRYPTO_HASH + select CRYPTO_SHA1 + select CRYPTO_SHA256 + help + Use VIA PadLock for SHA1/SHA256 algorithms. + + Available in VIA C7 and newer processors. + + If unsure say M. The compiled module will be + called padlock-sha. + +config CRYPTO_DEV_GEODE + tristate "Support for the Geode LX AES engine" + depends on X86_32 && PCI + select CRYPTO_ALGAPI + select CRYPTO_SKCIPHER + help + Say 'Y' here to use the AMD Geode LX processor on-board AES + engine for the CryptoAPI AES algorithm. + + To compile this driver as a module, choose M here: the module + will be called geode-aes. + +config ZCRYPT + tristate "Support for s390 cryptographic adapters" + depends on S390 + depends on AP + select HW_RANDOM + help + Select this option if you want to enable support for + s390 cryptographic adapters like Crypto Express 4 up + to 8 in Coprocessor (CEXxC), EP11 Coprocessor (CEXxP) + or Accelerator (CEXxA) mode. + +config PKEY + tristate "Kernel API for protected key handling" + depends on S390 + help + With this option enabled the pkey kernel modules provide an API + for creation and handling of protected keys. Other parts of the + kernel or userspace applications may use these functions. + + The protected key support is distributed into: + - A pkey base and API kernel module (pkey.ko) which offers the + infrastructure for the pkey handler kernel modules, the ioctl + and the sysfs API and the in-kernel API to the crypto cipher + implementations using protected key. + - A pkey pckmo kernel module (pkey-pckmo.ko) which is automatically + loaded when pckmo support (that is generation of protected keys + from clear key values) is available. + - A pkey CCA kernel module (pkey-cca.ko) which is automatically + loaded when a CEX crypto card is available. + - A pkey EP11 kernel module (pkey-ep11.ko) which is automatically + loaded when a CEX crypto card is available. + - A pkey UV kernel module (pkey-uv.ko) which is automatically + loaded when the Ultravisor feature is available within a + protected execution environment. + + Select this option if you want to enable the kernel and userspace + API for protected key handling. + +config PKEY_CCA + tristate "PKEY CCA support handler" + depends on PKEY + depends on ZCRYPT + help + This is the CCA support handler for deriving protected keys + from CCA (secure) keys. Also this handler provides an alternate + way to make protected keys from clear key values. + + The PKEY CCA support handler needs a Crypto Express card (CEX) + in CCA mode. + + If you have selected the PKEY option then you should also enable + this option unless you are sure you never need to derive protected + keys from CCA key material. + +config PKEY_EP11 + tristate "PKEY EP11 support handler" + depends on PKEY + depends on ZCRYPT + help + This is the EP11 support handler for deriving protected keys + from EP11 (secure) keys. Also this handler provides an alternate + way to make protected keys from clear key values. + + The PKEY EP11 support handler needs a Crypto Express card (CEX) + in EP11 mode. + + If you have selected the PKEY option then you should also enable + this option unless you are sure you never need to derive protected + keys from EP11 key material. + +config PKEY_PCKMO + tristate "PKEY PCKMO support handler" + depends on PKEY + help + This is the PCKMO support handler for deriving protected keys + from clear key values via invoking the PCKMO instruction. + + The PCKMO instruction can be enabled and disabled in the crypto + settings at the LPAR profile. This handler checks for availability + during initialization and if build as a kernel module unloads + itself if PCKMO is disabled. + + The PCKMO way of deriving protected keys from clear key material + is especially used during self test of protected key ciphers like + PAES but the CCA and EP11 handler provide alternate ways to + generate protected keys from clear key values. + + If you have selected the PKEY option then you should also enable + this option unless you are sure you never need to derive protected + keys from clear key values directly via PCKMO. + +config PKEY_UV + tristate "PKEY UV support handler" + depends on PKEY + depends on S390_UV_UAPI + help + This is the PKEY Ultravisor support handler for deriving protected + keys from secrets stored within the Ultravisor (UV). + + This module works together with the UV device and supports the + retrieval of protected keys from secrets stored within the + UV firmware layer. This service is only available within + a protected execution guest and thus this module will fail upon + modprobe if no protected execution environment is detected. + + Enable this option if you intend to run this kernel with an KVM + guest with protected execution and you want to use UV retrievable + secrets via PKEY API. + +config CRYPTO_PAES_S390 + tristate "PAES cipher algorithms" + depends on S390 + depends on ZCRYPT + depends on PKEY + select CRYPTO_ALGAPI + select CRYPTO_SKCIPHER + select CRYPTO_ENGINE + help + This is the s390 hardware accelerated implementation of the + AES cipher algorithms for use with protected key. + + Select this option if you want to use the paes cipher + for example to use protected key encrypted devices. + +config CRYPTO_PHMAC_S390 + tristate "PHMAC cipher algorithms" + depends on S390 + depends on PKEY + select CRYPTO_HASH + select CRYPTO_ENGINE + help + This is the s390 hardware accelerated implementation of the + protected key HMAC support for SHA224, SHA256, SHA384 and SHA512. + + Select this option if you want to use the phmac digests + for example to use dm-integrity with secure/protected keys. + +config S390_PRNG + tristate "Pseudo random number generator device driver" + depends on S390 + default "m" + help + Select this option if you want to use the s390 pseudo random number + generator. The PRNG is part of the cryptographic processor functions + and uses triple-DES to generate secure random numbers like the + ANSI X9.17 standard. User-space programs access the + pseudo-random-number device through the char device /dev/prandom. + + It is available as of z9. + +config CRYPTO_DEV_SL3516 + tristate "Storlink SL3516 crypto offloader" + depends on ARCH_GEMINI || COMPILE_TEST + depends on HAS_IOMEM && PM + select CRYPTO_SKCIPHER + select CRYPTO_ENGINE + select CRYPTO_ECB + select CRYPTO_AES + select HW_RANDOM + help + This option allows you to have support for SL3516 crypto offloader. + +config CRYPTO_DEV_SL3516_DEBUG + bool "Enable SL3516 stats" + depends on CRYPTO_DEV_SL3516 + depends on DEBUG_FS + help + Say y to enable SL3516 debug stats. + This will create /sys/kernel/debug/sl3516/stats for displaying + the number of requests per algorithm and other internal stats. + +config CRYPTO_DEV_HIFN_795X + tristate "Driver HIFN 795x crypto accelerator chips" + select CRYPTO_LIB_DES + select CRYPTO_SKCIPHER + select HW_RANDOM if CRYPTO_DEV_HIFN_795X_RNG + depends on PCI + depends on !ARCH_DMA_ADDR_T_64BIT + help + This option allows you to have support for HIFN 795x crypto adapters. + +config CRYPTO_DEV_HIFN_795X_RNG + bool "HIFN 795x random number generator" + depends on CRYPTO_DEV_HIFN_795X + help + Select this option if you want to enable the random number generator + on the HIFN 795x crypto adapters. + +source "drivers/crypto/caam/Kconfig" + +config CRYPTO_DEV_TALITOS + tristate "Talitos Freescale Security Engine (SEC)" + select CRYPTO_AEAD + select CRYPTO_AUTHENC + select CRYPTO_SKCIPHER + select CRYPTO_HASH + select CRYPTO_LIB_DES + select HW_RANDOM + depends on FSL_SOC + help + Say 'Y' here to use the Freescale Security Engine (SEC) + to offload cryptographic algorithm computation. + + The Freescale SEC is present on PowerQUICC 'E' processors, such + as the MPC8349E and MPC8548E. + + To compile this driver as a module, choose M here: the module + will be called talitos. + +config CRYPTO_DEV_TALITOS1 + bool "SEC1 (SEC 1.0 and SEC Lite 1.2)" + depends on CRYPTO_DEV_TALITOS + depends on PPC_8xx || PPC_82xx + default y + help + Say 'Y' here to use the Freescale Security Engine (SEC) version 1.0 + found on MPC82xx or the Freescale Security Engine (SEC Lite) + version 1.2 found on MPC8xx + +config CRYPTO_DEV_TALITOS2 + bool "SEC2+ (SEC version 2.0 or upper)" + depends on CRYPTO_DEV_TALITOS + default y if !PPC_8xx + help + Say 'Y' here to use the Freescale Security Engine (SEC) + version 2 and following as found on MPC83xx, MPC85xx, etc ... + +config CRYPTO_DEV_PPC4XX + tristate "Driver AMCC PPC4xx crypto accelerator" + depends on PPC && 4xx + select CRYPTO_HASH + select CRYPTO_AEAD + select CRYPTO_AES + select CRYPTO_LIB_AES + select CRYPTO_CCM + select CRYPTO_CTR + select CRYPTO_GCM + select CRYPTO_SKCIPHER + help + This option allows you to have support for AMCC crypto acceleration. + +config HW_RANDOM_PPC4XX + bool "PowerPC 4xx generic true random number generator support" + depends on CRYPTO_DEV_PPC4XX && HW_RANDOM=y + default y + help + This option provides the kernel-side support for the TRNG hardware + found in the security function of some PowerPC 4xx SoCs. + +config CRYPTO_DEV_OMAP + tristate "Support for OMAP crypto HW accelerators" + depends on ARCH_OMAP2PLUS || COMPILE_TEST + help + OMAP processors have various crypto HW accelerators. Select this if + you want to use the OMAP modules for any of the crypto algorithms. + +if CRYPTO_DEV_OMAP + +config CRYPTO_DEV_OMAP_SHAM + tristate "Support for OMAP MD5/SHA1/SHA2 hw accelerator" + depends on ARCH_OMAP2PLUS + select CRYPTO_ENGINE + select CRYPTO_SHA1 + select CRYPTO_MD5 + select CRYPTO_SHA256 + select CRYPTO_SHA512 + select CRYPTO_HMAC + help + OMAP processors have MD5/SHA1/SHA2 hw accelerator. Select this if you + want to use the OMAP module for MD5/SHA1/SHA2 algorithms. + +config CRYPTO_DEV_OMAP_AES + tristate "Support for OMAP AES hw engine" + depends on ARCH_OMAP2 || ARCH_OMAP3 || ARCH_OMAP2PLUS + select CRYPTO_AES + select CRYPTO_SKCIPHER + select CRYPTO_ENGINE + select CRYPTO_CBC + select CRYPTO_ECB + select CRYPTO_CTR + select CRYPTO_AEAD + help + OMAP processors have AES module accelerator. Select this if you + want to use the OMAP module for AES algorithms. + +config CRYPTO_DEV_OMAP_DES + tristate "Support for OMAP DES/3DES hw engine" + depends on ARCH_OMAP2PLUS || COMPILE_TEST + select CRYPTO_LIB_DES + select CRYPTO_SKCIPHER + select CRYPTO_ENGINE + help + OMAP processors have DES/3DES module accelerator. Select this if you + want to use the OMAP module for DES and 3DES algorithms. Currently + the ECB and CBC modes of operation are supported by the driver. Also + accesses made on unaligned boundaries are supported. + +endif # CRYPTO_DEV_OMAP + +config CRYPTO_DEV_SAHARA + tristate "Support for SAHARA crypto accelerator" + depends on ARCH_MXC && OF + select CRYPTO_SKCIPHER + select CRYPTO_AES + select CRYPTO_ECB + select CRYPTO_ENGINE + help + This option enables support for the SAHARA HW crypto accelerator + found in some Freescale i.MX chips. + +config CRYPTO_DEV_S5P + tristate "Support for Samsung S5PV210/Exynos crypto accelerator" + depends on ARCH_S5PV210 || ARCH_EXYNOS || COMPILE_TEST + depends on HAS_IOMEM + select CRYPTO_AES + select CRYPTO_SKCIPHER + help + This option allows you to have support for S5P crypto acceleration. + Select this to offload Samsung S5PV210 or S5PC110, Exynos from AES + algorithms execution. + +config CRYPTO_DEV_EXYNOS_HASH + bool "Support for Samsung Exynos HASH accelerator" + depends on CRYPTO_DEV_S5P + select CRYPTO_SHA1 + select CRYPTO_MD5 + select CRYPTO_SHA256 + help + Select this to offload Exynos from HASH MD5/SHA1/SHA256. + This will select software SHA1, MD5 and SHA256 as they are + needed for small and zero-size messages. + +config CRYPTO_DEV_NX + bool "Support for IBM PowerPC Nest (NX) cryptographic acceleration" + depends on PPC64 + help + This enables support for the NX hardware cryptographic accelerator + coprocessor that is in IBM PowerPC P7+ or later processors. This + does not actually enable any drivers, it only allows you to select + which acceleration type (encryption and/or compression) to enable. + +if CRYPTO_DEV_NX + source "drivers/crypto/nx/Kconfig" +endif + +config CRYPTO_DEV_ATMEL_AUTHENC + bool "Support for Atmel IPSEC/SSL hw accelerator" + depends on ARCH_AT91 || COMPILE_TEST + depends on CRYPTO_DEV_ATMEL_AES + help + Some Atmel processors can combine the AES and SHA hw accelerators + to enhance support of IPSEC/SSL. + Select this if you want to use the Atmel modules for + authenc(hmac(shaX),Y(cbc)) algorithms. + +config CRYPTO_DEV_ATMEL_AES + tristate "Support for Atmel AES hw accelerator" + depends on ARCH_MICROCHIP || COMPILE_TEST + select CRYPTO_AES + select CRYPTO_AEAD + select CRYPTO_SKCIPHER + select CRYPTO_AUTHENC if CRYPTO_DEV_ATMEL_AUTHENC + select CRYPTO_DEV_ATMEL_SHA if CRYPTO_DEV_ATMEL_AUTHENC + help + Some Atmel processors have AES hw accelerator. + Select this if you want to use the Atmel module for + AES algorithms. + + To compile this driver as a module, choose M here: the module + will be called atmel-aes. + +config CRYPTO_DEV_ATMEL_TDES + tristate "Support for Atmel DES/TDES hw accelerator" + depends on ARCH_AT91 || COMPILE_TEST + select CRYPTO_LIB_DES + select CRYPTO_SKCIPHER + help + Some Atmel processors have DES/TDES hw accelerator. + Select this if you want to use the Atmel module for + DES/TDES algorithms. + + To compile this driver as a module, choose M here: the module + will be called atmel-tdes. + +config CRYPTO_DEV_ATMEL_SHA + tristate "Support for Atmel SHA hw accelerator" + depends on ARCH_AT91 || COMPILE_TEST + select CRYPTO_HASH + help + Some Atmel processors have SHA1/SHA224/SHA256/SHA384/SHA512 + hw accelerator. + Select this if you want to use the Atmel module for + SHA1/SHA224/SHA256/SHA384/SHA512 algorithms. + + To compile this driver as a module, choose M here: the module + will be called atmel-sha. + +config CRYPTO_DEV_ATMEL_I2C + tristate + select BITREVERSE + +config CRYPTO_DEV_ATMEL_ECC + tristate "Support for Microchip / Atmel ECC hw accelerator" + depends on I2C + select CRYPTO_DEV_ATMEL_I2C + select CRYPTO_ECDH + select CRC16 + help + Microchip / Atmel ECC hw accelerator. + Select this if you want to use the Microchip / Atmel module for + ECDH algorithm. + + To compile this driver as a module, choose M here: the module + will be called atmel-ecc. + +config CRYPTO_DEV_ATMEL_SHA204A + tristate "Support for Microchip / Atmel SHA accelerator and RNG" + depends on I2C + select CRYPTO_DEV_ATMEL_I2C + select HW_RANDOM + select CRC16 + help + Microchip / Atmel SHA accelerator and RNG. + Select this if you want to use the Microchip / Atmel SHA204A + module as a random number generator. (Other functions of the + chip are currently not exposed by this driver) + + To compile this driver as a module, choose M here: the module + will be called atmel-sha204a. + +config CRYPTO_DEV_CCP + bool "Support for AMD Secure Processor" + depends on ((X86 && PCI) || (ARM64 && (OF_ADDRESS || ACPI))) && HAS_IOMEM + help + The AMD Secure Processor provides support for the Cryptographic Coprocessor + (CCP) and the Platform Security Processor (PSP) devices. + +if CRYPTO_DEV_CCP + source "drivers/crypto/ccp/Kconfig" +endif + +config CRYPTO_DEV_MXS_DCP + tristate "Support for Freescale MXS DCP" + depends on (ARCH_MXS || ARCH_MXC) + select STMP_DEVICE + select CRYPTO_CBC + select CRYPTO_ECB + select CRYPTO_AES + select CRYPTO_SKCIPHER + select CRYPTO_HASH + help + The Freescale i.MX23/i.MX28 has SHA1/SHA256 and AES128 CBC/ECB + co-processor on the die. + + To compile this driver as a module, choose M here: the module + will be called mxs-dcp. + +source "drivers/crypto/cavium/cpt/Kconfig" +source "drivers/crypto/cavium/nitrox/Kconfig" +source "drivers/crypto/marvell/Kconfig" +source "drivers/crypto/intel/Kconfig" + +config CRYPTO_DEV_QCE + tristate "Qualcomm crypto engine accelerator" + depends on (BROKEN && ARCH_QCOM) || COMPILE_TEST + depends on HAS_IOMEM + help + This driver supports Qualcomm crypto engine accelerator + hardware. To compile this driver as a module, choose M here. The + module will be called qcrypto. + + This driver does not have exclusive access to the + hardware, causing races with the secure world. It + is also slower than the CPU. + +config CRYPTO_DEV_QCE_SKCIPHER + bool + depends on CRYPTO_DEV_QCE + select CRYPTO_AES + select CRYPTO_LIB_DES + select CRYPTO_ECB + select CRYPTO_CBC + select CRYPTO_XTS + select CRYPTO_CTR + select CRYPTO_SKCIPHER + +config CRYPTO_DEV_QCE_SHA + bool + depends on CRYPTO_DEV_QCE + select CRYPTO_SHA1 + select CRYPTO_SHA256 + +config CRYPTO_DEV_QCE_AEAD + bool + depends on CRYPTO_DEV_QCE + select CRYPTO_AUTHENC + select CRYPTO_LIB_DES + +choice + prompt "Algorithms enabled for QCE acceleration" + default CRYPTO_DEV_QCE_ENABLE_ALL + depends on CRYPTO_DEV_QCE + help + This option allows to choose whether to build support for all algorithms + (default), hashes-only, or skciphers-only. + + The QCE engine does not appear to scale as well as the CPU to handle + multiple crypto requests. While the ipq40xx chips have 4-core CPUs, the + QCE handles only 2 requests in parallel. + + Ipsec throughput seems to improve when disabling either family of + algorithms, sharing the load with the CPU. Enabling skciphers-only + appears to work best. + + config CRYPTO_DEV_QCE_ENABLE_ALL + bool "All supported algorithms" + select CRYPTO_DEV_QCE_SKCIPHER + select CRYPTO_DEV_QCE_SHA + select CRYPTO_DEV_QCE_AEAD + help + Enable all supported algorithms: + - AES (CBC, CTR, ECB, XTS) + - 3DES (CBC, ECB) + - DES (CBC, ECB) + - SHA1, HMAC-SHA1 + - SHA256, HMAC-SHA256 + + config CRYPTO_DEV_QCE_ENABLE_SKCIPHER + bool "Symmetric-key ciphers only" + select CRYPTO_DEV_QCE_SKCIPHER + help + Enable symmetric-key ciphers only: + - AES (CBC, CTR, ECB, XTS) + - 3DES (ECB, CBC) + - DES (ECB, CBC) + + config CRYPTO_DEV_QCE_ENABLE_SHA + bool "Hash/HMAC only" + select CRYPTO_DEV_QCE_SHA + help + Enable hashes/HMAC algorithms only: + - SHA1, HMAC-SHA1 + - SHA256, HMAC-SHA256 + + config CRYPTO_DEV_QCE_ENABLE_AEAD + bool "AEAD algorithms only" + select CRYPTO_DEV_QCE_AEAD + help + Enable AEAD algorithms only: + - authenc() + - ccm(aes) + - rfc4309(ccm(aes)) +endchoice + +config CRYPTO_DEV_QCE_SW_MAX_LEN + int "Default maximum request size to use software for AES" + depends on CRYPTO_DEV_QCE && CRYPTO_DEV_QCE_SKCIPHER + default 512 + help + This sets the default maximum request size to perform AES requests + using software instead of the crypto engine. It can be changed by + setting the aes_sw_max_len parameter. + + Small blocks are processed faster in software than hardware. + Considering the 256-bit ciphers, software is 2-3 times faster than + qce at 256-bytes, 30% faster at 512, and about even at 768-bytes. + With 128-bit keys, the break-even point would be around 1024-bytes. + + The default is set a little lower, to 512 bytes, to balance the + cost in CPU usage. The minimum recommended setting is 16-bytes + (1 AES block), since AES-GCM will fail if you set it lower. + Setting this to zero will send all requests to the hardware. + + Note that 192-bit keys are not supported by the hardware and are + always processed by the software fallback, and all DES requests + are done by the hardware. + +config CRYPTO_DEV_IMGTEC_HASH + tristate "Imagination Technologies hardware hash accelerator" + depends on MIPS || COMPILE_TEST + select CRYPTO_MD5 + select CRYPTO_SHA1 + select CRYPTO_SHA256 + select CRYPTO_HASH + help + This driver interfaces with the Imagination Technologies + hardware hash accelerator. Supporting MD5/SHA1/SHA224/SHA256 + hashing algorithms. + +config CRYPTO_DEV_ROCKCHIP + tristate "Rockchip's Cryptographic Engine driver" + depends on OF && ARCH_ROCKCHIP + depends on PM + select CRYPTO_ECB + select CRYPTO_CBC + select CRYPTO_DES + select CRYPTO_AES + select CRYPTO_ENGINE + select CRYPTO_LIB_DES + select CRYPTO_MD5 + select CRYPTO_SHA1 + select CRYPTO_SHA256 + select CRYPTO_HASH + select CRYPTO_SKCIPHER + + help + This driver interfaces with the hardware crypto accelerator. + Supporting cbc/ecb chainmode, and aes/des/des3_ede cipher mode. + +config CRYPTO_DEV_ROCKCHIP_DEBUG + bool "Enable Rockchip crypto stats" + depends on CRYPTO_DEV_ROCKCHIP + depends on DEBUG_FS + help + Say y to enable Rockchip crypto debug stats. + This will create /sys/kernel/debug/rk3288_crypto/stats for displaying + the number of requests per algorithm and other internal stats. + +config CRYPTO_DEV_TEGRA + tristate "Enable Tegra Security Engine" + depends on TEGRA_HOST1X + select CRYPTO_ENGINE + + help + Select this to enable Tegra Security Engine which accelerates various + AES encryption/decryption and HASH algorithms. + +config CRYPTO_DEV_ZYNQMP_AES + tristate "Support for Xilinx ZynqMP AES hw accelerator" + depends on ZYNQMP_FIRMWARE || COMPILE_TEST + select CRYPTO_AES + select CRYPTO_ENGINE + select CRYPTO_AEAD + help + Xilinx ZynqMP has AES-GCM engine used for symmetric key + encryption and decryption. This driver interfaces with AES hw + accelerator. Select this if you want to use the ZynqMP module + for AES algorithms. + +config CRYPTO_DEV_ZYNQMP_SHA3 + tristate "Support for Xilinx ZynqMP SHA3 hardware accelerator" + depends on ZYNQMP_FIRMWARE || COMPILE_TEST + select CRYPTO_SHA3 + help + Xilinx ZynqMP has SHA3 engine used for secure hash calculation. + This driver interfaces with SHA3 hardware engine. + Select this if you want to use the ZynqMP module + for SHA3 hash computation. + +source "drivers/crypto/chelsio/Kconfig" + +source "drivers/crypto/virtio/Kconfig" + +config CRYPTO_DEV_BCM_SPU + tristate "Broadcom symmetric crypto/hash acceleration support" + depends on ARCH_BCM_IPROC + depends on MAILBOX + default m + select CRYPTO_AUTHENC + select CRYPTO_LIB_DES + select CRYPTO_MD5 + select CRYPTO_SHA1 + select CRYPTO_SHA256 + select CRYPTO_SHA512 + help + This driver provides support for Broadcom crypto acceleration using the + Secure Processing Unit (SPU). The SPU driver registers skcipher, + ahash, and aead algorithms with the kernel cryptographic API. + +source "drivers/crypto/stm32/Kconfig" + +config CRYPTO_DEV_SAFEXCEL + tristate "Inside Secure's SafeXcel cryptographic engine driver" + depends on (OF || PCI || COMPILE_TEST) && HAS_IOMEM + select CRYPTO_LIB_AES + select CRYPTO_AUTHENC + select CRYPTO_SKCIPHER + select CRYPTO_LIB_DES + select CRYPTO_HASH + select CRYPTO_HMAC + select CRYPTO_MD5 + select CRYPTO_SHA1 + select CRYPTO_SHA256 + select CRYPTO_SHA512 + select CRYPTO_CHACHA20POLY1305 + select CRYPTO_SHA3 + help + This driver interfaces with the SafeXcel EIP-97 and EIP-197 cryptographic + engines designed by Inside Secure. It currently accelerates DES, 3DES and + AES block ciphers in ECB and CBC mode, as well as SHA1, SHA224, SHA256, + SHA384 and SHA512 hash algorithms for both basic hash and HMAC. + Additionally, it accelerates combined AES-CBC/HMAC-SHA AEAD operations. + +config CRYPTO_DEV_ARTPEC6 + tristate "Support for Axis ARTPEC-6/7 hardware crypto acceleration." + depends on ARM && (ARCH_ARTPEC || COMPILE_TEST) + depends on OF + select CRYPTO_AEAD + select CRYPTO_AES + select CRYPTO_ALGAPI + select CRYPTO_SKCIPHER + select CRYPTO_CTR + select CRYPTO_HASH + select CRYPTO_SHA1 + select CRYPTO_SHA256 + select CRYPTO_SHA512 + help + Enables the driver for the on-chip crypto accelerator + of Axis ARTPEC SoCs. + + To compile this driver as a module, choose M here. + +config CRYPTO_DEV_CCREE + tristate "Support for ARM TrustZone CryptoCell family of security processors" + depends on CRYPTO && CRYPTO_HW && OF && HAS_DMA + depends on HAS_IOMEM + select CRYPTO_HASH + select CRYPTO_SKCIPHER + select CRYPTO_LIB_DES + select CRYPTO_AEAD + select CRYPTO_AUTHENC + select CRYPTO_SHA1 + select CRYPTO_MD5 + select CRYPTO_SHA256 + select CRYPTO_SHA512 + select CRYPTO_HMAC + select CRYPTO_AES + select CRYPTO_CBC + select CRYPTO_ECB + select CRYPTO_CTR + select CRYPTO_XTS + select CRYPTO_SM4_GENERIC + select CRYPTO_SM3 + help + Say 'Y' to enable a driver for the REE interface of the Arm + TrustZone CryptoCell family of processors. Currently the + CryptoCell 713, 703, 712, 710 and 630 are supported. + Choose this if you wish to use hardware acceleration of + cryptographic operations on the system REE. + If unsure say Y. + +source "drivers/crypto/hisilicon/Kconfig" + +source "drivers/crypto/amlogic/Kconfig" + +config CRYPTO_DEV_SA2UL + tristate "Support for TI security accelerator" + depends on ARCH_K3 || COMPILE_TEST + select CRYPTO_AES + select CRYPTO_ALGAPI + select CRYPTO_AUTHENC + select CRYPTO_DES + select CRYPTO_SHA1 + select CRYPTO_SHA256 + select CRYPTO_SHA512 + select HW_RANDOM + select SG_SPLIT + help + K3 devices include a security accelerator engine that may be + used for crypto offload. Select this if you want to use hardware + acceleration for cryptographic algorithms on these devices. + +source "drivers/crypto/aspeed/Kconfig" +source "drivers/crypto/starfive/Kconfig" +source "drivers/crypto/inside-secure/eip93/Kconfig" +source "drivers/crypto/ti/Kconfig" + +endif # CRYPTO_HW diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile new file mode 100644 index 000000000..2c33b83f3 --- /dev/null +++ b/drivers/crypto/Makefile @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_CRYPTO_DEV_ALLWINNER) += allwinner/ +obj-$(CONFIG_CRYPTO_DEV_ASPEED) += aspeed/ +obj-$(CONFIG_CRYPTO_DEV_ATMEL_AES) += atmel-aes.o +obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA) += atmel-sha.o +obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o +# __init ordering requires atmel-i2c being before atmel-ecc and atmel-sha204a. +obj-$(CONFIG_CRYPTO_DEV_ATMEL_I2C) += atmel-i2c.o +obj-$(CONFIG_CRYPTO_DEV_ATMEL_ECC) += atmel-ecc.o +obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA204A) += atmel-sha204a.o +obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/ +obj-$(CONFIG_CRYPTO_DEV_CCREE) += ccree/ +obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chelsio/ +obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_COMMON) += caam/ +obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o +obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o +obj-$(CONFIG_CRYPTO_DEV_IMGTEC_HASH) += img-hash.o +obj-$(CONFIG_CRYPTO_DEV_MARVELL) += marvell/ +obj-$(CONFIG_CRYPTO_DEV_MXS_DCP) += mxs-dcp.o +obj-$(CONFIG_CRYPTO_DEV_NX) += nx/ +obj-$(CONFIG_CRYPTO_DEV_OMAP) += omap-crypto.o +obj-$(CONFIG_CRYPTO_DEV_OMAP_AES) += omap-aes-driver.o +omap-aes-driver-objs := omap-aes.o omap-aes-gcm.o +obj-$(CONFIG_CRYPTO_DEV_OMAP_DES) += omap-des.o +obj-$(CONFIG_CRYPTO_DEV_OMAP_SHAM) += omap-sham.o +obj-$(CONFIG_CRYPTO_DEV_PADLOCK_AES) += padlock-aes.o +obj-$(CONFIG_CRYPTO_DEV_PADLOCK_SHA) += padlock-sha.o +obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += amcc/ +obj-$(CONFIG_CRYPTO_DEV_QCE) += qce/ +obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP) += rockchip/ +obj-$(CONFIG_CRYPTO_DEV_S5P) += s5p-sss.o +obj-$(CONFIG_CRYPTO_DEV_SA2UL) += sa2ul.o +obj-$(CONFIG_CRYPTO_DEV_SAHARA) += sahara.o +obj-$(CONFIG_CRYPTO_DEV_SL3516) += gemini/ +obj-y += stm32/ +obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o +obj-$(CONFIG_CRYPTO_DEV_TEGRA) += tegra/ +obj-$(CONFIG_CRYPTO_DEV_VIRTIO) += virtio/ +obj-$(CONFIG_CRYPTO_DEV_BCM_SPU) += bcm/ +obj-y += inside-secure/ +obj-$(CONFIG_CRYPTO_DEV_ARTPEC6) += axis/ +obj-y += xilinx/ +obj-y += hisilicon/ +obj-$(CONFIG_CRYPTO_DEV_AMLOGIC_GXL) += amlogic/ +obj-y += intel/ +obj-y += starfive/ +obj-y += cavium/ +obj-y += ti/ diff --git a/drivers/crypto/allwinner/Kconfig b/drivers/crypto/allwinner/Kconfig new file mode 100644 index 000000000..d86ae005f --- /dev/null +++ b/drivers/crypto/allwinner/Kconfig @@ -0,0 +1,115 @@ +config CRYPTO_DEV_ALLWINNER + bool "Support for Allwinner cryptographic offloader" + depends on ARCH_SUNXI || COMPILE_TEST + default y if ARCH_SUNXI + help + Say Y here to get to see options for Allwinner hardware crypto devices + +config CRYPTO_DEV_SUN4I_SS + tristate "Support for Allwinner Security System cryptographic accelerator" + depends on ARCH_SUNXI + depends on PM + depends on CRYPTO_DEV_ALLWINNER + select CRYPTO_MD5 + select CRYPTO_SHA1 + select CRYPTO_AES + select CRYPTO_LIB_DES + select CRYPTO_SKCIPHER + help + Some Allwinner SoC have a crypto accelerator named + Security System. Select this if you want to use it. + The Security System handle AES/DES/3DES ciphers in CBC mode + and SHA1 and MD5 hash algorithms. + + To compile this driver as a module, choose M here: the module + will be called sun4i-ss. + +config CRYPTO_DEV_SUN4I_SS_DEBUG + bool "Enable sun4i-ss stats" + depends on CRYPTO_DEV_SUN4I_SS + depends on DEBUG_FS + help + Say y to enable sun4i-ss debug stats. + This will create /sys/kernel/debug/sun4i-ss/stats for displaying + the number of requests per algorithm. + +config CRYPTO_DEV_SUN8I_CE + tristate "Support for Allwinner Crypto Engine cryptographic offloader" + select CRYPTO_SKCIPHER + select CRYPTO_ENGINE + select CRYPTO_ECB + select CRYPTO_CBC + select CRYPTO_AES + select CRYPTO_DES + depends on CRYPTO_DEV_ALLWINNER + depends on PM + help + Select y here to have support for the crypto Engine available on + Allwinner SoC H2+, H3, H5, H6, R40 and A64. + The Crypto Engine handle AES/3DES ciphers in ECB/CBC mode. + + To compile this driver as a module, choose M here: the module + will be called sun8i-ce. + +config CRYPTO_DEV_SUN8I_CE_DEBUG + bool "Enable sun8i-ce stats" + depends on CRYPTO_DEV_SUN8I_CE + depends on DEBUG_FS + help + Say y to enable sun8i-ce debug stats. + This will create /sys/kernel/debug/sun8i-ce/stats for displaying + the number of requests per flow and per algorithm. + +config CRYPTO_DEV_SUN8I_CE_HASH + bool "Enable support for hash on sun8i-ce" + depends on CRYPTO_DEV_SUN8I_CE + select CRYPTO_MD5 + select CRYPTO_SHA1 + select CRYPTO_SHA256 + select CRYPTO_SHA512 + help + Say y to enable support for hash algorithms. + +config CRYPTO_DEV_SUN8I_CE_TRNG + bool "Support for Allwinner Crypto Engine TRNG" + depends on CRYPTO_DEV_SUN8I_CE + select HW_RANDOM + help + Select this option if you want to provide kernel-side support for + the True Random Number Generator found in the Crypto Engine. + +config CRYPTO_DEV_SUN8I_SS + tristate "Support for Allwinner Security System cryptographic offloader" + select CRYPTO_SKCIPHER + select CRYPTO_ENGINE + select CRYPTO_ECB + select CRYPTO_CBC + select CRYPTO_AES + select CRYPTO_DES + depends on CRYPTO_DEV_ALLWINNER + depends on PM + help + Select y here to have support for the Security System available on + Allwinner SoC A80, A83T. + The Security System handle AES/3DES ciphers in ECB/CBC mode. + + To compile this driver as a module, choose M here: the module + will be called sun8i-ss. + +config CRYPTO_DEV_SUN8I_SS_DEBUG + bool "Enable sun8i-ss stats" + depends on CRYPTO_DEV_SUN8I_SS + depends on DEBUG_FS + help + Say y to enable sun8i-ss debug stats. + This will create /sys/kernel/debug/sun8i-ss/stats for displaying + the number of requests per flow and per algorithm. + +config CRYPTO_DEV_SUN8I_SS_HASH + bool "Enable support for hash on sun8i-ss" + depends on CRYPTO_DEV_SUN8I_SS + select CRYPTO_MD5 + select CRYPTO_SHA1 + select CRYPTO_SHA256 + help + Say y to enable support for hash algorithms. diff --git a/drivers/crypto/allwinner/Makefile b/drivers/crypto/allwinner/Makefile new file mode 100644 index 000000000..6effe864d --- /dev/null +++ b/drivers/crypto/allwinner/Makefile @@ -0,0 +1,3 @@ +obj-$(CONFIG_CRYPTO_DEV_SUN4I_SS) += sun4i-ss/ +obj-$(CONFIG_CRYPTO_DEV_SUN8I_CE) += sun8i-ce/ +obj-$(CONFIG_CRYPTO_DEV_SUN8I_SS) += sun8i-ss/ diff --git a/drivers/crypto/allwinner/sun4i-ss/Makefile b/drivers/crypto/allwinner/sun4i-ss/Makefile new file mode 100644 index 000000000..06a9ae81f --- /dev/null +++ b/drivers/crypto/allwinner/sun4i-ss/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_SUN4I_SS) += sun4i-ss.o +sun4i-ss-y += sun4i-ss-core.o sun4i-ss-hash.o sun4i-ss-cipher.o diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-cipher.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-cipher.c new file mode 100644 index 000000000..f2dd66706 --- /dev/null +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-cipher.c @@ -0,0 +1,644 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * sun4i-ss-cipher.c - hardware cryptographic accelerator for Allwinner A20 SoC + * + * Copyright (C) 2013-2015 Corentin LABBE + * + * This file add support for AES cipher with 128,192,256 bits + * keysize in CBC and ECB mode. + * Add support also for DES and 3DES in CBC and ECB mode. + * + * You could find the datasheet in Documentation/arch/arm/sunxi.rst + */ +#include "sun4i-ss.h" + +static int noinline_for_stack sun4i_ss_opti_poll(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_ss_ctx *ss = op->ss; + unsigned int ivsize = crypto_skcipher_ivsize(tfm); + struct sun4i_cipher_req_ctx *ctx = skcipher_request_ctx(areq); + u32 mode = ctx->mode; + /* when activating SS, the default FIFO space is SS_RX_DEFAULT(32) */ + u32 rx_cnt = SS_RX_DEFAULT; + u32 tx_cnt = 0; + u32 spaces; + u32 v; + int err = 0; + unsigned int i; + unsigned int ileft = areq->cryptlen; + unsigned int oleft = areq->cryptlen; + unsigned int todo; + unsigned long pi = 0, po = 0; /* progress for in and out */ + bool miter_err; + struct sg_mapping_iter mi, mo; + unsigned int oi, oo; /* offset for in and out */ + unsigned long flags; + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct sun4i_ss_alg_template *algt; + + if (!areq->cryptlen) + return 0; + + if (!areq->src || !areq->dst) { + dev_err_ratelimited(ss->dev, "ERROR: Some SGs are NULL\n"); + return -EINVAL; + } + + if (areq->iv && ivsize > 0 && mode & SS_DECRYPTION) { + scatterwalk_map_and_copy(ctx->backup_iv, areq->src, + areq->cryptlen - ivsize, ivsize, 0); + } + + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN4I_SS_DEBUG)) { + algt = container_of(alg, struct sun4i_ss_alg_template, alg.crypto); + algt->stat_opti++; + algt->stat_bytes += areq->cryptlen; + } + + spin_lock_irqsave(&ss->slock, flags); + + for (i = 0; i < op->keylen / 4; i++) + writesl(ss->base + SS_KEY0 + i * 4, &op->key[i], 1); + + if (areq->iv) { + for (i = 0; i < 4 && i < ivsize / 4; i++) { + v = *(u32 *)(areq->iv + i * 4); + writesl(ss->base + SS_IV0 + i * 4, &v, 1); + } + } + writel(mode, ss->base + SS_CTL); + + + ileft = areq->cryptlen / 4; + oleft = areq->cryptlen / 4; + oi = 0; + oo = 0; + do { + if (ileft) { + sg_miter_start(&mi, areq->src, sg_nents(areq->src), + SG_MITER_FROM_SG | SG_MITER_ATOMIC); + if (pi) + sg_miter_skip(&mi, pi); + miter_err = sg_miter_next(&mi); + if (!miter_err || !mi.addr) { + dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n"); + err = -EINVAL; + goto release_ss; + } + todo = min(rx_cnt, ileft); + todo = min_t(size_t, todo, (mi.length - oi) / 4); + if (todo) { + ileft -= todo; + writesl(ss->base + SS_RXFIFO, mi.addr + oi, todo); + oi += todo * 4; + } + if (oi == mi.length) { + pi += mi.length; + oi = 0; + } + sg_miter_stop(&mi); + } + + spaces = readl(ss->base + SS_FCSR); + rx_cnt = SS_RXFIFO_SPACES(spaces); + tx_cnt = SS_TXFIFO_SPACES(spaces); + + sg_miter_start(&mo, areq->dst, sg_nents(areq->dst), + SG_MITER_TO_SG | SG_MITER_ATOMIC); + if (po) + sg_miter_skip(&mo, po); + miter_err = sg_miter_next(&mo); + if (!miter_err || !mo.addr) { + dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n"); + err = -EINVAL; + goto release_ss; + } + todo = min(tx_cnt, oleft); + todo = min_t(size_t, todo, (mo.length - oo) / 4); + if (todo) { + oleft -= todo; + readsl(ss->base + SS_TXFIFO, mo.addr + oo, todo); + oo += todo * 4; + } + if (oo == mo.length) { + oo = 0; + po += mo.length; + } + sg_miter_stop(&mo); + } while (oleft); + + if (areq->iv) { + if (mode & SS_DECRYPTION) { + memcpy(areq->iv, ctx->backup_iv, ivsize); + memzero_explicit(ctx->backup_iv, ivsize); + } else { + scatterwalk_map_and_copy(areq->iv, areq->dst, areq->cryptlen - ivsize, + ivsize, 0); + } + } + +release_ss: + writel(0, ss->base + SS_CTL); + spin_unlock_irqrestore(&ss->slock, flags); + return err; +} + +static int noinline_for_stack sun4i_ss_cipher_poll_fallback(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *ctx = skcipher_request_ctx(areq); + int err; + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct sun4i_ss_alg_template *algt; + + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN4I_SS_DEBUG)) { + algt = container_of(alg, struct sun4i_ss_alg_template, alg.crypto); + algt->stat_fb++; + } + + skcipher_request_set_tfm(&ctx->fallback_req, op->fallback_tfm); + skcipher_request_set_callback(&ctx->fallback_req, areq->base.flags, + areq->base.complete, areq->base.data); + skcipher_request_set_crypt(&ctx->fallback_req, areq->src, areq->dst, + areq->cryptlen, areq->iv); + if (ctx->mode & SS_DECRYPTION) + err = crypto_skcipher_decrypt(&ctx->fallback_req); + else + err = crypto_skcipher_encrypt(&ctx->fallback_req); + + return err; +} + +/* Generic function that support SG with size not multiple of 4 */ +static int sun4i_ss_cipher_poll(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_ss_ctx *ss = op->ss; + int no_chunk = 1; + struct scatterlist *in_sg = areq->src; + struct scatterlist *out_sg = areq->dst; + unsigned int ivsize = crypto_skcipher_ivsize(tfm); + struct sun4i_cipher_req_ctx *ctx = skcipher_request_ctx(areq); + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct sun4i_ss_alg_template *algt; + u32 mode = ctx->mode; + /* when activating SS, the default FIFO space is SS_RX_DEFAULT(32) */ + u32 rx_cnt = SS_RX_DEFAULT; + u32 tx_cnt = 0; + u32 v; + u32 spaces; + int err = 0; + unsigned int i; + unsigned int ileft = areq->cryptlen; + unsigned int oleft = areq->cryptlen; + unsigned int todo; + struct sg_mapping_iter mi, mo; + unsigned long pi = 0, po = 0; /* progress for in and out */ + bool miter_err; + unsigned int oi, oo; /* offset for in and out */ + unsigned int ob = 0; /* offset in buf */ + unsigned int obo = 0; /* offset in bufo*/ + unsigned int obl = 0; /* length of data in bufo */ + unsigned long flags; + bool need_fallback = false; + + if (!areq->cryptlen) + return 0; + + if (!areq->src || !areq->dst) { + dev_err_ratelimited(ss->dev, "ERROR: Some SGs are NULL\n"); + return -EINVAL; + } + + algt = container_of(alg, struct sun4i_ss_alg_template, alg.crypto); + if (areq->cryptlen % algt->alg.crypto.base.cra_blocksize) + need_fallback = true; + + /* + * if we have only SGs with size multiple of 4, + * we can use the SS optimized function + */ + while (in_sg && no_chunk == 1) { + if ((in_sg->length | in_sg->offset) & 3u) + no_chunk = 0; + in_sg = sg_next(in_sg); + } + while (out_sg && no_chunk == 1) { + if ((out_sg->length | out_sg->offset) & 3u) + no_chunk = 0; + out_sg = sg_next(out_sg); + } + + if (no_chunk == 1 && !need_fallback) + return sun4i_ss_opti_poll(areq); + + if (need_fallback) + return sun4i_ss_cipher_poll_fallback(areq); + + if (areq->iv && ivsize > 0 && mode & SS_DECRYPTION) { + scatterwalk_map_and_copy(ctx->backup_iv, areq->src, + areq->cryptlen - ivsize, ivsize, 0); + } + + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN4I_SS_DEBUG)) { + algt->stat_req++; + algt->stat_bytes += areq->cryptlen; + } + + spin_lock_irqsave(&ss->slock, flags); + + for (i = 0; i < op->keylen / 4; i++) + writesl(ss->base + SS_KEY0 + i * 4, &op->key[i], 1); + + if (areq->iv) { + for (i = 0; i < 4 && i < ivsize / 4; i++) { + v = *(u32 *)(areq->iv + i * 4); + writesl(ss->base + SS_IV0 + i * 4, &v, 1); + } + } + writel(mode, ss->base + SS_CTL); + + ileft = areq->cryptlen; + oleft = areq->cryptlen; + oi = 0; + oo = 0; + + while (oleft) { + if (ileft) { + sg_miter_start(&mi, areq->src, sg_nents(areq->src), + SG_MITER_FROM_SG | SG_MITER_ATOMIC); + if (pi) + sg_miter_skip(&mi, pi); + miter_err = sg_miter_next(&mi); + if (!miter_err || !mi.addr) { + dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n"); + err = -EINVAL; + goto release_ss; + } + /* + * todo is the number of consecutive 4byte word that we + * can read from current SG + */ + todo = min(rx_cnt, ileft / 4); + todo = min_t(size_t, todo, (mi.length - oi) / 4); + if (todo && !ob) { + writesl(ss->base + SS_RXFIFO, mi.addr + oi, + todo); + ileft -= todo * 4; + oi += todo * 4; + } else { + /* + * not enough consecutive bytes, so we need to + * linearize in buf. todo is in bytes + * After that copy, if we have a multiple of 4 + * we need to be able to write all buf in one + * pass, so it is why we min() with rx_cnt + */ + todo = min(rx_cnt * 4 - ob, ileft); + todo = min_t(size_t, todo, mi.length - oi); + memcpy(ss->buf + ob, mi.addr + oi, todo); + ileft -= todo; + oi += todo; + ob += todo; + if (!(ob % 4)) { + writesl(ss->base + SS_RXFIFO, ss->buf, + ob / 4); + ob = 0; + } + } + if (oi == mi.length) { + pi += mi.length; + oi = 0; + } + sg_miter_stop(&mi); + } + + spaces = readl(ss->base + SS_FCSR); + rx_cnt = SS_RXFIFO_SPACES(spaces); + tx_cnt = SS_TXFIFO_SPACES(spaces); + + if (!tx_cnt) + continue; + sg_miter_start(&mo, areq->dst, sg_nents(areq->dst), + SG_MITER_TO_SG | SG_MITER_ATOMIC); + if (po) + sg_miter_skip(&mo, po); + miter_err = sg_miter_next(&mo); + if (!miter_err || !mo.addr) { + dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n"); + err = -EINVAL; + goto release_ss; + } + /* todo in 4bytes word */ + todo = min(tx_cnt, oleft / 4); + todo = min_t(size_t, todo, (mo.length - oo) / 4); + + if (todo) { + readsl(ss->base + SS_TXFIFO, mo.addr + oo, todo); + oleft -= todo * 4; + oo += todo * 4; + if (oo == mo.length) { + po += mo.length; + oo = 0; + } + } else { + /* + * read obl bytes in bufo, we read at maximum for + * emptying the device + */ + readsl(ss->base + SS_TXFIFO, ss->bufo, tx_cnt); + obl = tx_cnt * 4; + obo = 0; + do { + /* + * how many bytes we can copy ? + * no more than remaining SG size + * no more than remaining buffer + * no need to test against oleft + */ + todo = min_t(size_t, + mo.length - oo, obl - obo); + memcpy(mo.addr + oo, ss->bufo + obo, todo); + oleft -= todo; + obo += todo; + oo += todo; + if (oo == mo.length) { + po += mo.length; + sg_miter_next(&mo); + oo = 0; + } + } while (obo < obl); + /* bufo must be fully used here */ + } + sg_miter_stop(&mo); + } + if (areq->iv) { + if (mode & SS_DECRYPTION) { + memcpy(areq->iv, ctx->backup_iv, ivsize); + memzero_explicit(ctx->backup_iv, ivsize); + } else { + scatterwalk_map_and_copy(areq->iv, areq->dst, areq->cryptlen - ivsize, + ivsize, 0); + } + } + +release_ss: + writel(0, ss->base + SS_CTL); + spin_unlock_irqrestore(&ss->slock, flags); + + return err; +} + +/* CBC AES */ +int sun4i_ss_cbc_aes_encrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + + rctx->mode = SS_OP_AES | SS_CBC | SS_ENABLED | SS_ENCRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +int sun4i_ss_cbc_aes_decrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + + rctx->mode = SS_OP_AES | SS_CBC | SS_ENABLED | SS_DECRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +/* ECB AES */ +int sun4i_ss_ecb_aes_encrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + + rctx->mode = SS_OP_AES | SS_ECB | SS_ENABLED | SS_ENCRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +int sun4i_ss_ecb_aes_decrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + + rctx->mode = SS_OP_AES | SS_ECB | SS_ENABLED | SS_DECRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +/* CBC DES */ +int sun4i_ss_cbc_des_encrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + + rctx->mode = SS_OP_DES | SS_CBC | SS_ENABLED | SS_ENCRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +int sun4i_ss_cbc_des_decrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + + rctx->mode = SS_OP_DES | SS_CBC | SS_ENABLED | SS_DECRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +/* ECB DES */ +int sun4i_ss_ecb_des_encrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + + rctx->mode = SS_OP_DES | SS_ECB | SS_ENABLED | SS_ENCRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +int sun4i_ss_ecb_des_decrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + + rctx->mode = SS_OP_DES | SS_ECB | SS_ENABLED | SS_DECRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +/* CBC 3DES */ +int sun4i_ss_cbc_des3_encrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + + rctx->mode = SS_OP_3DES | SS_CBC | SS_ENABLED | SS_ENCRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +int sun4i_ss_cbc_des3_decrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + + rctx->mode = SS_OP_3DES | SS_CBC | SS_ENABLED | SS_DECRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +/* ECB 3DES */ +int sun4i_ss_ecb_des3_encrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + + rctx->mode = SS_OP_3DES | SS_ECB | SS_ENABLED | SS_ENCRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +int sun4i_ss_ecb_des3_decrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + + rctx->mode = SS_OP_3DES | SS_ECB | SS_ENABLED | SS_DECRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +int sun4i_ss_cipher_init(struct crypto_tfm *tfm) +{ + struct sun4i_tfm_ctx *op = crypto_tfm_ctx(tfm); + struct sun4i_ss_alg_template *algt; + const char *name = crypto_tfm_alg_name(tfm); + int err; + + memset(op, 0, sizeof(struct sun4i_tfm_ctx)); + + algt = container_of(tfm->__crt_alg, struct sun4i_ss_alg_template, + alg.crypto.base); + op->ss = algt->ss; + + op->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(op->fallback_tfm)) { + dev_err(op->ss->dev, "ERROR: Cannot allocate fallback for %s %ld\n", + name, PTR_ERR(op->fallback_tfm)); + return PTR_ERR(op->fallback_tfm); + } + + crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm), + sizeof(struct sun4i_cipher_req_ctx) + + crypto_skcipher_reqsize(op->fallback_tfm)); + + err = pm_runtime_resume_and_get(op->ss->dev); + if (err < 0) + goto error_pm; + + return 0; +error_pm: + crypto_free_skcipher(op->fallback_tfm); + return err; +} + +void sun4i_ss_cipher_exit(struct crypto_tfm *tfm) +{ + struct sun4i_tfm_ctx *op = crypto_tfm_ctx(tfm); + + crypto_free_skcipher(op->fallback_tfm); + pm_runtime_put(op->ss->dev); +} + +/* check and set the AES key, prepare the mode to be used */ +int sun4i_ss_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_ss_ctx *ss = op->ss; + + switch (keylen) { + case 128 / 8: + op->keymode = SS_AES_128BITS; + break; + case 192 / 8: + op->keymode = SS_AES_192BITS; + break; + case 256 / 8: + op->keymode = SS_AES_256BITS; + break; + default: + dev_dbg(ss->dev, "ERROR: Invalid keylen %u\n", keylen); + return -EINVAL; + } + op->keylen = keylen; + memcpy(op->key, key, keylen); + + crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); + + return crypto_skcipher_setkey(op->fallback_tfm, key, keylen); +} + +/* check and set the DES key, prepare the mode to be used */ +int sun4i_ss_des_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + int err; + + err = verify_skcipher_des_key(tfm, key); + if (err) + return err; + + op->keylen = keylen; + memcpy(op->key, key, keylen); + + crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); + + return crypto_skcipher_setkey(op->fallback_tfm, key, keylen); +} + +/* check and set the 3DES key, prepare the mode to be used */ +int sun4i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + int err; + + err = verify_skcipher_des3_key(tfm, key); + if (err) + return err; + + op->keylen = keylen; + memcpy(op->key, key, keylen); + + crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); + + return crypto_skcipher_setkey(op->fallback_tfm, key, keylen); +} diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c new file mode 100644 index 000000000..35ef0930e --- /dev/null +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c @@ -0,0 +1,522 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * sun4i-ss-core.c - hardware cryptographic accelerator for Allwinner A20 SoC + * + * Copyright (C) 2013-2015 Corentin LABBE + * + * Core file which registers crypto algorithms supported by the SS. + * + * You could find a link for the datasheet in Documentation/arch/arm/sunxi.rst + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sun4i-ss.h" + +static const struct ss_variant ss_a10_variant = { + .sha1_in_be = false, +}; + +static const struct ss_variant ss_a33_variant = { + .sha1_in_be = true, +}; + +static struct sun4i_ss_alg_template ss_algs[] = { +{ .type = CRYPTO_ALG_TYPE_AHASH, + .mode = SS_OP_MD5, + .alg.hash = { + .init = sun4i_hash_init, + .update = sun4i_hash_update, + .final = sun4i_hash_final, + .finup = sun4i_hash_finup, + .digest = sun4i_hash_digest, + .export = sun4i_hash_export_md5, + .import = sun4i_hash_import_md5, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct md5_state), + .base = { + .cra_name = "md5", + .cra_driver_name = "md5-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun4i_req_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun4i_hash_crainit, + .cra_exit = sun4i_hash_craexit, + } + } + } +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .mode = SS_OP_SHA1, + .alg.hash = { + .init = sun4i_hash_init, + .update = sun4i_hash_update, + .final = sun4i_hash_final, + .finup = sun4i_hash_finup, + .digest = sun4i_hash_digest, + .export = sun4i_hash_export_sha1, + .import = sun4i_hash_import_sha1, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct sha1_state), + .base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun4i_req_ctx), + .cra_module = THIS_MODULE, + .cra_init = sun4i_hash_crainit, + .cra_exit = sun4i_hash_craexit, + } + } + } +}, +{ .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.crypto = { + .setkey = sun4i_ss_aes_setkey, + .encrypt = sun4i_ss_cbc_aes_encrypt, + .decrypt = sun4i_ss_cbc_aes_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct sun4i_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 3, + .cra_init = sun4i_ss_cipher_init, + .cra_exit = sun4i_ss_cipher_exit, + } + } +}, +{ .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.crypto = { + .setkey = sun4i_ss_aes_setkey, + .encrypt = sun4i_ss_ecb_aes_encrypt, + .decrypt = sun4i_ss_ecb_aes_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct sun4i_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 3, + .cra_init = sun4i_ss_cipher_init, + .cra_exit = sun4i_ss_cipher_exit, + } + } +}, +{ .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.crypto = { + .setkey = sun4i_ss_des_setkey, + .encrypt = sun4i_ss_cbc_des_encrypt, + .decrypt = sun4i_ss_cbc_des_decrypt, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .base = { + .cra_name = "cbc(des)", + .cra_driver_name = "cbc-des-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct sun4i_req_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 3, + .cra_init = sun4i_ss_cipher_init, + .cra_exit = sun4i_ss_cipher_exit, + } + } +}, +{ .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.crypto = { + .setkey = sun4i_ss_des_setkey, + .encrypt = sun4i_ss_ecb_des_encrypt, + .decrypt = sun4i_ss_ecb_des_decrypt, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .base = { + .cra_name = "ecb(des)", + .cra_driver_name = "ecb-des-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct sun4i_req_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 3, + .cra_init = sun4i_ss_cipher_init, + .cra_exit = sun4i_ss_cipher_exit, + } + } +}, +{ .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.crypto = { + .setkey = sun4i_ss_des3_setkey, + .encrypt = sun4i_ss_cbc_des3_encrypt, + .decrypt = sun4i_ss_cbc_des3_decrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .base = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "cbc-des3-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct sun4i_req_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 3, + .cra_init = sun4i_ss_cipher_init, + .cra_exit = sun4i_ss_cipher_exit, + } + } +}, +{ .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.crypto = { + .setkey = sun4i_ss_des3_setkey, + .encrypt = sun4i_ss_ecb_des3_encrypt, + .decrypt = sun4i_ss_ecb_des3_decrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .base = { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "ecb-des3-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct sun4i_req_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 3, + .cra_init = sun4i_ss_cipher_init, + .cra_exit = sun4i_ss_cipher_exit, + } + } +}, +}; + +static int sun4i_ss_debugfs_show(struct seq_file *seq, void *v) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(ss_algs); i++) { + if (!ss_algs[i].ss) + continue; + switch (ss_algs[i].type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + seq_printf(seq, "%s %s reqs=%lu opti=%lu fallback=%lu tsize=%lu\n", + ss_algs[i].alg.crypto.base.cra_driver_name, + ss_algs[i].alg.crypto.base.cra_name, + ss_algs[i].stat_req, ss_algs[i].stat_opti, ss_algs[i].stat_fb, + ss_algs[i].stat_bytes); + break; + case CRYPTO_ALG_TYPE_AHASH: + seq_printf(seq, "%s %s reqs=%lu\n", + ss_algs[i].alg.hash.halg.base.cra_driver_name, + ss_algs[i].alg.hash.halg.base.cra_name, + ss_algs[i].stat_req); + break; + } + } + return 0; +} +DEFINE_SHOW_ATTRIBUTE(sun4i_ss_debugfs); + +/* + * Power management strategy: The device is suspended unless a TFM exists for + * one of the algorithms proposed by this driver. + */ +static int sun4i_ss_pm_suspend(struct device *dev) +{ + struct sun4i_ss_ctx *ss = dev_get_drvdata(dev); + + reset_control_assert(ss->reset); + + clk_disable_unprepare(ss->ssclk); + clk_disable_unprepare(ss->busclk); + return 0; +} + +static int sun4i_ss_pm_resume(struct device *dev) +{ + struct sun4i_ss_ctx *ss = dev_get_drvdata(dev); + + int err; + + err = clk_prepare_enable(ss->busclk); + if (err) { + dev_err(ss->dev, "Cannot prepare_enable busclk\n"); + goto err_enable; + } + + err = clk_prepare_enable(ss->ssclk); + if (err) { + dev_err(ss->dev, "Cannot prepare_enable ssclk\n"); + goto err_enable; + } + + err = reset_control_deassert(ss->reset); + if (err) { + dev_err(ss->dev, "Cannot deassert reset control\n"); + goto err_enable; + } + + return err; +err_enable: + sun4i_ss_pm_suspend(dev); + return err; +} + +static const struct dev_pm_ops sun4i_ss_pm_ops = { + SET_RUNTIME_PM_OPS(sun4i_ss_pm_suspend, sun4i_ss_pm_resume, NULL) +}; + +/* + * When power management is enabled, this function enables the PM and set the + * device as suspended + * When power management is disabled, this function just enables the device + */ +static int sun4i_ss_pm_init(struct sun4i_ss_ctx *ss) +{ + int err; + + pm_runtime_use_autosuspend(ss->dev); + pm_runtime_set_autosuspend_delay(ss->dev, 2000); + + err = pm_runtime_set_suspended(ss->dev); + if (err) + return err; + pm_runtime_enable(ss->dev); + return err; +} + +static void sun4i_ss_pm_exit(struct sun4i_ss_ctx *ss) +{ + pm_runtime_disable(ss->dev); +} + +static int sun4i_ss_probe(struct platform_device *pdev) +{ + u32 v; + int err, i; + unsigned long cr; + const unsigned long cr_ahb = 24 * 1000 * 1000; + const unsigned long cr_mod = 150 * 1000 * 1000; + struct sun4i_ss_ctx *ss; + + if (!pdev->dev.of_node) + return -ENODEV; + + ss = devm_kzalloc(&pdev->dev, sizeof(*ss), GFP_KERNEL); + if (!ss) + return -ENOMEM; + + ss->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(ss->base)) { + dev_err(&pdev->dev, "Cannot request MMIO\n"); + return PTR_ERR(ss->base); + } + + ss->variant = of_device_get_match_data(&pdev->dev); + if (!ss->variant) { + dev_err(&pdev->dev, "Missing Security System variant\n"); + return -EINVAL; + } + + ss->ssclk = devm_clk_get(&pdev->dev, "mod"); + if (IS_ERR(ss->ssclk)) { + err = PTR_ERR(ss->ssclk); + dev_err(&pdev->dev, "Cannot get SS clock err=%d\n", err); + return err; + } + dev_dbg(&pdev->dev, "clock ss acquired\n"); + + ss->busclk = devm_clk_get(&pdev->dev, "ahb"); + if (IS_ERR(ss->busclk)) { + err = PTR_ERR(ss->busclk); + dev_err(&pdev->dev, "Cannot get AHB SS clock err=%d\n", err); + return err; + } + dev_dbg(&pdev->dev, "clock ahb_ss acquired\n"); + + ss->reset = devm_reset_control_get_optional(&pdev->dev, "ahb"); + if (IS_ERR(ss->reset)) + return PTR_ERR(ss->reset); + if (!ss->reset) + dev_info(&pdev->dev, "no reset control found\n"); + + /* + * Check that clock have the correct rates given in the datasheet + * Try to set the clock to the maximum allowed + */ + err = clk_set_rate(ss->ssclk, cr_mod); + if (err) { + dev_err(&pdev->dev, "Cannot set clock rate to ssclk\n"); + return err; + } + + /* + * The only impact on clocks below requirement are bad performance, + * so do not print "errors" + * warn on Overclocked clocks + */ + cr = clk_get_rate(ss->busclk); + if (cr >= cr_ahb) + dev_dbg(&pdev->dev, "Clock bus %lu (%lu MHz) (must be >= %lu)\n", + cr, cr / 1000000, cr_ahb); + else + dev_warn(&pdev->dev, "Clock bus %lu (%lu MHz) (must be >= %lu)\n", + cr, cr / 1000000, cr_ahb); + + cr = clk_get_rate(ss->ssclk); + if (cr <= cr_mod) + if (cr < cr_mod) + dev_warn(&pdev->dev, "Clock ss %lu (%lu MHz) (must be <= %lu)\n", + cr, cr / 1000000, cr_mod); + else + dev_dbg(&pdev->dev, "Clock ss %lu (%lu MHz) (must be <= %lu)\n", + cr, cr / 1000000, cr_mod); + else + dev_warn(&pdev->dev, "Clock ss is at %lu (%lu MHz) (must be <= %lu)\n", + cr, cr / 1000000, cr_mod); + + ss->dev = &pdev->dev; + platform_set_drvdata(pdev, ss); + + spin_lock_init(&ss->slock); + + err = sun4i_ss_pm_init(ss); + if (err) + return err; + + /* + * Datasheet named it "Die Bonding ID" + * I expect to be a sort of Security System Revision number. + * Since the A80 seems to have an other version of SS + * this info could be useful + */ + + err = pm_runtime_resume_and_get(ss->dev); + if (err < 0) + goto error_pm; + + writel(SS_ENABLED, ss->base + SS_CTL); + v = readl(ss->base + SS_CTL); + v >>= 16; + v &= 0x07; + dev_info(&pdev->dev, "Die ID %d\n", v); + writel(0, ss->base + SS_CTL); + + pm_runtime_put_sync(ss->dev); + + for (i = 0; i < ARRAY_SIZE(ss_algs); i++) { + ss_algs[i].ss = ss; + switch (ss_algs[i].type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + err = crypto_register_skcipher(&ss_algs[i].alg.crypto); + if (err) { + dev_err(ss->dev, "Fail to register %s\n", + ss_algs[i].alg.crypto.base.cra_name); + goto error_alg; + } + break; + case CRYPTO_ALG_TYPE_AHASH: + err = crypto_register_ahash(&ss_algs[i].alg.hash); + if (err) { + dev_err(ss->dev, "Fail to register %s\n", + ss_algs[i].alg.hash.halg.base.cra_name); + goto error_alg; + } + break; + } + } + + /* Ignore error of debugfs */ + ss->dbgfs_dir = debugfs_create_dir("sun4i-ss", NULL); + ss->dbgfs_stats = debugfs_create_file("stats", 0444, ss->dbgfs_dir, ss, + &sun4i_ss_debugfs_fops); + + return 0; +error_alg: + i--; + for (; i >= 0; i--) { + switch (ss_algs[i].type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + crypto_unregister_skcipher(&ss_algs[i].alg.crypto); + break; + case CRYPTO_ALG_TYPE_AHASH: + crypto_unregister_ahash(&ss_algs[i].alg.hash); + break; + } + } +error_pm: + sun4i_ss_pm_exit(ss); + return err; +} + +static void sun4i_ss_remove(struct platform_device *pdev) +{ + int i; + struct sun4i_ss_ctx *ss = platform_get_drvdata(pdev); + + for (i = 0; i < ARRAY_SIZE(ss_algs); i++) { + switch (ss_algs[i].type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + crypto_unregister_skcipher(&ss_algs[i].alg.crypto); + break; + case CRYPTO_ALG_TYPE_AHASH: + crypto_unregister_ahash(&ss_algs[i].alg.hash); + break; + } + } + + sun4i_ss_pm_exit(ss); +} + +static const struct of_device_id a20ss_crypto_of_match_table[] = { + { .compatible = "allwinner,sun4i-a10-crypto", + .data = &ss_a10_variant + }, + { .compatible = "allwinner,sun8i-a33-crypto", + .data = &ss_a33_variant + }, + {} +}; +MODULE_DEVICE_TABLE(of, a20ss_crypto_of_match_table); + +static struct platform_driver sun4i_ss_driver = { + .probe = sun4i_ss_probe, + .remove = sun4i_ss_remove, + .driver = { + .name = "sun4i-ss", + .pm = &sun4i_ss_pm_ops, + .of_match_table = a20ss_crypto_of_match_table, + }, +}; + +module_platform_driver(sun4i_ss_driver); + +MODULE_ALIAS("platform:sun4i-ss"); +MODULE_DESCRIPTION("Allwinner Security System cryptographic accelerator"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Corentin LABBE "); diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c new file mode 100644 index 000000000..434f2b271 --- /dev/null +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c @@ -0,0 +1,545 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * sun4i-ss-hash.c - hardware cryptographic accelerator for Allwinner A20 SoC + * + * Copyright (C) 2013-2015 Corentin LABBE + * + * This file add support for MD5 and SHA1. + * + * You could find the datasheet in Documentation/arch/arm/sunxi.rst + */ +#include "sun4i-ss.h" +#include +#include + +/* This is a totally arbitrary value */ +#define SS_TIMEOUT 100 + +int sun4i_hash_crainit(struct crypto_tfm *tfm) +{ + struct sun4i_tfm_ctx *op = crypto_tfm_ctx(tfm); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg); + struct sun4i_ss_alg_template *algt; + int err; + + memset(op, 0, sizeof(struct sun4i_tfm_ctx)); + + algt = container_of(alg, struct sun4i_ss_alg_template, alg.hash); + op->ss = algt->ss; + + err = pm_runtime_resume_and_get(op->ss->dev); + if (err < 0) + return err; + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct sun4i_req_ctx)); + return 0; +} + +void sun4i_hash_craexit(struct crypto_tfm *tfm) +{ + struct sun4i_tfm_ctx *op = crypto_tfm_ctx(tfm); + + pm_runtime_put(op->ss->dev); +} + +/* sun4i_hash_init: initialize request context */ +int sun4i_hash_init(struct ahash_request *areq) +{ + struct sun4i_req_ctx *op = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun4i_ss_alg_template *algt; + + memset(op, 0, sizeof(struct sun4i_req_ctx)); + + algt = container_of(alg, struct sun4i_ss_alg_template, alg.hash); + op->mode = algt->mode; + + return 0; +} + +int sun4i_hash_export_md5(struct ahash_request *areq, void *out) +{ + struct sun4i_req_ctx *op = ahash_request_ctx(areq); + struct md5_state *octx = out; + int i; + + octx->byte_count = op->byte_count + op->len; + + memcpy(octx->block, op->buf, op->len); + + if (op->byte_count) { + for (i = 0; i < 4; i++) + octx->hash[i] = op->hash[i]; + } else { + octx->hash[0] = SHA1_H0; + octx->hash[1] = SHA1_H1; + octx->hash[2] = SHA1_H2; + octx->hash[3] = SHA1_H3; + } + + return 0; +} + +int sun4i_hash_import_md5(struct ahash_request *areq, const void *in) +{ + struct sun4i_req_ctx *op = ahash_request_ctx(areq); + const struct md5_state *ictx = in; + int i; + + sun4i_hash_init(areq); + + op->byte_count = ictx->byte_count & ~0x3F; + op->len = ictx->byte_count & 0x3F; + + memcpy(op->buf, ictx->block, op->len); + + for (i = 0; i < 4; i++) + op->hash[i] = ictx->hash[i]; + + return 0; +} + +int sun4i_hash_export_sha1(struct ahash_request *areq, void *out) +{ + struct sun4i_req_ctx *op = ahash_request_ctx(areq); + struct sha1_state *octx = out; + int i; + + octx->count = op->byte_count + op->len; + + memcpy(octx->buffer, op->buf, op->len); + + if (op->byte_count) { + for (i = 0; i < 5; i++) + octx->state[i] = op->hash[i]; + } else { + octx->state[0] = SHA1_H0; + octx->state[1] = SHA1_H1; + octx->state[2] = SHA1_H2; + octx->state[3] = SHA1_H3; + octx->state[4] = SHA1_H4; + } + + return 0; +} + +int sun4i_hash_import_sha1(struct ahash_request *areq, const void *in) +{ + struct sun4i_req_ctx *op = ahash_request_ctx(areq); + const struct sha1_state *ictx = in; + int i; + + sun4i_hash_init(areq); + + op->byte_count = ictx->count & ~0x3F; + op->len = ictx->count & 0x3F; + + memcpy(op->buf, ictx->buffer, op->len); + + for (i = 0; i < 5; i++) + op->hash[i] = ictx->state[i]; + + return 0; +} + +#define SS_HASH_UPDATE 1 +#define SS_HASH_FINAL 2 + +/* + * sun4i_hash_update: update hash engine + * + * Could be used for both SHA1 and MD5 + * Write data by step of 32bits and put then in the SS. + * + * Since we cannot leave partial data and hash state in the engine, + * we need to get the hash state at the end of this function. + * We can get the hash state every 64 bytes + * + * So the first work is to get the number of bytes to write to SS modulo 64 + * The extra bytes will go to a temporary buffer op->buf storing op->len bytes + * + * So at the begin of update() + * if op->len + areq->nbytes < 64 + * => all data will be written to wait buffer (op->buf) and end=0 + * if not, write all data from op->buf to the device and position end to + * complete to 64bytes + * + * example 1: + * update1 60o => op->len=60 + * update2 60o => need one more word to have 64 bytes + * end=4 + * so write all data from op->buf and one word of SGs + * write remaining data in op->buf + * final state op->len=56 + */ +static int sun4i_hash(struct ahash_request *areq) +{ + /* + * i is the total bytes read from SGs, to be compared to areq->nbytes + * i is important because we cannot rely on SG length since the sum of + * SG->length could be greater than areq->nbytes + * + * end is the position when we need to stop writing to the device, + * to be compared to i + * + * in_i: advancement in the current SG + */ + unsigned int i = 0, end, fill, min_fill, nwait, nbw = 0, j = 0, todo; + unsigned int in_i = 0; + u32 spaces, rx_cnt = SS_RX_DEFAULT, bf[32] = {0}, v, ivmode = 0; + struct sun4i_req_ctx *op = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun4i_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + struct sun4i_ss_ctx *ss = tfmctx->ss; + struct sun4i_ss_alg_template *algt; + struct scatterlist *in_sg = areq->src; + struct sg_mapping_iter mi; + int in_r, err = 0; + size_t copied = 0; + u32 wb = 0; + + dev_dbg(ss->dev, "%s %s bc=%llu len=%u mode=%x wl=%u h0=%0x", + __func__, crypto_tfm_alg_name(areq->base.tfm), + op->byte_count, areq->nbytes, op->mode, + op->len, op->hash[0]); + + if (unlikely(!areq->nbytes) && !(op->flags & SS_HASH_FINAL)) + return 0; + + /* protect against overflow */ + if (unlikely(areq->nbytes > UINT_MAX - op->len)) { + dev_err(ss->dev, "Cannot process too large request\n"); + return -EINVAL; + } + + if (op->len + areq->nbytes < 64 && !(op->flags & SS_HASH_FINAL)) { + /* linearize data to op->buf */ + copied = sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), + op->buf + op->len, areq->nbytes, 0); + op->len += copied; + return 0; + } + + spin_lock_bh(&ss->slock); + + /* + * if some data have been processed before, + * we need to restore the partial hash state + */ + if (op->byte_count) { + ivmode = SS_IV_ARBITRARY; + for (i = 0; i < crypto_ahash_digestsize(tfm) / 4; i++) + writel(op->hash[i], ss->base + SS_IV0 + i * 4); + } + /* Enable the device */ + writel(op->mode | SS_ENABLED | ivmode, ss->base + SS_CTL); + + if (!(op->flags & SS_HASH_UPDATE)) + goto hash_final; + + /* start of handling data */ + if (!(op->flags & SS_HASH_FINAL)) { + end = ((areq->nbytes + op->len) / 64) * 64 - op->len; + + if (end > areq->nbytes || areq->nbytes - end > 63) { + dev_err(ss->dev, "ERROR: Bound error %u %u\n", + end, areq->nbytes); + err = -EINVAL; + goto release_ss; + } + } else { + /* Since we have the flag final, we can go up to modulo 4 */ + if (areq->nbytes < 4) + end = 0; + else + end = ((areq->nbytes + op->len) / 4) * 4 - op->len; + } + + /* TODO if SGlen % 4 and !op->len then DMA */ + i = 1; + while (in_sg && i == 1) { + if (in_sg->length % 4) + i = 0; + in_sg = sg_next(in_sg); + } + if (i == 1 && !op->len && areq->nbytes) + dev_dbg(ss->dev, "We can DMA\n"); + + i = 0; + sg_miter_start(&mi, areq->src, sg_nents(areq->src), + SG_MITER_FROM_SG | SG_MITER_ATOMIC); + sg_miter_next(&mi); + in_i = 0; + + do { + /* + * we need to linearize in two case: + * - the buffer is already used + * - the SG does not have enough byte remaining ( < 4) + */ + if (op->len || (mi.length - in_i) < 4) { + /* + * if we have entered here we have two reason to stop + * - the buffer is full + * - reach the end + */ + while (op->len < 64 && i < end) { + /* how many bytes we can read from current SG */ + in_r = min(end - i, 64 - op->len); + in_r = min_t(size_t, mi.length - in_i, in_r); + memcpy(op->buf + op->len, mi.addr + in_i, in_r); + op->len += in_r; + i += in_r; + in_i += in_r; + if (in_i == mi.length) { + sg_miter_next(&mi); + in_i = 0; + } + } + if (op->len > 3 && !(op->len % 4)) { + /* write buf to the device */ + writesl(ss->base + SS_RXFIFO, op->buf, + op->len / 4); + op->byte_count += op->len; + op->len = 0; + } + } + if (mi.length - in_i > 3 && i < end) { + /* how many bytes we can read from current SG */ + in_r = min_t(size_t, mi.length - in_i, areq->nbytes - i); + in_r = min_t(size_t, ((mi.length - in_i) / 4) * 4, in_r); + /* how many bytes we can write in the device*/ + todo = min3((u32)(end - i) / 4, rx_cnt, (u32)in_r / 4); + writesl(ss->base + SS_RXFIFO, mi.addr + in_i, todo); + op->byte_count += todo * 4; + i += todo * 4; + in_i += todo * 4; + rx_cnt -= todo; + if (!rx_cnt) { + spaces = readl(ss->base + SS_FCSR); + rx_cnt = SS_RXFIFO_SPACES(spaces); + } + if (in_i == mi.length) { + sg_miter_next(&mi); + in_i = 0; + } + } + } while (i < end); + + /* + * Now we have written to the device all that we can, + * store the remaining bytes in op->buf + */ + if ((areq->nbytes - i) < 64) { + while (i < areq->nbytes && in_i < mi.length && op->len < 64) { + /* how many bytes we can read from current SG */ + in_r = min(areq->nbytes - i, 64 - op->len); + in_r = min_t(size_t, mi.length - in_i, in_r); + memcpy(op->buf + op->len, mi.addr + in_i, in_r); + op->len += in_r; + i += in_r; + in_i += in_r; + if (in_i == mi.length) { + sg_miter_next(&mi); + in_i = 0; + } + } + } + + sg_miter_stop(&mi); + + /* + * End of data process + * Now if we have the flag final go to finalize part + * If not, store the partial hash + */ + if (op->flags & SS_HASH_FINAL) + goto hash_final; + + writel(op->mode | SS_ENABLED | SS_DATA_END, ss->base + SS_CTL); + i = 0; + do { + v = readl(ss->base + SS_CTL); + i++; + } while (i < SS_TIMEOUT && (v & SS_DATA_END)); + if (unlikely(i >= SS_TIMEOUT)) { + dev_err_ratelimited(ss->dev, + "ERROR: hash end timeout %d>%d ctl=%x len=%u\n", + i, SS_TIMEOUT, v, areq->nbytes); + err = -EIO; + goto release_ss; + } + + /* + * The datasheet isn't very clear about when to retrieve the digest. The + * bit SS_DATA_END is cleared when the engine has processed the data and + * when the digest is computed *but* it doesn't mean the digest is + * available in the digest registers. Hence the delay to be sure we can + * read it. + */ + ndelay(1); + + for (i = 0; i < crypto_ahash_digestsize(tfm) / 4; i++) + op->hash[i] = readl(ss->base + SS_MD0 + i * 4); + + goto release_ss; + +/* + * hash_final: finalize hashing operation + * + * If we have some remaining bytes, we write them. + * Then ask the SS for finalizing the hashing operation + * + * I do not check RX FIFO size in this function since the size is 32 + * after each enabling and this function neither write more than 32 words. + * If we come from the update part, we cannot have more than + * 3 remaining bytes to write and SS is fast enough to not care about it. + */ + +hash_final: + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN4I_SS_DEBUG)) { + algt = container_of(alg, struct sun4i_ss_alg_template, alg.hash); + algt->stat_req++; + } + + /* write the remaining words of the wait buffer */ + if (op->len) { + nwait = op->len / 4; + if (nwait) { + writesl(ss->base + SS_RXFIFO, op->buf, nwait); + op->byte_count += 4 * nwait; + } + + nbw = op->len - 4 * nwait; + if (nbw) { + wb = le32_to_cpup((__le32 *)(op->buf + nwait * 4)); + wb &= GENMASK((nbw * 8) - 1, 0); + + op->byte_count += nbw; + } + } + + /* write the remaining bytes of the nbw buffer */ + wb |= ((1 << 7) << (nbw * 8)); + ((__le32 *)bf)[j++] = cpu_to_le32(wb); + + /* + * number of space to pad to obtain 64o minus 8(size) minus 4 (final 1) + * I take the operations from other MD5/SHA1 implementations + */ + + /* last block size */ + fill = 64 - (op->byte_count % 64); + min_fill = 2 * sizeof(u32) + (nbw ? 0 : sizeof(u32)); + + /* if we can't fill all data, jump to the next 64 block */ + if (fill < min_fill) + fill += 64; + + j += (fill - min_fill) / sizeof(u32); + + /* write the length of data */ + if (op->mode == SS_OP_SHA1) { + __be64 *bits = (__be64 *)&bf[j]; + *bits = cpu_to_be64(op->byte_count << 3); + j += 2; + } else { + __le64 *bits = (__le64 *)&bf[j]; + *bits = cpu_to_le64(op->byte_count << 3); + j += 2; + } + writesl(ss->base + SS_RXFIFO, bf, j); + + /* Tell the SS to stop the hashing */ + writel(op->mode | SS_ENABLED | SS_DATA_END, ss->base + SS_CTL); + + /* + * Wait for SS to finish the hash. + * The timeout could happen only in case of bad overclocking + * or driver bug. + */ + i = 0; + do { + v = readl(ss->base + SS_CTL); + i++; + } while (i < SS_TIMEOUT && (v & SS_DATA_END)); + if (unlikely(i >= SS_TIMEOUT)) { + dev_err_ratelimited(ss->dev, + "ERROR: hash end timeout %d>%d ctl=%x len=%u\n", + i, SS_TIMEOUT, v, areq->nbytes); + err = -EIO; + goto release_ss; + } + + /* + * The datasheet isn't very clear about when to retrieve the digest. The + * bit SS_DATA_END is cleared when the engine has processed the data and + * when the digest is computed *but* it doesn't mean the digest is + * available in the digest registers. Hence the delay to be sure we can + * read it. + */ + ndelay(1); + + /* Get the hash from the device */ + if (op->mode == SS_OP_SHA1) { + for (i = 0; i < 5; i++) { + v = readl(ss->base + SS_MD0 + i * 4); + if (ss->variant->sha1_in_be) + put_unaligned_le32(v, areq->result + i * 4); + else + put_unaligned_be32(v, areq->result + i * 4); + } + } else { + for (i = 0; i < 4; i++) { + v = readl(ss->base + SS_MD0 + i * 4); + put_unaligned_le32(v, areq->result + i * 4); + } + } + +release_ss: + writel(0, ss->base + SS_CTL); + spin_unlock_bh(&ss->slock); + return err; +} + +int sun4i_hash_final(struct ahash_request *areq) +{ + struct sun4i_req_ctx *op = ahash_request_ctx(areq); + + op->flags = SS_HASH_FINAL; + return sun4i_hash(areq); +} + +int sun4i_hash_update(struct ahash_request *areq) +{ + struct sun4i_req_ctx *op = ahash_request_ctx(areq); + + op->flags = SS_HASH_UPDATE; + return sun4i_hash(areq); +} + +/* sun4i_hash_finup: finalize hashing operation after an update */ +int sun4i_hash_finup(struct ahash_request *areq) +{ + struct sun4i_req_ctx *op = ahash_request_ctx(areq); + + op->flags = SS_HASH_UPDATE | SS_HASH_FINAL; + return sun4i_hash(areq); +} + +/* combo of init/update/final functions */ +int sun4i_hash_digest(struct ahash_request *areq) +{ + int err; + struct sun4i_req_ctx *op = ahash_request_ctx(areq); + + err = sun4i_hash_init(areq); + if (err) + return err; + + op->flags = SS_HASH_UPDATE | SS_HASH_FINAL; + return sun4i_hash(areq); +} diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h new file mode 100644 index 000000000..f7d1c79ac --- /dev/null +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h @@ -0,0 +1,216 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * sun4i-ss.h - hardware cryptographic accelerator for Allwinner A20 SoC + * + * Copyright (C) 2013-2015 Corentin LABBE + * + * Support AES cipher with 128,192,256 bits keysize. + * Support MD5 and SHA1 hash algorithms. + * Support DES and 3DES + * + * You could find the datasheet in Documentation/arch/arm/sunxi.rst + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SS_CTL 0x00 +#define SS_KEY0 0x04 +#define SS_KEY1 0x08 +#define SS_KEY2 0x0C +#define SS_KEY3 0x10 +#define SS_KEY4 0x14 +#define SS_KEY5 0x18 +#define SS_KEY6 0x1C +#define SS_KEY7 0x20 + +#define SS_IV0 0x24 +#define SS_IV1 0x28 +#define SS_IV2 0x2C +#define SS_IV3 0x30 + +#define SS_FCSR 0x44 + +#define SS_MD0 0x4C +#define SS_MD1 0x50 +#define SS_MD2 0x54 +#define SS_MD3 0x58 +#define SS_MD4 0x5C + +#define SS_RXFIFO 0x200 +#define SS_TXFIFO 0x204 + +/* SS_CTL configuration values */ + +/* IV mode for hash */ +#define SS_IV_ARBITRARY (1 << 14) + +/* SS operation mode - bits 12-13 */ +#define SS_ECB (0 << 12) +#define SS_CBC (1 << 12) +#define SS_CTS (3 << 12) + +/* Counter width for CNT mode - bits 10-11 */ +#define SS_CNT_16BITS (0 << 10) +#define SS_CNT_32BITS (1 << 10) +#define SS_CNT_64BITS (2 << 10) + +/* Key size for AES - bits 8-9 */ +#define SS_AES_128BITS (0 << 8) +#define SS_AES_192BITS (1 << 8) +#define SS_AES_256BITS (2 << 8) + +/* Operation direction - bit 7 */ +#define SS_ENCRYPTION (0 << 7) +#define SS_DECRYPTION (1 << 7) + +/* SS Method - bits 4-6 */ +#define SS_OP_AES (0 << 4) +#define SS_OP_DES (1 << 4) +#define SS_OP_3DES (2 << 4) +#define SS_OP_SHA1 (3 << 4) +#define SS_OP_MD5 (4 << 4) + +/* Data end bit - bit 2 */ +#define SS_DATA_END (1 << 2) + +/* SS Enable bit - bit 0 */ +#define SS_DISABLED (0 << 0) +#define SS_ENABLED (1 << 0) + +/* SS_FCSR configuration values */ +/* RX FIFO status - bit 30 */ +#define SS_RXFIFO_FREE (1 << 30) + +/* RX FIFO empty spaces - bits 24-29 */ +#define SS_RXFIFO_SPACES(val) (((val) >> 24) & 0x3f) + +/* TX FIFO status - bit 22 */ +#define SS_TXFIFO_AVAILABLE (1 << 22) + +/* TX FIFO available spaces - bits 16-21 */ +#define SS_TXFIFO_SPACES(val) (((val) >> 16) & 0x3f) + +#define SS_RX_MAX 32 +#define SS_RX_DEFAULT SS_RX_MAX +#define SS_TX_MAX 33 + +#define SS_RXFIFO_EMP_INT_PENDING (1 << 10) +#define SS_TXFIFO_AVA_INT_PENDING (1 << 8) +#define SS_RXFIFO_EMP_INT_ENABLE (1 << 2) +#define SS_TXFIFO_AVA_INT_ENABLE (1 << 0) + +/* + * struct ss_variant - Describe SS hardware variant + * @sha1_in_be: The SHA1 digest is given by SS in BE, and so need to be inverted. + */ +struct ss_variant { + bool sha1_in_be; +}; + +struct sun4i_ss_ctx { + const struct ss_variant *variant; + void __iomem *base; + int irq; + struct clk *busclk; + struct clk *ssclk; + struct reset_control *reset; + struct device *dev; + struct resource *res; + char buf[4 * SS_RX_MAX];/* buffer for linearize SG src */ + char bufo[4 * SS_TX_MAX]; /* buffer for linearize SG dst */ + spinlock_t slock; /* control the use of the device */ + struct dentry *dbgfs_dir; + struct dentry *dbgfs_stats; +}; + +struct sun4i_ss_alg_template { + u32 type; + u32 mode; + union { + struct skcipher_alg crypto; + struct ahash_alg hash; + } alg; + struct sun4i_ss_ctx *ss; + unsigned long stat_req; + unsigned long stat_fb; + unsigned long stat_bytes; + unsigned long stat_opti; +}; + +struct sun4i_tfm_ctx { + u32 key[AES_MAX_KEY_SIZE / 4];/* divided by sizeof(u32) */ + u32 keylen; + u32 keymode; + struct sun4i_ss_ctx *ss; + struct crypto_skcipher *fallback_tfm; +}; + +struct sun4i_cipher_req_ctx { + u32 mode; + u8 backup_iv[AES_BLOCK_SIZE]; + struct skcipher_request fallback_req; // keep at the end +}; + +struct sun4i_req_ctx { + u32 mode; + u64 byte_count; /* number of bytes "uploaded" to the device */ + u32 hash[5]; /* for storing SS_IVx register */ + char buf[64]; + unsigned int len; + int flags; +}; + +int sun4i_hash_crainit(struct crypto_tfm *tfm); +void sun4i_hash_craexit(struct crypto_tfm *tfm); +int sun4i_hash_init(struct ahash_request *areq); +int sun4i_hash_update(struct ahash_request *areq); +int sun4i_hash_final(struct ahash_request *areq); +int sun4i_hash_finup(struct ahash_request *areq); +int sun4i_hash_digest(struct ahash_request *areq); +int sun4i_hash_export_md5(struct ahash_request *areq, void *out); +int sun4i_hash_import_md5(struct ahash_request *areq, const void *in); +int sun4i_hash_export_sha1(struct ahash_request *areq, void *out); +int sun4i_hash_import_sha1(struct ahash_request *areq, const void *in); + +int sun4i_ss_cbc_aes_encrypt(struct skcipher_request *areq); +int sun4i_ss_cbc_aes_decrypt(struct skcipher_request *areq); +int sun4i_ss_ecb_aes_encrypt(struct skcipher_request *areq); +int sun4i_ss_ecb_aes_decrypt(struct skcipher_request *areq); + +int sun4i_ss_cbc_des_encrypt(struct skcipher_request *areq); +int sun4i_ss_cbc_des_decrypt(struct skcipher_request *areq); +int sun4i_ss_ecb_des_encrypt(struct skcipher_request *areq); +int sun4i_ss_ecb_des_decrypt(struct skcipher_request *areq); + +int sun4i_ss_cbc_des3_encrypt(struct skcipher_request *areq); +int sun4i_ss_cbc_des3_decrypt(struct skcipher_request *areq); +int sun4i_ss_ecb_des3_encrypt(struct skcipher_request *areq); +int sun4i_ss_ecb_des3_decrypt(struct skcipher_request *areq); + +int sun4i_ss_cipher_init(struct crypto_tfm *tfm); +void sun4i_ss_cipher_exit(struct crypto_tfm *tfm); +int sun4i_ss_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen); +int sun4i_ss_des_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen); +int sun4i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen); diff --git a/drivers/crypto/allwinner/sun8i-ce/Makefile b/drivers/crypto/allwinner/sun8i-ce/Makefile new file mode 100644 index 000000000..ea708b427 --- /dev/null +++ b/drivers/crypto/allwinner/sun8i-ce/Makefile @@ -0,0 +1,4 @@ +obj-$(CONFIG_CRYPTO_DEV_SUN8I_CE) += sun8i-ce.o +sun8i-ce-y += sun8i-ce-core.o sun8i-ce-cipher.o +sun8i-ce-$(CONFIG_CRYPTO_DEV_SUN8I_CE_HASH) += sun8i-ce-hash.o +sun8i-ce-$(CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG) += sun8i-ce-trng.o diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c new file mode 100644 index 000000000..021614b65 --- /dev/null +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c @@ -0,0 +1,512 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sun8i-ce-cipher.c - hardware cryptographic offloader for + * Allwinner H3/A64/H5/H2+/H6/R40 SoC + * + * Copyright (C) 2016-2019 Corentin LABBE + * + * This file add support for AES cipher with 128,192,256 bits keysize in + * CBC and ECB mode. + * + * You could find a link for the datasheet in Documentation/arch/arm/sunxi.rst + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "sun8i-ce.h" + +static int sun8i_ce_cipher_need_fallback(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct scatterlist *sg; + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct sun8i_ce_alg_template *algt; + unsigned int todo, len; + + algt = container_of(alg, struct sun8i_ce_alg_template, alg.skcipher.base); + + if (sg_nents_for_len(areq->src, areq->cryptlen) > MAX_SG || + sg_nents_for_len(areq->dst, areq->cryptlen) > MAX_SG) { + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_maxsg++; + + return true; + } + + if (areq->cryptlen < crypto_skcipher_ivsize(tfm)) { + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_leniv++; + + return true; + } + + if (areq->cryptlen == 0) { + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_len0++; + + return true; + } + + if (areq->cryptlen % 16) { + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_mod16++; + + return true; + } + + len = areq->cryptlen; + sg = areq->src; + while (sg) { + if (!IS_ALIGNED(sg->offset, sizeof(u32))) { + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_srcali++; + + return true; + } + todo = min(len, sg->length); + if (todo % 4) { + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_srclen++; + + return true; + } + len -= todo; + sg = sg_next(sg); + } + + len = areq->cryptlen; + sg = areq->dst; + while (sg) { + if (!IS_ALIGNED(sg->offset, sizeof(u32))) { + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_dstali++; + + return true; + } + todo = min(len, sg->length); + if (todo % 4) { + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_dstlen++; + + return true; + } + len -= todo; + sg = sg_next(sg); + } + return false; +} + +static int sun8i_ce_cipher_fallback(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + int err; + + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) { + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct sun8i_ce_alg_template *algt; + + algt = container_of(alg, struct sun8i_ce_alg_template, + alg.skcipher.base); + + algt->stat_fb++; + } + + skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm); + skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags, + areq->base.complete, areq->base.data); + skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst, + areq->cryptlen, areq->iv); + if (rctx->op_dir & CE_DECRYPTION) + err = crypto_skcipher_decrypt(&rctx->fallback_req); + else + err = crypto_skcipher_encrypt(&rctx->fallback_req); + return err; +} + +static int sun8i_ce_cipher_prepare(struct skcipher_request *areq, + struct ce_task *cet) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun8i_ce_dev *ce = op->ce; + struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct sun8i_ce_alg_template *algt; + struct scatterlist *sg; + unsigned int todo, len, offset, ivsize; + u32 common, sym; + int i; + int nr_sgs = 0; + int nr_sgd = 0; + int err = 0; + int ns = sg_nents_for_len(areq->src, areq->cryptlen); + int nd = sg_nents_for_len(areq->dst, areq->cryptlen); + + algt = container_of(alg, struct sun8i_ce_alg_template, alg.skcipher.base); + + dev_dbg(ce->dev, "%s %s %u %x IV(%p %u) key=%u\n", __func__, + crypto_tfm_alg_name(areq->base.tfm), + areq->cryptlen, + rctx->op_dir, areq->iv, crypto_skcipher_ivsize(tfm), + op->keylen); + + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_req++; + + memset(cet, 0, sizeof(struct ce_task)); + + cet->t_id = cpu_to_le32(rctx->flow); + common = ce->variant->alg_cipher[algt->ce_algo_id]; + common |= rctx->op_dir | CE_COMM_INT; + cet->t_common_ctl = cpu_to_le32(common); + /* CTS and recent CE (H6) need length in bytes, in word otherwise */ + if (ce->variant->cipher_t_dlen_in_bytes) + cet->t_dlen = cpu_to_le32(areq->cryptlen); + else + cet->t_dlen = cpu_to_le32(areq->cryptlen / 4); + + sym = ce->variant->op_mode[algt->ce_blockmode]; + len = op->keylen; + switch (len) { + case 128 / 8: + sym |= CE_AES_128BITS; + break; + case 192 / 8: + sym |= CE_AES_192BITS; + break; + case 256 / 8: + sym |= CE_AES_256BITS; + break; + } + + cet->t_sym_ctl = cpu_to_le32(sym); + cet->t_asym_ctl = 0; + + rctx->addr_key = dma_map_single(ce->dev, op->key, op->keylen, DMA_TO_DEVICE); + if (dma_mapping_error(ce->dev, rctx->addr_key)) { + dev_err(ce->dev, "Cannot DMA MAP KEY\n"); + err = -EFAULT; + goto theend; + } + cet->t_key = desc_addr_val_le32(ce, rctx->addr_key); + + ivsize = crypto_skcipher_ivsize(tfm); + if (areq->iv && ivsize > 0) { + if (rctx->op_dir & CE_DECRYPTION) { + offset = areq->cryptlen - ivsize; + scatterwalk_map_and_copy(rctx->backup_iv, areq->src, + offset, ivsize, 0); + } + memcpy(rctx->bounce_iv, areq->iv, ivsize); + rctx->addr_iv = dma_map_single(ce->dev, rctx->bounce_iv, ivsize, + DMA_TO_DEVICE); + if (dma_mapping_error(ce->dev, rctx->addr_iv)) { + dev_err(ce->dev, "Cannot DMA MAP IV\n"); + err = -ENOMEM; + goto theend_iv; + } + cet->t_iv = desc_addr_val_le32(ce, rctx->addr_iv); + } + + if (areq->src == areq->dst) { + nr_sgs = dma_map_sg(ce->dev, areq->src, ns, DMA_BIDIRECTIONAL); + if (nr_sgs <= 0 || nr_sgs > MAX_SG) { + dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); + err = -EINVAL; + goto theend_iv; + } + nr_sgd = nr_sgs; + } else { + nr_sgs = dma_map_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE); + if (nr_sgs <= 0 || nr_sgs > MAX_SG) { + dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); + err = -EINVAL; + goto theend_iv; + } + nr_sgd = dma_map_sg(ce->dev, areq->dst, nd, DMA_FROM_DEVICE); + if (nr_sgd <= 0 || nr_sgd > MAX_SG) { + dev_err(ce->dev, "Invalid sg number %d\n", nr_sgd); + err = -EINVAL; + goto theend_sgs; + } + } + + len = areq->cryptlen; + for_each_sg(areq->src, sg, nr_sgs, i) { + cet->t_src[i].addr = desc_addr_val_le32(ce, sg_dma_address(sg)); + todo = min(len, sg_dma_len(sg)); + cet->t_src[i].len = cpu_to_le32(todo / 4); + dev_dbg(ce->dev, "%s total=%u SG(%d %u off=%d) todo=%u\n", __func__, + areq->cryptlen, i, cet->t_src[i].len, sg->offset, todo); + len -= todo; + } + if (len > 0) { + dev_err(ce->dev, "remaining len %d\n", len); + err = -EINVAL; + goto theend_sgs; + } + + len = areq->cryptlen; + for_each_sg(areq->dst, sg, nr_sgd, i) { + cet->t_dst[i].addr = desc_addr_val_le32(ce, sg_dma_address(sg)); + todo = min(len, sg_dma_len(sg)); + cet->t_dst[i].len = cpu_to_le32(todo / 4); + dev_dbg(ce->dev, "%s total=%u SG(%d %u off=%d) todo=%u\n", __func__, + areq->cryptlen, i, cet->t_dst[i].len, sg->offset, todo); + len -= todo; + } + if (len > 0) { + dev_err(ce->dev, "remaining len %d\n", len); + err = -EINVAL; + goto theend_sgs; + } + + rctx->nr_sgs = ns; + rctx->nr_sgd = nd; + return 0; + +theend_sgs: + if (areq->src == areq->dst) { + dma_unmap_sg(ce->dev, areq->src, ns, DMA_BIDIRECTIONAL); + } else { + if (nr_sgs > 0) + dma_unmap_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE); + + if (nr_sgd > 0) + dma_unmap_sg(ce->dev, areq->dst, nd, DMA_FROM_DEVICE); + } + +theend_iv: + if (areq->iv && ivsize > 0) { + if (!dma_mapping_error(ce->dev, rctx->addr_iv)) + dma_unmap_single(ce->dev, rctx->addr_iv, ivsize, + DMA_TO_DEVICE); + + offset = areq->cryptlen - ivsize; + if (rctx->op_dir & CE_DECRYPTION) { + memcpy(areq->iv, rctx->backup_iv, ivsize); + memzero_explicit(rctx->backup_iv, ivsize); + } else { + scatterwalk_map_and_copy(areq->iv, areq->dst, offset, + ivsize, 0); + } + memzero_explicit(rctx->bounce_iv, ivsize); + } + + dma_unmap_single(ce->dev, rctx->addr_key, op->keylen, DMA_TO_DEVICE); + +theend: + return err; +} + +static void sun8i_ce_cipher_unprepare(struct skcipher_request *areq, + struct ce_task *cet) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun8i_ce_dev *ce = op->ce; + struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + unsigned int ivsize, offset; + int nr_sgs = rctx->nr_sgs; + int nr_sgd = rctx->nr_sgd; + + ivsize = crypto_skcipher_ivsize(tfm); + + if (areq->src == areq->dst) { + dma_unmap_sg(ce->dev, areq->src, nr_sgs, DMA_BIDIRECTIONAL); + } else { + if (nr_sgs > 0) + dma_unmap_sg(ce->dev, areq->src, nr_sgs, DMA_TO_DEVICE); + dma_unmap_sg(ce->dev, areq->dst, nr_sgd, DMA_FROM_DEVICE); + } + + if (areq->iv && ivsize > 0) { + if (cet->t_iv) + dma_unmap_single(ce->dev, rctx->addr_iv, ivsize, + DMA_TO_DEVICE); + offset = areq->cryptlen - ivsize; + if (rctx->op_dir & CE_DECRYPTION) { + memcpy(areq->iv, rctx->backup_iv, ivsize); + memzero_explicit(rctx->backup_iv, ivsize); + } else { + scatterwalk_map_and_copy(areq->iv, areq->dst, offset, + ivsize, 0); + } + memzero_explicit(rctx->bounce_iv, ivsize); + } + + dma_unmap_single(ce->dev, rctx->addr_key, op->keylen, DMA_TO_DEVICE); +} + +int sun8i_ce_cipher_do_one(struct crypto_engine *engine, void *areq) +{ + struct skcipher_request *req = skcipher_request_cast(areq); + struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(req); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct sun8i_cipher_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + struct sun8i_ce_dev *ce = ctx->ce; + struct sun8i_ce_flow *chan; + int err; + + chan = &ce->chanlist[rctx->flow]; + + err = sun8i_ce_cipher_prepare(req, chan->tl); + if (err) + return err; + + err = sun8i_ce_run_task(ce, rctx->flow, + crypto_tfm_alg_name(req->base.tfm)); + + sun8i_ce_cipher_unprepare(req, chan->tl); + + local_bh_disable(); + crypto_finalize_skcipher_request(engine, req, err); + local_bh_enable(); + + return 0; +} + +int sun8i_ce_skdecrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + struct crypto_engine *engine; + int e; + + rctx->op_dir = CE_DECRYPTION; + if (sun8i_ce_cipher_need_fallback(areq)) + return sun8i_ce_cipher_fallback(areq); + + e = sun8i_ce_get_engine_number(op->ce); + rctx->flow = e; + engine = op->ce->chanlist[e].engine; + + return crypto_transfer_skcipher_request_to_engine(engine, areq); +} + +int sun8i_ce_skencrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + struct crypto_engine *engine; + int e; + + rctx->op_dir = CE_ENCRYPTION; + if (sun8i_ce_cipher_need_fallback(areq)) + return sun8i_ce_cipher_fallback(areq); + + e = sun8i_ce_get_engine_number(op->ce); + rctx->flow = e; + engine = op->ce->chanlist[e].engine; + + return crypto_transfer_skcipher_request_to_engine(engine, areq); +} + +int sun8i_ce_cipher_init(struct crypto_tfm *tfm) +{ + struct sun8i_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm); + struct sun8i_ce_alg_template *algt; + const char *name = crypto_tfm_alg_name(tfm); + struct crypto_skcipher *sktfm = __crypto_skcipher_cast(tfm); + struct skcipher_alg *alg = crypto_skcipher_alg(sktfm); + int err; + + memset(op, 0, sizeof(struct sun8i_cipher_tfm_ctx)); + + algt = container_of(alg, struct sun8i_ce_alg_template, alg.skcipher.base); + op->ce = algt->ce; + + op->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(op->fallback_tfm)) { + dev_err(op->ce->dev, "ERROR: Cannot allocate fallback for %s %ld\n", + name, PTR_ERR(op->fallback_tfm)); + return PTR_ERR(op->fallback_tfm); + } + + crypto_skcipher_set_reqsize(sktfm, sizeof(struct sun8i_cipher_req_ctx) + + crypto_skcipher_reqsize(op->fallback_tfm)); + + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + memcpy(algt->fbname, + crypto_skcipher_driver_name(op->fallback_tfm), + CRYPTO_MAX_ALG_NAME); + + err = pm_runtime_resume_and_get(op->ce->dev); + if (err < 0) + goto error_pm; + + return 0; +error_pm: + crypto_free_skcipher(op->fallback_tfm); + return err; +} + +void sun8i_ce_cipher_exit(struct crypto_tfm *tfm) +{ + struct sun8i_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm); + + kfree_sensitive(op->key); + crypto_free_skcipher(op->fallback_tfm); + pm_runtime_put_sync_suspend(op->ce->dev); +} + +int sun8i_ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun8i_ce_dev *ce = op->ce; + + switch (keylen) { + case 128 / 8: + break; + case 192 / 8: + break; + case 256 / 8: + break; + default: + dev_dbg(ce->dev, "ERROR: Invalid keylen %u\n", keylen); + return -EINVAL; + } + kfree_sensitive(op->key); + op->keylen = keylen; + op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); + if (!op->key) + return -ENOMEM; + + crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); + + return crypto_skcipher_setkey(op->fallback_tfm, key, keylen); +} + +int sun8i_ce_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + int err; + + err = verify_skcipher_des3_key(tfm, key); + if (err) + return err; + + kfree_sensitive(op->key); + op->keylen = keylen; + op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); + if (!op->key) + return -ENOMEM; + + crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); + + return crypto_skcipher_setkey(op->fallback_tfm, key, keylen); +} diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c new file mode 100644 index 000000000..21762bd40 --- /dev/null +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c @@ -0,0 +1,1075 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sun8i-ce-core.c - hardware cryptographic offloader for + * Allwinner H3/A64/H5/H2+/H6/R40 SoC + * + * Copyright (C) 2015-2019 Corentin Labbe + * + * Core file which registers crypto algorithms supported by the CryptoEngine. + * + * You could find a link for the datasheet in Documentation/arch/arm/sunxi.rst + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sun8i-ce.h" + +/* + * mod clock is lower on H3 than other SoC due to some DMA timeout occurring + * with high value. + * If you want to tune mod clock, loading driver and passing selftest is + * insufficient, you need to test with some LUKS test (mount and write to it) + */ +static const struct ce_variant ce_h3_variant = { + .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, + }, + .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, + CE_ALG_SHA384, CE_ALG_SHA512 + }, + .op_mode = { CE_OP_ECB, CE_OP_CBC + }, + .ce_clks = { + { "bus", 0, 200000000 }, + { "mod", 50000000, 0 }, + }, + .esr = ESR_H3, + .trng = CE_ID_NOTSUPP, +}; + +static const struct ce_variant ce_h5_variant = { + .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, + }, + .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, + CE_ID_NOTSUPP, CE_ID_NOTSUPP + }, + .op_mode = { CE_OP_ECB, CE_OP_CBC + }, + .ce_clks = { + { "bus", 0, 200000000 }, + { "mod", 300000000, 0 }, + }, + .esr = ESR_H5, + .trng = CE_ID_NOTSUPP, +}; + +static const struct ce_variant ce_h6_variant = { + .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, + }, + .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, + CE_ALG_SHA384, CE_ALG_SHA512 + }, + .op_mode = { CE_OP_ECB, CE_OP_CBC + }, + .cipher_t_dlen_in_bytes = true, + .hash_t_dlen_in_bits = true, + .trng_t_dlen_in_bytes = true, + .ce_clks = { + { "bus", 0, 200000000 }, + { "mod", 300000000, 0 }, + { "ram", 0, 400000000 }, + }, + .esr = ESR_H6, + .trng = CE_ALG_TRNG_V2, +}; + +static const struct ce_variant ce_h616_variant = { + .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, + }, + .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, + CE_ALG_SHA384, CE_ALG_SHA512 + }, + .op_mode = { CE_OP_ECB, CE_OP_CBC + }, + .cipher_t_dlen_in_bytes = true, + .hash_t_dlen_in_bits = true, + .trng_t_dlen_in_bytes = true, + .needs_word_addresses = true, + .ce_clks = { + { "bus", 0, 200000000 }, + { "mod", 300000000, 0 }, + { "ram", 0, 400000000 }, + { "trng", 0, 0 }, + }, + .esr = ESR_H6, + .trng = CE_ALG_TRNG_V2, +}; + +static const struct ce_variant ce_a64_variant = { + .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, + }, + .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, + CE_ID_NOTSUPP, CE_ID_NOTSUPP + }, + .op_mode = { CE_OP_ECB, CE_OP_CBC + }, + .ce_clks = { + { "bus", 0, 200000000 }, + { "mod", 300000000, 0 }, + }, + .esr = ESR_A64, + .trng = CE_ID_NOTSUPP, +}; + +static const struct ce_variant ce_d1_variant = { + .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, + }, + .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, + CE_ALG_SHA384, CE_ALG_SHA512 + }, + .op_mode = { CE_OP_ECB, CE_OP_CBC + }, + .ce_clks = { + { "bus", 0, 200000000 }, + { "mod", 300000000, 0 }, + { "ram", 0, 400000000 }, + { "trng", 0, 0 }, + }, + .esr = ESR_D1, + .trng = CE_ALG_TRNG, +}; + +static const struct ce_variant ce_r40_variant = { + .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, + }, + .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, + CE_ID_NOTSUPP, CE_ID_NOTSUPP + }, + .op_mode = { CE_OP_ECB, CE_OP_CBC + }, + .ce_clks = { + { "bus", 0, 200000000 }, + { "mod", 300000000, 0 }, + }, + .esr = ESR_R40, + .trng = CE_ID_NOTSUPP, +}; + +static void sun8i_ce_dump_task_descriptors(struct sun8i_ce_flow *chan) +{ + print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4, + chan->tl, sizeof(struct ce_task), false); +} + +/* + * sun8i_ce_get_engine_number() get the next channel slot + * This is a simple round-robin way of getting the next channel + * The flow 3 is reserve for xRNG operations + */ +int sun8i_ce_get_engine_number(struct sun8i_ce_dev *ce) +{ + return atomic_inc_return(&ce->flow) % (MAXFLOW - 1); +} + +int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name) +{ + u32 v; + int err = 0; + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + ce->chanlist[flow].stat_req++; +#endif + + mutex_lock(&ce->mlock); + + v = readl(ce->base + CE_ICR); + v |= 1 << flow; + writel(v, ce->base + CE_ICR); + + reinit_completion(&ce->chanlist[flow].complete); + writel(desc_addr_val(ce, ce->chanlist[flow].t_phy), ce->base + CE_TDQ); + + ce->chanlist[flow].status = 0; + /* Be sure all data is written before enabling the task */ + wmb(); + + /* Only H6 needs to write a part of t_common_ctl along with "1", but since it is ignored + * on older SoCs, we have no reason to complicate things. + */ + v = 1 | ((le32_to_cpu(ce->chanlist[flow].tl->t_common_ctl) & 0x7F) << 8); + writel(v, ce->base + CE_TLR); + mutex_unlock(&ce->mlock); + + wait_for_completion_interruptible_timeout(&ce->chanlist[flow].complete, + msecs_to_jiffies(CE_DMA_TIMEOUT_MS)); + + if (ce->chanlist[flow].status == 0) { + dev_err(ce->dev, "DMA timeout for %s on flow %d\n", name, flow); + err = -EFAULT; + } + /* No need to lock for this read, the channel is locked so + * nothing could modify the error value for this channel + */ + v = readl(ce->base + CE_ESR); + switch (ce->variant->esr) { + case ESR_H3: + /* Sadly, the error bit is not per flow */ + if (v) { + dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow); + sun8i_ce_dump_task_descriptors(&ce->chanlist[flow]); + err = -EFAULT; + } + if (v & CE_ERR_ALGO_NOTSUP) + dev_err(ce->dev, "CE ERROR: algorithm not supported\n"); + if (v & CE_ERR_DATALEN) + dev_err(ce->dev, "CE ERROR: data length error\n"); + if (v & CE_ERR_KEYSRAM) + dev_err(ce->dev, "CE ERROR: keysram access error for AES\n"); + break; + case ESR_A64: + case ESR_D1: + case ESR_H5: + case ESR_R40: + v >>= (flow * 4); + v &= 0xF; + if (v) { + dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow); + sun8i_ce_dump_task_descriptors(&ce->chanlist[flow]); + err = -EFAULT; + } + if (v & CE_ERR_ALGO_NOTSUP) + dev_err(ce->dev, "CE ERROR: algorithm not supported\n"); + if (v & CE_ERR_DATALEN) + dev_err(ce->dev, "CE ERROR: data length error\n"); + if (v & CE_ERR_KEYSRAM) + dev_err(ce->dev, "CE ERROR: keysram access error for AES\n"); + break; + case ESR_H6: + v >>= (flow * 8); + v &= 0xFF; + if (v) { + dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow); + sun8i_ce_dump_task_descriptors(&ce->chanlist[flow]); + err = -EFAULT; + } + if (v & CE_ERR_ALGO_NOTSUP) + dev_err(ce->dev, "CE ERROR: algorithm not supported\n"); + if (v & CE_ERR_DATALEN) + dev_err(ce->dev, "CE ERROR: data length error\n"); + if (v & CE_ERR_KEYSRAM) + dev_err(ce->dev, "CE ERROR: keysram access error for AES\n"); + if (v & CE_ERR_ADDR_INVALID) + dev_err(ce->dev, "CE ERROR: address invalid\n"); + if (v & CE_ERR_KEYLADDER) + dev_err(ce->dev, "CE ERROR: key ladder configuration error\n"); + break; + } + + return err; +} + +static irqreturn_t ce_irq_handler(int irq, void *data) +{ + struct sun8i_ce_dev *ce = (struct sun8i_ce_dev *)data; + int flow = 0; + u32 p; + + p = readl(ce->base + CE_ISR); + for (flow = 0; flow < MAXFLOW; flow++) { + if (p & (BIT(flow))) { + writel(BIT(flow), ce->base + CE_ISR); + ce->chanlist[flow].status = 1; + complete(&ce->chanlist[flow].complete); + } + } + + return IRQ_HANDLED; +} + +static struct sun8i_ce_alg_template ce_algs[] = { +{ + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .ce_algo_id = CE_ID_CIPHER_AES, + .ce_blockmode = CE_ID_OP_CBC, + .alg.skcipher.base = { + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-sun8i-ce", + .cra_priority = 400, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct sun8i_cipher_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 0xf, + .cra_init = sun8i_ce_cipher_init, + .cra_exit = sun8i_ce_cipher_exit, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = sun8i_ce_aes_setkey, + .encrypt = sun8i_ce_skencrypt, + .decrypt = sun8i_ce_skdecrypt, + }, + .alg.skcipher.op = { + .do_one_request = sun8i_ce_cipher_do_one, + }, +}, +{ + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .ce_algo_id = CE_ID_CIPHER_AES, + .ce_blockmode = CE_ID_OP_ECB, + .alg.skcipher.base = { + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-sun8i-ce", + .cra_priority = 400, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct sun8i_cipher_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 0xf, + .cra_init = sun8i_ce_cipher_init, + .cra_exit = sun8i_ce_cipher_exit, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = sun8i_ce_aes_setkey, + .encrypt = sun8i_ce_skencrypt, + .decrypt = sun8i_ce_skdecrypt, + }, + .alg.skcipher.op = { + .do_one_request = sun8i_ce_cipher_do_one, + }, +}, +{ + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .ce_algo_id = CE_ID_CIPHER_DES3, + .ce_blockmode = CE_ID_OP_CBC, + .alg.skcipher.base = { + .base = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "cbc-des3-sun8i-ce", + .cra_priority = 400, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct sun8i_cipher_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 0xf, + .cra_init = sun8i_ce_cipher_init, + .cra_exit = sun8i_ce_cipher_exit, + }, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setkey = sun8i_ce_des3_setkey, + .encrypt = sun8i_ce_skencrypt, + .decrypt = sun8i_ce_skdecrypt, + }, + .alg.skcipher.op = { + .do_one_request = sun8i_ce_cipher_do_one, + }, +}, +{ + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .ce_algo_id = CE_ID_CIPHER_DES3, + .ce_blockmode = CE_ID_OP_ECB, + .alg.skcipher.base = { + .base = { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "ecb-des3-sun8i-ce", + .cra_priority = 400, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct sun8i_cipher_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 0xf, + .cra_init = sun8i_ce_cipher_init, + .cra_exit = sun8i_ce_cipher_exit, + }, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = sun8i_ce_des3_setkey, + .encrypt = sun8i_ce_skencrypt, + .decrypt = sun8i_ce_skdecrypt, + }, + .alg.skcipher.op = { + .do_one_request = sun8i_ce_cipher_do_one, + }, +}, +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_HASH +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ce_algo_id = CE_ID_HASH_MD5, + .alg.hash.base = { + .init = sun8i_ce_hash_init, + .update = sun8i_ce_hash_update, + .final = sun8i_ce_hash_final, + .finup = sun8i_ce_hash_finup, + .digest = sun8i_ce_hash_digest, + .export = sun8i_ce_hash_export, + .import = sun8i_ce_hash_import, + .init_tfm = sun8i_ce_hash_init_tfm, + .exit_tfm = sun8i_ce_hash_exit_tfm, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct md5_state), + .base = { + .cra_name = "md5", + .cra_driver_name = "md5-sun8i-ce", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), + .cra_module = THIS_MODULE, + } + } + }, + .alg.hash.op = { + .do_one_request = sun8i_ce_hash_run, + }, + +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ce_algo_id = CE_ID_HASH_SHA1, + .alg.hash.base = { + .init = sun8i_ce_hash_init, + .update = sun8i_ce_hash_update, + .final = sun8i_ce_hash_final, + .finup = sun8i_ce_hash_finup, + .digest = sun8i_ce_hash_digest, + .export = sun8i_ce_hash_export, + .import = sun8i_ce_hash_import, + .init_tfm = sun8i_ce_hash_init_tfm, + .exit_tfm = sun8i_ce_hash_exit_tfm, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct sha1_state), + .base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-sun8i-ce", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), + .cra_module = THIS_MODULE, + } + } + }, + .alg.hash.op = { + .do_one_request = sun8i_ce_hash_run, + }, +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ce_algo_id = CE_ID_HASH_SHA224, + .alg.hash.base = { + .init = sun8i_ce_hash_init, + .update = sun8i_ce_hash_update, + .final = sun8i_ce_hash_final, + .finup = sun8i_ce_hash_finup, + .digest = sun8i_ce_hash_digest, + .export = sun8i_ce_hash_export, + .import = sun8i_ce_hash_import, + .init_tfm = sun8i_ce_hash_init_tfm, + .exit_tfm = sun8i_ce_hash_exit_tfm, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha224", + .cra_driver_name = "sha224-sun8i-ce", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), + .cra_module = THIS_MODULE, + } + } + }, + .alg.hash.op = { + .do_one_request = sun8i_ce_hash_run, + }, +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ce_algo_id = CE_ID_HASH_SHA256, + .alg.hash.base = { + .init = sun8i_ce_hash_init, + .update = sun8i_ce_hash_update, + .final = sun8i_ce_hash_final, + .finup = sun8i_ce_hash_finup, + .digest = sun8i_ce_hash_digest, + .export = sun8i_ce_hash_export, + .import = sun8i_ce_hash_import, + .init_tfm = sun8i_ce_hash_init_tfm, + .exit_tfm = sun8i_ce_hash_exit_tfm, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-sun8i-ce", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), + .cra_module = THIS_MODULE, + } + } + }, + .alg.hash.op = { + .do_one_request = sun8i_ce_hash_run, + }, +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ce_algo_id = CE_ID_HASH_SHA384, + .alg.hash.base = { + .init = sun8i_ce_hash_init, + .update = sun8i_ce_hash_update, + .final = sun8i_ce_hash_final, + .finup = sun8i_ce_hash_finup, + .digest = sun8i_ce_hash_digest, + .export = sun8i_ce_hash_export, + .import = sun8i_ce_hash_import, + .init_tfm = sun8i_ce_hash_init_tfm, + .exit_tfm = sun8i_ce_hash_exit_tfm, + .halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct sha512_state), + .base = { + .cra_name = "sha384", + .cra_driver_name = "sha384-sun8i-ce", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), + .cra_module = THIS_MODULE, + } + } + }, + .alg.hash.op = { + .do_one_request = sun8i_ce_hash_run, + }, +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ce_algo_id = CE_ID_HASH_SHA512, + .alg.hash.base = { + .init = sun8i_ce_hash_init, + .update = sun8i_ce_hash_update, + .final = sun8i_ce_hash_final, + .finup = sun8i_ce_hash_finup, + .digest = sun8i_ce_hash_digest, + .export = sun8i_ce_hash_export, + .import = sun8i_ce_hash_import, + .init_tfm = sun8i_ce_hash_init_tfm, + .exit_tfm = sun8i_ce_hash_exit_tfm, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct sha512_state), + .base = { + .cra_name = "sha512", + .cra_driver_name = "sha512-sun8i-ce", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), + .cra_module = THIS_MODULE, + } + } + }, + .alg.hash.op = { + .do_one_request = sun8i_ce_hash_run, + }, +}, +#endif +}; + +static int sun8i_ce_debugfs_show(struct seq_file *seq, void *v) +{ + struct sun8i_ce_dev *ce __maybe_unused = seq->private; + unsigned int i; + + for (i = 0; i < MAXFLOW; i++) + seq_printf(seq, "Channel %d: nreq %lu\n", i, +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + ce->chanlist[i].stat_req); +#else + 0ul); +#endif + + for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { + if (!ce_algs[i].ce) + continue; + switch (ce_algs[i].type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n", + ce_algs[i].alg.skcipher.base.base.cra_driver_name, + ce_algs[i].alg.skcipher.base.base.cra_name, + ce_algs[i].stat_req, ce_algs[i].stat_fb); + seq_printf(seq, "\tLast fallback is: %s\n", + ce_algs[i].fbname); + seq_printf(seq, "\tFallback due to 0 length: %lu\n", + ce_algs[i].stat_fb_len0); + seq_printf(seq, "\tFallback due to length !mod16: %lu\n", + ce_algs[i].stat_fb_mod16); + seq_printf(seq, "\tFallback due to length < IV: %lu\n", + ce_algs[i].stat_fb_leniv); + seq_printf(seq, "\tFallback due to source alignment: %lu\n", + ce_algs[i].stat_fb_srcali); + seq_printf(seq, "\tFallback due to dest alignment: %lu\n", + ce_algs[i].stat_fb_dstali); + seq_printf(seq, "\tFallback due to source length: %lu\n", + ce_algs[i].stat_fb_srclen); + seq_printf(seq, "\tFallback due to dest length: %lu\n", + ce_algs[i].stat_fb_dstlen); + seq_printf(seq, "\tFallback due to SG numbers: %lu\n", + ce_algs[i].stat_fb_maxsg); + break; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_HASH + case CRYPTO_ALG_TYPE_AHASH: + seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n", + ce_algs[i].alg.hash.base.halg.base.cra_driver_name, + ce_algs[i].alg.hash.base.halg.base.cra_name, + ce_algs[i].stat_req, ce_algs[i].stat_fb); + seq_printf(seq, "\tLast fallback is: %s\n", + ce_algs[i].fbname); + seq_printf(seq, "\tFallback due to 0 length: %lu\n", + ce_algs[i].stat_fb_len0); + seq_printf(seq, "\tFallback due to length: %lu\n", + ce_algs[i].stat_fb_srclen); + seq_printf(seq, "\tFallback due to alignment: %lu\n", + ce_algs[i].stat_fb_srcali); + seq_printf(seq, "\tFallback due to SG numbers: %lu\n", + ce_algs[i].stat_fb_maxsg); + break; +#endif + } + } +#if defined(CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG) && \ + defined(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG) + seq_printf(seq, "HWRNG %lu %lu\n", + ce->hwrng_stat_req, ce->hwrng_stat_bytes); +#endif + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(sun8i_ce_debugfs); + +static void sun8i_ce_free_chanlist(struct sun8i_ce_dev *ce, int i) +{ + while (i >= 0) { + crypto_engine_exit(ce->chanlist[i].engine); + if (ce->chanlist[i].tl) + dma_free_coherent(ce->dev, sizeof(struct ce_task), + ce->chanlist[i].tl, + ce->chanlist[i].t_phy); + i--; + } +} + +/* + * Allocate the channel list structure + */ +static int sun8i_ce_allocate_chanlist(struct sun8i_ce_dev *ce) +{ + int i, err; + + ce->chanlist = devm_kcalloc(ce->dev, MAXFLOW, + sizeof(struct sun8i_ce_flow), GFP_KERNEL); + if (!ce->chanlist) + return -ENOMEM; + + for (i = 0; i < MAXFLOW; i++) { + init_completion(&ce->chanlist[i].complete); + + ce->chanlist[i].engine = crypto_engine_alloc_init(ce->dev, true); + if (!ce->chanlist[i].engine) { + dev_err(ce->dev, "Cannot allocate engine\n"); + i--; + err = -ENOMEM; + goto error_engine; + } + err = crypto_engine_start(ce->chanlist[i].engine); + if (err) { + dev_err(ce->dev, "Cannot start engine\n"); + goto error_engine; + } + ce->chanlist[i].tl = dma_alloc_coherent(ce->dev, + sizeof(struct ce_task), + &ce->chanlist[i].t_phy, + GFP_KERNEL); + if (!ce->chanlist[i].tl) { + dev_err(ce->dev, "Cannot get DMA memory for task %d\n", + i); + err = -ENOMEM; + goto error_engine; + } + } + return 0; +error_engine: + sun8i_ce_free_chanlist(ce, i); + return err; +} + +/* + * Power management strategy: The device is suspended unless a TFM exists for + * one of the algorithms proposed by this driver. + */ +static int sun8i_ce_pm_suspend(struct device *dev) +{ + struct sun8i_ce_dev *ce = dev_get_drvdata(dev); + int i; + + reset_control_assert(ce->reset); + for (i = 0; i < CE_MAX_CLOCKS; i++) + clk_disable_unprepare(ce->ceclks[i]); + return 0; +} + +static int sun8i_ce_pm_resume(struct device *dev) +{ + struct sun8i_ce_dev *ce = dev_get_drvdata(dev); + int err, i; + + for (i = 0; i < CE_MAX_CLOCKS; i++) { + if (!ce->variant->ce_clks[i].name) + continue; + err = clk_prepare_enable(ce->ceclks[i]); + if (err) { + dev_err(ce->dev, "Cannot prepare_enable %s\n", + ce->variant->ce_clks[i].name); + goto error; + } + } + err = reset_control_deassert(ce->reset); + if (err) { + dev_err(ce->dev, "Cannot deassert reset control\n"); + goto error; + } + return 0; +error: + sun8i_ce_pm_suspend(dev); + return err; +} + +static const struct dev_pm_ops sun8i_ce_pm_ops = { + SET_RUNTIME_PM_OPS(sun8i_ce_pm_suspend, sun8i_ce_pm_resume, NULL) +}; + +static int sun8i_ce_pm_init(struct sun8i_ce_dev *ce) +{ + int err; + + pm_runtime_use_autosuspend(ce->dev); + pm_runtime_set_autosuspend_delay(ce->dev, 2000); + + err = pm_runtime_set_suspended(ce->dev); + if (err) + return err; + + err = devm_pm_runtime_enable(ce->dev); + if (err) + return err; + + return 0; +} + +static int sun8i_ce_get_clks(struct sun8i_ce_dev *ce) +{ + unsigned long cr; + int err, i; + + for (i = 0; i < CE_MAX_CLOCKS; i++) { + if (!ce->variant->ce_clks[i].name) + continue; + ce->ceclks[i] = devm_clk_get(ce->dev, ce->variant->ce_clks[i].name); + if (IS_ERR(ce->ceclks[i])) { + err = PTR_ERR(ce->ceclks[i]); + dev_err(ce->dev, "Cannot get %s CE clock err=%d\n", + ce->variant->ce_clks[i].name, err); + return err; + } + cr = clk_get_rate(ce->ceclks[i]); + if (!cr) + return -EINVAL; + if (ce->variant->ce_clks[i].freq > 0 && + cr != ce->variant->ce_clks[i].freq) { + dev_info(ce->dev, "Set %s clock to %lu (%lu Mhz) from %lu (%lu Mhz)\n", + ce->variant->ce_clks[i].name, + ce->variant->ce_clks[i].freq, + ce->variant->ce_clks[i].freq / 1000000, + cr, cr / 1000000); + err = clk_set_rate(ce->ceclks[i], ce->variant->ce_clks[i].freq); + if (err) + dev_err(ce->dev, "Fail to set %s clk speed to %lu hz\n", + ce->variant->ce_clks[i].name, + ce->variant->ce_clks[i].freq); + } + if (ce->variant->ce_clks[i].max_freq > 0 && + cr > ce->variant->ce_clks[i].max_freq) + dev_warn(ce->dev, "Frequency for %s (%lu hz) is higher than datasheet's recommendation (%lu hz)", + ce->variant->ce_clks[i].name, cr, + ce->variant->ce_clks[i].max_freq); + } + return 0; +} + +static int sun8i_ce_register_algs(struct sun8i_ce_dev *ce) +{ + int ce_method, err, id; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { + ce_algs[i].ce = ce; + switch (ce_algs[i].type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + id = ce_algs[i].ce_algo_id; + ce_method = ce->variant->alg_cipher[id]; + if (ce_method == CE_ID_NOTSUPP) { + dev_dbg(ce->dev, + "DEBUG: Algo of %s not supported\n", + ce_algs[i].alg.skcipher.base.base.cra_name); + ce_algs[i].ce = NULL; + break; + } + id = ce_algs[i].ce_blockmode; + ce_method = ce->variant->op_mode[id]; + if (ce_method == CE_ID_NOTSUPP) { + dev_dbg(ce->dev, "DEBUG: Blockmode of %s not supported\n", + ce_algs[i].alg.skcipher.base.base.cra_name); + ce_algs[i].ce = NULL; + break; + } + dev_info(ce->dev, "Register %s\n", + ce_algs[i].alg.skcipher.base.base.cra_name); + err = crypto_engine_register_skcipher(&ce_algs[i].alg.skcipher); + if (err) { + dev_err(ce->dev, "ERROR: Fail to register %s\n", + ce_algs[i].alg.skcipher.base.base.cra_name); + ce_algs[i].ce = NULL; + return err; + } + break; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_HASH + case CRYPTO_ALG_TYPE_AHASH: + id = ce_algs[i].ce_algo_id; + ce_method = ce->variant->alg_hash[id]; + if (ce_method == CE_ID_NOTSUPP) { + dev_info(ce->dev, + "DEBUG: Algo of %s not supported\n", + ce_algs[i].alg.hash.base.halg.base.cra_name); + ce_algs[i].ce = NULL; + break; + } + dev_info(ce->dev, "Register %s\n", + ce_algs[i].alg.hash.base.halg.base.cra_name); + err = crypto_engine_register_ahash(&ce_algs[i].alg.hash); + if (err) { + dev_err(ce->dev, "ERROR: Fail to register %s\n", + ce_algs[i].alg.hash.base.halg.base.cra_name); + ce_algs[i].ce = NULL; + return err; + } + break; +#endif + default: + ce_algs[i].ce = NULL; + dev_err(ce->dev, "ERROR: tried to register an unknown algo\n"); + } + } + return 0; +} + +static void sun8i_ce_unregister_algs(struct sun8i_ce_dev *ce) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { + if (!ce_algs[i].ce) + continue; + switch (ce_algs[i].type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + dev_info(ce->dev, "Unregister %d %s\n", i, + ce_algs[i].alg.skcipher.base.base.cra_name); + crypto_engine_unregister_skcipher(&ce_algs[i].alg.skcipher); + break; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_HASH + case CRYPTO_ALG_TYPE_AHASH: + dev_info(ce->dev, "Unregister %d %s\n", i, + ce_algs[i].alg.hash.base.halg.base.cra_name); + crypto_engine_unregister_ahash(&ce_algs[i].alg.hash); + break; +#endif + } + } +} + +static int sun8i_ce_probe(struct platform_device *pdev) +{ + struct sun8i_ce_dev *ce; + int err, irq; + u32 v; + + ce = devm_kzalloc(&pdev->dev, sizeof(*ce), GFP_KERNEL); + if (!ce) + return -ENOMEM; + + ce->dev = &pdev->dev; + platform_set_drvdata(pdev, ce); + + ce->variant = of_device_get_match_data(&pdev->dev); + if (!ce->variant) { + dev_err(&pdev->dev, "Missing Crypto Engine variant\n"); + return -EINVAL; + } + + ce->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(ce->base)) + return PTR_ERR(ce->base); + + err = sun8i_ce_get_clks(ce); + if (err) + return err; + + /* Get Non Secure IRQ */ + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + ce->reset = devm_reset_control_get(&pdev->dev, NULL); + if (IS_ERR(ce->reset)) + return dev_err_probe(&pdev->dev, PTR_ERR(ce->reset), + "No reset control found\n"); + + mutex_init(&ce->mlock); + mutex_init(&ce->rnglock); + + err = sun8i_ce_allocate_chanlist(ce); + if (err) + return err; + + err = sun8i_ce_pm_init(ce); + if (err) + goto error_pm; + + err = devm_request_irq(&pdev->dev, irq, ce_irq_handler, 0, + "sun8i-ce-ns", ce); + if (err) + goto error_pm; + + err = sun8i_ce_register_algs(ce); + if (err) + goto error_alg; + + err = pm_runtime_resume_and_get(ce->dev); + if (err < 0) + goto error_alg; + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG + sun8i_ce_hwrng_register(ce); +#endif + + v = readl(ce->base + CE_CTR); + v >>= CE_DIE_ID_SHIFT; + v &= CE_DIE_ID_MASK; + dev_info(&pdev->dev, "CryptoEngine Die ID %x\n", v); + + pm_runtime_put_sync(ce->dev); + + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) { + struct dentry *dbgfs_dir; + struct dentry *dbgfs_stats __maybe_unused; + + /* Ignore error of debugfs */ + dbgfs_dir = debugfs_create_dir("sun8i-ce", NULL); + dbgfs_stats = debugfs_create_file("stats", 0444, + dbgfs_dir, ce, + &sun8i_ce_debugfs_fops); + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + ce->dbgfs_dir = dbgfs_dir; + ce->dbgfs_stats = dbgfs_stats; +#endif + } + + return 0; +error_alg: + sun8i_ce_unregister_algs(ce); +error_pm: + sun8i_ce_free_chanlist(ce, MAXFLOW - 1); + return err; +} + +static void sun8i_ce_remove(struct platform_device *pdev) +{ + struct sun8i_ce_dev *ce = platform_get_drvdata(pdev); + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG + sun8i_ce_hwrng_unregister(ce); +#endif + + sun8i_ce_unregister_algs(ce); + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + debugfs_remove_recursive(ce->dbgfs_dir); +#endif + + sun8i_ce_free_chanlist(ce, MAXFLOW - 1); +} + +static const struct of_device_id sun8i_ce_crypto_of_match_table[] = { + { .compatible = "allwinner,sun8i-h3-crypto", + .data = &ce_h3_variant }, + { .compatible = "allwinner,sun8i-r40-crypto", + .data = &ce_r40_variant }, + { .compatible = "allwinner,sun20i-d1-crypto", + .data = &ce_d1_variant }, + { .compatible = "allwinner,sun50i-a64-crypto", + .data = &ce_a64_variant }, + { .compatible = "allwinner,sun50i-h5-crypto", + .data = &ce_h5_variant }, + { .compatible = "allwinner,sun50i-h6-crypto", + .data = &ce_h6_variant }, + { .compatible = "allwinner,sun50i-h616-crypto", + .data = &ce_h616_variant }, + {} +}; +MODULE_DEVICE_TABLE(of, sun8i_ce_crypto_of_match_table); + +static struct platform_driver sun8i_ce_driver = { + .probe = sun8i_ce_probe, + .remove = sun8i_ce_remove, + .driver = { + .name = "sun8i-ce", + .pm = &sun8i_ce_pm_ops, + .of_match_table = sun8i_ce_crypto_of_match_table, + }, +}; + +module_platform_driver(sun8i_ce_driver); + +MODULE_DESCRIPTION("Allwinner Crypto Engine cryptographic offloader"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Corentin Labbe "); diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c new file mode 100644 index 000000000..d01594353 --- /dev/null +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c @@ -0,0 +1,486 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sun8i-ce-hash.c - hardware cryptographic offloader for + * Allwinner H3/A64/H5/H2+/H6/R40 SoC + * + * Copyright (C) 2015-2020 Corentin Labbe + * + * This file add support for MD5 and SHA1/SHA224/SHA256/SHA384/SHA512. + * + * You could find the datasheet in Documentation/arch/arm/sunxi.rst + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "sun8i-ce.h" + +static void sun8i_ce_hash_stat_fb_inc(struct crypto_ahash *tfm) +{ + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) { + struct sun8i_ce_alg_template *algt; + struct ahash_alg *alg = crypto_ahash_alg(tfm); + + algt = container_of(alg, struct sun8i_ce_alg_template, + alg.hash.base); + algt->stat_fb++; + } +} + +int sun8i_ce_hash_init_tfm(struct crypto_ahash *tfm) +{ + struct sun8i_ce_hash_tfm_ctx *op = crypto_ahash_ctx(tfm); + struct ahash_alg *alg = crypto_ahash_alg(tfm); + struct sun8i_ce_alg_template *algt; + int err; + + algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash.base); + op->ce = algt->ce; + + /* FALLBACK */ + op->fallback_tfm = crypto_alloc_ahash(crypto_ahash_alg_name(tfm), 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(op->fallback_tfm)) { + dev_err(algt->ce->dev, "Fallback driver could no be loaded\n"); + return PTR_ERR(op->fallback_tfm); + } + + crypto_ahash_set_statesize(tfm, + crypto_ahash_statesize(op->fallback_tfm)); + + crypto_ahash_set_reqsize(tfm, + sizeof(struct sun8i_ce_hash_reqctx) + + crypto_ahash_reqsize(op->fallback_tfm) + + CRYPTO_DMA_PADDING); + + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + memcpy(algt->fbname, + crypto_ahash_driver_name(op->fallback_tfm), + CRYPTO_MAX_ALG_NAME); + + err = pm_runtime_resume_and_get(op->ce->dev); + if (err < 0) + goto error_pm; + return 0; +error_pm: + crypto_free_ahash(op->fallback_tfm); + return err; +} + +void sun8i_ce_hash_exit_tfm(struct crypto_ahash *tfm) +{ + struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + crypto_free_ahash(tfmctx->fallback_tfm); + pm_runtime_put_sync_suspend(tfmctx->ce->dev); +} + +int sun8i_ce_hash_init(struct ahash_request *areq) +{ + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + memset(rctx, 0, sizeof(struct sun8i_ce_hash_reqctx)); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + + return crypto_ahash_init(&rctx->fallback_req); +} + +int sun8i_ce_hash_export(struct ahash_request *areq, void *out) +{ + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + + return crypto_ahash_export(&rctx->fallback_req, out); +} + +int sun8i_ce_hash_import(struct ahash_request *areq, const void *in) +{ + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + + return crypto_ahash_import(&rctx->fallback_req, in); +} + +int sun8i_ce_hash_final(struct ahash_request *areq) +{ + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + sun8i_ce_hash_stat_fb_inc(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, NULL, areq->result, 0); + + return crypto_ahash_final(&rctx->fallback_req); +} + +int sun8i_ce_hash_update(struct ahash_request *areq) +{ + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, NULL, areq->nbytes); + + return crypto_ahash_update(&rctx->fallback_req); +} + +int sun8i_ce_hash_finup(struct ahash_request *areq) +{ + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + sun8i_ce_hash_stat_fb_inc(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, areq->result, + areq->nbytes); + + return crypto_ahash_finup(&rctx->fallback_req); +} + +static int sun8i_ce_hash_digest_fb(struct ahash_request *areq) +{ + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + sun8i_ce_hash_stat_fb_inc(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, areq->result, + areq->nbytes); + + return crypto_ahash_digest(&rctx->fallback_req); +} + +static bool sun8i_ce_hash_need_fallback(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun8i_ce_alg_template *algt; + struct scatterlist *sg; + + algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash.base); + + if (areq->nbytes == 0) { + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_len0++; + + return true; + } + /* we need to reserve one SG for padding one */ + if (sg_nents_for_len(areq->src, areq->nbytes) > MAX_SG - 1) { + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_maxsg++; + + return true; + } + sg = areq->src; + while (sg) { + if (sg->length % 4) { + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_srclen++; + + return true; + } + if (!IS_ALIGNED(sg->offset, sizeof(u32))) { + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_fb_srcali++; + + return true; + } + sg = sg_next(sg); + } + return false; +} + +int sun8i_ce_hash_digest(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *ctx = crypto_ahash_ctx(tfm); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); + struct sun8i_ce_dev *ce = ctx->ce; + struct crypto_engine *engine; + int e; + + if (sun8i_ce_hash_need_fallback(areq)) + return sun8i_ce_hash_digest_fb(areq); + + e = sun8i_ce_get_engine_number(ce); + rctx->flow = e; + engine = ce->chanlist[e].engine; + + return crypto_transfer_hash_request_to_engine(engine, areq); +} + +static u64 hash_pad(__le32 *buf, unsigned int bufsize, u64 padi, u64 byte_count, bool le, int bs) +{ + u64 fill, min_fill, j, k; + __be64 *bebits; + __le64 *lebits; + + j = padi; + buf[j++] = cpu_to_le32(0x80); + + if (bs == 64) { + fill = 64 - (byte_count % 64); + min_fill = 2 * sizeof(u32) + sizeof(u32); + } else { + fill = 128 - (byte_count % 128); + min_fill = 4 * sizeof(u32) + sizeof(u32); + } + + if (fill < min_fill) + fill += bs; + + k = j; + j += (fill - min_fill) / sizeof(u32); + if (j * 4 > bufsize) { + pr_err("%s OVERFLOW %llu\n", __func__, j); + return 0; + } + for (; k < j; k++) + buf[k] = 0; + + if (le) { + /* MD5 */ + lebits = (__le64 *)&buf[j]; + *lebits = cpu_to_le64(byte_count << 3); + j += 2; + } else { + if (bs == 64) { + /* sha1 sha224 sha256 */ + bebits = (__be64 *)&buf[j]; + *bebits = cpu_to_be64(byte_count << 3); + j += 2; + } else { + /* sha384 sha512*/ + bebits = (__be64 *)&buf[j]; + *bebits = cpu_to_be64(byte_count >> 61); + j += 2; + bebits = (__be64 *)&buf[j]; + *bebits = cpu_to_be64(byte_count << 3); + j += 2; + } + } + if (j * 4 > bufsize) { + pr_err("%s OVERFLOW %llu\n", __func__, j); + return 0; + } + + return j; +} + +static int sun8i_ce_hash_prepare(struct ahash_request *areq, struct ce_task *cet) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); + struct sun8i_ce_alg_template *algt; + struct sun8i_ce_dev *ce; + struct scatterlist *sg; + int nr_sgs, err; + unsigned int len; + u32 common; + u64 byte_count; + __le32 *bf; + int j, i, todo; + u64 bs; + int digestsize; + + algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash.base); + ce = algt->ce; + + bs = crypto_ahash_blocksize(tfm); + digestsize = crypto_ahash_digestsize(tfm); + if (digestsize == SHA224_DIGEST_SIZE) + digestsize = SHA256_DIGEST_SIZE; + if (digestsize == SHA384_DIGEST_SIZE) + digestsize = SHA512_DIGEST_SIZE; + + bf = (__le32 *)rctx->pad; + + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) + algt->stat_req++; + + dev_dbg(ce->dev, "%s %s len=%d\n", __func__, crypto_tfm_alg_name(areq->base.tfm), areq->nbytes); + + memset(cet, 0, sizeof(struct ce_task)); + + cet->t_id = cpu_to_le32(rctx->flow); + common = ce->variant->alg_hash[algt->ce_algo_id]; + common |= CE_COMM_INT; + cet->t_common_ctl = cpu_to_le32(common); + + cet->t_sym_ctl = 0; + cet->t_asym_ctl = 0; + + rctx->nr_sgs = sg_nents_for_len(areq->src, areq->nbytes); + nr_sgs = dma_map_sg(ce->dev, areq->src, rctx->nr_sgs, DMA_TO_DEVICE); + if (nr_sgs <= 0 || nr_sgs > MAX_SG) { + dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); + err = -EINVAL; + goto err_out; + } + + len = areq->nbytes; + for_each_sg(areq->src, sg, nr_sgs, i) { + cet->t_src[i].addr = desc_addr_val_le32(ce, sg_dma_address(sg)); + todo = min(len, sg_dma_len(sg)); + cet->t_src[i].len = cpu_to_le32(todo / 4); + len -= todo; + } + if (len > 0) { + dev_err(ce->dev, "remaining len %d\n", len); + err = -EINVAL; + goto err_unmap_src; + } + + rctx->result_len = digestsize; + rctx->addr_res = dma_map_single(ce->dev, rctx->result, rctx->result_len, + DMA_FROM_DEVICE); + cet->t_dst[0].addr = desc_addr_val_le32(ce, rctx->addr_res); + cet->t_dst[0].len = cpu_to_le32(rctx->result_len / 4); + if (dma_mapping_error(ce->dev, rctx->addr_res)) { + dev_err(ce->dev, "DMA map dest\n"); + err = -EINVAL; + goto err_unmap_src; + } + + byte_count = areq->nbytes; + j = 0; + + switch (algt->ce_algo_id) { + case CE_ID_HASH_MD5: + j = hash_pad(bf, 2 * bs, j, byte_count, true, bs); + break; + case CE_ID_HASH_SHA1: + case CE_ID_HASH_SHA224: + case CE_ID_HASH_SHA256: + j = hash_pad(bf, 2 * bs, j, byte_count, false, bs); + break; + case CE_ID_HASH_SHA384: + case CE_ID_HASH_SHA512: + j = hash_pad(bf, 2 * bs, j, byte_count, false, bs); + break; + } + if (!j) { + err = -EINVAL; + goto err_unmap_result; + } + + rctx->pad_len = j * 4; + rctx->addr_pad = dma_map_single(ce->dev, rctx->pad, rctx->pad_len, + DMA_TO_DEVICE); + cet->t_src[i].addr = desc_addr_val_le32(ce, rctx->addr_pad); + cet->t_src[i].len = cpu_to_le32(j); + if (dma_mapping_error(ce->dev, rctx->addr_pad)) { + dev_err(ce->dev, "DMA error on padding SG\n"); + err = -EINVAL; + goto err_unmap_result; + } + + if (ce->variant->hash_t_dlen_in_bits) + cet->t_dlen = cpu_to_le32((areq->nbytes + j * 4) * 8); + else + cet->t_dlen = cpu_to_le32(areq->nbytes / 4 + j); + + return 0; + +err_unmap_result: + dma_unmap_single(ce->dev, rctx->addr_res, rctx->result_len, + DMA_FROM_DEVICE); + +err_unmap_src: + dma_unmap_sg(ce->dev, areq->src, rctx->nr_sgs, DMA_TO_DEVICE); + +err_out: + return err; +} + +static void sun8i_ce_hash_unprepare(struct ahash_request *areq, + struct ce_task *cet) +{ + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *ctx = crypto_ahash_ctx(tfm); + struct sun8i_ce_dev *ce = ctx->ce; + + dma_unmap_single(ce->dev, rctx->addr_pad, rctx->pad_len, DMA_TO_DEVICE); + dma_unmap_single(ce->dev, rctx->addr_res, rctx->result_len, + DMA_FROM_DEVICE); + dma_unmap_sg(ce->dev, areq->src, rctx->nr_sgs, DMA_TO_DEVICE); +} + +int sun8i_ce_hash_run(struct crypto_engine *engine, void *async_req) +{ + struct ahash_request *areq = ahash_request_cast(async_req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ce_hash_tfm_ctx *ctx = crypto_ahash_ctx(tfm); + struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx_dma(areq); + struct sun8i_ce_dev *ce = ctx->ce; + struct sun8i_ce_flow *chan; + int err; + + chan = &ce->chanlist[rctx->flow]; + + err = sun8i_ce_hash_prepare(areq, chan->tl); + if (err) + return err; + + err = sun8i_ce_run_task(ce, rctx->flow, crypto_ahash_alg_name(tfm)); + + sun8i_ce_hash_unprepare(areq, chan->tl); + + if (!err) + memcpy(areq->result, rctx->result, + crypto_ahash_digestsize(tfm)); + + local_bh_disable(); + crypto_finalize_hash_request(engine, async_req, err); + local_bh_enable(); + + return 0; +} diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c new file mode 100644 index 000000000..244529bf0 --- /dev/null +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sun8i-ce-trng.c - hardware cryptographic offloader for + * Allwinner H3/A64/H5/H2+/H6/R40 SoC + * + * Copyright (C) 2015-2020 Corentin Labbe + * + * This file handle the TRNG + * + * You could find a link for the datasheet in Documentation/arch/arm/sunxi.rst + */ +#include "sun8i-ce.h" +#include +#include +#include +/* + * Note that according to the algorithm ID, 2 versions of the TRNG exists, + * The first present in H3/H5/R40/A64 and the second present in H6. + * This file adds support for both, but only the second is working + * reliabily according to rngtest. + **/ + +static int sun8i_ce_trng_read(struct hwrng *rng, void *data, size_t max, bool wait) +{ + struct sun8i_ce_dev *ce; + dma_addr_t dma_dst; + int err = 0; + int flow = 3; + unsigned int todo; + struct sun8i_ce_flow *chan; + struct ce_task *cet; + u32 common; + void *d; + + ce = container_of(rng, struct sun8i_ce_dev, trng); + + /* round the data length to a multiple of 32*/ + todo = max + 32; + todo -= todo % 32; + + d = kzalloc(todo, GFP_KERNEL | GFP_DMA); + if (!d) + return -ENOMEM; + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + ce->hwrng_stat_req++; + ce->hwrng_stat_bytes += todo; +#endif + + dma_dst = dma_map_single(ce->dev, d, todo, DMA_FROM_DEVICE); + if (dma_mapping_error(ce->dev, dma_dst)) { + dev_err(ce->dev, "Cannot DMA MAP DST\n"); + err = -EFAULT; + goto err_dst; + } + + err = pm_runtime_resume_and_get(ce->dev); + if (err < 0) + goto err_pm; + + mutex_lock(&ce->rnglock); + chan = &ce->chanlist[flow]; + + cet = &chan->tl[0]; + memset(cet, 0, sizeof(struct ce_task)); + + cet->t_id = cpu_to_le32(flow); + common = ce->variant->trng | CE_COMM_INT; + cet->t_common_ctl = cpu_to_le32(common); + + /* recent CE (H6) need length in bytes, in word otherwise */ + if (ce->variant->trng_t_dlen_in_bytes) + cet->t_dlen = cpu_to_le32(todo); + else + cet->t_dlen = cpu_to_le32(todo / 4); + + cet->t_sym_ctl = 0; + cet->t_asym_ctl = 0; + + cet->t_dst[0].addr = desc_addr_val_le32(ce, dma_dst); + cet->t_dst[0].len = cpu_to_le32(todo / 4); + + err = sun8i_ce_run_task(ce, 3, "TRNG"); + mutex_unlock(&ce->rnglock); + + pm_runtime_put(ce->dev); + +err_pm: + dma_unmap_single(ce->dev, dma_dst, todo, DMA_FROM_DEVICE); + + if (!err) { + memcpy(data, d, max); + err = max; + } +err_dst: + kfree_sensitive(d); + return err; +} + +int sun8i_ce_hwrng_register(struct sun8i_ce_dev *ce) +{ + int ret; + + if (ce->variant->trng == CE_ID_NOTSUPP) { + dev_info(ce->dev, "TRNG not supported\n"); + return 0; + } + ce->trng.name = "sun8i Crypto Engine TRNG"; + ce->trng.read = sun8i_ce_trng_read; + + ret = hwrng_register(&ce->trng); + if (ret) + dev_err(ce->dev, "Fail to register the TRNG\n"); + return ret; +} + +void sun8i_ce_hwrng_unregister(struct sun8i_ce_dev *ce) +{ + if (ce->variant->trng == CE_ID_NOTSUPP) + return; + hwrng_unregister(&ce->trng); +} diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h new file mode 100644 index 000000000..468d99bf5 --- /dev/null +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h @@ -0,0 +1,379 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * sun8i-ce.h - hardware cryptographic offloader for + * Allwinner H3/A64/H5/H2+/H6 SoC + * + * Copyright (C) 2016-2019 Corentin LABBE + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* CE Registers */ +#define CE_TDQ 0x00 +#define CE_CTR 0x04 +#define CE_ICR 0x08 +#define CE_ISR 0x0C +#define CE_TLR 0x10 +#define CE_TSR 0x14 +#define CE_ESR 0x18 +#define CE_CSSGR 0x1C +#define CE_CDSGR 0x20 +#define CE_CSAR 0x24 +#define CE_CDAR 0x28 +#define CE_TPR 0x2C + +/* Used in struct ce_task */ +/* ce_task common */ +#define CE_ENCRYPTION 0 +#define CE_DECRYPTION BIT(8) + +#define CE_COMM_INT BIT(31) + +/* ce_task symmetric */ +#define CE_AES_128BITS 0 +#define CE_AES_192BITS 1 +#define CE_AES_256BITS 2 + +#define CE_OP_ECB 0 +#define CE_OP_CBC (1 << 8) + +#define CE_ALG_AES 0 +#define CE_ALG_DES 1 +#define CE_ALG_3DES 2 +#define CE_ALG_MD5 16 +#define CE_ALG_SHA1 17 +#define CE_ALG_SHA224 18 +#define CE_ALG_SHA256 19 +#define CE_ALG_SHA384 20 +#define CE_ALG_SHA512 21 +#define CE_ALG_TRNG 48 +#define CE_ALG_TRNG_V2 0x1c + +/* Used in ce_variant */ +#define CE_ID_NOTSUPP 0xFF + +#define CE_ID_CIPHER_AES 0 +#define CE_ID_CIPHER_DES 1 +#define CE_ID_CIPHER_DES3 2 +#define CE_ID_CIPHER_MAX 3 + +#define CE_ID_HASH_MD5 0 +#define CE_ID_HASH_SHA1 1 +#define CE_ID_HASH_SHA224 2 +#define CE_ID_HASH_SHA256 3 +#define CE_ID_HASH_SHA384 4 +#define CE_ID_HASH_SHA512 5 +#define CE_ID_HASH_MAX 6 + +#define CE_ID_OP_ECB 0 +#define CE_ID_OP_CBC 1 +#define CE_ID_OP_MAX 2 + +/* Used in CE registers */ +#define CE_ERR_ALGO_NOTSUP BIT(0) +#define CE_ERR_DATALEN BIT(1) +#define CE_ERR_KEYSRAM BIT(2) +#define CE_ERR_ADDR_INVALID BIT(5) +#define CE_ERR_KEYLADDER BIT(6) + +#define ESR_H3 0 +#define ESR_A64 1 +#define ESR_R40 2 +#define ESR_H5 3 +#define ESR_H6 4 +#define ESR_D1 5 + +#define CE_DIE_ID_SHIFT 16 +#define CE_DIE_ID_MASK 0x07 + +#define MAX_SG 8 + +#define CE_MAX_CLOCKS 4 +#define CE_DMA_TIMEOUT_MS 3000 + +#define MAXFLOW 4 + +#define CE_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE +#define CE_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE + +/* + * struct ce_clock - Describe clocks used by sun8i-ce + * @name: Name of clock needed by this variant + * @freq: Frequency to set for each clock + * @max_freq: Maximum frequency for each clock (generally given by datasheet) + */ +struct ce_clock { + const char *name; + unsigned long freq; + unsigned long max_freq; +}; + +/* + * struct ce_variant - Describe CE capability for each variant hardware + * @alg_cipher: list of supported ciphers. for each CE_ID_ this will give the + * coresponding CE_ALG_XXX value + * @alg_hash: list of supported hashes. for each CE_ID_ this will give the + * corresponding CE_ALG_XXX value + * @op_mode: list of supported block modes + * @cipher_t_dlen_in_bytes: Does the request size for cipher is in + * bytes or words + * @hash_t_dlen_in_bytes: Does the request size for hash is in + * bits or words + * @trng_t_dlen_in_bytes: Does the request size for TRNG is in + * bytes or words + * @ce_clks: list of clocks needed by this variant + * @esr: The type of error register + * @trng: The CE_ALG_XXX value for the TRNG + */ +struct ce_variant { + char alg_cipher[CE_ID_CIPHER_MAX]; + char alg_hash[CE_ID_HASH_MAX]; + u32 op_mode[CE_ID_OP_MAX]; + bool cipher_t_dlen_in_bytes; + bool hash_t_dlen_in_bits; + bool trng_t_dlen_in_bytes; + bool needs_word_addresses; + struct ce_clock ce_clks[CE_MAX_CLOCKS]; + int esr; + unsigned char trng; +}; + +struct sginfo { + __le32 addr; + __le32 len; +} __packed; + +/* + * struct ce_task - CE Task descriptor + * The structure of this descriptor could be found in the datasheet + */ +struct ce_task { + __le32 t_id; + __le32 t_common_ctl; + __le32 t_sym_ctl; + __le32 t_asym_ctl; + __le32 t_key; + __le32 t_iv; + __le32 t_ctr; + __le32 t_dlen; + struct sginfo t_src[MAX_SG]; + struct sginfo t_dst[MAX_SG]; + __le32 next; + __le32 reserved[3]; +} __packed __aligned(8); + +/* + * struct sun8i_ce_flow - Information used by each flow + * @engine: ptr to the crypto_engine for this flow + * @complete: completion for the current task on this flow + * @status: set to 1 by interrupt if task is done + * @t_phy: Physical address of task + * @tl: pointer to the current ce_task for this flow + * @stat_req: number of request done by this flow + */ +struct sun8i_ce_flow { + struct crypto_engine *engine; + struct completion complete; + int status; + dma_addr_t t_phy; + struct ce_task *tl; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + unsigned long stat_req; +#endif +}; + +/* + * struct sun8i_ce_dev - main container for all this driver information + * @base: base address of CE + * @ceclks: clocks used by CE + * @reset: pointer to reset controller + * @dev: the platform device + * @mlock: Control access to device registers + * @rnglock: Control access to the RNG (dedicated channel 3) + * @chanlist: array of all flow + * @flow: flow to use in next request + * @variant: pointer to variant specific data + * @dbgfs_dir: Debugfs dentry for statistic directory + * @dbgfs_stats: Debugfs dentry for statistic counters + */ +struct sun8i_ce_dev { + void __iomem *base; + struct clk *ceclks[CE_MAX_CLOCKS]; + struct reset_control *reset; + struct device *dev; + struct mutex mlock; + struct mutex rnglock; + struct sun8i_ce_flow *chanlist; + atomic_t flow; + const struct ce_variant *variant; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + struct dentry *dbgfs_dir; + struct dentry *dbgfs_stats; +#endif +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG + struct hwrng trng; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG + unsigned long hwrng_stat_req; + unsigned long hwrng_stat_bytes; +#endif +#endif +}; + +static inline u32 desc_addr_val(struct sun8i_ce_dev *dev, dma_addr_t addr) +{ + if (dev->variant->needs_word_addresses) + return addr / 4; + + return addr; +} + +static inline __le32 desc_addr_val_le32(struct sun8i_ce_dev *dev, + dma_addr_t addr) +{ + return cpu_to_le32(desc_addr_val(dev, addr)); +} + +/* + * struct sun8i_cipher_req_ctx - context for a skcipher request + * @op_dir: direction (encrypt vs decrypt) for this request + * @flow: the flow to use for this request + * @nr_sgs: The number of source SG (as given by dma_map_sg()) + * @nr_sgd: The number of destination SG (as given by dma_map_sg()) + * @addr_iv: The IV addr returned by dma_map_single, need to unmap later + * @addr_key: The key addr returned by dma_map_single, need to unmap later + * @bounce_iv: Current IV buffer + * @backup_iv: Next IV buffer + * @fallback_req: request struct for invoking the fallback skcipher TFM + */ +struct sun8i_cipher_req_ctx { + u32 op_dir; + int flow; + int nr_sgs; + int nr_sgd; + dma_addr_t addr_iv; + dma_addr_t addr_key; + u8 bounce_iv[AES_BLOCK_SIZE] __aligned(sizeof(u32)); + u8 backup_iv[AES_BLOCK_SIZE]; + struct skcipher_request fallback_req; // keep at the end +}; + +/* + * struct sun8i_cipher_tfm_ctx - context for a skcipher TFM + * @key: pointer to key data + * @keylen: len of the key + * @ce: pointer to the private data of driver handling this TFM + * @fallback_tfm: pointer to the fallback TFM + */ +struct sun8i_cipher_tfm_ctx { + u32 *key; + u32 keylen; + struct sun8i_ce_dev *ce; + struct crypto_skcipher *fallback_tfm; +}; + +/* + * struct sun8i_ce_hash_tfm_ctx - context for an ahash TFM + * @ce: pointer to the private data of driver handling this TFM + * @fallback_tfm: pointer to the fallback TFM + */ +struct sun8i_ce_hash_tfm_ctx { + struct sun8i_ce_dev *ce; + struct crypto_ahash *fallback_tfm; +}; + +/* + * struct sun8i_ce_hash_reqctx - context for an ahash request + * @fallback_req: pre-allocated fallback request + * @flow: the flow to use for this request + * @nr_sgs: number of entries in the source scatterlist + * @result_len: result length in bytes + * @pad_len: padding length in bytes + * @addr_res: DMA address of the result buffer, returned by dma_map_single() + * @addr_pad: DMA address of the padding buffer, returned by dma_map_single() + * @result: per-request result buffer + * @pad: per-request padding buffer (up to 2 blocks) + */ +struct sun8i_ce_hash_reqctx { + int flow; + int nr_sgs; + size_t result_len; + size_t pad_len; + dma_addr_t addr_res; + dma_addr_t addr_pad; + u8 result[CE_MAX_HASH_DIGEST_SIZE] __aligned(CRYPTO_DMA_ALIGN); + u8 pad[2 * CE_MAX_HASH_BLOCK_SIZE]; + struct ahash_request fallback_req; // keep at the end +}; + +/* + * struct sun8i_ce_alg_template - crypto_alg template + * @type: the CRYPTO_ALG_TYPE for this template + * @ce_algo_id: the CE_ID for this template + * @ce_blockmode: the type of block operation CE_ID + * @ce: pointer to the sun8i_ce_dev structure associated with + * this template + * @alg: one of sub struct must be used + * @stat_req: number of request done on this template + * @stat_fb: number of request which has fallbacked + * @stat_bytes: total data size done by this template + */ +struct sun8i_ce_alg_template { + u32 type; + u32 ce_algo_id; + u32 ce_blockmode; + struct sun8i_ce_dev *ce; + union { + struct skcipher_engine_alg skcipher; + struct ahash_engine_alg hash; + } alg; + unsigned long stat_req; + unsigned long stat_fb; + unsigned long stat_bytes; + unsigned long stat_fb_maxsg; + unsigned long stat_fb_leniv; + unsigned long stat_fb_len0; + unsigned long stat_fb_mod16; + unsigned long stat_fb_srcali; + unsigned long stat_fb_srclen; + unsigned long stat_fb_dstali; + unsigned long stat_fb_dstlen; + char fbname[CRYPTO_MAX_ALG_NAME]; +}; + +int sun8i_ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen); +int sun8i_ce_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen); +int sun8i_ce_cipher_init(struct crypto_tfm *tfm); +void sun8i_ce_cipher_exit(struct crypto_tfm *tfm); +int sun8i_ce_cipher_do_one(struct crypto_engine *engine, void *areq); +int sun8i_ce_skdecrypt(struct skcipher_request *areq); +int sun8i_ce_skencrypt(struct skcipher_request *areq); + +int sun8i_ce_get_engine_number(struct sun8i_ce_dev *ce); + +int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name); + +int sun8i_ce_hash_init_tfm(struct crypto_ahash *tfm); +void sun8i_ce_hash_exit_tfm(struct crypto_ahash *tfm); +int sun8i_ce_hash_init(struct ahash_request *areq); +int sun8i_ce_hash_export(struct ahash_request *areq, void *out); +int sun8i_ce_hash_import(struct ahash_request *areq, const void *in); +int sun8i_ce_hash_final(struct ahash_request *areq); +int sun8i_ce_hash_update(struct ahash_request *areq); +int sun8i_ce_hash_finup(struct ahash_request *areq); +int sun8i_ce_hash_digest(struct ahash_request *areq); +int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq); + +int sun8i_ce_hwrng_register(struct sun8i_ce_dev *ce); +void sun8i_ce_hwrng_unregister(struct sun8i_ce_dev *ce); diff --git a/drivers/crypto/allwinner/sun8i-ss/Makefile b/drivers/crypto/allwinner/sun8i-ss/Makefile new file mode 100644 index 000000000..2d6458a42 --- /dev/null +++ b/drivers/crypto/allwinner/sun8i-ss/Makefile @@ -0,0 +1,3 @@ +obj-$(CONFIG_CRYPTO_DEV_SUN8I_SS) += sun8i-ss.o +sun8i-ss-y += sun8i-ss-core.o sun8i-ss-cipher.o +sun8i-ss-$(CONFIG_CRYPTO_DEV_SUN8I_SS_HASH) += sun8i-ss-hash.o diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c new file mode 100644 index 000000000..8831bcb23 --- /dev/null +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c @@ -0,0 +1,486 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sun8i-ss-cipher.c - hardware cryptographic offloader for + * Allwinner A80/A83T SoC + * + * Copyright (C) 2016-2019 Corentin LABBE + * + * This file add support for AES cipher with 128,192,256 bits keysize in + * CBC and ECB mode. + * + * You could find a link for the datasheet in Documentation/arch/arm/sunxi.rst + */ + +#include +#include +#include +#include +#include +#include +#include +#include "sun8i-ss.h" + +static bool sun8i_ss_need_fallback(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct sun8i_ss_alg_template *algt = container_of(alg, struct sun8i_ss_alg_template, alg.skcipher.base); + struct scatterlist *in_sg = areq->src; + struct scatterlist *out_sg = areq->dst; + struct scatterlist *sg; + unsigned int todo, len; + + if (areq->cryptlen == 0 || areq->cryptlen % 16) { + algt->stat_fb_len++; + return true; + } + + if (sg_nents_for_len(areq->src, areq->cryptlen) > 8 || + sg_nents_for_len(areq->dst, areq->cryptlen) > 8) { + algt->stat_fb_sgnum++; + return true; + } + + len = areq->cryptlen; + sg = areq->src; + while (sg) { + todo = min(len, sg->length); + if ((todo % 16) != 0) { + algt->stat_fb_sglen++; + return true; + } + if (!IS_ALIGNED(sg->offset, 16)) { + algt->stat_fb_align++; + return true; + } + len -= todo; + sg = sg_next(sg); + } + len = areq->cryptlen; + sg = areq->dst; + while (sg) { + todo = min(len, sg->length); + if ((todo % 16) != 0) { + algt->stat_fb_sglen++; + return true; + } + if (!IS_ALIGNED(sg->offset, 16)) { + algt->stat_fb_align++; + return true; + } + len -= todo; + sg = sg_next(sg); + } + + /* SS need same numbers of SG (with same length) for source and destination */ + in_sg = areq->src; + out_sg = areq->dst; + while (in_sg && out_sg) { + if (in_sg->length != out_sg->length) + return true; + in_sg = sg_next(in_sg); + out_sg = sg_next(out_sg); + } + if (in_sg || out_sg) + return true; + return false; +} + +static int sun8i_ss_cipher_fallback(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + int err; + + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG)) { + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct sun8i_ss_alg_template *algt __maybe_unused; + + algt = container_of(alg, struct sun8i_ss_alg_template, + alg.skcipher.base); + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + algt->stat_fb++; +#endif + } + + skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm); + skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags, + areq->base.complete, areq->base.data); + skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst, + areq->cryptlen, areq->iv); + if (rctx->op_dir & SS_DECRYPTION) + err = crypto_skcipher_decrypt(&rctx->fallback_req); + else + err = crypto_skcipher_encrypt(&rctx->fallback_req); + return err; +} + +static int sun8i_ss_setup_ivs(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun8i_ss_dev *ss = op->ss; + struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + struct scatterlist *sg = areq->src; + unsigned int todo, offset; + unsigned int len = areq->cryptlen; + unsigned int ivsize = crypto_skcipher_ivsize(tfm); + struct sun8i_ss_flow *sf = &ss->flows[rctx->flow]; + int i = 0; + dma_addr_t a; + int err; + + rctx->ivlen = ivsize; + if (rctx->op_dir & SS_DECRYPTION) { + offset = areq->cryptlen - ivsize; + scatterwalk_map_and_copy(sf->biv, areq->src, offset, + ivsize, 0); + } + + /* we need to copy all IVs from source in case DMA is bi-directionnal */ + while (sg && len) { + if (sg->length == 0) { + sg = sg_next(sg); + continue; + } + if (i == 0) + memcpy(sf->iv[0], areq->iv, ivsize); + a = dma_map_single(ss->dev, sf->iv[i], ivsize, DMA_TO_DEVICE); + if (dma_mapping_error(ss->dev, a)) { + memzero_explicit(sf->iv[i], ivsize); + dev_err(ss->dev, "Cannot DMA MAP IV\n"); + err = -EFAULT; + goto dma_iv_error; + } + rctx->p_iv[i] = a; + /* we need to setup all others IVs only in the decrypt way */ + if (rctx->op_dir == SS_ENCRYPTION) + return 0; + todo = min(len, sg_dma_len(sg)); + len -= todo; + i++; + if (i < MAX_SG) { + offset = sg->length - ivsize; + scatterwalk_map_and_copy(sf->iv[i], sg, offset, ivsize, 0); + } + rctx->niv = i; + sg = sg_next(sg); + } + + return 0; +dma_iv_error: + i--; + while (i >= 0) { + dma_unmap_single(ss->dev, rctx->p_iv[i], ivsize, DMA_TO_DEVICE); + memzero_explicit(sf->iv[i], ivsize); + i--; + } + return err; +} + +static int sun8i_ss_cipher(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun8i_ss_dev *ss = op->ss; + struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct sun8i_ss_alg_template *algt; + struct sun8i_ss_flow *sf = &ss->flows[rctx->flow]; + struct scatterlist *sg; + unsigned int todo, len, offset, ivsize; + int nr_sgs = 0; + int nr_sgd = 0; + int err = 0; + int nsgs = sg_nents_for_len(areq->src, areq->cryptlen); + int nsgd = sg_nents_for_len(areq->dst, areq->cryptlen); + int i; + + algt = container_of(alg, struct sun8i_ss_alg_template, alg.skcipher.base); + + dev_dbg(ss->dev, "%s %s %u %x IV(%p %u) key=%u\n", __func__, + crypto_tfm_alg_name(areq->base.tfm), + areq->cryptlen, + rctx->op_dir, areq->iv, crypto_skcipher_ivsize(tfm), + op->keylen); + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + algt->stat_req++; +#endif + + rctx->op_mode = ss->variant->op_mode[algt->ss_blockmode]; + rctx->method = ss->variant->alg_cipher[algt->ss_algo_id]; + rctx->keylen = op->keylen; + + rctx->p_key = dma_map_single(ss->dev, op->key, op->keylen, DMA_TO_DEVICE); + if (dma_mapping_error(ss->dev, rctx->p_key)) { + dev_err(ss->dev, "Cannot DMA MAP KEY\n"); + err = -EFAULT; + goto theend; + } + + ivsize = crypto_skcipher_ivsize(tfm); + if (areq->iv && crypto_skcipher_ivsize(tfm) > 0) { + err = sun8i_ss_setup_ivs(areq); + if (err) + goto theend_key; + } + if (areq->src == areq->dst) { + nr_sgs = dma_map_sg(ss->dev, areq->src, nsgs, DMA_BIDIRECTIONAL); + if (nr_sgs <= 0 || nr_sgs > 8) { + dev_err(ss->dev, "Invalid sg number %d\n", nr_sgs); + err = -EINVAL; + goto theend_iv; + } + nr_sgd = nr_sgs; + } else { + nr_sgs = dma_map_sg(ss->dev, areq->src, nsgs, DMA_TO_DEVICE); + if (nr_sgs <= 0 || nr_sgs > 8) { + dev_err(ss->dev, "Invalid sg number %d\n", nr_sgs); + err = -EINVAL; + goto theend_iv; + } + nr_sgd = dma_map_sg(ss->dev, areq->dst, nsgd, DMA_FROM_DEVICE); + if (nr_sgd <= 0 || nr_sgd > 8) { + dev_err(ss->dev, "Invalid sg number %d\n", nr_sgd); + err = -EINVAL; + goto theend_sgs; + } + } + + len = areq->cryptlen; + i = 0; + sg = areq->src; + while (i < nr_sgs && sg && len) { + if (sg_dma_len(sg) == 0) + goto sgs_next; + rctx->t_src[i].addr = sg_dma_address(sg); + todo = min(len, sg_dma_len(sg)); + rctx->t_src[i].len = todo / 4; + dev_dbg(ss->dev, "%s total=%u SGS(%d %u off=%d) todo=%u\n", __func__, + areq->cryptlen, i, rctx->t_src[i].len, sg->offset, todo); + len -= todo; + i++; +sgs_next: + sg = sg_next(sg); + } + if (len > 0) { + dev_err(ss->dev, "remaining len %d\n", len); + err = -EINVAL; + goto theend_sgs; + } + + len = areq->cryptlen; + i = 0; + sg = areq->dst; + while (i < nr_sgd && sg && len) { + if (sg_dma_len(sg) == 0) + goto sgd_next; + rctx->t_dst[i].addr = sg_dma_address(sg); + todo = min(len, sg_dma_len(sg)); + rctx->t_dst[i].len = todo / 4; + dev_dbg(ss->dev, "%s total=%u SGD(%d %u off=%d) todo=%u\n", __func__, + areq->cryptlen, i, rctx->t_dst[i].len, sg->offset, todo); + len -= todo; + i++; +sgd_next: + sg = sg_next(sg); + } + if (len > 0) { + dev_err(ss->dev, "remaining len %d\n", len); + err = -EINVAL; + goto theend_sgs; + } + + err = sun8i_ss_run_task(ss, rctx, crypto_tfm_alg_name(areq->base.tfm)); + +theend_sgs: + if (areq->src == areq->dst) { + dma_unmap_sg(ss->dev, areq->src, nsgs, DMA_BIDIRECTIONAL); + } else { + dma_unmap_sg(ss->dev, areq->src, nsgs, DMA_TO_DEVICE); + dma_unmap_sg(ss->dev, areq->dst, nsgd, DMA_FROM_DEVICE); + } + +theend_iv: + if (areq->iv && ivsize > 0) { + for (i = 0; i < rctx->niv; i++) { + dma_unmap_single(ss->dev, rctx->p_iv[i], ivsize, DMA_TO_DEVICE); + memzero_explicit(sf->iv[i], ivsize); + } + + offset = areq->cryptlen - ivsize; + if (rctx->op_dir & SS_DECRYPTION) { + memcpy(areq->iv, sf->biv, ivsize); + memzero_explicit(sf->biv, ivsize); + } else { + scatterwalk_map_and_copy(areq->iv, areq->dst, offset, + ivsize, 0); + } + } + +theend_key: + dma_unmap_single(ss->dev, rctx->p_key, op->keylen, DMA_TO_DEVICE); + +theend: + + return err; +} + +int sun8i_ss_handle_cipher_request(struct crypto_engine *engine, void *areq) +{ + int err; + struct skcipher_request *breq = container_of(areq, struct skcipher_request, base); + + err = sun8i_ss_cipher(breq); + local_bh_disable(); + crypto_finalize_skcipher_request(engine, breq, err); + local_bh_enable(); + + return 0; +} + +int sun8i_ss_skdecrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + struct crypto_engine *engine; + int e; + + memset(rctx, 0, sizeof(struct sun8i_cipher_req_ctx)); + rctx->op_dir = SS_DECRYPTION; + + if (sun8i_ss_need_fallback(areq)) + return sun8i_ss_cipher_fallback(areq); + + e = sun8i_ss_get_engine_number(op->ss); + engine = op->ss->flows[e].engine; + rctx->flow = e; + + return crypto_transfer_skcipher_request_to_engine(engine, areq); +} + +int sun8i_ss_skencrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + struct crypto_engine *engine; + int e; + + memset(rctx, 0, sizeof(struct sun8i_cipher_req_ctx)); + rctx->op_dir = SS_ENCRYPTION; + + if (sun8i_ss_need_fallback(areq)) + return sun8i_ss_cipher_fallback(areq); + + e = sun8i_ss_get_engine_number(op->ss); + engine = op->ss->flows[e].engine; + rctx->flow = e; + + return crypto_transfer_skcipher_request_to_engine(engine, areq); +} + +int sun8i_ss_cipher_init(struct crypto_tfm *tfm) +{ + struct sun8i_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm); + struct sun8i_ss_alg_template *algt; + const char *name = crypto_tfm_alg_name(tfm); + struct crypto_skcipher *sktfm = __crypto_skcipher_cast(tfm); + struct skcipher_alg *alg = crypto_skcipher_alg(sktfm); + int err; + + memset(op, 0, sizeof(struct sun8i_cipher_tfm_ctx)); + + algt = container_of(alg, struct sun8i_ss_alg_template, alg.skcipher.base); + op->ss = algt->ss; + + op->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(op->fallback_tfm)) { + dev_err(op->ss->dev, "ERROR: Cannot allocate fallback for %s %ld\n", + name, PTR_ERR(op->fallback_tfm)); + return PTR_ERR(op->fallback_tfm); + } + + crypto_skcipher_set_reqsize(sktfm, sizeof(struct sun8i_cipher_req_ctx) + + crypto_skcipher_reqsize(op->fallback_tfm)); + + memcpy(algt->fbname, + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(op->fallback_tfm)), + CRYPTO_MAX_ALG_NAME); + + err = pm_runtime_resume_and_get(op->ss->dev); + if (err < 0) { + dev_err(op->ss->dev, "pm error %d\n", err); + goto error_pm; + } + + return 0; +error_pm: + crypto_free_skcipher(op->fallback_tfm); + return err; +} + +void sun8i_ss_cipher_exit(struct crypto_tfm *tfm) +{ + struct sun8i_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm); + + kfree_sensitive(op->key); + crypto_free_skcipher(op->fallback_tfm); + pm_runtime_put_sync(op->ss->dev); +} + +int sun8i_ss_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun8i_ss_dev *ss = op->ss; + + switch (keylen) { + case 128 / 8: + break; + case 192 / 8: + break; + case 256 / 8: + break; + default: + dev_dbg(ss->dev, "ERROR: Invalid keylen %u\n", keylen); + return -EINVAL; + } + kfree_sensitive(op->key); + op->keylen = keylen; + op->key = kmemdup(key, keylen, GFP_KERNEL); + if (!op->key) + return -ENOMEM; + + crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); + + return crypto_skcipher_setkey(op->fallback_tfm, key, keylen); +} + +int sun8i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun8i_ss_dev *ss = op->ss; + + if (unlikely(keylen != 3 * DES_KEY_SIZE)) { + dev_dbg(ss->dev, "Invalid keylen %u\n", keylen); + return -EINVAL; + } + + kfree_sensitive(op->key); + op->keylen = keylen; + op->key = kmemdup(key, keylen, GFP_KERNEL); + if (!op->key) + return -ENOMEM; + + crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); + + return crypto_skcipher_setkey(op->fallback_tfm, key, keylen); +} diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c new file mode 100644 index 000000000..2167dd9f4 --- /dev/null +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c @@ -0,0 +1,909 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sun8i-ss-core.c - hardware cryptographic offloader for + * Allwinner A80/A83T SoC + * + * Copyright (C) 2015-2019 Corentin Labbe + * + * Core file which registers crypto algorithms supported by the SecuritySystem + * + * You could find a link for the datasheet in Documentation/arch/arm/sunxi.rst + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sun8i-ss.h" + +static const struct ss_variant ss_a80_variant = { + .alg_cipher = { SS_ALG_AES, SS_ALG_DES, SS_ALG_3DES, + }, + .alg_hash = { SS_ID_NOTSUPP, SS_ID_NOTSUPP, SS_ID_NOTSUPP, SS_ID_NOTSUPP, + }, + .op_mode = { SS_OP_ECB, SS_OP_CBC, + }, + .ss_clks = { + { "bus", 0, 300 * 1000 * 1000 }, + { "mod", 0, 300 * 1000 * 1000 }, + } +}; + +static const struct ss_variant ss_a83t_variant = { + .alg_cipher = { SS_ALG_AES, SS_ALG_DES, SS_ALG_3DES, + }, + .alg_hash = { SS_ALG_MD5, SS_ALG_SHA1, SS_ALG_SHA224, SS_ALG_SHA256, + }, + .op_mode = { SS_OP_ECB, SS_OP_CBC, + }, + .ss_clks = { + { "bus", 0, 300 * 1000 * 1000 }, + { "mod", 0, 300 * 1000 * 1000 }, + } +}; + +/* + * sun8i_ss_get_engine_number() get the next channel slot + * This is a simple round-robin way of getting the next channel + */ +int sun8i_ss_get_engine_number(struct sun8i_ss_dev *ss) +{ + return atomic_inc_return(&ss->flow) % MAXFLOW; +} + +int sun8i_ss_run_task(struct sun8i_ss_dev *ss, struct sun8i_cipher_req_ctx *rctx, + const char *name) +{ + int flow = rctx->flow; + unsigned int ivlen = rctx->ivlen; + u32 v = SS_START; + int i; + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + ss->flows[flow].stat_req++; +#endif + + /* choose between stream0/stream1 */ + if (flow) + v |= SS_FLOW1; + else + v |= SS_FLOW0; + + v |= rctx->op_mode; + v |= rctx->method; + + if (rctx->op_dir) + v |= SS_DECRYPTION; + + switch (rctx->keylen) { + case 128 / 8: + v |= SS_AES_128BITS << 7; + break; + case 192 / 8: + v |= SS_AES_192BITS << 7; + break; + case 256 / 8: + v |= SS_AES_256BITS << 7; + break; + } + + for (i = 0; i < MAX_SG; i++) { + if (!rctx->t_dst[i].addr) + break; + + mutex_lock(&ss->mlock); + writel(rctx->p_key, ss->base + SS_KEY_ADR_REG); + + if (ivlen) { + if (rctx->op_dir == SS_ENCRYPTION) { + if (i == 0) + writel(rctx->p_iv[0], ss->base + SS_IV_ADR_REG); + else + writel(rctx->t_dst[i - 1].addr + rctx->t_dst[i - 1].len * 4 - ivlen, ss->base + SS_IV_ADR_REG); + } else { + writel(rctx->p_iv[i], ss->base + SS_IV_ADR_REG); + } + } + + dev_dbg(ss->dev, + "Processing SG %d on flow %d %s ctl=%x %d to %d method=%x opmode=%x opdir=%x srclen=%d\n", + i, flow, name, v, + rctx->t_src[i].len, rctx->t_dst[i].len, + rctx->method, rctx->op_mode, + rctx->op_dir, rctx->t_src[i].len); + + writel(rctx->t_src[i].addr, ss->base + SS_SRC_ADR_REG); + writel(rctx->t_dst[i].addr, ss->base + SS_DST_ADR_REG); + writel(rctx->t_src[i].len, ss->base + SS_LEN_ADR_REG); + + reinit_completion(&ss->flows[flow].complete); + ss->flows[flow].status = 0; + wmb(); + + writel(v, ss->base + SS_CTL_REG); + mutex_unlock(&ss->mlock); + wait_for_completion_interruptible_timeout(&ss->flows[flow].complete, + msecs_to_jiffies(2000)); + if (ss->flows[flow].status == 0) { + dev_err(ss->dev, "DMA timeout for %s\n", name); + return -EFAULT; + } + } + + return 0; +} + +static irqreturn_t ss_irq_handler(int irq, void *data) +{ + struct sun8i_ss_dev *ss = (struct sun8i_ss_dev *)data; + int flow = 0; + u32 p; + + p = readl(ss->base + SS_INT_STA_REG); + for (flow = 0; flow < MAXFLOW; flow++) { + if (p & (BIT(flow))) { + writel(BIT(flow), ss->base + SS_INT_STA_REG); + ss->flows[flow].status = 1; + complete(&ss->flows[flow].complete); + } + } + + return IRQ_HANDLED; +} + +static struct sun8i_ss_alg_template ss_algs[] = { +{ + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .ss_algo_id = SS_ID_CIPHER_AES, + .ss_blockmode = SS_ID_OP_CBC, + .alg.skcipher.base = { + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-sun8i-ss", + .cra_priority = 400, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct sun8i_cipher_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 0xf, + .cra_init = sun8i_ss_cipher_init, + .cra_exit = sun8i_ss_cipher_exit, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = sun8i_ss_aes_setkey, + .encrypt = sun8i_ss_skencrypt, + .decrypt = sun8i_ss_skdecrypt, + }, + .alg.skcipher.op = { + .do_one_request = sun8i_ss_handle_cipher_request, + }, +}, +{ + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .ss_algo_id = SS_ID_CIPHER_AES, + .ss_blockmode = SS_ID_OP_ECB, + .alg.skcipher.base = { + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-sun8i-ss", + .cra_priority = 400, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct sun8i_cipher_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 0xf, + .cra_init = sun8i_ss_cipher_init, + .cra_exit = sun8i_ss_cipher_exit, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = sun8i_ss_aes_setkey, + .encrypt = sun8i_ss_skencrypt, + .decrypt = sun8i_ss_skdecrypt, + }, + .alg.skcipher.op = { + .do_one_request = sun8i_ss_handle_cipher_request, + }, +}, +{ + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .ss_algo_id = SS_ID_CIPHER_DES3, + .ss_blockmode = SS_ID_OP_CBC, + .alg.skcipher.base = { + .base = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "cbc-des3-sun8i-ss", + .cra_priority = 400, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct sun8i_cipher_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 0xf, + .cra_init = sun8i_ss_cipher_init, + .cra_exit = sun8i_ss_cipher_exit, + }, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setkey = sun8i_ss_des3_setkey, + .encrypt = sun8i_ss_skencrypt, + .decrypt = sun8i_ss_skdecrypt, + }, + .alg.skcipher.op = { + .do_one_request = sun8i_ss_handle_cipher_request, + }, +}, +{ + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .ss_algo_id = SS_ID_CIPHER_DES3, + .ss_blockmode = SS_ID_OP_ECB, + .alg.skcipher.base = { + .base = { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "ecb-des3-sun8i-ss", + .cra_priority = 400, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct sun8i_cipher_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 0xf, + .cra_init = sun8i_ss_cipher_init, + .cra_exit = sun8i_ss_cipher_exit, + }, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = sun8i_ss_des3_setkey, + .encrypt = sun8i_ss_skencrypt, + .decrypt = sun8i_ss_skdecrypt, + }, + .alg.skcipher.op = { + .do_one_request = sun8i_ss_handle_cipher_request, + }, +}, +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_HASH +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ss_algo_id = SS_ID_HASH_MD5, + .alg.hash.base = { + .init = sun8i_ss_hash_init, + .update = sun8i_ss_hash_update, + .final = sun8i_ss_hash_final, + .finup = sun8i_ss_hash_finup, + .digest = sun8i_ss_hash_digest, + .export = sun8i_ss_hash_export, + .import = sun8i_ss_hash_import, + .init_tfm = sun8i_ss_hash_init_tfm, + .exit_tfm = sun8i_ss_hash_exit_tfm, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct md5_state), + .base = { + .cra_name = "md5", + .cra_driver_name = "md5-sun8i-ss", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ss_hash_tfm_ctx), + .cra_module = THIS_MODULE, + } + } + }, + .alg.hash.op = { + .do_one_request = sun8i_ss_hash_run, + }, +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ss_algo_id = SS_ID_HASH_SHA1, + .alg.hash.base = { + .init = sun8i_ss_hash_init, + .update = sun8i_ss_hash_update, + .final = sun8i_ss_hash_final, + .finup = sun8i_ss_hash_finup, + .digest = sun8i_ss_hash_digest, + .export = sun8i_ss_hash_export, + .import = sun8i_ss_hash_import, + .init_tfm = sun8i_ss_hash_init_tfm, + .exit_tfm = sun8i_ss_hash_exit_tfm, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct sha1_state), + .base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-sun8i-ss", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ss_hash_tfm_ctx), + .cra_module = THIS_MODULE, + } + } + }, + .alg.hash.op = { + .do_one_request = sun8i_ss_hash_run, + }, +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ss_algo_id = SS_ID_HASH_SHA224, + .alg.hash.base = { + .init = sun8i_ss_hash_init, + .update = sun8i_ss_hash_update, + .final = sun8i_ss_hash_final, + .finup = sun8i_ss_hash_finup, + .digest = sun8i_ss_hash_digest, + .export = sun8i_ss_hash_export, + .import = sun8i_ss_hash_import, + .init_tfm = sun8i_ss_hash_init_tfm, + .exit_tfm = sun8i_ss_hash_exit_tfm, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha224", + .cra_driver_name = "sha224-sun8i-ss", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ss_hash_tfm_ctx), + .cra_module = THIS_MODULE, + } + } + }, + .alg.hash.op = { + .do_one_request = sun8i_ss_hash_run, + }, +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ss_algo_id = SS_ID_HASH_SHA256, + .alg.hash.base = { + .init = sun8i_ss_hash_init, + .update = sun8i_ss_hash_update, + .final = sun8i_ss_hash_final, + .finup = sun8i_ss_hash_finup, + .digest = sun8i_ss_hash_digest, + .export = sun8i_ss_hash_export, + .import = sun8i_ss_hash_import, + .init_tfm = sun8i_ss_hash_init_tfm, + .exit_tfm = sun8i_ss_hash_exit_tfm, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-sun8i-ss", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ss_hash_tfm_ctx), + .cra_module = THIS_MODULE, + } + } + }, + .alg.hash.op = { + .do_one_request = sun8i_ss_hash_run, + }, +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .ss_algo_id = SS_ID_HASH_SHA1, + .alg.hash.base = { + .init = sun8i_ss_hash_init, + .update = sun8i_ss_hash_update, + .final = sun8i_ss_hash_final, + .finup = sun8i_ss_hash_finup, + .digest = sun8i_ss_hash_digest, + .export = sun8i_ss_hash_export, + .import = sun8i_ss_hash_import, + .init_tfm = sun8i_ss_hash_init_tfm, + .exit_tfm = sun8i_ss_hash_exit_tfm, + .setkey = sun8i_ss_hmac_setkey, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct sha1_state), + .base = { + .cra_name = "hmac(sha1)", + .cra_driver_name = "hmac-sha1-sun8i-ss", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun8i_ss_hash_tfm_ctx), + .cra_module = THIS_MODULE, + } + } + }, + .alg.hash.op = { + .do_one_request = sun8i_ss_hash_run, + }, +}, +#endif +}; + +static int sun8i_ss_debugfs_show(struct seq_file *seq, void *v) +{ + struct sun8i_ss_dev *ss __maybe_unused = seq->private; + unsigned int i; + + for (i = 0; i < MAXFLOW; i++) + seq_printf(seq, "Channel %d: nreq %lu\n", i, +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + ss->flows[i].stat_req); +#else + 0ul); +#endif + + for (i = 0; i < ARRAY_SIZE(ss_algs); i++) { + if (!ss_algs[i].ss) + continue; + switch (ss_algs[i].type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n", + ss_algs[i].alg.skcipher.base.base.cra_driver_name, + ss_algs[i].alg.skcipher.base.base.cra_name, + ss_algs[i].stat_req, ss_algs[i].stat_fb); + + seq_printf(seq, "\tLast fallback is: %s\n", + ss_algs[i].fbname); + seq_printf(seq, "\tFallback due to length: %lu\n", + ss_algs[i].stat_fb_len); + seq_printf(seq, "\tFallback due to SG length: %lu\n", + ss_algs[i].stat_fb_sglen); + seq_printf(seq, "\tFallback due to alignment: %lu\n", + ss_algs[i].stat_fb_align); + seq_printf(seq, "\tFallback due to SG numbers: %lu\n", + ss_algs[i].stat_fb_sgnum); + break; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_HASH + case CRYPTO_ALG_TYPE_AHASH: + seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n", + ss_algs[i].alg.hash.base.halg.base.cra_driver_name, + ss_algs[i].alg.hash.base.halg.base.cra_name, + ss_algs[i].stat_req, ss_algs[i].stat_fb); + seq_printf(seq, "\tLast fallback is: %s\n", + ss_algs[i].fbname); + seq_printf(seq, "\tFallback due to length: %lu\n", + ss_algs[i].stat_fb_len); + seq_printf(seq, "\tFallback due to SG length: %lu\n", + ss_algs[i].stat_fb_sglen); + seq_printf(seq, "\tFallback due to alignment: %lu\n", + ss_algs[i].stat_fb_align); + seq_printf(seq, "\tFallback due to SG numbers: %lu\n", + ss_algs[i].stat_fb_sgnum); + break; +#endif + } + } + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(sun8i_ss_debugfs); + +static void sun8i_ss_free_flows(struct sun8i_ss_dev *ss, int i) +{ + while (i >= 0) { + crypto_engine_exit(ss->flows[i].engine); + i--; + } +} + +/* + * Allocate the flow list structure + */ +static int allocate_flows(struct sun8i_ss_dev *ss) +{ + int i, j, err; + + ss->flows = devm_kcalloc(ss->dev, MAXFLOW, sizeof(struct sun8i_ss_flow), + GFP_KERNEL); + if (!ss->flows) + return -ENOMEM; + + for (i = 0; i < MAXFLOW; i++) { + init_completion(&ss->flows[i].complete); + + ss->flows[i].biv = devm_kmalloc(ss->dev, AES_BLOCK_SIZE, + GFP_KERNEL); + if (!ss->flows[i].biv) { + err = -ENOMEM; + goto error_engine; + } + + for (j = 0; j < MAX_SG; j++) { + ss->flows[i].iv[j] = devm_kmalloc(ss->dev, AES_BLOCK_SIZE, + GFP_KERNEL); + if (!ss->flows[i].iv[j]) { + err = -ENOMEM; + goto error_engine; + } + } + + /* the padding could be up to two block. */ + ss->flows[i].pad = devm_kmalloc(ss->dev, MAX_PAD_SIZE, + GFP_KERNEL); + if (!ss->flows[i].pad) { + err = -ENOMEM; + goto error_engine; + } + ss->flows[i].result = + devm_kmalloc(ss->dev, max(SHA256_DIGEST_SIZE, + dma_get_cache_alignment()), + GFP_KERNEL); + if (!ss->flows[i].result) { + err = -ENOMEM; + goto error_engine; + } + + ss->flows[i].engine = crypto_engine_alloc_init(ss->dev, true); + if (!ss->flows[i].engine) { + dev_err(ss->dev, "Cannot allocate engine\n"); + i--; + err = -ENOMEM; + goto error_engine; + } + err = crypto_engine_start(ss->flows[i].engine); + if (err) { + dev_err(ss->dev, "Cannot start engine\n"); + goto error_engine; + } + } + return 0; +error_engine: + sun8i_ss_free_flows(ss, i); + return err; +} + +/* + * Power management strategy: The device is suspended unless a TFM exists for + * one of the algorithms proposed by this driver. + */ +static int sun8i_ss_pm_suspend(struct device *dev) +{ + struct sun8i_ss_dev *ss = dev_get_drvdata(dev); + int i; + + reset_control_assert(ss->reset); + for (i = 0; i < SS_MAX_CLOCKS; i++) + clk_disable_unprepare(ss->ssclks[i]); + return 0; +} + +static int sun8i_ss_pm_resume(struct device *dev) +{ + struct sun8i_ss_dev *ss = dev_get_drvdata(dev); + int err, i; + + for (i = 0; i < SS_MAX_CLOCKS; i++) { + if (!ss->variant->ss_clks[i].name) + continue; + err = clk_prepare_enable(ss->ssclks[i]); + if (err) { + dev_err(ss->dev, "Cannot prepare_enable %s\n", + ss->variant->ss_clks[i].name); + goto error; + } + } + err = reset_control_deassert(ss->reset); + if (err) { + dev_err(ss->dev, "Cannot deassert reset control\n"); + goto error; + } + /* enable interrupts for all flows */ + writel(BIT(0) | BIT(1), ss->base + SS_INT_CTL_REG); + + return 0; +error: + sun8i_ss_pm_suspend(dev); + return err; +} + +static const struct dev_pm_ops sun8i_ss_pm_ops = { + SET_RUNTIME_PM_OPS(sun8i_ss_pm_suspend, sun8i_ss_pm_resume, NULL) +}; + +static int sun8i_ss_pm_init(struct sun8i_ss_dev *ss) +{ + int err; + + pm_runtime_use_autosuspend(ss->dev); + pm_runtime_set_autosuspend_delay(ss->dev, 2000); + + err = pm_runtime_set_suspended(ss->dev); + if (err) + return err; + pm_runtime_enable(ss->dev); + return err; +} + +static void sun8i_ss_pm_exit(struct sun8i_ss_dev *ss) +{ + pm_runtime_disable(ss->dev); +} + +static int sun8i_ss_register_algs(struct sun8i_ss_dev *ss) +{ + int ss_method, err, id; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(ss_algs); i++) { + ss_algs[i].ss = ss; + switch (ss_algs[i].type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + id = ss_algs[i].ss_algo_id; + ss_method = ss->variant->alg_cipher[id]; + if (ss_method == SS_ID_NOTSUPP) { + dev_info(ss->dev, + "DEBUG: Algo of %s not supported\n", + ss_algs[i].alg.skcipher.base.base.cra_name); + ss_algs[i].ss = NULL; + break; + } + id = ss_algs[i].ss_blockmode; + ss_method = ss->variant->op_mode[id]; + if (ss_method == SS_ID_NOTSUPP) { + dev_info(ss->dev, "DEBUG: Blockmode of %s not supported\n", + ss_algs[i].alg.skcipher.base.base.cra_name); + ss_algs[i].ss = NULL; + break; + } + dev_info(ss->dev, "DEBUG: Register %s\n", + ss_algs[i].alg.skcipher.base.base.cra_name); + err = crypto_engine_register_skcipher(&ss_algs[i].alg.skcipher); + if (err) { + dev_err(ss->dev, "Fail to register %s\n", + ss_algs[i].alg.skcipher.base.base.cra_name); + ss_algs[i].ss = NULL; + return err; + } + break; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_HASH + case CRYPTO_ALG_TYPE_AHASH: + id = ss_algs[i].ss_algo_id; + ss_method = ss->variant->alg_hash[id]; + if (ss_method == SS_ID_NOTSUPP) { + dev_info(ss->dev, + "DEBUG: Algo of %s not supported\n", + ss_algs[i].alg.hash.base.halg.base.cra_name); + ss_algs[i].ss = NULL; + break; + } + dev_info(ss->dev, "Register %s\n", + ss_algs[i].alg.hash.base.halg.base.cra_name); + err = crypto_engine_register_ahash(&ss_algs[i].alg.hash); + if (err) { + dev_err(ss->dev, "ERROR: Fail to register %s\n", + ss_algs[i].alg.hash.base.halg.base.cra_name); + ss_algs[i].ss = NULL; + return err; + } + break; +#endif + default: + ss_algs[i].ss = NULL; + dev_err(ss->dev, "ERROR: tried to register an unknown algo\n"); + } + } + return 0; +} + +static void sun8i_ss_unregister_algs(struct sun8i_ss_dev *ss) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(ss_algs); i++) { + if (!ss_algs[i].ss) + continue; + switch (ss_algs[i].type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + dev_info(ss->dev, "Unregister %d %s\n", i, + ss_algs[i].alg.skcipher.base.base.cra_name); + crypto_engine_unregister_skcipher(&ss_algs[i].alg.skcipher); + break; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_HASH + case CRYPTO_ALG_TYPE_AHASH: + dev_info(ss->dev, "Unregister %d %s\n", i, + ss_algs[i].alg.hash.base.halg.base.cra_name); + crypto_engine_unregister_ahash(&ss_algs[i].alg.hash); + break; +#endif + } + } +} + +static int sun8i_ss_get_clks(struct sun8i_ss_dev *ss) +{ + unsigned long cr; + int err, i; + + for (i = 0; i < SS_MAX_CLOCKS; i++) { + if (!ss->variant->ss_clks[i].name) + continue; + ss->ssclks[i] = devm_clk_get(ss->dev, ss->variant->ss_clks[i].name); + if (IS_ERR(ss->ssclks[i])) { + err = PTR_ERR(ss->ssclks[i]); + dev_err(ss->dev, "Cannot get %s SS clock err=%d\n", + ss->variant->ss_clks[i].name, err); + return err; + } + cr = clk_get_rate(ss->ssclks[i]); + if (!cr) + return -EINVAL; + if (ss->variant->ss_clks[i].freq > 0 && + cr != ss->variant->ss_clks[i].freq) { + dev_info(ss->dev, "Set %s clock to %lu (%lu Mhz) from %lu (%lu Mhz)\n", + ss->variant->ss_clks[i].name, + ss->variant->ss_clks[i].freq, + ss->variant->ss_clks[i].freq / 1000000, + cr, cr / 1000000); + err = clk_set_rate(ss->ssclks[i], ss->variant->ss_clks[i].freq); + if (err) + dev_err(ss->dev, "Fail to set %s clk speed to %lu hz\n", + ss->variant->ss_clks[i].name, + ss->variant->ss_clks[i].freq); + } + if (ss->variant->ss_clks[i].max_freq > 0 && + cr > ss->variant->ss_clks[i].max_freq) + dev_warn(ss->dev, "Frequency for %s (%lu hz) is higher than datasheet's recommendation (%lu hz)", + ss->variant->ss_clks[i].name, cr, + ss->variant->ss_clks[i].max_freq); + } + return 0; +} + +static int sun8i_ss_probe(struct platform_device *pdev) +{ + struct sun8i_ss_dev *ss; + int err, irq; + u32 v; + + ss = devm_kzalloc(&pdev->dev, sizeof(*ss), GFP_KERNEL); + if (!ss) + return -ENOMEM; + + ss->dev = &pdev->dev; + platform_set_drvdata(pdev, ss); + + ss->variant = of_device_get_match_data(&pdev->dev); + if (!ss->variant) { + dev_err(&pdev->dev, "Missing Crypto Engine variant\n"); + return -EINVAL; + } + + ss->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(ss->base)) + return PTR_ERR(ss->base); + + err = sun8i_ss_get_clks(ss); + if (err) + return err; + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + ss->reset = devm_reset_control_get(&pdev->dev, NULL); + if (IS_ERR(ss->reset)) + return dev_err_probe(&pdev->dev, PTR_ERR(ss->reset), + "No reset control found\n"); + + mutex_init(&ss->mlock); + + err = allocate_flows(ss); + if (err) + return err; + + err = sun8i_ss_pm_init(ss); + if (err) + goto error_pm; + + err = devm_request_irq(&pdev->dev, irq, ss_irq_handler, 0, "sun8i-ss", ss); + if (err) + goto error_irq; + + err = sun8i_ss_register_algs(ss); + if (err) + goto error_alg; + + err = pm_runtime_resume_and_get(ss->dev); + if (err < 0) + goto error_alg; + + v = readl(ss->base + SS_CTL_REG); + v >>= SS_DIE_ID_SHIFT; + v &= SS_DIE_ID_MASK; + dev_info(&pdev->dev, "Security System Die ID %x\n", v); + + pm_runtime_put_sync(ss->dev); + + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG)) { + struct dentry *dbgfs_dir __maybe_unused; + struct dentry *dbgfs_stats __maybe_unused; + + /* Ignore error of debugfs */ + dbgfs_dir = debugfs_create_dir("sun8i-ss", NULL); + dbgfs_stats = debugfs_create_file("stats", 0444, + dbgfs_dir, ss, + &sun8i_ss_debugfs_fops); + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + ss->dbgfs_dir = dbgfs_dir; + ss->dbgfs_stats = dbgfs_stats; +#endif + } + + return 0; +error_alg: + sun8i_ss_unregister_algs(ss); +error_irq: + sun8i_ss_pm_exit(ss); +error_pm: + sun8i_ss_free_flows(ss, MAXFLOW - 1); + return err; +} + +static void sun8i_ss_remove(struct platform_device *pdev) +{ + struct sun8i_ss_dev *ss = platform_get_drvdata(pdev); + + sun8i_ss_unregister_algs(ss); + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + debugfs_remove_recursive(ss->dbgfs_dir); +#endif + + sun8i_ss_free_flows(ss, MAXFLOW - 1); + + sun8i_ss_pm_exit(ss); +} + +static const struct of_device_id sun8i_ss_crypto_of_match_table[] = { + { .compatible = "allwinner,sun8i-a83t-crypto", + .data = &ss_a83t_variant }, + { .compatible = "allwinner,sun9i-a80-crypto", + .data = &ss_a80_variant }, + {} +}; +MODULE_DEVICE_TABLE(of, sun8i_ss_crypto_of_match_table); + +static struct platform_driver sun8i_ss_driver = { + .probe = sun8i_ss_probe, + .remove = sun8i_ss_remove, + .driver = { + .name = "sun8i-ss", + .pm = &sun8i_ss_pm_ops, + .of_match_table = sun8i_ss_crypto_of_match_table, + }, +}; + +module_platform_driver(sun8i_ss_driver); + +MODULE_DESCRIPTION("Allwinner SecuritySystem cryptographic offloader"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Corentin Labbe "); diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c new file mode 100644 index 000000000..36a1ebca2 --- /dev/null +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c @@ -0,0 +1,705 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sun8i-ss-hash.c - hardware cryptographic offloader for + * Allwinner A80/A83T SoC + * + * Copyright (C) 2015-2020 Corentin Labbe + * + * This file add support for MD5 and SHA1/SHA224/SHA256. + * + * You could find the datasheet in Documentation/arch/arm/sunxi.rst + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "sun8i-ss.h" + +static int sun8i_ss_hashkey(struct sun8i_ss_hash_tfm_ctx *tfmctx, const u8 *key, + unsigned int keylen) +{ + struct crypto_shash *xtfm; + int ret; + + xtfm = crypto_alloc_shash("sha1", 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(xtfm)) + return PTR_ERR(xtfm); + + ret = crypto_shash_tfm_digest(xtfm, key, keylen, tfmctx->key); + if (ret) + dev_err(tfmctx->ss->dev, "shash digest error ret=%d\n", ret); + + crypto_free_shash(xtfm); + return ret; +} + +int sun8i_ss_hmac_setkey(struct crypto_ahash *ahash, const u8 *key, + unsigned int keylen) +{ + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(ahash); + int digestsize, i; + int bs = crypto_ahash_blocksize(ahash); + int ret; + + digestsize = crypto_ahash_digestsize(ahash); + + if (keylen > bs) { + ret = sun8i_ss_hashkey(tfmctx, key, keylen); + if (ret) + return ret; + tfmctx->keylen = digestsize; + } else { + tfmctx->keylen = keylen; + memcpy(tfmctx->key, key, keylen); + } + + tfmctx->ipad = kzalloc(bs, GFP_KERNEL); + if (!tfmctx->ipad) + return -ENOMEM; + tfmctx->opad = kzalloc(bs, GFP_KERNEL); + if (!tfmctx->opad) { + ret = -ENOMEM; + goto err_opad; + } + + memset(tfmctx->key + tfmctx->keylen, 0, bs - tfmctx->keylen); + memcpy(tfmctx->ipad, tfmctx->key, tfmctx->keylen); + memcpy(tfmctx->opad, tfmctx->key, tfmctx->keylen); + for (i = 0; i < bs; i++) { + tfmctx->ipad[i] ^= HMAC_IPAD_VALUE; + tfmctx->opad[i] ^= HMAC_OPAD_VALUE; + } + + ret = crypto_ahash_setkey(tfmctx->fallback_tfm, key, keylen); + if (!ret) + return 0; + + memzero_explicit(tfmctx->key, keylen); + kfree_sensitive(tfmctx->opad); +err_opad: + kfree_sensitive(tfmctx->ipad); + return ret; +} + +int sun8i_ss_hash_init_tfm(struct crypto_ahash *tfm) +{ + struct sun8i_ss_hash_tfm_ctx *op = crypto_ahash_ctx(tfm); + struct ahash_alg *alg = crypto_ahash_alg(tfm); + struct sun8i_ss_alg_template *algt; + int err; + + algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash.base); + op->ss = algt->ss; + + /* FALLBACK */ + op->fallback_tfm = crypto_alloc_ahash(crypto_ahash_alg_name(tfm), 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(op->fallback_tfm)) { + dev_err(algt->ss->dev, "Fallback driver could no be loaded\n"); + return PTR_ERR(op->fallback_tfm); + } + + crypto_ahash_set_statesize(tfm, + crypto_ahash_statesize(op->fallback_tfm)); + + crypto_ahash_set_reqsize(tfm, + sizeof(struct sun8i_ss_hash_reqctx) + + crypto_ahash_reqsize(op->fallback_tfm)); + + memcpy(algt->fbname, crypto_ahash_driver_name(op->fallback_tfm), + CRYPTO_MAX_ALG_NAME); + + err = pm_runtime_get_sync(op->ss->dev); + if (err < 0) + goto error_pm; + return 0; +error_pm: + pm_runtime_put_noidle(op->ss->dev); + crypto_free_ahash(op->fallback_tfm); + return err; +} + +void sun8i_ss_hash_exit_tfm(struct crypto_ahash *tfm) +{ + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + kfree_sensitive(tfmctx->ipad); + kfree_sensitive(tfmctx->opad); + + crypto_free_ahash(tfmctx->fallback_tfm); + pm_runtime_put_sync_suspend(tfmctx->ss->dev); +} + +int sun8i_ss_hash_init(struct ahash_request *areq) +{ + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + memset(rctx, 0, sizeof(struct sun8i_ss_hash_reqctx)); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + + return crypto_ahash_init(&rctx->fallback_req); +} + +int sun8i_ss_hash_export(struct ahash_request *areq, void *out) +{ + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + + return crypto_ahash_export(&rctx->fallback_req, out); +} + +int sun8i_ss_hash_import(struct ahash_request *areq, const void *in) +{ + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + + return crypto_ahash_import(&rctx->fallback_req, in); +} + +int sun8i_ss_hash_final(struct ahash_request *areq) +{ + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, NULL, areq->result, 0); + + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG)) { + struct ahash_alg *alg = crypto_ahash_alg(tfm); + struct sun8i_ss_alg_template *algt __maybe_unused; + + algt = container_of(alg, struct sun8i_ss_alg_template, + alg.hash.base); + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + algt->stat_fb++; +#endif + } + + return crypto_ahash_final(&rctx->fallback_req); +} + +int sun8i_ss_hash_update(struct ahash_request *areq) +{ + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, NULL, areq->nbytes); + + return crypto_ahash_update(&rctx->fallback_req); +} + +int sun8i_ss_hash_finup(struct ahash_request *areq) +{ + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, areq->result, + areq->nbytes); + + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG)) { + struct ahash_alg *alg = crypto_ahash_alg(tfm); + struct sun8i_ss_alg_template *algt __maybe_unused; + + algt = container_of(alg, struct sun8i_ss_alg_template, + alg.hash.base); + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + algt->stat_fb++; +#endif + } + + return crypto_ahash_finup(&rctx->fallback_req); +} + +static int sun8i_ss_hash_digest_fb(struct ahash_request *areq) +{ + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, areq->result, + areq->nbytes); + + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG)) { + struct ahash_alg *alg = crypto_ahash_alg(tfm); + struct sun8i_ss_alg_template *algt __maybe_unused; + + algt = container_of(alg, struct sun8i_ss_alg_template, + alg.hash.base); + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + algt->stat_fb++; +#endif + } + + return crypto_ahash_digest(&rctx->fallback_req); +} + +static int sun8i_ss_run_hash_task(struct sun8i_ss_dev *ss, + struct sun8i_ss_hash_reqctx *rctx, + const char *name) +{ + int flow = rctx->flow; + u32 v = SS_START; + int i; + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + ss->flows[flow].stat_req++; +#endif + + /* choose between stream0/stream1 */ + if (flow) + v |= SS_FLOW1; + else + v |= SS_FLOW0; + + v |= rctx->method; + + for (i = 0; i < MAX_SG; i++) { + if (!rctx->t_dst[i].addr) + break; + + mutex_lock(&ss->mlock); + if (i > 0) { + v |= BIT(17); + writel(rctx->t_dst[i - 1].addr, ss->base + SS_KEY_ADR_REG); + writel(rctx->t_dst[i - 1].addr, ss->base + SS_IV_ADR_REG); + } + + dev_dbg(ss->dev, + "Processing SG %d on flow %d %s ctl=%x %d to %d method=%x src=%x dst=%x\n", + i, flow, name, v, + rctx->t_src[i].len, rctx->t_dst[i].len, + rctx->method, rctx->t_src[i].addr, rctx->t_dst[i].addr); + + writel(rctx->t_src[i].addr, ss->base + SS_SRC_ADR_REG); + writel(rctx->t_dst[i].addr, ss->base + SS_DST_ADR_REG); + writel(rctx->t_src[i].len, ss->base + SS_LEN_ADR_REG); + writel(BIT(0) | BIT(1), ss->base + SS_INT_CTL_REG); + + reinit_completion(&ss->flows[flow].complete); + ss->flows[flow].status = 0; + wmb(); + + writel(v, ss->base + SS_CTL_REG); + mutex_unlock(&ss->mlock); + wait_for_completion_interruptible_timeout(&ss->flows[flow].complete, + msecs_to_jiffies(2000)); + if (ss->flows[flow].status == 0) { + dev_err(ss->dev, "DMA timeout for %s\n", name); + return -EFAULT; + } + } + + return 0; +} + +static bool sun8i_ss_hash_need_fallback(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct ahash_alg *alg = crypto_ahash_alg(tfm); + struct sun8i_ss_alg_template *algt; + struct scatterlist *sg; + + algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash.base); + + if (areq->nbytes == 0) { + algt->stat_fb_len++; + return true; + } + + if (areq->nbytes >= MAX_PAD_SIZE - 64) { + algt->stat_fb_len++; + return true; + } + + /* we need to reserve one SG for the padding one */ + if (sg_nents(areq->src) > MAX_SG - 1) { + algt->stat_fb_sgnum++; + return true; + } + + sg = areq->src; + while (sg) { + /* SS can operate hash only on full block size + * since SS support only MD5,sha1,sha224 and sha256, blocksize + * is always 64 + */ + /* Only the last block could be bounced to the pad buffer */ + if (sg->length % 64 && sg_next(sg)) { + algt->stat_fb_sglen++; + return true; + } + if (!IS_ALIGNED(sg->offset, sizeof(u32))) { + algt->stat_fb_align++; + return true; + } + if (sg->length % 4) { + algt->stat_fb_sglen++; + return true; + } + sg = sg_next(sg); + } + return false; +} + +int sun8i_ss_hash_digest(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct ahash_alg *alg = crypto_ahash_alg(tfm); + struct sun8i_ss_alg_template *algt; + struct sun8i_ss_dev *ss; + struct crypto_engine *engine; + int e; + + if (sun8i_ss_hash_need_fallback(areq)) + return sun8i_ss_hash_digest_fb(areq); + + algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash.base); + ss = algt->ss; + + e = sun8i_ss_get_engine_number(ss); + rctx->flow = e; + engine = ss->flows[e].engine; + + return crypto_transfer_hash_request_to_engine(engine, areq); +} + +static u64 hash_pad(__le32 *buf, unsigned int bufsize, u64 padi, u64 byte_count, bool le, int bs) +{ + u64 fill, min_fill, j, k; + __be64 *bebits; + __le64 *lebits; + + j = padi; + buf[j++] = cpu_to_le32(0x80); + + if (bs == 64) { + fill = 64 - (byte_count % 64); + min_fill = 2 * sizeof(u32) + sizeof(u32); + } else { + fill = 128 - (byte_count % 128); + min_fill = 4 * sizeof(u32) + sizeof(u32); + } + + if (fill < min_fill) + fill += bs; + + k = j; + j += (fill - min_fill) / sizeof(u32); + if (j * 4 > bufsize) { + pr_err("%s OVERFLOW %llu\n", __func__, j); + return 0; + } + for (; k < j; k++) + buf[k] = 0; + + if (le) { + /* MD5 */ + lebits = (__le64 *)&buf[j]; + *lebits = cpu_to_le64(byte_count << 3); + j += 2; + } else { + if (bs == 64) { + /* sha1 sha224 sha256 */ + bebits = (__be64 *)&buf[j]; + *bebits = cpu_to_be64(byte_count << 3); + j += 2; + } else { + /* sha384 sha512*/ + bebits = (__be64 *)&buf[j]; + *bebits = cpu_to_be64(byte_count >> 61); + j += 2; + bebits = (__be64 *)&buf[j]; + *bebits = cpu_to_be64(byte_count << 3); + j += 2; + } + } + if (j * 4 > bufsize) { + pr_err("%s OVERFLOW %llu\n", __func__, j); + return 0; + } + + return j; +} + +/* sun8i_ss_hash_run - run an ahash request + * Send the data of the request to the SS along with an extra SG with padding + */ +int sun8i_ss_hash_run(struct crypto_engine *engine, void *breq) +{ + struct ahash_request *areq = container_of(breq, struct ahash_request, base); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); + struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); + struct ahash_alg *alg = crypto_ahash_alg(tfm); + struct sun8i_ss_alg_template *algt; + struct sun8i_ss_dev *ss; + struct scatterlist *sg; + int bs = crypto_ahash_blocksize(tfm); + int nr_sgs, err, digestsize; + unsigned int len; + u64 byte_count; + void *pad, *result; + int j, i, k, todo; + dma_addr_t addr_res, addr_pad, addr_xpad; + __le32 *bf; + /* HMAC step: + * 0: normal hashing + * 1: IPAD + * 2: OPAD + */ + int hmac = 0; + + algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash.base); + ss = algt->ss; + j = 0; + + digestsize = crypto_ahash_digestsize(tfm); + if (digestsize == SHA224_DIGEST_SIZE) + digestsize = SHA256_DIGEST_SIZE; + + result = ss->flows[rctx->flow].result; + pad = ss->flows[rctx->flow].pad; + bf = (__le32 *)pad; + + for (i = 0; i < MAX_SG; i++) { + rctx->t_dst[i].addr = 0; + rctx->t_dst[i].len = 0; + } + +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + algt->stat_req++; +#endif + + rctx->method = ss->variant->alg_hash[algt->ss_algo_id]; + + nr_sgs = dma_map_sg(ss->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE); + if (nr_sgs <= 0 || nr_sgs > MAX_SG) { + dev_err(ss->dev, "Invalid sg number %d\n", nr_sgs); + err = -EINVAL; + goto theend; + } + + addr_res = dma_map_single(ss->dev, result, digestsize, DMA_FROM_DEVICE); + if (dma_mapping_error(ss->dev, addr_res)) { + dev_err(ss->dev, "DMA map dest\n"); + err = -EINVAL; + goto err_dma_result; + } + + len = areq->nbytes; + sg = areq->src; + i = 0; + while (len > 0 && sg) { + if (sg_dma_len(sg) == 0) { + sg = sg_next(sg); + continue; + } + todo = min(len, sg_dma_len(sg)); + /* only the last SG could be with a size not modulo64 */ + if (todo % 64 == 0) { + rctx->t_src[i].addr = sg_dma_address(sg); + rctx->t_src[i].len = todo / 4; + rctx->t_dst[i].addr = addr_res; + rctx->t_dst[i].len = digestsize / 4; + len -= todo; + } else { + scatterwalk_map_and_copy(bf, sg, 0, todo, 0); + j += todo / 4; + len -= todo; + } + sg = sg_next(sg); + i++; + } + if (len > 0) { + dev_err(ss->dev, "remaining len %d\n", len); + err = -EINVAL; + goto theend; + } + + if (j > 0) + i--; + +retry: + byte_count = areq->nbytes; + if (tfmctx->keylen && hmac == 0) { + hmac = 1; + /* shift all SG one slot up, to free slot 0 for IPAD */ + for (k = 6; k >= 0; k--) { + rctx->t_src[k + 1].addr = rctx->t_src[k].addr; + rctx->t_src[k + 1].len = rctx->t_src[k].len; + rctx->t_dst[k + 1].addr = rctx->t_dst[k].addr; + rctx->t_dst[k + 1].len = rctx->t_dst[k].len; + } + addr_xpad = dma_map_single(ss->dev, tfmctx->ipad, bs, DMA_TO_DEVICE); + err = dma_mapping_error(ss->dev, addr_xpad); + if (err) { + dev_err(ss->dev, "Fail to create DMA mapping of ipad\n"); + goto err_dma_xpad; + } + rctx->t_src[0].addr = addr_xpad; + rctx->t_src[0].len = bs / 4; + rctx->t_dst[0].addr = addr_res; + rctx->t_dst[0].len = digestsize / 4; + i++; + byte_count = areq->nbytes + bs; + } + if (tfmctx->keylen && hmac == 2) { + for (i = 0; i < MAX_SG; i++) { + rctx->t_src[i].addr = 0; + rctx->t_src[i].len = 0; + rctx->t_dst[i].addr = 0; + rctx->t_dst[i].len = 0; + } + + addr_res = dma_map_single(ss->dev, result, digestsize, DMA_FROM_DEVICE); + if (dma_mapping_error(ss->dev, addr_res)) { + dev_err(ss->dev, "Fail to create DMA mapping of result\n"); + err = -EINVAL; + goto err_dma_result; + } + addr_xpad = dma_map_single(ss->dev, tfmctx->opad, bs, DMA_TO_DEVICE); + err = dma_mapping_error(ss->dev, addr_xpad); + if (err) { + dev_err(ss->dev, "Fail to create DMA mapping of opad\n"); + goto err_dma_xpad; + } + rctx->t_src[0].addr = addr_xpad; + rctx->t_src[0].len = bs / 4; + + memcpy(bf, result, digestsize); + j = digestsize / 4; + i = 1; + byte_count = digestsize + bs; + + rctx->t_dst[0].addr = addr_res; + rctx->t_dst[0].len = digestsize / 4; + } + + switch (algt->ss_algo_id) { + case SS_ID_HASH_MD5: + j = hash_pad(bf, 4096, j, byte_count, true, bs); + break; + case SS_ID_HASH_SHA1: + case SS_ID_HASH_SHA224: + case SS_ID_HASH_SHA256: + j = hash_pad(bf, 4096, j, byte_count, false, bs); + break; + } + if (!j) { + err = -EINVAL; + goto theend; + } + + addr_pad = dma_map_single(ss->dev, pad, j * 4, DMA_TO_DEVICE); + if (dma_mapping_error(ss->dev, addr_pad)) { + dev_err(ss->dev, "DMA error on padding SG\n"); + err = -EINVAL; + goto err_dma_pad; + } + rctx->t_src[i].addr = addr_pad; + rctx->t_src[i].len = j; + rctx->t_dst[i].addr = addr_res; + rctx->t_dst[i].len = digestsize / 4; + + err = sun8i_ss_run_hash_task(ss, rctx, crypto_tfm_alg_name(areq->base.tfm)); + + /* + * mini helper for checking dma map/unmap + * flow start for hmac = 0 (and HMAC = 1) + * HMAC = 0 + * MAP src + * MAP res + * + * retry: + * if hmac then hmac = 1 + * MAP xpad (ipad) + * if hmac == 2 + * MAP res + * MAP xpad (opad) + * MAP pad + * ACTION! + * UNMAP pad + * if hmac + * UNMAP xpad + * UNMAP res + * if hmac < 2 + * UNMAP SRC + * + * if hmac = 1 then hmac = 2 goto retry + */ + + dma_unmap_single(ss->dev, addr_pad, j * 4, DMA_TO_DEVICE); + +err_dma_pad: + if (hmac > 0) + dma_unmap_single(ss->dev, addr_xpad, bs, DMA_TO_DEVICE); +err_dma_xpad: + dma_unmap_single(ss->dev, addr_res, digestsize, DMA_FROM_DEVICE); +err_dma_result: + if (hmac < 2) + dma_unmap_sg(ss->dev, areq->src, sg_nents(areq->src), + DMA_TO_DEVICE); + if (hmac == 1 && !err) { + hmac = 2; + goto retry; + } + + if (!err) + memcpy(areq->result, result, crypto_ahash_digestsize(tfm)); +theend: + local_bh_disable(); + crypto_finalize_hash_request(engine, breq, err); + local_bh_enable(); + return 0; +} diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h new file mode 100644 index 000000000..289fb22ab --- /dev/null +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h @@ -0,0 +1,297 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * sun8i-ss.h - hardware cryptographic offloader for + * Allwinner A80/A83T SoC + * + * Copyright (C) 2016-2019 Corentin LABBE + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SS_START 1 + +#define SS_ENCRYPTION 0 +#define SS_DECRYPTION BIT(6) + +#define SS_ALG_AES 0 +#define SS_ALG_DES (1 << 2) +#define SS_ALG_3DES (2 << 2) +#define SS_ALG_MD5 (3 << 2) +#define SS_ALG_SHA1 (6 << 2) +#define SS_ALG_SHA224 (7 << 2) +#define SS_ALG_SHA256 (8 << 2) + +#define SS_CTL_REG 0x00 +#define SS_INT_CTL_REG 0x04 +#define SS_INT_STA_REG 0x08 +#define SS_KEY_ADR_REG 0x10 +#define SS_IV_ADR_REG 0x18 +#define SS_SRC_ADR_REG 0x20 +#define SS_DST_ADR_REG 0x28 +#define SS_LEN_ADR_REG 0x30 + +#define SS_ID_NOTSUPP 0xFF + +#define SS_ID_CIPHER_AES 0 +#define SS_ID_CIPHER_DES 1 +#define SS_ID_CIPHER_DES3 2 +#define SS_ID_CIPHER_MAX 3 + +#define SS_ID_OP_ECB 0 +#define SS_ID_OP_CBC 1 +#define SS_ID_OP_MAX 2 + +#define SS_AES_128BITS 0 +#define SS_AES_192BITS 1 +#define SS_AES_256BITS 2 + +#define SS_OP_ECB 0 +#define SS_OP_CBC (1 << 13) + +#define SS_ID_HASH_MD5 0 +#define SS_ID_HASH_SHA1 1 +#define SS_ID_HASH_SHA224 2 +#define SS_ID_HASH_SHA256 3 +#define SS_ID_HASH_MAX 4 + +#define SS_FLOW0 BIT(30) +#define SS_FLOW1 BIT(31) + +#define MAX_SG 8 + +#define MAXFLOW 2 + +#define SS_MAX_CLOCKS 2 + +#define SS_DIE_ID_SHIFT 20 +#define SS_DIE_ID_MASK 0x07 + +#define MAX_PAD_SIZE 4096 + +/* + * struct ss_clock - Describe clocks used by sun8i-ss + * @name: Name of clock needed by this variant + * @freq: Frequency to set for each clock + * @max_freq: Maximum frequency for each clock + */ +struct ss_clock { + const char *name; + unsigned long freq; + unsigned long max_freq; +}; + +/* + * struct ss_variant - Describe SS capability for each variant hardware + * @alg_cipher: list of supported ciphers. for each SS_ID_ this will give the + * coresponding SS_ALG_XXX value + * @alg_hash: list of supported hashes. for each SS_ID_ this will give the + * corresponding SS_ALG_XXX value + * @op_mode: list of supported block modes + * @ss_clks: list of clock needed by this variant + */ +struct ss_variant { + char alg_cipher[SS_ID_CIPHER_MAX]; + char alg_hash[SS_ID_HASH_MAX]; + u32 op_mode[SS_ID_OP_MAX]; + struct ss_clock ss_clks[SS_MAX_CLOCKS]; +}; + +struct sginfo { + u32 addr; + u32 len; +}; + +/* + * struct sun8i_ss_flow - Information used by each flow + * @engine: ptr to the crypto_engine for this flow + * @complete: completion for the current task on this flow + * @status: set to 1 by interrupt if task is done + * @stat_req: number of request done by this flow + * @iv: list of IV to use for each step + * @biv: buffer which contain the backuped IV + * @pad: padding buffer for hash operations + * @result: buffer for storing the result of hash operations + */ +struct sun8i_ss_flow { + struct crypto_engine *engine; + struct completion complete; + int status; + u8 *iv[MAX_SG]; + u8 *biv; + void *pad; + void *result; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + unsigned long stat_req; +#endif +}; + +/* + * struct sun8i_ss_dev - main container for all this driver information + * @base: base address of SS + * @ssclks: clocks used by SS + * @reset: pointer to reset controller + * @dev: the platform device + * @mlock: Control access to device registers + * @flows: array of all flow + * @flow: flow to use in next request + * @variant: pointer to variant specific data + * @dbgfs_dir: Debugfs dentry for statistic directory + * @dbgfs_stats: Debugfs dentry for statistic counters + */ +struct sun8i_ss_dev { + void __iomem *base; + struct clk *ssclks[SS_MAX_CLOCKS]; + struct reset_control *reset; + struct device *dev; + struct mutex mlock; + struct sun8i_ss_flow *flows; + atomic_t flow; + const struct ss_variant *variant; +#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG + struct dentry *dbgfs_dir; + struct dentry *dbgfs_stats; +#endif +}; + +/* + * struct sun8i_cipher_req_ctx - context for a skcipher request + * @t_src: list of mapped SGs with their size + * @t_dst: list of mapped SGs with their size + * @p_key: DMA address of the key + * @p_iv: DMA address of the IVs + * @niv: Number of IVs DMA mapped + * @method: current algorithm for this request + * @op_mode: op_mode for this request + * @op_dir: direction (encrypt vs decrypt) for this request + * @flow: the flow to use for this request + * @ivlen: size of IVs + * @keylen: keylen for this request + * @fallback_req: request struct for invoking the fallback skcipher TFM + */ +struct sun8i_cipher_req_ctx { + struct sginfo t_src[MAX_SG]; + struct sginfo t_dst[MAX_SG]; + u32 p_key; + u32 p_iv[MAX_SG]; + int niv; + u32 method; + u32 op_mode; + u32 op_dir; + int flow; + unsigned int ivlen; + unsigned int keylen; + struct skcipher_request fallback_req; // keep at the end +}; + +/* + * struct sun8i_cipher_tfm_ctx - context for a skcipher TFM + * @key: pointer to key data + * @keylen: len of the key + * @ss: pointer to the private data of driver handling this TFM + * @fallback_tfm: pointer to the fallback TFM + */ +struct sun8i_cipher_tfm_ctx { + u32 *key; + u32 keylen; + struct sun8i_ss_dev *ss; + struct crypto_skcipher *fallback_tfm; +}; + +/* + * struct sun8i_ss_hash_tfm_ctx - context for an ahash TFM + * @fallback_tfm: pointer to the fallback TFM + * @ss: pointer to the private data of driver handling this TFM + */ +struct sun8i_ss_hash_tfm_ctx { + struct crypto_ahash *fallback_tfm; + struct sun8i_ss_dev *ss; + u8 *ipad; + u8 *opad; + u8 key[SHA256_BLOCK_SIZE]; + int keylen; +}; + +/* + * struct sun8i_ss_hash_reqctx - context for an ahash request + * @t_src: list of DMA address and size for source SGs + * @t_dst: list of DMA address and size for destination SGs + * @fallback_req: pre-allocated fallback request + * @method: the register value for the algorithm used by this request + * @flow: the flow to use for this request + */ +struct sun8i_ss_hash_reqctx { + struct sginfo t_src[MAX_SG]; + struct sginfo t_dst[MAX_SG]; + u32 method; + int flow; + + /* Must be last as it ends in a flexible-array member. */ + struct ahash_request fallback_req; +}; + +/* + * struct sun8i_ss_alg_template - crypto_alg template + * @type: the CRYPTO_ALG_TYPE for this template + * @ss_algo_id: the SS_ID for this template + * @ss_blockmode: the type of block operation SS_ID + * @ss: pointer to the sun8i_ss_dev structure associated with + * this template + * @alg: one of sub struct must be used + * @stat_req: number of request done on this template + * @stat_fb: number of request which has fallbacked + * @stat_bytes: total data size done by this template + */ +struct sun8i_ss_alg_template { + u32 type; + u32 ss_algo_id; + u32 ss_blockmode; + struct sun8i_ss_dev *ss; + union { + struct skcipher_engine_alg skcipher; + struct ahash_engine_alg hash; + } alg; + unsigned long stat_req; + unsigned long stat_fb; + unsigned long stat_bytes; + unsigned long stat_fb_len; + unsigned long stat_fb_sglen; + unsigned long stat_fb_align; + unsigned long stat_fb_sgnum; + char fbname[CRYPTO_MAX_ALG_NAME]; +}; + +int sun8i_ss_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen); +int sun8i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen); +int sun8i_ss_cipher_init(struct crypto_tfm *tfm); +void sun8i_ss_cipher_exit(struct crypto_tfm *tfm); +int sun8i_ss_handle_cipher_request(struct crypto_engine *engine, void *areq); +int sun8i_ss_skdecrypt(struct skcipher_request *areq); +int sun8i_ss_skencrypt(struct skcipher_request *areq); + +int sun8i_ss_get_engine_number(struct sun8i_ss_dev *ss); + +int sun8i_ss_run_task(struct sun8i_ss_dev *ss, struct sun8i_cipher_req_ctx *rctx, const char *name); + +int sun8i_ss_hash_init_tfm(struct crypto_ahash *tfm); +void sun8i_ss_hash_exit_tfm(struct crypto_ahash *tfm); +int sun8i_ss_hash_init(struct ahash_request *areq); +int sun8i_ss_hash_export(struct ahash_request *areq, void *out); +int sun8i_ss_hash_import(struct ahash_request *areq, const void *in); +int sun8i_ss_hash_final(struct ahash_request *areq); +int sun8i_ss_hash_update(struct ahash_request *areq); +int sun8i_ss_hash_finup(struct ahash_request *areq); +int sun8i_ss_hash_digest(struct ahash_request *areq); +int sun8i_ss_hash_run(struct crypto_engine *engine, void *breq); +int sun8i_ss_hmac_setkey(struct crypto_ahash *ahash, const u8 *key, + unsigned int keylen); diff --git a/drivers/crypto/amcc/Makefile b/drivers/crypto/amcc/Makefile new file mode 100644 index 000000000..d66239913 --- /dev/null +++ b/drivers/crypto/amcc/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += crypto4xx.o +crypto4xx-y := crypto4xx_core.o crypto4xx_alg.o +crypto4xx-$(CONFIG_HW_RANDOM_PPC4XX) += crypto4xx_trng.o diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c new file mode 100644 index 000000000..3177dc4f5 --- /dev/null +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -0,0 +1,601 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * AMCC SoC PPC4xx Crypto Driver + * + * Copyright (c) 2008 Applied Micro Circuits Corporation. + * All rights reserved. James Hsiao + * + * This file implements the Linux crypto algorithms. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "crypto4xx_reg_def.h" +#include "crypto4xx_core.h" +#include "crypto4xx_sa.h" + +static void set_dynamic_sa_command_0(struct dynamic_sa_ctl *sa, u32 save_h, + u32 save_iv, u32 ld_h, u32 ld_iv, + u32 hdr_proc, u32 h, u32 c, u32 pad_type, + u32 op_grp, u32 op, u32 dir) +{ + sa->sa_command_0.w = 0; + sa->sa_command_0.bf.save_hash_state = save_h; + sa->sa_command_0.bf.save_iv = save_iv; + sa->sa_command_0.bf.load_hash_state = ld_h; + sa->sa_command_0.bf.load_iv = ld_iv; + sa->sa_command_0.bf.hdr_proc = hdr_proc; + sa->sa_command_0.bf.hash_alg = h; + sa->sa_command_0.bf.cipher_alg = c; + sa->sa_command_0.bf.pad_type = pad_type & 3; + sa->sa_command_0.bf.extend_pad = pad_type >> 2; + sa->sa_command_0.bf.op_group = op_grp; + sa->sa_command_0.bf.opcode = op; + sa->sa_command_0.bf.dir = dir; +} + +static void set_dynamic_sa_command_1(struct dynamic_sa_ctl *sa, u32 cm, + u32 hmac_mc, u32 cfb, u32 esn, + u32 sn_mask, u32 mute, u32 cp_pad, + u32 cp_pay, u32 cp_hdr) +{ + sa->sa_command_1.w = 0; + sa->sa_command_1.bf.crypto_mode31 = (cm & 4) >> 2; + sa->sa_command_1.bf.crypto_mode9_8 = cm & 3; + sa->sa_command_1.bf.feedback_mode = cfb; + sa->sa_command_1.bf.sa_rev = 1; + sa->sa_command_1.bf.hmac_muting = hmac_mc; + sa->sa_command_1.bf.extended_seq_num = esn; + sa->sa_command_1.bf.seq_num_mask = sn_mask; + sa->sa_command_1.bf.mutable_bit_proc = mute; + sa->sa_command_1.bf.copy_pad = cp_pad; + sa->sa_command_1.bf.copy_payload = cp_pay; + sa->sa_command_1.bf.copy_hdr = cp_hdr; +} + +static inline int crypto4xx_crypt(struct skcipher_request *req, + const unsigned int ivlen, bool decrypt, + bool check_blocksize) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher); + __le32 iv[AES_IV_SIZE / 4]; + + if (check_blocksize && !IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE)) + return -EINVAL; + + if (ivlen) + crypto4xx_memcpy_to_le32(iv, req->iv, ivlen); + + return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, + req->cryptlen, iv, ivlen, decrypt ? ctx->sa_in : ctx->sa_out, + ctx->sa_len, 0, NULL); +} + +int crypto4xx_encrypt_noiv_block(struct skcipher_request *req) +{ + return crypto4xx_crypt(req, 0, false, true); +} + +int crypto4xx_encrypt_iv_stream(struct skcipher_request *req) +{ + return crypto4xx_crypt(req, AES_IV_SIZE, false, false); +} + +int crypto4xx_decrypt_noiv_block(struct skcipher_request *req) +{ + return crypto4xx_crypt(req, 0, true, true); +} + +int crypto4xx_decrypt_iv_stream(struct skcipher_request *req) +{ + return crypto4xx_crypt(req, AES_IV_SIZE, true, false); +} + +int crypto4xx_encrypt_iv_block(struct skcipher_request *req) +{ + return crypto4xx_crypt(req, AES_IV_SIZE, false, true); +} + +int crypto4xx_decrypt_iv_block(struct skcipher_request *req) +{ + return crypto4xx_crypt(req, AES_IV_SIZE, true, true); +} + +/* + * AES Functions + */ +static int crypto4xx_setkey_aes(struct crypto_skcipher *cipher, + const u8 *key, + unsigned int keylen, + unsigned char cm, + u8 fb) +{ + struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher); + struct dynamic_sa_ctl *sa; + int rc; + + if (keylen != AES_KEYSIZE_256 && keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_128) + return -EINVAL; + + /* Create SA */ + if (ctx->sa_in || ctx->sa_out) + crypto4xx_free_sa(ctx); + + rc = crypto4xx_alloc_sa(ctx, SA_AES128_LEN + (keylen-16) / 4); + if (rc) + return rc; + + /* Setup SA */ + sa = ctx->sa_in; + + set_dynamic_sa_command_0(sa, SA_NOT_SAVE_HASH, (cm == CRYPTO_MODE_ECB ? + SA_NOT_SAVE_IV : SA_SAVE_IV), + SA_NOT_LOAD_HASH, (cm == CRYPTO_MODE_ECB ? + SA_LOAD_IV_FROM_SA : SA_LOAD_IV_FROM_STATE), + SA_NO_HEADER_PROC, SA_HASH_ALG_NULL, + SA_CIPHER_ALG_AES, SA_PAD_TYPE_ZERO, + SA_OP_GROUP_BASIC, SA_OPCODE_DECRYPT, + DIR_INBOUND); + + set_dynamic_sa_command_1(sa, cm, SA_HASH_MODE_HASH, + fb, SA_EXTENDED_SN_OFF, + SA_SEQ_MASK_OFF, SA_MC_ENABLE, + SA_NOT_COPY_PAD, SA_NOT_COPY_PAYLOAD, + SA_NOT_COPY_HDR); + crypto4xx_memcpy_to_le32(get_dynamic_sa_key_field(sa), + key, keylen); + sa->sa_contents.w = SA_AES_CONTENTS | (keylen << 2); + sa->sa_command_1.bf.key_len = keylen >> 3; + + memcpy(ctx->sa_out, ctx->sa_in, ctx->sa_len * 4); + sa = ctx->sa_out; + sa->sa_command_0.bf.dir = DIR_OUTBOUND; + /* + * SA_OPCODE_ENCRYPT is the same value as SA_OPCODE_DECRYPT. + * it's the DIR_(IN|OUT)BOUND that matters + */ + sa->sa_command_0.bf.opcode = SA_OPCODE_ENCRYPT; + + return 0; +} + +int crypto4xx_setkey_aes_cbc(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_CBC, + CRYPTO_FEEDBACK_MODE_NO_FB); +} + +int crypto4xx_setkey_aes_ecb(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_ECB, + CRYPTO_FEEDBACK_MODE_NO_FB); +} + +int crypto4xx_setkey_rfc3686(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher); + int rc; + + rc = crypto4xx_setkey_aes(cipher, key, keylen - CTR_RFC3686_NONCE_SIZE, + CRYPTO_MODE_CTR, CRYPTO_FEEDBACK_MODE_NO_FB); + if (rc) + return rc; + + ctx->iv_nonce = cpu_to_le32p((u32 *)&key[keylen - + CTR_RFC3686_NONCE_SIZE]); + + return 0; +} + +int crypto4xx_rfc3686_encrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher); + __le32 iv[AES_IV_SIZE / 4] = { + ctx->iv_nonce, + cpu_to_le32p((u32 *) req->iv), + cpu_to_le32p((u32 *) (req->iv + 4)), + cpu_to_le32(1) }; + + return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, + req->cryptlen, iv, AES_IV_SIZE, + ctx->sa_out, ctx->sa_len, 0, NULL); +} + +int crypto4xx_rfc3686_decrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher); + __le32 iv[AES_IV_SIZE / 4] = { + ctx->iv_nonce, + cpu_to_le32p((u32 *) req->iv), + cpu_to_le32p((u32 *) (req->iv + 4)), + cpu_to_le32(1) }; + + return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, + req->cryptlen, iv, AES_IV_SIZE, + ctx->sa_out, ctx->sa_len, 0, NULL); +} + +static int +crypto4xx_ctr_crypt(struct skcipher_request *req, bool encrypt) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher); + size_t iv_len = crypto_skcipher_ivsize(cipher); + unsigned int counter = be32_to_cpup((__be32 *)(req->iv + iv_len - 4)); + unsigned int nblks = ALIGN(req->cryptlen, AES_BLOCK_SIZE) / + AES_BLOCK_SIZE; + + /* + * The hardware uses only the last 32-bits as the counter while the + * kernel tests (aes_ctr_enc_tv_template[4] for example) expect that + * the whole IV is a counter. So fallback if the counter is going to + * overlow. + */ + if (counter + nblks < counter) { + SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->sw_cipher.cipher); + int ret; + + skcipher_request_set_sync_tfm(subreq, ctx->sw_cipher.cipher); + skcipher_request_set_callback(subreq, req->base.flags, + NULL, NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + ret = encrypt ? crypto_skcipher_encrypt(subreq) + : crypto_skcipher_decrypt(subreq); + skcipher_request_zero(subreq); + return ret; + } + + return encrypt ? crypto4xx_encrypt_iv_stream(req) + : crypto4xx_decrypt_iv_stream(req); +} + +static int crypto4xx_sk_setup_fallback(struct crypto4xx_ctx *ctx, + struct crypto_skcipher *cipher, + const u8 *key, + unsigned int keylen) +{ + crypto_sync_skcipher_clear_flags(ctx->sw_cipher.cipher, + CRYPTO_TFM_REQ_MASK); + crypto_sync_skcipher_set_flags(ctx->sw_cipher.cipher, + crypto_skcipher_get_flags(cipher) & CRYPTO_TFM_REQ_MASK); + return crypto_sync_skcipher_setkey(ctx->sw_cipher.cipher, key, keylen); +} + +int crypto4xx_setkey_aes_ctr(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher); + int rc; + + rc = crypto4xx_sk_setup_fallback(ctx, cipher, key, keylen); + if (rc) + return rc; + + return crypto4xx_setkey_aes(cipher, key, keylen, + CRYPTO_MODE_CTR, CRYPTO_FEEDBACK_MODE_NO_FB); +} + +int crypto4xx_encrypt_ctr(struct skcipher_request *req) +{ + return crypto4xx_ctr_crypt(req, true); +} + +int crypto4xx_decrypt_ctr(struct skcipher_request *req) +{ + return crypto4xx_ctr_crypt(req, false); +} + +static inline bool crypto4xx_aead_need_fallback(struct aead_request *req, + unsigned int len, + bool is_ccm, bool decrypt) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + + /* authsize has to be a multiple of 4 */ + if (aead->authsize & 3) + return true; + + /* + * hardware does not handle cases where plaintext + * is less than a block. + */ + if (len < AES_BLOCK_SIZE) + return true; + + /* assoc len needs to be a multiple of 4 and <= 1020 */ + if (req->assoclen & 0x3 || req->assoclen > 1020) + return true; + + /* CCM supports only counter field length of 2 and 4 bytes */ + if (is_ccm && !(req->iv[0] == 1 || req->iv[0] == 3)) + return true; + + return false; +} + +static int crypto4xx_aead_fallback(struct aead_request *req, + struct crypto4xx_ctx *ctx, bool do_decrypt) +{ + struct aead_request *subreq = aead_request_ctx(req); + + aead_request_set_tfm(subreq, ctx->sw_cipher.aead); + aead_request_set_callback(subreq, req->base.flags, + req->base.complete, req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, + req->iv); + aead_request_set_ad(subreq, req->assoclen); + return do_decrypt ? crypto_aead_decrypt(subreq) : + crypto_aead_encrypt(subreq); +} + +static int crypto4xx_aead_setup_fallback(struct crypto4xx_ctx *ctx, + struct crypto_aead *cipher, + const u8 *key, + unsigned int keylen) +{ + crypto_aead_clear_flags(ctx->sw_cipher.aead, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(ctx->sw_cipher.aead, + crypto_aead_get_flags(cipher) & CRYPTO_TFM_REQ_MASK); + return crypto_aead_setkey(ctx->sw_cipher.aead, key, keylen); +} + +/* + * AES-CCM Functions + */ + +int crypto4xx_setkey_aes_ccm(struct crypto_aead *cipher, const u8 *key, + unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(cipher); + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); + struct dynamic_sa_ctl *sa; + int rc = 0; + + rc = crypto4xx_aead_setup_fallback(ctx, cipher, key, keylen); + if (rc) + return rc; + + if (ctx->sa_in || ctx->sa_out) + crypto4xx_free_sa(ctx); + + rc = crypto4xx_alloc_sa(ctx, SA_AES128_CCM_LEN + (keylen - 16) / 4); + if (rc) + return rc; + + /* Setup SA */ + sa = (struct dynamic_sa_ctl *) ctx->sa_in; + sa->sa_contents.w = SA_AES_CCM_CONTENTS | (keylen << 2); + + set_dynamic_sa_command_0(sa, SA_SAVE_HASH, SA_NOT_SAVE_IV, + SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE, + SA_NO_HEADER_PROC, SA_HASH_ALG_CBC_MAC, + SA_CIPHER_ALG_AES, + SA_PAD_TYPE_ZERO, SA_OP_GROUP_BASIC, + SA_OPCODE_HASH_DECRYPT, DIR_INBOUND); + + set_dynamic_sa_command_1(sa, CRYPTO_MODE_CTR, SA_HASH_MODE_HASH, + CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF, + SA_SEQ_MASK_OFF, SA_MC_ENABLE, + SA_NOT_COPY_PAD, SA_COPY_PAYLOAD, + SA_NOT_COPY_HDR); + + sa->sa_command_1.bf.key_len = keylen >> 3; + + crypto4xx_memcpy_to_le32(get_dynamic_sa_key_field(sa), key, keylen); + + memcpy(ctx->sa_out, ctx->sa_in, ctx->sa_len * 4); + sa = (struct dynamic_sa_ctl *) ctx->sa_out; + + set_dynamic_sa_command_0(sa, SA_SAVE_HASH, SA_NOT_SAVE_IV, + SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE, + SA_NO_HEADER_PROC, SA_HASH_ALG_CBC_MAC, + SA_CIPHER_ALG_AES, + SA_PAD_TYPE_ZERO, SA_OP_GROUP_BASIC, + SA_OPCODE_ENCRYPT_HASH, DIR_OUTBOUND); + + set_dynamic_sa_command_1(sa, CRYPTO_MODE_CTR, SA_HASH_MODE_HASH, + CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF, + SA_SEQ_MASK_OFF, SA_MC_ENABLE, + SA_COPY_PAD, SA_COPY_PAYLOAD, + SA_NOT_COPY_HDR); + + sa->sa_command_1.bf.key_len = keylen >> 3; + return 0; +} + +static int crypto4xx_crypt_aes_ccm(struct aead_request *req, bool decrypt) +{ + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct crypto4xx_aead_reqctx *rctx = aead_request_ctx(req); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + __le32 iv[4]; + u32 tmp_sa[SA_AES128_CCM_LEN + 4]; + struct dynamic_sa_ctl *sa = (struct dynamic_sa_ctl *)tmp_sa; + unsigned int len = req->cryptlen; + + if (decrypt) + len -= crypto_aead_authsize(aead); + + if (crypto4xx_aead_need_fallback(req, len, true, decrypt)) + return crypto4xx_aead_fallback(req, ctx, decrypt); + + memcpy(tmp_sa, decrypt ? ctx->sa_in : ctx->sa_out, ctx->sa_len * 4); + sa->sa_command_0.bf.digest_len = crypto_aead_authsize(aead) >> 2; + + if (req->iv[0] == 1) { + /* CRYPTO_MODE_AES_ICM */ + sa->sa_command_1.bf.crypto_mode9_8 = 1; + } + + iv[3] = cpu_to_le32(0); + crypto4xx_memcpy_to_le32(iv, req->iv, 16 - (req->iv[0] + 1)); + + return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, + len, iv, sizeof(iv), + sa, ctx->sa_len, req->assoclen, rctx->dst); +} + +int crypto4xx_encrypt_aes_ccm(struct aead_request *req) +{ + return crypto4xx_crypt_aes_ccm(req, false); +} + +int crypto4xx_decrypt_aes_ccm(struct aead_request *req) +{ + return crypto4xx_crypt_aes_ccm(req, true); +} + +int crypto4xx_setauthsize_aead(struct crypto_aead *cipher, + unsigned int authsize) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(cipher); + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); + + return crypto_aead_setauthsize(ctx->sw_cipher.aead, authsize); +} + +/* + * AES-GCM Functions + */ + +static int crypto4xx_aes_gcm_validate_keylen(unsigned int keylen) +{ + switch (keylen) { + case 16: + case 24: + case 32: + return 0; + default: + return -EINVAL; + } +} + +static int crypto4xx_compute_gcm_hash_key_sw(__le32 *hash_start, const u8 *key, + unsigned int keylen) +{ + struct aes_enckey aes; + uint8_t src[16] = { 0 }; + int rc; + + rc = aes_prepareenckey(&aes, key, keylen); + if (rc) { + pr_err("aes_prepareenckey() failed: %d\n", rc); + return rc; + } + + aes_encrypt(&aes, src, src); + crypto4xx_memcpy_to_le32(hash_start, src, 16); + memzero_explicit(&aes, sizeof(aes)); + return 0; +} + +int crypto4xx_setkey_aes_gcm(struct crypto_aead *cipher, + const u8 *key, unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(cipher); + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); + struct dynamic_sa_ctl *sa; + int rc = 0; + + if (crypto4xx_aes_gcm_validate_keylen(keylen) != 0) + return -EINVAL; + + rc = crypto4xx_aead_setup_fallback(ctx, cipher, key, keylen); + if (rc) + return rc; + + if (ctx->sa_in || ctx->sa_out) + crypto4xx_free_sa(ctx); + + rc = crypto4xx_alloc_sa(ctx, SA_AES128_GCM_LEN + (keylen - 16) / 4); + if (rc) + return rc; + + sa = (struct dynamic_sa_ctl *) ctx->sa_in; + + sa->sa_contents.w = SA_AES_GCM_CONTENTS | (keylen << 2); + set_dynamic_sa_command_0(sa, SA_SAVE_HASH, SA_NOT_SAVE_IV, + SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE, + SA_NO_HEADER_PROC, SA_HASH_ALG_GHASH, + SA_CIPHER_ALG_AES, SA_PAD_TYPE_ZERO, + SA_OP_GROUP_BASIC, SA_OPCODE_HASH_DECRYPT, + DIR_INBOUND); + set_dynamic_sa_command_1(sa, CRYPTO_MODE_CTR, SA_HASH_MODE_HASH, + CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF, + SA_SEQ_MASK_ON, SA_MC_DISABLE, + SA_NOT_COPY_PAD, SA_COPY_PAYLOAD, + SA_NOT_COPY_HDR); + + sa->sa_command_1.bf.key_len = keylen >> 3; + + crypto4xx_memcpy_to_le32(get_dynamic_sa_key_field(sa), + key, keylen); + + rc = crypto4xx_compute_gcm_hash_key_sw(get_dynamic_sa_inner_digest(sa), + key, keylen); + if (rc) { + pr_err("GCM hash key setting failed = %d\n", rc); + goto err; + } + + memcpy(ctx->sa_out, ctx->sa_in, ctx->sa_len * 4); + sa = (struct dynamic_sa_ctl *) ctx->sa_out; + sa->sa_command_0.bf.dir = DIR_OUTBOUND; + sa->sa_command_0.bf.opcode = SA_OPCODE_ENCRYPT_HASH; + + return 0; +err: + crypto4xx_free_sa(ctx); + return rc; +} + +static inline int crypto4xx_crypt_aes_gcm(struct aead_request *req, + bool decrypt) +{ + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct crypto4xx_aead_reqctx *rctx = aead_request_ctx(req); + __le32 iv[4]; + unsigned int len = req->cryptlen; + + if (decrypt) + len -= crypto_aead_authsize(crypto_aead_reqtfm(req)); + + if (crypto4xx_aead_need_fallback(req, len, false, decrypt)) + return crypto4xx_aead_fallback(req, ctx, decrypt); + + crypto4xx_memcpy_to_le32(iv, req->iv, GCM_AES_IV_SIZE); + iv[3] = cpu_to_le32(1); + + return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, + len, iv, sizeof(iv), + decrypt ? ctx->sa_in : ctx->sa_out, + ctx->sa_len, req->assoclen, rctx->dst); +} + +int crypto4xx_encrypt_aes_gcm(struct aead_request *req) +{ + return crypto4xx_crypt_aes_gcm(req, false); +} + +int crypto4xx_decrypt_aes_gcm(struct aead_request *req) +{ + return crypto4xx_crypt_aes_gcm(req, true); +} diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c new file mode 100644 index 000000000..fd010bfb7 --- /dev/null +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -0,0 +1,1367 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * AMCC SoC PPC4xx Crypto Driver + * + * Copyright (c) 2008 Applied Micro Circuits Corporation. + * All rights reserved. James Hsiao + * + * This file implements AMCC crypto offload Linux device driver for use with + * Linux CryptoAPI. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "crypto4xx_reg_def.h" +#include "crypto4xx_core.h" +#include "crypto4xx_sa.h" +#include "crypto4xx_trng.h" + +#define PPC4XX_SEC_VERSION_STR "0.5" + +/* + * PPC4xx Crypto Engine Initialization Routine + */ +static void crypto4xx_hw_init(struct crypto4xx_device *dev) +{ + union ce_ring_size ring_size; + union ce_ring_control ring_ctrl; + union ce_part_ring_size part_ring_size; + union ce_io_threshold io_threshold; + u32 rand_num; + union ce_pe_dma_cfg pe_dma_cfg; + u32 device_ctrl; + + writel(PPC4XX_BYTE_ORDER, dev->ce_base + CRYPTO4XX_BYTE_ORDER_CFG); + /* setup pe dma, include reset sg, pdr and pe, then release reset */ + pe_dma_cfg.w = 0; + pe_dma_cfg.bf.bo_sgpd_en = 1; + pe_dma_cfg.bf.bo_data_en = 0; + pe_dma_cfg.bf.bo_sa_en = 1; + pe_dma_cfg.bf.bo_pd_en = 1; + pe_dma_cfg.bf.dynamic_sa_en = 1; + pe_dma_cfg.bf.reset_sg = 1; + pe_dma_cfg.bf.reset_pdr = 1; + pe_dma_cfg.bf.reset_pe = 1; + writel(pe_dma_cfg.w, dev->ce_base + CRYPTO4XX_PE_DMA_CFG); + /* un reset pe,sg and pdr */ + pe_dma_cfg.bf.pe_mode = 0; + pe_dma_cfg.bf.reset_sg = 0; + pe_dma_cfg.bf.reset_pdr = 0; + pe_dma_cfg.bf.reset_pe = 0; + pe_dma_cfg.bf.bo_td_en = 0; + writel(pe_dma_cfg.w, dev->ce_base + CRYPTO4XX_PE_DMA_CFG); + writel(dev->pdr_pa, dev->ce_base + CRYPTO4XX_PDR_BASE); + writel(dev->pdr_pa, dev->ce_base + CRYPTO4XX_RDR_BASE); + writel(PPC4XX_PRNG_CTRL_AUTO_EN, dev->ce_base + CRYPTO4XX_PRNG_CTRL); + get_random_bytes(&rand_num, sizeof(rand_num)); + writel(rand_num, dev->ce_base + CRYPTO4XX_PRNG_SEED_L); + get_random_bytes(&rand_num, sizeof(rand_num)); + writel(rand_num, dev->ce_base + CRYPTO4XX_PRNG_SEED_H); + ring_size.w = 0; + ring_size.bf.ring_offset = PPC4XX_PD_SIZE; + ring_size.bf.ring_size = PPC4XX_NUM_PD; + writel(ring_size.w, dev->ce_base + CRYPTO4XX_RING_SIZE); + ring_ctrl.w = 0; + writel(ring_ctrl.w, dev->ce_base + CRYPTO4XX_RING_CTRL); + device_ctrl = readl(dev->ce_base + CRYPTO4XX_DEVICE_CTRL); + device_ctrl |= PPC4XX_DC_3DES_EN; + writel(device_ctrl, dev->ce_base + CRYPTO4XX_DEVICE_CTRL); + writel(dev->gdr_pa, dev->ce_base + CRYPTO4XX_GATH_RING_BASE); + writel(dev->sdr_pa, dev->ce_base + CRYPTO4XX_SCAT_RING_BASE); + part_ring_size.w = 0; + part_ring_size.bf.sdr_size = PPC4XX_SDR_SIZE; + part_ring_size.bf.gdr_size = PPC4XX_GDR_SIZE; + writel(part_ring_size.w, dev->ce_base + CRYPTO4XX_PART_RING_SIZE); + writel(PPC4XX_SD_BUFFER_SIZE, dev->ce_base + CRYPTO4XX_PART_RING_CFG); + io_threshold.w = 0; + io_threshold.bf.output_threshold = PPC4XX_OUTPUT_THRESHOLD; + io_threshold.bf.input_threshold = PPC4XX_INPUT_THRESHOLD; + writel(io_threshold.w, dev->ce_base + CRYPTO4XX_IO_THRESHOLD); + writel(0, dev->ce_base + CRYPTO4XX_PDR_BASE_UADDR); + writel(0, dev->ce_base + CRYPTO4XX_RDR_BASE_UADDR); + writel(0, dev->ce_base + CRYPTO4XX_PKT_SRC_UADDR); + writel(0, dev->ce_base + CRYPTO4XX_PKT_DEST_UADDR); + writel(0, dev->ce_base + CRYPTO4XX_SA_UADDR); + writel(0, dev->ce_base + CRYPTO4XX_GATH_RING_BASE_UADDR); + writel(0, dev->ce_base + CRYPTO4XX_SCAT_RING_BASE_UADDR); + /* un reset pe,sg and pdr */ + pe_dma_cfg.bf.pe_mode = 1; + pe_dma_cfg.bf.reset_sg = 0; + pe_dma_cfg.bf.reset_pdr = 0; + pe_dma_cfg.bf.reset_pe = 0; + pe_dma_cfg.bf.bo_td_en = 0; + writel(pe_dma_cfg.w, dev->ce_base + CRYPTO4XX_PE_DMA_CFG); + /*clear all pending interrupt*/ + writel(PPC4XX_INTERRUPT_CLR, dev->ce_base + CRYPTO4XX_INT_CLR); + writel(PPC4XX_INT_DESCR_CNT, dev->ce_base + CRYPTO4XX_INT_DESCR_CNT); + writel(PPC4XX_INT_DESCR_CNT, dev->ce_base + CRYPTO4XX_INT_DESCR_CNT); + writel(PPC4XX_INT_CFG, dev->ce_base + CRYPTO4XX_INT_CFG); + if (dev->is_revb) { + writel(PPC4XX_INT_TIMEOUT_CNT_REVB << 10, + dev->ce_base + CRYPTO4XX_INT_TIMEOUT_CNT); + writel(PPC4XX_PD_DONE_INT | PPC4XX_TMO_ERR_INT, + dev->ce_base + CRYPTO4XX_INT_EN); + } else { + writel(PPC4XX_PD_DONE_INT, dev->ce_base + CRYPTO4XX_INT_EN); + } +} + +int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size) +{ + ctx->sa_in = kcalloc(size, 4, GFP_ATOMIC); + if (ctx->sa_in == NULL) + return -ENOMEM; + + ctx->sa_out = kcalloc(size, 4, GFP_ATOMIC); + if (ctx->sa_out == NULL) { + kfree(ctx->sa_in); + ctx->sa_in = NULL; + return -ENOMEM; + } + + ctx->sa_len = size; + + return 0; +} + +void crypto4xx_free_sa(struct crypto4xx_ctx *ctx) +{ + kfree(ctx->sa_in); + ctx->sa_in = NULL; + kfree(ctx->sa_out); + ctx->sa_out = NULL; + ctx->sa_len = 0; +} + +/* + * alloc memory for the gather ring + * no need to alloc buf for the ring + * gdr_tail, gdr_head and gdr_count are initialized by this function + */ +static u32 crypto4xx_build_pdr(struct crypto4xx_device *dev) +{ + int i; + dev->pdr = dma_alloc_coherent(dev->core_dev->device, + sizeof(struct ce_pd) * PPC4XX_NUM_PD, + &dev->pdr_pa, GFP_KERNEL); + if (!dev->pdr) + return -ENOMEM; + + dev->pdr_uinfo = kzalloc_objs(struct pd_uinfo, PPC4XX_NUM_PD); + if (!dev->pdr_uinfo) { + dma_free_coherent(dev->core_dev->device, + sizeof(struct ce_pd) * PPC4XX_NUM_PD, + dev->pdr, + dev->pdr_pa); + return -ENOMEM; + } + dev->shadow_sa_pool = dma_alloc_coherent(dev->core_dev->device, + sizeof(union shadow_sa_buf) * PPC4XX_NUM_PD, + &dev->shadow_sa_pool_pa, + GFP_KERNEL); + if (!dev->shadow_sa_pool) + return -ENOMEM; + + dev->shadow_sr_pool = dma_alloc_coherent(dev->core_dev->device, + sizeof(struct sa_state_record) * PPC4XX_NUM_PD, + &dev->shadow_sr_pool_pa, GFP_KERNEL); + if (!dev->shadow_sr_pool) + return -ENOMEM; + for (i = 0; i < PPC4XX_NUM_PD; i++) { + struct ce_pd *pd = &dev->pdr[i]; + struct pd_uinfo *pd_uinfo = &dev->pdr_uinfo[i]; + + pd->sa = dev->shadow_sa_pool_pa + + sizeof(union shadow_sa_buf) * i; + + /* alloc 256 bytes which is enough for any kind of dynamic sa */ + pd_uinfo->sa_va = &dev->shadow_sa_pool[i].sa; + + /* alloc state record */ + pd_uinfo->sr_va = &dev->shadow_sr_pool[i]; + pd_uinfo->sr_pa = dev->shadow_sr_pool_pa + + sizeof(struct sa_state_record) * i; + } + + return 0; +} + +static void crypto4xx_destroy_pdr(struct crypto4xx_device *dev) +{ + if (dev->pdr) + dma_free_coherent(dev->core_dev->device, + sizeof(struct ce_pd) * PPC4XX_NUM_PD, + dev->pdr, dev->pdr_pa); + + if (dev->shadow_sa_pool) + dma_free_coherent(dev->core_dev->device, + sizeof(union shadow_sa_buf) * PPC4XX_NUM_PD, + dev->shadow_sa_pool, dev->shadow_sa_pool_pa); + + if (dev->shadow_sr_pool) + dma_free_coherent(dev->core_dev->device, + sizeof(struct sa_state_record) * PPC4XX_NUM_PD, + dev->shadow_sr_pool, dev->shadow_sr_pool_pa); + + kfree(dev->pdr_uinfo); +} + +static u32 crypto4xx_get_pd_from_pdr_nolock(struct crypto4xx_device *dev) +{ + u32 retval; + u32 tmp; + + retval = dev->pdr_head; + tmp = (dev->pdr_head + 1) % PPC4XX_NUM_PD; + + if (tmp == dev->pdr_tail) + return ERING_WAS_FULL; + + dev->pdr_head = tmp; + + return retval; +} + +static u32 crypto4xx_put_pd_to_pdr(struct crypto4xx_device *dev, u32 idx) +{ + struct pd_uinfo *pd_uinfo = &dev->pdr_uinfo[idx]; + u32 tail; + unsigned long flags; + + spin_lock_irqsave(&dev->core_dev->lock, flags); + pd_uinfo->state = PD_ENTRY_FREE; + + if (dev->pdr_tail != PPC4XX_LAST_PD) + dev->pdr_tail++; + else + dev->pdr_tail = 0; + tail = dev->pdr_tail; + spin_unlock_irqrestore(&dev->core_dev->lock, flags); + + return tail; +} + +/* + * alloc memory for the gather ring + * no need to alloc buf for the ring + * gdr_tail, gdr_head and gdr_count are initialized by this function + */ +static u32 crypto4xx_build_gdr(struct crypto4xx_device *dev) +{ + dev->gdr = dma_alloc_coherent(dev->core_dev->device, + sizeof(struct ce_gd) * PPC4XX_NUM_GD, + &dev->gdr_pa, GFP_KERNEL); + if (!dev->gdr) + return -ENOMEM; + + return 0; +} + +static inline void crypto4xx_destroy_gdr(struct crypto4xx_device *dev) +{ + if (dev->gdr) + dma_free_coherent(dev->core_dev->device, + sizeof(struct ce_gd) * PPC4XX_NUM_GD, + dev->gdr, dev->gdr_pa); +} + +/* + * when this function is called. + * preemption or interrupt must be disabled + */ +static u32 crypto4xx_get_n_gd(struct crypto4xx_device *dev, int n) +{ + u32 retval; + u32 tmp; + + if (n >= PPC4XX_NUM_GD) + return ERING_WAS_FULL; + + retval = dev->gdr_head; + tmp = (dev->gdr_head + n) % PPC4XX_NUM_GD; + if (dev->gdr_head > dev->gdr_tail) { + if (tmp < dev->gdr_head && tmp >= dev->gdr_tail) + return ERING_WAS_FULL; + } else if (dev->gdr_head < dev->gdr_tail) { + if (tmp < dev->gdr_head || tmp >= dev->gdr_tail) + return ERING_WAS_FULL; + } + dev->gdr_head = tmp; + + return retval; +} + +static u32 crypto4xx_put_gd_to_gdr(struct crypto4xx_device *dev) +{ + unsigned long flags; + + spin_lock_irqsave(&dev->core_dev->lock, flags); + if (dev->gdr_tail == dev->gdr_head) { + spin_unlock_irqrestore(&dev->core_dev->lock, flags); + return 0; + } + + if (dev->gdr_tail != PPC4XX_LAST_GD) + dev->gdr_tail++; + else + dev->gdr_tail = 0; + + spin_unlock_irqrestore(&dev->core_dev->lock, flags); + + return 0; +} + +static inline struct ce_gd *crypto4xx_get_gdp(struct crypto4xx_device *dev, + dma_addr_t *gd_dma, u32 idx) +{ + *gd_dma = dev->gdr_pa + sizeof(struct ce_gd) * idx; + + return &dev->gdr[idx]; +} + +/* + * alloc memory for the scatter ring + * need to alloc buf for the ring + * sdr_tail, sdr_head and sdr_count are initialized by this function + */ +static u32 crypto4xx_build_sdr(struct crypto4xx_device *dev) +{ + int i; + + dev->scatter_buffer_va = + dma_alloc_coherent(dev->core_dev->device, + PPC4XX_SD_BUFFER_SIZE * PPC4XX_NUM_SD, + &dev->scatter_buffer_pa, GFP_KERNEL); + if (!dev->scatter_buffer_va) + return -ENOMEM; + + /* alloc memory for scatter descriptor ring */ + dev->sdr = dma_alloc_coherent(dev->core_dev->device, + sizeof(struct ce_sd) * PPC4XX_NUM_SD, + &dev->sdr_pa, GFP_KERNEL); + if (!dev->sdr) + return -ENOMEM; + + for (i = 0; i < PPC4XX_NUM_SD; i++) { + dev->sdr[i].ptr = dev->scatter_buffer_pa + + PPC4XX_SD_BUFFER_SIZE * i; + } + + return 0; +} + +static void crypto4xx_destroy_sdr(struct crypto4xx_device *dev) +{ + if (dev->sdr) + dma_free_coherent(dev->core_dev->device, + sizeof(struct ce_sd) * PPC4XX_NUM_SD, + dev->sdr, dev->sdr_pa); + + if (dev->scatter_buffer_va) + dma_free_coherent(dev->core_dev->device, + PPC4XX_SD_BUFFER_SIZE * PPC4XX_NUM_SD, + dev->scatter_buffer_va, + dev->scatter_buffer_pa); +} + +/* + * when this function is called. + * preemption or interrupt must be disabled + */ +static u32 crypto4xx_get_n_sd(struct crypto4xx_device *dev, int n) +{ + u32 retval; + u32 tmp; + + if (n >= PPC4XX_NUM_SD) + return ERING_WAS_FULL; + + retval = dev->sdr_head; + tmp = (dev->sdr_head + n) % PPC4XX_NUM_SD; + if (dev->sdr_head > dev->gdr_tail) { + if (tmp < dev->sdr_head && tmp >= dev->sdr_tail) + return ERING_WAS_FULL; + } else if (dev->sdr_head < dev->sdr_tail) { + if (tmp < dev->sdr_head || tmp >= dev->sdr_tail) + return ERING_WAS_FULL; + } /* the head = tail, or empty case is already take cared */ + dev->sdr_head = tmp; + + return retval; +} + +static u32 crypto4xx_put_sd_to_sdr(struct crypto4xx_device *dev) +{ + unsigned long flags; + + spin_lock_irqsave(&dev->core_dev->lock, flags); + if (dev->sdr_tail == dev->sdr_head) { + spin_unlock_irqrestore(&dev->core_dev->lock, flags); + return 0; + } + if (dev->sdr_tail != PPC4XX_LAST_SD) + dev->sdr_tail++; + else + dev->sdr_tail = 0; + spin_unlock_irqrestore(&dev->core_dev->lock, flags); + + return 0; +} + +static inline struct ce_sd *crypto4xx_get_sdp(struct crypto4xx_device *dev, + dma_addr_t *sd_dma, u32 idx) +{ + *sd_dma = dev->sdr_pa + sizeof(struct ce_sd) * idx; + + return &dev->sdr[idx]; +} + +static void crypto4xx_copy_pkt_to_dst(struct crypto4xx_device *dev, + struct ce_pd *pd, + struct pd_uinfo *pd_uinfo, + u32 nbytes, + struct scatterlist *dst) +{ + unsigned int first_sd = pd_uinfo->first_sd; + unsigned int last_sd; + unsigned int overflow = 0; + unsigned int to_copy; + unsigned int dst_start = 0; + + /* + * Because the scatter buffers are all neatly organized in one + * big continuous ringbuffer; scatterwalk_map_and_copy() can + * be instructed to copy a range of buffers in one go. + */ + + last_sd = (first_sd + pd_uinfo->num_sd); + if (last_sd > PPC4XX_LAST_SD) { + last_sd = PPC4XX_LAST_SD; + overflow = last_sd % PPC4XX_NUM_SD; + } + + while (nbytes) { + void *buf = dev->scatter_buffer_va + + first_sd * PPC4XX_SD_BUFFER_SIZE; + + to_copy = min(nbytes, PPC4XX_SD_BUFFER_SIZE * + (1 + last_sd - first_sd)); + scatterwalk_map_and_copy(buf, dst, dst_start, to_copy, 1); + nbytes -= to_copy; + + if (overflow) { + first_sd = 0; + last_sd = overflow; + dst_start += to_copy; + overflow = 0; + } + } +} + +static void crypto4xx_ret_sg_desc(struct crypto4xx_device *dev, + struct pd_uinfo *pd_uinfo) +{ + int i; + if (pd_uinfo->num_gd) { + for (i = 0; i < pd_uinfo->num_gd; i++) + crypto4xx_put_gd_to_gdr(dev); + pd_uinfo->first_gd = 0xffffffff; + pd_uinfo->num_gd = 0; + } + if (pd_uinfo->num_sd) { + for (i = 0; i < pd_uinfo->num_sd; i++) + crypto4xx_put_sd_to_sdr(dev); + + pd_uinfo->first_sd = 0xffffffff; + pd_uinfo->num_sd = 0; + } +} + +static void crypto4xx_cipher_done(struct crypto4xx_device *dev, + struct pd_uinfo *pd_uinfo, + struct ce_pd *pd) +{ + struct skcipher_request *req; + struct scatterlist *dst; + + req = skcipher_request_cast(pd_uinfo->async_req); + + if (pd_uinfo->sa_va->sa_command_0.bf.scatter) { + crypto4xx_copy_pkt_to_dst(dev, pd, pd_uinfo, + req->cryptlen, req->dst); + } else { + dst = pd_uinfo->dest_va; + dma_unmap_page(dev->core_dev->device, pd->dest, dst->length, + DMA_FROM_DEVICE); + } + + if (pd_uinfo->sa_va->sa_command_0.bf.save_iv == SA_SAVE_IV) { + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + + crypto4xx_memcpy_from_le32((u32 *)req->iv, + pd_uinfo->sr_va->save_iv, + crypto_skcipher_ivsize(skcipher)); + } + + crypto4xx_ret_sg_desc(dev, pd_uinfo); + + if (pd_uinfo->state & PD_ENTRY_BUSY) + skcipher_request_complete(req, -EINPROGRESS); + skcipher_request_complete(req, 0); +} + +static void crypto4xx_aead_done(struct crypto4xx_device *dev, + struct pd_uinfo *pd_uinfo, + struct ce_pd *pd) +{ + struct aead_request *aead_req = container_of(pd_uinfo->async_req, + struct aead_request, base); + struct scatterlist *dst = pd_uinfo->dest_va; + size_t cp_len = crypto_aead_authsize( + crypto_aead_reqtfm(aead_req)); + u32 icv[AES_BLOCK_SIZE]; + int err = 0; + + if (pd_uinfo->sa_va->sa_command_0.bf.scatter) { + crypto4xx_copy_pkt_to_dst(dev, pd, pd_uinfo, + pd->pd_ctl_len.bf.pkt_len, + dst); + } else { + dma_unmap_page(dev->core_dev->device, pd->dest, dst->length, + DMA_FROM_DEVICE); + } + + if (pd_uinfo->sa_va->sa_command_0.bf.dir == DIR_OUTBOUND) { + /* append icv at the end */ + crypto4xx_memcpy_from_le32(icv, pd_uinfo->sr_va->save_digest, + sizeof(icv)); + + scatterwalk_map_and_copy(icv, dst, aead_req->cryptlen, + cp_len, 1); + } else { + /* check icv at the end */ + scatterwalk_map_and_copy(icv, aead_req->src, + aead_req->assoclen + aead_req->cryptlen - + cp_len, cp_len, 0); + + crypto4xx_memcpy_from_le32(icv, icv, sizeof(icv)); + + if (crypto_memneq(icv, pd_uinfo->sr_va->save_digest, cp_len)) + err = -EBADMSG; + } + + crypto4xx_ret_sg_desc(dev, pd_uinfo); + + if (pd->pd_ctl.bf.status & 0xff) { + if (!__ratelimit(&dev->aead_ratelimit)) { + if (pd->pd_ctl.bf.status & 2) + pr_err("pad fail error\n"); + if (pd->pd_ctl.bf.status & 4) + pr_err("seqnum fail\n"); + if (pd->pd_ctl.bf.status & 8) + pr_err("error _notify\n"); + pr_err("aead return err status = 0x%02x\n", + pd->pd_ctl.bf.status & 0xff); + pr_err("pd pad_ctl = 0x%08x\n", + pd->pd_ctl.bf.pd_pad_ctl); + } + err = -EINVAL; + } + + if (pd_uinfo->state & PD_ENTRY_BUSY) + aead_request_complete(aead_req, -EINPROGRESS); + + aead_request_complete(aead_req, err); +} + +static void crypto4xx_pd_done(struct crypto4xx_device *dev, u32 idx) +{ + struct ce_pd *pd = &dev->pdr[idx]; + struct pd_uinfo *pd_uinfo = &dev->pdr_uinfo[idx]; + + switch (crypto_tfm_alg_type(pd_uinfo->async_req->tfm)) { + case CRYPTO_ALG_TYPE_SKCIPHER: + crypto4xx_cipher_done(dev, pd_uinfo, pd); + break; + case CRYPTO_ALG_TYPE_AEAD: + crypto4xx_aead_done(dev, pd_uinfo, pd); + break; + } +} + +static void crypto4xx_stop_all(struct crypto4xx_core_device *core_dev) +{ + crypto4xx_destroy_pdr(core_dev->dev); + crypto4xx_destroy_gdr(core_dev->dev); + crypto4xx_destroy_sdr(core_dev->dev); +} + +static u32 get_next_gd(u32 current) +{ + if (current != PPC4XX_LAST_GD) + return current + 1; + else + return 0; +} + +static u32 get_next_sd(u32 current) +{ + if (current != PPC4XX_LAST_SD) + return current + 1; + else + return 0; +} + +int crypto4xx_build_pd(struct crypto_async_request *req, + struct crypto4xx_ctx *ctx, + struct scatterlist *src, + struct scatterlist *dst, + const unsigned int datalen, + const void *iv, const u32 iv_len, + const struct dynamic_sa_ctl *req_sa, + const unsigned int sa_len, + const unsigned int assoclen, + struct scatterlist *_dst) +{ + struct crypto4xx_device *dev = ctx->dev; + struct dynamic_sa_ctl *sa; + struct ce_gd *gd; + struct ce_pd *pd; + u32 num_gd, num_sd; + u32 fst_gd = 0xffffffff; + u32 fst_sd = 0xffffffff; + u32 pd_entry; + unsigned long flags; + struct pd_uinfo *pd_uinfo; + unsigned int nbytes = datalen; + size_t offset_to_sr_ptr; + u32 gd_idx = 0; + int tmp; + bool is_busy, force_sd; + + /* + * There's a very subtile/disguised "bug" in the hardware that + * gets indirectly mentioned in 18.1.3.5 Encryption/Decryption + * of the hardware spec: + * *drum roll* the AES/(T)DES OFB and CFB modes are listed as + * operation modes for >>> "Block ciphers" <<<. + * + * To workaround this issue and stop the hardware from causing + * "overran dst buffer" on crypttexts that are not a multiple + * of 16 (AES_BLOCK_SIZE), we force the driver to use the + * scatter buffers. + */ + force_sd = (req_sa->sa_command_1.bf.crypto_mode9_8 == CRYPTO_MODE_CFB + || req_sa->sa_command_1.bf.crypto_mode9_8 == CRYPTO_MODE_OFB) + && (datalen % AES_BLOCK_SIZE); + + /* figure how many gd are needed */ + tmp = sg_nents_for_len(src, assoclen + datalen); + if (tmp < 0) { + dev_err(dev->core_dev->device, "Invalid number of src SG.\n"); + return tmp; + } + if (tmp == 1) + tmp = 0; + num_gd = tmp; + + if (assoclen) { + nbytes += assoclen; + dst = scatterwalk_ffwd(_dst, dst, assoclen); + } + + /* figure how many sd are needed */ + if (sg_is_last(dst) && force_sd == false) { + num_sd = 0; + } else { + if (datalen > PPC4XX_SD_BUFFER_SIZE) { + num_sd = datalen / PPC4XX_SD_BUFFER_SIZE; + if (datalen % PPC4XX_SD_BUFFER_SIZE) + num_sd++; + } else { + num_sd = 1; + } + } + + /* + * The follow section of code needs to be protected + * The gather ring and scatter ring needs to be consecutive + * In case of run out of any kind of descriptor, the descriptor + * already got must be return the original place. + */ + spin_lock_irqsave(&dev->core_dev->lock, flags); + /* + * Let the caller know to slow down, once more than 13/16ths = 81% + * of the available data contexts are being used simultaneously. + * + * With PPC4XX_NUM_PD = 256, this will leave a "backlog queue" for + * 31 more contexts. Before new requests have to be rejected. + */ + if (req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) { + is_busy = ((dev->pdr_head - dev->pdr_tail) % PPC4XX_NUM_PD) >= + ((PPC4XX_NUM_PD * 13) / 16); + } else { + /* + * To fix contention issues between ipsec (no blacklog) and + * dm-crypto (backlog) reserve 32 entries for "no backlog" + * data contexts. + */ + is_busy = ((dev->pdr_head - dev->pdr_tail) % PPC4XX_NUM_PD) >= + ((PPC4XX_NUM_PD * 15) / 16); + + if (is_busy) { + spin_unlock_irqrestore(&dev->core_dev->lock, flags); + return -EBUSY; + } + } + + if (num_gd) { + fst_gd = crypto4xx_get_n_gd(dev, num_gd); + if (fst_gd == ERING_WAS_FULL) { + spin_unlock_irqrestore(&dev->core_dev->lock, flags); + return -EAGAIN; + } + } + if (num_sd) { + fst_sd = crypto4xx_get_n_sd(dev, num_sd); + if (fst_sd == ERING_WAS_FULL) { + if (num_gd) + dev->gdr_head = fst_gd; + spin_unlock_irqrestore(&dev->core_dev->lock, flags); + return -EAGAIN; + } + } + pd_entry = crypto4xx_get_pd_from_pdr_nolock(dev); + if (pd_entry == ERING_WAS_FULL) { + if (num_gd) + dev->gdr_head = fst_gd; + if (num_sd) + dev->sdr_head = fst_sd; + spin_unlock_irqrestore(&dev->core_dev->lock, flags); + return -EAGAIN; + } + spin_unlock_irqrestore(&dev->core_dev->lock, flags); + + pd = &dev->pdr[pd_entry]; + pd->sa_len = sa_len; + + pd_uinfo = &dev->pdr_uinfo[pd_entry]; + pd_uinfo->num_gd = num_gd; + pd_uinfo->num_sd = num_sd; + pd_uinfo->dest_va = dst; + pd_uinfo->async_req = req; + + if (iv_len) + memcpy(pd_uinfo->sr_va->save_iv, iv, iv_len); + + sa = pd_uinfo->sa_va; + memcpy(sa, req_sa, sa_len * 4); + + sa->sa_command_1.bf.hash_crypto_offset = (assoclen >> 2); + offset_to_sr_ptr = get_dynamic_sa_offset_state_ptr_field(sa); + *(u32 *)((unsigned long)sa + offset_to_sr_ptr) = pd_uinfo->sr_pa; + + if (num_gd) { + dma_addr_t gd_dma; + struct scatterlist *sg; + + /* get first gd we are going to use */ + gd_idx = fst_gd; + pd_uinfo->first_gd = fst_gd; + gd = crypto4xx_get_gdp(dev, &gd_dma, gd_idx); + pd->src = gd_dma; + /* enable gather */ + sa->sa_command_0.bf.gather = 1; + /* walk the sg, and setup gather array */ + + sg = src; + while (nbytes) { + size_t len; + + len = min(sg->length, nbytes); + gd->ptr = dma_map_page(dev->core_dev->device, + sg_page(sg), sg->offset, len, DMA_TO_DEVICE); + gd->ctl_len.len = len; + gd->ctl_len.done = 0; + gd->ctl_len.ready = 1; + if (len >= nbytes) + break; + + nbytes -= sg->length; + gd_idx = get_next_gd(gd_idx); + gd = crypto4xx_get_gdp(dev, &gd_dma, gd_idx); + sg = sg_next(sg); + } + } else { + pd->src = (u32)dma_map_page(dev->core_dev->device, sg_page(src), + src->offset, min(nbytes, src->length), + DMA_TO_DEVICE); + /* + * Disable gather in sa command + */ + sa->sa_command_0.bf.gather = 0; + /* + * Indicate gather array is not used + */ + pd_uinfo->first_gd = 0xffffffff; + } + if (!num_sd) { + /* + * we know application give us dst a whole piece of memory + * no need to use scatter ring. + */ + pd_uinfo->first_sd = 0xffffffff; + sa->sa_command_0.bf.scatter = 0; + pd->dest = (u32)dma_map_page(dev->core_dev->device, + sg_page(dst), dst->offset, + min(datalen, dst->length), + DMA_TO_DEVICE); + } else { + dma_addr_t sd_dma; + struct ce_sd *sd = NULL; + + u32 sd_idx = fst_sd; + nbytes = datalen; + sa->sa_command_0.bf.scatter = 1; + pd_uinfo->first_sd = fst_sd; + sd = crypto4xx_get_sdp(dev, &sd_dma, sd_idx); + pd->dest = sd_dma; + /* setup scatter descriptor */ + sd->ctl.done = 0; + sd->ctl.rdy = 1; + /* sd->ptr should be setup by sd_init routine*/ + if (nbytes >= PPC4XX_SD_BUFFER_SIZE) + nbytes -= PPC4XX_SD_BUFFER_SIZE; + else + nbytes = 0; + while (nbytes) { + sd_idx = get_next_sd(sd_idx); + sd = crypto4xx_get_sdp(dev, &sd_dma, sd_idx); + /* setup scatter descriptor */ + sd->ctl.done = 0; + sd->ctl.rdy = 1; + if (nbytes >= PPC4XX_SD_BUFFER_SIZE) { + nbytes -= PPC4XX_SD_BUFFER_SIZE; + } else { + /* + * SD entry can hold PPC4XX_SD_BUFFER_SIZE, + * which is more than nbytes, so done. + */ + nbytes = 0; + } + } + } + + pd->pd_ctl.w = PD_CTL_HOST_READY | + ((crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AEAD) ? + PD_CTL_HASH_FINAL : 0); + pd->pd_ctl_len.w = 0x00400000 | (assoclen + datalen); + pd_uinfo->state = PD_ENTRY_INUSE | (is_busy ? PD_ENTRY_BUSY : 0); + + wmb(); + /* write any value to push engine to read a pd */ + writel(0, dev->ce_base + CRYPTO4XX_INT_DESCR_RD); + writel(1, dev->ce_base + CRYPTO4XX_INT_DESCR_RD); + return is_busy ? -EBUSY : -EINPROGRESS; +} + +/* + * Algorithm Registration Functions + */ +static void crypto4xx_ctx_init(struct crypto4xx_alg *amcc_alg, + struct crypto4xx_ctx *ctx) +{ + ctx->dev = amcc_alg->dev; + ctx->sa_in = NULL; + ctx->sa_out = NULL; + ctx->sa_len = 0; +} + +static int crypto4xx_sk_init(struct crypto_skcipher *sk) +{ + struct skcipher_alg *alg = crypto_skcipher_alg(sk); + struct crypto4xx_alg *amcc_alg; + struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(sk); + + if (alg->base.cra_flags & CRYPTO_ALG_NEED_FALLBACK) { + ctx->sw_cipher.cipher = + crypto_alloc_sync_skcipher(alg->base.cra_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->sw_cipher.cipher)) + return PTR_ERR(ctx->sw_cipher.cipher); + } + + amcc_alg = container_of(alg, struct crypto4xx_alg, alg.u.cipher); + crypto4xx_ctx_init(amcc_alg, ctx); + return 0; +} + +static void crypto4xx_common_exit(struct crypto4xx_ctx *ctx) +{ + crypto4xx_free_sa(ctx); +} + +static void crypto4xx_sk_exit(struct crypto_skcipher *sk) +{ + struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(sk); + + crypto4xx_common_exit(ctx); + if (ctx->sw_cipher.cipher) + crypto_free_sync_skcipher(ctx->sw_cipher.cipher); +} + +static int crypto4xx_aead_init(struct crypto_aead *tfm) +{ + struct aead_alg *alg = crypto_aead_alg(tfm); + struct crypto4xx_ctx *ctx = crypto_aead_ctx(tfm); + struct crypto4xx_alg *amcc_alg; + + ctx->sw_cipher.aead = crypto_alloc_aead(alg->base.cra_name, 0, + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC); + if (IS_ERR(ctx->sw_cipher.aead)) + return PTR_ERR(ctx->sw_cipher.aead); + + amcc_alg = container_of(alg, struct crypto4xx_alg, alg.u.aead); + crypto4xx_ctx_init(amcc_alg, ctx); + crypto_aead_set_reqsize(tfm, max(sizeof(struct aead_request) + 32 + + crypto_aead_reqsize(ctx->sw_cipher.aead), + sizeof(struct crypto4xx_aead_reqctx))); + return 0; +} + +static void crypto4xx_aead_exit(struct crypto_aead *tfm) +{ + struct crypto4xx_ctx *ctx = crypto_aead_ctx(tfm); + + crypto4xx_common_exit(ctx); + crypto_free_aead(ctx->sw_cipher.aead); +} + +static int crypto4xx_register_alg(struct crypto4xx_device *sec_dev, + struct crypto4xx_alg_common *crypto_alg, + int array_size) +{ + struct crypto4xx_alg *alg; + int i; + int rc = 0; + + for (i = 0; i < array_size; i++) { + alg = kzalloc_obj(struct crypto4xx_alg); + if (!alg) + return -ENOMEM; + + alg->alg = crypto_alg[i]; + alg->dev = sec_dev; + + switch (alg->alg.type) { + case CRYPTO_ALG_TYPE_AEAD: + rc = crypto_register_aead(&alg->alg.u.aead); + break; + + default: + rc = crypto_register_skcipher(&alg->alg.u.cipher); + break; + } + + if (rc) + kfree(alg); + else + list_add_tail(&alg->entry, &sec_dev->alg_list); + } + + return 0; +} + +static void crypto4xx_unregister_alg(struct crypto4xx_device *sec_dev) +{ + struct crypto4xx_alg *alg, *tmp; + + list_for_each_entry_safe(alg, tmp, &sec_dev->alg_list, entry) { + list_del(&alg->entry); + switch (alg->alg.type) { + case CRYPTO_ALG_TYPE_AEAD: + crypto_unregister_aead(&alg->alg.u.aead); + break; + + default: + crypto_unregister_skcipher(&alg->alg.u.cipher); + } + kfree(alg); + } +} + +static void crypto4xx_bh_tasklet_cb(unsigned long data) +{ + struct device *dev = (struct device *)data; + struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev); + struct pd_uinfo *pd_uinfo; + struct ce_pd *pd; + u32 tail = core_dev->dev->pdr_tail; + u32 head = core_dev->dev->pdr_head; + + do { + pd_uinfo = &core_dev->dev->pdr_uinfo[tail]; + pd = &core_dev->dev->pdr[tail]; + if ((pd_uinfo->state & PD_ENTRY_INUSE) && + ((READ_ONCE(pd->pd_ctl.w) & + (PD_CTL_PE_DONE | PD_CTL_HOST_READY)) == + PD_CTL_PE_DONE)) { + crypto4xx_pd_done(core_dev->dev, tail); + tail = crypto4xx_put_pd_to_pdr(core_dev->dev, tail); + } else { + /* if tail not done, break */ + break; + } + } while (head != tail); +} + +/* + * Top Half of isr. + */ +static inline irqreturn_t crypto4xx_interrupt_handler(int irq, void *data, + u32 clr_val) +{ + struct device *dev = data; + struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev); + + writel(clr_val, core_dev->dev->ce_base + CRYPTO4XX_INT_CLR); + tasklet_schedule(&core_dev->tasklet); + + return IRQ_HANDLED; +} + +static irqreturn_t crypto4xx_ce_interrupt_handler(int irq, void *data) +{ + return crypto4xx_interrupt_handler(irq, data, PPC4XX_INTERRUPT_CLR); +} + +static irqreturn_t crypto4xx_ce_interrupt_handler_revb(int irq, void *data) +{ + return crypto4xx_interrupt_handler(irq, data, PPC4XX_INTERRUPT_CLR | + PPC4XX_TMO_ERR_INT); +} + +/* + * Supported Crypto Algorithms + */ +static struct crypto4xx_alg_common crypto4xx_alg[] = { + /* Crypto AES modes */ + { .type = CRYPTO_ALG_TYPE_SKCIPHER, .u.cipher = { + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-ppc4xx", + .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct crypto4xx_ctx), + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_IV_SIZE, + .setkey = crypto4xx_setkey_aes_cbc, + .encrypt = crypto4xx_encrypt_iv_block, + .decrypt = crypto4xx_decrypt_iv_block, + .init = crypto4xx_sk_init, + .exit = crypto4xx_sk_exit, + } }, + { .type = CRYPTO_ALG_TYPE_SKCIPHER, .u.cipher = { + .base = { + .cra_name = "ctr(aes)", + .cra_driver_name = "ctr-aes-ppc4xx", + .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct crypto4xx_ctx), + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_IV_SIZE, + .setkey = crypto4xx_setkey_aes_ctr, + .encrypt = crypto4xx_encrypt_ctr, + .decrypt = crypto4xx_decrypt_ctr, + .init = crypto4xx_sk_init, + .exit = crypto4xx_sk_exit, + } }, + { .type = CRYPTO_ALG_TYPE_SKCIPHER, .u.cipher = { + .base = { + .cra_name = "rfc3686(ctr(aes))", + .cra_driver_name = "rfc3686-ctr-aes-ppc4xx", + .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct crypto4xx_ctx), + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + .setkey = crypto4xx_setkey_rfc3686, + .encrypt = crypto4xx_rfc3686_encrypt, + .decrypt = crypto4xx_rfc3686_decrypt, + .init = crypto4xx_sk_init, + .exit = crypto4xx_sk_exit, + } }, + { .type = CRYPTO_ALG_TYPE_SKCIPHER, .u.cipher = { + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-ppc4xx", + .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct crypto4xx_ctx), + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = crypto4xx_setkey_aes_ecb, + .encrypt = crypto4xx_encrypt_noiv_block, + .decrypt = crypto4xx_decrypt_noiv_block, + .init = crypto4xx_sk_init, + .exit = crypto4xx_sk_exit, + } }, + + /* AEAD */ + { .type = CRYPTO_ALG_TYPE_AEAD, .u.aead = { + .setkey = crypto4xx_setkey_aes_ccm, + .setauthsize = crypto4xx_setauthsize_aead, + .encrypt = crypto4xx_encrypt_aes_ccm, + .decrypt = crypto4xx_decrypt_aes_ccm, + .init = crypto4xx_aead_init, + .exit = crypto4xx_aead_exit, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = 16, + .base = { + .cra_name = "ccm(aes)", + .cra_driver_name = "ccm-aes-ppc4xx", + .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct crypto4xx_ctx), + .cra_module = THIS_MODULE, + }, + } }, + { .type = CRYPTO_ALG_TYPE_AEAD, .u.aead = { + .setkey = crypto4xx_setkey_aes_gcm, + .setauthsize = crypto4xx_setauthsize_aead, + .encrypt = crypto4xx_encrypt_aes_gcm, + .decrypt = crypto4xx_decrypt_aes_gcm, + .init = crypto4xx_aead_init, + .exit = crypto4xx_aead_exit, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = 16, + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "gcm-aes-ppc4xx", + .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct crypto4xx_ctx), + .cra_module = THIS_MODULE, + }, + } }, +}; + +/* + * Module Initialization Routine + */ +static int crypto4xx_probe(struct platform_device *ofdev) +{ + int rc; + struct device *dev = &ofdev->dev; + struct crypto4xx_core_device *core_dev; + struct device_node *np; + u32 pvr; + bool is_revb = true; + + np = of_find_compatible_node(NULL, NULL, "amcc,ppc460ex-crypto"); + if (np) { + mtdcri(SDR0, PPC460EX_SDR0_SRST, + mfdcri(SDR0, PPC460EX_SDR0_SRST) | PPC460EX_CE_RESET); + mtdcri(SDR0, PPC460EX_SDR0_SRST, + mfdcri(SDR0, PPC460EX_SDR0_SRST) & ~PPC460EX_CE_RESET); + } else { + np = of_find_compatible_node(NULL, NULL, "amcc,ppc405ex-crypto"); + if (np) { + mtdcri(SDR0, PPC405EX_SDR0_SRST, + mfdcri(SDR0, PPC405EX_SDR0_SRST) | PPC405EX_CE_RESET); + mtdcri(SDR0, PPC405EX_SDR0_SRST, + mfdcri(SDR0, PPC405EX_SDR0_SRST) & ~PPC405EX_CE_RESET); + is_revb = false; + } else { + np = of_find_compatible_node(NULL, NULL, "amcc,ppc460sx-crypto"); + if (np) { + mtdcri(SDR0, PPC460SX_SDR0_SRST, + mfdcri(SDR0, PPC460SX_SDR0_SRST) | PPC460SX_CE_RESET); + mtdcri(SDR0, PPC460SX_SDR0_SRST, + mfdcri(SDR0, PPC460SX_SDR0_SRST) & ~PPC460SX_CE_RESET); + } else { + printk(KERN_ERR "Crypto Function Not supported!\n"); + return -EINVAL; + } + } + } + + of_node_put(np); + + core_dev = devm_kzalloc( + &ofdev->dev, sizeof(struct crypto4xx_core_device), GFP_KERNEL); + if (!core_dev) + return -ENOMEM; + + dev_set_drvdata(dev, core_dev); + core_dev->ofdev = ofdev; + core_dev->dev = devm_kzalloc( + &ofdev->dev, sizeof(struct crypto4xx_device), GFP_KERNEL); + if (!core_dev->dev) + return -ENOMEM; + + core_dev->dev->ce_base = devm_platform_ioremap_resource(ofdev, 0); + if (IS_ERR(core_dev->dev->ce_base)) + return PTR_ERR(core_dev->dev->ce_base); + + /* + * Older version of 460EX/GT have a hardware bug. + * Hence they do not support H/W based security intr coalescing + */ + pvr = mfspr(SPRN_PVR); + if (is_revb && ((pvr >> 4) == 0x130218A)) { + u32 min = PVR_MIN(pvr); + + if (min < 4) { + dev_info(dev, "RevA detected - disable interrupt coalescing\n"); + is_revb = false; + } + } + + core_dev->dev->core_dev = core_dev; + core_dev->dev->is_revb = is_revb; + core_dev->device = dev; + spin_lock_init(&core_dev->lock); + INIT_LIST_HEAD(&core_dev->dev->alg_list); + ratelimit_default_init(&core_dev->dev->aead_ratelimit); + rc = crypto4xx_build_sdr(core_dev->dev); + if (rc) + goto err_build_sdr; + rc = crypto4xx_build_pdr(core_dev->dev); + if (rc) + goto err_build_sdr; + + rc = crypto4xx_build_gdr(core_dev->dev); + if (rc) + goto err_build_sdr; + + /* Init tasklet for bottom half processing */ + tasklet_init(&core_dev->tasklet, crypto4xx_bh_tasklet_cb, + (unsigned long) dev); + + /* Register for Crypto isr, Crypto Engine IRQ */ + core_dev->irq = platform_get_irq(ofdev, 0); + if (core_dev->irq < 0) { + rc = core_dev->irq; + goto err_tasklet; + } + rc = request_irq(core_dev->irq, + is_revb ? crypto4xx_ce_interrupt_handler_revb : + crypto4xx_ce_interrupt_handler, + 0, KBUILD_MODNAME, dev); + if (rc) + goto err_tasklet; + + /* need to setup pdr, rdr, gdr and sdr before this */ + crypto4xx_hw_init(core_dev->dev); + + /* Register security algorithms with Linux CryptoAPI */ + rc = crypto4xx_register_alg(core_dev->dev, crypto4xx_alg, + ARRAY_SIZE(crypto4xx_alg)); + if (rc) + goto err_irq; + + ppc4xx_trng_probe(core_dev); + return 0; + +err_irq: + free_irq(core_dev->irq, dev); +err_tasklet: + tasklet_kill(&core_dev->tasklet); +err_build_sdr: + crypto4xx_destroy_sdr(core_dev->dev); + crypto4xx_destroy_gdr(core_dev->dev); + crypto4xx_destroy_pdr(core_dev->dev); + return rc; +} + +static void crypto4xx_remove(struct platform_device *ofdev) +{ + struct device *dev = &ofdev->dev; + struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev); + + ppc4xx_trng_remove(core_dev); + + /* + * Free IRQ before killing the tasklet to prevent the interrupt + * handler from rescheduling the tasklet after it has been killed. + */ + free_irq(core_dev->irq, dev); + tasklet_kill(&core_dev->tasklet); + /* Un-register with Linux CryptoAPI */ + crypto4xx_unregister_alg(core_dev->dev); + /* Free all allocated memory */ + crypto4xx_stop_all(core_dev); +} + +static const struct of_device_id crypto4xx_match[] = { + { .compatible = "amcc,ppc4xx-crypto",}, + { }, +}; +MODULE_DEVICE_TABLE(of, crypto4xx_match); + +static struct platform_driver crypto4xx_driver = { + .driver = { + .name = KBUILD_MODNAME, + .of_match_table = crypto4xx_match, + }, + .probe = crypto4xx_probe, + .remove = crypto4xx_remove, +}; + +module_platform_driver(crypto4xx_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("James Hsiao "); +MODULE_DESCRIPTION("Driver for AMCC PPC4xx crypto accelerator"); diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h new file mode 100644 index 000000000..66a95733c --- /dev/null +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -0,0 +1,235 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * AMCC SoC PPC4xx Crypto Driver + * + * Copyright (c) 2008 Applied Micro Circuits Corporation. + * All rights reserved. James Hsiao + * + * This is the header file for AMCC Crypto offload Linux device driver for + * use with Linux CryptoAPI. + + */ + +#ifndef __CRYPTO4XX_CORE_H__ +#define __CRYPTO4XX_CORE_H__ + +#include +#include +#include +#include +#include "crypto4xx_reg_def.h" +#include "crypto4xx_sa.h" + +#define PPC460SX_SDR0_SRST 0x201 +#define PPC405EX_SDR0_SRST 0x200 +#define PPC460EX_SDR0_SRST 0x201 +#define PPC460EX_CE_RESET 0x08000000 +#define PPC460SX_CE_RESET 0x20000000 +#define PPC405EX_CE_RESET 0x00000008 + +#define CRYPTO4XX_CRYPTO_PRIORITY 300 +#define PPC4XX_NUM_PD 256 +#define PPC4XX_LAST_PD (PPC4XX_NUM_PD - 1) +#define PPC4XX_NUM_GD 1024 +#define PPC4XX_LAST_GD (PPC4XX_NUM_GD - 1) +#define PPC4XX_NUM_SD 256 +#define PPC4XX_LAST_SD (PPC4XX_NUM_SD - 1) +#define PPC4XX_SD_BUFFER_SIZE 2048 + +#define PD_ENTRY_BUSY BIT(1) +#define PD_ENTRY_INUSE BIT(0) +#define PD_ENTRY_FREE 0 +#define ERING_WAS_FULL 0xffffffff + +struct crypto4xx_device; + +union shadow_sa_buf { + struct dynamic_sa_ctl sa; + + /* alloc 256 bytes which is enough for any kind of dynamic sa */ + u8 buf[256]; +} __packed; + +struct pd_uinfo { + struct crypto4xx_device *dev; + u32 state; + u32 first_gd; /* first gather discriptor + used by this packet */ + u32 num_gd; /* number of gather discriptor + used by this packet */ + u32 first_sd; /* first scatter discriptor + used by this packet */ + u32 num_sd; /* number of scatter discriptors + used by this packet */ + struct dynamic_sa_ctl *sa_va; /* shadow sa */ + struct sa_state_record *sr_va; /* state record for shadow sa */ + u32 sr_pa; + struct scatterlist *dest_va; + struct crypto_async_request *async_req; /* base crypto request + for this packet */ +}; + +struct crypto4xx_device { + struct crypto4xx_core_device *core_dev; + void __iomem *ce_base; + void __iomem *trng_base; + + struct ce_pd *pdr; /* base address of packet descriptor ring */ + dma_addr_t pdr_pa; /* physical address of pdr_base_register */ + struct ce_gd *gdr; /* gather descriptor ring */ + dma_addr_t gdr_pa; /* physical address of gdr_base_register */ + struct ce_sd *sdr; /* scatter descriptor ring */ + dma_addr_t sdr_pa; /* physical address of sdr_base_register */ + void *scatter_buffer_va; + dma_addr_t scatter_buffer_pa; + + union shadow_sa_buf *shadow_sa_pool; + dma_addr_t shadow_sa_pool_pa; + struct sa_state_record *shadow_sr_pool; + dma_addr_t shadow_sr_pool_pa; + u32 pdr_tail; + u32 pdr_head; + u32 gdr_tail; + u32 gdr_head; + u32 sdr_tail; + u32 sdr_head; + struct pd_uinfo *pdr_uinfo; + struct list_head alg_list; /* List of algorithm supported + by this device */ + struct ratelimit_state aead_ratelimit; + bool is_revb; +}; + +struct crypto4xx_core_device { + struct device *device; + struct platform_device *ofdev; + struct crypto4xx_device *dev; + struct hwrng *trng; + u32 int_status; + int irq; + struct tasklet_struct tasklet; + spinlock_t lock; +}; + +struct crypto4xx_ctx { + struct crypto4xx_device *dev; + struct dynamic_sa_ctl *sa_in; + struct dynamic_sa_ctl *sa_out; + __le32 iv_nonce; + u32 sa_len; + union { + struct crypto_sync_skcipher *cipher; + struct crypto_aead *aead; + } sw_cipher; +}; + +struct crypto4xx_aead_reqctx { + struct scatterlist dst[2]; +}; + +struct crypto4xx_alg_common { + u32 type; + union { + struct skcipher_alg cipher; + struct aead_alg aead; + } u; +}; + +struct crypto4xx_alg { + struct list_head entry; + struct crypto4xx_alg_common alg; + struct crypto4xx_device *dev; +}; + +#if IS_ENABLED(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION >= 120000 +#define BUILD_PD_ACCESS __attribute__((access(read_only, 6, 7))) +#else +#define BUILD_PD_ACCESS +#endif + +int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size); +void crypto4xx_free_sa(struct crypto4xx_ctx *ctx); +int crypto4xx_build_pd(struct crypto_async_request *req, + struct crypto4xx_ctx *ctx, + struct scatterlist *src, + struct scatterlist *dst, + const unsigned int datalen, + const void *iv, const u32 iv_len, + const struct dynamic_sa_ctl *sa, + const unsigned int sa_len, + const unsigned int assoclen, + struct scatterlist *dst_tmp) BUILD_PD_ACCESS; +int crypto4xx_setkey_aes_cbc(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_setkey_aes_ctr(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_setkey_aes_ecb(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_setkey_rfc3686(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_encrypt_ctr(struct skcipher_request *req); +int crypto4xx_decrypt_ctr(struct skcipher_request *req); +int crypto4xx_encrypt_iv_stream(struct skcipher_request *req); +int crypto4xx_decrypt_iv_stream(struct skcipher_request *req); +int crypto4xx_encrypt_iv_block(struct skcipher_request *req); +int crypto4xx_decrypt_iv_block(struct skcipher_request *req); +int crypto4xx_encrypt_noiv_block(struct skcipher_request *req); +int crypto4xx_decrypt_noiv_block(struct skcipher_request *req); +int crypto4xx_rfc3686_encrypt(struct skcipher_request *req); +int crypto4xx_rfc3686_decrypt(struct skcipher_request *req); + +/* + * Note: Only use this function to copy items that is word aligned. + */ +static inline void crypto4xx_memcpy_swab32(u32 *dst, const void *buf, + size_t len) +{ + for (; len >= 4; buf += 4, len -= 4) + *dst++ = __swab32p((u32 *) buf); + + if (len) { + const u8 *tmp = (u8 *)buf; + + switch (len) { + case 3: + *dst = (tmp[2] << 16) | + (tmp[1] << 8) | + tmp[0]; + break; + case 2: + *dst = (tmp[1] << 8) | + tmp[0]; + break; + case 1: + *dst = tmp[0]; + break; + default: + break; + } + } +} + +static inline void crypto4xx_memcpy_from_le32(u32 *dst, const void *buf, + size_t len) +{ + crypto4xx_memcpy_swab32(dst, buf, len); +} + +static inline void crypto4xx_memcpy_to_le32(__le32 *dst, const void *buf, + size_t len) +{ + crypto4xx_memcpy_swab32((u32 *)dst, buf, len); +} + +int crypto4xx_setauthsize_aead(struct crypto_aead *ciper, + unsigned int authsize); +int crypto4xx_setkey_aes_ccm(struct crypto_aead *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_encrypt_aes_ccm(struct aead_request *req); +int crypto4xx_decrypt_aes_ccm(struct aead_request *req); +int crypto4xx_setkey_aes_gcm(struct crypto_aead *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_encrypt_aes_gcm(struct aead_request *req); +int crypto4xx_decrypt_aes_gcm(struct aead_request *req); + +#endif diff --git a/drivers/crypto/amcc/crypto4xx_reg_def.h b/drivers/crypto/amcc/crypto4xx_reg_def.h new file mode 100644 index 000000000..73d626308 --- /dev/null +++ b/drivers/crypto/amcc/crypto4xx_reg_def.h @@ -0,0 +1,271 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * AMCC SoC PPC4xx Crypto Driver + * + * Copyright (c) 2008 Applied Micro Circuits Corporation. + * All rights reserved. James Hsiao + * + * This filr defines the register set for Security Subsystem + */ + +#ifndef __CRYPTO4XX_REG_DEF_H__ +#define __CRYPTO4XX_REG_DEF_H__ + +/* CRYPTO4XX Register offset */ +#define CRYPTO4XX_DESCRIPTOR 0x00000000 +#define CRYPTO4XX_CTRL_STAT 0x00000000 +#define CRYPTO4XX_SOURCE 0x00000004 +#define CRYPTO4XX_DEST 0x00000008 +#define CRYPTO4XX_SA 0x0000000C +#define CRYPTO4XX_SA_LENGTH 0x00000010 +#define CRYPTO4XX_LENGTH 0x00000014 + +#define CRYPTO4XX_PE_DMA_CFG 0x00000040 +#define CRYPTO4XX_PE_DMA_STAT 0x00000044 +#define CRYPTO4XX_PDR_BASE 0x00000048 +#define CRYPTO4XX_RDR_BASE 0x0000004c +#define CRYPTO4XX_RING_SIZE 0x00000050 +#define CRYPTO4XX_RING_CTRL 0x00000054 +#define CRYPTO4XX_INT_RING_STAT 0x00000058 +#define CRYPTO4XX_EXT_RING_STAT 0x0000005c +#define CRYPTO4XX_IO_THRESHOLD 0x00000060 +#define CRYPTO4XX_GATH_RING_BASE 0x00000064 +#define CRYPTO4XX_SCAT_RING_BASE 0x00000068 +#define CRYPTO4XX_PART_RING_SIZE 0x0000006c +#define CRYPTO4XX_PART_RING_CFG 0x00000070 + +#define CRYPTO4XX_PDR_BASE_UADDR 0x00000080 +#define CRYPTO4XX_RDR_BASE_UADDR 0x00000084 +#define CRYPTO4XX_PKT_SRC_UADDR 0x00000088 +#define CRYPTO4XX_PKT_DEST_UADDR 0x0000008c +#define CRYPTO4XX_SA_UADDR 0x00000090 +#define CRYPTO4XX_GATH_RING_BASE_UADDR 0x000000A0 +#define CRYPTO4XX_SCAT_RING_BASE_UADDR 0x000000A4 + +#define CRYPTO4XX_SEQ_RD 0x00000408 +#define CRYPTO4XX_SEQ_MASK_RD 0x0000040C + +#define CRYPTO4XX_SA_CMD_0 0x00010600 +#define CRYPTO4XX_SA_CMD_1 0x00010604 + +#define CRYPTO4XX_STATE_PTR 0x000106dc +#define CRYPTO4XX_STATE_IV 0x00010700 +#define CRYPTO4XX_STATE_HASH_BYTE_CNT_0 0x00010710 +#define CRYPTO4XX_STATE_HASH_BYTE_CNT_1 0x00010714 + +#define CRYPTO4XX_STATE_IDIGEST_0 0x00010718 +#define CRYPTO4XX_STATE_IDIGEST_1 0x0001071c + +#define CRYPTO4XX_DATA_IN 0x00018000 +#define CRYPTO4XX_DATA_OUT 0x0001c000 + +#define CRYPTO4XX_INT_UNMASK_STAT 0x000500a0 +#define CRYPTO4XX_INT_MASK_STAT 0x000500a4 +#define CRYPTO4XX_INT_CLR 0x000500a4 +#define CRYPTO4XX_INT_EN 0x000500a8 + +#define CRYPTO4XX_INT_PKA 0x00000002 +#define CRYPTO4XX_INT_PDR_DONE 0x00008000 +#define CRYPTO4XX_INT_MA_WR_ERR 0x00020000 +#define CRYPTO4XX_INT_MA_RD_ERR 0x00010000 +#define CRYPTO4XX_INT_PE_ERR 0x00000200 +#define CRYPTO4XX_INT_USER_DMA_ERR 0x00000040 +#define CRYPTO4XX_INT_SLAVE_ERR 0x00000010 +#define CRYPTO4XX_INT_MASTER_ERR 0x00000008 +#define CRYPTO4XX_INT_ERROR 0x00030258 + +#define CRYPTO4XX_INT_CFG 0x000500ac +#define CRYPTO4XX_INT_DESCR_RD 0x000500b0 +#define CRYPTO4XX_INT_DESCR_CNT 0x000500b4 +#define CRYPTO4XX_INT_TIMEOUT_CNT 0x000500b8 + +#define CRYPTO4XX_DEVICE_CTRL 0x00060080 +#define CRYPTO4XX_DEVICE_ID 0x00060084 +#define CRYPTO4XX_DEVICE_INFO 0x00060088 +#define CRYPTO4XX_DMA_USER_SRC 0x00060094 +#define CRYPTO4XX_DMA_USER_DEST 0x00060098 +#define CRYPTO4XX_DMA_USER_CMD 0x0006009C + +#define CRYPTO4XX_DMA_CFG 0x000600d4 +#define CRYPTO4XX_BYTE_ORDER_CFG 0x000600d8 +#define CRYPTO4XX_ENDIAN_CFG 0x000600d8 + +#define CRYPTO4XX_PRNG_CTRL 0x00070004 +#define CRYPTO4XX_PRNG_SEED_L 0x00070008 +#define CRYPTO4XX_PRNG_SEED_H 0x0007000c +/* + * Initialize CRYPTO ENGINE registers, and memory bases. + */ +#define PPC4XX_PDR_POLL 0x3ff +#define PPC4XX_OUTPUT_THRESHOLD 2 +#define PPC4XX_INPUT_THRESHOLD 2 +#define PPC4XX_PD_SIZE 6 +#define PPC4XX_CTX_DONE_INT 0x2000 +#define PPC4XX_PD_DONE_INT 0x8000 +#define PPC4XX_TMO_ERR_INT 0x40000 +#define PPC4XX_BYTE_ORDER 0x22222 +#define PPC4XX_INTERRUPT_CLR 0x3ffff +#define PPC4XX_PRNG_CTRL_AUTO_EN 0x3 +#define PPC4XX_DC_3DES_EN 1 +#define PPC4XX_TRNG_EN 0x00020000 +#define PPC4XX_INT_DESCR_CNT 7 +#define PPC4XX_INT_TIMEOUT_CNT 0 +#define PPC4XX_INT_TIMEOUT_CNT_REVB 0x3FF +#define PPC4XX_INT_CFG 1 +/* + * all follow define are ad hoc + */ +#define PPC4XX_RING_RETRY 100 +#define PPC4XX_RING_POLL 100 +#define PPC4XX_SDR_SIZE PPC4XX_NUM_SD +#define PPC4XX_GDR_SIZE PPC4XX_NUM_GD + +/* + * Generic Security Association (SA) with all possible fields. These will + * never likely used except for reference purpose. These structure format + * can be not changed as the hardware expects them to be layout as defined. + * Field can be removed or reduced but ordering can not be changed. + */ +#define CRYPTO4XX_DMA_CFG_OFFSET 0x40 +union ce_pe_dma_cfg { + struct { + u32 rsv:7; + u32 dir_host:1; + u32 rsv1:2; + u32 bo_td_en:1; + u32 dis_pdr_upd:1; + u32 bo_sgpd_en:1; + u32 bo_data_en:1; + u32 bo_sa_en:1; + u32 bo_pd_en:1; + u32 rsv2:4; + u32 dynamic_sa_en:1; + u32 pdr_mode:2; + u32 pe_mode:1; + u32 rsv3:5; + u32 reset_sg:1; + u32 reset_pdr:1; + u32 reset_pe:1; + } bf; + u32 w; +} __attribute__((packed)); + +#define CRYPTO4XX_PDR_BASE_OFFSET 0x48 +#define CRYPTO4XX_RDR_BASE_OFFSET 0x4c +#define CRYPTO4XX_RING_SIZE_OFFSET 0x50 +union ce_ring_size { + struct { + u32 ring_offset:16; + u32 rsv:6; + u32 ring_size:10; + } bf; + u32 w; +} __attribute__((packed)); + +#define CRYPTO4XX_RING_CONTROL_OFFSET 0x54 +union ce_ring_control { + struct { + u32 continuous:1; + u32 rsv:5; + u32 ring_retry_divisor:10; + u32 rsv1:4; + u32 ring_poll_divisor:10; + } bf; + u32 w; +} __attribute__((packed)); + +#define CRYPTO4XX_IO_THRESHOLD_OFFSET 0x60 +union ce_io_threshold { + struct { + u32 rsv:6; + u32 output_threshold:10; + u32 rsv1:6; + u32 input_threshold:10; + } bf; + u32 w; +} __attribute__((packed)); + +#define CRYPTO4XX_GATHER_RING_BASE_OFFSET 0x64 +#define CRYPTO4XX_SCATTER_RING_BASE_OFFSET 0x68 + +union ce_part_ring_size { + struct { + u32 sdr_size:16; + u32 gdr_size:16; + } bf; + u32 w; +} __attribute__((packed)); + +#define MAX_BURST_SIZE_32 0 +#define MAX_BURST_SIZE_64 1 +#define MAX_BURST_SIZE_128 2 +#define MAX_BURST_SIZE_256 3 + +/* gather descriptor control length */ +struct gd_ctl_len { + u32 len:16; + u32 rsv:14; + u32 done:1; + u32 ready:1; +} __attribute__((packed)); + +struct ce_gd { + u32 ptr; + struct gd_ctl_len ctl_len; +} __attribute__((packed)); + +struct sd_ctl { + u32 ctl:30; + u32 done:1; + u32 rdy:1; +} __attribute__((packed)); + +struct ce_sd { + u32 ptr; + struct sd_ctl ctl; +} __attribute__((packed)); + +#define PD_PAD_CTL_32 0x10 +#define PD_PAD_CTL_64 0x20 +#define PD_PAD_CTL_128 0x40 +#define PD_PAD_CTL_256 0x80 +union ce_pd_ctl { + struct { + u32 pd_pad_ctl:8; + u32 status:8; + u32 next_hdr:8; + u32 rsv:2; + u32 cached_sa:1; + u32 hash_final:1; + u32 init_arc4:1; + u32 rsv1:1; + u32 pe_done:1; + u32 host_ready:1; + } bf; + u32 w; +} __attribute__((packed)); +#define PD_CTL_HASH_FINAL BIT(4) +#define PD_CTL_PE_DONE BIT(1) +#define PD_CTL_HOST_READY BIT(0) + +union ce_pd_ctl_len { + struct { + u32 bypass:8; + u32 pe_done:1; + u32 host_ready:1; + u32 rsv:2; + u32 pkt_len:20; + } bf; + u32 w; +} __attribute__((packed)); + +struct ce_pd { + union ce_pd_ctl pd_ctl; + u32 src; + u32 dest; + u32 sa; /* get from ctx->sa_dma_addr */ + u32 sa_len; /* only if dynamic sa is used */ + union ce_pd_ctl_len pd_ctl_len; + +} __attribute__((packed)); +#endif diff --git a/drivers/crypto/amcc/crypto4xx_sa.h b/drivers/crypto/amcc/crypto4xx_sa.h new file mode 100644 index 000000000..e98e4e7ab --- /dev/null +++ b/drivers/crypto/amcc/crypto4xx_sa.h @@ -0,0 +1,309 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * AMCC SoC PPC4xx Crypto Driver + * + * Copyright (c) 2008 Applied Micro Circuits Corporation. + * All rights reserved. James Hsiao + * + * This file defines the security context + * associate format. + */ + +#ifndef __CRYPTO4XX_SA_H__ +#define __CRYPTO4XX_SA_H__ + +#define AES_IV_SIZE 16 + +/* + * Contents of Dynamic Security Association (SA) with all possible fields + */ +union dynamic_sa_contents { + struct { + u32 arc4_state_ptr:1; + u32 arc4_ij_ptr:1; + u32 state_ptr:1; + u32 iv3:1; + u32 iv2:1; + u32 iv1:1; + u32 iv0:1; + u32 seq_num_mask3:1; + u32 seq_num_mask2:1; + u32 seq_num_mask1:1; + u32 seq_num_mask0:1; + u32 seq_num1:1; + u32 seq_num0:1; + u32 spi:1; + u32 outer_size:5; + u32 inner_size:5; + u32 key_size:4; + u32 cmd_size:4; + } bf; + u32 w; +} __attribute__((packed)); + +#define DIR_OUTBOUND 0 +#define DIR_INBOUND 1 +#define SA_OP_GROUP_BASIC 0 +#define SA_OPCODE_ENCRYPT 0 +#define SA_OPCODE_DECRYPT 0 +#define SA_OPCODE_ENCRYPT_HASH 1 +#define SA_OPCODE_HASH_DECRYPT 1 +#define SA_OPCODE_HASH 3 +#define SA_CIPHER_ALG_DES 0 +#define SA_CIPHER_ALG_3DES 1 +#define SA_CIPHER_ALG_ARC4 2 +#define SA_CIPHER_ALG_AES 3 +#define SA_CIPHER_ALG_KASUMI 4 +#define SA_CIPHER_ALG_NULL 15 + +#define SA_HASH_ALG_MD5 0 +#define SA_HASH_ALG_SHA1 1 +#define SA_HASH_ALG_GHASH 12 +#define SA_HASH_ALG_CBC_MAC 14 +#define SA_HASH_ALG_NULL 15 +#define SA_HASH_ALG_SHA1_DIGEST_SIZE 20 + +#define SA_LOAD_HASH_FROM_SA 0 +#define SA_LOAD_HASH_FROM_STATE 2 +#define SA_NOT_LOAD_HASH 3 +#define SA_LOAD_IV_FROM_SA 0 +#define SA_LOAD_IV_FROM_INPUT 1 +#define SA_LOAD_IV_FROM_STATE 2 +#define SA_LOAD_IV_GEN_IV 3 + +#define SA_PAD_TYPE_CONSTANT 2 +#define SA_PAD_TYPE_ZERO 3 +#define SA_PAD_TYPE_TLS 5 +#define SA_PAD_TYPE_DTLS 5 +#define SA_NOT_SAVE_HASH 0 +#define SA_SAVE_HASH 1 +#define SA_NOT_SAVE_IV 0 +#define SA_SAVE_IV 1 +#define SA_HEADER_PROC 1 +#define SA_NO_HEADER_PROC 0 + +union sa_command_0 { + struct { + u32 scatter:1; + u32 gather:1; + u32 save_hash_state:1; + u32 save_iv:1; + u32 load_hash_state:2; + u32 load_iv:2; + u32 digest_len:4; + u32 hdr_proc:1; + u32 extend_pad:1; + u32 stream_cipher_pad:1; + u32 rsv:1; + u32 hash_alg:4; + u32 cipher_alg:4; + u32 pad_type:2; + u32 op_group:2; + u32 dir:1; + u32 opcode:3; + } bf; + u32 w; +} __attribute__((packed)); + +#define CRYPTO_MODE_ECB 0 +#define CRYPTO_MODE_CBC 1 +#define CRYPTO_MODE_OFB 2 +#define CRYPTO_MODE_CFB 3 +#define CRYPTO_MODE_CTR 4 + +#define CRYPTO_FEEDBACK_MODE_NO_FB 0 +#define CRYPTO_FEEDBACK_MODE_64BIT_OFB 0 +#define CRYPTO_FEEDBACK_MODE_8BIT_CFB 1 +#define CRYPTO_FEEDBACK_MODE_1BIT_CFB 2 +#define CRYPTO_FEEDBACK_MODE_128BIT_CFB 3 + +#define SA_AES_KEY_LEN_128 2 +#define SA_AES_KEY_LEN_192 3 +#define SA_AES_KEY_LEN_256 4 + +#define SA_REV2 1 +/* + * The follow defines bits sa_command_1 + * In Basic hash mode this bit define simple hash or hmac. + * In IPsec mode, this bit define muting control. + */ +#define SA_HASH_MODE_HASH 0 +#define SA_HASH_MODE_HMAC 1 +#define SA_MC_ENABLE 0 +#define SA_MC_DISABLE 1 +#define SA_NOT_COPY_HDR 0 +#define SA_COPY_HDR 1 +#define SA_NOT_COPY_PAD 0 +#define SA_COPY_PAD 1 +#define SA_NOT_COPY_PAYLOAD 0 +#define SA_COPY_PAYLOAD 1 +#define SA_EXTENDED_SN_OFF 0 +#define SA_EXTENDED_SN_ON 1 +#define SA_SEQ_MASK_OFF 0 +#define SA_SEQ_MASK_ON 1 + +union sa_command_1 { + struct { + u32 crypto_mode31:1; + u32 save_arc4_state:1; + u32 arc4_stateful:1; + u32 key_len:5; + u32 hash_crypto_offset:8; + u32 sa_rev:2; + u32 byte_offset:1; + u32 hmac_muting:1; + u32 feedback_mode:2; + u32 crypto_mode9_8:2; + u32 extended_seq_num:1; + u32 seq_num_mask:1; + u32 mutable_bit_proc:1; + u32 ip_version:1; + u32 copy_pad:1; + u32 copy_payload:1; + u32 copy_hdr:1; + u32 rsv1:1; + } bf; + u32 w; +} __attribute__((packed)); + +struct dynamic_sa_ctl { + union dynamic_sa_contents sa_contents; + union sa_command_0 sa_command_0; + union sa_command_1 sa_command_1; +} __attribute__((packed)); + +/* + * State Record for Security Association (SA) + */ +struct sa_state_record { + __le32 save_iv[4]; + __le32 save_hash_byte_cnt[2]; + union { + u32 save_digest[16]; /* for MD5/SHA */ + __le32 save_digest_le32[16]; /* GHASH / CBC */ + }; +} __attribute__((packed)); + +/* + * Security Association (SA) for AES128 + * + */ +struct dynamic_sa_aes128 { + struct dynamic_sa_ctl ctrl; + __le32 key[4]; + __le32 iv[4]; /* for CBC, OFC, and CFB mode */ + u32 state_ptr; + u32 reserved; +} __attribute__((packed)); + +#define SA_AES128_LEN (sizeof(struct dynamic_sa_aes128)/4) +#define SA_AES128_CONTENTS 0x3e000042 + +/* + * Security Association (SA) for AES192 + */ +struct dynamic_sa_aes192 { + struct dynamic_sa_ctl ctrl; + __le32 key[6]; + __le32 iv[4]; /* for CBC, OFC, and CFB mode */ + u32 state_ptr; + u32 reserved; +} __attribute__((packed)); + +#define SA_AES192_LEN (sizeof(struct dynamic_sa_aes192)/4) +#define SA_AES192_CONTENTS 0x3e000062 + +/* + * Security Association (SA) for AES256 + */ +struct dynamic_sa_aes256 { + struct dynamic_sa_ctl ctrl; + __le32 key[8]; + __le32 iv[4]; /* for CBC, OFC, and CFB mode */ + u32 state_ptr; + u32 reserved; +} __attribute__((packed)); + +#define SA_AES256_LEN (sizeof(struct dynamic_sa_aes256)/4) +#define SA_AES256_CONTENTS 0x3e000082 +#define SA_AES_CONTENTS 0x3e000002 + +/* + * Security Association (SA) for AES128 CCM + */ +struct dynamic_sa_aes128_ccm { + struct dynamic_sa_ctl ctrl; + __le32 key[4]; + __le32 iv[4]; + u32 state_ptr; + u32 reserved; +} __packed; +#define SA_AES128_CCM_LEN (sizeof(struct dynamic_sa_aes128_ccm)/4) +#define SA_AES128_CCM_CONTENTS 0x3e000042 +#define SA_AES_CCM_CONTENTS 0x3e000002 + +/* + * Security Association (SA) for AES128_GCM + */ +struct dynamic_sa_aes128_gcm { + struct dynamic_sa_ctl ctrl; + __le32 key[4]; + __le32 inner_digest[4]; + __le32 iv[4]; + u32 state_ptr; + u32 reserved; +} __packed; + +#define SA_AES128_GCM_LEN (sizeof(struct dynamic_sa_aes128_gcm)/4) +#define SA_AES128_GCM_CONTENTS 0x3e000442 +#define SA_AES_GCM_CONTENTS 0x3e000402 + +/* + * Security Association (SA) for HASH160: HMAC-SHA1 + */ +struct dynamic_sa_hash160 { + struct dynamic_sa_ctl ctrl; + __le32 inner_digest[5]; + __le32 outer_digest[5]; + u32 state_ptr; + u32 reserved; +} __attribute__((packed)); +#define SA_HASH160_LEN (sizeof(struct dynamic_sa_hash160)/4) +#define SA_HASH160_CONTENTS 0x2000a502 + +static inline u32 +get_dynamic_sa_offset_state_ptr_field(struct dynamic_sa_ctl *cts) +{ + u32 offset; + + offset = cts->sa_contents.bf.key_size + + cts->sa_contents.bf.inner_size + + cts->sa_contents.bf.outer_size + + cts->sa_contents.bf.spi + + cts->sa_contents.bf.seq_num0 + + cts->sa_contents.bf.seq_num1 + + cts->sa_contents.bf.seq_num_mask0 + + cts->sa_contents.bf.seq_num_mask1 + + cts->sa_contents.bf.seq_num_mask2 + + cts->sa_contents.bf.seq_num_mask3 + + cts->sa_contents.bf.iv0 + + cts->sa_contents.bf.iv1 + + cts->sa_contents.bf.iv2 + + cts->sa_contents.bf.iv3; + + return sizeof(struct dynamic_sa_ctl) + offset * 4; +} + +static inline __le32 *get_dynamic_sa_key_field(struct dynamic_sa_ctl *cts) +{ + return (__le32 *) ((unsigned long)cts + sizeof(struct dynamic_sa_ctl)); +} + +static inline __le32 *get_dynamic_sa_inner_digest(struct dynamic_sa_ctl *cts) +{ + return (__le32 *) ((unsigned long)cts + + sizeof(struct dynamic_sa_ctl) + + cts->sa_contents.bf.key_size * 4); +} + +#endif diff --git a/drivers/crypto/amcc/crypto4xx_trng.c b/drivers/crypto/amcc/crypto4xx_trng.c new file mode 100644 index 000000000..cfd66b779 --- /dev/null +++ b/drivers/crypto/amcc/crypto4xx_trng.c @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Generic PowerPC 44x RNG driver + * + * Copyright 2011 IBM Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "crypto4xx_core.h" +#include "crypto4xx_trng.h" +#include "crypto4xx_reg_def.h" + +#define PPC4XX_TRNG_CTRL 0x0008 +#define PPC4XX_TRNG_CTRL_DALM 0x20 +#define PPC4XX_TRNG_STAT 0x0004 +#define PPC4XX_TRNG_STAT_B 0x1 +#define PPC4XX_TRNG_DATA 0x0000 + +static int ppc4xx_trng_data_present(struct hwrng *rng, int wait) +{ + struct crypto4xx_device *dev = (void *)rng->priv; + int busy, i, present = 0; + + for (i = 0; i < 20; i++) { + busy = (in_le32(dev->trng_base + PPC4XX_TRNG_STAT) & + PPC4XX_TRNG_STAT_B); + if (!busy || !wait) { + present = 1; + break; + } + udelay(10); + } + return present; +} + +static int ppc4xx_trng_data_read(struct hwrng *rng, u32 *data) +{ + struct crypto4xx_device *dev = (void *)rng->priv; + *data = in_le32(dev->trng_base + PPC4XX_TRNG_DATA); + return 4; +} + +static void ppc4xx_trng_enable(struct crypto4xx_device *dev, bool enable) +{ + u32 device_ctrl; + + device_ctrl = readl(dev->ce_base + CRYPTO4XX_DEVICE_CTRL); + if (enable) + device_ctrl |= PPC4XX_TRNG_EN; + else + device_ctrl &= ~PPC4XX_TRNG_EN; + writel(device_ctrl, dev->ce_base + CRYPTO4XX_DEVICE_CTRL); +} + +static const struct of_device_id ppc4xx_trng_match[] = { + { .compatible = "ppc4xx-rng", }, + { .compatible = "amcc,ppc460ex-rng", }, + { .compatible = "amcc,ppc440epx-rng", }, + {}, +}; + +void ppc4xx_trng_probe(struct crypto4xx_core_device *core_dev) +{ + struct crypto4xx_device *dev = core_dev->dev; + struct device_node *trng = NULL; + struct hwrng *rng = NULL; + int err; + + /* Find the TRNG device node and map it */ + trng = of_find_matching_node(NULL, ppc4xx_trng_match); + if (!trng || !of_device_is_available(trng)) { + of_node_put(trng); + return; + } + + dev->trng_base = of_iomap(trng, 0); + of_node_put(trng); + if (!dev->trng_base) + goto err_out; + + rng = kzalloc_obj(*rng); + if (!rng) + goto err_out; + + rng->name = KBUILD_MODNAME; + rng->data_present = ppc4xx_trng_data_present; + rng->data_read = ppc4xx_trng_data_read; + rng->priv = (unsigned long) dev; + core_dev->trng = rng; + ppc4xx_trng_enable(dev, true); + out_le32(dev->trng_base + PPC4XX_TRNG_CTRL, PPC4XX_TRNG_CTRL_DALM); + err = devm_hwrng_register(core_dev->device, core_dev->trng); + if (err) { + ppc4xx_trng_enable(dev, false); + dev_err(core_dev->device, "failed to register hwrng (%d).\n", + err); + goto err_out; + } + return; + +err_out: + iounmap(dev->trng_base); + kfree(rng); + dev->trng_base = NULL; + core_dev->trng = NULL; +} + +void ppc4xx_trng_remove(struct crypto4xx_core_device *core_dev) +{ + if (core_dev && core_dev->trng) { + struct crypto4xx_device *dev = core_dev->dev; + + devm_hwrng_unregister(core_dev->device, core_dev->trng); + ppc4xx_trng_enable(dev, false); + iounmap(dev->trng_base); + kfree(core_dev->trng); + } +} + +MODULE_ALIAS("ppc4xx_rng"); diff --git a/drivers/crypto/amcc/crypto4xx_trng.h b/drivers/crypto/amcc/crypto4xx_trng.h new file mode 100644 index 000000000..735671627 --- /dev/null +++ b/drivers/crypto/amcc/crypto4xx_trng.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * AMCC SoC PPC4xx Crypto Driver + * + * Copyright (c) 2008 Applied Micro Circuits Corporation. + * All rights reserved. James Hsiao + * + * This file defines the security context + * associate format. + */ + +#ifndef __CRYPTO4XX_TRNG_H__ +#define __CRYPTO4XX_TRNG_H__ + +#ifdef CONFIG_HW_RANDOM_PPC4XX +void ppc4xx_trng_probe(struct crypto4xx_core_device *core_dev); +void ppc4xx_trng_remove(struct crypto4xx_core_device *core_dev); +#else +static inline void ppc4xx_trng_probe( + struct crypto4xx_core_device *dev __maybe_unused) { } +static inline void ppc4xx_trng_remove( + struct crypto4xx_core_device *dev __maybe_unused) { } +#endif + +#endif diff --git a/drivers/crypto/amlogic/Kconfig b/drivers/crypto/amlogic/Kconfig new file mode 100644 index 000000000..cf2c676a7 --- /dev/null +++ b/drivers/crypto/amlogic/Kconfig @@ -0,0 +1,25 @@ +config CRYPTO_DEV_AMLOGIC_GXL + tristate "Support for amlogic cryptographic offloader" + depends on HAS_IOMEM + default m if ARCH_MESON + select CRYPTO_SKCIPHER + select CRYPTO_ENGINE + select CRYPTO_ECB + select CRYPTO_CBC + select CRYPTO_AES + help + Select y here to have support for the cryptographic offloader + available on Amlogic GXL SoC. + This hardware handles AES ciphers in ECB/CBC mode. + + To compile this driver as a module, choose M here: the module + will be called amlogic-gxl-crypto. + +config CRYPTO_DEV_AMLOGIC_GXL_DEBUG + bool "Enable amlogic stats" + depends on CRYPTO_DEV_AMLOGIC_GXL + depends on DEBUG_FS + help + Say y to enable amlogic-crypto debug stats. + This will create /sys/kernel/debug/gxl-crypto/stats for displaying + the number of requests per flow and per algorithm. diff --git a/drivers/crypto/amlogic/Makefile b/drivers/crypto/amlogic/Makefile new file mode 100644 index 000000000..39057e62c --- /dev/null +++ b/drivers/crypto/amlogic/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_CRYPTO_DEV_AMLOGIC_GXL) += amlogic-gxl-crypto.o +amlogic-gxl-crypto-y := amlogic-gxl-core.o amlogic-gxl-cipher.o diff --git a/drivers/crypto/amlogic/amlogic-gxl-cipher.c b/drivers/crypto/amlogic/amlogic-gxl-cipher.c new file mode 100644 index 000000000..29048da6f --- /dev/null +++ b/drivers/crypto/amlogic/amlogic-gxl-cipher.c @@ -0,0 +1,371 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * amlogic-cipher.c - hardware cryptographic offloader for Amlogic GXL SoC + * + * Copyright (C) 2018-2019 Corentin LABBE + * + * This file add support for AES cipher with 128,192,256 bits keysize in + * CBC and ECB mode. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "amlogic-gxl.h" + +static int get_engine_number(struct meson_dev *mc) +{ + return atomic_inc_return(&mc->flow) % MAXFLOW; +} + +static bool meson_cipher_need_fallback(struct skcipher_request *areq) +{ + struct scatterlist *src_sg = areq->src; + struct scatterlist *dst_sg = areq->dst; + + if (areq->cryptlen == 0) + return true; + + if (sg_nents(src_sg) != sg_nents(dst_sg)) + return true; + + /* KEY/IV descriptors use 3 desc */ + if (sg_nents(src_sg) > MAXDESC - 3 || sg_nents(dst_sg) > MAXDESC - 3) + return true; + + while (src_sg && dst_sg) { + if ((src_sg->length % 16) != 0) + return true; + if ((dst_sg->length % 16) != 0) + return true; + if (src_sg->length != dst_sg->length) + return true; + if (!IS_ALIGNED(src_sg->offset, sizeof(u32))) + return true; + if (!IS_ALIGNED(dst_sg->offset, sizeof(u32))) + return true; + src_sg = sg_next(src_sg); + dst_sg = sg_next(dst_sg); + } + + return false; +} + +static int meson_cipher_do_fallback(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct meson_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + int err; +#ifdef CONFIG_CRYPTO_DEV_AMLOGIC_GXL_DEBUG + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct meson_alg_template *algt; + + algt = container_of(alg, struct meson_alg_template, alg.skcipher.base); + algt->stat_fb++; +#endif + skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm); + skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags, + areq->base.complete, areq->base.data); + skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst, + areq->cryptlen, areq->iv); + + if (rctx->op_dir == MESON_DECRYPT) + err = crypto_skcipher_decrypt(&rctx->fallback_req); + else + err = crypto_skcipher_encrypt(&rctx->fallback_req); + return err; +} + +static int meson_cipher(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct meson_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + struct meson_dev *mc = op->mc; + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct meson_alg_template *algt; + int flow = rctx->flow; + unsigned int todo, eat, len; + struct scatterlist *src_sg = areq->src; + struct scatterlist *dst_sg = areq->dst; + struct meson_desc *desc; + int nr_sgs, nr_sgd; + int i, err = 0; + unsigned int keyivlen, ivsize, offset, tloffset; + dma_addr_t phykeyiv; + void *backup_iv = NULL, *bkeyiv; + u32 v; + + algt = container_of(alg, struct meson_alg_template, alg.skcipher.base); + + dev_dbg(mc->dev, "%s %s %u %x IV(%u) key=%u flow=%d\n", __func__, + crypto_tfm_alg_name(areq->base.tfm), + areq->cryptlen, + rctx->op_dir, crypto_skcipher_ivsize(tfm), + op->keylen, flow); + +#ifdef CONFIG_CRYPTO_DEV_AMLOGIC_GXL_DEBUG + algt->stat_req++; + mc->chanlist[flow].stat_req++; +#endif + + /* + * The hardware expect a list of meson_desc structures. + * The 2 first structures store key + * The third stores IV + */ + bkeyiv = kzalloc(48, GFP_KERNEL | GFP_DMA); + if (!bkeyiv) + return -ENOMEM; + + memcpy(bkeyiv, op->key, op->keylen); + keyivlen = op->keylen; + + ivsize = crypto_skcipher_ivsize(tfm); + if (areq->iv && ivsize > 0) { + if (ivsize > areq->cryptlen) { + dev_err(mc->dev, "invalid ivsize=%d vs len=%d\n", ivsize, areq->cryptlen); + err = -EINVAL; + goto theend; + } + memcpy(bkeyiv + 32, areq->iv, ivsize); + keyivlen = 48; + if (rctx->op_dir == MESON_DECRYPT) { + backup_iv = kzalloc(ivsize, GFP_KERNEL); + if (!backup_iv) { + err = -ENOMEM; + goto theend; + } + offset = areq->cryptlen - ivsize; + scatterwalk_map_and_copy(backup_iv, areq->src, offset, + ivsize, 0); + } + } + if (keyivlen == 24) + keyivlen = 32; + + phykeyiv = dma_map_single(mc->dev, bkeyiv, keyivlen, + DMA_TO_DEVICE); + err = dma_mapping_error(mc->dev, phykeyiv); + if (err) { + dev_err(mc->dev, "Cannot DMA MAP KEY IV\n"); + goto theend; + } + + tloffset = 0; + eat = 0; + i = 0; + while (keyivlen > eat) { + desc = &mc->chanlist[flow].tl[tloffset]; + memset(desc, 0, sizeof(struct meson_desc)); + todo = min(keyivlen - eat, 16u); + desc->t_src = cpu_to_le32(phykeyiv + i * 16); + desc->t_dst = cpu_to_le32(i * 16); + v = (MODE_KEY << 20) | DESC_OWN | 16; + desc->t_status = cpu_to_le32(v); + + eat += todo; + i++; + tloffset++; + } + + if (areq->src == areq->dst) { + nr_sgs = dma_map_sg(mc->dev, areq->src, sg_nents(areq->src), + DMA_BIDIRECTIONAL); + if (!nr_sgs) { + dev_err(mc->dev, "Invalid SG count %d\n", nr_sgs); + err = -EINVAL; + goto theend; + } + nr_sgd = nr_sgs; + } else { + nr_sgs = dma_map_sg(mc->dev, areq->src, sg_nents(areq->src), + DMA_TO_DEVICE); + if (!nr_sgs || nr_sgs > MAXDESC - 3) { + dev_err(mc->dev, "Invalid SG count %d\n", nr_sgs); + err = -EINVAL; + goto theend; + } + nr_sgd = dma_map_sg(mc->dev, areq->dst, sg_nents(areq->dst), + DMA_FROM_DEVICE); + if (!nr_sgd || nr_sgd > MAXDESC - 3) { + dev_err(mc->dev, "Invalid SG count %d\n", nr_sgd); + err = -EINVAL; + goto theend; + } + } + + src_sg = areq->src; + dst_sg = areq->dst; + len = areq->cryptlen; + while (src_sg) { + desc = &mc->chanlist[flow].tl[tloffset]; + memset(desc, 0, sizeof(struct meson_desc)); + + desc->t_src = cpu_to_le32(sg_dma_address(src_sg)); + desc->t_dst = cpu_to_le32(sg_dma_address(dst_sg)); + todo = min(len, sg_dma_len(src_sg)); + v = (op->keymode << 20) | DESC_OWN | todo | (algt->blockmode << 26); + if (rctx->op_dir) + v |= DESC_ENCRYPTION; + len -= todo; + + if (!sg_next(src_sg)) + v |= DESC_LAST; + desc->t_status = cpu_to_le32(v); + tloffset++; + src_sg = sg_next(src_sg); + dst_sg = sg_next(dst_sg); + } + + reinit_completion(&mc->chanlist[flow].complete); + mc->chanlist[flow].status = 0; + writel(mc->chanlist[flow].t_phy | 2, mc->base + (flow << 2)); + wait_for_completion_interruptible_timeout(&mc->chanlist[flow].complete, + msecs_to_jiffies(500)); + if (mc->chanlist[flow].status == 0) { + dev_err(mc->dev, "DMA timeout for flow %d\n", flow); + err = -EINVAL; + } + + dma_unmap_single(mc->dev, phykeyiv, keyivlen, DMA_TO_DEVICE); + + if (areq->src == areq->dst) { + dma_unmap_sg(mc->dev, areq->src, sg_nents(areq->src), DMA_BIDIRECTIONAL); + } else { + dma_unmap_sg(mc->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE); + dma_unmap_sg(mc->dev, areq->dst, sg_nents(areq->dst), DMA_FROM_DEVICE); + } + + if (areq->iv && ivsize > 0) { + if (rctx->op_dir == MESON_DECRYPT) { + memcpy(areq->iv, backup_iv, ivsize); + } else { + scatterwalk_map_and_copy(areq->iv, areq->dst, + areq->cryptlen - ivsize, + ivsize, 0); + } + } +theend: + kfree_sensitive(bkeyiv); + kfree_sensitive(backup_iv); + + return err; +} + +int meson_handle_cipher_request(struct crypto_engine *engine, void *areq) +{ + int err; + struct skcipher_request *breq = container_of(areq, struct skcipher_request, base); + + err = meson_cipher(breq); + local_bh_disable(); + crypto_finalize_skcipher_request(engine, breq, err); + local_bh_enable(); + + return 0; +} + +int meson_skdecrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct meson_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + struct crypto_engine *engine; + int e; + + rctx->op_dir = MESON_DECRYPT; + if (meson_cipher_need_fallback(areq)) + return meson_cipher_do_fallback(areq); + e = get_engine_number(op->mc); + engine = op->mc->chanlist[e].engine; + rctx->flow = e; + + return crypto_transfer_skcipher_request_to_engine(engine, areq); +} + +int meson_skencrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct meson_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + struct crypto_engine *engine; + int e; + + rctx->op_dir = MESON_ENCRYPT; + if (meson_cipher_need_fallback(areq)) + return meson_cipher_do_fallback(areq); + e = get_engine_number(op->mc); + engine = op->mc->chanlist[e].engine; + rctx->flow = e; + + return crypto_transfer_skcipher_request_to_engine(engine, areq); +} + +int meson_cipher_init(struct crypto_tfm *tfm) +{ + struct meson_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm); + struct meson_alg_template *algt; + const char *name = crypto_tfm_alg_name(tfm); + struct crypto_skcipher *sktfm = __crypto_skcipher_cast(tfm); + struct skcipher_alg *alg = crypto_skcipher_alg(sktfm); + + memset(op, 0, sizeof(struct meson_cipher_tfm_ctx)); + + algt = container_of(alg, struct meson_alg_template, alg.skcipher.base); + op->mc = algt->mc; + + op->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(op->fallback_tfm)) { + dev_err(op->mc->dev, "ERROR: Cannot allocate fallback for %s %ld\n", + name, PTR_ERR(op->fallback_tfm)); + return PTR_ERR(op->fallback_tfm); + } + + crypto_skcipher_set_reqsize(sktfm, sizeof(struct meson_cipher_req_ctx) + + crypto_skcipher_reqsize(op->fallback_tfm)); + + return 0; +} + +void meson_cipher_exit(struct crypto_tfm *tfm) +{ + struct meson_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm); + + kfree_sensitive(op->key); + crypto_free_skcipher(op->fallback_tfm); +} + +int meson_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct meson_dev *mc = op->mc; + + switch (keylen) { + case 128 / 8: + op->keymode = MODE_AES_128; + break; + case 192 / 8: + op->keymode = MODE_AES_192; + break; + case 256 / 8: + op->keymode = MODE_AES_256; + break; + default: + dev_dbg(mc->dev, "ERROR: Invalid keylen %u\n", keylen); + return -EINVAL; + } + kfree_sensitive(op->key); + op->keylen = keylen; + op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); + if (!op->key) + return -ENOMEM; + + return crypto_skcipher_setkey(op->fallback_tfm, key, keylen); +} diff --git a/drivers/crypto/amlogic/amlogic-gxl-core.c b/drivers/crypto/amlogic/amlogic-gxl-core.c new file mode 100644 index 000000000..169c6eeb5 --- /dev/null +++ b/drivers/crypto/amlogic/amlogic-gxl-core.c @@ -0,0 +1,332 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * amlgoic-core.c - hardware cryptographic offloader for Amlogic GXL SoC + * + * Copyright (C) 2018-2019 Corentin Labbe + * + * Core file which registers crypto algorithms supported by the hardware. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "amlogic-gxl.h" + +static irqreturn_t meson_irq_handler(int irq, void *data) +{ + struct meson_dev *mc = (struct meson_dev *)data; + int flow; + u32 p; + + for (flow = 0; flow < MAXFLOW; flow++) { + if (mc->irqs[flow] == irq) { + p = readl(mc->base + ((0x04 + flow) << 2)); + if (p) { + writel_relaxed(0xF, mc->base + ((0x4 + flow) << 2)); + mc->chanlist[flow].status = 1; + complete(&mc->chanlist[flow].complete); + return IRQ_HANDLED; + } + dev_err(mc->dev, "%s %d Got irq for flow %d but ctrl is empty\n", __func__, irq, flow); + } + } + + dev_err(mc->dev, "%s %d from unknown irq\n", __func__, irq); + return IRQ_HANDLED; +} + +static struct meson_alg_template mc_algs[] = { +{ + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .blockmode = MESON_OPMODE_CBC, + .alg.skcipher.base = { + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-gxl", + .cra_priority = 400, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct meson_cipher_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 0xf, + .cra_init = meson_cipher_init, + .cra_exit = meson_cipher_exit, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = meson_aes_setkey, + .encrypt = meson_skencrypt, + .decrypt = meson_skdecrypt, + }, + .alg.skcipher.op = { + .do_one_request = meson_handle_cipher_request, + }, +}, +{ + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .blockmode = MESON_OPMODE_ECB, + .alg.skcipher.base = { + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-gxl", + .cra_priority = 400, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct meson_cipher_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 0xf, + .cra_init = meson_cipher_init, + .cra_exit = meson_cipher_exit, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = meson_aes_setkey, + .encrypt = meson_skencrypt, + .decrypt = meson_skdecrypt, + }, + .alg.skcipher.op = { + .do_one_request = meson_handle_cipher_request, + }, +}, +}; + +static int meson_debugfs_show(struct seq_file *seq, void *v) +{ + struct meson_dev *mc __maybe_unused = seq->private; + int i; + + for (i = 0; i < MAXFLOW; i++) + seq_printf(seq, "Channel %d: nreq %lu\n", i, +#ifdef CONFIG_CRYPTO_DEV_AMLOGIC_GXL_DEBUG + mc->chanlist[i].stat_req); +#else + 0ul); +#endif + + for (i = 0; i < ARRAY_SIZE(mc_algs); i++) { + switch (mc_algs[i].type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + seq_printf(seq, "%s %s %lu %lu\n", + mc_algs[i].alg.skcipher.base.base.cra_driver_name, + mc_algs[i].alg.skcipher.base.base.cra_name, +#ifdef CONFIG_CRYPTO_DEV_AMLOGIC_GXL_DEBUG + mc_algs[i].stat_req, mc_algs[i].stat_fb); +#else + 0ul, 0ul); +#endif + break; + } + } + return 0; +} +DEFINE_SHOW_ATTRIBUTE(meson_debugfs); + +static void meson_free_chanlist(struct meson_dev *mc, int i) +{ + while (i >= 0) { + crypto_engine_exit(mc->chanlist[i].engine); + if (mc->chanlist[i].tl) + dma_free_coherent(mc->dev, sizeof(struct meson_desc) * MAXDESC, + mc->chanlist[i].tl, + mc->chanlist[i].t_phy); + i--; + } +} + +/* + * Allocate the channel list structure + */ +static int meson_allocate_chanlist(struct meson_dev *mc) +{ + int i, err; + + mc->chanlist = devm_kcalloc(mc->dev, MAXFLOW, + sizeof(struct meson_flow), GFP_KERNEL); + if (!mc->chanlist) + return -ENOMEM; + + for (i = 0; i < MAXFLOW; i++) { + init_completion(&mc->chanlist[i].complete); + + mc->chanlist[i].engine = crypto_engine_alloc_init(mc->dev, true); + if (!mc->chanlist[i].engine) { + dev_err(mc->dev, "Cannot allocate engine\n"); + i--; + err = -ENOMEM; + goto error_engine; + } + err = crypto_engine_start(mc->chanlist[i].engine); + if (err) { + dev_err(mc->dev, "Cannot start engine\n"); + goto error_engine; + } + mc->chanlist[i].tl = dma_alloc_coherent(mc->dev, + sizeof(struct meson_desc) * MAXDESC, + &mc->chanlist[i].t_phy, + GFP_KERNEL); + if (!mc->chanlist[i].tl) { + err = -ENOMEM; + goto error_engine; + } + } + return 0; +error_engine: + meson_free_chanlist(mc, i); + return err; +} + +static int meson_register_algs(struct meson_dev *mc) +{ + int err, i; + + for (i = 0; i < ARRAY_SIZE(mc_algs); i++) { + mc_algs[i].mc = mc; + switch (mc_algs[i].type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + err = crypto_engine_register_skcipher(&mc_algs[i].alg.skcipher); + if (err) { + dev_err(mc->dev, "Fail to register %s\n", + mc_algs[i].alg.skcipher.base.base.cra_name); + mc_algs[i].mc = NULL; + return err; + } + break; + } + } + + return 0; +} + +static void meson_unregister_algs(struct meson_dev *mc) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(mc_algs); i++) { + if (!mc_algs[i].mc) + continue; + switch (mc_algs[i].type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + crypto_engine_unregister_skcipher(&mc_algs[i].alg.skcipher); + break; + } + } +} + +static int meson_crypto_probe(struct platform_device *pdev) +{ + struct meson_dev *mc; + int err, i; + + mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL); + if (!mc) + return -ENOMEM; + + mc->dev = &pdev->dev; + platform_set_drvdata(pdev, mc); + + mc->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(mc->base)) + return PTR_ERR(mc->base); + + mc->busclk = devm_clk_get(&pdev->dev, "blkmv"); + if (IS_ERR(mc->busclk)) { + err = PTR_ERR(mc->busclk); + dev_err(&pdev->dev, "Cannot get core clock err=%d\n", err); + return err; + } + + for (i = 0; i < MAXFLOW; i++) { + mc->irqs[i] = platform_get_irq(pdev, i); + if (mc->irqs[i] < 0) + return mc->irqs[i]; + + err = devm_request_irq(&pdev->dev, mc->irqs[i], meson_irq_handler, 0, + "gxl-crypto", mc); + if (err < 0) + return err; + } + + err = clk_prepare_enable(mc->busclk); + if (err != 0) { + dev_err(&pdev->dev, "Cannot prepare_enable busclk\n"); + return err; + } + + err = meson_allocate_chanlist(mc); + if (err) + goto error_flow; + + err = meson_register_algs(mc); + if (err) + goto error_alg; + + if (IS_ENABLED(CONFIG_CRYPTO_DEV_AMLOGIC_GXL_DEBUG)) { + struct dentry *dbgfs_dir; + + dbgfs_dir = debugfs_create_dir("gxl-crypto", NULL); + debugfs_create_file("stats", 0444, dbgfs_dir, mc, &meson_debugfs_fops); + +#ifdef CONFIG_CRYPTO_DEV_AMLOGIC_GXL_DEBUG + mc->dbgfs_dir = dbgfs_dir; +#endif + } + + return 0; +error_alg: + meson_unregister_algs(mc); + meson_free_chanlist(mc, MAXFLOW - 1); +error_flow: + clk_disable_unprepare(mc->busclk); + return err; +} + +static void meson_crypto_remove(struct platform_device *pdev) +{ + struct meson_dev *mc = platform_get_drvdata(pdev); + +#ifdef CONFIG_CRYPTO_DEV_AMLOGIC_GXL_DEBUG + debugfs_remove_recursive(mc->dbgfs_dir); +#endif + + meson_unregister_algs(mc); + + meson_free_chanlist(mc, MAXFLOW - 1); + + clk_disable_unprepare(mc->busclk); +} + +static const struct of_device_id meson_crypto_of_match_table[] = { + { .compatible = "amlogic,gxl-crypto", }, + {} +}; +MODULE_DEVICE_TABLE(of, meson_crypto_of_match_table); + +static struct platform_driver meson_crypto_driver = { + .probe = meson_crypto_probe, + .remove = meson_crypto_remove, + .driver = { + .name = "gxl-crypto", + .of_match_table = meson_crypto_of_match_table, + }, +}; + +module_platform_driver(meson_crypto_driver); + +MODULE_DESCRIPTION("Amlogic GXL cryptographic offloader"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Corentin Labbe "); diff --git a/drivers/crypto/amlogic/amlogic-gxl.h b/drivers/crypto/amlogic/amlogic-gxl.h new file mode 100644 index 000000000..d68094ffb --- /dev/null +++ b/drivers/crypto/amlogic/amlogic-gxl.h @@ -0,0 +1,159 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * amlogic.h - hardware cryptographic offloader for Amlogic SoC + * + * Copyright (C) 2018-2019 Corentin LABBE + */ +#include +#include +#include +#include +#include +#include + +#define MODE_KEY 1 +#define MODE_AES_128 0x8 +#define MODE_AES_192 0x9 +#define MODE_AES_256 0xa + +#define MESON_DECRYPT 0 +#define MESON_ENCRYPT 1 + +#define MESON_OPMODE_ECB 0 +#define MESON_OPMODE_CBC 1 + +#define MAXFLOW 2 + +#define MAXDESC 64 + +#define DESC_LAST BIT(18) +#define DESC_ENCRYPTION BIT(28) +#define DESC_OWN BIT(31) + +/* + * struct meson_desc - Descriptor for DMA operations + * Note that without datasheet, some are unknown + * @t_status: Descriptor of the cipher operation (see description below) + * @t_src: Physical address of data to read + * @t_dst: Physical address of data to write + * t_status is segmented like this: + * @len: 0-16 length of data to operate + * @irq: 17 Ignored by hardware + * @eoc: 18 End means the descriptor is the last + * @loop: 19 Unknown + * @mode: 20-23 Type of algorithm (AES, SHA) + * @begin: 24 Unknown + * @end: 25 Unknown + * @op_mode: 26-27 Blockmode (CBC, ECB) + * @enc: 28 0 means decryption, 1 is for encryption + * @block: 29 Unknown + * @error: 30 Unknown + * @owner: 31 owner of the descriptor, 1 own by HW + */ +struct meson_desc { + __le32 t_status; + __le32 t_src; + __le32 t_dst; +}; + +/* + * struct meson_flow - Information used by each flow + * @engine: ptr to the crypto_engine for this flow + * @keylen: keylen for this flow operation + * @complete: completion for the current task on this flow + * @status: set to 1 by interrupt if task is done + * @t_phy: Physical address of task + * @tl: pointer to the current ce_task for this flow + * @stat_req: number of request done by this flow + */ +struct meson_flow { + struct crypto_engine *engine; + struct completion complete; + int status; + unsigned int keylen; + dma_addr_t t_phy; + struct meson_desc *tl; +#ifdef CONFIG_CRYPTO_DEV_AMLOGIC_GXL_DEBUG + unsigned long stat_req; +#endif +}; + +/* + * struct meson_dev - main container for all this driver information + * @base: base address of amlogic-crypto + * @busclk: bus clock for amlogic-crypto + * @dev: the platform device + * @chanlist: array of all flow + * @flow: flow to use in next request + * @irqs: IRQ numbers for amlogic-crypto + * @dbgfs_dir: Debugfs dentry for statistic directory + * @dbgfs_stats: Debugfs dentry for statistic counters + */ +struct meson_dev { + void __iomem *base; + struct clk *busclk; + struct device *dev; + struct meson_flow *chanlist; + atomic_t flow; + int irqs[MAXFLOW]; +#ifdef CONFIG_CRYPTO_DEV_AMLOGIC_GXL_DEBUG + struct dentry *dbgfs_dir; +#endif +}; + +/* + * struct meson_cipher_req_ctx - context for a skcipher request + * @op_dir: direction (encrypt vs decrypt) for this request + * @flow: the flow to use for this request + */ +struct meson_cipher_req_ctx { + u32 op_dir; + int flow; + struct skcipher_request fallback_req; // keep at the end +}; + +/* + * struct meson_cipher_tfm_ctx - context for a skcipher TFM + * @key: pointer to key data + * @keylen: len of the key + * @keymode: The keymode(type and size of key) associated with this TFM + * @mc: pointer to the private data of driver handling this TFM + * @fallback_tfm: pointer to the fallback TFM + */ +struct meson_cipher_tfm_ctx { + u32 *key; + u32 keylen; + u32 keymode; + struct meson_dev *mc; + struct crypto_skcipher *fallback_tfm; +}; + +/* + * struct meson_alg_template - crypto_alg template + * @type: the CRYPTO_ALG_TYPE for this template + * @blockmode: the type of block operation + * @mc: pointer to the meson_dev structure associated with this template + * @alg: one of sub struct must be used + * @stat_req: number of request done on this template + * @stat_fb: total of all data len done on this template + */ +struct meson_alg_template { + u32 type; + u32 blockmode; + union { + struct skcipher_engine_alg skcipher; + } alg; + struct meson_dev *mc; +#ifdef CONFIG_CRYPTO_DEV_AMLOGIC_GXL_DEBUG + unsigned long stat_req; + unsigned long stat_fb; +#endif +}; + +int meson_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen); +int meson_cipher_init(struct crypto_tfm *tfm); +void meson_cipher_exit(struct crypto_tfm *tfm); +int meson_skdecrypt(struct skcipher_request *areq); +int meson_skencrypt(struct skcipher_request *areq); +int meson_handle_cipher_request(struct crypto_engine *engine, void *areq); diff --git a/drivers/crypto/aspeed/Kconfig b/drivers/crypto/aspeed/Kconfig new file mode 100644 index 000000000..e93f2f82b --- /dev/null +++ b/drivers/crypto/aspeed/Kconfig @@ -0,0 +1,57 @@ +config CRYPTO_DEV_ASPEED + tristate "Support for Aspeed cryptographic engine driver" + depends on ARCH_ASPEED || COMPILE_TEST + select CRYPTO_ENGINE + help + Hash and Crypto Engine (HACE) is designed to accelerate the + throughput of hash data digest, encryption and decryption. + + Select y here to have support for the cryptographic driver + available on Aspeed SoC. + +config CRYPTO_DEV_ASPEED_DEBUG + bool "Enable Aspeed crypto debug messages" + depends on CRYPTO_DEV_ASPEED + help + Print Aspeed crypto debugging messages if you use this + option to ask for those messages. + Avoid enabling this option for production build to + minimize driver timing. + +config CRYPTO_DEV_ASPEED_HACE_HASH + bool "Enable Aspeed Hash & Crypto Engine (HACE) hash" + depends on CRYPTO_DEV_ASPEED + select CRYPTO_SHA1 + select CRYPTO_SHA256 + select CRYPTO_SHA512 + select CRYPTO_HMAC + help + Select here to enable Aspeed Hash & Crypto Engine (HACE) + hash driver. + Supports multiple message digest standards, including + SHA-1, SHA-224, SHA-256, SHA-384, SHA-512, and so on. + +config CRYPTO_DEV_ASPEED_HACE_CRYPTO + bool "Enable Aspeed Hash & Crypto Engine (HACE) crypto" + depends on CRYPTO_DEV_ASPEED + select CRYPTO_AES + select CRYPTO_DES + select CRYPTO_ECB + select CRYPTO_CBC + select CRYPTO_CTR + help + Select here to enable Aspeed Hash & Crypto Engine (HACE) + crypto driver. + Supports AES/DES symmetric-key encryption and decryption + with ECB/CBC/CTR options. + +config CRYPTO_DEV_ASPEED_ACRY + bool "Enable Aspeed ACRY RSA Engine" + depends on CRYPTO_DEV_ASPEED + select CRYPTO_ENGINE + select CRYPTO_RSA + help + Select here to enable Aspeed ECC/RSA Engine (ACRY) + RSA driver. + Supports 256 bits to 4096 bits RSA encryption/decryption + and signature/verification. diff --git a/drivers/crypto/aspeed/Makefile b/drivers/crypto/aspeed/Makefile new file mode 100644 index 000000000..15862752c --- /dev/null +++ b/drivers/crypto/aspeed/Makefile @@ -0,0 +1,11 @@ +hace-hash-$(CONFIG_CRYPTO_DEV_ASPEED_HACE_HASH) := aspeed-hace-hash.o +hace-crypto-$(CONFIG_CRYPTO_DEV_ASPEED_HACE_CRYPTO) := aspeed-hace-crypto.o + +obj-$(CONFIG_CRYPTO_DEV_ASPEED) += aspeed_crypto.o +aspeed_crypto-objs := aspeed-hace.o \ + $(hace-hash-y) \ + $(hace-crypto-y) + +aspeed_acry-$(CONFIG_CRYPTO_DEV_ASPEED_ACRY) += aspeed-acry.o + +obj-$(CONFIG_CRYPTO_DEV_ASPEED) += $(aspeed_acry-y) diff --git a/drivers/crypto/aspeed/aspeed-acry.c b/drivers/crypto/aspeed/aspeed-acry.c new file mode 100644 index 000000000..97327ba2c --- /dev/null +++ b/drivers/crypto/aspeed/aspeed-acry.c @@ -0,0 +1,816 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright 2021 Aspeed Technology Inc. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_CRYPTO_DEV_ASPEED_DEBUG +#define ACRY_DBG(d, fmt, ...) \ + dev_info((d)->dev, "%s() " fmt, __func__, ##__VA_ARGS__) +#else +#define ACRY_DBG(d, fmt, ...) \ + dev_dbg((d)->dev, "%s() " fmt, __func__, ##__VA_ARGS__) +#endif + +/***************************** + * * + * ACRY register definitions * + * * + * ***************************/ +#define ASPEED_ACRY_TRIGGER 0x000 /* ACRY Engine Control: trigger */ +#define ASPEED_ACRY_DMA_CMD 0x048 /* ACRY Engine Control: Command */ +#define ASPEED_ACRY_DMA_SRC_BASE 0x04C /* ACRY DRAM base address for DMA */ +#define ASPEED_ACRY_DMA_LEN 0x050 /* ACRY Data Length of DMA */ +#define ASPEED_ACRY_RSA_KEY_LEN 0x058 /* ACRY RSA Exp/Mod Key Length (Bits) */ +#define ASPEED_ACRY_INT_MASK 0x3F8 /* ACRY Interrupt Mask */ +#define ASPEED_ACRY_STATUS 0x3FC /* ACRY Interrupt Status */ + +/* rsa trigger */ +#define ACRY_CMD_RSA_TRIGGER BIT(0) +#define ACRY_CMD_DMA_RSA_TRIGGER BIT(1) + +/* rsa dma cmd */ +#define ACRY_CMD_DMA_SRAM_MODE_RSA (0x3 << 4) +#define ACRY_CMD_DMEM_AHB BIT(8) +#define ACRY_CMD_DMA_SRAM_AHB_ENGINE 0 + +/* rsa key len */ +#define RSA_E_BITS_LEN(x) ((x) << 16) +#define RSA_M_BITS_LEN(x) (x) + +/* acry isr */ +#define ACRY_RSA_ISR BIT(1) + +#define ASPEED_ACRY_BUFF_SIZE 0x1800 /* DMA buffer size */ +#define ASPEED_ACRY_SRAM_MAX_LEN 2048 /* ACRY SRAM maximum length (Bytes) */ +#define ASPEED_ACRY_RSA_MAX_KEY_LEN 512 /* ACRY RSA maximum key length (Bytes) */ + +#define CRYPTO_FLAGS_BUSY BIT(1) +#define BYTES_PER_DWORD 4 + +/***************************** + * * + * AHBC register definitions * + * * + * ***************************/ +#define AHBC_REGION_PROT 0x240 +#define REGION_ACRYM BIT(23) + +#define ast_acry_write(acry, val, offset) \ + writel((val), (acry)->regs + (offset)) + +#define ast_acry_read(acry, offset) \ + readl((acry)->regs + (offset)) + +struct aspeed_acry_dev; + +typedef int (*aspeed_acry_fn_t)(struct aspeed_acry_dev *); + +struct aspeed_acry_dev { + void __iomem *regs; + struct device *dev; + int irq; + struct clk *clk; + struct regmap *ahbc; + + struct akcipher_request *req; + struct tasklet_struct done_task; + aspeed_acry_fn_t resume; + unsigned long flags; + + /* ACRY output SRAM buffer */ + void __iomem *acry_sram; + + /* ACRY input DMA buffer */ + void *buf_addr; + dma_addr_t buf_dma_addr; + + struct crypto_engine *crypt_engine_rsa; + + /* ACRY SRAM memory mapped */ + int exp_dw_mapping[ASPEED_ACRY_RSA_MAX_KEY_LEN]; + int mod_dw_mapping[ASPEED_ACRY_RSA_MAX_KEY_LEN]; + int data_byte_mapping[ASPEED_ACRY_SRAM_MAX_LEN]; +}; + +struct aspeed_acry_ctx { + struct aspeed_acry_dev *acry_dev; + + struct rsa_key key; + int enc; + u8 *n; + u8 *e; + u8 *d; + size_t n_sz; + size_t e_sz; + size_t d_sz; + + aspeed_acry_fn_t trigger; + + struct crypto_akcipher *fallback_tfm; +}; + +struct aspeed_acry_alg { + struct aspeed_acry_dev *acry_dev; + struct akcipher_engine_alg akcipher; +}; + +enum aspeed_rsa_key_mode { + ASPEED_RSA_EXP_MODE = 0, + ASPEED_RSA_MOD_MODE, + ASPEED_RSA_DATA_MODE, +}; + +static inline struct akcipher_request * + akcipher_request_cast(struct crypto_async_request *req) +{ + return container_of(req, struct akcipher_request, base); +} + +static int aspeed_acry_do_fallback(struct akcipher_request *req) +{ + struct crypto_akcipher *cipher = crypto_akcipher_reqtfm(req); + struct aspeed_acry_ctx *ctx = akcipher_tfm_ctx(cipher); + int err; + + akcipher_request_set_tfm(req, ctx->fallback_tfm); + + if (ctx->enc) + err = crypto_akcipher_encrypt(req); + else + err = crypto_akcipher_decrypt(req); + + akcipher_request_set_tfm(req, cipher); + + return err; +} + +static bool aspeed_acry_need_fallback(struct akcipher_request *req) +{ + struct crypto_akcipher *cipher = crypto_akcipher_reqtfm(req); + struct aspeed_acry_ctx *ctx = akcipher_tfm_ctx(cipher); + + return ctx->key.n_sz > ASPEED_ACRY_RSA_MAX_KEY_LEN; +} + +static int aspeed_acry_handle_queue(struct aspeed_acry_dev *acry_dev, + struct akcipher_request *req) +{ + if (aspeed_acry_need_fallback(req)) { + ACRY_DBG(acry_dev, "SW fallback\n"); + return aspeed_acry_do_fallback(req); + } + + return crypto_transfer_akcipher_request_to_engine(acry_dev->crypt_engine_rsa, req); +} + +static int aspeed_acry_do_request(struct crypto_engine *engine, void *areq) +{ + struct akcipher_request *req = akcipher_request_cast(areq); + struct crypto_akcipher *cipher = crypto_akcipher_reqtfm(req); + struct aspeed_acry_ctx *ctx = akcipher_tfm_ctx(cipher); + struct aspeed_acry_dev *acry_dev = ctx->acry_dev; + + acry_dev->req = req; + acry_dev->flags |= CRYPTO_FLAGS_BUSY; + + return ctx->trigger(acry_dev); +} + +static int aspeed_acry_complete(struct aspeed_acry_dev *acry_dev, int err) +{ + struct akcipher_request *req = acry_dev->req; + + acry_dev->flags &= ~CRYPTO_FLAGS_BUSY; + + crypto_finalize_akcipher_request(acry_dev->crypt_engine_rsa, req, err); + + return err; +} + +/* + * Copy Data to DMA buffer for engine used. + */ +static void aspeed_acry_rsa_sg_copy_to_buffer(struct aspeed_acry_dev *acry_dev, + u8 *buf, struct scatterlist *src, + size_t nbytes) +{ + static u8 dram_buffer[ASPEED_ACRY_SRAM_MAX_LEN]; + int i = 0, j; + int data_idx; + + ACRY_DBG(acry_dev, "\n"); + + scatterwalk_map_and_copy(dram_buffer, src, 0, nbytes, 0); + + for (j = nbytes - 1; j >= 0; j--) { + data_idx = acry_dev->data_byte_mapping[i]; + buf[data_idx] = dram_buffer[j]; + i++; + } + + for (; i < ASPEED_ACRY_SRAM_MAX_LEN; i++) { + data_idx = acry_dev->data_byte_mapping[i]; + buf[data_idx] = 0; + } +} + +/* + * Copy Exp/Mod to DMA buffer for engine used. + * + * Params: + * - mode 0 : Exponential + * - mode 1 : Modulus + * + * Example: + * - DRAM memory layout: + * D[0], D[4], D[8], D[12] + * - ACRY SRAM memory layout should reverse the order of source data: + * D[12], D[8], D[4], D[0] + */ +static int aspeed_acry_rsa_ctx_copy(struct aspeed_acry_dev *acry_dev, void *buf, + const void *xbuf, size_t nbytes, + enum aspeed_rsa_key_mode mode) +{ + const u8 *src = xbuf; + __le32 *dw_buf = buf; + int nbits, ndw; + int i, j, idx; + u32 data = 0; + + ACRY_DBG(acry_dev, "nbytes:%zu, mode:%d\n", nbytes, mode); + + if (nbytes > ASPEED_ACRY_RSA_MAX_KEY_LEN) + return -ENOMEM; + + /* Remove the leading zeros */ + while (nbytes > 0 && src[0] == 0) { + src++; + nbytes--; + } + + nbits = nbytes * 8; + if (nbytes > 0) + nbits -= count_leading_zeros(src[0]) - (BITS_PER_LONG - 8); + + /* double-world alignment */ + ndw = DIV_ROUND_UP(nbytes, BYTES_PER_DWORD); + + if (nbytes > 0) { + i = BYTES_PER_DWORD - nbytes % BYTES_PER_DWORD; + i %= BYTES_PER_DWORD; + + for (j = ndw; j > 0; j--) { + for (; i < BYTES_PER_DWORD; i++) { + data <<= 8; + data |= *src++; + } + + i = 0; + + if (mode == ASPEED_RSA_EXP_MODE) + idx = acry_dev->exp_dw_mapping[j - 1]; + else /* mode == ASPEED_RSA_MOD_MODE */ + idx = acry_dev->mod_dw_mapping[j - 1]; + + dw_buf[idx] = cpu_to_le32(data); + } + } + + return nbits; +} + +static int aspeed_acry_rsa_transfer(struct aspeed_acry_dev *acry_dev) +{ + struct akcipher_request *req = acry_dev->req; + u8 __iomem *sram_buffer = acry_dev->acry_sram; + struct scatterlist *out_sg = req->dst; + static u8 dram_buffer[ASPEED_ACRY_SRAM_MAX_LEN]; + int leading_zero = 1; + int result_nbytes; + int i = 0, j; + int data_idx; + + /* Set Data Memory to AHB(CPU) Access Mode */ + ast_acry_write(acry_dev, ACRY_CMD_DMEM_AHB, ASPEED_ACRY_DMA_CMD); + + /* Disable ACRY SRAM protection */ + regmap_update_bits(acry_dev->ahbc, AHBC_REGION_PROT, + REGION_ACRYM, 0); + + result_nbytes = ASPEED_ACRY_SRAM_MAX_LEN; + + for (j = ASPEED_ACRY_SRAM_MAX_LEN - 1; j >= 0; j--) { + data_idx = acry_dev->data_byte_mapping[j]; + if (readb(sram_buffer + data_idx) == 0 && leading_zero) { + result_nbytes--; + } else { + leading_zero = 0; + dram_buffer[i] = readb(sram_buffer + data_idx); + i++; + } + } + + ACRY_DBG(acry_dev, "result_nbytes:%d, req->dst_len:%d\n", + result_nbytes, req->dst_len); + + if (result_nbytes <= req->dst_len) { + scatterwalk_map_and_copy(dram_buffer, out_sg, 0, result_nbytes, + 1); + req->dst_len = result_nbytes; + + } else { + dev_err(acry_dev->dev, "RSA engine error!\n"); + } + + memzero_explicit(acry_dev->buf_addr, ASPEED_ACRY_BUFF_SIZE); + + return aspeed_acry_complete(acry_dev, 0); +} + +static int aspeed_acry_rsa_trigger(struct aspeed_acry_dev *acry_dev) +{ + struct akcipher_request *req = acry_dev->req; + struct crypto_akcipher *cipher = crypto_akcipher_reqtfm(req); + struct aspeed_acry_ctx *ctx = akcipher_tfm_ctx(cipher); + int ne, nm; + + if (!ctx->n || !ctx->n_sz) { + dev_err(acry_dev->dev, "%s: key n is not set\n", __func__); + return -EINVAL; + } + + memzero_explicit(acry_dev->buf_addr, ASPEED_ACRY_BUFF_SIZE); + + /* Copy source data to DMA buffer */ + aspeed_acry_rsa_sg_copy_to_buffer(acry_dev, acry_dev->buf_addr, + req->src, req->src_len); + + nm = aspeed_acry_rsa_ctx_copy(acry_dev, acry_dev->buf_addr, ctx->n, + ctx->n_sz, ASPEED_RSA_MOD_MODE); + if (ctx->enc) { + if (!ctx->e || !ctx->e_sz) { + dev_err(acry_dev->dev, "%s: key e is not set\n", + __func__); + return -EINVAL; + } + /* Copy key e to DMA buffer */ + ne = aspeed_acry_rsa_ctx_copy(acry_dev, acry_dev->buf_addr, + ctx->e, ctx->e_sz, + ASPEED_RSA_EXP_MODE); + } else { + if (!ctx->d || !ctx->d_sz) { + dev_err(acry_dev->dev, "%s: key d is not set\n", + __func__); + return -EINVAL; + } + /* Copy key d to DMA buffer */ + ne = aspeed_acry_rsa_ctx_copy(acry_dev, acry_dev->buf_addr, + ctx->key.d, ctx->key.d_sz, + ASPEED_RSA_EXP_MODE); + } + + ast_acry_write(acry_dev, acry_dev->buf_dma_addr, + ASPEED_ACRY_DMA_SRC_BASE); + ast_acry_write(acry_dev, (ne << 16) + nm, + ASPEED_ACRY_RSA_KEY_LEN); + ast_acry_write(acry_dev, ASPEED_ACRY_BUFF_SIZE, + ASPEED_ACRY_DMA_LEN); + + acry_dev->resume = aspeed_acry_rsa_transfer; + + /* Enable ACRY SRAM protection */ + regmap_update_bits(acry_dev->ahbc, AHBC_REGION_PROT, + REGION_ACRYM, REGION_ACRYM); + + ast_acry_write(acry_dev, ACRY_RSA_ISR, ASPEED_ACRY_INT_MASK); + ast_acry_write(acry_dev, ACRY_CMD_DMA_SRAM_MODE_RSA | + ACRY_CMD_DMA_SRAM_AHB_ENGINE, ASPEED_ACRY_DMA_CMD); + + /* Trigger RSA engines */ + ast_acry_write(acry_dev, ACRY_CMD_RSA_TRIGGER | + ACRY_CMD_DMA_RSA_TRIGGER, ASPEED_ACRY_TRIGGER); + + return 0; +} + +static int aspeed_acry_rsa_enc(struct akcipher_request *req) +{ + struct crypto_akcipher *cipher = crypto_akcipher_reqtfm(req); + struct aspeed_acry_ctx *ctx = akcipher_tfm_ctx(cipher); + struct aspeed_acry_dev *acry_dev = ctx->acry_dev; + + ctx->trigger = aspeed_acry_rsa_trigger; + ctx->enc = 1; + + return aspeed_acry_handle_queue(acry_dev, req); +} + +static int aspeed_acry_rsa_dec(struct akcipher_request *req) +{ + struct crypto_akcipher *cipher = crypto_akcipher_reqtfm(req); + struct aspeed_acry_ctx *ctx = akcipher_tfm_ctx(cipher); + struct aspeed_acry_dev *acry_dev = ctx->acry_dev; + + ctx->trigger = aspeed_acry_rsa_trigger; + ctx->enc = 0; + + return aspeed_acry_handle_queue(acry_dev, req); +} + +static u8 *aspeed_rsa_key_copy(u8 *src, size_t len) +{ + return kmemdup(src, len, GFP_KERNEL); +} + +static int aspeed_rsa_set_n(struct aspeed_acry_ctx *ctx, u8 *value, + size_t len) +{ + ctx->n_sz = len; + ctx->n = aspeed_rsa_key_copy(value, len); + if (!ctx->n) + return -ENOMEM; + + return 0; +} + +static int aspeed_rsa_set_e(struct aspeed_acry_ctx *ctx, u8 *value, + size_t len) +{ + ctx->e_sz = len; + ctx->e = aspeed_rsa_key_copy(value, len); + if (!ctx->e) + return -ENOMEM; + + return 0; +} + +static int aspeed_rsa_set_d(struct aspeed_acry_ctx *ctx, u8 *value, + size_t len) +{ + ctx->d_sz = len; + ctx->d = aspeed_rsa_key_copy(value, len); + if (!ctx->d) + return -ENOMEM; + + return 0; +} + +static void aspeed_rsa_key_free(struct aspeed_acry_ctx *ctx) +{ + kfree_sensitive(ctx->n); + kfree_sensitive(ctx->e); + kfree_sensitive(ctx->d); + ctx->n_sz = 0; + ctx->e_sz = 0; + ctx->d_sz = 0; +} + +static int aspeed_acry_rsa_setkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen, int priv) +{ + struct aspeed_acry_ctx *ctx = akcipher_tfm_ctx(tfm); + struct aspeed_acry_dev *acry_dev = ctx->acry_dev; + int ret; + + if (priv) + ret = rsa_parse_priv_key(&ctx->key, key, keylen); + else + ret = rsa_parse_pub_key(&ctx->key, key, keylen); + + if (ret) { + dev_err(acry_dev->dev, "rsa parse key failed, ret:0x%x\n", + ret); + return ret; + } + + /* Aspeed engine supports up to 4096 bits, + * Use software fallback instead. + */ + if (ctx->key.n_sz > ASPEED_ACRY_RSA_MAX_KEY_LEN) + return 0; + + ret = aspeed_rsa_set_n(ctx, (u8 *)ctx->key.n, ctx->key.n_sz); + if (ret) + goto err; + + ret = aspeed_rsa_set_e(ctx, (u8 *)ctx->key.e, ctx->key.e_sz); + if (ret) + goto err; + + if (priv) { + ret = aspeed_rsa_set_d(ctx, (u8 *)ctx->key.d, ctx->key.d_sz); + if (ret) + goto err; + } + + return 0; + +err: + dev_err(acry_dev->dev, "rsa set key failed\n"); + aspeed_rsa_key_free(ctx); + + return ret; +} + +static int aspeed_acry_rsa_set_pub_key(struct crypto_akcipher *tfm, + const void *key, + unsigned int keylen) +{ + struct aspeed_acry_ctx *ctx = akcipher_tfm_ctx(tfm); + int ret; + + ret = crypto_akcipher_set_pub_key(ctx->fallback_tfm, key, keylen); + if (ret) + return ret; + + return aspeed_acry_rsa_setkey(tfm, key, keylen, 0); +} + +static int aspeed_acry_rsa_set_priv_key(struct crypto_akcipher *tfm, + const void *key, + unsigned int keylen) +{ + struct aspeed_acry_ctx *ctx = akcipher_tfm_ctx(tfm); + int ret; + + ret = crypto_akcipher_set_priv_key(ctx->fallback_tfm, key, keylen); + if (ret) + return ret; + + return aspeed_acry_rsa_setkey(tfm, key, keylen, 1); +} + +static unsigned int aspeed_acry_rsa_max_size(struct crypto_akcipher *tfm) +{ + struct aspeed_acry_ctx *ctx = akcipher_tfm_ctx(tfm); + + if (ctx->key.n_sz > ASPEED_ACRY_RSA_MAX_KEY_LEN) + return crypto_akcipher_maxsize(ctx->fallback_tfm); + + return ctx->n_sz; +} + +static int aspeed_acry_rsa_init_tfm(struct crypto_akcipher *tfm) +{ + struct aspeed_acry_ctx *ctx = akcipher_tfm_ctx(tfm); + struct akcipher_alg *alg = crypto_akcipher_alg(tfm); + const char *name = crypto_tfm_alg_name(&tfm->base); + struct aspeed_acry_alg *acry_alg; + + acry_alg = container_of(alg, struct aspeed_acry_alg, akcipher.base); + + ctx->acry_dev = acry_alg->acry_dev; + + ctx->fallback_tfm = crypto_alloc_akcipher(name, 0, CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fallback_tfm)) { + dev_err(ctx->acry_dev->dev, "ERROR: Cannot allocate fallback for %s %ld\n", + name, PTR_ERR(ctx->fallback_tfm)); + return PTR_ERR(ctx->fallback_tfm); + } + + return 0; +} + +static void aspeed_acry_rsa_exit_tfm(struct crypto_akcipher *tfm) +{ + struct aspeed_acry_ctx *ctx = akcipher_tfm_ctx(tfm); + + crypto_free_akcipher(ctx->fallback_tfm); +} + +static struct aspeed_acry_alg aspeed_acry_akcipher_algs[] = { + { + .akcipher.base = { + .encrypt = aspeed_acry_rsa_enc, + .decrypt = aspeed_acry_rsa_dec, + .set_pub_key = aspeed_acry_rsa_set_pub_key, + .set_priv_key = aspeed_acry_rsa_set_priv_key, + .max_size = aspeed_acry_rsa_max_size, + .init = aspeed_acry_rsa_init_tfm, + .exit = aspeed_acry_rsa_exit_tfm, + .base = { + .cra_name = "rsa", + .cra_driver_name = "aspeed-rsa", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct aspeed_acry_ctx), + }, + }, + .akcipher.op = { + .do_one_request = aspeed_acry_do_request, + }, + }, +}; + +static void aspeed_acry_register(struct aspeed_acry_dev *acry_dev) +{ + int i, rc; + + for (i = 0; i < ARRAY_SIZE(aspeed_acry_akcipher_algs); i++) { + aspeed_acry_akcipher_algs[i].acry_dev = acry_dev; + rc = crypto_engine_register_akcipher(&aspeed_acry_akcipher_algs[i].akcipher); + if (rc) { + ACRY_DBG(acry_dev, "Failed to register %s\n", + aspeed_acry_akcipher_algs[i].akcipher.base.base.cra_name); + } + } +} + +static void aspeed_acry_unregister(struct aspeed_acry_dev *acry_dev) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(aspeed_acry_akcipher_algs); i++) + crypto_engine_unregister_akcipher(&aspeed_acry_akcipher_algs[i].akcipher); +} + +/* ACRY interrupt service routine. */ +static irqreturn_t aspeed_acry_irq(int irq, void *dev) +{ + struct aspeed_acry_dev *acry_dev = (struct aspeed_acry_dev *)dev; + u32 sts; + + sts = ast_acry_read(acry_dev, ASPEED_ACRY_STATUS); + ast_acry_write(acry_dev, sts, ASPEED_ACRY_STATUS); + + ACRY_DBG(acry_dev, "irq sts:0x%x\n", sts); + + if (sts & ACRY_RSA_ISR) { + /* Stop RSA engine */ + ast_acry_write(acry_dev, 0, ASPEED_ACRY_TRIGGER); + + if (acry_dev->flags & CRYPTO_FLAGS_BUSY) + tasklet_schedule(&acry_dev->done_task); + else + dev_err(acry_dev->dev, "RSA no active requests.\n"); + } + + return IRQ_HANDLED; +} + +/* + * ACRY SRAM has its own memory layout. + * Set the DRAM to SRAM indexing for future used. + */ +static void aspeed_acry_sram_mapping(struct aspeed_acry_dev *acry_dev) +{ + int i, j = 0; + + for (i = 0; i < (ASPEED_ACRY_SRAM_MAX_LEN / BYTES_PER_DWORD); i++) { + acry_dev->exp_dw_mapping[i] = j; + acry_dev->mod_dw_mapping[i] = j + 4; + acry_dev->data_byte_mapping[(i * 4)] = (j + 8) * 4; + acry_dev->data_byte_mapping[(i * 4) + 1] = (j + 8) * 4 + 1; + acry_dev->data_byte_mapping[(i * 4) + 2] = (j + 8) * 4 + 2; + acry_dev->data_byte_mapping[(i * 4) + 3] = (j + 8) * 4 + 3; + j++; + j = j % 4 ? j : j + 8; + } +} + +static void aspeed_acry_done_task(unsigned long data) +{ + struct aspeed_acry_dev *acry_dev = (struct aspeed_acry_dev *)data; + + (void)acry_dev->resume(acry_dev); +} + +static const struct of_device_id aspeed_acry_of_matches[] = { + { .compatible = "aspeed,ast2600-acry", }, + {}, +}; + +static int aspeed_acry_probe(struct platform_device *pdev) +{ + struct aspeed_acry_dev *acry_dev; + struct device *dev = &pdev->dev; + int rc; + + acry_dev = devm_kzalloc(dev, sizeof(struct aspeed_acry_dev), + GFP_KERNEL); + if (!acry_dev) + return -ENOMEM; + + acry_dev->dev = dev; + + platform_set_drvdata(pdev, acry_dev); + + acry_dev->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(acry_dev->regs)) + return PTR_ERR(acry_dev->regs); + + acry_dev->acry_sram = devm_platform_ioremap_resource(pdev, 1); + if (IS_ERR(acry_dev->acry_sram)) + return PTR_ERR(acry_dev->acry_sram); + + /* Get irq number and register it */ + acry_dev->irq = platform_get_irq(pdev, 0); + if (acry_dev->irq < 0) + return acry_dev->irq; + + rc = devm_request_irq(dev, acry_dev->irq, aspeed_acry_irq, 0, + dev_name(dev), acry_dev); + if (rc) + return rc; + + acry_dev->clk = devm_clk_get_enabled(dev, NULL); + if (IS_ERR(acry_dev->clk)) { + dev_err(dev, "Failed to get acry clk\n"); + return PTR_ERR(acry_dev->clk); + } + + acry_dev->ahbc = syscon_regmap_lookup_by_phandle(dev->of_node, + "aspeed,ahbc"); + if (IS_ERR(acry_dev->ahbc)) { + dev_err(dev, "Failed to get AHBC regmap\n"); + return -ENODEV; + } + + /* Initialize crypto hardware engine structure for RSA */ + acry_dev->crypt_engine_rsa = crypto_engine_alloc_init(dev, true); + if (!acry_dev->crypt_engine_rsa) { + rc = -ENOMEM; + goto clk_exit; + } + + rc = crypto_engine_start(acry_dev->crypt_engine_rsa); + if (rc) + goto err_engine_rsa_start; + + tasklet_init(&acry_dev->done_task, aspeed_acry_done_task, + (unsigned long)acry_dev); + + /* Set Data Memory to AHB(CPU) Access Mode */ + ast_acry_write(acry_dev, ACRY_CMD_DMEM_AHB, ASPEED_ACRY_DMA_CMD); + + /* Initialize ACRY SRAM index */ + aspeed_acry_sram_mapping(acry_dev); + + acry_dev->buf_addr = dmam_alloc_coherent(dev, ASPEED_ACRY_BUFF_SIZE, + &acry_dev->buf_dma_addr, + GFP_KERNEL); + if (!acry_dev->buf_addr) { + rc = -ENOMEM; + goto err_engine_rsa_start; + } + + aspeed_acry_register(acry_dev); + + dev_info(dev, "Aspeed ACRY Accelerator successfully registered\n"); + + return 0; + +err_engine_rsa_start: + crypto_engine_exit(acry_dev->crypt_engine_rsa); +clk_exit: + + return rc; +} + +static void aspeed_acry_remove(struct platform_device *pdev) +{ + struct aspeed_acry_dev *acry_dev = platform_get_drvdata(pdev); + + aspeed_acry_unregister(acry_dev); + crypto_engine_exit(acry_dev->crypt_engine_rsa); + tasklet_kill(&acry_dev->done_task); +} + +MODULE_DEVICE_TABLE(of, aspeed_acry_of_matches); + +static struct platform_driver aspeed_acry_driver = { + .probe = aspeed_acry_probe, + .remove = aspeed_acry_remove, + .driver = { + .name = KBUILD_MODNAME, + .of_match_table = aspeed_acry_of_matches, + }, +}; + +module_platform_driver(aspeed_acry_driver); + +MODULE_AUTHOR("Neal Liu "); +MODULE_DESCRIPTION("ASPEED ACRY driver for hardware RSA Engine"); +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/aspeed/aspeed-hace-crypto.c b/drivers/crypto/aspeed/aspeed-hace-crypto.c new file mode 100644 index 000000000..fa201dae1 --- /dev/null +++ b/drivers/crypto/aspeed/aspeed-hace-crypto.c @@ -0,0 +1,955 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (c) 2021 Aspeed Technology Inc. + */ + +#include "aspeed-hace.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_CRYPTO_DEV_ASPEED_HACE_CRYPTO_DEBUG +#define CIPHER_DBG(h, fmt, ...) \ + dev_info((h)->dev, "%s() " fmt, __func__, ##__VA_ARGS__) +#else +#define CIPHER_DBG(h, fmt, ...) \ + dev_dbg((h)->dev, "%s() " fmt, __func__, ##__VA_ARGS__) +#endif + +static int aspeed_crypto_do_fallback(struct skcipher_request *areq) +{ + struct aspeed_cipher_reqctx *rctx = skcipher_request_ctx(areq); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); + int err; + + skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags, + areq->base.complete, areq->base.data); + skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst, + areq->cryptlen, areq->iv); + + if (rctx->enc_cmd & HACE_CMD_ENCRYPT) + err = crypto_skcipher_encrypt(&rctx->fallback_req); + else + err = crypto_skcipher_decrypt(&rctx->fallback_req); + + return err; +} + +static bool aspeed_crypto_need_fallback(struct skcipher_request *areq) +{ + struct aspeed_cipher_reqctx *rctx = skcipher_request_ctx(areq); + + if (areq->cryptlen == 0) + return true; + + if ((rctx->enc_cmd & HACE_CMD_DES_SELECT) && + !IS_ALIGNED(areq->cryptlen, DES_BLOCK_SIZE)) + return true; + + if ((!(rctx->enc_cmd & HACE_CMD_DES_SELECT)) && + !IS_ALIGNED(areq->cryptlen, AES_BLOCK_SIZE)) + return true; + + return false; +} + +static int aspeed_hace_crypto_handle_queue(struct aspeed_hace_dev *hace_dev, + struct skcipher_request *req) +{ + if (hace_dev->version == AST2500_VERSION && + aspeed_crypto_need_fallback(req)) { + CIPHER_DBG(hace_dev, "SW fallback\n"); + return aspeed_crypto_do_fallback(req); + } + + return crypto_transfer_skcipher_request_to_engine( + hace_dev->crypt_engine_crypto, req); +} + +static int aspeed_crypto_do_request(struct crypto_engine *engine, void *areq) +{ + struct skcipher_request *req = skcipher_request_cast(areq); + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); + struct aspeed_hace_dev *hace_dev = ctx->hace_dev; + struct aspeed_engine_crypto *crypto_engine; + int rc; + + crypto_engine = &hace_dev->crypto_engine; + crypto_engine->req = req; + crypto_engine->flags |= CRYPTO_FLAGS_BUSY; + + rc = ctx->start(hace_dev); + + if (rc != -EINPROGRESS) + return -EIO; + + return 0; +} + +static int aspeed_sk_complete(struct aspeed_hace_dev *hace_dev, int err) +{ + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + struct aspeed_cipher_reqctx *rctx; + struct skcipher_request *req; + + CIPHER_DBG(hace_dev, "\n"); + + req = crypto_engine->req; + rctx = skcipher_request_ctx(req); + + if (rctx->enc_cmd & HACE_CMD_IV_REQUIRE) { + if (rctx->enc_cmd & HACE_CMD_DES_SELECT) + memcpy(req->iv, crypto_engine->cipher_ctx + + DES_KEY_SIZE, DES_KEY_SIZE); + else + memcpy(req->iv, crypto_engine->cipher_ctx, + AES_BLOCK_SIZE); + } + + crypto_engine->flags &= ~CRYPTO_FLAGS_BUSY; + + crypto_finalize_skcipher_request(hace_dev->crypt_engine_crypto, req, + err); + + return err; +} + +static int aspeed_sk_transfer_sg(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + struct device *dev = hace_dev->dev; + struct aspeed_cipher_reqctx *rctx; + struct skcipher_request *req; + + CIPHER_DBG(hace_dev, "\n"); + + req = crypto_engine->req; + rctx = skcipher_request_ctx(req); + + if (req->src == req->dst) { + dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_BIDIRECTIONAL); + } else { + dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE); + dma_unmap_sg(dev, req->dst, rctx->dst_nents, DMA_FROM_DEVICE); + } + + return aspeed_sk_complete(hace_dev, 0); +} + +static int aspeed_sk_transfer(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + struct aspeed_cipher_reqctx *rctx; + struct skcipher_request *req; + struct scatterlist *out_sg; + int nbytes = 0; + int rc = 0; + + req = crypto_engine->req; + rctx = skcipher_request_ctx(req); + out_sg = req->dst; + + /* Copy output buffer to dst scatter-gather lists */ + nbytes = sg_copy_from_buffer(out_sg, rctx->dst_nents, + crypto_engine->cipher_addr, req->cryptlen); + if (!nbytes) { + dev_warn(hace_dev->dev, "invalid sg copy, %s:0x%x, %s:0x%x\n", + "nbytes", nbytes, "cryptlen", req->cryptlen); + rc = -EINVAL; + } + + CIPHER_DBG(hace_dev, "%s:%d, %s:%d, %s:%d, %s:%p\n", + "nbytes", nbytes, "req->cryptlen", req->cryptlen, + "nb_out_sg", rctx->dst_nents, + "cipher addr", crypto_engine->cipher_addr); + + return aspeed_sk_complete(hace_dev, rc); +} + +static int aspeed_sk_start(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + struct aspeed_cipher_reqctx *rctx; + struct skcipher_request *req; + struct scatterlist *in_sg; + int nbytes; + + req = crypto_engine->req; + rctx = skcipher_request_ctx(req); + in_sg = req->src; + + nbytes = sg_copy_to_buffer(in_sg, rctx->src_nents, + crypto_engine->cipher_addr, req->cryptlen); + + CIPHER_DBG(hace_dev, "%s:%d, %s:%d, %s:%d, %s:%p\n", + "nbytes", nbytes, "req->cryptlen", req->cryptlen, + "nb_in_sg", rctx->src_nents, + "cipher addr", crypto_engine->cipher_addr); + + if (!nbytes) { + dev_warn(hace_dev->dev, "invalid sg copy, %s:0x%x, %s:0x%x\n", + "nbytes", nbytes, "cryptlen", req->cryptlen); + return -EINVAL; + } + + crypto_engine->resume = aspeed_sk_transfer; + + /* Trigger engines */ + ast_hace_write(hace_dev, crypto_engine->cipher_dma_addr, + ASPEED_HACE_SRC); + ast_hace_write(hace_dev, crypto_engine->cipher_dma_addr, + ASPEED_HACE_DEST); + ast_hace_write(hace_dev, req->cryptlen, ASPEED_HACE_DATA_LEN); + ast_hace_write(hace_dev, rctx->enc_cmd, ASPEED_HACE_CMD); + + return -EINPROGRESS; +} + +static int aspeed_sk_start_sg(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + struct aspeed_sg_list *src_list, *dst_list; + dma_addr_t src_dma_addr, dst_dma_addr; + struct aspeed_cipher_reqctx *rctx; + struct skcipher_request *req; + struct scatterlist *s; + int src_sg_len; + int dst_sg_len; + int total, i; + int rc; + + CIPHER_DBG(hace_dev, "\n"); + + req = crypto_engine->req; + rctx = skcipher_request_ctx(req); + + rctx->enc_cmd |= HACE_CMD_DES_SG_CTRL | HACE_CMD_SRC_SG_CTRL | + HACE_CMD_AES_KEY_HW_EXP | HACE_CMD_MBUS_REQ_SYNC_EN; + + /* BIDIRECTIONAL */ + if (req->dst == req->src) { + src_sg_len = dma_map_sg(hace_dev->dev, req->src, + rctx->src_nents, DMA_BIDIRECTIONAL); + dst_sg_len = src_sg_len; + if (!src_sg_len) { + dev_warn(hace_dev->dev, "dma_map_sg() src error\n"); + return -EINVAL; + } + + } else { + src_sg_len = dma_map_sg(hace_dev->dev, req->src, + rctx->src_nents, DMA_TO_DEVICE); + if (!src_sg_len) { + dev_warn(hace_dev->dev, "dma_map_sg() src error\n"); + return -EINVAL; + } + + dst_sg_len = dma_map_sg(hace_dev->dev, req->dst, + rctx->dst_nents, DMA_FROM_DEVICE); + if (!dst_sg_len) { + dev_warn(hace_dev->dev, "dma_map_sg() dst error\n"); + rc = -EINVAL; + goto free_req_src; + } + } + + src_list = (struct aspeed_sg_list *)crypto_engine->cipher_addr; + src_dma_addr = crypto_engine->cipher_dma_addr; + total = req->cryptlen; + + for_each_sg(req->src, s, src_sg_len, i) { + u32 phy_addr = sg_dma_address(s); + u32 len = sg_dma_len(s); + + if (total > len) + total -= len; + else { + /* last sg list */ + len = total; + len |= BIT(31); + total = 0; + } + + src_list[i].phy_addr = cpu_to_le32(phy_addr); + src_list[i].len = cpu_to_le32(len); + } + + if (total != 0) { + rc = -EINVAL; + goto free_req; + } + + if (req->dst == req->src) { + dst_list = src_list; + dst_dma_addr = src_dma_addr; + + } else { + dst_list = (struct aspeed_sg_list *)crypto_engine->dst_sg_addr; + dst_dma_addr = crypto_engine->dst_sg_dma_addr; + total = req->cryptlen; + + for_each_sg(req->dst, s, dst_sg_len, i) { + u32 phy_addr = sg_dma_address(s); + u32 len = sg_dma_len(s); + + if (total > len) + total -= len; + else { + /* last sg list */ + len = total; + len |= BIT(31); + total = 0; + } + + dst_list[i].phy_addr = cpu_to_le32(phy_addr); + dst_list[i].len = cpu_to_le32(len); + + } + + dst_list[dst_sg_len].phy_addr = 0; + dst_list[dst_sg_len].len = 0; + } + + if (total != 0) { + rc = -EINVAL; + goto free_req; + } + + crypto_engine->resume = aspeed_sk_transfer_sg; + + /* Memory barrier to ensure all data setup before engine starts */ + mb(); + + /* Trigger engines */ + ast_hace_write(hace_dev, src_dma_addr, ASPEED_HACE_SRC); + ast_hace_write(hace_dev, dst_dma_addr, ASPEED_HACE_DEST); + ast_hace_write(hace_dev, req->cryptlen, ASPEED_HACE_DATA_LEN); + ast_hace_write(hace_dev, rctx->enc_cmd, ASPEED_HACE_CMD); + + return -EINPROGRESS; + +free_req: + if (req->dst == req->src) { + dma_unmap_sg(hace_dev->dev, req->src, rctx->src_nents, + DMA_BIDIRECTIONAL); + + } else { + dma_unmap_sg(hace_dev->dev, req->dst, rctx->dst_nents, + DMA_FROM_DEVICE); + dma_unmap_sg(hace_dev->dev, req->src, rctx->src_nents, + DMA_TO_DEVICE); + } + + return rc; + +free_req_src: + dma_unmap_sg(hace_dev->dev, req->src, rctx->src_nents, DMA_TO_DEVICE); + + return rc; +} + +static int aspeed_hace_skcipher_trigger(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + struct aspeed_cipher_reqctx *rctx; + struct crypto_skcipher *cipher; + struct aspeed_cipher_ctx *ctx; + struct skcipher_request *req; + + CIPHER_DBG(hace_dev, "\n"); + + req = crypto_engine->req; + rctx = skcipher_request_ctx(req); + cipher = crypto_skcipher_reqtfm(req); + ctx = crypto_skcipher_ctx(cipher); + + /* enable interrupt */ + rctx->enc_cmd |= HACE_CMD_ISR_EN; + + rctx->dst_nents = sg_nents(req->dst); + rctx->src_nents = sg_nents(req->src); + + ast_hace_write(hace_dev, crypto_engine->cipher_ctx_dma, + ASPEED_HACE_CONTEXT); + + if (rctx->enc_cmd & HACE_CMD_IV_REQUIRE) { + if (rctx->enc_cmd & HACE_CMD_DES_SELECT) + memcpy(crypto_engine->cipher_ctx + DES_BLOCK_SIZE, + req->iv, DES_BLOCK_SIZE); + else + memcpy(crypto_engine->cipher_ctx, req->iv, + AES_BLOCK_SIZE); + } + + if (hace_dev->version == AST2600_VERSION) { + memcpy(crypto_engine->cipher_ctx + 16, ctx->key, ctx->key_len); + + return aspeed_sk_start_sg(hace_dev); + } + + memcpy(crypto_engine->cipher_ctx + 16, ctx->key, AES_MAX_KEYLENGTH); + + return aspeed_sk_start(hace_dev); +} + +static int aspeed_des_crypt(struct skcipher_request *req, u32 cmd) +{ + struct aspeed_cipher_reqctx *rctx = skcipher_request_ctx(req); + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); + struct aspeed_hace_dev *hace_dev = ctx->hace_dev; + u32 crypto_alg = cmd & HACE_CMD_OP_MODE_MASK; + + CIPHER_DBG(hace_dev, "\n"); + + if (crypto_alg == HACE_CMD_CBC || crypto_alg == HACE_CMD_ECB) { + if (!IS_ALIGNED(req->cryptlen, DES_BLOCK_SIZE)) + return -EINVAL; + } + + rctx->enc_cmd = cmd | HACE_CMD_DES_SELECT | HACE_CMD_RI_WO_DATA_ENABLE | + HACE_CMD_DES | HACE_CMD_CONTEXT_LOAD_ENABLE | + HACE_CMD_CONTEXT_SAVE_ENABLE; + + return aspeed_hace_crypto_handle_queue(hace_dev, req); +} + +static int aspeed_des_setkey(struct crypto_skcipher *cipher, const u8 *key, + unsigned int keylen) +{ + struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); + struct aspeed_hace_dev *hace_dev = ctx->hace_dev; + int rc; + + CIPHER_DBG(hace_dev, "keylen: %d bits\n", keylen); + + if (keylen != DES_KEY_SIZE && keylen != DES3_EDE_KEY_SIZE) { + dev_warn(hace_dev->dev, "invalid keylen: %d bits\n", keylen); + return -EINVAL; + } + + if (keylen == DES_KEY_SIZE) { + rc = crypto_des_verify_key(tfm, key); + if (rc) + return rc; + + } else if (keylen == DES3_EDE_KEY_SIZE) { + rc = crypto_des3_ede_verify_key(tfm, key); + if (rc) + return rc; + } + + memcpy(ctx->key, key, keylen); + ctx->key_len = keylen; + + crypto_skcipher_clear_flags(ctx->fallback_tfm, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(ctx->fallback_tfm, cipher->base.crt_flags & + CRYPTO_TFM_REQ_MASK); + + return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen); +} + +static int aspeed_tdes_ctr_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CTR | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_tdes_ctr_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CTR | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_tdes_cbc_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CBC | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_tdes_cbc_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CBC | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_tdes_ecb_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_ECB | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_tdes_ecb_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_ECB | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_des_ctr_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CTR | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_des_ctr_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CTR | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_des_cbc_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CBC | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_des_cbc_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CBC | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_des_ecb_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_ECB | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_des_ecb_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_ECB | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_aes_crypt(struct skcipher_request *req, u32 cmd) +{ + struct aspeed_cipher_reqctx *rctx = skcipher_request_ctx(req); + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); + struct aspeed_hace_dev *hace_dev = ctx->hace_dev; + u32 crypto_alg = cmd & HACE_CMD_OP_MODE_MASK; + + if (crypto_alg == HACE_CMD_CBC || crypto_alg == HACE_CMD_ECB) { + if (!IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE)) + return -EINVAL; + } + + CIPHER_DBG(hace_dev, "%s\n", + (cmd & HACE_CMD_ENCRYPT) ? "encrypt" : "decrypt"); + + cmd |= HACE_CMD_AES_SELECT | HACE_CMD_RI_WO_DATA_ENABLE | + HACE_CMD_CONTEXT_LOAD_ENABLE | HACE_CMD_CONTEXT_SAVE_ENABLE; + + switch (ctx->key_len) { + case AES_KEYSIZE_128: + cmd |= HACE_CMD_AES128; + break; + case AES_KEYSIZE_192: + cmd |= HACE_CMD_AES192; + break; + case AES_KEYSIZE_256: + cmd |= HACE_CMD_AES256; + break; + default: + return -EINVAL; + } + + rctx->enc_cmd = cmd; + + return aspeed_hace_crypto_handle_queue(hace_dev, req); +} + +static int aspeed_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, + unsigned int keylen) +{ + struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); + struct aspeed_hace_dev *hace_dev = ctx->hace_dev; + struct crypto_aes_ctx gen_aes_key; + + CIPHER_DBG(hace_dev, "keylen: %d bits\n", (keylen * 8)); + + if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) + return -EINVAL; + + if (ctx->hace_dev->version == AST2500_VERSION) { + aes_expandkey(&gen_aes_key, key, keylen); + memcpy(ctx->key, gen_aes_key.key_enc, AES_MAX_KEYLENGTH); + + } else { + memcpy(ctx->key, key, keylen); + } + + ctx->key_len = keylen; + + crypto_skcipher_clear_flags(ctx->fallback_tfm, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(ctx->fallback_tfm, cipher->base.crt_flags & + CRYPTO_TFM_REQ_MASK); + + return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen); +} + +static int aspeed_aes_ctr_decrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CTR); +} + +static int aspeed_aes_ctr_encrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CTR); +} + +static int aspeed_aes_cbc_decrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CBC); +} + +static int aspeed_aes_cbc_encrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CBC); +} + +static int aspeed_aes_ecb_decrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_ECB); +} + +static int aspeed_aes_ecb_encrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_ECB); +} + +static int aspeed_crypto_cra_init(struct crypto_skcipher *tfm) +{ + struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + const char *name = crypto_tfm_alg_name(&tfm->base); + struct aspeed_hace_alg *crypto_alg; + + + crypto_alg = container_of(alg, struct aspeed_hace_alg, alg.skcipher.base); + ctx->hace_dev = crypto_alg->hace_dev; + ctx->start = aspeed_hace_skcipher_trigger; + + CIPHER_DBG(ctx->hace_dev, "%s\n", name); + + ctx->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fallback_tfm)) { + dev_err(ctx->hace_dev->dev, "ERROR: Cannot allocate fallback for %s %ld\n", + name, PTR_ERR(ctx->fallback_tfm)); + return PTR_ERR(ctx->fallback_tfm); + } + + crypto_skcipher_set_reqsize(tfm, sizeof(struct aspeed_cipher_reqctx) + + crypto_skcipher_reqsize(ctx->fallback_tfm)); + + return 0; +} + +static void aspeed_crypto_cra_exit(struct crypto_skcipher *tfm) +{ + struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); + struct aspeed_hace_dev *hace_dev = ctx->hace_dev; + + CIPHER_DBG(hace_dev, "%s\n", crypto_tfm_alg_name(&tfm->base)); + crypto_free_skcipher(ctx->fallback_tfm); +} + +static struct aspeed_hace_alg aspeed_crypto_algs[] = { + { + .alg.skcipher.base = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = aspeed_aes_setkey, + .encrypt = aspeed_aes_ecb_encrypt, + .decrypt = aspeed_aes_ecb_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "aspeed-ecb-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + }, + .alg.skcipher.op = { + .do_one_request = aspeed_crypto_do_request, + }, + }, + { + .alg.skcipher.base = { + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = aspeed_aes_setkey, + .encrypt = aspeed_aes_cbc_encrypt, + .decrypt = aspeed_aes_cbc_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "aspeed-cbc-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + }, + .alg.skcipher.op = { + .do_one_request = aspeed_crypto_do_request, + }, + }, + { + .alg.skcipher.base = { + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_des_ecb_encrypt, + .decrypt = aspeed_des_ecb_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "ecb(des)", + .cra_driver_name = "aspeed-ecb-des", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + }, + .alg.skcipher.op = { + .do_one_request = aspeed_crypto_do_request, + }, + }, + { + .alg.skcipher.base = { + .ivsize = DES_BLOCK_SIZE, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_des_cbc_encrypt, + .decrypt = aspeed_des_cbc_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "cbc(des)", + .cra_driver_name = "aspeed-cbc-des", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + }, + .alg.skcipher.op = { + .do_one_request = aspeed_crypto_do_request, + }, + }, + { + .alg.skcipher.base = { + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_tdes_ecb_encrypt, + .decrypt = aspeed_tdes_ecb_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "aspeed-ecb-tdes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + }, + .alg.skcipher.op = { + .do_one_request = aspeed_crypto_do_request, + }, + }, + { + .alg.skcipher.base = { + .ivsize = DES_BLOCK_SIZE, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_tdes_cbc_encrypt, + .decrypt = aspeed_tdes_cbc_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "aspeed-cbc-tdes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + }, + .alg.skcipher.op = { + .do_one_request = aspeed_crypto_do_request, + }, + }, +}; + +static struct aspeed_hace_alg aspeed_crypto_algs_g6[] = { + { + .alg.skcipher.base = { + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = aspeed_aes_setkey, + .encrypt = aspeed_aes_ctr_encrypt, + .decrypt = aspeed_aes_ctr_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "ctr(aes)", + .cra_driver_name = "aspeed-ctr-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + }, + .alg.skcipher.op = { + .do_one_request = aspeed_crypto_do_request, + }, + }, + { + .alg.skcipher.base = { + .ivsize = DES_BLOCK_SIZE, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_des_ctr_encrypt, + .decrypt = aspeed_des_ctr_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "ctr(des)", + .cra_driver_name = "aspeed-ctr-des", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + }, + .alg.skcipher.op = { + .do_one_request = aspeed_crypto_do_request, + }, + }, + { + .alg.skcipher.base = { + .ivsize = DES_BLOCK_SIZE, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_tdes_ctr_encrypt, + .decrypt = aspeed_tdes_ctr_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "ctr(des3_ede)", + .cra_driver_name = "aspeed-ctr-tdes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + }, + .alg.skcipher.op = { + .do_one_request = aspeed_crypto_do_request, + }, + }, + +}; + +void aspeed_unregister_hace_crypto_algs(struct aspeed_hace_dev *hace_dev) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(aspeed_crypto_algs); i++) + crypto_engine_unregister_skcipher(&aspeed_crypto_algs[i].alg.skcipher); + + if (hace_dev->version != AST2600_VERSION) + return; + + for (i = 0; i < ARRAY_SIZE(aspeed_crypto_algs_g6); i++) + crypto_engine_unregister_skcipher(&aspeed_crypto_algs_g6[i].alg.skcipher); +} + +void aspeed_register_hace_crypto_algs(struct aspeed_hace_dev *hace_dev) +{ + int rc, i; + + CIPHER_DBG(hace_dev, "\n"); + + for (i = 0; i < ARRAY_SIZE(aspeed_crypto_algs); i++) { + aspeed_crypto_algs[i].hace_dev = hace_dev; + rc = crypto_engine_register_skcipher(&aspeed_crypto_algs[i].alg.skcipher); + if (rc) { + CIPHER_DBG(hace_dev, "Failed to register %s\n", + aspeed_crypto_algs[i].alg.skcipher.base.base.cra_name); + } + } + + if (hace_dev->version != AST2600_VERSION) + return; + + for (i = 0; i < ARRAY_SIZE(aspeed_crypto_algs_g6); i++) { + aspeed_crypto_algs_g6[i].hace_dev = hace_dev; + rc = crypto_engine_register_skcipher(&aspeed_crypto_algs_g6[i].alg.skcipher); + if (rc) { + CIPHER_DBG(hace_dev, "Failed to register %s\n", + aspeed_crypto_algs_g6[i].alg.skcipher.base.base.cra_name); + } + } +} diff --git a/drivers/crypto/aspeed/aspeed-hace-hash.c b/drivers/crypto/aspeed/aspeed-hace-hash.c new file mode 100644 index 000000000..6f0d03cfb --- /dev/null +++ b/drivers/crypto/aspeed/aspeed-hace-hash.c @@ -0,0 +1,825 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (c) 2021 Aspeed Technology Inc. + */ + +#include "aspeed-hace.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_CRYPTO_DEV_ASPEED_DEBUG +#define AHASH_DBG(h, fmt, ...) \ + dev_info((h)->dev, "%s() " fmt, __func__, ##__VA_ARGS__) +#else +#define AHASH_DBG(h, fmt, ...) \ + dev_dbg((h)->dev, "%s() " fmt, __func__, ##__VA_ARGS__) +#endif + +/* Initialization Vectors for SHA-family */ +static const __be32 sha1_iv[8] = { + cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1), + cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3), + cpu_to_be32(SHA1_H4), 0, 0, 0 +}; + +static const __be32 sha224_iv[8] = { + cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1), + cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3), + cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5), + cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7), +}; + +static const __be32 sha256_iv[8] = { + cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1), + cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3), + cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5), + cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7), +}; + +static const __be64 sha384_iv[8] = { + cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1), + cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3), + cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5), + cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7) +}; + +static const __be64 sha512_iv[8] = { + cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1), + cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3), + cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5), + cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7) +}; + +static int aspeed_sham_init(struct ahash_request *req); +static int aspeed_ahash_req_update(struct aspeed_hace_dev *hace_dev); + +static int aspeed_sham_export(struct ahash_request *req, void *out) +{ + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + union { + u8 *u8; + u64 *u64; + } p = { .u8 = out }; + + memcpy(out, rctx->digest, rctx->ivsize); + p.u8 += rctx->ivsize; + put_unaligned(rctx->digcnt[0], p.u64++); + if (rctx->ivsize == 64) + put_unaligned(rctx->digcnt[1], p.u64); + return 0; +} + +static int aspeed_sham_import(struct ahash_request *req, const void *in) +{ + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + union { + const u8 *u8; + const u64 *u64; + } p = { .u8 = in }; + int err; + + err = aspeed_sham_init(req); + if (err) + return err; + + memcpy(rctx->digest, in, rctx->ivsize); + p.u8 += rctx->ivsize; + rctx->digcnt[0] = get_unaligned(p.u64++); + if (rctx->ivsize == 64) + rctx->digcnt[1] = get_unaligned(p.u64); + return 0; +} + +/* The purpose of this padding is to ensure that the padded message is a + * multiple of 512 bits (SHA1/SHA224/SHA256) or 1024 bits (SHA384/SHA512). + * The bit "1" is appended at the end of the message followed by + * "padlen-1" zero bits. Then a 64 bits block (SHA1/SHA224/SHA256) or + * 128 bits block (SHA384/SHA512) equals to the message length in bits + * is appended. + * + * For SHA1/SHA224/SHA256, padlen is calculated as followed: + * - if message length < 56 bytes then padlen = 56 - message length + * - else padlen = 64 + 56 - message length + * + * For SHA384/SHA512, padlen is calculated as followed: + * - if message length < 112 bytes then padlen = 112 - message length + * - else padlen = 128 + 112 - message length + */ +static int aspeed_ahash_fill_padding(struct aspeed_hace_dev *hace_dev, + struct aspeed_sham_reqctx *rctx, u8 *buf) +{ + unsigned int index, padlen, bitslen; + __be64 bits[2]; + + AHASH_DBG(hace_dev, "rctx flags:0x%x\n", (u32)rctx->flags); + + switch (rctx->flags & SHA_FLAGS_MASK) { + case SHA_FLAGS_SHA1: + case SHA_FLAGS_SHA224: + case SHA_FLAGS_SHA256: + bits[0] = cpu_to_be64(rctx->digcnt[0] << 3); + index = rctx->digcnt[0] & 0x3f; + padlen = (index < 56) ? (56 - index) : ((64 + 56) - index); + bitslen = 8; + break; + default: + bits[1] = cpu_to_be64(rctx->digcnt[0] << 3); + bits[0] = cpu_to_be64(rctx->digcnt[1] << 3 | + rctx->digcnt[0] >> 61); + index = rctx->digcnt[0] & 0x7f; + padlen = (index < 112) ? (112 - index) : ((128 + 112) - index); + bitslen = 16; + break; + } + buf[0] = 0x80; + memset(buf + 1, 0, padlen - 1); + memcpy(buf + padlen, bits, bitslen); + return padlen + bitslen; +} + +static void aspeed_ahash_update_counter(struct aspeed_sham_reqctx *rctx, + unsigned int len) +{ + rctx->offset += len; + rctx->digcnt[0] += len; + if (rctx->digcnt[0] < len) + rctx->digcnt[1]++; +} + +/* + * Prepare DMA buffer before hardware engine + * processing. + */ +static int aspeed_ahash_dma_prepare(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + unsigned int length, remain; + bool final = false; + + length = rctx->total - rctx->offset; + remain = length - round_down(length, rctx->block_size); + + AHASH_DBG(hace_dev, "length:0x%x, remain:0x%x\n", length, remain); + + if (length > ASPEED_HASH_SRC_DMA_BUF_LEN) + length = ASPEED_HASH_SRC_DMA_BUF_LEN; + else if (rctx->flags & SHA_FLAGS_FINUP) { + if (round_up(length, rctx->block_size) + rctx->block_size > + ASPEED_CRYPTO_SRC_DMA_BUF_LEN) + length = round_down(length - 1, rctx->block_size); + else + final = true; + } else + length -= remain; + memcpy_from_sglist(hash_engine->ahash_src_addr, rctx->src_sg, rctx->offset, length); + aspeed_ahash_update_counter(rctx, length); + if (final) + length += aspeed_ahash_fill_padding( + hace_dev, rctx, hash_engine->ahash_src_addr + length); + + rctx->digest_dma_addr = dma_map_single(hace_dev->dev, rctx->digest, + SHA512_DIGEST_SIZE, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(hace_dev->dev, rctx->digest_dma_addr)) { + dev_warn(hace_dev->dev, "dma_map() rctx digest error\n"); + return -ENOMEM; + } + + hash_engine->src_length = length; + hash_engine->src_dma = hash_engine->ahash_src_dma_addr; + hash_engine->digest_dma = rctx->digest_dma_addr; + + return 0; +} + +/* + * Prepare DMA buffer as SG list buffer before + * hardware engine processing. + */ +static int aspeed_ahash_dma_prepare_sg(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + bool final = rctx->flags & SHA_FLAGS_FINUP; + int remain, sg_len, i, max_sg_nents; + unsigned int length, offset, total; + struct aspeed_sg_list *src_list; + struct scatterlist *s; + int rc = 0; + + offset = rctx->offset; + length = rctx->total - offset; + remain = final ? 0 : length - round_down(length, rctx->block_size); + length -= remain; + + AHASH_DBG(hace_dev, "%s:0x%x, %s:0x%x, %s:0x%x\n", + "rctx total", rctx->total, + "length", length, "remain", remain); + + sg_len = dma_map_sg(hace_dev->dev, rctx->src_sg, rctx->src_nents, + DMA_TO_DEVICE); + if (!sg_len) { + dev_warn(hace_dev->dev, "dma_map_sg() src error\n"); + rc = -ENOMEM; + goto end; + } + + max_sg_nents = ASPEED_HASH_SRC_DMA_BUF_LEN / sizeof(*src_list) - final; + sg_len = min(sg_len, max_sg_nents); + src_list = (struct aspeed_sg_list *)hash_engine->ahash_src_addr; + rctx->digest_dma_addr = dma_map_single(hace_dev->dev, rctx->digest, + SHA512_DIGEST_SIZE, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(hace_dev->dev, rctx->digest_dma_addr)) { + dev_warn(hace_dev->dev, "dma_map() rctx digest error\n"); + rc = -ENOMEM; + goto free_src_sg; + } + + total = 0; + for_each_sg(rctx->src_sg, s, sg_len, i) { + u32 phy_addr = sg_dma_address(s); + u32 len = sg_dma_len(s); + + if (len <= offset) { + offset -= len; + continue; + } + + len -= offset; + phy_addr += offset; + offset = 0; + + if (length > len) + length -= len; + else { + /* Last sg list */ + len = length; + length = 0; + } + + total += len; + src_list[i].phy_addr = cpu_to_le32(phy_addr); + src_list[i].len = cpu_to_le32(len); + } + + if (length != 0) { + total = round_down(total, rctx->block_size); + final = false; + } + + aspeed_ahash_update_counter(rctx, total); + if (final) { + int len = aspeed_ahash_fill_padding(hace_dev, rctx, + rctx->buffer); + + total += len; + rctx->buffer_dma_addr = dma_map_single(hace_dev->dev, + rctx->buffer, + sizeof(rctx->buffer), + DMA_TO_DEVICE); + if (dma_mapping_error(hace_dev->dev, rctx->buffer_dma_addr)) { + dev_warn(hace_dev->dev, "dma_map() rctx buffer error\n"); + rc = -ENOMEM; + goto free_rctx_digest; + } + + src_list[i].phy_addr = cpu_to_le32(rctx->buffer_dma_addr); + src_list[i].len = cpu_to_le32(len); + i++; + } + src_list[i - 1].len |= cpu_to_le32(HASH_SG_LAST_LIST); + + hash_engine->src_length = total; + hash_engine->src_dma = hash_engine->ahash_src_dma_addr; + hash_engine->digest_dma = rctx->digest_dma_addr; + + return 0; + +free_rctx_digest: + dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr, + SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL); +free_src_sg: + dma_unmap_sg(hace_dev->dev, rctx->src_sg, rctx->src_nents, + DMA_TO_DEVICE); +end: + return rc; +} + +static int aspeed_ahash_complete(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + + AHASH_DBG(hace_dev, "\n"); + + dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr, + SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL); + + if (rctx->total - rctx->offset >= rctx->block_size || + (rctx->total != rctx->offset && rctx->flags & SHA_FLAGS_FINUP)) + return aspeed_ahash_req_update(hace_dev); + + hash_engine->flags &= ~CRYPTO_FLAGS_BUSY; + + if (rctx->flags & SHA_FLAGS_FINUP) + memcpy(req->result, rctx->digest, rctx->digsize); + + crypto_finalize_hash_request(hace_dev->crypt_engine_hash, req, + rctx->total - rctx->offset); + + return 0; +} + +/* + * Trigger hardware engines to do the math. + */ +static int aspeed_hace_ahash_trigger(struct aspeed_hace_dev *hace_dev, + aspeed_hace_fn_t resume) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + + AHASH_DBG(hace_dev, "src_dma:%pad, digest_dma:%pad, length:%zu\n", + &hash_engine->src_dma, &hash_engine->digest_dma, + hash_engine->src_length); + + rctx->cmd |= HASH_CMD_INT_ENABLE; + hash_engine->resume = resume; + + ast_hace_write(hace_dev, hash_engine->src_dma, ASPEED_HACE_HASH_SRC); + ast_hace_write(hace_dev, hash_engine->digest_dma, + ASPEED_HACE_HASH_DIGEST_BUFF); + ast_hace_write(hace_dev, hash_engine->digest_dma, + ASPEED_HACE_HASH_KEY_BUFF); + ast_hace_write(hace_dev, hash_engine->src_length, + ASPEED_HACE_HASH_DATA_LEN); + + /* Memory barrier to ensure all data setup before engine starts */ + mb(); + + ast_hace_write(hace_dev, rctx->cmd, ASPEED_HACE_HASH_CMD); + + return -EINPROGRESS; +} + +static int aspeed_ahash_update_resume_sg(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + + AHASH_DBG(hace_dev, "\n"); + + dma_unmap_sg(hace_dev->dev, rctx->src_sg, rctx->src_nents, + DMA_TO_DEVICE); + + if (rctx->flags & SHA_FLAGS_FINUP && rctx->total == rctx->offset) + dma_unmap_single(hace_dev->dev, rctx->buffer_dma_addr, + sizeof(rctx->buffer), DMA_TO_DEVICE); + + rctx->cmd &= ~HASH_CMD_HASH_SRC_SG_CTRL; + + return aspeed_ahash_complete(hace_dev); +} + +static int aspeed_ahash_req_update(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + aspeed_hace_fn_t resume; + int ret; + + AHASH_DBG(hace_dev, "\n"); + + if (hace_dev->version == AST2600_VERSION) { + rctx->cmd |= HASH_CMD_HASH_SRC_SG_CTRL; + resume = aspeed_ahash_update_resume_sg; + + } else { + resume = aspeed_ahash_complete; + } + + ret = hash_engine->dma_prepare(hace_dev); + if (ret) + return ret; + + return aspeed_hace_ahash_trigger(hace_dev, resume); +} + +static int aspeed_hace_hash_handle_queue(struct aspeed_hace_dev *hace_dev, + struct ahash_request *req) +{ + return crypto_transfer_hash_request_to_engine( + hace_dev->crypt_engine_hash, req); +} + +static noinline int aspeed_ahash_fallback(struct ahash_request *req) +{ + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + HASH_FBREQ_ON_STACK(fbreq, req); + u8 *state = rctx->buffer; + struct scatterlist sg[2]; + struct scatterlist *ssg; + int ret; + + ssg = scatterwalk_ffwd(sg, req->src, rctx->offset); + ahash_request_set_crypt(fbreq, ssg, req->result, + rctx->total - rctx->offset); + + ret = aspeed_sham_export(req, state) ?: + crypto_ahash_import_core(fbreq, state); + + if (rctx->flags & SHA_FLAGS_FINUP) + ret = ret ?: crypto_ahash_finup(fbreq); + else + ret = ret ?: crypto_ahash_update(fbreq) ?: + crypto_ahash_export_core(fbreq, state) ?: + aspeed_sham_import(req, state); + HASH_REQUEST_ZERO(fbreq); + return ret; +} + +static int aspeed_ahash_do_request(struct crypto_engine *engine, void *areq) +{ + struct ahash_request *req = ahash_request_cast(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct aspeed_hace_dev *hace_dev = tctx->hace_dev; + struct aspeed_engine_hash *hash_engine; + int ret; + + hash_engine = &hace_dev->hash_engine; + hash_engine->flags |= CRYPTO_FLAGS_BUSY; + + ret = aspeed_ahash_req_update(hace_dev); + if (ret != -EINPROGRESS) + return aspeed_ahash_fallback(req); + + return 0; +} + +static void aspeed_ahash_prepare_request(struct crypto_engine *engine, + void *areq) +{ + struct ahash_request *req = ahash_request_cast(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct aspeed_hace_dev *hace_dev = tctx->hace_dev; + struct aspeed_engine_hash *hash_engine; + + hash_engine = &hace_dev->hash_engine; + hash_engine->req = req; + + if (hace_dev->version == AST2600_VERSION) + hash_engine->dma_prepare = aspeed_ahash_dma_prepare_sg; + else + hash_engine->dma_prepare = aspeed_ahash_dma_prepare; +} + +static int aspeed_ahash_do_one(struct crypto_engine *engine, void *areq) +{ + aspeed_ahash_prepare_request(engine, areq); + return aspeed_ahash_do_request(engine, areq); +} + +static int aspeed_sham_update(struct ahash_request *req) +{ + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct aspeed_hace_dev *hace_dev = tctx->hace_dev; + + AHASH_DBG(hace_dev, "req->nbytes: %d\n", req->nbytes); + + rctx->total = req->nbytes; + rctx->src_sg = req->src; + rctx->offset = 0; + rctx->src_nents = sg_nents_for_len(req->src, req->nbytes); + + return aspeed_hace_hash_handle_queue(hace_dev, req); +} + +static int aspeed_sham_finup(struct ahash_request *req) +{ + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct aspeed_hace_dev *hace_dev = tctx->hace_dev; + + AHASH_DBG(hace_dev, "req->nbytes: %d\n", req->nbytes); + + rctx->flags |= SHA_FLAGS_FINUP; + + return aspeed_sham_update(req); +} + +static int aspeed_sham_init(struct ahash_request *req) +{ + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct aspeed_hace_dev *hace_dev = tctx->hace_dev; + + AHASH_DBG(hace_dev, "%s: digest size:%d\n", + crypto_tfm_alg_name(&tfm->base), + crypto_ahash_digestsize(tfm)); + + rctx->cmd = HASH_CMD_ACC_MODE; + rctx->flags = 0; + + switch (crypto_ahash_digestsize(tfm)) { + case SHA1_DIGEST_SIZE: + rctx->cmd |= HASH_CMD_SHA1 | HASH_CMD_SHA_SWAP; + rctx->flags |= SHA_FLAGS_SHA1; + rctx->digsize = SHA1_DIGEST_SIZE; + rctx->block_size = SHA1_BLOCK_SIZE; + rctx->ivsize = 32; + memcpy(rctx->digest, sha1_iv, rctx->ivsize); + break; + case SHA224_DIGEST_SIZE: + rctx->cmd |= HASH_CMD_SHA224 | HASH_CMD_SHA_SWAP; + rctx->flags |= SHA_FLAGS_SHA224; + rctx->digsize = SHA224_DIGEST_SIZE; + rctx->block_size = SHA224_BLOCK_SIZE; + rctx->ivsize = 32; + memcpy(rctx->digest, sha224_iv, rctx->ivsize); + break; + case SHA256_DIGEST_SIZE: + rctx->cmd |= HASH_CMD_SHA256 | HASH_CMD_SHA_SWAP; + rctx->flags |= SHA_FLAGS_SHA256; + rctx->digsize = SHA256_DIGEST_SIZE; + rctx->block_size = SHA256_BLOCK_SIZE; + rctx->ivsize = 32; + memcpy(rctx->digest, sha256_iv, rctx->ivsize); + break; + case SHA384_DIGEST_SIZE: + rctx->cmd |= HASH_CMD_SHA512_SER | HASH_CMD_SHA384 | + HASH_CMD_SHA_SWAP; + rctx->flags |= SHA_FLAGS_SHA384; + rctx->digsize = SHA384_DIGEST_SIZE; + rctx->block_size = SHA384_BLOCK_SIZE; + rctx->ivsize = 64; + memcpy(rctx->digest, sha384_iv, rctx->ivsize); + break; + case SHA512_DIGEST_SIZE: + rctx->cmd |= HASH_CMD_SHA512_SER | HASH_CMD_SHA512 | + HASH_CMD_SHA_SWAP; + rctx->flags |= SHA_FLAGS_SHA512; + rctx->digsize = SHA512_DIGEST_SIZE; + rctx->block_size = SHA512_BLOCK_SIZE; + rctx->ivsize = 64; + memcpy(rctx->digest, sha512_iv, rctx->ivsize); + break; + default: + dev_warn(tctx->hace_dev->dev, "digest size %d not support\n", + crypto_ahash_digestsize(tfm)); + return -EINVAL; + } + + rctx->total = 0; + rctx->digcnt[0] = 0; + rctx->digcnt[1] = 0; + + return 0; +} + +static int aspeed_sham_digest(struct ahash_request *req) +{ + return aspeed_sham_init(req) ? : aspeed_sham_finup(req); +} + +static int aspeed_sham_cra_init(struct crypto_ahash *tfm) +{ + struct ahash_alg *alg = crypto_ahash_alg(tfm); + struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct aspeed_hace_alg *ast_alg; + + ast_alg = container_of(alg, struct aspeed_hace_alg, alg.ahash.base); + tctx->hace_dev = ast_alg->hace_dev; + + return 0; +} + +static struct aspeed_hace_alg aspeed_ahash_algs[] = { + { + .alg.ahash.base = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .init_tfm = aspeed_sham_cra_init, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "sha1", + .cra_driver_name = "aspeed-sha1", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx), + .cra_reqsize = sizeof(struct aspeed_sham_reqctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + } + } + }, + .alg.ahash.op = { + .do_one_request = aspeed_ahash_do_one, + }, + }, + { + .alg.ahash.base = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .init_tfm = aspeed_sham_cra_init, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "sha256", + .cra_driver_name = "aspeed-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx), + .cra_reqsize = sizeof(struct aspeed_sham_reqctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + } + } + }, + .alg.ahash.op = { + .do_one_request = aspeed_ahash_do_one, + }, + }, + { + .alg.ahash.base = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .init_tfm = aspeed_sham_cra_init, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "sha224", + .cra_driver_name = "aspeed-sha224", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx), + .cra_reqsize = sizeof(struct aspeed_sham_reqctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + } + } + }, + .alg.ahash.op = { + .do_one_request = aspeed_ahash_do_one, + }, + }, +}; + +static struct aspeed_hace_alg aspeed_ahash_algs_g6[] = { + { + .alg.ahash.base = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .init_tfm = aspeed_sham_cra_init, + .halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "sha384", + .cra_driver_name = "aspeed-sha384", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx), + .cra_reqsize = sizeof(struct aspeed_sham_reqctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + } + } + }, + .alg.ahash.op = { + .do_one_request = aspeed_ahash_do_one, + }, + }, + { + .alg.ahash.base = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .init_tfm = aspeed_sham_cra_init, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "sha512", + .cra_driver_name = "aspeed-sha512", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx), + .cra_reqsize = sizeof(struct aspeed_sham_reqctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + } + } + }, + .alg.ahash.op = { + .do_one_request = aspeed_ahash_do_one, + }, + }, +}; + +void aspeed_unregister_hace_hash_algs(struct aspeed_hace_dev *hace_dev) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(aspeed_ahash_algs); i++) + crypto_engine_unregister_ahash(&aspeed_ahash_algs[i].alg.ahash); + + if (hace_dev->version != AST2600_VERSION) + return; + + for (i = 0; i < ARRAY_SIZE(aspeed_ahash_algs_g6); i++) + crypto_engine_unregister_ahash(&aspeed_ahash_algs_g6[i].alg.ahash); +} + +void aspeed_register_hace_hash_algs(struct aspeed_hace_dev *hace_dev) +{ + int rc, i; + + AHASH_DBG(hace_dev, "\n"); + + for (i = 0; i < ARRAY_SIZE(aspeed_ahash_algs); i++) { + aspeed_ahash_algs[i].hace_dev = hace_dev; + rc = crypto_engine_register_ahash(&aspeed_ahash_algs[i].alg.ahash); + if (rc) { + AHASH_DBG(hace_dev, "Failed to register %s\n", + aspeed_ahash_algs[i].alg.ahash.base.halg.base.cra_name); + } + } + + if (hace_dev->version != AST2600_VERSION) + return; + + for (i = 0; i < ARRAY_SIZE(aspeed_ahash_algs_g6); i++) { + aspeed_ahash_algs_g6[i].hace_dev = hace_dev; + rc = crypto_engine_register_ahash(&aspeed_ahash_algs_g6[i].alg.ahash); + if (rc) { + AHASH_DBG(hace_dev, "Failed to register %s\n", + aspeed_ahash_algs_g6[i].alg.ahash.base.halg.base.cra_name); + } + } +} diff --git a/drivers/crypto/aspeed/aspeed-hace.c b/drivers/crypto/aspeed/aspeed-hace.c new file mode 100644 index 000000000..6555d7b2a --- /dev/null +++ b/drivers/crypto/aspeed/aspeed-hace.c @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (c) 2021 Aspeed Technology Inc. + */ + +#include "aspeed-hace.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_CRYPTO_DEV_ASPEED_DEBUG +#define HACE_DBG(d, fmt, ...) \ + dev_info((d)->dev, "%s() " fmt, __func__, ##__VA_ARGS__) +#else +#define HACE_DBG(d, fmt, ...) \ + dev_dbg((d)->dev, "%s() " fmt, __func__, ##__VA_ARGS__) +#endif + +/* HACE interrupt service routine */ +static irqreturn_t aspeed_hace_irq(int irq, void *dev) +{ + struct aspeed_hace_dev *hace_dev = (struct aspeed_hace_dev *)dev; + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + u32 sts; + + sts = ast_hace_read(hace_dev, ASPEED_HACE_STS); + ast_hace_write(hace_dev, sts, ASPEED_HACE_STS); + + HACE_DBG(hace_dev, "irq status: 0x%x\n", sts); + + if (sts & HACE_HASH_ISR) { + if (hash_engine->flags & CRYPTO_FLAGS_BUSY) + tasklet_schedule(&hash_engine->done_task); + else + dev_warn(hace_dev->dev, "HASH no active requests.\n"); + } + + if (sts & HACE_CRYPTO_ISR) { + if (crypto_engine->flags & CRYPTO_FLAGS_BUSY) + tasklet_schedule(&crypto_engine->done_task); + else + dev_warn(hace_dev->dev, "CRYPTO no active requests.\n"); + } + + return IRQ_HANDLED; +} + +static void aspeed_hace_crypto_done_task(unsigned long data) +{ + struct aspeed_hace_dev *hace_dev = (struct aspeed_hace_dev *)data; + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + + crypto_engine->resume(hace_dev); +} + +static void aspeed_hace_hash_done_task(unsigned long data) +{ + struct aspeed_hace_dev *hace_dev = (struct aspeed_hace_dev *)data; + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + + hash_engine->resume(hace_dev); +} + +static void aspeed_hace_register(struct aspeed_hace_dev *hace_dev) +{ +#ifdef CONFIG_CRYPTO_DEV_ASPEED_HACE_HASH + aspeed_register_hace_hash_algs(hace_dev); +#endif +#ifdef CONFIG_CRYPTO_DEV_ASPEED_HACE_CRYPTO + aspeed_register_hace_crypto_algs(hace_dev); +#endif +} + +static void aspeed_hace_unregister(struct aspeed_hace_dev *hace_dev) +{ +#ifdef CONFIG_CRYPTO_DEV_ASPEED_HACE_HASH + aspeed_unregister_hace_hash_algs(hace_dev); +#endif +#ifdef CONFIG_CRYPTO_DEV_ASPEED_HACE_CRYPTO + aspeed_unregister_hace_crypto_algs(hace_dev); +#endif +} + +static const struct of_device_id aspeed_hace_of_matches[] = { + { .compatible = "aspeed,ast2500-hace", .data = (void *)5, }, + { .compatible = "aspeed,ast2600-hace", .data = (void *)6, }, + {}, +}; + +static int aspeed_hace_probe(struct platform_device *pdev) +{ + struct aspeed_engine_crypto *crypto_engine; + struct aspeed_engine_hash *hash_engine; + struct aspeed_hace_dev *hace_dev; + int rc; + + hace_dev = devm_kzalloc(&pdev->dev, sizeof(struct aspeed_hace_dev), + GFP_KERNEL); + if (!hace_dev) + return -ENOMEM; + + hace_dev->version = (uintptr_t)device_get_match_data(&pdev->dev); + if (!hace_dev->version) { + dev_err(&pdev->dev, "Failed to match hace dev id\n"); + return -EINVAL; + } + + hace_dev->dev = &pdev->dev; + hash_engine = &hace_dev->hash_engine; + crypto_engine = &hace_dev->crypto_engine; + + platform_set_drvdata(pdev, hace_dev); + + hace_dev->regs = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); + if (IS_ERR(hace_dev->regs)) + return PTR_ERR(hace_dev->regs); + + /* Get irq number and register it */ + hace_dev->irq = platform_get_irq(pdev, 0); + if (hace_dev->irq < 0) + return hace_dev->irq; + + rc = devm_request_irq(&pdev->dev, hace_dev->irq, aspeed_hace_irq, 0, + dev_name(&pdev->dev), hace_dev); + if (rc) + return rc; + + /* Get clk and enable it */ + hace_dev->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(hace_dev->clk)) { + dev_err(&pdev->dev, "Failed to get clk\n"); + return -ENODEV; + } + + rc = clk_prepare_enable(hace_dev->clk); + if (rc) { + dev_err(&pdev->dev, "Failed to enable clock 0x%x\n", rc); + return rc; + } + + /* Initialize crypto hardware engine structure for hash */ + hace_dev->crypt_engine_hash = crypto_engine_alloc_init(hace_dev->dev, + true); + if (!hace_dev->crypt_engine_hash) { + rc = -ENOMEM; + goto clk_exit; + } + + rc = crypto_engine_start(hace_dev->crypt_engine_hash); + if (rc) + goto err_engine_hash_start; + + tasklet_init(&hash_engine->done_task, aspeed_hace_hash_done_task, + (unsigned long)hace_dev); + + /* Initialize crypto hardware engine structure for crypto */ + hace_dev->crypt_engine_crypto = crypto_engine_alloc_init(hace_dev->dev, + true); + if (!hace_dev->crypt_engine_crypto) { + rc = -ENOMEM; + goto err_engine_hash_start; + } + + rc = crypto_engine_start(hace_dev->crypt_engine_crypto); + if (rc) + goto err_engine_crypto_start; + + tasklet_init(&crypto_engine->done_task, aspeed_hace_crypto_done_task, + (unsigned long)hace_dev); + + /* Allocate DMA buffer for hash engine input used */ + hash_engine->ahash_src_addr = + dmam_alloc_coherent(&pdev->dev, + ASPEED_HASH_SRC_DMA_BUF_LEN, + &hash_engine->ahash_src_dma_addr, + GFP_KERNEL); + if (!hash_engine->ahash_src_addr) { + dev_err(&pdev->dev, "Failed to allocate dma buffer\n"); + rc = -ENOMEM; + goto err_engine_crypto_start; + } + + /* Allocate DMA buffer for crypto engine context used */ + crypto_engine->cipher_ctx = + dmam_alloc_coherent(&pdev->dev, + PAGE_SIZE, + &crypto_engine->cipher_ctx_dma, + GFP_KERNEL); + if (!crypto_engine->cipher_ctx) { + dev_err(&pdev->dev, "Failed to allocate cipher ctx dma\n"); + rc = -ENOMEM; + goto err_engine_crypto_start; + } + + /* Allocate DMA buffer for crypto engine input used */ + crypto_engine->cipher_addr = + dmam_alloc_coherent(&pdev->dev, + ASPEED_CRYPTO_SRC_DMA_BUF_LEN, + &crypto_engine->cipher_dma_addr, + GFP_KERNEL); + if (!crypto_engine->cipher_addr) { + dev_err(&pdev->dev, "Failed to allocate cipher addr dma\n"); + rc = -ENOMEM; + goto err_engine_crypto_start; + } + + /* Allocate DMA buffer for crypto engine output used */ + if (hace_dev->version == AST2600_VERSION) { + crypto_engine->dst_sg_addr = + dmam_alloc_coherent(&pdev->dev, + ASPEED_CRYPTO_DST_DMA_BUF_LEN, + &crypto_engine->dst_sg_dma_addr, + GFP_KERNEL); + if (!crypto_engine->dst_sg_addr) { + dev_err(&pdev->dev, "Failed to allocate dst_sg dma\n"); + rc = -ENOMEM; + goto err_engine_crypto_start; + } + } + + aspeed_hace_register(hace_dev); + + dev_info(&pdev->dev, "Aspeed Crypto Accelerator successfully registered\n"); + + return 0; + +err_engine_crypto_start: + crypto_engine_exit(hace_dev->crypt_engine_crypto); +err_engine_hash_start: + crypto_engine_exit(hace_dev->crypt_engine_hash); +clk_exit: + clk_disable_unprepare(hace_dev->clk); + + return rc; +} + +static void aspeed_hace_remove(struct platform_device *pdev) +{ + struct aspeed_hace_dev *hace_dev = platform_get_drvdata(pdev); + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + + aspeed_hace_unregister(hace_dev); + + crypto_engine_exit(hace_dev->crypt_engine_hash); + crypto_engine_exit(hace_dev->crypt_engine_crypto); + + tasklet_kill(&hash_engine->done_task); + tasklet_kill(&crypto_engine->done_task); + + clk_disable_unprepare(hace_dev->clk); +} + +MODULE_DEVICE_TABLE(of, aspeed_hace_of_matches); + +static struct platform_driver aspeed_hace_driver = { + .probe = aspeed_hace_probe, + .remove = aspeed_hace_remove, + .driver = { + .name = KBUILD_MODNAME, + .of_match_table = aspeed_hace_of_matches, + }, +}; + +module_platform_driver(aspeed_hace_driver); + +MODULE_AUTHOR("Neal Liu "); +MODULE_DESCRIPTION("Aspeed HACE driver Crypto Accelerator"); +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/aspeed/aspeed-hace.h b/drivers/crypto/aspeed/aspeed-hace.h new file mode 100644 index 000000000..b1d07730d --- /dev/null +++ b/drivers/crypto/aspeed/aspeed-hace.h @@ -0,0 +1,274 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +#ifndef __ASPEED_HACE_H__ +#define __ASPEED_HACE_H__ + +#include +#include +#include +#include +#include +#include +#include +#include + +/***************************** + * * + * HACE register definitions * + * * + * ***************************/ +#define ASPEED_HACE_SRC 0x00 /* Crypto Data Source Base Address Register */ +#define ASPEED_HACE_DEST 0x04 /* Crypto Data Destination Base Address Register */ +#define ASPEED_HACE_CONTEXT 0x08 /* Crypto Context Buffer Base Address Register */ +#define ASPEED_HACE_DATA_LEN 0x0C /* Crypto Data Length Register */ +#define ASPEED_HACE_CMD 0x10 /* Crypto Engine Command Register */ + +/* G5 */ +#define ASPEED_HACE_TAG 0x18 /* HACE Tag Register */ +/* G6 */ +#define ASPEED_HACE_GCM_ADD_LEN 0x14 /* Crypto AES-GCM Additional Data Length Register */ +#define ASPEED_HACE_GCM_TAG_BASE_ADDR 0x18 /* Crypto AES-GCM Tag Write Buff Base Address Reg */ + +#define ASPEED_HACE_STS 0x1C /* HACE Status Register */ + +#define ASPEED_HACE_HASH_SRC 0x20 /* Hash Data Source Base Address Register */ +#define ASPEED_HACE_HASH_DIGEST_BUFF 0x24 /* Hash Digest Write Buffer Base Address Register */ +#define ASPEED_HACE_HASH_KEY_BUFF 0x28 /* Hash HMAC Key Buffer Base Address Register */ +#define ASPEED_HACE_HASH_DATA_LEN 0x2C /* Hash Data Length Register */ +#define ASPEED_HACE_HASH_CMD 0x30 /* Hash Engine Command Register */ + +/* crypto cmd */ +#define HACE_CMD_SINGLE_DES 0 +#define HACE_CMD_TRIPLE_DES BIT(17) +#define HACE_CMD_AES_SELECT 0 +#define HACE_CMD_DES_SELECT BIT(16) +#define HACE_CMD_ISR_EN BIT(12) +#define HACE_CMD_CONTEXT_SAVE_ENABLE (0) +#define HACE_CMD_CONTEXT_SAVE_DISABLE BIT(9) +#define HACE_CMD_AES (0) +#define HACE_CMD_DES (0) +#define HACE_CMD_RC4 BIT(8) +#define HACE_CMD_DECRYPT (0) +#define HACE_CMD_ENCRYPT BIT(7) + +#define HACE_CMD_ECB (0x0 << 4) +#define HACE_CMD_CBC (0x1 << 4) +#define HACE_CMD_CFB (0x2 << 4) +#define HACE_CMD_OFB (0x3 << 4) +#define HACE_CMD_CTR (0x4 << 4) +#define HACE_CMD_OP_MODE_MASK (0x7 << 4) + +#define HACE_CMD_AES128 (0x0 << 2) +#define HACE_CMD_AES192 (0x1 << 2) +#define HACE_CMD_AES256 (0x2 << 2) +#define HACE_CMD_OP_CASCADE (0x3) +#define HACE_CMD_OP_INDEPENDENT (0x1) + +/* G5 */ +#define HACE_CMD_RI_WO_DATA_ENABLE (0) +#define HACE_CMD_RI_WO_DATA_DISABLE BIT(11) +#define HACE_CMD_CONTEXT_LOAD_ENABLE (0) +#define HACE_CMD_CONTEXT_LOAD_DISABLE BIT(10) +/* G6 */ +#define HACE_CMD_AES_KEY_FROM_OTP BIT(24) +#define HACE_CMD_GHASH_TAG_XOR_EN BIT(23) +#define HACE_CMD_GHASH_PAD_LEN_INV BIT(22) +#define HACE_CMD_GCM_TAG_ADDR_SEL BIT(21) +#define HACE_CMD_MBUS_REQ_SYNC_EN BIT(20) +#define HACE_CMD_DES_SG_CTRL BIT(19) +#define HACE_CMD_SRC_SG_CTRL BIT(18) +#define HACE_CMD_CTR_IV_AES_96 (0x1 << 14) +#define HACE_CMD_CTR_IV_DES_32 (0x1 << 14) +#define HACE_CMD_CTR_IV_AES_64 (0x2 << 14) +#define HACE_CMD_CTR_IV_AES_32 (0x3 << 14) +#define HACE_CMD_AES_KEY_HW_EXP BIT(13) +#define HACE_CMD_GCM (0x5 << 4) + +/* interrupt status reg */ +#define HACE_CRYPTO_ISR BIT(12) +#define HACE_HASH_ISR BIT(9) +#define HACE_HASH_BUSY BIT(0) + +/* hash cmd reg */ +#define HASH_CMD_MBUS_REQ_SYNC_EN BIT(20) +#define HASH_CMD_HASH_SRC_SG_CTRL BIT(18) +#define HASH_CMD_SHA512_224 (0x3 << 10) +#define HASH_CMD_SHA512_256 (0x2 << 10) +#define HASH_CMD_SHA384 (0x1 << 10) +#define HASH_CMD_SHA512 (0) +#define HASH_CMD_INT_ENABLE BIT(9) +#define HASH_CMD_HMAC (0x1 << 7) +#define HASH_CMD_ACC_MODE (0x2 << 7) +#define HASH_CMD_HMAC_KEY (0x3 << 7) +#define HASH_CMD_SHA1 (0x2 << 4) +#define HASH_CMD_SHA224 (0x4 << 4) +#define HASH_CMD_SHA256 (0x5 << 4) +#define HASH_CMD_SHA512_SER (0x6 << 4) +#define HASH_CMD_SHA_SWAP (0x2 << 2) + +#define HASH_SG_LAST_LIST BIT(31) + +#define CRYPTO_FLAGS_BUSY BIT(1) + +#define SHA_OP_UPDATE 1 +#define SHA_OP_FINAL 2 + +#define SHA_FLAGS_SHA1 BIT(0) +#define SHA_FLAGS_SHA224 BIT(1) +#define SHA_FLAGS_SHA256 BIT(2) +#define SHA_FLAGS_SHA384 BIT(3) +#define SHA_FLAGS_SHA512 BIT(4) +#define SHA_FLAGS_SHA512_224 BIT(5) +#define SHA_FLAGS_SHA512_256 BIT(6) +#define SHA_FLAGS_FINUP BIT(9) +#define SHA_FLAGS_MASK (0xff) + +#define ASPEED_CRYPTO_SRC_DMA_BUF_LEN 0xa000 +#define ASPEED_CRYPTO_DST_DMA_BUF_LEN 0xa000 +#define ASPEED_CRYPTO_GCM_TAG_OFFSET 0x9ff0 +#define ASPEED_HASH_SRC_DMA_BUF_LEN 0xa000 +#define ASPEED_HASH_QUEUE_LENGTH 50 + +#define HACE_CMD_IV_REQUIRE (HACE_CMD_CBC | HACE_CMD_CFB | \ + HACE_CMD_OFB | HACE_CMD_CTR) + +struct aspeed_hace_dev; +struct scatterlist; + +typedef int (*aspeed_hace_fn_t)(struct aspeed_hace_dev *); + +struct aspeed_sg_list { + __le32 len; + __le32 phy_addr; +}; + +struct aspeed_engine_hash { + struct tasklet_struct done_task; + unsigned long flags; + struct ahash_request *req; + + /* input buffer */ + void *ahash_src_addr; + dma_addr_t ahash_src_dma_addr; + + dma_addr_t src_dma; + dma_addr_t digest_dma; + + size_t src_length; + + /* callback func */ + aspeed_hace_fn_t resume; + aspeed_hace_fn_t dma_prepare; +}; + +struct aspeed_sham_ctx { + struct aspeed_hace_dev *hace_dev; +}; + +struct aspeed_sham_reqctx { + /* DMA buffer written by hardware */ + u8 digest[SHA512_DIGEST_SIZE] __aligned(64); + + /* Software state sorted by size. */ + u64 digcnt[2]; + + unsigned long flags; /* final update flag should no use*/ + u32 cmd; /* trigger cmd */ + + /* walk state */ + struct scatterlist *src_sg; + int src_nents; + unsigned int offset; /* offset in current sg */ + unsigned int total; /* per update length */ + + size_t digsize; + size_t block_size; + size_t ivsize; + + dma_addr_t buffer_dma_addr; + dma_addr_t digest_dma_addr; + + /* This is DMA too but read-only for hardware. */ + u8 buffer[SHA512_BLOCK_SIZE + 16]; +}; + +struct aspeed_engine_crypto { + struct tasklet_struct done_task; + unsigned long flags; + struct skcipher_request *req; + + /* context buffer */ + void *cipher_ctx; + dma_addr_t cipher_ctx_dma; + + /* input buffer, could be single/scatter-gather lists */ + void *cipher_addr; + dma_addr_t cipher_dma_addr; + + /* output buffer, only used in scatter-gather lists */ + void *dst_sg_addr; + dma_addr_t dst_sg_dma_addr; + + /* callback func */ + aspeed_hace_fn_t resume; +}; + +struct aspeed_cipher_ctx { + struct aspeed_hace_dev *hace_dev; + int key_len; + u8 key[AES_MAX_KEYLENGTH]; + + /* callback func */ + aspeed_hace_fn_t start; + + struct crypto_skcipher *fallback_tfm; +}; + +struct aspeed_cipher_reqctx { + int enc_cmd; + int src_nents; + int dst_nents; + + struct skcipher_request fallback_req; /* keep at the end */ +}; + +struct aspeed_hace_dev { + void __iomem *regs; + struct device *dev; + int irq; + struct clk *clk; + unsigned long version; + + struct crypto_engine *crypt_engine_hash; + struct crypto_engine *crypt_engine_crypto; + + struct aspeed_engine_hash hash_engine; + struct aspeed_engine_crypto crypto_engine; +}; + +struct aspeed_hace_alg { + struct aspeed_hace_dev *hace_dev; + + const char *alg_base; + + union { + struct skcipher_engine_alg skcipher; + struct ahash_engine_alg ahash; + } alg; +}; + +enum aspeed_version { + AST2500_VERSION = 5, + AST2600_VERSION +}; + +#define ast_hace_write(hace, val, offset) \ + writel((val), (hace)->regs + (offset)) +#define ast_hace_read(hace, offset) \ + readl((hace)->regs + (offset)) + +void aspeed_register_hace_hash_algs(struct aspeed_hace_dev *hace_dev); +void aspeed_unregister_hace_hash_algs(struct aspeed_hace_dev *hace_dev); +void aspeed_register_hace_crypto_algs(struct aspeed_hace_dev *hace_dev); +void aspeed_unregister_hace_crypto_algs(struct aspeed_hace_dev *hace_dev); + +#endif diff --git a/drivers/crypto/atmel-aes-regs.h b/drivers/crypto/atmel-aes-regs.h new file mode 100644 index 000000000..62f9d3038 --- /dev/null +++ b/drivers/crypto/atmel-aes-regs.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ATMEL_AES_REGS_H__ +#define __ATMEL_AES_REGS_H__ + +#define AES_CR 0x00 +#define AES_CR_START (1 << 0) +#define AES_CR_SWRST (1 << 8) +#define AES_CR_LOADSEED (1 << 16) + +#define AES_MR 0x04 +#define AES_MR_CYPHER_DEC (0 << 0) +#define AES_MR_CYPHER_ENC (1 << 0) +#define AES_MR_GTAGEN (1 << 1) +#define AES_MR_DUALBUFF (1 << 3) +#define AES_MR_PROCDLY_MASK (0xF << 4) +#define AES_MR_PROCDLY_OFFSET 4 +#define AES_MR_SMOD_MASK (0x3 << 8) +#define AES_MR_SMOD_MANUAL (0x0 << 8) +#define AES_MR_SMOD_AUTO (0x1 << 8) +#define AES_MR_SMOD_IDATAR0 (0x2 << 8) +#define AES_MR_KEYSIZE_MASK (0x3 << 10) +#define AES_MR_KEYSIZE_128 (0x0 << 10) +#define AES_MR_KEYSIZE_192 (0x1 << 10) +#define AES_MR_KEYSIZE_256 (0x2 << 10) +#define AES_MR_OPMOD_MASK (0x7 << 12) +#define AES_MR_OPMOD_ECB (0x0 << 12) +#define AES_MR_OPMOD_CBC (0x1 << 12) +#define AES_MR_OPMOD_OFB (0x2 << 12) +#define AES_MR_OPMOD_CFB (0x3 << 12) +#define AES_MR_OPMOD_CTR (0x4 << 12) +#define AES_MR_OPMOD_GCM (0x5 << 12) +#define AES_MR_OPMOD_XTS (0x6 << 12) +#define AES_MR_LOD (0x1 << 15) +#define AES_MR_CFBS_MASK (0x7 << 16) +#define AES_MR_CFBS_128b (0x0 << 16) +#define AES_MR_CFBS_64b (0x1 << 16) +#define AES_MR_CFBS_32b (0x2 << 16) +#define AES_MR_CFBS_16b (0x3 << 16) +#define AES_MR_CFBS_8b (0x4 << 16) +#define AES_MR_CKEY_MASK (0xF << 20) +#define AES_MR_CKEY_OFFSET 20 +#define AES_MR_CMTYP_MASK (0x1F << 24) +#define AES_MR_CMTYP_OFFSET 24 + +#define AES_IER 0x10 +#define AES_IDR 0x14 +#define AES_IMR 0x18 +#define AES_ISR 0x1C +#define AES_INT_DATARDY (1 << 0) +#define AES_INT_URAD (1 << 8) +#define AES_INT_TAGRDY (1 << 16) +#define AES_ISR_URAT_MASK (0xF << 12) +#define AES_ISR_URAT_IDR_WR_PROC (0x0 << 12) +#define AES_ISR_URAT_ODR_RD_PROC (0x1 << 12) +#define AES_ISR_URAT_MR_WR_PROC (0x2 << 12) +#define AES_ISR_URAT_ODR_RD_SUBK (0x3 << 12) +#define AES_ISR_URAT_MR_WR_SUBK (0x4 << 12) +#define AES_ISR_URAT_WOR_RD (0x5 << 12) + +#define AES_KEYWR(x) (0x20 + ((x) * 0x04)) +#define AES_IDATAR(x) (0x40 + ((x) * 0x04)) +#define AES_ODATAR(x) (0x50 + ((x) * 0x04)) +#define AES_IVR(x) (0x60 + ((x) * 0x04)) + +#define AES_AADLENR 0x70 +#define AES_CLENR 0x74 +#define AES_GHASHR(x) (0x78 + ((x) * 0x04)) +#define AES_TAGR(x) (0x88 + ((x) * 0x04)) +#define AES_CTRR 0x98 +#define AES_GCMHR(x) (0x9c + ((x) * 0x04)) + +#define AES_EMR 0xb0 +#define AES_EMR_APEN BIT(0) /* Auto Padding Enable */ +#define AES_EMR_APM BIT(1) /* Auto Padding Mode */ +#define AES_EMR_APM_IPSEC 0x0 +#define AES_EMR_APM_SSL BIT(1) +#define AES_EMR_PLIPEN BIT(4) /* PLIP Enable */ +#define AES_EMR_PLIPD BIT(5) /* PLIP Decipher */ +#define AES_EMR_PADLEN_MASK (0xFu << 8) +#define AES_EMR_PADLEN_OFFSET 8 +#define AES_EMR_PADLEN(padlen) (((padlen) << AES_EMR_PADLEN_OFFSET) &\ + AES_EMR_PADLEN_MASK) +#define AES_EMR_NHEAD_MASK (0xFu << 16) +#define AES_EMR_NHEAD_OFFSET 16 +#define AES_EMR_NHEAD(nhead) (((nhead) << AES_EMR_NHEAD_OFFSET) &\ + AES_EMR_NHEAD_MASK) + +#define AES_TWR(x) (0xc0 + ((x) * 0x04)) +#define AES_ALPHAR(x) (0xd0 + ((x) * 0x04)) + +#define AES_HW_VERSION 0xFC + +#endif /* __ATMEL_AES_REGS_H__ */ diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c new file mode 100644 index 000000000..50d97cef7 --- /dev/null +++ b/drivers/crypto/atmel-aes.c @@ -0,0 +1,2459 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Cryptographic API. + * + * Support for ATMEL AES HW acceleration. + * + * Copyright (c) 2012 Eukréa Electromatique - ATMEL + * Author: Nicolas Royer + * + * Some ideas are from omap-aes.c driver. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "atmel-aes-regs.h" +#include "atmel-authenc.h" + +#define ATMEL_AES_PRIORITY 300 + +#define ATMEL_AES_BUFFER_ORDER 2 +#define ATMEL_AES_BUFFER_SIZE (PAGE_SIZE << ATMEL_AES_BUFFER_ORDER) + +#define SIZE_IN_WORDS(x) ((x) >> 2) + +/* AES flags */ +/* Reserve bits [18:16] [14:12] [1:0] for mode (same as for AES_MR) */ +#define AES_FLAGS_ENCRYPT AES_MR_CYPHER_ENC +#define AES_FLAGS_GTAGEN AES_MR_GTAGEN +#define AES_FLAGS_OPMODE_MASK (AES_MR_OPMOD_MASK | AES_MR_CFBS_MASK) +#define AES_FLAGS_ECB AES_MR_OPMOD_ECB +#define AES_FLAGS_CBC AES_MR_OPMOD_CBC +#define AES_FLAGS_CTR AES_MR_OPMOD_CTR +#define AES_FLAGS_GCM AES_MR_OPMOD_GCM +#define AES_FLAGS_XTS AES_MR_OPMOD_XTS + +#define AES_FLAGS_MODE_MASK (AES_FLAGS_OPMODE_MASK | \ + AES_FLAGS_ENCRYPT | \ + AES_FLAGS_GTAGEN) + +#define AES_FLAGS_BUSY BIT(3) +#define AES_FLAGS_DUMP_REG BIT(4) +#define AES_FLAGS_OWN_SHA BIT(5) + +#define AES_FLAGS_PERSISTENT AES_FLAGS_BUSY + +#define ATMEL_AES_QUEUE_LENGTH 50 + +#define ATMEL_AES_DMA_THRESHOLD 256 + + +struct atmel_aes_caps { + bool has_dualbuff; + bool has_gcm; + bool has_xts; + bool has_authenc; + u32 max_burst_size; +}; + +struct atmel_aes_dev; + + +typedef int (*atmel_aes_fn_t)(struct atmel_aes_dev *); + + +struct atmel_aes_base_ctx { + struct atmel_aes_dev *dd; + atmel_aes_fn_t start; + int keylen; + u32 key[AES_KEYSIZE_256 / sizeof(u32)]; + u16 block_size; + bool is_aead; +}; + +struct atmel_aes_ctx { + struct atmel_aes_base_ctx base; +}; + +struct atmel_aes_ctr_ctx { + struct atmel_aes_base_ctx base; + + __be32 iv[AES_BLOCK_SIZE / sizeof(u32)]; + size_t offset; + struct scatterlist src[2]; + struct scatterlist dst[2]; + u32 blocks; +}; + +struct atmel_aes_gcm_ctx { + struct atmel_aes_base_ctx base; + + struct scatterlist src[2]; + struct scatterlist dst[2]; + + __be32 j0[AES_BLOCK_SIZE / sizeof(u32)]; + u32 tag[AES_BLOCK_SIZE / sizeof(u32)]; + __be32 ghash[AES_BLOCK_SIZE / sizeof(u32)]; + size_t textlen; + + const __be32 *ghash_in; + __be32 *ghash_out; + atmel_aes_fn_t ghash_resume; +}; + +struct atmel_aes_xts_ctx { + struct atmel_aes_base_ctx base; + + u32 key2[AES_KEYSIZE_256 / sizeof(u32)]; + struct crypto_skcipher *fallback_tfm; +}; + +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) +struct atmel_aes_authenc_ctx { + struct atmel_aes_base_ctx base; + struct atmel_sha_authenc_ctx *auth; +}; +#endif + +struct atmel_aes_reqctx { + unsigned long mode; + u8 lastc[AES_BLOCK_SIZE]; + struct skcipher_request fallback_req; +}; + +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) +struct atmel_aes_authenc_reqctx { + struct atmel_aes_reqctx base; + + struct scatterlist src[2]; + struct scatterlist dst[2]; + size_t textlen; + u32 digest[SHA512_DIGEST_SIZE / sizeof(u32)]; + + /* auth_req MUST be place last. */ + struct ahash_request auth_req; +}; +#endif + +struct atmel_aes_dma { + struct dma_chan *chan; + struct scatterlist *sg; + int nents; + unsigned int remainder; + unsigned int sg_len; +}; + +struct atmel_aes_dev { + struct list_head list; + unsigned long phys_base; + void __iomem *io_base; + + struct crypto_async_request *areq; + struct atmel_aes_base_ctx *ctx; + + bool is_async; + atmel_aes_fn_t resume; + atmel_aes_fn_t cpu_transfer_complete; + + struct device *dev; + struct clk *iclk; + int irq; + + unsigned long flags; + + spinlock_t lock; + struct crypto_queue queue; + + struct tasklet_struct done_task; + struct tasklet_struct queue_task; + + size_t total; + size_t datalen; + u32 *data; + + struct atmel_aes_dma src; + struct atmel_aes_dma dst; + + size_t buflen; + void *buf; + struct scatterlist aligned_sg; + struct scatterlist *real_dst; + + struct atmel_aes_caps caps; + + u32 hw_version; +}; + +struct atmel_aes_drv { + struct list_head dev_list; + spinlock_t lock; +}; + +static struct atmel_aes_drv atmel_aes = { + .dev_list = LIST_HEAD_INIT(atmel_aes.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(atmel_aes.lock), +}; + +#ifdef VERBOSE_DEBUG +static const char *atmel_aes_reg_name(u32 offset, char *tmp, size_t sz) +{ + switch (offset) { + case AES_CR: + return "CR"; + + case AES_MR: + return "MR"; + + case AES_ISR: + return "ISR"; + + case AES_IMR: + return "IMR"; + + case AES_IER: + return "IER"; + + case AES_IDR: + return "IDR"; + + case AES_KEYWR(0): + case AES_KEYWR(1): + case AES_KEYWR(2): + case AES_KEYWR(3): + case AES_KEYWR(4): + case AES_KEYWR(5): + case AES_KEYWR(6): + case AES_KEYWR(7): + snprintf(tmp, sz, "KEYWR[%u]", (offset - AES_KEYWR(0)) >> 2); + break; + + case AES_IDATAR(0): + case AES_IDATAR(1): + case AES_IDATAR(2): + case AES_IDATAR(3): + snprintf(tmp, sz, "IDATAR[%u]", (offset - AES_IDATAR(0)) >> 2); + break; + + case AES_ODATAR(0): + case AES_ODATAR(1): + case AES_ODATAR(2): + case AES_ODATAR(3): + snprintf(tmp, sz, "ODATAR[%u]", (offset - AES_ODATAR(0)) >> 2); + break; + + case AES_IVR(0): + case AES_IVR(1): + case AES_IVR(2): + case AES_IVR(3): + snprintf(tmp, sz, "IVR[%u]", (offset - AES_IVR(0)) >> 2); + break; + + case AES_AADLENR: + return "AADLENR"; + + case AES_CLENR: + return "CLENR"; + + case AES_GHASHR(0): + case AES_GHASHR(1): + case AES_GHASHR(2): + case AES_GHASHR(3): + snprintf(tmp, sz, "GHASHR[%u]", (offset - AES_GHASHR(0)) >> 2); + break; + + case AES_TAGR(0): + case AES_TAGR(1): + case AES_TAGR(2): + case AES_TAGR(3): + snprintf(tmp, sz, "TAGR[%u]", (offset - AES_TAGR(0)) >> 2); + break; + + case AES_CTRR: + return "CTRR"; + + case AES_GCMHR(0): + case AES_GCMHR(1): + case AES_GCMHR(2): + case AES_GCMHR(3): + snprintf(tmp, sz, "GCMHR[%u]", (offset - AES_GCMHR(0)) >> 2); + break; + + case AES_EMR: + return "EMR"; + + case AES_TWR(0): + case AES_TWR(1): + case AES_TWR(2): + case AES_TWR(3): + snprintf(tmp, sz, "TWR[%u]", (offset - AES_TWR(0)) >> 2); + break; + + case AES_ALPHAR(0): + case AES_ALPHAR(1): + case AES_ALPHAR(2): + case AES_ALPHAR(3): + snprintf(tmp, sz, "ALPHAR[%u]", (offset - AES_ALPHAR(0)) >> 2); + break; + + default: + snprintf(tmp, sz, "0x%02x", offset); + break; + } + + return tmp; +} +#endif /* VERBOSE_DEBUG */ + +/* Shared functions */ + +static inline u32 atmel_aes_read(struct atmel_aes_dev *dd, u32 offset) +{ + u32 value = readl_relaxed(dd->io_base + offset); + +#ifdef VERBOSE_DEBUG + if (dd->flags & AES_FLAGS_DUMP_REG) { + char tmp[16]; + + dev_vdbg(dd->dev, "read 0x%08x from %s\n", value, + atmel_aes_reg_name(offset, tmp, sizeof(tmp))); + } +#endif /* VERBOSE_DEBUG */ + + return value; +} + +static inline void atmel_aes_write(struct atmel_aes_dev *dd, + u32 offset, u32 value) +{ +#ifdef VERBOSE_DEBUG + if (dd->flags & AES_FLAGS_DUMP_REG) { + char tmp[16]; + + dev_vdbg(dd->dev, "write 0x%08x into %s\n", value, + atmel_aes_reg_name(offset, tmp, sizeof(tmp))); + } +#endif /* VERBOSE_DEBUG */ + + writel_relaxed(value, dd->io_base + offset); +} + +static void atmel_aes_read_n(struct atmel_aes_dev *dd, u32 offset, + u32 *value, int count) +{ + for (; count--; value++, offset += 4) + *value = atmel_aes_read(dd, offset); +} + +static void atmel_aes_write_n(struct atmel_aes_dev *dd, u32 offset, + const u32 *value, int count) +{ + for (; count--; value++, offset += 4) + atmel_aes_write(dd, offset, *value); +} + +static inline void atmel_aes_read_block(struct atmel_aes_dev *dd, u32 offset, + void *value) +{ + atmel_aes_read_n(dd, offset, value, SIZE_IN_WORDS(AES_BLOCK_SIZE)); +} + +static inline void atmel_aes_write_block(struct atmel_aes_dev *dd, u32 offset, + const void *value) +{ + atmel_aes_write_n(dd, offset, value, SIZE_IN_WORDS(AES_BLOCK_SIZE)); +} + +static inline int atmel_aes_wait_for_data_ready(struct atmel_aes_dev *dd, + atmel_aes_fn_t resume) +{ + u32 isr = atmel_aes_read(dd, AES_ISR); + + if (unlikely(isr & AES_INT_DATARDY)) + return resume(dd); + + dd->resume = resume; + atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); + return -EINPROGRESS; +} + +static inline size_t atmel_aes_padlen(size_t len, size_t block_size) +{ + len &= block_size - 1; + return len ? block_size - len : 0; +} + +static struct atmel_aes_dev *atmel_aes_dev_alloc(struct atmel_aes_base_ctx *ctx) +{ + struct atmel_aes_dev *aes_dd; + + spin_lock_bh(&atmel_aes.lock); + /* One AES IP per SoC. */ + aes_dd = list_first_entry_or_null(&atmel_aes.dev_list, + struct atmel_aes_dev, list); + spin_unlock_bh(&atmel_aes.lock); + return aes_dd; +} + +static int atmel_aes_hw_init(struct atmel_aes_dev *dd) +{ + int err; + + err = clk_enable(dd->iclk); + if (err) + return err; + + atmel_aes_write(dd, AES_CR, AES_CR_SWRST); + atmel_aes_write(dd, AES_MR, 0xE << AES_MR_CKEY_OFFSET); + + return 0; +} + +static inline unsigned int atmel_aes_get_version(struct atmel_aes_dev *dd) +{ + return atmel_aes_read(dd, AES_HW_VERSION) & 0x00000fff; +} + +static int atmel_aes_hw_version_init(struct atmel_aes_dev *dd) +{ + int err; + + err = atmel_aes_hw_init(dd); + if (err) + return err; + + dd->hw_version = atmel_aes_get_version(dd); + + dev_info(dd->dev, "version: 0x%x\n", dd->hw_version); + + clk_disable(dd->iclk); + return 0; +} + +static inline void atmel_aes_set_mode(struct atmel_aes_dev *dd, + const struct atmel_aes_reqctx *rctx) +{ + /* Clear all but persistent flags and set request flags. */ + dd->flags = (dd->flags & AES_FLAGS_PERSISTENT) | rctx->mode; +} + +static inline bool atmel_aes_is_encrypt(const struct atmel_aes_dev *dd) +{ + return (dd->flags & AES_FLAGS_ENCRYPT); +} + +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) +static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err); +#endif + +static void atmel_aes_set_iv_as_last_ciphertext_block(struct atmel_aes_dev *dd) +{ + struct skcipher_request *req = skcipher_request_cast(dd->areq); + struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req); + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + unsigned int ivsize = crypto_skcipher_ivsize(skcipher); + + if (req->cryptlen < ivsize) + return; + + if (rctx->mode & AES_FLAGS_ENCRYPT) + scatterwalk_map_and_copy(req->iv, req->dst, + req->cryptlen - ivsize, ivsize, 0); + else + memcpy(req->iv, rctx->lastc, ivsize); +} + +static inline struct atmel_aes_ctr_ctx * +atmel_aes_ctr_ctx_cast(struct atmel_aes_base_ctx *ctx) +{ + return container_of(ctx, struct atmel_aes_ctr_ctx, base); +} + +static void atmel_aes_ctr_update_req_iv(struct atmel_aes_dev *dd) +{ + struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx); + struct skcipher_request *req = skcipher_request_cast(dd->areq); + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + unsigned int ivsize = crypto_skcipher_ivsize(skcipher); + int i; + + /* + * The CTR transfer works in fragments of data of maximum 1 MByte + * because of the 16 bit CTR counter embedded in the IP. When reaching + * here, ctx->blocks contains the number of blocks of the last fragment + * processed, there is no need to explicit cast it to u16. + */ + for (i = 0; i < ctx->blocks; i++) + crypto_inc((u8 *)ctx->iv, AES_BLOCK_SIZE); + + memcpy(req->iv, ctx->iv, ivsize); +} + +static inline int atmel_aes_complete(struct atmel_aes_dev *dd, int err) +{ + struct skcipher_request *req = skcipher_request_cast(dd->areq); + struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req); + +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) + if (dd->ctx->is_aead) + atmel_aes_authenc_complete(dd, err); +#endif + + clk_disable(dd->iclk); + dd->flags &= ~AES_FLAGS_BUSY; + + if (!err && !dd->ctx->is_aead && + (rctx->mode & AES_FLAGS_OPMODE_MASK) != AES_FLAGS_ECB) { + if ((rctx->mode & AES_FLAGS_OPMODE_MASK) != AES_FLAGS_CTR) + atmel_aes_set_iv_as_last_ciphertext_block(dd); + else + atmel_aes_ctr_update_req_iv(dd); + } + + if (dd->is_async) + crypto_request_complete(dd->areq, err); + + tasklet_schedule(&dd->queue_task); + + return err; +} + +static void atmel_aes_write_ctrl_key(struct atmel_aes_dev *dd, bool use_dma, + const __be32 *iv, const u32 *key, int keylen) +{ + u32 valmr = 0; + + /* MR register must be set before IV registers */ + if (keylen == AES_KEYSIZE_128) + valmr |= AES_MR_KEYSIZE_128; + else if (keylen == AES_KEYSIZE_192) + valmr |= AES_MR_KEYSIZE_192; + else + valmr |= AES_MR_KEYSIZE_256; + + valmr |= dd->flags & AES_FLAGS_MODE_MASK; + + if (use_dma) { + valmr |= AES_MR_SMOD_IDATAR0; + if (dd->caps.has_dualbuff) + valmr |= AES_MR_DUALBUFF; + } else { + valmr |= AES_MR_SMOD_AUTO; + } + + atmel_aes_write(dd, AES_MR, valmr); + + atmel_aes_write_n(dd, AES_KEYWR(0), key, SIZE_IN_WORDS(keylen)); + + if (iv && (valmr & AES_MR_OPMOD_MASK) != AES_MR_OPMOD_ECB) + atmel_aes_write_block(dd, AES_IVR(0), iv); +} + +static inline void atmel_aes_write_ctrl(struct atmel_aes_dev *dd, bool use_dma, + const __be32 *iv) + +{ + atmel_aes_write_ctrl_key(dd, use_dma, iv, + dd->ctx->key, dd->ctx->keylen); +} + +/* CPU transfer */ + +static int atmel_aes_cpu_transfer(struct atmel_aes_dev *dd) +{ + int err = 0; + u32 isr; + + for (;;) { + atmel_aes_read_block(dd, AES_ODATAR(0), dd->data); + dd->data += 4; + dd->datalen -= AES_BLOCK_SIZE; + + if (dd->datalen < AES_BLOCK_SIZE) + break; + + atmel_aes_write_block(dd, AES_IDATAR(0), dd->data); + + isr = atmel_aes_read(dd, AES_ISR); + if (!(isr & AES_INT_DATARDY)) { + dd->resume = atmel_aes_cpu_transfer; + atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); + return -EINPROGRESS; + } + } + + if (!sg_copy_from_buffer(dd->real_dst, sg_nents(dd->real_dst), + dd->buf, dd->total)) + err = -EINVAL; + + if (err) + return atmel_aes_complete(dd, err); + + return dd->cpu_transfer_complete(dd); +} + +static int atmel_aes_cpu_start(struct atmel_aes_dev *dd, + struct scatterlist *src, + struct scatterlist *dst, + size_t len, + atmel_aes_fn_t resume) +{ + size_t padlen = atmel_aes_padlen(len, AES_BLOCK_SIZE); + + if (unlikely(len == 0)) + return -EINVAL; + + sg_copy_to_buffer(src, sg_nents(src), dd->buf, len); + + dd->total = len; + dd->real_dst = dst; + dd->cpu_transfer_complete = resume; + dd->datalen = len + padlen; + dd->data = (u32 *)dd->buf; + atmel_aes_write_block(dd, AES_IDATAR(0), dd->data); + return atmel_aes_wait_for_data_ready(dd, atmel_aes_cpu_transfer); +} + + +/* DMA transfer */ + +static void atmel_aes_dma_callback(void *data); + +static bool atmel_aes_check_aligned(struct atmel_aes_dev *dd, + struct scatterlist *sg, + size_t len, + struct atmel_aes_dma *dma) +{ + int nents; + + if (!IS_ALIGNED(len, dd->ctx->block_size)) + return false; + + for (nents = 0; sg; sg = sg_next(sg), ++nents) { + if (!IS_ALIGNED(sg->offset, sizeof(u32))) + return false; + + if (len <= sg->length) { + if (!IS_ALIGNED(len, dd->ctx->block_size)) + return false; + + dma->nents = nents+1; + dma->remainder = sg->length - len; + sg->length = len; + return true; + } + + if (!IS_ALIGNED(sg->length, dd->ctx->block_size)) + return false; + + len -= sg->length; + } + + return false; +} + +static inline void atmel_aes_restore_sg(const struct atmel_aes_dma *dma) +{ + struct scatterlist *sg = dma->sg; + int nents = dma->nents; + + if (!dma->remainder) + return; + + while (--nents > 0 && sg) + sg = sg_next(sg); + + if (!sg) + return; + + sg->length += dma->remainder; +} + +static int atmel_aes_map(struct atmel_aes_dev *dd, + struct scatterlist *src, + struct scatterlist *dst, + size_t len) +{ + bool src_aligned, dst_aligned; + size_t padlen; + + dd->total = len; + dd->src.sg = src; + dd->dst.sg = dst; + dd->real_dst = dst; + + src_aligned = atmel_aes_check_aligned(dd, src, len, &dd->src); + if (src == dst) + dst_aligned = src_aligned; + else + dst_aligned = atmel_aes_check_aligned(dd, dst, len, &dd->dst); + if (!src_aligned || !dst_aligned) { + padlen = atmel_aes_padlen(len, dd->ctx->block_size); + + if (dd->buflen < len + padlen) + return -ENOMEM; + + if (!src_aligned) { + sg_copy_to_buffer(src, sg_nents(src), dd->buf, len); + dd->src.sg = &dd->aligned_sg; + dd->src.nents = 1; + dd->src.remainder = 0; + } + + if (!dst_aligned) { + dd->dst.sg = &dd->aligned_sg; + dd->dst.nents = 1; + dd->dst.remainder = 0; + } + + sg_init_table(&dd->aligned_sg, 1); + sg_set_buf(&dd->aligned_sg, dd->buf, len + padlen); + } + + if (dd->src.sg == dd->dst.sg) { + dd->src.sg_len = dma_map_sg(dd->dev, dd->src.sg, dd->src.nents, + DMA_BIDIRECTIONAL); + dd->dst.sg_len = dd->src.sg_len; + if (!dd->src.sg_len) + return -EFAULT; + } else { + dd->src.sg_len = dma_map_sg(dd->dev, dd->src.sg, dd->src.nents, + DMA_TO_DEVICE); + if (!dd->src.sg_len) + return -EFAULT; + + dd->dst.sg_len = dma_map_sg(dd->dev, dd->dst.sg, dd->dst.nents, + DMA_FROM_DEVICE); + if (!dd->dst.sg_len) { + dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents, + DMA_TO_DEVICE); + return -EFAULT; + } + } + + return 0; +} + +static void atmel_aes_unmap(struct atmel_aes_dev *dd) +{ + if (dd->src.sg == dd->dst.sg) { + dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents, + DMA_BIDIRECTIONAL); + + if (dd->src.sg != &dd->aligned_sg) + atmel_aes_restore_sg(&dd->src); + } else { + dma_unmap_sg(dd->dev, dd->dst.sg, dd->dst.nents, + DMA_FROM_DEVICE); + + if (dd->dst.sg != &dd->aligned_sg) + atmel_aes_restore_sg(&dd->dst); + + dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents, + DMA_TO_DEVICE); + + if (dd->src.sg != &dd->aligned_sg) + atmel_aes_restore_sg(&dd->src); + } + + if (dd->dst.sg == &dd->aligned_sg) + sg_copy_from_buffer(dd->real_dst, sg_nents(dd->real_dst), + dd->buf, dd->total); +} + +static int atmel_aes_dma_transfer_start(struct atmel_aes_dev *dd, + enum dma_slave_buswidth addr_width, + enum dma_transfer_direction dir, + u32 maxburst) +{ + struct dma_async_tx_descriptor *desc; + struct dma_slave_config config; + dma_async_tx_callback callback; + struct atmel_aes_dma *dma; + + memset(&config, 0, sizeof(config)); + config.src_addr_width = addr_width; + config.dst_addr_width = addr_width; + config.src_maxburst = maxburst; + config.dst_maxburst = maxburst; + + switch (dir) { + case DMA_MEM_TO_DEV: + dma = &dd->src; + callback = NULL; + config.dst_addr = dd->phys_base + AES_IDATAR(0); + break; + + case DMA_DEV_TO_MEM: + dma = &dd->dst; + callback = atmel_aes_dma_callback; + config.src_addr = dd->phys_base + AES_ODATAR(0); + break; + + default: + return -EINVAL; + } + + desc = dmaengine_prep_config_sg(dma->chan, dma->sg, dma->sg_len, dir, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK, + &config); + if (!desc) + return -ENOMEM; + + desc->callback = callback; + desc->callback_param = dd; + dmaengine_submit(desc); + dma_async_issue_pending(dma->chan); + + return 0; +} + +static int atmel_aes_dma_start(struct atmel_aes_dev *dd, + struct scatterlist *src, + struct scatterlist *dst, + size_t len, + atmel_aes_fn_t resume) +{ + enum dma_slave_buswidth addr_width; + u32 maxburst; + int err; + + switch (dd->ctx->block_size) { + case AES_BLOCK_SIZE: + addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + maxburst = dd->caps.max_burst_size; + break; + + default: + err = -EINVAL; + goto exit; + } + + err = atmel_aes_map(dd, src, dst, len); + if (err) + goto exit; + + dd->resume = resume; + + /* Set output DMA transfer first */ + err = atmel_aes_dma_transfer_start(dd, addr_width, DMA_DEV_TO_MEM, + maxburst); + if (err) + goto unmap; + + /* Then set input DMA transfer */ + err = atmel_aes_dma_transfer_start(dd, addr_width, DMA_MEM_TO_DEV, + maxburst); + if (err) + goto output_transfer_stop; + + return -EINPROGRESS; + +output_transfer_stop: + dmaengine_terminate_sync(dd->dst.chan); +unmap: + atmel_aes_unmap(dd); +exit: + return atmel_aes_complete(dd, err); +} + +static void atmel_aes_dma_callback(void *data) +{ + struct atmel_aes_dev *dd = data; + + atmel_aes_unmap(dd); + dd->is_async = true; + (void)dd->resume(dd); +} + +static int atmel_aes_handle_queue(struct atmel_aes_dev *dd, + struct crypto_async_request *new_areq) +{ + struct crypto_async_request *areq, *backlog; + struct atmel_aes_base_ctx *ctx; + unsigned long flags; + bool start_async; + int err, ret = 0; + + spin_lock_irqsave(&dd->lock, flags); + if (new_areq) + ret = crypto_enqueue_request(&dd->queue, new_areq); + if (dd->flags & AES_FLAGS_BUSY) { + spin_unlock_irqrestore(&dd->lock, flags); + return ret; + } + backlog = crypto_get_backlog(&dd->queue); + areq = crypto_dequeue_request(&dd->queue); + if (areq) + dd->flags |= AES_FLAGS_BUSY; + spin_unlock_irqrestore(&dd->lock, flags); + + if (!areq) + return ret; + + if (backlog) + crypto_request_complete(backlog, -EINPROGRESS); + + ctx = crypto_tfm_ctx(areq->tfm); + + dd->areq = areq; + dd->ctx = ctx; + start_async = (areq != new_areq); + dd->is_async = start_async; + + /* WARNING: ctx->start() MAY change dd->is_async. */ + err = ctx->start(dd); + return (start_async) ? ret : err; +} + + +/* AES async block ciphers */ + +static int atmel_aes_transfer_complete(struct atmel_aes_dev *dd) +{ + return atmel_aes_complete(dd, 0); +} + +static int atmel_aes_start(struct atmel_aes_dev *dd) +{ + struct skcipher_request *req = skcipher_request_cast(dd->areq); + struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req); + bool use_dma = (req->cryptlen >= ATMEL_AES_DMA_THRESHOLD || + dd->ctx->block_size != AES_BLOCK_SIZE); + int err; + + atmel_aes_set_mode(dd, rctx); + + err = atmel_aes_hw_init(dd); + if (err) + return atmel_aes_complete(dd, err); + + atmel_aes_write_ctrl(dd, use_dma, (void *)req->iv); + if (use_dma) + return atmel_aes_dma_start(dd, req->src, req->dst, + req->cryptlen, + atmel_aes_transfer_complete); + + return atmel_aes_cpu_start(dd, req->src, req->dst, req->cryptlen, + atmel_aes_transfer_complete); +} + +static int atmel_aes_ctr_transfer(struct atmel_aes_dev *dd) +{ + struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx); + struct skcipher_request *req = skcipher_request_cast(dd->areq); + struct scatterlist *src, *dst; + size_t datalen; + u32 ctr; + u16 start, end; + bool use_dma, fragmented = false; + + /* Check for transfer completion. */ + ctx->offset += dd->total; + if (ctx->offset >= req->cryptlen) + return atmel_aes_transfer_complete(dd); + + /* Compute data length. */ + datalen = req->cryptlen - ctx->offset; + ctx->blocks = DIV_ROUND_UP(datalen, AES_BLOCK_SIZE); + ctr = be32_to_cpu(ctx->iv[3]); + + /* Check 16bit counter overflow. */ + start = ctr & 0xffff; + end = start + ctx->blocks - 1; + + if (ctx->blocks >> 16 || end < start) { + ctr |= 0xffff; + datalen = AES_BLOCK_SIZE * (0x10000 - start); + fragmented = true; + } + + use_dma = (datalen >= ATMEL_AES_DMA_THRESHOLD); + + /* Jump to offset. */ + src = scatterwalk_ffwd(ctx->src, req->src, ctx->offset); + dst = ((req->src == req->dst) ? src : + scatterwalk_ffwd(ctx->dst, req->dst, ctx->offset)); + + /* Configure hardware. */ + atmel_aes_write_ctrl(dd, use_dma, ctx->iv); + if (unlikely(fragmented)) { + /* + * Increment the counter manually to cope with the hardware + * counter overflow. + */ + ctx->iv[3] = cpu_to_be32(ctr); + crypto_inc((u8 *)ctx->iv, AES_BLOCK_SIZE); + } + + if (use_dma) + return atmel_aes_dma_start(dd, src, dst, datalen, + atmel_aes_ctr_transfer); + + return atmel_aes_cpu_start(dd, src, dst, datalen, + atmel_aes_ctr_transfer); +} + +static int atmel_aes_ctr_start(struct atmel_aes_dev *dd) +{ + struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx); + struct skcipher_request *req = skcipher_request_cast(dd->areq); + struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req); + int err; + + atmel_aes_set_mode(dd, rctx); + + err = atmel_aes_hw_init(dd); + if (err) + return atmel_aes_complete(dd, err); + + memcpy(ctx->iv, req->iv, AES_BLOCK_SIZE); + ctx->offset = 0; + dd->total = 0; + return atmel_aes_ctr_transfer(dd); +} + +static int atmel_aes_xts_fallback(struct skcipher_request *req, bool enc) +{ + struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req); + struct atmel_aes_xts_ctx *ctx = crypto_skcipher_ctx( + crypto_skcipher_reqtfm(req)); + + skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + skcipher_request_set_callback(&rctx->fallback_req, req->base.flags, + req->base.complete, req->base.data); + skcipher_request_set_crypt(&rctx->fallback_req, req->src, req->dst, + req->cryptlen, req->iv); + + return enc ? crypto_skcipher_encrypt(&rctx->fallback_req) : + crypto_skcipher_decrypt(&rctx->fallback_req); +} + +static int atmel_aes_crypt(struct skcipher_request *req, unsigned long mode) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + struct atmel_aes_base_ctx *ctx = crypto_skcipher_ctx(skcipher); + struct atmel_aes_reqctx *rctx; + u32 opmode = mode & AES_FLAGS_OPMODE_MASK; + + if (opmode == AES_FLAGS_XTS) { + if (req->cryptlen < XTS_BLOCK_SIZE) + return -EINVAL; + + if (!IS_ALIGNED(req->cryptlen, XTS_BLOCK_SIZE)) + return atmel_aes_xts_fallback(req, + mode & AES_FLAGS_ENCRYPT); + } + + /* + * ECB, CBC or CTR mode require the plaintext and ciphertext + * to have a positve integer length. + */ + if (!req->cryptlen && opmode != AES_FLAGS_XTS) + return 0; + + if ((opmode == AES_FLAGS_ECB || opmode == AES_FLAGS_CBC) && + !IS_ALIGNED(req->cryptlen, crypto_skcipher_blocksize(skcipher))) + return -EINVAL; + + ctx->block_size = AES_BLOCK_SIZE; + ctx->is_aead = false; + + rctx = skcipher_request_ctx(req); + rctx->mode = mode; + + if (opmode != AES_FLAGS_ECB && + !(mode & AES_FLAGS_ENCRYPT)) { + unsigned int ivsize = crypto_skcipher_ivsize(skcipher); + + if (req->cryptlen >= ivsize) + scatterwalk_map_and_copy(rctx->lastc, req->src, + req->cryptlen - ivsize, + ivsize, 0); + } + + return atmel_aes_handle_queue(ctx->dd, &req->base); +} + +static int atmel_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct atmel_aes_base_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (keylen != AES_KEYSIZE_128 && + keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) + return -EINVAL; + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + return 0; +} + +static int atmel_aes_ecb_encrypt(struct skcipher_request *req) +{ + return atmel_aes_crypt(req, AES_FLAGS_ECB | AES_FLAGS_ENCRYPT); +} + +static int atmel_aes_ecb_decrypt(struct skcipher_request *req) +{ + return atmel_aes_crypt(req, AES_FLAGS_ECB); +} + +static int atmel_aes_cbc_encrypt(struct skcipher_request *req) +{ + return atmel_aes_crypt(req, AES_FLAGS_CBC | AES_FLAGS_ENCRYPT); +} + +static int atmel_aes_cbc_decrypt(struct skcipher_request *req) +{ + return atmel_aes_crypt(req, AES_FLAGS_CBC); +} + +static int atmel_aes_ctr_encrypt(struct skcipher_request *req) +{ + return atmel_aes_crypt(req, AES_FLAGS_CTR | AES_FLAGS_ENCRYPT); +} + +static int atmel_aes_ctr_decrypt(struct skcipher_request *req) +{ + return atmel_aes_crypt(req, AES_FLAGS_CTR); +} + +static int atmel_aes_init_tfm(struct crypto_skcipher *tfm) +{ + struct atmel_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + struct atmel_aes_dev *dd; + + dd = atmel_aes_dev_alloc(&ctx->base); + if (!dd) + return -ENODEV; + + crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx)); + ctx->base.dd = dd; + ctx->base.start = atmel_aes_start; + + return 0; +} + +static int atmel_aes_ctr_init_tfm(struct crypto_skcipher *tfm) +{ + struct atmel_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + struct atmel_aes_dev *dd; + + dd = atmel_aes_dev_alloc(&ctx->base); + if (!dd) + return -ENODEV; + + crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx)); + ctx->base.dd = dd; + ctx->base.start = atmel_aes_ctr_start; + + return 0; +} + +static struct skcipher_alg aes_algs[] = { +{ + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "atmel-ecb-aes", + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct atmel_aes_ctx), + + .init = atmel_aes_init_tfm, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = atmel_aes_setkey, + .encrypt = atmel_aes_ecb_encrypt, + .decrypt = atmel_aes_ecb_decrypt, +}, +{ + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "atmel-cbc-aes", + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct atmel_aes_ctx), + + .init = atmel_aes_init_tfm, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = atmel_aes_setkey, + .encrypt = atmel_aes_cbc_encrypt, + .decrypt = atmel_aes_cbc_decrypt, + .ivsize = AES_BLOCK_SIZE, +}, +{ + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "atmel-ctr-aes", + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct atmel_aes_ctr_ctx), + + .init = atmel_aes_ctr_init_tfm, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = atmel_aes_setkey, + .encrypt = atmel_aes_ctr_encrypt, + .decrypt = atmel_aes_ctr_decrypt, + .ivsize = AES_BLOCK_SIZE, +}, +}; + + +/* gcm aead functions */ + +static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd, + const u32 *data, size_t datalen, + const __be32 *ghash_in, __be32 *ghash_out, + atmel_aes_fn_t resume); +static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd); +static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd); + +static int atmel_aes_gcm_start(struct atmel_aes_dev *dd); +static int atmel_aes_gcm_process(struct atmel_aes_dev *dd); +static int atmel_aes_gcm_length(struct atmel_aes_dev *dd); +static int atmel_aes_gcm_data(struct atmel_aes_dev *dd); +static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd); +static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd); +static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd); + +static inline struct atmel_aes_gcm_ctx * +atmel_aes_gcm_ctx_cast(struct atmel_aes_base_ctx *ctx) +{ + return container_of(ctx, struct atmel_aes_gcm_ctx, base); +} + +static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd, + const u32 *data, size_t datalen, + const __be32 *ghash_in, __be32 *ghash_out, + atmel_aes_fn_t resume) +{ + struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); + + dd->data = (u32 *)data; + dd->datalen = datalen; + ctx->ghash_in = ghash_in; + ctx->ghash_out = ghash_out; + ctx->ghash_resume = resume; + + atmel_aes_write_ctrl(dd, false, NULL); + return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_ghash_init); +} + +static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd) +{ + struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); + + /* Set the data length. */ + atmel_aes_write(dd, AES_AADLENR, dd->total); + atmel_aes_write(dd, AES_CLENR, 0); + + /* If needed, overwrite the GCM Intermediate Hash Word Registers */ + if (ctx->ghash_in) + atmel_aes_write_block(dd, AES_GHASHR(0), ctx->ghash_in); + + return atmel_aes_gcm_ghash_finalize(dd); +} + +static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd) +{ + struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); + u32 isr; + + /* Write data into the Input Data Registers. */ + while (dd->datalen > 0) { + atmel_aes_write_block(dd, AES_IDATAR(0), dd->data); + dd->data += 4; + dd->datalen -= AES_BLOCK_SIZE; + + isr = atmel_aes_read(dd, AES_ISR); + if (!(isr & AES_INT_DATARDY)) { + dd->resume = atmel_aes_gcm_ghash_finalize; + atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); + return -EINPROGRESS; + } + } + + /* Read the computed hash from GHASHRx. */ + atmel_aes_read_block(dd, AES_GHASHR(0), ctx->ghash_out); + + return ctx->ghash_resume(dd); +} + + +static int atmel_aes_gcm_start(struct atmel_aes_dev *dd) +{ + struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); + struct aead_request *req = aead_request_cast(dd->areq); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct atmel_aes_reqctx *rctx = aead_request_ctx(req); + size_t ivsize = crypto_aead_ivsize(tfm); + size_t datalen, padlen; + const void *iv = req->iv; + u8 *data = dd->buf; + int err; + + atmel_aes_set_mode(dd, rctx); + + err = atmel_aes_hw_init(dd); + if (err) + return atmel_aes_complete(dd, err); + + if (likely(ivsize == GCM_AES_IV_SIZE)) { + memcpy(ctx->j0, iv, ivsize); + ctx->j0[3] = cpu_to_be32(1); + return atmel_aes_gcm_process(dd); + } + + padlen = atmel_aes_padlen(ivsize, AES_BLOCK_SIZE); + datalen = ivsize + padlen + AES_BLOCK_SIZE; + if (datalen > dd->buflen) + return atmel_aes_complete(dd, -EINVAL); + + memcpy(data, iv, ivsize); + memset(data + ivsize, 0, padlen + sizeof(u64)); + ((__be64 *)(data + datalen))[-1] = cpu_to_be64(ivsize * 8); + + return atmel_aes_gcm_ghash(dd, (const u32 *)data, datalen, + NULL, ctx->j0, atmel_aes_gcm_process); +} + +static int atmel_aes_gcm_process(struct atmel_aes_dev *dd) +{ + struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); + struct aead_request *req = aead_request_cast(dd->areq); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + bool enc = atmel_aes_is_encrypt(dd); + u32 authsize; + + /* Compute text length. */ + authsize = crypto_aead_authsize(tfm); + ctx->textlen = req->cryptlen - (enc ? 0 : authsize); + + /* + * According to tcrypt test suite, the GCM Automatic Tag Generation + * fails when both the message and its associated data are empty. + */ + if (likely(req->assoclen != 0 || ctx->textlen != 0)) + dd->flags |= AES_FLAGS_GTAGEN; + + atmel_aes_write_ctrl(dd, false, NULL); + return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_length); +} + +static int atmel_aes_gcm_length(struct atmel_aes_dev *dd) +{ + struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); + struct aead_request *req = aead_request_cast(dd->areq); + __be32 j0_lsw, *j0 = ctx->j0; + size_t padlen; + + /* Write incr32(J0) into IV. */ + j0_lsw = j0[3]; + be32_add_cpu(&j0[3], 1); + atmel_aes_write_block(dd, AES_IVR(0), j0); + j0[3] = j0_lsw; + + /* Set aad and text lengths. */ + atmel_aes_write(dd, AES_AADLENR, req->assoclen); + atmel_aes_write(dd, AES_CLENR, ctx->textlen); + + /* Check whether AAD are present. */ + if (unlikely(req->assoclen == 0)) { + dd->datalen = 0; + return atmel_aes_gcm_data(dd); + } + + /* Copy assoc data and add padding. */ + padlen = atmel_aes_padlen(req->assoclen, AES_BLOCK_SIZE); + if (unlikely(req->assoclen + padlen > dd->buflen)) + return atmel_aes_complete(dd, -EINVAL); + sg_copy_to_buffer(req->src, sg_nents(req->src), dd->buf, req->assoclen); + + /* Write assoc data into the Input Data register. */ + dd->data = (u32 *)dd->buf; + dd->datalen = req->assoclen + padlen; + return atmel_aes_gcm_data(dd); +} + +static int atmel_aes_gcm_data(struct atmel_aes_dev *dd) +{ + struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); + struct aead_request *req = aead_request_cast(dd->areq); + bool use_dma = (ctx->textlen >= ATMEL_AES_DMA_THRESHOLD); + struct scatterlist *src, *dst; + u32 isr, mr; + + /* Write AAD first. */ + while (dd->datalen > 0) { + atmel_aes_write_block(dd, AES_IDATAR(0), dd->data); + dd->data += 4; + dd->datalen -= AES_BLOCK_SIZE; + + isr = atmel_aes_read(dd, AES_ISR); + if (!(isr & AES_INT_DATARDY)) { + dd->resume = atmel_aes_gcm_data; + atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); + return -EINPROGRESS; + } + } + + /* GMAC only. */ + if (unlikely(ctx->textlen == 0)) + return atmel_aes_gcm_tag_init(dd); + + /* Prepare src and dst scatter lists to transfer cipher/plain texts */ + src = scatterwalk_ffwd(ctx->src, req->src, req->assoclen); + dst = ((req->src == req->dst) ? src : + scatterwalk_ffwd(ctx->dst, req->dst, req->assoclen)); + + if (use_dma) { + /* Update the Mode Register for DMA transfers. */ + mr = atmel_aes_read(dd, AES_MR); + mr &= ~(AES_MR_SMOD_MASK | AES_MR_DUALBUFF); + mr |= AES_MR_SMOD_IDATAR0; + if (dd->caps.has_dualbuff) + mr |= AES_MR_DUALBUFF; + atmel_aes_write(dd, AES_MR, mr); + + return atmel_aes_dma_start(dd, src, dst, ctx->textlen, + atmel_aes_gcm_tag_init); + } + + return atmel_aes_cpu_start(dd, src, dst, ctx->textlen, + atmel_aes_gcm_tag_init); +} + +static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd) +{ + struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); + struct aead_request *req = aead_request_cast(dd->areq); + __be64 *data = dd->buf; + + if (likely(dd->flags & AES_FLAGS_GTAGEN)) { + if (!(atmel_aes_read(dd, AES_ISR) & AES_INT_TAGRDY)) { + dd->resume = atmel_aes_gcm_tag_init; + atmel_aes_write(dd, AES_IER, AES_INT_TAGRDY); + return -EINPROGRESS; + } + + return atmel_aes_gcm_finalize(dd); + } + + /* Read the GCM Intermediate Hash Word Registers. */ + atmel_aes_read_block(dd, AES_GHASHR(0), ctx->ghash); + + data[0] = cpu_to_be64(req->assoclen * 8); + data[1] = cpu_to_be64(ctx->textlen * 8); + + return atmel_aes_gcm_ghash(dd, (const u32 *)data, AES_BLOCK_SIZE, + ctx->ghash, ctx->ghash, atmel_aes_gcm_tag); +} + +static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd) +{ + struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); + unsigned long flags; + + /* + * Change mode to CTR to complete the tag generation. + * Use J0 as Initialization Vector. + */ + flags = dd->flags; + dd->flags &= ~(AES_FLAGS_OPMODE_MASK | AES_FLAGS_GTAGEN); + dd->flags |= AES_FLAGS_CTR; + atmel_aes_write_ctrl(dd, false, ctx->j0); + dd->flags = flags; + + atmel_aes_write_block(dd, AES_IDATAR(0), ctx->ghash); + return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_finalize); +} + +static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd) +{ + struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); + struct aead_request *req = aead_request_cast(dd->areq); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + bool enc = atmel_aes_is_encrypt(dd); + u32 offset, authsize, itag[4], *otag = ctx->tag; + int err; + + /* Read the computed tag. */ + if (likely(dd->flags & AES_FLAGS_GTAGEN)) + atmel_aes_read_block(dd, AES_TAGR(0), ctx->tag); + else + atmel_aes_read_block(dd, AES_ODATAR(0), ctx->tag); + + offset = req->assoclen + ctx->textlen; + authsize = crypto_aead_authsize(tfm); + if (enc) { + scatterwalk_map_and_copy(otag, req->dst, offset, authsize, 1); + err = 0; + } else { + scatterwalk_map_and_copy(itag, req->src, offset, authsize, 0); + err = crypto_memneq(itag, otag, authsize) ? -EBADMSG : 0; + } + + return atmel_aes_complete(dd, err); +} + +static int atmel_aes_gcm_crypt(struct aead_request *req, + unsigned long mode) +{ + struct atmel_aes_base_ctx *ctx; + struct atmel_aes_reqctx *rctx; + + ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + ctx->block_size = AES_BLOCK_SIZE; + ctx->is_aead = true; + + rctx = aead_request_ctx(req); + rctx->mode = AES_FLAGS_GCM | mode; + + return atmel_aes_handle_queue(ctx->dd, &req->base); +} + +static int atmel_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct atmel_aes_base_ctx *ctx = crypto_aead_ctx(tfm); + + if (keylen != AES_KEYSIZE_256 && + keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_128) + return -EINVAL; + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + return 0; +} + +static int atmel_aes_gcm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + return crypto_gcm_check_authsize(authsize); +} + +static int atmel_aes_gcm_encrypt(struct aead_request *req) +{ + return atmel_aes_gcm_crypt(req, AES_FLAGS_ENCRYPT); +} + +static int atmel_aes_gcm_decrypt(struct aead_request *req) +{ + return atmel_aes_gcm_crypt(req, 0); +} + +static int atmel_aes_gcm_init(struct crypto_aead *tfm) +{ + struct atmel_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); + struct atmel_aes_dev *dd; + + dd = atmel_aes_dev_alloc(&ctx->base); + if (!dd) + return -ENODEV; + + crypto_aead_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx)); + ctx->base.dd = dd; + ctx->base.start = atmel_aes_gcm_start; + + return 0; +} + +static struct aead_alg aes_gcm_alg = { + .setkey = atmel_aes_gcm_setkey, + .setauthsize = atmel_aes_gcm_setauthsize, + .encrypt = atmel_aes_gcm_encrypt, + .decrypt = atmel_aes_gcm_decrypt, + .init = atmel_aes_gcm_init, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "atmel-gcm-aes", + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct atmel_aes_gcm_ctx), + }, +}; + + +/* xts functions */ + +static inline struct atmel_aes_xts_ctx * +atmel_aes_xts_ctx_cast(struct atmel_aes_base_ctx *ctx) +{ + return container_of(ctx, struct atmel_aes_xts_ctx, base); +} + +static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd); + +static int atmel_aes_xts_start(struct atmel_aes_dev *dd) +{ + struct atmel_aes_xts_ctx *ctx = atmel_aes_xts_ctx_cast(dd->ctx); + struct skcipher_request *req = skcipher_request_cast(dd->areq); + struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req); + unsigned long flags; + int err; + + atmel_aes_set_mode(dd, rctx); + + err = atmel_aes_hw_init(dd); + if (err) + return atmel_aes_complete(dd, err); + + /* Compute the tweak value from req->iv with ecb(aes). */ + flags = dd->flags; + dd->flags &= ~AES_FLAGS_MODE_MASK; + dd->flags |= (AES_FLAGS_ECB | AES_FLAGS_ENCRYPT); + atmel_aes_write_ctrl_key(dd, false, NULL, + ctx->key2, ctx->base.keylen); + dd->flags = flags; + + atmel_aes_write_block(dd, AES_IDATAR(0), req->iv); + return atmel_aes_wait_for_data_ready(dd, atmel_aes_xts_process_data); +} + +static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd) +{ + struct skcipher_request *req = skcipher_request_cast(dd->areq); + bool use_dma = (req->cryptlen >= ATMEL_AES_DMA_THRESHOLD); + u32 tweak[AES_BLOCK_SIZE / sizeof(u32)]; + static const __le32 one[AES_BLOCK_SIZE / sizeof(u32)] = {cpu_to_le32(1), }; + u8 *tweak_bytes = (u8 *)tweak; + int i; + + /* Read the computed ciphered tweak value. */ + atmel_aes_read_block(dd, AES_ODATAR(0), tweak); + /* + * Hardware quirk: + * the order of the ciphered tweak bytes need to be reversed before + * writing them into the ODATARx registers. + */ + for (i = 0; i < AES_BLOCK_SIZE/2; ++i) + swap(tweak_bytes[i], tweak_bytes[AES_BLOCK_SIZE - 1 - i]); + + /* Process the data. */ + atmel_aes_write_ctrl(dd, use_dma, NULL); + atmel_aes_write_block(dd, AES_TWR(0), tweak); + atmel_aes_write_block(dd, AES_ALPHAR(0), one); + if (use_dma) + return atmel_aes_dma_start(dd, req->src, req->dst, + req->cryptlen, + atmel_aes_transfer_complete); + + return atmel_aes_cpu_start(dd, req->src, req->dst, req->cryptlen, + atmel_aes_transfer_complete); +} + +static int atmel_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct atmel_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); + int err; + + err = xts_verify_key(tfm, key, keylen); + if (err) + return err; + + crypto_skcipher_clear_flags(ctx->fallback_tfm, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(ctx->fallback_tfm, tfm->base.crt_flags & + CRYPTO_TFM_REQ_MASK); + err = crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen); + if (err) + return err; + + memcpy(ctx->base.key, key, keylen/2); + memcpy(ctx->key2, key + keylen/2, keylen/2); + ctx->base.keylen = keylen/2; + + return 0; +} + +static int atmel_aes_xts_encrypt(struct skcipher_request *req) +{ + return atmel_aes_crypt(req, AES_FLAGS_XTS | AES_FLAGS_ENCRYPT); +} + +static int atmel_aes_xts_decrypt(struct skcipher_request *req) +{ + return atmel_aes_crypt(req, AES_FLAGS_XTS); +} + +static int atmel_aes_xts_init_tfm(struct crypto_skcipher *tfm) +{ + struct atmel_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); + struct atmel_aes_dev *dd; + const char *tfm_name = crypto_tfm_alg_name(&tfm->base); + + dd = atmel_aes_dev_alloc(&ctx->base); + if (!dd) + return -ENODEV; + + ctx->fallback_tfm = crypto_alloc_skcipher(tfm_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fallback_tfm)) + return PTR_ERR(ctx->fallback_tfm); + + crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx) + + crypto_skcipher_reqsize(ctx->fallback_tfm)); + ctx->base.dd = dd; + ctx->base.start = atmel_aes_xts_start; + + return 0; +} + +static void atmel_aes_xts_exit_tfm(struct crypto_skcipher *tfm) +{ + struct atmel_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); + + crypto_free_skcipher(ctx->fallback_tfm); +} + +static struct skcipher_alg aes_xts_alg = { + .base.cra_name = "xts(aes)", + .base.cra_driver_name = "atmel-xts-aes", + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct atmel_aes_xts_ctx), + .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = atmel_aes_xts_setkey, + .encrypt = atmel_aes_xts_encrypt, + .decrypt = atmel_aes_xts_decrypt, + .init = atmel_aes_xts_init_tfm, + .exit = atmel_aes_xts_exit_tfm, +}; + +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) +/* authenc aead functions */ + +static int atmel_aes_authenc_start(struct atmel_aes_dev *dd); +static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err, + bool is_async); +static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err, + bool is_async); +static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd); +static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err, + bool is_async); + +static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err) +{ + struct aead_request *req = aead_request_cast(dd->areq); + struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); + + if (err && (dd->flags & AES_FLAGS_OWN_SHA)) + atmel_sha_authenc_abort(&rctx->auth_req); + dd->flags &= ~AES_FLAGS_OWN_SHA; +} + +static int atmel_aes_authenc_start(struct atmel_aes_dev *dd) +{ + struct aead_request *req = aead_request_cast(dd->areq); + struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm); + int err; + + atmel_aes_set_mode(dd, &rctx->base); + + err = atmel_aes_hw_init(dd); + if (err) + return atmel_aes_complete(dd, err); + + return atmel_sha_authenc_schedule(&rctx->auth_req, ctx->auth, + atmel_aes_authenc_init, dd); +} + +static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err, + bool is_async) +{ + struct aead_request *req = aead_request_cast(dd->areq); + struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); + + if (is_async) + dd->is_async = true; + if (err) + return atmel_aes_complete(dd, err); + + /* If here, we've got the ownership of the SHA device. */ + dd->flags |= AES_FLAGS_OWN_SHA; + + /* Configure the SHA device. */ + return atmel_sha_authenc_init(&rctx->auth_req, + req->src, req->assoclen, + rctx->textlen, + atmel_aes_authenc_transfer, dd); +} + +static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err, + bool is_async) +{ + struct aead_request *req = aead_request_cast(dd->areq); + struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); + bool enc = atmel_aes_is_encrypt(dd); + struct scatterlist *src, *dst; + __be32 iv[AES_BLOCK_SIZE / sizeof(u32)]; + u32 emr; + + if (is_async) + dd->is_async = true; + if (err) + return atmel_aes_complete(dd, err); + + /* Prepare src and dst scatter-lists to transfer cipher/plain texts. */ + src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen); + dst = src; + + if (req->src != req->dst) + dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen); + + /* Configure the AES device. */ + memcpy(iv, req->iv, sizeof(iv)); + + /* + * Here we always set the 2nd parameter of atmel_aes_write_ctrl() to + * 'true' even if the data transfer is actually performed by the CPU (so + * not by the DMA) because we must force the AES_MR_SMOD bitfield to the + * value AES_MR_SMOD_IDATAR0. Indeed, both AES_MR_SMOD and SHA_MR_SMOD + * must be set to *_MR_SMOD_IDATAR0. + */ + atmel_aes_write_ctrl(dd, true, iv); + emr = AES_EMR_PLIPEN; + if (!enc) + emr |= AES_EMR_PLIPD; + atmel_aes_write(dd, AES_EMR, emr); + + /* Transfer data. */ + return atmel_aes_dma_start(dd, src, dst, rctx->textlen, + atmel_aes_authenc_digest); +} + +static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd) +{ + struct aead_request *req = aead_request_cast(dd->areq); + struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); + + /* atmel_sha_authenc_final() releases the SHA device. */ + dd->flags &= ~AES_FLAGS_OWN_SHA; + return atmel_sha_authenc_final(&rctx->auth_req, + rctx->digest, sizeof(rctx->digest), + atmel_aes_authenc_final, dd); +} + +static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err, + bool is_async) +{ + struct aead_request *req = aead_request_cast(dd->areq); + struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + bool enc = atmel_aes_is_encrypt(dd); + u32 idigest[SHA512_DIGEST_SIZE / sizeof(u32)], *odigest = rctx->digest; + u32 offs, authsize; + + if (is_async) + dd->is_async = true; + if (err) + goto complete; + + offs = req->assoclen + rctx->textlen; + authsize = crypto_aead_authsize(tfm); + if (enc) { + scatterwalk_map_and_copy(odigest, req->dst, offs, authsize, 1); + } else { + scatterwalk_map_and_copy(idigest, req->src, offs, authsize, 0); + if (crypto_memneq(idigest, odigest, authsize)) + err = -EBADMSG; + } + +complete: + return atmel_aes_complete(dd, err); +} + +static int atmel_aes_authenc_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm); + struct crypto_authenc_keys keys; + int err; + + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) + goto badkey; + + if (keys.enckeylen > sizeof(ctx->base.key)) + goto badkey; + + /* Save auth key. */ + err = atmel_sha_authenc_setkey(ctx->auth, + keys.authkey, keys.authkeylen, + crypto_aead_get_flags(tfm)); + if (err) { + memzero_explicit(&keys, sizeof(keys)); + return err; + } + + /* Save enc key. */ + ctx->base.keylen = keys.enckeylen; + memcpy(ctx->base.key, keys.enckey, keys.enckeylen); + + memzero_explicit(&keys, sizeof(keys)); + return 0; + +badkey: + memzero_explicit(&keys, sizeof(keys)); + return -EINVAL; +} + +static int atmel_aes_authenc_init_tfm(struct crypto_aead *tfm, + unsigned long auth_mode) +{ + struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm); + unsigned int auth_reqsize = atmel_sha_authenc_get_reqsize(); + struct atmel_aes_dev *dd; + + dd = atmel_aes_dev_alloc(&ctx->base); + if (!dd) + return -ENODEV; + + ctx->auth = atmel_sha_authenc_spawn(auth_mode); + if (IS_ERR(ctx->auth)) + return PTR_ERR(ctx->auth); + + crypto_aead_set_reqsize(tfm, (sizeof(struct atmel_aes_authenc_reqctx) + + auth_reqsize)); + ctx->base.dd = dd; + ctx->base.start = atmel_aes_authenc_start; + + return 0; +} + +static int atmel_aes_authenc_hmac_sha1_init_tfm(struct crypto_aead *tfm) +{ + return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA1); +} + +static int atmel_aes_authenc_hmac_sha224_init_tfm(struct crypto_aead *tfm) +{ + return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA224); +} + +static int atmel_aes_authenc_hmac_sha256_init_tfm(struct crypto_aead *tfm) +{ + return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA256); +} + +static int atmel_aes_authenc_hmac_sha384_init_tfm(struct crypto_aead *tfm) +{ + return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA384); +} + +static int atmel_aes_authenc_hmac_sha512_init_tfm(struct crypto_aead *tfm) +{ + return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA512); +} + +static void atmel_aes_authenc_exit_tfm(struct crypto_aead *tfm) +{ + struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm); + + atmel_sha_authenc_free(ctx->auth); +} + +static int atmel_aes_authenc_crypt(struct aead_request *req, + unsigned long mode) +{ + struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct atmel_aes_base_ctx *ctx = crypto_aead_ctx(tfm); + u32 authsize = crypto_aead_authsize(tfm); + bool enc = (mode & AES_FLAGS_ENCRYPT); + + /* Compute text length. */ + if (!enc && req->cryptlen < authsize) + return -EINVAL; + rctx->textlen = req->cryptlen - (enc ? 0 : authsize); + + /* + * Currently, empty messages are not supported yet: + * the SHA auto-padding can be used only on non-empty messages. + * Hence a special case needs to be implemented for empty message. + */ + if (!rctx->textlen && !req->assoclen) + return -EINVAL; + + rctx->base.mode = mode; + ctx->block_size = AES_BLOCK_SIZE; + ctx->is_aead = true; + + return atmel_aes_handle_queue(ctx->dd, &req->base); +} + +static int atmel_aes_authenc_cbc_aes_encrypt(struct aead_request *req) +{ + return atmel_aes_authenc_crypt(req, AES_FLAGS_CBC | AES_FLAGS_ENCRYPT); +} + +static int atmel_aes_authenc_cbc_aes_decrypt(struct aead_request *req) +{ + return atmel_aes_authenc_crypt(req, AES_FLAGS_CBC); +} + +static struct aead_alg aes_authenc_algs[] = { +{ + .setkey = atmel_aes_authenc_setkey, + .encrypt = atmel_aes_authenc_cbc_aes_encrypt, + .decrypt = atmel_aes_authenc_cbc_aes_decrypt, + .init = atmel_aes_authenc_hmac_sha1_init_tfm, + .exit = atmel_aes_authenc_exit_tfm, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "atmel-authenc-hmac-sha1-cbc-aes", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), + }, +}, +{ + .setkey = atmel_aes_authenc_setkey, + .encrypt = atmel_aes_authenc_cbc_aes_encrypt, + .decrypt = atmel_aes_authenc_cbc_aes_decrypt, + .init = atmel_aes_authenc_hmac_sha224_init_tfm, + .exit = atmel_aes_authenc_exit_tfm, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + + .base = { + .cra_name = "authenc(hmac(sha224),cbc(aes))", + .cra_driver_name = "atmel-authenc-hmac-sha224-cbc-aes", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), + }, +}, +{ + .setkey = atmel_aes_authenc_setkey, + .encrypt = atmel_aes_authenc_cbc_aes_encrypt, + .decrypt = atmel_aes_authenc_cbc_aes_decrypt, + .init = atmel_aes_authenc_hmac_sha256_init_tfm, + .exit = atmel_aes_authenc_exit_tfm, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "atmel-authenc-hmac-sha256-cbc-aes", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), + }, +}, +{ + .setkey = atmel_aes_authenc_setkey, + .encrypt = atmel_aes_authenc_cbc_aes_encrypt, + .decrypt = atmel_aes_authenc_cbc_aes_decrypt, + .init = atmel_aes_authenc_hmac_sha384_init_tfm, + .exit = atmel_aes_authenc_exit_tfm, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + + .base = { + .cra_name = "authenc(hmac(sha384),cbc(aes))", + .cra_driver_name = "atmel-authenc-hmac-sha384-cbc-aes", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), + }, +}, +{ + .setkey = atmel_aes_authenc_setkey, + .encrypt = atmel_aes_authenc_cbc_aes_encrypt, + .decrypt = atmel_aes_authenc_cbc_aes_decrypt, + .init = atmel_aes_authenc_hmac_sha512_init_tfm, + .exit = atmel_aes_authenc_exit_tfm, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + + .base = { + .cra_name = "authenc(hmac(sha512),cbc(aes))", + .cra_driver_name = "atmel-authenc-hmac-sha512-cbc-aes", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), + }, +}, +}; +#endif /* CONFIG_CRYPTO_DEV_ATMEL_AUTHENC */ + +/* Probe functions */ + +static int atmel_aes_buff_init(struct atmel_aes_dev *dd) +{ + dd->buf = (void *)__get_free_pages(GFP_KERNEL, ATMEL_AES_BUFFER_ORDER); + dd->buflen = ATMEL_AES_BUFFER_SIZE; + dd->buflen &= ~(AES_BLOCK_SIZE - 1); + + if (!dd->buf) { + dev_err(dd->dev, "unable to alloc pages.\n"); + return -ENOMEM; + } + + return 0; +} + +static void atmel_aes_buff_cleanup(struct atmel_aes_dev *dd) +{ + free_pages((unsigned long)dd->buf, ATMEL_AES_BUFFER_ORDER); +} + +static int atmel_aes_dma_init(struct atmel_aes_dev *dd) +{ + int ret; + + /* Try to grab 2 DMA channels */ + dd->src.chan = dma_request_chan(dd->dev, "tx"); + if (IS_ERR(dd->src.chan)) { + ret = PTR_ERR(dd->src.chan); + goto err_dma_in; + } + + dd->dst.chan = dma_request_chan(dd->dev, "rx"); + if (IS_ERR(dd->dst.chan)) { + ret = PTR_ERR(dd->dst.chan); + goto err_dma_out; + } + + return 0; + +err_dma_out: + dma_release_channel(dd->src.chan); +err_dma_in: + dev_err(dd->dev, "no DMA channel available\n"); + return ret; +} + +static void atmel_aes_dma_cleanup(struct atmel_aes_dev *dd) +{ + dma_release_channel(dd->dst.chan); + dma_release_channel(dd->src.chan); +} + +static void atmel_aes_queue_task(unsigned long data) +{ + struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data; + + atmel_aes_handle_queue(dd, NULL); +} + +static void atmel_aes_done_task(unsigned long data) +{ + struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data; + + dd->is_async = true; + (void)dd->resume(dd); +} + +static irqreturn_t atmel_aes_irq(int irq, void *dev_id) +{ + struct atmel_aes_dev *aes_dd = dev_id; + u32 reg; + + reg = atmel_aes_read(aes_dd, AES_ISR); + if (reg & atmel_aes_read(aes_dd, AES_IMR)) { + atmel_aes_write(aes_dd, AES_IDR, reg); + if (AES_FLAGS_BUSY & aes_dd->flags) + tasklet_schedule(&aes_dd->done_task); + else + dev_warn(aes_dd->dev, "AES interrupt when no active requests.\n"); + return IRQ_HANDLED; + } + + return IRQ_NONE; +} + +static void atmel_aes_unregister_algs(struct atmel_aes_dev *dd) +{ +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) + if (dd->caps.has_authenc) + crypto_unregister_aeads(aes_authenc_algs, + ARRAY_SIZE(aes_authenc_algs)); +#endif + + if (dd->caps.has_xts) + crypto_unregister_skcipher(&aes_xts_alg); + + if (dd->caps.has_gcm) + crypto_unregister_aead(&aes_gcm_alg); + + crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); +} + +static void atmel_aes_crypto_alg_init(struct crypto_alg *alg) +{ + alg->cra_flags |= CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; + alg->cra_alignmask = 0xf; + alg->cra_priority = ATMEL_AES_PRIORITY; + alg->cra_module = THIS_MODULE; +} + +static int atmel_aes_register_algs(struct atmel_aes_dev *dd) +{ + int err, i; + + for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { + atmel_aes_crypto_alg_init(&aes_algs[i].base); + + err = crypto_register_skcipher(&aes_algs[i]); + if (err) + goto err_aes_algs; + } + + if (dd->caps.has_gcm) { + atmel_aes_crypto_alg_init(&aes_gcm_alg.base); + + err = crypto_register_aead(&aes_gcm_alg); + if (err) + goto err_aes_gcm_alg; + } + + if (dd->caps.has_xts) { + atmel_aes_crypto_alg_init(&aes_xts_alg.base); + + err = crypto_register_skcipher(&aes_xts_alg); + if (err) + goto err_aes_xts_alg; + } + +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) + if (dd->caps.has_authenc) { + for (i = 0; i < ARRAY_SIZE(aes_authenc_algs); i++) { + atmel_aes_crypto_alg_init(&aes_authenc_algs[i].base); + + err = crypto_register_aead(&aes_authenc_algs[i]); + if (err) + goto err_aes_authenc_alg; + } + } +#endif + + return 0; + +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) + /* i = ARRAY_SIZE(aes_authenc_algs); */ +err_aes_authenc_alg: + crypto_unregister_aeads(aes_authenc_algs, i); + if (dd->caps.has_xts) + crypto_unregister_skcipher(&aes_xts_alg); +#endif +err_aes_xts_alg: + if (dd->caps.has_gcm) + crypto_unregister_aead(&aes_gcm_alg); +err_aes_gcm_alg: + i = ARRAY_SIZE(aes_algs); +err_aes_algs: + crypto_unregister_skciphers(aes_algs, i); + + return err; +} + +static void atmel_aes_get_cap(struct atmel_aes_dev *dd) +{ + dd->caps.has_dualbuff = 0; + dd->caps.has_gcm = 0; + dd->caps.has_xts = 0; + dd->caps.has_authenc = 0; + dd->caps.max_burst_size = 1; + + /* keep only major version number */ + switch (dd->hw_version & 0xff0) { + case 0x800: + case 0x700: + case 0x600: + case 0x500: + dd->caps.has_dualbuff = 1; + dd->caps.has_gcm = 1; + dd->caps.has_xts = 1; + dd->caps.has_authenc = 1; + dd->caps.max_burst_size = 4; + break; + case 0x200: + dd->caps.has_dualbuff = 1; + dd->caps.has_gcm = 1; + dd->caps.max_burst_size = 4; + break; + case 0x130: + dd->caps.has_dualbuff = 1; + dd->caps.max_burst_size = 4; + break; + case 0x120: + break; + default: + dev_warn(dd->dev, + "Unmanaged aes version, set minimum capabilities\n"); + break; + } +} + +static const struct of_device_id atmel_aes_dt_ids[] = { + { .compatible = "atmel,at91sam9g46-aes" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, atmel_aes_dt_ids); + +static int atmel_aes_probe(struct platform_device *pdev) +{ + struct atmel_aes_dev *aes_dd; + struct device *dev = &pdev->dev; + struct resource *aes_res; + int err; + + aes_dd = devm_kzalloc(&pdev->dev, sizeof(*aes_dd), GFP_KERNEL); + if (!aes_dd) + return -ENOMEM; + + aes_dd->dev = dev; + + platform_set_drvdata(pdev, aes_dd); + + INIT_LIST_HEAD(&aes_dd->list); + spin_lock_init(&aes_dd->lock); + + tasklet_init(&aes_dd->done_task, atmel_aes_done_task, + (unsigned long)aes_dd); + tasklet_init(&aes_dd->queue_task, atmel_aes_queue_task, + (unsigned long)aes_dd); + + crypto_init_queue(&aes_dd->queue, ATMEL_AES_QUEUE_LENGTH); + + aes_dd->io_base = devm_platform_get_and_ioremap_resource(pdev, 0, &aes_res); + if (IS_ERR(aes_dd->io_base)) { + err = PTR_ERR(aes_dd->io_base); + goto err_tasklet_kill; + } + aes_dd->phys_base = aes_res->start; + + /* Get the IRQ */ + aes_dd->irq = platform_get_irq(pdev, 0); + if (aes_dd->irq < 0) { + err = aes_dd->irq; + goto err_tasklet_kill; + } + + err = devm_request_irq(&pdev->dev, aes_dd->irq, atmel_aes_irq, + IRQF_SHARED, "atmel-aes", aes_dd); + if (err) + goto err_tasklet_kill; + + /* Initializing the clock */ + aes_dd->iclk = devm_clk_get_prepared(&pdev->dev, "aes_clk"); + if (IS_ERR(aes_dd->iclk)) { + dev_err(dev, "clock initialization failed.\n"); + err = PTR_ERR(aes_dd->iclk); + goto err_tasklet_kill; + } + + err = atmel_aes_hw_version_init(aes_dd); + if (err) + goto err_tasklet_kill; + + atmel_aes_get_cap(aes_dd); + +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) + if (aes_dd->caps.has_authenc && !atmel_sha_authenc_is_ready()) { + err = -EPROBE_DEFER; + goto err_tasklet_kill; + } +#endif + + err = atmel_aes_buff_init(aes_dd); + if (err) + goto err_tasklet_kill; + + err = atmel_aes_dma_init(aes_dd); + if (err) + goto err_buff_cleanup; + + spin_lock(&atmel_aes.lock); + list_add_tail(&aes_dd->list, &atmel_aes.dev_list); + spin_unlock(&atmel_aes.lock); + + err = atmel_aes_register_algs(aes_dd); + if (err) + goto err_algs; + + dev_info(dev, "Atmel AES - Using %s, %s for DMA transfers\n", + dma_chan_name(aes_dd->src.chan), + dma_chan_name(aes_dd->dst.chan)); + + return 0; + +err_algs: + spin_lock(&atmel_aes.lock); + list_del(&aes_dd->list); + spin_unlock(&atmel_aes.lock); + atmel_aes_dma_cleanup(aes_dd); +err_buff_cleanup: + atmel_aes_buff_cleanup(aes_dd); +err_tasklet_kill: + tasklet_kill(&aes_dd->done_task); + tasklet_kill(&aes_dd->queue_task); + + return err; +} + +static void atmel_aes_remove(struct platform_device *pdev) +{ + struct atmel_aes_dev *aes_dd; + + aes_dd = platform_get_drvdata(pdev); + + spin_lock(&atmel_aes.lock); + list_del(&aes_dd->list); + spin_unlock(&atmel_aes.lock); + + atmel_aes_unregister_algs(aes_dd); + + tasklet_kill(&aes_dd->done_task); + tasklet_kill(&aes_dd->queue_task); + + atmel_aes_dma_cleanup(aes_dd); + atmel_aes_buff_cleanup(aes_dd); +} + +static struct platform_driver atmel_aes_driver = { + .probe = atmel_aes_probe, + .remove = atmel_aes_remove, + .driver = { + .name = "atmel_aes", + .of_match_table = atmel_aes_dt_ids, + }, +}; + +module_platform_driver(atmel_aes_driver); + +MODULE_DESCRIPTION("Atmel AES hw acceleration support."); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique"); diff --git a/drivers/crypto/atmel-authenc.h b/drivers/crypto/atmel-authenc.h new file mode 100644 index 000000000..45171e89a --- /dev/null +++ b/drivers/crypto/atmel-authenc.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * API for Atmel Secure Protocol Layers Improved Performances (SPLIP) + * + * Copyright (C) 2016 Atmel Corporation + * + * Author: Cyrille Pitchen + * + * This driver is based on drivers/mtd/spi-nor/fsl-quadspi.c from Freescale. + */ + +#ifndef __ATMEL_AUTHENC_H__ +#define __ATMEL_AUTHENC_H__ + +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) + +#include +#include +#include +#include +#include "atmel-sha-regs.h" + +struct atmel_aes_dev; +typedef int (*atmel_aes_authenc_fn_t)(struct atmel_aes_dev *, int, bool); + +struct atmel_sha_authenc_ctx; + +bool atmel_sha_authenc_is_ready(void); +unsigned int atmel_sha_authenc_get_reqsize(void); + +struct atmel_sha_authenc_ctx *atmel_sha_authenc_spawn(unsigned long mode); +void atmel_sha_authenc_free(struct atmel_sha_authenc_ctx *auth); +int atmel_sha_authenc_setkey(struct atmel_sha_authenc_ctx *auth, + const u8 *key, unsigned int keylen, u32 flags); + +int atmel_sha_authenc_schedule(struct ahash_request *req, + struct atmel_sha_authenc_ctx *auth, + atmel_aes_authenc_fn_t cb, + struct atmel_aes_dev *dd); +int atmel_sha_authenc_init(struct ahash_request *req, + struct scatterlist *assoc, unsigned int assoclen, + unsigned int textlen, + atmel_aes_authenc_fn_t cb, + struct atmel_aes_dev *dd); +int atmel_sha_authenc_final(struct ahash_request *req, + u32 *digest, unsigned int digestlen, + atmel_aes_authenc_fn_t cb, + struct atmel_aes_dev *dd); +void atmel_sha_authenc_abort(struct ahash_request *req); + +#endif /* CONFIG_CRYPTO_DEV_ATMEL_AUTHENC */ + +#endif /* __ATMEL_AUTHENC_H__ */ diff --git a/drivers/crypto/atmel-ecc.c b/drivers/crypto/atmel-ecc.c new file mode 100644 index 000000000..075f38a67 --- /dev/null +++ b/drivers/crypto/atmel-ecc.c @@ -0,0 +1,391 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Microchip / Atmel ECC (I2C) driver. + * + * Copyright (c) 2017, Microchip Technology Inc. + * Author: Tudor Ambarus + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "atmel-i2c.h" + +static struct atmel_ecc_driver_data driver_data; + +/** + * struct atmel_ecdh_ctx - transformation context + * @client: I2C client device + * @fallback: ECDH fallback used for caller-provided private keys + * @public_key: cached public key for the device-generated private key + * @do_fallback: true when ECDH operations should use @fallback + * + * The caller must not invoke set_secret() while generate_public_key() + * or compute_shared_secret() are in flight. + */ +struct atmel_ecdh_ctx { + struct i2c_client *client; + struct crypto_kpp *fallback; + const u8 *public_key; + bool do_fallback; +}; + +static void atmel_ecdh_done(struct atmel_i2c_work_data *work_data, void *areq, + int status) +{ + struct kpp_request *req = areq; + struct atmel_i2c_cmd *cmd = &work_data->cmd; + size_t copied, n_sz; + + if (status) + goto free_work_data; + + /* copy only as much as requested, capped at 32 bytes */ + n_sz = min(ATMEL_ECC_NIST_P256_N_SIZE, req->dst_len); + + /* copy the shared secret */ + copied = sg_copy_from_buffer(req->dst, sg_nents_for_len(req->dst, n_sz), + &cmd->data[RSP_DATA_IDX], n_sz); + if (copied != n_sz) + status = -EINVAL; + +free_work_data: + kfree_sensitive(work_data); + kpp_request_complete(req, status); +} + +/* + * If no private key is provided, generate one in the device and cache + * the corresponding public key. The generated private key never leaves + * the device. + */ +static int atmel_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, + unsigned int len) +{ + struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm); + struct atmel_i2c_cmd *cmd; + void *public_key; + struct ecdh params; + int ret = -ENOMEM; + + kfree(ctx->public_key); + ctx->public_key = NULL; + ctx->do_fallback = false; + + if (crypto_ecdh_decode_key(buf, len, ¶ms) < 0) { + dev_err(&ctx->client->dev, "crypto_ecdh_decode_key failed\n"); + return -EINVAL; + } + + if (params.key_size) { + ret = crypto_kpp_set_secret(ctx->fallback, buf, len); + ctx->do_fallback = !ret; + return ret; + } + + cmd = kmalloc_obj(*cmd); + if (!cmd) + return -ENOMEM; + + public_key = kmalloc(ATMEL_ECC_PUBKEY_SIZE, GFP_KERNEL); + if (!public_key) + goto free_cmd; + + atmel_i2c_init_genkey_cmd(cmd, DATA_SLOT_2); + + ret = atmel_i2c_send_receive(ctx->client, cmd); + if (ret) { + kfree(public_key); + goto free_cmd; + } + + memcpy(public_key, &cmd->data[RSP_DATA_IDX], ATMEL_ECC_PUBKEY_SIZE); + ctx->public_key = public_key; + +free_cmd: + kfree(cmd); + return ret; +} + +static int atmel_ecdh_generate_public_key(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm); + size_t copied, nbytes; + + if (ctx->do_fallback) { + kpp_request_set_tfm(req, ctx->fallback); + return crypto_kpp_generate_public_key(req); + } + + if (!ctx->public_key) + return -EINVAL; + + /* copy only as much as requested, capped at 64 bytes */ + nbytes = min(ATMEL_ECC_PUBKEY_SIZE, req->dst_len); + + /* public key was saved at private key generation */ + copied = sg_copy_from_buffer(req->dst, + sg_nents_for_len(req->dst, nbytes), + ctx->public_key, nbytes); + if (copied != nbytes) + return -EINVAL; + + return 0; +} + +static int atmel_ecdh_compute_shared_secret(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm); + struct atmel_i2c_work_data *work_data; + gfp_t gfp; + int ret; + + if (ctx->do_fallback) { + kpp_request_set_tfm(req, ctx->fallback); + return crypto_kpp_compute_shared_secret(req); + } + + if (!ctx->public_key) + return -EINVAL; + + /* A P-256 public key must contain two 32-byte coordinates */ + if (req->src_len != ATMEL_ECC_PUBKEY_SIZE) + return -EINVAL; + + gfp = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : + GFP_ATOMIC; + + work_data = kmalloc_obj(*work_data, gfp); + if (!work_data) + return -ENOMEM; + + work_data->ctx = ctx; + work_data->client = ctx->client; + + ret = atmel_i2c_init_ecdh_cmd(&work_data->cmd, req->src); + if (ret) + goto free_work_data; + + atmel_i2c_enqueue(work_data, atmel_ecdh_done, req); + + return -EINPROGRESS; + +free_work_data: + kfree(work_data); + return ret; +} + +static struct i2c_client *atmel_ecc_i2c_client_alloc(void) +{ + struct atmel_i2c_client_priv *i2c_priv, *min_i2c_priv = NULL; + struct i2c_client *client = ERR_PTR(-ENODEV); + int min_tfm_cnt = INT_MAX; + int tfm_cnt; + + spin_lock(&driver_data.i2c_list_lock); + + if (list_empty(&driver_data.i2c_client_list)) { + spin_unlock(&driver_data.i2c_list_lock); + return ERR_PTR(-ENODEV); + } + + list_for_each_entry(i2c_priv, &driver_data.i2c_client_list, + i2c_client_list_node) { + tfm_cnt = atomic_read(&i2c_priv->tfm_count); + if (tfm_cnt < min_tfm_cnt) { + min_tfm_cnt = tfm_cnt; + min_i2c_priv = i2c_priv; + } + if (!min_tfm_cnt) + break; + } + + if (min_i2c_priv) { + atomic_inc(&min_i2c_priv->tfm_count); + client = min_i2c_priv->client; + } + + spin_unlock(&driver_data.i2c_list_lock); + + return client; +} + +static void atmel_ecc_i2c_client_free(struct i2c_client *client) +{ + struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); + + atomic_dec(&i2c_priv->tfm_count); +} + +static int atmel_ecdh_init_tfm(struct crypto_kpp *tfm) +{ + const char *alg = kpp_alg_name(tfm); + struct crypto_kpp *fallback; + struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm); + + ctx->client = atmel_ecc_i2c_client_alloc(); + if (IS_ERR(ctx->client)) { + pr_err("tfm - i2c_client binding failed\n"); + return PTR_ERR(ctx->client); + } + + fallback = crypto_alloc_kpp(alg, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(fallback)) { + dev_err(&ctx->client->dev, "Failed to allocate transformation for '%s': %ld\n", + alg, PTR_ERR(fallback)); + atmel_ecc_i2c_client_free(ctx->client); + return PTR_ERR(fallback); + } + + crypto_kpp_set_flags(fallback, crypto_kpp_get_flags(tfm)); + ctx->fallback = fallback; + + return 0; +} + +static void atmel_ecdh_exit_tfm(struct crypto_kpp *tfm) +{ + struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm); + + kfree(ctx->public_key); + crypto_free_kpp(ctx->fallback); + atmel_ecc_i2c_client_free(ctx->client); +} + +static unsigned int atmel_ecdh_max_size(struct crypto_kpp *tfm) +{ + struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm); + + return crypto_kpp_maxsize(ctx->fallback); +} + +static struct kpp_alg atmel_ecdh_nist_p256 = { + .set_secret = atmel_ecdh_set_secret, + .generate_public_key = atmel_ecdh_generate_public_key, + .compute_shared_secret = atmel_ecdh_compute_shared_secret, + .init = atmel_ecdh_init_tfm, + .exit = atmel_ecdh_exit_tfm, + .max_size = atmel_ecdh_max_size, + .base = { + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .cra_name = "ecdh-nist-p256", + .cra_driver_name = "atmel-ecdh", + .cra_priority = ATMEL_ECC_PRIORITY, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct atmel_ecdh_ctx), + }, +}; + +static int atmel_ecc_probe(struct i2c_client *client) +{ + struct atmel_i2c_client_priv *i2c_priv; + int ret; + + ret = atmel_i2c_probe(client); + if (ret) + return ret; + + i2c_priv = i2c_get_clientdata(client); + + spin_lock(&driver_data.i2c_list_lock); + list_add_tail(&i2c_priv->i2c_client_list_node, + &driver_data.i2c_client_list); + spin_unlock(&driver_data.i2c_list_lock); + + ret = crypto_register_kpp(&atmel_ecdh_nist_p256); + if (ret) { + spin_lock(&driver_data.i2c_list_lock); + list_del(&i2c_priv->i2c_client_list_node); + spin_unlock(&driver_data.i2c_list_lock); + + dev_err(&client->dev, "%s alg registration failed\n", + atmel_ecdh_nist_p256.base.cra_driver_name); + } else { + dev_info(&client->dev, "atmel ecc algorithms registered in /proc/crypto\n"); + } + + return ret; +} + +static void atmel_ecc_remove(struct i2c_client *client) +{ + struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); + + /* Return EBUSY if i2c client already allocated. */ + if (atomic_read(&i2c_priv->tfm_count)) { + /* + * After we return here, the memory backing the device is freed. + * That happens no matter what the return value of this function + * is because in the Linux device model there is no error + * handling for unbinding a driver. + * If there is still some action pending, it probably involves + * accessing the freed memory. + */ + dev_emerg(&client->dev, "Device is busy, expect memory corruption.\n"); + return; + } + + crypto_unregister_kpp(&atmel_ecdh_nist_p256); + + spin_lock(&driver_data.i2c_list_lock); + list_del(&i2c_priv->i2c_client_list_node); + spin_unlock(&driver_data.i2c_list_lock); +} + +static const struct of_device_id atmel_ecc_dt_ids[] = { + { .compatible = "atmel,atecc508a", }, + { .compatible = "atmel,atecc608b", }, + { } +}; +MODULE_DEVICE_TABLE(of, atmel_ecc_dt_ids); + +static const struct i2c_device_id atmel_ecc_id[] = { + { .name = "atecc508a" }, + { .name = "atecc608b" }, + { } +}; +MODULE_DEVICE_TABLE(i2c, atmel_ecc_id); + +static struct i2c_driver atmel_ecc_driver = { + .driver = { + .name = "atmel-ecc", + .of_match_table = atmel_ecc_dt_ids, + }, + .probe = atmel_ecc_probe, + .remove = atmel_ecc_remove, + .id_table = atmel_ecc_id, +}; + +static int __init atmel_ecc_init(void) +{ + spin_lock_init(&driver_data.i2c_list_lock); + INIT_LIST_HEAD(&driver_data.i2c_client_list); + return i2c_add_driver(&atmel_ecc_driver); +} + +static void __exit atmel_ecc_exit(void) +{ + atmel_i2c_flush_queue(); + i2c_del_driver(&atmel_ecc_driver); +} + +module_init(atmel_ecc_init); +module_exit(atmel_ecc_exit); + +MODULE_AUTHOR("Tudor Ambarus"); +MODULE_DESCRIPTION("Microchip / Atmel ECC (I2C) driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/crypto/atmel-i2c.c b/drivers/crypto/atmel-i2c.c new file mode 100644 index 000000000..24bded47a --- /dev/null +++ b/drivers/crypto/atmel-i2c.c @@ -0,0 +1,418 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Microchip / Atmel ECC (I2C) driver. + * + * Copyright (c) 2017, Microchip Technology Inc. + * Author: Tudor Ambarus + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "atmel-i2c.h" + +static const struct { + u8 value; + const char *error_text; +} error_list[] = { + { 0x01, "CheckMac or Verify miscompare" }, + { 0x03, "Parse Error" }, + { 0x05, "ECC Fault" }, + { 0x0F, "Execution Error" }, + { 0xEE, "Watchdog about to expire" }, + { 0xFF, "CRC or other communication error" }, +}; + +/** + * atmel_i2c_checksum() - Generate 16-bit CRC as required by ATMEL ECC. + * CRC16 verification of the count, opcode, param1, param2 and data bytes. + * The checksum is saved in little-endian format in the least significant + * two bytes of the command. CRC polynomial is 0x8005 and the initial register + * value should be zero. + * + * @cmd : structure used for communicating with the device. + */ +static void atmel_i2c_checksum(struct atmel_i2c_cmd *cmd) +{ + u8 *data = &cmd->count; + size_t len = cmd->count - CRC_SIZE; + __le16 *__crc16 = (__le16 *)(data + len); + + *__crc16 = cpu_to_le16(bitrev16(crc16(0, data, len))); +} + +void atmel_i2c_init_read_config_cmd(struct atmel_i2c_cmd *cmd) +{ + cmd->word_addr = COMMAND; + cmd->opcode = OPCODE_READ; + /* + * Read the word from Configuration zone that contains the lock bytes + * (UserExtra, Selector, LockValue, LockConfig). + */ + cmd->param1 = CONFIGURATION_ZONE; + cmd->param2 = cpu_to_le16(DEVICE_LOCK_ADDR); + cmd->count = READ_COUNT; + + atmel_i2c_checksum(cmd); + + cmd->msecs = MAX_EXEC_TIME_READ; + cmd->rxsize = READ_RSP_SIZE; +} +EXPORT_SYMBOL(atmel_i2c_init_read_config_cmd); + +int atmel_i2c_init_read_otp_cmd(struct atmel_i2c_cmd *cmd, u16 addr) +{ + if (addr >= OTP_ZONE_SIZE / 4) + return -EINVAL; + + cmd->word_addr = COMMAND; + cmd->opcode = OPCODE_READ; + /* + * Read the word from OTP zone that may contain e.g. serial + * numbers or similar if persistently pre-initialized and locked + */ + cmd->param1 = OTP_ZONE; + cmd->param2 = cpu_to_le16(addr); + cmd->count = READ_COUNT; + + atmel_i2c_checksum(cmd); + + cmd->msecs = MAX_EXEC_TIME_READ; + cmd->rxsize = READ_RSP_SIZE; + + return 0; +} +EXPORT_SYMBOL(atmel_i2c_init_read_otp_cmd); + +void atmel_i2c_init_random_cmd(struct atmel_i2c_cmd *cmd) +{ + cmd->word_addr = COMMAND; + cmd->opcode = OPCODE_RANDOM; + cmd->param1 = 0; + cmd->param2 = 0; + cmd->count = RANDOM_COUNT; + + atmel_i2c_checksum(cmd); + + cmd->msecs = MAX_EXEC_TIME_RANDOM; + cmd->rxsize = RANDOM_RSP_SIZE; +} +EXPORT_SYMBOL(atmel_i2c_init_random_cmd); + +void atmel_i2c_init_genkey_cmd(struct atmel_i2c_cmd *cmd, u16 keyid) +{ + cmd->word_addr = COMMAND; + cmd->count = GENKEY_COUNT; + cmd->opcode = OPCODE_GENKEY; + cmd->param1 = GENKEY_MODE_PRIVATE; + /* a random private key will be generated and stored in slot keyID */ + cmd->param2 = cpu_to_le16(keyid); + + atmel_i2c_checksum(cmd); + + cmd->msecs = MAX_EXEC_TIME_GENKEY; + cmd->rxsize = GENKEY_RSP_SIZE; +} +EXPORT_SYMBOL(atmel_i2c_init_genkey_cmd); + +int atmel_i2c_init_ecdh_cmd(struct atmel_i2c_cmd *cmd, + struct scatterlist *pubkey) +{ + size_t copied; + + cmd->word_addr = COMMAND; + cmd->count = ECDH_COUNT; + cmd->opcode = OPCODE_ECDH; + cmd->param1 = ECDH_PREFIX_MODE; + /* private key slot */ + cmd->param2 = cpu_to_le16(DATA_SLOT_2); + + /* + * The device only supports P-256. Its public key is encoded as + * two 32-byte coordinates. + */ + copied = sg_copy_to_buffer(pubkey, + sg_nents_for_len(pubkey, + ATMEL_ECC_PUBKEY_SIZE), + cmd->data, ATMEL_ECC_PUBKEY_SIZE); + if (copied != ATMEL_ECC_PUBKEY_SIZE) + return -EINVAL; + + atmel_i2c_checksum(cmd); + + cmd->msecs = MAX_EXEC_TIME_ECDH; + cmd->rxsize = ECDH_RSP_SIZE; + + return 0; +} +EXPORT_SYMBOL(atmel_i2c_init_ecdh_cmd); + +/* + * After wake and after execution of a command, there will be error, status, or + * result bytes in the device's output register that can be retrieved by the + * system. When the length of that group is four bytes, the codes returned are + * detailed in error_list. + */ +static int atmel_i2c_status(struct device *dev, u8 *status) +{ + size_t err_list_len = ARRAY_SIZE(error_list); + int i; + u8 err_id = status[1]; + + if (*status != STATUS_SIZE) + return 0; + + if (err_id == STATUS_WAKE_SUCCESSFUL || err_id == STATUS_NOERR) + return 0; + + for (i = 0; i < err_list_len; i++) + if (error_list[i].value == err_id) + break; + + /* if err_id is not in the error_list then ignore it */ + if (i != err_list_len) { + dev_err(dev, "%02x: %s:\n", err_id, error_list[i].error_text); + return err_id; + } + + return 0; +} + +static int atmel_i2c_wakeup(struct i2c_client *client) +{ + struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); + u8 status[STATUS_RSP_SIZE]; + int ret; + + /* + * The device ignores any levels or transitions on the SCL pin when the + * device is idle, asleep or during waking up. Don't check for error + * when waking up the device. + */ + i2c_transfer_buffer_flags(client, i2c_priv->wake_token, + i2c_priv->wake_token_sz, I2C_M_IGNORE_NAK); + + /* + * Wait to wake the device. Typical execution times for ecdh and genkey + * are around tens of milliseconds. Delta is chosen to 50 microseconds. + */ + usleep_range(TWHI_MIN, TWHI_MAX); + + ret = i2c_master_recv(client, status, STATUS_SIZE); + if (ret < 0) + return ret; + + return atmel_i2c_status(&client->dev, status); +} + +static int atmel_i2c_sleep(struct i2c_client *client) +{ + u8 sleep = SLEEP_TOKEN; + + return i2c_master_send(client, &sleep, 1); +} + +/* + * atmel_i2c_send_receive() - send a command to the device and receive its + * response. + * @client: i2c client device + * @cmd : structure used to communicate with the device + * + * After the device receives a Wake token, a watchdog counter starts within the + * device. After the watchdog timer expires, the device enters sleep mode + * regardless of whether some I/O transmission or command execution is in + * progress. If a command is attempted when insufficient time remains prior to + * watchdog timer execution, the device will return the watchdog timeout error + * code without attempting to execute the command. There is no way to reset the + * counter other than to put the device into sleep or idle mode and then + * wake it up again. + */ +int atmel_i2c_send_receive(struct i2c_client *client, struct atmel_i2c_cmd *cmd) +{ + struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); + int ret; + + mutex_lock(&i2c_priv->lock); + + ret = atmel_i2c_wakeup(client); + if (ret) + goto err; + + /* send the command */ + ret = i2c_master_send(client, (u8 *)cmd, cmd->count + WORD_ADDR_SIZE); + if (ret < 0) + goto err; + + /* delay the appropriate amount of time for command to execute */ + msleep(cmd->msecs); + + /* receive the response */ + ret = i2c_master_recv(client, cmd->data, cmd->rxsize); + if (ret < 0) + goto err; + + /* put the device into low-power mode */ + ret = atmel_i2c_sleep(client); + if (ret < 0) + goto err; + + mutex_unlock(&i2c_priv->lock); + return atmel_i2c_status(&client->dev, cmd->data); +err: + mutex_unlock(&i2c_priv->lock); + return ret; +} +EXPORT_SYMBOL(atmel_i2c_send_receive); + +static void atmel_i2c_work_handler(struct work_struct *work) +{ + struct atmel_i2c_work_data *work_data = + container_of(work, struct atmel_i2c_work_data, work); + struct atmel_i2c_cmd *cmd = &work_data->cmd; + struct i2c_client *client = work_data->client; + int status; + + status = atmel_i2c_send_receive(client, cmd); + work_data->cbk(work_data, work_data->areq, status); +} + +static struct workqueue_struct *atmel_wq; + +void atmel_i2c_enqueue(struct atmel_i2c_work_data *work_data, + void (*cbk)(struct atmel_i2c_work_data *work_data, + void *areq, int status), + void *areq) +{ + work_data->cbk = cbk; + work_data->areq = areq; + + INIT_WORK(&work_data->work, atmel_i2c_work_handler); + queue_work(atmel_wq, &work_data->work); +} +EXPORT_SYMBOL(atmel_i2c_enqueue); + +void atmel_i2c_flush_queue(void) +{ + flush_workqueue(atmel_wq); +} +EXPORT_SYMBOL(atmel_i2c_flush_queue); + +static inline size_t atmel_i2c_wake_token_sz(u32 bus_clk_rate) +{ + u32 no_of_bits = DIV_ROUND_UP(TWLO_USEC * bus_clk_rate, USEC_PER_SEC); + + /* return the size of the wake_token in bytes */ + return DIV_ROUND_UP(no_of_bits, 8); +} + +static int device_sanity_check(struct i2c_client *client) +{ + struct atmel_i2c_cmd *cmd; + int ret; + + cmd = kmalloc_obj(*cmd); + if (!cmd) + return -ENOMEM; + + atmel_i2c_init_read_config_cmd(cmd); + + ret = atmel_i2c_send_receive(client, cmd); + if (ret) + goto free_cmd; + + /* + * It is vital that the Configuration, Data and OTP zones be locked + * prior to release into the field of the system containing the device. + * Failure to lock these zones may permit modification of any secret + * keys and may lead to other security problems. + */ + if (cmd->data[LOCK_CONFIG_IDX] || cmd->data[LOCK_VALUE_IDX]) { + dev_err(&client->dev, "Configuration or Data and OTP zones are unlocked!\n"); + ret = -ENOTSUPP; + } + + /* fall through */ +free_cmd: + kfree(cmd); + return ret; +} + +int atmel_i2c_probe(struct i2c_client *client) +{ + struct atmel_i2c_client_priv *i2c_priv; + struct device *dev = &client->dev; + int ret; + u32 bus_clk_rate; + + if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) { + dev_err(dev, "I2C_FUNC_I2C not supported\n"); + return -ENODEV; + } + + bus_clk_rate = i2c_acpi_find_bus_speed(&client->adapter->dev); + if (!bus_clk_rate) { + ret = device_property_read_u32(&client->adapter->dev, + "clock-frequency", &bus_clk_rate); + if (ret) { + dev_err(dev, "failed to read clock-frequency property\n"); + return ret; + } + } + + if (bus_clk_rate > I2C_MAX_FAST_MODE_PLUS_FREQ) { + dev_err(dev, "%u exceeds maximum supported clock frequency (1MHz)\n", + bus_clk_rate); + return -EINVAL; + } + + i2c_priv = devm_kmalloc(dev, sizeof(*i2c_priv), GFP_KERNEL); + if (!i2c_priv) + return -ENOMEM; + + i2c_priv->client = client; + mutex_init(&i2c_priv->lock); + + /* + * WAKE_TOKEN_MAX_SIZE was calculated for the maximum bus_clk_rate - + * 1MHz. The previous bus_clk_rate check ensures us that wake_token_sz + * will always be smaller than or equal to WAKE_TOKEN_MAX_SIZE. + */ + i2c_priv->wake_token_sz = atmel_i2c_wake_token_sz(bus_clk_rate); + + memset(i2c_priv->wake_token, 0, sizeof(i2c_priv->wake_token)); + + atomic_set(&i2c_priv->tfm_count, 0); + + i2c_set_clientdata(client, i2c_priv); + + return device_sanity_check(client); +} +EXPORT_SYMBOL(atmel_i2c_probe); + +static int __init atmel_i2c_init(void) +{ + atmel_wq = alloc_workqueue("atmel_wq", WQ_PERCPU, 0); + return atmel_wq ? 0 : -ENOMEM; +} + +static void __exit atmel_i2c_exit(void) +{ + destroy_workqueue(atmel_wq); +} + +module_init(atmel_i2c_init); +module_exit(atmel_i2c_exit); + +MODULE_AUTHOR("Tudor Ambarus"); +MODULE_DESCRIPTION("Microchip / Atmel ECC (I2C) driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/crypto/atmel-i2c.h b/drivers/crypto/atmel-i2c.h new file mode 100644 index 000000000..72f04c156 --- /dev/null +++ b/drivers/crypto/atmel-i2c.h @@ -0,0 +1,192 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2017, Microchip Technology Inc. + * Author: Tudor Ambarus + */ + +#ifndef __ATMEL_I2C_H__ +#define __ATMEL_I2C_H__ + +#include +#include + +#define ATMEL_ECC_PRIORITY 300 + +#define COMMAND 0x03 /* packet function */ +#define SLEEP_TOKEN 0x01 +#define WAKE_TOKEN_MAX_SIZE 8 + +/* Definitions of Data and Command sizes */ +#define WORD_ADDR_SIZE 1 +#define COUNT_SIZE 1 +#define CRC_SIZE 2 +#define CMD_OVERHEAD_SIZE (COUNT_SIZE + CRC_SIZE) + +/* size in bytes of the n prime */ +#define ATMEL_ECC_NIST_P256_N_SIZE 32 +#define ATMEL_ECC_PUBKEY_SIZE (2 * ATMEL_ECC_NIST_P256_N_SIZE) + +#define STATUS_RSP_SIZE 4 +#define ECDH_RSP_SIZE (32 + CMD_OVERHEAD_SIZE) +#define GENKEY_RSP_SIZE (ATMEL_ECC_PUBKEY_SIZE + \ + CMD_OVERHEAD_SIZE) +#define READ_RSP_SIZE (4 + CMD_OVERHEAD_SIZE) +#define RANDOM_RSP_SIZE (32 + CMD_OVERHEAD_SIZE) +#define MAX_RSP_SIZE GENKEY_RSP_SIZE + +/** + * atmel_i2c_cmd - structure used for communicating with the device. + * @word_addr: indicates the function of the packet sent to the device. This + * byte should have a value of COMMAND for normal operation. + * @count : number of bytes to be transferred to (or from) the device. + * @opcode : the command code. + * @param1 : the first parameter; always present. + * @param2 : the second parameter; always present. + * @data : optional remaining input data. Includes a 2-byte CRC. + * @rxsize : size of the data received from i2c client. + * @msecs : command execution time in milliseconds + */ +struct atmel_i2c_cmd { + u8 word_addr; + u8 count; + u8 opcode; + u8 param1; + __le16 param2; + u8 data[MAX_RSP_SIZE]; + u8 msecs; + u16 rxsize; +} __packed; + +/* Status/Error codes */ +#define STATUS_SIZE 0x04 +#define STATUS_NOERR 0x00 +#define STATUS_WAKE_SUCCESSFUL 0x11 + +/* Definitions for eeprom organization */ +#define CONFIGURATION_ZONE 0 +#define OTP_ZONE 1 + +/* Definitions for eeprom zone sizes */ +#define OTP_ZONE_SIZE 64 + +/* Definitions for Indexes common to all commands */ +#define RSP_DATA_IDX 1 /* buffer index of data in response */ +#define DATA_SLOT_2 2 /* used for ECDH private key */ + +/* Definitions for the device lock state */ +#define DEVICE_LOCK_ADDR 0x15 +#define LOCK_VALUE_IDX (RSP_DATA_IDX + 2) +#define LOCK_CONFIG_IDX (RSP_DATA_IDX + 3) + +/* + * Wake High delay to data communication (microseconds). SDA should be stable + * high for this entire duration. + */ +#define TWHI_MIN 1500 +#define TWHI_MAX 1550 + +/* Wake Low duration */ +#define TWLO_USEC 60 + +/* Command execution time (milliseconds) */ +#define MAX_EXEC_TIME_ECDH 58 +#define MAX_EXEC_TIME_GENKEY 115 +#define MAX_EXEC_TIME_READ 1 +#define MAX_EXEC_TIME_RANDOM 50 + +/* Command opcode */ +#define OPCODE_ECDH 0x43 +#define OPCODE_GENKEY 0x40 +#define OPCODE_READ 0x02 +#define OPCODE_RANDOM 0x1b + +/* Definitions for the READ Command */ +#define READ_COUNT 7 + +/* Definitions for the RANDOM Command */ +#define RANDOM_COUNT 7 + +/* Definitions for the GenKey Command */ +#define GENKEY_COUNT 7 +#define GENKEY_MODE_PRIVATE 0x04 + +/* Definitions for the ECDH Command */ +#define ECDH_COUNT 71 +#define ECDH_PREFIX_MODE 0x00 + +/* Used for binding tfm objects to i2c clients. */ +struct atmel_ecc_driver_data { + struct list_head i2c_client_list; + spinlock_t i2c_list_lock; +} ____cacheline_aligned; + +/** + * atmel_i2c_client_priv - i2c_client private data + * @client : pointer to i2c client device + * @i2c_client_list_node: part of i2c_client_list + * @lock : lock for sending i2c commands + * @wake_token : wake token array of zeros + * @wake_token_sz : size in bytes of the wake_token + * @tfm_count : number of active crypto transformations on i2c client + * @hwrng : hold the hardware generated rng + * + * Reads and writes from/to the i2c client are sequential. The first byte + * transmitted to the device is treated as the byte size. Any attempt to send + * more than this number of bytes will cause the device to not ACK those bytes. + * After the host writes a single command byte to the input buffer, reads are + * prohibited until after the device completes command execution. Use a mutex + * when sending i2c commands. + */ +struct atmel_i2c_client_priv { + struct i2c_client *client; + struct list_head i2c_client_list_node; + struct mutex lock; + u8 wake_token[WAKE_TOKEN_MAX_SIZE]; + size_t wake_token_sz; + atomic_t tfm_count ____cacheline_aligned; + struct hwrng hwrng; +}; + +/** + * atmel_i2c_work_data - data structure representing the work + * @ctx : transformation context. + * @cbk : pointer to a callback function to be invoked upon completion of this + * request. This has the form: + * callback(struct atmel_i2c_work_data *work_data, void *areq, u8 status) + * where: + * @work_data: data structure representing the work + * @areq : optional pointer to an argument passed with the original + * request. + * @status : status returned from the i2c client device or i2c error. + * @areq: optional pointer to a user argument for use at callback time. + * @work: describes the task to be executed. + * @cmd : structure used for communicating with the device. + */ +struct atmel_i2c_work_data { + void *ctx; + struct i2c_client *client; + void (*cbk)(struct atmel_i2c_work_data *work_data, void *areq, + int status); + void *areq; + struct work_struct work; + struct atmel_i2c_cmd cmd; +}; + +int atmel_i2c_probe(struct i2c_client *client); + +void atmel_i2c_enqueue(struct atmel_i2c_work_data *work_data, + void (*cbk)(struct atmel_i2c_work_data *work_data, + void *areq, int status), + void *areq); +void atmel_i2c_flush_queue(void); + +int atmel_i2c_send_receive(struct i2c_client *client, struct atmel_i2c_cmd *cmd); + +void atmel_i2c_init_read_config_cmd(struct atmel_i2c_cmd *cmd); +int atmel_i2c_init_read_otp_cmd(struct atmel_i2c_cmd *cmd, u16 addr); +void atmel_i2c_init_random_cmd(struct atmel_i2c_cmd *cmd); +void atmel_i2c_init_genkey_cmd(struct atmel_i2c_cmd *cmd, u16 keyid); +int atmel_i2c_init_ecdh_cmd(struct atmel_i2c_cmd *cmd, + struct scatterlist *pubkey); + +#endif /* __ATMEL_I2C_H__ */ diff --git a/drivers/crypto/atmel-sha-regs.h b/drivers/crypto/atmel-sha-regs.h new file mode 100644 index 000000000..b2b5e634e --- /dev/null +++ b/drivers/crypto/atmel-sha-regs.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ATMEL_SHA_REGS_H__ +#define __ATMEL_SHA_REGS_H__ + +#define SHA_REG_DIGEST(x) (0x80 + ((x) * 0x04)) +#define SHA_REG_DIN(x) (0x40 + ((x) * 0x04)) + +#define SHA_CR 0x00 +#define SHA_CR_START (1 << 0) +#define SHA_CR_FIRST (1 << 4) +#define SHA_CR_SWRST (1 << 8) +#define SHA_CR_WUIHV (1 << 12) +#define SHA_CR_WUIEHV (1 << 13) + +#define SHA_MR 0x04 +#define SHA_MR_MODE_MASK (0x3 << 0) +#define SHA_MR_MODE_MANUAL 0x0 +#define SHA_MR_MODE_AUTO 0x1 +#define SHA_MR_MODE_PDC 0x2 +#define SHA_MR_MODE_IDATAR0 0x2 +#define SHA_MR_PROCDLY (1 << 4) +#define SHA_MR_UIHV (1 << 5) +#define SHA_MR_UIEHV (1 << 6) +#define SHA_MR_ALGO_MASK GENMASK(10, 8) +#define SHA_MR_ALGO_SHA1 (0 << 8) +#define SHA_MR_ALGO_SHA256 (1 << 8) +#define SHA_MR_ALGO_SHA384 (2 << 8) +#define SHA_MR_ALGO_SHA512 (3 << 8) +#define SHA_MR_ALGO_SHA224 (4 << 8) +#define SHA_MR_HMAC (1 << 11) +#define SHA_MR_DUALBUFF (1 << 16) + +#define SHA_FLAGS_ALGO_MASK SHA_MR_ALGO_MASK +#define SHA_FLAGS_SHA1 SHA_MR_ALGO_SHA1 +#define SHA_FLAGS_SHA256 SHA_MR_ALGO_SHA256 +#define SHA_FLAGS_SHA384 SHA_MR_ALGO_SHA384 +#define SHA_FLAGS_SHA512 SHA_MR_ALGO_SHA512 +#define SHA_FLAGS_SHA224 SHA_MR_ALGO_SHA224 +#define SHA_FLAGS_HMAC SHA_MR_HMAC +#define SHA_FLAGS_HMAC_SHA1 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA1) +#define SHA_FLAGS_HMAC_SHA256 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA256) +#define SHA_FLAGS_HMAC_SHA384 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA384) +#define SHA_FLAGS_HMAC_SHA512 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA512) +#define SHA_FLAGS_HMAC_SHA224 (SHA_FLAGS_HMAC | SHA_FLAGS_SHA224) +#define SHA_FLAGS_MODE_MASK (SHA_FLAGS_HMAC | SHA_FLAGS_ALGO_MASK) + +#define SHA_IER 0x10 +#define SHA_IDR 0x14 +#define SHA_IMR 0x18 +#define SHA_ISR 0x1C +#define SHA_INT_DATARDY (1 << 0) +#define SHA_INT_ENDTX (1 << 1) +#define SHA_INT_TXBUFE (1 << 2) +#define SHA_INT_URAD (1 << 8) +#define SHA_ISR_URAT_MASK (0x7 << 12) +#define SHA_ISR_URAT_IDR (0x0 << 12) +#define SHA_ISR_URAT_ODR (0x1 << 12) +#define SHA_ISR_URAT_MR (0x2 << 12) +#define SHA_ISR_URAT_WO (0x5 << 12) + +#define SHA_MSR 0x20 +#define SHA_BCR 0x30 + +#define SHA_HW_VERSION 0xFC + +#define SHA_TPR 0x108 +#define SHA_TCR 0x10C +#define SHA_TNPR 0x118 +#define SHA_TNCR 0x11C +#define SHA_PTCR 0x120 +#define SHA_PTCR_TXTEN (1 << 8) +#define SHA_PTCR_TXTDIS (1 << 9) +#define SHA_PTSR 0x124 +#define SHA_PTSR_TXTEN (1 << 8) + +#endif /* __ATMEL_SHA_REGS_H__ */ diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c new file mode 100644 index 000000000..48f54b615 --- /dev/null +++ b/drivers/crypto/atmel-sha.c @@ -0,0 +1,2690 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Cryptographic API. + * + * Support for ATMEL SHA1/SHA256 HW acceleration. + * + * Copyright (c) 2012 Eukréa Electromatique - ATMEL + * Author: Nicolas Royer + * + * Some ideas are from omap-sham.c drivers. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "atmel-sha-regs.h" +#include "atmel-authenc.h" + +#define ATMEL_SHA_PRIORITY 300 + +/* SHA flags */ +#define SHA_FLAGS_BUSY BIT(0) +#define SHA_FLAGS_FINAL BIT(1) +#define SHA_FLAGS_DMA_ACTIVE BIT(2) +#define SHA_FLAGS_OUTPUT_READY BIT(3) +#define SHA_FLAGS_INIT BIT(4) +#define SHA_FLAGS_CPU BIT(5) +#define SHA_FLAGS_DMA_READY BIT(6) +#define SHA_FLAGS_DUMP_REG BIT(7) + +/* bits[11:8] are reserved. */ + +#define SHA_FLAGS_FINUP BIT(16) +#define SHA_FLAGS_SG BIT(17) +#define SHA_FLAGS_ERROR BIT(23) +#define SHA_FLAGS_PAD BIT(24) +#define SHA_FLAGS_RESTORE BIT(25) +#define SHA_FLAGS_IDATAR0 BIT(26) +#define SHA_FLAGS_WAIT_DATARDY BIT(27) + +#define SHA_OP_INIT 0 +#define SHA_OP_UPDATE 1 +#define SHA_OP_FINAL 2 +#define SHA_OP_DIGEST 3 + +#define SHA_BUFFER_LEN (PAGE_SIZE / 16) + +#define ATMEL_SHA_DMA_THRESHOLD 56 + +struct atmel_sha_caps { + bool has_dma; + bool has_dualbuff; + bool has_sha224; + bool has_sha_384_512; + bool has_uihv; + bool has_hmac; +}; + +struct atmel_sha_dev; + +/* + * .statesize = sizeof(struct atmel_sha_reqctx) must be <= PAGE_SIZE / 8 as + * tested by the ahash_prepare_alg() function. + */ +struct atmel_sha_reqctx { + struct atmel_sha_dev *dd; + unsigned long flags; + unsigned long op; + + u8 digest[SHA512_DIGEST_SIZE] __aligned(sizeof(u32)); + u64 digcnt[2]; + size_t bufcnt; + size_t buflen; + dma_addr_t dma_addr; + + /* walk state */ + struct scatterlist *sg; + unsigned int offset; /* offset in current sg */ + unsigned int total; /* total request */ + + size_t block_size; + size_t hash_size; + + u8 buffer[SHA_BUFFER_LEN + SHA512_BLOCK_SIZE] __aligned(sizeof(u32)); +}; + +typedef int (*atmel_sha_fn_t)(struct atmel_sha_dev *); + +struct atmel_sha_ctx { + struct atmel_sha_dev *dd; + atmel_sha_fn_t start; + + unsigned long flags; +}; + +#define ATMEL_SHA_QUEUE_LENGTH 50 + +struct atmel_sha_dma { + struct dma_chan *chan; + struct dma_slave_config dma_conf; + struct scatterlist *sg; + int nents; + unsigned int last_sg_length; +}; + +struct atmel_sha_dev { + struct list_head list; + unsigned long phys_base; + struct device *dev; + struct clk *iclk; + int irq; + void __iomem *io_base; + + spinlock_t lock; + struct tasklet_struct done_task; + struct tasklet_struct queue_task; + + unsigned long flags; + struct crypto_queue queue; + struct ahash_request *req; + bool is_async; + bool force_complete; + atmel_sha_fn_t resume; + atmel_sha_fn_t cpu_transfer_complete; + + struct atmel_sha_dma dma_lch_in; + + struct atmel_sha_caps caps; + + struct scatterlist tmp; + + u32 hw_version; +}; + +struct atmel_sha_drv { + struct list_head dev_list; + spinlock_t lock; +}; + +static struct atmel_sha_drv atmel_sha = { + .dev_list = LIST_HEAD_INIT(atmel_sha.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(atmel_sha.lock), +}; + +#ifdef VERBOSE_DEBUG +static const char *atmel_sha_reg_name(u32 offset, char *tmp, size_t sz, bool wr) +{ + switch (offset) { + case SHA_CR: + return "CR"; + + case SHA_MR: + return "MR"; + + case SHA_IER: + return "IER"; + + case SHA_IDR: + return "IDR"; + + case SHA_IMR: + return "IMR"; + + case SHA_ISR: + return "ISR"; + + case SHA_MSR: + return "MSR"; + + case SHA_BCR: + return "BCR"; + + case SHA_REG_DIN(0): + case SHA_REG_DIN(1): + case SHA_REG_DIN(2): + case SHA_REG_DIN(3): + case SHA_REG_DIN(4): + case SHA_REG_DIN(5): + case SHA_REG_DIN(6): + case SHA_REG_DIN(7): + case SHA_REG_DIN(8): + case SHA_REG_DIN(9): + case SHA_REG_DIN(10): + case SHA_REG_DIN(11): + case SHA_REG_DIN(12): + case SHA_REG_DIN(13): + case SHA_REG_DIN(14): + case SHA_REG_DIN(15): + snprintf(tmp, sz, "IDATAR[%u]", (offset - SHA_REG_DIN(0)) >> 2); + break; + + case SHA_REG_DIGEST(0): + case SHA_REG_DIGEST(1): + case SHA_REG_DIGEST(2): + case SHA_REG_DIGEST(3): + case SHA_REG_DIGEST(4): + case SHA_REG_DIGEST(5): + case SHA_REG_DIGEST(6): + case SHA_REG_DIGEST(7): + case SHA_REG_DIGEST(8): + case SHA_REG_DIGEST(9): + case SHA_REG_DIGEST(10): + case SHA_REG_DIGEST(11): + case SHA_REG_DIGEST(12): + case SHA_REG_DIGEST(13): + case SHA_REG_DIGEST(14): + case SHA_REG_DIGEST(15): + if (wr) + snprintf(tmp, sz, "IDATAR[%u]", + 16u + ((offset - SHA_REG_DIGEST(0)) >> 2)); + else + snprintf(tmp, sz, "ODATAR[%u]", + (offset - SHA_REG_DIGEST(0)) >> 2); + break; + + case SHA_HW_VERSION: + return "HWVER"; + + default: + snprintf(tmp, sz, "0x%02x", offset); + break; + } + + return tmp; +} + +#endif /* VERBOSE_DEBUG */ + +static inline u32 atmel_sha_read(struct atmel_sha_dev *dd, u32 offset) +{ + u32 value = readl_relaxed(dd->io_base + offset); + +#ifdef VERBOSE_DEBUG + if (dd->flags & SHA_FLAGS_DUMP_REG) { + char tmp[16]; + + dev_vdbg(dd->dev, "read 0x%08x from %s\n", value, + atmel_sha_reg_name(offset, tmp, sizeof(tmp), false)); + } +#endif /* VERBOSE_DEBUG */ + + return value; +} + +static inline void atmel_sha_write(struct atmel_sha_dev *dd, + u32 offset, u32 value) +{ +#ifdef VERBOSE_DEBUG + if (dd->flags & SHA_FLAGS_DUMP_REG) { + char tmp[16]; + + dev_vdbg(dd->dev, "write 0x%08x into %s\n", value, + atmel_sha_reg_name(offset, tmp, sizeof(tmp), true)); + } +#endif /* VERBOSE_DEBUG */ + + writel_relaxed(value, dd->io_base + offset); +} + +static inline int atmel_sha_complete(struct atmel_sha_dev *dd, int err) +{ + struct ahash_request *req = dd->req; + + dd->flags &= ~(SHA_FLAGS_BUSY | SHA_FLAGS_FINAL | SHA_FLAGS_CPU | + SHA_FLAGS_DMA_READY | SHA_FLAGS_OUTPUT_READY | + SHA_FLAGS_DUMP_REG); + + clk_disable(dd->iclk); + + if ((dd->is_async || dd->force_complete) && req->base.complete) + ahash_request_complete(req, err); + + /* handle new request */ + tasklet_schedule(&dd->queue_task); + + return err; +} + +static size_t atmel_sha_append_sg(struct atmel_sha_reqctx *ctx) +{ + size_t count; + + while ((ctx->bufcnt < ctx->buflen) && ctx->total) { + count = min3(ctx->sg->length - ctx->offset, ctx->total, + ctx->buflen - ctx->bufcnt); + + if (count == 0) { + /* + * Check if count == 0 because the buffer is full or + * because the sg length is 0. In the latest case, + * check if there is another sg in the list, a 0 length + * sg doesn't necessarily mean the end of the sg list. + */ + if ((ctx->sg->length == 0) && !sg_is_last(ctx->sg)) { + ctx->sg = sg_next(ctx->sg); + continue; + } else { + break; + } + } + + scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, ctx->sg, + ctx->offset, count, 0); + + ctx->bufcnt += count; + ctx->offset += count; + ctx->total -= count; + + if (ctx->offset == ctx->sg->length) { + ctx->sg = sg_next(ctx->sg); + if (ctx->sg) + ctx->offset = 0; + else + ctx->total = 0; + } + } + + return 0; +} + +/* + * The purpose of this padding is to ensure that the padded message is a + * multiple of 512 bits (SHA1/SHA224/SHA256) or 1024 bits (SHA384/SHA512). + * The bit "1" is appended at the end of the message followed by + * "padlen-1" zero bits. Then a 64 bits block (SHA1/SHA224/SHA256) or + * 128 bits block (SHA384/SHA512) equals to the message length in bits + * is appended. + * + * For SHA1/SHA224/SHA256, padlen is calculated as followed: + * - if message length < 56 bytes then padlen = 56 - message length + * - else padlen = 64 + 56 - message length + * + * For SHA384/SHA512, padlen is calculated as followed: + * - if message length < 112 bytes then padlen = 112 - message length + * - else padlen = 128 + 112 - message length + */ +static void atmel_sha_fill_padding(struct atmel_sha_reqctx *ctx, int length) +{ + unsigned int index, padlen; + __be64 bits[2]; + u64 size[2]; + + size[0] = ctx->digcnt[0]; + size[1] = ctx->digcnt[1]; + + size[0] += ctx->bufcnt; + if (size[0] < ctx->bufcnt) + size[1]++; + + size[0] += length; + if (size[0] < length) + size[1]++; + + bits[1] = cpu_to_be64(size[0] << 3); + bits[0] = cpu_to_be64(size[1] << 3 | size[0] >> 61); + + switch (ctx->flags & SHA_FLAGS_ALGO_MASK) { + case SHA_FLAGS_SHA384: + case SHA_FLAGS_SHA512: + index = ctx->bufcnt & 0x7f; + padlen = (index < 112) ? (112 - index) : ((128+112) - index); + *(ctx->buffer + ctx->bufcnt) = 0x80; + memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen-1); + memcpy(ctx->buffer + ctx->bufcnt + padlen, bits, 16); + ctx->bufcnt += padlen + 16; + ctx->flags |= SHA_FLAGS_PAD; + break; + + default: + index = ctx->bufcnt & 0x3f; + padlen = (index < 56) ? (56 - index) : ((64+56) - index); + *(ctx->buffer + ctx->bufcnt) = 0x80; + memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen-1); + memcpy(ctx->buffer + ctx->bufcnt + padlen, &bits[1], 8); + ctx->bufcnt += padlen + 8; + ctx->flags |= SHA_FLAGS_PAD; + break; + } +} + +static struct atmel_sha_dev *atmel_sha_find_dev(struct atmel_sha_ctx *tctx) +{ + struct atmel_sha_dev *dd; + + spin_lock_bh(&atmel_sha.lock); + if (!tctx->dd) + tctx->dd = list_first_entry_or_null(&atmel_sha.dev_list, + struct atmel_sha_dev, list); + dd = tctx->dd; + spin_unlock_bh(&atmel_sha.lock); + + return dd; +} + +static int atmel_sha_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct atmel_sha_ctx *tctx = crypto_ahash_ctx(tfm); + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + struct atmel_sha_dev *dd = atmel_sha_find_dev(tctx); + + ctx->dd = dd; + + ctx->flags = 0; + + dev_dbg(dd->dev, "init: digest size: %u\n", + crypto_ahash_digestsize(tfm)); + + switch (crypto_ahash_digestsize(tfm)) { + case SHA1_DIGEST_SIZE: + ctx->flags |= SHA_FLAGS_SHA1; + ctx->block_size = SHA1_BLOCK_SIZE; + break; + case SHA224_DIGEST_SIZE: + ctx->flags |= SHA_FLAGS_SHA224; + ctx->block_size = SHA224_BLOCK_SIZE; + break; + case SHA256_DIGEST_SIZE: + ctx->flags |= SHA_FLAGS_SHA256; + ctx->block_size = SHA256_BLOCK_SIZE; + break; + case SHA384_DIGEST_SIZE: + ctx->flags |= SHA_FLAGS_SHA384; + ctx->block_size = SHA384_BLOCK_SIZE; + break; + case SHA512_DIGEST_SIZE: + ctx->flags |= SHA_FLAGS_SHA512; + ctx->block_size = SHA512_BLOCK_SIZE; + break; + default: + return -EINVAL; + } + + ctx->bufcnt = 0; + ctx->digcnt[0] = 0; + ctx->digcnt[1] = 0; + ctx->buflen = SHA_BUFFER_LEN; + + return 0; +} + +static void atmel_sha_write_ctrl(struct atmel_sha_dev *dd, int dma) +{ + struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); + u32 valmr = SHA_MR_MODE_AUTO; + unsigned int i, hashsize = 0; + + if (likely(dma)) { + if (!dd->caps.has_dma) + atmel_sha_write(dd, SHA_IER, SHA_INT_TXBUFE); + valmr = SHA_MR_MODE_PDC; + if (dd->caps.has_dualbuff) + valmr |= SHA_MR_DUALBUFF; + } else { + atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY); + } + + switch (ctx->flags & SHA_FLAGS_ALGO_MASK) { + case SHA_FLAGS_SHA1: + valmr |= SHA_MR_ALGO_SHA1; + hashsize = SHA1_DIGEST_SIZE; + break; + + case SHA_FLAGS_SHA224: + valmr |= SHA_MR_ALGO_SHA224; + hashsize = SHA256_DIGEST_SIZE; + break; + + case SHA_FLAGS_SHA256: + valmr |= SHA_MR_ALGO_SHA256; + hashsize = SHA256_DIGEST_SIZE; + break; + + case SHA_FLAGS_SHA384: + valmr |= SHA_MR_ALGO_SHA384; + hashsize = SHA512_DIGEST_SIZE; + break; + + case SHA_FLAGS_SHA512: + valmr |= SHA_MR_ALGO_SHA512; + hashsize = SHA512_DIGEST_SIZE; + break; + + default: + break; + } + + /* Setting CR_FIRST only for the first iteration */ + if (!(ctx->digcnt[0] || ctx->digcnt[1])) { + atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST); + } else if (dd->caps.has_uihv && (ctx->flags & SHA_FLAGS_RESTORE)) { + const u32 *hash = (const u32 *)ctx->digest; + + /* + * Restore the hardware context: update the User Initialize + * Hash Value (UIHV) with the value saved when the latest + * 'update' operation completed on this very same crypto + * request. + */ + ctx->flags &= ~SHA_FLAGS_RESTORE; + atmel_sha_write(dd, SHA_CR, SHA_CR_WUIHV); + for (i = 0; i < hashsize / sizeof(u32); ++i) + atmel_sha_write(dd, SHA_REG_DIN(i), hash[i]); + atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST); + valmr |= SHA_MR_UIHV; + } + /* + * WARNING: If the UIHV feature is not available, the hardware CANNOT + * process concurrent requests: the internal registers used to store + * the hash/digest are still set to the partial digest output values + * computed during the latest round. + */ + + atmel_sha_write(dd, SHA_MR, valmr); +} + +static inline int atmel_sha_wait_for_data_ready(struct atmel_sha_dev *dd, + atmel_sha_fn_t resume) +{ + u32 isr = atmel_sha_read(dd, SHA_ISR); + + if (unlikely(isr & SHA_INT_DATARDY)) + return resume(dd); + + dd->resume = resume; + atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY); + return -EINPROGRESS; +} + +static int atmel_sha_xmit_cpu(struct atmel_sha_dev *dd, const u8 *buf, + size_t length, int final) +{ + struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); + int count, len32; + const u32 *buffer = (const u32 *)buf; + + dev_dbg(dd->dev, "xmit_cpu: digcnt: 0x%llx 0x%llx, length: %zd, final: %d\n", + ctx->digcnt[1], ctx->digcnt[0], length, final); + + atmel_sha_write_ctrl(dd, 0); + + /* should be non-zero before next lines to disable clocks later */ + ctx->digcnt[0] += length; + if (ctx->digcnt[0] < length) + ctx->digcnt[1]++; + + if (final) + dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */ + + len32 = DIV_ROUND_UP(length, sizeof(u32)); + + dd->flags |= SHA_FLAGS_CPU; + + for (count = 0; count < len32; count++) + atmel_sha_write(dd, SHA_REG_DIN(count), buffer[count]); + + return -EINPROGRESS; +} + +static int atmel_sha_xmit_pdc(struct atmel_sha_dev *dd, dma_addr_t dma_addr1, + size_t length1, dma_addr_t dma_addr2, size_t length2, int final) +{ + struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); + int len32; + + dev_dbg(dd->dev, "xmit_pdc: digcnt: 0x%llx 0x%llx, length: %zd, final: %d\n", + ctx->digcnt[1], ctx->digcnt[0], length1, final); + + len32 = DIV_ROUND_UP(length1, sizeof(u32)); + atmel_sha_write(dd, SHA_PTCR, SHA_PTCR_TXTDIS); + atmel_sha_write(dd, SHA_TPR, dma_addr1); + atmel_sha_write(dd, SHA_TCR, len32); + + len32 = DIV_ROUND_UP(length2, sizeof(u32)); + atmel_sha_write(dd, SHA_TNPR, dma_addr2); + atmel_sha_write(dd, SHA_TNCR, len32); + + atmel_sha_write_ctrl(dd, 1); + + /* should be non-zero before next lines to disable clocks later */ + ctx->digcnt[0] += length1; + if (ctx->digcnt[0] < length1) + ctx->digcnt[1]++; + + if (final) + dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */ + + dd->flags |= SHA_FLAGS_DMA_ACTIVE; + + /* Start DMA transfer */ + atmel_sha_write(dd, SHA_PTCR, SHA_PTCR_TXTEN); + + return -EINPROGRESS; +} + +static void atmel_sha_dma_callback(void *data) +{ + struct atmel_sha_dev *dd = data; + + dd->is_async = true; + + /* dma_lch_in - completed - wait DATRDY */ + atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY); +} + +static int atmel_sha_xmit_dma(struct atmel_sha_dev *dd, dma_addr_t dma_addr1, + size_t length1, dma_addr_t dma_addr2, size_t length2, int final) +{ + struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); + struct dma_async_tx_descriptor *in_desc; + struct scatterlist sg[2]; + + dev_dbg(dd->dev, "xmit_dma: digcnt: 0x%llx 0x%llx, length: %zd, final: %d\n", + ctx->digcnt[1], ctx->digcnt[0], length1, final); + + dd->dma_lch_in.dma_conf.src_maxburst = 16; + dd->dma_lch_in.dma_conf.dst_maxburst = 16; + + dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf); + + if (length2) { + sg_init_table(sg, 2); + sg_dma_address(&sg[0]) = dma_addr1; + sg_dma_len(&sg[0]) = length1; + sg_dma_address(&sg[1]) = dma_addr2; + sg_dma_len(&sg[1]) = length2; + in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, sg, 2, + DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + } else { + sg_init_table(sg, 1); + sg_dma_address(&sg[0]) = dma_addr1; + sg_dma_len(&sg[0]) = length1; + in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, sg, 1, + DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + } + if (!in_desc) + return atmel_sha_complete(dd, -EINVAL); + + in_desc->callback = atmel_sha_dma_callback; + in_desc->callback_param = dd; + + atmel_sha_write_ctrl(dd, 1); + + /* should be non-zero before next lines to disable clocks later */ + ctx->digcnt[0] += length1; + if (ctx->digcnt[0] < length1) + ctx->digcnt[1]++; + + if (final) + dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */ + + dd->flags |= SHA_FLAGS_DMA_ACTIVE; + + /* Start DMA transfer */ + dmaengine_submit(in_desc); + dma_async_issue_pending(dd->dma_lch_in.chan); + + return -EINPROGRESS; +} + +static int atmel_sha_xmit_start(struct atmel_sha_dev *dd, dma_addr_t dma_addr1, + size_t length1, dma_addr_t dma_addr2, size_t length2, int final) +{ + if (dd->caps.has_dma) + return atmel_sha_xmit_dma(dd, dma_addr1, length1, + dma_addr2, length2, final); + else + return atmel_sha_xmit_pdc(dd, dma_addr1, length1, + dma_addr2, length2, final); +} + +static int atmel_sha_update_cpu(struct atmel_sha_dev *dd) +{ + struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); + int bufcnt; + + atmel_sha_append_sg(ctx); + atmel_sha_fill_padding(ctx, 0); + bufcnt = ctx->bufcnt; + ctx->bufcnt = 0; + + return atmel_sha_xmit_cpu(dd, ctx->buffer, bufcnt, 1); +} + +static int atmel_sha_xmit_dma_map(struct atmel_sha_dev *dd, + struct atmel_sha_reqctx *ctx, + size_t length, int final) +{ + ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, + ctx->buflen + ctx->block_size, DMA_TO_DEVICE); + if (dma_mapping_error(dd->dev, ctx->dma_addr)) { + dev_err(dd->dev, "dma %zu bytes error\n", ctx->buflen + + ctx->block_size); + return atmel_sha_complete(dd, -EINVAL); + } + + ctx->flags &= ~SHA_FLAGS_SG; + + /* next call does not fail... so no unmap in the case of error */ + return atmel_sha_xmit_start(dd, ctx->dma_addr, length, 0, 0, final); +} + +static int atmel_sha_update_dma_slow(struct atmel_sha_dev *dd) +{ + struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); + unsigned int final; + size_t count; + + atmel_sha_append_sg(ctx); + + final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total; + + dev_dbg(dd->dev, "slow: bufcnt: %zu, digcnt: 0x%llx 0x%llx, final: %d\n", + ctx->bufcnt, ctx->digcnt[1], ctx->digcnt[0], final); + + if (final) + atmel_sha_fill_padding(ctx, 0); + + if (final || (ctx->bufcnt == ctx->buflen)) { + count = ctx->bufcnt; + ctx->bufcnt = 0; + return atmel_sha_xmit_dma_map(dd, ctx, count, final); + } + + return 0; +} + +static int atmel_sha_update_dma_start(struct atmel_sha_dev *dd) +{ + struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); + unsigned int length, final, tail; + struct scatterlist *sg; + unsigned int count; + + if (!ctx->total) + return 0; + + if (ctx->bufcnt || ctx->offset) + return atmel_sha_update_dma_slow(dd); + + dev_dbg(dd->dev, "fast: digcnt: 0x%llx 0x%llx, bufcnt: %zd, total: %u\n", + ctx->digcnt[1], ctx->digcnt[0], ctx->bufcnt, ctx->total); + + sg = ctx->sg; + + if (!IS_ALIGNED(sg->offset, sizeof(u32))) + return atmel_sha_update_dma_slow(dd); + + if (!sg_is_last(sg) && !IS_ALIGNED(sg->length, ctx->block_size)) + /* size is not ctx->block_size aligned */ + return atmel_sha_update_dma_slow(dd); + + length = min(ctx->total, sg->length); + + if (sg_is_last(sg)) { + if (!(ctx->flags & SHA_FLAGS_FINUP)) { + /* not last sg must be ctx->block_size aligned */ + tail = length & (ctx->block_size - 1); + length -= tail; + } + } + + ctx->total -= length; + ctx->offset = length; /* offset where to start slow */ + + final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total; + + /* Add padding */ + if (final) { + tail = length & (ctx->block_size - 1); + length -= tail; + ctx->total += tail; + ctx->offset = length; /* offset where to start slow */ + + sg = ctx->sg; + atmel_sha_append_sg(ctx); + + atmel_sha_fill_padding(ctx, length); + + ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, + ctx->buflen + ctx->block_size, DMA_TO_DEVICE); + if (dma_mapping_error(dd->dev, ctx->dma_addr)) { + dev_err(dd->dev, "dma %zu bytes error\n", + ctx->buflen + ctx->block_size); + return atmel_sha_complete(dd, -EINVAL); + } + + if (length == 0) { + ctx->flags &= ~SHA_FLAGS_SG; + count = ctx->bufcnt; + ctx->bufcnt = 0; + return atmel_sha_xmit_start(dd, ctx->dma_addr, count, 0, + 0, final); + } else { + ctx->sg = sg; + if (!dma_map_sg(dd->dev, ctx->sg, 1, + DMA_TO_DEVICE)) { + dev_err(dd->dev, "dma_map_sg error\n"); + return atmel_sha_complete(dd, -EINVAL); + } + + ctx->flags |= SHA_FLAGS_SG; + + count = ctx->bufcnt; + ctx->bufcnt = 0; + return atmel_sha_xmit_start(dd, sg_dma_address(ctx->sg), + length, ctx->dma_addr, count, final); + } + } + + if (!dma_map_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE)) { + dev_err(dd->dev, "dma_map_sg error\n"); + return atmel_sha_complete(dd, -EINVAL); + } + + ctx->flags |= SHA_FLAGS_SG; + + /* next call does not fail... so no unmap in the case of error */ + return atmel_sha_xmit_start(dd, sg_dma_address(ctx->sg), length, 0, + 0, final); +} + +static void atmel_sha_update_dma_stop(struct atmel_sha_dev *dd) +{ + struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); + + if (ctx->flags & SHA_FLAGS_SG) { + dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE); + if (ctx->sg->length == ctx->offset) { + ctx->sg = sg_next(ctx->sg); + if (ctx->sg) + ctx->offset = 0; + } + if (ctx->flags & SHA_FLAGS_PAD) { + dma_unmap_single(dd->dev, ctx->dma_addr, + ctx->buflen + ctx->block_size, DMA_TO_DEVICE); + } + } else { + dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen + + ctx->block_size, DMA_TO_DEVICE); + } +} + +static int atmel_sha_update_req(struct atmel_sha_dev *dd) +{ + struct ahash_request *req = dd->req; + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + int err; + + dev_dbg(dd->dev, "update_req: total: %u, digcnt: 0x%llx 0x%llx\n", + ctx->total, ctx->digcnt[1], ctx->digcnt[0]); + + if (ctx->flags & SHA_FLAGS_CPU) + err = atmel_sha_update_cpu(dd); + else + err = atmel_sha_update_dma_start(dd); + + /* wait for dma completion before can take more data */ + dev_dbg(dd->dev, "update: err: %d, digcnt: 0x%llx 0%llx\n", + err, ctx->digcnt[1], ctx->digcnt[0]); + + return err; +} + +static int atmel_sha_final_req(struct atmel_sha_dev *dd) +{ + struct ahash_request *req = dd->req; + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + int err = 0; + int count; + + if (ctx->bufcnt >= ATMEL_SHA_DMA_THRESHOLD) { + atmel_sha_fill_padding(ctx, 0); + count = ctx->bufcnt; + ctx->bufcnt = 0; + err = atmel_sha_xmit_dma_map(dd, ctx, count, 1); + } + /* faster to handle last block with cpu */ + else { + atmel_sha_fill_padding(ctx, 0); + count = ctx->bufcnt; + ctx->bufcnt = 0; + err = atmel_sha_xmit_cpu(dd, ctx->buffer, count, 1); + } + + dev_dbg(dd->dev, "final_req: err: %d\n", err); + + return err; +} + +static void atmel_sha_copy_hash(struct ahash_request *req) +{ + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + u32 *hash = (u32 *)ctx->digest; + unsigned int i, hashsize; + + switch (ctx->flags & SHA_FLAGS_ALGO_MASK) { + case SHA_FLAGS_SHA1: + hashsize = SHA1_DIGEST_SIZE; + break; + + case SHA_FLAGS_SHA224: + case SHA_FLAGS_SHA256: + hashsize = SHA256_DIGEST_SIZE; + break; + + case SHA_FLAGS_SHA384: + case SHA_FLAGS_SHA512: + hashsize = SHA512_DIGEST_SIZE; + break; + + default: + /* Should not happen... */ + return; + } + + for (i = 0; i < hashsize / sizeof(u32); ++i) + hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i)); + ctx->flags |= SHA_FLAGS_RESTORE; +} + +static void atmel_sha_copy_ready_hash(struct ahash_request *req) +{ + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + + if (!req->result) + return; + + switch (ctx->flags & SHA_FLAGS_ALGO_MASK) { + default: + case SHA_FLAGS_SHA1: + memcpy(req->result, ctx->digest, SHA1_DIGEST_SIZE); + break; + + case SHA_FLAGS_SHA224: + memcpy(req->result, ctx->digest, SHA224_DIGEST_SIZE); + break; + + case SHA_FLAGS_SHA256: + memcpy(req->result, ctx->digest, SHA256_DIGEST_SIZE); + break; + + case SHA_FLAGS_SHA384: + memcpy(req->result, ctx->digest, SHA384_DIGEST_SIZE); + break; + + case SHA_FLAGS_SHA512: + memcpy(req->result, ctx->digest, SHA512_DIGEST_SIZE); + break; + } +} + +static int atmel_sha_finish(struct ahash_request *req) +{ + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + struct atmel_sha_dev *dd = ctx->dd; + + if (ctx->digcnt[0] || ctx->digcnt[1]) + atmel_sha_copy_ready_hash(req); + + dev_dbg(dd->dev, "digcnt: 0x%llx 0x%llx, bufcnt: %zd\n", ctx->digcnt[1], + ctx->digcnt[0], ctx->bufcnt); + + return 0; +} + +static void atmel_sha_finish_req(struct ahash_request *req, int err) +{ + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + struct atmel_sha_dev *dd = ctx->dd; + + if (!err) { + atmel_sha_copy_hash(req); + if (SHA_FLAGS_FINAL & dd->flags) + err = atmel_sha_finish(req); + } else { + ctx->flags |= SHA_FLAGS_ERROR; + } + + /* atomic operation is not needed here */ + (void)atmel_sha_complete(dd, err); +} + +static int atmel_sha_hw_init(struct atmel_sha_dev *dd) +{ + int err; + + err = clk_enable(dd->iclk); + if (err) + return err; + + if (!(SHA_FLAGS_INIT & dd->flags)) { + atmel_sha_write(dd, SHA_CR, SHA_CR_SWRST); + dd->flags |= SHA_FLAGS_INIT; + } + + return 0; +} + +static inline unsigned int atmel_sha_get_version(struct atmel_sha_dev *dd) +{ + return atmel_sha_read(dd, SHA_HW_VERSION) & 0x00000fff; +} + +static int atmel_sha_hw_version_init(struct atmel_sha_dev *dd) +{ + int err; + + err = atmel_sha_hw_init(dd); + if (err) + return err; + + dd->hw_version = atmel_sha_get_version(dd); + + dev_info(dd->dev, + "version: 0x%x\n", dd->hw_version); + + clk_disable(dd->iclk); + + return 0; +} + +static int atmel_sha_handle_queue(struct atmel_sha_dev *dd, + struct ahash_request *req) +{ + struct crypto_async_request *async_req, *backlog; + struct atmel_sha_ctx *ctx; + unsigned long flags; + bool start_async; + int err = 0, ret = 0; + + spin_lock_irqsave(&dd->lock, flags); + if (req) + ret = ahash_enqueue_request(&dd->queue, req); + + if (SHA_FLAGS_BUSY & dd->flags) { + spin_unlock_irqrestore(&dd->lock, flags); + return ret; + } + + backlog = crypto_get_backlog(&dd->queue); + async_req = crypto_dequeue_request(&dd->queue); + if (async_req) + dd->flags |= SHA_FLAGS_BUSY; + + spin_unlock_irqrestore(&dd->lock, flags); + + if (!async_req) + return ret; + + if (backlog) + crypto_request_complete(backlog, -EINPROGRESS); + + ctx = crypto_tfm_ctx(async_req->tfm); + + dd->req = ahash_request_cast(async_req); + start_async = (dd->req != req); + dd->is_async = start_async; + dd->force_complete = false; + + /* WARNING: ctx->start() MAY change dd->is_async. */ + err = ctx->start(dd); + return (start_async) ? ret : err; +} + +static int atmel_sha_done(struct atmel_sha_dev *dd); + +static int atmel_sha_start(struct atmel_sha_dev *dd) +{ + struct ahash_request *req = dd->req; + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + int err; + + dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %u\n", + ctx->op, req->nbytes); + + err = atmel_sha_hw_init(dd); + if (err) + return atmel_sha_complete(dd, err); + + /* + * atmel_sha_update_req() and atmel_sha_final_req() can return either: + * -EINPROGRESS: the hardware is busy and the SHA driver will resume + * its job later in the done_task. + * This is the main path. + * + * 0: the SHA driver can continue its job then release the hardware + * later, if needed, with atmel_sha_finish_req(). + * This is the alternate path. + * + * < 0: an error has occurred so atmel_sha_complete(dd, err) has already + * been called, hence the hardware has been released. + * The SHA driver must stop its job without calling + * atmel_sha_finish_req(), otherwise atmel_sha_complete() would be + * called a second time. + * + * Please note that currently, atmel_sha_final_req() never returns 0. + */ + + dd->resume = atmel_sha_done; + if (ctx->op == SHA_OP_UPDATE) { + err = atmel_sha_update_req(dd); + if (!err && (ctx->flags & SHA_FLAGS_FINUP)) + /* no final() after finup() */ + err = atmel_sha_final_req(dd); + } else if (ctx->op == SHA_OP_FINAL) { + err = atmel_sha_final_req(dd); + } + + if (!err) + /* done_task will not finish it, so do it here */ + atmel_sha_finish_req(req, err); + + dev_dbg(dd->dev, "exit, err: %d\n", err); + + return err; +} + +static int atmel_sha_enqueue(struct ahash_request *req, unsigned int op) +{ + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + struct atmel_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm); + struct atmel_sha_dev *dd = tctx->dd; + + ctx->op = op; + + return atmel_sha_handle_queue(dd, req); +} + +static int atmel_sha_update(struct ahash_request *req) +{ + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + + if (!req->nbytes) + return 0; + + ctx->total = req->nbytes; + ctx->sg = req->src; + ctx->offset = 0; + + if (ctx->flags & SHA_FLAGS_FINUP) { + if (ctx->bufcnt + ctx->total < ATMEL_SHA_DMA_THRESHOLD) + /* faster to use CPU for short transfers */ + ctx->flags |= SHA_FLAGS_CPU; + } else if (ctx->bufcnt + ctx->total < ctx->buflen) { + atmel_sha_append_sg(ctx); + return 0; + } + return atmel_sha_enqueue(req, SHA_OP_UPDATE); +} + +static int atmel_sha_final(struct ahash_request *req) +{ + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + + ctx->flags |= SHA_FLAGS_FINUP; + + if (ctx->flags & SHA_FLAGS_ERROR) + return 0; /* uncompleted hash is not needed */ + + if (ctx->flags & SHA_FLAGS_PAD) + /* copy ready hash (+ finalize hmac) */ + return atmel_sha_finish(req); + + return atmel_sha_enqueue(req, SHA_OP_FINAL); +} + +static int atmel_sha_finup(struct ahash_request *req) +{ + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + int err1, err2; + + ctx->flags |= SHA_FLAGS_FINUP; + + err1 = atmel_sha_update(req); + if (err1 == -EINPROGRESS || + (err1 == -EBUSY && (ahash_request_flags(req) & + CRYPTO_TFM_REQ_MAY_BACKLOG))) + return err1; + + /* + * final() has to be always called to cleanup resources + * even if udpate() failed, except EINPROGRESS + */ + err2 = atmel_sha_final(req); + + return err1 ?: err2; +} + +static int atmel_sha_digest(struct ahash_request *req) +{ + return atmel_sha_init(req) ?: atmel_sha_finup(req); +} + + +static int atmel_sha_export(struct ahash_request *req, void *out) +{ + const struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + + memcpy(out, ctx, sizeof(*ctx)); + return 0; +} + +static int atmel_sha_import(struct ahash_request *req, const void *in) +{ + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + + memcpy(ctx, in, sizeof(*ctx)); + return 0; +} + +static int atmel_sha_cra_init(struct crypto_tfm *tfm) +{ + struct atmel_sha_ctx *ctx = crypto_tfm_ctx(tfm); + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct atmel_sha_reqctx)); + ctx->start = atmel_sha_start; + + return 0; +} + +static void atmel_sha_alg_init(struct ahash_alg *alg) +{ + alg->halg.base.cra_priority = ATMEL_SHA_PRIORITY; + alg->halg.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY; + alg->halg.base.cra_ctxsize = sizeof(struct atmel_sha_ctx); + alg->halg.base.cra_module = THIS_MODULE; + alg->halg.base.cra_init = atmel_sha_cra_init; + + alg->halg.statesize = sizeof(struct atmel_sha_reqctx); + + alg->init = atmel_sha_init; + alg->update = atmel_sha_update; + alg->final = atmel_sha_final; + alg->finup = atmel_sha_finup; + alg->digest = atmel_sha_digest; + alg->export = atmel_sha_export; + alg->import = atmel_sha_import; +} + +static struct ahash_alg sha_1_256_algs[] = { +{ + .halg.base.cra_name = "sha1", + .halg.base.cra_driver_name = "atmel-sha1", + .halg.base.cra_blocksize = SHA1_BLOCK_SIZE, + + .halg.digestsize = SHA1_DIGEST_SIZE, +}, +{ + .halg.base.cra_name = "sha256", + .halg.base.cra_driver_name = "atmel-sha256", + .halg.base.cra_blocksize = SHA256_BLOCK_SIZE, + + .halg.digestsize = SHA256_DIGEST_SIZE, +}, +}; + +static struct ahash_alg sha_224_alg = { + .halg.base.cra_name = "sha224", + .halg.base.cra_driver_name = "atmel-sha224", + .halg.base.cra_blocksize = SHA224_BLOCK_SIZE, + + .halg.digestsize = SHA224_DIGEST_SIZE, +}; + +static struct ahash_alg sha_384_512_algs[] = { +{ + .halg.base.cra_name = "sha384", + .halg.base.cra_driver_name = "atmel-sha384", + .halg.base.cra_blocksize = SHA384_BLOCK_SIZE, + + .halg.digestsize = SHA384_DIGEST_SIZE, +}, +{ + .halg.base.cra_name = "sha512", + .halg.base.cra_driver_name = "atmel-sha512", + .halg.base.cra_blocksize = SHA512_BLOCK_SIZE, + + .halg.digestsize = SHA512_DIGEST_SIZE, +}, +}; + +static void atmel_sha_queue_task(unsigned long data) +{ + struct atmel_sha_dev *dd = (struct atmel_sha_dev *)data; + + atmel_sha_handle_queue(dd, NULL); +} + +static int atmel_sha_done(struct atmel_sha_dev *dd) +{ + int err = 0; + + if (SHA_FLAGS_CPU & dd->flags) { + if (SHA_FLAGS_OUTPUT_READY & dd->flags) { + dd->flags &= ~SHA_FLAGS_OUTPUT_READY; + goto finish; + } + } else if (SHA_FLAGS_DMA_READY & dd->flags) { + if (SHA_FLAGS_DMA_ACTIVE & dd->flags) { + dd->flags &= ~SHA_FLAGS_DMA_ACTIVE; + atmel_sha_update_dma_stop(dd); + } + if (SHA_FLAGS_OUTPUT_READY & dd->flags) { + /* hash or semi-hash ready */ + dd->flags &= ~(SHA_FLAGS_DMA_READY | + SHA_FLAGS_OUTPUT_READY); + err = atmel_sha_update_dma_start(dd); + if (err != -EINPROGRESS) + goto finish; + } + } + return err; + +finish: + /* finish curent request */ + atmel_sha_finish_req(dd->req, err); + + return err; +} + +static void atmel_sha_done_task(unsigned long data) +{ + struct atmel_sha_dev *dd = (struct atmel_sha_dev *)data; + + dd->is_async = true; + (void)dd->resume(dd); +} + +static irqreturn_t atmel_sha_irq(int irq, void *dev_id) +{ + struct atmel_sha_dev *sha_dd = dev_id; + u32 reg; + + reg = atmel_sha_read(sha_dd, SHA_ISR); + if (reg & atmel_sha_read(sha_dd, SHA_IMR)) { + atmel_sha_write(sha_dd, SHA_IDR, reg); + if (SHA_FLAGS_BUSY & sha_dd->flags) { + sha_dd->flags |= SHA_FLAGS_OUTPUT_READY; + if (!(SHA_FLAGS_CPU & sha_dd->flags)) + sha_dd->flags |= SHA_FLAGS_DMA_READY; + tasklet_schedule(&sha_dd->done_task); + } else { + dev_warn(sha_dd->dev, "SHA interrupt when no active requests.\n"); + } + return IRQ_HANDLED; + } + + return IRQ_NONE; +} + + +/* DMA transfer functions */ + +static bool atmel_sha_dma_check_aligned(struct atmel_sha_dev *dd, + struct scatterlist *sg, + size_t len) +{ + struct atmel_sha_dma *dma = &dd->dma_lch_in; + struct ahash_request *req = dd->req; + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + size_t bs = ctx->block_size; + int nents; + + for (nents = 0; sg; sg = sg_next(sg), ++nents) { + if (!IS_ALIGNED(sg->offset, sizeof(u32))) + return false; + + /* + * This is the last sg, the only one that is allowed to + * have an unaligned length. + */ + if (len <= sg->length) { + dma->nents = nents + 1; + dma->last_sg_length = sg->length; + sg->length = ALIGN(len, sizeof(u32)); + return true; + } + + /* All other sg lengths MUST be aligned to the block size. */ + if (!IS_ALIGNED(sg->length, bs)) + return false; + + len -= sg->length; + } + + return false; +} + +static void atmel_sha_dma_callback2(void *data) +{ + struct atmel_sha_dev *dd = data; + struct atmel_sha_dma *dma = &dd->dma_lch_in; + struct scatterlist *sg; + int nents; + + dma_unmap_sg(dd->dev, dma->sg, dma->nents, DMA_TO_DEVICE); + + sg = dma->sg; + for (nents = 0; nents < dma->nents - 1; ++nents) + sg = sg_next(sg); + sg->length = dma->last_sg_length; + + dd->is_async = true; + (void)atmel_sha_wait_for_data_ready(dd, dd->resume); +} + +static int atmel_sha_dma_start(struct atmel_sha_dev *dd, + struct scatterlist *src, + size_t len, + atmel_sha_fn_t resume) +{ + struct atmel_sha_dma *dma = &dd->dma_lch_in; + struct dma_slave_config *config = &dma->dma_conf; + struct dma_chan *chan = dma->chan; + struct dma_async_tx_descriptor *desc; + dma_cookie_t cookie; + unsigned int sg_len; + int err; + + dd->resume = resume; + + /* + * dma->nents has already been initialized by + * atmel_sha_dma_check_aligned(). + */ + dma->sg = src; + sg_len = dma_map_sg(dd->dev, dma->sg, dma->nents, DMA_TO_DEVICE); + if (!sg_len) { + err = -ENOMEM; + goto exit; + } + + config->src_maxburst = 16; + config->dst_maxburst = 16; + err = dmaengine_slave_config(chan, config); + if (err) + goto unmap_sg; + + desc = dmaengine_prep_slave_sg(chan, dma->sg, sg_len, DMA_MEM_TO_DEV, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!desc) { + err = -ENOMEM; + goto unmap_sg; + } + + desc->callback = atmel_sha_dma_callback2; + desc->callback_param = dd; + cookie = dmaengine_submit(desc); + err = dma_submit_error(cookie); + if (err) + goto unmap_sg; + + dma_async_issue_pending(chan); + + return -EINPROGRESS; + +unmap_sg: + dma_unmap_sg(dd->dev, dma->sg, dma->nents, DMA_TO_DEVICE); +exit: + return atmel_sha_complete(dd, err); +} + + +/* CPU transfer functions */ + +static int atmel_sha_cpu_transfer(struct atmel_sha_dev *dd) +{ + struct ahash_request *req = dd->req; + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + const u32 *words = (const u32 *)ctx->buffer; + size_t i, num_words; + u32 isr, din, din_inc; + + din_inc = (ctx->flags & SHA_FLAGS_IDATAR0) ? 0 : 1; + for (;;) { + /* Write data into the Input Data Registers. */ + num_words = DIV_ROUND_UP(ctx->bufcnt, sizeof(u32)); + for (i = 0, din = 0; i < num_words; ++i, din += din_inc) + atmel_sha_write(dd, SHA_REG_DIN(din), words[i]); + + ctx->offset += ctx->bufcnt; + ctx->total -= ctx->bufcnt; + + if (!ctx->total) + break; + + /* + * Prepare next block: + * Fill ctx->buffer now with the next data to be written into + * IDATARx: it gives time for the SHA hardware to process + * the current data so the SHA_INT_DATARDY flag might be set + * in SHA_ISR when polling this register at the beginning of + * the next loop. + */ + ctx->bufcnt = min_t(size_t, ctx->block_size, ctx->total); + scatterwalk_map_and_copy(ctx->buffer, ctx->sg, + ctx->offset, ctx->bufcnt, 0); + + /* Wait for hardware to be ready again. */ + isr = atmel_sha_read(dd, SHA_ISR); + if (!(isr & SHA_INT_DATARDY)) { + /* Not ready yet. */ + dd->resume = atmel_sha_cpu_transfer; + atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY); + return -EINPROGRESS; + } + } + + if (unlikely(!(ctx->flags & SHA_FLAGS_WAIT_DATARDY))) + return dd->cpu_transfer_complete(dd); + + return atmel_sha_wait_for_data_ready(dd, dd->cpu_transfer_complete); +} + +static int atmel_sha_cpu_start(struct atmel_sha_dev *dd, + struct scatterlist *sg, + unsigned int len, + bool idatar0_only, + bool wait_data_ready, + atmel_sha_fn_t resume) +{ + struct ahash_request *req = dd->req; + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + + if (!len) + return resume(dd); + + ctx->flags &= ~(SHA_FLAGS_IDATAR0 | SHA_FLAGS_WAIT_DATARDY); + + if (idatar0_only) + ctx->flags |= SHA_FLAGS_IDATAR0; + + if (wait_data_ready) + ctx->flags |= SHA_FLAGS_WAIT_DATARDY; + + ctx->sg = sg; + ctx->total = len; + ctx->offset = 0; + + /* Prepare the first block to be written. */ + ctx->bufcnt = min_t(size_t, ctx->block_size, ctx->total); + scatterwalk_map_and_copy(ctx->buffer, ctx->sg, + ctx->offset, ctx->bufcnt, 0); + + dd->cpu_transfer_complete = resume; + return atmel_sha_cpu_transfer(dd); +} + +static int atmel_sha_cpu_hash(struct atmel_sha_dev *dd, + const void *data, unsigned int datalen, + bool auto_padding, + atmel_sha_fn_t resume) +{ + struct ahash_request *req = dd->req; + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + u32 msglen = (auto_padding) ? datalen : 0; + u32 mr = SHA_MR_MODE_AUTO; + + if (!(IS_ALIGNED(datalen, ctx->block_size) || auto_padding)) + return atmel_sha_complete(dd, -EINVAL); + + mr |= (ctx->flags & SHA_FLAGS_ALGO_MASK); + atmel_sha_write(dd, SHA_MR, mr); + atmel_sha_write(dd, SHA_MSR, msglen); + atmel_sha_write(dd, SHA_BCR, msglen); + atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST); + + sg_init_one(&dd->tmp, data, datalen); + return atmel_sha_cpu_start(dd, &dd->tmp, datalen, false, true, resume); +} + + +/* hmac functions */ + +struct atmel_sha_hmac_key { + bool valid; + unsigned int keylen; + u8 buffer[SHA512_BLOCK_SIZE]; + u8 *keydup; +}; + +static inline void atmel_sha_hmac_key_init(struct atmel_sha_hmac_key *hkey) +{ + memset(hkey, 0, sizeof(*hkey)); +} + +static inline void atmel_sha_hmac_key_release(struct atmel_sha_hmac_key *hkey) +{ + kfree(hkey->keydup); + memset(hkey, 0, sizeof(*hkey)); +} + +static inline int atmel_sha_hmac_key_set(struct atmel_sha_hmac_key *hkey, + const u8 *key, + unsigned int keylen) +{ + atmel_sha_hmac_key_release(hkey); + + if (keylen > sizeof(hkey->buffer)) { + hkey->keydup = kmemdup(key, keylen, GFP_KERNEL); + if (!hkey->keydup) + return -ENOMEM; + + } else { + memcpy(hkey->buffer, key, keylen); + } + + hkey->valid = true; + hkey->keylen = keylen; + return 0; +} + +static inline bool atmel_sha_hmac_key_get(const struct atmel_sha_hmac_key *hkey, + const u8 **key, + unsigned int *keylen) +{ + if (!hkey->valid) + return false; + + *keylen = hkey->keylen; + *key = (hkey->keydup) ? hkey->keydup : hkey->buffer; + return true; +} + + +struct atmel_sha_hmac_ctx { + struct atmel_sha_ctx base; + + struct atmel_sha_hmac_key hkey; + u32 ipad[SHA512_BLOCK_SIZE / sizeof(u32)]; + u32 opad[SHA512_BLOCK_SIZE / sizeof(u32)]; + atmel_sha_fn_t resume; +}; + +static int atmel_sha_hmac_setup(struct atmel_sha_dev *dd, + atmel_sha_fn_t resume); +static int atmel_sha_hmac_prehash_key(struct atmel_sha_dev *dd, + const u8 *key, unsigned int keylen); +static int atmel_sha_hmac_prehash_key_done(struct atmel_sha_dev *dd); +static int atmel_sha_hmac_compute_ipad_hash(struct atmel_sha_dev *dd); +static int atmel_sha_hmac_compute_opad_hash(struct atmel_sha_dev *dd); +static int atmel_sha_hmac_setup_done(struct atmel_sha_dev *dd); + +static int atmel_sha_hmac_init_done(struct atmel_sha_dev *dd); +static int atmel_sha_hmac_final(struct atmel_sha_dev *dd); +static int atmel_sha_hmac_final_done(struct atmel_sha_dev *dd); +static int atmel_sha_hmac_digest2(struct atmel_sha_dev *dd); + +static int atmel_sha_hmac_setup(struct atmel_sha_dev *dd, + atmel_sha_fn_t resume) +{ + struct ahash_request *req = dd->req; + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm); + unsigned int keylen; + const u8 *key; + size_t bs; + + hmac->resume = resume; + switch (ctx->flags & SHA_FLAGS_ALGO_MASK) { + case SHA_FLAGS_SHA1: + ctx->block_size = SHA1_BLOCK_SIZE; + ctx->hash_size = SHA1_DIGEST_SIZE; + break; + + case SHA_FLAGS_SHA224: + ctx->block_size = SHA224_BLOCK_SIZE; + ctx->hash_size = SHA256_DIGEST_SIZE; + break; + + case SHA_FLAGS_SHA256: + ctx->block_size = SHA256_BLOCK_SIZE; + ctx->hash_size = SHA256_DIGEST_SIZE; + break; + + case SHA_FLAGS_SHA384: + ctx->block_size = SHA384_BLOCK_SIZE; + ctx->hash_size = SHA512_DIGEST_SIZE; + break; + + case SHA_FLAGS_SHA512: + ctx->block_size = SHA512_BLOCK_SIZE; + ctx->hash_size = SHA512_DIGEST_SIZE; + break; + + default: + return atmel_sha_complete(dd, -EINVAL); + } + bs = ctx->block_size; + + if (likely(!atmel_sha_hmac_key_get(&hmac->hkey, &key, &keylen))) + return resume(dd); + + /* Compute K' from K. */ + if (unlikely(keylen > bs)) + return atmel_sha_hmac_prehash_key(dd, key, keylen); + + /* Prepare ipad. */ + memcpy_and_pad(hmac->ipad, bs, key, keylen, 0); + return atmel_sha_hmac_compute_ipad_hash(dd); +} + +static int atmel_sha_hmac_prehash_key(struct atmel_sha_dev *dd, + const u8 *key, unsigned int keylen) +{ + return atmel_sha_cpu_hash(dd, key, keylen, true, + atmel_sha_hmac_prehash_key_done); +} + +static int atmel_sha_hmac_prehash_key_done(struct atmel_sha_dev *dd) +{ + struct ahash_request *req = dd->req; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm); + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + size_t ds = crypto_ahash_digestsize(tfm); + size_t bs = ctx->block_size; + size_t i, num_words = ds / sizeof(u32); + + /* Prepare ipad. */ + for (i = 0; i < num_words; ++i) + hmac->ipad[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i)); + memset((u8 *)hmac->ipad + ds, 0, bs - ds); + return atmel_sha_hmac_compute_ipad_hash(dd); +} + +static int atmel_sha_hmac_compute_ipad_hash(struct atmel_sha_dev *dd) +{ + struct ahash_request *req = dd->req; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm); + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + size_t bs = ctx->block_size; + size_t i, num_words = bs / sizeof(u32); + + unsafe_memcpy(hmac->opad, hmac->ipad, bs, + "fortified memcpy causes -Wrestrict warning"); + for (i = 0; i < num_words; ++i) { + hmac->ipad[i] ^= 0x36363636; + hmac->opad[i] ^= 0x5c5c5c5c; + } + + return atmel_sha_cpu_hash(dd, hmac->ipad, bs, false, + atmel_sha_hmac_compute_opad_hash); +} + +static int atmel_sha_hmac_compute_opad_hash(struct atmel_sha_dev *dd) +{ + struct ahash_request *req = dd->req; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm); + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + size_t bs = ctx->block_size; + size_t hs = ctx->hash_size; + size_t i, num_words = hs / sizeof(u32); + + for (i = 0; i < num_words; ++i) + hmac->ipad[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i)); + return atmel_sha_cpu_hash(dd, hmac->opad, bs, false, + atmel_sha_hmac_setup_done); +} + +static int atmel_sha_hmac_setup_done(struct atmel_sha_dev *dd) +{ + struct ahash_request *req = dd->req; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm); + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + size_t hs = ctx->hash_size; + size_t i, num_words = hs / sizeof(u32); + + for (i = 0; i < num_words; ++i) + hmac->opad[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i)); + atmel_sha_hmac_key_release(&hmac->hkey); + return hmac->resume(dd); +} + +static int atmel_sha_hmac_start(struct atmel_sha_dev *dd) +{ + struct ahash_request *req = dd->req; + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + int err; + + err = atmel_sha_hw_init(dd); + if (err) + return atmel_sha_complete(dd, err); + + switch (ctx->op) { + case SHA_OP_INIT: + err = atmel_sha_hmac_setup(dd, atmel_sha_hmac_init_done); + break; + + case SHA_OP_UPDATE: + dd->resume = atmel_sha_done; + err = atmel_sha_update_req(dd); + break; + + case SHA_OP_FINAL: + dd->resume = atmel_sha_hmac_final; + err = atmel_sha_final_req(dd); + break; + + case SHA_OP_DIGEST: + err = atmel_sha_hmac_setup(dd, atmel_sha_hmac_digest2); + break; + + default: + return atmel_sha_complete(dd, -EINVAL); + } + + return err; +} + +static int atmel_sha_hmac_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm); + + return atmel_sha_hmac_key_set(&hmac->hkey, key, keylen); +} + +static int atmel_sha_hmac_init(struct ahash_request *req) +{ + int err; + + err = atmel_sha_init(req); + if (err) + return err; + + return atmel_sha_enqueue(req, SHA_OP_INIT); +} + +static int atmel_sha_hmac_init_done(struct atmel_sha_dev *dd) +{ + struct ahash_request *req = dd->req; + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm); + size_t bs = ctx->block_size; + size_t hs = ctx->hash_size; + + ctx->bufcnt = 0; + ctx->digcnt[0] = bs; + ctx->digcnt[1] = 0; + ctx->flags |= SHA_FLAGS_RESTORE; + memcpy(ctx->digest, hmac->ipad, hs); + return atmel_sha_complete(dd, 0); +} + +static int atmel_sha_hmac_final(struct atmel_sha_dev *dd) +{ + struct ahash_request *req = dd->req; + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm); + u32 *digest = (u32 *)ctx->digest; + size_t ds = crypto_ahash_digestsize(tfm); + size_t bs = ctx->block_size; + size_t hs = ctx->hash_size; + size_t i, num_words; + u32 mr; + + /* Save d = SHA((K' + ipad) | msg). */ + num_words = ds / sizeof(u32); + for (i = 0; i < num_words; ++i) + digest[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i)); + + /* Restore context to finish computing SHA((K' + opad) | d). */ + atmel_sha_write(dd, SHA_CR, SHA_CR_WUIHV); + num_words = hs / sizeof(u32); + for (i = 0; i < num_words; ++i) + atmel_sha_write(dd, SHA_REG_DIN(i), hmac->opad[i]); + + mr = SHA_MR_MODE_AUTO | SHA_MR_UIHV; + mr |= (ctx->flags & SHA_FLAGS_ALGO_MASK); + atmel_sha_write(dd, SHA_MR, mr); + atmel_sha_write(dd, SHA_MSR, bs + ds); + atmel_sha_write(dd, SHA_BCR, ds); + atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST); + + sg_init_one(&dd->tmp, digest, ds); + return atmel_sha_cpu_start(dd, &dd->tmp, ds, false, true, + atmel_sha_hmac_final_done); +} + +static int atmel_sha_hmac_final_done(struct atmel_sha_dev *dd) +{ + /* + * req->result might not be sizeof(u32) aligned, so copy the + * digest into ctx->digest[] before memcpy() the data into + * req->result. + */ + atmel_sha_copy_hash(dd->req); + atmel_sha_copy_ready_hash(dd->req); + return atmel_sha_complete(dd, 0); +} + +static int atmel_sha_hmac_digest(struct ahash_request *req) +{ + int err; + + err = atmel_sha_init(req); + if (err) + return err; + + return atmel_sha_enqueue(req, SHA_OP_DIGEST); +} + +static int atmel_sha_hmac_digest2(struct atmel_sha_dev *dd) +{ + struct ahash_request *req = dd->req; + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm); + struct scatterlist *sgbuf; + size_t hs = ctx->hash_size; + size_t i, num_words = hs / sizeof(u32); + bool use_dma = false; + u32 mr; + + /* Special case for empty message. */ + if (!req->nbytes) { + req->nbytes = 0; + ctx->bufcnt = 0; + ctx->digcnt[0] = 0; + ctx->digcnt[1] = 0; + switch (ctx->flags & SHA_FLAGS_ALGO_MASK) { + case SHA_FLAGS_SHA1: + case SHA_FLAGS_SHA224: + case SHA_FLAGS_SHA256: + atmel_sha_fill_padding(ctx, 64); + break; + + case SHA_FLAGS_SHA384: + case SHA_FLAGS_SHA512: + atmel_sha_fill_padding(ctx, 128); + break; + } + sg_init_one(&dd->tmp, ctx->buffer, ctx->bufcnt); + } + + /* Check DMA threshold and alignment. */ + if (req->nbytes > ATMEL_SHA_DMA_THRESHOLD && + atmel_sha_dma_check_aligned(dd, req->src, req->nbytes)) + use_dma = true; + + /* Write both initial hash values to compute a HMAC. */ + atmel_sha_write(dd, SHA_CR, SHA_CR_WUIHV); + for (i = 0; i < num_words; ++i) + atmel_sha_write(dd, SHA_REG_DIN(i), hmac->ipad[i]); + + atmel_sha_write(dd, SHA_CR, SHA_CR_WUIEHV); + for (i = 0; i < num_words; ++i) + atmel_sha_write(dd, SHA_REG_DIN(i), hmac->opad[i]); + + /* Write the Mode, Message Size, Bytes Count then Control Registers. */ + mr = (SHA_MR_HMAC | SHA_MR_DUALBUFF); + mr |= ctx->flags & SHA_FLAGS_ALGO_MASK; + if (use_dma) + mr |= SHA_MR_MODE_IDATAR0; + else + mr |= SHA_MR_MODE_AUTO; + atmel_sha_write(dd, SHA_MR, mr); + + atmel_sha_write(dd, SHA_MSR, req->nbytes); + atmel_sha_write(dd, SHA_BCR, req->nbytes); + + atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST); + + /* Special case for empty message. */ + if (!req->nbytes) { + sgbuf = &dd->tmp; + req->nbytes = ctx->bufcnt; + } else { + sgbuf = req->src; + } + + /* Process data. */ + if (use_dma) + return atmel_sha_dma_start(dd, sgbuf, req->nbytes, + atmel_sha_hmac_final_done); + + return atmel_sha_cpu_start(dd, sgbuf, req->nbytes, false, true, + atmel_sha_hmac_final_done); +} + +static int atmel_sha_hmac_cra_init(struct crypto_tfm *tfm) +{ + struct atmel_sha_hmac_ctx *hmac = crypto_tfm_ctx(tfm); + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct atmel_sha_reqctx)); + hmac->base.start = atmel_sha_hmac_start; + atmel_sha_hmac_key_init(&hmac->hkey); + + return 0; +} + +static void atmel_sha_hmac_cra_exit(struct crypto_tfm *tfm) +{ + struct atmel_sha_hmac_ctx *hmac = crypto_tfm_ctx(tfm); + + atmel_sha_hmac_key_release(&hmac->hkey); +} + +static void atmel_sha_hmac_alg_init(struct ahash_alg *alg) +{ + alg->halg.base.cra_priority = ATMEL_SHA_PRIORITY; + alg->halg.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY; + alg->halg.base.cra_ctxsize = sizeof(struct atmel_sha_hmac_ctx); + alg->halg.base.cra_module = THIS_MODULE; + alg->halg.base.cra_init = atmel_sha_hmac_cra_init; + alg->halg.base.cra_exit = atmel_sha_hmac_cra_exit; + + alg->halg.statesize = sizeof(struct atmel_sha_reqctx); + + alg->init = atmel_sha_hmac_init; + alg->update = atmel_sha_update; + alg->final = atmel_sha_final; + alg->digest = atmel_sha_hmac_digest; + alg->setkey = atmel_sha_hmac_setkey; + alg->export = atmel_sha_export; + alg->import = atmel_sha_import; +} + +static struct ahash_alg sha_hmac_algs[] = { +{ + .halg.base.cra_name = "hmac(sha1)", + .halg.base.cra_driver_name = "atmel-hmac-sha1", + .halg.base.cra_blocksize = SHA1_BLOCK_SIZE, + + .halg.digestsize = SHA1_DIGEST_SIZE, +}, +{ + .halg.base.cra_name = "hmac(sha224)", + .halg.base.cra_driver_name = "atmel-hmac-sha224", + .halg.base.cra_blocksize = SHA224_BLOCK_SIZE, + + .halg.digestsize = SHA224_DIGEST_SIZE, +}, +{ + .halg.base.cra_name = "hmac(sha256)", + .halg.base.cra_driver_name = "atmel-hmac-sha256", + .halg.base.cra_blocksize = SHA256_BLOCK_SIZE, + + .halg.digestsize = SHA256_DIGEST_SIZE, +}, +{ + .halg.base.cra_name = "hmac(sha384)", + .halg.base.cra_driver_name = "atmel-hmac-sha384", + .halg.base.cra_blocksize = SHA384_BLOCK_SIZE, + + .halg.digestsize = SHA384_DIGEST_SIZE, +}, +{ + .halg.base.cra_name = "hmac(sha512)", + .halg.base.cra_driver_name = "atmel-hmac-sha512", + .halg.base.cra_blocksize = SHA512_BLOCK_SIZE, + + .halg.digestsize = SHA512_DIGEST_SIZE, +}, +}; + +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) +/* authenc functions */ + +static int atmel_sha_authenc_init2(struct atmel_sha_dev *dd); +static int atmel_sha_authenc_init_done(struct atmel_sha_dev *dd); +static int atmel_sha_authenc_final_done(struct atmel_sha_dev *dd); + + +struct atmel_sha_authenc_ctx { + struct crypto_ahash *tfm; +}; + +struct atmel_sha_authenc_reqctx { + struct atmel_sha_reqctx base; + + atmel_aes_authenc_fn_t cb; + struct atmel_aes_dev *aes_dev; + + /* _init() parameters. */ + struct scatterlist *assoc; + u32 assoclen; + u32 textlen; + + /* _final() parameters. */ + u32 *digest; + unsigned int digestlen; +}; + +static void atmel_sha_authenc_complete(void *data, int err) +{ + struct ahash_request *req = data; + struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req); + + authctx->cb(authctx->aes_dev, err, authctx->base.dd->is_async); +} + +static int atmel_sha_authenc_start(struct atmel_sha_dev *dd) +{ + struct ahash_request *req = dd->req; + struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req); + int err; + + /* + * Force atmel_sha_complete() to call req->base.complete(), ie + * atmel_sha_authenc_complete(), which in turn calls authctx->cb(). + */ + dd->force_complete = true; + + err = atmel_sha_hw_init(dd); + return authctx->cb(authctx->aes_dev, err, dd->is_async); +} + +bool atmel_sha_authenc_is_ready(void) +{ + struct atmel_sha_ctx dummy; + + dummy.dd = NULL; + return (atmel_sha_find_dev(&dummy) != NULL); +} +EXPORT_SYMBOL_GPL(atmel_sha_authenc_is_ready); + +unsigned int atmel_sha_authenc_get_reqsize(void) +{ + return sizeof(struct atmel_sha_authenc_reqctx); +} +EXPORT_SYMBOL_GPL(atmel_sha_authenc_get_reqsize); + +struct atmel_sha_authenc_ctx *atmel_sha_authenc_spawn(unsigned long mode) +{ + struct atmel_sha_authenc_ctx *auth; + struct crypto_ahash *tfm; + struct atmel_sha_ctx *tctx; + const char *name; + int err = -EINVAL; + + switch (mode & SHA_FLAGS_MODE_MASK) { + case SHA_FLAGS_HMAC_SHA1: + name = "atmel-hmac-sha1"; + break; + + case SHA_FLAGS_HMAC_SHA224: + name = "atmel-hmac-sha224"; + break; + + case SHA_FLAGS_HMAC_SHA256: + name = "atmel-hmac-sha256"; + break; + + case SHA_FLAGS_HMAC_SHA384: + name = "atmel-hmac-sha384"; + break; + + case SHA_FLAGS_HMAC_SHA512: + name = "atmel-hmac-sha512"; + break; + + default: + goto error; + } + + tfm = crypto_alloc_ahash(name, 0, 0); + if (IS_ERR(tfm)) { + err = PTR_ERR(tfm); + goto error; + } + tctx = crypto_ahash_ctx(tfm); + tctx->start = atmel_sha_authenc_start; + tctx->flags = mode; + + auth = kzalloc_obj(*auth); + if (!auth) { + err = -ENOMEM; + goto err_free_ahash; + } + auth->tfm = tfm; + + return auth; + +err_free_ahash: + crypto_free_ahash(tfm); +error: + return ERR_PTR(err); +} +EXPORT_SYMBOL_GPL(atmel_sha_authenc_spawn); + +void atmel_sha_authenc_free(struct atmel_sha_authenc_ctx *auth) +{ + if (auth) + crypto_free_ahash(auth->tfm); + kfree(auth); +} +EXPORT_SYMBOL_GPL(atmel_sha_authenc_free); + +int atmel_sha_authenc_setkey(struct atmel_sha_authenc_ctx *auth, + const u8 *key, unsigned int keylen, u32 flags) +{ + struct crypto_ahash *tfm = auth->tfm; + + crypto_ahash_clear_flags(tfm, CRYPTO_TFM_REQ_MASK); + crypto_ahash_set_flags(tfm, flags & CRYPTO_TFM_REQ_MASK); + return crypto_ahash_setkey(tfm, key, keylen); +} +EXPORT_SYMBOL_GPL(atmel_sha_authenc_setkey); + +int atmel_sha_authenc_schedule(struct ahash_request *req, + struct atmel_sha_authenc_ctx *auth, + atmel_aes_authenc_fn_t cb, + struct atmel_aes_dev *aes_dev) +{ + struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req); + struct atmel_sha_reqctx *ctx = &authctx->base; + struct crypto_ahash *tfm = auth->tfm; + struct atmel_sha_ctx *tctx = crypto_ahash_ctx(tfm); + struct atmel_sha_dev *dd; + + /* Reset request context (MUST be done first). */ + memset(authctx, 0, sizeof(*authctx)); + + /* Get SHA device. */ + dd = atmel_sha_find_dev(tctx); + if (!dd) + return cb(aes_dev, -ENODEV, false); + + /* Init request context. */ + ctx->dd = dd; + ctx->buflen = SHA_BUFFER_LEN; + authctx->cb = cb; + authctx->aes_dev = aes_dev; + ahash_request_set_tfm(req, tfm); + ahash_request_set_callback(req, 0, atmel_sha_authenc_complete, req); + + return atmel_sha_handle_queue(dd, req); +} +EXPORT_SYMBOL_GPL(atmel_sha_authenc_schedule); + +int atmel_sha_authenc_init(struct ahash_request *req, + struct scatterlist *assoc, unsigned int assoclen, + unsigned int textlen, + atmel_aes_authenc_fn_t cb, + struct atmel_aes_dev *aes_dev) +{ + struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req); + struct atmel_sha_reqctx *ctx = &authctx->base; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm); + struct atmel_sha_dev *dd = ctx->dd; + + if (unlikely(!IS_ALIGNED(assoclen, sizeof(u32)))) + return atmel_sha_complete(dd, -EINVAL); + + authctx->cb = cb; + authctx->aes_dev = aes_dev; + authctx->assoc = assoc; + authctx->assoclen = assoclen; + authctx->textlen = textlen; + + ctx->flags = hmac->base.flags; + return atmel_sha_hmac_setup(dd, atmel_sha_authenc_init2); +} +EXPORT_SYMBOL_GPL(atmel_sha_authenc_init); + +static int atmel_sha_authenc_init2(struct atmel_sha_dev *dd) +{ + struct ahash_request *req = dd->req; + struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req); + struct atmel_sha_reqctx *ctx = &authctx->base; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm); + size_t hs = ctx->hash_size; + size_t i, num_words = hs / sizeof(u32); + u32 mr, msg_size; + + atmel_sha_write(dd, SHA_CR, SHA_CR_WUIHV); + for (i = 0; i < num_words; ++i) + atmel_sha_write(dd, SHA_REG_DIN(i), hmac->ipad[i]); + + atmel_sha_write(dd, SHA_CR, SHA_CR_WUIEHV); + for (i = 0; i < num_words; ++i) + atmel_sha_write(dd, SHA_REG_DIN(i), hmac->opad[i]); + + mr = (SHA_MR_MODE_IDATAR0 | + SHA_MR_HMAC | + SHA_MR_DUALBUFF); + mr |= ctx->flags & SHA_FLAGS_ALGO_MASK; + atmel_sha_write(dd, SHA_MR, mr); + + msg_size = authctx->assoclen + authctx->textlen; + atmel_sha_write(dd, SHA_MSR, msg_size); + atmel_sha_write(dd, SHA_BCR, msg_size); + + atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST); + + /* Process assoc data. */ + return atmel_sha_cpu_start(dd, authctx->assoc, authctx->assoclen, + true, false, + atmel_sha_authenc_init_done); +} + +static int atmel_sha_authenc_init_done(struct atmel_sha_dev *dd) +{ + struct ahash_request *req = dd->req; + struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req); + + return authctx->cb(authctx->aes_dev, 0, dd->is_async); +} + +int atmel_sha_authenc_final(struct ahash_request *req, + u32 *digest, unsigned int digestlen, + atmel_aes_authenc_fn_t cb, + struct atmel_aes_dev *aes_dev) +{ + struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req); + struct atmel_sha_reqctx *ctx = &authctx->base; + struct atmel_sha_dev *dd = ctx->dd; + + switch (ctx->flags & SHA_FLAGS_ALGO_MASK) { + case SHA_FLAGS_SHA1: + authctx->digestlen = SHA1_DIGEST_SIZE; + break; + + case SHA_FLAGS_SHA224: + authctx->digestlen = SHA224_DIGEST_SIZE; + break; + + case SHA_FLAGS_SHA256: + authctx->digestlen = SHA256_DIGEST_SIZE; + break; + + case SHA_FLAGS_SHA384: + authctx->digestlen = SHA384_DIGEST_SIZE; + break; + + case SHA_FLAGS_SHA512: + authctx->digestlen = SHA512_DIGEST_SIZE; + break; + + default: + return atmel_sha_complete(dd, -EINVAL); + } + if (authctx->digestlen > digestlen) + authctx->digestlen = digestlen; + + authctx->cb = cb; + authctx->aes_dev = aes_dev; + authctx->digest = digest; + return atmel_sha_wait_for_data_ready(dd, + atmel_sha_authenc_final_done); +} +EXPORT_SYMBOL_GPL(atmel_sha_authenc_final); + +static int atmel_sha_authenc_final_done(struct atmel_sha_dev *dd) +{ + struct ahash_request *req = dd->req; + struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req); + size_t i, num_words = authctx->digestlen / sizeof(u32); + + for (i = 0; i < num_words; ++i) + authctx->digest[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i)); + + return atmel_sha_complete(dd, 0); +} + +void atmel_sha_authenc_abort(struct ahash_request *req) +{ + struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req); + struct atmel_sha_reqctx *ctx = &authctx->base; + struct atmel_sha_dev *dd = ctx->dd; + + /* Prevent atmel_sha_complete() from calling req->base.complete(). */ + dd->is_async = false; + dd->force_complete = false; + (void)atmel_sha_complete(dd, 0); +} +EXPORT_SYMBOL_GPL(atmel_sha_authenc_abort); + +#endif /* CONFIG_CRYPTO_DEV_ATMEL_AUTHENC */ + + +static void atmel_sha_unregister_algs(struct atmel_sha_dev *dd) +{ + if (dd->caps.has_hmac) + crypto_unregister_ahashes(sha_hmac_algs, + ARRAY_SIZE(sha_hmac_algs)); + + crypto_unregister_ahashes(sha_1_256_algs, ARRAY_SIZE(sha_1_256_algs)); + + if (dd->caps.has_sha224) + crypto_unregister_ahash(&sha_224_alg); + + if (dd->caps.has_sha_384_512) + crypto_unregister_ahashes(sha_384_512_algs, + ARRAY_SIZE(sha_384_512_algs)); +} + +static int atmel_sha_register_algs(struct atmel_sha_dev *dd) +{ + int err, i; + + for (i = 0; i < ARRAY_SIZE(sha_1_256_algs); i++) { + atmel_sha_alg_init(&sha_1_256_algs[i]); + + err = crypto_register_ahash(&sha_1_256_algs[i]); + if (err) + goto err_sha_1_256_algs; + } + + if (dd->caps.has_sha224) { + atmel_sha_alg_init(&sha_224_alg); + + err = crypto_register_ahash(&sha_224_alg); + if (err) + goto err_sha_224_algs; + } + + if (dd->caps.has_sha_384_512) { + for (i = 0; i < ARRAY_SIZE(sha_384_512_algs); i++) { + atmel_sha_alg_init(&sha_384_512_algs[i]); + + err = crypto_register_ahash(&sha_384_512_algs[i]); + if (err) + goto err_sha_384_512_algs; + } + } + + if (dd->caps.has_hmac) { + for (i = 0; i < ARRAY_SIZE(sha_hmac_algs); i++) { + atmel_sha_hmac_alg_init(&sha_hmac_algs[i]); + + err = crypto_register_ahash(&sha_hmac_algs[i]); + if (err) + goto err_sha_hmac_algs; + } + } + + return 0; + + /*i = ARRAY_SIZE(sha_hmac_algs);*/ +err_sha_hmac_algs: + crypto_unregister_ahashes(sha_hmac_algs, i); + i = ARRAY_SIZE(sha_384_512_algs); +err_sha_384_512_algs: + crypto_unregister_ahashes(sha_384_512_algs, i); + crypto_unregister_ahash(&sha_224_alg); +err_sha_224_algs: + i = ARRAY_SIZE(sha_1_256_algs); +err_sha_1_256_algs: + crypto_unregister_ahashes(sha_1_256_algs, i); + + return err; +} + +static int atmel_sha_dma_init(struct atmel_sha_dev *dd) +{ + dd->dma_lch_in.chan = dma_request_chan(dd->dev, "tx"); + if (IS_ERR(dd->dma_lch_in.chan)) { + return dev_err_probe(dd->dev, PTR_ERR(dd->dma_lch_in.chan), + "DMA channel is not available\n"); + } + + dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + + SHA_REG_DIN(0); + dd->dma_lch_in.dma_conf.src_maxburst = 1; + dd->dma_lch_in.dma_conf.src_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_in.dma_conf.dst_maxburst = 1; + dd->dma_lch_in.dma_conf.dst_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_in.dma_conf.device_fc = false; + + return 0; +} + +static void atmel_sha_dma_cleanup(struct atmel_sha_dev *dd) +{ + dma_release_channel(dd->dma_lch_in.chan); +} + +static void atmel_sha_get_cap(struct atmel_sha_dev *dd) +{ + + dd->caps.has_dma = 0; + dd->caps.has_dualbuff = 0; + dd->caps.has_sha224 = 0; + dd->caps.has_sha_384_512 = 0; + dd->caps.has_uihv = 0; + dd->caps.has_hmac = 0; + + /* keep only major version number */ + switch (dd->hw_version & 0xff0) { + case 0x800: + case 0x700: + case 0x600: + case 0x510: + dd->caps.has_dma = 1; + dd->caps.has_dualbuff = 1; + dd->caps.has_sha224 = 1; + dd->caps.has_sha_384_512 = 1; + dd->caps.has_uihv = 1; + dd->caps.has_hmac = 1; + break; + case 0x420: + dd->caps.has_dma = 1; + dd->caps.has_dualbuff = 1; + dd->caps.has_sha224 = 1; + dd->caps.has_sha_384_512 = 1; + dd->caps.has_uihv = 1; + break; + case 0x410: + dd->caps.has_dma = 1; + dd->caps.has_dualbuff = 1; + dd->caps.has_sha224 = 1; + dd->caps.has_sha_384_512 = 1; + break; + case 0x400: + dd->caps.has_dma = 1; + dd->caps.has_dualbuff = 1; + dd->caps.has_sha224 = 1; + break; + case 0x320: + break; + default: + dev_warn(dd->dev, + "Unmanaged sha version, set minimum capabilities\n"); + break; + } +} + +static const struct of_device_id atmel_sha_dt_ids[] = { + { .compatible = "atmel,at91sam9g46-sha" }, + { /* sentinel */ } +}; + +MODULE_DEVICE_TABLE(of, atmel_sha_dt_ids); + +static int atmel_sha_probe(struct platform_device *pdev) +{ + struct atmel_sha_dev *sha_dd; + struct device *dev = &pdev->dev; + struct resource *sha_res; + int err; + + sha_dd = devm_kzalloc(&pdev->dev, sizeof(*sha_dd), GFP_KERNEL); + if (!sha_dd) + return -ENOMEM; + + sha_dd->dev = dev; + + platform_set_drvdata(pdev, sha_dd); + + INIT_LIST_HEAD(&sha_dd->list); + spin_lock_init(&sha_dd->lock); + + tasklet_init(&sha_dd->done_task, atmel_sha_done_task, + (unsigned long)sha_dd); + tasklet_init(&sha_dd->queue_task, atmel_sha_queue_task, + (unsigned long)sha_dd); + + crypto_init_queue(&sha_dd->queue, ATMEL_SHA_QUEUE_LENGTH); + + sha_dd->io_base = devm_platform_get_and_ioremap_resource(pdev, 0, &sha_res); + if (IS_ERR(sha_dd->io_base)) { + err = PTR_ERR(sha_dd->io_base); + goto err_tasklet_kill; + } + sha_dd->phys_base = sha_res->start; + + /* Get the IRQ */ + sha_dd->irq = platform_get_irq(pdev, 0); + if (sha_dd->irq < 0) { + err = sha_dd->irq; + goto err_tasklet_kill; + } + + err = devm_request_irq(&pdev->dev, sha_dd->irq, atmel_sha_irq, + IRQF_SHARED, "atmel-sha", sha_dd); + if (err) + goto err_tasklet_kill; + + /* Initializing the clock */ + sha_dd->iclk = devm_clk_get_prepared(&pdev->dev, "sha_clk"); + if (IS_ERR(sha_dd->iclk)) { + dev_err(dev, "clock initialization failed.\n"); + err = PTR_ERR(sha_dd->iclk); + goto err_tasklet_kill; + } + + err = atmel_sha_hw_version_init(sha_dd); + if (err) + goto err_tasklet_kill; + + atmel_sha_get_cap(sha_dd); + + if (sha_dd->caps.has_dma) { + err = atmel_sha_dma_init(sha_dd); + if (err) + goto err_tasklet_kill; + + dev_info(dev, "using %s for DMA transfers\n", + dma_chan_name(sha_dd->dma_lch_in.chan)); + } + + spin_lock(&atmel_sha.lock); + list_add_tail(&sha_dd->list, &atmel_sha.dev_list); + spin_unlock(&atmel_sha.lock); + + err = atmel_sha_register_algs(sha_dd); + if (err) + goto err_algs; + + dev_info(dev, "Atmel SHA1/SHA256%s%s\n", + sha_dd->caps.has_sha224 ? "/SHA224" : "", + sha_dd->caps.has_sha_384_512 ? "/SHA384/SHA512" : ""); + + return 0; + +err_algs: + spin_lock(&atmel_sha.lock); + list_del(&sha_dd->list); + spin_unlock(&atmel_sha.lock); + if (sha_dd->caps.has_dma) + atmel_sha_dma_cleanup(sha_dd); +err_tasklet_kill: + tasklet_kill(&sha_dd->queue_task); + tasklet_kill(&sha_dd->done_task); + + return err; +} + +static void atmel_sha_remove(struct platform_device *pdev) +{ + struct atmel_sha_dev *sha_dd = platform_get_drvdata(pdev); + + spin_lock(&atmel_sha.lock); + list_del(&sha_dd->list); + spin_unlock(&atmel_sha.lock); + + atmel_sha_unregister_algs(sha_dd); + + tasklet_kill(&sha_dd->queue_task); + tasklet_kill(&sha_dd->done_task); + + if (sha_dd->caps.has_dma) + atmel_sha_dma_cleanup(sha_dd); +} + +static struct platform_driver atmel_sha_driver = { + .probe = atmel_sha_probe, + .remove = atmel_sha_remove, + .driver = { + .name = "atmel_sha", + .of_match_table = atmel_sha_dt_ids, + }, +}; + +module_platform_driver(atmel_sha_driver); + +MODULE_DESCRIPTION("Atmel SHA (1/256/224/384/512) hw acceleration support."); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique"); diff --git a/drivers/crypto/atmel-sha204a.c b/drivers/crypto/atmel-sha204a.c new file mode 100644 index 000000000..21a84c29c --- /dev/null +++ b/drivers/crypto/atmel-sha204a.c @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Microchip / Atmel SHA204A (I2C) driver. + * + * Copyright (c) 2019 Linaro, Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "atmel-i2c.h" + +/* + * According to review by Bill Cox [1], the ATSHA204 has very low entropy. + * [1] https://www.metzdowd.com/pipermail/cryptography/2014-December/023858.html + */ +static const unsigned short atsha204_quality = 1; + +static void atmel_sha204a_rng_done(struct atmel_i2c_work_data *work_data, + void *areq, int status) +{ + struct atmel_i2c_client_priv *i2c_priv = work_data->ctx; + struct hwrng *rng = areq; + + if (status) { + dev_warn_ratelimited(&i2c_priv->client->dev, + "i2c transaction failed (%d)\n", + status); + kfree_sensitive(work_data); + atomic_dec(&i2c_priv->tfm_count); + return; + } + + rng->priv = (unsigned long)work_data; + atomic_dec(&i2c_priv->tfm_count); +} + +static int atmel_sha204a_rng_read_nonblocking(struct hwrng *rng, void *data, + size_t max) +{ + struct atmel_i2c_client_priv *i2c_priv; + struct atmel_i2c_work_data *work_data; + + i2c_priv = container_of(rng, struct atmel_i2c_client_priv, hwrng); + + /* keep maximum 1 asynchronous read in flight at any time */ + if (!atomic_add_unless(&i2c_priv->tfm_count, 1, 1)) + return 0; + + if (rng->priv) { + work_data = (struct atmel_i2c_work_data *)rng->priv; + max = min(RANDOM_RSP_SIZE - CMD_OVERHEAD_SIZE, max); + memcpy(data, &work_data->cmd.data[RSP_DATA_IDX], max); + rng->priv = 0; + } else { + work_data = kmalloc_obj(*work_data, GFP_ATOMIC); + if (!work_data) { + atomic_dec(&i2c_priv->tfm_count); + return -ENOMEM; + } + work_data->ctx = i2c_priv; + work_data->client = i2c_priv->client; + + max = 0; + } + + atmel_i2c_init_random_cmd(&work_data->cmd); + atmel_i2c_enqueue(work_data, atmel_sha204a_rng_done, rng); + + return max; +} + +static int atmel_sha204a_rng_read(struct hwrng *rng, void *data, size_t max, + bool wait) +{ + struct atmel_i2c_client_priv *i2c_priv; + struct atmel_i2c_cmd cmd; + int ret; + + if (!wait) + return atmel_sha204a_rng_read_nonblocking(rng, data, max); + + i2c_priv = container_of(rng, struct atmel_i2c_client_priv, hwrng); + + atmel_i2c_init_random_cmd(&cmd); + + ret = atmel_i2c_send_receive(i2c_priv->client, &cmd); + if (ret) + goto out; + + max = min(RANDOM_RSP_SIZE - CMD_OVERHEAD_SIZE, max); + memcpy(data, &cmd.data[RSP_DATA_IDX], max); + ret = max; + +out: + memzero_explicit(&cmd, sizeof(cmd)); + return ret; +} + +static int atmel_sha204a_otp_read(struct i2c_client *client, u16 addr, u8 *otp) +{ + struct atmel_i2c_cmd cmd; + int ret; + + ret = atmel_i2c_init_read_otp_cmd(&cmd, addr); + if (ret < 0) { + dev_err(&client->dev, "failed, invalid otp address %04X\n", + addr); + return ret; + } + + ret = atmel_i2c_send_receive(client, &cmd); + if (ret < 0) { + dev_err(&client->dev, "failed to read otp at %04X\n", addr); + return ret; + } + + if (cmd.data[0] == 0xff) { + dev_err(&client->dev, "failed, device not ready\n"); + return -EIO; + } + + memcpy(otp, cmd.data+1, 4); + + return ret; +} + +static ssize_t otp_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + u16 addr; + u8 otp[OTP_ZONE_SIZE]; + struct i2c_client *client = to_i2c_client(dev); + ssize_t len = 0; + int i, ret; + + for (addr = 0; addr < OTP_ZONE_SIZE / 4; addr++) { + ret = atmel_sha204a_otp_read(client, addr, otp + addr * 4); + if (ret < 0) { + dev_err(dev, "failed to read otp zone\n"); + return ret; + } + } + + for (i = 0; i < OTP_ZONE_SIZE; i++) + len += sysfs_emit_at(buf, len, "%02X", otp[i]); + len += sysfs_emit_at(buf, len, "\n"); + return len; +} +static DEVICE_ATTR_RO(otp); + +static struct attribute *atmel_sha204a_attrs[] = { + &dev_attr_otp.attr, + NULL +}; + +static const struct attribute_group atmel_sha204a_groups = { + .name = "atsha204a", + .attrs = atmel_sha204a_attrs, +}; + +static int atmel_sha204a_probe(struct i2c_client *client) +{ + struct atmel_i2c_client_priv *i2c_priv; + const unsigned short *quality; + int ret; + + ret = atmel_i2c_probe(client); + if (ret) + return ret; + + i2c_priv = i2c_get_clientdata(client); + + memset(&i2c_priv->hwrng, 0, sizeof(i2c_priv->hwrng)); + + i2c_priv->hwrng.name = dev_name(&client->dev); + i2c_priv->hwrng.read = atmel_sha204a_rng_read; + + quality = i2c_get_match_data(client); + if (quality) + i2c_priv->hwrng.quality = *quality; + + ret = devm_hwrng_register(&client->dev, &i2c_priv->hwrng); + if (ret) { + dev_err(&client->dev, "failed to register RNG (%d)\n", ret); + return ret; + } + + ret = sysfs_create_group(&client->dev.kobj, &atmel_sha204a_groups); + if (ret) { + dev_err(&client->dev, "failed to create sysfs group (%d)\n", ret); + return ret; + } + + return ret; +} + +static void atmel_sha204a_remove(struct i2c_client *client) +{ + struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); + + sysfs_remove_group(&client->dev.kobj, &atmel_sha204a_groups); + devm_hwrng_unregister(&client->dev, &i2c_priv->hwrng); + atmel_i2c_flush_queue(); + + kfree_sensitive((void *)i2c_priv->hwrng.priv); +} + +static const struct of_device_id atmel_sha204a_dt_ids[] = { + { .compatible = "atmel,atsha204" }, + { .compatible = "atmel,atsha204a" }, + { } +}; +MODULE_DEVICE_TABLE(of, atmel_sha204a_dt_ids); + +static const struct i2c_device_id atmel_sha204a_id[] = { + { .name = "atsha204", .driver_data = (kernel_ulong_t)&atsha204_quality }, + { .name = "atsha204a", .driver_data = (kernel_ulong_t)NULL }, + { } +}; +MODULE_DEVICE_TABLE(i2c, atmel_sha204a_id); + +static struct i2c_driver atmel_sha204a_driver = { + .probe = atmel_sha204a_probe, + .remove = atmel_sha204a_remove, + .id_table = atmel_sha204a_id, + + .driver.name = "atmel-sha204a", + .driver.of_match_table = atmel_sha204a_dt_ids, +}; + +static int __init atmel_sha204a_init(void) +{ + return i2c_add_driver(&atmel_sha204a_driver); +} + +static void __exit atmel_sha204a_exit(void) +{ + atmel_i2c_flush_queue(); + i2c_del_driver(&atmel_sha204a_driver); +} + +module_init(atmel_sha204a_init); +module_exit(atmel_sha204a_exit); + +MODULE_AUTHOR("Ard Biesheuvel "); +MODULE_DESCRIPTION("Microchip / Atmel SHA204A (I2C) driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/crypto/atmel-tdes-regs.h b/drivers/crypto/atmel-tdes-regs.h new file mode 100644 index 000000000..fbd905720 --- /dev/null +++ b/drivers/crypto/atmel-tdes-regs.h @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ATMEL_TDES_REGS_H__ +#define __ATMEL_TDES_REGS_H__ + +#define TDES_CR 0x00 +#define TDES_CR_START (1 << 0) +#define TDES_CR_SWRST (1 << 8) +#define TDES_CR_LOADSEED (1 << 16) + +#define TDES_MR 0x04 +#define TDES_MR_CYPHER_DEC (0 << 0) +#define TDES_MR_CYPHER_ENC (1 << 0) +#define TDES_MR_TDESMOD_MASK (0x3 << 1) +#define TDES_MR_TDESMOD_DES (0x0 << 1) +#define TDES_MR_TDESMOD_TDES (0x1 << 1) +#define TDES_MR_TDESMOD_XTEA (0x2 << 1) +#define TDES_MR_KEYMOD_3KEY (0 << 4) +#define TDES_MR_KEYMOD_2KEY (1 << 4) +#define TDES_MR_SMOD_MASK (0x3 << 8) +#define TDES_MR_SMOD_MANUAL (0x0 << 8) +#define TDES_MR_SMOD_AUTO (0x1 << 8) +#define TDES_MR_SMOD_PDC (0x2 << 8) +#define TDES_MR_OPMOD_MASK (0x3 << 12) +#define TDES_MR_OPMOD_ECB (0x0 << 12) +#define TDES_MR_OPMOD_CBC (0x1 << 12) +#define TDES_MR_OPMOD_OFB (0x2 << 12) +#define TDES_MR_OPMOD_CFB (0x3 << 12) +#define TDES_MR_LOD (0x1 << 15) +#define TDES_MR_CFBS_MASK (0x3 << 16) +#define TDES_MR_CFBS_64b (0x0 << 16) +#define TDES_MR_CFBS_32b (0x1 << 16) +#define TDES_MR_CFBS_16b (0x2 << 16) +#define TDES_MR_CFBS_8b (0x3 << 16) +#define TDES_MR_CKEY_MASK (0xF << 20) +#define TDES_MR_CKEY_OFFSET 20 +#define TDES_MR_CTYPE_MASK (0x3F << 24) +#define TDES_MR_CTYPE_OFFSET 24 + +#define TDES_IER 0x10 +#define TDES_IDR 0x14 +#define TDES_IMR 0x18 +#define TDES_ISR 0x1C +#define TDES_INT_DATARDY (1 << 0) +#define TDES_INT_ENDRX (1 << 1) +#define TDES_INT_ENDTX (1 << 2) +#define TDES_INT_RXBUFF (1 << 3) +#define TDES_INT_TXBUFE (1 << 4) +#define TDES_INT_URAD (1 << 8) +#define TDES_ISR_URAT_MASK (0x3 << 12) +#define TDES_ISR_URAT_IDR (0x0 << 12) +#define TDES_ISR_URAT_ODR (0x1 << 12) +#define TDES_ISR_URAT_MR (0x2 << 12) +#define TDES_ISR_URAT_WO (0x3 << 12) + + +#define TDES_KEY1W1R 0x20 +#define TDES_KEY1W2R 0x24 +#define TDES_KEY2W1R 0x28 +#define TDES_KEY2W2R 0x2C +#define TDES_KEY3W1R 0x30 +#define TDES_KEY3W2R 0x34 +#define TDES_IDATA1R 0x40 +#define TDES_IDATA2R 0x44 +#define TDES_ODATA1R 0x50 +#define TDES_ODATA2R 0x54 +#define TDES_IV1R 0x60 +#define TDES_IV2R 0x64 + +#define TDES_XTEARNDR 0x70 +#define TDES_XTEARNDR_XTEA_RNDS_MASK (0x3F << 0) +#define TDES_XTEARNDR_XTEA_RNDS_OFFSET 0 + +#define TDES_HW_VERSION 0xFC + +#define TDES_RPR 0x100 +#define TDES_RCR 0x104 +#define TDES_TPR 0x108 +#define TDES_TCR 0x10C +#define TDES_RNPR 0x118 +#define TDES_RNCR 0x11C +#define TDES_TNPR 0x118 +#define TDES_TNCR 0x11C +#define TDES_PTCR 0x120 +#define TDES_PTCR_RXTEN (1 << 0) +#define TDES_PTCR_RXTDIS (1 << 1) +#define TDES_PTCR_TXTEN (1 << 8) +#define TDES_PTCR_TXTDIS (1 << 9) +#define TDES_PTSR 0x124 +#define TDES_PTSR_RXTEN (1 << 0) +#define TDES_PTSR_TXTEN (1 << 8) + +#endif /* __ATMEL_TDES_REGS_H__ */ diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c new file mode 100644 index 000000000..ffea1db74 --- /dev/null +++ b/drivers/crypto/atmel-tdes.c @@ -0,0 +1,1060 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Cryptographic API. + * + * Support for ATMEL DES/TDES HW acceleration. + * + * Copyright (c) 2012 Eukréa Electromatique - ATMEL + * Author: Nicolas Royer + * + * Some ideas are from omap-aes.c drivers. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "atmel-tdes-regs.h" + +#define ATMEL_TDES_PRIORITY 300 + +/* TDES flags */ +/* Reserve bits [17:16], [13:12], [2:0] for AES Mode Register */ +#define TDES_FLAGS_ENCRYPT TDES_MR_CYPHER_ENC +#define TDES_FLAGS_OPMODE_MASK (TDES_MR_OPMOD_MASK | TDES_MR_CFBS_MASK) +#define TDES_FLAGS_ECB TDES_MR_OPMOD_ECB +#define TDES_FLAGS_CBC TDES_MR_OPMOD_CBC + +#define TDES_FLAGS_MODE_MASK (TDES_FLAGS_OPMODE_MASK | TDES_FLAGS_ENCRYPT) + +#define TDES_FLAGS_INIT BIT(3) +#define TDES_FLAGS_FAST BIT(4) +#define TDES_FLAGS_BUSY BIT(5) +#define TDES_FLAGS_DMA BIT(6) + +#define ATMEL_TDES_QUEUE_LENGTH 50 + +struct atmel_tdes_caps { + bool has_dma; +}; + +struct atmel_tdes_dev; + +struct atmel_tdes_ctx { + struct atmel_tdes_dev *dd; + + int keylen; + u32 key[DES3_EDE_KEY_SIZE / sizeof(u32)]; + unsigned long flags; + + u16 block_size; +}; + +struct atmel_tdes_reqctx { + unsigned long mode; + u8 lastc[DES_BLOCK_SIZE]; +}; + +struct atmel_tdes_dma { + struct dma_chan *chan; + struct dma_slave_config dma_conf; +}; + +struct atmel_tdes_dev { + struct list_head list; + unsigned long phys_base; + void __iomem *io_base; + + struct atmel_tdes_ctx *ctx; + struct device *dev; + struct clk *iclk; + int irq; + + unsigned long flags; + + spinlock_t lock; + struct crypto_queue queue; + + struct tasklet_struct done_task; + struct tasklet_struct queue_task; + + struct skcipher_request *req; + size_t total; + + struct scatterlist *in_sg; + unsigned int nb_in_sg; + size_t in_offset; + struct scatterlist *out_sg; + unsigned int nb_out_sg; + size_t out_offset; + + size_t buflen; + size_t dma_size; + + void *buf_in; + int dma_in; + dma_addr_t dma_addr_in; + struct atmel_tdes_dma dma_lch_in; + + void *buf_out; + int dma_out; + dma_addr_t dma_addr_out; + struct atmel_tdes_dma dma_lch_out; + + struct atmel_tdes_caps caps; + + u32 hw_version; +}; + +struct atmel_tdes_drv { + struct list_head dev_list; + spinlock_t lock; +}; + +static struct atmel_tdes_drv atmel_tdes = { + .dev_list = LIST_HEAD_INIT(atmel_tdes.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(atmel_tdes.lock), +}; + +static int atmel_tdes_sg_copy(struct scatterlist **sg, size_t *offset, + void *buf, size_t buflen, size_t total, int out) +{ + size_t count, off = 0; + + while (buflen && total) { + count = min((*sg)->length - *offset, total); + count = min(count, buflen); + + if (!count) + return off; + + scatterwalk_map_and_copy(buf + off, *sg, *offset, count, out); + + off += count; + buflen -= count; + *offset += count; + total -= count; + + if (*offset == (*sg)->length) { + *sg = sg_next(*sg); + if (*sg) + *offset = 0; + else + total = 0; + } + } + + return off; +} + +static inline u32 atmel_tdes_read(struct atmel_tdes_dev *dd, u32 offset) +{ + return readl_relaxed(dd->io_base + offset); +} + +static inline void atmel_tdes_write(struct atmel_tdes_dev *dd, + u32 offset, u32 value) +{ + writel_relaxed(value, dd->io_base + offset); +} + +static void atmel_tdes_write_n(struct atmel_tdes_dev *dd, u32 offset, + const u32 *value, int count) +{ + for (; count--; value++, offset += 4) + atmel_tdes_write(dd, offset, *value); +} + +static struct atmel_tdes_dev *atmel_tdes_dev_alloc(void) +{ + struct atmel_tdes_dev *tdes_dd; + + spin_lock_bh(&atmel_tdes.lock); + /* One TDES IP per SoC. */ + tdes_dd = list_first_entry_or_null(&atmel_tdes.dev_list, + struct atmel_tdes_dev, list); + spin_unlock_bh(&atmel_tdes.lock); + return tdes_dd; +} + +static int atmel_tdes_hw_init(struct atmel_tdes_dev *dd) +{ + int err; + + err = clk_prepare_enable(dd->iclk); + if (err) + return err; + + if (!(dd->flags & TDES_FLAGS_INIT)) { + atmel_tdes_write(dd, TDES_CR, TDES_CR_SWRST); + dd->flags |= TDES_FLAGS_INIT; + } + + return 0; +} + +static inline unsigned int atmel_tdes_get_version(struct atmel_tdes_dev *dd) +{ + return atmel_tdes_read(dd, TDES_HW_VERSION) & 0x00000fff; +} + +static int atmel_tdes_hw_version_init(struct atmel_tdes_dev *dd) +{ + int err; + + err = atmel_tdes_hw_init(dd); + if (err) + return err; + + dd->hw_version = atmel_tdes_get_version(dd); + + dev_info(dd->dev, + "version: 0x%x\n", dd->hw_version); + + clk_disable_unprepare(dd->iclk); + + return 0; +} + +static void atmel_tdes_dma_callback(void *data) +{ + struct atmel_tdes_dev *dd = data; + + /* dma_lch_out - completed */ + tasklet_schedule(&dd->done_task); +} + +static int atmel_tdes_write_ctrl(struct atmel_tdes_dev *dd) +{ + int err; + u32 valmr = TDES_MR_SMOD_PDC; + + err = atmel_tdes_hw_init(dd); + + if (err) + return err; + + if (!dd->caps.has_dma) + atmel_tdes_write(dd, TDES_PTCR, + TDES_PTCR_TXTDIS | TDES_PTCR_RXTDIS); + + /* MR register must be set before IV registers */ + if (dd->ctx->keylen > (DES_KEY_SIZE << 1)) { + valmr |= TDES_MR_KEYMOD_3KEY; + valmr |= TDES_MR_TDESMOD_TDES; + } else if (dd->ctx->keylen > DES_KEY_SIZE) { + valmr |= TDES_MR_KEYMOD_2KEY; + valmr |= TDES_MR_TDESMOD_TDES; + } else { + valmr |= TDES_MR_TDESMOD_DES; + } + + valmr |= dd->flags & TDES_FLAGS_MODE_MASK; + + atmel_tdes_write(dd, TDES_MR, valmr); + + atmel_tdes_write_n(dd, TDES_KEY1W1R, dd->ctx->key, + dd->ctx->keylen >> 2); + + if (dd->req->iv && (valmr & TDES_MR_OPMOD_MASK) != TDES_MR_OPMOD_ECB) + atmel_tdes_write_n(dd, TDES_IV1R, (void *)dd->req->iv, 2); + + return 0; +} + +static int atmel_tdes_crypt_pdc_stop(struct atmel_tdes_dev *dd) +{ + size_t count; + + atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTDIS|TDES_PTCR_RXTDIS); + + if (dd->flags & TDES_FLAGS_FAST) { + dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); + dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); + } else { + dma_sync_single_for_cpu(dd->dev, dd->dma_addr_out, + dd->dma_size, DMA_FROM_DEVICE); + + count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset, + dd->buf_out, dd->buflen, dd->dma_size, 1); + if (count != dd->dma_size) { + dev_dbg(dd->dev, "not all data converted: %zu\n", count); + return -EINVAL; + } + } + + return 0; +} + +static int atmel_tdes_buff_init(struct atmel_tdes_dev *dd) +{ + int err = -ENOMEM; + + dd->buf_in = (void *)__get_free_page(GFP_KERNEL); + dd->buf_out = (void *)__get_free_page(GFP_KERNEL); + dd->buflen = PAGE_SIZE; + dd->buflen &= ~(DES_BLOCK_SIZE - 1); + + if (!dd->buf_in || !dd->buf_out) { + dev_dbg(dd->dev, "unable to alloc pages.\n"); + goto err_alloc; + } + + /* MAP here */ + dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in, + dd->buflen, DMA_TO_DEVICE); + err = dma_mapping_error(dd->dev, dd->dma_addr_in); + if (err) { + dev_dbg(dd->dev, "dma %zd bytes error\n", dd->buflen); + goto err_map_in; + } + + dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out, + dd->buflen, DMA_FROM_DEVICE); + err = dma_mapping_error(dd->dev, dd->dma_addr_out); + if (err) { + dev_dbg(dd->dev, "dma %zd bytes error\n", dd->buflen); + goto err_map_out; + } + + return 0; + +err_map_out: + dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, + DMA_TO_DEVICE); +err_map_in: +err_alloc: + free_page((unsigned long)dd->buf_out); + free_page((unsigned long)dd->buf_in); + return err; +} + +static void atmel_tdes_buff_cleanup(struct atmel_tdes_dev *dd) +{ + dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen, + DMA_FROM_DEVICE); + dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, + DMA_TO_DEVICE); + free_page((unsigned long)dd->buf_out); + free_page((unsigned long)dd->buf_in); +} + +static int atmel_tdes_crypt_pdc(struct atmel_tdes_dev *dd, + dma_addr_t dma_addr_in, + dma_addr_t dma_addr_out, int length) +{ + int len32; + + dd->dma_size = length; + + if (!(dd->flags & TDES_FLAGS_FAST)) { + dma_sync_single_for_device(dd->dev, dma_addr_in, length, + DMA_TO_DEVICE); + } + + len32 = DIV_ROUND_UP(length, sizeof(u32)); + + atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTDIS|TDES_PTCR_RXTDIS); + atmel_tdes_write(dd, TDES_TPR, dma_addr_in); + atmel_tdes_write(dd, TDES_TCR, len32); + atmel_tdes_write(dd, TDES_RPR, dma_addr_out); + atmel_tdes_write(dd, TDES_RCR, len32); + + /* Enable Interrupt */ + atmel_tdes_write(dd, TDES_IER, TDES_INT_ENDRX); + + /* Start DMA transfer */ + atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTEN | TDES_PTCR_RXTEN); + + return 0; +} + +static int atmel_tdes_crypt_dma(struct atmel_tdes_dev *dd, + dma_addr_t dma_addr_in, + dma_addr_t dma_addr_out, int length) +{ + struct scatterlist sg[2]; + struct dma_async_tx_descriptor *in_desc, *out_desc; + enum dma_slave_buswidth addr_width; + + dd->dma_size = length; + + if (!(dd->flags & TDES_FLAGS_FAST)) { + dma_sync_single_for_device(dd->dev, dma_addr_in, length, + DMA_TO_DEVICE); + } + + addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + + dd->dma_lch_in.dma_conf.dst_addr_width = addr_width; + dd->dma_lch_out.dma_conf.src_addr_width = addr_width; + + dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf); + dmaengine_slave_config(dd->dma_lch_out.chan, &dd->dma_lch_out.dma_conf); + + dd->flags |= TDES_FLAGS_DMA; + + sg_init_table(&sg[0], 1); + sg_dma_address(&sg[0]) = dma_addr_in; + sg_dma_len(&sg[0]) = length; + + sg_init_table(&sg[1], 1); + sg_dma_address(&sg[1]) = dma_addr_out; + sg_dma_len(&sg[1]) = length; + + in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, &sg[0], + 1, DMA_MEM_TO_DEV, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!in_desc) + return -EINVAL; + + out_desc = dmaengine_prep_slave_sg(dd->dma_lch_out.chan, &sg[1], + 1, DMA_DEV_TO_MEM, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!out_desc) + return -EINVAL; + + out_desc->callback = atmel_tdes_dma_callback; + out_desc->callback_param = dd; + + dmaengine_submit(out_desc); + dma_async_issue_pending(dd->dma_lch_out.chan); + + dmaengine_submit(in_desc); + dma_async_issue_pending(dd->dma_lch_in.chan); + + return 0; +} + +static int atmel_tdes_crypt_start(struct atmel_tdes_dev *dd) +{ + bool fast; + int err; + size_t count; + dma_addr_t addr_in, addr_out; + + fast = !dd->in_offset && !dd->out_offset && + dd->in_sg->length == dd->out_sg->length && + IS_ALIGNED(dd->in_sg->offset, sizeof(u32)) && + IS_ALIGNED(dd->out_sg->offset, sizeof(u32)) && + IS_ALIGNED(dd->in_sg->length, dd->ctx->block_size); + + if (fast) { + count = min(dd->total, dd->in_sg->length); + + err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); + if (!err) { + dev_dbg(dd->dev, "dma_map_sg() error\n"); + return -EINVAL; + } + + err = dma_map_sg(dd->dev, dd->out_sg, 1, + DMA_FROM_DEVICE); + if (!err) { + dev_dbg(dd->dev, "dma_map_sg() error\n"); + dma_unmap_sg(dd->dev, dd->in_sg, 1, + DMA_TO_DEVICE); + return -EINVAL; + } + + addr_in = sg_dma_address(dd->in_sg); + addr_out = sg_dma_address(dd->out_sg); + + dd->flags |= TDES_FLAGS_FAST; + + } else { + /* use cache buffers */ + count = atmel_tdes_sg_copy(&dd->in_sg, &dd->in_offset, + dd->buf_in, dd->buflen, dd->total, 0); + + addr_in = dd->dma_addr_in; + addr_out = dd->dma_addr_out; + + dd->flags &= ~TDES_FLAGS_FAST; + } + + dd->total -= count; + + if (dd->caps.has_dma) + err = atmel_tdes_crypt_dma(dd, addr_in, addr_out, count); + else + err = atmel_tdes_crypt_pdc(dd, addr_in, addr_out, count); + + if (err && (dd->flags & TDES_FLAGS_FAST)) { + dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); + dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); + } + + return err; +} + +static void +atmel_tdes_set_iv_as_last_ciphertext_block(struct atmel_tdes_dev *dd) +{ + struct skcipher_request *req = dd->req; + struct atmel_tdes_reqctx *rctx = skcipher_request_ctx(req); + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + unsigned int ivsize = crypto_skcipher_ivsize(skcipher); + + if (req->cryptlen < ivsize) + return; + + if (rctx->mode & TDES_FLAGS_ENCRYPT) + scatterwalk_map_and_copy(req->iv, req->dst, + req->cryptlen - ivsize, ivsize, 0); + else + memcpy(req->iv, rctx->lastc, ivsize); + +} + +static void atmel_tdes_finish_req(struct atmel_tdes_dev *dd, int err) +{ + struct skcipher_request *req = dd->req; + struct atmel_tdes_reqctx *rctx = skcipher_request_ctx(req); + + clk_disable_unprepare(dd->iclk); + + dd->flags &= ~TDES_FLAGS_BUSY; + + if (!err && (rctx->mode & TDES_FLAGS_OPMODE_MASK) != TDES_FLAGS_ECB) + atmel_tdes_set_iv_as_last_ciphertext_block(dd); + + skcipher_request_complete(req, err); +} + +static int atmel_tdes_handle_queue(struct atmel_tdes_dev *dd, + struct skcipher_request *req) +{ + struct crypto_async_request *async_req, *backlog; + struct atmel_tdes_ctx *ctx; + struct atmel_tdes_reqctx *rctx; + unsigned long flags; + int err, ret = 0; + + spin_lock_irqsave(&dd->lock, flags); + if (req) + ret = crypto_enqueue_request(&dd->queue, &req->base); + if (dd->flags & TDES_FLAGS_BUSY) { + spin_unlock_irqrestore(&dd->lock, flags); + return ret; + } + backlog = crypto_get_backlog(&dd->queue); + async_req = crypto_dequeue_request(&dd->queue); + if (async_req) + dd->flags |= TDES_FLAGS_BUSY; + spin_unlock_irqrestore(&dd->lock, flags); + + if (!async_req) + return ret; + + if (backlog) + crypto_request_complete(backlog, -EINPROGRESS); + + req = skcipher_request_cast(async_req); + + /* assign new request to device */ + dd->req = req; + dd->total = req->cryptlen; + dd->in_offset = 0; + dd->in_sg = req->src; + dd->out_offset = 0; + dd->out_sg = req->dst; + + rctx = skcipher_request_ctx(req); + ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + rctx->mode &= TDES_FLAGS_MODE_MASK; + dd->flags = (dd->flags & ~TDES_FLAGS_MODE_MASK) | rctx->mode; + dd->ctx = ctx; + + err = atmel_tdes_write_ctrl(dd); + if (!err) + err = atmel_tdes_crypt_start(dd); + if (err) { + /* des_task will not finish it, so do it here */ + atmel_tdes_finish_req(dd, err); + tasklet_schedule(&dd->queue_task); + } + + return ret; +} + +static int atmel_tdes_crypt_dma_stop(struct atmel_tdes_dev *dd) +{ + size_t count; + + if (dd->flags & TDES_FLAGS_FAST) { + dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); + dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); + } else { + dma_sync_single_for_cpu(dd->dev, dd->dma_addr_out, dd->dma_size, + DMA_FROM_DEVICE); + + count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset, + dd->buf_out, dd->buflen, + dd->dma_size, 1); + if (count != dd->dma_size) { + dev_dbg(dd->dev, "not all data converted: %zu\n", count); + return -EINVAL; + } + } + + return 0; +} + +static int atmel_tdes_crypt(struct skcipher_request *req, unsigned long mode) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + struct atmel_tdes_ctx *ctx = crypto_skcipher_ctx(skcipher); + struct atmel_tdes_reqctx *rctx = skcipher_request_ctx(req); + struct device *dev = ctx->dd->dev; + + if (!req->cryptlen) + return 0; + + if (!IS_ALIGNED(req->cryptlen, DES_BLOCK_SIZE)) { + dev_dbg(dev, "request size is not exact amount of DES blocks\n"); + return -EINVAL; + } + ctx->block_size = DES_BLOCK_SIZE; + + rctx->mode = mode; + + if ((mode & TDES_FLAGS_OPMODE_MASK) != TDES_FLAGS_ECB && + !(mode & TDES_FLAGS_ENCRYPT)) { + unsigned int ivsize = crypto_skcipher_ivsize(skcipher); + + if (req->cryptlen >= ivsize) + scatterwalk_map_and_copy(rctx->lastc, req->src, + req->cryptlen - ivsize, + ivsize, 0); + } + + return atmel_tdes_handle_queue(ctx->dd, req); +} + +static int atmel_tdes_dma_init(struct atmel_tdes_dev *dd) +{ + int ret; + + /* Try to grab 2 DMA channels */ + dd->dma_lch_in.chan = dma_request_chan(dd->dev, "tx"); + if (IS_ERR(dd->dma_lch_in.chan)) { + ret = PTR_ERR(dd->dma_lch_in.chan); + goto err_dma_in; + } + + dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + + TDES_IDATA1R; + dd->dma_lch_in.dma_conf.src_maxburst = 1; + dd->dma_lch_in.dma_conf.src_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_in.dma_conf.dst_maxburst = 1; + dd->dma_lch_in.dma_conf.dst_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_in.dma_conf.device_fc = false; + + dd->dma_lch_out.chan = dma_request_chan(dd->dev, "rx"); + if (IS_ERR(dd->dma_lch_out.chan)) { + ret = PTR_ERR(dd->dma_lch_out.chan); + goto err_dma_out; + } + + dd->dma_lch_out.dma_conf.src_addr = dd->phys_base + + TDES_ODATA1R; + dd->dma_lch_out.dma_conf.src_maxburst = 1; + dd->dma_lch_out.dma_conf.src_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_out.dma_conf.dst_maxburst = 1; + dd->dma_lch_out.dma_conf.dst_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_out.dma_conf.device_fc = false; + + return 0; + +err_dma_out: + dma_release_channel(dd->dma_lch_in.chan); +err_dma_in: + dev_err(dd->dev, "no DMA channel available\n"); + return ret; +} + +static void atmel_tdes_dma_cleanup(struct atmel_tdes_dev *dd) +{ + dma_release_channel(dd->dma_lch_in.chan); + dma_release_channel(dd->dma_lch_out.chan); +} + +static int atmel_des_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct atmel_tdes_ctx *ctx = crypto_skcipher_ctx(tfm); + int err; + + err = verify_skcipher_des_key(tfm, key); + if (err) + return err; + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + return 0; +} + +static int atmel_tdes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct atmel_tdes_ctx *ctx = crypto_skcipher_ctx(tfm); + int err; + + err = verify_skcipher_des3_key(tfm, key); + if (err) + return err; + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + return 0; +} + +static int atmel_tdes_ecb_encrypt(struct skcipher_request *req) +{ + return atmel_tdes_crypt(req, TDES_FLAGS_ECB | TDES_FLAGS_ENCRYPT); +} + +static int atmel_tdes_ecb_decrypt(struct skcipher_request *req) +{ + return atmel_tdes_crypt(req, TDES_FLAGS_ECB); +} + +static int atmel_tdes_cbc_encrypt(struct skcipher_request *req) +{ + return atmel_tdes_crypt(req, TDES_FLAGS_CBC | TDES_FLAGS_ENCRYPT); +} + +static int atmel_tdes_cbc_decrypt(struct skcipher_request *req) +{ + return atmel_tdes_crypt(req, TDES_FLAGS_CBC); +} + +static int atmel_tdes_init_tfm(struct crypto_skcipher *tfm) +{ + struct atmel_tdes_ctx *ctx = crypto_skcipher_ctx(tfm); + + ctx->dd = atmel_tdes_dev_alloc(); + if (!ctx->dd) + return -ENODEV; + + crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_tdes_reqctx)); + + return 0; +} + +static void atmel_tdes_skcipher_alg_init(struct skcipher_alg *alg) +{ + alg->base.cra_priority = ATMEL_TDES_PRIORITY; + alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; + alg->base.cra_ctxsize = sizeof(struct atmel_tdes_ctx); + alg->base.cra_module = THIS_MODULE; + + alg->init = atmel_tdes_init_tfm; +} + +static struct skcipher_alg tdes_algs[] = { +{ + .base.cra_name = "ecb(des)", + .base.cra_driver_name = "atmel-ecb-des", + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_alignmask = 0x7, + + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = atmel_des_setkey, + .encrypt = atmel_tdes_ecb_encrypt, + .decrypt = atmel_tdes_ecb_decrypt, +}, +{ + .base.cra_name = "cbc(des)", + .base.cra_driver_name = "atmel-cbc-des", + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_alignmask = 0x7, + + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = atmel_des_setkey, + .encrypt = atmel_tdes_cbc_encrypt, + .decrypt = atmel_tdes_cbc_decrypt, +}, +{ + .base.cra_name = "ecb(des3_ede)", + .base.cra_driver_name = "atmel-ecb-tdes", + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_alignmask = 0x7, + + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = atmel_tdes_setkey, + .encrypt = atmel_tdes_ecb_encrypt, + .decrypt = atmel_tdes_ecb_decrypt, +}, +{ + .base.cra_name = "cbc(des3_ede)", + .base.cra_driver_name = "atmel-cbc-tdes", + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_alignmask = 0x7, + + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = atmel_tdes_setkey, + .encrypt = atmel_tdes_cbc_encrypt, + .decrypt = atmel_tdes_cbc_decrypt, + .ivsize = DES_BLOCK_SIZE, +}, +}; + +static void atmel_tdes_queue_task(unsigned long data) +{ + struct atmel_tdes_dev *dd = (struct atmel_tdes_dev *)data; + + atmel_tdes_handle_queue(dd, NULL); +} + +static void atmel_tdes_done_task(unsigned long data) +{ + struct atmel_tdes_dev *dd = (struct atmel_tdes_dev *) data; + int err; + + if (!(dd->flags & TDES_FLAGS_DMA)) + err = atmel_tdes_crypt_pdc_stop(dd); + else + err = atmel_tdes_crypt_dma_stop(dd); + + if (dd->total && !err) { + if (dd->flags & TDES_FLAGS_FAST) { + dd->in_sg = sg_next(dd->in_sg); + dd->out_sg = sg_next(dd->out_sg); + if (!dd->in_sg || !dd->out_sg) + err = -EINVAL; + } + if (!err) + err = atmel_tdes_crypt_start(dd); + if (!err) + return; /* DMA started. Not finishing. */ + } + + atmel_tdes_finish_req(dd, err); + atmel_tdes_handle_queue(dd, NULL); +} + +static irqreturn_t atmel_tdes_irq(int irq, void *dev_id) +{ + struct atmel_tdes_dev *tdes_dd = dev_id; + u32 reg; + + reg = atmel_tdes_read(tdes_dd, TDES_ISR); + if (reg & atmel_tdes_read(tdes_dd, TDES_IMR)) { + atmel_tdes_write(tdes_dd, TDES_IDR, reg); + if (TDES_FLAGS_BUSY & tdes_dd->flags) + tasklet_schedule(&tdes_dd->done_task); + else + dev_warn(tdes_dd->dev, "TDES interrupt when no active requests.\n"); + return IRQ_HANDLED; + } + + return IRQ_NONE; +} + +static int atmel_tdes_register_algs(struct atmel_tdes_dev *dd) +{ + int err, i; + + for (i = 0; i < ARRAY_SIZE(tdes_algs); i++) { + atmel_tdes_skcipher_alg_init(&tdes_algs[i]); + + err = crypto_register_skcipher(&tdes_algs[i]); + if (err) { + crypto_unregister_skciphers(tdes_algs, i); + return err; + } + } + + return 0; +} + +static void atmel_tdes_get_cap(struct atmel_tdes_dev *dd) +{ + dd->caps.has_dma = 0; + + /* keep only major version number */ + switch (dd->hw_version & 0xf00) { + case 0x800: + case 0x700: + dd->caps.has_dma = 1; + break; + case 0x600: + break; + default: + dev_warn(dd->dev, + "Unmanaged tdes version, set minimum capabilities\n"); + break; + } +} + +static const struct of_device_id atmel_tdes_dt_ids[] = { + { .compatible = "atmel,at91sam9g46-tdes" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, atmel_tdes_dt_ids); + +static int atmel_tdes_probe(struct platform_device *pdev) +{ + struct atmel_tdes_dev *tdes_dd; + struct device *dev = &pdev->dev; + struct resource *tdes_res; + int err; + + tdes_dd = devm_kmalloc(&pdev->dev, sizeof(*tdes_dd), GFP_KERNEL); + if (!tdes_dd) + return -ENOMEM; + + tdes_dd->dev = dev; + + platform_set_drvdata(pdev, tdes_dd); + + INIT_LIST_HEAD(&tdes_dd->list); + spin_lock_init(&tdes_dd->lock); + + tasklet_init(&tdes_dd->done_task, atmel_tdes_done_task, + (unsigned long)tdes_dd); + tasklet_init(&tdes_dd->queue_task, atmel_tdes_queue_task, + (unsigned long)tdes_dd); + + crypto_init_queue(&tdes_dd->queue, ATMEL_TDES_QUEUE_LENGTH); + + tdes_dd->io_base = devm_platform_get_and_ioremap_resource(pdev, 0, &tdes_res); + if (IS_ERR(tdes_dd->io_base)) { + err = PTR_ERR(tdes_dd->io_base); + goto err_tasklet_kill; + } + tdes_dd->phys_base = tdes_res->start; + + /* Get the IRQ */ + tdes_dd->irq = platform_get_irq(pdev, 0); + if (tdes_dd->irq < 0) { + err = tdes_dd->irq; + goto err_tasklet_kill; + } + + err = devm_request_irq(&pdev->dev, tdes_dd->irq, atmel_tdes_irq, + IRQF_SHARED, "atmel-tdes", tdes_dd); + if (err) + goto err_tasklet_kill; + + /* Initializing the clock */ + tdes_dd->iclk = devm_clk_get(&pdev->dev, "tdes_clk"); + if (IS_ERR(tdes_dd->iclk)) { + dev_err(dev, "clock initialization failed.\n"); + err = PTR_ERR(tdes_dd->iclk); + goto err_tasklet_kill; + } + + err = atmel_tdes_hw_version_init(tdes_dd); + if (err) + goto err_tasklet_kill; + + atmel_tdes_get_cap(tdes_dd); + + err = atmel_tdes_buff_init(tdes_dd); + if (err) + goto err_tasklet_kill; + + if (tdes_dd->caps.has_dma) { + err = atmel_tdes_dma_init(tdes_dd); + if (err) + goto err_buff_cleanup; + + dev_info(dev, "using %s, %s for DMA transfers\n", + dma_chan_name(tdes_dd->dma_lch_in.chan), + dma_chan_name(tdes_dd->dma_lch_out.chan)); + } + + spin_lock(&atmel_tdes.lock); + list_add_tail(&tdes_dd->list, &atmel_tdes.dev_list); + spin_unlock(&atmel_tdes.lock); + + err = atmel_tdes_register_algs(tdes_dd); + if (err) + goto err_algs; + + dev_info(dev, "Atmel DES/TDES\n"); + + return 0; + +err_algs: + spin_lock(&atmel_tdes.lock); + list_del(&tdes_dd->list); + spin_unlock(&atmel_tdes.lock); + if (tdes_dd->caps.has_dma) + atmel_tdes_dma_cleanup(tdes_dd); +err_buff_cleanup: + atmel_tdes_buff_cleanup(tdes_dd); +err_tasklet_kill: + tasklet_kill(&tdes_dd->done_task); + tasklet_kill(&tdes_dd->queue_task); + + return err; +} + +static void atmel_tdes_remove(struct platform_device *pdev) +{ + struct atmel_tdes_dev *tdes_dd = platform_get_drvdata(pdev); + + spin_lock(&atmel_tdes.lock); + list_del(&tdes_dd->list); + spin_unlock(&atmel_tdes.lock); + + crypto_unregister_skciphers(tdes_algs, ARRAY_SIZE(tdes_algs)); + + tasklet_kill(&tdes_dd->done_task); + tasklet_kill(&tdes_dd->queue_task); + + if (tdes_dd->caps.has_dma) + atmel_tdes_dma_cleanup(tdes_dd); + + atmel_tdes_buff_cleanup(tdes_dd); +} + +static struct platform_driver atmel_tdes_driver = { + .probe = atmel_tdes_probe, + .remove = atmel_tdes_remove, + .driver = { + .name = "atmel_tdes", + .of_match_table = atmel_tdes_dt_ids, + }, +}; + +module_platform_driver(atmel_tdes_driver); + +MODULE_DESCRIPTION("Atmel DES/TDES hw acceleration support."); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique"); diff --git a/drivers/crypto/axis/Makefile b/drivers/crypto/axis/Makefile new file mode 100644 index 000000000..146e09e4c --- /dev/null +++ b/drivers/crypto/axis/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_ARTPEC6) := artpec6_crypto.o diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c new file mode 100644 index 000000000..2bf8d71c8 --- /dev/null +++ b/drivers/crypto/axis/artpec6_crypto.c @@ -0,0 +1,2980 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Driver for ARTPEC-6 crypto block using the kernel asynchronous crypto api. + * + * Copyright (C) 2014-2017 Axis Communications AB + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Max length of a line in all cache levels for Artpec SoCs. */ +#define ARTPEC_CACHE_LINE_MAX 32 + +#define PDMA_OUT_CFG 0x0000 +#define PDMA_OUT_BUF_CFG 0x0004 +#define PDMA_OUT_CMD 0x0008 +#define PDMA_OUT_DESCRQ_PUSH 0x0010 +#define PDMA_OUT_DESCRQ_STAT 0x0014 + +#define A6_PDMA_IN_CFG 0x0028 +#define A6_PDMA_IN_BUF_CFG 0x002c +#define A6_PDMA_IN_CMD 0x0030 +#define A6_PDMA_IN_STATQ_PUSH 0x0038 +#define A6_PDMA_IN_DESCRQ_PUSH 0x0044 +#define A6_PDMA_IN_DESCRQ_STAT 0x0048 +#define A6_PDMA_INTR_MASK 0x0068 +#define A6_PDMA_ACK_INTR 0x006c +#define A6_PDMA_MASKED_INTR 0x0074 + +#define A7_PDMA_IN_CFG 0x002c +#define A7_PDMA_IN_BUF_CFG 0x0030 +#define A7_PDMA_IN_CMD 0x0034 +#define A7_PDMA_IN_STATQ_PUSH 0x003c +#define A7_PDMA_IN_DESCRQ_PUSH 0x0048 +#define A7_PDMA_IN_DESCRQ_STAT 0x004C +#define A7_PDMA_INTR_MASK 0x006c +#define A7_PDMA_ACK_INTR 0x0070 +#define A7_PDMA_MASKED_INTR 0x0078 + +#define PDMA_OUT_CFG_EN BIT(0) + +#define PDMA_OUT_BUF_CFG_DATA_BUF_SIZE GENMASK(4, 0) +#define PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE GENMASK(9, 5) + +#define PDMA_OUT_CMD_START BIT(0) +#define A6_PDMA_OUT_CMD_STOP BIT(3) +#define A7_PDMA_OUT_CMD_STOP BIT(2) + +#define PDMA_OUT_DESCRQ_PUSH_LEN GENMASK(5, 0) +#define PDMA_OUT_DESCRQ_PUSH_ADDR GENMASK(31, 6) + +#define PDMA_OUT_DESCRQ_STAT_LEVEL GENMASK(3, 0) +#define PDMA_OUT_DESCRQ_STAT_SIZE GENMASK(7, 4) + +#define PDMA_IN_CFG_EN BIT(0) + +#define PDMA_IN_BUF_CFG_DATA_BUF_SIZE GENMASK(4, 0) +#define PDMA_IN_BUF_CFG_DESCR_BUF_SIZE GENMASK(9, 5) +#define PDMA_IN_BUF_CFG_STAT_BUF_SIZE GENMASK(14, 10) + +#define PDMA_IN_CMD_START BIT(0) +#define A6_PDMA_IN_CMD_FLUSH_STAT BIT(2) +#define A6_PDMA_IN_CMD_STOP BIT(3) +#define A7_PDMA_IN_CMD_FLUSH_STAT BIT(1) +#define A7_PDMA_IN_CMD_STOP BIT(2) + +#define PDMA_IN_STATQ_PUSH_LEN GENMASK(5, 0) +#define PDMA_IN_STATQ_PUSH_ADDR GENMASK(31, 6) + +#define PDMA_IN_DESCRQ_PUSH_LEN GENMASK(5, 0) +#define PDMA_IN_DESCRQ_PUSH_ADDR GENMASK(31, 6) + +#define PDMA_IN_DESCRQ_STAT_LEVEL GENMASK(3, 0) +#define PDMA_IN_DESCRQ_STAT_SIZE GENMASK(7, 4) + +#define A6_PDMA_INTR_MASK_IN_DATA BIT(2) +#define A6_PDMA_INTR_MASK_IN_EOP BIT(3) +#define A6_PDMA_INTR_MASK_IN_EOP_FLUSH BIT(4) + +#define A7_PDMA_INTR_MASK_IN_DATA BIT(3) +#define A7_PDMA_INTR_MASK_IN_EOP BIT(4) +#define A7_PDMA_INTR_MASK_IN_EOP_FLUSH BIT(5) + +#define A6_CRY_MD_OPER GENMASK(19, 16) + +#define A6_CRY_MD_HASH_SEL_CTX GENMASK(21, 20) +#define A6_CRY_MD_HASH_HMAC_FIN BIT(23) + +#define A6_CRY_MD_CIPHER_LEN GENMASK(21, 20) +#define A6_CRY_MD_CIPHER_DECR BIT(22) +#define A6_CRY_MD_CIPHER_TWEAK BIT(23) +#define A6_CRY_MD_CIPHER_DSEQ BIT(24) + +#define A7_CRY_MD_OPER GENMASK(11, 8) + +#define A7_CRY_MD_HASH_SEL_CTX GENMASK(13, 12) +#define A7_CRY_MD_HASH_HMAC_FIN BIT(15) + +#define A7_CRY_MD_CIPHER_LEN GENMASK(13, 12) +#define A7_CRY_MD_CIPHER_DECR BIT(14) +#define A7_CRY_MD_CIPHER_TWEAK BIT(15) +#define A7_CRY_MD_CIPHER_DSEQ BIT(16) + +/* DMA metadata constants */ +#define regk_crypto_aes_cbc 0x00000002 +#define regk_crypto_aes_ctr 0x00000003 +#define regk_crypto_aes_ecb 0x00000001 +#define regk_crypto_aes_gcm 0x00000004 +#define regk_crypto_aes_xts 0x00000005 +#define regk_crypto_cache 0x00000002 +#define a6_regk_crypto_dlkey 0x0000000a +#define a7_regk_crypto_dlkey 0x0000000e +#define regk_crypto_ext 0x00000001 +#define regk_crypto_hmac_sha1 0x00000007 +#define regk_crypto_hmac_sha256 0x00000009 +#define regk_crypto_init 0x00000000 +#define regk_crypto_key_128 0x00000000 +#define regk_crypto_key_192 0x00000001 +#define regk_crypto_key_256 0x00000002 +#define regk_crypto_null 0x00000000 +#define regk_crypto_sha1 0x00000006 +#define regk_crypto_sha256 0x00000008 + +/* DMA descriptor structures */ +struct pdma_descr_ctrl { + unsigned char short_descr : 1; + unsigned char pad1 : 1; + unsigned char eop : 1; + unsigned char intr : 1; + unsigned char short_len : 3; + unsigned char pad2 : 1; +} __packed; + +struct pdma_data_descr { + unsigned int len : 24; + unsigned int buf : 32; +} __packed; + +struct pdma_short_descr { + unsigned char data[7]; +} __packed; + +struct pdma_descr { + struct pdma_descr_ctrl ctrl; + union { + struct pdma_data_descr data; + struct pdma_short_descr shrt; + }; +}; + +struct pdma_stat_descr { + unsigned char pad1 : 1; + unsigned char pad2 : 1; + unsigned char eop : 1; + unsigned char pad3 : 5; + unsigned int len : 24; +}; + +/* Each descriptor array can hold max 64 entries */ +#define PDMA_DESCR_COUNT 64 + +#define MODULE_NAME "Artpec-6 CA" + +/* Hash modes (including HMAC variants) */ +#define ARTPEC6_CRYPTO_HASH_SHA1 1 +#define ARTPEC6_CRYPTO_HASH_SHA256 2 + +/* Crypto modes */ +#define ARTPEC6_CRYPTO_CIPHER_AES_ECB 1 +#define ARTPEC6_CRYPTO_CIPHER_AES_CBC 2 +#define ARTPEC6_CRYPTO_CIPHER_AES_CTR 3 +#define ARTPEC6_CRYPTO_CIPHER_AES_XTS 5 + +/* The PDMA is a DMA-engine tightly coupled with a ciphering engine. + * It operates on a descriptor array with up to 64 descriptor entries. + * The arrays must be 64 byte aligned in memory. + * + * The ciphering unit has no registers and is completely controlled by + * a 4-byte metadata that is inserted at the beginning of each dma packet. + * + * A dma packet is a sequence of descriptors terminated by setting the .eop + * field in the final descriptor of the packet. + * + * Multiple packets are used for providing context data, key data and + * the plain/ciphertext. + * + * PDMA Descriptors (Array) + * +------+------+------+~~+-------+------+---- + * | 0 | 1 | 2 |~~| 11 EOP| 12 | .... + * +--+---+--+---+----+-+~~+-------+----+-+---- + * | | | | | + * | | | | | + * __|__ +-------++-------++-------+ +----+ + * | MD | |Payload||Payload||Payload| | MD | + * +-----+ +-------++-------++-------+ +----+ + */ + +struct artpec6_crypto_bounce_buffer { + struct list_head list; + size_t length; + struct scatterlist *sg; + size_t offset; + /* buf is aligned to ARTPEC_CACHE_LINE_MAX and + * holds up to ARTPEC_CACHE_LINE_MAX bytes data. + */ + void *buf; +}; + +struct artpec6_crypto_dma_map { + dma_addr_t dma_addr; + size_t size; + enum dma_data_direction dir; +}; + +struct artpec6_crypto_dma_descriptors { + struct pdma_descr out[PDMA_DESCR_COUNT] __aligned(64); + struct pdma_descr in[PDMA_DESCR_COUNT] __aligned(64); + u32 stat[PDMA_DESCR_COUNT] __aligned(64); + struct list_head bounce_buffers; + /* Enough maps for all out/in buffers, and all three descr. arrays */ + struct artpec6_crypto_dma_map maps[PDMA_DESCR_COUNT * 2 + 2]; + dma_addr_t out_dma_addr; + dma_addr_t in_dma_addr; + dma_addr_t stat_dma_addr; + size_t out_cnt; + size_t in_cnt; + size_t map_count; +}; + +enum artpec6_crypto_variant { + ARTPEC6_CRYPTO = 1, + ARTPEC7_CRYPTO, +}; + +struct artpec6_crypto { + void __iomem *base; + spinlock_t queue_lock; + struct list_head queue; /* waiting for pdma fifo space */ + struct list_head pending; /* submitted to pdma fifo */ + struct tasklet_struct task; + struct kmem_cache *dma_cache; + int pending_count; + struct timer_list timer; + enum artpec6_crypto_variant variant; + void *pad_buffer; /* cache-aligned block padding buffer */ + void *zero_buffer; +}; + +enum artpec6_crypto_hash_flags { + HASH_FLAG_INIT_CTX = 2, + HASH_FLAG_UPDATE = 4, + HASH_FLAG_FINALIZE = 8, + HASH_FLAG_HMAC = 16, + HASH_FLAG_UPDATE_KEY = 32, +}; + +struct artpec6_crypto_req_common { + struct list_head list; + struct list_head complete_in_progress; + struct artpec6_crypto_dma_descriptors *dma; + struct crypto_async_request *req; + void (*complete)(struct crypto_async_request *req); + gfp_t gfp_flags; +}; + +struct artpec6_hash_request_context { + char partial_buffer[SHA256_BLOCK_SIZE]; + char partial_buffer_out[SHA256_BLOCK_SIZE]; + char key_buffer[SHA256_BLOCK_SIZE]; + char pad_buffer[SHA256_BLOCK_SIZE + 32]; + unsigned char digeststate[SHA256_DIGEST_SIZE]; + size_t partial_bytes; + u64 digcnt; + u32 key_md; + u32 hash_md; + enum artpec6_crypto_hash_flags hash_flags; + struct artpec6_crypto_req_common common; +}; + +struct artpec6_hash_export_state { + char partial_buffer[SHA256_BLOCK_SIZE]; + unsigned char digeststate[SHA256_DIGEST_SIZE]; + size_t partial_bytes; + u64 digcnt; + int oper; + unsigned int hash_flags; +}; + +struct artpec6_hashalg_context { + char hmac_key[SHA256_BLOCK_SIZE]; + size_t hmac_key_length; + struct crypto_shash *child_hash; +}; + +struct artpec6_crypto_request_context { + u32 cipher_md; + bool decrypt; + struct artpec6_crypto_req_common common; +}; + +struct artpec6_cryptotfm_context { + unsigned char aes_key[2*AES_MAX_KEY_SIZE]; + size_t key_length; + u32 key_md; + int crypto_type; + struct crypto_sync_skcipher *fallback; +}; + +struct artpec6_crypto_aead_hw_ctx { + __be64 aad_length_bits; + __be64 text_length_bits; + __u8 J0[AES_BLOCK_SIZE]; +}; + +struct artpec6_crypto_aead_req_ctx { + struct artpec6_crypto_aead_hw_ctx hw_ctx; + u32 cipher_md; + bool decrypt; + struct artpec6_crypto_req_common common; + __u8 decryption_tag[AES_BLOCK_SIZE] ____cacheline_aligned; +}; + +/* The crypto framework makes it hard to avoid this global. */ +static struct device *artpec6_crypto_dev; + +#ifdef CONFIG_FAULT_INJECTION +static DECLARE_FAULT_ATTR(artpec6_crypto_fail_status_read); +static DECLARE_FAULT_ATTR(artpec6_crypto_fail_dma_array_full); +#endif + +enum { + ARTPEC6_CRYPTO_PREPARE_HASH_NO_START, + ARTPEC6_CRYPTO_PREPARE_HASH_START, +}; + +static int artpec6_crypto_prepare_aead(struct aead_request *areq); +static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq); +static int artpec6_crypto_prepare_hash(struct ahash_request *areq); + +static void +artpec6_crypto_complete_crypto(struct crypto_async_request *req); +static void +artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req); +static void +artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req); +static void +artpec6_crypto_complete_aead(struct crypto_async_request *req); +static void +artpec6_crypto_complete_hash(struct crypto_async_request *req); + +static int +artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common); + +static void +artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common); + +struct artpec6_crypto_walk { + struct scatterlist *sg; + size_t offset; +}; + +static void artpec6_crypto_walk_init(struct artpec6_crypto_walk *awalk, + struct scatterlist *sg) +{ + awalk->sg = sg; + awalk->offset = 0; +} + +static size_t artpec6_crypto_walk_advance(struct artpec6_crypto_walk *awalk, + size_t nbytes) +{ + while (nbytes && awalk->sg) { + size_t piece; + + WARN_ON(awalk->offset > awalk->sg->length); + + piece = min(nbytes, (size_t)awalk->sg->length - awalk->offset); + nbytes -= piece; + awalk->offset += piece; + if (awalk->offset == awalk->sg->length) { + awalk->sg = sg_next(awalk->sg); + awalk->offset = 0; + } + + } + + return nbytes; +} + +static size_t +artpec6_crypto_walk_chunklen(const struct artpec6_crypto_walk *awalk) +{ + WARN_ON(awalk->sg->length == awalk->offset); + + return awalk->sg->length - awalk->offset; +} + +static dma_addr_t +artpec6_crypto_walk_chunk_phys(const struct artpec6_crypto_walk *awalk) +{ + return sg_phys(awalk->sg) + awalk->offset; +} + +static void +artpec6_crypto_copy_bounce_buffers(struct artpec6_crypto_req_common *common) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + struct artpec6_crypto_bounce_buffer *b; + struct artpec6_crypto_bounce_buffer *next; + + list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) { + pr_debug("bounce entry %p: %zu bytes @ %zu from %p\n", + b, b->length, b->offset, b->buf); + sg_pcopy_from_buffer(b->sg, + 1, + b->buf, + b->length, + b->offset); + + list_del(&b->list); + kfree(b); + } +} + +static inline bool artpec6_crypto_busy(void) +{ + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + int fifo_count = ac->pending_count; + + return fifo_count > 6; +} + +static int artpec6_crypto_submit(struct artpec6_crypto_req_common *req) +{ + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + int ret = -EBUSY; + + spin_lock_bh(&ac->queue_lock); + + if (!artpec6_crypto_busy()) { + list_add_tail(&req->list, &ac->pending); + artpec6_crypto_start_dma(req); + ret = -EINPROGRESS; + } else if (req->req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) { + list_add_tail(&req->list, &ac->queue); + } else { + artpec6_crypto_common_destroy(req); + } + + spin_unlock_bh(&ac->queue_lock); + + return ret; +} + +static void artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common) +{ + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + enum artpec6_crypto_variant variant = ac->variant; + void __iomem *base = ac->base; + struct artpec6_crypto_dma_descriptors *dma = common->dma; + u32 ind, statd, outd; + + /* Make descriptor content visible to the DMA before starting it. */ + wmb(); + + ind = FIELD_PREP(PDMA_IN_DESCRQ_PUSH_LEN, dma->in_cnt - 1) | + FIELD_PREP(PDMA_IN_DESCRQ_PUSH_ADDR, dma->in_dma_addr >> 6); + + statd = FIELD_PREP(PDMA_IN_STATQ_PUSH_LEN, dma->in_cnt - 1) | + FIELD_PREP(PDMA_IN_STATQ_PUSH_ADDR, dma->stat_dma_addr >> 6); + + outd = FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_LEN, dma->out_cnt - 1) | + FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_ADDR, dma->out_dma_addr >> 6); + + if (variant == ARTPEC6_CRYPTO) { + writel_relaxed(ind, base + A6_PDMA_IN_DESCRQ_PUSH); + writel_relaxed(statd, base + A6_PDMA_IN_STATQ_PUSH); + writel_relaxed(PDMA_IN_CMD_START, base + A6_PDMA_IN_CMD); + } else { + writel_relaxed(ind, base + A7_PDMA_IN_DESCRQ_PUSH); + writel_relaxed(statd, base + A7_PDMA_IN_STATQ_PUSH); + writel_relaxed(PDMA_IN_CMD_START, base + A7_PDMA_IN_CMD); + } + + writel_relaxed(outd, base + PDMA_OUT_DESCRQ_PUSH); + writel_relaxed(PDMA_OUT_CMD_START, base + PDMA_OUT_CMD); + + ac->pending_count++; +} + +static void +artpec6_crypto_init_dma_operation(struct artpec6_crypto_req_common *common) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + + dma->out_cnt = 0; + dma->in_cnt = 0; + dma->map_count = 0; + INIT_LIST_HEAD(&dma->bounce_buffers); +} + +static bool fault_inject_dma_descr(void) +{ +#ifdef CONFIG_FAULT_INJECTION + return should_fail(&artpec6_crypto_fail_dma_array_full, 1); +#else + return false; +#endif +} + +/** artpec6_crypto_setup_out_descr_phys - Setup an out channel with a + * physical address + * + * @addr: The physical address of the data buffer + * @len: The length of the data buffer + * @eop: True if this is the last buffer in the packet + * + * @return 0 on success or -ENOSPC if there are no more descriptors available + */ +static int +artpec6_crypto_setup_out_descr_phys(struct artpec6_crypto_req_common *common, + dma_addr_t addr, size_t len, bool eop) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + struct pdma_descr *d; + + if (dma->out_cnt >= PDMA_DESCR_COUNT || + fault_inject_dma_descr()) { + pr_err("No free OUT DMA descriptors available!\n"); + return -ENOSPC; + } + + d = &dma->out[dma->out_cnt++]; + memset(d, 0, sizeof(*d)); + + d->ctrl.short_descr = 0; + d->ctrl.eop = eop; + d->data.len = len; + d->data.buf = addr; + return 0; +} + +/** artpec6_crypto_setup_out_descr_short - Setup a short out descriptor + * + * @dst: The virtual address of the data + * @len: The length of the data, must be between 1 to 7 bytes + * @eop: True if this is the last buffer in the packet + * + * @return 0 on success + * -ENOSPC if no more descriptors are available + * -EINVAL if the data length exceeds 7 bytes + */ +static int +artpec6_crypto_setup_out_descr_short(struct artpec6_crypto_req_common *common, + void *dst, unsigned int len, bool eop) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + struct pdma_descr *d; + + if (dma->out_cnt >= PDMA_DESCR_COUNT || + fault_inject_dma_descr()) { + pr_err("No free OUT DMA descriptors available!\n"); + return -ENOSPC; + } else if (len > 7 || len < 1) { + return -EINVAL; + } + d = &dma->out[dma->out_cnt++]; + memset(d, 0, sizeof(*d)); + + d->ctrl.short_descr = 1; + d->ctrl.short_len = len; + d->ctrl.eop = eop; + memcpy(d->shrt.data, dst, len); + return 0; +} + +static int artpec6_crypto_dma_map_page(struct artpec6_crypto_req_common *common, + struct page *page, size_t offset, + size_t size, + enum dma_data_direction dir, + dma_addr_t *dma_addr_out) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + struct device *dev = artpec6_crypto_dev; + struct artpec6_crypto_dma_map *map; + dma_addr_t dma_addr; + + *dma_addr_out = 0; + + if (dma->map_count >= ARRAY_SIZE(dma->maps)) + return -ENOMEM; + + dma_addr = dma_map_page(dev, page, offset, size, dir); + if (dma_mapping_error(dev, dma_addr)) + return -ENOMEM; + + map = &dma->maps[dma->map_count++]; + map->size = size; + map->dma_addr = dma_addr; + map->dir = dir; + + *dma_addr_out = dma_addr; + + return 0; +} + +static int +artpec6_crypto_dma_map_single(struct artpec6_crypto_req_common *common, + void *ptr, size_t size, + enum dma_data_direction dir, + dma_addr_t *dma_addr_out) +{ + struct page *page = virt_to_page(ptr); + size_t offset = (uintptr_t)ptr & ~PAGE_MASK; + + return artpec6_crypto_dma_map_page(common, page, offset, size, dir, + dma_addr_out); +} + +static int +artpec6_crypto_dma_map_descs(struct artpec6_crypto_req_common *common) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + int ret; + + ret = artpec6_crypto_dma_map_single(common, dma->in, + sizeof(dma->in[0]) * dma->in_cnt, + DMA_TO_DEVICE, &dma->in_dma_addr); + if (ret) + return ret; + + ret = artpec6_crypto_dma_map_single(common, dma->out, + sizeof(dma->out[0]) * dma->out_cnt, + DMA_TO_DEVICE, &dma->out_dma_addr); + if (ret) + return ret; + + /* We only read one stat descriptor */ + dma->stat[dma->in_cnt - 1] = 0; + + /* + * DMA_BIDIRECTIONAL since we need our zeroing of the stat descriptor + * to be written. + */ + return artpec6_crypto_dma_map_single(common, + dma->stat, + sizeof(dma->stat[0]) * dma->in_cnt, + DMA_BIDIRECTIONAL, + &dma->stat_dma_addr); +} + +static void +artpec6_crypto_dma_unmap_all(struct artpec6_crypto_req_common *common) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + struct device *dev = artpec6_crypto_dev; + int i; + + for (i = 0; i < dma->map_count; i++) { + struct artpec6_crypto_dma_map *map = &dma->maps[i]; + + dma_unmap_page(dev, map->dma_addr, map->size, map->dir); + } + + dma->map_count = 0; +} + +/** artpec6_crypto_setup_out_descr - Setup an out descriptor + * + * @dst: The virtual address of the data + * @len: The length of the data + * @eop: True if this is the last buffer in the packet + * @use_short: If this is true and the data length is 7 bytes or less then + * a short descriptor will be used + * + * @return 0 on success + * Any errors from artpec6_crypto_setup_out_descr_short() or + * setup_out_descr_phys() + */ +static int +artpec6_crypto_setup_out_descr(struct artpec6_crypto_req_common *common, + void *dst, unsigned int len, bool eop, + bool use_short) +{ + dma_addr_t dma_addr; + int ret; + + if (use_short && len < 7) + return artpec6_crypto_setup_out_descr_short(common, dst, len, + eop); + + ret = artpec6_crypto_dma_map_single(common, dst, len, DMA_TO_DEVICE, + &dma_addr); + if (ret) + return ret; + + return artpec6_crypto_setup_out_descr_phys(common, dma_addr, len, eop); +} + +/** artpec6_crypto_setup_in_descr_phys - Setup an in channel with a + * physical address + * + * @addr: The physical address of the data buffer + * @len: The length of the data buffer + * @intr: True if an interrupt should be fired after HW processing of this + * descriptor + * + */ +static int +artpec6_crypto_setup_in_descr_phys(struct artpec6_crypto_req_common *common, + dma_addr_t addr, unsigned int len, bool intr) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + struct pdma_descr *d; + + if (dma->in_cnt >= PDMA_DESCR_COUNT || + fault_inject_dma_descr()) { + pr_err("No free IN DMA descriptors available!\n"); + return -ENOSPC; + } + d = &dma->in[dma->in_cnt++]; + memset(d, 0, sizeof(*d)); + + d->ctrl.intr = intr; + d->data.len = len; + d->data.buf = addr; + return 0; +} + +/** artpec6_crypto_setup_in_descr - Setup an in channel descriptor + * + * @buffer: The virtual address to of the data buffer + * @len: The length of the data buffer + * @last: If this is the last data buffer in the request (i.e. an interrupt + * is needed + * + * Short descriptors are not used for the in channel + */ +static int +artpec6_crypto_setup_in_descr(struct artpec6_crypto_req_common *common, + void *buffer, unsigned int len, bool last) +{ + dma_addr_t dma_addr; + int ret; + + ret = artpec6_crypto_dma_map_single(common, buffer, len, + DMA_FROM_DEVICE, &dma_addr); + if (ret) + return ret; + + return artpec6_crypto_setup_in_descr_phys(common, dma_addr, len, last); +} + +static struct artpec6_crypto_bounce_buffer * +artpec6_crypto_alloc_bounce(gfp_t flags) +{ + void *base; + size_t alloc_size = sizeof(struct artpec6_crypto_bounce_buffer) + + 2 * ARTPEC_CACHE_LINE_MAX; + struct artpec6_crypto_bounce_buffer *bbuf = kzalloc(alloc_size, flags); + + if (!bbuf) + return NULL; + + base = bbuf + 1; + bbuf->buf = PTR_ALIGN(base, ARTPEC_CACHE_LINE_MAX); + return bbuf; +} + +static int setup_bounce_buffer_in(struct artpec6_crypto_req_common *common, + struct artpec6_crypto_walk *walk, size_t size) +{ + struct artpec6_crypto_bounce_buffer *bbuf; + int ret; + + bbuf = artpec6_crypto_alloc_bounce(common->gfp_flags); + if (!bbuf) + return -ENOMEM; + + bbuf->length = size; + bbuf->sg = walk->sg; + bbuf->offset = walk->offset; + + ret = artpec6_crypto_setup_in_descr(common, bbuf->buf, size, false); + if (ret) { + kfree(bbuf); + return ret; + } + + pr_debug("BOUNCE %zu offset %zu\n", size, walk->offset); + list_add_tail(&bbuf->list, &common->dma->bounce_buffers); + return 0; +} + +static int +artpec6_crypto_setup_sg_descrs_in(struct artpec6_crypto_req_common *common, + struct artpec6_crypto_walk *walk, + size_t count) +{ + size_t chunk; + int ret; + dma_addr_t addr; + + while (walk->sg && count) { + chunk = min(count, artpec6_crypto_walk_chunklen(walk)); + addr = artpec6_crypto_walk_chunk_phys(walk); + + /* When destination buffers are not aligned to the cache line + * size we need bounce buffers. The DMA-API requires that the + * entire line is owned by the DMA buffer and this holds also + * for the case when coherent DMA is used. + */ + if (!IS_ALIGNED(addr, ARTPEC_CACHE_LINE_MAX)) { + chunk = min_t(dma_addr_t, chunk, + ALIGN(addr, ARTPEC_CACHE_LINE_MAX) - + addr); + + pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk); + ret = setup_bounce_buffer_in(common, walk, chunk); + } else if (chunk < ARTPEC_CACHE_LINE_MAX) { + pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk); + ret = setup_bounce_buffer_in(common, walk, chunk); + } else { + dma_addr_t dma_addr; + + chunk = chunk & ~(ARTPEC_CACHE_LINE_MAX-1); + + pr_debug("CHUNK %pad:%zu\n", &addr, chunk); + + ret = artpec6_crypto_dma_map_page(common, + sg_page(walk->sg), + walk->sg->offset + + walk->offset, + chunk, + DMA_FROM_DEVICE, + &dma_addr); + if (ret) + return ret; + + ret = artpec6_crypto_setup_in_descr_phys(common, + dma_addr, + chunk, false); + } + + if (ret) + return ret; + + count = count - chunk; + artpec6_crypto_walk_advance(walk, chunk); + } + + if (count) + pr_err("EOL unexpected %zu bytes left\n", count); + + return count ? -EINVAL : 0; +} + +static int +artpec6_crypto_setup_sg_descrs_out(struct artpec6_crypto_req_common *common, + struct artpec6_crypto_walk *walk, + size_t count) +{ + size_t chunk; + int ret; + dma_addr_t addr; + + while (walk->sg && count) { + chunk = min(count, artpec6_crypto_walk_chunklen(walk)); + addr = artpec6_crypto_walk_chunk_phys(walk); + + pr_debug("OUT-CHUNK %pad:%zu\n", &addr, chunk); + + if (addr & 3) { + char buf[3]; + + chunk = min_t(size_t, chunk, (4-(addr&3))); + + sg_pcopy_to_buffer(walk->sg, 1, buf, chunk, + walk->offset); + + ret = artpec6_crypto_setup_out_descr_short(common, buf, + chunk, + false); + } else { + dma_addr_t dma_addr; + + ret = artpec6_crypto_dma_map_page(common, + sg_page(walk->sg), + walk->sg->offset + + walk->offset, + chunk, + DMA_TO_DEVICE, + &dma_addr); + if (ret) + return ret; + + ret = artpec6_crypto_setup_out_descr_phys(common, + dma_addr, + chunk, false); + } + + if (ret) + return ret; + + count = count - chunk; + artpec6_crypto_walk_advance(walk, chunk); + } + + if (count) + pr_err("EOL unexpected %zu bytes left\n", count); + + return count ? -EINVAL : 0; +} + + +/** artpec6_crypto_terminate_out_descrs - Set the EOP on the last out descriptor + * + * If the out descriptor list is non-empty, then the eop flag on the + * last used out descriptor will be set. + * + * @return 0 on success + * -EINVAL if the out descriptor is empty or has overflown + */ +static int +artpec6_crypto_terminate_out_descrs(struct artpec6_crypto_req_common *common) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + struct pdma_descr *d; + + if (!dma->out_cnt || dma->out_cnt > PDMA_DESCR_COUNT) { + pr_err("%s: OUT descriptor list is %s\n", + MODULE_NAME, dma->out_cnt ? "empty" : "full"); + return -EINVAL; + + } + + d = &dma->out[dma->out_cnt-1]; + d->ctrl.eop = 1; + + return 0; +} + +/** artpec6_crypto_terminate_in_descrs - Set the interrupt flag on the last + * in descriptor + * + * See artpec6_crypto_terminate_out_descrs() for return values + */ +static int +artpec6_crypto_terminate_in_descrs(struct artpec6_crypto_req_common *common) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + struct pdma_descr *d; + + if (!dma->in_cnt || dma->in_cnt > PDMA_DESCR_COUNT) { + pr_err("%s: IN descriptor list is %s\n", + MODULE_NAME, dma->in_cnt ? "empty" : "full"); + return -EINVAL; + } + + d = &dma->in[dma->in_cnt-1]; + d->ctrl.intr = 1; + return 0; +} + +/** create_hash_pad - Create a Secure Hash conformant pad + * + * @dst: The destination buffer to write the pad. Must be at least 64 bytes + * @dgstlen: The total length of the hash digest in bytes + * @bitcount: The total length of the digest in bits + * + * @return The total number of padding bytes written to @dst + */ +static size_t +create_hash_pad(int oper, unsigned char *dst, u64 dgstlen, u64 bitcount) +{ + unsigned int mod, target, diff, pad_bytes, size_bytes; + __be64 bits = __cpu_to_be64(bitcount); + + switch (oper) { + case regk_crypto_sha1: + case regk_crypto_sha256: + case regk_crypto_hmac_sha1: + case regk_crypto_hmac_sha256: + target = 448 / 8; + mod = 512 / 8; + size_bytes = 8; + break; + default: + target = 896 / 8; + mod = 1024 / 8; + size_bytes = 16; + break; + } + + target -= 1; + diff = dgstlen & (mod - 1); + pad_bytes = diff > target ? target + mod - diff : target - diff; + + memset(dst + 1, 0, pad_bytes); + dst[0] = 0x80; + + if (size_bytes == 16) { + memset(dst + 1 + pad_bytes, 0, 8); + memcpy(dst + 1 + pad_bytes + 8, &bits, 8); + } else { + memcpy(dst + 1 + pad_bytes, &bits, 8); + } + + return pad_bytes + size_bytes + 1; +} + +static int artpec6_crypto_common_init(struct artpec6_crypto_req_common *common, + struct crypto_async_request *parent, + void (*complete)(struct crypto_async_request *req), + struct scatterlist *dstsg, unsigned int nbytes) +{ + gfp_t flags; + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + + flags = (parent->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + + common->gfp_flags = flags; + common->dma = kmem_cache_alloc(ac->dma_cache, flags); + if (!common->dma) + return -ENOMEM; + + common->req = parent; + common->complete = complete; + return 0; +} + +static void +artpec6_crypto_bounce_destroy(struct artpec6_crypto_dma_descriptors *dma) +{ + struct artpec6_crypto_bounce_buffer *b; + struct artpec6_crypto_bounce_buffer *next; + + list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) { + kfree(b); + } +} + +static int +artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common) +{ + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + + artpec6_crypto_dma_unmap_all(common); + artpec6_crypto_bounce_destroy(common->dma); + kmem_cache_free(ac->dma_cache, common->dma); + common->dma = NULL; + return 0; +} + +/* + * Ciphering functions. + */ +static int artpec6_crypto_encrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); + struct artpec6_crypto_request_context *req_ctx = NULL; + void (*complete)(struct crypto_async_request *req); + int ret; + + req_ctx = skcipher_request_ctx(req); + + switch (ctx->crypto_type) { + case ARTPEC6_CRYPTO_CIPHER_AES_CBC: + case ARTPEC6_CRYPTO_CIPHER_AES_ECB: + case ARTPEC6_CRYPTO_CIPHER_AES_XTS: + req_ctx->decrypt = 0; + break; + default: + break; + } + + switch (ctx->crypto_type) { + case ARTPEC6_CRYPTO_CIPHER_AES_CBC: + complete = artpec6_crypto_complete_cbc_encrypt; + break; + default: + complete = artpec6_crypto_complete_crypto; + break; + } + + ret = artpec6_crypto_common_init(&req_ctx->common, + &req->base, + complete, + req->dst, req->cryptlen); + if (ret) + return ret; + + ret = artpec6_crypto_prepare_crypto(req); + if (ret) { + artpec6_crypto_common_destroy(&req_ctx->common); + return ret; + } + + return artpec6_crypto_submit(&req_ctx->common); +} + +static int artpec6_crypto_decrypt(struct skcipher_request *req) +{ + int ret; + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); + struct artpec6_crypto_request_context *req_ctx = NULL; + void (*complete)(struct crypto_async_request *req); + + req_ctx = skcipher_request_ctx(req); + + switch (ctx->crypto_type) { + case ARTPEC6_CRYPTO_CIPHER_AES_CBC: + case ARTPEC6_CRYPTO_CIPHER_AES_ECB: + case ARTPEC6_CRYPTO_CIPHER_AES_XTS: + req_ctx->decrypt = 1; + break; + default: + break; + } + + + switch (ctx->crypto_type) { + case ARTPEC6_CRYPTO_CIPHER_AES_CBC: + complete = artpec6_crypto_complete_cbc_decrypt; + break; + default: + complete = artpec6_crypto_complete_crypto; + break; + } + + ret = artpec6_crypto_common_init(&req_ctx->common, &req->base, + complete, + req->dst, req->cryptlen); + if (ret) + return ret; + + ret = artpec6_crypto_prepare_crypto(req); + if (ret) { + artpec6_crypto_common_destroy(&req_ctx->common); + return ret; + } + + return artpec6_crypto_submit(&req_ctx->common); +} + +static int +artpec6_crypto_ctr_crypt(struct skcipher_request *req, bool encrypt) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); + size_t iv_len = crypto_skcipher_ivsize(cipher); + unsigned int counter = be32_to_cpup((__be32 *) + (req->iv + iv_len - 4)); + unsigned int nblks = ALIGN(req->cryptlen, AES_BLOCK_SIZE) / + AES_BLOCK_SIZE; + + /* + * The hardware uses only the last 32-bits as the counter while the + * kernel tests (aes_ctr_enc_tv_template[4] for example) expect that + * the whole IV is a counter. So fallback if the counter is going to + * overlow. + */ + if (counter + nblks < counter) { + int ret; + + pr_debug("counter %x will overflow (nblks %u), falling back\n", + counter, counter + nblks); + + ret = crypto_sync_skcipher_setkey(ctx->fallback, ctx->aes_key, + ctx->key_length); + if (ret) + return ret; + + { + SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback); + + skcipher_request_set_sync_tfm(subreq, ctx->fallback); + skcipher_request_set_callback(subreq, req->base.flags, + NULL, NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + ret = encrypt ? crypto_skcipher_encrypt(subreq) + : crypto_skcipher_decrypt(subreq); + skcipher_request_zero(subreq); + } + return ret; + } + + return encrypt ? artpec6_crypto_encrypt(req) + : artpec6_crypto_decrypt(req); +} + +static int artpec6_crypto_ctr_encrypt(struct skcipher_request *req) +{ + return artpec6_crypto_ctr_crypt(req, true); +} + +static int artpec6_crypto_ctr_decrypt(struct skcipher_request *req) +{ + return artpec6_crypto_ctr_crypt(req, false); +} + +/* + * AEAD functions + */ +static int artpec6_crypto_aead_init(struct crypto_aead *tfm) +{ + struct artpec6_cryptotfm_context *tfm_ctx = crypto_aead_ctx(tfm); + + memset(tfm_ctx, 0, sizeof(*tfm_ctx)); + + crypto_aead_set_reqsize(tfm, + sizeof(struct artpec6_crypto_aead_req_ctx)); + + return 0; +} + +static int artpec6_crypto_aead_set_key(struct crypto_aead *tfm, const u8 *key, + unsigned int len) +{ + struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(&tfm->base); + + if (len != 16 && len != 24 && len != 32) + return -EINVAL; + + ctx->key_length = len; + + memcpy(ctx->aes_key, key, len); + return 0; +} + +static int artpec6_crypto_aead_encrypt(struct aead_request *req) +{ + int ret; + struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req); + + req_ctx->decrypt = false; + ret = artpec6_crypto_common_init(&req_ctx->common, &req->base, + artpec6_crypto_complete_aead, + NULL, 0); + if (ret) + return ret; + + ret = artpec6_crypto_prepare_aead(req); + if (ret) { + artpec6_crypto_common_destroy(&req_ctx->common); + return ret; + } + + return artpec6_crypto_submit(&req_ctx->common); +} + +static int artpec6_crypto_aead_decrypt(struct aead_request *req) +{ + int ret; + struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req); + + req_ctx->decrypt = true; + if (req->cryptlen < AES_BLOCK_SIZE) + return -EINVAL; + + ret = artpec6_crypto_common_init(&req_ctx->common, + &req->base, + artpec6_crypto_complete_aead, + NULL, 0); + if (ret) + return ret; + + ret = artpec6_crypto_prepare_aead(req); + if (ret) { + artpec6_crypto_common_destroy(&req_ctx->common); + return ret; + } + + return artpec6_crypto_submit(&req_ctx->common); +} + +static int artpec6_crypto_prepare_hash(struct ahash_request *areq) +{ + struct artpec6_hashalg_context *ctx = crypto_tfm_ctx(areq->base.tfm); + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(areq); + size_t digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq)); + size_t contextsize = digestsize; + size_t blocksize = crypto_tfm_alg_blocksize( + crypto_ahash_tfm(crypto_ahash_reqtfm(areq))); + struct artpec6_crypto_req_common *common = &req_ctx->common; + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + enum artpec6_crypto_variant variant = ac->variant; + u32 sel_ctx; + bool ext_ctx = false; + bool run_hw = false; + int error = 0; + + artpec6_crypto_init_dma_operation(common); + + /* Upload HMAC key, it must be the first packet */ + if (req_ctx->hash_flags & HASH_FLAG_HMAC) { + if (variant == ARTPEC6_CRYPTO) { + req_ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, + a6_regk_crypto_dlkey); + } else { + req_ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, + a7_regk_crypto_dlkey); + } + + memcpy_and_pad(req_ctx->key_buffer, blocksize, ctx->hmac_key, + ctx->hmac_key_length, 0); + + error = artpec6_crypto_setup_out_descr(common, + (void *)&req_ctx->key_md, + sizeof(req_ctx->key_md), false, false); + if (error) + return error; + + error = artpec6_crypto_setup_out_descr(common, + req_ctx->key_buffer, blocksize, + true, false); + if (error) + return error; + } + + if (!(req_ctx->hash_flags & HASH_FLAG_INIT_CTX)) { + /* Restore context */ + sel_ctx = regk_crypto_ext; + ext_ctx = true; + } else { + sel_ctx = regk_crypto_init; + } + + if (variant == ARTPEC6_CRYPTO) { + req_ctx->hash_md &= ~A6_CRY_MD_HASH_SEL_CTX; + req_ctx->hash_md |= FIELD_PREP(A6_CRY_MD_HASH_SEL_CTX, sel_ctx); + + /* If this is the final round, set the final flag */ + if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) + req_ctx->hash_md |= A6_CRY_MD_HASH_HMAC_FIN; + } else { + req_ctx->hash_md &= ~A7_CRY_MD_HASH_SEL_CTX; + req_ctx->hash_md |= FIELD_PREP(A7_CRY_MD_HASH_SEL_CTX, sel_ctx); + + /* If this is the final round, set the final flag */ + if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) + req_ctx->hash_md |= A7_CRY_MD_HASH_HMAC_FIN; + } + + /* Setup up metadata descriptors */ + error = artpec6_crypto_setup_out_descr(common, + (void *)&req_ctx->hash_md, + sizeof(req_ctx->hash_md), false, false); + if (error) + return error; + + error = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); + if (error) + return error; + + if (ext_ctx) { + error = artpec6_crypto_setup_out_descr(common, + req_ctx->digeststate, + contextsize, false, false); + + if (error) + return error; + } + + if (req_ctx->hash_flags & HASH_FLAG_UPDATE) { + size_t done_bytes = 0; + size_t total_bytes = areq->nbytes + req_ctx->partial_bytes; + size_t ready_bytes = round_down(total_bytes, blocksize); + struct artpec6_crypto_walk walk; + + run_hw = ready_bytes > 0; + if (req_ctx->partial_bytes && ready_bytes) { + /* We have a partial buffer and will at least some bytes + * to the HW. Empty this partial buffer before tackling + * the SG lists + */ + memcpy(req_ctx->partial_buffer_out, + req_ctx->partial_buffer, + req_ctx->partial_bytes); + + error = artpec6_crypto_setup_out_descr(common, + req_ctx->partial_buffer_out, + req_ctx->partial_bytes, + false, true); + if (error) + return error; + + /* Reset partial buffer */ + done_bytes += req_ctx->partial_bytes; + req_ctx->partial_bytes = 0; + } + + artpec6_crypto_walk_init(&walk, areq->src); + + error = artpec6_crypto_setup_sg_descrs_out(common, &walk, + ready_bytes - + done_bytes); + if (error) + return error; + + if (walk.sg) { + size_t sg_skip = ready_bytes - done_bytes; + size_t sg_rem = areq->nbytes - sg_skip; + + sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), + req_ctx->partial_buffer + + req_ctx->partial_bytes, + sg_rem, sg_skip); + + req_ctx->partial_bytes += sg_rem; + } + + req_ctx->digcnt += ready_bytes; + req_ctx->hash_flags &= ~(HASH_FLAG_UPDATE); + } + + /* Finalize */ + if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) { + size_t hash_pad_len; + u64 digest_bits; + u32 oper; + + if (variant == ARTPEC6_CRYPTO) + oper = FIELD_GET(A6_CRY_MD_OPER, req_ctx->hash_md); + else + oper = FIELD_GET(A7_CRY_MD_OPER, req_ctx->hash_md); + + /* Write out the partial buffer if present */ + if (req_ctx->partial_bytes) { + memcpy(req_ctx->partial_buffer_out, + req_ctx->partial_buffer, + req_ctx->partial_bytes); + error = artpec6_crypto_setup_out_descr(common, + req_ctx->partial_buffer_out, + req_ctx->partial_bytes, + false, true); + if (error) + return error; + + req_ctx->digcnt += req_ctx->partial_bytes; + req_ctx->partial_bytes = 0; + } + + if (req_ctx->hash_flags & HASH_FLAG_HMAC) + digest_bits = 8 * (req_ctx->digcnt + blocksize); + else + digest_bits = 8 * req_ctx->digcnt; + + /* Add the hash pad */ + hash_pad_len = create_hash_pad(oper, req_ctx->pad_buffer, + req_ctx->digcnt, digest_bits); + error = artpec6_crypto_setup_out_descr(common, + req_ctx->pad_buffer, + hash_pad_len, false, + true); + req_ctx->digcnt = 0; + + if (error) + return error; + + /* Descriptor for the final result */ + error = artpec6_crypto_setup_in_descr(common, areq->result, + digestsize, + true); + if (error) + return error; + + } else { /* This is not the final operation for this request */ + if (!run_hw) + return ARTPEC6_CRYPTO_PREPARE_HASH_NO_START; + + /* Save the result to the context */ + error = artpec6_crypto_setup_in_descr(common, + req_ctx->digeststate, + contextsize, false); + if (error) + return error; + /* fall through */ + } + + req_ctx->hash_flags &= ~(HASH_FLAG_INIT_CTX | HASH_FLAG_UPDATE | + HASH_FLAG_FINALIZE); + + error = artpec6_crypto_terminate_in_descrs(common); + if (error) + return error; + + error = artpec6_crypto_terminate_out_descrs(common); + if (error) + return error; + + error = artpec6_crypto_dma_map_descs(common); + if (error) + return error; + + return ARTPEC6_CRYPTO_PREPARE_HASH_START; +} + + +static int artpec6_crypto_aes_ecb_init(struct crypto_skcipher *tfm) +{ + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); + + crypto_skcipher_set_reqsize(tfm, + sizeof(struct artpec6_crypto_request_context)); + ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_ECB; + + return 0; +} + +static int artpec6_crypto_aes_ctr_init(struct crypto_skcipher *tfm) +{ + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); + + ctx->fallback = + crypto_alloc_sync_skcipher(crypto_tfm_alg_name(&tfm->base), + 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fallback)) + return PTR_ERR(ctx->fallback); + + crypto_skcipher_set_reqsize(tfm, + sizeof(struct artpec6_crypto_request_context)); + ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CTR; + + return 0; +} + +static int artpec6_crypto_aes_cbc_init(struct crypto_skcipher *tfm) +{ + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); + + crypto_skcipher_set_reqsize(tfm, + sizeof(struct artpec6_crypto_request_context)); + ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CBC; + + return 0; +} + +static int artpec6_crypto_aes_xts_init(struct crypto_skcipher *tfm) +{ + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); + + crypto_skcipher_set_reqsize(tfm, + sizeof(struct artpec6_crypto_request_context)); + ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_XTS; + + return 0; +} + +static void artpec6_crypto_aes_exit(struct crypto_skcipher *tfm) +{ + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); + + memset(ctx, 0, sizeof(*ctx)); +} + +static void artpec6_crypto_aes_ctr_exit(struct crypto_skcipher *tfm) +{ + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); + + crypto_free_sync_skcipher(ctx->fallback); + artpec6_crypto_aes_exit(tfm); +} + +static int +artpec6_crypto_cipher_set_key(struct crypto_skcipher *cipher, const u8 *key, + unsigned int keylen) +{ + struct artpec6_cryptotfm_context *ctx = + crypto_skcipher_ctx(cipher); + + switch (keylen) { + case 16: + case 24: + case 32: + break; + default: + return -EINVAL; + } + + memcpy(ctx->aes_key, key, keylen); + ctx->key_length = keylen; + return 0; +} + +static int +artpec6_crypto_xts_set_key(struct crypto_skcipher *cipher, const u8 *key, + unsigned int keylen) +{ + struct artpec6_cryptotfm_context *ctx = + crypto_skcipher_ctx(cipher); + int ret; + + ret = xts_verify_key(cipher, key, keylen); + if (ret) + return ret; + + switch (keylen) { + case 32: + case 48: + case 64: + break; + default: + return -EINVAL; + } + + memcpy(ctx->aes_key, key, keylen); + ctx->key_length = keylen; + return 0; +} + +/** artpec6_crypto_process_crypto - Prepare an async block cipher crypto request + * + * @req: The asynch request to process + * + * @return 0 if the dma job was successfully prepared + * <0 on error + * + * This function sets up the PDMA descriptors for a block cipher request. + * + * The required padding is added for AES-CTR using a statically defined + * buffer. + * + * The PDMA descriptor list will be as follows: + * + * OUT: [KEY_MD][KEY][EOP][IV]...[data_n][AES-CTR_pad] + * IN: ...[data_n] + * + */ +static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq) +{ + int ret; + struct artpec6_crypto_walk walk; + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); + struct artpec6_crypto_request_context *req_ctx = NULL; + size_t iv_len = crypto_skcipher_ivsize(cipher); + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + enum artpec6_crypto_variant variant = ac->variant; + struct artpec6_crypto_req_common *common; + bool cipher_decr = false; + size_t cipher_klen; + u32 cipher_len = 0; /* Same as regk_crypto_key_128 for NULL crypto */ + u32 oper; + + req_ctx = skcipher_request_ctx(areq); + common = &req_ctx->common; + + artpec6_crypto_init_dma_operation(common); + + if (variant == ARTPEC6_CRYPTO) + ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, a6_regk_crypto_dlkey); + else + ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, a7_regk_crypto_dlkey); + + ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md, + sizeof(ctx->key_md), false, false); + if (ret) + return ret; + + ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key, + ctx->key_length, true, false); + if (ret) + return ret; + + req_ctx->cipher_md = 0; + + if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS) + cipher_klen = ctx->key_length/2; + else + cipher_klen = ctx->key_length; + + /* Metadata */ + switch (cipher_klen) { + case 16: + cipher_len = regk_crypto_key_128; + break; + case 24: + cipher_len = regk_crypto_key_192; + break; + case 32: + cipher_len = regk_crypto_key_256; + break; + default: + pr_err("%s: Invalid key length %zu!\n", + MODULE_NAME, ctx->key_length); + return -EINVAL; + } + + switch (ctx->crypto_type) { + case ARTPEC6_CRYPTO_CIPHER_AES_ECB: + oper = regk_crypto_aes_ecb; + cipher_decr = req_ctx->decrypt; + break; + + case ARTPEC6_CRYPTO_CIPHER_AES_CBC: + oper = regk_crypto_aes_cbc; + cipher_decr = req_ctx->decrypt; + break; + + case ARTPEC6_CRYPTO_CIPHER_AES_CTR: + oper = regk_crypto_aes_ctr; + cipher_decr = false; + break; + + case ARTPEC6_CRYPTO_CIPHER_AES_XTS: + oper = regk_crypto_aes_xts; + cipher_decr = req_ctx->decrypt; + + if (variant == ARTPEC6_CRYPTO) + req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DSEQ; + else + req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DSEQ; + break; + + default: + pr_err("%s: Invalid cipher mode %d!\n", + MODULE_NAME, ctx->crypto_type); + return -EINVAL; + } + + if (variant == ARTPEC6_CRYPTO) { + req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER, oper); + req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN, + cipher_len); + if (cipher_decr) + req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR; + } else { + req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER, oper); + req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN, + cipher_len); + if (cipher_decr) + req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR; + } + + ret = artpec6_crypto_setup_out_descr(common, + &req_ctx->cipher_md, + sizeof(req_ctx->cipher_md), + false, false); + if (ret) + return ret; + + ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); + if (ret) + return ret; + + if (iv_len) { + ret = artpec6_crypto_setup_out_descr(common, areq->iv, iv_len, + false, false); + if (ret) + return ret; + } + /* Data out */ + artpec6_crypto_walk_init(&walk, areq->src); + ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, areq->cryptlen); + if (ret) + return ret; + + /* Data in */ + artpec6_crypto_walk_init(&walk, areq->dst); + ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, areq->cryptlen); + if (ret) + return ret; + + /* CTR-mode padding required by the HW. */ + if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_CTR || + ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS) { + size_t pad = ALIGN(areq->cryptlen, AES_BLOCK_SIZE) - + areq->cryptlen; + + if (pad) { + ret = artpec6_crypto_setup_out_descr(common, + ac->pad_buffer, + pad, false, false); + if (ret) + return ret; + + ret = artpec6_crypto_setup_in_descr(common, + ac->pad_buffer, pad, + false); + if (ret) + return ret; + } + } + + ret = artpec6_crypto_terminate_out_descrs(common); + if (ret) + return ret; + + ret = artpec6_crypto_terminate_in_descrs(common); + if (ret) + return ret; + + return artpec6_crypto_dma_map_descs(common); +} + +static int artpec6_crypto_prepare_aead(struct aead_request *areq) +{ + size_t count; + int ret; + size_t input_length; + struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(areq->base.tfm); + struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq); + struct crypto_aead *cipher = crypto_aead_reqtfm(areq); + struct artpec6_crypto_req_common *common = &req_ctx->common; + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + enum artpec6_crypto_variant variant = ac->variant; + u32 md_cipher_len; + + artpec6_crypto_init_dma_operation(common); + + /* Key */ + if (variant == ARTPEC6_CRYPTO) { + ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, + a6_regk_crypto_dlkey); + } else { + ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, + a7_regk_crypto_dlkey); + } + ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md, + sizeof(ctx->key_md), false, false); + if (ret) + return ret; + + ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key, + ctx->key_length, true, false); + if (ret) + return ret; + + req_ctx->cipher_md = 0; + + switch (ctx->key_length) { + case 16: + md_cipher_len = regk_crypto_key_128; + break; + case 24: + md_cipher_len = regk_crypto_key_192; + break; + case 32: + md_cipher_len = regk_crypto_key_256; + break; + default: + return -EINVAL; + } + + if (variant == ARTPEC6_CRYPTO) { + req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER, + regk_crypto_aes_gcm); + req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN, + md_cipher_len); + if (req_ctx->decrypt) + req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR; + } else { + req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER, + regk_crypto_aes_gcm); + req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN, + md_cipher_len); + if (req_ctx->decrypt) + req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR; + } + + ret = artpec6_crypto_setup_out_descr(common, + (void *) &req_ctx->cipher_md, + sizeof(req_ctx->cipher_md), false, + false); + if (ret) + return ret; + + ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); + if (ret) + return ret; + + /* For the decryption, cryptlen includes the tag. */ + input_length = areq->cryptlen; + if (req_ctx->decrypt) + input_length -= crypto_aead_authsize(cipher); + + /* Prepare the context buffer */ + req_ctx->hw_ctx.aad_length_bits = + __cpu_to_be64(8*areq->assoclen); + + req_ctx->hw_ctx.text_length_bits = + __cpu_to_be64(8*input_length); + + memcpy(req_ctx->hw_ctx.J0, areq->iv, crypto_aead_ivsize(cipher)); + // The HW omits the initial increment of the counter field. + memcpy(req_ctx->hw_ctx.J0 + GCM_AES_IV_SIZE, "\x00\x00\x00\x01", 4); + + ret = artpec6_crypto_setup_out_descr(common, &req_ctx->hw_ctx, + sizeof(struct artpec6_crypto_aead_hw_ctx), false, false); + if (ret) + return ret; + + { + struct artpec6_crypto_walk walk; + + artpec6_crypto_walk_init(&walk, areq->src); + + /* Associated data */ + count = areq->assoclen; + ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count); + if (ret) + return ret; + + if (!IS_ALIGNED(areq->assoclen, 16)) { + size_t assoc_pad = 16 - (areq->assoclen % 16); + /* The HW mandates zero padding here */ + ret = artpec6_crypto_setup_out_descr(common, + ac->zero_buffer, + assoc_pad, false, + false); + if (ret) + return ret; + } + + /* Data to crypto */ + count = input_length; + ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count); + if (ret) + return ret; + + if (!IS_ALIGNED(input_length, 16)) { + size_t crypto_pad = 16 - (input_length % 16); + /* The HW mandates zero padding here */ + ret = artpec6_crypto_setup_out_descr(common, + ac->zero_buffer, + crypto_pad, + false, + false); + if (ret) + return ret; + } + } + + /* Data from crypto */ + { + struct artpec6_crypto_walk walk; + size_t output_len = areq->cryptlen; + + if (req_ctx->decrypt) + output_len -= crypto_aead_authsize(cipher); + + artpec6_crypto_walk_init(&walk, areq->dst); + + /* skip associated data in the output */ + count = artpec6_crypto_walk_advance(&walk, areq->assoclen); + if (count) + return -EINVAL; + + count = output_len; + ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, count); + if (ret) + return ret; + + /* Put padding between the cryptotext and the auth tag */ + if (!IS_ALIGNED(output_len, 16)) { + size_t crypto_pad = 16 - (output_len % 16); + + ret = artpec6_crypto_setup_in_descr(common, + ac->pad_buffer, + crypto_pad, false); + if (ret) + return ret; + } + + /* The authentication tag shall follow immediately after + * the output ciphertext. For decryption it is put in a context + * buffer for later compare against the input tag. + */ + + if (req_ctx->decrypt) { + ret = artpec6_crypto_setup_in_descr(common, + req_ctx->decryption_tag, AES_BLOCK_SIZE, false); + if (ret) + return ret; + + } else { + /* For encryption the requested tag size may be smaller + * than the hardware's generated tag. + */ + size_t authsize = crypto_aead_authsize(cipher); + + ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, + authsize); + if (ret) + return ret; + + if (authsize < AES_BLOCK_SIZE) { + count = AES_BLOCK_SIZE - authsize; + ret = artpec6_crypto_setup_in_descr(common, + ac->pad_buffer, + count, false); + if (ret) + return ret; + } + } + + } + + ret = artpec6_crypto_terminate_in_descrs(common); + if (ret) + return ret; + + ret = artpec6_crypto_terminate_out_descrs(common); + if (ret) + return ret; + + return artpec6_crypto_dma_map_descs(common); +} + +static void artpec6_crypto_process_queue(struct artpec6_crypto *ac, + struct list_head *completions) +{ + struct artpec6_crypto_req_common *req; + + while (!list_empty(&ac->queue) && !artpec6_crypto_busy()) { + req = list_first_entry(&ac->queue, + struct artpec6_crypto_req_common, + list); + list_move_tail(&req->list, &ac->pending); + artpec6_crypto_start_dma(req); + + list_add_tail(&req->complete_in_progress, completions); + } + + /* + * In some cases, the hardware can raise an in_eop_flush interrupt + * before actually updating the status, so we have an timer which will + * recheck the status on timeout. Since the cases are expected to be + * very rare, we use a relatively large timeout value. There should be + * no noticeable negative effect if we timeout spuriously. + */ + if (ac->pending_count) + mod_timer(&ac->timer, jiffies + msecs_to_jiffies(100)); + else + timer_delete(&ac->timer); +} + +static void artpec6_crypto_timeout(struct timer_list *t) +{ + struct artpec6_crypto *ac = timer_container_of(ac, t, timer); + + dev_info_ratelimited(artpec6_crypto_dev, "timeout\n"); + + tasklet_schedule(&ac->task); +} + +static void artpec6_crypto_task(unsigned long data) +{ + struct artpec6_crypto *ac = (struct artpec6_crypto *)data; + struct artpec6_crypto_req_common *req; + struct artpec6_crypto_req_common *n; + struct list_head complete_done; + struct list_head complete_in_progress; + + INIT_LIST_HEAD(&complete_done); + INIT_LIST_HEAD(&complete_in_progress); + + if (list_empty(&ac->pending)) { + pr_debug("Spurious IRQ\n"); + return; + } + + spin_lock(&ac->queue_lock); + + list_for_each_entry_safe(req, n, &ac->pending, list) { + struct artpec6_crypto_dma_descriptors *dma = req->dma; + u32 stat; + dma_addr_t stataddr; + + stataddr = dma->stat_dma_addr + 4 * (req->dma->in_cnt - 1); + dma_sync_single_for_cpu(artpec6_crypto_dev, + stataddr, + 4, + DMA_BIDIRECTIONAL); + + stat = req->dma->stat[req->dma->in_cnt-1]; + + /* A non-zero final status descriptor indicates + * this job has finished. + */ + pr_debug("Request %p status is %X\n", req, stat); + if (!stat) + break; + + /* Allow testing of timeout handling with fault injection */ +#ifdef CONFIG_FAULT_INJECTION + if (should_fail(&artpec6_crypto_fail_status_read, 1)) + continue; +#endif + + pr_debug("Completing request %p\n", req); + + list_move_tail(&req->list, &complete_done); + + ac->pending_count--; + } + + artpec6_crypto_process_queue(ac, &complete_in_progress); + + spin_unlock(&ac->queue_lock); + + /* Perform the completion callbacks without holding the queue lock + * to allow new request submissions from the callbacks. + */ + list_for_each_entry_safe(req, n, &complete_done, list) { + artpec6_crypto_dma_unmap_all(req); + artpec6_crypto_copy_bounce_buffers(req); + artpec6_crypto_common_destroy(req); + + req->complete(req->req); + } + + list_for_each_entry_safe(req, n, &complete_in_progress, + complete_in_progress) { + crypto_request_complete(req->req, -EINPROGRESS); + } +} + +static void artpec6_crypto_complete_crypto(struct crypto_async_request *req) +{ + crypto_request_complete(req, 0); +} + +static void +artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req) +{ + struct skcipher_request *cipher_req = container_of(req, + struct skcipher_request, base); + + scatterwalk_map_and_copy(cipher_req->iv, cipher_req->src, + cipher_req->cryptlen - AES_BLOCK_SIZE, + AES_BLOCK_SIZE, 0); + skcipher_request_complete(cipher_req, 0); +} + +static void +artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req) +{ + struct skcipher_request *cipher_req = container_of(req, + struct skcipher_request, base); + + scatterwalk_map_and_copy(cipher_req->iv, cipher_req->dst, + cipher_req->cryptlen - AES_BLOCK_SIZE, + AES_BLOCK_SIZE, 0); + skcipher_request_complete(cipher_req, 0); +} + +static void artpec6_crypto_complete_aead(struct crypto_async_request *req) +{ + int result = 0; + + /* Verify GCM hashtag. */ + struct aead_request *areq = container_of(req, + struct aead_request, base); + struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq); + + if (req_ctx->decrypt) { + u8 input_tag[AES_BLOCK_SIZE]; + unsigned int authsize = crypto_aead_authsize(aead); + + sg_pcopy_to_buffer(areq->src, + sg_nents(areq->src), + input_tag, + authsize, + areq->assoclen + areq->cryptlen - + authsize); + + if (crypto_memneq(req_ctx->decryption_tag, + input_tag, + authsize)) { + pr_debug("***EBADMSG:\n"); + print_hex_dump_debug("ref:", DUMP_PREFIX_ADDRESS, 32, 1, + input_tag, authsize, true); + print_hex_dump_debug("out:", DUMP_PREFIX_ADDRESS, 32, 1, + req_ctx->decryption_tag, + authsize, true); + + result = -EBADMSG; + } + } + + aead_request_complete(areq, result); +} + +static void artpec6_crypto_complete_hash(struct crypto_async_request *req) +{ + crypto_request_complete(req, 0); +} + + +/*------------------- Hash functions -----------------------------------------*/ +static int +artpec6_crypto_hash_set_key(struct crypto_ahash *tfm, + const u8 *key, unsigned int keylen) +{ + struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(&tfm->base); + size_t blocksize; + int ret; + + if (!keylen) { + pr_err("Invalid length (%d) of HMAC key\n", + keylen); + return -EINVAL; + } + + memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key)); + + blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + + if (keylen > blocksize) { + tfm_ctx->hmac_key_length = blocksize; + + ret = crypto_shash_tfm_digest(tfm_ctx->child_hash, key, keylen, + tfm_ctx->hmac_key); + if (ret) + return ret; + } else { + memcpy(tfm_ctx->hmac_key, key, keylen); + tfm_ctx->hmac_key_length = keylen; + } + + return 0; +} + +static int +artpec6_crypto_init_hash(struct ahash_request *req, u8 type, int hmac) +{ + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + enum artpec6_crypto_variant variant = ac->variant; + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + u32 oper; + + memset(req_ctx, 0, sizeof(*req_ctx)); + + req_ctx->hash_flags = HASH_FLAG_INIT_CTX; + if (hmac) + req_ctx->hash_flags |= (HASH_FLAG_HMAC | HASH_FLAG_UPDATE_KEY); + + switch (type) { + case ARTPEC6_CRYPTO_HASH_SHA1: + oper = hmac ? regk_crypto_hmac_sha1 : regk_crypto_sha1; + break; + case ARTPEC6_CRYPTO_HASH_SHA256: + oper = hmac ? regk_crypto_hmac_sha256 : regk_crypto_sha256; + break; + default: + pr_err("%s: Unsupported hash type 0x%x\n", MODULE_NAME, type); + return -EINVAL; + } + + if (variant == ARTPEC6_CRYPTO) + req_ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, oper); + else + req_ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, oper); + + return 0; +} + +static int artpec6_crypto_prepare_submit_hash(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + int ret; + + if (!req_ctx->common.dma) { + ret = artpec6_crypto_common_init(&req_ctx->common, + &req->base, + artpec6_crypto_complete_hash, + NULL, 0); + + if (ret) + return ret; + } + + ret = artpec6_crypto_prepare_hash(req); + switch (ret) { + case ARTPEC6_CRYPTO_PREPARE_HASH_START: + ret = artpec6_crypto_submit(&req_ctx->common); + break; + + case ARTPEC6_CRYPTO_PREPARE_HASH_NO_START: + ret = 0; + fallthrough; + + default: + artpec6_crypto_common_destroy(&req_ctx->common); + break; + } + + return ret; +} + +static int artpec6_crypto_hash_final(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + + req_ctx->hash_flags |= HASH_FLAG_FINALIZE; + + return artpec6_crypto_prepare_submit_hash(req); +} + +static int artpec6_crypto_hash_update(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + + req_ctx->hash_flags |= HASH_FLAG_UPDATE; + + return artpec6_crypto_prepare_submit_hash(req); +} + +static int artpec6_crypto_sha1_init(struct ahash_request *req) +{ + return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0); +} + +static int artpec6_crypto_sha1_digest(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + + artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0); + + req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; + + return artpec6_crypto_prepare_submit_hash(req); +} + +static int artpec6_crypto_sha256_init(struct ahash_request *req) +{ + return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0); +} + +static int artpec6_crypto_sha256_digest(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + + artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0); + req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; + + return artpec6_crypto_prepare_submit_hash(req); +} + +static int artpec6_crypto_hmac_sha256_init(struct ahash_request *req) +{ + return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1); +} + +static int artpec6_crypto_hmac_sha256_digest(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + + artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1); + req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; + + return artpec6_crypto_prepare_submit_hash(req); +} + +static int artpec6_crypto_ahash_init_common(struct crypto_tfm *tfm, + const char *base_hash_name) +{ + struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm); + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct artpec6_hash_request_context)); + memset(tfm_ctx, 0, sizeof(*tfm_ctx)); + + if (base_hash_name) { + struct crypto_shash *child; + + child = crypto_alloc_shash(base_hash_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + + if (IS_ERR(child)) + return PTR_ERR(child); + + tfm_ctx->child_hash = child; + } + + return 0; +} + +static int artpec6_crypto_ahash_init(struct crypto_tfm *tfm) +{ + return artpec6_crypto_ahash_init_common(tfm, NULL); +} + +static int artpec6_crypto_ahash_init_hmac_sha256(struct crypto_tfm *tfm) +{ + return artpec6_crypto_ahash_init_common(tfm, "sha256"); +} + +static void artpec6_crypto_ahash_exit(struct crypto_tfm *tfm) +{ + struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm); + + if (tfm_ctx->child_hash) + crypto_free_shash(tfm_ctx->child_hash); + + memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key)); + tfm_ctx->hmac_key_length = 0; +} + +static int artpec6_crypto_hash_export(struct ahash_request *req, void *out) +{ + const struct artpec6_hash_request_context *ctx = ahash_request_ctx(req); + struct artpec6_hash_export_state *state = out; + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + enum artpec6_crypto_variant variant = ac->variant; + + BUILD_BUG_ON(sizeof(state->partial_buffer) != + sizeof(ctx->partial_buffer)); + BUILD_BUG_ON(sizeof(state->digeststate) != sizeof(ctx->digeststate)); + + state->digcnt = ctx->digcnt; + state->partial_bytes = ctx->partial_bytes; + state->hash_flags = ctx->hash_flags; + + if (variant == ARTPEC6_CRYPTO) + state->oper = FIELD_GET(A6_CRY_MD_OPER, ctx->hash_md); + else + state->oper = FIELD_GET(A7_CRY_MD_OPER, ctx->hash_md); + + memcpy(state->partial_buffer, ctx->partial_buffer, + sizeof(state->partial_buffer)); + memcpy(state->digeststate, ctx->digeststate, + sizeof(state->digeststate)); + + return 0; +} + +static int artpec6_crypto_hash_import(struct ahash_request *req, const void *in) +{ + struct artpec6_hash_request_context *ctx = ahash_request_ctx(req); + const struct artpec6_hash_export_state *state = in; + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + enum artpec6_crypto_variant variant = ac->variant; + + memset(ctx, 0, sizeof(*ctx)); + + ctx->digcnt = state->digcnt; + ctx->partial_bytes = state->partial_bytes; + ctx->hash_flags = state->hash_flags; + + if (variant == ARTPEC6_CRYPTO) + ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, state->oper); + else + ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, state->oper); + + memcpy(ctx->partial_buffer, state->partial_buffer, + sizeof(state->partial_buffer)); + memcpy(ctx->digeststate, state->digeststate, + sizeof(state->digeststate)); + + return 0; +} + +static int init_crypto_hw(struct artpec6_crypto *ac) +{ + enum artpec6_crypto_variant variant = ac->variant; + void __iomem *base = ac->base; + u32 out_descr_buf_size; + u32 out_data_buf_size; + u32 in_data_buf_size; + u32 in_descr_buf_size; + u32 in_stat_buf_size; + u32 in, out; + + /* + * The PDMA unit contains 1984 bytes of internal memory for the OUT + * channels and 1024 bytes for the IN channel. This is an elastic + * memory used to internally store the descriptors and data. The values + * ares specified in 64 byte incremements. Trustzone buffers are not + * used at this stage. + */ + out_data_buf_size = 16; /* 1024 bytes for data */ + out_descr_buf_size = 15; /* 960 bytes for descriptors */ + in_data_buf_size = 8; /* 512 bytes for data */ + in_descr_buf_size = 4; /* 256 bytes for descriptors */ + in_stat_buf_size = 4; /* 256 bytes for stat descrs */ + + BUILD_BUG_ON_MSG((out_data_buf_size + + out_descr_buf_size) * 64 > 1984, + "Invalid OUT configuration"); + + BUILD_BUG_ON_MSG((in_data_buf_size + + in_descr_buf_size + + in_stat_buf_size) * 64 > 1024, + "Invalid IN configuration"); + + in = FIELD_PREP(PDMA_IN_BUF_CFG_DATA_BUF_SIZE, in_data_buf_size) | + FIELD_PREP(PDMA_IN_BUF_CFG_DESCR_BUF_SIZE, in_descr_buf_size) | + FIELD_PREP(PDMA_IN_BUF_CFG_STAT_BUF_SIZE, in_stat_buf_size); + + out = FIELD_PREP(PDMA_OUT_BUF_CFG_DATA_BUF_SIZE, out_data_buf_size) | + FIELD_PREP(PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE, out_descr_buf_size); + + writel_relaxed(out, base + PDMA_OUT_BUF_CFG); + writel_relaxed(PDMA_OUT_CFG_EN, base + PDMA_OUT_CFG); + + if (variant == ARTPEC6_CRYPTO) { + writel_relaxed(in, base + A6_PDMA_IN_BUF_CFG); + writel_relaxed(PDMA_IN_CFG_EN, base + A6_PDMA_IN_CFG); + writel_relaxed(A6_PDMA_INTR_MASK_IN_DATA | + A6_PDMA_INTR_MASK_IN_EOP_FLUSH, + base + A6_PDMA_INTR_MASK); + } else { + writel_relaxed(in, base + A7_PDMA_IN_BUF_CFG); + writel_relaxed(PDMA_IN_CFG_EN, base + A7_PDMA_IN_CFG); + writel_relaxed(A7_PDMA_INTR_MASK_IN_DATA | + A7_PDMA_INTR_MASK_IN_EOP_FLUSH, + base + A7_PDMA_INTR_MASK); + } + + return 0; +} + +static void artpec6_crypto_disable_hw(struct artpec6_crypto *ac) +{ + enum artpec6_crypto_variant variant = ac->variant; + void __iomem *base = ac->base; + + if (variant == ARTPEC6_CRYPTO) { + writel_relaxed(A6_PDMA_IN_CMD_STOP, base + A6_PDMA_IN_CMD); + writel_relaxed(0, base + A6_PDMA_IN_CFG); + writel_relaxed(A6_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD); + } else { + writel_relaxed(A7_PDMA_IN_CMD_STOP, base + A7_PDMA_IN_CMD); + writel_relaxed(0, base + A7_PDMA_IN_CFG); + writel_relaxed(A7_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD); + } + + writel_relaxed(0, base + PDMA_OUT_CFG); + +} + +static irqreturn_t artpec6_crypto_irq(int irq, void *dev_id) +{ + struct artpec6_crypto *ac = dev_id; + enum artpec6_crypto_variant variant = ac->variant; + void __iomem *base = ac->base; + u32 mask_in_data, mask_in_eop_flush; + u32 in_cmd_flush_stat, in_cmd_reg; + u32 ack_intr_reg; + u32 ack = 0; + u32 intr; + + if (variant == ARTPEC6_CRYPTO) { + intr = readl_relaxed(base + A6_PDMA_MASKED_INTR); + mask_in_data = A6_PDMA_INTR_MASK_IN_DATA; + mask_in_eop_flush = A6_PDMA_INTR_MASK_IN_EOP_FLUSH; + in_cmd_flush_stat = A6_PDMA_IN_CMD_FLUSH_STAT; + in_cmd_reg = A6_PDMA_IN_CMD; + ack_intr_reg = A6_PDMA_ACK_INTR; + } else { + intr = readl_relaxed(base + A7_PDMA_MASKED_INTR); + mask_in_data = A7_PDMA_INTR_MASK_IN_DATA; + mask_in_eop_flush = A7_PDMA_INTR_MASK_IN_EOP_FLUSH; + in_cmd_flush_stat = A7_PDMA_IN_CMD_FLUSH_STAT; + in_cmd_reg = A7_PDMA_IN_CMD; + ack_intr_reg = A7_PDMA_ACK_INTR; + } + + /* We get two interrupt notifications from each job. + * The in_data means all data was sent to memory and then + * we request a status flush command to write the per-job + * status to its status vector. This ensures that the + * tasklet can detect exactly how many submitted jobs + * that have finished. + */ + if (intr & mask_in_data) + ack |= mask_in_data; + + if (intr & mask_in_eop_flush) + ack |= mask_in_eop_flush; + else + writel_relaxed(in_cmd_flush_stat, base + in_cmd_reg); + + writel_relaxed(ack, base + ack_intr_reg); + + if (intr & mask_in_eop_flush) + tasklet_schedule(&ac->task); + + return IRQ_HANDLED; +} + +/*------------------- Algorithm definitions ----------------------------------*/ + +/* Hashes */ +static struct ahash_alg hash_algos[] = { + /* SHA-1 */ + { + .init = artpec6_crypto_sha1_init, + .update = artpec6_crypto_hash_update, + .final = artpec6_crypto_hash_final, + .digest = artpec6_crypto_sha1_digest, + .import = artpec6_crypto_hash_import, + .export = artpec6_crypto_hash_export, + .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.statesize = sizeof(struct artpec6_hash_export_state), + .halg.base = { + .cra_name = "sha1", + .cra_driver_name = "artpec-sha1", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct artpec6_hashalg_context), + .cra_module = THIS_MODULE, + .cra_init = artpec6_crypto_ahash_init, + .cra_exit = artpec6_crypto_ahash_exit, + } + }, + /* SHA-256 */ + { + .init = artpec6_crypto_sha256_init, + .update = artpec6_crypto_hash_update, + .final = artpec6_crypto_hash_final, + .digest = artpec6_crypto_sha256_digest, + .import = artpec6_crypto_hash_import, + .export = artpec6_crypto_hash_export, + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.statesize = sizeof(struct artpec6_hash_export_state), + .halg.base = { + .cra_name = "sha256", + .cra_driver_name = "artpec-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct artpec6_hashalg_context), + .cra_module = THIS_MODULE, + .cra_init = artpec6_crypto_ahash_init, + .cra_exit = artpec6_crypto_ahash_exit, + } + }, + /* HMAC SHA-256 */ + { + .init = artpec6_crypto_hmac_sha256_init, + .update = artpec6_crypto_hash_update, + .final = artpec6_crypto_hash_final, + .digest = artpec6_crypto_hmac_sha256_digest, + .import = artpec6_crypto_hash_import, + .export = artpec6_crypto_hash_export, + .setkey = artpec6_crypto_hash_set_key, + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.statesize = sizeof(struct artpec6_hash_export_state), + .halg.base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "artpec-hmac-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct artpec6_hashalg_context), + .cra_module = THIS_MODULE, + .cra_init = artpec6_crypto_ahash_init_hmac_sha256, + .cra_exit = artpec6_crypto_ahash_exit, + } + }, +}; + +/* Crypto */ +static struct skcipher_alg crypto_algos[] = { + /* AES - ECB */ + { + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "artpec6-ecb-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = artpec6_crypto_cipher_set_key, + .encrypt = artpec6_crypto_encrypt, + .decrypt = artpec6_crypto_decrypt, + .init = artpec6_crypto_aes_ecb_init, + .exit = artpec6_crypto_aes_exit, + }, + /* AES - CTR */ + { + .base = { + .cra_name = "ctr(aes)", + .cra_driver_name = "artpec6-ctr-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = artpec6_crypto_cipher_set_key, + .encrypt = artpec6_crypto_ctr_encrypt, + .decrypt = artpec6_crypto_ctr_decrypt, + .init = artpec6_crypto_aes_ctr_init, + .exit = artpec6_crypto_aes_ctr_exit, + }, + /* AES - CBC */ + { + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "artpec6-cbc-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = artpec6_crypto_cipher_set_key, + .encrypt = artpec6_crypto_encrypt, + .decrypt = artpec6_crypto_decrypt, + .init = artpec6_crypto_aes_cbc_init, + .exit = artpec6_crypto_aes_exit + }, + /* AES - XTS */ + { + .base = { + .cra_name = "xts(aes)", + .cra_driver_name = "artpec6-xts-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + }, + .min_keysize = 2*AES_MIN_KEY_SIZE, + .max_keysize = 2*AES_MAX_KEY_SIZE, + .ivsize = 16, + .setkey = artpec6_crypto_xts_set_key, + .encrypt = artpec6_crypto_encrypt, + .decrypt = artpec6_crypto_decrypt, + .init = artpec6_crypto_aes_xts_init, + .exit = artpec6_crypto_aes_exit, + }, +}; + +static struct aead_alg aead_algos[] = { + { + .init = artpec6_crypto_aead_init, + .setkey = artpec6_crypto_aead_set_key, + .encrypt = artpec6_crypto_aead_encrypt, + .decrypt = artpec6_crypto_aead_decrypt, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "artpec-gcm-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + }, + } +}; + +#ifdef CONFIG_DEBUG_FS + +static struct dentry *dbgfs_root; + +static void artpec6_crypto_init_debugfs(void) +{ + dbgfs_root = debugfs_create_dir("artpec6_crypto", NULL); + +#ifdef CONFIG_FAULT_INJECTION + fault_create_debugfs_attr("fail_status_read", dbgfs_root, + &artpec6_crypto_fail_status_read); + + fault_create_debugfs_attr("fail_dma_array_full", dbgfs_root, + &artpec6_crypto_fail_dma_array_full); +#endif +} + +static void artpec6_crypto_free_debugfs(void) +{ + debugfs_remove_recursive(dbgfs_root); + dbgfs_root = NULL; +} +#endif + +static const struct of_device_id artpec6_crypto_of_match[] = { + { .compatible = "axis,artpec6-crypto", .data = (void *)ARTPEC6_CRYPTO }, + { .compatible = "axis,artpec7-crypto", .data = (void *)ARTPEC7_CRYPTO }, + {} +}; +MODULE_DEVICE_TABLE(of, artpec6_crypto_of_match); + +static int artpec6_crypto_probe(struct platform_device *pdev) +{ + enum artpec6_crypto_variant variant; + struct artpec6_crypto *ac; + struct device *dev = &pdev->dev; + void __iomem *base; + int irq; + int err; + + if (artpec6_crypto_dev) + return -ENODEV; + + variant = (enum artpec6_crypto_variant)of_device_get_match_data(dev); + if (!variant) + return -EINVAL; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return -ENODEV; + + ac = devm_kzalloc(&pdev->dev, sizeof(struct artpec6_crypto), + GFP_KERNEL); + if (!ac) + return -ENOMEM; + + platform_set_drvdata(pdev, ac); + ac->variant = variant; + + spin_lock_init(&ac->queue_lock); + INIT_LIST_HEAD(&ac->queue); + INIT_LIST_HEAD(&ac->pending); + timer_setup(&ac->timer, artpec6_crypto_timeout, 0); + + ac->base = base; + + ac->dma_cache = kmem_cache_create("artpec6_crypto_dma", + sizeof(struct artpec6_crypto_dma_descriptors), + 64, + 0, + NULL); + if (!ac->dma_cache) + return -ENOMEM; + +#ifdef CONFIG_DEBUG_FS + artpec6_crypto_init_debugfs(); +#endif + + tasklet_init(&ac->task, artpec6_crypto_task, + (unsigned long)ac); + + ac->pad_buffer = devm_kcalloc(&pdev->dev, 2, ARTPEC_CACHE_LINE_MAX, + GFP_KERNEL); + if (!ac->pad_buffer) + return -ENOMEM; + ac->pad_buffer = PTR_ALIGN(ac->pad_buffer, ARTPEC_CACHE_LINE_MAX); + + ac->zero_buffer = devm_kcalloc(&pdev->dev, 2, ARTPEC_CACHE_LINE_MAX, + GFP_KERNEL); + if (!ac->zero_buffer) + return -ENOMEM; + ac->zero_buffer = PTR_ALIGN(ac->zero_buffer, ARTPEC_CACHE_LINE_MAX); + + err = init_crypto_hw(ac); + if (err) + goto free_cache; + + err = devm_request_irq(&pdev->dev, irq, artpec6_crypto_irq, 0, + "artpec6-crypto", ac); + if (err) + goto disable_hw; + + artpec6_crypto_dev = &pdev->dev; + + err = crypto_register_ahashes(hash_algos, ARRAY_SIZE(hash_algos)); + if (err) { + dev_err(dev, "Failed to register ahashes\n"); + goto disable_hw; + } + + err = crypto_register_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos)); + if (err) { + dev_err(dev, "Failed to register ciphers\n"); + goto unregister_ahashes; + } + + err = crypto_register_aeads(aead_algos, ARRAY_SIZE(aead_algos)); + if (err) { + dev_err(dev, "Failed to register aeads\n"); + goto unregister_algs; + } + + return 0; + +unregister_algs: + crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos)); +unregister_ahashes: + crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos)); +disable_hw: + artpec6_crypto_disable_hw(ac); +free_cache: + kmem_cache_destroy(ac->dma_cache); + return err; +} + +static void artpec6_crypto_remove(struct platform_device *pdev) +{ + struct artpec6_crypto *ac = platform_get_drvdata(pdev); + int irq = platform_get_irq(pdev, 0); + + crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos)); + crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos)); + crypto_unregister_aeads(aead_algos, ARRAY_SIZE(aead_algos)); + + tasklet_disable(&ac->task); + devm_free_irq(&pdev->dev, irq, ac); + tasklet_kill(&ac->task); + timer_delete_sync(&ac->timer); + + artpec6_crypto_disable_hw(ac); + + kmem_cache_destroy(ac->dma_cache); +#ifdef CONFIG_DEBUG_FS + artpec6_crypto_free_debugfs(); +#endif +} + +static struct platform_driver artpec6_crypto_driver = { + .probe = artpec6_crypto_probe, + .remove = artpec6_crypto_remove, + .driver = { + .name = "artpec6-crypto", + .of_match_table = artpec6_crypto_of_match, + }, +}; + +module_platform_driver(artpec6_crypto_driver); + +MODULE_AUTHOR("Axis Communications AB"); +MODULE_DESCRIPTION("ARTPEC-6 Crypto driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/bcm/Makefile b/drivers/crypto/bcm/Makefile new file mode 100644 index 000000000..8a2110b4e --- /dev/null +++ b/drivers/crypto/bcm/Makefile @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0-only +# File: drivers/crypto/bcm/Makefile +# +# Makefile for crypto acceleration files for Broadcom SPU driver +# +# Uncomment to enable debug tracing in the SPU driver. +# CFLAGS_util.o := -DDEBUG +# CFLAGS_cipher.o := -DDEBUG +# CFLAGS_spu.o := -DDEBUG +# CFLAGS_spu2.o := -DDEBUG + +obj-$(CONFIG_CRYPTO_DEV_BCM_SPU) := bcm_crypto_spu.o + +bcm_crypto_spu-objs := util.o spu.o spu2.o cipher.o diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c new file mode 100644 index 000000000..e9d9ca744 --- /dev/null +++ b/drivers/crypto/bcm/cipher.c @@ -0,0 +1,4709 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2016 Broadcom + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "util.h" +#include "cipher.h" +#include "spu.h" +#include "spum.h" +#include "spu2.h" + +/* ================= Device Structure ================== */ + +struct bcm_device_private iproc_priv; + +/* ==================== Parameters ===================== */ + +int flow_debug_logging; +module_param(flow_debug_logging, int, 0644); +MODULE_PARM_DESC(flow_debug_logging, "Enable Flow Debug Logging"); + +int packet_debug_logging; +module_param(packet_debug_logging, int, 0644); +MODULE_PARM_DESC(packet_debug_logging, "Enable Packet Debug Logging"); + +int debug_logging_sleep; +module_param(debug_logging_sleep, int, 0644); +MODULE_PARM_DESC(debug_logging_sleep, "Packet Debug Logging Sleep"); + +/* + * The value of these module parameters is used to set the priority for each + * algo type when this driver registers algos with the kernel crypto API. + * To use a priority other than the default, set the priority in the insmod or + * modprobe. Changing the module priority after init time has no effect. + * + * The default priorities are chosen to be lower (less preferred) than ARMv8 CE + * algos, but more preferred than generic software algos. + */ +static int cipher_pri = 150; +module_param(cipher_pri, int, 0644); +MODULE_PARM_DESC(cipher_pri, "Priority for cipher algos"); + +static int hash_pri = 100; +module_param(hash_pri, int, 0644); +MODULE_PARM_DESC(hash_pri, "Priority for hash algos"); + +static int aead_pri = 150; +module_param(aead_pri, int, 0644); +MODULE_PARM_DESC(aead_pri, "Priority for AEAD algos"); + +/* A type 3 BCM header, expected to precede the SPU header for SPU-M. + * Bits 3 and 4 in the first byte encode the channel number (the dma ringset). + * 0x60 - ring 0 + * 0x68 - ring 1 + * 0x70 - ring 2 + * 0x78 - ring 3 + */ +static char BCMHEADER[] = { 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28 }; +/* + * Some SPU hw does not use BCM header on SPU messages. So BCM_HDR_LEN + * is set dynamically after reading SPU type from device tree. + */ +#define BCM_HDR_LEN iproc_priv.bcm_hdr_len + +/* min and max time to sleep before retrying when mbox queue is full. usec */ +#define MBOX_SLEEP_MIN 800 +#define MBOX_SLEEP_MAX 1000 + +/** + * select_channel() - Select a SPU channel to handle a crypto request. Selects + * channel in round robin order. + * + * Return: channel index + */ +static u8 select_channel(void) +{ + u8 chan_idx = atomic_inc_return(&iproc_priv.next_chan); + + return chan_idx % iproc_priv.spu.num_chan; +} + +/** + * spu_skcipher_rx_sg_create() - Build up the scatterlist of buffers used to + * receive a SPU response message for an skcipher request. Includes buffers to + * catch SPU message headers and the response data. + * @mssg: mailbox message containing the receive sg + * @rctx: crypto request context + * @rx_frag_num: number of scatterlist elements required to hold the + * SPU response message + * @chunksize: Number of bytes of response data expected + * @stat_pad_len: Number of bytes required to pad the STAT field to + * a 4-byte boundary + * + * The scatterlist that gets allocated here is freed in spu_chunk_cleanup() + * when the request completes, whether the request is handled successfully or + * there is an error. + * + * Returns: + * 0 if successful + * < 0 if an error + */ +static int +spu_skcipher_rx_sg_create(struct brcm_message *mssg, + struct iproc_reqctx_s *rctx, + u8 rx_frag_num, + unsigned int chunksize, u32 stat_pad_len) +{ + struct spu_hw *spu = &iproc_priv.spu; + struct scatterlist *sg; /* used to build sgs in mbox message */ + struct iproc_ctx_s *ctx = rctx->ctx; + u32 datalen; /* Number of bytes of response data expected */ + + mssg->spu.dst = kmalloc_objs(struct scatterlist, rx_frag_num, rctx->gfp); + if (!mssg->spu.dst) + return -ENOMEM; + + sg = mssg->spu.dst; + sg_init_table(sg, rx_frag_num); + /* Space for SPU message header */ + sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len); + + /* If XTS tweak in payload, add buffer to receive encrypted tweak */ + if ((ctx->cipher.mode == CIPHER_MODE_XTS) && + spu->spu_xts_tweak_in_payload()) + sg_set_buf(sg++, rctx->msg_buf.c.supdt_tweak, + SPU_XTS_TWEAK_SIZE); + + /* Copy in each dst sg entry from request, up to chunksize */ + datalen = spu_msg_sg_add(&sg, &rctx->dst_sg, &rctx->dst_skip, + rctx->dst_nents, chunksize); + if (datalen < chunksize) { + pr_err("%s(): failed to copy dst sg to mbox msg. chunksize %u, datalen %u", + __func__, chunksize, datalen); + return -EFAULT; + } + + if (stat_pad_len) + sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len); + + memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN); + sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len()); + + return 0; +} + +/** + * spu_skcipher_tx_sg_create() - Build up the scatterlist of buffers used to + * send a SPU request message for an skcipher request. Includes SPU message + * headers and the request data. + * @mssg: mailbox message containing the transmit sg + * @rctx: crypto request context + * @tx_frag_num: number of scatterlist elements required to construct the + * SPU request message + * @chunksize: Number of bytes of request data + * @pad_len: Number of pad bytes + * + * The scatterlist that gets allocated here is freed in spu_chunk_cleanup() + * when the request completes, whether the request is handled successfully or + * there is an error. + * + * Returns: + * 0 if successful + * < 0 if an error + */ +static int +spu_skcipher_tx_sg_create(struct brcm_message *mssg, + struct iproc_reqctx_s *rctx, + u8 tx_frag_num, unsigned int chunksize, u32 pad_len) +{ + struct spu_hw *spu = &iproc_priv.spu; + struct scatterlist *sg; /* used to build sgs in mbox message */ + struct iproc_ctx_s *ctx = rctx->ctx; + u32 datalen; /* Number of bytes of response data expected */ + u32 stat_len; + + mssg->spu.src = kmalloc_objs(struct scatterlist, tx_frag_num, rctx->gfp); + if (unlikely(!mssg->spu.src)) + return -ENOMEM; + + sg = mssg->spu.src; + sg_init_table(sg, tx_frag_num); + + sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr, + BCM_HDR_LEN + ctx->spu_req_hdr_len); + + /* if XTS tweak in payload, copy from IV (where crypto API puts it) */ + if ((ctx->cipher.mode == CIPHER_MODE_XTS) && + spu->spu_xts_tweak_in_payload()) + sg_set_buf(sg++, rctx->msg_buf.iv_ctr, SPU_XTS_TWEAK_SIZE); + + /* Copy in each src sg entry from request, up to chunksize */ + datalen = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip, + rctx->src_nents, chunksize); + if (unlikely(datalen < chunksize)) { + pr_err("%s(): failed to copy src sg to mbox msg", + __func__); + return -EFAULT; + } + + if (pad_len) + sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len); + + stat_len = spu->spu_tx_status_len(); + if (stat_len) { + memset(rctx->msg_buf.tx_stat, 0, stat_len); + sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len); + } + return 0; +} + +static int mailbox_send_message(struct brcm_message *mssg, u32 flags, + u8 chan_idx) +{ + int err; + int retry_cnt = 0; + struct device *dev = &(iproc_priv.pdev->dev); + + err = mbox_send_message(iproc_priv.mbox[chan_idx], mssg); + if (flags & CRYPTO_TFM_REQ_MAY_SLEEP) { + while ((err == -ENOBUFS) && (retry_cnt < SPU_MB_RETRY_MAX)) { + /* + * Mailbox queue is full. Since MAY_SLEEP is set, assume + * not in atomic context and we can wait and try again. + */ + retry_cnt++; + usleep_range(MBOX_SLEEP_MIN, MBOX_SLEEP_MAX); + err = mbox_send_message(iproc_priv.mbox[chan_idx], + mssg); + atomic_inc(&iproc_priv.mb_no_spc); + } + } + if (err < 0) { + atomic_inc(&iproc_priv.mb_send_fail); + return err; + } + + /* Check error returned by mailbox controller */ + err = mssg->error; + if (unlikely(err < 0)) { + dev_err(dev, "message error %d", err); + /* Signal txdone for mailbox channel */ + } + + /* Signal txdone for mailbox channel */ + mbox_client_txdone(iproc_priv.mbox[chan_idx], err); + return err; +} + +/** + * handle_skcipher_req() - Submit as much of a block cipher request as fits in + * a single SPU request message, starting at the current position in the request + * data. + * @rctx: Crypto request context + * + * This may be called on the crypto API thread, or, when a request is so large + * it must be broken into multiple SPU messages, on the thread used to invoke + * the response callback. When requests are broken into multiple SPU + * messages, we assume subsequent messages depend on previous results, and + * thus always wait for previous results before submitting the next message. + * Because requests are submitted in lock step like this, there is no need + * to synchronize access to request data structures. + * + * Return: -EINPROGRESS: request has been accepted and result will be returned + * asynchronously + * Any other value indicates an error + */ +static int handle_skcipher_req(struct iproc_reqctx_s *rctx) +{ + struct spu_hw *spu = &iproc_priv.spu; + struct crypto_async_request *areq = rctx->parent; + struct skcipher_request *req = + container_of(areq, struct skcipher_request, base); + struct iproc_ctx_s *ctx = rctx->ctx; + struct spu_cipher_parms cipher_parms; + int err; + unsigned int chunksize; /* Num bytes of request to submit */ + int remaining; /* Bytes of request still to process */ + int chunk_start; /* Beginning of data for current SPU msg */ + + /* IV or ctr value to use in this SPU msg */ + u8 local_iv_ctr[MAX_IV_SIZE]; + u32 stat_pad_len; /* num bytes to align status field */ + u32 pad_len; /* total length of all padding */ + struct brcm_message *mssg; /* mailbox message */ + + /* number of entries in src and dst sg in mailbox message. */ + u8 rx_frag_num = 2; /* response header and STATUS */ + u8 tx_frag_num = 1; /* request header */ + + flow_log("%s\n", __func__); + + cipher_parms.alg = ctx->cipher.alg; + cipher_parms.mode = ctx->cipher.mode; + cipher_parms.type = ctx->cipher_type; + cipher_parms.key_len = ctx->enckeylen; + cipher_parms.key_buf = ctx->enckey; + cipher_parms.iv_buf = local_iv_ctr; + cipher_parms.iv_len = rctx->iv_ctr_len; + + mssg = &rctx->mb_mssg; + chunk_start = rctx->src_sent; + remaining = rctx->total_todo - chunk_start; + + /* determine the chunk we are breaking off and update the indexes */ + if ((ctx->max_payload != SPU_MAX_PAYLOAD_INF) && + (remaining > ctx->max_payload)) + chunksize = ctx->max_payload; + else + chunksize = remaining; + + rctx->src_sent += chunksize; + rctx->total_sent = rctx->src_sent; + + /* Count number of sg entries to be included in this request */ + rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, chunksize); + rctx->dst_nents = spu_sg_count(rctx->dst_sg, rctx->dst_skip, chunksize); + + if ((ctx->cipher.mode == CIPHER_MODE_CBC) && + rctx->is_encrypt && chunk_start) + /* + * Encrypting non-first first chunk. Copy last block of + * previous result to IV for this chunk. + */ + sg_copy_part_to_buf(req->dst, rctx->msg_buf.iv_ctr, + rctx->iv_ctr_len, + chunk_start - rctx->iv_ctr_len); + + if (rctx->iv_ctr_len) { + /* get our local copy of the iv */ + __builtin_memcpy(local_iv_ctr, rctx->msg_buf.iv_ctr, + rctx->iv_ctr_len); + + /* generate the next IV if possible */ + if ((ctx->cipher.mode == CIPHER_MODE_CBC) && + !rctx->is_encrypt) { + /* + * CBC Decrypt: next IV is the last ciphertext block in + * this chunk + */ + sg_copy_part_to_buf(req->src, rctx->msg_buf.iv_ctr, + rctx->iv_ctr_len, + rctx->src_sent - rctx->iv_ctr_len); + } else if (ctx->cipher.mode == CIPHER_MODE_CTR) { + /* + * The SPU hardware increments the counter once for + * each AES block of 16 bytes. So update the counter + * for the next chunk, if there is one. Note that for + * this chunk, the counter has already been copied to + * local_iv_ctr. We can assume a block size of 16, + * because we only support CTR mode for AES, not for + * any other cipher alg. + */ + add_to_ctr(rctx->msg_buf.iv_ctr, chunksize >> 4); + } + } + + if (ctx->max_payload == SPU_MAX_PAYLOAD_INF) + flow_log("max_payload infinite\n"); + else + flow_log("max_payload %u\n", ctx->max_payload); + + flow_log("sent:%u start:%u remains:%u size:%u\n", + rctx->src_sent, chunk_start, remaining, chunksize); + + /* Copy SPU header template created at setkey time */ + memcpy(rctx->msg_buf.bcm_spu_req_hdr, ctx->bcm_spu_req_hdr, + sizeof(rctx->msg_buf.bcm_spu_req_hdr)); + + spu->spu_cipher_req_finish(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN, + ctx->spu_req_hdr_len, !(rctx->is_encrypt), + &cipher_parms, chunksize); + + atomic64_add(chunksize, &iproc_priv.bytes_out); + + stat_pad_len = spu->spu_wordalign_padlen(chunksize); + if (stat_pad_len) + rx_frag_num++; + pad_len = stat_pad_len; + if (pad_len) { + tx_frag_num++; + spu->spu_request_pad(rctx->msg_buf.spu_req_pad, 0, + 0, ctx->auth.alg, ctx->auth.mode, + rctx->total_sent, stat_pad_len); + } + + spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN, + ctx->spu_req_hdr_len); + packet_log("payload:\n"); + dump_sg(rctx->src_sg, rctx->src_skip, chunksize); + packet_dump(" pad: ", rctx->msg_buf.spu_req_pad, pad_len); + + /* + * Build mailbox message containing SPU request msg and rx buffers + * to catch response message + */ + memset(mssg, 0, sizeof(*mssg)); + mssg->type = BRCM_MESSAGE_SPU; + mssg->ctx = rctx; /* Will be returned in response */ + + /* Create rx scatterlist to catch result */ + rx_frag_num += rctx->dst_nents; + + if ((ctx->cipher.mode == CIPHER_MODE_XTS) && + spu->spu_xts_tweak_in_payload()) + rx_frag_num++; /* extra sg to insert tweak */ + + err = spu_skcipher_rx_sg_create(mssg, rctx, rx_frag_num, chunksize, + stat_pad_len); + if (err) + return err; + + /* Create tx scatterlist containing SPU request message */ + tx_frag_num += rctx->src_nents; + if (spu->spu_tx_status_len()) + tx_frag_num++; + + if ((ctx->cipher.mode == CIPHER_MODE_XTS) && + spu->spu_xts_tweak_in_payload()) + tx_frag_num++; /* extra sg to insert tweak */ + + err = spu_skcipher_tx_sg_create(mssg, rctx, tx_frag_num, chunksize, + pad_len); + if (err) + return err; + + err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx); + if (unlikely(err < 0)) + return err; + + return -EINPROGRESS; +} + +/** + * handle_skcipher_resp() - Process a block cipher SPU response. Updates the + * total received count for the request and updates global stats. + * @rctx: Crypto request context + */ +static void handle_skcipher_resp(struct iproc_reqctx_s *rctx) +{ + struct spu_hw *spu = &iproc_priv.spu; + struct crypto_async_request *areq = rctx->parent; + struct skcipher_request *req = skcipher_request_cast(areq); + struct iproc_ctx_s *ctx = rctx->ctx; + u32 payload_len; + + /* See how much data was returned */ + payload_len = spu->spu_payload_length(rctx->msg_buf.spu_resp_hdr); + + /* + * In XTS mode, the first SPU_XTS_TWEAK_SIZE bytes may be the + * encrypted tweak ("i") value; we don't count those. + */ + if ((ctx->cipher.mode == CIPHER_MODE_XTS) && + spu->spu_xts_tweak_in_payload() && + (payload_len >= SPU_XTS_TWEAK_SIZE)) + payload_len -= SPU_XTS_TWEAK_SIZE; + + atomic64_add(payload_len, &iproc_priv.bytes_in); + + flow_log("%s() offset: %u, bd_len: %u BD:\n", + __func__, rctx->total_received, payload_len); + + dump_sg(req->dst, rctx->total_received, payload_len); + + rctx->total_received += payload_len; + if (rctx->total_received == rctx->total_todo) { + atomic_inc(&iproc_priv.op_counts[SPU_OP_CIPHER]); + atomic_inc( + &iproc_priv.cipher_cnt[ctx->cipher.alg][ctx->cipher.mode]); + } +} + +/** + * spu_ahash_rx_sg_create() - Build up the scatterlist of buffers used to + * receive a SPU response message for an ahash request. + * @mssg: mailbox message containing the receive sg + * @rctx: crypto request context + * @rx_frag_num: number of scatterlist elements required to hold the + * SPU response message + * @digestsize: length of hash digest, in bytes + * @stat_pad_len: Number of bytes required to pad the STAT field to + * a 4-byte boundary + * + * The scatterlist that gets allocated here is freed in spu_chunk_cleanup() + * when the request completes, whether the request is handled successfully or + * there is an error. + * + * Return: + * 0 if successful + * < 0 if an error + */ +static int +spu_ahash_rx_sg_create(struct brcm_message *mssg, + struct iproc_reqctx_s *rctx, + u8 rx_frag_num, unsigned int digestsize, + u32 stat_pad_len) +{ + struct spu_hw *spu = &iproc_priv.spu; + struct scatterlist *sg; /* used to build sgs in mbox message */ + struct iproc_ctx_s *ctx = rctx->ctx; + + mssg->spu.dst = kmalloc_objs(struct scatterlist, rx_frag_num, rctx->gfp); + if (!mssg->spu.dst) + return -ENOMEM; + + sg = mssg->spu.dst; + sg_init_table(sg, rx_frag_num); + /* Space for SPU message header */ + sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len); + + /* Space for digest */ + sg_set_buf(sg++, rctx->msg_buf.digest, digestsize); + + if (stat_pad_len) + sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len); + + memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN); + sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len()); + return 0; +} + +/** + * spu_ahash_tx_sg_create() - Build up the scatterlist of buffers used to send + * a SPU request message for an ahash request. Includes SPU message headers and + * the request data. + * @mssg: mailbox message containing the transmit sg + * @rctx: crypto request context + * @tx_frag_num: number of scatterlist elements required to construct the + * SPU request message + * @spu_hdr_len: length in bytes of SPU message header + * @hash_carry_len: Number of bytes of data carried over from previous req + * @new_data_len: Number of bytes of new request data + * @pad_len: Number of pad bytes + * + * The scatterlist that gets allocated here is freed in spu_chunk_cleanup() + * when the request completes, whether the request is handled successfully or + * there is an error. + * + * Return: + * 0 if successful + * < 0 if an error + */ +static int +spu_ahash_tx_sg_create(struct brcm_message *mssg, + struct iproc_reqctx_s *rctx, + u8 tx_frag_num, + u32 spu_hdr_len, + unsigned int hash_carry_len, + unsigned int new_data_len, u32 pad_len) +{ + struct spu_hw *spu = &iproc_priv.spu; + struct scatterlist *sg; /* used to build sgs in mbox message */ + u32 datalen; /* Number of bytes of response data expected */ + u32 stat_len; + + mssg->spu.src = kmalloc_objs(struct scatterlist, tx_frag_num, rctx->gfp); + if (!mssg->spu.src) + return -ENOMEM; + + sg = mssg->spu.src; + sg_init_table(sg, tx_frag_num); + + sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr, + BCM_HDR_LEN + spu_hdr_len); + + if (hash_carry_len) + sg_set_buf(sg++, rctx->hash_carry, hash_carry_len); + + if (new_data_len) { + /* Copy in each src sg entry from request, up to chunksize */ + datalen = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip, + rctx->src_nents, new_data_len); + if (datalen < new_data_len) { + pr_err("%s(): failed to copy src sg to mbox msg", + __func__); + return -EFAULT; + } + } + + if (pad_len) + sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len); + + stat_len = spu->spu_tx_status_len(); + if (stat_len) { + memset(rctx->msg_buf.tx_stat, 0, stat_len); + sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len); + } + + return 0; +} + +/** + * handle_ahash_req() - Process an asynchronous hash request from the crypto + * API. + * @rctx: Crypto request context + * + * Builds a SPU request message embedded in a mailbox message and submits the + * mailbox message on a selected mailbox channel. The SPU request message is + * constructed as a scatterlist, including entries from the crypto API's + * src scatterlist to avoid copying the data to be hashed. This function is + * called either on the thread from the crypto API, or, in the case that the + * crypto API request is too large to fit in a single SPU request message, + * on the thread that invokes the receive callback with a response message. + * Because some operations require the response from one chunk before the next + * chunk can be submitted, we always wait for the response for the previous + * chunk before submitting the next chunk. Because requests are submitted in + * lock step like this, there is no need to synchronize access to request data + * structures. + * + * Return: + * -EINPROGRESS: request has been submitted to SPU and response will be + * returned asynchronously + * -EAGAIN: non-final request included a small amount of data, which for + * efficiency we did not submit to the SPU, but instead stored + * to be submitted to the SPU with the next part of the request + * other: an error code + */ +static int handle_ahash_req(struct iproc_reqctx_s *rctx) +{ + struct spu_hw *spu = &iproc_priv.spu; + struct crypto_async_request *areq = rctx->parent; + struct ahash_request *req = ahash_request_cast(areq); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct crypto_tfm *tfm = crypto_ahash_tfm(ahash); + unsigned int blocksize = crypto_tfm_alg_blocksize(tfm); + struct iproc_ctx_s *ctx = rctx->ctx; + + /* number of bytes still to be hashed in this req */ + unsigned int nbytes_to_hash = 0; + int err; + unsigned int chunksize = 0; /* length of hash carry + new data */ + /* + * length of new data, not from hash carry, to be submitted in + * this hw request + */ + unsigned int new_data_len; + + unsigned int __maybe_unused chunk_start = 0; + u32 db_size; /* Length of data field, incl gcm and hash padding */ + int pad_len = 0; /* total pad len, including gcm, hash, stat padding */ + u32 data_pad_len = 0; /* length of GCM/CCM padding */ + u32 stat_pad_len = 0; /* length of padding to align STATUS word */ + struct brcm_message *mssg; /* mailbox message */ + struct spu_request_opts req_opts; + struct spu_cipher_parms cipher_parms; + struct spu_hash_parms hash_parms; + struct spu_aead_parms aead_parms; + unsigned int local_nbuf; + u32 spu_hdr_len; + unsigned int digestsize; + u16 rem = 0; + + /* + * number of entries in src and dst sg. Always includes SPU msg header. + * rx always includes a buffer to catch digest and STATUS. + */ + u8 rx_frag_num = 3; + u8 tx_frag_num = 1; + + flow_log("total_todo %u, total_sent %u\n", + rctx->total_todo, rctx->total_sent); + + memset(&req_opts, 0, sizeof(req_opts)); + memset(&cipher_parms, 0, sizeof(cipher_parms)); + memset(&hash_parms, 0, sizeof(hash_parms)); + memset(&aead_parms, 0, sizeof(aead_parms)); + + req_opts.bd_suppress = true; + hash_parms.alg = ctx->auth.alg; + hash_parms.mode = ctx->auth.mode; + hash_parms.type = HASH_TYPE_NONE; + hash_parms.key_buf = (u8 *)ctx->authkey; + hash_parms.key_len = ctx->authkeylen; + + /* + * For hash algorithms below assignment looks bit odd but + * it's needed for AES-XCBC and AES-CMAC hash algorithms + * to differentiate between 128, 192, 256 bit key values. + * Based on the key values, hash algorithm is selected. + * For example for 128 bit key, hash algorithm is AES-128. + */ + cipher_parms.type = ctx->cipher_type; + + mssg = &rctx->mb_mssg; + chunk_start = rctx->src_sent; + + /* + * Compute the amount remaining to hash. This may include data + * carried over from previous requests. + */ + nbytes_to_hash = rctx->total_todo - rctx->total_sent; + chunksize = nbytes_to_hash; + if ((ctx->max_payload != SPU_MAX_PAYLOAD_INF) && + (chunksize > ctx->max_payload)) + chunksize = ctx->max_payload; + + /* + * If this is not a final request and the request data is not a multiple + * of a full block, then simply park the extra data and prefix it to the + * data for the next request. + */ + if (!rctx->is_final) { + u8 *dest = rctx->hash_carry + rctx->hash_carry_len; + u16 new_len; /* len of data to add to hash carry */ + + rem = chunksize % blocksize; /* remainder */ + if (rem) { + /* chunksize not a multiple of blocksize */ + chunksize -= rem; + if (chunksize == 0) { + /* Don't have a full block to submit to hw */ + new_len = rem - rctx->hash_carry_len; + sg_copy_part_to_buf(req->src, dest, new_len, + rctx->src_sent); + rctx->hash_carry_len = rem; + flow_log("Exiting with hash carry len: %u\n", + rctx->hash_carry_len); + packet_dump(" buf: ", + rctx->hash_carry, + rctx->hash_carry_len); + return -EAGAIN; + } + } + } + + /* if we have hash carry, then prefix it to the data in this request */ + local_nbuf = rctx->hash_carry_len; + rctx->hash_carry_len = 0; + if (local_nbuf) + tx_frag_num++; + new_data_len = chunksize - local_nbuf; + + /* Count number of sg entries to be used in this request */ + rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, + new_data_len); + + /* AES hashing keeps key size in type field, so need to copy it here */ + if (hash_parms.alg == HASH_ALG_AES) + hash_parms.type = (enum hash_type)cipher_parms.type; + else + hash_parms.type = spu->spu_hash_type(rctx->total_sent); + + digestsize = spu->spu_digest_size(ctx->digestsize, ctx->auth.alg, + hash_parms.type); + hash_parms.digestsize = digestsize; + + /* update the indexes */ + rctx->total_sent += chunksize; + /* if you sent a prebuf then that wasn't from this req->src */ + rctx->src_sent += new_data_len; + + if ((rctx->total_sent == rctx->total_todo) && rctx->is_final) + hash_parms.pad_len = spu->spu_hash_pad_len(hash_parms.alg, + hash_parms.mode, + chunksize, + blocksize); + + /* + * If a non-first chunk, then include the digest returned from the + * previous chunk so that hw can add to it (except for AES types). + */ + if ((hash_parms.type == HASH_TYPE_UPDT) && + (hash_parms.alg != HASH_ALG_AES)) { + hash_parms.key_buf = rctx->incr_hash; + hash_parms.key_len = digestsize; + } + + atomic64_add(chunksize, &iproc_priv.bytes_out); + + flow_log("%s() final: %u nbuf: %u ", + __func__, rctx->is_final, local_nbuf); + + if (ctx->max_payload == SPU_MAX_PAYLOAD_INF) + flow_log("max_payload infinite\n"); + else + flow_log("max_payload %u\n", ctx->max_payload); + + flow_log("chunk_start: %u chunk_size: %u\n", chunk_start, chunksize); + + /* Prepend SPU header with type 3 BCM header */ + memcpy(rctx->msg_buf.bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN); + + hash_parms.prebuf_len = local_nbuf; + spu_hdr_len = spu->spu_create_request(rctx->msg_buf.bcm_spu_req_hdr + + BCM_HDR_LEN, + &req_opts, &cipher_parms, + &hash_parms, &aead_parms, + new_data_len); + + if (spu_hdr_len == 0) { + pr_err("Failed to create SPU request header\n"); + return -EFAULT; + } + + /* + * Determine total length of padding required. Put all padding in one + * buffer. + */ + data_pad_len = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode, chunksize); + db_size = spu_real_db_size(0, 0, local_nbuf, new_data_len, + 0, 0, hash_parms.pad_len); + if (spu->spu_tx_status_len()) + stat_pad_len = spu->spu_wordalign_padlen(db_size); + if (stat_pad_len) + rx_frag_num++; + pad_len = hash_parms.pad_len + data_pad_len + stat_pad_len; + if (pad_len) { + tx_frag_num++; + spu->spu_request_pad(rctx->msg_buf.spu_req_pad, data_pad_len, + hash_parms.pad_len, ctx->auth.alg, + ctx->auth.mode, rctx->total_sent, + stat_pad_len); + } + + spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN, + spu_hdr_len); + packet_dump(" prebuf: ", rctx->hash_carry, local_nbuf); + flow_log("Data:\n"); + dump_sg(rctx->src_sg, rctx->src_skip, new_data_len); + packet_dump(" pad: ", rctx->msg_buf.spu_req_pad, pad_len); + + /* + * Build mailbox message containing SPU request msg and rx buffers + * to catch response message + */ + memset(mssg, 0, sizeof(*mssg)); + mssg->type = BRCM_MESSAGE_SPU; + mssg->ctx = rctx; /* Will be returned in response */ + + /* Create rx scatterlist to catch result */ + err = spu_ahash_rx_sg_create(mssg, rctx, rx_frag_num, digestsize, + stat_pad_len); + if (err) + return err; + + /* Create tx scatterlist containing SPU request message */ + tx_frag_num += rctx->src_nents; + if (spu->spu_tx_status_len()) + tx_frag_num++; + err = spu_ahash_tx_sg_create(mssg, rctx, tx_frag_num, spu_hdr_len, + local_nbuf, new_data_len, pad_len); + if (err) + return err; + + err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx); + if (unlikely(err < 0)) + return err; + + return -EINPROGRESS; +} + +/** + * spu_hmac_outer_hash() - Request synchonous software compute of the outer hash + * for an HMAC request. + * @req: The HMAC request from the crypto API + * @ctx: The session context + * + * Return: 0 if synchronous hash operation successful + * -EINVAL if the hash algo is unrecognized + * any other value indicates an error + */ +static int spu_hmac_outer_hash(struct ahash_request *req, + struct iproc_ctx_s *ctx) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + unsigned int blocksize = + crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash)); + int rc; + + switch (ctx->auth.alg) { + case HASH_ALG_MD5: + rc = do_shash("md5", req->result, ctx->opad, blocksize, + req->result, ctx->digestsize, NULL, 0); + break; + case HASH_ALG_SHA1: + rc = do_shash("sha1", req->result, ctx->opad, blocksize, + req->result, ctx->digestsize, NULL, 0); + break; + case HASH_ALG_SHA224: + rc = do_shash("sha224", req->result, ctx->opad, blocksize, + req->result, ctx->digestsize, NULL, 0); + break; + case HASH_ALG_SHA256: + rc = do_shash("sha256", req->result, ctx->opad, blocksize, + req->result, ctx->digestsize, NULL, 0); + break; + case HASH_ALG_SHA384: + rc = do_shash("sha384", req->result, ctx->opad, blocksize, + req->result, ctx->digestsize, NULL, 0); + break; + case HASH_ALG_SHA512: + rc = do_shash("sha512", req->result, ctx->opad, blocksize, + req->result, ctx->digestsize, NULL, 0); + break; + default: + pr_err("%s() Error : unknown hmac type\n", __func__); + rc = -EINVAL; + } + return rc; +} + +/** + * ahash_req_done() - Process a hash result from the SPU hardware. + * @rctx: Crypto request context + * + * Return: 0 if successful + * < 0 if an error + */ +static int ahash_req_done(struct iproc_reqctx_s *rctx) +{ + struct spu_hw *spu = &iproc_priv.spu; + struct crypto_async_request *areq = rctx->parent; + struct ahash_request *req = ahash_request_cast(areq); + struct iproc_ctx_s *ctx = rctx->ctx; + int err; + + memcpy(req->result, rctx->msg_buf.digest, ctx->digestsize); + + if (spu->spu_type == SPU_TYPE_SPUM) { + /* byte swap the output from the UPDT function to network byte + * order + */ + if (ctx->auth.alg == HASH_ALG_MD5) { + __swab32s((u32 *)req->result); + __swab32s(((u32 *)req->result) + 1); + __swab32s(((u32 *)req->result) + 2); + __swab32s(((u32 *)req->result) + 3); + __swab32s(((u32 *)req->result) + 4); + } + } + + flow_dump(" digest ", req->result, ctx->digestsize); + + /* if this an HMAC then do the outer hash */ + if (rctx->is_sw_hmac) { + err = spu_hmac_outer_hash(req, ctx); + if (err < 0) + return err; + flow_dump(" hmac: ", req->result, ctx->digestsize); + } + + if (rctx->is_sw_hmac || ctx->auth.mode == HASH_MODE_HMAC) { + atomic_inc(&iproc_priv.op_counts[SPU_OP_HMAC]); + atomic_inc(&iproc_priv.hmac_cnt[ctx->auth.alg]); + } else { + atomic_inc(&iproc_priv.op_counts[SPU_OP_HASH]); + atomic_inc(&iproc_priv.hash_cnt[ctx->auth.alg]); + } + + return 0; +} + +/** + * handle_ahash_resp() - Process a SPU response message for a hash request. + * Checks if the entire crypto API request has been processed, and if so, + * invokes post processing on the result. + * @rctx: Crypto request context + */ +static void handle_ahash_resp(struct iproc_reqctx_s *rctx) +{ + struct iproc_ctx_s *ctx = rctx->ctx; + struct crypto_async_request *areq = rctx->parent; + struct ahash_request *req = ahash_request_cast(areq); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + unsigned int blocksize = + crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash)); + /* + * Save hash to use as input to next op if incremental. Might be copying + * too much, but that's easier than figuring out actual digest size here + */ + memcpy(rctx->incr_hash, rctx->msg_buf.digest, MAX_DIGEST_SIZE); + + flow_log("%s() blocksize:%u digestsize:%u\n", + __func__, blocksize, ctx->digestsize); + + atomic64_add(ctx->digestsize, &iproc_priv.bytes_in); + + if (rctx->is_final && (rctx->total_sent == rctx->total_todo)) + ahash_req_done(rctx); +} + +/** + * spu_aead_rx_sg_create() - Build up the scatterlist of buffers used to receive + * a SPU response message for an AEAD request. Includes buffers to catch SPU + * message headers and the response data. + * @mssg: mailbox message containing the receive sg + * @req: Crypto API request + * @rctx: crypto request context + * @rx_frag_num: number of scatterlist elements required to hold the + * SPU response message + * @assoc_len: Length of associated data included in the crypto request + * @ret_iv_len: Length of IV returned in response + * @resp_len: Number of bytes of response data expected to be written to + * dst buffer from crypto API + * @digestsize: Length of hash digest, in bytes + * @stat_pad_len: Number of bytes required to pad the STAT field to + * a 4-byte boundary + * + * The scatterlist that gets allocated here is freed in spu_chunk_cleanup() + * when the request completes, whether the request is handled successfully or + * there is an error. + * + * Returns: + * 0 if successful + * < 0 if an error + */ +static int spu_aead_rx_sg_create(struct brcm_message *mssg, + struct aead_request *req, + struct iproc_reqctx_s *rctx, + u8 rx_frag_num, + unsigned int assoc_len, + u32 ret_iv_len, unsigned int resp_len, + unsigned int digestsize, u32 stat_pad_len) +{ + struct spu_hw *spu = &iproc_priv.spu; + struct scatterlist *sg; /* used to build sgs in mbox message */ + struct iproc_ctx_s *ctx = rctx->ctx; + u32 datalen; /* Number of bytes of response data expected */ + u32 assoc_buf_len; + u8 data_padlen = 0; + + if (ctx->is_rfc4543) { + /* RFC4543: only pad after data, not after AAD */ + data_padlen = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode, + assoc_len + resp_len); + assoc_buf_len = assoc_len; + } else { + data_padlen = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode, + resp_len); + assoc_buf_len = spu->spu_assoc_resp_len(ctx->cipher.mode, + assoc_len, ret_iv_len, + rctx->is_encrypt); + } + + if (ctx->cipher.mode == CIPHER_MODE_CCM) + /* ICV (after data) must be in the next 32-bit word for CCM */ + data_padlen += spu->spu_wordalign_padlen(assoc_buf_len + + resp_len + + data_padlen); + + if (data_padlen) + /* have to catch gcm pad in separate buffer */ + rx_frag_num++; + + mssg->spu.dst = kmalloc_objs(struct scatterlist, rx_frag_num, rctx->gfp); + if (!mssg->spu.dst) + return -ENOMEM; + + sg = mssg->spu.dst; + sg_init_table(sg, rx_frag_num); + + /* Space for SPU message header */ + sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len); + + if (assoc_buf_len) { + /* + * Don't write directly to req->dst, because SPU may pad the + * assoc data in the response + */ + memset(rctx->msg_buf.a.resp_aad, 0, assoc_buf_len); + sg_set_buf(sg++, rctx->msg_buf.a.resp_aad, assoc_buf_len); + } + + if (resp_len) { + /* + * Copy in each dst sg entry from request, up to chunksize. + * dst sg catches just the data. digest caught in separate buf. + */ + datalen = spu_msg_sg_add(&sg, &rctx->dst_sg, &rctx->dst_skip, + rctx->dst_nents, resp_len); + if (datalen < (resp_len)) { + pr_err("%s(): failed to copy dst sg to mbox msg. expected len %u, datalen %u", + __func__, resp_len, datalen); + return -EFAULT; + } + } + + /* If GCM/CCM data is padded, catch padding in separate buffer */ + if (data_padlen) { + memset(rctx->msg_buf.a.gcmpad, 0, data_padlen); + sg_set_buf(sg++, rctx->msg_buf.a.gcmpad, data_padlen); + } + + /* Always catch ICV in separate buffer */ + sg_set_buf(sg++, rctx->msg_buf.digest, digestsize); + + flow_log("stat_pad_len %u\n", stat_pad_len); + if (stat_pad_len) { + memset(rctx->msg_buf.rx_stat_pad, 0, stat_pad_len); + sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len); + } + + memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN); + sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len()); + + return 0; +} + +/** + * spu_aead_tx_sg_create() - Build up the scatterlist of buffers used to send a + * SPU request message for an AEAD request. Includes SPU message headers and the + * request data. + * @mssg: mailbox message containing the transmit sg + * @rctx: crypto request context + * @tx_frag_num: number of scatterlist elements required to construct the + * SPU request message + * @spu_hdr_len: length of SPU message header in bytes + * @assoc: crypto API associated data scatterlist + * @assoc_len: length of associated data + * @assoc_nents: number of scatterlist entries containing assoc data + * @aead_iv_len: length of AEAD IV, if included + * @chunksize: Number of bytes of request data + * @aad_pad_len: Number of bytes of padding at end of AAD. For GCM/CCM. + * @pad_len: Number of pad bytes + * @incl_icv: If true, write separate ICV buffer after data and + * any padding + * + * The scatterlist that gets allocated here is freed in spu_chunk_cleanup() + * when the request completes, whether the request is handled successfully or + * there is an error. + * + * Return: + * 0 if successful + * < 0 if an error + */ +static int spu_aead_tx_sg_create(struct brcm_message *mssg, + struct iproc_reqctx_s *rctx, + u8 tx_frag_num, + u32 spu_hdr_len, + struct scatterlist *assoc, + unsigned int assoc_len, + int assoc_nents, + unsigned int aead_iv_len, + unsigned int chunksize, + u32 aad_pad_len, u32 pad_len, bool incl_icv) +{ + struct spu_hw *spu = &iproc_priv.spu; + struct scatterlist *sg; /* used to build sgs in mbox message */ + struct scatterlist *assoc_sg = assoc; + struct iproc_ctx_s *ctx = rctx->ctx; + u32 datalen; /* Number of bytes of data to write */ + u32 written; /* Number of bytes of data written */ + u32 assoc_offset = 0; + u32 stat_len; + + mssg->spu.src = kmalloc_objs(struct scatterlist, tx_frag_num, rctx->gfp); + if (!mssg->spu.src) + return -ENOMEM; + + sg = mssg->spu.src; + sg_init_table(sg, tx_frag_num); + + sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr, + BCM_HDR_LEN + spu_hdr_len); + + if (assoc_len) { + /* Copy in each associated data sg entry from request */ + written = spu_msg_sg_add(&sg, &assoc_sg, &assoc_offset, + assoc_nents, assoc_len); + if (written < assoc_len) { + pr_err("%s(): failed to copy assoc sg to mbox msg", + __func__); + return -EFAULT; + } + } + + if (aead_iv_len) + sg_set_buf(sg++, rctx->msg_buf.iv_ctr, aead_iv_len); + + if (aad_pad_len) { + memset(rctx->msg_buf.a.req_aad_pad, 0, aad_pad_len); + sg_set_buf(sg++, rctx->msg_buf.a.req_aad_pad, aad_pad_len); + } + + datalen = chunksize; + if ((chunksize > ctx->digestsize) && incl_icv) + datalen -= ctx->digestsize; + if (datalen) { + /* For aead, a single msg should consume the entire src sg */ + written = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip, + rctx->src_nents, datalen); + if (written < datalen) { + pr_err("%s(): failed to copy src sg to mbox msg", + __func__); + return -EFAULT; + } + } + + if (pad_len) { + memset(rctx->msg_buf.spu_req_pad, 0, pad_len); + sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len); + } + + if (incl_icv) + sg_set_buf(sg++, rctx->msg_buf.digest, ctx->digestsize); + + stat_len = spu->spu_tx_status_len(); + if (stat_len) { + memset(rctx->msg_buf.tx_stat, 0, stat_len); + sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len); + } + return 0; +} + +/** + * handle_aead_req() - Submit a SPU request message for the next chunk of the + * current AEAD request. + * @rctx: Crypto request context + * + * Unlike other operation types, we assume the length of the request fits in + * a single SPU request message. aead_enqueue() makes sure this is true. + * Comments for other op types regarding threads applies here as well. + * + * Unlike incremental hash ops, where the spu returns the entire hash for + * truncated algs like sha-224, the SPU returns just the truncated hash in + * response to aead requests. So digestsize is always ctx->digestsize here. + * + * Return: -EINPROGRESS: crypto request has been accepted and result will be + * returned asynchronously + * Any other value indicates an error + */ +static int handle_aead_req(struct iproc_reqctx_s *rctx) +{ + struct spu_hw *spu = &iproc_priv.spu; + struct crypto_async_request *areq = rctx->parent; + struct aead_request *req = container_of(areq, + struct aead_request, base); + struct iproc_ctx_s *ctx = rctx->ctx; + int err; + unsigned int chunksize; + unsigned int resp_len; + u32 spu_hdr_len; + u32 db_size; + u32 stat_pad_len; + u32 pad_len; + struct brcm_message *mssg; /* mailbox message */ + struct spu_request_opts req_opts; + struct spu_cipher_parms cipher_parms; + struct spu_hash_parms hash_parms; + struct spu_aead_parms aead_parms; + int assoc_nents = 0; + bool incl_icv = false; + unsigned int digestsize = ctx->digestsize; + + /* number of entries in src and dst sg. Always includes SPU msg header. + */ + u8 rx_frag_num = 2; /* and STATUS */ + u8 tx_frag_num = 1; + + /* doing the whole thing at once */ + chunksize = rctx->total_todo; + + flow_log("%s: chunksize %u\n", __func__, chunksize); + + memset(&req_opts, 0, sizeof(req_opts)); + memset(&hash_parms, 0, sizeof(hash_parms)); + memset(&aead_parms, 0, sizeof(aead_parms)); + + req_opts.is_inbound = !(rctx->is_encrypt); + req_opts.auth_first = ctx->auth_first; + req_opts.is_aead = true; + req_opts.is_esp = ctx->is_esp; + + cipher_parms.alg = ctx->cipher.alg; + cipher_parms.mode = ctx->cipher.mode; + cipher_parms.type = ctx->cipher_type; + cipher_parms.key_buf = ctx->enckey; + cipher_parms.key_len = ctx->enckeylen; + cipher_parms.iv_buf = rctx->msg_buf.iv_ctr; + cipher_parms.iv_len = rctx->iv_ctr_len; + + hash_parms.alg = ctx->auth.alg; + hash_parms.mode = ctx->auth.mode; + hash_parms.type = HASH_TYPE_NONE; + hash_parms.key_buf = (u8 *)ctx->authkey; + hash_parms.key_len = ctx->authkeylen; + hash_parms.digestsize = digestsize; + + if ((ctx->auth.alg == HASH_ALG_SHA224) && + (ctx->authkeylen < SHA224_DIGEST_SIZE)) + hash_parms.key_len = SHA224_DIGEST_SIZE; + + aead_parms.assoc_size = req->assoclen; + if (ctx->is_esp && !ctx->is_rfc4543) { + /* + * 8-byte IV is included assoc data in request. SPU2 + * expects AAD to include just SPI and seqno. So + * subtract off the IV len. + */ + aead_parms.assoc_size -= GCM_RFC4106_IV_SIZE; + + if (rctx->is_encrypt) { + aead_parms.return_iv = true; + aead_parms.ret_iv_len = GCM_RFC4106_IV_SIZE; + aead_parms.ret_iv_off = GCM_ESP_SALT_SIZE; + } + } else { + aead_parms.ret_iv_len = 0; + } + + /* + * Count number of sg entries from the crypto API request that are to + * be included in this mailbox message. For dst sg, don't count space + * for digest. Digest gets caught in a separate buffer and copied back + * to dst sg when processing response. + */ + rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, chunksize); + rctx->dst_nents = spu_sg_count(rctx->dst_sg, rctx->dst_skip, chunksize); + if (aead_parms.assoc_size) + assoc_nents = spu_sg_count(rctx->assoc, 0, + aead_parms.assoc_size); + + mssg = &rctx->mb_mssg; + + rctx->total_sent = chunksize; + rctx->src_sent = chunksize; + if (spu->spu_assoc_resp_len(ctx->cipher.mode, + aead_parms.assoc_size, + aead_parms.ret_iv_len, + rctx->is_encrypt)) + rx_frag_num++; + + aead_parms.iv_len = spu->spu_aead_ivlen(ctx->cipher.mode, + rctx->iv_ctr_len); + + if (ctx->auth.alg == HASH_ALG_AES) + hash_parms.type = (enum hash_type)ctx->cipher_type; + + /* General case AAD padding (CCM and RFC4543 special cases below) */ + aead_parms.aad_pad_len = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode, + aead_parms.assoc_size); + + /* General case data padding (CCM decrypt special case below) */ + aead_parms.data_pad_len = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode, + chunksize); + + if (ctx->cipher.mode == CIPHER_MODE_CCM) { + /* + * for CCM, AAD len + 2 (rather than AAD len) needs to be + * 128-bit aligned + */ + aead_parms.aad_pad_len = spu->spu_gcm_ccm_pad_len( + ctx->cipher.mode, + aead_parms.assoc_size + 2); + + /* + * And when decrypting CCM, need to pad without including + * size of ICV which is tacked on to end of chunk + */ + if (!rctx->is_encrypt) + aead_parms.data_pad_len = + spu->spu_gcm_ccm_pad_len(ctx->cipher.mode, + chunksize - digestsize); + + /* CCM also requires software to rewrite portions of IV: */ + spu->spu_ccm_update_iv(digestsize, &cipher_parms, req->assoclen, + chunksize, rctx->is_encrypt, + ctx->is_esp); + } + + if (ctx->is_rfc4543) { + /* + * RFC4543: data is included in AAD, so don't pad after AAD + * and pad data based on both AAD + data size + */ + aead_parms.aad_pad_len = 0; + if (!rctx->is_encrypt) + aead_parms.data_pad_len = spu->spu_gcm_ccm_pad_len( + ctx->cipher.mode, + aead_parms.assoc_size + chunksize - + digestsize); + else + aead_parms.data_pad_len = spu->spu_gcm_ccm_pad_len( + ctx->cipher.mode, + aead_parms.assoc_size + chunksize); + + req_opts.is_rfc4543 = true; + } + + if (spu_req_incl_icv(ctx->cipher.mode, rctx->is_encrypt)) { + incl_icv = true; + tx_frag_num++; + /* Copy ICV from end of src scatterlist to digest buf */ + sg_copy_part_to_buf(req->src, rctx->msg_buf.digest, digestsize, + req->assoclen + rctx->total_sent - + digestsize); + } + + atomic64_add(chunksize, &iproc_priv.bytes_out); + + flow_log("%s()-sent chunksize:%u\n", __func__, chunksize); + + /* Prepend SPU header with type 3 BCM header */ + memcpy(rctx->msg_buf.bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN); + + spu_hdr_len = spu->spu_create_request(rctx->msg_buf.bcm_spu_req_hdr + + BCM_HDR_LEN, &req_opts, + &cipher_parms, &hash_parms, + &aead_parms, chunksize); + + /* Determine total length of padding. Put all padding in one buffer. */ + db_size = spu_real_db_size(aead_parms.assoc_size, aead_parms.iv_len, 0, + chunksize, aead_parms.aad_pad_len, + aead_parms.data_pad_len, 0); + + stat_pad_len = spu->spu_wordalign_padlen(db_size); + + if (stat_pad_len) + rx_frag_num++; + pad_len = aead_parms.data_pad_len + stat_pad_len; + if (pad_len) { + tx_frag_num++; + spu->spu_request_pad(rctx->msg_buf.spu_req_pad, + aead_parms.data_pad_len, 0, + ctx->auth.alg, ctx->auth.mode, + rctx->total_sent, stat_pad_len); + } + + spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN, + spu_hdr_len); + dump_sg(rctx->assoc, 0, aead_parms.assoc_size); + packet_dump(" aead iv: ", rctx->msg_buf.iv_ctr, aead_parms.iv_len); + packet_log("BD:\n"); + dump_sg(rctx->src_sg, rctx->src_skip, chunksize); + packet_dump(" pad: ", rctx->msg_buf.spu_req_pad, pad_len); + + /* + * Build mailbox message containing SPU request msg and rx buffers + * to catch response message + */ + memset(mssg, 0, sizeof(*mssg)); + mssg->type = BRCM_MESSAGE_SPU; + mssg->ctx = rctx; /* Will be returned in response */ + + /* Create rx scatterlist to catch result */ + rx_frag_num += rctx->dst_nents; + resp_len = chunksize; + + /* + * Always catch ICV in separate buffer. Have to for GCM/CCM because of + * padding. Have to for SHA-224 and other truncated SHAs because SPU + * sends entire digest back. + */ + rx_frag_num++; + + if (((ctx->cipher.mode == CIPHER_MODE_GCM) || + (ctx->cipher.mode == CIPHER_MODE_CCM)) && !rctx->is_encrypt) { + /* + * Input is ciphertxt plus ICV, but ICV not incl + * in output. + */ + resp_len -= ctx->digestsize; + if (resp_len == 0) + /* no rx frags to catch output data */ + rx_frag_num -= rctx->dst_nents; + } + + err = spu_aead_rx_sg_create(mssg, req, rctx, rx_frag_num, + aead_parms.assoc_size, + aead_parms.ret_iv_len, resp_len, digestsize, + stat_pad_len); + if (err) + return err; + + /* Create tx scatterlist containing SPU request message */ + tx_frag_num += rctx->src_nents; + tx_frag_num += assoc_nents; + if (aead_parms.aad_pad_len) + tx_frag_num++; + if (aead_parms.iv_len) + tx_frag_num++; + if (spu->spu_tx_status_len()) + tx_frag_num++; + err = spu_aead_tx_sg_create(mssg, rctx, tx_frag_num, spu_hdr_len, + rctx->assoc, aead_parms.assoc_size, + assoc_nents, aead_parms.iv_len, chunksize, + aead_parms.aad_pad_len, pad_len, incl_icv); + if (err) + return err; + + err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx); + if (unlikely(err < 0)) + return err; + + return -EINPROGRESS; +} + +/** + * handle_aead_resp() - Process a SPU response message for an AEAD request. + * @rctx: Crypto request context + */ +static void handle_aead_resp(struct iproc_reqctx_s *rctx) +{ + struct spu_hw *spu = &iproc_priv.spu; + struct crypto_async_request *areq = rctx->parent; + struct aead_request *req = container_of(areq, + struct aead_request, base); + struct iproc_ctx_s *ctx = rctx->ctx; + u32 payload_len; + unsigned int icv_offset; + u32 result_len; + + /* See how much data was returned */ + payload_len = spu->spu_payload_length(rctx->msg_buf.spu_resp_hdr); + flow_log("payload_len %u\n", payload_len); + + /* only count payload */ + atomic64_add(payload_len, &iproc_priv.bytes_in); + + if (req->assoclen) + packet_dump(" assoc_data ", rctx->msg_buf.a.resp_aad, + req->assoclen); + + /* + * Copy the ICV back to the destination + * buffer. In decrypt case, SPU gives us back the digest, but crypto + * API doesn't expect ICV in dst buffer. + */ + result_len = req->cryptlen; + if (rctx->is_encrypt) { + icv_offset = req->assoclen + rctx->total_sent; + packet_dump(" ICV: ", rctx->msg_buf.digest, ctx->digestsize); + flow_log("copying ICV to dst sg at offset %u\n", icv_offset); + sg_copy_part_from_buf(req->dst, rctx->msg_buf.digest, + ctx->digestsize, icv_offset); + result_len += ctx->digestsize; + } + + packet_log("response data: "); + dump_sg(req->dst, req->assoclen, result_len); + + atomic_inc(&iproc_priv.op_counts[SPU_OP_AEAD]); + if (ctx->cipher.alg == CIPHER_ALG_AES) { + if (ctx->cipher.mode == CIPHER_MODE_CCM) + atomic_inc(&iproc_priv.aead_cnt[AES_CCM]); + else if (ctx->cipher.mode == CIPHER_MODE_GCM) + atomic_inc(&iproc_priv.aead_cnt[AES_GCM]); + else + atomic_inc(&iproc_priv.aead_cnt[AUTHENC]); + } else { + atomic_inc(&iproc_priv.aead_cnt[AUTHENC]); + } +} + +/** + * spu_chunk_cleanup() - Do cleanup after processing one chunk of a request + * @rctx: request context + * + * Mailbox scatterlists are allocated for each chunk. So free them after + * processing each chunk. + */ +static void spu_chunk_cleanup(struct iproc_reqctx_s *rctx) +{ + /* mailbox message used to tx request */ + struct brcm_message *mssg = &rctx->mb_mssg; + + kfree(mssg->spu.src); + kfree(mssg->spu.dst); + memset(mssg, 0, sizeof(struct brcm_message)); +} + +/** + * finish_req() - Used to invoke the complete callback from the requester when + * a request has been handled asynchronously. + * @rctx: Request context + * @err: Indicates whether the request was successful or not + * + * Ensures that cleanup has been done for request + */ +static void finish_req(struct iproc_reqctx_s *rctx, int err) +{ + struct crypto_async_request *areq = rctx->parent; + + flow_log("%s() err:%d\n\n", __func__, err); + + /* No harm done if already called */ + spu_chunk_cleanup(rctx); + + if (areq) + crypto_request_complete(areq, err); +} + +/** + * spu_rx_callback() - Callback from mailbox framework with a SPU response. + * @cl: mailbox client structure for SPU driver + * @msg: mailbox message containing SPU response + */ +static void spu_rx_callback(struct mbox_client *cl, void *msg) +{ + struct spu_hw *spu = &iproc_priv.spu; + struct brcm_message *mssg = msg; + struct iproc_reqctx_s *rctx; + int err; + + rctx = mssg->ctx; + if (unlikely(!rctx)) { + /* This is fatal */ + pr_err("%s(): no request context", __func__); + err = -EFAULT; + goto cb_finish; + } + + /* process the SPU status */ + err = spu->spu_status_process(rctx->msg_buf.rx_stat); + if (err != 0) { + if (err == SPU_INVALID_ICV) + atomic_inc(&iproc_priv.bad_icv); + err = -EBADMSG; + goto cb_finish; + } + + /* Process the SPU response message */ + switch (rctx->ctx->alg->type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + handle_skcipher_resp(rctx); + break; + case CRYPTO_ALG_TYPE_AHASH: + handle_ahash_resp(rctx); + break; + case CRYPTO_ALG_TYPE_AEAD: + handle_aead_resp(rctx); + break; + default: + err = -EINVAL; + goto cb_finish; + } + + /* + * If this response does not complete the request, then send the next + * request chunk. + */ + if (rctx->total_sent < rctx->total_todo) { + /* Deallocate anything specific to previous chunk */ + spu_chunk_cleanup(rctx); + + switch (rctx->ctx->alg->type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + err = handle_skcipher_req(rctx); + break; + case CRYPTO_ALG_TYPE_AHASH: + err = handle_ahash_req(rctx); + if (err == -EAGAIN) + /* + * we saved data in hash carry, but tell crypto + * API we successfully completed request. + */ + err = 0; + break; + case CRYPTO_ALG_TYPE_AEAD: + err = handle_aead_req(rctx); + break; + default: + err = -EINVAL; + } + + if (err == -EINPROGRESS) + /* Successfully submitted request for next chunk */ + return; + } + +cb_finish: + finish_req(rctx, err); +} + +/* ==================== Kernel Cryptographic API ==================== */ + +/** + * skcipher_enqueue() - Handle skcipher encrypt or decrypt request. + * @req: Crypto API request + * @encrypt: true if encrypting; false if decrypting + * + * Return: -EINPROGRESS if request accepted and result will be returned + * asynchronously + * < 0 if an error + */ +static int skcipher_enqueue(struct skcipher_request *req, bool encrypt) +{ + struct iproc_reqctx_s *rctx = skcipher_request_ctx(req); + struct iproc_ctx_s *ctx = + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + int err; + + flow_log("%s() enc:%u\n", __func__, encrypt); + + rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + rctx->parent = &req->base; + rctx->is_encrypt = encrypt; + rctx->bd_suppress = false; + rctx->total_todo = req->cryptlen; + rctx->src_sent = 0; + rctx->total_sent = 0; + rctx->total_received = 0; + rctx->ctx = ctx; + + /* Initialize current position in src and dst scatterlists */ + rctx->src_sg = req->src; + rctx->src_nents = 0; + rctx->src_skip = 0; + rctx->dst_sg = req->dst; + rctx->dst_nents = 0; + rctx->dst_skip = 0; + + if (ctx->cipher.mode == CIPHER_MODE_CBC || + ctx->cipher.mode == CIPHER_MODE_CTR || + ctx->cipher.mode == CIPHER_MODE_OFB || + ctx->cipher.mode == CIPHER_MODE_XTS || + ctx->cipher.mode == CIPHER_MODE_GCM || + ctx->cipher.mode == CIPHER_MODE_CCM) { + rctx->iv_ctr_len = + crypto_skcipher_ivsize(crypto_skcipher_reqtfm(req)); + memcpy(rctx->msg_buf.iv_ctr, req->iv, rctx->iv_ctr_len); + } else { + rctx->iv_ctr_len = 0; + } + + /* Choose a SPU to process this request */ + rctx->chan_idx = select_channel(); + err = handle_skcipher_req(rctx); + if (err != -EINPROGRESS) + /* synchronous result */ + spu_chunk_cleanup(rctx); + + return err; +} + +static int des_setkey(struct crypto_skcipher *cipher, const u8 *key, + unsigned int keylen) +{ + struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher); + int err; + + err = verify_skcipher_des_key(cipher, key); + if (err) + return err; + + ctx->cipher_type = CIPHER_TYPE_DES; + return 0; +} + +static int threedes_setkey(struct crypto_skcipher *cipher, const u8 *key, + unsigned int keylen) +{ + struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher); + int err; + + err = verify_skcipher_des3_key(cipher, key); + if (err) + return err; + + ctx->cipher_type = CIPHER_TYPE_3DES; + return 0; +} + +static int aes_setkey(struct crypto_skcipher *cipher, const u8 *key, + unsigned int keylen) +{ + struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher); + + if (ctx->cipher.mode == CIPHER_MODE_XTS) + /* XTS includes two keys of equal length */ + keylen = keylen / 2; + + switch (keylen) { + case AES_KEYSIZE_128: + ctx->cipher_type = CIPHER_TYPE_AES128; + break; + case AES_KEYSIZE_192: + ctx->cipher_type = CIPHER_TYPE_AES192; + break; + case AES_KEYSIZE_256: + ctx->cipher_type = CIPHER_TYPE_AES256; + break; + default: + return -EINVAL; + } + WARN_ON((ctx->max_payload != SPU_MAX_PAYLOAD_INF) && + ((ctx->max_payload % AES_BLOCK_SIZE) != 0)); + return 0; +} + +static int skcipher_setkey(struct crypto_skcipher *cipher, const u8 *key, + unsigned int keylen) +{ + struct spu_hw *spu = &iproc_priv.spu; + struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher); + struct spu_cipher_parms cipher_parms; + u32 alloc_len = 0; + int err; + + flow_log("skcipher_setkey() keylen: %d\n", keylen); + flow_dump(" key: ", key, keylen); + + switch (ctx->cipher.alg) { + case CIPHER_ALG_DES: + err = des_setkey(cipher, key, keylen); + break; + case CIPHER_ALG_3DES: + err = threedes_setkey(cipher, key, keylen); + break; + case CIPHER_ALG_AES: + err = aes_setkey(cipher, key, keylen); + break; + default: + pr_err("%s() Error: unknown cipher alg\n", __func__); + err = -EINVAL; + } + if (err) + return err; + + memcpy(ctx->enckey, key, keylen); + ctx->enckeylen = keylen; + + /* SPU needs XTS keys in the reverse order the crypto API presents */ + if ((ctx->cipher.alg == CIPHER_ALG_AES) && + (ctx->cipher.mode == CIPHER_MODE_XTS)) { + unsigned int xts_keylen = keylen / 2; + + memcpy(ctx->enckey, key + xts_keylen, xts_keylen); + memcpy(ctx->enckey + xts_keylen, key, xts_keylen); + } + + if (spu->spu_type == SPU_TYPE_SPUM) + alloc_len = BCM_HDR_LEN + SPU_HEADER_ALLOC_LEN; + else if (spu->spu_type == SPU_TYPE_SPU2) + alloc_len = BCM_HDR_LEN + SPU2_HEADER_ALLOC_LEN; + memset(ctx->bcm_spu_req_hdr, 0, alloc_len); + cipher_parms.iv_buf = NULL; + cipher_parms.iv_len = crypto_skcipher_ivsize(cipher); + flow_log("%s: iv_len %u\n", __func__, cipher_parms.iv_len); + + cipher_parms.alg = ctx->cipher.alg; + cipher_parms.mode = ctx->cipher.mode; + cipher_parms.type = ctx->cipher_type; + cipher_parms.key_buf = ctx->enckey; + cipher_parms.key_len = ctx->enckeylen; + + /* Prepend SPU request message with BCM header */ + memcpy(ctx->bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN); + ctx->spu_req_hdr_len = + spu->spu_cipher_req_init(ctx->bcm_spu_req_hdr + BCM_HDR_LEN, + &cipher_parms); + + ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen, + ctx->enckeylen, + false); + + atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_CIPHER]); + + return 0; +} + +static int skcipher_encrypt(struct skcipher_request *req) +{ + flow_log("skcipher_encrypt() nbytes:%u\n", req->cryptlen); + + return skcipher_enqueue(req, true); +} + +static int skcipher_decrypt(struct skcipher_request *req) +{ + flow_log("skcipher_decrypt() nbytes:%u\n", req->cryptlen); + return skcipher_enqueue(req, false); +} + +static int ahash_enqueue(struct ahash_request *req) +{ + struct iproc_reqctx_s *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm); + int err; + const char *alg_name; + + flow_log("ahash_enqueue() nbytes:%u\n", req->nbytes); + + rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + rctx->parent = &req->base; + rctx->ctx = ctx; + rctx->bd_suppress = true; + memset(&rctx->mb_mssg, 0, sizeof(struct brcm_message)); + + /* Initialize position in src scatterlist */ + rctx->src_sg = req->src; + rctx->src_skip = 0; + rctx->src_nents = 0; + rctx->dst_sg = NULL; + rctx->dst_skip = 0; + rctx->dst_nents = 0; + + /* SPU2 hardware does not compute hash of zero length data */ + if ((rctx->is_final == 1) && (rctx->total_todo == 0) && + (iproc_priv.spu.spu_type == SPU_TYPE_SPU2)) { + alg_name = crypto_ahash_alg_name(tfm); + flow_log("Doing %sfinal %s zero-len hash request in software\n", + rctx->is_final ? "" : "non-", alg_name); + err = do_shash((unsigned char *)alg_name, req->result, + NULL, 0, NULL, 0, ctx->authkey, + ctx->authkeylen); + if (err < 0) + flow_log("Hash request failed with error %d\n", err); + return err; + } + /* Choose a SPU to process this request */ + rctx->chan_idx = select_channel(); + + err = handle_ahash_req(rctx); + if (err != -EINPROGRESS) + /* synchronous result */ + spu_chunk_cleanup(rctx); + + if (err == -EAGAIN) + /* + * we saved data in hash carry, but tell crypto API + * we successfully completed request. + */ + err = 0; + + return err; +} + +static int __ahash_init(struct ahash_request *req) +{ + struct spu_hw *spu = &iproc_priv.spu; + struct iproc_reqctx_s *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm); + + flow_log("%s()\n", __func__); + + /* Initialize the context */ + rctx->hash_carry_len = 0; + rctx->is_final = 0; + + rctx->total_todo = 0; + rctx->src_sent = 0; + rctx->total_sent = 0; + rctx->total_received = 0; + + ctx->digestsize = crypto_ahash_digestsize(tfm); + /* If we add a hash whose digest is larger, catch it here. */ + WARN_ON(ctx->digestsize > MAX_DIGEST_SIZE); + + rctx->is_sw_hmac = false; + + ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen, 0, + true); + + return 0; +} + +/** + * spu_no_incr_hash() - Determine whether incremental hashing is supported. + * @ctx: Crypto session context + * + * SPU-2 does not support incremental hashing (we'll have to revisit and + * condition based on chip revision or device tree entry if future versions do + * support incremental hash) + * + * SPU-M also doesn't support incremental hashing of AES-XCBC + * + * Return: true if incremental hashing is not supported + * false otherwise + */ +static bool spu_no_incr_hash(struct iproc_ctx_s *ctx) +{ + struct spu_hw *spu = &iproc_priv.spu; + + if (spu->spu_type == SPU_TYPE_SPU2) + return true; + + if ((ctx->auth.alg == HASH_ALG_AES) && + (ctx->auth.mode == HASH_MODE_XCBC)) + return true; + + /* Otherwise, incremental hashing is supported */ + return false; +} + +static int ahash_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm); + const char *alg_name; + struct crypto_shash *hash; + int ret; + gfp_t gfp; + + if (spu_no_incr_hash(ctx)) { + /* + * If we get an incremental hashing request and it's not + * supported by the hardware, we need to handle it in software + * by calling synchronous hash functions. + */ + alg_name = crypto_ahash_alg_name(tfm); + hash = crypto_alloc_shash(alg_name, 0, 0); + if (IS_ERR(hash)) { + ret = PTR_ERR(hash); + goto err; + } + + gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + ctx->shash = kmalloc(sizeof(*ctx->shash) + + crypto_shash_descsize(hash), gfp); + if (!ctx->shash) { + ret = -ENOMEM; + goto err_hash; + } + ctx->shash->tfm = hash; + + /* Set the key using data we already have from setkey */ + if (ctx->authkeylen > 0) { + ret = crypto_shash_setkey(hash, ctx->authkey, + ctx->authkeylen); + if (ret) + goto err_shash; + } + + /* Initialize hash w/ this key and other params */ + ret = crypto_shash_init(ctx->shash); + if (ret) + goto err_shash; + } else { + /* Otherwise call the internal function which uses SPU hw */ + ret = __ahash_init(req); + } + + return ret; + +err_shash: + kfree(ctx->shash); +err_hash: + crypto_free_shash(hash); +err: + return ret; +} + +static int __ahash_update(struct ahash_request *req) +{ + struct iproc_reqctx_s *rctx = ahash_request_ctx(req); + + flow_log("ahash_update() nbytes:%u\n", req->nbytes); + + if (!req->nbytes) + return 0; + rctx->total_todo += req->nbytes; + rctx->src_sent = 0; + + return ahash_enqueue(req); +} + +static int ahash_update(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm); + u8 *tmpbuf; + int ret; + int nents; + gfp_t gfp; + + if (spu_no_incr_hash(ctx)) { + /* + * If we get an incremental hashing request and it's not + * supported by the hardware, we need to handle it in software + * by calling synchronous hash functions. + */ + if (req->src) + nents = sg_nents(req->src); + else + return -EINVAL; + + /* Copy data from req scatterlist to tmp buffer */ + gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + tmpbuf = kmalloc(req->nbytes, gfp); + if (!tmpbuf) + return -ENOMEM; + + if (sg_copy_to_buffer(req->src, nents, tmpbuf, req->nbytes) != + req->nbytes) { + kfree(tmpbuf); + return -EINVAL; + } + + /* Call synchronous update */ + ret = crypto_shash_update(ctx->shash, tmpbuf, req->nbytes); + kfree(tmpbuf); + } else { + /* Otherwise call the internal function which uses SPU hw */ + ret = __ahash_update(req); + } + + return ret; +} + +static int __ahash_final(struct ahash_request *req) +{ + struct iproc_reqctx_s *rctx = ahash_request_ctx(req); + + flow_log("ahash_final() nbytes:%u\n", req->nbytes); + + rctx->is_final = 1; + + return ahash_enqueue(req); +} + +static int ahash_final(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm); + int ret; + + if (spu_no_incr_hash(ctx)) { + /* + * If we get an incremental hashing request and it's not + * supported by the hardware, we need to handle it in software + * by calling synchronous hash functions. + */ + ret = crypto_shash_final(ctx->shash, req->result); + + /* Done with hash, can deallocate it now */ + crypto_free_shash(ctx->shash->tfm); + kfree(ctx->shash); + + } else { + /* Otherwise call the internal function which uses SPU hw */ + ret = __ahash_final(req); + } + + return ret; +} + +static int __ahash_finup(struct ahash_request *req) +{ + struct iproc_reqctx_s *rctx = ahash_request_ctx(req); + + flow_log("ahash_finup() nbytes:%u\n", req->nbytes); + + rctx->total_todo += req->nbytes; + rctx->src_sent = 0; + rctx->is_final = 1; + + return ahash_enqueue(req); +} + +static int ahash_finup(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm); + u8 *tmpbuf; + int ret; + int nents; + gfp_t gfp; + + if (spu_no_incr_hash(ctx)) { + /* + * If we get an incremental hashing request and it's not + * supported by the hardware, we need to handle it in software + * by calling synchronous hash functions. + */ + if (req->src) { + nents = sg_nents(req->src); + } else { + ret = -EINVAL; + goto ahash_finup_exit; + } + + /* Copy data from req scatterlist to tmp buffer */ + gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + tmpbuf = kmalloc(req->nbytes, gfp); + if (!tmpbuf) { + ret = -ENOMEM; + goto ahash_finup_exit; + } + + if (sg_copy_to_buffer(req->src, nents, tmpbuf, req->nbytes) != + req->nbytes) { + ret = -EINVAL; + goto ahash_finup_free; + } + + /* Call synchronous update */ + ret = crypto_shash_finup(ctx->shash, tmpbuf, req->nbytes, + req->result); + } else { + /* Otherwise call the internal function which uses SPU hw */ + return __ahash_finup(req); + } +ahash_finup_free: + kfree(tmpbuf); + +ahash_finup_exit: + /* Done with hash, can deallocate it now */ + crypto_free_shash(ctx->shash->tfm); + kfree(ctx->shash); + return ret; +} + +static int ahash_digest(struct ahash_request *req) +{ + int err; + + flow_log("ahash_digest() nbytes:%u\n", req->nbytes); + + /* whole thing at once */ + err = __ahash_init(req); + if (!err) + err = __ahash_finup(req); + + return err; +} + +static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key, + unsigned int keylen) +{ + struct iproc_ctx_s *ctx = crypto_ahash_ctx(ahash); + + flow_log("%s() ahash:%p key:%p keylen:%u\n", + __func__, ahash, key, keylen); + flow_dump(" key: ", key, keylen); + + if (ctx->auth.alg == HASH_ALG_AES) { + switch (keylen) { + case AES_KEYSIZE_128: + ctx->cipher_type = CIPHER_TYPE_AES128; + break; + case AES_KEYSIZE_192: + ctx->cipher_type = CIPHER_TYPE_AES192; + break; + case AES_KEYSIZE_256: + ctx->cipher_type = CIPHER_TYPE_AES256; + break; + default: + pr_err("%s() Error: Invalid key length\n", __func__); + return -EINVAL; + } + } else { + pr_err("%s() Error: unknown hash alg\n", __func__); + return -EINVAL; + } + memcpy(ctx->authkey, key, keylen); + ctx->authkeylen = keylen; + + return 0; +} + +static int ahash_export(struct ahash_request *req, void *out) +{ + const struct iproc_reqctx_s *rctx = ahash_request_ctx(req); + struct spu_hash_export_s *spu_exp = (struct spu_hash_export_s *)out; + + spu_exp->total_todo = rctx->total_todo; + spu_exp->total_sent = rctx->total_sent; + spu_exp->is_sw_hmac = rctx->is_sw_hmac; + memcpy(spu_exp->hash_carry, rctx->hash_carry, sizeof(rctx->hash_carry)); + spu_exp->hash_carry_len = rctx->hash_carry_len; + memcpy(spu_exp->incr_hash, rctx->incr_hash, sizeof(rctx->incr_hash)); + + return 0; +} + +static int ahash_import(struct ahash_request *req, const void *in) +{ + struct iproc_reqctx_s *rctx = ahash_request_ctx(req); + struct spu_hash_export_s *spu_exp = (struct spu_hash_export_s *)in; + + rctx->total_todo = spu_exp->total_todo; + rctx->total_sent = spu_exp->total_sent; + rctx->is_sw_hmac = spu_exp->is_sw_hmac; + memcpy(rctx->hash_carry, spu_exp->hash_carry, sizeof(rctx->hash_carry)); + rctx->hash_carry_len = spu_exp->hash_carry_len; + memcpy(rctx->incr_hash, spu_exp->incr_hash, sizeof(rctx->incr_hash)); + + return 0; +} + +static int ahash_hmac_setkey(struct crypto_ahash *ahash, const u8 *key, + unsigned int keylen) +{ + struct iproc_ctx_s *ctx = crypto_ahash_ctx(ahash); + unsigned int blocksize = + crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash)); + unsigned int digestsize = crypto_ahash_digestsize(ahash); + unsigned int index; + int rc; + + flow_log("%s() ahash:%p key:%p keylen:%u blksz:%u digestsz:%u\n", + __func__, ahash, key, keylen, blocksize, digestsize); + flow_dump(" key: ", key, keylen); + + if (keylen > blocksize) { + switch (ctx->auth.alg) { + case HASH_ALG_MD5: + rc = do_shash("md5", ctx->authkey, key, keylen, NULL, + 0, NULL, 0); + break; + case HASH_ALG_SHA1: + rc = do_shash("sha1", ctx->authkey, key, keylen, NULL, + 0, NULL, 0); + break; + case HASH_ALG_SHA224: + rc = do_shash("sha224", ctx->authkey, key, keylen, NULL, + 0, NULL, 0); + break; + case HASH_ALG_SHA256: + rc = do_shash("sha256", ctx->authkey, key, keylen, NULL, + 0, NULL, 0); + break; + case HASH_ALG_SHA384: + rc = do_shash("sha384", ctx->authkey, key, keylen, NULL, + 0, NULL, 0); + break; + case HASH_ALG_SHA512: + rc = do_shash("sha512", ctx->authkey, key, keylen, NULL, + 0, NULL, 0); + break; + case HASH_ALG_SHA3_224: + rc = do_shash("sha3-224", ctx->authkey, key, keylen, + NULL, 0, NULL, 0); + break; + case HASH_ALG_SHA3_256: + rc = do_shash("sha3-256", ctx->authkey, key, keylen, + NULL, 0, NULL, 0); + break; + case HASH_ALG_SHA3_384: + rc = do_shash("sha3-384", ctx->authkey, key, keylen, + NULL, 0, NULL, 0); + break; + case HASH_ALG_SHA3_512: + rc = do_shash("sha3-512", ctx->authkey, key, keylen, + NULL, 0, NULL, 0); + break; + default: + pr_err("%s() Error: unknown hash alg\n", __func__); + return -EINVAL; + } + if (rc < 0) { + pr_err("%s() Error %d computing shash for %s\n", + __func__, rc, hash_alg_name[ctx->auth.alg]); + return rc; + } + ctx->authkeylen = digestsize; + + flow_log(" keylen > digestsize... hashed\n"); + flow_dump(" newkey: ", ctx->authkey, ctx->authkeylen); + } else { + memcpy(ctx->authkey, key, keylen); + ctx->authkeylen = keylen; + } + + /* + * Full HMAC operation in SPUM is not verified, + * So keeping the generation of IPAD, OPAD and + * outer hashing in software. + */ + if (iproc_priv.spu.spu_type == SPU_TYPE_SPUM) { + memcpy_and_pad(ctx->ipad, blocksize, ctx->authkey, ctx->authkeylen, 0); + ctx->authkeylen = 0; + unsafe_memcpy(ctx->opad, ctx->ipad, blocksize, + "fortified memcpy causes -Wrestrict warning"); + + for (index = 0; index < blocksize; index++) { + ctx->ipad[index] ^= HMAC_IPAD_VALUE; + ctx->opad[index] ^= HMAC_OPAD_VALUE; + } + + flow_dump(" ipad: ", ctx->ipad, blocksize); + flow_dump(" opad: ", ctx->opad, blocksize); + } + ctx->digestsize = digestsize; + atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_HMAC]); + + return 0; +} + +static int ahash_hmac_init(struct ahash_request *req) +{ + int ret; + struct iproc_reqctx_s *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm); + unsigned int blocksize = + crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + + flow_log("ahash_hmac_init()\n"); + + /* init the context as a hash */ + ret = ahash_init(req); + if (ret) + return ret; + + if (!spu_no_incr_hash(ctx)) { + /* SPU-M can do incr hashing but needs sw for outer HMAC */ + rctx->is_sw_hmac = true; + ctx->auth.mode = HASH_MODE_HASH; + /* start with a prepended ipad */ + memcpy(rctx->hash_carry, ctx->ipad, blocksize); + rctx->hash_carry_len = blocksize; + rctx->total_todo += blocksize; + } + + return 0; +} + +static int ahash_hmac_update(struct ahash_request *req) +{ + flow_log("ahash_hmac_update() nbytes:%u\n", req->nbytes); + + if (!req->nbytes) + return 0; + + return ahash_update(req); +} + +static int ahash_hmac_final(struct ahash_request *req) +{ + flow_log("ahash_hmac_final() nbytes:%u\n", req->nbytes); + + return ahash_final(req); +} + +static int ahash_hmac_finup(struct ahash_request *req) +{ + flow_log("ahash_hmac_finupl() nbytes:%u\n", req->nbytes); + + return ahash_finup(req); +} + +static int ahash_hmac_digest(struct ahash_request *req) +{ + struct iproc_reqctx_s *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm); + unsigned int blocksize = + crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + + flow_log("ahash_hmac_digest() nbytes:%u\n", req->nbytes); + + /* Perform initialization and then call finup */ + __ahash_init(req); + + if (iproc_priv.spu.spu_type == SPU_TYPE_SPU2) { + /* + * SPU2 supports full HMAC implementation in the + * hardware, need not to generate IPAD, OPAD and + * outer hash in software. + * Only for hash key len > hash block size, SPU2 + * expects to perform hashing on the key, shorten + * it to digest size and feed it as hash key. + */ + rctx->is_sw_hmac = false; + ctx->auth.mode = HASH_MODE_HMAC; + } else { + rctx->is_sw_hmac = true; + ctx->auth.mode = HASH_MODE_HASH; + /* start with a prepended ipad */ + memcpy(rctx->hash_carry, ctx->ipad, blocksize); + rctx->hash_carry_len = blocksize; + rctx->total_todo += blocksize; + } + + return __ahash_finup(req); +} + +/* aead helpers */ + +static int aead_need_fallback(struct aead_request *req) +{ + struct iproc_reqctx_s *rctx = aead_request_ctx(req); + struct spu_hw *spu = &iproc_priv.spu; + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct iproc_ctx_s *ctx = crypto_aead_ctx(aead); + u32 payload_len; + + /* + * SPU hardware cannot handle the AES-GCM/CCM case where plaintext + * and AAD are both 0 bytes long. So use fallback in this case. + */ + if (((ctx->cipher.mode == CIPHER_MODE_GCM) || + (ctx->cipher.mode == CIPHER_MODE_CCM)) && + (req->assoclen == 0)) { + if ((rctx->is_encrypt && (req->cryptlen == 0)) || + (!rctx->is_encrypt && (req->cryptlen == ctx->digestsize))) { + flow_log("AES GCM/CCM needs fallback for 0 len req\n"); + return 1; + } + } + + /* SPU-M hardware only supports CCM digest size of 8, 12, or 16 bytes */ + if ((ctx->cipher.mode == CIPHER_MODE_CCM) && + (spu->spu_type == SPU_TYPE_SPUM) && + (ctx->digestsize != 8) && (ctx->digestsize != 12) && + (ctx->digestsize != 16)) { + flow_log("%s() AES CCM needs fallback for digest size %d\n", + __func__, ctx->digestsize); + return 1; + } + + /* + * SPU-M on NSP has an issue where AES-CCM hash is not correct + * when AAD size is 0 + */ + if ((ctx->cipher.mode == CIPHER_MODE_CCM) && + (spu->spu_subtype == SPU_SUBTYPE_SPUM_NSP) && + (req->assoclen == 0)) { + flow_log("%s() AES_CCM needs fallback for 0 len AAD on NSP\n", + __func__); + return 1; + } + + /* + * RFC4106 and RFC4543 cannot handle the case where AAD is other than + * 16 or 20 bytes long. So use fallback in this case. + */ + if (ctx->cipher.mode == CIPHER_MODE_GCM && + ctx->cipher.alg == CIPHER_ALG_AES && + rctx->iv_ctr_len == GCM_RFC4106_IV_SIZE && + req->assoclen != 16 && req->assoclen != 20) { + flow_log("RFC4106/RFC4543 needs fallback for assoclen" + " other than 16 or 20 bytes\n"); + return 1; + } + + payload_len = req->cryptlen; + if (spu->spu_type == SPU_TYPE_SPUM) + payload_len += req->assoclen; + + flow_log("%s() payload len: %u\n", __func__, payload_len); + + if (ctx->max_payload == SPU_MAX_PAYLOAD_INF) + return 0; + else + return payload_len > ctx->max_payload; +} + +static int aead_do_fallback(struct aead_request *req, bool is_encrypt) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct crypto_tfm *tfm = crypto_aead_tfm(aead); + struct iproc_reqctx_s *rctx = aead_request_ctx(req); + struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm); + struct aead_request *subreq; + + flow_log("%s() enc:%u\n", __func__, is_encrypt); + + if (!ctx->fallback_cipher) + return -EINVAL; + + subreq = &rctx->req; + aead_request_set_tfm(subreq, ctx->fallback_cipher); + aead_request_set_callback(subreq, aead_request_flags(req), + req->base.complete, req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, + req->iv); + aead_request_set_ad(subreq, req->assoclen); + + return is_encrypt ? crypto_aead_encrypt(req) : + crypto_aead_decrypt(req); +} + +static int aead_enqueue(struct aead_request *req, bool is_encrypt) +{ + struct iproc_reqctx_s *rctx = aead_request_ctx(req); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct iproc_ctx_s *ctx = crypto_aead_ctx(aead); + int err; + + flow_log("%s() enc:%u\n", __func__, is_encrypt); + + if (req->assoclen > MAX_ASSOC_SIZE) { + pr_err + ("%s() Error: associated data too long. (%u > %u bytes)\n", + __func__, req->assoclen, MAX_ASSOC_SIZE); + return -EINVAL; + } + + rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + rctx->parent = &req->base; + rctx->is_encrypt = is_encrypt; + rctx->bd_suppress = false; + rctx->total_todo = req->cryptlen; + rctx->src_sent = 0; + rctx->total_sent = 0; + rctx->total_received = 0; + rctx->is_sw_hmac = false; + rctx->ctx = ctx; + memset(&rctx->mb_mssg, 0, sizeof(struct brcm_message)); + + /* assoc data is at start of src sg */ + rctx->assoc = req->src; + + /* + * Init current position in src scatterlist to be after assoc data. + * src_skip set to buffer offset where data begins. (Assoc data could + * end in the middle of a buffer.) + */ + if (spu_sg_at_offset(req->src, req->assoclen, &rctx->src_sg, + &rctx->src_skip) < 0) { + pr_err("%s() Error: Unable to find start of src data\n", + __func__); + return -EINVAL; + } + + rctx->src_nents = 0; + rctx->dst_nents = 0; + if (req->dst == req->src) { + rctx->dst_sg = rctx->src_sg; + rctx->dst_skip = rctx->src_skip; + } else { + /* + * Expect req->dst to have room for assoc data followed by + * output data and ICV, if encrypt. So initialize dst_sg + * to point beyond assoc len offset. + */ + if (spu_sg_at_offset(req->dst, req->assoclen, &rctx->dst_sg, + &rctx->dst_skip) < 0) { + pr_err("%s() Error: Unable to find start of dst data\n", + __func__); + return -EINVAL; + } + } + + if (ctx->cipher.mode == CIPHER_MODE_CBC || + ctx->cipher.mode == CIPHER_MODE_CTR || + ctx->cipher.mode == CIPHER_MODE_OFB || + ctx->cipher.mode == CIPHER_MODE_XTS || + ctx->cipher.mode == CIPHER_MODE_GCM) { + rctx->iv_ctr_len = + ctx->salt_len + + crypto_aead_ivsize(crypto_aead_reqtfm(req)); + } else if (ctx->cipher.mode == CIPHER_MODE_CCM) { + rctx->iv_ctr_len = CCM_AES_IV_SIZE; + } else { + rctx->iv_ctr_len = 0; + } + + rctx->hash_carry_len = 0; + + flow_log(" src sg: %p\n", req->src); + flow_log(" rctx->src_sg: %p, src_skip %u\n", + rctx->src_sg, rctx->src_skip); + flow_log(" assoc: %p, assoclen %u\n", rctx->assoc, req->assoclen); + flow_log(" dst sg: %p\n", req->dst); + flow_log(" rctx->dst_sg: %p, dst_skip %u\n", + rctx->dst_sg, rctx->dst_skip); + flow_log(" iv_ctr_len:%u\n", rctx->iv_ctr_len); + flow_dump(" iv: ", req->iv, rctx->iv_ctr_len); + flow_log(" authkeylen:%u\n", ctx->authkeylen); + flow_log(" is_esp: %s\n", str_yes_no(ctx->is_esp)); + + if (ctx->max_payload == SPU_MAX_PAYLOAD_INF) + flow_log(" max_payload infinite"); + else + flow_log(" max_payload: %u\n", ctx->max_payload); + + if (unlikely(aead_need_fallback(req))) + return aead_do_fallback(req, is_encrypt); + + /* + * Do memory allocations for request after fallback check, because if we + * do fallback, we won't call finish_req() to dealloc. + */ + if (rctx->iv_ctr_len) { + if (ctx->salt_len) + memcpy(rctx->msg_buf.iv_ctr + ctx->salt_offset, + ctx->salt, ctx->salt_len); + memcpy(rctx->msg_buf.iv_ctr + ctx->salt_offset + ctx->salt_len, + req->iv, + rctx->iv_ctr_len - ctx->salt_len - ctx->salt_offset); + } + + rctx->chan_idx = select_channel(); + err = handle_aead_req(rctx); + if (err != -EINPROGRESS) + /* synchronous result */ + spu_chunk_cleanup(rctx); + + return err; +} + +static int aead_authenc_setkey(struct crypto_aead *cipher, + const u8 *key, unsigned int keylen) +{ + struct spu_hw *spu = &iproc_priv.spu; + struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher); + struct crypto_tfm *tfm = crypto_aead_tfm(cipher); + struct crypto_authenc_keys keys; + int ret; + + flow_log("%s() aead:%p key:%p keylen:%u\n", __func__, cipher, key, + keylen); + flow_dump(" key: ", key, keylen); + + ret = crypto_authenc_extractkeys(&keys, key, keylen); + if (ret) + goto badkey; + + if (keys.enckeylen > MAX_KEY_SIZE || + keys.authkeylen > MAX_KEY_SIZE) + goto badkey; + + ctx->enckeylen = keys.enckeylen; + ctx->authkeylen = keys.authkeylen; + + memcpy(ctx->enckey, keys.enckey, keys.enckeylen); + /* May end up padding auth key. So make sure it's zeroed. */ + memset(ctx->authkey, 0, sizeof(ctx->authkey)); + memcpy(ctx->authkey, keys.authkey, keys.authkeylen); + + switch (ctx->alg->cipher_info.alg) { + case CIPHER_ALG_DES: + if (verify_aead_des_key(cipher, keys.enckey, keys.enckeylen)) + return -EINVAL; + + ctx->cipher_type = CIPHER_TYPE_DES; + break; + case CIPHER_ALG_3DES: + if (verify_aead_des3_key(cipher, keys.enckey, keys.enckeylen)) + return -EINVAL; + + ctx->cipher_type = CIPHER_TYPE_3DES; + break; + case CIPHER_ALG_AES: + switch (ctx->enckeylen) { + case AES_KEYSIZE_128: + ctx->cipher_type = CIPHER_TYPE_AES128; + break; + case AES_KEYSIZE_192: + ctx->cipher_type = CIPHER_TYPE_AES192; + break; + case AES_KEYSIZE_256: + ctx->cipher_type = CIPHER_TYPE_AES256; + break; + default: + goto badkey; + } + break; + default: + pr_err("%s() Error: Unknown cipher alg\n", __func__); + return -EINVAL; + } + + flow_log(" enckeylen:%u authkeylen:%u\n", ctx->enckeylen, + ctx->authkeylen); + flow_dump(" enc: ", ctx->enckey, ctx->enckeylen); + flow_dump(" auth: ", ctx->authkey, ctx->authkeylen); + + /* setkey the fallback just in case we needto use it */ + if (ctx->fallback_cipher) { + flow_log(" running fallback setkey()\n"); + + ctx->fallback_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; + ctx->fallback_cipher->base.crt_flags |= + tfm->crt_flags & CRYPTO_TFM_REQ_MASK; + ret = crypto_aead_setkey(ctx->fallback_cipher, key, keylen); + if (ret) + flow_log(" fallback setkey() returned:%d\n", ret); + } + + ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen, + ctx->enckeylen, + false); + + atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_AEAD]); + + return ret; + +badkey: + ctx->enckeylen = 0; + ctx->authkeylen = 0; + ctx->digestsize = 0; + + return -EINVAL; +} + +static int aead_gcm_ccm_setkey(struct crypto_aead *cipher, + const u8 *key, unsigned int keylen) +{ + struct spu_hw *spu = &iproc_priv.spu; + struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher); + struct crypto_tfm *tfm = crypto_aead_tfm(cipher); + + int ret = 0; + + flow_log("%s() keylen:%u\n", __func__, keylen); + flow_dump(" key: ", key, keylen); + + if (!ctx->is_esp) + ctx->digestsize = keylen; + + ctx->enckeylen = keylen; + ctx->authkeylen = 0; + + switch (ctx->enckeylen) { + case AES_KEYSIZE_128: + ctx->cipher_type = CIPHER_TYPE_AES128; + break; + case AES_KEYSIZE_192: + ctx->cipher_type = CIPHER_TYPE_AES192; + break; + case AES_KEYSIZE_256: + ctx->cipher_type = CIPHER_TYPE_AES256; + break; + default: + goto badkey; + } + + memcpy(ctx->enckey, key, ctx->enckeylen); + + flow_log(" enckeylen:%u authkeylen:%u\n", ctx->enckeylen, + ctx->authkeylen); + flow_dump(" enc: ", ctx->enckey, ctx->enckeylen); + flow_dump(" auth: ", ctx->authkey, ctx->authkeylen); + + /* setkey the fallback just in case we need to use it */ + if (ctx->fallback_cipher) { + flow_log(" running fallback setkey()\n"); + + ctx->fallback_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; + ctx->fallback_cipher->base.crt_flags |= + tfm->crt_flags & CRYPTO_TFM_REQ_MASK; + ret = crypto_aead_setkey(ctx->fallback_cipher, key, + keylen + ctx->salt_len); + if (ret) + flow_log(" fallback setkey() returned:%d\n", ret); + } + + ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen, + ctx->enckeylen, + false); + + atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_AEAD]); + + flow_log(" enckeylen:%u authkeylen:%u\n", ctx->enckeylen, + ctx->authkeylen); + + return ret; + +badkey: + ctx->enckeylen = 0; + ctx->authkeylen = 0; + ctx->digestsize = 0; + + return -EINVAL; +} + +/** + * aead_gcm_esp_setkey() - setkey() operation for ESP variant of GCM AES. + * @cipher: AEAD structure + * @key: Key followed by 4 bytes of salt + * @keylen: Length of key plus salt, in bytes + * + * Extracts salt from key and stores it to be prepended to IV on each request. + * Digest is always 16 bytes + * + * Return: Value from generic gcm setkey. + */ +static int aead_gcm_esp_setkey(struct crypto_aead *cipher, + const u8 *key, unsigned int keylen) +{ + struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher); + + flow_log("%s\n", __func__); + + if (keylen < GCM_ESP_SALT_SIZE) + return -EINVAL; + + ctx->salt_len = GCM_ESP_SALT_SIZE; + ctx->salt_offset = GCM_ESP_SALT_OFFSET; + memcpy(ctx->salt, key + keylen - GCM_ESP_SALT_SIZE, GCM_ESP_SALT_SIZE); + keylen -= GCM_ESP_SALT_SIZE; + ctx->digestsize = GCM_ESP_DIGESTSIZE; + ctx->is_esp = true; + flow_dump("salt: ", ctx->salt, GCM_ESP_SALT_SIZE); + + return aead_gcm_ccm_setkey(cipher, key, keylen); +} + +/** + * rfc4543_gcm_esp_setkey() - setkey operation for RFC4543 variant of GCM/GMAC. + * @cipher: AEAD structure + * @key: Key followed by 4 bytes of salt + * @keylen: Length of key plus salt, in bytes + * + * Extracts salt from key and stores it to be prepended to IV on each request. + * Digest is always 16 bytes + * + * Return: Value from generic gcm setkey. + */ +static int rfc4543_gcm_esp_setkey(struct crypto_aead *cipher, + const u8 *key, unsigned int keylen) +{ + struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher); + + flow_log("%s\n", __func__); + + if (keylen < GCM_ESP_SALT_SIZE) + return -EINVAL; + + ctx->salt_len = GCM_ESP_SALT_SIZE; + ctx->salt_offset = GCM_ESP_SALT_OFFSET; + memcpy(ctx->salt, key + keylen - GCM_ESP_SALT_SIZE, GCM_ESP_SALT_SIZE); + keylen -= GCM_ESP_SALT_SIZE; + ctx->digestsize = GCM_ESP_DIGESTSIZE; + ctx->is_esp = true; + ctx->is_rfc4543 = true; + flow_dump("salt: ", ctx->salt, GCM_ESP_SALT_SIZE); + + return aead_gcm_ccm_setkey(cipher, key, keylen); +} + +/** + * aead_ccm_esp_setkey() - setkey() operation for ESP variant of CCM AES. + * @cipher: AEAD structure + * @key: Key followed by 4 bytes of salt + * @keylen: Length of key plus salt, in bytes + * + * Extracts salt from key and stores it to be prepended to IV on each request. + * Digest is always 16 bytes + * + * Return: Value from generic ccm setkey. + */ +static int aead_ccm_esp_setkey(struct crypto_aead *cipher, + const u8 *key, unsigned int keylen) +{ + struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher); + + flow_log("%s\n", __func__); + + if (keylen < CCM_ESP_SALT_SIZE) + return -EINVAL; + + ctx->salt_len = CCM_ESP_SALT_SIZE; + ctx->salt_offset = CCM_ESP_SALT_OFFSET; + memcpy(ctx->salt, key + keylen - CCM_ESP_SALT_SIZE, CCM_ESP_SALT_SIZE); + keylen -= CCM_ESP_SALT_SIZE; + ctx->is_esp = true; + flow_dump("salt: ", ctx->salt, CCM_ESP_SALT_SIZE); + + return aead_gcm_ccm_setkey(cipher, key, keylen); +} + +static int aead_setauthsize(struct crypto_aead *cipher, unsigned int authsize) +{ + struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher); + int ret = 0; + + flow_log("%s() authkeylen:%u authsize:%u\n", + __func__, ctx->authkeylen, authsize); + + ctx->digestsize = authsize; + + /* setkey the fallback just in case we needto use it */ + if (ctx->fallback_cipher) { + flow_log(" running fallback setauth()\n"); + + ret = crypto_aead_setauthsize(ctx->fallback_cipher, authsize); + if (ret) + flow_log(" fallback setauth() returned:%d\n", ret); + } + + return ret; +} + +static int aead_encrypt(struct aead_request *req) +{ + flow_log("%s() cryptlen:%u %08x\n", __func__, req->cryptlen, + req->cryptlen); + dump_sg(req->src, 0, req->cryptlen + req->assoclen); + flow_log(" assoc_len:%u\n", req->assoclen); + + return aead_enqueue(req, true); +} + +static int aead_decrypt(struct aead_request *req) +{ + flow_log("%s() cryptlen:%u\n", __func__, req->cryptlen); + dump_sg(req->src, 0, req->cryptlen + req->assoclen); + flow_log(" assoc_len:%u\n", req->assoclen); + + return aead_enqueue(req, false); +} + +/* ==================== Supported Cipher Algorithms ==================== */ + +static struct iproc_alg_s driver_algs[] = { + { + .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "gcm-aes-iproc", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK + }, + .setkey = aead_gcm_ccm_setkey, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_AES, + .mode = CIPHER_MODE_GCM, + }, + .auth_info = { + .alg = HASH_ALG_AES, + .mode = HASH_MODE_GCM, + }, + .auth_first = 0, + }, + { + .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "ccm(aes)", + .cra_driver_name = "ccm-aes-iproc", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK + }, + .setkey = aead_gcm_ccm_setkey, + .ivsize = CCM_AES_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_AES, + .mode = CIPHER_MODE_CCM, + }, + .auth_info = { + .alg = HASH_ALG_AES, + .mode = HASH_MODE_CCM, + }, + .auth_first = 0, + }, + { + .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "gcm-aes-esp-iproc", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK + }, + .setkey = aead_gcm_esp_setkey, + .ivsize = GCM_RFC4106_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_AES, + .mode = CIPHER_MODE_GCM, + }, + .auth_info = { + .alg = HASH_ALG_AES, + .mode = HASH_MODE_GCM, + }, + .auth_first = 0, + }, + { + .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "rfc4309(ccm(aes))", + .cra_driver_name = "ccm-aes-esp-iproc", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK + }, + .setkey = aead_ccm_esp_setkey, + .ivsize = CCM_AES_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_AES, + .mode = CIPHER_MODE_CCM, + }, + .auth_info = { + .alg = HASH_ALG_AES, + .mode = HASH_MODE_CCM, + }, + .auth_first = 0, + }, + { + .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "rfc4543(gcm(aes))", + .cra_driver_name = "gmac-aes-esp-iproc", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK + }, + .setkey = rfc4543_gcm_esp_setkey, + .ivsize = GCM_RFC4106_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_AES, + .mode = CIPHER_MODE_GCM, + }, + .auth_info = { + .alg = HASH_ALG_AES, + .mode = HASH_MODE_GCM, + }, + .auth_first = 0, + }, + { + .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(aes))", + .cra_driver_name = "authenc-hmac-md5-cbc-aes-iproc", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY + }, + .setkey = aead_authenc_setkey, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_AES, + .mode = CIPHER_MODE_CBC, + }, + .auth_info = { + .alg = HASH_ALG_MD5, + .mode = HASH_MODE_HMAC, + }, + .auth_first = 0, + }, + { + .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha1-cbc-aes-iproc", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY + }, + .setkey = aead_authenc_setkey, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_AES, + .mode = CIPHER_MODE_CBC, + }, + .auth_info = { + .alg = HASH_ALG_SHA1, + .mode = HASH_MODE_HMAC, + }, + .auth_first = 0, + }, + { + .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha256-cbc-aes-iproc", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY + }, + .setkey = aead_authenc_setkey, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_AES, + .mode = CIPHER_MODE_CBC, + }, + .auth_info = { + .alg = HASH_ALG_SHA256, + .mode = HASH_MODE_HMAC, + }, + .auth_first = 0, + }, + { + .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des))", + .cra_driver_name = "authenc-hmac-md5-cbc-des-iproc", + .cra_blocksize = DES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY + }, + .setkey = aead_authenc_setkey, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_DES, + .mode = CIPHER_MODE_CBC, + }, + .auth_info = { + .alg = HASH_ALG_MD5, + .mode = HASH_MODE_HMAC, + }, + .auth_first = 0, + }, + { + .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des))", + .cra_driver_name = "authenc-hmac-sha1-cbc-des-iproc", + .cra_blocksize = DES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY + }, + .setkey = aead_authenc_setkey, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_DES, + .mode = CIPHER_MODE_CBC, + }, + .auth_info = { + .alg = HASH_ALG_SHA1, + .mode = HASH_MODE_HMAC, + }, + .auth_first = 0, + }, + { + .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(des))", + .cra_driver_name = "authenc-hmac-sha224-cbc-des-iproc", + .cra_blocksize = DES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY + }, + .setkey = aead_authenc_setkey, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_DES, + .mode = CIPHER_MODE_CBC, + }, + .auth_info = { + .alg = HASH_ALG_SHA224, + .mode = HASH_MODE_HMAC, + }, + .auth_first = 0, + }, + { + .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(des))", + .cra_driver_name = "authenc-hmac-sha256-cbc-des-iproc", + .cra_blocksize = DES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY + }, + .setkey = aead_authenc_setkey, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_DES, + .mode = CIPHER_MODE_CBC, + }, + .auth_info = { + .alg = HASH_ALG_SHA256, + .mode = HASH_MODE_HMAC, + }, + .auth_first = 0, + }, + { + .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(des))", + .cra_driver_name = "authenc-hmac-sha384-cbc-des-iproc", + .cra_blocksize = DES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY + }, + .setkey = aead_authenc_setkey, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_DES, + .mode = CIPHER_MODE_CBC, + }, + .auth_info = { + .alg = HASH_ALG_SHA384, + .mode = HASH_MODE_HMAC, + }, + .auth_first = 0, + }, + { + .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(des))", + .cra_driver_name = "authenc-hmac-sha512-cbc-des-iproc", + .cra_blocksize = DES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY + }, + .setkey = aead_authenc_setkey, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_DES, + .mode = CIPHER_MODE_CBC, + }, + .auth_info = { + .alg = HASH_ALG_SHA512, + .mode = HASH_MODE_HMAC, + }, + .auth_first = 0, + }, + { + .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-md5-cbc-des3-iproc", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY + }, + .setkey = aead_authenc_setkey, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_3DES, + .mode = CIPHER_MODE_CBC, + }, + .auth_info = { + .alg = HASH_ALG_MD5, + .mode = HASH_MODE_HMAC, + }, + .auth_first = 0, + }, + { + .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha1-cbc-des3-iproc", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY + }, + .setkey = aead_authenc_setkey, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_3DES, + .mode = CIPHER_MODE_CBC, + }, + .auth_info = { + .alg = HASH_ALG_SHA1, + .mode = HASH_MODE_HMAC, + }, + .auth_first = 0, + }, + { + .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha224-cbc-des3-iproc", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY + }, + .setkey = aead_authenc_setkey, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_3DES, + .mode = CIPHER_MODE_CBC, + }, + .auth_info = { + .alg = HASH_ALG_SHA224, + .mode = HASH_MODE_HMAC, + }, + .auth_first = 0, + }, + { + .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha256-cbc-des3-iproc", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY + }, + .setkey = aead_authenc_setkey, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_3DES, + .mode = CIPHER_MODE_CBC, + }, + .auth_info = { + .alg = HASH_ALG_SHA256, + .mode = HASH_MODE_HMAC, + }, + .auth_first = 0, + }, + { + .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha384-cbc-des3-iproc", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY + }, + .setkey = aead_authenc_setkey, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_3DES, + .mode = CIPHER_MODE_CBC, + }, + .auth_info = { + .alg = HASH_ALG_SHA384, + .mode = HASH_MODE_HMAC, + }, + .auth_first = 0, + }, + { + .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha512-cbc-des3-iproc", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY + }, + .setkey = aead_authenc_setkey, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_3DES, + .mode = CIPHER_MODE_CBC, + }, + .auth_info = { + .alg = HASH_ALG_SHA512, + .mode = HASH_MODE_HMAC, + }, + .auth_first = 0, + }, + +/* SKCIPHER algorithms. */ + { + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .base.cra_name = "cbc(des)", + .base.cra_driver_name = "cbc-des-iproc", + .base.cra_blocksize = DES_BLOCK_SIZE, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_DES, + .mode = CIPHER_MODE_CBC, + }, + .auth_info = { + .alg = HASH_ALG_NONE, + .mode = HASH_MODE_NONE, + }, + }, + { + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .base.cra_name = "ecb(des)", + .base.cra_driver_name = "ecb-des-iproc", + .base.cra_blocksize = DES_BLOCK_SIZE, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = 0, + }, + .cipher_info = { + .alg = CIPHER_ALG_DES, + .mode = CIPHER_MODE_ECB, + }, + .auth_info = { + .alg = HASH_ALG_NONE, + .mode = HASH_MODE_NONE, + }, + }, + { + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .base.cra_name = "cbc(des3_ede)", + .base.cra_driver_name = "cbc-des3-iproc", + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_3DES, + .mode = CIPHER_MODE_CBC, + }, + .auth_info = { + .alg = HASH_ALG_NONE, + .mode = HASH_MODE_NONE, + }, + }, + { + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .base.cra_name = "ecb(des3_ede)", + .base.cra_driver_name = "ecb-des3-iproc", + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = 0, + }, + .cipher_info = { + .alg = CIPHER_ALG_3DES, + .mode = CIPHER_MODE_ECB, + }, + .auth_info = { + .alg = HASH_ALG_NONE, + .mode = HASH_MODE_NONE, + }, + }, + { + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-iproc", + .base.cra_blocksize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_AES, + .mode = CIPHER_MODE_CBC, + }, + .auth_info = { + .alg = HASH_ALG_NONE, + .mode = HASH_MODE_NONE, + }, + }, + { + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-iproc", + .base.cra_blocksize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = 0, + }, + .cipher_info = { + .alg = CIPHER_ALG_AES, + .mode = CIPHER_MODE_ECB, + }, + .auth_info = { + .alg = HASH_ALG_NONE, + .mode = HASH_MODE_NONE, + }, + }, + { + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "ctr-aes-iproc", + .base.cra_blocksize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_AES, + .mode = CIPHER_MODE_CTR, + }, + .auth_info = { + .alg = HASH_ALG_NONE, + .mode = HASH_MODE_NONE, + }, + }, +{ + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .base.cra_name = "xts(aes)", + .base.cra_driver_name = "xts-aes-iproc", + .base.cra_blocksize = AES_BLOCK_SIZE, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .cipher_info = { + .alg = CIPHER_ALG_AES, + .mode = CIPHER_MODE_XTS, + }, + .auth_info = { + .alg = HASH_ALG_NONE, + .mode = HASH_MODE_NONE, + }, + }, + +/* AHASH algorithms. */ + { + .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = MD5_DIGEST_SIZE, + .halg.base = { + .cra_name = "md5", + .cra_driver_name = "md5-iproc", + .cra_blocksize = MD5_BLOCK_WORDS * 4, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + } + }, + .cipher_info = { + .alg = CIPHER_ALG_NONE, + .mode = CIPHER_MODE_NONE, + }, + .auth_info = { + .alg = HASH_ALG_MD5, + .mode = HASH_MODE_HASH, + }, + }, + { + .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = MD5_DIGEST_SIZE, + .halg.base = { + .cra_name = "hmac(md5)", + .cra_driver_name = "hmac-md5-iproc", + .cra_blocksize = MD5_BLOCK_WORDS * 4, + } + }, + .cipher_info = { + .alg = CIPHER_ALG_NONE, + .mode = CIPHER_MODE_NONE, + }, + .auth_info = { + .alg = HASH_ALG_MD5, + .mode = HASH_MODE_HMAC, + }, + }, + {.type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-iproc", + .cra_blocksize = SHA1_BLOCK_SIZE, + } + }, + .cipher_info = { + .alg = CIPHER_ALG_NONE, + .mode = CIPHER_MODE_NONE, + }, + .auth_info = { + .alg = HASH_ALG_SHA1, + .mode = HASH_MODE_HASH, + }, + }, + {.type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.base = { + .cra_name = "hmac(sha1)", + .cra_driver_name = "hmac-sha1-iproc", + .cra_blocksize = SHA1_BLOCK_SIZE, + } + }, + .cipher_info = { + .alg = CIPHER_ALG_NONE, + .mode = CIPHER_MODE_NONE, + }, + .auth_info = { + .alg = HASH_ALG_SHA1, + .mode = HASH_MODE_HMAC, + }, + }, + {.type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA224_DIGEST_SIZE, + .halg.base = { + .cra_name = "sha224", + .cra_driver_name = "sha224-iproc", + .cra_blocksize = SHA224_BLOCK_SIZE, + } + }, + .cipher_info = { + .alg = CIPHER_ALG_NONE, + .mode = CIPHER_MODE_NONE, + }, + .auth_info = { + .alg = HASH_ALG_SHA224, + .mode = HASH_MODE_HASH, + }, + }, + {.type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA224_DIGEST_SIZE, + .halg.base = { + .cra_name = "hmac(sha224)", + .cra_driver_name = "hmac-sha224-iproc", + .cra_blocksize = SHA224_BLOCK_SIZE, + } + }, + .cipher_info = { + .alg = CIPHER_ALG_NONE, + .mode = CIPHER_MODE_NONE, + }, + .auth_info = { + .alg = HASH_ALG_SHA224, + .mode = HASH_MODE_HMAC, + }, + }, + {.type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-iproc", + .cra_blocksize = SHA256_BLOCK_SIZE, + } + }, + .cipher_info = { + .alg = CIPHER_ALG_NONE, + .mode = CIPHER_MODE_NONE, + }, + .auth_info = { + .alg = HASH_ALG_SHA256, + .mode = HASH_MODE_HASH, + }, + }, + {.type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "hmac-sha256-iproc", + .cra_blocksize = SHA256_BLOCK_SIZE, + } + }, + .cipher_info = { + .alg = CIPHER_ALG_NONE, + .mode = CIPHER_MODE_NONE, + }, + .auth_info = { + .alg = HASH_ALG_SHA256, + .mode = HASH_MODE_HMAC, + }, + }, + { + .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA384_DIGEST_SIZE, + .halg.base = { + .cra_name = "sha384", + .cra_driver_name = "sha384-iproc", + .cra_blocksize = SHA384_BLOCK_SIZE, + } + }, + .cipher_info = { + .alg = CIPHER_ALG_NONE, + .mode = CIPHER_MODE_NONE, + }, + .auth_info = { + .alg = HASH_ALG_SHA384, + .mode = HASH_MODE_HASH, + }, + }, + { + .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA384_DIGEST_SIZE, + .halg.base = { + .cra_name = "hmac(sha384)", + .cra_driver_name = "hmac-sha384-iproc", + .cra_blocksize = SHA384_BLOCK_SIZE, + } + }, + .cipher_info = { + .alg = CIPHER_ALG_NONE, + .mode = CIPHER_MODE_NONE, + }, + .auth_info = { + .alg = HASH_ALG_SHA384, + .mode = HASH_MODE_HMAC, + }, + }, + { + .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA512_DIGEST_SIZE, + .halg.base = { + .cra_name = "sha512", + .cra_driver_name = "sha512-iproc", + .cra_blocksize = SHA512_BLOCK_SIZE, + } + }, + .cipher_info = { + .alg = CIPHER_ALG_NONE, + .mode = CIPHER_MODE_NONE, + }, + .auth_info = { + .alg = HASH_ALG_SHA512, + .mode = HASH_MODE_HASH, + }, + }, + { + .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA512_DIGEST_SIZE, + .halg.base = { + .cra_name = "hmac(sha512)", + .cra_driver_name = "hmac-sha512-iproc", + .cra_blocksize = SHA512_BLOCK_SIZE, + } + }, + .cipher_info = { + .alg = CIPHER_ALG_NONE, + .mode = CIPHER_MODE_NONE, + }, + .auth_info = { + .alg = HASH_ALG_SHA512, + .mode = HASH_MODE_HMAC, + }, + }, + { + .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA3_224_DIGEST_SIZE, + .halg.base = { + .cra_name = "sha3-224", + .cra_driver_name = "sha3-224-iproc", + .cra_blocksize = SHA3_224_BLOCK_SIZE, + } + }, + .cipher_info = { + .alg = CIPHER_ALG_NONE, + .mode = CIPHER_MODE_NONE, + }, + .auth_info = { + .alg = HASH_ALG_SHA3_224, + .mode = HASH_MODE_HASH, + }, + }, + { + .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA3_224_DIGEST_SIZE, + .halg.base = { + .cra_name = "hmac(sha3-224)", + .cra_driver_name = "hmac-sha3-224-iproc", + .cra_blocksize = SHA3_224_BLOCK_SIZE, + } + }, + .cipher_info = { + .alg = CIPHER_ALG_NONE, + .mode = CIPHER_MODE_NONE, + }, + .auth_info = { + .alg = HASH_ALG_SHA3_224, + .mode = HASH_MODE_HMAC + }, + }, + { + .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA3_256_DIGEST_SIZE, + .halg.base = { + .cra_name = "sha3-256", + .cra_driver_name = "sha3-256-iproc", + .cra_blocksize = SHA3_256_BLOCK_SIZE, + } + }, + .cipher_info = { + .alg = CIPHER_ALG_NONE, + .mode = CIPHER_MODE_NONE, + }, + .auth_info = { + .alg = HASH_ALG_SHA3_256, + .mode = HASH_MODE_HASH, + }, + }, + { + .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA3_256_DIGEST_SIZE, + .halg.base = { + .cra_name = "hmac(sha3-256)", + .cra_driver_name = "hmac-sha3-256-iproc", + .cra_blocksize = SHA3_256_BLOCK_SIZE, + } + }, + .cipher_info = { + .alg = CIPHER_ALG_NONE, + .mode = CIPHER_MODE_NONE, + }, + .auth_info = { + .alg = HASH_ALG_SHA3_256, + .mode = HASH_MODE_HMAC, + }, + }, + { + .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA3_384_DIGEST_SIZE, + .halg.base = { + .cra_name = "sha3-384", + .cra_driver_name = "sha3-384-iproc", + .cra_blocksize = SHA3_224_BLOCK_SIZE, + } + }, + .cipher_info = { + .alg = CIPHER_ALG_NONE, + .mode = CIPHER_MODE_NONE, + }, + .auth_info = { + .alg = HASH_ALG_SHA3_384, + .mode = HASH_MODE_HASH, + }, + }, + { + .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA3_384_DIGEST_SIZE, + .halg.base = { + .cra_name = "hmac(sha3-384)", + .cra_driver_name = "hmac-sha3-384-iproc", + .cra_blocksize = SHA3_384_BLOCK_SIZE, + } + }, + .cipher_info = { + .alg = CIPHER_ALG_NONE, + .mode = CIPHER_MODE_NONE, + }, + .auth_info = { + .alg = HASH_ALG_SHA3_384, + .mode = HASH_MODE_HMAC, + }, + }, + { + .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA3_512_DIGEST_SIZE, + .halg.base = { + .cra_name = "sha3-512", + .cra_driver_name = "sha3-512-iproc", + .cra_blocksize = SHA3_512_BLOCK_SIZE, + } + }, + .cipher_info = { + .alg = CIPHER_ALG_NONE, + .mode = CIPHER_MODE_NONE, + }, + .auth_info = { + .alg = HASH_ALG_SHA3_512, + .mode = HASH_MODE_HASH, + }, + }, + { + .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA3_512_DIGEST_SIZE, + .halg.base = { + .cra_name = "hmac(sha3-512)", + .cra_driver_name = "hmac-sha3-512-iproc", + .cra_blocksize = SHA3_512_BLOCK_SIZE, + } + }, + .cipher_info = { + .alg = CIPHER_ALG_NONE, + .mode = CIPHER_MODE_NONE, + }, + .auth_info = { + .alg = HASH_ALG_SHA3_512, + .mode = HASH_MODE_HMAC, + }, + }, + { + .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = AES_BLOCK_SIZE, + .halg.base = { + .cra_name = "xcbc(aes)", + .cra_driver_name = "xcbc-aes-iproc", + .cra_blocksize = AES_BLOCK_SIZE, + } + }, + .cipher_info = { + .alg = CIPHER_ALG_NONE, + .mode = CIPHER_MODE_NONE, + }, + .auth_info = { + .alg = HASH_ALG_AES, + .mode = HASH_MODE_XCBC, + }, + }, + { + .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = AES_BLOCK_SIZE, + .halg.base = { + .cra_name = "cmac(aes)", + .cra_driver_name = "cmac-aes-iproc", + .cra_blocksize = AES_BLOCK_SIZE, + } + }, + .cipher_info = { + .alg = CIPHER_ALG_NONE, + .mode = CIPHER_MODE_NONE, + }, + .auth_info = { + .alg = HASH_ALG_AES, + .mode = HASH_MODE_CMAC, + }, + }, +}; + +static int generic_cra_init(struct crypto_tfm *tfm, + struct iproc_alg_s *cipher_alg) +{ + struct spu_hw *spu = &iproc_priv.spu; + struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm); + unsigned int blocksize = crypto_tfm_alg_blocksize(tfm); + + flow_log("%s()\n", __func__); + + ctx->alg = cipher_alg; + ctx->cipher = cipher_alg->cipher_info; + ctx->auth = cipher_alg->auth_info; + ctx->auth_first = cipher_alg->auth_first; + ctx->max_payload = spu->spu_ctx_max_payload(ctx->cipher.alg, + ctx->cipher.mode, + blocksize); + ctx->fallback_cipher = NULL; + + ctx->enckeylen = 0; + ctx->authkeylen = 0; + + atomic_inc(&iproc_priv.stream_count); + atomic_inc(&iproc_priv.session_count); + + return 0; +} + +static int skcipher_init_tfm(struct crypto_skcipher *skcipher) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher); + struct skcipher_alg *alg = crypto_skcipher_alg(skcipher); + struct iproc_alg_s *cipher_alg; + + flow_log("%s()\n", __func__); + + crypto_skcipher_set_reqsize(skcipher, sizeof(struct iproc_reqctx_s)); + + cipher_alg = container_of(alg, struct iproc_alg_s, alg.skcipher); + return generic_cra_init(tfm, cipher_alg); +} + +static int ahash_cra_init(struct crypto_tfm *tfm) +{ + int err; + struct crypto_alg *alg = tfm->__crt_alg; + struct iproc_alg_s *cipher_alg; + + cipher_alg = container_of(__crypto_ahash_alg(alg), struct iproc_alg_s, + alg.hash); + + err = generic_cra_init(tfm, cipher_alg); + flow_log("%s()\n", __func__); + + /* + * export state size has to be < 512 bytes. So don't include msg bufs + * in state size. + */ + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct iproc_reqctx_s)); + + return err; +} + +static int aead_cra_init(struct crypto_aead *aead) +{ + unsigned int reqsize = sizeof(struct iproc_reqctx_s); + struct crypto_tfm *tfm = crypto_aead_tfm(aead); + struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm); + struct crypto_alg *alg = tfm->__crt_alg; + struct aead_alg *aalg = container_of(alg, struct aead_alg, base); + struct iproc_alg_s *cipher_alg = container_of(aalg, struct iproc_alg_s, + alg.aead); + + int err = generic_cra_init(tfm, cipher_alg); + + flow_log("%s()\n", __func__); + + ctx->is_esp = false; + ctx->salt_len = 0; + ctx->salt_offset = 0; + + /* random first IV */ + get_random_bytes(ctx->iv, MAX_IV_SIZE); + flow_dump(" iv: ", ctx->iv, MAX_IV_SIZE); + + if (err) + goto out; + + if (!(alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK)) + goto reqsize; + + flow_log("%s() creating fallback cipher\n", __func__); + + ctx->fallback_cipher = crypto_alloc_aead(alg->cra_name, 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fallback_cipher)) { + pr_err("%s() Error: failed to allocate fallback for %s\n", + __func__, alg->cra_name); + return PTR_ERR(ctx->fallback_cipher); + } + + reqsize += crypto_aead_reqsize(ctx->fallback_cipher); + +reqsize: + crypto_aead_set_reqsize(aead, reqsize); + +out: + return err; +} + +static void generic_cra_exit(struct crypto_tfm *tfm) +{ + atomic_dec(&iproc_priv.session_count); +} + +static void skcipher_exit_tfm(struct crypto_skcipher *tfm) +{ + generic_cra_exit(crypto_skcipher_tfm(tfm)); +} + +static void aead_cra_exit(struct crypto_aead *aead) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(aead); + struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm); + + generic_cra_exit(tfm); + + if (ctx->fallback_cipher) { + crypto_free_aead(ctx->fallback_cipher); + ctx->fallback_cipher = NULL; + } +} + +/** + * spu_functions_register() - Specify hardware-specific SPU functions based on + * SPU type read from device tree. + * @dev: device structure + * @spu_type: SPU hardware generation + * @spu_subtype: SPU hardware version + */ +static void spu_functions_register(struct device *dev, + enum spu_spu_type spu_type, + enum spu_spu_subtype spu_subtype) +{ + struct spu_hw *spu = &iproc_priv.spu; + + if (spu_type == SPU_TYPE_SPUM) { + dev_dbg(dev, "Registering SPUM functions"); + spu->spu_dump_msg_hdr = spum_dump_msg_hdr; + spu->spu_payload_length = spum_payload_length; + spu->spu_response_hdr_len = spum_response_hdr_len; + spu->spu_hash_pad_len = spum_hash_pad_len; + spu->spu_gcm_ccm_pad_len = spum_gcm_ccm_pad_len; + spu->spu_assoc_resp_len = spum_assoc_resp_len; + spu->spu_aead_ivlen = spum_aead_ivlen; + spu->spu_hash_type = spum_hash_type; + spu->spu_digest_size = spum_digest_size; + spu->spu_create_request = spum_create_request; + spu->spu_cipher_req_init = spum_cipher_req_init; + spu->spu_cipher_req_finish = spum_cipher_req_finish; + spu->spu_request_pad = spum_request_pad; + spu->spu_tx_status_len = spum_tx_status_len; + spu->spu_rx_status_len = spum_rx_status_len; + spu->spu_status_process = spum_status_process; + spu->spu_xts_tweak_in_payload = spum_xts_tweak_in_payload; + spu->spu_ccm_update_iv = spum_ccm_update_iv; + spu->spu_wordalign_padlen = spum_wordalign_padlen; + if (spu_subtype == SPU_SUBTYPE_SPUM_NS2) + spu->spu_ctx_max_payload = spum_ns2_ctx_max_payload; + else + spu->spu_ctx_max_payload = spum_nsp_ctx_max_payload; + } else { + dev_dbg(dev, "Registering SPU2 functions"); + spu->spu_dump_msg_hdr = spu2_dump_msg_hdr; + spu->spu_ctx_max_payload = spu2_ctx_max_payload; + spu->spu_payload_length = spu2_payload_length; + spu->spu_response_hdr_len = spu2_response_hdr_len; + spu->spu_hash_pad_len = spu2_hash_pad_len; + spu->spu_gcm_ccm_pad_len = spu2_gcm_ccm_pad_len; + spu->spu_assoc_resp_len = spu2_assoc_resp_len; + spu->spu_aead_ivlen = spu2_aead_ivlen; + spu->spu_hash_type = spu2_hash_type; + spu->spu_digest_size = spu2_digest_size; + spu->spu_create_request = spu2_create_request; + spu->spu_cipher_req_init = spu2_cipher_req_init; + spu->spu_cipher_req_finish = spu2_cipher_req_finish; + spu->spu_request_pad = spu2_request_pad; + spu->spu_tx_status_len = spu2_tx_status_len; + spu->spu_rx_status_len = spu2_rx_status_len; + spu->spu_status_process = spu2_status_process; + spu->spu_xts_tweak_in_payload = spu2_xts_tweak_in_payload; + spu->spu_ccm_update_iv = spu2_ccm_update_iv; + spu->spu_wordalign_padlen = spu2_wordalign_padlen; + } +} + +/** + * spu_mb_init() - Initialize mailbox client. Request ownership of a mailbox + * channel for the SPU being probed. + * @dev: SPU driver device structure + * + * Return: 0 if successful + * < 0 otherwise + */ +static int spu_mb_init(struct device *dev) +{ + struct mbox_client *mcl = &iproc_priv.mcl; + int err, i; + + iproc_priv.mbox = devm_kcalloc(dev, iproc_priv.spu.num_chan, + sizeof(struct mbox_chan *), GFP_KERNEL); + if (!iproc_priv.mbox) + return -ENOMEM; + + mcl->dev = dev; + mcl->tx_block = false; + mcl->tx_tout = 0; + mcl->knows_txdone = true; + mcl->rx_callback = spu_rx_callback; + mcl->tx_done = NULL; + + for (i = 0; i < iproc_priv.spu.num_chan; i++) { + iproc_priv.mbox[i] = mbox_request_channel(mcl, i); + if (IS_ERR(iproc_priv.mbox[i])) { + err = PTR_ERR(iproc_priv.mbox[i]); + dev_err(dev, + "Mbox channel %d request failed with err %d", + i, err); + iproc_priv.mbox[i] = NULL; + goto free_channels; + } + } + + return 0; +free_channels: + for (i = 0; i < iproc_priv.spu.num_chan; i++) { + if (iproc_priv.mbox[i]) + mbox_free_channel(iproc_priv.mbox[i]); + } + + return err; +} + +static void spu_mb_release(struct platform_device *pdev) +{ + int i; + + for (i = 0; i < iproc_priv.spu.num_chan; i++) + mbox_free_channel(iproc_priv.mbox[i]); +} + +static void spu_counters_init(void) +{ + int i; + int j; + + atomic_set(&iproc_priv.session_count, 0); + atomic_set(&iproc_priv.stream_count, 0); + atomic_set(&iproc_priv.next_chan, (int)iproc_priv.spu.num_chan); + atomic64_set(&iproc_priv.bytes_in, 0); + atomic64_set(&iproc_priv.bytes_out, 0); + for (i = 0; i < SPU_OP_NUM; i++) { + atomic_set(&iproc_priv.op_counts[i], 0); + atomic_set(&iproc_priv.setkey_cnt[i], 0); + } + for (i = 0; i < CIPHER_ALG_LAST; i++) + for (j = 0; j < CIPHER_MODE_LAST; j++) + atomic_set(&iproc_priv.cipher_cnt[i][j], 0); + + for (i = 0; i < HASH_ALG_LAST; i++) { + atomic_set(&iproc_priv.hash_cnt[i], 0); + atomic_set(&iproc_priv.hmac_cnt[i], 0); + } + for (i = 0; i < AEAD_TYPE_LAST; i++) + atomic_set(&iproc_priv.aead_cnt[i], 0); + + atomic_set(&iproc_priv.mb_no_spc, 0); + atomic_set(&iproc_priv.mb_send_fail, 0); + atomic_set(&iproc_priv.bad_icv, 0); +} + +static int spu_register_skcipher(struct iproc_alg_s *driver_alg) +{ + struct skcipher_alg *crypto = &driver_alg->alg.skcipher; + int err; + + crypto->base.cra_module = THIS_MODULE; + crypto->base.cra_priority = cipher_pri; + crypto->base.cra_alignmask = 0; + crypto->base.cra_ctxsize = sizeof(struct iproc_ctx_s); + crypto->base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY; + + crypto->init = skcipher_init_tfm; + crypto->exit = skcipher_exit_tfm; + crypto->setkey = skcipher_setkey; + crypto->encrypt = skcipher_encrypt; + crypto->decrypt = skcipher_decrypt; + + err = crypto_register_skcipher(crypto); + /* Mark alg as having been registered, if successful */ + if (err == 0) + driver_alg->registered = true; + pr_debug(" registered skcipher %s\n", crypto->base.cra_driver_name); + return err; +} + +static int spu_register_ahash(struct iproc_alg_s *driver_alg) +{ + struct spu_hw *spu = &iproc_priv.spu; + struct ahash_alg *hash = &driver_alg->alg.hash; + int err; + + /* AES-XCBC is the only AES hash type currently supported on SPU-M */ + if ((driver_alg->auth_info.alg == HASH_ALG_AES) && + (driver_alg->auth_info.mode != HASH_MODE_XCBC) && + (spu->spu_type == SPU_TYPE_SPUM)) + return 0; + + /* SHA3 algorithm variants are not registered for SPU-M or SPU2. */ + if ((driver_alg->auth_info.alg >= HASH_ALG_SHA3_224) && + (spu->spu_subtype != SPU_SUBTYPE_SPU2_V2)) + return 0; + + hash->halg.base.cra_module = THIS_MODULE; + hash->halg.base.cra_priority = hash_pri; + hash->halg.base.cra_alignmask = 0; + hash->halg.base.cra_ctxsize = sizeof(struct iproc_ctx_s); + hash->halg.base.cra_init = ahash_cra_init; + hash->halg.base.cra_exit = generic_cra_exit; + hash->halg.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY; + hash->halg.statesize = sizeof(struct spu_hash_export_s); + + if (driver_alg->auth_info.mode != HASH_MODE_HMAC) { + hash->init = ahash_init; + hash->update = ahash_update; + hash->final = ahash_final; + hash->finup = ahash_finup; + hash->digest = ahash_digest; + if ((driver_alg->auth_info.alg == HASH_ALG_AES) && + ((driver_alg->auth_info.mode == HASH_MODE_XCBC) || + (driver_alg->auth_info.mode == HASH_MODE_CMAC))) { + hash->setkey = ahash_setkey; + } + } else { + hash->setkey = ahash_hmac_setkey; + hash->init = ahash_hmac_init; + hash->update = ahash_hmac_update; + hash->final = ahash_hmac_final; + hash->finup = ahash_hmac_finup; + hash->digest = ahash_hmac_digest; + } + hash->export = ahash_export; + hash->import = ahash_import; + + err = crypto_register_ahash(hash); + /* Mark alg as having been registered, if successful */ + if (err == 0) + driver_alg->registered = true; + pr_debug(" registered ahash %s\n", + hash->halg.base.cra_driver_name); + return err; +} + +static int spu_register_aead(struct iproc_alg_s *driver_alg) +{ + struct aead_alg *aead = &driver_alg->alg.aead; + int err; + + aead->base.cra_module = THIS_MODULE; + aead->base.cra_priority = aead_pri; + aead->base.cra_alignmask = 0; + aead->base.cra_ctxsize = sizeof(struct iproc_ctx_s); + + aead->base.cra_flags |= CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY; + /* setkey set in alg initialization */ + aead->setauthsize = aead_setauthsize; + aead->encrypt = aead_encrypt; + aead->decrypt = aead_decrypt; + aead->init = aead_cra_init; + aead->exit = aead_cra_exit; + + err = crypto_register_aead(aead); + /* Mark alg as having been registered, if successful */ + if (err == 0) + driver_alg->registered = true; + pr_debug(" registered aead %s\n", aead->base.cra_driver_name); + return err; +} + +/* register crypto algorithms the device supports */ +static int spu_algs_register(struct device *dev) +{ + int i, j; + int err; + + for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { + switch (driver_algs[i].type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + err = spu_register_skcipher(&driver_algs[i]); + break; + case CRYPTO_ALG_TYPE_AHASH: + err = spu_register_ahash(&driver_algs[i]); + break; + case CRYPTO_ALG_TYPE_AEAD: + err = spu_register_aead(&driver_algs[i]); + break; + default: + dev_err(dev, + "iproc-crypto: unknown alg type: %d", + driver_algs[i].type); + err = -EINVAL; + } + + if (err) { + dev_err(dev, "alg registration failed with error %d\n", + err); + goto err_algs; + } + } + + return 0; + +err_algs: + for (j = 0; j < i; j++) { + /* Skip any algorithm not registered */ + if (!driver_algs[j].registered) + continue; + switch (driver_algs[j].type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + crypto_unregister_skcipher(&driver_algs[j].alg.skcipher); + driver_algs[j].registered = false; + break; + case CRYPTO_ALG_TYPE_AHASH: + crypto_unregister_ahash(&driver_algs[j].alg.hash); + driver_algs[j].registered = false; + break; + case CRYPTO_ALG_TYPE_AEAD: + crypto_unregister_aead(&driver_algs[j].alg.aead); + driver_algs[j].registered = false; + break; + } + } + return err; +} + +/* ==================== Kernel Platform API ==================== */ + +static struct spu_type_subtype spum_ns2_types = { + SPU_TYPE_SPUM, SPU_SUBTYPE_SPUM_NS2 +}; + +static struct spu_type_subtype spum_nsp_types = { + SPU_TYPE_SPUM, SPU_SUBTYPE_SPUM_NSP +}; + +static struct spu_type_subtype spu2_types = { + SPU_TYPE_SPU2, SPU_SUBTYPE_SPU2_V1 +}; + +static struct spu_type_subtype spu2_v2_types = { + SPU_TYPE_SPU2, SPU_SUBTYPE_SPU2_V2 +}; + +static const struct of_device_id bcm_spu_dt_ids[] = { + { + .compatible = "brcm,spum-crypto", + .data = &spum_ns2_types, + }, + { + .compatible = "brcm,spum-nsp-crypto", + .data = &spum_nsp_types, + }, + { + .compatible = "brcm,spu2-crypto", + .data = &spu2_types, + }, + { + .compatible = "brcm,spu2-v2-crypto", + .data = &spu2_v2_types, + }, + { /* sentinel */ } +}; + +MODULE_DEVICE_TABLE(of, bcm_spu_dt_ids); + +static int spu_dt_read(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct spu_hw *spu = &iproc_priv.spu; + struct resource *spu_ctrl_regs; + const struct spu_type_subtype *matched_spu_type; + struct device_node *dn = pdev->dev.of_node; + int err, i; + + /* Count number of mailbox channels */ + spu->num_chan = of_count_phandle_with_args(dn, "mboxes", "#mbox-cells"); + + matched_spu_type = of_device_get_match_data(dev); + if (!matched_spu_type) { + dev_err(dev, "Failed to match device\n"); + return -ENODEV; + } + + spu->spu_type = matched_spu_type->type; + spu->spu_subtype = matched_spu_type->subtype; + + for (i = 0; (i < MAX_SPUS) && ((spu_ctrl_regs = + platform_get_resource(pdev, IORESOURCE_MEM, i)) != NULL); i++) { + + spu->reg_vbase[i] = devm_ioremap_resource(dev, spu_ctrl_regs); + if (IS_ERR(spu->reg_vbase[i])) { + err = PTR_ERR(spu->reg_vbase[i]); + dev_err(dev, "Failed to map registers: %d\n", + err); + spu->reg_vbase[i] = NULL; + return err; + } + } + spu->num_spu = i; + dev_dbg(dev, "Device has %d SPUs", spu->num_spu); + + return 0; +} + +static int bcm_spu_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct spu_hw *spu = &iproc_priv.spu; + int err; + + iproc_priv.pdev = pdev; + platform_set_drvdata(iproc_priv.pdev, + &iproc_priv); + + err = spu_dt_read(pdev); + if (err < 0) + goto failure; + + err = spu_mb_init(dev); + if (err < 0) + goto failure; + + if (spu->spu_type == SPU_TYPE_SPUM) + iproc_priv.bcm_hdr_len = 8; + else if (spu->spu_type == SPU_TYPE_SPU2) + iproc_priv.bcm_hdr_len = 0; + + spu_functions_register(dev, spu->spu_type, spu->spu_subtype); + + spu_counters_init(); + + spu_setup_debugfs(); + + err = spu_algs_register(dev); + if (err < 0) + goto fail_reg; + + return 0; + +fail_reg: + spu_free_debugfs(); +failure: + spu_mb_release(pdev); + dev_err(dev, "%s failed with error %d.\n", __func__, err); + + return err; +} + +static void bcm_spu_remove(struct platform_device *pdev) +{ + int i; + struct device *dev = &pdev->dev; + char *cdn; + + for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { + /* + * Not all algorithms were registered, depending on whether + * hardware is SPU or SPU2. So here we make sure to skip + * those algorithms that were not previously registered. + */ + if (!driver_algs[i].registered) + continue; + + switch (driver_algs[i].type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + crypto_unregister_skcipher(&driver_algs[i].alg.skcipher); + dev_dbg(dev, " unregistered cipher %s\n", + driver_algs[i].alg.skcipher.base.cra_driver_name); + driver_algs[i].registered = false; + break; + case CRYPTO_ALG_TYPE_AHASH: + crypto_unregister_ahash(&driver_algs[i].alg.hash); + cdn = driver_algs[i].alg.hash.halg.base.cra_driver_name; + dev_dbg(dev, " unregistered hash %s\n", cdn); + driver_algs[i].registered = false; + break; + case CRYPTO_ALG_TYPE_AEAD: + crypto_unregister_aead(&driver_algs[i].alg.aead); + dev_dbg(dev, " unregistered aead %s\n", + driver_algs[i].alg.aead.base.cra_driver_name); + driver_algs[i].registered = false; + break; + } + } + spu_free_debugfs(); + spu_mb_release(pdev); +} + +/* ===== Kernel Module API ===== */ + +static struct platform_driver bcm_spu_pdriver = { + .driver = { + .name = "brcm-spu-crypto", + .of_match_table = bcm_spu_dt_ids, + }, + .probe = bcm_spu_probe, + .remove = bcm_spu_remove, +}; +module_platform_driver(bcm_spu_pdriver); + +MODULE_AUTHOR("Rob Rice "); +MODULE_DESCRIPTION("Broadcom symmetric crypto offload driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/crypto/bcm/cipher.h b/drivers/crypto/bcm/cipher.h new file mode 100644 index 000000000..e36881c98 --- /dev/null +++ b/drivers/crypto/bcm/cipher.h @@ -0,0 +1,469 @@ + +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2016 Broadcom + */ + +#ifndef _CIPHER_H +#define _CIPHER_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "spu.h" +#include "spum.h" +#include "spu2.h" + +/* Driver supports up to MAX_SPUS SPU blocks */ +#define MAX_SPUS 16 + +#define ARC4_STATE_SIZE 4 + +#define CCM_AES_IV_SIZE 16 +#define CCM_ESP_IV_SIZE 8 +#define RFC4543_ICV_SIZE 16 + +#define MAX_KEY_SIZE ARC4_MAX_KEY_SIZE +#define MAX_IV_SIZE AES_BLOCK_SIZE +#define MAX_DIGEST_SIZE SHA3_512_DIGEST_SIZE +#define MAX_ASSOC_SIZE 512 + +/* size of salt value for AES-GCM-ESP and AES-CCM-ESP */ +#define GCM_ESP_SALT_SIZE 4 +#define CCM_ESP_SALT_SIZE 3 +#define MAX_SALT_SIZE GCM_ESP_SALT_SIZE +#define GCM_ESP_SALT_OFFSET 0 +#define CCM_ESP_SALT_OFFSET 1 + +#define GCM_ESP_DIGESTSIZE 16 + +#define MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE + +/* + * Maximum number of bytes from a non-final hash request that can be deferred + * until more data is available. With new crypto API framework, this + * can be no more than one block of data. + */ +#define HASH_CARRY_MAX MAX_HASH_BLOCK_SIZE + +/* Force at least 4-byte alignment of all SPU message fields */ +#define SPU_MSG_ALIGN 4 + +/* Number of times to resend mailbox message if mb queue is full */ +#define SPU_MB_RETRY_MAX 1000 + +/* op_counts[] indexes */ +enum op_type { + SPU_OP_CIPHER, + SPU_OP_HASH, + SPU_OP_HMAC, + SPU_OP_AEAD, + SPU_OP_NUM +}; + +enum spu_spu_type { + SPU_TYPE_SPUM, + SPU_TYPE_SPU2, +}; + +/* + * SPUM_NS2 and SPUM_NSP are the SPU-M block on Northstar 2 and Northstar Plus, + * respectively. + */ +enum spu_spu_subtype { + SPU_SUBTYPE_SPUM_NS2, + SPU_SUBTYPE_SPUM_NSP, + SPU_SUBTYPE_SPU2_V1, + SPU_SUBTYPE_SPU2_V2 +}; + +struct spu_type_subtype { + enum spu_spu_type type; + enum spu_spu_subtype subtype; +}; + +struct cipher_op { + enum spu_cipher_alg alg; + enum spu_cipher_mode mode; +}; + +struct auth_op { + enum hash_alg alg; + enum hash_mode mode; +}; + +struct iproc_alg_s { + u32 type; + union { + struct skcipher_alg skcipher; + struct ahash_alg hash; + struct aead_alg aead; + } alg; + struct cipher_op cipher_info; + struct auth_op auth_info; + bool auth_first; + bool registered; +}; + +/* + * Buffers for a SPU request/reply message pair. All part of one structure to + * allow a single alloc per request. + */ +struct spu_msg_buf { + /* Request message fragments */ + + /* + * SPU request message header. For SPU-M, holds MH, EMH, SCTX, BDESC, + * and BD header. For SPU2, holds FMD, OMD. + */ + u8 bcm_spu_req_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)]; + + /* IV or counter. Size to include salt. Also used for XTS tweek. */ + u8 iv_ctr[ALIGN(2 * AES_BLOCK_SIZE, SPU_MSG_ALIGN)]; + + /* Hash digest. request and response. */ + u8 digest[ALIGN(MAX_DIGEST_SIZE, SPU_MSG_ALIGN)]; + + /* SPU request message padding */ + u8 spu_req_pad[ALIGN(SPU_PAD_LEN_MAX, SPU_MSG_ALIGN)]; + + /* SPU-M request message STATUS field */ + u8 tx_stat[ALIGN(SPU_TX_STATUS_LEN, SPU_MSG_ALIGN)]; + + /* Response message fragments */ + + /* SPU response message header */ + u8 spu_resp_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)]; + + /* SPU response message STATUS field padding */ + u8 rx_stat_pad[ALIGN(SPU_STAT_PAD_MAX, SPU_MSG_ALIGN)]; + + /* SPU response message STATUS field */ + u8 rx_stat[ALIGN(SPU_RX_STATUS_LEN, SPU_MSG_ALIGN)]; + + union { + /* Buffers only used for skcipher */ + struct { + /* + * Field used for either SUPDT when RC4 is used + * -OR- tweak value when XTS/AES is used + */ + u8 supdt_tweak[ALIGN(SPU_SUPDT_LEN, SPU_MSG_ALIGN)]; + } c; + + /* Buffers only used for aead */ + struct { + /* SPU response pad for GCM data */ + u8 gcmpad[ALIGN(AES_BLOCK_SIZE, SPU_MSG_ALIGN)]; + + /* SPU request msg padding for GCM AAD */ + u8 req_aad_pad[ALIGN(SPU_PAD_LEN_MAX, SPU_MSG_ALIGN)]; + + /* SPU response data to be discarded */ + u8 resp_aad[ALIGN(MAX_ASSOC_SIZE + MAX_IV_SIZE, + SPU_MSG_ALIGN)]; + } a; + }; +}; + +struct iproc_ctx_s { + u8 enckey[MAX_KEY_SIZE + ARC4_STATE_SIZE]; + unsigned int enckeylen; + + u8 authkey[MAX_KEY_SIZE + ARC4_STATE_SIZE]; + unsigned int authkeylen; + + u8 salt[MAX_SALT_SIZE]; + unsigned int salt_len; + unsigned int salt_offset; + u8 iv[MAX_IV_SIZE]; + + unsigned int digestsize; + + struct iproc_alg_s *alg; + bool is_esp; + + struct cipher_op cipher; + enum spu_cipher_type cipher_type; + + struct auth_op auth; + bool auth_first; + + /* + * The maximum length in bytes of the payload in a SPU message for this + * context. For SPU-M, the payload is the combination of AAD and data. + * For SPU2, the payload is just data. A value of SPU_MAX_PAYLOAD_INF + * indicates that there is no limit to the length of the SPU message + * payload. + */ + unsigned int max_payload; + + struct crypto_aead *fallback_cipher; + + /* auth_type is determined during processing of request */ + + u8 ipad[MAX_HASH_BLOCK_SIZE]; + u8 opad[MAX_HASH_BLOCK_SIZE]; + + /* + * Buffer to hold SPU message header template. Template is created at + * setkey time for skcipher requests, since most of the fields in the + * header are known at that time. At request time, just fill in a few + * missing pieces related to length of data in the request and IVs, etc. + */ + u8 bcm_spu_req_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)]; + + /* Length of SPU request header */ + u16 spu_req_hdr_len; + + /* Expected length of SPU response header */ + u16 spu_resp_hdr_len; + + /* + * shash descriptor - needed to perform incremental hashing in + * software, when hw doesn't support it. + */ + struct shash_desc *shash; + + bool is_rfc4543; /* RFC 4543 style of GMAC */ +}; + +/* state from iproc_reqctx_s necessary for hash state export/import */ +struct spu_hash_export_s { + unsigned int total_todo; + unsigned int total_sent; + u8 hash_carry[HASH_CARRY_MAX]; + unsigned int hash_carry_len; + u8 incr_hash[MAX_DIGEST_SIZE]; + bool is_sw_hmac; +}; + +struct iproc_reqctx_s { + /* general context */ + struct crypto_async_request *parent; + + /* only valid after enqueue() */ + struct iproc_ctx_s *ctx; + + u8 chan_idx; /* Mailbox channel to be used to submit this request */ + + /* total todo, rx'd, and sent for this request */ + unsigned int total_todo; + unsigned int total_received; /* only valid for skcipher */ + unsigned int total_sent; + + /* + * num bytes sent to hw from the src sg in this request. This can differ + * from total_sent for incremental hashing. total_sent includes previous + * init() and update() data. src_sent does not. + */ + unsigned int src_sent; + + /* + * For AEAD requests, start of associated data. This will typically + * point to the beginning of the src scatterlist from the request, + * since assoc data is at the beginning of the src scatterlist rather + * than in its own sg. + */ + struct scatterlist *assoc; + + /* + * scatterlist entry and offset to start of data for next chunk. Crypto + * API src scatterlist for AEAD starts with AAD, if present. For first + * chunk, src_sg is sg entry at beginning of input data (after AAD). + * src_skip begins at the offset in that sg entry where data begins. + */ + struct scatterlist *src_sg; + int src_nents; /* Number of src entries with data */ + u32 src_skip; /* bytes of current sg entry already used */ + + /* + * Same for destination. For AEAD, if there is AAD, output data must + * be written at offset following AAD. + */ + struct scatterlist *dst_sg; + int dst_nents; /* Number of dst entries with data */ + u32 dst_skip; /* bytes of current sg entry already written */ + + /* Mailbox message used to send this request to PDC driver */ + struct brcm_message mb_mssg; + + bool bd_suppress; /* suppress BD field in SPU response? */ + + /* cipher context */ + bool is_encrypt; + + /* + * CBC mode: IV. CTR mode: counter. Else empty. Used as a DMA + * buffer for AEAD requests. So allocate as DMAable memory. If IV + * concatenated with salt, includes the salt. + */ + u8 *iv_ctr; + /* Length of IV or counter, in bytes */ + unsigned int iv_ctr_len; + + /* + * Hash requests can be of any size, whether initial, update, or final. + * A non-final request must be submitted to the SPU as an integral + * number of blocks. This may leave data at the end of the request + * that is not a full block. Since the request is non-final, it cannot + * be padded. So, we write the remainder to this hash_carry buffer and + * hold it until the next request arrives. The carry data is then + * submitted at the beginning of the data in the next SPU msg. + * hash_carry_len is the number of bytes currently in hash_carry. These + * fields are only used for ahash requests. + */ + u8 hash_carry[HASH_CARRY_MAX]; + unsigned int hash_carry_len; + unsigned int is_final; /* is this the final for the hash op? */ + + /* + * Digest from incremental hash is saved here to include in next hash + * operation. Cannot be stored in req->result for truncated hashes, + * since result may be sized for final digest. Cannot be saved in + * msg_buf because that gets deleted between incremental hash ops + * and is not saved as part of export(). + */ + u8 incr_hash[MAX_DIGEST_SIZE]; + + /* hmac context */ + bool is_sw_hmac; + + gfp_t gfp; + + /* Buffers used to build SPU request and response messages */ + struct spu_msg_buf msg_buf; + + struct aead_request req; +}; + +/* + * Structure encapsulates a set of function pointers specific to the type of + * SPU hardware running. These functions handling creation and parsing of + * SPU request messages and SPU response messages. Includes hardware-specific + * values read from device tree. + */ +struct spu_hw { + void (*spu_dump_msg_hdr)(u8 *buf, unsigned int buf_len); + u32 (*spu_ctx_max_payload)(enum spu_cipher_alg cipher_alg, + enum spu_cipher_mode cipher_mode, + unsigned int blocksize); + u32 (*spu_payload_length)(u8 *spu_hdr); + u16 (*spu_response_hdr_len)(u16 auth_key_len, u16 enc_key_len, + bool is_hash); + u16 (*spu_hash_pad_len)(enum hash_alg hash_alg, + enum hash_mode hash_mode, u32 chunksize, + u16 hash_block_size); + u32 (*spu_gcm_ccm_pad_len)(enum spu_cipher_mode cipher_mode, + unsigned int data_size); + u32 (*spu_assoc_resp_len)(enum spu_cipher_mode cipher_mode, + unsigned int assoc_len, + unsigned int iv_len, bool is_encrypt); + u8 (*spu_aead_ivlen)(enum spu_cipher_mode cipher_mode, + u16 iv_len); + enum hash_type (*spu_hash_type)(u32 src_sent); + u32 (*spu_digest_size)(u32 digest_size, enum hash_alg alg, + enum hash_type); + u32 (*spu_create_request)(u8 *spu_hdr, + struct spu_request_opts *req_opts, + struct spu_cipher_parms *cipher_parms, + struct spu_hash_parms *hash_parms, + struct spu_aead_parms *aead_parms, + unsigned int data_size); + u16 (*spu_cipher_req_init)(u8 *spu_hdr, + struct spu_cipher_parms *cipher_parms); + void (*spu_cipher_req_finish)(u8 *spu_hdr, + u16 spu_req_hdr_len, + unsigned int is_inbound, + struct spu_cipher_parms *cipher_parms, + unsigned int data_size); + void (*spu_request_pad)(u8 *pad_start, u32 gcm_padding, + u32 hash_pad_len, enum hash_alg auth_alg, + enum hash_mode auth_mode, + unsigned int total_sent, u32 status_padding); + u8 (*spu_xts_tweak_in_payload)(void); + u8 (*spu_tx_status_len)(void); + u8 (*spu_rx_status_len)(void); + int (*spu_status_process)(u8 *statp); + void (*spu_ccm_update_iv)(unsigned int digestsize, + struct spu_cipher_parms *cipher_parms, + unsigned int assoclen, unsigned int chunksize, + bool is_encrypt, bool is_esp); + u32 (*spu_wordalign_padlen)(u32 data_size); + + /* The base virtual address of the SPU hw registers */ + void __iomem *reg_vbase[MAX_SPUS]; + + /* Version of the SPU hardware */ + enum spu_spu_type spu_type; + + /* Sub-version of the SPU hardware */ + enum spu_spu_subtype spu_subtype; + + /* The number of SPUs on this platform */ + u32 num_spu; + + /* The number of SPU channels on this platform */ + u32 num_chan; +}; + +struct bcm_device_private { + struct platform_device *pdev; + + struct spu_hw spu; + + atomic_t session_count; /* number of streams active */ + atomic_t stream_count; /* monotonic counter for streamID's */ + + /* Length of BCM header. Set to 0 when hw does not expect BCM HEADER. */ + u8 bcm_hdr_len; + + /* The index of the channel to use for the next crypto request */ + atomic_t next_chan; + + struct dentry *debugfs_dir; + struct dentry *debugfs_stats; + + /* Number of request bytes processed and result bytes returned */ + atomic64_t bytes_in; + atomic64_t bytes_out; + + /* Number of operations of each type */ + atomic_t op_counts[SPU_OP_NUM]; + + atomic_t cipher_cnt[CIPHER_ALG_LAST][CIPHER_MODE_LAST]; + atomic_t hash_cnt[HASH_ALG_LAST]; + atomic_t hmac_cnt[HASH_ALG_LAST]; + atomic_t aead_cnt[AEAD_TYPE_LAST]; + + /* Number of calls to setkey() for each operation type */ + atomic_t setkey_cnt[SPU_OP_NUM]; + + /* Number of times request was resubmitted because mb was full */ + atomic_t mb_no_spc; + + /* Number of mailbox send failures */ + atomic_t mb_send_fail; + + /* Number of ICV check failures for AEAD messages */ + atomic_t bad_icv; + + struct mbox_client mcl; + + /* Array of mailbox channel pointers, one for each channel */ + struct mbox_chan **mbox; +}; + +extern struct bcm_device_private iproc_priv; + +#endif diff --git a/drivers/crypto/bcm/spu.c b/drivers/crypto/bcm/spu.c new file mode 100644 index 000000000..86c227caa --- /dev/null +++ b/drivers/crypto/bcm/spu.c @@ -0,0 +1,1209 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2016 Broadcom + */ + +#include +#include + +#include "util.h" +#include "spu.h" +#include "spum.h" +#include "cipher.h" + +char *hash_alg_name[] = { "None", "md5", "sha1", "sha224", "sha256", "aes", + "sha384", "sha512", "sha3_224", "sha3_256", "sha3_384", "sha3_512" }; + +char *aead_alg_name[] = { "ccm(aes)", "gcm(aes)", "authenc" }; + +/* Assumes SPU-M messages are in big endian */ +void spum_dump_msg_hdr(u8 *buf, unsigned int buf_len) +{ + u8 *ptr = buf; + struct SPUHEADER *spuh = (struct SPUHEADER *)buf; + unsigned int hash_key_len = 0; + unsigned int hash_state_len = 0; + unsigned int cipher_key_len = 0; + unsigned int iv_len; + u32 pflags; + u32 cflags; + u32 ecf; + u32 cipher_alg; + u32 cipher_mode; + u32 cipher_type; + u32 hash_alg; + u32 hash_mode; + u32 hash_type; + u32 sctx_size; /* SCTX length in words */ + u32 sctx_pl_len; /* SCTX payload length in bytes */ + + packet_log("\n"); + packet_log("SPU Message header %p len: %u\n", buf, buf_len); + + /* ========== Decode MH ========== */ + packet_log(" MH 0x%08x\n", be32_to_cpup((__be32 *)ptr)); + if (spuh->mh.flags & MH_SCTX_PRES) + packet_log(" SCTX present\n"); + if (spuh->mh.flags & MH_BDESC_PRES) + packet_log(" BDESC present\n"); + if (spuh->mh.flags & MH_MFM_PRES) + packet_log(" MFM present\n"); + if (spuh->mh.flags & MH_BD_PRES) + packet_log(" BD present\n"); + if (spuh->mh.flags & MH_HASH_PRES) + packet_log(" HASH present\n"); + if (spuh->mh.flags & MH_SUPDT_PRES) + packet_log(" SUPDT present\n"); + packet_log(" Opcode 0x%02x\n", spuh->mh.op_code); + + ptr += sizeof(spuh->mh) + sizeof(spuh->emh); /* skip emh. unused */ + + /* ========== Decode SCTX ========== */ + if (spuh->mh.flags & MH_SCTX_PRES) { + pflags = be32_to_cpu(spuh->sa.proto_flags); + packet_log(" SCTX[0] 0x%08x\n", pflags); + sctx_size = pflags & SCTX_SIZE; + packet_log(" Size %u words\n", sctx_size); + + cflags = be32_to_cpu(spuh->sa.cipher_flags); + packet_log(" SCTX[1] 0x%08x\n", cflags); + packet_log(" Inbound:%lu (1:decrypt/vrfy 0:encrypt/auth)\n", + (cflags & CIPHER_INBOUND) >> CIPHER_INBOUND_SHIFT); + packet_log(" Order:%lu (1:AuthFirst 0:EncFirst)\n", + (cflags & CIPHER_ORDER) >> CIPHER_ORDER_SHIFT); + packet_log(" ICV_IS_512:%lx\n", + (cflags & ICV_IS_512) >> ICV_IS_512_SHIFT); + cipher_alg = (cflags & CIPHER_ALG) >> CIPHER_ALG_SHIFT; + cipher_mode = (cflags & CIPHER_MODE) >> CIPHER_MODE_SHIFT; + cipher_type = (cflags & CIPHER_TYPE) >> CIPHER_TYPE_SHIFT; + packet_log(" Crypto Alg:%u Mode:%u Type:%u\n", + cipher_alg, cipher_mode, cipher_type); + hash_alg = (cflags & HASH_ALG) >> HASH_ALG_SHIFT; + hash_mode = (cflags & HASH_MODE) >> HASH_MODE_SHIFT; + hash_type = (cflags & HASH_TYPE) >> HASH_TYPE_SHIFT; + packet_log(" Hash Alg:%x Mode:%x Type:%x\n", + hash_alg, hash_mode, hash_type); + packet_log(" UPDT_Offset:%u\n", cflags & UPDT_OFST); + + ecf = be32_to_cpu(spuh->sa.ecf); + packet_log(" SCTX[2] 0x%08x\n", ecf); + packet_log(" WriteICV:%lu CheckICV:%lu ICV_SIZE:%u ", + (ecf & INSERT_ICV) >> INSERT_ICV_SHIFT, + (ecf & CHECK_ICV) >> CHECK_ICV_SHIFT, + (ecf & ICV_SIZE) >> ICV_SIZE_SHIFT); + packet_log("BD_SUPPRESS:%lu\n", + (ecf & BD_SUPPRESS) >> BD_SUPPRESS_SHIFT); + packet_log(" SCTX_IV:%lu ExplicitIV:%lu GenIV:%lu ", + (ecf & SCTX_IV) >> SCTX_IV_SHIFT, + (ecf & EXPLICIT_IV) >> EXPLICIT_IV_SHIFT, + (ecf & GEN_IV) >> GEN_IV_SHIFT); + packet_log("IV_OV_OFST:%lu EXP_IV_SIZE:%u\n", + (ecf & IV_OFFSET) >> IV_OFFSET_SHIFT, + ecf & EXP_IV_SIZE); + + ptr += sizeof(struct SCTX); + + if (hash_alg && hash_mode) { + char *name = "NONE"; + + switch (hash_alg) { + case HASH_ALG_MD5: + hash_key_len = 16; + name = "MD5"; + break; + case HASH_ALG_SHA1: + hash_key_len = 20; + name = "SHA1"; + break; + case HASH_ALG_SHA224: + hash_key_len = 28; + name = "SHA224"; + break; + case HASH_ALG_SHA256: + hash_key_len = 32; + name = "SHA256"; + break; + case HASH_ALG_SHA384: + hash_key_len = 48; + name = "SHA384"; + break; + case HASH_ALG_SHA512: + hash_key_len = 64; + name = "SHA512"; + break; + case HASH_ALG_AES: + hash_key_len = 0; + name = "AES"; + break; + case HASH_ALG_NONE: + break; + } + + packet_log(" Auth Key Type:%s Length:%u Bytes\n", + name, hash_key_len); + packet_dump(" KEY: ", ptr, hash_key_len); + ptr += hash_key_len; + } else if ((hash_alg == HASH_ALG_AES) && + (hash_mode == HASH_MODE_XCBC)) { + char *name = "NONE"; + + switch (cipher_type) { + case CIPHER_TYPE_AES128: + hash_key_len = 16; + name = "AES128-XCBC"; + break; + case CIPHER_TYPE_AES192: + hash_key_len = 24; + name = "AES192-XCBC"; + break; + case CIPHER_TYPE_AES256: + hash_key_len = 32; + name = "AES256-XCBC"; + break; + } + packet_log(" Auth Key Type:%s Length:%u Bytes\n", + name, hash_key_len); + packet_dump(" KEY: ", ptr, hash_key_len); + ptr += hash_key_len; + } + + if (hash_alg && (hash_mode == HASH_MODE_NONE) && + (hash_type == HASH_TYPE_UPDT)) { + char *name = "NONE"; + + switch (hash_alg) { + case HASH_ALG_MD5: + hash_state_len = 16; + name = "MD5"; + break; + case HASH_ALG_SHA1: + hash_state_len = 20; + name = "SHA1"; + break; + case HASH_ALG_SHA224: + hash_state_len = 32; + name = "SHA224"; + break; + case HASH_ALG_SHA256: + hash_state_len = 32; + name = "SHA256"; + break; + case HASH_ALG_SHA384: + hash_state_len = 48; + name = "SHA384"; + break; + case HASH_ALG_SHA512: + hash_state_len = 64; + name = "SHA512"; + break; + case HASH_ALG_AES: + hash_state_len = 0; + name = "AES"; + break; + case HASH_ALG_NONE: + break; + } + + packet_log(" Auth State Type:%s Length:%u Bytes\n", + name, hash_state_len); + packet_dump(" State: ", ptr, hash_state_len); + ptr += hash_state_len; + } + + if (cipher_alg) { + char *name = "NONE"; + + switch (cipher_alg) { + case CIPHER_ALG_DES: + cipher_key_len = 8; + name = "DES"; + break; + case CIPHER_ALG_3DES: + cipher_key_len = 24; + name = "3DES"; + break; + case CIPHER_ALG_AES: + switch (cipher_type) { + case CIPHER_TYPE_AES128: + cipher_key_len = 16; + name = "AES128"; + break; + case CIPHER_TYPE_AES192: + cipher_key_len = 24; + name = "AES192"; + break; + case CIPHER_TYPE_AES256: + cipher_key_len = 32; + name = "AES256"; + break; + } + break; + case CIPHER_ALG_NONE: + break; + } + + packet_log(" Cipher Key Type:%s Length:%u Bytes\n", + name, cipher_key_len); + + /* XTS has two keys */ + if (cipher_mode == CIPHER_MODE_XTS) { + packet_dump(" KEY2: ", ptr, cipher_key_len); + ptr += cipher_key_len; + packet_dump(" KEY1: ", ptr, cipher_key_len); + ptr += cipher_key_len; + + cipher_key_len *= 2; + } else { + packet_dump(" KEY: ", ptr, cipher_key_len); + ptr += cipher_key_len; + } + + if (ecf & SCTX_IV) { + sctx_pl_len = sctx_size * sizeof(u32) - + sizeof(struct SCTX); + iv_len = sctx_pl_len - + (hash_key_len + hash_state_len + + cipher_key_len); + packet_log(" IV Length:%u Bytes\n", iv_len); + packet_dump(" IV: ", ptr, iv_len); + ptr += iv_len; + } + } + } + + /* ========== Decode BDESC ========== */ + if (spuh->mh.flags & MH_BDESC_PRES) { + struct BDESC_HEADER *bdesc = (struct BDESC_HEADER *)ptr; + + packet_log(" BDESC[0] 0x%08x\n", be32_to_cpup((__be32 *)ptr)); + packet_log(" OffsetMAC:%u LengthMAC:%u\n", + be16_to_cpu(bdesc->offset_mac), + be16_to_cpu(bdesc->length_mac)); + ptr += sizeof(u32); + + packet_log(" BDESC[1] 0x%08x\n", be32_to_cpup((__be32 *)ptr)); + packet_log(" OffsetCrypto:%u LengthCrypto:%u\n", + be16_to_cpu(bdesc->offset_crypto), + be16_to_cpu(bdesc->length_crypto)); + ptr += sizeof(u32); + + packet_log(" BDESC[2] 0x%08x\n", be32_to_cpup((__be32 *)ptr)); + packet_log(" OffsetICV:%u OffsetIV:%u\n", + be16_to_cpu(bdesc->offset_icv), + be16_to_cpu(bdesc->offset_iv)); + ptr += sizeof(u32); + } + + /* ========== Decode BD ========== */ + if (spuh->mh.flags & MH_BD_PRES) { + struct BD_HEADER *bd = (struct BD_HEADER *)ptr; + + packet_log(" BD[0] 0x%08x\n", be32_to_cpup((__be32 *)ptr)); + packet_log(" Size:%ubytes PrevLength:%u\n", + be16_to_cpu(bd->size), be16_to_cpu(bd->prev_length)); + ptr += 4; + } + + /* Double check sanity */ + if (buf + buf_len != ptr) { + packet_log(" Packet parsed incorrectly. "); + packet_log("buf:%p buf_len:%u buf+buf_len:%p ptr:%p\n", + buf, buf_len, buf + buf_len, ptr); + } + + packet_log("\n"); +} + +/** + * spum_ns2_ctx_max_payload() - Determine the max length of the payload for a + * SPU message for a given cipher and hash alg context. + * @cipher_alg: The cipher algorithm + * @cipher_mode: The cipher mode + * @blocksize: The size of a block of data for this algo + * + * The max payload must be a multiple of the blocksize so that if a request is + * too large to fit in a single SPU message, the request can be broken into + * max_payload sized chunks. Each chunk must be a multiple of blocksize. + * + * Return: Max payload length in bytes + */ +u32 spum_ns2_ctx_max_payload(enum spu_cipher_alg cipher_alg, + enum spu_cipher_mode cipher_mode, + unsigned int blocksize) +{ + u32 max_payload = SPUM_NS2_MAX_PAYLOAD; + u32 excess; + + /* In XTS on SPU-M, we'll need to insert tweak before input data */ + if (cipher_mode == CIPHER_MODE_XTS) + max_payload -= SPU_XTS_TWEAK_SIZE; + + excess = max_payload % blocksize; + + return max_payload - excess; +} + +/** + * spum_nsp_ctx_max_payload() - Determine the max length of the payload for a + * SPU message for a given cipher and hash alg context. + * @cipher_alg: The cipher algorithm + * @cipher_mode: The cipher mode + * @blocksize: The size of a block of data for this algo + * + * The max payload must be a multiple of the blocksize so that if a request is + * too large to fit in a single SPU message, the request can be broken into + * max_payload sized chunks. Each chunk must be a multiple of blocksize. + * + * Return: Max payload length in bytes + */ +u32 spum_nsp_ctx_max_payload(enum spu_cipher_alg cipher_alg, + enum spu_cipher_mode cipher_mode, + unsigned int blocksize) +{ + u32 max_payload = SPUM_NSP_MAX_PAYLOAD; + u32 excess; + + /* In XTS on SPU-M, we'll need to insert tweak before input data */ + if (cipher_mode == CIPHER_MODE_XTS) + max_payload -= SPU_XTS_TWEAK_SIZE; + + excess = max_payload % blocksize; + + return max_payload - excess; +} + +/** spum_payload_length() - Given a SPU-M message header, extract the payload + * length. + * @spu_hdr: Start of SPU header + * + * Assumes just MH, EMH, BD (no SCTX, BDESC. Works for response frames. + * + * Return: payload length in bytes + */ +u32 spum_payload_length(u8 *spu_hdr) +{ + struct BD_HEADER *bd; + u32 pl_len; + + /* Find BD header. skip MH, EMH */ + bd = (struct BD_HEADER *)(spu_hdr + 8); + pl_len = be16_to_cpu(bd->size); + + return pl_len; +} + +/** + * spum_response_hdr_len() - Given the length of the hash key and encryption + * key, determine the expected length of a SPU response header. + * @auth_key_len: authentication key length (bytes) + * @enc_key_len: encryption key length (bytes) + * @is_hash: true if response message is for a hash operation + * + * Return: length of SPU response header (bytes) + */ +u16 spum_response_hdr_len(u16 auth_key_len, u16 enc_key_len, bool is_hash) +{ + if (is_hash) + return SPU_HASH_RESP_HDR_LEN; + else + return SPU_RESP_HDR_LEN; +} + +/** + * spum_hash_pad_len() - Calculate the length of hash padding required to extend + * data to a full block size. + * @hash_alg: hash algorithm + * @hash_mode: hash mode + * @chunksize: length of data, in bytes + * @hash_block_size: size of a block of data for hash algorithm + * + * Reserve space for 1 byte (0x80) start of pad and the total length as u64 + * + * Return: length of hash pad in bytes + */ +u16 spum_hash_pad_len(enum hash_alg hash_alg, enum hash_mode hash_mode, + u32 chunksize, u16 hash_block_size) +{ + unsigned int length_len; + unsigned int used_space_last_block; + int hash_pad_len; + + /* AES-XCBC hash requires just padding to next block boundary */ + if ((hash_alg == HASH_ALG_AES) && (hash_mode == HASH_MODE_XCBC)) { + used_space_last_block = chunksize % hash_block_size; + hash_pad_len = hash_block_size - used_space_last_block; + if (hash_pad_len >= hash_block_size) + hash_pad_len -= hash_block_size; + return hash_pad_len; + } + + used_space_last_block = chunksize % hash_block_size + 1; + if ((hash_alg == HASH_ALG_SHA384) || (hash_alg == HASH_ALG_SHA512)) + length_len = 2 * sizeof(u64); + else + length_len = sizeof(u64); + + used_space_last_block += length_len; + hash_pad_len = hash_block_size - used_space_last_block; + if (hash_pad_len < 0) + hash_pad_len += hash_block_size; + + hash_pad_len += 1 + length_len; + return hash_pad_len; +} + +/** + * spum_gcm_ccm_pad_len() - Determine the required length of GCM or CCM padding. + * @cipher_mode: Algo type + * @data_size: Length of plaintext (bytes) + * + * Return: Length of padding, in bytes + */ +u32 spum_gcm_ccm_pad_len(enum spu_cipher_mode cipher_mode, + unsigned int data_size) +{ + u32 pad_len = 0; + u32 m1 = SPU_GCM_CCM_ALIGN - 1; + + if ((cipher_mode == CIPHER_MODE_GCM) || + (cipher_mode == CIPHER_MODE_CCM)) + pad_len = ((data_size + m1) & ~m1) - data_size; + + return pad_len; +} + +/** + * spum_assoc_resp_len() - Determine the size of the receive buffer required to + * catch associated data. + * @cipher_mode: cipher mode + * @assoc_len: length of associated data (bytes) + * @iv_len: length of IV (bytes) + * @is_encrypt: true if encrypting. false if decrypting. + * + * Return: length of associated data in response message (bytes) + */ +u32 spum_assoc_resp_len(enum spu_cipher_mode cipher_mode, + unsigned int assoc_len, unsigned int iv_len, + bool is_encrypt) +{ + u32 buflen = 0; + u32 pad; + + if (assoc_len) + buflen = assoc_len; + + if (cipher_mode == CIPHER_MODE_GCM) { + /* AAD needs to be padded in responses too */ + pad = spum_gcm_ccm_pad_len(cipher_mode, buflen); + buflen += pad; + } + if (cipher_mode == CIPHER_MODE_CCM) { + /* + * AAD needs to be padded in responses too + * for CCM, len + 2 needs to be 128-bit aligned. + */ + pad = spum_gcm_ccm_pad_len(cipher_mode, buflen + 2); + buflen += pad; + } + + return buflen; +} + +/** + * spum_aead_ivlen() - Calculate the length of the AEAD IV to be included + * in a SPU request after the AAD and before the payload. + * @cipher_mode: cipher mode + * @iv_len: initialization vector length in bytes + * + * In Linux ~4.2 and later, the assoc_data sg includes the IV. So no need + * to include the IV as a separate field in the SPU request msg. + * + * Return: Length of AEAD IV in bytes + */ +u8 spum_aead_ivlen(enum spu_cipher_mode cipher_mode, u16 iv_len) +{ + return 0; +} + +/** + * spum_hash_type() - Determine the type of hash operation. + * @src_sent: The number of bytes in the current request that have already + * been sent to the SPU to be hashed. + * + * We do not use HASH_TYPE_FULL for requests that fit in a single SPU message. + * Using FULL causes failures (such as when the string to be hashed is empty). + * For similar reasons, we never use HASH_TYPE_FIN. Instead, submit messages + * as INIT or UPDT and do the hash padding in sw. + */ +enum hash_type spum_hash_type(u32 src_sent) +{ + return src_sent ? HASH_TYPE_UPDT : HASH_TYPE_INIT; +} + +/** + * spum_digest_size() - Determine the size of a hash digest to expect the SPU to + * return. + * @alg_digest_size: Number of bytes in the final digest for the given algo + * @alg: The hash algorithm + * @htype: Type of hash operation (init, update, full, etc) + * + * When doing incremental hashing for an algorithm with a truncated hash + * (e.g., SHA224), the SPU returns the full digest so that it can be fed back as + * a partial result for the next chunk. + */ +u32 spum_digest_size(u32 alg_digest_size, enum hash_alg alg, + enum hash_type htype) +{ + u32 digestsize = alg_digest_size; + + /* SPU returns complete digest when doing incremental hash and truncated + * hash algo. + */ + if ((htype == HASH_TYPE_INIT) || (htype == HASH_TYPE_UPDT)) { + if (alg == HASH_ALG_SHA224) + digestsize = SHA256_DIGEST_SIZE; + else if (alg == HASH_ALG_SHA384) + digestsize = SHA512_DIGEST_SIZE; + } + return digestsize; +} + +/** + * spum_create_request() - Build a SPU request message header, up to and + * including the BD header. Construct the message starting at spu_hdr. Caller + * should allocate this buffer in DMA-able memory at least SPU_HEADER_ALLOC_LEN + * bytes long. + * @spu_hdr: Start of buffer where SPU request header is to be written + * @req_opts: SPU request message options + * @cipher_parms: Parameters related to cipher algorithm + * @hash_parms: Parameters related to hash algorithm + * @aead_parms: Parameters related to AEAD operation + * @data_size: Length of data to be encrypted or authenticated. If AEAD, does + * not include length of AAD. + * + * Return: the length of the SPU header in bytes. 0 if an error occurs. + */ +u32 spum_create_request(u8 *spu_hdr, + struct spu_request_opts *req_opts, + struct spu_cipher_parms *cipher_parms, + struct spu_hash_parms *hash_parms, + struct spu_aead_parms *aead_parms, + unsigned int data_size) +{ + struct SPUHEADER *spuh; + struct BDESC_HEADER *bdesc; + struct BD_HEADER *bd; + + u8 *ptr; + u32 protocol_bits = 0; + u32 cipher_bits = 0; + u32 ecf_bits = 0; + u8 sctx_words = 0; + unsigned int buf_len = 0; + + /* size of the cipher payload */ + unsigned int cipher_len = hash_parms->prebuf_len + data_size + + hash_parms->pad_len; + + /* offset of prebuf or data from end of BD header */ + unsigned int cipher_offset = aead_parms->assoc_size + + aead_parms->iv_len + aead_parms->aad_pad_len; + + /* total size of the DB data (without STAT word padding) */ + unsigned int real_db_size = spu_real_db_size(aead_parms->assoc_size, + aead_parms->iv_len, + hash_parms->prebuf_len, + data_size, + aead_parms->aad_pad_len, + aead_parms->data_pad_len, + hash_parms->pad_len); + + unsigned int auth_offset = 0; + unsigned int offset_iv = 0; + + /* size/offset of the auth payload */ + unsigned int auth_len; + + auth_len = real_db_size; + + if (req_opts->is_aead && req_opts->is_inbound) + cipher_len -= hash_parms->digestsize; + + if (req_opts->is_aead && req_opts->is_inbound) + auth_len -= hash_parms->digestsize; + + if ((hash_parms->alg == HASH_ALG_AES) && + (hash_parms->mode == HASH_MODE_XCBC)) { + auth_len -= hash_parms->pad_len; + cipher_len -= hash_parms->pad_len; + } + + flow_log("%s()\n", __func__); + flow_log(" in:%u authFirst:%u\n", + req_opts->is_inbound, req_opts->auth_first); + flow_log(" %s. cipher alg:%u mode:%u type %u\n", + spu_alg_name(cipher_parms->alg, cipher_parms->mode), + cipher_parms->alg, cipher_parms->mode, cipher_parms->type); + flow_log(" key: %d\n", cipher_parms->key_len); + flow_dump(" key: ", cipher_parms->key_buf, cipher_parms->key_len); + flow_log(" iv: %d\n", cipher_parms->iv_len); + flow_dump(" iv: ", cipher_parms->iv_buf, cipher_parms->iv_len); + flow_log(" auth alg:%u mode:%u type %u\n", + hash_parms->alg, hash_parms->mode, hash_parms->type); + flow_log(" digestsize: %u\n", hash_parms->digestsize); + flow_log(" authkey: %d\n", hash_parms->key_len); + flow_dump(" authkey: ", hash_parms->key_buf, hash_parms->key_len); + flow_log(" assoc_size:%u\n", aead_parms->assoc_size); + flow_log(" prebuf_len:%u\n", hash_parms->prebuf_len); + flow_log(" data_size:%u\n", data_size); + flow_log(" hash_pad_len:%u\n", hash_parms->pad_len); + flow_log(" real_db_size:%u\n", real_db_size); + flow_log(" auth_offset:%u auth_len:%u cipher_offset:%u cipher_len:%u\n", + auth_offset, auth_len, cipher_offset, cipher_len); + flow_log(" aead_iv: %u\n", aead_parms->iv_len); + + /* starting out: zero the header (plus some) */ + ptr = spu_hdr; + memset(ptr, 0, sizeof(struct SPUHEADER)); + + /* format master header word */ + /* Do not set the next bit even though the datasheet says to */ + spuh = (struct SPUHEADER *)ptr; + ptr += sizeof(struct SPUHEADER); + buf_len += sizeof(struct SPUHEADER); + + spuh->mh.op_code = SPU_CRYPTO_OPERATION_GENERIC; + spuh->mh.flags |= (MH_SCTX_PRES | MH_BDESC_PRES | MH_BD_PRES); + + /* Format sctx word 0 (protocol_bits) */ + sctx_words = 3; /* size in words */ + + /* Format sctx word 1 (cipher_bits) */ + if (req_opts->is_inbound) + cipher_bits |= CIPHER_INBOUND; + if (req_opts->auth_first) + cipher_bits |= CIPHER_ORDER; + + /* Set the crypto parameters in the cipher.flags */ + cipher_bits |= cipher_parms->alg << CIPHER_ALG_SHIFT; + cipher_bits |= cipher_parms->mode << CIPHER_MODE_SHIFT; + cipher_bits |= cipher_parms->type << CIPHER_TYPE_SHIFT; + + /* Set the auth parameters in the cipher.flags */ + cipher_bits |= hash_parms->alg << HASH_ALG_SHIFT; + cipher_bits |= hash_parms->mode << HASH_MODE_SHIFT; + cipher_bits |= hash_parms->type << HASH_TYPE_SHIFT; + + /* + * Format sctx extensions if required, and update main fields if + * required) + */ + if (hash_parms->alg) { + /* Write the authentication key material if present */ + if (hash_parms->key_len) { + memcpy(ptr, hash_parms->key_buf, hash_parms->key_len); + ptr += hash_parms->key_len; + buf_len += hash_parms->key_len; + sctx_words += hash_parms->key_len / 4; + } + + if ((cipher_parms->mode == CIPHER_MODE_GCM) || + (cipher_parms->mode == CIPHER_MODE_CCM)) + /* unpadded length */ + offset_iv = aead_parms->assoc_size; + + /* if GCM/CCM we need to write ICV into the payload */ + if (!req_opts->is_inbound) { + if ((cipher_parms->mode == CIPHER_MODE_GCM) || + (cipher_parms->mode == CIPHER_MODE_CCM)) + ecf_bits |= 1 << INSERT_ICV_SHIFT; + } else { + ecf_bits |= CHECK_ICV; + } + + /* Inform the SPU of the ICV size (in words) */ + if (hash_parms->digestsize == 64) + cipher_bits |= ICV_IS_512; + else + ecf_bits |= + (hash_parms->digestsize / 4) << ICV_SIZE_SHIFT; + } + + if (req_opts->bd_suppress) + ecf_bits |= BD_SUPPRESS; + + /* copy the encryption keys in the SAD entry */ + if (cipher_parms->alg) { + if (cipher_parms->key_len) { + memcpy(ptr, cipher_parms->key_buf, + cipher_parms->key_len); + ptr += cipher_parms->key_len; + buf_len += cipher_parms->key_len; + sctx_words += cipher_parms->key_len / 4; + } + + /* + * if encrypting then set IV size, use SCTX IV unless no IV + * given here + */ + if (cipher_parms->iv_buf && cipher_parms->iv_len) { + /* Use SCTX IV */ + ecf_bits |= SCTX_IV; + + /* cipher iv provided so put it in here */ + memcpy(ptr, cipher_parms->iv_buf, cipher_parms->iv_len); + + ptr += cipher_parms->iv_len; + buf_len += cipher_parms->iv_len; + sctx_words += cipher_parms->iv_len / 4; + } + } + + /* + * RFC4543 (GMAC/ESP) requires data to be sent as part of AAD + * so we need to override the BDESC parameters. + */ + if (req_opts->is_rfc4543) { + if (req_opts->is_inbound) + data_size -= hash_parms->digestsize; + offset_iv = aead_parms->assoc_size + data_size; + cipher_len = 0; + cipher_offset = offset_iv; + auth_len = cipher_offset + aead_parms->data_pad_len; + } + + /* write in the total sctx length now that we know it */ + protocol_bits |= sctx_words; + + /* Endian adjust the SCTX */ + spuh->sa.proto_flags = cpu_to_be32(protocol_bits); + spuh->sa.cipher_flags = cpu_to_be32(cipher_bits); + spuh->sa.ecf = cpu_to_be32(ecf_bits); + + /* === create the BDESC section === */ + bdesc = (struct BDESC_HEADER *)ptr; + + bdesc->offset_mac = cpu_to_be16(auth_offset); + bdesc->length_mac = cpu_to_be16(auth_len); + bdesc->offset_crypto = cpu_to_be16(cipher_offset); + bdesc->length_crypto = cpu_to_be16(cipher_len); + + /* + * CCM in SPU-M requires that ICV not be in same 32-bit word as data or + * padding. So account for padding as necessary. + */ + if (cipher_parms->mode == CIPHER_MODE_CCM) + auth_len += spum_wordalign_padlen(auth_len); + + bdesc->offset_icv = cpu_to_be16(auth_len); + bdesc->offset_iv = cpu_to_be16(offset_iv); + + ptr += sizeof(struct BDESC_HEADER); + buf_len += sizeof(struct BDESC_HEADER); + + /* === no MFM section === */ + + /* === create the BD section === */ + + /* add the BD header */ + bd = (struct BD_HEADER *)ptr; + bd->size = cpu_to_be16(real_db_size); + bd->prev_length = 0; + + ptr += sizeof(struct BD_HEADER); + buf_len += sizeof(struct BD_HEADER); + + packet_dump(" SPU request header: ", spu_hdr, buf_len); + + return buf_len; +} + +/** + * spum_cipher_req_init() - Build a SPU request message header, up to and + * including the BD header. + * @spu_hdr: Start of SPU request header (MH) + * @cipher_parms: Parameters that describe the cipher request + * + * Construct the message starting at spu_hdr. Caller should allocate this buffer + * in DMA-able memory at least SPU_HEADER_ALLOC_LEN bytes long. + * + * Return: the length of the SPU header in bytes. 0 if an error occurs. + */ +u16 spum_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms) +{ + struct SPUHEADER *spuh; + u32 protocol_bits = 0; + u32 cipher_bits = 0; + u32 ecf_bits = 0; + u8 sctx_words = 0; + + flow_log("%s()\n", __func__); + flow_log(" cipher alg:%u mode:%u type %u\n", cipher_parms->alg, + cipher_parms->mode, cipher_parms->type); + flow_log(" cipher_iv_len: %u\n", cipher_parms->iv_len); + flow_log(" key: %d\n", cipher_parms->key_len); + flow_dump(" key: ", cipher_parms->key_buf, cipher_parms->key_len); + + /* starting out: zero the header (plus some) */ + memset(spu_hdr, 0, sizeof(struct SPUHEADER)); + + /* format master header word */ + /* Do not set the next bit even though the datasheet says to */ + spuh = (struct SPUHEADER *)spu_hdr; + + spuh->mh.op_code = SPU_CRYPTO_OPERATION_GENERIC; + spuh->mh.flags |= (MH_SCTX_PRES | MH_BDESC_PRES | MH_BD_PRES); + + /* Format sctx word 0 (protocol_bits) */ + sctx_words = 3; /* size in words */ + + /* copy the encryption keys in the SAD entry */ + if (cipher_parms->alg) { + if (cipher_parms->key_len) + sctx_words += cipher_parms->key_len / 4; + + /* + * if encrypting then set IV size, use SCTX IV unless no IV + * given here + */ + if (cipher_parms->iv_len) { + /* Use SCTX IV */ + ecf_bits |= SCTX_IV; + sctx_words += cipher_parms->iv_len / 4; + } + } + + /* Set the crypto parameters in the cipher.flags */ + cipher_bits |= cipher_parms->alg << CIPHER_ALG_SHIFT; + cipher_bits |= cipher_parms->mode << CIPHER_MODE_SHIFT; + cipher_bits |= cipher_parms->type << CIPHER_TYPE_SHIFT; + + /* copy the encryption keys in the SAD entry */ + if (cipher_parms->alg && cipher_parms->key_len) + memcpy(spuh + 1, cipher_parms->key_buf, cipher_parms->key_len); + + /* write in the total sctx length now that we know it */ + protocol_bits |= sctx_words; + + /* Endian adjust the SCTX */ + spuh->sa.proto_flags = cpu_to_be32(protocol_bits); + + /* Endian adjust the SCTX */ + spuh->sa.cipher_flags = cpu_to_be32(cipher_bits); + spuh->sa.ecf = cpu_to_be32(ecf_bits); + + packet_dump(" SPU request header: ", spu_hdr, + sizeof(struct SPUHEADER)); + + return sizeof(struct SPUHEADER) + cipher_parms->key_len + + cipher_parms->iv_len + sizeof(struct BDESC_HEADER) + + sizeof(struct BD_HEADER); +} + +/** + * spum_cipher_req_finish() - Finish building a SPU request message header for a + * block cipher request. Assumes much of the header was already filled in at + * setkey() time in spu_cipher_req_init(). + * @spu_hdr: Start of the request message header (MH field) + * @spu_req_hdr_len: Length in bytes of the SPU request header + * @is_inbound: 0 encrypt, 1 decrypt + * @cipher_parms: Parameters describing cipher operation to be performed + * @data_size: Length of the data in the BD field + * + * Assumes much of the header was already filled in at setkey() time in + * spum_cipher_req_init(). + * spum_cipher_req_init() fills in the encryption key. + */ +void spum_cipher_req_finish(u8 *spu_hdr, + u16 spu_req_hdr_len, + unsigned int is_inbound, + struct spu_cipher_parms *cipher_parms, + unsigned int data_size) +{ + struct SPUHEADER *spuh; + struct BDESC_HEADER *bdesc; + struct BD_HEADER *bd; + u8 *bdesc_ptr = spu_hdr + spu_req_hdr_len - + (sizeof(struct BD_HEADER) + sizeof(struct BDESC_HEADER)); + + u32 cipher_bits; + + flow_log("%s()\n", __func__); + flow_log(" in: %u\n", is_inbound); + flow_log(" cipher alg: %u, cipher_type: %u\n", cipher_parms->alg, + cipher_parms->type); + + /* + * In XTS mode, API puts "i" parameter (block tweak) in IV. For + * SPU-M, should be in start of the BD; tx_sg_create() copies it there. + * IV in SPU msg for SPU-M should be 0, since that's the "j" parameter + * (block ctr within larger data unit) - given we can send entire disk + * block (<= 4KB) in 1 SPU msg, don't need to use this parameter. + */ + if (cipher_parms->mode == CIPHER_MODE_XTS) + memset(cipher_parms->iv_buf, 0, cipher_parms->iv_len); + + flow_log(" iv len: %d\n", cipher_parms->iv_len); + flow_dump(" iv: ", cipher_parms->iv_buf, cipher_parms->iv_len); + flow_log(" data_size: %u\n", data_size); + + /* format master header word */ + /* Do not set the next bit even though the datasheet says to */ + spuh = (struct SPUHEADER *)spu_hdr; + + /* cipher_bits was initialized at setkey time */ + cipher_bits = be32_to_cpu(spuh->sa.cipher_flags); + + /* Format sctx word 1 (cipher_bits) */ + if (is_inbound) + cipher_bits |= CIPHER_INBOUND; + else + cipher_bits &= ~CIPHER_INBOUND; + + if (cipher_parms->alg && cipher_parms->iv_buf && cipher_parms->iv_len) + /* cipher iv provided so put it in here */ + memcpy(bdesc_ptr - cipher_parms->iv_len, cipher_parms->iv_buf, + cipher_parms->iv_len); + + spuh->sa.cipher_flags = cpu_to_be32(cipher_bits); + + /* === create the BDESC section === */ + bdesc = (struct BDESC_HEADER *)bdesc_ptr; + bdesc->offset_mac = 0; + bdesc->length_mac = 0; + bdesc->offset_crypto = 0; + + /* XTS mode, data_size needs to include tweak parameter */ + if (cipher_parms->mode == CIPHER_MODE_XTS) + bdesc->length_crypto = cpu_to_be16(data_size + + SPU_XTS_TWEAK_SIZE); + else + bdesc->length_crypto = cpu_to_be16(data_size); + + bdesc->offset_icv = 0; + bdesc->offset_iv = 0; + + /* === no MFM section === */ + + /* === create the BD section === */ + /* add the BD header */ + bd = (struct BD_HEADER *)(bdesc_ptr + sizeof(struct BDESC_HEADER)); + bd->size = cpu_to_be16(data_size); + + /* XTS mode, data_size needs to include tweak parameter */ + if (cipher_parms->mode == CIPHER_MODE_XTS) + bd->size = cpu_to_be16(data_size + SPU_XTS_TWEAK_SIZE); + else + bd->size = cpu_to_be16(data_size); + + bd->prev_length = 0; + + packet_dump(" SPU request header: ", spu_hdr, spu_req_hdr_len); +} + +/** + * spum_request_pad() - Create pad bytes at the end of the data. + * @pad_start: Start of buffer where pad bytes are to be written + * @gcm_ccm_padding: length of GCM/CCM padding, in bytes + * @hash_pad_len: Number of bytes of padding extend data to full block + * @auth_alg: authentication algorithm + * @auth_mode: authentication mode + * @total_sent: length inserted at end of hash pad + * @status_padding: Number of bytes of padding to align STATUS word + * + * There may be three forms of pad: + * 1. GCM/CCM pad - for GCM/CCM mode ciphers, pad to 16-byte alignment + * 2. hash pad - pad to a block length, with 0x80 data terminator and + * size at the end + * 3. STAT pad - to ensure the STAT field is 4-byte aligned + */ +void spum_request_pad(u8 *pad_start, + u32 gcm_ccm_padding, + u32 hash_pad_len, + enum hash_alg auth_alg, + enum hash_mode auth_mode, + unsigned int total_sent, u32 status_padding) +{ + u8 *ptr = pad_start; + + /* fix data alignent for GCM/CCM */ + if (gcm_ccm_padding > 0) { + flow_log(" GCM: padding to 16 byte alignment: %u bytes\n", + gcm_ccm_padding); + memset(ptr, 0, gcm_ccm_padding); + ptr += gcm_ccm_padding; + } + + if (hash_pad_len > 0) { + /* clear the padding section */ + memset(ptr, 0, hash_pad_len); + + if ((auth_alg == HASH_ALG_AES) && + (auth_mode == HASH_MODE_XCBC)) { + /* AES/XCBC just requires padding to be 0s */ + ptr += hash_pad_len; + } else { + /* terminate the data */ + *ptr = 0x80; + ptr += (hash_pad_len - sizeof(u64)); + + /* add the size at the end as required per alg */ + if (auth_alg == HASH_ALG_MD5) + *(__le64 *)ptr = cpu_to_le64(total_sent * 8ull); + else /* SHA1, SHA2-224, SHA2-256 */ + *(__be64 *)ptr = cpu_to_be64(total_sent * 8ull); + ptr += sizeof(u64); + } + } + + /* pad to a 4byte alignment for STAT */ + if (status_padding > 0) { + flow_log(" STAT: padding to 4 byte alignment: %u bytes\n", + status_padding); + + memset(ptr, 0, status_padding); + ptr += status_padding; + } +} + +/** + * spum_xts_tweak_in_payload() - Indicate that SPUM DOES place the XTS tweak + * field in the packet payload (rather than using IV) + * + * Return: 1 + */ +u8 spum_xts_tweak_in_payload(void) +{ + return 1; +} + +/** + * spum_tx_status_len() - Return the length of the STATUS field in a SPU + * response message. + * + * Return: Length of STATUS field in bytes. + */ +u8 spum_tx_status_len(void) +{ + return SPU_TX_STATUS_LEN; +} + +/** + * spum_rx_status_len() - Return the length of the STATUS field in a SPU + * response message. + * + * Return: Length of STATUS field in bytes. + */ +u8 spum_rx_status_len(void) +{ + return SPU_RX_STATUS_LEN; +} + +/** + * spum_status_process() - Process the status from a SPU response message. + * @statp: start of STATUS word + * Return: + * 0 - if status is good and response should be processed + * !0 - status indicates an error and response is invalid + */ +int spum_status_process(u8 *statp) +{ + u32 status; + + status = __be32_to_cpu(*(__be32 *)statp); + flow_log("SPU response STATUS %#08x\n", status); + if (status & SPU_STATUS_ERROR_FLAG) { + pr_err("%s() Warning: Error result from SPU: %#08x\n", + __func__, status); + if (status & SPU_STATUS_INVALID_ICV) + return SPU_INVALID_ICV; + return -EBADMSG; + } + return 0; +} + +/** + * spum_ccm_update_iv() - Update the IV as per the requirements for CCM mode. + * + * @digestsize: Digest size of this request + * @cipher_parms: (pointer to) cipher parmaeters, includes IV buf & IV len + * @assoclen: Length of AAD data + * @chunksize: length of input data to be sent in this req + * @is_encrypt: true if this is an output/encrypt operation + * @is_esp: true if this is an ESP / RFC4309 operation + * + */ +void spum_ccm_update_iv(unsigned int digestsize, + struct spu_cipher_parms *cipher_parms, + unsigned int assoclen, + unsigned int chunksize, + bool is_encrypt, + bool is_esp) +{ + u8 L; /* L from CCM algorithm, length of plaintext data */ + u8 mprime; /* M' from CCM algo, (M - 2) / 2, where M=authsize */ + u8 adata; + + if (cipher_parms->iv_len != CCM_AES_IV_SIZE) { + pr_err("%s(): Invalid IV len %d for CCM mode, should be %d\n", + __func__, cipher_parms->iv_len, CCM_AES_IV_SIZE); + return; + } + + /* + * IV needs to be formatted as follows: + * + * | Byte 0 | Bytes 1 - N | Bytes (N+1) - 15 | + * | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | Bits 7 - 0 | Bits 7 - 0 | + * | 0 |Ad?|(M - 2) / 2| L - 1 | Nonce | Plaintext Length | + * + * Ad? = 1 if AAD present, 0 if not present + * M = size of auth field, 8, 12, or 16 bytes (SPU-M) -or- + * 4, 6, 8, 10, 12, 14, 16 bytes (SPU2) + * L = Size of Plaintext Length field; Nonce size = 15 - L + * + * It appears that the crypto API already expects the L-1 portion + * to be set in the first byte of the IV, which implicitly determines + * the nonce size, and also fills in the nonce. But the other bits + * in byte 0 as well as the plaintext length need to be filled in. + * + * In rfc4309/esp mode, L is not already in the supplied IV and + * we need to fill it in, as well as move the IV data to be after + * the salt + */ + if (is_esp) { + L = CCM_ESP_L_VALUE; /* RFC4309 has fixed L */ + } else { + /* L' = plaintext length - 1 so Plaintext length is L' + 1 */ + L = ((cipher_parms->iv_buf[0] & CCM_B0_L_PRIME) >> + CCM_B0_L_PRIME_SHIFT) + 1; + } + + mprime = (digestsize - 2) >> 1; /* M' = (M - 2) / 2 */ + adata = (assoclen > 0); /* adata = 1 if any associated data */ + + cipher_parms->iv_buf[0] = (adata << CCM_B0_ADATA_SHIFT) | + (mprime << CCM_B0_M_PRIME_SHIFT) | + ((L - 1) << CCM_B0_L_PRIME_SHIFT); + + /* Nonce is already filled in by crypto API, and is 15 - L bytes */ + + /* Don't include digest in plaintext size when decrypting */ + if (!is_encrypt) + chunksize -= digestsize; + + /* Fill in length of plaintext, formatted to be L bytes long */ + format_value_ccm(chunksize, &cipher_parms->iv_buf[15 - L + 1], L); +} + +/** + * spum_wordalign_padlen() - Given the length of a data field, determine the + * padding required to align the data following this field on a 4-byte boundary. + * @data_size: length of data field in bytes + * + * Return: length of status field padding, in bytes + */ +u32 spum_wordalign_padlen(u32 data_size) +{ + return ((data_size + 3) & ~3) - data_size; +} diff --git a/drivers/crypto/bcm/spu.h b/drivers/crypto/bcm/spu.h new file mode 100644 index 000000000..1c386a2d5 --- /dev/null +++ b/drivers/crypto/bcm/spu.h @@ -0,0 +1,276 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2016 Broadcom + */ + +/* + * This file contains the definition of SPU messages. There are currently two + * SPU message formats: SPU-M and SPU2. The hardware uses different values to + * identify the same things in SPU-M vs SPU2. So this file defines values that + * are hardware independent. Software can use these values for any version of + * SPU hardware. These values are used in APIs in spu.c. Functions internal to + * spu.c and spu2.c convert these to hardware-specific values. + */ + +#ifndef _SPU_H +#define _SPU_H + +#include +#include +#include +#include + +enum spu_cipher_alg { + CIPHER_ALG_NONE = 0x0, + CIPHER_ALG_RC4 = 0x1, + CIPHER_ALG_DES = 0x2, + CIPHER_ALG_3DES = 0x3, + CIPHER_ALG_AES = 0x4, + CIPHER_ALG_LAST = 0x5 +}; + +enum spu_cipher_mode { + CIPHER_MODE_NONE = 0x0, + CIPHER_MODE_ECB = 0x0, + CIPHER_MODE_CBC = 0x1, + CIPHER_MODE_OFB = 0x2, + CIPHER_MODE_CFB = 0x3, + CIPHER_MODE_CTR = 0x4, + CIPHER_MODE_CCM = 0x5, + CIPHER_MODE_GCM = 0x6, + CIPHER_MODE_XTS = 0x7, + CIPHER_MODE_LAST = 0x8 +}; + +enum spu_cipher_type { + CIPHER_TYPE_NONE = 0x0, + CIPHER_TYPE_DES = 0x0, + CIPHER_TYPE_3DES = 0x0, + CIPHER_TYPE_INIT = 0x0, /* used for ARC4 */ + CIPHER_TYPE_AES128 = 0x0, + CIPHER_TYPE_AES192 = 0x1, + CIPHER_TYPE_UPDT = 0x1, /* used for ARC4 */ + CIPHER_TYPE_AES256 = 0x2, +}; + +enum hash_alg { + HASH_ALG_NONE = 0x0, + HASH_ALG_MD5 = 0x1, + HASH_ALG_SHA1 = 0x2, + HASH_ALG_SHA224 = 0x3, + HASH_ALG_SHA256 = 0x4, + HASH_ALG_AES = 0x5, + HASH_ALG_SHA384 = 0x6, + HASH_ALG_SHA512 = 0x7, + /* Keep SHA3 algorithms at the end always */ + HASH_ALG_SHA3_224 = 0x8, + HASH_ALG_SHA3_256 = 0x9, + HASH_ALG_SHA3_384 = 0xa, + HASH_ALG_SHA3_512 = 0xb, + HASH_ALG_LAST +}; + +enum hash_mode { + HASH_MODE_NONE = 0x0, + HASH_MODE_HASH = 0x0, + HASH_MODE_XCBC = 0x0, + HASH_MODE_CMAC = 0x1, + HASH_MODE_CTXT = 0x1, + HASH_MODE_HMAC = 0x2, + HASH_MODE_RABIN = 0x4, + HASH_MODE_FHMAC = 0x6, + HASH_MODE_CCM = 0x5, + HASH_MODE_GCM = 0x6, +}; + +enum hash_type { + HASH_TYPE_NONE = 0x0, + HASH_TYPE_FULL = 0x0, + HASH_TYPE_INIT = 0x1, + HASH_TYPE_UPDT = 0x2, + HASH_TYPE_FIN = 0x3, + HASH_TYPE_AES128 = 0x0, + HASH_TYPE_AES192 = 0x1, + HASH_TYPE_AES256 = 0x2 +}; + +enum aead_type { + AES_CCM, + AES_GCM, + AUTHENC, + AEAD_TYPE_LAST +}; + +extern char *hash_alg_name[HASH_ALG_LAST]; +extern char *aead_alg_name[AEAD_TYPE_LAST]; + +struct spu_request_opts { + bool is_inbound; + bool auth_first; + bool is_aead; + bool is_esp; + bool bd_suppress; + bool is_rfc4543; +}; + +struct spu_cipher_parms { + enum spu_cipher_alg alg; + enum spu_cipher_mode mode; + enum spu_cipher_type type; + u8 *key_buf; + u16 key_len; + /* iv_buf and iv_len include salt, if applicable */ + u8 *iv_buf; + u16 iv_len; +}; + +struct spu_hash_parms { + enum hash_alg alg; + enum hash_mode mode; + enum hash_type type; + u8 digestsize; + u8 *key_buf; + u16 key_len; + u16 prebuf_len; + /* length of hash pad. signed, needs to handle roll-overs */ + int pad_len; +}; + +struct spu_aead_parms { + u32 assoc_size; + u16 iv_len; /* length of IV field between assoc data and data */ + u8 aad_pad_len; /* For AES GCM/CCM, length of padding after AAD */ + u8 data_pad_len;/* For AES GCM/CCM, length of padding after data */ + bool return_iv; /* True if SPU should return an IV */ + u32 ret_iv_len; /* Length in bytes of returned IV */ + u32 ret_iv_off; /* Offset into full IV if partial IV returned */ +}; + +/************** SPU sizes ***************/ + +#define SPU_RX_STATUS_LEN 4 + +/* Max length of padding for 4-byte alignment of STATUS field */ +#define SPU_STAT_PAD_MAX 4 + +/* Max length of pad fragment. 4 is for 4-byte alignment of STATUS field */ +#define SPU_PAD_LEN_MAX (SPU_GCM_CCM_ALIGN + MAX_HASH_BLOCK_SIZE + \ + SPU_STAT_PAD_MAX) + +/* GCM and CCM require 16-byte alignment */ +#define SPU_GCM_CCM_ALIGN 16 + +/* Length up SUPDT field in SPU response message for RC4 */ +#define SPU_SUPDT_LEN 260 + +/* SPU status error codes. These used as common error codes across all + * SPU variants. + */ +#define SPU_INVALID_ICV 1 + +/* Indicates no limit to the length of the payload in a SPU message */ +#define SPU_MAX_PAYLOAD_INF 0xFFFFFFFF + +/* Size of XTS tweak ("i" parameter), in bytes */ +#define SPU_XTS_TWEAK_SIZE 16 + +/* CCM B_0 field definitions, common for SPU-M and SPU2 */ +#define CCM_B0_ADATA 0x40 +#define CCM_B0_ADATA_SHIFT 6 +#define CCM_B0_M_PRIME 0x38 +#define CCM_B0_M_PRIME_SHIFT 3 +#define CCM_B0_L_PRIME 0x07 +#define CCM_B0_L_PRIME_SHIFT 0 +#define CCM_ESP_L_VALUE 4 + +/** + * spu_req_incl_icv() - Return true if SPU request message should include the + * ICV as a separate buffer. + * @cipher_mode: the cipher mode being requested + * @is_encrypt: true if encrypting. false if decrypting. + * + * Return: true if ICV to be included as separate buffer + */ +static __always_inline bool spu_req_incl_icv(enum spu_cipher_mode cipher_mode, + bool is_encrypt) +{ + if ((cipher_mode == CIPHER_MODE_GCM) && !is_encrypt) + return true; + if ((cipher_mode == CIPHER_MODE_CCM) && !is_encrypt) + return true; + + return false; +} + +static __always_inline u32 spu_real_db_size(u32 assoc_size, + u32 aead_iv_buf_len, + u32 prebuf_len, + u32 data_size, + u32 aad_pad_len, + u32 gcm_pad_len, + u32 hash_pad_len) +{ + return assoc_size + aead_iv_buf_len + prebuf_len + data_size + + aad_pad_len + gcm_pad_len + hash_pad_len; +} + +/************** SPU Functions Prototypes **************/ + +void spum_dump_msg_hdr(u8 *buf, unsigned int buf_len); + +u32 spum_ns2_ctx_max_payload(enum spu_cipher_alg cipher_alg, + enum spu_cipher_mode cipher_mode, + unsigned int blocksize); +u32 spum_nsp_ctx_max_payload(enum spu_cipher_alg cipher_alg, + enum spu_cipher_mode cipher_mode, + unsigned int blocksize); +u32 spum_payload_length(u8 *spu_hdr); +u16 spum_response_hdr_len(u16 auth_key_len, u16 enc_key_len, bool is_hash); +u16 spum_hash_pad_len(enum hash_alg hash_alg, enum hash_mode hash_mode, + u32 chunksize, u16 hash_block_size); +u32 spum_gcm_ccm_pad_len(enum spu_cipher_mode cipher_mode, + unsigned int data_size); +u32 spum_assoc_resp_len(enum spu_cipher_mode cipher_mode, + unsigned int assoc_len, unsigned int iv_len, + bool is_encrypt); +u8 spum_aead_ivlen(enum spu_cipher_mode cipher_mode, u16 iv_len); +bool spu_req_incl_icv(enum spu_cipher_mode cipher_mode, bool is_encrypt); +enum hash_type spum_hash_type(u32 src_sent); +u32 spum_digest_size(u32 alg_digest_size, enum hash_alg alg, + enum hash_type htype); + +u32 spum_create_request(u8 *spu_hdr, + struct spu_request_opts *req_opts, + struct spu_cipher_parms *cipher_parms, + struct spu_hash_parms *hash_parms, + struct spu_aead_parms *aead_parms, + unsigned int data_size); + +u16 spum_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms); + +void spum_cipher_req_finish(u8 *spu_hdr, + u16 spu_req_hdr_len, + unsigned int is_inbound, + struct spu_cipher_parms *cipher_parms, + unsigned int data_size); + +void spum_request_pad(u8 *pad_start, + u32 gcm_padding, + u32 hash_pad_len, + enum hash_alg auth_alg, + enum hash_mode auth_mode, + unsigned int total_sent, u32 status_padding); + +u8 spum_xts_tweak_in_payload(void); +u8 spum_tx_status_len(void); +u8 spum_rx_status_len(void); +int spum_status_process(u8 *statp); + +void spum_ccm_update_iv(unsigned int digestsize, + struct spu_cipher_parms *cipher_parms, + unsigned int assoclen, + unsigned int chunksize, + bool is_encrypt, + bool is_esp); +u32 spum_wordalign_padlen(u32 data_size); +#endif diff --git a/drivers/crypto/bcm/spu2.c b/drivers/crypto/bcm/spu2.c new file mode 100644 index 000000000..ce322cf1b --- /dev/null +++ b/drivers/crypto/bcm/spu2.c @@ -0,0 +1,1385 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2016 Broadcom + */ + +/* + * This file works with the SPU2 version of the SPU. SPU2 has different message + * formats than the previous version of the SPU. All SPU message format + * differences should be hidden in the spux.c,h files. + */ + +#include +#include +#include + +#include "util.h" +#include "spu.h" +#include "spu2.h" + +#define SPU2_TX_STATUS_LEN 0 /* SPU2 has no STATUS in input packet */ + +/* + * Controlled by pkt_stat_cnt field in CRYPTO_SS_SPU0_CORE_SPU2_CONTROL0 + * register. Defaults to 2. + */ +#define SPU2_RX_STATUS_LEN 2 + +enum spu2_proto_sel { + SPU2_PROTO_RESV = 0, + SPU2_MACSEC_SECTAG8_ECB = 1, + SPU2_MACSEC_SECTAG8_SCB = 2, + SPU2_MACSEC_SECTAG16 = 3, + SPU2_MACSEC_SECTAG16_8_XPN = 4, + SPU2_IPSEC = 5, + SPU2_IPSEC_ESN = 6, + SPU2_TLS_CIPHER = 7, + SPU2_TLS_AEAD = 8, + SPU2_DTLS_CIPHER = 9, + SPU2_DTLS_AEAD = 10 +}; + +static char *spu2_cipher_type_names[] = { "None", "AES128", "AES192", "AES256", + "DES", "3DES" +}; + +static char *spu2_cipher_mode_names[] = { "ECB", "CBC", "CTR", "CFB", "OFB", + "XTS", "CCM", "GCM" +}; + +static char *spu2_hash_type_names[] = { "None", "AES128", "AES192", "AES256", + "Reserved", "Reserved", "MD5", "SHA1", "SHA224", "SHA256", "SHA384", + "SHA512", "SHA512/224", "SHA512/256", "SHA3-224", "SHA3-256", + "SHA3-384", "SHA3-512" +}; + +static char *spu2_hash_mode_names[] = { "CMAC", "CBC-MAC", "XCBC-MAC", "HMAC", + "Rabin", "CCM", "GCM", "Reserved" +}; + +static char *spu2_ciph_type_name(enum spu2_cipher_type cipher_type) +{ + if (cipher_type >= SPU2_CIPHER_TYPE_LAST) + return "Reserved"; + return spu2_cipher_type_names[cipher_type]; +} + +static char *spu2_ciph_mode_name(enum spu2_cipher_mode cipher_mode) +{ + if (cipher_mode >= SPU2_CIPHER_MODE_LAST) + return "Reserved"; + return spu2_cipher_mode_names[cipher_mode]; +} + +static char *spu2_hash_type_name(enum spu2_hash_type hash_type) +{ + if (hash_type >= SPU2_HASH_TYPE_LAST) + return "Reserved"; + return spu2_hash_type_names[hash_type]; +} + +static char *spu2_hash_mode_name(enum spu2_hash_mode hash_mode) +{ + if (hash_mode >= SPU2_HASH_MODE_LAST) + return "Reserved"; + return spu2_hash_mode_names[hash_mode]; +} + +/* + * Convert from a software cipher mode value to the corresponding value + * for SPU2. + */ +static int spu2_cipher_mode_xlate(enum spu_cipher_mode cipher_mode, + enum spu2_cipher_mode *spu2_mode) +{ + switch (cipher_mode) { + case CIPHER_MODE_ECB: + *spu2_mode = SPU2_CIPHER_MODE_ECB; + break; + case CIPHER_MODE_CBC: + *spu2_mode = SPU2_CIPHER_MODE_CBC; + break; + case CIPHER_MODE_OFB: + *spu2_mode = SPU2_CIPHER_MODE_OFB; + break; + case CIPHER_MODE_CFB: + *spu2_mode = SPU2_CIPHER_MODE_CFB; + break; + case CIPHER_MODE_CTR: + *spu2_mode = SPU2_CIPHER_MODE_CTR; + break; + case CIPHER_MODE_CCM: + *spu2_mode = SPU2_CIPHER_MODE_CCM; + break; + case CIPHER_MODE_GCM: + *spu2_mode = SPU2_CIPHER_MODE_GCM; + break; + case CIPHER_MODE_XTS: + *spu2_mode = SPU2_CIPHER_MODE_XTS; + break; + default: + return -EINVAL; + } + return 0; +} + +/** + * spu2_cipher_xlate() - Convert a cipher {alg/mode/type} triple to a SPU2 + * cipher type and mode. + * @cipher_alg: [in] cipher algorithm value from software enumeration + * @cipher_mode: [in] cipher mode value from software enumeration + * @cipher_type: [in] cipher type value from software enumeration + * @spu2_type: [out] cipher type value used by spu2 hardware + * @spu2_mode: [out] cipher mode value used by spu2 hardware + * + * Return: 0 if successful + */ +static int spu2_cipher_xlate(enum spu_cipher_alg cipher_alg, + enum spu_cipher_mode cipher_mode, + enum spu_cipher_type cipher_type, + enum spu2_cipher_type *spu2_type, + enum spu2_cipher_mode *spu2_mode) +{ + int err; + + err = spu2_cipher_mode_xlate(cipher_mode, spu2_mode); + if (err) { + flow_log("Invalid cipher mode %d\n", cipher_mode); + return err; + } + + switch (cipher_alg) { + case CIPHER_ALG_NONE: + *spu2_type = SPU2_CIPHER_TYPE_NONE; + break; + case CIPHER_ALG_RC4: + /* SPU2 does not support RC4 */ + err = -EINVAL; + *spu2_type = SPU2_CIPHER_TYPE_NONE; + break; + case CIPHER_ALG_DES: + *spu2_type = SPU2_CIPHER_TYPE_DES; + break; + case CIPHER_ALG_3DES: + *spu2_type = SPU2_CIPHER_TYPE_3DES; + break; + case CIPHER_ALG_AES: + switch (cipher_type) { + case CIPHER_TYPE_AES128: + *spu2_type = SPU2_CIPHER_TYPE_AES128; + break; + case CIPHER_TYPE_AES192: + *spu2_type = SPU2_CIPHER_TYPE_AES192; + break; + case CIPHER_TYPE_AES256: + *spu2_type = SPU2_CIPHER_TYPE_AES256; + break; + default: + err = -EINVAL; + } + break; + case CIPHER_ALG_LAST: + default: + err = -EINVAL; + break; + } + + if (err) + flow_log("Invalid cipher alg %d or type %d\n", + cipher_alg, cipher_type); + return err; +} + +/* + * Convert from a software hash mode value to the corresponding value + * for SPU2. Note that HASH_MODE_NONE and HASH_MODE_XCBC have the same value. + */ +static int spu2_hash_mode_xlate(enum hash_mode hash_mode, + enum spu2_hash_mode *spu2_mode) +{ + switch (hash_mode) { + case HASH_MODE_XCBC: + *spu2_mode = SPU2_HASH_MODE_XCBC_MAC; + break; + case HASH_MODE_CMAC: + *spu2_mode = SPU2_HASH_MODE_CMAC; + break; + case HASH_MODE_HMAC: + *spu2_mode = SPU2_HASH_MODE_HMAC; + break; + case HASH_MODE_CCM: + *spu2_mode = SPU2_HASH_MODE_CCM; + break; + case HASH_MODE_GCM: + *spu2_mode = SPU2_HASH_MODE_GCM; + break; + default: + return -EINVAL; + } + return 0; +} + +/** + * spu2_hash_xlate() - Convert a hash {alg/mode/type} triple to a SPU2 hash type + * and mode. + * @hash_alg: [in] hash algorithm value from software enumeration + * @hash_mode: [in] hash mode value from software enumeration + * @hash_type: [in] hash type value from software enumeration + * @ciph_type: [in] cipher type value from software enumeration + * @spu2_type: [out] hash type value used by SPU2 hardware + * @spu2_mode: [out] hash mode value used by SPU2 hardware + * + * Return: 0 if successful + */ +static int +spu2_hash_xlate(enum hash_alg hash_alg, enum hash_mode hash_mode, + enum hash_type hash_type, enum spu_cipher_type ciph_type, + enum spu2_hash_type *spu2_type, enum spu2_hash_mode *spu2_mode) +{ + int err; + + err = spu2_hash_mode_xlate(hash_mode, spu2_mode); + if (err) { + flow_log("Invalid hash mode %d\n", hash_mode); + return err; + } + + switch (hash_alg) { + case HASH_ALG_NONE: + *spu2_type = SPU2_HASH_TYPE_NONE; + break; + case HASH_ALG_MD5: + *spu2_type = SPU2_HASH_TYPE_MD5; + break; + case HASH_ALG_SHA1: + *spu2_type = SPU2_HASH_TYPE_SHA1; + break; + case HASH_ALG_SHA224: + *spu2_type = SPU2_HASH_TYPE_SHA224; + break; + case HASH_ALG_SHA256: + *spu2_type = SPU2_HASH_TYPE_SHA256; + break; + case HASH_ALG_SHA384: + *spu2_type = SPU2_HASH_TYPE_SHA384; + break; + case HASH_ALG_SHA512: + *spu2_type = SPU2_HASH_TYPE_SHA512; + break; + case HASH_ALG_AES: + switch (ciph_type) { + case CIPHER_TYPE_AES128: + *spu2_type = SPU2_HASH_TYPE_AES128; + break; + case CIPHER_TYPE_AES192: + *spu2_type = SPU2_HASH_TYPE_AES192; + break; + case CIPHER_TYPE_AES256: + *spu2_type = SPU2_HASH_TYPE_AES256; + break; + default: + err = -EINVAL; + } + break; + case HASH_ALG_SHA3_224: + *spu2_type = SPU2_HASH_TYPE_SHA3_224; + break; + case HASH_ALG_SHA3_256: + *spu2_type = SPU2_HASH_TYPE_SHA3_256; + break; + case HASH_ALG_SHA3_384: + *spu2_type = SPU2_HASH_TYPE_SHA3_384; + break; + case HASH_ALG_SHA3_512: + *spu2_type = SPU2_HASH_TYPE_SHA3_512; + break; + case HASH_ALG_LAST: + default: + err = -EINVAL; + break; + } + + if (err) + flow_log("Invalid hash alg %d or type %d\n", + hash_alg, hash_type); + return err; +} + +/* Dump FMD ctrl0. The ctrl0 input is in host byte order */ +static void spu2_dump_fmd_ctrl0(u64 ctrl0) +{ + enum spu2_cipher_type ciph_type; + enum spu2_cipher_mode ciph_mode; + enum spu2_hash_type hash_type; + enum spu2_hash_mode hash_mode; + char *ciph_name; + char *ciph_mode_name; + char *hash_name; + char *hash_mode_name; + u8 cfb; + u8 proto; + + packet_log(" FMD CTRL0 %#16llx\n", ctrl0); + if (ctrl0 & SPU2_CIPH_ENCRYPT_EN) + packet_log(" encrypt\n"); + else + packet_log(" decrypt\n"); + + ciph_type = (ctrl0 & SPU2_CIPH_TYPE) >> SPU2_CIPH_TYPE_SHIFT; + ciph_name = spu2_ciph_type_name(ciph_type); + packet_log(" Cipher type: %s\n", ciph_name); + + if (ciph_type != SPU2_CIPHER_TYPE_NONE) { + ciph_mode = (ctrl0 & SPU2_CIPH_MODE) >> SPU2_CIPH_MODE_SHIFT; + ciph_mode_name = spu2_ciph_mode_name(ciph_mode); + packet_log(" Cipher mode: %s\n", ciph_mode_name); + } + + cfb = (ctrl0 & SPU2_CFB_MASK) >> SPU2_CFB_MASK_SHIFT; + packet_log(" CFB %#x\n", cfb); + + proto = (ctrl0 & SPU2_PROTO_SEL) >> SPU2_PROTO_SEL_SHIFT; + packet_log(" protocol %#x\n", proto); + + if (ctrl0 & SPU2_HASH_FIRST) + packet_log(" hash first\n"); + else + packet_log(" cipher first\n"); + + if (ctrl0 & SPU2_CHK_TAG) + packet_log(" check tag\n"); + + hash_type = (ctrl0 & SPU2_HASH_TYPE) >> SPU2_HASH_TYPE_SHIFT; + hash_name = spu2_hash_type_name(hash_type); + packet_log(" Hash type: %s\n", hash_name); + + if (hash_type != SPU2_HASH_TYPE_NONE) { + hash_mode = (ctrl0 & SPU2_HASH_MODE) >> SPU2_HASH_MODE_SHIFT; + hash_mode_name = spu2_hash_mode_name(hash_mode); + packet_log(" Hash mode: %s\n", hash_mode_name); + } + + if (ctrl0 & SPU2_CIPH_PAD_EN) { + packet_log(" Cipher pad: %#2llx\n", + (ctrl0 & SPU2_CIPH_PAD) >> SPU2_CIPH_PAD_SHIFT); + } +} + +/* Dump FMD ctrl1. The ctrl1 input is in host byte order */ +static void spu2_dump_fmd_ctrl1(u64 ctrl1) +{ + u8 hash_key_len; + u8 ciph_key_len; + u8 ret_iv_len; + u8 iv_offset; + u8 iv_len; + u8 hash_tag_len; + u8 ret_md; + + packet_log(" FMD CTRL1 %#16llx\n", ctrl1); + if (ctrl1 & SPU2_TAG_LOC) + packet_log(" Tag after payload\n"); + + packet_log(" Msg includes "); + if (ctrl1 & SPU2_HAS_FR_DATA) + packet_log("FD "); + if (ctrl1 & SPU2_HAS_AAD1) + packet_log("AAD1 "); + if (ctrl1 & SPU2_HAS_NAAD) + packet_log("NAAD "); + if (ctrl1 & SPU2_HAS_AAD2) + packet_log("AAD2 "); + if (ctrl1 & SPU2_HAS_ESN) + packet_log("ESN "); + packet_log("\n"); + + hash_key_len = (ctrl1 & SPU2_HASH_KEY_LEN) >> SPU2_HASH_KEY_LEN_SHIFT; + packet_log(" Hash key len %u\n", hash_key_len); + + ciph_key_len = (ctrl1 & SPU2_CIPH_KEY_LEN) >> SPU2_CIPH_KEY_LEN_SHIFT; + packet_log(" Cipher key len %u\n", ciph_key_len); + + if (ctrl1 & SPU2_GENIV) + packet_log(" Generate IV\n"); + + if (ctrl1 & SPU2_HASH_IV) + packet_log(" IV included in hash\n"); + + if (ctrl1 & SPU2_RET_IV) + packet_log(" Return IV in output before payload\n"); + + ret_iv_len = (ctrl1 & SPU2_RET_IV_LEN) >> SPU2_RET_IV_LEN_SHIFT; + packet_log(" Length of returned IV %u bytes\n", + ret_iv_len ? ret_iv_len : 16); + + iv_offset = (ctrl1 & SPU2_IV_OFFSET) >> SPU2_IV_OFFSET_SHIFT; + packet_log(" IV offset %u\n", iv_offset); + + iv_len = (ctrl1 & SPU2_IV_LEN) >> SPU2_IV_LEN_SHIFT; + packet_log(" Input IV len %u bytes\n", iv_len); + + hash_tag_len = (ctrl1 & SPU2_HASH_TAG_LEN) >> SPU2_HASH_TAG_LEN_SHIFT; + packet_log(" Hash tag length %u bytes\n", hash_tag_len); + + packet_log(" Return "); + ret_md = (ctrl1 & SPU2_RETURN_MD) >> SPU2_RETURN_MD_SHIFT; + if (ret_md) + packet_log("FMD "); + if (ret_md == SPU2_RET_FMD_OMD) + packet_log("OMD "); + else if (ret_md == SPU2_RET_FMD_OMD_IV) + packet_log("OMD IV "); + if (ctrl1 & SPU2_RETURN_FD) + packet_log("FD "); + if (ctrl1 & SPU2_RETURN_AAD1) + packet_log("AAD1 "); + if (ctrl1 & SPU2_RETURN_NAAD) + packet_log("NAAD "); + if (ctrl1 & SPU2_RETURN_AAD2) + packet_log("AAD2 "); + if (ctrl1 & SPU2_RETURN_PAY) + packet_log("Payload"); + packet_log("\n"); +} + +/* Dump FMD ctrl2. The ctrl2 input is in host byte order */ +static void spu2_dump_fmd_ctrl2(u64 ctrl2) +{ + packet_log(" FMD CTRL2 %#16llx\n", ctrl2); + + packet_log(" AAD1 offset %llu length %llu bytes\n", + ctrl2 & SPU2_AAD1_OFFSET, + (ctrl2 & SPU2_AAD1_LEN) >> SPU2_AAD1_LEN_SHIFT); + packet_log(" AAD2 offset %llu\n", + (ctrl2 & SPU2_AAD2_OFFSET) >> SPU2_AAD2_OFFSET_SHIFT); + packet_log(" Payload offset %llu\n", + (ctrl2 & SPU2_PL_OFFSET) >> SPU2_PL_OFFSET_SHIFT); +} + +/* Dump FMD ctrl3. The ctrl3 input is in host byte order */ +static void spu2_dump_fmd_ctrl3(u64 ctrl3) +{ + packet_log(" FMD CTRL3 %#16llx\n", ctrl3); + + packet_log(" Payload length %llu bytes\n", ctrl3 & SPU2_PL_LEN); + packet_log(" TLS length %llu bytes\n", + (ctrl3 & SPU2_TLS_LEN) >> SPU2_TLS_LEN_SHIFT); +} + +static void spu2_dump_fmd(struct SPU2_FMD *fmd) +{ + spu2_dump_fmd_ctrl0(le64_to_cpu(fmd->ctrl0)); + spu2_dump_fmd_ctrl1(le64_to_cpu(fmd->ctrl1)); + spu2_dump_fmd_ctrl2(le64_to_cpu(fmd->ctrl2)); + spu2_dump_fmd_ctrl3(le64_to_cpu(fmd->ctrl3)); +} + +static void spu2_dump_omd(u8 *omd, u16 hash_key_len, u16 ciph_key_len, + u16 hash_iv_len, u16 ciph_iv_len) +{ + u8 *ptr = omd; + + packet_log(" OMD:\n"); + + if (hash_key_len) { + packet_log(" Hash Key Length %u bytes\n", hash_key_len); + packet_dump(" KEY: ", ptr, hash_key_len); + ptr += hash_key_len; + } + + if (ciph_key_len) { + packet_log(" Cipher Key Length %u bytes\n", ciph_key_len); + packet_dump(" KEY: ", ptr, ciph_key_len); + ptr += ciph_key_len; + } + + if (hash_iv_len) { + packet_log(" Hash IV Length %u bytes\n", hash_iv_len); + packet_dump(" hash IV: ", ptr, hash_iv_len); + ptr += hash_iv_len; + } + + if (ciph_iv_len) { + packet_log(" Cipher IV Length %u bytes\n", ciph_iv_len); + packet_dump(" cipher IV: ", ptr, ciph_iv_len); + } +} + +/* Dump a SPU2 header for debug */ +void spu2_dump_msg_hdr(u8 *buf, unsigned int buf_len) +{ + struct SPU2_FMD *fmd = (struct SPU2_FMD *)buf; + u8 *omd; + u64 ctrl1; + u16 hash_key_len; + u16 ciph_key_len; + u16 hash_iv_len; + u16 ciph_iv_len; + u16 omd_len; + + packet_log("\n"); + packet_log("SPU2 message header %p len: %u\n", buf, buf_len); + + spu2_dump_fmd(fmd); + omd = (u8 *)(fmd + 1); + + ctrl1 = le64_to_cpu(fmd->ctrl1); + hash_key_len = (ctrl1 & SPU2_HASH_KEY_LEN) >> SPU2_HASH_KEY_LEN_SHIFT; + ciph_key_len = (ctrl1 & SPU2_CIPH_KEY_LEN) >> SPU2_CIPH_KEY_LEN_SHIFT; + hash_iv_len = 0; + ciph_iv_len = (ctrl1 & SPU2_IV_LEN) >> SPU2_IV_LEN_SHIFT; + spu2_dump_omd(omd, hash_key_len, ciph_key_len, hash_iv_len, + ciph_iv_len); + + /* Double check sanity */ + omd_len = hash_key_len + ciph_key_len + hash_iv_len + ciph_iv_len; + if (FMD_SIZE + omd_len != buf_len) { + packet_log + (" Packet parsed incorrectly. buf_len %u, sum of MD %zu\n", + buf_len, FMD_SIZE + omd_len); + } + packet_log("\n"); +} + +/** + * spu2_fmd_init() - At setkey time, initialize the fixed meta data for + * subsequent skcipher requests for this context. + * @fmd: Start of FMD field to be written + * @spu2_type: Cipher algorithm + * @spu2_mode: Cipher mode + * @cipher_key_len: Length of cipher key, in bytes + * @cipher_iv_len: Length of cipher initialization vector, in bytes + * + * Return: 0 (success) + */ +static int spu2_fmd_init(struct SPU2_FMD *fmd, + enum spu2_cipher_type spu2_type, + enum spu2_cipher_mode spu2_mode, + u32 cipher_key_len, u32 cipher_iv_len) +{ + u64 ctrl0; + u64 ctrl1; + u64 ctrl2; + u64 ctrl3; + u32 aad1_offset; + u32 aad2_offset; + u16 aad1_len = 0; + u64 payload_offset; + + ctrl0 = (spu2_type << SPU2_CIPH_TYPE_SHIFT) | + (spu2_mode << SPU2_CIPH_MODE_SHIFT); + + ctrl1 = (cipher_key_len << SPU2_CIPH_KEY_LEN_SHIFT) | + ((u64)cipher_iv_len << SPU2_IV_LEN_SHIFT) | + ((u64)SPU2_RET_FMD_ONLY << SPU2_RETURN_MD_SHIFT) | SPU2_RETURN_PAY; + + /* + * AAD1 offset is from start of FD. FD length is always 0 for this + * driver. So AAD1_offset is always 0. + */ + aad1_offset = 0; + aad2_offset = aad1_offset; + payload_offset = 0; + ctrl2 = aad1_offset | + (aad1_len << SPU2_AAD1_LEN_SHIFT) | + (aad2_offset << SPU2_AAD2_OFFSET_SHIFT) | + (payload_offset << SPU2_PL_OFFSET_SHIFT); + + ctrl3 = 0; + + fmd->ctrl0 = cpu_to_le64(ctrl0); + fmd->ctrl1 = cpu_to_le64(ctrl1); + fmd->ctrl2 = cpu_to_le64(ctrl2); + fmd->ctrl3 = cpu_to_le64(ctrl3); + + return 0; +} + +/** + * spu2_fmd_ctrl0_write() - Write ctrl0 field in fixed metadata (FMD) field of + * SPU request packet. + * @fmd: Start of FMD field to be written + * @is_inbound: true if decrypting. false if encrypting. + * @auth_first: true if alg authenticates before encrypting + * @protocol: protocol selector + * @cipher_type: cipher algorithm + * @cipher_mode: cipher mode + * @auth_type: authentication type + * @auth_mode: authentication mode + */ +static void spu2_fmd_ctrl0_write(struct SPU2_FMD *fmd, + bool is_inbound, bool auth_first, + enum spu2_proto_sel protocol, + enum spu2_cipher_type cipher_type, + enum spu2_cipher_mode cipher_mode, + enum spu2_hash_type auth_type, + enum spu2_hash_mode auth_mode) +{ + u64 ctrl0 = 0; + + if ((cipher_type != SPU2_CIPHER_TYPE_NONE) && !is_inbound) + ctrl0 |= SPU2_CIPH_ENCRYPT_EN; + + ctrl0 |= ((u64)cipher_type << SPU2_CIPH_TYPE_SHIFT) | + ((u64)cipher_mode << SPU2_CIPH_MODE_SHIFT); + + if (protocol) + ctrl0 |= (u64)protocol << SPU2_PROTO_SEL_SHIFT; + + if (auth_first) + ctrl0 |= SPU2_HASH_FIRST; + + if (is_inbound && (auth_type != SPU2_HASH_TYPE_NONE)) + ctrl0 |= SPU2_CHK_TAG; + + ctrl0 |= (((u64)auth_type << SPU2_HASH_TYPE_SHIFT) | + ((u64)auth_mode << SPU2_HASH_MODE_SHIFT)); + + fmd->ctrl0 = cpu_to_le64(ctrl0); +} + +/** + * spu2_fmd_ctrl1_write() - Write ctrl1 field in fixed metadata (FMD) field of + * SPU request packet. + * @fmd: Start of FMD field to be written + * @is_inbound: true if decrypting. false if encrypting. + * @assoc_size: Length of additional associated data, in bytes + * @auth_key_len: Length of authentication key, in bytes + * @cipher_key_len: Length of cipher key, in bytes + * @gen_iv: If true, hw generates IV and returns in response + * @hash_iv: IV participates in hash. Used for IPSEC and TLS. + * @return_iv: Return IV in output packet before payload + * @ret_iv_len: Length of IV returned from SPU, in bytes + * @ret_iv_offset: Offset into full IV of start of returned IV + * @cipher_iv_len: Length of input cipher IV, in bytes + * @digest_size: Length of digest (aka, hash tag or ICV), in bytes + * @return_payload: Return payload in SPU response + * @return_md : return metadata in SPU response + * + * Packet can have AAD2 w/o AAD1. For algorithms currently supported, + * associated data goes in AAD2. + */ +static void spu2_fmd_ctrl1_write(struct SPU2_FMD *fmd, bool is_inbound, + u64 assoc_size, + u64 auth_key_len, u64 cipher_key_len, + bool gen_iv, bool hash_iv, bool return_iv, + u64 ret_iv_len, u64 ret_iv_offset, + u64 cipher_iv_len, u64 digest_size, + bool return_payload, bool return_md) +{ + u64 ctrl1 = 0; + + if (is_inbound && digest_size) + ctrl1 |= SPU2_TAG_LOC; + + if (assoc_size) { + ctrl1 |= SPU2_HAS_AAD2; + ctrl1 |= SPU2_RETURN_AAD2; /* need aad2 for gcm aes esp */ + } + + if (auth_key_len) + ctrl1 |= ((auth_key_len << SPU2_HASH_KEY_LEN_SHIFT) & + SPU2_HASH_KEY_LEN); + + if (cipher_key_len) + ctrl1 |= ((cipher_key_len << SPU2_CIPH_KEY_LEN_SHIFT) & + SPU2_CIPH_KEY_LEN); + + if (gen_iv) + ctrl1 |= SPU2_GENIV; + + if (hash_iv) + ctrl1 |= SPU2_HASH_IV; + + if (return_iv) { + ctrl1 |= SPU2_RET_IV; + ctrl1 |= ret_iv_len << SPU2_RET_IV_LEN_SHIFT; + ctrl1 |= ret_iv_offset << SPU2_IV_OFFSET_SHIFT; + } + + ctrl1 |= ((cipher_iv_len << SPU2_IV_LEN_SHIFT) & SPU2_IV_LEN); + + if (digest_size) + ctrl1 |= ((digest_size << SPU2_HASH_TAG_LEN_SHIFT) & + SPU2_HASH_TAG_LEN); + + /* Let's ask for the output pkt to include FMD, but don't need to + * get keys and IVs back in OMD. + */ + if (return_md) + ctrl1 |= ((u64)SPU2_RET_FMD_ONLY << SPU2_RETURN_MD_SHIFT); + else + ctrl1 |= ((u64)SPU2_RET_NO_MD << SPU2_RETURN_MD_SHIFT); + + /* Crypto API does not get assoc data back. So no need for AAD2. */ + + if (return_payload) + ctrl1 |= SPU2_RETURN_PAY; + + fmd->ctrl1 = cpu_to_le64(ctrl1); +} + +/** + * spu2_fmd_ctrl2_write() - Set the ctrl2 field in the fixed metadata field of + * SPU2 header. + * @fmd: Start of FMD field to be written + * @cipher_offset: Number of bytes from Start of Packet (end of FD field) where + * data to be encrypted or decrypted begins + * @auth_key_len: Length of authentication key, in bytes + * @auth_iv_len: Length of authentication initialization vector, in bytes + * @cipher_key_len: Length of cipher key, in bytes + * @cipher_iv_len: Length of cipher IV, in bytes + */ +static void spu2_fmd_ctrl2_write(struct SPU2_FMD *fmd, u64 cipher_offset, + u64 auth_key_len, u64 auth_iv_len, + u64 cipher_key_len, u64 cipher_iv_len) +{ + u64 ctrl2; + u64 aad1_offset; + u64 aad2_offset; + u16 aad1_len = 0; + u64 payload_offset; + + /* AAD1 offset is from start of FD. FD length always 0. */ + aad1_offset = 0; + + aad2_offset = aad1_offset; + payload_offset = cipher_offset; + ctrl2 = aad1_offset | + (aad1_len << SPU2_AAD1_LEN_SHIFT) | + (aad2_offset << SPU2_AAD2_OFFSET_SHIFT) | + (payload_offset << SPU2_PL_OFFSET_SHIFT); + + fmd->ctrl2 = cpu_to_le64(ctrl2); +} + +/** + * spu2_fmd_ctrl3_write() - Set the ctrl3 field in FMD + * @fmd: Fixed meta data. First field in SPU2 msg header. + * @payload_len: Length of payload, in bytes + */ +static void spu2_fmd_ctrl3_write(struct SPU2_FMD *fmd, u64 payload_len) +{ + u64 ctrl3; + + ctrl3 = payload_len & SPU2_PL_LEN; + + fmd->ctrl3 = cpu_to_le64(ctrl3); +} + +/** + * spu2_ctx_max_payload() - Determine the maximum length of the payload for a + * SPU message for a given cipher and hash alg context. + * @cipher_alg: The cipher algorithm + * @cipher_mode: The cipher mode + * @blocksize: The size of a block of data for this algo + * + * For SPU2, the hardware generally ignores the PayloadLen field in ctrl3 of + * FMD and just keeps computing until it receives a DMA descriptor with the EOF + * flag set. So we consider the max payload to be infinite. AES CCM is an + * exception. + * + * Return: Max payload length in bytes + */ +u32 spu2_ctx_max_payload(enum spu_cipher_alg cipher_alg, + enum spu_cipher_mode cipher_mode, + unsigned int blocksize) +{ + if ((cipher_alg == CIPHER_ALG_AES) && + (cipher_mode == CIPHER_MODE_CCM)) { + u32 excess = SPU2_MAX_PAYLOAD % blocksize; + + return SPU2_MAX_PAYLOAD - excess; + } else { + return SPU_MAX_PAYLOAD_INF; + } +} + +/** + * spu2_payload_length() - Given a SPU2 message header, extract the payload + * length. + * @spu_hdr: Start of SPU message header (FMD) + * + * Return: payload length, in bytes + */ +u32 spu2_payload_length(u8 *spu_hdr) +{ + struct SPU2_FMD *fmd = (struct SPU2_FMD *)spu_hdr; + u32 pl_len; + u64 ctrl3; + + ctrl3 = le64_to_cpu(fmd->ctrl3); + pl_len = ctrl3 & SPU2_PL_LEN; + + return pl_len; +} + +/** + * spu2_response_hdr_len() - Determine the expected length of a SPU response + * header. + * @auth_key_len: Length of authentication key, in bytes + * @enc_key_len: Length of encryption key, in bytes + * @is_hash: Unused + * + * For SPU2, includes just FMD. OMD is never requested. + * + * Return: Length of FMD, in bytes + */ +u16 spu2_response_hdr_len(u16 auth_key_len, u16 enc_key_len, bool is_hash) +{ + return FMD_SIZE; +} + +/** + * spu2_hash_pad_len() - Calculate the length of hash padding required to extend + * data to a full block size. + * @hash_alg: hash algorithm + * @hash_mode: hash mode + * @chunksize: length of data, in bytes + * @hash_block_size: size of a hash block, in bytes + * + * SPU2 hardware does all hash padding + * + * Return: length of hash pad in bytes + */ +u16 spu2_hash_pad_len(enum hash_alg hash_alg, enum hash_mode hash_mode, + u32 chunksize, u16 hash_block_size) +{ + return 0; +} + +/** + * spu2_gcm_ccm_pad_len() - Determine the length of GCM/CCM padding for either + * the AAD field or the data. + * @cipher_mode: Unused + * @data_size: Unused + * + * Return: 0. Unlike SPU-M, SPU2 hardware does any GCM/CCM padding required. + */ +u32 spu2_gcm_ccm_pad_len(enum spu_cipher_mode cipher_mode, + unsigned int data_size) +{ + return 0; +} + +/** + * spu2_assoc_resp_len() - Determine the size of the AAD2 buffer needed to catch + * associated data in a SPU2 output packet. + * @cipher_mode: cipher mode + * @assoc_len: length of additional associated data, in bytes + * @iv_len: length of initialization vector, in bytes + * @is_encrypt: true if encrypting. false if decrypt. + * + * Return: Length of buffer to catch associated data in response + */ +u32 spu2_assoc_resp_len(enum spu_cipher_mode cipher_mode, + unsigned int assoc_len, unsigned int iv_len, + bool is_encrypt) +{ + u32 resp_len = assoc_len; + + if (is_encrypt) + /* gcm aes esp has to write 8-byte IV in response */ + resp_len += iv_len; + return resp_len; +} + +/** + * spu2_aead_ivlen() - Calculate the length of the AEAD IV to be included + * in a SPU request after the AAD and before the payload. + * @cipher_mode: cipher mode + * @iv_len: initialization vector length in bytes + * + * For SPU2, AEAD IV is included in OMD and does not need to be repeated + * prior to the payload. + * + * Return: Length of AEAD IV in bytes + */ +u8 spu2_aead_ivlen(enum spu_cipher_mode cipher_mode, u16 iv_len) +{ + return 0; +} + +/** + * spu2_hash_type() - Determine the type of hash operation. + * @src_sent: The number of bytes in the current request that have already + * been sent to the SPU to be hashed. + * + * SPU2 always does a FULL hash operation + */ +enum hash_type spu2_hash_type(u32 src_sent) +{ + return HASH_TYPE_FULL; +} + +/** + * spu2_digest_size() - Determine the size of a hash digest to expect the SPU to + * return. + * @alg_digest_size: Number of bytes in the final digest for the given algo + * @alg: The hash algorithm + * @htype: Type of hash operation (init, update, full, etc) + * + */ +u32 spu2_digest_size(u32 alg_digest_size, enum hash_alg alg, + enum hash_type htype) +{ + return alg_digest_size; +} + +/** + * spu2_create_request() - Build a SPU2 request message header, includint FMD and + * OMD. + * @spu_hdr: Start of buffer where SPU request header is to be written + * @req_opts: SPU request message options + * @cipher_parms: Parameters related to cipher algorithm + * @hash_parms: Parameters related to hash algorithm + * @aead_parms: Parameters related to AEAD operation + * @data_size: Length of data to be encrypted or authenticated. If AEAD, does + * not include length of AAD. + * + * Construct the message starting at spu_hdr. Caller should allocate this buffer + * in DMA-able memory at least SPU_HEADER_ALLOC_LEN bytes long. + * + * Return: the length of the SPU header in bytes. 0 if an error occurs. + */ +u32 spu2_create_request(u8 *spu_hdr, + struct spu_request_opts *req_opts, + struct spu_cipher_parms *cipher_parms, + struct spu_hash_parms *hash_parms, + struct spu_aead_parms *aead_parms, + unsigned int data_size) +{ + struct SPU2_FMD *fmd; + u8 *ptr; + unsigned int buf_len; + int err; + enum spu2_cipher_type spu2_ciph_type = SPU2_CIPHER_TYPE_NONE; + enum spu2_cipher_mode spu2_ciph_mode; + enum spu2_hash_type spu2_auth_type = SPU2_HASH_TYPE_NONE; + enum spu2_hash_mode spu2_auth_mode; + bool return_md = true; + enum spu2_proto_sel proto = SPU2_PROTO_RESV; + + /* size of the payload */ + unsigned int payload_len = + hash_parms->prebuf_len + data_size + hash_parms->pad_len - + ((req_opts->is_aead && req_opts->is_inbound) ? + hash_parms->digestsize : 0); + + /* offset of prebuf or data from start of AAD2 */ + unsigned int cipher_offset = aead_parms->assoc_size + + aead_parms->aad_pad_len + aead_parms->iv_len; + + /* total size of the data following OMD (without STAT word padding) */ + unsigned int real_db_size = spu_real_db_size(aead_parms->assoc_size, + aead_parms->iv_len, + hash_parms->prebuf_len, + data_size, + aead_parms->aad_pad_len, + aead_parms->data_pad_len, + hash_parms->pad_len); + unsigned int assoc_size = aead_parms->assoc_size; + + if (req_opts->is_aead && + (cipher_parms->alg == CIPHER_ALG_AES) && + (cipher_parms->mode == CIPHER_MODE_GCM)) + /* + * On SPU 2, aes gcm cipher first on encrypt, auth first on + * decrypt + */ + req_opts->auth_first = req_opts->is_inbound; + + /* and do opposite for ccm (auth 1st on encrypt) */ + if (req_opts->is_aead && + (cipher_parms->alg == CIPHER_ALG_AES) && + (cipher_parms->mode == CIPHER_MODE_CCM)) + req_opts->auth_first = !req_opts->is_inbound; + + flow_log("%s()\n", __func__); + flow_log(" in:%u authFirst:%u\n", + req_opts->is_inbound, req_opts->auth_first); + flow_log(" cipher alg:%u mode:%u type %u\n", cipher_parms->alg, + cipher_parms->mode, cipher_parms->type); + flow_log(" is_esp: %s\n", str_yes_no(req_opts->is_esp)); + flow_log(" key: %d\n", cipher_parms->key_len); + flow_dump(" key: ", cipher_parms->key_buf, cipher_parms->key_len); + flow_log(" iv: %d\n", cipher_parms->iv_len); + flow_dump(" iv: ", cipher_parms->iv_buf, cipher_parms->iv_len); + flow_log(" auth alg:%u mode:%u type %u\n", + hash_parms->alg, hash_parms->mode, hash_parms->type); + flow_log(" digestsize: %u\n", hash_parms->digestsize); + flow_log(" authkey: %d\n", hash_parms->key_len); + flow_dump(" authkey: ", hash_parms->key_buf, hash_parms->key_len); + flow_log(" assoc_size:%u\n", assoc_size); + flow_log(" prebuf_len:%u\n", hash_parms->prebuf_len); + flow_log(" data_size:%u\n", data_size); + flow_log(" hash_pad_len:%u\n", hash_parms->pad_len); + flow_log(" real_db_size:%u\n", real_db_size); + flow_log(" cipher_offset:%u payload_len:%u\n", + cipher_offset, payload_len); + flow_log(" aead_iv: %u\n", aead_parms->iv_len); + + /* Convert to spu2 values for cipher alg, hash alg */ + err = spu2_cipher_xlate(cipher_parms->alg, cipher_parms->mode, + cipher_parms->type, + &spu2_ciph_type, &spu2_ciph_mode); + + /* If we are doing GCM hashing only - either via rfc4543 transform + * or because we happen to do GCM with AAD only and no payload - we + * need to configure hardware to use hash key rather than cipher key + * and put data into payload. This is because unlike SPU-M, running + * GCM cipher with 0 size payload is not permitted. + */ + if ((req_opts->is_rfc4543) || + ((spu2_ciph_mode == SPU2_CIPHER_MODE_GCM) && + (payload_len == 0))) { + /* Use hashing (only) and set up hash key */ + spu2_ciph_type = SPU2_CIPHER_TYPE_NONE; + hash_parms->key_len = cipher_parms->key_len; + memcpy(hash_parms->key_buf, cipher_parms->key_buf, + cipher_parms->key_len); + cipher_parms->key_len = 0; + + if (req_opts->is_rfc4543) + payload_len += assoc_size; + else + payload_len = assoc_size; + cipher_offset = 0; + assoc_size = 0; + } + + if (err) + return 0; + + flow_log("spu2 cipher type %s, cipher mode %s\n", + spu2_ciph_type_name(spu2_ciph_type), + spu2_ciph_mode_name(spu2_ciph_mode)); + + err = spu2_hash_xlate(hash_parms->alg, hash_parms->mode, + hash_parms->type, + cipher_parms->type, + &spu2_auth_type, &spu2_auth_mode); + if (err) + return 0; + + flow_log("spu2 hash type %s, hash mode %s\n", + spu2_hash_type_name(spu2_auth_type), + spu2_hash_mode_name(spu2_auth_mode)); + + fmd = (struct SPU2_FMD *)spu_hdr; + + spu2_fmd_ctrl0_write(fmd, req_opts->is_inbound, req_opts->auth_first, + proto, spu2_ciph_type, spu2_ciph_mode, + spu2_auth_type, spu2_auth_mode); + + spu2_fmd_ctrl1_write(fmd, req_opts->is_inbound, assoc_size, + hash_parms->key_len, cipher_parms->key_len, + false, false, + aead_parms->return_iv, aead_parms->ret_iv_len, + aead_parms->ret_iv_off, + cipher_parms->iv_len, hash_parms->digestsize, + !req_opts->bd_suppress, return_md); + + spu2_fmd_ctrl2_write(fmd, cipher_offset, hash_parms->key_len, 0, + cipher_parms->key_len, cipher_parms->iv_len); + + spu2_fmd_ctrl3_write(fmd, payload_len); + + ptr = (u8 *)(fmd + 1); + buf_len = sizeof(struct SPU2_FMD); + + /* Write OMD */ + if (hash_parms->key_len) { + memcpy(ptr, hash_parms->key_buf, hash_parms->key_len); + ptr += hash_parms->key_len; + buf_len += hash_parms->key_len; + } + if (cipher_parms->key_len) { + memcpy(ptr, cipher_parms->key_buf, cipher_parms->key_len); + ptr += cipher_parms->key_len; + buf_len += cipher_parms->key_len; + } + if (cipher_parms->iv_len) { + memcpy(ptr, cipher_parms->iv_buf, cipher_parms->iv_len); + ptr += cipher_parms->iv_len; + buf_len += cipher_parms->iv_len; + } + + packet_dump(" SPU request header: ", spu_hdr, buf_len); + + return buf_len; +} + +/** + * spu2_cipher_req_init() - Build an skcipher SPU2 request message header, + * including FMD and OMD. + * @spu_hdr: Location of start of SPU request (FMD field) + * @cipher_parms: Parameters describing cipher request + * + * Called at setkey time to initialize a msg header that can be reused for all + * subsequent skcipher requests. Construct the message starting at spu_hdr. + * Caller should allocate this buffer in DMA-able memory at least + * SPU_HEADER_ALLOC_LEN bytes long. + * + * Return: the total length of the SPU header (FMD and OMD) in bytes. 0 if an + * error occurs. + */ +u16 spu2_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms) +{ + struct SPU2_FMD *fmd; + u8 *omd; + enum spu2_cipher_type spu2_type = SPU2_CIPHER_TYPE_NONE; + enum spu2_cipher_mode spu2_mode; + int err; + + flow_log("%s()\n", __func__); + flow_log(" cipher alg:%u mode:%u type %u\n", cipher_parms->alg, + cipher_parms->mode, cipher_parms->type); + flow_log(" cipher_iv_len: %u\n", cipher_parms->iv_len); + flow_log(" key: %d\n", cipher_parms->key_len); + flow_dump(" key: ", cipher_parms->key_buf, cipher_parms->key_len); + + /* Convert to spu2 values */ + err = spu2_cipher_xlate(cipher_parms->alg, cipher_parms->mode, + cipher_parms->type, &spu2_type, &spu2_mode); + if (err) + return 0; + + flow_log("spu2 cipher type %s, cipher mode %s\n", + spu2_ciph_type_name(spu2_type), + spu2_ciph_mode_name(spu2_mode)); + + /* Construct the FMD header */ + fmd = (struct SPU2_FMD *)spu_hdr; + err = spu2_fmd_init(fmd, spu2_type, spu2_mode, cipher_parms->key_len, + cipher_parms->iv_len); + if (err) + return 0; + + /* Write cipher key to OMD */ + omd = (u8 *)(fmd + 1); + if (cipher_parms->key_buf && cipher_parms->key_len) + memcpy(omd, cipher_parms->key_buf, cipher_parms->key_len); + + packet_dump(" SPU request header: ", spu_hdr, + FMD_SIZE + cipher_parms->key_len + cipher_parms->iv_len); + + return FMD_SIZE + cipher_parms->key_len + cipher_parms->iv_len; +} + +/** + * spu2_cipher_req_finish() - Finish building a SPU request message header for a + * block cipher request. + * @spu_hdr: Start of the request message header (MH field) + * @spu_req_hdr_len: Length in bytes of the SPU request header + * @is_inbound: 0 encrypt, 1 decrypt + * @cipher_parms: Parameters describing cipher operation to be performed + * @data_size: Length of the data in the BD field + * + * Assumes much of the header was already filled in at setkey() time in + * spu_cipher_req_init(). + * spu_cipher_req_init() fills in the encryption key. + */ +void spu2_cipher_req_finish(u8 *spu_hdr, + u16 spu_req_hdr_len, + unsigned int is_inbound, + struct spu_cipher_parms *cipher_parms, + unsigned int data_size) +{ + struct SPU2_FMD *fmd; + u8 *omd; /* start of optional metadata */ + u64 ctrl0; + u64 ctrl3; + + flow_log("%s()\n", __func__); + flow_log(" in: %u\n", is_inbound); + flow_log(" cipher alg: %u, cipher_type: %u\n", cipher_parms->alg, + cipher_parms->type); + flow_log(" iv len: %d\n", cipher_parms->iv_len); + flow_dump(" iv: ", cipher_parms->iv_buf, cipher_parms->iv_len); + flow_log(" data_size: %u\n", data_size); + + fmd = (struct SPU2_FMD *)spu_hdr; + omd = (u8 *)(fmd + 1); + + /* + * FMD ctrl0 was initialized at setkey time. update it to indicate + * whether we are encrypting or decrypting. + */ + ctrl0 = le64_to_cpu(fmd->ctrl0); + if (is_inbound) + ctrl0 &= ~SPU2_CIPH_ENCRYPT_EN; /* decrypt */ + else + ctrl0 |= SPU2_CIPH_ENCRYPT_EN; /* encrypt */ + fmd->ctrl0 = cpu_to_le64(ctrl0); + + if (cipher_parms->alg && cipher_parms->iv_buf && cipher_parms->iv_len) { + /* cipher iv provided so put it in here */ + memcpy(omd + cipher_parms->key_len, cipher_parms->iv_buf, + cipher_parms->iv_len); + } + + ctrl3 = le64_to_cpu(fmd->ctrl3); + data_size &= SPU2_PL_LEN; + ctrl3 |= data_size; + fmd->ctrl3 = cpu_to_le64(ctrl3); + + packet_dump(" SPU request header: ", spu_hdr, spu_req_hdr_len); +} + +/** + * spu2_request_pad() - Create pad bytes at the end of the data. + * @pad_start: Start of buffer where pad bytes are to be written + * @gcm_padding: Length of GCM padding, in bytes + * @hash_pad_len: Number of bytes of padding extend data to full block + * @auth_alg: Authentication algorithm + * @auth_mode: Authentication mode + * @total_sent: Length inserted at end of hash pad + * @status_padding: Number of bytes of padding to align STATUS word + * + * There may be three forms of pad: + * 1. GCM pad - for GCM mode ciphers, pad to 16-byte alignment + * 2. hash pad - pad to a block length, with 0x80 data terminator and + * size at the end + * 3. STAT pad - to ensure the STAT field is 4-byte aligned + */ +void spu2_request_pad(u8 *pad_start, u32 gcm_padding, u32 hash_pad_len, + enum hash_alg auth_alg, enum hash_mode auth_mode, + unsigned int total_sent, u32 status_padding) +{ + u8 *ptr = pad_start; + + /* fix data alignent for GCM */ + if (gcm_padding > 0) { + flow_log(" GCM: padding to 16 byte alignment: %u bytes\n", + gcm_padding); + memset(ptr, 0, gcm_padding); + ptr += gcm_padding; + } + + if (hash_pad_len > 0) { + /* clear the padding section */ + memset(ptr, 0, hash_pad_len); + + /* terminate the data */ + *ptr = 0x80; + ptr += (hash_pad_len - sizeof(u64)); + + /* add the size at the end as required per alg */ + if (auth_alg == HASH_ALG_MD5) + *(__le64 *)ptr = cpu_to_le64(total_sent * 8ull); + else /* SHA1, SHA2-224, SHA2-256 */ + *(__be64 *)ptr = cpu_to_be64(total_sent * 8ull); + ptr += sizeof(u64); + } + + /* pad to a 4byte alignment for STAT */ + if (status_padding > 0) { + flow_log(" STAT: padding to 4 byte alignment: %u bytes\n", + status_padding); + + memset(ptr, 0, status_padding); + ptr += status_padding; + } +} + +/** + * spu2_xts_tweak_in_payload() - Indicate that SPU2 does NOT place the XTS + * tweak field in the packet payload (it uses IV instead) + * + * Return: 0 + */ +u8 spu2_xts_tweak_in_payload(void) +{ + return 0; +} + +/** + * spu2_tx_status_len() - Return the length of the STATUS field in a SPU + * response message. + * + * Return: Length of STATUS field in bytes. + */ +u8 spu2_tx_status_len(void) +{ + return SPU2_TX_STATUS_LEN; +} + +/** + * spu2_rx_status_len() - Return the length of the STATUS field in a SPU + * response message. + * + * Return: Length of STATUS field in bytes. + */ +u8 spu2_rx_status_len(void) +{ + return SPU2_RX_STATUS_LEN; +} + +/** + * spu2_status_process() - Process the status from a SPU response message. + * @statp: start of STATUS word + * + * Return: 0 - if status is good and response should be processed + * !0 - status indicates an error and response is invalid + */ +int spu2_status_process(u8 *statp) +{ + /* SPU2 status is 2 bytes by default - SPU_RX_STATUS_LEN */ + u16 status = le16_to_cpu(*(__le16 *)statp); + + if (status == 0) + return 0; + + flow_log("rx status is %#x\n", status); + if (status == SPU2_INVALID_ICV) + return SPU_INVALID_ICV; + + return -EBADMSG; +} + +/** + * spu2_ccm_update_iv() - Update the IV as per the requirements for CCM mode. + * + * @digestsize: Digest size of this request + * @cipher_parms: (pointer to) cipher parmaeters, includes IV buf & IV len + * @assoclen: Length of AAD data + * @chunksize: length of input data to be sent in this req + * @is_encrypt: true if this is an output/encrypt operation + * @is_esp: true if this is an ESP / RFC4309 operation + * + */ +void spu2_ccm_update_iv(unsigned int digestsize, + struct spu_cipher_parms *cipher_parms, + unsigned int assoclen, unsigned int chunksize, + bool is_encrypt, bool is_esp) +{ + int L; /* size of length field, in bytes */ + + /* + * In RFC4309 mode, L is fixed at 4 bytes; otherwise, IV from + * testmgr contains (L-1) in bottom 3 bits of first byte, + * per RFC 3610. + */ + if (is_esp) + L = CCM_ESP_L_VALUE; + else + L = ((cipher_parms->iv_buf[0] & CCM_B0_L_PRIME) >> + CCM_B0_L_PRIME_SHIFT) + 1; + + /* SPU2 doesn't want these length bytes nor the first byte... */ + cipher_parms->iv_len -= (1 + L); + memmove(cipher_parms->iv_buf, &cipher_parms->iv_buf[1], + cipher_parms->iv_len); +} + +/** + * spu2_wordalign_padlen() - SPU2 does not require padding. + * @data_size: length of data field in bytes + * + * Return: length of status field padding, in bytes (always 0 on SPU2) + */ +u32 spu2_wordalign_padlen(u32 data_size) +{ + return 0; +} diff --git a/drivers/crypto/bcm/spu2.h b/drivers/crypto/bcm/spu2.h new file mode 100644 index 000000000..a76d4e054 --- /dev/null +++ b/drivers/crypto/bcm/spu2.h @@ -0,0 +1,216 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2016 Broadcom + */ + +/* + * This file contains SPU message definitions specific to SPU2. + */ + +#ifndef _SPU2_H +#define _SPU2_H + +enum spu2_cipher_type { + SPU2_CIPHER_TYPE_NONE = 0x0, + SPU2_CIPHER_TYPE_AES128 = 0x1, + SPU2_CIPHER_TYPE_AES192 = 0x2, + SPU2_CIPHER_TYPE_AES256 = 0x3, + SPU2_CIPHER_TYPE_DES = 0x4, + SPU2_CIPHER_TYPE_3DES = 0x5, + SPU2_CIPHER_TYPE_LAST +}; + +enum spu2_cipher_mode { + SPU2_CIPHER_MODE_ECB = 0x0, + SPU2_CIPHER_MODE_CBC = 0x1, + SPU2_CIPHER_MODE_CTR = 0x2, + SPU2_CIPHER_MODE_CFB = 0x3, + SPU2_CIPHER_MODE_OFB = 0x4, + SPU2_CIPHER_MODE_XTS = 0x5, + SPU2_CIPHER_MODE_CCM = 0x6, + SPU2_CIPHER_MODE_GCM = 0x7, + SPU2_CIPHER_MODE_LAST +}; + +enum spu2_hash_type { + SPU2_HASH_TYPE_NONE = 0x0, + SPU2_HASH_TYPE_AES128 = 0x1, + SPU2_HASH_TYPE_AES192 = 0x2, + SPU2_HASH_TYPE_AES256 = 0x3, + SPU2_HASH_TYPE_MD5 = 0x6, + SPU2_HASH_TYPE_SHA1 = 0x7, + SPU2_HASH_TYPE_SHA224 = 0x8, + SPU2_HASH_TYPE_SHA256 = 0x9, + SPU2_HASH_TYPE_SHA384 = 0xa, + SPU2_HASH_TYPE_SHA512 = 0xb, + SPU2_HASH_TYPE_SHA512_224 = 0xc, + SPU2_HASH_TYPE_SHA512_256 = 0xd, + SPU2_HASH_TYPE_SHA3_224 = 0xe, + SPU2_HASH_TYPE_SHA3_256 = 0xf, + SPU2_HASH_TYPE_SHA3_384 = 0x10, + SPU2_HASH_TYPE_SHA3_512 = 0x11, + SPU2_HASH_TYPE_LAST +}; + +enum spu2_hash_mode { + SPU2_HASH_MODE_CMAC = 0x0, + SPU2_HASH_MODE_CBC_MAC = 0x1, + SPU2_HASH_MODE_XCBC_MAC = 0x2, + SPU2_HASH_MODE_HMAC = 0x3, + SPU2_HASH_MODE_RABIN = 0x4, + SPU2_HASH_MODE_CCM = 0x5, + SPU2_HASH_MODE_GCM = 0x6, + SPU2_HASH_MODE_RESERVED = 0x7, + SPU2_HASH_MODE_LAST +}; + +enum spu2_ret_md_opts { + SPU2_RET_NO_MD = 0, /* return no metadata */ + SPU2_RET_FMD_OMD = 1, /* return both FMD and OMD */ + SPU2_RET_FMD_ONLY = 2, /* return only FMD */ + SPU2_RET_FMD_OMD_IV = 3, /* return FMD and OMD with just IVs */ +}; + +/* Fixed Metadata format */ +struct SPU2_FMD { + __le64 ctrl0; + __le64 ctrl1; + __le64 ctrl2; + __le64 ctrl3; +}; + +#define FMD_SIZE sizeof(struct SPU2_FMD) + +/* Fixed part of request message header length in bytes. Just FMD. */ +#define SPU2_REQ_FIXED_LEN FMD_SIZE +#define SPU2_HEADER_ALLOC_LEN (SPU_REQ_FIXED_LEN + \ + 2 * MAX_KEY_SIZE + 2 * MAX_IV_SIZE) + +/* FMD ctrl0 field masks */ +#define SPU2_CIPH_ENCRYPT_EN 0x1 /* 0: decrypt, 1: encrypt */ +#define SPU2_CIPH_TYPE 0xF0 /* one of spu2_cipher_type */ +#define SPU2_CIPH_TYPE_SHIFT 4 +#define SPU2_CIPH_MODE 0xF00 /* one of spu2_cipher_mode */ +#define SPU2_CIPH_MODE_SHIFT 8 +#define SPU2_CFB_MASK 0x7000 /* cipher feedback mask */ +#define SPU2_CFB_MASK_SHIFT 12 +#define SPU2_PROTO_SEL 0xF00000 /* MACsec, IPsec, TLS... */ +#define SPU2_PROTO_SEL_SHIFT 20 +#define SPU2_HASH_FIRST 0x1000000 /* 1: hash input is input pkt + * data + */ +#define SPU2_CHK_TAG 0x2000000 /* 1: check digest provided */ +#define SPU2_HASH_TYPE 0x1F0000000 /* one of spu2_hash_type */ +#define SPU2_HASH_TYPE_SHIFT 28 +#define SPU2_HASH_MODE 0xF000000000 /* one of spu2_hash_mode */ +#define SPU2_HASH_MODE_SHIFT 36 +#define SPU2_CIPH_PAD_EN 0x100000000000 /* 1: Add pad to end of payload for + * enc + */ +#define SPU2_CIPH_PAD 0xFF000000000000 /* cipher pad value */ +#define SPU2_CIPH_PAD_SHIFT 48 + +/* FMD ctrl1 field masks */ +#define SPU2_TAG_LOC 0x1 /* 1: end of payload, 0: undef */ +#define SPU2_HAS_FR_DATA 0x2 /* 1: msg has frame data */ +#define SPU2_HAS_AAD1 0x4 /* 1: msg has AAD1 field */ +#define SPU2_HAS_NAAD 0x8 /* 1: msg has NAAD field */ +#define SPU2_HAS_AAD2 0x10 /* 1: msg has AAD2 field */ +#define SPU2_HAS_ESN 0x20 /* 1: msg has ESN field */ +#define SPU2_HASH_KEY_LEN 0xFF00 /* len of hash key in bytes. + * HMAC only. + */ +#define SPU2_HASH_KEY_LEN_SHIFT 8 +#define SPU2_CIPH_KEY_LEN 0xFF00000 /* len of cipher key in bytes */ +#define SPU2_CIPH_KEY_LEN_SHIFT 20 +#define SPU2_GENIV 0x10000000 /* 1: hw generates IV */ +#define SPU2_HASH_IV 0x20000000 /* 1: IV incl in hash */ +#define SPU2_RET_IV 0x40000000 /* 1: return IV in output msg + * b4 payload + */ +#define SPU2_RET_IV_LEN 0xF00000000 /* length in bytes of IV returned. + * 0 = 16 bytes + */ +#define SPU2_RET_IV_LEN_SHIFT 32 +#define SPU2_IV_OFFSET 0xF000000000 /* gen IV offset */ +#define SPU2_IV_OFFSET_SHIFT 36 +#define SPU2_IV_LEN 0x1F0000000000 /* length of input IV in bytes */ +#define SPU2_IV_LEN_SHIFT 40 +#define SPU2_HASH_TAG_LEN 0x7F000000000000 /* hash tag length in bytes */ +#define SPU2_HASH_TAG_LEN_SHIFT 48 +#define SPU2_RETURN_MD 0x300000000000000 /* return metadata */ +#define SPU2_RETURN_MD_SHIFT 56 +#define SPU2_RETURN_FD 0x400000000000000 +#define SPU2_RETURN_AAD1 0x800000000000000 +#define SPU2_RETURN_NAAD 0x1000000000000000 +#define SPU2_RETURN_AAD2 0x2000000000000000 +#define SPU2_RETURN_PAY 0x4000000000000000 /* return payload */ + +/* FMD ctrl2 field masks */ +#define SPU2_AAD1_OFFSET 0xFFF /* byte offset of AAD1 field */ +#define SPU2_AAD1_LEN 0xFF000 /* length of AAD1 in bytes */ +#define SPU2_AAD1_LEN_SHIFT 12 +#define SPU2_AAD2_OFFSET 0xFFF00000 /* byte offset of AAD2 field */ +#define SPU2_AAD2_OFFSET_SHIFT 20 +#define SPU2_PL_OFFSET 0xFFFFFFFF00000000 /* payload offset from AAD2 */ +#define SPU2_PL_OFFSET_SHIFT 32 + +/* FMD ctrl3 field masks */ +#define SPU2_PL_LEN 0xFFFFFFFF /* payload length in bytes */ +#define SPU2_TLS_LEN 0xFFFF00000000 /* TLS encrypt: cipher len + * TLS decrypt: compressed len + */ +#define SPU2_TLS_LEN_SHIFT 32 + +/* + * Max value that can be represented in the Payload Length field of the + * ctrl3 word of FMD. + */ +#define SPU2_MAX_PAYLOAD SPU2_PL_LEN + +/* Error values returned in STATUS field of response messages */ +#define SPU2_INVALID_ICV 1 + +void spu2_dump_msg_hdr(u8 *buf, unsigned int buf_len); +u32 spu2_ctx_max_payload(enum spu_cipher_alg cipher_alg, + enum spu_cipher_mode cipher_mode, + unsigned int blocksize); +u32 spu2_payload_length(u8 *spu_hdr); +u16 spu2_response_hdr_len(u16 auth_key_len, u16 enc_key_len, bool is_hash); +u16 spu2_hash_pad_len(enum hash_alg hash_alg, enum hash_mode hash_mode, + u32 chunksize, u16 hash_block_size); +u32 spu2_gcm_ccm_pad_len(enum spu_cipher_mode cipher_mode, + unsigned int data_size); +u32 spu2_assoc_resp_len(enum spu_cipher_mode cipher_mode, + unsigned int assoc_len, unsigned int iv_len, + bool is_encrypt); +u8 spu2_aead_ivlen(enum spu_cipher_mode cipher_mode, + u16 iv_len); +enum hash_type spu2_hash_type(u32 src_sent); +u32 spu2_digest_size(u32 alg_digest_size, enum hash_alg alg, + enum hash_type htype); +u32 spu2_create_request(u8 *spu_hdr, + struct spu_request_opts *req_opts, + struct spu_cipher_parms *cipher_parms, + struct spu_hash_parms *hash_parms, + struct spu_aead_parms *aead_parms, + unsigned int data_size); +u16 spu2_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms); +void spu2_cipher_req_finish(u8 *spu_hdr, + u16 spu_req_hdr_len, + unsigned int is_inbound, + struct spu_cipher_parms *cipher_parms, + unsigned int data_size); +void spu2_request_pad(u8 *pad_start, u32 gcm_padding, u32 hash_pad_len, + enum hash_alg auth_alg, enum hash_mode auth_mode, + unsigned int total_sent, u32 status_padding); +u8 spu2_xts_tweak_in_payload(void); +u8 spu2_tx_status_len(void); +u8 spu2_rx_status_len(void); +int spu2_status_process(u8 *statp); +void spu2_ccm_update_iv(unsigned int digestsize, + struct spu_cipher_parms *cipher_parms, + unsigned int assoclen, unsigned int chunksize, + bool is_encrypt, bool is_esp); +u32 spu2_wordalign_padlen(u32 data_size); +#endif diff --git a/drivers/crypto/bcm/spum.h b/drivers/crypto/bcm/spum.h new file mode 100644 index 000000000..f062f7580 --- /dev/null +++ b/drivers/crypto/bcm/spum.h @@ -0,0 +1,163 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2016 Broadcom + */ + +/* + * This file contains SPU message definitions specific to SPU-M. + */ + +#ifndef _SPUM_H_ +#define _SPUM_H_ + +#define SPU_CRYPTO_OPERATION_GENERIC 0x1 + +/* Length of STATUS field in tx and rx packets */ +#define SPU_TX_STATUS_LEN 4 + +/* SPU-M error codes */ +#define SPU_STATUS_MASK 0x0000FF00 +#define SPU_STATUS_SUCCESS 0x00000000 +#define SPU_STATUS_INVALID_ICV 0x00000100 + +#define SPU_STATUS_ERROR_FLAG 0x00020000 + +/* Request message. MH + EMH + BDESC + BD header */ +#define SPU_REQ_FIXED_LEN 24 + +/* + * Max length of a SPU message header. Used to allocate a buffer where + * the SPU message header is constructed. Can be used for either a SPU-M + * header or a SPU2 header. + * For SPU-M, sum of the following: + * MH - 4 bytes + * EMH - 4 + * SCTX - 3 + + * max auth key len - 64 + * max cipher key len - 264 (RC4) + * max IV len - 16 + * BDESC - 12 + * BD header - 4 + * Total: 371 + * + * For SPU2, FMD_SIZE (32) plus lengths of hash and cipher keys, + * hash and cipher IVs. If SPU2 does not support RC4, then + */ +#define SPU_HEADER_ALLOC_LEN (SPU_REQ_FIXED_LEN + MAX_KEY_SIZE + \ + MAX_KEY_SIZE + MAX_IV_SIZE) + +/* + * Response message header length. Normally MH, EMH, BD header, but when + * BD_SUPPRESS is used for hash requests, there is no BD header. + */ +#define SPU_RESP_HDR_LEN 12 +#define SPU_HASH_RESP_HDR_LEN 8 + +/* + * Max value that can be represented in the Payload Length field of the BD + * header. This is a 16-bit field. + */ +#define SPUM_NS2_MAX_PAYLOAD (BIT(16) - 1) + +/* + * NSP SPU is limited to ~9KB because of FA2 FIFO size limitations; + * Set MAX_PAYLOAD to 8k to allow for addition of header, digest, etc. + * and stay within limitation. + */ + +#define SPUM_NSP_MAX_PAYLOAD 8192 + +/* Buffer Descriptor Header [BDESC]. SPU in big-endian mode. */ +struct BDESC_HEADER { + __be16 offset_mac; /* word 0 [31-16] */ + __be16 length_mac; /* word 0 [15-0] */ + __be16 offset_crypto; /* word 1 [31-16] */ + __be16 length_crypto; /* word 1 [15-0] */ + __be16 offset_icv; /* word 2 [31-16] */ + __be16 offset_iv; /* word 2 [15-0] */ +}; + +/* Buffer Data Header [BD]. SPU in big-endian mode. */ +struct BD_HEADER { + __be16 size; + __be16 prev_length; +}; + +/* Command Context Header. SPU-M in big endian mode. */ +struct MHEADER { + u8 flags; /* [31:24] */ + u8 op_code; /* [23:16] */ + u16 reserved; /* [15:0] */ +}; + +/* MH header flags bits */ +#define MH_SUPDT_PRES BIT(0) +#define MH_HASH_PRES BIT(2) +#define MH_BD_PRES BIT(3) +#define MH_MFM_PRES BIT(4) +#define MH_BDESC_PRES BIT(5) +#define MH_SCTX_PRES BIT(7) + +/* SCTX word 0 bit offsets and fields masks */ +#define SCTX_SIZE 0x000000FF + +/* SCTX word 1 bit shifts and field masks */ +#define UPDT_OFST 0x000000FF /* offset of SCTX updateable fld */ +#define HASH_TYPE 0x00000300 /* hash alg operation type */ +#define HASH_TYPE_SHIFT 8 +#define HASH_MODE 0x00001C00 /* one of spu2_hash_mode */ +#define HASH_MODE_SHIFT 10 +#define HASH_ALG 0x0000E000 /* hash algorithm */ +#define HASH_ALG_SHIFT 13 +#define CIPHER_TYPE 0x00030000 /* encryption operation type */ +#define CIPHER_TYPE_SHIFT 16 +#define CIPHER_MODE 0x001C0000 /* encryption mode */ +#define CIPHER_MODE_SHIFT 18 +#define CIPHER_ALG 0x00E00000 /* encryption algo */ +#define CIPHER_ALG_SHIFT 21 +#define ICV_IS_512 BIT(27) +#define ICV_IS_512_SHIFT 27 +#define CIPHER_ORDER BIT(30) +#define CIPHER_ORDER_SHIFT 30 +#define CIPHER_INBOUND BIT(31) +#define CIPHER_INBOUND_SHIFT 31 + +/* SCTX word 2 bit shifts and field masks */ +#define EXP_IV_SIZE 0x7 +#define IV_OFFSET BIT(3) +#define IV_OFFSET_SHIFT 3 +#define GEN_IV BIT(5) +#define GEN_IV_SHIFT 5 +#define EXPLICIT_IV BIT(6) +#define EXPLICIT_IV_SHIFT 6 +#define SCTX_IV BIT(7) +#define SCTX_IV_SHIFT 7 +#define ICV_SIZE 0x0F00 +#define ICV_SIZE_SHIFT 8 +#define CHECK_ICV BIT(12) +#define CHECK_ICV_SHIFT 12 +#define INSERT_ICV BIT(13) +#define INSERT_ICV_SHIFT 13 +#define BD_SUPPRESS BIT(19) +#define BD_SUPPRESS_SHIFT 19 + +/* Generic Mode Security Context Structure [SCTX] */ +struct SCTX { +/* word 0: protocol flags */ + __be32 proto_flags; + +/* word 1: cipher flags */ + __be32 cipher_flags; + +/* word 2: Extended cipher flags */ + __be32 ecf; + +}; + +struct SPUHEADER { + struct MHEADER mh; + u32 emh; + struct SCTX sa; +}; + +#endif /* _SPUM_H_ */ diff --git a/drivers/crypto/bcm/util.c b/drivers/crypto/bcm/util.c new file mode 100644 index 000000000..d5d9cabea --- /dev/null +++ b/drivers/crypto/bcm/util.c @@ -0,0 +1,530 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2016 Broadcom + */ + +#include + +#include "cipher.h" +#include "util.h" + +/* offset of SPU_OFIFO_CTRL register */ +#define SPU_OFIFO_CTRL 0x40 +#define SPU_FIFO_WATERMARK 0x1FF + +/** + * spu_sg_at_offset() - Find the scatterlist entry at a given distance from the + * start of a scatterlist. + * @sg: [in] Start of a scatterlist + * @skip: [in] Distance from the start of the scatterlist, in bytes + * @sge: [out] Scatterlist entry at skip bytes from start + * @sge_offset: [out] Number of bytes from start of sge buffer to get to + * requested distance. + * + * Return: 0 if entry found at requested distance + * < 0 otherwise + */ +int spu_sg_at_offset(struct scatterlist *sg, unsigned int skip, + struct scatterlist **sge, unsigned int *sge_offset) +{ + /* byte index from start of sg to the end of the previous entry */ + unsigned int index = 0; + /* byte index from start of sg to the end of the current entry */ + unsigned int next_index; + + next_index = sg->length; + while (next_index <= skip) { + sg = sg_next(sg); + index = next_index; + if (!sg) + return -EINVAL; + next_index += sg->length; + } + + *sge_offset = skip - index; + *sge = sg; + return 0; +} + +/* Copy len bytes of sg data, starting at offset skip, to a dest buffer */ +void sg_copy_part_to_buf(struct scatterlist *src, u8 *dest, + unsigned int len, unsigned int skip) +{ + size_t copied; + unsigned int nents = sg_nents(src); + + copied = sg_pcopy_to_buffer(src, nents, dest, len, skip); + if (copied != len) { + flow_log("%s copied %u bytes of %u requested. ", + __func__, (u32)copied, len); + flow_log("sg with %u entries and skip %u\n", nents, skip); + } +} + +/* + * Copy data into a scatterlist starting at a specified offset in the + * scatterlist. Specifically, copy len bytes of data in the buffer src + * into the scatterlist dest, starting skip bytes into the scatterlist. + */ +void sg_copy_part_from_buf(struct scatterlist *dest, u8 *src, + unsigned int len, unsigned int skip) +{ + size_t copied; + unsigned int nents = sg_nents(dest); + + copied = sg_pcopy_from_buffer(dest, nents, src, len, skip); + if (copied != len) { + flow_log("%s copied %u bytes of %u requested. ", + __func__, (u32)copied, len); + flow_log("sg with %u entries and skip %u\n", nents, skip); + } +} + +/** + * spu_sg_count() - Determine number of elements in scatterlist to provide a + * specified number of bytes. + * @sg_list: scatterlist to examine + * @skip: index of starting point + * @nbytes: consider elements of scatterlist until reaching this number of + * bytes + * + * Return: the number of sg entries contributing to nbytes of data + */ +int spu_sg_count(struct scatterlist *sg_list, unsigned int skip, int nbytes) +{ + struct scatterlist *sg; + int sg_nents = 0; + unsigned int offset; + + if (!sg_list) + return 0; + + if (spu_sg_at_offset(sg_list, skip, &sg, &offset) < 0) + return 0; + + while (sg && (nbytes > 0)) { + sg_nents++; + nbytes -= (sg->length - offset); + offset = 0; + sg = sg_next(sg); + } + return sg_nents; +} + +/** + * spu_msg_sg_add() - Copy scatterlist entries from one sg to another, up to a + * given length. + * @to_sg: scatterlist to copy to + * @from_sg: scatterlist to copy from + * @from_skip: number of bytes to skip in from_sg. Non-zero when previous + * request included part of the buffer in entry in from_sg. + * Assumes from_skip < from_sg->length. + * @from_nents: number of entries in from_sg + * @length: number of bytes to copy. may reach this limit before exhausting + * from_sg. + * + * Copies the entries themselves, not the data in the entries. Assumes to_sg has + * enough entries. Does not limit the size of an individual buffer in to_sg. + * + * to_sg, from_sg, skip are all updated to end of copy + * + * Return: Number of bytes copied + */ +u32 spu_msg_sg_add(struct scatterlist **to_sg, + struct scatterlist **from_sg, u32 *from_skip, + u8 from_nents, u32 length) +{ + struct scatterlist *sg; /* an entry in from_sg */ + struct scatterlist *to = *to_sg; + struct scatterlist *from = *from_sg; + u32 skip = *from_skip; + u32 offset; + int i; + u32 entry_len = 0; + u32 frag_len = 0; /* length of entry added to to_sg */ + u32 copied = 0; /* number of bytes copied so far */ + + if (length == 0) + return 0; + + for_each_sg(from, sg, from_nents, i) { + /* number of bytes in this from entry not yet used */ + entry_len = sg->length - skip; + frag_len = min(entry_len, length - copied); + offset = sg->offset + skip; + if (frag_len) + sg_set_page(to++, sg_page(sg), frag_len, offset); + copied += frag_len; + if (copied == entry_len) { + /* used up all of from entry */ + skip = 0; /* start at beginning of next entry */ + } + if (copied == length) + break; + } + *to_sg = to; + *from_sg = sg; + if (frag_len < entry_len) + *from_skip = skip + frag_len; + else + *from_skip = 0; + + return copied; +} + +void add_to_ctr(u8 *ctr_pos, unsigned int increment) +{ + __be64 *high_be = (__be64 *)ctr_pos; + __be64 *low_be = high_be + 1; + u64 orig_low = __be64_to_cpu(*low_be); + u64 new_low = orig_low + (u64)increment; + + *low_be = __cpu_to_be64(new_low); + if (new_low < orig_low) + /* there was a carry from the low 8 bytes */ + *high_be = __cpu_to_be64(__be64_to_cpu(*high_be) + 1); +} + +struct sdesc { + struct shash_desc shash; + char ctx[]; +}; + +/** + * do_shash() - Do a synchronous hash operation in software + * @name: The name of the hash algorithm + * @result: Buffer where digest is to be written + * @data1: First part of data to hash. May be NULL. + * @data1_len: Length of data1, in bytes + * @data2: Second part of data to hash. May be NULL. + * @data2_len: Length of data2, in bytes + * @key: Key (if keyed hash) + * @key_len: Length of key, in bytes (or 0 if non-keyed hash) + * + * Note that the crypto API will not select this driver's own transform because + * this driver only registers asynchronous algos. + * + * Return: 0 if hash successfully stored in result + * < 0 otherwise + */ +int do_shash(unsigned char *name, unsigned char *result, + const u8 *data1, unsigned int data1_len, + const u8 *data2, unsigned int data2_len, + const u8 *key, unsigned int key_len) +{ + int rc; + unsigned int size; + struct crypto_shash *hash; + struct sdesc *sdesc; + + hash = crypto_alloc_shash(name, 0, 0); + if (IS_ERR(hash)) { + rc = PTR_ERR(hash); + pr_err("%s: Crypto %s allocation error %d\n", __func__, name, rc); + return rc; + } + + size = sizeof(struct shash_desc) + crypto_shash_descsize(hash); + sdesc = kmalloc(size, GFP_KERNEL); + if (!sdesc) { + rc = -ENOMEM; + goto do_shash_err; + } + sdesc->shash.tfm = hash; + + if (key_len > 0) { + rc = crypto_shash_setkey(hash, key, key_len); + if (rc) { + pr_err("%s: Could not setkey %s shash\n", __func__, name); + goto do_shash_err; + } + } + + rc = crypto_shash_init(&sdesc->shash); + if (rc) { + pr_err("%s: Could not init %s shash\n", __func__, name); + goto do_shash_err; + } + rc = crypto_shash_update(&sdesc->shash, data1, data1_len); + if (rc) { + pr_err("%s: Could not update1\n", __func__); + goto do_shash_err; + } + if (data2 && data2_len) { + rc = crypto_shash_update(&sdesc->shash, data2, data2_len); + if (rc) { + pr_err("%s: Could not update2\n", __func__); + goto do_shash_err; + } + } + rc = crypto_shash_final(&sdesc->shash, result); + if (rc) + pr_err("%s: Could not generate %s hash\n", __func__, name); + +do_shash_err: + crypto_free_shash(hash); + kfree(sdesc); + + return rc; +} + +#ifdef DEBUG +/* Dump len bytes of a scatterlist starting at skip bytes into the sg */ +void __dump_sg(struct scatterlist *sg, unsigned int skip, unsigned int len) +{ + u8 dbuf[16]; + unsigned int idx = skip; + unsigned int num_out = 0; /* number of bytes dumped so far */ + unsigned int count; + + if (packet_debug_logging) { + while (num_out < len) { + count = (len - num_out > 16) ? 16 : len - num_out; + sg_copy_part_to_buf(sg, dbuf, count, idx); + num_out += count; + print_hex_dump(KERN_ALERT, " sg: ", DUMP_PREFIX_NONE, + 4, 1, dbuf, count, false); + idx += 16; + } + } + if (debug_logging_sleep) + msleep(debug_logging_sleep); +} +#endif + +/* Returns the name for a given cipher alg/mode */ +char *spu_alg_name(enum spu_cipher_alg alg, enum spu_cipher_mode mode) +{ + switch (alg) { + case CIPHER_ALG_RC4: + return "rc4"; + case CIPHER_ALG_AES: + switch (mode) { + case CIPHER_MODE_CBC: + return "cbc(aes)"; + case CIPHER_MODE_ECB: + return "ecb(aes)"; + case CIPHER_MODE_OFB: + return "ofb(aes)"; + case CIPHER_MODE_CFB: + return "cfb(aes)"; + case CIPHER_MODE_CTR: + return "ctr(aes)"; + case CIPHER_MODE_XTS: + return "xts(aes)"; + case CIPHER_MODE_GCM: + return "gcm(aes)"; + default: + return "aes"; + } + break; + case CIPHER_ALG_DES: + switch (mode) { + case CIPHER_MODE_CBC: + return "cbc(des)"; + case CIPHER_MODE_ECB: + return "ecb(des)"; + case CIPHER_MODE_CTR: + return "ctr(des)"; + default: + return "des"; + } + break; + case CIPHER_ALG_3DES: + switch (mode) { + case CIPHER_MODE_CBC: + return "cbc(des3_ede)"; + case CIPHER_MODE_ECB: + return "ecb(des3_ede)"; + case CIPHER_MODE_CTR: + return "ctr(des3_ede)"; + default: + return "3des"; + } + break; + default: + return "other"; + } +} + +static ssize_t spu_debugfs_read(struct file *filp, char __user *ubuf, + size_t count, loff_t *offp) +{ + struct bcm_device_private *ipriv; + char *buf; + ssize_t ret, out_offset, out_count; + int i; + u32 fifo_len; + u32 spu_ofifo_ctrl; + u32 alg; + u32 mode; + u32 op_cnt; + + out_count = 2048; + + buf = kmalloc(out_count, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + ipriv = filp->private_data; + out_offset = 0; + out_offset += scnprintf(buf + out_offset, out_count - out_offset, + "Number of SPUs.........%u\n", + ipriv->spu.num_spu); + out_offset += scnprintf(buf + out_offset, out_count - out_offset, + "Current sessions.......%u\n", + atomic_read(&ipriv->session_count)); + out_offset += scnprintf(buf + out_offset, out_count - out_offset, + "Session count..........%u\n", + atomic_read(&ipriv->stream_count)); + out_offset += scnprintf(buf + out_offset, out_count - out_offset, + "Cipher setkey..........%u\n", + atomic_read(&ipriv->setkey_cnt[SPU_OP_CIPHER])); + out_offset += scnprintf(buf + out_offset, out_count - out_offset, + "Cipher Ops.............%u\n", + atomic_read(&ipriv->op_counts[SPU_OP_CIPHER])); + for (alg = 0; alg < CIPHER_ALG_LAST; alg++) { + for (mode = 0; mode < CIPHER_MODE_LAST; mode++) { + op_cnt = atomic_read(&ipriv->cipher_cnt[alg][mode]); + if (op_cnt) { + out_offset += scnprintf(buf + out_offset, + out_count - out_offset, + " %-13s%11u\n", + spu_alg_name(alg, mode), op_cnt); + } + } + } + out_offset += scnprintf(buf + out_offset, out_count - out_offset, + "Hash Ops...............%u\n", + atomic_read(&ipriv->op_counts[SPU_OP_HASH])); + for (alg = 0; alg < HASH_ALG_LAST; alg++) { + op_cnt = atomic_read(&ipriv->hash_cnt[alg]); + if (op_cnt) { + out_offset += scnprintf(buf + out_offset, + out_count - out_offset, + " %-13s%11u\n", + hash_alg_name[alg], op_cnt); + } + } + out_offset += scnprintf(buf + out_offset, out_count - out_offset, + "HMAC setkey............%u\n", + atomic_read(&ipriv->setkey_cnt[SPU_OP_HMAC])); + out_offset += scnprintf(buf + out_offset, out_count - out_offset, + "HMAC Ops...............%u\n", + atomic_read(&ipriv->op_counts[SPU_OP_HMAC])); + for (alg = 0; alg < HASH_ALG_LAST; alg++) { + op_cnt = atomic_read(&ipriv->hmac_cnt[alg]); + if (op_cnt) { + out_offset += scnprintf(buf + out_offset, + out_count - out_offset, + " %-13s%11u\n", + hash_alg_name[alg], op_cnt); + } + } + out_offset += scnprintf(buf + out_offset, out_count - out_offset, + "AEAD setkey............%u\n", + atomic_read(&ipriv->setkey_cnt[SPU_OP_AEAD])); + + out_offset += scnprintf(buf + out_offset, out_count - out_offset, + "AEAD Ops...............%u\n", + atomic_read(&ipriv->op_counts[SPU_OP_AEAD])); + for (alg = 0; alg < AEAD_TYPE_LAST; alg++) { + op_cnt = atomic_read(&ipriv->aead_cnt[alg]); + if (op_cnt) { + out_offset += scnprintf(buf + out_offset, + out_count - out_offset, + " %-13s%11u\n", + aead_alg_name[alg], op_cnt); + } + } + out_offset += scnprintf(buf + out_offset, out_count - out_offset, + "Bytes of req data......%llu\n", + (u64)atomic64_read(&ipriv->bytes_out)); + out_offset += scnprintf(buf + out_offset, out_count - out_offset, + "Bytes of resp data.....%llu\n", + (u64)atomic64_read(&ipriv->bytes_in)); + out_offset += scnprintf(buf + out_offset, out_count - out_offset, + "Mailbox full...........%u\n", + atomic_read(&ipriv->mb_no_spc)); + out_offset += scnprintf(buf + out_offset, out_count - out_offset, + "Mailbox send failures..%u\n", + atomic_read(&ipriv->mb_send_fail)); + out_offset += scnprintf(buf + out_offset, out_count - out_offset, + "Check ICV errors.......%u\n", + atomic_read(&ipriv->bad_icv)); + if (ipriv->spu.spu_type == SPU_TYPE_SPUM) + for (i = 0; i < ipriv->spu.num_spu; i++) { + spu_ofifo_ctrl = ioread32(ipriv->spu.reg_vbase[i] + + SPU_OFIFO_CTRL); + fifo_len = spu_ofifo_ctrl & SPU_FIFO_WATERMARK; + out_offset += scnprintf(buf + out_offset, + out_count - out_offset, + "SPU %d output FIFO high water.....%u\n", + i, fifo_len); + } + + if (out_offset > out_count) + out_offset = out_count; + + ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset); + kfree(buf); + return ret; +} + +static const struct file_operations spu_debugfs_stats = { + .owner = THIS_MODULE, + .open = simple_open, + .read = spu_debugfs_read, +}; + +/* + * Create the debug FS directories. If the top-level directory has not yet + * been created, create it now. Create a stats file in this directory for + * a SPU. + */ +void spu_setup_debugfs(void) +{ + if (!debugfs_initialized()) + return; + + if (!iproc_priv.debugfs_dir) + iproc_priv.debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, + NULL); + + if (!iproc_priv.debugfs_stats) + /* Create file with permissions S_IRUSR */ + debugfs_create_file("stats", 0400, iproc_priv.debugfs_dir, + &iproc_priv, &spu_debugfs_stats); +} + +void spu_free_debugfs(void) +{ + debugfs_remove_recursive(iproc_priv.debugfs_dir); + iproc_priv.debugfs_dir = NULL; +} + +/** + * format_value_ccm() - Format a value into a buffer, using a specified number + * of bytes (i.e. maybe writing value X into a 4 byte + * buffer, or maybe into a 12 byte buffer), as per the + * SPU CCM spec. + * + * @val: value to write (up to max of unsigned int) + * @buf: (pointer to) buffer to write the value + * @len: number of bytes to use (0 to 255) + * + */ +void format_value_ccm(unsigned int val, u8 *buf, u8 len) +{ + int i; + + /* First clear full output buffer */ + memset(buf, 0, len); + + /* Then, starting from right side, fill in with data */ + for (i = 0; i < len; i++) { + buf[len - i - 1] = (val >> (8 * i)) & 0xff; + if (i >= 3) + break; /* Only handle up to 32 bits of 'val' */ + } +} diff --git a/drivers/crypto/bcm/util.h b/drivers/crypto/bcm/util.h new file mode 100644 index 000000000..61c256384 --- /dev/null +++ b/drivers/crypto/bcm/util.h @@ -0,0 +1,113 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2016 Broadcom + */ + +#ifndef _UTIL_H +#define _UTIL_H + +#include +#include + +#include "spu.h" + +extern int flow_debug_logging; +extern int packet_debug_logging; +extern int debug_logging_sleep; + +#ifdef DEBUG +#define flow_log(...) \ + do { \ + if (flow_debug_logging) { \ + printk(__VA_ARGS__); \ + if (debug_logging_sleep) \ + msleep(debug_logging_sleep); \ + } \ + } while (0) +#define flow_dump(msg, var, var_len) \ + do { \ + if (flow_debug_logging) { \ + print_hex_dump(KERN_ALERT, msg, DUMP_PREFIX_NONE, \ + 16, 1, var, var_len, false); \ + if (debug_logging_sleep) \ + msleep(debug_logging_sleep); \ + } \ + } while (0) + +#define packet_log(...) \ + do { \ + if (packet_debug_logging) { \ + printk(__VA_ARGS__); \ + if (debug_logging_sleep) \ + msleep(debug_logging_sleep); \ + } \ + } while (0) +#define packet_dump(msg, var, var_len) \ + do { \ + if (packet_debug_logging) { \ + print_hex_dump(KERN_ALERT, msg, DUMP_PREFIX_NONE, \ + 16, 1, var, var_len, false); \ + if (debug_logging_sleep) \ + msleep(debug_logging_sleep); \ + } \ + } while (0) + +void __dump_sg(struct scatterlist *sg, unsigned int skip, unsigned int len); + +#define dump_sg(sg, skip, len) __dump_sg(sg, skip, len) + +#else /* !DEBUG_ON */ + +static inline void flow_log(const char *format, ...) +{ +} + +static inline void flow_dump(const char *msg, const void *var, size_t var_len) +{ +} + +static inline void packet_log(const char *format, ...) +{ +} + +static inline void packet_dump(const char *msg, const void *var, size_t var_len) +{ +} + +static inline void dump_sg(struct scatterlist *sg, unsigned int skip, + unsigned int len) +{ +} + +#endif /* DEBUG_ON */ + +int spu_sg_at_offset(struct scatterlist *sg, unsigned int skip, + struct scatterlist **sge, unsigned int *sge_offset); + +/* Copy sg data, from skip, length len, to dest */ +void sg_copy_part_to_buf(struct scatterlist *src, u8 *dest, + unsigned int len, unsigned int skip); +/* Copy src into scatterlist from offset, length len */ +void sg_copy_part_from_buf(struct scatterlist *dest, u8 *src, + unsigned int len, unsigned int skip); + +int spu_sg_count(struct scatterlist *sg_list, unsigned int skip, int nbytes); +u32 spu_msg_sg_add(struct scatterlist **to_sg, + struct scatterlist **from_sg, u32 *skip, + u8 from_nents, u32 tot_len); + +void add_to_ctr(u8 *ctr_pos, unsigned int increment); + +/* produce a message digest from data of length n bytes */ +int do_shash(unsigned char *name, unsigned char *result, + const u8 *data1, unsigned int data1_len, + const u8 *data2, unsigned int data2_len, + const u8 *key, unsigned int key_len); + +char *spu_alg_name(enum spu_cipher_alg alg, enum spu_cipher_mode mode); + +void spu_setup_debugfs(void); +void spu_free_debugfs(void); +void format_value_ccm(unsigned int val, u8 *buf, u8 len); + +#endif diff --git a/drivers/crypto/caam/Kconfig b/drivers/crypto/caam/Kconfig new file mode 100644 index 000000000..ec57bf6aa --- /dev/null +++ b/drivers/crypto/caam/Kconfig @@ -0,0 +1,188 @@ +# SPDX-License-Identifier: GPL-2.0 +config CRYPTO_DEV_FSL_CAAM_COMMON + tristate + +config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC + tristate + +config CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC + tristate + +config CRYPTO_DEV_FSL_CAAM + tristate "Freescale CAAM-Multicore platform driver backend" + depends on FSL_SOC || ARCH_MXC || ARCH_LAYERSCAPE || COMPILE_TEST + select SOC_BUS + select CRYPTO_DEV_FSL_CAAM_COMMON + imply FSL_MC_BUS + help + Enables the driver module for Freescale's Cryptographic Accelerator + and Assurance Module (CAAM), also known as the SEC version 4 (SEC4). + This module creates job ring devices, and configures h/w + to operate as a DPAA component automatically, depending + on h/w feature availability. + + To compile this driver as a module, choose M here: the module + will be called caam. + +if CRYPTO_DEV_FSL_CAAM + +config CRYPTO_DEV_FSL_CAAM_DEBUG + bool "Enable debug output in CAAM driver" + help + Selecting this will enable printing of various debug + information in the CAAM driver. + +menuconfig CRYPTO_DEV_FSL_CAAM_JR + tristate "Freescale CAAM Job Ring driver backend" + select CRYPTO_ENGINE + default y + help + Enables the driver module for Job Rings which are part of + Freescale's Cryptographic Accelerator + and Assurance Module (CAAM). This module adds a job ring operation + interface. + + To compile this driver as a module, choose M here: the module + will be called caam_jr. + +if CRYPTO_DEV_FSL_CAAM_JR + +config CRYPTO_DEV_FSL_CAAM_RINGSIZE + int "Job Ring size" + range 2 9 + default "9" + help + Select size of Job Rings as a power of 2, within the + range 2-9 (ring size 4-512). + Examples: + 2 => 4 + 3 => 8 + 4 => 16 + 5 => 32 + 6 => 64 + 7 => 128 + 8 => 256 + 9 => 512 + +config CRYPTO_DEV_FSL_CAAM_INTC + bool "Job Ring interrupt coalescing" + help + Enable the Job Ring's interrupt coalescing feature. + + Note: the driver already provides adequate + interrupt coalescing in software. + +config CRYPTO_DEV_FSL_CAAM_INTC_COUNT_THLD + int "Job Ring interrupt coalescing count threshold" + depends on CRYPTO_DEV_FSL_CAAM_INTC + range 1 255 + default 255 + help + Select number of descriptor completions to queue before + raising an interrupt, in the range 1-255. Note that a selection + of 1 functionally defeats the coalescing feature, and a selection + equal or greater than the job ring size will force timeouts. + +config CRYPTO_DEV_FSL_CAAM_INTC_TIME_THLD + int "Job Ring interrupt coalescing timer threshold" + depends on CRYPTO_DEV_FSL_CAAM_INTC + range 1 65535 + default 2048 + help + Select number of bus clocks/64 to timeout in the case that one or + more descriptor completions are queued without reaching the count + threshold. Range is 1-65535. + +config CRYPTO_DEV_FSL_CAAM_CRYPTO_API + bool "Register algorithm implementations with the Crypto API" + default y + select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC + select CRYPTO_AEAD + select CRYPTO_AUTHENC + select CRYPTO_SKCIPHER + select CRYPTO_LIB_DES + select CRYPTO_XTS + help + Selecting this will offload crypto for users of the + scatterlist crypto API (such as the linux native IPSec + stack) to the SEC4 via job ring. + +config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI + bool "Queue Interface as Crypto API backend" + depends on FSL_DPAA && NET + default y + select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC + select CRYPTO_AUTHENC + select CRYPTO_SKCIPHER + select CRYPTO_DES + select CRYPTO_XTS + help + Selecting this will use CAAM Queue Interface (QI) for sending + & receiving crypto jobs to/from CAAM. This gives better performance + than job ring interface when the number of cores are more than the + number of job rings assigned to the kernel. The number of portals + assigned to the kernel should also be more than the number of + job rings. + +config CRYPTO_DEV_FSL_CAAM_AHASH_API + bool "Register hash algorithm implementations with Crypto API" + default y + select CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC + select CRYPTO_HASH + help + Selecting this will offload ahash for users of the + scatterlist crypto API to the SEC4 via job ring. + +config CRYPTO_DEV_FSL_CAAM_PKC_API + bool "Register public key cryptography implementations with Crypto API" + default y + select CRYPTO_RSA + help + Selecting this will allow SEC Public key support for RSA. + Supported cryptographic primitives: encryption, decryption, + signature and verification. + +config CRYPTO_DEV_FSL_CAAM_RNG_API + bool "Register caam device for hwrng API" + default y + select HW_RANDOM + help + Selecting this will register the SEC4 hardware rng to + the hw_random API for supplying the kernel entropy pool. + +config CRYPTO_DEV_FSL_CAAM_BLOB_GEN + bool + +config CRYPTO_DEV_FSL_CAAM_RNG_TEST + bool "Test caam rng" + select CRYPTO_DEV_FSL_CAAM_RNG_API + help + Selecting this will enable a self-test to run for the + caam RNG. + This test is several minutes long and executes + just before the RNG is registered with the hw_random API. + +endif # CRYPTO_DEV_FSL_CAAM_JR + +endif # CRYPTO_DEV_FSL_CAAM + +config CRYPTO_DEV_FSL_DPAA2_CAAM + tristate "QorIQ DPAA2 CAAM (DPSECI) driver" + depends on FSL_MC_DPIO + depends on NETDEVICES + select CRYPTO_DEV_FSL_CAAM_COMMON + select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC + select CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC + select CRYPTO_SKCIPHER + select CRYPTO_AUTHENC + select CRYPTO_AEAD + select CRYPTO_HASH + select CRYPTO_DES + select CRYPTO_XTS + help + CAAM driver for QorIQ Data Path Acceleration Architecture 2. + It handles DPSECI DPAA2 objects that sit on the Management Complex + (MC) fsl-mc bus. + + To compile this as a module, choose M here: the module + will be called dpaa2_caam. diff --git a/drivers/crypto/caam/Makefile b/drivers/crypto/caam/Makefile new file mode 100644 index 000000000..a17e3cf14 --- /dev/null +++ b/drivers/crypto/caam/Makefile @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the CAAM backend and dependent components +# +ifeq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_DEBUG), y) + ccflags-y := -DDEBUG +endif + +ccflags-y += -DVERSION=\"\" + +obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_COMMON) += error.o +obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam.o +obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_JR) += caam_jr.o +obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC) += caamalg_desc.o +obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC) += caamhash_desc.o + +caam-y := ctrl.o +caam_jr-y := jr.o key_gen.o +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += caamalg_qi.o +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caampkc.o pkc_desc.o +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_BLOB_GEN) += blob_gen.o + +caam-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += qi.o +caam-$(CONFIG_DEBUG_FS) += debugfs.o + +obj-$(CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM) += dpaa2_caam.o + +dpaa2_caam-y := caamalg_qi2.o dpseci.o +dpaa2_caam-$(CONFIG_DEBUG_FS) += dpseci-debugfs.o diff --git a/drivers/crypto/caam/blob_gen.c b/drivers/crypto/caam/blob_gen.c new file mode 100644 index 000000000..c18dbac56 --- /dev/null +++ b/drivers/crypto/caam/blob_gen.c @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2015 Pengutronix, Steffen Trumtrar + * Copyright (C) 2021 Pengutronix, Ahmad Fatoum + * Copyright 2024-2025 NXP + */ + +#define pr_fmt(fmt) "caam blob_gen: " fmt + +#include +#include +#include +#include + +#include "compat.h" +#include "desc_constr.h" +#include "desc.h" +#include "error.h" +#include "intern.h" +#include "jr.h" +#include "regs.h" + +#define CAAM_BLOB_DESC_BYTES_MAX \ + /* Command to initialize & stating length of descriptor */ \ + (CAAM_CMD_SZ + \ + /* Command to append the key-modifier + key-modifier data */ \ + CAAM_CMD_SZ + CAAM_BLOB_KEYMOD_LENGTH + \ + /* Command to include input key + pointer to the input key */ \ + CAAM_CMD_SZ + CAAM_PTR_SZ_MAX + \ + /* Command to include output key + pointer to the output key */ \ + CAAM_CMD_SZ + CAAM_PTR_SZ_MAX + \ + /* Command describing the operation to perform */ \ + CAAM_CMD_SZ) + +struct caam_blob_priv { + struct device jrdev; +}; + +struct caam_blob_job_result { + int err; + struct completion completion; +}; + +static void caam_blob_job_done(struct device *dev, u32 *desc, u32 err, void *context) +{ + struct caam_blob_job_result *res = context; + int ecode = 0; + + dev_dbg(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); + + if (err) + ecode = caam_jr_strstatus(dev, err); + + res->err = ecode; + + /* + * Upon completion, desc points to a buffer containing a CAAM job + * descriptor which encapsulates data into an externally-storable + * blob. + */ + complete(&res->completion); +} + +static u32 check_caam_state(struct device *jrdev) +{ + const struct caam_drv_private *ctrlpriv; + + ctrlpriv = dev_get_drvdata(jrdev->parent); + return FIELD_GET(CSTA_MOO, rd_reg32(&ctrlpriv->jr[0]->perfmon.status)); +} + +int caam_process_blob(struct caam_blob_priv *priv, + struct caam_blob_info *info, bool encap) +{ + struct caam_blob_job_result testres; + struct device *jrdev = &priv->jrdev; + dma_addr_t dma_in, dma_out; + int op = OP_PCLID_BLOB; + int hwbk_caam_ovhd = 0; + size_t output_len; + u32 *desc; + u32 moo; + int ret; + int len; + + if (info->key_mod_len > CAAM_BLOB_KEYMOD_LENGTH) + return -EINVAL; + + if (encap) { + op |= OP_TYPE_ENCAP_PROTOCOL; + output_len = info->input_len + CAAM_BLOB_OVERHEAD; + } else { + op |= OP_TYPE_DECAP_PROTOCOL; + output_len = info->input_len - CAAM_BLOB_OVERHEAD; + info->output_len = output_len; + } + + if (encap && info->pkey_info.is_pkey) { + op |= OP_PCL_BLOB_BLACK; + if (info->pkey_info.key_enc_algo == CAAM_ENC_ALGO_CCM) { + op |= OP_PCL_BLOB_EKT; + hwbk_caam_ovhd = CAAM_CCM_OVERHEAD; + } + if ((info->input_len + hwbk_caam_ovhd) > MAX_KEY_SIZE) + return -EINVAL; + + len = info->input_len + hwbk_caam_ovhd; + } else { + len = info->input_len; + } + + desc = kzalloc(CAAM_BLOB_DESC_BYTES_MAX, GFP_KERNEL); + if (!desc) + return -ENOMEM; + + dma_in = dma_map_single(jrdev, info->input, len, + encap ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, dma_in)) { + dev_err(jrdev, "unable to map input DMA buffer\n"); + ret = -ENOMEM; + goto out_free; + } + + dma_out = dma_map_single(jrdev, info->output, output_len, + DMA_FROM_DEVICE); + if (dma_mapping_error(jrdev, dma_out)) { + dev_err(jrdev, "unable to map output DMA buffer\n"); + ret = -ENOMEM; + goto out_unmap_in; + } + + moo = check_caam_state(jrdev); + if (moo != CSTA_MOO_SECURE && moo != CSTA_MOO_TRUSTED) + dev_warn(jrdev, + "using insecure test key, enable HAB to use unique device key!\n"); + + /* + * A data blob is encrypted using a blob key (BK); a random number. + * The BK is used as an AES-CCM key. The initial block (B0) and the + * initial counter (Ctr0) are generated automatically and stored in + * Class 1 Context DWords 0+1+2+3. The random BK is stored in the + * Class 1 Key Register. Operation Mode is set to AES-CCM. + */ + init_job_desc(desc, 0); + + if (encap && info->pkey_info.is_pkey) { + /*!1. key command used to load class 1 key register + * from input plain key. + */ + append_key(desc, dma_in, info->input_len, + CLASS_1 | KEY_DEST_CLASS_REG); + /*!2. Fifostore to store protected key from class 1 key register. */ + if (info->pkey_info.key_enc_algo == CAAM_ENC_ALGO_CCM) { + append_fifo_store(desc, dma_in, info->input_len, + LDST_CLASS_1_CCB | + FIFOST_TYPE_KEY_CCM_JKEK); + } else { + append_fifo_store(desc, dma_in, info->input_len, + LDST_CLASS_1_CCB | + FIFOST_TYPE_KEY_KEK); + } + /* + * JUMP_OFFSET specifies the offset of the JUMP target from + * the JUMP command's address in the descriptor buffer. + */ + append_jump(desc, JUMP_COND_NOP | BIT(0) << JUMP_OFFSET_SHIFT); + } + + /*!3. Load class 2 key with key modifier. */ + append_key_as_imm(desc, info->key_mod, info->key_mod_len, + info->key_mod_len, CLASS_2 | KEY_DEST_CLASS_REG); + + /*!4. SEQ IN PTR Command. */ + append_seq_in_ptr(desc, dma_in, info->input_len, 0); + + /*!5. SEQ OUT PTR Command. */ + append_seq_out_ptr(desc, dma_out, output_len, 0); + + /*!6. Blob encapsulation/decapsulation PROTOCOL Command. */ + append_operation(desc, op); + + print_hex_dump_debug("data@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 1, info->input, + len, false); + print_hex_dump_debug("jobdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 1, desc, + desc_bytes(desc), false); + + testres.err = 0; + init_completion(&testres.completion); + + ret = caam_jr_enqueue(jrdev, desc, caam_blob_job_done, &testres); + if (ret == -EINPROGRESS) { + wait_for_completion(&testres.completion); + ret = testres.err; + print_hex_dump_debug("output@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 1, info->output, + output_len, false); + } + + if (ret == 0) + info->output_len = output_len; + + dma_unmap_single(jrdev, dma_out, output_len, DMA_FROM_DEVICE); +out_unmap_in: + dma_unmap_single(jrdev, dma_in, len, + encap ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); +out_free: + kfree(desc); + return ret; +} +EXPORT_SYMBOL(caam_process_blob); + +struct caam_blob_priv *caam_blob_gen_init(void) +{ + struct caam_drv_private *ctrlpriv; + struct device *jrdev; + + /* + * caam_blob_gen_init() may expectedly fail with -ENODEV, e.g. when + * CAAM driver didn't probe or when SoC lacks BLOB support. An + * error would be harsh in this case, so we stick to info level. + */ + + jrdev = caam_jr_alloc(); + if (IS_ERR(jrdev)) { + pr_info("job ring requested, but none currently available\n"); + return ERR_PTR(-ENODEV); + } + + ctrlpriv = dev_get_drvdata(jrdev->parent); + if (!ctrlpriv->blob_present) { + dev_info(jrdev, "no hardware blob generation support\n"); + caam_jr_free(jrdev); + return ERR_PTR(-ENODEV); + } + + return container_of(jrdev, struct caam_blob_priv, jrdev); +} +EXPORT_SYMBOL(caam_blob_gen_init); + +void caam_blob_gen_exit(struct caam_blob_priv *priv) +{ + caam_jr_free(&priv->jrdev); +} +EXPORT_SYMBOL(caam_blob_gen_exit); diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c new file mode 100644 index 000000000..ddbd60cf3 --- /dev/null +++ b/drivers/crypto/caam/caamalg.c @@ -0,0 +1,3988 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * caam - Freescale FSL CAAM support for crypto API + * + * Copyright 2008-2011 Freescale Semiconductor, Inc. + * Copyright 2016-2019, 2023, 2025 NXP + * + * Based on talitos crypto API driver. + * + * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008): + * + * --------------- --------------- + * | JobDesc #1 |-------------------->| ShareDesc | + * | *(packet 1) | | (PDB) | + * --------------- |------------->| (hashKey) | + * . | | (cipherKey) | + * . | |-------->| (operation) | + * --------------- | | --------------- + * | JobDesc #2 |------| | + * | *(packet 2) | | + * --------------- | + * . | + * . | + * --------------- | + * | JobDesc #3 |------------ + * | *(packet 3) | + * --------------- + * + * The SharedDesc never changes for a connection unless rekeyed, but + * each packet will likely be in a different place. So all we need + * to know to process the packet is where the input is, where the + * output goes, and what context we want to process with. Context is + * in the SharedDesc, packet references in the JobDesc. + * + * So, a job desc looks like: + * + * --------------------- + * | Header | + * | ShareDesc Pointer | + * | SEQ_OUT_PTR | + * | (output buffer) | + * | (output length) | + * | SEQ_IN_PTR | + * | (input buffer) | + * | (input length) | + * --------------------- + */ + +#include "compat.h" + +#include "regs.h" +#include "intern.h" +#include "desc_constr.h" +#include "jr.h" +#include "error.h" +#include "sg_sw_sec4.h" +#include "key_gen.h" +#include "caamalg_desc.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * crypto alg + */ +#define CAAM_CRA_PRIORITY 3000 +/* max key is sum of AES_MAX_KEY_SIZE, max split key size */ +#define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ + CTR_RFC3686_NONCE_SIZE + \ + SHA512_DIGEST_SIZE * 2) + +#define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2) +#define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ + CAAM_CMD_SZ * 4) +#define AUTHENC_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ + CAAM_CMD_SZ * 5) + +#define CHACHAPOLY_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + CAAM_CMD_SZ * 6) + +#define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN_MIN) +#define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) + +struct caam_alg_entry { + int class1_alg_type; + int class2_alg_type; + bool rfc3686; + bool geniv; + bool nodkp; +}; + +struct caam_aead_alg { + struct aead_engine_alg aead; + struct caam_alg_entry caam; + bool registered; +}; + +struct caam_skcipher_alg { + struct skcipher_engine_alg skcipher; + struct caam_alg_entry caam; + bool registered; +}; + +/* + * per-session context + */ +struct caam_ctx { + u32 sh_desc_enc[DESC_MAX_USED_LEN]; + u32 sh_desc_dec[DESC_MAX_USED_LEN]; + u8 key[CAAM_MAX_KEY_SIZE]; + dma_addr_t sh_desc_enc_dma; + dma_addr_t sh_desc_dec_dma; + dma_addr_t key_dma; + u8 protected_key[CAAM_MAX_KEY_SIZE]; + dma_addr_t protected_key_dma; + enum dma_data_direction dir; + struct device *jrdev; + struct alginfo adata; + struct alginfo cdata; + unsigned int authsize; + bool xts_key_fallback; + bool is_blob; + struct crypto_skcipher *fallback; +}; + +struct caam_skcipher_req_ctx { + struct skcipher_edesc *edesc; + struct skcipher_request fallback_req; +}; + +struct caam_aead_req_ctx { + struct aead_edesc *edesc; +}; + +static int aead_null_set_sh_desc(struct crypto_aead *aead) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *jrdev = ctx->jrdev; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); + u32 *desc; + int rem_bytes = CAAM_DESC_BYTES_MAX - AEAD_DESC_JOB_IO_LEN - + ctx->adata.keylen_pad; + + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_AEAD_NULL_ENC_LEN) { + ctx->adata.key_inline = true; + ctx->adata.key_virt = ctx->key; + } else { + ctx->adata.key_inline = false; + ctx->adata.key_dma = ctx->key_dma; + } + + /* aead_encrypt shared descriptor */ + desc = ctx->sh_desc_enc; + cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize, + ctrlpriv->era); + dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, + desc_bytes(desc), ctx->dir); + + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_AEAD_NULL_DEC_LEN) { + ctx->adata.key_inline = true; + ctx->adata.key_virt = ctx->key; + } else { + ctx->adata.key_inline = false; + ctx->adata.key_dma = ctx->key_dma; + } + + /* aead_decrypt shared descriptor */ + desc = ctx->sh_desc_dec; + cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize, + ctrlpriv->era); + dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, + desc_bytes(desc), ctx->dir); + + return 0; +} + +static int aead_set_sh_desc(struct crypto_aead *aead) +{ + struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), + struct caam_aead_alg, + aead.base); + unsigned int ivsize = crypto_aead_ivsize(aead); + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *jrdev = ctx->jrdev; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); + u32 ctx1_iv_off = 0; + u32 *desc, *nonce = NULL; + u32 inl_mask; + unsigned int data_len[2]; + const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == + OP_ALG_AAI_CTR_MOD128); + const bool is_rfc3686 = alg->caam.rfc3686; + + if (!ctx->authsize) + return 0; + + /* NULL encryption / decryption */ + if (!ctx->cdata.keylen) + return aead_null_set_sh_desc(aead); + + /* + * AES-CTR needs to load IV in CONTEXT1 reg + * at an offset of 128bits (16bytes) + * CONTEXT1[255:128] = IV + */ + if (ctr_mode) + ctx1_iv_off = 16; + + /* + * RFC3686 specific: + * CONTEXT1[255:128] = {NONCE, IV, COUNTER} + */ + if (is_rfc3686) { + ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; + nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + + ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); + } + + /* + * In case |user key| > |derived key|, using DKP + * would result in invalid opcodes (last bytes of user key) in + * the resulting descriptor. Use DKP instead => both + * virtual and dma key addresses are needed. + */ + ctx->adata.key_virt = ctx->key; + ctx->adata.key_dma = ctx->key_dma; + + ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; + ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; + + data_len[0] = ctx->adata.keylen_pad; + data_len[1] = ctx->cdata.keylen; + + if (alg->caam.geniv) + goto skip_enc; + + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + if (desc_inline_query(DESC_AEAD_ENC_LEN + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), + AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, + ARRAY_SIZE(data_len)) < 0) + return -EINVAL; + + ctx->adata.key_inline = !!(inl_mask & 1); + ctx->cdata.key_inline = !!(inl_mask & 2); + + /* aead_encrypt shared descriptor */ + desc = ctx->sh_desc_enc; + cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ivsize, + ctx->authsize, is_rfc3686, nonce, ctx1_iv_off, + false, ctrlpriv->era); + dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, + desc_bytes(desc), ctx->dir); + +skip_enc: + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + if (desc_inline_query(DESC_AEAD_DEC_LEN + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), + AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, + ARRAY_SIZE(data_len)) < 0) + return -EINVAL; + + ctx->adata.key_inline = !!(inl_mask & 1); + ctx->cdata.key_inline = !!(inl_mask & 2); + + /* aead_decrypt shared descriptor */ + desc = ctx->sh_desc_dec; + cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize, + ctx->authsize, alg->caam.geniv, is_rfc3686, + nonce, ctx1_iv_off, false, ctrlpriv->era); + dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, + desc_bytes(desc), ctx->dir); + + if (!alg->caam.geniv) + goto skip_givenc; + + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + if (desc_inline_query(DESC_AEAD_GIVENC_LEN + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), + AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, + ARRAY_SIZE(data_len)) < 0) + return -EINVAL; + + ctx->adata.key_inline = !!(inl_mask & 1); + ctx->cdata.key_inline = !!(inl_mask & 2); + + /* aead_givencrypt shared descriptor */ + desc = ctx->sh_desc_enc; + cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize, + ctx->authsize, is_rfc3686, nonce, + ctx1_iv_off, false, ctrlpriv->era); + dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, + desc_bytes(desc), ctx->dir); + +skip_givenc: + return 0; +} + +static int aead_setauthsize(struct crypto_aead *authenc, + unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); + + ctx->authsize = authsize; + aead_set_sh_desc(authenc); + + return 0; +} + +static int gcm_set_sh_desc(struct crypto_aead *aead) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *jrdev = ctx->jrdev; + unsigned int ivsize = crypto_aead_ivsize(aead); + u32 *desc; + int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - + ctx->cdata.keylen; + + if (!ctx->cdata.keylen || !ctx->authsize) + return 0; + + /* + * AES GCM encrypt shared descriptor + * Job Descriptor and Shared Descriptor + * must fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_GCM_ENC_LEN) { + ctx->cdata.key_inline = true; + ctx->cdata.key_virt = ctx->key; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } + + desc = ctx->sh_desc_enc; + cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, false); + dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, + desc_bytes(desc), ctx->dir); + + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_GCM_DEC_LEN) { + ctx->cdata.key_inline = true; + ctx->cdata.key_virt = ctx->key; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } + + desc = ctx->sh_desc_dec; + cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, false); + dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, + desc_bytes(desc), ctx->dir); + + return 0; +} + +static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); + int err; + + err = crypto_gcm_check_authsize(authsize); + if (err) + return err; + + ctx->authsize = authsize; + gcm_set_sh_desc(authenc); + + return 0; +} + +static int rfc4106_set_sh_desc(struct crypto_aead *aead) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *jrdev = ctx->jrdev; + unsigned int ivsize = crypto_aead_ivsize(aead); + u32 *desc; + int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - + ctx->cdata.keylen; + + if (!ctx->cdata.keylen || !ctx->authsize) + return 0; + + /* + * RFC4106 encrypt shared descriptor + * Job Descriptor and Shared Descriptor + * must fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_RFC4106_ENC_LEN) { + ctx->cdata.key_inline = true; + ctx->cdata.key_virt = ctx->key; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } + + desc = ctx->sh_desc_enc; + cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize, + false); + dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, + desc_bytes(desc), ctx->dir); + + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_RFC4106_DEC_LEN) { + ctx->cdata.key_inline = true; + ctx->cdata.key_virt = ctx->key; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } + + desc = ctx->sh_desc_dec; + cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize, + false); + dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, + desc_bytes(desc), ctx->dir); + + return 0; +} + +static int rfc4106_setauthsize(struct crypto_aead *authenc, + unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); + int err; + + err = crypto_rfc4106_check_authsize(authsize); + if (err) + return err; + + ctx->authsize = authsize; + rfc4106_set_sh_desc(authenc); + + return 0; +} + +static int rfc4543_set_sh_desc(struct crypto_aead *aead) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *jrdev = ctx->jrdev; + unsigned int ivsize = crypto_aead_ivsize(aead); + u32 *desc; + int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - + ctx->cdata.keylen; + + if (!ctx->cdata.keylen || !ctx->authsize) + return 0; + + /* + * RFC4543 encrypt shared descriptor + * Job Descriptor and Shared Descriptor + * must fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_RFC4543_ENC_LEN) { + ctx->cdata.key_inline = true; + ctx->cdata.key_virt = ctx->key; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } + + desc = ctx->sh_desc_enc; + cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize, + false); + dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, + desc_bytes(desc), ctx->dir); + + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_RFC4543_DEC_LEN) { + ctx->cdata.key_inline = true; + ctx->cdata.key_virt = ctx->key; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } + + desc = ctx->sh_desc_dec; + cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize, + false); + dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, + desc_bytes(desc), ctx->dir); + + return 0; +} + +static int rfc4543_setauthsize(struct crypto_aead *authenc, + unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); + + if (authsize != 16) + return -EINVAL; + + ctx->authsize = authsize; + rfc4543_set_sh_desc(authenc); + + return 0; +} + +static int chachapoly_set_sh_desc(struct crypto_aead *aead) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *jrdev = ctx->jrdev; + unsigned int ivsize = crypto_aead_ivsize(aead); + u32 *desc; + + if (!ctx->cdata.keylen || !ctx->authsize) + return 0; + + desc = ctx->sh_desc_enc; + cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, + ctx->authsize, true, false); + dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, + desc_bytes(desc), ctx->dir); + + desc = ctx->sh_desc_dec; + cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, + ctx->authsize, false, false); + dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, + desc_bytes(desc), ctx->dir); + + return 0; +} + +static int chachapoly_setauthsize(struct crypto_aead *aead, + unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + + if (authsize != POLY1305_DIGEST_SIZE) + return -EINVAL; + + ctx->authsize = authsize; + return chachapoly_set_sh_desc(aead); +} + +static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + unsigned int ivsize = crypto_aead_ivsize(aead); + unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize; + + if (keylen != CHACHA_KEY_SIZE + saltlen) + return -EINVAL; + + memcpy(ctx->key, key, keylen); + ctx->cdata.key_virt = ctx->key; + ctx->cdata.keylen = keylen - saltlen; + + return chachapoly_set_sh_desc(aead); +} + +static int aead_setkey(struct crypto_aead *aead, + const u8 *key, unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *jrdev = ctx->jrdev; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); + struct crypto_authenc_keys keys; + int ret = 0; + + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) + goto badkey; + + dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n", + keys.authkeylen + keys.enckeylen, keys.enckeylen, + keys.authkeylen); + print_hex_dump_devel("key in @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); + + /* + * If DKP is supported, use it in the shared descriptor to generate + * the split key. + */ + if (ctrlpriv->era >= 6) { + ctx->adata.keylen = keys.authkeylen; + ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & + OP_ALG_ALGSEL_MASK); + + if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) + goto badkey; + + memcpy(ctx->key, keys.authkey, keys.authkeylen); + memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, + keys.enckeylen); + dma_sync_single_for_device(jrdev, ctx->key_dma, + ctx->adata.keylen_pad + + keys.enckeylen, ctx->dir); + goto skip_split_key; + } + + ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, keys.authkey, + keys.authkeylen, CAAM_MAX_KEY_SIZE - + keys.enckeylen); + if (ret) { + goto badkey; + } + + /* postpend encryption key to auth split key */ + memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); + dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + + keys.enckeylen, ctx->dir); + + print_hex_dump_devel("ctx.key@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, + ctx->adata.keylen_pad + keys.enckeylen, 1); + +skip_split_key: + ctx->cdata.keylen = keys.enckeylen; + memzero_explicit(&keys, sizeof(keys)); + return aead_set_sh_desc(aead); +badkey: + memzero_explicit(&keys, sizeof(keys)); + return -EINVAL; +} + +static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct crypto_authenc_keys keys; + int err; + + err = crypto_authenc_extractkeys(&keys, key, keylen); + if (unlikely(err)) + return err; + + err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?: + aead_setkey(aead, key, keylen); + + memzero_explicit(&keys, sizeof(keys)); + return err; +} + +static int gcm_setkey(struct crypto_aead *aead, + const u8 *key, unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *jrdev = ctx->jrdev; + int err; + + err = aes_check_keylen(keylen); + if (err) + return err; + + print_hex_dump_devel("key in @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); + + memcpy(ctx->key, key, keylen); + dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir); + ctx->cdata.keylen = keylen; + + return gcm_set_sh_desc(aead); +} + +static int rfc4106_setkey(struct crypto_aead *aead, + const u8 *key, unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *jrdev = ctx->jrdev; + int err; + + err = aes_check_keylen(keylen - 4); + if (err) + return err; + + print_hex_dump_devel("key in @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); + + memcpy(ctx->key, key, keylen); + + /* + * The last four bytes of the key material are used as the salt value + * in the nonce. Update the AES key length. + */ + ctx->cdata.keylen = keylen - 4; + dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, + ctx->dir); + return rfc4106_set_sh_desc(aead); +} + +static int rfc4543_setkey(struct crypto_aead *aead, + const u8 *key, unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *jrdev = ctx->jrdev; + int err; + + err = aes_check_keylen(keylen - 4); + if (err) + return err; + + print_hex_dump_devel("key in @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); + + memcpy(ctx->key, key, keylen); + + /* + * The last four bytes of the key material are used as the salt value + * in the nonce. Update the AES key length. + */ + ctx->cdata.keylen = keylen - 4; + dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, + ctx->dir); + return rfc4543_set_sh_desc(aead); +} + +static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, + unsigned int keylen, const u32 ctx1_iv_off) +{ + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); + struct caam_skcipher_alg *alg = + container_of(crypto_skcipher_alg(skcipher), typeof(*alg), + skcipher.base); + struct device *jrdev = ctx->jrdev; + unsigned int ivsize = crypto_skcipher_ivsize(skcipher); + u32 *desc; + const bool is_rfc3686 = alg->caam.rfc3686; + + print_hex_dump_devel("key in @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); + + /* Here keylen is actual key length */ + ctx->cdata.keylen = keylen; + ctx->cdata.key_virt = key; + ctx->cdata.key_inline = true; + /* Here protected key len is plain key length */ + ctx->cdata.plain_keylen = keylen; + ctx->cdata.key_cmd_opt = 0; + + + /* skcipher_encrypt shared descriptor */ + desc = ctx->sh_desc_enc; + cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686, + ctx1_iv_off); + dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, + desc_bytes(desc), ctx->dir); + + /* skcipher_decrypt shared descriptor */ + desc = ctx->sh_desc_dec; + cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686, + ctx1_iv_off); + dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, + desc_bytes(desc), ctx->dir); + + return 0; +} + +static int paes_skcipher_setkey(struct crypto_skcipher *skcipher, + const u8 *key, + unsigned int keylen) +{ + struct caam_pkey_info *pkey_info = (struct caam_pkey_info *)key; + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); + struct device *jrdev = ctx->jrdev; + int err; + + ctx->cdata.key_inline = false; + + keylen = keylen - CAAM_PKEY_HEADER; + + /* Retrieve the length of key */ + ctx->cdata.plain_keylen = pkey_info->plain_key_sz; + + /* Retrieve the length of blob*/ + ctx->cdata.keylen = keylen; + + /* Retrieve the address of the blob */ + ctx->cdata.key_virt = pkey_info->key_buf; + + /* Validate key length for AES algorithms */ + err = aes_check_keylen(ctx->cdata.plain_keylen); + if (err) { + dev_err(jrdev, "bad key length\n"); + return err; + } + + /* set command option */ + ctx->cdata.key_cmd_opt |= KEY_ENC; + + /* check if the Protected-Key is CCM key */ + if (pkey_info->key_enc_algo == CAAM_ENC_ALGO_CCM) + ctx->cdata.key_cmd_opt |= KEY_EKT; + + memcpy(ctx->key, ctx->cdata.key_virt, keylen); + dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); + ctx->cdata.key_dma = ctx->key_dma; + + if (pkey_info->key_enc_algo == CAAM_ENC_ALGO_CCM) + ctx->protected_key_dma = dma_map_single(jrdev, ctx->protected_key, + ctx->cdata.plain_keylen + + CAAM_CCM_OVERHEAD, + DMA_FROM_DEVICE); + else + ctx->protected_key_dma = dma_map_single(jrdev, ctx->protected_key, + ctx->cdata.plain_keylen, + DMA_FROM_DEVICE); + + ctx->cdata.protected_key_dma = ctx->protected_key_dma; + ctx->is_blob = true; + + return 0; +} + +static int aes_skcipher_setkey(struct crypto_skcipher *skcipher, + const u8 *key, unsigned int keylen) +{ + int err; + + err = aes_check_keylen(keylen); + if (err) + return err; + + return skcipher_setkey(skcipher, key, keylen, 0); +} + +static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher, + const u8 *key, unsigned int keylen) +{ + u32 ctx1_iv_off; + int err; + + /* + * RFC3686 specific: + * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} + * | *key = {KEY, NONCE} + */ + ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; + keylen -= CTR_RFC3686_NONCE_SIZE; + + err = aes_check_keylen(keylen); + if (err) + return err; + + return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); +} + +static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher, + const u8 *key, unsigned int keylen) +{ + u32 ctx1_iv_off; + int err; + + /* + * AES-CTR needs to load IV in CONTEXT1 reg + * at an offset of 128bits (16bytes) + * CONTEXT1[255:128] = IV + */ + ctx1_iv_off = 16; + + err = aes_check_keylen(keylen); + if (err) + return err; + + return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); +} + +static int des_skcipher_setkey(struct crypto_skcipher *skcipher, + const u8 *key, unsigned int keylen) +{ + return verify_skcipher_des_key(skcipher, key) ?: + skcipher_setkey(skcipher, key, keylen, 0); +} + +static int des3_skcipher_setkey(struct crypto_skcipher *skcipher, + const u8 *key, unsigned int keylen) +{ + return verify_skcipher_des3_key(skcipher, key) ?: + skcipher_setkey(skcipher, key, keylen, 0); +} + +static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, + unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); + struct device *jrdev = ctx->jrdev; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); + u32 *desc; + int err; + + err = xts_verify_key(skcipher, key, keylen); + if (err) { + dev_dbg(jrdev, "key size mismatch\n"); + return err; + } + + if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256) + ctx->xts_key_fallback = true; + + if (ctrlpriv->era <= 8 || ctx->xts_key_fallback) { + err = crypto_skcipher_setkey(ctx->fallback, key, keylen); + if (err) + return err; + } + + ctx->cdata.keylen = keylen; + ctx->cdata.key_virt = key; + ctx->cdata.key_inline = true; + + /* xts_skcipher_encrypt shared descriptor */ + desc = ctx->sh_desc_enc; + cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata); + dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, + desc_bytes(desc), ctx->dir); + + /* xts_skcipher_decrypt shared descriptor */ + desc = ctx->sh_desc_dec; + cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata); + dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, + desc_bytes(desc), ctx->dir); + + return 0; +} + +/* + * aead_edesc - s/w-extended aead descriptor + * @src_nents: number of segments in input s/w scatterlist + * @dst_nents: number of segments in output s/w scatterlist + * @mapped_src_nents: number of segments in input h/w link table + * @mapped_dst_nents: number of segments in output h/w link table + * @sec4_sg_bytes: length of dma mapped sec4_sg space + * @bklog: stored to determine if the request needs backlog + * @sec4_sg_dma: bus physical mapped address of h/w link table + * @sec4_sg: pointer to h/w link table + * @hw_desc: the h/w job descriptor followed by any referenced link tables + */ +struct aead_edesc { + int src_nents; + int dst_nents; + int mapped_src_nents; + int mapped_dst_nents; + int sec4_sg_bytes; + bool bklog; + dma_addr_t sec4_sg_dma; + struct sec4_sg_entry *sec4_sg; + u32 hw_desc[]; +}; + +/* + * skcipher_edesc - s/w-extended skcipher descriptor + * @src_nents: number of segments in input s/w scatterlist + * @dst_nents: number of segments in output s/w scatterlist + * @mapped_src_nents: number of segments in input h/w link table + * @mapped_dst_nents: number of segments in output h/w link table + * @iv_dma: dma address of iv for checking continuity and link table + * @sec4_sg_bytes: length of dma mapped sec4_sg space + * @bklog: stored to determine if the request needs backlog + * @sec4_sg_dma: bus physical mapped address of h/w link table + * @sec4_sg: pointer to h/w link table + * @hw_desc: the h/w job descriptor followed by any referenced link tables + * and IV + */ +struct skcipher_edesc { + int src_nents; + int dst_nents; + int mapped_src_nents; + int mapped_dst_nents; + dma_addr_t iv_dma; + int sec4_sg_bytes; + bool bklog; + dma_addr_t sec4_sg_dma; + struct sec4_sg_entry *sec4_sg; + u32 hw_desc[]; +}; + +static void caam_unmap(struct device *dev, struct scatterlist *src, + struct scatterlist *dst, int src_nents, + int dst_nents, + dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma, + int sec4_sg_bytes) +{ + if (dst != src) { + if (src_nents) + dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); + if (dst_nents) + dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); + } else { + dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); + } + + if (iv_dma) + dma_unmap_single(dev, iv_dma, ivsize, DMA_BIDIRECTIONAL); + if (sec4_sg_bytes) + dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes, + DMA_TO_DEVICE); +} + +static void aead_unmap(struct device *dev, + struct aead_edesc *edesc, + struct aead_request *req) +{ + caam_unmap(dev, req->src, req->dst, + edesc->src_nents, edesc->dst_nents, 0, 0, + edesc->sec4_sg_dma, edesc->sec4_sg_bytes); +} + +static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, + struct skcipher_request *req) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + int ivsize = crypto_skcipher_ivsize(skcipher); + + caam_unmap(dev, req->src, req->dst, + edesc->src_nents, edesc->dst_nents, + edesc->iv_dma, ivsize, + edesc->sec4_sg_dma, edesc->sec4_sg_bytes); +} + +static void aead_crypt_done(struct device *jrdev, u32 *desc, u32 err, + void *context) +{ + struct aead_request *req = context; + struct caam_aead_req_ctx *rctx = aead_request_ctx(req); + struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev); + struct aead_edesc *edesc; + int ecode = 0; + bool has_bklog; + + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); + + edesc = rctx->edesc; + has_bklog = edesc->bklog; + + if (err) + ecode = caam_jr_strstatus(jrdev, err); + + aead_unmap(jrdev, edesc, req); + + kfree(edesc); + + /* + * If no backlog flag, the completion of the request is done + * by CAAM, not crypto engine. + */ + if (!has_bklog) + aead_request_complete(req, ecode); + else + crypto_finalize_aead_request(jrp->engine, req, ecode); +} + +static inline u8 *skcipher_edesc_iv(struct skcipher_edesc *edesc) +{ + + return PTR_ALIGN((u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes, + dma_get_cache_alignment()); +} + +static void skcipher_crypt_done(struct device *jrdev, u32 *desc, u32 err, + void *context) +{ + struct skcipher_request *req = context; + struct skcipher_edesc *edesc; + struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev); + int ivsize = crypto_skcipher_ivsize(skcipher); + int ecode = 0; + bool has_bklog; + + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); + + edesc = rctx->edesc; + has_bklog = edesc->bklog; + if (err) + ecode = caam_jr_strstatus(jrdev, err); + + skcipher_unmap(jrdev, edesc, req); + + /* + * The crypto API expects us to set the IV (req->iv) to the last + * ciphertext block (CBC mode) or last counter (CTR mode). + * This is used e.g. by the CTS mode. + */ + if (ivsize && !ecode) { + memcpy(req->iv, skcipher_edesc_iv(edesc), ivsize); + + print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->iv, + ivsize, 1); + } + + caam_dump_sg("dst @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->dst, + edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); + + kfree(edesc); + + /* + * If no backlog flag, the completion of the request is done + * by CAAM, not crypto engine. + */ + if (!has_bklog) + skcipher_request_complete(req, ecode); + else + crypto_finalize_skcipher_request(jrp->engine, req, ecode); +} + +/* + * Fill in aead job descriptor + */ +static void init_aead_job(struct aead_request *req, + struct aead_edesc *edesc, + bool all_contig, bool encrypt) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + int authsize = ctx->authsize; + u32 *desc = edesc->hw_desc; + u32 out_options, in_options; + dma_addr_t dst_dma, src_dma; + int len, sec4_sg_index = 0; + dma_addr_t ptr; + u32 *sh_desc; + + sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec; + ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma; + + len = desc_len(sh_desc); + init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); + + if (all_contig) { + src_dma = edesc->mapped_src_nents ? sg_dma_address(req->src) : + 0; + in_options = 0; + } else { + src_dma = edesc->sec4_sg_dma; + sec4_sg_index += edesc->mapped_src_nents; + in_options = LDST_SGF; + } + + append_seq_in_ptr(desc, src_dma, req->assoclen + req->cryptlen, + in_options); + + dst_dma = src_dma; + out_options = in_options; + + if (unlikely(req->src != req->dst)) { + if (!edesc->mapped_dst_nents) { + dst_dma = 0; + out_options = 0; + } else if (edesc->mapped_dst_nents == 1) { + dst_dma = sg_dma_address(req->dst); + out_options = 0; + } else { + dst_dma = edesc->sec4_sg_dma + + sec4_sg_index * + sizeof(struct sec4_sg_entry); + out_options = LDST_SGF; + } + } + + if (encrypt) + append_seq_out_ptr(desc, dst_dma, + req->assoclen + req->cryptlen + authsize, + out_options); + else + append_seq_out_ptr(desc, dst_dma, + req->assoclen + req->cryptlen - authsize, + out_options); +} + +static void init_gcm_job(struct aead_request *req, + struct aead_edesc *edesc, + bool all_contig, bool encrypt) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + unsigned int ivsize = crypto_aead_ivsize(aead); + u32 *desc = edesc->hw_desc; + bool generic_gcm = (ivsize == GCM_AES_IV_SIZE); + unsigned int last; + + init_aead_job(req, edesc, all_contig, encrypt); + append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); + + /* BUG This should not be specific to generic GCM. */ + last = 0; + if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen)) + last = FIFOLD_TYPE_LAST1; + + /* Read GCM IV */ + append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | GCM_AES_IV_SIZE | last); + /* Append Salt */ + if (!generic_gcm) + append_data(desc, ctx->key + ctx->cdata.keylen, 4); + /* Append IV */ + append_data(desc, req->iv, ivsize); + /* End of blank commands */ +} + +static void init_chachapoly_job(struct aead_request *req, + struct aead_edesc *edesc, bool all_contig, + bool encrypt) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + unsigned int ivsize = crypto_aead_ivsize(aead); + unsigned int assoclen = req->assoclen; + u32 *desc = edesc->hw_desc; + u32 ctx_iv_off = 4; + + init_aead_job(req, edesc, all_contig, encrypt); + + if (ivsize != CHACHAPOLY_IV_SIZE) { + /* IPsec specific: CONTEXT1[223:128] = {NONCE, IV} */ + ctx_iv_off += 4; + + /* + * The associated data comes already with the IV but we need + * to skip it when we authenticate or encrypt... + */ + assoclen -= ivsize; + } + + append_math_add_imm_u32(desc, REG3, ZERO, IMM, assoclen); + + /* + * For IPsec load the IV further in the same register. + * For RFC7539 simply load the 12 bytes nonce in a single operation + */ + append_load_as_imm(desc, req->iv, ivsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + ctx_iv_off << LDST_OFFSET_SHIFT); +} + +static void init_authenc_job(struct aead_request *req, + struct aead_edesc *edesc, + bool all_contig, bool encrypt) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), + struct caam_aead_alg, + aead.base); + unsigned int ivsize = crypto_aead_ivsize(aead); + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); + const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == + OP_ALG_AAI_CTR_MOD128); + const bool is_rfc3686 = alg->caam.rfc3686; + u32 *desc = edesc->hw_desc; + u32 ivoffset = 0; + + /* + * AES-CTR needs to load IV in CONTEXT1 reg + * at an offset of 128bits (16bytes) + * CONTEXT1[255:128] = IV + */ + if (ctr_mode) + ivoffset = 16; + + /* + * RFC3686 specific: + * CONTEXT1[255:128] = {NONCE, IV, COUNTER} + */ + if (is_rfc3686) + ivoffset = 16 + CTR_RFC3686_NONCE_SIZE; + + init_aead_job(req, edesc, all_contig, encrypt); + + /* + * {REG3, DPOVRD} = assoclen, depending on whether MATH command supports + * having DPOVRD as destination. + */ + if (ctrlpriv->era < 3) + append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); + else + append_math_add_imm_u32(desc, DPOVRD, ZERO, IMM, req->assoclen); + + if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv)) + append_load_as_imm(desc, req->iv, ivsize, + LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + (ivoffset << LDST_OFFSET_SHIFT)); +} + +/* + * Fill in skcipher job descriptor + */ +static void init_skcipher_job(struct skcipher_request *req, + struct skcipher_edesc *edesc, + const bool encrypt) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); + struct device *jrdev = ctx->jrdev; + int ivsize = crypto_skcipher_ivsize(skcipher); + u32 *desc = !ctx->is_blob ? edesc->hw_desc : + (u32 *)((u8 *)edesc->hw_desc + CAAM_DESC_BYTES_MAX); + dma_addr_t desc_dma; + u32 *sh_desc; + u32 in_options = 0, out_options = 0; + dma_addr_t src_dma, dst_dma, ptr; + int len, sec4_sg_index = 0; + + print_hex_dump_debug("presciv@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); + dev_dbg(jrdev, "asked=%d, cryptlen%d\n", + (int)edesc->src_nents > 1 ? 100 : req->cryptlen, req->cryptlen); + + caam_dump_sg("src @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->src, + edesc->src_nents > 1 ? 100 : req->cryptlen, 1); + + + if (ivsize || edesc->mapped_src_nents > 1) { + src_dma = edesc->sec4_sg_dma; + sec4_sg_index = edesc->mapped_src_nents + !!ivsize; + in_options = LDST_SGF; + } else { + src_dma = sg_dma_address(req->src); + } + + if (likely(req->src == req->dst)) { + dst_dma = src_dma + !!ivsize * sizeof(struct sec4_sg_entry); + out_options = in_options; + } else if (!ivsize && edesc->mapped_dst_nents == 1) { + dst_dma = sg_dma_address(req->dst); + } else { + dst_dma = edesc->sec4_sg_dma + sec4_sg_index * + sizeof(struct sec4_sg_entry); + out_options = LDST_SGF; + } + + if (ctx->is_blob) { + cnstr_desc_skcipher_enc_dec(desc, &ctx->cdata, + src_dma, dst_dma, req->cryptlen + ivsize, + in_options, out_options, + ivsize, encrypt); + + desc_dma = dma_map_single(jrdev, desc, desc_bytes(desc), DMA_TO_DEVICE); + + cnstr_desc_protected_blob_decap(edesc->hw_desc, &ctx->cdata, desc_dma); + } else { + sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec; + ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma; + + len = desc_len(sh_desc); + init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); + append_seq_in_ptr(desc, src_dma, req->cryptlen + ivsize, in_options); + + append_seq_out_ptr(desc, dst_dma, req->cryptlen + ivsize, out_options); + } +} + +/* + * allocate and map the aead extended descriptor + */ +static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, + int desc_bytes, bool *all_contig_ptr, + bool encrypt) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *jrdev = ctx->jrdev; + struct caam_aead_req_ctx *rctx = aead_request_ctx(req); + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; + int src_len, dst_len = 0; + struct aead_edesc *edesc; + int sec4_sg_index, sec4_sg_len, sec4_sg_bytes; + unsigned int authsize = ctx->authsize; + + if (unlikely(req->dst != req->src)) { + src_len = req->assoclen + req->cryptlen; + dst_len = src_len + (encrypt ? authsize : (-authsize)); + + src_nents = sg_nents_for_len(req->src, src_len); + if (unlikely(src_nents < 0)) { + dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", + src_len); + return ERR_PTR(src_nents); + } + + dst_nents = sg_nents_for_len(req->dst, dst_len); + if (unlikely(dst_nents < 0)) { + dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", + dst_len); + return ERR_PTR(dst_nents); + } + } else { + src_len = req->assoclen + req->cryptlen + + (encrypt ? authsize : 0); + + src_nents = sg_nents_for_len(req->src, src_len); + if (unlikely(src_nents < 0)) { + dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", + src_len); + return ERR_PTR(src_nents); + } + } + + if (likely(req->src == req->dst)) { + mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, + DMA_BIDIRECTIONAL); + if (unlikely(!mapped_src_nents)) { + dev_err(jrdev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + } else { + /* Cover also the case of null (zero length) input data */ + if (src_nents) { + mapped_src_nents = dma_map_sg(jrdev, req->src, + src_nents, DMA_TO_DEVICE); + if (unlikely(!mapped_src_nents)) { + dev_err(jrdev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + } else { + mapped_src_nents = 0; + } + + /* Cover also the case of null (zero length) output data */ + if (dst_nents) { + mapped_dst_nents = dma_map_sg(jrdev, req->dst, + dst_nents, + DMA_FROM_DEVICE); + if (unlikely(!mapped_dst_nents)) { + dev_err(jrdev, "unable to map destination\n"); + dma_unmap_sg(jrdev, req->src, src_nents, + DMA_TO_DEVICE); + return ERR_PTR(-ENOMEM); + } + } else { + mapped_dst_nents = 0; + } + } + + /* + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond + * the end of the table by allocating more S/G entries. + */ + sec4_sg_len = mapped_src_nents > 1 ? mapped_src_nents : 0; + if (mapped_dst_nents > 1) + sec4_sg_len += pad_sg_nents(mapped_dst_nents); + else + sec4_sg_len = pad_sg_nents(sec4_sg_len); + + sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); + + /* allocate space for base edesc and hw desc commands, link tables */ + edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, flags); + if (!edesc) { + caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0, + 0, 0, 0); + return ERR_PTR(-ENOMEM); + } + + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + edesc->mapped_src_nents = mapped_src_nents; + edesc->mapped_dst_nents = mapped_dst_nents; + edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + + desc_bytes; + + rctx->edesc = edesc; + + *all_contig_ptr = !(mapped_src_nents > 1); + + sec4_sg_index = 0; + if (mapped_src_nents > 1) { + sg_to_sec4_sg_last(req->src, src_len, + edesc->sec4_sg + sec4_sg_index, 0); + sec4_sg_index += mapped_src_nents; + } + if (mapped_dst_nents > 1) { + sg_to_sec4_sg_last(req->dst, dst_len, + edesc->sec4_sg + sec4_sg_index, 0); + } + + if (!sec4_sg_bytes) + return edesc; + + edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, + sec4_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { + dev_err(jrdev, "unable to map S/G table\n"); + aead_unmap(jrdev, edesc, req); + kfree(edesc); + return ERR_PTR(-ENOMEM); + } + + edesc->sec4_sg_bytes = sec4_sg_bytes; + + return edesc; +} + +static int aead_enqueue_req(struct device *jrdev, struct aead_request *req) +{ + struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev); + struct caam_aead_req_ctx *rctx = aead_request_ctx(req); + struct aead_edesc *edesc = rctx->edesc; + u32 *desc = edesc->hw_desc; + int ret; + + /* + * Only the backlog request are sent to crypto-engine since the others + * can be handled by CAAM, if free, especially since JR has up to 1024 + * entries (more than the 10 entries from crypto-engine). + */ + if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG) + ret = crypto_transfer_aead_request_to_engine(jrpriv->engine, + req); + else + ret = caam_jr_enqueue(jrdev, desc, aead_crypt_done, req); + + if ((ret != -EINPROGRESS) && (ret != -EBUSY)) { + aead_unmap(jrdev, edesc, req); + kfree(rctx->edesc); + } + + return ret; +} + +static inline int chachapoly_crypt(struct aead_request *req, bool encrypt) +{ + struct aead_edesc *edesc; + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *jrdev = ctx->jrdev; + bool all_contig; + u32 *desc; + + edesc = aead_edesc_alloc(req, CHACHAPOLY_DESC_JOB_IO_LEN, &all_contig, + encrypt); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + desc = edesc->hw_desc; + + init_chachapoly_job(req, edesc, all_contig, encrypt); + print_hex_dump_debug("chachapoly jobdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); + + return aead_enqueue_req(jrdev, req); +} + +static int chachapoly_encrypt(struct aead_request *req) +{ + return chachapoly_crypt(req, true); +} + +static int chachapoly_decrypt(struct aead_request *req) +{ + return chachapoly_crypt(req, false); +} + +static inline int aead_crypt(struct aead_request *req, bool encrypt) +{ + struct aead_edesc *edesc; + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *jrdev = ctx->jrdev; + bool all_contig; + + /* allocate extended descriptor */ + edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, + &all_contig, encrypt); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + /* Create and submit job descriptor */ + init_authenc_job(req, edesc, all_contig, encrypt); + + print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); + + return aead_enqueue_req(jrdev, req); +} + +static int aead_encrypt(struct aead_request *req) +{ + return aead_crypt(req, true); +} + +static int aead_decrypt(struct aead_request *req) +{ + return aead_crypt(req, false); +} + +static int aead_do_one_req(struct crypto_engine *engine, void *areq) +{ + struct aead_request *req = aead_request_cast(areq); + struct caam_ctx *ctx = crypto_aead_ctx_dma(crypto_aead_reqtfm(req)); + struct caam_aead_req_ctx *rctx = aead_request_ctx(req); + u32 *desc = rctx->edesc->hw_desc; + int ret; + + rctx->edesc->bklog = true; + + ret = caam_jr_enqueue(ctx->jrdev, desc, aead_crypt_done, req); + + if (ret == -ENOSPC && engine->retry_support) + return ret; + + if (ret != -EINPROGRESS) { + aead_unmap(ctx->jrdev, rctx->edesc, req); + kfree(rctx->edesc); + } else { + ret = 0; + } + + return ret; +} + +static inline int gcm_crypt(struct aead_request *req, bool encrypt) +{ + struct aead_edesc *edesc; + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *jrdev = ctx->jrdev; + bool all_contig; + + /* allocate extended descriptor */ + edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, + encrypt); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + /* Create and submit job descriptor */ + init_gcm_job(req, edesc, all_contig, encrypt); + + print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); + + return aead_enqueue_req(jrdev, req); +} + +static int gcm_encrypt(struct aead_request *req) +{ + return gcm_crypt(req, true); +} + +static int gcm_decrypt(struct aead_request *req) +{ + return gcm_crypt(req, false); +} + +static int ipsec_gcm_encrypt(struct aead_request *req) +{ + return crypto_ipsec_check_assoclen(req->assoclen) ? : gcm_encrypt(req); +} + +static int ipsec_gcm_decrypt(struct aead_request *req) +{ + return crypto_ipsec_check_assoclen(req->assoclen) ? : gcm_decrypt(req); +} + +/* + * allocate and map the skcipher extended descriptor for skcipher + */ +static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, + int desc_bytes) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); + struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); + struct device *jrdev = ctx->jrdev; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; + struct skcipher_edesc *edesc; + dma_addr_t iv_dma = 0; + u8 *iv; + int ivsize = crypto_skcipher_ivsize(skcipher); + int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes; + unsigned int aligned_size; + + src_nents = sg_nents_for_len(req->src, req->cryptlen); + if (unlikely(src_nents < 0)) { + dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", + req->cryptlen); + return ERR_PTR(src_nents); + } + + if (req->dst != req->src) { + dst_nents = sg_nents_for_len(req->dst, req->cryptlen); + if (unlikely(dst_nents < 0)) { + dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", + req->cryptlen); + return ERR_PTR(dst_nents); + } + } + + if (likely(req->src == req->dst)) { + mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, + DMA_BIDIRECTIONAL); + if (unlikely(!mapped_src_nents)) { + dev_err(jrdev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + } else { + mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, + DMA_TO_DEVICE); + if (unlikely(!mapped_src_nents)) { + dev_err(jrdev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents, + DMA_FROM_DEVICE); + if (unlikely(!mapped_dst_nents)) { + dev_err(jrdev, "unable to map destination\n"); + dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); + return ERR_PTR(-ENOMEM); + } + } + + if (!ivsize && mapped_src_nents == 1) + sec4_sg_ents = 0; // no need for an input hw s/g table + else + sec4_sg_ents = mapped_src_nents + !!ivsize; + dst_sg_idx = sec4_sg_ents; + + /* + * Input, output HW S/G tables: [IV, src][dst, IV] + * IV entries point to the same buffer + * If src == dst, S/G entries are reused (S/G tables overlap) + * + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond + * the end of the table by allocating more S/G entries. Logic: + * if (output S/G) + * pad output S/G, if needed + * else if (input S/G) ... + * pad input S/G, if needed + */ + if (ivsize || mapped_dst_nents > 1) { + if (req->src == req->dst) + sec4_sg_ents = !!ivsize + pad_sg_nents(sec4_sg_ents); + else + sec4_sg_ents += pad_sg_nents(mapped_dst_nents + + !!ivsize); + } else { + sec4_sg_ents = pad_sg_nents(sec4_sg_ents); + } + + sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry); + + /* + * allocate space for base edesc and hw desc commands, link tables, IV + */ + aligned_size = sizeof(*edesc) + desc_bytes + sec4_sg_bytes; + aligned_size = ALIGN(aligned_size, dma_get_cache_alignment()); + aligned_size += ~(ARCH_KMALLOC_MINALIGN - 1) & + (dma_get_cache_alignment() - 1); + aligned_size += ALIGN(ivsize, dma_get_cache_alignment()); + edesc = kzalloc(aligned_size, flags); + if (!edesc) { + dev_err(jrdev, "could not allocate extended descriptor\n"); + caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0, + 0, 0, 0); + return ERR_PTR(-ENOMEM); + } + + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + edesc->mapped_src_nents = mapped_src_nents; + edesc->mapped_dst_nents = mapped_dst_nents; + edesc->sec4_sg_bytes = sec4_sg_bytes; + edesc->sec4_sg = (struct sec4_sg_entry *)((u8 *)edesc->hw_desc + + desc_bytes); + rctx->edesc = edesc; + + /* Make sure IV is located in a DMAable area */ + if (ivsize) { + iv = skcipher_edesc_iv(edesc); + memcpy(iv, req->iv, ivsize); + + iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_BIDIRECTIONAL); + if (dma_mapping_error(jrdev, iv_dma)) { + dev_err(jrdev, "unable to map IV\n"); + caam_unmap(jrdev, req->src, req->dst, src_nents, + dst_nents, 0, 0, 0, 0); + kfree(edesc); + return ERR_PTR(-ENOMEM); + } + + dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0); + } + if (dst_sg_idx) + sg_to_sec4_sg(req->src, req->cryptlen, edesc->sec4_sg + + !!ivsize, 0); + + if (req->src != req->dst && (ivsize || mapped_dst_nents > 1)) + sg_to_sec4_sg(req->dst, req->cryptlen, edesc->sec4_sg + + dst_sg_idx, 0); + + if (ivsize) + dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx + + mapped_dst_nents, iv_dma, ivsize, 0); + + if (ivsize || mapped_dst_nents > 1) + sg_to_sec4_set_last(edesc->sec4_sg + dst_sg_idx + + mapped_dst_nents - 1 + !!ivsize); + + if (sec4_sg_bytes) { + edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, + sec4_sg_bytes, + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { + dev_err(jrdev, "unable to map S/G table\n"); + caam_unmap(jrdev, req->src, req->dst, src_nents, + dst_nents, iv_dma, ivsize, 0, 0); + kfree(edesc); + return ERR_PTR(-ENOMEM); + } + } + + edesc->iv_dma = iv_dma; + + print_hex_dump_debug("skcipher sec4_sg@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, + sec4_sg_bytes, 1); + + return edesc; +} + +static int skcipher_do_one_req(struct crypto_engine *engine, void *areq) +{ + struct skcipher_request *req = skcipher_request_cast(areq); + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(crypto_skcipher_reqtfm(req)); + struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); + u32 *desc = rctx->edesc->hw_desc; + int ret; + + rctx->edesc->bklog = true; + + ret = caam_jr_enqueue(ctx->jrdev, desc, skcipher_crypt_done, req); + + if (ret == -ENOSPC && engine->retry_support) + return ret; + + if (ret != -EINPROGRESS) { + skcipher_unmap(ctx->jrdev, rctx->edesc, req); + kfree(rctx->edesc); + } else { + ret = 0; + } + + return ret; +} + +static inline bool xts_skcipher_ivsize(struct skcipher_request *req) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + unsigned int ivsize = crypto_skcipher_ivsize(skcipher); + + return !!get_unaligned((u64 *)(req->iv + (ivsize / 2))); +} + +static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) +{ + struct skcipher_edesc *edesc; + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); + struct device *jrdev = ctx->jrdev; + struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev); + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); + u32 *desc; + int ret = 0; + int len; + + /* + * XTS is expected to return an error even for input length = 0 + * Note that the case input length < block size will be caught during + * HW offloading and return an error. + */ + if (!req->cryptlen && !ctx->fallback) + return 0; + + if (ctx->fallback && ((ctrlpriv->era <= 8 && xts_skcipher_ivsize(req)) || + ctx->xts_key_fallback)) { + struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); + + skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback); + skcipher_request_set_callback(&rctx->fallback_req, + req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(&rctx->fallback_req, req->src, + req->dst, req->cryptlen, req->iv); + + return encrypt ? crypto_skcipher_encrypt(&rctx->fallback_req) : + crypto_skcipher_decrypt(&rctx->fallback_req); + } + + len = DESC_JOB_IO_LEN * CAAM_CMD_SZ; + if (ctx->is_blob) + len += CAAM_DESC_BYTES_MAX; + + /* allocate extended descriptor */ + edesc = skcipher_edesc_alloc(req, len); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + /* Create and submit job descriptor*/ + init_skcipher_job(req, edesc, encrypt); + + print_hex_dump_debug("skcipher jobdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); + + desc = edesc->hw_desc; + /* + * Only the backlog request are sent to crypto-engine since the others + * can be handled by CAAM, if free, especially since JR has up to 1024 + * entries (more than the 10 entries from crypto-engine). + */ + if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG) + ret = crypto_transfer_skcipher_request_to_engine(jrpriv->engine, + req); + else + ret = caam_jr_enqueue(jrdev, desc, skcipher_crypt_done, req); + + if ((ret != -EINPROGRESS) && (ret != -EBUSY)) { + skcipher_unmap(jrdev, edesc, req); + kfree(edesc); + } + + return ret; +} + +static int skcipher_encrypt(struct skcipher_request *req) +{ + return skcipher_crypt(req, true); +} + +static int skcipher_decrypt(struct skcipher_request *req) +{ + return skcipher_crypt(req, false); +} + +static struct caam_skcipher_alg driver_algs[] = { + { + .skcipher.base = { + .base = { + .cra_name = "cbc(paes)", + .cra_driver_name = "cbc-paes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = paes_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE + CAAM_BLOB_OVERHEAD + + CAAM_PKEY_HEADER, + .max_keysize = AES_MAX_KEY_SIZE + CAAM_BLOB_OVERHEAD + + CAAM_PKEY_HEADER, + .ivsize = AES_BLOCK_SIZE, + }, + .skcipher.op = { + .do_one_request = skcipher_do_one_req, + }, + .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + }, + { + .skcipher.base = { + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aes_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .skcipher.op = { + .do_one_request = skcipher_do_one_req, + }, + .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + }, + { + .skcipher.base = { + .base = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "cbc-3des-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + }, + .skcipher.op = { + .do_one_request = skcipher_do_one_req, + }, + .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + }, + { + .skcipher.base = { + .base = { + .cra_name = "cbc(des)", + .cra_driver_name = "cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = des_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + }, + .skcipher.op = { + .do_one_request = skcipher_do_one_req, + }, + .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + }, + { + .skcipher.base = { + .base = { + .cra_name = "ctr(aes)", + .cra_driver_name = "ctr-aes-caam", + .cra_blocksize = 1, + }, + .setkey = ctr_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .chunksize = AES_BLOCK_SIZE, + }, + .skcipher.op = { + .do_one_request = skcipher_do_one_req, + }, + .caam.class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + }, + { + .skcipher.base = { + .base = { + .cra_name = "rfc3686(ctr(aes))", + .cra_driver_name = "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, + }, + .setkey = rfc3686_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + .chunksize = AES_BLOCK_SIZE, + }, + .skcipher.op = { + .do_one_request = skcipher_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .rfc3686 = true, + }, + }, + { + .skcipher.base = { + .base = { + .cra_name = "xts(aes)", + .cra_driver_name = "xts-aes-caam", + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = xts_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .skcipher.op = { + .do_one_request = skcipher_do_one_req, + }, + .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, + }, + { + .skcipher.base = { + .base = { + .cra_name = "ecb(des)", + .cra_driver_name = "ecb-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = des_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + }, + .skcipher.op = { + .do_one_request = skcipher_do_one_req, + }, + .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_ECB, + }, + { + .skcipher.base = { + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aes_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + }, + .skcipher.op = { + .do_one_request = skcipher_do_one_req, + }, + .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_ECB, + }, + { + .skcipher.base = { + .base = { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "ecb-des3-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + }, + .skcipher.op = { + .do_one_request = skcipher_do_one_req, + }, + .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_ECB, + }, +}; + +static struct caam_aead_alg driver_aeads[] = { + { + .aead.base = { + .base = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "rfc4106-gcm-aes-caam", + .cra_blocksize = 1, + }, + .setkey = rfc4106_setkey, + .setauthsize = rfc4106_setauthsize, + .encrypt = ipsec_gcm_encrypt, + .decrypt = ipsec_gcm_decrypt, + .ivsize = GCM_RFC4106_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + .nodkp = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "rfc4543(gcm(aes))", + .cra_driver_name = "rfc4543-gcm-aes-caam", + .cra_blocksize = 1, + }, + .setkey = rfc4543_setkey, + .setauthsize = rfc4543_setauthsize, + .encrypt = ipsec_gcm_encrypt, + .decrypt = ipsec_gcm_decrypt, + .ivsize = GCM_RFC4543_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + .nodkp = true, + }, + }, + /* Galois Counter Mode */ + { + .aead.base = { + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "gcm-aes-caam", + .cra_blocksize = 1, + }, + .setkey = gcm_setkey, + .setauthsize = gcm_setauthsize, + .encrypt = gcm_encrypt, + .decrypt = gcm_decrypt, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + .nodkp = true, + }, + }, + /* single-pass ipsec_esp descriptor */ + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(md5)," + "ecb(cipher_null))", + .cra_driver_name = "authenc-hmac-md5-" + "ecb-cipher_null-caam", + .cra_blocksize = NULL_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = NULL_IV_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha1)," + "ecb(cipher_null))", + .cra_driver_name = "authenc-hmac-sha1-" + "ecb-cipher_null-caam", + .cra_blocksize = NULL_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = NULL_IV_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha224)," + "ecb(cipher_null))", + .cra_driver_name = "authenc-hmac-sha224-" + "ecb-cipher_null-caam", + .cra_blocksize = NULL_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = NULL_IV_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha256)," + "ecb(cipher_null))", + .cra_driver_name = "authenc-hmac-sha256-" + "ecb-cipher_null-caam", + .cra_blocksize = NULL_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = NULL_IV_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha384)," + "ecb(cipher_null))", + .cra_driver_name = "authenc-hmac-sha384-" + "ecb-cipher_null-caam", + .cra_blocksize = NULL_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = NULL_IV_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha512)," + "ecb(cipher_null))", + .cra_driver_name = "authenc-hmac-sha512-" + "ecb-cipher_null-caam", + .cra_blocksize = NULL_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = NULL_IV_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(aes))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "echainiv(authenc(hmac(md5)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-hmac-md5-" + "cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha1)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha1-cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha224)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha224-cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha256)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha256-cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha384)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha384-cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha512)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha512-cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead.base = { + .base = { + .cra_name = "echainiv(authenc(hmac(md5)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-hmac-md5-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha1)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha1)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha1-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha224)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha224)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha224-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha256)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha256)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha256-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha384)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha384)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha384-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha512)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha512)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha512-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "echainiv(authenc(hmac(md5)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-hmac-md5-" + "cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha1)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha1-cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(des))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha224)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha224-cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(des))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha256)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha256-cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(des))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha384)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha384-cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(des))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha512)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha512-cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(md5)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-md5-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "seqiv(authenc(" + "hmac(md5),rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-md5-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + .geniv = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha1)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha1-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "seqiv(authenc(" + "hmac(sha1),rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha1-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + .geniv = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha224)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha224-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "seqiv(authenc(" + "hmac(sha224),rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha224-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + .geniv = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha256)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha256-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "seqiv(authenc(hmac(sha256)," + "rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha256-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + .geniv = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha384)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha384-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "seqiv(authenc(hmac(sha384)," + "rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha384-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + .geniv = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "authenc(hmac(sha512)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha512-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "seqiv(authenc(hmac(sha512)," + "rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha512-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + .geniv = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "rfc7539(chacha20,poly1305)", + .cra_driver_name = "rfc7539-chacha20-poly1305-" + "caam", + .cra_blocksize = 1, + }, + .setkey = chachapoly_setkey, + .setauthsize = chachapoly_setauthsize, + .encrypt = chachapoly_encrypt, + .decrypt = chachapoly_decrypt, + .ivsize = CHACHAPOLY_IV_SIZE, + .maxauthsize = POLY1305_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | + OP_ALG_AAI_AEAD, + .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | + OP_ALG_AAI_AEAD, + .nodkp = true, + }, + }, + { + .aead.base = { + .base = { + .cra_name = "rfc7539esp(chacha20,poly1305)", + .cra_driver_name = "rfc7539esp-chacha20-" + "poly1305-caam", + .cra_blocksize = 1, + }, + .setkey = chachapoly_setkey, + .setauthsize = chachapoly_setauthsize, + .encrypt = chachapoly_encrypt, + .decrypt = chachapoly_decrypt, + .ivsize = 8, + .maxauthsize = POLY1305_DIGEST_SIZE, + }, + .aead.op = { + .do_one_request = aead_do_one_req, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | + OP_ALG_AAI_AEAD, + .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | + OP_ALG_AAI_AEAD, + .nodkp = true, + }, + }, +}; + +static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, + bool uses_dkp) +{ + dma_addr_t dma_addr; + struct caam_drv_private *priv; + const size_t sh_desc_enc_offset = offsetof(struct caam_ctx, + sh_desc_enc); + + ctx->jrdev = caam_jr_alloc(); + if (IS_ERR(ctx->jrdev)) { + pr_err("Job Ring Device allocation for transform failed\n"); + return PTR_ERR(ctx->jrdev); + } + + priv = dev_get_drvdata(ctx->jrdev->parent); + if (priv->era >= 6 && uses_dkp) + ctx->dir = DMA_BIDIRECTIONAL; + else + ctx->dir = DMA_TO_DEVICE; + + dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_enc, + offsetof(struct caam_ctx, + sh_desc_enc_dma) - + sh_desc_enc_offset, + ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); + if (dma_mapping_error(ctx->jrdev, dma_addr)) { + dev_err(ctx->jrdev, "unable to map key, shared descriptors\n"); + caam_jr_free(ctx->jrdev); + return -ENOMEM; + } + + ctx->sh_desc_enc_dma = dma_addr; + ctx->sh_desc_dec_dma = dma_addr + offsetof(struct caam_ctx, + sh_desc_dec) - + sh_desc_enc_offset; + ctx->key_dma = dma_addr + offsetof(struct caam_ctx, key) - + sh_desc_enc_offset; + + /* copy descriptor header template value */ + ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; + ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; + + return 0; +} + +static int caam_cra_init(struct crypto_skcipher *tfm) +{ + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct caam_skcipher_alg *caam_alg = + container_of(alg, typeof(*caam_alg), skcipher.base); + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm); + u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; + int ret = 0; + + if (alg_aai == OP_ALG_AAI_XTS) { + const char *tfm_name = crypto_tfm_alg_name(&tfm->base); + struct crypto_skcipher *fallback; + + fallback = crypto_alloc_skcipher(tfm_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(fallback)) { + pr_err("Failed to allocate %s fallback: %ld\n", + tfm_name, PTR_ERR(fallback)); + return PTR_ERR(fallback); + } + + ctx->fallback = fallback; + crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx) + + crypto_skcipher_reqsize(fallback)); + } else { + crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx)); + } + + ret = caam_init_common(ctx, &caam_alg->caam, false); + if (ret && ctx->fallback) + crypto_free_skcipher(ctx->fallback); + + return ret; +} + +static int caam_aead_init(struct crypto_aead *tfm) +{ + struct aead_alg *alg = crypto_aead_alg(tfm); + struct caam_aead_alg *caam_alg = + container_of(alg, struct caam_aead_alg, aead.base); + struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm); + + crypto_aead_set_reqsize(tfm, sizeof(struct caam_aead_req_ctx)); + + return caam_init_common(ctx, &caam_alg->caam, !caam_alg->caam.nodkp); +} + +static void caam_exit_common(struct caam_ctx *ctx) +{ + dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_enc_dma, + offsetof(struct caam_ctx, sh_desc_enc_dma) - + offsetof(struct caam_ctx, sh_desc_enc), + ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); + caam_jr_free(ctx->jrdev); +} + +static void caam_cra_exit(struct crypto_skcipher *tfm) +{ + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm); + + if (ctx->fallback) + crypto_free_skcipher(ctx->fallback); + caam_exit_common(ctx); +} + +static void caam_aead_exit(struct crypto_aead *tfm) +{ + caam_exit_common(crypto_aead_ctx_dma(tfm)); +} + +void caam_algapi_exit(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { + struct caam_aead_alg *t_alg = driver_aeads + i; + + if (t_alg->registered) + crypto_engine_unregister_aead(&t_alg->aead); + } + + for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { + struct caam_skcipher_alg *t_alg = driver_algs + i; + + if (t_alg->registered) + crypto_engine_unregister_skcipher(&t_alg->skcipher); + } +} + +static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) +{ + struct skcipher_alg *alg = &t_alg->skcipher.base; + + alg->base.cra_module = THIS_MODULE; + alg->base.cra_priority = CAAM_CRA_PRIORITY; + alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding(); + alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY); + + alg->init = caam_cra_init; + alg->exit = caam_cra_exit; +} + +static void caam_aead_alg_init(struct caam_aead_alg *t_alg) +{ + struct aead_alg *alg = &t_alg->aead.base; + + alg->base.cra_module = THIS_MODULE; + alg->base.cra_priority = CAAM_CRA_PRIORITY; + alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding(); + alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY; + + alg->init = caam_aead_init; + alg->exit = caam_aead_exit; +} + +int caam_algapi_init(struct device *ctrldev) +{ + struct caam_drv_private *priv = dev_get_drvdata(ctrldev); + int i = 0, err = 0; + u32 aes_vid, aes_inst, des_inst, md_vid, md_inst, ccha_inst, ptha_inst; + unsigned int md_limit = SHA512_DIGEST_SIZE; + bool registered = false, gcm_support; + + /* + * Register crypto algorithms the device supports. + * First, detect presence and attributes of DES, AES, and MD blocks. + */ + if (priv->era < 10) { + struct caam_perfmon __iomem *perfmon = &priv->jr[0]->perfmon; + u32 cha_vid, cha_inst, aes_rn; + + cha_vid = rd_reg32(&perfmon->cha_id_ls); + aes_vid = cha_vid & CHA_ID_LS_AES_MASK; + md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; + + cha_inst = rd_reg32(&perfmon->cha_num_ls); + des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> + CHA_ID_LS_DES_SHIFT; + aes_inst = cha_inst & CHA_ID_LS_AES_MASK; + md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; + ccha_inst = 0; + ptha_inst = 0; + + aes_rn = rd_reg32(&perfmon->cha_rev_ls) & CHA_ID_LS_AES_MASK; + gcm_support = !(aes_vid == CHA_VER_VID_AES_LP && aes_rn < 8); + } else { + struct version_regs __iomem *vreg = &priv->jr[0]->vreg; + u32 aesa, mdha; + + aesa = rd_reg32(&vreg->aesa); + mdha = rd_reg32(&vreg->mdha); + + aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT; + md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT; + + des_inst = rd_reg32(&vreg->desa) & CHA_VER_NUM_MASK; + aes_inst = aesa & CHA_VER_NUM_MASK; + md_inst = mdha & CHA_VER_NUM_MASK; + ccha_inst = rd_reg32(&vreg->ccha) & CHA_VER_NUM_MASK; + ptha_inst = rd_reg32(&vreg->ptha) & CHA_VER_NUM_MASK; + + gcm_support = aesa & CHA_VER_MISC_AES_GCM; + } + + /* If MD is present, limit digest size based on LP256 */ + if (md_inst && md_vid == CHA_VER_VID_MD_LP256) + md_limit = SHA256_DIGEST_SIZE; + + for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { + struct caam_skcipher_alg *t_alg = driver_algs + i; + u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK; + + /* Skip DES algorithms if not supported by device */ + if (!des_inst && + ((alg_sel == OP_ALG_ALGSEL_3DES) || + (alg_sel == OP_ALG_ALGSEL_DES))) + continue; + + /* Skip AES algorithms if not supported by device */ + if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) + continue; + + /* + * Check support for AES modes not available + * on LP devices. + */ + if (aes_vid == CHA_VER_VID_AES_LP && + (t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK) == + OP_ALG_AAI_XTS) + continue; + + caam_skcipher_alg_init(t_alg); + + err = crypto_engine_register_skcipher(&t_alg->skcipher); + if (err) { + pr_warn("%s alg registration failed\n", + t_alg->skcipher.base.base.cra_driver_name); + continue; + } + + t_alg->registered = true; + registered = true; + } + + for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { + struct caam_aead_alg *t_alg = driver_aeads + i; + u32 c1_alg_sel = t_alg->caam.class1_alg_type & + OP_ALG_ALGSEL_MASK; + u32 c2_alg_sel = t_alg->caam.class2_alg_type & + OP_ALG_ALGSEL_MASK; + u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; + + /* Skip DES algorithms if not supported by device */ + if (!des_inst && + ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || + (c1_alg_sel == OP_ALG_ALGSEL_DES))) + continue; + + /* Skip AES algorithms if not supported by device */ + if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES)) + continue; + + /* Skip CHACHA20 algorithms if not supported by device */ + if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 && !ccha_inst) + continue; + + /* Skip POLY1305 algorithms if not supported by device */ + if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 && !ptha_inst) + continue; + + /* Skip GCM algorithms if not supported by device */ + if (c1_alg_sel == OP_ALG_ALGSEL_AES && + alg_aai == OP_ALG_AAI_GCM && !gcm_support) + continue; + + /* + * Skip algorithms requiring message digests + * if MD or MD size is not supported by device. + */ + if (is_mdha(c2_alg_sel) && + (!md_inst || t_alg->aead.base.maxauthsize > md_limit)) + continue; + + caam_aead_alg_init(t_alg); + + err = crypto_engine_register_aead(&t_alg->aead); + if (err) { + pr_warn("%s alg registration failed\n", + t_alg->aead.base.base.cra_driver_name); + continue; + } + + t_alg->registered = true; + registered = true; + } + + if (registered) + pr_info("caam algorithms registered in /proc/crypto\n"); + + return err; +} diff --git a/drivers/crypto/caam/caamalg_desc.c b/drivers/crypto/caam/caamalg_desc.c new file mode 100644 index 000000000..04c1105eb --- /dev/null +++ b/drivers/crypto/caam/caamalg_desc.c @@ -0,0 +1,1725 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Shared descriptors for aead, skcipher algorithms + * + * Copyright 2016-2019, 2025 NXP + */ + +#include "compat.h" +#include "desc_constr.h" +#include "caamalg_desc.h" +#include + +/* + * For aead functions, read payload and write payload, + * both of which are specified in req->src and req->dst + */ +static inline void aead_append_src_dst(u32 *desc, u32 msg_type) +{ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | + KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH); +} + +/* Set DK bit in class 1 operation if shared */ +static inline void append_dec_op1(u32 *desc, u32 type) +{ + u32 *jump_cmd, *uncond_jump_cmd; + + /* DK bit is valid only for AES */ + if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) { + append_operation(desc, type | OP_ALG_AS_INITFINAL | + OP_ALG_DECRYPT); + return; + } + + jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); + append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT); + uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL); + set_jump_tgt_here(desc, jump_cmd); + append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT | + OP_ALG_AAI_DK); + set_jump_tgt_here(desc, uncond_jump_cmd); +} + +/** + * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor + * (non-protocol) with no (null) encryption. + * @desc: pointer to buffer used for descriptor construction + * @adata: pointer to authentication transform definitions. + * A split key is required for SEC Era < 6; the size of the split key + * is specified in this case. Valid algorithm values - one of + * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed + * with OP_ALG_AAI_HMAC_PRECOMP. + * @icvsize: integrity check value (ICV) size (truncated or full) + * @era: SEC Era + */ +void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata, + unsigned int icvsize, int era) +{ + u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* Skip if already shared */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + if (era < 6) { + if (adata->key_inline) + append_key_as_imm(desc, adata->key_virt, + adata->keylen_pad, adata->keylen, + CLASS_2 | KEY_DEST_MDHA_SPLIT | + KEY_ENC); + else + append_key(desc, adata->key_dma, adata->keylen, + CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); + } else { + append_proto_dkp(desc, adata); + } + set_jump_tgt_here(desc, key_jump_cmd); + + /* assoclen + cryptlen = seqinlen */ + append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); + + /* Prepare to read and write cryptlen + assoclen bytes */ + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + + /* + * MOVE_LEN opcode is not available in all SEC HW revisions, + * thus need to do some magic, i.e. self-patch the descriptor + * buffer. + */ + read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | + MOVE_DEST_MATH3 | + (0x6 << MOVE_LEN_SHIFT)); + write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | + MOVE_DEST_DESCBUF | + MOVE_WAITCOMP | + (0x8 << MOVE_LEN_SHIFT)); + + /* Class 2 operation */ + append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + + /* Read and write cryptlen bytes */ + aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); + + set_move_tgt_here(desc, read_move_cmd); + set_move_tgt_here(desc, write_move_cmd); + append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); + append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | + MOVE_AUX_LS); + + /* Write ICV */ + append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | + LDST_SRCDST_BYTE_CONTEXT); + + print_hex_dump_debug("aead null enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); +} +EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap); + +/** + * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor + * (non-protocol) with no (null) decryption. + * @desc: pointer to buffer used for descriptor construction + * @adata: pointer to authentication transform definitions. + * A split key is required for SEC Era < 6; the size of the split key + * is specified in this case. Valid algorithm values - one of + * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed + * with OP_ALG_AAI_HMAC_PRECOMP. + * @icvsize: integrity check value (ICV) size (truncated or full) + * @era: SEC Era + */ +void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata, + unsigned int icvsize, int era) +{ + u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* Skip if already shared */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + if (era < 6) { + if (adata->key_inline) + append_key_as_imm(desc, adata->key_virt, + adata->keylen_pad, adata->keylen, + CLASS_2 | KEY_DEST_MDHA_SPLIT | + KEY_ENC); + else + append_key(desc, adata->key_dma, adata->keylen, + CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); + } else { + append_proto_dkp(desc, adata); + } + set_jump_tgt_here(desc, key_jump_cmd); + + /* Class 2 operation */ + append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_DECRYPT | OP_ALG_ICV_ON); + + /* assoclen + cryptlen = seqoutlen */ + append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); + + /* Prepare to read and write cryptlen + assoclen bytes */ + append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); + + /* + * MOVE_LEN opcode is not available in all SEC HW revisions, + * thus need to do some magic, i.e. self-patch the descriptor + * buffer. + */ + read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | + MOVE_DEST_MATH2 | + (0x6 << MOVE_LEN_SHIFT)); + write_move_cmd = append_move(desc, MOVE_SRC_MATH2 | + MOVE_DEST_DESCBUF | + MOVE_WAITCOMP | + (0x8 << MOVE_LEN_SHIFT)); + + /* Read and write cryptlen bytes */ + aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); + + /* + * Insert a NOP here, since we need at least 4 instructions between + * code patching the descriptor buffer and the location being patched. + */ + jump_cmd = append_jump(desc, JUMP_TEST_ALL); + set_jump_tgt_here(desc, jump_cmd); + + set_move_tgt_here(desc, read_move_cmd); + set_move_tgt_here(desc, write_move_cmd); + append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); + append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | + MOVE_AUX_LS); + append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); + + /* Load ICV */ + append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 | + FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); + + print_hex_dump_debug("aead null dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); +} +EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap); + +static void init_sh_desc_key_aead(u32 * const desc, + struct alginfo * const cdata, + struct alginfo * const adata, + const bool is_rfc3686, u32 *nonce, int era) +{ + u32 *key_jump_cmd; + unsigned int enckeylen = cdata->keylen; + + /* Note: Context registers are saved. */ + init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); + + /* Skip if already shared */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + + /* + * RFC3686 specific: + * | key = {AUTH_KEY, ENC_KEY, NONCE} + * | enckeylen = encryption key size + nonce size + */ + if (is_rfc3686) + enckeylen -= CTR_RFC3686_NONCE_SIZE; + + if (era < 6) { + if (adata->key_inline) + append_key_as_imm(desc, adata->key_virt, + adata->keylen_pad, adata->keylen, + CLASS_2 | KEY_DEST_MDHA_SPLIT | + KEY_ENC); + else + append_key(desc, adata->key_dma, adata->keylen, + CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); + } else { + append_proto_dkp(desc, adata); + } + + if (cdata->key_inline) + append_key_as_imm(desc, cdata->key_virt, enckeylen, + enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + else + append_key(desc, cdata->key_dma, enckeylen, CLASS_1 | + KEY_DEST_CLASS_REG); + + /* Load Counter into CONTEXT1 reg */ + if (is_rfc3686) { + append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, + LDST_CLASS_IND_CCB | + LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); + append_move(desc, + MOVE_SRC_OUTFIFO | + MOVE_DEST_CLASS1CTX | + (16 << MOVE_OFFSET_SHIFT) | + (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); + } + + set_jump_tgt_here(desc, key_jump_cmd); +} + +/** + * cnstr_shdsc_aead_encap - IPSec ESP encapsulation shared descriptor + * (non-protocol). + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed + * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. + * @adata: pointer to authentication transform definitions. + * A split key is required for SEC Era < 6; the size of the split key + * is specified in this case. Valid algorithm values - one of + * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed + * with OP_ALG_AAI_HMAC_PRECOMP. + * @ivsize: initialization vector size + * @icvsize: integrity check value (ICV) size (truncated or full) + * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template + * @nonce: pointer to rfc3686 nonce + * @ctx1_iv_off: IV offset in CONTEXT1 register + * @is_qi: true when called from caam/qi + * @era: SEC Era + */ +void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata, + struct alginfo *adata, unsigned int ivsize, + unsigned int icvsize, const bool is_rfc3686, + u32 *nonce, const u32 ctx1_iv_off, const bool is_qi, + int era) +{ + /* Note: Context registers are saved. */ + init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era); + + /* Class 2 operation */ + append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + + if (is_qi) { + u32 *wait_load_cmd; + + /* REG3 = assoclen */ + append_seq_load(desc, 4, LDST_CLASS_DECO | + LDST_SRCDST_WORD_DECO_MATH3 | + (4 << LDST_OFFSET_SHIFT)); + + wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_CALM | JUMP_COND_NCP | + JUMP_COND_NOP | JUMP_COND_NIP | + JUMP_COND_NIFP); + set_jump_tgt_here(desc, wait_load_cmd); + + append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + (ctx1_iv_off << LDST_OFFSET_SHIFT)); + } + + /* Read and write assoclen bytes */ + if (is_qi || era < 3) { + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + } else { + append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ); + } + + /* Skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); + + /* read assoc before reading payload */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | + FIFOLDST_VLF); + + /* Load Counter into CONTEXT1 reg */ + if (is_rfc3686) + append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << + LDST_OFFSET_SHIFT)); + + /* Class 1 operation */ + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + + /* Read and write cryptlen bytes */ + append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); + + /* Write ICV */ + append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | + LDST_SRCDST_BYTE_CONTEXT); + + print_hex_dump_debug("aead enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); +} +EXPORT_SYMBOL(cnstr_shdsc_aead_encap); + +/** + * cnstr_shdsc_aead_decap - IPSec ESP decapsulation shared descriptor + * (non-protocol). + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed + * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. + * @adata: pointer to authentication transform definitions. + * A split key is required for SEC Era < 6; the size of the split key + * is specified in this case. Valid algorithm values - one of + * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed + * with OP_ALG_AAI_HMAC_PRECOMP. + * @ivsize: initialization vector size + * @icvsize: integrity check value (ICV) size (truncated or full) + * @geniv: whether to generate Encrypted Chain IV + * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template + * @nonce: pointer to rfc3686 nonce + * @ctx1_iv_off: IV offset in CONTEXT1 register + * @is_qi: true when called from caam/qi + * @era: SEC Era + */ +void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata, + struct alginfo *adata, unsigned int ivsize, + unsigned int icvsize, const bool geniv, + const bool is_rfc3686, u32 *nonce, + const u32 ctx1_iv_off, const bool is_qi, int era) +{ + /* Note: Context registers are saved. */ + init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era); + + /* Class 2 operation */ + append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_DECRYPT | OP_ALG_ICV_ON); + + if (is_qi) { + u32 *wait_load_cmd; + + /* REG3 = assoclen */ + append_seq_load(desc, 4, LDST_CLASS_DECO | + LDST_SRCDST_WORD_DECO_MATH3 | + (4 << LDST_OFFSET_SHIFT)); + + wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_CALM | JUMP_COND_NCP | + JUMP_COND_NOP | JUMP_COND_NIP | + JUMP_COND_NIFP); + set_jump_tgt_here(desc, wait_load_cmd); + + if (!geniv) + append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + (ctx1_iv_off << LDST_OFFSET_SHIFT)); + } + + /* Read and write assoclen bytes */ + if (is_qi || era < 3) { + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + if (geniv) + append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM, + ivsize); + else + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, + CAAM_CMD_SZ); + } else { + append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ); + if (geniv) + append_math_add_imm_u32(desc, VARSEQOUTLEN, DPOVRD, IMM, + ivsize); + else + append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, + CAAM_CMD_SZ); + } + + /* Skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); + + /* read assoc before reading payload */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | + KEY_VLF); + + if (geniv) { + append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + (ctx1_iv_off << LDST_OFFSET_SHIFT)); + append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | + (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize); + } + + /* Load Counter into CONTEXT1 reg */ + if (is_rfc3686) + append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << + LDST_OFFSET_SHIFT)); + + /* Choose operation */ + if (ctx1_iv_off) + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_DECRYPT); + else + append_dec_op1(desc, cdata->algtype); + + /* Read and write cryptlen bytes */ + append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); + aead_append_src_dst(desc, FIFOLD_TYPE_MSG); + + /* Load ICV */ + append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 | + FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); + + print_hex_dump_debug("aead dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); +} +EXPORT_SYMBOL(cnstr_shdsc_aead_decap); + +/** + * cnstr_shdsc_aead_givencap - IPSec ESP encapsulation shared descriptor + * (non-protocol) with HW-generated initialization + * vector. + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed + * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. + * @adata: pointer to authentication transform definitions. + * A split key is required for SEC Era < 6; the size of the split key + * is specified in this case. Valid algorithm values - one of + * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed + * with OP_ALG_AAI_HMAC_PRECOMP. + * @ivsize: initialization vector size + * @icvsize: integrity check value (ICV) size (truncated or full) + * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template + * @nonce: pointer to rfc3686 nonce + * @ctx1_iv_off: IV offset in CONTEXT1 register + * @is_qi: true when called from caam/qi + * @era: SEC Era + */ +void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata, + struct alginfo *adata, unsigned int ivsize, + unsigned int icvsize, const bool is_rfc3686, + u32 *nonce, const u32 ctx1_iv_off, + const bool is_qi, int era) +{ + u32 geniv, moveiv; + u32 *wait_cmd; + + /* Note: Context registers are saved. */ + init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era); + + if (is_qi) { + u32 *wait_load_cmd; + + /* REG3 = assoclen */ + append_seq_load(desc, 4, LDST_CLASS_DECO | + LDST_SRCDST_WORD_DECO_MATH3 | + (4 << LDST_OFFSET_SHIFT)); + + wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_CALM | JUMP_COND_NCP | + JUMP_COND_NOP | JUMP_COND_NIP | + JUMP_COND_NIFP); + set_jump_tgt_here(desc, wait_load_cmd); + } + + if (is_rfc3686) { + if (is_qi) + append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + (ctx1_iv_off << LDST_OFFSET_SHIFT)); + + goto copy_iv; + } + + /* Generate IV */ + geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | + NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | + NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT); + append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | + LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); + append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); + append_move(desc, MOVE_WAITCOMP | + MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | + (ctx1_iv_off << MOVE_OFFSET_SHIFT) | + (ivsize << MOVE_LEN_SHIFT)); + append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); + +copy_iv: + /* Copy IV to class 1 context */ + append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | + (ctx1_iv_off << MOVE_OFFSET_SHIFT) | + (ivsize << MOVE_LEN_SHIFT)); + + /* Return to encryption */ + append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + + /* Read and write assoclen bytes */ + if (is_qi || era < 3) { + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + } else { + append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ); + } + + /* Skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); + + /* read assoc before reading payload */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | + KEY_VLF); + + /* Copy iv from outfifo to class 2 fifo */ + moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 | + NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT); + append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB | + LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); + append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB | + LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); + + /* Load Counter into CONTEXT1 reg */ + if (is_rfc3686) + append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << + LDST_OFFSET_SHIFT)); + + /* Class 1 operation */ + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + + /* Will write ivsize + cryptlen */ + append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + + /* Not need to reload iv */ + append_seq_fifo_load(desc, ivsize, + FIFOLD_CLASS_SKIP); + + /* Will read cryptlen */ + append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + + /* + * Wait for IV transfer (ofifo -> class2) to finish before starting + * ciphertext transfer (ofifo -> external memory). + */ + wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NIFP); + set_jump_tgt_here(desc, wait_cmd); + + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF | + FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH); + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); + + /* Write ICV */ + append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | + LDST_SRCDST_BYTE_CONTEXT); + + print_hex_dump_debug("aead givenc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); +} +EXPORT_SYMBOL(cnstr_shdsc_aead_givencap); + +/** + * cnstr_shdsc_gcm_encap - gcm encapsulation shared descriptor + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. + * @ivsize: initialization vector size + * @icvsize: integrity check value (ICV) size (truncated or full) + * @is_qi: true when called from caam/qi + */ +void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, unsigned int icvsize, + const bool is_qi) +{ + u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1, + *zero_assoc_jump_cmd2; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* skip key loading if they are loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + if (cdata->key_inline) + append_key_as_imm(desc, cdata->key_virt, cdata->keylen, + cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + else + append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | + KEY_DEST_CLASS_REG); + set_jump_tgt_here(desc, key_jump_cmd); + + /* class 1 operation */ + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + + if (is_qi) { + u32 *wait_load_cmd; + + /* REG3 = assoclen */ + append_seq_load(desc, 4, LDST_CLASS_DECO | + LDST_SRCDST_WORD_DECO_MATH3 | + (4 << LDST_OFFSET_SHIFT)); + + wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_CALM | JUMP_COND_NCP | + JUMP_COND_NOP | JUMP_COND_NIP | + JUMP_COND_NIFP); + set_jump_tgt_here(desc, wait_load_cmd); + + append_math_sub_imm_u32(desc, VARSEQOUTLEN, SEQINLEN, IMM, + ivsize); + } else { + append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, + CAAM_CMD_SZ); + } + + /* if assoclen + cryptlen is ZERO, skip to ICV write */ + zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + + if (is_qi) + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); + + /* if assoclen is ZERO, skip reading the assoc data */ + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + + /* skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); + + /* cryptlen = seqinlen - assoclen */ + append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ); + + /* if cryptlen is ZERO jump to zero-payload commands */ + zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + + /* read assoc data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); + set_jump_tgt_here(desc, zero_assoc_jump_cmd1); + + append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + + /* write encrypted data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); + + /* read payload data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); + + /* jump to ICV writing */ + if (is_qi) + append_jump(desc, JUMP_TEST_ALL | 4); + else + append_jump(desc, JUMP_TEST_ALL | 2); + + /* zero-payload commands */ + set_jump_tgt_here(desc, zero_payload_jump_cmd); + + /* read assoc data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1); + if (is_qi) + /* jump to ICV writing */ + append_jump(desc, JUMP_TEST_ALL | 2); + + /* There is no input data */ + set_jump_tgt_here(desc, zero_assoc_jump_cmd2); + + if (is_qi) + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | + FIFOLD_TYPE_LAST1); + + /* write ICV */ + append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT); + + print_hex_dump_debug("gcm enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); +} +EXPORT_SYMBOL(cnstr_shdsc_gcm_encap); + +/** + * cnstr_shdsc_gcm_decap - gcm decapsulation shared descriptor + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. + * @ivsize: initialization vector size + * @icvsize: integrity check value (ICV) size (truncated or full) + * @is_qi: true when called from caam/qi + */ +void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, unsigned int icvsize, + const bool is_qi) +{ + u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* skip key loading if they are loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | + JUMP_TEST_ALL | JUMP_COND_SHRD); + if (cdata->key_inline) + append_key_as_imm(desc, cdata->key_virt, cdata->keylen, + cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + else + append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | + KEY_DEST_CLASS_REG); + set_jump_tgt_here(desc, key_jump_cmd); + + /* class 1 operation */ + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_DECRYPT | OP_ALG_ICV_ON); + + if (is_qi) { + u32 *wait_load_cmd; + + /* REG3 = assoclen */ + append_seq_load(desc, 4, LDST_CLASS_DECO | + LDST_SRCDST_WORD_DECO_MATH3 | + (4 << LDST_OFFSET_SHIFT)); + + wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_CALM | JUMP_COND_NCP | + JUMP_COND_NOP | JUMP_COND_NIP | + JUMP_COND_NIFP); + set_jump_tgt_here(desc, wait_load_cmd); + + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); + } + + /* if assoclen is ZERO, skip reading the assoc data */ + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + + /* skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); + + /* read assoc data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); + + set_jump_tgt_here(desc, zero_assoc_jump_cmd1); + + /* cryptlen = seqoutlen - assoclen */ + append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); + + /* jump to zero-payload command if cryptlen is zero */ + zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + + append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); + + /* store encrypted data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); + + /* read payload data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); + + /* zero-payload command */ + set_jump_tgt_here(desc, zero_payload_jump_cmd); + + /* read ICV */ + append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); + + print_hex_dump_debug("gcm dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); +} +EXPORT_SYMBOL(cnstr_shdsc_gcm_decap); + +/** + * cnstr_shdsc_rfc4106_encap - IPSec ESP gcm encapsulation shared descriptor + * (non-protocol). + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. + * @ivsize: initialization vector size + * @icvsize: integrity check value (ICV) size (truncated or full) + * @is_qi: true when called from caam/qi + * + * Input sequence: AAD | PTXT + * Output sequence: AAD | CTXT | ICV + * AAD length (assoclen), which includes the IV length, is available in Math3. + */ +void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, unsigned int icvsize, + const bool is_qi) +{ + u32 *key_jump_cmd, *zero_cryptlen_jump_cmd, *skip_instructions; + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* Skip key loading if it is loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + if (cdata->key_inline) + append_key_as_imm(desc, cdata->key_virt, cdata->keylen, + cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + else + append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | + KEY_DEST_CLASS_REG); + set_jump_tgt_here(desc, key_jump_cmd); + + /* Class 1 operation */ + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + + if (is_qi) { + u32 *wait_load_cmd; + + /* REG3 = assoclen */ + append_seq_load(desc, 4, LDST_CLASS_DECO | + LDST_SRCDST_WORD_DECO_MATH3 | + (4 << LDST_OFFSET_SHIFT)); + + wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_CALM | JUMP_COND_NCP | + JUMP_COND_NOP | JUMP_COND_NIP | + JUMP_COND_NIFP); + set_jump_tgt_here(desc, wait_load_cmd); + + /* Read salt and IV */ + append_fifo_load_as_imm(desc, (void *)(cdata->key_virt + + cdata->keylen), 4, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV); + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); + } + + append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + + /* Skip AAD */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); + + /* Read cryptlen and set this value into VARSEQOUTLEN */ + append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ); + + /* If cryptlen is ZERO jump to AAD command */ + zero_cryptlen_jump_cmd = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + + /* Read AAD data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); + + /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA); + + /* Skip IV */ + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP); + append_math_add(desc, VARSEQINLEN, VARSEQOUTLEN, REG0, CAAM_CMD_SZ); + + /* Write encrypted data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); + + /* Read payload data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); + + /* Jump instructions to avoid double reading of AAD */ + skip_instructions = append_jump(desc, JUMP_TEST_ALL); + + /* There is no input data, cryptlen = 0 */ + set_jump_tgt_here(desc, zero_cryptlen_jump_cmd); + + /* Read AAD */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1); + + set_jump_tgt_here(desc, skip_instructions); + + /* Write ICV */ + append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT); + + print_hex_dump_debug("rfc4106 enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); +} +EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap); + +/** + * cnstr_shdsc_rfc4106_decap - IPSec ESP gcm decapsulation shared descriptor + * (non-protocol). + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. + * @ivsize: initialization vector size + * @icvsize: integrity check value (ICV) size (truncated or full) + * @is_qi: true when called from caam/qi + */ +void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, unsigned int icvsize, + const bool is_qi) +{ + u32 *key_jump_cmd; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* Skip key loading if it is loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + if (cdata->key_inline) + append_key_as_imm(desc, cdata->key_virt, cdata->keylen, + cdata->keylen, CLASS_1 | + KEY_DEST_CLASS_REG); + else + append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | + KEY_DEST_CLASS_REG); + set_jump_tgt_here(desc, key_jump_cmd); + + /* Class 1 operation */ + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_DECRYPT | OP_ALG_ICV_ON); + + if (is_qi) { + u32 *wait_load_cmd; + + /* REG3 = assoclen */ + append_seq_load(desc, 4, LDST_CLASS_DECO | + LDST_SRCDST_WORD_DECO_MATH3 | + (4 << LDST_OFFSET_SHIFT)); + + wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_CALM | JUMP_COND_NCP | + JUMP_COND_NOP | JUMP_COND_NIP | + JUMP_COND_NIFP); + set_jump_tgt_here(desc, wait_load_cmd); + + /* Read salt and IV */ + append_fifo_load_as_imm(desc, (void *)(cdata->key_virt + + cdata->keylen), 4, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV); + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); + } + + append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + + /* Read assoc data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); + + /* Skip IV */ + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP); + + /* Will read cryptlen bytes */ + append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ); + + /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); + + /* Skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); + + /* Will write cryptlen bytes */ + append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); + + /* Store payload data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); + + /* Read encrypted data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); + + /* Read ICV */ + append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); + + print_hex_dump_debug("rfc4106 dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); +} +EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap); + +/** + * cnstr_shdsc_rfc4543_encap - IPSec ESP gmac encapsulation shared descriptor + * (non-protocol). + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. + * @ivsize: initialization vector size + * @icvsize: integrity check value (ICV) size (truncated or full) + * @is_qi: true when called from caam/qi + */ +void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, unsigned int icvsize, + const bool is_qi) +{ + u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* Skip key loading if it is loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + if (cdata->key_inline) + append_key_as_imm(desc, cdata->key_virt, cdata->keylen, + cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + else + append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | + KEY_DEST_CLASS_REG); + set_jump_tgt_here(desc, key_jump_cmd); + + /* Class 1 operation */ + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + + if (is_qi) { + /* assoclen is not needed, skip it */ + append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP); + + /* Read salt and IV */ + append_fifo_load_as_imm(desc, (void *)(cdata->key_virt + + cdata->keylen), 4, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV); + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); + } + + /* assoclen + cryptlen = seqinlen */ + append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); + + /* + * MOVE_LEN opcode is not available in all SEC HW revisions, + * thus need to do some magic, i.e. self-patch the descriptor + * buffer. + */ + read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | + (0x6 << MOVE_LEN_SHIFT)); + write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | + (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP); + + /* Will read assoclen + cryptlen bytes */ + append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + + /* Will write assoclen + cryptlen bytes */ + append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + + /* Read and write assoclen + cryptlen bytes */ + aead_append_src_dst(desc, FIFOLD_TYPE_AAD); + + set_move_tgt_here(desc, read_move_cmd); + set_move_tgt_here(desc, write_move_cmd); + append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); + /* Move payload data to OFIFO */ + append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); + + /* Write ICV */ + append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT); + + print_hex_dump_debug("rfc4543 enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); +} +EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap); + +/** + * cnstr_shdsc_rfc4543_decap - IPSec ESP gmac decapsulation shared descriptor + * (non-protocol). + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. + * @ivsize: initialization vector size + * @icvsize: integrity check value (ICV) size (truncated or full) + * @is_qi: true when called from caam/qi + */ +void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, unsigned int icvsize, + const bool is_qi) +{ + u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* Skip key loading if it is loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + if (cdata->key_inline) + append_key_as_imm(desc, cdata->key_virt, cdata->keylen, + cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + else + append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | + KEY_DEST_CLASS_REG); + set_jump_tgt_here(desc, key_jump_cmd); + + /* Class 1 operation */ + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_DECRYPT | OP_ALG_ICV_ON); + + if (is_qi) { + /* assoclen is not needed, skip it */ + append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP); + + /* Read salt and IV */ + append_fifo_load_as_imm(desc, (void *)(cdata->key_virt + + cdata->keylen), 4, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV); + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); + } + + /* assoclen + cryptlen = seqoutlen */ + append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ); + + /* + * MOVE_LEN opcode is not available in all SEC HW revisions, + * thus need to do some magic, i.e. self-patch the descriptor + * buffer. + */ + read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | + (0x6 << MOVE_LEN_SHIFT)); + write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | + (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP); + + /* Will read assoclen + cryptlen bytes */ + append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); + + /* Will write assoclen + cryptlen bytes */ + append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); + + /* Store payload data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); + + /* In-snoop assoclen + cryptlen data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1); + + set_move_tgt_here(desc, read_move_cmd); + set_move_tgt_here(desc, write_move_cmd); + append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); + /* Move payload data to OFIFO */ + append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); + append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); + + /* Read ICV */ + append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); + + print_hex_dump_debug("rfc4543 dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); +} +EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap); + +/** + * cnstr_shdsc_chachapoly - Chacha20 + Poly1305 generic AEAD (rfc7539) and + * IPsec ESP (rfc7634, a.k.a. rfc7539esp) shared + * descriptor (non-protocol). + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_ALG_ALGSEL_CHACHA20 ANDed with + * OP_ALG_AAI_AEAD. + * @adata: pointer to authentication transform definitions + * Valid algorithm values - OP_ALG_ALGSEL_POLY1305 ANDed with + * OP_ALG_AAI_AEAD. + * @ivsize: initialization vector size + * @icvsize: integrity check value (ICV) size (truncated or full) + * @encap: true if encapsulation, false if decapsulation + * @is_qi: true when called from caam/qi + */ +void cnstr_shdsc_chachapoly(u32 * const desc, struct alginfo *cdata, + struct alginfo *adata, unsigned int ivsize, + unsigned int icvsize, const bool encap, + const bool is_qi) +{ + u32 *key_jump_cmd, *wait_cmd; + u32 nfifo; + const bool is_ipsec = (ivsize != CHACHAPOLY_IV_SIZE); + + /* Note: Context registers are saved. */ + init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); + + /* skip key loading if they are loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + + append_key_as_imm(desc, cdata->key_virt, cdata->keylen, cdata->keylen, + CLASS_1 | KEY_DEST_CLASS_REG); + + /* For IPsec load the salt from keymat in the context register */ + if (is_ipsec) + append_load_as_imm(desc, cdata->key_virt + cdata->keylen, 4, + LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT | + 4 << LDST_OFFSET_SHIFT); + + set_jump_tgt_here(desc, key_jump_cmd); + + /* Class 2 and 1 operations: Poly & ChaCha */ + if (encap) { + append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + } else { + append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_DECRYPT | OP_ALG_ICV_ON); + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_DECRYPT); + } + + if (is_qi) { + u32 *wait_load_cmd; + u32 ctx1_iv_off = is_ipsec ? 8 : 4; + + /* REG3 = assoclen */ + append_seq_load(desc, 4, LDST_CLASS_DECO | + LDST_SRCDST_WORD_DECO_MATH3 | + 4 << LDST_OFFSET_SHIFT); + + wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_CALM | JUMP_COND_NCP | + JUMP_COND_NOP | JUMP_COND_NIP | + JUMP_COND_NIFP); + set_jump_tgt_here(desc, wait_load_cmd); + + append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + ctx1_iv_off << LDST_OFFSET_SHIFT); + } + + /* + * MAGIC with NFIFO + * Read associated data from the input and send them to class1 and + * class2 alignment blocks. From class1 send data to output fifo and + * then write it to memory since we don't need to encrypt AD. + */ + nfifo = NFIFOENTRY_DEST_BOTH | NFIFOENTRY_FC1 | NFIFOENTRY_FC2 | + NFIFOENTRY_DTYPE_POLY | NFIFOENTRY_BND; + append_load_imm_u32(desc, nfifo, LDST_CLASS_IND_CCB | + LDST_SRCDST_WORD_INFO_FIFO_SM | LDLEN_MATH3); + + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + append_seq_fifo_load(desc, 0, FIFOLD_TYPE_NOINFOFIFO | + FIFOLD_CLASS_CLASS1 | LDST_VLF); + append_move_len(desc, MOVE_AUX_LS | MOVE_SRC_AUX_ABLK | + MOVE_DEST_OUTFIFO | MOVELEN_MRSEL_MATH3); + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | LDST_VLF); + + /* IPsec - copy IV at the output */ + if (is_ipsec) + append_seq_fifo_store(desc, ivsize, FIFOST_TYPE_METADATA | + 0x2 << 25); + + wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TYPE_LOCAL | + JUMP_COND_NOP | JUMP_TEST_ALL); + set_jump_tgt_here(desc, wait_cmd); + + if (encap) { + /* Read and write cryptlen bytes */ + append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, + CAAM_CMD_SZ); + aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); + + /* Write ICV */ + append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | + LDST_SRCDST_BYTE_CONTEXT); + } else { + /* Read and write cryptlen bytes */ + append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, + CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, + CAAM_CMD_SZ); + aead_append_src_dst(desc, FIFOLD_TYPE_MSG); + + /* Load ICV for verification */ + append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 | + FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); + } + + print_hex_dump_debug("chachapoly shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); +} +EXPORT_SYMBOL(cnstr_shdsc_chachapoly); + +/* For skcipher encrypt and decrypt, read from req->src and write to req->dst */ +static inline void skcipher_append_src_dst(u32 *desc) +{ + append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | + KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); +} + +void cnstr_desc_skcipher_enc_dec(u32 * const desc, struct alginfo *cdata, + dma_addr_t src, dma_addr_t dst, unsigned int data_sz, + unsigned int in_options, unsigned int out_options, + unsigned int ivsize, const bool encrypt) +{ + u32 options = cdata->algtype | OP_ALG_AS_INIT; + + if (encrypt) + options |= OP_ALG_ENCRYPT; + else + options |= OP_ALG_DECRYPT; + + init_job_desc(desc, 0); + + append_jump(desc, JUMP_JSL | JUMP_TYPE_LOCAL | + JUMP_COND_NOP | JUMP_TEST_ALL | 1); + + append_key(desc, cdata->protected_key_dma, cdata->plain_keylen, + CLASS_1 | KEY_DEST_CLASS_REG | cdata->key_cmd_opt); + + append_seq_in_ptr(desc, src, data_sz, in_options); + + append_seq_out_ptr(desc, dst, data_sz, out_options); + + /* Load IV, if there is one */ + if (ivsize) + append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB); + + append_operation(desc, options); + + skcipher_append_src_dst(desc); + + /* Store IV */ + if (ivsize) + append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB); + + print_hex_dump_debug("skcipher_enc_dec job desc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); +} +EXPORT_SYMBOL(cnstr_desc_skcipher_enc_dec); + +void cnstr_desc_protected_blob_decap(u32 * const desc, struct alginfo *cdata, + dma_addr_t next_desc_addr) +{ + u32 protected_store; + + init_job_desc(desc, 0); + + /* Load key modifier */ + append_load_as_imm(desc, KEYMOD, sizeof(KEYMOD) - 1, + LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_KEY); + + append_seq_in_ptr_intlen(desc, cdata->key_dma, + cdata->plain_keylen + CAAM_BLOB_OVERHEAD, 0); + + append_seq_out_ptr_intlen(desc, cdata->protected_key_dma, + cdata->plain_keylen, 0); + + protected_store = OP_PCLID_BLOB | OP_PCL_BLOB_BLACK; + if ((cdata->key_cmd_opt >> KEY_EKT_OFFSET) & 1) + protected_store |= OP_PCL_BLOB_EKT; + + append_operation(desc, OP_TYPE_DECAP_PROTOCOL | protected_store); + + if (next_desc_addr) { + append_jump(desc, JUMP_TYPE_NONLOCAL | JUMP_TEST_ALL); + append_ptr(desc, next_desc_addr); + } + + print_hex_dump_debug("protected blob decap job desc@" __stringify(__LINE__) ":", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +} +EXPORT_SYMBOL(cnstr_desc_protected_blob_decap); + +/** + * cnstr_shdsc_skcipher_encap - skcipher encapsulation shared descriptor + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed + * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128 + * - OP_ALG_ALGSEL_CHACHA20 + * @ivsize: initialization vector size + * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template + * @ctx1_iv_off: IV offset in CONTEXT1 register + */ +void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, const bool is_rfc3686, + const u32 ctx1_iv_off) +{ + u32 *key_jump_cmd; + u32 options = cdata->algtype | OP_ALG_AS_INIT | OP_ALG_ENCRYPT; + bool is_chacha20 = ((cdata->algtype & OP_ALG_ALGSEL_MASK) == + OP_ALG_ALGSEL_CHACHA20); + + init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); + /* Skip if already shared */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + + /* Load class1 key only */ + append_key_as_imm(desc, cdata->key_virt, cdata->keylen, + cdata->plain_keylen, CLASS_1 | KEY_DEST_CLASS_REG + | cdata->key_cmd_opt); + + /* Load nonce into CONTEXT1 reg */ + if (is_rfc3686) { + const u8 *nonce = cdata->key_virt + cdata->keylen; + + append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, + LDST_CLASS_IND_CCB | + LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | + MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) | + (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); + } + + set_jump_tgt_here(desc, key_jump_cmd); + + /* Load IV, if there is one */ + if (ivsize) + append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB | (ctx1_iv_off << + LDST_OFFSET_SHIFT)); + + /* Load counter into CONTEXT1 reg */ + if (is_rfc3686) + append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << + LDST_OFFSET_SHIFT)); + + /* Load operation */ + if (is_chacha20) + options |= OP_ALG_AS_FINALIZE; + append_operation(desc, options); + + /* Perform operation */ + skcipher_append_src_dst(desc); + + /* Store IV */ + if (!is_chacha20 && ivsize) + append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB | (ctx1_iv_off << + LDST_OFFSET_SHIFT)); + + print_hex_dump_debug("skcipher enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); +} +EXPORT_SYMBOL(cnstr_shdsc_skcipher_encap); + +/** + * cnstr_shdsc_skcipher_decap - skcipher decapsulation shared descriptor + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed + * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128 + * - OP_ALG_ALGSEL_CHACHA20 + * @ivsize: initialization vector size + * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template + * @ctx1_iv_off: IV offset in CONTEXT1 register + */ +void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, const bool is_rfc3686, + const u32 ctx1_iv_off) +{ + u32 *key_jump_cmd; + bool is_chacha20 = ((cdata->algtype & OP_ALG_ALGSEL_MASK) == + OP_ALG_ALGSEL_CHACHA20); + + init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); + /* Skip if already shared */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + + /* Load class1 key only */ + append_key_as_imm(desc, cdata->key_virt, cdata->keylen, + cdata->plain_keylen, CLASS_1 | KEY_DEST_CLASS_REG + | cdata->key_cmd_opt); + + /* Load nonce into CONTEXT1 reg */ + if (is_rfc3686) { + const u8 *nonce = cdata->key_virt + cdata->keylen; + + append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, + LDST_CLASS_IND_CCB | + LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | + MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) | + (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); + } + + set_jump_tgt_here(desc, key_jump_cmd); + + /* Load IV, if there is one */ + if (ivsize) + append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB | (ctx1_iv_off << + LDST_OFFSET_SHIFT)); + + /* Load counter into CONTEXT1 reg */ + if (is_rfc3686) + append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << + LDST_OFFSET_SHIFT)); + + /* Choose operation */ + if (ctx1_iv_off) + append_operation(desc, cdata->algtype | OP_ALG_AS_INIT | + OP_ALG_DECRYPT); + else + append_dec_op1(desc, cdata->algtype); + + /* Perform operation */ + skcipher_append_src_dst(desc); + + /* Store IV */ + if (!is_chacha20 && ivsize) + append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB | (ctx1_iv_off << + LDST_OFFSET_SHIFT)); + + print_hex_dump_debug("skcipher dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); +} +EXPORT_SYMBOL(cnstr_shdsc_skcipher_decap); + +/** + * cnstr_shdsc_xts_skcipher_encap - xts skcipher encapsulation shared descriptor + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS. + */ +void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata) +{ + /* + * Set sector size to a big value, practically disabling + * sector size segmentation in xts implementation. We cannot + * take full advantage of this HW feature with existing + * crypto API / dm-crypt SW architecture. + */ + __be64 sector_size = cpu_to_be64(BIT(15)); + u32 *key_jump_cmd; + + init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); + /* Skip if already shared */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + + /* Load class1 keys only */ + append_key_as_imm(desc, cdata->key_virt, cdata->keylen, + cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + + /* Load sector size with index 40 bytes (0x28) */ + append_load_as_imm(desc, (void *)§or_size, 8, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + (0x28 << LDST_OFFSET_SHIFT)); + + set_jump_tgt_here(desc, key_jump_cmd); + + /* + * create sequence for loading the sector index / 16B tweak value + * Lower 8B of IV - sector index / tweak lower half + * Upper 8B of IV - upper half of 16B tweak + */ + append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (0x20 << LDST_OFFSET_SHIFT)); + append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (0x30 << LDST_OFFSET_SHIFT)); + + /* Load operation */ + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + + /* Perform operation */ + skcipher_append_src_dst(desc); + + /* Store lower 8B and upper 8B of IV */ + append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (0x20 << LDST_OFFSET_SHIFT)); + append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (0x30 << LDST_OFFSET_SHIFT)); + + print_hex_dump_debug("xts skcipher enc shdesc@" __stringify(__LINE__) + ": ", DUMP_PREFIX_ADDRESS, 16, 4, + desc, desc_bytes(desc), 1); +} +EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_encap); + +/** + * cnstr_shdsc_xts_skcipher_decap - xts skcipher decapsulation shared descriptor + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS. + */ +void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata) +{ + /* + * Set sector size to a big value, practically disabling + * sector size segmentation in xts implementation. We cannot + * take full advantage of this HW feature with existing + * crypto API / dm-crypt SW architecture. + */ + __be64 sector_size = cpu_to_be64(BIT(15)); + u32 *key_jump_cmd; + + init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); + /* Skip if already shared */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + + /* Load class1 key only */ + append_key_as_imm(desc, cdata->key_virt, cdata->keylen, + cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + + /* Load sector size with index 40 bytes (0x28) */ + append_load_as_imm(desc, (void *)§or_size, 8, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + (0x28 << LDST_OFFSET_SHIFT)); + + set_jump_tgt_here(desc, key_jump_cmd); + + /* + * create sequence for loading the sector index / 16B tweak value + * Lower 8B of IV - sector index / tweak lower half + * Upper 8B of IV - upper half of 16B tweak + */ + append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (0x20 << LDST_OFFSET_SHIFT)); + append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (0x30 << LDST_OFFSET_SHIFT)); + /* Load operation */ + append_dec_op1(desc, cdata->algtype); + + /* Perform operation */ + skcipher_append_src_dst(desc); + + /* Store lower 8B and upper 8B of IV */ + append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (0x20 << LDST_OFFSET_SHIFT)); + append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (0x30 << LDST_OFFSET_SHIFT)); + + print_hex_dump_debug("xts skcipher dec shdesc@" __stringify(__LINE__) + ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +} +EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_decap); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("FSL CAAM descriptor support"); +MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); diff --git a/drivers/crypto/caam/caamalg_desc.h b/drivers/crypto/caam/caamalg_desc.h new file mode 100644 index 000000000..323490a4a --- /dev/null +++ b/drivers/crypto/caam/caamalg_desc.h @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Shared descriptors for aead, skcipher algorithms + * + * Copyright 2016, 2025 NXP + */ + +#ifndef _CAAMALG_DESC_H_ +#define _CAAMALG_DESC_H_ + +/* length of descriptors text */ +#define DESC_AEAD_BASE (4 * CAAM_CMD_SZ) +#define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 11 * CAAM_CMD_SZ) +#define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ) +#define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 8 * CAAM_CMD_SZ) +#define DESC_QI_AEAD_ENC_LEN (DESC_AEAD_ENC_LEN + 3 * CAAM_CMD_SZ) +#define DESC_QI_AEAD_DEC_LEN (DESC_AEAD_DEC_LEN + 3 * CAAM_CMD_SZ) +#define DESC_QI_AEAD_GIVENC_LEN (DESC_AEAD_GIVENC_LEN + 3 * CAAM_CMD_SZ) + +/* Note: Nonce is counted in cdata.keylen */ +#define DESC_AEAD_CTR_RFC3686_LEN (4 * CAAM_CMD_SZ) + +#define DESC_AEAD_NULL_BASE (3 * CAAM_CMD_SZ) +#define DESC_AEAD_NULL_ENC_LEN (DESC_AEAD_NULL_BASE + 11 * CAAM_CMD_SZ) +#define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 13 * CAAM_CMD_SZ) + +#define DESC_GCM_BASE (3 * CAAM_CMD_SZ) +#define DESC_GCM_ENC_LEN (DESC_GCM_BASE + 16 * CAAM_CMD_SZ) +#define DESC_GCM_DEC_LEN (DESC_GCM_BASE + 12 * CAAM_CMD_SZ) +#define DESC_QI_GCM_ENC_LEN (DESC_GCM_ENC_LEN + 6 * CAAM_CMD_SZ) +#define DESC_QI_GCM_DEC_LEN (DESC_GCM_DEC_LEN + 3 * CAAM_CMD_SZ) + +#define DESC_RFC4106_BASE (3 * CAAM_CMD_SZ) +#define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 16 * CAAM_CMD_SZ) +#define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ) +#define DESC_QI_RFC4106_ENC_LEN (DESC_RFC4106_ENC_LEN + 5 * CAAM_CMD_SZ) +#define DESC_QI_RFC4106_DEC_LEN (DESC_RFC4106_DEC_LEN + 5 * CAAM_CMD_SZ) + +#define DESC_RFC4543_BASE (3 * CAAM_CMD_SZ) +#define DESC_RFC4543_ENC_LEN (DESC_RFC4543_BASE + 11 * CAAM_CMD_SZ) +#define DESC_RFC4543_DEC_LEN (DESC_RFC4543_BASE + 12 * CAAM_CMD_SZ) +#define DESC_QI_RFC4543_ENC_LEN (DESC_RFC4543_ENC_LEN + 4 * CAAM_CMD_SZ) +#define DESC_QI_RFC4543_DEC_LEN (DESC_RFC4543_DEC_LEN + 4 * CAAM_CMD_SZ) + +#define DESC_SKCIPHER_BASE (3 * CAAM_CMD_SZ) +#define DESC_SKCIPHER_ENC_LEN (DESC_SKCIPHER_BASE + \ + 21 * CAAM_CMD_SZ) +#define DESC_SKCIPHER_DEC_LEN (DESC_SKCIPHER_BASE + \ + 16 * CAAM_CMD_SZ) + +/* Key modifier for CAAM Protected blobs */ +#define KEYMOD "SECURE_KEY" + +void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata, + unsigned int icvsize, int era); + +void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata, + unsigned int icvsize, int era); + +void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata, + struct alginfo *adata, unsigned int ivsize, + unsigned int icvsize, const bool is_rfc3686, + u32 *nonce, const u32 ctx1_iv_off, + const bool is_qi, int era); + +void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata, + struct alginfo *adata, unsigned int ivsize, + unsigned int icvsize, const bool geniv, + const bool is_rfc3686, u32 *nonce, + const u32 ctx1_iv_off, const bool is_qi, int era); + +void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata, + struct alginfo *adata, unsigned int ivsize, + unsigned int icvsize, const bool is_rfc3686, + u32 *nonce, const u32 ctx1_iv_off, + const bool is_qi, int era); + +void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, unsigned int icvsize, + const bool is_qi); + +void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, unsigned int icvsize, + const bool is_qi); + +void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, unsigned int icvsize, + const bool is_qi); + +void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, unsigned int icvsize, + const bool is_qi); + +void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, unsigned int icvsize, + const bool is_qi); + +void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, unsigned int icvsize, + const bool is_qi); + +void cnstr_shdsc_chachapoly(u32 * const desc, struct alginfo *cdata, + struct alginfo *adata, unsigned int ivsize, + unsigned int icvsize, const bool encap, + const bool is_qi); + +void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, const bool is_rfc3686, + const u32 ctx1_iv_off); + +void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, const bool is_rfc3686, + const u32 ctx1_iv_off); + +void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata); + +void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata); + +void cnstr_desc_protected_blob_decap(u32 * const desc, struct alginfo *cdata, + dma_addr_t next_desc); + +void cnstr_desc_skcipher_enc_dec(u32 * const desc, struct alginfo *cdata, + dma_addr_t src, dma_addr_t dst, unsigned int data_sz, + unsigned int in_options, unsigned int out_options, + unsigned int ivsize, const bool encrypt); + +#endif /* _CAAMALG_DESC_H_ */ diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c new file mode 100644 index 000000000..aa779caac --- /dev/null +++ b/drivers/crypto/caam/caamalg_qi.c @@ -0,0 +1,2743 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Freescale FSL CAAM support for crypto API over QI backend. + * Based on caamalg.c + * + * Copyright 2013-2016 Freescale Semiconductor, Inc. + * Copyright 2016-2019 NXP + */ + +#include "compat.h" +#include "ctrl.h" +#include "regs.h" +#include "intern.h" +#include "desc_constr.h" +#include "error.h" +#include "sg_sw_qm.h" +#include "key_gen.h" +#include "qi.h" +#include "jr.h" +#include "caamalg_desc.h" +#include +#include +#include +#include +#include +#include +#include + +/* + * crypto alg + */ +#define CAAM_CRA_PRIORITY 2000 +/* max key is sum of AES_MAX_KEY_SIZE, max split key size */ +#define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ + SHA512_DIGEST_SIZE * 2) + +#define DESC_MAX_USED_BYTES (DESC_QI_AEAD_GIVENC_LEN + \ + CAAM_MAX_KEY_SIZE) +#define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) + +struct caam_alg_entry { + int class1_alg_type; + int class2_alg_type; + bool rfc3686; + bool geniv; + bool nodkp; +}; + +struct caam_aead_alg { + struct aead_alg aead; + struct caam_alg_entry caam; + bool registered; +}; + +struct caam_skcipher_alg { + struct skcipher_alg skcipher; + struct caam_alg_entry caam; + bool registered; +}; + +/* + * per-session context + */ +struct caam_ctx { + struct device *jrdev; + u32 sh_desc_enc[DESC_MAX_USED_LEN]; + u32 sh_desc_dec[DESC_MAX_USED_LEN]; + u8 key[CAAM_MAX_KEY_SIZE]; + dma_addr_t key_dma; + enum dma_data_direction dir; + struct alginfo adata; + struct alginfo cdata; + unsigned int authsize; + struct device *qidev; + spinlock_t lock; /* Protects multiple init of driver context */ + struct caam_drv_ctx *drv_ctx[NUM_OP]; + bool xts_key_fallback; + struct crypto_skcipher *fallback; +}; + +struct caam_skcipher_req_ctx { + struct skcipher_request fallback_req; +}; + +static int aead_set_sh_desc(struct crypto_aead *aead) +{ + struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), + typeof(*alg), aead); + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + unsigned int ivsize = crypto_aead_ivsize(aead); + u32 ctx1_iv_off = 0; + u32 *nonce = NULL; + unsigned int data_len[2]; + u32 inl_mask; + const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == + OP_ALG_AAI_CTR_MOD128); + const bool is_rfc3686 = alg->caam.rfc3686; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); + + if (!ctx->cdata.keylen || !ctx->authsize) + return 0; + + /* + * AES-CTR needs to load IV in CONTEXT1 reg + * at an offset of 128bits (16bytes) + * CONTEXT1[255:128] = IV + */ + if (ctr_mode) + ctx1_iv_off = 16; + + /* + * RFC3686 specific: + * CONTEXT1[255:128] = {NONCE, IV, COUNTER} + */ + if (is_rfc3686) { + ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; + nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + + ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); + } + + /* + * In case |user key| > |derived key|, using DKP would result + * in invalid opcodes (last bytes of user key) in the resulting + * descriptor. Use DKP instead => both virtual and dma key + * addresses are needed. + */ + ctx->adata.key_virt = ctx->key; + ctx->adata.key_dma = ctx->key_dma; + + ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; + ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; + + data_len[0] = ctx->adata.keylen_pad; + data_len[1] = ctx->cdata.keylen; + + if (alg->caam.geniv) + goto skip_enc; + + /* aead_encrypt shared descriptor */ + if (desc_inline_query(DESC_QI_AEAD_ENC_LEN + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), + DESC_JOB_IO_LEN, data_len, &inl_mask, + ARRAY_SIZE(data_len)) < 0) + return -EINVAL; + + ctx->adata.key_inline = !!(inl_mask & 1); + ctx->cdata.key_inline = !!(inl_mask & 2); + + cnstr_shdsc_aead_encap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, + ivsize, ctx->authsize, is_rfc3686, nonce, + ctx1_iv_off, true, ctrlpriv->era); + +skip_enc: + /* aead_decrypt shared descriptor */ + if (desc_inline_query(DESC_QI_AEAD_DEC_LEN + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), + DESC_JOB_IO_LEN, data_len, &inl_mask, + ARRAY_SIZE(data_len)) < 0) + return -EINVAL; + + ctx->adata.key_inline = !!(inl_mask & 1); + ctx->cdata.key_inline = !!(inl_mask & 2); + + cnstr_shdsc_aead_decap(ctx->sh_desc_dec, &ctx->cdata, &ctx->adata, + ivsize, ctx->authsize, alg->caam.geniv, + is_rfc3686, nonce, ctx1_iv_off, true, + ctrlpriv->era); + + if (!alg->caam.geniv) + goto skip_givenc; + + /* aead_givencrypt shared descriptor */ + if (desc_inline_query(DESC_QI_AEAD_GIVENC_LEN + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), + DESC_JOB_IO_LEN, data_len, &inl_mask, + ARRAY_SIZE(data_len)) < 0) + return -EINVAL; + + ctx->adata.key_inline = !!(inl_mask & 1); + ctx->cdata.key_inline = !!(inl_mask & 2); + + cnstr_shdsc_aead_givencap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, + ivsize, ctx->authsize, is_rfc3686, nonce, + ctx1_iv_off, true, ctrlpriv->era); + +skip_givenc: + return 0; +} + +static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); + + ctx->authsize = authsize; + aead_set_sh_desc(authenc); + + return 0; +} + +static int aead_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *jrdev = ctx->jrdev; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); + struct crypto_authenc_keys keys; + int ret = 0; + + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) + goto badkey; + + dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n", + keys.authkeylen + keys.enckeylen, keys.enckeylen, + keys.authkeylen); + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); + + /* + * If DKP is supported, use it in the shared descriptor to generate + * the split key. + */ + if (ctrlpriv->era >= 6) { + ctx->adata.keylen = keys.authkeylen; + ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & + OP_ALG_ALGSEL_MASK); + + if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) + goto badkey; + + memcpy(ctx->key, keys.authkey, keys.authkeylen); + memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, + keys.enckeylen); + dma_sync_single_for_device(jrdev->parent, ctx->key_dma, + ctx->adata.keylen_pad + + keys.enckeylen, ctx->dir); + goto skip_split_key; + } + + ret = gen_split_key(jrdev, ctx->key, &ctx->adata, keys.authkey, + keys.authkeylen, CAAM_MAX_KEY_SIZE - + keys.enckeylen); + if (ret) + goto badkey; + + /* postpend encryption key to auth split key */ + memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); + dma_sync_single_for_device(jrdev->parent, ctx->key_dma, + ctx->adata.keylen_pad + keys.enckeylen, + ctx->dir); + + print_hex_dump_devel("ctx.key@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, + ctx->adata.keylen_pad + keys.enckeylen, 1); + +skip_split_key: + ctx->cdata.keylen = keys.enckeylen; + + ret = aead_set_sh_desc(aead); + if (ret) + goto badkey; + + /* Now update the driver contexts with the new shared descriptor */ + if (ctx->drv_ctx[ENCRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], + ctx->sh_desc_enc); + if (ret) { + dev_err(jrdev, "driver enc context update failed\n"); + goto badkey; + } + } + + if (ctx->drv_ctx[DECRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], + ctx->sh_desc_dec); + if (ret) { + dev_err(jrdev, "driver dec context update failed\n"); + goto badkey; + } + } + + memzero_explicit(&keys, sizeof(keys)); + return ret; +badkey: + memzero_explicit(&keys, sizeof(keys)); + return -EINVAL; +} + +static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct crypto_authenc_keys keys; + int err; + + err = crypto_authenc_extractkeys(&keys, key, keylen); + if (unlikely(err)) + return err; + + err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?: + aead_setkey(aead, key, keylen); + + memzero_explicit(&keys, sizeof(keys)); + return err; +} + +static int gcm_set_sh_desc(struct crypto_aead *aead) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + unsigned int ivsize = crypto_aead_ivsize(aead); + int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - + ctx->cdata.keylen; + + if (!ctx->cdata.keylen || !ctx->authsize) + return 0; + + /* + * Job Descriptor and Shared Descriptor + * must fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_QI_GCM_ENC_LEN) { + ctx->cdata.key_inline = true; + ctx->cdata.key_virt = ctx->key; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } + + cnstr_shdsc_gcm_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, + ctx->authsize, true); + + /* + * Job Descriptor and Shared Descriptor + * must fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_QI_GCM_DEC_LEN) { + ctx->cdata.key_inline = true; + ctx->cdata.key_virt = ctx->key; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } + + cnstr_shdsc_gcm_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, + ctx->authsize, true); + + return 0; +} + +static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); + int err; + + err = crypto_gcm_check_authsize(authsize); + if (err) + return err; + + ctx->authsize = authsize; + gcm_set_sh_desc(authenc); + + return 0; +} + +static int gcm_setkey(struct crypto_aead *aead, + const u8 *key, unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *jrdev = ctx->jrdev; + int ret; + + ret = aes_check_keylen(keylen); + if (ret) + return ret; + + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); + + memcpy(ctx->key, key, keylen); + dma_sync_single_for_device(jrdev->parent, ctx->key_dma, keylen, + ctx->dir); + ctx->cdata.keylen = keylen; + + ret = gcm_set_sh_desc(aead); + if (ret) + return ret; + + /* Now update the driver contexts with the new shared descriptor */ + if (ctx->drv_ctx[ENCRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], + ctx->sh_desc_enc); + if (ret) { + dev_err(jrdev, "driver enc context update failed\n"); + return ret; + } + } + + if (ctx->drv_ctx[DECRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], + ctx->sh_desc_dec); + if (ret) { + dev_err(jrdev, "driver dec context update failed\n"); + return ret; + } + } + + return 0; +} + +static int rfc4106_set_sh_desc(struct crypto_aead *aead) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + unsigned int ivsize = crypto_aead_ivsize(aead); + int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - + ctx->cdata.keylen; + + if (!ctx->cdata.keylen || !ctx->authsize) + return 0; + + ctx->cdata.key_virt = ctx->key; + + /* + * Job Descriptor and Shared Descriptor + * must fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) { + ctx->cdata.key_inline = true; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } + + cnstr_shdsc_rfc4106_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, + ctx->authsize, true); + + /* + * Job Descriptor and Shared Descriptor + * must fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) { + ctx->cdata.key_inline = true; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } + + cnstr_shdsc_rfc4106_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, + ctx->authsize, true); + + return 0; +} + +static int rfc4106_setauthsize(struct crypto_aead *authenc, + unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); + int err; + + err = crypto_rfc4106_check_authsize(authsize); + if (err) + return err; + + ctx->authsize = authsize; + rfc4106_set_sh_desc(authenc); + + return 0; +} + +static int rfc4106_setkey(struct crypto_aead *aead, + const u8 *key, unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *jrdev = ctx->jrdev; + int ret; + + ret = aes_check_keylen(keylen - 4); + if (ret) + return ret; + + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); + + memcpy(ctx->key, key, keylen); + /* + * The last four bytes of the key material are used as the salt value + * in the nonce. Update the AES key length. + */ + ctx->cdata.keylen = keylen - 4; + dma_sync_single_for_device(jrdev->parent, ctx->key_dma, + ctx->cdata.keylen, ctx->dir); + + ret = rfc4106_set_sh_desc(aead); + if (ret) + return ret; + + /* Now update the driver contexts with the new shared descriptor */ + if (ctx->drv_ctx[ENCRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], + ctx->sh_desc_enc); + if (ret) { + dev_err(jrdev, "driver enc context update failed\n"); + return ret; + } + } + + if (ctx->drv_ctx[DECRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], + ctx->sh_desc_dec); + if (ret) { + dev_err(jrdev, "driver dec context update failed\n"); + return ret; + } + } + + return 0; +} + +static int rfc4543_set_sh_desc(struct crypto_aead *aead) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + unsigned int ivsize = crypto_aead_ivsize(aead); + int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - + ctx->cdata.keylen; + + if (!ctx->cdata.keylen || !ctx->authsize) + return 0; + + ctx->cdata.key_virt = ctx->key; + + /* + * Job Descriptor and Shared Descriptor + * must fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) { + ctx->cdata.key_inline = true; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } + + cnstr_shdsc_rfc4543_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, + ctx->authsize, true); + + /* + * Job Descriptor and Shared Descriptor + * must fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) { + ctx->cdata.key_inline = true; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } + + cnstr_shdsc_rfc4543_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, + ctx->authsize, true); + + return 0; +} + +static int rfc4543_setauthsize(struct crypto_aead *authenc, + unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); + + if (authsize != 16) + return -EINVAL; + + ctx->authsize = authsize; + rfc4543_set_sh_desc(authenc); + + return 0; +} + +static int rfc4543_setkey(struct crypto_aead *aead, + const u8 *key, unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *jrdev = ctx->jrdev; + int ret; + + ret = aes_check_keylen(keylen - 4); + if (ret) + return ret; + + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); + + memcpy(ctx->key, key, keylen); + /* + * The last four bytes of the key material are used as the salt value + * in the nonce. Update the AES key length. + */ + ctx->cdata.keylen = keylen - 4; + dma_sync_single_for_device(jrdev->parent, ctx->key_dma, + ctx->cdata.keylen, ctx->dir); + + ret = rfc4543_set_sh_desc(aead); + if (ret) + return ret; + + /* Now update the driver contexts with the new shared descriptor */ + if (ctx->drv_ctx[ENCRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], + ctx->sh_desc_enc); + if (ret) { + dev_err(jrdev, "driver enc context update failed\n"); + return ret; + } + } + + if (ctx->drv_ctx[DECRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], + ctx->sh_desc_dec); + if (ret) { + dev_err(jrdev, "driver dec context update failed\n"); + return ret; + } + } + + return 0; +} + +static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, + unsigned int keylen, const u32 ctx1_iv_off) +{ + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); + struct caam_skcipher_alg *alg = + container_of(crypto_skcipher_alg(skcipher), typeof(*alg), + skcipher); + struct device *jrdev = ctx->jrdev; + unsigned int ivsize = crypto_skcipher_ivsize(skcipher); + const bool is_rfc3686 = alg->caam.rfc3686; + int ret = 0; + + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); + + ctx->cdata.keylen = keylen; + ctx->cdata.key_virt = key; + ctx->cdata.key_inline = true; + + /* skcipher encrypt, decrypt shared descriptors */ + cnstr_shdsc_skcipher_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, + is_rfc3686, ctx1_iv_off); + cnstr_shdsc_skcipher_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, + is_rfc3686, ctx1_iv_off); + + /* Now update the driver contexts with the new shared descriptor */ + if (ctx->drv_ctx[ENCRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], + ctx->sh_desc_enc); + if (ret) { + dev_err(jrdev, "driver enc context update failed\n"); + return -EINVAL; + } + } + + if (ctx->drv_ctx[DECRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], + ctx->sh_desc_dec); + if (ret) { + dev_err(jrdev, "driver dec context update failed\n"); + return -EINVAL; + } + } + + return ret; +} + +static int aes_skcipher_setkey(struct crypto_skcipher *skcipher, + const u8 *key, unsigned int keylen) +{ + int err; + + err = aes_check_keylen(keylen); + if (err) + return err; + + return skcipher_setkey(skcipher, key, keylen, 0); +} + +static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher, + const u8 *key, unsigned int keylen) +{ + u32 ctx1_iv_off; + int err; + + /* + * RFC3686 specific: + * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} + * | *key = {KEY, NONCE} + */ + ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; + keylen -= CTR_RFC3686_NONCE_SIZE; + + err = aes_check_keylen(keylen); + if (err) + return err; + + return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); +} + +static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher, + const u8 *key, unsigned int keylen) +{ + u32 ctx1_iv_off; + int err; + + /* + * AES-CTR needs to load IV in CONTEXT1 reg + * at an offset of 128bits (16bytes) + * CONTEXT1[255:128] = IV + */ + ctx1_iv_off = 16; + + err = aes_check_keylen(keylen); + if (err) + return err; + + return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); +} + +static int des3_skcipher_setkey(struct crypto_skcipher *skcipher, + const u8 *key, unsigned int keylen) +{ + return verify_skcipher_des3_key(skcipher, key) ?: + skcipher_setkey(skcipher, key, keylen, 0); +} + +static int des_skcipher_setkey(struct crypto_skcipher *skcipher, + const u8 *key, unsigned int keylen) +{ + return verify_skcipher_des_key(skcipher, key) ?: + skcipher_setkey(skcipher, key, keylen, 0); +} + +static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, + unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); + struct device *jrdev = ctx->jrdev; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); + int ret = 0; + int err; + + err = xts_verify_key(skcipher, key, keylen); + if (err) { + dev_dbg(jrdev, "key size mismatch\n"); + return err; + } + + if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256) + ctx->xts_key_fallback = true; + + if (ctrlpriv->era <= 8 || ctx->xts_key_fallback) { + err = crypto_skcipher_setkey(ctx->fallback, key, keylen); + if (err) + return err; + } + + ctx->cdata.keylen = keylen; + ctx->cdata.key_virt = key; + ctx->cdata.key_inline = true; + + /* xts skcipher encrypt, decrypt shared descriptors */ + cnstr_shdsc_xts_skcipher_encap(ctx->sh_desc_enc, &ctx->cdata); + cnstr_shdsc_xts_skcipher_decap(ctx->sh_desc_dec, &ctx->cdata); + + /* Now update the driver contexts with the new shared descriptor */ + if (ctx->drv_ctx[ENCRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], + ctx->sh_desc_enc); + if (ret) { + dev_err(jrdev, "driver enc context update failed\n"); + return -EINVAL; + } + } + + if (ctx->drv_ctx[DECRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], + ctx->sh_desc_dec); + if (ret) { + dev_err(jrdev, "driver dec context update failed\n"); + return -EINVAL; + } + } + + return ret; +} + +/* + * aead_edesc - s/w-extended aead descriptor + * @src_nents: number of segments in input scatterlist + * @dst_nents: number of segments in output scatterlist + * @iv_dma: dma address of iv for checking continuity and link table + * @qm_sg_bytes: length of dma mapped h/w link table + * @qm_sg_dma: bus physical mapped address of h/w link table + * @assoclen: associated data length, in CAAM endianness + * @assoclen_dma: bus physical mapped address of req->assoclen + * @drv_req: driver-specific request structure + * @sgt: the h/w link table, followed by IV + */ +struct aead_edesc { + int src_nents; + int dst_nents; + dma_addr_t iv_dma; + int qm_sg_bytes; + dma_addr_t qm_sg_dma; + unsigned int assoclen; + dma_addr_t assoclen_dma; + struct caam_drv_req drv_req; + struct qm_sg_entry sgt[]; +}; + +/* + * skcipher_edesc - s/w-extended skcipher descriptor + * @src_nents: number of segments in input scatterlist + * @dst_nents: number of segments in output scatterlist + * @iv_dma: dma address of iv for checking continuity and link table + * @qm_sg_bytes: length of dma mapped h/w link table + * @qm_sg_dma: bus physical mapped address of h/w link table + * @drv_req: driver-specific request structure + * @sgt: the h/w link table, followed by IV + */ +struct skcipher_edesc { + int src_nents; + int dst_nents; + dma_addr_t iv_dma; + int qm_sg_bytes; + dma_addr_t qm_sg_dma; + struct caam_drv_req drv_req; + struct qm_sg_entry sgt[]; +}; + +static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx, + enum optype type) +{ + /* + * This function is called on the fast path with values of 'type' + * known at compile time. Invalid arguments are not expected and + * thus no checks are made. + */ + struct caam_drv_ctx *drv_ctx = ctx->drv_ctx[type]; + u32 *desc; + + if (unlikely(!drv_ctx)) { + spin_lock(&ctx->lock); + + /* Read again to check if some other core init drv_ctx */ + drv_ctx = ctx->drv_ctx[type]; + if (!drv_ctx) { + int cpu; + + if (type == ENCRYPT) + desc = ctx->sh_desc_enc; + else /* (type == DECRYPT) */ + desc = ctx->sh_desc_dec; + + cpu = smp_processor_id(); + drv_ctx = caam_drv_ctx_init(ctx->qidev, &cpu, desc); + if (!IS_ERR(drv_ctx)) + drv_ctx->op_type = type; + + ctx->drv_ctx[type] = drv_ctx; + } + + spin_unlock(&ctx->lock); + } + + return drv_ctx; +} + +static void caam_unmap(struct device *dev, struct scatterlist *src, + struct scatterlist *dst, int src_nents, + int dst_nents, dma_addr_t iv_dma, int ivsize, + enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma, + int qm_sg_bytes) +{ + if (dst != src) { + if (src_nents) + dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); + if (dst_nents) + dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); + } else { + dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); + } + + if (iv_dma) + dma_unmap_single(dev, iv_dma, ivsize, iv_dir); + if (qm_sg_bytes) + dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); +} + +static void aead_unmap(struct device *dev, + struct aead_edesc *edesc, + struct aead_request *req) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + int ivsize = crypto_aead_ivsize(aead); + + caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, + edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma, + edesc->qm_sg_bytes); + dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); +} + +static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, + struct skcipher_request *req) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + int ivsize = crypto_skcipher_ivsize(skcipher); + + caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, + edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma, + edesc->qm_sg_bytes); +} + +static void aead_done(struct caam_drv_req *drv_req, u32 status) +{ + struct device *qidev; + struct aead_edesc *edesc; + struct aead_request *aead_req = drv_req->app_ctx; + struct crypto_aead *aead = crypto_aead_reqtfm(aead_req); + struct caam_ctx *caam_ctx = crypto_aead_ctx_dma(aead); + int ecode = 0; + + qidev = caam_ctx->qidev; + + if (unlikely(status)) + ecode = caam_jr_strstatus(qidev, status); + + edesc = container_of(drv_req, typeof(*edesc), drv_req); + aead_unmap(qidev, edesc, aead_req); + + aead_request_complete(aead_req, ecode); + qi_cache_free(edesc); +} + +/* + * allocate and map the aead extended descriptor + */ +static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, + bool encrypt) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), + typeof(*alg), aead); + struct device *qidev = ctx->qidev; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; + int src_len, dst_len = 0; + struct aead_edesc *edesc; + dma_addr_t qm_sg_dma, iv_dma = 0; + int ivsize = 0; + unsigned int authsize = ctx->authsize; + int qm_sg_index = 0, qm_sg_ents = 0, qm_sg_bytes; + int in_len, out_len; + struct qm_sg_entry *sg_table, *fd_sgt; + struct caam_drv_ctx *drv_ctx; + + drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT); + if (IS_ERR(drv_ctx)) + return ERR_CAST(drv_ctx); + + /* allocate space for base edesc and hw desc commands, link tables */ + edesc = qi_cache_alloc(flags); + if (unlikely(!edesc)) { + dev_err(qidev, "could not allocate extended descriptor\n"); + return ERR_PTR(-ENOMEM); + } + + if (likely(req->src == req->dst)) { + src_len = req->assoclen + req->cryptlen + + (encrypt ? authsize : 0); + + src_nents = sg_nents_for_len(req->src, src_len); + if (unlikely(src_nents < 0)) { + dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", + src_len); + qi_cache_free(edesc); + return ERR_PTR(src_nents); + } + + mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, + DMA_BIDIRECTIONAL); + if (unlikely(!mapped_src_nents)) { + dev_err(qidev, "unable to map source\n"); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + } else { + src_len = req->assoclen + req->cryptlen; + dst_len = src_len + (encrypt ? authsize : (-authsize)); + + src_nents = sg_nents_for_len(req->src, src_len); + if (unlikely(src_nents < 0)) { + dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", + src_len); + qi_cache_free(edesc); + return ERR_PTR(src_nents); + } + + dst_nents = sg_nents_for_len(req->dst, dst_len); + if (unlikely(dst_nents < 0)) { + dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", + dst_len); + qi_cache_free(edesc); + return ERR_PTR(dst_nents); + } + + if (src_nents) { + mapped_src_nents = dma_map_sg(qidev, req->src, + src_nents, DMA_TO_DEVICE); + if (unlikely(!mapped_src_nents)) { + dev_err(qidev, "unable to map source\n"); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + } else { + mapped_src_nents = 0; + } + + if (dst_nents) { + mapped_dst_nents = dma_map_sg(qidev, req->dst, + dst_nents, + DMA_FROM_DEVICE); + if (unlikely(!mapped_dst_nents)) { + dev_err(qidev, "unable to map destination\n"); + dma_unmap_sg(qidev, req->src, src_nents, + DMA_TO_DEVICE); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + } else { + mapped_dst_nents = 0; + } + } + + if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) + ivsize = crypto_aead_ivsize(aead); + + /* + * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. + * Input is not contiguous. + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond + * the end of the table by allocating more S/G entries. Logic: + * if (src != dst && output S/G) + * pad output S/G, if needed + * else if (src == dst && S/G) + * overlapping S/Gs; pad one of them + * else if (input S/G) ... + * pad input S/G, if needed + */ + qm_sg_ents = 1 + !!ivsize + mapped_src_nents; + if (mapped_dst_nents > 1) + qm_sg_ents += pad_sg_nents(mapped_dst_nents); + else if ((req->src == req->dst) && (mapped_src_nents > 1)) + qm_sg_ents = max(pad_sg_nents(qm_sg_ents), + 1 + !!ivsize + pad_sg_nents(mapped_src_nents)); + else + qm_sg_ents = pad_sg_nents(qm_sg_ents); + + sg_table = &edesc->sgt[0]; + qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); + if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize > + CAAM_QI_MEMCACHE_SIZE)) { + dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n", + qm_sg_ents, ivsize); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, + 0, DMA_NONE, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + if (ivsize) { + u8 *iv = (u8 *)(sg_table + qm_sg_ents); + + /* Make sure IV is located in a DMAable area */ + memcpy(iv, req->iv, ivsize); + + iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE); + if (dma_mapping_error(qidev, iv_dma)) { + dev_err(qidev, "unable to map IV\n"); + caam_unmap(qidev, req->src, req->dst, src_nents, + dst_nents, 0, 0, DMA_NONE, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + } + + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + edesc->iv_dma = iv_dma; + edesc->drv_req.app_ctx = req; + edesc->drv_req.cbk = aead_done; + edesc->drv_req.drv_ctx = drv_ctx; + + edesc->assoclen = cpu_to_caam32(req->assoclen); + edesc->assoclen_dma = dma_map_single(qidev, &edesc->assoclen, 4, + DMA_TO_DEVICE); + if (dma_mapping_error(qidev, edesc->assoclen_dma)) { + dev_err(qidev, "unable to map assoclen\n"); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0); + qm_sg_index++; + if (ivsize) { + dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); + qm_sg_index++; + } + sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0); + qm_sg_index += mapped_src_nents; + + if (mapped_dst_nents > 1) + sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0); + + qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(qidev, qm_sg_dma)) { + dev_err(qidev, "unable to map S/G table\n"); + dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + edesc->qm_sg_dma = qm_sg_dma; + edesc->qm_sg_bytes = qm_sg_bytes; + + out_len = req->assoclen + req->cryptlen + + (encrypt ? ctx->authsize : (-ctx->authsize)); + in_len = 4 + ivsize + req->assoclen + req->cryptlen; + + fd_sgt = &edesc->drv_req.fd_sgt[0]; + dma_to_qm_sg_one_last_ext(&fd_sgt[1], qm_sg_dma, in_len, 0); + + if (req->dst == req->src) { + if (mapped_src_nents == 1) + dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->src), + out_len, 0); + else + dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + + (1 + !!ivsize) * sizeof(*sg_table), + out_len, 0); + } else if (mapped_dst_nents <= 1) { + dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), out_len, + 0); + } else { + dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + sizeof(*sg_table) * + qm_sg_index, out_len, 0); + } + + return edesc; +} + +static inline int aead_crypt(struct aead_request *req, bool encrypt) +{ + struct aead_edesc *edesc; + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + int ret; + + if (unlikely(caam_congested)) + return -EAGAIN; + + /* allocate extended descriptor */ + edesc = aead_edesc_alloc(req, encrypt); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + /* Create and submit job descriptor */ + ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); + if (!ret) { + ret = -EINPROGRESS; + } else { + aead_unmap(ctx->qidev, edesc, req); + qi_cache_free(edesc); + } + + return ret; +} + +static int aead_encrypt(struct aead_request *req) +{ + return aead_crypt(req, true); +} + +static int aead_decrypt(struct aead_request *req) +{ + return aead_crypt(req, false); +} + +static int ipsec_gcm_encrypt(struct aead_request *req) +{ + return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_crypt(req, + true); +} + +static int ipsec_gcm_decrypt(struct aead_request *req) +{ + return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_crypt(req, + false); +} + +static inline u8 *skcipher_edesc_iv(struct skcipher_edesc *edesc) +{ + return PTR_ALIGN((u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, + dma_get_cache_alignment()); +} + +static void skcipher_done(struct caam_drv_req *drv_req, u32 status) +{ + struct skcipher_edesc *edesc; + struct skcipher_request *req = drv_req->app_ctx; + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + struct caam_ctx *caam_ctx = crypto_skcipher_ctx_dma(skcipher); + struct device *qidev = caam_ctx->qidev; + int ivsize = crypto_skcipher_ivsize(skcipher); + int ecode = 0; + + dev_dbg(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status); + + edesc = container_of(drv_req, typeof(*edesc), drv_req); + + if (status) + ecode = caam_jr_strstatus(qidev, status); + + print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->iv, + edesc->src_nents > 1 ? 100 : ivsize, 1); + caam_dump_sg("dst @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->dst, + edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); + + skcipher_unmap(qidev, edesc, req); + + /* + * The crypto API expects us to set the IV (req->iv) to the last + * ciphertext block (CBC mode) or last counter (CTR mode). + * This is used e.g. by the CTS mode. + */ + if (!ecode) + memcpy(req->iv, skcipher_edesc_iv(edesc), ivsize); + + qi_cache_free(edesc); + skcipher_request_complete(req, ecode); +} + +static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, + bool encrypt) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); + struct device *qidev = ctx->qidev; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; + struct skcipher_edesc *edesc; + dma_addr_t iv_dma; + u8 *iv; + int ivsize = crypto_skcipher_ivsize(skcipher); + int dst_sg_idx, qm_sg_ents, qm_sg_bytes; + struct qm_sg_entry *sg_table, *fd_sgt; + struct caam_drv_ctx *drv_ctx; + unsigned int len; + + drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT); + if (IS_ERR(drv_ctx)) + return ERR_CAST(drv_ctx); + + src_nents = sg_nents_for_len(req->src, req->cryptlen); + if (unlikely(src_nents < 0)) { + dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", + req->cryptlen); + return ERR_PTR(src_nents); + } + + if (unlikely(req->src != req->dst)) { + dst_nents = sg_nents_for_len(req->dst, req->cryptlen); + if (unlikely(dst_nents < 0)) { + dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", + req->cryptlen); + return ERR_PTR(dst_nents); + } + + mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, + DMA_TO_DEVICE); + if (unlikely(!mapped_src_nents)) { + dev_err(qidev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + + mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents, + DMA_FROM_DEVICE); + if (unlikely(!mapped_dst_nents)) { + dev_err(qidev, "unable to map destination\n"); + dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); + return ERR_PTR(-ENOMEM); + } + } else { + mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, + DMA_BIDIRECTIONAL); + if (unlikely(!mapped_src_nents)) { + dev_err(qidev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + } + + qm_sg_ents = 1 + mapped_src_nents; + dst_sg_idx = qm_sg_ents; + + /* + * Input, output HW S/G tables: [IV, src][dst, IV] + * IV entries point to the same buffer + * If src == dst, S/G entries are reused (S/G tables overlap) + * + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond + * the end of the table by allocating more S/G entries. + */ + if (req->src != req->dst) + qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1); + else + qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents); + + qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry); + + len = offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes; + len = ALIGN(len, dma_get_cache_alignment()); + len += ivsize; + + if (unlikely(len > CAAM_QI_MEMCACHE_SIZE)) { + dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n", + qm_sg_ents, ivsize); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, + 0, DMA_NONE, 0, 0); + return ERR_PTR(-ENOMEM); + } + + /* allocate space for base edesc, link tables and IV */ + edesc = qi_cache_alloc(flags); + if (unlikely(!edesc)) { + dev_err(qidev, "could not allocate extended descriptor\n"); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, + 0, DMA_NONE, 0, 0); + return ERR_PTR(-ENOMEM); + } + + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + edesc->qm_sg_bytes = qm_sg_bytes; + edesc->drv_req.app_ctx = req; + edesc->drv_req.cbk = skcipher_done; + edesc->drv_req.drv_ctx = drv_ctx; + + /* Make sure IV is located in a DMAable area */ + sg_table = &edesc->sgt[0]; + iv = skcipher_edesc_iv(edesc); + memcpy(iv, req->iv, ivsize); + + iv_dma = dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL); + if (dma_mapping_error(qidev, iv_dma)) { + dev_err(qidev, "unable to map IV\n"); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, + 0, DMA_NONE, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + edesc->iv_dma = iv_dma; + + dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); + sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0); + + if (req->src != req->dst) + sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0); + + dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma, + ivsize, 0); + + edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes, + DMA_TO_DEVICE); + if (dma_mapping_error(qidev, edesc->qm_sg_dma)) { + dev_err(qidev, "unable to map S/G table\n"); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + fd_sgt = &edesc->drv_req.fd_sgt[0]; + + dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma, + ivsize + req->cryptlen, 0); + + if (req->src == req->dst) + dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + + sizeof(*sg_table), req->cryptlen + ivsize, + 0); + else + dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx * + sizeof(*sg_table), req->cryptlen + ivsize, + 0); + + return edesc; +} + +static inline bool xts_skcipher_ivsize(struct skcipher_request *req) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + unsigned int ivsize = crypto_skcipher_ivsize(skcipher); + + return !!get_unaligned((u64 *)(req->iv + (ivsize / 2))); +} + +static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) +{ + struct skcipher_edesc *edesc; + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); + struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); + int ret; + + /* + * XTS is expected to return an error even for input length = 0 + * Note that the case input length < block size will be caught during + * HW offloading and return an error. + */ + if (!req->cryptlen && !ctx->fallback) + return 0; + + if (ctx->fallback && ((ctrlpriv->era <= 8 && xts_skcipher_ivsize(req)) || + ctx->xts_key_fallback)) { + struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); + + skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback); + skcipher_request_set_callback(&rctx->fallback_req, + req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(&rctx->fallback_req, req->src, + req->dst, req->cryptlen, req->iv); + + return encrypt ? crypto_skcipher_encrypt(&rctx->fallback_req) : + crypto_skcipher_decrypt(&rctx->fallback_req); + } + + if (unlikely(caam_congested)) + return -EAGAIN; + + /* allocate extended descriptor */ + edesc = skcipher_edesc_alloc(req, encrypt); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); + if (!ret) { + ret = -EINPROGRESS; + } else { + skcipher_unmap(ctx->qidev, edesc, req); + qi_cache_free(edesc); + } + + return ret; +} + +static int skcipher_encrypt(struct skcipher_request *req) +{ + return skcipher_crypt(req, true); +} + +static int skcipher_decrypt(struct skcipher_request *req) +{ + return skcipher_crypt(req, false); +} + +static struct caam_skcipher_alg driver_algs[] = { + { + .skcipher = { + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aes_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + }, + { + .skcipher = { + .base = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "cbc-3des-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + }, + .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + }, + { + .skcipher = { + .base = { + .cra_name = "cbc(des)", + .cra_driver_name = "cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = des_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + }, + .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + }, + { + .skcipher = { + .base = { + .cra_name = "ctr(aes)", + .cra_driver_name = "ctr-aes-caam-qi", + .cra_blocksize = 1, + }, + .setkey = ctr_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .chunksize = AES_BLOCK_SIZE, + }, + .caam.class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + }, + { + .skcipher = { + .base = { + .cra_name = "rfc3686(ctr(aes))", + .cra_driver_name = "rfc3686-ctr-aes-caam-qi", + .cra_blocksize = 1, + }, + .setkey = rfc3686_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + .chunksize = AES_BLOCK_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .rfc3686 = true, + }, + }, + { + .skcipher = { + .base = { + .cra_name = "xts(aes)", + .cra_driver_name = "xts-aes-caam-qi", + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = xts_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, + }, +}; + +static struct caam_aead_alg driver_aeads[] = { + { + .aead = { + .base = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "rfc4106-gcm-aes-caam-qi", + .cra_blocksize = 1, + }, + .setkey = rfc4106_setkey, + .setauthsize = rfc4106_setauthsize, + .encrypt = ipsec_gcm_encrypt, + .decrypt = ipsec_gcm_decrypt, + .ivsize = 8, + .maxauthsize = AES_BLOCK_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + .nodkp = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "rfc4543(gcm(aes))", + .cra_driver_name = "rfc4543-gcm-aes-caam-qi", + .cra_blocksize = 1, + }, + .setkey = rfc4543_setkey, + .setauthsize = rfc4543_setauthsize, + .encrypt = ipsec_gcm_encrypt, + .decrypt = ipsec_gcm_decrypt, + .ivsize = 8, + .maxauthsize = AES_BLOCK_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + .nodkp = true, + }, + }, + /* Galois Counter Mode */ + { + .aead = { + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "gcm-aes-caam-qi", + .cra_blocksize = 1, + }, + .setkey = gcm_setkey, + .setauthsize = gcm_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = 12, + .maxauthsize = AES_BLOCK_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + .nodkp = true, + } + }, + /* single-pass ipsec_esp descriptor */ + { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(aes))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(md5)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-hmac-md5-" + "cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha1)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha1-cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha224)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha224-cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha256)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha256-cbc-aes-" + "caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha384)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha384-cbc-aes-" + "caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha512)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha512-cbc-aes-" + "caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(md5)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-hmac-md5-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha1)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha1-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha224)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha224-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha256)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha256-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha384)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha384-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha512)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha512-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(md5)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-hmac-md5-" + "cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha1)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha1-cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(des))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha224)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha224-cbc-des-" + "caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(des))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha256)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha256-cbc-des-" + "caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(des))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha384)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha384-cbc-des-" + "caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(des))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha512)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha512-cbc-des-" + "caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, +}; + +static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, + bool uses_dkp) +{ + struct caam_drv_private *priv; + struct device *dev; + + /* + * distribute tfms across job rings to ensure in-order + * crypto request processing per tfm + */ + ctx->jrdev = caam_jr_alloc(); + if (IS_ERR(ctx->jrdev)) { + pr_err("Job Ring Device allocation for transform failed\n"); + return PTR_ERR(ctx->jrdev); + } + + dev = ctx->jrdev->parent; + priv = dev_get_drvdata(dev); + if (priv->era >= 6 && uses_dkp) + ctx->dir = DMA_BIDIRECTIONAL; + else + ctx->dir = DMA_TO_DEVICE; + + ctx->key_dma = dma_map_single(dev, ctx->key, sizeof(ctx->key), + ctx->dir); + if (dma_mapping_error(dev, ctx->key_dma)) { + dev_err(dev, "unable to map key\n"); + caam_jr_free(ctx->jrdev); + return -ENOMEM; + } + + /* copy descriptor header template value */ + ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; + ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; + + ctx->qidev = dev; + + spin_lock_init(&ctx->lock); + ctx->drv_ctx[ENCRYPT] = NULL; + ctx->drv_ctx[DECRYPT] = NULL; + + return 0; +} + +static int caam_cra_init(struct crypto_skcipher *tfm) +{ + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct caam_skcipher_alg *caam_alg = + container_of(alg, typeof(*caam_alg), skcipher); + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm); + u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; + int ret = 0; + + if (alg_aai == OP_ALG_AAI_XTS) { + const char *tfm_name = crypto_tfm_alg_name(&tfm->base); + struct crypto_skcipher *fallback; + + fallback = crypto_alloc_skcipher(tfm_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(fallback)) { + pr_err("Failed to allocate %s fallback: %ld\n", + tfm_name, PTR_ERR(fallback)); + return PTR_ERR(fallback); + } + + ctx->fallback = fallback; + crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx) + + crypto_skcipher_reqsize(fallback)); + } + + ret = caam_init_common(ctx, &caam_alg->caam, false); + if (ret && ctx->fallback) + crypto_free_skcipher(ctx->fallback); + + return ret; +} + +static int caam_aead_init(struct crypto_aead *tfm) +{ + struct aead_alg *alg = crypto_aead_alg(tfm); + struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg), + aead); + struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm); + + return caam_init_common(ctx, &caam_alg->caam, !caam_alg->caam.nodkp); +} + +static void caam_exit_common(struct caam_ctx *ctx) +{ + caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]); + caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]); + + dma_unmap_single(ctx->jrdev->parent, ctx->key_dma, sizeof(ctx->key), + ctx->dir); + + caam_jr_free(ctx->jrdev); +} + +static void caam_cra_exit(struct crypto_skcipher *tfm) +{ + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm); + + if (ctx->fallback) + crypto_free_skcipher(ctx->fallback); + caam_exit_common(ctx); +} + +static void caam_aead_exit(struct crypto_aead *tfm) +{ + caam_exit_common(crypto_aead_ctx_dma(tfm)); +} + +void caam_qi_algapi_exit(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { + struct caam_aead_alg *t_alg = driver_aeads + i; + + if (t_alg->registered) + crypto_unregister_aead(&t_alg->aead); + } + + for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { + struct caam_skcipher_alg *t_alg = driver_algs + i; + + if (t_alg->registered) + crypto_unregister_skcipher(&t_alg->skcipher); + } +} + +static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) +{ + struct skcipher_alg *alg = &t_alg->skcipher; + + alg->base.cra_module = THIS_MODULE; + alg->base.cra_priority = CAAM_CRA_PRIORITY; + alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding(); + alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY); + + alg->init = caam_cra_init; + alg->exit = caam_cra_exit; +} + +static void caam_aead_alg_init(struct caam_aead_alg *t_alg) +{ + struct aead_alg *alg = &t_alg->aead; + + alg->base.cra_module = THIS_MODULE; + alg->base.cra_priority = CAAM_CRA_PRIORITY; + alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding(); + alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY; + + alg->init = caam_aead_init; + alg->exit = caam_aead_exit; +} + +int caam_qi_algapi_init(struct device *ctrldev) +{ + struct caam_drv_private *priv = dev_get_drvdata(ctrldev); + int i = 0, err = 0; + u32 aes_vid, aes_inst, des_inst, md_vid, md_inst; + unsigned int md_limit = SHA512_DIGEST_SIZE; + bool registered = false; + + /* Make sure this runs only on (DPAA 1.x) QI */ + if (!priv->qi_present || caam_dpaa2) + return 0; + + /* + * Register crypto algorithms the device supports. + * First, detect presence and attributes of DES, AES, and MD blocks. + */ + if (priv->era < 10) { + u32 cha_vid, cha_inst; + + cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); + aes_vid = cha_vid & CHA_ID_LS_AES_MASK; + md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; + + cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); + des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> + CHA_ID_LS_DES_SHIFT; + aes_inst = cha_inst & CHA_ID_LS_AES_MASK; + md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; + } else { + u32 aesa, mdha; + + aesa = rd_reg32(&priv->ctrl->vreg.aesa); + mdha = rd_reg32(&priv->ctrl->vreg.mdha); + + aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT; + md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT; + + des_inst = rd_reg32(&priv->ctrl->vreg.desa) & CHA_VER_NUM_MASK; + aes_inst = aesa & CHA_VER_NUM_MASK; + md_inst = mdha & CHA_VER_NUM_MASK; + } + + /* If MD is present, limit digest size based on LP256 */ + if (md_inst && md_vid == CHA_VER_VID_MD_LP256) + md_limit = SHA256_DIGEST_SIZE; + + for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { + struct caam_skcipher_alg *t_alg = driver_algs + i; + u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK; + + /* Skip DES algorithms if not supported by device */ + if (!des_inst && + ((alg_sel == OP_ALG_ALGSEL_3DES) || + (alg_sel == OP_ALG_ALGSEL_DES))) + continue; + + /* Skip AES algorithms if not supported by device */ + if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) + continue; + + caam_skcipher_alg_init(t_alg); + + err = crypto_register_skcipher(&t_alg->skcipher); + if (err) { + dev_warn(ctrldev, "%s alg registration failed\n", + t_alg->skcipher.base.cra_driver_name); + continue; + } + + t_alg->registered = true; + registered = true; + } + + for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { + struct caam_aead_alg *t_alg = driver_aeads + i; + u32 c1_alg_sel = t_alg->caam.class1_alg_type & + OP_ALG_ALGSEL_MASK; + u32 c2_alg_sel = t_alg->caam.class2_alg_type & + OP_ALG_ALGSEL_MASK; + u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; + + /* Skip DES algorithms if not supported by device */ + if (!des_inst && + ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || + (c1_alg_sel == OP_ALG_ALGSEL_DES))) + continue; + + /* Skip AES algorithms if not supported by device */ + if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES)) + continue; + + /* + * Check support for AES algorithms not available + * on LP devices. + */ + if (aes_vid == CHA_VER_VID_AES_LP && alg_aai == OP_ALG_AAI_GCM) + continue; + + /* + * Skip algorithms requiring message digests + * if MD or MD size is not supported by device. + */ + if (c2_alg_sel && + (!md_inst || (t_alg->aead.maxauthsize > md_limit))) + continue; + + caam_aead_alg_init(t_alg); + + err = crypto_register_aead(&t_alg->aead); + if (err) { + pr_warn("%s alg registration failed\n", + t_alg->aead.base.cra_driver_name); + continue; + } + + t_alg->registered = true; + registered = true; + } + + if (registered) + dev_info(ctrldev, "algorithms registered in /proc/crypto\n"); + + return err; +} diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c new file mode 100644 index 000000000..6b47bcc16 --- /dev/null +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -0,0 +1,5565 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright 2015-2016 Freescale Semiconductor Inc. + * Copyright 2017-2019 NXP + */ + +#include "compat.h" +#include "regs.h" +#include "caamalg_qi2.h" +#include "dpseci_cmd.h" +#include "desc_constr.h" +#include "error.h" +#include "sg_sw_sec4.h" +#include "sg_sw_qm2.h" +#include "key_gen.h" +#include "caamalg_desc.h" +#include "caamhash_desc.h" +#include "dpseci-debugfs.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CAAM_CRA_PRIORITY 2000 + +/* max key is sum of AES_MAX_KEY_SIZE, max split key size */ +#define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \ + SHA512_DIGEST_SIZE * 2) + +/* + * This is a cache of buffers, from which the users of CAAM QI driver + * can allocate short buffers. It's speedier than doing kmalloc on the hotpath. + * NOTE: A more elegant solution would be to have some headroom in the frames + * being processed. This can be added by the dpaa2-eth driver. This would + * pose a problem for userspace application processing which cannot + * know of this limitation. So for now, this will work. + * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here + */ +static struct kmem_cache *qi_cache; + +struct caam_alg_entry { + struct device *dev; + int class1_alg_type; + int class2_alg_type; + bool rfc3686; + bool geniv; + bool nodkp; +}; + +struct caam_aead_alg { + struct aead_alg aead; + struct caam_alg_entry caam; + bool registered; +}; + +struct caam_skcipher_alg { + struct skcipher_alg skcipher; + struct caam_alg_entry caam; + bool registered; +}; + +/** + * struct caam_ctx - per-session context + * @flc: Flow Contexts array + * @key: [authentication key], encryption key + * @flc_dma: I/O virtual addresses of the Flow Contexts + * @key_dma: I/O virtual address of the key + * @dir: DMA direction for mapping key and Flow Contexts + * @dev: dpseci device + * @adata: authentication algorithm details + * @cdata: encryption algorithm details + * @authsize: authentication tag (a.k.a. ICV / MAC) size + * @xts_key_fallback: true if fallback tfm needs to be used due + * to unsupported xts key lengths + * @fallback: xts fallback tfm + */ +struct caam_ctx { + struct caam_flc flc[NUM_OP]; + u8 key[CAAM_MAX_KEY_SIZE]; + dma_addr_t flc_dma[NUM_OP]; + dma_addr_t key_dma; + enum dma_data_direction dir; + struct device *dev; + struct alginfo adata; + struct alginfo cdata; + unsigned int authsize; + bool xts_key_fallback; + struct crypto_skcipher *fallback; +}; + +static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv, + dma_addr_t iova_addr) +{ + phys_addr_t phys_addr; + + phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) : + iova_addr; + + return phys_to_virt(phys_addr); +} + +/* + * qi_cache_zalloc - Allocate buffers from CAAM-QI cache + * + * Allocate data on the hotpath. Instead of using kzalloc, one can use the + * services of the CAAM QI memory cache (backed by kmem_cache). The buffers + * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for + * hosting 16 SG entries. + * + * @flags - flags that would be used for the equivalent kmalloc(..) call + * + * Returns a pointer to a retrieved buffer on success or NULL on failure. + */ +static inline void *qi_cache_zalloc(gfp_t flags) +{ + return kmem_cache_zalloc(qi_cache, flags); +} + +/* + * qi_cache_free - Frees buffers allocated from CAAM-QI cache + * + * @obj - buffer previously allocated by qi_cache_zalloc + * + * No checking is being done, the call is a passthrough call to + * kmem_cache_free(...) + */ +static inline void qi_cache_free(void *obj) +{ + kmem_cache_free(qi_cache, obj); +} + +static struct caam_request *to_caam_req(struct crypto_async_request *areq) +{ + switch (crypto_tfm_alg_type(areq->tfm)) { + case CRYPTO_ALG_TYPE_SKCIPHER: + return skcipher_request_ctx_dma(skcipher_request_cast(areq)); + case CRYPTO_ALG_TYPE_AEAD: + return aead_request_ctx_dma( + container_of(areq, struct aead_request, base)); + case CRYPTO_ALG_TYPE_AHASH: + return ahash_request_ctx_dma(ahash_request_cast(areq)); + default: + return ERR_PTR(-EINVAL); + } +} + +static void caam_unmap(struct device *dev, struct scatterlist *src, + struct scatterlist *dst, int src_nents, + int dst_nents, dma_addr_t iv_dma, int ivsize, + enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma, + int qm_sg_bytes) +{ + if (dst != src) { + if (src_nents) + dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); + if (dst_nents) + dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); + } else { + dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); + } + + if (iv_dma) + dma_unmap_single(dev, iv_dma, ivsize, iv_dir); + + if (qm_sg_bytes) + dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); +} + +static int aead_set_sh_desc(struct crypto_aead *aead) +{ + struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), + typeof(*alg), aead); + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + unsigned int ivsize = crypto_aead_ivsize(aead); + struct device *dev = ctx->dev; + struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); + struct caam_flc *flc; + u32 *desc; + u32 ctx1_iv_off = 0; + u32 *nonce = NULL; + unsigned int data_len[2]; + u32 inl_mask; + const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == + OP_ALG_AAI_CTR_MOD128); + const bool is_rfc3686 = alg->caam.rfc3686; + + if (!ctx->cdata.keylen || !ctx->authsize) + return 0; + + /* + * AES-CTR needs to load IV in CONTEXT1 reg + * at an offset of 128bits (16bytes) + * CONTEXT1[255:128] = IV + */ + if (ctr_mode) + ctx1_iv_off = 16; + + /* + * RFC3686 specific: + * CONTEXT1[255:128] = {NONCE, IV, COUNTER} + */ + if (is_rfc3686) { + ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; + nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + + ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); + } + + /* + * In case |user key| > |derived key|, using DKP would result + * in invalid opcodes (last bytes of user key) in the resulting + * descriptor. Use DKP instead => both virtual and dma key + * addresses are needed. + */ + ctx->adata.key_virt = ctx->key; + ctx->adata.key_dma = ctx->key_dma; + + ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; + ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; + + data_len[0] = ctx->adata.keylen_pad; + data_len[1] = ctx->cdata.keylen; + + /* aead_encrypt shared descriptor */ + if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN : + DESC_QI_AEAD_ENC_LEN) + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), + DESC_JOB_IO_LEN, data_len, &inl_mask, + ARRAY_SIZE(data_len)) < 0) + return -EINVAL; + + ctx->adata.key_inline = !!(inl_mask & 1); + ctx->cdata.key_inline = !!(inl_mask & 2); + + flc = &ctx->flc[ENCRYPT]; + desc = flc->sh_desc; + + if (alg->caam.geniv) + cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, + ivsize, ctx->authsize, is_rfc3686, + nonce, ctx1_iv_off, true, + priv->sec_attr.era); + else + cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, + ivsize, ctx->authsize, is_rfc3686, nonce, + ctx1_iv_off, true, priv->sec_attr.era); + + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], + sizeof(flc->flc) + desc_bytes(desc), + ctx->dir); + + /* aead_decrypt shared descriptor */ + if (desc_inline_query(DESC_QI_AEAD_DEC_LEN + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), + DESC_JOB_IO_LEN, data_len, &inl_mask, + ARRAY_SIZE(data_len)) < 0) + return -EINVAL; + + ctx->adata.key_inline = !!(inl_mask & 1); + ctx->cdata.key_inline = !!(inl_mask & 2); + + flc = &ctx->flc[DECRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, + ivsize, ctx->authsize, alg->caam.geniv, + is_rfc3686, nonce, ctx1_iv_off, true, + priv->sec_attr.era); + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], + sizeof(flc->flc) + desc_bytes(desc), + ctx->dir); + + return 0; +} + +static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); + + ctx->authsize = authsize; + aead_set_sh_desc(authenc); + + return 0; +} + +static int aead_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *dev = ctx->dev; + struct crypto_authenc_keys keys; + + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) + goto badkey; + + dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n", + keys.authkeylen + keys.enckeylen, keys.enckeylen, + keys.authkeylen); + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); + + ctx->adata.keylen = keys.authkeylen; + ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & + OP_ALG_ALGSEL_MASK); + + if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) + goto badkey; + + memcpy(ctx->key, keys.authkey, keys.authkeylen); + memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); + dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad + + keys.enckeylen, ctx->dir); + print_hex_dump_devel("ctx.key@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, + ctx->adata.keylen_pad + keys.enckeylen, 1); + + ctx->cdata.keylen = keys.enckeylen; + + memzero_explicit(&keys, sizeof(keys)); + return aead_set_sh_desc(aead); +badkey: + memzero_explicit(&keys, sizeof(keys)); + return -EINVAL; +} + +static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct crypto_authenc_keys keys; + int err; + + err = crypto_authenc_extractkeys(&keys, key, keylen); + if (unlikely(err)) + goto out; + + err = -EINVAL; + if (keys.enckeylen != DES3_EDE_KEY_SIZE) + goto out; + + err = crypto_des3_ede_verify_key(crypto_aead_tfm(aead), keys.enckey) ?: + aead_setkey(aead, key, keylen); + +out: + memzero_explicit(&keys, sizeof(keys)); + return err; +} + +static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, + bool encrypt) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_request *req_ctx = aead_request_ctx_dma(req); + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), + typeof(*alg), aead); + struct device *dev = ctx->dev; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; + int src_len, dst_len = 0; + struct aead_edesc *edesc; + dma_addr_t qm_sg_dma, iv_dma = 0; + int ivsize = 0; + unsigned int authsize = ctx->authsize; + int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes; + int in_len, out_len; + struct dpaa2_sg_entry *sg_table; + + /* allocate space for base edesc, link tables and IV */ + edesc = qi_cache_zalloc(flags); + if (unlikely(!edesc)) { + dev_err(dev, "could not allocate extended descriptor\n"); + return ERR_PTR(-ENOMEM); + } + + if (unlikely(req->dst != req->src)) { + src_len = req->assoclen + req->cryptlen; + dst_len = src_len + (encrypt ? authsize : (-authsize)); + + src_nents = sg_nents_for_len(req->src, src_len); + if (unlikely(src_nents < 0)) { + dev_err(dev, "Insufficient bytes (%d) in src S/G\n", + src_len); + qi_cache_free(edesc); + return ERR_PTR(src_nents); + } + + dst_nents = sg_nents_for_len(req->dst, dst_len); + if (unlikely(dst_nents < 0)) { + dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", + dst_len); + qi_cache_free(edesc); + return ERR_PTR(dst_nents); + } + + if (src_nents) { + mapped_src_nents = dma_map_sg(dev, req->src, src_nents, + DMA_TO_DEVICE); + if (unlikely(!mapped_src_nents)) { + dev_err(dev, "unable to map source\n"); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + } else { + mapped_src_nents = 0; + } + + if (dst_nents) { + mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, + DMA_FROM_DEVICE); + if (unlikely(!mapped_dst_nents)) { + dev_err(dev, "unable to map destination\n"); + dma_unmap_sg(dev, req->src, src_nents, + DMA_TO_DEVICE); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + } else { + mapped_dst_nents = 0; + } + } else { + src_len = req->assoclen + req->cryptlen + + (encrypt ? authsize : 0); + + src_nents = sg_nents_for_len(req->src, src_len); + if (unlikely(src_nents < 0)) { + dev_err(dev, "Insufficient bytes (%d) in src S/G\n", + src_len); + qi_cache_free(edesc); + return ERR_PTR(src_nents); + } + + mapped_src_nents = dma_map_sg(dev, req->src, src_nents, + DMA_BIDIRECTIONAL); + if (unlikely(!mapped_src_nents)) { + dev_err(dev, "unable to map source\n"); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + } + + if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) + ivsize = crypto_aead_ivsize(aead); + + /* + * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. + * Input is not contiguous. + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond + * the end of the table by allocating more S/G entries. Logic: + * if (src != dst && output S/G) + * pad output S/G, if needed + * else if (src == dst && S/G) + * overlapping S/Gs; pad one of them + * else if (input S/G) ... + * pad input S/G, if needed + */ + qm_sg_nents = 1 + !!ivsize + mapped_src_nents; + if (mapped_dst_nents > 1) + qm_sg_nents += pad_sg_nents(mapped_dst_nents); + else if ((req->src == req->dst) && (mapped_src_nents > 1)) + qm_sg_nents = max(pad_sg_nents(qm_sg_nents), + 1 + !!ivsize + + pad_sg_nents(mapped_src_nents)); + else + qm_sg_nents = pad_sg_nents(qm_sg_nents); + + sg_table = &edesc->sgt[0]; + qm_sg_bytes = qm_sg_nents * sizeof(*sg_table); + if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize > + CAAM_QI_MEMCACHE_SIZE)) { + dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", + qm_sg_nents, ivsize); + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, + 0, DMA_NONE, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + if (ivsize) { + u8 *iv = (u8 *)(sg_table + qm_sg_nents); + + /* Make sure IV is located in a DMAable area */ + memcpy(iv, req->iv, ivsize); + + iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); + if (dma_mapping_error(dev, iv_dma)) { + dev_err(dev, "unable to map IV\n"); + caam_unmap(dev, req->src, req->dst, src_nents, + dst_nents, 0, 0, DMA_NONE, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + } + + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + edesc->iv_dma = iv_dma; + + if ((alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK) == + OP_ALG_ALGSEL_CHACHA20 && ivsize != CHACHAPOLY_IV_SIZE) + /* + * The associated data comes already with the IV but we need + * to skip it when we authenticate or encrypt... + */ + edesc->assoclen = cpu_to_caam32(req->assoclen - ivsize); + else + edesc->assoclen = cpu_to_caam32(req->assoclen); + edesc->assoclen_dma = dma_map_single(dev, &edesc->assoclen, 4, + DMA_TO_DEVICE); + if (dma_mapping_error(dev, edesc->assoclen_dma)) { + dev_err(dev, "unable to map assoclen\n"); + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0); + qm_sg_index++; + if (ivsize) { + dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); + qm_sg_index++; + } + sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0); + qm_sg_index += mapped_src_nents; + + if (mapped_dst_nents > 1) + sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0); + + qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(dev, qm_sg_dma)) { + dev_err(dev, "unable to map S/G table\n"); + dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + edesc->qm_sg_dma = qm_sg_dma; + edesc->qm_sg_bytes = qm_sg_bytes; + + out_len = req->assoclen + req->cryptlen + + (encrypt ? ctx->authsize : (-ctx->authsize)); + in_len = 4 + ivsize + req->assoclen + req->cryptlen; + + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); + dpaa2_fl_set_final(in_fle, true); + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(in_fle, qm_sg_dma); + dpaa2_fl_set_len(in_fle, in_len); + + if (req->dst == req->src) { + if (mapped_src_nents == 1) { + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src)); + } else { + dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(out_fle, qm_sg_dma + + (1 + !!ivsize) * sizeof(*sg_table)); + } + } else if (!mapped_dst_nents) { + /* + * crypto engine requires the output entry to be present when + * "frame list" FD is used. + * Since engine does not support FMT=2'b11 (unused entry type), + * leaving out_fle zeroized is the best option. + */ + goto skip_out_fle; + } else if (mapped_dst_nents == 1) { + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); + } else { + dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index * + sizeof(*sg_table)); + } + + dpaa2_fl_set_len(out_fle, out_len); + +skip_out_fle: + return edesc; +} + +static int chachapoly_set_sh_desc(struct crypto_aead *aead) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + unsigned int ivsize = crypto_aead_ivsize(aead); + struct device *dev = ctx->dev; + struct caam_flc *flc; + u32 *desc; + + if (!ctx->cdata.keylen || !ctx->authsize) + return 0; + + flc = &ctx->flc[ENCRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, + ctx->authsize, true, true); + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], + sizeof(flc->flc) + desc_bytes(desc), + ctx->dir); + + flc = &ctx->flc[DECRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, + ctx->authsize, false, true); + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], + sizeof(flc->flc) + desc_bytes(desc), + ctx->dir); + + return 0; +} + +static int chachapoly_setauthsize(struct crypto_aead *aead, + unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + + if (authsize != POLY1305_DIGEST_SIZE) + return -EINVAL; + + ctx->authsize = authsize; + return chachapoly_set_sh_desc(aead); +} + +static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + unsigned int ivsize = crypto_aead_ivsize(aead); + unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize; + + if (keylen != CHACHA_KEY_SIZE + saltlen) + return -EINVAL; + + memcpy(ctx->key, key, keylen); + ctx->cdata.key_virt = ctx->key; + ctx->cdata.keylen = keylen - saltlen; + + return chachapoly_set_sh_desc(aead); +} + +static int gcm_set_sh_desc(struct crypto_aead *aead) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *dev = ctx->dev; + unsigned int ivsize = crypto_aead_ivsize(aead); + struct caam_flc *flc; + u32 *desc; + int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - + ctx->cdata.keylen; + + if (!ctx->cdata.keylen || !ctx->authsize) + return 0; + + /* + * AES GCM encrypt shared descriptor + * Job Descriptor and Shared Descriptor + * must fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_QI_GCM_ENC_LEN) { + ctx->cdata.key_inline = true; + ctx->cdata.key_virt = ctx->key; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } + + flc = &ctx->flc[ENCRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true); + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], + sizeof(flc->flc) + desc_bytes(desc), + ctx->dir); + + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_QI_GCM_DEC_LEN) { + ctx->cdata.key_inline = true; + ctx->cdata.key_virt = ctx->key; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } + + flc = &ctx->flc[DECRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true); + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], + sizeof(flc->flc) + desc_bytes(desc), + ctx->dir); + + return 0; +} + +static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); + int err; + + err = crypto_gcm_check_authsize(authsize); + if (err) + return err; + + ctx->authsize = authsize; + gcm_set_sh_desc(authenc); + + return 0; +} + +static int gcm_setkey(struct crypto_aead *aead, + const u8 *key, unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *dev = ctx->dev; + int ret; + + ret = aes_check_keylen(keylen); + if (ret) + return ret; + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); + + memcpy(ctx->key, key, keylen); + dma_sync_single_for_device(dev, ctx->key_dma, keylen, ctx->dir); + ctx->cdata.keylen = keylen; + + return gcm_set_sh_desc(aead); +} + +static int rfc4106_set_sh_desc(struct crypto_aead *aead) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *dev = ctx->dev; + unsigned int ivsize = crypto_aead_ivsize(aead); + struct caam_flc *flc; + u32 *desc; + int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - + ctx->cdata.keylen; + + if (!ctx->cdata.keylen || !ctx->authsize) + return 0; + + ctx->cdata.key_virt = ctx->key; + + /* + * RFC4106 encrypt shared descriptor + * Job Descriptor and Shared Descriptor + * must fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) { + ctx->cdata.key_inline = true; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } + + flc = &ctx->flc[ENCRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize, + true); + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], + sizeof(flc->flc) + desc_bytes(desc), + ctx->dir); + + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) { + ctx->cdata.key_inline = true; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } + + flc = &ctx->flc[DECRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize, + true); + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], + sizeof(flc->flc) + desc_bytes(desc), + ctx->dir); + + return 0; +} + +static int rfc4106_setauthsize(struct crypto_aead *authenc, + unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); + int err; + + err = crypto_rfc4106_check_authsize(authsize); + if (err) + return err; + + ctx->authsize = authsize; + rfc4106_set_sh_desc(authenc); + + return 0; +} + +static int rfc4106_setkey(struct crypto_aead *aead, + const u8 *key, unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *dev = ctx->dev; + int ret; + + ret = aes_check_keylen(keylen - 4); + if (ret) + return ret; + + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); + + memcpy(ctx->key, key, keylen); + /* + * The last four bytes of the key material are used as the salt value + * in the nonce. Update the AES key length. + */ + ctx->cdata.keylen = keylen - 4; + dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, + ctx->dir); + + return rfc4106_set_sh_desc(aead); +} + +static int rfc4543_set_sh_desc(struct crypto_aead *aead) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *dev = ctx->dev; + unsigned int ivsize = crypto_aead_ivsize(aead); + struct caam_flc *flc; + u32 *desc; + int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - + ctx->cdata.keylen; + + if (!ctx->cdata.keylen || !ctx->authsize) + return 0; + + ctx->cdata.key_virt = ctx->key; + + /* + * RFC4543 encrypt shared descriptor + * Job Descriptor and Shared Descriptor + * must fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) { + ctx->cdata.key_inline = true; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } + + flc = &ctx->flc[ENCRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize, + true); + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], + sizeof(flc->flc) + desc_bytes(desc), + ctx->dir); + + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) { + ctx->cdata.key_inline = true; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } + + flc = &ctx->flc[DECRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize, + true); + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], + sizeof(flc->flc) + desc_bytes(desc), + ctx->dir); + + return 0; +} + +static int rfc4543_setauthsize(struct crypto_aead *authenc, + unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); + + if (authsize != 16) + return -EINVAL; + + ctx->authsize = authsize; + rfc4543_set_sh_desc(authenc); + + return 0; +} + +static int rfc4543_setkey(struct crypto_aead *aead, + const u8 *key, unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct device *dev = ctx->dev; + int ret; + + ret = aes_check_keylen(keylen - 4); + if (ret) + return ret; + + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); + + memcpy(ctx->key, key, keylen); + /* + * The last four bytes of the key material are used as the salt value + * in the nonce. Update the AES key length. + */ + ctx->cdata.keylen = keylen - 4; + dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, + ctx->dir); + + return rfc4543_set_sh_desc(aead); +} + +static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, + unsigned int keylen, const u32 ctx1_iv_off) +{ + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); + struct caam_skcipher_alg *alg = + container_of(crypto_skcipher_alg(skcipher), + struct caam_skcipher_alg, skcipher); + struct device *dev = ctx->dev; + struct caam_flc *flc; + unsigned int ivsize = crypto_skcipher_ivsize(skcipher); + u32 *desc; + const bool is_rfc3686 = alg->caam.rfc3686; + + print_hex_dump_devel("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); + + ctx->cdata.keylen = keylen; + ctx->cdata.key_virt = key; + ctx->cdata.key_inline = true; + + /* skcipher_encrypt shared descriptor */ + flc = &ctx->flc[ENCRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686, + ctx1_iv_off); + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], + sizeof(flc->flc) + desc_bytes(desc), + ctx->dir); + + /* skcipher_decrypt shared descriptor */ + flc = &ctx->flc[DECRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686, + ctx1_iv_off); + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], + sizeof(flc->flc) + desc_bytes(desc), + ctx->dir); + + return 0; +} + +static int aes_skcipher_setkey(struct crypto_skcipher *skcipher, + const u8 *key, unsigned int keylen) +{ + int err; + + err = aes_check_keylen(keylen); + if (err) + return err; + + return skcipher_setkey(skcipher, key, keylen, 0); +} + +static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher, + const u8 *key, unsigned int keylen) +{ + u32 ctx1_iv_off; + int err; + + /* + * RFC3686 specific: + * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} + * | *key = {KEY, NONCE} + */ + ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; + keylen -= CTR_RFC3686_NONCE_SIZE; + + err = aes_check_keylen(keylen); + if (err) + return err; + + return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); +} + +static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher, + const u8 *key, unsigned int keylen) +{ + u32 ctx1_iv_off; + int err; + + /* + * AES-CTR needs to load IV in CONTEXT1 reg + * at an offset of 128bits (16bytes) + * CONTEXT1[255:128] = IV + */ + ctx1_iv_off = 16; + + err = aes_check_keylen(keylen); + if (err) + return err; + + return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); +} + +static int chacha20_skcipher_setkey(struct crypto_skcipher *skcipher, + const u8 *key, unsigned int keylen) +{ + if (keylen != CHACHA_KEY_SIZE) + return -EINVAL; + + return skcipher_setkey(skcipher, key, keylen, 0); +} + +static int des_skcipher_setkey(struct crypto_skcipher *skcipher, + const u8 *key, unsigned int keylen) +{ + return verify_skcipher_des_key(skcipher, key) ?: + skcipher_setkey(skcipher, key, keylen, 0); +} + +static int des3_skcipher_setkey(struct crypto_skcipher *skcipher, + const u8 *key, unsigned int keylen) +{ + return verify_skcipher_des3_key(skcipher, key) ?: + skcipher_setkey(skcipher, key, keylen, 0); +} + +static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, + unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); + struct device *dev = ctx->dev; + struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); + struct caam_flc *flc; + u32 *desc; + int err; + + err = xts_verify_key(skcipher, key, keylen); + if (err) { + dev_dbg(dev, "key size mismatch\n"); + return err; + } + + if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256) + ctx->xts_key_fallback = true; + + if (priv->sec_attr.era <= 8 || ctx->xts_key_fallback) { + err = crypto_skcipher_setkey(ctx->fallback, key, keylen); + if (err) + return err; + } + + ctx->cdata.keylen = keylen; + ctx->cdata.key_virt = key; + ctx->cdata.key_inline = true; + + /* xts_skcipher_encrypt shared descriptor */ + flc = &ctx->flc[ENCRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata); + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], + sizeof(flc->flc) + desc_bytes(desc), + ctx->dir); + + /* xts_skcipher_decrypt shared descriptor */ + flc = &ctx->flc[DECRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata); + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], + sizeof(flc->flc) + desc_bytes(desc), + ctx->dir); + + return 0; +} + +static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + struct caam_request *req_ctx = skcipher_request_ctx_dma(req); + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); + struct device *dev = ctx->dev; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; + struct skcipher_edesc *edesc; + dma_addr_t iv_dma; + u8 *iv; + int ivsize = crypto_skcipher_ivsize(skcipher); + int dst_sg_idx, qm_sg_ents, qm_sg_bytes; + struct dpaa2_sg_entry *sg_table; + + src_nents = sg_nents_for_len(req->src, req->cryptlen); + if (unlikely(src_nents < 0)) { + dev_err(dev, "Insufficient bytes (%d) in src S/G\n", + req->cryptlen); + return ERR_PTR(src_nents); + } + + if (unlikely(req->dst != req->src)) { + dst_nents = sg_nents_for_len(req->dst, req->cryptlen); + if (unlikely(dst_nents < 0)) { + dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", + req->cryptlen); + return ERR_PTR(dst_nents); + } + + mapped_src_nents = dma_map_sg(dev, req->src, src_nents, + DMA_TO_DEVICE); + if (unlikely(!mapped_src_nents)) { + dev_err(dev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + + mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, + DMA_FROM_DEVICE); + if (unlikely(!mapped_dst_nents)) { + dev_err(dev, "unable to map destination\n"); + dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); + return ERR_PTR(-ENOMEM); + } + } else { + mapped_src_nents = dma_map_sg(dev, req->src, src_nents, + DMA_BIDIRECTIONAL); + if (unlikely(!mapped_src_nents)) { + dev_err(dev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + } + + qm_sg_ents = 1 + mapped_src_nents; + dst_sg_idx = qm_sg_ents; + + /* + * Input, output HW S/G tables: [IV, src][dst, IV] + * IV entries point to the same buffer + * If src == dst, S/G entries are reused (S/G tables overlap) + * + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond + * the end of the table by allocating more S/G entries. + */ + if (req->src != req->dst) + qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1); + else + qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents); + + qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry); + if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes + + ivsize > CAAM_QI_MEMCACHE_SIZE)) { + dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", + qm_sg_ents, ivsize); + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, + 0, DMA_NONE, 0, 0); + return ERR_PTR(-ENOMEM); + } + + /* allocate space for base edesc, link tables and IV */ + edesc = qi_cache_zalloc(flags); + if (unlikely(!edesc)) { + dev_err(dev, "could not allocate extended descriptor\n"); + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, + 0, DMA_NONE, 0, 0); + return ERR_PTR(-ENOMEM); + } + + /* Make sure IV is located in a DMAable area */ + sg_table = &edesc->sgt[0]; + iv = (u8 *)(sg_table + qm_sg_ents); + memcpy(iv, req->iv, ivsize); + + iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, iv_dma)) { + dev_err(dev, "unable to map IV\n"); + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, + 0, DMA_NONE, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + edesc->iv_dma = iv_dma; + edesc->qm_sg_bytes = qm_sg_bytes; + + dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); + sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0); + + if (req->src != req->dst) + sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0); + + dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma, + ivsize, 0); + + edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes, + DMA_TO_DEVICE); + if (dma_mapping_error(dev, edesc->qm_sg_dma)) { + dev_err(dev, "unable to map S/G table\n"); + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); + dpaa2_fl_set_final(in_fle, true); + dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize); + dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize); + + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); + + dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); + + if (req->src == req->dst) + dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + + sizeof(*sg_table)); + else + dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx * + sizeof(*sg_table)); + + return edesc; +} + +static void aead_unmap(struct device *dev, struct aead_edesc *edesc, + struct aead_request *req) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + int ivsize = crypto_aead_ivsize(aead); + + caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, + edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma, + edesc->qm_sg_bytes); + dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); +} + +static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, + struct skcipher_request *req) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + int ivsize = crypto_skcipher_ivsize(skcipher); + + caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, + edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma, + edesc->qm_sg_bytes); +} + +static void aead_encrypt_done(void *cbk_ctx, u32 status) +{ + struct crypto_async_request *areq = cbk_ctx; + struct aead_request *req = container_of(areq, struct aead_request, + base); + struct caam_request *req_ctx = to_caam_req(areq); + struct aead_edesc *edesc = req_ctx->edesc; + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + int ecode = 0; + + dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); + + if (unlikely(status)) + ecode = caam_qi2_strstatus(ctx->dev, status); + + aead_unmap(ctx->dev, edesc, req); + qi_cache_free(edesc); + aead_request_complete(req, ecode); +} + +static void aead_decrypt_done(void *cbk_ctx, u32 status) +{ + struct crypto_async_request *areq = cbk_ctx; + struct aead_request *req = container_of(areq, struct aead_request, + base); + struct caam_request *req_ctx = to_caam_req(areq); + struct aead_edesc *edesc = req_ctx->edesc; + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + int ecode = 0; + + dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); + + if (unlikely(status)) + ecode = caam_qi2_strstatus(ctx->dev, status); + + aead_unmap(ctx->dev, edesc, req); + qi_cache_free(edesc); + aead_request_complete(req, ecode); +} + +static int aead_encrypt(struct aead_request *req) +{ + struct aead_edesc *edesc; + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct caam_request *caam_req = aead_request_ctx_dma(req); + int ret; + + /* allocate extended descriptor */ + edesc = aead_edesc_alloc(req, true); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + caam_req->flc = &ctx->flc[ENCRYPT]; + caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; + caam_req->cbk = aead_encrypt_done; + caam_req->ctx = &req->base; + caam_req->edesc = edesc; + ret = dpaa2_caam_enqueue(ctx->dev, caam_req); + if (ret != -EINPROGRESS && + !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { + aead_unmap(ctx->dev, edesc, req); + qi_cache_free(edesc); + } + + return ret; +} + +static int aead_decrypt(struct aead_request *req) +{ + struct aead_edesc *edesc; + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); + struct caam_request *caam_req = aead_request_ctx_dma(req); + int ret; + + /* allocate extended descriptor */ + edesc = aead_edesc_alloc(req, false); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + caam_req->flc = &ctx->flc[DECRYPT]; + caam_req->flc_dma = ctx->flc_dma[DECRYPT]; + caam_req->cbk = aead_decrypt_done; + caam_req->ctx = &req->base; + caam_req->edesc = edesc; + ret = dpaa2_caam_enqueue(ctx->dev, caam_req); + if (ret != -EINPROGRESS && + !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { + aead_unmap(ctx->dev, edesc, req); + qi_cache_free(edesc); + } + + return ret; +} + +static int ipsec_gcm_encrypt(struct aead_request *req) +{ + return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_encrypt(req); +} + +static int ipsec_gcm_decrypt(struct aead_request *req) +{ + return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_decrypt(req); +} + +static void skcipher_encrypt_done(void *cbk_ctx, u32 status) +{ + struct crypto_async_request *areq = cbk_ctx; + struct skcipher_request *req = skcipher_request_cast(areq); + struct caam_request *req_ctx = to_caam_req(areq); + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); + struct skcipher_edesc *edesc = req_ctx->edesc; + int ecode = 0; + int ivsize = crypto_skcipher_ivsize(skcipher); + + dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); + + if (unlikely(status)) + ecode = caam_qi2_strstatus(ctx->dev, status); + + print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->iv, + edesc->src_nents > 1 ? 100 : ivsize, 1); + caam_dump_sg("dst @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->dst, + edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); + + skcipher_unmap(ctx->dev, edesc, req); + + /* + * The crypto API expects us to set the IV (req->iv) to the last + * ciphertext block (CBC mode) or last counter (CTR mode). + * This is used e.g. by the CTS mode. + */ + if (!ecode) + memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, + ivsize); + + qi_cache_free(edesc); + skcipher_request_complete(req, ecode); +} + +static void skcipher_decrypt_done(void *cbk_ctx, u32 status) +{ + struct crypto_async_request *areq = cbk_ctx; + struct skcipher_request *req = skcipher_request_cast(areq); + struct caam_request *req_ctx = to_caam_req(areq); + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); + struct skcipher_edesc *edesc = req_ctx->edesc; + int ecode = 0; + int ivsize = crypto_skcipher_ivsize(skcipher); + + dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); + + if (unlikely(status)) + ecode = caam_qi2_strstatus(ctx->dev, status); + + print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->iv, + edesc->src_nents > 1 ? 100 : ivsize, 1); + caam_dump_sg("dst @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->dst, + edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); + + skcipher_unmap(ctx->dev, edesc, req); + + /* + * The crypto API expects us to set the IV (req->iv) to the last + * ciphertext block (CBC mode) or last counter (CTR mode). + * This is used e.g. by the CTS mode. + */ + if (!ecode) + memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, + ivsize); + + qi_cache_free(edesc); + skcipher_request_complete(req, ecode); +} + +static inline bool xts_skcipher_ivsize(struct skcipher_request *req) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + unsigned int ivsize = crypto_skcipher_ivsize(skcipher); + + return !!get_unaligned((u64 *)(req->iv + (ivsize / 2))); +} + +static int skcipher_encrypt(struct skcipher_request *req) +{ + struct skcipher_edesc *edesc; + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); + struct caam_request *caam_req = skcipher_request_ctx_dma(req); + struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); + int ret; + + /* + * XTS is expected to return an error even for input length = 0 + * Note that the case input length < block size will be caught during + * HW offloading and return an error. + */ + if (!req->cryptlen && !ctx->fallback) + return 0; + + if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) || + ctx->xts_key_fallback)) { + skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback); + skcipher_request_set_callback(&caam_req->fallback_req, + req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(&caam_req->fallback_req, req->src, + req->dst, req->cryptlen, req->iv); + + return crypto_skcipher_encrypt(&caam_req->fallback_req); + } + + /* allocate extended descriptor */ + edesc = skcipher_edesc_alloc(req); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + caam_req->flc = &ctx->flc[ENCRYPT]; + caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; + caam_req->cbk = skcipher_encrypt_done; + caam_req->ctx = &req->base; + caam_req->edesc = edesc; + ret = dpaa2_caam_enqueue(ctx->dev, caam_req); + if (ret != -EINPROGRESS && + !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { + skcipher_unmap(ctx->dev, edesc, req); + qi_cache_free(edesc); + } + + return ret; +} + +static int skcipher_decrypt(struct skcipher_request *req) +{ + struct skcipher_edesc *edesc; + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); + struct caam_request *caam_req = skcipher_request_ctx_dma(req); + struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); + int ret; + + /* + * XTS is expected to return an error even for input length = 0 + * Note that the case input length < block size will be caught during + * HW offloading and return an error. + */ + if (!req->cryptlen && !ctx->fallback) + return 0; + + if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) || + ctx->xts_key_fallback)) { + skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback); + skcipher_request_set_callback(&caam_req->fallback_req, + req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(&caam_req->fallback_req, req->src, + req->dst, req->cryptlen, req->iv); + + return crypto_skcipher_decrypt(&caam_req->fallback_req); + } + + /* allocate extended descriptor */ + edesc = skcipher_edesc_alloc(req); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + caam_req->flc = &ctx->flc[DECRYPT]; + caam_req->flc_dma = ctx->flc_dma[DECRYPT]; + caam_req->cbk = skcipher_decrypt_done; + caam_req->ctx = &req->base; + caam_req->edesc = edesc; + ret = dpaa2_caam_enqueue(ctx->dev, caam_req); + if (ret != -EINPROGRESS && + !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { + skcipher_unmap(ctx->dev, edesc, req); + qi_cache_free(edesc); + } + + return ret; +} + +static int caam_cra_init(struct caam_ctx *ctx, struct caam_alg_entry *caam, + bool uses_dkp) +{ + dma_addr_t dma_addr; + int i; + + /* copy descriptor header template value */ + ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; + ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; + + ctx->dev = caam->dev; + ctx->dir = uses_dkp ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; + + dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, + offsetof(struct caam_ctx, flc_dma), + ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); + if (dma_mapping_error(ctx->dev, dma_addr)) { + dev_err(ctx->dev, "unable to map key, shared descriptors\n"); + return -ENOMEM; + } + + for (i = 0; i < NUM_OP; i++) + ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); + ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]); + + return 0; +} + +static int caam_cra_init_skcipher(struct crypto_skcipher *tfm) +{ + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct caam_skcipher_alg *caam_alg = + container_of(alg, typeof(*caam_alg), skcipher); + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm); + u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; + int ret = 0; + + if (alg_aai == OP_ALG_AAI_XTS) { + const char *tfm_name = crypto_tfm_alg_name(&tfm->base); + struct crypto_skcipher *fallback; + + fallback = crypto_alloc_skcipher(tfm_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(fallback)) { + dev_err(caam_alg->caam.dev, + "Failed to allocate %s fallback: %ld\n", + tfm_name, PTR_ERR(fallback)); + return PTR_ERR(fallback); + } + + ctx->fallback = fallback; + crypto_skcipher_set_reqsize_dma( + tfm, sizeof(struct caam_request) + + crypto_skcipher_reqsize(fallback)); + } else { + crypto_skcipher_set_reqsize_dma(tfm, + sizeof(struct caam_request)); + } + + ret = caam_cra_init(ctx, &caam_alg->caam, false); + if (ret && ctx->fallback) + crypto_free_skcipher(ctx->fallback); + + return ret; +} + +static int caam_cra_init_aead(struct crypto_aead *tfm) +{ + struct aead_alg *alg = crypto_aead_alg(tfm); + struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg), + aead); + + crypto_aead_set_reqsize_dma(tfm, sizeof(struct caam_request)); + return caam_cra_init(crypto_aead_ctx_dma(tfm), &caam_alg->caam, + !caam_alg->caam.nodkp); +} + +static void caam_exit_common(struct caam_ctx *ctx) +{ + dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], + offsetof(struct caam_ctx, flc_dma), ctx->dir, + DMA_ATTR_SKIP_CPU_SYNC); +} + +static void caam_cra_exit(struct crypto_skcipher *tfm) +{ + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm); + + if (ctx->fallback) + crypto_free_skcipher(ctx->fallback); + caam_exit_common(ctx); +} + +static void caam_cra_exit_aead(struct crypto_aead *tfm) +{ + caam_exit_common(crypto_aead_ctx_dma(tfm)); +} + +static struct caam_skcipher_alg driver_algs[] = { + { + .skcipher = { + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aes_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + }, + { + .skcipher = { + .base = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "cbc-3des-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + }, + .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + }, + { + .skcipher = { + .base = { + .cra_name = "cbc(des)", + .cra_driver_name = "cbc-des-caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = des_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + }, + .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + }, + { + .skcipher = { + .base = { + .cra_name = "ctr(aes)", + .cra_driver_name = "ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = ctr_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .chunksize = AES_BLOCK_SIZE, + }, + .caam.class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + }, + { + .skcipher = { + .base = { + .cra_name = "rfc3686(ctr(aes))", + .cra_driver_name = "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = rfc3686_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + .chunksize = AES_BLOCK_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .rfc3686 = true, + }, + }, + { + .skcipher = { + .base = { + .cra_name = "xts(aes)", + .cra_driver_name = "xts-aes-caam-qi2", + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = xts_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, + }, + { + .skcipher = { + .base = { + .cra_name = "chacha20", + .cra_driver_name = "chacha20-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = chacha20_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = CHACHA_KEY_SIZE, + .max_keysize = CHACHA_KEY_SIZE, + .ivsize = CHACHA_IV_SIZE, + }, + .caam.class1_alg_type = OP_ALG_ALGSEL_CHACHA20, + }, +}; + +static struct caam_aead_alg driver_aeads[] = { + { + .aead = { + .base = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "rfc4106-gcm-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = rfc4106_setkey, + .setauthsize = rfc4106_setauthsize, + .encrypt = ipsec_gcm_encrypt, + .decrypt = ipsec_gcm_decrypt, + .ivsize = 8, + .maxauthsize = AES_BLOCK_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + .nodkp = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "rfc4543(gcm(aes))", + .cra_driver_name = "rfc4543-gcm-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = rfc4543_setkey, + .setauthsize = rfc4543_setauthsize, + .encrypt = ipsec_gcm_encrypt, + .decrypt = ipsec_gcm_decrypt, + .ivsize = 8, + .maxauthsize = AES_BLOCK_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + .nodkp = true, + }, + }, + /* Galois Counter Mode */ + { + .aead = { + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "gcm-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = gcm_setkey, + .setauthsize = gcm_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = 12, + .maxauthsize = AES_BLOCK_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + .nodkp = true, + } + }, + /* single-pass ipsec_esp descriptor */ + { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(aes))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-aes-caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(md5)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-hmac-md5-" + "cbc-aes-caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-aes-caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha1)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha1-cbc-aes-caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-aes-caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha224)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha224-cbc-aes-caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-aes-caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha256)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha256-cbc-aes-" + "caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-aes-caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha384)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha384-cbc-aes-" + "caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-aes-caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha512)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha512-cbc-aes-" + "caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-des3_ede-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(md5)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-hmac-md5-" + "cbc-des3_ede-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-des3_ede-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha1)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha1-" + "cbc-des3_ede-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-des3_ede-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha224)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha224-" + "cbc-des3_ede-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-des3_ede-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha256)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha256-" + "cbc-des3_ede-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-des3_ede-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha384)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha384-" + "cbc-des3_ede-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-des3_ede-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha512)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha512-" + "cbc-des3_ede-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = des3_aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-des-caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(md5)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-hmac-md5-" + "cbc-des-caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-des-caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha1)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha1-cbc-des-caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(des))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-des-caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha224)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha224-cbc-des-" + "caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(des))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-des-caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha256)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha256-cbc-des-" + "caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(des))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-des-caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha384)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha384-cbc-des-" + "caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(des))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-des-caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha512)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha512-cbc-des-" + "caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-md5-" + "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "seqiv(authenc(" + "hmac(md5),rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-md5-" + "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + .geniv = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha1-" + "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "seqiv(authenc(" + "hmac(sha1),rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha1-" + "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + .geniv = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha224-" + "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "seqiv(authenc(" + "hmac(sha224),rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha224-" + "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + .geniv = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha256-" + "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "seqiv(authenc(hmac(sha256)," + "rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha256-" + "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + .geniv = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha384-" + "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "seqiv(authenc(hmac(sha384)," + "rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha384-" + "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + .geniv = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "rfc7539(chacha20,poly1305)", + .cra_driver_name = "rfc7539-chacha20-poly1305-" + "caam-qi2", + .cra_blocksize = 1, + }, + .setkey = chachapoly_setkey, + .setauthsize = chachapoly_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CHACHAPOLY_IV_SIZE, + .maxauthsize = POLY1305_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | + OP_ALG_AAI_AEAD, + .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | + OP_ALG_AAI_AEAD, + .nodkp = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "rfc7539esp(chacha20,poly1305)", + .cra_driver_name = "rfc7539esp-chacha20-" + "poly1305-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = chachapoly_setkey, + .setauthsize = chachapoly_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = 8, + .maxauthsize = POLY1305_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | + OP_ALG_AAI_AEAD, + .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | + OP_ALG_AAI_AEAD, + .nodkp = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha512-" + "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "seqiv(authenc(hmac(sha512)," + "rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha512-" + "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + .geniv = true, + }, + }, +}; + +static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) +{ + struct skcipher_alg *alg = &t_alg->skcipher; + + alg->base.cra_module = THIS_MODULE; + alg->base.cra_priority = CAAM_CRA_PRIORITY; + alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding(); + alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY); + + alg->init = caam_cra_init_skcipher; + alg->exit = caam_cra_exit; +} + +static void caam_aead_alg_init(struct caam_aead_alg *t_alg) +{ + struct aead_alg *alg = &t_alg->aead; + + alg->base.cra_module = THIS_MODULE; + alg->base.cra_priority = CAAM_CRA_PRIORITY; + alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding(); + alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY; + + alg->init = caam_cra_init_aead; + alg->exit = caam_cra_exit_aead; +} + +/* max hash key is max split key size */ +#define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2) + +#define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE + +/* caam context sizes for hashes: running digest + 8 */ +#define HASH_MSG_LEN 8 +#define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE) + +enum hash_optype { + UPDATE = 0, + UPDATE_FIRST, + FINALIZE, + DIGEST, + HASH_NUM_OP +}; + +/** + * struct caam_hash_ctx - ahash per-session context + * @flc: Flow Contexts array + * @key: authentication key + * @flc_dma: I/O virtual addresses of the Flow Contexts + * @dev: dpseci device + * @ctx_len: size of Context Register + * @adata: hashing algorithm details + */ +struct caam_hash_ctx { + struct caam_flc flc[HASH_NUM_OP]; + u8 key[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; + dma_addr_t flc_dma[HASH_NUM_OP]; + struct device *dev; + int ctx_len; + struct alginfo adata; +}; + +/* ahash state */ +struct caam_hash_state { + struct caam_request caam_req; + dma_addr_t buf_dma; + dma_addr_t ctx_dma; + int ctx_dma_len; + u8 buf[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; + int buflen; + int next_buflen; + u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned; + int (*update)(struct ahash_request *req); + int (*final)(struct ahash_request *req); + int (*finup)(struct ahash_request *req); +}; + +struct caam_export_state { + u8 buf[CAAM_MAX_HASH_BLOCK_SIZE]; + u8 caam_ctx[MAX_CTX_LEN]; + int buflen; + int (*update)(struct ahash_request *req); + int (*final)(struct ahash_request *req); + int (*finup)(struct ahash_request *req); +}; + +/* Map current buffer in state (if length > 0) and put it in link table */ +static inline int buf_map_to_qm_sg(struct device *dev, + struct dpaa2_sg_entry *qm_sg, + struct caam_hash_state *state) +{ + int buflen = state->buflen; + + if (!buflen) + return 0; + + state->buf_dma = dma_map_single(dev, state->buf, buflen, + DMA_TO_DEVICE); + if (dma_mapping_error(dev, state->buf_dma)) { + dev_err(dev, "unable to map buf\n"); + state->buf_dma = 0; + return -ENOMEM; + } + + dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0); + + return 0; +} + +/* Map state->caam_ctx, and add it to link table */ +static inline int ctx_map_to_qm_sg(struct device *dev, + struct caam_hash_state *state, int ctx_len, + struct dpaa2_sg_entry *qm_sg, u32 flag) +{ + state->ctx_dma_len = ctx_len; + state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag); + if (dma_mapping_error(dev, state->ctx_dma)) { + dev_err(dev, "unable to map ctx\n"); + state->ctx_dma = 0; + return -ENOMEM; + } + + dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0); + + return 0; +} + +static int ahash_set_sh_desc(struct crypto_ahash *ahash) +{ + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + int digestsize = crypto_ahash_digestsize(ahash); + struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); + struct caam_flc *flc; + u32 *desc; + + /* ahash_update shared descriptor */ + flc = &ctx->flc[UPDATE]; + desc = flc->sh_desc; + cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len, + ctx->ctx_len, true, priv->sec_attr.era); + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ + dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE], + desc_bytes(desc), DMA_BIDIRECTIONAL); + print_hex_dump_debug("ahash update shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); + + /* ahash_update_first shared descriptor */ + flc = &ctx->flc[UPDATE_FIRST]; + desc = flc->sh_desc; + cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len, + ctx->ctx_len, false, priv->sec_attr.era); + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ + dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST], + desc_bytes(desc), DMA_BIDIRECTIONAL); + print_hex_dump_debug("ahash update first shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); + + /* ahash_final shared descriptor */ + flc = &ctx->flc[FINALIZE]; + desc = flc->sh_desc; + cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize, + ctx->ctx_len, true, priv->sec_attr.era); + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ + dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE], + desc_bytes(desc), DMA_BIDIRECTIONAL); + print_hex_dump_debug("ahash final shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); + + /* ahash_digest shared descriptor */ + flc = &ctx->flc[DIGEST]; + desc = flc->sh_desc; + cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize, + ctx->ctx_len, false, priv->sec_attr.era); + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ + dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST], + desc_bytes(desc), DMA_BIDIRECTIONAL); + print_hex_dump_debug("ahash digest shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); + + return 0; +} + +struct split_key_sh_result { + struct completion completion; + int err; + struct device *dev; +}; + +static void split_key_sh_done(void *cbk_ctx, u32 err) +{ + struct split_key_sh_result *res = cbk_ctx; + + dev_dbg(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); + + res->err = err ? caam_qi2_strstatus(res->dev, err) : 0; + complete(&res->completion); +} + +/* Digest hash size if it is too large */ +static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key, + u32 digestsize) +{ + struct caam_request *req_ctx; + u32 *desc; + struct split_key_sh_result result; + dma_addr_t key_dma; + struct caam_flc *flc; + dma_addr_t flc_dma; + int ret = -ENOMEM; + struct dpaa2_fl_entry *in_fle, *out_fle; + + req_ctx = kzalloc_obj(*req_ctx); + if (!req_ctx) + return -ENOMEM; + + in_fle = &req_ctx->fd_flt[1]; + out_fle = &req_ctx->fd_flt[0]; + + flc = kzalloc_obj(*flc); + if (!flc) + goto err_flc; + + key_dma = dma_map_single(ctx->dev, key, *keylen, DMA_BIDIRECTIONAL); + if (dma_mapping_error(ctx->dev, key_dma)) { + dev_err(ctx->dev, "unable to map key memory\n"); + goto err_key_dma; + } + + desc = flc->sh_desc; + + init_sh_desc(desc, 0); + + /* descriptor to perform unkeyed hash on key_in */ + append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT | + OP_ALG_AS_INITFINAL); + append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 | + FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG); + append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | + LDST_SRCDST_BYTE_CONTEXT); + + flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ + flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) + + desc_bytes(desc), DMA_TO_DEVICE); + if (dma_mapping_error(ctx->dev, flc_dma)) { + dev_err(ctx->dev, "unable to map shared descriptor\n"); + goto err_flc_dma; + } + + dpaa2_fl_set_final(in_fle, true); + dpaa2_fl_set_format(in_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(in_fle, key_dma); + dpaa2_fl_set_len(in_fle, *keylen); + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, key_dma); + dpaa2_fl_set_len(out_fle, digestsize); + + print_hex_dump_devel("key_in@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1); + print_hex_dump_debug("shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); + + result.err = 0; + init_completion(&result.completion); + result.dev = ctx->dev; + + req_ctx->flc = flc; + req_ctx->flc_dma = flc_dma; + req_ctx->cbk = split_key_sh_done; + req_ctx->ctx = &result; + + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); + if (ret == -EINPROGRESS) { + /* in progress */ + wait_for_completion(&result.completion); + ret = result.err; + print_hex_dump_devel("digested key@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, + digestsize, 1); + } + + dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc), + DMA_TO_DEVICE); +err_flc_dma: + dma_unmap_single(ctx->dev, key_dma, *keylen, DMA_BIDIRECTIONAL); +err_key_dma: + kfree(flc); +err_flc: + kfree(req_ctx); + + *keylen = digestsize; + + return ret; +} + +static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key, + unsigned int keylen) +{ + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base); + unsigned int digestsize = crypto_ahash_digestsize(ahash); + int ret; + u8 *hashed_key = NULL; + + dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize); + + if (keylen > blocksize) { + unsigned int aligned_len = + ALIGN(keylen, dma_get_cache_alignment()); + + if (aligned_len < keylen) + return -EOVERFLOW; + + hashed_key = kmalloc(aligned_len, GFP_KERNEL); + if (!hashed_key) + return -ENOMEM; + memcpy(hashed_key, key, keylen); + ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize); + if (ret) + goto bad_free_key; + key = hashed_key; + } + + ctx->adata.keylen = keylen; + ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & + OP_ALG_ALGSEL_MASK); + if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE) + goto bad_free_key; + + ctx->adata.key_virt = key; + ctx->adata.key_inline = true; + + /* + * In case |user key| > |derived key|, using DKP would result + * in invalid opcodes (last bytes of user key) in the resulting + * descriptor. Use DKP instead => both virtual and dma key + * addresses are needed. + */ + if (keylen > ctx->adata.keylen_pad) { + memcpy(ctx->key, key, keylen); + dma_sync_single_for_device(ctx->dev, ctx->adata.key_dma, + ctx->adata.keylen_pad, + DMA_TO_DEVICE); + } + + ret = ahash_set_sh_desc(ahash); + kfree(hashed_key); + return ret; +bad_free_key: + kfree(hashed_key); + return -EINVAL; +} + +static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc, + struct ahash_request *req) +{ + struct caam_hash_state *state = ahash_request_ctx_dma(req); + + if (edesc->src_nents) + dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); + + if (edesc->qm_sg_bytes) + dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes, + DMA_TO_DEVICE); + + if (state->buf_dma) { + dma_unmap_single(dev, state->buf_dma, state->buflen, + DMA_TO_DEVICE); + state->buf_dma = 0; + } +} + +static inline void ahash_unmap_ctx(struct device *dev, + struct ahash_edesc *edesc, + struct ahash_request *req, u32 flag) +{ + struct caam_hash_state *state = ahash_request_ctx_dma(req); + + if (state->ctx_dma) { + dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag); + state->ctx_dma = 0; + } + ahash_unmap(dev, edesc, req); +} + +static void ahash_done(void *cbk_ctx, u32 status) +{ + struct crypto_async_request *areq = cbk_ctx; + struct ahash_request *req = ahash_request_cast(areq); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct ahash_edesc *edesc = state->caam_req.edesc; + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + int digestsize = crypto_ahash_digestsize(ahash); + int ecode = 0; + + dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); + + if (unlikely(status)) + ecode = caam_qi2_strstatus(ctx->dev, status); + + ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); + memcpy(req->result, state->caam_ctx, digestsize); + qi_cache_free(edesc); + + print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, + ctx->ctx_len, 1); + + ahash_request_complete(req, ecode); +} + +static void ahash_done_bi(void *cbk_ctx, u32 status) +{ + struct crypto_async_request *areq = cbk_ctx; + struct ahash_request *req = ahash_request_cast(areq); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct ahash_edesc *edesc = state->caam_req.edesc; + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + int ecode = 0; + + dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); + + if (unlikely(status)) + ecode = caam_qi2_strstatus(ctx->dev, status); + + ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); + qi_cache_free(edesc); + + scatterwalk_map_and_copy(state->buf, req->src, + req->nbytes - state->next_buflen, + state->next_buflen, 0); + state->buflen = state->next_buflen; + + print_hex_dump_debug("buf@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->buf, + state->buflen, 1); + + print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, + ctx->ctx_len, 1); + if (req->result) + print_hex_dump_debug("result@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->result, + crypto_ahash_digestsize(ahash), 1); + + ahash_request_complete(req, ecode); +} + +static void ahash_done_ctx_src(void *cbk_ctx, u32 status) +{ + struct crypto_async_request *areq = cbk_ctx; + struct ahash_request *req = ahash_request_cast(areq); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct ahash_edesc *edesc = state->caam_req.edesc; + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + int digestsize = crypto_ahash_digestsize(ahash); + int ecode = 0; + + dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); + + if (unlikely(status)) + ecode = caam_qi2_strstatus(ctx->dev, status); + + ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); + memcpy(req->result, state->caam_ctx, digestsize); + qi_cache_free(edesc); + + print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, + ctx->ctx_len, 1); + + ahash_request_complete(req, ecode); +} + +static void ahash_done_ctx_dst(void *cbk_ctx, u32 status) +{ + struct crypto_async_request *areq = cbk_ctx; + struct ahash_request *req = ahash_request_cast(areq); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct ahash_edesc *edesc = state->caam_req.edesc; + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + int ecode = 0; + + dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); + + if (unlikely(status)) + ecode = caam_qi2_strstatus(ctx->dev, status); + + ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); + qi_cache_free(edesc); + + scatterwalk_map_and_copy(state->buf, req->src, + req->nbytes - state->next_buflen, + state->next_buflen, 0); + state->buflen = state->next_buflen; + + print_hex_dump_debug("buf@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->buf, + state->buflen, 1); + + print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, + ctx->ctx_len, 1); + if (req->result) + print_hex_dump_debug("result@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->result, + crypto_ahash_digestsize(ahash), 1); + + ahash_request_complete(req, ecode); +} + +static int ahash_update_ctx(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct caam_request *req_ctx = &state->caam_req; + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + u8 *buf = state->buf; + int *buflen = &state->buflen; + int *next_buflen = &state->next_buflen; + int in_len = *buflen + req->nbytes, to_hash; + int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index; + struct ahash_edesc *edesc; + int ret = 0; + + *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); + to_hash = in_len - *next_buflen; + + if (to_hash) { + struct dpaa2_sg_entry *sg_table; + int src_len = req->nbytes - *next_buflen; + + src_nents = sg_nents_for_len(req->src, src_len); + if (src_nents < 0) { + dev_err(ctx->dev, "Invalid number of src SG.\n"); + return src_nents; + } + + if (src_nents) { + mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, + DMA_TO_DEVICE); + if (!mapped_nents) { + dev_err(ctx->dev, "unable to DMA map source\n"); + return -ENOMEM; + } + } else { + mapped_nents = 0; + } + + /* allocate space for base edesc and link tables */ + edesc = qi_cache_zalloc(flags); + if (!edesc) { + dma_unmap_sg(ctx->dev, req->src, src_nents, + DMA_TO_DEVICE); + return -ENOMEM; + } + + edesc->src_nents = src_nents; + qm_sg_src_index = 1 + (*buflen ? 1 : 0); + qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) * + sizeof(*sg_table); + sg_table = &edesc->sgt[0]; + + ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, + DMA_BIDIRECTIONAL); + if (ret) + goto unmap_ctx; + + ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); + if (ret) + goto unmap_ctx; + + if (mapped_nents) { + sg_to_qm_sg_last(req->src, src_len, + sg_table + qm_sg_src_index, 0); + } else { + dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, + true); + } + + edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, + qm_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { + dev_err(ctx->dev, "unable to map S/G table\n"); + ret = -ENOMEM; + goto unmap_ctx; + } + edesc->qm_sg_bytes = qm_sg_bytes; + + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); + dpaa2_fl_set_final(in_fle, true); + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); + dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash); + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, state->ctx_dma); + dpaa2_fl_set_len(out_fle, ctx->ctx_len); + + req_ctx->flc = &ctx->flc[UPDATE]; + req_ctx->flc_dma = ctx->flc_dma[UPDATE]; + req_ctx->cbk = ahash_done_bi; + req_ctx->ctx = &req->base; + req_ctx->edesc = edesc; + + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); + if (ret != -EINPROGRESS && + !(ret == -EBUSY && + req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + goto unmap_ctx; + } else if (*next_buflen) { + scatterwalk_map_and_copy(buf + *buflen, req->src, 0, + req->nbytes, 0); + *buflen = *next_buflen; + + print_hex_dump_debug("buf@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, buf, + *buflen, 1); + } + + return ret; +unmap_ctx: + ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); + qi_cache_free(edesc); + return ret; +} + +static int ahash_final_ctx(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct caam_request *req_ctx = &state->caam_req; + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int buflen = state->buflen; + int qm_sg_bytes; + int digestsize = crypto_ahash_digestsize(ahash); + struct ahash_edesc *edesc; + struct dpaa2_sg_entry *sg_table; + int ret; + + /* allocate space for base edesc and link tables */ + edesc = qi_cache_zalloc(flags); + if (!edesc) + return -ENOMEM; + + qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table); + sg_table = &edesc->sgt[0]; + + ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, + DMA_BIDIRECTIONAL); + if (ret) + goto unmap_ctx; + + ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); + if (ret) + goto unmap_ctx; + + dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true); + + edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, + DMA_TO_DEVICE); + if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { + dev_err(ctx->dev, "unable to map S/G table\n"); + ret = -ENOMEM; + goto unmap_ctx; + } + edesc->qm_sg_bytes = qm_sg_bytes; + + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); + dpaa2_fl_set_final(in_fle, true); + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); + dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen); + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, state->ctx_dma); + dpaa2_fl_set_len(out_fle, digestsize); + + req_ctx->flc = &ctx->flc[FINALIZE]; + req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; + req_ctx->cbk = ahash_done_ctx_src; + req_ctx->ctx = &req->base; + req_ctx->edesc = edesc; + + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); + if (ret == -EINPROGRESS || + (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + return ret; + +unmap_ctx: + ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); + qi_cache_free(edesc); + return ret; +} + +static int ahash_finup_ctx(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct caam_request *req_ctx = &state->caam_req; + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int buflen = state->buflen; + int qm_sg_bytes, qm_sg_src_index; + int src_nents, mapped_nents; + int digestsize = crypto_ahash_digestsize(ahash); + struct ahash_edesc *edesc; + struct dpaa2_sg_entry *sg_table; + int ret; + + src_nents = sg_nents_for_len(req->src, req->nbytes); + if (src_nents < 0) { + dev_err(ctx->dev, "Invalid number of src SG.\n"); + return src_nents; + } + + if (src_nents) { + mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, + DMA_TO_DEVICE); + if (!mapped_nents) { + dev_err(ctx->dev, "unable to DMA map source\n"); + return -ENOMEM; + } + } else { + mapped_nents = 0; + } + + /* allocate space for base edesc and link tables */ + edesc = qi_cache_zalloc(flags); + if (!edesc) { + dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); + return -ENOMEM; + } + + edesc->src_nents = src_nents; + qm_sg_src_index = 1 + (buflen ? 1 : 0); + qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) * + sizeof(*sg_table); + sg_table = &edesc->sgt[0]; + + ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, + DMA_BIDIRECTIONAL); + if (ret) + goto unmap_ctx; + + ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); + if (ret) + goto unmap_ctx; + + sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0); + + edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, + DMA_TO_DEVICE); + if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { + dev_err(ctx->dev, "unable to map S/G table\n"); + ret = -ENOMEM; + goto unmap_ctx; + } + edesc->qm_sg_bytes = qm_sg_bytes; + + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); + dpaa2_fl_set_final(in_fle, true); + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); + dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes); + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, state->ctx_dma); + dpaa2_fl_set_len(out_fle, digestsize); + + req_ctx->flc = &ctx->flc[FINALIZE]; + req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; + req_ctx->cbk = ahash_done_ctx_src; + req_ctx->ctx = &req->base; + req_ctx->edesc = edesc; + + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); + if (ret == -EINPROGRESS || + (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + return ret; + +unmap_ctx: + ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); + qi_cache_free(edesc); + return ret; +} + +static int ahash_digest(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct caam_request *req_ctx = &state->caam_req; + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int digestsize = crypto_ahash_digestsize(ahash); + int src_nents, mapped_nents; + struct ahash_edesc *edesc; + int ret = -ENOMEM; + + state->buf_dma = 0; + + src_nents = sg_nents_for_len(req->src, req->nbytes); + if (src_nents < 0) { + dev_err(ctx->dev, "Invalid number of src SG.\n"); + return src_nents; + } + + if (src_nents) { + mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, + DMA_TO_DEVICE); + if (!mapped_nents) { + dev_err(ctx->dev, "unable to map source for DMA\n"); + return ret; + } + } else { + mapped_nents = 0; + } + + /* allocate space for base edesc and link tables */ + edesc = qi_cache_zalloc(flags); + if (!edesc) { + dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); + return ret; + } + + edesc->src_nents = src_nents; + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); + + if (mapped_nents > 1) { + int qm_sg_bytes; + struct dpaa2_sg_entry *sg_table = &edesc->sgt[0]; + + qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table); + sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0); + edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, + qm_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { + dev_err(ctx->dev, "unable to map S/G table\n"); + goto unmap; + } + edesc->qm_sg_bytes = qm_sg_bytes; + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); + } else { + dpaa2_fl_set_format(in_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); + } + + state->ctx_dma_len = digestsize; + state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, + DMA_FROM_DEVICE); + if (dma_mapping_error(ctx->dev, state->ctx_dma)) { + dev_err(ctx->dev, "unable to map ctx\n"); + state->ctx_dma = 0; + goto unmap; + } + + dpaa2_fl_set_final(in_fle, true); + dpaa2_fl_set_len(in_fle, req->nbytes); + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, state->ctx_dma); + dpaa2_fl_set_len(out_fle, digestsize); + + req_ctx->flc = &ctx->flc[DIGEST]; + req_ctx->flc_dma = ctx->flc_dma[DIGEST]; + req_ctx->cbk = ahash_done; + req_ctx->ctx = &req->base; + req_ctx->edesc = edesc; + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); + if (ret == -EINPROGRESS || + (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + return ret; + +unmap: + ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); + qi_cache_free(edesc); + return ret; +} + +static int ahash_final_no_ctx(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct caam_request *req_ctx = &state->caam_req; + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + u8 *buf = state->buf; + int buflen = state->buflen; + int digestsize = crypto_ahash_digestsize(ahash); + struct ahash_edesc *edesc; + int ret = -ENOMEM; + + /* allocate space for base edesc and link tables */ + edesc = qi_cache_zalloc(flags); + if (!edesc) + return ret; + + if (buflen) { + state->buf_dma = dma_map_single(ctx->dev, buf, buflen, + DMA_TO_DEVICE); + if (dma_mapping_error(ctx->dev, state->buf_dma)) { + dev_err(ctx->dev, "unable to map src\n"); + goto unmap; + } + } + + state->ctx_dma_len = digestsize; + state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, + DMA_FROM_DEVICE); + if (dma_mapping_error(ctx->dev, state->ctx_dma)) { + dev_err(ctx->dev, "unable to map ctx\n"); + state->ctx_dma = 0; + goto unmap; + } + + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); + dpaa2_fl_set_final(in_fle, true); + /* + * crypto engine requires the input entry to be present when + * "frame list" FD is used. + * Since engine does not support FMT=2'b11 (unused entry type), leaving + * in_fle zeroized (except for "Final" flag) is the best option. + */ + if (buflen) { + dpaa2_fl_set_format(in_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(in_fle, state->buf_dma); + dpaa2_fl_set_len(in_fle, buflen); + } + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, state->ctx_dma); + dpaa2_fl_set_len(out_fle, digestsize); + + req_ctx->flc = &ctx->flc[DIGEST]; + req_ctx->flc_dma = ctx->flc_dma[DIGEST]; + req_ctx->cbk = ahash_done; + req_ctx->ctx = &req->base; + req_ctx->edesc = edesc; + + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); + if (ret == -EINPROGRESS || + (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + return ret; + +unmap: + ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); + qi_cache_free(edesc); + return ret; +} + +static int ahash_update_no_ctx(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct caam_request *req_ctx = &state->caam_req; + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + u8 *buf = state->buf; + int *buflen = &state->buflen; + int *next_buflen = &state->next_buflen; + int in_len = *buflen + req->nbytes, to_hash; + int qm_sg_bytes, src_nents, mapped_nents; + struct ahash_edesc *edesc; + int ret = 0; + + *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); + to_hash = in_len - *next_buflen; + + if (to_hash) { + struct dpaa2_sg_entry *sg_table; + int src_len = req->nbytes - *next_buflen; + + src_nents = sg_nents_for_len(req->src, src_len); + if (src_nents < 0) { + dev_err(ctx->dev, "Invalid number of src SG.\n"); + return src_nents; + } + + if (src_nents) { + mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, + DMA_TO_DEVICE); + if (!mapped_nents) { + dev_err(ctx->dev, "unable to DMA map source\n"); + return -ENOMEM; + } + } else { + mapped_nents = 0; + } + + /* allocate space for base edesc and link tables */ + edesc = qi_cache_zalloc(flags); + if (!edesc) { + dma_unmap_sg(ctx->dev, req->src, src_nents, + DMA_TO_DEVICE); + return -ENOMEM; + } + + edesc->src_nents = src_nents; + qm_sg_bytes = pad_sg_nents(1 + mapped_nents) * + sizeof(*sg_table); + sg_table = &edesc->sgt[0]; + + ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); + if (ret) + goto unmap_ctx; + + sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0); + + edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, + qm_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { + dev_err(ctx->dev, "unable to map S/G table\n"); + ret = -ENOMEM; + goto unmap_ctx; + } + edesc->qm_sg_bytes = qm_sg_bytes; + + state->ctx_dma_len = ctx->ctx_len; + state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, + ctx->ctx_len, DMA_FROM_DEVICE); + if (dma_mapping_error(ctx->dev, state->ctx_dma)) { + dev_err(ctx->dev, "unable to map ctx\n"); + state->ctx_dma = 0; + ret = -ENOMEM; + goto unmap_ctx; + } + + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); + dpaa2_fl_set_final(in_fle, true); + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); + dpaa2_fl_set_len(in_fle, to_hash); + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, state->ctx_dma); + dpaa2_fl_set_len(out_fle, ctx->ctx_len); + + req_ctx->flc = &ctx->flc[UPDATE_FIRST]; + req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; + req_ctx->cbk = ahash_done_ctx_dst; + req_ctx->ctx = &req->base; + req_ctx->edesc = edesc; + + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); + if (ret != -EINPROGRESS && + !(ret == -EBUSY && + req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + goto unmap_ctx; + + state->update = ahash_update_ctx; + state->finup = ahash_finup_ctx; + state->final = ahash_final_ctx; + } else if (*next_buflen) { + scatterwalk_map_and_copy(buf + *buflen, req->src, 0, + req->nbytes, 0); + *buflen = *next_buflen; + + print_hex_dump_debug("buf@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, buf, + *buflen, 1); + } + + return ret; +unmap_ctx: + ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE); + qi_cache_free(edesc); + return ret; +} + +static int ahash_finup_no_ctx(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct caam_request *req_ctx = &state->caam_req; + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int buflen = state->buflen; + int qm_sg_bytes, src_nents, mapped_nents; + int digestsize = crypto_ahash_digestsize(ahash); + struct ahash_edesc *edesc; + struct dpaa2_sg_entry *sg_table; + int ret = -ENOMEM; + + src_nents = sg_nents_for_len(req->src, req->nbytes); + if (src_nents < 0) { + dev_err(ctx->dev, "Invalid number of src SG.\n"); + return src_nents; + } + + if (src_nents) { + mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, + DMA_TO_DEVICE); + if (!mapped_nents) { + dev_err(ctx->dev, "unable to DMA map source\n"); + return ret; + } + } else { + mapped_nents = 0; + } + + /* allocate space for base edesc and link tables */ + edesc = qi_cache_zalloc(flags); + if (!edesc) { + dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); + return ret; + } + + edesc->src_nents = src_nents; + qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table); + sg_table = &edesc->sgt[0]; + + ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); + if (ret) + goto unmap; + + sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0); + + edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, + DMA_TO_DEVICE); + if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { + dev_err(ctx->dev, "unable to map S/G table\n"); + ret = -ENOMEM; + goto unmap; + } + edesc->qm_sg_bytes = qm_sg_bytes; + + state->ctx_dma_len = digestsize; + state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, + DMA_FROM_DEVICE); + if (dma_mapping_error(ctx->dev, state->ctx_dma)) { + dev_err(ctx->dev, "unable to map ctx\n"); + state->ctx_dma = 0; + ret = -ENOMEM; + goto unmap; + } + + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); + dpaa2_fl_set_final(in_fle, true); + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); + dpaa2_fl_set_len(in_fle, buflen + req->nbytes); + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, state->ctx_dma); + dpaa2_fl_set_len(out_fle, digestsize); + + req_ctx->flc = &ctx->flc[DIGEST]; + req_ctx->flc_dma = ctx->flc_dma[DIGEST]; + req_ctx->cbk = ahash_done; + req_ctx->ctx = &req->base; + req_ctx->edesc = edesc; + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); + if (ret != -EINPROGRESS && + !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + goto unmap; + + return ret; +unmap: + ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); + qi_cache_free(edesc); + return ret; +} + +static int ahash_update_first(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct caam_request *req_ctx = &state->caam_req; + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + u8 *buf = state->buf; + int *buflen = &state->buflen; + int *next_buflen = &state->next_buflen; + int to_hash; + int src_nents, mapped_nents; + struct ahash_edesc *edesc; + int ret = 0; + + *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) - + 1); + to_hash = req->nbytes - *next_buflen; + + if (to_hash) { + struct dpaa2_sg_entry *sg_table; + int src_len = req->nbytes - *next_buflen; + + src_nents = sg_nents_for_len(req->src, src_len); + if (src_nents < 0) { + dev_err(ctx->dev, "Invalid number of src SG.\n"); + return src_nents; + } + + if (src_nents) { + mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, + DMA_TO_DEVICE); + if (!mapped_nents) { + dev_err(ctx->dev, "unable to map source for DMA\n"); + return -ENOMEM; + } + } else { + mapped_nents = 0; + } + + /* allocate space for base edesc and link tables */ + edesc = qi_cache_zalloc(flags); + if (!edesc) { + dma_unmap_sg(ctx->dev, req->src, src_nents, + DMA_TO_DEVICE); + return -ENOMEM; + } + + edesc->src_nents = src_nents; + sg_table = &edesc->sgt[0]; + + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); + dpaa2_fl_set_final(in_fle, true); + dpaa2_fl_set_len(in_fle, to_hash); + + if (mapped_nents > 1) { + int qm_sg_bytes; + + sg_to_qm_sg_last(req->src, src_len, sg_table, 0); + qm_sg_bytes = pad_sg_nents(mapped_nents) * + sizeof(*sg_table); + edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, + qm_sg_bytes, + DMA_TO_DEVICE); + if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { + dev_err(ctx->dev, "unable to map S/G table\n"); + ret = -ENOMEM; + goto unmap_ctx; + } + edesc->qm_sg_bytes = qm_sg_bytes; + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); + } else { + dpaa2_fl_set_format(in_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); + } + + state->ctx_dma_len = ctx->ctx_len; + state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, + ctx->ctx_len, DMA_FROM_DEVICE); + if (dma_mapping_error(ctx->dev, state->ctx_dma)) { + dev_err(ctx->dev, "unable to map ctx\n"); + state->ctx_dma = 0; + ret = -ENOMEM; + goto unmap_ctx; + } + + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, state->ctx_dma); + dpaa2_fl_set_len(out_fle, ctx->ctx_len); + + req_ctx->flc = &ctx->flc[UPDATE_FIRST]; + req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; + req_ctx->cbk = ahash_done_ctx_dst; + req_ctx->ctx = &req->base; + req_ctx->edesc = edesc; + + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); + if (ret != -EINPROGRESS && + !(ret == -EBUSY && req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG)) + goto unmap_ctx; + + state->update = ahash_update_ctx; + state->finup = ahash_finup_ctx; + state->final = ahash_final_ctx; + } else if (*next_buflen) { + state->update = ahash_update_no_ctx; + state->finup = ahash_finup_no_ctx; + state->final = ahash_final_no_ctx; + scatterwalk_map_and_copy(buf, req->src, 0, + req->nbytes, 0); + *buflen = *next_buflen; + + print_hex_dump_debug("buf@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, buf, + *buflen, 1); + } + + return ret; +unmap_ctx: + ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE); + qi_cache_free(edesc); + return ret; +} + +static int ahash_finup_first(struct ahash_request *req) +{ + return ahash_digest(req); +} + +static int ahash_init(struct ahash_request *req) +{ + struct caam_hash_state *state = ahash_request_ctx_dma(req); + + state->update = ahash_update_first; + state->finup = ahash_finup_first; + state->final = ahash_final_no_ctx; + + state->ctx_dma = 0; + state->ctx_dma_len = 0; + state->buf_dma = 0; + state->buflen = 0; + state->next_buflen = 0; + + return 0; +} + +static int ahash_update(struct ahash_request *req) +{ + struct caam_hash_state *state = ahash_request_ctx_dma(req); + + return state->update(req); +} + +static int ahash_finup(struct ahash_request *req) +{ + struct caam_hash_state *state = ahash_request_ctx_dma(req); + + return state->finup(req); +} + +static int ahash_final(struct ahash_request *req) +{ + struct caam_hash_state *state = ahash_request_ctx_dma(req); + + return state->final(req); +} + +static int ahash_export(struct ahash_request *req, void *out) +{ + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct caam_export_state *export = out; + u8 *buf = state->buf; + int len = state->buflen; + + memcpy(export->buf, buf, len); + memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx)); + export->buflen = len; + export->update = state->update; + export->final = state->final; + export->finup = state->finup; + + return 0; +} + +static int ahash_import(struct ahash_request *req, const void *in) +{ + struct caam_hash_state *state = ahash_request_ctx_dma(req); + const struct caam_export_state *export = in; + + memset(state, 0, sizeof(*state)); + memcpy(state->buf, export->buf, export->buflen); + memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx)); + state->buflen = export->buflen; + state->update = export->update; + state->final = export->final; + state->finup = export->finup; + + return 0; +} + +struct caam_hash_template { + char name[CRYPTO_MAX_ALG_NAME]; + char driver_name[CRYPTO_MAX_ALG_NAME]; + char hmac_name[CRYPTO_MAX_ALG_NAME]; + char hmac_driver_name[CRYPTO_MAX_ALG_NAME]; + unsigned int blocksize; + struct ahash_alg template_ahash; + u32 alg_type; +}; + +/* ahash descriptors */ +static struct caam_hash_template driver_hash[] = { + { + .name = "sha1", + .driver_name = "sha1-caam-qi2", + .hmac_name = "hmac(sha1)", + .hmac_driver_name = "hmac-sha1-caam-qi2", + .blocksize = SHA1_BLOCK_SIZE, + .template_ahash = { + .init = ahash_init, + .update = ahash_update, + .final = ahash_final, + .finup = ahash_finup, + .digest = ahash_digest, + .export = ahash_export, + .import = ahash_import, + .setkey = ahash_setkey, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct caam_export_state), + }, + }, + .alg_type = OP_ALG_ALGSEL_SHA1, + }, { + .name = "sha224", + .driver_name = "sha224-caam-qi2", + .hmac_name = "hmac(sha224)", + .hmac_driver_name = "hmac-sha224-caam-qi2", + .blocksize = SHA224_BLOCK_SIZE, + .template_ahash = { + .init = ahash_init, + .update = ahash_update, + .final = ahash_final, + .finup = ahash_finup, + .digest = ahash_digest, + .export = ahash_export, + .import = ahash_import, + .setkey = ahash_setkey, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct caam_export_state), + }, + }, + .alg_type = OP_ALG_ALGSEL_SHA224, + }, { + .name = "sha256", + .driver_name = "sha256-caam-qi2", + .hmac_name = "hmac(sha256)", + .hmac_driver_name = "hmac-sha256-caam-qi2", + .blocksize = SHA256_BLOCK_SIZE, + .template_ahash = { + .init = ahash_init, + .update = ahash_update, + .final = ahash_final, + .finup = ahash_finup, + .digest = ahash_digest, + .export = ahash_export, + .import = ahash_import, + .setkey = ahash_setkey, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct caam_export_state), + }, + }, + .alg_type = OP_ALG_ALGSEL_SHA256, + }, { + .name = "sha384", + .driver_name = "sha384-caam-qi2", + .hmac_name = "hmac(sha384)", + .hmac_driver_name = "hmac-sha384-caam-qi2", + .blocksize = SHA384_BLOCK_SIZE, + .template_ahash = { + .init = ahash_init, + .update = ahash_update, + .final = ahash_final, + .finup = ahash_finup, + .digest = ahash_digest, + .export = ahash_export, + .import = ahash_import, + .setkey = ahash_setkey, + .halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct caam_export_state), + }, + }, + .alg_type = OP_ALG_ALGSEL_SHA384, + }, { + .name = "sha512", + .driver_name = "sha512-caam-qi2", + .hmac_name = "hmac(sha512)", + .hmac_driver_name = "hmac-sha512-caam-qi2", + .blocksize = SHA512_BLOCK_SIZE, + .template_ahash = { + .init = ahash_init, + .update = ahash_update, + .final = ahash_final, + .finup = ahash_finup, + .digest = ahash_digest, + .export = ahash_export, + .import = ahash_import, + .setkey = ahash_setkey, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct caam_export_state), + }, + }, + .alg_type = OP_ALG_ALGSEL_SHA512, + }, { + .name = "md5", + .driver_name = "md5-caam-qi2", + .hmac_name = "hmac(md5)", + .hmac_driver_name = "hmac-md5-caam-qi2", + .blocksize = MD5_BLOCK_WORDS * 4, + .template_ahash = { + .init = ahash_init, + .update = ahash_update, + .final = ahash_final, + .finup = ahash_finup, + .digest = ahash_digest, + .export = ahash_export, + .import = ahash_import, + .setkey = ahash_setkey, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct caam_export_state), + }, + }, + .alg_type = OP_ALG_ALGSEL_MD5, + } +}; + +struct caam_hash_alg { + struct list_head entry; + struct device *dev; + int alg_type; + bool is_hmac; + struct ahash_alg ahash_alg; +}; + +static int caam_hash_cra_init(struct crypto_tfm *tfm) +{ + struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); + struct crypto_alg *base = tfm->__crt_alg; + struct hash_alg_common *halg = + container_of(base, struct hash_alg_common, base); + struct ahash_alg *alg = + container_of(halg, struct ahash_alg, halg); + struct caam_hash_alg *caam_hash = + container_of(alg, struct caam_hash_alg, ahash_alg); + struct caam_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm); + /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */ + static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE, + HASH_MSG_LEN + SHA1_DIGEST_SIZE, + HASH_MSG_LEN + 32, + HASH_MSG_LEN + SHA256_DIGEST_SIZE, + HASH_MSG_LEN + 64, + HASH_MSG_LEN + SHA512_DIGEST_SIZE }; + dma_addr_t dma_addr; + int i; + + ctx->dev = caam_hash->dev; + + if (caam_hash->is_hmac) { + ctx->adata.key_dma = dma_map_single_attrs(ctx->dev, ctx->key, + ARRAY_SIZE(ctx->key), + DMA_TO_DEVICE, + DMA_ATTR_SKIP_CPU_SYNC); + if (dma_mapping_error(ctx->dev, ctx->adata.key_dma)) { + dev_err(ctx->dev, "unable to map key\n"); + return -ENOMEM; + } + } + + dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc), + DMA_BIDIRECTIONAL, + DMA_ATTR_SKIP_CPU_SYNC); + if (dma_mapping_error(ctx->dev, dma_addr)) { + dev_err(ctx->dev, "unable to map shared descriptors\n"); + if (ctx->adata.key_dma) + dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma, + ARRAY_SIZE(ctx->key), + DMA_TO_DEVICE, + DMA_ATTR_SKIP_CPU_SYNC); + return -ENOMEM; + } + + for (i = 0; i < HASH_NUM_OP; i++) + ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); + + /* copy descriptor header template value */ + ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; + + ctx->ctx_len = runninglen[(ctx->adata.algtype & + OP_ALG_ALGSEL_SUBMASK) >> + OP_ALG_ALGSEL_SHIFT]; + + crypto_ahash_set_reqsize_dma(ahash, sizeof(struct caam_hash_state)); + + /* + * For keyed hash algorithms shared descriptors + * will be created later in setkey() callback + */ + return caam_hash->is_hmac ? 0 : ahash_set_sh_desc(ahash); +} + +static void caam_hash_cra_exit(struct crypto_tfm *tfm) +{ + struct caam_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm); + + dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc), + DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC); + if (ctx->adata.key_dma) + dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma, + ARRAY_SIZE(ctx->key), DMA_TO_DEVICE, + DMA_ATTR_SKIP_CPU_SYNC); +} + +static struct caam_hash_alg *caam_hash_alloc(struct device *dev, + struct caam_hash_template *template, bool keyed) +{ + struct caam_hash_alg *t_alg; + struct ahash_alg *halg; + struct crypto_alg *alg; + + t_alg = kzalloc_obj(*t_alg); + if (!t_alg) + return ERR_PTR(-ENOMEM); + + t_alg->ahash_alg = template->template_ahash; + halg = &t_alg->ahash_alg; + alg = &halg->halg.base; + + if (keyed) { + strscpy(alg->cra_name, template->hmac_name); + strscpy(alg->cra_driver_name, template->hmac_driver_name); + t_alg->is_hmac = true; + } else { + strscpy(alg->cra_name, template->name); + strscpy(alg->cra_driver_name, template->driver_name); + t_alg->ahash_alg.setkey = NULL; + t_alg->is_hmac = false; + } + alg->cra_module = THIS_MODULE; + alg->cra_init = caam_hash_cra_init; + alg->cra_exit = caam_hash_cra_exit; + alg->cra_ctxsize = sizeof(struct caam_hash_ctx) + crypto_dma_padding(); + alg->cra_priority = CAAM_CRA_PRIORITY; + alg->cra_blocksize = template->blocksize; + alg->cra_alignmask = 0; + alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY; + + t_alg->alg_type = template->alg_type; + t_alg->dev = dev; + + return t_alg; +} + +static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx) +{ + struct dpaa2_caam_priv_per_cpu *ppriv; + + ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx); + napi_schedule_irqoff(&ppriv->napi); +} + +static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv) +{ + struct device *dev = priv->dev; + struct dpaa2_io_notification_ctx *nctx; + struct dpaa2_caam_priv_per_cpu *ppriv; + int err, i = 0, cpu; + + for_each_online_cpu(cpu) { + ppriv = per_cpu_ptr(priv->ppriv, cpu); + ppriv->priv = priv; + nctx = &ppriv->nctx; + nctx->is_cdan = 0; + nctx->id = ppriv->rsp_fqid; + nctx->desired_cpu = cpu; + nctx->cb = dpaa2_caam_fqdan_cb; + + /* Register notification callbacks */ + ppriv->dpio = dpaa2_io_service_select(cpu); + err = dpaa2_io_service_register(ppriv->dpio, nctx, dev); + if (unlikely(err)) { + dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu); + nctx->cb = NULL; + /* + * If no affine DPIO for this core, there's probably + * none available for next cores either. Signal we want + * to retry later, in case the DPIO devices weren't + * probed yet. + */ + err = -EPROBE_DEFER; + goto err; + } + + ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE, + dev); + if (unlikely(!ppriv->store)) { + dev_err(dev, "dpaa2_io_store_create() failed\n"); + err = -ENOMEM; + goto err; + } + + if (++i == priv->num_pairs) + break; + } + + return 0; + +err: + for_each_online_cpu(cpu) { + ppriv = per_cpu_ptr(priv->ppriv, cpu); + if (!ppriv->nctx.cb) + break; + dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, dev); + } + + for_each_online_cpu(cpu) { + ppriv = per_cpu_ptr(priv->ppriv, cpu); + if (!ppriv->store) + break; + dpaa2_io_store_destroy(ppriv->store); + } + + return err; +} + +static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv) +{ + struct dpaa2_caam_priv_per_cpu *ppriv; + int i = 0, cpu; + + for_each_online_cpu(cpu) { + ppriv = per_cpu_ptr(priv->ppriv, cpu); + dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, + priv->dev); + dpaa2_io_store_destroy(ppriv->store); + + if (++i == priv->num_pairs) + return; + } +} + +static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv) +{ + struct dpseci_rx_queue_cfg rx_queue_cfg; + struct device *dev = priv->dev; + struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); + struct dpaa2_caam_priv_per_cpu *ppriv; + int err = 0, i = 0, cpu; + + /* Configure Rx queues */ + for_each_online_cpu(cpu) { + ppriv = per_cpu_ptr(priv->ppriv, cpu); + + rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST | + DPSECI_QUEUE_OPT_USER_CTX; + rx_queue_cfg.order_preservation_en = 0; + rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO; + rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id; + /* + * Rx priority (WQ) doesn't really matter, since we use + * pull mode, i.e. volatile dequeues from specific FQs + */ + rx_queue_cfg.dest_cfg.priority = 0; + rx_queue_cfg.user_ctx = ppriv->nctx.qman64; + + err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, + &rx_queue_cfg); + if (err) { + dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n", + err); + return err; + } + + if (++i == priv->num_pairs) + break; + } + + return err; +} + +static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv) +{ + struct device *dev = priv->dev; + + if (!priv->cscn_mem) + return; + + dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); + kfree(priv->cscn_mem); +} + +static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv) +{ + struct device *dev = priv->dev; + struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); + struct dpaa2_caam_priv_per_cpu *ppriv; + int i, err; + + if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) { + err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle); + if (err) + dev_err(dev, "dpseci_reset() failed\n"); + } + + for_each_cpu(i, priv->clean_mask) { + ppriv = per_cpu_ptr(priv->ppriv, i); + free_netdev(ppriv->net_dev); + } + free_cpumask_var(priv->clean_mask); + + dpaa2_dpseci_congestion_free(priv); + dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); +} + +static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv, + const struct dpaa2_fd *fd) +{ + struct caam_request *req; + u32 fd_err; + + if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) { + dev_err(priv->dev, "Only Frame List FD format is supported!\n"); + return; + } + + fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK; + if (unlikely(fd_err)) + dev_err_ratelimited(priv->dev, "FD error: %08x\n", fd_err); + + /* + * FD[ADDR] is guaranteed to be valid, irrespective of errors reported + * in FD[ERR] or FD[FRC]. + */ + req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd)); + dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt), + DMA_BIDIRECTIONAL); + req->cbk(req->ctx, dpaa2_fd_get_frc(fd)); +} + +static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv) +{ + int err; + + /* Retry while portal is busy */ + do { + err = dpaa2_io_service_pull_fq(ppriv->dpio, ppriv->rsp_fqid, + ppriv->store); + } while (err == -EBUSY); + + if (unlikely(err)) + dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err); + + return err; +} + +static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv) +{ + struct dpaa2_dq *dq; + int cleaned = 0, is_last; + + do { + dq = dpaa2_io_store_next(ppriv->store, &is_last); + if (unlikely(!dq)) { + if (unlikely(!is_last)) { + dev_dbg(ppriv->priv->dev, + "FQ %d returned no valid frames\n", + ppriv->rsp_fqid); + /* + * MUST retry until we get some sort of + * valid response token (be it "empty dequeue" + * or a valid frame). + */ + continue; + } + break; + } + + /* Process FD */ + dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq)); + cleaned++; + } while (!is_last); + + return cleaned; +} + +static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget) +{ + struct dpaa2_caam_priv_per_cpu *ppriv; + struct dpaa2_caam_priv *priv; + int err, cleaned = 0, store_cleaned; + + ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi); + priv = ppriv->priv; + + if (unlikely(dpaa2_caam_pull_fq(ppriv))) + return 0; + + do { + store_cleaned = dpaa2_caam_store_consume(ppriv); + cleaned += store_cleaned; + + if (store_cleaned == 0 || + cleaned > budget - DPAA2_CAAM_STORE_SIZE) + break; + + /* Try to dequeue some more */ + err = dpaa2_caam_pull_fq(ppriv); + if (unlikely(err)) + break; + } while (1); + + if (cleaned < budget) { + napi_complete_done(napi, cleaned); + err = dpaa2_io_service_rearm(ppriv->dpio, &ppriv->nctx); + if (unlikely(err)) + dev_err(priv->dev, "Notification rearm failed: %d\n", + err); + } + + return cleaned; +} + +static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv, + u16 token) +{ + struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 }; + struct device *dev = priv->dev; + unsigned int alignmask; + int err; + + /* + * Congestion group feature supported starting with DPSECI API v5.1 + * and only when object has been created with this capability. + */ + if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) || + !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG)) + return 0; + + alignmask = DPAA2_CSCN_ALIGN - 1; + alignmask |= dma_get_cache_alignment() - 1; + priv->cscn_mem = kzalloc(ALIGN(DPAA2_CSCN_SIZE, alignmask + 1), + GFP_KERNEL); + if (!priv->cscn_mem) + return -ENOMEM; + + priv->cscn_dma = dma_map_single(dev, priv->cscn_mem, + DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); + if (dma_mapping_error(dev, priv->cscn_dma)) { + dev_err(dev, "Error mapping CSCN memory area\n"); + err = -ENOMEM; + goto err_dma_map; + } + + cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES; + cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH; + cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH; + cong_notif_cfg.message_ctx = (uintptr_t)priv; + cong_notif_cfg.message_iova = priv->cscn_dma; + cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER | + DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT | + DPSECI_CGN_MODE_COHERENT_WRITE; + + err = dpseci_set_congestion_notification(priv->mc_io, 0, token, + &cong_notif_cfg); + if (err) { + dev_err(dev, "dpseci_set_congestion_notification failed\n"); + goto err_set_cong; + } + + return 0; + +err_set_cong: + dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); +err_dma_map: + kfree(priv->cscn_mem); + + return err; +} + +static void free_dpaa2_pcpu_netdev(struct dpaa2_caam_priv *priv, const cpumask_t *cpus) +{ + struct dpaa2_caam_priv_per_cpu *ppriv; + int i; + + for_each_cpu(i, cpus) { + ppriv = per_cpu_ptr(priv->ppriv, i); + free_netdev(ppriv->net_dev); + } +} + +static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) +{ + struct device *dev = &ls_dev->dev; + struct dpaa2_caam_priv *priv; + struct dpaa2_caam_priv_per_cpu *ppriv; + int err, cpu; + u8 i; + + err = -ENOMEM; + priv = dev_get_drvdata(dev); + + if (!zalloc_cpumask_var(&priv->clean_mask, GFP_KERNEL)) + goto err_cpumask; + + priv->dev = dev; + priv->dpsec_id = ls_dev->obj_desc.id; + + /* Get a handle for the DPSECI this interface is associate with */ + err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle); + if (err) { + dev_err(dev, "dpseci_open() failed: %d\n", err); + goto err_open; + } + + err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver, + &priv->minor_ver); + if (err) { + dev_err(dev, "dpseci_get_api_version() failed\n"); + goto err_get_vers; + } + + dev_info(dev, "dpseci v%d.%d\n", priv->major_ver, priv->minor_ver); + + if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) { + err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle); + if (err) { + dev_err(dev, "dpseci_reset() failed\n"); + goto err_get_vers; + } + } + + err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle, + &priv->dpseci_attr); + if (err) { + dev_err(dev, "dpseci_get_attributes() failed\n"); + goto err_get_vers; + } + + err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle, + &priv->sec_attr); + if (err) { + dev_err(dev, "dpseci_get_sec_attr() failed\n"); + goto err_get_vers; + } + + err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle); + if (err) { + dev_err(dev, "setup_congestion() failed\n"); + goto err_get_vers; + } + + priv->num_pairs = min(priv->dpseci_attr.num_rx_queues, + priv->dpseci_attr.num_tx_queues); + if (priv->num_pairs > num_online_cpus()) { + dev_warn(dev, "%d queues won't be used\n", + priv->num_pairs - num_online_cpus()); + priv->num_pairs = num_online_cpus(); + } + + for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) { + err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, + &priv->rx_queue_attr[i]); + if (err) { + dev_err(dev, "dpseci_get_rx_queue() failed\n"); + goto err_get_rx_queue; + } + } + + for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) { + err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, + &priv->tx_queue_attr[i]); + if (err) { + dev_err(dev, "dpseci_get_tx_queue() failed\n"); + goto err_get_rx_queue; + } + } + + i = 0; + for_each_online_cpu(cpu) { + u8 j; + + j = i % priv->num_pairs; + + ppriv = per_cpu_ptr(priv->ppriv, cpu); + ppriv->req_fqid = priv->tx_queue_attr[j].fqid; + + /* + * Allow all cores to enqueue, while only some of them + * will take part in dequeuing. + */ + if (++i > priv->num_pairs) + continue; + + ppriv->rsp_fqid = priv->rx_queue_attr[j].fqid; + ppriv->prio = j; + + dev_dbg(dev, "pair %d: rx queue %d, tx queue %d\n", j, + priv->rx_queue_attr[j].fqid, + priv->tx_queue_attr[j].fqid); + + ppriv->net_dev = alloc_netdev_dummy(0); + if (!ppriv->net_dev) { + err = -ENOMEM; + goto err_alloc_netdev; + } + cpumask_set_cpu(cpu, priv->clean_mask); + ppriv->net_dev->dev = *dev; + + netif_napi_add_tx_weight(ppriv->net_dev, &ppriv->napi, + dpaa2_dpseci_poll, + DPAA2_CAAM_NAPI_WEIGHT); + } + + return 0; + +err_alloc_netdev: + free_dpaa2_pcpu_netdev(priv, priv->clean_mask); +err_get_rx_queue: + dpaa2_dpseci_congestion_free(priv); +err_get_vers: + dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); +err_open: + free_cpumask_var(priv->clean_mask); +err_cpumask: + return err; +} + +static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv) +{ + struct device *dev = priv->dev; + struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); + struct dpaa2_caam_priv_per_cpu *ppriv; + int i; + + for (i = 0; i < priv->num_pairs; i++) { + ppriv = per_cpu_ptr(priv->ppriv, i); + napi_enable(&ppriv->napi); + } + + return dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle); +} + +static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv) +{ + struct device *dev = priv->dev; + struct dpaa2_caam_priv_per_cpu *ppriv; + struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); + int i, err = 0, enabled; + + err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle); + if (err) { + dev_err(dev, "dpseci_disable() failed\n"); + return err; + } + + err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled); + if (err) { + dev_err(dev, "dpseci_is_enabled() failed\n"); + return err; + } + + dev_dbg(dev, "disable: %s\n", str_false_true(enabled)); + + for (i = 0; i < priv->num_pairs; i++) { + ppriv = per_cpu_ptr(priv->ppriv, i); + napi_disable(&ppriv->napi); + netif_napi_del(&ppriv->napi); + } + + return 0; +} + +static struct list_head hash_list; + +static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev) +{ + struct device *dev; + struct dpaa2_caam_priv *priv; + int i, err = 0; + bool registered = false; + + /* + * There is no way to get CAAM endianness - there is no direct register + * space access and MC f/w does not provide this attribute. + * All DPAA2-based SoCs have little endian CAAM, thus hard-code this + * property. + */ + caam_little_end = true; + + caam_imx = false; + + dev = &dpseci_dev->dev; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + dev_set_drvdata(dev, priv); + + priv->domain = iommu_get_domain_for_dev(dev); + + qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE, + 0, 0, NULL); + if (!qi_cache) { + dev_err(dev, "Can't allocate SEC cache\n"); + return -ENOMEM; + } + + err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49)); + if (err) { + dev_err(dev, "dma_set_mask_and_coherent() failed\n"); + goto err_dma_mask; + } + + /* Obtain a MC portal */ + err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io); + if (err) { + if (err == -ENXIO) + err = -EPROBE_DEFER; + else + dev_err(dev, "MC portal allocation failed\n"); + + goto err_dma_mask; + } + + priv->ppriv = alloc_percpu(*priv->ppriv); + if (!priv->ppriv) { + dev_err(dev, "alloc_percpu() failed\n"); + err = -ENOMEM; + goto err_alloc_ppriv; + } + + /* DPSECI initialization */ + err = dpaa2_dpseci_setup(dpseci_dev); + if (err) { + dev_err(dev, "dpaa2_dpseci_setup() failed\n"); + goto err_dpseci_setup; + } + + /* DPIO */ + err = dpaa2_dpseci_dpio_setup(priv); + if (err) { + dev_err_probe(dev, err, "dpaa2_dpseci_dpio_setup() failed\n"); + goto err_dpio_setup; + } + + /* DPSECI binding to DPIO */ + err = dpaa2_dpseci_bind(priv); + if (err) { + dev_err(dev, "dpaa2_dpseci_bind() failed\n"); + goto err_bind; + } + + /* DPSECI enable */ + err = dpaa2_dpseci_enable(priv); + if (err) { + dev_err(dev, "dpaa2_dpseci_enable() failed\n"); + goto err_bind; + } + + dpaa2_dpseci_debugfs_init(priv); + + /* register crypto algorithms the device supports */ + for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { + struct caam_skcipher_alg *t_alg = driver_algs + i; + u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK; + + /* Skip DES algorithms if not supported by device */ + if (!priv->sec_attr.des_acc_num && + (alg_sel == OP_ALG_ALGSEL_3DES || + alg_sel == OP_ALG_ALGSEL_DES)) + continue; + + /* Skip AES algorithms if not supported by device */ + if (!priv->sec_attr.aes_acc_num && + alg_sel == OP_ALG_ALGSEL_AES) + continue; + + /* Skip CHACHA20 algorithms if not supported by device */ + if (alg_sel == OP_ALG_ALGSEL_CHACHA20 && + !priv->sec_attr.ccha_acc_num) + continue; + + t_alg->caam.dev = dev; + caam_skcipher_alg_init(t_alg); + + err = crypto_register_skcipher(&t_alg->skcipher); + if (err) { + dev_warn(dev, "%s alg registration failed: %d\n", + t_alg->skcipher.base.cra_driver_name, err); + continue; + } + + t_alg->registered = true; + registered = true; + } + + for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { + struct caam_aead_alg *t_alg = driver_aeads + i; + u32 c1_alg_sel = t_alg->caam.class1_alg_type & + OP_ALG_ALGSEL_MASK; + u32 c2_alg_sel = t_alg->caam.class2_alg_type & + OP_ALG_ALGSEL_MASK; + + /* Skip DES algorithms if not supported by device */ + if (!priv->sec_attr.des_acc_num && + (c1_alg_sel == OP_ALG_ALGSEL_3DES || + c1_alg_sel == OP_ALG_ALGSEL_DES)) + continue; + + /* Skip AES algorithms if not supported by device */ + if (!priv->sec_attr.aes_acc_num && + c1_alg_sel == OP_ALG_ALGSEL_AES) + continue; + + /* Skip CHACHA20 algorithms if not supported by device */ + if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 && + !priv->sec_attr.ccha_acc_num) + continue; + + /* Skip POLY1305 algorithms if not supported by device */ + if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 && + !priv->sec_attr.ptha_acc_num) + continue; + + /* + * Skip algorithms requiring message digests + * if MD not supported by device. + */ + if ((c2_alg_sel & ~OP_ALG_ALGSEL_SUBMASK) == 0x40 && + !priv->sec_attr.md_acc_num) + continue; + + t_alg->caam.dev = dev; + caam_aead_alg_init(t_alg); + + err = crypto_register_aead(&t_alg->aead); + if (err) { + dev_warn(dev, "%s alg registration failed: %d\n", + t_alg->aead.base.cra_driver_name, err); + continue; + } + + t_alg->registered = true; + registered = true; + } + if (registered) + dev_info(dev, "algorithms registered in /proc/crypto\n"); + + /* register hash algorithms the device supports */ + INIT_LIST_HEAD(&hash_list); + + /* + * Skip registration of any hashing algorithms if MD block + * is not present. + */ + if (!priv->sec_attr.md_acc_num) + return 0; + + for (i = 0; i < ARRAY_SIZE(driver_hash); i++) { + struct caam_hash_alg *t_alg; + struct caam_hash_template *alg = driver_hash + i; + + /* register hmac version */ + t_alg = caam_hash_alloc(dev, alg, true); + if (IS_ERR(t_alg)) { + err = PTR_ERR(t_alg); + dev_warn(dev, "%s hash alg allocation failed: %d\n", + alg->hmac_driver_name, err); + continue; + } + + err = crypto_register_ahash(&t_alg->ahash_alg); + if (err) { + dev_warn(dev, "%s alg registration failed: %d\n", + t_alg->ahash_alg.halg.base.cra_driver_name, + err); + kfree(t_alg); + } else { + list_add_tail(&t_alg->entry, &hash_list); + } + + /* register unkeyed version */ + t_alg = caam_hash_alloc(dev, alg, false); + if (IS_ERR(t_alg)) { + err = PTR_ERR(t_alg); + dev_warn(dev, "%s alg allocation failed: %d\n", + alg->driver_name, err); + continue; + } + + err = crypto_register_ahash(&t_alg->ahash_alg); + if (err) { + dev_warn(dev, "%s alg registration failed: %d\n", + t_alg->ahash_alg.halg.base.cra_driver_name, + err); + kfree(t_alg); + } else { + list_add_tail(&t_alg->entry, &hash_list); + } + } + if (!list_empty(&hash_list)) + dev_info(dev, "hash algorithms registered in /proc/crypto\n"); + + return err; + +err_bind: + dpaa2_dpseci_dpio_free(priv); +err_dpio_setup: + dpaa2_dpseci_free(priv); +err_dpseci_setup: + free_percpu(priv->ppriv); +err_alloc_ppriv: + fsl_mc_portal_free(priv->mc_io); +err_dma_mask: + kmem_cache_destroy(qi_cache); + + return err; +} + +static void __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev) +{ + struct device *dev; + struct dpaa2_caam_priv *priv; + int i; + + dev = &ls_dev->dev; + priv = dev_get_drvdata(dev); + + dpaa2_dpseci_debugfs_exit(priv); + + for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { + struct caam_aead_alg *t_alg = driver_aeads + i; + + if (t_alg->registered) + crypto_unregister_aead(&t_alg->aead); + } + + for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { + struct caam_skcipher_alg *t_alg = driver_algs + i; + + if (t_alg->registered) + crypto_unregister_skcipher(&t_alg->skcipher); + } + + if (hash_list.next) { + struct caam_hash_alg *t_hash_alg, *p; + + list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) { + crypto_unregister_ahash(&t_hash_alg->ahash_alg); + list_del(&t_hash_alg->entry); + kfree(t_hash_alg); + } + } + + dpaa2_dpseci_disable(priv); + dpaa2_dpseci_dpio_free(priv); + dpaa2_dpseci_free(priv); + free_percpu(priv->ppriv); + fsl_mc_portal_free(priv->mc_io); + kmem_cache_destroy(qi_cache); +} + +int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req) +{ + struct dpaa2_fd fd; + struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); + struct dpaa2_caam_priv_per_cpu *ppriv; + int err = 0, i; + + if (IS_ERR(req)) + return PTR_ERR(req); + + if (priv->cscn_mem) { + dma_sync_single_for_cpu(priv->dev, priv->cscn_dma, + DPAA2_CSCN_SIZE, + DMA_FROM_DEVICE); + if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem))) { + dev_dbg_ratelimited(dev, "Dropping request\n"); + return -EBUSY; + } + } + + dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma); + + req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt), + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, req->fd_flt_dma)) { + dev_err(dev, "DMA mapping error for QI enqueue request\n"); + goto err_out; + } + + memset(&fd, 0, sizeof(fd)); + dpaa2_fd_set_format(&fd, dpaa2_fd_list); + dpaa2_fd_set_addr(&fd, req->fd_flt_dma); + dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1])); + dpaa2_fd_set_flc(&fd, req->flc_dma); + + ppriv = raw_cpu_ptr(priv->ppriv); + for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) { + err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid, + &fd); + if (err != -EBUSY) + break; + + cpu_relax(); + } + + if (unlikely(err)) { + dev_err_ratelimited(dev, "Error enqueuing frame: %d\n", err); + goto err_out; + } + + return -EINPROGRESS; + +err_out: + dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt), + DMA_BIDIRECTIONAL); + return -EIO; +} +EXPORT_SYMBOL(dpaa2_caam_enqueue); + +static const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = { + { + .vendor = FSL_MC_VENDOR_FREESCALE, + .obj_type = "dpseci", + }, + { .vendor = 0x0 } +}; +MODULE_DEVICE_TABLE(fslmc, dpaa2_caam_match_id_table); + +static struct fsl_mc_driver dpaa2_caam_driver = { + .driver = { + .name = KBUILD_MODNAME, + .owner = THIS_MODULE, + }, + .probe = dpaa2_caam_probe, + .remove = dpaa2_caam_remove, + .match_id_table = dpaa2_caam_match_id_table +}; + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Freescale Semiconductor, Inc"); +MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver"); + +module_fsl_mc_driver(dpaa2_caam_driver); diff --git a/drivers/crypto/caam/caamalg_qi2.h b/drivers/crypto/caam/caamalg_qi2.h new file mode 100644 index 000000000..8e65b4b28 --- /dev/null +++ b/drivers/crypto/caam/caamalg_qi2.h @@ -0,0 +1,200 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */ +/* + * Copyright 2015-2016 Freescale Semiconductor Inc. + * Copyright 2017-2018 NXP + */ + +#ifndef _CAAMALG_QI2_H_ +#define _CAAMALG_QI2_H_ + +#include +#include +#include +#include +#include +#include +#include "dpseci.h" +#include "desc_constr.h" + +#define DPAA2_CAAM_STORE_SIZE 16 +/* NAPI weight *must* be a multiple of the store size. */ +#define DPAA2_CAAM_NAPI_WEIGHT 512 + +/* The congestion entrance threshold was chosen so that on LS2088 + * we support the maximum throughput for the available memory + */ +#define DPAA2_SEC_CONG_ENTRY_THRESH (128 * 1024 * 1024) +#define DPAA2_SEC_CONG_EXIT_THRESH (DPAA2_SEC_CONG_ENTRY_THRESH * 9 / 10) + +/** + * dpaa2_caam_priv - driver private data + * @dpseci_id: DPSECI object unique ID + * @major_ver: DPSECI major version + * @minor_ver: DPSECI minor version + * @dpseci_attr: DPSECI attributes + * @sec_attr: SEC engine attributes + * @rx_queue_attr: array of Rx queue attributes + * @tx_queue_attr: array of Tx queue attributes + * @cscn_mem: pointer to memory region containing the congestion SCN + * it's size is larger than to accommodate alignment + * @cscn_dma: dma address used by the QMAN to write CSCN messages + * @dev: device associated with the DPSECI object + * @mc_io: pointer to MC portal's I/O object + * @domain: IOMMU domain + * @ppriv: per CPU pointers to privata data + * @clean_mask: CPU mask of CPUs that have allocated netdevs + */ +struct dpaa2_caam_priv { + int dpsec_id; + + u16 major_ver; + u16 minor_ver; + + struct dpseci_attr dpseci_attr; + struct dpseci_sec_attr sec_attr; + struct dpseci_rx_queue_attr rx_queue_attr[DPSECI_MAX_QUEUE_NUM]; + struct dpseci_tx_queue_attr tx_queue_attr[DPSECI_MAX_QUEUE_NUM]; + int num_pairs; + + /* congestion */ + void *cscn_mem; + dma_addr_t cscn_dma; + + struct device *dev; + struct fsl_mc_io *mc_io; + struct iommu_domain *domain; + + struct dpaa2_caam_priv_per_cpu __percpu *ppriv; + struct dentry *dfs_root; + cpumask_var_t clean_mask; +}; + +/** + * dpaa2_caam_priv_per_cpu - per CPU private data + * @napi: napi structure + * @net_dev: netdev used by napi + * @req_fqid: (virtual) request (Tx / enqueue) FQID + * @rsp_fqid: (virtual) response (Rx / dequeue) FQID + * @prio: internal queue number - index for dpaa2_caam_priv.*_queue_attr + * @nctx: notification context of response FQ + * @store: where dequeued frames are stored + * @priv: backpointer to dpaa2_caam_priv + * @dpio: portal used for data path operations + */ +struct dpaa2_caam_priv_per_cpu { + struct napi_struct napi; + struct net_device *net_dev; + int req_fqid; + int rsp_fqid; + int prio; + struct dpaa2_io_notification_ctx nctx; + struct dpaa2_io_store *store; + struct dpaa2_caam_priv *priv; + struct dpaa2_io *dpio; +}; + +/* Length of a single buffer in the QI driver memory cache */ +#define CAAM_QI_MEMCACHE_SIZE 512 + +/* + * aead_edesc - s/w-extended aead descriptor + * @src_nents: number of segments in input scatterlist + * @dst_nents: number of segments in output scatterlist + * @iv_dma: dma address of iv for checking continuity and link table + * @qm_sg_bytes: length of dma mapped h/w link table + * @qm_sg_dma: bus physical mapped address of h/w link table + * @assoclen: associated data length, in CAAM endianness + * @assoclen_dma: bus physical mapped address of req->assoclen + * @sgt: the h/w link table, followed by IV + */ +struct aead_edesc { + int src_nents; + int dst_nents; + dma_addr_t iv_dma; + int qm_sg_bytes; + dma_addr_t qm_sg_dma; + unsigned int assoclen; + dma_addr_t assoclen_dma; + struct dpaa2_sg_entry sgt[]; +}; + +/* + * skcipher_edesc - s/w-extended skcipher descriptor + * @src_nents: number of segments in input scatterlist + * @dst_nents: number of segments in output scatterlist + * @iv_dma: dma address of iv for checking continuity and link table + * @qm_sg_bytes: length of dma mapped qm_sg space + * @qm_sg_dma: I/O virtual address of h/w link table + * @sgt: the h/w link table, followed by IV + */ +struct skcipher_edesc { + int src_nents; + int dst_nents; + dma_addr_t iv_dma; + int qm_sg_bytes; + dma_addr_t qm_sg_dma; + struct dpaa2_sg_entry sgt[]; +}; + +/* + * ahash_edesc - s/w-extended ahash descriptor + * @qm_sg_dma: I/O virtual address of h/w link table + * @src_nents: number of segments in input scatterlist + * @qm_sg_bytes: length of dma mapped qm_sg space + * @sgt: pointer to h/w link table + */ +struct ahash_edesc { + dma_addr_t qm_sg_dma; + int src_nents; + int qm_sg_bytes; + struct dpaa2_sg_entry sgt[]; +}; + +/** + * caam_flc - Flow Context (FLC) + * @flc: Flow Context options + * @sh_desc: Shared Descriptor + */ +struct caam_flc { + u32 flc[16]; + u32 sh_desc[MAX_SDLEN]; +} __aligned(CRYPTO_DMA_ALIGN); + +enum optype { + ENCRYPT = 0, + DECRYPT, + NUM_OP +}; + +/** + * caam_request - the request structure the driver application should fill while + * submitting a job to driver. + * @fd_flt: Frame list table defining input and output + * fd_flt[0] - FLE pointing to output buffer + * fd_flt[1] - FLE pointing to input buffer + * @fd_flt_dma: DMA address for the frame list table + * @flc: Flow Context + * @flc_dma: I/O virtual address of Flow Context + * @cbk: Callback function to invoke when job is completed + * @ctx: arbit context attached with request by the application + * @edesc: extended descriptor; points to one of {skcipher,aead}_edesc + */ +struct caam_request { + struct dpaa2_fl_entry fd_flt[2] __aligned(CRYPTO_DMA_ALIGN); + dma_addr_t fd_flt_dma; + struct caam_flc *flc; + dma_addr_t flc_dma; + void (*cbk)(void *ctx, u32 err); + void *ctx; + void *edesc; + struct skcipher_request fallback_req; +}; + +/** + * dpaa2_caam_enqueue() - enqueue a crypto request + * @dev: device associated with the DPSECI object + * @req: pointer to caam_request + */ +int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req); + +#endif /* _CAAMALG_QI2_H_ */ diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c new file mode 100644 index 000000000..3cd718103 --- /dev/null +++ b/drivers/crypto/caam/caamhash.c @@ -0,0 +1,2029 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * caam - Freescale FSL CAAM support for ahash functions of crypto API + * + * Copyright 2011 Freescale Semiconductor, Inc. + * Copyright 2018-2019, 2023 NXP + * + * Based on caamalg.c crypto API driver. + * + * relationship of digest job descriptor or first job descriptor after init to + * shared descriptors: + * + * --------------- --------------- + * | JobDesc #1 |-------------------->| ShareDesc | + * | *(packet 1) | | (hashKey) | + * --------------- | (operation) | + * --------------- + * + * relationship of subsequent job descriptors to shared descriptors: + * + * --------------- --------------- + * | JobDesc #2 |-------------------->| ShareDesc | + * | *(packet 2) | |------------->| (hashKey) | + * --------------- | |-------->| (operation) | + * . | | | (load ctx2) | + * . | | --------------- + * --------------- | | + * | JobDesc #3 |------| | + * | *(packet 3) | | + * --------------- | + * . | + * . | + * --------------- | + * | JobDesc #4 |------------ + * | *(packet 4) | + * --------------- + * + * The SharedDesc never changes for a connection unless rekeyed, but + * each packet will likely be in a different place. So all we need + * to know to process the packet is where the input is, where the + * output goes, and what context we want to process with. Context is + * in the SharedDesc, packet references in the JobDesc. + * + * So, a job desc looks like: + * + * --------------------- + * | Header | + * | ShareDesc Pointer | + * | SEQ_OUT_PTR | + * | (output buffer) | + * | (output length) | + * | SEQ_IN_PTR | + * | (input buffer) | + * | (input length) | + * --------------------- + */ + +#include "compat.h" + +#include "regs.h" +#include "intern.h" +#include "desc_constr.h" +#include "jr.h" +#include "error.h" +#include "sg_sw_sec4.h" +#include "key_gen.h" +#include "caamhash_desc.h" +#include +#include +#include +#include +#include +#include +#include + +#define CAAM_CRA_PRIORITY 3000 + +/* max hash key is max split key size */ +#define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2) + +#define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE +#define CAAM_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE + +#define DESC_HASH_MAX_USED_BYTES (DESC_AHASH_FINAL_LEN + \ + CAAM_MAX_HASH_KEY_SIZE) +#define DESC_HASH_MAX_USED_LEN (DESC_HASH_MAX_USED_BYTES / CAAM_CMD_SZ) + +/* caam context sizes for hashes: running digest + 8 */ +#define HASH_MSG_LEN 8 +#define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE) + +static struct list_head hash_list; + +/* ahash per-session context */ +struct caam_hash_ctx { + u32 sh_desc_update[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; + u32 sh_desc_update_first[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; + u32 sh_desc_fin[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; + u32 sh_desc_digest[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; + u8 key[CAAM_MAX_HASH_KEY_SIZE] ____cacheline_aligned; + dma_addr_t sh_desc_update_dma ____cacheline_aligned; + dma_addr_t sh_desc_update_first_dma; + dma_addr_t sh_desc_fin_dma; + dma_addr_t sh_desc_digest_dma; + enum dma_data_direction dir; + enum dma_data_direction key_dir; + struct device *jrdev; + int ctx_len; + struct alginfo adata; +}; + +/* ahash state */ +struct caam_hash_state { + dma_addr_t buf_dma; + dma_addr_t ctx_dma; + int ctx_dma_len; + u8 buf[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; + int buflen; + int next_buflen; + u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned; + int (*update)(struct ahash_request *req) ____cacheline_aligned; + int (*final)(struct ahash_request *req); + int (*finup)(struct ahash_request *req); + struct ahash_edesc *edesc; + void (*ahash_op_done)(struct device *jrdev, u32 *desc, u32 err, + void *context); +}; + +struct caam_export_state { + u8 buf[CAAM_MAX_HASH_BLOCK_SIZE]; + u8 caam_ctx[MAX_CTX_LEN]; + int buflen; + int (*update)(struct ahash_request *req); + int (*final)(struct ahash_request *req); + int (*finup)(struct ahash_request *req); +}; + +static inline bool is_cmac_aes(u32 algtype) +{ + return (algtype & (OP_ALG_ALGSEL_MASK | OP_ALG_AAI_MASK)) == + (OP_ALG_ALGSEL_AES | OP_ALG_AAI_CMAC); +} +/* Common job descriptor seq in/out ptr routines */ + +/* Map state->caam_ctx, and append seq_out_ptr command that points to it */ +static inline int map_seq_out_ptr_ctx(u32 *desc, struct device *jrdev, + struct caam_hash_state *state, + int ctx_len) +{ + state->ctx_dma_len = ctx_len; + state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, + ctx_len, DMA_FROM_DEVICE); + if (dma_mapping_error(jrdev, state->ctx_dma)) { + dev_err(jrdev, "unable to map ctx\n"); + state->ctx_dma = 0; + return -ENOMEM; + } + + append_seq_out_ptr(desc, state->ctx_dma, ctx_len, 0); + + return 0; +} + +/* Map current buffer in state (if length > 0) and put it in link table */ +static inline int buf_map_to_sec4_sg(struct device *jrdev, + struct sec4_sg_entry *sec4_sg, + struct caam_hash_state *state) +{ + int buflen = state->buflen; + + if (!buflen) + return 0; + + state->buf_dma = dma_map_single(jrdev, state->buf, buflen, + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, state->buf_dma)) { + dev_err(jrdev, "unable to map buf\n"); + state->buf_dma = 0; + return -ENOMEM; + } + + dma_to_sec4_sg_one(sec4_sg, state->buf_dma, buflen, 0); + + return 0; +} + +/* Map state->caam_ctx, and add it to link table */ +static inline int ctx_map_to_sec4_sg(struct device *jrdev, + struct caam_hash_state *state, int ctx_len, + struct sec4_sg_entry *sec4_sg, u32 flag) +{ + state->ctx_dma_len = ctx_len; + state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, ctx_len, flag); + if (dma_mapping_error(jrdev, state->ctx_dma)) { + dev_err(jrdev, "unable to map ctx\n"); + state->ctx_dma = 0; + return -ENOMEM; + } + + dma_to_sec4_sg_one(sec4_sg, state->ctx_dma, ctx_len, 0); + + return 0; +} + +static int ahash_set_sh_desc(struct crypto_ahash *ahash) +{ + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + int digestsize = crypto_ahash_digestsize(ahash); + struct device *jrdev = ctx->jrdev; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); + u32 *desc; + + ctx->adata.key_virt = ctx->key; + + /* ahash_update shared descriptor */ + desc = ctx->sh_desc_update; + cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len, + ctx->ctx_len, true, ctrlpriv->era); + dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma, + desc_bytes(desc), ctx->dir); + + print_hex_dump_debug("ahash update shdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); + + /* ahash_update_first shared descriptor */ + desc = ctx->sh_desc_update_first; + cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len, + ctx->ctx_len, false, ctrlpriv->era); + dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma, + desc_bytes(desc), ctx->dir); + print_hex_dump_debug("ahash update first shdesc@"__stringify(__LINE__) + ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); + + /* ahash_final shared descriptor */ + desc = ctx->sh_desc_fin; + cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize, + ctx->ctx_len, true, ctrlpriv->era); + dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma, + desc_bytes(desc), ctx->dir); + + print_hex_dump_debug("ahash final shdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); + + /* ahash_digest shared descriptor */ + desc = ctx->sh_desc_digest; + cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize, + ctx->ctx_len, false, ctrlpriv->era); + dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma, + desc_bytes(desc), ctx->dir); + + print_hex_dump_debug("ahash digest shdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); + + return 0; +} + +static int axcbc_set_sh_desc(struct crypto_ahash *ahash) +{ + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + int digestsize = crypto_ahash_digestsize(ahash); + struct device *jrdev = ctx->jrdev; + u32 *desc; + + /* shared descriptor for ahash_update */ + desc = ctx->sh_desc_update; + cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_UPDATE, + ctx->ctx_len, ctx->ctx_len); + dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma, + desc_bytes(desc), ctx->dir); + print_hex_dump_debug("axcbc update shdesc@" __stringify(__LINE__)" : ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); + + /* shared descriptor for ahash_{final,finup} */ + desc = ctx->sh_desc_fin; + cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, + digestsize, ctx->ctx_len); + dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma, + desc_bytes(desc), ctx->dir); + print_hex_dump_debug("axcbc finup shdesc@" __stringify(__LINE__)" : ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); + + /* key is immediate data for INIT and INITFINAL states */ + ctx->adata.key_virt = ctx->key; + + /* shared descriptor for first invocation of ahash_update */ + desc = ctx->sh_desc_update_first; + cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len, + ctx->ctx_len); + dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma, + desc_bytes(desc), ctx->dir); + print_hex_dump_debug("axcbc update first shdesc@" __stringify(__LINE__) + " : ", DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); + + /* shared descriptor for ahash_digest */ + desc = ctx->sh_desc_digest; + cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, + digestsize, ctx->ctx_len); + dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma, + desc_bytes(desc), ctx->dir); + print_hex_dump_debug("axcbc digest shdesc@" __stringify(__LINE__)" : ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); + return 0; +} + +static int acmac_set_sh_desc(struct crypto_ahash *ahash) +{ + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + int digestsize = crypto_ahash_digestsize(ahash); + struct device *jrdev = ctx->jrdev; + u32 *desc; + + /* shared descriptor for ahash_update */ + desc = ctx->sh_desc_update; + cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_UPDATE, + ctx->ctx_len, ctx->ctx_len); + dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma, + desc_bytes(desc), ctx->dir); + print_hex_dump_debug("acmac update shdesc@" __stringify(__LINE__)" : ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); + + /* shared descriptor for ahash_{final,finup} */ + desc = ctx->sh_desc_fin; + cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, + digestsize, ctx->ctx_len); + dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma, + desc_bytes(desc), ctx->dir); + print_hex_dump_debug("acmac finup shdesc@" __stringify(__LINE__)" : ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); + + /* shared descriptor for first invocation of ahash_update */ + desc = ctx->sh_desc_update_first; + cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len, + ctx->ctx_len); + dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma, + desc_bytes(desc), ctx->dir); + print_hex_dump_debug("acmac update first shdesc@" __stringify(__LINE__) + " : ", DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); + + /* shared descriptor for ahash_digest */ + desc = ctx->sh_desc_digest; + cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, + digestsize, ctx->ctx_len); + dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma, + desc_bytes(desc), ctx->dir); + print_hex_dump_debug("acmac digest shdesc@" __stringify(__LINE__)" : ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); + + return 0; +} + +/* Digest hash size if it is too large */ +static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key, + u32 digestsize) +{ + struct device *jrdev = ctx->jrdev; + u32 *desc; + struct split_key_result result; + dma_addr_t key_dma; + int ret; + + desc = kmalloc(CAAM_CMD_SZ * 8 + CAAM_PTR_SZ * 2, GFP_KERNEL); + if (!desc) + return -ENOMEM; + + init_job_desc(desc, 0); + + key_dma = dma_map_single(jrdev, key, *keylen, DMA_BIDIRECTIONAL); + if (dma_mapping_error(jrdev, key_dma)) { + dev_err(jrdev, "unable to map key memory\n"); + kfree(desc); + return -ENOMEM; + } + + /* Job descriptor to perform unkeyed hash on key_in */ + append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT | + OP_ALG_AS_INITFINAL); + append_seq_in_ptr(desc, key_dma, *keylen, 0); + append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 | + FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG); + append_seq_out_ptr(desc, key_dma, digestsize, 0); + append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | + LDST_SRCDST_BYTE_CONTEXT); + + print_hex_dump_devel("key_in@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1); + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); + + result.err = 0; + init_completion(&result.completion); + + ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result); + if (ret == -EINPROGRESS) { + /* in progress */ + wait_for_completion(&result.completion); + ret = result.err; + + print_hex_dump_devel("digested key@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, + digestsize, 1); + } + dma_unmap_single(jrdev, key_dma, *keylen, DMA_BIDIRECTIONAL); + + *keylen = digestsize; + + kfree(desc); + + return ret; +} + +static int ahash_setkey(struct crypto_ahash *ahash, + const u8 *key, unsigned int keylen) +{ + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + struct device *jrdev = ctx->jrdev; + int blocksize = crypto_tfm_alg_blocksize(&ahash->base); + int digestsize = crypto_ahash_digestsize(ahash); + struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); + int ret; + u8 *hashed_key = NULL; + + dev_dbg(jrdev, "keylen %d\n", keylen); + + if (keylen > blocksize) { + unsigned int aligned_len = + ALIGN(keylen, dma_get_cache_alignment()); + + if (aligned_len < keylen) + return -EOVERFLOW; + + hashed_key = kmalloc(aligned_len, GFP_KERNEL); + if (!hashed_key) + return -ENOMEM; + memcpy(hashed_key, key, keylen); + ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize); + if (ret) + goto bad_free_key; + key = hashed_key; + } + + /* + * If DKP is supported, use it in the shared descriptor to generate + * the split key. + */ + if (ctrlpriv->era >= 6) { + ctx->adata.key_inline = true; + ctx->adata.keylen = keylen; + ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & + OP_ALG_ALGSEL_MASK); + + if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE) + goto bad_free_key; + + memcpy(ctx->key, key, keylen); + + /* + * In case |user key| > |derived key|, using DKP + * would result in invalid opcodes (last bytes of user key) in + * the resulting descriptor. Use DKP instead => both + * virtual and dma key addresses are needed. + */ + if (keylen > ctx->adata.keylen_pad) + dma_sync_single_for_device(ctx->jrdev, + ctx->adata.key_dma, + ctx->adata.keylen_pad, + DMA_TO_DEVICE); + } else { + ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key, + keylen, CAAM_MAX_HASH_KEY_SIZE); + if (ret) + goto bad_free_key; + } + + kfree(hashed_key); + return ahash_set_sh_desc(ahash); + bad_free_key: + kfree(hashed_key); + return -EINVAL; +} + +static int axcbc_setkey(struct crypto_ahash *ahash, const u8 *key, + unsigned int keylen) +{ + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + struct device *jrdev = ctx->jrdev; + + if (keylen != AES_KEYSIZE_128) + return -EINVAL; + + memcpy(ctx->key, key, keylen); + dma_sync_single_for_device(jrdev, ctx->adata.key_dma, keylen, + DMA_TO_DEVICE); + ctx->adata.keylen = keylen; + + print_hex_dump_devel("axcbc ctx.key@" __stringify(__LINE__)" : ", + DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, keylen, 1); + + return axcbc_set_sh_desc(ahash); +} + +static int acmac_setkey(struct crypto_ahash *ahash, const u8 *key, + unsigned int keylen) +{ + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + int err; + + err = aes_check_keylen(keylen); + if (err) + return err; + + /* key is immediate data for all cmac shared descriptors */ + ctx->adata.key_virt = key; + ctx->adata.keylen = keylen; + + print_hex_dump_devel("acmac ctx.key@" __stringify(__LINE__)" : ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); + + return acmac_set_sh_desc(ahash); +} + +/* + * ahash_edesc - s/w-extended ahash descriptor + * @sec4_sg_dma: physical mapped address of h/w link table + * @src_nents: number of segments in input scatterlist + * @sec4_sg_bytes: length of dma mapped sec4_sg space + * @bklog: stored to determine if the request needs backlog + * @hw_desc: the h/w job descriptor followed by any referenced link tables + * @sec4_sg: h/w link table + */ +struct ahash_edesc { + dma_addr_t sec4_sg_dma; + int src_nents; + int sec4_sg_bytes; + bool bklog; + u32 hw_desc[DESC_JOB_IO_LEN_MAX / sizeof(u32)] ____cacheline_aligned; + struct sec4_sg_entry sec4_sg[]; +}; + +static inline void ahash_unmap(struct device *dev, + struct ahash_edesc *edesc, + struct ahash_request *req, int dst_len) +{ + struct caam_hash_state *state = ahash_request_ctx_dma(req); + + if (edesc->src_nents) + dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); + + if (edesc->sec4_sg_bytes) + dma_unmap_single(dev, edesc->sec4_sg_dma, + edesc->sec4_sg_bytes, DMA_TO_DEVICE); + + if (state->buf_dma) { + dma_unmap_single(dev, state->buf_dma, state->buflen, + DMA_TO_DEVICE); + state->buf_dma = 0; + } +} + +static inline void ahash_unmap_ctx(struct device *dev, + struct ahash_edesc *edesc, + struct ahash_request *req, int dst_len, u32 flag) +{ + struct caam_hash_state *state = ahash_request_ctx_dma(req); + + if (state->ctx_dma) { + dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag); + state->ctx_dma = 0; + } + ahash_unmap(dev, edesc, req, dst_len); +} + +static inline void ahash_done_cpy(struct device *jrdev, u32 *desc, u32 err, + void *context, enum dma_data_direction dir) +{ + struct ahash_request *req = context; + struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev); + struct ahash_edesc *edesc; + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + int digestsize = crypto_ahash_digestsize(ahash); + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + int ecode = 0; + bool has_bklog; + + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); + + edesc = state->edesc; + has_bklog = edesc->bklog; + + if (err) + ecode = caam_jr_strstatus(jrdev, err); + + ahash_unmap_ctx(jrdev, edesc, req, digestsize, dir); + memcpy(req->result, state->caam_ctx, digestsize); + kfree(edesc); + + print_hex_dump_debug("ctx@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, + ctx->ctx_len, 1); + + /* + * If no backlog flag, the completion of the request is done + * by CAAM, not crypto engine. + */ + if (!has_bklog) + ahash_request_complete(req, ecode); + else + crypto_finalize_hash_request(jrp->engine, req, ecode); +} + +static void ahash_done(struct device *jrdev, u32 *desc, u32 err, + void *context) +{ + ahash_done_cpy(jrdev, desc, err, context, DMA_FROM_DEVICE); +} + +static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err, + void *context) +{ + ahash_done_cpy(jrdev, desc, err, context, DMA_BIDIRECTIONAL); +} + +static inline void ahash_done_switch(struct device *jrdev, u32 *desc, u32 err, + void *context, enum dma_data_direction dir) +{ + struct ahash_request *req = context; + struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev); + struct ahash_edesc *edesc; + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + struct caam_hash_state *state = ahash_request_ctx_dma(req); + int digestsize = crypto_ahash_digestsize(ahash); + int ecode = 0; + bool has_bklog; + + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); + + edesc = state->edesc; + has_bklog = edesc->bklog; + if (err) + ecode = caam_jr_strstatus(jrdev, err); + + ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, dir); + kfree(edesc); + + scatterwalk_map_and_copy(state->buf, req->src, + req->nbytes - state->next_buflen, + state->next_buflen, 0); + state->buflen = state->next_buflen; + + print_hex_dump_debug("buf@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->buf, + state->buflen, 1); + + print_hex_dump_debug("ctx@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, + ctx->ctx_len, 1); + if (req->result) + print_hex_dump_debug("result@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->result, + digestsize, 1); + + /* + * If no backlog flag, the completion of the request is done + * by CAAM, not crypto engine. + */ + if (!has_bklog) + ahash_request_complete(req, ecode); + else + crypto_finalize_hash_request(jrp->engine, req, ecode); + +} + +static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err, + void *context) +{ + ahash_done_switch(jrdev, desc, err, context, DMA_BIDIRECTIONAL); +} + +static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err, + void *context) +{ + ahash_done_switch(jrdev, desc, err, context, DMA_FROM_DEVICE); +} + +/* + * Allocate an enhanced descriptor, which contains the hardware descriptor + * and space for hardware scatter table containing sg_num entries. + */ +static struct ahash_edesc *ahash_edesc_alloc(struct ahash_request *req, + int sg_num, u32 *sh_desc, + dma_addr_t sh_desc_dma) +{ + struct caam_hash_state *state = ahash_request_ctx_dma(req); + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + struct ahash_edesc *edesc; + + sg_num = pad_sg_nents(sg_num); + edesc = kzalloc_flex(*edesc, sec4_sg, sg_num, flags); + if (!edesc) + return NULL; + + state->edesc = edesc; + + init_job_desc_shared(edesc->hw_desc, sh_desc_dma, desc_len(sh_desc), + HDR_SHARE_DEFER | HDR_REVERSE); + + return edesc; +} + +static int ahash_edesc_add_src(struct caam_hash_ctx *ctx, + struct ahash_edesc *edesc, + struct ahash_request *req, int nents, + unsigned int first_sg, + unsigned int first_bytes, size_t to_hash) +{ + dma_addr_t src_dma; + u32 options; + + if (nents > 1 || first_sg) { + struct sec4_sg_entry *sg = edesc->sec4_sg; + unsigned int sgsize = sizeof(*sg) * + pad_sg_nents(first_sg + nents); + + sg_to_sec4_sg_last(req->src, to_hash, sg + first_sg, 0); + + src_dma = dma_map_single(ctx->jrdev, sg, sgsize, DMA_TO_DEVICE); + if (dma_mapping_error(ctx->jrdev, src_dma)) { + dev_err(ctx->jrdev, "unable to map S/G table\n"); + return -ENOMEM; + } + + edesc->sec4_sg_bytes = sgsize; + edesc->sec4_sg_dma = src_dma; + options = LDST_SGF; + } else { + src_dma = sg_dma_address(req->src); + options = 0; + } + + append_seq_in_ptr(edesc->hw_desc, src_dma, first_bytes + to_hash, + options); + + return 0; +} + +static int ahash_do_one_req(struct crypto_engine *engine, void *areq) +{ + struct ahash_request *req = ahash_request_cast(areq); + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(crypto_ahash_reqtfm(req)); + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct device *jrdev = ctx->jrdev; + u32 *desc = state->edesc->hw_desc; + int ret; + + state->edesc->bklog = true; + + ret = caam_jr_enqueue(jrdev, desc, state->ahash_op_done, req); + + if (ret == -ENOSPC && engine->retry_support) + return ret; + + if (ret != -EINPROGRESS) { + ahash_unmap(jrdev, state->edesc, req, 0); + kfree(state->edesc); + } else { + ret = 0; + } + + return ret; +} + +static int ahash_enqueue_req(struct device *jrdev, + void (*cbk)(struct device *jrdev, u32 *desc, + u32 err, void *context), + struct ahash_request *req, + int dst_len, enum dma_data_direction dir) +{ + struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev); + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct ahash_edesc *edesc = state->edesc; + u32 *desc = edesc->hw_desc; + int ret; + + state->ahash_op_done = cbk; + + /* + * Only the backlog request are sent to crypto-engine since the others + * can be handled by CAAM, if free, especially since JR has up to 1024 + * entries (more than the 10 entries from crypto-engine). + */ + if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG) + ret = crypto_transfer_hash_request_to_engine(jrpriv->engine, + req); + else + ret = caam_jr_enqueue(jrdev, desc, cbk, req); + + if ((ret != -EINPROGRESS) && (ret != -EBUSY)) { + ahash_unmap_ctx(jrdev, edesc, req, dst_len, dir); + kfree(edesc); + } + + return ret; +} + +/* submit update job descriptor */ +static int ahash_update_ctx(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct device *jrdev = ctx->jrdev; + u8 *buf = state->buf; + int *buflen = &state->buflen; + int *next_buflen = &state->next_buflen; + int blocksize = crypto_ahash_blocksize(ahash); + int in_len = *buflen + req->nbytes, to_hash; + u32 *desc; + int src_nents, mapped_nents, sec4_sg_bytes, sec4_sg_src_index; + struct ahash_edesc *edesc; + int ret = 0; + + *next_buflen = in_len & (blocksize - 1); + to_hash = in_len - *next_buflen; + + /* + * For XCBC and CMAC, if to_hash is multiple of block size, + * keep last block in internal buffer + */ + if ((is_xcbc_aes(ctx->adata.algtype) || + is_cmac_aes(ctx->adata.algtype)) && to_hash >= blocksize && + (*next_buflen == 0)) { + *next_buflen = blocksize; + to_hash -= blocksize; + } + + if (to_hash) { + int pad_nents; + int src_len = req->nbytes - *next_buflen; + + src_nents = sg_nents_for_len(req->src, src_len); + if (src_nents < 0) { + dev_err(jrdev, "Invalid number of src SG.\n"); + return src_nents; + } + + if (src_nents) { + mapped_nents = dma_map_sg(jrdev, req->src, src_nents, + DMA_TO_DEVICE); + if (!mapped_nents) { + dev_err(jrdev, "unable to DMA map source\n"); + return -ENOMEM; + } + } else { + mapped_nents = 0; + } + + sec4_sg_src_index = 1 + (*buflen ? 1 : 0); + pad_nents = pad_sg_nents(sec4_sg_src_index + mapped_nents); + sec4_sg_bytes = pad_nents * sizeof(struct sec4_sg_entry); + + /* + * allocate space for base edesc and hw desc commands, + * link tables + */ + edesc = ahash_edesc_alloc(req, pad_nents, ctx->sh_desc_update, + ctx->sh_desc_update_dma); + if (!edesc) { + dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); + return -ENOMEM; + } + + edesc->src_nents = src_nents; + edesc->sec4_sg_bytes = sec4_sg_bytes; + + ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len, + edesc->sec4_sg, DMA_BIDIRECTIONAL); + if (ret) + goto unmap_ctx; + + ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state); + if (ret) + goto unmap_ctx; + + if (mapped_nents) + sg_to_sec4_sg_last(req->src, src_len, + edesc->sec4_sg + sec4_sg_src_index, + 0); + else + sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index - + 1); + + desc = edesc->hw_desc; + + edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, + sec4_sg_bytes, + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { + dev_err(jrdev, "unable to map S/G table\n"); + ret = -ENOMEM; + goto unmap_ctx; + } + + append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + + to_hash, LDST_SGF); + + append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0); + + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); + + ret = ahash_enqueue_req(jrdev, ahash_done_bi, req, + ctx->ctx_len, DMA_BIDIRECTIONAL); + } else if (*next_buflen) { + scatterwalk_map_and_copy(buf + *buflen, req->src, 0, + req->nbytes, 0); + *buflen = *next_buflen; + + print_hex_dump_debug("buf@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, buf, + *buflen, 1); + } + + return ret; +unmap_ctx: + ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL); + kfree(edesc); + return ret; +} + +static int ahash_final_ctx(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct device *jrdev = ctx->jrdev; + int buflen = state->buflen; + u32 *desc; + int sec4_sg_bytes; + int digestsize = crypto_ahash_digestsize(ahash); + struct ahash_edesc *edesc; + int ret; + + sec4_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * + sizeof(struct sec4_sg_entry); + + /* allocate space for base edesc and hw desc commands, link tables */ + edesc = ahash_edesc_alloc(req, 4, ctx->sh_desc_fin, + ctx->sh_desc_fin_dma); + if (!edesc) + return -ENOMEM; + + desc = edesc->hw_desc; + + edesc->sec4_sg_bytes = sec4_sg_bytes; + + ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len, + edesc->sec4_sg, DMA_BIDIRECTIONAL); + if (ret) + goto unmap_ctx; + + ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state); + if (ret) + goto unmap_ctx; + + sg_to_sec4_set_last(edesc->sec4_sg + (buflen ? 1 : 0)); + + edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, + sec4_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { + dev_err(jrdev, "unable to map S/G table\n"); + ret = -ENOMEM; + goto unmap_ctx; + } + + append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + buflen, + LDST_SGF); + append_seq_out_ptr(desc, state->ctx_dma, digestsize, 0); + + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); + + return ahash_enqueue_req(jrdev, ahash_done_ctx_src, req, + digestsize, DMA_BIDIRECTIONAL); + unmap_ctx: + ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_BIDIRECTIONAL); + kfree(edesc); + return ret; +} + +static int ahash_finup_ctx(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct device *jrdev = ctx->jrdev; + int buflen = state->buflen; + u32 *desc; + int sec4_sg_src_index; + int src_nents, mapped_nents; + int digestsize = crypto_ahash_digestsize(ahash); + struct ahash_edesc *edesc; + int ret; + + src_nents = sg_nents_for_len(req->src, req->nbytes); + if (src_nents < 0) { + dev_err(jrdev, "Invalid number of src SG.\n"); + return src_nents; + } + + if (src_nents) { + mapped_nents = dma_map_sg(jrdev, req->src, src_nents, + DMA_TO_DEVICE); + if (!mapped_nents) { + dev_err(jrdev, "unable to DMA map source\n"); + return -ENOMEM; + } + } else { + mapped_nents = 0; + } + + sec4_sg_src_index = 1 + (buflen ? 1 : 0); + + /* allocate space for base edesc and hw desc commands, link tables */ + edesc = ahash_edesc_alloc(req, sec4_sg_src_index + mapped_nents, + ctx->sh_desc_fin, ctx->sh_desc_fin_dma); + if (!edesc) { + dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); + return -ENOMEM; + } + + desc = edesc->hw_desc; + + edesc->src_nents = src_nents; + + ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len, + edesc->sec4_sg, DMA_BIDIRECTIONAL); + if (ret) + goto unmap_ctx; + + ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state); + if (ret) + goto unmap_ctx; + + ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, + sec4_sg_src_index, ctx->ctx_len + buflen, + req->nbytes); + if (ret) + goto unmap_ctx; + + append_seq_out_ptr(desc, state->ctx_dma, digestsize, 0); + + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); + + return ahash_enqueue_req(jrdev, ahash_done_ctx_src, req, + digestsize, DMA_BIDIRECTIONAL); + unmap_ctx: + ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_BIDIRECTIONAL); + kfree(edesc); + return ret; +} + +static int ahash_digest(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct device *jrdev = ctx->jrdev; + u32 *desc; + int digestsize = crypto_ahash_digestsize(ahash); + int src_nents, mapped_nents; + struct ahash_edesc *edesc; + int ret; + + state->buf_dma = 0; + + src_nents = sg_nents_for_len(req->src, req->nbytes); + if (src_nents < 0) { + dev_err(jrdev, "Invalid number of src SG.\n"); + return src_nents; + } + + if (src_nents) { + mapped_nents = dma_map_sg(jrdev, req->src, src_nents, + DMA_TO_DEVICE); + if (!mapped_nents) { + dev_err(jrdev, "unable to map source for DMA\n"); + return -ENOMEM; + } + } else { + mapped_nents = 0; + } + + /* allocate space for base edesc and hw desc commands, link tables */ + edesc = ahash_edesc_alloc(req, mapped_nents > 1 ? mapped_nents : 0, + ctx->sh_desc_digest, ctx->sh_desc_digest_dma); + if (!edesc) { + dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); + return -ENOMEM; + } + + edesc->src_nents = src_nents; + + ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0, + req->nbytes); + if (ret) { + ahash_unmap(jrdev, edesc, req, digestsize); + kfree(edesc); + return ret; + } + + desc = edesc->hw_desc; + + ret = map_seq_out_ptr_ctx(desc, jrdev, state, digestsize); + if (ret) { + ahash_unmap(jrdev, edesc, req, digestsize); + kfree(edesc); + return -ENOMEM; + } + + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); + + return ahash_enqueue_req(jrdev, ahash_done, req, digestsize, + DMA_FROM_DEVICE); +} + +/* submit ahash final if it the first job descriptor */ +static int ahash_final_no_ctx(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct device *jrdev = ctx->jrdev; + u8 *buf = state->buf; + int buflen = state->buflen; + u32 *desc; + int digestsize = crypto_ahash_digestsize(ahash); + struct ahash_edesc *edesc; + int ret; + + /* allocate space for base edesc and hw desc commands, link tables */ + edesc = ahash_edesc_alloc(req, 0, ctx->sh_desc_digest, + ctx->sh_desc_digest_dma); + if (!edesc) + return -ENOMEM; + + desc = edesc->hw_desc; + + if (buflen) { + state->buf_dma = dma_map_single(jrdev, buf, buflen, + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, state->buf_dma)) { + dev_err(jrdev, "unable to map src\n"); + goto unmap; + } + + append_seq_in_ptr(desc, state->buf_dma, buflen, 0); + } + + ret = map_seq_out_ptr_ctx(desc, jrdev, state, digestsize); + if (ret) + goto unmap; + + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); + + return ahash_enqueue_req(jrdev, ahash_done, req, + digestsize, DMA_FROM_DEVICE); + unmap: + ahash_unmap(jrdev, edesc, req, digestsize); + kfree(edesc); + return -ENOMEM; +} + +/* submit ahash update if it the first job descriptor after update */ +static int ahash_update_no_ctx(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct device *jrdev = ctx->jrdev; + u8 *buf = state->buf; + int *buflen = &state->buflen; + int *next_buflen = &state->next_buflen; + int blocksize = crypto_ahash_blocksize(ahash); + int in_len = *buflen + req->nbytes, to_hash; + int sec4_sg_bytes, src_nents, mapped_nents; + struct ahash_edesc *edesc; + u32 *desc; + int ret = 0; + + *next_buflen = in_len & (blocksize - 1); + to_hash = in_len - *next_buflen; + + /* + * For XCBC and CMAC, if to_hash is multiple of block size, + * keep last block in internal buffer + */ + if ((is_xcbc_aes(ctx->adata.algtype) || + is_cmac_aes(ctx->adata.algtype)) && to_hash >= blocksize && + (*next_buflen == 0)) { + *next_buflen = blocksize; + to_hash -= blocksize; + } + + if (to_hash) { + int pad_nents; + int src_len = req->nbytes - *next_buflen; + + src_nents = sg_nents_for_len(req->src, src_len); + if (src_nents < 0) { + dev_err(jrdev, "Invalid number of src SG.\n"); + return src_nents; + } + + if (src_nents) { + mapped_nents = dma_map_sg(jrdev, req->src, src_nents, + DMA_TO_DEVICE); + if (!mapped_nents) { + dev_err(jrdev, "unable to DMA map source\n"); + return -ENOMEM; + } + } else { + mapped_nents = 0; + } + + pad_nents = pad_sg_nents(1 + mapped_nents); + sec4_sg_bytes = pad_nents * sizeof(struct sec4_sg_entry); + + /* + * allocate space for base edesc and hw desc commands, + * link tables + */ + edesc = ahash_edesc_alloc(req, pad_nents, + ctx->sh_desc_update_first, + ctx->sh_desc_update_first_dma); + if (!edesc) { + dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); + return -ENOMEM; + } + + edesc->src_nents = src_nents; + edesc->sec4_sg_bytes = sec4_sg_bytes; + + ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state); + if (ret) + goto unmap_ctx; + + sg_to_sec4_sg_last(req->src, src_len, edesc->sec4_sg + 1, 0); + + desc = edesc->hw_desc; + + edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, + sec4_sg_bytes, + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { + dev_err(jrdev, "unable to map S/G table\n"); + ret = -ENOMEM; + goto unmap_ctx; + } + + append_seq_in_ptr(desc, edesc->sec4_sg_dma, to_hash, LDST_SGF); + + ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len); + if (ret) + goto unmap_ctx; + + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); + + ret = ahash_enqueue_req(jrdev, ahash_done_ctx_dst, req, + ctx->ctx_len, DMA_TO_DEVICE); + if ((ret != -EINPROGRESS) && (ret != -EBUSY)) + return ret; + state->update = ahash_update_ctx; + state->finup = ahash_finup_ctx; + state->final = ahash_final_ctx; + } else if (*next_buflen) { + scatterwalk_map_and_copy(buf + *buflen, req->src, 0, + req->nbytes, 0); + *buflen = *next_buflen; + + print_hex_dump_debug("buf@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, buf, + *buflen, 1); + } + + return ret; + unmap_ctx: + ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE); + kfree(edesc); + return ret; +} + +/* submit ahash finup if it the first job descriptor after update */ +static int ahash_finup_no_ctx(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct device *jrdev = ctx->jrdev; + int buflen = state->buflen; + u32 *desc; + int sec4_sg_bytes, sec4_sg_src_index, src_nents, mapped_nents; + int digestsize = crypto_ahash_digestsize(ahash); + struct ahash_edesc *edesc; + int ret; + + src_nents = sg_nents_for_len(req->src, req->nbytes); + if (src_nents < 0) { + dev_err(jrdev, "Invalid number of src SG.\n"); + return src_nents; + } + + if (src_nents) { + mapped_nents = dma_map_sg(jrdev, req->src, src_nents, + DMA_TO_DEVICE); + if (!mapped_nents) { + dev_err(jrdev, "unable to DMA map source\n"); + return -ENOMEM; + } + } else { + mapped_nents = 0; + } + + sec4_sg_src_index = 2; + sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) * + sizeof(struct sec4_sg_entry); + + /* allocate space for base edesc and hw desc commands, link tables */ + edesc = ahash_edesc_alloc(req, sec4_sg_src_index + mapped_nents, + ctx->sh_desc_digest, ctx->sh_desc_digest_dma); + if (!edesc) { + dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); + return -ENOMEM; + } + + desc = edesc->hw_desc; + + edesc->src_nents = src_nents; + edesc->sec4_sg_bytes = sec4_sg_bytes; + + ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state); + if (ret) + goto unmap; + + ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 1, buflen, + req->nbytes); + if (ret) { + dev_err(jrdev, "unable to map S/G table\n"); + goto unmap; + } + + ret = map_seq_out_ptr_ctx(desc, jrdev, state, digestsize); + if (ret) + goto unmap; + + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); + + return ahash_enqueue_req(jrdev, ahash_done, req, + digestsize, DMA_FROM_DEVICE); + unmap: + ahash_unmap(jrdev, edesc, req, digestsize); + kfree(edesc); + return -ENOMEM; + +} + +/* submit first update job descriptor after init */ +static int ahash_update_first(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct device *jrdev = ctx->jrdev; + u8 *buf = state->buf; + int *buflen = &state->buflen; + int *next_buflen = &state->next_buflen; + int to_hash; + int blocksize = crypto_ahash_blocksize(ahash); + u32 *desc; + int src_nents, mapped_nents; + struct ahash_edesc *edesc; + int ret = 0; + + *next_buflen = req->nbytes & (blocksize - 1); + to_hash = req->nbytes - *next_buflen; + + /* + * For XCBC and CMAC, if to_hash is multiple of block size, + * keep last block in internal buffer + */ + if ((is_xcbc_aes(ctx->adata.algtype) || + is_cmac_aes(ctx->adata.algtype)) && to_hash >= blocksize && + (*next_buflen == 0)) { + *next_buflen = blocksize; + to_hash -= blocksize; + } + + if (to_hash) { + src_nents = sg_nents_for_len(req->src, + req->nbytes - *next_buflen); + if (src_nents < 0) { + dev_err(jrdev, "Invalid number of src SG.\n"); + return src_nents; + } + + if (src_nents) { + mapped_nents = dma_map_sg(jrdev, req->src, src_nents, + DMA_TO_DEVICE); + if (!mapped_nents) { + dev_err(jrdev, "unable to map source for DMA\n"); + return -ENOMEM; + } + } else { + mapped_nents = 0; + } + + /* + * allocate space for base edesc and hw desc commands, + * link tables + */ + edesc = ahash_edesc_alloc(req, mapped_nents > 1 ? + mapped_nents : 0, + ctx->sh_desc_update_first, + ctx->sh_desc_update_first_dma); + if (!edesc) { + dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); + return -ENOMEM; + } + + edesc->src_nents = src_nents; + + ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0, + to_hash); + if (ret) + goto unmap_ctx; + + desc = edesc->hw_desc; + + ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len); + if (ret) + goto unmap_ctx; + + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); + + ret = ahash_enqueue_req(jrdev, ahash_done_ctx_dst, req, + ctx->ctx_len, DMA_TO_DEVICE); + if ((ret != -EINPROGRESS) && (ret != -EBUSY)) + return ret; + state->update = ahash_update_ctx; + state->finup = ahash_finup_ctx; + state->final = ahash_final_ctx; + } else if (*next_buflen) { + state->update = ahash_update_no_ctx; + state->finup = ahash_finup_no_ctx; + state->final = ahash_final_no_ctx; + scatterwalk_map_and_copy(buf, req->src, 0, + req->nbytes, 0); + *buflen = *next_buflen; + + print_hex_dump_debug("buf@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, buf, + *buflen, 1); + } + + return ret; + unmap_ctx: + ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE); + kfree(edesc); + return ret; +} + +static int ahash_finup_first(struct ahash_request *req) +{ + return ahash_digest(req); +} + +static int ahash_init(struct ahash_request *req) +{ + struct caam_hash_state *state = ahash_request_ctx_dma(req); + + state->update = ahash_update_first; + state->finup = ahash_finup_first; + state->final = ahash_final_no_ctx; + + state->ctx_dma = 0; + state->ctx_dma_len = 0; + state->buf_dma = 0; + state->buflen = 0; + state->next_buflen = 0; + + return 0; +} + +static int ahash_update(struct ahash_request *req) +{ + struct caam_hash_state *state = ahash_request_ctx_dma(req); + + return state->update(req); +} + +static int ahash_finup(struct ahash_request *req) +{ + struct caam_hash_state *state = ahash_request_ctx_dma(req); + + return state->finup(req); +} + +static int ahash_final(struct ahash_request *req) +{ + struct caam_hash_state *state = ahash_request_ctx_dma(req); + + return state->final(req); +} + +static int ahash_export(struct ahash_request *req, void *out) +{ + struct caam_hash_state *state = ahash_request_ctx_dma(req); + struct caam_export_state *export = out; + u8 *buf = state->buf; + int len = state->buflen; + + memcpy(export->buf, buf, len); + memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx)); + export->buflen = len; + export->update = state->update; + export->final = state->final; + export->finup = state->finup; + + return 0; +} + +static int ahash_import(struct ahash_request *req, const void *in) +{ + struct caam_hash_state *state = ahash_request_ctx_dma(req); + const struct caam_export_state *export = in; + + memset(state, 0, sizeof(*state)); + memcpy(state->buf, export->buf, export->buflen); + memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx)); + state->buflen = export->buflen; + state->update = export->update; + state->final = export->final; + state->finup = export->finup; + + return 0; +} + +struct caam_hash_template { + char name[CRYPTO_MAX_ALG_NAME]; + char driver_name[CRYPTO_MAX_ALG_NAME]; + char hmac_name[CRYPTO_MAX_ALG_NAME]; + char hmac_driver_name[CRYPTO_MAX_ALG_NAME]; + unsigned int blocksize; + struct ahash_alg template_ahash; + u32 alg_type; +}; + +/* ahash descriptors */ +static struct caam_hash_template driver_hash[] = { + { + .name = "sha1", + .driver_name = "sha1-caam", + .hmac_name = "hmac(sha1)", + .hmac_driver_name = "hmac-sha1-caam", + .blocksize = SHA1_BLOCK_SIZE, + .template_ahash = { + .init = ahash_init, + .update = ahash_update, + .final = ahash_final, + .finup = ahash_finup, + .digest = ahash_digest, + .export = ahash_export, + .import = ahash_import, + .setkey = ahash_setkey, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct caam_export_state), + }, + }, + .alg_type = OP_ALG_ALGSEL_SHA1, + }, { + .name = "sha224", + .driver_name = "sha224-caam", + .hmac_name = "hmac(sha224)", + .hmac_driver_name = "hmac-sha224-caam", + .blocksize = SHA224_BLOCK_SIZE, + .template_ahash = { + .init = ahash_init, + .update = ahash_update, + .final = ahash_final, + .finup = ahash_finup, + .digest = ahash_digest, + .export = ahash_export, + .import = ahash_import, + .setkey = ahash_setkey, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct caam_export_state), + }, + }, + .alg_type = OP_ALG_ALGSEL_SHA224, + }, { + .name = "sha256", + .driver_name = "sha256-caam", + .hmac_name = "hmac(sha256)", + .hmac_driver_name = "hmac-sha256-caam", + .blocksize = SHA256_BLOCK_SIZE, + .template_ahash = { + .init = ahash_init, + .update = ahash_update, + .final = ahash_final, + .finup = ahash_finup, + .digest = ahash_digest, + .export = ahash_export, + .import = ahash_import, + .setkey = ahash_setkey, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct caam_export_state), + }, + }, + .alg_type = OP_ALG_ALGSEL_SHA256, + }, { + .name = "sha384", + .driver_name = "sha384-caam", + .hmac_name = "hmac(sha384)", + .hmac_driver_name = "hmac-sha384-caam", + .blocksize = SHA384_BLOCK_SIZE, + .template_ahash = { + .init = ahash_init, + .update = ahash_update, + .final = ahash_final, + .finup = ahash_finup, + .digest = ahash_digest, + .export = ahash_export, + .import = ahash_import, + .setkey = ahash_setkey, + .halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct caam_export_state), + }, + }, + .alg_type = OP_ALG_ALGSEL_SHA384, + }, { + .name = "sha512", + .driver_name = "sha512-caam", + .hmac_name = "hmac(sha512)", + .hmac_driver_name = "hmac-sha512-caam", + .blocksize = SHA512_BLOCK_SIZE, + .template_ahash = { + .init = ahash_init, + .update = ahash_update, + .final = ahash_final, + .finup = ahash_finup, + .digest = ahash_digest, + .export = ahash_export, + .import = ahash_import, + .setkey = ahash_setkey, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct caam_export_state), + }, + }, + .alg_type = OP_ALG_ALGSEL_SHA512, + }, { + .name = "md5", + .driver_name = "md5-caam", + .hmac_name = "hmac(md5)", + .hmac_driver_name = "hmac-md5-caam", + .blocksize = MD5_BLOCK_WORDS * 4, + .template_ahash = { + .init = ahash_init, + .update = ahash_update, + .final = ahash_final, + .finup = ahash_finup, + .digest = ahash_digest, + .export = ahash_export, + .import = ahash_import, + .setkey = ahash_setkey, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct caam_export_state), + }, + }, + .alg_type = OP_ALG_ALGSEL_MD5, + }, { + .hmac_name = "xcbc(aes)", + .hmac_driver_name = "xcbc-aes-caam", + .blocksize = AES_BLOCK_SIZE, + .template_ahash = { + .init = ahash_init, + .update = ahash_update, + .final = ahash_final, + .finup = ahash_finup, + .digest = ahash_digest, + .export = ahash_export, + .import = ahash_import, + .setkey = axcbc_setkey, + .halg = { + .digestsize = AES_BLOCK_SIZE, + .statesize = sizeof(struct caam_export_state), + }, + }, + .alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XCBC_MAC, + }, { + .hmac_name = "cmac(aes)", + .hmac_driver_name = "cmac-aes-caam", + .blocksize = AES_BLOCK_SIZE, + .template_ahash = { + .init = ahash_init, + .update = ahash_update, + .final = ahash_final, + .finup = ahash_finup, + .digest = ahash_digest, + .export = ahash_export, + .import = ahash_import, + .setkey = acmac_setkey, + .halg = { + .digestsize = AES_BLOCK_SIZE, + .statesize = sizeof(struct caam_export_state), + }, + }, + .alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CMAC, + }, +}; + +struct caam_hash_alg { + struct list_head entry; + int alg_type; + bool is_hmac; + struct ahash_engine_alg ahash_alg; +}; + +static int caam_hash_cra_init(struct crypto_tfm *tfm) +{ + struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); + struct crypto_alg *base = tfm->__crt_alg; + struct hash_alg_common *halg = + container_of(base, struct hash_alg_common, base); + struct ahash_alg *alg = + container_of(halg, struct ahash_alg, halg); + struct caam_hash_alg *caam_hash = + container_of(alg, struct caam_hash_alg, ahash_alg.base); + struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */ + static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE, + HASH_MSG_LEN + SHA1_DIGEST_SIZE, + HASH_MSG_LEN + 32, + HASH_MSG_LEN + SHA256_DIGEST_SIZE, + HASH_MSG_LEN + 64, + HASH_MSG_LEN + SHA512_DIGEST_SIZE }; + const size_t sh_desc_update_offset = offsetof(struct caam_hash_ctx, + sh_desc_update); + dma_addr_t dma_addr; + struct caam_drv_private *priv; + + /* + * Get a Job ring from Job Ring driver to ensure in-order + * crypto request processing per tfm + */ + ctx->jrdev = caam_jr_alloc(); + if (IS_ERR(ctx->jrdev)) { + pr_err("Job Ring Device allocation for transform failed\n"); + return PTR_ERR(ctx->jrdev); + } + + priv = dev_get_drvdata(ctx->jrdev->parent); + + if (is_xcbc_aes(caam_hash->alg_type)) { + ctx->dir = DMA_TO_DEVICE; + ctx->key_dir = DMA_BIDIRECTIONAL; + ctx->adata.algtype = OP_TYPE_CLASS1_ALG | caam_hash->alg_type; + ctx->ctx_len = 48; + } else if (is_cmac_aes(caam_hash->alg_type)) { + ctx->dir = DMA_TO_DEVICE; + ctx->key_dir = DMA_NONE; + ctx->adata.algtype = OP_TYPE_CLASS1_ALG | caam_hash->alg_type; + ctx->ctx_len = 32; + } else { + if (priv->era >= 6) { + ctx->dir = DMA_BIDIRECTIONAL; + ctx->key_dir = caam_hash->is_hmac ? DMA_TO_DEVICE : DMA_NONE; + } else { + ctx->dir = DMA_TO_DEVICE; + ctx->key_dir = DMA_NONE; + } + ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; + ctx->ctx_len = runninglen[(ctx->adata.algtype & + OP_ALG_ALGSEL_SUBMASK) >> + OP_ALG_ALGSEL_SHIFT]; + } + + if (ctx->key_dir != DMA_NONE) { + ctx->adata.key_dma = dma_map_single_attrs(ctx->jrdev, ctx->key, + ARRAY_SIZE(ctx->key), + ctx->key_dir, + DMA_ATTR_SKIP_CPU_SYNC); + if (dma_mapping_error(ctx->jrdev, ctx->adata.key_dma)) { + dev_err(ctx->jrdev, "unable to map key\n"); + caam_jr_free(ctx->jrdev); + return -ENOMEM; + } + } + + dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_update, + offsetof(struct caam_hash_ctx, key) - + sh_desc_update_offset, + ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); + if (dma_mapping_error(ctx->jrdev, dma_addr)) { + dev_err(ctx->jrdev, "unable to map shared descriptors\n"); + + if (ctx->key_dir != DMA_NONE) + dma_unmap_single_attrs(ctx->jrdev, ctx->adata.key_dma, + ARRAY_SIZE(ctx->key), + ctx->key_dir, + DMA_ATTR_SKIP_CPU_SYNC); + + caam_jr_free(ctx->jrdev); + return -ENOMEM; + } + + ctx->sh_desc_update_dma = dma_addr; + ctx->sh_desc_update_first_dma = dma_addr + + offsetof(struct caam_hash_ctx, + sh_desc_update_first) - + sh_desc_update_offset; + ctx->sh_desc_fin_dma = dma_addr + offsetof(struct caam_hash_ctx, + sh_desc_fin) - + sh_desc_update_offset; + ctx->sh_desc_digest_dma = dma_addr + offsetof(struct caam_hash_ctx, + sh_desc_digest) - + sh_desc_update_offset; + + crypto_ahash_set_reqsize_dma(ahash, sizeof(struct caam_hash_state)); + + /* + * For keyed hash algorithms shared descriptors + * will be created later in setkey() callback + */ + return caam_hash->is_hmac ? 0 : ahash_set_sh_desc(ahash); +} + +static void caam_hash_cra_exit(struct crypto_tfm *tfm) +{ + struct caam_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm); + + dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_update_dma, + offsetof(struct caam_hash_ctx, key) - + offsetof(struct caam_hash_ctx, sh_desc_update), + ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); + if (ctx->key_dir != DMA_NONE) + dma_unmap_single_attrs(ctx->jrdev, ctx->adata.key_dma, + ARRAY_SIZE(ctx->key), ctx->key_dir, + DMA_ATTR_SKIP_CPU_SYNC); + caam_jr_free(ctx->jrdev); +} + +void caam_algapi_hash_exit(void) +{ + struct caam_hash_alg *t_alg, *n; + + if (!hash_list.next) + return; + + list_for_each_entry_safe(t_alg, n, &hash_list, entry) { + crypto_engine_unregister_ahash(&t_alg->ahash_alg); + list_del(&t_alg->entry); + kfree(t_alg); + } +} + +static struct caam_hash_alg * +caam_hash_alloc(struct caam_hash_template *template, + bool keyed) +{ + struct caam_hash_alg *t_alg; + struct ahash_alg *halg; + struct crypto_alg *alg; + + t_alg = kzalloc_obj(*t_alg); + if (!t_alg) + return ERR_PTR(-ENOMEM); + + t_alg->ahash_alg.base = template->template_ahash; + halg = &t_alg->ahash_alg.base; + alg = &halg->halg.base; + + if (keyed) { + strscpy(alg->cra_name, template->hmac_name); + strscpy(alg->cra_driver_name, template->hmac_driver_name); + t_alg->is_hmac = true; + } else { + strscpy(alg->cra_name, template->name); + strscpy(alg->cra_driver_name, template->driver_name); + halg->setkey = NULL; + t_alg->is_hmac = false; + } + alg->cra_module = THIS_MODULE; + alg->cra_init = caam_hash_cra_init; + alg->cra_exit = caam_hash_cra_exit; + alg->cra_ctxsize = sizeof(struct caam_hash_ctx) + crypto_dma_padding(); + alg->cra_priority = CAAM_CRA_PRIORITY; + alg->cra_blocksize = template->blocksize; + alg->cra_alignmask = 0; + alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY; + + t_alg->alg_type = template->alg_type; + t_alg->ahash_alg.op.do_one_request = ahash_do_one_req; + + return t_alg; +} + +int caam_algapi_hash_init(struct device *ctrldev) +{ + int i = 0, err = 0; + struct caam_drv_private *priv = dev_get_drvdata(ctrldev); + unsigned int md_limit = SHA512_DIGEST_SIZE; + u32 md_inst, md_vid; + + /* + * Register crypto algorithms the device supports. First, identify + * presence and attributes of MD block. + */ + if (priv->era < 10) { + struct caam_perfmon __iomem *perfmon = &priv->jr[0]->perfmon; + + md_vid = (rd_reg32(&perfmon->cha_id_ls) & + CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; + md_inst = (rd_reg32(&perfmon->cha_num_ls) & + CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; + } else { + u32 mdha = rd_reg32(&priv->jr[0]->vreg.mdha); + + md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT; + md_inst = mdha & CHA_VER_NUM_MASK; + } + + /* + * Skip registration of any hashing algorithms if MD block + * is not present. + */ + if (!md_inst) + return 0; + + /* Limit digest size based on LP256 */ + if (md_vid == CHA_VER_VID_MD_LP256) + md_limit = SHA256_DIGEST_SIZE; + + INIT_LIST_HEAD(&hash_list); + + /* register crypto algorithms the device supports */ + for (i = 0; i < ARRAY_SIZE(driver_hash); i++) { + struct caam_hash_alg *t_alg; + struct caam_hash_template *alg = driver_hash + i; + + /* If MD size is not supported by device, skip registration */ + if (is_mdha(alg->alg_type) && + alg->template_ahash.halg.digestsize > md_limit) + continue; + + /* register hmac version */ + t_alg = caam_hash_alloc(alg, true); + if (IS_ERR(t_alg)) { + err = PTR_ERR(t_alg); + pr_warn("%s alg allocation failed\n", + alg->hmac_driver_name); + continue; + } + + err = crypto_engine_register_ahash(&t_alg->ahash_alg); + if (err) { + pr_warn("%s alg registration failed: %d\n", + t_alg->ahash_alg.base.halg.base.cra_driver_name, + err); + kfree(t_alg); + } else + list_add_tail(&t_alg->entry, &hash_list); + + if ((alg->alg_type & OP_ALG_ALGSEL_MASK) == OP_ALG_ALGSEL_AES) + continue; + + /* register unkeyed version */ + t_alg = caam_hash_alloc(alg, false); + if (IS_ERR(t_alg)) { + err = PTR_ERR(t_alg); + pr_warn("%s alg allocation failed\n", alg->driver_name); + continue; + } + + err = crypto_engine_register_ahash(&t_alg->ahash_alg); + if (err) { + pr_warn("%s alg registration failed: %d\n", + t_alg->ahash_alg.base.halg.base.cra_driver_name, + err); + kfree(t_alg); + } else + list_add_tail(&t_alg->entry, &hash_list); + } + + return err; +} diff --git a/drivers/crypto/caam/caamhash_desc.c b/drivers/crypto/caam/caamhash_desc.c new file mode 100644 index 000000000..619564509 --- /dev/null +++ b/drivers/crypto/caam/caamhash_desc.c @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Shared descriptors for ahash algorithms + * + * Copyright 2017-2019 NXP + */ + +#include "compat.h" +#include "desc_constr.h" +#include "caamhash_desc.h" + +/** + * cnstr_shdsc_ahash - ahash shared descriptor + * @desc: pointer to buffer used for descriptor construction + * @adata: pointer to authentication transform definitions. + * A split key is required for SEC Era < 6; the size of the split key + * is specified in this case. + * Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, SHA224, + * SHA256, SHA384, SHA512}. + * @state: algorithm state OP_ALG_AS_{INIT, FINALIZE, INITFINALIZE, UPDATE} + * @digestsize: algorithm's digest size + * @ctx_len: size of Context Register + * @import_ctx: true if previous Context Register needs to be restored + * must be true for ahash update and final + * must be false for ahash first and digest + * @era: SEC Era + */ +void cnstr_shdsc_ahash(u32 * const desc, struct alginfo *adata, u32 state, + int digestsize, int ctx_len, bool import_ctx, int era) +{ + u32 op = adata->algtype; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* Append key if it has been set; ahash update excluded */ + if (state != OP_ALG_AS_UPDATE && adata->keylen) { + u32 *skip_key_load; + + /* Skip key loading if already shared */ + skip_key_load = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + + if (era < 6) + append_key_as_imm(desc, adata->key_virt, + adata->keylen_pad, + adata->keylen, CLASS_2 | + KEY_DEST_MDHA_SPLIT | KEY_ENC); + else + append_proto_dkp(desc, adata); + + set_jump_tgt_here(desc, skip_key_load); + + op |= OP_ALG_AAI_HMAC_PRECOMP; + } + + /* If needed, import context from software */ + if (import_ctx) + append_seq_load(desc, ctx_len, LDST_CLASS_2_CCB | + LDST_SRCDST_BYTE_CONTEXT); + + /* Class 2 operation */ + append_operation(desc, op | state | OP_ALG_ENCRYPT); + + /* + * Load from buf and/or src and write to req->result or state->context + * Calculate remaining bytes to read + */ + append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + /* Read remaining bytes */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 | + FIFOLD_TYPE_MSG | KEY_VLF); + /* Store class2 context bytes */ + append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | + LDST_SRCDST_BYTE_CONTEXT); +} +EXPORT_SYMBOL(cnstr_shdsc_ahash); + +/** + * cnstr_shdsc_sk_hash - shared descriptor for symmetric key cipher-based + * hash algorithms + * @desc: pointer to buffer used for descriptor construction + * @adata: pointer to authentication transform definitions. + * @state: algorithm state OP_ALG_AS_{INIT, FINALIZE, INITFINALIZE, UPDATE} + * @digestsize: algorithm's digest size + * @ctx_len: size of Context Register + */ +void cnstr_shdsc_sk_hash(u32 * const desc, struct alginfo *adata, u32 state, + int digestsize, int ctx_len) +{ + u32 *skip_key_load; + + init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); + + /* Skip loading of key, context if already shared */ + skip_key_load = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); + + if (state == OP_ALG_AS_INIT || state == OP_ALG_AS_INITFINAL) { + append_key_as_imm(desc, adata->key_virt, adata->keylen, + adata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + } else { /* UPDATE, FINALIZE */ + if (is_xcbc_aes(adata->algtype)) + /* Load K1 */ + append_key(desc, adata->key_dma, adata->keylen, + CLASS_1 | KEY_DEST_CLASS_REG | KEY_ENC); + else /* CMAC */ + append_key_as_imm(desc, adata->key_virt, adata->keylen, + adata->keylen, CLASS_1 | + KEY_DEST_CLASS_REG); + /* Restore context */ + append_seq_load(desc, ctx_len, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT); + } + + set_jump_tgt_here(desc, skip_key_load); + + /* Class 1 operation */ + append_operation(desc, adata->algtype | state | OP_ALG_ENCRYPT); + + /* + * Load from buf and/or src and write to req->result or state->context + * Calculate remaining bytes to read + */ + append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + + /* Read remaining bytes */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_LAST1 | + FIFOLD_TYPE_MSG | FIFOLDST_VLF); + + /* + * Save context: + * - xcbc: partial hash, keys K2 and K3 + * - cmac: partial hash, constant L = E(K,0) + */ + append_seq_store(desc, digestsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT); + if (is_xcbc_aes(adata->algtype) && state == OP_ALG_AS_INIT) + /* Save K1 */ + append_fifo_store(desc, adata->key_dma, adata->keylen, + LDST_CLASS_1_CCB | FIFOST_TYPE_KEY_KEK); +} +EXPORT_SYMBOL(cnstr_shdsc_sk_hash); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DESCRIPTION("FSL CAAM ahash descriptors support"); +MODULE_AUTHOR("NXP Semiconductors"); diff --git a/drivers/crypto/caam/caamhash_desc.h b/drivers/crypto/caam/caamhash_desc.h new file mode 100644 index 000000000..4f369b8cb --- /dev/null +++ b/drivers/crypto/caam/caamhash_desc.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */ +/* + * Shared descriptors for ahash algorithms + * + * Copyright 2017 NXP + */ + +#ifndef _CAAMHASH_DESC_H_ +#define _CAAMHASH_DESC_H_ + +/* length of descriptors text */ +#define DESC_AHASH_BASE (3 * CAAM_CMD_SZ) +#define DESC_AHASH_UPDATE_LEN (6 * CAAM_CMD_SZ) +#define DESC_AHASH_UPDATE_FIRST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ) +#define DESC_AHASH_FINAL_LEN (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ) +#define DESC_AHASH_DIGEST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ) + +static inline bool is_xcbc_aes(u32 algtype) +{ + return (algtype & (OP_ALG_ALGSEL_MASK | OP_ALG_AAI_MASK)) == + (OP_ALG_ALGSEL_AES | OP_ALG_AAI_XCBC_MAC); +} + +void cnstr_shdsc_ahash(u32 * const desc, struct alginfo *adata, u32 state, + int digestsize, int ctx_len, bool import_ctx, int era); + +void cnstr_shdsc_sk_hash(u32 * const desc, struct alginfo *adata, u32 state, + int digestsize, int ctx_len); +#endif /* _CAAMHASH_DESC_H_ */ diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c new file mode 100644 index 000000000..cb001aa1d --- /dev/null +++ b/drivers/crypto/caam/caampkc.c @@ -0,0 +1,1230 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * caam - Freescale FSL CAAM support for Public Key Cryptography + * + * Copyright 2016 Freescale Semiconductor, Inc. + * Copyright 2018-2019, 2023 NXP + * + * There is no Shared Descriptor for PKC so that the Job Descriptor must carry + * all the desired key parameters, input and output pointers. + */ +#include "compat.h" +#include "regs.h" +#include "intern.h" +#include "jr.h" +#include "error.h" +#include "desc_constr.h" +#include "sg_sw_sec4.h" +#include "caampkc.h" +#include +#include +#include +#include +#include +#include + +#define DESC_RSA_PUB_LEN (2 * CAAM_CMD_SZ + SIZEOF_RSA_PUB_PDB) +#define DESC_RSA_PRIV_F1_LEN (2 * CAAM_CMD_SZ + \ + SIZEOF_RSA_PRIV_F1_PDB) +#define DESC_RSA_PRIV_F2_LEN (2 * CAAM_CMD_SZ + \ + SIZEOF_RSA_PRIV_F2_PDB) +#define DESC_RSA_PRIV_F3_LEN (2 * CAAM_CMD_SZ + \ + SIZEOF_RSA_PRIV_F3_PDB) +#define CAAM_RSA_MAX_INPUT_SIZE 512 /* for a 4096-bit modulus */ + +/* buffer filled with zeros, used for padding */ +static u8 *zero_buffer; + +/* + * variable used to avoid double free of resources in case + * algorithm registration was unsuccessful + */ +static bool init_done; + +struct caam_akcipher_alg { + struct akcipher_engine_alg akcipher; + bool registered; +}; + +static void rsa_io_unmap(struct device *dev, struct rsa_edesc *edesc, + struct akcipher_request *req) +{ + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + + dma_unmap_sg(dev, req->dst, edesc->dst_nents, DMA_FROM_DEVICE); + dma_unmap_sg(dev, req_ctx->fixup_src, edesc->src_nents, DMA_TO_DEVICE); + + if (edesc->sec4_sg_bytes) + dma_unmap_single(dev, edesc->sec4_sg_dma, edesc->sec4_sg_bytes, + DMA_TO_DEVICE); +} + +static void rsa_pub_unmap(struct device *dev, struct rsa_edesc *edesc, + struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + struct caam_rsa_key *key = &ctx->key; + struct rsa_pub_pdb *pdb = &edesc->pdb.pub; + + dma_unmap_single(dev, pdb->n_dma, key->n_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->e_dma, key->e_sz, DMA_TO_DEVICE); +} + +static void rsa_priv_f1_unmap(struct device *dev, struct rsa_edesc *edesc, + struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + struct caam_rsa_key *key = &ctx->key; + struct rsa_priv_f1_pdb *pdb = &edesc->pdb.priv_f1; + + dma_unmap_single(dev, pdb->n_dma, key->n_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->d_dma, key->d_sz, DMA_TO_DEVICE); +} + +static void rsa_priv_f2_unmap(struct device *dev, struct rsa_edesc *edesc, + struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + struct caam_rsa_key *key = &ctx->key; + struct rsa_priv_f2_pdb *pdb = &edesc->pdb.priv_f2; + size_t p_sz = key->p_sz; + size_t q_sz = key->q_sz; + + dma_unmap_single(dev, pdb->d_dma, key->d_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->p_dma, p_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->q_dma, q_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_BIDIRECTIONAL); + dma_unmap_single(dev, pdb->tmp2_dma, q_sz, DMA_BIDIRECTIONAL); +} + +static void rsa_priv_f3_unmap(struct device *dev, struct rsa_edesc *edesc, + struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + struct caam_rsa_key *key = &ctx->key; + struct rsa_priv_f3_pdb *pdb = &edesc->pdb.priv_f3; + size_t p_sz = key->p_sz; + size_t q_sz = key->q_sz; + + dma_unmap_single(dev, pdb->p_dma, p_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->q_dma, q_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->dp_dma, p_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->dq_dma, q_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->c_dma, p_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_BIDIRECTIONAL); + dma_unmap_single(dev, pdb->tmp2_dma, q_sz, DMA_BIDIRECTIONAL); +} + +/* RSA Job Completion handler */ +static void rsa_pub_done(struct device *dev, u32 *desc, u32 err, void *context) +{ + struct akcipher_request *req = context; + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); + struct rsa_edesc *edesc; + int ecode = 0; + bool has_bklog; + + if (err) + ecode = caam_jr_strstatus(dev, err); + + edesc = req_ctx->edesc; + has_bklog = edesc->bklog; + + rsa_pub_unmap(dev, edesc, req); + rsa_io_unmap(dev, edesc, req); + kfree(edesc); + + /* + * If no backlog flag, the completion of the request is done + * by CAAM, not crypto engine. + */ + if (!has_bklog) + akcipher_request_complete(req, ecode); + else + crypto_finalize_akcipher_request(jrp->engine, req, ecode); +} + +static void rsa_priv_f_done(struct device *dev, u32 *desc, u32 err, + void *context) +{ + struct akcipher_request *req = context; + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + struct caam_rsa_key *key = &ctx->key; + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + struct rsa_edesc *edesc; + int ecode = 0; + bool has_bklog; + + if (err) + ecode = caam_jr_strstatus(dev, err); + + edesc = req_ctx->edesc; + has_bklog = edesc->bklog; + + switch (key->priv_form) { + case FORM1: + rsa_priv_f1_unmap(dev, edesc, req); + break; + case FORM2: + rsa_priv_f2_unmap(dev, edesc, req); + break; + case FORM3: + rsa_priv_f3_unmap(dev, edesc, req); + } + + rsa_io_unmap(dev, edesc, req); + kfree(edesc); + + /* + * If no backlog flag, the completion of the request is done + * by CAAM, not crypto engine. + */ + if (!has_bklog) + akcipher_request_complete(req, ecode); + else + crypto_finalize_akcipher_request(jrp->engine, req, ecode); +} + +/** + * caam_rsa_count_leading_zeros - Count leading zeros, need it to strip, + * from a given scatterlist + * + * @sgl : scatterlist to count zeros from + * @nbytes: number of zeros, in bytes, to strip + * @flags : operation flags + */ +static int caam_rsa_count_leading_zeros(struct scatterlist *sgl, + unsigned int nbytes, + unsigned int flags) +{ + struct sg_mapping_iter miter; + int lzeros, ents; + unsigned int len; + unsigned int tbytes = nbytes; + const u8 *buff; + + ents = sg_nents_for_len(sgl, nbytes); + if (ents < 0) + return ents; + + sg_miter_start(&miter, sgl, ents, SG_MITER_FROM_SG | flags); + + lzeros = 0; + len = 0; + while (nbytes > 0) { + /* do not strip more than given bytes */ + while (len && !*buff && lzeros < nbytes) { + lzeros++; + len--; + buff++; + } + + if (len && *buff) + break; + + if (!sg_miter_next(&miter)) + break; + + buff = miter.addr; + len = miter.length; + + nbytes -= lzeros; + lzeros = 0; + } + + miter.consumed = lzeros; + sg_miter_stop(&miter); + nbytes -= lzeros; + + return tbytes - nbytes; +} + +static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, + size_t desclen) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + struct device *dev = ctx->dev; + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + struct caam_rsa_key *key = &ctx->key; + struct rsa_edesc *edesc; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int sg_flags = (flags == GFP_ATOMIC) ? SG_MITER_ATOMIC : 0; + int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; + int src_nents, dst_nents; + int mapped_src_nents, mapped_dst_nents; + unsigned int diff_size = 0; + int lzeros; + + if (req->src_len > key->n_sz) { + /* + * strip leading zeros and + * return the number of zeros to skip + */ + lzeros = caam_rsa_count_leading_zeros(req->src, req->src_len - + key->n_sz, sg_flags); + if (lzeros < 0) + return ERR_PTR(lzeros); + + req_ctx->fixup_src = scatterwalk_ffwd(req_ctx->src, req->src, + lzeros); + req_ctx->fixup_src_len = req->src_len - lzeros; + } else { + /* + * input src is less then n key modulus, + * so there will be zero padding + */ + diff_size = key->n_sz - req->src_len; + req_ctx->fixup_src = req->src; + req_ctx->fixup_src_len = req->src_len; + } + + src_nents = sg_nents_for_len(req_ctx->fixup_src, + req_ctx->fixup_src_len); + dst_nents = sg_nents_for_len(req->dst, req->dst_len); + + mapped_src_nents = dma_map_sg(dev, req_ctx->fixup_src, src_nents, + DMA_TO_DEVICE); + if (unlikely(!mapped_src_nents)) { + dev_err(dev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, + DMA_FROM_DEVICE); + if (unlikely(!mapped_dst_nents)) { + dev_err(dev, "unable to map destination\n"); + goto src_fail; + } + + if (!diff_size && mapped_src_nents == 1) + sec4_sg_len = 0; /* no need for an input hw s/g table */ + else + sec4_sg_len = mapped_src_nents + !!diff_size; + sec4_sg_index = sec4_sg_len; + + if (mapped_dst_nents > 1) + sec4_sg_len += pad_sg_nents(mapped_dst_nents); + else + sec4_sg_len = pad_sg_nents(sec4_sg_len); + + sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); + + /* allocate space for base edesc, hw desc commands and link tables */ + edesc = kzalloc(sizeof(*edesc) + desclen + sec4_sg_bytes, flags); + if (!edesc) + goto dst_fail; + + edesc->sec4_sg = (void *)edesc + sizeof(*edesc) + desclen; + if (diff_size) + dma_to_sec4_sg_one(edesc->sec4_sg, ctx->padding_dma, diff_size, + 0); + + if (sec4_sg_index) + sg_to_sec4_sg_last(req_ctx->fixup_src, req_ctx->fixup_src_len, + edesc->sec4_sg + !!diff_size, 0); + + if (mapped_dst_nents > 1) + sg_to_sec4_sg_last(req->dst, req->dst_len, + edesc->sec4_sg + sec4_sg_index, 0); + + /* Save nents for later use in Job Descriptor */ + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + + req_ctx->edesc = edesc; + + if (!sec4_sg_bytes) + return edesc; + + edesc->mapped_src_nents = mapped_src_nents; + edesc->mapped_dst_nents = mapped_dst_nents; + + edesc->sec4_sg_dma = dma_map_single(dev, edesc->sec4_sg, + sec4_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(dev, edesc->sec4_sg_dma)) { + dev_err(dev, "unable to map S/G table\n"); + goto sec4_sg_fail; + } + + edesc->sec4_sg_bytes = sec4_sg_bytes; + + print_hex_dump_debug("caampkc sec4_sg@" __stringify(__LINE__) ": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, + edesc->sec4_sg_bytes, 1); + + return edesc; + +sec4_sg_fail: + kfree(edesc); +dst_fail: + dma_unmap_sg(dev, req->dst, dst_nents, DMA_FROM_DEVICE); +src_fail: + dma_unmap_sg(dev, req_ctx->fixup_src, src_nents, DMA_TO_DEVICE); + return ERR_PTR(-ENOMEM); +} + +static int akcipher_do_one_req(struct crypto_engine *engine, void *areq) +{ + struct akcipher_request *req = container_of(areq, + struct akcipher_request, + base); + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + struct device *jrdev = ctx->dev; + u32 *desc = req_ctx->edesc->hw_desc; + int ret; + + req_ctx->edesc->bklog = true; + + ret = caam_jr_enqueue(jrdev, desc, req_ctx->akcipher_op_done, req); + + if (ret == -ENOSPC && engine->retry_support) + return ret; + + if (ret != -EINPROGRESS) { + rsa_pub_unmap(jrdev, req_ctx->edesc, req); + rsa_io_unmap(jrdev, req_ctx->edesc, req); + kfree(req_ctx->edesc); + } else { + ret = 0; + } + + return ret; +} + +static int set_rsa_pub_pdb(struct akcipher_request *req, + struct rsa_edesc *edesc) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + struct caam_rsa_key *key = &ctx->key; + struct device *dev = ctx->dev; + struct rsa_pub_pdb *pdb = &edesc->pdb.pub; + int sec4_sg_index = 0; + + pdb->n_dma = dma_map_single(dev, key->n, key->n_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->n_dma)) { + dev_err(dev, "Unable to map RSA modulus memory\n"); + return -ENOMEM; + } + + pdb->e_dma = dma_map_single(dev, key->e, key->e_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->e_dma)) { + dev_err(dev, "Unable to map RSA public exponent memory\n"); + dma_unmap_single(dev, pdb->n_dma, key->n_sz, DMA_TO_DEVICE); + return -ENOMEM; + } + + if (edesc->mapped_src_nents > 1) { + pdb->sgf |= RSA_PDB_SGF_F; + pdb->f_dma = edesc->sec4_sg_dma; + sec4_sg_index += edesc->mapped_src_nents; + } else { + pdb->f_dma = sg_dma_address(req_ctx->fixup_src); + } + + if (edesc->mapped_dst_nents > 1) { + pdb->sgf |= RSA_PDB_SGF_G; + pdb->g_dma = edesc->sec4_sg_dma + + sec4_sg_index * sizeof(struct sec4_sg_entry); + } else { + pdb->g_dma = sg_dma_address(req->dst); + } + + pdb->sgf |= (key->e_sz << RSA_PDB_E_SHIFT) | key->n_sz; + pdb->f_len = req_ctx->fixup_src_len; + + return 0; +} + +static int set_rsa_priv_f1_pdb(struct akcipher_request *req, + struct rsa_edesc *edesc) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + struct caam_rsa_key *key = &ctx->key; + struct device *dev = ctx->dev; + struct rsa_priv_f1_pdb *pdb = &edesc->pdb.priv_f1; + int sec4_sg_index = 0; + + pdb->n_dma = dma_map_single(dev, key->n, key->n_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->n_dma)) { + dev_err(dev, "Unable to map modulus memory\n"); + return -ENOMEM; + } + + pdb->d_dma = dma_map_single(dev, key->d, key->d_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->d_dma)) { + dev_err(dev, "Unable to map RSA private exponent memory\n"); + dma_unmap_single(dev, pdb->n_dma, key->n_sz, DMA_TO_DEVICE); + return -ENOMEM; + } + + if (edesc->mapped_src_nents > 1) { + pdb->sgf |= RSA_PRIV_PDB_SGF_G; + pdb->g_dma = edesc->sec4_sg_dma; + sec4_sg_index += edesc->mapped_src_nents; + + } else { + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + + pdb->g_dma = sg_dma_address(req_ctx->fixup_src); + } + + if (edesc->mapped_dst_nents > 1) { + pdb->sgf |= RSA_PRIV_PDB_SGF_F; + pdb->f_dma = edesc->sec4_sg_dma + + sec4_sg_index * sizeof(struct sec4_sg_entry); + } else { + pdb->f_dma = sg_dma_address(req->dst); + } + + pdb->sgf |= (key->d_sz << RSA_PDB_D_SHIFT) | key->n_sz; + + return 0; +} + +static int set_rsa_priv_f2_pdb(struct akcipher_request *req, + struct rsa_edesc *edesc) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + struct caam_rsa_key *key = &ctx->key; + struct device *dev = ctx->dev; + struct rsa_priv_f2_pdb *pdb = &edesc->pdb.priv_f2; + int sec4_sg_index = 0; + size_t p_sz = key->p_sz; + size_t q_sz = key->q_sz; + + pdb->d_dma = dma_map_single(dev, key->d, key->d_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->d_dma)) { + dev_err(dev, "Unable to map RSA private exponent memory\n"); + return -ENOMEM; + } + + pdb->p_dma = dma_map_single(dev, key->p, p_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->p_dma)) { + dev_err(dev, "Unable to map RSA prime factor p memory\n"); + goto unmap_d; + } + + pdb->q_dma = dma_map_single(dev, key->q, q_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->q_dma)) { + dev_err(dev, "Unable to map RSA prime factor q memory\n"); + goto unmap_p; + } + + pdb->tmp1_dma = dma_map_single(dev, key->tmp1, p_sz, DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, pdb->tmp1_dma)) { + dev_err(dev, "Unable to map RSA tmp1 memory\n"); + goto unmap_q; + } + + pdb->tmp2_dma = dma_map_single(dev, key->tmp2, q_sz, DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, pdb->tmp2_dma)) { + dev_err(dev, "Unable to map RSA tmp2 memory\n"); + goto unmap_tmp1; + } + + if (edesc->mapped_src_nents > 1) { + pdb->sgf |= RSA_PRIV_PDB_SGF_G; + pdb->g_dma = edesc->sec4_sg_dma; + sec4_sg_index += edesc->mapped_src_nents; + } else { + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + + pdb->g_dma = sg_dma_address(req_ctx->fixup_src); + } + + if (edesc->mapped_dst_nents > 1) { + pdb->sgf |= RSA_PRIV_PDB_SGF_F; + pdb->f_dma = edesc->sec4_sg_dma + + sec4_sg_index * sizeof(struct sec4_sg_entry); + } else { + pdb->f_dma = sg_dma_address(req->dst); + } + + pdb->sgf |= (key->d_sz << RSA_PDB_D_SHIFT) | key->n_sz; + pdb->p_q_len = (q_sz << RSA_PDB_Q_SHIFT) | p_sz; + + return 0; + +unmap_tmp1: + dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_BIDIRECTIONAL); +unmap_q: + dma_unmap_single(dev, pdb->q_dma, q_sz, DMA_TO_DEVICE); +unmap_p: + dma_unmap_single(dev, pdb->p_dma, p_sz, DMA_TO_DEVICE); +unmap_d: + dma_unmap_single(dev, pdb->d_dma, key->d_sz, DMA_TO_DEVICE); + + return -ENOMEM; +} + +static int set_rsa_priv_f3_pdb(struct akcipher_request *req, + struct rsa_edesc *edesc) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + struct caam_rsa_key *key = &ctx->key; + struct device *dev = ctx->dev; + struct rsa_priv_f3_pdb *pdb = &edesc->pdb.priv_f3; + int sec4_sg_index = 0; + size_t p_sz = key->p_sz; + size_t q_sz = key->q_sz; + + pdb->p_dma = dma_map_single(dev, key->p, p_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->p_dma)) { + dev_err(dev, "Unable to map RSA prime factor p memory\n"); + return -ENOMEM; + } + + pdb->q_dma = dma_map_single(dev, key->q, q_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->q_dma)) { + dev_err(dev, "Unable to map RSA prime factor q memory\n"); + goto unmap_p; + } + + pdb->dp_dma = dma_map_single(dev, key->dp, p_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->dp_dma)) { + dev_err(dev, "Unable to map RSA exponent dp memory\n"); + goto unmap_q; + } + + pdb->dq_dma = dma_map_single(dev, key->dq, q_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->dq_dma)) { + dev_err(dev, "Unable to map RSA exponent dq memory\n"); + goto unmap_dp; + } + + pdb->c_dma = dma_map_single(dev, key->qinv, p_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->c_dma)) { + dev_err(dev, "Unable to map RSA CRT coefficient qinv memory\n"); + goto unmap_dq; + } + + pdb->tmp1_dma = dma_map_single(dev, key->tmp1, p_sz, DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, pdb->tmp1_dma)) { + dev_err(dev, "Unable to map RSA tmp1 memory\n"); + goto unmap_qinv; + } + + pdb->tmp2_dma = dma_map_single(dev, key->tmp2, q_sz, DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, pdb->tmp2_dma)) { + dev_err(dev, "Unable to map RSA tmp2 memory\n"); + goto unmap_tmp1; + } + + if (edesc->mapped_src_nents > 1) { + pdb->sgf |= RSA_PRIV_PDB_SGF_G; + pdb->g_dma = edesc->sec4_sg_dma; + sec4_sg_index += edesc->mapped_src_nents; + } else { + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + + pdb->g_dma = sg_dma_address(req_ctx->fixup_src); + } + + if (edesc->mapped_dst_nents > 1) { + pdb->sgf |= RSA_PRIV_PDB_SGF_F; + pdb->f_dma = edesc->sec4_sg_dma + + sec4_sg_index * sizeof(struct sec4_sg_entry); + } else { + pdb->f_dma = sg_dma_address(req->dst); + } + + pdb->sgf |= key->n_sz; + pdb->p_q_len = (q_sz << RSA_PDB_Q_SHIFT) | p_sz; + + return 0; + +unmap_tmp1: + dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_BIDIRECTIONAL); +unmap_qinv: + dma_unmap_single(dev, pdb->c_dma, p_sz, DMA_TO_DEVICE); +unmap_dq: + dma_unmap_single(dev, pdb->dq_dma, q_sz, DMA_TO_DEVICE); +unmap_dp: + dma_unmap_single(dev, pdb->dp_dma, p_sz, DMA_TO_DEVICE); +unmap_q: + dma_unmap_single(dev, pdb->q_dma, q_sz, DMA_TO_DEVICE); +unmap_p: + dma_unmap_single(dev, pdb->p_dma, p_sz, DMA_TO_DEVICE); + + return -ENOMEM; +} + +static int akcipher_enqueue_req(struct device *jrdev, + void (*cbk)(struct device *jrdev, u32 *desc, + u32 err, void *context), + struct akcipher_request *req) +{ + struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev); + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + struct caam_rsa_key *key = &ctx->key; + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + struct rsa_edesc *edesc = req_ctx->edesc; + u32 *desc = edesc->hw_desc; + int ret; + + req_ctx->akcipher_op_done = cbk; + /* + * Only the backlog request are sent to crypto-engine since the others + * can be handled by CAAM, if free, especially since JR has up to 1024 + * entries (more than the 10 entries from crypto-engine). + */ + if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG) + ret = crypto_transfer_akcipher_request_to_engine(jrpriv->engine, + req); + else + ret = caam_jr_enqueue(jrdev, desc, cbk, req); + + if ((ret != -EINPROGRESS) && (ret != -EBUSY)) { + switch (key->priv_form) { + case FORM1: + rsa_priv_f1_unmap(jrdev, edesc, req); + break; + case FORM2: + rsa_priv_f2_unmap(jrdev, edesc, req); + break; + case FORM3: + rsa_priv_f3_unmap(jrdev, edesc, req); + break; + default: + rsa_pub_unmap(jrdev, edesc, req); + } + rsa_io_unmap(jrdev, edesc, req); + kfree(edesc); + } + + return ret; +} + +static int caam_rsa_enc(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + struct caam_rsa_key *key = &ctx->key; + struct device *jrdev = ctx->dev; + struct rsa_edesc *edesc; + int ret; + + if (unlikely(!key->n || !key->e)) + return -EINVAL; + + if (req->dst_len < key->n_sz) { + req->dst_len = key->n_sz; + dev_err(jrdev, "Output buffer length less than parameter n\n"); + return -EOVERFLOW; + } + + /* Allocate extended descriptor */ + edesc = rsa_edesc_alloc(req, DESC_RSA_PUB_LEN); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + /* Set RSA Encrypt Protocol Data Block */ + ret = set_rsa_pub_pdb(req, edesc); + if (ret) + goto init_fail; + + /* Initialize Job Descriptor */ + init_rsa_pub_desc(edesc->hw_desc, &edesc->pdb.pub); + + return akcipher_enqueue_req(jrdev, rsa_pub_done, req); + +init_fail: + rsa_io_unmap(jrdev, edesc, req); + kfree(edesc); + return ret; +} + +static int caam_rsa_dec_priv_f1(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + struct device *jrdev = ctx->dev; + struct rsa_edesc *edesc; + int ret; + + /* Allocate extended descriptor */ + edesc = rsa_edesc_alloc(req, DESC_RSA_PRIV_F1_LEN); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + /* Set RSA Decrypt Protocol Data Block - Private Key Form #1 */ + ret = set_rsa_priv_f1_pdb(req, edesc); + if (ret) + goto init_fail; + + /* Initialize Job Descriptor */ + init_rsa_priv_f1_desc(edesc->hw_desc, &edesc->pdb.priv_f1); + + return akcipher_enqueue_req(jrdev, rsa_priv_f_done, req); + +init_fail: + rsa_io_unmap(jrdev, edesc, req); + kfree(edesc); + return ret; +} + +static int caam_rsa_dec_priv_f2(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + struct device *jrdev = ctx->dev; + struct rsa_edesc *edesc; + int ret; + + /* Allocate extended descriptor */ + edesc = rsa_edesc_alloc(req, DESC_RSA_PRIV_F2_LEN); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + /* Set RSA Decrypt Protocol Data Block - Private Key Form #2 */ + ret = set_rsa_priv_f2_pdb(req, edesc); + if (ret) + goto init_fail; + + /* Initialize Job Descriptor */ + init_rsa_priv_f2_desc(edesc->hw_desc, &edesc->pdb.priv_f2); + + return akcipher_enqueue_req(jrdev, rsa_priv_f_done, req); + +init_fail: + rsa_io_unmap(jrdev, edesc, req); + kfree(edesc); + return ret; +} + +static int caam_rsa_dec_priv_f3(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + struct device *jrdev = ctx->dev; + struct rsa_edesc *edesc; + int ret; + + /* Allocate extended descriptor */ + edesc = rsa_edesc_alloc(req, DESC_RSA_PRIV_F3_LEN); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + /* Set RSA Decrypt Protocol Data Block - Private Key Form #3 */ + ret = set_rsa_priv_f3_pdb(req, edesc); + if (ret) + goto init_fail; + + /* Initialize Job Descriptor */ + init_rsa_priv_f3_desc(edesc->hw_desc, &edesc->pdb.priv_f3); + + return akcipher_enqueue_req(jrdev, rsa_priv_f_done, req); + +init_fail: + rsa_io_unmap(jrdev, edesc, req); + kfree(edesc); + return ret; +} + +static int caam_rsa_dec(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + struct caam_rsa_key *key = &ctx->key; + int ret; + + if (unlikely(!key->n || !key->d)) + return -EINVAL; + + if (req->dst_len < key->n_sz) { + req->dst_len = key->n_sz; + dev_err(ctx->dev, "Output buffer length less than parameter n\n"); + return -EOVERFLOW; + } + + if (key->priv_form == FORM3) + ret = caam_rsa_dec_priv_f3(req); + else if (key->priv_form == FORM2) + ret = caam_rsa_dec_priv_f2(req); + else + ret = caam_rsa_dec_priv_f1(req); + + return ret; +} + +static void caam_rsa_free_key(struct caam_rsa_key *key) +{ + kfree_sensitive(key->d); + kfree_sensitive(key->p); + kfree_sensitive(key->q); + kfree_sensitive(key->dp); + kfree_sensitive(key->dq); + kfree_sensitive(key->qinv); + kfree_sensitive(key->tmp1); + kfree_sensitive(key->tmp2); + kfree(key->e); + kfree(key->n); + memset(key, 0, sizeof(*key)); +} + +static void caam_rsa_drop_leading_zeros(const u8 **ptr, size_t *nbytes) +{ + while (!**ptr && *nbytes) { + (*ptr)++; + (*nbytes)--; + } +} + +/** + * caam_read_rsa_crt - Used for reading dP, dQ, qInv CRT members. + * dP, dQ and qInv could decode to less than corresponding p, q length, as the + * BER-encoding requires that the minimum number of bytes be used to encode the + * integer. dP, dQ, qInv decoded values have to be zero-padded to appropriate + * length. + * + * @ptr : pointer to {dP, dQ, qInv} CRT member + * @nbytes: length in bytes of {dP, dQ, qInv} CRT member + * @dstlen: length in bytes of corresponding p or q prime factor + */ +static u8 *caam_read_rsa_crt(const u8 *ptr, size_t nbytes, size_t dstlen) +{ + u8 *dst; + + caam_rsa_drop_leading_zeros(&ptr, &nbytes); + if (!nbytes) + return NULL; + + dst = kzalloc(dstlen, GFP_KERNEL); + if (!dst) + return NULL; + + memcpy(dst + (dstlen - nbytes), ptr, nbytes); + + return dst; +} + +/** + * caam_read_raw_data - Read a raw byte stream as a positive integer. + * The function skips buffer's leading zeros, copies the remained data + * to a buffer allocated in the GFP_KERNEL zone and returns + * the address of the new buffer. + * + * @buf : The data to read + * @nbytes: The amount of data to read + */ +static inline u8 *caam_read_raw_data(const u8 *buf, size_t *nbytes) +{ + + caam_rsa_drop_leading_zeros(&buf, nbytes); + if (!*nbytes) + return NULL; + + return kmemdup(buf, *nbytes, GFP_KERNEL); +} + +static int caam_rsa_check_key_length(unsigned int len) +{ + if (len > 4096) + return -EINVAL; + return 0; +} + +static int caam_rsa_set_pub_key(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + struct rsa_key raw_key = {NULL}; + struct caam_rsa_key *rsa_key = &ctx->key; + int ret; + + /* Free the old RSA key if any */ + caam_rsa_free_key(rsa_key); + + ret = rsa_parse_pub_key(&raw_key, key, keylen); + if (ret) + return ret; + + /* Copy key in DMA zone */ + rsa_key->e = kmemdup(raw_key.e, raw_key.e_sz, GFP_KERNEL); + if (!rsa_key->e) + goto err; + + /* + * Skip leading zeros and copy the positive integer to a buffer + * allocated in the GFP_KERNEL zone. The decryption descriptor + * expects a positive integer for the RSA modulus and uses its length as + * decryption output length. + */ + rsa_key->n = caam_read_raw_data(raw_key.n, &raw_key.n_sz); + if (!rsa_key->n) + goto err; + + if (caam_rsa_check_key_length(raw_key.n_sz << 3)) { + caam_rsa_free_key(rsa_key); + return -EINVAL; + } + + rsa_key->e_sz = raw_key.e_sz; + rsa_key->n_sz = raw_key.n_sz; + + return 0; +err: + caam_rsa_free_key(rsa_key); + return -ENOMEM; +} + +static int caam_rsa_set_priv_key_form(struct caam_rsa_ctx *ctx, + struct rsa_key *raw_key) +{ + struct caam_rsa_key *rsa_key = &ctx->key; + size_t p_sz = raw_key->p_sz; + size_t q_sz = raw_key->q_sz; + unsigned aligned_size; + + rsa_key->p = caam_read_raw_data(raw_key->p, &p_sz); + if (!rsa_key->p) + return -ENOMEM; + rsa_key->p_sz = p_sz; + + rsa_key->q = caam_read_raw_data(raw_key->q, &q_sz); + if (!rsa_key->q) + goto free_p; + rsa_key->q_sz = q_sz; + + aligned_size = ALIGN(raw_key->p_sz, dma_get_cache_alignment()); + rsa_key->tmp1 = kzalloc(aligned_size, GFP_KERNEL); + if (!rsa_key->tmp1) + goto free_q; + + aligned_size = ALIGN(raw_key->q_sz, dma_get_cache_alignment()); + rsa_key->tmp2 = kzalloc(aligned_size, GFP_KERNEL); + if (!rsa_key->tmp2) + goto free_tmp1; + + rsa_key->priv_form = FORM2; + + rsa_key->dp = caam_read_rsa_crt(raw_key->dp, raw_key->dp_sz, p_sz); + if (!rsa_key->dp) + goto free_tmp2; + + rsa_key->dq = caam_read_rsa_crt(raw_key->dq, raw_key->dq_sz, q_sz); + if (!rsa_key->dq) + goto free_dp; + + rsa_key->qinv = caam_read_rsa_crt(raw_key->qinv, raw_key->qinv_sz, + q_sz); + if (!rsa_key->qinv) + goto free_dq; + + rsa_key->priv_form = FORM3; + + return 0; + +free_dq: + kfree_sensitive(rsa_key->dq); +free_dp: + kfree_sensitive(rsa_key->dp); +free_tmp2: + kfree_sensitive(rsa_key->tmp2); +free_tmp1: + kfree_sensitive(rsa_key->tmp1); +free_q: + kfree_sensitive(rsa_key->q); +free_p: + kfree_sensitive(rsa_key->p); + return -ENOMEM; +} + +static int caam_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + struct rsa_key raw_key = {NULL}; + struct caam_rsa_key *rsa_key = &ctx->key; + int ret; + + /* Free the old RSA key if any */ + caam_rsa_free_key(rsa_key); + + ret = rsa_parse_priv_key(&raw_key, key, keylen); + if (ret) + return ret; + + /* Copy key in DMA zone */ + rsa_key->d = kmemdup(raw_key.d, raw_key.d_sz, GFP_KERNEL); + if (!rsa_key->d) + goto err; + + rsa_key->e = kmemdup(raw_key.e, raw_key.e_sz, GFP_KERNEL); + if (!rsa_key->e) + goto err; + + /* + * Skip leading zeros and copy the positive integer to a buffer + * allocated in the GFP_KERNEL zone. The decryption descriptor + * expects a positive integer for the RSA modulus and uses its length as + * decryption output length. + */ + rsa_key->n = caam_read_raw_data(raw_key.n, &raw_key.n_sz); + if (!rsa_key->n) + goto err; + + if (caam_rsa_check_key_length(raw_key.n_sz << 3)) { + caam_rsa_free_key(rsa_key); + return -EINVAL; + } + + rsa_key->d_sz = raw_key.d_sz; + rsa_key->e_sz = raw_key.e_sz; + rsa_key->n_sz = raw_key.n_sz; + + ret = caam_rsa_set_priv_key_form(ctx, &raw_key); + if (ret) + goto err; + + return 0; + +err: + caam_rsa_free_key(rsa_key); + return -ENOMEM; +} + +static unsigned int caam_rsa_max_size(struct crypto_akcipher *tfm) +{ + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + + return ctx->key.n_sz; +} + +/* Per session pkc's driver context creation function */ +static int caam_rsa_init_tfm(struct crypto_akcipher *tfm) +{ + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + + akcipher_set_reqsize(tfm, sizeof(struct caam_rsa_req_ctx)); + + ctx->dev = caam_jr_alloc(); + + if (IS_ERR(ctx->dev)) { + pr_err("Job Ring Device allocation for transform failed\n"); + return PTR_ERR(ctx->dev); + } + + ctx->padding_dma = dma_map_single(ctx->dev, zero_buffer, + CAAM_RSA_MAX_INPUT_SIZE - 1, + DMA_TO_DEVICE); + if (dma_mapping_error(ctx->dev, ctx->padding_dma)) { + dev_err(ctx->dev, "unable to map padding\n"); + caam_jr_free(ctx->dev); + return -ENOMEM; + } + + return 0; +} + +/* Per session pkc's driver context cleanup function */ +static void caam_rsa_exit_tfm(struct crypto_akcipher *tfm) +{ + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + struct caam_rsa_key *key = &ctx->key; + + dma_unmap_single(ctx->dev, ctx->padding_dma, CAAM_RSA_MAX_INPUT_SIZE - + 1, DMA_TO_DEVICE); + caam_rsa_free_key(key); + caam_jr_free(ctx->dev); +} + +static struct caam_akcipher_alg caam_rsa = { + .akcipher.base = { + .encrypt = caam_rsa_enc, + .decrypt = caam_rsa_dec, + .set_pub_key = caam_rsa_set_pub_key, + .set_priv_key = caam_rsa_set_priv_key, + .max_size = caam_rsa_max_size, + .init = caam_rsa_init_tfm, + .exit = caam_rsa_exit_tfm, + .base = { + .cra_name = "rsa", + .cra_driver_name = "rsa-caam", + .cra_priority = 3000, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct caam_rsa_ctx) + + CRYPTO_DMA_PADDING, + }, + }, + .akcipher.op = { + .do_one_request = akcipher_do_one_req, + }, +}; + +/* Public Key Cryptography module initialization handler */ +int caam_pkc_init(struct device *ctrldev) +{ + struct caam_drv_private *priv = dev_get_drvdata(ctrldev); + u32 pk_inst, pkha; + int err; + init_done = false; + + /* Determine public key hardware accelerator presence. */ + if (priv->era < 10) { + pk_inst = (rd_reg32(&priv->jr[0]->perfmon.cha_num_ls) & + CHA_ID_LS_PK_MASK) >> CHA_ID_LS_PK_SHIFT; + } else { + pkha = rd_reg32(&priv->jr[0]->vreg.pkha); + pk_inst = pkha & CHA_VER_NUM_MASK; + + /* + * Newer CAAMs support partially disabled functionality. If this is the + * case, the number is non-zero, but this bit is set to indicate that + * no encryption or decryption is supported. Only signing and verifying + * is supported. + */ + if (pkha & CHA_VER_MISC_PKHA_NO_CRYPT) + pk_inst = 0; + } + + /* Do not register algorithms if PKHA is not present. */ + if (!pk_inst) + return 0; + + /* allocate zero buffer, used for padding input */ + zero_buffer = kzalloc(CAAM_RSA_MAX_INPUT_SIZE - 1, GFP_KERNEL); + if (!zero_buffer) + return -ENOMEM; + + err = crypto_engine_register_akcipher(&caam_rsa.akcipher); + + if (err) { + kfree(zero_buffer); + dev_warn(ctrldev, "%s alg registration failed\n", + caam_rsa.akcipher.base.base.cra_driver_name); + } else { + init_done = true; + caam_rsa.registered = true; + dev_info(ctrldev, "caam pkc algorithms registered in /proc/crypto\n"); + } + + return err; +} + +void caam_pkc_exit(void) +{ + if (!init_done) + return; + + if (caam_rsa.registered) + crypto_engine_unregister_akcipher(&caam_rsa.akcipher); + + kfree(zero_buffer); +} diff --git a/drivers/crypto/caam/caampkc.h b/drivers/crypto/caam/caampkc.h new file mode 100644 index 000000000..96d03704c --- /dev/null +++ b/drivers/crypto/caam/caampkc.h @@ -0,0 +1,155 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * caam - Freescale FSL CAAM support for Public Key Cryptography descriptors + * + * Copyright 2016 Freescale Semiconductor, Inc. + * + * There is no Shared Descriptor for PKC so that the Job Descriptor must carry + * all the desired key parameters, input and output pointers. + */ + +#ifndef _PKC_DESC_H_ +#define _PKC_DESC_H_ +#include "compat.h" +#include "pdb.h" + +/** + * caam_priv_key_form - CAAM RSA private key representation + * CAAM RSA private key may have either of three forms. + * + * 1. The first representation consists of the pair (n, d), where the + * components have the following meanings: + * n the RSA modulus + * d the RSA private exponent + * + * 2. The second representation consists of the triplet (p, q, d), where the + * components have the following meanings: + * p the first prime factor of the RSA modulus n + * q the second prime factor of the RSA modulus n + * d the RSA private exponent + * + * 3. The third representation consists of the quintuple (p, q, dP, dQ, qInv), + * where the components have the following meanings: + * p the first prime factor of the RSA modulus n + * q the second prime factor of the RSA modulus n + * dP the first factors's CRT exponent + * dQ the second factors's CRT exponent + * qInv the (first) CRT coefficient + * + * The benefit of using the third or the second key form is lower computational + * cost for the decryption and signature operations. + */ +enum caam_priv_key_form { + FORM1, + FORM2, + FORM3 +}; + +/** + * caam_rsa_key - CAAM RSA key structure. Keys are allocated in DMA zone. + * @n : RSA modulus raw byte stream + * @e : RSA public exponent raw byte stream + * @d : RSA private exponent raw byte stream + * @p : RSA prime factor p of RSA modulus n + * @q : RSA prime factor q of RSA modulus n + * @dp : RSA CRT exponent of p + * @dp : RSA CRT exponent of q + * @qinv : RSA CRT coefficient + * @tmp1 : CAAM uses this temporary buffer as internal state buffer. + * It is assumed to be as long as p. + * @tmp2 : CAAM uses this temporary buffer as internal state buffer. + * It is assumed to be as long as q. + * @n_sz : length in bytes of RSA modulus n + * @e_sz : length in bytes of RSA public exponent + * @d_sz : length in bytes of RSA private exponent + * @p_sz : length in bytes of RSA prime factor p of RSA modulus n + * @q_sz : length in bytes of RSA prime factor q of RSA modulus n + * @priv_form : CAAM RSA private key representation + */ +struct caam_rsa_key { + u8 *n; + u8 *e; + u8 *d; + u8 *p; + u8 *q; + u8 *dp; + u8 *dq; + u8 *qinv; + u8 *tmp1; + u8 *tmp2; + size_t n_sz; + size_t e_sz; + size_t d_sz; + size_t p_sz; + size_t q_sz; + enum caam_priv_key_form priv_form; +}; + +/** + * caam_rsa_ctx - per session context. + * @key : RSA key in DMA zone + * @dev : device structure + * @padding_dma : dma address of padding, for adding it to the input + */ +struct caam_rsa_ctx { + struct caam_rsa_key key; + struct device *dev; + dma_addr_t padding_dma; + +}; + +/** + * caam_rsa_req_ctx - per request context. + * @src : input scatterlist (stripped of leading zeros) + * @fixup_src : input scatterlist (that might be stripped of leading zeros) + * @fixup_src_len : length of the fixup_src input scatterlist + * @edesc : s/w-extended rsa descriptor + * @akcipher_op_done : callback used when operation is done + */ +struct caam_rsa_req_ctx { + struct scatterlist src[2]; + struct scatterlist *fixup_src; + unsigned int fixup_src_len; + struct rsa_edesc *edesc; + void (*akcipher_op_done)(struct device *jrdev, u32 *desc, u32 err, + void *context); +}; + +/** + * rsa_edesc - s/w-extended rsa descriptor + * @src_nents : number of segments in input s/w scatterlist + * @dst_nents : number of segments in output s/w scatterlist + * @mapped_src_nents: number of segments in input h/w link table + * @mapped_dst_nents: number of segments in output h/w link table + * @sec4_sg_bytes : length of h/w link table + * @bklog : stored to determine if the request needs backlog + * @sec4_sg_dma : dma address of h/w link table + * @sec4_sg : pointer to h/w link table + * @pdb : specific RSA Protocol Data Block (PDB) + * @hw_desc : descriptor followed by link tables if any + */ +struct rsa_edesc { + int src_nents; + int dst_nents; + int mapped_src_nents; + int mapped_dst_nents; + int sec4_sg_bytes; + bool bklog; + dma_addr_t sec4_sg_dma; + struct sec4_sg_entry *sec4_sg; + union { + struct rsa_pub_pdb pub; + struct rsa_priv_f1_pdb priv_f1; + struct rsa_priv_f2_pdb priv_f2; + struct rsa_priv_f3_pdb priv_f3; + } pdb; + u32 hw_desc[]; +}; + +/* Descriptor construction primitives. */ +void init_rsa_pub_desc(u32 *desc, struct rsa_pub_pdb *pdb); +void init_rsa_priv_f1_desc(u32 *desc, struct rsa_priv_f1_pdb *pdb); +void init_rsa_priv_f2_desc(u32 *desc, struct rsa_priv_f2_pdb *pdb); +void init_rsa_priv_f3_desc(u32 *desc, struct rsa_priv_f3_pdb *pdb); + +#endif diff --git a/drivers/crypto/caam/caamrng.c b/drivers/crypto/caam/caamrng.c new file mode 100644 index 000000000..0eb43c862 --- /dev/null +++ b/drivers/crypto/caam/caamrng.c @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * caam - Freescale FSL CAAM support for hw_random + * + * Copyright 2011 Freescale Semiconductor, Inc. + * Copyright 2018-2019, 2023 NXP + * + * Based on caamalg.c crypto API driver. + * + */ + +#include +#include +#include +#include +#include +#include + +#include "compat.h" + +#include "regs.h" +#include "intern.h" +#include "desc_constr.h" +#include "jr.h" +#include "error.h" + +#define CAAM_RNG_MAX_FIFO_STORE_SIZE 16 + +/* + * Length of used descriptors, see caam_init_desc() + */ +#define CAAM_RNG_DESC_LEN (CAAM_CMD_SZ + \ + CAAM_CMD_SZ + \ + CAAM_CMD_SZ + CAAM_PTR_SZ_MAX) + +/* rng per-device context */ +struct caam_rng_ctx { + struct hwrng rng; + struct device *jrdev; + struct device *ctrldev; + void *desc_async; + void *desc_sync; + struct work_struct worker; + struct kfifo fifo; +}; + +struct caam_rng_job_ctx { + struct completion *done; + int *err; +}; + +static struct caam_rng_ctx *to_caam_rng_ctx(struct hwrng *r) +{ + return (struct caam_rng_ctx *)r->priv; +} + +static void caam_rng_done(struct device *jrdev, u32 *desc, u32 err, + void *context) +{ + struct caam_rng_job_ctx *jctx = context; + + if (err) + *jctx->err = caam_jr_strstatus(jrdev, err); + + complete(jctx->done); +} + +static u32 *caam_init_desc(u32 *desc, dma_addr_t dst_dma) +{ + init_job_desc(desc, 0); /* + 1 cmd_sz */ + /* Generate random bytes: + 1 cmd_sz */ + append_operation(desc, OP_ALG_ALGSEL_RNG | OP_TYPE_CLASS1_ALG | + OP_ALG_PR_ON); + /* Store bytes: + 1 cmd_sz + caam_ptr_sz */ + append_fifo_store(desc, dst_dma, + CAAM_RNG_MAX_FIFO_STORE_SIZE, FIFOST_TYPE_RNGSTORE); + + print_hex_dump_debug("rng job desc@: ", DUMP_PREFIX_ADDRESS, + 16, 4, desc, desc_bytes(desc), 1); + + return desc; +} + +static int caam_rng_read_one(struct device *jrdev, + void *dst, int len, + void *desc, + struct completion *done) +{ + dma_addr_t dst_dma; + int err, ret = 0; + struct caam_rng_job_ctx jctx = { + .done = done, + .err = &ret, + }; + + len = CAAM_RNG_MAX_FIFO_STORE_SIZE; + + dst_dma = dma_map_single(jrdev, dst, len, DMA_FROM_DEVICE); + if (dma_mapping_error(jrdev, dst_dma)) { + dev_err(jrdev, "unable to map destination memory\n"); + return -ENOMEM; + } + + init_completion(done); + err = caam_jr_enqueue(jrdev, + caam_init_desc(desc, dst_dma), + caam_rng_done, &jctx); + if (err == -EINPROGRESS) { + wait_for_completion(done); + err = 0; + } + + dma_unmap_single(jrdev, dst_dma, len, DMA_FROM_DEVICE); + + return err ?: (ret ?: len); +} + +static void caam_rng_fill_async(struct caam_rng_ctx *ctx) +{ + struct scatterlist sg[1]; + struct completion done; + int len, nents; + + sg_init_table(sg, ARRAY_SIZE(sg)); + nents = kfifo_dma_in_prepare(&ctx->fifo, sg, ARRAY_SIZE(sg), + CAAM_RNG_MAX_FIFO_STORE_SIZE); + if (!nents) + return; + + len = caam_rng_read_one(ctx->jrdev, sg_virt(&sg[0]), + sg[0].length, + ctx->desc_async, + &done); + if (len < 0) + return; + + kfifo_dma_in_finish(&ctx->fifo, len); +} + +static void caam_rng_worker(struct work_struct *work) +{ + struct caam_rng_ctx *ctx = container_of(work, struct caam_rng_ctx, + worker); + caam_rng_fill_async(ctx); +} + +static int caam_read(struct hwrng *rng, void *dst, size_t max, bool wait) +{ + struct caam_rng_ctx *ctx = to_caam_rng_ctx(rng); + int out; + + if (wait) { + struct completion done; + + return caam_rng_read_one(ctx->jrdev, dst, max, + ctx->desc_sync, &done); + } + + out = kfifo_out(&ctx->fifo, dst, max); + if (kfifo_is_empty(&ctx->fifo)) + schedule_work(&ctx->worker); + + return out; +} + +static void caam_cleanup(struct hwrng *rng) +{ + struct caam_rng_ctx *ctx = to_caam_rng_ctx(rng); + + flush_work(&ctx->worker); + caam_jr_free(ctx->jrdev); + kfifo_free(&ctx->fifo); +} + +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_TEST +static inline void test_len(struct hwrng *rng, size_t len, bool wait) +{ + u8 *buf; + int read_len; + struct caam_rng_ctx *ctx = to_caam_rng_ctx(rng); + struct device *dev = ctx->ctrldev; + + buf = kcalloc(CAAM_RNG_MAX_FIFO_STORE_SIZE, sizeof(u8), GFP_KERNEL); + if (!buf) { + return; + } + while (len > 0) { + read_len = rng->read(rng, buf, len, wait); + + if (read_len < 0 || (read_len == 0 && wait)) { + dev_err(dev, "RNG Read FAILED received %d bytes\n", + read_len); + kfree(buf); + return; + } + + print_hex_dump_debug("random bytes@: ", + DUMP_PREFIX_ADDRESS, 16, 4, + buf, read_len, 1); + + len = len - read_len; + } + + kfree(buf); +} + +static inline void test_mode_once(struct hwrng *rng, bool wait) +{ + test_len(rng, 32, wait); + test_len(rng, 64, wait); + test_len(rng, 128, wait); +} + +static void self_test(struct hwrng *rng) +{ + pr_info("Executing RNG SELF-TEST with wait\n"); + test_mode_once(rng, true); +} +#endif + +static int caam_init(struct hwrng *rng) +{ + struct caam_rng_ctx *ctx = to_caam_rng_ctx(rng); + int err; + + ctx->desc_sync = devm_kzalloc(ctx->ctrldev, CAAM_RNG_DESC_LEN, + GFP_KERNEL); + if (!ctx->desc_sync) + return -ENOMEM; + + ctx->desc_async = devm_kzalloc(ctx->ctrldev, CAAM_RNG_DESC_LEN, + GFP_KERNEL); + if (!ctx->desc_async) + return -ENOMEM; + + if (kfifo_alloc(&ctx->fifo, ALIGN(CAAM_RNG_MAX_FIFO_STORE_SIZE, + dma_get_cache_alignment()), + GFP_KERNEL)) + return -ENOMEM; + + INIT_WORK(&ctx->worker, caam_rng_worker); + + ctx->jrdev = caam_jr_alloc(); + err = PTR_ERR_OR_ZERO(ctx->jrdev); + if (err) { + kfifo_free(&ctx->fifo); + pr_err("Job Ring Device allocation for transform failed\n"); + return err; + } + + /* + * Fill async buffer to have early randomness data for + * hw_random + */ + caam_rng_fill_async(ctx); + + return 0; +} + +int caam_rng_init(struct device *ctrldev); + +void caam_rng_exit(struct device *ctrldev) +{ + devres_release_group(ctrldev, caam_rng_init); +} + +int caam_rng_init(struct device *ctrldev) +{ + struct caam_rng_ctx *ctx; + u32 rng_inst; + struct caam_drv_private *priv = dev_get_drvdata(ctrldev); + int ret; + + /* Check for an instantiated RNG before registration */ + if (priv->era < 10) + rng_inst = (rd_reg32(&priv->jr[0]->perfmon.cha_num_ls) & + CHA_ID_LS_RNG_MASK) >> CHA_ID_LS_RNG_SHIFT; + else + rng_inst = rd_reg32(&priv->jr[0]->vreg.rng) & CHA_VER_NUM_MASK; + + if (!rng_inst) + return 0; + + if (!devres_open_group(ctrldev, caam_rng_init, GFP_KERNEL)) + return -ENOMEM; + + ctx = devm_kzalloc(ctrldev, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + ctx->ctrldev = ctrldev; + + ctx->rng.name = "rng-caam"; + ctx->rng.init = caam_init; + ctx->rng.cleanup = caam_cleanup; + ctx->rng.read = caam_read; + ctx->rng.priv = (unsigned long)ctx; + + dev_info(ctrldev, "registering rng-caam\n"); + + ret = devm_hwrng_register(ctrldev, &ctx->rng); + if (ret) { + caam_rng_exit(ctrldev); + return ret; + } + +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_TEST + self_test(&ctx->rng); +#endif + + devres_close_group(ctrldev, caam_rng_init); + return 0; +} diff --git a/drivers/crypto/caam/compat.h b/drivers/crypto/caam/compat.h new file mode 100644 index 000000000..c4f797641 --- /dev/null +++ b/drivers/crypto/caam/compat.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright 2008-2011 Freescale Semiconductor, Inc. + */ + +#ifndef CAAM_COMPAT_H +#define CAAM_COMPAT_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#endif /* !defined(CAAM_COMPAT_H) */ diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c new file mode 100644 index 000000000..320be5d77 --- /dev/null +++ b/drivers/crypto/caam/ctrl.c @@ -0,0 +1,1172 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* * CAAM control-plane driver backend + * Controller-level driver, kernel property detection, initialization + * + * Copyright 2008-2012 Freescale Semiconductor, Inc. + * Copyright 2018-2019, 2023 NXP + */ + +#include +#include +#include +#include +#include +#include + +#include "compat.h" +#include "debugfs.h" +#include "regs.h" +#include "intern.h" +#include "jr.h" +#include "desc_constr.h" +#include "ctrl.h" + +bool caam_dpaa2; +EXPORT_SYMBOL(caam_dpaa2); + +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI +#include "qi.h" +#endif + +/* + * Descriptor to instantiate RNG State Handle 0 in normal mode and + * load the JDKEK, TDKEK and TDSK registers + */ +static void build_instantiation_desc(u32 *desc, int handle, int do_sk) +{ + u32 *jump_cmd, op_flags; + + init_job_desc(desc, 0); + + op_flags = OP_TYPE_CLASS1_ALG | OP_ALG_ALGSEL_RNG | + (handle << OP_ALG_AAI_SHIFT) | OP_ALG_AS_INIT | + OP_ALG_PR_ON; + + /* INIT RNG in non-test mode */ + append_operation(desc, op_flags); + + if (!handle && do_sk) { + /* + * For SH0, Secure Keys must be generated as well + */ + + /* wait for done */ + jump_cmd = append_jump(desc, JUMP_CLASS_CLASS1); + set_jump_tgt_here(desc, jump_cmd); + + /* + * load 1 to clear written reg: + * resets the done interrupt and returns the RNG to idle. + */ + append_load_imm_u32(desc, 1, LDST_SRCDST_WORD_CLRW); + + /* Initialize State Handle */ + append_operation(desc, OP_TYPE_CLASS1_ALG | OP_ALG_ALGSEL_RNG | + OP_ALG_AAI_RNG4_SK); + } + + append_jump(desc, JUMP_CLASS_CLASS1 | JUMP_TYPE_HALT); +} + +/* Descriptor for deinstantiation of State Handle 0 of the RNG block. */ +static void build_deinstantiation_desc(u32 *desc, int handle) +{ + init_job_desc(desc, 0); + + /* Uninstantiate State Handle 0 */ + append_operation(desc, OP_TYPE_CLASS1_ALG | OP_ALG_ALGSEL_RNG | + (handle << OP_ALG_AAI_SHIFT) | OP_ALG_AS_INITFINAL); + + append_jump(desc, JUMP_CLASS_CLASS1 | JUMP_TYPE_HALT); +} + +#ifdef CONFIG_OF +static const struct of_device_id imx8m_machine_match[] = { + { .compatible = "fsl,imx8mm", }, + { .compatible = "fsl,imx8mn", }, + { .compatible = "fsl,imx8mp", }, + { .compatible = "fsl,imx8mq", }, + { .compatible = "fsl,imx8ulp", }, + { } +}; +#endif + +/* + * run_descriptor_deco0 - runs a descriptor on DECO0, under direct control of + * the software (no JR/QI used). + * @ctrldev - pointer to device + * @status - descriptor status, after being run + * + * Return: - 0 if no error occurred + * - -ENODEV if the DECO couldn't be acquired + * - -EAGAIN if an error occurred while executing the descriptor + */ +static inline int run_descriptor_deco0(struct device *ctrldev, u32 *desc, + u32 *status) +{ + struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev); + struct caam_ctrl __iomem *ctrl = ctrlpriv->ctrl; + struct caam_deco __iomem *deco = ctrlpriv->deco; + unsigned int timeout = 100000; + u32 deco_dbg_reg, deco_state, flags; + int i; + + + if (ctrlpriv->virt_en == 1 || + /* + * Apparently on i.MX8M{Q,M,N,P} it doesn't matter if virt_en == 1 + * and the following steps should be performed regardless + */ + of_match_node(imx8m_machine_match, of_root)) { + clrsetbits_32(&ctrl->deco_rsr, 0, DECORSR_JR0); + + while (!(rd_reg32(&ctrl->deco_rsr) & DECORSR_VALID) && + --timeout) + cpu_relax(); + + timeout = 100000; + } + + clrsetbits_32(&ctrl->deco_rq, 0, DECORR_RQD0ENABLE); + + while (!(rd_reg32(&ctrl->deco_rq) & DECORR_DEN0) && + --timeout) + cpu_relax(); + + if (!timeout) { + dev_err(ctrldev, "failed to acquire DECO 0\n"); + clrsetbits_32(&ctrl->deco_rq, DECORR_RQD0ENABLE, 0); + return -ENODEV; + } + + for (i = 0; i < desc_len(desc); i++) + wr_reg32(&deco->descbuf[i], caam32_to_cpu(*(desc + i))); + + flags = DECO_JQCR_WHL; + /* + * If the descriptor length is longer than 4 words, then the + * FOUR bit in JRCTRL register must be set. + */ + if (desc_len(desc) >= 4) + flags |= DECO_JQCR_FOUR; + + /* Instruct the DECO to execute it */ + clrsetbits_32(&deco->jr_ctl_hi, 0, flags); + + timeout = 10000000; + do { + deco_dbg_reg = rd_reg32(&deco->desc_dbg); + + if (ctrlpriv->era < 10) + deco_state = (deco_dbg_reg & DESC_DBG_DECO_STAT_MASK) >> + DESC_DBG_DECO_STAT_SHIFT; + else + deco_state = (rd_reg32(&deco->dbg_exec) & + DESC_DER_DECO_STAT_MASK) >> + DESC_DER_DECO_STAT_SHIFT; + + /* + * If an error occurred in the descriptor, then + * the DECO status field will be set to 0x0D + */ + if (deco_state == DECO_STAT_HOST_ERR) + break; + + cpu_relax(); + } while ((deco_dbg_reg & DESC_DBG_DECO_STAT_VALID) && --timeout); + + *status = rd_reg32(&deco->op_status_hi) & + DECO_OP_STATUS_HI_ERR_MASK; + + if (ctrlpriv->virt_en == 1) + clrsetbits_32(&ctrl->deco_rsr, DECORSR_JR0, 0); + + /* Mark the DECO as free */ + clrsetbits_32(&ctrl->deco_rq, DECORR_RQD0ENABLE, 0); + + if (!timeout) + return -EAGAIN; + + return 0; +} + +/* + * deinstantiate_rng - builds and executes a descriptor on DECO0, + * which deinitializes the RNG block. + * @ctrldev - pointer to device + * @state_handle_mask - bitmask containing the instantiation status + * for the RNG4 state handles which exist in + * the RNG4 block: 1 if it's been instantiated + * + * Return: - 0 if no error occurred + * - -ENOMEM if there isn't enough memory to allocate the descriptor + * - -ENODEV if DECO0 couldn't be acquired + * - -EAGAIN if an error occurred when executing the descriptor + */ +static int deinstantiate_rng(struct device *ctrldev, int state_handle_mask) +{ + u32 *desc, status; + int sh_idx, ret = 0; + + desc = kmalloc(CAAM_CMD_SZ * 3, GFP_KERNEL); + if (!desc) + return -ENOMEM; + + for (sh_idx = 0; sh_idx < RNG4_MAX_HANDLES; sh_idx++) { + /* + * If the corresponding bit is set, then it means the state + * handle was initialized by us, and thus it needs to be + * deinitialized as well + */ + if ((1 << sh_idx) & state_handle_mask) { + /* + * Create the descriptor for deinstantating this state + * handle + */ + build_deinstantiation_desc(desc, sh_idx); + + /* Try to run it through DECO0 */ + ret = run_descriptor_deco0(ctrldev, desc, &status); + + if (ret || + (status && status != JRSTA_SSRC_JUMP_HALT_CC)) { + dev_err(ctrldev, + "Failed to deinstantiate RNG4 SH%d\n", + sh_idx); + break; + } + dev_info(ctrldev, "Deinstantiated RNG4 SH%d\n", sh_idx); + } + } + + kfree(desc); + + return ret; +} + +static void devm_deinstantiate_rng(void *data) +{ + struct device *ctrldev = data; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev); + + /* + * De-initialize RNG state handles initialized by this driver. + * In case of SoCs with Management Complex, RNG is managed by MC f/w. + */ + if (ctrlpriv->rng4_sh_init) + deinstantiate_rng(ctrldev, ctrlpriv->rng4_sh_init); +} + +/* + * instantiate_rng - builds and executes a descriptor on DECO0, + * which initializes the RNG block. + * @ctrldev - pointer to device + * @state_handle_mask - bitmask containing the instantiation status + * for the RNG4 state handles which exist in + * the RNG4 block: 1 if it's been instantiated + * by an external entry, 0 otherwise. + * @gen_sk - generate data to be loaded into the JDKEK, TDKEK and TDSK; + * Caution: this can be done only once; if the keys need to be + * regenerated, a POR is required + * + * Return: - 0 if no error occurred + * - -ENOMEM if there isn't enough memory to allocate the descriptor + * - -ENODEV if DECO0 couldn't be acquired + * - -EAGAIN if an error occurred when executing the descriptor + * f.i. there was a RNG hardware error due to not "good enough" + * entropy being acquired. + */ +static int instantiate_rng(struct device *ctrldev, int state_handle_mask, + int gen_sk) +{ + struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev); + struct caam_ctrl __iomem *ctrl; + u32 *desc, status = 0, rdsta_val; + int ret = 0, sh_idx; + + ctrl = (struct caam_ctrl __iomem *)ctrlpriv->ctrl; + desc = kmalloc(CAAM_CMD_SZ * 7, GFP_KERNEL); + if (!desc) + return -ENOMEM; + + for (sh_idx = 0; sh_idx < RNG4_MAX_HANDLES; sh_idx++) { + const u32 rdsta_if = RDSTA_IF0 << sh_idx; + const u32 rdsta_pr = RDSTA_PR0 << sh_idx; + const u32 rdsta_mask = rdsta_if | rdsta_pr; + + /* Clear the contents before using the descriptor */ + memset(desc, 0x00, CAAM_CMD_SZ * 7); + + /* + * If the corresponding bit is set, this state handle + * was initialized by somebody else, so it's left alone. + */ + if (rdsta_if & state_handle_mask) { + if (rdsta_pr & state_handle_mask) + continue; + + dev_info(ctrldev, + "RNG4 SH%d was previously instantiated without prediction resistance. Tearing it down\n", + sh_idx); + + ret = deinstantiate_rng(ctrldev, rdsta_if); + if (ret) + break; + } + + /* Create the descriptor for instantiating RNG State Handle */ + build_instantiation_desc(desc, sh_idx, gen_sk); + + /* Try to run it through DECO0 */ + ret = run_descriptor_deco0(ctrldev, desc, &status); + + /* + * If ret is not 0, or descriptor status is not 0, then + * something went wrong. No need to try the next state + * handle (if available), bail out here. + * Also, if for some reason, the State Handle didn't get + * instantiated although the descriptor has finished + * without any error (HW optimizations for later + * CAAM eras), then try again. + */ + if (ret) + break; + + rdsta_val = rd_reg32(&ctrl->r4tst[0].rdsta) & RDSTA_MASK; + if ((status && status != JRSTA_SSRC_JUMP_HALT_CC) || + (rdsta_val & rdsta_mask) != rdsta_mask) { + ret = -EAGAIN; + break; + } + + dev_info(ctrldev, "Instantiated RNG4 SH%d\n", sh_idx); + } + + kfree(desc); + + if (ret) + return ret; + + return devm_add_action_or_reset(ctrldev, devm_deinstantiate_rng, ctrldev); +} + +/* + * kick_trng - sets the various parameters for enabling the initialization + * of the RNG4 block in CAAM + * @dev - pointer to the controller device + * @ent_delay - Defines the length (in system clocks) of each entropy sample. + */ +static void kick_trng(struct device *dev, int ent_delay) +{ + struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev); + struct caam_ctrl __iomem *ctrl; + struct rng4tst __iomem *r4tst; + u32 val, rtsdctl; + + ctrl = (struct caam_ctrl __iomem *)ctrlpriv->ctrl; + r4tst = &ctrl->r4tst[0]; + + /* + * Setting both RTMCTL:PRGM and RTMCTL:TRNG_ACC causes TRNG to + * properly invalidate the entropy in the entropy register and + * force re-generation. + */ + clrsetbits_32(&r4tst->rtmctl, 0, RTMCTL_PRGM | RTMCTL_ACC); + + /* + * Performance-wise, it does not make sense to + * set the delay to a value that is lower + * than the last one that worked (i.e. the state handles + * were instantiated properly). + */ + rtsdctl = rd_reg32(&r4tst->rtsdctl); + val = (rtsdctl & RTSDCTL_ENT_DLY_MASK) >> RTSDCTL_ENT_DLY_SHIFT; + if (ent_delay > val) { + val = ent_delay; + /* min. freq. count, equal to 1/4 of the entropy sample length */ + wr_reg32(&r4tst->rtfrqmin, val >> 2); + /* disable maximum frequency count */ + wr_reg32(&r4tst->rtfrqmax, RTFRQMAX_DISABLE); + } + + wr_reg32(&r4tst->rtsdctl, (val << RTSDCTL_ENT_DLY_SHIFT) | + RTSDCTL_SAMP_SIZE_VAL); + + /* + * To avoid reprogramming the self-test parameters over and over again, + * use RTSDCTL[SAMP_SIZE] as an indicator. + */ + if ((rtsdctl & RTSDCTL_SAMP_SIZE_MASK) != RTSDCTL_SAMP_SIZE_VAL) { + wr_reg32(&r4tst->rtscmisc, (2 << 16) | 32); + wr_reg32(&r4tst->rtpkrrng, 570); + wr_reg32(&r4tst->rtpkrmax, 1600); + wr_reg32(&r4tst->rtscml, (122 << 16) | 317); + wr_reg32(&r4tst->rtscrl[0], (80 << 16) | 107); + wr_reg32(&r4tst->rtscrl[1], (57 << 16) | 62); + wr_reg32(&r4tst->rtscrl[2], (39 << 16) | 39); + wr_reg32(&r4tst->rtscrl[3], (27 << 16) | 26); + wr_reg32(&r4tst->rtscrl[4], (19 << 16) | 18); + wr_reg32(&r4tst->rtscrl[5], (18 << 16) | 17); + } + + /* + * select raw sampling in both entropy shifter + * and statistical checker; ; put RNG4 into run mode + */ + clrsetbits_32(&r4tst->rtmctl, RTMCTL_PRGM | RTMCTL_ACC, + RTMCTL_SAMP_MODE_RAW_ES_SC); +} + +static int caam_get_era_from_hw(struct caam_perfmon __iomem *perfmon) +{ + static const struct { + u16 ip_id; + u8 maj_rev; + u8 era; + } id[] = { + {0x0A10, 1, 1}, + {0x0A10, 2, 2}, + {0x0A12, 1, 3}, + {0x0A14, 1, 3}, + {0x0A14, 2, 4}, + {0x0A16, 1, 4}, + {0x0A10, 3, 4}, + {0x0A11, 1, 4}, + {0x0A18, 1, 4}, + {0x0A11, 2, 5}, + {0x0A12, 2, 5}, + {0x0A13, 1, 5}, + {0x0A1C, 1, 5} + }; + u32 ccbvid, id_ms; + u8 maj_rev, era; + u16 ip_id; + int i; + + ccbvid = rd_reg32(&perfmon->ccb_id); + era = (ccbvid & CCBVID_ERA_MASK) >> CCBVID_ERA_SHIFT; + if (era) /* This is '0' prior to CAAM ERA-6 */ + return era; + + id_ms = rd_reg32(&perfmon->caam_id_ms); + ip_id = (id_ms & SECVID_MS_IPID_MASK) >> SECVID_MS_IPID_SHIFT; + maj_rev = (id_ms & SECVID_MS_MAJ_REV_MASK) >> SECVID_MS_MAJ_REV_SHIFT; + + for (i = 0; i < ARRAY_SIZE(id); i++) + if (id[i].ip_id == ip_id && id[i].maj_rev == maj_rev) + return id[i].era; + + return -ENOTSUPP; +} + +/** + * caam_get_era() - Return the ERA of the SEC on SoC, based + * on "sec-era" optional property in the DTS. This property is updated + * by u-boot. + * In case this property is not passed an attempt to retrieve the CAAM + * era via register reads will be made. + * + * @perfmon: Performance Monitor Registers + */ +static int caam_get_era(struct caam_perfmon __iomem *perfmon) +{ + struct device_node *caam_node; + int ret; + u32 prop; + + caam_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); + ret = of_property_read_u32(caam_node, "fsl,sec-era", &prop); + of_node_put(caam_node); + + if (!ret) + return prop; + else + return caam_get_era_from_hw(perfmon); +} + +/* + * ERRATA: imx6 devices (imx6D, imx6Q, imx6DL, imx6S, imx6DP and imx6QP) + * have an issue wherein AXI bus transactions may not occur in the correct + * order. This isn't a problem running single descriptors, but can be if + * running multiple concurrent descriptors. Reworking the driver to throttle + * to single requests is impractical, thus the workaround is to limit the AXI + * pipeline to a depth of 1 (from it's default of 4) to preclude this situation + * from occurring. + */ +static void handle_imx6_err005766(u32 __iomem *mcr) +{ + if (of_machine_is_compatible("fsl,imx6q") || + of_machine_is_compatible("fsl,imx6dl") || + of_machine_is_compatible("fsl,imx6qp")) + clrsetbits_32(mcr, MCFGR_AXIPIPE_MASK, + 1 << MCFGR_AXIPIPE_SHIFT); +} + +static const struct of_device_id caam_match[] = { + { + .compatible = "fsl,sec-v4.0", + }, + { + .compatible = "fsl,sec4.0", + }, + {}, +}; +MODULE_DEVICE_TABLE(of, caam_match); + +struct caam_imx_data { + bool page0_access; + const struct clk_bulk_data *clks; + int num_clks; +}; + +static const struct clk_bulk_data caam_imx6_clks[] = { + { .id = "ipg" }, + { .id = "mem" }, + { .id = "aclk" }, + { .id = "emi_slow" }, +}; + +static const struct caam_imx_data caam_imx6_data = { + .page0_access = true, + .clks = caam_imx6_clks, + .num_clks = ARRAY_SIZE(caam_imx6_clks), +}; + +static const struct clk_bulk_data caam_imx7_clks[] = { + { .id = "ipg" }, + { .id = "aclk" }, +}; + +static const struct caam_imx_data caam_imx7_data = { + .page0_access = true, + .clks = caam_imx7_clks, + .num_clks = ARRAY_SIZE(caam_imx7_clks), +}; + +static const struct clk_bulk_data caam_imx6ul_clks[] = { + { .id = "ipg" }, + { .id = "mem" }, + { .id = "aclk" }, +}; + +static const struct caam_imx_data caam_imx6ul_data = { + .page0_access = true, + .clks = caam_imx6ul_clks, + .num_clks = ARRAY_SIZE(caam_imx6ul_clks), +}; + +static const struct clk_bulk_data caam_vf610_clks[] = { + { .id = "ipg" }, +}; + +static const struct caam_imx_data caam_vf610_data = { + .page0_access = true, + .clks = caam_vf610_clks, + .num_clks = ARRAY_SIZE(caam_vf610_clks), +}; + +static const struct caam_imx_data caam_imx8ulp_data; + +static const struct soc_device_attribute caam_imx_soc_table[] = { + { .soc_id = "i.MX6UL", .data = &caam_imx6ul_data }, + { .soc_id = "i.MX6*", .data = &caam_imx6_data }, + { .soc_id = "i.MX7*", .data = &caam_imx7_data }, + { .soc_id = "i.MX8M*", .data = &caam_imx7_data }, + { .soc_id = "i.MX8ULP", .data = &caam_imx8ulp_data }, + { .soc_id = "i.MX8Q*", .data = &caam_imx8ulp_data }, + { .soc_id = "VF*", .data = &caam_vf610_data }, + { .family = "Freescale i.MX" }, + { /* sentinel */ } +}; + +static void disable_clocks(void *data) +{ + struct caam_drv_private *ctrlpriv = data; + + clk_bulk_disable_unprepare(ctrlpriv->num_clks, ctrlpriv->clks); +} + +static int init_clocks(struct device *dev, const struct caam_imx_data *data) +{ + struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev); + int ret; + + ctrlpriv->num_clks = data->num_clks; + ctrlpriv->clks = devm_kmemdup_array(dev, data->clks, + data->num_clks, sizeof(*data->clks), + GFP_KERNEL); + if (!ctrlpriv->clks) + return -ENOMEM; + + ret = devm_clk_bulk_get(dev, ctrlpriv->num_clks, ctrlpriv->clks); + if (ret) { + dev_err(dev, + "Failed to request all necessary clocks\n"); + return ret; + } + + ret = clk_bulk_prepare_enable(ctrlpriv->num_clks, ctrlpriv->clks); + if (ret) { + dev_err(dev, + "Failed to prepare/enable all necessary clocks\n"); + return ret; + } + + return devm_add_action_or_reset(dev, disable_clocks, ctrlpriv); +} + +static void caam_remove_debugfs(void *root) +{ + debugfs_remove_recursive(root); +} + +#ifdef CONFIG_FSL_MC_BUS +static bool check_version(struct fsl_mc_version *mc_version, u32 major, + u32 minor, u32 revision) +{ + if (mc_version->major > major) + return true; + + if (mc_version->major == major) { + if (mc_version->minor > minor) + return true; + + if (mc_version->minor == minor && + mc_version->revision > revision) + return true; + } + + return false; +} +#endif + +static bool needs_entropy_delay_adjustment(void) +{ + if (of_machine_is_compatible("fsl,imx6sx")) + return true; + return false; +} + +static int caam_ctrl_rng_init(struct device *dev) +{ + struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev); + struct caam_ctrl __iomem *ctrl = ctrlpriv->ctrl; + int ret, gen_sk, ent_delay = RTSDCTL_ENT_DLY_MIN; + u8 rng_vid; + + if (ctrlpriv->era < 10) { + struct caam_perfmon __iomem *perfmon; + + perfmon = ctrlpriv->total_jobrs ? + (struct caam_perfmon __iomem *)&ctrlpriv->jr[0]->perfmon : + (struct caam_perfmon __iomem *)&ctrl->perfmon; + + rng_vid = (rd_reg32(&perfmon->cha_id_ls) & + CHA_ID_LS_RNG_MASK) >> CHA_ID_LS_RNG_SHIFT; + } else { + struct version_regs __iomem *vreg; + + vreg = ctrlpriv->total_jobrs ? + (struct version_regs __iomem *)&ctrlpriv->jr[0]->vreg : + (struct version_regs __iomem *)&ctrl->vreg; + + rng_vid = (rd_reg32(&vreg->rng) & CHA_VER_VID_MASK) >> + CHA_VER_VID_SHIFT; + } + + /* + * If SEC has RNG version >= 4 and RNG state handle has not been + * already instantiated, do RNG instantiation + * In case of SoCs with Management Complex, RNG is managed by MC f/w. + */ + if (!(ctrlpriv->mc_en && ctrlpriv->pr_support) && rng_vid >= 4) { + ctrlpriv->rng4_sh_init = + rd_reg32(&ctrl->r4tst[0].rdsta); + /* + * If the secure keys (TDKEK, JDKEK, TDSK), were already + * generated, signal this to the function that is instantiating + * the state handles. An error would occur if RNG4 attempts + * to regenerate these keys before the next POR. + */ + gen_sk = ctrlpriv->rng4_sh_init & RDSTA_SKVN ? 0 : 1; + ctrlpriv->rng4_sh_init &= RDSTA_MASK; + do { + int inst_handles = + rd_reg32(&ctrl->r4tst[0].rdsta) & RDSTA_MASK; + /* + * If either SH were instantiated by somebody else + * (e.g. u-boot) then it is assumed that the entropy + * parameters are properly set and thus the function + * setting these (kick_trng(...)) is skipped. + * Also, if a handle was instantiated, do not change + * the TRNG parameters. + */ + if (needs_entropy_delay_adjustment()) + ent_delay = 12000; + if (!inst_handles) { + dev_info(dev, + "Entropy delay = %u\n", + ent_delay); + kick_trng(dev, ent_delay); + ent_delay = ent_delay * 2; + } + /* + * if instantiate_rng(...) fails, the loop will rerun + * and the kick_trng(...) function will modify the + * upper and lower limits of the entropy sampling + * interval, leading to a successful initialization of + * the RNG. + */ + ret = instantiate_rng(dev, inst_handles, + gen_sk); + /* + * Entropy delay is determined via TRNG characterization. + * TRNG characterization is run across different voltages + * and temperatures. + * If worst case value for ent_dly is identified, + * the loop can be skipped for that platform. + */ + if (needs_entropy_delay_adjustment()) + break; + if (ret == -EAGAIN) + /* + * if here, the loop will rerun, + * so don't hog the CPU + */ + cpu_relax(); + } while ((ret == -EAGAIN) && (ent_delay < RTSDCTL_ENT_DLY_MAX)); + if (ret) { + dev_err(dev, "failed to instantiate RNG"); + return ret; + } + /* + * Set handles initialized by this module as the complement of + * the already initialized ones + */ + ctrlpriv->rng4_sh_init = ~ctrlpriv->rng4_sh_init & RDSTA_MASK; + + /* Enable RDB bit so that RNG works faster */ + clrsetbits_32(&ctrl->scfgr, 0, SCFGR_RDBENABLE); + } + + return 0; +} + +/* Indicate if the internal state of the CAAM is lost during PM */ +static int caam_off_during_pm(void) +{ + bool not_off_during_pm = of_machine_is_compatible("fsl,imx6q") || + of_machine_is_compatible("fsl,imx6qp") || + of_machine_is_compatible("fsl,imx6dl"); + + return not_off_during_pm ? 0 : 1; +} + +static void caam_state_save(struct device *dev) +{ + struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev); + struct caam_ctl_state *state = &ctrlpriv->state; + struct caam_ctrl __iomem *ctrl = ctrlpriv->ctrl; + u32 deco_inst, jr_inst; + int i; + + state->mcr = rd_reg32(&ctrl->mcr); + state->scfgr = rd_reg32(&ctrl->scfgr); + + deco_inst = (rd_reg32(&ctrl->perfmon.cha_num_ms) & + CHA_ID_MS_DECO_MASK) >> CHA_ID_MS_DECO_SHIFT; + for (i = 0; i < deco_inst; i++) { + state->deco_mid[i].liodn_ms = + rd_reg32(&ctrl->deco_mid[i].liodn_ms); + state->deco_mid[i].liodn_ls = + rd_reg32(&ctrl->deco_mid[i].liodn_ls); + } + + jr_inst = (rd_reg32(&ctrl->perfmon.cha_num_ms) & + CHA_ID_MS_JR_MASK) >> CHA_ID_MS_JR_SHIFT; + for (i = 0; i < jr_inst; i++) { + state->jr_mid[i].liodn_ms = + rd_reg32(&ctrl->jr_mid[i].liodn_ms); + state->jr_mid[i].liodn_ls = + rd_reg32(&ctrl->jr_mid[i].liodn_ls); + } +} + +static void caam_state_restore(const struct device *dev) +{ + const struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev); + const struct caam_ctl_state *state = &ctrlpriv->state; + struct caam_ctrl __iomem *ctrl = ctrlpriv->ctrl; + u32 deco_inst, jr_inst; + int i; + + wr_reg32(&ctrl->mcr, state->mcr); + wr_reg32(&ctrl->scfgr, state->scfgr); + + deco_inst = (rd_reg32(&ctrl->perfmon.cha_num_ms) & + CHA_ID_MS_DECO_MASK) >> CHA_ID_MS_DECO_SHIFT; + for (i = 0; i < deco_inst; i++) { + wr_reg32(&ctrl->deco_mid[i].liodn_ms, + state->deco_mid[i].liodn_ms); + wr_reg32(&ctrl->deco_mid[i].liodn_ls, + state->deco_mid[i].liodn_ls); + } + + jr_inst = (rd_reg32(&ctrl->perfmon.cha_num_ms) & + CHA_ID_MS_JR_MASK) >> CHA_ID_MS_JR_SHIFT; + for (i = 0; i < jr_inst; i++) { + wr_reg32(&ctrl->jr_mid[i].liodn_ms, + state->jr_mid[i].liodn_ms); + wr_reg32(&ctrl->jr_mid[i].liodn_ls, + state->jr_mid[i].liodn_ls); + } + + if (ctrlpriv->virt_en == 1) + clrsetbits_32(&ctrl->jrstart, 0, JRSTART_JR0_START | + JRSTART_JR1_START | JRSTART_JR2_START | + JRSTART_JR3_START); +} + +static int caam_ctrl_suspend(struct device *dev) +{ + const struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev); + + if (ctrlpriv->caam_off_during_pm && !ctrlpriv->no_page0) + caam_state_save(dev); + + return 0; +} + +static int caam_ctrl_resume(struct device *dev) +{ + struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev); + int ret = 0; + + if (ctrlpriv->caam_off_during_pm && !ctrlpriv->no_page0) { + caam_state_restore(dev); + + /* HW and rng will be reset so deinstantiation can be removed */ + devm_remove_action(dev, devm_deinstantiate_rng, dev); + ret = caam_ctrl_rng_init(dev); + } + + return ret; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(caam_ctrl_pm_ops, caam_ctrl_suspend, caam_ctrl_resume); + +/* Probe routine for CAAM top (controller) level */ +static int caam_probe(struct platform_device *pdev) +{ + int ret, ring; + u64 caam_id; + const struct soc_device_attribute *imx_soc_match; + struct device *dev; + struct device_node *nprop, *np; + struct caam_ctrl __iomem *ctrl; + struct caam_drv_private *ctrlpriv; + struct caam_perfmon __iomem *perfmon; + struct dentry *dfs_root; + u32 scfgr, comp_params; + int pg_size; + int BLOCK_OFFSET = 0; + bool reg_access = true; + const struct caam_imx_data *imx_soc_data; + + ctrlpriv = devm_kzalloc(&pdev->dev, sizeof(*ctrlpriv), GFP_KERNEL); + if (!ctrlpriv) + return -ENOMEM; + + dev = &pdev->dev; + dev_set_drvdata(dev, ctrlpriv); + nprop = pdev->dev.of_node; + + imx_soc_match = soc_device_match(caam_imx_soc_table); + if (!imx_soc_match && of_match_node(imx8m_machine_match, of_root)) + return -EPROBE_DEFER; + + caam_imx = (bool)imx_soc_match; + + ctrlpriv->caam_off_during_pm = caam_imx && caam_off_during_pm(); + + if (imx_soc_match) { + /* + * Until Layerscape and i.MX OP-TEE get in sync, + * only i.MX OP-TEE use cases disallow access to + * caam page 0 (controller) registers. + */ + np = of_find_compatible_node(NULL, NULL, "linaro,optee-tz"); + ctrlpriv->optee_en = !!np; + of_node_put(np); + + reg_access = !ctrlpriv->optee_en; + + if (!imx_soc_match->data) { + dev_err(dev, "No clock data provided for i.MX SoC"); + return -EINVAL; + } + + imx_soc_data = imx_soc_match->data; + reg_access = reg_access && imx_soc_data->page0_access; + ctrlpriv->no_page0 = !reg_access; + /* + * CAAM clocks cannot be controlled from kernel. + */ + if (!imx_soc_data->num_clks) + goto iomap_ctrl; + + ret = init_clocks(dev, imx_soc_match->data); + if (ret) + return ret; + } + +iomap_ctrl: + /* Get configuration properties from device tree */ + /* First, get register page */ + ctrl = devm_of_iomap(dev, nprop, 0, NULL); + ret = PTR_ERR_OR_ZERO(ctrl); + if (ret) { + dev_err(dev, "caam: of_iomap() failed\n"); + return ret; + } + + ring = 0; + for_each_available_child_of_node(nprop, np) + if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") || + of_device_is_compatible(np, "fsl,sec4.0-job-ring")) { + u32 reg; + + if (of_property_read_u32_index(np, "reg", 0, ®)) { + dev_err(dev, "%s read reg property error\n", + np->full_name); + continue; + } + + ctrlpriv->jr[ring] = (struct caam_job_ring __iomem __force *) + ((__force uint8_t *)ctrl + reg); + + ctrlpriv->total_jobrs++; + ring++; + } + + /* + * Wherever possible, instead of accessing registers from the global page, + * use the alias registers in the first (cf. DT nodes order) + * job ring's page. + */ + perfmon = ring ? (struct caam_perfmon __iomem *)&ctrlpriv->jr[0]->perfmon : + (struct caam_perfmon __iomem *)&ctrl->perfmon; + + caam_little_end = !(bool)(rd_reg32(&perfmon->status) & + (CSTA_PLEND | CSTA_ALT_PLEND)); + comp_params = rd_reg32(&perfmon->comp_parms_ms); + if (reg_access && comp_params & CTPR_MS_PS && + rd_reg32(&ctrl->mcr) & MCFGR_LONG_PTR) + caam_ptr_sz = sizeof(u64); + else + caam_ptr_sz = sizeof(u32); + caam_dpaa2 = !!(comp_params & CTPR_MS_DPAA2); + ctrlpriv->qi_present = !!(comp_params & CTPR_MS_QI_MASK); + +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI + /* If (DPAA 1.x) QI present, check whether dependencies are available */ + if (ctrlpriv->qi_present && !caam_dpaa2) { + ret = qman_is_probed(); + if (!ret) { + return -EPROBE_DEFER; + } else if (ret < 0) { + dev_err(dev, "failing probe due to qman probe error\n"); + return -ENODEV; + } + + ret = qman_portals_probed(); + if (!ret) { + return -EPROBE_DEFER; + } else if (ret < 0) { + dev_err(dev, "failing probe due to qman portals probe error\n"); + return -ENODEV; + } + } +#endif + + /* Allocating the BLOCK_OFFSET based on the supported page size on + * the platform + */ + pg_size = (comp_params & CTPR_MS_PG_SZ_MASK) >> CTPR_MS_PG_SZ_SHIFT; + if (pg_size == 0) + BLOCK_OFFSET = PG_SIZE_4K; + else + BLOCK_OFFSET = PG_SIZE_64K; + + ctrlpriv->ctrl = (struct caam_ctrl __iomem __force *)ctrl; + ctrlpriv->assure = (struct caam_assurance __iomem __force *) + ((__force uint8_t *)ctrl + + BLOCK_OFFSET * ASSURE_BLOCK_NUMBER + ); + ctrlpriv->deco = (struct caam_deco __iomem __force *) + ((__force uint8_t *)ctrl + + BLOCK_OFFSET * DECO_BLOCK_NUMBER + ); + + /* Get the IRQ of the controller (for security violations only) */ + ctrlpriv->secvio_irq = irq_of_parse_and_map(nprop, 0); + np = of_find_compatible_node(NULL, NULL, "fsl,qoriq-mc"); + ctrlpriv->mc_en = !!np; + of_node_put(np); + +#ifdef CONFIG_FSL_MC_BUS + if (ctrlpriv->mc_en) { + struct fsl_mc_version *mc_version; + + mc_version = fsl_mc_get_version(); + if (mc_version) + ctrlpriv->pr_support = check_version(mc_version, 10, 20, + 0); + else + return -EPROBE_DEFER; + } +#endif + + if (!reg_access) + goto set_dma_mask; + + /* + * Enable DECO watchdogs and, if this is a PHYS_ADDR_T_64BIT kernel, + * long pointers in master configuration register. + * In case of SoCs with Management Complex, MC f/w performs + * the configuration. + */ + if (!ctrlpriv->mc_en) + clrsetbits_32(&ctrl->mcr, MCFGR_AWCACHE_MASK, + MCFGR_AWCACHE_CACH | MCFGR_AWCACHE_BUFF | + MCFGR_WDENABLE | MCFGR_LARGE_BURST); + + handle_imx6_err005766(&ctrl->mcr); + + /* + * Read the Compile Time parameters and SCFGR to determine + * if virtualization is enabled for this platform + */ + scfgr = rd_reg32(&ctrl->scfgr); + + ctrlpriv->virt_en = 0; + if (comp_params & CTPR_MS_VIRT_EN_INCL) { + /* VIRT_EN_INCL = 1 & VIRT_EN_POR = 1 or + * VIRT_EN_INCL = 1 & VIRT_EN_POR = 0 & SCFGR_VIRT_EN = 1 + */ + if ((comp_params & CTPR_MS_VIRT_EN_POR) || + (!(comp_params & CTPR_MS_VIRT_EN_POR) && + (scfgr & SCFGR_VIRT_EN))) + ctrlpriv->virt_en = 1; + } else { + /* VIRT_EN_INCL = 0 && VIRT_EN_POR_VALUE = 1 */ + if (comp_params & CTPR_MS_VIRT_EN_POR) + ctrlpriv->virt_en = 1; + } + + if (ctrlpriv->virt_en == 1) + clrsetbits_32(&ctrl->jrstart, 0, JRSTART_JR0_START | + JRSTART_JR1_START | JRSTART_JR2_START | + JRSTART_JR3_START); + +set_dma_mask: + ret = dma_set_mask_and_coherent(dev, caam_get_dma_mask(dev)); + if (ret) { + dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", ret); + return ret; + } + + ctrlpriv->era = caam_get_era(perfmon); + ctrlpriv->domain = iommu_get_domain_for_dev(dev); + + dfs_root = debugfs_create_dir(dev_name(dev), NULL); + if (IS_ENABLED(CONFIG_DEBUG_FS)) { + ret = devm_add_action_or_reset(dev, caam_remove_debugfs, + dfs_root); + if (ret) + return ret; + } + + caam_debugfs_init(ctrlpriv, perfmon, dfs_root); + + /* Check to see if (DPAA 1.x) QI present. If so, enable */ + if (ctrlpriv->qi_present && !caam_dpaa2) { + ctrlpriv->qi = (struct caam_queue_if __iomem __force *) + ((__force uint8_t *)ctrl + + BLOCK_OFFSET * QI_BLOCK_NUMBER + ); + /* This is all that's required to physically enable QI */ + wr_reg32(&ctrlpriv->qi->qi_control_lo, QICTL_DQEN); + + /* If QMAN driver is present, init CAAM-QI backend */ +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI + ret = caam_qi_init(pdev); + if (ret) + dev_err(dev, "caam qi i/f init failed: %d\n", ret); +#endif + } + + /* If no QI and no rings specified, quit and go home */ + if ((!ctrlpriv->qi_present) && (!ctrlpriv->total_jobrs)) { + dev_err(dev, "no queues configured, terminating\n"); + return -ENOMEM; + } + + comp_params = rd_reg32(&perfmon->comp_parms_ls); + ctrlpriv->blob_present = !!(comp_params & CTPR_LS_BLOB); + + /* + * Some SoCs like the LS1028A (non-E) indicate CTPR_LS_BLOB support, + * but fail when actually using it due to missing AES support, so + * check both here. + */ + if (ctrlpriv->era < 10) { + ctrlpriv->blob_present = ctrlpriv->blob_present && + (rd_reg32(&perfmon->cha_num_ls) & CHA_ID_LS_AES_MASK); + } else { + struct version_regs __iomem *vreg; + + vreg = ctrlpriv->total_jobrs ? + (struct version_regs __iomem *)&ctrlpriv->jr[0]->vreg : + (struct version_regs __iomem *)&ctrl->vreg; + + ctrlpriv->blob_present = ctrlpriv->blob_present && + (rd_reg32(&vreg->aesa) & CHA_VER_MISC_AES_NUM_MASK); + } + + if (reg_access) { + ret = caam_ctrl_rng_init(dev); + if (ret) + return ret; + } + + caam_id = (u64)rd_reg32(&perfmon->caam_id_ms) << 32 | + (u64)rd_reg32(&perfmon->caam_id_ls); + + /* Report "alive" for developer to see */ + dev_info(dev, "device ID = 0x%016llx (Era %d)\n", caam_id, + ctrlpriv->era); + dev_info(dev, "job rings = %d, qi = %d\n", + ctrlpriv->total_jobrs, ctrlpriv->qi_present); + + ret = devm_of_platform_populate(dev); + if (ret) + dev_err(dev, "JR platform devices creation error\n"); + + return ret; +} + +static struct platform_driver caam_driver = { + .driver = { + .name = "caam", + .of_match_table = caam_match, + .pm = pm_ptr(&caam_ctrl_pm_ops), + }, + .probe = caam_probe, +}; + +module_platform_driver(caam_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("FSL CAAM request backend"); +MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); diff --git a/drivers/crypto/caam/ctrl.h b/drivers/crypto/caam/ctrl.h new file mode 100644 index 000000000..f3ecd6792 --- /dev/null +++ b/drivers/crypto/caam/ctrl.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * CAAM control-plane driver backend public-level include definitions + * + * Copyright 2012 Freescale Semiconductor, Inc. + */ + +#ifndef CTRL_H +#define CTRL_H + +/* Prototypes for backend-level services exposed to APIs */ +extern bool caam_dpaa2; + +#endif /* CTRL_H */ diff --git a/drivers/crypto/caam/debugfs.c b/drivers/crypto/caam/debugfs.c new file mode 100644 index 000000000..718352b7a --- /dev/null +++ b/drivers/crypto/caam/debugfs.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* Copyright 2019, 2023 NXP */ + +#include +#include "compat.h" +#include "debugfs.h" +#include "regs.h" +#include "intern.h" + +static int caam_debugfs_u64_get(void *data, u64 *val) +{ + *val = caam64_to_cpu(*(u64 *)data); + return 0; +} + +static int caam_debugfs_u32_get(void *data, u64 *val) +{ + *val = caam32_to_cpu(*(u32 *)data); + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(caam_fops_u32_ro, caam_debugfs_u32_get, NULL, "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(caam_fops_u64_ro, caam_debugfs_u64_get, NULL, "%llu\n"); + +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI +/* + * This is a counter for the number of times the congestion group (where all + * the request and response queueus are) reached congestion. Incremented + * each time the congestion callback is called with congested == true. + */ +static u64 times_congested; + +void caam_debugfs_qi_congested(void) +{ + times_congested++; +} + +void caam_debugfs_qi_init(struct caam_drv_private *ctrlpriv) +{ + debugfs_create_file("qi_congested", 0444, ctrlpriv->ctl, + ×_congested, &caam_fops_u64_ro); +} +#endif + +void caam_debugfs_init(struct caam_drv_private *ctrlpriv, + struct caam_perfmon __force *perfmon, + struct dentry *root) +{ + /* + * FIXME: needs better naming distinction, as some amalgamation of + * "caam" and nprop->full_name. The OF name isn't distinctive, + * but does separate instances + */ + + ctrlpriv->ctl = debugfs_create_dir("ctl", root); + + debugfs_create_file("rq_dequeued", 0444, ctrlpriv->ctl, + &perfmon->req_dequeued, &caam_fops_u64_ro); + debugfs_create_file("ob_rq_encrypted", 0444, ctrlpriv->ctl, + &perfmon->ob_enc_req, &caam_fops_u64_ro); + debugfs_create_file("ib_rq_decrypted", 0444, ctrlpriv->ctl, + &perfmon->ib_dec_req, &caam_fops_u64_ro); + debugfs_create_file("ob_bytes_encrypted", 0444, ctrlpriv->ctl, + &perfmon->ob_enc_bytes, &caam_fops_u64_ro); + debugfs_create_file("ob_bytes_protected", 0444, ctrlpriv->ctl, + &perfmon->ob_prot_bytes, &caam_fops_u64_ro); + debugfs_create_file("ib_bytes_decrypted", 0444, ctrlpriv->ctl, + &perfmon->ib_dec_bytes, &caam_fops_u64_ro); + debugfs_create_file("ib_bytes_validated", 0444, ctrlpriv->ctl, + &perfmon->ib_valid_bytes, &caam_fops_u64_ro); + + /* Controller level - global status values */ + debugfs_create_file("fault_addr", 0444, ctrlpriv->ctl, + &perfmon->faultaddr, &caam_fops_u32_ro); + debugfs_create_file("fault_detail", 0444, ctrlpriv->ctl, + &perfmon->faultdetail, &caam_fops_u32_ro); + debugfs_create_file("fault_status", 0444, ctrlpriv->ctl, + &perfmon->status, &caam_fops_u32_ro); + + if (ctrlpriv->optee_en) + return; + + /* Internal covering keys (useful in non-secure mode only) */ + ctrlpriv->ctl_kek_wrap.data = (__force void *)&ctrlpriv->ctrl->kek[0]; + ctrlpriv->ctl_kek_wrap.size = KEK_KEY_SIZE * sizeof(u32); + debugfs_create_blob("kek", 0444, ctrlpriv->ctl, + &ctrlpriv->ctl_kek_wrap); + + ctrlpriv->ctl_tkek_wrap.data = (__force void *)&ctrlpriv->ctrl->tkek[0]; + ctrlpriv->ctl_tkek_wrap.size = KEK_KEY_SIZE * sizeof(u32); + debugfs_create_blob("tkek", 0444, ctrlpriv->ctl, + &ctrlpriv->ctl_tkek_wrap); + + ctrlpriv->ctl_tdsk_wrap.data = (__force void *)&ctrlpriv->ctrl->tdsk[0]; + ctrlpriv->ctl_tdsk_wrap.size = KEK_KEY_SIZE * sizeof(u32); + debugfs_create_blob("tdsk", 0444, ctrlpriv->ctl, + &ctrlpriv->ctl_tdsk_wrap); +} diff --git a/drivers/crypto/caam/debugfs.h b/drivers/crypto/caam/debugfs.h new file mode 100644 index 000000000..ef238c71f --- /dev/null +++ b/drivers/crypto/caam/debugfs.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */ +/* Copyright 2019, 2023 NXP */ + +#ifndef CAAM_DEBUGFS_H +#define CAAM_DEBUGFS_H + +struct dentry; +struct caam_drv_private; +struct caam_perfmon; + +#ifdef CONFIG_DEBUG_FS +void caam_debugfs_init(struct caam_drv_private *ctrlpriv, + struct caam_perfmon __force *perfmon, struct dentry *root); +#else +static inline void caam_debugfs_init(struct caam_drv_private *ctrlpriv, + struct caam_perfmon __force *perfmon, + struct dentry *root) +{} +#endif + +#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) +void caam_debugfs_qi_congested(void); +void caam_debugfs_qi_init(struct caam_drv_private *ctrlpriv); +#else +static inline void caam_debugfs_qi_congested(void) {} +static inline void caam_debugfs_qi_init(struct caam_drv_private *ctrlpriv) {} +#endif + +#endif /* CAAM_DEBUGFS_H */ diff --git a/drivers/crypto/caam/desc.h b/drivers/crypto/caam/desc.h new file mode 100644 index 000000000..c28e94fcb --- /dev/null +++ b/drivers/crypto/caam/desc.h @@ -0,0 +1,1694 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * CAAM descriptor composition header + * Definitions to support CAAM descriptor instruction generation + * + * Copyright 2008-2011 Freescale Semiconductor, Inc. + * Copyright 2018, 2025 NXP + */ + +#ifndef DESC_H +#define DESC_H + +/* + * 16-byte hardware scatter/gather table + * An 8-byte table exists in the hardware spec, but has never been + * implemented to date. The 8/16 option is selected at RTL-compile-time. + * and this selection is visible in the Compile Time Parameters Register + */ + +#define SEC4_SG_LEN_EXT 0x80000000 /* Entry points to table */ +#define SEC4_SG_LEN_FIN 0x40000000 /* Last entry in table */ +#define SEC4_SG_BPID_MASK 0x000000ff +#define SEC4_SG_BPID_SHIFT 16 +#define SEC4_SG_LEN_MASK 0x3fffffff /* Excludes EXT and FINAL */ +#define SEC4_SG_OFFSET_MASK 0x00001fff + +/* Max size of any CAAM descriptor in 32-bit words, inclusive of header */ +#define MAX_CAAM_DESCSIZE 64 + +/* Block size of any entity covered/uncovered with a KEK/TKEK */ +#define KEK_BLOCKSIZE 16 + +/* + * Supported descriptor command types as they show up + * inside a descriptor command word. + */ +#define CMD_SHIFT 27 +#define CMD_MASK 0xf8000000 + +#define CMD_KEY (0x00 << CMD_SHIFT) +#define CMD_SEQ_KEY (0x01 << CMD_SHIFT) +#define CMD_LOAD (0x02 << CMD_SHIFT) +#define CMD_SEQ_LOAD (0x03 << CMD_SHIFT) +#define CMD_FIFO_LOAD (0x04 << CMD_SHIFT) +#define CMD_SEQ_FIFO_LOAD (0x05 << CMD_SHIFT) +#define CMD_STORE (0x0a << CMD_SHIFT) +#define CMD_SEQ_STORE (0x0b << CMD_SHIFT) +#define CMD_FIFO_STORE (0x0c << CMD_SHIFT) +#define CMD_SEQ_FIFO_STORE (0x0d << CMD_SHIFT) +#define CMD_MOVE_LEN (0x0e << CMD_SHIFT) +#define CMD_MOVE (0x0f << CMD_SHIFT) +#define CMD_OPERATION (0x10 << CMD_SHIFT) +#define CMD_SIGNATURE (0x12 << CMD_SHIFT) +#define CMD_JUMP (0x14 << CMD_SHIFT) +#define CMD_MATH (0x15 << CMD_SHIFT) +#define CMD_DESC_HDR (0x16 << CMD_SHIFT) +#define CMD_SHARED_DESC_HDR (0x17 << CMD_SHIFT) +#define CMD_SEQ_IN_PTR (0x1e << CMD_SHIFT) +#define CMD_SEQ_OUT_PTR (0x1f << CMD_SHIFT) + +/* General-purpose class selector for all commands */ +#define CLASS_SHIFT 25 +#define CLASS_MASK (0x03 << CLASS_SHIFT) + +#define CLASS_NONE (0x00 << CLASS_SHIFT) +#define CLASS_1 (0x01 << CLASS_SHIFT) +#define CLASS_2 (0x02 << CLASS_SHIFT) +#define CLASS_BOTH (0x03 << CLASS_SHIFT) + +/* + * Descriptor header command constructs + * Covers shared, job, and trusted descriptor headers + */ + +/* + * Do Not Run - marks a descriptor inexecutable if there was + * a preceding error somewhere + */ +#define HDR_DNR 0x01000000 + +/* + * ONE - should always be set. Combination of ONE (always + * set) and ZRO (always clear) forms an endianness sanity check + */ +#define HDR_ONE 0x00800000 +#define HDR_ZRO 0x00008000 + +/* Start Index or SharedDesc Length */ +#define HDR_START_IDX_SHIFT 16 +#define HDR_START_IDX_MASK (0x3f << HDR_START_IDX_SHIFT) + +/* If shared descriptor header, 6-bit length */ +#define HDR_DESCLEN_SHR_MASK 0x3f + +/* If non-shared header, 7-bit length */ +#define HDR_DESCLEN_MASK 0x7f + +/* This is a TrustedDesc (if not SharedDesc) */ +#define HDR_TRUSTED 0x00004000 + +/* Make into TrustedDesc (if not SharedDesc) */ +#define HDR_MAKE_TRUSTED 0x00002000 + +/* Save context if self-shared (if SharedDesc) */ +#define HDR_SAVECTX 0x00001000 + +/* Next item points to SharedDesc */ +#define HDR_SHARED 0x00001000 + +/* + * Reverse Execution Order - execute JobDesc first, then + * execute SharedDesc (normally SharedDesc goes first). + */ +#define HDR_REVERSE 0x00000800 + +/* Propagate DNR property to SharedDesc */ +#define HDR_PROP_DNR 0x00000800 + +/* JobDesc/SharedDesc share property */ +#define HDR_SD_SHARE_SHIFT 8 +#define HDR_SD_SHARE_MASK (0x03 << HDR_SD_SHARE_SHIFT) +#define HDR_JD_SHARE_SHIFT 8 +#define HDR_JD_SHARE_MASK (0x07 << HDR_JD_SHARE_SHIFT) + +#define HDR_SHARE_NEVER (0x00 << HDR_SD_SHARE_SHIFT) +#define HDR_SHARE_WAIT (0x01 << HDR_SD_SHARE_SHIFT) +#define HDR_SHARE_SERIAL (0x02 << HDR_SD_SHARE_SHIFT) +#define HDR_SHARE_ALWAYS (0x03 << HDR_SD_SHARE_SHIFT) +#define HDR_SHARE_DEFER (0x04 << HDR_SD_SHARE_SHIFT) + +/* JobDesc/SharedDesc descriptor length */ +#define HDR_JD_LENGTH_MASK 0x7f +#define HDR_SD_LENGTH_MASK 0x3f + +/* + * KEY/SEQ_KEY Command Constructs + */ + +/* Key Destination Class: 01 = Class 1, 02 - Class 2 */ +#define KEY_DEST_CLASS_SHIFT 25 /* use CLASS_1 or CLASS_2 */ +#define KEY_DEST_CLASS_MASK (0x03 << KEY_DEST_CLASS_SHIFT) + +/* Scatter-Gather Table/Variable Length Field */ +#define KEY_SGF 0x01000000 +#define KEY_VLF 0x01000000 + +/* Immediate - Key follows command in the descriptor */ +#define KEY_IMM 0x00800000 + +/* + * Encrypted - Key is encrypted either with the KEK, or + * with the TDKEK if TK is set + */ +#define KEY_ENC 0x00400000 + +/* + * No Write Back - Do not allow key to be FIFO STOREd + */ +#define KEY_NWB 0x00200000 + +/* + * Enhanced Encryption of Key + */ +#define KEY_EKT 0x00100000 +#define KEY_EKT_OFFSET 20 + +/* + * Encrypted with Trusted Key + */ +#define KEY_TK 0x00008000 + +/* + * KDEST - Key Destination: 0 - class key register, + * 1 - PKHA 'e', 2 - AFHA Sbox, 3 - MDHA split-key + */ +#define KEY_DEST_SHIFT 16 +#define KEY_DEST_MASK (0x03 << KEY_DEST_SHIFT) + +#define KEY_DEST_CLASS_REG (0x00 << KEY_DEST_SHIFT) +#define KEY_DEST_PKHA_E (0x01 << KEY_DEST_SHIFT) +#define KEY_DEST_AFHA_SBOX (0x02 << KEY_DEST_SHIFT) +#define KEY_DEST_MDHA_SPLIT (0x03 << KEY_DEST_SHIFT) + +/* Length in bytes */ +#define KEY_LENGTH_MASK 0x000003ff + +/* + * LOAD/SEQ_LOAD/STORE/SEQ_STORE Command Constructs + */ + +/* + * Load/Store Destination: 0 = class independent CCB, + * 1 = class 1 CCB, 2 = class 2 CCB, 3 = DECO + */ +#define LDST_CLASS_SHIFT 25 +#define LDST_CLASS_MASK (0x03 << LDST_CLASS_SHIFT) +#define LDST_CLASS_IND_CCB (0x00 << LDST_CLASS_SHIFT) +#define LDST_CLASS_1_CCB (0x01 << LDST_CLASS_SHIFT) +#define LDST_CLASS_2_CCB (0x02 << LDST_CLASS_SHIFT) +#define LDST_CLASS_DECO (0x03 << LDST_CLASS_SHIFT) + +/* Scatter-Gather Table/Variable Length Field */ +#define LDST_SGF 0x01000000 +#define LDST_VLF LDST_SGF + +/* Immediate - Key follows this command in descriptor */ +#define LDST_IMM_MASK 1 +#define LDST_IMM_SHIFT 23 +#define LDST_IMM (LDST_IMM_MASK << LDST_IMM_SHIFT) + +/* SRC/DST - Destination for LOAD, Source for STORE */ +#define LDST_SRCDST_SHIFT 16 +#define LDST_SRCDST_MASK (0x7f << LDST_SRCDST_SHIFT) + +#define LDST_SRCDST_BYTE_CONTEXT (0x20 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_BYTE_KEY (0x40 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_BYTE_INFIFO (0x7c << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_BYTE_OUTFIFO (0x7e << LDST_SRCDST_SHIFT) + +#define LDST_SRCDST_WORD_MODE_REG (0x00 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_KEYSZ_REG (0x01 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DATASZ_REG (0x02 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_ICVSZ_REG (0x03 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_CHACTRL (0x06 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DECOCTRL (0x06 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_IRQCTRL (0x07 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DECO_PCLOVRD (0x07 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_CLRW (0x08 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DECO_MATH0 (0x08 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_STAT (0x09 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DECO_MATH1 (0x09 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DECO_MATH2 (0x0a << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DECO_AAD_SZ (0x0b << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DECO_MATH3 (0x0b << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_CLASS1_IV_SZ (0x0c << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_ALTDS_CLASS1 (0x0f << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_PKHA_A_SZ (0x10 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_PKHA_B_SZ (0x11 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_PKHA_N_SZ (0x12 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_PKHA_E_SZ (0x13 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_CLASS_CTX (0x20 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DESCBUF (0x40 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DESCBUF_JOB (0x41 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DESCBUF_SHARED (0x42 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DESCBUF_JOB_WE (0x45 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DESCBUF_SHARED_WE (0x46 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_INFO_FIFO_SM (0x71 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_INFO_FIFO (0x7a << LDST_SRCDST_SHIFT) + +/* Offset in source/destination */ +#define LDST_OFFSET_SHIFT 8 +#define LDST_OFFSET_MASK (0xff << LDST_OFFSET_SHIFT) + +/* LDOFF definitions used when DST = LDST_SRCDST_WORD_DECOCTRL */ +/* These could also be shifted by LDST_OFFSET_SHIFT - this reads better */ +#define LDOFF_CHG_SHARE_SHIFT 0 +#define LDOFF_CHG_SHARE_MASK (0x3 << LDOFF_CHG_SHARE_SHIFT) +#define LDOFF_CHG_SHARE_NEVER (0x1 << LDOFF_CHG_SHARE_SHIFT) +#define LDOFF_CHG_SHARE_OK_PROP (0x2 << LDOFF_CHG_SHARE_SHIFT) +#define LDOFF_CHG_SHARE_OK_NO_PROP (0x3 << LDOFF_CHG_SHARE_SHIFT) + +#define LDOFF_ENABLE_AUTO_NFIFO (1 << 2) +#define LDOFF_DISABLE_AUTO_NFIFO (1 << 3) + +#define LDOFF_CHG_NONSEQLIODN_SHIFT 4 +#define LDOFF_CHG_NONSEQLIODN_MASK (0x3 << LDOFF_CHG_NONSEQLIODN_SHIFT) +#define LDOFF_CHG_NONSEQLIODN_SEQ (0x1 << LDOFF_CHG_NONSEQLIODN_SHIFT) +#define LDOFF_CHG_NONSEQLIODN_NON_SEQ (0x2 << LDOFF_CHG_NONSEQLIODN_SHIFT) +#define LDOFF_CHG_NONSEQLIODN_TRUSTED (0x3 << LDOFF_CHG_NONSEQLIODN_SHIFT) + +#define LDOFF_CHG_SEQLIODN_SHIFT 6 +#define LDOFF_CHG_SEQLIODN_MASK (0x3 << LDOFF_CHG_SEQLIODN_SHIFT) +#define LDOFF_CHG_SEQLIODN_SEQ (0x1 << LDOFF_CHG_SEQLIODN_SHIFT) +#define LDOFF_CHG_SEQLIODN_NON_SEQ (0x2 << LDOFF_CHG_SEQLIODN_SHIFT) +#define LDOFF_CHG_SEQLIODN_TRUSTED (0x3 << LDOFF_CHG_SEQLIODN_SHIFT) + +/* Data length in bytes */ +#define LDST_LEN_SHIFT 0 +#define LDST_LEN_MASK (0xff << LDST_LEN_SHIFT) + +/* Special Length definitions when dst=deco-ctrl */ +#define LDLEN_ENABLE_OSL_COUNT (1 << 7) +#define LDLEN_RST_CHA_OFIFO_PTR (1 << 6) +#define LDLEN_RST_OFIFO (1 << 5) +#define LDLEN_SET_OFIFO_OFF_VALID (1 << 4) +#define LDLEN_SET_OFIFO_OFF_RSVD (1 << 3) +#define LDLEN_SET_OFIFO_OFFSET_SHIFT 0 +#define LDLEN_SET_OFIFO_OFFSET_MASK (3 << LDLEN_SET_OFIFO_OFFSET_SHIFT) + +/* Special Length definitions when dst=sm, nfifo-{sm,m} */ +#define LDLEN_MATH0 0 +#define LDLEN_MATH1 1 +#define LDLEN_MATH2 2 +#define LDLEN_MATH3 3 + +/* + * FIFO_LOAD/FIFO_STORE/SEQ_FIFO_LOAD/SEQ_FIFO_STORE + * Command Constructs + */ + +/* + * Load Destination: 0 = skip (SEQ_FIFO_LOAD only), + * 1 = Load for Class1, 2 = Load for Class2, 3 = Load both + * Store Source: 0 = normal, 1 = Class1key, 2 = Class2key + */ +#define FIFOLD_CLASS_SHIFT 25 +#define FIFOLD_CLASS_MASK (0x03 << FIFOLD_CLASS_SHIFT) +#define FIFOLD_CLASS_SKIP (0x00 << FIFOLD_CLASS_SHIFT) +#define FIFOLD_CLASS_CLASS1 (0x01 << FIFOLD_CLASS_SHIFT) +#define FIFOLD_CLASS_CLASS2 (0x02 << FIFOLD_CLASS_SHIFT) +#define FIFOLD_CLASS_BOTH (0x03 << FIFOLD_CLASS_SHIFT) + +#define FIFOST_CLASS_SHIFT 25 +#define FIFOST_CLASS_MASK (0x03 << FIFOST_CLASS_SHIFT) +#define FIFOST_CLASS_NORMAL (0x00 << FIFOST_CLASS_SHIFT) +#define FIFOST_CLASS_CLASS1KEY (0x01 << FIFOST_CLASS_SHIFT) +#define FIFOST_CLASS_CLASS2KEY (0x02 << FIFOST_CLASS_SHIFT) + +/* + * Scatter-Gather Table/Variable Length Field + * If set for FIFO_LOAD, refers to a SG table. Within + * SEQ_FIFO_LOAD, is variable input sequence + */ +#define FIFOLDST_SGF_SHIFT 24 +#define FIFOLDST_SGF_MASK (1 << FIFOLDST_SGF_SHIFT) +#define FIFOLDST_VLF_MASK (1 << FIFOLDST_SGF_SHIFT) +#define FIFOLDST_SGF (1 << FIFOLDST_SGF_SHIFT) +#define FIFOLDST_VLF (1 << FIFOLDST_SGF_SHIFT) + +/* Immediate - Data follows command in descriptor */ +#define FIFOLD_IMM_SHIFT 23 +#define FIFOLD_IMM_MASK (1 << FIFOLD_IMM_SHIFT) +#define FIFOLD_IMM (1 << FIFOLD_IMM_SHIFT) + +/* Continue - Not the last FIFO store to come */ +#define FIFOST_CONT_SHIFT 23 +#define FIFOST_CONT_MASK (1 << FIFOST_CONT_SHIFT) + +/* + * Extended Length - use 32-bit extended length that + * follows the pointer field. Illegal with IMM set + */ +#define FIFOLDST_EXT_SHIFT 22 +#define FIFOLDST_EXT_MASK (1 << FIFOLDST_EXT_SHIFT) +#define FIFOLDST_EXT (1 << FIFOLDST_EXT_SHIFT) + +/* Input data type.*/ +#define FIFOLD_TYPE_SHIFT 16 +#define FIFOLD_CONT_TYPE_SHIFT 19 /* shift past last-flush bits */ +#define FIFOLD_TYPE_MASK (0x3f << FIFOLD_TYPE_SHIFT) + +/* PK types */ +#define FIFOLD_TYPE_PK (0x00 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_MASK (0x30 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_TYPEMASK (0x0f << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_A0 (0x00 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_A1 (0x01 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_A2 (0x02 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_A3 (0x03 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_B0 (0x04 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_B1 (0x05 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_B2 (0x06 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_B3 (0x07 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_N (0x08 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_A (0x0c << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_B (0x0d << FIFOLD_TYPE_SHIFT) + +/* Other types. Need to OR in last/flush bits as desired */ +#define FIFOLD_TYPE_MSG_MASK (0x38 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_MSG (0x10 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_MSG1OUT2 (0x18 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_IV (0x20 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_BITDATA (0x28 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_AAD (0x30 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_ICV (0x38 << FIFOLD_TYPE_SHIFT) + +/* Last/Flush bits for use with "other" types above */ +#define FIFOLD_TYPE_ACT_MASK (0x07 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_NOACTION (0x00 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_FLUSH1 (0x01 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_LAST1 (0x02 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_LAST2FLUSH (0x03 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_LAST2 (0x04 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_LAST2FLUSH1 (0x05 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_LASTBOTH (0x06 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_LASTBOTHFL (0x07 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_NOINFOFIFO (0x0F << FIFOLD_TYPE_SHIFT) + +#define FIFOLDST_LEN_MASK 0xffff +#define FIFOLDST_EXT_LEN_MASK 0xffffffff + +/* Output data types */ +#define FIFOST_TYPE_SHIFT 16 +#define FIFOST_TYPE_MASK (0x3f << FIFOST_TYPE_SHIFT) + +#define FIFOST_TYPE_PKHA_A0 (0x00 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_PKHA_A1 (0x01 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_PKHA_A2 (0x02 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_PKHA_A3 (0x03 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_PKHA_B0 (0x04 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_PKHA_B1 (0x05 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_PKHA_B2 (0x06 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_PKHA_B3 (0x07 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_PKHA_N (0x08 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_PKHA_A (0x0c << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_PKHA_B (0x0d << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_KEY_CCM_JKEK (0x14 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_AF_SBOX_JKEK (0x20 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_AF_SBOX_TKEK (0x21 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_PKHA_E_JKEK (0x22 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_PKHA_E_TKEK (0x23 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_KEY_KEK (0x24 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_KEY_TKEK (0x25 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_SPLIT_KEK (0x26 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_SPLIT_TKEK (0x27 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_OUTFIFO_KEK (0x28 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_OUTFIFO_TKEK (0x29 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_MESSAGE_DATA (0x30 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_RNGSTORE (0x34 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_RNGFIFO (0x35 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_METADATA (0x3e << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_SKIP (0x3f << FIFOST_TYPE_SHIFT) + +/* + * OPERATION Command Constructs + */ + +/* Operation type selectors - OP TYPE */ +#define OP_TYPE_SHIFT 24 +#define OP_TYPE_MASK (0x07 << OP_TYPE_SHIFT) + +#define OP_TYPE_UNI_PROTOCOL (0x00 << OP_TYPE_SHIFT) +#define OP_TYPE_PK (0x01 << OP_TYPE_SHIFT) +#define OP_TYPE_CLASS1_ALG (0x02 << OP_TYPE_SHIFT) +#define OP_TYPE_CLASS2_ALG (0x04 << OP_TYPE_SHIFT) +#define OP_TYPE_DECAP_PROTOCOL (0x06 << OP_TYPE_SHIFT) +#define OP_TYPE_ENCAP_PROTOCOL (0x07 << OP_TYPE_SHIFT) + +/* ProtocolID selectors - PROTID */ +#define OP_PCLID_SHIFT 16 +#define OP_PCLID_MASK (0xff << 16) + +/* Assuming OP_TYPE = OP_TYPE_UNI_PROTOCOL */ +#define OP_PCLID_IKEV1_PRF (0x01 << OP_PCLID_SHIFT) +#define OP_PCLID_IKEV2_PRF (0x02 << OP_PCLID_SHIFT) +#define OP_PCLID_SSL30_PRF (0x08 << OP_PCLID_SHIFT) +#define OP_PCLID_TLS10_PRF (0x09 << OP_PCLID_SHIFT) +#define OP_PCLID_TLS11_PRF (0x0a << OP_PCLID_SHIFT) +#define OP_PCLID_DTLS10_PRF (0x0c << OP_PCLID_SHIFT) +#define OP_PCLID_PRF (0x06 << OP_PCLID_SHIFT) +#define OP_PCLID_BLOB (0x0d << OP_PCLID_SHIFT) +#define OP_PCLID_SECRETKEY (0x11 << OP_PCLID_SHIFT) +#define OP_PCLID_PUBLICKEYPAIR (0x14 << OP_PCLID_SHIFT) +#define OP_PCLID_DSASIGN (0x15 << OP_PCLID_SHIFT) +#define OP_PCLID_DSAVERIFY (0x16 << OP_PCLID_SHIFT) +#define OP_PCLID_RSAENC_PUBKEY (0x18 << OP_PCLID_SHIFT) +#define OP_PCLID_RSADEC_PRVKEY (0x19 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_MD5 (0x20 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_SHA1 (0x21 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_SHA224 (0x22 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_SHA256 (0x23 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_SHA384 (0x24 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_SHA512 (0x25 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_RIF_MD5 (0x60 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_RIF_SHA1 (0x61 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_RIF_SHA224 (0x62 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_RIF_SHA256 (0x63 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_RIF_SHA384 (0x64 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_RIF_SHA512 (0x65 << OP_PCLID_SHIFT) + +/* Assuming OP_TYPE = OP_TYPE_DECAP_PROTOCOL/ENCAP_PROTOCOL */ +#define OP_PCLID_IPSEC (0x01 << OP_PCLID_SHIFT) +#define OP_PCLID_SRTP (0x02 << OP_PCLID_SHIFT) +#define OP_PCLID_MACSEC (0x03 << OP_PCLID_SHIFT) +#define OP_PCLID_WIFI (0x04 << OP_PCLID_SHIFT) +#define OP_PCLID_WIMAX (0x05 << OP_PCLID_SHIFT) +#define OP_PCLID_SSL30 (0x08 << OP_PCLID_SHIFT) +#define OP_PCLID_TLS10 (0x09 << OP_PCLID_SHIFT) +#define OP_PCLID_TLS11 (0x0a << OP_PCLID_SHIFT) +#define OP_PCLID_TLS12 (0x0b << OP_PCLID_SHIFT) +#define OP_PCLID_DTLS (0x0c << OP_PCLID_SHIFT) + +/* + * ProtocolInfo selectors + */ +#define OP_PCLINFO_MASK 0xffff + +/* for OP_PCLID_IPSEC */ +#define OP_PCL_IPSEC_CIPHER_MASK 0xff00 +#define OP_PCL_IPSEC_AUTH_MASK 0x00ff + +#define OP_PCL_IPSEC_DES_IV64 0x0100 +#define OP_PCL_IPSEC_DES 0x0200 +#define OP_PCL_IPSEC_3DES 0x0300 +#define OP_PCL_IPSEC_AES_CBC 0x0c00 +#define OP_PCL_IPSEC_AES_CTR 0x0d00 +#define OP_PCL_IPSEC_AES_XTS 0x1600 +#define OP_PCL_IPSEC_AES_CCM8 0x0e00 +#define OP_PCL_IPSEC_AES_CCM12 0x0f00 +#define OP_PCL_IPSEC_AES_CCM16 0x1000 +#define OP_PCL_IPSEC_AES_GCM8 0x1200 +#define OP_PCL_IPSEC_AES_GCM12 0x1300 +#define OP_PCL_IPSEC_AES_GCM16 0x1400 + +#define OP_PCL_IPSEC_HMAC_NULL 0x0000 +#define OP_PCL_IPSEC_HMAC_MD5_96 0x0001 +#define OP_PCL_IPSEC_HMAC_SHA1_96 0x0002 +#define OP_PCL_IPSEC_AES_XCBC_MAC_96 0x0005 +#define OP_PCL_IPSEC_HMAC_MD5_128 0x0006 +#define OP_PCL_IPSEC_HMAC_SHA1_160 0x0007 +#define OP_PCL_IPSEC_HMAC_SHA2_256_128 0x000c +#define OP_PCL_IPSEC_HMAC_SHA2_384_192 0x000d +#define OP_PCL_IPSEC_HMAC_SHA2_512_256 0x000e + +/* For SRTP - OP_PCLID_SRTP */ +#define OP_PCL_SRTP_CIPHER_MASK 0xff00 +#define OP_PCL_SRTP_AUTH_MASK 0x00ff + +#define OP_PCL_SRTP_AES_CTR 0x0d00 + +#define OP_PCL_SRTP_HMAC_SHA1_160 0x0007 + +/* For SSL 3.0 - OP_PCLID_SSL30 */ +#define OP_PCL_SSL30_AES_128_CBC_SHA 0x002f +#define OP_PCL_SSL30_AES_128_CBC_SHA_2 0x0030 +#define OP_PCL_SSL30_AES_128_CBC_SHA_3 0x0031 +#define OP_PCL_SSL30_AES_128_CBC_SHA_4 0x0032 +#define OP_PCL_SSL30_AES_128_CBC_SHA_5 0x0033 +#define OP_PCL_SSL30_AES_128_CBC_SHA_6 0x0034 +#define OP_PCL_SSL30_AES_128_CBC_SHA_7 0x008c +#define OP_PCL_SSL30_AES_128_CBC_SHA_8 0x0090 +#define OP_PCL_SSL30_AES_128_CBC_SHA_9 0x0094 +#define OP_PCL_SSL30_AES_128_CBC_SHA_10 0xc004 +#define OP_PCL_SSL30_AES_128_CBC_SHA_11 0xc009 +#define OP_PCL_SSL30_AES_128_CBC_SHA_12 0xc00e +#define OP_PCL_SSL30_AES_128_CBC_SHA_13 0xc013 +#define OP_PCL_SSL30_AES_128_CBC_SHA_14 0xc018 +#define OP_PCL_SSL30_AES_128_CBC_SHA_15 0xc01d +#define OP_PCL_SSL30_AES_128_CBC_SHA_16 0xc01e +#define OP_PCL_SSL30_AES_128_CBC_SHA_17 0xc01f + +#define OP_PCL_SSL30_AES_256_CBC_SHA 0x0035 +#define OP_PCL_SSL30_AES_256_CBC_SHA_2 0x0036 +#define OP_PCL_SSL30_AES_256_CBC_SHA_3 0x0037 +#define OP_PCL_SSL30_AES_256_CBC_SHA_4 0x0038 +#define OP_PCL_SSL30_AES_256_CBC_SHA_5 0x0039 +#define OP_PCL_SSL30_AES_256_CBC_SHA_6 0x003a +#define OP_PCL_SSL30_AES_256_CBC_SHA_7 0x008d +#define OP_PCL_SSL30_AES_256_CBC_SHA_8 0x0091 +#define OP_PCL_SSL30_AES_256_CBC_SHA_9 0x0095 +#define OP_PCL_SSL30_AES_256_CBC_SHA_10 0xc005 +#define OP_PCL_SSL30_AES_256_CBC_SHA_11 0xc00a +#define OP_PCL_SSL30_AES_256_CBC_SHA_12 0xc00f +#define OP_PCL_SSL30_AES_256_CBC_SHA_13 0xc014 +#define OP_PCL_SSL30_AES_256_CBC_SHA_14 0xc019 +#define OP_PCL_SSL30_AES_256_CBC_SHA_15 0xc020 +#define OP_PCL_SSL30_AES_256_CBC_SHA_16 0xc021 +#define OP_PCL_SSL30_AES_256_CBC_SHA_17 0xc022 + +#define OP_PCL_SSL30_3DES_EDE_CBC_MD5 0x0023 + +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA 0x001f +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_2 0x008b +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_3 0x008f +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_4 0x0093 +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_5 0x000a +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_6 0x000d +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_7 0x0010 +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_8 0x0013 +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_9 0x0016 +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_10 0x001b +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_11 0xc003 +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_12 0xc008 +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_13 0xc00d +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_14 0xc012 +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_15 0xc017 +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_16 0xc01a +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_17 0xc01b +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_18 0xc01c + +#define OP_PCL_SSL30_DES40_CBC_MD5 0x0029 + +#define OP_PCL_SSL30_DES_CBC_MD5 0x0022 + +#define OP_PCL_SSL30_DES40_CBC_SHA 0x0008 +#define OP_PCL_SSL30_DES40_CBC_SHA_2 0x000b +#define OP_PCL_SSL30_DES40_CBC_SHA_3 0x000e +#define OP_PCL_SSL30_DES40_CBC_SHA_4 0x0011 +#define OP_PCL_SSL30_DES40_CBC_SHA_5 0x0014 +#define OP_PCL_SSL30_DES40_CBC_SHA_6 0x0019 +#define OP_PCL_SSL30_DES40_CBC_SHA_7 0x0026 + +#define OP_PCL_SSL30_DES_CBC_SHA 0x001e +#define OP_PCL_SSL30_DES_CBC_SHA_2 0x0009 +#define OP_PCL_SSL30_DES_CBC_SHA_3 0x000c +#define OP_PCL_SSL30_DES_CBC_SHA_4 0x000f +#define OP_PCL_SSL30_DES_CBC_SHA_5 0x0012 +#define OP_PCL_SSL30_DES_CBC_SHA_6 0x0015 +#define OP_PCL_SSL30_DES_CBC_SHA_7 0x001a + +#define OP_PCL_SSL30_RC4_128_MD5 0x0024 +#define OP_PCL_SSL30_RC4_128_MD5_2 0x0004 +#define OP_PCL_SSL30_RC4_128_MD5_3 0x0018 + +#define OP_PCL_SSL30_RC4_40_MD5 0x002b +#define OP_PCL_SSL30_RC4_40_MD5_2 0x0003 +#define OP_PCL_SSL30_RC4_40_MD5_3 0x0017 + +#define OP_PCL_SSL30_RC4_128_SHA 0x0020 +#define OP_PCL_SSL30_RC4_128_SHA_2 0x008a +#define OP_PCL_SSL30_RC4_128_SHA_3 0x008e +#define OP_PCL_SSL30_RC4_128_SHA_4 0x0092 +#define OP_PCL_SSL30_RC4_128_SHA_5 0x0005 +#define OP_PCL_SSL30_RC4_128_SHA_6 0xc002 +#define OP_PCL_SSL30_RC4_128_SHA_7 0xc007 +#define OP_PCL_SSL30_RC4_128_SHA_8 0xc00c +#define OP_PCL_SSL30_RC4_128_SHA_9 0xc011 +#define OP_PCL_SSL30_RC4_128_SHA_10 0xc016 + +#define OP_PCL_SSL30_RC4_40_SHA 0x0028 + + +/* For TLS 1.0 - OP_PCLID_TLS10 */ +#define OP_PCL_TLS10_AES_128_CBC_SHA 0x002f +#define OP_PCL_TLS10_AES_128_CBC_SHA_2 0x0030 +#define OP_PCL_TLS10_AES_128_CBC_SHA_3 0x0031 +#define OP_PCL_TLS10_AES_128_CBC_SHA_4 0x0032 +#define OP_PCL_TLS10_AES_128_CBC_SHA_5 0x0033 +#define OP_PCL_TLS10_AES_128_CBC_SHA_6 0x0034 +#define OP_PCL_TLS10_AES_128_CBC_SHA_7 0x008c +#define OP_PCL_TLS10_AES_128_CBC_SHA_8 0x0090 +#define OP_PCL_TLS10_AES_128_CBC_SHA_9 0x0094 +#define OP_PCL_TLS10_AES_128_CBC_SHA_10 0xc004 +#define OP_PCL_TLS10_AES_128_CBC_SHA_11 0xc009 +#define OP_PCL_TLS10_AES_128_CBC_SHA_12 0xc00e +#define OP_PCL_TLS10_AES_128_CBC_SHA_13 0xc013 +#define OP_PCL_TLS10_AES_128_CBC_SHA_14 0xc018 +#define OP_PCL_TLS10_AES_128_CBC_SHA_15 0xc01d +#define OP_PCL_TLS10_AES_128_CBC_SHA_16 0xc01e +#define OP_PCL_TLS10_AES_128_CBC_SHA_17 0xc01f + +#define OP_PCL_TLS10_AES_256_CBC_SHA 0x0035 +#define OP_PCL_TLS10_AES_256_CBC_SHA_2 0x0036 +#define OP_PCL_TLS10_AES_256_CBC_SHA_3 0x0037 +#define OP_PCL_TLS10_AES_256_CBC_SHA_4 0x0038 +#define OP_PCL_TLS10_AES_256_CBC_SHA_5 0x0039 +#define OP_PCL_TLS10_AES_256_CBC_SHA_6 0x003a +#define OP_PCL_TLS10_AES_256_CBC_SHA_7 0x008d +#define OP_PCL_TLS10_AES_256_CBC_SHA_8 0x0091 +#define OP_PCL_TLS10_AES_256_CBC_SHA_9 0x0095 +#define OP_PCL_TLS10_AES_256_CBC_SHA_10 0xc005 +#define OP_PCL_TLS10_AES_256_CBC_SHA_11 0xc00a +#define OP_PCL_TLS10_AES_256_CBC_SHA_12 0xc00f +#define OP_PCL_TLS10_AES_256_CBC_SHA_13 0xc014 +#define OP_PCL_TLS10_AES_256_CBC_SHA_14 0xc019 +#define OP_PCL_TLS10_AES_256_CBC_SHA_15 0xc020 +#define OP_PCL_TLS10_AES_256_CBC_SHA_16 0xc021 +#define OP_PCL_TLS10_AES_256_CBC_SHA_17 0xc022 + +/* #define OP_PCL_TLS10_3DES_EDE_CBC_MD5 0x0023 */ + +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA 0x001f +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_2 0x008b +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_3 0x008f +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_4 0x0093 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_5 0x000a +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_6 0x000d +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_7 0x0010 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_8 0x0013 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_9 0x0016 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_10 0x001b +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_11 0xc003 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_12 0xc008 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_13 0xc00d +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_14 0xc012 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_15 0xc017 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_16 0xc01a +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_17 0xc01b +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_18 0xc01c + +#define OP_PCL_TLS10_DES40_CBC_MD5 0x0029 + +#define OP_PCL_TLS10_DES_CBC_MD5 0x0022 + +#define OP_PCL_TLS10_DES40_CBC_SHA 0x0008 +#define OP_PCL_TLS10_DES40_CBC_SHA_2 0x000b +#define OP_PCL_TLS10_DES40_CBC_SHA_3 0x000e +#define OP_PCL_TLS10_DES40_CBC_SHA_4 0x0011 +#define OP_PCL_TLS10_DES40_CBC_SHA_5 0x0014 +#define OP_PCL_TLS10_DES40_CBC_SHA_6 0x0019 +#define OP_PCL_TLS10_DES40_CBC_SHA_7 0x0026 + + +#define OP_PCL_TLS10_DES_CBC_SHA 0x001e +#define OP_PCL_TLS10_DES_CBC_SHA_2 0x0009 +#define OP_PCL_TLS10_DES_CBC_SHA_3 0x000c +#define OP_PCL_TLS10_DES_CBC_SHA_4 0x000f +#define OP_PCL_TLS10_DES_CBC_SHA_5 0x0012 +#define OP_PCL_TLS10_DES_CBC_SHA_6 0x0015 +#define OP_PCL_TLS10_DES_CBC_SHA_7 0x001a + +#define OP_PCL_TLS10_RC4_128_MD5 0x0024 +#define OP_PCL_TLS10_RC4_128_MD5_2 0x0004 +#define OP_PCL_TLS10_RC4_128_MD5_3 0x0018 + +#define OP_PCL_TLS10_RC4_40_MD5 0x002b +#define OP_PCL_TLS10_RC4_40_MD5_2 0x0003 +#define OP_PCL_TLS10_RC4_40_MD5_3 0x0017 + +#define OP_PCL_TLS10_RC4_128_SHA 0x0020 +#define OP_PCL_TLS10_RC4_128_SHA_2 0x008a +#define OP_PCL_TLS10_RC4_128_SHA_3 0x008e +#define OP_PCL_TLS10_RC4_128_SHA_4 0x0092 +#define OP_PCL_TLS10_RC4_128_SHA_5 0x0005 +#define OP_PCL_TLS10_RC4_128_SHA_6 0xc002 +#define OP_PCL_TLS10_RC4_128_SHA_7 0xc007 +#define OP_PCL_TLS10_RC4_128_SHA_8 0xc00c +#define OP_PCL_TLS10_RC4_128_SHA_9 0xc011 +#define OP_PCL_TLS10_RC4_128_SHA_10 0xc016 + +#define OP_PCL_TLS10_RC4_40_SHA 0x0028 + +#define OP_PCL_TLS10_3DES_EDE_CBC_MD5 0xff23 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA160 0xff30 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA224 0xff34 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA256 0xff36 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA384 0xff33 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA512 0xff35 +#define OP_PCL_TLS10_AES_128_CBC_SHA160 0xff80 +#define OP_PCL_TLS10_AES_128_CBC_SHA224 0xff84 +#define OP_PCL_TLS10_AES_128_CBC_SHA256 0xff86 +#define OP_PCL_TLS10_AES_128_CBC_SHA384 0xff83 +#define OP_PCL_TLS10_AES_128_CBC_SHA512 0xff85 +#define OP_PCL_TLS10_AES_192_CBC_SHA160 0xff20 +#define OP_PCL_TLS10_AES_192_CBC_SHA224 0xff24 +#define OP_PCL_TLS10_AES_192_CBC_SHA256 0xff26 +#define OP_PCL_TLS10_AES_192_CBC_SHA384 0xff23 +#define OP_PCL_TLS10_AES_192_CBC_SHA512 0xff25 +#define OP_PCL_TLS10_AES_256_CBC_SHA160 0xff60 +#define OP_PCL_TLS10_AES_256_CBC_SHA224 0xff64 +#define OP_PCL_TLS10_AES_256_CBC_SHA256 0xff66 +#define OP_PCL_TLS10_AES_256_CBC_SHA384 0xff63 +#define OP_PCL_TLS10_AES_256_CBC_SHA512 0xff65 + + + +/* For TLS 1.1 - OP_PCLID_TLS11 */ +#define OP_PCL_TLS11_AES_128_CBC_SHA 0x002f +#define OP_PCL_TLS11_AES_128_CBC_SHA_2 0x0030 +#define OP_PCL_TLS11_AES_128_CBC_SHA_3 0x0031 +#define OP_PCL_TLS11_AES_128_CBC_SHA_4 0x0032 +#define OP_PCL_TLS11_AES_128_CBC_SHA_5 0x0033 +#define OP_PCL_TLS11_AES_128_CBC_SHA_6 0x0034 +#define OP_PCL_TLS11_AES_128_CBC_SHA_7 0x008c +#define OP_PCL_TLS11_AES_128_CBC_SHA_8 0x0090 +#define OP_PCL_TLS11_AES_128_CBC_SHA_9 0x0094 +#define OP_PCL_TLS11_AES_128_CBC_SHA_10 0xc004 +#define OP_PCL_TLS11_AES_128_CBC_SHA_11 0xc009 +#define OP_PCL_TLS11_AES_128_CBC_SHA_12 0xc00e +#define OP_PCL_TLS11_AES_128_CBC_SHA_13 0xc013 +#define OP_PCL_TLS11_AES_128_CBC_SHA_14 0xc018 +#define OP_PCL_TLS11_AES_128_CBC_SHA_15 0xc01d +#define OP_PCL_TLS11_AES_128_CBC_SHA_16 0xc01e +#define OP_PCL_TLS11_AES_128_CBC_SHA_17 0xc01f + +#define OP_PCL_TLS11_AES_256_CBC_SHA 0x0035 +#define OP_PCL_TLS11_AES_256_CBC_SHA_2 0x0036 +#define OP_PCL_TLS11_AES_256_CBC_SHA_3 0x0037 +#define OP_PCL_TLS11_AES_256_CBC_SHA_4 0x0038 +#define OP_PCL_TLS11_AES_256_CBC_SHA_5 0x0039 +#define OP_PCL_TLS11_AES_256_CBC_SHA_6 0x003a +#define OP_PCL_TLS11_AES_256_CBC_SHA_7 0x008d +#define OP_PCL_TLS11_AES_256_CBC_SHA_8 0x0091 +#define OP_PCL_TLS11_AES_256_CBC_SHA_9 0x0095 +#define OP_PCL_TLS11_AES_256_CBC_SHA_10 0xc005 +#define OP_PCL_TLS11_AES_256_CBC_SHA_11 0xc00a +#define OP_PCL_TLS11_AES_256_CBC_SHA_12 0xc00f +#define OP_PCL_TLS11_AES_256_CBC_SHA_13 0xc014 +#define OP_PCL_TLS11_AES_256_CBC_SHA_14 0xc019 +#define OP_PCL_TLS11_AES_256_CBC_SHA_15 0xc020 +#define OP_PCL_TLS11_AES_256_CBC_SHA_16 0xc021 +#define OP_PCL_TLS11_AES_256_CBC_SHA_17 0xc022 + +/* #define OP_PCL_TLS11_3DES_EDE_CBC_MD5 0x0023 */ + +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA 0x001f +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_2 0x008b +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_3 0x008f +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_4 0x0093 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_5 0x000a +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_6 0x000d +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_7 0x0010 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_8 0x0013 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_9 0x0016 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_10 0x001b +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_11 0xc003 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_12 0xc008 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_13 0xc00d +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_14 0xc012 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_15 0xc017 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_16 0xc01a +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_17 0xc01b +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_18 0xc01c + +#define OP_PCL_TLS11_DES40_CBC_MD5 0x0029 + +#define OP_PCL_TLS11_DES_CBC_MD5 0x0022 + +#define OP_PCL_TLS11_DES40_CBC_SHA 0x0008 +#define OP_PCL_TLS11_DES40_CBC_SHA_2 0x000b +#define OP_PCL_TLS11_DES40_CBC_SHA_3 0x000e +#define OP_PCL_TLS11_DES40_CBC_SHA_4 0x0011 +#define OP_PCL_TLS11_DES40_CBC_SHA_5 0x0014 +#define OP_PCL_TLS11_DES40_CBC_SHA_6 0x0019 +#define OP_PCL_TLS11_DES40_CBC_SHA_7 0x0026 + +#define OP_PCL_TLS11_DES_CBC_SHA 0x001e +#define OP_PCL_TLS11_DES_CBC_SHA_2 0x0009 +#define OP_PCL_TLS11_DES_CBC_SHA_3 0x000c +#define OP_PCL_TLS11_DES_CBC_SHA_4 0x000f +#define OP_PCL_TLS11_DES_CBC_SHA_5 0x0012 +#define OP_PCL_TLS11_DES_CBC_SHA_6 0x0015 +#define OP_PCL_TLS11_DES_CBC_SHA_7 0x001a + +#define OP_PCL_TLS11_RC4_128_MD5 0x0024 +#define OP_PCL_TLS11_RC4_128_MD5_2 0x0004 +#define OP_PCL_TLS11_RC4_128_MD5_3 0x0018 + +#define OP_PCL_TLS11_RC4_40_MD5 0x002b +#define OP_PCL_TLS11_RC4_40_MD5_2 0x0003 +#define OP_PCL_TLS11_RC4_40_MD5_3 0x0017 + +#define OP_PCL_TLS11_RC4_128_SHA 0x0020 +#define OP_PCL_TLS11_RC4_128_SHA_2 0x008a +#define OP_PCL_TLS11_RC4_128_SHA_3 0x008e +#define OP_PCL_TLS11_RC4_128_SHA_4 0x0092 +#define OP_PCL_TLS11_RC4_128_SHA_5 0x0005 +#define OP_PCL_TLS11_RC4_128_SHA_6 0xc002 +#define OP_PCL_TLS11_RC4_128_SHA_7 0xc007 +#define OP_PCL_TLS11_RC4_128_SHA_8 0xc00c +#define OP_PCL_TLS11_RC4_128_SHA_9 0xc011 +#define OP_PCL_TLS11_RC4_128_SHA_10 0xc016 + +#define OP_PCL_TLS11_RC4_40_SHA 0x0028 + +#define OP_PCL_TLS11_3DES_EDE_CBC_MD5 0xff23 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA160 0xff30 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA224 0xff34 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA256 0xff36 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA384 0xff33 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA512 0xff35 +#define OP_PCL_TLS11_AES_128_CBC_SHA160 0xff80 +#define OP_PCL_TLS11_AES_128_CBC_SHA224 0xff84 +#define OP_PCL_TLS11_AES_128_CBC_SHA256 0xff86 +#define OP_PCL_TLS11_AES_128_CBC_SHA384 0xff83 +#define OP_PCL_TLS11_AES_128_CBC_SHA512 0xff85 +#define OP_PCL_TLS11_AES_192_CBC_SHA160 0xff20 +#define OP_PCL_TLS11_AES_192_CBC_SHA224 0xff24 +#define OP_PCL_TLS11_AES_192_CBC_SHA256 0xff26 +#define OP_PCL_TLS11_AES_192_CBC_SHA384 0xff23 +#define OP_PCL_TLS11_AES_192_CBC_SHA512 0xff25 +#define OP_PCL_TLS11_AES_256_CBC_SHA160 0xff60 +#define OP_PCL_TLS11_AES_256_CBC_SHA224 0xff64 +#define OP_PCL_TLS11_AES_256_CBC_SHA256 0xff66 +#define OP_PCL_TLS11_AES_256_CBC_SHA384 0xff63 +#define OP_PCL_TLS11_AES_256_CBC_SHA512 0xff65 + + +/* For TLS 1.2 - OP_PCLID_TLS12 */ +#define OP_PCL_TLS12_AES_128_CBC_SHA 0x002f +#define OP_PCL_TLS12_AES_128_CBC_SHA_2 0x0030 +#define OP_PCL_TLS12_AES_128_CBC_SHA_3 0x0031 +#define OP_PCL_TLS12_AES_128_CBC_SHA_4 0x0032 +#define OP_PCL_TLS12_AES_128_CBC_SHA_5 0x0033 +#define OP_PCL_TLS12_AES_128_CBC_SHA_6 0x0034 +#define OP_PCL_TLS12_AES_128_CBC_SHA_7 0x008c +#define OP_PCL_TLS12_AES_128_CBC_SHA_8 0x0090 +#define OP_PCL_TLS12_AES_128_CBC_SHA_9 0x0094 +#define OP_PCL_TLS12_AES_128_CBC_SHA_10 0xc004 +#define OP_PCL_TLS12_AES_128_CBC_SHA_11 0xc009 +#define OP_PCL_TLS12_AES_128_CBC_SHA_12 0xc00e +#define OP_PCL_TLS12_AES_128_CBC_SHA_13 0xc013 +#define OP_PCL_TLS12_AES_128_CBC_SHA_14 0xc018 +#define OP_PCL_TLS12_AES_128_CBC_SHA_15 0xc01d +#define OP_PCL_TLS12_AES_128_CBC_SHA_16 0xc01e +#define OP_PCL_TLS12_AES_128_CBC_SHA_17 0xc01f + +#define OP_PCL_TLS12_AES_256_CBC_SHA 0x0035 +#define OP_PCL_TLS12_AES_256_CBC_SHA_2 0x0036 +#define OP_PCL_TLS12_AES_256_CBC_SHA_3 0x0037 +#define OP_PCL_TLS12_AES_256_CBC_SHA_4 0x0038 +#define OP_PCL_TLS12_AES_256_CBC_SHA_5 0x0039 +#define OP_PCL_TLS12_AES_256_CBC_SHA_6 0x003a +#define OP_PCL_TLS12_AES_256_CBC_SHA_7 0x008d +#define OP_PCL_TLS12_AES_256_CBC_SHA_8 0x0091 +#define OP_PCL_TLS12_AES_256_CBC_SHA_9 0x0095 +#define OP_PCL_TLS12_AES_256_CBC_SHA_10 0xc005 +#define OP_PCL_TLS12_AES_256_CBC_SHA_11 0xc00a +#define OP_PCL_TLS12_AES_256_CBC_SHA_12 0xc00f +#define OP_PCL_TLS12_AES_256_CBC_SHA_13 0xc014 +#define OP_PCL_TLS12_AES_256_CBC_SHA_14 0xc019 +#define OP_PCL_TLS12_AES_256_CBC_SHA_15 0xc020 +#define OP_PCL_TLS12_AES_256_CBC_SHA_16 0xc021 +#define OP_PCL_TLS12_AES_256_CBC_SHA_17 0xc022 + +/* #define OP_PCL_TLS12_3DES_EDE_CBC_MD5 0x0023 */ + +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA 0x001f +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_2 0x008b +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_3 0x008f +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_4 0x0093 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_5 0x000a +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_6 0x000d +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_7 0x0010 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_8 0x0013 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_9 0x0016 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_10 0x001b +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_11 0xc003 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_12 0xc008 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_13 0xc00d +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_14 0xc012 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_15 0xc017 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_16 0xc01a +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_17 0xc01b +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_18 0xc01c + +#define OP_PCL_TLS12_DES40_CBC_MD5 0x0029 + +#define OP_PCL_TLS12_DES_CBC_MD5 0x0022 + +#define OP_PCL_TLS12_DES40_CBC_SHA 0x0008 +#define OP_PCL_TLS12_DES40_CBC_SHA_2 0x000b +#define OP_PCL_TLS12_DES40_CBC_SHA_3 0x000e +#define OP_PCL_TLS12_DES40_CBC_SHA_4 0x0011 +#define OP_PCL_TLS12_DES40_CBC_SHA_5 0x0014 +#define OP_PCL_TLS12_DES40_CBC_SHA_6 0x0019 +#define OP_PCL_TLS12_DES40_CBC_SHA_7 0x0026 + +#define OP_PCL_TLS12_DES_CBC_SHA 0x001e +#define OP_PCL_TLS12_DES_CBC_SHA_2 0x0009 +#define OP_PCL_TLS12_DES_CBC_SHA_3 0x000c +#define OP_PCL_TLS12_DES_CBC_SHA_4 0x000f +#define OP_PCL_TLS12_DES_CBC_SHA_5 0x0012 +#define OP_PCL_TLS12_DES_CBC_SHA_6 0x0015 +#define OP_PCL_TLS12_DES_CBC_SHA_7 0x001a + +#define OP_PCL_TLS12_RC4_128_MD5 0x0024 +#define OP_PCL_TLS12_RC4_128_MD5_2 0x0004 +#define OP_PCL_TLS12_RC4_128_MD5_3 0x0018 + +#define OP_PCL_TLS12_RC4_40_MD5 0x002b +#define OP_PCL_TLS12_RC4_40_MD5_2 0x0003 +#define OP_PCL_TLS12_RC4_40_MD5_3 0x0017 + +#define OP_PCL_TLS12_RC4_128_SHA 0x0020 +#define OP_PCL_TLS12_RC4_128_SHA_2 0x008a +#define OP_PCL_TLS12_RC4_128_SHA_3 0x008e +#define OP_PCL_TLS12_RC4_128_SHA_4 0x0092 +#define OP_PCL_TLS12_RC4_128_SHA_5 0x0005 +#define OP_PCL_TLS12_RC4_128_SHA_6 0xc002 +#define OP_PCL_TLS12_RC4_128_SHA_7 0xc007 +#define OP_PCL_TLS12_RC4_128_SHA_8 0xc00c +#define OP_PCL_TLS12_RC4_128_SHA_9 0xc011 +#define OP_PCL_TLS12_RC4_128_SHA_10 0xc016 + +#define OP_PCL_TLS12_RC4_40_SHA 0x0028 + +/* #define OP_PCL_TLS12_AES_128_CBC_SHA256 0x003c */ +#define OP_PCL_TLS12_AES_128_CBC_SHA256_2 0x003e +#define OP_PCL_TLS12_AES_128_CBC_SHA256_3 0x003f +#define OP_PCL_TLS12_AES_128_CBC_SHA256_4 0x0040 +#define OP_PCL_TLS12_AES_128_CBC_SHA256_5 0x0067 +#define OP_PCL_TLS12_AES_128_CBC_SHA256_6 0x006c + +/* #define OP_PCL_TLS12_AES_256_CBC_SHA256 0x003d */ +#define OP_PCL_TLS12_AES_256_CBC_SHA256_2 0x0068 +#define OP_PCL_TLS12_AES_256_CBC_SHA256_3 0x0069 +#define OP_PCL_TLS12_AES_256_CBC_SHA256_4 0x006a +#define OP_PCL_TLS12_AES_256_CBC_SHA256_5 0x006b +#define OP_PCL_TLS12_AES_256_CBC_SHA256_6 0x006d + +/* AEAD_AES_xxx_CCM/GCM remain to be defined... */ + +#define OP_PCL_TLS12_3DES_EDE_CBC_MD5 0xff23 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA160 0xff30 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA224 0xff34 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA256 0xff36 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA384 0xff33 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA512 0xff35 +#define OP_PCL_TLS12_AES_128_CBC_SHA160 0xff80 +#define OP_PCL_TLS12_AES_128_CBC_SHA224 0xff84 +#define OP_PCL_TLS12_AES_128_CBC_SHA256 0xff86 +#define OP_PCL_TLS12_AES_128_CBC_SHA384 0xff83 +#define OP_PCL_TLS12_AES_128_CBC_SHA512 0xff85 +#define OP_PCL_TLS12_AES_192_CBC_SHA160 0xff20 +#define OP_PCL_TLS12_AES_192_CBC_SHA224 0xff24 +#define OP_PCL_TLS12_AES_192_CBC_SHA256 0xff26 +#define OP_PCL_TLS12_AES_192_CBC_SHA384 0xff23 +#define OP_PCL_TLS12_AES_192_CBC_SHA512 0xff25 +#define OP_PCL_TLS12_AES_256_CBC_SHA160 0xff60 +#define OP_PCL_TLS12_AES_256_CBC_SHA224 0xff64 +#define OP_PCL_TLS12_AES_256_CBC_SHA256 0xff66 +#define OP_PCL_TLS12_AES_256_CBC_SHA384 0xff63 +#define OP_PCL_TLS12_AES_256_CBC_SHA512 0xff65 + +/* Blob protocol protinfo bits */ + +#define OP_PCL_BLOB_BLACK 0x0004 +#define OP_PCL_BLOB_EKT 0x0100 + +/* For DTLS - OP_PCLID_DTLS */ + +#define OP_PCL_DTLS_AES_128_CBC_SHA 0x002f +#define OP_PCL_DTLS_AES_128_CBC_SHA_2 0x0030 +#define OP_PCL_DTLS_AES_128_CBC_SHA_3 0x0031 +#define OP_PCL_DTLS_AES_128_CBC_SHA_4 0x0032 +#define OP_PCL_DTLS_AES_128_CBC_SHA_5 0x0033 +#define OP_PCL_DTLS_AES_128_CBC_SHA_6 0x0034 +#define OP_PCL_DTLS_AES_128_CBC_SHA_7 0x008c +#define OP_PCL_DTLS_AES_128_CBC_SHA_8 0x0090 +#define OP_PCL_DTLS_AES_128_CBC_SHA_9 0x0094 +#define OP_PCL_DTLS_AES_128_CBC_SHA_10 0xc004 +#define OP_PCL_DTLS_AES_128_CBC_SHA_11 0xc009 +#define OP_PCL_DTLS_AES_128_CBC_SHA_12 0xc00e +#define OP_PCL_DTLS_AES_128_CBC_SHA_13 0xc013 +#define OP_PCL_DTLS_AES_128_CBC_SHA_14 0xc018 +#define OP_PCL_DTLS_AES_128_CBC_SHA_15 0xc01d +#define OP_PCL_DTLS_AES_128_CBC_SHA_16 0xc01e +#define OP_PCL_DTLS_AES_128_CBC_SHA_17 0xc01f + +#define OP_PCL_DTLS_AES_256_CBC_SHA 0x0035 +#define OP_PCL_DTLS_AES_256_CBC_SHA_2 0x0036 +#define OP_PCL_DTLS_AES_256_CBC_SHA_3 0x0037 +#define OP_PCL_DTLS_AES_256_CBC_SHA_4 0x0038 +#define OP_PCL_DTLS_AES_256_CBC_SHA_5 0x0039 +#define OP_PCL_DTLS_AES_256_CBC_SHA_6 0x003a +#define OP_PCL_DTLS_AES_256_CBC_SHA_7 0x008d +#define OP_PCL_DTLS_AES_256_CBC_SHA_8 0x0091 +#define OP_PCL_DTLS_AES_256_CBC_SHA_9 0x0095 +#define OP_PCL_DTLS_AES_256_CBC_SHA_10 0xc005 +#define OP_PCL_DTLS_AES_256_CBC_SHA_11 0xc00a +#define OP_PCL_DTLS_AES_256_CBC_SHA_12 0xc00f +#define OP_PCL_DTLS_AES_256_CBC_SHA_13 0xc014 +#define OP_PCL_DTLS_AES_256_CBC_SHA_14 0xc019 +#define OP_PCL_DTLS_AES_256_CBC_SHA_15 0xc020 +#define OP_PCL_DTLS_AES_256_CBC_SHA_16 0xc021 +#define OP_PCL_DTLS_AES_256_CBC_SHA_17 0xc022 + +/* #define OP_PCL_DTLS_3DES_EDE_CBC_MD5 0x0023 */ + +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA 0x001f +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_2 0x008b +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_3 0x008f +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_4 0x0093 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_5 0x000a +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_6 0x000d +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_7 0x0010 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_8 0x0013 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_9 0x0016 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_10 0x001b +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_11 0xc003 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_12 0xc008 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_13 0xc00d +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_14 0xc012 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_15 0xc017 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_16 0xc01a +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_17 0xc01b +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_18 0xc01c + +#define OP_PCL_DTLS_DES40_CBC_MD5 0x0029 + +#define OP_PCL_DTLS_DES_CBC_MD5 0x0022 + +#define OP_PCL_DTLS_DES40_CBC_SHA 0x0008 +#define OP_PCL_DTLS_DES40_CBC_SHA_2 0x000b +#define OP_PCL_DTLS_DES40_CBC_SHA_3 0x000e +#define OP_PCL_DTLS_DES40_CBC_SHA_4 0x0011 +#define OP_PCL_DTLS_DES40_CBC_SHA_5 0x0014 +#define OP_PCL_DTLS_DES40_CBC_SHA_6 0x0019 +#define OP_PCL_DTLS_DES40_CBC_SHA_7 0x0026 + + +#define OP_PCL_DTLS_DES_CBC_SHA 0x001e +#define OP_PCL_DTLS_DES_CBC_SHA_2 0x0009 +#define OP_PCL_DTLS_DES_CBC_SHA_3 0x000c +#define OP_PCL_DTLS_DES_CBC_SHA_4 0x000f +#define OP_PCL_DTLS_DES_CBC_SHA_5 0x0012 +#define OP_PCL_DTLS_DES_CBC_SHA_6 0x0015 +#define OP_PCL_DTLS_DES_CBC_SHA_7 0x001a + + +#define OP_PCL_DTLS_3DES_EDE_CBC_MD5 0xff23 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA160 0xff30 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA224 0xff34 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA256 0xff36 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA384 0xff33 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA512 0xff35 +#define OP_PCL_DTLS_AES_128_CBC_SHA160 0xff80 +#define OP_PCL_DTLS_AES_128_CBC_SHA224 0xff84 +#define OP_PCL_DTLS_AES_128_CBC_SHA256 0xff86 +#define OP_PCL_DTLS_AES_128_CBC_SHA384 0xff83 +#define OP_PCL_DTLS_AES_128_CBC_SHA512 0xff85 +#define OP_PCL_DTLS_AES_192_CBC_SHA160 0xff20 +#define OP_PCL_DTLS_AES_192_CBC_SHA224 0xff24 +#define OP_PCL_DTLS_AES_192_CBC_SHA256 0xff26 +#define OP_PCL_DTLS_AES_192_CBC_SHA384 0xff23 +#define OP_PCL_DTLS_AES_192_CBC_SHA512 0xff25 +#define OP_PCL_DTLS_AES_256_CBC_SHA160 0xff60 +#define OP_PCL_DTLS_AES_256_CBC_SHA224 0xff64 +#define OP_PCL_DTLS_AES_256_CBC_SHA256 0xff66 +#define OP_PCL_DTLS_AES_256_CBC_SHA384 0xff63 +#define OP_PCL_DTLS_AES_256_CBC_SHA512 0xff65 + +/* 802.16 WiMAX protinfos */ +#define OP_PCL_WIMAX_OFDM 0x0201 +#define OP_PCL_WIMAX_OFDMA 0x0231 + +/* 802.11 WiFi protinfos */ +#define OP_PCL_WIFI 0xac04 + +/* MacSec protinfos */ +#define OP_PCL_MACSEC 0x0001 + +/* Derived Key Protocol (DKP) Protinfo */ +#define OP_PCL_DKP_SRC_SHIFT 14 +#define OP_PCL_DKP_SRC_MASK (3 << OP_PCL_DKP_SRC_SHIFT) +#define OP_PCL_DKP_SRC_IMM (0 << OP_PCL_DKP_SRC_SHIFT) +#define OP_PCL_DKP_SRC_SEQ (1 << OP_PCL_DKP_SRC_SHIFT) +#define OP_PCL_DKP_SRC_PTR (2 << OP_PCL_DKP_SRC_SHIFT) +#define OP_PCL_DKP_SRC_SGF (3 << OP_PCL_DKP_SRC_SHIFT) +#define OP_PCL_DKP_DST_SHIFT 12 +#define OP_PCL_DKP_DST_MASK (3 << OP_PCL_DKP_DST_SHIFT) +#define OP_PCL_DKP_DST_IMM (0 << OP_PCL_DKP_DST_SHIFT) +#define OP_PCL_DKP_DST_SEQ (1 << OP_PCL_DKP_DST_SHIFT) +#define OP_PCL_DKP_DST_PTR (2 << OP_PCL_DKP_DST_SHIFT) +#define OP_PCL_DKP_DST_SGF (3 << OP_PCL_DKP_DST_SHIFT) +#define OP_PCL_DKP_KEY_SHIFT 0 +#define OP_PCL_DKP_KEY_MASK (0xfff << OP_PCL_DKP_KEY_SHIFT) + +/* PKI unidirectional protocol protinfo bits */ +#define OP_PCL_PKPROT_TEST 0x0008 +#define OP_PCL_PKPROT_DECRYPT 0x0004 +#define OP_PCL_PKPROT_ECC 0x0002 +#define OP_PCL_PKPROT_F2M 0x0001 + +/* For non-protocol/alg-only op commands */ +#define OP_ALG_TYPE_SHIFT 24 +#define OP_ALG_TYPE_MASK (0x7 << OP_ALG_TYPE_SHIFT) +#define OP_ALG_TYPE_CLASS1 (2 << OP_ALG_TYPE_SHIFT) +#define OP_ALG_TYPE_CLASS2 (4 << OP_ALG_TYPE_SHIFT) + +/* version register fields */ +#define OP_VER_CCHA_NUM 0x000000ff /* Number CCHAs instantiated */ +#define OP_VER_CCHA_MISC 0x0000ff00 /* CCHA Miscellaneous Information */ +#define OP_VER_CCHA_REV 0x00ff0000 /* CCHA Revision Number */ +#define OP_VER_CCHA_VID 0xff000000 /* CCHA Version ID */ + +#define OP_ALG_ALGSEL_SHIFT 16 +#define OP_ALG_ALGSEL_MASK (0xff << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_SUBMASK (0x0f << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_AES (0x10 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_DES (0x20 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_3DES (0x21 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_ARC4 (0x30 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_CHA_MDHA (0x40 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_MD5 (0x40 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_SHA1 (0x41 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_SHA224 (0x42 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_SHA256 (0x43 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_SHA384 (0x44 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_SHA512 (0x45 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_RNG (0x50 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_SNOW (0x60 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_SNOW_F8 (0x60 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_KASUMI (0x70 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_CRC (0x90 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_SNOW_F9 (0xA0 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_CHACHA20 (0xD0 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_POLY1305 (0xE0 << OP_ALG_ALGSEL_SHIFT) + +#define OP_ALG_AAI_SHIFT 4 +#define OP_ALG_AAI_MASK (0x1ff << OP_ALG_AAI_SHIFT) + +/* blockcipher AAI set */ +#define OP_ALG_AAI_CTR_MOD128 (0x00 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD8 (0x01 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD16 (0x02 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD24 (0x03 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD32 (0x04 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD40 (0x05 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD48 (0x06 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD56 (0x07 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD64 (0x08 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD72 (0x09 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD80 (0x0a << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD88 (0x0b << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD96 (0x0c << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD104 (0x0d << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD112 (0x0e << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD120 (0x0f << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CBC (0x10 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_ECB (0x20 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CFB (0x30 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_OFB (0x40 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_XTS (0x50 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CMAC (0x60 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_XCBC_MAC (0x70 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CCM (0x80 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_GCM (0x90 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CBC_XCBCMAC (0xa0 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_XCBCMAC (0xb0 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CHECKODD (0x80 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_DK (0x100 << OP_ALG_AAI_SHIFT) + +/* randomizer AAI set */ +#define OP_ALG_AAI_RNG (0x00 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_RNG_NZB (0x10 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_RNG_OBP (0x20 << OP_ALG_AAI_SHIFT) + +/* RNG4 AAI set */ +#define OP_ALG_AAI_RNG4_SH_0 (0x00 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_RNG4_SH_1 (0x01 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_RNG4_PS (0x40 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_RNG4_AI (0x80 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_RNG4_SK (0x100 << OP_ALG_AAI_SHIFT) + +/* Chacha20 AAI set */ +#define OP_ALG_AAI_AEAD (0x002 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_KEYSTREAM (0x001 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_BC8 (0x008 << OP_ALG_AAI_SHIFT) + +/* hmac/smac AAI set */ +#define OP_ALG_AAI_HASH (0x00 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_HMAC (0x01 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_SMAC (0x02 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_HMAC_PRECOMP (0x04 << OP_ALG_AAI_SHIFT) + +/* CRC AAI set*/ +#define OP_ALG_AAI_802 (0x01 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_3385 (0x02 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CUST_POLY (0x04 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_DIS (0x10 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_DOS (0x20 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_DOC (0x40 << OP_ALG_AAI_SHIFT) + +/* Kasumi/SNOW AAI set */ +#define OP_ALG_AAI_F8 (0xc0 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_F9 (0xc8 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_GSM (0x10 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_EDGE (0x20 << OP_ALG_AAI_SHIFT) + +#define OP_ALG_AS_SHIFT 2 +#define OP_ALG_AS_MASK (0x3 << OP_ALG_AS_SHIFT) +#define OP_ALG_AS_UPDATE (0 << OP_ALG_AS_SHIFT) +#define OP_ALG_AS_INIT (1 << OP_ALG_AS_SHIFT) +#define OP_ALG_AS_FINALIZE (2 << OP_ALG_AS_SHIFT) +#define OP_ALG_AS_INITFINAL (3 << OP_ALG_AS_SHIFT) + +#define OP_ALG_ICV_SHIFT 1 +#define OP_ALG_ICV_MASK (1 << OP_ALG_ICV_SHIFT) +#define OP_ALG_ICV_OFF (0 << OP_ALG_ICV_SHIFT) +#define OP_ALG_ICV_ON (1 << OP_ALG_ICV_SHIFT) + +#define OP_ALG_PR_ON BIT(1) + +#define OP_ALG_DIR_SHIFT 0 +#define OP_ALG_DIR_MASK 1 +#define OP_ALG_DECRYPT 0 +#define OP_ALG_ENCRYPT 1 + +/* PKHA algorithm type set */ +#define OP_ALG_PK 0x00800000 +#define OP_ALG_PK_FUN_MASK 0x3f /* clrmem, modmath, or cpymem */ + +/* PKHA mode clear memory functions */ +#define OP_ALG_PKMODE_A_RAM 0x80000 +#define OP_ALG_PKMODE_B_RAM 0x40000 +#define OP_ALG_PKMODE_E_RAM 0x20000 +#define OP_ALG_PKMODE_N_RAM 0x10000 +#define OP_ALG_PKMODE_CLEARMEM 0x00001 + +/* PKHA mode modular-arithmetic functions */ +#define OP_ALG_PKMODE_MOD_IN_MONTY 0x80000 +#define OP_ALG_PKMODE_MOD_OUT_MONTY 0x40000 +#define OP_ALG_PKMODE_MOD_F2M 0x20000 +#define OP_ALG_PKMODE_MOD_R2_IN 0x10000 +#define OP_ALG_PKMODE_PRJECTV 0x00800 +#define OP_ALG_PKMODE_TIME_EQ 0x400 +#define OP_ALG_PKMODE_OUT_B 0x000 +#define OP_ALG_PKMODE_OUT_A 0x100 +#define OP_ALG_PKMODE_MOD_ADD 0x002 +#define OP_ALG_PKMODE_MOD_SUB_AB 0x003 +#define OP_ALG_PKMODE_MOD_SUB_BA 0x004 +#define OP_ALG_PKMODE_MOD_MULT 0x005 +#define OP_ALG_PKMODE_MOD_EXPO 0x006 +#define OP_ALG_PKMODE_MOD_REDUCT 0x007 +#define OP_ALG_PKMODE_MOD_INV 0x008 +#define OP_ALG_PKMODE_MOD_ECC_ADD 0x009 +#define OP_ALG_PKMODE_MOD_ECC_DBL 0x00a +#define OP_ALG_PKMODE_MOD_ECC_MULT 0x00b +#define OP_ALG_PKMODE_MOD_MONT_CNST 0x00c +#define OP_ALG_PKMODE_MOD_CRT_CNST 0x00d +#define OP_ALG_PKMODE_MOD_GCD 0x00e +#define OP_ALG_PKMODE_MOD_PRIMALITY 0x00f + +/* PKHA mode copy-memory functions */ +#define OP_ALG_PKMODE_SRC_REG_SHIFT 17 +#define OP_ALG_PKMODE_SRC_REG_MASK (7 << OP_ALG_PKMODE_SRC_REG_SHIFT) +#define OP_ALG_PKMODE_DST_REG_SHIFT 10 +#define OP_ALG_PKMODE_DST_REG_MASK (7 << OP_ALG_PKMODE_DST_REG_SHIFT) +#define OP_ALG_PKMODE_SRC_SEG_SHIFT 8 +#define OP_ALG_PKMODE_SRC_SEG_MASK (3 << OP_ALG_PKMODE_SRC_SEG_SHIFT) +#define OP_ALG_PKMODE_DST_SEG_SHIFT 6 +#define OP_ALG_PKMODE_DST_SEG_MASK (3 << OP_ALG_PKMODE_DST_SEG_SHIFT) + +#define OP_ALG_PKMODE_SRC_REG_A (0 << OP_ALG_PKMODE_SRC_REG_SHIFT) +#define OP_ALG_PKMODE_SRC_REG_B (1 << OP_ALG_PKMODE_SRC_REG_SHIFT) +#define OP_ALG_PKMODE_SRC_REG_N (3 << OP_ALG_PKMODE_SRC_REG_SHIFT) +#define OP_ALG_PKMODE_DST_REG_A (0 << OP_ALG_PKMODE_DST_REG_SHIFT) +#define OP_ALG_PKMODE_DST_REG_B (1 << OP_ALG_PKMODE_DST_REG_SHIFT) +#define OP_ALG_PKMODE_DST_REG_E (2 << OP_ALG_PKMODE_DST_REG_SHIFT) +#define OP_ALG_PKMODE_DST_REG_N (3 << OP_ALG_PKMODE_DST_REG_SHIFT) +#define OP_ALG_PKMODE_SRC_SEG_0 (0 << OP_ALG_PKMODE_SRC_SEG_SHIFT) +#define OP_ALG_PKMODE_SRC_SEG_1 (1 << OP_ALG_PKMODE_SRC_SEG_SHIFT) +#define OP_ALG_PKMODE_SRC_SEG_2 (2 << OP_ALG_PKMODE_SRC_SEG_SHIFT) +#define OP_ALG_PKMODE_SRC_SEG_3 (3 << OP_ALG_PKMODE_SRC_SEG_SHIFT) +#define OP_ALG_PKMODE_DST_SEG_0 (0 << OP_ALG_PKMODE_DST_SEG_SHIFT) +#define OP_ALG_PKMODE_DST_SEG_1 (1 << OP_ALG_PKMODE_DST_SEG_SHIFT) +#define OP_ALG_PKMODE_DST_SEG_2 (2 << OP_ALG_PKMODE_DST_SEG_SHIFT) +#define OP_ALG_PKMODE_DST_SEG_3 (3 << OP_ALG_PKMODE_DST_SEG_SHIFT) +#define OP_ALG_PKMODE_CPYMEM_N_SZ 0x80 +#define OP_ALG_PKMODE_CPYMEM_SRC_SZ 0x81 + +/* + * SEQ_IN_PTR Command Constructs + */ + +/* Release Buffers */ +#define SQIN_RBS 0x04000000 + +/* Sequence pointer is really a descriptor */ +#define SQIN_INL 0x02000000 + +/* Sequence pointer is a scatter-gather table */ +#define SQIN_SGF 0x01000000 + +/* Appends to a previous pointer */ +#define SQIN_PRE 0x00800000 + +/* Use extended length following pointer */ +#define SQIN_EXT 0x00400000 + +/* Restore sequence with pointer/length */ +#define SQIN_RTO 0x00200000 + +/* Replace job descriptor */ +#define SQIN_RJD 0x00100000 + +#define SQIN_LEN_SHIFT 0 +#define SQIN_LEN_MASK (0xffff << SQIN_LEN_SHIFT) + +/* + * SEQ_OUT_PTR Command Constructs + */ + +/* Sequence pointer is a scatter-gather table */ +#define SQOUT_SGF 0x01000000 + +/* Appends to a previous pointer */ +#define SQOUT_PRE SQIN_PRE + +/* Restore sequence with pointer/length */ +#define SQOUT_RTO SQIN_RTO + +/* Use extended length following pointer */ +#define SQOUT_EXT 0x00400000 + +#define SQOUT_LEN_SHIFT 0 +#define SQOUT_LEN_MASK (0xffff << SQOUT_LEN_SHIFT) + + +/* + * SIGNATURE Command Constructs + */ + +/* TYPE field is all that's relevant */ +#define SIGN_TYPE_SHIFT 16 +#define SIGN_TYPE_MASK (0x0f << SIGN_TYPE_SHIFT) + +#define SIGN_TYPE_FINAL (0x00 << SIGN_TYPE_SHIFT) +#define SIGN_TYPE_FINAL_RESTORE (0x01 << SIGN_TYPE_SHIFT) +#define SIGN_TYPE_FINAL_NONZERO (0x02 << SIGN_TYPE_SHIFT) +#define SIGN_TYPE_IMM_2 (0x0a << SIGN_TYPE_SHIFT) +#define SIGN_TYPE_IMM_3 (0x0b << SIGN_TYPE_SHIFT) +#define SIGN_TYPE_IMM_4 (0x0c << SIGN_TYPE_SHIFT) + +/* + * MOVE Command Constructs + */ + +#define MOVE_AUX_SHIFT 25 +#define MOVE_AUX_MASK (3 << MOVE_AUX_SHIFT) +#define MOVE_AUX_MS (2 << MOVE_AUX_SHIFT) +#define MOVE_AUX_LS (1 << MOVE_AUX_SHIFT) + +#define MOVE_WAITCOMP_SHIFT 24 +#define MOVE_WAITCOMP_MASK (1 << MOVE_WAITCOMP_SHIFT) +#define MOVE_WAITCOMP (1 << MOVE_WAITCOMP_SHIFT) + +#define MOVE_SRC_SHIFT 20 +#define MOVE_SRC_MASK (0x0f << MOVE_SRC_SHIFT) +#define MOVE_SRC_CLASS1CTX (0x00 << MOVE_SRC_SHIFT) +#define MOVE_SRC_CLASS2CTX (0x01 << MOVE_SRC_SHIFT) +#define MOVE_SRC_OUTFIFO (0x02 << MOVE_SRC_SHIFT) +#define MOVE_SRC_DESCBUF (0x03 << MOVE_SRC_SHIFT) +#define MOVE_SRC_MATH0 (0x04 << MOVE_SRC_SHIFT) +#define MOVE_SRC_MATH1 (0x05 << MOVE_SRC_SHIFT) +#define MOVE_SRC_MATH2 (0x06 << MOVE_SRC_SHIFT) +#define MOVE_SRC_MATH3 (0x07 << MOVE_SRC_SHIFT) +#define MOVE_SRC_INFIFO (0x08 << MOVE_SRC_SHIFT) +#define MOVE_SRC_INFIFO_CL (0x09 << MOVE_SRC_SHIFT) +#define MOVE_SRC_AUX_ABLK (0x0a << MOVE_SRC_SHIFT) + +#define MOVE_DEST_SHIFT 16 +#define MOVE_DEST_MASK (0x0f << MOVE_DEST_SHIFT) +#define MOVE_DEST_CLASS1CTX (0x00 << MOVE_DEST_SHIFT) +#define MOVE_DEST_CLASS2CTX (0x01 << MOVE_DEST_SHIFT) +#define MOVE_DEST_OUTFIFO (0x02 << MOVE_DEST_SHIFT) +#define MOVE_DEST_DESCBUF (0x03 << MOVE_DEST_SHIFT) +#define MOVE_DEST_MATH0 (0x04 << MOVE_DEST_SHIFT) +#define MOVE_DEST_MATH1 (0x05 << MOVE_DEST_SHIFT) +#define MOVE_DEST_MATH2 (0x06 << MOVE_DEST_SHIFT) +#define MOVE_DEST_MATH3 (0x07 << MOVE_DEST_SHIFT) +#define MOVE_DEST_CLASS1INFIFO (0x08 << MOVE_DEST_SHIFT) +#define MOVE_DEST_CLASS2INFIFO (0x09 << MOVE_DEST_SHIFT) +#define MOVE_DEST_INFIFO_NOINFO (0x0a << MOVE_DEST_SHIFT) +#define MOVE_DEST_PK_A (0x0c << MOVE_DEST_SHIFT) +#define MOVE_DEST_CLASS1KEY (0x0d << MOVE_DEST_SHIFT) +#define MOVE_DEST_CLASS2KEY (0x0e << MOVE_DEST_SHIFT) + +#define MOVE_OFFSET_SHIFT 8 +#define MOVE_OFFSET_MASK (0xff << MOVE_OFFSET_SHIFT) + +#define MOVE_LEN_SHIFT 0 +#define MOVE_LEN_MASK (0xff << MOVE_LEN_SHIFT) + +#define MOVELEN_MRSEL_SHIFT 0 +#define MOVELEN_MRSEL_MASK (0x3 << MOVE_LEN_SHIFT) +#define MOVELEN_MRSEL_MATH0 (0 << MOVELEN_MRSEL_SHIFT) +#define MOVELEN_MRSEL_MATH1 (1 << MOVELEN_MRSEL_SHIFT) +#define MOVELEN_MRSEL_MATH2 (2 << MOVELEN_MRSEL_SHIFT) +#define MOVELEN_MRSEL_MATH3 (3 << MOVELEN_MRSEL_SHIFT) + +/* + * MATH Command Constructs + */ + +#define MATH_IFB_SHIFT 26 +#define MATH_IFB_MASK (1 << MATH_IFB_SHIFT) +#define MATH_IFB (1 << MATH_IFB_SHIFT) + +#define MATH_NFU_SHIFT 25 +#define MATH_NFU_MASK (1 << MATH_NFU_SHIFT) +#define MATH_NFU (1 << MATH_NFU_SHIFT) + +#define MATH_STL_SHIFT 24 +#define MATH_STL_MASK (1 << MATH_STL_SHIFT) +#define MATH_STL (1 << MATH_STL_SHIFT) + +/* Function selectors */ +#define MATH_FUN_SHIFT 20 +#define MATH_FUN_MASK (0x0f << MATH_FUN_SHIFT) +#define MATH_FUN_ADD (0x00 << MATH_FUN_SHIFT) +#define MATH_FUN_ADDC (0x01 << MATH_FUN_SHIFT) +#define MATH_FUN_SUB (0x02 << MATH_FUN_SHIFT) +#define MATH_FUN_SUBB (0x03 << MATH_FUN_SHIFT) +#define MATH_FUN_OR (0x04 << MATH_FUN_SHIFT) +#define MATH_FUN_AND (0x05 << MATH_FUN_SHIFT) +#define MATH_FUN_XOR (0x06 << MATH_FUN_SHIFT) +#define MATH_FUN_LSHIFT (0x07 << MATH_FUN_SHIFT) +#define MATH_FUN_RSHIFT (0x08 << MATH_FUN_SHIFT) +#define MATH_FUN_SHLD (0x09 << MATH_FUN_SHIFT) +#define MATH_FUN_ZBYT (0x0a << MATH_FUN_SHIFT) + +/* Source 0 selectors */ +#define MATH_SRC0_SHIFT 16 +#define MATH_SRC0_MASK (0x0f << MATH_SRC0_SHIFT) +#define MATH_SRC0_REG0 (0x00 << MATH_SRC0_SHIFT) +#define MATH_SRC0_REG1 (0x01 << MATH_SRC0_SHIFT) +#define MATH_SRC0_REG2 (0x02 << MATH_SRC0_SHIFT) +#define MATH_SRC0_REG3 (0x03 << MATH_SRC0_SHIFT) +#define MATH_SRC0_IMM (0x04 << MATH_SRC0_SHIFT) +#define MATH_SRC0_DPOVRD (0x07 << MATH_SRC0_SHIFT) +#define MATH_SRC0_SEQINLEN (0x08 << MATH_SRC0_SHIFT) +#define MATH_SRC0_SEQOUTLEN (0x09 << MATH_SRC0_SHIFT) +#define MATH_SRC0_VARSEQINLEN (0x0a << MATH_SRC0_SHIFT) +#define MATH_SRC0_VARSEQOUTLEN (0x0b << MATH_SRC0_SHIFT) +#define MATH_SRC0_ZERO (0x0c << MATH_SRC0_SHIFT) + +/* Source 1 selectors */ +#define MATH_SRC1_SHIFT 12 +#define MATH_SRC1_MASK (0x0f << MATH_SRC1_SHIFT) +#define MATH_SRC1_REG0 (0x00 << MATH_SRC1_SHIFT) +#define MATH_SRC1_REG1 (0x01 << MATH_SRC1_SHIFT) +#define MATH_SRC1_REG2 (0x02 << MATH_SRC1_SHIFT) +#define MATH_SRC1_REG3 (0x03 << MATH_SRC1_SHIFT) +#define MATH_SRC1_IMM (0x04 << MATH_SRC1_SHIFT) +#define MATH_SRC1_DPOVRD (0x07 << MATH_SRC1_SHIFT) +#define MATH_SRC1_INFIFO (0x0a << MATH_SRC1_SHIFT) +#define MATH_SRC1_OUTFIFO (0x0b << MATH_SRC1_SHIFT) +#define MATH_SRC1_ONE (0x0c << MATH_SRC1_SHIFT) + +/* Destination selectors */ +#define MATH_DEST_SHIFT 8 +#define MATH_DEST_MASK (0x0f << MATH_DEST_SHIFT) +#define MATH_DEST_REG0 (0x00 << MATH_DEST_SHIFT) +#define MATH_DEST_REG1 (0x01 << MATH_DEST_SHIFT) +#define MATH_DEST_REG2 (0x02 << MATH_DEST_SHIFT) +#define MATH_DEST_REG3 (0x03 << MATH_DEST_SHIFT) +#define MATH_DEST_DPOVRD (0x07 << MATH_DEST_SHIFT) +#define MATH_DEST_SEQINLEN (0x08 << MATH_DEST_SHIFT) +#define MATH_DEST_SEQOUTLEN (0x09 << MATH_DEST_SHIFT) +#define MATH_DEST_VARSEQINLEN (0x0a << MATH_DEST_SHIFT) +#define MATH_DEST_VARSEQOUTLEN (0x0b << MATH_DEST_SHIFT) +#define MATH_DEST_NONE (0x0f << MATH_DEST_SHIFT) + +/* Length selectors */ +#define MATH_LEN_SHIFT 0 +#define MATH_LEN_MASK (0x0f << MATH_LEN_SHIFT) +#define MATH_LEN_1BYTE 0x01 +#define MATH_LEN_2BYTE 0x02 +#define MATH_LEN_4BYTE 0x04 +#define MATH_LEN_8BYTE 0x08 + +/* + * JUMP Command Constructs + */ + +#define JUMP_CLASS_SHIFT 25 +#define JUMP_CLASS_MASK (3 << JUMP_CLASS_SHIFT) +#define JUMP_CLASS_NONE 0 +#define JUMP_CLASS_CLASS1 (1 << JUMP_CLASS_SHIFT) +#define JUMP_CLASS_CLASS2 (2 << JUMP_CLASS_SHIFT) +#define JUMP_CLASS_BOTH (3 << JUMP_CLASS_SHIFT) + +#define JUMP_JSL_SHIFT 24 +#define JUMP_JSL_MASK (1 << JUMP_JSL_SHIFT) +#define JUMP_JSL (1 << JUMP_JSL_SHIFT) + +#define JUMP_TYPE_SHIFT 22 +#define JUMP_TYPE_LOCAL (0x00 << JUMP_TYPE_SHIFT) +#define JUMP_TYPE_NONLOCAL (0x01 << JUMP_TYPE_SHIFT) +#define JUMP_TYPE_HALT (0x02 << JUMP_TYPE_SHIFT) +#define JUMP_TYPE_HALT_USER (0x03 << JUMP_TYPE_SHIFT) + +#define JUMP_TEST_SHIFT 16 +#define JUMP_TEST_MASK (0x03 << JUMP_TEST_SHIFT) +#define JUMP_TEST_ALL (0x00 << JUMP_TEST_SHIFT) +#define JUMP_TEST_INVALL (0x01 << JUMP_TEST_SHIFT) +#define JUMP_TEST_ANY (0x02 << JUMP_TEST_SHIFT) +#define JUMP_TEST_INVANY (0x03 << JUMP_TEST_SHIFT) + +/* Condition codes. JSL bit is factored in */ +#define JUMP_COND_SHIFT 8 +#define JUMP_COND_MASK (0x100ff << JUMP_COND_SHIFT) +#define JUMP_COND_PK_0 (0x80 << JUMP_COND_SHIFT) +#define JUMP_COND_PK_GCD_1 (0x40 << JUMP_COND_SHIFT) +#define JUMP_COND_PK_PRIME (0x20 << JUMP_COND_SHIFT) +#define JUMP_COND_MATH_N (0x08 << JUMP_COND_SHIFT) +#define JUMP_COND_MATH_Z (0x04 << JUMP_COND_SHIFT) +#define JUMP_COND_MATH_C (0x02 << JUMP_COND_SHIFT) +#define JUMP_COND_MATH_NV (0x01 << JUMP_COND_SHIFT) + +#define JUMP_COND_JRP ((0x80 << JUMP_COND_SHIFT) | JUMP_JSL) +#define JUMP_COND_SHRD ((0x40 << JUMP_COND_SHIFT) | JUMP_JSL) +#define JUMP_COND_SELF ((0x20 << JUMP_COND_SHIFT) | JUMP_JSL) +#define JUMP_COND_CALM ((0x10 << JUMP_COND_SHIFT) | JUMP_JSL) +#define JUMP_COND_NIP ((0x08 << JUMP_COND_SHIFT) | JUMP_JSL) +#define JUMP_COND_NIFP ((0x04 << JUMP_COND_SHIFT) | JUMP_JSL) +#define JUMP_COND_NOP ((0x02 << JUMP_COND_SHIFT) | JUMP_JSL) +#define JUMP_COND_NCP ((0x01 << JUMP_COND_SHIFT) | JUMP_JSL) + +#define JUMP_OFFSET_SHIFT 0 +#define JUMP_OFFSET_MASK (0xff << JUMP_OFFSET_SHIFT) + +/* + * NFIFO ENTRY + * Data Constructs + * + */ +#define NFIFOENTRY_DEST_SHIFT 30 +#define NFIFOENTRY_DEST_MASK (3 << NFIFOENTRY_DEST_SHIFT) +#define NFIFOENTRY_DEST_DECO (0 << NFIFOENTRY_DEST_SHIFT) +#define NFIFOENTRY_DEST_CLASS1 (1 << NFIFOENTRY_DEST_SHIFT) +#define NFIFOENTRY_DEST_CLASS2 (2 << NFIFOENTRY_DEST_SHIFT) +#define NFIFOENTRY_DEST_BOTH (3 << NFIFOENTRY_DEST_SHIFT) + +#define NFIFOENTRY_LC2_SHIFT 29 +#define NFIFOENTRY_LC2_MASK (1 << NFIFOENTRY_LC2_SHIFT) +#define NFIFOENTRY_LC2 (1 << NFIFOENTRY_LC2_SHIFT) + +#define NFIFOENTRY_LC1_SHIFT 28 +#define NFIFOENTRY_LC1_MASK (1 << NFIFOENTRY_LC1_SHIFT) +#define NFIFOENTRY_LC1 (1 << NFIFOENTRY_LC1_SHIFT) + +#define NFIFOENTRY_FC2_SHIFT 27 +#define NFIFOENTRY_FC2_MASK (1 << NFIFOENTRY_FC2_SHIFT) +#define NFIFOENTRY_FC2 (1 << NFIFOENTRY_FC2_SHIFT) + +#define NFIFOENTRY_FC1_SHIFT 26 +#define NFIFOENTRY_FC1_MASK (1 << NFIFOENTRY_FC1_SHIFT) +#define NFIFOENTRY_FC1 (1 << NFIFOENTRY_FC1_SHIFT) + +#define NFIFOENTRY_STYPE_SHIFT 24 +#define NFIFOENTRY_STYPE_MASK (3 << NFIFOENTRY_STYPE_SHIFT) +#define NFIFOENTRY_STYPE_DFIFO (0 << NFIFOENTRY_STYPE_SHIFT) +#define NFIFOENTRY_STYPE_OFIFO (1 << NFIFOENTRY_STYPE_SHIFT) +#define NFIFOENTRY_STYPE_PAD (2 << NFIFOENTRY_STYPE_SHIFT) +#define NFIFOENTRY_STYPE_SNOOP (3 << NFIFOENTRY_STYPE_SHIFT) + +#define NFIFOENTRY_DTYPE_SHIFT 20 +#define NFIFOENTRY_DTYPE_MASK (0xF << NFIFOENTRY_DTYPE_SHIFT) + +#define NFIFOENTRY_DTYPE_SBOX (0x0 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_AAD (0x1 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_IV (0x2 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_SAD (0x3 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_ICV (0xA << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_POLY (0xB << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_SKIP (0xE << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_MSG (0xF << NFIFOENTRY_DTYPE_SHIFT) + +#define NFIFOENTRY_DTYPE_PK_A0 (0x0 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_PK_A1 (0x1 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_PK_A2 (0x2 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_PK_A3 (0x3 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_PK_B0 (0x4 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_PK_B1 (0x5 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_PK_B2 (0x6 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_PK_B3 (0x7 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_PK_N (0x8 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_PK_E (0x9 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_PK_A (0xC << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_PK_B (0xD << NFIFOENTRY_DTYPE_SHIFT) + + +#define NFIFOENTRY_BND_SHIFT 19 +#define NFIFOENTRY_BND_MASK (1 << NFIFOENTRY_BND_SHIFT) +#define NFIFOENTRY_BND (1 << NFIFOENTRY_BND_SHIFT) + +#define NFIFOENTRY_PTYPE_SHIFT 16 +#define NFIFOENTRY_PTYPE_MASK (0x7 << NFIFOENTRY_PTYPE_SHIFT) + +#define NFIFOENTRY_PTYPE_ZEROS (0x0 << NFIFOENTRY_PTYPE_SHIFT) +#define NFIFOENTRY_PTYPE_RND_NOZEROS (0x1 << NFIFOENTRY_PTYPE_SHIFT) +#define NFIFOENTRY_PTYPE_INCREMENT (0x2 << NFIFOENTRY_PTYPE_SHIFT) +#define NFIFOENTRY_PTYPE_RND (0x3 << NFIFOENTRY_PTYPE_SHIFT) +#define NFIFOENTRY_PTYPE_ZEROS_NZ (0x4 << NFIFOENTRY_PTYPE_SHIFT) +#define NFIFOENTRY_PTYPE_RND_NZ_LZ (0x5 << NFIFOENTRY_PTYPE_SHIFT) +#define NFIFOENTRY_PTYPE_N (0x6 << NFIFOENTRY_PTYPE_SHIFT) +#define NFIFOENTRY_PTYPE_RND_NZ_N (0x7 << NFIFOENTRY_PTYPE_SHIFT) + +#define NFIFOENTRY_OC_SHIFT 15 +#define NFIFOENTRY_OC_MASK (1 << NFIFOENTRY_OC_SHIFT) +#define NFIFOENTRY_OC (1 << NFIFOENTRY_OC_SHIFT) + +#define NFIFOENTRY_AST_SHIFT 14 +#define NFIFOENTRY_AST_MASK (1 << NFIFOENTRY_OC_SHIFT) +#define NFIFOENTRY_AST (1 << NFIFOENTRY_OC_SHIFT) + +#define NFIFOENTRY_BM_SHIFT 11 +#define NFIFOENTRY_BM_MASK (1 << NFIFOENTRY_BM_SHIFT) +#define NFIFOENTRY_BM (1 << NFIFOENTRY_BM_SHIFT) + +#define NFIFOENTRY_PS_SHIFT 10 +#define NFIFOENTRY_PS_MASK (1 << NFIFOENTRY_PS_SHIFT) +#define NFIFOENTRY_PS (1 << NFIFOENTRY_PS_SHIFT) + +#define NFIFOENTRY_DLEN_SHIFT 0 +#define NFIFOENTRY_DLEN_MASK (0xFFF << NFIFOENTRY_DLEN_SHIFT) + +#define NFIFOENTRY_PLEN_SHIFT 0 +#define NFIFOENTRY_PLEN_MASK (0xFF << NFIFOENTRY_PLEN_SHIFT) + +/* Append Load Immediate Command */ +#define FD_CMD_APPEND_LOAD_IMMEDIATE 0x80000000 + +/* Set SEQ LIODN equal to the Non-SEQ LIODN for the job */ +#define FD_CMD_SET_SEQ_LIODN_EQUAL_NONSEQ_LIODN 0x40000000 + +/* Frame Descriptor Command for Replacement Job Descriptor */ +#define FD_CMD_REPLACE_JOB_DESC 0x20000000 + +#endif /* DESC_H */ diff --git a/drivers/crypto/caam/desc_constr.h b/drivers/crypto/caam/desc_constr.h new file mode 100644 index 000000000..2a29dd2c9 --- /dev/null +++ b/drivers/crypto/caam/desc_constr.h @@ -0,0 +1,612 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * caam descriptor construction helper functions + * + * Copyright 2008-2012 Freescale Semiconductor, Inc. + * Copyright 2019, 2025 NXP + */ + +#ifndef DESC_CONSTR_H +#define DESC_CONSTR_H + +#include "desc.h" +#include "regs.h" + +#define IMMEDIATE (1 << 23) +#define CAAM_CMD_SZ sizeof(u32) +#define CAAM_PTR_SZ caam_ptr_sz +#define CAAM_PTR_SZ_MAX sizeof(dma_addr_t) +#define CAAM_PTR_SZ_MIN sizeof(u32) +#define CAAM_DESC_BYTES_MAX (CAAM_CMD_SZ * MAX_CAAM_DESCSIZE) +#define __DESC_JOB_IO_LEN(n) (CAAM_CMD_SZ * 5 + (n) * 3) +#define DESC_JOB_IO_LEN __DESC_JOB_IO_LEN(CAAM_PTR_SZ) +#define DESC_JOB_IO_LEN_MAX __DESC_JOB_IO_LEN(CAAM_PTR_SZ_MAX) +#define DESC_JOB_IO_LEN_MIN __DESC_JOB_IO_LEN(CAAM_PTR_SZ_MIN) + +/* + * The CAAM QI hardware constructs a job descriptor which points + * to shared descriptor (as pointed by context_a of FQ to CAAM). + * When the job descriptor is executed by deco, the whole job + * descriptor together with shared descriptor gets loaded in + * deco buffer which is 64 words long (each 32-bit). + * + * The job descriptor constructed by QI hardware has layout: + * + * HEADER (1 word) + * Shdesc ptr (1 or 2 words) + * SEQ_OUT_PTR (1 word) + * Out ptr (1 or 2 words) + * Out length (1 word) + * SEQ_IN_PTR (1 word) + * In ptr (1 or 2 words) + * In length (1 word) + * + * The shdesc ptr is used to fetch shared descriptor contents + * into deco buffer. + * + * Apart from shdesc contents, the total number of words that + * get loaded in deco buffer are '8' or '11'. The remaining words + * in deco buffer can be used for storing shared descriptor. + */ +#define MAX_SDLEN ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN_MIN) / CAAM_CMD_SZ) + +#ifdef DEBUG +#define PRINT_POS do { printk(KERN_DEBUG "%02d: %s\n", desc_len(desc),\ + &__func__[sizeof("append")]); } while (0) +#else +#define PRINT_POS +#endif + +#define SET_OK_NO_PROP_ERRORS (IMMEDIATE | LDST_CLASS_DECO | \ + LDST_SRCDST_WORD_DECOCTRL | \ + (LDOFF_CHG_SHARE_OK_NO_PROP << \ + LDST_OFFSET_SHIFT)) +#define DISABLE_AUTO_INFO_FIFO (IMMEDIATE | LDST_CLASS_DECO | \ + LDST_SRCDST_WORD_DECOCTRL | \ + (LDOFF_DISABLE_AUTO_NFIFO << LDST_OFFSET_SHIFT)) +#define ENABLE_AUTO_INFO_FIFO (IMMEDIATE | LDST_CLASS_DECO | \ + LDST_SRCDST_WORD_DECOCTRL | \ + (LDOFF_ENABLE_AUTO_NFIFO << LDST_OFFSET_SHIFT)) + +extern bool caam_little_end; +extern size_t caam_ptr_sz; + +/* + * HW fetches 4 S/G table entries at a time, irrespective of how many entries + * are in the table. It's SW's responsibility to make sure these accesses + * do not have side effects. + */ +static inline int pad_sg_nents(int sg_nents) +{ + return ALIGN(sg_nents, 4); +} + +static inline int desc_len(u32 * const desc) +{ + return caam32_to_cpu(*desc) & HDR_DESCLEN_MASK; +} + +static inline int desc_bytes(void * const desc) +{ + return desc_len(desc) * CAAM_CMD_SZ; +} + +static inline u32 *desc_end(u32 * const desc) +{ + return desc + desc_len(desc); +} + +static inline void *sh_desc_pdb(u32 * const desc) +{ + return desc + 1; +} + +static inline void init_desc(u32 * const desc, u32 options) +{ + *desc = cpu_to_caam32((options | HDR_ONE) + 1); +} + +static inline void init_sh_desc(u32 * const desc, u32 options) +{ + PRINT_POS; + init_desc(desc, CMD_SHARED_DESC_HDR | options); +} + +static inline void init_sh_desc_pdb(u32 * const desc, u32 options, + size_t pdb_bytes) +{ + u32 pdb_len = (pdb_bytes + CAAM_CMD_SZ - 1) / CAAM_CMD_SZ; + + init_sh_desc(desc, (((pdb_len + 1) << HDR_START_IDX_SHIFT) + pdb_len) | + options); +} + +static inline void init_job_desc(u32 * const desc, u32 options) +{ + init_desc(desc, CMD_DESC_HDR | options); +} + +static inline void init_job_desc_pdb(u32 * const desc, u32 options, + size_t pdb_bytes) +{ + u32 pdb_len = (pdb_bytes + CAAM_CMD_SZ - 1) / CAAM_CMD_SZ; + + init_job_desc(desc, (((pdb_len + 1) << HDR_START_IDX_SHIFT)) | options); +} + +static inline void append_ptr(u32 * const desc, dma_addr_t ptr) +{ + if (caam_ptr_sz == sizeof(dma_addr_t)) { + dma_addr_t *offset = (dma_addr_t *)desc_end(desc); + + *offset = cpu_to_caam_dma(ptr); + } else { + u32 *offset = (u32 *)desc_end(desc); + + *offset = cpu_to_caam_dma(ptr); + } + + (*desc) = cpu_to_caam32(caam32_to_cpu(*desc) + + CAAM_PTR_SZ / CAAM_CMD_SZ); +} + +static inline void init_job_desc_shared(u32 * const desc, dma_addr_t ptr, + int len, u32 options) +{ + PRINT_POS; + init_job_desc(desc, HDR_SHARED | options | + (len << HDR_START_IDX_SHIFT)); + append_ptr(desc, ptr); +} + +static inline void append_data(u32 * const desc, const void *data, int len) +{ + u32 *offset = desc_end(desc); + + /* Avoid gcc warning: memcpy with data == NULL */ + if (!IS_ENABLED(CONFIG_CRYPTO_DEV_FSL_CAAM_DEBUG) || data) + memcpy(offset, data, len); + + (*desc) = cpu_to_caam32(caam32_to_cpu(*desc) + + (len + CAAM_CMD_SZ - 1) / CAAM_CMD_SZ); +} + +static inline void append_cmd(u32 * const desc, u32 command) +{ + u32 *cmd = desc_end(desc); + + *cmd = cpu_to_caam32(command); + + (*desc) = cpu_to_caam32(caam32_to_cpu(*desc) + 1); +} + +#define append_u32 append_cmd + +static inline void append_u64(u32 * const desc, u64 data) +{ + u32 *offset = desc_end(desc); + + /* Only 32-bit alignment is guaranteed in descriptor buffer */ + if (caam_little_end) { + *offset = cpu_to_caam32(lower_32_bits(data)); + *(++offset) = cpu_to_caam32(upper_32_bits(data)); + } else { + *offset = cpu_to_caam32(upper_32_bits(data)); + *(++offset) = cpu_to_caam32(lower_32_bits(data)); + } + + (*desc) = cpu_to_caam32(caam32_to_cpu(*desc) + 2); +} + +/* Write command without affecting header, and return pointer to next word */ +static inline u32 *write_cmd(u32 * const desc, u32 command) +{ + *desc = cpu_to_caam32(command); + + return desc + 1; +} + +static inline void append_cmd_ptr(u32 * const desc, dma_addr_t ptr, int len, + u32 command) +{ + append_cmd(desc, command | len); + append_ptr(desc, ptr); +} + +/* Write length after pointer, rather than inside command */ +static inline void append_cmd_ptr_extlen(u32 * const desc, dma_addr_t ptr, + unsigned int len, u32 command) +{ + append_cmd(desc, command); + if (!(command & (SQIN_RTO | SQIN_PRE))) + append_ptr(desc, ptr); + append_cmd(desc, len); +} + +static inline void append_cmd_data(u32 * const desc, const void *data, int len, + u32 command) +{ + append_cmd(desc, command | IMMEDIATE | len); + append_data(desc, data, len); +} + +#define APPEND_CMD_RET(cmd, op) \ +static inline u32 *append_##cmd(u32 * const desc, u32 options) \ +{ \ + u32 *cmd = desc_end(desc); \ + PRINT_POS; \ + append_cmd(desc, CMD_##op | options); \ + return cmd; \ +} +APPEND_CMD_RET(jump, JUMP) +APPEND_CMD_RET(move, MOVE) +APPEND_CMD_RET(move_len, MOVE_LEN) + +static inline void set_jump_tgt_here(u32 * const desc, u32 *jump_cmd) +{ + *jump_cmd = cpu_to_caam32(caam32_to_cpu(*jump_cmd) | + (desc_len(desc) - (jump_cmd - desc))); +} + +static inline void set_move_tgt_here(u32 * const desc, u32 *move_cmd) +{ + u32 val = caam32_to_cpu(*move_cmd); + + val &= ~MOVE_OFFSET_MASK; + val |= (desc_len(desc) << (MOVE_OFFSET_SHIFT + 2)) & MOVE_OFFSET_MASK; + *move_cmd = cpu_to_caam32(val); +} + +#define APPEND_CMD(cmd, op) \ +static inline void append_##cmd(u32 * const desc, u32 options) \ +{ \ + PRINT_POS; \ + append_cmd(desc, CMD_##op | options); \ +} +APPEND_CMD(operation, OPERATION) + +#define APPEND_CMD_LEN(cmd, op) \ +static inline void append_##cmd(u32 * const desc, unsigned int len, \ + u32 options) \ +{ \ + PRINT_POS; \ + append_cmd(desc, CMD_##op | len | options); \ +} + +APPEND_CMD_LEN(seq_load, SEQ_LOAD) +APPEND_CMD_LEN(seq_store, SEQ_STORE) +APPEND_CMD_LEN(seq_fifo_load, SEQ_FIFO_LOAD) +APPEND_CMD_LEN(seq_fifo_store, SEQ_FIFO_STORE) + +#define APPEND_CMD_PTR(cmd, op) \ +static inline void append_##cmd(u32 * const desc, dma_addr_t ptr, \ + unsigned int len, u32 options) \ +{ \ + PRINT_POS; \ + append_cmd_ptr(desc, ptr, len, CMD_##op | options); \ +} +APPEND_CMD_PTR(key, KEY) +APPEND_CMD_PTR(load, LOAD) +APPEND_CMD_PTR(fifo_load, FIFO_LOAD) +APPEND_CMD_PTR(fifo_store, FIFO_STORE) + +static inline void append_store(u32 * const desc, dma_addr_t ptr, + unsigned int len, u32 options) +{ + u32 cmd_src; + + cmd_src = options & LDST_SRCDST_MASK; + + append_cmd(desc, CMD_STORE | options | len); + + /* The following options do not require pointer */ + if (!(cmd_src == LDST_SRCDST_WORD_DESCBUF_SHARED || + cmd_src == LDST_SRCDST_WORD_DESCBUF_JOB || + cmd_src == LDST_SRCDST_WORD_DESCBUF_JOB_WE || + cmd_src == LDST_SRCDST_WORD_DESCBUF_SHARED_WE)) + append_ptr(desc, ptr); +} + +#define APPEND_SEQ_PTR_INTLEN(cmd, op) \ +static inline void append_seq_##cmd##_ptr_intlen(u32 * const desc, \ + dma_addr_t ptr, \ + unsigned int len, \ + u32 options) \ +{ \ + PRINT_POS; \ + if (options & (SQIN_RTO | SQIN_PRE)) \ + append_cmd(desc, CMD_SEQ_##op##_PTR | len | options); \ + else \ + append_cmd_ptr(desc, ptr, len, CMD_SEQ_##op##_PTR | options); \ +} +APPEND_SEQ_PTR_INTLEN(in, IN) +APPEND_SEQ_PTR_INTLEN(out, OUT) + +#define APPEND_CMD_PTR_TO_IMM(cmd, op) \ +static inline void append_##cmd##_as_imm(u32 * const desc, const void *data, \ + unsigned int len, u32 options) \ +{ \ + PRINT_POS; \ + append_cmd_data(desc, data, len, CMD_##op | options); \ +} +APPEND_CMD_PTR_TO_IMM(load, LOAD); +APPEND_CMD_PTR_TO_IMM(fifo_load, FIFO_LOAD); + +#define APPEND_CMD_PTR_EXTLEN(cmd, op) \ +static inline void append_##cmd##_extlen(u32 * const desc, dma_addr_t ptr, \ + unsigned int len, u32 options) \ +{ \ + PRINT_POS; \ + append_cmd_ptr_extlen(desc, ptr, len, CMD_##op | SQIN_EXT | options); \ +} +APPEND_CMD_PTR_EXTLEN(seq_in_ptr, SEQ_IN_PTR) +APPEND_CMD_PTR_EXTLEN(seq_out_ptr, SEQ_OUT_PTR) + +/* + * Determine whether to store length internally or externally depending on + * the size of its type + */ +#define APPEND_CMD_PTR_LEN(cmd, op, type) \ +static inline void append_##cmd(u32 * const desc, dma_addr_t ptr, \ + type len, u32 options) \ +{ \ + PRINT_POS; \ + if (sizeof(type) > sizeof(u16)) \ + append_##cmd##_extlen(desc, ptr, len, options); \ + else \ + append_##cmd##_intlen(desc, ptr, len, options); \ +} +APPEND_CMD_PTR_LEN(seq_in_ptr, SEQ_IN_PTR, u32) +APPEND_CMD_PTR_LEN(seq_out_ptr, SEQ_OUT_PTR, u32) + +/* + * 2nd variant for commands whose specified immediate length differs + * from length of immediate data provided, e.g., split keys + */ +#define APPEND_CMD_PTR_TO_IMM2(cmd, op) \ +static inline void append_##cmd##_as_imm(u32 * const desc, const void *data, \ + unsigned int data_len, \ + unsigned int len, u32 options) \ +{ \ + PRINT_POS; \ + append_cmd(desc, CMD_##op | IMMEDIATE | len | options); \ + append_data(desc, data, data_len); \ +} +APPEND_CMD_PTR_TO_IMM2(key, KEY); + +#define APPEND_CMD_RAW_IMM(cmd, op, type) \ +static inline void append_##cmd##_imm_##type(u32 * const desc, type immediate, \ + u32 options) \ +{ \ + PRINT_POS; \ + if (options & LDST_LEN_MASK) \ + append_cmd(desc, CMD_##op | IMMEDIATE | options); \ + else \ + append_cmd(desc, CMD_##op | IMMEDIATE | options | \ + sizeof(type)); \ + append_cmd(desc, immediate); \ +} +APPEND_CMD_RAW_IMM(load, LOAD, u32); + +/* + * ee - endianness + * size - size of immediate type in bytes + */ +#define APPEND_CMD_RAW_IMM2(cmd, op, ee, size) \ +static inline void append_##cmd##_imm_##ee##size(u32 *desc, \ + u##size immediate, \ + u32 options) \ +{ \ + __##ee##size data = cpu_to_##ee##size(immediate); \ + PRINT_POS; \ + append_cmd(desc, CMD_##op | IMMEDIATE | options | sizeof(data)); \ + append_data(desc, &data, sizeof(data)); \ +} + +APPEND_CMD_RAW_IMM2(load, LOAD, be, 32); + +/* + * Append math command. Only the last part of destination and source need to + * be specified + */ +#define APPEND_MATH(op, desc, dest, src_0, src_1, len) \ +append_cmd(desc, CMD_MATH | MATH_FUN_##op | MATH_DEST_##dest | \ + MATH_SRC0_##src_0 | MATH_SRC1_##src_1 | (u32)len); + +#define append_math_add(desc, dest, src0, src1, len) \ + APPEND_MATH(ADD, desc, dest, src0, src1, len) +#define append_math_sub(desc, dest, src0, src1, len) \ + APPEND_MATH(SUB, desc, dest, src0, src1, len) +#define append_math_add_c(desc, dest, src0, src1, len) \ + APPEND_MATH(ADDC, desc, dest, src0, src1, len) +#define append_math_sub_b(desc, dest, src0, src1, len) \ + APPEND_MATH(SUBB, desc, dest, src0, src1, len) +#define append_math_and(desc, dest, src0, src1, len) \ + APPEND_MATH(AND, desc, dest, src0, src1, len) +#define append_math_or(desc, dest, src0, src1, len) \ + APPEND_MATH(OR, desc, dest, src0, src1, len) +#define append_math_xor(desc, dest, src0, src1, len) \ + APPEND_MATH(XOR, desc, dest, src0, src1, len) +#define append_math_lshift(desc, dest, src0, src1, len) \ + APPEND_MATH(LSHIFT, desc, dest, src0, src1, len) +#define append_math_rshift(desc, dest, src0, src1, len) \ + APPEND_MATH(RSHIFT, desc, dest, src0, src1, len) +#define append_math_ldshift(desc, dest, src0, src1, len) \ + APPEND_MATH(SHLD, desc, dest, src0, src1, len) + +/* Exactly one source is IMM. Data is passed in as u32 value */ +#define APPEND_MATH_IMM_u32(op, desc, dest, src_0, src_1, data) \ +do { \ + APPEND_MATH(op, desc, dest, src_0, src_1, CAAM_CMD_SZ); \ + append_cmd(desc, data); \ +} while (0) + +#define append_math_add_imm_u32(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u32(ADD, desc, dest, src0, src1, data) +#define append_math_sub_imm_u32(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u32(SUB, desc, dest, src0, src1, data) +#define append_math_add_c_imm_u32(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u32(ADDC, desc, dest, src0, src1, data) +#define append_math_sub_b_imm_u32(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u32(SUBB, desc, dest, src0, src1, data) +#define append_math_and_imm_u32(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u32(AND, desc, dest, src0, src1, data) +#define append_math_or_imm_u32(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u32(OR, desc, dest, src0, src1, data) +#define append_math_xor_imm_u32(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u32(XOR, desc, dest, src0, src1, data) +#define append_math_lshift_imm_u32(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u32(LSHIFT, desc, dest, src0, src1, data) +#define append_math_rshift_imm_u32(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u32(RSHIFT, desc, dest, src0, src1, data) + +/* Exactly one source is IMM. Data is passed in as u64 value */ +#define APPEND_MATH_IMM_u64(op, desc, dest, src_0, src_1, data) \ +do { \ + u32 upper = (data >> 16) >> 16; \ + APPEND_MATH(op, desc, dest, src_0, src_1, CAAM_CMD_SZ * 2 | \ + (upper ? 0 : MATH_IFB)); \ + if (upper) \ + append_u64(desc, data); \ + else \ + append_u32(desc, lower_32_bits(data)); \ +} while (0) + +#define append_math_add_imm_u64(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u64(ADD, desc, dest, src0, src1, data) +#define append_math_sub_imm_u64(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u64(SUB, desc, dest, src0, src1, data) +#define append_math_add_c_imm_u64(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u64(ADDC, desc, dest, src0, src1, data) +#define append_math_sub_b_imm_u64(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u64(SUBB, desc, dest, src0, src1, data) +#define append_math_and_imm_u64(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u64(AND, desc, dest, src0, src1, data) +#define append_math_or_imm_u64(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u64(OR, desc, dest, src0, src1, data) +#define append_math_xor_imm_u64(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u64(XOR, desc, dest, src0, src1, data) +#define append_math_lshift_imm_u64(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u64(LSHIFT, desc, dest, src0, src1, data) +#define append_math_rshift_imm_u64(desc, dest, src0, src1, data) \ + APPEND_MATH_IMM_u64(RSHIFT, desc, dest, src0, src1, data) + +/** + * struct alginfo - Container for algorithm details + * @algtype: algorithm selector; for valid values, see documentation of the + * functions where it is used. + * @keylen: length of the provided algorithm key, in bytes + * @keylen_pad: padded length of the provided algorithm key, in bytes + * @key_dma: dma (bus) address where algorithm key resides + * @protected_key_dma: dma (bus) address where protected key resides + * @key_virt: virtual address where algorithm key resides + * @key_inline: true - key can be inlined in the descriptor; false - key is + * referenced by the descriptor + * @plain_keylen: size of the key to be loaded by the CAAM + * @key_cmd_opt: optional parameters for KEY command + */ +struct alginfo { + u32 algtype; + unsigned int keylen; + unsigned int keylen_pad; + dma_addr_t key_dma; + dma_addr_t protected_key_dma; + const void *key_virt; + bool key_inline; + u32 plain_keylen; + u32 key_cmd_opt; +}; + +/** + * desc_inline_query() - Provide indications on which data items can be inlined + * and which shall be referenced in a shared descriptor. + * @sd_base_len: Shared descriptor base length - bytes consumed by the commands, + * excluding the data items to be inlined (or corresponding + * pointer if an item is not inlined). Each cnstr_* function that + * generates descriptors should have a define mentioning + * corresponding length. + * @jd_len: Maximum length of the job descriptor(s) that will be used + * together with the shared descriptor. + * @data_len: Array of lengths of the data items trying to be inlined + * @inl_mask: 32bit mask with bit x = 1 if data item x can be inlined, 0 + * otherwise. + * @count: Number of data items (size of @data_len array); must be <= 32 + * + * Return: 0 if data can be inlined / referenced, negative value if not. If 0, + * check @inl_mask for details. + */ +static inline int desc_inline_query(unsigned int sd_base_len, + unsigned int jd_len, unsigned int *data_len, + u32 *inl_mask, unsigned int count) +{ + int rem_bytes = (int)(CAAM_DESC_BYTES_MAX - sd_base_len - jd_len); + unsigned int i; + + *inl_mask = 0; + for (i = 0; (i < count) && (rem_bytes > 0); i++) { + if (rem_bytes - (int)(data_len[i] + + (count - i - 1) * CAAM_PTR_SZ) >= 0) { + rem_bytes -= data_len[i]; + *inl_mask |= (1 << i); + } else { + rem_bytes -= CAAM_PTR_SZ; + } + } + + return (rem_bytes >= 0) ? 0 : -1; +} + +/** + * append_proto_dkp - Derived Key Protocol (DKP): key -> split key + * @desc: pointer to buffer used for descriptor construction + * @adata: pointer to authentication transform definitions. + * keylen should be the length of initial key, while keylen_pad + * the length of the derived (split) key. + * Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, SHA224, + * SHA256, SHA384, SHA512}. + */ +static inline void append_proto_dkp(u32 * const desc, struct alginfo *adata) +{ + u32 protid; + + /* + * Quick & dirty translation from OP_ALG_ALGSEL_{MD5, SHA*} + * to OP_PCLID_DKP_{MD5, SHA*} + */ + protid = (adata->algtype & OP_ALG_ALGSEL_SUBMASK) | + (0x20 << OP_ALG_ALGSEL_SHIFT); + + if (adata->key_inline) { + int words; + + if (adata->keylen > adata->keylen_pad) { + append_operation(desc, OP_TYPE_UNI_PROTOCOL | protid | + OP_PCL_DKP_SRC_PTR | + OP_PCL_DKP_DST_IMM | adata->keylen); + append_ptr(desc, adata->key_dma); + + words = (ALIGN(adata->keylen_pad, CAAM_CMD_SZ) - + CAAM_PTR_SZ) / CAAM_CMD_SZ; + } else { + append_operation(desc, OP_TYPE_UNI_PROTOCOL | protid | + OP_PCL_DKP_SRC_IMM | + OP_PCL_DKP_DST_IMM | adata->keylen); + append_data(desc, adata->key_virt, adata->keylen); + + words = (ALIGN(adata->keylen_pad, CAAM_CMD_SZ) - + ALIGN(adata->keylen, CAAM_CMD_SZ)) / + CAAM_CMD_SZ; + } + + /* Reserve space in descriptor buffer for the derived key */ + if (words) + (*desc) = cpu_to_caam32(caam32_to_cpu(*desc) + words); + } else { + append_operation(desc, OP_TYPE_UNI_PROTOCOL | protid | + OP_PCL_DKP_SRC_PTR | OP_PCL_DKP_DST_PTR | + adata->keylen); + append_ptr(desc, adata->key_dma); + } +} + +#endif /* DESC_CONSTR_H */ diff --git a/drivers/crypto/caam/dpseci-debugfs.c b/drivers/crypto/caam/dpseci-debugfs.c new file mode 100644 index 000000000..020a9d8a8 --- /dev/null +++ b/drivers/crypto/caam/dpseci-debugfs.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* Copyright 2019 NXP */ + +#include +#include +#include +#include "dpseci-debugfs.h" + +static int dpseci_dbg_fqs_show(struct seq_file *file, void *offset) +{ + struct dpaa2_caam_priv *priv = file->private; + u32 fqid, fcnt, bcnt; + int i, err; + + seq_printf(file, "FQ stats for %s:\n", dev_name(priv->dev)); + seq_printf(file, "%s%16s%16s\n", + "Rx-VFQID", + "Pending frames", + "Pending bytes"); + + for (i = 0; i < priv->num_pairs; i++) { + fqid = priv->rx_queue_attr[i].fqid; + err = dpaa2_io_query_fq_count(NULL, fqid, &fcnt, &bcnt); + if (err) + continue; + + seq_printf(file, "%5d%16u%16u\n", fqid, fcnt, bcnt); + } + + seq_printf(file, "%s%16s%16s\n", + "Tx-VFQID", + "Pending frames", + "Pending bytes"); + + for (i = 0; i < priv->num_pairs; i++) { + fqid = priv->tx_queue_attr[i].fqid; + err = dpaa2_io_query_fq_count(NULL, fqid, &fcnt, &bcnt); + if (err) + continue; + + seq_printf(file, "%5d%16u%16u\n", fqid, fcnt, bcnt); + } + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(dpseci_dbg_fqs); + +void dpaa2_dpseci_debugfs_init(struct dpaa2_caam_priv *priv) +{ + priv->dfs_root = debugfs_create_dir(dev_name(priv->dev), NULL); + + debugfs_create_file("fq_stats", 0444, priv->dfs_root, priv, + &dpseci_dbg_fqs_fops); +} + +void dpaa2_dpseci_debugfs_exit(struct dpaa2_caam_priv *priv) +{ + debugfs_remove_recursive(priv->dfs_root); +} diff --git a/drivers/crypto/caam/dpseci-debugfs.h b/drivers/crypto/caam/dpseci-debugfs.h new file mode 100644 index 000000000..bc22af7be --- /dev/null +++ b/drivers/crypto/caam/dpseci-debugfs.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */ +/* Copyright 2019 NXP */ + +#ifndef DPSECI_DEBUGFS_H +#define DPSECI_DEBUGFS_H + +#include +#include "caamalg_qi2.h" + +#ifdef CONFIG_DEBUG_FS +void dpaa2_dpseci_debugfs_init(struct dpaa2_caam_priv *priv); +void dpaa2_dpseci_debugfs_exit(struct dpaa2_caam_priv *priv); +#else +static inline void dpaa2_dpseci_debugfs_init(struct dpaa2_caam_priv *priv) {} +static inline void dpaa2_dpseci_debugfs_exit(struct dpaa2_caam_priv *priv) {} +#endif /* CONFIG_DEBUG_FS */ + +#endif /* DPSECI_DEBUGFS_H */ diff --git a/drivers/crypto/caam/dpseci.c b/drivers/crypto/caam/dpseci.c new file mode 100644 index 000000000..039df6c57 --- /dev/null +++ b/drivers/crypto/caam/dpseci.c @@ -0,0 +1,444 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) +/* + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2017-2018 NXP + */ + +#include +#include "dpseci.h" +#include "dpseci_cmd.h" + +/** + * dpseci_open() - Open a control session for the specified object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @dpseci_id: DPSECI unique ID + * @token: Returned token; use in subsequent API calls + * + * This function can be used to open a control session for an already created + * object; an object may have been declared statically in the DPL + * or created dynamically. + * This function returns a unique authentication token, associated with the + * specific object ID and the specific MC portal; this token must be used in all + * subsequent commands for this specific object. + * + * Return: '0' on success, error code otherwise + */ +int dpseci_open(struct fsl_mc_io *mc_io, u32 cmd_flags, int dpseci_id, + u16 *token) +{ + struct fsl_mc_command cmd = { 0 }; + struct dpseci_cmd_open *cmd_params; + int err; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_OPEN, + cmd_flags, + 0); + cmd_params = (struct dpseci_cmd_open *)cmd.params; + cmd_params->dpseci_id = cpu_to_le32(dpseci_id); + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + *token = mc_cmd_hdr_read_token(&cmd); + + return 0; +} + +/** + * dpseci_close() - Close the control session of the object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * + * After this function is called, no further operations are allowed on the + * object without opening a new control session. + * + * Return: '0' on success, error code otherwise + */ +int dpseci_close(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) +{ + struct fsl_mc_command cmd = { 0 }; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_CLOSE, + cmd_flags, + token); + return mc_send_command(mc_io, &cmd); +} + +/** + * dpseci_enable() - Enable the DPSECI, allow sending and receiving frames + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * + * Return: '0' on success, error code otherwise + */ +int dpseci_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) +{ + struct fsl_mc_command cmd = { 0 }; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_ENABLE, + cmd_flags, + token); + return mc_send_command(mc_io, &cmd); +} + +/** + * dpseci_disable() - Disable the DPSECI, stop sending and receiving frames + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * + * Return: '0' on success, error code otherwise + */ +int dpseci_disable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) +{ + struct fsl_mc_command cmd = { 0 }; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_DISABLE, + cmd_flags, + token); + + return mc_send_command(mc_io, &cmd); +} + +/** + * dpseci_reset() - Reset the DPSECI, returns the object to initial state + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * + * Return: '0' on success, error code otherwise + */ +int dpseci_reset(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) +{ + struct fsl_mc_command cmd = { 0 }; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_RESET, + cmd_flags, + token); + return mc_send_command(mc_io, &cmd); +} + +/** + * dpseci_is_enabled() - Check if the DPSECI is enabled. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @en: Returns '1' if object is enabled; '0' otherwise + * + * Return: '0' on success, error code otherwise + */ +int dpseci_is_enabled(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + int *en) +{ + struct fsl_mc_command cmd = { 0 }; + struct dpseci_rsp_is_enabled *rsp_params; + int err; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_IS_ENABLED, + cmd_flags, + token); + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + rsp_params = (struct dpseci_rsp_is_enabled *)cmd.params; + *en = dpseci_get_field(rsp_params->is_enabled, ENABLE); + + return 0; +} + +/** + * dpseci_get_attributes() - Retrieve DPSECI attributes + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @attr: Returned object's attributes + * + * Return: '0' on success, error code otherwise + */ +int dpseci_get_attributes(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + struct dpseci_attr *attr) +{ + struct fsl_mc_command cmd = { 0 }; + struct dpseci_rsp_get_attributes *rsp_params; + int err; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_ATTR, + cmd_flags, + token); + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + rsp_params = (struct dpseci_rsp_get_attributes *)cmd.params; + attr->id = le32_to_cpu(rsp_params->id); + attr->num_tx_queues = rsp_params->num_tx_queues; + attr->num_rx_queues = rsp_params->num_rx_queues; + attr->options = le32_to_cpu(rsp_params->options); + + return 0; +} + +/** + * dpseci_set_rx_queue() - Set Rx queue configuration + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @queue: Select the queue relative to number of priorities configured at + * DPSECI creation; use DPSECI_ALL_QUEUES to configure all + * Rx queues identically. + * @cfg: Rx queue configuration + * + * Return: '0' on success, error code otherwise + */ +int dpseci_set_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 queue, const struct dpseci_rx_queue_cfg *cfg) +{ + struct fsl_mc_command cmd = { 0 }; + struct dpseci_cmd_queue *cmd_params; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_SET_RX_QUEUE, + cmd_flags, + token); + cmd_params = (struct dpseci_cmd_queue *)cmd.params; + cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id); + cmd_params->priority = cfg->dest_cfg.priority; + cmd_params->queue = queue; + dpseci_set_field(cmd_params->dest_type, DEST_TYPE, + cfg->dest_cfg.dest_type); + cmd_params->user_ctx = cpu_to_le64(cfg->user_ctx); + cmd_params->options = cpu_to_le32(cfg->options); + dpseci_set_field(cmd_params->order_preservation_en, ORDER_PRESERVATION, + cfg->order_preservation_en); + + return mc_send_command(mc_io, &cmd); +} + +/** + * dpseci_get_rx_queue() - Retrieve Rx queue attributes + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @queue: Select the queue relative to number of priorities configured at + * DPSECI creation + * @attr: Returned Rx queue attributes + * + * Return: '0' on success, error code otherwise + */ +int dpseci_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 queue, struct dpseci_rx_queue_attr *attr) +{ + struct fsl_mc_command cmd = { 0 }; + struct dpseci_cmd_queue *cmd_params; + int err; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_RX_QUEUE, + cmd_flags, + token); + cmd_params = (struct dpseci_cmd_queue *)cmd.params; + cmd_params->queue = queue; + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + attr->dest_cfg.dest_id = le32_to_cpu(cmd_params->dest_id); + attr->dest_cfg.priority = cmd_params->priority; + attr->dest_cfg.dest_type = dpseci_get_field(cmd_params->dest_type, + DEST_TYPE); + attr->user_ctx = le64_to_cpu(cmd_params->user_ctx); + attr->fqid = le32_to_cpu(cmd_params->fqid); + attr->order_preservation_en = + dpseci_get_field(cmd_params->order_preservation_en, + ORDER_PRESERVATION); + + return 0; +} + +/** + * dpseci_get_tx_queue() - Retrieve Tx queue attributes + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @queue: Select the queue relative to number of priorities configured at + * DPSECI creation + * @attr: Returned Tx queue attributes + * + * Return: '0' on success, error code otherwise + */ +int dpseci_get_tx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 queue, struct dpseci_tx_queue_attr *attr) +{ + struct fsl_mc_command cmd = { 0 }; + struct dpseci_cmd_queue *cmd_params; + struct dpseci_rsp_get_tx_queue *rsp_params; + int err; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_TX_QUEUE, + cmd_flags, + token); + cmd_params = (struct dpseci_cmd_queue *)cmd.params; + cmd_params->queue = queue; + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + rsp_params = (struct dpseci_rsp_get_tx_queue *)cmd.params; + attr->fqid = le32_to_cpu(rsp_params->fqid); + attr->priority = rsp_params->priority; + + return 0; +} + +/** + * dpseci_get_sec_attr() - Retrieve SEC accelerator attributes + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @attr: Returned SEC attributes + * + * Return: '0' on success, error code otherwise + */ +int dpseci_get_sec_attr(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + struct dpseci_sec_attr *attr) +{ + struct fsl_mc_command cmd = { 0 }; + struct dpseci_rsp_get_sec_attr *rsp_params; + int err; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_SEC_ATTR, + cmd_flags, + token); + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + rsp_params = (struct dpseci_rsp_get_sec_attr *)cmd.params; + attr->ip_id = le16_to_cpu(rsp_params->ip_id); + attr->major_rev = rsp_params->major_rev; + attr->minor_rev = rsp_params->minor_rev; + attr->era = rsp_params->era; + attr->deco_num = rsp_params->deco_num; + attr->zuc_auth_acc_num = rsp_params->zuc_auth_acc_num; + attr->zuc_enc_acc_num = rsp_params->zuc_enc_acc_num; + attr->snow_f8_acc_num = rsp_params->snow_f8_acc_num; + attr->snow_f9_acc_num = rsp_params->snow_f9_acc_num; + attr->crc_acc_num = rsp_params->crc_acc_num; + attr->pk_acc_num = rsp_params->pk_acc_num; + attr->kasumi_acc_num = rsp_params->kasumi_acc_num; + attr->rng_acc_num = rsp_params->rng_acc_num; + attr->md_acc_num = rsp_params->md_acc_num; + attr->arc4_acc_num = rsp_params->arc4_acc_num; + attr->des_acc_num = rsp_params->des_acc_num; + attr->aes_acc_num = rsp_params->aes_acc_num; + attr->ccha_acc_num = rsp_params->ccha_acc_num; + attr->ptha_acc_num = rsp_params->ptha_acc_num; + + return 0; +} + +/** + * dpseci_get_api_version() - Get Data Path SEC Interface API version + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @major_ver: Major version of data path sec API + * @minor_ver: Minor version of data path sec API + * + * Return: '0' on success, error code otherwise + */ +int dpseci_get_api_version(struct fsl_mc_io *mc_io, u32 cmd_flags, + u16 *major_ver, u16 *minor_ver) +{ + struct fsl_mc_command cmd = { 0 }; + struct dpseci_rsp_get_api_version *rsp_params; + int err; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_API_VERSION, + cmd_flags, 0); + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + rsp_params = (struct dpseci_rsp_get_api_version *)cmd.params; + *major_ver = le16_to_cpu(rsp_params->major); + *minor_ver = le16_to_cpu(rsp_params->minor); + + return 0; +} + +/** + * dpseci_set_congestion_notification() - Set congestion group + * notification configuration + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @cfg: congestion notification configuration + * + * Return: '0' on success, error code otherwise + */ +int dpseci_set_congestion_notification(struct fsl_mc_io *mc_io, u32 cmd_flags, + u16 token, const struct dpseci_congestion_notification_cfg *cfg) +{ + struct fsl_mc_command cmd = { 0 }; + struct dpseci_cmd_congestion_notification *cmd_params; + + cmd.header = mc_encode_cmd_header( + DPSECI_CMDID_SET_CONGESTION_NOTIFICATION, + cmd_flags, + token); + cmd_params = (struct dpseci_cmd_congestion_notification *)cmd.params; + cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id); + cmd_params->notification_mode = cpu_to_le16(cfg->notification_mode); + cmd_params->priority = cfg->dest_cfg.priority; + dpseci_set_field(cmd_params->options, CGN_DEST_TYPE, + cfg->dest_cfg.dest_type); + dpseci_set_field(cmd_params->options, CGN_UNITS, cfg->units); + cmd_params->message_iova = cpu_to_le64(cfg->message_iova); + cmd_params->message_ctx = cpu_to_le64(cfg->message_ctx); + cmd_params->threshold_entry = cpu_to_le32(cfg->threshold_entry); + cmd_params->threshold_exit = cpu_to_le32(cfg->threshold_exit); + + return mc_send_command(mc_io, &cmd); +} + +/** + * dpseci_get_congestion_notification() - Get congestion group notification + * configuration + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @cfg: congestion notification configuration + * + * Return: '0' on success, error code otherwise + */ +int dpseci_get_congestion_notification(struct fsl_mc_io *mc_io, u32 cmd_flags, + u16 token, struct dpseci_congestion_notification_cfg *cfg) +{ + struct fsl_mc_command cmd = { 0 }; + struct dpseci_cmd_congestion_notification *rsp_params; + int err; + + cmd.header = mc_encode_cmd_header( + DPSECI_CMDID_GET_CONGESTION_NOTIFICATION, + cmd_flags, + token); + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + rsp_params = (struct dpseci_cmd_congestion_notification *)cmd.params; + cfg->dest_cfg.dest_id = le32_to_cpu(rsp_params->dest_id); + cfg->notification_mode = le16_to_cpu(rsp_params->notification_mode); + cfg->dest_cfg.priority = rsp_params->priority; + cfg->dest_cfg.dest_type = dpseci_get_field(rsp_params->options, + CGN_DEST_TYPE); + cfg->units = dpseci_get_field(rsp_params->options, CGN_UNITS); + cfg->message_iova = le64_to_cpu(rsp_params->message_iova); + cfg->message_ctx = le64_to_cpu(rsp_params->message_ctx); + cfg->threshold_entry = le32_to_cpu(rsp_params->threshold_entry); + cfg->threshold_exit = le32_to_cpu(rsp_params->threshold_exit); + + return 0; +} diff --git a/drivers/crypto/caam/dpseci.h b/drivers/crypto/caam/dpseci.h new file mode 100644 index 000000000..6dcd9be81 --- /dev/null +++ b/drivers/crypto/caam/dpseci.h @@ -0,0 +1,335 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */ +/* + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2017-2018 NXP + */ +#ifndef _DPSECI_H_ +#define _DPSECI_H_ + +/* + * Data Path SEC Interface API + * Contains initialization APIs and runtime control APIs for DPSECI + */ + +struct fsl_mc_io; + +/** + * General DPSECI macros + */ + +/** + * Maximum number of Tx/Rx queues per DPSECI object + */ +#define DPSECI_MAX_QUEUE_NUM 16 + +/** + * All queues considered; see dpseci_set_rx_queue() + */ +#define DPSECI_ALL_QUEUES (u8)(-1) + +int dpseci_open(struct fsl_mc_io *mc_io, u32 cmd_flags, int dpseci_id, + u16 *token); + +int dpseci_close(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); + +/** + * Enable the Congestion Group support + */ +#define DPSECI_OPT_HAS_CG 0x000020 + +/** + * struct dpseci_cfg - Structure representing DPSECI configuration + * @options: Any combination of the following flags: + * DPSECI_OPT_HAS_CG + * @num_tx_queues: num of queues towards the SEC + * @num_rx_queues: num of queues back from the SEC + * @priorities: Priorities for the SEC hardware processing; + * each place in the array is the priority of the tx queue + * towards the SEC; + * valid priorities are configured with values 1-8; + */ +struct dpseci_cfg { + u32 options; + u8 num_tx_queues; + u8 num_rx_queues; + u8 priorities[DPSECI_MAX_QUEUE_NUM]; +}; + +int dpseci_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); + +int dpseci_disable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); + +int dpseci_reset(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); + +int dpseci_is_enabled(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + int *en); + +/** + * struct dpseci_attr - Structure representing DPSECI attributes + * @id: DPSECI object ID + * @num_tx_queues: number of queues towards the SEC + * @num_rx_queues: number of queues back from the SEC + * @options: any combination of the following flags: + * DPSECI_OPT_HAS_CG + */ +struct dpseci_attr { + int id; + u8 num_tx_queues; + u8 num_rx_queues; + u32 options; +}; + +int dpseci_get_attributes(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + struct dpseci_attr *attr); + +/** + * enum dpseci_dest - DPSECI destination types + * @DPSECI_DEST_NONE: Unassigned destination; The queue is set in parked mode + * and does not generate FQDAN notifications; user is expected to dequeue + * from the queue based on polling or other user-defined method + * @DPSECI_DEST_DPIO: The queue is set in schedule mode and generates FQDAN + * notifications to the specified DPIO; user is expected to dequeue from + * the queue only after notification is received + * @DPSECI_DEST_DPCON: The queue is set in schedule mode and does not generate + * FQDAN notifications, but is connected to the specified DPCON object; + * user is expected to dequeue from the DPCON channel + */ +enum dpseci_dest { + DPSECI_DEST_NONE = 0, + DPSECI_DEST_DPIO, + DPSECI_DEST_DPCON +}; + +/** + * struct dpseci_dest_cfg - Structure representing DPSECI destination parameters + * @dest_type: Destination type + * @dest_id: Either DPIO ID or DPCON ID, depending on the destination type + * @priority: Priority selection within the DPIO or DPCON channel; valid values + * are 0-1 or 0-7, depending on the number of priorities in that channel; + * not relevant for 'DPSECI_DEST_NONE' option + */ +struct dpseci_dest_cfg { + enum dpseci_dest dest_type; + int dest_id; + u8 priority; +}; + +/** + * DPSECI queue modification options + */ + +/** + * Select to modify the user's context associated with the queue + */ +#define DPSECI_QUEUE_OPT_USER_CTX 0x00000001 + +/** + * Select to modify the queue's destination + */ +#define DPSECI_QUEUE_OPT_DEST 0x00000002 + +/** + * Select to modify the queue's order preservation + */ +#define DPSECI_QUEUE_OPT_ORDER_PRESERVATION 0x00000004 + +/** + * struct dpseci_rx_queue_cfg - DPSECI RX queue configuration + * @options: Flags representing the suggested modifications to the queue; + * Use any combination of 'DPSECI_QUEUE_OPT_' flags + * @order_preservation_en: order preservation configuration for the rx queue + * valid only if 'DPSECI_QUEUE_OPT_ORDER_PRESERVATION' is contained in 'options' + * @user_ctx: User context value provided in the frame descriptor of each + * dequeued frame; valid only if 'DPSECI_QUEUE_OPT_USER_CTX' is contained + * in 'options' + * @dest_cfg: Queue destination parameters; valid only if + * 'DPSECI_QUEUE_OPT_DEST' is contained in 'options' + */ +struct dpseci_rx_queue_cfg { + u32 options; + int order_preservation_en; + u64 user_ctx; + struct dpseci_dest_cfg dest_cfg; +}; + +int dpseci_set_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 queue, const struct dpseci_rx_queue_cfg *cfg); + +/** + * struct dpseci_rx_queue_attr - Structure representing attributes of Rx queues + * @user_ctx: User context value provided in the frame descriptor of each + * dequeued frame + * @order_preservation_en: Status of the order preservation configuration on the + * queue + * @dest_cfg: Queue destination configuration + * @fqid: Virtual FQID value to be used for dequeue operations + */ +struct dpseci_rx_queue_attr { + u64 user_ctx; + int order_preservation_en; + struct dpseci_dest_cfg dest_cfg; + u32 fqid; +}; + +int dpseci_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 queue, struct dpseci_rx_queue_attr *attr); + +/** + * struct dpseci_tx_queue_attr - Structure representing attributes of Tx queues + * @fqid: Virtual FQID to be used for sending frames to SEC hardware + * @priority: SEC hardware processing priority for the queue + */ +struct dpseci_tx_queue_attr { + u32 fqid; + u8 priority; +}; + +int dpseci_get_tx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 queue, struct dpseci_tx_queue_attr *attr); + +/** + * struct dpseci_sec_attr - Structure representing attributes of the SEC + * hardware accelerator + * @ip_id: ID for SEC + * @major_rev: Major revision number for SEC + * @minor_rev: Minor revision number for SEC + * @era: SEC Era + * @deco_num: The number of copies of the DECO that are implemented in this + * version of SEC + * @zuc_auth_acc_num: The number of copies of ZUCA that are implemented in this + * version of SEC + * @zuc_enc_acc_num: The number of copies of ZUCE that are implemented in this + * version of SEC + * @snow_f8_acc_num: The number of copies of the SNOW-f8 module that are + * implemented in this version of SEC + * @snow_f9_acc_num: The number of copies of the SNOW-f9 module that are + * implemented in this version of SEC + * @crc_acc_num: The number of copies of the CRC module that are implemented in + * this version of SEC + * @pk_acc_num: The number of copies of the Public Key module that are + * implemented in this version of SEC + * @kasumi_acc_num: The number of copies of the Kasumi module that are + * implemented in this version of SEC + * @rng_acc_num: The number of copies of the Random Number Generator that are + * implemented in this version of SEC + * @md_acc_num: The number of copies of the MDHA (Hashing module) that are + * implemented in this version of SEC + * @arc4_acc_num: The number of copies of the ARC4 module that are implemented + * in this version of SEC + * @des_acc_num: The number of copies of the DES module that are implemented in + * this version of SEC + * @aes_acc_num: The number of copies of the AES module that are implemented in + * this version of SEC + * @ccha_acc_num: The number of copies of the ChaCha20 module that are + * implemented in this version of SEC. + * @ptha_acc_num: The number of copies of the Poly1305 module that are + * implemented in this version of SEC. + **/ +struct dpseci_sec_attr { + u16 ip_id; + u8 major_rev; + u8 minor_rev; + u8 era; + u8 deco_num; + u8 zuc_auth_acc_num; + u8 zuc_enc_acc_num; + u8 snow_f8_acc_num; + u8 snow_f9_acc_num; + u8 crc_acc_num; + u8 pk_acc_num; + u8 kasumi_acc_num; + u8 rng_acc_num; + u8 md_acc_num; + u8 arc4_acc_num; + u8 des_acc_num; + u8 aes_acc_num; + u8 ccha_acc_num; + u8 ptha_acc_num; +}; + +int dpseci_get_sec_attr(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + struct dpseci_sec_attr *attr); + +int dpseci_get_api_version(struct fsl_mc_io *mc_io, u32 cmd_flags, + u16 *major_ver, u16 *minor_ver); + +/** + * enum dpseci_congestion_unit - DPSECI congestion units + * @DPSECI_CONGESTION_UNIT_BYTES: bytes units + * @DPSECI_CONGESTION_UNIT_FRAMES: frames units + */ +enum dpseci_congestion_unit { + DPSECI_CONGESTION_UNIT_BYTES = 0, + DPSECI_CONGESTION_UNIT_FRAMES +}; + +/** + * CSCN message is written to message_iova once entering a + * congestion state (see 'threshold_entry') + */ +#define DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER 0x00000001 + +/** + * CSCN message is written to message_iova once exiting a + * congestion state (see 'threshold_exit') + */ +#define DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT 0x00000002 + +/** + * CSCN write will attempt to allocate into a cache (coherent write); + * valid only if 'DPSECI_CGN_MODE_WRITE_MEM_' is selected + */ +#define DPSECI_CGN_MODE_COHERENT_WRITE 0x00000004 + +/** + * if 'dpseci_dest_cfg.dest_type != DPSECI_DEST_NONE' CSCN message is sent to + * DPIO/DPCON's WQ channel once entering a congestion state + * (see 'threshold_entry') + */ +#define DPSECI_CGN_MODE_NOTIFY_DEST_ON_ENTER 0x00000008 + +/** + * if 'dpseci_dest_cfg.dest_type != DPSECI_DEST_NONE' CSCN message is sent to + * DPIO/DPCON's WQ channel once exiting a congestion state + * (see 'threshold_exit') + */ +#define DPSECI_CGN_MODE_NOTIFY_DEST_ON_EXIT 0x00000010 + +/** + * if 'dpseci_dest_cfg.dest_type != DPSECI_DEST_NONE' when the CSCN is written + * to the sw-portal's DQRR, the DQRI interrupt is asserted immediately + * (if enabled) + */ +#define DPSECI_CGN_MODE_INTR_COALESCING_DISABLED 0x00000020 + +/** + * struct dpseci_congestion_notification_cfg - congestion notification + * configuration + * @units: units type + * @threshold_entry: above this threshold we enter a congestion state. + * set it to '0' to disable it + * @threshold_exit: below this threshold we exit the congestion state. + * @message_ctx: The context that will be part of the CSCN message + * @message_iova: I/O virtual address (must be in DMA-able memory), + * must be 16B aligned; + * @dest_cfg: CSCN can be send to either DPIO or DPCON WQ channel + * @notification_mode: Mask of available options; use 'DPSECI_CGN_MODE_' + * values + */ +struct dpseci_congestion_notification_cfg { + enum dpseci_congestion_unit units; + u32 threshold_entry; + u32 threshold_exit; + u64 message_ctx; + u64 message_iova; + struct dpseci_dest_cfg dest_cfg; + u16 notification_mode; +}; + +int dpseci_set_congestion_notification(struct fsl_mc_io *mc_io, u32 cmd_flags, + u16 token, const struct dpseci_congestion_notification_cfg *cfg); + +int dpseci_get_congestion_notification(struct fsl_mc_io *mc_io, u32 cmd_flags, + u16 token, struct dpseci_congestion_notification_cfg *cfg); + +#endif /* _DPSECI_H_ */ diff --git a/drivers/crypto/caam/dpseci_cmd.h b/drivers/crypto/caam/dpseci_cmd.h new file mode 100644 index 000000000..71a007c85 --- /dev/null +++ b/drivers/crypto/caam/dpseci_cmd.h @@ -0,0 +1,150 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */ +/* + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2017-2018 NXP + */ + +#ifndef _DPSECI_CMD_H_ +#define _DPSECI_CMD_H_ + +/* DPSECI Version */ +#define DPSECI_VER_MAJOR 5 +#define DPSECI_VER_MINOR 3 + +#define DPSECI_VER(maj, min) (((maj) << 16) | (min)) +#define DPSECI_VERSION DPSECI_VER(DPSECI_VER_MAJOR, DPSECI_VER_MINOR) + +/* Command versioning */ +#define DPSECI_CMD_BASE_VERSION 1 +#define DPSECI_CMD_BASE_VERSION_V2 2 +#define DPSECI_CMD_ID_OFFSET 4 + +#define DPSECI_CMD_V1(id) (((id) << DPSECI_CMD_ID_OFFSET) | \ + DPSECI_CMD_BASE_VERSION) + +#define DPSECI_CMD_V2(id) (((id) << DPSECI_CMD_ID_OFFSET) | \ + DPSECI_CMD_BASE_VERSION_V2) + +/* Command IDs */ +#define DPSECI_CMDID_CLOSE DPSECI_CMD_V1(0x800) +#define DPSECI_CMDID_OPEN DPSECI_CMD_V1(0x809) +#define DPSECI_CMDID_GET_API_VERSION DPSECI_CMD_V1(0xa09) + +#define DPSECI_CMDID_ENABLE DPSECI_CMD_V1(0x002) +#define DPSECI_CMDID_DISABLE DPSECI_CMD_V1(0x003) +#define DPSECI_CMDID_GET_ATTR DPSECI_CMD_V1(0x004) +#define DPSECI_CMDID_RESET DPSECI_CMD_V1(0x005) +#define DPSECI_CMDID_IS_ENABLED DPSECI_CMD_V1(0x006) + +#define DPSECI_CMDID_SET_RX_QUEUE DPSECI_CMD_V1(0x194) +#define DPSECI_CMDID_GET_RX_QUEUE DPSECI_CMD_V1(0x196) +#define DPSECI_CMDID_GET_TX_QUEUE DPSECI_CMD_V1(0x197) +#define DPSECI_CMDID_GET_SEC_ATTR DPSECI_CMD_V2(0x198) +#define DPSECI_CMDID_SET_CONGESTION_NOTIFICATION DPSECI_CMD_V1(0x170) +#define DPSECI_CMDID_GET_CONGESTION_NOTIFICATION DPSECI_CMD_V1(0x171) + +/* Macros for accessing command fields smaller than 1 byte */ +#define DPSECI_MASK(field) \ + GENMASK(DPSECI_##field##_SHIFT + DPSECI_##field##_SIZE - 1, \ + DPSECI_##field##_SHIFT) + +#define dpseci_set_field(var, field, val) \ + ((var) |= (((val) << DPSECI_##field##_SHIFT) & DPSECI_MASK(field))) + +#define dpseci_get_field(var, field) \ + (((var) & DPSECI_MASK(field)) >> DPSECI_##field##_SHIFT) + +struct dpseci_cmd_open { + __le32 dpseci_id; +}; + +#define DPSECI_ENABLE_SHIFT 0 +#define DPSECI_ENABLE_SIZE 1 + +struct dpseci_rsp_is_enabled { + u8 is_enabled; +}; + +struct dpseci_rsp_get_attributes { + __le32 id; + __le32 pad0; + u8 num_tx_queues; + u8 num_rx_queues; + u8 pad1[6]; + __le32 options; +}; + +#define DPSECI_DEST_TYPE_SHIFT 0 +#define DPSECI_DEST_TYPE_SIZE 4 + +#define DPSECI_ORDER_PRESERVATION_SHIFT 0 +#define DPSECI_ORDER_PRESERVATION_SIZE 1 + +struct dpseci_cmd_queue { + __le32 dest_id; + u8 priority; + u8 queue; + u8 dest_type; + u8 pad; + __le64 user_ctx; + union { + __le32 options; + __le32 fqid; + }; + u8 order_preservation_en; +}; + +struct dpseci_rsp_get_tx_queue { + __le32 pad; + __le32 fqid; + u8 priority; +}; + +struct dpseci_rsp_get_sec_attr { + __le16 ip_id; + u8 major_rev; + u8 minor_rev; + u8 era; + u8 pad0[3]; + u8 deco_num; + u8 zuc_auth_acc_num; + u8 zuc_enc_acc_num; + u8 pad1; + u8 snow_f8_acc_num; + u8 snow_f9_acc_num; + u8 crc_acc_num; + u8 pad2; + u8 pk_acc_num; + u8 kasumi_acc_num; + u8 rng_acc_num; + u8 pad3; + u8 md_acc_num; + u8 arc4_acc_num; + u8 des_acc_num; + u8 aes_acc_num; + u8 ccha_acc_num; + u8 ptha_acc_num; +}; + +struct dpseci_rsp_get_api_version { + __le16 major; + __le16 minor; +}; + +#define DPSECI_CGN_DEST_TYPE_SHIFT 0 +#define DPSECI_CGN_DEST_TYPE_SIZE 4 +#define DPSECI_CGN_UNITS_SHIFT 4 +#define DPSECI_CGN_UNITS_SIZE 2 + +struct dpseci_cmd_congestion_notification { + __le32 dest_id; + __le16 notification_mode; + u8 priority; + u8 options; + __le64 message_iova; + __le64 message_ctx; + __le32 threshold_entry; + __le32 threshold_exit; +}; + +#endif /* _DPSECI_CMD_H_ */ diff --git a/drivers/crypto/caam/error.c b/drivers/crypto/caam/error.c new file mode 100644 index 000000000..72db90176 --- /dev/null +++ b/drivers/crypto/caam/error.c @@ -0,0 +1,393 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * CAAM Error Reporting + * + * Copyright 2009-2011 Freescale Semiconductor, Inc. + */ + +#include "compat.h" +#include "regs.h" +#include "desc.h" +#include "error.h" + +#ifdef DEBUG +#include + +void caam_dump_sg(const char *prefix_str, int prefix_type, + int rowsize, int groupsize, struct scatterlist *sg, + size_t tlen, bool ascii) +{ + struct scatterlist *it; + void *it_page; + size_t len; + void *buf; + + for (it = sg; it && tlen > 0 ; it = sg_next(it)) { + /* + * make sure the scatterlist's page + * has a valid virtual memory mapping + */ + it_page = kmap_atomic(sg_page(it)); + if (unlikely(!it_page)) { + pr_err("caam_dump_sg: kmap failed\n"); + return; + } + + buf = it_page + it->offset; + len = min_t(size_t, tlen, it->length); + print_hex_dump_debug(prefix_str, prefix_type, rowsize, + groupsize, buf, len, ascii); + tlen -= len; + + kunmap_atomic(it_page); + } +} +#else +void caam_dump_sg(const char *prefix_str, int prefix_type, + int rowsize, int groupsize, struct scatterlist *sg, + size_t tlen, bool ascii) +{} +#endif /* DEBUG */ +EXPORT_SYMBOL(caam_dump_sg); + +bool caam_little_end; +EXPORT_SYMBOL(caam_little_end); + +bool caam_imx; +EXPORT_SYMBOL(caam_imx); + +size_t caam_ptr_sz; +EXPORT_SYMBOL(caam_ptr_sz); + +static const struct { + u8 value; + const char *error_text; +} desc_error_list[] = { + { 0x00, "No error." }, + { 0x01, "SGT Length Error. The descriptor is trying to read more data than is contained in the SGT table." }, + { 0x02, "SGT Null Entry Error." }, + { 0x03, "Job Ring Control Error. There is a bad value in the Job Ring Control register." }, + { 0x04, "Invalid Descriptor Command. The Descriptor Command field is invalid." }, + { 0x05, "Reserved." }, + { 0x06, "Invalid KEY Command" }, + { 0x07, "Invalid LOAD Command" }, + { 0x08, "Invalid STORE Command" }, + { 0x09, "Invalid OPERATION Command" }, + { 0x0A, "Invalid FIFO LOAD Command" }, + { 0x0B, "Invalid FIFO STORE Command" }, + { 0x0C, "Invalid MOVE/MOVE_LEN Command" }, + { 0x0D, "Invalid JUMP Command. A nonlocal JUMP Command is invalid because the target is not a Job Header Command, or the jump is from a Trusted Descriptor to a Job Descriptor, or because the target Descriptor contains a Shared Descriptor." }, + { 0x0E, "Invalid MATH Command" }, + { 0x0F, "Invalid SIGNATURE Command" }, + { 0x10, "Invalid Sequence Command. A SEQ IN PTR OR SEQ OUT PTR Command is invalid or a SEQ KEY, SEQ LOAD, SEQ FIFO LOAD, or SEQ FIFO STORE decremented the input or output sequence length below 0. This error may result if a built-in PROTOCOL Command has encountered a malformed PDU." }, + { 0x11, "Skip data type invalid. The type must be 0xE or 0xF."}, + { 0x12, "Shared Descriptor Header Error" }, + { 0x13, "Header Error. Invalid length or parity, or certain other problems." }, + { 0x14, "Burster Error. Burster has gotten to an illegal state" }, + { 0x15, "Context Register Length Error. The descriptor is trying to read or write past the end of the Context Register. A SEQ LOAD or SEQ STORE with the VLF bit set was executed with too large a length in the variable length register (VSOL for SEQ STORE or VSIL for SEQ LOAD)." }, + { 0x16, "DMA Error" }, + { 0x17, "Reserved." }, + { 0x1A, "Job failed due to JR reset" }, + { 0x1B, "Job failed due to Fail Mode" }, + { 0x1C, "DECO Watchdog timer timeout error" }, + { 0x1D, "DECO tried to copy a key from another DECO but the other DECO's Key Registers were locked" }, + { 0x1E, "DECO attempted to copy data from a DECO that had an unmasked Descriptor error" }, + { 0x1F, "LIODN error. DECO was trying to share from itself or from another DECO but the two Non-SEQ LIODN values didn't match or the 'shared from' DECO's Descriptor required that the SEQ LIODNs be the same and they aren't." }, + { 0x20, "DECO has completed a reset initiated via the DRR register" }, + { 0x21, "Nonce error. When using EKT (CCM) key encryption option in the FIFO STORE Command, the Nonce counter reached its maximum value and this encryption mode can no longer be used." }, + { 0x22, "Meta data is too large (> 511 bytes) for TLS decap (input frame; block ciphers) and IPsec decap (output frame, when doing the next header byte update) and DCRC (output frame)." }, + { 0x23, "Read Input Frame error" }, + { 0x24, "JDKEK, TDKEK or TDSK not loaded error" }, + { 0x80, "DNR (do not run) error" }, + { 0x81, "undefined protocol command" }, + { 0x82, "invalid setting in PDB" }, + { 0x83, "Anti-replay LATE error" }, + { 0x84, "Anti-replay REPLAY error" }, + { 0x85, "Sequence number overflow" }, + { 0x86, "Sigver invalid signature" }, + { 0x87, "DSA Sign Illegal test descriptor" }, + { 0x88, "Protocol Format Error - A protocol has seen an error in the format of data received. When running RSA, this means that formatting with random padding was used, and did not follow the form: 0x00, 0x02, 8-to-N bytes of non-zero pad, 0x00, F data." }, + { 0x89, "Protocol Size Error - A protocol has seen an error in size. When running RSA, pdb size N < (size of F) when no formatting is used; or pdb size N < (F + 11) when formatting is used." }, + { 0xC1, "Blob Command error: Undefined mode" }, + { 0xC2, "Blob Command error: Secure Memory Blob mode error" }, + { 0xC4, "Blob Command error: Black Blob key or input size error" }, + { 0xC5, "Blob Command error: Invalid key destination" }, + { 0xC8, "Blob Command error: Trusted/Secure mode error" }, + { 0xF0, "IPsec TTL or hop limit field either came in as 0, or was decremented to 0" }, + { 0xF1, "3GPP HFN matches or exceeds the Threshold" }, +}; + +static const struct { + u8 value; + const char *error_text; +} qi_error_list[] = { + { 0x00, "No error" }, + { 0x1F, "Job terminated by FQ or ICID flush" }, + { 0x20, "FD format error"}, + { 0x21, "FD command format error"}, + { 0x23, "FL format error"}, + { 0x25, "CRJD specified in FD, but not enabled in FLC"}, + { 0x30, "Max. buffer size too small"}, + { 0x31, "DHR exceeds max. buffer size (allocate mode, S/G format)"}, + { 0x32, "SGT exceeds max. buffer size (allocate mode, S/G format"}, + { 0x33, "Size over/underflow (allocate mode)"}, + { 0x34, "Size over/underflow (reuse mode)"}, + { 0x35, "Length exceeds max. short length (allocate mode, S/G/ format)"}, + { 0x36, "Memory footprint exceeds max. value (allocate mode, S/G/ format)"}, + { 0x41, "SBC frame format not supported (allocate mode)"}, + { 0x42, "Pool 0 invalid / pool 1 size < pool 0 size (allocate mode)"}, + { 0x43, "Annotation output enabled but ASAR = 0 (allocate mode)"}, + { 0x44, "Unsupported or reserved frame format or SGHR = 1 (reuse mode)"}, + { 0x45, "DHR correction underflow (reuse mode, single buffer format)"}, + { 0x46, "Annotation length exceeds offset (reuse mode)"}, + { 0x48, "Annotation output enabled but ASA limited by ASAR (reuse mode)"}, + { 0x49, "Data offset correction exceeds input frame data length (reuse mode)"}, + { 0x4B, "Annotation output enabled but ASA cannot be expanded (frame list)"}, + { 0x51, "Unsupported IF reuse mode"}, + { 0x52, "Unsupported FL use mode"}, + { 0x53, "Unsupported RJD use mode"}, + { 0x54, "Unsupported inline descriptor use mode"}, + { 0xC0, "Table buffer pool 0 depletion"}, + { 0xC1, "Table buffer pool 1 depletion"}, + { 0xC2, "Data buffer pool 0 depletion, no OF allocated"}, + { 0xC3, "Data buffer pool 1 depletion, no OF allocated"}, + { 0xC4, "Data buffer pool 0 depletion, partial OF allocated"}, + { 0xC5, "Data buffer pool 1 depletion, partial OF allocated"}, + { 0xD0, "FLC read error"}, + { 0xD1, "FL read error"}, + { 0xD2, "FL write error"}, + { 0xD3, "OF SGT write error"}, + { 0xD4, "PTA read error"}, + { 0xD5, "PTA write error"}, + { 0xD6, "OF SGT F-bit write error"}, + { 0xD7, "ASA write error"}, + { 0xE1, "FLC[ICR]=0 ICID error"}, + { 0xE2, "FLC[ICR]=1 ICID error"}, + { 0xE4, "source of ICID flush not trusted (BDI = 0)"}, +}; + +static const char * const cha_id_list[] = { + "", + "AES", + "DES", + "ARC4", + "MDHA", + "RNG", + "SNOW f8", + "Kasumi f8/9", + "PKHA", + "CRCA", + "SNOW f9", + "ZUCE", + "ZUCA", +}; + +static const char * const err_id_list[] = { + "No error.", + "Mode error.", + "Data size error.", + "Key size error.", + "PKHA A memory size error.", + "PKHA B memory size error.", + "Data arrived out of sequence error.", + "PKHA divide-by-zero error.", + "PKHA modulus even error.", + "DES key parity error.", + "ICV check failed.", + "Hardware error.", + "Unsupported CCM AAD size.", + "Class 1 CHA is not reset", + "Invalid CHA combination was selected", + "Invalid CHA selected.", +}; + +static const char * const rng_err_id_list[] = { + "", + "", + "", + "Instantiate", + "Not instantiated", + "Test instantiate", + "Prediction resistance", + "Prediction resistance and test request", + "Uninstantiate", + "Secure key generation", + "", + "Hardware error", + "Continuous check" +}; + +static int report_ccb_status(struct device *jrdev, const u32 status, + const char *error) +{ + u8 cha_id = (status & JRSTA_CCBERR_CHAID_MASK) >> + JRSTA_CCBERR_CHAID_SHIFT; + u8 err_id = status & JRSTA_CCBERR_ERRID_MASK; + u8 idx = (status & JRSTA_DECOERR_INDEX_MASK) >> + JRSTA_DECOERR_INDEX_SHIFT; + char *idx_str; + const char *cha_str = "unidentified cha_id value 0x"; + char cha_err_code[3] = { 0 }; + const char *err_str = "unidentified err_id value 0x"; + char err_err_code[3] = { 0 }; + + if (status & JRSTA_DECOERR_JUMP) + idx_str = "jump tgt desc idx"; + else + idx_str = "desc idx"; + + if (cha_id < ARRAY_SIZE(cha_id_list)) + cha_str = cha_id_list[cha_id]; + else + snprintf(cha_err_code, sizeof(cha_err_code), "%02x", cha_id); + + if ((cha_id << JRSTA_CCBERR_CHAID_SHIFT) == JRSTA_CCBERR_CHAID_RNG && + err_id < ARRAY_SIZE(rng_err_id_list) && + strlen(rng_err_id_list[err_id])) { + /* RNG-only error */ + err_str = rng_err_id_list[err_id]; + } else { + err_str = err_id_list[err_id]; + } + + /* + * CCB ICV check failures are part of normal operation life; + * we leave the upper layers to do what they want with them. + */ + if (err_id == JRSTA_CCBERR_ERRID_ICVCHK) + return -EBADMSG; + + dev_err_ratelimited(jrdev, "%08x: %s: %s %d: %s%s: %s%s\n", status, + error, idx_str, idx, cha_str, cha_err_code, + err_str, err_err_code); + + return -EINVAL; +} + +static int report_jump_status(struct device *jrdev, const u32 status, + const char *error) +{ + dev_err(jrdev, "%08x: %s: %s() not implemented\n", + status, error, __func__); + + return -EINVAL; +} + +static int report_deco_status(struct device *jrdev, const u32 status, + const char *error) +{ + u8 err_id = status & JRSTA_DECOERR_ERROR_MASK; + u8 idx = (status & JRSTA_DECOERR_INDEX_MASK) >> + JRSTA_DECOERR_INDEX_SHIFT; + char *idx_str; + const char *err_str = "unidentified error value 0x"; + char err_err_code[3] = { 0 }; + int i; + + if (status & JRSTA_DECOERR_JUMP) + idx_str = "jump tgt desc idx"; + else + idx_str = "desc idx"; + + for (i = 0; i < ARRAY_SIZE(desc_error_list); i++) + if (desc_error_list[i].value == err_id) + break; + + if (i != ARRAY_SIZE(desc_error_list) && desc_error_list[i].error_text) + err_str = desc_error_list[i].error_text; + else + snprintf(err_err_code, sizeof(err_err_code), "%02x", err_id); + + dev_err(jrdev, "%08x: %s: %s %d: %s%s\n", + status, error, idx_str, idx, err_str, err_err_code); + + return -EINVAL; +} + +static int report_qi_status(struct device *qidev, const u32 status, + const char *error) +{ + u8 err_id = status & JRSTA_QIERR_ERROR_MASK; + const char *err_str = "unidentified error value 0x"; + char err_err_code[3] = { 0 }; + int i; + + for (i = 0; i < ARRAY_SIZE(qi_error_list); i++) + if (qi_error_list[i].value == err_id) + break; + + if (i != ARRAY_SIZE(qi_error_list) && qi_error_list[i].error_text) + err_str = qi_error_list[i].error_text; + else + snprintf(err_err_code, sizeof(err_err_code), "%02x", err_id); + + dev_err(qidev, "%08x: %s: %s%s\n", + status, error, err_str, err_err_code); + + return -EINVAL; +} + +static int report_jr_status(struct device *jrdev, const u32 status, + const char *error) +{ + dev_err(jrdev, "%08x: %s: %s() not implemented\n", + status, error, __func__); + + return -EINVAL; +} + +static int report_cond_code_status(struct device *jrdev, const u32 status, + const char *error) +{ + dev_err(jrdev, "%08x: %s: %s() not implemented\n", + status, error, __func__); + + return -EINVAL; +} + +int caam_strstatus(struct device *jrdev, u32 status, bool qi_v2) +{ + static const struct stat_src { + int (*report_ssed)(struct device *jrdev, const u32 status, + const char *error); + const char *error; + } status_src[16] = { + { NULL, "No error" }, + { NULL, NULL }, + { report_ccb_status, "CCB" }, + { report_jump_status, "Jump" }, + { report_deco_status, "DECO" }, + { report_qi_status, "Queue Manager Interface" }, + { report_jr_status, "Job Ring" }, + { report_cond_code_status, "Condition Code" }, + { NULL, NULL }, + { NULL, NULL }, + { NULL, NULL }, + { NULL, NULL }, + { NULL, NULL }, + { NULL, NULL }, + { NULL, NULL }, + { NULL, NULL }, + }; + u32 ssrc = status >> JRSTA_SSRC_SHIFT; + const char *error = status_src[ssrc].error; + + /* + * If there is an error handling function, call it to report the error. + * Otherwise print the error source name. + */ + if (status_src[ssrc].report_ssed) + return status_src[ssrc].report_ssed(jrdev, status, error); + + if (error) + dev_err(jrdev, "%d: %s\n", ssrc, error); + else + dev_err(jrdev, "%d: unknown error source\n", ssrc); + + return -EINVAL; +} +EXPORT_SYMBOL(caam_strstatus); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("FSL CAAM error reporting"); +MODULE_AUTHOR("Freescale Semiconductor"); diff --git a/drivers/crypto/caam/error.h b/drivers/crypto/caam/error.h new file mode 100644 index 000000000..16809fa8f --- /dev/null +++ b/drivers/crypto/caam/error.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * CAAM Error Reporting code header + * + * Copyright 2009-2011 Freescale Semiconductor, Inc. + */ + +#ifndef CAAM_ERROR_H +#define CAAM_ERROR_H + +#include "desc.h" + +#define CAAM_ERROR_STR_MAX 302 + +int caam_strstatus(struct device *dev, u32 status, bool qi_v2); + +#define caam_jr_strstatus(jrdev, status) caam_strstatus(jrdev, status, false) +#define caam_qi2_strstatus(qidev, status) caam_strstatus(qidev, status, true) + +void caam_dump_sg(const char *prefix_str, int prefix_type, + int rowsize, int groupsize, struct scatterlist *sg, + size_t tlen, bool ascii); + +static inline bool is_mdha(u32 algtype) +{ + return (algtype & OP_ALG_ALGSEL_MASK & ~OP_ALG_ALGSEL_SUBMASK) == + OP_ALG_CHA_MDHA; +} +#endif /* CAAM_ERROR_H */ diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h new file mode 100644 index 000000000..6e48bf7d6 --- /dev/null +++ b/drivers/crypto/caam/intern.h @@ -0,0 +1,251 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * CAAM/SEC 4.x driver backend + * Private/internal definitions between modules + * + * Copyright 2008-2011 Freescale Semiconductor, Inc. + * Copyright 2019, 2023 NXP + */ + +#ifndef INTERN_H +#define INTERN_H + +#include "ctrl.h" +#include + +/* Currently comes from Kconfig param as a ^2 (driver-required) */ +#define JOBR_DEPTH (1 << CONFIG_CRYPTO_DEV_FSL_CAAM_RINGSIZE) + +/* + * Maximum size for crypto-engine software queue based on Job Ring + * size (JOBR_DEPTH) and a THRESHOLD (reserved for the non-crypto-API + * requests that are not passed through crypto-engine) + */ +#define THRESHOLD 15 +#define CRYPTO_ENGINE_MAX_QLEN (JOBR_DEPTH - THRESHOLD) + +/* Kconfig params for interrupt coalescing if selected (else zero) */ +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_INTC +#define JOBR_INTC JRCFG_ICEN +#define JOBR_INTC_TIME_THLD CONFIG_CRYPTO_DEV_FSL_CAAM_INTC_TIME_THLD +#define JOBR_INTC_COUNT_THLD CONFIG_CRYPTO_DEV_FSL_CAAM_INTC_COUNT_THLD +#else +#define JOBR_INTC 0 +#define JOBR_INTC_TIME_THLD 0 +#define JOBR_INTC_COUNT_THLD 0 +#endif + +/* + * Storage for tracking each in-process entry moving across a ring + * Each entry on an output ring needs one of these + */ +struct caam_jrentry_info { + void (*callbk)(struct device *dev, u32 *desc, u32 status, void *arg); + void *cbkarg; /* Argument per ring entry */ + u32 *desc_addr_virt; /* Stored virt addr for postprocessing */ + dma_addr_t desc_addr_dma; /* Stored bus addr for done matching */ + u32 desc_size; /* Stored size for postprocessing, header derived */ +}; + +struct caam_jr_state { + dma_addr_t inpbusaddr; + dma_addr_t outbusaddr; +}; + +struct caam_jr_dequeue_params { + struct device *dev; + int enable_itr; +}; + +/* Private sub-storage for a single JobR */ +struct caam_drv_private_jr { + struct list_head list_node; /* Job Ring device list */ + struct device *dev; + int ridx; + struct caam_job_ring __iomem *rregs; /* JobR's register space */ + struct tasklet_struct irqtask; + struct caam_jr_dequeue_params tasklet_params; + int irq; /* One per queue */ + bool hwrng; + + /* Number of scatterlist crypt transforms active on the JobR */ + atomic_t tfm_count ____cacheline_aligned; + + /* Job ring info */ + struct caam_jrentry_info *entinfo; /* Alloc'ed 1 per ring entry */ + spinlock_t inplock ____cacheline_aligned; /* Input ring index lock */ + u32 inpring_avail; /* Number of free entries in input ring */ + int head; /* entinfo (s/w ring) head index */ + void *inpring; /* Base of input ring, alloc + * DMA-safe */ + int out_ring_read_index; /* Output index "tail" */ + int tail; /* entinfo (s/w ring) tail index */ + void *outring; /* Base of output ring, DMA-safe */ + struct crypto_engine *engine; + + struct caam_jr_state state; /* State of the JR during PM */ +}; + +struct caam_ctl_state { + struct masterid deco_mid[16]; + struct masterid jr_mid[4]; + u32 mcr; + u32 scfgr; +}; + +/* + * Driver-private storage for a single CAAM block instance + */ +struct caam_drv_private { + /* Physical-presence section */ + struct caam_ctrl __iomem *ctrl; /* controller region */ + struct caam_deco __iomem *deco; /* DECO/CCB views */ + struct caam_assurance __iomem *assure; + struct caam_queue_if __iomem *qi; /* QI control region */ + struct caam_job_ring __iomem *jr[4]; /* JobR's register space */ + + struct iommu_domain *domain; + + /* + * Detected geometry block. Filled in from device tree if powerpc, + * or from register-based version detection code + */ + u8 total_jobrs; /* Total Job Rings in device */ + u8 qi_present; /* Nonzero if QI present in device */ + u8 blob_present; /* Nonzero if BLOB support present in device */ + u8 mc_en; /* Nonzero if MC f/w is active */ + u8 optee_en; /* Nonzero if OP-TEE f/w is active */ + u8 no_page0; /* Nonzero if register page 0 is not controlled by Linux */ + bool pr_support; /* RNG prediction resistance available */ + int secvio_irq; /* Security violation interrupt number */ + int virt_en; /* Virtualization enabled in CAAM */ + int era; /* CAAM Era (internal HW revision) */ + +#define RNG4_MAX_HANDLES 2 + /* RNG4 block */ + u32 rng4_sh_init; /* This bitmap shows which of the State + Handles of the RNG4 block are initialized + by this driver */ + + struct clk_bulk_data *clks; + int num_clks; + /* + * debugfs entries for developer view into driver/device + * variables at runtime. + */ +#ifdef CONFIG_DEBUG_FS + struct dentry *ctl; /* controller dir */ + struct debugfs_blob_wrapper ctl_kek_wrap, ctl_tkek_wrap, ctl_tdsk_wrap; +#endif + + int caam_off_during_pm; /* If the CAAM is reset after suspend */ + struct caam_ctl_state state; /* State of the CTL during PM */ +}; + +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API + +int caam_algapi_init(struct device *dev); +void caam_algapi_exit(void); + +#else + +static inline int caam_algapi_init(struct device *dev) +{ + return 0; +} + +static inline void caam_algapi_exit(void) +{ +} + +#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API */ + +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API + +int caam_algapi_hash_init(struct device *dev); +void caam_algapi_hash_exit(void); + +#else + +static inline int caam_algapi_hash_init(struct device *dev) +{ + return 0; +} + +static inline void caam_algapi_hash_exit(void) +{ +} + +#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API */ + +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API + +int caam_pkc_init(struct device *dev); +void caam_pkc_exit(void); + +#else + +static inline int caam_pkc_init(struct device *dev) +{ + return 0; +} + +static inline void caam_pkc_exit(void) +{ +} + +#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API */ + +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API + +int caam_rng_init(struct device *dev); +void caam_rng_exit(struct device *dev); + +#else + +static inline int caam_rng_init(struct device *dev) +{ + return 0; +} + +static inline void caam_rng_exit(struct device *dev) {} + +#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API */ + +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI + +int caam_qi_algapi_init(struct device *dev); +void caam_qi_algapi_exit(void); + +#else + +static inline int caam_qi_algapi_init(struct device *dev) +{ + return 0; +} + +static inline void caam_qi_algapi_exit(void) +{ +} + +#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI */ + +static inline u64 caam_get_dma_mask(struct device *dev) +{ + struct device_node *nprop = dev->of_node; + + if (caam_ptr_sz != sizeof(u64)) + return DMA_BIT_MASK(32); + + if (caam_dpaa2) + return DMA_BIT_MASK(49); + + if (of_device_is_compatible(nprop, "fsl,sec-v5.0-job-ring") || + of_device_is_compatible(nprop, "fsl,sec-v5.0")) + return DMA_BIT_MASK(40); + + return DMA_BIT_MASK(36); +} + + +#endif /* INTERN_H */ diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c new file mode 100644 index 000000000..cf725a6ac --- /dev/null +++ b/drivers/crypto/caam/jr.c @@ -0,0 +1,835 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * CAAM/SEC 4.x transport/backend driver + * JobR backend functionality + * + * Copyright 2008-2012 Freescale Semiconductor, Inc. + * Copyright 2019, 2023 NXP + */ + +#include +#include +#include + +#include "compat.h" +#include "ctrl.h" +#include "regs.h" +#include "jr.h" +#include "desc.h" +#include "intern.h" + +struct jr_driver_data { + /* List of Physical JobR's with the Driver */ + struct list_head jr_list; + spinlock_t jr_alloc_lock; /* jr_list lock */ +} ____cacheline_aligned; + +static struct jr_driver_data driver_data; +static DEFINE_MUTEX(algs_lock); +static unsigned int active_devs; + +static void register_algs(struct caam_drv_private_jr *jrpriv, + struct device *dev) +{ + mutex_lock(&algs_lock); + + if (++active_devs != 1) + goto algs_unlock; + + caam_algapi_init(dev); + caam_algapi_hash_init(dev); + caam_pkc_init(dev); + jrpriv->hwrng = !caam_rng_init(dev); + caam_qi_algapi_init(dev); + +algs_unlock: + mutex_unlock(&algs_lock); +} + +static void unregister_algs(void) +{ + mutex_lock(&algs_lock); + + if (--active_devs != 0) + goto algs_unlock; + + caam_qi_algapi_exit(); + caam_pkc_exit(); + caam_algapi_hash_exit(); + caam_algapi_exit(); + +algs_unlock: + mutex_unlock(&algs_lock); +} + +static void caam_jr_crypto_engine_exit(void *data) +{ + struct device *jrdev = data; + struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev); + + /* Free the resources of crypto-engine */ + crypto_engine_exit(jrpriv->engine); +} + +/* + * Put the CAAM in quiesce, ie stop + * + * Must be called with itr disabled + */ +static int caam_jr_stop_processing(struct device *dev, u32 jrcr_bits) +{ + struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); + unsigned int timeout = 100000; + + /* Check the current status */ + if (rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_INPROGRESS) + goto wait_quiesce_completion; + + /* Reset the field */ + clrsetbits_32(&jrp->rregs->jrintstatus, JRINT_ERR_HALT_MASK, 0); + + /* initiate flush / park (required prior to reset) */ + wr_reg32(&jrp->rregs->jrcommand, jrcr_bits); + +wait_quiesce_completion: + while (((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) == + JRINT_ERR_HALT_INPROGRESS) && --timeout) + cpu_relax(); + + if ((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) != + JRINT_ERR_HALT_COMPLETE || timeout == 0) { + dev_err(dev, "failed to flush job ring %d\n", jrp->ridx); + return -EIO; + } + + return 0; +} + +/* + * Flush the job ring, so the jobs running will be stopped, jobs queued will be + * invalidated and the CAAM will no longer fetch fron input ring. + * + * Must be called with itr disabled + */ +static int caam_jr_flush(struct device *dev) +{ + return caam_jr_stop_processing(dev, JRCR_RESET); +} + +/* The resume can be used after a park or a flush if CAAM has not been reset */ +static int caam_jr_restart_processing(struct device *dev) +{ + struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); + u32 halt_status = rd_reg32(&jrp->rregs->jrintstatus) & + JRINT_ERR_HALT_MASK; + + /* Check that the flush/park is completed */ + if (halt_status != JRINT_ERR_HALT_COMPLETE) + return -1; + + /* Resume processing of jobs */ + clrsetbits_32(&jrp->rregs->jrintstatus, 0, JRINT_ERR_HALT_COMPLETE); + + return 0; +} + +static int caam_reset_hw_jr(struct device *dev) +{ + struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); + unsigned int timeout = 100000; + int err; + /* + * mask interrupts since we are going to poll + * for reset completion status + */ + clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JRCFG_IMSK); + err = caam_jr_flush(dev); + if (err) + return err; + + /* initiate reset */ + wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET); + while ((rd_reg32(&jrp->rregs->jrcommand) & JRCR_RESET) && --timeout) + cpu_relax(); + + if (timeout == 0) { + dev_err(dev, "failed to reset job ring %d\n", jrp->ridx); + return -EIO; + } + + /* unmask interrupts */ + clrsetbits_32(&jrp->rregs->rconfig_lo, JRCFG_IMSK, 0); + + return 0; +} + +/* + * Shutdown JobR independent of platform property code + */ +static int caam_jr_shutdown(struct device *dev) +{ + struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); + int ret; + + ret = caam_reset_hw_jr(dev); + + tasklet_kill(&jrp->irqtask); + + return ret; +} + +static void caam_jr_remove(struct platform_device *pdev) +{ + int ret; + struct device *jrdev; + struct caam_drv_private_jr *jrpriv; + + jrdev = &pdev->dev; + jrpriv = dev_get_drvdata(jrdev); + + if (jrpriv->hwrng) + caam_rng_exit(jrdev->parent); + + /* + * If a job ring is still allocated there is trouble ahead. Once + * caam_jr_remove() returned, jrpriv will be freed and the registers + * will get unmapped. So any user of such a job ring will probably + * crash. + */ + if (atomic_read(&jrpriv->tfm_count)) { + dev_alert(jrdev, "Device is busy; consumers might start to crash\n"); + return; + } + + /* Unregister JR-based RNG & crypto algorithms */ + unregister_algs(); + + /* Remove the node from Physical JobR list maintained by driver */ + spin_lock(&driver_data.jr_alloc_lock); + list_del(&jrpriv->list_node); + spin_unlock(&driver_data.jr_alloc_lock); + + /* Release ring */ + ret = caam_jr_shutdown(jrdev); + if (ret) + dev_err(jrdev, "Failed to shut down job ring\n"); +} + +/* Main per-ring interrupt handler */ +static irqreturn_t caam_jr_interrupt(int irq, void *st_dev) +{ + struct device *dev = st_dev; + struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); + u32 irqstate; + + /* + * Check the output ring for ready responses, kick + * tasklet if jobs done. + */ + irqstate = rd_reg32(&jrp->rregs->jrintstatus); + if (!(irqstate & JRINT_JR_INT)) + return IRQ_NONE; + + /* + * If JobR error, we got more development work to do + * Flag a bug now, but we really need to shut down and + * restart the queue (and fix code). + */ + if (irqstate & JRINT_JR_ERROR) { + dev_err(dev, "job ring error: irqstate: %08x\n", irqstate); + BUG(); + } + + /* mask valid interrupts */ + clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JRCFG_IMSK); + + /* Have valid interrupt at this point, just ACK and trigger */ + wr_reg32(&jrp->rregs->jrintstatus, irqstate); + + preempt_disable(); + tasklet_schedule(&jrp->irqtask); + preempt_enable(); + + return IRQ_HANDLED; +} + +/* Deferred service handler, run as interrupt-fired tasklet */ +static void caam_jr_dequeue(unsigned long devarg) +{ + int hw_idx, sw_idx, i, head, tail; + struct caam_jr_dequeue_params *params = (void *)devarg; + struct device *dev = params->dev; + struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); + void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg); + u32 *userdesc, userstatus; + void *userarg; + u32 outring_used = 0; + + while (outring_used || + (outring_used = rd_reg32(&jrp->rregs->outring_used))) { + + head = READ_ONCE(jrp->head); + + sw_idx = tail = jrp->tail; + hw_idx = jrp->out_ring_read_index; + + for (i = 0; CIRC_CNT(head, tail + i, JOBR_DEPTH) >= 1; i++) { + sw_idx = (tail + i) & (JOBR_DEPTH - 1); + + if (jr_outentry_desc(jrp->outring, hw_idx) == + caam_dma_to_cpu(jrp->entinfo[sw_idx].desc_addr_dma)) + break; /* found */ + } + /* we should never fail to find a matching descriptor */ + BUG_ON(CIRC_CNT(head, tail + i, JOBR_DEPTH) <= 0); + + /* Unmap just-run descriptor so we can post-process */ + dma_unmap_single(dev, + caam_dma_to_cpu(jr_outentry_desc(jrp->outring, + hw_idx)), + jrp->entinfo[sw_idx].desc_size, + DMA_TO_DEVICE); + + /* mark completed, avoid matching on a recycled desc addr */ + jrp->entinfo[sw_idx].desc_addr_dma = 0; + + /* Stash callback params */ + usercall = jrp->entinfo[sw_idx].callbk; + userarg = jrp->entinfo[sw_idx].cbkarg; + userdesc = jrp->entinfo[sw_idx].desc_addr_virt; + userstatus = caam32_to_cpu(jr_outentry_jrstatus(jrp->outring, + hw_idx)); + + /* + * Make sure all information from the job has been obtained + * before telling CAAM that the job has been removed from the + * output ring. + */ + mb(); + + /* set done */ + wr_reg32(&jrp->rregs->outring_rmvd, 1); + + jrp->out_ring_read_index = (jrp->out_ring_read_index + 1) & + (JOBR_DEPTH - 1); + + /* + * if this job completed out-of-order, do not increment + * the tail. Otherwise, increment tail by 1 plus the + * number of subsequent jobs already completed out-of-order + */ + if (sw_idx == tail) { + do { + tail = (tail + 1) & (JOBR_DEPTH - 1); + } while (CIRC_CNT(head, tail, JOBR_DEPTH) >= 1 && + jrp->entinfo[tail].desc_addr_dma == 0); + + jrp->tail = tail; + } + + /* Finally, execute user's callback */ + usercall(dev, userdesc, userstatus, userarg); + outring_used--; + } + + if (params->enable_itr) + /* reenable / unmask IRQs */ + clrsetbits_32(&jrp->rregs->rconfig_lo, JRCFG_IMSK, 0); +} + +/** + * caam_jr_alloc() - Alloc a job ring for someone to use as needed. + * + * returns : pointer to the newly allocated physical + * JobR dev can be written to if successful. + **/ +struct device *caam_jr_alloc(void) +{ + struct caam_drv_private_jr *jrpriv, *min_jrpriv = NULL; + struct device *dev = ERR_PTR(-ENODEV); + int min_tfm_cnt = INT_MAX; + int tfm_cnt; + + spin_lock(&driver_data.jr_alloc_lock); + + if (list_empty(&driver_data.jr_list)) { + spin_unlock(&driver_data.jr_alloc_lock); + return ERR_PTR(-ENODEV); + } + + list_for_each_entry(jrpriv, &driver_data.jr_list, list_node) { + tfm_cnt = atomic_read(&jrpriv->tfm_count); + if (tfm_cnt < min_tfm_cnt) { + min_tfm_cnt = tfm_cnt; + min_jrpriv = jrpriv; + } + if (!min_tfm_cnt) + break; + } + + if (min_jrpriv) { + atomic_inc(&min_jrpriv->tfm_count); + dev = min_jrpriv->dev; + } + spin_unlock(&driver_data.jr_alloc_lock); + + return dev; +} +EXPORT_SYMBOL(caam_jr_alloc); + +/** + * caam_jr_free() - Free the Job Ring + * @rdev: points to the dev that identifies the Job ring to + * be released. + **/ +void caam_jr_free(struct device *rdev) +{ + struct caam_drv_private_jr *jrpriv = dev_get_drvdata(rdev); + + atomic_dec(&jrpriv->tfm_count); +} +EXPORT_SYMBOL(caam_jr_free); + +/** + * caam_jr_enqueue() - Enqueue a job descriptor head. Returns -EINPROGRESS + * if OK, -ENOSPC if the queue is full, -EIO if it cannot map the caller's + * descriptor. + * @dev: struct device of the job ring to be used + * @desc: points to a job descriptor that execute our request. All + * descriptors (and all referenced data) must be in a DMAable + * region, and all data references must be physical addresses + * accessible to CAAM (i.e. within a PAMU window granted + * to it). + * @cbk: pointer to a callback function to be invoked upon completion + * of this request. This has the form: + * callback(struct device *dev, u32 *desc, u32 stat, void *arg) + * where: + * dev: contains the job ring device that processed this + * response. + * desc: descriptor that initiated the request, same as + * "desc" being argued to caam_jr_enqueue(). + * status: untranslated status received from CAAM. See the + * reference manual for a detailed description of + * error meaning, or see the JRSTA definitions in the + * register header file + * areq: optional pointer to an argument passed with the + * original request + * @areq: optional pointer to a user argument for use at callback + * time. + **/ +int caam_jr_enqueue(struct device *dev, u32 *desc, + void (*cbk)(struct device *dev, u32 *desc, + u32 status, void *areq), + void *areq) +{ + struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); + struct caam_jrentry_info *head_entry; + int head, tail, desc_size; + dma_addr_t desc_dma; + + desc_size = (caam32_to_cpu(*desc) & HDR_JD_LENGTH_MASK) * sizeof(u32); + desc_dma = dma_map_single(dev, desc, desc_size, DMA_TO_DEVICE); + if (dma_mapping_error(dev, desc_dma)) { + dev_err(dev, "caam_jr_enqueue(): can't map jobdesc\n"); + return -EIO; + } + + spin_lock_bh(&jrp->inplock); + + head = jrp->head; + tail = READ_ONCE(jrp->tail); + + if (!jrp->inpring_avail || + CIRC_SPACE(head, tail, JOBR_DEPTH) <= 0) { + spin_unlock_bh(&jrp->inplock); + dma_unmap_single(dev, desc_dma, desc_size, DMA_TO_DEVICE); + return -ENOSPC; + } + + head_entry = &jrp->entinfo[head]; + head_entry->desc_addr_virt = desc; + head_entry->desc_size = desc_size; + head_entry->callbk = (void *)cbk; + head_entry->cbkarg = areq; + head_entry->desc_addr_dma = desc_dma; + + jr_inpentry_set(jrp->inpring, head, cpu_to_caam_dma(desc_dma)); + + /* + * Guarantee that the descriptor's DMA address has been written to + * the next slot in the ring before the write index is updated, since + * other cores may update this index independently. + * + * Under heavy DDR load, smp_wmb() or dma_wmb() fail to make the input + * ring be updated before the CAAM starts reading it. So, CAAM will + * process, again, an old descriptor address and will put it in the + * output ring. This will make caam_jr_dequeue() to fail, since this + * old descriptor is not in the software ring. + * To fix this, use wmb() which works on the full system instead of + * inner/outer shareable domains. + */ + wmb(); + + jrp->head = (head + 1) & (JOBR_DEPTH - 1); + + /* + * Ensure that all job information has been written before + * notifying CAAM that a new job was added to the input ring + * using a memory barrier. The wr_reg32() uses api iowrite32() + * to do the register write. iowrite32() issues a memory barrier + * before the write operation. + */ + + wr_reg32(&jrp->rregs->inpring_jobadd, 1); + + jrp->inpring_avail--; + if (!jrp->inpring_avail) + jrp->inpring_avail = rd_reg32(&jrp->rregs->inpring_avail); + + spin_unlock_bh(&jrp->inplock); + + return -EINPROGRESS; +} +EXPORT_SYMBOL(caam_jr_enqueue); + +static void caam_jr_init_hw(struct device *dev, dma_addr_t inpbusaddr, + dma_addr_t outbusaddr) +{ + struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); + + wr_reg64(&jrp->rregs->inpring_base, inpbusaddr); + wr_reg64(&jrp->rregs->outring_base, outbusaddr); + wr_reg32(&jrp->rregs->inpring_size, JOBR_DEPTH); + wr_reg32(&jrp->rregs->outring_size, JOBR_DEPTH); + + /* Select interrupt coalescing parameters */ + clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JOBR_INTC | + (JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) | + (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT)); +} + +static void caam_jr_reset_index(struct caam_drv_private_jr *jrp) +{ + jrp->out_ring_read_index = 0; + jrp->head = 0; + jrp->tail = 0; +} + +/* + * Init JobR independent of platform property detection + */ +static int caam_jr_init(struct device *dev) +{ + struct caam_drv_private_jr *jrp; + dma_addr_t inpbusaddr, outbusaddr; + int i, error; + + jrp = dev_get_drvdata(dev); + + error = caam_reset_hw_jr(dev); + if (error) + return error; + + jrp->inpring = dmam_alloc_coherent(dev, SIZEOF_JR_INPENTRY * + JOBR_DEPTH, &inpbusaddr, + GFP_KERNEL); + if (!jrp->inpring) + return -ENOMEM; + + jrp->outring = dmam_alloc_coherent(dev, SIZEOF_JR_OUTENTRY * + JOBR_DEPTH, &outbusaddr, + GFP_KERNEL); + if (!jrp->outring) + return -ENOMEM; + + jrp->entinfo = devm_kcalloc(dev, JOBR_DEPTH, sizeof(*jrp->entinfo), + GFP_KERNEL); + if (!jrp->entinfo) + return -ENOMEM; + + for (i = 0; i < JOBR_DEPTH; i++) + jrp->entinfo[i].desc_addr_dma = !0; + + /* Setup rings */ + caam_jr_reset_index(jrp); + jrp->inpring_avail = JOBR_DEPTH; + caam_jr_init_hw(dev, inpbusaddr, outbusaddr); + + spin_lock_init(&jrp->inplock); + + jrp->tasklet_params.dev = dev; + jrp->tasklet_params.enable_itr = 1; + tasklet_init(&jrp->irqtask, caam_jr_dequeue, + (unsigned long)&jrp->tasklet_params); + + /* Connect job ring interrupt handler. */ + error = devm_request_irq(dev, jrp->irq, caam_jr_interrupt, IRQF_SHARED, + dev_name(dev), dev); + if (error) { + dev_err(dev, "can't connect JobR %d interrupt (%d)\n", + jrp->ridx, jrp->irq); + tasklet_kill(&jrp->irqtask); + } + + return error; +} + +/* + * Probe routine for each detected JobR subsystem. + */ +static int caam_jr_probe(struct platform_device *pdev) +{ + struct device *jrdev; + struct caam_drv_private_jr *jrpriv; + static int total_jobrs; + void __iomem *ctrl; + struct resource *r; + int error; + int irq; + + /* + * The job rings live inside the register window of their parent + * fsl,sec-v4.0 node, which caam_probe() already reserves (and maps) + * via devm_of_iomap(). A requested region that overlaps that + * reservation, e.g. from devm_platform_ioremap_resource(), would + * therefore fail with -EBUSY, so map the registers without claiming + * the region here. + */ + r = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!r) { + dev_err(&pdev->dev, "platform_get_resource() failed\n"); + return -EINVAL; + } + + ctrl = devm_ioremap(&pdev->dev, r->start, resource_size(r)); + if (!ctrl) { + dev_err(&pdev->dev, "devm_ioremap() failed\n"); + return -ENOMEM; + } + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + jrdev = &pdev->dev; + jrpriv = devm_kzalloc(jrdev, sizeof(*jrpriv), GFP_KERNEL); + if (!jrpriv) + return -ENOMEM; + + dev_set_drvdata(jrdev, jrpriv); + + /* save ring identity relative to detection */ + jrpriv->ridx = total_jobrs++; + + jrpriv->rregs = ctrl; + + error = dma_set_mask_and_coherent(jrdev, caam_get_dma_mask(jrdev)); + if (error) { + dev_err(jrdev, "dma_set_mask_and_coherent failed (%d)\n", + error); + return error; + } + + /* Initialize crypto engine */ + jrpriv->engine = crypto_engine_alloc_init_and_set(jrdev, true, false, + CRYPTO_ENGINE_MAX_QLEN); + if (!jrpriv->engine) { + dev_err(jrdev, "Could not init crypto-engine\n"); + return -ENOMEM; + } + + error = devm_add_action_or_reset(jrdev, caam_jr_crypto_engine_exit, + jrdev); + if (error) + return error; + + /* Start crypto engine */ + error = crypto_engine_start(jrpriv->engine); + if (error) { + dev_err(jrdev, "Could not start crypto-engine\n"); + return error; + } + + /* Identify the interrupt */ + jrpriv->irq = irq; + + /* Now do the platform independent part */ + error = caam_jr_init(jrdev); /* now turn on hardware */ + if (error) + return error; + + jrpriv->dev = jrdev; + spin_lock(&driver_data.jr_alloc_lock); + list_add_tail(&jrpriv->list_node, &driver_data.jr_list); + spin_unlock(&driver_data.jr_alloc_lock); + + atomic_set(&jrpriv->tfm_count, 0); + + device_init_wakeup(&pdev->dev, 1); + device_set_wakeup_enable(&pdev->dev, false); + + register_algs(jrpriv, jrdev->parent); + + return 0; +} + +static void caam_jr_get_hw_state(struct device *dev) +{ + struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); + + jrp->state.inpbusaddr = rd_reg64(&jrp->rregs->inpring_base); + jrp->state.outbusaddr = rd_reg64(&jrp->rregs->outring_base); +} + +static int caam_jr_suspend(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct caam_drv_private_jr *jrpriv = platform_get_drvdata(pdev); + struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev->parent); + struct caam_jr_dequeue_params suspend_params = { + .dev = dev, + .enable_itr = 0, + }; + + /* Remove the node from Physical JobR list maintained by driver */ + spin_lock(&driver_data.jr_alloc_lock); + list_del(&jrpriv->list_node); + spin_unlock(&driver_data.jr_alloc_lock); + + if (jrpriv->hwrng) + caam_rng_exit(dev->parent); + + if (ctrlpriv->caam_off_during_pm) { + int err; + + tasklet_disable(&jrpriv->irqtask); + + /* mask itr to call flush */ + clrsetbits_32(&jrpriv->rregs->rconfig_lo, 0, JRCFG_IMSK); + + /* Invalid job in process */ + err = caam_jr_flush(dev); + if (err) { + dev_err(dev, "Failed to flush\n"); + return err; + } + + /* Dequeing jobs flushed */ + caam_jr_dequeue((unsigned long)&suspend_params); + + /* Save state */ + caam_jr_get_hw_state(dev); + } else if (device_may_wakeup(&pdev->dev)) { + enable_irq_wake(jrpriv->irq); + } + + return 0; +} + +static int caam_jr_resume(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct caam_drv_private_jr *jrpriv = platform_get_drvdata(pdev); + struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev->parent); + + if (ctrlpriv->caam_off_during_pm) { + u64 inp_addr; + int err; + + /* + * Check if the CAAM has been resetted checking the address of + * the input ring + */ + inp_addr = rd_reg64(&jrpriv->rregs->inpring_base); + if (inp_addr != 0) { + /* JR still has some configuration */ + if (inp_addr == jrpriv->state.inpbusaddr) { + /* JR has not been resetted */ + err = caam_jr_restart_processing(dev); + if (err) { + dev_err(dev, + "Restart processing failed\n"); + return err; + } + + tasklet_enable(&jrpriv->irqtask); + + clrsetbits_32(&jrpriv->rregs->rconfig_lo, + JRCFG_IMSK, 0); + + goto add_jr; + } else if (ctrlpriv->optee_en) { + /* JR has been used by OPTEE, reset it */ + err = caam_reset_hw_jr(dev); + if (err) { + dev_err(dev, "Failed to reset JR\n"); + return err; + } + } else { + /* No explanation, return error */ + return -EIO; + } + } + + caam_jr_reset_index(jrpriv); + caam_jr_init_hw(dev, jrpriv->state.inpbusaddr, + jrpriv->state.outbusaddr); + + tasklet_enable(&jrpriv->irqtask); + } else if (device_may_wakeup(&pdev->dev)) { + disable_irq_wake(jrpriv->irq); + } + +add_jr: + spin_lock(&driver_data.jr_alloc_lock); + list_add_tail(&jrpriv->list_node, &driver_data.jr_list); + spin_unlock(&driver_data.jr_alloc_lock); + + if (jrpriv->hwrng) + jrpriv->hwrng = !caam_rng_init(dev->parent); + + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(caam_jr_pm_ops, caam_jr_suspend, caam_jr_resume); + +static const struct of_device_id caam_jr_match[] = { + { + .compatible = "fsl,sec-v4.0-job-ring", + }, + { + .compatible = "fsl,sec4.0-job-ring", + }, + {}, +}; +MODULE_DEVICE_TABLE(of, caam_jr_match); + +static struct platform_driver caam_jr_driver = { + .driver = { + .name = "caam_jr", + .of_match_table = caam_jr_match, + .pm = pm_ptr(&caam_jr_pm_ops), + }, + .probe = caam_jr_probe, + .remove = caam_jr_remove, + .shutdown = caam_jr_remove, +}; + +static int __init jr_driver_init(void) +{ + spin_lock_init(&driver_data.jr_alloc_lock); + INIT_LIST_HEAD(&driver_data.jr_list); + return platform_driver_register(&caam_jr_driver); +} + +static void __exit jr_driver_exit(void) +{ + platform_driver_unregister(&caam_jr_driver); +} + +module_init(jr_driver_init); +module_exit(jr_driver_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("FSL CAAM JR request backend"); +MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); diff --git a/drivers/crypto/caam/jr.h b/drivers/crypto/caam/jr.h new file mode 100644 index 000000000..eab611530 --- /dev/null +++ b/drivers/crypto/caam/jr.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * CAAM public-level include definitions for the JobR backend + * + * Copyright 2008-2011 Freescale Semiconductor, Inc. + */ + +#ifndef JR_H +#define JR_H + +/* Prototypes for backend-level services exposed to APIs */ +struct device *caam_jr_alloc(void); +void caam_jr_free(struct device *rdev); +int caam_jr_enqueue(struct device *dev, u32 *desc, + void (*cbk)(struct device *dev, u32 *desc, u32 status, + void *areq), + void *areq); + +#endif /* JR_H */ diff --git a/drivers/crypto/caam/key_gen.c b/drivers/crypto/caam/key_gen.c new file mode 100644 index 000000000..de2fcc387 --- /dev/null +++ b/drivers/crypto/caam/key_gen.c @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * CAAM/SEC 4.x functions for handling key-generation jobs + * + * Copyright 2008-2011 Freescale Semiconductor, Inc. + * + */ +#include "compat.h" +#include "jr.h" +#include "error.h" +#include "desc_constr.h" +#include "key_gen.h" + +void split_key_done(struct device *dev, u32 *desc, u32 err, + void *context) +{ + struct split_key_result *res = context; + int ecode = 0; + + dev_dbg(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); + + if (err) + ecode = caam_jr_strstatus(dev, err); + + res->err = ecode; + + complete(&res->completion); +} +EXPORT_SYMBOL(split_key_done); +/* +get a split ipad/opad key + +Split key generation----------------------------------------------- + +[00] 0xb0810008 jobdesc: stidx=1 share=never len=8 +[01] 0x04000014 key: class2->keyreg len=20 + @0xffe01000 +[03] 0x84410014 operation: cls2-op sha1 hmac init dec +[04] 0x24940000 fifold: class2 msgdata-last2 len=0 imm +[05] 0xa4000001 jump: class2 local all ->1 [06] +[06] 0x64260028 fifostr: class2 mdsplit-jdk len=40 + @0xffe04000 +*/ +int gen_split_key(struct device *jrdev, u8 *key_out, + struct alginfo * const adata, const u8 *key_in, u32 keylen, + int max_keylen) +{ + u32 *desc; + struct split_key_result result; + dma_addr_t dma_addr; + unsigned int local_max; + int ret = -ENOMEM; + + adata->keylen = split_key_len(adata->algtype & OP_ALG_ALGSEL_MASK); + adata->keylen_pad = split_key_pad_len(adata->algtype & + OP_ALG_ALGSEL_MASK); + local_max = max(keylen, adata->keylen_pad); + + dev_dbg(jrdev, "split keylen %d split keylen padded %d\n", + adata->keylen, adata->keylen_pad); + print_hex_dump_devel("ctx.key@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1); + + if (local_max > max_keylen) + return -EINVAL; + + desc = kmalloc(CAAM_CMD_SZ * 6 + CAAM_PTR_SZ * 2, GFP_KERNEL); + if (!desc) { + dev_err(jrdev, "unable to allocate key input memory\n"); + return ret; + } + + memcpy(key_out, key_in, keylen); + + dma_addr = dma_map_single(jrdev, key_out, local_max, DMA_BIDIRECTIONAL); + if (dma_mapping_error(jrdev, dma_addr)) { + dev_err(jrdev, "unable to map key memory\n"); + goto out_free; + } + + init_job_desc(desc, 0); + append_key(desc, dma_addr, keylen, CLASS_2 | KEY_DEST_CLASS_REG); + + /* Sets MDHA up into an HMAC-INIT */ + append_operation(desc, (adata->algtype & OP_ALG_ALGSEL_MASK) | + OP_ALG_AAI_HMAC | OP_TYPE_CLASS2_ALG | OP_ALG_DECRYPT | + OP_ALG_AS_INIT); + + /* + * do a FIFO_LOAD of zero, this will trigger the internal key expansion + * into both pads inside MDHA + */ + append_fifo_load_as_imm(desc, NULL, 0, LDST_CLASS_2_CCB | + FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST2); + + /* + * FIFO_STORE with the explicit split-key content store + * (0x26 output type) + */ + append_fifo_store(desc, dma_addr, adata->keylen, + LDST_CLASS_2_CCB | FIFOST_TYPE_SPLIT_KEK); + + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); + + result.err = 0; + init_completion(&result.completion); + + ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result); + if (ret == -EINPROGRESS) { + /* in progress */ + wait_for_completion(&result.completion); + ret = result.err; + + print_hex_dump_devel("ctx.key@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key_out, + adata->keylen_pad, 1); + } + + dma_unmap_single(jrdev, dma_addr, local_max, DMA_BIDIRECTIONAL); +out_free: + kfree(desc); + return ret; +} +EXPORT_SYMBOL(gen_split_key); diff --git a/drivers/crypto/caam/key_gen.h b/drivers/crypto/caam/key_gen.h new file mode 100644 index 000000000..818f78f6f --- /dev/null +++ b/drivers/crypto/caam/key_gen.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * CAAM/SEC 4.x definitions for handling key-generation jobs + * + * Copyright 2008-2011 Freescale Semiconductor, Inc. + * + */ + +/** + * split_key_len - Compute MDHA split key length for a given algorithm + * @hash: Hashing algorithm selection, one of OP_ALG_ALGSEL_* - MD5, SHA1, + * SHA224, SHA384, SHA512. + * + * Return: MDHA split key length + */ +static inline u32 split_key_len(u32 hash) +{ + /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ + static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; + u32 idx; + + idx = (hash & OP_ALG_ALGSEL_SUBMASK) >> OP_ALG_ALGSEL_SHIFT; + + return (u32)(mdpadlen[idx] * 2); +} + +/** + * split_key_pad_len - Compute MDHA split key pad length for a given algorithm + * @hash: Hashing algorithm selection, one of OP_ALG_ALGSEL_* - MD5, SHA1, + * SHA224, SHA384, SHA512. + * + * Return: MDHA split key pad length + */ +static inline u32 split_key_pad_len(u32 hash) +{ + return ALIGN(split_key_len(hash), 16); +} + +struct split_key_result { + struct completion completion; + int err; +}; + +void split_key_done(struct device *dev, u32 *desc, u32 err, void *context); + +int gen_split_key(struct device *jrdev, u8 *key_out, + struct alginfo * const adata, const u8 *key_in, u32 keylen, + int max_keylen); diff --git a/drivers/crypto/caam/pdb.h b/drivers/crypto/caam/pdb.h new file mode 100644 index 000000000..4b1bcf53f --- /dev/null +++ b/drivers/crypto/caam/pdb.h @@ -0,0 +1,599 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * CAAM Protocol Data Block (PDB) definition header file + * + * Copyright 2008-2016 Freescale Semiconductor, Inc. + * + */ + +#ifndef CAAM_PDB_H +#define CAAM_PDB_H +#include "compat.h" + +/* + * PDB- IPSec ESP Header Modification Options + */ +#define PDBHMO_ESP_DECAP_SHIFT 28 +#define PDBHMO_ESP_ENCAP_SHIFT 28 +/* + * Encap and Decap - Decrement TTL (Hop Limit) - Based on the value of the + * Options Byte IP version (IPvsn) field: + * if IPv4, decrement the inner IP header TTL field (byte 8); + * if IPv6 decrement the inner IP header Hop Limit field (byte 7). +*/ +#define PDBHMO_ESP_DECAP_DEC_TTL (0x02 << PDBHMO_ESP_DECAP_SHIFT) +#define PDBHMO_ESP_ENCAP_DEC_TTL (0x02 << PDBHMO_ESP_ENCAP_SHIFT) +/* + * Decap - DiffServ Copy - Copy the IPv4 TOS or IPv6 Traffic Class byte + * from the outer IP header to the inner IP header. + */ +#define PDBHMO_ESP_DIFFSERV (0x01 << PDBHMO_ESP_DECAP_SHIFT) +/* + * Encap- Copy DF bit -if an IPv4 tunnel mode outer IP header is coming from + * the PDB, copy the DF bit from the inner IP header to the outer IP header. + */ +#define PDBHMO_ESP_DFBIT (0x04 << PDBHMO_ESP_ENCAP_SHIFT) + +#define PDBNH_ESP_ENCAP_SHIFT 16 +#define PDBNH_ESP_ENCAP_MASK (0xff << PDBNH_ESP_ENCAP_SHIFT) + +#define PDBHDRLEN_ESP_DECAP_SHIFT 16 +#define PDBHDRLEN_MASK (0x0fff << PDBHDRLEN_ESP_DECAP_SHIFT) + +#define PDB_NH_OFFSET_SHIFT 8 +#define PDB_NH_OFFSET_MASK (0xff << PDB_NH_OFFSET_SHIFT) + +/* + * PDB - IPSec ESP Encap/Decap Options + */ +#define PDBOPTS_ESP_ARSNONE 0x00 /* no antireplay window */ +#define PDBOPTS_ESP_ARS32 0x40 /* 32-entry antireplay window */ +#define PDBOPTS_ESP_ARS128 0x80 /* 128-entry antireplay window */ +#define PDBOPTS_ESP_ARS64 0xc0 /* 64-entry antireplay window */ +#define PDBOPTS_ESP_ARS_MASK 0xc0 /* antireplay window mask */ +#define PDBOPTS_ESP_IVSRC 0x20 /* IV comes from internal random gen */ +#define PDBOPTS_ESP_ESN 0x10 /* extended sequence included */ +#define PDBOPTS_ESP_OUTFMT 0x08 /* output only decapsulation (decap) */ +#define PDBOPTS_ESP_IPHDRSRC 0x08 /* IP header comes from PDB (encap) */ +#define PDBOPTS_ESP_INCIPHDR 0x04 /* Prepend IP header to output frame */ +#define PDBOPTS_ESP_IPVSN 0x02 /* process IPv6 header */ +#define PDBOPTS_ESP_AOFL 0x04 /* adjust out frame len (decap, SEC>=5.3)*/ +#define PDBOPTS_ESP_TUNNEL 0x01 /* tunnel mode next-header byte */ +#define PDBOPTS_ESP_IPV6 0x02 /* ip header version is V6 */ +#define PDBOPTS_ESP_DIFFSERV 0x40 /* copy TOS/TC from inner iphdr */ +#define PDBOPTS_ESP_UPDATE_CSUM 0x80 /* encap-update ip header checksum */ +#define PDBOPTS_ESP_VERIFY_CSUM 0x20 /* decap-validate ip header checksum */ + +/* + * General IPSec encap/decap PDB definitions + */ + +/** + * ipsec_encap_cbc - PDB part for IPsec CBC encapsulation + * @iv: 16-byte array initialization vector + */ +struct ipsec_encap_cbc { + u8 iv[16]; +}; + +/** + * ipsec_encap_ctr - PDB part for IPsec CTR encapsulation + * @ctr_nonce: 4-byte array nonce + * @ctr_initial: initial count constant + * @iv: initialization vector + */ +struct ipsec_encap_ctr { + u8 ctr_nonce[4]; + u32 ctr_initial; + u64 iv; +}; + +/** + * ipsec_encap_ccm - PDB part for IPsec CCM encapsulation + * @salt: 3-byte array salt (lower 24 bits) + * @ccm_opt: CCM algorithm options - MSB-LSB description: + * b0_flags (8b) - CCM B0; use 0x5B for 8-byte ICV, 0x6B for 12-byte ICV, + * 0x7B for 16-byte ICV (cf. RFC4309, RFC3610) + * ctr_flags (8b) - counter flags; constant equal to 0x3 + * ctr_initial (16b) - initial count constant + * @iv: initialization vector + */ +struct ipsec_encap_ccm { + u8 salt[4]; + u32 ccm_opt; + u64 iv; +}; + +/** + * ipsec_encap_gcm - PDB part for IPsec GCM encapsulation + * @salt: 3-byte array salt (lower 24 bits) + * @rsvd: reserved, do not use + * @iv: initialization vector + */ +struct ipsec_encap_gcm { + u8 salt[4]; + u32 rsvd1; + u64 iv; +}; + +/** + * ipsec_encap_pdb - PDB for IPsec encapsulation + * @options: MSB-LSB description + * hmo (header manipulation options) - 4b + * reserved - 4b + * next header - 8b + * next header offset - 8b + * option flags (depend on selected algorithm) - 8b + * @seq_num_ext_hi: (optional) IPsec Extended Sequence Number (ESN) + * @seq_num: IPsec sequence number + * @spi: IPsec SPI (Security Parameters Index) + * @ip_hdr_len: optional IP Header length (in bytes) + * reserved - 16b + * Opt. IP Hdr Len - 16b + * @ip_hdr: optional IP Header content + */ +struct ipsec_encap_pdb { + u32 options; + u32 seq_num_ext_hi; + u32 seq_num; + union { + struct ipsec_encap_cbc cbc; + struct ipsec_encap_ctr ctr; + struct ipsec_encap_ccm ccm; + struct ipsec_encap_gcm gcm; + }; + u32 spi; + u32 ip_hdr_len; + u32 ip_hdr[]; +}; + +/** + * ipsec_decap_cbc - PDB part for IPsec CBC decapsulation + * @rsvd: reserved, do not use + */ +struct ipsec_decap_cbc { + u32 rsvd[2]; +}; + +/** + * ipsec_decap_ctr - PDB part for IPsec CTR decapsulation + * @ctr_nonce: 4-byte array nonce + * @ctr_initial: initial count constant + */ +struct ipsec_decap_ctr { + u8 ctr_nonce[4]; + u32 ctr_initial; +}; + +/** + * ipsec_decap_ccm - PDB part for IPsec CCM decapsulation + * @salt: 3-byte salt (lower 24 bits) + * @ccm_opt: CCM algorithm options - MSB-LSB description: + * b0_flags (8b) - CCM B0; use 0x5B for 8-byte ICV, 0x6B for 12-byte ICV, + * 0x7B for 16-byte ICV (cf. RFC4309, RFC3610) + * ctr_flags (8b) - counter flags; constant equal to 0x3 + * ctr_initial (16b) - initial count constant + */ +struct ipsec_decap_ccm { + u8 salt[4]; + u32 ccm_opt; +}; + +/** + * ipsec_decap_gcm - PDB part for IPsec GCN decapsulation + * @salt: 4-byte salt + * @rsvd: reserved, do not use + */ +struct ipsec_decap_gcm { + u8 salt[4]; + u32 resvd; +}; + +/** + * ipsec_decap_pdb - PDB for IPsec decapsulation + * @options: MSB-LSB description + * hmo (header manipulation options) - 4b + * IP header length - 12b + * next header offset - 8b + * option flags (depend on selected algorithm) - 8b + * @seq_num_ext_hi: (optional) IPsec Extended Sequence Number (ESN) + * @seq_num: IPsec sequence number + * @anti_replay: Anti-replay window; size depends on ARS (option flags) + */ +struct ipsec_decap_pdb { + u32 options; + union { + struct ipsec_decap_cbc cbc; + struct ipsec_decap_ctr ctr; + struct ipsec_decap_ccm ccm; + struct ipsec_decap_gcm gcm; + }; + u32 seq_num_ext_hi; + u32 seq_num; + __be32 anti_replay[4]; +}; + +/* + * IPSec ESP Datapath Protocol Override Register (DPOVRD) + */ +struct ipsec_deco_dpovrd { +#define IPSEC_ENCAP_DECO_DPOVRD_USE 0x80 + u8 ovrd_ecn; + u8 ip_hdr_len; + u8 nh_offset; + u8 next_header; /* reserved if decap */ +}; + +/* + * IEEE 802.11i WiFi Protocol Data Block + */ +#define WIFI_PDBOPTS_FCS 0x01 +#define WIFI_PDBOPTS_AR 0x40 + +struct wifi_encap_pdb { + u16 mac_hdr_len; + u8 rsvd; + u8 options; + u8 iv_flags; + u8 pri; + u16 pn1; + u32 pn2; + u16 frm_ctrl_mask; + u16 seq_ctrl_mask; + u8 rsvd1[2]; + u8 cnst; + u8 key_id; + u8 ctr_flags; + u8 rsvd2; + u16 ctr_init; +}; + +struct wifi_decap_pdb { + u16 mac_hdr_len; + u8 rsvd; + u8 options; + u8 iv_flags; + u8 pri; + u16 pn1; + u32 pn2; + u16 frm_ctrl_mask; + u16 seq_ctrl_mask; + u8 rsvd1[4]; + u8 ctr_flags; + u8 rsvd2; + u16 ctr_init; +}; + +/* + * IEEE 802.16 WiMAX Protocol Data Block + */ +#define WIMAX_PDBOPTS_FCS 0x01 +#define WIMAX_PDBOPTS_AR 0x40 /* decap only */ + +struct wimax_encap_pdb { + u8 rsvd[3]; + u8 options; + u32 nonce; + u8 b0_flags; + u8 ctr_flags; + u16 ctr_init; + /* begin DECO writeback region */ + u32 pn; + /* end DECO writeback region */ +}; + +struct wimax_decap_pdb { + u8 rsvd[3]; + u8 options; + u32 nonce; + u8 iv_flags; + u8 ctr_flags; + u16 ctr_init; + /* begin DECO writeback region */ + u32 pn; + u8 rsvd1[2]; + u16 antireplay_len; + u64 antireplay_scorecard; + /* end DECO writeback region */ +}; + +/* + * IEEE 801.AE MacSEC Protocol Data Block + */ +#define MACSEC_PDBOPTS_FCS 0x01 +#define MACSEC_PDBOPTS_AR 0x40 /* used in decap only */ + +struct macsec_encap_pdb { + u16 aad_len; + u8 rsvd; + u8 options; + u64 sci; + u16 ethertype; + u8 tci_an; + u8 rsvd1; + /* begin DECO writeback region */ + u32 pn; + /* end DECO writeback region */ +}; + +struct macsec_decap_pdb { + u16 aad_len; + u8 rsvd; + u8 options; + u64 sci; + u8 rsvd1[3]; + /* begin DECO writeback region */ + u8 antireplay_len; + u32 pn; + u64 antireplay_scorecard; + /* end DECO writeback region */ +}; + +/* + * SSL/TLS/DTLS Protocol Data Blocks + */ + +#define TLS_PDBOPTS_ARS32 0x40 +#define TLS_PDBOPTS_ARS64 0xc0 +#define TLS_PDBOPTS_OUTFMT 0x08 +#define TLS_PDBOPTS_IV_WRTBK 0x02 /* 1.1/1.2/DTLS only */ +#define TLS_PDBOPTS_EXP_RND_IV 0x01 /* 1.1/1.2/DTLS only */ + +struct tls_block_encap_pdb { + u8 type; + u8 version[2]; + u8 options; + u64 seq_num; + u32 iv[4]; +}; + +struct tls_stream_encap_pdb { + u8 type; + u8 version[2]; + u8 options; + u64 seq_num; + u8 i; + u8 j; + u8 rsvd1[2]; +}; + +struct dtls_block_encap_pdb { + u8 type; + u8 version[2]; + u8 options; + u16 epoch; + u16 seq_num[3]; + u32 iv[4]; +}; + +struct tls_block_decap_pdb { + u8 rsvd[3]; + u8 options; + u64 seq_num; + u32 iv[4]; +}; + +struct tls_stream_decap_pdb { + u8 rsvd[3]; + u8 options; + u64 seq_num; + u8 i; + u8 j; + u8 rsvd1[2]; +}; + +struct dtls_block_decap_pdb { + u8 rsvd[3]; + u8 options; + u16 epoch; + u16 seq_num[3]; + u32 iv[4]; + u64 antireplay_scorecard; +}; + +/* + * SRTP Protocol Data Blocks + */ +#define SRTP_PDBOPTS_MKI 0x08 +#define SRTP_PDBOPTS_AR 0x40 + +struct srtp_encap_pdb { + u8 x_len; + u8 mki_len; + u8 n_tag; + u8 options; + u32 cnst0; + u8 rsvd[2]; + u16 cnst1; + u16 salt[7]; + u16 cnst2; + u32 rsvd1; + u32 roc; + u32 opt_mki; +}; + +struct srtp_decap_pdb { + u8 x_len; + u8 mki_len; + u8 n_tag; + u8 options; + u32 cnst0; + u8 rsvd[2]; + u16 cnst1; + u16 salt[7]; + u16 cnst2; + u16 rsvd1; + u16 seq_num; + u32 roc; + u64 antireplay_scorecard; +}; + +/* + * DSA/ECDSA Protocol Data Blocks + * Two of these exist: DSA-SIGN, and DSA-VERIFY. They are similar + * except for the treatment of "w" for verify, "s" for sign, + * and the placement of "a,b". + */ +#define DSA_PDB_SGF_SHIFT 24 +#define DSA_PDB_SGF_MASK (0xff << DSA_PDB_SGF_SHIFT) +#define DSA_PDB_SGF_Q (0x80 << DSA_PDB_SGF_SHIFT) +#define DSA_PDB_SGF_R (0x40 << DSA_PDB_SGF_SHIFT) +#define DSA_PDB_SGF_G (0x20 << DSA_PDB_SGF_SHIFT) +#define DSA_PDB_SGF_W (0x10 << DSA_PDB_SGF_SHIFT) +#define DSA_PDB_SGF_S (0x10 << DSA_PDB_SGF_SHIFT) +#define DSA_PDB_SGF_F (0x08 << DSA_PDB_SGF_SHIFT) +#define DSA_PDB_SGF_C (0x04 << DSA_PDB_SGF_SHIFT) +#define DSA_PDB_SGF_D (0x02 << DSA_PDB_SGF_SHIFT) +#define DSA_PDB_SGF_AB_SIGN (0x02 << DSA_PDB_SGF_SHIFT) +#define DSA_PDB_SGF_AB_VERIFY (0x01 << DSA_PDB_SGF_SHIFT) + +#define DSA_PDB_L_SHIFT 7 +#define DSA_PDB_L_MASK (0x3ff << DSA_PDB_L_SHIFT) + +#define DSA_PDB_N_MASK 0x7f + +struct dsa_sign_pdb { + u32 sgf_ln; /* Use DSA_PDB_ definitions per above */ + u8 *q; + u8 *r; + u8 *g; /* or Gx,y */ + u8 *s; + u8 *f; + u8 *c; + u8 *d; + u8 *ab; /* ECC only */ + u8 *u; +}; + +struct dsa_verify_pdb { + u32 sgf_ln; + u8 *q; + u8 *r; + u8 *g; /* or Gx,y */ + u8 *w; /* or Wx,y */ + u8 *f; + u8 *c; + u8 *d; + u8 *tmp; /* temporary data block */ + u8 *ab; /* only used if ECC processing */ +}; + +/* RSA Protocol Data Block */ +#define RSA_PDB_SGF_SHIFT 28 +#define RSA_PDB_E_SHIFT 12 +#define RSA_PDB_E_MASK (0xFFF << RSA_PDB_E_SHIFT) +#define RSA_PDB_D_SHIFT 12 +#define RSA_PDB_D_MASK (0xFFF << RSA_PDB_D_SHIFT) +#define RSA_PDB_Q_SHIFT 12 +#define RSA_PDB_Q_MASK (0xFFF << RSA_PDB_Q_SHIFT) + +#define RSA_PDB_SGF_F (0x8 << RSA_PDB_SGF_SHIFT) +#define RSA_PDB_SGF_G (0x4 << RSA_PDB_SGF_SHIFT) +#define RSA_PRIV_PDB_SGF_F (0x4 << RSA_PDB_SGF_SHIFT) +#define RSA_PRIV_PDB_SGF_G (0x8 << RSA_PDB_SGF_SHIFT) + +#define RSA_PRIV_KEY_FRM_1 0 +#define RSA_PRIV_KEY_FRM_2 1 +#define RSA_PRIV_KEY_FRM_3 2 + +/** + * RSA Encrypt Protocol Data Block + * @sgf: scatter-gather field + * @f_dma: dma address of input data + * @g_dma: dma address of encrypted output data + * @n_dma: dma address of RSA modulus + * @e_dma: dma address of RSA public exponent + * @f_len: length in octets of the input data + */ +struct rsa_pub_pdb { + u32 sgf; + dma_addr_t f_dma; + dma_addr_t g_dma; + dma_addr_t n_dma; + dma_addr_t e_dma; + u32 f_len; +}; + +#define SIZEOF_RSA_PUB_PDB (2 * sizeof(u32) + 4 * caam_ptr_sz) + +/** + * RSA Decrypt PDB - Private Key Form #1 + * @sgf: scatter-gather field + * @g_dma: dma address of encrypted input data + * @f_dma: dma address of output data + * @n_dma: dma address of RSA modulus + * @d_dma: dma address of RSA private exponent + */ +struct rsa_priv_f1_pdb { + u32 sgf; + dma_addr_t g_dma; + dma_addr_t f_dma; + dma_addr_t n_dma; + dma_addr_t d_dma; +}; + +#define SIZEOF_RSA_PRIV_F1_PDB (sizeof(u32) + 4 * caam_ptr_sz) + +/** + * RSA Decrypt PDB - Private Key Form #2 + * @sgf : scatter-gather field + * @g_dma : dma address of encrypted input data + * @f_dma : dma address of output data + * @d_dma : dma address of RSA private exponent + * @p_dma : dma address of RSA prime factor p of RSA modulus n + * @q_dma : dma address of RSA prime factor q of RSA modulus n + * @tmp1_dma: dma address of temporary buffer. CAAM uses this temporary buffer + * as internal state buffer. It is assumed to be as long as p. + * @tmp2_dma: dma address of temporary buffer. CAAM uses this temporary buffer + * as internal state buffer. It is assumed to be as long as q. + * @p_q_len : length in bytes of first two prime factors of the RSA modulus n + */ +struct rsa_priv_f2_pdb { + u32 sgf; + dma_addr_t g_dma; + dma_addr_t f_dma; + dma_addr_t d_dma; + dma_addr_t p_dma; + dma_addr_t q_dma; + dma_addr_t tmp1_dma; + dma_addr_t tmp2_dma; + u32 p_q_len; +}; + +#define SIZEOF_RSA_PRIV_F2_PDB (2 * sizeof(u32) + 7 * caam_ptr_sz) + +/** + * RSA Decrypt PDB - Private Key Form #3 + * This is the RSA Chinese Reminder Theorem (CRT) form for two prime factors of + * the RSA modulus. + * @sgf : scatter-gather field + * @g_dma : dma address of encrypted input data + * @f_dma : dma address of output data + * @c_dma : dma address of RSA CRT coefficient + * @p_dma : dma address of RSA prime factor p of RSA modulus n + * @q_dma : dma address of RSA prime factor q of RSA modulus n + * @dp_dma : dma address of RSA CRT exponent of RSA prime factor p + * @dp_dma : dma address of RSA CRT exponent of RSA prime factor q + * @tmp1_dma: dma address of temporary buffer. CAAM uses this temporary buffer + * as internal state buffer. It is assumed to be as long as p. + * @tmp2_dma: dma address of temporary buffer. CAAM uses this temporary buffer + * as internal state buffer. It is assumed to be as long as q. + * @p_q_len : length in bytes of first two prime factors of the RSA modulus n + */ +struct rsa_priv_f3_pdb { + u32 sgf; + dma_addr_t g_dma; + dma_addr_t f_dma; + dma_addr_t c_dma; + dma_addr_t p_dma; + dma_addr_t q_dma; + dma_addr_t dp_dma; + dma_addr_t dq_dma; + dma_addr_t tmp1_dma; + dma_addr_t tmp2_dma; + u32 p_q_len; +}; + +#define SIZEOF_RSA_PRIV_F3_PDB (2 * sizeof(u32) + 9 * caam_ptr_sz) + +#endif diff --git a/drivers/crypto/caam/pkc_desc.c b/drivers/crypto/caam/pkc_desc.c new file mode 100644 index 000000000..0d5ee762e --- /dev/null +++ b/drivers/crypto/caam/pkc_desc.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * caam - Freescale FSL CAAM support for Public Key Cryptography descriptors + * + * Copyright 2016 Freescale Semiconductor, Inc. + * + * There is no Shared Descriptor for PKC so that the Job Descriptor must carry + * all the desired key parameters, input and output pointers. + */ +#include "caampkc.h" +#include "desc_constr.h" + +/* Descriptor for RSA Public operation */ +void init_rsa_pub_desc(u32 *desc, struct rsa_pub_pdb *pdb) +{ + init_job_desc_pdb(desc, 0, SIZEOF_RSA_PUB_PDB); + append_cmd(desc, pdb->sgf); + append_ptr(desc, pdb->f_dma); + append_ptr(desc, pdb->g_dma); + append_ptr(desc, pdb->n_dma); + append_ptr(desc, pdb->e_dma); + append_cmd(desc, pdb->f_len); + append_operation(desc, OP_TYPE_UNI_PROTOCOL | OP_PCLID_RSAENC_PUBKEY); +} + +/* Descriptor for RSA Private operation - Private Key Form #1 */ +void init_rsa_priv_f1_desc(u32 *desc, struct rsa_priv_f1_pdb *pdb) +{ + init_job_desc_pdb(desc, 0, SIZEOF_RSA_PRIV_F1_PDB); + append_cmd(desc, pdb->sgf); + append_ptr(desc, pdb->g_dma); + append_ptr(desc, pdb->f_dma); + append_ptr(desc, pdb->n_dma); + append_ptr(desc, pdb->d_dma); + append_operation(desc, OP_TYPE_UNI_PROTOCOL | OP_PCLID_RSADEC_PRVKEY | + RSA_PRIV_KEY_FRM_1); +} + +/* Descriptor for RSA Private operation - Private Key Form #2 */ +void init_rsa_priv_f2_desc(u32 *desc, struct rsa_priv_f2_pdb *pdb) +{ + init_job_desc_pdb(desc, 0, SIZEOF_RSA_PRIV_F2_PDB); + append_cmd(desc, pdb->sgf); + append_ptr(desc, pdb->g_dma); + append_ptr(desc, pdb->f_dma); + append_ptr(desc, pdb->d_dma); + append_ptr(desc, pdb->p_dma); + append_ptr(desc, pdb->q_dma); + append_ptr(desc, pdb->tmp1_dma); + append_ptr(desc, pdb->tmp2_dma); + append_cmd(desc, pdb->p_q_len); + append_operation(desc, OP_TYPE_UNI_PROTOCOL | OP_PCLID_RSADEC_PRVKEY | + RSA_PRIV_KEY_FRM_2); +} + +/* Descriptor for RSA Private operation - Private Key Form #3 */ +void init_rsa_priv_f3_desc(u32 *desc, struct rsa_priv_f3_pdb *pdb) +{ + init_job_desc_pdb(desc, 0, SIZEOF_RSA_PRIV_F3_PDB); + append_cmd(desc, pdb->sgf); + append_ptr(desc, pdb->g_dma); + append_ptr(desc, pdb->f_dma); + append_ptr(desc, pdb->c_dma); + append_ptr(desc, pdb->p_dma); + append_ptr(desc, pdb->q_dma); + append_ptr(desc, pdb->dp_dma); + append_ptr(desc, pdb->dq_dma); + append_ptr(desc, pdb->tmp1_dma); + append_ptr(desc, pdb->tmp2_dma); + append_cmd(desc, pdb->p_q_len); + append_operation(desc, OP_TYPE_UNI_PROTOCOL | OP_PCLID_RSADEC_PRVKEY | + RSA_PRIV_KEY_FRM_3); +} diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c new file mode 100644 index 000000000..c4bfe8e4a --- /dev/null +++ b/drivers/crypto/caam/qi.c @@ -0,0 +1,813 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * CAAM/SEC 4.x QI transport/backend driver + * Queue Interface backend functionality + * + * Copyright 2013-2016 Freescale Semiconductor, Inc. + * Copyright 2016-2017, 2019-2020 NXP + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "debugfs.h" +#include "regs.h" +#include "qi.h" +#include "desc.h" +#include "intern.h" +#include "desc_constr.h" + +#define PREHDR_RSLS_SHIFT 31 +#define PREHDR_ABS BIT(25) + +/* + * Use a reasonable backlog of frames (per CPU) as congestion threshold, + * so that resources used by the in-flight buffers do not become a memory hog. + */ +#define MAX_RSP_FQ_BACKLOG_PER_CPU 256 + +#define CAAM_QI_ENQUEUE_RETRIES 10000 + +#define CAAM_NAPI_WEIGHT 63 + +/* + * caam_napi - struct holding CAAM NAPI-related params + * @irqtask: IRQ task for QI backend + * @p: QMan portal + */ +struct caam_napi { + struct napi_struct irqtask; + struct qman_portal *p; +}; + +/* + * caam_qi_pcpu_priv - percpu private data structure to main list of pending + * responses expected on each cpu. + * @caam_napi: CAAM NAPI params + * @net_dev: netdev used by NAPI + * @rsp_fq: response FQ from CAAM + */ +struct caam_qi_pcpu_priv { + struct caam_napi caam_napi; + struct net_device *net_dev; + struct qman_fq *rsp_fq; +} ____cacheline_aligned; + +static DEFINE_PER_CPU(struct caam_qi_pcpu_priv, pcpu_qipriv); +static DEFINE_PER_CPU(int, last_cpu); + +/* + * caam_qi_priv - CAAM QI backend private params + * @cgr: QMan congestion group + */ +struct caam_qi_priv { + struct qman_cgr cgr; +}; + +static struct caam_qi_priv qipriv ____cacheline_aligned; + +/* + * This is written by only one core - the one that initialized the CGR - and + * read by multiple cores (all the others). + */ +bool caam_congested __read_mostly; +EXPORT_SYMBOL(caam_congested); + +/* + * This is a cache of buffers, from which the users of CAAM QI driver + * can allocate short (CAAM_QI_MEMCACHE_SIZE) buffers. It's faster than + * doing malloc on the hotpath. + * NOTE: A more elegant solution would be to have some headroom in the frames + * being processed. This could be added by the dpaa-ethernet driver. + * This would pose a problem for userspace application processing which + * cannot know of this limitation. So for now, this will work. + * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here + */ +static struct kmem_cache *qi_cache; + +static void *caam_iova_to_virt(struct iommu_domain *domain, + dma_addr_t iova_addr) +{ + phys_addr_t phys_addr; + + phys_addr = domain ? iommu_iova_to_phys(domain, iova_addr) : iova_addr; + + return phys_to_virt(phys_addr); +} + +int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req) +{ + struct qm_fd fd; + dma_addr_t addr; + int ret; + int num_retries = 0; + + qm_fd_clear_fd(&fd); + qm_fd_set_compound(&fd, qm_sg_entry_get_len(&req->fd_sgt[1])); + + addr = dma_map_single(qidev, req->fd_sgt, sizeof(req->fd_sgt), + DMA_BIDIRECTIONAL); + if (dma_mapping_error(qidev, addr)) { + dev_err(qidev, "DMA mapping error for QI enqueue request\n"); + return -EIO; + } + qm_fd_addr_set64(&fd, addr); + + do { + refcount_inc(&req->drv_ctx->refcnt); + ret = qman_enqueue(req->drv_ctx->req_fq, &fd); + if (likely(!ret)) + return 0; + + refcount_dec(&req->drv_ctx->refcnt); + if (ret != -EBUSY) + break; + num_retries++; + } while (num_retries < CAAM_QI_ENQUEUE_RETRIES); + + dev_err(qidev, "qman_enqueue failed: %d\n", ret); + + return ret; +} +EXPORT_SYMBOL(caam_qi_enqueue); + +static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq, + const union qm_mr_entry *msg) +{ + const struct qm_fd *fd; + struct caam_drv_req *drv_req; + struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev->dev); + struct caam_drv_private *priv = dev_get_drvdata(qidev); + + fd = &msg->ern.fd; + + drv_req = caam_iova_to_virt(priv->domain, qm_fd_addr_get64(fd)); + if (!drv_req) { + dev_err(qidev, + "Can't find original request for CAAM response\n"); + return; + } + + refcount_dec(&drv_req->drv_ctx->refcnt); + + if (qm_fd_get_format(fd) != qm_fd_compound) { + dev_err(qidev, "Non-compound FD from CAAM\n"); + return; + } + + dma_unmap_single(drv_req->drv_ctx->qidev, qm_fd_addr(fd), + sizeof(drv_req->fd_sgt), DMA_BIDIRECTIONAL); + + if (fd->status) + drv_req->cbk(drv_req, be32_to_cpu(fd->status)); + else + drv_req->cbk(drv_req, JRSTA_SSRC_QI); +} + +static struct qman_fq *create_caam_req_fq(struct device *qidev, + struct qman_fq *rsp_fq, + dma_addr_t hwdesc, + int fq_sched_flag) +{ + int ret; + struct qman_fq *req_fq; + struct qm_mcc_initfq opts; + + req_fq = kzalloc_obj(*req_fq, GFP_ATOMIC); + if (!req_fq) + return ERR_PTR(-ENOMEM); + + req_fq->cb.ern = caam_fq_ern_cb; + req_fq->cb.fqs = NULL; + + ret = qman_create_fq(0, QMAN_FQ_FLAG_DYNAMIC_FQID | + QMAN_FQ_FLAG_TO_DCPORTAL, req_fq); + if (ret) { + dev_err(qidev, "Failed to create session req FQ\n"); + goto create_req_fq_fail; + } + + memset(&opts, 0, sizeof(opts)); + opts.we_mask = cpu_to_be16(QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_DESTWQ | + QM_INITFQ_WE_CONTEXTB | + QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CGID); + opts.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_CPCSTASH | QM_FQCTRL_CGE); + qm_fqd_set_destwq(&opts.fqd, qm_channel_caam, 2); + opts.fqd.context_b = cpu_to_be32(qman_fq_fqid(rsp_fq)); + qm_fqd_context_a_set64(&opts.fqd, hwdesc); + opts.fqd.cgid = qipriv.cgr.cgrid; + + ret = qman_init_fq(req_fq, fq_sched_flag, &opts); + if (ret) { + dev_err(qidev, "Failed to init session req FQ\n"); + goto init_req_fq_fail; + } + + dev_dbg(qidev, "Allocated request FQ %u for CPU %u\n", req_fq->fqid, + smp_processor_id()); + return req_fq; + +init_req_fq_fail: + qman_destroy_fq(req_fq); +create_req_fq_fail: + kfree(req_fq); + return ERR_PTR(ret); +} + +static int empty_retired_fq(struct device *qidev, struct qman_fq *fq) +{ + int ret; + + ret = qman_volatile_dequeue(fq, QMAN_VOLATILE_FLAG_WAIT_INT | + QMAN_VOLATILE_FLAG_FINISH, + QM_VDQCR_PRECEDENCE_VDQCR | + QM_VDQCR_NUMFRAMES_TILLEMPTY); + if (ret) { + dev_err(qidev, "Volatile dequeue fail for FQ: %u\n", fq->fqid); + return ret; + } + + do { + struct qman_portal *p; + + p = qman_get_affine_portal(smp_processor_id()); + qman_p_poll_dqrr(p, 16); + } while (fq->flags & QMAN_FQ_STATE_NE); + + return 0; +} + +static int kill_fq(struct device *qidev, struct qman_fq *fq) +{ + u32 flags; + int ret; + + ret = qman_retire_fq(fq, &flags); + if (ret < 0) { + dev_err(qidev, "qman_retire_fq failed: %d\n", ret); + return ret; + } + + if (!ret) + goto empty_fq; + + /* Async FQ retirement condition */ + if (ret == 1) { + /* Retry till FQ gets in retired state */ + do { + msleep(20); + } while (fq->state != qman_fq_state_retired); + + WARN_ON(fq->flags & QMAN_FQ_STATE_BLOCKOOS); + WARN_ON(fq->flags & QMAN_FQ_STATE_ORL); + } + +empty_fq: + if (fq->flags & QMAN_FQ_STATE_NE) { + ret = empty_retired_fq(qidev, fq); + if (ret) { + dev_err(qidev, "empty_retired_fq fail for FQ: %u\n", + fq->fqid); + return ret; + } + } + + ret = qman_oos_fq(fq); + if (ret) + dev_err(qidev, "OOS of FQID: %u failed\n", fq->fqid); + + qman_destroy_fq(fq); + kfree(fq); + + return ret; +} + +static int empty_caam_fq(struct qman_fq *fq, struct caam_drv_ctx *drv_ctx) +{ + int ret; + int retries = 10; + struct qm_mcr_queryfq_np np; + + /* Wait till the older CAAM FQ get empty */ + do { + ret = qman_query_fq_np(fq, &np); + if (ret) + return ret; + + if (!qm_mcr_np_get(&np, frm_cnt)) + break; + + msleep(20); + } while (1); + + /* Wait until pending jobs from this FQ are processed by CAAM */ + do { + if (refcount_read(&drv_ctx->refcnt) == 1) + break; + + msleep(20); + } while (--retries); + + if (!retries) + dev_warn_once(drv_ctx->qidev, "%d frames from FQID %u still pending in CAAM\n", + refcount_read(&drv_ctx->refcnt), fq->fqid); + + return 0; +} + +int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc) +{ + int ret; + u32 num_words; + struct qman_fq *new_fq, *old_fq; + struct device *qidev = drv_ctx->qidev; + + num_words = desc_len(sh_desc); + if (num_words > MAX_SDLEN) { + dev_err(qidev, "Invalid descriptor len: %d words\n", num_words); + return -EINVAL; + } + + /* Note down older req FQ */ + old_fq = drv_ctx->req_fq; + + /* Create a new req FQ in parked state */ + new_fq = create_caam_req_fq(drv_ctx->qidev, drv_ctx->rsp_fq, + drv_ctx->context_a, 0); + if (IS_ERR(new_fq)) { + dev_err(qidev, "FQ allocation for shdesc update failed\n"); + return PTR_ERR(new_fq); + } + + /* Hook up new FQ to context so that new requests keep queuing */ + drv_ctx->req_fq = new_fq; + + /* Empty and remove the older FQ */ + ret = empty_caam_fq(old_fq, drv_ctx); + if (ret) { + dev_err(qidev, "Old CAAM FQ empty failed: %d\n", ret); + + /* We can revert to older FQ */ + drv_ctx->req_fq = old_fq; + + if (kill_fq(qidev, new_fq)) + dev_warn(qidev, "New CAAM FQ kill failed\n"); + + return ret; + } + + /* + * Re-initialise pre-header. Set RSLS and SDLEN. + * Update the shared descriptor for driver context. + */ + drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) | + num_words); + drv_ctx->prehdr[1] = cpu_to_caam32(PREHDR_ABS); + memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc)); + dma_sync_single_for_device(qidev, drv_ctx->context_a, + sizeof(drv_ctx->sh_desc) + + sizeof(drv_ctx->prehdr), + DMA_BIDIRECTIONAL); + + /* Put the new FQ in scheduled state */ + ret = qman_schedule_fq(new_fq); + if (ret) { + dev_err(qidev, "Fail to sched new CAAM FQ, ecode = %d\n", ret); + + /* + * We can kill new FQ and revert to old FQ. + * Since the desc is already modified, it is success case + */ + + drv_ctx->req_fq = old_fq; + + if (kill_fq(qidev, new_fq)) + dev_warn(qidev, "New CAAM FQ kill failed\n"); + } else if (kill_fq(qidev, old_fq)) { + dev_warn(qidev, "Old CAAM FQ kill failed\n"); + } + + return 0; +} +EXPORT_SYMBOL(caam_drv_ctx_update); + +struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev, + int *cpu, + u32 *sh_desc) +{ + size_t size; + u32 num_words; + dma_addr_t hwdesc; + struct caam_drv_ctx *drv_ctx; + const cpumask_t *cpus = qman_affine_cpus(); + + num_words = desc_len(sh_desc); + if (num_words > MAX_SDLEN) { + dev_err(qidev, "Invalid descriptor len: %d words\n", + num_words); + return ERR_PTR(-EINVAL); + } + + drv_ctx = kzalloc_obj(*drv_ctx, GFP_ATOMIC); + if (!drv_ctx) + return ERR_PTR(-ENOMEM); + + /* + * Initialise pre-header - set RSLS and SDLEN - and shared descriptor + * and dma-map them. + */ + drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) | + num_words); + drv_ctx->prehdr[1] = cpu_to_caam32(PREHDR_ABS); + memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc)); + size = sizeof(drv_ctx->prehdr) + sizeof(drv_ctx->sh_desc); + hwdesc = dma_map_single(qidev, drv_ctx->prehdr, size, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(qidev, hwdesc)) { + dev_err(qidev, "DMA map error for preheader + shdesc\n"); + kfree(drv_ctx); + return ERR_PTR(-ENOMEM); + } + drv_ctx->context_a = hwdesc; + + /* If given CPU does not own the portal, choose another one that does */ + if (!cpumask_test_cpu(*cpu, cpus)) { + int *pcpu = &get_cpu_var(last_cpu); + + *pcpu = cpumask_next_wrap(*pcpu, cpus); + *cpu = *pcpu; + put_cpu_var(last_cpu); + } + drv_ctx->cpu = *cpu; + + /* Find response FQ hooked with this CPU */ + drv_ctx->rsp_fq = per_cpu(pcpu_qipriv.rsp_fq, drv_ctx->cpu); + + /* Attach request FQ */ + drv_ctx->req_fq = create_caam_req_fq(qidev, drv_ctx->rsp_fq, hwdesc, + QMAN_INITFQ_FLAG_SCHED); + if (IS_ERR(drv_ctx->req_fq)) { + dev_err(qidev, "create_caam_req_fq failed\n"); + dma_unmap_single(qidev, hwdesc, size, DMA_BIDIRECTIONAL); + kfree(drv_ctx); + return ERR_PTR(-ENOMEM); + } + + /* init reference counter used to track references to request FQ */ + refcount_set(&drv_ctx->refcnt, 1); + + drv_ctx->qidev = qidev; + return drv_ctx; +} +EXPORT_SYMBOL(caam_drv_ctx_init); + +void *qi_cache_alloc(gfp_t flags) +{ + return kmem_cache_alloc(qi_cache, flags); +} +EXPORT_SYMBOL(qi_cache_alloc); + +void qi_cache_free(void *obj) +{ + kmem_cache_free(qi_cache, obj); +} +EXPORT_SYMBOL(qi_cache_free); + +static int caam_qi_poll(struct napi_struct *napi, int budget) +{ + struct caam_napi *np = container_of(napi, struct caam_napi, irqtask); + + int cleaned = qman_p_poll_dqrr(np->p, budget); + + if (cleaned < budget) { + napi_complete(napi); + qman_p_irqsource_add(np->p, QM_PIRQ_DQRI); + } + + return cleaned; +} + +void caam_drv_ctx_rel(struct caam_drv_ctx *drv_ctx) +{ + if (IS_ERR_OR_NULL(drv_ctx)) + return; + + /* Remove request FQ */ + if (kill_fq(drv_ctx->qidev, drv_ctx->req_fq)) + dev_err(drv_ctx->qidev, "Crypto session req FQ kill failed\n"); + + dma_unmap_single(drv_ctx->qidev, drv_ctx->context_a, + sizeof(drv_ctx->sh_desc) + sizeof(drv_ctx->prehdr), + DMA_BIDIRECTIONAL); + kfree(drv_ctx); +} +EXPORT_SYMBOL(caam_drv_ctx_rel); + +static void caam_qi_shutdown(void *data) +{ + int i; + struct device *qidev = data; + struct caam_qi_priv *priv = &qipriv; + const cpumask_t *cpus = qman_affine_cpus(); + + for_each_cpu(i, cpus) { + struct napi_struct *irqtask; + + irqtask = &per_cpu_ptr(&pcpu_qipriv.caam_napi, i)->irqtask; + napi_disable(irqtask); + netif_napi_del(irqtask); + + if (kill_fq(qidev, per_cpu(pcpu_qipriv.rsp_fq, i))) + dev_err(qidev, "Rsp FQ kill failed, cpu: %d\n", i); + free_netdev(per_cpu(pcpu_qipriv.net_dev, i)); + } + + qman_delete_cgr_safe(&priv->cgr); + qman_release_cgrid(priv->cgr.cgrid); + + kmem_cache_destroy(qi_cache); +} + +static void cgr_cb(struct qman_portal *qm, struct qman_cgr *cgr, int congested) +{ + caam_congested = congested; + + if (congested) { + caam_debugfs_qi_congested(); + + pr_debug_ratelimited("CAAM entered congestion\n"); + + } else { + pr_debug_ratelimited("CAAM exited congestion\n"); + } +} + +static int caam_qi_napi_schedule(struct qman_portal *p, struct caam_napi *np, + bool sched_napi) +{ + if (sched_napi) { + /* Disable QMan IRQ source and invoke NAPI */ + qman_p_irqsource_remove(p, QM_PIRQ_DQRI); + np->p = p; + napi_schedule(&np->irqtask); + return 1; + } + return 0; +} + +static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p, + struct qman_fq *rsp_fq, + const struct qm_dqrr_entry *dqrr, + bool sched_napi) +{ + struct caam_napi *caam_napi = raw_cpu_ptr(&pcpu_qipriv.caam_napi); + struct caam_drv_req *drv_req; + const struct qm_fd *fd; + struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev->dev); + struct caam_drv_private *priv = dev_get_drvdata(qidev); + u32 status; + + if (caam_qi_napi_schedule(p, caam_napi, sched_napi)) + return qman_cb_dqrr_stop; + + fd = &dqrr->fd; + + drv_req = caam_iova_to_virt(priv->domain, qm_fd_addr_get64(fd)); + if (unlikely(!drv_req)) { + dev_err(qidev, + "Can't find original request for caam response\n"); + return qman_cb_dqrr_consume; + } + + refcount_dec(&drv_req->drv_ctx->refcnt); + + status = be32_to_cpu(fd->status); + if (unlikely(status)) { + u32 ssrc = status & JRSTA_SSRC_MASK; + u8 err_id = status & JRSTA_CCBERR_ERRID_MASK; + + if (ssrc != JRSTA_SSRC_CCB_ERROR || + err_id != JRSTA_CCBERR_ERRID_ICVCHK) + dev_err_ratelimited(qidev, + "Error: %#x in CAAM response FD\n", + status); + } + + if (unlikely(qm_fd_get_format(fd) != qm_fd_compound)) { + dev_err(qidev, "Non-compound FD from CAAM\n"); + return qman_cb_dqrr_consume; + } + + dma_unmap_single(drv_req->drv_ctx->qidev, qm_fd_addr(fd), + sizeof(drv_req->fd_sgt), DMA_BIDIRECTIONAL); + + drv_req->cbk(drv_req, status); + return qman_cb_dqrr_consume; +} + +static int alloc_rsp_fq_cpu(struct device *qidev, unsigned int cpu) +{ + struct qm_mcc_initfq opts; + struct qman_fq *fq; + int ret; + + fq = kzalloc_obj(*fq); + if (!fq) + return -ENOMEM; + + fq->cb.dqrr = caam_rsp_fq_dqrr_cb; + + ret = qman_create_fq(0, QMAN_FQ_FLAG_NO_ENQUEUE | + QMAN_FQ_FLAG_DYNAMIC_FQID, fq); + if (ret) { + dev_err(qidev, "Rsp FQ create failed\n"); + kfree(fq); + return -ENODEV; + } + + memset(&opts, 0, sizeof(opts)); + opts.we_mask = cpu_to_be16(QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_DESTWQ | + QM_INITFQ_WE_CONTEXTB | + QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CGID); + opts.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_CTXASTASHING | + QM_FQCTRL_CPCSTASH | QM_FQCTRL_CGE); + qm_fqd_set_destwq(&opts.fqd, qman_affine_channel(cpu), 3); + opts.fqd.cgid = qipriv.cgr.cgrid; + opts.fqd.context_a.stashing.exclusive = QM_STASHING_EXCL_CTX | + QM_STASHING_EXCL_DATA; + qm_fqd_set_stashing(&opts.fqd, 0, 1, 1); + + ret = qman_init_fq(fq, QMAN_INITFQ_FLAG_SCHED, &opts); + if (ret) { + dev_err(qidev, "Rsp FQ init failed\n"); + kfree(fq); + return -ENODEV; + } + + per_cpu(pcpu_qipriv.rsp_fq, cpu) = fq; + + dev_dbg(qidev, "Allocated response FQ %u for CPU %u", fq->fqid, cpu); + return 0; +} + +static int init_cgr(struct device *qidev) +{ + int ret; + struct qm_mcc_initcgr opts; + const u64 val = (u64)cpumask_weight(qman_affine_cpus()) * + MAX_RSP_FQ_BACKLOG_PER_CPU; + + ret = qman_alloc_cgrid(&qipriv.cgr.cgrid); + if (ret) { + dev_err(qidev, "CGR alloc failed for rsp FQs: %d\n", ret); + return ret; + } + + qipriv.cgr.cb = cgr_cb; + memset(&opts, 0, sizeof(opts)); + opts.we_mask = cpu_to_be16(QM_CGR_WE_CSCN_EN | QM_CGR_WE_CS_THRES | + QM_CGR_WE_MODE); + opts.cgr.cscn_en = QM_CGR_EN; + opts.cgr.mode = QMAN_CGR_MODE_FRAME; + qm_cgr_cs_thres_set64(&opts.cgr.cs_thres, val, 1); + + ret = qman_create_cgr(&qipriv.cgr, QMAN_CGR_FLAG_USE_INIT, &opts); + if (ret) { + dev_err(qidev, "Error %d creating CAAM CGRID: %u\n", ret, + qipriv.cgr.cgrid); + return ret; + } + + dev_dbg(qidev, "Congestion threshold set to %llu\n", val); + return 0; +} + +static int alloc_rsp_fqs(struct device *qidev) +{ + int ret, i; + const cpumask_t *cpus = qman_affine_cpus(); + + /*Now create response FQs*/ + for_each_cpu(i, cpus) { + ret = alloc_rsp_fq_cpu(qidev, i); + if (ret) { + dev_err(qidev, "CAAM rsp FQ alloc failed, cpu: %u", i); + return ret; + } + } + + return 0; +} + +static void free_rsp_fqs(void) +{ + int i; + const cpumask_t *cpus = qman_affine_cpus(); + + for_each_cpu(i, cpus) + kfree(per_cpu(pcpu_qipriv.rsp_fq, i)); +} + +static void free_caam_qi_pcpu_netdev(const cpumask_t *cpus) +{ + struct caam_qi_pcpu_priv *priv; + int i; + + for_each_cpu(i, cpus) { + priv = per_cpu_ptr(&pcpu_qipriv, i); + free_netdev(priv->net_dev); + } +} + +int caam_qi_init(struct platform_device *caam_pdev) +{ + int err, i; + struct device *qidev = &caam_pdev->dev; + struct caam_drv_private *ctrlpriv; + const cpumask_t *cpus = qman_affine_cpus(); + cpumask_var_t clean_mask; + + err = -ENOMEM; + if (!zalloc_cpumask_var(&clean_mask, GFP_KERNEL)) + goto fail_cpumask; + + ctrlpriv = dev_get_drvdata(qidev); + + /* Initialize the congestion detection */ + err = init_cgr(qidev); + if (err) { + dev_err(qidev, "CGR initialization failed: %d\n", err); + goto fail_cgr; + } + + /* Initialise response FQs */ + err = alloc_rsp_fqs(qidev); + if (err) { + dev_err(qidev, "Can't allocate CAAM response FQs: %d\n", err); + goto fail_fqs; + } + + /* + * Enable the NAPI contexts on each of the core which has an affine + * portal. + */ + for_each_cpu(i, cpus) { + struct caam_qi_pcpu_priv *priv = per_cpu_ptr(&pcpu_qipriv, i); + struct caam_napi *caam_napi = &priv->caam_napi; + struct napi_struct *irqtask = &caam_napi->irqtask; + struct net_device *net_dev; + + net_dev = alloc_netdev_dummy(0); + if (!net_dev) { + err = -ENOMEM; + goto fail; + } + cpumask_set_cpu(i, clean_mask); + priv->net_dev = net_dev; + net_dev->dev = *qidev; + + netif_napi_add_tx_weight(net_dev, irqtask, caam_qi_poll, + CAAM_NAPI_WEIGHT); + + napi_enable(irqtask); + } + + qi_cache = kmem_cache_create("caamqicache", CAAM_QI_MEMCACHE_SIZE, + dma_get_cache_alignment(), 0, NULL); + if (!qi_cache) { + dev_err(qidev, "Can't allocate CAAM cache\n"); + err = -ENOMEM; + goto fail; + } + + caam_debugfs_qi_init(ctrlpriv); + + err = devm_add_action_or_reset(qidev, caam_qi_shutdown, qidev); + if (err) + goto fail2; + + dev_info(qidev, "Linux CAAM Queue I/F driver initialised\n"); + goto free_cpumask; + +fail2: + kmem_cache_destroy(qi_cache); +fail: + free_caam_qi_pcpu_netdev(clean_mask); +fail_fqs: + free_rsp_fqs(); + qman_delete_cgr_safe(&qipriv.cgr); + qman_release_cgrid(qipriv.cgr.cgrid); +fail_cgr: +free_cpumask: + free_cpumask_var(clean_mask); +fail_cpumask: + return err; +} diff --git a/drivers/crypto/caam/qi.h b/drivers/crypto/caam/qi.h new file mode 100644 index 000000000..a96e3d213 --- /dev/null +++ b/drivers/crypto/caam/qi.h @@ -0,0 +1,183 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Public definitions for the CAAM/QI (Queue Interface) backend. + * + * Copyright 2013-2016 Freescale Semiconductor, Inc. + * Copyright 2016-2017, 2020 NXP + */ + +#ifndef __QI_H__ +#define __QI_H__ + +#include +#include +#include +#include "compat.h" +#include "desc.h" +#include "desc_constr.h" + +/* Length of a single buffer in the QI driver memory cache */ +#define CAAM_QI_MEMCACHE_SIZE 768 + +extern bool caam_congested __read_mostly; + +/* + * This is the request structure the driver application should fill while + * submitting a job to driver. + */ +struct caam_drv_req; + +/* + * caam_qi_cbk - application's callback function invoked by the driver when the + * request has been successfully processed. + * @drv_req: original request that was submitted + * @status: completion status of request (0 - success, non-zero - error code) + */ +typedef void (*caam_qi_cbk)(struct caam_drv_req *drv_req, u32 status); + +enum optype { + ENCRYPT, + DECRYPT, + NUM_OP +}; + +/** + * caam_drv_ctx - CAAM/QI backend driver context + * + * The jobs are processed by the driver against a driver context. + * With every cryptographic context, a driver context is attached. + * The driver context contains data for private use by driver. + * For the applications, this is an opaque structure. + * + * @prehdr: preheader placed before shrd desc + * @sh_desc: shared descriptor + * @context_a: shared descriptor dma address + * @req_fq: to-CAAM request frame queue + * @rsp_fq: from-CAAM response frame queue + * @refcnt: reference counter incremented for each frame enqueued in to-CAAM FQ + * @cpu: cpu on which to receive CAAM response + * @op_type: operation type + * @qidev: device pointer for CAAM/QI backend + */ +struct caam_drv_ctx { + struct { + u32 prehdr[2]; + u32 sh_desc[MAX_SDLEN]; + } __aligned(CRYPTO_DMA_ALIGN); + dma_addr_t context_a; + struct qman_fq *req_fq; + struct qman_fq *rsp_fq; + refcount_t refcnt; + int cpu; + enum optype op_type; + struct device *qidev; +}; + +/** + * caam_drv_req - The request structure the driver application should fill while + * submitting a job to driver. + * @fd_sgt: QMan S/G pointing to output (fd_sgt[0]) and input (fd_sgt[1]) + * buffers. + * @cbk: callback function to invoke when job is completed + * @app_ctx: arbitrary context attached with request by the application + * + * The fields mentioned below should not be used by application. + * These are for private use by driver. + * + * @hdr__: linked list header to maintain list of outstanding requests to CAAM + * @hwaddr: DMA address for the S/G table. + */ +struct caam_drv_req { + struct qm_sg_entry fd_sgt[2]; + struct caam_drv_ctx *drv_ctx; + caam_qi_cbk cbk; + void *app_ctx; +} __aligned(CRYPTO_DMA_ALIGN); + +/** + * caam_drv_ctx_init - Initialise a CAAM/QI driver context + * + * A CAAM/QI driver context must be attached with each cryptographic context. + * This function allocates memory for CAAM/QI context and returns a handle to + * the application. This handle must be submitted along with each enqueue + * request to the driver by the application. + * + * @cpu: CPU where the application prefers to the driver to receive CAAM + * responses. The request completion callback would be issued from this + * CPU. + * @sh_desc: shared descriptor pointer to be attached with CAAM/QI driver + * context. + * + * Returns a driver context on success or negative error code on failure. + */ +struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev, int *cpu, + u32 *sh_desc); + +/** + * caam_qi_enqueue - Submit a request to QI backend driver. + * + * The request structure must be properly filled as described above. + * + * @qidev: device pointer for QI backend + * @req: CAAM QI request structure + * + * Returns 0 on success or negative error code on failure. + */ +int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req); + +/** + * caam_drv_ctx_busy - Check if there are too many jobs pending with CAAM + * or too many CAAM responses are pending to be processed. + * @drv_ctx: driver context for which job is to be submitted + * + * Returns caam congestion status 'true/false' + */ +bool caam_drv_ctx_busy(struct caam_drv_ctx *drv_ctx); + +/** + * caam_drv_ctx_update - Update QI driver context + * + * Invoked when shared descriptor is required to be change in driver context. + * + * @drv_ctx: driver context to be updated + * @sh_desc: new shared descriptor pointer to be updated in QI driver context + * + * Returns 0 on success or negative error code on failure. + */ +int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc); + +/** + * caam_drv_ctx_rel - Release a QI driver context + * @drv_ctx: context to be released + */ +void caam_drv_ctx_rel(struct caam_drv_ctx *drv_ctx); + +int caam_qi_init(struct platform_device *pdev); + +/** + * qi_cache_alloc - Allocate buffers from CAAM-QI cache + * + * Invoked when a user of the CAAM-QI (i.e. caamalg-qi) needs data which has + * to be allocated on the hotpath. Instead of using malloc, one can use the + * services of the CAAM QI memory cache (backed by kmem_cache). The buffers + * will have a size of 256B, which is sufficient for hosting 16 SG entries. + * + * @flags: flags that would be used for the equivalent malloc(..) call + * + * Returns a pointer to a retrieved buffer on success or NULL on failure. + */ +void *qi_cache_alloc(gfp_t flags); + +/** + * qi_cache_free - Frees buffers allocated from CAAM-QI cache + * + * Invoked when a user of the CAAM-QI (i.e. caamalg-qi) no longer needs + * the buffer previously allocated by a qi_cache_alloc call. + * No checking is being done, the call is a passthrough call to + * kmem_cache_free(...) + * + * @obj: object previously allocated using qi_cache_alloc() + */ +void qi_cache_free(void *obj); + +#endif /* __QI_H__ */ diff --git a/drivers/crypto/caam/regs.h b/drivers/crypto/caam/regs.h new file mode 100644 index 000000000..873df9de9 --- /dev/null +++ b/drivers/crypto/caam/regs.h @@ -0,0 +1,1036 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * CAAM hardware register-level view + * + * Copyright 2008-2011 Freescale Semiconductor, Inc. + * Copyright 2018, 2023 NXP + */ + +#ifndef REGS_H +#define REGS_H + +#include +#include +#include +#include + +/* + * Architecture-specific register access methods + * + * CAAM's bus-addressable registers are 64 bits internally. + * They have been wired to be safely accessible on 32-bit + * architectures, however. Registers were organized such + * that (a) they can be contained in 32 bits, (b) if not, then they + * can be treated as two 32-bit entities, or finally (c) if they + * must be treated as a single 64-bit value, then this can safely + * be done with two 32-bit cycles. + * + * For 32-bit operations on 64-bit values, CAAM follows the same + * 64-bit register access conventions as it's predecessors, in that + * writes are "triggered" by a write to the register at the numerically + * higher address, thus, a full 64-bit write cycle requires a write + * to the lower address, followed by a write to the higher address, + * which will latch/execute the write cycle. + * + * For example, let's assume a SW reset of CAAM through the master + * configuration register. + * - SWRST is in bit 31 of MCFG. + * - MCFG begins at base+0x0000. + * - Bits 63-32 are a 32-bit word at base+0x0000 (numerically-lower) + * - Bits 31-0 are a 32-bit word at base+0x0004 (numerically-higher) + * + * (and on Power, the convention is 0-31, 32-63, I know...) + * + * Assuming a 64-bit write to this MCFG to perform a software reset + * would then require a write of 0 to base+0x0000, followed by a + * write of 0x80000000 to base+0x0004, which would "execute" the + * reset. + * + * Of course, since MCFG 63-32 is all zero, we could cheat and simply + * write 0x8000000 to base+0x0004, and the reset would work fine. + * However, since CAAM does contain some write-and-read-intended + * 64-bit registers, this code defines 64-bit access methods for + * the sake of internal consistency and simplicity, and so that a + * clean transition to 64-bit is possible when it becomes necessary. + * + * There are limitations to this that the developer must recognize. + * 32-bit architectures cannot enforce an atomic-64 operation, + * Therefore: + * + * - On writes, since the HW is assumed to latch the cycle on the + * write of the higher-numeric-address word, then ordered + * writes work OK. + * + * - For reads, where a register contains a relevant value of more + * that 32 bits, the hardware employs logic to latch the other + * "half" of the data until read, ensuring an accurate value. + * This is of particular relevance when dealing with CAAM's + * performance counters. + * + */ + +extern bool caam_little_end; +extern bool caam_imx; +extern size_t caam_ptr_sz; + +#define caam_to_cpu(len) \ +static inline u##len caam##len ## _to_cpu(u##len val) \ +{ \ + if (caam_little_end) \ + return le##len ## _to_cpu((__force __le##len)val); \ + else \ + return be##len ## _to_cpu((__force __be##len)val); \ +} + +#define cpu_to_caam(len) \ +static inline u##len cpu_to_caam##len(u##len val) \ +{ \ + if (caam_little_end) \ + return (__force u##len)cpu_to_le##len(val); \ + else \ + return (__force u##len)cpu_to_be##len(val); \ +} + +caam_to_cpu(16) +caam_to_cpu(32) +caam_to_cpu(64) +cpu_to_caam(16) +cpu_to_caam(32) +cpu_to_caam(64) + +static inline void wr_reg32(void __iomem *reg, u32 data) +{ + if (caam_little_end) + iowrite32(data, reg); + else + iowrite32be(data, reg); +} + +static inline u32 rd_reg32(void __iomem *reg) +{ + if (caam_little_end) + return ioread32(reg); + + return ioread32be(reg); +} + +static inline void clrsetbits_32(void __iomem *reg, u32 clear, u32 set) +{ + if (caam_little_end) + iowrite32((ioread32(reg) & ~clear) | set, reg); + else + iowrite32be((ioread32be(reg) & ~clear) | set, reg); +} + +/* + * The only users of these wr/rd_reg64 functions is the Job Ring (JR). + * The DMA address registers in the JR are handled differently depending on + * platform: + * + * 1. All BE CAAM platforms and i.MX platforms (LE CAAM): + * + * base + 0x0000 : most-significant 32 bits + * base + 0x0004 : least-significant 32 bits + * + * The 32-bit version of this core therefore has to write to base + 0x0004 + * to set the 32-bit wide DMA address. + * + * 2. All other LE CAAM platforms (LS1021A etc.) + * base + 0x0000 : least-significant 32 bits + * base + 0x0004 : most-significant 32 bits + */ +static inline void wr_reg64(void __iomem *reg, u64 data) +{ + if (caam_little_end) { + if (caam_imx) { + iowrite32(data >> 32, (u32 __iomem *)(reg)); + iowrite32(data, (u32 __iomem *)(reg) + 1); + } else { + iowrite64(data, reg); + } + } else { + iowrite64be(data, reg); + } +} + +static inline u64 rd_reg64(void __iomem *reg) +{ + if (caam_little_end) { + if (caam_imx) { + u32 low, high; + + high = ioread32(reg); + low = ioread32(reg + sizeof(u32)); + + return low + ((u64)high << 32); + } else { + return ioread64(reg); + } + } else { + return ioread64be(reg); + } +} + +static inline u64 cpu_to_caam_dma64(dma_addr_t value) +{ + if (caam_imx) { + u64 ret_val = (u64)cpu_to_caam32(lower_32_bits(value)) << 32; + + if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT)) + ret_val |= (u64)cpu_to_caam32(upper_32_bits(value)); + + return ret_val; + } + + return cpu_to_caam64(value); +} + +static inline u64 caam_dma64_to_cpu(u64 value) +{ + if (caam_imx) + return (((u64)caam32_to_cpu(lower_32_bits(value)) << 32) | + (u64)caam32_to_cpu(upper_32_bits(value))); + + return caam64_to_cpu(value); +} + +static inline u64 cpu_to_caam_dma(u64 value) +{ + if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) && + caam_ptr_sz == sizeof(u64)) + return cpu_to_caam_dma64(value); + else + return cpu_to_caam32(value); +} + +static inline u64 caam_dma_to_cpu(u64 value) +{ + if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) && + caam_ptr_sz == sizeof(u64)) + return caam_dma64_to_cpu(value); + else + return caam32_to_cpu(value); +} + +/* + * jr_outentry + * Represents each entry in a JobR output ring + */ + +static inline void jr_outentry_get(void *outring, int hw_idx, dma_addr_t *desc, + u32 *jrstatus) +{ + + if (caam_ptr_sz == sizeof(u32)) { + struct { + u32 desc; + u32 jrstatus; + } __packed *outentry = outring; + + *desc = outentry[hw_idx].desc; + *jrstatus = outentry[hw_idx].jrstatus; + } else { + struct { + dma_addr_t desc;/* Pointer to completed descriptor */ + u32 jrstatus; /* Status for completed descriptor */ + } __packed *outentry = outring; + + *desc = outentry[hw_idx].desc; + *jrstatus = outentry[hw_idx].jrstatus; + } +} + +#define SIZEOF_JR_OUTENTRY (caam_ptr_sz + sizeof(u32)) + +static inline dma_addr_t jr_outentry_desc(void *outring, int hw_idx) +{ + dma_addr_t desc; + u32 unused; + + jr_outentry_get(outring, hw_idx, &desc, &unused); + + return desc; +} + +static inline u32 jr_outentry_jrstatus(void *outring, int hw_idx) +{ + dma_addr_t unused; + u32 jrstatus; + + jr_outentry_get(outring, hw_idx, &unused, &jrstatus); + + return jrstatus; +} + +static inline void jr_inpentry_set(void *inpring, int hw_idx, dma_addr_t val) +{ + if (caam_ptr_sz == sizeof(u32)) { + u32 *inpentry = inpring; + + inpentry[hw_idx] = val; + } else { + dma_addr_t *inpentry = inpring; + + inpentry[hw_idx] = val; + } +} + +#define SIZEOF_JR_INPENTRY caam_ptr_sz + + +/* Version registers (Era 10+) e80-eff */ +struct version_regs { + u32 crca; /* CRCA_VERSION */ + u32 afha; /* AFHA_VERSION */ + u32 kfha; /* KFHA_VERSION */ + u32 pkha; /* PKHA_VERSION */ + u32 aesa; /* AESA_VERSION */ + u32 mdha; /* MDHA_VERSION */ + u32 desa; /* DESA_VERSION */ + u32 snw8a; /* SNW8A_VERSION */ + u32 snw9a; /* SNW9A_VERSION */ + u32 zuce; /* ZUCE_VERSION */ + u32 zuca; /* ZUCA_VERSION */ + u32 ccha; /* CCHA_VERSION */ + u32 ptha; /* PTHA_VERSION */ + u32 rng; /* RNG_VERSION */ + u32 trng; /* TRNG_VERSION */ + u32 aaha; /* AAHA_VERSION */ + u32 rsvd[10]; + u32 sr; /* SR_VERSION */ + u32 dma; /* DMA_VERSION */ + u32 ai; /* AI_VERSION */ + u32 qi; /* QI_VERSION */ + u32 jr; /* JR_VERSION */ + u32 deco; /* DECO_VERSION */ +}; + +/* Version registers bitfields */ + +/* Number of CHAs instantiated */ +#define CHA_VER_NUM_MASK 0xffull +/* CHA Miscellaneous Information */ +#define CHA_VER_MISC_SHIFT 8 +#define CHA_VER_MISC_MASK (0xffull << CHA_VER_MISC_SHIFT) +/* CHA Revision Number */ +#define CHA_VER_REV_SHIFT 16 +#define CHA_VER_REV_MASK (0xffull << CHA_VER_REV_SHIFT) +/* CHA Version ID */ +#define CHA_VER_VID_SHIFT 24 +#define CHA_VER_VID_MASK (0xffull << CHA_VER_VID_SHIFT) + +/* CHA Miscellaneous Information - AESA_MISC specific */ +#define CHA_VER_MISC_AES_NUM_MASK GENMASK(7, 0) +#define CHA_VER_MISC_AES_GCM BIT(1 + CHA_VER_MISC_SHIFT) + +/* CHA Miscellaneous Information - PKHA_MISC specific */ +#define CHA_VER_MISC_PKHA_NO_CRYPT BIT(7 + CHA_VER_MISC_SHIFT) + +/* + * caam_perfmon - Performance Monitor/Secure Memory Status/ + * CAAM Global Status/Component Version IDs + * + * Spans f00-fff wherever instantiated + */ + +/* Number of DECOs */ +#define CHA_NUM_MS_DECONUM_SHIFT 24 +#define CHA_NUM_MS_DECONUM_MASK (0xfull << CHA_NUM_MS_DECONUM_SHIFT) + +/* + * CHA version IDs / instantiation bitfields (< Era 10) + * Defined for use with the cha_id fields in perfmon, but the same shift/mask + * selectors can be used to pull out the number of instantiated blocks within + * cha_num fields in perfmon because the locations are the same. + */ +#define CHA_ID_LS_AES_SHIFT 0 +#define CHA_ID_LS_AES_MASK (0xfull << CHA_ID_LS_AES_SHIFT) + +#define CHA_ID_LS_DES_SHIFT 4 +#define CHA_ID_LS_DES_MASK (0xfull << CHA_ID_LS_DES_SHIFT) + +#define CHA_ID_LS_ARC4_SHIFT 8 +#define CHA_ID_LS_ARC4_MASK (0xfull << CHA_ID_LS_ARC4_SHIFT) + +#define CHA_ID_LS_MD_SHIFT 12 +#define CHA_ID_LS_MD_MASK (0xfull << CHA_ID_LS_MD_SHIFT) + +#define CHA_ID_LS_RNG_SHIFT 16 +#define CHA_ID_LS_RNG_MASK (0xfull << CHA_ID_LS_RNG_SHIFT) + +#define CHA_ID_LS_SNW8_SHIFT 20 +#define CHA_ID_LS_SNW8_MASK (0xfull << CHA_ID_LS_SNW8_SHIFT) + +#define CHA_ID_LS_KAS_SHIFT 24 +#define CHA_ID_LS_KAS_MASK (0xfull << CHA_ID_LS_KAS_SHIFT) + +#define CHA_ID_LS_PK_SHIFT 28 +#define CHA_ID_LS_PK_MASK (0xfull << CHA_ID_LS_PK_SHIFT) + +#define CHA_ID_MS_CRC_SHIFT 0 +#define CHA_ID_MS_CRC_MASK (0xfull << CHA_ID_MS_CRC_SHIFT) + +#define CHA_ID_MS_SNW9_SHIFT 4 +#define CHA_ID_MS_SNW9_MASK (0xfull << CHA_ID_MS_SNW9_SHIFT) + +#define CHA_ID_MS_DECO_SHIFT 24 +#define CHA_ID_MS_DECO_MASK (0xfull << CHA_ID_MS_DECO_SHIFT) + +#define CHA_ID_MS_JR_SHIFT 28 +#define CHA_ID_MS_JR_MASK (0xfull << CHA_ID_MS_JR_SHIFT) + +/* Specific CHA version IDs */ +#define CHA_VER_VID_AES_LP 0x3ull +#define CHA_VER_VID_AES_HP 0x4ull +#define CHA_VER_VID_MD_LP256 0x0ull +#define CHA_VER_VID_MD_LP512 0x1ull +#define CHA_VER_VID_MD_HP 0x2ull + +struct sec_vid { + u16 ip_id; + u8 maj_rev; + u8 min_rev; +}; + +struct caam_perfmon { + /* Performance Monitor Registers f00-f9f */ + u64 req_dequeued; /* PC_REQ_DEQ - Dequeued Requests */ + u64 ob_enc_req; /* PC_OB_ENC_REQ - Outbound Encrypt Requests */ + u64 ib_dec_req; /* PC_IB_DEC_REQ - Inbound Decrypt Requests */ + u64 ob_enc_bytes; /* PC_OB_ENCRYPT - Outbound Bytes Encrypted */ + u64 ob_prot_bytes; /* PC_OB_PROTECT - Outbound Bytes Protected */ + u64 ib_dec_bytes; /* PC_IB_DECRYPT - Inbound Bytes Decrypted */ + u64 ib_valid_bytes; /* PC_IB_VALIDATED Inbound Bytes Validated */ + u64 rsvd[13]; + + /* CAAM Hardware Instantiation Parameters fa0-fbf */ + u32 cha_rev_ms; /* CRNR - CHA Rev No. Most significant half*/ + u32 cha_rev_ls; /* CRNR - CHA Rev No. Least significant half*/ +#define CTPR_MS_QI_SHIFT 25 +#define CTPR_MS_QI_MASK (0x1ull << CTPR_MS_QI_SHIFT) +#define CTPR_MS_PS BIT(17) +#define CTPR_MS_DPAA2 BIT(13) +#define CTPR_MS_VIRT_EN_INCL 0x00000001 +#define CTPR_MS_VIRT_EN_POR 0x00000002 +#define CTPR_MS_PG_SZ_MASK 0x10 +#define CTPR_MS_PG_SZ_SHIFT 4 + u32 comp_parms_ms; /* CTPR - Compile Parameters Register */ +#define CTPR_LS_BLOB BIT(1) + u32 comp_parms_ls; /* CTPR - Compile Parameters Register */ + u64 rsvd1[2]; + + /* CAAM Global Status fc0-fdf */ + u64 faultaddr; /* FAR - Fault Address */ + u32 faultliodn; /* FALR - Fault Address LIODN */ + u32 faultdetail; /* FADR - Fault Addr Detail */ + u32 rsvd2; +#define CSTA_PLEND BIT(10) +#define CSTA_ALT_PLEND BIT(18) +#define CSTA_MOO GENMASK(9, 8) +#define CSTA_MOO_SECURE 1 +#define CSTA_MOO_TRUSTED 2 + u32 status; /* CSTA - CAAM Status */ + u64 rsvd3; + + /* Component Instantiation Parameters fe0-fff */ + u32 rtic_id; /* RVID - RTIC Version ID */ +#define CCBVID_ERA_MASK 0xff000000 +#define CCBVID_ERA_SHIFT 24 + u32 ccb_id; /* CCBVID - CCB Version ID */ + u32 cha_id_ms; /* CHAVID - CHA Version ID Most Significant*/ + u32 cha_id_ls; /* CHAVID - CHA Version ID Least Significant*/ + u32 cha_num_ms; /* CHANUM - CHA Number Most Significant */ + u32 cha_num_ls; /* CHANUM - CHA Number Least Significant*/ +#define SECVID_MS_IPID_MASK 0xffff0000 +#define SECVID_MS_IPID_SHIFT 16 +#define SECVID_MS_MAJ_REV_MASK 0x0000ff00 +#define SECVID_MS_MAJ_REV_SHIFT 8 + u32 caam_id_ms; /* CAAMVID - CAAM Version ID MS */ + u32 caam_id_ls; /* CAAMVID - CAAM Version ID LS */ +}; + +/* LIODN programming for DMA configuration */ +#define MSTRID_LOCK_LIODN 0x80000000 +#define MSTRID_LOCK_MAKETRUSTED 0x00010000 /* only for JR masterid */ + +#define MSTRID_LIODN_MASK 0x0fff +struct masterid { + u32 liodn_ms; /* lock and make-trusted control bits */ + u32 liodn_ls; /* LIODN for non-sequence and seq access */ +}; + +/* RNGB test mode (replicated twice in some configurations) */ +/* Padded out to 0x100 */ +struct rngtst { + u32 mode; /* RTSTMODEx - Test mode */ + u32 rsvd1[3]; + u32 reset; /* RTSTRESETx - Test reset control */ + u32 rsvd2[3]; + u32 status; /* RTSTSSTATUSx - Test status */ + u32 rsvd3; + u32 errstat; /* RTSTERRSTATx - Test error status */ + u32 rsvd4; + u32 errctl; /* RTSTERRCTLx - Test error control */ + u32 rsvd5; + u32 entropy; /* RTSTENTROPYx - Test entropy */ + u32 rsvd6[15]; + u32 verifctl; /* RTSTVERIFCTLx - Test verification control */ + u32 rsvd7; + u32 verifstat; /* RTSTVERIFSTATx - Test verification status */ + u32 rsvd8; + u32 verifdata; /* RTSTVERIFDx - Test verification data */ + u32 rsvd9; + u32 xkey; /* RTSTXKEYx - Test XKEY */ + u32 rsvd10; + u32 oscctctl; /* RTSTOSCCTCTLx - Test osc. counter control */ + u32 rsvd11; + u32 oscct; /* RTSTOSCCTx - Test oscillator counter */ + u32 rsvd12; + u32 oscctstat; /* RTSTODCCTSTATx - Test osc counter status */ + u32 rsvd13[2]; + u32 ofifo[4]; /* RTSTOFIFOx - Test output FIFO */ + u32 rsvd14[15]; +}; + +/* RNG4 TRNG test registers */ +struct rng4tst { +#define RTMCTL_ACC BIT(5) /* TRNG access mode */ +#define RTMCTL_PRGM BIT(16) /* 1 -> program mode, 0 -> run mode */ +#define RTMCTL_SAMP_MODE_VON_NEUMANN_ES_SC 0 /* use von Neumann data in + both entropy shifter and + statistical checker */ +#define RTMCTL_SAMP_MODE_RAW_ES_SC 1 /* use raw data in both + entropy shifter and + statistical checker */ +#define RTMCTL_SAMP_MODE_VON_NEUMANN_ES_RAW_SC 2 /* use von Neumann data in + entropy shifter, raw data + in statistical checker */ +#define RTMCTL_SAMP_MODE_INVALID 3 /* invalid combination */ + u32 rtmctl; /* misc. control register */ + u32 rtscmisc; /* statistical check misc. register */ + u32 rtpkrrng; /* poker range register */ + union { + u32 rtpkrmax; /* PRGM=1: poker max. limit register */ + u32 rtpkrsq; /* PRGM=0: poker square calc. result register */ + }; +#define RTSDCTL_ENT_DLY_SHIFT 16 +#define RTSDCTL_ENT_DLY_MASK (0xffff << RTSDCTL_ENT_DLY_SHIFT) +#define RTSDCTL_ENT_DLY_MIN 3200 +#define RTSDCTL_ENT_DLY_MAX 12800 +#define RTSDCTL_SAMP_SIZE_MASK 0xffff +#define RTSDCTL_SAMP_SIZE_VAL 512 + u32 rtsdctl; /* seed control register */ + union { + u32 rtsblim; /* PRGM=1: sparse bit limit register */ + u32 rttotsam; /* PRGM=0: total samples register */ + }; + u32 rtfrqmin; /* frequency count min. limit register */ +#define RTFRQMAX_DISABLE (1 << 20) + union { + u32 rtfrqmax; /* PRGM=1: freq. count max. limit register */ + u32 rtfrqcnt; /* PRGM=0: freq. count register */ + }; + union { + u32 rtscmc; /* statistical check run monobit count */ + u32 rtscml; /* statistical check run monobit limit */ + }; + union { + u32 rtscrc[6]; /* statistical check run length count */ + u32 rtscrl[6]; /* statistical check run length limit */ + }; + u32 rsvd1[33]; +#define RDSTA_SKVT 0x80000000 +#define RDSTA_SKVN 0x40000000 +#define RDSTA_PR0 BIT(4) +#define RDSTA_PR1 BIT(5) +#define RDSTA_IF0 0x00000001 +#define RDSTA_IF1 0x00000002 +#define RDSTA_MASK (RDSTA_PR1 | RDSTA_PR0 | RDSTA_IF1 | RDSTA_IF0) + u32 rdsta; + u32 rsvd2[15]; +}; + +/* + * caam_ctrl - basic core configuration + * starts base + 0x0000 padded out to 0x1000 + */ + +#define KEK_KEY_SIZE 8 +#define TKEK_KEY_SIZE 8 +#define TDSK_KEY_SIZE 8 + +#define DECO_RESET 1 /* Use with DECO reset/availability regs */ +#define DECO_RESET_0 (DECO_RESET << 0) +#define DECO_RESET_1 (DECO_RESET << 1) +#define DECO_RESET_2 (DECO_RESET << 2) +#define DECO_RESET_3 (DECO_RESET << 3) +#define DECO_RESET_4 (DECO_RESET << 4) + +struct caam_ctrl { + /* Basic Configuration Section 000-01f */ + /* Read/Writable */ + u32 rsvd1; + u32 mcr; /* MCFG Master Config Register */ + u32 rsvd2; + u32 scfgr; /* SCFGR, Security Config Register */ + + /* Bus Access Configuration Section 010-11f */ + /* Read/Writable */ + struct masterid jr_mid[4]; /* JRxLIODNR - JobR LIODN setup */ + u32 rsvd3[11]; + u32 jrstart; /* JRSTART - Job Ring Start Register */ + struct masterid rtic_mid[4]; /* RTICxLIODNR - RTIC LIODN setup */ + u32 rsvd4[5]; + u32 deco_rsr; /* DECORSR - Deco Request Source */ + u32 rsvd11; + u32 deco_rq; /* DECORR - DECO Request */ + struct masterid deco_mid[16]; /* DECOxLIODNR - 1 per DECO */ + + /* DECO Availability/Reset Section 120-3ff */ + u32 deco_avail; /* DAR - DECO availability */ + u32 deco_reset; /* DRR - DECO reset */ + u32 rsvd6[182]; + + /* Key Encryption/Decryption Configuration 400-5ff */ + /* Read/Writable only while in Non-secure mode */ + u32 kek[KEK_KEY_SIZE]; /* JDKEKR - Key Encryption Key */ + u32 tkek[TKEK_KEY_SIZE]; /* TDKEKR - Trusted Desc KEK */ + u32 tdsk[TDSK_KEY_SIZE]; /* TDSKR - Trusted Desc Signing Key */ + u32 rsvd7[32]; + u64 sknonce; /* SKNR - Secure Key Nonce */ + u32 rsvd8[70]; + + /* RNG Test/Verification/Debug Access 600-7ff */ + /* (Useful in Test/Debug modes only...) */ + union { + struct rngtst rtst[2]; + struct rng4tst r4tst[2]; + }; + + u32 rsvd9[416]; + + /* Version registers - introduced with era 10 e80-eff */ + struct version_regs vreg; + /* Performance Monitor f00-fff */ + struct caam_perfmon perfmon; +}; + +/* + * Controller master config register defs + */ +#define MCFGR_SWRESET 0x80000000 /* software reset */ +#define MCFGR_WDENABLE 0x40000000 /* DECO watchdog enable */ +#define MCFGR_WDFAIL 0x20000000 /* DECO watchdog force-fail */ +#define MCFGR_DMA_RESET 0x10000000 +#define MCFGR_LONG_PTR 0x00010000 /* Use >32-bit desc addressing */ +#define SCFGR_RDBENABLE 0x00000400 +#define SCFGR_VIRT_EN 0x00008000 +#define DECORR_RQD0ENABLE 0x00000001 /* Enable DECO0 for direct access */ +#define DECORSR_JR0 0x00000001 /* JR to supply TZ, SDID, ICID */ +#define DECORSR_VALID 0x80000000 +#define DECORR_DEN0 0x00010000 /* DECO0 available for access*/ + +/* AXI read cache control */ +#define MCFGR_ARCACHE_SHIFT 12 +#define MCFGR_ARCACHE_MASK (0xf << MCFGR_ARCACHE_SHIFT) +#define MCFGR_ARCACHE_BUFF (0x1 << MCFGR_ARCACHE_SHIFT) +#define MCFGR_ARCACHE_CACH (0x2 << MCFGR_ARCACHE_SHIFT) +#define MCFGR_ARCACHE_RALL (0x4 << MCFGR_ARCACHE_SHIFT) + +/* AXI write cache control */ +#define MCFGR_AWCACHE_SHIFT 8 +#define MCFGR_AWCACHE_MASK (0xf << MCFGR_AWCACHE_SHIFT) +#define MCFGR_AWCACHE_BUFF (0x1 << MCFGR_AWCACHE_SHIFT) +#define MCFGR_AWCACHE_CACH (0x2 << MCFGR_AWCACHE_SHIFT) +#define MCFGR_AWCACHE_WALL (0x8 << MCFGR_AWCACHE_SHIFT) + +/* AXI pipeline depth */ +#define MCFGR_AXIPIPE_SHIFT 4 +#define MCFGR_AXIPIPE_MASK (0xf << MCFGR_AXIPIPE_SHIFT) + +#define MCFGR_AXIPRI 0x00000008 /* Assert AXI priority sideband */ +#define MCFGR_LARGE_BURST 0x00000004 /* 128/256-byte burst size */ +#define MCFGR_BURST_64 0x00000001 /* 64-byte burst size */ + +/* JRSTART register offsets */ +#define JRSTART_JR0_START 0x00000001 /* Start Job ring 0 */ +#define JRSTART_JR1_START 0x00000002 /* Start Job ring 1 */ +#define JRSTART_JR2_START 0x00000004 /* Start Job ring 2 */ +#define JRSTART_JR3_START 0x00000008 /* Start Job ring 3 */ + +/* + * caam_job_ring - direct job ring setup + * 1-4 possible per instantiation, base + 1000/2000/3000/4000 + * Padded out to 0x1000 + */ +struct caam_job_ring { + /* Input ring */ + u64 inpring_base; /* IRBAx - Input desc ring baseaddr */ + u32 rsvd1; + u32 inpring_size; /* IRSx - Input ring size */ + u32 rsvd2; + u32 inpring_avail; /* IRSAx - Input ring room remaining */ + u32 rsvd3; + u32 inpring_jobadd; /* IRJAx - Input ring jobs added */ + + /* Output Ring */ + u64 outring_base; /* ORBAx - Output status ring base addr */ + u32 rsvd4; + u32 outring_size; /* ORSx - Output ring size */ + u32 rsvd5; + u32 outring_rmvd; /* ORJRx - Output ring jobs removed */ + u32 rsvd6; + u32 outring_used; /* ORSFx - Output ring slots full */ + + /* Status/Configuration */ + u32 rsvd7; + u32 jroutstatus; /* JRSTAx - JobR output status */ + u32 rsvd8; + u32 jrintstatus; /* JRINTx - JobR interrupt status */ + u32 rconfig_hi; /* JRxCFG - Ring configuration */ + u32 rconfig_lo; + + /* Indices. CAAM maintains as "heads" of each queue */ + u32 rsvd9; + u32 inp_rdidx; /* IRRIx - Input ring read index */ + u32 rsvd10; + u32 out_wtidx; /* ORWIx - Output ring write index */ + + /* Command/control */ + u32 rsvd11; + u32 jrcommand; /* JRCRx - JobR command */ + + u32 rsvd12[900]; + + /* Version registers - introduced with era 10 e80-eff */ + struct version_regs vreg; + /* Performance Monitor f00-fff */ + struct caam_perfmon perfmon; +}; + +#define JR_RINGSIZE_MASK 0x03ff +/* + * jrstatus - Job Ring Output Status + * All values in lo word + * Also note, same values written out as status through QI + * in the command/status field of a frame descriptor + */ +#define JRSTA_SSRC_SHIFT 28 +#define JRSTA_SSRC_MASK 0xf0000000 + +#define JRSTA_SSRC_NONE 0x00000000 +#define JRSTA_SSRC_CCB_ERROR 0x20000000 +#define JRSTA_SSRC_JUMP_HALT_USER 0x30000000 +#define JRSTA_SSRC_DECO 0x40000000 +#define JRSTA_SSRC_QI 0x50000000 +#define JRSTA_SSRC_JRERROR 0x60000000 +#define JRSTA_SSRC_JUMP_HALT_CC 0x70000000 + +#define JRSTA_DECOERR_JUMP 0x08000000 +#define JRSTA_DECOERR_INDEX_SHIFT 8 +#define JRSTA_DECOERR_INDEX_MASK 0xff00 +#define JRSTA_DECOERR_ERROR_MASK 0x00ff + +#define JRSTA_DECOERR_NONE 0x00 +#define JRSTA_DECOERR_LINKLEN 0x01 +#define JRSTA_DECOERR_LINKPTR 0x02 +#define JRSTA_DECOERR_JRCTRL 0x03 +#define JRSTA_DECOERR_DESCCMD 0x04 +#define JRSTA_DECOERR_ORDER 0x05 +#define JRSTA_DECOERR_KEYCMD 0x06 +#define JRSTA_DECOERR_LOADCMD 0x07 +#define JRSTA_DECOERR_STORECMD 0x08 +#define JRSTA_DECOERR_OPCMD 0x09 +#define JRSTA_DECOERR_FIFOLDCMD 0x0a +#define JRSTA_DECOERR_FIFOSTCMD 0x0b +#define JRSTA_DECOERR_MOVECMD 0x0c +#define JRSTA_DECOERR_JUMPCMD 0x0d +#define JRSTA_DECOERR_MATHCMD 0x0e +#define JRSTA_DECOERR_SHASHCMD 0x0f +#define JRSTA_DECOERR_SEQCMD 0x10 +#define JRSTA_DECOERR_DECOINTERNAL 0x11 +#define JRSTA_DECOERR_SHDESCHDR 0x12 +#define JRSTA_DECOERR_HDRLEN 0x13 +#define JRSTA_DECOERR_BURSTER 0x14 +#define JRSTA_DECOERR_DESCSIGNATURE 0x15 +#define JRSTA_DECOERR_DMA 0x16 +#define JRSTA_DECOERR_BURSTFIFO 0x17 +#define JRSTA_DECOERR_JRRESET 0x1a +#define JRSTA_DECOERR_JOBFAIL 0x1b +#define JRSTA_DECOERR_DNRERR 0x80 +#define JRSTA_DECOERR_UNDEFPCL 0x81 +#define JRSTA_DECOERR_PDBERR 0x82 +#define JRSTA_DECOERR_ANRPLY_LATE 0x83 +#define JRSTA_DECOERR_ANRPLY_REPLAY 0x84 +#define JRSTA_DECOERR_SEQOVF 0x85 +#define JRSTA_DECOERR_INVSIGN 0x86 +#define JRSTA_DECOERR_DSASIGN 0x87 + +#define JRSTA_QIERR_ERROR_MASK 0x00ff + +#define JRSTA_CCBERR_JUMP 0x08000000 +#define JRSTA_CCBERR_INDEX_MASK 0xff00 +#define JRSTA_CCBERR_INDEX_SHIFT 8 +#define JRSTA_CCBERR_CHAID_MASK 0x00f0 +#define JRSTA_CCBERR_CHAID_SHIFT 4 +#define JRSTA_CCBERR_ERRID_MASK 0x000f + +#define JRSTA_CCBERR_CHAID_AES (0x01 << JRSTA_CCBERR_CHAID_SHIFT) +#define JRSTA_CCBERR_CHAID_DES (0x02 << JRSTA_CCBERR_CHAID_SHIFT) +#define JRSTA_CCBERR_CHAID_ARC4 (0x03 << JRSTA_CCBERR_CHAID_SHIFT) +#define JRSTA_CCBERR_CHAID_MD (0x04 << JRSTA_CCBERR_CHAID_SHIFT) +#define JRSTA_CCBERR_CHAID_RNG (0x05 << JRSTA_CCBERR_CHAID_SHIFT) +#define JRSTA_CCBERR_CHAID_SNOW (0x06 << JRSTA_CCBERR_CHAID_SHIFT) +#define JRSTA_CCBERR_CHAID_KASUMI (0x07 << JRSTA_CCBERR_CHAID_SHIFT) +#define JRSTA_CCBERR_CHAID_PK (0x08 << JRSTA_CCBERR_CHAID_SHIFT) +#define JRSTA_CCBERR_CHAID_CRC (0x09 << JRSTA_CCBERR_CHAID_SHIFT) + +#define JRSTA_CCBERR_ERRID_NONE 0x00 +#define JRSTA_CCBERR_ERRID_MODE 0x01 +#define JRSTA_CCBERR_ERRID_DATASIZ 0x02 +#define JRSTA_CCBERR_ERRID_KEYSIZ 0x03 +#define JRSTA_CCBERR_ERRID_PKAMEMSZ 0x04 +#define JRSTA_CCBERR_ERRID_PKBMEMSZ 0x05 +#define JRSTA_CCBERR_ERRID_SEQUENCE 0x06 +#define JRSTA_CCBERR_ERRID_PKDIVZRO 0x07 +#define JRSTA_CCBERR_ERRID_PKMODEVN 0x08 +#define JRSTA_CCBERR_ERRID_KEYPARIT 0x09 +#define JRSTA_CCBERR_ERRID_ICVCHK 0x0a +#define JRSTA_CCBERR_ERRID_HARDWARE 0x0b +#define JRSTA_CCBERR_ERRID_CCMAAD 0x0c +#define JRSTA_CCBERR_ERRID_INVCHA 0x0f + +#define JRINT_ERR_INDEX_MASK 0x3fff0000 +#define JRINT_ERR_INDEX_SHIFT 16 +#define JRINT_ERR_TYPE_MASK 0xf00 +#define JRINT_ERR_TYPE_SHIFT 8 +#define JRINT_ERR_HALT_MASK 0xc +#define JRINT_ERR_HALT_SHIFT 2 +#define JRINT_ERR_HALT_INPROGRESS 0x4 +#define JRINT_ERR_HALT_COMPLETE 0x8 +#define JRINT_JR_ERROR 0x02 +#define JRINT_JR_INT 0x01 + +#define JRINT_ERR_TYPE_WRITE 1 +#define JRINT_ERR_TYPE_BAD_INPADDR 3 +#define JRINT_ERR_TYPE_BAD_OUTADDR 4 +#define JRINT_ERR_TYPE_INV_INPWRT 5 +#define JRINT_ERR_TYPE_INV_OUTWRT 6 +#define JRINT_ERR_TYPE_RESET 7 +#define JRINT_ERR_TYPE_REMOVE_OFL 8 +#define JRINT_ERR_TYPE_ADD_OFL 9 + +#define JRCFG_SOE 0x04 +#define JRCFG_ICEN 0x02 +#define JRCFG_IMSK 0x01 +#define JRCFG_ICDCT_SHIFT 8 +#define JRCFG_ICTT_SHIFT 16 + +#define JRCR_RESET 0x01 + +/* + * caam_assurance - Assurance Controller View + * base + 0x6000 padded out to 0x1000 + */ + +struct rtic_element { + u64 address; + u32 rsvd; + u32 length; +}; + +struct rtic_block { + struct rtic_element element[2]; +}; + +struct rtic_memhash { + u32 memhash_be[32]; + u32 memhash_le[32]; +}; + +struct caam_assurance { + /* Status/Command/Watchdog */ + u32 rsvd1; + u32 status; /* RSTA - Status */ + u32 rsvd2; + u32 cmd; /* RCMD - Command */ + u32 rsvd3; + u32 ctrl; /* RCTL - Control */ + u32 rsvd4; + u32 throttle; /* RTHR - Throttle */ + u32 rsvd5[2]; + u64 watchdog; /* RWDOG - Watchdog Timer */ + u32 rsvd6; + u32 rend; /* REND - Endian corrections */ + u32 rsvd7[50]; + + /* Block access/configuration @ 100/110/120/130 */ + struct rtic_block memblk[4]; /* Memory Blocks A-D */ + u32 rsvd8[32]; + + /* Block hashes @ 200/300/400/500 */ + struct rtic_memhash hash[4]; /* Block hash values A-D */ + u32 rsvd_3[640]; +}; + +/* + * caam_queue_if - QI configuration and control + * starts base + 0x7000, padded out to 0x1000 long + */ + +struct caam_queue_if { + u32 qi_control_hi; /* QICTL - QI Control */ + u32 qi_control_lo; + u32 rsvd1; + u32 qi_status; /* QISTA - QI Status */ + u32 qi_deq_cfg_hi; /* QIDQC - QI Dequeue Configuration */ + u32 qi_deq_cfg_lo; + u32 qi_enq_cfg_hi; /* QISEQC - QI Enqueue Command */ + u32 qi_enq_cfg_lo; + u32 rsvd2[1016]; +}; + +/* QI control bits - low word */ +#define QICTL_DQEN 0x01 /* Enable frame pop */ +#define QICTL_STOP 0x02 /* Stop dequeue/enqueue */ +#define QICTL_SOE 0x04 /* Stop on error */ + +/* QI control bits - high word */ +#define QICTL_MBSI 0x01 +#define QICTL_MHWSI 0x02 +#define QICTL_MWSI 0x04 +#define QICTL_MDWSI 0x08 +#define QICTL_CBSI 0x10 /* CtrlDataByteSwapInput */ +#define QICTL_CHWSI 0x20 /* CtrlDataHalfSwapInput */ +#define QICTL_CWSI 0x40 /* CtrlDataWordSwapInput */ +#define QICTL_CDWSI 0x80 /* CtrlDataDWordSwapInput */ +#define QICTL_MBSO 0x0100 +#define QICTL_MHWSO 0x0200 +#define QICTL_MWSO 0x0400 +#define QICTL_MDWSO 0x0800 +#define QICTL_CBSO 0x1000 /* CtrlDataByteSwapOutput */ +#define QICTL_CHWSO 0x2000 /* CtrlDataHalfSwapOutput */ +#define QICTL_CWSO 0x4000 /* CtrlDataWordSwapOutput */ +#define QICTL_CDWSO 0x8000 /* CtrlDataDWordSwapOutput */ +#define QICTL_DMBS 0x010000 +#define QICTL_EPO 0x020000 + +/* QI status bits */ +#define QISTA_PHRDERR 0x01 /* PreHeader Read Error */ +#define QISTA_CFRDERR 0x02 /* Compound Frame Read Error */ +#define QISTA_OFWRERR 0x04 /* Output Frame Read Error */ +#define QISTA_BPDERR 0x08 /* Buffer Pool Depleted */ +#define QISTA_BTSERR 0x10 /* Buffer Undersize */ +#define QISTA_CFWRERR 0x20 /* Compound Frame Write Err */ +#define QISTA_STOPD 0x80000000 /* QI Stopped (see QICTL) */ + +/* deco_sg_table - DECO view of scatter/gather table */ +struct deco_sg_table { + u64 addr; /* Segment Address */ + u32 elen; /* E, F bits + 30-bit length */ + u32 bpid_offset; /* Buffer Pool ID + 16-bit length */ +}; + +/* + * caam_deco - descriptor controller - CHA cluster block + * + * Only accessible when direct DECO access is turned on + * (done in DECORR, via MID programmed in DECOxMID + * + * 5 typical, base + 0x8000/9000/a000/b000 + * Padded out to 0x1000 long + */ +struct caam_deco { + u32 rsvd1; + u32 cls1_mode; /* CxC1MR - Class 1 Mode */ + u32 rsvd2; + u32 cls1_keysize; /* CxC1KSR - Class 1 Key Size */ + u32 cls1_datasize_hi; /* CxC1DSR - Class 1 Data Size */ + u32 cls1_datasize_lo; + u32 rsvd3; + u32 cls1_icvsize; /* CxC1ICVSR - Class 1 ICV size */ + u32 rsvd4[5]; + u32 cha_ctrl; /* CCTLR - CHA control */ + u32 rsvd5; + u32 irq_crtl; /* CxCIRQ - CCB interrupt done/error/clear */ + u32 rsvd6; + u32 clr_written; /* CxCWR - Clear-Written */ + u32 ccb_status_hi; /* CxCSTA - CCB Status/Error */ + u32 ccb_status_lo; + u32 rsvd7[3]; + u32 aad_size; /* CxAADSZR - Current AAD Size */ + u32 rsvd8; + u32 cls1_iv_size; /* CxC1IVSZR - Current Class 1 IV Size */ + u32 rsvd9[7]; + u32 pkha_a_size; /* PKASZRx - Size of PKHA A */ + u32 rsvd10; + u32 pkha_b_size; /* PKBSZRx - Size of PKHA B */ + u32 rsvd11; + u32 pkha_n_size; /* PKNSZRx - Size of PKHA N */ + u32 rsvd12; + u32 pkha_e_size; /* PKESZRx - Size of PKHA E */ + u32 rsvd13[24]; + u32 cls1_ctx[16]; /* CxC1CTXR - Class 1 Context @100 */ + u32 rsvd14[48]; + u32 cls1_key[8]; /* CxC1KEYR - Class 1 Key @200 */ + u32 rsvd15[121]; + u32 cls2_mode; /* CxC2MR - Class 2 Mode */ + u32 rsvd16; + u32 cls2_keysize; /* CxX2KSR - Class 2 Key Size */ + u32 cls2_datasize_hi; /* CxC2DSR - Class 2 Data Size */ + u32 cls2_datasize_lo; + u32 rsvd17; + u32 cls2_icvsize; /* CxC2ICVSZR - Class 2 ICV Size */ + u32 rsvd18[56]; + u32 cls2_ctx[18]; /* CxC2CTXR - Class 2 Context @500 */ + u32 rsvd19[46]; + u32 cls2_key[32]; /* CxC2KEYR - Class2 Key @600 */ + u32 rsvd20[84]; + u32 inp_infofifo_hi; /* CxIFIFO - Input Info FIFO @7d0 */ + u32 inp_infofifo_lo; + u32 rsvd21[2]; + u64 inp_datafifo; /* CxDFIFO - Input Data FIFO */ + u32 rsvd22[2]; + u64 out_datafifo; /* CxOFIFO - Output Data FIFO */ + u32 rsvd23[2]; + u32 jr_ctl_hi; /* CxJRR - JobR Control Register @800 */ + u32 jr_ctl_lo; + u64 jr_descaddr; /* CxDADR - JobR Descriptor Address */ +#define DECO_OP_STATUS_HI_ERR_MASK 0xF00000FF + u32 op_status_hi; /* DxOPSTA - DECO Operation Status */ + u32 op_status_lo; + u32 rsvd24[2]; + u32 liodn; /* DxLSR - DECO LIODN Status - non-seq */ + u32 td_liodn; /* DxLSR - DECO LIODN Status - trustdesc */ + u32 rsvd26[6]; + u64 math[4]; /* DxMTH - Math register */ + u32 rsvd27[8]; + struct deco_sg_table gthr_tbl[4]; /* DxGTR - Gather Tables */ + u32 rsvd28[16]; + struct deco_sg_table sctr_tbl[4]; /* DxSTR - Scatter Tables */ + u32 rsvd29[48]; + u32 descbuf[64]; /* DxDESB - Descriptor buffer */ + u32 rscvd30[193]; +#define DESC_DBG_DECO_STAT_VALID 0x80000000 +#define DESC_DBG_DECO_STAT_MASK 0x00F00000 +#define DESC_DBG_DECO_STAT_SHIFT 20 + u32 desc_dbg; /* DxDDR - DECO Debug Register */ + u32 rsvd31[13]; +#define DESC_DER_DECO_STAT_MASK 0x000F0000 +#define DESC_DER_DECO_STAT_SHIFT 16 + u32 dbg_exec; /* DxDER - DECO Debug Exec Register */ + u32 rsvd32[112]; +}; + +#define DECO_STAT_HOST_ERR 0xD + +#define DECO_JQCR_WHL 0x20000000 +#define DECO_JQCR_FOUR 0x10000000 + +#define JR_BLOCK_NUMBER 1 +#define ASSURE_BLOCK_NUMBER 6 +#define QI_BLOCK_NUMBER 7 +#define DECO_BLOCK_NUMBER 8 +#define PG_SIZE_4K 0x1000 +#define PG_SIZE_64K 0x10000 +#endif /* REGS_H */ diff --git a/drivers/crypto/caam/sg_sw_qm.h b/drivers/crypto/caam/sg_sw_qm.h new file mode 100644 index 000000000..d56cc7efb --- /dev/null +++ b/drivers/crypto/caam/sg_sw_qm.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */ +/* + * Copyright 2013-2016 Freescale Semiconductor, Inc. + * Copyright 2016-2017 NXP + */ + +#ifndef __SG_SW_QM_H +#define __SG_SW_QM_H + +#include +#include "regs.h" + +static inline void __dma_to_qm_sg(struct qm_sg_entry *qm_sg_ptr, dma_addr_t dma, + u16 offset) +{ + qm_sg_entry_set64(qm_sg_ptr, dma); + qm_sg_ptr->__reserved2 = 0; + qm_sg_ptr->bpid = 0; + qm_sg_ptr->offset = cpu_to_be16(offset & QM_SG_OFF_MASK); +} + +static inline void dma_to_qm_sg_one(struct qm_sg_entry *qm_sg_ptr, + dma_addr_t dma, u32 len, u16 offset) +{ + __dma_to_qm_sg(qm_sg_ptr, dma, offset); + qm_sg_entry_set_len(qm_sg_ptr, len); +} + +static inline void dma_to_qm_sg_one_last(struct qm_sg_entry *qm_sg_ptr, + dma_addr_t dma, u32 len, u16 offset) +{ + __dma_to_qm_sg(qm_sg_ptr, dma, offset); + qm_sg_entry_set_f(qm_sg_ptr, len); +} + +static inline void dma_to_qm_sg_one_ext(struct qm_sg_entry *qm_sg_ptr, + dma_addr_t dma, u32 len, u16 offset) +{ + __dma_to_qm_sg(qm_sg_ptr, dma, offset); + qm_sg_ptr->cfg = cpu_to_be32(QM_SG_EXT | (len & QM_SG_LEN_MASK)); +} + +static inline void dma_to_qm_sg_one_last_ext(struct qm_sg_entry *qm_sg_ptr, + dma_addr_t dma, u32 len, + u16 offset) +{ + __dma_to_qm_sg(qm_sg_ptr, dma, offset); + qm_sg_ptr->cfg = cpu_to_be32(QM_SG_EXT | QM_SG_FIN | + (len & QM_SG_LEN_MASK)); +} + +/* + * convert scatterlist to h/w link table format + * but does not have final bit; instead, returns last entry + */ +static inline struct qm_sg_entry * +sg_to_qm_sg(struct scatterlist *sg, int len, + struct qm_sg_entry *qm_sg_ptr, u16 offset) +{ + int ent_len; + + while (len) { + ent_len = min_t(int, sg_dma_len(sg), len); + + dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), ent_len, + offset); + qm_sg_ptr++; + sg = sg_next(sg); + len -= ent_len; + } + return qm_sg_ptr - 1; +} + +/* + * convert scatterlist to h/w link table format + * scatterlist must have been previously dma mapped + */ +static inline void sg_to_qm_sg_last(struct scatterlist *sg, int len, + struct qm_sg_entry *qm_sg_ptr, u16 offset) +{ + qm_sg_ptr = sg_to_qm_sg(sg, len, qm_sg_ptr, offset); + qm_sg_entry_set_f(qm_sg_ptr, qm_sg_entry_get_len(qm_sg_ptr)); +} + +#endif /* __SG_SW_QM_H */ diff --git a/drivers/crypto/caam/sg_sw_qm2.h b/drivers/crypto/caam/sg_sw_qm2.h new file mode 100644 index 000000000..b8b737d2b --- /dev/null +++ b/drivers/crypto/caam/sg_sw_qm2.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */ +/* + * Copyright 2015-2016 Freescale Semiconductor, Inc. + * Copyright 2017 NXP + */ + +#ifndef _SG_SW_QM2_H_ +#define _SG_SW_QM2_H_ + +#include + +static inline void dma_to_qm_sg_one(struct dpaa2_sg_entry *qm_sg_ptr, + dma_addr_t dma, u32 len, u16 offset) +{ + dpaa2_sg_set_addr(qm_sg_ptr, dma); + dpaa2_sg_set_format(qm_sg_ptr, dpaa2_sg_single); + dpaa2_sg_set_final(qm_sg_ptr, false); + dpaa2_sg_set_len(qm_sg_ptr, len); + dpaa2_sg_set_bpid(qm_sg_ptr, 0); + dpaa2_sg_set_offset(qm_sg_ptr, offset); +} + +/* + * convert scatterlist to h/w link table format + * but does not have final bit; instead, returns last entry + */ +static inline struct dpaa2_sg_entry * +sg_to_qm_sg(struct scatterlist *sg, int len, + struct dpaa2_sg_entry *qm_sg_ptr, u16 offset) +{ + int ent_len; + + while (len) { + ent_len = min_t(int, sg_dma_len(sg), len); + + dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), ent_len, + offset); + qm_sg_ptr++; + sg = sg_next(sg); + len -= ent_len; + } + return qm_sg_ptr - 1; +} + +/* + * convert scatterlist to h/w link table format + * scatterlist must have been previously dma mapped + */ +static inline void sg_to_qm_sg_last(struct scatterlist *sg, int len, + struct dpaa2_sg_entry *qm_sg_ptr, + u16 offset) +{ + qm_sg_ptr = sg_to_qm_sg(sg, len, qm_sg_ptr, offset); + dpaa2_sg_set_final(qm_sg_ptr, true); +} + +#endif /* _SG_SW_QM2_H_ */ diff --git a/drivers/crypto/caam/sg_sw_sec4.h b/drivers/crypto/caam/sg_sw_sec4.h new file mode 100644 index 000000000..07e1ee992 --- /dev/null +++ b/drivers/crypto/caam/sg_sw_sec4.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * CAAM/SEC 4.x functions for using scatterlists in caam driver + * + * Copyright 2008-2011 Freescale Semiconductor, Inc. + * + */ + +#ifndef _SG_SW_SEC4_H_ +#define _SG_SW_SEC4_H_ + +#include "ctrl.h" +#include "regs.h" +#include "sg_sw_qm2.h" +#include + +struct sec4_sg_entry { + u64 ptr; + u32 len; + u32 bpid_offset; +}; + +/* + * convert single dma address to h/w link table format + */ +static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr, + dma_addr_t dma, u32 len, u16 offset) +{ + if (caam_dpaa2) { + dma_to_qm_sg_one((struct dpaa2_sg_entry *)sec4_sg_ptr, dma, len, + offset); + } else { + sec4_sg_ptr->ptr = cpu_to_caam_dma64(dma); + sec4_sg_ptr->len = cpu_to_caam32(len); + sec4_sg_ptr->bpid_offset = cpu_to_caam32(offset & + SEC4_SG_OFFSET_MASK); + } + + print_hex_dump_debug("sec4_sg_ptr@: ", DUMP_PREFIX_ADDRESS, 16, 4, + sec4_sg_ptr, sizeof(struct sec4_sg_entry), 1); +} + +/* + * convert scatterlist to h/w link table format + * but does not have final bit; instead, returns last entry + */ +static inline struct sec4_sg_entry * +sg_to_sec4_sg(struct scatterlist *sg, int len, + struct sec4_sg_entry *sec4_sg_ptr, u16 offset) +{ + int ent_len; + + while (len) { + ent_len = min_t(int, sg_dma_len(sg), len); + + dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg), ent_len, + offset); + sec4_sg_ptr++; + sg = sg_next(sg); + len -= ent_len; + } + return sec4_sg_ptr - 1; +} + +static inline void sg_to_sec4_set_last(struct sec4_sg_entry *sec4_sg_ptr) +{ + if (caam_dpaa2) + dpaa2_sg_set_final((struct dpaa2_sg_entry *)sec4_sg_ptr, true); + else + sec4_sg_ptr->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); +} + +/* + * convert scatterlist to h/w link table format + * scatterlist must have been previously dma mapped + */ +static inline void sg_to_sec4_sg_last(struct scatterlist *sg, int len, + struct sec4_sg_entry *sec4_sg_ptr, + u16 offset) +{ + sec4_sg_ptr = sg_to_sec4_sg(sg, len, sec4_sg_ptr, offset); + sg_to_sec4_set_last(sec4_sg_ptr); +} + +#endif /* _SG_SW_SEC4_H_ */ diff --git a/drivers/crypto/cavium/Makefile b/drivers/crypto/cavium/Makefile new file mode 100644 index 000000000..75227c587 --- /dev/null +++ b/drivers/crypto/cavium/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for Cavium crypto device drivers +# +obj-$(CONFIG_CRYPTO_DEV_CPT) += cpt/ +obj-$(CONFIG_CRYPTO_DEV_NITROX) += nitrox/ diff --git a/drivers/crypto/cavium/cpt/Kconfig b/drivers/crypto/cavium/cpt/Kconfig new file mode 100644 index 000000000..b9874058d --- /dev/null +++ b/drivers/crypto/cavium/cpt/Kconfig @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Cavium crypto device configuration +# + +config CRYPTO_DEV_CPT + tristate + +config CAVIUM_CPT + tristate "Cavium Cryptographic Accelerator driver" + depends on ARCH_THUNDER || COMPILE_TEST + depends on PCI_MSI && 64BIT + select CRYPTO_DEV_CPT + help + Support for Cavium CPT block found in octeon-tx series of + processors. + + To compile this as a module, choose M here. diff --git a/drivers/crypto/cavium/cpt/Makefile b/drivers/crypto/cavium/cpt/Makefile new file mode 100644 index 000000000..0f04f1b5c --- /dev/null +++ b/drivers/crypto/cavium/cpt/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CAVIUM_CPT) += cptpf.o cptvf.o +cptpf-objs := cptpf_main.o cptpf_mbox.o +cptvf-objs := cptvf_main.o cptvf_reqmanager.o cptvf_mbox.o cptvf_algs.o diff --git a/drivers/crypto/cavium/cpt/cpt_common.h b/drivers/crypto/cavium/cpt/cpt_common.h new file mode 100644 index 000000000..eb3361518 --- /dev/null +++ b/drivers/crypto/cavium/cpt/cpt_common.h @@ -0,0 +1,153 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2016 Cavium, Inc. + */ + +#ifndef __CPT_COMMON_H +#define __CPT_COMMON_H + +#include +#include +#include + +#include "cpt_hw_types.h" + +/* Device ID */ +#define CPT_81XX_PCI_PF_DEVICE_ID 0xa040 +#define CPT_81XX_PCI_VF_DEVICE_ID 0xa041 + +/* flags to indicate the features supported */ +#define CPT_FLAG_SRIOV_ENABLED BIT(1) +#define CPT_FLAG_VF_DRIVER BIT(2) +#define CPT_FLAG_DEVICE_READY BIT(3) + +#define cpt_sriov_enabled(cpt) ((cpt)->flags & CPT_FLAG_SRIOV_ENABLED) +#define cpt_vf_driver(cpt) ((cpt)->flags & CPT_FLAG_VF_DRIVER) +#define cpt_device_ready(cpt) ((cpt)->flags & CPT_FLAG_DEVICE_READY) + +#define CPT_MBOX_MSG_TYPE_ACK 1 +#define CPT_MBOX_MSG_TYPE_NACK 2 +#define CPT_MBOX_MSG_TIMEOUT 2000 +#define VF_STATE_DOWN 0 +#define VF_STATE_UP 1 + +/* + * CPT Registers map for 81xx + */ + +/* PF registers */ +#define CPTX_PF_CONSTANTS(a) (0x0ll + ((u64)(a) << 36)) +#define CPTX_PF_RESET(a) (0x100ll + ((u64)(a) << 36)) +#define CPTX_PF_DIAG(a) (0x120ll + ((u64)(a) << 36)) +#define CPTX_PF_BIST_STATUS(a) (0x160ll + ((u64)(a) << 36)) +#define CPTX_PF_ECC0_CTL(a) (0x200ll + ((u64)(a) << 36)) +#define CPTX_PF_ECC0_FLIP(a) (0x210ll + ((u64)(a) << 36)) +#define CPTX_PF_ECC0_INT(a) (0x220ll + ((u64)(a) << 36)) +#define CPTX_PF_ECC0_INT_W1S(a) (0x230ll + ((u64)(a) << 36)) +#define CPTX_PF_ECC0_ENA_W1S(a) (0x240ll + ((u64)(a) << 36)) +#define CPTX_PF_ECC0_ENA_W1C(a) (0x250ll + ((u64)(a) << 36)) +#define CPTX_PF_MBOX_INTX(a, b) \ + (0x400ll + ((u64)(a) << 36) + ((b) << 3)) +#define CPTX_PF_MBOX_INT_W1SX(a, b) \ + (0x420ll + ((u64)(a) << 36) + ((b) << 3)) +#define CPTX_PF_MBOX_ENA_W1CX(a, b) \ + (0x440ll + ((u64)(a) << 36) + ((b) << 3)) +#define CPTX_PF_MBOX_ENA_W1SX(a, b) \ + (0x460ll + ((u64)(a) << 36) + ((b) << 3)) +#define CPTX_PF_EXEC_INT(a) (0x500ll + 0x1000000000ll * ((a) & 0x1)) +#define CPTX_PF_EXEC_INT_W1S(a) (0x520ll + ((u64)(a) << 36)) +#define CPTX_PF_EXEC_ENA_W1C(a) (0x540ll + ((u64)(a) << 36)) +#define CPTX_PF_EXEC_ENA_W1S(a) (0x560ll + ((u64)(a) << 36)) +#define CPTX_PF_GX_EN(a, b) \ + (0x600ll + ((u64)(a) << 36) + ((b) << 3)) +#define CPTX_PF_EXEC_INFO(a) (0x700ll + ((u64)(a) << 36)) +#define CPTX_PF_EXEC_BUSY(a) (0x800ll + ((u64)(a) << 36)) +#define CPTX_PF_EXEC_INFO0(a) (0x900ll + ((u64)(a) << 36)) +#define CPTX_PF_EXEC_INFO1(a) (0x910ll + ((u64)(a) << 36)) +#define CPTX_PF_INST_REQ_PC(a) (0x10000ll + ((u64)(a) << 36)) +#define CPTX_PF_INST_LATENCY_PC(a) \ + (0x10020ll + ((u64)(a) << 36)) +#define CPTX_PF_RD_REQ_PC(a) (0x10040ll + ((u64)(a) << 36)) +#define CPTX_PF_RD_LATENCY_PC(a) (0x10060ll + ((u64)(a) << 36)) +#define CPTX_PF_RD_UC_PC(a) (0x10080ll + ((u64)(a) << 36)) +#define CPTX_PF_ACTIVE_CYCLES_PC(a) (0x10100ll + ((u64)(a) << 36)) +#define CPTX_PF_EXE_CTL(a) (0x4000000ll + ((u64)(a) << 36)) +#define CPTX_PF_EXE_STATUS(a) (0x4000008ll + ((u64)(a) << 36)) +#define CPTX_PF_EXE_CLK(a) (0x4000010ll + ((u64)(a) << 36)) +#define CPTX_PF_EXE_DBG_CTL(a) (0x4000018ll + ((u64)(a) << 36)) +#define CPTX_PF_EXE_DBG_DATA(a) (0x4000020ll + ((u64)(a) << 36)) +#define CPTX_PF_EXE_BIST_STATUS(a) (0x4000028ll + ((u64)(a) << 36)) +#define CPTX_PF_EXE_REQ_TIMER(a) (0x4000030ll + ((u64)(a) << 36)) +#define CPTX_PF_EXE_MEM_CTL(a) (0x4000038ll + ((u64)(a) << 36)) +#define CPTX_PF_EXE_PERF_CTL(a) (0x4001000ll + ((u64)(a) << 36)) +#define CPTX_PF_EXE_DBG_CNTX(a, b) \ + (0x4001100ll + ((u64)(a) << 36) + ((b) << 3)) +#define CPTX_PF_EXE_PERF_EVENT_CNT(a) (0x4001180ll + ((u64)(a) << 36)) +#define CPTX_PF_EXE_EPCI_INBX_CNT(a, b) \ + (0x4001200ll + ((u64)(a) << 36) + ((b) << 3)) +#define CPTX_PF_EXE_EPCI_OUTBX_CNT(a, b) \ + (0x4001240ll + ((u64)(a) << 36) + ((b) << 3)) +#define CPTX_PF_ENGX_UCODE_BASE(a, b) \ + (0x4002000ll + ((u64)(a) << 36) + ((b) << 3)) +#define CPTX_PF_QX_CTL(a, b) \ + (0x8000000ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_PF_QX_GMCTL(a, b) \ + (0x8000020ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_PF_QX_CTL2(a, b) \ + (0x8000100ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_PF_VFX_MBOXX(a, b, c) \ + (0x8001000ll + ((u64)(a) << 36) + ((b) << 20) + ((c) << 8)) + +/* VF registers */ +#define CPTX_VQX_CTL(a, b) (0x100ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_SADDR(a, b) (0x200ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_DONE_WAIT(a, b) (0x400ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_INPROG(a, b) (0x410ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_DONE(a, b) (0x420ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_DONE_ACK(a, b) (0x440ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_DONE_INT_W1S(a, b) (0x460ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_DONE_INT_W1C(a, b) (0x468ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_DONE_ENA_W1S(a, b) (0x470ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_DONE_ENA_W1C(a, b) (0x478ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_MISC_INT(a, b) (0x500ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_MISC_INT_W1S(a, b) (0x508ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_MISC_ENA_W1S(a, b) (0x510ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_MISC_ENA_W1C(a, b) (0x518ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VQX_DOORBELL(a, b) (0x600ll + ((u64)(a) << 36) + ((b) << 20)) +#define CPTX_VFX_PF_MBOXX(a, b, c) \ + (0x1000ll + ((u64)(a) << 36) + ((b) << 20) + ((c) << 3)) + +enum vftype { + AE_TYPES = 1, + SE_TYPES = 2, + BAD_CPT_TYPES, +}; + +/* Max CPT devices supported */ +enum cpt_mbox_opcode { + CPT_MSG_VF_UP = 1, + CPT_MSG_VF_DOWN, + CPT_MSG_READY, + CPT_MSG_QLEN, + CPT_MSG_QBIND_GRP, + CPT_MSG_VQ_PRIORITY, +}; + +/* CPT mailbox structure */ +struct cpt_mbox { + u64 msg; /* Message type MBOX[0] */ + u64 data;/* Data MBOX[1] */ +}; + +/* Register read/write APIs */ +static inline void cpt_write_csr64(u8 __iomem *hw_addr, u64 offset, + u64 val) +{ + writeq(val, hw_addr + offset); +} + +static inline u64 cpt_read_csr64(u8 __iomem *hw_addr, u64 offset) +{ + return readq(hw_addr + offset); +} +#endif /* __CPT_COMMON_H */ diff --git a/drivers/crypto/cavium/cpt/cpt_hw_types.h b/drivers/crypto/cavium/cpt/cpt_hw_types.h new file mode 100644 index 000000000..ae4791a8e --- /dev/null +++ b/drivers/crypto/cavium/cpt/cpt_hw_types.h @@ -0,0 +1,655 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2016 Cavium, Inc. + */ + +#ifndef __CPT_HW_TYPES_H +#define __CPT_HW_TYPES_H + +#include "cpt_common.h" + +/** + * Enumeration cpt_comp_e + * + * CPT Completion Enumeration + * Enumerates the values of CPT_RES_S[COMPCODE]. + */ +enum cpt_comp_e { + CPT_COMP_E_NOTDONE = 0x00, + CPT_COMP_E_GOOD = 0x01, + CPT_COMP_E_FAULT = 0x02, + CPT_COMP_E_SWERR = 0x03, + CPT_COMP_E_LAST_ENTRY = 0xFF +}; + +/** + * Structure cpt_inst_s + * + * CPT Instruction Structure + * This structure specifies the instruction layout. Instructions are + * stored in memory as little-endian unless CPT()_PF_Q()_CTL[INST_BE] is set. + * cpt_inst_s_s + * Word 0 + * doneint:1 Done interrupt. + * 0 = No interrupts related to this instruction. + * 1 = When the instruction completes, CPT()_VQ()_DONE[DONE] will be + * incremented,and based on the rules described there an interrupt may + * occur. + * Word 1 + * res_addr [127: 64] Result IOVA. + * If nonzero, specifies where to write CPT_RES_S. + * If zero, no result structure will be written. + * Address must be 16-byte aligned. + * Bits <63:49> are ignored by hardware; software should use a + * sign-extended bit <48> for forward compatibility. + * Word 2 + * grp:10 [171:162] If [WQ_PTR] is nonzero, the SSO guest-group to use when + * CPT submits work SSO. + * For the SSO to not discard the add-work request, FPA_PF_MAP() must map + * [GRP] and CPT()_PF_Q()_GMCTL[GMID] as valid. + * tt:2 [161:160] If [WQ_PTR] is nonzero, the SSO tag type to use when CPT + * submits work to SSO + * tag:32 [159:128] If [WQ_PTR] is nonzero, the SSO tag to use when CPT + * submits work to SSO. + * Word 3 + * wq_ptr [255:192] If [WQ_PTR] is nonzero, it is a pointer to a + * work-queue entry that CPT submits work to SSO after all context, + * output data, and result write operations are visible to other + * CNXXXX units and the cores. Bits <2:0> must be zero. + * Bits <63:49> are ignored by hardware; software should + * use a sign-extended bit <48> for forward compatibility. + * Internal: + * Bits <63:49>, <2:0> are ignored by hardware, treated as always 0x0. + * Word 4 + * ei0; [319:256] Engine instruction word 0. Passed to the AE/SE. + * Word 5 + * ei1; [383:320] Engine instruction word 1. Passed to the AE/SE. + * Word 6 + * ei2; [447:384] Engine instruction word 1. Passed to the AE/SE. + * Word 7 + * ei3; [511:448] Engine instruction word 1. Passed to the AE/SE. + * + */ +union cpt_inst_s { + u64 u[8]; + struct cpt_inst_s_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_17_63:47; + u64 doneint:1; + u64 reserved_0_1:16; +#else /* Word 0 - Little Endian */ + u64 reserved_0_15:16; + u64 doneint:1; + u64 reserved_17_63:47; +#endif /* Word 0 - End */ + u64 res_addr; +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 2 - Big Endian */ + u64 reserved_172_19:20; + u64 grp:10; + u64 tt:2; + u64 tag:32; +#else /* Word 2 - Little Endian */ + u64 tag:32; + u64 tt:2; + u64 grp:10; + u64 reserved_172_191:20; +#endif /* Word 2 - End */ + u64 wq_ptr; + u64 ei0; + u64 ei1; + u64 ei2; + u64 ei3; + } s; +}; + +/** + * Structure cpt_res_s + * + * CPT Result Structure + * The CPT coprocessor writes the result structure after it completes a + * CPT_INST_S instruction. The result structure is exactly 16 bytes, and + * each instruction completion produces exactly one result structure. + * + * This structure is stored in memory as little-endian unless + * CPT()_PF_Q()_CTL[INST_BE] is set. + * cpt_res_s_s + * Word 0 + * doneint:1 [16:16] Done interrupt. This bit is copied from the + * corresponding instruction's CPT_INST_S[DONEINT]. + * compcode:8 [7:0] Indicates completion/error status of the CPT coprocessor + * for the associated instruction, as enumerated by CPT_COMP_E. + * Core software may write the memory location containing [COMPCODE] to + * 0x0 before ringing the doorbell, and then poll for completion by + * checking for a nonzero value. + * Once the core observes a nonzero [COMPCODE] value in this case,the CPT + * coprocessor will have also completed L2/DRAM write operations. + * Word 1 + * reserved + * + */ +union cpt_res_s { + u64 u[2]; + struct cpt_res_s_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_17_63:47; + u64 doneint:1; + u64 reserved_8_15:8; + u64 compcode:8; +#else /* Word 0 - Little Endian */ + u64 compcode:8; + u64 reserved_8_15:8; + u64 doneint:1; + u64 reserved_17_63:47; +#endif /* Word 0 - End */ + u64 reserved_64_127; + } s; +}; + +/** + * Register (NCB) cpt#_pf_bist_status + * + * CPT PF Control Bist Status Register + * This register has the BIST status of memories. Each bit is the BIST result + * of an individual memory (per bit, 0 = pass and 1 = fail). + * cptx_pf_bist_status_s + * Word0 + * bstatus [29:0](RO/H) BIST status. One bit per memory, enumerated by + * CPT_RAMS_E. + */ +union cptx_pf_bist_status { + u64 u; + struct cptx_pf_bist_status_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_30_63:34; + u64 bstatus:30; +#else /* Word 0 - Little Endian */ + u64 bstatus:30; + u64 reserved_30_63:34; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_pf_constants + * + * CPT PF Constants Register + * This register contains implementation-related parameters of CPT in CNXXXX. + * cptx_pf_constants_s + * Word 0 + * reserved_40_63:24 [63:40] Reserved. + * epcis:8 [39:32](RO) Number of EPCI busses. + * grps:8 [31:24](RO) Number of engine groups implemented. + * ae:8 [23:16](RO/H) Number of AEs. In CNXXXX, for CPT0 returns 0x0, + * for CPT1 returns 0x18, or less if there are fuse-disables. + * se:8 [15:8](RO/H) Number of SEs. In CNXXXX, for CPT0 returns 0x30, + * or less if there are fuse-disables, for CPT1 returns 0x0. + * vq:8 [7:0](RO) Number of VQs. + */ +union cptx_pf_constants { + u64 u; + struct cptx_pf_constants_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_40_63:24; + u64 epcis:8; + u64 grps:8; + u64 ae:8; + u64 se:8; + u64 vq:8; +#else /* Word 0 - Little Endian */ + u64 vq:8; + u64 se:8; + u64 ae:8; + u64 grps:8; + u64 epcis:8; + u64 reserved_40_63:24; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_pf_exe_bist_status + * + * CPT PF Engine Bist Status Register + * This register has the BIST status of each engine. Each bit is the + * BIST result of an individual engine (per bit, 0 = pass and 1 = fail). + * cptx_pf_exe_bist_status_s + * Word0 + * reserved_48_63:16 [63:48] reserved + * bstatus:48 [47:0](RO/H) BIST status. One bit per engine. + * + */ +union cptx_pf_exe_bist_status { + u64 u; + struct cptx_pf_exe_bist_status_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_48_63:16; + u64 bstatus:48; +#else /* Word 0 - Little Endian */ + u64 bstatus:48; + u64 reserved_48_63:16; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_pf_q#_ctl + * + * CPT Queue Control Register + * This register configures queues. This register should be changed only + * when quiescent (see CPT()_VQ()_INPROG[INFLIGHT]). + * cptx_pf_qx_ctl_s + * Word0 + * reserved_60_63:4 [63:60] reserved. + * aura:12; [59:48](R/W) Guest-aura for returning this queue's + * instruction-chunk buffers to FPA. Only used when [INST_FREE] is set. + * For the FPA to not discard the request, FPA_PF_MAP() must map + * [AURA] and CPT()_PF_Q()_GMCTL[GMID] as valid. + * reserved_45_47:3 [47:45] reserved. + * size:13 [44:32](R/W) Command-buffer size, in number of 64-bit words per + * command buffer segment. Must be 8*n + 1, where n is the number of + * instructions per buffer segment. + * reserved_11_31:21 [31:11] Reserved. + * cont_err:1 [10:10](R/W) Continue on error. + * 0 = When CPT()_VQ()_MISC_INT[NWRP], CPT()_VQ()_MISC_INT[IRDE] or + * CPT()_VQ()_MISC_INT[DOVF] are set by hardware or software via + * CPT()_VQ()_MISC_INT_W1S, then CPT()_VQ()_CTL[ENA] is cleared. Due to + * pipelining, additional instructions may have been processed between the + * instruction causing the error and the next instruction in the disabled + * queue (the instruction at CPT()_VQ()_SADDR). + * 1 = Ignore errors and continue processing instructions. + * For diagnostic use only. + * inst_free:1 [9:9](R/W) Instruction FPA free. When set, when CPT reaches the + * end of an instruction chunk, that chunk will be freed to the FPA. + * inst_be:1 [8:8](R/W) Instruction big-endian control. When set, instructions, + * instruction next chunk pointers, and result structures are stored in + * big-endian format in memory. + * iqb_ldwb:1 [7:7](R/W) Instruction load don't write back. + * 0 = The hardware issues NCB transient load (LDT) towards the cache, + * which if the line hits and it is dirty will cause the line to be + * written back before being replaced. + * 1 = The hardware issues NCB LDWB read-and-invalidate command towards + * the cache when fetching the last word of instructions; as a result the + * line will not be written back when replaced. This improves + * performance, but software must not read the instructions after they are + * posted to the hardware. Reads that do not consume the last word of a + * cache line always use LDI. + * reserved_4_6:3 [6:4] Reserved. + * grp:3; [3:1](R/W) Engine group. + * pri:1; [0:0](R/W) Queue priority. + * 1 = This queue has higher priority. Round-robin between higher + * priority queues. + * 0 = This queue has lower priority. Round-robin between lower + * priority queues. + */ +union cptx_pf_qx_ctl { + u64 u; + struct cptx_pf_qx_ctl_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_60_63:4; + u64 aura:12; + u64 reserved_45_47:3; + u64 size:13; + u64 reserved_11_31:21; + u64 cont_err:1; + u64 inst_free:1; + u64 inst_be:1; + u64 iqb_ldwb:1; + u64 reserved_4_6:3; + u64 grp:3; + u64 pri:1; +#else /* Word 0 - Little Endian */ + u64 pri:1; + u64 grp:3; + u64 reserved_4_6:3; + u64 iqb_ldwb:1; + u64 inst_be:1; + u64 inst_free:1; + u64 cont_err:1; + u64 reserved_11_31:21; + u64 size:13; + u64 reserved_45_47:3; + u64 aura:12; + u64 reserved_60_63:4; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_vq#_saddr + * + * CPT Queue Starting Buffer Address Registers + * These registers set the instruction buffer starting address. + * cptx_vqx_saddr_s + * Word0 + * reserved_49_63:15 [63:49] Reserved. + * ptr:43 [48:6](R/W/H) Instruction buffer IOVA <48:6> (64-byte aligned). + * When written, it is the initial buffer starting address; when read, + * it is the next read pointer to be requested from L2C. The PTR field + * is overwritten with the next pointer each time that the command buffer + * segment is exhausted. New commands will then be read from the newly + * specified command buffer pointer. + * reserved_0_5:6 [5:0] Reserved. + * + */ +union cptx_vqx_saddr { + u64 u; + struct cptx_vqx_saddr_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_49_63:15; + u64 ptr:43; + u64 reserved_0_5:6; +#else /* Word 0 - Little Endian */ + u64 reserved_0_5:6; + u64 ptr:43; + u64 reserved_49_63:15; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_vq#_misc_ena_w1s + * + * CPT Queue Misc Interrupt Enable Set Register + * This register sets interrupt enable bits. + * cptx_vqx_misc_ena_w1s_s + * Word0 + * reserved_5_63:59 [63:5] Reserved. + * swerr:1 [4:4](R/W1S/H) Reads or sets enable for + * CPT(0..1)_VQ(0..63)_MISC_INT[SWERR]. + * nwrp:1 [3:3](R/W1S/H) Reads or sets enable for + * CPT(0..1)_VQ(0..63)_MISC_INT[NWRP]. + * irde:1 [2:2](R/W1S/H) Reads or sets enable for + * CPT(0..1)_VQ(0..63)_MISC_INT[IRDE]. + * dovf:1 [1:1](R/W1S/H) Reads or sets enable for + * CPT(0..1)_VQ(0..63)_MISC_INT[DOVF]. + * mbox:1 [0:0](R/W1S/H) Reads or sets enable for + * CPT(0..1)_VQ(0..63)_MISC_INT[MBOX]. + * + */ +union cptx_vqx_misc_ena_w1s { + u64 u; + struct cptx_vqx_misc_ena_w1s_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_5_63:59; + u64 swerr:1; + u64 nwrp:1; + u64 irde:1; + u64 dovf:1; + u64 mbox:1; +#else /* Word 0 - Little Endian */ + u64 mbox:1; + u64 dovf:1; + u64 irde:1; + u64 nwrp:1; + u64 swerr:1; + u64 reserved_5_63:59; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_vq#_doorbell + * + * CPT Queue Doorbell Registers + * Doorbells for the CPT instruction queues. + * cptx_vqx_doorbell_s + * Word0 + * reserved_20_63:44 [63:20] Reserved. + * dbell_cnt:20 [19:0](R/W/H) Number of instruction queue 64-bit words to add + * to the CPT instruction doorbell count. Readback value is the + * current number of pending doorbell requests. If counter overflows + * CPT()_VQ()_MISC_INT[DBELL_DOVF] is set. To reset the count back to + * zero, write one to clear CPT()_VQ()_MISC_INT_ENA_W1C[DBELL_DOVF], + * then write a value of 2^20 minus the read [DBELL_CNT], then write one + * to CPT()_VQ()_MISC_INT_W1C[DBELL_DOVF] and + * CPT()_VQ()_MISC_INT_ENA_W1S[DBELL_DOVF]. Must be a multiple of 8. + * All CPT instructions are 8 words and require a doorbell count of + * multiple of 8. + */ +union cptx_vqx_doorbell { + u64 u; + struct cptx_vqx_doorbell_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_20_63:44; + u64 dbell_cnt:20; +#else /* Word 0 - Little Endian */ + u64 dbell_cnt:20; + u64 reserved_20_63:44; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_vq#_inprog + * + * CPT Queue In Progress Count Registers + * These registers contain the per-queue instruction in flight registers. + * cptx_vqx_inprog_s + * Word0 + * reserved_8_63:56 [63:8] Reserved. + * inflight:8 [7:0](RO/H) Inflight count. Counts the number of instructions + * for the VF for which CPT is fetching, executing or responding to + * instructions. However this does not include any interrupts that are + * awaiting software handling (CPT()_VQ()_DONE[DONE] != 0x0). + * A queue may not be reconfigured until: + * 1. CPT()_VQ()_CTL[ENA] is cleared by software. + * 2. [INFLIGHT] is polled until equals to zero. + */ +union cptx_vqx_inprog { + u64 u; + struct cptx_vqx_inprog_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_8_63:56; + u64 inflight:8; +#else /* Word 0 - Little Endian */ + u64 inflight:8; + u64 reserved_8_63:56; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_vq#_misc_int + * + * CPT Queue Misc Interrupt Register + * These registers contain the per-queue miscellaneous interrupts. + * cptx_vqx_misc_int_s + * Word 0 + * reserved_5_63:59 [63:5] Reserved. + * swerr:1 [4:4](R/W1C/H) Software error from engines. + * nwrp:1 [3:3](R/W1C/H) NCB result write response error. + * irde:1 [2:2](R/W1C/H) Instruction NCB read response error. + * dovf:1 [1:1](R/W1C/H) Doorbell overflow. + * mbox:1 [0:0](R/W1C/H) PF to VF mailbox interrupt. Set when + * CPT()_VF()_PF_MBOX(0) is written. + * + */ +union cptx_vqx_misc_int { + u64 u; + struct cptx_vqx_misc_int_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_5_63:59; + u64 swerr:1; + u64 nwrp:1; + u64 irde:1; + u64 dovf:1; + u64 mbox:1; +#else /* Word 0 - Little Endian */ + u64 mbox:1; + u64 dovf:1; + u64 irde:1; + u64 nwrp:1; + u64 swerr:1; + u64 reserved_5_63:59; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_vq#_done_ack + * + * CPT Queue Done Count Ack Registers + * This register is written by software to acknowledge interrupts. + * cptx_vqx_done_ack_s + * Word0 + * reserved_20_63:44 [63:20] Reserved. + * done_ack:20 [19:0](R/W/H) Number of decrements to CPT()_VQ()_DONE[DONE]. + * Reads CPT()_VQ()_DONE[DONE]. Written by software to acknowledge + * interrupts. If CPT()_VQ()_DONE[DONE] is still nonzero the interrupt + * will be re-sent if the conditions described in CPT()_VQ()_DONE[DONE] + * are satisfied. + * + */ +union cptx_vqx_done_ack { + u64 u; + struct cptx_vqx_done_ack_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_20_63:44; + u64 done_ack:20; +#else /* Word 0 - Little Endian */ + u64 done_ack:20; + u64 reserved_20_63:44; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_vq#_done + * + * CPT Queue Done Count Registers + * These registers contain the per-queue instruction done count. + * cptx_vqx_done_s + * Word0 + * reserved_20_63:44 [63:20] Reserved. + * done:20 [19:0](R/W/H) Done count. When CPT_INST_S[DONEINT] set and that + * instruction completes, CPT()_VQ()_DONE[DONE] is incremented when the + * instruction finishes. Write to this field are for diagnostic use only; + * instead software writes CPT()_VQ()_DONE_ACK with the number of + * decrements for this field. + * Interrupts are sent as follows: + * * When CPT()_VQ()_DONE[DONE] = 0, then no results are pending, the + * interrupt coalescing timer is held to zero, and an interrupt is not + * sent. + * * When CPT()_VQ()_DONE[DONE] != 0, then the interrupt coalescing timer + * counts. If the counter is >= CPT()_VQ()_DONE_WAIT[TIME_WAIT]*1024, or + * CPT()_VQ()_DONE[DONE] >= CPT()_VQ()_DONE_WAIT[NUM_WAIT], i.e. enough + * time has passed or enough results have arrived, then the interrupt is + * sent. + * * When CPT()_VQ()_DONE_ACK is written (or CPT()_VQ()_DONE is written + * but this is not typical), the interrupt coalescing timer restarts. + * Note after decrementing this interrupt equation is recomputed, + * for example if CPT()_VQ()_DONE[DONE] >= CPT()_VQ()_DONE_WAIT[NUM_WAIT] + * and because the timer is zero, the interrupt will be resent immediately. + * (This covers the race case between software acknowledging an interrupt + * and a result returning.) + * * When CPT()_VQ()_DONE_ENA_W1S[DONE] = 0, interrupts are not sent, + * but the counting described above still occurs. + * Since CPT instructions complete out-of-order, if software is using + * completion interrupts the suggested scheme is to request a DONEINT on + * each request, and when an interrupt arrives perform a "greedy" scan for + * completions; even if a later command is acknowledged first this will + * not result in missing a completion. + * Software is responsible for making sure [DONE] does not overflow; + * for example by insuring there are not more than 2^20-1 instructions in + * flight that may request interrupts. + * + */ +union cptx_vqx_done { + u64 u; + struct cptx_vqx_done_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_20_63:44; + u64 done:20; +#else /* Word 0 - Little Endian */ + u64 done:20; + u64 reserved_20_63:44; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_vq#_done_wait + * + * CPT Queue Done Interrupt Coalescing Wait Registers + * Specifies the per queue interrupt coalescing settings. + * cptx_vqx_done_wait_s + * Word0 + * reserved_48_63:16 [63:48] Reserved. + * time_wait:16; [47:32](R/W) Time hold-off. When CPT()_VQ()_DONE[DONE] = 0 + * or CPT()_VQ()_DONE_ACK is written a timer is cleared. When the timer + * reaches [TIME_WAIT]*1024 then interrupt coalescing ends. + * see CPT()_VQ()_DONE[DONE]. If 0x0, time coalescing is disabled. + * reserved_20_31:12 [31:20] Reserved. + * num_wait:20 [19:0](R/W) Number of messages hold-off. + * When CPT()_VQ()_DONE[DONE] >= [NUM_WAIT] then interrupt coalescing ends + * see CPT()_VQ()_DONE[DONE]. If 0x0, same behavior as 0x1. + * + */ +union cptx_vqx_done_wait { + u64 u; + struct cptx_vqx_done_wait_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_48_63:16; + u64 time_wait:16; + u64 reserved_20_31:12; + u64 num_wait:20; +#else /* Word 0 - Little Endian */ + u64 num_wait:20; + u64 reserved_20_31:12; + u64 time_wait:16; + u64 reserved_48_63:16; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_vq#_done_ena_w1s + * + * CPT Queue Done Interrupt Enable Set Registers + * Write 1 to these registers will enable the DONEINT interrupt for the queue. + * cptx_vqx_done_ena_w1s_s + * Word0 + * reserved_1_63:63 [63:1] Reserved. + * done:1 [0:0](R/W1S/H) Write 1 will enable DONEINT for this queue. + * Write 0 has no effect. Read will return the enable bit. + */ +union cptx_vqx_done_ena_w1s { + u64 u; + struct cptx_vqx_done_ena_w1s_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_1_63:63; + u64 done:1; +#else /* Word 0 - Little Endian */ + u64 done:1; + u64 reserved_1_63:63; +#endif /* Word 0 - End */ + } s; +}; + +/** + * Register (NCB) cpt#_vq#_ctl + * + * CPT VF Queue Control Registers + * This register configures queues. This register should be changed (other than + * clearing [ENA]) only when quiescent (see CPT()_VQ()_INPROG[INFLIGHT]). + * cptx_vqx_ctl_s + * Word0 + * reserved_1_63:63 [63:1] Reserved. + * ena:1 [0:0](R/W/H) Enables the logical instruction queue. + * See also CPT()_PF_Q()_CTL[CONT_ERR] and CPT()_VQ()_INPROG[INFLIGHT]. + * 1 = Queue is enabled. + * 0 = Queue is disabled. + */ +union cptx_vqx_ctl { + u64 u; + struct cptx_vqx_ctl_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_1_63:63; + u64 ena:1; +#else /* Word 0 - Little Endian */ + u64 ena:1; + u64 reserved_1_63:63; +#endif /* Word 0 - End */ + } s; +}; +#endif /*__CPT_HW_TYPES_H*/ diff --git a/drivers/crypto/cavium/cpt/cptpf.h b/drivers/crypto/cavium/cpt/cptpf.h new file mode 100644 index 000000000..c88e177a3 --- /dev/null +++ b/drivers/crypto/cavium/cpt/cptpf.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2016 Cavium, Inc. + */ + +#ifndef __CPTPF_H +#define __CPTPF_H + +#include "cpt_common.h" + +#define CSR_DELAY 30 +#define CPT_MAX_CORE_GROUPS 8 +#define CPT_MAX_SE_CORES 10 +#define CPT_MAX_AE_CORES 6 +#define CPT_MAX_TOTAL_CORES (CPT_MAX_SE_CORES + CPT_MAX_AE_CORES) +#define CPT_MAX_VF_NUM 16 +#define CPT_PF_MSIX_VECTORS 3 +#define CPT_PF_INT_VEC_E_MBOXX(a) (0x02 + (a)) +#define CPT_UCODE_VERSION_SZ 32 +struct cpt_device; + +struct microcode { + u8 is_mc_valid; + u8 is_ae; + u8 group; + u8 num_cores; + u32 code_size; + u64 core_mask; + u8 version[CPT_UCODE_VERSION_SZ]; + /* Base info */ + dma_addr_t phys_base; + void *code; +}; + +struct cpt_vf_info { + u8 state; + u8 priority; + u8 id; + u32 qlen; +}; + +/** + * cpt device structure + */ +struct cpt_device { + u16 flags; /* Flags to hold device status bits */ + u8 num_vf_en; /* Number of VFs enabled (0...CPT_MAX_VF_NUM) */ + struct cpt_vf_info vfinfo[CPT_MAX_VF_NUM]; /* Per VF info */ + + void __iomem *reg_base; /* Register start address */ + struct pci_dev *pdev; /* pci device handle */ + + struct microcode mcode[CPT_MAX_CORE_GROUPS]; + u8 next_mc_idx; /* next microcode index */ + u8 next_group; + u8 max_se_cores; + u8 max_ae_cores; +}; + +void cpt_mbox_intr_handler(struct cpt_device *cpt, int mbx); +#endif /* __CPTPF_H */ diff --git a/drivers/crypto/cavium/cpt/cptpf_main.c b/drivers/crypto/cavium/cpt/cptpf_main.c new file mode 100644 index 000000000..9358c1c04 --- /dev/null +++ b/drivers/crypto/cavium/cpt/cptpf_main.c @@ -0,0 +1,669 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2016 Cavium, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "cptpf.h" + +#define DRV_NAME "thunder-cpt" +#define DRV_VERSION "1.0" + +static u32 num_vfs = 4; /* Default 4 VF enabled */ +module_param(num_vfs, uint, 0444); +MODULE_PARM_DESC(num_vfs, "Number of VFs to enable(1-16)"); + +/* + * Disable cores specified by coremask + */ +static void cpt_disable_cores(struct cpt_device *cpt, u64 coremask, + u8 type, u8 grp) +{ + u64 pf_exe_ctl; + u32 timeout = 100; + u64 grpmask = 0; + struct device *dev = &cpt->pdev->dev; + + if (type == AE_TYPES) + coremask = (coremask << cpt->max_se_cores); + + /* Disengage the cores from groups */ + grpmask = cpt_read_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp)); + cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp), + (grpmask & ~coremask)); + udelay(CSR_DELAY); + grp = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXEC_BUSY(0)); + while (grp & coremask) { + dev_err(dev, "Cores still busy %llx", coremask); + grp = cpt_read_csr64(cpt->reg_base, + CPTX_PF_EXEC_BUSY(0)); + if (!timeout--) + break; + + udelay(CSR_DELAY); + } + + /* Disable the cores */ + pf_exe_ctl = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0)); + cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0), + (pf_exe_ctl & ~coremask)); + udelay(CSR_DELAY); +} + +/* + * Enable cores specified by coremask + */ +static void cpt_enable_cores(struct cpt_device *cpt, u64 coremask, + u8 type) +{ + u64 pf_exe_ctl; + + if (type == AE_TYPES) + coremask = (coremask << cpt->max_se_cores); + + pf_exe_ctl = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0)); + cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0), + (pf_exe_ctl | coremask)); + udelay(CSR_DELAY); +} + +static void cpt_configure_group(struct cpt_device *cpt, u8 grp, + u64 coremask, u8 type) +{ + u64 pf_gx_en = 0; + + if (type == AE_TYPES) + coremask = (coremask << cpt->max_se_cores); + + pf_gx_en = cpt_read_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp)); + cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp), + (pf_gx_en | coremask)); + udelay(CSR_DELAY); +} + +static void cpt_disable_mbox_interrupts(struct cpt_device *cpt) +{ + /* Clear mbox(0) interupts for all vfs */ + cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_ENA_W1CX(0, 0), ~0ull); +} + +static void cpt_disable_ecc_interrupts(struct cpt_device *cpt) +{ + /* Clear ecc(0) interupts for all vfs */ + cpt_write_csr64(cpt->reg_base, CPTX_PF_ECC0_ENA_W1C(0), ~0ull); +} + +static void cpt_disable_exec_interrupts(struct cpt_device *cpt) +{ + /* Clear exec interupts for all vfs */ + cpt_write_csr64(cpt->reg_base, CPTX_PF_EXEC_ENA_W1C(0), ~0ull); +} + +static void cpt_disable_all_interrupts(struct cpt_device *cpt) +{ + cpt_disable_mbox_interrupts(cpt); + cpt_disable_ecc_interrupts(cpt); + cpt_disable_exec_interrupts(cpt); +} + +static void cpt_enable_mbox_interrupts(struct cpt_device *cpt) +{ + /* Set mbox(0) interupts for all vfs */ + cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_ENA_W1SX(0, 0), ~0ull); +} + +static int cpt_load_microcode(struct cpt_device *cpt, struct microcode *mcode) +{ + int ret = 0, core = 0, shift = 0; + u32 total_cores = 0; + struct device *dev = &cpt->pdev->dev; + + if (!mcode || !mcode->code) { + dev_err(dev, "Either the mcode is null or data is NULL\n"); + return -EINVAL; + } + + if (mcode->code_size == 0) { + dev_err(dev, "microcode size is 0\n"); + return -EINVAL; + } + + /* Assumes 0-9 are SE cores for UCODE_BASE registers and + * AE core bases follow + */ + if (mcode->is_ae) { + core = CPT_MAX_SE_CORES; /* start couting from 10 */ + total_cores = CPT_MAX_TOTAL_CORES; /* upto 15 */ + } else { + core = 0; /* start couting from 0 */ + total_cores = CPT_MAX_SE_CORES; /* upto 9 */ + } + + /* Point to microcode for each core of the group */ + for (; core < total_cores ; core++, shift++) { + if (mcode->core_mask & (1 << shift)) { + cpt_write_csr64(cpt->reg_base, + CPTX_PF_ENGX_UCODE_BASE(0, core), + (u64)mcode->phys_base); + } + } + return ret; +} + +static int do_cpt_init(struct cpt_device *cpt, struct microcode *mcode) +{ + int ret = 0; + struct device *dev = &cpt->pdev->dev; + + /* Make device not ready */ + cpt->flags &= ~CPT_FLAG_DEVICE_READY; + /* Disable All PF interrupts */ + cpt_disable_all_interrupts(cpt); + /* Calculate mcode group and coremasks */ + if (mcode->is_ae) { + if (mcode->num_cores > cpt->max_ae_cores) { + dev_err(dev, "Requested for more cores than available AE cores\n"); + ret = -EINVAL; + goto cpt_init_fail; + } + + if (cpt->next_group >= CPT_MAX_CORE_GROUPS) { + dev_err(dev, "Can't load, all eight microcode groups in use"); + return -ENFILE; + } + + mcode->group = cpt->next_group; + /* Convert requested cores to mask */ + mcode->core_mask = GENMASK(mcode->num_cores, 0); + cpt_disable_cores(cpt, mcode->core_mask, AE_TYPES, + mcode->group); + /* Load microcode for AE engines */ + ret = cpt_load_microcode(cpt, mcode); + if (ret) { + dev_err(dev, "Microcode load Failed for %s\n", + mcode->version); + goto cpt_init_fail; + } + cpt->next_group++; + /* Configure group mask for the mcode */ + cpt_configure_group(cpt, mcode->group, mcode->core_mask, + AE_TYPES); + /* Enable AE cores for the group mask */ + cpt_enable_cores(cpt, mcode->core_mask, AE_TYPES); + } else { + if (mcode->num_cores > cpt->max_se_cores) { + dev_err(dev, "Requested for more cores than available SE cores\n"); + ret = -EINVAL; + goto cpt_init_fail; + } + if (cpt->next_group >= CPT_MAX_CORE_GROUPS) { + dev_err(dev, "Can't load, all eight microcode groups in use"); + return -ENFILE; + } + + mcode->group = cpt->next_group; + /* Covert requested cores to mask */ + mcode->core_mask = GENMASK(mcode->num_cores, 0); + cpt_disable_cores(cpt, mcode->core_mask, SE_TYPES, + mcode->group); + /* Load microcode for SE engines */ + ret = cpt_load_microcode(cpt, mcode); + if (ret) { + dev_err(dev, "Microcode load Failed for %s\n", + mcode->version); + goto cpt_init_fail; + } + cpt->next_group++; + /* Configure group mask for the mcode */ + cpt_configure_group(cpt, mcode->group, mcode->core_mask, + SE_TYPES); + /* Enable SE cores for the group mask */ + cpt_enable_cores(cpt, mcode->core_mask, SE_TYPES); + } + + /* Enabled PF mailbox interrupts */ + cpt_enable_mbox_interrupts(cpt); + cpt->flags |= CPT_FLAG_DEVICE_READY; + + return ret; + +cpt_init_fail: + /* Enabled PF mailbox interrupts */ + cpt_enable_mbox_interrupts(cpt); + + return ret; +} + +struct ucode_header { + u8 version[CPT_UCODE_VERSION_SZ]; + __be32 code_length; + u32 data_length; + u64 sram_address; +}; + +static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae) +{ + const struct firmware *fw_entry; + struct device *dev = &cpt->pdev->dev; + struct ucode_header *ucode; + unsigned int code_length; + struct microcode *mcode; + int j, ret = 0; + + ret = request_firmware(&fw_entry, fw, dev); + if (ret) + return ret; + + ucode = (struct ucode_header *)fw_entry->data; + mcode = &cpt->mcode[cpt->next_mc_idx]; + memcpy(mcode->version, (u8 *)fw_entry->data, CPT_UCODE_VERSION_SZ); + code_length = ntohl(ucode->code_length); + if (code_length == 0 || code_length >= INT_MAX / 2) { + ret = -EINVAL; + goto fw_release; + } + mcode->code_size = code_length * 2; + + mcode->is_ae = is_ae; + mcode->core_mask = 0ULL; + mcode->num_cores = is_ae ? 6 : 10; + + /* Allocate DMAable space */ + mcode->code = dma_alloc_coherent(&cpt->pdev->dev, mcode->code_size, + &mcode->phys_base, GFP_KERNEL); + if (!mcode->code) { + dev_err(dev, "Unable to allocate space for microcode"); + ret = -ENOMEM; + goto fw_release; + } + + memcpy((void *)mcode->code, (void *)(fw_entry->data + sizeof(*ucode)), + mcode->code_size); + + /* Byte swap 64-bit */ + for (j = 0; j < (mcode->code_size / 8); j++) + ((__be64 *)mcode->code)[j] = cpu_to_be64(((u64 *)mcode->code)[j]); + /* MC needs 16-bit swap */ + for (j = 0; j < (mcode->code_size / 2); j++) + ((__be16 *)mcode->code)[j] = cpu_to_be16(((u16 *)mcode->code)[j]); + + dev_dbg(dev, "mcode->code_size = %u\n", mcode->code_size); + dev_dbg(dev, "mcode->is_ae = %u\n", mcode->is_ae); + dev_dbg(dev, "mcode->num_cores = %u\n", mcode->num_cores); + dev_dbg(dev, "mcode->code = %llx\n", (u64)mcode->code); + dev_dbg(dev, "mcode->phys_base = %llx\n", mcode->phys_base); + + ret = do_cpt_init(cpt, mcode); + if (ret) { + dma_free_coherent(&cpt->pdev->dev, mcode->code_size, + mcode->code, mcode->phys_base); + dev_err(dev, "do_cpt_init failed with ret: %d\n", ret); + goto fw_release; + } + + dev_info(dev, "Microcode Loaded %s\n", mcode->version); + mcode->is_mc_valid = 1; + cpt->next_mc_idx++; + +fw_release: + release_firmware(fw_entry); + + return ret; +} + +static int cpt_ucode_load(struct cpt_device *cpt) +{ + int ret = 0; + struct device *dev = &cpt->pdev->dev; + + ret = cpt_ucode_load_fw(cpt, "cpt8x-mc-ae.out", true); + if (ret) { + dev_err(dev, "ae:cpt_ucode_load failed with ret: %d\n", ret); + return ret; + } + ret = cpt_ucode_load_fw(cpt, "cpt8x-mc-se.out", false); + if (ret) { + dev_err(dev, "se:cpt_ucode_load failed with ret: %d\n", ret); + return ret; + } + + return ret; +} + +static irqreturn_t cpt_mbx0_intr_handler(int irq, void *cpt_irq) +{ + struct cpt_device *cpt = (struct cpt_device *)cpt_irq; + + cpt_mbox_intr_handler(cpt, 0); + + return IRQ_HANDLED; +} + +static void cpt_reset(struct cpt_device *cpt) +{ + cpt_write_csr64(cpt->reg_base, CPTX_PF_RESET(0), 1); +} + +static void cpt_find_max_enabled_cores(struct cpt_device *cpt) +{ + union cptx_pf_constants pf_cnsts = {0}; + + pf_cnsts.u = cpt_read_csr64(cpt->reg_base, CPTX_PF_CONSTANTS(0)); + cpt->max_se_cores = pf_cnsts.s.se; + cpt->max_ae_cores = pf_cnsts.s.ae; +} + +static u32 cpt_check_bist_status(struct cpt_device *cpt) +{ + union cptx_pf_bist_status bist_sts = {0}; + + bist_sts.u = cpt_read_csr64(cpt->reg_base, + CPTX_PF_BIST_STATUS(0)); + + return bist_sts.u; +} + +static u64 cpt_check_exe_bist_status(struct cpt_device *cpt) +{ + union cptx_pf_exe_bist_status bist_sts = {0}; + + bist_sts.u = cpt_read_csr64(cpt->reg_base, + CPTX_PF_EXE_BIST_STATUS(0)); + + return bist_sts.u; +} + +static void cpt_disable_all_cores(struct cpt_device *cpt) +{ + u32 grp, timeout = 100; + struct device *dev = &cpt->pdev->dev; + + /* Disengage the cores from groups */ + for (grp = 0; grp < CPT_MAX_CORE_GROUPS; grp++) { + cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp), 0); + udelay(CSR_DELAY); + } + + grp = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXEC_BUSY(0)); + while (grp) { + dev_err(dev, "Cores still busy"); + grp = cpt_read_csr64(cpt->reg_base, + CPTX_PF_EXEC_BUSY(0)); + if (!timeout--) + break; + + udelay(CSR_DELAY); + } + /* Disable the cores */ + cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0), 0); +} + +/* + * Ensure all cores are disengaged from all groups by + * calling cpt_disable_all_cores() before calling this + * function. + */ +static void cpt_unload_microcode(struct cpt_device *cpt) +{ + u32 grp = 0, core; + + /* Free microcode bases and reset group masks */ + for (grp = 0; grp < CPT_MAX_CORE_GROUPS; grp++) { + struct microcode *mcode = &cpt->mcode[grp]; + + if (cpt->mcode[grp].code) + dma_free_coherent(&cpt->pdev->dev, mcode->code_size, + mcode->code, mcode->phys_base); + mcode->code = NULL; + } + /* Clear UCODE_BASE registers for all engines */ + for (core = 0; core < CPT_MAX_TOTAL_CORES; core++) + cpt_write_csr64(cpt->reg_base, + CPTX_PF_ENGX_UCODE_BASE(0, core), 0ull); +} + +static int cpt_device_init(struct cpt_device *cpt) +{ + u64 bist; + struct device *dev = &cpt->pdev->dev; + + /* Reset the PF when probed first */ + cpt_reset(cpt); + msleep(100); + + /*Check BIST status*/ + bist = (u64)cpt_check_bist_status(cpt); + if (bist) { + dev_err(dev, "RAM BIST failed with code 0x%llx", bist); + return -ENODEV; + } + + bist = cpt_check_exe_bist_status(cpt); + if (bist) { + dev_err(dev, "Engine BIST failed with code 0x%llx", bist); + return -ENODEV; + } + + /*Get CLK frequency*/ + /*Get max enabled cores */ + cpt_find_max_enabled_cores(cpt); + /*Disable all cores*/ + cpt_disable_all_cores(cpt); + /*Reset device parameters*/ + cpt->next_mc_idx = 0; + cpt->next_group = 0; + /* PF is ready */ + cpt->flags |= CPT_FLAG_DEVICE_READY; + + return 0; +} + +static int cpt_register_interrupts(struct cpt_device *cpt) +{ + int ret; + struct device *dev = &cpt->pdev->dev; + + /* Enable MSI-X */ + ret = pci_alloc_irq_vectors(cpt->pdev, CPT_PF_MSIX_VECTORS, + CPT_PF_MSIX_VECTORS, PCI_IRQ_MSIX); + if (ret < 0) { + dev_err(&cpt->pdev->dev, "Request for #%d msix vectors failed\n", + CPT_PF_MSIX_VECTORS); + return ret; + } + + /* Register mailbox interrupt handlers */ + ret = request_irq(pci_irq_vector(cpt->pdev, CPT_PF_INT_VEC_E_MBOXX(0)), + cpt_mbx0_intr_handler, 0, "CPT Mbox0", cpt); + if (ret) + goto fail; + + /* Enable mailbox interrupt */ + cpt_enable_mbox_interrupts(cpt); + return 0; + +fail: + dev_err(dev, "Request irq failed\n"); + pci_disable_msix(cpt->pdev); + return ret; +} + +static void cpt_unregister_interrupts(struct cpt_device *cpt) +{ + free_irq(pci_irq_vector(cpt->pdev, CPT_PF_INT_VEC_E_MBOXX(0)), cpt); + pci_disable_msix(cpt->pdev); +} + +static int cpt_sriov_init(struct cpt_device *cpt, int num_vfs) +{ + int pos = 0; + int err; + u16 total_vf_cnt; + struct pci_dev *pdev = cpt->pdev; + + pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV); + if (!pos) { + dev_err(&pdev->dev, "SRIOV capability is not found in PCIe config space\n"); + return -ENODEV; + } + + cpt->num_vf_en = num_vfs; /* User requested VFs */ + pci_read_config_word(pdev, (pos + PCI_SRIOV_TOTAL_VF), &total_vf_cnt); + if (total_vf_cnt < cpt->num_vf_en) + cpt->num_vf_en = total_vf_cnt; + + if (!total_vf_cnt) + return 0; + + /*Enabled the available VFs */ + err = pci_enable_sriov(pdev, cpt->num_vf_en); + if (err) { + dev_err(&pdev->dev, "SRIOV enable failed, num VF is %d\n", + cpt->num_vf_en); + cpt->num_vf_en = 0; + return err; + } + + /* TODO: Optionally enable static VQ priorities feature */ + + dev_info(&pdev->dev, "SRIOV enabled, number of VF available %d\n", + cpt->num_vf_en); + + cpt->flags |= CPT_FLAG_SRIOV_ENABLED; + + return 0; +} + +static int cpt_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct device *dev = &pdev->dev; + struct cpt_device *cpt; + int err; + + if (num_vfs > 16 || num_vfs < 4) { + dev_warn(dev, "Invalid vf count %d, Resetting it to 4(default)\n", + num_vfs); + num_vfs = 4; + } + + cpt = devm_kzalloc(dev, sizeof(*cpt), GFP_KERNEL); + if (!cpt) + return -ENOMEM; + + pci_set_drvdata(pdev, cpt); + cpt->pdev = pdev; + err = pci_enable_device(pdev); + if (err) { + dev_err(dev, "Failed to enable PCI device\n"); + pci_set_drvdata(pdev, NULL); + return err; + } + + err = pci_request_regions(pdev, DRV_NAME); + if (err) { + dev_err(dev, "PCI request regions failed 0x%x\n", err); + goto cpt_err_disable_device; + } + + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); + if (err) { + dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); + goto cpt_err_release_regions; + } + + /* MAP PF's configuration registers */ + cpt->reg_base = pcim_iomap(pdev, 0, 0); + if (!cpt->reg_base) { + dev_err(dev, "Cannot map config register space, aborting\n"); + err = -ENOMEM; + goto cpt_err_release_regions; + } + + /* CPT device HW initialization */ + cpt_device_init(cpt); + + /* Register interrupts */ + err = cpt_register_interrupts(cpt); + if (err) + goto cpt_err_release_regions; + + err = cpt_ucode_load(cpt); + if (err) + goto cpt_err_unregister_interrupts; + + /* Configure SRIOV */ + err = cpt_sriov_init(cpt, num_vfs); + if (err) + goto cpt_err_unregister_interrupts; + + return 0; + +cpt_err_unregister_interrupts: + cpt_unregister_interrupts(cpt); +cpt_err_release_regions: + pci_release_regions(pdev); +cpt_err_disable_device: + pci_disable_device(pdev); + pci_set_drvdata(pdev, NULL); + return err; +} + +static void cpt_remove(struct pci_dev *pdev) +{ + struct cpt_device *cpt = pci_get_drvdata(pdev); + + /* Disengage SE and AE cores from all groups*/ + cpt_disable_all_cores(cpt); + /* Unload microcodes */ + cpt_unload_microcode(cpt); + cpt_unregister_interrupts(cpt); + pci_disable_sriov(pdev); + pci_release_regions(pdev); + pci_disable_device(pdev); + pci_set_drvdata(pdev, NULL); +} + +static void cpt_shutdown(struct pci_dev *pdev) +{ + struct cpt_device *cpt = pci_get_drvdata(pdev); + + if (!cpt) + return; + + dev_info(&pdev->dev, "Shutdown device %x:%x.\n", + (u32)pdev->vendor, (u32)pdev->device); + + cpt_unregister_interrupts(cpt); + pci_release_regions(pdev); + pci_disable_device(pdev); + pci_set_drvdata(pdev, NULL); +} + +/* Supported devices */ +static const struct pci_device_id cpt_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, CPT_81XX_PCI_PF_DEVICE_ID) }, + { 0, } /* end of table */ +}; +MODULE_DEVICE_TABLE(pci, cpt_id_table); + +static struct pci_driver cpt_pci_driver = { + .name = DRV_NAME, + .id_table = cpt_id_table, + .probe = cpt_probe, + .remove = cpt_remove, + .shutdown = cpt_shutdown, +}; + +module_pci_driver(cpt_pci_driver); + +MODULE_AUTHOR("George Cherian "); +MODULE_DESCRIPTION("Cavium Thunder CPT Physical Function Driver"); +MODULE_LICENSE("GPL v2"); +MODULE_VERSION(DRV_VERSION); diff --git a/drivers/crypto/cavium/cpt/cptpf_mbox.c b/drivers/crypto/cavium/cpt/cptpf_mbox.c new file mode 100644 index 000000000..f01b863d6 --- /dev/null +++ b/drivers/crypto/cavium/cpt/cptpf_mbox.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2016 Cavium, Inc. + */ +#include +#include "cptpf.h" + +static void cpt_send_msg_to_vf(struct cpt_device *cpt, int vf, + struct cpt_mbox *mbx) +{ + /* Writing mbox(0) causes interrupt */ + cpt_write_csr64(cpt->reg_base, CPTX_PF_VFX_MBOXX(0, vf, 1), + mbx->data); + cpt_write_csr64(cpt->reg_base, CPTX_PF_VFX_MBOXX(0, vf, 0), mbx->msg); +} + +/* ACKs VF's mailbox message + * @vf: VF to which ACK to be sent + */ +static void cpt_mbox_send_ack(struct cpt_device *cpt, int vf, + struct cpt_mbox *mbx) +{ + mbx->data = 0ull; + mbx->msg = CPT_MBOX_MSG_TYPE_ACK; + cpt_send_msg_to_vf(cpt, vf, mbx); +} + +static void cpt_clear_mbox_intr(struct cpt_device *cpt, u32 vf) +{ + /* W1C for the VF */ + cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_INTX(0, 0), (1 << vf)); +} + +/* + * Configure QLEN/Chunk sizes for VF + */ +static void cpt_cfg_qlen_for_vf(struct cpt_device *cpt, int vf, u32 size) +{ + union cptx_pf_qx_ctl pf_qx_ctl; + + pf_qx_ctl.u = cpt_read_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, vf)); + pf_qx_ctl.s.size = size; + pf_qx_ctl.s.cont_err = true; + cpt_write_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, vf), pf_qx_ctl.u); +} + +/* + * Configure VQ priority + */ +static void cpt_cfg_vq_priority(struct cpt_device *cpt, int vf, u32 pri) +{ + union cptx_pf_qx_ctl pf_qx_ctl; + + pf_qx_ctl.u = cpt_read_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, vf)); + pf_qx_ctl.s.pri = pri; + cpt_write_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, vf), pf_qx_ctl.u); +} + +static int cpt_bind_vq_to_grp(struct cpt_device *cpt, u8 q, u8 grp) +{ + struct microcode *mcode = cpt->mcode; + union cptx_pf_qx_ctl pf_qx_ctl; + struct device *dev = &cpt->pdev->dev; + + if (q >= CPT_MAX_VF_NUM) { + dev_err(dev, "Queues are more than cores in the group"); + return -EINVAL; + } + if (grp >= CPT_MAX_CORE_GROUPS) { + dev_err(dev, "Request group is more than possible groups"); + return -EINVAL; + } + if (grp >= cpt->next_mc_idx) { + dev_err(dev, "Request group is higher than available functional groups"); + return -EINVAL; + } + pf_qx_ctl.u = cpt_read_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, q)); + pf_qx_ctl.s.grp = mcode[grp].group; + cpt_write_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, q), pf_qx_ctl.u); + dev_dbg(dev, "VF %d TYPE %s", q, (mcode[grp].is_ae ? "AE" : "SE")); + + return mcode[grp].is_ae ? AE_TYPES : SE_TYPES; +} + +/* Interrupt handler to handle mailbox messages from VFs */ +static void cpt_handle_mbox_intr(struct cpt_device *cpt, int vf) +{ + struct cpt_vf_info *vfx = &cpt->vfinfo[vf]; + struct cpt_mbox mbx = {}; + int vftype; + struct device *dev = &cpt->pdev->dev; + /* + * MBOX[0] contains msg + * MBOX[1] contains data + */ + mbx.msg = cpt_read_csr64(cpt->reg_base, CPTX_PF_VFX_MBOXX(0, vf, 0)); + mbx.data = cpt_read_csr64(cpt->reg_base, CPTX_PF_VFX_MBOXX(0, vf, 1)); + dev_dbg(dev, "%s: Mailbox msg 0x%llx from VF%d", __func__, mbx.msg, vf); + switch (mbx.msg) { + case CPT_MSG_VF_UP: + vfx->state = VF_STATE_UP; + try_module_get(THIS_MODULE); + cpt_mbox_send_ack(cpt, vf, &mbx); + break; + case CPT_MSG_READY: + mbx.msg = CPT_MSG_READY; + mbx.data = vf; + cpt_send_msg_to_vf(cpt, vf, &mbx); + break; + case CPT_MSG_VF_DOWN: + /* First msg in VF teardown sequence */ + vfx->state = VF_STATE_DOWN; + module_put(THIS_MODULE); + cpt_mbox_send_ack(cpt, vf, &mbx); + break; + case CPT_MSG_QLEN: + vfx->qlen = mbx.data; + cpt_cfg_qlen_for_vf(cpt, vf, vfx->qlen); + cpt_mbox_send_ack(cpt, vf, &mbx); + break; + case CPT_MSG_QBIND_GRP: + vftype = cpt_bind_vq_to_grp(cpt, vf, (u8)mbx.data); + if ((vftype != AE_TYPES) && (vftype != SE_TYPES)) + dev_err(dev, "Queue %d binding to group %llu failed", + vf, mbx.data); + else { + dev_dbg(dev, "Queue %d binding to group %llu successful", + vf, mbx.data); + mbx.msg = CPT_MSG_QBIND_GRP; + mbx.data = vftype; + cpt_send_msg_to_vf(cpt, vf, &mbx); + } + break; + case CPT_MSG_VQ_PRIORITY: + vfx->priority = mbx.data; + cpt_cfg_vq_priority(cpt, vf, vfx->priority); + cpt_mbox_send_ack(cpt, vf, &mbx); + break; + default: + dev_err(&cpt->pdev->dev, "Invalid msg from VF%d, msg 0x%llx\n", + vf, mbx.msg); + break; + } +} + +void cpt_mbox_intr_handler (struct cpt_device *cpt, int mbx) +{ + u64 intr; + u8 vf; + + intr = cpt_read_csr64(cpt->reg_base, CPTX_PF_MBOX_INTX(0, 0)); + dev_dbg(&cpt->pdev->dev, "PF interrupt Mbox%d 0x%llx\n", mbx, intr); + for (vf = 0; vf < CPT_MAX_VF_NUM; vf++) { + if (intr & (1ULL << vf)) { + dev_dbg(&cpt->pdev->dev, "Intr from VF %d\n", vf); + cpt_handle_mbox_intr(cpt, vf); + cpt_clear_mbox_intr(cpt, vf); + } + } +} diff --git a/drivers/crypto/cavium/cpt/cptvf.h b/drivers/crypto/cavium/cpt/cptvf.h new file mode 100644 index 000000000..c695676ad --- /dev/null +++ b/drivers/crypto/cavium/cpt/cptvf.h @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2016 Cavium, Inc. + */ + +#ifndef __CPTVF_H +#define __CPTVF_H + +#include +#include "cpt_common.h" + +/* Default command queue length */ +#define CPT_CMD_QLEN 2046 +#define CPT_CMD_QCHUNK_SIZE 1023 + +/* Default command timeout in seconds */ +#define CPT_COMMAND_TIMEOUT 4 +#define CPT_TIMER_THOLD 0xFFFF +#define CPT_NUM_QS_PER_VF 1 +#define CPT_INST_SIZE 64 +#define CPT_NEXT_CHUNK_PTR_SIZE 8 + +#define CPT_VF_MSIX_VECTORS 2 +#define CPT_VF_INTR_MBOX_MASK BIT(0) +#define CPT_VF_INTR_DOVF_MASK BIT(1) +#define CPT_VF_INTR_IRDE_MASK BIT(2) +#define CPT_VF_INTR_NWRP_MASK BIT(3) +#define CPT_VF_INTR_SERR_MASK BIT(4) +#define DMA_DIRECT_DIRECT 0 /* Input DIRECT, Output DIRECT */ +#define DMA_GATHER_SCATTER 1 +#define FROM_DPTR 1 + +/** + * Enumeration cpt_vf_int_vec_e + * + * CPT VF MSI-X Vector Enumeration + * Enumerates the MSI-X interrupt vectors. + */ +enum cpt_vf_int_vec_e { + CPT_VF_INT_VEC_E_MISC = 0x00, + CPT_VF_INT_VEC_E_DONE = 0x01 +}; + +struct command_chunk { + u8 *head; + dma_addr_t dma_addr; + u32 size; /* Chunk size, max CPT_INST_CHUNK_MAX_SIZE */ + struct hlist_node nextchunk; +}; + +struct command_queue { + spinlock_t lock; /* command queue lock */ + u32 idx; /* Command queue host write idx */ + u32 nchunks; /* Number of command chunks */ + struct command_chunk *qhead; /* Command queue head, instructions + * are inserted here + */ + struct hlist_head chead; +}; + +struct command_qinfo { + u32 cmd_size; + u32 qchunksize; /* Command queue chunk size */ + struct command_queue queue[CPT_NUM_QS_PER_VF]; +}; + +struct pending_entry { + u8 busy; /* Entry status (free/busy) */ + + volatile u64 *completion_addr; /* Completion address */ + void *post_arg; + void (*callback)(int, void *); /* Kernel ASYNC request callabck */ + void *callback_arg; /* Kernel ASYNC request callabck arg */ +}; + +struct pending_queue { + struct pending_entry *head; /* head of the queue */ + u32 front; /* Process work from here */ + u32 rear; /* Append new work here */ + atomic64_t pending_count; + spinlock_t lock; /* Queue lock */ +}; + +struct pending_qinfo { + u32 nr_queues; /* Number of queues supported */ + u32 qlen; /* Queue length */ + struct pending_queue queue[CPT_NUM_QS_PER_VF]; +}; + +#define for_each_pending_queue(qinfo, q, i) \ + for (i = 0, q = &qinfo->queue[i]; i < qinfo->nr_queues; i++, \ + q = &qinfo->queue[i]) + +struct cpt_vf { + u16 flags; /* Flags to hold device status bits */ + u8 vfid; /* Device Index 0...CPT_MAX_VF_NUM */ + u8 vftype; /* VF type of SE_TYPE(1) or AE_TYPE(1) */ + u8 vfgrp; /* VF group (0 - 8) */ + u8 node; /* Operating node: Bits (46:44) in BAR0 address */ + u8 priority; /* VF priority ring: 1-High proirity round + * robin ring;0-Low priority round robin ring; + */ + struct pci_dev *pdev; /* pci device handle */ + void __iomem *reg_base; /* Register start address */ + void *wqe_info; /* BH worker info */ + /* MSI-X */ + cpumask_var_t affinity_mask[CPT_VF_MSIX_VECTORS]; + /* Command and Pending queues */ + u32 qsize; + u32 nr_queues; + struct command_qinfo cqinfo; /* Command queue information */ + struct pending_qinfo pqinfo; /* Pending queue information */ + /* VF-PF mailbox communication */ + bool pf_acked; + bool pf_nacked; +}; + +int cptvf_send_vf_up(struct cpt_vf *cptvf); +int cptvf_send_vf_down(struct cpt_vf *cptvf); +int cptvf_send_vf_to_grp_msg(struct cpt_vf *cptvf); +int cptvf_send_vf_priority_msg(struct cpt_vf *cptvf); +int cptvf_send_vq_size_msg(struct cpt_vf *cptvf); +int cptvf_check_pf_ready(struct cpt_vf *cptvf); +void cptvf_handle_mbox_intr(struct cpt_vf *cptvf); +void cvm_crypto_exit(void); +int cvm_crypto_init(struct cpt_vf *cptvf); +void vq_post_process(struct cpt_vf *cptvf, u32 qno); +void cptvf_write_vq_doorbell(struct cpt_vf *cptvf, u32 val); +#endif /* __CPTVF_H */ diff --git a/drivers/crypto/cavium/cpt/cptvf_algs.c b/drivers/crypto/cavium/cpt/cptvf_algs.c new file mode 100644 index 000000000..219fe9be7 --- /dev/null +++ b/drivers/crypto/cavium/cpt/cptvf_algs.c @@ -0,0 +1,462 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/* + * Copyright (C) 2016 Cavium, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cptvf.h" +#include "cptvf_algs.h" + +struct cpt_device_handle { + void *cdev[MAX_DEVICES]; + u32 dev_count; +}; + +static struct cpt_device_handle dev_handle; + +static void cvm_callback(u32 status, void *arg) +{ + struct crypto_async_request *req = (struct crypto_async_request *)arg; + + crypto_request_complete(req, !status); +} + +static inline void update_input_iv(struct cpt_request_info *req_info, + u8 *iv, u32 enc_iv_len, + u32 *argcnt) +{ + /* Setting the iv information */ + req_info->in[*argcnt].vptr = (void *)iv; + req_info->in[*argcnt].size = enc_iv_len; + req_info->req.dlen += enc_iv_len; + + ++(*argcnt); +} + +static inline void update_output_iv(struct cpt_request_info *req_info, + u8 *iv, u32 enc_iv_len, + u32 *argcnt) +{ + /* Setting the iv information */ + req_info->out[*argcnt].vptr = (void *)iv; + req_info->out[*argcnt].size = enc_iv_len; + req_info->rlen += enc_iv_len; + + ++(*argcnt); +} + +static inline void update_input_data(struct cpt_request_info *req_info, + struct scatterlist *inp_sg, + u32 nbytes, u32 *argcnt) +{ + req_info->req.dlen += nbytes; + + while (nbytes) { + u32 len = min(nbytes, inp_sg->length); + u8 *ptr = sg_virt(inp_sg); + + req_info->in[*argcnt].vptr = (void *)ptr; + req_info->in[*argcnt].size = len; + nbytes -= len; + + ++(*argcnt); + ++inp_sg; + } +} + +static inline void update_output_data(struct cpt_request_info *req_info, + struct scatterlist *outp_sg, + u32 nbytes, u32 *argcnt) +{ + req_info->rlen += nbytes; + + while (nbytes) { + u32 len = min(nbytes, outp_sg->length); + u8 *ptr = sg_virt(outp_sg); + + req_info->out[*argcnt].vptr = (void *)ptr; + req_info->out[*argcnt].size = len; + nbytes -= len; + ++(*argcnt); + ++outp_sg; + } +} + +static inline u32 create_ctx_hdr(struct skcipher_request *req, u32 enc, + u32 *argcnt) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(tfm); + struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req); + struct fc_context *fctx = &rctx->fctx; + u32 enc_iv_len = crypto_skcipher_ivsize(tfm); + struct cpt_request_info *req_info = &rctx->cpt_req; + __be64 *ctrl_flags = NULL; + __be64 *offset_control; + + req_info->ctrl.s.grp = 0; + req_info->ctrl.s.dma_mode = DMA_GATHER_SCATTER; + req_info->ctrl.s.se_req = SE_CORE_REQ; + + req_info->req.opcode.s.major = MAJOR_OP_FC | + DMA_MODE_FLAG(DMA_GATHER_SCATTER); + if (enc) + req_info->req.opcode.s.minor = 2; + else + req_info->req.opcode.s.minor = 3; + + req_info->req.param1 = req->cryptlen; /* Encryption Data length */ + req_info->req.param2 = 0; /*Auth data length */ + + fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type; + fctx->enc.enc_ctrl.e.aes_key = ctx->key_type; + fctx->enc.enc_ctrl.e.iv_source = FROM_DPTR; + + if (ctx->cipher_type == AES_XTS) + memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2); + else + memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len); + ctrl_flags = (__be64 *)&fctx->enc.enc_ctrl.flags; + *ctrl_flags = cpu_to_be64(fctx->enc.enc_ctrl.flags); + + offset_control = (__be64 *)&rctx->control_word; + *offset_control = cpu_to_be64(((u64)(enc_iv_len) << 16)); + /* Storing Packet Data Information in offset + * Control Word First 8 bytes + */ + req_info->in[*argcnt].vptr = (u8 *)offset_control; + req_info->in[*argcnt].size = CONTROL_WORD_LEN; + req_info->req.dlen += CONTROL_WORD_LEN; + ++(*argcnt); + + req_info->in[*argcnt].vptr = (u8 *)fctx; + req_info->in[*argcnt].size = sizeof(struct fc_context); + req_info->req.dlen += sizeof(struct fc_context); + + ++(*argcnt); + + return 0; +} + +static inline u32 create_input_list(struct skcipher_request *req, u32 enc, + u32 enc_iv_len) +{ + struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req); + struct cpt_request_info *req_info = &rctx->cpt_req; + u32 argcnt = 0; + + create_ctx_hdr(req, enc, &argcnt); + update_input_iv(req_info, req->iv, enc_iv_len, &argcnt); + update_input_data(req_info, req->src, req->cryptlen, &argcnt); + req_info->incnt = argcnt; + + return 0; +} + +static inline void store_cb_info(struct skcipher_request *req, + struct cpt_request_info *req_info) +{ + req_info->callback = (void *)cvm_callback; + req_info->callback_arg = (void *)&req->base; +} + +static inline void create_output_list(struct skcipher_request *req, + u32 enc_iv_len) +{ + struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req); + struct cpt_request_info *req_info = &rctx->cpt_req; + u32 argcnt = 0; + + /* OUTPUT Buffer Processing + * AES encryption/decryption output would be + * received in the following format + * + * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----| + * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ] + */ + /* Reading IV information */ + update_output_iv(req_info, req->iv, enc_iv_len, &argcnt); + update_output_data(req_info, req->dst, req->cryptlen, &argcnt); + req_info->outcnt = argcnt; +} + +static inline int cvm_enc_dec(struct skcipher_request *req, u32 enc) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req); + u32 enc_iv_len = crypto_skcipher_ivsize(tfm); + struct fc_context *fctx = &rctx->fctx; + struct cpt_request_info *req_info = &rctx->cpt_req; + void *cdev = NULL; + int status; + + memset(req_info, 0, sizeof(struct cpt_request_info)); + req_info->may_sleep = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) != 0; + memset(fctx, 0, sizeof(struct fc_context)); + create_input_list(req, enc, enc_iv_len); + create_output_list(req, enc_iv_len); + store_cb_info(req, req_info); + cdev = dev_handle.cdev[smp_processor_id()]; + status = cptvf_do_request(cdev, req_info); + /* We perform an asynchronous send and once + * the request is completed the driver would + * intimate through registered call back functions + */ + + if (status) + return status; + else + return -EINPROGRESS; +} + +static int cvm_encrypt(struct skcipher_request *req) +{ + return cvm_enc_dec(req, true); +} + +static int cvm_decrypt(struct skcipher_request *req) +{ + return cvm_enc_dec(req, false); +} + +static int cvm_xts_setkey(struct crypto_skcipher *cipher, const u8 *key, + u32 keylen) +{ + struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(cipher); + int err; + const u8 *key1 = key; + const u8 *key2 = key + (keylen / 2); + + err = xts_verify_key(cipher, key, keylen); + if (err) + return err; + ctx->key_len = keylen; + memcpy(ctx->enc_key, key1, keylen / 2); + memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2); + ctx->cipher_type = AES_XTS; + switch (ctx->key_len) { + case 32: + ctx->key_type = AES_128_BIT; + break; + case 64: + ctx->key_type = AES_256_BIT; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int cvm_validate_keylen(struct cvm_enc_ctx *ctx, u32 keylen) +{ + if ((keylen == 16) || (keylen == 24) || (keylen == 32)) { + ctx->key_len = keylen; + switch (ctx->key_len) { + case 16: + ctx->key_type = AES_128_BIT; + break; + case 24: + ctx->key_type = AES_192_BIT; + break; + case 32: + ctx->key_type = AES_256_BIT; + break; + default: + return -EINVAL; + } + + if (ctx->cipher_type == DES3_CBC) + ctx->key_type = 0; + + return 0; + } + + return -EINVAL; +} + +static int cvm_setkey(struct crypto_skcipher *cipher, const u8 *key, + u32 keylen, u8 cipher_type) +{ + struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(cipher); + + ctx->cipher_type = cipher_type; + if (!cvm_validate_keylen(ctx, keylen)) { + memcpy(ctx->enc_key, key, keylen); + return 0; + } else { + return -EINVAL; + } +} + +static int cvm_cbc_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, + u32 keylen) +{ + return cvm_setkey(cipher, key, keylen, AES_CBC); +} + +static int cvm_ecb_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, + u32 keylen) +{ + return cvm_setkey(cipher, key, keylen, AES_ECB); +} + +static int cvm_cbc_des3_setkey(struct crypto_skcipher *cipher, const u8 *key, + u32 keylen) +{ + return verify_skcipher_des3_key(cipher, key) ?: + cvm_setkey(cipher, key, keylen, DES3_CBC); +} + +static int cvm_ecb_des3_setkey(struct crypto_skcipher *cipher, const u8 *key, + u32 keylen) +{ + return verify_skcipher_des3_key(cipher, key) ?: + cvm_setkey(cipher, key, keylen, DES3_ECB); +} + +static int cvm_enc_dec_init(struct crypto_skcipher *tfm) +{ + crypto_skcipher_set_reqsize_dma(tfm, sizeof(struct cvm_req_ctx)); + + return 0; +} + +static struct skcipher_alg algs[] = { { + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct cvm_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_name = "xts(aes)", + .base.cra_driver_name = "cavium-xts-aes", + .base.cra_module = THIS_MODULE, + + .ivsize = AES_BLOCK_SIZE, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .setkey = cvm_xts_setkey, + .encrypt = cvm_encrypt, + .decrypt = cvm_decrypt, + .init = cvm_enc_dec_init, +}, { + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct cvm_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cavium-cbc-aes", + .base.cra_module = THIS_MODULE, + + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = cvm_cbc_aes_setkey, + .encrypt = cvm_encrypt, + .decrypt = cvm_decrypt, + .init = cvm_enc_dec_init, +}, { + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct cvm_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "cavium-ecb-aes", + .base.cra_module = THIS_MODULE, + + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = cvm_ecb_aes_setkey, + .encrypt = cvm_encrypt, + .decrypt = cvm_decrypt, + .init = cvm_enc_dec_init, +}, { + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct cvm_des3_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_name = "cbc(des3_ede)", + .base.cra_driver_name = "cavium-cbc-des3_ede", + .base.cra_module = THIS_MODULE, + + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = cvm_cbc_des3_setkey, + .encrypt = cvm_encrypt, + .decrypt = cvm_decrypt, + .init = cvm_enc_dec_init, +}, { + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct cvm_des3_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_name = "ecb(des3_ede)", + .base.cra_driver_name = "cavium-ecb-des3_ede", + .base.cra_module = THIS_MODULE, + + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = cvm_ecb_des3_setkey, + .encrypt = cvm_encrypt, + .decrypt = cvm_decrypt, + .init = cvm_enc_dec_init, +} }; + +static inline int cav_register_algs(void) +{ + return crypto_register_skciphers(algs, ARRAY_SIZE(algs)); +} + +static inline void cav_unregister_algs(void) +{ + crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); +} + +int cvm_crypto_init(struct cpt_vf *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + u32 dev_count; + + dev_count = dev_handle.dev_count; + dev_handle.cdev[dev_count] = cptvf; + dev_handle.dev_count++; + + if (dev_count == 3) { + if (cav_register_algs()) { + dev_err(&pdev->dev, "Error in registering crypto algorithms\n"); + return -EINVAL; + } + } + + return 0; +} + +void cvm_crypto_exit(void) +{ + u32 dev_count; + + dev_count = --dev_handle.dev_count; + if (!dev_count) + cav_unregister_algs(); +} diff --git a/drivers/crypto/cavium/cpt/cptvf_algs.h b/drivers/crypto/cavium/cpt/cptvf_algs.h new file mode 100644 index 000000000..61a149fe8 --- /dev/null +++ b/drivers/crypto/cavium/cpt/cptvf_algs.h @@ -0,0 +1,117 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2016 Cavium, Inc. + */ + +#ifndef _CPTVF_ALGS_H_ +#define _CPTVF_ALGS_H_ + +#include "request_manager.h" + +#define MAX_DEVICES 16 +#define MAJOR_OP_FC 0x33 +#define MAX_ENC_KEY_SIZE 32 +#define MAX_HASH_KEY_SIZE 64 +#define MAX_KEY_SIZE (MAX_ENC_KEY_SIZE + MAX_HASH_KEY_SIZE) +#define CONTROL_WORD_LEN 8 +#define KEY2_OFFSET 48 + +#define DMA_MODE_FLAG(dma_mode) \ + (((dma_mode) == DMA_GATHER_SCATTER) ? (1 << 7) : 0) + +enum req_type { + AE_CORE_REQ, + SE_CORE_REQ, +}; + +enum cipher_type { + DES3_CBC = 0x1, + DES3_ECB = 0x2, + AES_CBC = 0x3, + AES_ECB = 0x4, + AES_CFB = 0x5, + AES_CTR = 0x6, + AES_GCM = 0x7, + AES_XTS = 0x8 +}; + +enum aes_type { + AES_128_BIT = 0x1, + AES_192_BIT = 0x2, + AES_256_BIT = 0x3 +}; + +union encr_ctrl { + u64 flags; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 enc_cipher:4; + u64 reserved1:1; + u64 aes_key:2; + u64 iv_source:1; + u64 hash_type:4; + u64 reserved2:3; + u64 auth_input_type:1; + u64 mac_len:8; + u64 reserved3:8; + u64 encr_offset:16; + u64 iv_offset:8; + u64 auth_offset:8; +#else + u64 auth_offset:8; + u64 iv_offset:8; + u64 encr_offset:16; + u64 reserved3:8; + u64 mac_len:8; + u64 auth_input_type:1; + u64 reserved2:3; + u64 hash_type:4; + u64 iv_source:1; + u64 aes_key:2; + u64 reserved1:1; + u64 enc_cipher:4; +#endif + } e; +}; + +struct cvm_cipher { + const char *name; + u8 value; +}; + +struct enc_context { + union encr_ctrl enc_ctrl; + u8 encr_key[32]; + u8 encr_iv[16]; +}; + +struct fchmac_context { + u8 ipad[64]; + u8 opad[64]; /* or OPAD */ +}; + +struct fc_context { + struct enc_context enc; + struct fchmac_context hmac; +}; + +struct cvm_enc_ctx { + u32 key_len; + u8 enc_key[MAX_KEY_SIZE]; + u8 cipher_type:4; + u8 key_type:2; +}; + +struct cvm_des3_ctx { + u32 key_len; + u8 des3_key[MAX_KEY_SIZE]; +}; + +struct cvm_req_ctx { + struct cpt_request_info cpt_req; + u64 control_word; + struct fc_context fctx; +}; + +int cptvf_do_request(void *cptvf, struct cpt_request_info *req); +#endif /*_CPTVF_ALGS_H_*/ diff --git a/drivers/crypto/cavium/cpt/cptvf_main.c b/drivers/crypto/cavium/cpt/cptvf_main.c new file mode 100644 index 000000000..db76df14c --- /dev/null +++ b/drivers/crypto/cavium/cpt/cptvf_main.c @@ -0,0 +1,856 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2016 Cavium, Inc. + */ + +#include +#include + +#include "cptvf.h" + +#define DRV_NAME "thunder-cptvf" +#define DRV_VERSION "1.0" + +struct cptvf_wqe { + struct tasklet_struct twork; + void *cptvf; + u32 qno; +}; + +struct cptvf_wqe_info { + struct cptvf_wqe vq_wqe[CPT_NUM_QS_PER_VF]; +}; + +static void vq_work_handler(unsigned long data) +{ + struct cptvf_wqe_info *cwqe_info = (struct cptvf_wqe_info *)data; + struct cptvf_wqe *cwqe = &cwqe_info->vq_wqe[0]; + + vq_post_process(cwqe->cptvf, cwqe->qno); +} + +static int init_worker_threads(struct cpt_vf *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + struct cptvf_wqe_info *cwqe_info; + int i; + + cwqe_info = kzalloc_obj(*cwqe_info); + if (!cwqe_info) + return -ENOMEM; + + if (cptvf->nr_queues) { + dev_info(&pdev->dev, "Creating VQ worker threads (%d)\n", + cptvf->nr_queues); + } + + for (i = 0; i < cptvf->nr_queues; i++) { + tasklet_init(&cwqe_info->vq_wqe[i].twork, vq_work_handler, + (u64)cwqe_info); + cwqe_info->vq_wqe[i].qno = i; + cwqe_info->vq_wqe[i].cptvf = cptvf; + } + + cptvf->wqe_info = cwqe_info; + + return 0; +} + +static void cleanup_worker_threads(struct cpt_vf *cptvf) +{ + struct cptvf_wqe_info *cwqe_info; + struct pci_dev *pdev = cptvf->pdev; + int i; + + cwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info; + if (!cwqe_info) + return; + + if (cptvf->nr_queues) { + dev_info(&pdev->dev, "Cleaning VQ worker threads (%u)\n", + cptvf->nr_queues); + } + + for (i = 0; i < cptvf->nr_queues; i++) + tasklet_kill(&cwqe_info->vq_wqe[i].twork); + + kfree_sensitive(cwqe_info); + cptvf->wqe_info = NULL; +} + +static void free_pending_queues(struct pending_qinfo *pqinfo) +{ + int i; + struct pending_queue *queue; + + for_each_pending_queue(pqinfo, queue, i) { + if (!queue->head) + continue; + + /* free single queue */ + kfree_sensitive((queue->head)); + + queue->front = 0; + queue->rear = 0; + + return; + } + + pqinfo->qlen = 0; + pqinfo->nr_queues = 0; +} + +static int alloc_pending_queues(struct pending_qinfo *pqinfo, u32 qlen, + u32 nr_queues) +{ + u32 i; + int ret; + struct pending_queue *queue = NULL; + + pqinfo->nr_queues = nr_queues; + pqinfo->qlen = qlen; + + for_each_pending_queue(pqinfo, queue, i) { + queue->head = kzalloc_objs(*queue->head, qlen); + if (!queue->head) { + ret = -ENOMEM; + goto pending_qfail; + } + + queue->front = 0; + queue->rear = 0; + atomic64_set((&queue->pending_count), (0)); + + /* init queue spin lock */ + spin_lock_init(&queue->lock); + } + + return 0; + +pending_qfail: + free_pending_queues(pqinfo); + + return ret; +} + +static int init_pending_queues(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues) +{ + struct pci_dev *pdev = cptvf->pdev; + int ret; + + if (!nr_queues) + return 0; + + ret = alloc_pending_queues(&cptvf->pqinfo, qlen, nr_queues); + if (ret) { + dev_err(&pdev->dev, "failed to setup pending queues (%u)\n", + nr_queues); + return ret; + } + + return 0; +} + +static void cleanup_pending_queues(struct cpt_vf *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + + if (!cptvf->nr_queues) + return; + + dev_info(&pdev->dev, "Cleaning VQ pending queue (%u)\n", + cptvf->nr_queues); + free_pending_queues(&cptvf->pqinfo); +} + +static void free_command_queues(struct cpt_vf *cptvf, + struct command_qinfo *cqinfo) +{ + int i; + struct command_queue *queue = NULL; + struct command_chunk *chunk = NULL; + struct pci_dev *pdev = cptvf->pdev; + struct hlist_node *node; + + /* clean up for each queue */ + for (i = 0; i < cptvf->nr_queues; i++) { + queue = &cqinfo->queue[i]; + if (hlist_empty(&cqinfo->queue[i].chead)) + continue; + + hlist_for_each_entry_safe(chunk, node, &cqinfo->queue[i].chead, + nextchunk) { + dma_free_coherent(&pdev->dev, + chunk->size + CPT_NEXT_CHUNK_PTR_SIZE, + chunk->head, + chunk->dma_addr); + chunk->head = NULL; + chunk->dma_addr = 0; + hlist_del(&chunk->nextchunk); + kfree_sensitive(chunk); + } + + queue->nchunks = 0; + queue->idx = 0; + } + + /* common cleanup */ + cqinfo->cmd_size = 0; +} + +static int alloc_command_queues(struct cpt_vf *cptvf, + struct command_qinfo *cqinfo, size_t cmd_size, + u32 qlen) +{ + int i; + size_t q_size; + struct command_queue *queue = NULL; + struct pci_dev *pdev = cptvf->pdev; + + /* common init */ + cqinfo->cmd_size = cmd_size; + /* Qsize in dwords, needed for SADDR config, 1-next chunk pointer */ + cptvf->qsize = min(qlen, cqinfo->qchunksize) * + CPT_NEXT_CHUNK_PTR_SIZE + 1; + /* Qsize in bytes to create space for alignment */ + q_size = qlen * cqinfo->cmd_size; + + /* per queue initialization */ + for (i = 0; i < cptvf->nr_queues; i++) { + size_t c_size = 0; + size_t rem_q_size = q_size; + struct command_chunk *curr = NULL, *first = NULL, *last = NULL; + u32 qcsize_bytes = cqinfo->qchunksize * cqinfo->cmd_size; + + queue = &cqinfo->queue[i]; + INIT_HLIST_HEAD(&cqinfo->queue[i].chead); + do { + curr = kzalloc_obj(*curr); + if (!curr) + goto cmd_qfail; + + c_size = (rem_q_size > qcsize_bytes) ? qcsize_bytes : + rem_q_size; + curr->head = dma_alloc_coherent(&pdev->dev, + c_size + CPT_NEXT_CHUNK_PTR_SIZE, + &curr->dma_addr, + GFP_KERNEL); + if (!curr->head) { + dev_err(&pdev->dev, "Command Q (%d) chunk (%d) allocation failed\n", + i, queue->nchunks); + kfree(curr); + goto cmd_qfail; + } + + curr->size = c_size; + if (queue->nchunks == 0) { + hlist_add_head(&curr->nextchunk, + &cqinfo->queue[i].chead); + first = curr; + } else { + hlist_add_behind(&curr->nextchunk, + &last->nextchunk); + } + + queue->nchunks++; + rem_q_size -= c_size; + if (last) + *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr; + + last = curr; + } while (rem_q_size); + + /* Make the queue circular */ + /* Tie back last chunk entry to head */ + curr = first; + *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr; + queue->qhead = curr; + spin_lock_init(&queue->lock); + } + return 0; + +cmd_qfail: + free_command_queues(cptvf, cqinfo); + return -ENOMEM; +} + +static int init_command_queues(struct cpt_vf *cptvf, u32 qlen) +{ + struct pci_dev *pdev = cptvf->pdev; + int ret; + + /* setup AE command queues */ + ret = alloc_command_queues(cptvf, &cptvf->cqinfo, CPT_INST_SIZE, + qlen); + if (ret) { + dev_err(&pdev->dev, "failed to allocate AE command queues (%u)\n", + cptvf->nr_queues); + return ret; + } + + return ret; +} + +static void cleanup_command_queues(struct cpt_vf *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + + if (!cptvf->nr_queues) + return; + + dev_info(&pdev->dev, "Cleaning VQ command queue (%u)\n", + cptvf->nr_queues); + free_command_queues(cptvf, &cptvf->cqinfo); +} + +static void cptvf_sw_cleanup(struct cpt_vf *cptvf) +{ + cleanup_worker_threads(cptvf); + cleanup_pending_queues(cptvf); + cleanup_command_queues(cptvf); +} + +static int cptvf_sw_init(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues) +{ + struct pci_dev *pdev = cptvf->pdev; + int ret = 0; + u32 max_dev_queues = 0; + + max_dev_queues = CPT_NUM_QS_PER_VF; + /* possible cpus */ + nr_queues = min_t(u32, nr_queues, max_dev_queues); + cptvf->nr_queues = nr_queues; + + ret = init_command_queues(cptvf, qlen); + if (ret) { + dev_err(&pdev->dev, "Failed to setup command queues (%u)\n", + nr_queues); + return ret; + } + + ret = init_pending_queues(cptvf, qlen, nr_queues); + if (ret) { + dev_err(&pdev->dev, "Failed to setup pending queues (%u)\n", + nr_queues); + goto setup_pqfail; + } + + /* Create worker threads for BH processing */ + ret = init_worker_threads(cptvf); + if (ret) { + dev_err(&pdev->dev, "Failed to setup worker threads\n"); + goto init_work_fail; + } + + return 0; + +init_work_fail: + cleanup_worker_threads(cptvf); + cleanup_pending_queues(cptvf); + +setup_pqfail: + cleanup_command_queues(cptvf); + + return ret; +} + +static void cptvf_free_irq_affinity(struct cpt_vf *cptvf, int vec) +{ + irq_set_affinity_hint(pci_irq_vector(cptvf->pdev, vec), NULL); + free_cpumask_var(cptvf->affinity_mask[vec]); +} + +static void cptvf_write_vq_ctl(struct cpt_vf *cptvf, bool val) +{ + union cptx_vqx_ctl vqx_ctl; + + vqx_ctl.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0)); + vqx_ctl.s.ena = val; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0), vqx_ctl.u); +} + +void cptvf_write_vq_doorbell(struct cpt_vf *cptvf, u32 val) +{ + union cptx_vqx_doorbell vqx_dbell; + + vqx_dbell.u = cpt_read_csr64(cptvf->reg_base, + CPTX_VQX_DOORBELL(0, 0)); + vqx_dbell.s.dbell_cnt = val * 8; /* Num of Instructions * 8 words */ + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DOORBELL(0, 0), + vqx_dbell.u); +} + +static void cptvf_write_vq_inprog(struct cpt_vf *cptvf, u8 val) +{ + union cptx_vqx_inprog vqx_inprg; + + vqx_inprg.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0)); + vqx_inprg.s.inflight = val; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0), vqx_inprg.u); +} + +static void cptvf_write_vq_done_numwait(struct cpt_vf *cptvf, u32 val) +{ + union cptx_vqx_done_wait vqx_dwait; + + vqx_dwait.u = cpt_read_csr64(cptvf->reg_base, + CPTX_VQX_DONE_WAIT(0, 0)); + vqx_dwait.s.num_wait = val; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0), + vqx_dwait.u); +} + +static void cptvf_write_vq_done_timewait(struct cpt_vf *cptvf, u16 time) +{ + union cptx_vqx_done_wait vqx_dwait; + + vqx_dwait.u = cpt_read_csr64(cptvf->reg_base, + CPTX_VQX_DONE_WAIT(0, 0)); + vqx_dwait.s.time_wait = time; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0), + vqx_dwait.u); +} + +static void cptvf_enable_swerr_interrupts(struct cpt_vf *cptvf) +{ + union cptx_vqx_misc_ena_w1s vqx_misc_ena; + + vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base, + CPTX_VQX_MISC_ENA_W1S(0, 0)); + /* Set mbox(0) interupts for the requested vf */ + vqx_misc_ena.s.swerr = 1; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0), + vqx_misc_ena.u); +} + +static void cptvf_enable_mbox_interrupts(struct cpt_vf *cptvf) +{ + union cptx_vqx_misc_ena_w1s vqx_misc_ena; + + vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base, + CPTX_VQX_MISC_ENA_W1S(0, 0)); + /* Set mbox(0) interupts for the requested vf */ + vqx_misc_ena.s.mbox = 1; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0), + vqx_misc_ena.u); +} + +static void cptvf_enable_done_interrupts(struct cpt_vf *cptvf) +{ + union cptx_vqx_done_ena_w1s vqx_done_ena; + + vqx_done_ena.u = cpt_read_csr64(cptvf->reg_base, + CPTX_VQX_DONE_ENA_W1S(0, 0)); + /* Set DONE interrupt for the requested vf */ + vqx_done_ena.s.done = 1; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ENA_W1S(0, 0), + vqx_done_ena.u); +} + +static void cptvf_clear_dovf_intr(struct cpt_vf *cptvf) +{ + union cptx_vqx_misc_int vqx_misc_int; + + vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, + CPTX_VQX_MISC_INT(0, 0)); + /* W1C for the VF */ + vqx_misc_int.s.dovf = 1; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), + vqx_misc_int.u); +} + +static void cptvf_clear_irde_intr(struct cpt_vf *cptvf) +{ + union cptx_vqx_misc_int vqx_misc_int; + + vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, + CPTX_VQX_MISC_INT(0, 0)); + /* W1C for the VF */ + vqx_misc_int.s.irde = 1; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), + vqx_misc_int.u); +} + +static void cptvf_clear_nwrp_intr(struct cpt_vf *cptvf) +{ + union cptx_vqx_misc_int vqx_misc_int; + + vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, + CPTX_VQX_MISC_INT(0, 0)); + /* W1C for the VF */ + vqx_misc_int.s.nwrp = 1; + cpt_write_csr64(cptvf->reg_base, + CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u); +} + +static void cptvf_clear_mbox_intr(struct cpt_vf *cptvf) +{ + union cptx_vqx_misc_int vqx_misc_int; + + vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, + CPTX_VQX_MISC_INT(0, 0)); + /* W1C for the VF */ + vqx_misc_int.s.mbox = 1; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), + vqx_misc_int.u); +} + +static void cptvf_clear_swerr_intr(struct cpt_vf *cptvf) +{ + union cptx_vqx_misc_int vqx_misc_int; + + vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, + CPTX_VQX_MISC_INT(0, 0)); + /* W1C for the VF */ + vqx_misc_int.s.swerr = 1; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), + vqx_misc_int.u); +} + +static u64 cptvf_read_vf_misc_intr_status(struct cpt_vf *cptvf) +{ + return cpt_read_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0)); +} + +static irqreturn_t cptvf_misc_intr_handler(int irq, void *cptvf_irq) +{ + struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq; + struct pci_dev *pdev = cptvf->pdev; + u64 intr; + + intr = cptvf_read_vf_misc_intr_status(cptvf); + /*Check for MISC interrupt types*/ + if (likely(intr & CPT_VF_INTR_MBOX_MASK)) { + dev_dbg(&pdev->dev, "Mailbox interrupt 0x%llx on CPT VF %d\n", + intr, cptvf->vfid); + cptvf_handle_mbox_intr(cptvf); + cptvf_clear_mbox_intr(cptvf); + } else if (unlikely(intr & CPT_VF_INTR_DOVF_MASK)) { + cptvf_clear_dovf_intr(cptvf); + /*Clear doorbell count*/ + cptvf_write_vq_doorbell(cptvf, 0); + dev_err(&pdev->dev, "Doorbell overflow error interrupt 0x%llx on CPT VF %d\n", + intr, cptvf->vfid); + } else if (unlikely(intr & CPT_VF_INTR_IRDE_MASK)) { + cptvf_clear_irde_intr(cptvf); + dev_err(&pdev->dev, "Instruction NCB read error interrupt 0x%llx on CPT VF %d\n", + intr, cptvf->vfid); + } else if (unlikely(intr & CPT_VF_INTR_NWRP_MASK)) { + cptvf_clear_nwrp_intr(cptvf); + dev_err(&pdev->dev, "NCB response write error interrupt 0x%llx on CPT VF %d\n", + intr, cptvf->vfid); + } else if (unlikely(intr & CPT_VF_INTR_SERR_MASK)) { + cptvf_clear_swerr_intr(cptvf); + dev_err(&pdev->dev, "Software error interrupt 0x%llx on CPT VF %d\n", + intr, cptvf->vfid); + } else { + dev_err(&pdev->dev, "Unhandled interrupt in CPT VF %d\n", + cptvf->vfid); + } + + return IRQ_HANDLED; +} + +static inline struct cptvf_wqe *get_cptvf_vq_wqe(struct cpt_vf *cptvf, + int qno) +{ + struct cptvf_wqe_info *nwqe_info; + + if (unlikely(qno >= cptvf->nr_queues)) + return NULL; + nwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info; + + return &nwqe_info->vq_wqe[qno]; +} + +static inline u32 cptvf_read_vq_done_count(struct cpt_vf *cptvf) +{ + union cptx_vqx_done vqx_done; + + vqx_done.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_DONE(0, 0)); + return vqx_done.s.done; +} + +static inline void cptvf_write_vq_done_ack(struct cpt_vf *cptvf, + u32 ackcnt) +{ + union cptx_vqx_done_ack vqx_dack_cnt; + + vqx_dack_cnt.u = cpt_read_csr64(cptvf->reg_base, + CPTX_VQX_DONE_ACK(0, 0)); + vqx_dack_cnt.s.done_ack = ackcnt; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ACK(0, 0), + vqx_dack_cnt.u); +} + +static irqreturn_t cptvf_done_intr_handler(int irq, void *cptvf_irq) +{ + struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq; + struct pci_dev *pdev = cptvf->pdev; + /* Read the number of completions */ + u32 intr = cptvf_read_vq_done_count(cptvf); + + if (intr) { + struct cptvf_wqe *wqe; + + /* Acknowledge the number of + * scheduled completions for processing + */ + cptvf_write_vq_done_ack(cptvf, intr); + wqe = get_cptvf_vq_wqe(cptvf, 0); + if (unlikely(!wqe)) { + dev_err(&pdev->dev, "No work to schedule for VF (%d)", + cptvf->vfid); + return IRQ_NONE; + } + tasklet_hi_schedule(&wqe->twork); + } + + return IRQ_HANDLED; +} + +static void cptvf_set_irq_affinity(struct cpt_vf *cptvf, int vec) +{ + struct pci_dev *pdev = cptvf->pdev; + int cpu; + + if (!zalloc_cpumask_var(&cptvf->affinity_mask[vec], + GFP_KERNEL)) { + dev_err(&pdev->dev, "Allocation failed for affinity_mask for VF %d", + cptvf->vfid); + return; + } + + cpu = cptvf->vfid % num_online_cpus(); + cpumask_set_cpu(cpumask_local_spread(cpu, cptvf->node), + cptvf->affinity_mask[vec]); + irq_set_affinity_hint(pci_irq_vector(pdev, vec), + cptvf->affinity_mask[vec]); +} + +static void cptvf_write_vq_saddr(struct cpt_vf *cptvf, u64 val) +{ + union cptx_vqx_saddr vqx_saddr; + + vqx_saddr.u = val; + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_SADDR(0, 0), vqx_saddr.u); +} + +static void cptvf_device_init(struct cpt_vf *cptvf) +{ + u64 base_addr = 0; + + /* Disable the VQ */ + cptvf_write_vq_ctl(cptvf, 0); + /* Reset the doorbell */ + cptvf_write_vq_doorbell(cptvf, 0); + /* Clear inflight */ + cptvf_write_vq_inprog(cptvf, 0); + /* Write VQ SADDR */ + /* TODO: for now only one queue, so hard coded */ + base_addr = (u64)(cptvf->cqinfo.queue[0].qhead->dma_addr); + cptvf_write_vq_saddr(cptvf, base_addr); + /* Configure timerhold / coalescence */ + cptvf_write_vq_done_timewait(cptvf, CPT_TIMER_THOLD); + cptvf_write_vq_done_numwait(cptvf, 1); + /* Enable the VQ */ + cptvf_write_vq_ctl(cptvf, 1); + /* Flag the VF ready */ + cptvf->flags |= CPT_FLAG_DEVICE_READY; +} + +static int cptvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct device *dev = &pdev->dev; + struct cpt_vf *cptvf; + int err; + + cptvf = devm_kzalloc(dev, sizeof(*cptvf), GFP_KERNEL); + if (!cptvf) + return -ENOMEM; + + pci_set_drvdata(pdev, cptvf); + cptvf->pdev = pdev; + err = pci_enable_device(pdev); + if (err) { + dev_err(dev, "Failed to enable PCI device\n"); + pci_set_drvdata(pdev, NULL); + return err; + } + + err = pci_request_regions(pdev, DRV_NAME); + if (err) { + dev_err(dev, "PCI request regions failed 0x%x\n", err); + goto cptvf_err_disable_device; + } + /* Mark as VF driver */ + cptvf->flags |= CPT_FLAG_VF_DRIVER; + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); + if (err) { + dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); + goto cptvf_err_release_regions; + } + + /* MAP PF's configuration registers */ + cptvf->reg_base = pcim_iomap(pdev, 0, 0); + if (!cptvf->reg_base) { + dev_err(dev, "Cannot map config register space, aborting\n"); + err = -ENOMEM; + goto cptvf_err_release_regions; + } + + cptvf->node = dev_to_node(&pdev->dev); + err = pci_alloc_irq_vectors(pdev, CPT_VF_MSIX_VECTORS, + CPT_VF_MSIX_VECTORS, PCI_IRQ_MSIX); + if (err < 0) { + dev_err(dev, "Request for #%d msix vectors failed\n", + CPT_VF_MSIX_VECTORS); + goto cptvf_err_release_regions; + } + + err = request_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), + cptvf_misc_intr_handler, 0, "CPT VF misc intr", + cptvf); + if (err) { + dev_err(dev, "Request misc irq failed"); + goto cptvf_free_vectors; + } + + /* Enable mailbox interrupt */ + cptvf_enable_mbox_interrupts(cptvf); + cptvf_enable_swerr_interrupts(cptvf); + + /* Check ready with PF */ + /* Gets chip ID / device Id from PF if ready */ + err = cptvf_check_pf_ready(cptvf); + if (err) { + dev_err(dev, "PF not responding to READY msg"); + goto cptvf_free_misc_irq; + } + + /* CPT VF software resources initialization */ + cptvf->cqinfo.qchunksize = CPT_CMD_QCHUNK_SIZE; + err = cptvf_sw_init(cptvf, CPT_CMD_QLEN, CPT_NUM_QS_PER_VF); + if (err) { + dev_err(dev, "cptvf_sw_init() failed"); + goto cptvf_free_misc_irq; + } + /* Convey VQ LEN to PF */ + err = cptvf_send_vq_size_msg(cptvf); + if (err) { + dev_err(dev, "PF not responding to QLEN msg"); + goto cptvf_free_misc_irq; + } + + /* CPT VF device initialization */ + cptvf_device_init(cptvf); + /* Send msg to PF to assign currnet Q to required group */ + cptvf->vfgrp = 1; + err = cptvf_send_vf_to_grp_msg(cptvf); + if (err) { + dev_err(dev, "PF not responding to VF_GRP msg"); + goto cptvf_free_misc_irq; + } + + cptvf->priority = 1; + err = cptvf_send_vf_priority_msg(cptvf); + if (err) { + dev_err(dev, "PF not responding to VF_PRIO msg"); + goto cptvf_free_misc_irq; + } + + err = request_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE), + cptvf_done_intr_handler, 0, "CPT VF done intr", + cptvf); + if (err) { + dev_err(dev, "Request done irq failed\n"); + goto cptvf_free_misc_irq; + } + + /* Enable mailbox interrupt */ + cptvf_enable_done_interrupts(cptvf); + + /* Set irq affinity masks */ + cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC); + cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE); + + err = cptvf_send_vf_up(cptvf); + if (err) { + dev_err(dev, "PF not responding to UP msg"); + goto cptvf_free_irq_affinity; + } + err = cvm_crypto_init(cptvf); + if (err) { + dev_err(dev, "Algorithm register failed\n"); + goto cptvf_free_irq_affinity; + } + return 0; + +cptvf_free_irq_affinity: + cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE); + cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC); +cptvf_free_misc_irq: + free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf); +cptvf_free_vectors: + pci_free_irq_vectors(cptvf->pdev); +cptvf_err_release_regions: + pci_release_regions(pdev); +cptvf_err_disable_device: + pci_disable_device(pdev); + pci_set_drvdata(pdev, NULL); + + return err; +} + +static void cptvf_remove(struct pci_dev *pdev) +{ + struct cpt_vf *cptvf = pci_get_drvdata(pdev); + + if (!cptvf) { + dev_err(&pdev->dev, "Invalid CPT-VF device\n"); + return; + } + + /* Convey DOWN to PF */ + if (cptvf_send_vf_down(cptvf)) { + dev_err(&pdev->dev, "PF not responding to DOWN msg"); + } else { + cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE); + cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC); + free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE), cptvf); + free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf); + pci_free_irq_vectors(cptvf->pdev); + cptvf_sw_cleanup(cptvf); + pci_set_drvdata(pdev, NULL); + pci_release_regions(pdev); + pci_disable_device(pdev); + cvm_crypto_exit(); + } +} + +static void cptvf_shutdown(struct pci_dev *pdev) +{ + cptvf_remove(pdev); +} + +/* Supported devices */ +static const struct pci_device_id cptvf_id_table[] = { + { PCI_VDEVICE(CAVIUM, CPT_81XX_PCI_VF_DEVICE_ID) }, + { } /* end of table */ +}; +MODULE_DEVICE_TABLE(pci, cptvf_id_table); + +static struct pci_driver cptvf_pci_driver = { + .name = DRV_NAME, + .id_table = cptvf_id_table, + .probe = cptvf_probe, + .remove = cptvf_remove, + .shutdown = cptvf_shutdown, +}; + +module_pci_driver(cptvf_pci_driver); + +MODULE_AUTHOR("George Cherian "); +MODULE_DESCRIPTION("Cavium Thunder CPT Virtual Function Driver"); +MODULE_LICENSE("GPL v2"); +MODULE_VERSION(DRV_VERSION); diff --git a/drivers/crypto/cavium/cpt/cptvf_mbox.c b/drivers/crypto/cavium/cpt/cptvf_mbox.c new file mode 100644 index 000000000..1267e1eba --- /dev/null +++ b/drivers/crypto/cavium/cpt/cptvf_mbox.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2016 Cavium, Inc. + */ + +#include "cptvf.h" + +static void cptvf_send_msg_to_pf(struct cpt_vf *cptvf, struct cpt_mbox *mbx) +{ + /* Writing mbox(1) causes interrupt */ + cpt_write_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 0), + mbx->msg); + cpt_write_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 1), + mbx->data); +} + +/* Interrupt handler to handle mailbox messages from VFs */ +void cptvf_handle_mbox_intr(struct cpt_vf *cptvf) +{ + struct cpt_mbox mbx = {}; + + /* + * MBOX[0] contains msg + * MBOX[1] contains data + */ + mbx.msg = cpt_read_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 0)); + mbx.data = cpt_read_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 1)); + dev_dbg(&cptvf->pdev->dev, "%s: Mailbox msg 0x%llx from PF\n", + __func__, mbx.msg); + switch (mbx.msg) { + case CPT_MSG_READY: + { + cptvf->pf_acked = true; + cptvf->vfid = mbx.data; + dev_dbg(&cptvf->pdev->dev, "Received VFID %d\n", cptvf->vfid); + break; + } + case CPT_MSG_QBIND_GRP: + cptvf->pf_acked = true; + cptvf->vftype = mbx.data; + dev_dbg(&cptvf->pdev->dev, "VF %d type %s group %d\n", + cptvf->vfid, ((mbx.data == SE_TYPES) ? "SE" : "AE"), + cptvf->vfgrp); + break; + case CPT_MBOX_MSG_TYPE_ACK: + cptvf->pf_acked = true; + break; + case CPT_MBOX_MSG_TYPE_NACK: + cptvf->pf_nacked = true; + break; + default: + dev_err(&cptvf->pdev->dev, "Invalid msg from PF, msg 0x%llx\n", + mbx.msg); + break; + } +} + +static int cptvf_send_msg_to_pf_timeout(struct cpt_vf *cptvf, + struct cpt_mbox *mbx) +{ + int timeout = CPT_MBOX_MSG_TIMEOUT; + int sleep = 10; + + cptvf->pf_acked = false; + cptvf->pf_nacked = false; + cptvf_send_msg_to_pf(cptvf, mbx); + /* Wait for previous message to be acked, timeout 2sec */ + while (!cptvf->pf_acked) { + if (cptvf->pf_nacked) + return -EINVAL; + msleep(sleep); + if (cptvf->pf_acked) + break; + timeout -= sleep; + if (!timeout) { + dev_err(&cptvf->pdev->dev, "PF didn't ack to mbox msg %llx from VF%u\n", + (mbx->msg & 0xFF), cptvf->vfid); + return -EBUSY; + } + } + + return 0; +} + +/* + * Checks if VF is able to comminicate with PF + * and also gets the CPT number this VF is associated to. + */ +int cptvf_check_pf_ready(struct cpt_vf *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + struct cpt_mbox mbx = {}; + + mbx.msg = CPT_MSG_READY; + if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) { + dev_err(&pdev->dev, "PF didn't respond to READY msg\n"); + return -EBUSY; + } + + return 0; +} + +/* + * Communicate VQs size to PF to program CPT(0)_PF_Q(0-15)_CTL of the VF. + * Must be ACKed. + */ +int cptvf_send_vq_size_msg(struct cpt_vf *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + struct cpt_mbox mbx = {}; + + mbx.msg = CPT_MSG_QLEN; + mbx.data = cptvf->qsize; + if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) { + dev_err(&pdev->dev, "PF didn't respond to vq_size msg\n"); + return -EBUSY; + } + + return 0; +} + +/* + * Communicate VF group required to PF and get the VQ binded to that group + */ +int cptvf_send_vf_to_grp_msg(struct cpt_vf *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + struct cpt_mbox mbx = {}; + + mbx.msg = CPT_MSG_QBIND_GRP; + /* Convey group of the VF */ + mbx.data = cptvf->vfgrp; + if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) { + dev_err(&pdev->dev, "PF didn't respond to vf_type msg\n"); + return -EBUSY; + } + + return 0; +} + +/* + * Communicate VF group required to PF and get the VQ binded to that group + */ +int cptvf_send_vf_priority_msg(struct cpt_vf *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + struct cpt_mbox mbx = {}; + + mbx.msg = CPT_MSG_VQ_PRIORITY; + /* Convey group of the VF */ + mbx.data = cptvf->priority; + if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) { + dev_err(&pdev->dev, "PF didn't respond to vf_type msg\n"); + return -EBUSY; + } + return 0; +} + +/* + * Communicate to PF that VF is UP and running + */ +int cptvf_send_vf_up(struct cpt_vf *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + struct cpt_mbox mbx = {}; + + mbx.msg = CPT_MSG_VF_UP; + if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) { + dev_err(&pdev->dev, "PF didn't respond to UP msg\n"); + return -EBUSY; + } + + return 0; +} + +/* + * Communicate to PF that VF is DOWN and running + */ +int cptvf_send_vf_down(struct cpt_vf *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + struct cpt_mbox mbx = {}; + + mbx.msg = CPT_MSG_VF_DOWN; + if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) { + dev_err(&pdev->dev, "PF didn't respond to DOWN msg\n"); + return -EBUSY; + } + + return 0; +} diff --git a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c new file mode 100644 index 000000000..de305cbec --- /dev/null +++ b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c @@ -0,0 +1,575 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2016 Cavium, Inc. + */ + +#include "cptvf.h" +#include "cptvf_algs.h" +#include "request_manager.h" + +/** + * get_free_pending_entry - get free entry from pending queue + * @q: pending queue + * @qlen: queue length + */ +static struct pending_entry *get_free_pending_entry(struct pending_queue *q, + int qlen) +{ + struct pending_entry *ent = NULL; + + ent = &q->head[q->rear]; + if (unlikely(ent->busy)) { + ent = NULL; + goto no_free_entry; + } + + q->rear++; + if (unlikely(q->rear == qlen)) + q->rear = 0; + +no_free_entry: + return ent; +} + +static inline void pending_queue_inc_front(struct pending_qinfo *pqinfo, + int qno) +{ + struct pending_queue *queue = &pqinfo->queue[qno]; + + queue->front++; + if (unlikely(queue->front == pqinfo->qlen)) + queue->front = 0; +} + +static int setup_sgio_components(struct cpt_vf *cptvf, struct buf_ptr *list, + int buf_count, u8 *buffer) +{ + int ret = 0, i, j; + int components; + struct sglist_component *sg_ptr = NULL; + struct pci_dev *pdev = cptvf->pdev; + + if (unlikely(!list)) { + dev_err(&pdev->dev, "Input List pointer is NULL\n"); + return -EFAULT; + } + + for (i = 0; i < buf_count; i++) { + if (likely(list[i].vptr)) { + list[i].dma_addr = dma_map_single(&pdev->dev, + list[i].vptr, + list[i].size, + DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(&pdev->dev, + list[i].dma_addr))) { + dev_err(&pdev->dev, "DMA map kernel buffer failed for component: %d\n", + i); + ret = -EIO; + goto sg_cleanup; + } + } + } + + components = buf_count / 4; + sg_ptr = (struct sglist_component *)buffer; + for (i = 0; i < components; i++) { + sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size); + sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size); + sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size); + sg_ptr->u.s.len3 = cpu_to_be16(list[i * 4 + 3].size); + sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr); + sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr); + sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr); + sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr); + sg_ptr++; + } + + components = buf_count % 4; + + switch (components) { + case 3: + sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size); + sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr); + fallthrough; + case 2: + sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size); + sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr); + fallthrough; + case 1: + sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size); + sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr); + break; + default: + break; + } + + return ret; + +sg_cleanup: + for (j = 0; j < i; j++) { + if (list[j].dma_addr) { + dma_unmap_single(&pdev->dev, list[j].dma_addr, + list[j].size, DMA_BIDIRECTIONAL); + } + + list[j].dma_addr = 0; + } + + return ret; +} + +static inline int setup_sgio_list(struct cpt_vf *cptvf, + struct cpt_info_buffer *info, + struct cpt_request_info *req) +{ + u16 g_sz_bytes = 0, s_sz_bytes = 0; + int ret = 0; + struct pci_dev *pdev = cptvf->pdev; + + if (req->incnt > MAX_SG_IN_CNT || req->outcnt > MAX_SG_OUT_CNT) { + dev_err(&pdev->dev, "Request SG components are higher than supported\n"); + ret = -EINVAL; + goto scatter_gather_clean; + } + + /* Setup gather (input) components */ + g_sz_bytes = ((req->incnt + 3) / 4) * sizeof(struct sglist_component); + info->gather_components = kzalloc(g_sz_bytes, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC); + if (!info->gather_components) { + ret = -ENOMEM; + goto scatter_gather_clean; + } + + ret = setup_sgio_components(cptvf, req->in, + req->incnt, + info->gather_components); + if (ret) { + dev_err(&pdev->dev, "Failed to setup gather list\n"); + ret = -EFAULT; + goto scatter_gather_clean; + } + + /* Setup scatter (output) components */ + s_sz_bytes = ((req->outcnt + 3) / 4) * sizeof(struct sglist_component); + info->scatter_components = kzalloc(s_sz_bytes, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC); + if (!info->scatter_components) { + ret = -ENOMEM; + goto scatter_gather_clean; + } + + ret = setup_sgio_components(cptvf, req->out, + req->outcnt, + info->scatter_components); + if (ret) { + dev_err(&pdev->dev, "Failed to setup gather list\n"); + ret = -EFAULT; + goto scatter_gather_clean; + } + + /* Create and initialize DPTR */ + info->dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE; + info->in_buffer = kzalloc(info->dlen, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC); + if (!info->in_buffer) { + ret = -ENOMEM; + goto scatter_gather_clean; + } + + ((__be16 *)info->in_buffer)[0] = cpu_to_be16(req->outcnt); + ((__be16 *)info->in_buffer)[1] = cpu_to_be16(req->incnt); + ((__be16 *)info->in_buffer)[2] = 0; + ((__be16 *)info->in_buffer)[3] = 0; + + memcpy(&info->in_buffer[8], info->gather_components, + g_sz_bytes); + memcpy(&info->in_buffer[8 + g_sz_bytes], + info->scatter_components, s_sz_bytes); + + info->dptr_baddr = dma_map_single(&pdev->dev, + (void *)info->in_buffer, + info->dlen, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(&pdev->dev, info->dptr_baddr)) { + dev_err(&pdev->dev, "Mapping DPTR Failed %d\n", info->dlen); + ret = -EIO; + goto scatter_gather_clean; + } + + /* Create and initialize RPTR */ + info->out_buffer = kzalloc(COMPLETION_CODE_SIZE, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC); + if (!info->out_buffer) { + ret = -ENOMEM; + goto scatter_gather_clean; + } + + *((u64 *)info->out_buffer) = ~((u64)COMPLETION_CODE_INIT); + info->alternate_caddr = (u64 *)info->out_buffer; + info->rptr_baddr = dma_map_single(&pdev->dev, + (void *)info->out_buffer, + COMPLETION_CODE_SIZE, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(&pdev->dev, info->rptr_baddr)) { + dev_err(&pdev->dev, "Mapping RPTR Failed %d\n", + COMPLETION_CODE_SIZE); + ret = -EIO; + goto scatter_gather_clean; + } + + return 0; + +scatter_gather_clean: + return ret; +} + +static int send_cpt_command(struct cpt_vf *cptvf, union cpt_inst_s *cmd, + u32 qno) +{ + struct pci_dev *pdev = cptvf->pdev; + struct command_qinfo *qinfo = NULL; + struct command_queue *queue; + struct command_chunk *chunk; + u8 *ent; + int ret = 0; + + if (unlikely(qno >= cptvf->nr_queues)) { + dev_err(&pdev->dev, "Invalid queue (qno: %d, nr_queues: %d)\n", + qno, cptvf->nr_queues); + return -EINVAL; + } + + qinfo = &cptvf->cqinfo; + queue = &qinfo->queue[qno]; + /* lock command queue */ + spin_lock(&queue->lock); + ent = &queue->qhead->head[queue->idx * qinfo->cmd_size]; + memcpy(ent, (void *)cmd, qinfo->cmd_size); + + if (++queue->idx >= queue->qhead->size / 64) { + hlist_for_each_entry(chunk, &queue->chead, nextchunk) { + if (chunk == queue->qhead) { + continue; + } else { + queue->qhead = chunk; + break; + } + } + queue->idx = 0; + } + /* make sure all memory stores are done before ringing doorbell */ + smp_wmb(); + cptvf_write_vq_doorbell(cptvf, 1); + /* unlock command queue */ + spin_unlock(&queue->lock); + + return ret; +} + +static void do_request_cleanup(struct cpt_vf *cptvf, + struct cpt_info_buffer *info) +{ + int i; + struct pci_dev *pdev = cptvf->pdev; + struct cpt_request_info *req; + + if (info->dptr_baddr) + dma_unmap_single(&pdev->dev, info->dptr_baddr, + info->dlen, DMA_BIDIRECTIONAL); + + if (info->rptr_baddr) + dma_unmap_single(&pdev->dev, info->rptr_baddr, + COMPLETION_CODE_SIZE, DMA_BIDIRECTIONAL); + + if (info->comp_baddr) + dma_unmap_single(&pdev->dev, info->comp_baddr, + sizeof(union cpt_res_s), DMA_BIDIRECTIONAL); + + if (info->req) { + req = info->req; + for (i = 0; i < req->outcnt; i++) { + if (req->out[i].dma_addr) + dma_unmap_single(&pdev->dev, + req->out[i].dma_addr, + req->out[i].size, + DMA_BIDIRECTIONAL); + } + + for (i = 0; i < req->incnt; i++) { + if (req->in[i].dma_addr) + dma_unmap_single(&pdev->dev, + req->in[i].dma_addr, + req->in[i].size, + DMA_BIDIRECTIONAL); + } + } + + kfree_sensitive(info->scatter_components); + kfree_sensitive(info->gather_components); + kfree_sensitive(info->out_buffer); + kfree_sensitive(info->in_buffer); + kfree_sensitive((void *)info->completion_addr); + kfree_sensitive(info); +} + +static void do_post_process(struct cpt_vf *cptvf, struct cpt_info_buffer *info) +{ + struct pci_dev *pdev = cptvf->pdev; + + if (!info) { + dev_err(&pdev->dev, "incorrect cpt_info_buffer for post processing\n"); + return; + } + + do_request_cleanup(cptvf, info); +} + +static inline void process_pending_queue(struct cpt_vf *cptvf, + struct pending_qinfo *pqinfo, + int qno) +{ + struct pci_dev *pdev = cptvf->pdev; + struct pending_queue *pqueue = &pqinfo->queue[qno]; + struct pending_entry *pentry = NULL; + struct cpt_info_buffer *info = NULL; + union cpt_res_s *status = NULL; + unsigned char ccode; + + while (1) { + spin_lock_bh(&pqueue->lock); + pentry = &pqueue->head[pqueue->front]; + if (unlikely(!pentry->busy)) { + spin_unlock_bh(&pqueue->lock); + break; + } + + info = (struct cpt_info_buffer *)pentry->post_arg; + if (unlikely(!info)) { + dev_err(&pdev->dev, "Pending Entry post arg NULL\n"); + pending_queue_inc_front(pqinfo, qno); + spin_unlock_bh(&pqueue->lock); + continue; + } + + status = (union cpt_res_s *)pentry->completion_addr; + ccode = status->s.compcode; + if ((status->s.compcode == CPT_COMP_E_FAULT) || + (status->s.compcode == CPT_COMP_E_SWERR)) { + dev_err(&pdev->dev, "Request failed with %s\n", + (status->s.compcode == CPT_COMP_E_FAULT) ? + "DMA Fault" : "Software error"); + pentry->completion_addr = NULL; + pentry->busy = false; + atomic64_dec((&pqueue->pending_count)); + pentry->post_arg = NULL; + pending_queue_inc_front(pqinfo, qno); + do_request_cleanup(cptvf, info); + spin_unlock_bh(&pqueue->lock); + break; + } else if (status->s.compcode == COMPLETION_CODE_INIT) { + /* check for timeout */ + if (time_after_eq(jiffies, + (info->time_in + + (CPT_COMMAND_TIMEOUT * HZ)))) { + dev_err(&pdev->dev, "Request timed out"); + pentry->completion_addr = NULL; + pentry->busy = false; + atomic64_dec((&pqueue->pending_count)); + pentry->post_arg = NULL; + pending_queue_inc_front(pqinfo, qno); + do_request_cleanup(cptvf, info); + spin_unlock_bh(&pqueue->lock); + break; + } else if ((*info->alternate_caddr == + (~COMPLETION_CODE_INIT)) && + (info->extra_time < TIME_IN_RESET_COUNT)) { + info->time_in = jiffies; + info->extra_time++; + spin_unlock_bh(&pqueue->lock); + break; + } + } + + pentry->completion_addr = NULL; + pentry->busy = false; + pentry->post_arg = NULL; + atomic64_dec((&pqueue->pending_count)); + pending_queue_inc_front(pqinfo, qno); + spin_unlock_bh(&pqueue->lock); + + do_post_process(info->cptvf, info); + /* + * Calling callback after we find + * that the request has been serviced + */ + pentry->callback(ccode, pentry->callback_arg); + } +} + +int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req) +{ + int ret = 0, clear = 0, queue = 0; + struct cpt_info_buffer *info = NULL; + struct cptvf_request *cpt_req = NULL; + union ctrl_info *ctrl = NULL; + union cpt_res_s *result = NULL; + struct pending_entry *pentry = NULL; + struct pending_queue *pqueue = NULL; + struct pci_dev *pdev = cptvf->pdev; + u8 group = 0; + struct cpt_vq_command vq_cmd; + union cpt_inst_s cptinst; + + info = kzalloc_obj(*info, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC); + if (unlikely(!info)) { + dev_err(&pdev->dev, "Unable to allocate memory for info_buffer\n"); + return -ENOMEM; + } + + cpt_req = (struct cptvf_request *)&req->req; + ctrl = (union ctrl_info *)&req->ctrl; + + info->cptvf = cptvf; + group = ctrl->s.grp; + ret = setup_sgio_list(cptvf, info, req); + if (ret) { + dev_err(&pdev->dev, "Setting up SG list failed"); + goto request_cleanup; + } + + cpt_req->dlen = info->dlen; + /* + * Get buffer for union cpt_res_s response + * structure and its physical address + */ + info->completion_addr = kzalloc(sizeof(union cpt_res_s), req->may_sleep ? GFP_KERNEL : GFP_ATOMIC); + if (unlikely(!info->completion_addr)) { + dev_err(&pdev->dev, "Unable to allocate memory for completion_addr\n"); + ret = -ENOMEM; + goto request_cleanup; + } + + result = (union cpt_res_s *)info->completion_addr; + result->s.compcode = COMPLETION_CODE_INIT; + info->comp_baddr = dma_map_single(&pdev->dev, + (void *)info->completion_addr, + sizeof(union cpt_res_s), + DMA_BIDIRECTIONAL); + if (dma_mapping_error(&pdev->dev, info->comp_baddr)) { + dev_err(&pdev->dev, "mapping compptr Failed %lu\n", + sizeof(union cpt_res_s)); + ret = -EFAULT; + goto request_cleanup; + } + + /* Fill the VQ command */ + vq_cmd.cmd.u64 = 0; + vq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags); + vq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1); + vq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2); + vq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen); + + vq_cmd.dptr = info->dptr_baddr; + vq_cmd.rptr = info->rptr_baddr; + vq_cmd.cptr.u64 = 0; + vq_cmd.cptr.s.grp = group; + /* Get Pending Entry to submit command */ + /* Always queue 0, because 1 queue per VF */ + queue = 0; + pqueue = &cptvf->pqinfo.queue[queue]; + + if (atomic64_read(&pqueue->pending_count) > PENDING_THOLD) { + dev_err(&pdev->dev, "pending threshold reached\n"); + process_pending_queue(cptvf, &cptvf->pqinfo, queue); + } + +get_pending_entry: + spin_lock_bh(&pqueue->lock); + pentry = get_free_pending_entry(pqueue, cptvf->pqinfo.qlen); + if (unlikely(!pentry)) { + spin_unlock_bh(&pqueue->lock); + if (clear == 0) { + process_pending_queue(cptvf, &cptvf->pqinfo, queue); + clear = 1; + goto get_pending_entry; + } + dev_err(&pdev->dev, "Get free entry failed\n"); + dev_err(&pdev->dev, "queue: %d, rear: %d, front: %d\n", + queue, pqueue->rear, pqueue->front); + ret = -EFAULT; + goto request_cleanup; + } + + pentry->completion_addr = info->completion_addr; + pentry->post_arg = (void *)info; + pentry->callback = req->callback; + pentry->callback_arg = req->callback_arg; + info->pentry = pentry; + pentry->busy = true; + atomic64_inc(&pqueue->pending_count); + + /* Send CPT command */ + info->pentry = pentry; + info->time_in = jiffies; + info->req = req; + + /* Create the CPT_INST_S type command for HW interpretation */ + cptinst.s.doneint = true; + cptinst.s.res_addr = (u64)info->comp_baddr; + cptinst.s.tag = 0; + cptinst.s.grp = 0; + cptinst.s.wq_ptr = 0; + cptinst.s.ei0 = vq_cmd.cmd.u64; + cptinst.s.ei1 = vq_cmd.dptr; + cptinst.s.ei2 = vq_cmd.rptr; + cptinst.s.ei3 = vq_cmd.cptr.u64; + + ret = send_cpt_command(cptvf, &cptinst, queue); + spin_unlock_bh(&pqueue->lock); + if (unlikely(ret)) { + dev_err(&pdev->dev, "Send command failed for AE\n"); + ret = -EFAULT; + goto request_cleanup; + } + + return 0; + +request_cleanup: + dev_dbg(&pdev->dev, "Failed to submit CPT command\n"); + do_request_cleanup(cptvf, info); + + return ret; +} + +void vq_post_process(struct cpt_vf *cptvf, u32 qno) +{ + struct pci_dev *pdev = cptvf->pdev; + + if (unlikely(qno > cptvf->nr_queues)) { + dev_err(&pdev->dev, "Request for post processing on invalid pending queue: %u\n", + qno); + return; + } + + process_pending_queue(cptvf, &cptvf->pqinfo, qno); +} + +int cptvf_do_request(void *vfdev, struct cpt_request_info *req) +{ + struct cpt_vf *cptvf = (struct cpt_vf *)vfdev; + struct pci_dev *pdev = cptvf->pdev; + + if (!cpt_device_ready(cptvf)) { + dev_err(&pdev->dev, "CPT Device is not ready"); + return -ENODEV; + } + + if ((cptvf->vftype == SE_TYPES) && (!req->ctrl.s.se_req)) { + dev_err(&pdev->dev, "CPTVF-%d of SE TYPE got AE request", + cptvf->vfid); + return -EINVAL; + } else if ((cptvf->vftype == AE_TYPES) && (req->ctrl.s.se_req)) { + dev_err(&pdev->dev, "CPTVF-%d of AE TYPE got SE request", + cptvf->vfid); + return -EINVAL; + } + + return process_request(cptvf, req); +} diff --git a/drivers/crypto/cavium/cpt/request_manager.h b/drivers/crypto/cavium/cpt/request_manager.h new file mode 100644 index 000000000..8d40e4ba3 --- /dev/null +++ b/drivers/crypto/cavium/cpt/request_manager.h @@ -0,0 +1,146 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2016 Cavium, Inc. + */ + +#ifndef __REQUEST_MANAGER_H +#define __REQUEST_MANAGER_H + +#include "cpt_common.h" + +#define TIME_IN_RESET_COUNT 5 +#define COMPLETION_CODE_SIZE 8 +#define COMPLETION_CODE_INIT 0 +#define PENDING_THOLD 100 +#define MAX_SG_IN_CNT 12 +#define MAX_SG_OUT_CNT 13 +#define SG_LIST_HDR_SIZE 8 +#define MAX_BUF_CNT 16 + +union ctrl_info { + u32 flags; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u32 reserved0:26; + u32 grp:3; /* Group bits */ + u32 dma_mode:2; /* DMA mode */ + u32 se_req:1;/* To SE core */ +#else + u32 se_req:1; /* To SE core */ + u32 dma_mode:2; /* DMA mode */ + u32 grp:3; /* Group bits */ + u32 reserved0:26; +#endif + } s; +}; + +union opcode_info { + u16 flags; + struct { + u8 major; + u8 minor; + } s; +}; + +struct cptvf_request { + union opcode_info opcode; + u16 param1; + u16 param2; + u16 dlen; +}; + +struct buf_ptr { + u8 *vptr; + dma_addr_t dma_addr; + u16 size; +}; + +struct cpt_request_info { + u8 incnt; /* Number of input buffers */ + u8 outcnt; /* Number of output buffers */ + u16 rlen; /* Output length */ + union ctrl_info ctrl; /* User control information */ + struct cptvf_request req; /* Request Information (Core specific) */ + + bool may_sleep; + + struct buf_ptr in[MAX_BUF_CNT]; + struct buf_ptr out[MAX_BUF_CNT]; + + void (*callback)(int, void *); /* Kernel ASYNC request callabck */ + void *callback_arg; /* Kernel ASYNC request callabck arg */ +}; + +struct sglist_component { + union { + u64 len; + struct { + __be16 len0; + __be16 len1; + __be16 len2; + __be16 len3; + } s; + } u; + __be64 ptr0; + __be64 ptr1; + __be64 ptr2; + __be64 ptr3; +}; + +struct cpt_info_buffer { + struct cpt_vf *cptvf; + unsigned long time_in; + u8 extra_time; + + struct cpt_request_info *req; + dma_addr_t dptr_baddr; + u32 dlen; + dma_addr_t rptr_baddr; + dma_addr_t comp_baddr; + u8 *in_buffer; + u8 *out_buffer; + u8 *gather_components; + u8 *scatter_components; + + struct pending_entry *pentry; + volatile u64 *completion_addr; + volatile u64 *alternate_caddr; +}; + +/* + * CPT_INST_S software command definitions + * Words EI (0-3) + */ +union vq_cmd_word0 { + u64 u64; + struct { + __be16 opcode; + __be16 param1; + __be16 param2; + __be16 dlen; + } s; +}; + +union vq_cmd_word3 { + u64 u64; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 grp:3; + u64 cptr:61; +#else + u64 cptr:61; + u64 grp:3; +#endif + } s; +}; + +struct cpt_vq_command { + union vq_cmd_word0 cmd; + u64 dptr; + u64 rptr; + union vq_cmd_word3 cptr; +}; + +void vq_post_process(struct cpt_vf *cptvf, u32 qno); +int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req); +#endif /* __REQUEST_MANAGER_H */ diff --git a/drivers/crypto/cavium/nitrox/Kconfig b/drivers/crypto/cavium/nitrox/Kconfig new file mode 100644 index 000000000..7dc008332 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/Kconfig @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Cavium NITROX Crypto Device configuration +# +config CRYPTO_DEV_NITROX + tristate + select CRYPTO_SKCIPHER + select CRYPTO_AES + select CRYPTO_LIB_DES + select FW_LOADER + +config CRYPTO_DEV_NITROX_CNN55XX + tristate "Support for Cavium CNN55XX driver" + depends on PCI_MSI && 64BIT + select CRYPTO_DEV_NITROX + help + Support for Cavium NITROX family CNN55XX driver + for accelerating crypto workloads. + + To compile this as a module, choose M here: the module + will be called n5pf. diff --git a/drivers/crypto/cavium/nitrox/Makefile b/drivers/crypto/cavium/nitrox/Makefile new file mode 100644 index 000000000..f83991aaf --- /dev/null +++ b/drivers/crypto/cavium/nitrox/Makefile @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_CRYPTO_DEV_NITROX_CNN55XX) += n5pf.o + +n5pf-objs := nitrox_main.o \ + nitrox_isr.o \ + nitrox_lib.o \ + nitrox_hal.o \ + nitrox_reqmgr.o \ + nitrox_algs.o \ + nitrox_mbx.o \ + nitrox_skcipher.o \ + nitrox_aead.o + +n5pf-$(CONFIG_PCI_IOV) += nitrox_sriov.o +n5pf-$(CONFIG_DEBUG_FS) += nitrox_debugfs.o diff --git a/drivers/crypto/cavium/nitrox/nitrox_aead.c b/drivers/crypto/cavium/nitrox/nitrox_aead.c new file mode 100644 index 000000000..b0e530341 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_aead.c @@ -0,0 +1,567 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "nitrox_dev.h" +#include "nitrox_common.h" +#include "nitrox_req.h" + +#define GCM_AES_SALT_SIZE 4 + +union gph_p3 { + struct { +#ifdef __BIG_ENDIAN_BITFIELD + u16 iv_offset : 8; + u16 auth_offset : 8; +#else + u16 auth_offset : 8; + u16 iv_offset : 8; +#endif + }; + u16 param; +}; + +static int nitrox_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + int aes_keylen; + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + struct flexi_crypto_context *fctx; + union fc_ctx_flags flags; + + aes_keylen = flexi_aes_keylen(keylen); + if (aes_keylen < 0) + return -EINVAL; + + /* fill crypto context */ + fctx = nctx->u.fctx; + flags.fu = be64_to_cpu(fctx->flags.f); + flags.w0.aes_keylen = aes_keylen; + fctx->flags.f = cpu_to_be64(flags.fu); + + /* copy enc key to context */ + memset(&fctx->crypto, 0, sizeof(fctx->crypto)); + memcpy(fctx->crypto.u.key, key, keylen); + + return 0; +} + +static int nitrox_aead_setauthsize(struct crypto_aead *aead, + unsigned int authsize) +{ + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + struct flexi_crypto_context *fctx = nctx->u.fctx; + union fc_ctx_flags flags; + + flags.fu = be64_to_cpu(fctx->flags.f); + flags.w0.mac_len = authsize; + fctx->flags.f = cpu_to_be64(flags.fu); + + aead->authsize = authsize; + + return 0; +} + +static int nitrox_aes_gcm_setauthsize(struct crypto_aead *aead, + unsigned int authsize) +{ + switch (authsize) { + case 4: + case 8: + case 12: + case 13: + case 14: + case 15: + case 16: + break; + default: + return -EINVAL; + } + + return nitrox_aead_setauthsize(aead, authsize); +} + +static int alloc_src_sglist(struct nitrox_kcrypt_request *nkreq, + struct scatterlist *src, char *iv, int ivsize, + int buflen) +{ + int nents = sg_nents_for_len(src, buflen); + int ret; + + if (nents < 0) + return nents; + + /* IV entry */ + nents += 1; + /* Allocate buffer to hold IV and input scatterlist array */ + ret = alloc_src_req_buf(nkreq, nents, ivsize); + if (ret) + return ret; + + nitrox_creq_copy_iv(nkreq->src, iv, ivsize); + nitrox_creq_set_src_sg(nkreq, nents, ivsize, src, buflen); + + return 0; +} + +static int alloc_dst_sglist(struct nitrox_kcrypt_request *nkreq, + struct scatterlist *dst, int ivsize, int buflen) +{ + int nents = sg_nents_for_len(dst, buflen); + int ret; + + if (nents < 0) + return nents; + + /* IV, ORH, COMPLETION entries */ + nents += 3; + /* Allocate buffer to hold ORH, COMPLETION and output scatterlist + * array + */ + ret = alloc_dst_req_buf(nkreq, nents); + if (ret) + return ret; + + nitrox_creq_set_orh(nkreq); + nitrox_creq_set_comp(nkreq); + nitrox_creq_set_dst_sg(nkreq, nents, ivsize, dst, buflen); + + return 0; +} + +static void free_src_sglist(struct nitrox_kcrypt_request *nkreq) +{ + kfree(nkreq->src); +} + +static void free_dst_sglist(struct nitrox_kcrypt_request *nkreq) +{ + kfree(nkreq->dst); +} + +static int nitrox_set_creq(struct nitrox_aead_rctx *rctx) +{ + struct se_crypto_request *creq = &rctx->nkreq.creq; + union gph_p3 param3; + int ret; + + creq->flags = rctx->flags; + creq->gfp = (rctx->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : + GFP_ATOMIC; + + creq->ctrl.value = 0; + creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC; + creq->ctrl.s.arg = rctx->ctrl_arg; + + creq->gph.param0 = cpu_to_be16(rctx->cryptlen); + creq->gph.param1 = cpu_to_be16(rctx->cryptlen + rctx->assoclen); + creq->gph.param2 = cpu_to_be16(rctx->ivsize + rctx->assoclen); + param3.iv_offset = 0; + param3.auth_offset = rctx->ivsize; + creq->gph.param3 = cpu_to_be16(param3.param); + + creq->ctx_handle = rctx->ctx_handle; + creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context); + + ret = alloc_src_sglist(&rctx->nkreq, rctx->src, rctx->iv, rctx->ivsize, + rctx->srclen); + if (ret) + return ret; + + ret = alloc_dst_sglist(&rctx->nkreq, rctx->dst, rctx->ivsize, + rctx->dstlen); + if (ret) { + free_src_sglist(&rctx->nkreq); + return ret; + } + + return 0; +} + +static void nitrox_aead_callback(void *arg, int err) +{ + struct aead_request *areq = arg; + struct nitrox_aead_rctx *rctx = aead_request_ctx(areq); + + free_src_sglist(&rctx->nkreq); + free_dst_sglist(&rctx->nkreq); + if (err) { + pr_err_ratelimited("request failed status 0x%0x\n", err); + err = -EINVAL; + } + + aead_request_complete(areq, err); +} + +static inline bool nitrox_aes_gcm_assoclen_supported(unsigned int assoclen) +{ + if (assoclen <= 512) + return true; + + return false; +} + +static int nitrox_aes_gcm_enc(struct aead_request *areq) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + struct nitrox_aead_rctx *rctx = aead_request_ctx(areq); + struct se_crypto_request *creq = &rctx->nkreq.creq; + struct flexi_crypto_context *fctx = nctx->u.fctx; + int ret; + + if (!nitrox_aes_gcm_assoclen_supported(areq->assoclen)) + return -EINVAL; + + memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE); + + rctx->cryptlen = areq->cryptlen; + rctx->assoclen = areq->assoclen; + rctx->srclen = areq->assoclen + areq->cryptlen; + rctx->dstlen = rctx->srclen + aead->authsize; + rctx->iv = &areq->iv[GCM_AES_SALT_SIZE]; + rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE; + rctx->flags = areq->base.flags; + rctx->ctx_handle = nctx->u.ctx_handle; + rctx->src = areq->src; + rctx->dst = areq->dst; + rctx->ctrl_arg = ENCRYPT; + ret = nitrox_set_creq(rctx); + if (ret) + return ret; + + /* send the crypto request */ + return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback, + areq); +} + +static int nitrox_aes_gcm_dec(struct aead_request *areq) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + struct nitrox_aead_rctx *rctx = aead_request_ctx(areq); + struct se_crypto_request *creq = &rctx->nkreq.creq; + struct flexi_crypto_context *fctx = nctx->u.fctx; + int ret; + + if (!nitrox_aes_gcm_assoclen_supported(areq->assoclen)) + return -EINVAL; + + memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE); + + rctx->cryptlen = areq->cryptlen - aead->authsize; + rctx->assoclen = areq->assoclen; + rctx->srclen = areq->cryptlen + areq->assoclen; + rctx->dstlen = rctx->srclen - aead->authsize; + rctx->iv = &areq->iv[GCM_AES_SALT_SIZE]; + rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE; + rctx->flags = areq->base.flags; + rctx->ctx_handle = nctx->u.ctx_handle; + rctx->src = areq->src; + rctx->dst = areq->dst; + rctx->ctrl_arg = DECRYPT; + ret = nitrox_set_creq(rctx); + if (ret) + return ret; + + /* send the crypto request */ + return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback, + areq); +} + +static int nitrox_aead_init(struct crypto_aead *aead) +{ + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + struct crypto_ctx_hdr *chdr; + + /* get the first device */ + nctx->ndev = nitrox_get_first_device(); + if (!nctx->ndev) + return -ENODEV; + + /* allocate nitrox crypto context */ + chdr = crypto_alloc_context(nctx->ndev); + if (!chdr) { + nitrox_put_device(nctx->ndev); + return -ENOMEM; + } + nctx->chdr = chdr; + nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr + + sizeof(struct ctx_hdr)); + nctx->u.fctx->flags.f = 0; + + return 0; +} + +static int nitrox_gcm_common_init(struct crypto_aead *aead) +{ + int ret; + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + union fc_ctx_flags *flags; + + ret = nitrox_aead_init(aead); + if (ret) + return ret; + + flags = &nctx->u.fctx->flags; + flags->w0.cipher_type = CIPHER_AES_GCM; + flags->w0.hash_type = AUTH_NULL; + flags->w0.iv_source = IV_FROM_DPTR; + /* ask microcode to calculate ipad/opad */ + flags->w0.auth_input_type = 1; + flags->f = cpu_to_be64(flags->fu); + + return 0; +} + +static int nitrox_aes_gcm_init(struct crypto_aead *aead) +{ + int ret; + + ret = nitrox_gcm_common_init(aead); + if (ret) + return ret; + + crypto_aead_set_reqsize(aead, + sizeof(struct aead_request) + + sizeof(struct nitrox_aead_rctx)); + + return 0; +} + +static void nitrox_aead_exit(struct crypto_aead *aead) +{ + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + + /* free the nitrox crypto context */ + if (nctx->u.ctx_handle) { + struct flexi_crypto_context *fctx = nctx->u.fctx; + + memzero_explicit(&fctx->crypto, sizeof(struct crypto_keys)); + memzero_explicit(&fctx->auth, sizeof(struct auth_keys)); + crypto_free_context((void *)nctx->chdr); + } + nitrox_put_device(nctx->ndev); + + nctx->u.ctx_handle = 0; + nctx->ndev = NULL; +} + +static int nitrox_rfc4106_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + struct flexi_crypto_context *fctx = nctx->u.fctx; + int ret; + + if (keylen < GCM_AES_SALT_SIZE) + return -EINVAL; + + keylen -= GCM_AES_SALT_SIZE; + ret = nitrox_aes_gcm_setkey(aead, key, keylen); + if (ret) + return ret; + + memcpy(fctx->crypto.iv, key + keylen, GCM_AES_SALT_SIZE); + return 0; +} + +static int nitrox_rfc4106_setauthsize(struct crypto_aead *aead, + unsigned int authsize) +{ + switch (authsize) { + case 8: + case 12: + case 16: + break; + default: + return -EINVAL; + } + + return nitrox_aead_setauthsize(aead, authsize); +} + +static int nitrox_rfc4106_set_aead_rctx_sglist(struct aead_request *areq) +{ + struct nitrox_rfc4106_rctx *rctx = aead_request_ctx_dma(areq); + struct nitrox_aead_rctx *aead_rctx = &rctx->base; + unsigned int assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE; + struct scatterlist *sg; + + if (areq->assoclen != 16 && areq->assoclen != 20) + return -EINVAL; + + scatterwalk_map_and_copy(rctx->assoc, areq->src, 0, assoclen, 0); + sg_init_table(rctx->src, 3); + sg_set_buf(rctx->src, rctx->assoc, assoclen); + sg = scatterwalk_ffwd(rctx->src + 1, areq->src, areq->assoclen); + if (sg != rctx->src + 1) + sg_chain(rctx->src, 2, sg); + + if (areq->src != areq->dst) { + sg_init_table(rctx->dst, 3); + sg_set_buf(rctx->dst, rctx->assoc, assoclen); + sg = scatterwalk_ffwd(rctx->dst + 1, areq->dst, areq->assoclen); + if (sg != rctx->dst + 1) + sg_chain(rctx->dst, 2, sg); + } + + aead_rctx->src = rctx->src; + aead_rctx->dst = (areq->src == areq->dst) ? rctx->src : rctx->dst; + + return 0; +} + +static void nitrox_rfc4106_callback(void *arg, int err) +{ + struct aead_request *areq = arg; + struct nitrox_rfc4106_rctx *rctx = aead_request_ctx_dma(areq); + struct nitrox_kcrypt_request *nkreq = &rctx->base.nkreq; + + free_src_sglist(nkreq); + free_dst_sglist(nkreq); + if (err) { + pr_err_ratelimited("request failed status 0x%0x\n", err); + err = -EINVAL; + } + + aead_request_complete(areq, err); +} + +static int nitrox_rfc4106_enc(struct aead_request *areq) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + struct nitrox_rfc4106_rctx *rctx = aead_request_ctx_dma(areq); + struct nitrox_aead_rctx *aead_rctx = &rctx->base; + struct se_crypto_request *creq = &aead_rctx->nkreq.creq; + int ret; + + aead_rctx->cryptlen = areq->cryptlen; + aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE; + aead_rctx->srclen = aead_rctx->assoclen + aead_rctx->cryptlen; + aead_rctx->dstlen = aead_rctx->srclen + aead->authsize; + aead_rctx->iv = areq->iv; + aead_rctx->ivsize = GCM_RFC4106_IV_SIZE; + aead_rctx->flags = areq->base.flags; + aead_rctx->ctx_handle = nctx->u.ctx_handle; + aead_rctx->ctrl_arg = ENCRYPT; + + ret = nitrox_rfc4106_set_aead_rctx_sglist(areq); + if (ret) + return ret; + + ret = nitrox_set_creq(aead_rctx); + if (ret) + return ret; + + /* send the crypto request */ + return nitrox_process_se_request(nctx->ndev, creq, + nitrox_rfc4106_callback, areq); +} + +static int nitrox_rfc4106_dec(struct aead_request *areq) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx_dma(aead); + struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq); + struct nitrox_aead_rctx *aead_rctx = &rctx->base; + struct se_crypto_request *creq = &aead_rctx->nkreq.creq; + int ret; + + aead_rctx->cryptlen = areq->cryptlen - aead->authsize; + aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE; + aead_rctx->srclen = + areq->cryptlen - GCM_RFC4106_IV_SIZE + areq->assoclen; + aead_rctx->dstlen = aead_rctx->srclen - aead->authsize; + aead_rctx->iv = areq->iv; + aead_rctx->ivsize = GCM_RFC4106_IV_SIZE; + aead_rctx->flags = areq->base.flags; + aead_rctx->ctx_handle = nctx->u.ctx_handle; + aead_rctx->ctrl_arg = DECRYPT; + + ret = nitrox_rfc4106_set_aead_rctx_sglist(areq); + if (ret) + return ret; + + ret = nitrox_set_creq(aead_rctx); + if (ret) + return ret; + + /* send the crypto request */ + return nitrox_process_se_request(nctx->ndev, creq, + nitrox_rfc4106_callback, areq); +} + +static int nitrox_rfc4106_init(struct crypto_aead *aead) +{ + int ret; + + ret = nitrox_gcm_common_init(aead); + if (ret) + return ret; + + crypto_aead_set_reqsize_dma(aead, sizeof(struct aead_request) + + sizeof(struct nitrox_rfc4106_rctx)); + + return 0; +} + +static struct aead_alg nitrox_aeads[] = { { + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "n5_aes_gcm", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .setkey = nitrox_aes_gcm_setkey, + .setauthsize = nitrox_aes_gcm_setauthsize, + .encrypt = nitrox_aes_gcm_enc, + .decrypt = nitrox_aes_gcm_dec, + .init = nitrox_aes_gcm_init, + .exit = nitrox_aead_exit, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, +}, { + .base = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "n5_rfc4106", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .setkey = nitrox_rfc4106_setkey, + .setauthsize = nitrox_rfc4106_setauthsize, + .encrypt = nitrox_rfc4106_enc, + .decrypt = nitrox_rfc4106_dec, + .init = nitrox_rfc4106_init, + .exit = nitrox_aead_exit, + .ivsize = GCM_RFC4106_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, +} }; + +int nitrox_register_aeads(void) +{ + return crypto_register_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads)); +} + +void nitrox_unregister_aeads(void) +{ + crypto_unregister_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads)); +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_algs.c b/drivers/crypto/cavium/nitrox/nitrox_algs.c new file mode 100644 index 000000000..d646ae5f2 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_algs.c @@ -0,0 +1,24 @@ +#include "nitrox_common.h" + +int nitrox_crypto_register(void) +{ + int err; + + err = nitrox_register_skciphers(); + if (err) + return err; + + err = nitrox_register_aeads(); + if (err) { + nitrox_unregister_skciphers(); + return err; + } + + return 0; +} + +void nitrox_crypto_unregister(void) +{ + nitrox_unregister_aeads(); + nitrox_unregister_skciphers(); +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_common.h b/drivers/crypto/cavium/nitrox/nitrox_common.h new file mode 100644 index 000000000..e4be69d7e --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_common.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NITROX_COMMON_H +#define __NITROX_COMMON_H + +#include "nitrox_dev.h" +#include "nitrox_req.h" + +int nitrox_crypto_register(void); +void nitrox_crypto_unregister(void); +int nitrox_register_aeads(void); +void nitrox_unregister_aeads(void); +int nitrox_register_skciphers(void); +void nitrox_unregister_skciphers(void); +void *crypto_alloc_context(struct nitrox_device *ndev); +void crypto_free_context(void *ctx); +struct nitrox_device *nitrox_get_first_device(void); +void nitrox_put_device(struct nitrox_device *ndev); + +int nitrox_common_sw_init(struct nitrox_device *ndev); +void nitrox_common_sw_cleanup(struct nitrox_device *ndev); + +void pkt_slc_resp_tasklet(unsigned long data); +int nitrox_process_se_request(struct nitrox_device *ndev, + struct se_crypto_request *req, + completion_t cb, + void *cb_arg); +void backlog_qflush_work(struct work_struct *work); + + +#endif /* __NITROX_COMMON_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_csr.h b/drivers/crypto/cavium/nitrox/nitrox_csr.h new file mode 100644 index 000000000..1c8715ae0 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_csr.h @@ -0,0 +1,1439 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NITROX_CSR_H +#define __NITROX_CSR_H + +#include +#include + +/* EMU clusters */ +#define NR_CLUSTERS 4 +/* Maximum cores per cluster, + * varies based on partname + */ +#define AE_CORES_PER_CLUSTER 20 +#define SE_CORES_PER_CLUSTER 16 + +#define AE_MAX_CORES (AE_CORES_PER_CLUSTER * NR_CLUSTERS) +#define SE_MAX_CORES (SE_CORES_PER_CLUSTER * NR_CLUSTERS) +#define ZIP_MAX_CORES 5 + +/* BIST registers */ +#define EMU_BIST_STATUSX(_i) (0x1402700 + ((_i) * 0x40000)) +#define UCD_BIST_STATUS 0x12C0070 +#define NPS_CORE_BIST_REG 0x10000E8 +#define NPS_CORE_NPC_BIST_REG 0x1000128 +#define NPS_PKT_SLC_BIST_REG 0x1040088 +#define NPS_PKT_IN_BIST_REG 0x1040100 +#define POM_BIST_REG 0x11C0100 +#define BMI_BIST_REG 0x1140080 +#define EFL_CORE_BIST_REGX(_i) (0x1240100 + ((_i) * 0x400)) +#define EFL_TOP_BIST_STAT 0x1241090 +#define BMO_BIST_REG 0x1180080 +#define LBC_BIST_STATUS 0x1200020 +#define PEM_BIST_STATUSX(_i) (0x1080468 | ((_i) << 18)) + +/* EMU registers */ +#define EMU_SE_ENABLEX(_i) (0x1400000 + ((_i) * 0x40000)) +#define EMU_AE_ENABLEX(_i) (0x1400008 + ((_i) * 0x40000)) +#define EMU_WD_INT_ENA_W1SX(_i) (0x1402318 + ((_i) * 0x40000)) +#define EMU_GE_INT_ENA_W1SX(_i) (0x1402518 + ((_i) * 0x40000)) +#define EMU_FUSE_MAPX(_i) (0x1402708 + ((_i) * 0x40000)) + +/* UCD registers */ +#define UCD_SE_EID_UCODE_BLOCK_NUMX(_i) (0x12C0000 + ((_i) * 0x1000)) +#define UCD_AE_EID_UCODE_BLOCK_NUMX(_i) (0x12C0008 + ((_i) * 0x800)) +#define UCD_UCODE_LOAD_BLOCK_NUM 0x12C0010 +#define UCD_UCODE_LOAD_IDX_DATAX(_i) (0x12C0018 + ((_i) * 0x20)) +#define UCD_SE_CNTX(_i) (0x12C0040 + ((_i) * 0x1000)) +#define UCD_AE_CNTX(_i) (0x12C0048 + ((_i) * 0x800)) + +/* AQM registers */ +#define AQM_CTL 0x1300000 +#define AQM_INT 0x1300008 +#define AQM_DBELL_OVF_LO 0x1300010 +#define AQM_DBELL_OVF_HI 0x1300018 +#define AQM_DBELL_OVF_LO_W1S 0x1300020 +#define AQM_DBELL_OVF_LO_ENA_W1C 0x1300028 +#define AQM_DBELL_OVF_LO_ENA_W1S 0x1300030 +#define AQM_DBELL_OVF_HI_W1S 0x1300038 +#define AQM_DBELL_OVF_HI_ENA_W1C 0x1300040 +#define AQM_DBELL_OVF_HI_ENA_W1S 0x1300048 +#define AQM_DMA_RD_ERR_LO 0x1300050 +#define AQM_DMA_RD_ERR_HI 0x1300058 +#define AQM_DMA_RD_ERR_LO_W1S 0x1300060 +#define AQM_DMA_RD_ERR_LO_ENA_W1C 0x1300068 +#define AQM_DMA_RD_ERR_LO_ENA_W1S 0x1300070 +#define AQM_DMA_RD_ERR_HI_W1S 0x1300078 +#define AQM_DMA_RD_ERR_HI_ENA_W1C 0x1300080 +#define AQM_DMA_RD_ERR_HI_ENA_W1S 0x1300088 +#define AQM_EXEC_NA_LO 0x1300090 +#define AQM_EXEC_NA_HI 0x1300098 +#define AQM_EXEC_NA_LO_W1S 0x13000A0 +#define AQM_EXEC_NA_LO_ENA_W1C 0x13000A8 +#define AQM_EXEC_NA_LO_ENA_W1S 0x13000B0 +#define AQM_EXEC_NA_HI_W1S 0x13000B8 +#define AQM_EXEC_NA_HI_ENA_W1C 0x13000C0 +#define AQM_EXEC_NA_HI_ENA_W1S 0x13000C8 +#define AQM_EXEC_ERR_LO 0x13000D0 +#define AQM_EXEC_ERR_HI 0x13000D8 +#define AQM_EXEC_ERR_LO_W1S 0x13000E0 +#define AQM_EXEC_ERR_LO_ENA_W1C 0x13000E8 +#define AQM_EXEC_ERR_LO_ENA_W1S 0x13000F0 +#define AQM_EXEC_ERR_HI_W1S 0x13000F8 +#define AQM_EXEC_ERR_HI_ENA_W1C 0x1300100 +#define AQM_EXEC_ERR_HI_ENA_W1S 0x1300108 +#define AQM_ECC_INT 0x1300110 +#define AQM_ECC_INT_W1S 0x1300118 +#define AQM_ECC_INT_ENA_W1C 0x1300120 +#define AQM_ECC_INT_ENA_W1S 0x1300128 +#define AQM_ECC_CTL 0x1300130 +#define AQM_BIST_STATUS 0x1300138 +#define AQM_CMD_INF_THRX(x) (0x1300400 + ((x) * 0x8)) +#define AQM_CMD_INFX(x) (0x1300800 + ((x) * 0x8)) +#define AQM_GRP_EXECMSK_LOX(x) (0x1300C00 + ((x) * 0x10)) +#define AQM_GRP_EXECMSK_HIX(x) (0x1300C08 + ((x) * 0x10)) +#define AQM_ACTIVITY_STAT_LO 0x1300C80 +#define AQM_ACTIVITY_STAT_HI 0x1300C88 +#define AQM_Q_CMD_PROCX(x) (0x1301000 + ((x) * 0x8)) +#define AQM_PERF_CTL_LO 0x1301400 +#define AQM_PERF_CTL_HI 0x1301408 +#define AQM_PERF_CNT 0x1301410 + +#define AQMQ_DRBLX(x) (0x20000 + ((x) * 0x40000)) +#define AQMQ_QSZX(x) (0x20008 + ((x) * 0x40000)) +#define AQMQ_BADRX(x) (0x20010 + ((x) * 0x40000)) +#define AQMQ_NXT_CMDX(x) (0x20018 + ((x) * 0x40000)) +#define AQMQ_CMD_CNTX(x) (0x20020 + ((x) * 0x40000)) +#define AQMQ_CMP_THRX(x) (0x20028 + ((x) * 0x40000)) +#define AQMQ_CMP_CNTX(x) (0x20030 + ((x) * 0x40000)) +#define AQMQ_TIM_LDX(x) (0x20038 + ((x) * 0x40000)) +#define AQMQ_TIMERX(x) (0x20040 + ((x) * 0x40000)) +#define AQMQ_ENX(x) (0x20048 + ((x) * 0x40000)) +#define AQMQ_ACTIVITY_STATX(x) (0x20050 + ((x) * 0x40000)) +#define AQM_VF_CMP_STATX(x) (0x28000 + ((x) * 0x40000)) + +/* NPS core registers */ +#define NPS_CORE_GBL_VFCFG 0x1000000 +#define NPS_CORE_CONTROL 0x1000008 +#define NPS_CORE_INT_ACTIVE 0x1000080 +#define NPS_CORE_INT 0x10000A0 +#define NPS_CORE_INT_ENA_W1S 0x10000B8 +#define NPS_STATS_PKT_DMA_RD_CNT 0x1000180 +#define NPS_STATS_PKT_DMA_WR_CNT 0x1000190 + +/* NPS packet registers */ +#define NPS_PKT_INT 0x1040018 +#define NPS_PKT_MBOX_INT_LO 0x1040020 +#define NPS_PKT_MBOX_INT_LO_ENA_W1C 0x1040030 +#define NPS_PKT_MBOX_INT_LO_ENA_W1S 0x1040038 +#define NPS_PKT_MBOX_INT_HI 0x1040040 +#define NPS_PKT_MBOX_INT_HI_ENA_W1C 0x1040050 +#define NPS_PKT_MBOX_INT_HI_ENA_W1S 0x1040058 +#define NPS_PKT_IN_RERR_HI 0x1040108 +#define NPS_PKT_IN_RERR_HI_ENA_W1S 0x1040120 +#define NPS_PKT_IN_RERR_LO 0x1040128 +#define NPS_PKT_IN_RERR_LO_ENA_W1S 0x1040140 +#define NPS_PKT_IN_ERR_TYPE 0x1040148 +#define NPS_PKT_IN_ERR_TYPE_ENA_W1S 0x1040160 +#define NPS_PKT_IN_INSTR_CTLX(_i) (0x10060 + ((_i) * 0x40000)) +#define NPS_PKT_IN_INSTR_BADDRX(_i) (0x10068 + ((_i) * 0x40000)) +#define NPS_PKT_IN_INSTR_RSIZEX(_i) (0x10070 + ((_i) * 0x40000)) +#define NPS_PKT_IN_DONE_CNTSX(_i) (0x10080 + ((_i) * 0x40000)) +#define NPS_PKT_IN_INSTR_BAOFF_DBELLX(_i) (0x10078 + ((_i) * 0x40000)) +#define NPS_PKT_IN_INT_LEVELSX(_i) (0x10088 + ((_i) * 0x40000)) + +#define NPS_PKT_SLC_RERR_HI 0x1040208 +#define NPS_PKT_SLC_RERR_HI_ENA_W1S 0x1040220 +#define NPS_PKT_SLC_RERR_LO 0x1040228 +#define NPS_PKT_SLC_RERR_LO_ENA_W1S 0x1040240 +#define NPS_PKT_SLC_ERR_TYPE 0x1040248 +#define NPS_PKT_SLC_ERR_TYPE_ENA_W1S 0x1040260 +/* Mailbox PF->VF PF Accessible Data registers */ +#define NPS_PKT_MBOX_PF_VF_PFDATAX(_i) (0x1040800 + ((_i) * 0x8)) +#define NPS_PKT_MBOX_VF_PF_PFDATAX(_i) (0x1040C00 + ((_i) * 0x8)) + +#define NPS_PKT_SLC_CTLX(_i) (0x10000 + ((_i) * 0x40000)) +#define NPS_PKT_SLC_CNTSX(_i) (0x10008 + ((_i) * 0x40000)) +#define NPS_PKT_SLC_INT_LEVELSX(_i) (0x10010 + ((_i) * 0x40000)) + +/* POM registers */ +#define POM_INT_ENA_W1S 0x11C0018 +#define POM_GRP_EXECMASKX(_i) (0x11C1100 | ((_i) * 8)) +#define POM_INT 0x11C0000 +#define POM_PERF_CTL 0x11CC400 + +/* BMI registers */ +#define BMI_INT 0x1140000 +#define BMI_CTL 0x1140020 +#define BMI_INT_ENA_W1S 0x1140018 +#define BMI_NPS_PKT_CNT 0x1140070 + +/* EFL registers */ +#define EFL_CORE_INT_ENA_W1SX(_i) (0x1240018 + ((_i) * 0x400)) +#define EFL_CORE_VF_ERR_INT0X(_i) (0x1240050 + ((_i) * 0x400)) +#define EFL_CORE_VF_ERR_INT0_ENA_W1SX(_i) (0x1240068 + ((_i) * 0x400)) +#define EFL_CORE_VF_ERR_INT1X(_i) (0x1240070 + ((_i) * 0x400)) +#define EFL_CORE_VF_ERR_INT1_ENA_W1SX(_i) (0x1240088 + ((_i) * 0x400)) +#define EFL_CORE_SE_ERR_INTX(_i) (0x12400A0 + ((_i) * 0x400)) +#define EFL_RNM_CTL_STATUS 0x1241800 +#define EFL_CORE_INTX(_i) (0x1240000 + ((_i) * 0x400)) + +/* BMO registers */ +#define BMO_CTL2 0x1180028 +#define BMO_NPS_SLC_PKT_CNT 0x1180078 + +/* LBC registers */ +#define LBC_INT 0x1200000 +#define LBC_INVAL_CTL 0x1201010 +#define LBC_PLM_VF1_64_INT 0x1202008 +#define LBC_INVAL_STATUS 0x1202010 +#define LBC_INT_ENA_W1S 0x1203000 +#define LBC_PLM_VF1_64_INT_ENA_W1S 0x1205008 +#define LBC_PLM_VF65_128_INT 0x1206008 +#define LBC_ELM_VF1_64_INT 0x1208000 +#define LBC_PLM_VF65_128_INT_ENA_W1S 0x1209008 +#define LBC_ELM_VF1_64_INT_ENA_W1S 0x120B000 +#define LBC_ELM_VF65_128_INT 0x120C000 +#define LBC_ELM_VF65_128_INT_ENA_W1S 0x120F000 + +#define RST_BOOT 0x10C1600 +#define FUS_DAT1 0x10C1408 + +/* PEM registers */ +#define PEM0_INT 0x1080428 + +/** + * struct ucd_core_eid_ucode_block_num - Core Eid to Ucode Blk Mapping Registers + * @ucode_len: Ucode length identifier 32KB or 64KB + * @ucode_blk: Ucode Block Number + */ +union ucd_core_eid_ucode_block_num { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_4_63 : 60; + u64 ucode_len : 1; + u64 ucode_blk : 3; +#else + u64 ucode_blk : 3; + u64 ucode_len : 1; + u64 raz_4_63 : 60; +#endif + }; +}; + +/** + * struct aqm_grp_execmsk_lo - Available AE engines for the group + * @exec_0_to_39: AE engines 0 to 39 status + */ +union aqm_grp_execmsk_lo { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_40_63 : 24; + u64 exec_0_to_39 : 40; +#else + u64 exec_0_to_39 : 40; + u64 raz_40_63 : 24; +#endif + }; +}; + +/** + * struct aqm_grp_execmsk_hi - Available AE engines for the group + * @exec_40_to_79: AE engines 40 to 79 status + */ +union aqm_grp_execmsk_hi { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_40_63 : 24; + u64 exec_40_to_79 : 40; +#else + u64 exec_40_to_79 : 40; + u64 raz_40_63 : 24; +#endif + }; +}; + +/** + * struct aqmq_drbl - AQM Queue Doorbell Counter Registers + * @dbell_count: Doorbell Counter + */ +union aqmq_drbl { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_32_63 : 32; + u64 dbell_count : 32; +#else + u64 dbell_count : 32; + u64 raz_32_63 : 32; +#endif + }; +}; + +/** + * struct aqmq_qsz - AQM Queue Host Queue Size Registers + * @host_queue_size: Size, in numbers of 'aqmq_command_s' command + * of the Host Ring. + */ +union aqmq_qsz { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_32_63 : 32; + u64 host_queue_size : 32; +#else + u64 host_queue_size : 32; + u64 raz_32_63 : 32; +#endif + }; +}; + +/** + * struct aqmq_cmp_thr - AQM Queue Commands Completed Threshold Registers + * @commands_completed_threshold: Count of 'aqmq_command_s' commands executed + * by AE engines for which completion interrupt is asserted. + */ +union aqmq_cmp_thr { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_32_63 : 32; + u64 commands_completed_threshold : 32; +#else + u64 commands_completed_threshold : 32; + u64 raz_32_63 : 32; +#endif + }; +}; + +/** + * struct aqmq_cmp_cnt - AQM Queue Commands Completed Count Registers + * @resend: Bit to request completion interrupt Resend. + * @completion_status: Command completion status of the ring. + * @commands_completed_count: Count of 'aqmq_command_s' commands executed by + * AE engines. + */ +union aqmq_cmp_cnt { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_34_63 : 30; + u64 resend : 1; + u64 completion_status : 1; + u64 commands_completed_count : 32; +#else + u64 commands_completed_count : 32; + u64 completion_status : 1; + u64 resend : 1; + u64 raz_34_63 : 30; +#endif + }; +}; + +/** + * struct aqmq_en - AQM Queue Enable Registers + * @queue_status: 1 = AQMQ is enabled, 0 = AQMQ is disabled + */ +union aqmq_en { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_1_63 : 63; + u64 queue_enable : 1; +#else + u64 queue_enable : 1; + u64 raz_1_63 : 63; +#endif + }; +}; + +/** + * struct aqmq_activity_stat - AQM Queue Activity Status Registers + * @queue_active: 1 = AQMQ is active, 0 = AQMQ is quiescent + */ +union aqmq_activity_stat { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_1_63 : 63; + u64 queue_active : 1; +#else + u64 queue_active : 1; + u64 raz_1_63 : 63; +#endif + }; +}; + +/** + * struct emu_fuse_map - EMU Fuse Map Registers + * @ae_fuse: Fuse settings for AE 19..0 + * @se_fuse: Fuse settings for SE 15..0 + * + * A set bit indicates the unit is fuse disabled. + */ +union emu_fuse_map { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 valid : 1; + u64 raz_52_62 : 11; + u64 ae_fuse : 20; + u64 raz_16_31 : 16; + u64 se_fuse : 16; +#else + u64 se_fuse : 16; + u64 raz_16_31 : 16; + u64 ae_fuse : 20; + u64 raz_52_62 : 11; + u64 valid : 1; +#endif + } s; +}; + +/** + * struct emu_se_enable - Symmetric Engine Enable Registers + * @enable: Individual enables for each of the clusters + * 16 symmetric engines. + */ +union emu_se_enable { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 48; + u64 enable : 16; +#else + u64 enable : 16; + u64 raz : 48; +#endif + } s; +}; + +/** + * struct emu_ae_enable - EMU Asymmetric engines. + * @enable: Individual enables for each of the cluster's + * 20 Asymmetric Engines. + */ +union emu_ae_enable { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 44; + u64 enable : 20; +#else + u64 enable : 20; + u64 raz : 44; +#endif + } s; +}; + +/** + * struct emu_wd_int_ena_w1s - EMU Interrupt Enable Registers + * @ae_wd: Reads or sets enable for EMU(0..3)_WD_INT[AE_WD] + * @se_wd: Reads or sets enable for EMU(0..3)_WD_INT[SE_WD] + */ +union emu_wd_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz2 : 12; + u64 ae_wd : 20; + u64 raz1 : 16; + u64 se_wd : 16; +#else + u64 se_wd : 16; + u64 raz1 : 16; + u64 ae_wd : 20; + u64 raz2 : 12; +#endif + } s; +}; + +/** + * struct emu_ge_int_ena_w1s - EMU Interrupt Enable set registers + * @ae_ge: Reads or sets enable for EMU(0..3)_GE_INT[AE_GE] + * @se_ge: Reads or sets enable for EMU(0..3)_GE_INT[SE_GE] + */ +union emu_ge_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_52_63 : 12; + u64 ae_ge : 20; + u64 raz_16_31: 16; + u64 se_ge : 16; +#else + u64 se_ge : 16; + u64 raz_16_31: 16; + u64 ae_ge : 20; + u64 raz_52_63 : 12; +#endif + } s; +}; + +/** + * struct nps_pkt_slc_ctl - Solicited Packet Out Control Registers + * @rh: Indicates whether to remove or include the response header + * 1 = Include, 0 = Remove + * @z: If set, 8 trailing 0x00 bytes will be added to the end of the + * outgoing packet. + * @enb: Enable for this port. + */ +union nps_pkt_slc_ctl { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 raz : 61; + u64 rh : 1; + u64 z : 1; + u64 enb : 1; +#else + u64 enb : 1; + u64 z : 1; + u64 rh : 1; + u64 raz : 61; +#endif + } s; +}; + +/** + * struct nps_pkt_slc_cnts - Solicited Packet Out Count Registers + * @slc_int: Returns a 1 when: + * NPS_PKT_SLC(i)_CNTS[CNT] > NPS_PKT_SLC(i)_INT_LEVELS[CNT], or + * NPS_PKT_SLC(i)_CNTS[TIMER] > NPS_PKT_SLC(i)_INT_LEVELS[TIMET]. + * To clear the bit, the CNTS register must be written to clear. + * @in_int: Returns a 1 when: + * NPS_PKT_IN(i)_DONE_CNTS[CNT] > NPS_PKT_IN(i)_INT_LEVELS[CNT]. + * To clear the bit, the DONE_CNTS register must be written to clear. + * @mbox_int: Returns a 1 when: + * NPS_PKT_MBOX_PF_VF(i)_INT[INTR] is set. To clear the bit, + * write NPS_PKT_MBOX_PF_VF(i)_INT[INTR] with 1. + * @timer: Timer, incremented every 2048 coprocessor clock cycles + * when [CNT] is not zero. The hardware clears both [TIMER] and + * [INT] when [CNT] goes to 0. + * @cnt: Packet counter. Hardware adds to [CNT] as it sends packets out. + * On a write to this CSR, hardware subtracts the amount written to the + * [CNT] field from [CNT]. + */ +union nps_pkt_slc_cnts { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 slc_int : 1; + u64 uns_int : 1; + u64 in_int : 1; + u64 mbox_int : 1; + u64 resend : 1; + u64 raz : 5; + u64 timer : 22; + u64 cnt : 32; +#else + u64 cnt : 32; + u64 timer : 22; + u64 raz : 5; + u64 resend : 1; + u64 mbox_int : 1; + u64 in_int : 1; + u64 uns_int : 1; + u64 slc_int : 1; +#endif + } s; +}; + +/** + * struct nps_pkt_slc_int_levels - Solicited Packet Out Interrupt Levels + * Registers. + * @bmode: Determines whether NPS_PKT_SLC_CNTS[CNT] is a byte or + * packet counter. + * @timet: Output port counter time interrupt threshold. + * @cnt: Output port counter interrupt threshold. + */ +union nps_pkt_slc_int_levels { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 bmode : 1; + u64 raz : 9; + u64 timet : 22; + u64 cnt : 32; +#else + u64 cnt : 32; + u64 timet : 22; + u64 raz : 9; + u64 bmode : 1; +#endif + } s; +}; + +/** + * struct nps_pkt_inst - NPS Packet Interrupt Register + * @in_err: Set when any NPS_PKT_IN_RERR_HI/LO bit and + * corresponding NPS_PKT_IN_RERR_*_ENA_* bit are bot set. + * @uns_err: Set when any NSP_PKT_UNS_RERR_HI/LO bit and + * corresponding NPS_PKT_UNS_RERR_*_ENA_* bit are both set. + * @slc_er: Set when any NSP_PKT_SLC_RERR_HI/LO bit and + * corresponding NPS_PKT_SLC_RERR_*_ENA_* bit are both set. + */ +union nps_pkt_int { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 raz : 54; + u64 uns_wto : 1; + u64 in_err : 1; + u64 uns_err : 1; + u64 slc_err : 1; + u64 in_dbe : 1; + u64 in_sbe : 1; + u64 uns_dbe : 1; + u64 uns_sbe : 1; + u64 slc_dbe : 1; + u64 slc_sbe : 1; +#else + u64 slc_sbe : 1; + u64 slc_dbe : 1; + u64 uns_sbe : 1; + u64 uns_dbe : 1; + u64 in_sbe : 1; + u64 in_dbe : 1; + u64 slc_err : 1; + u64 uns_err : 1; + u64 in_err : 1; + u64 uns_wto : 1; + u64 raz : 54; +#endif + } s; +}; + +/** + * struct nps_pkt_in_done_cnts - Input instruction ring counts registers + * @slc_cnt: Returns a 1 when: + * NPS_PKT_SLC(i)_CNTS[CNT] > NPS_PKT_SLC(i)_INT_LEVELS[CNT], or + * NPS_PKT_SLC(i)_CNTS[TIMER] > NPS_PKT_SCL(i)_INT_LEVELS[TIMET] + * To clear the bit, the CNTS register must be + * written to clear the underlying condition + * @uns_int: Return a 1 when: + * NPS_PKT_UNS(i)_CNTS[CNT] > NPS_PKT_UNS(i)_INT_LEVELS[CNT], or + * NPS_PKT_UNS(i)_CNTS[TIMER] > NPS_PKT_UNS(i)_INT_LEVELS[TIMET] + * To clear the bit, the CNTS register must be + * written to clear the underlying condition + * @in_int: Returns a 1 when: + * NPS_PKT_IN(i)_DONE_CNTS[CNT] > NPS_PKT_IN(i)_INT_LEVELS[CNT] + * To clear the bit, the DONE_CNTS register + * must be written to clear the underlying condition + * @mbox_int: Returns a 1 when: + * NPS_PKT_MBOX_PF_VF(i)_INT[INTR] is set. + * To clear the bit, write NPS_PKT_MBOX_PF_VF(i)_INT[INTR] + * with 1. + * @resend: A write of 1 will resend an MSI-X interrupt message if any + * of the following conditions are true for this ring "i". + * NPS_PKT_SLC(i)_CNTS[CNT] > NPS_PKT_SLC(i)_INT_LEVELS[CNT] + * NPS_PKT_SLC(i)_CNTS[TIMER] > NPS_PKT_SLC(i)_INT_LEVELS[TIMET] + * NPS_PKT_UNS(i)_CNTS[CNT] > NPS_PKT_UNS(i)_INT_LEVELS[CNT] + * NPS_PKT_UNS(i)_CNTS[TIMER] > NPS_PKT_UNS(i)_INT_LEVELS[TIMET] + * NPS_PKT_IN(i)_DONE_CNTS[CNT] > NPS_PKT_IN(i)_INT_LEVELS[CNT] + * NPS_PKT_MBOX_PF_VF(i)_INT[INTR] is set + * @cnt: Packet counter. Hardware adds to [CNT] as it reads + * packets. On a write to this CSR, hardware substracts the + * amount written to the [CNT] field from [CNT], which will + * clear PKT_IN(i)_INT_STATUS[INTR] if [CNT] becomes <= + * NPS_PKT_IN(i)_INT_LEVELS[CNT]. This register should be + * cleared before enabling a ring by reading the current + * value and writing it back. + */ +union nps_pkt_in_done_cnts { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 slc_int : 1; + u64 uns_int : 1; + u64 in_int : 1; + u64 mbox_int : 1; + u64 resend : 1; + u64 raz : 27; + u64 cnt : 32; +#else + u64 cnt : 32; + u64 raz : 27; + u64 resend : 1; + u64 mbox_int : 1; + u64 in_int : 1; + u64 uns_int : 1; + u64 slc_int : 1; +#endif + } s; +}; + +/** + * struct nps_pkt_in_instr_ctl - Input Instruction Ring Control Registers. + * @is64b: If 1, the ring uses 64-byte instructions. If 0, the + * ring uses 32-byte instructions. + * @enb: Enable for the input ring. + */ +union nps_pkt_in_instr_ctl { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 62; + u64 is64b : 1; + u64 enb : 1; +#else + u64 enb : 1; + u64 is64b : 1; + u64 raz : 62; +#endif + } s; +}; + +/** + * struct nps_pkt_in_instr_rsize - Input instruction ring size registers + * @rsize: Ring size (number of instructions) + */ +union nps_pkt_in_instr_rsize { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 32; + u64 rsize : 32; +#else + u64 rsize : 32; + u64 raz : 32; +#endif + } s; +}; + +/** + * struct nps_pkt_in_instr_baoff_dbell - Input instruction ring + * base address offset and doorbell registers + * @aoff: Address offset. The offset from the NPS_PKT_IN_INSTR_BADDR + * where the next pointer is read. + * @dbell: Pointer list doorbell count. Write operations to this field + * increments the present value here. Read operations return the + * present value. + */ +union nps_pkt_in_instr_baoff_dbell { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 aoff : 32; + u64 dbell : 32; +#else + u64 dbell : 32; + u64 aoff : 32; +#endif + } s; +}; + +/** + * struct nps_core_int_ena_w1s - NPS core interrupt enable set register + * @host_nps_wr_err: Reads or sets enable for + * NPS_CORE_INT[HOST_NPS_WR_ERR]. + * @npco_dma_malform: Reads or sets enable for + * NPS_CORE_INT[NPCO_DMA_MALFORM]. + * @exec_wr_timeout: Reads or sets enable for + * NPS_CORE_INT[EXEC_WR_TIMEOUT]. + * @host_wr_timeout: Reads or sets enable for + * NPS_CORE_INT[HOST_WR_TIMEOUT]. + * @host_wr_err: Reads or sets enable for + * NPS_CORE_INT[HOST_WR_ERR] + */ +union nps_core_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz4 : 55; + u64 host_nps_wr_err : 1; + u64 npco_dma_malform : 1; + u64 exec_wr_timeout : 1; + u64 host_wr_timeout : 1; + u64 host_wr_err : 1; + u64 raz3 : 1; + u64 raz2 : 1; + u64 raz1 : 1; + u64 raz0 : 1; +#else + u64 raz0 : 1; + u64 raz1 : 1; + u64 raz2 : 1; + u64 raz3 : 1; + u64 host_wr_err : 1; + u64 host_wr_timeout : 1; + u64 exec_wr_timeout : 1; + u64 npco_dma_malform : 1; + u64 host_nps_wr_err : 1; + u64 raz4 : 55; +#endif + } s; +}; + +/** + * struct nps_core_gbl_vfcfg - Global VF Configuration Register. + * @ilk_disable: When set, this bit indicates that the ILK interface has + * been disabled. + * @obaf: BMO allocation control + * 0 = allocate per queue + * 1 = allocate per VF + * @ibaf: BMI allocation control + * 0 = allocate per queue + * 1 = allocate per VF + * @zaf: ZIP allocation control + * 0 = allocate per queue + * 1 = allocate per VF + * @aeaf: AE allocation control + * 0 = allocate per queue + * 1 = allocate per VF + * @seaf: SE allocation control + * 0 = allocation per queue + * 1 = allocate per VF + * @cfg: VF/PF mode. + */ +union nps_core_gbl_vfcfg { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz :55; + u64 ilk_disable :1; + u64 obaf :1; + u64 ibaf :1; + u64 zaf :1; + u64 aeaf :1; + u64 seaf :1; + u64 cfg :3; +#else + u64 cfg :3; + u64 seaf :1; + u64 aeaf :1; + u64 zaf :1; + u64 ibaf :1; + u64 obaf :1; + u64 ilk_disable :1; + u64 raz :55; +#endif + } s; +}; + +/** + * struct nps_core_int_active - NPS Core Interrupt Active Register + * @resend: Resend MSI-X interrupt if needs to handle interrupts + * Sofware can set this bit and then exit the ISR. + * @ocla: Set when any OCLA(0)_INT and corresponding OCLA(0_INT_ENA_W1C + * bit are set + * @mbox: Set when any NPS_PKT_MBOX_INT_LO/HI and corresponding + * NPS_PKT_MBOX_INT_LO_ENA_W1C/HI_ENA_W1C bits are set + * @emu: bit i is set in [EMU] when any EMU(i)_INT bit is set + * @bmo: Set when any BMO_INT bit is set + * @bmi: Set when any BMI_INT bit is set or when any non-RO + * BMI_INT and corresponding BMI_INT_ENA_W1C bits are both set + * @aqm: Set when any AQM_INT bit is set + * @zqm: Set when any ZQM_INT bit is set + * @efl: Set when any EFL_INT RO bit is set or when any non-RO EFL_INT + * and corresponding EFL_INT_ENA_W1C bits are both set + * @ilk: Set when any ILK_INT bit is set + * @lbc: Set when any LBC_INT RO bit is set or when any non-RO LBC_INT + * and corresponding LBC_INT_ENA_W1C bits are bot set + * @pem: Set when any PEM(0)_INT RO bit is set or when any non-RO + * PEM(0)_INT and corresponding PEM(0)_INT_ENA_W1C bit are both set + * @ucd: Set when any UCD_INT bit is set + * @zctl: Set when any ZIP_INT RO bit is set or when any non-RO ZIP_INT + * and corresponding ZIP_INT_ENA_W1C bits are both set + * @lbm: Set when any LBM_INT bit is set + * @nps_pkt: Set when any NPS_PKT_INT bit is set + * @nps_core: Set when any NPS_CORE_INT RO bit is set or when non-RO + * NPS_CORE_INT and corresponding NSP_CORE_INT_ENA_W1C bits are both set + */ +union nps_core_int_active { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 resend : 1; + u64 raz : 43; + u64 ocla : 1; + u64 mbox : 1; + u64 emu : 4; + u64 bmo : 1; + u64 bmi : 1; + u64 aqm : 1; + u64 zqm : 1; + u64 efl : 1; + u64 ilk : 1; + u64 lbc : 1; + u64 pem : 1; + u64 pom : 1; + u64 ucd : 1; + u64 zctl : 1; + u64 lbm : 1; + u64 nps_pkt : 1; + u64 nps_core : 1; +#else + u64 nps_core : 1; + u64 nps_pkt : 1; + u64 lbm : 1; + u64 zctl: 1; + u64 ucd : 1; + u64 pom : 1; + u64 pem : 1; + u64 lbc : 1; + u64 ilk : 1; + u64 efl : 1; + u64 zqm : 1; + u64 aqm : 1; + u64 bmi : 1; + u64 bmo : 1; + u64 emu : 4; + u64 mbox : 1; + u64 ocla : 1; + u64 raz : 43; + u64 resend : 1; +#endif + } s; +}; + +/** + * struct efl_core_int - EFL Interrupt Registers + * @epci_decode_err: EPCI decoded a transacation that was unknown + * This error should only occurred when there is a micrcode/SE error + * and should be considered fatal + * @ae_err: An AE uncorrectable error occurred. + * See EFL_CORE(0..3)_AE_ERR_INT + * @se_err: An SE uncorrectable error occurred. + * See EFL_CORE(0..3)_SE_ERR_INT + * @dbe: Double-bit error occurred in EFL + * @sbe: Single-bit error occurred in EFL + * @d_left: Asserted when new POM-Header-BMI-data is + * being sent to an Exec, and that Exec has Not read all BMI + * data associated with the previous POM header + * @len_ovr: Asserted when an Exec-Read is issued that is more than + * 14 greater in length that the BMI data left to be read + */ +union efl_core_int { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 57; + u64 epci_decode_err : 1; + u64 ae_err : 1; + u64 se_err : 1; + u64 dbe : 1; + u64 sbe : 1; + u64 d_left : 1; + u64 len_ovr : 1; +#else + u64 len_ovr : 1; + u64 d_left : 1; + u64 sbe : 1; + u64 dbe : 1; + u64 se_err : 1; + u64 ae_err : 1; + u64 epci_decode_err : 1; + u64 raz : 57; +#endif + } s; +}; + +/** + * struct efl_core_int_ena_w1s - EFL core interrupt enable set register + * @epci_decode_err: Reads or sets enable for + * EFL_CORE(0..3)_INT[EPCI_DECODE_ERR]. + * @d_left: Reads or sets enable for + * EFL_CORE(0..3)_INT[D_LEFT]. + * @len_ovr: Reads or sets enable for + * EFL_CORE(0..3)_INT[LEN_OVR]. + */ +union efl_core_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_7_63 : 57; + u64 epci_decode_err : 1; + u64 raz_2_5 : 4; + u64 d_left : 1; + u64 len_ovr : 1; +#else + u64 len_ovr : 1; + u64 d_left : 1; + u64 raz_2_5 : 4; + u64 epci_decode_err : 1; + u64 raz_7_63 : 57; +#endif + } s; +}; + +/** + * struct efl_rnm_ctl_status - RNM Control and Status Register + * @ent_sel: Select input to RNM FIFO + * @exp_ent: Exported entropy enable for random number generator + * @rng_rst: Reset to RNG. Setting this bit to 1 cancels the generation + * of the current random number. + * @rnm_rst: Reset the RNM. Setting this bit to 1 clears all sorted numbers + * in the random number memory. + * @rng_en: Enabled the output of the RNG. + * @ent_en: Entropy enable for random number generator. + */ +union efl_rnm_ctl_status { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_9_63 : 55; + u64 ent_sel : 4; + u64 exp_ent : 1; + u64 rng_rst : 1; + u64 rnm_rst : 1; + u64 rng_en : 1; + u64 ent_en : 1; +#else + u64 ent_en : 1; + u64 rng_en : 1; + u64 rnm_rst : 1; + u64 rng_rst : 1; + u64 exp_ent : 1; + u64 ent_sel : 4; + u64 raz_9_63 : 55; +#endif + } s; +}; + +/** + * struct bmi_ctl - BMI control register + * @ilk_hdrq_thrsh: Maximum number of header queue locations + * that ILK packets may consume. When the threshold is + * exceeded ILK_XOFF is sent to the BMI_X2P_ARB. + * @nps_hdrq_thrsh: Maximum number of header queue locations + * that NPS packets may consume. When the threshold is + * exceeded NPS_XOFF is sent to the BMI_X2P_ARB. + * @totl_hdrq_thrsh: Maximum number of header queue locations + * that the sum of ILK and NPS packets may consume. + * @ilk_free_thrsh: Maximum number of buffers that ILK packet + * flows may consume before ILK_XOFF is sent to the BMI_X2P_ARB. + * @nps_free_thrsh: Maximum number of buffers that NPS packet + * flows may consume before NPS XOFF is sent to the BMI_X2p_ARB. + * @totl_free_thrsh: Maximum number of buffers that bot ILK and NPS + * packet flows may consume before both NPS_XOFF and ILK_XOFF + * are asserted to the BMI_X2P_ARB. + * @max_pkt_len: Maximum packet length, integral number of 256B + * buffers. + */ +union bmi_ctl { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_56_63 : 8; + u64 ilk_hdrq_thrsh : 8; + u64 nps_hdrq_thrsh : 8; + u64 totl_hdrq_thrsh : 8; + u64 ilk_free_thrsh : 8; + u64 nps_free_thrsh : 8; + u64 totl_free_thrsh : 8; + u64 max_pkt_len : 8; +#else + u64 max_pkt_len : 8; + u64 totl_free_thrsh : 8; + u64 nps_free_thrsh : 8; + u64 ilk_free_thrsh : 8; + u64 totl_hdrq_thrsh : 8; + u64 nps_hdrq_thrsh : 8; + u64 ilk_hdrq_thrsh : 8; + u64 raz_56_63 : 8; +#endif + } s; +}; + +/** + * struct bmi_int_ena_w1s - BMI interrupt enable set register + * @ilk_req_oflw: Reads or sets enable for + * BMI_INT[ILK_REQ_OFLW]. + * @nps_req_oflw: Reads or sets enable for + * BMI_INT[NPS_REQ_OFLW]. + * @fpf_undrrn: Reads or sets enable for + * BMI_INT[FPF_UNDRRN]. + * @eop_err_ilk: Reads or sets enable for + * BMI_INT[EOP_ERR_ILK]. + * @eop_err_nps: Reads or sets enable for + * BMI_INT[EOP_ERR_NPS]. + * @sop_err_ilk: Reads or sets enable for + * BMI_INT[SOP_ERR_ILK]. + * @sop_err_nps: Reads or sets enable for + * BMI_INT[SOP_ERR_NPS]. + * @pkt_rcv_err_ilk: Reads or sets enable for + * BMI_INT[PKT_RCV_ERR_ILK]. + * @pkt_rcv_err_nps: Reads or sets enable for + * BMI_INT[PKT_RCV_ERR_NPS]. + * @max_len_err_ilk: Reads or sets enable for + * BMI_INT[MAX_LEN_ERR_ILK]. + * @max_len_err_nps: Reads or sets enable for + * BMI_INT[MAX_LEN_ERR_NPS]. + */ +union bmi_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_13_63 : 51; + u64 ilk_req_oflw : 1; + u64 nps_req_oflw : 1; + u64 raz_10 : 1; + u64 raz_9 : 1; + u64 fpf_undrrn : 1; + u64 eop_err_ilk : 1; + u64 eop_err_nps : 1; + u64 sop_err_ilk : 1; + u64 sop_err_nps : 1; + u64 pkt_rcv_err_ilk : 1; + u64 pkt_rcv_err_nps : 1; + u64 max_len_err_ilk : 1; + u64 max_len_err_nps : 1; +#else + u64 max_len_err_nps : 1; + u64 max_len_err_ilk : 1; + u64 pkt_rcv_err_nps : 1; + u64 pkt_rcv_err_ilk : 1; + u64 sop_err_nps : 1; + u64 sop_err_ilk : 1; + u64 eop_err_nps : 1; + u64 eop_err_ilk : 1; + u64 fpf_undrrn : 1; + u64 raz_9 : 1; + u64 raz_10 : 1; + u64 nps_req_oflw : 1; + u64 ilk_req_oflw : 1; + u64 raz_13_63 : 51; +#endif + } s; +}; + +/** + * struct bmo_ctl2 - BMO Control2 Register + * @arb_sel: Determines P2X Arbitration + * @ilk_buf_thrsh: Maximum number of buffers that the + * ILK packet flows may consume before ILK XOFF is + * asserted to the POM. + * @nps_slc_buf_thrsh: Maximum number of buffers that the + * NPS_SLC packet flow may consume before NPS_SLC XOFF is + * asserted to the POM. + * @nps_uns_buf_thrsh: Maximum number of buffers that the + * NPS_UNS packet flow may consume before NPS_UNS XOFF is + * asserted to the POM. + * @totl_buf_thrsh: Maximum number of buffers that ILK, NPS_UNS and + * NPS_SLC packet flows may consume before NPS_UNS XOFF, NSP_SLC and + * ILK_XOFF are all asserted POM. + */ +union bmo_ctl2 { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 arb_sel : 1; + u64 raz_32_62 : 31; + u64 ilk_buf_thrsh : 8; + u64 nps_slc_buf_thrsh : 8; + u64 nps_uns_buf_thrsh : 8; + u64 totl_buf_thrsh : 8; +#else + u64 totl_buf_thrsh : 8; + u64 nps_uns_buf_thrsh : 8; + u64 nps_slc_buf_thrsh : 8; + u64 ilk_buf_thrsh : 8; + u64 raz_32_62 : 31; + u64 arb_sel : 1; +#endif + } s; +}; + +/** + * struct pom_int_ena_w1s - POM interrupt enable set register + * @illegal_intf: Reads or sets enable for POM_INT[ILLEGAL_INTF]. + * @illegal_dport: Reads or sets enable for POM_INT[ILLEGAL_DPORT]. + */ +union pom_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz2 : 60; + u64 illegal_intf : 1; + u64 illegal_dport : 1; + u64 raz1 : 1; + u64 raz0 : 1; +#else + u64 raz0 : 1; + u64 raz1 : 1; + u64 illegal_dport : 1; + u64 illegal_intf : 1; + u64 raz2 : 60; +#endif + } s; +}; + +/** + * struct lbc_inval_ctl - LBC invalidation control register + * @wait_timer: Wait timer for wait state. [WAIT_TIMER] must + * always be written with its reset value. + * @cam_inval_start: Software should write [CAM_INVAL_START]=1 + * to initiate an LBC cache invalidation. After this, software + * should read LBC_INVAL_STATUS until LBC_INVAL_STATUS[DONE] is set. + * LBC hardware clears [CAVM_INVAL_START] before software can + * observed LBC_INVAL_STATUS[DONE] to be set + */ +union lbc_inval_ctl { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz2 : 48; + u64 wait_timer : 8; + u64 raz1 : 6; + u64 cam_inval_start : 1; + u64 raz0 : 1; +#else + u64 raz0 : 1; + u64 cam_inval_start : 1; + u64 raz1 : 6; + u64 wait_timer : 8; + u64 raz2 : 48; +#endif + } s; +}; + +/** + * struct lbc_int_ena_w1s - LBC interrupt enable set register + * @cam_hard_err: Reads or sets enable for LBC_INT[CAM_HARD_ERR]. + * @cam_inval_abort: Reads or sets enable for LBC_INT[CAM_INVAL_ABORT]. + * @over_fetch_err: Reads or sets enable for LBC_INT[OVER_FETCH_ERR]. + * @cache_line_to_err: Reads or sets enable for + * LBC_INT[CACHE_LINE_TO_ERR]. + * @cam_soft_err: Reads or sets enable for + * LBC_INT[CAM_SOFT_ERR]. + * @dma_rd_err: Reads or sets enable for + * LBC_INT[DMA_RD_ERR]. + */ +union lbc_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_10_63 : 54; + u64 cam_hard_err : 1; + u64 cam_inval_abort : 1; + u64 over_fetch_err : 1; + u64 cache_line_to_err : 1; + u64 raz_2_5 : 4; + u64 cam_soft_err : 1; + u64 dma_rd_err : 1; +#else + u64 dma_rd_err : 1; + u64 cam_soft_err : 1; + u64 raz_2_5 : 4; + u64 cache_line_to_err : 1; + u64 over_fetch_err : 1; + u64 cam_inval_abort : 1; + u64 cam_hard_err : 1; + u64 raz_10_63 : 54; +#endif + } s; +}; + +/** + * struct lbc_int - LBC interrupt summary register + * @cam_hard_err: indicates a fatal hardware error. + * It requires system reset. + * When [CAM_HARD_ERR] is set, LBC stops logging any new information in + * LBC_POM_MISS_INFO_LOG, + * LBC_POM_MISS_ADDR_LOG, + * LBC_EFL_MISS_INFO_LOG, and + * LBC_EFL_MISS_ADDR_LOG. + * Software should sample them. + * @cam_inval_abort: indicates a fatal hardware error. + * System reset is required. + * @over_fetch_err: indicates a fatal hardware error + * System reset is required + * @cache_line_to_err: is a debug feature. + * This timeout interrupt bit tells the software that + * a cacheline in LBC has non-zero usage and the context + * has not been used for greater than the + * LBC_TO_CNT[TO_CNT] time interval. + * @sbe: Memory SBE error. This is recoverable via ECC. + * See LBC_ECC_INT for more details. + * @dbe: Memory DBE error. This is a fatal and requires a + * system reset. + * @pref_dat_len_mismatch_err: Summary bit for context length + * mismatch errors. + * @rd_dat_len_mismatch_err: Summary bit for SE read data length + * greater than data prefect length errors. + * @cam_soft_err: is recoverable. Software must complete a + * LBC_INVAL_CTL[CAM_INVAL_START] invalidation sequence and + * then clear [CAM_SOFT_ERR]. + * @dma_rd_err: A context prefect read of host memory returned with + * a read error. + */ +union lbc_int { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_10_63 : 54; + u64 cam_hard_err : 1; + u64 cam_inval_abort : 1; + u64 over_fetch_err : 1; + u64 cache_line_to_err : 1; + u64 sbe : 1; + u64 dbe : 1; + u64 pref_dat_len_mismatch_err : 1; + u64 rd_dat_len_mismatch_err : 1; + u64 cam_soft_err : 1; + u64 dma_rd_err : 1; +#else + u64 dma_rd_err : 1; + u64 cam_soft_err : 1; + u64 rd_dat_len_mismatch_err : 1; + u64 pref_dat_len_mismatch_err : 1; + u64 dbe : 1; + u64 sbe : 1; + u64 cache_line_to_err : 1; + u64 over_fetch_err : 1; + u64 cam_inval_abort : 1; + u64 cam_hard_err : 1; + u64 raz_10_63 : 54; +#endif + } s; +}; + +/** + * struct lbc_inval_status: LBC Invalidation status register + * @cam_clean_entry_complete_cnt: The number of entries that are + * cleaned up successfully. + * @cam_clean_entry_cnt: The number of entries that have the CAM + * inval command issued. + * @cam_inval_state: cam invalidation FSM state + * @cam_inval_abort: cam invalidation abort + * @cam_rst_rdy: lbc_cam reset ready + * @done: LBC clears [DONE] when + * LBC_INVAL_CTL[CAM_INVAL_START] is written with a one, + * and sets [DONE] when it completes the invalidation + * sequence. + */ +union lbc_inval_status { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz3 : 23; + u64 cam_clean_entry_complete_cnt : 9; + u64 raz2 : 7; + u64 cam_clean_entry_cnt : 9; + u64 raz1 : 5; + u64 cam_inval_state : 3; + u64 raz0 : 5; + u64 cam_inval_abort : 1; + u64 cam_rst_rdy : 1; + u64 done : 1; +#else + u64 done : 1; + u64 cam_rst_rdy : 1; + u64 cam_inval_abort : 1; + u64 raz0 : 5; + u64 cam_inval_state : 3; + u64 raz1 : 5; + u64 cam_clean_entry_cnt : 9; + u64 raz2 : 7; + u64 cam_clean_entry_complete_cnt : 9; + u64 raz3 : 23; +#endif + } s; +}; + +/** + * struct rst_boot: RST Boot Register + * @jtcsrdis: when set, internal CSR access via JTAG TAP controller + * is disabled + * @jt_tst_mode: JTAG test mode + * @io_supply: I/O power supply setting based on IO_VDD_SELECT pin: + * 0x1 = 1.8V + * 0x2 = 2.5V + * 0x4 = 3.3V + * All other values are reserved + * @pnr_mul: clock multiplier + * @lboot: last boot cause mask, resets only with PLL_DC_OK + * @rboot: determines whether core 0 remains in reset after + * chip cold or warm or soft reset + * @rboot_pin: read only access to REMOTE_BOOT pin + */ +union rst_boot { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_63 : 1; + u64 jtcsrdis : 1; + u64 raz_59_61 : 3; + u64 jt_tst_mode : 1; + u64 raz_40_57 : 18; + u64 io_supply : 3; + u64 raz_30_36 : 7; + u64 pnr_mul : 6; + u64 raz_12_23 : 12; + u64 lboot : 10; + u64 rboot : 1; + u64 rboot_pin : 1; +#else + u64 rboot_pin : 1; + u64 rboot : 1; + u64 lboot : 10; + u64 raz_12_23 : 12; + u64 pnr_mul : 6; + u64 raz_30_36 : 7; + u64 io_supply : 3; + u64 raz_40_57 : 18; + u64 jt_tst_mode : 1; + u64 raz_59_61 : 3; + u64 jtcsrdis : 1; + u64 raz_63 : 1; +#endif + }; +}; + +/** + * struct fus_dat1: Fuse Data 1 Register + * @pll_mul: main clock PLL multiplier hardware limit + * @pll_half_dis: main clock PLL control + * @efus_lck: efuse lockdown + * @zip_info: ZIP information + * @bar2_sz_conf: when zero, BAR2 size conforms to + * PCIe specification + * @efus_ign: efuse ignore + * @nozip: ZIP disable + * @pll_alt_matrix: select alternate PLL matrix + * @pll_bwadj_denom: select CLKF denominator for + * BWADJ value + * @chip_id: chip ID + */ +union fus_dat1 { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_57_63 : 7; + u64 pll_mul : 3; + u64 pll_half_dis : 1; + u64 raz_43_52 : 10; + u64 efus_lck : 3; + u64 raz_26_39 : 14; + u64 zip_info : 5; + u64 bar2_sz_conf : 1; + u64 efus_ign : 1; + u64 nozip : 1; + u64 raz_11_17 : 7; + u64 pll_alt_matrix : 1; + u64 pll_bwadj_denom : 2; + u64 chip_id : 8; +#else + u64 chip_id : 8; + u64 pll_bwadj_denom : 2; + u64 pll_alt_matrix : 1; + u64 raz_11_17 : 7; + u64 nozip : 1; + u64 efus_ign : 1; + u64 bar2_sz_conf : 1; + u64 zip_info : 5; + u64 raz_26_39 : 14; + u64 efus_lck : 3; + u64 raz_43_52 : 10; + u64 pll_half_dis : 1; + u64 pll_mul : 3; + u64 raz_57_63 : 7; +#endif + }; +}; + +#endif /* __NITROX_CSR_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_debugfs.c b/drivers/crypto/cavium/nitrox/nitrox_debugfs.c new file mode 100644 index 000000000..741572a01 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_debugfs.c @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +#include "nitrox_csr.h" +#include "nitrox_debugfs.h" +#include "nitrox_dev.h" + +static int firmware_show(struct seq_file *s, void *v) +{ + struct nitrox_device *ndev = s->private; + + seq_printf(s, "Version: %s\n", ndev->hw.fw_name[0]); + seq_printf(s, "Version: %s\n", ndev->hw.fw_name[1]); + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(firmware); + +static int device_show(struct seq_file *s, void *v) +{ + struct nitrox_device *ndev = s->private; + + seq_printf(s, "NITROX [%d]\n", ndev->idx); + seq_printf(s, " Part Name: %s\n", ndev->hw.partname); + seq_printf(s, " Frequency: %d MHz\n", ndev->hw.freq); + seq_printf(s, " Device ID: 0x%0x\n", ndev->hw.device_id); + seq_printf(s, " Revision ID: 0x%0x\n", ndev->hw.revision_id); + seq_printf(s, " Cores: [AE=%u SE=%u ZIP=%u]\n", + ndev->hw.ae_cores, ndev->hw.se_cores, ndev->hw.zip_cores); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(device); + +static int stats_show(struct seq_file *s, void *v) +{ + struct nitrox_device *ndev = s->private; + + seq_printf(s, "NITROX [%d] Request Statistics\n", ndev->idx); + seq_printf(s, " Posted: %llu\n", + (u64)atomic64_read(&ndev->stats.posted)); + seq_printf(s, " Completed: %llu\n", + (u64)atomic64_read(&ndev->stats.completed)); + seq_printf(s, " Dropped: %llu\n", + (u64)atomic64_read(&ndev->stats.dropped)); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(stats); + +void nitrox_debugfs_exit(struct nitrox_device *ndev) +{ + debugfs_remove_recursive(ndev->debugfs_dir); + ndev->debugfs_dir = NULL; +} + +void nitrox_debugfs_init(struct nitrox_device *ndev) +{ + struct dentry *dir; + + dir = debugfs_create_dir(KBUILD_MODNAME, NULL); + + ndev->debugfs_dir = dir; + debugfs_create_file("firmware", 0400, dir, ndev, &firmware_fops); + debugfs_create_file("device", 0400, dir, ndev, &device_fops); + debugfs_create_file("stats", 0400, dir, ndev, &stats_fops); +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_debugfs.h b/drivers/crypto/cavium/nitrox/nitrox_debugfs.h new file mode 100644 index 000000000..09c4cf251 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_debugfs.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NITROX_DEBUGFS_H +#define __NITROX_DEBUGFS_H + +#include "nitrox_dev.h" + +#ifdef CONFIG_DEBUG_FS +void nitrox_debugfs_init(struct nitrox_device *ndev); +void nitrox_debugfs_exit(struct nitrox_device *ndev); +#else +static inline void nitrox_debugfs_init(struct nitrox_device *ndev) +{ +} + +static inline void nitrox_debugfs_exit(struct nitrox_device *ndev) +{ +} +#endif /* !CONFIG_DEBUG_FS */ + +#endif /* __NITROX_DEBUGFS_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_dev.h b/drivers/crypto/cavium/nitrox/nitrox_dev.h new file mode 100644 index 000000000..c2d0c23fb --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_dev.h @@ -0,0 +1,302 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NITROX_DEV_H +#define __NITROX_DEV_H + +#include +#include +#include +#include + +#define VERSION_LEN 32 +/* Maximum queues in PF mode */ +#define MAX_PF_QUEUES 64 +/* Maximum device queues */ +#define MAX_DEV_QUEUES (MAX_PF_QUEUES) +/* Maximum UCD Blocks */ +#define CNN55XX_MAX_UCD_BLOCKS 8 + +/** + * struct nitrox_cmdq - NITROX command queue + * @cmd_qlock: command queue lock + * @resp_qlock: response queue lock + * @backlog_qlock: backlog queue lock + * @ndev: NITROX device + * @response_head: submitted request list + * @backlog_head: backlog queue + * @dbell_csr_addr: doorbell register address for this queue + * @compl_cnt_csr_addr: completion count register address of the slc port + * @base: command queue base address + * @dma: dma address of the base + * @pending_count: request pending at device + * @backlog_count: backlog request count + * @write_idx: next write index for the command + * @instr_size: command size + * @qno: command queue number + * @qsize: command queue size + * @unalign_base: unaligned base address + * @unalign_dma: unaligned dma address + */ +struct nitrox_cmdq { + spinlock_t cmd_qlock; + spinlock_t resp_qlock; + spinlock_t backlog_qlock; + + struct nitrox_device *ndev; + struct list_head response_head; + struct list_head backlog_head; + + u8 __iomem *dbell_csr_addr; + u8 __iomem *compl_cnt_csr_addr; + u8 *base; + dma_addr_t dma; + + struct work_struct backlog_qflush; + + atomic_t pending_count; + atomic_t backlog_count; + + int write_idx; + u8 instr_size; + u8 qno; + u32 qsize; + + u8 *unalign_base; + dma_addr_t unalign_dma; +}; + +/** + * struct nitrox_hw - NITROX hardware information + * @partname: partname ex: CNN55xxx-xxx + * @fw_name: firmware version + * @freq: NITROX frequency + * @vendor_id: vendor ID + * @device_id: device ID + * @revision_id: revision ID + * @se_cores: number of symmetric cores + * @ae_cores: number of asymmetric cores + * @zip_cores: number of zip cores + */ +struct nitrox_hw { + char partname[IFNAMSIZ * 2]; + char fw_name[CNN55XX_MAX_UCD_BLOCKS][VERSION_LEN]; + + int freq; + u16 vendor_id; + u16 device_id; + u8 revision_id; + + u8 se_cores; + u8 ae_cores; + u8 zip_cores; +}; + +struct nitrox_stats { + atomic64_t posted; + atomic64_t completed; + atomic64_t dropped; +}; + +#define IRQ_NAMESZ 32 + +struct nitrox_q_vector { + char name[IRQ_NAMESZ]; + bool valid; + int ring; + struct tasklet_struct resp_tasklet; + union { + struct nitrox_cmdq *cmdq; + struct nitrox_device *ndev; + }; +}; + +enum mcode_type { + MCODE_TYPE_INVALID, + MCODE_TYPE_AE, + MCODE_TYPE_SE_SSL, + MCODE_TYPE_SE_IPSEC, +}; + +/** + * mbox_msg - Mailbox message data + * @type: message type + * @opcode: message opcode + * @data: message data + */ +union mbox_msg { + u64 value; + struct { + u64 type: 2; + u64 opcode: 6; + u64 data: 58; + }; + struct { + u64 type: 2; + u64 opcode: 6; + u64 chipid: 8; + u64 vfid: 8; + } id; + struct { + u64 type: 2; + u64 opcode: 6; + u64 count: 4; + u64 info: 40; + u64 next_se_grp: 3; + u64 next_ae_grp: 3; + } mcode_info; +}; + +/** + * nitrox_vfdev - NITROX VF device instance in PF + * @state: VF device state + * @vfno: VF number + * @nr_queues: number of queues enabled in VF + * @ring: ring to communicate with VF + * @msg: Mailbox message data from VF + * @mbx_resp: Mailbox counters + */ +struct nitrox_vfdev { + atomic_t state; + int vfno; + int nr_queues; + int ring; + union mbox_msg msg; + atomic64_t mbx_resp; +}; + +/** + * struct nitrox_iov - SR-IOV information + * @num_vfs: number of VF(s) enabled + * @max_vf_queues: Maximum number of queues allowed for VF + * @vfdev: VF(s) devices + * @pf2vf_wq: workqueue for PF2VF communication + * @msix: MSI-X entry for PF in SR-IOV case + */ +struct nitrox_iov { + int num_vfs; + int max_vf_queues; + struct nitrox_vfdev *vfdev; + struct workqueue_struct *pf2vf_wq; + struct msix_entry msix; +}; + +/* + * NITROX Device states + */ +enum ndev_state { + __NDEV_NOT_READY, + __NDEV_READY, + __NDEV_IN_RESET, +}; + +/* NITROX support modes for VF(s) */ +enum vf_mode { + __NDEV_MODE_PF, + __NDEV_MODE_VF16, + __NDEV_MODE_VF32, + __NDEV_MODE_VF64, + __NDEV_MODE_VF128, +}; + +#define __NDEV_SRIOV_BIT 0 + +/* command queue size */ +#define DEFAULT_CMD_QLEN 2048 +/* command timeout in milliseconds */ +#define CMD_TIMEOUT 2000 + +#define DEV(ndev) ((struct device *)(&(ndev)->pdev->dev)) + +#define NITROX_CSR_ADDR(ndev, offset) \ + ((ndev)->bar_addr + (offset)) + +/** + * struct nitrox_device - NITROX Device Information. + * @list: pointer to linked list of devices + * @bar_addr: iomap address + * @pdev: PCI device information + * @state: NITROX device state + * @flags: flags to indicate device the features + * @timeout: Request timeout in jiffies + * @refcnt: Device usage count + * @idx: device index (0..N) + * @node: NUMA node id attached + * @qlen: Command queue length + * @nr_queues: Number of command queues + * @mode: Device mode PF/VF + * @ctx_pool: DMA pool for crypto context + * @pkt_inq: Packet input rings + * @aqmq: AQM command queues + * @qvec: MSI-X queue vectors information + * @iov: SR-IOV informatin + * @num_vecs: number of MSI-X vectors + * @stats: request statistics + * @hw: hardware information + * @debugfs_dir: debugfs directory + */ +struct nitrox_device { + struct list_head list; + + u8 __iomem *bar_addr; + struct pci_dev *pdev; + + atomic_t state; + unsigned long flags; + unsigned long timeout; + refcount_t refcnt; + + u8 idx; + int node; + u16 qlen; + u16 nr_queues; + enum vf_mode mode; + + struct dma_pool *ctx_pool; + struct nitrox_cmdq *pkt_inq; + struct nitrox_cmdq *aqmq[MAX_DEV_QUEUES] ____cacheline_aligned_in_smp; + + struct nitrox_q_vector *qvec; + struct nitrox_iov iov; + int num_vecs; + + struct nitrox_stats stats; + struct nitrox_hw hw; +#if IS_ENABLED(CONFIG_DEBUG_FS) + struct dentry *debugfs_dir; +#endif +}; + +/** + * nitrox_read_csr - Read from device register + * @ndev: NITROX device + * @offset: offset of the register to read + * + * Returns: value read + */ +static inline u64 nitrox_read_csr(struct nitrox_device *ndev, u64 offset) +{ + return readq(ndev->bar_addr + offset); +} + +/** + * nitrox_write_csr - Write to device register + * @ndev: NITROX device + * @offset: offset of the register to write + * @value: value to write + */ +static inline void nitrox_write_csr(struct nitrox_device *ndev, u64 offset, + u64 value) +{ + writeq(value, (ndev->bar_addr + offset)); +} + +static inline bool nitrox_ready(struct nitrox_device *ndev) +{ + return atomic_read(&ndev->state) == __NDEV_READY; +} + +static inline bool nitrox_vfdev_ready(struct nitrox_vfdev *vfdev) +{ + return atomic_read(&vfdev->state) == __NDEV_READY; +} + +#endif /* __NITROX_DEV_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_hal.c b/drivers/crypto/cavium/nitrox/nitrox_hal.c new file mode 100644 index 000000000..e36c1741b --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_hal.c @@ -0,0 +1,680 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +#include "nitrox_dev.h" +#include "nitrox_csr.h" +#include "nitrox_hal.h" + +#define PLL_REF_CLK 50 +#define MAX_CSR_RETRIES 10 + +/** + * emu_enable_cores - Enable EMU cluster cores. + * @ndev: NITROX device + */ +static void emu_enable_cores(struct nitrox_device *ndev) +{ + union emu_se_enable emu_se; + union emu_ae_enable emu_ae; + int i; + + /* AE cores 20 per cluster */ + emu_ae.value = 0; + emu_ae.s.enable = 0xfffff; + + /* SE cores 16 per cluster */ + emu_se.value = 0; + emu_se.s.enable = 0xffff; + + /* enable per cluster cores */ + for (i = 0; i < NR_CLUSTERS; i++) { + nitrox_write_csr(ndev, EMU_AE_ENABLEX(i), emu_ae.value); + nitrox_write_csr(ndev, EMU_SE_ENABLEX(i), emu_se.value); + } +} + +/** + * nitrox_config_emu_unit - configure EMU unit. + * @ndev: NITROX device + */ +void nitrox_config_emu_unit(struct nitrox_device *ndev) +{ + union emu_wd_int_ena_w1s emu_wd_int; + union emu_ge_int_ena_w1s emu_ge_int; + u64 offset; + int i; + + /* enable cores */ + emu_enable_cores(ndev); + + /* enable general error and watch dog interrupts */ + emu_ge_int.value = 0; + emu_ge_int.s.se_ge = 0xffff; + emu_ge_int.s.ae_ge = 0xfffff; + emu_wd_int.value = 0; + emu_wd_int.s.se_wd = 1; + + for (i = 0; i < NR_CLUSTERS; i++) { + offset = EMU_WD_INT_ENA_W1SX(i); + nitrox_write_csr(ndev, offset, emu_wd_int.value); + offset = EMU_GE_INT_ENA_W1SX(i); + nitrox_write_csr(ndev, offset, emu_ge_int.value); + } +} + +static void reset_pkt_input_ring(struct nitrox_device *ndev, int ring) +{ + union nps_pkt_in_instr_ctl pkt_in_ctl; + union nps_pkt_in_done_cnts pkt_in_cnts; + int max_retries = MAX_CSR_RETRIES; + u64 offset; + + /* step 1: disable the ring, clear enable bit */ + offset = NPS_PKT_IN_INSTR_CTLX(ring); + pkt_in_ctl.value = nitrox_read_csr(ndev, offset); + pkt_in_ctl.s.enb = 0; + nitrox_write_csr(ndev, offset, pkt_in_ctl.value); + + /* step 2: wait to clear [ENB] */ + usleep_range(100, 150); + do { + pkt_in_ctl.value = nitrox_read_csr(ndev, offset); + if (!pkt_in_ctl.s.enb) + break; + udelay(50); + } while (max_retries--); + + /* step 3: clear done counts */ + offset = NPS_PKT_IN_DONE_CNTSX(ring); + pkt_in_cnts.value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, pkt_in_cnts.value); + usleep_range(50, 100); +} + +void enable_pkt_input_ring(struct nitrox_device *ndev, int ring) +{ + union nps_pkt_in_instr_ctl pkt_in_ctl; + int max_retries = MAX_CSR_RETRIES; + u64 offset; + + /* 64-byte instruction size */ + offset = NPS_PKT_IN_INSTR_CTLX(ring); + pkt_in_ctl.value = nitrox_read_csr(ndev, offset); + pkt_in_ctl.s.is64b = 1; + pkt_in_ctl.s.enb = 1; + nitrox_write_csr(ndev, offset, pkt_in_ctl.value); + + /* wait for set [ENB] */ + do { + pkt_in_ctl.value = nitrox_read_csr(ndev, offset); + if (pkt_in_ctl.s.enb) + break; + udelay(50); + } while (max_retries--); +} + +/** + * nitrox_config_pkt_input_rings - configure Packet Input Rings + * @ndev: NITROX device + */ +void nitrox_config_pkt_input_rings(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < ndev->nr_queues; i++) { + struct nitrox_cmdq *cmdq = &ndev->pkt_inq[i]; + union nps_pkt_in_instr_rsize pkt_in_rsize; + union nps_pkt_in_instr_baoff_dbell pkt_in_dbell; + u64 offset; + + reset_pkt_input_ring(ndev, i); + + /** + * step 4: + * configure ring base address 16-byte aligned, + * size and interrupt threshold. + */ + offset = NPS_PKT_IN_INSTR_BADDRX(i); + nitrox_write_csr(ndev, offset, cmdq->dma); + + /* configure ring size */ + offset = NPS_PKT_IN_INSTR_RSIZEX(i); + pkt_in_rsize.value = 0; + pkt_in_rsize.s.rsize = ndev->qlen; + nitrox_write_csr(ndev, offset, pkt_in_rsize.value); + + /* set high threshold for pkt input ring interrupts */ + offset = NPS_PKT_IN_INT_LEVELSX(i); + nitrox_write_csr(ndev, offset, 0xffffffff); + + /* step 5: clear off door bell counts */ + offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(i); + pkt_in_dbell.value = 0; + pkt_in_dbell.s.dbell = 0xffffffff; + nitrox_write_csr(ndev, offset, pkt_in_dbell.value); + + /* enable the ring */ + enable_pkt_input_ring(ndev, i); + } +} + +static void reset_pkt_solicit_port(struct nitrox_device *ndev, int port) +{ + union nps_pkt_slc_ctl pkt_slc_ctl; + union nps_pkt_slc_cnts pkt_slc_cnts; + int max_retries = MAX_CSR_RETRIES; + u64 offset; + + /* step 1: disable slc port */ + offset = NPS_PKT_SLC_CTLX(port); + pkt_slc_ctl.value = nitrox_read_csr(ndev, offset); + pkt_slc_ctl.s.enb = 0; + nitrox_write_csr(ndev, offset, pkt_slc_ctl.value); + + /* step 2 */ + usleep_range(100, 150); + /* wait to clear [ENB] */ + do { + pkt_slc_ctl.value = nitrox_read_csr(ndev, offset); + if (!pkt_slc_ctl.s.enb) + break; + udelay(50); + } while (max_retries--); + + /* step 3: clear slc counters */ + offset = NPS_PKT_SLC_CNTSX(port); + pkt_slc_cnts.value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, pkt_slc_cnts.value); + usleep_range(50, 100); +} + +void enable_pkt_solicit_port(struct nitrox_device *ndev, int port) +{ + union nps_pkt_slc_ctl pkt_slc_ctl; + int max_retries = MAX_CSR_RETRIES; + u64 offset; + + offset = NPS_PKT_SLC_CTLX(port); + pkt_slc_ctl.value = 0; + pkt_slc_ctl.s.enb = 1; + /* + * 8 trailing 0x00 bytes will be added + * to the end of the outgoing packet. + */ + pkt_slc_ctl.s.z = 1; + /* enable response header */ + pkt_slc_ctl.s.rh = 1; + nitrox_write_csr(ndev, offset, pkt_slc_ctl.value); + + /* wait to set [ENB] */ + do { + pkt_slc_ctl.value = nitrox_read_csr(ndev, offset); + if (pkt_slc_ctl.s.enb) + break; + udelay(50); + } while (max_retries--); +} + +static void config_pkt_solicit_port(struct nitrox_device *ndev, int port) +{ + union nps_pkt_slc_int_levels pkt_slc_int; + u64 offset; + + reset_pkt_solicit_port(ndev, port); + + /* step 4: configure interrupt levels */ + offset = NPS_PKT_SLC_INT_LEVELSX(port); + pkt_slc_int.value = 0; + /* time interrupt threshold */ + pkt_slc_int.s.timet = 0x3fffff; + nitrox_write_csr(ndev, offset, pkt_slc_int.value); + + /* enable the solicit port */ + enable_pkt_solicit_port(ndev, port); +} + +void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < ndev->nr_queues; i++) + config_pkt_solicit_port(ndev, i); +} + +/** + * enable_nps_core_interrupts - enable NPS core interrutps + * @ndev: NITROX device. + * + * This includes NPS core interrupts. + */ +static void enable_nps_core_interrupts(struct nitrox_device *ndev) +{ + union nps_core_int_ena_w1s core_int; + + /* NPS core interrutps */ + core_int.value = 0; + core_int.s.host_wr_err = 1; + core_int.s.host_wr_timeout = 1; + core_int.s.exec_wr_timeout = 1; + core_int.s.npco_dma_malform = 1; + core_int.s.host_nps_wr_err = 1; + nitrox_write_csr(ndev, NPS_CORE_INT_ENA_W1S, core_int.value); +} + +void nitrox_config_nps_core_unit(struct nitrox_device *ndev) +{ + union nps_core_gbl_vfcfg core_gbl_vfcfg; + + /* endian control information */ + nitrox_write_csr(ndev, NPS_CORE_CONTROL, 1ULL); + + /* disable ILK interface */ + core_gbl_vfcfg.value = 0; + core_gbl_vfcfg.s.ilk_disable = 1; + core_gbl_vfcfg.s.cfg = __NDEV_MODE_PF; + nitrox_write_csr(ndev, NPS_CORE_GBL_VFCFG, core_gbl_vfcfg.value); + + /* enable nps core interrupts */ + enable_nps_core_interrupts(ndev); +} + +/** + * enable_nps_pkt_interrupts - enable NPS packet interrutps + * @ndev: NITROX device. + * + * This includes NPS packet in and slc interrupts. + */ +static void enable_nps_pkt_interrupts(struct nitrox_device *ndev) +{ + /* NPS packet in ring interrupts */ + nitrox_write_csr(ndev, NPS_PKT_IN_RERR_LO_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, NPS_PKT_IN_RERR_HI_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, NPS_PKT_IN_ERR_TYPE_ENA_W1S, (~0ULL)); + /* NPS packet slc port interrupts */ + nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_HI_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_LO_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, NPS_PKT_SLC_ERR_TYPE_ENA_W1S, (~0uLL)); +} + +void nitrox_config_nps_pkt_unit(struct nitrox_device *ndev) +{ + /* config input and solicit ports */ + nitrox_config_pkt_input_rings(ndev); + nitrox_config_pkt_solicit_ports(ndev); + + /* enable nps packet interrupts */ + enable_nps_pkt_interrupts(ndev); +} + +static void reset_aqm_ring(struct nitrox_device *ndev, int ring) +{ + union aqmq_en aqmq_en_reg; + union aqmq_activity_stat activity_stat; + union aqmq_cmp_cnt cmp_cnt; + int max_retries = MAX_CSR_RETRIES; + u64 offset; + + /* step 1: disable the queue */ + offset = AQMQ_ENX(ring); + aqmq_en_reg.value = 0; + aqmq_en_reg.queue_enable = 0; + nitrox_write_csr(ndev, offset, aqmq_en_reg.value); + + /* step 2: wait for AQMQ_ACTIVITY_STATX[QUEUE_ACTIVE] to clear */ + usleep_range(100, 150); + offset = AQMQ_ACTIVITY_STATX(ring); + do { + activity_stat.value = nitrox_read_csr(ndev, offset); + if (!activity_stat.queue_active) + break; + udelay(50); + } while (max_retries--); + + /* step 3: clear commands completed count */ + offset = AQMQ_CMP_CNTX(ring); + cmp_cnt.value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, cmp_cnt.value); + usleep_range(50, 100); +} + +void enable_aqm_ring(struct nitrox_device *ndev, int ring) +{ + union aqmq_en aqmq_en_reg; + u64 offset; + + offset = AQMQ_ENX(ring); + aqmq_en_reg.value = 0; + aqmq_en_reg.queue_enable = 1; + nitrox_write_csr(ndev, offset, aqmq_en_reg.value); + usleep_range(50, 100); +} + +void nitrox_config_aqm_rings(struct nitrox_device *ndev) +{ + int ring; + + for (ring = 0; ring < ndev->nr_queues; ring++) { + struct nitrox_cmdq *cmdq = ndev->aqmq[ring]; + union aqmq_drbl drbl; + union aqmq_qsz qsize; + union aqmq_cmp_thr cmp_thr; + u64 offset; + + /* steps 1 - 3 */ + reset_aqm_ring(ndev, ring); + + /* step 4: clear doorbell count of ring */ + offset = AQMQ_DRBLX(ring); + drbl.value = 0; + drbl.dbell_count = 0xFFFFFFFF; + nitrox_write_csr(ndev, offset, drbl.value); + + /* step 5: configure host ring details */ + + /* set host address for next command of ring */ + offset = AQMQ_NXT_CMDX(ring); + nitrox_write_csr(ndev, offset, 0ULL); + + /* set host address of ring base */ + offset = AQMQ_BADRX(ring); + nitrox_write_csr(ndev, offset, cmdq->dma); + + /* set ring size */ + offset = AQMQ_QSZX(ring); + qsize.value = 0; + qsize.host_queue_size = ndev->qlen; + nitrox_write_csr(ndev, offset, qsize.value); + + /* set command completion threshold */ + offset = AQMQ_CMP_THRX(ring); + cmp_thr.value = 0; + cmp_thr.commands_completed_threshold = 1; + nitrox_write_csr(ndev, offset, cmp_thr.value); + + /* step 6: enable the queue */ + enable_aqm_ring(ndev, ring); + } +} + +static void enable_aqm_interrupts(struct nitrox_device *ndev) +{ + /* clear interrupt enable bits */ + nitrox_write_csr(ndev, AQM_DBELL_OVF_LO_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, AQM_DBELL_OVF_HI_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, AQM_DMA_RD_ERR_LO_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, AQM_DMA_RD_ERR_HI_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, AQM_EXEC_NA_LO_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, AQM_EXEC_NA_HI_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, AQM_EXEC_ERR_LO_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, AQM_EXEC_ERR_HI_ENA_W1S, (~0ULL)); +} + +void nitrox_config_aqm_unit(struct nitrox_device *ndev) +{ + /* config aqm command queues */ + nitrox_config_aqm_rings(ndev); + + /* enable aqm interrupts */ + enable_aqm_interrupts(ndev); +} + +void nitrox_config_pom_unit(struct nitrox_device *ndev) +{ + union pom_int_ena_w1s pom_int; + int i; + + /* enable pom interrupts */ + pom_int.value = 0; + pom_int.s.illegal_dport = 1; + nitrox_write_csr(ndev, POM_INT_ENA_W1S, pom_int.value); + + /* enable perf counters */ + for (i = 0; i < ndev->hw.se_cores; i++) + nitrox_write_csr(ndev, POM_PERF_CTL, BIT_ULL(i)); +} + +/** + * nitrox_config_rand_unit - enable NITROX random number unit + * @ndev: NITROX device + */ +void nitrox_config_rand_unit(struct nitrox_device *ndev) +{ + union efl_rnm_ctl_status efl_rnm_ctl; + u64 offset; + + offset = EFL_RNM_CTL_STATUS; + efl_rnm_ctl.value = nitrox_read_csr(ndev, offset); + efl_rnm_ctl.s.ent_en = 1; + efl_rnm_ctl.s.rng_en = 1; + nitrox_write_csr(ndev, offset, efl_rnm_ctl.value); +} + +void nitrox_config_efl_unit(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < NR_CLUSTERS; i++) { + union efl_core_int_ena_w1s efl_core_int; + u64 offset; + + /* EFL core interrupts */ + offset = EFL_CORE_INT_ENA_W1SX(i); + efl_core_int.value = 0; + efl_core_int.s.len_ovr = 1; + efl_core_int.s.d_left = 1; + efl_core_int.s.epci_decode_err = 1; + nitrox_write_csr(ndev, offset, efl_core_int.value); + + offset = EFL_CORE_VF_ERR_INT0_ENA_W1SX(i); + nitrox_write_csr(ndev, offset, (~0ULL)); + offset = EFL_CORE_VF_ERR_INT1_ENA_W1SX(i); + nitrox_write_csr(ndev, offset, (~0ULL)); + } +} + +void nitrox_config_bmi_unit(struct nitrox_device *ndev) +{ + union bmi_ctl bmi_ctl; + union bmi_int_ena_w1s bmi_int_ena; + u64 offset; + + /* no threshold limits for PCIe */ + offset = BMI_CTL; + bmi_ctl.value = nitrox_read_csr(ndev, offset); + bmi_ctl.s.max_pkt_len = 0xff; + bmi_ctl.s.nps_free_thrsh = 0xff; + bmi_ctl.s.nps_hdrq_thrsh = 0x7a; + nitrox_write_csr(ndev, offset, bmi_ctl.value); + + /* enable interrupts */ + offset = BMI_INT_ENA_W1S; + bmi_int_ena.value = 0; + bmi_int_ena.s.max_len_err_nps = 1; + bmi_int_ena.s.pkt_rcv_err_nps = 1; + bmi_int_ena.s.fpf_undrrn = 1; + nitrox_write_csr(ndev, offset, bmi_int_ena.value); +} + +void nitrox_config_bmo_unit(struct nitrox_device *ndev) +{ + union bmo_ctl2 bmo_ctl2; + u64 offset; + + /* no threshold limits for PCIe */ + offset = BMO_CTL2; + bmo_ctl2.value = nitrox_read_csr(ndev, offset); + bmo_ctl2.s.nps_slc_buf_thrsh = 0xff; + nitrox_write_csr(ndev, offset, bmo_ctl2.value); +} + +void invalidate_lbc(struct nitrox_device *ndev) +{ + union lbc_inval_ctl lbc_ctl; + union lbc_inval_status lbc_stat; + int max_retries = MAX_CSR_RETRIES; + u64 offset; + + /* invalidate LBC */ + offset = LBC_INVAL_CTL; + lbc_ctl.value = nitrox_read_csr(ndev, offset); + lbc_ctl.s.cam_inval_start = 1; + nitrox_write_csr(ndev, offset, lbc_ctl.value); + + offset = LBC_INVAL_STATUS; + do { + lbc_stat.value = nitrox_read_csr(ndev, offset); + if (lbc_stat.s.done) + break; + udelay(50); + } while (max_retries--); +} + +void nitrox_config_lbc_unit(struct nitrox_device *ndev) +{ + union lbc_int_ena_w1s lbc_int_ena; + u64 offset; + + invalidate_lbc(ndev); + + /* enable interrupts */ + offset = LBC_INT_ENA_W1S; + lbc_int_ena.value = 0; + lbc_int_ena.s.dma_rd_err = 1; + lbc_int_ena.s.over_fetch_err = 1; + lbc_int_ena.s.cam_inval_abort = 1; + lbc_int_ena.s.cam_hard_err = 1; + nitrox_write_csr(ndev, offset, lbc_int_ena.value); + + offset = LBC_PLM_VF1_64_INT_ENA_W1S; + nitrox_write_csr(ndev, offset, (~0ULL)); + offset = LBC_PLM_VF65_128_INT_ENA_W1S; + nitrox_write_csr(ndev, offset, (~0ULL)); + + offset = LBC_ELM_VF1_64_INT_ENA_W1S; + nitrox_write_csr(ndev, offset, (~0ULL)); + offset = LBC_ELM_VF65_128_INT_ENA_W1S; + nitrox_write_csr(ndev, offset, (~0ULL)); +} + +void config_nps_core_vfcfg_mode(struct nitrox_device *ndev, enum vf_mode mode) +{ + union nps_core_gbl_vfcfg vfcfg; + + vfcfg.value = nitrox_read_csr(ndev, NPS_CORE_GBL_VFCFG); + vfcfg.s.cfg = mode & 0x7; + + nitrox_write_csr(ndev, NPS_CORE_GBL_VFCFG, vfcfg.value); +} + +static const char *get_core_option(u8 se_cores, u8 ae_cores) +{ + const char *option = ""; + + if (ae_cores == AE_MAX_CORES) { + switch (se_cores) { + case SE_MAX_CORES: + option = "60"; + break; + case 40: + option = "60s"; + break; + } + } else if (ae_cores == (AE_MAX_CORES / 2)) { + option = "30"; + } else { + option = "60i"; + } + + return option; +} + +static const char *get_feature_option(u8 zip_cores, int core_freq) +{ + if (zip_cores == 0) + return ""; + else if (zip_cores < ZIP_MAX_CORES) + return "-C15"; + + if (core_freq >= 850) + return "-C45"; + else if (core_freq >= 750) + return "-C35"; + else if (core_freq >= 550) + return "-C25"; + + return ""; +} + +void nitrox_get_hwinfo(struct nitrox_device *ndev) +{ + union emu_fuse_map emu_fuse; + union rst_boot rst_boot; + union fus_dat1 fus_dat1; + unsigned char name[IFNAMSIZ * 2] = {}; + int i, dead_cores; + u64 offset; + + /* get core frequency */ + offset = RST_BOOT; + rst_boot.value = nitrox_read_csr(ndev, offset); + ndev->hw.freq = (rst_boot.pnr_mul + 3) * PLL_REF_CLK; + + for (i = 0; i < NR_CLUSTERS; i++) { + offset = EMU_FUSE_MAPX(i); + emu_fuse.value = nitrox_read_csr(ndev, offset); + if (emu_fuse.s.valid) { + dead_cores = hweight32(emu_fuse.s.ae_fuse); + ndev->hw.ae_cores += AE_CORES_PER_CLUSTER - dead_cores; + dead_cores = hweight16(emu_fuse.s.se_fuse); + ndev->hw.se_cores += SE_CORES_PER_CLUSTER - dead_cores; + } + } + /* find zip hardware availability */ + offset = FUS_DAT1; + fus_dat1.value = nitrox_read_csr(ndev, offset); + if (!fus_dat1.nozip) { + dead_cores = hweight8(fus_dat1.zip_info); + ndev->hw.zip_cores = ZIP_MAX_CORES - dead_cores; + } + + /* determine the partname + * CNN55--- + */ + snprintf(name, sizeof(name), "CNN55%s-%3dBG676%s-1.%u", + get_core_option(ndev->hw.se_cores, ndev->hw.ae_cores), + ndev->hw.freq, + get_feature_option(ndev->hw.zip_cores, ndev->hw.freq), + ndev->hw.revision_id); + + /* copy partname */ + strscpy(ndev->hw.partname, name); +} + +void enable_pf2vf_mbox_interrupts(struct nitrox_device *ndev) +{ + u64 value = ~0ULL; + u64 reg_addr; + + /* Mailbox interrupt low enable set register */ + reg_addr = NPS_PKT_MBOX_INT_LO_ENA_W1S; + nitrox_write_csr(ndev, reg_addr, value); + + /* Mailbox interrupt high enable set register */ + reg_addr = NPS_PKT_MBOX_INT_HI_ENA_W1S; + nitrox_write_csr(ndev, reg_addr, value); +} + +void disable_pf2vf_mbox_interrupts(struct nitrox_device *ndev) +{ + u64 value = ~0ULL; + u64 reg_addr; + + /* Mailbox interrupt low enable clear register */ + reg_addr = NPS_PKT_MBOX_INT_LO_ENA_W1C; + nitrox_write_csr(ndev, reg_addr, value); + + /* Mailbox interrupt high enable clear register */ + reg_addr = NPS_PKT_MBOX_INT_HI_ENA_W1C; + nitrox_write_csr(ndev, reg_addr, value); +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_hal.h b/drivers/crypto/cavium/nitrox/nitrox_hal.h new file mode 100644 index 000000000..48b0af039 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_hal.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NITROX_HAL_H +#define __NITROX_HAL_H + +#include "nitrox_dev.h" + +void nitrox_config_aqm_rings(struct nitrox_device *ndev); +void nitrox_config_aqm_unit(struct nitrox_device *ndev); +void nitrox_config_emu_unit(struct nitrox_device *ndev); +void nitrox_config_pkt_input_rings(struct nitrox_device *ndev); +void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev); +void nitrox_config_nps_core_unit(struct nitrox_device *ndev); +void nitrox_config_nps_pkt_unit(struct nitrox_device *ndev); +void nitrox_config_pom_unit(struct nitrox_device *ndev); +void nitrox_config_rand_unit(struct nitrox_device *ndev); +void nitrox_config_efl_unit(struct nitrox_device *ndev); +void nitrox_config_bmi_unit(struct nitrox_device *ndev); +void nitrox_config_bmo_unit(struct nitrox_device *ndev); +void nitrox_config_lbc_unit(struct nitrox_device *ndev); +void invalidate_lbc(struct nitrox_device *ndev); +void enable_aqm_ring(struct nitrox_device *ndev, int qno); +void enable_pkt_input_ring(struct nitrox_device *ndev, int ring); +void enable_pkt_solicit_port(struct nitrox_device *ndev, int port); +void config_nps_core_vfcfg_mode(struct nitrox_device *ndev, enum vf_mode mode); +void nitrox_get_hwinfo(struct nitrox_device *ndev); +void enable_pf2vf_mbox_interrupts(struct nitrox_device *ndev); +void disable_pf2vf_mbox_interrupts(struct nitrox_device *ndev); + +#endif /* __NITROX_HAL_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_isr.c b/drivers/crypto/cavium/nitrox/nitrox_isr.c new file mode 100644 index 000000000..cf09f7451 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_isr.c @@ -0,0 +1,458 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include + +#include "nitrox_dev.h" +#include "nitrox_csr.h" +#include "nitrox_common.h" +#include "nitrox_hal.h" +#include "nitrox_isr.h" +#include "nitrox_mbx.h" + +/* + * One vector for each type of ring + * - NPS packet ring, AQMQ ring and ZQMQ ring + */ +#define NR_RING_VECTORS 3 +#define NR_NON_RING_VECTORS 1 +/* base entry for packet ring/port */ +#define PKT_RING_MSIX_BASE 0 +#define NON_RING_MSIX_BASE 192 + +/** + * nps_pkt_slc_isr - IRQ handler for NPS solicit port + * @irq: irq number + * @data: argument + */ +static irqreturn_t nps_pkt_slc_isr(int irq, void *data) +{ + struct nitrox_q_vector *qvec = data; + union nps_pkt_slc_cnts slc_cnts; + struct nitrox_cmdq *cmdq = qvec->cmdq; + + slc_cnts.value = readq(cmdq->compl_cnt_csr_addr); + /* New packet on SLC output port */ + if (slc_cnts.s.slc_int) + tasklet_hi_schedule(&qvec->resp_tasklet); + + return IRQ_HANDLED; +} + +static void clear_nps_core_err_intr(struct nitrox_device *ndev) +{ + u64 value; + + /* Write 1 to clear */ + value = nitrox_read_csr(ndev, NPS_CORE_INT); + nitrox_write_csr(ndev, NPS_CORE_INT, value); + + dev_err_ratelimited(DEV(ndev), "NSP_CORE_INT 0x%016llx\n", value); +} + +static void clear_nps_pkt_err_intr(struct nitrox_device *ndev) +{ + union nps_pkt_int pkt_int; + unsigned long value, offset; + int i; + + pkt_int.value = nitrox_read_csr(ndev, NPS_PKT_INT); + dev_err_ratelimited(DEV(ndev), "NPS_PKT_INT 0x%016llx\n", + pkt_int.value); + + if (pkt_int.s.slc_err) { + offset = NPS_PKT_SLC_ERR_TYPE; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_SLC_ERR_TYPE 0x%016lx\n", value); + + offset = NPS_PKT_SLC_RERR_LO; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + /* enable the solicit ports */ + for_each_set_bit(i, &value, BITS_PER_LONG) + enable_pkt_solicit_port(ndev, i); + + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_SLC_RERR_LO 0x%016lx\n", value); + + offset = NPS_PKT_SLC_RERR_HI; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_SLC_RERR_HI 0x%016lx\n", value); + } + + if (pkt_int.s.in_err) { + offset = NPS_PKT_IN_ERR_TYPE; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_IN_ERR_TYPE 0x%016lx\n", value); + offset = NPS_PKT_IN_RERR_LO; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + /* enable the input ring */ + for_each_set_bit(i, &value, BITS_PER_LONG) + enable_pkt_input_ring(ndev, i); + + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_IN_RERR_LO 0x%016lx\n", value); + + offset = NPS_PKT_IN_RERR_HI; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_IN_RERR_HI 0x%016lx\n", value); + } +} + +static void clear_pom_err_intr(struct nitrox_device *ndev) +{ + u64 value; + + value = nitrox_read_csr(ndev, POM_INT); + nitrox_write_csr(ndev, POM_INT, value); + dev_err_ratelimited(DEV(ndev), "POM_INT 0x%016llx\n", value); +} + +static void clear_pem_err_intr(struct nitrox_device *ndev) +{ + u64 value; + + value = nitrox_read_csr(ndev, PEM0_INT); + nitrox_write_csr(ndev, PEM0_INT, value); + dev_err_ratelimited(DEV(ndev), "PEM(0)_INT 0x%016llx\n", value); +} + +static void clear_lbc_err_intr(struct nitrox_device *ndev) +{ + union lbc_int lbc_int; + u64 value, offset; + int i; + + lbc_int.value = nitrox_read_csr(ndev, LBC_INT); + dev_err_ratelimited(DEV(ndev), "LBC_INT 0x%016llx\n", lbc_int.value); + + if (lbc_int.s.dma_rd_err) { + for (i = 0; i < NR_CLUSTERS; i++) { + offset = EFL_CORE_VF_ERR_INT0X(i); + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + offset = EFL_CORE_VF_ERR_INT1X(i); + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + } + } + + if (lbc_int.s.cam_soft_err) { + dev_err_ratelimited(DEV(ndev), "CAM_SOFT_ERR, invalidating LBC\n"); + invalidate_lbc(ndev); + } + + if (lbc_int.s.pref_dat_len_mismatch_err) { + offset = LBC_PLM_VF1_64_INT; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + offset = LBC_PLM_VF65_128_INT; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + } + + if (lbc_int.s.rd_dat_len_mismatch_err) { + offset = LBC_ELM_VF1_64_INT; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + offset = LBC_ELM_VF65_128_INT; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + } + nitrox_write_csr(ndev, LBC_INT, lbc_int.value); +} + +static void clear_efl_err_intr(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < NR_CLUSTERS; i++) { + union efl_core_int core_int; + u64 value, offset; + + offset = EFL_CORE_INTX(i); + core_int.value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, core_int.value); + dev_err_ratelimited(DEV(ndev), "ELF_CORE(%d)_INT 0x%016llx\n", + i, core_int.value); + if (core_int.s.se_err) { + offset = EFL_CORE_SE_ERR_INTX(i); + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + } + } +} + +static void clear_bmi_err_intr(struct nitrox_device *ndev) +{ + u64 value; + + value = nitrox_read_csr(ndev, BMI_INT); + nitrox_write_csr(ndev, BMI_INT, value); + dev_err_ratelimited(DEV(ndev), "BMI_INT 0x%016llx\n", value); +} + +static void nps_core_int_tasklet(unsigned long data) +{ + struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data); + struct nitrox_device *ndev = qvec->ndev; + + /* if pf mode do queue recovery */ + if (ndev->mode == __NDEV_MODE_PF) { + } else { + /** + * if VF(s) enabled communicate the error information + * to VF(s) + */ + } +} + +/* + * nps_core_int_isr - interrupt handler for NITROX errors and + * mailbox communication + */ +static irqreturn_t nps_core_int_isr(int irq, void *data) +{ + struct nitrox_q_vector *qvec = data; + struct nitrox_device *ndev = qvec->ndev; + union nps_core_int_active core_int; + + core_int.value = nitrox_read_csr(ndev, NPS_CORE_INT_ACTIVE); + + if (core_int.s.nps_core) + clear_nps_core_err_intr(ndev); + + if (core_int.s.nps_pkt) + clear_nps_pkt_err_intr(ndev); + + if (core_int.s.pom) + clear_pom_err_intr(ndev); + + if (core_int.s.pem) + clear_pem_err_intr(ndev); + + if (core_int.s.lbc) + clear_lbc_err_intr(ndev); + + if (core_int.s.efl) + clear_efl_err_intr(ndev); + + if (core_int.s.bmi) + clear_bmi_err_intr(ndev); + + /* Mailbox interrupt */ + if (core_int.s.mbox) + nitrox_pf2vf_mbox_handler(ndev); + + /* If more work callback the ISR, set resend */ + core_int.s.resend = 1; + nitrox_write_csr(ndev, NPS_CORE_INT_ACTIVE, core_int.value); + + return IRQ_HANDLED; +} + +void nitrox_unregister_interrupts(struct nitrox_device *ndev) +{ + struct pci_dev *pdev = ndev->pdev; + int i; + + for (i = 0; i < ndev->num_vecs; i++) { + struct nitrox_q_vector *qvec; + int vec; + + qvec = ndev->qvec + i; + if (!qvec->valid) + continue; + + /* get the vector number */ + vec = pci_irq_vector(pdev, i); + irq_set_affinity_hint(vec, NULL); + free_irq(vec, qvec); + + tasklet_disable(&qvec->resp_tasklet); + tasklet_kill(&qvec->resp_tasklet); + qvec->valid = false; + } + kfree(ndev->qvec); + ndev->qvec = NULL; + pci_free_irq_vectors(pdev); +} + +int nitrox_register_interrupts(struct nitrox_device *ndev) +{ + struct pci_dev *pdev = ndev->pdev; + struct nitrox_q_vector *qvec; + int nr_vecs, vec, cpu; + int ret, i; + + /* + * PF MSI-X vectors + * + * Entry 0: NPS PKT ring 0 + * Entry 1: AQMQ ring 0 + * Entry 2: ZQM ring 0 + * Entry 3: NPS PKT ring 1 + * Entry 4: AQMQ ring 1 + * Entry 5: ZQM ring 1 + * .... + * Entry 192: NPS_CORE_INT_ACTIVE + */ + nr_vecs = pci_msix_vec_count(pdev); + if (nr_vecs < 0) { + dev_err(DEV(ndev), "Error in getting vec count %d\n", nr_vecs); + return nr_vecs; + } + + /* Enable MSI-X */ + ret = pci_alloc_irq_vectors(pdev, nr_vecs, nr_vecs, PCI_IRQ_MSIX); + if (ret < 0) { + dev_err(DEV(ndev), "msix vectors %d alloc failed\n", nr_vecs); + return ret; + } + ndev->num_vecs = nr_vecs; + + ndev->qvec = kzalloc_objs(*qvec, nr_vecs); + if (!ndev->qvec) { + pci_free_irq_vectors(pdev); + return -ENOMEM; + } + + /* request irqs for packet rings/ports */ + for (i = PKT_RING_MSIX_BASE; i < (nr_vecs - 1); i += NR_RING_VECTORS) { + qvec = &ndev->qvec[i]; + + qvec->ring = i / NR_RING_VECTORS; + if (qvec->ring >= ndev->nr_queues) + break; + + qvec->cmdq = &ndev->pkt_inq[qvec->ring]; + snprintf(qvec->name, IRQ_NAMESZ, "nitrox-pkt%d", qvec->ring); + /* get the vector number */ + vec = pci_irq_vector(pdev, i); + ret = request_irq(vec, nps_pkt_slc_isr, 0, qvec->name, qvec); + if (ret) { + dev_err(DEV(ndev), "irq failed for pkt ring/port%d\n", + qvec->ring); + goto irq_fail; + } + cpu = qvec->ring % num_online_cpus(); + irq_set_affinity_hint(vec, get_cpu_mask(cpu)); + + tasklet_init(&qvec->resp_tasklet, pkt_slc_resp_tasklet, + (unsigned long)qvec); + qvec->valid = true; + } + + /* request irqs for non ring vectors */ + i = NON_RING_MSIX_BASE; + qvec = &ndev->qvec[i]; + qvec->ndev = ndev; + + snprintf(qvec->name, IRQ_NAMESZ, "nitrox-core-int%d", i); + /* get the vector number */ + vec = pci_irq_vector(pdev, i); + ret = request_irq(vec, nps_core_int_isr, 0, qvec->name, qvec); + if (ret) { + dev_err(DEV(ndev), "irq failed for nitrox-core-int%d\n", i); + goto irq_fail; + } + cpu = num_online_cpus(); + irq_set_affinity_hint(vec, get_cpu_mask(cpu)); + + tasklet_init(&qvec->resp_tasklet, nps_core_int_tasklet, + (unsigned long)qvec); + qvec->valid = true; + + return 0; + +irq_fail: + nitrox_unregister_interrupts(ndev); + return ret; +} + +void nitrox_sriov_unregister_interrupts(struct nitrox_device *ndev) +{ + struct pci_dev *pdev = ndev->pdev; + int i; + + for (i = 0; i < ndev->num_vecs; i++) { + struct nitrox_q_vector *qvec; + int vec; + + qvec = ndev->qvec + i; + if (!qvec->valid) + continue; + + vec = ndev->iov.msix.vector; + irq_set_affinity_hint(vec, NULL); + free_irq(vec, qvec); + + tasklet_disable(&qvec->resp_tasklet); + tasklet_kill(&qvec->resp_tasklet); + qvec->valid = false; + } + kfree(ndev->qvec); + ndev->qvec = NULL; + pci_disable_msix(pdev); +} + +int nitrox_sriov_register_interupts(struct nitrox_device *ndev) +{ + struct pci_dev *pdev = ndev->pdev; + struct nitrox_q_vector *qvec; + int vec, cpu; + int ret; + + /** + * only non ring vectors i.e Entry 192 is available + * for PF in SR-IOV mode. + */ + ndev->iov.msix.entry = NON_RING_MSIX_BASE; + ret = pci_enable_msix_exact(pdev, &ndev->iov.msix, NR_NON_RING_VECTORS); + if (ret) { + dev_err(DEV(ndev), "failed to allocate nps-core-int%d\n", + NON_RING_MSIX_BASE); + return ret; + } + + qvec = kzalloc_objs(*qvec, NR_NON_RING_VECTORS); + if (!qvec) { + pci_disable_msix(pdev); + return -ENOMEM; + } + qvec->ndev = ndev; + + ndev->qvec = qvec; + ndev->num_vecs = NR_NON_RING_VECTORS; + snprintf(qvec->name, IRQ_NAMESZ, "nitrox-core-int%d", + NON_RING_MSIX_BASE); + + vec = ndev->iov.msix.vector; + ret = request_irq(vec, nps_core_int_isr, 0, qvec->name, qvec); + if (ret) { + dev_err(DEV(ndev), "irq failed for nitrox-core-int%d\n", + NON_RING_MSIX_BASE); + goto iov_irq_fail; + } + cpu = num_online_cpus(); + irq_set_affinity_hint(vec, get_cpu_mask(cpu)); + + tasklet_init(&qvec->resp_tasklet, nps_core_int_tasklet, + (unsigned long)qvec); + qvec->valid = true; + + return 0; + +iov_irq_fail: + nitrox_sriov_unregister_interrupts(ndev); + return ret; +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_isr.h b/drivers/crypto/cavium/nitrox/nitrox_isr.h new file mode 100644 index 000000000..2bb123cd2 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_isr.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NITROX_ISR_H +#define __NITROX_ISR_H + +#include "nitrox_dev.h" + +int nitrox_register_interrupts(struct nitrox_device *ndev); +void nitrox_unregister_interrupts(struct nitrox_device *ndev); +int nitrox_sriov_register_interupts(struct nitrox_device *ndev); +void nitrox_sriov_unregister_interrupts(struct nitrox_device *ndev); + +#ifdef CONFIG_PCI_IOV +int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs); +#else +static inline int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs) +{ + return 0; +} +#endif + +#endif /* __NITROX_ISR_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_lib.c b/drivers/crypto/cavium/nitrox/nitrox_lib.c new file mode 100644 index 000000000..d35303857 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_lib.c @@ -0,0 +1,300 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nitrox_dev.h" +#include "nitrox_common.h" +#include "nitrox_req.h" +#include "nitrox_csr.h" + +#define CRYPTO_CTX_SIZE 256 + +/* packet input ring alignments */ +#define PKTIN_Q_ALIGN_BYTES 16 +/* AQM Queue input alignments */ +#define AQM_Q_ALIGN_BYTES 32 + +static int nitrox_cmdq_init(struct nitrox_cmdq *cmdq, int align_bytes) +{ + struct nitrox_device *ndev = cmdq->ndev; + + cmdq->qsize = (ndev->qlen * cmdq->instr_size) + align_bytes; + cmdq->unalign_base = dma_alloc_coherent(DEV(ndev), cmdq->qsize, + &cmdq->unalign_dma, + GFP_KERNEL); + if (!cmdq->unalign_base) + return -ENOMEM; + + cmdq->dma = PTR_ALIGN(cmdq->unalign_dma, align_bytes); + cmdq->base = cmdq->unalign_base + (cmdq->dma - cmdq->unalign_dma); + cmdq->write_idx = 0; + + spin_lock_init(&cmdq->cmd_qlock); + spin_lock_init(&cmdq->resp_qlock); + spin_lock_init(&cmdq->backlog_qlock); + + INIT_LIST_HEAD(&cmdq->response_head); + INIT_LIST_HEAD(&cmdq->backlog_head); + INIT_WORK(&cmdq->backlog_qflush, backlog_qflush_work); + + atomic_set(&cmdq->pending_count, 0); + atomic_set(&cmdq->backlog_count, 0); + return 0; +} + +static void nitrox_cmdq_reset(struct nitrox_cmdq *cmdq) +{ + cmdq->write_idx = 0; + atomic_set(&cmdq->pending_count, 0); + atomic_set(&cmdq->backlog_count, 0); +} + +static void nitrox_cmdq_cleanup(struct nitrox_cmdq *cmdq) +{ + struct nitrox_device *ndev; + + if (!cmdq) + return; + + if (!cmdq->unalign_base) + return; + + ndev = cmdq->ndev; + cancel_work_sync(&cmdq->backlog_qflush); + + dma_free_coherent(DEV(ndev), cmdq->qsize, + cmdq->unalign_base, cmdq->unalign_dma); + nitrox_cmdq_reset(cmdq); + + cmdq->dbell_csr_addr = NULL; + cmdq->compl_cnt_csr_addr = NULL; + cmdq->unalign_base = NULL; + cmdq->base = NULL; + cmdq->unalign_dma = 0; + cmdq->dma = 0; + cmdq->qsize = 0; + cmdq->instr_size = 0; +} + +static void nitrox_free_aqm_queues(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < ndev->nr_queues; i++) { + nitrox_cmdq_cleanup(ndev->aqmq[i]); + kfree_sensitive(ndev->aqmq[i]); + ndev->aqmq[i] = NULL; + } +} + +static int nitrox_alloc_aqm_queues(struct nitrox_device *ndev) +{ + int i, err; + + for (i = 0; i < ndev->nr_queues; i++) { + struct nitrox_cmdq *cmdq; + u64 offset; + + cmdq = kzalloc_node(sizeof(*cmdq), GFP_KERNEL, ndev->node); + if (!cmdq) { + err = -ENOMEM; + goto aqmq_fail; + } + + cmdq->ndev = ndev; + cmdq->qno = i; + cmdq->instr_size = sizeof(struct aqmq_command_s); + + /* AQM Queue Doorbell Counter Register Address */ + offset = AQMQ_DRBLX(i); + cmdq->dbell_csr_addr = NITROX_CSR_ADDR(ndev, offset); + /* AQM Queue Commands Completed Count Register Address */ + offset = AQMQ_CMD_CNTX(i); + cmdq->compl_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset); + + err = nitrox_cmdq_init(cmdq, AQM_Q_ALIGN_BYTES); + if (err) { + kfree_sensitive(cmdq); + goto aqmq_fail; + } + ndev->aqmq[i] = cmdq; + } + + return 0; + +aqmq_fail: + nitrox_free_aqm_queues(ndev); + return err; +} + +static void nitrox_free_pktin_queues(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < ndev->nr_queues; i++) { + struct nitrox_cmdq *cmdq = &ndev->pkt_inq[i]; + + nitrox_cmdq_cleanup(cmdq); + } + kfree(ndev->pkt_inq); + ndev->pkt_inq = NULL; +} + +static int nitrox_alloc_pktin_queues(struct nitrox_device *ndev) +{ + int i, err; + + ndev->pkt_inq = kcalloc_node(ndev->nr_queues, + sizeof(struct nitrox_cmdq), + GFP_KERNEL, ndev->node); + if (!ndev->pkt_inq) + return -ENOMEM; + + for (i = 0; i < ndev->nr_queues; i++) { + struct nitrox_cmdq *cmdq; + u64 offset; + + cmdq = &ndev->pkt_inq[i]; + cmdq->ndev = ndev; + cmdq->qno = i; + cmdq->instr_size = sizeof(struct nps_pkt_instr); + + /* packet input ring doorbell address */ + offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(i); + cmdq->dbell_csr_addr = NITROX_CSR_ADDR(ndev, offset); + /* packet solicit port completion count address */ + offset = NPS_PKT_SLC_CNTSX(i); + cmdq->compl_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset); + + err = nitrox_cmdq_init(cmdq, PKTIN_Q_ALIGN_BYTES); + if (err) + goto pktq_fail; + } + return 0; + +pktq_fail: + nitrox_free_pktin_queues(ndev); + return err; +} + +static int create_crypto_dma_pool(struct nitrox_device *ndev) +{ + size_t size; + + /* Crypto context pool, 16 byte aligned */ + size = CRYPTO_CTX_SIZE + sizeof(struct ctx_hdr); + ndev->ctx_pool = dma_pool_create("nitrox-context", + DEV(ndev), size, 16, 0); + if (!ndev->ctx_pool) + return -ENOMEM; + + return 0; +} + +static void destroy_crypto_dma_pool(struct nitrox_device *ndev) +{ + if (!ndev->ctx_pool) + return; + + dma_pool_destroy(ndev->ctx_pool); + ndev->ctx_pool = NULL; +} + +/* + * crypto_alloc_context - Allocate crypto context from pool + * @ndev: NITROX Device + */ +void *crypto_alloc_context(struct nitrox_device *ndev) +{ + struct ctx_hdr *ctx; + struct crypto_ctx_hdr *chdr; + void *vaddr; + dma_addr_t dma; + + chdr = kmalloc_obj(*chdr); + if (!chdr) + return NULL; + + vaddr = dma_pool_zalloc(ndev->ctx_pool, GFP_KERNEL, &dma); + if (!vaddr) { + kfree(chdr); + return NULL; + } + + /* fill meta data */ + ctx = vaddr; + ctx->pool = ndev->ctx_pool; + ctx->dma = dma; + ctx->ctx_dma = dma + sizeof(struct ctx_hdr); + + chdr->pool = ndev->ctx_pool; + chdr->dma = dma; + chdr->vaddr = vaddr; + + return chdr; +} + +/** + * crypto_free_context - Free crypto context to pool + * @ctx: context to free + */ +void crypto_free_context(void *ctx) +{ + struct crypto_ctx_hdr *ctxp; + + if (!ctx) + return; + + ctxp = ctx; + dma_pool_free(ctxp->pool, ctxp->vaddr, ctxp->dma); + kfree(ctxp); +} + +/** + * nitrox_common_sw_init - allocate software resources. + * @ndev: NITROX device + * + * Allocates crypto context pools and command queues etc. + * + * Return: 0 on success, or a negative error code on error. + */ +int nitrox_common_sw_init(struct nitrox_device *ndev) +{ + int err = 0; + + /* per device crypto context pool */ + err = create_crypto_dma_pool(ndev); + if (err) + return err; + + err = nitrox_alloc_pktin_queues(ndev); + if (err) + destroy_crypto_dma_pool(ndev); + + err = nitrox_alloc_aqm_queues(ndev); + if (err) { + nitrox_free_pktin_queues(ndev); + destroy_crypto_dma_pool(ndev); + } + + return err; +} + +/** + * nitrox_common_sw_cleanup - free software resources. + * @ndev: NITROX device + */ +void nitrox_common_sw_cleanup(struct nitrox_device *ndev) +{ + nitrox_free_aqm_queues(ndev); + nitrox_free_pktin_queues(ndev); + destroy_crypto_dma_pool(ndev); +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c new file mode 100644 index 000000000..e474c84d8 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_main.c @@ -0,0 +1,582 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include +#include +#include + +#include "nitrox_dev.h" +#include "nitrox_common.h" +#include "nitrox_csr.h" +#include "nitrox_hal.h" +#include "nitrox_isr.h" +#include "nitrox_debugfs.h" + +#define CNN55XX_DEV_ID 0x12 +#define UCODE_HLEN 48 +#define DEFAULT_SE_GROUP 0 +#define DEFAULT_AE_GROUP 0 + +#define DRIVER_VERSION "1.2" +#define CNN55XX_UCD_BLOCK_SIZE 32768 +#define CNN55XX_MAX_UCODE_SIZE (CNN55XX_UCD_BLOCK_SIZE * 2) +#define FW_DIR "cavium/" +/* SE microcode */ +#define SE_FW FW_DIR "cnn55xx_se.fw" +/* AE microcode */ +#define AE_FW FW_DIR "cnn55xx_ae.fw" + +static const char nitrox_driver_name[] = "CNN55XX"; + +static LIST_HEAD(ndevlist); +static DEFINE_MUTEX(devlist_lock); +static unsigned int num_devices; + +/* + * nitrox_pci_tbl - PCI Device ID Table + */ +static const struct pci_device_id nitrox_pci_tbl[] = { + { PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID) }, + /* required last entry */ + { } +}; +MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl); + +static unsigned int qlen = DEFAULT_CMD_QLEN; +module_param(qlen, uint, 0644); +MODULE_PARM_DESC(qlen, "Command queue length - default 2048"); + +/** + * struct ucode - Firmware Header + * @id: microcode ID + * @version: firmware version + * @code_size: code section size + * @raz: alignment + * @code: code section + */ +struct ucode { + u8 id; + char version[VERSION_LEN - 1]; + __be32 code_size; + u8 raz[12]; + u64 code[]; +}; + +/* + * write_to_ucd_unit - Write Firmware to NITROX UCD unit + */ +static void write_to_ucd_unit(struct nitrox_device *ndev, u32 ucode_size, + u64 *ucode_data, int block_num) +{ + u32 code_size; + u64 offset, data; + int i = 0; + + /* + * UCD structure + * + * ------------- + * | BLK 7 | + * ------------- + * | BLK 6 | + * ------------- + * | ... | + * ------------- + * | BLK 0 | + * ------------- + * Total of 8 blocks, each size 32KB + */ + + /* set the block number */ + offset = UCD_UCODE_LOAD_BLOCK_NUM; + nitrox_write_csr(ndev, offset, block_num); + + code_size = roundup(ucode_size, 16); + while (code_size) { + data = ucode_data[i]; + /* write 8 bytes at a time */ + offset = UCD_UCODE_LOAD_IDX_DATAX(i); + nitrox_write_csr(ndev, offset, data); + code_size -= 8; + i++; + } + + usleep_range(300, 400); +} + +static int nitrox_load_fw(struct nitrox_device *ndev) +{ + const struct firmware *fw; + const char *fw_name; + struct ucode *ucode; + u64 *ucode_data; + u64 offset; + union ucd_core_eid_ucode_block_num core_2_eid_val; + union aqm_grp_execmsk_lo aqm_grp_execmask_lo; + union aqm_grp_execmsk_hi aqm_grp_execmask_hi; + u32 ucode_size; + int ret, i = 0; + + fw_name = SE_FW; + dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name); + + ret = request_firmware(&fw, fw_name, DEV(ndev)); + if (ret < 0) { + dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name); + return ret; + } + + ucode = (struct ucode *)fw->data; + + ucode_size = be32_to_cpu(ucode->code_size) * 2; + if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) { + dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n", + ucode_size, fw_name); + release_firmware(fw); + return -EINVAL; + } + ucode_data = ucode->code; + + /* copy the firmware version */ + memcpy(&ndev->hw.fw_name[0][0], ucode->version, (VERSION_LEN - 2)); + ndev->hw.fw_name[0][VERSION_LEN - 1] = '\0'; + + /* Load SE Firmware on UCD Block 0 */ + write_to_ucd_unit(ndev, ucode_size, ucode_data, 0); + + release_firmware(fw); + + /* put all SE cores in DEFAULT_SE_GROUP */ + offset = POM_GRP_EXECMASKX(DEFAULT_SE_GROUP); + nitrox_write_csr(ndev, offset, (~0ULL)); + + /* write block number and firmware length + * bit:<2:0> block number + * bit:3 is set SE uses 32KB microcode + * bit:3 is clear SE uses 64KB microcode + */ + core_2_eid_val.value = 0ULL; + core_2_eid_val.ucode_blk = 0; + if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE) + core_2_eid_val.ucode_len = 1; + else + core_2_eid_val.ucode_len = 0; + + for (i = 0; i < ndev->hw.se_cores; i++) { + offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i); + nitrox_write_csr(ndev, offset, core_2_eid_val.value); + } + + + fw_name = AE_FW; + dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name); + + ret = request_firmware(&fw, fw_name, DEV(ndev)); + if (ret < 0) { + dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name); + return ret; + } + + ucode = (struct ucode *)fw->data; + + ucode_size = be32_to_cpu(ucode->code_size) * 2; + if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) { + dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n", + ucode_size, fw_name); + release_firmware(fw); + return -EINVAL; + } + ucode_data = ucode->code; + + /* copy the firmware version */ + memcpy(&ndev->hw.fw_name[1][0], ucode->version, (VERSION_LEN - 2)); + ndev->hw.fw_name[1][VERSION_LEN - 1] = '\0'; + + /* Load AE Firmware on UCD Block 2 */ + write_to_ucd_unit(ndev, ucode_size, ucode_data, 2); + + release_firmware(fw); + + /* put all AE cores in DEFAULT_AE_GROUP */ + offset = AQM_GRP_EXECMSK_LOX(DEFAULT_AE_GROUP); + aqm_grp_execmask_lo.exec_0_to_39 = 0xFFFFFFFFFFULL; + nitrox_write_csr(ndev, offset, aqm_grp_execmask_lo.value); + offset = AQM_GRP_EXECMSK_HIX(DEFAULT_AE_GROUP); + aqm_grp_execmask_hi.exec_40_to_79 = 0xFFFFFFFFFFULL; + nitrox_write_csr(ndev, offset, aqm_grp_execmask_hi.value); + + /* write block number and firmware length + * bit:<2:0> block number + * bit:3 is set AE uses 32KB microcode + * bit:3 is clear AE uses 64KB microcode + */ + core_2_eid_val.value = 0ULL; + core_2_eid_val.ucode_blk = 2; + if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE) + core_2_eid_val.ucode_len = 1; + else + core_2_eid_val.ucode_len = 0; + + for (i = 0; i < ndev->hw.ae_cores; i++) { + offset = UCD_AE_EID_UCODE_BLOCK_NUMX(i); + nitrox_write_csr(ndev, offset, core_2_eid_val.value); + } + + return 0; +} + +/** + * nitrox_add_to_devlist - add NITROX device to global device list + * @ndev: NITROX device + */ +static int nitrox_add_to_devlist(struct nitrox_device *ndev) +{ + struct nitrox_device *dev; + int ret = 0; + + INIT_LIST_HEAD(&ndev->list); + refcount_set(&ndev->refcnt, 1); + + mutex_lock(&devlist_lock); + list_for_each_entry(dev, &ndevlist, list) { + if (dev == ndev) { + ret = -EEXIST; + goto unlock; + } + } + ndev->idx = num_devices++; + list_add_tail(&ndev->list, &ndevlist); +unlock: + mutex_unlock(&devlist_lock); + return ret; +} + +/** + * nitrox_remove_from_devlist - remove NITROX device from + * global device list + * @ndev: NITROX device + */ +static void nitrox_remove_from_devlist(struct nitrox_device *ndev) +{ + mutex_lock(&devlist_lock); + list_del(&ndev->list); + num_devices--; + mutex_unlock(&devlist_lock); +} + +struct nitrox_device *nitrox_get_first_device(void) +{ + struct nitrox_device *ndev = NULL, *iter; + + mutex_lock(&devlist_lock); + list_for_each_entry(iter, &ndevlist, list) { + if (nitrox_ready(iter)) { + ndev = iter; + break; + } + } + mutex_unlock(&devlist_lock); + if (!ndev) + return NULL; + + refcount_inc(&ndev->refcnt); + /* barrier to sync with other cpus */ + smp_mb__after_atomic(); + return ndev; +} + +void nitrox_put_device(struct nitrox_device *ndev) +{ + if (!ndev) + return; + + refcount_dec(&ndev->refcnt); + /* barrier to sync with other cpus */ + smp_mb__after_atomic(); +} + +static int nitrox_device_flr(struct pci_dev *pdev) +{ + int pos = 0; + + pos = pci_save_state(pdev); + if (pos) { + dev_err(&pdev->dev, "Failed to save pci state\n"); + return -ENOMEM; + } + + pcie_reset_flr(pdev, PCI_RESET_DO_RESET); + + pci_restore_state(pdev); + + return 0; +} + +static int nitrox_pf_sw_init(struct nitrox_device *ndev) +{ + int err; + + err = nitrox_common_sw_init(ndev); + if (err) + return err; + + err = nitrox_register_interrupts(ndev); + if (err) + nitrox_common_sw_cleanup(ndev); + + return err; +} + +static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev) +{ + nitrox_unregister_interrupts(ndev); + nitrox_common_sw_cleanup(ndev); +} + +/** + * nitrox_bist_check - Check NITROX BIST registers status + * @ndev: NITROX device + */ +static int nitrox_bist_check(struct nitrox_device *ndev) +{ + u64 value = 0; + int i; + + for (i = 0; i < NR_CLUSTERS; i++) { + value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i)); + value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i)); + } + value += nitrox_read_csr(ndev, UCD_BIST_STATUS); + value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG); + value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG); + value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG); + value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG); + value += nitrox_read_csr(ndev, POM_BIST_REG); + value += nitrox_read_csr(ndev, BMI_BIST_REG); + value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT); + value += nitrox_read_csr(ndev, BMO_BIST_REG); + value += nitrox_read_csr(ndev, LBC_BIST_STATUS); + value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0)); + if (value) + return -EIO; + return 0; +} + +static int nitrox_pf_hw_init(struct nitrox_device *ndev) +{ + int err; + + err = nitrox_bist_check(ndev); + if (err) { + dev_err(&ndev->pdev->dev, "BIST check failed\n"); + return err; + } + /* get cores information */ + nitrox_get_hwinfo(ndev); + + nitrox_config_nps_core_unit(ndev); + nitrox_config_aqm_unit(ndev); + nitrox_config_nps_pkt_unit(ndev); + nitrox_config_pom_unit(ndev); + nitrox_config_efl_unit(ndev); + /* configure IO units */ + nitrox_config_bmi_unit(ndev); + nitrox_config_bmo_unit(ndev); + /* configure Local Buffer Cache */ + nitrox_config_lbc_unit(ndev); + nitrox_config_rand_unit(ndev); + + /* load firmware on cores */ + err = nitrox_load_fw(ndev); + if (err) + return err; + + nitrox_config_emu_unit(ndev); + + return 0; +} + +/** + * nitrox_probe - NITROX Initialization function. + * @pdev: PCI device information struct + * @id: entry in nitrox_pci_tbl + * + * Return: 0, if the driver is bound to the device, or + * a negative error if there is failure. + */ +static int nitrox_probe(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + struct nitrox_device *ndev; + int err; + + dev_info_once(&pdev->dev, "%s driver version %s\n", + nitrox_driver_name, DRIVER_VERSION); + + err = pci_enable_device_mem(pdev); + if (err) + return err; + + /* do FLR */ + err = nitrox_device_flr(pdev); + if (err) { + dev_err(&pdev->dev, "FLR failed\n"); + goto flr_fail; + } + + if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { + dev_dbg(&pdev->dev, "DMA to 64-BIT address\n"); + } else { + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (err) { + dev_err(&pdev->dev, "DMA configuration failed\n"); + goto flr_fail; + } + } + + err = pci_request_mem_regions(pdev, nitrox_driver_name); + if (err) + goto flr_fail; + pci_set_master(pdev); + + ndev = kzalloc_obj(*ndev); + if (!ndev) { + err = -ENOMEM; + goto ndev_fail; + } + + pci_set_drvdata(pdev, ndev); + ndev->pdev = pdev; + + /* add to device list */ + nitrox_add_to_devlist(ndev); + + ndev->hw.vendor_id = pdev->vendor; + ndev->hw.device_id = pdev->device; + ndev->hw.revision_id = pdev->revision; + /* command timeout in jiffies */ + ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT); + ndev->node = dev_to_node(&pdev->dev); + if (ndev->node == NUMA_NO_NODE) + ndev->node = 0; + + ndev->bar_addr = ioremap(pci_resource_start(pdev, 0), + pci_resource_len(pdev, 0)); + if (!ndev->bar_addr) { + err = -EIO; + goto ioremap_err; + } + /* allocate command queus based on cpus, max queues are 64 */ + ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus()); + ndev->qlen = qlen; + + err = nitrox_pf_sw_init(ndev); + if (err) + goto pf_sw_fail; + + err = nitrox_pf_hw_init(ndev); + if (err) + goto pf_hw_fail; + + nitrox_debugfs_init(ndev); + + /* clear the statistics */ + atomic64_set(&ndev->stats.posted, 0); + atomic64_set(&ndev->stats.completed, 0); + atomic64_set(&ndev->stats.dropped, 0); + + atomic_set(&ndev->state, __NDEV_READY); + /* barrier to sync with other cpus */ + smp_mb__after_atomic(); + + err = nitrox_crypto_register(); + if (err) + goto crypto_fail; + + return 0; + +crypto_fail: + nitrox_debugfs_exit(ndev); + atomic_set(&ndev->state, __NDEV_NOT_READY); + /* barrier to sync with other cpus */ + smp_mb__after_atomic(); +pf_hw_fail: + nitrox_pf_sw_cleanup(ndev); +pf_sw_fail: + iounmap(ndev->bar_addr); +ioremap_err: + nitrox_remove_from_devlist(ndev); + kfree(ndev); + pci_set_drvdata(pdev, NULL); +ndev_fail: + pci_release_mem_regions(pdev); +flr_fail: + pci_disable_device(pdev); + return err; +} + +/** + * nitrox_remove - Unbind the driver from the device. + * @pdev: PCI device information struct + */ +static void nitrox_remove(struct pci_dev *pdev) +{ + struct nitrox_device *ndev = pci_get_drvdata(pdev); + + if (!ndev) + return; + + if (!refcount_dec_and_test(&ndev->refcnt)) { + dev_err(DEV(ndev), "Device refcnt not zero (%d)\n", + refcount_read(&ndev->refcnt)); + return; + } + + dev_info(DEV(ndev), "Removing Device %x:%x\n", + ndev->hw.vendor_id, ndev->hw.device_id); + + atomic_set(&ndev->state, __NDEV_NOT_READY); + /* barrier to sync with other cpus */ + smp_mb__after_atomic(); + + nitrox_remove_from_devlist(ndev); + + /* disable SR-IOV */ + nitrox_sriov_configure(pdev, 0); + nitrox_crypto_unregister(); + nitrox_debugfs_exit(ndev); + nitrox_pf_sw_cleanup(ndev); + + iounmap(ndev->bar_addr); + kfree(ndev); + + pci_set_drvdata(pdev, NULL); + pci_release_mem_regions(pdev); + pci_disable_device(pdev); +} + +static void nitrox_shutdown(struct pci_dev *pdev) +{ + pci_set_drvdata(pdev, NULL); + pci_release_mem_regions(pdev); + pci_disable_device(pdev); +} + +static struct pci_driver nitrox_driver = { + .name = nitrox_driver_name, + .id_table = nitrox_pci_tbl, + .probe = nitrox_probe, + .remove = nitrox_remove, + .shutdown = nitrox_shutdown, + .sriov_configure = nitrox_sriov_configure, +}; + +module_pci_driver(nitrox_driver); + +MODULE_AUTHOR("Srikanth Jampala "); +MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " "); +MODULE_LICENSE("GPL"); +MODULE_VERSION(DRIVER_VERSION); +MODULE_FIRMWARE(SE_FW); diff --git a/drivers/crypto/cavium/nitrox/nitrox_mbx.c b/drivers/crypto/cavium/nitrox/nitrox_mbx.c new file mode 100644 index 000000000..944d8b882 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_mbx.c @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +#include "nitrox_csr.h" +#include "nitrox_hal.h" +#include "nitrox_dev.h" +#include "nitrox_mbx.h" + +#define RING_TO_VFNO(_x, _y) ((_x) / (_y)) + +/* + * mbx_msg_type - Mailbox message types + */ +enum mbx_msg_type { + MBX_MSG_TYPE_NOP, + MBX_MSG_TYPE_REQ, + MBX_MSG_TYPE_ACK, + MBX_MSG_TYPE_NACK, +}; + +/* + * mbx_msg_opcode - Mailbox message opcodes + */ +enum mbx_msg_opcode { + MSG_OP_VF_MODE = 1, + MSG_OP_VF_UP, + MSG_OP_VF_DOWN, + MSG_OP_CHIPID_VFID, + MSG_OP_MCODE_INFO = 11, +}; + +struct pf2vf_work { + struct nitrox_vfdev *vfdev; + struct nitrox_device *ndev; + struct work_struct pf2vf_resp; +}; + +static inline u64 pf2vf_read_mbox(struct nitrox_device *ndev, int ring) +{ + u64 reg_addr; + + reg_addr = NPS_PKT_MBOX_VF_PF_PFDATAX(ring); + return nitrox_read_csr(ndev, reg_addr); +} + +static inline void pf2vf_write_mbox(struct nitrox_device *ndev, u64 value, + int ring) +{ + u64 reg_addr; + + reg_addr = NPS_PKT_MBOX_PF_VF_PFDATAX(ring); + nitrox_write_csr(ndev, reg_addr, value); +} + +static void pf2vf_send_response(struct nitrox_device *ndev, + struct nitrox_vfdev *vfdev) +{ + union mbox_msg msg; + + msg.value = vfdev->msg.value; + + switch (vfdev->msg.opcode) { + case MSG_OP_VF_MODE: + msg.data = ndev->mode; + break; + case MSG_OP_VF_UP: + vfdev->nr_queues = vfdev->msg.data; + atomic_set(&vfdev->state, __NDEV_READY); + break; + case MSG_OP_CHIPID_VFID: + msg.id.chipid = ndev->idx; + msg.id.vfid = vfdev->vfno; + break; + case MSG_OP_VF_DOWN: + vfdev->nr_queues = 0; + atomic_set(&vfdev->state, __NDEV_NOT_READY); + break; + case MSG_OP_MCODE_INFO: + msg.data = 0; + msg.mcode_info.count = 2; + msg.mcode_info.info = MCODE_TYPE_SE_SSL | (MCODE_TYPE_AE << 5); + msg.mcode_info.next_se_grp = 1; + msg.mcode_info.next_ae_grp = 1; + break; + default: + msg.type = MBX_MSG_TYPE_NOP; + break; + } + + if (msg.type == MBX_MSG_TYPE_NOP) + return; + + /* send ACK to VF */ + msg.type = MBX_MSG_TYPE_ACK; + pf2vf_write_mbox(ndev, msg.value, vfdev->ring); + + vfdev->msg.value = 0; + atomic64_inc(&vfdev->mbx_resp); +} + +static void pf2vf_resp_handler(struct work_struct *work) +{ + struct pf2vf_work *pf2vf_resp = container_of(work, struct pf2vf_work, + pf2vf_resp); + struct nitrox_vfdev *vfdev = pf2vf_resp->vfdev; + struct nitrox_device *ndev = pf2vf_resp->ndev; + + switch (vfdev->msg.type) { + case MBX_MSG_TYPE_REQ: + /* process the request from VF */ + pf2vf_send_response(ndev, vfdev); + break; + case MBX_MSG_TYPE_ACK: + case MBX_MSG_TYPE_NACK: + break; + } + + kfree(pf2vf_resp); +} + +void nitrox_pf2vf_mbox_handler(struct nitrox_device *ndev) +{ + DECLARE_BITMAP(csr, BITS_PER_TYPE(u64)); + struct nitrox_vfdev *vfdev; + struct pf2vf_work *pfwork; + u64 value, reg_addr; + u32 i; + int vfno; + + /* loop for VF(0..63) */ + reg_addr = NPS_PKT_MBOX_INT_LO; + value = nitrox_read_csr(ndev, reg_addr); + bitmap_from_u64(csr, value); + for_each_set_bit(i, csr, BITS_PER_TYPE(csr)) { + /* get the vfno from ring */ + vfno = RING_TO_VFNO(i, ndev->iov.max_vf_queues); + vfdev = ndev->iov.vfdev + vfno; + vfdev->ring = i; + /* fill the vf mailbox data */ + vfdev->msg.value = pf2vf_read_mbox(ndev, vfdev->ring); + pfwork = kzalloc_obj(*pfwork, GFP_ATOMIC); + if (!pfwork) + continue; + + pfwork->vfdev = vfdev; + pfwork->ndev = ndev; + INIT_WORK(&pfwork->pf2vf_resp, pf2vf_resp_handler); + queue_work(ndev->iov.pf2vf_wq, &pfwork->pf2vf_resp); + /* clear the corresponding vf bit */ + nitrox_write_csr(ndev, reg_addr, BIT_ULL(i)); + } + + /* loop for VF(64..127) */ + reg_addr = NPS_PKT_MBOX_INT_HI; + value = nitrox_read_csr(ndev, reg_addr); + bitmap_from_u64(csr, value); + for_each_set_bit(i, csr, BITS_PER_TYPE(csr)) { + /* get the vfno from ring */ + vfno = RING_TO_VFNO(i + 64, ndev->iov.max_vf_queues); + vfdev = ndev->iov.vfdev + vfno; + vfdev->ring = (i + 64); + /* fill the vf mailbox data */ + vfdev->msg.value = pf2vf_read_mbox(ndev, vfdev->ring); + + pfwork = kzalloc_obj(*pfwork, GFP_ATOMIC); + if (!pfwork) + continue; + + pfwork->vfdev = vfdev; + pfwork->ndev = ndev; + INIT_WORK(&pfwork->pf2vf_resp, pf2vf_resp_handler); + queue_work(ndev->iov.pf2vf_wq, &pfwork->pf2vf_resp); + /* clear the corresponding vf bit */ + nitrox_write_csr(ndev, reg_addr, BIT_ULL(i)); + } +} + +int nitrox_mbox_init(struct nitrox_device *ndev) +{ + struct nitrox_vfdev *vfdev; + int i; + + ndev->iov.vfdev = kzalloc_objs(struct nitrox_vfdev, ndev->iov.num_vfs); + if (!ndev->iov.vfdev) + return -ENOMEM; + + for (i = 0; i < ndev->iov.num_vfs; i++) { + vfdev = ndev->iov.vfdev + i; + vfdev->vfno = i; + } + + /* allocate pf2vf response workqueue */ + ndev->iov.pf2vf_wq = alloc_workqueue("nitrox_pf2vf", WQ_PERCPU, 0); + if (!ndev->iov.pf2vf_wq) { + kfree(ndev->iov.vfdev); + ndev->iov.vfdev = NULL; + return -ENOMEM; + } + /* enable pf2vf mailbox interrupts */ + enable_pf2vf_mbox_interrupts(ndev); + + return 0; +} + +void nitrox_mbox_cleanup(struct nitrox_device *ndev) +{ + /* disable pf2vf mailbox interrupts */ + disable_pf2vf_mbox_interrupts(ndev); + /* destroy workqueue */ + if (ndev->iov.pf2vf_wq) + destroy_workqueue(ndev->iov.pf2vf_wq); + + kfree(ndev->iov.vfdev); + ndev->iov.pf2vf_wq = NULL; + ndev->iov.vfdev = NULL; +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_mbx.h b/drivers/crypto/cavium/nitrox/nitrox_mbx.h new file mode 100644 index 000000000..7c93d0282 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_mbx.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NITROX_MBX_H +#define __NITROX_MBX_H + +int nitrox_mbox_init(struct nitrox_device *ndev); +void nitrox_mbox_cleanup(struct nitrox_device *ndev); +void nitrox_pf2vf_mbox_handler(struct nitrox_device *ndev); + +#endif /* __NITROX_MBX_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_req.h b/drivers/crypto/cavium/nitrox/nitrox_req.h new file mode 100644 index 000000000..6bf088bcd --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_req.h @@ -0,0 +1,728 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NITROX_REQ_H +#define __NITROX_REQ_H + +#include +#include + +#include "nitrox_dev.h" + +#define PENDING_SIG 0xFFFFFFFFFFFFFFFFUL +#define PRIO 4001 + +typedef void (*sereq_completion_t)(void *req, int err); + +/** + * struct gphdr - General purpose Header + * @param0: first parameter. + * @param1: second parameter. + * @param2: third parameter. + * @param3: fourth parameter. + * + * Params tell the iv and enc/dec data offsets. + */ +struct gphdr { + __be16 param0; + __be16 param1; + __be16 param2; + __be16 param3; +}; + +/** + * struct se_req_ctrl - SE request information. + * @arg: Minor number of the opcode + * @ctxc: Context control. + * @unca: Uncertainity enabled. + * @info: Additional information for SE cores. + * @ctxl: Context length in bytes. + * @uddl: User defined data length + */ +union se_req_ctrl { + u64 value; + struct { + u64 raz : 22; + u64 arg : 8; + u64 ctxc : 2; + u64 unca : 1; + u64 info : 3; + u64 unc : 8; + u64 ctxl : 12; + u64 uddl : 8; + } s; +}; + +#define MAX_IV_LEN 16 + +/** + * struct se_crypto_request - SE crypto request structure. + * @opcode: Request opcode (enc/dec) + * @flags: flags from crypto subsystem + * @ctx_handle: Crypto context handle. + * @gph: GP Header + * @ctrl: Request Information. + * @orh: ORH address + * @comp: completion address + * @src: Input sglist + * @dst: Output sglist + */ +struct se_crypto_request { + u8 opcode; + gfp_t gfp; + u32 flags; + u64 ctx_handle; + + struct gphdr gph; + union se_req_ctrl ctrl; + u64 *orh; + u64 *comp; + + struct scatterlist *src; + struct scatterlist *dst; +}; + +/* Crypto opcodes */ +#define FLEXI_CRYPTO_ENCRYPT_HMAC 0x33 +#define ENCRYPT 0 +#define DECRYPT 1 + +/* IV from context */ +#define IV_FROM_CTX 0 +/* IV from Input data */ +#define IV_FROM_DPTR 1 + +/** + * cipher opcodes for firmware + */ +enum flexi_cipher { + CIPHER_NULL = 0, + CIPHER_3DES_CBC, + CIPHER_3DES_ECB, + CIPHER_AES_CBC, + CIPHER_AES_ECB, + CIPHER_AES_CFB, + CIPHER_AES_CTR, + CIPHER_AES_GCM, + CIPHER_AES_XTS, + CIPHER_AES_CCM, + CIPHER_AES_CBC_CTS, + CIPHER_AES_ECB_CTS, + CIPHER_INVALID +}; + +enum flexi_auth { + AUTH_NULL = 0, + AUTH_MD5, + AUTH_SHA1, + AUTH_SHA2_SHA224, + AUTH_SHA2_SHA256, + AUTH_SHA2_SHA384, + AUTH_SHA2_SHA512, + AUTH_GMAC, + AUTH_INVALID +}; + +/** + * struct crypto_keys - Crypto keys + * @key: Encryption key or KEY1 for AES-XTS + * @iv: Encryption IV or Tweak for AES-XTS + */ +struct crypto_keys { + union { + u8 key[AES_MAX_KEY_SIZE]; + u8 key1[AES_MAX_KEY_SIZE]; + } u; + u8 iv[AES_BLOCK_SIZE]; +}; + +/** + * struct auth_keys - Authentication keys + * @ipad: IPAD or KEY2 for AES-XTS + * @opad: OPAD or AUTH KEY if auth_input_type = 1 + */ +struct auth_keys { + union { + u8 ipad[64]; + u8 key2[64]; + } u; + u8 opad[64]; +}; + +union fc_ctx_flags { + __be64 f; + u64 fu; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 cipher_type : 4; + u64 reserved_59 : 1; + u64 aes_keylen : 2; + u64 iv_source : 1; + u64 hash_type : 4; + u64 reserved_49_51 : 3; + u64 auth_input_type: 1; + u64 mac_len : 8; + u64 reserved_0_39 : 40; +#else + u64 reserved_0_39 : 40; + u64 mac_len : 8; + u64 auth_input_type: 1; + u64 reserved_49_51 : 3; + u64 hash_type : 4; + u64 iv_source : 1; + u64 aes_keylen : 2; + u64 reserved_59 : 1; + u64 cipher_type : 4; +#endif + } w0; +}; +/** + * struct flexi_crypto_context - Crypto context + * @cipher_type: Encryption cipher type + * @aes_keylen: AES key length + * @iv_source: Encryption IV source + * @hash_type: Authentication type + * @auth_input_type: Authentication input type + * 1 - Authentication IV and KEY, microcode calculates OPAD/IPAD + * 0 - Authentication OPAD/IPAD + * @mac_len: mac length + * @crypto: Crypto keys + * @auth: Authentication keys + */ +struct flexi_crypto_context { + union fc_ctx_flags flags; + struct crypto_keys crypto; + struct auth_keys auth; +}; + +struct crypto_ctx_hdr { + struct dma_pool *pool; + dma_addr_t dma; + void *vaddr; +}; + +struct nitrox_crypto_ctx { + struct nitrox_device *ndev; + union { + u64 ctx_handle; + struct flexi_crypto_context *fctx; + } u; + struct crypto_ctx_hdr *chdr; + sereq_completion_t callback; +}; + +struct nitrox_kcrypt_request { + struct se_crypto_request creq; + u8 *src; + u8 *dst; + u8 *iv_out; +}; + +/** + * struct nitrox_aead_rctx - AEAD request context + * @nkreq: Base request context + * @cryptlen: Encryption/Decryption data length + * @assoclen: AAD length + * @srclen: Input buffer length + * @dstlen: Output buffer length + * @iv: IV data + * @ivsize: IV data length + * @flags: AEAD req flags + * @ctx_handle: Device context handle + * @src: Source sglist + * @dst: Destination sglist + * @ctrl_arg: Identifies the request type (ENCRYPT/DECRYPT) + */ +struct nitrox_aead_rctx { + struct nitrox_kcrypt_request nkreq; + unsigned int cryptlen; + unsigned int assoclen; + unsigned int srclen; + unsigned int dstlen; + u8 *iv; + int ivsize; + u32 flags; + u64 ctx_handle; + struct scatterlist *src; + struct scatterlist *dst; + u8 ctrl_arg; +}; + +/** + * struct nitrox_rfc4106_rctx - rfc4106 cipher request context + * @base: AEAD request context + * @src: Source sglist + * @dst: Destination sglist + * @assoc: AAD + */ +struct nitrox_rfc4106_rctx { + struct nitrox_aead_rctx base; + struct scatterlist src[3]; + struct scatterlist dst[3]; + u8 assoc[20]; +}; + +/** + * struct pkt_instr_hdr - Packet Instruction Header + * @g: Gather used + * When [G] is set and [GSZ] != 0, the instruction is + * indirect gather instruction. + * When [G] is set and [GSZ] = 0, the instruction is + * direct gather instruction. + * @gsz: Number of pointers in the indirect gather list + * @ihi: When set hardware duplicates the 1st 8 bytes of pkt_instr_hdr + * and adds them to the packet after the pkt_instr_hdr but before any UDD + * @ssz: Not used by the input hardware. But can become slc_store_int[SSZ] + * when [IHI] is set. + * @fsz: The number of front data bytes directly included in the + * PCIe instruction. + * @tlen: The length of the input packet in bytes, include: + * - 16B pkt_hdr + * - Inline context bytes if any, + * - UDD if any, + * - packet payload bytes + */ +union pkt_instr_hdr { + __be64 bev; + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 raz_48_63 : 16; + u64 g : 1; + u64 gsz : 7; + u64 ihi : 1; + u64 ssz : 7; + u64 raz_30_31 : 2; + u64 fsz : 6; + u64 raz_16_23 : 8; + u64 tlen : 16; +#else + u64 tlen : 16; + u64 raz_16_23 : 8; + u64 fsz : 6; + u64 raz_30_31 : 2; + u64 ssz : 7; + u64 ihi : 1; + u64 gsz : 7; + u64 g : 1; + u64 raz_48_63 : 16; +#endif + } s; +}; + +/** + * struct pkt_hdr - Packet Input Header + * @opcode: Request opcode (Major) + * @arg: Request opcode (Minor) + * @ctxc: Context control. + * @unca: When set [UNC] is the uncertainty count for an input packet. + * The hardware uses uncertainty counts to predict + * output buffer use and avoid deadlock. + * @info: Not used by input hardware. Available for use + * during SE processing. + * @destport: The expected destination port/ring/channel for the packet. + * @unc: Uncertainty count for an input packet. + * @grp: SE group that will process the input packet. + * @ctxl: Context Length in 64-bit words. + * @uddl: User-defined data (UDD) length in bytes. + * @ctxp: Context pointer. CTXP<63,2:0> must be zero in all cases. + */ +union pkt_hdr { + __be64 bev[2]; + u64 value[2]; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 opcode : 8; + u64 arg : 8; + u64 ctxc : 2; + u64 unca : 1; + u64 raz_44 : 1; + u64 info : 3; + u64 destport : 9; + u64 unc : 8; + u64 raz_19_23 : 5; + u64 grp : 3; + u64 raz_15 : 1; + u64 ctxl : 7; + u64 uddl : 8; +#else + u64 uddl : 8; + u64 ctxl : 7; + u64 raz_15 : 1; + u64 grp : 3; + u64 raz_19_23 : 5; + u64 unc : 8; + u64 destport : 9; + u64 info : 3; + u64 raz_44 : 1; + u64 unca : 1; + u64 ctxc : 2; + u64 arg : 8; + u64 opcode : 8; +#endif + __be64 ctxp; + } s; +}; + +/** + * struct slc_store_info - Solicited Paceket Output Store Information. + * @ssz: The number of scatterlist pointers for the solicited output port + * packet. + * @rptr: The result pointer for the solicited output port packet. + * If [SSZ]=0, [RPTR] must point directly to a buffer on the remote + * host that is large enough to hold the entire output packet. + * If [SSZ]!=0, [RPTR] must point to an array of ([SSZ]+3)/4 + * sglist components at [RPTR] on the remote host. + */ +union slc_store_info { + __be64 bev[2]; + u64 value[2]; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 raz_39_63 : 25; + u64 ssz : 7; + u64 raz_0_31 : 32; +#else + u64 raz_0_31 : 32; + u64 ssz : 7; + u64 raz_39_63 : 25; +#endif + __be64 rptr; + } s; +}; + +/** + * struct nps_pkt_instr - NPS Packet Instruction of SE cores. + * @dptr0 : Input pointer points to buffer in remote host. + * @ih: Packet Instruction Header (8 bytes) + * @irh: Packet Input Header (16 bytes) + * @slc: Solicited Packet Output Store Information (16 bytes) + * @fdata: Front data + * + * 64-Byte Instruction Format + */ +struct nps_pkt_instr { + __be64 dptr0; + union pkt_instr_hdr ih; + union pkt_hdr irh; + union slc_store_info slc; + u64 fdata[2]; +}; + +/** + * struct aqmq_command_s - The 32 byte command for AE processing. + * @opcode: Request opcode + * @param1: Request control parameter 1 + * @param2: Request control parameter 2 + * @dlen: Input length + * @dptr: Input pointer points to buffer in remote host + * @rptr: Result pointer points to buffer in remote host + * @grp: AQM Group (0..7) + * @cptr: Context pointer + */ +struct aqmq_command_s { + __be16 opcode; + __be16 param1; + __be16 param2; + __be16 dlen; + __be64 dptr; + __be64 rptr; + union { + __be64 word3; +#if defined(__BIG_ENDIAN_BITFIELD) + u64 grp : 3; + u64 cptr : 61; +#else + u64 cptr : 61; + u64 grp : 3; +#endif + }; +}; + +/** + * struct ctx_hdr - Book keeping data about the crypto context + * @pool: Pool used to allocate crypto context + * @dma: Base DMA address of the crypto context + * @ctx_dma: Actual usable crypto context for NITROX + */ +struct ctx_hdr { + struct dma_pool *pool; + dma_addr_t dma; + dma_addr_t ctx_dma; +}; + +/* + * struct sglist_component - SG list component format + * @len0: The number of bytes at [PTR0] on the remote host. + * @len1: The number of bytes at [PTR1] on the remote host. + * @len2: The number of bytes at [PTR2] on the remote host. + * @len3: The number of bytes at [PTR3] on the remote host. + * @dma0: First pointer point to buffer in remote host. + * @dma1: Second pointer point to buffer in remote host. + * @dma2: Third pointer point to buffer in remote host. + * @dma3: Fourth pointer point to buffer in remote host. + */ +struct nitrox_sgcomp { + __be16 len[4]; + __be64 dma[4]; +}; + +/* + * strutct nitrox_sgtable - SG list information + * @sgmap_cnt: Number of buffers mapped + * @total_bytes: Total bytes in sglist. + * @sgcomp_len: Total sglist components length. + * @sgcomp_dma: DMA address of sglist component. + * @sg: crypto request buffer. + * @sgcomp: sglist component for NITROX. + */ +struct nitrox_sgtable { + u8 sgmap_cnt; + u16 total_bytes; + u32 sgcomp_len; + dma_addr_t sgcomp_dma; + struct scatterlist *sg; + struct nitrox_sgcomp *sgcomp; +}; + +/* Response Header Length */ +#define ORH_HLEN 8 +/* Completion bytes Length */ +#define COMP_HLEN 8 + +struct resp_hdr { + u64 *orh; + u64 *completion; +}; + +typedef void (*completion_t)(void *arg, int err); + +/** + * struct nitrox_softreq - Represents the NIROX Request. + * @response: response list entry + * @backlog: Backlog list entry + * @ndev: Device used to submit the request + * @cmdq: Command queue for submission + * @resp: Response headers + * @instr: 64B instruction + * @in: SG table for input + * @out SG table for output + * @tstamp: Request submitted time in jiffies + * @callback: callback after request completion/timeout + * @cb_arg: callback argument + */ +struct nitrox_softreq { + struct list_head response; + struct list_head backlog; + + u32 flags; + gfp_t gfp; + atomic_t status; + + struct nitrox_device *ndev; + struct nitrox_cmdq *cmdq; + + struct nps_pkt_instr instr; + struct resp_hdr resp; + struct nitrox_sgtable in; + struct nitrox_sgtable out; + + unsigned long tstamp; + + completion_t callback; + void *cb_arg; +}; + +static inline int flexi_aes_keylen(int keylen) +{ + int aes_keylen; + + switch (keylen) { + case AES_KEYSIZE_128: + aes_keylen = 1; + break; + case AES_KEYSIZE_192: + aes_keylen = 2; + break; + case AES_KEYSIZE_256: + aes_keylen = 3; + break; + default: + aes_keylen = -EINVAL; + break; + } + return aes_keylen; +} + +static inline void *alloc_req_buf(int nents, int extralen, gfp_t gfp) +{ + size_t size; + + size = sizeof(struct scatterlist) * nents; + size += extralen; + + return kzalloc(size, gfp); +} + +/** + * create_single_sg - Point SG entry to the data + * @sg: Destination SG list + * @buf: Data + * @buflen: Data length + * + * Returns next free entry in the destination SG list + **/ +static inline struct scatterlist *create_single_sg(struct scatterlist *sg, + void *buf, int buflen) +{ + sg_set_buf(sg, buf, buflen); + sg++; + return sg; +} + +/** + * create_multi_sg - Create multiple sg entries with buflen data length from + * source sglist + * @to_sg: Destination SG list + * @from_sg: Source SG list + * @buflen: Data length + * + * Returns next free entry in the destination SG list + **/ +static inline struct scatterlist *create_multi_sg(struct scatterlist *to_sg, + struct scatterlist *from_sg, + int buflen) +{ + struct scatterlist *sg = to_sg; + unsigned int sglen; + + for (; buflen && from_sg; buflen -= sglen) { + sglen = from_sg->length; + if (sglen > buflen) + sglen = buflen; + + sg_set_buf(sg, sg_virt(from_sg), sglen); + from_sg = sg_next(from_sg); + sg++; + } + + return sg; +} + +static inline void set_orh_value(u64 *orh) +{ + WRITE_ONCE(*orh, PENDING_SIG); +} + +static inline void set_comp_value(u64 *comp) +{ + WRITE_ONCE(*comp, PENDING_SIG); +} + +static inline int alloc_src_req_buf(struct nitrox_kcrypt_request *nkreq, + int nents, int ivsize) +{ + struct se_crypto_request *creq = &nkreq->creq; + + nkreq->src = alloc_req_buf(nents, ivsize, creq->gfp); + if (!nkreq->src) + return -ENOMEM; + + return 0; +} + +static inline void nitrox_creq_copy_iv(char *dst, char *src, int size) +{ + memcpy(dst, src, size); +} + +static inline struct scatterlist *nitrox_creq_src_sg(char *iv, int ivsize) +{ + return (struct scatterlist *)(iv + ivsize); +} + +static inline void nitrox_creq_set_src_sg(struct nitrox_kcrypt_request *nkreq, + int nents, int ivsize, + struct scatterlist *src, int buflen) +{ + char *iv = nkreq->src; + struct scatterlist *sg; + struct se_crypto_request *creq = &nkreq->creq; + + creq->src = nitrox_creq_src_sg(iv, ivsize); + sg = creq->src; + sg_init_table(sg, nents); + + /* Input format: + * +----+----------------+ + * | IV | SRC sg entries | + * +----+----------------+ + */ + + /* IV */ + sg = create_single_sg(sg, iv, ivsize); + /* SRC entries */ + create_multi_sg(sg, src, buflen); +} + +static inline int alloc_dst_req_buf(struct nitrox_kcrypt_request *nkreq, + int nents) +{ + int extralen = ORH_HLEN + COMP_HLEN; + struct se_crypto_request *creq = &nkreq->creq; + + nkreq->dst = alloc_req_buf(nents, extralen, creq->gfp); + if (!nkreq->dst) + return -ENOMEM; + + return 0; +} + +static inline void nitrox_creq_set_orh(struct nitrox_kcrypt_request *nkreq) +{ + struct se_crypto_request *creq = &nkreq->creq; + + creq->orh = (u64 *)(nkreq->dst); + set_orh_value(creq->orh); +} + +static inline void nitrox_creq_set_comp(struct nitrox_kcrypt_request *nkreq) +{ + struct se_crypto_request *creq = &nkreq->creq; + + creq->comp = (u64 *)(nkreq->dst + ORH_HLEN); + set_comp_value(creq->comp); +} + +static inline struct scatterlist *nitrox_creq_dst_sg(char *dst) +{ + return (struct scatterlist *)(dst + ORH_HLEN + COMP_HLEN); +} + +static inline void nitrox_creq_set_dst_sg(struct nitrox_kcrypt_request *nkreq, + int nents, int ivsize, + struct scatterlist *dst, int buflen) +{ + struct se_crypto_request *creq = &nkreq->creq; + struct scatterlist *sg; + char *iv = nkreq->src; + + creq->dst = nitrox_creq_dst_sg(nkreq->dst); + sg = creq->dst; + sg_init_table(sg, nents); + + /* Output format: + * +-----+----+----------------+-----------------+ + * | ORH | IV | DST sg entries | COMPLETION Bytes| + * +-----+----+----------------+-----------------+ + */ + + /* ORH */ + sg = create_single_sg(sg, creq->orh, ORH_HLEN); + /* IV */ + sg = create_single_sg(sg, iv, ivsize); + /* DST entries */ + sg = create_multi_sg(sg, dst, buflen); + /* COMPLETION Bytes */ + create_single_sg(sg, creq->comp, COMP_HLEN); +} + +#endif /* __NITROX_REQ_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c new file mode 100644 index 000000000..6a1bdae73 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c @@ -0,0 +1,607 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include + +#include "nitrox_common.h" +#include "nitrox_dev.h" +#include "nitrox_req.h" +#include "nitrox_csr.h" + +/* SLC_STORE_INFO */ +#define MIN_UDD_LEN 16 +/* PKT_IN_HDR + SLC_STORE_INFO */ +#define FDATA_SIZE 32 +/* Base destination port for the solicited requests */ +#define SOLICIT_BASE_DPORT 256 + +#define REQ_NOT_POSTED 1 +#define REQ_BACKLOG 2 +#define REQ_POSTED 3 + +/* + * Response codes from SE microcode + * 0x00 - Success + * Completion with no error + * 0x43 - ERR_GC_DATA_LEN_INVALID + * Invalid Data length if Encryption Data length is + * less than 16 bytes for AES-XTS and AES-CTS. + * 0x45 - ERR_GC_CTX_LEN_INVALID + * Invalid context length: CTXL != 23 words. + * 0x4F - ERR_GC_DOCSIS_CIPHER_INVALID + * DOCSIS support is enabled with other than + * AES/DES-CBC mode encryption. + * 0x50 - ERR_GC_DOCSIS_OFFSET_INVALID + * Authentication offset is other than 0 with + * Encryption IV source = 0. + * Authentication offset is other than 8 (DES)/16 (AES) + * with Encryption IV source = 1 + * 0x51 - ERR_GC_CRC32_INVALID_SELECTION + * CRC32 is enabled for other than DOCSIS encryption. + * 0x52 - ERR_GC_AES_CCM_FLAG_INVALID + * Invalid flag options in AES-CCM IV. + */ + +static inline int incr_index(int index, int count, int max) +{ + if ((index + count) >= max) + index = index + count - max; + else + index += count; + + return index; +} + +static void softreq_unmap_sgbufs(struct nitrox_softreq *sr) +{ + struct nitrox_device *ndev = sr->ndev; + struct device *dev = DEV(ndev); + + + dma_unmap_sg(dev, sr->in.sg, sg_nents(sr->in.sg), + DMA_BIDIRECTIONAL); + dma_unmap_single(dev, sr->in.sgcomp_dma, sr->in.sgcomp_len, + DMA_TO_DEVICE); + kfree(sr->in.sgcomp); + sr->in.sg = NULL; + sr->in.sgmap_cnt = 0; + + dma_unmap_sg(dev, sr->out.sg, sg_nents(sr->out.sg), + DMA_BIDIRECTIONAL); + dma_unmap_single(dev, sr->out.sgcomp_dma, sr->out.sgcomp_len, + DMA_TO_DEVICE); + kfree(sr->out.sgcomp); + sr->out.sg = NULL; + sr->out.sgmap_cnt = 0; +} + +static void softreq_destroy(struct nitrox_softreq *sr) +{ + softreq_unmap_sgbufs(sr); + kfree(sr); +} + +/** + * create_sg_component - create SG componets for N5 device. + * @sr: Request structure + * @sgtbl: SG table + * @map_nents: number of dma mapped entries + * + * Component structure + * + * 63 48 47 32 31 16 15 0 + * -------------------------------------- + * | LEN0 | LEN1 | LEN2 | LEN3 | + * |------------------------------------- + * | PTR0 | + * -------------------------------------- + * | PTR1 | + * -------------------------------------- + * | PTR2 | + * -------------------------------------- + * | PTR3 | + * -------------------------------------- + * + * Returns 0 if success or a negative errno code on error. + */ +static int create_sg_component(struct nitrox_softreq *sr, + struct nitrox_sgtable *sgtbl, int map_nents) +{ + struct nitrox_device *ndev = sr->ndev; + struct nitrox_sgcomp *sgcomp; + struct scatterlist *sg; + dma_addr_t dma; + size_t sz_comp; + int i, j, nr_sgcomp; + + nr_sgcomp = roundup(map_nents, 4) / 4; + + /* each component holds 4 dma pointers */ + sz_comp = nr_sgcomp * sizeof(*sgcomp); + sgcomp = kzalloc(sz_comp, sr->gfp); + if (!sgcomp) + return -ENOMEM; + + sgtbl->sgcomp = sgcomp; + + sg = sgtbl->sg; + /* populate device sg component */ + for (i = 0; i < nr_sgcomp; i++) { + for (j = 0; j < 4 && sg; j++) { + sgcomp[i].len[j] = cpu_to_be16(sg_dma_len(sg)); + sgcomp[i].dma[j] = cpu_to_be64(sg_dma_address(sg)); + sg = sg_next(sg); + } + } + /* map the device sg component */ + dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE); + if (dma_mapping_error(DEV(ndev), dma)) { + kfree(sgtbl->sgcomp); + sgtbl->sgcomp = NULL; + return -ENOMEM; + } + + sgtbl->sgcomp_dma = dma; + sgtbl->sgcomp_len = sz_comp; + + return 0; +} + +/** + * dma_map_inbufs - DMA map input sglist and creates sglist component + * for N5 device. + * @sr: Request structure + * @req: Crypto request structre + * + * Returns 0 if successful or a negative errno code on error. + */ +static int dma_map_inbufs(struct nitrox_softreq *sr, + struct se_crypto_request *req) +{ + struct device *dev = DEV(sr->ndev); + struct scatterlist *sg; + int i, nents, ret = 0; + + nents = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_BIDIRECTIONAL); + if (!nents) + return -EINVAL; + + for_each_sg(req->src, sg, nents, i) + sr->in.total_bytes += sg_dma_len(sg); + + sr->in.sg = req->src; + sr->in.sgmap_cnt = nents; + ret = create_sg_component(sr, &sr->in, sr->in.sgmap_cnt); + if (ret) + goto incomp_err; + + return 0; + +incomp_err: + dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_BIDIRECTIONAL); + sr->in.sgmap_cnt = 0; + return ret; +} + +static int dma_map_outbufs(struct nitrox_softreq *sr, + struct se_crypto_request *req) +{ + struct device *dev = DEV(sr->ndev); + int nents, ret = 0; + + nents = dma_map_sg(dev, req->dst, sg_nents(req->dst), + DMA_BIDIRECTIONAL); + if (!nents) + return -EINVAL; + + sr->out.sg = req->dst; + sr->out.sgmap_cnt = nents; + ret = create_sg_component(sr, &sr->out, sr->out.sgmap_cnt); + if (ret) + goto outcomp_map_err; + + return 0; + +outcomp_map_err: + dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_BIDIRECTIONAL); + sr->out.sgmap_cnt = 0; + sr->out.sg = NULL; + return ret; +} + +static inline int softreq_map_iobuf(struct nitrox_softreq *sr, + struct se_crypto_request *creq) +{ + int ret; + + ret = dma_map_inbufs(sr, creq); + if (ret) + return ret; + + ret = dma_map_outbufs(sr, creq); + if (ret) + softreq_unmap_sgbufs(sr); + + return ret; +} + +static inline void backlog_list_add(struct nitrox_softreq *sr, + struct nitrox_cmdq *cmdq) +{ + INIT_LIST_HEAD(&sr->backlog); + + spin_lock_bh(&cmdq->backlog_qlock); + list_add_tail(&sr->backlog, &cmdq->backlog_head); + atomic_inc(&cmdq->backlog_count); + atomic_set(&sr->status, REQ_BACKLOG); + spin_unlock_bh(&cmdq->backlog_qlock); +} + +static inline void response_list_add(struct nitrox_softreq *sr, + struct nitrox_cmdq *cmdq) +{ + INIT_LIST_HEAD(&sr->response); + + spin_lock_bh(&cmdq->resp_qlock); + list_add_tail(&sr->response, &cmdq->response_head); + spin_unlock_bh(&cmdq->resp_qlock); +} + +static inline void response_list_del(struct nitrox_softreq *sr, + struct nitrox_cmdq *cmdq) +{ + spin_lock_bh(&cmdq->resp_qlock); + list_del(&sr->response); + spin_unlock_bh(&cmdq->resp_qlock); +} + +static struct nitrox_softreq * +get_first_response_entry(struct nitrox_cmdq *cmdq) +{ + return list_first_entry_or_null(&cmdq->response_head, + struct nitrox_softreq, response); +} + +static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen) +{ + if (atomic_inc_return(&cmdq->pending_count) > qlen) { + atomic_dec(&cmdq->pending_count); + /* sync with other cpus */ + smp_mb__after_atomic(); + return true; + } + /* sync with other cpus */ + smp_mb__after_atomic(); + return false; +} + +/** + * post_se_instr - Post SE instruction to Packet Input ring + * @sr: Request structure + * @cmdq: Command queue structure + * + * Returns 0 if successful or a negative error code, + * if no space in ring. + */ +static void post_se_instr(struct nitrox_softreq *sr, + struct nitrox_cmdq *cmdq) +{ + struct nitrox_device *ndev = sr->ndev; + int idx; + u8 *ent; + + spin_lock_bh(&cmdq->cmd_qlock); + + idx = cmdq->write_idx; + /* copy the instruction */ + ent = cmdq->base + (idx * cmdq->instr_size); + memcpy(ent, &sr->instr, cmdq->instr_size); + + atomic_set(&sr->status, REQ_POSTED); + response_list_add(sr, cmdq); + sr->tstamp = jiffies; + /* flush the command queue updates */ + dma_wmb(); + + /* Ring doorbell with count 1 */ + writeq(1, cmdq->dbell_csr_addr); + + cmdq->write_idx = incr_index(idx, 1, ndev->qlen); + + spin_unlock_bh(&cmdq->cmd_qlock); + + /* increment the posted command count */ + atomic64_inc(&ndev->stats.posted); +} + +static int post_backlog_cmds(struct nitrox_cmdq *cmdq) +{ + struct nitrox_device *ndev = cmdq->ndev; + struct nitrox_softreq *sr, *tmp; + int ret = 0; + + if (!atomic_read(&cmdq->backlog_count)) + return 0; + + spin_lock_bh(&cmdq->backlog_qlock); + + list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) { + /* submit until space available */ + if (unlikely(cmdq_full(cmdq, ndev->qlen))) { + ret = -ENOSPC; + break; + } + /* delete from backlog list */ + list_del(&sr->backlog); + atomic_dec(&cmdq->backlog_count); + /* sync with other cpus */ + smp_mb__after_atomic(); + + /* post the command */ + post_se_instr(sr, cmdq); + } + spin_unlock_bh(&cmdq->backlog_qlock); + + return ret; +} + +static int nitrox_enqueue_request(struct nitrox_softreq *sr) +{ + struct nitrox_cmdq *cmdq = sr->cmdq; + struct nitrox_device *ndev = sr->ndev; + + /* try to post backlog requests */ + post_backlog_cmds(cmdq); + + if (unlikely(cmdq_full(cmdq, ndev->qlen))) { + if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { + /* increment drop count */ + atomic64_inc(&ndev->stats.dropped); + return -ENOSPC; + } + /* add to backlog list */ + backlog_list_add(sr, cmdq); + return -EINPROGRESS; + } + post_se_instr(sr, cmdq); + + return -EINPROGRESS; +} + +/** + * nitrox_process_se_request - Send request to SE core + * @ndev: NITROX device + * @req: Crypto request + * @callback: Completion callback + * @cb_arg: Completion callback arguments + * + * Returns 0 on success, or a negative error code. + */ +int nitrox_process_se_request(struct nitrox_device *ndev, + struct se_crypto_request *req, + completion_t callback, + void *cb_arg) +{ + struct nitrox_softreq *sr; + dma_addr_t ctx_handle = 0; + int qno, ret = 0; + + if (!nitrox_ready(ndev)) + return -ENODEV; + + sr = kzalloc_obj(*sr, req->gfp); + if (!sr) + return -ENOMEM; + + sr->ndev = ndev; + sr->flags = req->flags; + sr->gfp = req->gfp; + sr->callback = callback; + sr->cb_arg = cb_arg; + + atomic_set(&sr->status, REQ_NOT_POSTED); + + sr->resp.orh = req->orh; + sr->resp.completion = req->comp; + + ret = softreq_map_iobuf(sr, req); + if (ret) { + kfree(sr); + return ret; + } + + /* get the context handle */ + if (req->ctx_handle) { + struct ctx_hdr *hdr; + u8 *ctx_ptr; + + ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle; + hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr)); + ctx_handle = hdr->ctx_dma; + } + + /* select the queue */ + qno = smp_processor_id() % ndev->nr_queues; + + sr->cmdq = &ndev->pkt_inq[qno]; + + /* + * 64-Byte Instruction Format + * + * ---------------------- + * | DPTR0 | 8 bytes + * ---------------------- + * | PKT_IN_INSTR_HDR | 8 bytes + * ---------------------- + * | PKT_IN_HDR | 16 bytes + * ---------------------- + * | SLC_INFO | 16 bytes + * ---------------------- + * | Front data | 16 bytes + * ---------------------- + */ + + /* fill the packet instruction */ + /* word 0 */ + sr->instr.dptr0 = cpu_to_be64(sr->in.sgcomp_dma); + + /* word 1 */ + sr->instr.ih.value = 0; + sr->instr.ih.s.g = 1; + sr->instr.ih.s.gsz = sr->in.sgmap_cnt; + sr->instr.ih.s.ssz = sr->out.sgmap_cnt; + sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr); + sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes; + sr->instr.ih.bev = cpu_to_be64(sr->instr.ih.value); + + /* word 2 */ + sr->instr.irh.value[0] = 0; + sr->instr.irh.s.uddl = MIN_UDD_LEN; + /* context length in 64-bit words */ + sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8); + /* offset from solicit base port 256 */ + sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno; + sr->instr.irh.s.ctxc = req->ctrl.s.ctxc; + sr->instr.irh.s.arg = req->ctrl.s.arg; + sr->instr.irh.s.opcode = req->opcode; + sr->instr.irh.bev[0] = cpu_to_be64(sr->instr.irh.value[0]); + + /* word 3 */ + sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle); + + /* word 4 */ + sr->instr.slc.value[0] = 0; + sr->instr.slc.s.ssz = sr->out.sgmap_cnt; + sr->instr.slc.bev[0] = cpu_to_be64(sr->instr.slc.value[0]); + + /* word 5 */ + sr->instr.slc.s.rptr = cpu_to_be64(sr->out.sgcomp_dma); + + /* + * No conversion for front data, + * It goes into payload + * put GP Header in front data + */ + sr->instr.fdata[0] = *((u64 *)&req->gph); + sr->instr.fdata[1] = 0; + + ret = nitrox_enqueue_request(sr); + if (ret == -ENOSPC) + goto send_fail; + + return ret; + +send_fail: + softreq_destroy(sr); + return ret; +} + +static inline int cmd_timeout(unsigned long tstamp, unsigned long timeout) +{ + return time_after_eq(jiffies, (tstamp + timeout)); +} + +void backlog_qflush_work(struct work_struct *work) +{ + struct nitrox_cmdq *cmdq; + + cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush); + post_backlog_cmds(cmdq); +} + +static bool sr_completed(struct nitrox_softreq *sr) +{ + u64 orh = READ_ONCE(*sr->resp.orh); + unsigned long timeout = jiffies + msecs_to_jiffies(1); + + if ((orh != PENDING_SIG) && (orh & 0xff)) + return true; + + while (READ_ONCE(*sr->resp.completion) == PENDING_SIG) { + if (time_after(jiffies, timeout)) { + pr_err("comp not done\n"); + return false; + } + } + + return true; +} + +/** + * process_response_list - process completed requests + * @cmdq: Command queue structure + * + * Returns the number of responses processed. + */ +static void process_response_list(struct nitrox_cmdq *cmdq) +{ + struct nitrox_device *ndev = cmdq->ndev; + struct nitrox_softreq *sr; + int req_completed = 0, err = 0, budget; + completion_t callback; + void *cb_arg; + + /* check all pending requests */ + budget = atomic_read(&cmdq->pending_count); + + while (req_completed < budget) { + sr = get_first_response_entry(cmdq); + if (!sr) + break; + + if (atomic_read(&sr->status) != REQ_POSTED) + break; + + /* check orh and completion bytes updates */ + if (!sr_completed(sr)) { + /* request not completed, check for timeout */ + if (!cmd_timeout(sr->tstamp, ndev->timeout)) + break; + dev_err_ratelimited(DEV(ndev), + "Request timeout, orh 0x%016llx\n", + READ_ONCE(*sr->resp.orh)); + } + atomic_dec(&cmdq->pending_count); + atomic64_inc(&ndev->stats.completed); + /* sync with other cpus */ + smp_mb__after_atomic(); + /* remove from response list */ + response_list_del(sr, cmdq); + /* ORH error code */ + err = READ_ONCE(*sr->resp.orh) & 0xff; + callback = sr->callback; + cb_arg = sr->cb_arg; + softreq_destroy(sr); + if (callback) + callback(cb_arg, err); + + req_completed++; + } +} + +/* + * pkt_slc_resp_tasklet - post processing of SE responses + */ +void pkt_slc_resp_tasklet(unsigned long data) +{ + struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data); + struct nitrox_cmdq *cmdq = qvec->cmdq; + union nps_pkt_slc_cnts slc_cnts; + + /* read completion count */ + slc_cnts.value = readq(cmdq->compl_cnt_csr_addr); + /* resend the interrupt if more work to do */ + slc_cnts.s.resend = 1; + + process_response_list(cmdq); + + /* + * clear the interrupt with resend bit enabled, + * MSI-X interrupt generates if Completion count > Threshold + */ + writeq(slc_cnts.value, cmdq->compl_cnt_csr_addr); + + if (atomic_read(&cmdq->backlog_count)) + schedule_work(&cmdq->backlog_qflush); +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_skcipher.c b/drivers/crypto/cavium/nitrox/nitrox_skcipher.c new file mode 100644 index 000000000..6e5e667ba --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_skcipher.c @@ -0,0 +1,531 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "nitrox_dev.h" +#include "nitrox_common.h" +#include "nitrox_req.h" + +struct nitrox_cipher { + const char *name; + enum flexi_cipher value; +}; + +/* + * supported cipher list + */ +static const struct nitrox_cipher flexi_cipher_table[] = { + { "null", CIPHER_NULL }, + { "cbc(des3_ede)", CIPHER_3DES_CBC }, + { "ecb(des3_ede)", CIPHER_3DES_ECB }, + { "cbc(aes)", CIPHER_AES_CBC }, + { "ecb(aes)", CIPHER_AES_ECB }, + { "cfb(aes)", CIPHER_AES_CFB }, + { "rfc3686(ctr(aes))", CIPHER_AES_CTR }, + { "xts(aes)", CIPHER_AES_XTS }, + { "cts(cbc(aes))", CIPHER_AES_CBC_CTS }, + { NULL, CIPHER_INVALID } +}; + +static enum flexi_cipher flexi_cipher_type(const char *name) +{ + const struct nitrox_cipher *cipher = flexi_cipher_table; + + while (cipher->name) { + if (!strcmp(cipher->name, name)) + break; + cipher++; + } + return cipher->value; +} + +static void free_src_sglist(struct skcipher_request *skreq) +{ + struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); + + kfree(nkreq->src); +} + +static void free_dst_sglist(struct skcipher_request *skreq) +{ + struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); + + kfree(nkreq->dst); +} + +static void nitrox_skcipher_callback(void *arg, int err) +{ + struct skcipher_request *skreq = arg; + + free_src_sglist(skreq); + free_dst_sglist(skreq); + if (err) { + pr_err_ratelimited("request failed status 0x%0x\n", err); + err = -EINVAL; + } + + skcipher_request_complete(skreq, err); +} + +static void nitrox_cbc_cipher_callback(void *arg, int err) +{ + struct skcipher_request *skreq = arg; + struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq); + int ivsize = crypto_skcipher_ivsize(cipher); + unsigned int start = skreq->cryptlen - ivsize; + + if (err) { + nitrox_skcipher_callback(arg, err); + return; + } + + if (nkreq->creq.ctrl.s.arg == ENCRYPT) { + scatterwalk_map_and_copy(skreq->iv, skreq->dst, start, ivsize, + 0); + } else { + if (skreq->src != skreq->dst) { + scatterwalk_map_and_copy(skreq->iv, skreq->src, start, + ivsize, 0); + } else { + memcpy(skreq->iv, nkreq->iv_out, ivsize); + kfree(nkreq->iv_out); + } + } + + nitrox_skcipher_callback(arg, err); +} + +static int nitrox_skcipher_init(struct crypto_skcipher *tfm) +{ + struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm); + struct crypto_ctx_hdr *chdr; + + /* get the first device */ + nctx->ndev = nitrox_get_first_device(); + if (!nctx->ndev) + return -ENODEV; + + /* allocate nitrox crypto context */ + chdr = crypto_alloc_context(nctx->ndev); + if (!chdr) { + nitrox_put_device(nctx->ndev); + return -ENOMEM; + } + + nctx->callback = nitrox_skcipher_callback; + nctx->chdr = chdr; + nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr + + sizeof(struct ctx_hdr)); + crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(tfm) + + sizeof(struct nitrox_kcrypt_request)); + return 0; +} + +static int nitrox_cbc_init(struct crypto_skcipher *tfm) +{ + int err; + struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm); + + err = nitrox_skcipher_init(tfm); + if (err) + return err; + + nctx->callback = nitrox_cbc_cipher_callback; + return 0; +} + +static void nitrox_skcipher_exit(struct crypto_skcipher *tfm) +{ + struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm); + + /* free the nitrox crypto context */ + if (nctx->u.ctx_handle) { + struct flexi_crypto_context *fctx = nctx->u.fctx; + + memzero_explicit(&fctx->crypto, sizeof(struct crypto_keys)); + memzero_explicit(&fctx->auth, sizeof(struct auth_keys)); + crypto_free_context((void *)nctx->chdr); + } + nitrox_put_device(nctx->ndev); + + nctx->u.ctx_handle = 0; + nctx->ndev = NULL; +} + +static inline int nitrox_skcipher_setkey(struct crypto_skcipher *cipher, + int aes_keylen, const u8 *key, + unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); + struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm); + struct flexi_crypto_context *fctx; + union fc_ctx_flags *flags; + enum flexi_cipher cipher_type; + const char *name; + + name = crypto_tfm_alg_name(tfm); + cipher_type = flexi_cipher_type(name); + if (unlikely(cipher_type == CIPHER_INVALID)) { + pr_err("unsupported cipher: %s\n", name); + return -EINVAL; + } + + /* fill crypto context */ + fctx = nctx->u.fctx; + flags = &fctx->flags; + flags->f = 0; + flags->w0.cipher_type = cipher_type; + flags->w0.aes_keylen = aes_keylen; + flags->w0.iv_source = IV_FROM_DPTR; + flags->f = cpu_to_be64(*(u64 *)&flags->w0); + /* copy the key to context */ + memcpy(fctx->crypto.u.key, key, keylen); + + return 0; +} + +static int nitrox_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, + unsigned int keylen) +{ + int aes_keylen; + + aes_keylen = flexi_aes_keylen(keylen); + if (aes_keylen < 0) + return -EINVAL; + return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen); +} + +static int alloc_src_sglist(struct skcipher_request *skreq, int ivsize) +{ + struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); + int nents = sg_nents(skreq->src) + 1; + int ret; + + /* Allocate buffer to hold IV and input scatterlist array */ + ret = alloc_src_req_buf(nkreq, nents, ivsize); + if (ret) + return ret; + + nitrox_creq_copy_iv(nkreq->src, skreq->iv, ivsize); + nitrox_creq_set_src_sg(nkreq, nents, ivsize, skreq->src, + skreq->cryptlen); + + return 0; +} + +static int alloc_dst_sglist(struct skcipher_request *skreq, int ivsize) +{ + struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); + int nents = sg_nents(skreq->dst) + 3; + int ret; + + /* Allocate buffer to hold ORH, COMPLETION and output scatterlist + * array + */ + ret = alloc_dst_req_buf(nkreq, nents); + if (ret) + return ret; + + nitrox_creq_set_orh(nkreq); + nitrox_creq_set_comp(nkreq); + nitrox_creq_set_dst_sg(nkreq, nents, ivsize, skreq->dst, + skreq->cryptlen); + + return 0; +} + +static int nitrox_skcipher_crypt(struct skcipher_request *skreq, bool enc) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq); + struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(cipher); + struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); + int ivsize = crypto_skcipher_ivsize(cipher); + struct se_crypto_request *creq; + int ret; + + creq = &nkreq->creq; + creq->flags = skreq->base.flags; + creq->gfp = (skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + + /* fill the request */ + creq->ctrl.value = 0; + creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC; + creq->ctrl.s.arg = (enc ? ENCRYPT : DECRYPT); + /* param0: length of the data to be encrypted */ + creq->gph.param0 = cpu_to_be16(skreq->cryptlen); + creq->gph.param1 = 0; + /* param2: encryption data offset */ + creq->gph.param2 = cpu_to_be16(ivsize); + creq->gph.param3 = 0; + + creq->ctx_handle = nctx->u.ctx_handle; + creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context); + + ret = alloc_src_sglist(skreq, ivsize); + if (ret) + return ret; + + ret = alloc_dst_sglist(skreq, ivsize); + if (ret) { + free_src_sglist(skreq); + return ret; + } + + /* send the crypto request */ + return nitrox_process_se_request(nctx->ndev, creq, nctx->callback, + skreq); +} + +static int nitrox_cbc_decrypt(struct skcipher_request *skreq) +{ + struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq); + int ivsize = crypto_skcipher_ivsize(cipher); + gfp_t flags = (skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + unsigned int start = skreq->cryptlen - ivsize; + + if (skreq->src != skreq->dst) + return nitrox_skcipher_crypt(skreq, false); + + nkreq->iv_out = kmalloc(ivsize, flags); + if (!nkreq->iv_out) + return -ENOMEM; + + scatterwalk_map_and_copy(nkreq->iv_out, skreq->src, start, ivsize, 0); + return nitrox_skcipher_crypt(skreq, false); +} + +static int nitrox_aes_encrypt(struct skcipher_request *skreq) +{ + return nitrox_skcipher_crypt(skreq, true); +} + +static int nitrox_aes_decrypt(struct skcipher_request *skreq) +{ + return nitrox_skcipher_crypt(skreq, false); +} + +static int nitrox_3des_setkey(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + return verify_skcipher_des3_key(cipher, key) ?: + nitrox_skcipher_setkey(cipher, 0, key, keylen); +} + +static int nitrox_3des_encrypt(struct skcipher_request *skreq) +{ + return nitrox_skcipher_crypt(skreq, true); +} + +static int nitrox_3des_decrypt(struct skcipher_request *skreq) +{ + return nitrox_skcipher_crypt(skreq, false); +} + +static int nitrox_aes_xts_setkey(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(cipher); + struct flexi_crypto_context *fctx; + int aes_keylen, ret; + + ret = xts_verify_key(cipher, key, keylen); + if (ret) + return ret; + + keylen /= 2; + + aes_keylen = flexi_aes_keylen(keylen); + if (aes_keylen < 0) + return -EINVAL; + + fctx = nctx->u.fctx; + /* copy KEY2 */ + memcpy(fctx->auth.u.key2, (key + keylen), keylen); + + return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen); +} + +static int nitrox_aes_ctr_rfc3686_setkey(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(cipher); + struct flexi_crypto_context *fctx; + int aes_keylen; + + if (keylen < CTR_RFC3686_NONCE_SIZE) + return -EINVAL; + + fctx = nctx->u.fctx; + + memcpy(fctx->crypto.iv, key + (keylen - CTR_RFC3686_NONCE_SIZE), + CTR_RFC3686_NONCE_SIZE); + + keylen -= CTR_RFC3686_NONCE_SIZE; + + aes_keylen = flexi_aes_keylen(keylen); + if (aes_keylen < 0) + return -EINVAL; + return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen); +} + +static struct skcipher_alg nitrox_skciphers[] = { { + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "n5_cbc(aes)", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = nitrox_aes_setkey, + .encrypt = nitrox_aes_encrypt, + .decrypt = nitrox_cbc_decrypt, + .init = nitrox_cbc_init, + .exit = nitrox_skcipher_exit, +}, { + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "n5_ecb(aes)", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = nitrox_aes_setkey, + .encrypt = nitrox_aes_encrypt, + .decrypt = nitrox_aes_decrypt, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, +}, { + .base = { + .cra_name = "xts(aes)", + .cra_driver_name = "n5_xts(aes)", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = nitrox_aes_xts_setkey, + .encrypt = nitrox_aes_encrypt, + .decrypt = nitrox_aes_decrypt, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, +}, { + .base = { + .cra_name = "rfc3686(ctr(aes))", + .cra_driver_name = "n5_rfc3686(ctr(aes))", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, + .setkey = nitrox_aes_ctr_rfc3686_setkey, + .encrypt = nitrox_aes_encrypt, + .decrypt = nitrox_aes_decrypt, +}, { + .base = { + .cra_name = "cts(cbc(aes))", + .cra_driver_name = "n5_cts(cbc(aes))", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = nitrox_aes_setkey, + .encrypt = nitrox_aes_encrypt, + .decrypt = nitrox_aes_decrypt, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, +}, { + .base = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "n5_cbc(des3_ede)", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setkey = nitrox_3des_setkey, + .encrypt = nitrox_3des_encrypt, + .decrypt = nitrox_cbc_decrypt, + .init = nitrox_cbc_init, + .exit = nitrox_skcipher_exit, +}, { + .base = { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "n5_ecb(des3_ede)", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setkey = nitrox_3des_setkey, + .encrypt = nitrox_3des_encrypt, + .decrypt = nitrox_3des_decrypt, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, +} + +}; + +int nitrox_register_skciphers(void) +{ + return crypto_register_skciphers(nitrox_skciphers, + ARRAY_SIZE(nitrox_skciphers)); +} + +void nitrox_unregister_skciphers(void) +{ + crypto_unregister_skciphers(nitrox_skciphers, + ARRAY_SIZE(nitrox_skciphers)); +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_sriov.c b/drivers/crypto/cavium/nitrox/nitrox_sriov.c new file mode 100644 index 000000000..43287f847 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_sriov.c @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +#include "nitrox_dev.h" +#include "nitrox_hal.h" +#include "nitrox_common.h" +#include "nitrox_isr.h" +#include "nitrox_mbx.h" + +/** + * num_vfs_valid - validate VF count + * @num_vfs: number of VF(s) + */ +static inline bool num_vfs_valid(int num_vfs) +{ + bool valid = false; + + switch (num_vfs) { + case 16: + case 32: + case 64: + case 128: + valid = true; + break; + } + + return valid; +} + +static inline enum vf_mode num_vfs_to_mode(int num_vfs) +{ + enum vf_mode mode = 0; + + switch (num_vfs) { + case 0: + mode = __NDEV_MODE_PF; + break; + case 16: + mode = __NDEV_MODE_VF16; + break; + case 32: + mode = __NDEV_MODE_VF32; + break; + case 64: + mode = __NDEV_MODE_VF64; + break; + case 128: + mode = __NDEV_MODE_VF128; + break; + } + + return mode; +} + +static inline int vf_mode_to_nr_queues(enum vf_mode mode) +{ + int nr_queues = 0; + + switch (mode) { + case __NDEV_MODE_PF: + nr_queues = MAX_PF_QUEUES; + break; + case __NDEV_MODE_VF16: + nr_queues = 8; + break; + case __NDEV_MODE_VF32: + nr_queues = 4; + break; + case __NDEV_MODE_VF64: + nr_queues = 2; + break; + case __NDEV_MODE_VF128: + nr_queues = 1; + break; + } + + return nr_queues; +} + +static void nitrox_pf_cleanup(struct nitrox_device *ndev) +{ + /* PF has no queues in SR-IOV mode */ + atomic_set(&ndev->state, __NDEV_NOT_READY); + /* unregister crypto algorithms */ + nitrox_crypto_unregister(); + + /* cleanup PF resources */ + nitrox_unregister_interrupts(ndev); + nitrox_common_sw_cleanup(ndev); +} + +/** + * nitrox_pf_reinit - re-initialize PF resources once SR-IOV is disabled + * @ndev: NITROX device + */ +static int nitrox_pf_reinit(struct nitrox_device *ndev) +{ + int err; + + /* allocate resources for PF */ + err = nitrox_common_sw_init(ndev); + if (err) + return err; + + err = nitrox_register_interrupts(ndev); + if (err) { + nitrox_common_sw_cleanup(ndev); + return err; + } + + /* configure the AQM queues */ + nitrox_config_aqm_rings(ndev); + + /* configure the packet queues */ + nitrox_config_pkt_input_rings(ndev); + nitrox_config_pkt_solicit_ports(ndev); + + /* set device to ready state */ + atomic_set(&ndev->state, __NDEV_READY); + + /* register crypto algorithms */ + return nitrox_crypto_register(); +} + +static void nitrox_sriov_cleanup(struct nitrox_device *ndev) +{ + /* unregister interrupts for PF in SR-IOV */ + nitrox_sriov_unregister_interrupts(ndev); + nitrox_mbox_cleanup(ndev); +} + +static int nitrox_sriov_init(struct nitrox_device *ndev) +{ + int ret; + + /* register interrupts for PF in SR-IOV */ + ret = nitrox_sriov_register_interupts(ndev); + if (ret) + return ret; + + ret = nitrox_mbox_init(ndev); + if (ret) + goto sriov_init_fail; + + return 0; + +sriov_init_fail: + nitrox_sriov_cleanup(ndev); + return ret; +} + +static int nitrox_sriov_enable(struct pci_dev *pdev, int num_vfs) +{ + struct nitrox_device *ndev = pci_get_drvdata(pdev); + int err; + + if (!num_vfs_valid(num_vfs)) { + dev_err(DEV(ndev), "Invalid num_vfs %d\n", num_vfs); + return -EINVAL; + } + + if (pci_num_vf(pdev) == num_vfs) + return num_vfs; + + err = pci_enable_sriov(pdev, num_vfs); + if (err) { + dev_err(DEV(ndev), "failed to enable PCI sriov %d\n", err); + return err; + } + dev_info(DEV(ndev), "Enabled VF(s) %d\n", num_vfs); + + ndev->mode = num_vfs_to_mode(num_vfs); + ndev->iov.num_vfs = num_vfs; + ndev->iov.max_vf_queues = vf_mode_to_nr_queues(ndev->mode); + /* set bit in flags */ + set_bit(__NDEV_SRIOV_BIT, &ndev->flags); + + /* cleanup PF resources */ + nitrox_pf_cleanup(ndev); + + /* PF SR-IOV mode initialization */ + err = nitrox_sriov_init(ndev); + if (err) + goto iov_fail; + + config_nps_core_vfcfg_mode(ndev, ndev->mode); + return num_vfs; + +iov_fail: + pci_disable_sriov(pdev); + /* clear bit in flags */ + clear_bit(__NDEV_SRIOV_BIT, &ndev->flags); + ndev->iov.num_vfs = 0; + ndev->mode = __NDEV_MODE_PF; + /* reset back to working mode in PF */ + nitrox_pf_reinit(ndev); + return err; +} + +static int nitrox_sriov_disable(struct pci_dev *pdev) +{ + struct nitrox_device *ndev = pci_get_drvdata(pdev); + + if (!test_bit(__NDEV_SRIOV_BIT, &ndev->flags)) + return 0; + + if (pci_vfs_assigned(pdev)) { + dev_warn(DEV(ndev), "VFs are attached to VM. Can't disable SR-IOV\n"); + return -EPERM; + } + pci_disable_sriov(pdev); + /* clear bit in flags */ + clear_bit(__NDEV_SRIOV_BIT, &ndev->flags); + + ndev->iov.num_vfs = 0; + ndev->iov.max_vf_queues = 0; + ndev->mode = __NDEV_MODE_PF; + + /* cleanup PF SR-IOV resources */ + nitrox_sriov_cleanup(ndev); + + config_nps_core_vfcfg_mode(ndev, ndev->mode); + + return nitrox_pf_reinit(ndev); +} + +int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs) +{ + if (!num_vfs) + return nitrox_sriov_disable(pdev); + + return nitrox_sriov_enable(pdev, num_vfs); +} diff --git a/drivers/crypto/ccp/Kconfig b/drivers/crypto/ccp/Kconfig new file mode 100644 index 000000000..f16a0f611 --- /dev/null +++ b/drivers/crypto/ccp/Kconfig @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: GPL-2.0-only +config CRYPTO_DEV_CCP_DD + tristate "Secure Processor device driver" + depends on CPU_SUP_AMD || ARM64 + default m + help + Provides AMD Secure Processor device driver. + If you choose 'M' here, this module will be called ccp. + +config CRYPTO_DEV_SP_CCP + bool "Cryptographic Coprocessor device" + default y + depends on CRYPTO_DEV_CCP_DD && DMADEVICES + select HW_RANDOM + select DMA_ENGINE + select CRYPTO_SHA1 + select CRYPTO_SHA256 + help + Provides the support for AMD Cryptographic Coprocessor (CCP) device + which can be used to offload encryption operations such as SHA, AES + and more. + +config CRYPTO_DEV_CCP_CRYPTO + tristate "Encryption and hashing offload support" + default m + depends on CRYPTO_DEV_CCP_DD + depends on CRYPTO_DEV_SP_CCP + select CRYPTO_HASH + select CRYPTO_SKCIPHER + select CRYPTO_AUTHENC + select CRYPTO_RSA + select CRYPTO_LIB_AES + help + Support for using the cryptographic API with the AMD Cryptographic + Coprocessor. This module supports offload of SHA and AES algorithms. + If you choose 'M' here, this module will be called ccp_crypto. + +config CRYPTO_DEV_SP_PSP + bool "Platform Security Processor (PSP) device" + default y + depends on CRYPTO_DEV_CCP_DD && X86_64 && AMD_IOMMU + select PCI_TSM if PCI + help + Provide support for the AMD Platform Security Processor (PSP). + The PSP is a dedicated processor that provides support for key + management commands in Secure Encrypted Virtualization (SEV) mode, + along with software-based Trusted Execution Environment (TEE) to + enable third-party trusted applications. + +config CRYPTO_DEV_CCP_DEBUGFS + bool "Enable CCP Internals in DebugFS" + default n + depends on CRYPTO_DEV_SP_CCP + help + Expose CCP device information such as operation statistics, feature + information, and descriptor queue contents. diff --git a/drivers/crypto/ccp/Makefile b/drivers/crypto/ccp/Makefile new file mode 100644 index 000000000..0424e0856 --- /dev/null +++ b/drivers/crypto/ccp/Makefile @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_CRYPTO_DEV_CCP_DD) += ccp.o +ccp-objs := sp-dev.o sp-platform.o +ccp-$(CONFIG_CRYPTO_DEV_SP_CCP) += ccp-dev.o \ + ccp-ops.o \ + ccp-dev-v3.o \ + ccp-dev-v5.o \ + ccp-dmaengine.o +ccp-$(CONFIG_CRYPTO_DEV_CCP_DEBUGFS) += ccp-debugfs.o +ccp-$(CONFIG_PCI) += sp-pci.o +ccp-$(CONFIG_CRYPTO_DEV_SP_PSP) += psp-dev.o \ + sev-dev.o \ + tee-dev.o \ + platform-access.o \ + dbc.o \ + hsti.o \ + sfs.o + +ifeq ($(CONFIG_PCI_TSM),y) +ccp-$(CONFIG_CRYPTO_DEV_SP_PSP) += sev-dev-tsm.o sev-dev-tio.o +endif + +obj-$(CONFIG_CRYPTO_DEV_CCP_CRYPTO) += ccp-crypto.o +ccp-crypto-objs := ccp-crypto-main.o \ + ccp-crypto-aes.o \ + ccp-crypto-aes-cmac.o \ + ccp-crypto-aes-xts.o \ + ccp-crypto-aes-galois.o \ + ccp-crypto-des3.o \ + ccp-crypto-rsa.o \ + ccp-crypto-sha.o diff --git a/drivers/crypto/ccp/ccp-crypto-aes-cmac.c b/drivers/crypto/ccp/ccp-crypto-aes-cmac.c new file mode 100644 index 000000000..ead7566d7 --- /dev/null +++ b/drivers/crypto/ccp/ccp-crypto-aes-cmac.c @@ -0,0 +1,402 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Cryptographic Coprocessor (CCP) AES CMAC crypto API support + * + * Copyright (C) 2013,2018 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ccp-crypto.h" + +static int ccp_aes_cmac_complete(struct crypto_async_request *async_req, + int ret) +{ + struct ahash_request *req = ahash_request_cast(async_req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx_dma(req); + unsigned int digest_size = crypto_ahash_digestsize(tfm); + + if (ret) + goto e_free; + + if (rctx->hash_rem) { + /* Save remaining data to buffer */ + unsigned int offset = rctx->nbytes - rctx->hash_rem; + + scatterwalk_map_and_copy(rctx->buf, rctx->src, + offset, rctx->hash_rem, 0); + rctx->buf_count = rctx->hash_rem; + } else { + rctx->buf_count = 0; + } + + /* Update result area if supplied */ + if (req->result && rctx->final) + memcpy(req->result, rctx->iv, digest_size); + +e_free: + sg_free_table(&rctx->data_sg); + + return ret; +} + +static int ccp_do_cmac_update(struct ahash_request *req, unsigned int nbytes, + unsigned int final) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ccp_ctx *ctx = crypto_ahash_ctx_dma(tfm); + struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx_dma(req); + struct scatterlist *sg, *cmac_key_sg = NULL; + unsigned int block_size = + crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + unsigned int need_pad, sg_count; + gfp_t gfp; + u64 len; + int ret; + + if (!ctx->u.aes.key_len) + return -EINVAL; + + if (nbytes) + rctx->null_msg = 0; + + len = (u64)rctx->buf_count + (u64)nbytes; + + if (!final && (len <= block_size)) { + scatterwalk_map_and_copy(rctx->buf + rctx->buf_count, req->src, + 0, nbytes, 0); + rctx->buf_count += nbytes; + + return 0; + } + + rctx->src = req->src; + rctx->nbytes = nbytes; + + rctx->final = final; + rctx->hash_rem = final ? 0 : len & (block_size - 1); + rctx->hash_cnt = len - rctx->hash_rem; + if (!final && !rctx->hash_rem) { + /* CCP can't do zero length final, so keep some data around */ + rctx->hash_cnt -= block_size; + rctx->hash_rem = block_size; + } + + if (final && (rctx->null_msg || (len & (block_size - 1)))) + need_pad = 1; + else + need_pad = 0; + + sg_init_one(&rctx->iv_sg, rctx->iv, sizeof(rctx->iv)); + + /* Build the data scatterlist table - allocate enough entries for all + * possible data pieces (buffer, input data, padding) + */ + sg_count = (nbytes) ? sg_nents(req->src) + 2 : 2; + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + ret = sg_alloc_table(&rctx->data_sg, sg_count, gfp); + if (ret) + return ret; + + sg = NULL; + if (rctx->buf_count) { + sg_init_one(&rctx->buf_sg, rctx->buf, rctx->buf_count); + sg = ccp_crypto_sg_table_add(&rctx->data_sg, &rctx->buf_sg); + if (!sg) { + ret = -EINVAL; + goto e_free; + } + } + + if (nbytes) { + sg = ccp_crypto_sg_table_add(&rctx->data_sg, req->src); + if (!sg) { + ret = -EINVAL; + goto e_free; + } + } + + if (need_pad) { + int pad_length = block_size - (len & (block_size - 1)); + + rctx->hash_cnt += pad_length; + + memset(rctx->pad, 0, sizeof(rctx->pad)); + rctx->pad[0] = 0x80; + sg_init_one(&rctx->pad_sg, rctx->pad, pad_length); + sg = ccp_crypto_sg_table_add(&rctx->data_sg, &rctx->pad_sg); + if (!sg) { + ret = -EINVAL; + goto e_free; + } + } + if (sg) { + sg_mark_end(sg); + sg = rctx->data_sg.sgl; + } + + /* Initialize the K1/K2 scatterlist */ + if (final) + cmac_key_sg = (need_pad) ? &ctx->u.aes.k2_sg + : &ctx->u.aes.k1_sg; + + memset(&rctx->cmd, 0, sizeof(rctx->cmd)); + INIT_LIST_HEAD(&rctx->cmd.entry); + rctx->cmd.engine = CCP_ENGINE_AES; + rctx->cmd.u.aes.type = ctx->u.aes.type; + rctx->cmd.u.aes.mode = ctx->u.aes.mode; + rctx->cmd.u.aes.action = CCP_AES_ACTION_ENCRYPT; + rctx->cmd.u.aes.key = &ctx->u.aes.key_sg; + rctx->cmd.u.aes.key_len = ctx->u.aes.key_len; + rctx->cmd.u.aes.iv = &rctx->iv_sg; + rctx->cmd.u.aes.iv_len = AES_BLOCK_SIZE; + rctx->cmd.u.aes.src = sg; + rctx->cmd.u.aes.src_len = rctx->hash_cnt; + rctx->cmd.u.aes.dst = NULL; + rctx->cmd.u.aes.cmac_key = cmac_key_sg; + rctx->cmd.u.aes.cmac_key_len = ctx->u.aes.kn_len; + rctx->cmd.u.aes.cmac_final = final; + + ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd); + + return ret; + +e_free: + sg_free_table(&rctx->data_sg); + + return ret; +} + +static int ccp_aes_cmac_init(struct ahash_request *req) +{ + struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx_dma(req); + + memset(rctx, 0, sizeof(*rctx)); + + rctx->null_msg = 1; + + return 0; +} + +static int ccp_aes_cmac_update(struct ahash_request *req) +{ + return ccp_do_cmac_update(req, req->nbytes, 0); +} + +static int ccp_aes_cmac_final(struct ahash_request *req) +{ + return ccp_do_cmac_update(req, 0, 1); +} + +static int ccp_aes_cmac_finup(struct ahash_request *req) +{ + return ccp_do_cmac_update(req, req->nbytes, 1); +} + +static int ccp_aes_cmac_digest(struct ahash_request *req) +{ + int ret; + + ret = ccp_aes_cmac_init(req); + if (ret) + return ret; + + return ccp_aes_cmac_finup(req); +} + +static int ccp_aes_cmac_export(struct ahash_request *req, void *out) +{ + struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx_dma(req); + struct ccp_aes_cmac_exp_ctx state; + + /* Don't let anything leak to 'out' */ + memset(&state, 0, sizeof(state)); + + state.null_msg = rctx->null_msg; + memcpy(state.iv, rctx->iv, sizeof(state.iv)); + state.buf_count = rctx->buf_count; + memcpy(state.buf, rctx->buf, sizeof(state.buf)); + + /* 'out' may not be aligned so memcpy from local variable */ + memcpy(out, &state, sizeof(state)); + + return 0; +} + +static int ccp_aes_cmac_import(struct ahash_request *req, const void *in) +{ + struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx_dma(req); + struct ccp_aes_cmac_exp_ctx state; + + /* 'in' may not be aligned so memcpy to local variable */ + memcpy(&state, in, sizeof(state)); + + memset(rctx, 0, sizeof(*rctx)); + rctx->null_msg = state.null_msg; + memcpy(rctx->iv, state.iv, sizeof(rctx->iv)); + rctx->buf_count = state.buf_count; + memcpy(rctx->buf, state.buf, sizeof(rctx->buf)); + + return 0; +} + +static int ccp_aes_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int key_len) +{ + struct ccp_ctx *ctx = crypto_ahash_ctx_dma(tfm); + struct ccp_crypto_ahash_alg *alg = + ccp_crypto_ahash_alg(crypto_ahash_tfm(tfm)); + u64 k0_hi, k0_lo, k1_hi, k1_lo, k2_hi, k2_lo; + u64 rb_hi = 0x00, rb_lo = 0x87; + struct aes_enckey aes; + __be64 *gk; + int ret; + + switch (key_len) { + case AES_KEYSIZE_128: + ctx->u.aes.type = CCP_AES_TYPE_128; + break; + case AES_KEYSIZE_192: + ctx->u.aes.type = CCP_AES_TYPE_192; + break; + case AES_KEYSIZE_256: + ctx->u.aes.type = CCP_AES_TYPE_256; + break; + default: + return -EINVAL; + } + ctx->u.aes.mode = alg->mode; + + /* Set to zero until complete */ + ctx->u.aes.key_len = 0; + + /* Set the key for the AES cipher used to generate the keys */ + ret = aes_prepareenckey(&aes, key, key_len); + if (ret) + return ret; + + /* Encrypt a block of zeroes - use key area in context */ + memset(ctx->u.aes.key, 0, sizeof(ctx->u.aes.key)); + aes_encrypt(&aes, ctx->u.aes.key, ctx->u.aes.key); + memzero_explicit(&aes, sizeof(aes)); + + /* Generate K1 and K2 */ + k0_hi = be64_to_cpu(*((__be64 *)ctx->u.aes.key)); + k0_lo = be64_to_cpu(*((__be64 *)ctx->u.aes.key + 1)); + + k1_hi = (k0_hi << 1) | (k0_lo >> 63); + k1_lo = k0_lo << 1; + if (ctx->u.aes.key[0] & 0x80) { + k1_hi ^= rb_hi; + k1_lo ^= rb_lo; + } + gk = (__be64 *)ctx->u.aes.k1; + *gk = cpu_to_be64(k1_hi); + gk++; + *gk = cpu_to_be64(k1_lo); + + k2_hi = (k1_hi << 1) | (k1_lo >> 63); + k2_lo = k1_lo << 1; + if (ctx->u.aes.k1[0] & 0x80) { + k2_hi ^= rb_hi; + k2_lo ^= rb_lo; + } + gk = (__be64 *)ctx->u.aes.k2; + *gk = cpu_to_be64(k2_hi); + gk++; + *gk = cpu_to_be64(k2_lo); + + ctx->u.aes.kn_len = sizeof(ctx->u.aes.k1); + sg_init_one(&ctx->u.aes.k1_sg, ctx->u.aes.k1, sizeof(ctx->u.aes.k1)); + sg_init_one(&ctx->u.aes.k2_sg, ctx->u.aes.k2, sizeof(ctx->u.aes.k2)); + + /* Save the supplied key */ + memset(ctx->u.aes.key, 0, sizeof(ctx->u.aes.key)); + memcpy(ctx->u.aes.key, key, key_len); + ctx->u.aes.key_len = key_len; + sg_init_one(&ctx->u.aes.key_sg, ctx->u.aes.key, key_len); + + return ret; +} + +static int ccp_aes_cmac_cra_init(struct crypto_tfm *tfm) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx_dma(tfm); + struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); + + ctx->complete = ccp_aes_cmac_complete; + ctx->u.aes.key_len = 0; + + crypto_ahash_set_reqsize_dma(ahash, + sizeof(struct ccp_aes_cmac_req_ctx)); + + return 0; +} + +int ccp_register_aes_cmac_algs(struct list_head *head) +{ + struct ccp_crypto_ahash_alg *ccp_alg; + struct ahash_alg *alg; + struct hash_alg_common *halg; + struct crypto_alg *base; + int ret; + + ccp_alg = kzalloc_obj(*ccp_alg); + if (!ccp_alg) + return -ENOMEM; + + INIT_LIST_HEAD(&ccp_alg->entry); + ccp_alg->mode = CCP_AES_MODE_CMAC; + + alg = &ccp_alg->alg; + alg->init = ccp_aes_cmac_init; + alg->update = ccp_aes_cmac_update; + alg->final = ccp_aes_cmac_final; + alg->finup = ccp_aes_cmac_finup; + alg->digest = ccp_aes_cmac_digest; + alg->export = ccp_aes_cmac_export; + alg->import = ccp_aes_cmac_import; + alg->setkey = ccp_aes_cmac_setkey; + + halg = &alg->halg; + halg->digestsize = AES_BLOCK_SIZE; + halg->statesize = sizeof(struct ccp_aes_cmac_exp_ctx); + + base = &halg->base; + snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "cmac(aes)"); + snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "cmac-aes-ccp"); + base->cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK; + base->cra_blocksize = AES_BLOCK_SIZE; + base->cra_ctxsize = sizeof(struct ccp_ctx) + crypto_dma_padding(); + base->cra_priority = CCP_CRA_PRIORITY; + base->cra_init = ccp_aes_cmac_cra_init; + base->cra_module = THIS_MODULE; + + ret = crypto_register_ahash(alg); + if (ret) { + pr_err("%s ahash algorithm registration error (%d)\n", + base->cra_name, ret); + kfree(ccp_alg); + return ret; + } + + list_add(&ccp_alg->entry, head); + + return 0; +} diff --git a/drivers/crypto/ccp/ccp-crypto-aes-galois.c b/drivers/crypto/ccp/ccp-crypto-aes-galois.c new file mode 100644 index 000000000..fc14c2e73 --- /dev/null +++ b/drivers/crypto/ccp/ccp-crypto-aes-galois.c @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Cryptographic Coprocessor (CCP) AES GCM crypto API support + * + * Copyright (C) 2016,2017 Advanced Micro Devices, Inc. + * + * Author: Gary R Hook + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ccp-crypto.h" + +static int ccp_aes_gcm_complete(struct crypto_async_request *async_req, int ret) +{ + return ret; +} + +static int ccp_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int key_len) +{ + struct ccp_ctx *ctx = crypto_aead_ctx_dma(tfm); + + switch (key_len) { + case AES_KEYSIZE_128: + ctx->u.aes.type = CCP_AES_TYPE_128; + break; + case AES_KEYSIZE_192: + ctx->u.aes.type = CCP_AES_TYPE_192; + break; + case AES_KEYSIZE_256: + ctx->u.aes.type = CCP_AES_TYPE_256; + break; + default: + return -EINVAL; + } + + ctx->u.aes.mode = CCP_AES_MODE_GCM; + ctx->u.aes.key_len = key_len; + + memcpy(ctx->u.aes.key, key, key_len); + sg_init_one(&ctx->u.aes.key_sg, ctx->u.aes.key, key_len); + + return 0; +} + +static int ccp_aes_gcm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + switch (authsize) { + case 16: + case 15: + case 14: + case 13: + case 12: + case 8: + case 4: + break; + default: + return -EINVAL; + } + + return 0; +} + +static int ccp_aes_gcm_crypt(struct aead_request *req, bool encrypt) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct ccp_ctx *ctx = crypto_aead_ctx_dma(tfm); + struct ccp_aes_req_ctx *rctx = aead_request_ctx_dma(req); + struct scatterlist *iv_sg = NULL; + unsigned int iv_len = 0; + int i; + int ret = 0; + + if (!ctx->u.aes.key_len) + return -EINVAL; + + if (ctx->u.aes.mode != CCP_AES_MODE_GCM) + return -EINVAL; + + if (!req->iv) + return -EINVAL; + + /* + * 5 parts: + * plaintext/ciphertext input + * AAD + * key + * IV + * Destination+tag buffer + */ + + /* Prepare the IV: 12 bytes + an integer (counter) */ + memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); + for (i = 0; i < 3; i++) + rctx->iv[i + GCM_AES_IV_SIZE] = 0; + rctx->iv[AES_BLOCK_SIZE - 1] = 1; + + /* Set up a scatterlist for the IV */ + iv_sg = &rctx->iv_sg; + iv_len = AES_BLOCK_SIZE; + sg_init_one(iv_sg, rctx->iv, iv_len); + + /* The AAD + plaintext are concatenated in the src buffer */ + memset(&rctx->cmd, 0, sizeof(rctx->cmd)); + INIT_LIST_HEAD(&rctx->cmd.entry); + rctx->cmd.engine = CCP_ENGINE_AES; + rctx->cmd.u.aes.authsize = crypto_aead_authsize(tfm); + rctx->cmd.u.aes.type = ctx->u.aes.type; + rctx->cmd.u.aes.mode = ctx->u.aes.mode; + rctx->cmd.u.aes.action = encrypt; + rctx->cmd.u.aes.key = &ctx->u.aes.key_sg; + rctx->cmd.u.aes.key_len = ctx->u.aes.key_len; + rctx->cmd.u.aes.iv = iv_sg; + rctx->cmd.u.aes.iv_len = iv_len; + rctx->cmd.u.aes.src = req->src; + rctx->cmd.u.aes.src_len = req->cryptlen; + rctx->cmd.u.aes.aad_len = req->assoclen; + + /* The cipher text + the tag are in the dst buffer */ + rctx->cmd.u.aes.dst = req->dst; + + ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd); + + return ret; +} + +static int ccp_aes_gcm_encrypt(struct aead_request *req) +{ + return ccp_aes_gcm_crypt(req, CCP_AES_ACTION_ENCRYPT); +} + +static int ccp_aes_gcm_decrypt(struct aead_request *req) +{ + return ccp_aes_gcm_crypt(req, CCP_AES_ACTION_DECRYPT); +} + +static int ccp_aes_gcm_cra_init(struct crypto_aead *tfm) +{ + struct ccp_ctx *ctx = crypto_aead_ctx_dma(tfm); + + ctx->complete = ccp_aes_gcm_complete; + ctx->u.aes.key_len = 0; + + crypto_aead_set_reqsize_dma(tfm, sizeof(struct ccp_aes_req_ctx)); + + return 0; +} + +static void ccp_aes_gcm_cra_exit(struct crypto_tfm *tfm) +{ +} + +static struct aead_alg ccp_aes_gcm_defaults = { + .setkey = ccp_aes_gcm_setkey, + .setauthsize = ccp_aes_gcm_setauthsize, + .encrypt = ccp_aes_gcm_encrypt, + .decrypt = ccp_aes_gcm_decrypt, + .init = ccp_aes_gcm_cra_init, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .base = { + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ccp_ctx) + CRYPTO_DMA_PADDING, + .cra_priority = CCP_CRA_PRIORITY, + .cra_exit = ccp_aes_gcm_cra_exit, + .cra_module = THIS_MODULE, + }, +}; + +struct ccp_aes_aead_def { + enum ccp_aes_mode mode; + unsigned int version; + const char *name; + const char *driver_name; + unsigned int blocksize; + unsigned int ivsize; + struct aead_alg *alg_defaults; +}; + +static struct ccp_aes_aead_def aes_aead_algs[] = { + { + .mode = CCP_AES_MODE_GHASH, + .version = CCP_VERSION(5, 0), + .name = "gcm(aes)", + .driver_name = "gcm-aes-ccp", + .blocksize = 1, + .ivsize = AES_BLOCK_SIZE, + .alg_defaults = &ccp_aes_gcm_defaults, + }, +}; + +static int ccp_register_aes_aead(struct list_head *head, + const struct ccp_aes_aead_def *def) +{ + struct ccp_crypto_aead *ccp_aead; + struct aead_alg *alg; + int ret; + + ccp_aead = kzalloc_obj(*ccp_aead); + if (!ccp_aead) + return -ENOMEM; + + INIT_LIST_HEAD(&ccp_aead->entry); + + ccp_aead->mode = def->mode; + + /* Copy the defaults and override as necessary */ + alg = &ccp_aead->alg; + *alg = *def->alg_defaults; + strscpy(alg->base.cra_name, def->name); + strscpy(alg->base.cra_driver_name, def->driver_name); + alg->base.cra_blocksize = def->blocksize; + + ret = crypto_register_aead(alg); + if (ret) { + pr_err("%s aead algorithm registration error (%d)\n", + alg->base.cra_name, ret); + kfree(ccp_aead); + return ret; + } + + list_add(&ccp_aead->entry, head); + + return 0; +} + +int ccp_register_aes_aeads(struct list_head *head) +{ + int i, ret; + unsigned int ccpversion = ccp_version(); + + for (i = 0; i < ARRAY_SIZE(aes_aead_algs); i++) { + if (aes_aead_algs[i].version > ccpversion) + continue; + ret = ccp_register_aes_aead(head, &aes_aead_algs[i]); + if (ret) + return ret; + } + + return 0; +} diff --git a/drivers/crypto/ccp/ccp-crypto-aes-xts.c b/drivers/crypto/ccp/ccp-crypto-aes-xts.c new file mode 100644 index 000000000..8e5913728 --- /dev/null +++ b/drivers/crypto/ccp/ccp-crypto-aes-xts.c @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Cryptographic Coprocessor (CCP) AES XTS crypto API support + * + * Copyright (C) 2013,2017 Advanced Micro Devices, Inc. + * + * Author: Gary R Hook + * Author: Tom Lendacky + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ccp-crypto.h" + +struct ccp_aes_xts_def { + const char *name; + const char *drv_name; +}; + +static const struct ccp_aes_xts_def aes_xts_algs[] = { + { + .name = "xts(aes)", + .drv_name = "xts-aes-ccp", + }, +}; + +struct ccp_unit_size_map { + unsigned int size; + u32 value; +}; + +static struct ccp_unit_size_map xts_unit_sizes[] = { + { + .size = 16, + .value = CCP_XTS_AES_UNIT_SIZE_16, + }, + { + .size = 512, + .value = CCP_XTS_AES_UNIT_SIZE_512, + }, + { + .size = 1024, + .value = CCP_XTS_AES_UNIT_SIZE_1024, + }, + { + .size = 2048, + .value = CCP_XTS_AES_UNIT_SIZE_2048, + }, + { + .size = 4096, + .value = CCP_XTS_AES_UNIT_SIZE_4096, + }, +}; + +static int ccp_aes_xts_complete(struct crypto_async_request *async_req, int ret) +{ + struct skcipher_request *req = skcipher_request_cast(async_req); + struct ccp_aes_req_ctx *rctx = skcipher_request_ctx_dma(req); + + if (ret) + return ret; + + memcpy(req->iv, rctx->iv, AES_BLOCK_SIZE); + + return 0; +} + +static int ccp_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int key_len) +{ + struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm); + unsigned int ccpversion = ccp_version(); + int ret; + + ret = xts_verify_key(tfm, key, key_len); + if (ret) + return ret; + + /* Version 3 devices support 128-bit keys; version 5 devices can + * accommodate 128- and 256-bit keys. + */ + switch (key_len) { + case AES_KEYSIZE_128 * 2: + memcpy(ctx->u.aes.key, key, key_len); + break; + case AES_KEYSIZE_256 * 2: + if (ccpversion > CCP_VERSION(3, 0)) + memcpy(ctx->u.aes.key, key, key_len); + break; + } + ctx->u.aes.key_len = key_len / 2; + sg_init_one(&ctx->u.aes.key_sg, ctx->u.aes.key, key_len); + + return crypto_skcipher_setkey(ctx->u.aes.tfm_skcipher, key, key_len); +} + +static int ccp_aes_xts_crypt(struct skcipher_request *req, + unsigned int encrypt) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm); + struct ccp_aes_req_ctx *rctx = skcipher_request_ctx_dma(req); + unsigned int ccpversion = ccp_version(); + unsigned int fallback = 0; + unsigned int unit; + u32 unit_size; + int ret; + + if (!ctx->u.aes.key_len) + return -EINVAL; + + if (!req->iv) + return -EINVAL; + + /* Check conditions under which the CCP can fulfill a request. The + * device can handle input plaintext of a length that is a multiple + * of the unit_size, bug the crypto implementation only supports + * the unit_size being equal to the input length. This limits the + * number of scenarios we can handle. + */ + unit_size = CCP_XTS_AES_UNIT_SIZE__LAST; + for (unit = 0; unit < ARRAY_SIZE(xts_unit_sizes); unit++) { + if (req->cryptlen == xts_unit_sizes[unit].size) { + unit_size = unit; + break; + } + } + /* The CCP has restrictions on block sizes. Also, a version 3 device + * only supports AES-128 operations; version 5 CCPs support both + * AES-128 and -256 operations. + */ + if (unit_size == CCP_XTS_AES_UNIT_SIZE__LAST) + fallback = 1; + if ((ccpversion < CCP_VERSION(5, 0)) && + (ctx->u.aes.key_len != AES_KEYSIZE_128)) + fallback = 1; + if ((ctx->u.aes.key_len != AES_KEYSIZE_128) && + (ctx->u.aes.key_len != AES_KEYSIZE_256)) + fallback = 1; + if (fallback) { + /* Use the fallback to process the request for any + * unsupported unit sizes or key sizes + */ + skcipher_request_set_tfm(&rctx->fallback_req, + ctx->u.aes.tfm_skcipher); + skcipher_request_set_callback(&rctx->fallback_req, + req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(&rctx->fallback_req, req->src, + req->dst, req->cryptlen, req->iv); + ret = encrypt ? crypto_skcipher_encrypt(&rctx->fallback_req) : + crypto_skcipher_decrypt(&rctx->fallback_req); + return ret; + } + + memcpy(rctx->iv, req->iv, AES_BLOCK_SIZE); + sg_init_one(&rctx->iv_sg, rctx->iv, AES_BLOCK_SIZE); + + memset(&rctx->cmd, 0, sizeof(rctx->cmd)); + INIT_LIST_HEAD(&rctx->cmd.entry); + rctx->cmd.engine = CCP_ENGINE_XTS_AES_128; + rctx->cmd.u.xts.type = CCP_AES_TYPE_128; + rctx->cmd.u.xts.action = (encrypt) ? CCP_AES_ACTION_ENCRYPT + : CCP_AES_ACTION_DECRYPT; + rctx->cmd.u.xts.unit_size = unit_size; + rctx->cmd.u.xts.key = &ctx->u.aes.key_sg; + rctx->cmd.u.xts.key_len = ctx->u.aes.key_len; + rctx->cmd.u.xts.iv = &rctx->iv_sg; + rctx->cmd.u.xts.iv_len = AES_BLOCK_SIZE; + rctx->cmd.u.xts.src = req->src; + rctx->cmd.u.xts.src_len = req->cryptlen; + rctx->cmd.u.xts.dst = req->dst; + + ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd); + + return ret; +} + +static int ccp_aes_xts_encrypt(struct skcipher_request *req) +{ + return ccp_aes_xts_crypt(req, 1); +} + +static int ccp_aes_xts_decrypt(struct skcipher_request *req) +{ + return ccp_aes_xts_crypt(req, 0); +} + +static int ccp_aes_xts_init_tfm(struct crypto_skcipher *tfm) +{ + struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm); + struct crypto_skcipher *fallback_tfm; + + ctx->complete = ccp_aes_xts_complete; + ctx->u.aes.key_len = 0; + + fallback_tfm = crypto_alloc_skcipher("xts(aes)", 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(fallback_tfm)) { + pr_warn("could not load fallback driver xts(aes)\n"); + return PTR_ERR(fallback_tfm); + } + ctx->u.aes.tfm_skcipher = fallback_tfm; + + crypto_skcipher_set_reqsize_dma(tfm, + sizeof(struct ccp_aes_req_ctx) + + crypto_skcipher_reqsize(fallback_tfm)); + + return 0; +} + +static void ccp_aes_xts_exit_tfm(struct crypto_skcipher *tfm) +{ + struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm); + + crypto_free_skcipher(ctx->u.aes.tfm_skcipher); +} + +static int ccp_register_aes_xts_alg(struct list_head *head, + const struct ccp_aes_xts_def *def) +{ + struct ccp_crypto_skcipher_alg *ccp_alg; + struct skcipher_alg *alg; + int ret; + + ccp_alg = kzalloc_obj(*ccp_alg); + if (!ccp_alg) + return -ENOMEM; + + INIT_LIST_HEAD(&ccp_alg->entry); + + alg = &ccp_alg->alg; + + strscpy(alg->base.cra_name, def->name); + strscpy(alg->base.cra_driver_name, def->drv_name); + alg->base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK; + alg->base.cra_blocksize = AES_BLOCK_SIZE; + alg->base.cra_ctxsize = sizeof(struct ccp_ctx) + + crypto_dma_padding(); + alg->base.cra_priority = CCP_CRA_PRIORITY; + alg->base.cra_module = THIS_MODULE; + + alg->setkey = ccp_aes_xts_setkey; + alg->encrypt = ccp_aes_xts_encrypt; + alg->decrypt = ccp_aes_xts_decrypt; + alg->min_keysize = AES_MIN_KEY_SIZE * 2; + alg->max_keysize = AES_MAX_KEY_SIZE * 2; + alg->ivsize = AES_BLOCK_SIZE; + alg->init = ccp_aes_xts_init_tfm; + alg->exit = ccp_aes_xts_exit_tfm; + + ret = crypto_register_skcipher(alg); + if (ret) { + pr_err("%s skcipher algorithm registration error (%d)\n", + alg->base.cra_name, ret); + kfree(ccp_alg); + return ret; + } + + list_add(&ccp_alg->entry, head); + + return 0; +} + +int ccp_register_aes_xts_algs(struct list_head *head) +{ + int i, ret; + + for (i = 0; i < ARRAY_SIZE(aes_xts_algs); i++) { + ret = ccp_register_aes_xts_alg(head, &aes_xts_algs[i]); + if (ret) + return ret; + } + + return 0; +} diff --git a/drivers/crypto/ccp/ccp-crypto-aes.c b/drivers/crypto/ccp/ccp-crypto-aes.c new file mode 100644 index 000000000..f475819b6 --- /dev/null +++ b/drivers/crypto/ccp/ccp-crypto-aes.c @@ -0,0 +1,343 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Cryptographic Coprocessor (CCP) AES crypto API support + * + * Copyright (C) 2013-2019 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ccp-crypto.h" + +static int ccp_aes_complete(struct crypto_async_request *async_req, int ret) +{ + struct skcipher_request *req = skcipher_request_cast(async_req); + struct ccp_ctx *ctx = crypto_skcipher_ctx_dma( + crypto_skcipher_reqtfm(req)); + struct ccp_aes_req_ctx *rctx = skcipher_request_ctx_dma(req); + + if (ret) + return ret; + + if (ctx->u.aes.mode != CCP_AES_MODE_ECB) { + size_t ivsize = crypto_skcipher_ivsize(crypto_skcipher_reqtfm(req)); + + memcpy(req->iv, rctx->iv, ivsize); + } + + return 0; +} + +static int ccp_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int key_len) +{ + struct ccp_crypto_skcipher_alg *alg = ccp_crypto_skcipher_alg(tfm); + struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm); + + switch (key_len) { + case AES_KEYSIZE_128: + ctx->u.aes.type = CCP_AES_TYPE_128; + break; + case AES_KEYSIZE_192: + ctx->u.aes.type = CCP_AES_TYPE_192; + break; + case AES_KEYSIZE_256: + ctx->u.aes.type = CCP_AES_TYPE_256; + break; + default: + return -EINVAL; + } + ctx->u.aes.mode = alg->mode; + ctx->u.aes.key_len = key_len; + + memcpy(ctx->u.aes.key, key, key_len); + sg_init_one(&ctx->u.aes.key_sg, ctx->u.aes.key, key_len); + + return 0; +} + +static int ccp_aes_crypt(struct skcipher_request *req, bool encrypt) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm); + struct ccp_aes_req_ctx *rctx = skcipher_request_ctx_dma(req); + struct scatterlist *iv_sg = NULL; + unsigned int iv_len = 0; + + if (!ctx->u.aes.key_len) + return -EINVAL; + + if (((ctx->u.aes.mode == CCP_AES_MODE_ECB) || + (ctx->u.aes.mode == CCP_AES_MODE_CBC)) && + (req->cryptlen & (AES_BLOCK_SIZE - 1))) + return -EINVAL; + + if (ctx->u.aes.mode != CCP_AES_MODE_ECB) { + if (!req->iv) + return -EINVAL; + + memcpy(rctx->iv, req->iv, AES_BLOCK_SIZE); + iv_sg = &rctx->iv_sg; + iv_len = AES_BLOCK_SIZE; + sg_init_one(iv_sg, rctx->iv, iv_len); + } + + memset(&rctx->cmd, 0, sizeof(rctx->cmd)); + INIT_LIST_HEAD(&rctx->cmd.entry); + rctx->cmd.engine = CCP_ENGINE_AES; + rctx->cmd.u.aes.type = ctx->u.aes.type; + rctx->cmd.u.aes.mode = ctx->u.aes.mode; + rctx->cmd.u.aes.action = + (encrypt) ? CCP_AES_ACTION_ENCRYPT : CCP_AES_ACTION_DECRYPT; + rctx->cmd.u.aes.key = &ctx->u.aes.key_sg; + rctx->cmd.u.aes.key_len = ctx->u.aes.key_len; + rctx->cmd.u.aes.iv = iv_sg; + rctx->cmd.u.aes.iv_len = iv_len; + rctx->cmd.u.aes.src = req->src; + rctx->cmd.u.aes.src_len = req->cryptlen; + rctx->cmd.u.aes.dst = req->dst; + + return ccp_crypto_enqueue_request(&req->base, &rctx->cmd); +} + +static int ccp_aes_encrypt(struct skcipher_request *req) +{ + return ccp_aes_crypt(req, true); +} + +static int ccp_aes_decrypt(struct skcipher_request *req) +{ + return ccp_aes_crypt(req, false); +} + +static int ccp_aes_init_tfm(struct crypto_skcipher *tfm) +{ + struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm); + + ctx->complete = ccp_aes_complete; + ctx->u.aes.key_len = 0; + + crypto_skcipher_set_reqsize(tfm, sizeof(struct ccp_aes_req_ctx)); + + return 0; +} + +static int ccp_aes_rfc3686_complete(struct crypto_async_request *async_req, + int ret) +{ + struct skcipher_request *req = skcipher_request_cast(async_req); + struct ccp_aes_req_ctx *rctx = skcipher_request_ctx_dma(req); + + /* Restore the original pointer */ + req->iv = rctx->rfc3686_info; + + return ccp_aes_complete(async_req, ret); +} + +static int ccp_aes_rfc3686_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int key_len) +{ + struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm); + + if (key_len < CTR_RFC3686_NONCE_SIZE) + return -EINVAL; + + key_len -= CTR_RFC3686_NONCE_SIZE; + memcpy(ctx->u.aes.nonce, key + key_len, CTR_RFC3686_NONCE_SIZE); + + return ccp_aes_setkey(tfm, key, key_len); +} + +static int ccp_aes_rfc3686_crypt(struct skcipher_request *req, bool encrypt) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm); + struct ccp_aes_req_ctx *rctx = skcipher_request_ctx_dma(req); + u8 *iv; + + /* Initialize the CTR block */ + iv = rctx->rfc3686_iv; + memcpy(iv, ctx->u.aes.nonce, CTR_RFC3686_NONCE_SIZE); + + iv += CTR_RFC3686_NONCE_SIZE; + memcpy(iv, req->iv, CTR_RFC3686_IV_SIZE); + + iv += CTR_RFC3686_IV_SIZE; + *(__be32 *)iv = cpu_to_be32(1); + + /* Point to the new IV */ + rctx->rfc3686_info = req->iv; + req->iv = rctx->rfc3686_iv; + + return ccp_aes_crypt(req, encrypt); +} + +static int ccp_aes_rfc3686_encrypt(struct skcipher_request *req) +{ + return ccp_aes_rfc3686_crypt(req, true); +} + +static int ccp_aes_rfc3686_decrypt(struct skcipher_request *req) +{ + return ccp_aes_rfc3686_crypt(req, false); +} + +static int ccp_aes_rfc3686_init_tfm(struct crypto_skcipher *tfm) +{ + struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm); + + ctx->complete = ccp_aes_rfc3686_complete; + ctx->u.aes.key_len = 0; + + crypto_skcipher_set_reqsize_dma(tfm, sizeof(struct ccp_aes_req_ctx)); + + return 0; +} + +static const struct skcipher_alg ccp_aes_defaults = { + .setkey = ccp_aes_setkey, + .encrypt = ccp_aes_encrypt, + .decrypt = ccp_aes_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .init = ccp_aes_init_tfm, + + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ccp_ctx) + CRYPTO_DMA_PADDING, + .base.cra_priority = CCP_CRA_PRIORITY, + .base.cra_module = THIS_MODULE, +}; + +static const struct skcipher_alg ccp_aes_rfc3686_defaults = { + .setkey = ccp_aes_rfc3686_setkey, + .encrypt = ccp_aes_rfc3686_encrypt, + .decrypt = ccp_aes_rfc3686_decrypt, + .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .init = ccp_aes_rfc3686_init_tfm, + + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = CTR_RFC3686_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ccp_ctx) + CRYPTO_DMA_PADDING, + .base.cra_priority = CCP_CRA_PRIORITY, + .base.cra_module = THIS_MODULE, +}; + +struct ccp_aes_def { + enum ccp_aes_mode mode; + unsigned int version; + const char *name; + const char *driver_name; + unsigned int blocksize; + unsigned int ivsize; + const struct skcipher_alg *alg_defaults; +}; + +static struct ccp_aes_def aes_algs[] = { + { + .mode = CCP_AES_MODE_ECB, + .version = CCP_VERSION(3, 0), + .name = "ecb(aes)", + .driver_name = "ecb-aes-ccp", + .blocksize = AES_BLOCK_SIZE, + .ivsize = 0, + .alg_defaults = &ccp_aes_defaults, + }, + { + .mode = CCP_AES_MODE_CBC, + .version = CCP_VERSION(3, 0), + .name = "cbc(aes)", + .driver_name = "cbc-aes-ccp", + .blocksize = AES_BLOCK_SIZE, + .ivsize = AES_BLOCK_SIZE, + .alg_defaults = &ccp_aes_defaults, + }, + { + .mode = CCP_AES_MODE_CTR, + .version = CCP_VERSION(3, 0), + .name = "ctr(aes)", + .driver_name = "ctr-aes-ccp", + .blocksize = 1, + .ivsize = AES_BLOCK_SIZE, + .alg_defaults = &ccp_aes_defaults, + }, + { + .mode = CCP_AES_MODE_CTR, + .version = CCP_VERSION(3, 0), + .name = "rfc3686(ctr(aes))", + .driver_name = "rfc3686-ctr-aes-ccp", + .blocksize = 1, + .ivsize = CTR_RFC3686_IV_SIZE, + .alg_defaults = &ccp_aes_rfc3686_defaults, + }, +}; + +static int ccp_register_aes_alg(struct list_head *head, + const struct ccp_aes_def *def) +{ + struct ccp_crypto_skcipher_alg *ccp_alg; + struct skcipher_alg *alg; + int ret; + + ccp_alg = kzalloc_obj(*ccp_alg); + if (!ccp_alg) + return -ENOMEM; + + INIT_LIST_HEAD(&ccp_alg->entry); + + ccp_alg->mode = def->mode; + + /* Copy the defaults and override as necessary */ + alg = &ccp_alg->alg; + *alg = *def->alg_defaults; + strscpy(alg->base.cra_name, def->name); + strscpy(alg->base.cra_driver_name, def->driver_name); + alg->base.cra_blocksize = def->blocksize; + alg->ivsize = def->ivsize; + + ret = crypto_register_skcipher(alg); + if (ret) { + pr_err("%s skcipher algorithm registration error (%d)\n", + alg->base.cra_name, ret); + kfree(ccp_alg); + return ret; + } + + list_add(&ccp_alg->entry, head); + + return 0; +} + +int ccp_register_aes_algs(struct list_head *head) +{ + int i, ret; + unsigned int ccpversion = ccp_version(); + + for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { + if (aes_algs[i].version > ccpversion) + continue; + ret = ccp_register_aes_alg(head, &aes_algs[i]); + if (ret) + return ret; + } + + return 0; +} diff --git a/drivers/crypto/ccp/ccp-crypto-des3.c b/drivers/crypto/ccp/ccp-crypto-des3.c new file mode 100644 index 000000000..e26b431a5 --- /dev/null +++ b/drivers/crypto/ccp/ccp-crypto-des3.c @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Cryptographic Coprocessor (CCP) DES3 crypto API support + * + * Copyright (C) 2016,2017 Advanced Micro Devices, Inc. + * + * Author: Gary R Hook + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ccp-crypto.h" + +static int ccp_des3_complete(struct crypto_async_request *async_req, int ret) +{ + struct skcipher_request *req = skcipher_request_cast(async_req); + struct ccp_ctx *ctx = crypto_skcipher_ctx_dma( + crypto_skcipher_reqtfm(req)); + struct ccp_des3_req_ctx *rctx = skcipher_request_ctx_dma(req); + + if (ret) + return ret; + + if (ctx->u.des3.mode != CCP_DES3_MODE_ECB) + memcpy(req->iv, rctx->iv, DES3_EDE_BLOCK_SIZE); + + return 0; +} + +static int ccp_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int key_len) +{ + struct ccp_crypto_skcipher_alg *alg = ccp_crypto_skcipher_alg(tfm); + struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm); + int err; + + err = verify_skcipher_des3_key(tfm, key); + if (err) + return err; + + /* It's not clear that there is any support for a keysize of 112. + * If needed, the caller should make K1 == K3 + */ + ctx->u.des3.type = CCP_DES3_TYPE_168; + ctx->u.des3.mode = alg->mode; + ctx->u.des3.key_len = key_len; + + memcpy(ctx->u.des3.key, key, key_len); + sg_init_one(&ctx->u.des3.key_sg, ctx->u.des3.key, key_len); + + return 0; +} + +static int ccp_des3_crypt(struct skcipher_request *req, bool encrypt) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm); + struct ccp_des3_req_ctx *rctx = skcipher_request_ctx_dma(req); + struct scatterlist *iv_sg = NULL; + unsigned int iv_len = 0; + + if (!ctx->u.des3.key_len) + return -EINVAL; + + if (((ctx->u.des3.mode == CCP_DES3_MODE_ECB) || + (ctx->u.des3.mode == CCP_DES3_MODE_CBC)) && + (req->cryptlen & (DES3_EDE_BLOCK_SIZE - 1))) + return -EINVAL; + + if (ctx->u.des3.mode != CCP_DES3_MODE_ECB) { + if (!req->iv) + return -EINVAL; + + memcpy(rctx->iv, req->iv, DES3_EDE_BLOCK_SIZE); + iv_sg = &rctx->iv_sg; + iv_len = DES3_EDE_BLOCK_SIZE; + sg_init_one(iv_sg, rctx->iv, iv_len); + } + + memset(&rctx->cmd, 0, sizeof(rctx->cmd)); + INIT_LIST_HEAD(&rctx->cmd.entry); + rctx->cmd.engine = CCP_ENGINE_DES3; + rctx->cmd.u.des3.type = ctx->u.des3.type; + rctx->cmd.u.des3.mode = ctx->u.des3.mode; + rctx->cmd.u.des3.action = (encrypt) + ? CCP_DES3_ACTION_ENCRYPT + : CCP_DES3_ACTION_DECRYPT; + rctx->cmd.u.des3.key = &ctx->u.des3.key_sg; + rctx->cmd.u.des3.key_len = ctx->u.des3.key_len; + rctx->cmd.u.des3.iv = iv_sg; + rctx->cmd.u.des3.iv_len = iv_len; + rctx->cmd.u.des3.src = req->src; + rctx->cmd.u.des3.src_len = req->cryptlen; + rctx->cmd.u.des3.dst = req->dst; + + return ccp_crypto_enqueue_request(&req->base, &rctx->cmd); +} + +static int ccp_des3_encrypt(struct skcipher_request *req) +{ + return ccp_des3_crypt(req, true); +} + +static int ccp_des3_decrypt(struct skcipher_request *req) +{ + return ccp_des3_crypt(req, false); +} + +static int ccp_des3_init_tfm(struct crypto_skcipher *tfm) +{ + struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm); + + ctx->complete = ccp_des3_complete; + ctx->u.des3.key_len = 0; + + crypto_skcipher_set_reqsize_dma(tfm, sizeof(struct ccp_des3_req_ctx)); + + return 0; +} + +static const struct skcipher_alg ccp_des3_defaults = { + .setkey = ccp_des3_setkey, + .encrypt = ccp_des3_encrypt, + .decrypt = ccp_des3_decrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .init = ccp_des3_init_tfm, + + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ccp_ctx) + CRYPTO_DMA_PADDING, + .base.cra_priority = CCP_CRA_PRIORITY, + .base.cra_module = THIS_MODULE, +}; + +struct ccp_des3_def { + enum ccp_des3_mode mode; + unsigned int version; + const char *name; + const char *driver_name; + unsigned int blocksize; + unsigned int ivsize; + const struct skcipher_alg *alg_defaults; +}; + +static const struct ccp_des3_def des3_algs[] = { + { + .mode = CCP_DES3_MODE_ECB, + .version = CCP_VERSION(5, 0), + .name = "ecb(des3_ede)", + .driver_name = "ecb-des3-ccp", + .blocksize = DES3_EDE_BLOCK_SIZE, + .ivsize = 0, + .alg_defaults = &ccp_des3_defaults, + }, + { + .mode = CCP_DES3_MODE_CBC, + .version = CCP_VERSION(5, 0), + .name = "cbc(des3_ede)", + .driver_name = "cbc-des3-ccp", + .blocksize = DES3_EDE_BLOCK_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .alg_defaults = &ccp_des3_defaults, + }, +}; + +static int ccp_register_des3_alg(struct list_head *head, + const struct ccp_des3_def *def) +{ + struct ccp_crypto_skcipher_alg *ccp_alg; + struct skcipher_alg *alg; + int ret; + + ccp_alg = kzalloc_obj(*ccp_alg); + if (!ccp_alg) + return -ENOMEM; + + INIT_LIST_HEAD(&ccp_alg->entry); + + ccp_alg->mode = def->mode; + + /* Copy the defaults and override as necessary */ + alg = &ccp_alg->alg; + *alg = *def->alg_defaults; + strscpy(alg->base.cra_name, def->name); + strscpy(alg->base.cra_driver_name, def->driver_name); + alg->base.cra_blocksize = def->blocksize; + alg->ivsize = def->ivsize; + + ret = crypto_register_skcipher(alg); + if (ret) { + pr_err("%s skcipher algorithm registration error (%d)\n", + alg->base.cra_name, ret); + kfree(ccp_alg); + return ret; + } + + list_add(&ccp_alg->entry, head); + + return 0; +} + +int ccp_register_des3_algs(struct list_head *head) +{ + int i, ret; + unsigned int ccpversion = ccp_version(); + + for (i = 0; i < ARRAY_SIZE(des3_algs); i++) { + if (des3_algs[i].version > ccpversion) + continue; + ret = ccp_register_des3_alg(head, &des3_algs[i]); + if (ret) + return ret; + } + + return 0; +} diff --git a/drivers/crypto/ccp/ccp-crypto-main.c b/drivers/crypto/ccp/ccp-crypto-main.c new file mode 100644 index 000000000..698e39ab1 --- /dev/null +++ b/drivers/crypto/ccp/ccp-crypto-main.c @@ -0,0 +1,426 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Cryptographic Coprocessor (CCP) crypto API support + * + * Copyright (C) 2013,2017 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ccp-crypto.h" + +MODULE_AUTHOR("Tom Lendacky "); +MODULE_LICENSE("GPL"); +MODULE_VERSION("1.0.0"); +MODULE_DESCRIPTION("AMD Cryptographic Coprocessor crypto API support"); + +static unsigned int aes_disable; +module_param(aes_disable, uint, 0444); +MODULE_PARM_DESC(aes_disable, "Disable use of AES - any non-zero value"); + +static unsigned int sha_disable; +module_param(sha_disable, uint, 0444); +MODULE_PARM_DESC(sha_disable, "Disable use of SHA - any non-zero value"); + +static unsigned int des3_disable; +module_param(des3_disable, uint, 0444); +MODULE_PARM_DESC(des3_disable, "Disable use of 3DES - any non-zero value"); + +static unsigned int rsa_disable; +module_param(rsa_disable, uint, 0444); +MODULE_PARM_DESC(rsa_disable, "Disable use of RSA - any non-zero value"); + +/* List heads for the supported algorithms */ +static LIST_HEAD(hash_algs); +static LIST_HEAD(skcipher_algs); +static LIST_HEAD(aead_algs); +static LIST_HEAD(akcipher_algs); + +/* For any tfm, requests for that tfm must be returned on the order + * received. With multiple queues available, the CCP can process more + * than one cmd at a time. Therefore we must maintain a cmd list to insure + * the proper ordering of requests on a given tfm. + */ +struct ccp_crypto_queue { + struct list_head cmds; + struct list_head *backlog; + unsigned int cmd_count; +}; + +#define CCP_CRYPTO_MAX_QLEN 100 + +static struct ccp_crypto_queue req_queue; +static DEFINE_SPINLOCK(req_queue_lock); + +struct ccp_crypto_cmd { + struct list_head entry; + + struct ccp_cmd *cmd; + + /* Save the crypto_tfm and crypto_async_request addresses + * separately to avoid any reference to a possibly invalid + * crypto_async_request structure after invoking the request + * callback + */ + struct crypto_async_request *req; + struct crypto_tfm *tfm; + + /* Used for held command processing to determine state */ + int ret; +}; + +static inline bool ccp_crypto_success(int err) +{ + if (err && (err != -EINPROGRESS) && (err != -EBUSY)) + return false; + + return true; +} + +static struct ccp_crypto_cmd *ccp_crypto_cmd_complete( + struct ccp_crypto_cmd *crypto_cmd, struct ccp_crypto_cmd **backlog) +{ + struct ccp_crypto_cmd *held = NULL, *tmp; + unsigned long flags; + + *backlog = NULL; + + spin_lock_irqsave(&req_queue_lock, flags); + + /* Held cmds will be after the current cmd in the queue so start + * searching for a cmd with a matching tfm for submission. + */ + tmp = crypto_cmd; + list_for_each_entry_continue(tmp, &req_queue.cmds, entry) { + if (crypto_cmd->tfm != tmp->tfm) + continue; + held = tmp; + break; + } + + /* Process the backlog: + * Because cmds can be executed from any point in the cmd list + * special precautions have to be taken when handling the backlog. + */ + if (req_queue.backlog != &req_queue.cmds) { + /* Skip over this cmd if it is the next backlog cmd */ + if (req_queue.backlog == &crypto_cmd->entry) + req_queue.backlog = crypto_cmd->entry.next; + + *backlog = container_of(req_queue.backlog, + struct ccp_crypto_cmd, entry); + req_queue.backlog = req_queue.backlog->next; + + /* Skip over this cmd if it is now the next backlog cmd */ + if (req_queue.backlog == &crypto_cmd->entry) + req_queue.backlog = crypto_cmd->entry.next; + } + + /* Remove the cmd entry from the list of cmds */ + req_queue.cmd_count--; + list_del(&crypto_cmd->entry); + + spin_unlock_irqrestore(&req_queue_lock, flags); + + return held; +} + +static void ccp_crypto_complete(void *data, int err) +{ + struct ccp_crypto_cmd *crypto_cmd = data; + struct ccp_crypto_cmd *held, *next, *backlog; + struct crypto_async_request *req = crypto_cmd->req; + struct ccp_ctx *ctx = crypto_tfm_ctx_dma(req->tfm); + int ret; + + if (err == -EINPROGRESS) { + /* Only propagate the -EINPROGRESS if necessary */ + if (crypto_cmd->ret == -EBUSY) { + crypto_cmd->ret = -EINPROGRESS; + crypto_request_complete(req, -EINPROGRESS); + } + + return; + } + + /* Operation has completed - update the queue before invoking + * the completion callbacks and retrieve the next cmd (cmd with + * a matching tfm) that can be submitted to the CCP. + */ + held = ccp_crypto_cmd_complete(crypto_cmd, &backlog); + if (backlog) { + backlog->ret = -EINPROGRESS; + crypto_request_complete(backlog->req, -EINPROGRESS); + } + + /* Transition the state from -EBUSY to -EINPROGRESS first */ + if (crypto_cmd->ret == -EBUSY) + crypto_request_complete(req, -EINPROGRESS); + + /* Completion callbacks */ + ret = err; + if (ctx->complete) + ret = ctx->complete(req, ret); + crypto_request_complete(req, ret); + + /* Submit the next cmd */ + while (held) { + /* Since we have already queued the cmd, we must indicate that + * we can backlog so as not to "lose" this request. + */ + held->cmd->flags |= CCP_CMD_MAY_BACKLOG; + ret = ccp_enqueue_cmd(held->cmd); + if (ccp_crypto_success(ret)) + break; + + /* Error occurred, report it and get the next entry */ + ctx = crypto_tfm_ctx_dma(held->req->tfm); + if (ctx->complete) + ret = ctx->complete(held->req, ret); + crypto_request_complete(held->req, ret); + + next = ccp_crypto_cmd_complete(held, &backlog); + if (backlog) { + backlog->ret = -EINPROGRESS; + crypto_request_complete(backlog->req, -EINPROGRESS); + } + + kfree(held); + held = next; + } + + kfree(crypto_cmd); +} + +static int ccp_crypto_enqueue_cmd(struct ccp_crypto_cmd *crypto_cmd) +{ + struct ccp_crypto_cmd *active = NULL, *tmp; + unsigned long flags; + bool free_cmd = true; + int ret; + + spin_lock_irqsave(&req_queue_lock, flags); + + /* Check if the cmd can/should be queued */ + if (req_queue.cmd_count >= CCP_CRYPTO_MAX_QLEN) { + if (!(crypto_cmd->cmd->flags & CCP_CMD_MAY_BACKLOG)) { + ret = -ENOSPC; + goto e_lock; + } + } + + /* Look for an entry with the same tfm. If there is a cmd + * with the same tfm in the list then the current cmd cannot + * be submitted to the CCP yet. + */ + list_for_each_entry(tmp, &req_queue.cmds, entry) { + if (crypto_cmd->tfm != tmp->tfm) + continue; + active = tmp; + break; + } + + ret = -EINPROGRESS; + if (!active) { + ret = ccp_enqueue_cmd(crypto_cmd->cmd); + if (!ccp_crypto_success(ret)) + goto e_lock; /* Error, don't queue it */ + } + + if (req_queue.cmd_count >= CCP_CRYPTO_MAX_QLEN) { + ret = -EBUSY; + if (req_queue.backlog == &req_queue.cmds) + req_queue.backlog = &crypto_cmd->entry; + } + crypto_cmd->ret = ret; + + req_queue.cmd_count++; + list_add_tail(&crypto_cmd->entry, &req_queue.cmds); + + free_cmd = false; + +e_lock: + spin_unlock_irqrestore(&req_queue_lock, flags); + + if (free_cmd) + kfree(crypto_cmd); + + return ret; +} + +/** + * ccp_crypto_enqueue_request - queue an crypto async request for processing + * by the CCP + * + * @req: crypto_async_request struct to be processed + * @cmd: ccp_cmd struct to be sent to the CCP + */ +int ccp_crypto_enqueue_request(struct crypto_async_request *req, + struct ccp_cmd *cmd) +{ + struct ccp_crypto_cmd *crypto_cmd; + gfp_t gfp; + + gfp = req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; + + crypto_cmd = kzalloc_obj(*crypto_cmd, gfp); + if (!crypto_cmd) + return -ENOMEM; + + /* The tfm pointer must be saved and not referenced from the + * crypto_async_request (req) pointer because it is used after + * completion callback for the request and the req pointer + * might not be valid anymore. + */ + crypto_cmd->cmd = cmd; + crypto_cmd->req = req; + crypto_cmd->tfm = req->tfm; + + cmd->callback = ccp_crypto_complete; + cmd->data = crypto_cmd; + + if (req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) + cmd->flags |= CCP_CMD_MAY_BACKLOG; + else + cmd->flags &= ~CCP_CMD_MAY_BACKLOG; + + return ccp_crypto_enqueue_cmd(crypto_cmd); +} + +struct scatterlist *ccp_crypto_sg_table_add(struct sg_table *table, + struct scatterlist *sg_add) +{ + struct scatterlist *sg, *sg_last = NULL; + + for (sg = table->sgl; sg; sg = sg_next(sg)) + if (!sg_page(sg)) + break; + if (WARN_ON(!sg)) + return NULL; + + for (; sg && sg_add; sg = sg_next(sg), sg_add = sg_next(sg_add)) { + sg_set_page(sg, sg_page(sg_add), sg_add->length, + sg_add->offset); + sg_last = sg; + } + if (WARN_ON(sg_add)) + return NULL; + + return sg_last; +} + +static int ccp_register_algs(void) +{ + int ret; + + if (!aes_disable) { + ret = ccp_register_aes_algs(&skcipher_algs); + if (ret) + return ret; + + ret = ccp_register_aes_cmac_algs(&hash_algs); + if (ret) + return ret; + + ret = ccp_register_aes_xts_algs(&skcipher_algs); + if (ret) + return ret; + + ret = ccp_register_aes_aeads(&aead_algs); + if (ret) + return ret; + } + + if (!des3_disable) { + ret = ccp_register_des3_algs(&skcipher_algs); + if (ret) + return ret; + } + + if (!sha_disable) { + ret = ccp_register_sha_algs(&hash_algs); + if (ret) + return ret; + } + + if (!rsa_disable) { + ret = ccp_register_rsa_algs(&akcipher_algs); + if (ret) + return ret; + } + + return 0; +} + +static void ccp_unregister_algs(void) +{ + struct ccp_crypto_ahash_alg *ahash_alg, *ahash_tmp; + struct ccp_crypto_skcipher_alg *ablk_alg, *ablk_tmp; + struct ccp_crypto_aead *aead_alg, *aead_tmp; + struct ccp_crypto_akcipher_alg *akc_alg, *akc_tmp; + + list_for_each_entry_safe(ahash_alg, ahash_tmp, &hash_algs, entry) { + crypto_unregister_ahash(&ahash_alg->alg); + list_del(&ahash_alg->entry); + kfree(ahash_alg); + } + + list_for_each_entry_safe(ablk_alg, ablk_tmp, &skcipher_algs, entry) { + crypto_unregister_skcipher(&ablk_alg->alg); + list_del(&ablk_alg->entry); + kfree(ablk_alg); + } + + list_for_each_entry_safe(aead_alg, aead_tmp, &aead_algs, entry) { + crypto_unregister_aead(&aead_alg->alg); + list_del(&aead_alg->entry); + kfree(aead_alg); + } + + list_for_each_entry_safe(akc_alg, akc_tmp, &akcipher_algs, entry) { + crypto_unregister_akcipher(&akc_alg->alg); + list_del(&akc_alg->entry); + kfree(akc_alg); + } +} + +static int __init ccp_crypto_init(void) +{ + int ret; + + ret = ccp_present(); + if (ret) { + pr_err("Cannot load: there are no available CCPs\n"); + return ret; + } + + INIT_LIST_HEAD(&req_queue.cmds); + req_queue.backlog = &req_queue.cmds; + req_queue.cmd_count = 0; + + ret = ccp_register_algs(); + if (ret) + ccp_unregister_algs(); + + return ret; +} + +static void __exit ccp_crypto_exit(void) +{ + ccp_unregister_algs(); +} + +module_init(ccp_crypto_init); +module_exit(ccp_crypto_exit); diff --git a/drivers/crypto/ccp/ccp-crypto-rsa.c b/drivers/crypto/ccp/ccp-crypto-rsa.c new file mode 100644 index 000000000..287d7f620 --- /dev/null +++ b/drivers/crypto/ccp/ccp-crypto-rsa.c @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Cryptographic Coprocessor (CCP) RSA crypto API support + * + * Copyright (C) 2017 Advanced Micro Devices, Inc. + * + * Author: Gary R Hook + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ccp-crypto.h" + +static inline struct akcipher_request *akcipher_request_cast( + struct crypto_async_request *req) +{ + return container_of(req, struct akcipher_request, base); +} + +static inline int ccp_copy_and_save_keypart(u8 **kpbuf, unsigned int *kplen, + const u8 *buf, size_t sz) +{ + int nskip; + + for (nskip = 0; nskip < sz; nskip++) + if (buf[nskip]) + break; + *kplen = sz - nskip; + *kpbuf = kmemdup(buf + nskip, *kplen, GFP_KERNEL); + if (!*kpbuf) + return -ENOMEM; + + return 0; +} + +static int ccp_rsa_complete(struct crypto_async_request *async_req, int ret) +{ + struct akcipher_request *req = akcipher_request_cast(async_req); + struct ccp_rsa_req_ctx *rctx = akcipher_request_ctx_dma(req); + + if (ret) + return ret; + + req->dst_len = rctx->cmd.u.rsa.key_size >> 3; + + return 0; +} + +static unsigned int ccp_rsa_maxsize(struct crypto_akcipher *tfm) +{ + struct ccp_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + + return ctx->u.rsa.n_len; +} + +static int ccp_rsa_crypt(struct akcipher_request *req, bool encrypt) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct ccp_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + struct ccp_rsa_req_ctx *rctx = akcipher_request_ctx_dma(req); + int ret = 0; + + memset(&rctx->cmd, 0, sizeof(rctx->cmd)); + INIT_LIST_HEAD(&rctx->cmd.entry); + rctx->cmd.engine = CCP_ENGINE_RSA; + + rctx->cmd.u.rsa.key_size = ctx->u.rsa.key_len; /* in bits */ + if (encrypt) { + rctx->cmd.u.rsa.exp = &ctx->u.rsa.e_sg; + rctx->cmd.u.rsa.exp_len = ctx->u.rsa.e_len; + } else { + rctx->cmd.u.rsa.exp = &ctx->u.rsa.d_sg; + rctx->cmd.u.rsa.exp_len = ctx->u.rsa.d_len; + } + rctx->cmd.u.rsa.mod = &ctx->u.rsa.n_sg; + rctx->cmd.u.rsa.mod_len = ctx->u.rsa.n_len; + rctx->cmd.u.rsa.src = req->src; + rctx->cmd.u.rsa.src_len = req->src_len; + rctx->cmd.u.rsa.dst = req->dst; + + ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd); + + return ret; +} + +static int ccp_rsa_encrypt(struct akcipher_request *req) +{ + return ccp_rsa_crypt(req, true); +} + +static int ccp_rsa_decrypt(struct akcipher_request *req) +{ + return ccp_rsa_crypt(req, false); +} + +static int ccp_check_key_length(unsigned int len) +{ + /* In bits */ + if (len < 8 || len > 4096) + return -EINVAL; + return 0; +} + +static void ccp_rsa_free_key_bufs(struct ccp_ctx *ctx) +{ + /* Clean up old key data */ + kfree_sensitive(ctx->u.rsa.e_buf); + ctx->u.rsa.e_buf = NULL; + ctx->u.rsa.e_len = 0; + kfree_sensitive(ctx->u.rsa.n_buf); + ctx->u.rsa.n_buf = NULL; + ctx->u.rsa.n_len = 0; + kfree_sensitive(ctx->u.rsa.d_buf); + ctx->u.rsa.d_buf = NULL; + ctx->u.rsa.d_len = 0; +} + +static int ccp_rsa_setkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen, bool private) +{ + struct ccp_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + struct rsa_key raw_key; + int ret; + + ccp_rsa_free_key_bufs(ctx); + memset(&raw_key, 0, sizeof(raw_key)); + + /* Code borrowed from crypto/rsa.c */ + if (private) + ret = rsa_parse_priv_key(&raw_key, key, keylen); + else + ret = rsa_parse_pub_key(&raw_key, key, keylen); + if (ret) + goto n_key; + + ret = ccp_copy_and_save_keypart(&ctx->u.rsa.n_buf, &ctx->u.rsa.n_len, + raw_key.n, raw_key.n_sz); + if (ret) + goto key_err; + sg_init_one(&ctx->u.rsa.n_sg, ctx->u.rsa.n_buf, ctx->u.rsa.n_len); + + ctx->u.rsa.key_len = ctx->u.rsa.n_len << 3; /* convert to bits */ + if (ccp_check_key_length(ctx->u.rsa.key_len)) { + ret = -EINVAL; + goto key_err; + } + + ret = ccp_copy_and_save_keypart(&ctx->u.rsa.e_buf, &ctx->u.rsa.e_len, + raw_key.e, raw_key.e_sz); + if (ret) + goto key_err; + sg_init_one(&ctx->u.rsa.e_sg, ctx->u.rsa.e_buf, ctx->u.rsa.e_len); + + if (private) { + ret = ccp_copy_and_save_keypart(&ctx->u.rsa.d_buf, + &ctx->u.rsa.d_len, + raw_key.d, raw_key.d_sz); + if (ret) + goto key_err; + sg_init_one(&ctx->u.rsa.d_sg, + ctx->u.rsa.d_buf, ctx->u.rsa.d_len); + } + + return 0; + +key_err: + ccp_rsa_free_key_bufs(ctx); + +n_key: + return ret; +} + +static int ccp_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + return ccp_rsa_setkey(tfm, key, keylen, true); +} + +static int ccp_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + return ccp_rsa_setkey(tfm, key, keylen, false); +} + +static int ccp_rsa_init_tfm(struct crypto_akcipher *tfm) +{ + struct ccp_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + + akcipher_set_reqsize_dma(tfm, sizeof(struct ccp_rsa_req_ctx)); + ctx->complete = ccp_rsa_complete; + + return 0; +} + +static void ccp_rsa_exit_tfm(struct crypto_akcipher *tfm) +{ + struct ccp_ctx *ctx = akcipher_tfm_ctx_dma(tfm); + + ccp_rsa_free_key_bufs(ctx); +} + +static struct akcipher_alg ccp_rsa_defaults = { + .encrypt = ccp_rsa_encrypt, + .decrypt = ccp_rsa_decrypt, + .set_pub_key = ccp_rsa_setpubkey, + .set_priv_key = ccp_rsa_setprivkey, + .max_size = ccp_rsa_maxsize, + .init = ccp_rsa_init_tfm, + .exit = ccp_rsa_exit_tfm, + .base = { + .cra_name = "rsa", + .cra_driver_name = "rsa-ccp", + .cra_priority = CCP_CRA_PRIORITY, + .cra_module = THIS_MODULE, + .cra_ctxsize = 2 * sizeof(struct ccp_ctx) + CRYPTO_DMA_PADDING, + }, +}; + +struct ccp_rsa_def { + unsigned int version; + const char *name; + const char *driver_name; + unsigned int reqsize; + struct akcipher_alg *alg_defaults; +}; + +static struct ccp_rsa_def rsa_algs[] = { + { + .version = CCP_VERSION(3, 0), + .name = "rsa", + .driver_name = "rsa-ccp", + .reqsize = sizeof(struct ccp_rsa_req_ctx), + .alg_defaults = &ccp_rsa_defaults, + } +}; + +static int ccp_register_rsa_alg(struct list_head *head, + const struct ccp_rsa_def *def) +{ + struct ccp_crypto_akcipher_alg *ccp_alg; + struct akcipher_alg *alg; + int ret; + + ccp_alg = kzalloc_obj(*ccp_alg); + if (!ccp_alg) + return -ENOMEM; + + INIT_LIST_HEAD(&ccp_alg->entry); + + alg = &ccp_alg->alg; + *alg = *def->alg_defaults; + strscpy(alg->base.cra_name, def->name); + strscpy(alg->base.cra_driver_name, def->driver_name); + ret = crypto_register_akcipher(alg); + if (ret) { + pr_err("%s akcipher algorithm registration error (%d)\n", + alg->base.cra_name, ret); + kfree(ccp_alg); + return ret; + } + + list_add(&ccp_alg->entry, head); + + return 0; +} + +int ccp_register_rsa_algs(struct list_head *head) +{ + int i, ret; + unsigned int ccpversion = ccp_version(); + + /* Register the RSA algorithm in standard mode + * This works for CCP v3 and later + */ + for (i = 0; i < ARRAY_SIZE(rsa_algs); i++) { + if (rsa_algs[i].version > ccpversion) + continue; + ret = ccp_register_rsa_alg(head, &rsa_algs[i]); + if (ret) + return ret; + } + + return 0; +} diff --git a/drivers/crypto/ccp/ccp-crypto-sha.c b/drivers/crypto/ccp/ccp-crypto-sha.c new file mode 100644 index 000000000..ff9bb253d --- /dev/null +++ b/drivers/crypto/ccp/ccp-crypto-sha.c @@ -0,0 +1,529 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Cryptographic Coprocessor (CCP) SHA crypto API support + * + * Copyright (C) 2013,2018 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky + * Author: Gary R Hook + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ccp-crypto.h" + +static int ccp_sha_complete(struct crypto_async_request *async_req, int ret) +{ + struct ahash_request *req = ahash_request_cast(async_req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ccp_sha_req_ctx *rctx = ahash_request_ctx_dma(req); + unsigned int digest_size = crypto_ahash_digestsize(tfm); + + if (ret) + goto e_free; + + if (rctx->hash_rem) { + /* Save remaining data to buffer */ + unsigned int offset = rctx->nbytes - rctx->hash_rem; + + scatterwalk_map_and_copy(rctx->buf, rctx->src, + offset, rctx->hash_rem, 0); + rctx->buf_count = rctx->hash_rem; + } else { + rctx->buf_count = 0; + } + + /* Update result area if supplied */ + if (req->result && rctx->final) + memcpy(req->result, rctx->ctx, digest_size); + +e_free: + sg_free_table(&rctx->data_sg); + + return ret; +} + +static int ccp_do_sha_update(struct ahash_request *req, unsigned int nbytes, + unsigned int final) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ccp_ctx *ctx = crypto_ahash_ctx_dma(tfm); + struct ccp_sha_req_ctx *rctx = ahash_request_ctx_dma(req); + struct scatterlist *sg; + unsigned int block_size = + crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + unsigned int sg_count; + gfp_t gfp; + u64 len; + int ret; + + len = (u64)rctx->buf_count + (u64)nbytes; + + if (!final && (len <= block_size)) { + scatterwalk_map_and_copy(rctx->buf + rctx->buf_count, req->src, + 0, nbytes, 0); + rctx->buf_count += nbytes; + + return 0; + } + + rctx->src = req->src; + rctx->nbytes = nbytes; + + rctx->final = final; + rctx->hash_rem = final ? 0 : len & (block_size - 1); + rctx->hash_cnt = len - rctx->hash_rem; + if (!final && !rctx->hash_rem) { + /* CCP can't do zero length final, so keep some data around */ + rctx->hash_cnt -= block_size; + rctx->hash_rem = block_size; + } + + /* Initialize the context scatterlist */ + sg_init_one(&rctx->ctx_sg, rctx->ctx, sizeof(rctx->ctx)); + + sg = NULL; + if (rctx->buf_count && nbytes) { + /* Build the data scatterlist table - allocate enough entries + * for both data pieces (buffer and input data) + */ + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + sg_count = sg_nents(req->src) + 1; + ret = sg_alloc_table(&rctx->data_sg, sg_count, gfp); + if (ret) + return ret; + + sg_init_one(&rctx->buf_sg, rctx->buf, rctx->buf_count); + sg = ccp_crypto_sg_table_add(&rctx->data_sg, &rctx->buf_sg); + if (!sg) { + ret = -EINVAL; + goto e_free; + } + sg = ccp_crypto_sg_table_add(&rctx->data_sg, req->src); + if (!sg) { + ret = -EINVAL; + goto e_free; + } + sg_mark_end(sg); + + sg = rctx->data_sg.sgl; + } else if (rctx->buf_count) { + sg_init_one(&rctx->buf_sg, rctx->buf, rctx->buf_count); + + sg = &rctx->buf_sg; + } else if (nbytes) { + sg = req->src; + } + + rctx->msg_bits += (rctx->hash_cnt << 3); /* Total in bits */ + + memset(&rctx->cmd, 0, sizeof(rctx->cmd)); + INIT_LIST_HEAD(&rctx->cmd.entry); + rctx->cmd.engine = CCP_ENGINE_SHA; + rctx->cmd.u.sha.type = rctx->type; + rctx->cmd.u.sha.ctx = &rctx->ctx_sg; + + switch (rctx->type) { + case CCP_SHA_TYPE_1: + rctx->cmd.u.sha.ctx_len = SHA1_DIGEST_SIZE; + break; + case CCP_SHA_TYPE_224: + rctx->cmd.u.sha.ctx_len = SHA224_DIGEST_SIZE; + break; + case CCP_SHA_TYPE_256: + rctx->cmd.u.sha.ctx_len = SHA256_DIGEST_SIZE; + break; + case CCP_SHA_TYPE_384: + rctx->cmd.u.sha.ctx_len = SHA384_DIGEST_SIZE; + break; + case CCP_SHA_TYPE_512: + rctx->cmd.u.sha.ctx_len = SHA512_DIGEST_SIZE; + break; + default: + /* Should never get here */ + break; + } + + rctx->cmd.u.sha.src = sg; + rctx->cmd.u.sha.src_len = rctx->hash_cnt; + rctx->cmd.u.sha.opad = ctx->u.sha.key_len ? + &ctx->u.sha.opad_sg : NULL; + rctx->cmd.u.sha.opad_len = ctx->u.sha.key_len ? + ctx->u.sha.opad_count : 0; + rctx->cmd.u.sha.first = rctx->first; + rctx->cmd.u.sha.final = rctx->final; + rctx->cmd.u.sha.msg_bits = rctx->msg_bits; + + rctx->first = 0; + + ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd); + + return ret; + +e_free: + sg_free_table(&rctx->data_sg); + + return ret; +} + +static int ccp_sha_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ccp_ctx *ctx = crypto_ahash_ctx_dma(tfm); + struct ccp_sha_req_ctx *rctx = ahash_request_ctx_dma(req); + struct ccp_crypto_ahash_alg *alg = + ccp_crypto_ahash_alg(crypto_ahash_tfm(tfm)); + unsigned int block_size = + crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + + memset(rctx, 0, sizeof(*rctx)); + + rctx->type = alg->type; + rctx->first = 1; + + if (ctx->u.sha.key_len) { + /* Buffer the HMAC key for first update */ + memcpy(rctx->buf, ctx->u.sha.ipad, block_size); + rctx->buf_count = block_size; + } + + return 0; +} + +static int ccp_sha_update(struct ahash_request *req) +{ + return ccp_do_sha_update(req, req->nbytes, 0); +} + +static int ccp_sha_final(struct ahash_request *req) +{ + return ccp_do_sha_update(req, 0, 1); +} + +static int ccp_sha_finup(struct ahash_request *req) +{ + return ccp_do_sha_update(req, req->nbytes, 1); +} + +static int ccp_sha_digest(struct ahash_request *req) +{ + int ret; + + ret = ccp_sha_init(req); + if (ret) + return ret; + + return ccp_sha_finup(req); +} + +static int ccp_sha_export(struct ahash_request *req, void *out) +{ + struct ccp_sha_req_ctx *rctx = ahash_request_ctx_dma(req); + struct ccp_sha_exp_ctx state; + + /* Don't let anything leak to 'out' */ + memset(&state, 0, sizeof(state)); + + state.type = rctx->type; + state.msg_bits = rctx->msg_bits; + state.first = rctx->first; + memcpy(state.ctx, rctx->ctx, sizeof(state.ctx)); + state.buf_count = rctx->buf_count; + memcpy(state.buf, rctx->buf, sizeof(state.buf)); + + /* 'out' may not be aligned so memcpy from local variable */ + memcpy(out, &state, sizeof(state)); + + return 0; +} + +static int ccp_sha_import(struct ahash_request *req, const void *in) +{ + struct ccp_sha_req_ctx *rctx = ahash_request_ctx_dma(req); + struct ccp_sha_exp_ctx state; + + /* 'in' may not be aligned so memcpy to local variable */ + memcpy(&state, in, sizeof(state)); + + memset(rctx, 0, sizeof(*rctx)); + rctx->type = state.type; + rctx->msg_bits = state.msg_bits; + rctx->first = state.first; + memcpy(rctx->ctx, state.ctx, sizeof(rctx->ctx)); + rctx->buf_count = state.buf_count; + memcpy(rctx->buf, state.buf, sizeof(rctx->buf)); + + return 0; +} + +static int ccp_sha_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int key_len) +{ + struct ccp_ctx *ctx = crypto_ahash_ctx_dma(tfm); + struct crypto_shash *shash = ctx->u.sha.hmac_tfm; + unsigned int block_size = crypto_shash_blocksize(shash); + unsigned int digest_size = crypto_shash_digestsize(shash); + int i, ret; + + /* Set to zero until complete */ + ctx->u.sha.key_len = 0; + + /* Clear key area to provide zero padding for keys smaller + * than the block size + */ + memset(ctx->u.sha.key, 0, sizeof(ctx->u.sha.key)); + + if (key_len > block_size) { + /* Must hash the input key */ + ret = crypto_shash_tfm_digest(shash, key, key_len, + ctx->u.sha.key); + if (ret) + return -EINVAL; + + key_len = digest_size; + } else { + memcpy(ctx->u.sha.key, key, key_len); + } + + for (i = 0; i < block_size; i++) { + ctx->u.sha.ipad[i] = ctx->u.sha.key[i] ^ HMAC_IPAD_VALUE; + ctx->u.sha.opad[i] = ctx->u.sha.key[i] ^ HMAC_OPAD_VALUE; + } + + sg_init_one(&ctx->u.sha.opad_sg, ctx->u.sha.opad, block_size); + ctx->u.sha.opad_count = block_size; + + ctx->u.sha.key_len = key_len; + + return 0; +} + +static int ccp_sha_cra_init(struct crypto_tfm *tfm) +{ + struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); + struct ccp_ctx *ctx = crypto_ahash_ctx_dma(ahash); + + ctx->complete = ccp_sha_complete; + ctx->u.sha.key_len = 0; + + crypto_ahash_set_reqsize_dma(ahash, sizeof(struct ccp_sha_req_ctx)); + + return 0; +} + +static void ccp_sha_cra_exit(struct crypto_tfm *tfm) +{ +} + +static int ccp_hmac_sha_cra_init(struct crypto_tfm *tfm) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx_dma(tfm); + struct ccp_crypto_ahash_alg *alg = ccp_crypto_ahash_alg(tfm); + struct crypto_shash *hmac_tfm; + + hmac_tfm = crypto_alloc_shash(alg->child_alg, 0, 0); + if (IS_ERR(hmac_tfm)) { + pr_warn("could not load driver %s need for HMAC support\n", + alg->child_alg); + return PTR_ERR(hmac_tfm); + } + + ctx->u.sha.hmac_tfm = hmac_tfm; + + return ccp_sha_cra_init(tfm); +} + +static void ccp_hmac_sha_cra_exit(struct crypto_tfm *tfm) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx_dma(tfm); + + if (ctx->u.sha.hmac_tfm) + crypto_free_shash(ctx->u.sha.hmac_tfm); + + ccp_sha_cra_exit(tfm); +} + +struct ccp_sha_def { + unsigned int version; + const char *name; + const char *drv_name; + enum ccp_sha_type type; + u32 digest_size; + u32 block_size; +}; + +static struct ccp_sha_def sha_algs[] = { + { + .version = CCP_VERSION(3, 0), + .name = "sha1", + .drv_name = "sha1-ccp", + .type = CCP_SHA_TYPE_1, + .digest_size = SHA1_DIGEST_SIZE, + .block_size = SHA1_BLOCK_SIZE, + }, + { + .version = CCP_VERSION(3, 0), + .name = "sha224", + .drv_name = "sha224-ccp", + .type = CCP_SHA_TYPE_224, + .digest_size = SHA224_DIGEST_SIZE, + .block_size = SHA224_BLOCK_SIZE, + }, + { + .version = CCP_VERSION(3, 0), + .name = "sha256", + .drv_name = "sha256-ccp", + .type = CCP_SHA_TYPE_256, + .digest_size = SHA256_DIGEST_SIZE, + .block_size = SHA256_BLOCK_SIZE, + }, + { + .version = CCP_VERSION(5, 0), + .name = "sha384", + .drv_name = "sha384-ccp", + .type = CCP_SHA_TYPE_384, + .digest_size = SHA384_DIGEST_SIZE, + .block_size = SHA384_BLOCK_SIZE, + }, + { + .version = CCP_VERSION(5, 0), + .name = "sha512", + .drv_name = "sha512-ccp", + .type = CCP_SHA_TYPE_512, + .digest_size = SHA512_DIGEST_SIZE, + .block_size = SHA512_BLOCK_SIZE, + }, +}; + +static int ccp_register_hmac_alg(struct list_head *head, + const struct ccp_sha_def *def, + const struct ccp_crypto_ahash_alg *base_alg) +{ + struct ccp_crypto_ahash_alg *ccp_alg; + struct ahash_alg *alg; + struct hash_alg_common *halg; + struct crypto_alg *base; + int ret; + + ccp_alg = kzalloc_obj(*ccp_alg); + if (!ccp_alg) + return -ENOMEM; + + /* Copy the base algorithm and only change what's necessary */ + *ccp_alg = *base_alg; + INIT_LIST_HEAD(&ccp_alg->entry); + + strscpy(ccp_alg->child_alg, def->name); + + alg = &ccp_alg->alg; + alg->setkey = ccp_sha_setkey; + + halg = &alg->halg; + + base = &halg->base; + snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)", def->name); + snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s", + def->drv_name); + base->cra_init = ccp_hmac_sha_cra_init; + base->cra_exit = ccp_hmac_sha_cra_exit; + + ret = crypto_register_ahash(alg); + if (ret) { + pr_err("%s ahash algorithm registration error (%d)\n", + base->cra_name, ret); + kfree(ccp_alg); + return ret; + } + + list_add(&ccp_alg->entry, head); + + return ret; +} + +static int ccp_register_sha_alg(struct list_head *head, + const struct ccp_sha_def *def) +{ + struct ccp_crypto_ahash_alg *ccp_alg; + struct ahash_alg *alg; + struct hash_alg_common *halg; + struct crypto_alg *base; + int ret; + + ccp_alg = kzalloc_obj(*ccp_alg); + if (!ccp_alg) + return -ENOMEM; + + INIT_LIST_HEAD(&ccp_alg->entry); + + ccp_alg->type = def->type; + + alg = &ccp_alg->alg; + alg->init = ccp_sha_init; + alg->update = ccp_sha_update; + alg->final = ccp_sha_final; + alg->finup = ccp_sha_finup; + alg->digest = ccp_sha_digest; + alg->export = ccp_sha_export; + alg->import = ccp_sha_import; + + halg = &alg->halg; + halg->digestsize = def->digest_size; + halg->statesize = sizeof(struct ccp_sha_exp_ctx); + + base = &halg->base; + strscpy(base->cra_name, def->name); + strscpy(base->cra_driver_name, def->drv_name); + base->cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK; + base->cra_blocksize = def->block_size; + base->cra_ctxsize = sizeof(struct ccp_ctx) + crypto_dma_padding(); + base->cra_priority = CCP_CRA_PRIORITY; + base->cra_init = ccp_sha_cra_init; + base->cra_exit = ccp_sha_cra_exit; + base->cra_module = THIS_MODULE; + + ret = crypto_register_ahash(alg); + if (ret) { + pr_err("%s ahash algorithm registration error (%d)\n", + base->cra_name, ret); + kfree(ccp_alg); + return ret; + } + + list_add(&ccp_alg->entry, head); + + ret = ccp_register_hmac_alg(head, def, ccp_alg); + + return ret; +} + +int ccp_register_sha_algs(struct list_head *head) +{ + int i, ret; + unsigned int ccpversion = ccp_version(); + + for (i = 0; i < ARRAY_SIZE(sha_algs); i++) { + if (sha_algs[i].version > ccpversion) + continue; + ret = ccp_register_sha_alg(head, &sha_algs[i]); + if (ret) + return ret; + } + + return 0; +} diff --git a/drivers/crypto/ccp/ccp-crypto.h b/drivers/crypto/ccp/ccp-crypto.h new file mode 100644 index 000000000..e42450d07 --- /dev/null +++ b/drivers/crypto/ccp/ccp-crypto.h @@ -0,0 +1,286 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * AMD Cryptographic Coprocessor (CCP) crypto API support + * + * Copyright (C) 2013,2017 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky + */ + +#ifndef __CCP_CRYPTO_H__ +#define __CCP_CRYPTO_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* We want the module name in front of our messages */ +#undef pr_fmt +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#define CCP_LOG_LEVEL KERN_INFO + +#define CCP_CRA_PRIORITY 300 + +struct ccp_crypto_skcipher_alg { + struct list_head entry; + + u32 mode; + + struct skcipher_alg alg; +}; + +struct ccp_crypto_aead { + struct list_head entry; + + u32 mode; + + struct aead_alg alg; +}; + +struct ccp_crypto_ahash_alg { + struct list_head entry; + + const __be32 *init; + u32 type; + u32 mode; + + /* Child algorithm used for HMAC, CMAC, etc */ + char child_alg[CRYPTO_MAX_ALG_NAME]; + + struct ahash_alg alg; +}; + +struct ccp_crypto_akcipher_alg { + struct list_head entry; + + struct akcipher_alg alg; +}; + +static inline struct ccp_crypto_skcipher_alg * + ccp_crypto_skcipher_alg(struct crypto_skcipher *tfm) +{ + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + + return container_of(alg, struct ccp_crypto_skcipher_alg, alg); +} + +static inline struct ccp_crypto_ahash_alg * + ccp_crypto_ahash_alg(struct crypto_tfm *tfm) +{ + struct crypto_alg *alg = tfm->__crt_alg; + struct ahash_alg *ahash_alg; + + ahash_alg = container_of(alg, struct ahash_alg, halg.base); + + return container_of(ahash_alg, struct ccp_crypto_ahash_alg, alg); +} + +/***** AES related defines *****/ +struct ccp_aes_ctx { + /* Fallback cipher for XTS with unsupported unit sizes */ + struct crypto_skcipher *tfm_skcipher; + + enum ccp_engine engine; + enum ccp_aes_type type; + enum ccp_aes_mode mode; + + struct scatterlist key_sg; + unsigned int key_len; + u8 key[AES_MAX_KEY_SIZE * 2]; + + u8 nonce[CTR_RFC3686_NONCE_SIZE]; + + /* CMAC key structures */ + struct scatterlist k1_sg; + struct scatterlist k2_sg; + unsigned int kn_len; + u8 k1[AES_BLOCK_SIZE]; + u8 k2[AES_BLOCK_SIZE]; +}; + +struct ccp_aes_req_ctx { + struct scatterlist iv_sg; + u8 iv[AES_BLOCK_SIZE]; + + struct scatterlist tag_sg; + u8 tag[AES_BLOCK_SIZE]; + + /* Fields used for RFC3686 requests */ + u8 *rfc3686_info; + u8 rfc3686_iv[AES_BLOCK_SIZE]; + + struct ccp_cmd cmd; + + struct skcipher_request fallback_req; // keep at the end +}; + +struct ccp_aes_cmac_req_ctx { + unsigned int null_msg; + unsigned int final; + + struct scatterlist *src; + unsigned int nbytes; + + u64 hash_cnt; + unsigned int hash_rem; + + struct sg_table data_sg; + + struct scatterlist iv_sg; + u8 iv[AES_BLOCK_SIZE]; + + struct scatterlist buf_sg; + unsigned int buf_count; + u8 buf[AES_BLOCK_SIZE]; + + struct scatterlist pad_sg; + unsigned int pad_count; + u8 pad[AES_BLOCK_SIZE]; + + struct ccp_cmd cmd; +}; + +struct ccp_aes_cmac_exp_ctx { + unsigned int null_msg; + + u8 iv[AES_BLOCK_SIZE]; + + unsigned int buf_count; + u8 buf[AES_BLOCK_SIZE]; +}; + +/***** 3DES related defines *****/ +struct ccp_des3_ctx { + enum ccp_engine engine; + enum ccp_des3_type type; + enum ccp_des3_mode mode; + + struct scatterlist key_sg; + unsigned int key_len; + u8 key[AES_MAX_KEY_SIZE]; +}; + +struct ccp_des3_req_ctx { + struct scatterlist iv_sg; + u8 iv[AES_BLOCK_SIZE]; + + struct ccp_cmd cmd; +}; + +/* SHA-related defines + * These values must be large enough to accommodate any variant + */ +#define MAX_SHA_CONTEXT_SIZE SHA512_DIGEST_SIZE +#define MAX_SHA_BLOCK_SIZE SHA512_BLOCK_SIZE + +struct ccp_sha_ctx { + struct scatterlist opad_sg; + unsigned int opad_count; + + unsigned int key_len; + u8 key[MAX_SHA_BLOCK_SIZE]; + u8 ipad[MAX_SHA_BLOCK_SIZE]; + u8 opad[MAX_SHA_BLOCK_SIZE]; + struct crypto_shash *hmac_tfm; +}; + +struct ccp_sha_req_ctx { + enum ccp_sha_type type; + + u64 msg_bits; + + unsigned int first; + unsigned int final; + + struct scatterlist *src; + unsigned int nbytes; + + u64 hash_cnt; + unsigned int hash_rem; + + struct sg_table data_sg; + + struct scatterlist ctx_sg; + u8 ctx[MAX_SHA_CONTEXT_SIZE]; + + struct scatterlist buf_sg; + unsigned int buf_count; + u8 buf[MAX_SHA_BLOCK_SIZE]; + + /* CCP driver command */ + struct ccp_cmd cmd; +}; + +struct ccp_sha_exp_ctx { + enum ccp_sha_type type; + + u64 msg_bits; + + unsigned int first; + + u8 ctx[MAX_SHA_CONTEXT_SIZE]; + + unsigned int buf_count; + u8 buf[MAX_SHA_BLOCK_SIZE]; +}; + +/***** RSA related defines *****/ + +struct ccp_rsa_ctx { + unsigned int key_len; /* in bits */ + struct scatterlist e_sg; + u8 *e_buf; + unsigned int e_len; + struct scatterlist n_sg; + u8 *n_buf; + unsigned int n_len; + struct scatterlist d_sg; + u8 *d_buf; + unsigned int d_len; +}; + +struct ccp_rsa_req_ctx { + struct ccp_cmd cmd; +}; + +#define CCP_RSA_MAXMOD (4 * 1024 / 8) +#define CCP5_RSA_MAXMOD (16 * 1024 / 8) + +/***** Common Context Structure *****/ +struct ccp_ctx { + int (*complete)(struct crypto_async_request *req, int ret); + + union { + struct ccp_aes_ctx aes; + struct ccp_rsa_ctx rsa; + struct ccp_sha_ctx sha; + struct ccp_des3_ctx des3; + } u; +}; + +int ccp_crypto_enqueue_request(struct crypto_async_request *req, + struct ccp_cmd *cmd); +struct scatterlist *ccp_crypto_sg_table_add(struct sg_table *table, + struct scatterlist *sg_add); + +int ccp_register_aes_algs(struct list_head *head); +int ccp_register_aes_cmac_algs(struct list_head *head); +int ccp_register_aes_xts_algs(struct list_head *head); +int ccp_register_aes_aeads(struct list_head *head); +int ccp_register_sha_algs(struct list_head *head); +int ccp_register_des3_algs(struct list_head *head); +int ccp_register_rsa_algs(struct list_head *head); + +#endif diff --git a/drivers/crypto/ccp/ccp-debugfs.c b/drivers/crypto/ccp/ccp-debugfs.c new file mode 100644 index 000000000..dc26bc22c --- /dev/null +++ b/drivers/crypto/ccp/ccp-debugfs.c @@ -0,0 +1,326 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Cryptographic Coprocessor (CCP) driver + * + * Copyright (C) 2017 Advanced Micro Devices, Inc. + * + * Author: Gary R Hook + */ + +#include +#include + +#include "ccp-dev.h" + +/* DebugFS helpers */ +#define OBUFP (obuf + oboff) +#define OBUFLEN 512 +#define OBUFSPC (OBUFLEN - oboff) +#define OSCNPRINTF(fmt, ...) \ + scnprintf(OBUFP, OBUFSPC, fmt, ## __VA_ARGS__) + +#define BUFLEN 63 + +#define RI_VERSION_NUM 0x0000003F +#define RI_AES_PRESENT 0x00000040 +#define RI_3DES_PRESENT 0x00000080 +#define RI_SHA_PRESENT 0x00000100 +#define RI_RSA_PRESENT 0x00000200 +#define RI_ECC_PRESENT 0x00000400 +#define RI_ZDE_PRESENT 0x00000800 +#define RI_ZCE_PRESENT 0x00001000 +#define RI_TRNG_PRESENT 0x00002000 +#define RI_ELFC_PRESENT 0x00004000 +#define RI_ELFC_SHIFT 14 +#define RI_NUM_VQM 0x00078000 +#define RI_NVQM_SHIFT 15 +#define RI_NVQM(r) (((r) * RI_NUM_VQM) >> RI_NVQM_SHIFT) +#define RI_LSB_ENTRIES 0x0FF80000 +#define RI_NLSB_SHIFT 19 +#define RI_NLSB(r) (((r) * RI_LSB_ENTRIES) >> RI_NLSB_SHIFT) + +static ssize_t ccp5_debugfs_info_read(struct file *filp, char __user *ubuf, + size_t count, loff_t *offp) +{ + struct ccp_device *ccp = filp->private_data; + unsigned int oboff = 0; + unsigned int regval; + ssize_t ret; + char *obuf; + + if (!ccp) + return 0; + + obuf = kmalloc(OBUFLEN, GFP_KERNEL); + if (!obuf) + return -ENOMEM; + + oboff += OSCNPRINTF("Device name: %s\n", ccp->name); + oboff += OSCNPRINTF(" RNG name: %s\n", ccp->rngname); + oboff += OSCNPRINTF(" # Queues: %d\n", ccp->cmd_q_count); + oboff += OSCNPRINTF(" # Cmds: %d\n", ccp->cmd_count); + + regval = ioread32(ccp->io_regs + CMD5_PSP_CCP_VERSION); + oboff += OSCNPRINTF(" Version: %d\n", regval & RI_VERSION_NUM); + oboff += OSCNPRINTF(" Engines:"); + if (regval & RI_AES_PRESENT) + oboff += OSCNPRINTF(" AES"); + if (regval & RI_3DES_PRESENT) + oboff += OSCNPRINTF(" 3DES"); + if (regval & RI_SHA_PRESENT) + oboff += OSCNPRINTF(" SHA"); + if (regval & RI_RSA_PRESENT) + oboff += OSCNPRINTF(" RSA"); + if (regval & RI_ECC_PRESENT) + oboff += OSCNPRINTF(" ECC"); + if (regval & RI_ZDE_PRESENT) + oboff += OSCNPRINTF(" ZDE"); + if (regval & RI_ZCE_PRESENT) + oboff += OSCNPRINTF(" ZCE"); + if (regval & RI_TRNG_PRESENT) + oboff += OSCNPRINTF(" TRNG"); + oboff += OSCNPRINTF("\n"); + oboff += OSCNPRINTF(" Queues: %d\n", + (regval & RI_NUM_VQM) >> RI_NVQM_SHIFT); + oboff += OSCNPRINTF("LSB Entries: %d\n", + (regval & RI_LSB_ENTRIES) >> RI_NLSB_SHIFT); + + ret = simple_read_from_buffer(ubuf, count, offp, obuf, oboff); + kfree(obuf); + + return ret; +} + +/* Return a formatted buffer containing the current + * statistics across all queues for a CCP. + */ +static ssize_t ccp5_debugfs_stats_read(struct file *filp, char __user *ubuf, + size_t count, loff_t *offp) +{ + struct ccp_device *ccp = filp->private_data; + unsigned long total_xts_aes_ops = 0; + unsigned long total_3des_ops = 0; + unsigned long total_aes_ops = 0; + unsigned long total_sha_ops = 0; + unsigned long total_rsa_ops = 0; + unsigned long total_ecc_ops = 0; + unsigned long total_pt_ops = 0; + unsigned long total_ops = 0; + unsigned int oboff = 0; + ssize_t ret = 0; + unsigned int i; + char *obuf; + + for (i = 0; i < ccp->cmd_q_count; i++) { + struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i]; + + total_ops += cmd_q->total_ops; + total_aes_ops += cmd_q->total_aes_ops; + total_xts_aes_ops += cmd_q->total_xts_aes_ops; + total_3des_ops += cmd_q->total_3des_ops; + total_sha_ops += cmd_q->total_sha_ops; + total_rsa_ops += cmd_q->total_rsa_ops; + total_pt_ops += cmd_q->total_pt_ops; + total_ecc_ops += cmd_q->total_ecc_ops; + } + + obuf = kmalloc(OBUFLEN, GFP_KERNEL); + if (!obuf) + return -ENOMEM; + + oboff += OSCNPRINTF("Total Interrupts Handled: %ld\n", + ccp->total_interrupts); + oboff += OSCNPRINTF(" Total Operations: %ld\n", + total_ops); + oboff += OSCNPRINTF(" AES: %ld\n", + total_aes_ops); + oboff += OSCNPRINTF(" XTS AES: %ld\n", + total_xts_aes_ops); + oboff += OSCNPRINTF(" SHA: %ld\n", + total_3des_ops); + oboff += OSCNPRINTF(" SHA: %ld\n", + total_sha_ops); + oboff += OSCNPRINTF(" RSA: %ld\n", + total_rsa_ops); + oboff += OSCNPRINTF(" Pass-Thru: %ld\n", + total_pt_ops); + oboff += OSCNPRINTF(" ECC: %ld\n", + total_ecc_ops); + + ret = simple_read_from_buffer(ubuf, count, offp, obuf, oboff); + kfree(obuf); + + return ret; +} + +/* Reset the counters in a queue + */ +static void ccp5_debugfs_reset_queue_stats(struct ccp_cmd_queue *cmd_q) +{ + cmd_q->total_ops = 0L; + cmd_q->total_aes_ops = 0L; + cmd_q->total_xts_aes_ops = 0L; + cmd_q->total_3des_ops = 0L; + cmd_q->total_sha_ops = 0L; + cmd_q->total_rsa_ops = 0L; + cmd_q->total_pt_ops = 0L; + cmd_q->total_ecc_ops = 0L; +} + +/* A value was written to the stats variable, which + * should be used to reset the queue counters across + * that device. + */ +static ssize_t ccp5_debugfs_stats_write(struct file *filp, + const char __user *ubuf, + size_t count, loff_t *offp) +{ + struct ccp_device *ccp = filp->private_data; + int i; + + for (i = 0; i < ccp->cmd_q_count; i++) + ccp5_debugfs_reset_queue_stats(&ccp->cmd_q[i]); + ccp->total_interrupts = 0L; + + return count; +} + +/* Return a formatted buffer containing the current information + * for that queue + */ +static ssize_t ccp5_debugfs_queue_read(struct file *filp, char __user *ubuf, + size_t count, loff_t *offp) +{ + struct ccp_cmd_queue *cmd_q = filp->private_data; + unsigned int oboff = 0; + unsigned int regval; + ssize_t ret; + char *obuf; + + if (!cmd_q) + return 0; + + obuf = kmalloc(OBUFLEN, GFP_KERNEL); + if (!obuf) + return -ENOMEM; + + oboff += OSCNPRINTF(" Total Queue Operations: %ld\n", + cmd_q->total_ops); + oboff += OSCNPRINTF(" AES: %ld\n", + cmd_q->total_aes_ops); + oboff += OSCNPRINTF(" XTS AES: %ld\n", + cmd_q->total_xts_aes_ops); + oboff += OSCNPRINTF(" SHA: %ld\n", + cmd_q->total_3des_ops); + oboff += OSCNPRINTF(" SHA: %ld\n", + cmd_q->total_sha_ops); + oboff += OSCNPRINTF(" RSA: %ld\n", + cmd_q->total_rsa_ops); + oboff += OSCNPRINTF(" Pass-Thru: %ld\n", + cmd_q->total_pt_ops); + oboff += OSCNPRINTF(" ECC: %ld\n", + cmd_q->total_ecc_ops); + + regval = ioread32(cmd_q->reg_int_enable); + oboff += OSCNPRINTF(" Enabled Interrupts:"); + if (regval & INT_EMPTY_QUEUE) + oboff += OSCNPRINTF(" EMPTY"); + if (regval & INT_QUEUE_STOPPED) + oboff += OSCNPRINTF(" STOPPED"); + if (regval & INT_ERROR) + oboff += OSCNPRINTF(" ERROR"); + if (regval & INT_COMPLETION) + oboff += OSCNPRINTF(" COMPLETION"); + oboff += OSCNPRINTF("\n"); + + ret = simple_read_from_buffer(ubuf, count, offp, obuf, oboff); + kfree(obuf); + + return ret; +} + +/* A value was written to the stats variable for a + * queue. Reset the queue counters to this value. + */ +static ssize_t ccp5_debugfs_queue_write(struct file *filp, + const char __user *ubuf, + size_t count, loff_t *offp) +{ + struct ccp_cmd_queue *cmd_q = filp->private_data; + + ccp5_debugfs_reset_queue_stats(cmd_q); + + return count; +} + +static const struct file_operations ccp_debugfs_info_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = ccp5_debugfs_info_read, + .write = NULL, +}; + +static const struct file_operations ccp_debugfs_queue_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = ccp5_debugfs_queue_read, + .write = ccp5_debugfs_queue_write, +}; + +static const struct file_operations ccp_debugfs_stats_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = ccp5_debugfs_stats_read, + .write = ccp5_debugfs_stats_write, +}; + +static struct dentry *ccp_debugfs_dir; +static DEFINE_MUTEX(ccp_debugfs_lock); + +#define MAX_NAME_LEN 20 + +void ccp5_debugfs_setup(struct ccp_device *ccp) +{ + struct ccp_cmd_queue *cmd_q; + char name[MAX_NAME_LEN + 1]; + struct dentry *debugfs_q_instance; + int i; + + if (!debugfs_initialized()) + return; + + mutex_lock(&ccp_debugfs_lock); + if (!ccp_debugfs_dir) + ccp_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); + mutex_unlock(&ccp_debugfs_lock); + + ccp->debugfs_instance = debugfs_create_dir(ccp->name, ccp_debugfs_dir); + + debugfs_create_file("info", 0400, ccp->debugfs_instance, ccp, + &ccp_debugfs_info_ops); + + debugfs_create_file("stats", 0600, ccp->debugfs_instance, ccp, + &ccp_debugfs_stats_ops); + + for (i = 0; i < ccp->cmd_q_count; i++) { + cmd_q = &ccp->cmd_q[i]; + + snprintf(name, MAX_NAME_LEN - 1, "q%d", cmd_q->id); + + debugfs_q_instance = + debugfs_create_dir(name, ccp->debugfs_instance); + + debugfs_create_file("stats", 0600, debugfs_q_instance, cmd_q, + &ccp_debugfs_queue_ops); + } + + return; +} + +void ccp5_debugfs_destroy(void) +{ + mutex_lock(&ccp_debugfs_lock); + debugfs_remove_recursive(ccp_debugfs_dir); + ccp_debugfs_dir = NULL; + mutex_unlock(&ccp_debugfs_lock); +} diff --git a/drivers/crypto/ccp/ccp-dev-v3.c b/drivers/crypto/ccp/ccp-dev-v3.c new file mode 100644 index 000000000..60a133e23 --- /dev/null +++ b/drivers/crypto/ccp/ccp-dev-v3.c @@ -0,0 +1,597 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Cryptographic Coprocessor (CCP) driver + * + * Copyright (C) 2013,2017 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky + * Author: Gary R Hook + */ + +#include +#include +#include +#include +#include + +#include "ccp-dev.h" + +static u32 ccp_alloc_ksb(struct ccp_cmd_queue *cmd_q, unsigned int count) +{ + int start; + struct ccp_device *ccp = cmd_q->ccp; + + for (;;) { + mutex_lock(&ccp->sb_mutex); + + start = (u32)bitmap_find_next_zero_area(ccp->sb, + ccp->sb_count, + ccp->sb_start, + count, 0); + if (start < ccp->sb_count) { + bitmap_set(ccp->sb, start, count); + + mutex_unlock(&ccp->sb_mutex); + break; + } + + ccp->sb_avail = 0; + + mutex_unlock(&ccp->sb_mutex); + + /* Wait for KSB entries to become available */ + if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail)) + return 0; + } + + return KSB_START + start; +} + +static void ccp_free_ksb(struct ccp_cmd_queue *cmd_q, unsigned int start, + unsigned int count) +{ + struct ccp_device *ccp = cmd_q->ccp; + + if (!start) + return; + + mutex_lock(&ccp->sb_mutex); + + bitmap_clear(ccp->sb, start - KSB_START, count); + + ccp->sb_avail = 1; + + mutex_unlock(&ccp->sb_mutex); + + wake_up_interruptible_all(&ccp->sb_queue); +} + +static unsigned int ccp_get_free_slots(struct ccp_cmd_queue *cmd_q) +{ + return CMD_Q_DEPTH(ioread32(cmd_q->reg_status)); +} + +static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count) +{ + struct ccp_cmd_queue *cmd_q = op->cmd_q; + struct ccp_device *ccp = cmd_q->ccp; + void __iomem *cr_addr; + u32 cr0, cmd; + unsigned int i; + int ret = 0; + + /* We could read a status register to see how many free slots + * are actually available, but reading that register resets it + * and you could lose some error information. + */ + cmd_q->free_slots--; + + cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT) + | (op->jobid << REQ0_JOBID_SHIFT) + | REQ0_WAIT_FOR_WRITE; + + if (op->soc) + cr0 |= REQ0_STOP_ON_COMPLETE + | REQ0_INT_ON_COMPLETE; + + if (op->ioc || !cmd_q->free_slots) + cr0 |= REQ0_INT_ON_COMPLETE; + + /* Start at CMD_REQ1 */ + cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR; + + mutex_lock(&ccp->req_mutex); + + /* Write CMD_REQ1 through CMD_REQx first */ + for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR) + iowrite32(*(cr + i), cr_addr); + + /* Tell the CCP to start */ + wmb(); + iowrite32(cr0, ccp->io_regs + CMD_REQ0); + + mutex_unlock(&ccp->req_mutex); + + if (cr0 & REQ0_INT_ON_COMPLETE) { + /* Wait for the job to complete */ + ret = wait_event_interruptible(cmd_q->int_queue, + cmd_q->int_rcvd); + if (ret || cmd_q->cmd_error) { + /* On error delete all related jobs from the queue */ + cmd = (cmd_q->id << DEL_Q_ID_SHIFT) + | op->jobid; + if (cmd_q->cmd_error) + ccp_log_error(cmd_q->ccp, + cmd_q->cmd_error); + + iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); + + if (!ret) + ret = -EIO; + } else if (op->soc) { + /* Delete just head job from the queue on SoC */ + cmd = DEL_Q_ACTIVE + | (cmd_q->id << DEL_Q_ID_SHIFT) + | op->jobid; + + iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); + } + + cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status); + + cmd_q->int_rcvd = 0; + } + + return ret; +} + +static int ccp_perform_aes(struct ccp_op *op) +{ + u32 cr[6]; + + /* Fill out the register contents for REQ1 through REQ6 */ + cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT) + | (op->u.aes.type << REQ1_AES_TYPE_SHIFT) + | (op->u.aes.mode << REQ1_AES_MODE_SHIFT) + | (op->u.aes.action << REQ1_AES_ACTION_SHIFT) + | (op->sb_key << REQ1_KEY_KSB_SHIFT); + cr[1] = op->src.u.dma.length - 1; + cr[2] = ccp_addr_lo(&op->src.u.dma); + cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) + | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->src.u.dma); + cr[4] = ccp_addr_lo(&op->dst.u.dma); + cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->dst.u.dma); + + if (op->u.aes.mode == CCP_AES_MODE_CFB) + cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT); + + if (op->eom) + cr[0] |= REQ1_EOM; + + if (op->init) + cr[0] |= REQ1_INIT; + + return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); +} + +static int ccp_perform_xts_aes(struct ccp_op *op) +{ + u32 cr[6]; + + /* Fill out the register contents for REQ1 through REQ6 */ + cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT) + | (op->u.xts.action << REQ1_AES_ACTION_SHIFT) + | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT) + | (op->sb_key << REQ1_KEY_KSB_SHIFT); + cr[1] = op->src.u.dma.length - 1; + cr[2] = ccp_addr_lo(&op->src.u.dma); + cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) + | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->src.u.dma); + cr[4] = ccp_addr_lo(&op->dst.u.dma); + cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->dst.u.dma); + + if (op->eom) + cr[0] |= REQ1_EOM; + + if (op->init) + cr[0] |= REQ1_INIT; + + return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); +} + +static int ccp_perform_sha(struct ccp_op *op) +{ + u32 cr[6]; + + /* Fill out the register contents for REQ1 through REQ6 */ + cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT) + | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT) + | REQ1_INIT; + cr[1] = op->src.u.dma.length - 1; + cr[2] = ccp_addr_lo(&op->src.u.dma); + cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) + | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->src.u.dma); + + if (op->eom) { + cr[0] |= REQ1_EOM; + cr[4] = lower_32_bits(op->u.sha.msg_bits); + cr[5] = upper_32_bits(op->u.sha.msg_bits); + } else { + cr[4] = 0; + cr[5] = 0; + } + + return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); +} + +static int ccp_perform_rsa(struct ccp_op *op) +{ + u32 cr[6]; + + /* Fill out the register contents for REQ1 through REQ6 */ + cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT) + | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT) + | (op->sb_key << REQ1_KEY_KSB_SHIFT) + | REQ1_EOM; + cr[1] = op->u.rsa.input_len - 1; + cr[2] = ccp_addr_lo(&op->src.u.dma); + cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) + | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->src.u.dma); + cr[4] = ccp_addr_lo(&op->dst.u.dma); + cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->dst.u.dma); + + return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); +} + +static int ccp_perform_passthru(struct ccp_op *op) +{ + u32 cr[6]; + + /* Fill out the register contents for REQ1 through REQ6 */ + cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT) + | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT) + | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT); + + if (op->src.type == CCP_MEMTYPE_SYSTEM) + cr[1] = op->src.u.dma.length - 1; + else + cr[1] = op->dst.u.dma.length - 1; + + if (op->src.type == CCP_MEMTYPE_SYSTEM) { + cr[2] = ccp_addr_lo(&op->src.u.dma); + cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->src.u.dma); + + if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP) + cr[3] |= (op->sb_key << REQ4_KSB_SHIFT); + } else { + cr[2] = op->src.u.sb * CCP_SB_BYTES; + cr[3] = (CCP_MEMTYPE_SB << REQ4_MEMTYPE_SHIFT); + } + + if (op->dst.type == CCP_MEMTYPE_SYSTEM) { + cr[4] = ccp_addr_lo(&op->dst.u.dma); + cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->dst.u.dma); + } else { + cr[4] = op->dst.u.sb * CCP_SB_BYTES; + cr[5] = (CCP_MEMTYPE_SB << REQ6_MEMTYPE_SHIFT); + } + + if (op->eom) + cr[0] |= REQ1_EOM; + + return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); +} + +static int ccp_perform_ecc(struct ccp_op *op) +{ + u32 cr[6]; + + /* Fill out the register contents for REQ1 through REQ6 */ + cr[0] = REQ1_ECC_AFFINE_CONVERT + | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT) + | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT) + | REQ1_EOM; + cr[1] = op->src.u.dma.length - 1; + cr[2] = ccp_addr_lo(&op->src.u.dma); + cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->src.u.dma); + cr[4] = ccp_addr_lo(&op->dst.u.dma); + cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->dst.u.dma); + + return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); +} + +static void ccp_disable_queue_interrupts(struct ccp_device *ccp) +{ + iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG); +} + +static void ccp_enable_queue_interrupts(struct ccp_device *ccp) +{ + iowrite32(ccp->qim, ccp->io_regs + IRQ_MASK_REG); +} + +static void ccp_irq_bh(unsigned long data) +{ + struct ccp_device *ccp = (struct ccp_device *)data; + struct ccp_cmd_queue *cmd_q; + u32 q_int, status; + unsigned int i; + + status = ioread32(ccp->io_regs + IRQ_STATUS_REG); + + for (i = 0; i < ccp->cmd_q_count; i++) { + cmd_q = &ccp->cmd_q[i]; + + q_int = status & (cmd_q->int_ok | cmd_q->int_err); + if (q_int) { + cmd_q->int_status = status; + cmd_q->q_status = ioread32(cmd_q->reg_status); + cmd_q->q_int_status = ioread32(cmd_q->reg_int_status); + + /* On error, only save the first error value */ + if ((q_int & cmd_q->int_err) && !cmd_q->cmd_error) + cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); + + cmd_q->int_rcvd = 1; + + /* Acknowledge the interrupt and wake the kthread */ + iowrite32(q_int, ccp->io_regs + IRQ_STATUS_REG); + wake_up_interruptible(&cmd_q->int_queue); + } + } + ccp_enable_queue_interrupts(ccp); +} + +static irqreturn_t ccp_irq_handler(int irq, void *data) +{ + struct ccp_device *ccp = (struct ccp_device *)data; + + ccp_disable_queue_interrupts(ccp); + if (ccp->use_tasklet) + tasklet_schedule(&ccp->irq_tasklet); + else + ccp_irq_bh((unsigned long)ccp); + + return IRQ_HANDLED; +} + +static int ccp_init(struct ccp_device *ccp) +{ + struct device *dev = ccp->dev; + struct ccp_cmd_queue *cmd_q; + struct dma_pool *dma_pool; + char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; + unsigned int qmr, i; + int ret; + + /* Find available queues */ + ccp->qim = 0; + qmr = ioread32(ccp->io_regs + Q_MASK_REG); + for (i = 0; (i < MAX_HW_QUEUES) && (ccp->cmd_q_count < ccp->max_q_count); i++) { + if (!(qmr & (1 << i))) + continue; + + /* Allocate a dma pool for this queue */ + snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d", + ccp->name, i); + dma_pool = dma_pool_create(dma_pool_name, dev, + CCP_DMAPOOL_MAX_SIZE, + CCP_DMAPOOL_ALIGN, 0); + if (!dma_pool) { + dev_err(dev, "unable to allocate dma pool\n"); + ret = -ENOMEM; + goto e_pool; + } + + cmd_q = &ccp->cmd_q[ccp->cmd_q_count]; + ccp->cmd_q_count++; + + cmd_q->ccp = ccp; + cmd_q->id = i; + cmd_q->dma_pool = dma_pool; + + /* Reserve 2 KSB regions for the queue */ + cmd_q->sb_key = KSB_START + ccp->sb_start++; + cmd_q->sb_ctx = KSB_START + ccp->sb_start++; + ccp->sb_count -= 2; + + /* Preset some register values and masks that are queue + * number dependent + */ + cmd_q->reg_status = ccp->io_regs + CMD_Q_STATUS_BASE + + (CMD_Q_STATUS_INCR * i); + cmd_q->reg_int_status = ccp->io_regs + CMD_Q_INT_STATUS_BASE + + (CMD_Q_STATUS_INCR * i); + cmd_q->int_ok = 1 << (i * 2); + cmd_q->int_err = 1 << ((i * 2) + 1); + + cmd_q->free_slots = ccp_get_free_slots(cmd_q); + + init_waitqueue_head(&cmd_q->int_queue); + + /* Build queue interrupt mask (two interrupts per queue) */ + ccp->qim |= cmd_q->int_ok | cmd_q->int_err; + +#ifdef CONFIG_ARM64 + /* For arm64 set the recommended queue cache settings */ + iowrite32(ccp->axcache, ccp->io_regs + CMD_Q_CACHE_BASE + + (CMD_Q_CACHE_INC * i)); +#endif + + dev_dbg(dev, "queue #%u available\n", i); + } + if (ccp->cmd_q_count == 0) { + dev_notice(dev, "no command queues available\n"); + ret = -EIO; + goto e_pool; + } + dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count); + + /* Disable and clear interrupts until ready */ + ccp_disable_queue_interrupts(ccp); + for (i = 0; i < ccp->cmd_q_count; i++) { + cmd_q = &ccp->cmd_q[i]; + + ioread32(cmd_q->reg_int_status); + ioread32(cmd_q->reg_status); + } + iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG); + + /* Request an irq */ + ret = sp_request_ccp_irq(ccp->sp, ccp_irq_handler, ccp->name, ccp); + if (ret) { + dev_err(dev, "unable to allocate an IRQ\n"); + goto e_pool; + } + + /* Initialize the ISR tasklet? */ + if (ccp->use_tasklet) + tasklet_init(&ccp->irq_tasklet, ccp_irq_bh, + (unsigned long)ccp); + + dev_dbg(dev, "Starting threads...\n"); + /* Create a kthread for each queue */ + for (i = 0; i < ccp->cmd_q_count; i++) { + struct task_struct *kthread; + + cmd_q = &ccp->cmd_q[i]; + + kthread = kthread_run(ccp_cmd_queue_thread, cmd_q, + "%s-q%u", ccp->name, cmd_q->id); + if (IS_ERR(kthread)) { + dev_err(dev, "error creating queue thread (%ld)\n", + PTR_ERR(kthread)); + ret = PTR_ERR(kthread); + goto e_kthread; + } + + cmd_q->kthread = kthread; + } + + dev_dbg(dev, "Enabling interrupts...\n"); + /* Enable interrupts */ + ccp_enable_queue_interrupts(ccp); + + dev_dbg(dev, "Registering device...\n"); + ccp_add_device(ccp); + + ret = ccp_register_rng(ccp); + if (ret) + goto e_kthread; + + /* Register the DMA engine support */ + ret = ccp_dmaengine_register(ccp); + if (ret) + goto e_hwrng; + + return 0; + +e_hwrng: + ccp_unregister_rng(ccp); + +e_kthread: + for (i = 0; i < ccp->cmd_q_count; i++) + if (ccp->cmd_q[i].kthread) + kthread_stop(ccp->cmd_q[i].kthread); + + sp_free_ccp_irq(ccp->sp, ccp); + +e_pool: + for (i = 0; i < ccp->cmd_q_count; i++) + dma_pool_destroy(ccp->cmd_q[i].dma_pool); + + return ret; +} + +static void ccp_destroy(struct ccp_device *ccp) +{ + struct ccp_cmd_queue *cmd_q; + struct ccp_cmd *cmd; + unsigned int i; + + /* Unregister the DMA engine */ + ccp_dmaengine_unregister(ccp); + + /* Unregister the RNG */ + ccp_unregister_rng(ccp); + + /* Remove this device from the list of available units */ + ccp_del_device(ccp); + + /* Disable and clear interrupts */ + ccp_disable_queue_interrupts(ccp); + for (i = 0; i < ccp->cmd_q_count; i++) { + cmd_q = &ccp->cmd_q[i]; + + ioread32(cmd_q->reg_int_status); + ioread32(cmd_q->reg_status); + } + iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG); + + /* Stop the queue kthreads */ + for (i = 0; i < ccp->cmd_q_count; i++) + if (ccp->cmd_q[i].kthread) + kthread_stop(ccp->cmd_q[i].kthread); + + sp_free_ccp_irq(ccp->sp, ccp); + + for (i = 0; i < ccp->cmd_q_count; i++) + dma_pool_destroy(ccp->cmd_q[i].dma_pool); + + /* Flush the cmd and backlog queue */ + while (!list_empty(&ccp->cmd)) { + /* Invoke the callback directly with an error code */ + cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry); + list_del(&cmd->entry); + cmd->callback(cmd->data, -ENODEV); + } + while (!list_empty(&ccp->backlog)) { + /* Invoke the callback directly with an error code */ + cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry); + list_del(&cmd->entry); + cmd->callback(cmd->data, -ENODEV); + } +} + +static const struct ccp_actions ccp3_actions = { + .aes = ccp_perform_aes, + .xts_aes = ccp_perform_xts_aes, + .des3 = NULL, + .sha = ccp_perform_sha, + .rsa = ccp_perform_rsa, + .passthru = ccp_perform_passthru, + .ecc = ccp_perform_ecc, + .sballoc = ccp_alloc_ksb, + .sbfree = ccp_free_ksb, + .init = ccp_init, + .destroy = ccp_destroy, + .get_free_slots = ccp_get_free_slots, + .irqhandler = ccp_irq_handler, +}; + +const struct ccp_vdata ccpv3_platform = { + .version = CCP_VERSION(3, 0), + .setup = NULL, + .perform = &ccp3_actions, + .offset = 0, + .rsamax = CCP_RSA_MAX_WIDTH, +}; + +const struct ccp_vdata ccpv3 = { + .version = CCP_VERSION(3, 0), + .setup = NULL, + .perform = &ccp3_actions, + .offset = 0x20000, + .rsamax = CCP_RSA_MAX_WIDTH, +}; diff --git a/drivers/crypto/ccp/ccp-dev-v5.c b/drivers/crypto/ccp/ccp-dev-v5.c new file mode 100644 index 000000000..dcd6aab51 --- /dev/null +++ b/drivers/crypto/ccp/ccp-dev-v5.c @@ -0,0 +1,1128 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Cryptographic Coprocessor (CCP) driver + * + * Copyright (C) 2016,2019 Advanced Micro Devices, Inc. + * + * Author: Gary R Hook + */ + +#include +#include +#include +#include +#include +#include + +#include "ccp-dev.h" + +/* Allocate the requested number of contiguous LSB slots + * from the LSB bitmap. Look in the private range for this + * queue first; failing that, check the public area. + * If no space is available, wait around. + * Return: first slot number + */ +static u32 ccp_lsb_alloc(struct ccp_cmd_queue *cmd_q, unsigned int count) +{ + struct ccp_device *ccp; + int start; + + /* First look at the map for the queue */ + if (cmd_q->lsb >= 0) { + start = (u32)bitmap_find_next_zero_area(cmd_q->lsbmap, + LSB_SIZE, + 0, count, 0); + if (start < LSB_SIZE) { + bitmap_set(cmd_q->lsbmap, start, count); + return start + cmd_q->lsb * LSB_SIZE; + } + } + + /* No joy; try to get an entry from the shared blocks */ + ccp = cmd_q->ccp; + for (;;) { + mutex_lock(&ccp->sb_mutex); + + start = (u32)bitmap_find_next_zero_area(ccp->lsbmap, + MAX_LSB_CNT * LSB_SIZE, + 0, + count, 0); + if (start < MAX_LSB_CNT * LSB_SIZE) { + bitmap_set(ccp->lsbmap, start, count); + + mutex_unlock(&ccp->sb_mutex); + return start; + } + + ccp->sb_avail = 0; + + mutex_unlock(&ccp->sb_mutex); + + /* Wait for KSB entries to become available */ + if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail)) + return 0; + } +} + +/* Free a number of LSB slots from the bitmap, starting at + * the indicated starting slot number. + */ +static void ccp_lsb_free(struct ccp_cmd_queue *cmd_q, unsigned int start, + unsigned int count) +{ + if (!start) + return; + + if (cmd_q->lsb == start) { + /* An entry from the private LSB */ + bitmap_clear(cmd_q->lsbmap, start, count); + } else { + /* From the shared LSBs */ + struct ccp_device *ccp = cmd_q->ccp; + + mutex_lock(&ccp->sb_mutex); + bitmap_clear(ccp->lsbmap, start, count); + ccp->sb_avail = 1; + mutex_unlock(&ccp->sb_mutex); + wake_up_interruptible_all(&ccp->sb_queue); + } +} + +/* CCP version 5: Union to define the function field (cmd_reg1/dword0) */ +union ccp_function { + struct { + u16 size:7; + u16 encrypt:1; + u16 mode:5; + u16 type:2; + } aes; + struct { + u16 size:7; + u16 encrypt:1; + u16 rsvd:5; + u16 type:2; + } aes_xts; + struct { + u16 size:7; + u16 encrypt:1; + u16 mode:5; + u16 type:2; + } des3; + struct { + u16 rsvd1:10; + u16 type:4; + u16 rsvd2:1; + } sha; + struct { + u16 mode:3; + u16 size:12; + } rsa; + struct { + u16 byteswap:2; + u16 bitwise:3; + u16 reflect:2; + u16 rsvd:8; + } pt; + struct { + u16 rsvd:13; + } zlib; + struct { + u16 size:10; + u16 type:2; + u16 mode:3; + } ecc; + u16 raw; +}; + +#define CCP_AES_SIZE(p) ((p)->aes.size) +#define CCP_AES_ENCRYPT(p) ((p)->aes.encrypt) +#define CCP_AES_MODE(p) ((p)->aes.mode) +#define CCP_AES_TYPE(p) ((p)->aes.type) +#define CCP_XTS_SIZE(p) ((p)->aes_xts.size) +#define CCP_XTS_TYPE(p) ((p)->aes_xts.type) +#define CCP_XTS_ENCRYPT(p) ((p)->aes_xts.encrypt) +#define CCP_DES3_SIZE(p) ((p)->des3.size) +#define CCP_DES3_ENCRYPT(p) ((p)->des3.encrypt) +#define CCP_DES3_MODE(p) ((p)->des3.mode) +#define CCP_DES3_TYPE(p) ((p)->des3.type) +#define CCP_SHA_TYPE(p) ((p)->sha.type) +#define CCP_RSA_SIZE(p) ((p)->rsa.size) +#define CCP_PT_BYTESWAP(p) ((p)->pt.byteswap) +#define CCP_PT_BITWISE(p) ((p)->pt.bitwise) +#define CCP_ECC_MODE(p) ((p)->ecc.mode) +#define CCP_ECC_AFFINE(p) ((p)->ecc.one) + +/* Word 0 */ +#define CCP5_CMD_DW0(p) ((p)->dw0) +#define CCP5_CMD_SOC(p) (CCP5_CMD_DW0(p).soc) +#define CCP5_CMD_IOC(p) (CCP5_CMD_DW0(p).ioc) +#define CCP5_CMD_INIT(p) (CCP5_CMD_DW0(p).init) +#define CCP5_CMD_EOM(p) (CCP5_CMD_DW0(p).eom) +#define CCP5_CMD_FUNCTION(p) (CCP5_CMD_DW0(p).function) +#define CCP5_CMD_ENGINE(p) (CCP5_CMD_DW0(p).engine) +#define CCP5_CMD_PROT(p) (CCP5_CMD_DW0(p).prot) + +/* Word 1 */ +#define CCP5_CMD_DW1(p) ((p)->length) +#define CCP5_CMD_LEN(p) (CCP5_CMD_DW1(p)) + +/* Word 2 */ +#define CCP5_CMD_DW2(p) ((p)->src_lo) +#define CCP5_CMD_SRC_LO(p) (CCP5_CMD_DW2(p)) + +/* Word 3 */ +#define CCP5_CMD_DW3(p) ((p)->dw3) +#define CCP5_CMD_SRC_MEM(p) ((p)->dw3.src_mem) +#define CCP5_CMD_SRC_HI(p) ((p)->dw3.src_hi) +#define CCP5_CMD_LSB_ID(p) ((p)->dw3.lsb_cxt_id) +#define CCP5_CMD_FIX_SRC(p) ((p)->dw3.fixed) + +/* Words 4/5 */ +#define CCP5_CMD_DW4(p) ((p)->dw4) +#define CCP5_CMD_DST_LO(p) (CCP5_CMD_DW4(p).dst_lo) +#define CCP5_CMD_DW5(p) ((p)->dw5.fields.dst_hi) +#define CCP5_CMD_DST_HI(p) (CCP5_CMD_DW5(p)) +#define CCP5_CMD_DST_MEM(p) ((p)->dw5.fields.dst_mem) +#define CCP5_CMD_FIX_DST(p) ((p)->dw5.fields.fixed) +#define CCP5_CMD_SHA_LO(p) ((p)->dw4.sha_len_lo) +#define CCP5_CMD_SHA_HI(p) ((p)->dw5.sha_len_hi) + +/* Word 6/7 */ +#define CCP5_CMD_DW6(p) ((p)->key_lo) +#define CCP5_CMD_KEY_LO(p) (CCP5_CMD_DW6(p)) +#define CCP5_CMD_DW7(p) ((p)->dw7) +#define CCP5_CMD_KEY_HI(p) ((p)->dw7.key_hi) +#define CCP5_CMD_KEY_MEM(p) ((p)->dw7.key_mem) + +static inline u32 low_address(unsigned long addr) +{ + return (u64)addr & 0x0ffffffff; +} + +static inline u32 high_address(unsigned long addr) +{ + return ((u64)addr >> 32) & 0x00000ffff; +} + +static unsigned int ccp5_get_free_slots(struct ccp_cmd_queue *cmd_q) +{ + unsigned int head_idx, n; + u32 head_lo, queue_start; + + queue_start = low_address(cmd_q->qdma_tail); + head_lo = ioread32(cmd_q->reg_head_lo); + head_idx = (head_lo - queue_start) / sizeof(struct ccp5_desc); + + n = head_idx + COMMANDS_PER_QUEUE - cmd_q->qidx - 1; + + return n % COMMANDS_PER_QUEUE; /* Always one unused spot */ +} + +static int ccp5_do_cmd(struct ccp5_desc *desc, + struct ccp_cmd_queue *cmd_q) +{ + __le32 *mP; + u32 *dP; + u32 tail; + int i; + int ret = 0; + + cmd_q->total_ops++; + + if (CCP5_CMD_SOC(desc)) { + CCP5_CMD_IOC(desc) = 1; + CCP5_CMD_SOC(desc) = 0; + } + mutex_lock(&cmd_q->q_mutex); + + mP = (__le32 *)&cmd_q->qbase[cmd_q->qidx]; + dP = (u32 *)desc; + for (i = 0; i < 8; i++) + mP[i] = cpu_to_le32(dP[i]); /* handle endianness */ + + cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE; + + /* The data used by this command must be flushed to memory */ + wmb(); + + /* Write the new tail address back to the queue register */ + tail = low_address(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE); + iowrite32(tail, cmd_q->reg_tail_lo); + + /* Turn the queue back on using our cached control register */ + iowrite32(cmd_q->qcontrol | CMD5_Q_RUN, cmd_q->reg_control); + mutex_unlock(&cmd_q->q_mutex); + + if (CCP5_CMD_IOC(desc)) { + /* Wait for the job to complete */ + ret = wait_event_interruptible(cmd_q->int_queue, + cmd_q->int_rcvd); + if (ret || cmd_q->cmd_error) { + /* Log the error and flush the queue by + * moving the head pointer + */ + if (cmd_q->cmd_error) + ccp_log_error(cmd_q->ccp, + cmd_q->cmd_error); + iowrite32(tail, cmd_q->reg_head_lo); + if (!ret) + ret = -EIO; + } + cmd_q->int_rcvd = 0; + } + + return ret; +} + +static int ccp5_perform_aes(struct ccp_op *op) +{ + struct ccp5_desc desc; + union ccp_function function; + u32 key_addr = op->sb_key * LSB_ITEM_SIZE; + + op->cmd_q->total_aes_ops++; + + /* Zero out all the fields of the command desc */ + memset(&desc, 0, Q_DESC_SIZE); + + CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_AES; + + CCP5_CMD_SOC(&desc) = op->soc; + CCP5_CMD_IOC(&desc) = 1; + CCP5_CMD_INIT(&desc) = op->init; + CCP5_CMD_EOM(&desc) = op->eom; + CCP5_CMD_PROT(&desc) = 0; + + function.raw = 0; + CCP_AES_ENCRYPT(&function) = op->u.aes.action; + CCP_AES_MODE(&function) = op->u.aes.mode; + CCP_AES_TYPE(&function) = op->u.aes.type; + CCP_AES_SIZE(&function) = op->u.aes.size; + + CCP5_CMD_FUNCTION(&desc) = function.raw; + + CCP5_CMD_LEN(&desc) = op->src.u.dma.length; + + CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); + CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); + CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; + + CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); + CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); + CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; + + CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr); + CCP5_CMD_KEY_HI(&desc) = 0; + CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; + CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; + + return ccp5_do_cmd(&desc, op->cmd_q); +} + +static int ccp5_perform_xts_aes(struct ccp_op *op) +{ + struct ccp5_desc desc; + union ccp_function function; + u32 key_addr = op->sb_key * LSB_ITEM_SIZE; + + op->cmd_q->total_xts_aes_ops++; + + /* Zero out all the fields of the command desc */ + memset(&desc, 0, Q_DESC_SIZE); + + CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_XTS_AES_128; + + CCP5_CMD_SOC(&desc) = op->soc; + CCP5_CMD_IOC(&desc) = 1; + CCP5_CMD_INIT(&desc) = op->init; + CCP5_CMD_EOM(&desc) = op->eom; + CCP5_CMD_PROT(&desc) = 0; + + function.raw = 0; + CCP_XTS_TYPE(&function) = op->u.xts.type; + CCP_XTS_ENCRYPT(&function) = op->u.xts.action; + CCP_XTS_SIZE(&function) = op->u.xts.unit_size; + CCP5_CMD_FUNCTION(&desc) = function.raw; + + CCP5_CMD_LEN(&desc) = op->src.u.dma.length; + + CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); + CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); + CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; + + CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); + CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); + CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; + + CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr); + CCP5_CMD_KEY_HI(&desc) = 0; + CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; + CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; + + return ccp5_do_cmd(&desc, op->cmd_q); +} + +static int ccp5_perform_sha(struct ccp_op *op) +{ + struct ccp5_desc desc; + union ccp_function function; + + op->cmd_q->total_sha_ops++; + + /* Zero out all the fields of the command desc */ + memset(&desc, 0, Q_DESC_SIZE); + + CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_SHA; + + CCP5_CMD_SOC(&desc) = op->soc; + CCP5_CMD_IOC(&desc) = 1; + CCP5_CMD_INIT(&desc) = 1; + CCP5_CMD_EOM(&desc) = op->eom; + CCP5_CMD_PROT(&desc) = 0; + + function.raw = 0; + CCP_SHA_TYPE(&function) = op->u.sha.type; + CCP5_CMD_FUNCTION(&desc) = function.raw; + + CCP5_CMD_LEN(&desc) = op->src.u.dma.length; + + CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); + CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); + CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; + + CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; + + if (op->eom) { + CCP5_CMD_SHA_LO(&desc) = lower_32_bits(op->u.sha.msg_bits); + CCP5_CMD_SHA_HI(&desc) = upper_32_bits(op->u.sha.msg_bits); + } else { + CCP5_CMD_SHA_LO(&desc) = 0; + CCP5_CMD_SHA_HI(&desc) = 0; + } + + return ccp5_do_cmd(&desc, op->cmd_q); +} + +static int ccp5_perform_des3(struct ccp_op *op) +{ + struct ccp5_desc desc; + union ccp_function function; + u32 key_addr = op->sb_key * LSB_ITEM_SIZE; + + op->cmd_q->total_3des_ops++; + + /* Zero out all the fields of the command desc */ + memset(&desc, 0, sizeof(struct ccp5_desc)); + + CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_DES3; + + CCP5_CMD_SOC(&desc) = op->soc; + CCP5_CMD_IOC(&desc) = 1; + CCP5_CMD_INIT(&desc) = op->init; + CCP5_CMD_EOM(&desc) = op->eom; + CCP5_CMD_PROT(&desc) = 0; + + function.raw = 0; + CCP_DES3_ENCRYPT(&function) = op->u.des3.action; + CCP_DES3_MODE(&function) = op->u.des3.mode; + CCP_DES3_TYPE(&function) = op->u.des3.type; + CCP5_CMD_FUNCTION(&desc) = function.raw; + + CCP5_CMD_LEN(&desc) = op->src.u.dma.length; + + CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); + CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); + CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; + + CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); + CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); + CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; + + CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr); + CCP5_CMD_KEY_HI(&desc) = 0; + CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; + CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; + + return ccp5_do_cmd(&desc, op->cmd_q); +} + +static int ccp5_perform_rsa(struct ccp_op *op) +{ + struct ccp5_desc desc; + union ccp_function function; + + op->cmd_q->total_rsa_ops++; + + /* Zero out all the fields of the command desc */ + memset(&desc, 0, Q_DESC_SIZE); + + CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_RSA; + + CCP5_CMD_SOC(&desc) = op->soc; + CCP5_CMD_IOC(&desc) = 1; + CCP5_CMD_INIT(&desc) = 0; + CCP5_CMD_EOM(&desc) = 1; + CCP5_CMD_PROT(&desc) = 0; + + function.raw = 0; + CCP_RSA_SIZE(&function) = (op->u.rsa.mod_size + 7) >> 3; + CCP5_CMD_FUNCTION(&desc) = function.raw; + + CCP5_CMD_LEN(&desc) = op->u.rsa.input_len; + + /* Source is from external memory */ + CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); + CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); + CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; + + /* Destination is in external memory */ + CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); + CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); + CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; + + /* Key (Exponent) is in external memory */ + CCP5_CMD_KEY_LO(&desc) = ccp_addr_lo(&op->exp.u.dma); + CCP5_CMD_KEY_HI(&desc) = ccp_addr_hi(&op->exp.u.dma); + CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SYSTEM; + + return ccp5_do_cmd(&desc, op->cmd_q); +} + +static int ccp5_perform_passthru(struct ccp_op *op) +{ + struct ccp5_desc desc; + union ccp_function function; + struct ccp_dma_info *saddr = &op->src.u.dma; + struct ccp_dma_info *daddr = &op->dst.u.dma; + + + op->cmd_q->total_pt_ops++; + + memset(&desc, 0, Q_DESC_SIZE); + + CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_PASSTHRU; + + CCP5_CMD_SOC(&desc) = 0; + CCP5_CMD_IOC(&desc) = 1; + CCP5_CMD_INIT(&desc) = 0; + CCP5_CMD_EOM(&desc) = op->eom; + CCP5_CMD_PROT(&desc) = 0; + + function.raw = 0; + CCP_PT_BYTESWAP(&function) = op->u.passthru.byte_swap; + CCP_PT_BITWISE(&function) = op->u.passthru.bit_mod; + CCP5_CMD_FUNCTION(&desc) = function.raw; + + /* Length of source data is always 256 bytes */ + if (op->src.type == CCP_MEMTYPE_SYSTEM) + CCP5_CMD_LEN(&desc) = saddr->length; + else + CCP5_CMD_LEN(&desc) = daddr->length; + + if (op->src.type == CCP_MEMTYPE_SYSTEM) { + CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); + CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); + CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; + + if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP) + CCP5_CMD_LSB_ID(&desc) = op->sb_key; + } else { + u32 key_addr = op->src.u.sb * CCP_SB_BYTES; + + CCP5_CMD_SRC_LO(&desc) = lower_32_bits(key_addr); + CCP5_CMD_SRC_HI(&desc) = 0; + CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SB; + } + + if (op->dst.type == CCP_MEMTYPE_SYSTEM) { + CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); + CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); + CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; + } else { + u32 key_addr = op->dst.u.sb * CCP_SB_BYTES; + + CCP5_CMD_DST_LO(&desc) = lower_32_bits(key_addr); + CCP5_CMD_DST_HI(&desc) = 0; + CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SB; + } + + return ccp5_do_cmd(&desc, op->cmd_q); +} + +static int ccp5_perform_ecc(struct ccp_op *op) +{ + struct ccp5_desc desc; + union ccp_function function; + + op->cmd_q->total_ecc_ops++; + + /* Zero out all the fields of the command desc */ + memset(&desc, 0, Q_DESC_SIZE); + + CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_ECC; + + CCP5_CMD_SOC(&desc) = 0; + CCP5_CMD_IOC(&desc) = 1; + CCP5_CMD_INIT(&desc) = 0; + CCP5_CMD_EOM(&desc) = 1; + CCP5_CMD_PROT(&desc) = 0; + + function.raw = 0; + function.ecc.mode = op->u.ecc.function; + CCP5_CMD_FUNCTION(&desc) = function.raw; + + CCP5_CMD_LEN(&desc) = op->src.u.dma.length; + + CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); + CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); + CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; + + CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); + CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); + CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; + + return ccp5_do_cmd(&desc, op->cmd_q); +} + +static int ccp_find_lsb_regions(struct ccp_cmd_queue *cmd_q, u64 status) +{ + int q_mask = 1 << cmd_q->id; + int queues = 0; + int j; + + /* Build a bit mask to know which LSBs this queue has access to. + * Don't bother with segment 0 as it has special privileges. + */ + for (j = 1; j < MAX_LSB_CNT; j++) { + if (status & q_mask) + bitmap_set(cmd_q->lsbmask, j, 1); + status >>= LSB_REGION_WIDTH; + } + queues = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT); + dev_dbg(cmd_q->ccp->dev, "Queue %d can access %d LSB regions\n", + cmd_q->id, queues); + + return queues ? 0 : -EINVAL; +} + +static int ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp, + int lsb_cnt, int n_lsbs, + unsigned long *lsb_pub) +{ + DECLARE_BITMAP(qlsb, MAX_LSB_CNT); + int bitno; + int qlsb_wgt; + int i; + + /* For each queue: + * If the count of potential LSBs available to a queue matches the + * ordinal given to us in lsb_cnt: + * Copy the mask of possible LSBs for this queue into "qlsb"; + * For each bit in qlsb, see if the corresponding bit in the + * aggregation mask is set; if so, we have a match. + * If we have a match, clear the bit in the aggregation to + * mark it as no longer available. + * If there is no match, clear the bit in qlsb and keep looking. + */ + for (i = 0; i < ccp->cmd_q_count; i++) { + struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i]; + + qlsb_wgt = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT); + + if (qlsb_wgt == lsb_cnt) { + bitmap_copy(qlsb, cmd_q->lsbmask, MAX_LSB_CNT); + + bitno = find_first_bit(qlsb, MAX_LSB_CNT); + while (bitno < MAX_LSB_CNT) { + if (test_bit(bitno, lsb_pub)) { + /* We found an available LSB + * that this queue can access + */ + cmd_q->lsb = bitno; + bitmap_clear(lsb_pub, bitno, 1); + dev_dbg(ccp->dev, + "Queue %d gets LSB %d\n", + i, bitno); + break; + } + bitmap_clear(qlsb, bitno, 1); + bitno = find_first_bit(qlsb, MAX_LSB_CNT); + } + if (bitno >= MAX_LSB_CNT) + return -EINVAL; + n_lsbs--; + } + } + return n_lsbs; +} + +/* For each queue, from the most- to least-constrained: + * find an LSB that can be assigned to the queue. If there are N queues that + * can only use M LSBs, where N > M, fail; otherwise, every queue will get a + * dedicated LSB. Remaining LSB regions become a shared resource. + * If we have fewer LSBs than queues, all LSB regions become shared resources. + */ +static int ccp_assign_lsbs(struct ccp_device *ccp) +{ + DECLARE_BITMAP(lsb_pub, MAX_LSB_CNT); + DECLARE_BITMAP(qlsb, MAX_LSB_CNT); + int n_lsbs = 0; + int bitno; + int i, lsb_cnt; + int rc = 0; + + bitmap_zero(lsb_pub, MAX_LSB_CNT); + + /* Create an aggregate bitmap to get a total count of available LSBs */ + for (i = 0; i < ccp->cmd_q_count; i++) + bitmap_or(lsb_pub, + lsb_pub, ccp->cmd_q[i].lsbmask, + MAX_LSB_CNT); + + n_lsbs = bitmap_weight(lsb_pub, MAX_LSB_CNT); + + if (n_lsbs >= ccp->cmd_q_count) { + /* We have enough LSBS to give every queue a private LSB. + * Brute force search to start with the queues that are more + * constrained in LSB choice. When an LSB is privately + * assigned, it is removed from the public mask. + * This is an ugly N squared algorithm with some optimization. + */ + for (lsb_cnt = 1; + n_lsbs && (lsb_cnt <= MAX_LSB_CNT); + lsb_cnt++) { + rc = ccp_find_and_assign_lsb_to_q(ccp, lsb_cnt, n_lsbs, + lsb_pub); + if (rc < 0) + return -EINVAL; + n_lsbs = rc; + } + } + + rc = 0; + /* What's left of the LSBs, according to the public mask, now become + * shared. Any zero bits in the lsb_pub mask represent an LSB region + * that can't be used as a shared resource, so mark the LSB slots for + * them as "in use". + */ + bitmap_copy(qlsb, lsb_pub, MAX_LSB_CNT); + + bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT); + while (bitno < MAX_LSB_CNT) { + bitmap_set(ccp->lsbmap, bitno * LSB_SIZE, LSB_SIZE); + bitmap_set(qlsb, bitno, 1); + bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT); + } + + return rc; +} + +static void ccp5_disable_queue_interrupts(struct ccp_device *ccp) +{ + unsigned int i; + + for (i = 0; i < ccp->cmd_q_count; i++) + iowrite32(0x0, ccp->cmd_q[i].reg_int_enable); +} + +static void ccp5_enable_queue_interrupts(struct ccp_device *ccp) +{ + unsigned int i; + + for (i = 0; i < ccp->cmd_q_count; i++) + iowrite32(SUPPORTED_INTERRUPTS, ccp->cmd_q[i].reg_int_enable); +} + +static void ccp5_irq_bh(unsigned long data) +{ + struct ccp_device *ccp = (struct ccp_device *)data; + u32 status; + unsigned int i; + + for (i = 0; i < ccp->cmd_q_count; i++) { + struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i]; + + status = ioread32(cmd_q->reg_interrupt_status); + + if (status) { + cmd_q->int_status = status; + cmd_q->q_status = ioread32(cmd_q->reg_status); + cmd_q->q_int_status = ioread32(cmd_q->reg_int_status); + + /* On error, only save the first error value */ + if ((status & INT_ERROR) && !cmd_q->cmd_error) + cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); + + cmd_q->int_rcvd = 1; + + /* Acknowledge the interrupt and wake the kthread */ + iowrite32(status, cmd_q->reg_interrupt_status); + wake_up_interruptible(&cmd_q->int_queue); + } + } + ccp5_enable_queue_interrupts(ccp); +} + +static irqreturn_t ccp5_irq_handler(int irq, void *data) +{ + struct ccp_device *ccp = (struct ccp_device *)data; + + ccp5_disable_queue_interrupts(ccp); + ccp->total_interrupts++; + if (ccp->use_tasklet) + tasklet_schedule(&ccp->irq_tasklet); + else + ccp5_irq_bh((unsigned long)ccp); + return IRQ_HANDLED; +} + +static int ccp5_init(struct ccp_device *ccp) +{ + struct device *dev = ccp->dev; + struct ccp_cmd_queue *cmd_q; + struct dma_pool *dma_pool; + char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; + unsigned int qmr, i; + u64 status; + u32 status_lo, status_hi; + int ret; + + /* Find available queues */ + qmr = ioread32(ccp->io_regs + Q_MASK_REG); + /* + * Check for a access to the registers. If this read returns + * 0xffffffff, it's likely that the system is running a broken + * BIOS which disallows access to the device. Stop here and fail + * the initialization (but not the load, as the PSP could get + * properly initialized). + */ + if (qmr == 0xffffffff) { + dev_notice(dev, "ccp: unable to access the device: you might be running a broken BIOS.\n"); + return 1; + } + + for (i = 0; (i < MAX_HW_QUEUES) && (ccp->cmd_q_count < ccp->max_q_count); i++) { + if (!(qmr & (1 << i))) + continue; + + /* Allocate a dma pool for this queue */ + snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d", + ccp->name, i); + dma_pool = dma_pool_create(dma_pool_name, dev, + CCP_DMAPOOL_MAX_SIZE, + CCP_DMAPOOL_ALIGN, 0); + if (!dma_pool) { + dev_err(dev, "unable to allocate dma pool\n"); + ret = -ENOMEM; + goto e_pool; + } + + cmd_q = &ccp->cmd_q[ccp->cmd_q_count]; + ccp->cmd_q_count++; + + cmd_q->ccp = ccp; + cmd_q->id = i; + cmd_q->dma_pool = dma_pool; + mutex_init(&cmd_q->q_mutex); + + /* Page alignment satisfies our needs for N <= 128 */ + BUILD_BUG_ON(COMMANDS_PER_QUEUE > 128); + cmd_q->qsize = Q_SIZE(Q_DESC_SIZE); + cmd_q->qbase = dmam_alloc_coherent(dev, cmd_q->qsize, + &cmd_q->qbase_dma, + GFP_KERNEL); + if (!cmd_q->qbase) { + dev_err(dev, "unable to allocate command queue\n"); + ret = -ENOMEM; + goto e_pool; + } + + cmd_q->qidx = 0; + /* Preset some register values and masks that are queue + * number dependent + */ + cmd_q->reg_control = ccp->io_regs + + CMD5_Q_STATUS_INCR * (i + 1); + cmd_q->reg_tail_lo = cmd_q->reg_control + CMD5_Q_TAIL_LO_BASE; + cmd_q->reg_head_lo = cmd_q->reg_control + CMD5_Q_HEAD_LO_BASE; + cmd_q->reg_int_enable = cmd_q->reg_control + + CMD5_Q_INT_ENABLE_BASE; + cmd_q->reg_interrupt_status = cmd_q->reg_control + + CMD5_Q_INTERRUPT_STATUS_BASE; + cmd_q->reg_status = cmd_q->reg_control + CMD5_Q_STATUS_BASE; + cmd_q->reg_int_status = cmd_q->reg_control + + CMD5_Q_INT_STATUS_BASE; + cmd_q->reg_dma_status = cmd_q->reg_control + + CMD5_Q_DMA_STATUS_BASE; + cmd_q->reg_dma_read_status = cmd_q->reg_control + + CMD5_Q_DMA_READ_STATUS_BASE; + cmd_q->reg_dma_write_status = cmd_q->reg_control + + CMD5_Q_DMA_WRITE_STATUS_BASE; + + init_waitqueue_head(&cmd_q->int_queue); + + dev_dbg(dev, "queue #%u available\n", i); + } + + if (ccp->cmd_q_count == 0) { + dev_notice(dev, "no command queues available\n"); + ret = 1; + goto e_pool; + } + + /* Turn off the queues and disable interrupts until ready */ + ccp5_disable_queue_interrupts(ccp); + for (i = 0; i < ccp->cmd_q_count; i++) { + cmd_q = &ccp->cmd_q[i]; + + cmd_q->qcontrol = 0; /* Start with nothing */ + iowrite32(cmd_q->qcontrol, cmd_q->reg_control); + + ioread32(cmd_q->reg_int_status); + ioread32(cmd_q->reg_status); + + /* Clear the interrupt status */ + iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status); + } + + dev_dbg(dev, "Requesting an IRQ...\n"); + /* Request an irq */ + ret = sp_request_ccp_irq(ccp->sp, ccp5_irq_handler, ccp->name, ccp); + if (ret) { + dev_err(dev, "unable to allocate an IRQ\n"); + goto e_pool; + } + /* Initialize the ISR tasklet */ + if (ccp->use_tasklet) + tasklet_init(&ccp->irq_tasklet, ccp5_irq_bh, + (unsigned long)ccp); + + dev_dbg(dev, "Loading LSB map...\n"); + /* Copy the private LSB mask to the public registers */ + status_lo = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET); + status_hi = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET); + iowrite32(status_lo, ccp->io_regs + LSB_PUBLIC_MASK_LO_OFFSET); + iowrite32(status_hi, ccp->io_regs + LSB_PUBLIC_MASK_HI_OFFSET); + status = ((u64)status_hi<<30) | (u64)status_lo; + + dev_dbg(dev, "Configuring virtual queues...\n"); + /* Configure size of each virtual queue accessible to host */ + for (i = 0; i < ccp->cmd_q_count; i++) { + u32 dma_addr_lo; + u32 dma_addr_hi; + + cmd_q = &ccp->cmd_q[i]; + + cmd_q->qcontrol &= ~(CMD5_Q_SIZE << CMD5_Q_SHIFT); + cmd_q->qcontrol |= QUEUE_SIZE_VAL << CMD5_Q_SHIFT; + + cmd_q->qdma_tail = cmd_q->qbase_dma; + dma_addr_lo = low_address(cmd_q->qdma_tail); + iowrite32((u32)dma_addr_lo, cmd_q->reg_tail_lo); + iowrite32((u32)dma_addr_lo, cmd_q->reg_head_lo); + + dma_addr_hi = high_address(cmd_q->qdma_tail); + cmd_q->qcontrol |= (dma_addr_hi << 16); + iowrite32(cmd_q->qcontrol, cmd_q->reg_control); + + /* Find the LSB regions accessible to the queue */ + ccp_find_lsb_regions(cmd_q, status); + cmd_q->lsb = -1; /* Unassigned value */ + } + + dev_dbg(dev, "Assigning LSBs...\n"); + ret = ccp_assign_lsbs(ccp); + if (ret) { + dev_err(dev, "Unable to assign LSBs (%d)\n", ret); + goto e_irq; + } + + /* Optimization: pre-allocate LSB slots for each queue */ + for (i = 0; i < ccp->cmd_q_count; i++) { + ccp->cmd_q[i].sb_key = ccp_lsb_alloc(&ccp->cmd_q[i], 2); + ccp->cmd_q[i].sb_ctx = ccp_lsb_alloc(&ccp->cmd_q[i], 2); + } + + dev_dbg(dev, "Starting threads...\n"); + /* Create a kthread for each queue */ + for (i = 0; i < ccp->cmd_q_count; i++) { + struct task_struct *kthread; + + cmd_q = &ccp->cmd_q[i]; + + kthread = kthread_run(ccp_cmd_queue_thread, cmd_q, + "%s-q%u", ccp->name, cmd_q->id); + if (IS_ERR(kthread)) { + dev_err(dev, "error creating queue thread (%ld)\n", + PTR_ERR(kthread)); + ret = PTR_ERR(kthread); + goto e_kthread; + } + + cmd_q->kthread = kthread; + } + + dev_dbg(dev, "Enabling interrupts...\n"); + ccp5_enable_queue_interrupts(ccp); + + dev_dbg(dev, "Registering device...\n"); + /* Put this on the unit list to make it available */ + ccp_add_device(ccp); + + ret = ccp_register_rng(ccp); + if (ret) + goto e_kthread; + + /* Register the DMA engine support */ + ret = ccp_dmaengine_register(ccp); + if (ret) + goto e_hwrng; + +#ifdef CONFIG_CRYPTO_DEV_CCP_DEBUGFS + /* Set up debugfs entries */ + ccp5_debugfs_setup(ccp); +#endif + + return 0; + +e_hwrng: + ccp_unregister_rng(ccp); + +e_kthread: + for (i = 0; i < ccp->cmd_q_count; i++) + if (ccp->cmd_q[i].kthread) + kthread_stop(ccp->cmd_q[i].kthread); + +e_irq: + sp_free_ccp_irq(ccp->sp, ccp); + +e_pool: + for (i = 0; i < ccp->cmd_q_count; i++) + dma_pool_destroy(ccp->cmd_q[i].dma_pool); + + return ret; +} + +static void ccp5_destroy(struct ccp_device *ccp) +{ + struct ccp_cmd_queue *cmd_q; + struct ccp_cmd *cmd; + unsigned int i; + + /* Unregister the DMA engine */ + ccp_dmaengine_unregister(ccp); + + /* Unregister the RNG */ + ccp_unregister_rng(ccp); + + /* Remove this device from the list of available units first */ + ccp_del_device(ccp); + +#ifdef CONFIG_CRYPTO_DEV_CCP_DEBUGFS + /* We're in the process of tearing down the entire driver; + * when all the devices are gone clean up debugfs + */ + if (ccp_present()) + ccp5_debugfs_destroy(); +#endif + + /* Disable and clear interrupts */ + ccp5_disable_queue_interrupts(ccp); + for (i = 0; i < ccp->cmd_q_count; i++) { + cmd_q = &ccp->cmd_q[i]; + + /* Turn off the run bit */ + iowrite32(cmd_q->qcontrol & ~CMD5_Q_RUN, cmd_q->reg_control); + + /* Clear the interrupt status */ + iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status); + ioread32(cmd_q->reg_int_status); + ioread32(cmd_q->reg_status); + } + + /* Stop the queue kthreads */ + for (i = 0; i < ccp->cmd_q_count; i++) + if (ccp->cmd_q[i].kthread) + kthread_stop(ccp->cmd_q[i].kthread); + + sp_free_ccp_irq(ccp->sp, ccp); + + /* Flush the cmd and backlog queue */ + while (!list_empty(&ccp->cmd)) { + /* Invoke the callback directly with an error code */ + cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry); + list_del(&cmd->entry); + cmd->callback(cmd->data, -ENODEV); + } + while (!list_empty(&ccp->backlog)) { + /* Invoke the callback directly with an error code */ + cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry); + list_del(&cmd->entry); + cmd->callback(cmd->data, -ENODEV); + } +} + +static void ccp5_config(struct ccp_device *ccp) +{ + /* Public side */ + iowrite32(0x0, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET); +} + +static void ccp5other_config(struct ccp_device *ccp) +{ + int i; + u32 rnd; + + /* We own all of the queues on the NTB CCP */ + + iowrite32(0x00012D57, ccp->io_regs + CMD5_TRNG_CTL_OFFSET); + iowrite32(0x00000003, ccp->io_regs + CMD5_CONFIG_0_OFFSET); + for (i = 0; i < 12; i++) { + rnd = ioread32(ccp->io_regs + TRNG_OUT_REG); + iowrite32(rnd, ccp->io_regs + CMD5_AES_MASK_OFFSET); + } + + iowrite32(0x0000001F, ccp->io_regs + CMD5_QUEUE_MASK_OFFSET); + iowrite32(0x00005B6D, ccp->io_regs + CMD5_QUEUE_PRIO_OFFSET); + iowrite32(0x00000000, ccp->io_regs + CMD5_CMD_TIMEOUT_OFFSET); + + iowrite32(0x3FFFFFFF, ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET); + iowrite32(0x000003FF, ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET); + + iowrite32(0x00108823, ccp->io_regs + CMD5_CLK_GATE_CTL_OFFSET); + + ccp5_config(ccp); +} + +/* Version 5 adds some function, but is essentially the same as v5 */ +static const struct ccp_actions ccp5_actions = { + .aes = ccp5_perform_aes, + .xts_aes = ccp5_perform_xts_aes, + .sha = ccp5_perform_sha, + .des3 = ccp5_perform_des3, + .rsa = ccp5_perform_rsa, + .passthru = ccp5_perform_passthru, + .ecc = ccp5_perform_ecc, + .sballoc = ccp_lsb_alloc, + .sbfree = ccp_lsb_free, + .init = ccp5_init, + .destroy = ccp5_destroy, + .get_free_slots = ccp5_get_free_slots, +}; + +const struct ccp_vdata ccpv5a = { + .version = CCP_VERSION(5, 0), + .setup = ccp5_config, + .perform = &ccp5_actions, + .offset = 0x0, + .rsamax = CCP5_RSA_MAX_WIDTH, +}; + +const struct ccp_vdata ccpv5b = { + .version = CCP_VERSION(5, 0), + .dma_chan_attr = DMA_PRIVATE, + .setup = ccp5other_config, + .perform = &ccp5_actions, + .offset = 0x0, + .rsamax = CCP5_RSA_MAX_WIDTH, +}; diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c new file mode 100644 index 000000000..246801912 --- /dev/null +++ b/drivers/crypto/ccp/ccp-dev.c @@ -0,0 +1,670 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Cryptographic Coprocessor (CCP) driver + * + * Copyright (C) 2013,2019 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky + * Author: Gary R Hook + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_X86 +#include +#endif +#include + +#include "ccp-dev.h" + +#define MAX_CCPS 32 + +/* Limit CCP use to a specifed number of queues per device */ +static unsigned int nqueues; +module_param(nqueues, uint, 0444); +MODULE_PARM_DESC(nqueues, "Number of queues per CCP (minimum 1; default: all available)"); + +/* Limit the maximum number of configured CCPs */ +static atomic_t dev_count = ATOMIC_INIT(0); +static unsigned int max_devs = MAX_CCPS; +module_param(max_devs, uint, 0444); +MODULE_PARM_DESC(max_devs, "Maximum number of CCPs to enable (default: all; 0 disables all CCPs)"); + +struct ccp_tasklet_data { + struct completion completion; + struct ccp_cmd *cmd; +}; + +/* Human-readable error strings */ +#define CCP_MAX_ERROR_CODE 64 +static char *ccp_error_codes[] = { + "", + "ILLEGAL_ENGINE", + "ILLEGAL_KEY_ID", + "ILLEGAL_FUNCTION_TYPE", + "ILLEGAL_FUNCTION_MODE", + "ILLEGAL_FUNCTION_ENCRYPT", + "ILLEGAL_FUNCTION_SIZE", + "Zlib_MISSING_INIT_EOM", + "ILLEGAL_FUNCTION_RSVD", + "ILLEGAL_BUFFER_LENGTH", + "VLSB_FAULT", + "ILLEGAL_MEM_ADDR", + "ILLEGAL_MEM_SEL", + "ILLEGAL_CONTEXT_ID", + "ILLEGAL_KEY_ADDR", + "0xF Reserved", + "Zlib_ILLEGAL_MULTI_QUEUE", + "Zlib_ILLEGAL_JOBID_CHANGE", + "CMD_TIMEOUT", + "IDMA0_AXI_SLVERR", + "IDMA0_AXI_DECERR", + "0x15 Reserved", + "IDMA1_AXI_SLAVE_FAULT", + "IDMA1_AIXI_DECERR", + "0x18 Reserved", + "ZLIBVHB_AXI_SLVERR", + "ZLIBVHB_AXI_DECERR", + "0x1B Reserved", + "ZLIB_UNEXPECTED_EOM", + "ZLIB_EXTRA_DATA", + "ZLIB_BTYPE", + "ZLIB_UNDEFINED_SYMBOL", + "ZLIB_UNDEFINED_DISTANCE_S", + "ZLIB_CODE_LENGTH_SYMBOL", + "ZLIB _VHB_ILLEGAL_FETCH", + "ZLIB_UNCOMPRESSED_LEN", + "ZLIB_LIMIT_REACHED", + "ZLIB_CHECKSUM_MISMATCH0", + "ODMA0_AXI_SLVERR", + "ODMA0_AXI_DECERR", + "0x28 Reserved", + "ODMA1_AXI_SLVERR", + "ODMA1_AXI_DECERR", +}; + +void ccp_log_error(struct ccp_device *d, unsigned int e) +{ + if (WARN_ON(e >= CCP_MAX_ERROR_CODE)) + return; + + if (e < ARRAY_SIZE(ccp_error_codes)) + dev_err(d->dev, "CCP error %d: %s\n", e, ccp_error_codes[e]); + else + dev_err(d->dev, "CCP error %d: Unknown Error\n", e); +} + +/* List of CCPs, CCP count, read-write access lock, and access functions + * + * Lock structure: get ccp_unit_lock for reading whenever we need to + * examine the CCP list. While holding it for reading we can acquire + * the RR lock to update the round-robin next-CCP pointer. The unit lock + * must be acquired before the RR lock. + * + * If the unit-lock is acquired for writing, we have total control over + * the list, so there's no value in getting the RR lock. + */ +static DEFINE_RWLOCK(ccp_unit_lock); +static LIST_HEAD(ccp_units); + +/* Round-robin counter */ +static DEFINE_SPINLOCK(ccp_rr_lock); +static struct ccp_device *ccp_rr; + +/** + * ccp_add_device - add a CCP device to the list + * + * @ccp: ccp_device struct pointer + * + * Put this CCP on the unit list, which makes it available + * for use. + * + * Returns zero if a CCP device is present, -ENODEV otherwise. + */ +void ccp_add_device(struct ccp_device *ccp) +{ + unsigned long flags; + + write_lock_irqsave(&ccp_unit_lock, flags); + list_add_tail(&ccp->entry, &ccp_units); + if (!ccp_rr) + /* We already have the list lock (we're first) so this + * pointer can't change on us. Set its initial value. + */ + ccp_rr = ccp; + write_unlock_irqrestore(&ccp_unit_lock, flags); +} + +/** + * ccp_del_device - remove a CCP device from the list + * + * @ccp: ccp_device struct pointer + * + * Remove this unit from the list of devices. If the next device + * up for use is this one, adjust the pointer. If this is the last + * device, NULL the pointer. + */ +void ccp_del_device(struct ccp_device *ccp) +{ + unsigned long flags; + + write_lock_irqsave(&ccp_unit_lock, flags); + if (ccp_rr == ccp) { + /* ccp_unit_lock is read/write; any read access + * will be suspended while we make changes to the + * list and RR pointer. + */ + if (list_is_last(&ccp_rr->entry, &ccp_units)) + ccp_rr = list_first_entry(&ccp_units, struct ccp_device, + entry); + else + ccp_rr = list_next_entry(ccp_rr, entry); + } + list_del(&ccp->entry); + if (list_empty(&ccp_units)) + ccp_rr = NULL; + write_unlock_irqrestore(&ccp_unit_lock, flags); +} + + + +int ccp_register_rng(struct ccp_device *ccp) +{ + int ret = 0; + + dev_dbg(ccp->dev, "Registering RNG...\n"); + /* Register an RNG */ + ccp->hwrng.name = ccp->rngname; + ccp->hwrng.read = ccp_trng_read; + ret = hwrng_register(&ccp->hwrng); + if (ret) + dev_err(ccp->dev, "error registering hwrng (%d)\n", ret); + + return ret; +} + +void ccp_unregister_rng(struct ccp_device *ccp) +{ + if (ccp->hwrng.name) + hwrng_unregister(&ccp->hwrng); +} + +static struct ccp_device *ccp_get_device(void) +{ + unsigned long flags; + struct ccp_device *dp = NULL; + + /* We round-robin through the unit list. + * The (ccp_rr) pointer refers to the next unit to use. + */ + read_lock_irqsave(&ccp_unit_lock, flags); + if (!list_empty(&ccp_units)) { + spin_lock(&ccp_rr_lock); + dp = ccp_rr; + if (list_is_last(&ccp_rr->entry, &ccp_units)) + ccp_rr = list_first_entry(&ccp_units, struct ccp_device, + entry); + else + ccp_rr = list_next_entry(ccp_rr, entry); + spin_unlock(&ccp_rr_lock); + } + read_unlock_irqrestore(&ccp_unit_lock, flags); + + return dp; +} + +/** + * ccp_present - check if a CCP device is present + * + * Returns zero if a CCP device is present, -ENODEV otherwise. + */ +int ccp_present(void) +{ + unsigned long flags; + int ret; + + read_lock_irqsave(&ccp_unit_lock, flags); + ret = list_empty(&ccp_units); + read_unlock_irqrestore(&ccp_unit_lock, flags); + + return ret ? -ENODEV : 0; +} +EXPORT_SYMBOL_GPL(ccp_present); + +/** + * ccp_version - get the version of the CCP device + * + * Returns the version from the first unit on the list; + * otherwise a zero if no CCP device is present + */ +unsigned int ccp_version(void) +{ + struct ccp_device *dp; + unsigned long flags; + int ret = 0; + + read_lock_irqsave(&ccp_unit_lock, flags); + if (!list_empty(&ccp_units)) { + dp = list_first_entry(&ccp_units, struct ccp_device, entry); + ret = dp->vdata->version; + } + read_unlock_irqrestore(&ccp_unit_lock, flags); + + return ret; +} +EXPORT_SYMBOL_GPL(ccp_version); + +/** + * ccp_enqueue_cmd - queue an operation for processing by the CCP + * + * @cmd: ccp_cmd struct to be processed + * + * Queue a cmd to be processed by the CCP. If queueing the cmd + * would exceed the defined length of the cmd queue the cmd will + * only be queued if the CCP_CMD_MAY_BACKLOG flag is set and will + * result in a return code of -EBUSY. + * + * The callback routine specified in the ccp_cmd struct will be + * called to notify the caller of completion (if the cmd was not + * backlogged) or advancement out of the backlog. If the cmd has + * advanced out of the backlog the "err" value of the callback + * will be -EINPROGRESS. Any other "err" value during callback is + * the result of the operation. + * + * The cmd has been successfully queued if: + * the return code is -EINPROGRESS or + * the return code is -EBUSY and CCP_CMD_MAY_BACKLOG flag is set + */ +int ccp_enqueue_cmd(struct ccp_cmd *cmd) +{ + struct ccp_device *ccp; + unsigned long flags; + unsigned int i; + int ret; + + /* Some commands might need to be sent to a specific device */ + ccp = cmd->ccp ? cmd->ccp : ccp_get_device(); + + if (!ccp) + return -ENODEV; + + /* Caller must supply a callback routine */ + if (!cmd->callback) + return -EINVAL; + + cmd->ccp = ccp; + + spin_lock_irqsave(&ccp->cmd_lock, flags); + + i = ccp->cmd_q_count; + + if (ccp->cmd_count >= MAX_CMD_QLEN) { + if (cmd->flags & CCP_CMD_MAY_BACKLOG) { + ret = -EBUSY; + list_add_tail(&cmd->entry, &ccp->backlog); + } else { + ret = -ENOSPC; + } + } else { + ret = -EINPROGRESS; + ccp->cmd_count++; + list_add_tail(&cmd->entry, &ccp->cmd); + + /* Find an idle queue */ + if (!ccp->suspending) { + for (i = 0; i < ccp->cmd_q_count; i++) { + if (ccp->cmd_q[i].active) + continue; + + break; + } + } + } + + spin_unlock_irqrestore(&ccp->cmd_lock, flags); + + /* If we found an idle queue, wake it up */ + if (i < ccp->cmd_q_count) + wake_up_process(ccp->cmd_q[i].kthread); + + return ret; +} +EXPORT_SYMBOL_GPL(ccp_enqueue_cmd); + +static void ccp_do_cmd_backlog(struct work_struct *work) +{ + struct ccp_cmd *cmd = container_of(work, struct ccp_cmd, work); + struct ccp_device *ccp = cmd->ccp; + unsigned long flags; + unsigned int i; + + cmd->callback(cmd->data, -EINPROGRESS); + + spin_lock_irqsave(&ccp->cmd_lock, flags); + + ccp->cmd_count++; + list_add_tail(&cmd->entry, &ccp->cmd); + + /* Find an idle queue */ + for (i = 0; i < ccp->cmd_q_count; i++) { + if (ccp->cmd_q[i].active) + continue; + + break; + } + + spin_unlock_irqrestore(&ccp->cmd_lock, flags); + + /* If we found an idle queue, wake it up */ + if (i < ccp->cmd_q_count) + wake_up_process(ccp->cmd_q[i].kthread); +} + +static struct ccp_cmd *ccp_dequeue_cmd(struct ccp_cmd_queue *cmd_q) +{ + struct ccp_device *ccp = cmd_q->ccp; + struct ccp_cmd *cmd = NULL; + struct ccp_cmd *backlog = NULL; + unsigned long flags; + + spin_lock_irqsave(&ccp->cmd_lock, flags); + + cmd_q->active = 0; + + if (ccp->suspending) { + cmd_q->suspended = 1; + + spin_unlock_irqrestore(&ccp->cmd_lock, flags); + wake_up_interruptible(&ccp->suspend_queue); + + return NULL; + } + + if (ccp->cmd_count) { + cmd_q->active = 1; + + cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry); + list_del(&cmd->entry); + + ccp->cmd_count--; + } + + if (!list_empty(&ccp->backlog)) { + backlog = list_first_entry(&ccp->backlog, struct ccp_cmd, + entry); + list_del(&backlog->entry); + } + + spin_unlock_irqrestore(&ccp->cmd_lock, flags); + + if (backlog) { + INIT_WORK(&backlog->work, ccp_do_cmd_backlog); + schedule_work(&backlog->work); + } + + return cmd; +} + +static void ccp_do_cmd_complete(unsigned long data) +{ + struct ccp_tasklet_data *tdata = (struct ccp_tasklet_data *)data; + struct ccp_cmd *cmd = tdata->cmd; + + cmd->callback(cmd->data, cmd->ret); + + complete(&tdata->completion); +} + +/** + * ccp_cmd_queue_thread - create a kernel thread to manage a CCP queue + * + * @data: thread-specific data + */ +int ccp_cmd_queue_thread(void *data) +{ + struct ccp_cmd_queue *cmd_q = (struct ccp_cmd_queue *)data; + struct ccp_cmd *cmd; + struct ccp_tasklet_data tdata; + struct tasklet_struct tasklet; + + tasklet_init(&tasklet, ccp_do_cmd_complete, (unsigned long)&tdata); + + set_current_state(TASK_INTERRUPTIBLE); + while (!kthread_should_stop()) { + schedule(); + + set_current_state(TASK_INTERRUPTIBLE); + + cmd = ccp_dequeue_cmd(cmd_q); + if (!cmd) + continue; + + __set_current_state(TASK_RUNNING); + + /* Execute the command */ + cmd->ret = ccp_run_cmd(cmd_q, cmd); + + /* Schedule the completion callback */ + tdata.cmd = cmd; + init_completion(&tdata.completion); + tasklet_schedule(&tasklet); + wait_for_completion(&tdata.completion); + } + + __set_current_state(TASK_RUNNING); + + return 0; +} + +/** + * ccp_alloc_struct - allocate and initialize the ccp_device struct + * + * @sp: sp_device struct of the CCP + */ +struct ccp_device *ccp_alloc_struct(struct sp_device *sp) +{ + struct device *dev = sp->dev; + struct ccp_device *ccp; + + ccp = devm_kzalloc(dev, sizeof(*ccp), GFP_KERNEL); + if (!ccp) + return NULL; + ccp->dev = dev; + ccp->sp = sp; + ccp->axcache = sp->axcache; + + INIT_LIST_HEAD(&ccp->cmd); + INIT_LIST_HEAD(&ccp->backlog); + + spin_lock_init(&ccp->cmd_lock); + mutex_init(&ccp->req_mutex); + mutex_init(&ccp->sb_mutex); + ccp->sb_count = KSB_COUNT; + ccp->sb_start = 0; + + /* Initialize the wait queues */ + init_waitqueue_head(&ccp->sb_queue); + init_waitqueue_head(&ccp->suspend_queue); + + snprintf(ccp->name, MAX_CCP_NAME_LEN, "ccp-%u", sp->ord); + snprintf(ccp->rngname, MAX_CCP_NAME_LEN, "ccp-%u-rng", sp->ord); + + return ccp; +} + +int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait) +{ + struct ccp_device *ccp = container_of(rng, struct ccp_device, hwrng); + u32 trng_value; + int len = min(sizeof(trng_value), max); + + /* Locking is provided by the caller so we can update device + * hwrng-related fields safely + */ + trng_value = ioread32(ccp->io_regs + TRNG_OUT_REG); + if (!trng_value) { + /* Zero is returned if not data is available or if a + * bad-entropy error is present. Assume an error if + * we exceed TRNG_RETRIES reads of zero. + */ + if (ccp->hwrng_retries++ > TRNG_RETRIES) + return -EIO; + + return 0; + } + + /* Reset the counter and save the rng value */ + ccp->hwrng_retries = 0; + memcpy(data, &trng_value, len); + + return len; +} + +bool ccp_queues_suspended(struct ccp_device *ccp) +{ + unsigned int suspended = 0; + unsigned long flags; + unsigned int i; + + spin_lock_irqsave(&ccp->cmd_lock, flags); + + for (i = 0; i < ccp->cmd_q_count; i++) + if (ccp->cmd_q[i].suspended) + suspended++; + + spin_unlock_irqrestore(&ccp->cmd_lock, flags); + + return ccp->cmd_q_count == suspended; +} + +void ccp_dev_suspend(struct sp_device *sp) +{ + struct ccp_device *ccp = sp->ccp_data; + unsigned long flags; + unsigned int i; + + /* If there's no device there's nothing to do */ + if (!ccp) + return; + + spin_lock_irqsave(&ccp->cmd_lock, flags); + + ccp->suspending = 1; + + /* Wake all the queue kthreads to prepare for suspend */ + for (i = 0; i < ccp->cmd_q_count; i++) + wake_up_process(ccp->cmd_q[i].kthread); + + spin_unlock_irqrestore(&ccp->cmd_lock, flags); + + /* Wait for all queue kthreads to say they're done */ + while (!ccp_queues_suspended(ccp)) + wait_event_interruptible(ccp->suspend_queue, + ccp_queues_suspended(ccp)); +} + +void ccp_dev_resume(struct sp_device *sp) +{ + struct ccp_device *ccp = sp->ccp_data; + unsigned long flags; + unsigned int i; + + /* If there's no device there's nothing to do */ + if (!ccp) + return; + + spin_lock_irqsave(&ccp->cmd_lock, flags); + + ccp->suspending = 0; + + /* Wake up all the kthreads */ + for (i = 0; i < ccp->cmd_q_count; i++) { + ccp->cmd_q[i].suspended = 0; + wake_up_process(ccp->cmd_q[i].kthread); + } + + spin_unlock_irqrestore(&ccp->cmd_lock, flags); +} + +int ccp_dev_init(struct sp_device *sp) +{ + struct device *dev = sp->dev; + struct ccp_device *ccp; + int ret; + + /* + * Check how many we have so far, and stop after reaching + * that number + */ + if (atomic_inc_return(&dev_count) > max_devs) + return 0; /* don't fail the load */ + + ret = -ENOMEM; + ccp = ccp_alloc_struct(sp); + if (!ccp) + goto e_err; + sp->ccp_data = ccp; + + if (!nqueues || (nqueues > MAX_HW_QUEUES)) + ccp->max_q_count = MAX_HW_QUEUES; + else + ccp->max_q_count = nqueues; + + ccp->vdata = (struct ccp_vdata *)sp->dev_vdata->ccp_vdata; + if (!ccp->vdata || !ccp->vdata->version) { + ret = -ENODEV; + dev_err(dev, "missing driver data\n"); + goto e_err; + } + + ccp->use_tasklet = sp->use_tasklet; + + ccp->io_regs = sp->io_map + ccp->vdata->offset; + if (ccp->vdata->setup) + ccp->vdata->setup(ccp); + + ret = ccp->vdata->perform->init(ccp); + if (ret) { + /* A positive number means that the device cannot be initialized, + * but no additional message is required. + */ + if (ret > 0) + goto e_quiet; + + /* An unexpected problem occurred, and should be reported in the log */ + goto e_err; + } + + dev_notice(dev, "ccp enabled\n"); + + return 0; + +e_err: + dev_notice(dev, "ccp initialization failed\n"); + +e_quiet: + sp->ccp_data = NULL; + + return ret; +} + +void ccp_dev_destroy(struct sp_device *sp) +{ + struct ccp_device *ccp = sp->ccp_data; + + if (!ccp) + return; + + ccp->vdata->perform->destroy(ccp); +} diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h new file mode 100644 index 000000000..83350e2d9 --- /dev/null +++ b/drivers/crypto/ccp/ccp-dev.h @@ -0,0 +1,673 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * AMD Cryptographic Coprocessor (CCP) driver + * + * Copyright (C) 2013,2017 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky + * Author: Gary R Hook + */ + +#ifndef __CCP_DEV_H__ +#define __CCP_DEV_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sp-dev.h" + +#define MAX_CCP_NAME_LEN 16 +#define MAX_DMAPOOL_NAME_LEN 32 + +#define MAX_HW_QUEUES 5 +#define MAX_CMD_QLEN 100 + +#define TRNG_RETRIES 10 + +#define CACHE_NONE 0x00 +#define CACHE_WB_NO_ALLOC 0xb7 + +/****** Register Mappings ******/ +#define Q_MASK_REG 0x000 +#define TRNG_OUT_REG 0x00c +#define IRQ_MASK_REG 0x040 +#define IRQ_STATUS_REG 0x200 + +#define DEL_CMD_Q_JOB 0x124 +#define DEL_Q_ACTIVE 0x00000200 +#define DEL_Q_ID_SHIFT 6 + +#define CMD_REQ0 0x180 +#define CMD_REQ_INCR 0x04 + +#define CMD_Q_STATUS_BASE 0x210 +#define CMD_Q_INT_STATUS_BASE 0x214 +#define CMD_Q_STATUS_INCR 0x20 + +#define CMD_Q_CACHE_BASE 0x228 +#define CMD_Q_CACHE_INC 0x20 + +#define CMD_Q_ERROR(__qs) ((__qs) & 0x0000003f) +#define CMD_Q_DEPTH(__qs) (((__qs) >> 12) & 0x0000000f) + +/* ------------------------ CCP Version 5 Specifics ------------------------ */ +#define CMD5_QUEUE_MASK_OFFSET 0x00 +#define CMD5_QUEUE_PRIO_OFFSET 0x04 +#define CMD5_REQID_CONFIG_OFFSET 0x08 +#define CMD5_CMD_TIMEOUT_OFFSET 0x10 +#define LSB_PUBLIC_MASK_LO_OFFSET 0x18 +#define LSB_PUBLIC_MASK_HI_OFFSET 0x1C +#define LSB_PRIVATE_MASK_LO_OFFSET 0x20 +#define LSB_PRIVATE_MASK_HI_OFFSET 0x24 +#define CMD5_PSP_CCP_VERSION 0x100 + +#define CMD5_Q_CONTROL_BASE 0x0000 +#define CMD5_Q_TAIL_LO_BASE 0x0004 +#define CMD5_Q_HEAD_LO_BASE 0x0008 +#define CMD5_Q_INT_ENABLE_BASE 0x000C +#define CMD5_Q_INTERRUPT_STATUS_BASE 0x0010 + +#define CMD5_Q_STATUS_BASE 0x0100 +#define CMD5_Q_INT_STATUS_BASE 0x0104 +#define CMD5_Q_DMA_STATUS_BASE 0x0108 +#define CMD5_Q_DMA_READ_STATUS_BASE 0x010C +#define CMD5_Q_DMA_WRITE_STATUS_BASE 0x0110 +#define CMD5_Q_ABORT_BASE 0x0114 +#define CMD5_Q_AX_CACHE_BASE 0x0118 + +#define CMD5_CONFIG_0_OFFSET 0x6000 +#define CMD5_TRNG_CTL_OFFSET 0x6008 +#define CMD5_AES_MASK_OFFSET 0x6010 +#define CMD5_CLK_GATE_CTL_OFFSET 0x603C + +/* Address offset between two virtual queue registers */ +#define CMD5_Q_STATUS_INCR 0x1000 + +/* Bit masks */ +#define CMD5_Q_RUN 0x1 +#define CMD5_Q_HALT 0x2 +#define CMD5_Q_MEM_LOCATION 0x4 +#define CMD5_Q_SIZE 0x1F +#define CMD5_Q_SHIFT 3 +#define COMMANDS_PER_QUEUE 16 +#define QUEUE_SIZE_VAL ((ffs(COMMANDS_PER_QUEUE) - 2) & \ + CMD5_Q_SIZE) +#define Q_PTR_MASK (2 << (QUEUE_SIZE_VAL + 5) - 1) +#define Q_DESC_SIZE sizeof(struct ccp5_desc) +#define Q_SIZE(n) (COMMANDS_PER_QUEUE*(n)) + +#define INT_COMPLETION 0x1 +#define INT_ERROR 0x2 +#define INT_QUEUE_STOPPED 0x4 +#define INT_EMPTY_QUEUE 0x8 +#define SUPPORTED_INTERRUPTS (INT_COMPLETION | INT_ERROR) + +#define LSB_REGION_WIDTH 5 +#define MAX_LSB_CNT 8 + +#define LSB_SIZE 16 +#define LSB_ITEM_SIZE 32 +#define PLSB_MAP_SIZE (LSB_SIZE) +#define SLSB_MAP_SIZE (MAX_LSB_CNT * LSB_SIZE) + +#define LSB_ENTRY_NUMBER(LSB_ADDR) (LSB_ADDR / LSB_ITEM_SIZE) + +/* ------------------------ CCP Version 3 Specifics ------------------------ */ +#define REQ0_WAIT_FOR_WRITE 0x00000004 +#define REQ0_INT_ON_COMPLETE 0x00000002 +#define REQ0_STOP_ON_COMPLETE 0x00000001 + +#define REQ0_CMD_Q_SHIFT 9 +#define REQ0_JOBID_SHIFT 3 + +/****** REQ1 Related Values ******/ +#define REQ1_PROTECT_SHIFT 27 +#define REQ1_ENGINE_SHIFT 23 +#define REQ1_KEY_KSB_SHIFT 2 + +#define REQ1_EOM 0x00000002 +#define REQ1_INIT 0x00000001 + +/* AES Related Values */ +#define REQ1_AES_TYPE_SHIFT 21 +#define REQ1_AES_MODE_SHIFT 18 +#define REQ1_AES_ACTION_SHIFT 17 +#define REQ1_AES_CFB_SIZE_SHIFT 10 + +/* XTS-AES Related Values */ +#define REQ1_XTS_AES_SIZE_SHIFT 10 + +/* SHA Related Values */ +#define REQ1_SHA_TYPE_SHIFT 21 + +/* RSA Related Values */ +#define REQ1_RSA_MOD_SIZE_SHIFT 10 + +/* Pass-Through Related Values */ +#define REQ1_PT_BW_SHIFT 12 +#define REQ1_PT_BS_SHIFT 10 + +/* ECC Related Values */ +#define REQ1_ECC_AFFINE_CONVERT 0x00200000 +#define REQ1_ECC_FUNCTION_SHIFT 18 + +/****** REQ4 Related Values ******/ +#define REQ4_KSB_SHIFT 18 +#define REQ4_MEMTYPE_SHIFT 16 + +/****** REQ6 Related Values ******/ +#define REQ6_MEMTYPE_SHIFT 16 + +/****** Key Storage Block ******/ +#define KSB_START 77 +#define KSB_END 127 +#define KSB_COUNT (KSB_END - KSB_START + 1) +#define CCP_SB_BITS 256 + +#define CCP_JOBID_MASK 0x0000003f + +/* ------------------------ General CCP Defines ------------------------ */ + +#define CCP_DMA_DFLT 0x0 +#define CCP_DMA_PRIV 0x1 +#define CCP_DMA_PUB 0x2 + +#define CCP_DMAPOOL_MAX_SIZE 64 +#define CCP_DMAPOOL_ALIGN BIT(5) + +#define CCP_REVERSE_BUF_SIZE 64 + +#define CCP_AES_KEY_SB_COUNT 1 +#define CCP_AES_CTX_SB_COUNT 1 + +#define CCP_XTS_AES_KEY_SB_COUNT 1 +#define CCP5_XTS_AES_KEY_SB_COUNT 2 +#define CCP_XTS_AES_CTX_SB_COUNT 1 + +#define CCP_DES3_KEY_SB_COUNT 1 +#define CCP_DES3_CTX_SB_COUNT 1 + +#define CCP_SHA_SB_COUNT 1 + +#define CCP_RSA_MAX_WIDTH 4096 +#define CCP5_RSA_MAX_WIDTH 16384 + +#define CCP_PASSTHRU_BLOCKSIZE 256 +#define CCP_PASSTHRU_MASKSIZE 32 +#define CCP_PASSTHRU_SB_COUNT 1 + +#define CCP_ECC_MODULUS_BYTES 48 /* 384-bits */ +#define CCP_ECC_MAX_OPERANDS 6 +#define CCP_ECC_MAX_OUTPUTS 3 +#define CCP_ECC_SRC_BUF_SIZE 448 +#define CCP_ECC_DST_BUF_SIZE 192 +#define CCP_ECC_OPERAND_SIZE 64 +#define CCP_ECC_OUTPUT_SIZE 64 +#define CCP_ECC_RESULT_OFFSET 60 +#define CCP_ECC_RESULT_SUCCESS 0x0001 + +#define CCP_SB_BYTES 32 + +struct ccp_op; +struct ccp_device; +struct ccp_cmd; +struct ccp_fns; + +struct ccp_dma_cmd { + struct list_head entry; + + struct ccp_cmd ccp_cmd; +}; + +struct ccp_dma_desc { + struct list_head entry; + + struct ccp_device *ccp; + + struct list_head pending; + struct list_head active; + + enum dma_status status; + struct dma_async_tx_descriptor tx_desc; + size_t len; +}; + +struct ccp_dma_chan { + struct ccp_device *ccp; + + spinlock_t lock; + struct list_head created; + struct list_head pending; + struct list_head active; + struct list_head complete; + + struct tasklet_struct cleanup_tasklet; + + enum dma_status status; + struct dma_chan dma_chan; +}; + +struct ccp_cmd_queue { + struct ccp_device *ccp; + + /* Queue identifier */ + u32 id; + + /* Queue dma pool */ + struct dma_pool *dma_pool; + + /* Queue base address (not neccessarily aligned)*/ + struct ccp5_desc *qbase; + + /* Aligned queue start address (per requirement) */ + struct mutex q_mutex ____cacheline_aligned; + unsigned int qidx; + + /* Version 5 has different requirements for queue memory */ + unsigned int qsize; + dma_addr_t qbase_dma; + dma_addr_t qdma_tail; + + /* Per-queue reserved storage block(s) */ + u32 sb_key; + u32 sb_ctx; + + /* Bitmap of LSBs that can be accessed by this queue */ + DECLARE_BITMAP(lsbmask, MAX_LSB_CNT); + /* Private LSB that is assigned to this queue, or -1 if none. + * Bitmap for my private LSB, unused otherwise + */ + int lsb; + DECLARE_BITMAP(lsbmap, PLSB_MAP_SIZE); + + /* Queue processing thread */ + struct task_struct *kthread; + unsigned int active; + unsigned int suspended; + + /* Number of free command slots available */ + unsigned int free_slots; + + /* Interrupt masks */ + u32 int_ok; + u32 int_err; + + /* Register addresses for queue */ + void __iomem *reg_control; + void __iomem *reg_tail_lo; + void __iomem *reg_head_lo; + void __iomem *reg_int_enable; + void __iomem *reg_interrupt_status; + void __iomem *reg_status; + void __iomem *reg_int_status; + void __iomem *reg_dma_status; + void __iomem *reg_dma_read_status; + void __iomem *reg_dma_write_status; + u32 qcontrol; /* Cached control register */ + + /* Status values from job */ + u32 int_status; + u32 q_status; + u32 q_int_status; + u32 cmd_error; + + /* Interrupt wait queue */ + wait_queue_head_t int_queue; + unsigned int int_rcvd; + + /* Per-queue Statistics */ + unsigned long total_ops; + unsigned long total_aes_ops; + unsigned long total_xts_aes_ops; + unsigned long total_3des_ops; + unsigned long total_sha_ops; + unsigned long total_rsa_ops; + unsigned long total_pt_ops; + unsigned long total_ecc_ops; +} ____cacheline_aligned; + +struct ccp_device { + struct list_head entry; + + struct ccp_vdata *vdata; + unsigned int ord; + char name[MAX_CCP_NAME_LEN]; + char rngname[MAX_CCP_NAME_LEN]; + + struct device *dev; + struct sp_device *sp; + + /* Bus specific device information + */ + void *dev_specific; + unsigned int qim; + unsigned int irq; + bool use_tasklet; + struct tasklet_struct irq_tasklet; + + /* I/O area used for device communication. The register mapping + * starts at an offset into the mapped bar. + * The CMD_REQx registers and the Delete_Cmd_Queue_Job register + * need to be protected while a command queue thread is accessing + * them. + */ + struct mutex req_mutex ____cacheline_aligned; + void __iomem *io_regs; + + /* Master lists that all cmds are queued on. Because there can be + * more than one CCP command queue that can process a cmd a separate + * backlog list is needed so that the backlog completion call + * completes before the cmd is available for execution. + */ + spinlock_t cmd_lock ____cacheline_aligned; + unsigned int cmd_count; + struct list_head cmd; + struct list_head backlog; + + /* The command queues. These represent the queues available on the + * CCP that are available for processing cmds + */ + struct ccp_cmd_queue cmd_q[MAX_HW_QUEUES]; + unsigned int cmd_q_count; + unsigned int max_q_count; + + /* Support for the CCP True RNG + */ + struct hwrng hwrng; + unsigned int hwrng_retries; + + /* Support for the CCP DMA capabilities + */ + struct dma_device dma_dev; + struct ccp_dma_chan *ccp_dma_chan; + struct kmem_cache *dma_cmd_cache; + struct kmem_cache *dma_desc_cache; + + /* A counter used to generate job-ids for cmds submitted to the CCP + */ + atomic_t current_id ____cacheline_aligned; + + /* The v3 CCP uses key storage blocks (SB) to maintain context for + * certain operations. To prevent multiple cmds from using the same + * SB range a command queue reserves an SB range for the duration of + * the cmd. Each queue, will however, reserve 2 SB blocks for + * operations that only require single SB entries (eg. AES context/iv + * and key) in order to avoid allocation contention. This will reserve + * at most 10 SB entries, leaving 40 SB entries available for dynamic + * allocation. + * + * The v5 CCP Local Storage Block (LSB) is broken up into 8 + * memrory ranges, each of which can be enabled for access by one + * or more queues. Device initialization takes this into account, + * and attempts to assign one region for exclusive use by each + * available queue; the rest are then aggregated as "public" use. + * If there are fewer regions than queues, all regions are shared + * amongst all queues. + */ + struct mutex sb_mutex ____cacheline_aligned; + DECLARE_BITMAP(sb, KSB_COUNT); + wait_queue_head_t sb_queue; + unsigned int sb_avail; + unsigned int sb_count; + u32 sb_start; + + /* Bitmap of shared LSBs, if any */ + DECLARE_BITMAP(lsbmap, SLSB_MAP_SIZE); + + /* Suspend support */ + unsigned int suspending; + wait_queue_head_t suspend_queue; + + /* DMA caching attribute support */ + unsigned int axcache; + + /* Device Statistics */ + unsigned long total_interrupts; + + /* DebugFS info */ + struct dentry *debugfs_instance; +}; + +enum ccp_memtype { + CCP_MEMTYPE_SYSTEM = 0, + CCP_MEMTYPE_SB, + CCP_MEMTYPE_LOCAL, + CCP_MEMTYPE__LAST, +}; +#define CCP_MEMTYPE_LSB CCP_MEMTYPE_KSB + + +struct ccp_dma_info { + dma_addr_t address; + unsigned int offset; + unsigned int length; + enum dma_data_direction dir; +} __packed __aligned(4); + +struct ccp_dm_workarea { + struct device *dev; + struct dma_pool *dma_pool; + + u8 *address; + struct ccp_dma_info dma; + unsigned int length; +}; + +struct ccp_sg_workarea { + struct scatterlist *sg; + int nents; + unsigned int sg_used; + + struct scatterlist *dma_sg; + struct scatterlist *dma_sg_head; + struct device *dma_dev; + unsigned int dma_count; + enum dma_data_direction dma_dir; + + u64 bytes_left; +}; + +struct ccp_data { + struct ccp_sg_workarea sg_wa; + struct ccp_dm_workarea dm_wa; +}; + +struct ccp_mem { + enum ccp_memtype type; + union { + struct ccp_dma_info dma; + u32 sb; + } u; +}; + +struct ccp_aes_op { + enum ccp_aes_type type; + enum ccp_aes_mode mode; + enum ccp_aes_action action; + unsigned int size; +}; + +struct ccp_xts_aes_op { + enum ccp_aes_type type; + enum ccp_aes_action action; + enum ccp_xts_aes_unit_size unit_size; +}; + +struct ccp_des3_op { + enum ccp_des3_type type; + enum ccp_des3_mode mode; + enum ccp_des3_action action; +}; + +struct ccp_sha_op { + enum ccp_sha_type type; + u64 msg_bits; +}; + +struct ccp_rsa_op { + u32 mod_size; + u32 input_len; +}; + +struct ccp_passthru_op { + enum ccp_passthru_bitwise bit_mod; + enum ccp_passthru_byteswap byte_swap; +}; + +struct ccp_ecc_op { + enum ccp_ecc_function function; +}; + +struct ccp_op { + struct ccp_cmd_queue *cmd_q; + + u32 jobid; + u32 ioc; + u32 soc; + u32 sb_key; + u32 sb_ctx; + u32 init; + u32 eom; + + struct ccp_mem src; + struct ccp_mem dst; + struct ccp_mem exp; + + union { + struct ccp_aes_op aes; + struct ccp_xts_aes_op xts; + struct ccp_des3_op des3; + struct ccp_sha_op sha; + struct ccp_rsa_op rsa; + struct ccp_passthru_op passthru; + struct ccp_ecc_op ecc; + } u; +}; + +static inline u32 ccp_addr_lo(struct ccp_dma_info *info) +{ + return lower_32_bits(info->address + info->offset); +} + +static inline u32 ccp_addr_hi(struct ccp_dma_info *info) +{ + return upper_32_bits(info->address + info->offset) & 0x0000ffff; +} + +/** + * descriptor for version 5 CPP commands + * 8 32-bit words: + * word 0: function; engine; control bits + * word 1: length of source data + * word 2: low 32 bits of source pointer + * word 3: upper 16 bits of source pointer; source memory type + * word 4: low 32 bits of destination pointer + * word 5: upper 16 bits of destination pointer; destination memory type + * word 6: low 32 bits of key pointer + * word 7: upper 16 bits of key pointer; key memory type + */ +struct dword0 { + unsigned int soc:1; + unsigned int ioc:1; + unsigned int rsvd1:1; + unsigned int init:1; + unsigned int eom:1; /* AES/SHA only */ + unsigned int function:15; + unsigned int engine:4; + unsigned int prot:1; + unsigned int rsvd2:7; +}; + +struct dword3 { + unsigned int src_hi:16; + unsigned int src_mem:2; + unsigned int lsb_cxt_id:8; + unsigned int rsvd1:5; + unsigned int fixed:1; +}; + +union dword4 { + u32 dst_lo; /* NON-SHA */ + u32 sha_len_lo; /* SHA */ +}; + +union dword5 { + struct { + unsigned int dst_hi:16; + unsigned int dst_mem:2; + unsigned int rsvd1:13; + unsigned int fixed:1; + } fields; + u32 sha_len_hi; +}; + +struct dword7 { + unsigned int key_hi:16; + unsigned int key_mem:2; + unsigned int rsvd1:14; +}; + +struct ccp5_desc { + struct dword0 dw0; + u32 length; + u32 src_lo; + struct dword3 dw3; + union dword4 dw4; + union dword5 dw5; + u32 key_lo; + struct dword7 dw7; +}; + +void ccp_add_device(struct ccp_device *ccp); +void ccp_del_device(struct ccp_device *ccp); + +extern void ccp_log_error(struct ccp_device *, unsigned int); + +struct ccp_device *ccp_alloc_struct(struct sp_device *sp); +bool ccp_queues_suspended(struct ccp_device *ccp); +int ccp_cmd_queue_thread(void *data); +int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait); + +int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd); + +int ccp_register_rng(struct ccp_device *ccp); +void ccp_unregister_rng(struct ccp_device *ccp); +int ccp_dmaengine_register(struct ccp_device *ccp); +void ccp_dmaengine_unregister(struct ccp_device *ccp); + +void ccp5_debugfs_setup(struct ccp_device *ccp); +void ccp5_debugfs_destroy(void); + +/* Structure for computation functions that are device-specific */ +struct ccp_actions { + int (*aes)(struct ccp_op *); + int (*xts_aes)(struct ccp_op *); + int (*des3)(struct ccp_op *); + int (*sha)(struct ccp_op *); + int (*rsa)(struct ccp_op *); + int (*passthru)(struct ccp_op *); + int (*ecc)(struct ccp_op *); + u32 (*sballoc)(struct ccp_cmd_queue *, unsigned int); + void (*sbfree)(struct ccp_cmd_queue *, unsigned int, unsigned int); + unsigned int (*get_free_slots)(struct ccp_cmd_queue *); + int (*init)(struct ccp_device *); + void (*destroy)(struct ccp_device *); + irqreturn_t (*irqhandler)(int, void *); +}; + +extern const struct ccp_vdata ccpv3_platform; +extern const struct ccp_vdata ccpv3; +extern const struct ccp_vdata ccpv5a; +extern const struct ccp_vdata ccpv5b; + +#endif diff --git a/drivers/crypto/ccp/ccp-dmaengine.c b/drivers/crypto/ccp/ccp-dmaengine.c new file mode 100644 index 000000000..b386a7063 --- /dev/null +++ b/drivers/crypto/ccp/ccp-dmaengine.c @@ -0,0 +1,791 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Cryptographic Coprocessor (CCP) driver + * + * Copyright (C) 2016,2019 Advanced Micro Devices, Inc. + * + * Author: Gary R Hook + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "ccp-dev.h" +#include "../../dma/dmaengine.h" + +#define CCP_DMA_WIDTH(_mask) \ +({ \ + u64 mask = _mask + 1; \ + (mask == 0) ? 64 : fls64(mask); \ +}) + +/* The CCP as a DMA provider can be configured for public or private + * channels. Default is specified in the vdata for the device (PCI ID). + * This module parameter will override for all channels on all devices: + * dma_chan_attr = 0x2 to force all channels public + * = 0x1 to force all channels private + * = 0x0 to defer to the vdata setting + * = any other value: warning, revert to 0x0 + */ +static unsigned int dma_chan_attr = CCP_DMA_DFLT; +module_param(dma_chan_attr, uint, 0444); +MODULE_PARM_DESC(dma_chan_attr, "Set DMA channel visibility: 0 (default) = device defaults, 1 = make private, 2 = make public"); + +static unsigned int dmaengine = 1; +module_param(dmaengine, uint, 0444); +MODULE_PARM_DESC(dmaengine, "Register services with the DMA subsystem (any non-zero value, default: 1)"); + +static unsigned int ccp_get_dma_chan_attr(struct ccp_device *ccp) +{ + switch (dma_chan_attr) { + case CCP_DMA_DFLT: + return ccp->vdata->dma_chan_attr; + + case CCP_DMA_PRIV: + return DMA_PRIVATE; + + case CCP_DMA_PUB: + return 0; + + default: + dev_info_once(ccp->dev, "Invalid value for dma_chan_attr: %d\n", + dma_chan_attr); + return ccp->vdata->dma_chan_attr; + } +} + +static void ccp_free_cmd_resources(struct ccp_device *ccp, + struct list_head *list) +{ + struct ccp_dma_cmd *cmd, *ctmp; + + list_for_each_entry_safe(cmd, ctmp, list, entry) { + list_del(&cmd->entry); + kmem_cache_free(ccp->dma_cmd_cache, cmd); + } +} + +static void ccp_free_desc_resources(struct ccp_device *ccp, + struct list_head *list) +{ + struct ccp_dma_desc *desc, *dtmp; + + list_for_each_entry_safe(desc, dtmp, list, entry) { + ccp_free_cmd_resources(ccp, &desc->active); + ccp_free_cmd_resources(ccp, &desc->pending); + + list_del(&desc->entry); + kmem_cache_free(ccp->dma_desc_cache, desc); + } +} + +static void ccp_free_chan_resources(struct dma_chan *dma_chan) +{ + struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, + dma_chan); + unsigned long flags; + + dev_dbg(chan->ccp->dev, "%s - chan=%p\n", __func__, chan); + + spin_lock_irqsave(&chan->lock, flags); + + ccp_free_desc_resources(chan->ccp, &chan->complete); + ccp_free_desc_resources(chan->ccp, &chan->active); + ccp_free_desc_resources(chan->ccp, &chan->pending); + ccp_free_desc_resources(chan->ccp, &chan->created); + + spin_unlock_irqrestore(&chan->lock, flags); +} + +static void ccp_cleanup_desc_resources(struct ccp_device *ccp, + struct list_head *list) +{ + struct ccp_dma_desc *desc, *dtmp; + + list_for_each_entry_safe_reverse(desc, dtmp, list, entry) { + if (!async_tx_test_ack(&desc->tx_desc)) + continue; + + dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc); + + ccp_free_cmd_resources(ccp, &desc->active); + ccp_free_cmd_resources(ccp, &desc->pending); + + list_del(&desc->entry); + kmem_cache_free(ccp->dma_desc_cache, desc); + } +} + +static void ccp_do_cleanup(unsigned long data) +{ + struct ccp_dma_chan *chan = (struct ccp_dma_chan *)data; + unsigned long flags; + + dev_dbg(chan->ccp->dev, "%s - chan=%s\n", __func__, + dma_chan_name(&chan->dma_chan)); + + spin_lock_irqsave(&chan->lock, flags); + + ccp_cleanup_desc_resources(chan->ccp, &chan->complete); + + spin_unlock_irqrestore(&chan->lock, flags); +} + +static int ccp_issue_next_cmd(struct ccp_dma_desc *desc) +{ + struct ccp_dma_cmd *cmd; + int ret; + + cmd = list_first_entry(&desc->pending, struct ccp_dma_cmd, entry); + list_move(&cmd->entry, &desc->active); + + dev_dbg(desc->ccp->dev, "%s - tx %d, cmd=%p\n", __func__, + desc->tx_desc.cookie, cmd); + + ret = ccp_enqueue_cmd(&cmd->ccp_cmd); + if (!ret || (ret == -EINPROGRESS) || (ret == -EBUSY)) + return 0; + + dev_dbg(desc->ccp->dev, "%s - error: ret=%d, tx %d, cmd=%p\n", __func__, + ret, desc->tx_desc.cookie, cmd); + + return ret; +} + +static void ccp_free_active_cmd(struct ccp_dma_desc *desc) +{ + struct ccp_dma_cmd *cmd; + + cmd = list_first_entry_or_null(&desc->active, struct ccp_dma_cmd, + entry); + if (!cmd) + return; + + dev_dbg(desc->ccp->dev, "%s - freeing tx %d cmd=%p\n", + __func__, desc->tx_desc.cookie, cmd); + + list_del(&cmd->entry); + kmem_cache_free(desc->ccp->dma_cmd_cache, cmd); +} + +static struct ccp_dma_desc *__ccp_next_dma_desc(struct ccp_dma_chan *chan, + struct ccp_dma_desc *desc) +{ + /* Move current DMA descriptor to the complete list */ + if (desc) + list_move(&desc->entry, &chan->complete); + + /* Get the next DMA descriptor on the active list */ + desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc, + entry); + + return desc; +} + +static struct ccp_dma_desc *ccp_handle_active_desc(struct ccp_dma_chan *chan, + struct ccp_dma_desc *desc) +{ + struct dma_async_tx_descriptor *tx_desc; + unsigned long flags; + + /* Loop over descriptors until one is found with commands */ + do { + if (desc) { + /* Remove the DMA command from the list and free it */ + ccp_free_active_cmd(desc); + + if (!list_empty(&desc->pending)) { + /* No errors, keep going */ + if (desc->status != DMA_ERROR) + return desc; + + /* Error, free remaining commands and move on */ + ccp_free_cmd_resources(desc->ccp, + &desc->pending); + } + + tx_desc = &desc->tx_desc; + } else { + tx_desc = NULL; + } + + spin_lock_irqsave(&chan->lock, flags); + + if (desc) { + if (desc->status != DMA_ERROR) + desc->status = DMA_COMPLETE; + + dev_dbg(desc->ccp->dev, + "%s - tx %d complete, status=%u\n", __func__, + desc->tx_desc.cookie, desc->status); + + dma_cookie_complete(tx_desc); + dma_descriptor_unmap(tx_desc); + } + + desc = __ccp_next_dma_desc(chan, desc); + + spin_unlock_irqrestore(&chan->lock, flags); + + if (tx_desc) { + dmaengine_desc_get_callback_invoke(tx_desc, NULL); + + dma_run_dependencies(tx_desc); + } + } while (desc); + + return NULL; +} + +static struct ccp_dma_desc *__ccp_pending_to_active(struct ccp_dma_chan *chan) +{ + struct ccp_dma_desc *desc; + + if (list_empty(&chan->pending)) + return NULL; + + desc = list_empty(&chan->active) + ? list_first_entry(&chan->pending, struct ccp_dma_desc, entry) + : NULL; + + list_splice_tail_init(&chan->pending, &chan->active); + + return desc; +} + +static void ccp_cmd_callback(void *data, int err) +{ + struct ccp_dma_desc *desc = data; + struct ccp_dma_chan *chan; + int ret; + + if (err == -EINPROGRESS) + return; + + chan = container_of(desc->tx_desc.chan, struct ccp_dma_chan, + dma_chan); + + dev_dbg(chan->ccp->dev, "%s - tx %d callback, err=%d\n", + __func__, desc->tx_desc.cookie, err); + + if (err) + desc->status = DMA_ERROR; + + while (true) { + /* Check for DMA descriptor completion */ + desc = ccp_handle_active_desc(chan, desc); + + /* Don't submit cmd if no descriptor or DMA is paused */ + if (!desc || (chan->status == DMA_PAUSED)) + break; + + ret = ccp_issue_next_cmd(desc); + if (!ret) + break; + + desc->status = DMA_ERROR; + } + + tasklet_schedule(&chan->cleanup_tasklet); +} + +static dma_cookie_t ccp_tx_submit(struct dma_async_tx_descriptor *tx_desc) +{ + struct ccp_dma_desc *desc = container_of(tx_desc, struct ccp_dma_desc, + tx_desc); + struct ccp_dma_chan *chan; + dma_cookie_t cookie; + unsigned long flags; + + chan = container_of(tx_desc->chan, struct ccp_dma_chan, dma_chan); + + spin_lock_irqsave(&chan->lock, flags); + + cookie = dma_cookie_assign(tx_desc); + list_move_tail(&desc->entry, &chan->pending); + + spin_unlock_irqrestore(&chan->lock, flags); + + dev_dbg(chan->ccp->dev, "%s - added tx descriptor %d to pending list\n", + __func__, cookie); + + return cookie; +} + +static struct ccp_dma_cmd *ccp_alloc_dma_cmd(struct ccp_dma_chan *chan) +{ + struct ccp_dma_cmd *cmd; + + cmd = kmem_cache_alloc(chan->ccp->dma_cmd_cache, GFP_NOWAIT); + if (cmd) + memset(cmd, 0, sizeof(*cmd)); + + return cmd; +} + +static struct ccp_dma_desc *ccp_alloc_dma_desc(struct ccp_dma_chan *chan, + unsigned long flags) +{ + struct ccp_dma_desc *desc; + + desc = kmem_cache_zalloc(chan->ccp->dma_desc_cache, GFP_NOWAIT); + if (!desc) + return NULL; + + dma_async_tx_descriptor_init(&desc->tx_desc, &chan->dma_chan); + desc->tx_desc.flags = flags; + desc->tx_desc.tx_submit = ccp_tx_submit; + desc->ccp = chan->ccp; + INIT_LIST_HEAD(&desc->entry); + INIT_LIST_HEAD(&desc->pending); + INIT_LIST_HEAD(&desc->active); + desc->status = DMA_IN_PROGRESS; + + return desc; +} + +static struct ccp_dma_desc *ccp_create_desc(struct dma_chan *dma_chan, + struct scatterlist *dst_sg, + unsigned int dst_nents, + struct scatterlist *src_sg, + unsigned int src_nents, + unsigned long flags) +{ + struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, + dma_chan); + struct ccp_device *ccp = chan->ccp; + struct ccp_dma_desc *desc; + struct ccp_dma_cmd *cmd; + struct ccp_cmd *ccp_cmd; + struct ccp_passthru_nomap_engine *ccp_pt; + unsigned int src_offset, src_len; + unsigned int dst_offset, dst_len; + unsigned int len; + unsigned long sflags; + size_t total_len; + + if (!dst_sg || !src_sg) + return NULL; + + if (!dst_nents || !src_nents) + return NULL; + + desc = ccp_alloc_dma_desc(chan, flags); + if (!desc) + return NULL; + + total_len = 0; + + src_len = sg_dma_len(src_sg); + src_offset = 0; + + dst_len = sg_dma_len(dst_sg); + dst_offset = 0; + + while (true) { + if (!src_len) { + src_nents--; + if (!src_nents) + break; + + src_sg = sg_next(src_sg); + if (!src_sg) + break; + + src_len = sg_dma_len(src_sg); + src_offset = 0; + continue; + } + + if (!dst_len) { + dst_nents--; + if (!dst_nents) + break; + + dst_sg = sg_next(dst_sg); + if (!dst_sg) + break; + + dst_len = sg_dma_len(dst_sg); + dst_offset = 0; + continue; + } + + len = min(dst_len, src_len); + + cmd = ccp_alloc_dma_cmd(chan); + if (!cmd) + goto err; + + ccp_cmd = &cmd->ccp_cmd; + ccp_cmd->ccp = chan->ccp; + ccp_pt = &ccp_cmd->u.passthru_nomap; + ccp_cmd->flags = CCP_CMD_MAY_BACKLOG; + ccp_cmd->flags |= CCP_CMD_PASSTHRU_NO_DMA_MAP; + ccp_cmd->engine = CCP_ENGINE_PASSTHRU; + ccp_pt->bit_mod = CCP_PASSTHRU_BITWISE_NOOP; + ccp_pt->byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP; + ccp_pt->src_dma = sg_dma_address(src_sg) + src_offset; + ccp_pt->dst_dma = sg_dma_address(dst_sg) + dst_offset; + ccp_pt->src_len = len; + ccp_pt->final = 1; + ccp_cmd->callback = ccp_cmd_callback; + ccp_cmd->data = desc; + + list_add_tail(&cmd->entry, &desc->pending); + + dev_dbg(ccp->dev, + "%s - cmd=%p, src=%pad, dst=%pad, len=%llu\n", __func__, + cmd, &ccp_pt->src_dma, + &ccp_pt->dst_dma, ccp_pt->src_len); + + total_len += len; + + src_len -= len; + src_offset += len; + + dst_len -= len; + dst_offset += len; + } + + desc->len = total_len; + + if (list_empty(&desc->pending)) + goto err; + + dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc); + + spin_lock_irqsave(&chan->lock, sflags); + + list_add_tail(&desc->entry, &chan->created); + + spin_unlock_irqrestore(&chan->lock, sflags); + + return desc; + +err: + ccp_free_cmd_resources(ccp, &desc->pending); + kmem_cache_free(ccp->dma_desc_cache, desc); + + return NULL; +} + +static struct dma_async_tx_descriptor *ccp_prep_dma_memcpy( + struct dma_chan *dma_chan, dma_addr_t dst, dma_addr_t src, size_t len, + unsigned long flags) +{ + struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, + dma_chan); + struct ccp_dma_desc *desc; + struct scatterlist dst_sg, src_sg; + + dev_dbg(chan->ccp->dev, + "%s - src=%pad, dst=%pad, len=%zu, flags=%#lx\n", + __func__, &src, &dst, len, flags); + + sg_init_table(&dst_sg, 1); + sg_dma_address(&dst_sg) = dst; + sg_dma_len(&dst_sg) = len; + + sg_init_table(&src_sg, 1); + sg_dma_address(&src_sg) = src; + sg_dma_len(&src_sg) = len; + + desc = ccp_create_desc(dma_chan, &dst_sg, 1, &src_sg, 1, flags); + if (!desc) + return NULL; + + return &desc->tx_desc; +} + +static struct dma_async_tx_descriptor *ccp_prep_dma_interrupt( + struct dma_chan *dma_chan, unsigned long flags) +{ + struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, + dma_chan); + struct ccp_dma_desc *desc; + + desc = ccp_alloc_dma_desc(chan, flags); + if (!desc) + return NULL; + + return &desc->tx_desc; +} + +static void ccp_issue_pending(struct dma_chan *dma_chan) +{ + struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, + dma_chan); + struct ccp_dma_desc *desc; + unsigned long flags; + + dev_dbg(chan->ccp->dev, "%s\n", __func__); + + spin_lock_irqsave(&chan->lock, flags); + + desc = __ccp_pending_to_active(chan); + + spin_unlock_irqrestore(&chan->lock, flags); + + /* If there was nothing active, start processing */ + if (desc) + ccp_cmd_callback(desc, 0); +} + +static enum dma_status ccp_tx_status(struct dma_chan *dma_chan, + dma_cookie_t cookie, + struct dma_tx_state *state) +{ + struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, + dma_chan); + struct ccp_dma_desc *desc; + enum dma_status ret; + unsigned long flags; + + if (chan->status == DMA_PAUSED) { + ret = DMA_PAUSED; + goto out; + } + + ret = dma_cookie_status(dma_chan, cookie, state); + if (ret == DMA_COMPLETE) { + spin_lock_irqsave(&chan->lock, flags); + + /* Get status from complete chain, if still there */ + list_for_each_entry(desc, &chan->complete, entry) { + if (desc->tx_desc.cookie != cookie) + continue; + + ret = desc->status; + break; + } + + spin_unlock_irqrestore(&chan->lock, flags); + } + +out: + dev_dbg(chan->ccp->dev, "%s - %u\n", __func__, ret); + + return ret; +} + +static int ccp_pause(struct dma_chan *dma_chan) +{ + struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, + dma_chan); + + chan->status = DMA_PAUSED; + + /*TODO: Wait for active DMA to complete before returning? */ + + return 0; +} + +static int ccp_resume(struct dma_chan *dma_chan) +{ + struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, + dma_chan); + struct ccp_dma_desc *desc; + unsigned long flags; + + spin_lock_irqsave(&chan->lock, flags); + + desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc, + entry); + + spin_unlock_irqrestore(&chan->lock, flags); + + /* Indicate the channel is running again */ + chan->status = DMA_IN_PROGRESS; + + /* If there was something active, re-start */ + if (desc) + ccp_cmd_callback(desc, 0); + + return 0; +} + +static int ccp_terminate_all(struct dma_chan *dma_chan) +{ + struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, + dma_chan); + unsigned long flags; + + dev_dbg(chan->ccp->dev, "%s\n", __func__); + + /*TODO: Wait for active DMA to complete before continuing */ + + spin_lock_irqsave(&chan->lock, flags); + + /*TODO: Purge the complete list? */ + ccp_free_desc_resources(chan->ccp, &chan->active); + ccp_free_desc_resources(chan->ccp, &chan->pending); + ccp_free_desc_resources(chan->ccp, &chan->created); + + spin_unlock_irqrestore(&chan->lock, flags); + + return 0; +} + +static void ccp_dma_release(struct ccp_device *ccp) +{ + struct ccp_dma_chan *chan; + struct dma_chan *dma_chan; + unsigned int i; + + for (i = 0; i < ccp->cmd_q_count; i++) { + chan = ccp->ccp_dma_chan + i; + dma_chan = &chan->dma_chan; + + tasklet_kill(&chan->cleanup_tasklet); + list_del_rcu(&dma_chan->device_node); + } +} + +static void ccp_dma_release_channels(struct ccp_device *ccp) +{ + struct ccp_dma_chan *chan; + struct dma_chan *dma_chan; + unsigned int i; + + for (i = 0; i < ccp->cmd_q_count; i++) { + chan = ccp->ccp_dma_chan + i; + dma_chan = &chan->dma_chan; + + if (dma_chan->client_count) + dma_release_channel(dma_chan); + } +} + +int ccp_dmaengine_register(struct ccp_device *ccp) +{ + struct ccp_dma_chan *chan; + struct dma_device *dma_dev = &ccp->dma_dev; + struct dma_chan *dma_chan; + char *dma_cmd_cache_name; + char *dma_desc_cache_name; + unsigned int i; + int ret; + + if (!dmaengine) + return 0; + + ccp->ccp_dma_chan = devm_kcalloc(ccp->dev, ccp->cmd_q_count, + sizeof(*(ccp->ccp_dma_chan)), + GFP_KERNEL); + if (!ccp->ccp_dma_chan) + return -ENOMEM; + + dma_cmd_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL, + "%s-dmaengine-cmd-cache", + ccp->name); + if (!dma_cmd_cache_name) + return -ENOMEM; + + ccp->dma_cmd_cache = kmem_cache_create(dma_cmd_cache_name, + sizeof(struct ccp_dma_cmd), + sizeof(void *), + SLAB_HWCACHE_ALIGN, NULL); + if (!ccp->dma_cmd_cache) + return -ENOMEM; + + dma_desc_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL, + "%s-dmaengine-desc-cache", + ccp->name); + if (!dma_desc_cache_name) { + ret = -ENOMEM; + goto err_cache; + } + + ccp->dma_desc_cache = kmem_cache_create(dma_desc_cache_name, + sizeof(struct ccp_dma_desc), + sizeof(void *), + SLAB_HWCACHE_ALIGN, NULL); + if (!ccp->dma_desc_cache) { + ret = -ENOMEM; + goto err_cache; + } + + dma_dev->dev = ccp->dev; + dma_dev->src_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev)); + dma_dev->dst_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev)); + dma_dev->directions = DMA_MEM_TO_MEM; + dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; + dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); + dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); + + /* The DMA channels for this device can be set to public or private, + * and overridden by the module parameter dma_chan_attr. + * Default: according to the value in vdata (dma_chan_attr=0) + * dma_chan_attr=0x1: all channels private (override vdata) + * dma_chan_attr=0x2: all channels public (override vdata) + */ + if (ccp_get_dma_chan_attr(ccp) == DMA_PRIVATE) + dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); + + INIT_LIST_HEAD(&dma_dev->channels); + for (i = 0; i < ccp->cmd_q_count; i++) { + chan = ccp->ccp_dma_chan + i; + dma_chan = &chan->dma_chan; + + chan->ccp = ccp; + + spin_lock_init(&chan->lock); + INIT_LIST_HEAD(&chan->created); + INIT_LIST_HEAD(&chan->pending); + INIT_LIST_HEAD(&chan->active); + INIT_LIST_HEAD(&chan->complete); + + tasklet_init(&chan->cleanup_tasklet, ccp_do_cleanup, + (unsigned long)chan); + + dma_chan->device = dma_dev; + dma_cookie_init(dma_chan); + + list_add_tail(&dma_chan->device_node, &dma_dev->channels); + } + + dma_dev->device_free_chan_resources = ccp_free_chan_resources; + dma_dev->device_prep_dma_memcpy = ccp_prep_dma_memcpy; + dma_dev->device_prep_dma_interrupt = ccp_prep_dma_interrupt; + dma_dev->device_issue_pending = ccp_issue_pending; + dma_dev->device_tx_status = ccp_tx_status; + dma_dev->device_pause = ccp_pause; + dma_dev->device_resume = ccp_resume; + dma_dev->device_terminate_all = ccp_terminate_all; + + ret = dma_async_device_register(dma_dev); + if (ret) + goto err_reg; + + return 0; + +err_reg: + ccp_dma_release(ccp); + kmem_cache_destroy(ccp->dma_desc_cache); + +err_cache: + kmem_cache_destroy(ccp->dma_cmd_cache); + + return ret; +} + +void ccp_dmaengine_unregister(struct ccp_device *ccp) +{ + struct dma_device *dma_dev = &ccp->dma_dev; + + if (!dmaengine) + return; + + ccp_dma_release_channels(ccp); + dma_async_device_unregister(dma_dev); + ccp_dma_release(ccp); + + kmem_cache_destroy(ccp->dma_desc_cache); + kmem_cache_destroy(ccp->dma_cmd_cache); +} diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c new file mode 100644 index 000000000..f35803c37 --- /dev/null +++ b/drivers/crypto/ccp/ccp-ops.c @@ -0,0 +1,2525 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Cryptographic Coprocessor (CCP) driver + * + * Copyright (C) 2013-2019 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky + * Author: Gary R Hook + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ccp-dev.h" + +/* SHA initial context values */ +static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = { + cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1), + cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3), + cpu_to_be32(SHA1_H4), +}; + +static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = { + cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1), + cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3), + cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5), + cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7), +}; + +static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = { + cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1), + cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3), + cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5), + cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7), +}; + +static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = { + cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1), + cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3), + cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5), + cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7), +}; + +static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = { + cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1), + cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3), + cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5), + cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7), +}; + +#define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \ + ccp_gen_jobid(ccp) : 0) + +static u32 ccp_gen_jobid(struct ccp_device *ccp) +{ + return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK; +} + +static void ccp_sg_free(struct ccp_sg_workarea *wa) +{ + if (wa->dma_count) + dma_unmap_sg(wa->dma_dev, wa->dma_sg_head, wa->nents, wa->dma_dir); + + wa->dma_count = 0; +} + +static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev, + struct scatterlist *sg, u64 len, + enum dma_data_direction dma_dir) +{ + memset(wa, 0, sizeof(*wa)); + + wa->sg = sg; + if (!sg) + return 0; + + wa->nents = sg_nents_for_len(sg, len); + if (wa->nents < 0) + return wa->nents; + + wa->bytes_left = len; + wa->sg_used = 0; + + if (len == 0) + return 0; + + if (dma_dir == DMA_NONE) + return 0; + + wa->dma_sg = sg; + wa->dma_sg_head = sg; + wa->dma_dev = dev; + wa->dma_dir = dma_dir; + wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir); + if (!wa->dma_count) + return -ENOMEM; + + return 0; +} + +static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len) +{ + unsigned int nbytes = min_t(u64, len, wa->bytes_left); + unsigned int sg_combined_len = 0; + + if (!wa->sg) + return; + + wa->sg_used += nbytes; + wa->bytes_left -= nbytes; + if (wa->sg_used == sg_dma_len(wa->dma_sg)) { + /* Advance to the next DMA scatterlist entry */ + wa->dma_sg = sg_next(wa->dma_sg); + + /* In the case that the DMA mapped scatterlist has entries + * that have been merged, the non-DMA mapped scatterlist + * must be advanced multiple times for each merged entry. + * This ensures that the current non-DMA mapped entry + * corresponds to the current DMA mapped entry. + */ + do { + sg_combined_len += wa->sg->length; + wa->sg = sg_next(wa->sg); + } while (wa->sg_used > sg_combined_len); + + wa->sg_used = 0; + } +} + +static void ccp_dm_free(struct ccp_dm_workarea *wa) +{ + if (wa->length <= CCP_DMAPOOL_MAX_SIZE) { + if (wa->address) + dma_pool_free(wa->dma_pool, wa->address, + wa->dma.address); + } else { + if (wa->dma.address) + dma_unmap_single(wa->dev, wa->dma.address, wa->length, + wa->dma.dir); + kfree(wa->address); + } + + wa->address = NULL; + wa->dma.address = 0; +} + +static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa, + struct ccp_cmd_queue *cmd_q, + unsigned int len, + enum dma_data_direction dir) +{ + memset(wa, 0, sizeof(*wa)); + + if (!len) + return 0; + + wa->dev = cmd_q->ccp->dev; + wa->length = len; + + if (len <= CCP_DMAPOOL_MAX_SIZE) { + wa->dma_pool = cmd_q->dma_pool; + + wa->address = dma_pool_zalloc(wa->dma_pool, GFP_KERNEL, + &wa->dma.address); + if (!wa->address) + return -ENOMEM; + + wa->dma.length = CCP_DMAPOOL_MAX_SIZE; + + } else { + wa->address = kzalloc(len, GFP_KERNEL); + if (!wa->address) + return -ENOMEM; + + wa->dma.address = dma_map_single(wa->dev, wa->address, len, + dir); + if (dma_mapping_error(wa->dev, wa->dma.address)) { + kfree(wa->address); + wa->address = NULL; + return -ENOMEM; + } + + wa->dma.length = len; + } + wa->dma.dir = dir; + + return 0; +} + +static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, + struct scatterlist *sg, unsigned int sg_offset, + unsigned int len) +{ + WARN_ON(!wa->address); + + if (len > (wa->length - wa_offset)) + return -EINVAL; + + scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, + 0); + return 0; +} + +static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, + struct scatterlist *sg, unsigned int sg_offset, + unsigned int len) +{ + WARN_ON(!wa->address); + + scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, + 1); +} + +static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa, + unsigned int wa_offset, + struct scatterlist *sg, + unsigned int sg_offset, + unsigned int len) +{ + u8 *p, *q; + int rc; + + rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len); + if (rc) + return rc; + + p = wa->address + wa_offset; + q = p + len - 1; + while (p < q) { + *p = *p ^ *q; + *q = *p ^ *q; + *p = *p ^ *q; + p++; + q--; + } + return 0; +} + +static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa, + unsigned int wa_offset, + struct scatterlist *sg, + unsigned int sg_offset, + unsigned int len) +{ + u8 *p, *q; + + p = wa->address + wa_offset; + q = p + len - 1; + while (p < q) { + *p = *p ^ *q; + *q = *p ^ *q; + *p = *p ^ *q; + p++; + q--; + } + + ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len); +} + +static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q) +{ + ccp_dm_free(&data->dm_wa); + ccp_sg_free(&data->sg_wa); +} + +static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q, + struct scatterlist *sg, u64 sg_len, + unsigned int dm_len, + enum dma_data_direction dir) +{ + int ret; + + memset(data, 0, sizeof(*data)); + + ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len, + dir); + if (ret) + goto e_err; + + ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir); + if (ret) + goto e_err; + + return 0; + +e_err: + ccp_free_data(data, cmd_q); + + return ret; +} + +static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from) +{ + struct ccp_sg_workarea *sg_wa = &data->sg_wa; + struct ccp_dm_workarea *dm_wa = &data->dm_wa; + unsigned int buf_count, nbytes; + + /* Clear the buffer if setting it */ + if (!from) + memset(dm_wa->address, 0, dm_wa->length); + + if (!sg_wa->sg) + return 0; + + /* Perform the copy operation + * nbytes will always be <= UINT_MAX because dm_wa->length is + * an unsigned int + */ + nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length); + scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used, + nbytes, from); + + /* Update the structures and generate the count */ + buf_count = 0; + while (sg_wa->bytes_left && (buf_count < dm_wa->length)) { + nbytes = min(sg_dma_len(sg_wa->dma_sg) - sg_wa->sg_used, + dm_wa->length - buf_count); + nbytes = min_t(u64, sg_wa->bytes_left, nbytes); + + buf_count += nbytes; + ccp_update_sg_workarea(sg_wa, nbytes); + } + + return buf_count; +} + +static unsigned int ccp_fill_queue_buf(struct ccp_data *data) +{ + return ccp_queue_buf(data, 0); +} + +static unsigned int ccp_empty_queue_buf(struct ccp_data *data) +{ + return ccp_queue_buf(data, 1); +} + +static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst, + struct ccp_op *op, unsigned int block_size, + bool blocksize_op) +{ + unsigned int sg_src_len, sg_dst_len, op_len; + + /* The CCP can only DMA from/to one address each per operation. This + * requires that we find the smallest DMA area between the source + * and destination. The resulting len values will always be <= UINT_MAX + * because the dma length is an unsigned int. + */ + sg_src_len = sg_dma_len(src->sg_wa.dma_sg) - src->sg_wa.sg_used; + sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len); + + if (dst) { + sg_dst_len = sg_dma_len(dst->sg_wa.dma_sg) - dst->sg_wa.sg_used; + sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len); + op_len = min(sg_src_len, sg_dst_len); + } else { + op_len = sg_src_len; + } + + /* The data operation length will be at least block_size in length + * or the smaller of available sg room remaining for the source or + * the destination + */ + op_len = max(op_len, block_size); + + /* Unless we have to buffer data, there's no reason to wait */ + op->soc = 0; + + if (sg_src_len < block_size) { + /* Not enough data in the sg element, so it + * needs to be buffered into a blocksize chunk + */ + int cp_len = ccp_fill_queue_buf(src); + + op->soc = 1; + op->src.u.dma.address = src->dm_wa.dma.address; + op->src.u.dma.offset = 0; + op->src.u.dma.length = (blocksize_op) ? block_size : cp_len; + } else { + /* Enough data in the sg element, but we need to + * adjust for any previously copied data + */ + op->src.u.dma.address = sg_dma_address(src->sg_wa.dma_sg); + op->src.u.dma.offset = src->sg_wa.sg_used; + op->src.u.dma.length = op_len & ~(block_size - 1); + + ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length); + } + + if (dst) { + if (sg_dst_len < block_size) { + /* Not enough room in the sg element or we're on the + * last piece of data (when using padding), so the + * output needs to be buffered into a blocksize chunk + */ + op->soc = 1; + op->dst.u.dma.address = dst->dm_wa.dma.address; + op->dst.u.dma.offset = 0; + op->dst.u.dma.length = op->src.u.dma.length; + } else { + /* Enough room in the sg element, but we need to + * adjust for any previously used area + */ + op->dst.u.dma.address = sg_dma_address(dst->sg_wa.dma_sg); + op->dst.u.dma.offset = dst->sg_wa.sg_used; + op->dst.u.dma.length = op->src.u.dma.length; + } + } +} + +static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst, + struct ccp_op *op) +{ + op->init = 0; + + if (dst) { + if (op->dst.u.dma.address == dst->dm_wa.dma.address) + ccp_empty_queue_buf(dst); + else + ccp_update_sg_workarea(&dst->sg_wa, + op->dst.u.dma.length); + } +} + +static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q, + struct ccp_dm_workarea *wa, u32 jobid, u32 sb, + u32 byte_swap, bool from) +{ + struct ccp_op op; + + memset(&op, 0, sizeof(op)); + + op.cmd_q = cmd_q; + op.jobid = jobid; + op.eom = 1; + + if (from) { + op.soc = 1; + op.src.type = CCP_MEMTYPE_SB; + op.src.u.sb = sb; + op.dst.type = CCP_MEMTYPE_SYSTEM; + op.dst.u.dma.address = wa->dma.address; + op.dst.u.dma.length = wa->length; + } else { + op.src.type = CCP_MEMTYPE_SYSTEM; + op.src.u.dma.address = wa->dma.address; + op.src.u.dma.length = wa->length; + op.dst.type = CCP_MEMTYPE_SB; + op.dst.u.sb = sb; + } + + op.u.passthru.byte_swap = byte_swap; + + return cmd_q->ccp->vdata->perform->passthru(&op); +} + +static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q, + struct ccp_dm_workarea *wa, u32 jobid, u32 sb, + u32 byte_swap) +{ + return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false); +} + +static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q, + struct ccp_dm_workarea *wa, u32 jobid, u32 sb, + u32 byte_swap) +{ + return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true); +} + +static noinline_for_stack int +ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + struct ccp_aes_engine *aes = &cmd->u.aes; + struct ccp_dm_workarea key, ctx; + struct ccp_data src; + struct ccp_op op; + unsigned int dm_offset; + int ret; + + if (!((aes->key_len == AES_KEYSIZE_128) || + (aes->key_len == AES_KEYSIZE_192) || + (aes->key_len == AES_KEYSIZE_256))) + return -EINVAL; + + if (aes->src_len & (AES_BLOCK_SIZE - 1)) + return -EINVAL; + + if (aes->iv_len != AES_BLOCK_SIZE) + return -EINVAL; + + if (!aes->key || !aes->iv || !aes->src) + return -EINVAL; + + if (aes->cmac_final) { + if (aes->cmac_key_len != AES_BLOCK_SIZE) + return -EINVAL; + + if (!aes->cmac_key) + return -EINVAL; + } + + BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1); + BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1); + + ret = -EIO; + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = CCP_NEW_JOBID(cmd_q->ccp); + op.sb_key = cmd_q->sb_key; + op.sb_ctx = cmd_q->sb_ctx; + op.init = 1; + op.u.aes.type = aes->type; + op.u.aes.mode = aes->mode; + op.u.aes.action = aes->action; + + /* All supported key sizes fit in a single (32-byte) SB entry + * and must be in little endian format. Use the 256-bit byte + * swap passthru option to convert from big endian to little + * endian. + */ + ret = ccp_init_dm_workarea(&key, cmd_q, + CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES, + DMA_TO_DEVICE); + if (ret) + return ret; + + dm_offset = CCP_SB_BYTES - aes->key_len; + ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); + if (ret) + goto e_key; + ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_key; + } + + /* The AES context fits in a single (32-byte) SB entry and + * must be in little endian format. Use the 256-bit byte swap + * passthru option to convert from big endian to little endian. + */ + ret = ccp_init_dm_workarea(&ctx, cmd_q, + CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, + DMA_BIDIRECTIONAL); + if (ret) + goto e_key; + + dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; + ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); + if (ret) + goto e_ctx; + ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_ctx; + } + + /* Send data to the CCP AES engine */ + ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, + AES_BLOCK_SIZE, DMA_TO_DEVICE); + if (ret) + goto e_ctx; + + while (src.sg_wa.bytes_left) { + ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true); + if (aes->cmac_final && !src.sg_wa.bytes_left) { + op.eom = 1; + + /* Push the K1/K2 key to the CCP now */ + ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, + op.sb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_src; + } + + ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0, + aes->cmac_key_len); + if (ret) + goto e_src; + ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_src; + } + } + + ret = cmd_q->ccp->vdata->perform->aes(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_src; + } + + ccp_process_data(&src, NULL, &op); + } + + /* Retrieve the AES context - convert from LE to BE using + * 32-byte (256-bit) byteswapping + */ + ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_src; + } + + /* ...but we only need AES_BLOCK_SIZE bytes */ + dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; + ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); + +e_src: + ccp_free_data(&src, cmd_q); + +e_ctx: + ccp_dm_free(&ctx); + +e_key: + ccp_dm_free(&key); + + return ret; +} + +static noinline_for_stack int +ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + struct ccp_aes_engine *aes = &cmd->u.aes; + struct { + struct ccp_dm_workarea key; + struct ccp_dm_workarea ctx; + struct ccp_dm_workarea final; + struct ccp_dm_workarea tag; + struct ccp_data src; + struct ccp_data dst; + struct ccp_data aad; + struct ccp_op op; + } *wa __free(kfree) = kzalloc_obj(*wa); + unsigned int dm_offset; + unsigned int authsize; + unsigned int jobid; + unsigned int ilen; + bool in_place = true; /* Default value */ + __be64 *final; + int ret; + + struct scatterlist *p_inp, sg_inp[2]; + struct scatterlist *p_tag, sg_tag[2]; + struct scatterlist *p_outp, sg_outp[2]; + struct scatterlist *p_aad; + + if (!wa) + return -ENOMEM; + + if (!aes->iv) + return -EINVAL; + + if (!((aes->key_len == AES_KEYSIZE_128) || + (aes->key_len == AES_KEYSIZE_192) || + (aes->key_len == AES_KEYSIZE_256))) + return -EINVAL; + + if (!aes->key) /* Gotta have a key SGL */ + return -EINVAL; + + /* Zero defaults to 16 bytes, the maximum size */ + authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE; + switch (authsize) { + case 16: + case 15: + case 14: + case 13: + case 12: + case 8: + case 4: + break; + default: + return -EINVAL; + } + + /* First, decompose the source buffer into AAD & PT, + * and the destination buffer into AAD, CT & tag, or + * the input into CT & tag. + * It is expected that the input and output SGs will + * be valid, even if the AAD and input lengths are 0. + */ + p_aad = aes->src; + p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len); + p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len); + if (aes->action == CCP_AES_ACTION_ENCRYPT) { + ilen = aes->src_len; + p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen); + } else { + /* Input length for decryption includes tag */ + ilen = aes->src_len - authsize; + p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen); + } + + jobid = CCP_NEW_JOBID(cmd_q->ccp); + + memset(&wa->op, 0, sizeof(wa->op)); + wa->op.cmd_q = cmd_q; + wa->op.jobid = jobid; + wa->op.sb_key = cmd_q->sb_key; /* Pre-allocated */ + wa->op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ + wa->op.init = 1; + wa->op.u.aes.type = aes->type; + + /* Copy the key to the LSB */ + ret = ccp_init_dm_workarea(&wa->key, cmd_q, + CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, + DMA_TO_DEVICE); + if (ret) + return ret; + + dm_offset = CCP_SB_BYTES - aes->key_len; + ret = ccp_set_dm_area(&wa->key, dm_offset, aes->key, 0, aes->key_len); + if (ret) + goto e_key; + ret = ccp_copy_to_sb(cmd_q, &wa->key, wa->op.jobid, wa->op.sb_key, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_key; + } + + /* Copy the context (IV) to the LSB. + * There is an assumption here that the IV is 96 bits in length, plus + * a nonce of 32 bits. If no IV is present, use a zeroed buffer. + */ + ret = ccp_init_dm_workarea(&wa->ctx, cmd_q, + CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, + DMA_BIDIRECTIONAL); + if (ret) + goto e_key; + + dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len; + ret = ccp_set_dm_area(&wa->ctx, dm_offset, aes->iv, 0, aes->iv_len); + if (ret) + goto e_ctx; + + ret = ccp_copy_to_sb(cmd_q, &wa->ctx, wa->op.jobid, wa->op.sb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_ctx; + } + + wa->op.init = 1; + if (aes->aad_len > 0) { + /* Step 1: Run a GHASH over the Additional Authenticated Data */ + ret = ccp_init_data(&wa->aad, cmd_q, p_aad, aes->aad_len, + AES_BLOCK_SIZE, + DMA_TO_DEVICE); + if (ret) + goto e_ctx; + + wa->op.u.aes.mode = CCP_AES_MODE_GHASH; + wa->op.u.aes.action = CCP_AES_GHASHAAD; + + while (wa->aad.sg_wa.bytes_left) { + ccp_prepare_data(&wa->aad, NULL, &wa->op, AES_BLOCK_SIZE, true); + + ret = cmd_q->ccp->vdata->perform->aes(&wa->op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_aad; + } + + ccp_process_data(&wa->aad, NULL, &wa->op); + wa->op.init = 0; + } + } + + wa->op.u.aes.mode = CCP_AES_MODE_GCTR; + wa->op.u.aes.action = aes->action; + + if (ilen > 0) { + /* Step 2: Run a GCTR over the plaintext */ + in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false; + + ret = ccp_init_data(&wa->src, cmd_q, p_inp, ilen, + AES_BLOCK_SIZE, + in_place ? DMA_BIDIRECTIONAL + : DMA_TO_DEVICE); + if (ret) + goto e_aad; + + if (in_place) { + wa->dst = wa->src; + } else { + ret = ccp_init_data(&wa->dst, cmd_q, p_outp, ilen, + AES_BLOCK_SIZE, DMA_FROM_DEVICE); + if (ret) + goto e_src; + } + + wa->op.soc = 0; + wa->op.eom = 0; + wa->op.init = 1; + while (wa->src.sg_wa.bytes_left) { + ccp_prepare_data(&wa->src, &wa->dst, &wa->op, AES_BLOCK_SIZE, true); + if (!wa->src.sg_wa.bytes_left) { + unsigned int nbytes = ilen % AES_BLOCK_SIZE; + + if (nbytes) { + wa->op.eom = 1; + wa->op.u.aes.size = (nbytes * 8) - 1; + } + } + + ret = cmd_q->ccp->vdata->perform->aes(&wa->op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + ccp_process_data(&wa->src, &wa->dst, &wa->op); + wa->op.init = 0; + } + } + + /* Step 3: Update the IV portion of the context with the original IV */ + ret = ccp_copy_from_sb(cmd_q, &wa->ctx, wa->op.jobid, wa->op.sb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + ret = ccp_set_dm_area(&wa->ctx, dm_offset, aes->iv, 0, aes->iv_len); + if (ret) + goto e_dst; + + ret = ccp_copy_to_sb(cmd_q, &wa->ctx, wa->op.jobid, wa->op.sb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + /* Step 4: Concatenate the lengths of the AAD and source, and + * hash that 16 byte buffer. + */ + ret = ccp_init_dm_workarea(&wa->final, cmd_q, AES_BLOCK_SIZE, + DMA_BIDIRECTIONAL); + if (ret) + goto e_dst; + final = (__be64 *)wa->final.address; + final[0] = cpu_to_be64(aes->aad_len * 8); + final[1] = cpu_to_be64(ilen * 8); + + memset(&wa->op, 0, sizeof(wa->op)); + wa->op.cmd_q = cmd_q; + wa->op.jobid = jobid; + wa->op.sb_key = cmd_q->sb_key; /* Pre-allocated */ + wa->op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ + wa->op.init = 1; + wa->op.u.aes.type = aes->type; + wa->op.u.aes.mode = CCP_AES_MODE_GHASH; + wa->op.u.aes.action = CCP_AES_GHASHFINAL; + wa->op.src.type = CCP_MEMTYPE_SYSTEM; + wa->op.src.u.dma.address = wa->final.dma.address; + wa->op.src.u.dma.length = AES_BLOCK_SIZE; + wa->op.dst.type = CCP_MEMTYPE_SYSTEM; + wa->op.dst.u.dma.address = wa->final.dma.address; + wa->op.dst.u.dma.length = AES_BLOCK_SIZE; + wa->op.eom = 1; + wa->op.u.aes.size = 0; + ret = cmd_q->ccp->vdata->perform->aes(&wa->op); + if (ret) + goto e_final_wa; + + if (aes->action == CCP_AES_ACTION_ENCRYPT) { + /* Put the ciphered tag after the ciphertext. */ + ccp_get_dm_area(&wa->final, 0, p_tag, 0, authsize); + } else { + /* Does this ciphered tag match the input? */ + ret = ccp_init_dm_workarea(&wa->tag, cmd_q, authsize, + DMA_BIDIRECTIONAL); + if (ret) + goto e_final_wa; + ret = ccp_set_dm_area(&wa->tag, 0, p_tag, 0, authsize); + if (ret) { + ccp_dm_free(&wa->tag); + goto e_final_wa; + } + + ret = crypto_memneq(wa->tag.address, wa->final.address, + authsize) ? -EBADMSG : 0; + ccp_dm_free(&wa->tag); + } + +e_final_wa: + ccp_dm_free(&wa->final); + +e_dst: + if (ilen > 0 && !in_place) + ccp_free_data(&wa->dst, cmd_q); + +e_src: + if (ilen > 0) + ccp_free_data(&wa->src, cmd_q); + +e_aad: + if (aes->aad_len) + ccp_free_data(&wa->aad, cmd_q); + +e_ctx: + ccp_dm_free(&wa->ctx); + +e_key: + ccp_dm_free(&wa->key); + + return ret; +} + +static noinline_for_stack int +ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + struct ccp_aes_engine *aes = &cmd->u.aes; + struct ccp_dm_workarea key, ctx; + struct ccp_data src, dst; + struct ccp_op op; + unsigned int dm_offset; + bool in_place = false; + int ret; + + if (!((aes->key_len == AES_KEYSIZE_128) || + (aes->key_len == AES_KEYSIZE_192) || + (aes->key_len == AES_KEYSIZE_256))) + return -EINVAL; + + if (((aes->mode == CCP_AES_MODE_ECB) || + (aes->mode == CCP_AES_MODE_CBC)) && + (aes->src_len & (AES_BLOCK_SIZE - 1))) + return -EINVAL; + + if (!aes->key || !aes->src || !aes->dst) + return -EINVAL; + + if (aes->mode != CCP_AES_MODE_ECB) { + if (aes->iv_len != AES_BLOCK_SIZE) + return -EINVAL; + + if (!aes->iv) + return -EINVAL; + } + + BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1); + BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1); + + ret = -EIO; + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = CCP_NEW_JOBID(cmd_q->ccp); + op.sb_key = cmd_q->sb_key; + op.sb_ctx = cmd_q->sb_ctx; + op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1; + op.u.aes.type = aes->type; + op.u.aes.mode = aes->mode; + op.u.aes.action = aes->action; + + /* All supported key sizes fit in a single (32-byte) SB entry + * and must be in little endian format. Use the 256-bit byte + * swap passthru option to convert from big endian to little + * endian. + */ + ret = ccp_init_dm_workarea(&key, cmd_q, + CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES, + DMA_TO_DEVICE); + if (ret) + return ret; + + dm_offset = CCP_SB_BYTES - aes->key_len; + ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); + if (ret) + goto e_key; + ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_key; + } + + /* The AES context fits in a single (32-byte) SB entry and + * must be in little endian format. Use the 256-bit byte swap + * passthru option to convert from big endian to little endian. + */ + ret = ccp_init_dm_workarea(&ctx, cmd_q, + CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, + DMA_BIDIRECTIONAL); + if (ret) + goto e_key; + + if (aes->mode != CCP_AES_MODE_ECB) { + /* Load the AES context - convert to LE */ + dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; + ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); + if (ret) + goto e_ctx; + ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_ctx; + } + } + switch (aes->mode) { + case CCP_AES_MODE_CFB: /* CFB128 only */ + case CCP_AES_MODE_CTR: + op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1; + break; + default: + op.u.aes.size = 0; + } + + /* Prepare the input and output data workareas. For in-place + * operations we need to set the dma direction to BIDIRECTIONAL + * and copy the src workarea to the dst workarea. + */ + if (sg_virt(aes->src) == sg_virt(aes->dst)) + in_place = true; + + ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, + AES_BLOCK_SIZE, + in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); + if (ret) + goto e_ctx; + + if (in_place) { + dst = src; + } else { + ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len, + AES_BLOCK_SIZE, DMA_FROM_DEVICE); + if (ret) + goto e_src; + } + + /* Send data to the CCP AES engine */ + while (src.sg_wa.bytes_left) { + ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true); + if (!src.sg_wa.bytes_left) { + op.eom = 1; + + /* Since we don't retrieve the AES context in ECB + * mode we have to wait for the operation to complete + * on the last piece of data + */ + if (aes->mode == CCP_AES_MODE_ECB) + op.soc = 1; + } + + ret = cmd_q->ccp->vdata->perform->aes(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + ccp_process_data(&src, &dst, &op); + } + + if (aes->mode != CCP_AES_MODE_ECB) { + /* Retrieve the AES context - convert from LE to BE using + * 32-byte (256-bit) byteswapping + */ + ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + /* ...but we only need AES_BLOCK_SIZE bytes */ + dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; + ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); + } + +e_dst: + if (!in_place) + ccp_free_data(&dst, cmd_q); + +e_src: + ccp_free_data(&src, cmd_q); + +e_ctx: + ccp_dm_free(&ctx); + +e_key: + ccp_dm_free(&key); + + return ret; +} + +static noinline_for_stack int +ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + struct ccp_xts_aes_engine *xts = &cmd->u.xts; + struct ccp_dm_workarea key, ctx; + struct ccp_data src, dst; + struct ccp_op op; + unsigned int unit_size, dm_offset; + bool in_place = false; + unsigned int sb_count; + enum ccp_aes_type aestype; + int ret; + + switch (xts->unit_size) { + case CCP_XTS_AES_UNIT_SIZE_16: + unit_size = 16; + break; + case CCP_XTS_AES_UNIT_SIZE_512: + unit_size = 512; + break; + case CCP_XTS_AES_UNIT_SIZE_1024: + unit_size = 1024; + break; + case CCP_XTS_AES_UNIT_SIZE_2048: + unit_size = 2048; + break; + case CCP_XTS_AES_UNIT_SIZE_4096: + unit_size = 4096; + break; + + default: + return -EINVAL; + } + + if (xts->key_len == AES_KEYSIZE_128) + aestype = CCP_AES_TYPE_128; + else if (xts->key_len == AES_KEYSIZE_256) + aestype = CCP_AES_TYPE_256; + else + return -EINVAL; + + if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1))) + return -EINVAL; + + if (xts->iv_len != AES_BLOCK_SIZE) + return -EINVAL; + + if (!xts->key || !xts->iv || !xts->src || !xts->dst) + return -EINVAL; + + BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1); + BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1); + + ret = -EIO; + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = CCP_NEW_JOBID(cmd_q->ccp); + op.sb_key = cmd_q->sb_key; + op.sb_ctx = cmd_q->sb_ctx; + op.init = 1; + op.u.xts.type = aestype; + op.u.xts.action = xts->action; + op.u.xts.unit_size = xts->unit_size; + + /* A version 3 device only supports 128-bit keys, which fits into a + * single SB entry. A version 5 device uses a 512-bit vector, so two + * SB entries. + */ + if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) + sb_count = CCP_XTS_AES_KEY_SB_COUNT; + else + sb_count = CCP5_XTS_AES_KEY_SB_COUNT; + ret = ccp_init_dm_workarea(&key, cmd_q, + sb_count * CCP_SB_BYTES, + DMA_TO_DEVICE); + if (ret) + return ret; + + if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) { + /* All supported key sizes must be in little endian format. + * Use the 256-bit byte swap passthru option to convert from + * big endian to little endian. + */ + dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128; + ret = ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len); + if (ret) + goto e_key; + ret = ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len); + if (ret) + goto e_key; + } else { + /* Version 5 CCPs use a 512-bit space for the key: each portion + * occupies 256 bits, or one entire slot, and is zero-padded. + */ + unsigned int pad; + + dm_offset = CCP_SB_BYTES; + pad = dm_offset - xts->key_len; + ret = ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len); + if (ret) + goto e_key; + ret = ccp_set_dm_area(&key, dm_offset + pad, xts->key, + xts->key_len, xts->key_len); + if (ret) + goto e_key; + } + ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_key; + } + + /* The AES context fits in a single (32-byte) SB entry and + * for XTS is already in little endian format so no byte swapping + * is needed. + */ + ret = ccp_init_dm_workarea(&ctx, cmd_q, + CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES, + DMA_BIDIRECTIONAL); + if (ret) + goto e_key; + + ret = ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len); + if (ret) + goto e_ctx; + ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, + CCP_PASSTHRU_BYTESWAP_NOOP); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_ctx; + } + + /* Prepare the input and output data workareas. For in-place + * operations we need to set the dma direction to BIDIRECTIONAL + * and copy the src workarea to the dst workarea. + */ + if (sg_virt(xts->src) == sg_virt(xts->dst)) + in_place = true; + + ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len, + unit_size, + in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); + if (ret) + goto e_ctx; + + if (in_place) { + dst = src; + } else { + ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len, + unit_size, DMA_FROM_DEVICE); + if (ret) + goto e_src; + } + + /* Send data to the CCP AES engine */ + while (src.sg_wa.bytes_left) { + ccp_prepare_data(&src, &dst, &op, unit_size, true); + if (!src.sg_wa.bytes_left) + op.eom = 1; + + ret = cmd_q->ccp->vdata->perform->xts_aes(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + ccp_process_data(&src, &dst, &op); + } + + /* Retrieve the AES context - convert from LE to BE using + * 32-byte (256-bit) byteswapping + */ + ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + /* ...but we only need AES_BLOCK_SIZE bytes */ + dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; + ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len); + +e_dst: + if (!in_place) + ccp_free_data(&dst, cmd_q); + +e_src: + ccp_free_data(&src, cmd_q); + +e_ctx: + ccp_dm_free(&ctx); + +e_key: + ccp_dm_free(&key); + + return ret; +} + +static noinline_for_stack int +ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + struct ccp_des3_engine *des3 = &cmd->u.des3; + + struct ccp_dm_workarea key, ctx; + struct ccp_data src, dst; + struct ccp_op op; + unsigned int dm_offset; + unsigned int len_singlekey; + bool in_place = false; + int ret; + + /* Error checks */ + if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) + return -EINVAL; + + if (!cmd_q->ccp->vdata->perform->des3) + return -EINVAL; + + if (des3->key_len != DES3_EDE_KEY_SIZE) + return -EINVAL; + + if (((des3->mode == CCP_DES3_MODE_ECB) || + (des3->mode == CCP_DES3_MODE_CBC)) && + (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1))) + return -EINVAL; + + if (!des3->key || !des3->src || !des3->dst) + return -EINVAL; + + if (des3->mode != CCP_DES3_MODE_ECB) { + if (des3->iv_len != DES3_EDE_BLOCK_SIZE) + return -EINVAL; + + if (!des3->iv) + return -EINVAL; + } + + /* Zero out all the fields of the command desc */ + memset(&op, 0, sizeof(op)); + + /* Set up the Function field */ + op.cmd_q = cmd_q; + op.jobid = CCP_NEW_JOBID(cmd_q->ccp); + op.sb_key = cmd_q->sb_key; + + op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1; + op.u.des3.type = des3->type; + op.u.des3.mode = des3->mode; + op.u.des3.action = des3->action; + + /* + * All supported key sizes fit in a single (32-byte) KSB entry and + * (like AES) must be in little endian format. Use the 256-bit byte + * swap passthru option to convert from big endian to little endian. + */ + ret = ccp_init_dm_workarea(&key, cmd_q, + CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES, + DMA_TO_DEVICE); + if (ret) + return ret; + + /* + * The contents of the key triplet are in the reverse order of what + * is required by the engine. Copy the 3 pieces individually to put + * them where they belong. + */ + dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */ + + len_singlekey = des3->key_len / 3; + ret = ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey, + des3->key, 0, len_singlekey); + if (ret) + goto e_key; + ret = ccp_set_dm_area(&key, dm_offset + len_singlekey, + des3->key, len_singlekey, len_singlekey); + if (ret) + goto e_key; + ret = ccp_set_dm_area(&key, dm_offset, + des3->key, 2 * len_singlekey, len_singlekey); + if (ret) + goto e_key; + + /* Copy the key to the SB */ + ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_key; + } + + /* + * The DES3 context fits in a single (32-byte) KSB entry and + * must be in little endian format. Use the 256-bit byte swap + * passthru option to convert from big endian to little endian. + */ + if (des3->mode != CCP_DES3_MODE_ECB) { + op.sb_ctx = cmd_q->sb_ctx; + + ret = ccp_init_dm_workarea(&ctx, cmd_q, + CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES, + DMA_BIDIRECTIONAL); + if (ret) + goto e_key; + + /* Load the context into the LSB */ + dm_offset = CCP_SB_BYTES - des3->iv_len; + ret = ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0, + des3->iv_len); + if (ret) + goto e_ctx; + + ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_ctx; + } + } + + /* + * Prepare the input and output data workareas. For in-place + * operations we need to set the dma direction to BIDIRECTIONAL + * and copy the src workarea to the dst workarea. + */ + if (sg_virt(des3->src) == sg_virt(des3->dst)) + in_place = true; + + ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len, + DES3_EDE_BLOCK_SIZE, + in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); + if (ret) + goto e_ctx; + + if (in_place) + dst = src; + else { + ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len, + DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE); + if (ret) + goto e_src; + } + + /* Send data to the CCP DES3 engine */ + while (src.sg_wa.bytes_left) { + ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true); + if (!src.sg_wa.bytes_left) { + op.eom = 1; + + /* Since we don't retrieve the context in ECB mode + * we have to wait for the operation to complete + * on the last piece of data + */ + op.soc = 0; + } + + ret = cmd_q->ccp->vdata->perform->des3(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + ccp_process_data(&src, &dst, &op); + } + + if (des3->mode != CCP_DES3_MODE_ECB) { + /* Retrieve the context and make BE */ + ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */ + ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0, + DES3_EDE_BLOCK_SIZE); + } +e_dst: + if (!in_place) + ccp_free_data(&dst, cmd_q); + +e_src: + ccp_free_data(&src, cmd_q); + +e_ctx: + if (des3->mode != CCP_DES3_MODE_ECB) + ccp_dm_free(&ctx); + +e_key: + ccp_dm_free(&key); + + return ret; +} + +static noinline_for_stack int +ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + struct ccp_sha_engine *sha = &cmd->u.sha; + struct ccp_dm_workarea ctx; + struct ccp_data src; + struct ccp_op op; + unsigned int ioffset, ooffset; + unsigned int digest_size; + int sb_count; + const void *init; + u64 block_size; + int ctx_size; + int ret; + + switch (sha->type) { + case CCP_SHA_TYPE_1: + if (sha->ctx_len < SHA1_DIGEST_SIZE) + return -EINVAL; + block_size = SHA1_BLOCK_SIZE; + break; + case CCP_SHA_TYPE_224: + if (sha->ctx_len < SHA224_DIGEST_SIZE) + return -EINVAL; + block_size = SHA224_BLOCK_SIZE; + break; + case CCP_SHA_TYPE_256: + if (sha->ctx_len < SHA256_DIGEST_SIZE) + return -EINVAL; + block_size = SHA256_BLOCK_SIZE; + break; + case CCP_SHA_TYPE_384: + if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0) + || sha->ctx_len < SHA384_DIGEST_SIZE) + return -EINVAL; + block_size = SHA384_BLOCK_SIZE; + break; + case CCP_SHA_TYPE_512: + if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0) + || sha->ctx_len < SHA512_DIGEST_SIZE) + return -EINVAL; + block_size = SHA512_BLOCK_SIZE; + break; + default: + return -EINVAL; + } + + if (!sha->ctx) + return -EINVAL; + + if (!sha->final && (sha->src_len & (block_size - 1))) + return -EINVAL; + + /* The version 3 device can't handle zero-length input */ + if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) { + + if (!sha->src_len) { + unsigned int digest_len; + const u8 *sha_zero; + + /* Not final, just return */ + if (!sha->final) + return 0; + + /* CCP can't do a zero length sha operation so the + * caller must buffer the data. + */ + if (sha->msg_bits) + return -EINVAL; + + /* The CCP cannot perform zero-length sha operations + * so the caller is required to buffer data for the + * final operation. However, a sha operation for a + * message with a total length of zero is valid so + * known values are required to supply the result. + */ + switch (sha->type) { + case CCP_SHA_TYPE_1: + sha_zero = sha1_zero_message_hash; + digest_len = SHA1_DIGEST_SIZE; + break; + case CCP_SHA_TYPE_224: + sha_zero = sha224_zero_message_hash; + digest_len = SHA224_DIGEST_SIZE; + break; + case CCP_SHA_TYPE_256: + sha_zero = sha256_zero_message_hash; + digest_len = SHA256_DIGEST_SIZE; + break; + default: + return -EINVAL; + } + + scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0, + digest_len, 1); + + return 0; + } + } + + /* Set variables used throughout */ + switch (sha->type) { + case CCP_SHA_TYPE_1: + digest_size = SHA1_DIGEST_SIZE; + init = (void *) ccp_sha1_init; + ctx_size = SHA1_DIGEST_SIZE; + sb_count = 1; + if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0)) + ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE; + else + ooffset = ioffset = 0; + break; + case CCP_SHA_TYPE_224: + digest_size = SHA224_DIGEST_SIZE; + init = (void *) ccp_sha224_init; + ctx_size = SHA256_DIGEST_SIZE; + sb_count = 1; + ioffset = 0; + if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0)) + ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE; + else + ooffset = 0; + break; + case CCP_SHA_TYPE_256: + digest_size = SHA256_DIGEST_SIZE; + init = (void *) ccp_sha256_init; + ctx_size = SHA256_DIGEST_SIZE; + sb_count = 1; + ooffset = ioffset = 0; + break; + case CCP_SHA_TYPE_384: + digest_size = SHA384_DIGEST_SIZE; + init = (void *) ccp_sha384_init; + ctx_size = SHA512_DIGEST_SIZE; + sb_count = 2; + ioffset = 0; + ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE; + break; + case CCP_SHA_TYPE_512: + digest_size = SHA512_DIGEST_SIZE; + init = (void *) ccp_sha512_init; + ctx_size = SHA512_DIGEST_SIZE; + sb_count = 2; + ooffset = ioffset = 0; + break; + default: + ret = -EINVAL; + goto e_data; + } + + /* For zero-length plaintext the src pointer is ignored; + * otherwise both parts must be valid + */ + if (sha->src_len && !sha->src) + return -EINVAL; + + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = CCP_NEW_JOBID(cmd_q->ccp); + op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ + op.u.sha.type = sha->type; + op.u.sha.msg_bits = sha->msg_bits; + + /* For SHA1/224/256 the context fits in a single (32-byte) SB entry; + * SHA384/512 require 2 adjacent SB slots, with the right half in the + * first slot, and the left half in the second. Each portion must then + * be in little endian format: use the 256-bit byte swap option. + */ + ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES, + DMA_BIDIRECTIONAL); + if (ret) + return ret; + if (sha->first) { + switch (sha->type) { + case CCP_SHA_TYPE_1: + case CCP_SHA_TYPE_224: + case CCP_SHA_TYPE_256: + memcpy(ctx.address + ioffset, init, ctx_size); + break; + case CCP_SHA_TYPE_384: + case CCP_SHA_TYPE_512: + memcpy(ctx.address + ctx_size / 2, init, + ctx_size / 2); + memcpy(ctx.address, init + ctx_size / 2, + ctx_size / 2); + break; + default: + ret = -EINVAL; + goto e_ctx; + } + } else { + /* Restore the context */ + ret = ccp_set_dm_area(&ctx, 0, sha->ctx, 0, + sb_count * CCP_SB_BYTES); + if (ret) + goto e_ctx; + } + + ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_ctx; + } + + if (sha->src) { + /* Send data to the CCP SHA engine; block_size is set above */ + ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len, + block_size, DMA_TO_DEVICE); + if (ret) + goto e_ctx; + + while (src.sg_wa.bytes_left) { + ccp_prepare_data(&src, NULL, &op, block_size, false); + if (sha->final && !src.sg_wa.bytes_left) + op.eom = 1; + + ret = cmd_q->ccp->vdata->perform->sha(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_data; + } + + ccp_process_data(&src, NULL, &op); + } + } else { + op.eom = 1; + ret = cmd_q->ccp->vdata->perform->sha(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_data; + } + } + + /* Retrieve the SHA context - convert from LE to BE using + * 32-byte (256-bit) byteswapping to BE + */ + ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_data; + } + + if (sha->final) { + /* Finishing up, so get the digest */ + switch (sha->type) { + case CCP_SHA_TYPE_1: + case CCP_SHA_TYPE_224: + case CCP_SHA_TYPE_256: + ccp_get_dm_area(&ctx, ooffset, + sha->ctx, 0, + digest_size); + break; + case CCP_SHA_TYPE_384: + case CCP_SHA_TYPE_512: + ccp_get_dm_area(&ctx, 0, + sha->ctx, LSB_ITEM_SIZE - ooffset, + LSB_ITEM_SIZE); + ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset, + sha->ctx, 0, + LSB_ITEM_SIZE - ooffset); + break; + default: + ret = -EINVAL; + goto e_data; + } + } else { + /* Stash the context */ + ccp_get_dm_area(&ctx, 0, sha->ctx, 0, + sb_count * CCP_SB_BYTES); + } + + if (sha->final && sha->opad) { + /* HMAC operation, recursively perform final SHA */ + struct ccp_cmd hmac_cmd; + struct scatterlist sg; + u8 *hmac_buf; + + if (sha->opad_len != block_size) { + ret = -EINVAL; + goto e_data; + } + + hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL); + if (!hmac_buf) { + ret = -ENOMEM; + goto e_data; + } + sg_init_one(&sg, hmac_buf, block_size + digest_size); + + scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0); + switch (sha->type) { + case CCP_SHA_TYPE_1: + case CCP_SHA_TYPE_224: + case CCP_SHA_TYPE_256: + memcpy(hmac_buf + block_size, + ctx.address + ooffset, + digest_size); + break; + case CCP_SHA_TYPE_384: + case CCP_SHA_TYPE_512: + memcpy(hmac_buf + block_size, + ctx.address + LSB_ITEM_SIZE + ooffset, + LSB_ITEM_SIZE); + memcpy(hmac_buf + block_size + + (LSB_ITEM_SIZE - ooffset), + ctx.address, + LSB_ITEM_SIZE); + break; + default: + kfree(hmac_buf); + ret = -EINVAL; + goto e_data; + } + + memset(&hmac_cmd, 0, sizeof(hmac_cmd)); + hmac_cmd.engine = CCP_ENGINE_SHA; + hmac_cmd.u.sha.type = sha->type; + hmac_cmd.u.sha.ctx = sha->ctx; + hmac_cmd.u.sha.ctx_len = sha->ctx_len; + hmac_cmd.u.sha.src = &sg; + hmac_cmd.u.sha.src_len = block_size + digest_size; + hmac_cmd.u.sha.opad = NULL; + hmac_cmd.u.sha.opad_len = 0; + hmac_cmd.u.sha.first = 1; + hmac_cmd.u.sha.final = 1; + hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3; + + ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd); + if (ret) + cmd->engine_error = hmac_cmd.engine_error; + + kfree(hmac_buf); + } + +e_data: + if (sha->src) + ccp_free_data(&src, cmd_q); + +e_ctx: + ccp_dm_free(&ctx); + + return ret; +} + +static noinline_for_stack int +ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + struct ccp_rsa_engine *rsa = &cmd->u.rsa; + struct ccp_dm_workarea exp, src, dst; + struct ccp_op op; + unsigned int sb_count, i_len, o_len; + int ret; + + /* Check against the maximum allowable size, in bits */ + if (rsa->key_size > cmd_q->ccp->vdata->rsamax) + return -EINVAL; + + if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst) + return -EINVAL; + + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = CCP_NEW_JOBID(cmd_q->ccp); + + /* The RSA modulus must precede the message being acted upon, so + * it must be copied to a DMA area where the message and the + * modulus can be concatenated. Therefore the input buffer + * length required is twice the output buffer length (which + * must be a multiple of 256-bits). Compute o_len, i_len in bytes. + * Buffer sizes must be a multiple of 32 bytes; rounding up may be + * required. + */ + o_len = 32 * ((rsa->key_size + 255) / 256); + i_len = o_len * 2; + + sb_count = 0; + if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) { + /* sb_count is the number of storage block slots required + * for the modulus. + */ + sb_count = o_len / CCP_SB_BYTES; + op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q, + sb_count); + if (!op.sb_key) + return -EIO; + } else { + /* A version 5 device allows a modulus size that will not fit + * in the LSB, so the command will transfer it from memory. + * Set the sb key to the default, even though it's not used. + */ + op.sb_key = cmd_q->sb_key; + } + + /* The RSA exponent must be in little endian format. Reverse its + * byte order. + */ + ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE); + if (ret) + goto e_sb; + + ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len); + if (ret) + goto e_exp; + + if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) { + /* Copy the exponent to the local storage block, using + * as many 32-byte blocks as were allocated above. It's + * already little endian, so no further change is required. + */ + ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key, + CCP_PASSTHRU_BYTESWAP_NOOP); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_exp; + } + } else { + /* The exponent can be retrieved from memory via DMA. */ + op.exp.u.dma.address = exp.dma.address; + op.exp.u.dma.offset = 0; + } + + /* Concatenate the modulus and the message. Both the modulus and + * the operands must be in little endian format. Since the input + * is in big endian format it must be converted. + */ + ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE); + if (ret) + goto e_exp; + + ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len); + if (ret) + goto e_src; + ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len); + if (ret) + goto e_src; + + /* Prepare the output area for the operation */ + ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE); + if (ret) + goto e_src; + + op.soc = 1; + op.src.u.dma.address = src.dma.address; + op.src.u.dma.offset = 0; + op.src.u.dma.length = i_len; + op.dst.u.dma.address = dst.dma.address; + op.dst.u.dma.offset = 0; + op.dst.u.dma.length = o_len; + + op.u.rsa.mod_size = rsa->key_size; + op.u.rsa.input_len = i_len; + + ret = cmd_q->ccp->vdata->perform->rsa(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len); + +e_dst: + ccp_dm_free(&dst); + +e_src: + ccp_dm_free(&src); + +e_exp: + ccp_dm_free(&exp); + +e_sb: + if (sb_count) + cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count); + + return ret; +} + +static noinline_for_stack int +ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + struct ccp_passthru_engine *pt = &cmd->u.passthru; + struct ccp_dm_workarea mask; + struct ccp_data src, dst; + struct ccp_op op; + bool in_place = false; + unsigned int i; + int ret = 0; + + if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) + return -EINVAL; + + if (!pt->src || !pt->dst) + return -EINVAL; + + if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { + if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) + return -EINVAL; + if (!pt->mask) + return -EINVAL; + } + + BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1); + + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = CCP_NEW_JOBID(cmd_q->ccp); + + if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { + /* Load the mask */ + op.sb_key = cmd_q->sb_key; + + ret = ccp_init_dm_workarea(&mask, cmd_q, + CCP_PASSTHRU_SB_COUNT * + CCP_SB_BYTES, + DMA_TO_DEVICE); + if (ret) + return ret; + + ret = ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len); + if (ret) + goto e_mask; + ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key, + CCP_PASSTHRU_BYTESWAP_NOOP); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_mask; + } + } + + /* Prepare the input and output data workareas. For in-place + * operations we need to set the dma direction to BIDIRECTIONAL + * and copy the src workarea to the dst workarea. + */ + if (sg_virt(pt->src) == sg_virt(pt->dst)) + in_place = true; + + ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len, + CCP_PASSTHRU_MASKSIZE, + in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); + if (ret) + goto e_mask; + + if (in_place) { + dst = src; + } else { + ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len, + CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE); + if (ret) + goto e_src; + } + + /* Send data to the CCP Passthru engine + * Because the CCP engine works on a single source and destination + * dma address at a time, each entry in the source scatterlist + * (after the dma_map_sg call) must be less than or equal to the + * (remaining) length in the destination scatterlist entry and the + * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE + */ + dst.sg_wa.sg_used = 0; + for (i = 1; i <= src.sg_wa.dma_count; i++) { + if (!dst.sg_wa.sg || + (sg_dma_len(dst.sg_wa.sg) < sg_dma_len(src.sg_wa.sg))) { + ret = -EINVAL; + goto e_dst; + } + + if (i == src.sg_wa.dma_count) { + op.eom = 1; + op.soc = 1; + } + + op.src.type = CCP_MEMTYPE_SYSTEM; + op.src.u.dma.address = sg_dma_address(src.sg_wa.sg); + op.src.u.dma.offset = 0; + op.src.u.dma.length = sg_dma_len(src.sg_wa.sg); + + op.dst.type = CCP_MEMTYPE_SYSTEM; + op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg); + op.dst.u.dma.offset = dst.sg_wa.sg_used; + op.dst.u.dma.length = op.src.u.dma.length; + + ret = cmd_q->ccp->vdata->perform->passthru(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + dst.sg_wa.sg_used += sg_dma_len(src.sg_wa.sg); + if (dst.sg_wa.sg_used == sg_dma_len(dst.sg_wa.sg)) { + dst.sg_wa.sg = sg_next(dst.sg_wa.sg); + dst.sg_wa.sg_used = 0; + } + src.sg_wa.sg = sg_next(src.sg_wa.sg); + } + +e_dst: + if (!in_place) + ccp_free_data(&dst, cmd_q); + +e_src: + ccp_free_data(&src, cmd_q); + +e_mask: + if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) + ccp_dm_free(&mask); + + return ret; +} + +static noinline_for_stack int +ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q, + struct ccp_cmd *cmd) +{ + struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap; + struct ccp_dm_workarea mask; + struct ccp_op op; + int ret; + + if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) + return -EINVAL; + + if (!pt->src_dma || !pt->dst_dma) + return -EINVAL; + + if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { + if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) + return -EINVAL; + if (!pt->mask) + return -EINVAL; + } + + BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1); + + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = CCP_NEW_JOBID(cmd_q->ccp); + + if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { + /* Load the mask */ + op.sb_key = cmd_q->sb_key; + + mask.length = pt->mask_len; + mask.dma.address = pt->mask; + mask.dma.length = pt->mask_len; + + ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key, + CCP_PASSTHRU_BYTESWAP_NOOP); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + return ret; + } + } + + /* Send data to the CCP Passthru engine */ + op.eom = 1; + op.soc = 1; + + op.src.type = CCP_MEMTYPE_SYSTEM; + op.src.u.dma.address = pt->src_dma; + op.src.u.dma.offset = 0; + op.src.u.dma.length = pt->src_len; + + op.dst.type = CCP_MEMTYPE_SYSTEM; + op.dst.u.dma.address = pt->dst_dma; + op.dst.u.dma.offset = 0; + op.dst.u.dma.length = pt->src_len; + + ret = cmd_q->ccp->vdata->perform->passthru(&op); + if (ret) + cmd->engine_error = cmd_q->cmd_error; + + return ret; +} + +static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + struct ccp_ecc_engine *ecc = &cmd->u.ecc; + struct ccp_dm_workarea src, dst; + struct ccp_op op; + int ret; + u8 *save; + + if (!ecc->u.mm.operand_1 || + (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + + if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) + if (!ecc->u.mm.operand_2 || + (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + + if (!ecc->u.mm.result || + (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = CCP_NEW_JOBID(cmd_q->ccp); + + /* Concatenate the modulus and the operands. Both the modulus and + * the operands must be in little endian format. Since the input + * is in big endian format it must be converted and placed in a + * fixed length buffer. + */ + ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, + DMA_TO_DEVICE); + if (ret) + return ret; + + /* Save the workarea address since it is updated in order to perform + * the concatenation + */ + save = src.address; + + /* Copy the ECC modulus */ + ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len); + if (ret) + goto e_src; + src.address += CCP_ECC_OPERAND_SIZE; + + /* Copy the first operand */ + ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0, + ecc->u.mm.operand_1_len); + if (ret) + goto e_src; + src.address += CCP_ECC_OPERAND_SIZE; + + if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) { + /* Copy the second operand */ + ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0, + ecc->u.mm.operand_2_len); + if (ret) + goto e_src; + src.address += CCP_ECC_OPERAND_SIZE; + } + + /* Restore the workarea address */ + src.address = save; + + /* Prepare the output area for the operation */ + ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, + DMA_FROM_DEVICE); + if (ret) + goto e_src; + + op.soc = 1; + op.src.u.dma.address = src.dma.address; + op.src.u.dma.offset = 0; + op.src.u.dma.length = src.length; + op.dst.u.dma.address = dst.dma.address; + op.dst.u.dma.offset = 0; + op.dst.u.dma.length = dst.length; + + op.u.ecc.function = cmd->u.ecc.function; + + ret = cmd_q->ccp->vdata->perform->ecc(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + ecc->ecc_result = le16_to_cpup( + (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); + if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { + ret = -EIO; + goto e_dst; + } + + /* Save the ECC result */ + ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0, + CCP_ECC_MODULUS_BYTES); + +e_dst: + ccp_dm_free(&dst); + +e_src: + ccp_dm_free(&src); + + return ret; +} + +static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + struct ccp_ecc_engine *ecc = &cmd->u.ecc; + struct ccp_dm_workarea src, dst; + struct ccp_op op; + int ret; + u8 *save; + + if (!ecc->u.pm.point_1.x || + (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) || + !ecc->u.pm.point_1.y || + (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + + if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { + if (!ecc->u.pm.point_2.x || + (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) || + !ecc->u.pm.point_2.y || + (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + } else { + if (!ecc->u.pm.domain_a || + (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + + if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) + if (!ecc->u.pm.scalar || + (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + } + + if (!ecc->u.pm.result.x || + (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) || + !ecc->u.pm.result.y || + (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = CCP_NEW_JOBID(cmd_q->ccp); + + /* Concatenate the modulus and the operands. Both the modulus and + * the operands must be in little endian format. Since the input + * is in big endian format it must be converted and placed in a + * fixed length buffer. + */ + ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, + DMA_TO_DEVICE); + if (ret) + return ret; + + /* Save the workarea address since it is updated in order to perform + * the concatenation + */ + save = src.address; + + /* Copy the ECC modulus */ + ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len); + if (ret) + goto e_src; + src.address += CCP_ECC_OPERAND_SIZE; + + /* Copy the first point X and Y coordinate */ + ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0, + ecc->u.pm.point_1.x_len); + if (ret) + goto e_src; + src.address += CCP_ECC_OPERAND_SIZE; + ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0, + ecc->u.pm.point_1.y_len); + if (ret) + goto e_src; + src.address += CCP_ECC_OPERAND_SIZE; + + /* Set the first point Z coordinate to 1 */ + *src.address = 0x01; + src.address += CCP_ECC_OPERAND_SIZE; + + if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { + /* Copy the second point X and Y coordinate */ + ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0, + ecc->u.pm.point_2.x_len); + if (ret) + goto e_src; + src.address += CCP_ECC_OPERAND_SIZE; + ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0, + ecc->u.pm.point_2.y_len); + if (ret) + goto e_src; + src.address += CCP_ECC_OPERAND_SIZE; + + /* Set the second point Z coordinate to 1 */ + *src.address = 0x01; + src.address += CCP_ECC_OPERAND_SIZE; + } else { + /* Copy the Domain "a" parameter */ + ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0, + ecc->u.pm.domain_a_len); + if (ret) + goto e_src; + src.address += CCP_ECC_OPERAND_SIZE; + + if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) { + /* Copy the scalar value */ + ret = ccp_reverse_set_dm_area(&src, 0, + ecc->u.pm.scalar, 0, + ecc->u.pm.scalar_len); + if (ret) + goto e_src; + src.address += CCP_ECC_OPERAND_SIZE; + } + } + + /* Restore the workarea address */ + src.address = save; + + /* Prepare the output area for the operation */ + ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, + DMA_FROM_DEVICE); + if (ret) + goto e_src; + + op.soc = 1; + op.src.u.dma.address = src.dma.address; + op.src.u.dma.offset = 0; + op.src.u.dma.length = src.length; + op.dst.u.dma.address = dst.dma.address; + op.dst.u.dma.offset = 0; + op.dst.u.dma.length = dst.length; + + op.u.ecc.function = cmd->u.ecc.function; + + ret = cmd_q->ccp->vdata->perform->ecc(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + ecc->ecc_result = le16_to_cpup( + (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); + if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { + ret = -EIO; + goto e_dst; + } + + /* Save the workarea address since it is updated as we walk through + * to copy the point math result + */ + save = dst.address; + + /* Save the ECC result X and Y coordinates */ + ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0, + CCP_ECC_MODULUS_BYTES); + dst.address += CCP_ECC_OUTPUT_SIZE; + ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0, + CCP_ECC_MODULUS_BYTES); + + /* Restore the workarea address */ + dst.address = save; + +e_dst: + ccp_dm_free(&dst); + +e_src: + ccp_dm_free(&src); + + return ret; +} + +static noinline_for_stack int +ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + struct ccp_ecc_engine *ecc = &cmd->u.ecc; + + ecc->ecc_result = 0; + + if (!ecc->mod || + (ecc->mod_len > CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + + switch (ecc->function) { + case CCP_ECC_FUNCTION_MMUL_384BIT: + case CCP_ECC_FUNCTION_MADD_384BIT: + case CCP_ECC_FUNCTION_MINV_384BIT: + return ccp_run_ecc_mm_cmd(cmd_q, cmd); + + case CCP_ECC_FUNCTION_PADD_384BIT: + case CCP_ECC_FUNCTION_PMUL_384BIT: + case CCP_ECC_FUNCTION_PDBL_384BIT: + return ccp_run_ecc_pm_cmd(cmd_q, cmd); + + default: + return -EINVAL; + } +} + +int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + int ret; + + cmd->engine_error = 0; + cmd_q->cmd_error = 0; + cmd_q->int_rcvd = 0; + cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q); + + switch (cmd->engine) { + case CCP_ENGINE_AES: + switch (cmd->u.aes.mode) { + case CCP_AES_MODE_CMAC: + ret = ccp_run_aes_cmac_cmd(cmd_q, cmd); + break; + case CCP_AES_MODE_GCM: + ret = ccp_run_aes_gcm_cmd(cmd_q, cmd); + break; + default: + ret = ccp_run_aes_cmd(cmd_q, cmd); + break; + } + break; + case CCP_ENGINE_XTS_AES_128: + ret = ccp_run_xts_aes_cmd(cmd_q, cmd); + break; + case CCP_ENGINE_DES3: + ret = ccp_run_des3_cmd(cmd_q, cmd); + break; + case CCP_ENGINE_SHA: + ret = ccp_run_sha_cmd(cmd_q, cmd); + break; + case CCP_ENGINE_RSA: + ret = ccp_run_rsa_cmd(cmd_q, cmd); + break; + case CCP_ENGINE_PASSTHRU: + if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP) + ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd); + else + ret = ccp_run_passthru_cmd(cmd_q, cmd); + break; + case CCP_ENGINE_ECC: + ret = ccp_run_ecc_cmd(cmd_q, cmd); + break; + default: + ret = -EINVAL; + } + + return ret; +} diff --git a/drivers/crypto/ccp/dbc.c b/drivers/crypto/ccp/dbc.c new file mode 100644 index 000000000..410084a90 --- /dev/null +++ b/drivers/crypto/ccp/dbc.c @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Secure Processor Dynamic Boost Control interface + * + * Copyright (C) 2023 Advanced Micro Devices, Inc. + * + * Author: Mario Limonciello + */ + +#include + +#include "dbc.h" + +#define DBC_DEFAULT_TIMEOUT (10 * MSEC_PER_SEC) +struct error_map { + u32 psp; + int ret; +}; + +#define DBC_ERROR_ACCESS_DENIED 0x0001 +#define DBC_ERROR_EXCESS_DATA 0x0004 +#define DBC_ERROR_BAD_PARAMETERS 0x0006 +#define DBC_ERROR_BAD_STATE 0x0007 +#define DBC_ERROR_NOT_IMPLEMENTED 0x0009 +#define DBC_ERROR_BUSY 0x000D +#define DBC_ERROR_MESSAGE_FAILURE 0x0307 +#define DBC_ERROR_OVERFLOW 0x300F +#define DBC_ERROR_SIGNATURE_INVALID 0x3072 + +static struct error_map error_codes[] = { + {DBC_ERROR_ACCESS_DENIED, -EACCES}, + {DBC_ERROR_EXCESS_DATA, -E2BIG}, + {DBC_ERROR_BAD_PARAMETERS, -EINVAL}, + {DBC_ERROR_BAD_STATE, -EAGAIN}, + {DBC_ERROR_MESSAGE_FAILURE, -ENOENT}, + {DBC_ERROR_NOT_IMPLEMENTED, -ENOENT}, + {DBC_ERROR_BUSY, -EBUSY}, + {DBC_ERROR_OVERFLOW, -ENFILE}, + {DBC_ERROR_SIGNATURE_INVALID, -EPERM}, + {0x0, 0x0}, +}; + +static inline int send_dbc_cmd_thru_ext(struct psp_dbc_device *dbc_dev, int msg) +{ + dbc_dev->mbox->ext_req.header.sub_cmd_id = msg; + + return psp_extended_mailbox_cmd(dbc_dev->psp, + DBC_DEFAULT_TIMEOUT, + (struct psp_ext_request *)dbc_dev->mbox); +} + +static inline int send_dbc_cmd_thru_pa(struct psp_dbc_device *dbc_dev, int msg) +{ + return psp_send_platform_access_msg(msg, + (struct psp_request *)dbc_dev->mbox); +} + +static int send_dbc_cmd(struct psp_dbc_device *dbc_dev, int msg) +{ + int ret; + + *dbc_dev->result = 0; + ret = dbc_dev->use_ext ? send_dbc_cmd_thru_ext(dbc_dev, msg) : + send_dbc_cmd_thru_pa(dbc_dev, msg); + if (ret == -EIO) { + int i; + + dev_dbg(dbc_dev->dev, + "msg 0x%x failed with PSP error: 0x%x\n", + msg, *dbc_dev->result); + + for (i = 0; error_codes[i].psp; i++) { + if (*dbc_dev->result == error_codes[i].psp) + return error_codes[i].ret; + } + } + + return ret; +} + +static int send_dbc_nonce(struct psp_dbc_device *dbc_dev) +{ + int ret; + + *dbc_dev->payload_size = dbc_dev->header_size + sizeof(struct dbc_user_nonce); + ret = send_dbc_cmd(dbc_dev, PSP_DYNAMIC_BOOST_GET_NONCE); + if (ret == -EAGAIN) { + dev_dbg(dbc_dev->dev, "retrying get nonce\n"); + ret = send_dbc_cmd(dbc_dev, PSP_DYNAMIC_BOOST_GET_NONCE); + } + + return ret; +} + +static int send_dbc_parameter(struct psp_dbc_device *dbc_dev) +{ + struct dbc_user_param *user_param = (struct dbc_user_param *)dbc_dev->payload; + + switch (user_param->msg_index) { + case PARAM_SET_FMAX_CAP: + case PARAM_SET_PWR_CAP: + case PARAM_SET_GFX_MODE: + return send_dbc_cmd(dbc_dev, PSP_DYNAMIC_BOOST_SET_PARAMETER); + case PARAM_GET_FMAX_CAP: + case PARAM_GET_PWR_CAP: + case PARAM_GET_CURR_TEMP: + case PARAM_GET_FMAX_MAX: + case PARAM_GET_FMAX_MIN: + case PARAM_GET_SOC_PWR_MAX: + case PARAM_GET_SOC_PWR_MIN: + case PARAM_GET_SOC_PWR_CUR: + case PARAM_GET_GFX_MODE: + return send_dbc_cmd(dbc_dev, PSP_DYNAMIC_BOOST_GET_PARAMETER); + } + + return -EINVAL; +} + +void dbc_dev_destroy(struct psp_device *psp) +{ + struct psp_dbc_device *dbc_dev = psp->dbc_data; + + if (!dbc_dev) + return; + + misc_deregister(&dbc_dev->char_dev); + mutex_destroy(&dbc_dev->ioctl_mutex); + psp->dbc_data = NULL; +} + +static long dbc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct psp_device *psp_master = psp_get_master_device(); + void __user *argp = (void __user *)arg; + struct psp_dbc_device *dbc_dev; + int ret; + + if (!psp_master || !psp_master->dbc_data) + return -ENODEV; + dbc_dev = psp_master->dbc_data; + + guard(mutex)(&dbc_dev->ioctl_mutex); + + switch (cmd) { + case DBCIOCNONCE: + if (copy_from_user(dbc_dev->payload, argp, sizeof(struct dbc_user_nonce))) + return -EFAULT; + + ret = send_dbc_nonce(dbc_dev); + if (ret) + return ret; + + if (copy_to_user(argp, dbc_dev->payload, sizeof(struct dbc_user_nonce))) + return -EFAULT; + break; + case DBCIOCUID: + if (copy_from_user(dbc_dev->payload, argp, sizeof(struct dbc_user_setuid))) + return -EFAULT; + + *dbc_dev->payload_size = dbc_dev->header_size + sizeof(struct dbc_user_setuid); + ret = send_dbc_cmd(dbc_dev, PSP_DYNAMIC_BOOST_SET_UID); + if (ret) + return ret; + + if (copy_to_user(argp, dbc_dev->payload, sizeof(struct dbc_user_setuid))) + return -EFAULT; + break; + case DBCIOCPARAM: + if (copy_from_user(dbc_dev->payload, argp, sizeof(struct dbc_user_param))) + return -EFAULT; + + *dbc_dev->payload_size = dbc_dev->header_size + sizeof(struct dbc_user_param); + ret = send_dbc_parameter(dbc_dev); + if (ret) + return ret; + + if (copy_to_user(argp, dbc_dev->payload, sizeof(struct dbc_user_param))) + return -EFAULT; + break; + default: + return -EINVAL; + } + + return 0; +} + +static const struct file_operations dbc_fops = { + .owner = THIS_MODULE, + .unlocked_ioctl = dbc_ioctl, +}; + +int dbc_dev_init(struct psp_device *psp) +{ + struct device *dev = psp->dev; + struct psp_dbc_device *dbc_dev; + int ret; + + dbc_dev = devm_kzalloc(dev, sizeof(*dbc_dev), GFP_KERNEL); + if (!dbc_dev) + return -ENOMEM; + + BUILD_BUG_ON(sizeof(union dbc_buffer) > PAGE_SIZE); + dbc_dev->mbox = (void *)devm_get_free_pages(dev, GFP_KERNEL | __GFP_ZERO, 0); + if (!dbc_dev->mbox) { + ret = -ENOMEM; + goto cleanup_dev; + } + + psp->dbc_data = dbc_dev; + dbc_dev->dev = dev; + dbc_dev->psp = psp; + + if (psp->capability.dbc_thru_ext) { + dbc_dev->use_ext = true; + dbc_dev->payload_size = &dbc_dev->mbox->ext_req.header.payload_size; + dbc_dev->result = &dbc_dev->mbox->ext_req.header.status; + dbc_dev->payload = &dbc_dev->mbox->ext_req.buf; + dbc_dev->header_size = sizeof(struct psp_ext_req_buffer_hdr); + } else { + dbc_dev->payload_size = &dbc_dev->mbox->pa_req.header.payload_size; + dbc_dev->result = &dbc_dev->mbox->pa_req.header.status; + dbc_dev->payload = &dbc_dev->mbox->pa_req.buf; + dbc_dev->header_size = sizeof(struct psp_req_buffer_hdr); + } + + ret = send_dbc_nonce(dbc_dev); + if (ret == -EACCES) { + dev_dbg(dbc_dev->dev, + "dynamic boost control was previously authenticated\n"); + ret = 0; + } + dev_dbg(dbc_dev->dev, "dynamic boost control is %savailable\n", + ret ? "un" : ""); + if (ret) { + ret = 0; + goto cleanup_mbox; + } + + dbc_dev->char_dev.minor = MISC_DYNAMIC_MINOR; + dbc_dev->char_dev.name = "dbc"; + dbc_dev->char_dev.fops = &dbc_fops; + dbc_dev->char_dev.mode = 0600; + ret = misc_register(&dbc_dev->char_dev); + if (ret) + goto cleanup_mbox; + + mutex_init(&dbc_dev->ioctl_mutex); + + return 0; + +cleanup_mbox: + devm_free_pages(dev, (unsigned long)dbc_dev->mbox); + +cleanup_dev: + psp->dbc_data = NULL; + devm_kfree(dev, dbc_dev); + + return ret; +} diff --git a/drivers/crypto/ccp/dbc.h b/drivers/crypto/ccp/dbc.h new file mode 100644 index 000000000..e0fecbe92 --- /dev/null +++ b/drivers/crypto/ccp/dbc.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * AMD Platform Security Processor (PSP) Dynamic Boost Control support + * + * Copyright (C) 2023 Advanced Micro Devices, Inc. + * + * Author: Mario Limonciello + */ + +#ifndef __DBC_H__ +#define __DBC_H__ + +#include + +#include +#include +#include + +#include "psp-dev.h" + +struct psp_dbc_device { + struct device *dev; + struct psp_device *psp; + + union dbc_buffer *mbox; + + struct mutex ioctl_mutex; + + struct miscdevice char_dev; + + /* used to abstract communication path */ + bool use_ext; + u32 header_size; + u32 *payload_size; + u32 *result; + void *payload; +}; + +union dbc_buffer { + struct psp_request pa_req; + struct psp_ext_request ext_req; +}; + +void dbc_dev_destroy(struct psp_device *psp); +int dbc_dev_init(struct psp_device *psp); + +#endif /* __DBC_H */ diff --git a/drivers/crypto/ccp/hsti.c b/drivers/crypto/ccp/hsti.c new file mode 100644 index 000000000..df09e0322 --- /dev/null +++ b/drivers/crypto/ccp/hsti.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Secure Processor device driver, security attributes + * + * Copyright (C) 2023-2024 Advanced Micro Devices, Inc. + * + * Author: Mario Limonciello + */ + +#include + +#include "psp-dev.h" +#include "hsti.h" + +#define PSP_CAPABILITY_PSP_SECURITY_OFFSET 8 + +struct hsti_request { + struct psp_req_buffer_hdr header; + u32 hsti; +} __packed; + +#define security_attribute_show(name) \ +static ssize_t name##_show(struct device *d, struct device_attribute *attr, \ + char *buf) \ +{ \ + struct sp_device *sp = dev_get_drvdata(d); \ + struct psp_device *psp = sp->psp_data; \ + return sysfs_emit(buf, "%d\n", psp->capability.name); \ +} + +security_attribute_show(fused_part) +static DEVICE_ATTR_RO(fused_part); +security_attribute_show(boot_integrity) +static DEVICE_ATTR_RO(boot_integrity); +security_attribute_show(debug_lock_on) +static DEVICE_ATTR_RO(debug_lock_on); +security_attribute_show(tsme_status) +static DEVICE_ATTR_RO(tsme_status); +security_attribute_show(anti_rollback_status) +static DEVICE_ATTR_RO(anti_rollback_status); +security_attribute_show(rpmc_production_enabled) +static DEVICE_ATTR_RO(rpmc_production_enabled); +security_attribute_show(rpmc_spirom_available) +static DEVICE_ATTR_RO(rpmc_spirom_available); +security_attribute_show(hsp_tpm_available) +static DEVICE_ATTR_RO(hsp_tpm_available); +security_attribute_show(rom_armor_enforced) +static DEVICE_ATTR_RO(rom_armor_enforced); + +static struct attribute *psp_security_attrs[] = { + &dev_attr_fused_part.attr, + &dev_attr_boot_integrity.attr, + &dev_attr_debug_lock_on.attr, + &dev_attr_tsme_status.attr, + &dev_attr_anti_rollback_status.attr, + &dev_attr_rpmc_production_enabled.attr, + &dev_attr_rpmc_spirom_available.attr, + &dev_attr_hsp_tpm_available.attr, + &dev_attr_rom_armor_enforced.attr, + NULL +}; + +static umode_t psp_security_is_visible(struct kobject *kobj, struct attribute *attr, int idx) +{ + struct device *dev = kobj_to_dev(kobj); + struct sp_device *sp = dev_get_drvdata(dev); + struct psp_device *psp = sp->psp_data; + + if (psp && psp->capability.security_reporting) + return 0444; + + return 0; +} + +struct attribute_group psp_security_attr_group = { + .attrs = psp_security_attrs, + .is_visible = psp_security_is_visible, +}; + +static int psp_populate_hsti(struct psp_device *psp) +{ + struct hsti_request *req; + int ret; + + /* Are the security attributes already reported? */ + if (psp->capability.security_reporting) + return 0; + + /* Allocate command-response buffer */ + req = kzalloc_obj(*req); + if (!req) + return -ENOMEM; + + req->header.payload_size = sizeof(*req); + + ret = psp_send_platform_access_msg(PSP_CMD_HSTI_QUERY, (struct psp_request *)req); + if (ret) + goto out; + + if (req->header.status != 0) { + dev_dbg(psp->dev, "failed to populate HSTI state: %d\n", req->header.status); + ret = -EINVAL; + goto out; + } + + psp->capability.security_reporting = 1; + psp->capability.raw |= req->hsti << PSP_CAPABILITY_PSP_SECURITY_OFFSET; + +out: + kfree(req); + + return ret; +} + +int psp_init_hsti(struct psp_device *psp) +{ + int ret; + + if (PSP_FEATURE(psp, HSTI)) { + ret = psp_populate_hsti(psp); + if (ret) + return ret; + } + + /* + * At this stage, if security information hasn't been populated by + * either the PSP or by the driver through the platform command, + * then there is nothing more to do. + */ + if (!psp->capability.security_reporting) + return 0; + + if (psp->capability.tsme_status) { + if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) + dev_notice(psp->dev, "psp: Both TSME and SME are active, SME is unnecessary when TSME is active.\n"); + else + dev_notice(psp->dev, "psp: TSME enabled\n"); + } + + return 0; +} diff --git a/drivers/crypto/ccp/hsti.h b/drivers/crypto/ccp/hsti.h new file mode 100644 index 000000000..6a70f922d --- /dev/null +++ b/drivers/crypto/ccp/hsti.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * AMD Secure Processor device driver, security attributes + * + * Copyright (C) 2023-2024 Advanced Micro Devices, Inc. + * + * Author: Mario Limonciello + */ + +#ifndef __HSTI_H +#define __HSTI_H + +extern struct attribute_group psp_security_attr_group; + +int psp_init_hsti(struct psp_device *psp); + +#endif /* __HSTI_H */ diff --git a/drivers/crypto/ccp/platform-access.c b/drivers/crypto/ccp/platform-access.c new file mode 100644 index 000000000..1b8ed3389 --- /dev/null +++ b/drivers/crypto/ccp/platform-access.c @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * AMD Platform Security Processor (PSP) Platform Access interface + * + * Copyright (C) 2023 Advanced Micro Devices, Inc. + * + * Author: Mario Limonciello + * + * Some of this code is adapted from drivers/i2c/busses/i2c-designware-amdpsp.c + * developed by Jan Dabros and Copyright (C) 2022 Google Inc. + * + */ + +#include +#include +#include +#include + +#include "platform-access.h" + +#define PSP_CMD_TIMEOUT_US (500 * USEC_PER_MSEC) +#define DOORBELL_CMDRESP_STS GENMASK(7, 0) + +/* Recovery field should be equal 0 to start sending commands */ +static int check_recovery(u32 __iomem *cmd) +{ + return FIELD_GET(PSP_CMDRESP_RECOVERY, ioread32(cmd)); +} + +static int wait_cmd(u32 __iomem *cmd) +{ + u32 tmp, expected; + + /* Expect mbox_cmd to be cleared and ready bit to be set by PSP */ + expected = FIELD_PREP(PSP_CMDRESP_RESP, 1); + + /* + * Check for readiness of PSP mailbox in a tight loop in order to + * process further as soon as command was consumed. + */ + return readl_poll_timeout(cmd, tmp, (tmp & expected), 0, + PSP_CMD_TIMEOUT_US); +} + +int psp_check_platform_access_status(void) +{ + struct psp_device *psp = psp_get_master_device(); + + if (!psp || !psp->platform_access_data) + return -ENODEV; + + return 0; +} +EXPORT_SYMBOL(psp_check_platform_access_status); + +int psp_send_platform_access_msg(enum psp_platform_access_msg msg, + struct psp_request *req) +{ + struct psp_device *psp = psp_get_master_device(); + u32 __iomem *cmd, *lo, *hi; + struct psp_platform_access_device *pa_dev; + phys_addr_t req_addr; + u32 cmd_reg; + int ret; + + if (!psp || !psp->platform_access_data) + return -ENODEV; + + pa_dev = psp->platform_access_data; + + if (!pa_dev->vdata->cmdresp_reg || !pa_dev->vdata->cmdbuff_addr_lo_reg || + !pa_dev->vdata->cmdbuff_addr_hi_reg) + return -ENODEV; + + cmd = psp->io_regs + pa_dev->vdata->cmdresp_reg; + lo = psp->io_regs + pa_dev->vdata->cmdbuff_addr_lo_reg; + hi = psp->io_regs + pa_dev->vdata->cmdbuff_addr_hi_reg; + + mutex_lock(&pa_dev->mailbox_mutex); + + if (check_recovery(cmd)) { + dev_dbg(psp->dev, "platform mailbox is in recovery\n"); + ret = -EBUSY; + goto unlock; + } + + if (wait_cmd(cmd)) { + dev_dbg(psp->dev, "platform mailbox is not done processing command\n"); + ret = -EBUSY; + goto unlock; + } + + /* + * Fill mailbox with address of command-response buffer, which will be + * used for sending i2c requests as well as reading status returned by + * PSP. Use physical address of buffer, since PSP will map this region. + */ + req_addr = __psp_pa(req); + iowrite32(lower_32_bits(req_addr), lo); + iowrite32(upper_32_bits(req_addr), hi); + + print_hex_dump_debug("->psp ", DUMP_PREFIX_OFFSET, 16, 2, req, + req->header.payload_size, false); + + /* Write command register to trigger processing */ + cmd_reg = FIELD_PREP(PSP_CMDRESP_CMD, msg); + iowrite32(cmd_reg, cmd); + + if (wait_cmd(cmd)) { + ret = -ETIMEDOUT; + goto unlock; + } + + /* Ensure it was triggered by this driver */ + if (ioread32(lo) != lower_32_bits(req_addr) || + ioread32(hi) != upper_32_bits(req_addr)) { + ret = -EBUSY; + goto unlock; + } + + /* + * Read status from PSP. If status is non-zero, it indicates an error + * occurred during "processing" of the command. + * If status is zero, it indicates the command was "processed" + * successfully, but the result of the command is in the payload. + * Return both cases to the caller as -EIO to investigate. + */ + cmd_reg = ioread32(cmd); + if (FIELD_GET(PSP_CMDRESP_STS, cmd_reg)) + req->header.status = FIELD_GET(PSP_CMDRESP_STS, cmd_reg); + if (req->header.status) { + ret = -EIO; + goto unlock; + } + + print_hex_dump_debug("<-psp ", DUMP_PREFIX_OFFSET, 16, 2, req, + req->header.payload_size, false); + + ret = 0; + +unlock: + mutex_unlock(&pa_dev->mailbox_mutex); + + return ret; +} +EXPORT_SYMBOL_GPL(psp_send_platform_access_msg); + +int psp_ring_platform_doorbell(int msg, u32 *result) +{ + struct psp_device *psp = psp_get_master_device(); + struct psp_platform_access_device *pa_dev; + u32 __iomem *button, *cmd; + int ret, val; + + if (!psp || !psp->platform_access_data) + return -ENODEV; + + pa_dev = psp->platform_access_data; + button = psp->io_regs + pa_dev->vdata->doorbell_button_reg; + cmd = psp->io_regs + pa_dev->vdata->doorbell_cmd_reg; + + mutex_lock(&pa_dev->doorbell_mutex); + + if (wait_cmd(cmd)) { + dev_err(psp->dev, "doorbell command not done processing\n"); + ret = -EBUSY; + goto unlock; + } + + iowrite32(FIELD_PREP(DOORBELL_CMDRESP_STS, msg), cmd); + iowrite32(PSP_DRBL_RING, button); + + if (wait_cmd(cmd)) { + ret = -ETIMEDOUT; + goto unlock; + } + + val = FIELD_GET(DOORBELL_CMDRESP_STS, ioread32(cmd)); + if (val) { + if (result) + *result = val; + ret = -EIO; + goto unlock; + } + + ret = 0; +unlock: + mutex_unlock(&pa_dev->doorbell_mutex); + + return ret; +} +EXPORT_SYMBOL_GPL(psp_ring_platform_doorbell); + +void platform_access_dev_destroy(struct psp_device *psp) +{ + struct psp_platform_access_device *pa_dev = psp->platform_access_data; + + if (!pa_dev) + return; + + mutex_destroy(&pa_dev->mailbox_mutex); + mutex_destroy(&pa_dev->doorbell_mutex); + psp->platform_access_data = NULL; +} + +int platform_access_dev_init(struct psp_device *psp) +{ + struct device *dev = psp->dev; + struct psp_platform_access_device *pa_dev; + + pa_dev = devm_kzalloc(dev, sizeof(*pa_dev), GFP_KERNEL); + if (!pa_dev) + return -ENOMEM; + + psp->platform_access_data = pa_dev; + pa_dev->psp = psp; + pa_dev->dev = dev; + + pa_dev->vdata = (struct platform_access_vdata *)psp->vdata->platform_access; + + mutex_init(&pa_dev->mailbox_mutex); + mutex_init(&pa_dev->doorbell_mutex); + + dev_dbg(dev, "platform access enabled\n"); + + return 0; +} diff --git a/drivers/crypto/ccp/platform-access.h b/drivers/crypto/ccp/platform-access.h new file mode 100644 index 000000000..a83f03beb --- /dev/null +++ b/drivers/crypto/ccp/platform-access.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * AMD Platform Security Processor (PSP) Platform Access interface + * + * Copyright (C) 2023 Advanced Micro Devices, Inc. + * + * Author: Mario Limonciello + */ + +#ifndef __PSP_PLATFORM_ACCESS_H__ +#define __PSP_PLATFORM_ACCESS_H__ + +#include +#include +#include +#include + +#include "psp-dev.h" + +struct psp_platform_access_device { + struct device *dev; + struct psp_device *psp; + + struct platform_access_vdata *vdata; + + struct mutex mailbox_mutex; + struct mutex doorbell_mutex; + + void *platform_access_data; +}; + +void platform_access_dev_destroy(struct psp_device *psp); +int platform_access_dev_init(struct psp_device *psp); + +#endif /* __PSP_PLATFORM_ACCESS_H__ */ diff --git a/drivers/crypto/ccp/psp-dev.c b/drivers/crypto/ccp/psp-dev.c new file mode 100644 index 000000000..b14ce5106 --- /dev/null +++ b/drivers/crypto/ccp/psp-dev.c @@ -0,0 +1,381 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Platform Security Processor (PSP) interface + * + * Copyright (C) 2016,2019 Advanced Micro Devices, Inc. + * + * Author: Brijesh Singh + */ + +#include +#include +#include +#include +#include + +#include "sp-dev.h" +#include "psp-dev.h" +#include "sev-dev.h" +#include "tee-dev.h" +#include "sfs.h" +#include "platform-access.h" +#include "dbc.h" +#include "hsti.h" + +struct psp_device *psp_master; + +#define PSP_C2PMSG_17_CMDRESP_CMD GENMASK(19, 16) + +static int psp_mailbox_poll(const void __iomem *cmdresp_reg, unsigned int *cmdresp, + unsigned int timeout_msecs) +{ + while (true) { + *cmdresp = ioread32(cmdresp_reg); + if (FIELD_GET(PSP_CMDRESP_RESP, *cmdresp)) + return 0; + + if (!timeout_msecs--) + break; + + usleep_range(1000, 1100); + } + + return -ETIMEDOUT; +} + +int psp_mailbox_command(struct psp_device *psp, enum psp_cmd cmd, void *cmdbuff, + unsigned int timeout_msecs, unsigned int *cmdresp) +{ + void __iomem *cmdresp_reg, *cmdbuff_lo_reg, *cmdbuff_hi_reg; + int ret; + + if (!psp || !psp->vdata || !psp->vdata->cmdresp_reg || + !psp->vdata->cmdbuff_addr_lo_reg || !psp->vdata->cmdbuff_addr_hi_reg) + return -ENODEV; + + cmdresp_reg = psp->io_regs + psp->vdata->cmdresp_reg; + cmdbuff_lo_reg = psp->io_regs + psp->vdata->cmdbuff_addr_lo_reg; + cmdbuff_hi_reg = psp->io_regs + psp->vdata->cmdbuff_addr_hi_reg; + + mutex_lock(&psp->mailbox_mutex); + + /* Ensure mailbox is ready for a command */ + ret = -EBUSY; + if (psp_mailbox_poll(cmdresp_reg, cmdresp, 0)) + goto unlock; + + if (cmdbuff) { + iowrite32(lower_32_bits(__psp_pa(cmdbuff)), cmdbuff_lo_reg); + iowrite32(upper_32_bits(__psp_pa(cmdbuff)), cmdbuff_hi_reg); + } + + *cmdresp = FIELD_PREP(PSP_C2PMSG_17_CMDRESP_CMD, cmd); + iowrite32(*cmdresp, cmdresp_reg); + + ret = psp_mailbox_poll(cmdresp_reg, cmdresp, timeout_msecs); + +unlock: + mutex_unlock(&psp->mailbox_mutex); + + return ret; +} + +int psp_extended_mailbox_cmd(struct psp_device *psp, unsigned int timeout_msecs, + struct psp_ext_request *req) +{ + unsigned int reg; + int ret; + + print_hex_dump_debug("->psp ", DUMP_PREFIX_OFFSET, 16, 2, req, + req->header.payload_size, false); + + ret = psp_mailbox_command(psp, PSP_CMD_TEE_EXTENDED_CMD, (void *)req, + timeout_msecs, ®); + if (ret) { + return ret; + } else if (FIELD_GET(PSP_CMDRESP_STS, reg)) { + req->header.status = FIELD_GET(PSP_CMDRESP_STS, reg); + return -EIO; + } + + print_hex_dump_debug("<-psp ", DUMP_PREFIX_OFFSET, 16, 2, req, + req->header.payload_size, false); + + return 0; +} + +static struct psp_device *psp_alloc_struct(struct sp_device *sp) +{ + struct device *dev = sp->dev; + struct psp_device *psp; + + psp = devm_kzalloc(dev, sizeof(*psp), GFP_KERNEL); + if (!psp) + return NULL; + + psp->dev = dev; + psp->sp = sp; + + snprintf(psp->name, sizeof(psp->name), "psp-%u", sp->ord); + + return psp; +} + +static irqreturn_t psp_irq_handler(int irq, void *data) +{ + struct psp_device *psp = data; + unsigned int status; + + /* Read the interrupt status: */ + status = ioread32(psp->io_regs + psp->vdata->intsts_reg); + + /* Clear the interrupt status by writing the same value we read. */ + iowrite32(status, psp->io_regs + psp->vdata->intsts_reg); + + /* invoke subdevice interrupt handlers */ + if (status) { + if (psp->sev_irq_handler) + psp->sev_irq_handler(irq, psp->sev_irq_data, status); + } + + return IRQ_HANDLED; +} + +static unsigned int psp_get_capability(struct psp_device *psp) +{ + unsigned int val = ioread32(psp->io_regs + psp->vdata->feature_reg); + + /* + * Check for a access to the registers. If this read returns + * 0xffffffff, it's likely that the system is running a broken + * BIOS which disallows access to the device. Stop here and + * fail the PSP initialization (but not the load, as the CCP + * could get properly initialized). + */ + if (val == 0xffffffff) { + dev_notice(psp->dev, "psp: unable to access the device: you might be running a broken BIOS.\n"); + return -ENODEV; + } + psp->capability.raw = val; + + return 0; +} + +static int psp_check_sev_support(struct psp_device *psp) +{ + /* Check if device supports SEV feature */ + if (!psp->capability.sev) { + dev_dbg(psp->dev, "psp does not support SEV\n"); + return -ENODEV; + } + + return 0; +} + +static int psp_check_tee_support(struct psp_device *psp) +{ + /* Check if device supports TEE feature */ + if (!psp->capability.tee) { + dev_dbg(psp->dev, "psp does not support TEE\n"); + return -ENODEV; + } + + return 0; +} + +static int psp_check_sfs_support(struct psp_device *psp) +{ + /* Check if device supports SFS feature */ + if (!psp->capability.sfs) { + dev_dbg(psp->dev, "psp does not support SFS\n"); + return -ENODEV; + } + + return 0; +} + +static int psp_init(struct psp_device *psp) +{ + int ret; + + if (!psp_check_sev_support(psp)) { + ret = sev_dev_init(psp); + if (ret) + return ret; + } + + if (!psp_check_tee_support(psp)) { + ret = tee_dev_init(psp); + if (ret) + return ret; + } + + if (!psp_check_sfs_support(psp)) { + ret = sfs_dev_init(psp); + if (ret) + return ret; + } + + if (psp->vdata->platform_access) { + ret = platform_access_dev_init(psp); + if (ret) + return ret; + } + + /* dbc must come after platform access as it tests the feature */ + if (PSP_FEATURE(psp, DBC) || + psp->capability.dbc_thru_ext) { + ret = dbc_dev_init(psp); + if (ret) + return ret; + } + + /* HSTI uses platform access on some systems. */ + ret = psp_init_hsti(psp); + if (ret) + return ret; + + return 0; +} + +int psp_dev_init(struct sp_device *sp) +{ + struct device *dev = sp->dev; + struct psp_device *psp; + int ret; + + ret = -ENOMEM; + psp = psp_alloc_struct(sp); + if (!psp) + goto e_err; + + sp->psp_data = psp; + + psp->vdata = (struct psp_vdata *)sp->dev_vdata->psp_vdata; + if (!psp->vdata) { + ret = -ENODEV; + dev_err(dev, "missing driver data\n"); + goto e_err; + } + + psp->io_regs = sp->io_map; + mutex_init(&psp->mailbox_mutex); + + ret = psp_get_capability(psp); + if (ret) + goto e_disable; + + /* Disable and clear interrupts until ready */ + iowrite32(0, psp->io_regs + psp->vdata->inten_reg); + iowrite32(-1, psp->io_regs + psp->vdata->intsts_reg); + + /* Request an irq */ + ret = sp_request_psp_irq(psp->sp, psp_irq_handler, psp->name, psp); + if (ret) { + dev_err(dev, "psp: unable to allocate an IRQ\n"); + goto e_err; + } + + /* master device must be set for platform access */ + if (psp->sp->set_psp_master_device) + psp->sp->set_psp_master_device(psp->sp); + + ret = psp_init(psp); + if (ret) + goto e_irq; + + /* Enable interrupt */ + iowrite32(-1, psp->io_regs + psp->vdata->inten_reg); + + dev_notice(dev, "psp enabled\n"); + + return 0; + +e_irq: + if (sp->clear_psp_master_device) + sp->clear_psp_master_device(sp); + + sp_free_psp_irq(psp->sp, psp); +e_err: + sp->psp_data = NULL; + + dev_notice(dev, "psp initialization failed\n"); + + return ret; + +e_disable: + sp->psp_data = NULL; + + return ret; +} + +void psp_dev_destroy(struct sp_device *sp) +{ + struct psp_device *psp = sp->psp_data; + + if (!psp) + return; + + dbc_dev_destroy(psp); + + platform_access_dev_destroy(psp); + + sfs_dev_destroy(psp); + + tee_dev_destroy(psp); + + sev_dev_destroy(psp); + + sp_free_psp_irq(sp, psp); + + if (sp->clear_psp_master_device) + sp->clear_psp_master_device(sp); +} + +void psp_set_sev_irq_handler(struct psp_device *psp, psp_irq_handler_t handler, + void *data) +{ + psp->sev_irq_data = data; + psp->sev_irq_handler = handler; +} + +void psp_clear_sev_irq_handler(struct psp_device *psp) +{ + psp_set_sev_irq_handler(psp, NULL, NULL); +} + +struct psp_device *psp_get_master_device(void) +{ + struct sp_device *sp = sp_get_psp_master_device(); + + return sp ? sp->psp_data : NULL; +} + +int psp_restore(struct sp_device *sp) +{ + struct psp_device *psp = sp->psp_data; + int ret = 0; + + if (psp->tee_data) + ret = tee_restore(psp); + + return ret; +} + +void psp_pci_init(void) +{ + psp_master = psp_get_master_device(); + + if (!psp_master) + return; + + sev_pci_init(); +} + +void psp_pci_exit(void) +{ + if (!psp_master) + return; + + sev_pci_exit(); +} diff --git a/drivers/crypto/ccp/psp-dev.h b/drivers/crypto/ccp/psp-dev.h new file mode 100644 index 000000000..4e370e76b --- /dev/null +++ b/drivers/crypto/ccp/psp-dev.h @@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * AMD Platform Security Processor (PSP) interface driver + * + * Copyright (C) 2017-2019 Advanced Micro Devices, Inc. + * + * Author: Brijesh Singh + */ + +#ifndef __PSP_DEV_H__ +#define __PSP_DEV_H__ + +#include +#include +#include +#include +#include +#include +#include + +#include "sp-dev.h" + +#define MAX_PSP_NAME_LEN 16 + +extern struct psp_device *psp_master; + +typedef void (*psp_irq_handler_t)(int, void *, unsigned int); + +union psp_cap_register { + unsigned int raw; + struct { + unsigned int sev :1, + tee :1, + dbc_thru_ext :1, + sfs :1, + rsvd1 :3, + security_reporting :1, + fused_part :1, + boot_integrity :1, + debug_lock_on :1, + rsvd3 :2, + tsme_status :1, + rsvd4 :1, + anti_rollback_status :1, + rpmc_production_enabled :1, + rpmc_spirom_available :1, + hsp_tpm_available :1, + rom_armor_enforced :1, + rsvd5 :12; + }; +}; + +struct psp_device { + struct list_head entry; + + struct psp_vdata *vdata; + char name[MAX_PSP_NAME_LEN]; + + struct device *dev; + struct sp_device *sp; + + void __iomem *io_regs; + struct mutex mailbox_mutex; + + psp_irq_handler_t sev_irq_handler; + void *sev_irq_data; + + void *sev_data; + void *tee_data; + void *platform_access_data; + void *dbc_data; + void *sfs_data; + + union psp_cap_register capability; +}; + +void psp_set_sev_irq_handler(struct psp_device *psp, psp_irq_handler_t handler, + void *data); +void psp_clear_sev_irq_handler(struct psp_device *psp); + +struct psp_device *psp_get_master_device(void); + +/** + * enum psp_cmd - PSP mailbox commands + * @PSP_CMD_TEE_RING_INIT: Initialize TEE ring buffer + * @PSP_CMD_TEE_RING_DESTROY: Destroy TEE ring buffer + * @PSP_CMD_TEE_EXTENDED_CMD: Extended command + * @PSP_CMD_MAX: Maximum command id + */ +enum psp_cmd { + PSP_CMD_TEE_RING_INIT = 1, + PSP_CMD_TEE_RING_DESTROY = 2, + PSP_CMD_TEE_EXTENDED_CMD = 14, + PSP_CMD_MAX = 15, +}; + +int psp_mailbox_command(struct psp_device *psp, enum psp_cmd cmd, void *cmdbuff, + unsigned int timeout_msecs, unsigned int *cmdresp); + +/** + * struct psp_ext_req_buffer_hdr - Structure of the extended command header + * @payload_size: total payload size + * @sub_cmd_id: extended command ID + * @status: status of command execution (out) + */ +struct psp_ext_req_buffer_hdr { + u32 payload_size; + u32 sub_cmd_id; + u32 status; +} __packed; + +struct psp_ext_request { + struct psp_ext_req_buffer_hdr header; + void *buf; +} __packed; + +/** + * enum psp_sub_cmd - PSP mailbox sub commands + * @PSP_SUB_CMD_DBC_GET_NONCE: Get nonce from DBC + * @PSP_SUB_CMD_DBC_SET_UID: Set UID for DBC + * @PSP_SUB_CMD_DBC_GET_PARAMETER: Get parameter from DBC + * @PSP_SUB_CMD_DBC_SET_PARAMETER: Set parameter for DBC + * @PSP_SUB_CMD_SFS_GET_FW_VERS: Get firmware versions for ASP and other MP + * @PSP_SUB_CMD_SFS_UPDATE: Command to load, verify and execute SFS package + */ +enum psp_sub_cmd { + PSP_SUB_CMD_DBC_GET_NONCE = PSP_DYNAMIC_BOOST_GET_NONCE, + PSP_SUB_CMD_DBC_SET_UID = PSP_DYNAMIC_BOOST_SET_UID, + PSP_SUB_CMD_DBC_GET_PARAMETER = PSP_DYNAMIC_BOOST_GET_PARAMETER, + PSP_SUB_CMD_DBC_SET_PARAMETER = PSP_DYNAMIC_BOOST_SET_PARAMETER, + PSP_SUB_CMD_SFS_GET_FW_VERS = PSP_SFS_GET_FW_VERSIONS, + PSP_SUB_CMD_SFS_UPDATE = PSP_SFS_UPDATE, +}; + +int psp_extended_mailbox_cmd(struct psp_device *psp, unsigned int timeout_msecs, + struct psp_ext_request *req); +#endif /* __PSP_DEV_H */ diff --git a/drivers/crypto/ccp/sev-dev-tio.c b/drivers/crypto/ccp/sev-dev-tio.c new file mode 100644 index 000000000..c84e23982 --- /dev/null +++ b/drivers/crypto/ccp/sev-dev-tio.c @@ -0,0 +1,864 @@ +// SPDX-License-Identifier: GPL-2.0-only + +// Interface to PSP for CCP/SEV-TIO/SNP-VM + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "sev-dev.h" +#include "sev-dev-tio.h" + +#define to_tio_status(dev_data) \ + (container_of((dev_data), struct tio_dsm, data)->sev->tio_status) + +#define SLA_PAGE_TYPE_DATA 0 +#define SLA_PAGE_TYPE_SCATTER 1 +#define SLA_PAGE_SIZE_4K 0 +#define SLA_PAGE_SIZE_2M 1 +#define SLA_SZ(s) ((s).page_size == SLA_PAGE_SIZE_2M ? SZ_2M : SZ_4K) +#define SLA_SCATTER_LEN(s) (SLA_SZ(s) / sizeof(struct sla_addr_t)) +#define SLA_EOL ((struct sla_addr_t) { .pfn = ((1UL << 40) - 1) }) +#define SLA_NULL ((struct sla_addr_t) { 0 }) +#define IS_SLA_NULL(s) ((s).sla == SLA_NULL.sla) +#define IS_SLA_EOL(s) ((s).sla == SLA_EOL.sla) + +static phys_addr_t sla_to_pa(struct sla_addr_t sla) +{ + u64 pfn = sla.pfn; + u64 pa = pfn << PAGE_SHIFT; + + return pa; +} + +static void *sla_to_va(struct sla_addr_t sla) +{ + void *va = __va(__sme_clr(sla_to_pa(sla))); + + return va; +} + +#define sla_to_pfn(sla) (__pa(sla_to_va(sla)) >> PAGE_SHIFT) +#define sla_to_page(sla) virt_to_page(sla_to_va(sla)) + +static struct sla_addr_t make_sla(struct page *pg, bool stp) +{ + u64 pa = __sme_set(page_to_phys(pg)); + struct sla_addr_t ret = { + .pfn = pa >> PAGE_SHIFT, + .page_size = SLA_PAGE_SIZE_4K, /* Do not do SLA_PAGE_SIZE_2M ATM */ + .page_type = stp ? SLA_PAGE_TYPE_SCATTER : SLA_PAGE_TYPE_DATA + }; + + return ret; +} + +/* the BUFFER Structure */ +#define SLA_BUFFER_FLAG_ENCRYPTION BIT(0) + +/* + * struct sla_buffer_hdr - Scatter list address buffer header + * + * @capacity_sz: Total capacity of the buffer in bytes + * @payload_sz: Size of buffer payload in bytes, must be multiple of 32B + * @flags: Buffer flags (SLA_BUFFER_FLAG_ENCRYPTION: buffer is encrypted) + * @iv: Initialization vector used for encryption + * @authtag: Authentication tag for encrypted buffer + */ +struct sla_buffer_hdr { + u32 capacity_sz; + u32 payload_sz; /* The size of BUFFER_PAYLOAD in bytes. Must be multiple of 32B */ + u32 flags; + u8 reserved1[4]; + u8 iv[16]; /* IV used for the encryption of this buffer */ + u8 authtag[16]; /* Authentication tag for this buffer */ + u8 reserved2[16]; +} __packed; + +enum spdm_data_type_t { + DOBJ_DATA_TYPE_SPDM = 0x1, + DOBJ_DATA_TYPE_SECURE_SPDM = 0x2, +}; + +struct spdm_dobj_hdr_req { + struct spdm_dobj_hdr hdr; /* hdr.id == SPDM_DOBJ_ID_REQ */ + u8 data_type; /* spdm_data_type_t */ + u8 reserved2[5]; +} __packed; + +struct spdm_dobj_hdr_resp { + struct spdm_dobj_hdr hdr; /* hdr.id == SPDM_DOBJ_ID_RESP */ + u8 data_type; /* spdm_data_type_t */ + u8 reserved2[5]; +} __packed; + +/* Defined in sev-dev-tio.h so sev-dev-tsm.c can read types of blobs */ +struct spdm_dobj_hdr_cert; +struct spdm_dobj_hdr_meas; +struct spdm_dobj_hdr_report; + +/* Used in all SPDM-aware TIO commands */ +struct spdm_ctrl { + struct sla_addr_t req; + struct sla_addr_t resp; + struct sla_addr_t scratch; + struct sla_addr_t output; +} __packed; + +static size_t sla_dobj_id_to_size(u8 id) +{ + size_t n; + + BUILD_BUG_ON(sizeof(struct spdm_dobj_hdr_resp) != 0x10); + switch (id) { + case SPDM_DOBJ_ID_REQ: + n = sizeof(struct spdm_dobj_hdr_req); + break; + case SPDM_DOBJ_ID_RESP: + n = sizeof(struct spdm_dobj_hdr_resp); + break; + default: + WARN_ON(1); + n = 0; + break; + } + + return n; +} + +#define SPDM_DOBJ_HDR_SIZE(hdr) sla_dobj_id_to_size((hdr)->id) +#define SPDM_DOBJ_DATA(hdr) ((u8 *)(hdr) + SPDM_DOBJ_HDR_SIZE(hdr)) +#define SPDM_DOBJ_LEN(hdr) ((hdr)->length - SPDM_DOBJ_HDR_SIZE(hdr)) + +#define sla_to_dobj_resp_hdr(buf) ((struct spdm_dobj_hdr_resp *) \ + sla_to_dobj_hdr_check((buf), SPDM_DOBJ_ID_RESP)) +#define sla_to_dobj_req_hdr(buf) ((struct spdm_dobj_hdr_req *) \ + sla_to_dobj_hdr_check((buf), SPDM_DOBJ_ID_REQ)) + +static struct spdm_dobj_hdr *sla_to_dobj_hdr(struct sla_buffer_hdr *buf) +{ + if (!buf) + return NULL; + + return (struct spdm_dobj_hdr *) &buf[1]; +} + +static struct spdm_dobj_hdr *sla_to_dobj_hdr_check(struct sla_buffer_hdr *buf, u32 check_dobjid) +{ + struct spdm_dobj_hdr *hdr = sla_to_dobj_hdr(buf); + + if (WARN_ON_ONCE(!hdr)) + return NULL; + + if (hdr->id != check_dobjid) { + pr_err("! ERROR: expected %d, found %d\n", check_dobjid, hdr->id); + return NULL; + } + + return hdr; +} + +static void *sla_to_data(struct sla_buffer_hdr *buf, u32 dobjid) +{ + struct spdm_dobj_hdr *hdr = sla_to_dobj_hdr(buf); + + if (WARN_ON_ONCE(dobjid != SPDM_DOBJ_ID_REQ && dobjid != SPDM_DOBJ_ID_RESP)) + return NULL; + + if (!hdr) + return NULL; + + return (u8 *) hdr + sla_dobj_id_to_size(dobjid); +} + +/* + * struct sev_data_tio_status - SEV_CMD_TIO_STATUS command + * + * @length: Length of this command buffer in bytes + * @status_paddr: System physical address of the TIO_STATUS structure + */ +struct sev_data_tio_status { + u32 length; + u8 reserved[4]; + u64 status_paddr; +} __packed; + +/* TIO_INIT */ +struct sev_data_tio_init { + u32 length; + u8 reserved[12]; +} __packed; + +/* + * struct sev_data_tio_dev_create - TIO_DEV_CREATE command + * + * @length: Length in bytes of this command buffer + * @dev_ctx_sla: Scatter list address pointing to a buffer to be used as a device context buffer + * @device_id: PCIe Routing Identifier of the device to connect to + * @root_port_id: PCIe Routing Identifier of the root port of the device + * @segment_id: PCIe Segment Identifier of the device to connect to + */ +struct sev_data_tio_dev_create { + u32 length; + u8 reserved1[4]; + struct sla_addr_t dev_ctx_sla; + u16 device_id; + u16 root_port_id; + u8 segment_id; + u8 reserved2[11]; +} __packed; + +/* + * struct sev_data_tio_dev_connect - TIO_DEV_CONNECT command + * + * @length: Length in bytes of this command buffer + * @spdm_ctrl: SPDM control structure defined in Section 5.1 + * @dev_ctx_sla: Scatter list address of the device context buffer + * @tc_mask: Bitmask of the traffic classes to initialize for SEV-TIO usage. + * Setting the kth bit of the TC_MASK to 1 indicates that the traffic + * class k will be initialized + * @cert_slot: Slot number of the certificate requested for constructing the SPDM session + * @ide_stream_id: IDE stream IDs to be associated with this device. + * Valid only if corresponding bit in TC_MASK is set + */ +struct sev_data_tio_dev_connect { + u32 length; + u8 reserved1[4]; + struct spdm_ctrl spdm_ctrl; + u8 reserved2[8]; + struct sla_addr_t dev_ctx_sla; + u8 tc_mask; + u8 cert_slot; + u8 reserved3[6]; + u8 ide_stream_id[8]; + u8 reserved4[8]; +} __packed; + +/* + * struct sev_data_tio_dev_disconnect - TIO_DEV_DISCONNECT command + * + * @length: Length in bytes of this command buffer + * @flags: Command flags (TIO_DEV_DISCONNECT_FLAG_FORCE: force disconnect) + * @spdm_ctrl: SPDM control structure defined in Section 5.1 + * @dev_ctx_sla: Scatter list address of the device context buffer + */ +#define TIO_DEV_DISCONNECT_FLAG_FORCE BIT(0) + +struct sev_data_tio_dev_disconnect { + u32 length; + u32 flags; + struct spdm_ctrl spdm_ctrl; + struct sla_addr_t dev_ctx_sla; +} __packed; + +/* + * struct sev_data_tio_dev_meas - TIO_DEV_MEASUREMENTS command + * + * @length: Length in bytes of this command buffer + * @flags: Command flags (TIO_DEV_MEAS_FLAG_RAW_BITSTREAM: request raw measurements) + * @spdm_ctrl: SPDM control structure defined in Section 5.1 + * @dev_ctx_sla: Scatter list address of the device context buffer + * @meas_nonce: Nonce for measurement freshness verification + */ +#define TIO_DEV_MEAS_FLAG_RAW_BITSTREAM BIT(0) + +struct sev_data_tio_dev_meas { + u32 length; + u32 flags; + struct spdm_ctrl spdm_ctrl; + struct sla_addr_t dev_ctx_sla; + u8 meas_nonce[32]; +} __packed; + +/* + * struct sev_data_tio_dev_certs - TIO_DEV_CERTIFICATES command + * + * @length: Length in bytes of this command buffer + * @spdm_ctrl: SPDM control structure defined in Section 5.1 + * @dev_ctx_sla: Scatter list address of the device context buffer + */ +struct sev_data_tio_dev_certs { + u32 length; + u8 reserved[4]; + struct spdm_ctrl spdm_ctrl; + struct sla_addr_t dev_ctx_sla; +} __packed; + +/* + * struct sev_data_tio_dev_reclaim - TIO_DEV_RECLAIM command + * + * @length: Length in bytes of this command buffer + * @dev_ctx_sla: Scatter list address of the device context buffer + * + * This command reclaims resources associated with a device context. + */ +struct sev_data_tio_dev_reclaim { + u32 length; + u8 reserved[4]; + struct sla_addr_t dev_ctx_sla; +} __packed; + +static struct sla_buffer_hdr *sla_buffer_map(struct sla_addr_t sla) +{ + struct sla_buffer_hdr *buf; + + BUILD_BUG_ON(sizeof(struct sla_buffer_hdr) != 0x40); + if (IS_SLA_NULL(sla)) + return NULL; + + if (sla.page_type == SLA_PAGE_TYPE_SCATTER) { + struct sla_addr_t *scatter = sla_to_va(sla); + unsigned int i, npages = 0; + + for (i = 0; i < SLA_SCATTER_LEN(sla); ++i) { + if (WARN_ON_ONCE(SLA_SZ(scatter[i]) > SZ_4K)) + return NULL; + + if (WARN_ON_ONCE(scatter[i].page_type == SLA_PAGE_TYPE_SCATTER)) + return NULL; + + if (IS_SLA_EOL(scatter[i])) { + npages = i; + break; + } + } + if (WARN_ON_ONCE(!npages)) + return NULL; + + struct page **pp = kmalloc_objs(pp[0], npages); + + if (!pp) + return NULL; + + for (i = 0; i < npages; ++i) + pp[i] = sla_to_page(scatter[i]); + + buf = vm_map_ram(pp, npages, 0); + kfree(pp); + } else { + struct page *pg = sla_to_page(sla); + + buf = vm_map_ram(&pg, 1, 0); + } + + return buf; +} + +static void sla_buffer_unmap(struct sla_addr_t sla, struct sla_buffer_hdr *buf) +{ + if (!buf) + return; + + if (sla.page_type == SLA_PAGE_TYPE_SCATTER) { + struct sla_addr_t *scatter = sla_to_va(sla); + unsigned int i, npages = 0; + + for (i = 0; i < SLA_SCATTER_LEN(sla); ++i) { + if (IS_SLA_EOL(scatter[i])) { + npages = i; + break; + } + } + if (!npages) + return; + + vm_unmap_ram(buf, npages); + } else { + vm_unmap_ram(buf, 1); + } +} + +static void dobj_response_init(struct sla_buffer_hdr *buf) +{ + struct spdm_dobj_hdr *dobj = sla_to_dobj_hdr(buf); + + dobj->id = SPDM_DOBJ_ID_RESP; + dobj->version.major = 0x1; + dobj->version.minor = 0; + dobj->length = 0; + buf->payload_sz = sla_dobj_id_to_size(dobj->id) + dobj->length; +} + +static void sla_free(struct sla_addr_t sla, size_t len, bool firmware_state) +{ + unsigned int npages = PAGE_ALIGN(len) >> PAGE_SHIFT; + struct sla_addr_t *scatter = NULL; + int ret = 0, i; + + if (IS_SLA_NULL(sla)) + return; + + if (firmware_state) { + if (sla.page_type == SLA_PAGE_TYPE_SCATTER) { + scatter = sla_to_va(sla); + + for (i = 0; i < npages; ++i) { + if (IS_SLA_EOL(scatter[i])) + break; + + ret = snp_reclaim_pages(sla_to_pa(scatter[i]), 1, false); + if (ret) + break; + } + } else { + ret = snp_reclaim_pages(sla_to_pa(sla), 1, false); + } + } + + if (WARN_ON(ret)) + return; + + if (scatter) { + for (i = 0; i < npages; ++i) { + if (IS_SLA_EOL(scatter[i])) + break; + free_page((unsigned long)sla_to_va(scatter[i])); + } + } + + free_page((unsigned long)sla_to_va(sla)); +} + +static struct sla_addr_t sla_alloc(size_t len, bool firmware_state) +{ + unsigned long i, npages = PAGE_ALIGN(len) >> PAGE_SHIFT; + struct sla_addr_t *scatter = NULL; + struct sla_addr_t ret = SLA_NULL; + struct sla_buffer_hdr *buf; + struct page *pg; + + if (npages == 0) + return ret; + + if (WARN_ON_ONCE(npages > ((PAGE_SIZE / sizeof(struct sla_addr_t)) + 1))) + return ret; + + BUILD_BUG_ON(PAGE_SIZE < SZ_4K); + + if (npages > 1) { + pg = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!pg) + return SLA_NULL; + + ret = make_sla(pg, true); + scatter = page_to_virt(pg); + for (i = 0; i < npages; ++i) { + pg = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!pg) + goto no_reclaim_exit; + + scatter[i] = make_sla(pg, false); + } + scatter[i] = SLA_EOL; + } else { + pg = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!pg) + return SLA_NULL; + + ret = make_sla(pg, false); + } + + buf = sla_buffer_map(ret); + if (!buf) + goto no_reclaim_exit; + + buf->capacity_sz = (npages << PAGE_SHIFT); + sla_buffer_unmap(ret, buf); + + if (firmware_state) { + if (scatter) { + for (i = 0; i < npages; ++i) { + if (rmp_make_private(sla_to_pfn(scatter[i]), 0, + PG_LEVEL_4K, 0, true)) + goto free_exit; + } + } else { + if (rmp_make_private(sla_to_pfn(ret), 0, PG_LEVEL_4K, 0, true)) + goto no_reclaim_exit; + } + } + + return ret; + +no_reclaim_exit: + firmware_state = false; +free_exit: + sla_free(ret, len, firmware_state); + return SLA_NULL; +} + +/* Expands a buffer, only firmware owned buffers allowed for now */ +static int sla_expand(struct sla_addr_t *sla, size_t *len) +{ + struct sla_buffer_hdr *oldbuf = sla_buffer_map(*sla), *newbuf; + struct sla_addr_t oldsla = *sla, newsla; + size_t oldlen = *len, newlen; + + if (!oldbuf) + return -EFAULT; + + newlen = oldbuf->capacity_sz; + if (oldbuf->capacity_sz == oldlen) { + /* This buffer does not require expansion, must be another buffer */ + sla_buffer_unmap(oldsla, oldbuf); + return 1; + } + + pr_notice("Expanding BUFFER from %ld to %ld bytes\n", oldlen, newlen); + + newsla = sla_alloc(newlen, true); + if (IS_SLA_NULL(newsla)) + return -ENOMEM; + + newbuf = sla_buffer_map(newsla); + if (!newbuf) { + sla_free(newsla, newlen, true); + return -EFAULT; + } + + memcpy(newbuf, oldbuf, oldlen); + + sla_buffer_unmap(newsla, newbuf); + sla_free(oldsla, oldlen, true); + *sla = newsla; + *len = newlen; + + return 0; +} + +static int sev_tio_do_cmd(int cmd, void *data, size_t data_len, int *psp_ret, + struct tsm_dsm_tio *dev_data) +{ + int rc; + + *psp_ret = 0; + rc = sev_do_cmd(cmd, data, psp_ret); + + if (WARN_ON(!rc && *psp_ret == SEV_RET_SPDM_REQUEST)) + return -EIO; + + if (rc == 0 && *psp_ret == SEV_RET_EXPAND_BUFFER_LENGTH_REQUEST) { + int rc1, rc2; + + rc1 = sla_expand(&dev_data->output, &dev_data->output_len); + if (rc1 < 0) + return rc1; + + rc2 = sla_expand(&dev_data->scratch, &dev_data->scratch_len); + if (rc2 < 0) + return rc2; + + if (!rc1 && !rc2) + /* Neither buffer requires expansion, this is wrong */ + return -EFAULT; + + *psp_ret = 0; + rc = sev_do_cmd(cmd, data, psp_ret); + } + + if ((rc == 0 || rc == -EIO) && *psp_ret == SEV_RET_SPDM_REQUEST) { + struct spdm_dobj_hdr_resp *resp_hdr; + struct spdm_dobj_hdr_req *req_hdr; + struct sev_tio_status *tio_status = to_tio_status(dev_data); + size_t resp_len = tio_status->spdm_req_size_max - + (sla_dobj_id_to_size(SPDM_DOBJ_ID_RESP) + sizeof(struct sla_buffer_hdr)); + + if (!dev_data->cmd) { + if (WARN_ON_ONCE(!data_len || (data_len != *(u32 *) data))) + return -EINVAL; + if (WARN_ON(data_len > sizeof(dev_data->cmd_data))) + return -EFAULT; + memcpy(dev_data->cmd_data, data, data_len); + memset(&dev_data->cmd_data[data_len], 0xFF, + sizeof(dev_data->cmd_data) - data_len); + dev_data->cmd = cmd; + } + + req_hdr = sla_to_dobj_req_hdr(dev_data->reqbuf); + resp_hdr = sla_to_dobj_resp_hdr(dev_data->respbuf); + switch (req_hdr->data_type) { + case DOBJ_DATA_TYPE_SPDM: + rc = PCI_DOE_FEATURE_CMA; + break; + case DOBJ_DATA_TYPE_SECURE_SPDM: + rc = PCI_DOE_FEATURE_SSESSION; + break; + default: + return -EINVAL; + } + resp_hdr->data_type = req_hdr->data_type; + dev_data->spdm.req_len = req_hdr->hdr.length - + sla_dobj_id_to_size(SPDM_DOBJ_ID_REQ); + dev_data->spdm.rsp_len = resp_len; + } else if (dev_data && dev_data->cmd) { + /* For either error or success just stop the bouncing */ + memset(dev_data->cmd_data, 0, sizeof(dev_data->cmd_data)); + dev_data->cmd = 0; + } + + return rc; +} + +int sev_tio_continue(struct tsm_dsm_tio *dev_data) +{ + struct spdm_dobj_hdr_resp *resp_hdr; + int ret; + + if (!dev_data || !dev_data->cmd) + return -EINVAL; + + resp_hdr = sla_to_dobj_resp_hdr(dev_data->respbuf); + resp_hdr->hdr.length = ALIGN(sla_dobj_id_to_size(SPDM_DOBJ_ID_RESP) + + dev_data->spdm.rsp_len, 32); + dev_data->respbuf->payload_sz = resp_hdr->hdr.length; + + ret = sev_tio_do_cmd(dev_data->cmd, dev_data->cmd_data, 0, + &dev_data->psp_ret, dev_data); + if (ret) + return ret; + + if (dev_data->psp_ret != SEV_RET_SUCCESS) + return -EINVAL; + + return 0; +} + +static void spdm_ctrl_init(struct spdm_ctrl *ctrl, struct tsm_dsm_tio *dev_data) +{ + ctrl->req = dev_data->req; + ctrl->resp = dev_data->resp; + ctrl->scratch = dev_data->scratch; + ctrl->output = dev_data->output; +} + +static void spdm_ctrl_free(struct tsm_dsm_tio *dev_data) +{ + struct sev_tio_status *tio_status = to_tio_status(dev_data); + size_t len = tio_status->spdm_req_size_max - + (sla_dobj_id_to_size(SPDM_DOBJ_ID_RESP) + + sizeof(struct sla_buffer_hdr)); + struct tsm_spdm *spdm = &dev_data->spdm; + + sla_buffer_unmap(dev_data->resp, dev_data->respbuf); + sla_buffer_unmap(dev_data->req, dev_data->reqbuf); + spdm->rsp = NULL; + spdm->req = NULL; + sla_free(dev_data->req, len, true); + sla_free(dev_data->resp, len, false); + sla_free(dev_data->scratch, tio_status->spdm_scratch_size_max, true); + + dev_data->req.sla = 0; + dev_data->resp.sla = 0; + dev_data->scratch.sla = 0; + dev_data->respbuf = NULL; + dev_data->reqbuf = NULL; + sla_free(dev_data->output, tio_status->spdm_out_size_max, true); +} + +static int spdm_ctrl_alloc(struct tsm_dsm_tio *dev_data) +{ + struct sev_tio_status *tio_status = to_tio_status(dev_data); + struct tsm_spdm *spdm = &dev_data->spdm; + int ret; + + dev_data->req = sla_alloc(tio_status->spdm_req_size_max, true); + dev_data->resp = sla_alloc(tio_status->spdm_req_size_max, false); + dev_data->scratch_len = tio_status->spdm_scratch_size_max; + dev_data->scratch = sla_alloc(dev_data->scratch_len, true); + dev_data->output_len = tio_status->spdm_out_size_max; + dev_data->output = sla_alloc(dev_data->output_len, true); + + if (IS_SLA_NULL(dev_data->req) || IS_SLA_NULL(dev_data->resp) || + IS_SLA_NULL(dev_data->scratch) || IS_SLA_NULL(dev_data->dev_ctx)) { + ret = -ENOMEM; + goto free_spdm_exit; + } + + dev_data->reqbuf = sla_buffer_map(dev_data->req); + dev_data->respbuf = sla_buffer_map(dev_data->resp); + if (!dev_data->reqbuf || !dev_data->respbuf) { + ret = -EFAULT; + goto free_spdm_exit; + } + + spdm->req = sla_to_data(dev_data->reqbuf, SPDM_DOBJ_ID_REQ); + spdm->rsp = sla_to_data(dev_data->respbuf, SPDM_DOBJ_ID_RESP); + if (!spdm->req || !spdm->rsp) { + ret = -EFAULT; + goto free_spdm_exit; + } + + dobj_response_init(dev_data->respbuf); + + return 0; + +free_spdm_exit: + spdm_ctrl_free(dev_data); + return ret; +} + +int sev_tio_init_locked(void *tio_status_page) +{ + struct sev_tio_status *tio_status = tio_status_page; + struct sev_data_tio_status data_status = { + .length = sizeof(data_status), + }; + int ret, psp_ret; + + data_status.status_paddr = __psp_pa(tio_status_page); + ret = __sev_do_cmd_locked(SEV_CMD_TIO_STATUS, &data_status, &psp_ret); + if (ret) + return ret; + + if (tio_status->length < offsetofend(struct sev_tio_status, tdictx_size) || + tio_status->reserved) + return -EFAULT; + + if (!tio_status->tio_en && !tio_status->tio_init_done) + return -ENOENT; + + if (tio_status->tio_init_done) + return -EBUSY; + + struct sev_data_tio_init ti = { .length = sizeof(ti) }; + + ret = __sev_do_cmd_locked(SEV_CMD_TIO_INIT, &ti, &psp_ret); + if (ret) + return ret; + + ret = __sev_do_cmd_locked(SEV_CMD_TIO_STATUS, &data_status, &psp_ret); + if (ret) + return ret; + + return 0; +} + +int sev_tio_dev_create(struct tsm_dsm_tio *dev_data, u16 device_id, + u16 root_port_id, u8 segment_id) +{ + struct sev_tio_status *tio_status = to_tio_status(dev_data); + struct sev_data_tio_dev_create create = { + .length = sizeof(create), + .device_id = device_id, + .root_port_id = root_port_id, + .segment_id = segment_id, + }; + void *data_pg; + int ret; + + dev_data->dev_ctx = sla_alloc(tio_status->devctx_size, true); + if (IS_SLA_NULL(dev_data->dev_ctx)) + return -ENOMEM; + + data_pg = snp_alloc_firmware_page(GFP_KERNEL_ACCOUNT); + if (!data_pg) { + ret = -ENOMEM; + goto free_ctx_exit; + } + + create.dev_ctx_sla = dev_data->dev_ctx; + ret = sev_do_cmd(SEV_CMD_TIO_DEV_CREATE, &create, &dev_data->psp_ret); + if (ret) + goto free_data_pg_exit; + + dev_data->data_pg = data_pg; + + return 0; + +free_data_pg_exit: + snp_free_firmware_page(data_pg); +free_ctx_exit: + sla_free(create.dev_ctx_sla, tio_status->devctx_size, true); + return ret; +} + +int sev_tio_dev_reclaim(struct tsm_dsm_tio *dev_data) +{ + struct sev_tio_status *tio_status = to_tio_status(dev_data); + struct sev_data_tio_dev_reclaim r = { + .length = sizeof(r), + .dev_ctx_sla = dev_data->dev_ctx, + }; + int ret; + + if (dev_data->data_pg) { + snp_free_firmware_page(dev_data->data_pg); + dev_data->data_pg = NULL; + } + + if (IS_SLA_NULL(dev_data->dev_ctx)) + return 0; + + ret = sev_do_cmd(SEV_CMD_TIO_DEV_RECLAIM, &r, &dev_data->psp_ret); + + sla_free(dev_data->dev_ctx, tio_status->devctx_size, true); + dev_data->dev_ctx = SLA_NULL; + + spdm_ctrl_free(dev_data); + + return ret; +} + +int sev_tio_dev_connect(struct tsm_dsm_tio *dev_data, u8 tc_mask, u8 ids[8], u8 cert_slot) +{ + struct sev_data_tio_dev_connect connect = { + .length = sizeof(connect), + .tc_mask = tc_mask, + .cert_slot = cert_slot, + .dev_ctx_sla = dev_data->dev_ctx, + .ide_stream_id = { + ids[0], ids[1], ids[2], ids[3], + ids[4], ids[5], ids[6], ids[7] + }, + }; + int ret; + + if (WARN_ON(IS_SLA_NULL(dev_data->dev_ctx))) + return -EFAULT; + if (!(tc_mask & 1)) + return -EINVAL; + + ret = spdm_ctrl_alloc(dev_data); + if (ret) + return ret; + + spdm_ctrl_init(&connect.spdm_ctrl, dev_data); + + return sev_tio_do_cmd(SEV_CMD_TIO_DEV_CONNECT, &connect, sizeof(connect), + &dev_data->psp_ret, dev_data); +} + +int sev_tio_dev_disconnect(struct tsm_dsm_tio *dev_data, bool force) +{ + struct sev_data_tio_dev_disconnect dc = { + .length = sizeof(dc), + .dev_ctx_sla = dev_data->dev_ctx, + .flags = force ? TIO_DEV_DISCONNECT_FLAG_FORCE : 0, + }; + + if (WARN_ON_ONCE(IS_SLA_NULL(dev_data->dev_ctx))) + return -EFAULT; + + spdm_ctrl_init(&dc.spdm_ctrl, dev_data); + + return sev_tio_do_cmd(SEV_CMD_TIO_DEV_DISCONNECT, &dc, sizeof(dc), + &dev_data->psp_ret, dev_data); +} + +int sev_tio_cmd_buffer_len(int cmd) +{ + switch (cmd) { + case SEV_CMD_TIO_STATUS: return sizeof(struct sev_data_tio_status); + case SEV_CMD_TIO_INIT: return sizeof(struct sev_data_tio_init); + case SEV_CMD_TIO_DEV_CREATE: return sizeof(struct sev_data_tio_dev_create); + case SEV_CMD_TIO_DEV_RECLAIM: return sizeof(struct sev_data_tio_dev_reclaim); + case SEV_CMD_TIO_DEV_CONNECT: return sizeof(struct sev_data_tio_dev_connect); + case SEV_CMD_TIO_DEV_DISCONNECT: return sizeof(struct sev_data_tio_dev_disconnect); + default: return 0; + } +} diff --git a/drivers/crypto/ccp/sev-dev-tio.h b/drivers/crypto/ccp/sev-dev-tio.h new file mode 100644 index 000000000..67512b3db --- /dev/null +++ b/drivers/crypto/ccp/sev-dev-tio.h @@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __PSP_SEV_TIO_H__ +#define __PSP_SEV_TIO_H__ + +#include +#include +#include +#include + +struct sla_addr_t { + union { + u64 sla; + struct { + u64 page_type :1, + page_size :1, + reserved1 :10, + pfn :40, + reserved2 :12; + }; + }; +} __packed; + +#define SEV_TIO_MAX_COMMAND_LENGTH 128 + +/* SPDM control structure for DOE */ +struct tsm_spdm { + unsigned long req_len; + void *req; + unsigned long rsp_len; + void *rsp; +}; + +/* Describes TIO device */ +struct tsm_dsm_tio { + u8 cert_slot; + struct sla_addr_t dev_ctx; + struct sla_addr_t req; + struct sla_addr_t resp; + struct sla_addr_t scratch; + struct sla_addr_t output; + size_t output_len; + size_t scratch_len; + struct tsm_spdm spdm; + struct sla_buffer_hdr *reqbuf; /* vmap'ed @req for DOE */ + struct sla_buffer_hdr *respbuf; /* vmap'ed @resp for DOE */ + + int cmd; + int psp_ret; + u8 cmd_data[SEV_TIO_MAX_COMMAND_LENGTH]; + void *data_pg; /* Data page for DEV_STATUS/TDI_STATUS/TDI_INFO/ASID_FENCE */ + +#define TIO_IDE_MAX_TC 8 + struct pci_ide *ide[TIO_IDE_MAX_TC]; +}; + +/* Describes TSM structure for PF0 pointed by pci_dev->tsm */ +struct tio_dsm { + struct pci_tsm_pf0 tsm; + struct tsm_dsm_tio data; + struct sev_device *sev; +}; + +/* Data object IDs */ +#define SPDM_DOBJ_ID_NONE 0 +#define SPDM_DOBJ_ID_REQ 1 +#define SPDM_DOBJ_ID_RESP 2 + +struct spdm_dobj_hdr { + u32 id; /* Data object type identifier */ + u32 length; /* Length of the data object, INCLUDING THIS HEADER */ + struct { /* Version of the data object structure */ + u8 minor; + u8 major; + } version; +} __packed; + +/** + * struct sev_tio_status - TIO_STATUS command's info_paddr buffer + * + * @length: Length of this structure in bytes + * @tio_en: Indicates that SNP_INIT_EX initialized the RMP for SEV-TIO + * @tio_init_done: Indicates TIO_INIT has been invoked + * @spdm_req_size_min: Minimum SPDM request buffer size in bytes + * @spdm_req_size_max: Maximum SPDM request buffer size in bytes + * @spdm_scratch_size_min: Minimum SPDM scratch buffer size in bytes + * @spdm_scratch_size_max: Maximum SPDM scratch buffer size in bytes + * @spdm_out_size_min: Minimum SPDM output buffer size in bytes + * @spdm_out_size_max: Maximum for the SPDM output buffer size in bytes + * @spdm_rsp_size_min: Minimum SPDM response buffer size in bytes + * @spdm_rsp_size_max: Maximum SPDM response buffer size in bytes + * @devctx_size: Size of a device context buffer in bytes + * @tdictx_size: Size of a TDI context buffer in bytes + * @tio_crypto_alg: TIO crypto algorithms supported + */ +struct sev_tio_status { + u32 length; + u32 tio_en :1, + tio_init_done :1, + reserved :30; + u32 spdm_req_size_min; + u32 spdm_req_size_max; + u32 spdm_scratch_size_min; + u32 spdm_scratch_size_max; + u32 spdm_out_size_min; + u32 spdm_out_size_max; + u32 spdm_rsp_size_min; + u32 spdm_rsp_size_max; + u32 devctx_size; + u32 tdictx_size; + u32 tio_crypto_alg; + u8 reserved2[12]; +} __packed; + +int sev_tio_init_locked(void *tio_status_page); +int sev_tio_continue(struct tsm_dsm_tio *dev_data); + +int sev_tio_dev_create(struct tsm_dsm_tio *dev_data, u16 device_id, u16 root_port_id, + u8 segment_id); +int sev_tio_dev_connect(struct tsm_dsm_tio *dev_data, u8 tc_mask, u8 ids[8], u8 cert_slot); +int sev_tio_dev_disconnect(struct tsm_dsm_tio *dev_data, bool force); +int sev_tio_dev_reclaim(struct tsm_dsm_tio *dev_data); + +#endif /* __PSP_SEV_TIO_H__ */ diff --git a/drivers/crypto/ccp/sev-dev-tsm.c b/drivers/crypto/ccp/sev-dev-tsm.c new file mode 100644 index 000000000..46f2539d2 --- /dev/null +++ b/drivers/crypto/ccp/sev-dev-tsm.c @@ -0,0 +1,399 @@ +// SPDX-License-Identifier: GPL-2.0-only + +// Interface to CCP/SEV-TIO for generic PCIe TDISP module + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "psp-dev.h" +#include "sev-dev.h" +#include "sev-dev-tio.h" + +MODULE_IMPORT_NS("PCI_IDE"); + +#define dev_to_sp(dev) ((struct sp_device *)dev_get_drvdata(dev)) +#define dev_to_psp(dev) ((struct psp_device *)(dev_to_sp(dev)->psp_data)) +#define dev_to_sev(dev) ((struct sev_device *)(dev_to_psp(dev)->sev_data)) +#define tsm_dev_to_sev(tsmdev) dev_to_sev((tsmdev)->dev.parent) + +#define pdev_to_tio_dsm(pdev) (container_of((pdev)->tsm, struct tio_dsm, tsm.base_tsm)) + +static int sev_tio_spdm_cmd(struct tio_dsm *dsm, int ret) +{ + struct tsm_dsm_tio *dev_data = &dsm->data; + struct tsm_spdm *spdm = &dev_data->spdm; + + /* Check the main command handler response before entering the loop */ + if (ret == 0 && dev_data->psp_ret != SEV_RET_SUCCESS) + return -EINVAL; + + if (ret <= 0) + return ret; + + /* ret > 0 means "SPDM requested" */ + while (ret == PCI_DOE_FEATURE_CMA || ret == PCI_DOE_FEATURE_SSESSION) { + ret = pci_doe(dsm->tsm.doe_mb, PCI_VENDOR_ID_PCI_SIG, ret, + spdm->req, spdm->req_len, spdm->rsp, spdm->rsp_len); + if (ret < 0) + break; + + WARN_ON_ONCE(ret == 0); /* The response should never be empty */ + spdm->rsp_len = ret; + ret = sev_tio_continue(dev_data); + } + + return ret; +} + +static int stream_enable(struct pci_ide *ide) +{ + struct pci_dev *rp = pcie_find_root_port(ide->pdev); + int ret; + + ret = pci_ide_stream_enable(ide->pdev, ide); + if (ret && ret != -ENXIO) + return ret; + + ret = pci_ide_stream_enable(rp, ide); + if (ret && ret != -ENXIO) + pci_ide_stream_disable(ide->pdev, ide); + + return ret; +} + +static int streams_enable(struct pci_ide **ide) +{ + int ret = 0; + + for (int i = 0; i < TIO_IDE_MAX_TC; ++i) { + if (ide[i]) { + ret = stream_enable(ide[i]); + if (ret) + break; + } + } + + return ret; +} + +static void stream_disable(struct pci_ide *ide) +{ + pci_ide_stream_disable(ide->pdev, ide); + pci_ide_stream_disable(pcie_find_root_port(ide->pdev), ide); +} + +static void streams_disable(struct pci_ide **ide) +{ + for (int i = 0; i < TIO_IDE_MAX_TC; ++i) + if (ide[i]) + stream_disable(ide[i]); +} + +static void stream_setup(struct pci_ide *ide) +{ + struct pci_dev *rp = pcie_find_root_port(ide->pdev); + + ide->partner[PCI_IDE_EP].rid_start = 0; + ide->partner[PCI_IDE_EP].rid_end = 0xffff; + ide->partner[PCI_IDE_RP].rid_start = 0; + ide->partner[PCI_IDE_RP].rid_end = 0xffff; + + ide->pdev->ide_cfg = 0; + ide->pdev->ide_tee_limit = 1; + rp->ide_cfg = 1; + rp->ide_tee_limit = 0; + + pci_warn(ide->pdev, "Forcing CFG/TEE for %s", pci_name(rp)); + pci_ide_stream_setup(ide->pdev, ide); + pci_ide_stream_setup(rp, ide); +} + +static u8 streams_setup(struct pci_ide **ide, u8 *ids) +{ + bool def = false; + u8 tc_mask = 0; + int i; + + for (i = 0; i < TIO_IDE_MAX_TC; ++i) { + if (!ide[i]) { + ids[i] = 0xFF; + continue; + } + + tc_mask |= BIT(i); + ids[i] = ide[i]->stream_id; + + if (!def) { + struct pci_ide_partner *settings; + + settings = pci_ide_to_settings(ide[i]->pdev, ide[i]); + settings->default_stream = 1; + def = true; + } + + stream_setup(ide[i]); + } + + return tc_mask; +} + +static int streams_register(struct pci_ide **ide) +{ + int ret = 0, i; + + for (i = 0; i < TIO_IDE_MAX_TC; ++i) { + if (ide[i]) { + ret = pci_ide_stream_register(ide[i]); + if (ret) + break; + } + } + + return ret; +} + +static void streams_unregister(struct pci_ide **ide) +{ + for (int i = 0; i < TIO_IDE_MAX_TC; ++i) + if (ide[i]) + pci_ide_stream_unregister(ide[i]); +} + +static void stream_teardown(struct pci_ide *ide) +{ + pci_ide_stream_teardown(ide->pdev, ide); + pci_ide_stream_teardown(pcie_find_root_port(ide->pdev), ide); +} + +static void streams_teardown(struct pci_ide **ide) +{ + for (int i = 0; i < TIO_IDE_MAX_TC; ++i) { + if (ide[i]) { + stream_teardown(ide[i]); + pci_ide_stream_free(ide[i]); + ide[i] = NULL; + } + } +} + +static int stream_alloc(struct pci_dev *pdev, struct pci_ide **ide, + unsigned int tc) +{ + struct pci_ide *ide1; + + if (ide[tc]) { + pci_err(pdev, "Stream for class=%d already registered", tc); + return -EBUSY; + } + + ide1 = pci_ide_stream_alloc(pdev); + if (!ide1) + return -EFAULT; + + ide1->stream_id = ide1->host_bridge_stream; + + ide[tc] = ide1; + + return 0; +} + +static struct pci_tsm *tio_pf0_probe(struct pci_dev *pdev, struct sev_device *sev) +{ + struct tio_dsm *dsm __free(kfree) = kzalloc_obj(*dsm); + int rc; + + if (!dsm) + return NULL; + + rc = pci_tsm_pf0_constructor(pdev, &dsm->tsm, sev->tsmdev); + if (rc) + return NULL; + + pci_dbg(pdev, "TSM enabled\n"); + dsm->sev = sev; + return &no_free_ptr(dsm)->tsm.base_tsm; +} + +static struct pci_tsm *dsm_probe(struct tsm_dev *tsmdev, struct pci_dev *pdev) +{ + struct sev_device *sev = tsm_dev_to_sev(tsmdev); + + if (is_pci_tsm_pf0(pdev)) + return tio_pf0_probe(pdev, sev); + return NULL; +} + +static void dsm_remove(struct pci_tsm *tsm) +{ + struct pci_dev *pdev = tsm->pdev; + + pci_dbg(pdev, "TSM disabled\n"); + + if (is_pci_tsm_pf0(pdev)) { + struct tio_dsm *dsm = container_of(tsm, struct tio_dsm, tsm.base_tsm); + + pci_tsm_pf0_destructor(&dsm->tsm); + kfree(dsm); + } +} + +static int dsm_create(struct tio_dsm *dsm) +{ + struct pci_dev *pdev = dsm->tsm.base_tsm.pdev; + struct pci_dev *rootport; + u8 segment_id; + u16 device_id; + u16 root_port_id; + u32 lnkcap = 0; + + if (!pdev->bus) + return -ENODEV; + + segment_id = pci_domain_nr(pdev->bus); + rootport = pcie_find_root_port(pdev); + device_id = pci_dev_id(pdev); + + if (pci_read_config_dword(rootport, pci_pcie_cap(rootport) + PCI_EXP_LNKCAP, + &lnkcap)) + return -ENODEV; + + root_port_id = FIELD_GET(PCI_EXP_LNKCAP_PN, lnkcap); + + return sev_tio_dev_create(&dsm->data, device_id, root_port_id, segment_id); +} + +static int dsm_connect(struct pci_dev *pdev) +{ + struct tio_dsm *dsm = pdev_to_tio_dsm(pdev); + struct tsm_dsm_tio *dev_data = &dsm->data; + u8 ids[TIO_IDE_MAX_TC]; + u8 tc_mask; + int ret; + + if (pci_find_doe_mailbox(pdev, PCI_VENDOR_ID_PCI_SIG, + PCI_DOE_FEATURE_SSESSION) != dsm->tsm.doe_mb) { + pci_err(pdev, "CMA DOE MB must support SSESSION\n"); + return -EFAULT; + } + + ret = stream_alloc(pdev, dev_data->ide, 0); + if (ret) + return ret; + + ret = dsm_create(dsm); + if (ret) + goto ide_free_exit; + + tc_mask = streams_setup(dev_data->ide, ids); + + ret = sev_tio_dev_connect(dev_data, tc_mask, ids, dev_data->cert_slot); + ret = sev_tio_spdm_cmd(dsm, ret); + if (ret) + goto free_exit; + + streams_enable(dev_data->ide); + + ret = streams_register(dev_data->ide); + if (ret) + goto free_exit; + + return 0; + +free_exit: + sev_tio_dev_reclaim(dev_data); + + streams_disable(dev_data->ide); +ide_free_exit: + + streams_teardown(dev_data->ide); + + return ret; +} + +static void dsm_disconnect(struct pci_dev *pdev) +{ + bool force = SYSTEM_HALT <= system_state && system_state <= SYSTEM_RESTART; + struct tio_dsm *dsm = pdev_to_tio_dsm(pdev); + struct tsm_dsm_tio *dev_data = &dsm->data; + int ret; + + ret = sev_tio_dev_disconnect(dev_data, force); + ret = sev_tio_spdm_cmd(dsm, ret); + if (ret && !force) { + ret = sev_tio_dev_disconnect(dev_data, true); + sev_tio_spdm_cmd(dsm, ret); + } + + sev_tio_dev_reclaim(dev_data); + + streams_disable(dev_data->ide); + streams_unregister(dev_data->ide); + streams_teardown(dev_data->ide); +} + +static struct pci_tsm_ops sev_tsm_ops = { + .probe = dsm_probe, + .remove = dsm_remove, + .connect = dsm_connect, + .disconnect = dsm_disconnect, +}; + +void sev_tsm_init_locked(struct sev_device *sev, void *tio_status_page) +{ + struct sev_tio_status *t = kzalloc_obj(*t); + struct tsm_dev *tsmdev; + int ret; + + WARN_ON(sev->tio_status); + + if (!t) + return; + + ret = sev_tio_init_locked(tio_status_page); + if (ret) { + pr_warn("SEV-TIO STATUS failed with %d\n", ret); + goto error_exit; + } + + tsmdev = tsm_register(sev->dev, &sev_tsm_ops); + if (IS_ERR(tsmdev)) + goto error_exit; + + memcpy(t, tio_status_page, sizeof(*t)); + + pr_notice("SEV-TIO status: EN=%d INIT_DONE=%d rq=%d..%d rs=%d..%d " + "scr=%d..%d out=%d..%d dev=%d tdi=%d algos=%x\n", + t->tio_en, t->tio_init_done, + t->spdm_req_size_min, t->spdm_req_size_max, + t->spdm_rsp_size_min, t->spdm_rsp_size_max, + t->spdm_scratch_size_min, t->spdm_scratch_size_max, + t->spdm_out_size_min, t->spdm_out_size_max, + t->devctx_size, t->tdictx_size, + t->tio_crypto_alg); + + sev->tsmdev = tsmdev; + sev->tio_status = t; + + return; + +error_exit: + pr_err("Failed to enable SEV-TIO: ret=%d en=%d initdone=%d SEV=%d\n", + ret, t->tio_en, t->tio_init_done, boot_cpu_has(X86_FEATURE_SEV)); + kfree(t); +} + +void sev_tsm_uninit(struct sev_device *sev) +{ + if (sev->tsmdev) + tsm_unregister(sev->tsmdev); + + sev->tsmdev = NULL; +} diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c new file mode 100644 index 000000000..f833cb7e4 --- /dev/null +++ b/drivers/crypto/ccp/sev-dev.c @@ -0,0 +1,3168 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Secure Encrypted Virtualization (SEV) interface + * + * Copyright (C) 2016,2019 Advanced Micro Devices, Inc. + * + * Author: Brijesh Singh + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "psp-dev.h" +#include "sev-dev.h" + +#define DEVICE_NAME "sev" +#define SEV_FW_FILE "amd/sev.fw" +#define SEV_FW_NAME_SIZE 64 + +/* Minimum firmware version required for the SEV-SNP support */ +#define SNP_MIN_API_MAJOR 1 +#define SNP_MIN_API_MINOR 51 + +/* + * Maximum number of firmware-writable buffers that might be specified + * in the parameters of a legacy SEV command buffer. + */ +#define CMD_BUF_FW_WRITABLE_MAX 2 + +/* Leave room in the descriptor array for an end-of-list indicator. */ +#define CMD_BUF_DESC_MAX (CMD_BUF_FW_WRITABLE_MAX + 1) + +static DEFINE_MUTEX(sev_cmd_mutex); +static struct sev_misc_dev *misc_dev; + +static int psp_cmd_timeout = 100; +module_param(psp_cmd_timeout, int, 0644); +MODULE_PARM_DESC(psp_cmd_timeout, " default timeout value, in seconds, for PSP commands"); + +static int psp_probe_timeout = 5; +module_param(psp_probe_timeout, int, 0644); +MODULE_PARM_DESC(psp_probe_timeout, " default timeout value, in seconds, during PSP device probe"); + +static char *init_ex_path; +module_param(init_ex_path, charp, 0444); +MODULE_PARM_DESC(init_ex_path, " Path for INIT_EX data; if set try INIT_EX"); + +static bool psp_init_on_probe = true; +module_param(psp_init_on_probe, bool, 0444); +MODULE_PARM_DESC(psp_init_on_probe, " if true, the PSP will be initialized on module init. Else the PSP will be initialized on the first command requiring it"); + +#if IS_ENABLED(CONFIG_PCI_TSM) +static bool sev_tio_enabled = true; +module_param_named(tio, sev_tio_enabled, bool, 0444); +MODULE_PARM_DESC(tio, "Enables TIO in SNP_INIT_EX"); +#else +static const bool sev_tio_enabled = false; +#endif + +MODULE_FIRMWARE("amd/amd_sev_fam17h_model0xh.sbin"); /* 1st gen EPYC */ +MODULE_FIRMWARE("amd/amd_sev_fam17h_model3xh.sbin"); /* 2nd gen EPYC */ +MODULE_FIRMWARE("amd/amd_sev_fam19h_model0xh.sbin"); /* 3rd gen EPYC */ +MODULE_FIRMWARE("amd/amd_sev_fam19h_model1xh.sbin"); /* 4th gen EPYC */ + +static bool psp_dead; +static int psp_timeout; + +enum snp_hv_fixed_pages_state { + ALLOCATED, + HV_FIXED, +}; + +struct snp_hv_fixed_pages_entry { + struct list_head list; + struct page *page; + unsigned int order; + bool free; + enum snp_hv_fixed_pages_state page_state; +}; + +static LIST_HEAD(snp_hv_fixed_pages); + +/* Trusted Memory Region (TMR): + * The TMR is a 1MB area that must be 1MB aligned. Use the page allocator + * to allocate the memory, which will return aligned memory for the specified + * allocation order. + * + * When SEV-SNP is enabled the TMR needs to be 2MB aligned and 2MB sized. + */ +#define SEV_TMR_SIZE (1024 * 1024) +#define SNP_TMR_SIZE (2 * 1024 * 1024) + +static void *sev_es_tmr; +static size_t sev_es_tmr_size = SEV_TMR_SIZE; + +/* INIT_EX NV Storage: + * The NV Storage is a 32Kb area and must be 4Kb page aligned. Use the page + * allocator to allocate the memory, which will return aligned memory for the + * specified allocation order. + */ +#define NV_LENGTH (32 * 1024) +static void *sev_init_ex_buffer; + +static void __sev_firmware_shutdown(struct sev_device *sev, bool panic); + +static int snp_shutdown_on_panic(struct notifier_block *nb, + unsigned long reason, void *arg); + +static struct notifier_block snp_panic_notifier = { + .notifier_call = snp_shutdown_on_panic, +}; + +static inline bool sev_version_greater_or_equal(u8 maj, u8 min) +{ + struct sev_device *sev = psp_master->sev_data; + + if (sev->api_major > maj) + return true; + + if (sev->api_major == maj && sev->api_minor >= min) + return true; + + return false; +} + +static void sev_irq_handler(int irq, void *data, unsigned int status) +{ + struct sev_device *sev = data; + int reg; + + /* Check if it is command completion: */ + if (!(status & SEV_CMD_COMPLETE)) + return; + + /* Check if it is SEV command completion: */ + reg = ioread32(sev->io_regs + sev->vdata->cmdresp_reg); + if (FIELD_GET(PSP_CMDRESP_RESP, reg)) { + sev->int_rcvd = 1; + wake_up(&sev->int_queue); + } +} + +static int sev_wait_cmd_ioc(struct sev_device *sev, + unsigned int *reg, unsigned int timeout) +{ + int ret; + + /* + * If invoked during panic handling, local interrupts are disabled, + * so the PSP command completion interrupt can't be used. Poll for + * PSP command completion instead. + */ + if (irqs_disabled()) { + unsigned long timeout_usecs = (timeout * USEC_PER_SEC) / 10; + + /* Poll for SEV command completion: */ + while (timeout_usecs--) { + *reg = ioread32(sev->io_regs + sev->vdata->cmdresp_reg); + if (*reg & PSP_CMDRESP_RESP) + return 0; + + udelay(10); + } + return -ETIMEDOUT; + } + + ret = wait_event_timeout(sev->int_queue, + sev->int_rcvd, timeout * HZ); + if (!ret) + return -ETIMEDOUT; + + *reg = ioread32(sev->io_regs + sev->vdata->cmdresp_reg); + + return 0; +} + +static int sev_cmd_buffer_len(int cmd) +{ + switch (cmd) { + case SEV_CMD_INIT: return sizeof(struct sev_data_init); + case SEV_CMD_INIT_EX: return sizeof(struct sev_data_init_ex); + case SEV_CMD_SNP_SHUTDOWN_EX: return sizeof(struct sev_data_snp_shutdown_ex); + case SEV_CMD_SNP_INIT_EX: return sizeof(struct sev_data_snp_init_ex); + case SEV_CMD_PLATFORM_STATUS: return sizeof(struct sev_user_data_status); + case SEV_CMD_PEK_CSR: return sizeof(struct sev_data_pek_csr); + case SEV_CMD_PEK_CERT_IMPORT: return sizeof(struct sev_data_pek_cert_import); + case SEV_CMD_PDH_CERT_EXPORT: return sizeof(struct sev_data_pdh_cert_export); + case SEV_CMD_LAUNCH_START: return sizeof(struct sev_data_launch_start); + case SEV_CMD_LAUNCH_UPDATE_DATA: return sizeof(struct sev_data_launch_update_data); + case SEV_CMD_LAUNCH_UPDATE_VMSA: return sizeof(struct sev_data_launch_update_vmsa); + case SEV_CMD_LAUNCH_FINISH: return sizeof(struct sev_data_launch_finish); + case SEV_CMD_LAUNCH_MEASURE: return sizeof(struct sev_data_launch_measure); + case SEV_CMD_ACTIVATE: return sizeof(struct sev_data_activate); + case SEV_CMD_DEACTIVATE: return sizeof(struct sev_data_deactivate); + case SEV_CMD_DECOMMISSION: return sizeof(struct sev_data_decommission); + case SEV_CMD_GUEST_STATUS: return sizeof(struct sev_data_guest_status); + case SEV_CMD_DBG_DECRYPT: return sizeof(struct sev_data_dbg); + case SEV_CMD_DBG_ENCRYPT: return sizeof(struct sev_data_dbg); + case SEV_CMD_SEND_START: return sizeof(struct sev_data_send_start); + case SEV_CMD_SEND_UPDATE_DATA: return sizeof(struct sev_data_send_update_data); + case SEV_CMD_SEND_UPDATE_VMSA: return sizeof(struct sev_data_send_update_vmsa); + case SEV_CMD_SEND_FINISH: return sizeof(struct sev_data_send_finish); + case SEV_CMD_RECEIVE_START: return sizeof(struct sev_data_receive_start); + case SEV_CMD_RECEIVE_FINISH: return sizeof(struct sev_data_receive_finish); + case SEV_CMD_RECEIVE_UPDATE_DATA: return sizeof(struct sev_data_receive_update_data); + case SEV_CMD_RECEIVE_UPDATE_VMSA: return sizeof(struct sev_data_receive_update_vmsa); + case SEV_CMD_LAUNCH_UPDATE_SECRET: return sizeof(struct sev_data_launch_secret); + case SEV_CMD_DOWNLOAD_FIRMWARE: return sizeof(struct sev_data_download_firmware); + case SEV_CMD_GET_ID: return sizeof(struct sev_data_get_id); + case SEV_CMD_ATTESTATION_REPORT: return sizeof(struct sev_data_attestation_report); + case SEV_CMD_SEND_CANCEL: return sizeof(struct sev_data_send_cancel); + case SEV_CMD_SNP_GCTX_CREATE: return sizeof(struct sev_data_snp_addr); + case SEV_CMD_SNP_LAUNCH_START: return sizeof(struct sev_data_snp_launch_start); + case SEV_CMD_SNP_LAUNCH_UPDATE: return sizeof(struct sev_data_snp_launch_update); + case SEV_CMD_SNP_ACTIVATE: return sizeof(struct sev_data_snp_activate); + case SEV_CMD_SNP_DECOMMISSION: return sizeof(struct sev_data_snp_addr); + case SEV_CMD_SNP_PAGE_RECLAIM: return sizeof(struct sev_data_snp_page_reclaim); + case SEV_CMD_SNP_GUEST_STATUS: return sizeof(struct sev_data_snp_guest_status); + case SEV_CMD_SNP_LAUNCH_FINISH: return sizeof(struct sev_data_snp_launch_finish); + case SEV_CMD_SNP_DBG_DECRYPT: return sizeof(struct sev_data_snp_dbg); + case SEV_CMD_SNP_DBG_ENCRYPT: return sizeof(struct sev_data_snp_dbg); + case SEV_CMD_SNP_VERIFY_MITIGATION: return sizeof(struct sev_data_snp_verify_mitigation); + case SEV_CMD_SNP_PAGE_UNSMASH: return sizeof(struct sev_data_snp_page_unsmash); + case SEV_CMD_SNP_PLATFORM_STATUS: return sizeof(struct sev_data_snp_addr); + case SEV_CMD_SNP_GUEST_REQUEST: return sizeof(struct sev_data_snp_guest_request); + case SEV_CMD_SNP_CONFIG: return sizeof(struct sev_user_data_snp_config); + case SEV_CMD_SNP_COMMIT: return sizeof(struct sev_data_snp_commit); + case SEV_CMD_SNP_FEATURE_INFO: return sizeof(struct sev_data_snp_feature_info); + case SEV_CMD_SNP_VLEK_LOAD: return sizeof(struct sev_user_data_snp_vlek_load); + default: return sev_tio_cmd_buffer_len(cmd); + } + + return 0; +} + +static struct file *open_file_as_root(const char *filename, int flags, umode_t mode) +{ + CLASS(prepare_creds, cred)(); + if (!cred) + return ERR_PTR(-ENOMEM); + + cred->fsuid = GLOBAL_ROOT_UID; + + scoped_with_init_fs() { + scoped_with_creds(cred) + return filp_open(filename, flags, mode); + } +} + +static int sev_read_init_ex_file(void) +{ + struct sev_device *sev = psp_master->sev_data; + struct file *fp; + ssize_t nread; + + lockdep_assert_held(&sev_cmd_mutex); + + if (!sev_init_ex_buffer) + return -EOPNOTSUPP; + + fp = open_file_as_root(init_ex_path, O_RDONLY, 0); + if (IS_ERR(fp)) { + int ret = PTR_ERR(fp); + + if (ret == -ENOENT) { + dev_info(sev->dev, + "SEV: %s does not exist and will be created later.\n", + init_ex_path); + ret = 0; + } else { + dev_err(sev->dev, + "SEV: could not open %s for read, error %d\n", + init_ex_path, ret); + } + return ret; + } + + nread = kernel_read(fp, sev_init_ex_buffer, NV_LENGTH, NULL); + if (nread != NV_LENGTH) { + dev_info(sev->dev, + "SEV: could not read %u bytes to non volatile memory area, ret %ld\n", + NV_LENGTH, nread); + } + + dev_dbg(sev->dev, "SEV: read %ld bytes from NV file\n", nread); + filp_close(fp, NULL); + + return 0; +} + +static int sev_write_init_ex_file(void) +{ + struct sev_device *sev = psp_master->sev_data; + struct file *fp; + loff_t offset = 0; + ssize_t nwrite; + + lockdep_assert_held(&sev_cmd_mutex); + + if (!sev_init_ex_buffer) + return 0; + + fp = open_file_as_root(init_ex_path, O_CREAT | O_WRONLY, 0600); + if (IS_ERR(fp)) { + int ret = PTR_ERR(fp); + + dev_err(sev->dev, + "SEV: could not open file for write, error %d\n", + ret); + return ret; + } + + nwrite = kernel_write(fp, sev_init_ex_buffer, NV_LENGTH, &offset); + vfs_fsync(fp, 0); + filp_close(fp, NULL); + + if (nwrite != NV_LENGTH) { + dev_err(sev->dev, + "SEV: failed to write %u bytes to non volatile memory area, ret %ld\n", + NV_LENGTH, nwrite); + return -EIO; + } + + dev_dbg(sev->dev, "SEV: write successful to NV file\n"); + + return 0; +} + +static int sev_write_init_ex_file_if_required(int cmd_id) +{ + lockdep_assert_held(&sev_cmd_mutex); + + if (!sev_init_ex_buffer) + return 0; + + /* + * Only a few platform commands modify the SPI/NV area, but none of the + * non-platform commands do. Only INIT(_EX), PLATFORM_RESET, PEK_GEN, + * PEK_CERT_IMPORT, and PDH_GEN do. + */ + switch (cmd_id) { + case SEV_CMD_FACTORY_RESET: + case SEV_CMD_INIT_EX: + case SEV_CMD_PDH_GEN: + case SEV_CMD_PEK_CERT_IMPORT: + case SEV_CMD_PEK_GEN: + break; + default: + return 0; + } + + return sev_write_init_ex_file(); +} + +int snp_reclaim_pages(unsigned long paddr, unsigned int npages, bool locked) +{ + int ret, err, i; + + paddr = __sme_clr(ALIGN_DOWN(paddr, PAGE_SIZE)); + + for (i = 0; i < npages; i++, paddr += PAGE_SIZE) { + struct sev_data_snp_page_reclaim data = {0}; + + data.paddr = paddr; + + if (locked) + ret = __sev_do_cmd_locked(SEV_CMD_SNP_PAGE_RECLAIM, &data, &err); + else + ret = sev_do_cmd(SEV_CMD_SNP_PAGE_RECLAIM, &data, &err); + + if (ret) + goto cleanup; + + ret = rmp_make_shared(__phys_to_pfn(paddr), PG_LEVEL_4K); + if (ret) + goto cleanup; + } + + return 0; + +cleanup: + /* + * If there was a failure reclaiming the page then it is no longer safe + * to release it back to the system; leak it instead. + */ + snp_leak_pages(__phys_to_pfn(paddr), npages - i); + return ret; +} +EXPORT_SYMBOL_GPL(snp_reclaim_pages); + +static int rmp_mark_pages_firmware(unsigned long paddr, unsigned int npages, bool locked) +{ + unsigned long pfn = __sme_clr(paddr) >> PAGE_SHIFT; + int rc, i; + + for (i = 0; i < npages; i++, pfn++) { + rc = rmp_make_private(pfn, 0, PG_LEVEL_4K, 0, true); + if (rc) + goto cleanup; + } + + return 0; + +cleanup: + /* + * Try unrolling the firmware state changes by + * reclaiming the pages which were already changed to the + * firmware state. + */ + snp_reclaim_pages(paddr, i, locked); + + return rc; +} + +static struct page *__snp_alloc_firmware_pages(gfp_t gfp_mask, int order, bool locked) +{ + unsigned long npages = 1ul << order, paddr; + struct sev_device *sev; + struct page *page; + + if (!psp_master || !psp_master->sev_data) + return NULL; + + page = alloc_pages(gfp_mask, order); + if (!page) + return NULL; + + /* If SEV-SNP is initialized then add the page in RMP table. */ + sev = psp_master->sev_data; + if (!sev->snp_initialized) + return page; + + paddr = __pa((unsigned long)page_address(page)); + if (rmp_mark_pages_firmware(paddr, npages, locked)) + return NULL; + + return page; +} + +void *snp_alloc_firmware_page(gfp_t gfp_mask) +{ + struct page *page; + + page = __snp_alloc_firmware_pages(gfp_mask, 0, false); + + return page ? page_address(page) : NULL; +} +EXPORT_SYMBOL_GPL(snp_alloc_firmware_page); + +static void __snp_free_firmware_pages(struct page *page, int order, bool locked) +{ + struct sev_device *sev = psp_master->sev_data; + unsigned long paddr, npages = 1ul << order; + + if (!page) + return; + + paddr = __pa((unsigned long)page_address(page)); + if (sev->snp_initialized && + snp_reclaim_pages(paddr, npages, locked)) + return; + + __free_pages(page, order); +} + +void snp_free_firmware_page(void *addr) +{ + if (!addr) + return; + + __snp_free_firmware_pages(virt_to_page(addr), 0, false); +} +EXPORT_SYMBOL_GPL(snp_free_firmware_page); + +static void *sev_fw_alloc(unsigned long len) +{ + struct page *page; + + page = __snp_alloc_firmware_pages(GFP_KERNEL, get_order(len), true); + if (!page) + return NULL; + + return page_address(page); +} + +/** + * struct cmd_buf_desc - descriptors for managing legacy SEV command address + * parameters corresponding to buffers that may be written to by firmware. + * + * @paddr_ptr: pointer to the address parameter in the command buffer which may + * need to be saved/restored depending on whether a bounce buffer + * is used. In the case of a bounce buffer, the command buffer + * needs to be updated with the address of the new bounce buffer + * snp_map_cmd_buf_desc() has allocated specifically for it. Must + * be NULL if this descriptor is only an end-of-list indicator. + * + * @paddr_orig: storage for the original address parameter, which can be used to + * restore the original value in @paddr_ptr in cases where it is + * replaced with the address of a bounce buffer. + * + * @len: length of buffer located at the address originally stored at @paddr_ptr + * + * @guest_owned: true if the address corresponds to guest-owned pages, in which + * case bounce buffers are not needed. + */ +struct cmd_buf_desc { + u64 *paddr_ptr; + u64 paddr_orig; + u32 len; + bool guest_owned; +}; + +/* + * If a legacy SEV command parameter is a memory address, those pages in + * turn need to be transitioned to/from firmware-owned before/after + * executing the firmware command. + * + * Additionally, in cases where those pages are not guest-owned, a bounce + * buffer is needed in place of the original memory address parameter. + * + * A set of descriptors are used to keep track of this handling, and + * initialized here based on the specific commands being executed. + */ +static void snp_populate_cmd_buf_desc_list(int cmd, void *cmd_buf, + struct cmd_buf_desc *desc_list) +{ + switch (cmd) { + case SEV_CMD_PDH_CERT_EXPORT: { + struct sev_data_pdh_cert_export *data = cmd_buf; + + desc_list[0].paddr_ptr = &data->pdh_cert_address; + desc_list[0].len = data->pdh_cert_len; + desc_list[1].paddr_ptr = &data->cert_chain_address; + desc_list[1].len = data->cert_chain_len; + break; + } + case SEV_CMD_GET_ID: { + struct sev_data_get_id *data = cmd_buf; + + desc_list[0].paddr_ptr = &data->address; + desc_list[0].len = data->len; + break; + } + case SEV_CMD_PEK_CSR: { + struct sev_data_pek_csr *data = cmd_buf; + + desc_list[0].paddr_ptr = &data->address; + desc_list[0].len = data->len; + break; + } + case SEV_CMD_LAUNCH_UPDATE_DATA: { + struct sev_data_launch_update_data *data = cmd_buf; + + desc_list[0].paddr_ptr = &data->address; + desc_list[0].len = data->len; + desc_list[0].guest_owned = true; + break; + } + case SEV_CMD_LAUNCH_UPDATE_VMSA: { + struct sev_data_launch_update_vmsa *data = cmd_buf; + + desc_list[0].paddr_ptr = &data->address; + desc_list[0].len = data->len; + desc_list[0].guest_owned = true; + break; + } + case SEV_CMD_LAUNCH_MEASURE: { + struct sev_data_launch_measure *data = cmd_buf; + + desc_list[0].paddr_ptr = &data->address; + desc_list[0].len = data->len; + break; + } + case SEV_CMD_LAUNCH_UPDATE_SECRET: { + struct sev_data_launch_secret *data = cmd_buf; + + desc_list[0].paddr_ptr = &data->guest_address; + desc_list[0].len = data->guest_len; + desc_list[0].guest_owned = true; + break; + } + case SEV_CMD_DBG_DECRYPT: { + struct sev_data_dbg *data = cmd_buf; + + desc_list[0].paddr_ptr = &data->dst_addr; + desc_list[0].len = data->len; + desc_list[0].guest_owned = true; + break; + } + case SEV_CMD_DBG_ENCRYPT: { + struct sev_data_dbg *data = cmd_buf; + + desc_list[0].paddr_ptr = &data->dst_addr; + desc_list[0].len = data->len; + desc_list[0].guest_owned = true; + break; + } + case SEV_CMD_ATTESTATION_REPORT: { + struct sev_data_attestation_report *data = cmd_buf; + + desc_list[0].paddr_ptr = &data->address; + desc_list[0].len = data->len; + break; + } + case SEV_CMD_SEND_START: { + struct sev_data_send_start *data = cmd_buf; + + desc_list[0].paddr_ptr = &data->session_address; + desc_list[0].len = data->session_len; + break; + } + case SEV_CMD_SEND_UPDATE_DATA: { + struct sev_data_send_update_data *data = cmd_buf; + + desc_list[0].paddr_ptr = &data->hdr_address; + desc_list[0].len = data->hdr_len; + desc_list[1].paddr_ptr = &data->trans_address; + desc_list[1].len = data->trans_len; + break; + } + case SEV_CMD_SEND_UPDATE_VMSA: { + struct sev_data_send_update_vmsa *data = cmd_buf; + + desc_list[0].paddr_ptr = &data->hdr_address; + desc_list[0].len = data->hdr_len; + desc_list[1].paddr_ptr = &data->trans_address; + desc_list[1].len = data->trans_len; + break; + } + case SEV_CMD_RECEIVE_UPDATE_DATA: { + struct sev_data_receive_update_data *data = cmd_buf; + + desc_list[0].paddr_ptr = &data->guest_address; + desc_list[0].len = data->guest_len; + desc_list[0].guest_owned = true; + break; + } + case SEV_CMD_RECEIVE_UPDATE_VMSA: { + struct sev_data_receive_update_vmsa *data = cmd_buf; + + desc_list[0].paddr_ptr = &data->guest_address; + desc_list[0].len = data->guest_len; + desc_list[0].guest_owned = true; + break; + } + default: + break; + } +} + +static int snp_map_cmd_buf_desc(struct cmd_buf_desc *desc) +{ + unsigned int npages; + + if (!desc->len) + return 0; + + /* Allocate a bounce buffer if this isn't a guest owned page. */ + if (!desc->guest_owned) { + struct page *page; + + page = alloc_pages(GFP_KERNEL_ACCOUNT, get_order(desc->len)); + if (!page) { + pr_warn("Failed to allocate bounce buffer for SEV legacy command.\n"); + return -ENOMEM; + } + + desc->paddr_orig = *desc->paddr_ptr; + *desc->paddr_ptr = __psp_pa(page_to_virt(page)); + } + + npages = PAGE_ALIGN(desc->len) >> PAGE_SHIFT; + + /* Transition the buffer to firmware-owned. */ + if (rmp_mark_pages_firmware(*desc->paddr_ptr, npages, true)) { + pr_warn("Error moving pages to firmware-owned state for SEV legacy command.\n"); + return -EFAULT; + } + + return 0; +} + +static int snp_unmap_cmd_buf_desc(struct cmd_buf_desc *desc) +{ + unsigned int npages; + + if (!desc->len) + return 0; + + npages = PAGE_ALIGN(desc->len) >> PAGE_SHIFT; + + /* Transition the buffers back to hypervisor-owned. */ + if (snp_reclaim_pages(*desc->paddr_ptr, npages, true)) { + pr_warn("Failed to reclaim firmware-owned pages while issuing SEV legacy command.\n"); + return -EFAULT; + } + + /* Copy data from bounce buffer and then free it. */ + if (!desc->guest_owned) { + void *bounce_buf = __va(__sme_clr(*desc->paddr_ptr)); + void *dst_buf = __va(__sme_clr(desc->paddr_orig)); + + memcpy(dst_buf, bounce_buf, desc->len); + __free_pages(virt_to_page(bounce_buf), get_order(desc->len)); + + /* Restore the original address in the command buffer. */ + *desc->paddr_ptr = desc->paddr_orig; + } + + return 0; +} + +static int snp_map_cmd_buf_desc_list(int cmd, void *cmd_buf, struct cmd_buf_desc *desc_list) +{ + int i; + + snp_populate_cmd_buf_desc_list(cmd, cmd_buf, desc_list); + + for (i = 0; i < CMD_BUF_DESC_MAX; i++) { + struct cmd_buf_desc *desc = &desc_list[i]; + + if (!desc->paddr_ptr) + break; + + if (snp_map_cmd_buf_desc(desc)) + goto err_unmap; + } + + return 0; + +err_unmap: + for (i--; i >= 0; i--) + snp_unmap_cmd_buf_desc(&desc_list[i]); + + return -EFAULT; +} + +static int snp_unmap_cmd_buf_desc_list(struct cmd_buf_desc *desc_list) +{ + int i, ret = 0; + + for (i = 0; i < CMD_BUF_DESC_MAX; i++) { + struct cmd_buf_desc *desc = &desc_list[i]; + + if (!desc->paddr_ptr) + break; + + if (snp_unmap_cmd_buf_desc(&desc_list[i])) + ret = -EFAULT; + } + + return ret; +} + +static bool sev_cmd_buf_writable(int cmd) +{ + switch (cmd) { + case SEV_CMD_PLATFORM_STATUS: + case SEV_CMD_GUEST_STATUS: + case SEV_CMD_LAUNCH_START: + case SEV_CMD_RECEIVE_START: + case SEV_CMD_LAUNCH_MEASURE: + case SEV_CMD_SEND_START: + case SEV_CMD_SEND_UPDATE_DATA: + case SEV_CMD_SEND_UPDATE_VMSA: + case SEV_CMD_PEK_CSR: + case SEV_CMD_PDH_CERT_EXPORT: + case SEV_CMD_GET_ID: + case SEV_CMD_ATTESTATION_REPORT: + return true; + default: + return false; + } +} + +/* After SNP is INIT'ed, the behavior of legacy SEV commands is changed. */ +static bool snp_legacy_handling_needed(int cmd) +{ + struct sev_device *sev = psp_master->sev_data; + + return cmd < SEV_CMD_SNP_INIT && sev->snp_initialized; +} + +static int snp_prep_cmd_buf(int cmd, void *cmd_buf, struct cmd_buf_desc *desc_list) +{ + if (!snp_legacy_handling_needed(cmd)) + return 0; + + if (snp_map_cmd_buf_desc_list(cmd, cmd_buf, desc_list)) + return -EFAULT; + + /* + * Before command execution, the command buffer needs to be put into + * the firmware-owned state. + */ + if (sev_cmd_buf_writable(cmd)) { + if (rmp_mark_pages_firmware(__pa(cmd_buf), 1, true)) + return -EFAULT; + } + + return 0; +} + +static int snp_reclaim_cmd_buf(int cmd, void *cmd_buf) +{ + if (!snp_legacy_handling_needed(cmd)) + return 0; + + /* + * After command completion, the command buffer needs to be put back + * into the hypervisor-owned state. + */ + if (sev_cmd_buf_writable(cmd)) + if (snp_reclaim_pages(__pa(cmd_buf), 1, true)) + return -EFAULT; + + return 0; +} + +int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret) +{ + struct cmd_buf_desc desc_list[CMD_BUF_DESC_MAX] = {0}; + struct psp_device *psp = psp_master; + struct sev_device *sev; + unsigned int cmdbuff_hi, cmdbuff_lo; + unsigned int phys_lsb, phys_msb; + unsigned int reg; + void *cmd_buf; + int buf_len; + int ret = 0; + + if (!psp || !psp->sev_data) + return -ENODEV; + + if (psp_dead) + return -EBUSY; + + sev = psp->sev_data; + + buf_len = sev_cmd_buffer_len(cmd); + if (WARN_ON_ONCE(!data != !buf_len)) + return -EINVAL; + + /* + * Copy the incoming data to driver's scratch buffer as __pa() will not + * work for some memory, e.g. vmalloc'd addresses, and @data may not be + * physically contiguous. + */ + if (data) { + /* + * Commands are generally issued one at a time and require the + * sev_cmd_mutex, but there could be recursive firmware requests + * due to SEV_CMD_SNP_PAGE_RECLAIM needing to be issued while + * preparing buffers for another command. This is the only known + * case of nesting in the current code, so exactly one + * additional command buffer is available for that purpose. + */ + if (!sev->cmd_buf_active) { + cmd_buf = sev->cmd_buf; + sev->cmd_buf_active = true; + } else if (!sev->cmd_buf_backup_active) { + cmd_buf = sev->cmd_buf_backup; + sev->cmd_buf_backup_active = true; + } else { + dev_err(sev->dev, + "SEV: too many firmware commands in progress, no command buffers available.\n"); + return -EBUSY; + } + + memcpy(cmd_buf, data, buf_len); + + /* + * The behavior of the SEV-legacy commands is altered when the + * SNP firmware is in the INIT state. + */ + ret = snp_prep_cmd_buf(cmd, cmd_buf, desc_list); + if (ret) { + dev_err(sev->dev, + "SEV: failed to prepare buffer for legacy command 0x%x. Error: %d\n", + cmd, ret); + return ret; + } + } else { + cmd_buf = sev->cmd_buf; + } + + /* Get the physical address of the command buffer */ + phys_lsb = data ? lower_32_bits(__psp_pa(cmd_buf)) : 0; + phys_msb = data ? upper_32_bits(__psp_pa(cmd_buf)) : 0; + + dev_dbg(sev->dev, "sev command id %#x buffer 0x%08x%08x timeout %us\n", + cmd, phys_msb, phys_lsb, psp_timeout); + + print_hex_dump_debug("(in): ", DUMP_PREFIX_OFFSET, 16, 2, data, + buf_len, false); + + iowrite32(phys_lsb, sev->io_regs + sev->vdata->cmdbuff_addr_lo_reg); + iowrite32(phys_msb, sev->io_regs + sev->vdata->cmdbuff_addr_hi_reg); + + sev->int_rcvd = 0; + + reg = FIELD_PREP(SEV_CMDRESP_CMD, cmd); + + /* + * If invoked during panic handling, local interrupts are disabled so + * the PSP command completion interrupt can't be used. + * sev_wait_cmd_ioc() already checks for interrupts disabled and + * polls for PSP command completion. Ensure we do not request an + * interrupt from the PSP if irqs disabled. + */ + if (!irqs_disabled()) + reg |= SEV_CMDRESP_IOC; + + iowrite32(reg, sev->io_regs + sev->vdata->cmdresp_reg); + + /* wait for command completion */ + ret = sev_wait_cmd_ioc(sev, ®, psp_timeout); + if (ret) { + if (psp_ret) + *psp_ret = 0; + + dev_err(sev->dev, "sev command %#x timed out, disabling PSP\n", cmd); + psp_dead = true; + + return ret; + } + + psp_timeout = psp_cmd_timeout; + + if (psp_ret) + *psp_ret = FIELD_GET(PSP_CMDRESP_STS, reg); + + if (FIELD_GET(PSP_CMDRESP_STS, reg)) { + dev_dbg(sev->dev, "sev command %#x failed (%#010lx)\n", + cmd, FIELD_GET(PSP_CMDRESP_STS, reg)); + + /* + * PSP firmware may report additional error information in the + * command buffer registers on error. Print contents of command + * buffer registers if they changed. + */ + cmdbuff_hi = ioread32(sev->io_regs + sev->vdata->cmdbuff_addr_hi_reg); + cmdbuff_lo = ioread32(sev->io_regs + sev->vdata->cmdbuff_addr_lo_reg); + if (cmdbuff_hi != phys_msb || cmdbuff_lo != phys_lsb) { + dev_dbg(sev->dev, "Additional error information reported in cmdbuff:"); + dev_dbg(sev->dev, " cmdbuff hi: %#010x\n", cmdbuff_hi); + dev_dbg(sev->dev, " cmdbuff lo: %#010x\n", cmdbuff_lo); + } + ret = -EIO; + } else { + ret = sev_write_init_ex_file_if_required(cmd); + } + + /* + * Copy potential output from the PSP back to data. Do this even on + * failure in case the caller wants to glean something from the error. + */ + if (data) { + int ret_reclaim; + /* + * Restore the page state after the command completes. + */ + ret_reclaim = snp_reclaim_cmd_buf(cmd, cmd_buf); + if (ret_reclaim) { + dev_err(sev->dev, + "SEV: failed to reclaim buffer for legacy command %#x. Error: %d\n", + cmd, ret_reclaim); + return ret_reclaim; + } + + memcpy(data, cmd_buf, buf_len); + + if (sev->cmd_buf_backup_active) + sev->cmd_buf_backup_active = false; + else + sev->cmd_buf_active = false; + + if (snp_unmap_cmd_buf_desc_list(desc_list)) + return -EFAULT; + } + + print_hex_dump_debug("(out): ", DUMP_PREFIX_OFFSET, 16, 2, data, + buf_len, false); + + return ret; +} + +int sev_do_cmd(int cmd, void *data, int *psp_ret) +{ + int rc; + + mutex_lock(&sev_cmd_mutex); + rc = __sev_do_cmd_locked(cmd, data, psp_ret); + mutex_unlock(&sev_cmd_mutex); + + return rc; +} +EXPORT_SYMBOL_GPL(sev_do_cmd); + +static int __sev_init_locked(int *error) +{ + struct sev_data_init data; + + memset(&data, 0, sizeof(data)); + if (sev_es_tmr) { + /* + * Do not include the encryption mask on the physical + * address of the TMR (firmware should clear it anyway). + */ + data.tmr_address = __pa(sev_es_tmr); + + data.flags |= SEV_INIT_FLAGS_SEV_ES; + data.tmr_len = sev_es_tmr_size; + } + + return __sev_do_cmd_locked(SEV_CMD_INIT, &data, error); +} + +static int __sev_init_ex_locked(int *error) +{ + struct sev_data_init_ex data; + + memset(&data, 0, sizeof(data)); + data.length = sizeof(data); + data.nv_address = __psp_pa(sev_init_ex_buffer); + data.nv_len = NV_LENGTH; + + if (sev_es_tmr) { + /* + * Do not include the encryption mask on the physical + * address of the TMR (firmware should clear it anyway). + */ + data.tmr_address = __pa(sev_es_tmr); + + data.flags |= SEV_INIT_FLAGS_SEV_ES; + data.tmr_len = sev_es_tmr_size; + } + + return __sev_do_cmd_locked(SEV_CMD_INIT_EX, &data, error); +} + +static inline int __sev_do_init_locked(int *psp_ret) +{ + if (sev_init_ex_buffer) + return __sev_init_ex_locked(psp_ret); + else + return __sev_init_locked(psp_ret); +} + +/* Hypervisor Fixed pages API interface */ +static void snp_hv_fixed_pages_state_update(struct sev_device *sev, + enum snp_hv_fixed_pages_state page_state) +{ + struct snp_hv_fixed_pages_entry *entry; + + /* List is protected by sev_cmd_mutex */ + lockdep_assert_held(&sev_cmd_mutex); + + if (list_empty(&snp_hv_fixed_pages)) + return; + + list_for_each_entry(entry, &snp_hv_fixed_pages, list) + entry->page_state = page_state; +} + +/* + * Allocate HV_FIXED pages in 2MB aligned sizes to ensure the whole + * 2MB pages are marked as HV_FIXED. + */ +struct page *snp_alloc_hv_fixed_pages(unsigned int num_2mb_pages) +{ + struct psp_device *psp_master = psp_get_master_device(); + struct snp_hv_fixed_pages_entry *entry; + unsigned int order; + struct page *page; + + if (!psp_master) + return NULL; + + order = get_order(PMD_SIZE * num_2mb_pages); + + /* + * SNP_INIT_EX is protected by sev_cmd_mutex, therefore this list + * also needs to be protected using the same mutex. + */ + guard(mutex)(&sev_cmd_mutex); + + /* + * This API uses SNP_INIT_EX to transition allocated pages to HV_Fixed + * page state, fail if SNP is already initialized. + */ + if (psp_master->sev_data && + ((struct sev_device *)psp_master->sev_data)->snp_initialized) + return NULL; + + /* Re-use freed pages that match the request */ + list_for_each_entry(entry, &snp_hv_fixed_pages, list) { + /* Hypervisor fixed page allocator implements exact fit policy */ + if (entry->order == order && entry->free) { + entry->free = false; + memset(page_address(entry->page), 0, + (1 << entry->order) * PAGE_SIZE); + return entry->page; + } + } + + page = alloc_pages(GFP_KERNEL | __GFP_ZERO, order); + if (!page) + return NULL; + + entry = kzalloc_obj(*entry); + if (!entry) { + __free_pages(page, order); + return NULL; + } + + entry->page = page; + entry->order = order; + list_add_tail(&entry->list, &snp_hv_fixed_pages); + + return page; +} + +void snp_free_hv_fixed_pages(struct page *page) +{ + struct psp_device *psp_master = psp_get_master_device(); + struct snp_hv_fixed_pages_entry *entry, *nentry; + + if (!psp_master) + return; + + /* + * SNP_INIT_EX is protected by sev_cmd_mutex, therefore this list + * also needs to be protected using the same mutex. + */ + guard(mutex)(&sev_cmd_mutex); + + list_for_each_entry_safe(entry, nentry, &snp_hv_fixed_pages, list) { + if (entry->page != page) + continue; + + /* + * HV_FIXED page state cannot be changed until reboot + * and they cannot be used by an SNP guest, so they cannot + * be returned back to the page allocator. + * Mark the pages as free internally to allow possible re-use. + */ + if (entry->page_state == HV_FIXED) { + entry->free = true; + } else { + __free_pages(page, entry->order); + list_del(&entry->list); + kfree(entry); + } + return; + } +} + +static void snp_add_hv_fixed_pages(struct sev_device *sev, struct sev_data_range_list *range_list) +{ + struct snp_hv_fixed_pages_entry *entry; + struct sev_data_range *range; + int num_elements; + + lockdep_assert_held(&sev_cmd_mutex); + + if (list_empty(&snp_hv_fixed_pages)) + return; + + num_elements = list_count_nodes(&snp_hv_fixed_pages) + + range_list->num_elements; + + /* + * Ensure the list of HV_FIXED pages that will be passed to firmware + * do not exceed the page-sized argument buffer. + */ + if (num_elements * sizeof(*range) + sizeof(*range_list) > PAGE_SIZE) { + dev_warn(sev->dev, "Additional HV_Fixed pages cannot be accommodated, omitting\n"); + return; + } + + range = &range_list->ranges[range_list->num_elements]; + list_for_each_entry(entry, &snp_hv_fixed_pages, list) { + range->base = page_to_pfn(entry->page) << PAGE_SHIFT; + range->page_count = 1 << entry->order; + range++; + } + range_list->num_elements = num_elements; +} + +static void snp_leak_hv_fixed_pages(void) +{ + struct snp_hv_fixed_pages_entry *entry, *nentry; + + /* List is protected by sev_cmd_mutex */ + lockdep_assert_held(&sev_cmd_mutex); + + if (list_empty(&snp_hv_fixed_pages)) + return; + + list_for_each_entry_safe(entry, nentry, &snp_hv_fixed_pages, list) { + if (entry->free && entry->page_state != HV_FIXED) + __free_pages(entry->page, entry->order); + else + __snp_leak_pages(page_to_pfn(entry->page), + 1 << entry->order, false); + + list_del(&entry->list); + kfree(entry); + } +} + +bool sev_is_snp_ciphertext_hiding_supported(void) +{ + struct psp_device *psp = psp_master; + struct sev_device *sev; + + if (!psp || !psp->sev_data) + return false; + + sev = psp->sev_data; + + /* + * Feature information indicates if CipherTextHiding feature is + * supported by the SEV firmware and additionally platform status + * indicates if CipherTextHiding feature is enabled in the + * Platform BIOS. + */ + return ((sev->snp_feat_info_0.ecx & SNP_CIPHER_TEXT_HIDING_SUPPORTED) && + sev->snp_plat_status.ciphertext_hiding_cap); +} +EXPORT_SYMBOL_GPL(sev_is_snp_ciphertext_hiding_supported); + +static int snp_get_platform_data(struct sev_device *sev, int *error) +{ + struct sev_data_snp_feature_info snp_feat_info; + struct snp_feature_info *feat_info; + struct sev_data_snp_addr buf; + struct page *page; + int rc; + + /* + * This function is expected to be called before SNP is + * initialized. + */ + if (sev->snp_initialized) + return -EINVAL; + + buf.address = __psp_pa(&sev->snp_plat_status); + rc = sev_do_cmd(SEV_CMD_SNP_PLATFORM_STATUS, &buf, error); + if (rc) { + dev_err(sev->dev, "SNP PLATFORM_STATUS command failed, ret = %d, error = %#x\n", + rc, *error); + return rc; + } + + sev->api_major = sev->snp_plat_status.api_major; + sev->api_minor = sev->snp_plat_status.api_minor; + sev->build = sev->snp_plat_status.build_id; + + /* + * Do feature discovery of the currently loaded firmware, + * and cache feature information from CPUID 0x8000_0024, + * sub-function 0. + */ + if (!sev->snp_plat_status.feature_info) + return 0; + + /* + * Use dynamically allocated structure for the SNP_FEATURE_INFO + * command to ensure structure is 8-byte aligned, and does not + * cross a page boundary. + */ + page = alloc_page(GFP_KERNEL); + if (!page) + return -ENOMEM; + + feat_info = page_address(page); + snp_feat_info.length = sizeof(snp_feat_info); + snp_feat_info.ecx_in = 0; + snp_feat_info.feature_info_paddr = __psp_pa(feat_info); + + rc = sev_do_cmd(SEV_CMD_SNP_FEATURE_INFO, &snp_feat_info, error); + if (!rc) + sev->snp_feat_info_0 = *feat_info; + else + dev_err(sev->dev, "SNP FEATURE_INFO command failed, ret = %d, error = %#x\n", + rc, *error); + + __free_page(page); + + return rc; +} + +static int snp_filter_reserved_mem_regions(struct resource *rs, void *arg) +{ + struct sev_data_range_list *range_list = arg; + struct sev_data_range *range = &range_list->ranges[range_list->num_elements]; + size_t size; + + /* + * Ensure the list of HV_FIXED pages passed to the firmware including + * the one about to be written to do not exceed the page-sized argument + * buffer. + */ + if (((range_list->num_elements + 1) * sizeof(struct sev_data_range) + + sizeof(struct sev_data_range_list)) > PAGE_SIZE) + return -E2BIG; + + switch (rs->desc) { + case E820_TYPE_RESERVED: + case E820_TYPE_PMEM: + case E820_TYPE_ACPI: + range->base = rs->start & PAGE_MASK; + size = PAGE_ALIGN((rs->end + 1) - rs->start); + range->page_count = size >> PAGE_SHIFT; + range_list->num_elements++; + break; + default: + break; + } + + return 0; +} + +#ifdef CONFIG_SYSFS +static int snp_verify_mitigation(u16 command, u64 vector, + struct sev_data_snp_verify_mitigation_dst *dst) +{ + struct sev_data_snp_verify_mitigation_dst *mit_dst = NULL; + struct sev_data_snp_verify_mitigation data = {0}; + struct sev_device *sev = psp_master->sev_data; + int ret, error = 0; + + mit_dst = snp_alloc_firmware_page(GFP_KERNEL | __GFP_ZERO); + if (!mit_dst) + return -ENOMEM; + + data.length = sizeof(data); + data.subcommand = command; + data.vector = vector; + data.dst_paddr = __psp_pa(mit_dst); + data.dst_paddr_en = true; + + ret = sev_do_cmd(SEV_CMD_SNP_VERIFY_MITIGATION, &data, &error); + if (!ret) + memcpy(dst, mit_dst, sizeof(*mit_dst)); + else + dev_err(sev->dev, "SNP_VERIFY_MITIGATION command failed, ret = %d, error = %#x\n", + ret, error); + + snp_free_firmware_page(mit_dst); + + return ret; +} + +static ssize_t supported_mitigations_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct sev_data_snp_verify_mitigation_dst dst; + int ret; + + ret = snp_verify_mitigation(SNP_MIT_SUBCMD_REQ_STATUS, 0, &dst); + if (ret) + return ret; + + return sysfs_emit(buf, "0x%llx\n", dst.mit_supported_vector); +} + +static struct kobj_attribute supported_attr = + __ATTR_RO_MODE(supported_mitigations, 0400); + +static ssize_t verified_mitigations_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct sev_data_snp_verify_mitigation_dst dst; + int ret; + + ret = snp_verify_mitigation(SNP_MIT_SUBCMD_REQ_STATUS, 0, &dst); + if (ret) + return ret; + + return sysfs_emit(buf, "0x%llx\n", dst.mit_verified_vector); +} + +static ssize_t verified_mitigations_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct sev_data_snp_verify_mitigation_dst dst; + struct sev_device *sev = psp_master->sev_data; + u64 vector; + int ret; + + ret = kstrtoull(buf, 0, &vector); + if (ret) + return ret; + + /* + * The firmware verifies a single mitigation per call. Reject vectors + * with more than one bit set early to avoid a guaranteed-to-fail call + */ + if (hweight64(vector) != 1) + return -EINVAL; + + ret = snp_verify_mitigation(SNP_MIT_SUBCMD_REQ_VERIFY, vector, &dst); + if (ret) + return ret; + + if (dst.mit_failure_status) { + dev_err(sev->dev, "Verify Mitigation - failure status: 0x%x\n", + dst.mit_failure_status); + return -EINVAL; + } + + return count; +} + +static struct kobj_attribute verified_attr = + __ATTR_RW_MODE(verified_mitigations, 0600); + +static struct attribute *mitigation_attrs[] = { + &supported_attr.attr, + &verified_attr.attr, + NULL +}; + +static const struct attribute_group mit_attr_group = { + .attrs = mitigation_attrs, +}; + +static void sev_snp_register_verify_mitigation(struct sev_device *sev) +{ + int rc; + + if (!(sev->snp_feat_info_0.ecx & SNP_VERIFY_MITIGATION_SUPPORTED) || + sev->verify_mit) + return; + + if (!sev->sev_kobj) { + sev->sev_kobj = kobject_create_and_add("sev", firmware_kobj); + if (!sev->sev_kobj) + return; + } + + sev->verify_mit = kobject_create_and_add("vulnerabilities", sev->sev_kobj); + if (!sev->verify_mit) + goto err_sev_kobj; + + rc = sysfs_create_group(sev->verify_mit, &mit_attr_group); + if (rc) + goto err_verify_mit; + + return; + +err_verify_mit: + kobject_put(sev->verify_mit); + sev->verify_mit = NULL; +err_sev_kobj: + kobject_put(sev->sev_kobj); + sev->sev_kobj = NULL; +} + +static void sev_snp_unregister_verify_mitigation(struct sev_device *sev) +{ + if (sev->verify_mit) { + sysfs_remove_group(sev->verify_mit, &mit_attr_group); + kobject_put(sev->verify_mit); + sev->verify_mit = NULL; + } + + if (sev->sev_kobj) { + kobject_put(sev->sev_kobj); + sev->sev_kobj = NULL; + } +} +#else // CONFIG_SYSFS +static void sev_snp_register_verify_mitigation(struct sev_device *sev) { } +static void sev_snp_unregister_verify_mitigation(struct sev_device *sev) { } +#endif // CONFIG_SYSFS + +static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid) +{ + struct sev_data_range_list *snp_range_list __free(kfree) = NULL; + struct psp_device *psp = psp_master; + struct sev_data_snp_init_ex data = {}; + struct sev_device *sev; + void *arg = &data; + int cmd, rc = 0; + + if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) + return -ENODEV; + + sev = psp->sev_data; + + if (sev->snp_initialized) + return 0; + + if (!sev_version_greater_or_equal(SNP_MIN_API_MAJOR, SNP_MIN_API_MINOR)) { + dev_dbg(sev->dev, "SEV-SNP support requires firmware version >= %d:%d\n", + SNP_MIN_API_MAJOR, SNP_MIN_API_MINOR); + return -EOPNOTSUPP; + } + + rc = snp_prepare(); + if (rc) + return rc; + + /* + * Starting in SNP firmware v1.52, the SNP_INIT_EX command takes a list + * of system physical address ranges to convert into HV-fixed page + * states during the RMP initialization. For instance, the memory that + * UEFI reserves should be included in the that list. This allows system + * components that occasionally write to memory (e.g. logging to UEFI + * reserved regions) to not fail due to RMP initialization and SNP + * enablement. + * + */ + if (sev_version_greater_or_equal(SNP_MIN_API_MAJOR, 52)) { + bool tio_supp = !!(sev->snp_feat_info_0.ebx & SNP_SEV_TIO_SUPPORTED); + + /* + * Firmware checks that the pages containing the ranges enumerated + * in the RANGES structure are either in the default page state or in the + * firmware page state. + */ + snp_range_list = kzalloc(PAGE_SIZE, GFP_KERNEL); + if (!snp_range_list) { + dev_err(sev->dev, + "SEV: SNP_INIT_EX range list memory allocation failed\n"); + return -ENOMEM; + } + + /* + * Retrieve all reserved memory regions from the e820 memory map + * to be setup as HV-fixed pages. + */ + rc = walk_iomem_res_desc(IORES_DESC_NONE, IORESOURCE_MEM, 0, ~0, + snp_range_list, snp_filter_reserved_mem_regions); + if (rc) { + dev_err(sev->dev, + "SEV: SNP_INIT_EX walk_iomem_res_desc failed rc = %d\n", rc); + return rc; + } + + /* + * Add HV_Fixed pages from other PSP sub-devices, such as SFS to the + * HV_Fixed page list. + */ + snp_add_hv_fixed_pages(sev, snp_range_list); + + if (max_snp_asid) { + data.ciphertext_hiding_en = 1; + data.max_snp_asid = max_snp_asid; + } + + data.init_rmp = 1; + data.list_paddr_en = 1; + data.list_paddr = __psp_pa(snp_range_list); + + data.tio_en = tio_supp && sev_tio_enabled && amd_iommu_sev_tio_supported(); + + /* + * When psp_init_on_probe is disabled, the userspace calling + * SEV ioctl can inadvertently shut down SNP and SEV-TIO causing + * unexpected state loss. + */ + if (data.tio_en && !psp_init_on_probe) + dev_warn(sev->dev, "SEV-TIO as incompatible with psp_init_on_probe=0\n"); + + cmd = SEV_CMD_SNP_INIT_EX; + } else { + cmd = SEV_CMD_SNP_INIT; + arg = NULL; + } + + /* + * The following sequence must be issued before launching the first SNP + * guest to ensure all dirty cache lines are flushed, including from + * updates to the RMP table itself via the RMPUPDATE instruction: + * + * - WBINVD on all running CPUs + * - SEV_CMD_SNP_INIT[_EX] firmware command + * - WBINVD on all running CPUs + * - SEV_CMD_SNP_DF_FLUSH firmware command + */ + wbinvd_on_all_cpus(); + + rc = __sev_do_cmd_locked(cmd, arg, error); + if (rc) { + dev_err(sev->dev, "SEV-SNP: %s failed rc %d, error %#x\n", + cmd == SEV_CMD_SNP_INIT_EX ? "SNP_INIT_EX" : "SNP_INIT", + rc, *error); + return rc; + } + + /* Prepare for first SNP guest launch after INIT. */ + wbinvd_on_all_cpus(); + rc = __sev_do_cmd_locked(SEV_CMD_SNP_DF_FLUSH, NULL, error); + if (rc) { + dev_err(sev->dev, "SEV-SNP: SNP_DF_FLUSH failed rc %d, error %#x\n", + rc, *error); + return rc; + } + + snp_hv_fixed_pages_state_update(sev, HV_FIXED); + sev->snp_initialized = true; + dev_dbg(sev->dev, "SEV-SNP firmware initialized, SEV-TIO is %s\n", + data.tio_en ? "enabled" : "disabled"); + + dev_info(sev->dev, "SEV-SNP API:%d.%d build:%d\n", sev->api_major, + sev->api_minor, sev->build); + + atomic_notifier_chain_register(&panic_notifier_list, + &snp_panic_notifier); + + if (data.tio_en) { + struct page *page; + + /* + * This executes with the sev_cmd_mutex held so down the stack + * snp_reclaim_pages(locked=false) might be needed (which is extremely + * unlikely) but will cause a deadlock. + * Instead of exporting __snp_alloc_firmware_pages(), allocate a page + * for this one call here. + */ + page = __snp_alloc_firmware_pages(GFP_KERNEL_ACCOUNT | __GFP_ZERO, + 0, true); + if (page) { + void *tio_status = page_address(page); + + sev_tsm_init_locked(sev, tio_status); + + __snp_free_firmware_pages(page, 0, true); + } + } + + sev_es_tmr_size = SNP_TMR_SIZE; + + return 0; +} + +static void __sev_platform_init_handle_tmr(struct sev_device *sev) +{ + if (sev_es_tmr) + return; + + /* Obtain the TMR memory area for SEV-ES use */ + sev_es_tmr = sev_fw_alloc(sev_es_tmr_size); + if (sev_es_tmr) { + /* Must flush the cache before giving it to the firmware */ + if (!sev->snp_initialized) + clflush_cache_range(sev_es_tmr, sev_es_tmr_size); + } else { + dev_warn(sev->dev, "SEV: TMR allocation failed, SEV-ES support unavailable\n"); + } +} + +/* + * If an init_ex_path is provided allocate a buffer for the file and + * read in the contents. Additionally, if SNP is initialized, convert + * the buffer pages to firmware pages. + */ +static int __sev_platform_init_handle_init_ex_path(struct sev_device *sev) +{ + struct page *page; + int rc; + + if (!init_ex_path) + return 0; + + if (sev_init_ex_buffer) + return 0; + + page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(NV_LENGTH)); + if (!page) { + dev_err(sev->dev, "SEV: INIT_EX NV memory allocation failed\n"); + return -ENOMEM; + } + + sev_init_ex_buffer = page_address(page); + + rc = sev_read_init_ex_file(); + if (rc) + goto err_free; + + /* If SEV-SNP is initialized, transition to firmware page. */ + if (sev->snp_initialized) { + unsigned long npages; + + npages = 1UL << get_order(NV_LENGTH); + if (rmp_mark_pages_firmware(__pa(sev_init_ex_buffer), npages, true)) { + dev_err(sev->dev, "SEV: INIT_EX NV memory page state change failed.\n"); + rc = -ENOMEM; + /* + * Pages can be in an inconsistent state, don't release them back to the + * system. + */ + goto err_reset; + } + } + + return 0; + +err_free: + __free_pages(page, get_order(NV_LENGTH)); +err_reset: + sev_init_ex_buffer = NULL; + return rc; +} + +static int __sev_platform_init_locked(int *error) +{ + int rc, psp_ret, dfflush_error; + struct sev_device *sev; + + psp_ret = dfflush_error = SEV_RET_NO_FW_CALL; + + if (!psp_master || !psp_master->sev_data) + return -ENODEV; + + sev = psp_master->sev_data; + + if (sev->sev_plat_status.state == SEV_STATE_INIT) + return 0; + + __sev_platform_init_handle_tmr(sev); + + rc = __sev_platform_init_handle_init_ex_path(sev); + if (rc) + return rc; + + rc = __sev_do_init_locked(&psp_ret); + if (rc && psp_ret == SEV_RET_SECURE_DATA_INVALID) { + /* + * Initialization command returned an integrity check failure + * status code, meaning that firmware load and validation of SEV + * related persistent data has failed. Retrying the + * initialization function should succeed by replacing the state + * with a reset state. + */ + dev_err(sev->dev, +"SEV: retrying INIT command because of SECURE_DATA_INVALID error. Retrying once to reset PSP SEV state."); + rc = __sev_do_init_locked(&psp_ret); + } + + if (error) + *error = psp_ret; + + if (rc) { + dev_err(sev->dev, "SEV: %s failed %#x, rc %d\n", + sev_init_ex_buffer ? "INIT_EX" : "INIT", psp_ret, rc); + return rc; + } + + sev->sev_plat_status.state = SEV_STATE_INIT; + + /* Prepare for first SEV guest launch after INIT */ + wbinvd_on_all_cpus(); + rc = __sev_do_cmd_locked(SEV_CMD_DF_FLUSH, NULL, &dfflush_error); + if (rc) { + dev_err(sev->dev, "SEV: DF_FLUSH failed %#x, rc %d\n", + dfflush_error, rc); + return rc; + } + + dev_dbg(sev->dev, "SEV firmware initialized\n"); + + dev_info(sev->dev, "SEV API:%d.%d build:%d\n", sev->api_major, + sev->api_minor, sev->build); + + return 0; +} + +static int _sev_platform_init_locked(struct sev_platform_init_args *args) +{ + struct sev_device *sev; + int rc; + + if (!psp_master || !psp_master->sev_data) + return -ENODEV; + + /* + * Skip SNP/SEV initialization under a kdump kernel as SEV/SNP + * may already be initialized in the previous kernel. Since no + * SNP/SEV guests are run under a kdump kernel, there is no + * need to initialize SNP or SEV during kdump boot. + */ + if (is_kdump_kernel()) + return 0; + + sev = psp_master->sev_data; + + if (sev->sev_plat_status.state == SEV_STATE_INIT) + return 0; + + rc = __sev_snp_init_locked(&args->error, args->max_snp_asid); + if (rc && rc != -ENODEV) + return rc; + + /* Defer legacy SEV/SEV-ES support if allowed by caller/module. */ + if (args->probe && !psp_init_on_probe) + return 0; + + return __sev_platform_init_locked(&args->error); +} + +int sev_platform_init(struct sev_platform_init_args *args) +{ + int rc; + + mutex_lock(&sev_cmd_mutex); + rc = _sev_platform_init_locked(args); + mutex_unlock(&sev_cmd_mutex); + + /* + * Register the sysfs interface outside the sev_cmd_mutex. The + * _show()/_store() handlers issue SEV commands that acquire the + * sev_cmd_mutex, so creating (and on the shutdown path, removing) the + * sysfs group must stay outside that lock. sysfs provides its own + * synchronization between group creation/removal and concurrent + * attribute access. + */ + if (!rc) + sev_snp_register_verify_mitigation(psp_master->sev_data); + + return rc; +} +EXPORT_SYMBOL_GPL(sev_platform_init); + +static int __sev_platform_shutdown_locked(int *error) +{ + struct psp_device *psp = psp_master; + struct sev_device *sev; + int ret; + + if (!psp || !psp->sev_data) + return 0; + + sev = psp->sev_data; + + if (sev->sev_plat_status.state == SEV_STATE_UNINIT) + return 0; + + ret = __sev_do_cmd_locked(SEV_CMD_SHUTDOWN, NULL, error); + if (ret) { + dev_err(sev->dev, "SEV: failed to SHUTDOWN error %#x, rc %d\n", + *error, ret); + return ret; + } + + sev->sev_plat_status.state = SEV_STATE_UNINIT; + dev_dbg(sev->dev, "SEV firmware shutdown\n"); + + return ret; +} + +static int sev_get_platform_state(int *state, int *error) +{ + struct sev_user_data_status data; + int rc; + + rc = __sev_do_cmd_locked(SEV_CMD_PLATFORM_STATUS, &data, error); + if (rc) + return rc; + + *state = data.state; + return rc; +} + +static int sev_move_to_init_state(struct sev_issue_cmd *argp, bool *shutdown_required) +{ + int rc; + + rc = __sev_platform_init_locked(&argp->error); + if (rc) + return rc; + + *shutdown_required = true; + + return 0; +} + +static int sev_ioctl_do_reset(struct sev_issue_cmd *argp, bool writable) +{ + int state, rc; + + if (!writable) + return -EPERM; + + /* + * The SEV spec requires that FACTORY_RESET must be issued in + * UNINIT state. Before we go further lets check if any guest is + * active. + * + * If FW is in WORKING state then deny the request otherwise issue + * SHUTDOWN command do INIT -> UNINIT before issuing the FACTORY_RESET. + * + */ + rc = sev_get_platform_state(&state, &argp->error); + if (rc) + return rc; + + if (state == SEV_STATE_WORKING) + return -EBUSY; + + if (state == SEV_STATE_INIT) { + rc = __sev_platform_shutdown_locked(&argp->error); + if (rc) + return rc; + } + + return __sev_do_cmd_locked(SEV_CMD_FACTORY_RESET, NULL, &argp->error); +} + +static int sev_ioctl_do_platform_status(struct sev_issue_cmd *argp) +{ + struct sev_user_data_status data; + int ret; + + memset(&data, 0, sizeof(data)); + + ret = __sev_do_cmd_locked(SEV_CMD_PLATFORM_STATUS, &data, &argp->error); + if (ret) + return ret; + + if (copy_to_user((void __user *)argp->data, &data, sizeof(data))) + ret = -EFAULT; + + return ret; +} + +static int sev_ioctl_do_pek_pdh_gen(int cmd, struct sev_issue_cmd *argp, bool writable) +{ + struct sev_device *sev = psp_master->sev_data; + bool shutdown_required = false; + int rc; + + if (!writable) + return -EPERM; + + if (sev->sev_plat_status.state == SEV_STATE_UNINIT) { + rc = sev_move_to_init_state(argp, &shutdown_required); + if (rc) + return rc; + } + + rc = __sev_do_cmd_locked(cmd, NULL, &argp->error); + + if (shutdown_required) + __sev_firmware_shutdown(sev, false); + + return rc; +} + +static int sev_ioctl_do_pek_csr(struct sev_issue_cmd *argp, bool writable) +{ + struct sev_device *sev = psp_master->sev_data; + struct sev_user_data_pek_csr input; + bool shutdown_required = false; + struct sev_data_pek_csr data; + void __user *input_address; + void *blob = NULL; + int ret; + + if (!writable) + return -EPERM; + + if (copy_from_user(&input, (void __user *)argp->data, sizeof(input))) + return -EFAULT; + + memset(&data, 0, sizeof(data)); + + /* userspace wants to query CSR length */ + if (!input.address || !input.length) + goto cmd; + + /* allocate a physically contiguous buffer to store the CSR blob */ + input_address = (void __user *)input.address; + if (input.length > SEV_FW_BLOB_MAX_SIZE) + return -EFAULT; + + blob = kzalloc(input.length, GFP_KERNEL); + if (!blob) + return -ENOMEM; + + data.address = __psp_pa(blob); + data.len = input.length; + +cmd: + if (sev->sev_plat_status.state == SEV_STATE_UNINIT) { + ret = sev_move_to_init_state(argp, &shutdown_required); + if (ret) + goto e_free_blob; + } + + ret = __sev_do_cmd_locked(SEV_CMD_PEK_CSR, &data, &argp->error); + + /* + * Firmware will returns the length of the CSR blob (either the minimum + * required length or the actual length written), return it to the user. + */ + input.length = data.len; + + if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) { + ret = -EFAULT; + goto e_free_blob; + } + + if (ret || WARN_ON_ONCE(argp->error)) + goto e_free_blob; + + if (blob) { + if (copy_to_user(input_address, blob, input.length)) + ret = -EFAULT; + } + +e_free_blob: + if (shutdown_required) + __sev_firmware_shutdown(sev, false); + + kfree(blob); + return ret; +} + +void *psp_copy_user_blob(u64 uaddr, u32 len) +{ + if (!uaddr || !len) + return ERR_PTR(-EINVAL); + + /* verify that blob length does not exceed our limit */ + if (len > SEV_FW_BLOB_MAX_SIZE) + return ERR_PTR(-EINVAL); + + return memdup_user((void __user *)uaddr, len); +} +EXPORT_SYMBOL_GPL(psp_copy_user_blob); + +static int sev_get_api_version(void) +{ + struct sev_device *sev = psp_master->sev_data; + struct sev_user_data_status status; + int error = 0, ret; + + /* + * Cache SNP platform status and SNP feature information + * if SNP is available. + */ + if (cc_platform_has(CC_ATTR_HOST_SEV_SNP)) { + ret = snp_get_platform_data(sev, &error); + if (ret) + return 1; + } + + ret = sev_platform_status(&status, &error); + if (ret) { + dev_err(sev->dev, + "SEV: failed to get status. Error: %#x\n", error); + return 1; + } + + /* Cache SEV platform status */ + sev->sev_plat_status = status; + + sev->api_major = status.api_major; + sev->api_minor = status.api_minor; + sev->build = status.build; + + return 0; +} + +static int sev_get_firmware(struct device *dev, + const struct firmware **firmware) +{ + char fw_name_specific[SEV_FW_NAME_SIZE]; + char fw_name_subset[SEV_FW_NAME_SIZE]; + + snprintf(fw_name_specific, sizeof(fw_name_specific), + "amd/amd_sev_fam%.2xh_model%.2xh.sbin", + boot_cpu_data.x86, boot_cpu_data.x86_model); + + snprintf(fw_name_subset, sizeof(fw_name_subset), + "amd/amd_sev_fam%.2xh_model%.1xxh.sbin", + boot_cpu_data.x86, (boot_cpu_data.x86_model & 0xf0) >> 4); + + /* Check for SEV FW for a particular model. + * Ex. amd_sev_fam17h_model00h.sbin for Family 17h Model 00h + * + * or + * + * Check for SEV FW common to a subset of models. + * Ex. amd_sev_fam17h_model0xh.sbin for + * Family 17h Model 00h -- Family 17h Model 0Fh + * + * or + * + * Fall-back to using generic name: sev.fw + */ + if ((firmware_request_nowarn(firmware, fw_name_specific, dev) >= 0) || + (firmware_request_nowarn(firmware, fw_name_subset, dev) >= 0) || + (firmware_request_nowarn(firmware, SEV_FW_FILE, dev) >= 0)) + return 0; + + return -ENOENT; +} + +/* Don't fail if SEV FW couldn't be updated. Continue with existing SEV FW */ +static int sev_update_firmware(struct device *dev) +{ + struct sev_data_download_firmware data; + const struct firmware *firmware; + int ret, error, order; + struct page *p; + void *fw_blob; + + if (!sev_version_greater_or_equal(0, 15)) { + dev_dbg(dev, "DOWNLOAD_FIRMWARE not supported\n"); + return -1; + } + + if (sev_get_firmware(dev, &firmware) == -ENOENT) { + dev_dbg(dev, "No SEV firmware file present\n"); + return -1; + } + + order = get_order(firmware->size); + p = alloc_pages(GFP_KERNEL, order); + if (!p) { + ret = -1; + goto fw_err; + } + + /* + * Copy firmware data to a kernel allocated contiguous + * memory region. + */ + fw_blob = page_address(p); + memcpy(fw_blob, firmware->data, firmware->size); + + data.address = __psp_pa(fw_blob); + data.len = firmware->size; + + ret = sev_do_cmd(SEV_CMD_DOWNLOAD_FIRMWARE, &data, &error); + + /* + * A quirk for fixing the committed TCB version, when upgrading from + * earlier firmware version than 1.50. + */ + if (!ret && !sev_version_greater_or_equal(1, 50)) + ret = sev_do_cmd(SEV_CMD_DOWNLOAD_FIRMWARE, &data, &error); + + if (ret) + dev_dbg(dev, "Failed to update SEV firmware: %#x\n", error); + + __free_pages(p, order); + +fw_err: + release_firmware(firmware); + + return ret; +} + +static int __sev_snp_shutdown_locked(int *error, bool panic) +{ + struct psp_device *psp = psp_master; + struct sev_device *sev; + struct sev_data_snp_shutdown_ex data; + int ret; + + if (!psp || !psp->sev_data) + return 0; + + sev = psp->sev_data; + + if (!sev->snp_initialized) + return 0; + + memset(&data, 0, sizeof(data)); + data.len = sizeof(data); + data.iommu_snp_shutdown = 1; + if (sev->snp_feat_info_0.ecx & SNP_X86_SHUTDOWN_SUPPORTED) + data.x86_snp_shutdown = 1; + + /* + * If invoked during panic handling, local interrupts are disabled + * and all CPUs are stopped, so wbinvd_on_all_cpus() can't be called. + * In that case, a wbinvd() is done on remote CPUs via the NMI + * callback, so only a local wbinvd() is needed here. + */ + if (!panic) + wbinvd_on_all_cpus(); + else + wbinvd(); + + ret = __sev_do_cmd_locked(SEV_CMD_SNP_SHUTDOWN_EX, &data, error); + /* SHUTDOWN may require DF_FLUSH */ + if (*error == SEV_RET_DFFLUSH_REQUIRED) { + int dfflush_error = SEV_RET_NO_FW_CALL; + + ret = __sev_do_cmd_locked(SEV_CMD_SNP_DF_FLUSH, NULL, &dfflush_error); + if (ret) { + dev_err(sev->dev, "SEV-SNP DF_FLUSH failed, ret = %d, error = %#x\n", + ret, dfflush_error); + return ret; + } + /* reissue the shutdown command */ + ret = __sev_do_cmd_locked(SEV_CMD_SNP_SHUTDOWN_EX, &data, + error); + } + if (ret) { + dev_err(sev->dev, "SEV-SNP firmware shutdown failed, rc %d, error %#x\n", + ret, *error); + return ret; + } + + if (data.x86_snp_shutdown) { + if (!panic) + snp_shutdown(); + snp_hv_fixed_pages_state_update(sev, ALLOCATED); + } else { + /* + * SNP_SHUTDOWN_EX with IOMMU_SNP_SHUTDOWN set to 1 disables SNP + * enforcement by the IOMMU and also transitions all pages + * associated with the IOMMU to the Reclaim state. + * Firmware was transitioning the IOMMU pages to Hypervisor state + * before version 1.53. But, accounting for the number of assigned + * 4kB pages in a 2M page was done incorrectly by not transitioning + * to the Reclaim state. This resulted in RMP #PF when later accessing + * the 2M page containing those pages during kexec boot. Hence, the + * firmware now transitions these pages to Reclaim state and hypervisor + * needs to transition these pages to shared state. SNP Firmware + * version 1.53 and above are needed for kexec boot. + */ + ret = amd_iommu_snp_disable(); + if (ret) { + dev_err(sev->dev, "SNP IOMMU shutdown failed\n"); + return ret; + } + } + + snp_leak_hv_fixed_pages(); + sev->snp_initialized = false; + dev_dbg(sev->dev, "SEV-SNP firmware shutdown\n"); + + /* + * __sev_snp_shutdown_locked() deadlocks when it tries to unregister + * itself during panic as the panic notifier is called with RCU read + * lock held and notifier unregistration does RCU synchronization. + */ + if (!panic) + atomic_notifier_chain_unregister(&panic_notifier_list, + &snp_panic_notifier); + + /* Reset TMR size back to default */ + sev_es_tmr_size = SEV_TMR_SIZE; + + return ret; +} + +static int sev_ioctl_do_pek_import(struct sev_issue_cmd *argp, bool writable) +{ + struct sev_device *sev = psp_master->sev_data; + struct sev_user_data_pek_cert_import input; + struct sev_data_pek_cert_import data; + bool shutdown_required = false; + void *pek_blob, *oca_blob; + int ret; + + if (!writable) + return -EPERM; + + if (copy_from_user(&input, (void __user *)argp->data, sizeof(input))) + return -EFAULT; + + /* copy PEK certificate blobs from userspace */ + pek_blob = psp_copy_user_blob(input.pek_cert_address, input.pek_cert_len); + if (IS_ERR(pek_blob)) + return PTR_ERR(pek_blob); + + data.reserved = 0; + data.pek_cert_address = __psp_pa(pek_blob); + data.pek_cert_len = input.pek_cert_len; + + /* copy PEK certificate blobs from userspace */ + oca_blob = psp_copy_user_blob(input.oca_cert_address, input.oca_cert_len); + if (IS_ERR(oca_blob)) { + ret = PTR_ERR(oca_blob); + goto e_free_pek; + } + + data.oca_cert_address = __psp_pa(oca_blob); + data.oca_cert_len = input.oca_cert_len; + + /* If platform is not in INIT state then transition it to INIT */ + if (sev->sev_plat_status.state != SEV_STATE_INIT) { + ret = sev_move_to_init_state(argp, &shutdown_required); + if (ret) + goto e_free_oca; + } + + ret = __sev_do_cmd_locked(SEV_CMD_PEK_CERT_IMPORT, &data, &argp->error); + +e_free_oca: + if (shutdown_required) + __sev_firmware_shutdown(sev, false); + + kfree(oca_blob); +e_free_pek: + kfree(pek_blob); + return ret; +} + +static int sev_ioctl_do_get_id2(struct sev_issue_cmd *argp) +{ + struct sev_user_data_get_id2 input; + struct sev_data_get_id data; + void __user *input_address; + void *id_blob = NULL; + int ret; + + /* SEV GET_ID is available from SEV API v0.16 and up */ + if (!sev_version_greater_or_equal(0, 16)) + return -ENOTSUPP; + + if (copy_from_user(&input, (void __user *)argp->data, sizeof(input))) + return -EFAULT; + + input_address = (void __user *)input.address; + + if (input.address && input.length) { + /* + * The length of the ID shouldn't be assumed by software since + * it may change in the future. The allocation size is limited + * to 1 << (PAGE_SHIFT + MAX_PAGE_ORDER) by the page allocator. + * If the allocation fails, simply return ENOMEM rather than + * warning in the kernel log. + */ + id_blob = kzalloc(input.length, GFP_KERNEL | __GFP_NOWARN); + if (!id_blob) + return -ENOMEM; + + data.address = __psp_pa(id_blob); + data.len = input.length; + } else { + data.address = 0; + data.len = 0; + } + + ret = __sev_do_cmd_locked(SEV_CMD_GET_ID, &data, &argp->error); + + /* + * Firmware will return the length of the ID value (either the minimum + * required length or the actual length written), return it to the user. + */ + input.length = data.len; + + if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) { + ret = -EFAULT; + goto e_free; + } + + if (ret || WARN_ON_ONCE(argp->error)) + goto e_free; + + if (id_blob) { + if (copy_to_user(input_address, id_blob, data.len)) { + ret = -EFAULT; + goto e_free; + } + } + +e_free: + kfree(id_blob); + + return ret; +} + +static int sev_ioctl_do_get_id(struct sev_issue_cmd *argp) +{ + struct sev_data_get_id *data; + u64 data_size, user_size; + void *id_blob, *mem; + int ret; + + /* SEV GET_ID available from SEV API v0.16 and up */ + if (!sev_version_greater_or_equal(0, 16)) + return -ENOTSUPP; + + /* SEV FW expects the buffer it fills with the ID to be + * 8-byte aligned. Memory allocated should be enough to + * hold data structure + alignment padding + memory + * where SEV FW writes the ID. + */ + data_size = ALIGN(sizeof(struct sev_data_get_id), 8); + user_size = sizeof(struct sev_user_data_get_id); + + mem = kzalloc(data_size + user_size, GFP_KERNEL); + if (!mem) + return -ENOMEM; + + data = mem; + id_blob = mem + data_size; + + data->address = __psp_pa(id_blob); + data->len = user_size; + + ret = __sev_do_cmd_locked(SEV_CMD_GET_ID, data, &argp->error); + if (!ret) { + if (copy_to_user((void __user *)argp->data, id_blob, data->len)) + ret = -EFAULT; + } + + kfree(mem); + + return ret; +} + +static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable) +{ + struct sev_device *sev = psp_master->sev_data; + struct sev_user_data_pdh_cert_export input; + void *pdh_blob = NULL, *cert_blob = NULL; + struct sev_data_pdh_cert_export data; + void __user *input_cert_chain_address; + void __user *input_pdh_cert_address; + bool shutdown_required = false; + int ret; + + if (copy_from_user(&input, (void __user *)argp->data, sizeof(input))) + return -EFAULT; + + memset(&data, 0, sizeof(data)); + + input_pdh_cert_address = (void __user *)input.pdh_cert_address; + input_cert_chain_address = (void __user *)input.cert_chain_address; + + /* Userspace wants to query the certificate length. */ + if (!input.pdh_cert_address || + !input.pdh_cert_len || + !input.cert_chain_address || + !input.cert_chain_len) + goto cmd; + + /* Allocate a physically contiguous buffer to store the PDH blob. */ + if (input.pdh_cert_len > SEV_FW_BLOB_MAX_SIZE) + return -EFAULT; + + /* Allocate a physically contiguous buffer to store the cert chain blob. */ + if (input.cert_chain_len > SEV_FW_BLOB_MAX_SIZE) + return -EFAULT; + + pdh_blob = kzalloc(input.pdh_cert_len, GFP_KERNEL); + if (!pdh_blob) + return -ENOMEM; + + data.pdh_cert_address = __psp_pa(pdh_blob); + data.pdh_cert_len = input.pdh_cert_len; + + cert_blob = kzalloc(input.cert_chain_len, GFP_KERNEL); + if (!cert_blob) { + ret = -ENOMEM; + goto e_free_pdh; + } + + data.cert_chain_address = __psp_pa(cert_blob); + data.cert_chain_len = input.cert_chain_len; + +cmd: + /* If platform is not in INIT state then transition it to INIT. */ + if (sev->sev_plat_status.state != SEV_STATE_INIT) { + if (!writable) { + ret = -EPERM; + goto e_free_cert; + } + ret = sev_move_to_init_state(argp, &shutdown_required); + if (ret) + goto e_free_cert; + } + + ret = __sev_do_cmd_locked(SEV_CMD_PDH_CERT_EXPORT, &data, &argp->error); + + /* + * Firmware will return the length of the blobs (either the minimum + * required length or the actual length written), return 'em to the user. + */ + input.cert_chain_len = data.cert_chain_len; + input.pdh_cert_len = data.pdh_cert_len; + + if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) { + ret = -EFAULT; + goto e_free_cert; + } + + if (ret || WARN_ON_ONCE(argp->error)) + goto e_free_cert; + + if (pdh_blob) { + if (copy_to_user(input_pdh_cert_address, + pdh_blob, input.pdh_cert_len)) { + ret = -EFAULT; + goto e_free_cert; + } + } + + if (cert_blob) { + if (copy_to_user(input_cert_chain_address, + cert_blob, input.cert_chain_len)) + ret = -EFAULT; + } + +e_free_cert: + if (shutdown_required) + __sev_firmware_shutdown(sev, false); + + kfree(cert_blob); +e_free_pdh: + kfree(pdh_blob); + return ret; +} + +static int __sev_do_snp_platform_status(struct sev_user_data_snp_status *status, + int *error) +{ + struct sev_device *sev = psp_master->sev_data; + struct sev_data_snp_addr buf; + struct page *status_page; + void *data; + int ret; + + status_page = alloc_page(GFP_KERNEL_ACCOUNT); + if (!status_page) + return -ENOMEM; + + data = page_address(status_page); + + /* + * SNP_PLATFORM_STATUS can be executed in any SNP state. But if executed + * when SNP has been initialized, the status page must be firmware-owned. + */ + if (sev->snp_initialized) { + /* + * Firmware expects the status page to be in Firmware state, + * otherwise it will report an error INVALID_PAGE_STATE. + */ + if (rmp_mark_pages_firmware(__pa(data), 1, true)) { + ret = -EFAULT; + goto cleanup; + } + } + + buf.address = __psp_pa(data); + ret = __sev_do_cmd_locked(SEV_CMD_SNP_PLATFORM_STATUS, &buf, error); + + if (sev->snp_initialized) { + /* + * The status page will be in Reclaim state on success, or left + * in Firmware state on failure. Use snp_reclaim_pages() to + * transition either case back to Hypervisor-owned state. + */ + if (snp_reclaim_pages(__pa(data), 1, true)) + return -EFAULT; + } + + if (ret) + goto cleanup; + + memcpy(status, data, sizeof(*status)); + +cleanup: + __free_pages(status_page, 0); + return ret; +} + +static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp) +{ + struct sev_user_data_snp_status status; + int ret; + + if (!argp->data) + return -EINVAL; + + ret = __sev_do_snp_platform_status(&status, &argp->error); + if (ret < 0) + return ret; + + if (copy_to_user((void __user *)argp->data, &status, + sizeof(struct sev_user_data_snp_status))) + ret = -EFAULT; + + return ret; +} + +static int sev_ioctl_do_snp_commit(struct sev_issue_cmd *argp) +{ + struct sev_data_snp_commit buf; + int ret; + + buf.len = sizeof(buf); + + ret = __sev_do_cmd_locked(SEV_CMD_SNP_COMMIT, &buf, &argp->error); + + return ret; +} + +static int sev_ioctl_do_snp_set_config(struct sev_issue_cmd *argp, bool writable) +{ + struct sev_device *sev = psp_master->sev_data; + struct sev_user_data_snp_config config; + + if (!argp->data) + return -EINVAL; + + if (!writable) + return -EPERM; + + if (!sev->snp_initialized) + return -ENODEV; + + if (copy_from_user(&config, (void __user *)argp->data, sizeof(config))) + return -EFAULT; + + return __sev_do_cmd_locked(SEV_CMD_SNP_CONFIG, &config, &argp->error); +} + +static int sev_ioctl_do_snp_vlek_load(struct sev_issue_cmd *argp, bool writable) +{ + struct sev_device *sev = psp_master->sev_data; + struct sev_user_data_snp_vlek_load input; + void *blob; + int ret; + + if (!argp->data) + return -EINVAL; + + if (!writable) + return -EPERM; + + if (!sev->snp_initialized) + return -ENODEV; + + if (copy_from_user(&input, u64_to_user_ptr(argp->data), sizeof(input))) + return -EFAULT; + + if (input.len != sizeof(input) || input.vlek_wrapped_version != 0) + return -EINVAL; + + blob = psp_copy_user_blob(input.vlek_wrapped_address, + sizeof(struct sev_user_data_snp_wrapped_vlek_hashstick)); + if (IS_ERR(blob)) + return PTR_ERR(blob); + + input.vlek_wrapped_address = __psp_pa(blob); + + ret = __sev_do_cmd_locked(SEV_CMD_SNP_VLEK_LOAD, &input, &argp->error); + kfree(blob); + + return ret; +} + +static long sev_ioctl(struct file *file, unsigned int ioctl, unsigned long arg) +{ + void __user *argp = (void __user *)arg; + struct sev_issue_cmd input; + int ret = -EFAULT; + bool writable = file->f_mode & FMODE_WRITE; + + if (!psp_master || !psp_master->sev_data) + return -ENODEV; + + if (ioctl != SEV_ISSUE_CMD) + return -EINVAL; + + if (copy_from_user(&input, argp, sizeof(struct sev_issue_cmd))) + return -EFAULT; + + if (input.cmd > SEV_MAX) + return -EINVAL; + + mutex_lock(&sev_cmd_mutex); + + switch (input.cmd) { + + case SEV_FACTORY_RESET: + ret = sev_ioctl_do_reset(&input, writable); + break; + case SEV_PLATFORM_STATUS: + ret = sev_ioctl_do_platform_status(&input); + break; + case SEV_PEK_GEN: + ret = sev_ioctl_do_pek_pdh_gen(SEV_CMD_PEK_GEN, &input, writable); + break; + case SEV_PDH_GEN: + ret = sev_ioctl_do_pek_pdh_gen(SEV_CMD_PDH_GEN, &input, writable); + break; + case SEV_PEK_CSR: + ret = sev_ioctl_do_pek_csr(&input, writable); + break; + case SEV_PEK_CERT_IMPORT: + ret = sev_ioctl_do_pek_import(&input, writable); + break; + case SEV_PDH_CERT_EXPORT: + ret = sev_ioctl_do_pdh_export(&input, writable); + break; + case SEV_GET_ID: + pr_warn_once("SEV_GET_ID command is deprecated, use SEV_GET_ID2\n"); + ret = sev_ioctl_do_get_id(&input); + break; + case SEV_GET_ID2: + ret = sev_ioctl_do_get_id2(&input); + break; + case SNP_PLATFORM_STATUS: + ret = sev_ioctl_do_snp_platform_status(&input); + break; + case SNP_COMMIT: + ret = sev_ioctl_do_snp_commit(&input); + break; + case SNP_SET_CONFIG: + ret = sev_ioctl_do_snp_set_config(&input, writable); + break; + case SNP_VLEK_LOAD: + ret = sev_ioctl_do_snp_vlek_load(&input, writable); + break; + default: + ret = -EINVAL; + goto out; + } + + if (copy_to_user(argp, &input, sizeof(struct sev_issue_cmd))) + ret = -EFAULT; +out: + mutex_unlock(&sev_cmd_mutex); + + return ret; +} + +static const struct file_operations sev_fops = { + .owner = THIS_MODULE, + .unlocked_ioctl = sev_ioctl, +}; + +int sev_platform_status(struct sev_user_data_status *data, int *error) +{ + return sev_do_cmd(SEV_CMD_PLATFORM_STATUS, data, error); +} +EXPORT_SYMBOL_GPL(sev_platform_status); + +int sev_guest_deactivate(struct sev_data_deactivate *data, int *error) +{ + return sev_do_cmd(SEV_CMD_DEACTIVATE, data, error); +} +EXPORT_SYMBOL_GPL(sev_guest_deactivate); + +int sev_guest_activate(struct sev_data_activate *data, int *error) +{ + return sev_do_cmd(SEV_CMD_ACTIVATE, data, error); +} +EXPORT_SYMBOL_GPL(sev_guest_activate); + +int sev_guest_decommission(struct sev_data_decommission *data, int *error) +{ + return sev_do_cmd(SEV_CMD_DECOMMISSION, data, error); +} +EXPORT_SYMBOL_GPL(sev_guest_decommission); + +int sev_guest_df_flush(int *error) +{ + return sev_do_cmd(SEV_CMD_DF_FLUSH, NULL, error); +} +EXPORT_SYMBOL_GPL(sev_guest_df_flush); + +static void sev_exit(struct kref *ref) +{ + misc_deregister(&misc_dev->misc); + kfree(misc_dev); + misc_dev = NULL; +} + +static int sev_misc_init(struct sev_device *sev) +{ + struct device *dev = sev->dev; + int ret; + + /* + * SEV feature support can be detected on multiple devices but the SEV + * FW commands must be issued on the master. During probe, we do not + * know the master hence we create /dev/sev on the first device probe. + * sev_do_cmd() finds the right master device to which to issue the + * command to the firmware. + */ + if (!misc_dev) { + struct miscdevice *misc; + + misc_dev = kzalloc_obj(*misc_dev); + if (!misc_dev) + return -ENOMEM; + + misc = &misc_dev->misc; + misc->minor = MISC_DYNAMIC_MINOR; + misc->name = DEVICE_NAME; + misc->fops = &sev_fops; + + ret = misc_register(misc); + if (ret) + return ret; + + kref_init(&misc_dev->refcount); + } else { + kref_get(&misc_dev->refcount); + } + + init_waitqueue_head(&sev->int_queue); + sev->misc = misc_dev; + dev_dbg(dev, "registered SEV device\n"); + + return 0; +} + +int sev_dev_init(struct psp_device *psp) +{ + struct device *dev = psp->dev; + struct sev_device *sev; + int ret = -ENOMEM; + + if (!boot_cpu_has(X86_FEATURE_SEV)) { + dev_info_once(dev, "SEV: memory encryption not enabled by BIOS\n"); + return 0; + } + + sev = devm_kzalloc(dev, sizeof(*sev), GFP_KERNEL); + if (!sev) + goto e_err; + + sev->cmd_buf = (void *)devm_get_free_pages(dev, GFP_KERNEL, 1); + if (!sev->cmd_buf) + goto e_sev; + + sev->cmd_buf_backup = (uint8_t *)sev->cmd_buf + PAGE_SIZE; + + psp->sev_data = sev; + + sev->dev = dev; + sev->psp = psp; + + sev->io_regs = psp->io_regs; + + sev->vdata = (struct sev_vdata *)psp->vdata->sev; + if (!sev->vdata) { + ret = -ENODEV; + dev_err(dev, "sev: missing driver data\n"); + goto e_buf; + } + + psp_set_sev_irq_handler(psp, sev_irq_handler, sev); + + ret = sev_misc_init(sev); + if (ret) + goto e_irq; + + dev_notice(dev, "sev enabled\n"); + + return 0; + +e_irq: + psp_clear_sev_irq_handler(psp); +e_buf: + devm_free_pages(dev, (unsigned long)sev->cmd_buf); +e_sev: + devm_kfree(dev, sev); +e_err: + psp->sev_data = NULL; + + dev_notice(dev, "sev initialization failed\n"); + + return ret; +} + +static void __sev_firmware_shutdown(struct sev_device *sev, bool panic) +{ + int error; + + __sev_platform_shutdown_locked(&error); + + if (sev_es_tmr) { + /* + * The TMR area was encrypted, flush it from the cache. + * + * If invoked during panic handling, local interrupts are + * disabled and all CPUs are stopped, so wbinvd_on_all_cpus() + * can't be used. In that case, wbinvd() is done on remote CPUs + * via the NMI callback, and done for this CPU later during + * SNP shutdown, so wbinvd_on_all_cpus() can be skipped. + */ + if (!panic) + wbinvd_on_all_cpus(); + + __snp_free_firmware_pages(virt_to_page(sev_es_tmr), + get_order(sev_es_tmr_size), + true); + sev_es_tmr = NULL; + } + + if (sev_init_ex_buffer) { + __snp_free_firmware_pages(virt_to_page(sev_init_ex_buffer), + get_order(NV_LENGTH), + true); + sev_init_ex_buffer = NULL; + } + + __sev_snp_shutdown_locked(&error, panic); +} + +static void sev_firmware_shutdown(struct sev_device *sev) +{ + /* + * Calling without sev_cmd_mutex held as TSM will likely try disconnecting + * IDE and this ends up calling sev_do_cmd() which locks sev_cmd_mutex. + */ + if (sev->tio_status) + sev_tsm_uninit(sev); + + /* + * Remove the sysfs interface before taking the sev_cmd_mutex. + * sysfs_remove_group() waits for in-flight _show()/_store() handlers + * to drain, and those handlers issue SNP_VERIFY_MITIGATION via + * sev_do_cmd() which acquires the sev_cmd_mutex. Removing the group + * while holding the mutex could therefore deadlock. + */ + sev_snp_unregister_verify_mitigation(sev); + + mutex_lock(&sev_cmd_mutex); + + __sev_firmware_shutdown(sev, false); + + kfree(sev->tio_status); + sev->tio_status = NULL; + + mutex_unlock(&sev_cmd_mutex); +} + +void sev_platform_shutdown(void) +{ + if (!psp_master || !psp_master->sev_data) + return; + + sev_firmware_shutdown(psp_master->sev_data); +} +EXPORT_SYMBOL_GPL(sev_platform_shutdown); + +u64 sev_get_snp_policy_bits(void) +{ + struct psp_device *psp = psp_master; + struct sev_device *sev; + u64 policy_bits; + + if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) + return 0; + + if (!psp || !psp->sev_data) + return 0; + + sev = psp->sev_data; + + policy_bits = SNP_POLICY_MASK_BASE; + + if (sev->snp_plat_status.feature_info) { + if (sev->snp_feat_info_0.ecx & SNP_RAPL_DISABLE_SUPPORTED) + policy_bits |= SNP_POLICY_MASK_RAPL_DIS; + + if (sev->snp_feat_info_0.ecx & SNP_CIPHER_TEXT_HIDING_SUPPORTED) + policy_bits |= SNP_POLICY_MASK_CIPHERTEXT_HIDING_DRAM; + + if (sev->snp_feat_info_0.ecx & SNP_AES_256_XTS_POLICY_SUPPORTED) + policy_bits |= SNP_POLICY_MASK_MEM_AES_256_XTS; + + if (sev->snp_feat_info_0.ecx & SNP_CXL_ALLOW_POLICY_SUPPORTED) + policy_bits |= SNP_POLICY_MASK_CXL_ALLOW; + + if (sev_version_greater_or_equal(1, 58)) + policy_bits |= SNP_POLICY_MASK_PAGE_SWAP_DISABLE; + } + + return policy_bits; +} +EXPORT_SYMBOL_GPL(sev_get_snp_policy_bits); + +void sev_dev_destroy(struct psp_device *psp) +{ + struct sev_device *sev = psp->sev_data; + + if (!sev) + return; + + sev_firmware_shutdown(sev); + + if (sev->misc) + kref_put(&misc_dev->refcount, sev_exit); + + psp_clear_sev_irq_handler(psp); +} + +static int snp_shutdown_on_panic(struct notifier_block *nb, + unsigned long reason, void *arg) +{ + struct sev_device *sev = psp_master->sev_data; + + /* + * If sev_cmd_mutex is already acquired, then it's likely + * another PSP command is in flight and issuing a shutdown + * would fail in unexpected ways. Rather than create even + * more confusion during a panic, just bail out here. + */ + if (mutex_is_locked(&sev_cmd_mutex)) + return NOTIFY_DONE; + + __sev_firmware_shutdown(sev, true); + + return NOTIFY_DONE; +} + +int sev_issue_cmd_external_user(struct file *filep, unsigned int cmd, + void *data, int *error) +{ + if (!filep || filep->f_op != &sev_fops) + return -EBADF; + + return sev_do_cmd(cmd, data, error); +} +EXPORT_SYMBOL_GPL(sev_issue_cmd_external_user); + +void sev_pci_init(void) +{ + struct sev_device *sev = psp_master->sev_data; + u8 api_major, api_minor, build; + + if (!sev) + return; + + psp_timeout = psp_probe_timeout; + + if (sev_get_api_version()) + goto err; + + api_major = sev->api_major; + api_minor = sev->api_minor; + build = sev->build; + + if (sev_update_firmware(sev->dev) == 0) + sev_get_api_version(); + + if (api_major != sev->api_major || api_minor != sev->api_minor || + build != sev->build) + dev_info(sev->dev, "SEV firmware updated from %d.%d.%d to %d.%d.%d\n", + api_major, api_minor, build, + sev->api_major, sev->api_minor, sev->build); + + return; + +err: + sev_dev_destroy(psp_master); + + psp_master->sev_data = NULL; +} + +void sev_pci_exit(void) +{ + struct sev_device *sev = psp_master->sev_data; + + if (!sev) + return; + + sev_firmware_shutdown(sev); +} + +static int get_v1_svn(struct sev_device *sev) +{ + struct sev_snp_tcb_version_genoa_milan *tcb; + struct sev_user_data_snp_status status; + int ret, error = 0; + + mutex_lock(&sev_cmd_mutex); + ret = __sev_do_snp_platform_status(&status, &error); + mutex_unlock(&sev_cmd_mutex); + if (ret < 0) + return ret; + + tcb = (struct sev_snp_tcb_version_genoa_milan *)&status + .current_tcb_version; + return tcb->snp; +} + +static int get_v2_svn(struct sev_device *sev) +{ + struct sev_user_data_snp_status status; + struct sev_snp_tcb_version_turin *tcb; + int ret, error = 0; + + mutex_lock(&sev_cmd_mutex); + ret = __sev_do_snp_platform_status(&status, &error); + mutex_unlock(&sev_cmd_mutex); + if (ret < 0) + return ret; + + tcb = (struct sev_snp_tcb_version_turin *)&status + .current_tcb_version; + return tcb->snp; +} + +static bool sev_firmware_allows_es(struct sev_device *sev) +{ + /* Documented in AMD-SB-3023 */ + if (boot_cpu_has(X86_FEATURE_ZEN4) || boot_cpu_has(X86_FEATURE_ZEN3)) + return get_v1_svn(sev) < 0x1b; + else if (boot_cpu_has(X86_FEATURE_ZEN5)) + return get_v2_svn(sev) < 0x4; + else + return true; +} + +int sev_firmware_supported_vm_types(void) +{ + int supported_vm_types = 0; + struct sev_device *sev; + + if (!psp_master || !psp_master->sev_data) + return supported_vm_types; + sev = psp_master->sev_data; + + supported_vm_types |= BIT(KVM_X86_SEV_VM); + supported_vm_types |= BIT(KVM_X86_SEV_ES_VM); + + if (!sev->snp_initialized) + return supported_vm_types; + + supported_vm_types |= BIT(KVM_X86_SNP_VM); + + if (!sev_firmware_allows_es(sev)) + supported_vm_types &= ~BIT(KVM_X86_SEV_ES_VM); + + return supported_vm_types; + +} +EXPORT_SYMBOL_FOR_MODULES(sev_firmware_supported_vm_types, "kvm-amd"); diff --git a/drivers/crypto/ccp/sev-dev.h b/drivers/crypto/ccp/sev-dev.h new file mode 100644 index 000000000..d5e596606 --- /dev/null +++ b/drivers/crypto/ccp/sev-dev.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * AMD Platform Security Processor (PSP) interface driver + * + * Copyright (C) 2017-2019 Advanced Micro Devices, Inc. + * + * Author: Brijesh Singh + */ + +#ifndef __SEV_DEV_H__ +#define __SEV_DEV_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SEV_CMDRESP_CMD GENMASK(26, 16) +#define SEV_CMD_COMPLETE BIT(1) +#define SEV_CMDRESP_IOC BIT(0) + +struct sev_misc_dev { + struct kref refcount; + struct miscdevice misc; +}; + +struct sev_tio_status; + +struct sev_device { + struct device *dev; + struct psp_device *psp; + + void __iomem *io_regs; + + struct sev_vdata *vdata; + + unsigned int int_rcvd; + wait_queue_head_t int_queue; + struct sev_misc_dev *misc; + + u8 api_major; + u8 api_minor; + u8 build; + + void *cmd_buf; + void *cmd_buf_backup; + bool cmd_buf_active; + bool cmd_buf_backup_active; + + bool snp_initialized; + + struct kobject *sev_kobj; + struct kobject *verify_mit; + + struct sev_user_data_status sev_plat_status; + + struct sev_user_data_snp_status snp_plat_status; + struct snp_feature_info snp_feat_info_0; + + struct tsm_dev *tsmdev; + struct sev_tio_status *tio_status; +}; + +int sev_dev_init(struct psp_device *psp); +void sev_dev_destroy(struct psp_device *psp); + +int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret); + +void sev_pci_init(void); +void sev_pci_exit(void); + +struct page *snp_alloc_hv_fixed_pages(unsigned int num_2mb_pages); +void snp_free_hv_fixed_pages(struct page *page); + +void sev_tsm_init_locked(struct sev_device *sev, void *tio_status_page); +void sev_tsm_uninit(struct sev_device *sev); +int sev_tio_cmd_buffer_len(int cmd); + +#endif /* __SEV_DEV_H */ diff --git a/drivers/crypto/ccp/sfs.c b/drivers/crypto/ccp/sfs.c new file mode 100644 index 000000000..a47778397 --- /dev/null +++ b/drivers/crypto/ccp/sfs.c @@ -0,0 +1,311 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Secure Processor Seamless Firmware Servicing support. + * + * Copyright (C) 2025 Advanced Micro Devices, Inc. + * + * Author: Ashish Kalra + */ + +#include + +#include "sfs.h" +#include "sev-dev.h" + +#define SFS_DEFAULT_TIMEOUT (10 * MSEC_PER_SEC) +#define SFS_MAX_PAYLOAD_SIZE (2 * 1024 * 1024) +#define SFS_NUM_2MB_PAGES_CMDBUF (SFS_MAX_PAYLOAD_SIZE / PMD_SIZE) +#define SFS_NUM_PAGES_CMDBUF (SFS_MAX_PAYLOAD_SIZE / PAGE_SIZE) + +static DEFINE_MUTEX(sfs_ioctl_mutex); + +static struct sfs_misc_dev *misc_dev; + +static int send_sfs_cmd(struct sfs_device *sfs_dev, int msg) +{ + int ret; + + sfs_dev->command_buf->hdr.status = 0; + sfs_dev->command_buf->hdr.sub_cmd_id = msg; + + ret = psp_extended_mailbox_cmd(sfs_dev->psp, + SFS_DEFAULT_TIMEOUT, + (struct psp_ext_request *)sfs_dev->command_buf); + if (ret == -EIO) { + dev_dbg(sfs_dev->dev, + "msg 0x%x failed with PSP error: 0x%x, extended status: 0x%x\n", + msg, sfs_dev->command_buf->hdr.status, + *(u32 *)sfs_dev->command_buf->buf); + } + + return ret; +} + +static int send_sfs_get_fw_versions(struct sfs_device *sfs_dev) +{ + /* + * SFS_GET_FW_VERSIONS command needs the output buffer to be + * initialized to 0xC7 in every byte. + */ + memset(sfs_dev->command_buf->sfs_buffer, 0xc7, PAGE_SIZE); + sfs_dev->command_buf->hdr.payload_size = 2 * PAGE_SIZE; + + return send_sfs_cmd(sfs_dev, PSP_SFS_GET_FW_VERSIONS); +} + +static int send_sfs_update_package(struct sfs_device *sfs_dev, const char *payload_name) +{ + char payload_path[PAYLOAD_NAME_SIZE + sizeof("amd/")]; + const struct firmware *firmware; + unsigned long package_size; + int ret; + + /* Sanitize userspace provided payload name */ + if (!strnchr(payload_name, PAYLOAD_NAME_SIZE, '\0')) + return -EINVAL; + + snprintf(payload_path, sizeof(payload_path), "amd/%s", payload_name); + + ret = firmware_request_nowarn(&firmware, payload_path, sfs_dev->dev); + if (ret < 0) { + dev_warn_ratelimited(sfs_dev->dev, "firmware request failed for %s (%d)\n", + payload_path, ret); + return -ENOENT; + } + + /* + * SFS Update Package command's input buffer contains TEE_EXT_CMD_BUFFER + * followed by the Update Package and it should be 64KB aligned. + */ + package_size = ALIGN(firmware->size + PAGE_SIZE, 0x10000U); + + /* + * SFS command buffer is a pre-allocated 2MB buffer, fail update package + * if SFS payload is larger than the pre-allocated command buffer. + */ + if (package_size > SFS_MAX_PAYLOAD_SIZE) { + dev_warn_ratelimited(sfs_dev->dev, + "SFS payload size %ld larger than maximum supported payload size of %u\n", + package_size, SFS_MAX_PAYLOAD_SIZE); + release_firmware(firmware); + return -E2BIG; + } + + /* + * Copy firmware data to a HV_Fixed memory region. + */ + memcpy(sfs_dev->command_buf->sfs_buffer, firmware->data, firmware->size); + sfs_dev->command_buf->hdr.payload_size = package_size; + + release_firmware(firmware); + + return send_sfs_cmd(sfs_dev, PSP_SFS_UPDATE); +} + +static long sfs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct sfs_user_get_fw_versions __user *sfs_get_fw_versions; + struct sfs_user_update_package __user *sfs_update_package; + struct psp_device *psp_master = psp_get_master_device(); + char payload_name[PAYLOAD_NAME_SIZE]; + struct sfs_device *sfs_dev; + int ret = 0; + + if (!psp_master || !psp_master->sfs_data) + return -ENODEV; + + sfs_dev = psp_master->sfs_data; + + guard(mutex)(&sfs_ioctl_mutex); + + switch (cmd) { + case SFSIOCFWVERS: + dev_dbg(sfs_dev->dev, "in SFSIOCFWVERS\n"); + + sfs_get_fw_versions = (struct sfs_user_get_fw_versions __user *)arg; + + ret = send_sfs_get_fw_versions(sfs_dev); + if (ret && ret != -EIO) + return ret; + + /* + * Return SFS status and extended status back to userspace + * if PSP status indicated success or command error. + */ + if (copy_to_user(&sfs_get_fw_versions->blob, sfs_dev->command_buf->sfs_buffer, + PAGE_SIZE)) + return -EFAULT; + if (copy_to_user(&sfs_get_fw_versions->sfs_status, + &sfs_dev->command_buf->hdr.status, + sizeof(sfs_get_fw_versions->sfs_status))) + return -EFAULT; + if (copy_to_user(&sfs_get_fw_versions->sfs_extended_status, + &sfs_dev->command_buf->buf, + sizeof(sfs_get_fw_versions->sfs_extended_status))) + return -EFAULT; + break; + case SFSIOCUPDATEPKG: + dev_dbg(sfs_dev->dev, "in SFSIOCUPDATEPKG\n"); + + sfs_update_package = (struct sfs_user_update_package __user *)arg; + + if (copy_from_user(payload_name, sfs_update_package->payload_name, + PAYLOAD_NAME_SIZE)) + return -EFAULT; + + ret = send_sfs_update_package(sfs_dev, payload_name); + if (ret && ret != -EIO) + return ret; + + /* + * Return SFS status and extended status back to userspace + * if PSP status indicated success or command error. + */ + if (copy_to_user(&sfs_update_package->sfs_status, + &sfs_dev->command_buf->hdr.status, + sizeof(sfs_update_package->sfs_status))) + return -EFAULT; + if (copy_to_user(&sfs_update_package->sfs_extended_status, + &sfs_dev->command_buf->buf, + sizeof(sfs_update_package->sfs_extended_status))) + return -EFAULT; + break; + default: + ret = -EINVAL; + } + + return ret; +} + +static const struct file_operations sfs_fops = { + .owner = THIS_MODULE, + .unlocked_ioctl = sfs_ioctl, +}; + +static void sfs_exit(struct kref *ref) +{ + misc_deregister(&misc_dev->misc); + kfree(misc_dev); + misc_dev = NULL; +} + +void sfs_dev_destroy(struct psp_device *psp) +{ + struct sfs_device *sfs_dev = psp->sfs_data; + + if (!sfs_dev) + return; + + /* + * Change SFS command buffer back to the default "Write-Back" type. + */ + set_memory_wb((unsigned long)sfs_dev->command_buf, SFS_NUM_PAGES_CMDBUF); + + snp_free_hv_fixed_pages(sfs_dev->page); + + if (sfs_dev->misc) + kref_put(&misc_dev->refcount, sfs_exit); + + psp->sfs_data = NULL; +} + +/* Based on sev_misc_init() */ +static int sfs_misc_init(struct sfs_device *sfs) +{ + struct device *dev = sfs->dev; + int ret; + + /* + * SFS feature support can be detected on multiple devices but the SFS + * FW commands must be issued on the master. During probe, we do not + * know the master hence we create /dev/sfs on the first device probe. + */ + if (!misc_dev) { + struct miscdevice *misc; + + misc_dev = kzalloc_obj(*misc_dev); + if (!misc_dev) + return -ENOMEM; + + misc = &misc_dev->misc; + misc->minor = MISC_DYNAMIC_MINOR; + misc->name = "sfs"; + misc->fops = &sfs_fops; + misc->mode = 0600; + + ret = misc_register(misc); + if (ret) + return ret; + + kref_init(&misc_dev->refcount); + } else { + kref_get(&misc_dev->refcount); + } + + sfs->misc = misc_dev; + dev_dbg(dev, "registered SFS device\n"); + + return 0; +} + +int sfs_dev_init(struct psp_device *psp) +{ + struct device *dev = psp->dev; + struct sfs_device *sfs_dev; + struct page *page; + int ret = -ENOMEM; + + sfs_dev = devm_kzalloc(dev, sizeof(*sfs_dev), GFP_KERNEL); + if (!sfs_dev) + return -ENOMEM; + + /* + * Pre-allocate 2MB command buffer for all SFS commands using + * SNP HV_Fixed page allocator which also transitions the + * SFS command buffer to HV_Fixed page state if SNP is enabled. + */ + page = snp_alloc_hv_fixed_pages(SFS_NUM_2MB_PAGES_CMDBUF); + if (!page) { + dev_dbg(dev, "Command Buffer HV-Fixed page allocation failed\n"); + goto cleanup_dev; + } + sfs_dev->page = page; + sfs_dev->command_buf = page_address(page); + + dev_dbg(dev, "Command buffer 0x%px to be marked as HV_Fixed\n", sfs_dev->command_buf); + + /* + * SFS command buffer must be mapped as non-cacheable. + */ + ret = set_memory_uc((unsigned long)sfs_dev->command_buf, SFS_NUM_PAGES_CMDBUF); + if (ret) { + dev_dbg(dev, "Set memory uc failed\n"); + goto cleanup_cmd_buf; + } + + dev_dbg(dev, "Command buffer 0x%px marked uncacheable\n", sfs_dev->command_buf); + + psp->sfs_data = sfs_dev; + sfs_dev->dev = dev; + sfs_dev->psp = psp; + + ret = sfs_misc_init(sfs_dev); + if (ret) + goto cleanup_mem_attr; + + dev_notice(sfs_dev->dev, "SFS support is available\n"); + + return 0; + +cleanup_mem_attr: + set_memory_wb((unsigned long)sfs_dev->command_buf, SFS_NUM_PAGES_CMDBUF); + +cleanup_cmd_buf: + snp_free_hv_fixed_pages(page); + +cleanup_dev: + psp->sfs_data = NULL; + devm_kfree(dev, sfs_dev); + + return ret; +} diff --git a/drivers/crypto/ccp/sfs.h b/drivers/crypto/ccp/sfs.h new file mode 100644 index 000000000..97704c210 --- /dev/null +++ b/drivers/crypto/ccp/sfs.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * AMD Platform Security Processor (PSP) Seamless Firmware (SFS) Support. + * + * Copyright (C) 2025 Advanced Micro Devices, Inc. + * + * Author: Ashish Kalra + */ + +#ifndef __SFS_H__ +#define __SFS_H__ + +#include + +#include +#include +#include +#include +#include + +#include "psp-dev.h" + +struct sfs_misc_dev { + struct kref refcount; + struct miscdevice misc; +}; + +struct sfs_command { + struct psp_ext_req_buffer_hdr hdr; + u8 buf[PAGE_SIZE - sizeof(struct psp_ext_req_buffer_hdr)]; + u8 sfs_buffer[]; +} __packed; + +struct sfs_device { + struct device *dev; + struct psp_device *psp; + + struct page *page; + struct sfs_command *command_buf; + + struct sfs_misc_dev *misc; +}; + +void sfs_dev_destroy(struct psp_device *psp); +int sfs_dev_init(struct psp_device *psp); + +#endif /* __SFS_H__ */ diff --git a/drivers/crypto/ccp/sp-dev.c b/drivers/crypto/ccp/sp-dev.c new file mode 100644 index 000000000..f204aa5df --- /dev/null +++ b/drivers/crypto/ccp/sp-dev.c @@ -0,0 +1,325 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Secure Processor driver + * + * Copyright (C) 2017-2018 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky + * Author: Gary R Hook + * Author: Brijesh Singh + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sev-dev.h" +#include "ccp-dev.h" +#include "sp-dev.h" + +MODULE_AUTHOR("Tom Lendacky "); +MODULE_AUTHOR("Gary R Hook "); +MODULE_LICENSE("GPL"); +MODULE_VERSION("1.1.0"); +MODULE_DESCRIPTION("AMD Secure Processor driver"); + +/* List of SPs, SP count, read-write access lock, and access functions + * + * Lock structure: get sp_unit_lock for reading whenever we need to + * examine the SP list. + */ +static DEFINE_RWLOCK(sp_unit_lock); +static LIST_HEAD(sp_units); + +/* Ever-increasing value to produce unique unit numbers */ +static atomic_t sp_ordinal; + +static void sp_add_device(struct sp_device *sp) +{ + unsigned long flags; + + write_lock_irqsave(&sp_unit_lock, flags); + + list_add_tail(&sp->entry, &sp_units); + + write_unlock_irqrestore(&sp_unit_lock, flags); +} + +static void sp_del_device(struct sp_device *sp) +{ + unsigned long flags; + + write_lock_irqsave(&sp_unit_lock, flags); + + list_del(&sp->entry); + + write_unlock_irqrestore(&sp_unit_lock, flags); +} + +static irqreturn_t sp_irq_handler(int irq, void *data) +{ + struct sp_device *sp = data; + + if (sp->ccp_irq_handler) + sp->ccp_irq_handler(irq, sp->ccp_irq_data); + + if (sp->psp_irq_handler) + sp->psp_irq_handler(irq, sp->psp_irq_data); + + return IRQ_HANDLED; +} + +int sp_request_ccp_irq(struct sp_device *sp, irq_handler_t handler, + const char *name, void *data) +{ + int ret; + + if ((sp->psp_irq == sp->ccp_irq) && sp->dev_vdata->psp_vdata) { + /* Need a common routine to manage all interrupts */ + sp->ccp_irq_data = data; + sp->ccp_irq_handler = handler; + + if (!sp->irq_registered) { + ret = request_irq(sp->ccp_irq, sp_irq_handler, 0, + sp->name, sp); + if (ret) + return ret; + + sp->irq_registered = true; + } + } else { + /* Each sub-device can manage it's own interrupt */ + ret = request_irq(sp->ccp_irq, handler, 0, name, data); + if (ret) + return ret; + } + + return 0; +} + +int sp_request_psp_irq(struct sp_device *sp, irq_handler_t handler, + const char *name, void *data) +{ + int ret; + + if ((sp->psp_irq == sp->ccp_irq) && sp->dev_vdata->ccp_vdata) { + /* Need a common routine to manage all interrupts */ + sp->psp_irq_data = data; + sp->psp_irq_handler = handler; + + if (!sp->irq_registered) { + ret = request_irq(sp->psp_irq, sp_irq_handler, 0, + sp->name, sp); + if (ret) + return ret; + + sp->irq_registered = true; + } + } else { + /* Each sub-device can manage it's own interrupt */ + ret = request_irq(sp->psp_irq, handler, 0, name, data); + if (ret) + return ret; + } + + return 0; +} + +void sp_free_ccp_irq(struct sp_device *sp, void *data) +{ + if ((sp->psp_irq == sp->ccp_irq) && sp->dev_vdata->psp_vdata) { + /* Using common routine to manage all interrupts */ + if (!sp->psp_irq_handler) { + /* Nothing else using it, so free it */ + free_irq(sp->ccp_irq, sp); + + sp->irq_registered = false; + } + + sp->ccp_irq_handler = NULL; + sp->ccp_irq_data = NULL; + } else { + /* Each sub-device can manage it's own interrupt */ + free_irq(sp->ccp_irq, data); + } +} + +void sp_free_psp_irq(struct sp_device *sp, void *data) +{ + if ((sp->psp_irq == sp->ccp_irq) && sp->dev_vdata->ccp_vdata) { + /* Using common routine to manage all interrupts */ + if (!sp->ccp_irq_handler) { + /* Nothing else using it, so free it */ + free_irq(sp->psp_irq, sp); + + sp->irq_registered = false; + } + + sp->psp_irq_handler = NULL; + sp->psp_irq_data = NULL; + } else { + /* Each sub-device can manage it's own interrupt */ + free_irq(sp->psp_irq, data); + } +} + +/** + * sp_alloc_struct - allocate and initialize the sp_device struct + * + * @dev: device struct of the SP + */ +struct sp_device *sp_alloc_struct(struct device *dev) +{ + struct sp_device *sp; + + sp = devm_kzalloc(dev, sizeof(*sp), GFP_KERNEL); + if (!sp) + return NULL; + + sp->dev = dev; + sp->ord = atomic_inc_return(&sp_ordinal); + snprintf(sp->name, SP_MAX_NAME_LEN, "sp-%u", sp->ord); + + return sp; +} + +int sp_init(struct sp_device *sp) +{ + sp_add_device(sp); + + if (sp->dev_vdata->ccp_vdata) + ccp_dev_init(sp); + + if (sp->dev_vdata->psp_vdata) + psp_dev_init(sp); + return 0; +} + +void sp_destroy(struct sp_device *sp) +{ + if (sp->dev_vdata->ccp_vdata) + ccp_dev_destroy(sp); + + if (sp->dev_vdata->psp_vdata) + psp_dev_destroy(sp); + + sp_del_device(sp); +} + +int sp_suspend(struct sp_device *sp) +{ + if (sp->dev_vdata->ccp_vdata) { + ccp_dev_suspend(sp); + } + + return 0; +} + +int sp_resume(struct sp_device *sp) +{ + if (sp->dev_vdata->ccp_vdata) { + ccp_dev_resume(sp); + } + + return 0; +} + +int sp_restore(struct sp_device *sp) +{ + if (sp->psp_data) { + int ret = psp_restore(sp); + + if (ret) + return ret; + } + + return sp_resume(sp); +} + +struct sp_device *sp_get_psp_master_device(void) +{ + struct sp_device *i, *ret = NULL; + unsigned long flags; + + write_lock_irqsave(&sp_unit_lock, flags); + if (list_empty(&sp_units)) + goto unlock; + + list_for_each_entry(i, &sp_units, entry) { + if (i->psp_data && i->get_psp_master_device) { + ret = i->get_psp_master_device(); + break; + } + } + +unlock: + write_unlock_irqrestore(&sp_unit_lock, flags); + return ret; +} + +static int __init sp_mod_init(void) +{ +#ifdef CONFIG_X86 + static bool initialized; + int ret; + + if (initialized) + return 0; + + ret = sp_pci_init(); + if (ret) + return ret; + +#ifdef CONFIG_CRYPTO_DEV_SP_PSP + psp_pci_init(); +#endif + + initialized = true; + + return 0; +#endif + +#ifdef CONFIG_ARM64 + int ret; + + ret = sp_platform_init(); + if (ret) + return ret; + + return 0; +#endif + + return -ENODEV; +} + +#if IS_BUILTIN(CONFIG_KVM_AMD) && IS_ENABLED(CONFIG_KVM_AMD_SEV) +int __init sev_module_init(void) +{ + return sp_mod_init(); +} +#endif + +static void __exit sp_mod_exit(void) +{ +#ifdef CONFIG_X86 + +#ifdef CONFIG_CRYPTO_DEV_SP_PSP + psp_pci_exit(); +#endif + + sp_pci_exit(); +#endif + +#ifdef CONFIG_ARM64 + sp_platform_exit(); +#endif +} + +module_init(sp_mod_init); +module_exit(sp_mod_exit); diff --git a/drivers/crypto/ccp/sp-dev.h b/drivers/crypto/ccp/sp-dev.h new file mode 100644 index 000000000..a83751cfd --- /dev/null +++ b/drivers/crypto/ccp/sp-dev.h @@ -0,0 +1,190 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * AMD Secure Processor driver + * + * Copyright (C) 2017-2019 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky + * Author: Gary R Hook + * Author: Brijesh Singh + */ + +#ifndef __SP_DEV_H__ +#define __SP_DEV_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SP_MAX_NAME_LEN 32 + +#define CACHE_NONE 0x00 +#define CACHE_WB_NO_ALLOC 0xb7 + +#define PLATFORM_FEATURE_DBC 0x1 +#define PLATFORM_FEATURE_HSTI 0x2 + +#define PSP_FEATURE(psp, feat) (psp->vdata && psp->vdata->platform_features & PLATFORM_FEATURE_##feat) + +/* Structure to hold CCP device data */ +struct ccp_device; +struct ccp_vdata { + const unsigned int version; + const unsigned int dma_chan_attr; + void (*setup)(struct ccp_device *); + const struct ccp_actions *perform; + const unsigned int offset; + const unsigned int rsamax; +}; + +struct sev_vdata { + const unsigned int cmdresp_reg; + const unsigned int cmdbuff_addr_lo_reg; + const unsigned int cmdbuff_addr_hi_reg; +}; + +struct tee_vdata { + const unsigned int cmdresp_reg; + const unsigned int cmdbuff_addr_lo_reg; + const unsigned int cmdbuff_addr_hi_reg; + const unsigned int ring_wptr_reg; + const unsigned int ring_rptr_reg; + const unsigned int info_reg; +}; + +struct platform_access_vdata { + const unsigned int cmdresp_reg; + const unsigned int cmdbuff_addr_lo_reg; + const unsigned int cmdbuff_addr_hi_reg; + const unsigned int doorbell_button_reg; + const unsigned int doorbell_cmd_reg; + +}; + +struct psp_vdata { + const struct sev_vdata *sev; + const struct tee_vdata *tee; + const struct platform_access_vdata *platform_access; + const unsigned int cmdresp_reg; + const unsigned int cmdbuff_addr_lo_reg; + const unsigned int cmdbuff_addr_hi_reg; + const unsigned int feature_reg; + const unsigned int inten_reg; + const unsigned int intsts_reg; + const unsigned int bootloader_info_reg; + const unsigned int platform_features; +}; + +/* Structure to hold SP device data */ +struct sp_dev_vdata { + const unsigned int bar; + + const struct ccp_vdata *ccp_vdata; + const struct psp_vdata *psp_vdata; +}; + +struct sp_device { + struct list_head entry; + + struct device *dev; + + const struct sp_dev_vdata *dev_vdata; + unsigned int ord; + char name[SP_MAX_NAME_LEN]; + + /* Bus specific device information */ + void *dev_specific; + + /* I/O area used for device communication. */ + void __iomem *io_map; + + /* DMA caching attribute support */ + unsigned int axcache; + + /* get and set master device */ + struct sp_device*(*get_psp_master_device)(void); + void (*set_psp_master_device)(struct sp_device *); + void (*clear_psp_master_device)(struct sp_device *); + + bool irq_registered; + bool use_tasklet; + + unsigned int ccp_irq; + irq_handler_t ccp_irq_handler; + void *ccp_irq_data; + + unsigned int psp_irq; + irq_handler_t psp_irq_handler; + void *psp_irq_data; + + void *ccp_data; + void *psp_data; +}; + +int sp_pci_init(void); +void sp_pci_exit(void); + +int sp_platform_init(void); +void sp_platform_exit(void); + +struct sp_device *sp_alloc_struct(struct device *dev); + +int sp_init(struct sp_device *sp); +void sp_destroy(struct sp_device *sp); + +int sp_suspend(struct sp_device *sp); +int sp_resume(struct sp_device *sp); +int sp_restore(struct sp_device *sp); +int sp_request_ccp_irq(struct sp_device *sp, irq_handler_t handler, + const char *name, void *data); +void sp_free_ccp_irq(struct sp_device *sp, void *data); +int sp_request_psp_irq(struct sp_device *sp, irq_handler_t handler, + const char *name, void *data); +void sp_free_psp_irq(struct sp_device *sp, void *data); +struct sp_device *sp_get_psp_master_device(void); + +#ifdef CONFIG_CRYPTO_DEV_SP_CCP + +int ccp_dev_init(struct sp_device *sp); +void ccp_dev_destroy(struct sp_device *sp); + +void ccp_dev_suspend(struct sp_device *sp); +void ccp_dev_resume(struct sp_device *sp); + +#else /* !CONFIG_CRYPTO_DEV_SP_CCP */ + +static inline int ccp_dev_init(struct sp_device *sp) +{ + return 0; +} +static inline void ccp_dev_destroy(struct sp_device *sp) { } +static inline void ccp_dev_suspend(struct sp_device *sp) { } +static inline void ccp_dev_resume(struct sp_device *sp) { } +#endif /* CONFIG_CRYPTO_DEV_SP_CCP */ + +#ifdef CONFIG_CRYPTO_DEV_SP_PSP + +int psp_dev_init(struct sp_device *sp); +void psp_pci_init(void); +void psp_dev_destroy(struct sp_device *sp); +void psp_pci_exit(void); +int psp_restore(struct sp_device *sp); + +#else /* !CONFIG_CRYPTO_DEV_SP_PSP */ + +static inline int psp_dev_init(struct sp_device *sp) { return 0; } +static inline void psp_pci_init(void) { } +static inline void psp_dev_destroy(struct sp_device *sp) { } +static inline void psp_pci_exit(void) { } +static inline int psp_restore(struct sp_device *sp) { return 0; } + +#endif /* CONFIG_CRYPTO_DEV_SP_PSP */ + +#endif diff --git a/drivers/crypto/ccp/sp-pci.c b/drivers/crypto/ccp/sp-pci.c new file mode 100644 index 000000000..ede6ff9ad --- /dev/null +++ b/drivers/crypto/ccp/sp-pci.c @@ -0,0 +1,600 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Secure Processor device driver + * + * Copyright (C) 2013,2019 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky + * Author: Gary R Hook + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ccp-dev.h" +#include "psp-dev.h" +#include "hsti.h" + +/* used for version string AA.BB.CC.DD */ +#define AA GENMASK(31, 24) +#define BB GENMASK(23, 16) +#define CC GENMASK(15, 8) +#define DD GENMASK(7, 0) + +#define MSIX_VECTORS 2 + +struct sp_pci { + int msix_count; + struct msix_entry msix_entry[MSIX_VECTORS]; +}; +static struct sp_device *sp_dev_master; + +#define version_attribute_show(name, _offset) \ +static ssize_t name##_show(struct device *d, struct device_attribute *attr, \ + char *buf) \ +{ \ + struct sp_device *sp = dev_get_drvdata(d); \ + struct psp_device *psp = sp->psp_data; \ + unsigned int val = ioread32(psp->io_regs + _offset); \ + return sysfs_emit(buf, "%02lx.%02lx.%02lx.%02lx\n", \ + FIELD_GET(AA, val), \ + FIELD_GET(BB, val), \ + FIELD_GET(CC, val), \ + FIELD_GET(DD, val)); \ +} + +version_attribute_show(bootloader_version, psp->vdata->bootloader_info_reg) +static DEVICE_ATTR_RO(bootloader_version); +version_attribute_show(tee_version, psp->vdata->tee->info_reg) +static DEVICE_ATTR_RO(tee_version); + +static struct attribute *psp_firmware_attrs[] = { + &dev_attr_bootloader_version.attr, + &dev_attr_tee_version.attr, + NULL, +}; + +static umode_t psp_firmware_is_visible(struct kobject *kobj, struct attribute *attr, int idx) +{ + struct device *dev = kobj_to_dev(kobj); + struct sp_device *sp = dev_get_drvdata(dev); + struct psp_device *psp = sp->psp_data; + unsigned int val = 0xffffffff; + + if (!psp) + return 0; + + if (attr == &dev_attr_bootloader_version.attr && + psp->vdata->bootloader_info_reg) + val = ioread32(psp->io_regs + psp->vdata->bootloader_info_reg); + + if (attr == &dev_attr_tee_version.attr && psp->capability.tee && + psp->vdata->tee->info_reg) + val = ioread32(psp->io_regs + psp->vdata->tee->info_reg); + + /* If platform disallows accessing this register it will be all f's */ + if (val != 0xffffffff) + return 0444; + + return 0; +} + +static struct attribute_group psp_firmware_attr_group = { + .attrs = psp_firmware_attrs, + .is_visible = psp_firmware_is_visible, +}; + +static const struct attribute_group *psp_groups[] = { +#ifdef CONFIG_CRYPTO_DEV_SP_PSP + &psp_security_attr_group, +#endif + &psp_firmware_attr_group, + NULL, +}; + +static int sp_get_msix_irqs(struct sp_device *sp) +{ + struct sp_pci *sp_pci = sp->dev_specific; + struct device *dev = sp->dev; + struct pci_dev *pdev = to_pci_dev(dev); + int v, ret; + + for (v = 0; v < ARRAY_SIZE(sp_pci->msix_entry); v++) + sp_pci->msix_entry[v].entry = v; + + ret = pci_enable_msix_range(pdev, sp_pci->msix_entry, 1, v); + if (ret < 0) + return ret; + + sp_pci->msix_count = ret; + sp->use_tasklet = true; + + sp->psp_irq = sp_pci->msix_entry[0].vector; + sp->ccp_irq = (sp_pci->msix_count > 1) ? sp_pci->msix_entry[1].vector + : sp_pci->msix_entry[0].vector; + return 0; +} + +static int sp_get_msi_irq(struct sp_device *sp) +{ + struct device *dev = sp->dev; + struct pci_dev *pdev = to_pci_dev(dev); + int ret; + + ret = pci_enable_msi(pdev); + if (ret) + return ret; + + sp->ccp_irq = pdev->irq; + sp->psp_irq = pdev->irq; + + return 0; +} + +static int sp_get_irqs(struct sp_device *sp) +{ + struct device *dev = sp->dev; + int ret; + + ret = sp_get_msix_irqs(sp); + if (!ret) + return 0; + + /* Couldn't get MSI-X vectors, try MSI */ + dev_notice(dev, "could not enable MSI-X (%d), trying MSI\n", ret); + ret = sp_get_msi_irq(sp); + if (!ret) + return 0; + + /* Couldn't get MSI interrupt */ + dev_notice(dev, "could not enable MSI (%d)\n", ret); + + return ret; +} + +static void sp_free_irqs(struct sp_device *sp) +{ + struct sp_pci *sp_pci = sp->dev_specific; + struct device *dev = sp->dev; + struct pci_dev *pdev = to_pci_dev(dev); + + if (sp_pci->msix_count) + pci_disable_msix(pdev); + else if (sp->psp_irq) + pci_disable_msi(pdev); + + sp->ccp_irq = 0; + sp->psp_irq = 0; +} + +static bool sp_pci_is_master(struct sp_device *sp) +{ + struct device *dev_cur, *dev_new; + struct pci_dev *pdev_cur, *pdev_new; + + dev_new = sp->dev; + dev_cur = sp_dev_master->dev; + + pdev_new = to_pci_dev(dev_new); + pdev_cur = to_pci_dev(dev_cur); + + if (pci_domain_nr(pdev_new->bus) != pci_domain_nr(pdev_cur->bus)) + return pci_domain_nr(pdev_new->bus) < pci_domain_nr(pdev_cur->bus); + + if (pdev_new->bus->number != pdev_cur->bus->number) + return pdev_new->bus->number < pdev_cur->bus->number; + + if (PCI_SLOT(pdev_new->devfn) != PCI_SLOT(pdev_cur->devfn)) + return PCI_SLOT(pdev_new->devfn) < PCI_SLOT(pdev_cur->devfn); + + if (PCI_FUNC(pdev_new->devfn) != PCI_FUNC(pdev_cur->devfn)) + return PCI_FUNC(pdev_new->devfn) < PCI_FUNC(pdev_cur->devfn); + + return false; +} + +static void psp_set_master(struct sp_device *sp) +{ + if (!sp_dev_master) { + sp_dev_master = sp; + return; + } + + if (sp_pci_is_master(sp)) + sp_dev_master = sp; +} + +static struct sp_device *psp_get_master(void) +{ + return sp_dev_master; +} + +static void psp_clear_master(struct sp_device *sp) +{ + if (sp == sp_dev_master) { + sp_dev_master = NULL; + dev_dbg(sp->dev, "Cleared sp_dev_master\n"); + } +} + +static int sp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct sp_device *sp; + struct sp_pci *sp_pci; + struct device *dev = &pdev->dev; + void __iomem * const *iomap_table; + int bar_mask; + int ret; + + ret = -ENOMEM; + sp = sp_alloc_struct(dev); + if (!sp) + goto e_err; + + sp_pci = devm_kzalloc(dev, sizeof(*sp_pci), GFP_KERNEL); + if (!sp_pci) + goto e_err; + + sp->dev_specific = sp_pci; + sp->dev_vdata = (struct sp_dev_vdata *)id->driver_data; + if (!sp->dev_vdata) { + ret = -ENODEV; + dev_err(dev, "missing driver data\n"); + goto e_err; + } + + ret = pcim_enable_device(pdev); + if (ret) { + dev_err(dev, "pcim_enable_device failed (%d)\n", ret); + goto e_err; + } + + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); + ret = pcim_iomap_regions(pdev, bar_mask, "ccp"); + if (ret) { + dev_err(dev, "pcim_iomap_regions failed (%d)\n", ret); + goto e_err; + } + + iomap_table = pcim_iomap_table(pdev); + if (!iomap_table) { + dev_err(dev, "pcim_iomap_table failed\n"); + ret = -ENOMEM; + goto e_err; + } + + sp->io_map = iomap_table[sp->dev_vdata->bar]; + if (!sp->io_map) { + dev_err(dev, "ioremap failed\n"); + ret = -ENOMEM; + goto e_err; + } + + ret = sp_get_irqs(sp); + if (ret) + goto e_err; + + pci_set_master(pdev); + sp->set_psp_master_device = psp_set_master; + sp->get_psp_master_device = psp_get_master; + sp->clear_psp_master_device = psp_clear_master; + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); + if (ret) { + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + if (ret) { + dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", + ret); + goto free_irqs; + } + } + + dev_set_drvdata(dev, sp); + + ret = sp_init(sp); + if (ret) + goto free_irqs; + + return 0; + +free_irqs: + sp_free_irqs(sp); +e_err: + dev_notice(dev, "initialization failed\n"); + return ret; +} + +static void sp_pci_shutdown(struct pci_dev *pdev) +{ + struct device *dev = &pdev->dev; + struct sp_device *sp = dev_get_drvdata(dev); + + if (!sp) + return; + + sp_destroy(sp); +} + +static void sp_pci_remove(struct pci_dev *pdev) +{ + struct device *dev = &pdev->dev; + struct sp_device *sp = dev_get_drvdata(dev); + + if (!sp) + return; + + sp_destroy(sp); + + sp_free_irqs(sp); +} + +static int __maybe_unused sp_pci_suspend(struct device *dev) +{ + struct sp_device *sp = dev_get_drvdata(dev); + + return sp_suspend(sp); +} + +static int __maybe_unused sp_pci_resume(struct device *dev) +{ + struct sp_device *sp = dev_get_drvdata(dev); + + return sp_resume(sp); +} + +static int __maybe_unused sp_pci_restore(struct device *dev) +{ + struct sp_device *sp = dev_get_drvdata(dev); + + return sp_restore(sp); +} + +#ifdef CONFIG_CRYPTO_DEV_SP_PSP +static const struct sev_vdata sevv1 = { + .cmdresp_reg = 0x10580, /* C2PMSG_32 */ + .cmdbuff_addr_lo_reg = 0x105e0, /* C2PMSG_56 */ + .cmdbuff_addr_hi_reg = 0x105e4, /* C2PMSG_57 */ +}; + +static const struct sev_vdata sevv2 = { + .cmdresp_reg = 0x10980, /* C2PMSG_32 */ + .cmdbuff_addr_lo_reg = 0x109e0, /* C2PMSG_56 */ + .cmdbuff_addr_hi_reg = 0x109e4, /* C2PMSG_57 */ +}; + +static const struct tee_vdata teev1 = { + .ring_wptr_reg = 0x10550, /* C2PMSG_20 */ + .ring_rptr_reg = 0x10554, /* C2PMSG_21 */ + .info_reg = 0x109e8, /* C2PMSG_58 */ +}; + +static const struct tee_vdata teev2 = { + .ring_wptr_reg = 0x10950, /* C2PMSG_20 */ + .ring_rptr_reg = 0x10954, /* C2PMSG_21 */ + .info_reg = 0x109e8, /* C2PMSG_58 */ +}; + +static const struct platform_access_vdata pa_v1 = { + .cmdresp_reg = 0x10570, /* C2PMSG_28 */ + .cmdbuff_addr_lo_reg = 0x10574, /* C2PMSG_29 */ + .cmdbuff_addr_hi_reg = 0x10578, /* C2PMSG_30 */ + .doorbell_button_reg = 0x10a24, /* C2PMSG_73 */ + .doorbell_cmd_reg = 0x10a40, /* C2PMSG_80 */ +}; + +static const struct platform_access_vdata pa_v2 = { + .doorbell_button_reg = 0x10a24, /* C2PMSG_73 */ + .doorbell_cmd_reg = 0x10a40, /* C2PMSG_80 */ +}; + +static const struct psp_vdata pspv1 = { + .sev = &sevv1, + .bootloader_info_reg = 0x105ec, /* C2PMSG_59 */ + .feature_reg = 0x105fc, /* C2PMSG_63 */ + .inten_reg = 0x10610, /* P2CMSG_INTEN */ + .intsts_reg = 0x10614, /* P2CMSG_INTSTS */ +}; + +static const struct psp_vdata pspv2 = { + .sev = &sevv2, + .platform_access = &pa_v1, + .bootloader_info_reg = 0x109ec, /* C2PMSG_59 */ + .feature_reg = 0x109fc, /* C2PMSG_63 */ + .inten_reg = 0x10690, /* P2CMSG_INTEN */ + .intsts_reg = 0x10694, /* P2CMSG_INTSTS */ + .platform_features = PLATFORM_FEATURE_HSTI, +}; + +static const struct psp_vdata pspv3 = { + .tee = &teev1, + .platform_access = &pa_v1, + .cmdresp_reg = 0x10544, /* C2PMSG_17 */ + .cmdbuff_addr_lo_reg = 0x10548, /* C2PMSG_18 */ + .cmdbuff_addr_hi_reg = 0x1054c, /* C2PMSG_19 */ + .bootloader_info_reg = 0x109ec, /* C2PMSG_59 */ + .feature_reg = 0x109fc, /* C2PMSG_63 */ + .inten_reg = 0x10690, /* P2CMSG_INTEN */ + .intsts_reg = 0x10694, /* P2CMSG_INTSTS */ + .platform_features = PLATFORM_FEATURE_DBC | + PLATFORM_FEATURE_HSTI, +}; + +static const struct psp_vdata pspv4 = { + .sev = &sevv2, + .tee = &teev1, + .cmdresp_reg = 0x10544, /* C2PMSG_17 */ + .cmdbuff_addr_lo_reg = 0x10548, /* C2PMSG_18 */ + .cmdbuff_addr_hi_reg = 0x1054c, /* C2PMSG_19 */ + .bootloader_info_reg = 0x109ec, /* C2PMSG_59 */ + .feature_reg = 0x109fc, /* C2PMSG_63 */ + .inten_reg = 0x10690, /* P2CMSG_INTEN */ + .intsts_reg = 0x10694, /* P2CMSG_INTSTS */ +}; + +static const struct psp_vdata pspv5 = { + .tee = &teev2, + .platform_access = &pa_v2, + .cmdresp_reg = 0x10944, /* C2PMSG_17 */ + .cmdbuff_addr_lo_reg = 0x10948, /* C2PMSG_18 */ + .cmdbuff_addr_hi_reg = 0x1094c, /* C2PMSG_19 */ + .bootloader_info_reg = 0x109ec, /* C2PMSG_59 */ + .feature_reg = 0x109fc, /* C2PMSG_63 */ + .inten_reg = 0x10510, /* P2CMSG_INTEN */ + .intsts_reg = 0x10514, /* P2CMSG_INTSTS */ +}; + +static const struct psp_vdata pspv6 = { + .sev = &sevv2, + .tee = &teev2, + .cmdresp_reg = 0x10944, /* C2PMSG_17 */ + .cmdbuff_addr_lo_reg = 0x10948, /* C2PMSG_18 */ + .cmdbuff_addr_hi_reg = 0x1094c, /* C2PMSG_19 */ + .bootloader_info_reg = 0x109ec, /* C2PMSG_59 */ + .feature_reg = 0x109fc, /* C2PMSG_63 */ + .inten_reg = 0x10510, /* P2CMSG_INTEN */ + .intsts_reg = 0x10514, /* P2CMSG_INTSTS */ +}; + +static const struct psp_vdata pspv7 = { + .tee = &teev2, + .cmdresp_reg = 0x10944, /* C2PMSG_17 */ + .cmdbuff_addr_lo_reg = 0x10948, /* C2PMSG_18 */ + .cmdbuff_addr_hi_reg = 0x1094c, /* C2PMSG_19 */ + .bootloader_info_reg = 0x109ec, /* C2PMSG_59 */ + .feature_reg = 0x109fc, /* C2PMSG_63 */ + .inten_reg = 0x10510, /* P2CMSG_INTEN */ + .intsts_reg = 0x10514, /* P2CMSG_INTSTS */ +}; + +#endif + +static const struct sp_dev_vdata dev_vdata[] = { + { /* 0 */ + .bar = 2, +#ifdef CONFIG_CRYPTO_DEV_SP_CCP + .ccp_vdata = &ccpv3, +#endif + }, + { /* 1 */ + .bar = 2, +#ifdef CONFIG_CRYPTO_DEV_SP_CCP + .ccp_vdata = &ccpv5a, +#endif +#ifdef CONFIG_CRYPTO_DEV_SP_PSP + .psp_vdata = &pspv1, +#endif + }, + { /* 2 */ + .bar = 2, +#ifdef CONFIG_CRYPTO_DEV_SP_CCP + .ccp_vdata = &ccpv5b, +#endif + }, + { /* 3 */ + .bar = 2, +#ifdef CONFIG_CRYPTO_DEV_SP_CCP + .ccp_vdata = &ccpv5a, +#endif +#ifdef CONFIG_CRYPTO_DEV_SP_PSP + .psp_vdata = &pspv2, +#endif + }, + { /* 4 */ + .bar = 2, +#ifdef CONFIG_CRYPTO_DEV_SP_CCP + .ccp_vdata = &ccpv5a, +#endif +#ifdef CONFIG_CRYPTO_DEV_SP_PSP + .psp_vdata = &pspv3, +#endif + }, + { /* 5 */ + .bar = 2, +#ifdef CONFIG_CRYPTO_DEV_SP_PSP + .psp_vdata = &pspv4, +#endif + }, + { /* 6 */ + .bar = 2, +#ifdef CONFIG_CRYPTO_DEV_SP_PSP + .psp_vdata = &pspv3, +#endif + }, + { /* 7 */ + .bar = 2, +#ifdef CONFIG_CRYPTO_DEV_SP_PSP + .psp_vdata = &pspv5, +#endif + }, + { /* 8 */ + .bar = 2, +#ifdef CONFIG_CRYPTO_DEV_SP_PSP + .psp_vdata = &pspv6, +#endif + }, + { /* 9 */ + .bar = 2, +#ifdef CONFIG_CRYPTO_DEV_SP_PSP + .psp_vdata = &pspv7, +#endif + }, + +}; +static const struct pci_device_id sp_pci_table[] = { + { PCI_VDEVICE(AMD, 0x1537), .driver_data = (kernel_ulong_t)&dev_vdata[0] }, + { PCI_VDEVICE(AMD, 0x1456), .driver_data = (kernel_ulong_t)&dev_vdata[1] }, + { PCI_VDEVICE(AMD, 0x1468), .driver_data = (kernel_ulong_t)&dev_vdata[2] }, + { PCI_VDEVICE(AMD, 0x1486), .driver_data = (kernel_ulong_t)&dev_vdata[3] }, + { PCI_VDEVICE(AMD, 0x15DF), .driver_data = (kernel_ulong_t)&dev_vdata[4] }, + { PCI_VDEVICE(AMD, 0x14CA), .driver_data = (kernel_ulong_t)&dev_vdata[5] }, + { PCI_VDEVICE(AMD, 0x15C7), .driver_data = (kernel_ulong_t)&dev_vdata[6] }, + { PCI_VDEVICE(AMD, 0x1649), .driver_data = (kernel_ulong_t)&dev_vdata[6] }, + { PCI_VDEVICE(AMD, 0x1134), .driver_data = (kernel_ulong_t)&dev_vdata[7] }, + { PCI_VDEVICE(AMD, 0x17E0), .driver_data = (kernel_ulong_t)&dev_vdata[7] }, + { PCI_VDEVICE(AMD, 0x156E), .driver_data = (kernel_ulong_t)&dev_vdata[8] }, + { PCI_VDEVICE(AMD, 0x17D8), .driver_data = (kernel_ulong_t)&dev_vdata[8] }, + { PCI_VDEVICE(AMD, 0x115A), .driver_data = (kernel_ulong_t)&dev_vdata[9] }, + /* Last entry must be zero */ + { } +}; +MODULE_DEVICE_TABLE(pci, sp_pci_table); + +static const struct dev_pm_ops sp_pci_pm_ops = { + .suspend = pm_sleep_ptr(sp_pci_suspend), + .resume = pm_sleep_ptr(sp_pci_resume), + .freeze = pm_sleep_ptr(sp_pci_suspend), + .thaw = pm_sleep_ptr(sp_pci_resume), + .poweroff = pm_sleep_ptr(sp_pci_suspend), + .restore_early = pm_sleep_ptr(sp_pci_restore), +}; + +static struct pci_driver sp_pci_driver = { + .name = "ccp", + .id_table = sp_pci_table, + .probe = sp_pci_probe, + .remove = sp_pci_remove, + .shutdown = sp_pci_shutdown, + .driver.pm = &sp_pci_pm_ops, + .dev_groups = psp_groups, +}; + +int sp_pci_init(void) +{ + return pci_register_driver(&sp_pci_driver); +} + +void sp_pci_exit(void) +{ + pci_unregister_driver(&sp_pci_driver); +} diff --git a/drivers/crypto/ccp/sp-platform.c b/drivers/crypto/ccp/sp-platform.c new file mode 100644 index 000000000..3f9843fa7 --- /dev/null +++ b/drivers/crypto/ccp/sp-platform.c @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Secure Processor device driver + * + * Copyright (C) 2014,2018 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ccp-dev.h" + +struct sp_platform { + int coherent; + unsigned int irq_count; +}; + +static const struct sp_dev_vdata dev_vdata[] = { + { + .bar = 0, +#ifdef CONFIG_CRYPTO_DEV_SP_CCP + .ccp_vdata = &ccpv3_platform, +#endif + }, +}; + +static const struct acpi_device_id sp_acpi_match[] = { + { "AMDI0C00", (kernel_ulong_t)&dev_vdata[0] }, + { }, +}; +MODULE_DEVICE_TABLE(acpi, sp_acpi_match); + +static const struct of_device_id sp_of_match[] = { + { .compatible = "amd,ccp-seattle-v1a", + .data = (const void *)&dev_vdata[0] }, + { }, +}; +MODULE_DEVICE_TABLE(of, sp_of_match); + +static const struct sp_dev_vdata *sp_get_acpi_version(struct platform_device *pdev) +{ + const struct acpi_device_id *match; + + match = acpi_match_device(sp_acpi_match, &pdev->dev); + if (match && match->driver_data) + return (const struct sp_dev_vdata *)match->driver_data; + + return NULL; +} + +static int sp_get_irqs(struct sp_device *sp) +{ + struct sp_platform *sp_platform = sp->dev_specific; + struct device *dev = sp->dev; + struct platform_device *pdev = to_platform_device(dev); + int ret; + + sp_platform->irq_count = platform_irq_count(pdev); + + ret = platform_get_irq(pdev, 0); + if (ret < 0) { + dev_notice(dev, "unable to get IRQ (%d)\n", ret); + return ret; + } + + sp->psp_irq = ret; + if (sp_platform->irq_count == 1) { + sp->ccp_irq = ret; + } else { + ret = platform_get_irq(pdev, 1); + if (ret < 0) { + dev_notice(dev, "unable to get IRQ (%d)\n", ret); + return ret; + } + + sp->ccp_irq = ret; + } + + return 0; +} + +static int sp_platform_probe(struct platform_device *pdev) +{ + struct sp_device *sp; + struct sp_platform *sp_platform; + struct device *dev = &pdev->dev; + enum dev_dma_attr attr; + int ret; + + ret = -ENOMEM; + sp = sp_alloc_struct(dev); + if (!sp) + goto e_err; + + sp_platform = devm_kzalloc(dev, sizeof(*sp_platform), GFP_KERNEL); + if (!sp_platform) + goto e_err; + + sp->dev_specific = sp_platform; + sp->dev_vdata = pdev->dev.of_node ? of_device_get_match_data(&pdev->dev) + : sp_get_acpi_version(pdev); + if (!sp->dev_vdata) { + ret = -ENODEV; + dev_err(dev, "missing driver data\n"); + goto e_err; + } + + sp->io_map = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(sp->io_map)) { + ret = PTR_ERR(sp->io_map); + goto e_err; + } + + attr = device_get_dma_attr(dev); + if (attr == DEV_DMA_NOT_SUPPORTED) { + dev_err(dev, "DMA is not supported"); + goto e_err; + } + + sp_platform->coherent = (attr == DEV_DMA_COHERENT); + if (sp_platform->coherent) + sp->axcache = CACHE_WB_NO_ALLOC; + else + sp->axcache = CACHE_NONE; + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); + if (ret) { + dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", ret); + goto e_err; + } + + ret = sp_get_irqs(sp); + if (ret) + goto e_err; + + dev_set_drvdata(dev, sp); + + ret = sp_init(sp); + if (ret) + goto e_err; + + dev_notice(dev, "enabled\n"); + + return 0; + +e_err: + dev_notice(dev, "initialization failed\n"); + return ret; +} + +static void sp_platform_remove(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct sp_device *sp = dev_get_drvdata(dev); + + sp_destroy(sp); + + dev_notice(dev, "disabled\n"); +} + +#ifdef CONFIG_PM +static int sp_platform_suspend(struct platform_device *pdev, + pm_message_t state) +{ + struct device *dev = &pdev->dev; + struct sp_device *sp = dev_get_drvdata(dev); + + return sp_suspend(sp); +} + +static int sp_platform_resume(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct sp_device *sp = dev_get_drvdata(dev); + + return sp_resume(sp); +} +#endif + +static struct platform_driver sp_platform_driver = { + .driver = { + .name = "ccp", + .acpi_match_table = sp_acpi_match, + .of_match_table = sp_of_match, + }, + .probe = sp_platform_probe, + .remove = sp_platform_remove, +#ifdef CONFIG_PM + .suspend = sp_platform_suspend, + .resume = sp_platform_resume, +#endif +}; + +int sp_platform_init(void) +{ + return platform_driver_register(&sp_platform_driver); +} + +void sp_platform_exit(void) +{ + platform_driver_unregister(&sp_platform_driver); +} diff --git a/drivers/crypto/ccp/tee-dev.c b/drivers/crypto/ccp/tee-dev.c new file mode 100644 index 000000000..3e3645980 --- /dev/null +++ b/drivers/crypto/ccp/tee-dev.c @@ -0,0 +1,399 @@ +// SPDX-License-Identifier: MIT +/* + * AMD Trusted Execution Environment (TEE) interface + * + * Author: Rijo Thomas + * Author: Devaraj Rangasamy + * + * Copyright (C) 2019,2021 Advanced Micro Devices, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "psp-dev.h" +#include "tee-dev.h" + +static bool psp_dead; + +static int tee_alloc_ring(struct psp_tee_device *tee, int ring_size) +{ + struct ring_buf_manager *rb_mgr = &tee->rb_mgr; + void *start_addr; + + if (!ring_size) + return -EINVAL; + + /* We need actual physical address instead of DMA address, since + * Trusted OS running on AMD Secure Processor will map this region + */ + start_addr = (void *)__get_free_pages(GFP_KERNEL, get_order(ring_size)); + if (!start_addr) + return -ENOMEM; + + memset(start_addr, 0x0, ring_size); + rb_mgr->ring_start = start_addr; + rb_mgr->ring_size = ring_size; + rb_mgr->ring_pa = __psp_pa(start_addr); + mutex_init(&rb_mgr->mutex); + + return 0; +} + +static void tee_free_ring(struct psp_tee_device *tee) +{ + struct ring_buf_manager *rb_mgr = &tee->rb_mgr; + + if (!rb_mgr->ring_start) + return; + + free_pages((unsigned long)rb_mgr->ring_start, + get_order(rb_mgr->ring_size)); + + rb_mgr->ring_start = NULL; + rb_mgr->ring_size = 0; + rb_mgr->ring_pa = 0; + mutex_destroy(&rb_mgr->mutex); +} + +static +struct tee_init_ring_cmd *tee_alloc_cmd_buffer(struct psp_tee_device *tee) +{ + struct tee_init_ring_cmd *cmd; + + cmd = kzalloc_obj(*cmd); + if (!cmd) + return NULL; + + cmd->hi_addr = upper_32_bits(tee->rb_mgr.ring_pa); + cmd->low_addr = lower_32_bits(tee->rb_mgr.ring_pa); + cmd->size = tee->rb_mgr.ring_size; + + dev_dbg(tee->dev, "tee: ring address: high = 0x%x low = 0x%x size = %u\n", + cmd->hi_addr, cmd->low_addr, cmd->size); + + return cmd; +} + +static inline void tee_free_cmd_buffer(struct tee_init_ring_cmd *cmd) +{ + kfree(cmd); +} + +static bool tee_send_destroy_cmd(struct psp_tee_device *tee) +{ + unsigned int reg; + int ret; + + ret = psp_mailbox_command(tee->psp, PSP_CMD_TEE_RING_DESTROY, NULL, + TEE_DEFAULT_CMD_TIMEOUT, ®); + if (ret) { + dev_err(tee->dev, "tee: ring destroy command timed out, disabling TEE support\n"); + psp_dead = true; + return false; + } + + if (FIELD_GET(PSP_CMDRESP_STS, reg)) { + dev_err(tee->dev, "tee: ring destroy command failed (%#010lx)\n", + FIELD_GET(PSP_CMDRESP_STS, reg)); + psp_dead = true; + return false; + } + + return true; +} + +static int tee_init_ring(struct psp_tee_device *tee) +{ + int ring_size = MAX_RING_BUFFER_ENTRIES * sizeof(struct tee_ring_cmd); + struct tee_init_ring_cmd *cmd; + bool retry = false; + unsigned int reg; + int ret; + + BUILD_BUG_ON(sizeof(struct tee_ring_cmd) != 1024); + + ret = tee_alloc_ring(tee, ring_size); + if (ret) { + dev_err(tee->dev, "tee: ring allocation failed %d\n", ret); + return ret; + } + + tee->rb_mgr.wptr = 0; + + cmd = tee_alloc_cmd_buffer(tee); + if (!cmd) { + tee_free_ring(tee); + return -ENOMEM; + } + + /* Send command buffer details to Trusted OS by writing to + * CPU-PSP message registers + */ +retry_init: + ret = psp_mailbox_command(tee->psp, PSP_CMD_TEE_RING_INIT, cmd, + TEE_DEFAULT_CMD_TIMEOUT, ®); + if (ret) { + dev_err(tee->dev, "tee: ring init command timed out, disabling TEE support\n"); + tee_free_ring(tee); + psp_dead = true; + goto free_buf; + } + + if (FIELD_GET(PSP_CMDRESP_STS, reg)) { + /* + * During the hibernate resume sequence driver may have gotten loaded + * but the ring not properly destroyed. If the ring doesn't work, try + * to destroy and re-init once. + */ + if (!retry && FIELD_GET(PSP_CMDRESP_STS, reg) == PSP_TEE_STS_RING_BUSY) { + dev_info(tee->dev, "tee: ring init command failed with busy status, retrying\n"); + if (tee_send_destroy_cmd(tee)) { + retry = true; + goto retry_init; + } + } + dev_err(tee->dev, "tee: ring init command failed (%#010lx)\n", + FIELD_GET(PSP_CMDRESP_STS, reg)); + tee_free_ring(tee); + psp_dead = true; + ret = -EIO; + } + +free_buf: + tee_free_cmd_buffer(cmd); + + return ret; +} + +static void tee_destroy_ring(struct psp_tee_device *tee) +{ + if (!tee->rb_mgr.ring_start) + return; + + if (psp_dead) + goto free_ring; + + tee_send_destroy_cmd(tee); + +free_ring: + tee_free_ring(tee); +} + +int tee_dev_init(struct psp_device *psp) +{ + struct device *dev = psp->dev; + struct psp_tee_device *tee; + int ret; + + ret = -ENOMEM; + tee = devm_kzalloc(dev, sizeof(*tee), GFP_KERNEL); + if (!tee) + goto e_err; + + psp->tee_data = tee; + + tee->dev = dev; + tee->psp = psp; + + tee->io_regs = psp->io_regs; + + tee->vdata = (struct tee_vdata *)psp->vdata->tee; + if (!tee->vdata) { + ret = -ENODEV; + dev_err(dev, "tee: missing driver data\n"); + goto e_err; + } + + ret = tee_init_ring(tee); + if (ret) { + dev_err(dev, "tee: failed to init ring buffer\n"); + goto e_err; + } + + dev_notice(dev, "tee enabled\n"); + + return 0; + +e_err: + psp->tee_data = NULL; + + dev_notice(dev, "tee initialization failed\n"); + + return ret; +} + +void tee_dev_destroy(struct psp_device *psp) +{ + struct psp_tee_device *tee = psp->tee_data; + + if (!tee) + return; + + tee_destroy_ring(tee); +} + +static int tee_submit_cmd(struct psp_tee_device *tee, enum tee_cmd_id cmd_id, + void *buf, size_t len, struct tee_ring_cmd **resp) +{ + struct tee_ring_cmd *cmd; + int nloop = 1000, ret = 0; + u32 rptr; + + *resp = NULL; + + mutex_lock(&tee->rb_mgr.mutex); + + /* Loop until empty entry found in ring buffer */ + do { + /* Get pointer to ring buffer command entry */ + cmd = (struct tee_ring_cmd *) + (tee->rb_mgr.ring_start + tee->rb_mgr.wptr); + + rptr = ioread32(tee->io_regs + tee->vdata->ring_rptr_reg); + + /* Check if ring buffer is full or command entry is waiting + * for response from TEE + */ + if (!(tee->rb_mgr.wptr + sizeof(struct tee_ring_cmd) == rptr || + cmd->flag == CMD_WAITING_FOR_RESPONSE)) + break; + + dev_dbg(tee->dev, "tee: ring buffer full. rptr = %u wptr = %u\n", + rptr, tee->rb_mgr.wptr); + + /* Wait if ring buffer is full or TEE is processing data */ + mutex_unlock(&tee->rb_mgr.mutex); + schedule_timeout_interruptible(msecs_to_jiffies(10)); + mutex_lock(&tee->rb_mgr.mutex); + + } while (--nloop); + + if (!nloop && + (tee->rb_mgr.wptr + sizeof(struct tee_ring_cmd) == rptr || + cmd->flag == CMD_WAITING_FOR_RESPONSE)) { + dev_err(tee->dev, "tee: ring buffer full. rptr = %u wptr = %u response flag %u\n", + rptr, tee->rb_mgr.wptr, cmd->flag); + ret = -EBUSY; + goto unlock; + } + + /* Do not submit command if PSP got disabled while processing any + * command in another thread + */ + if (psp_dead) { + ret = -EBUSY; + goto unlock; + } + + /* Write command data into ring buffer */ + cmd->cmd_id = cmd_id; + cmd->cmd_state = TEE_CMD_STATE_INIT; + memset(&cmd->buf[0], 0, sizeof(cmd->buf)); + memcpy(&cmd->buf[0], buf, len); + + /* Indicate driver is waiting for response */ + cmd->flag = CMD_WAITING_FOR_RESPONSE; + + /* Update local copy of write pointer */ + tee->rb_mgr.wptr += sizeof(struct tee_ring_cmd); + if (tee->rb_mgr.wptr >= tee->rb_mgr.ring_size) + tee->rb_mgr.wptr = 0; + + /* Trigger interrupt to Trusted OS */ + iowrite32(tee->rb_mgr.wptr, tee->io_regs + tee->vdata->ring_wptr_reg); + + /* The response is provided by Trusted OS in same + * location as submitted data entry within ring buffer. + */ + *resp = cmd; + +unlock: + mutex_unlock(&tee->rb_mgr.mutex); + + return ret; +} + +static int tee_wait_cmd_completion(struct psp_tee_device *tee, + struct tee_ring_cmd *resp, + unsigned int timeout) +{ + /* ~1ms sleep per loop => nloop = timeout * 1000 */ + int nloop = timeout * 1000; + + while (--nloop) { + if (resp->cmd_state == TEE_CMD_STATE_COMPLETED) + return 0; + + usleep_range(1000, 1100); + } + + dev_err(tee->dev, "tee: command 0x%x timed out, disabling PSP\n", + resp->cmd_id); + + psp_dead = true; + + return -ETIMEDOUT; +} + +int psp_tee_process_cmd(enum tee_cmd_id cmd_id, void *buf, size_t len, + u32 *status) +{ + struct psp_device *psp = psp_get_master_device(); + struct psp_tee_device *tee; + struct tee_ring_cmd *resp; + int ret; + + if (!buf || !status || !len || len > sizeof(resp->buf)) + return -EINVAL; + + *status = 0; + + if (!psp || !psp->tee_data) + return -ENODEV; + + if (psp_dead) + return -EBUSY; + + tee = psp->tee_data; + + ret = tee_submit_cmd(tee, cmd_id, buf, len, &resp); + if (ret) + return ret; + + ret = tee_wait_cmd_completion(tee, resp, TEE_DEFAULT_RING_TIMEOUT); + if (ret) { + resp->flag = CMD_RESPONSE_TIMEDOUT; + return ret; + } + + memcpy(buf, &resp->buf[0], len); + *status = resp->status; + + resp->flag = CMD_RESPONSE_COPIED; + + return 0; +} +EXPORT_SYMBOL(psp_tee_process_cmd); + +int psp_check_tee_status(void) +{ + struct psp_device *psp = psp_get_master_device(); + + if (!psp || !psp->tee_data) + return -ENODEV; + + return 0; +} +EXPORT_SYMBOL(psp_check_tee_status); + +int tee_restore(struct psp_device *psp) +{ + return tee_init_ring(psp->tee_data); +} diff --git a/drivers/crypto/ccp/tee-dev.h b/drivers/crypto/ccp/tee-dev.h new file mode 100644 index 000000000..c23416cb7 --- /dev/null +++ b/drivers/crypto/ccp/tee-dev.h @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright (C) 2019,2021 Advanced Micro Devices, Inc. + * + * Author: Rijo Thomas + * Author: Devaraj Rangasamy + * + */ + +/* This file describes the TEE communication interface between host and AMD + * Secure Processor + */ + +#ifndef __TEE_DEV_H__ +#define __TEE_DEV_H__ + +#include +#include + +#define TEE_DEFAULT_CMD_TIMEOUT (10 * MSEC_PER_SEC) +#define TEE_DEFAULT_RING_TIMEOUT 10 +#define MAX_BUFFER_SIZE 988 + +/** + * struct tee_init_ring_cmd - Command to init TEE ring buffer + * @low_addr: bits [31:0] of the physical address of ring buffer + * @hi_addr: bits [63:32] of the physical address of ring buffer + * @size: size of ring buffer in bytes + */ +struct tee_init_ring_cmd { + u32 low_addr; + u32 hi_addr; + u32 size; +}; + +#define MAX_RING_BUFFER_ENTRIES 32 + +/** + * struct ring_buf_manager - Helper structure to manage ring buffer. + * @ring_start: starting address of ring buffer + * @ring_size: size of ring buffer in bytes + * @ring_pa: physical address of ring buffer + * @wptr: index to the last written entry in ring buffer + */ +struct ring_buf_manager { + struct mutex mutex; /* synchronizes access to ring buffer */ + void *ring_start; + u32 ring_size; + phys_addr_t ring_pa; + u32 wptr; +}; + +struct psp_tee_device { + struct device *dev; + struct psp_device *psp; + void __iomem *io_regs; + struct tee_vdata *vdata; + struct ring_buf_manager rb_mgr; +}; + +/** + * enum tee_cmd_state - TEE command states for the ring buffer interface + * @TEE_CMD_STATE_INIT: initial state of command when sent from host + * @TEE_CMD_STATE_PROCESS: command being processed by TEE environment + * @TEE_CMD_STATE_COMPLETED: command processing completed + */ +enum tee_cmd_state { + TEE_CMD_STATE_INIT, + TEE_CMD_STATE_PROCESS, + TEE_CMD_STATE_COMPLETED, +}; + +/** + * enum cmd_resp_state - TEE command's response status maintained by driver + * @CMD_RESPONSE_INVALID: initial state when no command is written to ring + * @CMD_WAITING_FOR_RESPONSE: driver waiting for response from TEE + * @CMD_RESPONSE_TIMEDOUT: failed to get response from TEE + * @CMD_RESPONSE_COPIED: driver has copied response from TEE + */ +enum cmd_resp_state { + CMD_RESPONSE_INVALID, + CMD_WAITING_FOR_RESPONSE, + CMD_RESPONSE_TIMEDOUT, + CMD_RESPONSE_COPIED, +}; + +/** + * struct tee_ring_cmd - Structure of the command buffer in TEE ring + * @cmd_id: refers to &enum tee_cmd_id. Command id for the ring buffer + * interface + * @cmd_state: refers to &enum tee_cmd_state + * @status: status of TEE command execution + * @res0: reserved region + * @pdata: private data (currently unused) + * @res1: reserved region + * @buf: TEE command specific buffer + * @flag: refers to &enum cmd_resp_state + */ +struct tee_ring_cmd { + u32 cmd_id; + u32 cmd_state; + u32 status; + u32 res0[1]; + u64 pdata; + u32 res1[2]; + u8 buf[MAX_BUFFER_SIZE]; + u32 flag; + + /* Total size: 1024 bytes */ +} __packed; + +int tee_dev_init(struct psp_device *psp); +void tee_dev_destroy(struct psp_device *psp); +int tee_restore(struct psp_device *psp); + +#endif /* __TEE_DEV_H__ */ diff --git a/drivers/crypto/ccree/Makefile b/drivers/crypto/ccree/Makefile new file mode 100644 index 000000000..5cfda508e --- /dev/null +++ b/drivers/crypto/ccree/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2012-2019 ARM Limited (or its affiliates). + +obj-$(CONFIG_CRYPTO_DEV_CCREE) := ccree.o +ccree-y := cc_driver.o cc_buffer_mgr.o cc_request_mgr.o cc_cipher.o cc_hash.o cc_aead.o cc_sram_mgr.o +ccree-$(CONFIG_CRYPTO_FIPS) += cc_fips.o +ccree-$(CONFIG_DEBUG_FS) += cc_debugfs.o +ccree-$(CONFIG_PM) += cc_pm.o diff --git a/drivers/crypto/ccree/cc_aead.c b/drivers/crypto/ccree/cc_aead.c new file mode 100644 index 000000000..088c46030 --- /dev/null +++ b/drivers/crypto/ccree/cc_aead.c @@ -0,0 +1,2667 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "cc_driver.h" +#include "cc_buffer_mgr.h" +#include "cc_aead.h" +#include "cc_request_mgr.h" +#include "cc_hash.h" +#include "cc_sram_mgr.h" + +#define template_aead template_u.aead + +#define MAX_AEAD_SETKEY_SEQ 12 +#define MAX_AEAD_PROCESS_SEQ 23 + +#define MAX_HMAC_DIGEST_SIZE (SHA256_DIGEST_SIZE) +#define MAX_HMAC_BLOCK_SIZE (SHA256_BLOCK_SIZE) + +#define MAX_NONCE_SIZE CTR_RFC3686_NONCE_SIZE + +struct cc_aead_handle { + u32 sram_workspace_addr; + struct list_head aead_list; +}; + +struct cc_hmac_s { + u8 *padded_authkey; + u8 *ipad_opad; /* IPAD, OPAD*/ + dma_addr_t padded_authkey_dma_addr; + dma_addr_t ipad_opad_dma_addr; +}; + +struct cc_xcbc_s { + u8 *xcbc_keys; /* K1,K2,K3 */ + dma_addr_t xcbc_keys_dma_addr; +}; + +struct cc_aead_ctx { + struct cc_drvdata *drvdata; + u8 ctr_nonce[MAX_NONCE_SIZE]; /* used for ctr3686 iv and aes ccm */ + u8 *enckey; + dma_addr_t enckey_dma_addr; + union { + struct cc_hmac_s hmac; + struct cc_xcbc_s xcbc; + } auth_state; + unsigned int enc_keylen; + unsigned int auth_keylen; + unsigned int authsize; /* Actual (reduced?) size of the MAC/ICv */ + unsigned int hash_len; + enum drv_cipher_mode cipher_mode; + enum cc_flow_mode flow_mode; + enum drv_hash_mode auth_mode; +}; + +static void cc_aead_exit(struct crypto_aead *tfm) +{ + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct device *dev = drvdata_to_dev(ctx->drvdata); + + dev_dbg(dev, "Clearing context @%p for %s\n", crypto_aead_ctx(tfm), + crypto_tfm_alg_name(&tfm->base)); + + /* Unmap enckey buffer */ + if (ctx->enckey) { + dma_free_coherent(dev, AES_MAX_KEY_SIZE, ctx->enckey, + ctx->enckey_dma_addr); + dev_dbg(dev, "Freed enckey DMA buffer enckey_dma_addr=%pad\n", + &ctx->enckey_dma_addr); + ctx->enckey_dma_addr = 0; + ctx->enckey = NULL; + } + + if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */ + struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc; + + if (xcbc->xcbc_keys) { + dma_free_coherent(dev, CC_AES_128_BIT_KEY_SIZE * 3, + xcbc->xcbc_keys, + xcbc->xcbc_keys_dma_addr); + } + dev_dbg(dev, "Freed xcbc_keys DMA buffer xcbc_keys_dma_addr=%pad\n", + &xcbc->xcbc_keys_dma_addr); + xcbc->xcbc_keys_dma_addr = 0; + xcbc->xcbc_keys = NULL; + } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC auth. */ + struct cc_hmac_s *hmac = &ctx->auth_state.hmac; + + if (hmac->ipad_opad) { + dma_free_coherent(dev, 2 * MAX_HMAC_DIGEST_SIZE, + hmac->ipad_opad, + hmac->ipad_opad_dma_addr); + dev_dbg(dev, "Freed ipad_opad DMA buffer ipad_opad_dma_addr=%pad\n", + &hmac->ipad_opad_dma_addr); + hmac->ipad_opad_dma_addr = 0; + hmac->ipad_opad = NULL; + } + if (hmac->padded_authkey) { + dma_free_coherent(dev, MAX_HMAC_BLOCK_SIZE, + hmac->padded_authkey, + hmac->padded_authkey_dma_addr); + dev_dbg(dev, "Freed padded_authkey DMA buffer padded_authkey_dma_addr=%pad\n", + &hmac->padded_authkey_dma_addr); + hmac->padded_authkey_dma_addr = 0; + hmac->padded_authkey = NULL; + } + } +} + +static unsigned int cc_get_aead_hash_len(struct crypto_aead *tfm) +{ + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + + return cc_get_default_hash_len(ctx->drvdata); +} + +static int cc_aead_init(struct crypto_aead *tfm) +{ + struct aead_alg *alg = crypto_aead_alg(tfm); + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct cc_crypto_alg *cc_alg = + container_of(alg, struct cc_crypto_alg, aead_alg); + struct device *dev = drvdata_to_dev(cc_alg->drvdata); + + dev_dbg(dev, "Initializing context @%p for %s\n", ctx, + crypto_tfm_alg_name(&tfm->base)); + + /* Initialize modes in instance */ + ctx->cipher_mode = cc_alg->cipher_mode; + ctx->flow_mode = cc_alg->flow_mode; + ctx->auth_mode = cc_alg->auth_mode; + ctx->drvdata = cc_alg->drvdata; + crypto_aead_set_reqsize_dma(tfm, sizeof(struct aead_req_ctx)); + + /* Allocate key buffer, cache line aligned */ + ctx->enckey = dma_alloc_coherent(dev, AES_MAX_KEY_SIZE, + &ctx->enckey_dma_addr, GFP_KERNEL); + if (!ctx->enckey) { + dev_err(dev, "Failed allocating key buffer\n"); + goto init_failed; + } + dev_dbg(dev, "Allocated enckey buffer in context ctx->enckey=@%p\n", + ctx->enckey); + + /* Set default authlen value */ + + if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */ + struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc; + const unsigned int key_size = CC_AES_128_BIT_KEY_SIZE * 3; + + /* Allocate dma-coherent buffer for XCBC's K1+K2+K3 */ + /* (and temporary for user key - up to 256b) */ + xcbc->xcbc_keys = dma_alloc_coherent(dev, key_size, + &xcbc->xcbc_keys_dma_addr, + GFP_KERNEL); + if (!xcbc->xcbc_keys) { + dev_err(dev, "Failed allocating buffer for XCBC keys\n"); + goto init_failed; + } + } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC authentication */ + struct cc_hmac_s *hmac = &ctx->auth_state.hmac; + const unsigned int digest_size = 2 * MAX_HMAC_DIGEST_SIZE; + dma_addr_t *pkey_dma = &hmac->padded_authkey_dma_addr; + + /* Allocate dma-coherent buffer for IPAD + OPAD */ + hmac->ipad_opad = dma_alloc_coherent(dev, digest_size, + &hmac->ipad_opad_dma_addr, + GFP_KERNEL); + + if (!hmac->ipad_opad) { + dev_err(dev, "Failed allocating IPAD/OPAD buffer\n"); + goto init_failed; + } + + dev_dbg(dev, "Allocated authkey buffer in context ctx->authkey=@%p\n", + hmac->ipad_opad); + + hmac->padded_authkey = dma_alloc_coherent(dev, + MAX_HMAC_BLOCK_SIZE, + pkey_dma, + GFP_KERNEL); + + if (!hmac->padded_authkey) { + dev_err(dev, "failed to allocate padded_authkey\n"); + goto init_failed; + } + } else { + ctx->auth_state.hmac.ipad_opad = NULL; + ctx->auth_state.hmac.padded_authkey = NULL; + } + ctx->hash_len = cc_get_aead_hash_len(tfm); + + return 0; + +init_failed: + cc_aead_exit(tfm); + return -ENOMEM; +} + +static void cc_aead_complete(struct device *dev, void *cc_req, int err) +{ + struct aead_request *areq = (struct aead_request *)cc_req; + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(areq); + struct crypto_aead *tfm = crypto_aead_reqtfm(cc_req); + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + + /* BACKLOG notification */ + if (err == -EINPROGRESS) + goto done; + + cc_unmap_aead_request(dev, areq); + + /* Restore ordinary iv pointer */ + areq->iv = areq_ctx->backup_iv; + + if (err) + goto done; + + if (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { + if (memcmp(areq_ctx->mac_buf, areq_ctx->icv_virt_addr, + ctx->authsize) != 0) { + dev_dbg(dev, "Payload authentication failure, (auth-size=%d, cipher=%d)\n", + ctx->authsize, ctx->cipher_mode); + /* In case of payload authentication failure, MUST NOT + * revealed the decrypted message --> zero its memory. + */ + sg_zero_buffer(areq->dst, sg_nents(areq->dst), + areq->cryptlen, areq->assoclen); + err = -EBADMSG; + } + /*ENCRYPT*/ + } else if (areq_ctx->is_icv_fragmented) { + u32 skip = areq->cryptlen + areq_ctx->dst_offset; + + cc_copy_sg_portion(dev, areq_ctx->mac_buf, areq_ctx->dst_sgl, + skip, (skip + ctx->authsize), + CC_SG_FROM_BUF); + } +done: + aead_request_complete(areq, err); +} + +static unsigned int xcbc_setkey(struct cc_hw_desc *desc, + struct cc_aead_ctx *ctx) +{ + /* Load the AES key */ + hw_desc_init(&desc[0]); + /* We are using for the source/user key the same buffer + * as for the output keys, * because after this key loading it + * is not needed anymore + */ + set_din_type(&desc[0], DMA_DLLI, + ctx->auth_state.xcbc.xcbc_keys_dma_addr, ctx->auth_keylen, + NS_BIT); + set_cipher_mode(&desc[0], DRV_CIPHER_ECB); + set_cipher_config0(&desc[0], DRV_CRYPTO_DIRECTION_ENCRYPT); + set_key_size_aes(&desc[0], ctx->auth_keylen); + set_flow_mode(&desc[0], S_DIN_to_AES); + set_setup_mode(&desc[0], SETUP_LOAD_KEY0); + + hw_desc_init(&desc[1]); + set_din_const(&desc[1], 0x01010101, CC_AES_128_BIT_KEY_SIZE); + set_flow_mode(&desc[1], DIN_AES_DOUT); + set_dout_dlli(&desc[1], ctx->auth_state.xcbc.xcbc_keys_dma_addr, + AES_KEYSIZE_128, NS_BIT, 0); + + hw_desc_init(&desc[2]); + set_din_const(&desc[2], 0x02020202, CC_AES_128_BIT_KEY_SIZE); + set_flow_mode(&desc[2], DIN_AES_DOUT); + set_dout_dlli(&desc[2], (ctx->auth_state.xcbc.xcbc_keys_dma_addr + + AES_KEYSIZE_128), + AES_KEYSIZE_128, NS_BIT, 0); + + hw_desc_init(&desc[3]); + set_din_const(&desc[3], 0x03030303, CC_AES_128_BIT_KEY_SIZE); + set_flow_mode(&desc[3], DIN_AES_DOUT); + set_dout_dlli(&desc[3], (ctx->auth_state.xcbc.xcbc_keys_dma_addr + + 2 * AES_KEYSIZE_128), + AES_KEYSIZE_128, NS_BIT, 0); + + return 4; +} + +static unsigned int hmac_setkey(struct cc_hw_desc *desc, + struct cc_aead_ctx *ctx) +{ + unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST }; + unsigned int digest_ofs = 0; + unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? + DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; + unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ? + CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE; + struct cc_hmac_s *hmac = &ctx->auth_state.hmac; + + unsigned int idx = 0; + int i; + + /* calc derived HMAC key */ + for (i = 0; i < 2; i++) { + /* Load hash initial state */ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], hash_mode); + set_din_sram(&desc[idx], + cc_larval_digest_addr(ctx->drvdata, + ctx->auth_mode), + digest_size); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); + idx++; + + /* Load the hash current length*/ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], hash_mode); + set_din_const(&desc[idx], 0, ctx->hash_len); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); + idx++; + + /* Prepare ipad key */ + hw_desc_init(&desc[idx]); + set_xor_val(&desc[idx], hmac_pad_const[i]); + set_cipher_mode(&desc[idx], hash_mode); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); + idx++; + + /* Perform HASH update */ + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, + hmac->padded_authkey_dma_addr, + SHA256_BLOCK_SIZE, NS_BIT); + set_cipher_mode(&desc[idx], hash_mode); + set_xor_active(&desc[idx]); + set_flow_mode(&desc[idx], DIN_HASH); + idx++; + + /* Get the digset */ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], hash_mode); + set_dout_dlli(&desc[idx], + (hmac->ipad_opad_dma_addr + digest_ofs), + digest_size, NS_BIT, 0); + set_flow_mode(&desc[idx], S_HASH_to_DOUT); + set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); + set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); + idx++; + + digest_ofs += digest_size; + } + + return idx; +} + +static int validate_keys_sizes(struct cc_aead_ctx *ctx) +{ + struct device *dev = drvdata_to_dev(ctx->drvdata); + + dev_dbg(dev, "enc_keylen=%u authkeylen=%u\n", + ctx->enc_keylen, ctx->auth_keylen); + + switch (ctx->auth_mode) { + case DRV_HASH_SHA1: + case DRV_HASH_SHA256: + break; + case DRV_HASH_XCBC_MAC: + if (ctx->auth_keylen != AES_KEYSIZE_128 && + ctx->auth_keylen != AES_KEYSIZE_192 && + ctx->auth_keylen != AES_KEYSIZE_256) + return -ENOTSUPP; + break; + case DRV_HASH_NULL: /* Not authenc (e.g., CCM) - no auth_key) */ + if (ctx->auth_keylen > 0) + return -EINVAL; + break; + default: + dev_dbg(dev, "Invalid auth_mode=%d\n", ctx->auth_mode); + return -EINVAL; + } + /* Check cipher key size */ + if (ctx->flow_mode == S_DIN_to_DES) { + if (ctx->enc_keylen != DES3_EDE_KEY_SIZE) { + dev_dbg(dev, "Invalid cipher(3DES) key size: %u\n", + ctx->enc_keylen); + return -EINVAL; + } + } else { /* Default assumed to be AES ciphers */ + if (ctx->enc_keylen != AES_KEYSIZE_128 && + ctx->enc_keylen != AES_KEYSIZE_192 && + ctx->enc_keylen != AES_KEYSIZE_256) { + dev_dbg(dev, "Invalid cipher(AES) key size: %u\n", + ctx->enc_keylen); + return -EINVAL; + } + } + + return 0; /* All tests of keys sizes passed */ +} + +/* This function prepers the user key so it can pass to the hmac processing + * (copy to intenral buffer or hash in case of key longer than block + */ +static int cc_get_plain_hmac_key(struct crypto_aead *tfm, const u8 *authkey, + unsigned int keylen) +{ + dma_addr_t key_dma_addr = 0; + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct device *dev = drvdata_to_dev(ctx->drvdata); + u32 larval_addr; + struct cc_crypto_req cc_req = {}; + unsigned int blocksize; + unsigned int digestsize; + unsigned int hashmode; + unsigned int idx = 0; + int rc = 0; + u8 *key = NULL; + struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ]; + dma_addr_t padded_authkey_dma_addr = + ctx->auth_state.hmac.padded_authkey_dma_addr; + + switch (ctx->auth_mode) { /* auth_key required and >0 */ + case DRV_HASH_SHA1: + blocksize = SHA1_BLOCK_SIZE; + digestsize = SHA1_DIGEST_SIZE; + hashmode = DRV_HASH_HW_SHA1; + break; + case DRV_HASH_SHA256: + default: + blocksize = SHA256_BLOCK_SIZE; + digestsize = SHA256_DIGEST_SIZE; + hashmode = DRV_HASH_HW_SHA256; + } + + if (keylen != 0) { + + key = kmemdup(authkey, keylen, GFP_KERNEL); + if (!key) + return -ENOMEM; + + key_dma_addr = dma_map_single(dev, key, keylen, DMA_TO_DEVICE); + if (dma_mapping_error(dev, key_dma_addr)) { + dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n", + key, keylen); + kfree_sensitive(key); + return -ENOMEM; + } + if (keylen > blocksize) { + /* Load hash initial state */ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], hashmode); + larval_addr = cc_larval_digest_addr(ctx->drvdata, + ctx->auth_mode); + set_din_sram(&desc[idx], larval_addr, digestsize); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); + idx++; + + /* Load the hash current length*/ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], hashmode); + set_din_const(&desc[idx], 0, ctx->hash_len); + set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); + idx++; + + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, + key_dma_addr, keylen, NS_BIT); + set_flow_mode(&desc[idx], DIN_HASH); + idx++; + + /* Get hashed key */ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], hashmode); + set_dout_dlli(&desc[idx], padded_authkey_dma_addr, + digestsize, NS_BIT, 0); + set_flow_mode(&desc[idx], S_HASH_to_DOUT); + set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); + set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); + set_cipher_config0(&desc[idx], + HASH_DIGEST_RESULT_LITTLE_ENDIAN); + idx++; + + hw_desc_init(&desc[idx]); + set_din_const(&desc[idx], 0, (blocksize - digestsize)); + set_flow_mode(&desc[idx], BYPASS); + set_dout_dlli(&desc[idx], (padded_authkey_dma_addr + + digestsize), (blocksize - digestsize), + NS_BIT, 0); + idx++; + } else { + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, key_dma_addr, + keylen, NS_BIT); + set_flow_mode(&desc[idx], BYPASS); + set_dout_dlli(&desc[idx], padded_authkey_dma_addr, + keylen, NS_BIT, 0); + idx++; + + if ((blocksize - keylen) != 0) { + hw_desc_init(&desc[idx]); + set_din_const(&desc[idx], 0, + (blocksize - keylen)); + set_flow_mode(&desc[idx], BYPASS); + set_dout_dlli(&desc[idx], + (padded_authkey_dma_addr + + keylen), + (blocksize - keylen), NS_BIT, 0); + idx++; + } + } + } else { + hw_desc_init(&desc[idx]); + set_din_const(&desc[idx], 0, (blocksize - keylen)); + set_flow_mode(&desc[idx], BYPASS); + set_dout_dlli(&desc[idx], padded_authkey_dma_addr, + blocksize, NS_BIT, 0); + idx++; + } + + rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx); + if (rc) + dev_err(dev, "send_request() failed (rc=%d)\n", rc); + + if (key_dma_addr) + dma_unmap_single(dev, key_dma_addr, keylen, DMA_TO_DEVICE); + + kfree_sensitive(key); + + return rc; +} + +static int cc_aead_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct cc_crypto_req cc_req = {}; + struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ]; + unsigned int seq_len = 0; + struct device *dev = drvdata_to_dev(ctx->drvdata); + const u8 *enckey, *authkey; + int rc; + + dev_dbg(dev, "Setting key in context @%p for %s. key=%p keylen=%u\n", + ctx, crypto_tfm_alg_name(crypto_aead_tfm(tfm)), key, keylen); + + /* STAT_PHASE_0: Init and sanity checks */ + + if (ctx->auth_mode != DRV_HASH_NULL) { /* authenc() alg. */ + struct crypto_authenc_keys keys; + + rc = crypto_authenc_extractkeys(&keys, key, keylen); + if (rc) + return rc; + enckey = keys.enckey; + authkey = keys.authkey; + ctx->enc_keylen = keys.enckeylen; + ctx->auth_keylen = keys.authkeylen; + + if (ctx->cipher_mode == DRV_CIPHER_CTR) { + /* the nonce is stored in bytes at end of key */ + if (ctx->enc_keylen < + (AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE)) + return -EINVAL; + /* Copy nonce from last 4 bytes in CTR key to + * first 4 bytes in CTR IV + */ + memcpy(ctx->ctr_nonce, enckey + ctx->enc_keylen - + CTR_RFC3686_NONCE_SIZE, CTR_RFC3686_NONCE_SIZE); + /* Set CTR key size */ + ctx->enc_keylen -= CTR_RFC3686_NONCE_SIZE; + } + } else { /* non-authenc - has just one key */ + enckey = key; + authkey = NULL; + ctx->enc_keylen = keylen; + ctx->auth_keylen = 0; + } + + rc = validate_keys_sizes(ctx); + if (rc) + return rc; + + /* STAT_PHASE_1: Copy key to ctx */ + + /* Get key material */ + memcpy(ctx->enckey, enckey, ctx->enc_keylen); + if (ctx->enc_keylen == 24) + memset(ctx->enckey + 24, 0, CC_AES_KEY_SIZE_MAX - 24); + if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { + memcpy(ctx->auth_state.xcbc.xcbc_keys, authkey, + ctx->auth_keylen); + } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC */ + rc = cc_get_plain_hmac_key(tfm, authkey, ctx->auth_keylen); + if (rc) + return rc; + } + + /* STAT_PHASE_2: Create sequence */ + + switch (ctx->auth_mode) { + case DRV_HASH_SHA1: + case DRV_HASH_SHA256: + seq_len = hmac_setkey(desc, ctx); + break; + case DRV_HASH_XCBC_MAC: + seq_len = xcbc_setkey(desc, ctx); + break; + case DRV_HASH_NULL: /* non-authenc modes, e.g., CCM */ + break; /* No auth. key setup */ + default: + dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode); + return -ENOTSUPP; + } + + /* STAT_PHASE_3: Submit sequence to HW */ + + if (seq_len > 0) { /* For CCM there is no sequence to setup the key */ + rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, seq_len); + if (rc) { + dev_err(dev, "send_request() failed (rc=%d)\n", rc); + return rc; + } + } + + /* Update STAT_PHASE_3 */ + return rc; +} + +static int cc_des3_aead_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct crypto_authenc_keys keys; + int err; + + err = crypto_authenc_extractkeys(&keys, key, keylen); + if (unlikely(err)) + return err; + + err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?: + cc_aead_setkey(aead, key, keylen); + + memzero_explicit(&keys, sizeof(keys)); + return err; +} + +static int cc_rfc4309_ccm_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + + if (keylen < 3) + return -EINVAL; + + keylen -= 3; + memcpy(ctx->ctr_nonce, key + keylen, 3); + + return cc_aead_setkey(tfm, key, keylen); +} + +static int cc_aead_setauthsize(struct crypto_aead *authenc, + unsigned int authsize) +{ + struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc); + struct device *dev = drvdata_to_dev(ctx->drvdata); + + /* Unsupported auth. sizes */ + if (authsize == 0 || + authsize > crypto_aead_maxauthsize(authenc)) { + return -ENOTSUPP; + } + + ctx->authsize = authsize; + dev_dbg(dev, "authlen=%d\n", ctx->authsize); + + return 0; +} + +static int cc_rfc4309_ccm_setauthsize(struct crypto_aead *authenc, + unsigned int authsize) +{ + switch (authsize) { + case 8: + case 12: + case 16: + break; + default: + return -EINVAL; + } + + return cc_aead_setauthsize(authenc, authsize); +} + +static int cc_ccm_setauthsize(struct crypto_aead *authenc, + unsigned int authsize) +{ + switch (authsize) { + case 4: + case 6: + case 8: + case 10: + case 12: + case 14: + case 16: + break; + default: + return -EINVAL; + } + + return cc_aead_setauthsize(authenc, authsize); +} + +static void cc_set_assoc_desc(struct aead_request *areq, unsigned int flow_mode, + struct cc_hw_desc desc[], unsigned int *seq_size) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(areq); + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(areq); + enum cc_req_dma_buf_type assoc_dma_type = areq_ctx->assoc_buff_type; + unsigned int idx = *seq_size; + struct device *dev = drvdata_to_dev(ctx->drvdata); + + switch (assoc_dma_type) { + case CC_DMA_BUF_DLLI: + dev_dbg(dev, "ASSOC buffer type DLLI\n"); + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, sg_dma_address(areq->src), + areq_ctx->assoclen, NS_BIT); + set_flow_mode(&desc[idx], flow_mode); + if (ctx->auth_mode == DRV_HASH_XCBC_MAC && + areq_ctx->cryptlen > 0) + set_din_not_last_indication(&desc[idx]); + break; + case CC_DMA_BUF_MLLI: + dev_dbg(dev, "ASSOC buffer type MLLI\n"); + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_MLLI, areq_ctx->assoc.sram_addr, + areq_ctx->assoc.mlli_nents, NS_BIT); + set_flow_mode(&desc[idx], flow_mode); + if (ctx->auth_mode == DRV_HASH_XCBC_MAC && + areq_ctx->cryptlen > 0) + set_din_not_last_indication(&desc[idx]); + break; + case CC_DMA_BUF_NULL: + default: + dev_err(dev, "Invalid ASSOC buffer type\n"); + } + + *seq_size = (++idx); +} + +static void cc_proc_authen_desc(struct aead_request *areq, + unsigned int flow_mode, + struct cc_hw_desc desc[], + unsigned int *seq_size, int direct) +{ + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(areq); + enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type; + unsigned int idx = *seq_size; + struct crypto_aead *tfm = crypto_aead_reqtfm(areq); + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct device *dev = drvdata_to_dev(ctx->drvdata); + + switch (data_dma_type) { + case CC_DMA_BUF_DLLI: + { + struct scatterlist *cipher = + (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? + areq_ctx->dst_sgl : areq_ctx->src_sgl; + + unsigned int offset = + (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? + areq_ctx->dst_offset : areq_ctx->src_offset; + dev_dbg(dev, "AUTHENC: SRC/DST buffer type DLLI\n"); + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, + (sg_dma_address(cipher) + offset), + areq_ctx->cryptlen, NS_BIT); + set_flow_mode(&desc[idx], flow_mode); + break; + } + case CC_DMA_BUF_MLLI: + { + /* DOUBLE-PASS flow (as default) + * assoc. + iv + data -compact in one table + * if assoclen is ZERO only IV perform + */ + u32 mlli_addr = areq_ctx->assoc.sram_addr; + u32 mlli_nents = areq_ctx->assoc.mlli_nents; + + if (areq_ctx->is_single_pass) { + if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { + mlli_addr = areq_ctx->dst.sram_addr; + mlli_nents = areq_ctx->dst.mlli_nents; + } else { + mlli_addr = areq_ctx->src.sram_addr; + mlli_nents = areq_ctx->src.mlli_nents; + } + } + + dev_dbg(dev, "AUTHENC: SRC/DST buffer type MLLI\n"); + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_MLLI, mlli_addr, mlli_nents, + NS_BIT); + set_flow_mode(&desc[idx], flow_mode); + break; + } + case CC_DMA_BUF_NULL: + default: + dev_err(dev, "AUTHENC: Invalid SRC/DST buffer type\n"); + } + + *seq_size = (++idx); +} + +static void cc_proc_cipher_desc(struct aead_request *areq, + unsigned int flow_mode, + struct cc_hw_desc desc[], + unsigned int *seq_size) +{ + unsigned int idx = *seq_size; + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(areq); + enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type; + struct crypto_aead *tfm = crypto_aead_reqtfm(areq); + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct device *dev = drvdata_to_dev(ctx->drvdata); + + if (areq_ctx->cryptlen == 0) + return; /*null processing*/ + + switch (data_dma_type) { + case CC_DMA_BUF_DLLI: + dev_dbg(dev, "CIPHER: SRC/DST buffer type DLLI\n"); + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, + (sg_dma_address(areq_ctx->src_sgl) + + areq_ctx->src_offset), areq_ctx->cryptlen, + NS_BIT); + set_dout_dlli(&desc[idx], + (sg_dma_address(areq_ctx->dst_sgl) + + areq_ctx->dst_offset), + areq_ctx->cryptlen, NS_BIT, 0); + set_flow_mode(&desc[idx], flow_mode); + break; + case CC_DMA_BUF_MLLI: + dev_dbg(dev, "CIPHER: SRC/DST buffer type MLLI\n"); + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_MLLI, areq_ctx->src.sram_addr, + areq_ctx->src.mlli_nents, NS_BIT); + set_dout_mlli(&desc[idx], areq_ctx->dst.sram_addr, + areq_ctx->dst.mlli_nents, NS_BIT, 0); + set_flow_mode(&desc[idx], flow_mode); + break; + case CC_DMA_BUF_NULL: + default: + dev_err(dev, "CIPHER: Invalid SRC/DST buffer type\n"); + } + + *seq_size = (++idx); +} + +static void cc_proc_digest_desc(struct aead_request *req, + struct cc_hw_desc desc[], + unsigned int *seq_size) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); + unsigned int idx = *seq_size; + unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? + DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; + int direct = req_ctx->gen_ctx.op_type; + + /* Get final ICV result */ + if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { + hw_desc_init(&desc[idx]); + set_flow_mode(&desc[idx], S_HASH_to_DOUT); + set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); + set_dout_dlli(&desc[idx], req_ctx->icv_dma_addr, ctx->authsize, + NS_BIT, 1); + set_queue_last_ind(ctx->drvdata, &desc[idx]); + if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { + set_aes_not_hash_mode(&desc[idx]); + set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); + } else { + set_cipher_config0(&desc[idx], + HASH_DIGEST_RESULT_LITTLE_ENDIAN); + set_cipher_mode(&desc[idx], hash_mode); + } + } else { /*Decrypt*/ + /* Get ICV out from hardware */ + hw_desc_init(&desc[idx]); + set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); + set_flow_mode(&desc[idx], S_HASH_to_DOUT); + set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, + ctx->authsize, NS_BIT, 1); + set_queue_last_ind(ctx->drvdata, &desc[idx]); + set_cipher_config0(&desc[idx], + HASH_DIGEST_RESULT_LITTLE_ENDIAN); + set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); + if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { + set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); + set_aes_not_hash_mode(&desc[idx]); + } else { + set_cipher_mode(&desc[idx], hash_mode); + } + } + + *seq_size = (++idx); +} + +static void cc_set_cipher_desc(struct aead_request *req, + struct cc_hw_desc desc[], + unsigned int *seq_size) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); + unsigned int hw_iv_size = req_ctx->hw_iv_size; + unsigned int idx = *seq_size; + int direct = req_ctx->gen_ctx.op_type; + + /* Setup cipher state */ + hw_desc_init(&desc[idx]); + set_cipher_config0(&desc[idx], direct); + set_flow_mode(&desc[idx], ctx->flow_mode); + set_din_type(&desc[idx], DMA_DLLI, req_ctx->gen_ctx.iv_dma_addr, + hw_iv_size, NS_BIT); + if (ctx->cipher_mode == DRV_CIPHER_CTR) + set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); + else + set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); + set_cipher_mode(&desc[idx], ctx->cipher_mode); + idx++; + + /* Setup enc. key */ + hw_desc_init(&desc[idx]); + set_cipher_config0(&desc[idx], direct); + set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); + set_flow_mode(&desc[idx], ctx->flow_mode); + if (ctx->flow_mode == S_DIN_to_AES) { + set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, + ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX : + ctx->enc_keylen), NS_BIT); + set_key_size_aes(&desc[idx], ctx->enc_keylen); + } else { + set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, + ctx->enc_keylen, NS_BIT); + set_key_size_des(&desc[idx], ctx->enc_keylen); + } + set_cipher_mode(&desc[idx], ctx->cipher_mode); + idx++; + + *seq_size = idx; +} + +static void cc_proc_cipher(struct aead_request *req, struct cc_hw_desc desc[], + unsigned int *seq_size, unsigned int data_flow_mode) +{ + struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); + int direct = req_ctx->gen_ctx.op_type; + unsigned int idx = *seq_size; + + if (req_ctx->cryptlen == 0) + return; /*null processing*/ + + cc_set_cipher_desc(req, desc, &idx); + cc_proc_cipher_desc(req, data_flow_mode, desc, &idx); + if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { + /* We must wait for DMA to write all cipher */ + hw_desc_init(&desc[idx]); + set_din_no_dma(&desc[idx], 0, 0xfffff0); + set_dout_no_dma(&desc[idx], 0, 0, 1); + idx++; + } + + *seq_size = idx; +} + +static void cc_set_hmac_desc(struct aead_request *req, struct cc_hw_desc desc[], + unsigned int *seq_size) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? + DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; + unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ? + CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE; + unsigned int idx = *seq_size; + + /* Loading hash ipad xor key state */ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], hash_mode); + set_din_type(&desc[idx], DMA_DLLI, + ctx->auth_state.hmac.ipad_opad_dma_addr, digest_size, + NS_BIT); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); + idx++; + + /* Load init. digest len (64 bytes) */ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], hash_mode); + set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode), + ctx->hash_len); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); + idx++; + + *seq_size = idx; +} + +static void cc_set_xcbc_desc(struct aead_request *req, struct cc_hw_desc desc[], + unsigned int *seq_size) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + unsigned int idx = *seq_size; + + /* Loading MAC state */ + hw_desc_init(&desc[idx]); + set_din_const(&desc[idx], 0, CC_AES_BLOCK_SIZE); + set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); + set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); + set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); + set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_aes_not_hash_mode(&desc[idx]); + idx++; + + /* Setup XCBC MAC K1 */ + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, + ctx->auth_state.xcbc.xcbc_keys_dma_addr, + AES_KEYSIZE_128, NS_BIT); + set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); + set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); + set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); + set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_aes_not_hash_mode(&desc[idx]); + idx++; + + /* Setup XCBC MAC K2 */ + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, + (ctx->auth_state.xcbc.xcbc_keys_dma_addr + + AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT); + set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); + set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); + set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); + set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_aes_not_hash_mode(&desc[idx]); + idx++; + + /* Setup XCBC MAC K3 */ + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, + (ctx->auth_state.xcbc.xcbc_keys_dma_addr + + 2 * AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT); + set_setup_mode(&desc[idx], SETUP_LOAD_STATE2); + set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); + set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); + set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_aes_not_hash_mode(&desc[idx]); + idx++; + + *seq_size = idx; +} + +static void cc_proc_header_desc(struct aead_request *req, + struct cc_hw_desc desc[], + unsigned int *seq_size) +{ + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); + unsigned int idx = *seq_size; + + /* Hash associated data */ + if (areq_ctx->assoclen > 0) + cc_set_assoc_desc(req, DIN_HASH, desc, &idx); + + /* Hash IV */ + *seq_size = idx; +} + +static void cc_proc_scheme_desc(struct aead_request *req, + struct cc_hw_desc desc[], + unsigned int *seq_size) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct cc_aead_handle *aead_handle = ctx->drvdata->aead_handle; + unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? + DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; + unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ? + CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE; + unsigned int idx = *seq_size; + + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], hash_mode); + set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr, + ctx->hash_len); + set_flow_mode(&desc[idx], S_HASH_to_DOUT); + set_setup_mode(&desc[idx], SETUP_WRITE_STATE1); + set_cipher_do(&desc[idx], DO_PAD); + idx++; + + /* Get final ICV result */ + hw_desc_init(&desc[idx]); + set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr, + digest_size); + set_flow_mode(&desc[idx], S_HASH_to_DOUT); + set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); + set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN); + set_cipher_mode(&desc[idx], hash_mode); + idx++; + + /* Loading hash opad xor key state */ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], hash_mode); + set_din_type(&desc[idx], DMA_DLLI, + (ctx->auth_state.hmac.ipad_opad_dma_addr + digest_size), + digest_size, NS_BIT); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); + idx++; + + /* Load init. digest len (64 bytes) */ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], hash_mode); + set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode), + ctx->hash_len); + set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); + idx++; + + /* Perform HASH update */ + hw_desc_init(&desc[idx]); + set_din_sram(&desc[idx], aead_handle->sram_workspace_addr, + digest_size); + set_flow_mode(&desc[idx], DIN_HASH); + idx++; + + *seq_size = idx; +} + +static void cc_mlli_to_sram(struct aead_request *req, + struct cc_hw_desc desc[], unsigned int *seq_size) +{ + struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct device *dev = drvdata_to_dev(ctx->drvdata); + + if ((req_ctx->assoc_buff_type == CC_DMA_BUF_MLLI || + req_ctx->data_buff_type == CC_DMA_BUF_MLLI || + !req_ctx->is_single_pass) && req_ctx->mlli_params.mlli_len) { + dev_dbg(dev, "Copy-to-sram: mlli_dma=%08x, mlli_size=%u\n", + ctx->drvdata->mlli_sram_addr, + req_ctx->mlli_params.mlli_len); + /* Copy MLLI table host-to-sram */ + hw_desc_init(&desc[*seq_size]); + set_din_type(&desc[*seq_size], DMA_DLLI, + req_ctx->mlli_params.mlli_dma_addr, + req_ctx->mlli_params.mlli_len, NS_BIT); + set_dout_sram(&desc[*seq_size], + ctx->drvdata->mlli_sram_addr, + req_ctx->mlli_params.mlli_len); + set_flow_mode(&desc[*seq_size], BYPASS); + (*seq_size)++; + } +} + +static enum cc_flow_mode cc_get_data_flow(enum drv_crypto_direction direct, + enum cc_flow_mode setup_flow_mode, + bool is_single_pass) +{ + enum cc_flow_mode data_flow_mode; + + if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { + if (setup_flow_mode == S_DIN_to_AES) + data_flow_mode = is_single_pass ? + AES_to_HASH_and_DOUT : DIN_AES_DOUT; + else + data_flow_mode = is_single_pass ? + DES_to_HASH_and_DOUT : DIN_DES_DOUT; + } else { /* Decrypt */ + if (setup_flow_mode == S_DIN_to_AES) + data_flow_mode = is_single_pass ? + AES_and_HASH : DIN_AES_DOUT; + else + data_flow_mode = is_single_pass ? + DES_and_HASH : DIN_DES_DOUT; + } + + return data_flow_mode; +} + +static void cc_hmac_authenc(struct aead_request *req, struct cc_hw_desc desc[], + unsigned int *seq_size) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); + int direct = req_ctx->gen_ctx.op_type; + unsigned int data_flow_mode = + cc_get_data_flow(direct, ctx->flow_mode, + req_ctx->is_single_pass); + + if (req_ctx->is_single_pass) { + /* + * Single-pass flow + */ + cc_set_hmac_desc(req, desc, seq_size); + cc_set_cipher_desc(req, desc, seq_size); + cc_proc_header_desc(req, desc, seq_size); + cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size); + cc_proc_scheme_desc(req, desc, seq_size); + cc_proc_digest_desc(req, desc, seq_size); + return; + } + + /* + * Double-pass flow + * Fallback for unsupported single-pass modes, + * i.e. using assoc. data of non-word-multiple + */ + if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { + /* encrypt first.. */ + cc_proc_cipher(req, desc, seq_size, data_flow_mode); + /* authenc after..*/ + cc_set_hmac_desc(req, desc, seq_size); + cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); + cc_proc_scheme_desc(req, desc, seq_size); + cc_proc_digest_desc(req, desc, seq_size); + + } else { /*DECRYPT*/ + /* authenc first..*/ + cc_set_hmac_desc(req, desc, seq_size); + cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); + cc_proc_scheme_desc(req, desc, seq_size); + /* decrypt after.. */ + cc_proc_cipher(req, desc, seq_size, data_flow_mode); + /* read the digest result with setting the completion bit + * must be after the cipher operation + */ + cc_proc_digest_desc(req, desc, seq_size); + } +} + +static void +cc_xcbc_authenc(struct aead_request *req, struct cc_hw_desc desc[], + unsigned int *seq_size) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); + int direct = req_ctx->gen_ctx.op_type; + unsigned int data_flow_mode = + cc_get_data_flow(direct, ctx->flow_mode, + req_ctx->is_single_pass); + + if (req_ctx->is_single_pass) { + /* + * Single-pass flow + */ + cc_set_xcbc_desc(req, desc, seq_size); + cc_set_cipher_desc(req, desc, seq_size); + cc_proc_header_desc(req, desc, seq_size); + cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size); + cc_proc_digest_desc(req, desc, seq_size); + return; + } + + /* + * Double-pass flow + * Fallback for unsupported single-pass modes, + * i.e. using assoc. data of non-word-multiple + */ + if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { + /* encrypt first.. */ + cc_proc_cipher(req, desc, seq_size, data_flow_mode); + /* authenc after.. */ + cc_set_xcbc_desc(req, desc, seq_size); + cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); + cc_proc_digest_desc(req, desc, seq_size); + } else { /*DECRYPT*/ + /* authenc first.. */ + cc_set_xcbc_desc(req, desc, seq_size); + cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); + /* decrypt after..*/ + cc_proc_cipher(req, desc, seq_size, data_flow_mode); + /* read the digest result with setting the completion bit + * must be after the cipher operation + */ + cc_proc_digest_desc(req, desc, seq_size); + } +} + +static int validate_data_size(struct cc_aead_ctx *ctx, + enum drv_crypto_direction direct, + struct aead_request *req) +{ + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); + struct device *dev = drvdata_to_dev(ctx->drvdata); + unsigned int assoclen = areq_ctx->assoclen; + unsigned int cipherlen = (direct == DRV_CRYPTO_DIRECTION_DECRYPT) ? + (req->cryptlen - ctx->authsize) : req->cryptlen; + + if (direct == DRV_CRYPTO_DIRECTION_DECRYPT && + req->cryptlen < ctx->authsize) + goto data_size_err; + + areq_ctx->is_single_pass = true; /*defaulted to fast flow*/ + + switch (ctx->flow_mode) { + case S_DIN_to_AES: + if (ctx->cipher_mode == DRV_CIPHER_CBC && + !IS_ALIGNED(cipherlen, AES_BLOCK_SIZE)) + goto data_size_err; + if (ctx->cipher_mode == DRV_CIPHER_CCM) + break; + if (ctx->cipher_mode == DRV_CIPHER_GCTR) { + if (areq_ctx->plaintext_authenticate_only) + areq_ctx->is_single_pass = false; + break; + } + + if (!IS_ALIGNED(assoclen, sizeof(u32))) + areq_ctx->is_single_pass = false; + + if (ctx->cipher_mode == DRV_CIPHER_CTR && + !IS_ALIGNED(cipherlen, sizeof(u32))) + areq_ctx->is_single_pass = false; + + break; + case S_DIN_to_DES: + if (!IS_ALIGNED(cipherlen, DES_BLOCK_SIZE)) + goto data_size_err; + if (!IS_ALIGNED(assoclen, DES_BLOCK_SIZE)) + areq_ctx->is_single_pass = false; + break; + default: + dev_err(dev, "Unexpected flow mode (%d)\n", ctx->flow_mode); + goto data_size_err; + } + + return 0; + +data_size_err: + return -EINVAL; +} + +static unsigned int format_ccm_a0(u8 *pa0_buff, u32 header_size) +{ + unsigned int len = 0; + + if (header_size == 0) + return 0; + + if (header_size < ((1UL << 16) - (1UL << 8))) { + len = 2; + + pa0_buff[0] = (header_size >> 8) & 0xFF; + pa0_buff[1] = header_size & 0xFF; + } else { + len = 6; + + pa0_buff[0] = 0xFF; + pa0_buff[1] = 0xFE; + pa0_buff[2] = (header_size >> 24) & 0xFF; + pa0_buff[3] = (header_size >> 16) & 0xFF; + pa0_buff[4] = (header_size >> 8) & 0xFF; + pa0_buff[5] = header_size & 0xFF; + } + + return len; +} + +static int set_msg_len(u8 *block, unsigned int msglen, unsigned int csize) +{ + __be32 data; + + memset(block, 0, csize); + block += csize; + + if (csize >= 4) + csize = 4; + else if (msglen > (1 << (8 * csize))) + return -EOVERFLOW; + + data = cpu_to_be32(msglen); + memcpy(block - csize, (u8 *)&data + 4 - csize, csize); + + return 0; +} + +static int cc_ccm(struct aead_request *req, struct cc_hw_desc desc[], + unsigned int *seq_size) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); + unsigned int idx = *seq_size; + unsigned int cipher_flow_mode; + dma_addr_t mac_result; + + if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { + cipher_flow_mode = AES_to_HASH_and_DOUT; + mac_result = req_ctx->mac_buf_dma_addr; + } else { /* Encrypt */ + cipher_flow_mode = AES_and_HASH; + mac_result = req_ctx->icv_dma_addr; + } + + /* load key */ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], DRV_CIPHER_CTR); + set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, + ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX : + ctx->enc_keylen), NS_BIT); + set_key_size_aes(&desc[idx], ctx->enc_keylen); + set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); + set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); + set_flow_mode(&desc[idx], S_DIN_to_AES); + idx++; + + /* load ctr state */ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], DRV_CIPHER_CTR); + set_key_size_aes(&desc[idx], ctx->enc_keylen); + set_din_type(&desc[idx], DMA_DLLI, + req_ctx->gen_ctx.iv_dma_addr, AES_BLOCK_SIZE, NS_BIT); + set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); + set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); + set_flow_mode(&desc[idx], S_DIN_to_AES); + idx++; + + /* load MAC key */ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC); + set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, + ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX : + ctx->enc_keylen), NS_BIT); + set_key_size_aes(&desc[idx], ctx->enc_keylen); + set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); + set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_aes_not_hash_mode(&desc[idx]); + idx++; + + /* load MAC state */ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC); + set_key_size_aes(&desc[idx], ctx->enc_keylen); + set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr, + AES_BLOCK_SIZE, NS_BIT); + set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); + set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_aes_not_hash_mode(&desc[idx]); + idx++; + + /* process assoc data */ + if (req_ctx->assoclen > 0) { + cc_set_assoc_desc(req, DIN_HASH, desc, &idx); + } else { + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, + sg_dma_address(&req_ctx->ccm_adata_sg), + AES_BLOCK_SIZE + req_ctx->ccm_hdr_size, NS_BIT); + set_flow_mode(&desc[idx], DIN_HASH); + idx++; + } + + /* process the cipher */ + if (req_ctx->cryptlen) + cc_proc_cipher_desc(req, cipher_flow_mode, desc, &idx); + + /* Read temporal MAC */ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC); + set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, ctx->authsize, + NS_BIT, 0); + set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); + set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN); + set_flow_mode(&desc[idx], S_HASH_to_DOUT); + set_aes_not_hash_mode(&desc[idx]); + idx++; + + /* load AES-CTR state (for last MAC calculation)*/ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], DRV_CIPHER_CTR); + set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); + set_din_type(&desc[idx], DMA_DLLI, req_ctx->ccm_iv0_dma_addr, + AES_BLOCK_SIZE, NS_BIT); + set_key_size_aes(&desc[idx], ctx->enc_keylen); + set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); + set_flow_mode(&desc[idx], S_DIN_to_AES); + idx++; + + hw_desc_init(&desc[idx]); + set_din_no_dma(&desc[idx], 0, 0xfffff0); + set_dout_no_dma(&desc[idx], 0, 0, 1); + idx++; + + /* encrypt the "T" value and store MAC in mac_state */ + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr, + ctx->authsize, NS_BIT); + set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1); + set_queue_last_ind(ctx->drvdata, &desc[idx]); + set_flow_mode(&desc[idx], DIN_AES_DOUT); + idx++; + + *seq_size = idx; + return 0; +} + +static int config_ccm_adata(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct device *dev = drvdata_to_dev(ctx->drvdata); + struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); + //unsigned int size_of_a = 0, rem_a_size = 0; + unsigned int lp = req->iv[0]; + /* Note: The code assume that req->iv[0] already contains the value + * of L' of RFC3610 + */ + unsigned int l = lp + 1; /* This is L' of RFC 3610. */ + unsigned int m = ctx->authsize; /* This is M' of RFC 3610. */ + u8 *b0 = req_ctx->ccm_config + CCM_B0_OFFSET; + u8 *a0 = req_ctx->ccm_config + CCM_A0_OFFSET; + u8 *ctr_count_0 = req_ctx->ccm_config + CCM_CTR_COUNT_0_OFFSET; + unsigned int cryptlen = (req_ctx->gen_ctx.op_type == + DRV_CRYPTO_DIRECTION_ENCRYPT) ? + req->cryptlen : + (req->cryptlen - ctx->authsize); + int rc; + + memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE); + memset(req_ctx->ccm_config, 0, AES_BLOCK_SIZE * 3); + + /* taken from crypto/ccm.c */ + /* 2 <= L <= 8, so 1 <= L' <= 7. */ + if (l < 2 || l > 8) { + dev_dbg(dev, "illegal iv value %X\n", req->iv[0]); + return -EINVAL; + } + memcpy(b0, req->iv, AES_BLOCK_SIZE); + + /* format control info per RFC 3610 and + * NIST Special Publication 800-38C + */ + *b0 |= (8 * ((m - 2) / 2)); + if (req_ctx->assoclen > 0) + *b0 |= 64; /* Enable bit 6 if Adata exists. */ + + rc = set_msg_len(b0 + 16 - l, cryptlen, l); /* Write L'. */ + if (rc) { + dev_err(dev, "message len overflow detected"); + return rc; + } + /* END of "taken from crypto/ccm.c" */ + + /* l(a) - size of associated data. */ + req_ctx->ccm_hdr_size = format_ccm_a0(a0, req_ctx->assoclen); + + memset(req->iv + 15 - req->iv[0], 0, req->iv[0] + 1); + req->iv[15] = 1; + + memcpy(ctr_count_0, req->iv, AES_BLOCK_SIZE); + ctr_count_0[15] = 0; + + return 0; +} + +static void cc_proc_rfc4309_ccm(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); + + /* L' */ + memset(areq_ctx->ctr_iv, 0, AES_BLOCK_SIZE); + /* For RFC 4309, always use 4 bytes for message length + * (at most 2^32-1 bytes). + */ + areq_ctx->ctr_iv[0] = 3; + + /* In RFC 4309 there is an 11-bytes nonce+IV part, + * that we build here. + */ + memcpy(areq_ctx->ctr_iv + CCM_BLOCK_NONCE_OFFSET, ctx->ctr_nonce, + CCM_BLOCK_NONCE_SIZE); + memcpy(areq_ctx->ctr_iv + CCM_BLOCK_IV_OFFSET, req->iv, + CCM_BLOCK_IV_SIZE); + req->iv = areq_ctx->ctr_iv; +} + +static void cc_set_ghash_desc(struct aead_request *req, + struct cc_hw_desc desc[], unsigned int *seq_size) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); + unsigned int idx = *seq_size; + + /* load key to AES*/ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], DRV_CIPHER_ECB); + set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); + set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, + ctx->enc_keylen, NS_BIT); + set_key_size_aes(&desc[idx], ctx->enc_keylen); + set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); + set_flow_mode(&desc[idx], S_DIN_to_AES); + idx++; + + /* process one zero block to generate hkey */ + hw_desc_init(&desc[idx]); + set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE); + set_dout_dlli(&desc[idx], req_ctx->hkey_dma_addr, AES_BLOCK_SIZE, + NS_BIT, 0); + set_flow_mode(&desc[idx], DIN_AES_DOUT); + idx++; + + /* Memory Barrier */ + hw_desc_init(&desc[idx]); + set_din_no_dma(&desc[idx], 0, 0xfffff0); + set_dout_no_dma(&desc[idx], 0, 0, 1); + idx++; + + /* Load GHASH subkey */ + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, req_ctx->hkey_dma_addr, + AES_BLOCK_SIZE, NS_BIT); + set_dout_no_dma(&desc[idx], 0, 0, 1); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_aes_not_hash_mode(&desc[idx]); + set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); + set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); + set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); + idx++; + + /* Configure Hash Engine to work with GHASH. + * Since it was not possible to extend HASH submodes to add GHASH, + * The following command is necessary in order to + * select GHASH (according to HW designers) + */ + hw_desc_init(&desc[idx]); + set_din_no_dma(&desc[idx], 0, 0xfffff0); + set_dout_no_dma(&desc[idx], 0, 0, 1); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_aes_not_hash_mode(&desc[idx]); + set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); + set_cipher_do(&desc[idx], 1); //1=AES_SK RKEK + set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); + set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); + set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); + idx++; + + /* Load GHASH initial STATE (which is 0). (for any hash there is an + * initial state) + */ + hw_desc_init(&desc[idx]); + set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE); + set_dout_no_dma(&desc[idx], 0, 0, 1); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_aes_not_hash_mode(&desc[idx]); + set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); + set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); + set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); + idx++; + + *seq_size = idx; +} + +static void cc_set_gctr_desc(struct aead_request *req, struct cc_hw_desc desc[], + unsigned int *seq_size) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); + unsigned int idx = *seq_size; + + /* load key to AES*/ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); + set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); + set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, + ctx->enc_keylen, NS_BIT); + set_key_size_aes(&desc[idx], ctx->enc_keylen); + set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); + set_flow_mode(&desc[idx], S_DIN_to_AES); + idx++; + + if (req_ctx->cryptlen && !req_ctx->plaintext_authenticate_only) { + /* load AES/CTR initial CTR value inc by 2*/ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); + set_key_size_aes(&desc[idx], ctx->enc_keylen); + set_din_type(&desc[idx], DMA_DLLI, + req_ctx->gcm_iv_inc2_dma_addr, AES_BLOCK_SIZE, + NS_BIT); + set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); + set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); + set_flow_mode(&desc[idx], S_DIN_to_AES); + idx++; + } + + *seq_size = idx; +} + +static void cc_proc_gcm_result(struct aead_request *req, + struct cc_hw_desc desc[], + unsigned int *seq_size) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); + dma_addr_t mac_result; + unsigned int idx = *seq_size; + + if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { + mac_result = req_ctx->mac_buf_dma_addr; + } else { /* Encrypt */ + mac_result = req_ctx->icv_dma_addr; + } + + /* process(ghash) gcm_block_len */ + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_block_len_dma_addr, + AES_BLOCK_SIZE, NS_BIT); + set_flow_mode(&desc[idx], DIN_HASH); + idx++; + + /* Store GHASH state after GHASH(Associated Data + Cipher +LenBlock) */ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); + set_din_no_dma(&desc[idx], 0, 0xfffff0); + set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, AES_BLOCK_SIZE, + NS_BIT, 0); + set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); + set_flow_mode(&desc[idx], S_HASH_to_DOUT); + set_aes_not_hash_mode(&desc[idx]); + + idx++; + + /* load AES/CTR initial CTR value inc by 1*/ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); + set_key_size_aes(&desc[idx], ctx->enc_keylen); + set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_iv_inc1_dma_addr, + AES_BLOCK_SIZE, NS_BIT); + set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); + set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); + set_flow_mode(&desc[idx], S_DIN_to_AES); + idx++; + + /* Memory Barrier */ + hw_desc_init(&desc[idx]); + set_din_no_dma(&desc[idx], 0, 0xfffff0); + set_dout_no_dma(&desc[idx], 0, 0, 1); + idx++; + + /* process GCTR on stored GHASH and store MAC in mac_state*/ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); + set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr, + AES_BLOCK_SIZE, NS_BIT); + set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1); + set_queue_last_ind(ctx->drvdata, &desc[idx]); + set_flow_mode(&desc[idx], DIN_AES_DOUT); + idx++; + + *seq_size = idx; +} + +static int cc_gcm(struct aead_request *req, struct cc_hw_desc desc[], + unsigned int *seq_size) +{ + struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); + unsigned int cipher_flow_mode; + + //in RFC4543 no data to encrypt. just copy data from src to dest. + if (req_ctx->plaintext_authenticate_only) { + cc_proc_cipher_desc(req, BYPASS, desc, seq_size); + cc_set_ghash_desc(req, desc, seq_size); + /* process(ghash) assoc data */ + cc_set_assoc_desc(req, DIN_HASH, desc, seq_size); + cc_set_gctr_desc(req, desc, seq_size); + cc_proc_gcm_result(req, desc, seq_size); + return 0; + } + + if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { + cipher_flow_mode = AES_and_HASH; + } else { /* Encrypt */ + cipher_flow_mode = AES_to_HASH_and_DOUT; + } + + // for gcm and rfc4106. + cc_set_ghash_desc(req, desc, seq_size); + /* process(ghash) assoc data */ + if (req_ctx->assoclen > 0) + cc_set_assoc_desc(req, DIN_HASH, desc, seq_size); + cc_set_gctr_desc(req, desc, seq_size); + /* process(gctr+ghash) */ + if (req_ctx->cryptlen) + cc_proc_cipher_desc(req, cipher_flow_mode, desc, seq_size); + cc_proc_gcm_result(req, desc, seq_size); + + return 0; +} + +static int config_gcm_context(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); + struct device *dev = drvdata_to_dev(ctx->drvdata); + + unsigned int cryptlen = (req_ctx->gen_ctx.op_type == + DRV_CRYPTO_DIRECTION_ENCRYPT) ? + req->cryptlen : + (req->cryptlen - ctx->authsize); + __be32 counter = cpu_to_be32(2); + + dev_dbg(dev, "%s() cryptlen = %d, req_ctx->assoclen = %d ctx->authsize = %d\n", + __func__, cryptlen, req_ctx->assoclen, ctx->authsize); + + memset(req_ctx->hkey, 0, AES_BLOCK_SIZE); + + memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE); + + memcpy(req->iv + 12, &counter, 4); + memcpy(req_ctx->gcm_iv_inc2, req->iv, 16); + + counter = cpu_to_be32(1); + memcpy(req->iv + 12, &counter, 4); + memcpy(req_ctx->gcm_iv_inc1, req->iv, 16); + + if (!req_ctx->plaintext_authenticate_only) { + __be64 temp64; + + temp64 = cpu_to_be64(req_ctx->assoclen * 8); + memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64)); + temp64 = cpu_to_be64(cryptlen * 8); + memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8); + } else { + /* rfc4543=> all data(AAD,IV,Plain) are considered additional + * data that is nothing is encrypted. + */ + __be64 temp64; + + temp64 = cpu_to_be64((req_ctx->assoclen + cryptlen) * 8); + memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64)); + temp64 = 0; + memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8); + } + + return 0; +} + +static void cc_proc_rfc4_gcm(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); + + memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_NONCE_OFFSET, + ctx->ctr_nonce, GCM_BLOCK_RFC4_NONCE_SIZE); + memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_IV_OFFSET, req->iv, + GCM_BLOCK_RFC4_IV_SIZE); + req->iv = areq_ctx->ctr_iv; +} + +static int cc_proc_aead(struct aead_request *req, + enum drv_crypto_direction direct) +{ + int rc = 0; + int seq_len = 0; + struct cc_hw_desc desc[MAX_AEAD_PROCESS_SEQ]; + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); + struct device *dev = drvdata_to_dev(ctx->drvdata); + struct cc_crypto_req cc_req = {}; + + dev_dbg(dev, "%s context=%p req=%p iv=%p src=%p src_ofs=%d dst=%p dst_ofs=%d cryptolen=%d\n", + ((direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? "Enc" : "Dec"), + ctx, req, req->iv, sg_virt(req->src), req->src->offset, + sg_virt(req->dst), req->dst->offset, req->cryptlen); + + /* STAT_PHASE_0: Init and sanity checks */ + + /* Check data length according to mode */ + if (validate_data_size(ctx, direct, req)) { + dev_err(dev, "Unsupported crypt/assoc len %d/%d.\n", + req->cryptlen, areq_ctx->assoclen); + return -EINVAL; + } + + /* Setup request structure */ + cc_req.user_cb = cc_aead_complete; + cc_req.user_arg = req; + + /* Setup request context */ + areq_ctx->gen_ctx.op_type = direct; + areq_ctx->req_authsize = ctx->authsize; + areq_ctx->cipher_mode = ctx->cipher_mode; + + /* STAT_PHASE_1: Map buffers */ + + if (ctx->cipher_mode == DRV_CIPHER_CTR) { + /* Build CTR IV - Copy nonce from last 4 bytes in + * CTR key to first 4 bytes in CTR IV + */ + memcpy(areq_ctx->ctr_iv, ctx->ctr_nonce, + CTR_RFC3686_NONCE_SIZE); + memcpy(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE, req->iv, + CTR_RFC3686_IV_SIZE); + /* Initialize counter portion of counter block */ + *(__be32 *)(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE + + CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); + + /* Replace with counter iv */ + req->iv = areq_ctx->ctr_iv; + areq_ctx->hw_iv_size = CTR_RFC3686_BLOCK_SIZE; + } else if ((ctx->cipher_mode == DRV_CIPHER_CCM) || + (ctx->cipher_mode == DRV_CIPHER_GCTR)) { + areq_ctx->hw_iv_size = AES_BLOCK_SIZE; + if (areq_ctx->ctr_iv != req->iv) { + memcpy(areq_ctx->ctr_iv, req->iv, + crypto_aead_ivsize(tfm)); + req->iv = areq_ctx->ctr_iv; + } + } else { + areq_ctx->hw_iv_size = crypto_aead_ivsize(tfm); + } + + if (ctx->cipher_mode == DRV_CIPHER_CCM) { + rc = config_ccm_adata(req); + if (rc) { + dev_dbg(dev, "config_ccm_adata() returned with a failure %d!", + rc); + goto exit; + } + } else { + areq_ctx->ccm_hdr_size = ccm_header_size_null; + } + + if (ctx->cipher_mode == DRV_CIPHER_GCTR) { + rc = config_gcm_context(req); + if (rc) { + dev_dbg(dev, "config_gcm_context() returned with a failure %d!", + rc); + goto exit; + } + } + + rc = cc_map_aead_request(ctx->drvdata, req); + if (rc) { + dev_err(dev, "map_request() failed\n"); + goto exit; + } + + /* STAT_PHASE_2: Create sequence */ + + /* Load MLLI tables to SRAM if necessary */ + cc_mlli_to_sram(req, desc, &seq_len); + + switch (ctx->auth_mode) { + case DRV_HASH_SHA1: + case DRV_HASH_SHA256: + cc_hmac_authenc(req, desc, &seq_len); + break; + case DRV_HASH_XCBC_MAC: + cc_xcbc_authenc(req, desc, &seq_len); + break; + case DRV_HASH_NULL: + if (ctx->cipher_mode == DRV_CIPHER_CCM) + cc_ccm(req, desc, &seq_len); + if (ctx->cipher_mode == DRV_CIPHER_GCTR) + cc_gcm(req, desc, &seq_len); + break; + default: + dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode); + cc_unmap_aead_request(dev, req); + rc = -ENOTSUPP; + goto exit; + } + + /* STAT_PHASE_3: Lock HW and push sequence */ + + rc = cc_send_request(ctx->drvdata, &cc_req, desc, seq_len, &req->base); + + if (rc != -EINPROGRESS && rc != -EBUSY) { + dev_err(dev, "send_request() failed (rc=%d)\n", rc); + cc_unmap_aead_request(dev, req); + } + +exit: + return rc; +} + +static int cc_aead_encrypt(struct aead_request *req) +{ + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); + int rc; + + memset(areq_ctx, 0, sizeof(*areq_ctx)); + + /* No generated IV required */ + areq_ctx->backup_iv = req->iv; + areq_ctx->assoclen = req->assoclen; + + rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); + if (rc != -EINPROGRESS && rc != -EBUSY) + req->iv = areq_ctx->backup_iv; + + return rc; +} + +static int cc_rfc4309_ccm_encrypt(struct aead_request *req) +{ + /* Very similar to cc_aead_encrypt() above. */ + + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); + int rc; + + rc = crypto_ipsec_check_assoclen(req->assoclen); + if (rc) + goto out; + + memset(areq_ctx, 0, sizeof(*areq_ctx)); + + /* No generated IV required */ + areq_ctx->backup_iv = req->iv; + areq_ctx->assoclen = req->assoclen - CCM_BLOCK_IV_SIZE; + + cc_proc_rfc4309_ccm(req); + + rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); + if (rc != -EINPROGRESS && rc != -EBUSY) + req->iv = areq_ctx->backup_iv; +out: + return rc; +} + +static int cc_aead_decrypt(struct aead_request *req) +{ + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); + int rc; + + memset(areq_ctx, 0, sizeof(*areq_ctx)); + + /* No generated IV required */ + areq_ctx->backup_iv = req->iv; + areq_ctx->assoclen = req->assoclen; + + rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); + if (rc != -EINPROGRESS && rc != -EBUSY) + req->iv = areq_ctx->backup_iv; + + return rc; +} + +static int cc_rfc4309_ccm_decrypt(struct aead_request *req) +{ + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); + int rc; + + rc = crypto_ipsec_check_assoclen(req->assoclen); + if (rc) + goto out; + + memset(areq_ctx, 0, sizeof(*areq_ctx)); + + /* No generated IV required */ + areq_ctx->backup_iv = req->iv; + areq_ctx->assoclen = req->assoclen - CCM_BLOCK_IV_SIZE; + + cc_proc_rfc4309_ccm(req); + + rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); + if (rc != -EINPROGRESS && rc != -EBUSY) + req->iv = areq_ctx->backup_iv; + +out: + return rc; +} + +static int cc_rfc4106_gcm_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct device *dev = drvdata_to_dev(ctx->drvdata); + + dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key); + + if (keylen < 4) + return -EINVAL; + + keylen -= 4; + memcpy(ctx->ctr_nonce, key + keylen, 4); + + return cc_aead_setkey(tfm, key, keylen); +} + +static int cc_rfc4543_gcm_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct device *dev = drvdata_to_dev(ctx->drvdata); + + dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key); + + if (keylen < 4) + return -EINVAL; + + keylen -= 4; + memcpy(ctx->ctr_nonce, key + keylen, 4); + + return cc_aead_setkey(tfm, key, keylen); +} + +static int cc_gcm_setauthsize(struct crypto_aead *authenc, + unsigned int authsize) +{ + switch (authsize) { + case 4: + case 8: + case 12: + case 13: + case 14: + case 15: + case 16: + break; + default: + return -EINVAL; + } + + return cc_aead_setauthsize(authenc, authsize); +} + +static int cc_rfc4106_gcm_setauthsize(struct crypto_aead *authenc, + unsigned int authsize) +{ + struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc); + struct device *dev = drvdata_to_dev(ctx->drvdata); + + dev_dbg(dev, "authsize %d\n", authsize); + + switch (authsize) { + case 8: + case 12: + case 16: + break; + default: + return -EINVAL; + } + + return cc_aead_setauthsize(authenc, authsize); +} + +static int cc_rfc4543_gcm_setauthsize(struct crypto_aead *authenc, + unsigned int authsize) +{ + struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc); + struct device *dev = drvdata_to_dev(ctx->drvdata); + + dev_dbg(dev, "authsize %d\n", authsize); + + if (authsize != 16) + return -EINVAL; + + return cc_aead_setauthsize(authenc, authsize); +} + +static int cc_rfc4106_gcm_encrypt(struct aead_request *req) +{ + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); + int rc; + + rc = crypto_ipsec_check_assoclen(req->assoclen); + if (rc) + goto out; + + memset(areq_ctx, 0, sizeof(*areq_ctx)); + + /* No generated IV required */ + areq_ctx->backup_iv = req->iv; + areq_ctx->assoclen = req->assoclen - GCM_BLOCK_RFC4_IV_SIZE; + + cc_proc_rfc4_gcm(req); + + rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); + if (rc != -EINPROGRESS && rc != -EBUSY) + req->iv = areq_ctx->backup_iv; +out: + return rc; +} + +static int cc_rfc4543_gcm_encrypt(struct aead_request *req) +{ + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); + int rc; + + rc = crypto_ipsec_check_assoclen(req->assoclen); + if (rc) + goto out; + + memset(areq_ctx, 0, sizeof(*areq_ctx)); + + //plaintext is not encrypted with rfc4543 + areq_ctx->plaintext_authenticate_only = true; + + /* No generated IV required */ + areq_ctx->backup_iv = req->iv; + areq_ctx->assoclen = req->assoclen; + + cc_proc_rfc4_gcm(req); + + rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); + if (rc != -EINPROGRESS && rc != -EBUSY) + req->iv = areq_ctx->backup_iv; +out: + return rc; +} + +static int cc_rfc4106_gcm_decrypt(struct aead_request *req) +{ + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); + int rc; + + rc = crypto_ipsec_check_assoclen(req->assoclen); + if (rc) + goto out; + + memset(areq_ctx, 0, sizeof(*areq_ctx)); + + /* No generated IV required */ + areq_ctx->backup_iv = req->iv; + areq_ctx->assoclen = req->assoclen - GCM_BLOCK_RFC4_IV_SIZE; + + cc_proc_rfc4_gcm(req); + + rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); + if (rc != -EINPROGRESS && rc != -EBUSY) + req->iv = areq_ctx->backup_iv; +out: + return rc; +} + +static int cc_rfc4543_gcm_decrypt(struct aead_request *req) +{ + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); + int rc; + + rc = crypto_ipsec_check_assoclen(req->assoclen); + if (rc) + goto out; + + memset(areq_ctx, 0, sizeof(*areq_ctx)); + + //plaintext is not decrypted with rfc4543 + areq_ctx->plaintext_authenticate_only = true; + + /* No generated IV required */ + areq_ctx->backup_iv = req->iv; + areq_ctx->assoclen = req->assoclen; + + cc_proc_rfc4_gcm(req); + + rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); + if (rc != -EINPROGRESS && rc != -EBUSY) + req->iv = areq_ctx->backup_iv; +out: + return rc; +} + +/* aead alg */ +static struct cc_alg_template aead_algs[] = { + { + .name = "authenc(hmac(sha1),cbc(aes))", + .driver_name = "authenc-hmac-sha1-cbc-aes-ccree", + .blocksize = AES_BLOCK_SIZE, + .template_aead = { + .setkey = cc_aead_setkey, + .setauthsize = cc_aead_setauthsize, + .encrypt = cc_aead_encrypt, + .decrypt = cc_aead_decrypt, + .init = cc_aead_init, + .exit = cc_aead_exit, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .cipher_mode = DRV_CIPHER_CBC, + .flow_mode = S_DIN_to_AES, + .auth_mode = DRV_HASH_SHA1, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "authenc(hmac(sha1),cbc(des3_ede))", + .driver_name = "authenc-hmac-sha1-cbc-des3-ccree", + .blocksize = DES3_EDE_BLOCK_SIZE, + .template_aead = { + .setkey = cc_des3_aead_setkey, + .setauthsize = cc_aead_setauthsize, + .encrypt = cc_aead_encrypt, + .decrypt = cc_aead_decrypt, + .init = cc_aead_init, + .exit = cc_aead_exit, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .cipher_mode = DRV_CIPHER_CBC, + .flow_mode = S_DIN_to_DES, + .auth_mode = DRV_HASH_SHA1, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "authenc(hmac(sha256),cbc(aes))", + .driver_name = "authenc-hmac-sha256-cbc-aes-ccree", + .blocksize = AES_BLOCK_SIZE, + .template_aead = { + .setkey = cc_aead_setkey, + .setauthsize = cc_aead_setauthsize, + .encrypt = cc_aead_encrypt, + .decrypt = cc_aead_decrypt, + .init = cc_aead_init, + .exit = cc_aead_exit, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .cipher_mode = DRV_CIPHER_CBC, + .flow_mode = S_DIN_to_AES, + .auth_mode = DRV_HASH_SHA256, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "authenc(hmac(sha256),cbc(des3_ede))", + .driver_name = "authenc-hmac-sha256-cbc-des3-ccree", + .blocksize = DES3_EDE_BLOCK_SIZE, + .template_aead = { + .setkey = cc_des3_aead_setkey, + .setauthsize = cc_aead_setauthsize, + .encrypt = cc_aead_encrypt, + .decrypt = cc_aead_decrypt, + .init = cc_aead_init, + .exit = cc_aead_exit, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .cipher_mode = DRV_CIPHER_CBC, + .flow_mode = S_DIN_to_DES, + .auth_mode = DRV_HASH_SHA256, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "authenc(xcbc(aes),cbc(aes))", + .driver_name = "authenc-xcbc-aes-cbc-aes-ccree", + .blocksize = AES_BLOCK_SIZE, + .template_aead = { + .setkey = cc_aead_setkey, + .setauthsize = cc_aead_setauthsize, + .encrypt = cc_aead_encrypt, + .decrypt = cc_aead_decrypt, + .init = cc_aead_init, + .exit = cc_aead_exit, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_CBC, + .flow_mode = S_DIN_to_AES, + .auth_mode = DRV_HASH_XCBC_MAC, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", + .driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-ccree", + .blocksize = 1, + .template_aead = { + .setkey = cc_aead_setkey, + .setauthsize = cc_aead_setauthsize, + .encrypt = cc_aead_encrypt, + .decrypt = cc_aead_decrypt, + .init = cc_aead_init, + .exit = cc_aead_exit, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .cipher_mode = DRV_CIPHER_CTR, + .flow_mode = S_DIN_to_AES, + .auth_mode = DRV_HASH_SHA1, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", + .driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-ccree", + .blocksize = 1, + .template_aead = { + .setkey = cc_aead_setkey, + .setauthsize = cc_aead_setauthsize, + .encrypt = cc_aead_encrypt, + .decrypt = cc_aead_decrypt, + .init = cc_aead_init, + .exit = cc_aead_exit, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .cipher_mode = DRV_CIPHER_CTR, + .flow_mode = S_DIN_to_AES, + .auth_mode = DRV_HASH_SHA256, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "authenc(xcbc(aes),rfc3686(ctr(aes)))", + .driver_name = "authenc-xcbc-aes-rfc3686-ctr-aes-ccree", + .blocksize = 1, + .template_aead = { + .setkey = cc_aead_setkey, + .setauthsize = cc_aead_setauthsize, + .encrypt = cc_aead_encrypt, + .decrypt = cc_aead_decrypt, + .init = cc_aead_init, + .exit = cc_aead_exit, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_CTR, + .flow_mode = S_DIN_to_AES, + .auth_mode = DRV_HASH_XCBC_MAC, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "ccm(aes)", + .driver_name = "ccm-aes-ccree", + .blocksize = 1, + .template_aead = { + .setkey = cc_aead_setkey, + .setauthsize = cc_ccm_setauthsize, + .encrypt = cc_aead_encrypt, + .decrypt = cc_aead_decrypt, + .init = cc_aead_init, + .exit = cc_aead_exit, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_CCM, + .flow_mode = S_DIN_to_AES, + .auth_mode = DRV_HASH_NULL, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "rfc4309(ccm(aes))", + .driver_name = "rfc4309-ccm-aes-ccree", + .blocksize = 1, + .template_aead = { + .setkey = cc_rfc4309_ccm_setkey, + .setauthsize = cc_rfc4309_ccm_setauthsize, + .encrypt = cc_rfc4309_ccm_encrypt, + .decrypt = cc_rfc4309_ccm_decrypt, + .init = cc_aead_init, + .exit = cc_aead_exit, + .ivsize = CCM_BLOCK_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_CCM, + .flow_mode = S_DIN_to_AES, + .auth_mode = DRV_HASH_NULL, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "gcm(aes)", + .driver_name = "gcm-aes-ccree", + .blocksize = 1, + .template_aead = { + .setkey = cc_aead_setkey, + .setauthsize = cc_gcm_setauthsize, + .encrypt = cc_aead_encrypt, + .decrypt = cc_aead_decrypt, + .init = cc_aead_init, + .exit = cc_aead_exit, + .ivsize = 12, + .maxauthsize = AES_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_GCTR, + .flow_mode = S_DIN_to_AES, + .auth_mode = DRV_HASH_NULL, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "rfc4106(gcm(aes))", + .driver_name = "rfc4106-gcm-aes-ccree", + .blocksize = 1, + .template_aead = { + .setkey = cc_rfc4106_gcm_setkey, + .setauthsize = cc_rfc4106_gcm_setauthsize, + .encrypt = cc_rfc4106_gcm_encrypt, + .decrypt = cc_rfc4106_gcm_decrypt, + .init = cc_aead_init, + .exit = cc_aead_exit, + .ivsize = GCM_BLOCK_RFC4_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_GCTR, + .flow_mode = S_DIN_to_AES, + .auth_mode = DRV_HASH_NULL, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "rfc4543(gcm(aes))", + .driver_name = "rfc4543-gcm-aes-ccree", + .blocksize = 1, + .template_aead = { + .setkey = cc_rfc4543_gcm_setkey, + .setauthsize = cc_rfc4543_gcm_setauthsize, + .encrypt = cc_rfc4543_gcm_encrypt, + .decrypt = cc_rfc4543_gcm_decrypt, + .init = cc_aead_init, + .exit = cc_aead_exit, + .ivsize = GCM_BLOCK_RFC4_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_GCTR, + .flow_mode = S_DIN_to_AES, + .auth_mode = DRV_HASH_NULL, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, +}; + +static struct cc_crypto_alg *cc_create_aead_alg(struct cc_alg_template *tmpl, + struct device *dev) +{ + struct cc_crypto_alg *t_alg; + struct aead_alg *alg; + + t_alg = devm_kzalloc(dev, sizeof(*t_alg), GFP_KERNEL); + if (!t_alg) + return ERR_PTR(-ENOMEM); + + alg = &tmpl->template_aead; + + if (strscpy(alg->base.cra_name, tmpl->name) < 0) + return ERR_PTR(-EINVAL); + if (strscpy(alg->base.cra_driver_name, tmpl->driver_name) < 0) + return ERR_PTR(-EINVAL); + + alg->base.cra_module = THIS_MODULE; + alg->base.cra_priority = CC_CRA_PRIO; + + alg->base.cra_ctxsize = sizeof(struct cc_aead_ctx); + alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; + alg->base.cra_blocksize = tmpl->blocksize; + alg->init = cc_aead_init; + alg->exit = cc_aead_exit; + + t_alg->aead_alg = *alg; + + t_alg->cipher_mode = tmpl->cipher_mode; + t_alg->flow_mode = tmpl->flow_mode; + t_alg->auth_mode = tmpl->auth_mode; + + return t_alg; +} + +int cc_aead_free(struct cc_drvdata *drvdata) +{ + struct cc_crypto_alg *t_alg, *n; + struct cc_aead_handle *aead_handle = drvdata->aead_handle; + + /* Remove registered algs */ + list_for_each_entry_safe(t_alg, n, &aead_handle->aead_list, entry) { + crypto_unregister_aead(&t_alg->aead_alg); + list_del(&t_alg->entry); + } + + return 0; +} + +int cc_aead_alloc(struct cc_drvdata *drvdata) +{ + struct cc_aead_handle *aead_handle; + struct cc_crypto_alg *t_alg; + int rc = -ENOMEM; + int alg; + struct device *dev = drvdata_to_dev(drvdata); + + aead_handle = devm_kmalloc(dev, sizeof(*aead_handle), GFP_KERNEL); + if (!aead_handle) { + rc = -ENOMEM; + goto fail0; + } + + INIT_LIST_HEAD(&aead_handle->aead_list); + drvdata->aead_handle = aead_handle; + + aead_handle->sram_workspace_addr = cc_sram_alloc(drvdata, + MAX_HMAC_DIGEST_SIZE); + + if (aead_handle->sram_workspace_addr == NULL_SRAM_ADDR) { + rc = -ENOMEM; + goto fail1; + } + + /* Linux crypto */ + for (alg = 0; alg < ARRAY_SIZE(aead_algs); alg++) { + if ((aead_algs[alg].min_hw_rev > drvdata->hw_rev) || + !(drvdata->std_bodies & aead_algs[alg].std_body)) + continue; + + t_alg = cc_create_aead_alg(&aead_algs[alg], dev); + if (IS_ERR(t_alg)) { + rc = PTR_ERR(t_alg); + dev_err(dev, "%s alg allocation failed\n", + aead_algs[alg].driver_name); + goto fail1; + } + t_alg->drvdata = drvdata; + rc = crypto_register_aead(&t_alg->aead_alg); + if (rc) { + dev_err(dev, "%s alg registration failed\n", + t_alg->aead_alg.base.cra_driver_name); + goto fail1; + } + + list_add_tail(&t_alg->entry, &aead_handle->aead_list); + dev_dbg(dev, "Registered %s\n", + t_alg->aead_alg.base.cra_driver_name); + } + + return 0; + +fail1: + cc_aead_free(drvdata); +fail0: + return rc; +} diff --git a/drivers/crypto/ccree/cc_aead.h b/drivers/crypto/ccree/cc_aead.h new file mode 100644 index 000000000..b69591550 --- /dev/null +++ b/drivers/crypto/ccree/cc_aead.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ + +/* \file cc_aead.h + * ARM CryptoCell AEAD Crypto API + */ + +#ifndef __CC_AEAD_H__ +#define __CC_AEAD_H__ + +#include +#include +#include + +/* mac_cmp - HW writes 8 B but all bytes hold the same value */ +#define ICV_CMP_SIZE 8 +#define CCM_CONFIG_BUF_SIZE (AES_BLOCK_SIZE * 3) +#define MAX_MAC_SIZE SHA256_DIGEST_SIZE + +/* defines for AES GCM configuration buffer */ +#define GCM_BLOCK_LEN_SIZE 8 + +#define GCM_BLOCK_RFC4_IV_OFFSET 4 +#define GCM_BLOCK_RFC4_IV_SIZE 8 /* IV size for rfc's */ +#define GCM_BLOCK_RFC4_NONCE_OFFSET 0 +#define GCM_BLOCK_RFC4_NONCE_SIZE 4 + +/* Offsets into AES CCM configuration buffer */ +#define CCM_B0_OFFSET 0 +#define CCM_A0_OFFSET 16 +#define CCM_CTR_COUNT_0_OFFSET 32 +/* CCM B0 and CTR_COUNT constants. */ +#define CCM_BLOCK_NONCE_OFFSET 1 /* Nonce offset inside B0 and CTR_COUNT */ +#define CCM_BLOCK_NONCE_SIZE 3 /* Nonce size inside B0 and CTR_COUNT */ +#define CCM_BLOCK_IV_OFFSET 4 /* IV offset inside B0 and CTR_COUNT */ +#define CCM_BLOCK_IV_SIZE 8 /* IV size inside B0 and CTR_COUNT */ + +enum aead_ccm_header_size { + ccm_header_size_null = -1, + ccm_header_size_zero = 0, + ccm_header_size_2 = 2, + ccm_header_size_6 = 6, + ccm_header_size_max = S32_MAX +}; + +struct aead_req_ctx { + /* Allocate cache line although only 4 bytes are needed to + * assure next field falls @ cache line + * Used for both: digest HW compare and CCM/GCM MAC value + */ + u8 mac_buf[MAX_MAC_SIZE] ____cacheline_aligned; + u8 ctr_iv[AES_BLOCK_SIZE] ____cacheline_aligned; + + //used in gcm + u8 gcm_iv_inc1[AES_BLOCK_SIZE] ____cacheline_aligned; + u8 gcm_iv_inc2[AES_BLOCK_SIZE] ____cacheline_aligned; + u8 hkey[AES_BLOCK_SIZE] ____cacheline_aligned; + struct { + u8 len_a[GCM_BLOCK_LEN_SIZE] ____cacheline_aligned; + u8 len_c[GCM_BLOCK_LEN_SIZE]; + } gcm_len_block; + + u8 ccm_config[CCM_CONFIG_BUF_SIZE] ____cacheline_aligned; + /* HW actual size input */ + unsigned int hw_iv_size ____cacheline_aligned; + /* used to prevent cache coherence problem */ + u8 backup_mac[MAX_MAC_SIZE]; + u8 *backup_iv; /* store orig iv */ + u32 assoclen; /* size of AAD buffer to authenticate */ + dma_addr_t mac_buf_dma_addr; /* internal ICV DMA buffer */ + /* buffer for internal ccm configurations */ + dma_addr_t ccm_iv0_dma_addr; + dma_addr_t icv_dma_addr; /* Phys. address of ICV */ + + //used in gcm + /* buffer for internal gcm configurations */ + dma_addr_t gcm_iv_inc1_dma_addr; + /* buffer for internal gcm configurations */ + dma_addr_t gcm_iv_inc2_dma_addr; + dma_addr_t hkey_dma_addr; /* Phys. address of hkey */ + dma_addr_t gcm_block_len_dma_addr; /* Phys. address of gcm block len */ + + u8 *icv_virt_addr; /* Virt. address of ICV */ + struct async_gen_req_ctx gen_ctx; + struct cc_mlli assoc; + struct cc_mlli src; + struct cc_mlli dst; + struct scatterlist *src_sgl; + struct scatterlist *dst_sgl; + unsigned int src_offset; + unsigned int dst_offset; + enum cc_req_dma_buf_type assoc_buff_type; + enum cc_req_dma_buf_type data_buff_type; + struct mlli_params mlli_params; + unsigned int cryptlen; + struct scatterlist ccm_adata_sg; + enum aead_ccm_header_size ccm_hdr_size; + unsigned int req_authsize; + enum drv_cipher_mode cipher_mode; + bool is_icv_fragmented; + bool is_single_pass; + bool plaintext_authenticate_only; //for gcm_rfc4543 +}; + +int cc_aead_alloc(struct cc_drvdata *drvdata); +int cc_aead_free(struct cc_drvdata *drvdata); + +#endif /*__CC_AEAD_H__*/ diff --git a/drivers/crypto/ccree/cc_buffer_mgr.c b/drivers/crypto/ccree/cc_buffer_mgr.c new file mode 100644 index 000000000..dc7e0cd51 --- /dev/null +++ b/drivers/crypto/ccree/cc_buffer_mgr.c @@ -0,0 +1,1397 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ + +#include +#include +#include +#include +#include + +#include "cc_buffer_mgr.h" +#include "cc_lli_defs.h" +#include "cc_cipher.h" +#include "cc_hash.h" +#include "cc_aead.h" + +union buffer_array_entry { + struct scatterlist *sgl; + dma_addr_t buffer_dma; +}; + +struct buffer_array { + unsigned int num_of_buffers; + union buffer_array_entry entry[MAX_NUM_OF_BUFFERS_IN_MLLI]; + unsigned int offset[MAX_NUM_OF_BUFFERS_IN_MLLI]; + int nents[MAX_NUM_OF_BUFFERS_IN_MLLI]; + int total_data_len[MAX_NUM_OF_BUFFERS_IN_MLLI]; + bool is_last[MAX_NUM_OF_BUFFERS_IN_MLLI]; + u32 *mlli_nents[MAX_NUM_OF_BUFFERS_IN_MLLI]; +}; + +static inline char *cc_dma_buf_type(enum cc_req_dma_buf_type type) +{ + switch (type) { + case CC_DMA_BUF_NULL: + return "BUF_NULL"; + case CC_DMA_BUF_DLLI: + return "BUF_DLLI"; + case CC_DMA_BUF_MLLI: + return "BUF_MLLI"; + default: + return "BUF_INVALID"; + } +} + +/** + * cc_copy_mac() - Copy MAC to temporary location + * + * @dev: device object + * @req: aead request object + * @dir: [IN] copy from/to sgl + */ +static void cc_copy_mac(struct device *dev, struct aead_request *req, + enum cc_sg_cpy_direct dir) +{ + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); + u32 skip = req->assoclen + req->cryptlen; + + cc_copy_sg_portion(dev, areq_ctx->backup_mac, req->src, + (skip - areq_ctx->req_authsize), skip, dir); +} + +/** + * cc_get_sgl_nents() - Get scatterlist number of entries. + * + * @dev: Device object + * @sg_list: SG list + * @nbytes: [IN] Total SGL data bytes. + * @lbytes: [OUT] Returns the amount of bytes at the last entry + * + * Return: + * Number of entries in the scatterlist + */ +static unsigned int cc_get_sgl_nents(struct device *dev, + struct scatterlist *sg_list, + unsigned int nbytes, u32 *lbytes) +{ + unsigned int nents = 0; + + *lbytes = 0; + + while (nbytes && sg_list) { + nents++; + /* get the number of bytes in the last entry */ + *lbytes = nbytes; + nbytes -= (sg_list->length > nbytes) ? + nbytes : sg_list->length; + sg_list = sg_next(sg_list); + } + + dev_dbg(dev, "nents %d last bytes %d\n", nents, *lbytes); + return nents; +} + +/** + * cc_copy_sg_portion() - Copy scatter list data, + * from to_skip to end, to dest and vice versa + * + * @dev: Device object + * @dest: Buffer to copy to/from + * @sg: SG list + * @to_skip: Number of bytes to skip before copying + * @end: Offset of last byte to copy + * @direct: Transfer direction (true == from SG list to buffer, false == from + * buffer to SG list) + */ +void cc_copy_sg_portion(struct device *dev, u8 *dest, struct scatterlist *sg, + u32 to_skip, u32 end, enum cc_sg_cpy_direct direct) +{ + u32 nents; + + nents = sg_nents_for_len(sg, end); + sg_copy_buffer(sg, nents, dest, (end - to_skip + 1), to_skip, + (direct == CC_SG_TO_BUF)); +} + +static int cc_render_buff_to_mlli(struct device *dev, dma_addr_t buff_dma, + u32 buff_size, u32 *curr_nents, + u32 **mlli_entry_pp) +{ + u32 *mlli_entry_p = *mlli_entry_pp; + u32 new_nents; + + /* Verify there is no memory overflow*/ + new_nents = (*curr_nents + buff_size / CC_MAX_MLLI_ENTRY_SIZE + 1); + if (new_nents > MAX_NUM_OF_TOTAL_MLLI_ENTRIES) { + dev_err(dev, "Too many mlli entries. current %d max %d\n", + new_nents, MAX_NUM_OF_TOTAL_MLLI_ENTRIES); + return -ENOMEM; + } + + /*handle buffer longer than 64 kbytes */ + while (buff_size > CC_MAX_MLLI_ENTRY_SIZE) { + cc_lli_set_addr(mlli_entry_p, buff_dma); + cc_lli_set_size(mlli_entry_p, CC_MAX_MLLI_ENTRY_SIZE); + dev_dbg(dev, "entry[%d]: single_buff=0x%08X size=%08X\n", + *curr_nents, mlli_entry_p[LLI_WORD0_OFFSET], + mlli_entry_p[LLI_WORD1_OFFSET]); + buff_dma += CC_MAX_MLLI_ENTRY_SIZE; + buff_size -= CC_MAX_MLLI_ENTRY_SIZE; + mlli_entry_p = mlli_entry_p + 2; + (*curr_nents)++; + } + /*Last entry */ + cc_lli_set_addr(mlli_entry_p, buff_dma); + cc_lli_set_size(mlli_entry_p, buff_size); + dev_dbg(dev, "entry[%d]: single_buff=0x%08X size=%08X\n", + *curr_nents, mlli_entry_p[LLI_WORD0_OFFSET], + mlli_entry_p[LLI_WORD1_OFFSET]); + mlli_entry_p = mlli_entry_p + 2; + *mlli_entry_pp = mlli_entry_p; + (*curr_nents)++; + return 0; +} + +static int cc_render_sg_to_mlli(struct device *dev, struct scatterlist *sgl, + u32 sgl_data_len, u32 sgl_offset, + u32 *curr_nents, u32 **mlli_entry_pp) +{ + struct scatterlist *curr_sgl = sgl; + u32 *mlli_entry_p = *mlli_entry_pp; + s32 rc = 0; + + for ( ; (curr_sgl && sgl_data_len); + curr_sgl = sg_next(curr_sgl)) { + u32 entry_data_len = + (sgl_data_len > sg_dma_len(curr_sgl) - sgl_offset) ? + sg_dma_len(curr_sgl) - sgl_offset : + sgl_data_len; + sgl_data_len -= entry_data_len; + rc = cc_render_buff_to_mlli(dev, sg_dma_address(curr_sgl) + + sgl_offset, entry_data_len, + curr_nents, &mlli_entry_p); + if (rc) + return rc; + + sgl_offset = 0; + } + *mlli_entry_pp = mlli_entry_p; + return 0; +} + +static int cc_generate_mlli(struct device *dev, struct buffer_array *sg_data, + struct mlli_params *mlli_params, gfp_t flags) +{ + u32 *mlli_p; + u32 total_nents = 0, prev_total_nents = 0; + int rc = 0, i; + + dev_dbg(dev, "NUM of SG's = %d\n", sg_data->num_of_buffers); + + /* Allocate memory from the pointed pool */ + mlli_params->mlli_virt_addr = + dma_pool_alloc(mlli_params->curr_pool, flags, + &mlli_params->mlli_dma_addr); + if (!mlli_params->mlli_virt_addr) { + dev_err(dev, "dma_pool_alloc() failed\n"); + rc = -ENOMEM; + goto build_mlli_exit; + } + /* Point to start of MLLI */ + mlli_p = mlli_params->mlli_virt_addr; + /* go over all SG's and link it to one MLLI table */ + for (i = 0; i < sg_data->num_of_buffers; i++) { + union buffer_array_entry *entry = &sg_data->entry[i]; + u32 tot_len = sg_data->total_data_len[i]; + u32 offset = sg_data->offset[i]; + + rc = cc_render_sg_to_mlli(dev, entry->sgl, tot_len, offset, + &total_nents, &mlli_p); + if (rc) + return rc; + + /* set last bit in the current table */ + if (sg_data->mlli_nents[i]) { + /*Calculate the current MLLI table length for the + *length field in the descriptor + */ + *sg_data->mlli_nents[i] += + (total_nents - prev_total_nents); + prev_total_nents = total_nents; + } + } + + /* Set MLLI size for the bypass operation */ + mlli_params->mlli_len = (total_nents * LLI_ENTRY_BYTE_SIZE); + + dev_dbg(dev, "MLLI params: virt_addr=%p dma_addr=%pad mlli_len=0x%X\n", + mlli_params->mlli_virt_addr, &mlli_params->mlli_dma_addr, + mlli_params->mlli_len); + +build_mlli_exit: + return rc; +} + +static void cc_add_sg_entry(struct device *dev, struct buffer_array *sgl_data, + unsigned int nents, struct scatterlist *sgl, + unsigned int data_len, unsigned int data_offset, + bool is_last_table, u32 *mlli_nents) +{ + unsigned int index = sgl_data->num_of_buffers; + + dev_dbg(dev, "index=%u nents=%u sgl=%p data_len=0x%08X is_last=%d\n", + index, nents, sgl, data_len, is_last_table); + sgl_data->nents[index] = nents; + sgl_data->entry[index].sgl = sgl; + sgl_data->offset[index] = data_offset; + sgl_data->total_data_len[index] = data_len; + sgl_data->is_last[index] = is_last_table; + sgl_data->mlli_nents[index] = mlli_nents; + if (sgl_data->mlli_nents[index]) + *sgl_data->mlli_nents[index] = 0; + sgl_data->num_of_buffers++; +} + +static int cc_map_sg(struct device *dev, struct scatterlist *sg, + unsigned int nbytes, int direction, u32 *nents, + u32 max_sg_nents, u32 *lbytes, u32 *mapped_nents) +{ + int ret = 0; + + if (!nbytes) { + *mapped_nents = 0; + *lbytes = 0; + *nents = 0; + return 0; + } + + *nents = cc_get_sgl_nents(dev, sg, nbytes, lbytes); + if (*nents > max_sg_nents) { + *nents = 0; + dev_err(dev, "Too many fragments. current %d max %d\n", + *nents, max_sg_nents); + return -ENOMEM; + } + + ret = dma_map_sg(dev, sg, *nents, direction); + if (!ret) { + *nents = 0; + dev_err(dev, "dma_map_sg() sg buffer failed %d\n", ret); + return -ENOMEM; + } + + *mapped_nents = ret; + + return 0; +} + +static int +cc_set_aead_conf_buf(struct device *dev, struct aead_req_ctx *areq_ctx, + u8 *config_data, struct buffer_array *sg_data, + unsigned int assoclen) +{ + dev_dbg(dev, " handle additional data config set to DLLI\n"); + /* create sg for the current buffer */ + sg_init_one(&areq_ctx->ccm_adata_sg, config_data, + AES_BLOCK_SIZE + areq_ctx->ccm_hdr_size); + if (dma_map_sg(dev, &areq_ctx->ccm_adata_sg, 1, DMA_TO_DEVICE) != 1) { + dev_err(dev, "dma_map_sg() config buffer failed\n"); + return -ENOMEM; + } + dev_dbg(dev, "Mapped curr_buff: dma_address=%pad page=%p addr=%p offset=%u length=%u\n", + &sg_dma_address(&areq_ctx->ccm_adata_sg), + sg_page(&areq_ctx->ccm_adata_sg), + sg_virt(&areq_ctx->ccm_adata_sg), + areq_ctx->ccm_adata_sg.offset, areq_ctx->ccm_adata_sg.length); + /* prepare for case of MLLI */ + if (assoclen > 0) { + cc_add_sg_entry(dev, sg_data, 1, &areq_ctx->ccm_adata_sg, + (AES_BLOCK_SIZE + areq_ctx->ccm_hdr_size), + 0, false, NULL); + } + return 0; +} + +static int cc_set_hash_buf(struct device *dev, struct ahash_req_ctx *areq_ctx, + u8 *curr_buff, u32 curr_buff_cnt, + struct buffer_array *sg_data) +{ + dev_dbg(dev, " handle curr buff %x set to DLLI\n", curr_buff_cnt); + /* create sg for the current buffer */ + sg_init_one(areq_ctx->buff_sg, curr_buff, curr_buff_cnt); + if (dma_map_sg(dev, areq_ctx->buff_sg, 1, DMA_TO_DEVICE) != 1) { + dev_err(dev, "dma_map_sg() src buffer failed\n"); + return -ENOMEM; + } + dev_dbg(dev, "Mapped curr_buff: dma_address=%pad page=%p addr=%p offset=%u length=%u\n", + &sg_dma_address(areq_ctx->buff_sg), sg_page(areq_ctx->buff_sg), + sg_virt(areq_ctx->buff_sg), areq_ctx->buff_sg->offset, + areq_ctx->buff_sg->length); + areq_ctx->data_dma_buf_type = CC_DMA_BUF_DLLI; + areq_ctx->curr_sg = areq_ctx->buff_sg; + areq_ctx->in_nents = 0; + /* prepare for case of MLLI */ + cc_add_sg_entry(dev, sg_data, 1, areq_ctx->buff_sg, curr_buff_cnt, 0, + false, NULL); + return 0; +} + +void cc_unmap_cipher_request(struct device *dev, void *ctx, + unsigned int ivsize, struct scatterlist *src, + struct scatterlist *dst) +{ + struct cipher_req_ctx *req_ctx = (struct cipher_req_ctx *)ctx; + + if (req_ctx->gen_ctx.iv_dma_addr) { + dev_dbg(dev, "Unmapped iv: iv_dma_addr=%pad iv_size=%u\n", + &req_ctx->gen_ctx.iv_dma_addr, ivsize); + dma_unmap_single(dev, req_ctx->gen_ctx.iv_dma_addr, + ivsize, DMA_BIDIRECTIONAL); + } + /* Release pool */ + if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI && + req_ctx->mlli_params.mlli_virt_addr) { + dma_pool_free(req_ctx->mlli_params.curr_pool, + req_ctx->mlli_params.mlli_virt_addr, + req_ctx->mlli_params.mlli_dma_addr); + } + + if (src != dst) { + dma_unmap_sg(dev, src, req_ctx->in_nents, DMA_TO_DEVICE); + dma_unmap_sg(dev, dst, req_ctx->out_nents, DMA_FROM_DEVICE); + dev_dbg(dev, "Unmapped req->dst=%p\n", sg_virt(dst)); + dev_dbg(dev, "Unmapped req->src=%p\n", sg_virt(src)); + } else { + dma_unmap_sg(dev, src, req_ctx->in_nents, DMA_BIDIRECTIONAL); + dev_dbg(dev, "Unmapped req->src=%p\n", sg_virt(src)); + } +} + +int cc_map_cipher_request(struct cc_drvdata *drvdata, void *ctx, + unsigned int ivsize, unsigned int nbytes, + void *info, struct scatterlist *src, + struct scatterlist *dst, gfp_t flags) +{ + struct cipher_req_ctx *req_ctx = (struct cipher_req_ctx *)ctx; + struct mlli_params *mlli_params = &req_ctx->mlli_params; + struct device *dev = drvdata_to_dev(drvdata); + struct buffer_array sg_data; + u32 dummy = 0; + int rc = 0; + u32 mapped_nents = 0; + int src_direction = (src != dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL); + + req_ctx->dma_buf_type = CC_DMA_BUF_DLLI; + mlli_params->curr_pool = NULL; + sg_data.num_of_buffers = 0; + + /* Map IV buffer */ + if (ivsize) { + dump_byte_array("iv", info, ivsize); + req_ctx->gen_ctx.iv_dma_addr = + dma_map_single(dev, info, ivsize, DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, req_ctx->gen_ctx.iv_dma_addr)) { + dev_err(dev, "Mapping iv %u B at va=%p for DMA failed\n", + ivsize, info); + return -ENOMEM; + } + dev_dbg(dev, "Mapped iv %u B at va=%p to dma=%pad\n", + ivsize, info, &req_ctx->gen_ctx.iv_dma_addr); + } else { + req_ctx->gen_ctx.iv_dma_addr = 0; + } + + /* Map the src SGL */ + rc = cc_map_sg(dev, src, nbytes, src_direction, &req_ctx->in_nents, + LLI_MAX_NUM_OF_DATA_ENTRIES, &dummy, &mapped_nents); + if (rc) + goto cipher_exit; + if (mapped_nents > 1) + req_ctx->dma_buf_type = CC_DMA_BUF_MLLI; + + if (src == dst) { + /* Handle inplace operation */ + if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI) { + req_ctx->out_nents = 0; + cc_add_sg_entry(dev, &sg_data, req_ctx->in_nents, src, + nbytes, 0, true, + &req_ctx->in_mlli_nents); + } + } else { + /* Map the dst sg */ + rc = cc_map_sg(dev, dst, nbytes, DMA_FROM_DEVICE, + &req_ctx->out_nents, LLI_MAX_NUM_OF_DATA_ENTRIES, + &dummy, &mapped_nents); + if (rc) + goto cipher_exit; + if (mapped_nents > 1) + req_ctx->dma_buf_type = CC_DMA_BUF_MLLI; + + if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI) { + cc_add_sg_entry(dev, &sg_data, req_ctx->in_nents, src, + nbytes, 0, true, + &req_ctx->in_mlli_nents); + cc_add_sg_entry(dev, &sg_data, req_ctx->out_nents, dst, + nbytes, 0, true, + &req_ctx->out_mlli_nents); + } + } + + if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI) { + mlli_params->curr_pool = drvdata->mlli_buffs_pool; + rc = cc_generate_mlli(dev, &sg_data, mlli_params, flags); + if (rc) + goto cipher_exit; + } + + dev_dbg(dev, "areq_ctx->dma_buf_type = %s\n", + cc_dma_buf_type(req_ctx->dma_buf_type)); + + return 0; + +cipher_exit: + cc_unmap_cipher_request(dev, req_ctx, ivsize, src, dst); + return rc; +} + +void cc_unmap_aead_request(struct device *dev, struct aead_request *req) +{ + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); + unsigned int hw_iv_size = areq_ctx->hw_iv_size; + struct cc_drvdata *drvdata = dev_get_drvdata(dev); + int src_direction = (req->src != req->dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL); + + if (areq_ctx->mac_buf_dma_addr) { + dma_unmap_single(dev, areq_ctx->mac_buf_dma_addr, + MAX_MAC_SIZE, DMA_BIDIRECTIONAL); + } + + if (areq_ctx->cipher_mode == DRV_CIPHER_GCTR) { + if (areq_ctx->hkey_dma_addr) { + dma_unmap_single(dev, areq_ctx->hkey_dma_addr, + AES_BLOCK_SIZE, DMA_BIDIRECTIONAL); + } + + if (areq_ctx->gcm_block_len_dma_addr) { + dma_unmap_single(dev, areq_ctx->gcm_block_len_dma_addr, + AES_BLOCK_SIZE, DMA_TO_DEVICE); + } + + if (areq_ctx->gcm_iv_inc1_dma_addr) { + dma_unmap_single(dev, areq_ctx->gcm_iv_inc1_dma_addr, + AES_BLOCK_SIZE, DMA_TO_DEVICE); + } + + if (areq_ctx->gcm_iv_inc2_dma_addr) { + dma_unmap_single(dev, areq_ctx->gcm_iv_inc2_dma_addr, + AES_BLOCK_SIZE, DMA_TO_DEVICE); + } + } + + if (areq_ctx->ccm_hdr_size != ccm_header_size_null) { + if (areq_ctx->ccm_iv0_dma_addr) { + dma_unmap_single(dev, areq_ctx->ccm_iv0_dma_addr, + AES_BLOCK_SIZE, DMA_TO_DEVICE); + } + + dma_unmap_sg(dev, &areq_ctx->ccm_adata_sg, 1, DMA_TO_DEVICE); + } + if (areq_ctx->gen_ctx.iv_dma_addr) { + dma_unmap_single(dev, areq_ctx->gen_ctx.iv_dma_addr, + hw_iv_size, DMA_BIDIRECTIONAL); + kfree_sensitive(areq_ctx->gen_ctx.iv); + } + + /* Release pool */ + if ((areq_ctx->assoc_buff_type == CC_DMA_BUF_MLLI || + areq_ctx->data_buff_type == CC_DMA_BUF_MLLI) && + (areq_ctx->mlli_params.mlli_virt_addr)) { + dev_dbg(dev, "free MLLI buffer: dma=%pad virt=%p\n", + &areq_ctx->mlli_params.mlli_dma_addr, + areq_ctx->mlli_params.mlli_virt_addr); + dma_pool_free(areq_ctx->mlli_params.curr_pool, + areq_ctx->mlli_params.mlli_virt_addr, + areq_ctx->mlli_params.mlli_dma_addr); + } + + dev_dbg(dev, "Unmapping src sgl: req->src=%p areq_ctx->src.nents=%u areq_ctx->assoc.nents=%u assoclen:%u cryptlen=%u\n", + sg_virt(req->src), areq_ctx->src.nents, areq_ctx->assoc.nents, + areq_ctx->assoclen, req->cryptlen); + + dma_unmap_sg(dev, req->src, areq_ctx->src.mapped_nents, src_direction); + if (req->src != req->dst) { + dev_dbg(dev, "Unmapping dst sgl: req->dst=%p\n", + sg_virt(req->dst)); + dma_unmap_sg(dev, req->dst, areq_ctx->dst.mapped_nents, DMA_FROM_DEVICE); + } + if (drvdata->coherent && + areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT && + req->src == req->dst) { + /* copy back mac from temporary location to deal with possible + * data memory overriding that caused by cache coherence + * problem. + */ + cc_copy_mac(dev, req, CC_SG_FROM_BUF); + } +} + +static bool cc_is_icv_frag(unsigned int sgl_nents, unsigned int authsize, + u32 last_entry_data_size) +{ + return ((sgl_nents > 1) && (last_entry_data_size < authsize)); +} + +static int cc_aead_chain_iv(struct cc_drvdata *drvdata, + struct aead_request *req, + struct buffer_array *sg_data, + bool is_last, bool do_chain) +{ + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); + unsigned int hw_iv_size = areq_ctx->hw_iv_size; + struct device *dev = drvdata_to_dev(drvdata); + gfp_t flags = cc_gfp_flags(&req->base); + int rc = 0; + + if (!req->iv) { + areq_ctx->gen_ctx.iv_dma_addr = 0; + areq_ctx->gen_ctx.iv = NULL; + goto chain_iv_exit; + } + + areq_ctx->gen_ctx.iv = kmemdup(req->iv, hw_iv_size, flags); + if (!areq_ctx->gen_ctx.iv) + return -ENOMEM; + + areq_ctx->gen_ctx.iv_dma_addr = + dma_map_single(dev, areq_ctx->gen_ctx.iv, hw_iv_size, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, areq_ctx->gen_ctx.iv_dma_addr)) { + dev_err(dev, "Mapping iv %u B at va=%p for DMA failed\n", + hw_iv_size, req->iv); + kfree_sensitive(areq_ctx->gen_ctx.iv); + areq_ctx->gen_ctx.iv = NULL; + rc = -ENOMEM; + goto chain_iv_exit; + } + + dev_dbg(dev, "Mapped iv %u B at va=%p to dma=%pad\n", + hw_iv_size, req->iv, &areq_ctx->gen_ctx.iv_dma_addr); + +chain_iv_exit: + return rc; +} + +static int cc_aead_chain_assoc(struct cc_drvdata *drvdata, + struct aead_request *req, + struct buffer_array *sg_data, + bool is_last, bool do_chain) +{ + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); + int rc = 0; + int mapped_nents = 0; + struct device *dev = drvdata_to_dev(drvdata); + + if (!sg_data) { + rc = -EINVAL; + goto chain_assoc_exit; + } + + if (areq_ctx->assoclen == 0) { + areq_ctx->assoc_buff_type = CC_DMA_BUF_NULL; + areq_ctx->assoc.nents = 0; + areq_ctx->assoc.mlli_nents = 0; + dev_dbg(dev, "Chain assoc of length 0: buff_type=%s nents=%u\n", + cc_dma_buf_type(areq_ctx->assoc_buff_type), + areq_ctx->assoc.nents); + goto chain_assoc_exit; + } + + mapped_nents = sg_nents_for_len(req->src, areq_ctx->assoclen); + if (mapped_nents < 0) + return mapped_nents; + + if (mapped_nents > LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES) { + dev_err(dev, "Too many fragments. current %d max %d\n", + mapped_nents, LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES); + return -ENOMEM; + } + areq_ctx->assoc.nents = mapped_nents; + + /* in CCM case we have additional entry for + * ccm header configurations + */ + if (areq_ctx->ccm_hdr_size != ccm_header_size_null) { + if ((mapped_nents + 1) > LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES) { + dev_err(dev, "CCM case.Too many fragments. Current %d max %d\n", + (areq_ctx->assoc.nents + 1), + LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES); + rc = -ENOMEM; + goto chain_assoc_exit; + } + } + + if (mapped_nents == 1 && areq_ctx->ccm_hdr_size == ccm_header_size_null) + areq_ctx->assoc_buff_type = CC_DMA_BUF_DLLI; + else + areq_ctx->assoc_buff_type = CC_DMA_BUF_MLLI; + + if (do_chain || areq_ctx->assoc_buff_type == CC_DMA_BUF_MLLI) { + dev_dbg(dev, "Chain assoc: buff_type=%s nents=%u\n", + cc_dma_buf_type(areq_ctx->assoc_buff_type), + areq_ctx->assoc.nents); + cc_add_sg_entry(dev, sg_data, areq_ctx->assoc.nents, req->src, + areq_ctx->assoclen, 0, is_last, + &areq_ctx->assoc.mlli_nents); + areq_ctx->assoc_buff_type = CC_DMA_BUF_MLLI; + } + +chain_assoc_exit: + return rc; +} + +static void cc_prepare_aead_data_dlli(struct aead_request *req, + u32 *src_last_bytes, u32 *dst_last_bytes) +{ + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); + enum drv_crypto_direction direct = areq_ctx->gen_ctx.op_type; + unsigned int authsize = areq_ctx->req_authsize; + struct scatterlist *sg; + ssize_t offset; + + areq_ctx->is_icv_fragmented = false; + + if ((req->src == req->dst) || direct == DRV_CRYPTO_DIRECTION_DECRYPT) { + sg = areq_ctx->src_sgl; + offset = *src_last_bytes - authsize; + } else { + sg = areq_ctx->dst_sgl; + offset = *dst_last_bytes - authsize; + } + + areq_ctx->icv_dma_addr = sg_dma_address(sg) + offset; + areq_ctx->icv_virt_addr = sg_virt(sg) + offset; +} + +static void cc_prepare_aead_data_mlli(struct cc_drvdata *drvdata, + struct aead_request *req, + struct buffer_array *sg_data, + u32 *src_last_bytes, u32 *dst_last_bytes, + bool is_last_table) +{ + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); + enum drv_crypto_direction direct = areq_ctx->gen_ctx.op_type; + unsigned int authsize = areq_ctx->req_authsize; + struct device *dev = drvdata_to_dev(drvdata); + struct scatterlist *sg; + + if (req->src == req->dst) { + /*INPLACE*/ + cc_add_sg_entry(dev, sg_data, areq_ctx->src.nents, + areq_ctx->src_sgl, areq_ctx->cryptlen, + areq_ctx->src_offset, is_last_table, + &areq_ctx->src.mlli_nents); + + areq_ctx->is_icv_fragmented = + cc_is_icv_frag(areq_ctx->src.nents, authsize, + *src_last_bytes); + + if (areq_ctx->is_icv_fragmented) { + /* Backup happens only when ICV is fragmented, ICV + * verification is made by CPU compare in order to + * simplify MAC verification upon request completion + */ + if (direct == DRV_CRYPTO_DIRECTION_DECRYPT) { + /* In coherent platforms (e.g. ACP) + * already copying ICV for any + * INPLACE-DECRYPT operation, hence + * we must neglect this code. + */ + if (!drvdata->coherent) + cc_copy_mac(dev, req, CC_SG_TO_BUF); + + areq_ctx->icv_virt_addr = areq_ctx->backup_mac; + } else { + areq_ctx->icv_virt_addr = areq_ctx->mac_buf; + areq_ctx->icv_dma_addr = + areq_ctx->mac_buf_dma_addr; + } + } else { /* Contig. ICV */ + sg = &areq_ctx->src_sgl[areq_ctx->src.nents - 1]; + /*Should hanlde if the sg is not contig.*/ + areq_ctx->icv_dma_addr = sg_dma_address(sg) + + (*src_last_bytes - authsize); + areq_ctx->icv_virt_addr = sg_virt(sg) + + (*src_last_bytes - authsize); + } + + } else if (direct == DRV_CRYPTO_DIRECTION_DECRYPT) { + /*NON-INPLACE and DECRYPT*/ + cc_add_sg_entry(dev, sg_data, areq_ctx->src.nents, + areq_ctx->src_sgl, areq_ctx->cryptlen, + areq_ctx->src_offset, is_last_table, + &areq_ctx->src.mlli_nents); + cc_add_sg_entry(dev, sg_data, areq_ctx->dst.nents, + areq_ctx->dst_sgl, areq_ctx->cryptlen, + areq_ctx->dst_offset, is_last_table, + &areq_ctx->dst.mlli_nents); + + areq_ctx->is_icv_fragmented = + cc_is_icv_frag(areq_ctx->src.nents, authsize, + *src_last_bytes); + /* Backup happens only when ICV is fragmented, ICV + + * verification is made by CPU compare in order to simplify + * MAC verification upon request completion + */ + if (areq_ctx->is_icv_fragmented) { + cc_copy_mac(dev, req, CC_SG_TO_BUF); + areq_ctx->icv_virt_addr = areq_ctx->backup_mac; + + } else { /* Contig. ICV */ + sg = &areq_ctx->src_sgl[areq_ctx->src.nents - 1]; + /*Should hanlde if the sg is not contig.*/ + areq_ctx->icv_dma_addr = sg_dma_address(sg) + + (*src_last_bytes - authsize); + areq_ctx->icv_virt_addr = sg_virt(sg) + + (*src_last_bytes - authsize); + } + + } else { + /*NON-INPLACE and ENCRYPT*/ + cc_add_sg_entry(dev, sg_data, areq_ctx->dst.nents, + areq_ctx->dst_sgl, areq_ctx->cryptlen, + areq_ctx->dst_offset, is_last_table, + &areq_ctx->dst.mlli_nents); + cc_add_sg_entry(dev, sg_data, areq_ctx->src.nents, + areq_ctx->src_sgl, areq_ctx->cryptlen, + areq_ctx->src_offset, is_last_table, + &areq_ctx->src.mlli_nents); + + areq_ctx->is_icv_fragmented = + cc_is_icv_frag(areq_ctx->dst.nents, authsize, + *dst_last_bytes); + + if (!areq_ctx->is_icv_fragmented) { + sg = &areq_ctx->dst_sgl[areq_ctx->dst.nents - 1]; + /* Contig. ICV */ + areq_ctx->icv_dma_addr = sg_dma_address(sg) + + (*dst_last_bytes - authsize); + areq_ctx->icv_virt_addr = sg_virt(sg) + + (*dst_last_bytes - authsize); + } else { + areq_ctx->icv_dma_addr = areq_ctx->mac_buf_dma_addr; + areq_ctx->icv_virt_addr = areq_ctx->mac_buf; + } + } +} + +static int cc_aead_chain_data(struct cc_drvdata *drvdata, + struct aead_request *req, + struct buffer_array *sg_data, + bool is_last_table, bool do_chain) +{ + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); + struct device *dev = drvdata_to_dev(drvdata); + enum drv_crypto_direction direct = areq_ctx->gen_ctx.op_type; + unsigned int authsize = areq_ctx->req_authsize; + unsigned int src_last_bytes = 0, dst_last_bytes = 0; + int rc = 0; + u32 src_mapped_nents = 0, dst_mapped_nents = 0; + u32 offset = 0; + /* non-inplace mode */ + unsigned int size_for_map = req->assoclen + req->cryptlen; + u32 sg_index = 0; + u32 size_to_skip = req->assoclen; + struct scatterlist *sgl; + + offset = size_to_skip; + + if (!sg_data) + return -EINVAL; + + areq_ctx->src_sgl = req->src; + areq_ctx->dst_sgl = req->dst; + + size_for_map += (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? + authsize : 0; + src_mapped_nents = cc_get_sgl_nents(dev, req->src, size_for_map, + &src_last_bytes); + sg_index = areq_ctx->src_sgl->length; + //check where the data starts + while (src_mapped_nents && (sg_index <= size_to_skip)) { + src_mapped_nents--; + offset -= areq_ctx->src_sgl->length; + sgl = sg_next(areq_ctx->src_sgl); + if (!sgl) + break; + areq_ctx->src_sgl = sgl; + sg_index += areq_ctx->src_sgl->length; + } + if (src_mapped_nents > LLI_MAX_NUM_OF_DATA_ENTRIES) { + dev_err(dev, "Too many fragments. current %d max %d\n", + src_mapped_nents, LLI_MAX_NUM_OF_DATA_ENTRIES); + return -ENOMEM; + } + + areq_ctx->src.nents = src_mapped_nents; + + areq_ctx->src_offset = offset; + + if (req->src != req->dst) { + size_for_map = req->assoclen + req->cryptlen; + + if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) + size_for_map += authsize; + else + size_for_map -= authsize; + + rc = cc_map_sg(dev, req->dst, size_for_map, DMA_FROM_DEVICE, + &areq_ctx->dst.mapped_nents, + LLI_MAX_NUM_OF_DATA_ENTRIES, &dst_last_bytes, + &dst_mapped_nents); + if (rc) + goto chain_data_exit; + } + + dst_mapped_nents = cc_get_sgl_nents(dev, req->dst, size_for_map, + &dst_last_bytes); + sg_index = areq_ctx->dst_sgl->length; + offset = size_to_skip; + + //check where the data starts + while (dst_mapped_nents && sg_index <= size_to_skip) { + dst_mapped_nents--; + offset -= areq_ctx->dst_sgl->length; + sgl = sg_next(areq_ctx->dst_sgl); + if (!sgl) + break; + areq_ctx->dst_sgl = sgl; + sg_index += areq_ctx->dst_sgl->length; + } + if (dst_mapped_nents > LLI_MAX_NUM_OF_DATA_ENTRIES) { + dev_err(dev, "Too many fragments. current %d max %d\n", + dst_mapped_nents, LLI_MAX_NUM_OF_DATA_ENTRIES); + return -ENOMEM; + } + areq_ctx->dst.nents = dst_mapped_nents; + areq_ctx->dst_offset = offset; + if (src_mapped_nents > 1 || + dst_mapped_nents > 1 || + do_chain) { + areq_ctx->data_buff_type = CC_DMA_BUF_MLLI; + cc_prepare_aead_data_mlli(drvdata, req, sg_data, + &src_last_bytes, &dst_last_bytes, + is_last_table); + } else { + areq_ctx->data_buff_type = CC_DMA_BUF_DLLI; + cc_prepare_aead_data_dlli(req, &src_last_bytes, + &dst_last_bytes); + } + +chain_data_exit: + return rc; +} + +static void cc_update_aead_mlli_nents(struct cc_drvdata *drvdata, + struct aead_request *req) +{ + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); + u32 curr_mlli_size = 0; + + if (areq_ctx->assoc_buff_type == CC_DMA_BUF_MLLI) { + areq_ctx->assoc.sram_addr = drvdata->mlli_sram_addr; + curr_mlli_size = areq_ctx->assoc.mlli_nents * + LLI_ENTRY_BYTE_SIZE; + } + + if (areq_ctx->data_buff_type == CC_DMA_BUF_MLLI) { + /*Inplace case dst nents equal to src nents*/ + if (req->src == req->dst) { + areq_ctx->dst.mlli_nents = areq_ctx->src.mlli_nents; + areq_ctx->src.sram_addr = drvdata->mlli_sram_addr + + curr_mlli_size; + areq_ctx->dst.sram_addr = areq_ctx->src.sram_addr; + if (!areq_ctx->is_single_pass) + areq_ctx->assoc.mlli_nents += + areq_ctx->src.mlli_nents; + } else { + if (areq_ctx->gen_ctx.op_type == + DRV_CRYPTO_DIRECTION_DECRYPT) { + areq_ctx->src.sram_addr = + drvdata->mlli_sram_addr + + curr_mlli_size; + areq_ctx->dst.sram_addr = + areq_ctx->src.sram_addr + + areq_ctx->src.mlli_nents * + LLI_ENTRY_BYTE_SIZE; + if (!areq_ctx->is_single_pass) + areq_ctx->assoc.mlli_nents += + areq_ctx->src.mlli_nents; + } else { + areq_ctx->dst.sram_addr = + drvdata->mlli_sram_addr + + curr_mlli_size; + areq_ctx->src.sram_addr = + areq_ctx->dst.sram_addr + + areq_ctx->dst.mlli_nents * + LLI_ENTRY_BYTE_SIZE; + if (!areq_ctx->is_single_pass) + areq_ctx->assoc.mlli_nents += + areq_ctx->dst.mlli_nents; + } + } + } +} + +int cc_map_aead_request(struct cc_drvdata *drvdata, struct aead_request *req) +{ + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); + struct mlli_params *mlli_params = &areq_ctx->mlli_params; + struct device *dev = drvdata_to_dev(drvdata); + struct buffer_array sg_data; + unsigned int authsize = areq_ctx->req_authsize; + int rc = 0; + dma_addr_t dma_addr; + u32 mapped_nents = 0; + u32 dummy = 0; /*used for the assoc data fragments */ + u32 size_to_map; + gfp_t flags = cc_gfp_flags(&req->base); + + mlli_params->curr_pool = NULL; + sg_data.num_of_buffers = 0; + + /* copy mac to a temporary location to deal with possible + * data memory overriding that caused by cache coherence problem. + */ + if (drvdata->coherent && + areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT && + req->src == req->dst) + cc_copy_mac(dev, req, CC_SG_TO_BUF); + + /* cacluate the size for cipher remove ICV in decrypt*/ + areq_ctx->cryptlen = (areq_ctx->gen_ctx.op_type == + DRV_CRYPTO_DIRECTION_ENCRYPT) ? + req->cryptlen : + (req->cryptlen - authsize); + + dma_addr = dma_map_single(dev, areq_ctx->mac_buf, MAX_MAC_SIZE, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, dma_addr)) { + dev_err(dev, "Mapping mac_buf %u B at va=%p for DMA failed\n", + MAX_MAC_SIZE, areq_ctx->mac_buf); + rc = -ENOMEM; + goto aead_map_failure; + } + areq_ctx->mac_buf_dma_addr = dma_addr; + + if (areq_ctx->ccm_hdr_size != ccm_header_size_null) { + void *addr = areq_ctx->ccm_config + CCM_CTR_COUNT_0_OFFSET; + + dma_addr = dma_map_single(dev, addr, AES_BLOCK_SIZE, + DMA_TO_DEVICE); + + if (dma_mapping_error(dev, dma_addr)) { + dev_err(dev, "Mapping mac_buf %u B at va=%p for DMA failed\n", + AES_BLOCK_SIZE, addr); + areq_ctx->ccm_iv0_dma_addr = 0; + rc = -ENOMEM; + goto aead_map_failure; + } + areq_ctx->ccm_iv0_dma_addr = dma_addr; + + rc = cc_set_aead_conf_buf(dev, areq_ctx, areq_ctx->ccm_config, + &sg_data, areq_ctx->assoclen); + if (rc) + goto aead_map_failure; + } + + if (areq_ctx->cipher_mode == DRV_CIPHER_GCTR) { + dma_addr = dma_map_single(dev, areq_ctx->hkey, AES_BLOCK_SIZE, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, dma_addr)) { + dev_err(dev, "Mapping hkey %u B at va=%p for DMA failed\n", + AES_BLOCK_SIZE, areq_ctx->hkey); + rc = -ENOMEM; + goto aead_map_failure; + } + areq_ctx->hkey_dma_addr = dma_addr; + + dma_addr = dma_map_single(dev, &areq_ctx->gcm_len_block, + AES_BLOCK_SIZE, DMA_TO_DEVICE); + if (dma_mapping_error(dev, dma_addr)) { + dev_err(dev, "Mapping gcm_len_block %u B at va=%p for DMA failed\n", + AES_BLOCK_SIZE, &areq_ctx->gcm_len_block); + rc = -ENOMEM; + goto aead_map_failure; + } + areq_ctx->gcm_block_len_dma_addr = dma_addr; + + dma_addr = dma_map_single(dev, areq_ctx->gcm_iv_inc1, + AES_BLOCK_SIZE, DMA_TO_DEVICE); + + if (dma_mapping_error(dev, dma_addr)) { + dev_err(dev, "Mapping gcm_iv_inc1 %u B at va=%p for DMA failed\n", + AES_BLOCK_SIZE, (areq_ctx->gcm_iv_inc1)); + areq_ctx->gcm_iv_inc1_dma_addr = 0; + rc = -ENOMEM; + goto aead_map_failure; + } + areq_ctx->gcm_iv_inc1_dma_addr = dma_addr; + + dma_addr = dma_map_single(dev, areq_ctx->gcm_iv_inc2, + AES_BLOCK_SIZE, DMA_TO_DEVICE); + + if (dma_mapping_error(dev, dma_addr)) { + dev_err(dev, "Mapping gcm_iv_inc2 %u B at va=%p for DMA failed\n", + AES_BLOCK_SIZE, (areq_ctx->gcm_iv_inc2)); + areq_ctx->gcm_iv_inc2_dma_addr = 0; + rc = -ENOMEM; + goto aead_map_failure; + } + areq_ctx->gcm_iv_inc2_dma_addr = dma_addr; + } + + size_to_map = req->cryptlen + req->assoclen; + /* If we do in-place encryption, we also need the auth tag */ + if ((areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_ENCRYPT) && + (req->src == req->dst)) { + size_to_map += authsize; + } + + rc = cc_map_sg(dev, req->src, size_to_map, + (req->src != req->dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL), + &areq_ctx->src.mapped_nents, + (LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES + + LLI_MAX_NUM_OF_DATA_ENTRIES), + &dummy, &mapped_nents); + if (rc) + goto aead_map_failure; + + if (areq_ctx->is_single_pass) { + /* + * Create MLLI table for: + * (1) Assoc. data + * (2) Src/Dst SGLs + * Note: IV is contg. buffer (not an SGL) + */ + rc = cc_aead_chain_assoc(drvdata, req, &sg_data, true, false); + if (rc) + goto aead_map_failure; + rc = cc_aead_chain_iv(drvdata, req, &sg_data, true, false); + if (rc) + goto aead_map_failure; + rc = cc_aead_chain_data(drvdata, req, &sg_data, true, false); + if (rc) + goto aead_map_failure; + } else { /* DOUBLE-PASS flow */ + /* + * Prepare MLLI table(s) in this order: + * + * If ENCRYPT/DECRYPT (inplace): + * (1) MLLI table for assoc + * (2) IV entry (chained right after end of assoc) + * (3) MLLI for src/dst (inplace operation) + * + * If ENCRYPT (non-inplace) + * (1) MLLI table for assoc + * (2) IV entry (chained right after end of assoc) + * (3) MLLI for dst + * (4) MLLI for src + * + * If DECRYPT (non-inplace) + * (1) MLLI table for assoc + * (2) IV entry (chained right after end of assoc) + * (3) MLLI for src + * (4) MLLI for dst + */ + rc = cc_aead_chain_assoc(drvdata, req, &sg_data, false, true); + if (rc) + goto aead_map_failure; + rc = cc_aead_chain_iv(drvdata, req, &sg_data, false, true); + if (rc) + goto aead_map_failure; + rc = cc_aead_chain_data(drvdata, req, &sg_data, true, true); + if (rc) + goto aead_map_failure; + } + + /* Mlli support -start building the MLLI according to the above + * results + */ + if (areq_ctx->assoc_buff_type == CC_DMA_BUF_MLLI || + areq_ctx->data_buff_type == CC_DMA_BUF_MLLI) { + mlli_params->curr_pool = drvdata->mlli_buffs_pool; + rc = cc_generate_mlli(dev, &sg_data, mlli_params, flags); + if (rc) + goto aead_map_failure; + + cc_update_aead_mlli_nents(drvdata, req); + dev_dbg(dev, "assoc params mn %d\n", + areq_ctx->assoc.mlli_nents); + dev_dbg(dev, "src params mn %d\n", areq_ctx->src.mlli_nents); + dev_dbg(dev, "dst params mn %d\n", areq_ctx->dst.mlli_nents); + } + return 0; + +aead_map_failure: + cc_unmap_aead_request(dev, req); + return rc; +} + +int cc_map_hash_request_final(struct cc_drvdata *drvdata, void *ctx, + struct scatterlist *src, unsigned int nbytes, + bool do_update, gfp_t flags) +{ + struct ahash_req_ctx *areq_ctx = (struct ahash_req_ctx *)ctx; + struct device *dev = drvdata_to_dev(drvdata); + u8 *curr_buff = cc_hash_buf(areq_ctx); + u32 *curr_buff_cnt = cc_hash_buf_cnt(areq_ctx); + struct mlli_params *mlli_params = &areq_ctx->mlli_params; + struct buffer_array sg_data; + int rc = 0; + u32 dummy = 0; + u32 mapped_nents = 0; + + dev_dbg(dev, "final params : curr_buff=%p curr_buff_cnt=0x%X nbytes = 0x%X src=%p curr_index=%u\n", + curr_buff, *curr_buff_cnt, nbytes, src, areq_ctx->buff_index); + /* Init the type of the dma buffer */ + areq_ctx->data_dma_buf_type = CC_DMA_BUF_NULL; + mlli_params->curr_pool = NULL; + sg_data.num_of_buffers = 0; + areq_ctx->in_nents = 0; + + if (nbytes == 0 && *curr_buff_cnt == 0) { + /* nothing to do */ + return 0; + } + + /* map the previous buffer */ + if (*curr_buff_cnt) { + rc = cc_set_hash_buf(dev, areq_ctx, curr_buff, *curr_buff_cnt, + &sg_data); + if (rc) + return rc; + } + + if (src && nbytes > 0 && do_update) { + rc = cc_map_sg(dev, src, nbytes, DMA_TO_DEVICE, + &areq_ctx->in_nents, LLI_MAX_NUM_OF_DATA_ENTRIES, + &dummy, &mapped_nents); + if (rc) + goto unmap_curr_buff; + if (src && mapped_nents == 1 && + areq_ctx->data_dma_buf_type == CC_DMA_BUF_NULL) { + memcpy(areq_ctx->buff_sg, src, + sizeof(struct scatterlist)); + areq_ctx->buff_sg->length = nbytes; + areq_ctx->curr_sg = areq_ctx->buff_sg; + areq_ctx->data_dma_buf_type = CC_DMA_BUF_DLLI; + } else { + areq_ctx->data_dma_buf_type = CC_DMA_BUF_MLLI; + } + } + + /*build mlli */ + if (areq_ctx->data_dma_buf_type == CC_DMA_BUF_MLLI) { + mlli_params->curr_pool = drvdata->mlli_buffs_pool; + /* add the src data to the sg_data */ + cc_add_sg_entry(dev, &sg_data, areq_ctx->in_nents, src, nbytes, + 0, true, &areq_ctx->mlli_nents); + rc = cc_generate_mlli(dev, &sg_data, mlli_params, flags); + if (rc) + goto fail_unmap_din; + } + /* change the buffer index for the unmap function */ + areq_ctx->buff_index = (areq_ctx->buff_index ^ 1); + dev_dbg(dev, "areq_ctx->data_dma_buf_type = %s\n", + cc_dma_buf_type(areq_ctx->data_dma_buf_type)); + return 0; + +fail_unmap_din: + dma_unmap_sg(dev, src, areq_ctx->in_nents, DMA_TO_DEVICE); + +unmap_curr_buff: + if (*curr_buff_cnt) + dma_unmap_sg(dev, areq_ctx->buff_sg, 1, DMA_TO_DEVICE); + + return rc; +} + +int cc_map_hash_request_update(struct cc_drvdata *drvdata, void *ctx, + struct scatterlist *src, unsigned int nbytes, + unsigned int block_size, gfp_t flags) +{ + struct ahash_req_ctx *areq_ctx = (struct ahash_req_ctx *)ctx; + struct device *dev = drvdata_to_dev(drvdata); + u8 *curr_buff = cc_hash_buf(areq_ctx); + u32 *curr_buff_cnt = cc_hash_buf_cnt(areq_ctx); + u8 *next_buff = cc_next_buf(areq_ctx); + u32 *next_buff_cnt = cc_next_buf_cnt(areq_ctx); + struct mlli_params *mlli_params = &areq_ctx->mlli_params; + unsigned int update_data_len; + u32 total_in_len = nbytes + *curr_buff_cnt; + struct buffer_array sg_data; + unsigned int swap_index = 0; + int rc = 0; + u32 dummy = 0; + u32 mapped_nents = 0; + int sg_nents; + + dev_dbg(dev, " update params : curr_buff=%p curr_buff_cnt=0x%X nbytes=0x%X src=%p curr_index=%u\n", + curr_buff, *curr_buff_cnt, nbytes, src, areq_ctx->buff_index); + /* Init the type of the dma buffer */ + areq_ctx->data_dma_buf_type = CC_DMA_BUF_NULL; + mlli_params->curr_pool = NULL; + areq_ctx->curr_sg = NULL; + sg_data.num_of_buffers = 0; + areq_ctx->in_nents = 0; + + if (total_in_len < block_size) { + dev_dbg(dev, " less than one block: curr_buff=%p *curr_buff_cnt=0x%X copy_to=%p\n", + curr_buff, *curr_buff_cnt, &curr_buff[*curr_buff_cnt]); + sg_nents = sg_nents_for_len(src, nbytes); + if (sg_nents < 0) + return sg_nents; + areq_ctx->in_nents = sg_nents; + sg_copy_to_buffer(src, areq_ctx->in_nents, + &curr_buff[*curr_buff_cnt], nbytes); + *curr_buff_cnt += nbytes; + return 1; + } + + /* Calculate the residue size*/ + *next_buff_cnt = total_in_len & (block_size - 1); + /* update data len */ + update_data_len = total_in_len - *next_buff_cnt; + + dev_dbg(dev, " temp length : *next_buff_cnt=0x%X update_data_len=0x%X\n", + *next_buff_cnt, update_data_len); + + /* Copy the new residue to next buffer */ + if (*next_buff_cnt) { + dev_dbg(dev, " handle residue: next buff %p skip data %u residue %u\n", + next_buff, (update_data_len - *curr_buff_cnt), + *next_buff_cnt); + cc_copy_sg_portion(dev, next_buff, src, + (update_data_len - *curr_buff_cnt), + nbytes, CC_SG_TO_BUF); + /* change the buffer index for next operation */ + swap_index = 1; + } + + if (*curr_buff_cnt) { + rc = cc_set_hash_buf(dev, areq_ctx, curr_buff, *curr_buff_cnt, + &sg_data); + if (rc) + return rc; + /* change the buffer index for next operation */ + swap_index = 1; + } + + if (update_data_len > *curr_buff_cnt) { + rc = cc_map_sg(dev, src, (update_data_len - *curr_buff_cnt), + DMA_TO_DEVICE, &areq_ctx->in_nents, + LLI_MAX_NUM_OF_DATA_ENTRIES, &dummy, + &mapped_nents); + if (rc) + goto unmap_curr_buff; + if (mapped_nents == 1 && + areq_ctx->data_dma_buf_type == CC_DMA_BUF_NULL) { + /* only one entry in the SG and no previous data */ + memcpy(areq_ctx->buff_sg, src, + sizeof(struct scatterlist)); + areq_ctx->buff_sg->length = update_data_len; + areq_ctx->data_dma_buf_type = CC_DMA_BUF_DLLI; + areq_ctx->curr_sg = areq_ctx->buff_sg; + } else { + areq_ctx->data_dma_buf_type = CC_DMA_BUF_MLLI; + } + } + + if (areq_ctx->data_dma_buf_type == CC_DMA_BUF_MLLI) { + mlli_params->curr_pool = drvdata->mlli_buffs_pool; + /* add the src data to the sg_data */ + cc_add_sg_entry(dev, &sg_data, areq_ctx->in_nents, src, + (update_data_len - *curr_buff_cnt), 0, true, + &areq_ctx->mlli_nents); + rc = cc_generate_mlli(dev, &sg_data, mlli_params, flags); + if (rc) + goto fail_unmap_din; + } + areq_ctx->buff_index = (areq_ctx->buff_index ^ swap_index); + + return 0; + +fail_unmap_din: + dma_unmap_sg(dev, src, areq_ctx->in_nents, DMA_TO_DEVICE); + +unmap_curr_buff: + if (*curr_buff_cnt) + dma_unmap_sg(dev, areq_ctx->buff_sg, 1, DMA_TO_DEVICE); + + return rc; +} + +void cc_unmap_hash_request(struct device *dev, void *ctx, + struct scatterlist *src, bool do_revert) +{ + struct ahash_req_ctx *areq_ctx = (struct ahash_req_ctx *)ctx; + u32 *prev_len = cc_next_buf_cnt(areq_ctx); + + /*In case a pool was set, a table was + *allocated and should be released + */ + if (areq_ctx->mlli_params.curr_pool) { + dev_dbg(dev, "free MLLI buffer: dma=%pad virt=%p\n", + &areq_ctx->mlli_params.mlli_dma_addr, + areq_ctx->mlli_params.mlli_virt_addr); + dma_pool_free(areq_ctx->mlli_params.curr_pool, + areq_ctx->mlli_params.mlli_virt_addr, + areq_ctx->mlli_params.mlli_dma_addr); + } + + if (src && areq_ctx->in_nents) { + dev_dbg(dev, "Unmapped sg src: virt=%p dma=%pad len=0x%X\n", + sg_virt(src), &sg_dma_address(src), sg_dma_len(src)); + dma_unmap_sg(dev, src, + areq_ctx->in_nents, DMA_TO_DEVICE); + } + + if (*prev_len) { + dev_dbg(dev, "Unmapped buffer: areq_ctx->buff_sg=%p dma=%pad len 0x%X\n", + sg_virt(areq_ctx->buff_sg), + &sg_dma_address(areq_ctx->buff_sg), + sg_dma_len(areq_ctx->buff_sg)); + dma_unmap_sg(dev, areq_ctx->buff_sg, 1, DMA_TO_DEVICE); + if (!do_revert) { + /* clean the previous data length for update + * operation + */ + *prev_len = 0; + } else { + areq_ctx->buff_index ^= 1; + } + } +} + +int cc_buffer_mgr_init(struct cc_drvdata *drvdata) +{ + struct device *dev = drvdata_to_dev(drvdata); + + drvdata->mlli_buffs_pool = + dma_pool_create("dx_single_mlli_tables", dev, + MAX_NUM_OF_TOTAL_MLLI_ENTRIES * + LLI_ENTRY_BYTE_SIZE, + MLLI_TABLE_MIN_ALIGNMENT, 0); + + if (!drvdata->mlli_buffs_pool) + return -ENOMEM; + + return 0; +} + +int cc_buffer_mgr_fini(struct cc_drvdata *drvdata) +{ + dma_pool_destroy(drvdata->mlli_buffs_pool); + return 0; +} diff --git a/drivers/crypto/ccree/cc_buffer_mgr.h b/drivers/crypto/ccree/cc_buffer_mgr.h new file mode 100644 index 000000000..653441b65 --- /dev/null +++ b/drivers/crypto/ccree/cc_buffer_mgr.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ + +/* \file cc_buffer_mgr.h + * Buffer Manager + */ + +#ifndef __CC_BUFFER_MGR_H__ +#define __CC_BUFFER_MGR_H__ + +#include + +#include "cc_driver.h" + +enum cc_req_dma_buf_type { + CC_DMA_BUF_NULL = 0, + CC_DMA_BUF_DLLI, + CC_DMA_BUF_MLLI +}; + +enum cc_sg_cpy_direct { + CC_SG_TO_BUF = 0, + CC_SG_FROM_BUF = 1 +}; + +struct cc_mlli { + u32 sram_addr; + unsigned int mapped_nents; + unsigned int nents; //sg nents + unsigned int mlli_nents; //mlli nents might be different than the above +}; + +struct mlli_params { + struct dma_pool *curr_pool; + void *mlli_virt_addr; + dma_addr_t mlli_dma_addr; + u32 mlli_len; +}; + +int cc_buffer_mgr_init(struct cc_drvdata *drvdata); + +int cc_buffer_mgr_fini(struct cc_drvdata *drvdata); + +int cc_map_cipher_request(struct cc_drvdata *drvdata, void *ctx, + unsigned int ivsize, unsigned int nbytes, + void *info, struct scatterlist *src, + struct scatterlist *dst, gfp_t flags); + +void cc_unmap_cipher_request(struct device *dev, void *ctx, unsigned int ivsize, + struct scatterlist *src, struct scatterlist *dst); + +int cc_map_aead_request(struct cc_drvdata *drvdata, struct aead_request *req); + +void cc_unmap_aead_request(struct device *dev, struct aead_request *req); + +int cc_map_hash_request_final(struct cc_drvdata *drvdata, void *ctx, + struct scatterlist *src, unsigned int nbytes, + bool do_update, gfp_t flags); + +int cc_map_hash_request_update(struct cc_drvdata *drvdata, void *ctx, + struct scatterlist *src, unsigned int nbytes, + unsigned int block_size, gfp_t flags); + +void cc_unmap_hash_request(struct device *dev, void *ctx, + struct scatterlist *src, bool do_revert); + +void cc_copy_sg_portion(struct device *dev, u8 *dest, struct scatterlist *sg, + u32 to_skip, u32 end, enum cc_sg_cpy_direct direct); + +#endif /*__BUFFER_MGR_H__*/ diff --git a/drivers/crypto/ccree/cc_cipher.c b/drivers/crypto/ccree/cc_cipher.c new file mode 100644 index 000000000..5339b3796 --- /dev/null +++ b/drivers/crypto/ccree/cc_cipher.c @@ -0,0 +1,1471 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cc_driver.h" +#include "cc_lli_defs.h" +#include "cc_buffer_mgr.h" +#include "cc_cipher.h" +#include "cc_request_mgr.h" + +#define MAX_SKCIPHER_SEQ_LEN 6 + +#define template_skcipher template_u.skcipher + +struct cc_user_key_info { + u8 *key; + dma_addr_t key_dma_addr; +}; + +struct cc_hw_key_info { + enum cc_hw_crypto_key key1_slot; + enum cc_hw_crypto_key key2_slot; +}; + +struct cc_cpp_key_info { + u8 slot; + enum cc_cpp_alg alg; +}; + +enum cc_key_type { + CC_UNPROTECTED_KEY, /* User key */ + CC_HW_PROTECTED_KEY, /* HW (FDE) key */ + CC_POLICY_PROTECTED_KEY, /* CPP key */ + CC_INVALID_PROTECTED_KEY /* Invalid key */ +}; + +struct cc_cipher_ctx { + struct cc_drvdata *drvdata; + int keylen; + int cipher_mode; + int flow_mode; + unsigned int flags; + enum cc_key_type key_type; + struct cc_user_key_info user; + union { + struct cc_hw_key_info hw; + struct cc_cpp_key_info cpp; + }; + struct crypto_shash *shash_tfm; + struct crypto_skcipher *fallback_tfm; + bool fallback_on; +}; + +static void cc_cipher_complete(struct device *dev, void *cc_req, int err); + +static inline enum cc_key_type cc_key_type(struct crypto_tfm *tfm) +{ + struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); + + return ctx_p->key_type; +} + +static int validate_keys_sizes(struct cc_cipher_ctx *ctx_p, u32 size) +{ + switch (ctx_p->flow_mode) { + case S_DIN_to_AES: + switch (size) { + case CC_AES_128_BIT_KEY_SIZE: + case CC_AES_192_BIT_KEY_SIZE: + if (ctx_p->cipher_mode != DRV_CIPHER_XTS) + return 0; + break; + case CC_AES_256_BIT_KEY_SIZE: + return 0; + case (CC_AES_192_BIT_KEY_SIZE * 2): + case (CC_AES_256_BIT_KEY_SIZE * 2): + if (ctx_p->cipher_mode == DRV_CIPHER_XTS || + ctx_p->cipher_mode == DRV_CIPHER_ESSIV) + return 0; + break; + default: + break; + } + break; + case S_DIN_to_DES: + if (size == DES3_EDE_KEY_SIZE || size == DES_KEY_SIZE) + return 0; + break; + case S_DIN_to_SM4: + if (size == SM4_KEY_SIZE) + return 0; + break; + default: + break; + } + return -EINVAL; +} + +static int validate_data_size(struct cc_cipher_ctx *ctx_p, + unsigned int size) +{ + switch (ctx_p->flow_mode) { + case S_DIN_to_AES: + switch (ctx_p->cipher_mode) { + case DRV_CIPHER_XTS: + case DRV_CIPHER_CBC_CTS: + if (size >= AES_BLOCK_SIZE) + return 0; + break; + case DRV_CIPHER_OFB: + case DRV_CIPHER_CTR: + return 0; + case DRV_CIPHER_ECB: + case DRV_CIPHER_CBC: + case DRV_CIPHER_ESSIV: + if (IS_ALIGNED(size, AES_BLOCK_SIZE)) + return 0; + break; + default: + break; + } + break; + case S_DIN_to_DES: + if (IS_ALIGNED(size, DES_BLOCK_SIZE)) + return 0; + break; + case S_DIN_to_SM4: + switch (ctx_p->cipher_mode) { + case DRV_CIPHER_CTR: + return 0; + case DRV_CIPHER_ECB: + case DRV_CIPHER_CBC: + if (IS_ALIGNED(size, SM4_BLOCK_SIZE)) + return 0; + break; + default: + break; + } + break; + default: + break; + } + return -EINVAL; +} + +static int cc_cipher_init(struct crypto_tfm *tfm) +{ + struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); + struct cc_crypto_alg *cc_alg = + container_of(tfm->__crt_alg, struct cc_crypto_alg, + skcipher_alg.base); + struct device *dev = drvdata_to_dev(cc_alg->drvdata); + unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize; + unsigned int fallback_req_size = 0; + + dev_dbg(dev, "Initializing context @%p for %s\n", ctx_p, + crypto_tfm_alg_name(tfm)); + + ctx_p->cipher_mode = cc_alg->cipher_mode; + ctx_p->flow_mode = cc_alg->flow_mode; + ctx_p->drvdata = cc_alg->drvdata; + + if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { + const char *name = crypto_tfm_alg_name(tfm); + + /* Alloc hash tfm for essiv */ + ctx_p->shash_tfm = crypto_alloc_shash("sha256", 0, 0); + if (IS_ERR(ctx_p->shash_tfm)) { + dev_err(dev, "Error allocating hash tfm for ESSIV.\n"); + return PTR_ERR(ctx_p->shash_tfm); + } + max_key_buf_size <<= 1; + + /* Alloc fallback tfm or essiv when key size != 256 bit */ + ctx_p->fallback_tfm = + crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC); + + if (IS_ERR(ctx_p->fallback_tfm)) { + /* Note we're still allowing registration with no fallback since it's + * better to have most modes supported than none at all. + */ + dev_warn(dev, "Error allocating fallback algo %s. Some modes may be available.\n", + name); + ctx_p->fallback_tfm = NULL; + } else { + fallback_req_size = crypto_skcipher_reqsize(ctx_p->fallback_tfm); + } + } + + crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm), + sizeof(struct cipher_req_ctx) + fallback_req_size); + + /* Allocate key buffer, cache line aligned */ + ctx_p->user.key = kzalloc(max_key_buf_size, GFP_KERNEL); + if (!ctx_p->user.key) + goto free_fallback; + + dev_dbg(dev, "Allocated key buffer in context. key=@%p\n", + ctx_p->user.key); + + /* Map key buffer */ + ctx_p->user.key_dma_addr = dma_map_single(dev, ctx_p->user.key, + max_key_buf_size, + DMA_TO_DEVICE); + if (dma_mapping_error(dev, ctx_p->user.key_dma_addr)) { + dev_err(dev, "Mapping Key %u B at va=%p for DMA failed\n", + max_key_buf_size, ctx_p->user.key); + goto free_key; + } + dev_dbg(dev, "Mapped key %u B at va=%p to dma=%pad\n", + max_key_buf_size, ctx_p->user.key, &ctx_p->user.key_dma_addr); + + return 0; + +free_key: + kfree(ctx_p->user.key); +free_fallback: + crypto_free_skcipher(ctx_p->fallback_tfm); + crypto_free_shash(ctx_p->shash_tfm); + + return -ENOMEM; +} + +static void cc_cipher_exit(struct crypto_tfm *tfm) +{ + struct crypto_alg *alg = tfm->__crt_alg; + struct cc_crypto_alg *cc_alg = + container_of(alg, struct cc_crypto_alg, + skcipher_alg.base); + unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize; + struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); + struct device *dev = drvdata_to_dev(ctx_p->drvdata); + + dev_dbg(dev, "Clearing context @%p for %s\n", + crypto_tfm_ctx(tfm), crypto_tfm_alg_name(tfm)); + + if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { + /* Free hash tfm for essiv */ + crypto_free_shash(ctx_p->shash_tfm); + ctx_p->shash_tfm = NULL; + crypto_free_skcipher(ctx_p->fallback_tfm); + ctx_p->fallback_tfm = NULL; + } + + /* Unmap key buffer */ + dma_unmap_single(dev, ctx_p->user.key_dma_addr, max_key_buf_size, + DMA_TO_DEVICE); + dev_dbg(dev, "Unmapped key buffer key_dma_addr=%pad\n", + &ctx_p->user.key_dma_addr); + + /* Free key buffer in context */ + dev_dbg(dev, "Free key buffer in context. key=@%p\n", ctx_p->user.key); + kfree_sensitive(ctx_p->user.key); +} + +static enum cc_hw_crypto_key cc_slot_to_hw_key(u8 slot_num) +{ + switch (slot_num) { + case 0: + return KFDE0_KEY; + case 1: + return KFDE1_KEY; + case 2: + return KFDE2_KEY; + case 3: + return KFDE3_KEY; + } + return END_OF_KEYS; +} + +static u8 cc_slot_to_cpp_key(u8 slot_num) +{ + return (slot_num - CC_FIRST_CPP_KEY_SLOT); +} + +static inline enum cc_key_type cc_slot_to_key_type(u8 slot_num) +{ + if (slot_num >= CC_FIRST_HW_KEY_SLOT && slot_num <= CC_LAST_HW_KEY_SLOT) + return CC_HW_PROTECTED_KEY; + else if (slot_num >= CC_FIRST_CPP_KEY_SLOT && + slot_num <= CC_LAST_CPP_KEY_SLOT) + return CC_POLICY_PROTECTED_KEY; + else + return CC_INVALID_PROTECTED_KEY; +} + +static int cc_cipher_sethkey(struct crypto_skcipher *sktfm, const u8 *key, + unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(sktfm); + struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); + struct device *dev = drvdata_to_dev(ctx_p->drvdata); + struct cc_hkey_info hki; + + dev_dbg(dev, "Setting HW key in context @%p for %s. keylen=%u\n", + ctx_p, crypto_tfm_alg_name(tfm), keylen); + dump_byte_array("key", key, keylen); + + /* STAT_PHASE_0: Init and sanity checks */ + + /* This check the size of the protected key token */ + if (keylen != sizeof(hki)) { + dev_err(dev, "Unsupported protected key size %d.\n", keylen); + return -EINVAL; + } + + memcpy(&hki, key, keylen); + + /* The real key len for crypto op is the size of the HW key + * referenced by the HW key slot, not the hardware key token + */ + keylen = hki.keylen; + + if (validate_keys_sizes(ctx_p, keylen)) { + dev_dbg(dev, "Unsupported key size %d.\n", keylen); + return -EINVAL; + } + + ctx_p->keylen = keylen; + ctx_p->fallback_on = false; + + switch (cc_slot_to_key_type(hki.hw_key1)) { + case CC_HW_PROTECTED_KEY: + if (ctx_p->flow_mode == S_DIN_to_SM4) { + dev_err(dev, "Only AES HW protected keys are supported\n"); + return -EINVAL; + } + + ctx_p->hw.key1_slot = cc_slot_to_hw_key(hki.hw_key1); + if (ctx_p->hw.key1_slot == END_OF_KEYS) { + dev_err(dev, "Unsupported hw key1 number (%d)\n", + hki.hw_key1); + return -EINVAL; + } + + if (ctx_p->cipher_mode == DRV_CIPHER_XTS || + ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { + if (hki.hw_key1 == hki.hw_key2) { + dev_err(dev, "Illegal hw key numbers (%d,%d)\n", + hki.hw_key1, hki.hw_key2); + return -EINVAL; + } + + ctx_p->hw.key2_slot = cc_slot_to_hw_key(hki.hw_key2); + if (ctx_p->hw.key2_slot == END_OF_KEYS) { + dev_err(dev, "Unsupported hw key2 number (%d)\n", + hki.hw_key2); + return -EINVAL; + } + } + + ctx_p->key_type = CC_HW_PROTECTED_KEY; + dev_dbg(dev, "HW protected key %d/%d set\n.", + ctx_p->hw.key1_slot, ctx_p->hw.key2_slot); + break; + + case CC_POLICY_PROTECTED_KEY: + if (ctx_p->drvdata->hw_rev < CC_HW_REV_713) { + dev_err(dev, "CPP keys not supported in this hardware revision.\n"); + return -EINVAL; + } + + if (ctx_p->cipher_mode != DRV_CIPHER_CBC && + ctx_p->cipher_mode != DRV_CIPHER_CTR) { + dev_err(dev, "CPP keys only supported in CBC or CTR modes.\n"); + return -EINVAL; + } + + ctx_p->cpp.slot = cc_slot_to_cpp_key(hki.hw_key1); + if (ctx_p->flow_mode == S_DIN_to_AES) + ctx_p->cpp.alg = CC_CPP_AES; + else /* Must be SM4 since due to sethkey registration */ + ctx_p->cpp.alg = CC_CPP_SM4; + ctx_p->key_type = CC_POLICY_PROTECTED_KEY; + dev_dbg(dev, "policy protected key alg: %d slot: %d.\n", + ctx_p->cpp.alg, ctx_p->cpp.slot); + break; + + default: + dev_err(dev, "Unsupported protected key (%d)\n", hki.hw_key1); + return -EINVAL; + } + + return 0; +} + +static int cc_cipher_setkey(struct crypto_skcipher *sktfm, const u8 *key, + unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(sktfm); + struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); + struct device *dev = drvdata_to_dev(ctx_p->drvdata); + struct cc_crypto_alg *cc_alg = + container_of(tfm->__crt_alg, struct cc_crypto_alg, + skcipher_alg.base); + unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize; + + dev_dbg(dev, "Setting key in context @%p for %s. keylen=%u\n", + ctx_p, crypto_tfm_alg_name(tfm), keylen); + dump_byte_array("key", key, keylen); + + /* STAT_PHASE_0: Init and sanity checks */ + + if (validate_keys_sizes(ctx_p, keylen)) { + dev_dbg(dev, "Invalid key size %d.\n", keylen); + return -EINVAL; + } + + if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { + + /* We only support 256 bit ESSIV-CBC-AES keys */ + if (keylen != AES_KEYSIZE_256) { + unsigned int flags = crypto_tfm_get_flags(tfm) & CRYPTO_TFM_REQ_MASK; + + if (likely(ctx_p->fallback_tfm)) { + ctx_p->fallback_on = true; + crypto_skcipher_clear_flags(ctx_p->fallback_tfm, + CRYPTO_TFM_REQ_MASK); + crypto_skcipher_clear_flags(ctx_p->fallback_tfm, flags); + return crypto_skcipher_setkey(ctx_p->fallback_tfm, key, keylen); + } + + dev_dbg(dev, "Unsupported key size %d and no fallback.\n", keylen); + return -EINVAL; + } + + /* Internal ESSIV key buffer is double sized */ + max_key_buf_size <<= 1; + } + + ctx_p->fallback_on = false; + ctx_p->key_type = CC_UNPROTECTED_KEY; + + /* + * Verify DES weak keys + * Note that we're dropping the expanded key since the + * HW does the expansion on its own. + */ + if (ctx_p->flow_mode == S_DIN_to_DES) { + if ((keylen == DES3_EDE_KEY_SIZE && + verify_skcipher_des3_key(sktfm, key)) || + verify_skcipher_des_key(sktfm, key)) { + dev_dbg(dev, "weak DES key"); + return -EINVAL; + } + } + + if (ctx_p->cipher_mode == DRV_CIPHER_XTS && + xts_verify_key(sktfm, key, keylen)) { + dev_dbg(dev, "weak XTS key"); + return -EINVAL; + } + + /* STAT_PHASE_1: Copy key to ctx */ + dma_sync_single_for_cpu(dev, ctx_p->user.key_dma_addr, + max_key_buf_size, DMA_TO_DEVICE); + + memcpy(ctx_p->user.key, key, keylen); + + if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { + /* sha256 for key2 - use sw implementation */ + int err; + + err = crypto_shash_tfm_digest(ctx_p->shash_tfm, + ctx_p->user.key, keylen, + ctx_p->user.key + keylen); + if (err) { + dev_err(dev, "Failed to hash ESSIV key.\n"); + return err; + } + + keylen <<= 1; + } + dma_sync_single_for_device(dev, ctx_p->user.key_dma_addr, + max_key_buf_size, DMA_TO_DEVICE); + ctx_p->keylen = keylen; + + dev_dbg(dev, "return safely"); + return 0; +} + +static int cc_out_setup_mode(struct cc_cipher_ctx *ctx_p) +{ + switch (ctx_p->flow_mode) { + case S_DIN_to_AES: + return S_AES_to_DOUT; + case S_DIN_to_DES: + return S_DES_to_DOUT; + case S_DIN_to_SM4: + return S_SM4_to_DOUT; + default: + return ctx_p->flow_mode; + } +} + +static void cc_setup_readiv_desc(struct crypto_tfm *tfm, + struct cipher_req_ctx *req_ctx, + unsigned int ivsize, struct cc_hw_desc desc[], + unsigned int *seq_size) +{ + struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); + struct device *dev = drvdata_to_dev(ctx_p->drvdata); + int cipher_mode = ctx_p->cipher_mode; + int flow_mode = cc_out_setup_mode(ctx_p); + int direction = req_ctx->gen_ctx.op_type; + dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr; + + if (ctx_p->key_type == CC_POLICY_PROTECTED_KEY) + return; + + switch (cipher_mode) { + case DRV_CIPHER_ECB: + break; + case DRV_CIPHER_CBC: + case DRV_CIPHER_CBC_CTS: + case DRV_CIPHER_CTR: + case DRV_CIPHER_OFB: + /* Read next IV */ + hw_desc_init(&desc[*seq_size]); + set_dout_dlli(&desc[*seq_size], iv_dma_addr, ivsize, NS_BIT, 1); + set_cipher_config0(&desc[*seq_size], direction); + set_flow_mode(&desc[*seq_size], flow_mode); + set_cipher_mode(&desc[*seq_size], cipher_mode); + if (cipher_mode == DRV_CIPHER_CTR || + cipher_mode == DRV_CIPHER_OFB) { + set_setup_mode(&desc[*seq_size], SETUP_WRITE_STATE1); + } else { + set_setup_mode(&desc[*seq_size], SETUP_WRITE_STATE0); + } + set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]); + (*seq_size)++; + break; + case DRV_CIPHER_XTS: + case DRV_CIPHER_ESSIV: + /* IV */ + hw_desc_init(&desc[*seq_size]); + set_setup_mode(&desc[*seq_size], SETUP_WRITE_STATE1); + set_cipher_mode(&desc[*seq_size], cipher_mode); + set_cipher_config0(&desc[*seq_size], direction); + set_flow_mode(&desc[*seq_size], flow_mode); + set_dout_dlli(&desc[*seq_size], iv_dma_addr, CC_AES_BLOCK_SIZE, + NS_BIT, 1); + set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]); + (*seq_size)++; + break; + default: + dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode); + } +} + + +static void cc_setup_state_desc(struct crypto_tfm *tfm, + struct cipher_req_ctx *req_ctx, + unsigned int ivsize, unsigned int nbytes, + struct cc_hw_desc desc[], + unsigned int *seq_size) +{ + struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); + struct device *dev = drvdata_to_dev(ctx_p->drvdata); + int cipher_mode = ctx_p->cipher_mode; + int flow_mode = ctx_p->flow_mode; + int direction = req_ctx->gen_ctx.op_type; + dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr; + + switch (cipher_mode) { + case DRV_CIPHER_ECB: + break; + case DRV_CIPHER_CBC: + case DRV_CIPHER_CBC_CTS: + case DRV_CIPHER_CTR: + case DRV_CIPHER_OFB: + /* Load IV */ + hw_desc_init(&desc[*seq_size]); + set_din_type(&desc[*seq_size], DMA_DLLI, iv_dma_addr, ivsize, + NS_BIT); + set_cipher_config0(&desc[*seq_size], direction); + set_flow_mode(&desc[*seq_size], flow_mode); + set_cipher_mode(&desc[*seq_size], cipher_mode); + if (cipher_mode == DRV_CIPHER_CTR || + cipher_mode == DRV_CIPHER_OFB) { + set_setup_mode(&desc[*seq_size], SETUP_LOAD_STATE1); + } else { + set_setup_mode(&desc[*seq_size], SETUP_LOAD_STATE0); + } + (*seq_size)++; + break; + case DRV_CIPHER_XTS: + case DRV_CIPHER_ESSIV: + break; + default: + dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode); + } +} + + +static void cc_setup_xex_state_desc(struct crypto_tfm *tfm, + struct cipher_req_ctx *req_ctx, + unsigned int ivsize, unsigned int nbytes, + struct cc_hw_desc desc[], + unsigned int *seq_size) +{ + struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); + struct device *dev = drvdata_to_dev(ctx_p->drvdata); + int cipher_mode = ctx_p->cipher_mode; + int flow_mode = ctx_p->flow_mode; + int direction = req_ctx->gen_ctx.op_type; + dma_addr_t key_dma_addr = ctx_p->user.key_dma_addr; + unsigned int key_len = (ctx_p->keylen / 2); + dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr; + unsigned int key_offset = key_len; + + switch (cipher_mode) { + case DRV_CIPHER_ECB: + break; + case DRV_CIPHER_CBC: + case DRV_CIPHER_CBC_CTS: + case DRV_CIPHER_CTR: + case DRV_CIPHER_OFB: + break; + case DRV_CIPHER_XTS: + case DRV_CIPHER_ESSIV: + + if (cipher_mode == DRV_CIPHER_ESSIV) + key_len = SHA256_DIGEST_SIZE; + + /* load XEX key */ + hw_desc_init(&desc[*seq_size]); + set_cipher_mode(&desc[*seq_size], cipher_mode); + set_cipher_config0(&desc[*seq_size], direction); + if (cc_key_type(tfm) == CC_HW_PROTECTED_KEY) { + set_hw_crypto_key(&desc[*seq_size], + ctx_p->hw.key2_slot); + } else { + set_din_type(&desc[*seq_size], DMA_DLLI, + (key_dma_addr + key_offset), + key_len, NS_BIT); + } + set_xex_data_unit_size(&desc[*seq_size], nbytes); + set_flow_mode(&desc[*seq_size], S_DIN_to_AES2); + set_key_size_aes(&desc[*seq_size], key_len); + set_setup_mode(&desc[*seq_size], SETUP_LOAD_XEX_KEY); + (*seq_size)++; + + /* Load IV */ + hw_desc_init(&desc[*seq_size]); + set_setup_mode(&desc[*seq_size], SETUP_LOAD_STATE1); + set_cipher_mode(&desc[*seq_size], cipher_mode); + set_cipher_config0(&desc[*seq_size], direction); + set_key_size_aes(&desc[*seq_size], key_len); + set_flow_mode(&desc[*seq_size], flow_mode); + set_din_type(&desc[*seq_size], DMA_DLLI, iv_dma_addr, + CC_AES_BLOCK_SIZE, NS_BIT); + (*seq_size)++; + break; + default: + dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode); + } +} + +static int cc_out_flow_mode(struct cc_cipher_ctx *ctx_p) +{ + switch (ctx_p->flow_mode) { + case S_DIN_to_AES: + return DIN_AES_DOUT; + case S_DIN_to_DES: + return DIN_DES_DOUT; + case S_DIN_to_SM4: + return DIN_SM4_DOUT; + default: + return ctx_p->flow_mode; + } +} + +static void cc_setup_key_desc(struct crypto_tfm *tfm, + struct cipher_req_ctx *req_ctx, + unsigned int nbytes, struct cc_hw_desc desc[], + unsigned int *seq_size) +{ + struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); + struct device *dev = drvdata_to_dev(ctx_p->drvdata); + int cipher_mode = ctx_p->cipher_mode; + int flow_mode = ctx_p->flow_mode; + int direction = req_ctx->gen_ctx.op_type; + dma_addr_t key_dma_addr = ctx_p->user.key_dma_addr; + unsigned int key_len = ctx_p->keylen; + unsigned int din_size; + + switch (cipher_mode) { + case DRV_CIPHER_CBC: + case DRV_CIPHER_CBC_CTS: + case DRV_CIPHER_CTR: + case DRV_CIPHER_OFB: + case DRV_CIPHER_ECB: + /* Load key */ + hw_desc_init(&desc[*seq_size]); + set_cipher_mode(&desc[*seq_size], cipher_mode); + set_cipher_config0(&desc[*seq_size], direction); + + if (cc_key_type(tfm) == CC_POLICY_PROTECTED_KEY) { + /* We use the AES key size coding for all CPP algs */ + set_key_size_aes(&desc[*seq_size], key_len); + set_cpp_crypto_key(&desc[*seq_size], ctx_p->cpp.slot); + flow_mode = cc_out_flow_mode(ctx_p); + } else { + if (flow_mode == S_DIN_to_AES) { + if (cc_key_type(tfm) == CC_HW_PROTECTED_KEY) { + set_hw_crypto_key(&desc[*seq_size], + ctx_p->hw.key1_slot); + } else { + /* CC_POLICY_UNPROTECTED_KEY + * Invalid keys are filtered out in + * sethkey() + */ + din_size = (key_len == 24) ? + AES_MAX_KEY_SIZE : key_len; + + set_din_type(&desc[*seq_size], DMA_DLLI, + key_dma_addr, din_size, + NS_BIT); + } + set_key_size_aes(&desc[*seq_size], key_len); + } else { + /*des*/ + set_din_type(&desc[*seq_size], DMA_DLLI, + key_dma_addr, key_len, NS_BIT); + set_key_size_des(&desc[*seq_size], key_len); + } + set_setup_mode(&desc[*seq_size], SETUP_LOAD_KEY0); + } + set_flow_mode(&desc[*seq_size], flow_mode); + (*seq_size)++; + break; + case DRV_CIPHER_XTS: + case DRV_CIPHER_ESSIV: + /* Load AES key */ + hw_desc_init(&desc[*seq_size]); + set_cipher_mode(&desc[*seq_size], cipher_mode); + set_cipher_config0(&desc[*seq_size], direction); + if (cc_key_type(tfm) == CC_HW_PROTECTED_KEY) { + set_hw_crypto_key(&desc[*seq_size], + ctx_p->hw.key1_slot); + } else { + set_din_type(&desc[*seq_size], DMA_DLLI, key_dma_addr, + (key_len / 2), NS_BIT); + } + set_key_size_aes(&desc[*seq_size], (key_len / 2)); + set_flow_mode(&desc[*seq_size], flow_mode); + set_setup_mode(&desc[*seq_size], SETUP_LOAD_KEY0); + (*seq_size)++; + break; + default: + dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode); + } +} + +static void cc_setup_mlli_desc(struct crypto_tfm *tfm, + struct cipher_req_ctx *req_ctx, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes, void *areq, + struct cc_hw_desc desc[], unsigned int *seq_size) +{ + struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); + struct device *dev = drvdata_to_dev(ctx_p->drvdata); + + if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI) { + /* bypass */ + dev_dbg(dev, " bypass params addr %pad length 0x%X addr 0x%08X\n", + &req_ctx->mlli_params.mlli_dma_addr, + req_ctx->mlli_params.mlli_len, + ctx_p->drvdata->mlli_sram_addr); + hw_desc_init(&desc[*seq_size]); + set_din_type(&desc[*seq_size], DMA_DLLI, + req_ctx->mlli_params.mlli_dma_addr, + req_ctx->mlli_params.mlli_len, NS_BIT); + set_dout_sram(&desc[*seq_size], + ctx_p->drvdata->mlli_sram_addr, + req_ctx->mlli_params.mlli_len); + set_flow_mode(&desc[*seq_size], BYPASS); + (*seq_size)++; + } +} + +static void cc_setup_flow_desc(struct crypto_tfm *tfm, + struct cipher_req_ctx *req_ctx, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes, struct cc_hw_desc desc[], + unsigned int *seq_size) +{ + struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); + struct device *dev = drvdata_to_dev(ctx_p->drvdata); + unsigned int flow_mode = cc_out_flow_mode(ctx_p); + bool last_desc = (ctx_p->key_type == CC_POLICY_PROTECTED_KEY || + ctx_p->cipher_mode == DRV_CIPHER_ECB); + + /* Process */ + if (req_ctx->dma_buf_type == CC_DMA_BUF_DLLI) { + dev_dbg(dev, " data params addr %pad length 0x%X\n", + &sg_dma_address(src), nbytes); + dev_dbg(dev, " data params addr %pad length 0x%X\n", + &sg_dma_address(dst), nbytes); + hw_desc_init(&desc[*seq_size]); + set_din_type(&desc[*seq_size], DMA_DLLI, sg_dma_address(src), + nbytes, NS_BIT); + set_dout_dlli(&desc[*seq_size], sg_dma_address(dst), + nbytes, NS_BIT, (!last_desc ? 0 : 1)); + if (last_desc) + set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]); + + set_flow_mode(&desc[*seq_size], flow_mode); + (*seq_size)++; + } else { + hw_desc_init(&desc[*seq_size]); + set_din_type(&desc[*seq_size], DMA_MLLI, + ctx_p->drvdata->mlli_sram_addr, + req_ctx->in_mlli_nents, NS_BIT); + if (req_ctx->out_nents == 0) { + dev_dbg(dev, " din/dout params addr 0x%08X addr 0x%08X\n", + ctx_p->drvdata->mlli_sram_addr, + ctx_p->drvdata->mlli_sram_addr); + set_dout_mlli(&desc[*seq_size], + ctx_p->drvdata->mlli_sram_addr, + req_ctx->in_mlli_nents, NS_BIT, + (!last_desc ? 0 : 1)); + } else { + dev_dbg(dev, " din/dout params addr 0x%08X addr 0x%08X\n", + ctx_p->drvdata->mlli_sram_addr, + ctx_p->drvdata->mlli_sram_addr + + (u32)LLI_ENTRY_BYTE_SIZE * req_ctx->in_nents); + set_dout_mlli(&desc[*seq_size], + (ctx_p->drvdata->mlli_sram_addr + + (LLI_ENTRY_BYTE_SIZE * + req_ctx->in_mlli_nents)), + req_ctx->out_mlli_nents, NS_BIT, + (!last_desc ? 0 : 1)); + } + if (last_desc) + set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]); + + set_flow_mode(&desc[*seq_size], flow_mode); + (*seq_size)++; + } +} + +static void cc_cipher_complete(struct device *dev, void *cc_req, int err) +{ + struct skcipher_request *req = (struct skcipher_request *)cc_req; + struct scatterlist *dst = req->dst; + struct scatterlist *src = req->src; + struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); + struct crypto_skcipher *sk_tfm = crypto_skcipher_reqtfm(req); + unsigned int ivsize = crypto_skcipher_ivsize(sk_tfm); + + if (err != -EINPROGRESS) { + /* Not a BACKLOG notification */ + cc_unmap_cipher_request(dev, req_ctx, ivsize, src, dst); + memcpy(req->iv, req_ctx->iv, ivsize); + kfree_sensitive(req_ctx->iv); + } + + skcipher_request_complete(req, err); +} + +static int cc_cipher_process(struct skcipher_request *req, + enum drv_crypto_direction direction) +{ + struct crypto_skcipher *sk_tfm = crypto_skcipher_reqtfm(req); + struct crypto_tfm *tfm = crypto_skcipher_tfm(sk_tfm); + struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); + unsigned int ivsize = crypto_skcipher_ivsize(sk_tfm); + struct scatterlist *dst = req->dst; + struct scatterlist *src = req->src; + unsigned int nbytes = req->cryptlen; + void *iv = req->iv; + struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); + struct device *dev = drvdata_to_dev(ctx_p->drvdata); + struct cc_hw_desc desc[MAX_SKCIPHER_SEQ_LEN]; + struct cc_crypto_req cc_req = {}; + int rc; + unsigned int seq_len = 0; + gfp_t flags = cc_gfp_flags(&req->base); + + dev_dbg(dev, "%s req=%p iv=%p nbytes=%d\n", + ((direction == DRV_CRYPTO_DIRECTION_ENCRYPT) ? + "Encrypt" : "Decrypt"), req, iv, nbytes); + + /* STAT_PHASE_0: Init and sanity checks */ + + if (validate_data_size(ctx_p, nbytes)) { + dev_dbg(dev, "Unsupported data size %d.\n", nbytes); + rc = -EINVAL; + goto exit_process; + } + if (nbytes == 0) { + /* No data to process is valid */ + rc = 0; + goto exit_process; + } + + if (ctx_p->fallback_on) { + struct skcipher_request *subreq = skcipher_request_ctx(req); + + *subreq = *req; + skcipher_request_set_tfm(subreq, ctx_p->fallback_tfm); + if (direction == DRV_CRYPTO_DIRECTION_ENCRYPT) + return crypto_skcipher_encrypt(subreq); + else + return crypto_skcipher_decrypt(subreq); + } + + /* The IV we are handed may be allocated from the stack so + * we must copy it to a DMAable buffer before use. + */ + req_ctx->iv = kmemdup(iv, ivsize, flags); + if (!req_ctx->iv) { + rc = -ENOMEM; + goto exit_process; + } + + /* Setup request structure */ + cc_req.user_cb = cc_cipher_complete; + cc_req.user_arg = req; + + /* Setup CPP operation details */ + if (ctx_p->key_type == CC_POLICY_PROTECTED_KEY) { + cc_req.cpp.is_cpp = true; + cc_req.cpp.alg = ctx_p->cpp.alg; + cc_req.cpp.slot = ctx_p->cpp.slot; + } + + /* Setup request context */ + req_ctx->gen_ctx.op_type = direction; + + /* STAT_PHASE_1: Map buffers */ + + rc = cc_map_cipher_request(ctx_p->drvdata, req_ctx, ivsize, nbytes, + req_ctx->iv, src, dst, flags); + if (rc) { + dev_err(dev, "map_request() failed\n"); + goto exit_process; + } + + /* STAT_PHASE_2: Create sequence */ + + /* Setup state (IV) */ + cc_setup_state_desc(tfm, req_ctx, ivsize, nbytes, desc, &seq_len); + /* Setup MLLI line, if needed */ + cc_setup_mlli_desc(tfm, req_ctx, dst, src, nbytes, req, desc, &seq_len); + /* Setup key */ + cc_setup_key_desc(tfm, req_ctx, nbytes, desc, &seq_len); + /* Setup state (IV and XEX key) */ + cc_setup_xex_state_desc(tfm, req_ctx, ivsize, nbytes, desc, &seq_len); + /* Data processing */ + cc_setup_flow_desc(tfm, req_ctx, dst, src, nbytes, desc, &seq_len); + /* Read next IV */ + cc_setup_readiv_desc(tfm, req_ctx, ivsize, desc, &seq_len); + + /* STAT_PHASE_3: Lock HW and push sequence */ + + rc = cc_send_request(ctx_p->drvdata, &cc_req, desc, seq_len, + &req->base); + if (rc != -EINPROGRESS && rc != -EBUSY) { + /* Failed to send the request or request completed + * synchronously + */ + cc_unmap_cipher_request(dev, req_ctx, ivsize, src, dst); + } + +exit_process: + if (rc != -EINPROGRESS && rc != -EBUSY) { + kfree_sensitive(req_ctx->iv); + } + + return rc; +} + +static int cc_cipher_encrypt(struct skcipher_request *req) +{ + struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); + + memset(req_ctx, 0, sizeof(*req_ctx)); + + return cc_cipher_process(req, DRV_CRYPTO_DIRECTION_ENCRYPT); +} + +static int cc_cipher_decrypt(struct skcipher_request *req) +{ + struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); + + memset(req_ctx, 0, sizeof(*req_ctx)); + + return cc_cipher_process(req, DRV_CRYPTO_DIRECTION_DECRYPT); +} + +/* Block cipher alg */ +static const struct cc_alg_template skcipher_algs[] = { + { + .name = "xts(paes)", + .driver_name = "xts-paes-ccree", + .blocksize = 1, + .template_skcipher = { + .setkey = cc_cipher_sethkey, + .encrypt = cc_cipher_encrypt, + .decrypt = cc_cipher_decrypt, + .min_keysize = CC_HW_KEY_SIZE, + .max_keysize = CC_HW_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_XTS, + .flow_mode = S_DIN_to_AES, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + .sec_func = true, + }, + { + .name = "essiv(cbc(paes),sha256)", + .driver_name = "essiv-paes-ccree", + .blocksize = AES_BLOCK_SIZE, + .template_skcipher = { + .setkey = cc_cipher_sethkey, + .encrypt = cc_cipher_encrypt, + .decrypt = cc_cipher_decrypt, + .min_keysize = CC_HW_KEY_SIZE, + .max_keysize = CC_HW_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_ESSIV, + .flow_mode = S_DIN_to_AES, + .min_hw_rev = CC_HW_REV_712, + .std_body = CC_STD_NIST, + .sec_func = true, + }, + { + .name = "ecb(paes)", + .driver_name = "ecb-paes-ccree", + .blocksize = AES_BLOCK_SIZE, + .template_skcipher = { + .setkey = cc_cipher_sethkey, + .encrypt = cc_cipher_encrypt, + .decrypt = cc_cipher_decrypt, + .min_keysize = CC_HW_KEY_SIZE, + .max_keysize = CC_HW_KEY_SIZE, + .ivsize = 0, + }, + .cipher_mode = DRV_CIPHER_ECB, + .flow_mode = S_DIN_to_AES, + .min_hw_rev = CC_HW_REV_712, + .std_body = CC_STD_NIST, + .sec_func = true, + }, + { + .name = "cbc(paes)", + .driver_name = "cbc-paes-ccree", + .blocksize = AES_BLOCK_SIZE, + .template_skcipher = { + .setkey = cc_cipher_sethkey, + .encrypt = cc_cipher_encrypt, + .decrypt = cc_cipher_decrypt, + .min_keysize = CC_HW_KEY_SIZE, + .max_keysize = CC_HW_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_CBC, + .flow_mode = S_DIN_to_AES, + .min_hw_rev = CC_HW_REV_712, + .std_body = CC_STD_NIST, + .sec_func = true, + }, + { + .name = "cts(cbc(paes))", + .driver_name = "cts-cbc-paes-ccree", + .blocksize = AES_BLOCK_SIZE, + .template_skcipher = { + .setkey = cc_cipher_sethkey, + .encrypt = cc_cipher_encrypt, + .decrypt = cc_cipher_decrypt, + .min_keysize = CC_HW_KEY_SIZE, + .max_keysize = CC_HW_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_CBC_CTS, + .flow_mode = S_DIN_to_AES, + .min_hw_rev = CC_HW_REV_712, + .std_body = CC_STD_NIST, + .sec_func = true, + }, + { + .name = "ctr(paes)", + .driver_name = "ctr-paes-ccree", + .blocksize = 1, + .template_skcipher = { + .setkey = cc_cipher_sethkey, + .encrypt = cc_cipher_encrypt, + .decrypt = cc_cipher_decrypt, + .min_keysize = CC_HW_KEY_SIZE, + .max_keysize = CC_HW_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_CTR, + .flow_mode = S_DIN_to_AES, + .min_hw_rev = CC_HW_REV_712, + .std_body = CC_STD_NIST, + .sec_func = true, + }, + { + /* See https://www.mail-archive.com/linux-crypto@vger.kernel.org/msg40576.html + * for the reason why this differs from the generic + * implementation. + */ + .name = "xts(aes)", + .driver_name = "xts-aes-ccree", + .blocksize = 1, + .template_skcipher = { + .setkey = cc_cipher_setkey, + .encrypt = cc_cipher_encrypt, + .decrypt = cc_cipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE * 2, + .max_keysize = AES_MAX_KEY_SIZE * 2, + .ivsize = AES_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_XTS, + .flow_mode = S_DIN_to_AES, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "essiv(cbc(aes),sha256)", + .driver_name = "essiv-aes-ccree", + .blocksize = AES_BLOCK_SIZE, + .template_skcipher = { + .setkey = cc_cipher_setkey, + .encrypt = cc_cipher_encrypt, + .decrypt = cc_cipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_ESSIV, + .flow_mode = S_DIN_to_AES, + .min_hw_rev = CC_HW_REV_712, + .std_body = CC_STD_NIST, + }, + { + .name = "ecb(aes)", + .driver_name = "ecb-aes-ccree", + .blocksize = AES_BLOCK_SIZE, + .template_skcipher = { + .setkey = cc_cipher_setkey, + .encrypt = cc_cipher_encrypt, + .decrypt = cc_cipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = 0, + }, + .cipher_mode = DRV_CIPHER_ECB, + .flow_mode = S_DIN_to_AES, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "cbc(aes)", + .driver_name = "cbc-aes-ccree", + .blocksize = AES_BLOCK_SIZE, + .template_skcipher = { + .setkey = cc_cipher_setkey, + .encrypt = cc_cipher_encrypt, + .decrypt = cc_cipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_CBC, + .flow_mode = S_DIN_to_AES, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "cts(cbc(aes))", + .driver_name = "cts-cbc-aes-ccree", + .blocksize = AES_BLOCK_SIZE, + .template_skcipher = { + .setkey = cc_cipher_setkey, + .encrypt = cc_cipher_encrypt, + .decrypt = cc_cipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_CBC_CTS, + .flow_mode = S_DIN_to_AES, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "ctr(aes)", + .driver_name = "ctr-aes-ccree", + .blocksize = 1, + .template_skcipher = { + .setkey = cc_cipher_setkey, + .encrypt = cc_cipher_encrypt, + .decrypt = cc_cipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_CTR, + .flow_mode = S_DIN_to_AES, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "cbc(des3_ede)", + .driver_name = "cbc-3des-ccree", + .blocksize = DES3_EDE_BLOCK_SIZE, + .template_skcipher = { + .setkey = cc_cipher_setkey, + .encrypt = cc_cipher_encrypt, + .decrypt = cc_cipher_decrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_CBC, + .flow_mode = S_DIN_to_DES, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "ecb(des3_ede)", + .driver_name = "ecb-3des-ccree", + .blocksize = DES3_EDE_BLOCK_SIZE, + .template_skcipher = { + .setkey = cc_cipher_setkey, + .encrypt = cc_cipher_encrypt, + .decrypt = cc_cipher_decrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = 0, + }, + .cipher_mode = DRV_CIPHER_ECB, + .flow_mode = S_DIN_to_DES, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "cbc(des)", + .driver_name = "cbc-des-ccree", + .blocksize = DES_BLOCK_SIZE, + .template_skcipher = { + .setkey = cc_cipher_setkey, + .encrypt = cc_cipher_encrypt, + .decrypt = cc_cipher_decrypt, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_CBC, + .flow_mode = S_DIN_to_DES, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "ecb(des)", + .driver_name = "ecb-des-ccree", + .blocksize = DES_BLOCK_SIZE, + .template_skcipher = { + .setkey = cc_cipher_setkey, + .encrypt = cc_cipher_encrypt, + .decrypt = cc_cipher_decrypt, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = 0, + }, + .cipher_mode = DRV_CIPHER_ECB, + .flow_mode = S_DIN_to_DES, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "cbc(sm4)", + .driver_name = "cbc-sm4-ccree", + .blocksize = SM4_BLOCK_SIZE, + .template_skcipher = { + .setkey = cc_cipher_setkey, + .encrypt = cc_cipher_encrypt, + .decrypt = cc_cipher_decrypt, + .min_keysize = SM4_KEY_SIZE, + .max_keysize = SM4_KEY_SIZE, + .ivsize = SM4_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_CBC, + .flow_mode = S_DIN_to_SM4, + .min_hw_rev = CC_HW_REV_713, + .std_body = CC_STD_OSCCA, + }, + { + .name = "ecb(sm4)", + .driver_name = "ecb-sm4-ccree", + .blocksize = SM4_BLOCK_SIZE, + .template_skcipher = { + .setkey = cc_cipher_setkey, + .encrypt = cc_cipher_encrypt, + .decrypt = cc_cipher_decrypt, + .min_keysize = SM4_KEY_SIZE, + .max_keysize = SM4_KEY_SIZE, + .ivsize = 0, + }, + .cipher_mode = DRV_CIPHER_ECB, + .flow_mode = S_DIN_to_SM4, + .min_hw_rev = CC_HW_REV_713, + .std_body = CC_STD_OSCCA, + }, + { + .name = "ctr(sm4)", + .driver_name = "ctr-sm4-ccree", + .blocksize = 1, + .template_skcipher = { + .setkey = cc_cipher_setkey, + .encrypt = cc_cipher_encrypt, + .decrypt = cc_cipher_decrypt, + .min_keysize = SM4_KEY_SIZE, + .max_keysize = SM4_KEY_SIZE, + .ivsize = SM4_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_CTR, + .flow_mode = S_DIN_to_SM4, + .min_hw_rev = CC_HW_REV_713, + .std_body = CC_STD_OSCCA, + }, + { + .name = "cbc(psm4)", + .driver_name = "cbc-psm4-ccree", + .blocksize = SM4_BLOCK_SIZE, + .template_skcipher = { + .setkey = cc_cipher_sethkey, + .encrypt = cc_cipher_encrypt, + .decrypt = cc_cipher_decrypt, + .min_keysize = CC_HW_KEY_SIZE, + .max_keysize = CC_HW_KEY_SIZE, + .ivsize = SM4_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_CBC, + .flow_mode = S_DIN_to_SM4, + .min_hw_rev = CC_HW_REV_713, + .std_body = CC_STD_OSCCA, + .sec_func = true, + }, + { + .name = "ctr(psm4)", + .driver_name = "ctr-psm4-ccree", + .blocksize = SM4_BLOCK_SIZE, + .template_skcipher = { + .setkey = cc_cipher_sethkey, + .encrypt = cc_cipher_encrypt, + .decrypt = cc_cipher_decrypt, + .min_keysize = CC_HW_KEY_SIZE, + .max_keysize = CC_HW_KEY_SIZE, + .ivsize = SM4_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_CTR, + .flow_mode = S_DIN_to_SM4, + .min_hw_rev = CC_HW_REV_713, + .std_body = CC_STD_OSCCA, + .sec_func = true, + }, +}; + +static struct cc_crypto_alg *cc_create_alg(const struct cc_alg_template *tmpl, + struct device *dev) +{ + struct cc_crypto_alg *t_alg; + struct skcipher_alg *alg; + + t_alg = devm_kzalloc(dev, sizeof(*t_alg), GFP_KERNEL); + if (!t_alg) + return ERR_PTR(-ENOMEM); + + alg = &t_alg->skcipher_alg; + + memcpy(alg, &tmpl->template_skcipher, sizeof(*alg)); + + if (strscpy(alg->base.cra_name, tmpl->name) < 0) + return ERR_PTR(-EINVAL); + if (strscpy(alg->base.cra_driver_name, tmpl->driver_name) < 0) + return ERR_PTR(-EINVAL); + + alg->base.cra_module = THIS_MODULE; + alg->base.cra_priority = CC_CRA_PRIO; + alg->base.cra_blocksize = tmpl->blocksize; + alg->base.cra_alignmask = 0; + alg->base.cra_ctxsize = sizeof(struct cc_cipher_ctx); + + alg->base.cra_init = cc_cipher_init; + alg->base.cra_exit = cc_cipher_exit; + alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; + + t_alg->cipher_mode = tmpl->cipher_mode; + t_alg->flow_mode = tmpl->flow_mode; + + return t_alg; +} + +int cc_cipher_free(struct cc_drvdata *drvdata) +{ + struct cc_crypto_alg *t_alg, *n; + + /* Remove registered algs */ + list_for_each_entry_safe(t_alg, n, &drvdata->alg_list, entry) { + crypto_unregister_skcipher(&t_alg->skcipher_alg); + list_del(&t_alg->entry); + } + return 0; +} + +int cc_cipher_alloc(struct cc_drvdata *drvdata) +{ + struct cc_crypto_alg *t_alg; + struct device *dev = drvdata_to_dev(drvdata); + int rc = -ENOMEM; + int alg; + + INIT_LIST_HEAD(&drvdata->alg_list); + + /* Linux crypto */ + dev_dbg(dev, "Number of algorithms = %zu\n", + ARRAY_SIZE(skcipher_algs)); + for (alg = 0; alg < ARRAY_SIZE(skcipher_algs); alg++) { + if ((skcipher_algs[alg].min_hw_rev > drvdata->hw_rev) || + !(drvdata->std_bodies & skcipher_algs[alg].std_body) || + (drvdata->sec_disabled && skcipher_algs[alg].sec_func)) + continue; + + dev_dbg(dev, "creating %s\n", skcipher_algs[alg].driver_name); + t_alg = cc_create_alg(&skcipher_algs[alg], dev); + if (IS_ERR(t_alg)) { + rc = PTR_ERR(t_alg); + dev_err(dev, "%s alg allocation failed\n", + skcipher_algs[alg].driver_name); + goto fail0; + } + t_alg->drvdata = drvdata; + + dev_dbg(dev, "registering %s\n", + skcipher_algs[alg].driver_name); + rc = crypto_register_skcipher(&t_alg->skcipher_alg); + dev_dbg(dev, "%s alg registration rc = %x\n", + t_alg->skcipher_alg.base.cra_driver_name, rc); + if (rc) { + dev_err(dev, "%s alg registration failed\n", + t_alg->skcipher_alg.base.cra_driver_name); + goto fail0; + } + + list_add_tail(&t_alg->entry, &drvdata->alg_list); + dev_dbg(dev, "Registered %s\n", + t_alg->skcipher_alg.base.cra_driver_name); + } + return 0; + +fail0: + cc_cipher_free(drvdata); + return rc; +} diff --git a/drivers/crypto/ccree/cc_cipher.h b/drivers/crypto/ccree/cc_cipher.h new file mode 100644 index 000000000..da3a38707 --- /dev/null +++ b/drivers/crypto/ccree/cc_cipher.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ + +/* \file cc_cipher.h + * ARM CryptoCell Cipher Crypto API + */ + +#ifndef __CC_CIPHER_H__ +#define __CC_CIPHER_H__ + +#include +#include +#include "cc_driver.h" +#include "cc_buffer_mgr.h" + +struct cipher_req_ctx { + struct async_gen_req_ctx gen_ctx; + enum cc_req_dma_buf_type dma_buf_type; + u32 in_nents; + u32 in_mlli_nents; + u32 out_nents; + u32 out_mlli_nents; + u8 *iv; + struct mlli_params mlli_params; +}; + +int cc_cipher_alloc(struct cc_drvdata *drvdata); + +int cc_cipher_free(struct cc_drvdata *drvdata); + +struct cc_hkey_info { + u16 keylen; + u8 hw_key1; + u8 hw_key2; +} __packed; + +#define CC_HW_KEY_SIZE sizeof(struct cc_hkey_info) + +#endif /*__CC_CIPHER_H__*/ diff --git a/drivers/crypto/ccree/cc_crypto_ctx.h b/drivers/crypto/ccree/cc_crypto_ctx.h new file mode 100644 index 000000000..bd9a1c089 --- /dev/null +++ b/drivers/crypto/ccree/cc_crypto_ctx.h @@ -0,0 +1,142 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ + +#ifndef _CC_CRYPTO_CTX_H_ +#define _CC_CRYPTO_CTX_H_ + +#include + +#define CC_DRV_DES_IV_SIZE 8 +#define CC_DRV_DES_BLOCK_SIZE 8 + +#define CC_DRV_DES_ONE_KEY_SIZE 8 +#define CC_DRV_DES_DOUBLE_KEY_SIZE 16 +#define CC_DRV_DES_TRIPLE_KEY_SIZE 24 +#define CC_DRV_DES_KEY_SIZE_MAX CC_DRV_DES_TRIPLE_KEY_SIZE + +#define CC_AES_IV_SIZE 16 +#define CC_AES_IV_SIZE_WORDS (CC_AES_IV_SIZE >> 2) + +#define CC_AES_BLOCK_SIZE 16 +#define CC_AES_BLOCK_SIZE_WORDS 4 + +#define CC_AES_128_BIT_KEY_SIZE 16 +#define CC_AES_128_BIT_KEY_SIZE_WORDS (CC_AES_128_BIT_KEY_SIZE >> 2) +#define CC_AES_192_BIT_KEY_SIZE 24 +#define CC_AES_192_BIT_KEY_SIZE_WORDS (CC_AES_192_BIT_KEY_SIZE >> 2) +#define CC_AES_256_BIT_KEY_SIZE 32 +#define CC_AES_256_BIT_KEY_SIZE_WORDS (CC_AES_256_BIT_KEY_SIZE >> 2) +#define CC_AES_KEY_SIZE_MAX CC_AES_256_BIT_KEY_SIZE +#define CC_AES_KEY_SIZE_WORDS_MAX (CC_AES_KEY_SIZE_MAX >> 2) + +#define CC_MD5_DIGEST_SIZE 16 +#define CC_SHA1_DIGEST_SIZE 20 +#define CC_SHA224_DIGEST_SIZE 28 +#define CC_SHA256_DIGEST_SIZE 32 +#define CC_SHA256_DIGEST_SIZE_IN_WORDS 8 +#define CC_SHA384_DIGEST_SIZE 48 +#define CC_SHA512_DIGEST_SIZE 64 + +#define CC_SHA1_BLOCK_SIZE 64 +#define CC_SHA1_BLOCK_SIZE_IN_WORDS 16 +#define CC_MD5_BLOCK_SIZE 64 +#define CC_MD5_BLOCK_SIZE_IN_WORDS 16 +#define CC_SHA224_BLOCK_SIZE 64 +#define CC_SHA256_BLOCK_SIZE 64 +#define CC_SHA256_BLOCK_SIZE_IN_WORDS 16 +#define CC_SHA1_224_256_BLOCK_SIZE 64 +#define CC_SHA384_BLOCK_SIZE 128 +#define CC_SHA512_BLOCK_SIZE 128 + +#define CC_DIGEST_SIZE_MAX CC_SHA512_DIGEST_SIZE +#define CC_HASH_BLOCK_SIZE_MAX CC_SHA512_BLOCK_SIZE /*1024b*/ + +#define CC_HMAC_BLOCK_SIZE_MAX CC_HASH_BLOCK_SIZE_MAX + +#define CC_DRV_ALG_MAX_BLOCK_SIZE CC_HASH_BLOCK_SIZE_MAX + +#define CC_CPP_NUM_SLOTS 8 +#define CC_CPP_NUM_ALGS 2 + +enum cc_cpp_alg { + CC_CPP_SM4 = 1, + CC_CPP_AES = 0 +}; + +enum drv_engine_type { + DRV_ENGINE_NULL = 0, + DRV_ENGINE_AES = 1, + DRV_ENGINE_DES = 2, + DRV_ENGINE_HASH = 3, + DRV_ENGINE_RC4 = 4, + DRV_ENGINE_DOUT = 5, + DRV_ENGINE_RESERVE32B = S32_MAX, +}; + +enum drv_crypto_alg { + DRV_CRYPTO_ALG_NULL = -1, + DRV_CRYPTO_ALG_AES = 0, + DRV_CRYPTO_ALG_DES = 1, + DRV_CRYPTO_ALG_HASH = 2, + DRV_CRYPTO_ALG_C2 = 3, + DRV_CRYPTO_ALG_HMAC = 4, + DRV_CRYPTO_ALG_AEAD = 5, + DRV_CRYPTO_ALG_BYPASS = 6, + DRV_CRYPTO_ALG_NUM = 7, + DRV_CRYPTO_ALG_RESERVE32B = S32_MAX +}; + +enum drv_crypto_direction { + DRV_CRYPTO_DIRECTION_NULL = -1, + DRV_CRYPTO_DIRECTION_ENCRYPT = 0, + DRV_CRYPTO_DIRECTION_DECRYPT = 1, + DRV_CRYPTO_DIRECTION_DECRYPT_ENCRYPT = 3, + DRV_CRYPTO_DIRECTION_RESERVE32B = S32_MAX +}; + +enum drv_cipher_mode { + DRV_CIPHER_NULL_MODE = -1, + DRV_CIPHER_ECB = 0, + DRV_CIPHER_CBC = 1, + DRV_CIPHER_CTR = 2, + DRV_CIPHER_CBC_MAC = 3, + DRV_CIPHER_XTS = 4, + DRV_CIPHER_XCBC_MAC = 5, + DRV_CIPHER_OFB = 6, + DRV_CIPHER_CMAC = 7, + DRV_CIPHER_CCM = 8, + DRV_CIPHER_CBC_CTS = 11, + DRV_CIPHER_GCTR = 12, + DRV_CIPHER_ESSIV = 13, + DRV_CIPHER_RESERVE32B = S32_MAX +}; + +enum drv_hash_mode { + DRV_HASH_NULL = -1, + DRV_HASH_SHA1 = 0, + DRV_HASH_SHA256 = 1, + DRV_HASH_SHA224 = 2, + DRV_HASH_SHA512 = 3, + DRV_HASH_SHA384 = 4, + DRV_HASH_MD5 = 5, + DRV_HASH_CBC_MAC = 6, + DRV_HASH_XCBC_MAC = 7, + DRV_HASH_CMAC = 8, + DRV_HASH_SM3 = 9, + DRV_HASH_MODE_NUM = 10, + DRV_HASH_RESERVE32B = S32_MAX +}; + +enum drv_hash_hw_mode { + DRV_HASH_HW_MD5 = 0, + DRV_HASH_HW_SHA1 = 1, + DRV_HASH_HW_SHA256 = 2, + DRV_HASH_HW_SHA224 = 10, + DRV_HASH_HW_SHA512 = 4, + DRV_HASH_HW_SHA384 = 12, + DRV_HASH_HW_GHASH = 6, + DRV_HASH_HW_SM3 = 14, + DRV_HASH_HW_RESERVE32B = S32_MAX +}; + +#endif /* _CC_CRYPTO_CTX_H_ */ diff --git a/drivers/crypto/ccree/cc_debugfs.c b/drivers/crypto/ccree/cc_debugfs.c new file mode 100644 index 000000000..8f008f024 --- /dev/null +++ b/drivers/crypto/ccree/cc_debugfs.c @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2012-2019 ARM Limited or its affiliates. */ + +#include +#include +#include +#include "cc_driver.h" +#include "cc_crypto_ctx.h" +#include "cc_debugfs.h" + +#define CC_DEBUG_REG(_X) { \ + .name = __stringify(_X),\ + .offset = CC_REG(_X) \ + } + +/* + * This is a global var for the dentry of the + * debugfs ccree/ dir. It is not tied down to + * a specific instance of ccree, hence it is + * global. + */ +static struct dentry *cc_debugfs_dir; + +static struct debugfs_reg32 ver_sig_regs[] = { + { .name = "SIGNATURE" }, /* Must be 0th */ + { .name = "VERSION" }, /* Must be 1st */ +}; + +static const struct debugfs_reg32 pid_cid_regs[] = { + CC_DEBUG_REG(PERIPHERAL_ID_0), + CC_DEBUG_REG(PERIPHERAL_ID_1), + CC_DEBUG_REG(PERIPHERAL_ID_2), + CC_DEBUG_REG(PERIPHERAL_ID_3), + CC_DEBUG_REG(PERIPHERAL_ID_4), + CC_DEBUG_REG(COMPONENT_ID_0), + CC_DEBUG_REG(COMPONENT_ID_1), + CC_DEBUG_REG(COMPONENT_ID_2), + CC_DEBUG_REG(COMPONENT_ID_3), +}; + +static const struct debugfs_reg32 debug_regs[] = { + CC_DEBUG_REG(HOST_IRR), + CC_DEBUG_REG(HOST_POWER_DOWN_EN), + CC_DEBUG_REG(AXIM_MON_ERR), + CC_DEBUG_REG(DSCRPTR_QUEUE_CONTENT), + CC_DEBUG_REG(HOST_IMR), + CC_DEBUG_REG(AXIM_CFG), + CC_DEBUG_REG(AXIM_CACHE_PARAMS), + CC_DEBUG_REG(GPR_HOST), + CC_DEBUG_REG(AXIM_MON_COMP), +}; + +void __init cc_debugfs_global_init(void) +{ + cc_debugfs_dir = debugfs_create_dir("ccree", NULL); +} + +void cc_debugfs_global_fini(void) +{ + debugfs_remove(cc_debugfs_dir); +} + +int cc_debugfs_init(struct cc_drvdata *drvdata) +{ + struct device *dev = drvdata_to_dev(drvdata); + struct debugfs_regset32 *regset, *verset; + + regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL); + if (!regset) + return -ENOMEM; + + regset->regs = debug_regs; + regset->nregs = ARRAY_SIZE(debug_regs); + regset->base = drvdata->cc_base; + regset->dev = dev; + + drvdata->dir = debugfs_create_dir(drvdata->plat_dev->name, + cc_debugfs_dir); + + debugfs_create_regset32("regs", 0400, drvdata->dir, regset); + debugfs_create_bool("coherent", 0400, drvdata->dir, &drvdata->coherent); + + verset = devm_kzalloc(dev, sizeof(*verset), GFP_KERNEL); + /* Failing here is not important enough to fail the module load */ + if (!verset) + return 0; + + if (drvdata->hw_rev <= CC_HW_REV_712) { + ver_sig_regs[0].offset = drvdata->sig_offset; + ver_sig_regs[1].offset = drvdata->ver_offset; + verset->regs = ver_sig_regs; + verset->nregs = ARRAY_SIZE(ver_sig_regs); + } else { + verset->regs = pid_cid_regs; + verset->nregs = ARRAY_SIZE(pid_cid_regs); + } + verset->base = drvdata->cc_base; + verset->dev = dev; + + debugfs_create_regset32("version", 0400, drvdata->dir, verset); + return 0; +} + +void cc_debugfs_fini(struct cc_drvdata *drvdata) +{ + debugfs_remove_recursive(drvdata->dir); +} diff --git a/drivers/crypto/ccree/cc_debugfs.h b/drivers/crypto/ccree/cc_debugfs.h new file mode 100644 index 000000000..664ff402e --- /dev/null +++ b/drivers/crypto/ccree/cc_debugfs.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ + +#ifndef __CC_DEBUGFS_H__ +#define __CC_DEBUGFS_H__ + +#ifdef CONFIG_DEBUG_FS +void cc_debugfs_global_init(void); +void cc_debugfs_global_fini(void); + +int cc_debugfs_init(struct cc_drvdata *drvdata); +void cc_debugfs_fini(struct cc_drvdata *drvdata); + +#else + +static inline void cc_debugfs_global_init(void) {} +static inline void cc_debugfs_global_fini(void) {} + +static inline int cc_debugfs_init(struct cc_drvdata *drvdata) +{ + return 0; +} + +static inline void cc_debugfs_fini(struct cc_drvdata *drvdata) {} + +#endif + +#endif /*__CC_SYSFS_H__*/ diff --git a/drivers/crypto/ccree/cc_driver.c b/drivers/crypto/ccree/cc_driver.c new file mode 100644 index 000000000..28a23cd57 --- /dev/null +++ b/drivers/crypto/ccree/cc_driver.c @@ -0,0 +1,674 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2012-2019 ARM Limited or its affiliates. */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cc_driver.h" +#include "cc_request_mgr.h" +#include "cc_buffer_mgr.h" +#include "cc_debugfs.h" +#include "cc_cipher.h" +#include "cc_aead.h" +#include "cc_hash.h" +#include "cc_sram_mgr.h" +#include "cc_pm.h" +#include "cc_fips.h" + +bool cc_dump_desc; +module_param_named(dump_desc, cc_dump_desc, bool, 0600); +MODULE_PARM_DESC(cc_dump_desc, "Dump descriptors to kernel log as debugging aid"); +bool cc_dump_bytes; +module_param_named(dump_bytes, cc_dump_bytes, bool, 0600); +MODULE_PARM_DESC(cc_dump_bytes, "Dump buffers to kernel log as debugging aid"); + +static bool cc_sec_disable; +module_param_named(sec_disable, cc_sec_disable, bool, 0600); +MODULE_PARM_DESC(cc_sec_disable, "Disable security functions"); + +struct cc_hw_data { + char *name; + enum cc_hw_rev rev; + u32 sig; + u32 cidr_0123; + u32 pidr_0124; + int std_bodies; +}; + +#define CC_NUM_IDRS 4 +#define CC_HW_RESET_LOOP_COUNT 10 + +/* Note: PIDR3 holds CMOD/Rev so ignored for HW identification purposes */ +static const u32 pidr_0124_offsets[CC_NUM_IDRS] = { + CC_REG(PERIPHERAL_ID_0), CC_REG(PERIPHERAL_ID_1), + CC_REG(PERIPHERAL_ID_2), CC_REG(PERIPHERAL_ID_4) +}; + +static const u32 cidr_0123_offsets[CC_NUM_IDRS] = { + CC_REG(COMPONENT_ID_0), CC_REG(COMPONENT_ID_1), + CC_REG(COMPONENT_ID_2), CC_REG(COMPONENT_ID_3) +}; + +/* Hardware revisions defs. */ + +/* The 703 is a OSCCA only variant of the 713 */ +static const struct cc_hw_data cc703_hw = { + .name = "703", .rev = CC_HW_REV_713, .cidr_0123 = 0xB105F00DU, + .pidr_0124 = 0x040BB0D0U, .std_bodies = CC_STD_OSCCA +}; + +static const struct cc_hw_data cc713_hw = { + .name = "713", .rev = CC_HW_REV_713, .cidr_0123 = 0xB105F00DU, + .pidr_0124 = 0x040BB0D0U, .std_bodies = CC_STD_ALL +}; + +static const struct cc_hw_data cc712_hw = { + .name = "712", .rev = CC_HW_REV_712, .sig = 0xDCC71200U, + .std_bodies = CC_STD_ALL +}; + +static const struct cc_hw_data cc710_hw = { + .name = "710", .rev = CC_HW_REV_710, .sig = 0xDCC63200U, + .std_bodies = CC_STD_ALL +}; + +static const struct cc_hw_data cc630p_hw = { + .name = "630P", .rev = CC_HW_REV_630, .sig = 0xDCC63000U, + .std_bodies = CC_STD_ALL +}; + +static const struct of_device_id arm_ccree_dev_of_match[] = { + { .compatible = "arm,cryptocell-703-ree", .data = &cc703_hw }, + { .compatible = "arm,cryptocell-713-ree", .data = &cc713_hw }, + { .compatible = "arm,cryptocell-712-ree", .data = &cc712_hw }, + { .compatible = "arm,cryptocell-710-ree", .data = &cc710_hw }, + { .compatible = "arm,cryptocell-630p-ree", .data = &cc630p_hw }, + {} +}; +MODULE_DEVICE_TABLE(of, arm_ccree_dev_of_match); + +static void init_cc_cache_params(struct cc_drvdata *drvdata) +{ + struct device *dev = drvdata_to_dev(drvdata); + u32 cache_params, ace_const, val; + u64 mask; + + /* compute CC_AXIM_CACHE_PARAMS */ + cache_params = cc_ioread(drvdata, CC_REG(AXIM_CACHE_PARAMS)); + dev_dbg(dev, "Cache params previous: 0x%08X\n", cache_params); + + /* non cached or write-back, write allocate */ + val = drvdata->coherent ? 0xb : 0x2; + + mask = CC_GENMASK(CC_AXIM_CACHE_PARAMS_AWCACHE); + cache_params &= ~mask; + cache_params |= FIELD_PREP(mask, val); + + mask = CC_GENMASK(CC_AXIM_CACHE_PARAMS_AWCACHE_LAST); + cache_params &= ~mask; + cache_params |= FIELD_PREP(mask, val); + + mask = CC_GENMASK(CC_AXIM_CACHE_PARAMS_ARCACHE); + cache_params &= ~mask; + cache_params |= FIELD_PREP(mask, val); + + drvdata->cache_params = cache_params; + + dev_dbg(dev, "Cache params current: 0x%08X\n", cache_params); + + if (drvdata->hw_rev <= CC_HW_REV_710) + return; + + /* compute CC_AXIM_ACE_CONST */ + ace_const = cc_ioread(drvdata, CC_REG(AXIM_ACE_CONST)); + dev_dbg(dev, "ACE-const previous: 0x%08X\n", ace_const); + + /* system or outer-sharable */ + val = drvdata->coherent ? 0x2 : 0x3; + + mask = CC_GENMASK(CC_AXIM_ACE_CONST_ARDOMAIN); + ace_const &= ~mask; + ace_const |= FIELD_PREP(mask, val); + + mask = CC_GENMASK(CC_AXIM_ACE_CONST_AWDOMAIN); + ace_const &= ~mask; + ace_const |= FIELD_PREP(mask, val); + + dev_dbg(dev, "ACE-const current: 0x%08X\n", ace_const); + + drvdata->ace_const = ace_const; +} + +static u32 cc_read_idr(struct cc_drvdata *drvdata, const u32 *idr_offsets) +{ + int i; + union { + u8 regs[CC_NUM_IDRS]; + __le32 val; + } idr; + + for (i = 0; i < CC_NUM_IDRS; ++i) + idr.regs[i] = cc_ioread(drvdata, idr_offsets[i]); + + return le32_to_cpu(idr.val); +} + +void __dump_byte_array(const char *name, const u8 *buf, size_t len) +{ + char prefix[64]; + + if (!buf) + return; + + snprintf(prefix, sizeof(prefix), "%s[%zu]: ", name, len); + + print_hex_dump(KERN_DEBUG, prefix, DUMP_PREFIX_ADDRESS, 16, 1, buf, + len, false); +} + +static irqreturn_t cc_isr(int irq, void *dev_id) +{ + struct cc_drvdata *drvdata = (struct cc_drvdata *)dev_id; + struct device *dev = drvdata_to_dev(drvdata); + u32 irr; + u32 imr; + + /* STAT_OP_TYPE_GENERIC STAT_PHASE_0: Interrupt */ + /* if driver suspended return, probably shared interrupt */ + if (pm_runtime_suspended(dev)) + return IRQ_NONE; + + /* read the interrupt status */ + irr = cc_ioread(drvdata, CC_REG(HOST_IRR)); + dev_dbg(dev, "Got IRR=0x%08X\n", irr); + + if (irr == 0) /* Probably shared interrupt line */ + return IRQ_NONE; + + imr = cc_ioread(drvdata, CC_REG(HOST_IMR)); + + /* clear interrupt - must be before processing events */ + cc_iowrite(drvdata, CC_REG(HOST_ICR), irr); + + drvdata->irq = irr; + /* Completion interrupt - most probable */ + if (irr & drvdata->comp_mask) { + /* Mask all completion interrupts - will be unmasked in + * deferred service handler + */ + cc_iowrite(drvdata, CC_REG(HOST_IMR), imr | drvdata->comp_mask); + irr &= ~drvdata->comp_mask; + complete_request(drvdata); + } +#ifdef CONFIG_CRYPTO_FIPS + /* TEE FIPS interrupt */ + if (irr & CC_GPR0_IRQ_MASK) { + /* Mask interrupt - will be unmasked in Deferred service + * handler + */ + cc_iowrite(drvdata, CC_REG(HOST_IMR), imr | CC_GPR0_IRQ_MASK); + irr &= ~CC_GPR0_IRQ_MASK; + fips_handler(drvdata); + } +#endif + /* AXI error interrupt */ + if (irr & CC_AXI_ERR_IRQ_MASK) { + u32 axi_err; + + /* Read the AXI error ID */ + axi_err = cc_ioread(drvdata, CC_REG(AXIM_MON_ERR)); + dev_dbg(dev, "AXI completion error: axim_mon_err=0x%08X\n", + axi_err); + + irr &= ~CC_AXI_ERR_IRQ_MASK; + } + + if (irr) { + dev_dbg_ratelimited(dev, "IRR includes unknown cause bits (0x%08X)\n", + irr); + /* Just warning */ + } + + return IRQ_HANDLED; +} + +bool cc_wait_for_reset_completion(struct cc_drvdata *drvdata) +{ + unsigned int val; + unsigned int i; + + /* 712/710/63 has no reset completion indication, always return true */ + if (drvdata->hw_rev <= CC_HW_REV_712) + return true; + + for (i = 0; i < CC_HW_RESET_LOOP_COUNT; i++) { + /* in cc7x3 NVM_IS_IDLE indicates that CC reset is + * completed and device is fully functional + */ + val = cc_ioread(drvdata, CC_REG(NVM_IS_IDLE)); + if (val & CC_NVM_IS_IDLE_MASK) { + /* hw indicate reset completed */ + return true; + } + /* allow scheduling other process on the processor */ + schedule(); + } + /* reset not completed */ + return false; +} + +int init_cc_regs(struct cc_drvdata *drvdata) +{ + unsigned int val; + struct device *dev = drvdata_to_dev(drvdata); + + /* Unmask all AXI interrupt sources AXI_CFG1 register */ + /* AXI interrupt config are obsoleted startign at cc7x3 */ + if (drvdata->hw_rev <= CC_HW_REV_712) { + val = cc_ioread(drvdata, CC_REG(AXIM_CFG)); + cc_iowrite(drvdata, CC_REG(AXIM_CFG), val & ~CC_AXI_IRQ_MASK); + dev_dbg(dev, "AXIM_CFG=0x%08X\n", + cc_ioread(drvdata, CC_REG(AXIM_CFG))); + } + + /* Clear all pending interrupts */ + val = cc_ioread(drvdata, CC_REG(HOST_IRR)); + dev_dbg(dev, "IRR=0x%08X\n", val); + cc_iowrite(drvdata, CC_REG(HOST_ICR), val); + + /* Unmask relevant interrupt cause */ + val = drvdata->comp_mask | CC_AXI_ERR_IRQ_MASK; + + if (drvdata->hw_rev >= CC_HW_REV_712) + val |= CC_GPR0_IRQ_MASK; + + cc_iowrite(drvdata, CC_REG(HOST_IMR), ~val); + + cc_iowrite(drvdata, CC_REG(AXIM_CACHE_PARAMS), drvdata->cache_params); + if (drvdata->hw_rev >= CC_HW_REV_712) + cc_iowrite(drvdata, CC_REG(AXIM_ACE_CONST), drvdata->ace_const); + + return 0; +} + +static int init_cc_resources(struct platform_device *plat_dev) +{ + struct resource *req_mem_cc_regs = NULL; + struct cc_drvdata *new_drvdata; + struct device *dev = &plat_dev->dev; + struct device_node *np = dev->of_node; + u32 val, hw_rev_pidr, sig_cidr; + u64 dma_mask; + const struct cc_hw_data *hw_rev; + struct clk *clk; + int irq; + int rc = 0; + + new_drvdata = devm_kzalloc(dev, sizeof(*new_drvdata), GFP_KERNEL); + if (!new_drvdata) + return -ENOMEM; + + hw_rev = of_device_get_match_data(dev); + new_drvdata->hw_rev_name = hw_rev->name; + new_drvdata->hw_rev = hw_rev->rev; + new_drvdata->std_bodies = hw_rev->std_bodies; + + if (hw_rev->rev >= CC_HW_REV_712) { + new_drvdata->axim_mon_offset = CC_REG(AXIM_MON_COMP); + new_drvdata->sig_offset = CC_REG(HOST_SIGNATURE_712); + new_drvdata->ver_offset = CC_REG(HOST_VERSION_712); + } else { + new_drvdata->axim_mon_offset = CC_REG(AXIM_MON_COMP8); + new_drvdata->sig_offset = CC_REG(HOST_SIGNATURE_630); + new_drvdata->ver_offset = CC_REG(HOST_VERSION_630); + } + + new_drvdata->comp_mask = CC_COMP_IRQ_MASK; + + platform_set_drvdata(plat_dev, new_drvdata); + new_drvdata->plat_dev = plat_dev; + + clk = devm_clk_get_optional(dev, NULL); + if (IS_ERR(clk)) + return dev_err_probe(dev, PTR_ERR(clk), "Error getting clock\n"); + new_drvdata->clk = clk; + + new_drvdata->coherent = of_dma_is_coherent(np); + + /* Get device resources */ + /* First CC registers space */ + /* Map registers space */ + new_drvdata->cc_base = devm_platform_get_and_ioremap_resource(plat_dev, + 0, &req_mem_cc_regs); + if (IS_ERR(new_drvdata->cc_base)) + return PTR_ERR(new_drvdata->cc_base); + + dev_dbg(dev, "Got MEM resource (%s): %pR\n", req_mem_cc_regs->name, + req_mem_cc_regs); + dev_dbg(dev, "CC registers mapped from %pa to 0x%p\n", + &req_mem_cc_regs->start, new_drvdata->cc_base); + + /* Then IRQ */ + irq = platform_get_irq(plat_dev, 0); + if (irq < 0) + return irq; + + init_completion(&new_drvdata->hw_queue_avail); + + if (!dev->dma_mask) + dev->dma_mask = &dev->coherent_dma_mask; + + dma_mask = DMA_BIT_MASK(DMA_BIT_MASK_LEN); + rc = dma_set_coherent_mask(dev, dma_mask); + if (rc) { + dev_err(dev, "Failed in dma_set_coherent_mask, mask=%llx\n", + dma_mask); + return rc; + } + + rc = clk_prepare_enable(new_drvdata->clk); + if (rc) { + dev_err(dev, "Failed to enable clock"); + return rc; + } + + new_drvdata->sec_disabled = cc_sec_disable; + + pm_runtime_set_autosuspend_delay(dev, CC_SUSPEND_TIMEOUT); + pm_runtime_use_autosuspend(dev); + pm_runtime_set_active(dev); + pm_runtime_enable(dev); + rc = pm_runtime_get_sync(dev); + if (rc < 0) { + dev_err(dev, "pm_runtime_get_sync() failed: %d\n", rc); + goto post_pm_err; + } + + /* Wait for Cryptocell reset completion */ + if (!cc_wait_for_reset_completion(new_drvdata)) { + dev_err(dev, "Cryptocell reset not completed"); + } + + if (hw_rev->rev <= CC_HW_REV_712) { + /* Verify correct mapping */ + val = cc_ioread(new_drvdata, new_drvdata->sig_offset); + if (val != hw_rev->sig) { + dev_err(dev, "Invalid CC signature: SIGNATURE=0x%08X != expected=0x%08X\n", + val, hw_rev->sig); + rc = -EINVAL; + goto post_pm_err; + } + sig_cidr = val; + hw_rev_pidr = cc_ioread(new_drvdata, new_drvdata->ver_offset); + } else { + /* Verify correct mapping */ + val = cc_read_idr(new_drvdata, pidr_0124_offsets); + if (val != hw_rev->pidr_0124) { + dev_err(dev, "Invalid CC PIDR: PIDR0124=0x%08X != expected=0x%08X\n", + val, hw_rev->pidr_0124); + rc = -EINVAL; + goto post_pm_err; + } + hw_rev_pidr = val; + + val = cc_read_idr(new_drvdata, cidr_0123_offsets); + if (val != hw_rev->cidr_0123) { + dev_err(dev, "Invalid CC CIDR: CIDR0123=0x%08X != expected=0x%08X\n", + val, hw_rev->cidr_0123); + rc = -EINVAL; + goto post_pm_err; + } + sig_cidr = val; + + /* Check HW engine configuration */ + val = cc_ioread(new_drvdata, CC_REG(HOST_REMOVE_INPUT_PINS)); + switch (val) { + case CC_PINS_FULL: + /* This is fine */ + break; + case CC_PINS_SLIM: + if (new_drvdata->std_bodies & CC_STD_NIST) { + dev_warn(dev, "703 mode forced due to HW configuration.\n"); + new_drvdata->std_bodies = CC_STD_OSCCA; + } + break; + default: + dev_err(dev, "Unsupported engines configuration.\n"); + rc = -EINVAL; + goto post_pm_err; + } + + /* Check security disable state */ + val = cc_ioread(new_drvdata, CC_REG(SECURITY_DISABLED)); + val &= CC_SECURITY_DISABLED_MASK; + new_drvdata->sec_disabled |= !!val; + + if (!new_drvdata->sec_disabled) { + new_drvdata->comp_mask |= CC_CPP_SM4_ABORT_MASK; + if (new_drvdata->std_bodies & CC_STD_NIST) + new_drvdata->comp_mask |= CC_CPP_AES_ABORT_MASK; + } + } + + if (new_drvdata->sec_disabled) + dev_info(dev, "Security Disabled mode is in effect. Security functions disabled.\n"); + + /* Display HW versions */ + dev_info(dev, "ARM CryptoCell %s Driver: HW version 0x%08X/0x%8X, Driver version %s\n", + hw_rev->name, hw_rev_pidr, sig_cidr, DRV_MODULE_VERSION); + /* register the driver isr function */ + rc = devm_request_irq(dev, irq, cc_isr, IRQF_SHARED, "ccree", + new_drvdata); + if (rc) + goto post_pm_err; + dev_dbg(dev, "Registered to IRQ: %d\n", irq); + + init_cc_cache_params(new_drvdata); + + rc = init_cc_regs(new_drvdata); + if (rc) { + dev_err(dev, "init_cc_regs failed\n"); + goto post_pm_err; + } + + rc = cc_debugfs_init(new_drvdata); + if (rc) { + dev_err(dev, "Failed registering debugfs interface\n"); + goto post_regs_err; + } + + rc = cc_fips_init(new_drvdata); + if (rc) { + dev_err(dev, "cc_fips_init failed 0x%x\n", rc); + goto post_debugfs_err; + } + rc = cc_sram_mgr_init(new_drvdata); + if (rc) { + dev_err(dev, "cc_sram_mgr_init failed\n"); + goto post_fips_init_err; + } + + new_drvdata->mlli_sram_addr = + cc_sram_alloc(new_drvdata, MAX_MLLI_BUFF_SIZE); + if (new_drvdata->mlli_sram_addr == NULL_SRAM_ADDR) { + rc = -ENOMEM; + goto post_fips_init_err; + } + + rc = cc_req_mgr_init(new_drvdata); + if (rc) { + dev_err(dev, "cc_req_mgr_init failed\n"); + goto post_fips_init_err; + } + + rc = cc_buffer_mgr_init(new_drvdata); + if (rc) { + dev_err(dev, "cc_buffer_mgr_init failed\n"); + goto post_req_mgr_err; + } + + /* hash must be allocated first due to use of send_request_init() + * and dependency of AEAD on it + */ + rc = cc_hash_alloc(new_drvdata); + if (rc) { + dev_err(dev, "cc_hash_alloc failed\n"); + goto post_buf_mgr_err; + } + + /* Allocate crypto algs */ + rc = cc_cipher_alloc(new_drvdata); + if (rc) { + dev_err(dev, "cc_cipher_alloc failed\n"); + goto post_hash_err; + } + + rc = cc_aead_alloc(new_drvdata); + if (rc) { + dev_err(dev, "cc_aead_alloc failed\n"); + goto post_cipher_err; + } + + /* If we got here and FIPS mode is enabled + * it means all FIPS test passed, so let TEE + * know we're good. + */ + cc_set_ree_fips_status(new_drvdata, true); + + pm_runtime_put(dev); + return 0; + +post_cipher_err: + cc_cipher_free(new_drvdata); +post_hash_err: + cc_hash_free(new_drvdata); +post_buf_mgr_err: + cc_buffer_mgr_fini(new_drvdata); +post_req_mgr_err: + cc_req_mgr_fini(new_drvdata); +post_fips_init_err: + cc_fips_fini(new_drvdata); +post_debugfs_err: + cc_debugfs_fini(new_drvdata); +post_regs_err: + fini_cc_regs(new_drvdata); +post_pm_err: + pm_runtime_put_noidle(dev); + pm_runtime_disable(dev); + pm_runtime_set_suspended(dev); + clk_disable_unprepare(new_drvdata->clk); + return rc; +} + +void fini_cc_regs(struct cc_drvdata *drvdata) +{ + /* Mask all interrupts */ + cc_iowrite(drvdata, CC_REG(HOST_IMR), 0xFFFFFFFF); +} + +static void cleanup_cc_resources(struct platform_device *plat_dev) +{ + struct device *dev = &plat_dev->dev; + struct cc_drvdata *drvdata = + (struct cc_drvdata *)platform_get_drvdata(plat_dev); + + cc_aead_free(drvdata); + cc_cipher_free(drvdata); + cc_hash_free(drvdata); + cc_buffer_mgr_fini(drvdata); + cc_req_mgr_fini(drvdata); + cc_fips_fini(drvdata); + cc_debugfs_fini(drvdata); + fini_cc_regs(drvdata); + pm_runtime_put_noidle(dev); + pm_runtime_disable(dev); + pm_runtime_set_suspended(dev); + clk_disable_unprepare(drvdata->clk); +} + +unsigned int cc_get_default_hash_len(struct cc_drvdata *drvdata) +{ + if (drvdata->hw_rev >= CC_HW_REV_712) + return HASH_LEN_SIZE_712; + else + return HASH_LEN_SIZE_630; +} + +static int ccree_probe(struct platform_device *plat_dev) +{ + int rc; + struct device *dev = &plat_dev->dev; + + /* Map registers space */ + rc = init_cc_resources(plat_dev); + if (rc) + return rc; + + dev_info(dev, "ARM ccree device initialized\n"); + + return 0; +} + +static void ccree_remove(struct platform_device *plat_dev) +{ + struct device *dev = &plat_dev->dev; + + dev_dbg(dev, "Releasing ccree resources...\n"); + + cleanup_cc_resources(plat_dev); + + dev_info(dev, "ARM ccree device terminated\n"); +} + +static struct platform_driver ccree_driver = { + .driver = { + .name = "ccree", + .of_match_table = arm_ccree_dev_of_match, +#ifdef CONFIG_PM + .pm = &ccree_pm, +#endif + }, + .probe = ccree_probe, + .remove = ccree_remove, +}; + +static int __init ccree_init(void) +{ + int rc; + + cc_debugfs_global_init(); + + rc = platform_driver_register(&ccree_driver); + if (rc) { + cc_debugfs_global_fini(); + return rc; + } + + return 0; +} +module_init(ccree_init); + +static void __exit ccree_exit(void) +{ + platform_driver_unregister(&ccree_driver); + cc_debugfs_global_fini(); +} +module_exit(ccree_exit); + +/* Module description */ +MODULE_DESCRIPTION("ARM TrustZone CryptoCell REE Driver"); +MODULE_VERSION(DRV_MODULE_VERSION); +MODULE_AUTHOR("ARM"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/crypto/ccree/cc_driver.h b/drivers/crypto/ccree/cc_driver.h new file mode 100644 index 000000000..f49579aa1 --- /dev/null +++ b/drivers/crypto/ccree/cc_driver.h @@ -0,0 +1,235 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ + +/* \file cc_driver.h + * ARM CryptoCell Linux Crypto Driver + */ + +#ifndef __CC_DRIVER_H__ +#define __CC_DRIVER_H__ + +#ifdef COMP_IN_WQ +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cc_host_regs.h" +#include "cc_crypto_ctx.h" +#include "cc_hw_queue_defs.h" +#include "cc_sram_mgr.h" + +extern bool cc_dump_desc; +extern bool cc_dump_bytes; + +#define DRV_MODULE_VERSION "5.0" + +enum cc_hw_rev { + CC_HW_REV_630 = 630, + CC_HW_REV_710 = 710, + CC_HW_REV_712 = 712, + CC_HW_REV_713 = 713 +}; + +enum cc_std_body { + CC_STD_NIST = 0x1, + CC_STD_OSCCA = 0x2, + CC_STD_ALL = 0x3 +}; + +#define CC_PINS_FULL 0x0 +#define CC_PINS_SLIM 0x9F + +/* Maximum DMA mask supported by IP */ +#define DMA_BIT_MASK_LEN 48 + +#define CC_AXI_IRQ_MASK ((1 << CC_AXIM_CFG_BRESPMASK_BIT_SHIFT) | \ + (1 << CC_AXIM_CFG_RRESPMASK_BIT_SHIFT) | \ + (1 << CC_AXIM_CFG_INFLTMASK_BIT_SHIFT) | \ + (1 << CC_AXIM_CFG_COMPMASK_BIT_SHIFT)) + +#define CC_AXI_ERR_IRQ_MASK BIT(CC_HOST_IRR_AXI_ERR_INT_BIT_SHIFT) + +#define CC_COMP_IRQ_MASK BIT(CC_HOST_IRR_AXIM_COMP_INT_BIT_SHIFT) + +#define CC_SECURITY_DISABLED_MASK BIT(CC_SECURITY_DISABLED_VALUE_BIT_SHIFT) + +#define CC_NVM_IS_IDLE_MASK BIT(CC_NVM_IS_IDLE_VALUE_BIT_SHIFT) + +#define AXIM_MON_COMP_VALUE CC_GENMASK(CC_AXIM_MON_COMP_VALUE) + +#define CC_CPP_AES_ABORT_MASK ( \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_AES_0_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_AES_1_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_AES_2_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_AES_3_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_AES_4_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_AES_5_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_AES_6_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_AES_7_MASK_BIT_SHIFT)) + +#define CC_CPP_SM4_ABORT_MASK ( \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_SM_0_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_SM_1_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_SM_2_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_SM_3_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_SM_4_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_SM_5_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_SM_6_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_SM_7_MASK_BIT_SHIFT)) + +/* Register name mangling macro */ +#define CC_REG(reg_name) CC_ ## reg_name ## _REG_OFFSET + +/* TEE FIPS status interrupt */ +#define CC_GPR0_IRQ_MASK BIT(CC_HOST_IRR_GPR0_BIT_SHIFT) + +#define CC_CRA_PRIO 400 + +#define MIN_HW_QUEUE_SIZE 50 /* Minimum size required for proper function */ + +#define MAX_REQUEST_QUEUE_SIZE 4096 +#define MAX_MLLI_BUFF_SIZE 2080 + +/* Definitions for HW descriptors DIN/DOUT fields */ +#define NS_BIT 1 +#define AXI_ID 0 +/* AXI_ID is not actually the AXI ID of the transaction but the value of AXI_ID + * field in the HW descriptor. The DMA engine +8 that value. + */ + +struct cc_cpp_req { + bool is_cpp; + enum cc_cpp_alg alg; + u8 slot; +}; + +#define CC_MAX_IVGEN_DMA_ADDRESSES 3 +struct cc_crypto_req { + void (*user_cb)(struct device *dev, void *req, int err); + void *user_arg; + struct completion seq_compl; /* request completion */ + struct cc_cpp_req cpp; +}; + +/** + * struct cc_drvdata - driver private data context + * @cc_base: virt address of the CC registers + * @irq: bitmap indicating source of last interrupt + */ +struct cc_drvdata { + void __iomem *cc_base; + int irq; + struct completion hw_queue_avail; /* wait for HW queue availability */ + struct platform_device *plat_dev; + u32 mlli_sram_addr; + struct dma_pool *mlli_buffs_pool; + struct list_head alg_list; + void *hash_handle; + void *aead_handle; + void *request_mgr_handle; + void *fips_handle; + u32 sram_free_offset; /* offset to non-allocated area in SRAM */ + struct dentry *dir; /* for debugfs */ + struct clk *clk; + bool coherent; + char *hw_rev_name; + enum cc_hw_rev hw_rev; + u32 axim_mon_offset; + u32 sig_offset; + u32 ver_offset; + int std_bodies; + bool sec_disabled; + u32 comp_mask; + u32 cache_params; + u32 ace_const; +}; + +struct cc_crypto_alg { + struct list_head entry; + int cipher_mode; + int flow_mode; /* Note: currently, refers to the cipher mode only. */ + int auth_mode; + struct cc_drvdata *drvdata; + struct skcipher_alg skcipher_alg; + struct aead_alg aead_alg; +}; + +struct cc_alg_template { + char name[CRYPTO_MAX_ALG_NAME]; + char driver_name[CRYPTO_MAX_ALG_NAME]; + unsigned int blocksize; + union { + struct skcipher_alg skcipher; + struct aead_alg aead; + } template_u; + int cipher_mode; + int flow_mode; /* Note: currently, refers to the cipher mode only. */ + int auth_mode; + u32 min_hw_rev; + enum cc_std_body std_body; + bool sec_func; + unsigned int data_unit; + struct cc_drvdata *drvdata; +}; + +struct async_gen_req_ctx { + dma_addr_t iv_dma_addr; + u8 *iv; + enum drv_crypto_direction op_type; +}; + +static inline struct device *drvdata_to_dev(struct cc_drvdata *drvdata) +{ + return &drvdata->plat_dev->dev; +} + +void __dump_byte_array(const char *name, const u8 *buf, size_t len); +static inline void dump_byte_array(const char *name, const u8 *the_array, + size_t size) +{ + if (cc_dump_bytes) + __dump_byte_array(name, the_array, size); +} + +bool cc_wait_for_reset_completion(struct cc_drvdata *drvdata); +int init_cc_regs(struct cc_drvdata *drvdata); +void fini_cc_regs(struct cc_drvdata *drvdata); +unsigned int cc_get_default_hash_len(struct cc_drvdata *drvdata); + +static inline void cc_iowrite(struct cc_drvdata *drvdata, u32 reg, u32 val) +{ + iowrite32(val, (drvdata->cc_base + reg)); +} + +static inline u32 cc_ioread(struct cc_drvdata *drvdata, u32 reg) +{ + return ioread32(drvdata->cc_base + reg); +} + +static inline gfp_t cc_gfp_flags(struct crypto_async_request *req) +{ + return (req->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; +} + +static inline void set_queue_last_ind(struct cc_drvdata *drvdata, + struct cc_hw_desc *pdesc) +{ + if (drvdata->hw_rev >= CC_HW_REV_712) + set_queue_last_ind_bit(pdesc); +} + +#endif /*__CC_DRIVER_H__*/ diff --git a/drivers/crypto/ccree/cc_fips.c b/drivers/crypto/ccree/cc_fips.c new file mode 100644 index 000000000..702aefc21 --- /dev/null +++ b/drivers/crypto/ccree/cc_fips.c @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ + +#include +#include +#include + +#include "cc_driver.h" +#include "cc_fips.h" + +static void fips_dsr(unsigned long devarg); + +struct cc_fips_handle { + struct tasklet_struct tasklet; + struct notifier_block nb; + struct cc_drvdata *drvdata; +}; + +/* The function called once at driver entry point to check + * whether TEE FIPS error occurred. + */ +static bool cc_get_tee_fips_status(struct cc_drvdata *drvdata) +{ + u32 reg; + + reg = cc_ioread(drvdata, CC_REG(GPR_HOST)); + /* Did the TEE report status? */ + if (reg & CC_FIPS_SYNC_TEE_STATUS) + /* Yes. Is it OK? */ + return (reg & CC_FIPS_SYNC_MODULE_OK); + + /* No. It's either not in use or will be reported later */ + return true; +} + +/* + * This function should push the FIPS REE library status towards the TEE library + * by writing the error state to HOST_GPR0 register. + */ +void cc_set_ree_fips_status(struct cc_drvdata *drvdata, bool status) +{ + int val = CC_FIPS_SYNC_REE_STATUS; + + if (drvdata->hw_rev < CC_HW_REV_712) + return; + + val |= (status ? CC_FIPS_SYNC_MODULE_OK : CC_FIPS_SYNC_MODULE_ERROR); + + cc_iowrite(drvdata, CC_REG(HOST_GPR0), val); +} + +/* Push REE side FIPS test failure to TEE side */ +static int cc_ree_fips_failure(struct notifier_block *nb, unsigned long unused1, + void *unused2) +{ + struct cc_fips_handle *fips_h = + container_of(nb, struct cc_fips_handle, nb); + struct cc_drvdata *drvdata = fips_h->drvdata; + struct device *dev = drvdata_to_dev(drvdata); + + cc_set_ree_fips_status(drvdata, false); + dev_info(dev, "Notifying TEE of FIPS test failure...\n"); + + return NOTIFY_OK; +} + +void cc_fips_fini(struct cc_drvdata *drvdata) +{ + struct cc_fips_handle *fips_h = drvdata->fips_handle; + + if (drvdata->hw_rev < CC_HW_REV_712 || !fips_h) + return; + + atomic_notifier_chain_unregister(&fips_fail_notif_chain, &fips_h->nb); + + /* Kill tasklet */ + tasklet_kill(&fips_h->tasklet); + drvdata->fips_handle = NULL; +} + +void fips_handler(struct cc_drvdata *drvdata) +{ + struct cc_fips_handle *fips_handle_ptr = drvdata->fips_handle; + + if (drvdata->hw_rev < CC_HW_REV_712) + return; + + tasklet_schedule(&fips_handle_ptr->tasklet); +} + +static inline void tee_fips_error(struct device *dev) +{ + if (fips_enabled) + panic("ccree: TEE reported cryptographic error in fips mode!\n"); + else + dev_err(dev, "TEE reported error!\n"); +} + +/* + * This function check if cryptocell tee fips error occurred + * and in such case triggers system error + */ +void cc_tee_handle_fips_error(struct cc_drvdata *p_drvdata) +{ + struct device *dev = drvdata_to_dev(p_drvdata); + + if (!cc_get_tee_fips_status(p_drvdata)) + tee_fips_error(dev); +} + +/* Deferred service handler, run as interrupt-fired tasklet */ +static void fips_dsr(unsigned long devarg) +{ + struct cc_drvdata *drvdata = (struct cc_drvdata *)devarg; + u32 irq, val; + + irq = (drvdata->irq & (CC_GPR0_IRQ_MASK)); + + if (irq) { + cc_tee_handle_fips_error(drvdata); + } + + /* after verifying that there is nothing to do, + * unmask AXI completion interrupt. + */ + val = (CC_REG(HOST_IMR) & ~irq); + cc_iowrite(drvdata, CC_REG(HOST_IMR), val); +} + +/* The function called once at driver entry point .*/ +int cc_fips_init(struct cc_drvdata *p_drvdata) +{ + struct cc_fips_handle *fips_h; + struct device *dev = drvdata_to_dev(p_drvdata); + + if (p_drvdata->hw_rev < CC_HW_REV_712) + return 0; + + fips_h = devm_kzalloc(dev, sizeof(*fips_h), GFP_KERNEL); + if (!fips_h) + return -ENOMEM; + + p_drvdata->fips_handle = fips_h; + + dev_dbg(dev, "Initializing fips tasklet\n"); + tasklet_init(&fips_h->tasklet, fips_dsr, (unsigned long)p_drvdata); + fips_h->drvdata = p_drvdata; + fips_h->nb.notifier_call = cc_ree_fips_failure; + atomic_notifier_chain_register(&fips_fail_notif_chain, &fips_h->nb); + + cc_tee_handle_fips_error(p_drvdata); + + return 0; +} diff --git a/drivers/crypto/ccree/cc_fips.h b/drivers/crypto/ccree/cc_fips.h new file mode 100644 index 000000000..fc33eeb4d --- /dev/null +++ b/drivers/crypto/ccree/cc_fips.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ + +#ifndef __CC_FIPS_H__ +#define __CC_FIPS_H__ + +#ifdef CONFIG_CRYPTO_FIPS + +enum cc_fips_status { + CC_FIPS_SYNC_MODULE_OK = 0x0, + CC_FIPS_SYNC_MODULE_ERROR = 0x1, + CC_FIPS_SYNC_REE_STATUS = 0x4, + CC_FIPS_SYNC_TEE_STATUS = 0x8, + CC_FIPS_SYNC_STATUS_RESERVE32B = S32_MAX +}; + +int cc_fips_init(struct cc_drvdata *p_drvdata); +void cc_fips_fini(struct cc_drvdata *drvdata); +void fips_handler(struct cc_drvdata *drvdata); +void cc_set_ree_fips_status(struct cc_drvdata *drvdata, bool ok); +void cc_tee_handle_fips_error(struct cc_drvdata *p_drvdata); + +#else /* CONFIG_CRYPTO_FIPS */ + +static inline int cc_fips_init(struct cc_drvdata *p_drvdata) +{ + return 0; +} + +static inline void cc_fips_fini(struct cc_drvdata *drvdata) {} +static inline void cc_set_ree_fips_status(struct cc_drvdata *drvdata, + bool ok) {} +static inline void fips_handler(struct cc_drvdata *drvdata) {} +static inline void cc_tee_handle_fips_error(struct cc_drvdata *p_drvdata) {} + +#endif /* CONFIG_CRYPTO_FIPS */ + +#endif /*__CC_FIPS_H__*/ diff --git a/drivers/crypto/ccree/cc_hash.c b/drivers/crypto/ccree/cc_hash.c new file mode 100644 index 000000000..5de721a2c --- /dev/null +++ b/drivers/crypto/ccree/cc_hash.c @@ -0,0 +1,2313 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cc_driver.h" +#include "cc_request_mgr.h" +#include "cc_buffer_mgr.h" +#include "cc_hash.h" +#include "cc_sram_mgr.h" + +#define CC_MAX_HASH_SEQ_LEN 12 +#define CC_MAX_OPAD_KEYS_SIZE CC_MAX_HASH_BLCK_SIZE +#define CC_SM3_HASH_LEN_SIZE 8 + +struct cc_hash_handle { + u32 digest_len_sram_addr; /* const value in SRAM*/ + u32 larval_digest_sram_addr; /* const value in SRAM */ + struct list_head hash_list; +}; + +static const u32 cc_digest_len_init[] = { + 0x00000040, 0x00000000, 0x00000000, 0x00000000 }; +static const u32 cc_md5_init[] = { + SHA1_H3, SHA1_H2, SHA1_H1, SHA1_H0 }; +static const u32 cc_sha1_init[] = { + SHA1_H4, SHA1_H3, SHA1_H2, SHA1_H1, SHA1_H0 }; +static const u32 cc_sha224_init[] = { + SHA224_H7, SHA224_H6, SHA224_H5, SHA224_H4, + SHA224_H3, SHA224_H2, SHA224_H1, SHA224_H0 }; +static const u32 cc_sha256_init[] = { + SHA256_H7, SHA256_H6, SHA256_H5, SHA256_H4, + SHA256_H3, SHA256_H2, SHA256_H1, SHA256_H0 }; +static const u32 cc_digest_len_sha512_init[] = { + 0x00000080, 0x00000000, 0x00000000, 0x00000000 }; + +/* + * Due to the way the HW works, every double word in the SHA384 and SHA512 + * larval hashes must be stored in hi/lo order + */ +#define hilo(x) upper_32_bits(x), lower_32_bits(x) +static const u32 cc_sha384_init[] = { + hilo(SHA384_H7), hilo(SHA384_H6), hilo(SHA384_H5), hilo(SHA384_H4), + hilo(SHA384_H3), hilo(SHA384_H2), hilo(SHA384_H1), hilo(SHA384_H0) }; +static const u32 cc_sha512_init[] = { + hilo(SHA512_H7), hilo(SHA512_H6), hilo(SHA512_H5), hilo(SHA512_H4), + hilo(SHA512_H3), hilo(SHA512_H2), hilo(SHA512_H1), hilo(SHA512_H0) }; + +static const u32 cc_sm3_init[] = { + SM3_IVH, SM3_IVG, SM3_IVF, SM3_IVE, + SM3_IVD, SM3_IVC, SM3_IVB, SM3_IVA }; + +static void cc_setup_xcbc(struct ahash_request *areq, struct cc_hw_desc desc[], + unsigned int *seq_size); + +static void cc_setup_cmac(struct ahash_request *areq, struct cc_hw_desc desc[], + unsigned int *seq_size); + +static const void *cc_larval_digest(struct device *dev, u32 mode); + +struct cc_hash_alg { + struct list_head entry; + int hash_mode; + int hw_mode; + int inter_digestsize; + struct cc_drvdata *drvdata; + struct ahash_alg ahash_alg; +}; + +struct hash_key_req_ctx { + u32 keylen; + dma_addr_t key_dma_addr; + u8 *key; +}; + +/* hash per-session context */ +struct cc_hash_ctx { + struct cc_drvdata *drvdata; + /* holds the origin digest; the digest after "setkey" if HMAC,* + * the initial digest if HASH. + */ + u8 digest_buff[CC_MAX_HASH_DIGEST_SIZE] ____cacheline_aligned; + u8 opad_tmp_keys_buff[CC_MAX_OPAD_KEYS_SIZE] ____cacheline_aligned; + + dma_addr_t opad_tmp_keys_dma_addr ____cacheline_aligned; + dma_addr_t digest_buff_dma_addr; + /* use for hmac with key large then mode block size */ + struct hash_key_req_ctx key_params; + int hash_mode; + int hw_mode; + int inter_digestsize; + unsigned int hash_len; + struct completion setkey_comp; + bool is_hmac; +}; + +static void cc_set_desc(struct ahash_req_ctx *areq_ctx, struct cc_hash_ctx *ctx, + unsigned int flow_mode, struct cc_hw_desc desc[], + bool is_not_last_data, unsigned int *seq_size); + +static void cc_set_endianity(u32 mode, struct cc_hw_desc *desc) +{ + if (mode == DRV_HASH_MD5 || mode == DRV_HASH_SHA384 || + mode == DRV_HASH_SHA512) { + set_bytes_swap(desc, 1); + } else { + set_cipher_config0(desc, HASH_DIGEST_RESULT_LITTLE_ENDIAN); + } +} + +static int cc_map_result(struct device *dev, struct ahash_req_ctx *state, + unsigned int digestsize) +{ + state->digest_result_dma_addr = + dma_map_single(dev, state->digest_result_buff, + digestsize, DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, state->digest_result_dma_addr)) { + dev_err(dev, "Mapping digest result buffer %u B for DMA failed\n", + digestsize); + return -ENOMEM; + } + dev_dbg(dev, "Mapped digest result buffer %u B at va=%p to dma=%pad\n", + digestsize, state->digest_result_buff, + &state->digest_result_dma_addr); + + return 0; +} + +static void cc_init_req(struct device *dev, struct ahash_req_ctx *state, + struct cc_hash_ctx *ctx) +{ + bool is_hmac = ctx->is_hmac; + + memset(state, 0, sizeof(*state)); + + if (is_hmac) { + if (ctx->hw_mode != DRV_CIPHER_XCBC_MAC && + ctx->hw_mode != DRV_CIPHER_CMAC) { + dma_sync_single_for_cpu(dev, ctx->digest_buff_dma_addr, + ctx->inter_digestsize, + DMA_BIDIRECTIONAL); + + memcpy(state->digest_buff, ctx->digest_buff, + ctx->inter_digestsize); + if (ctx->hash_mode == DRV_HASH_SHA512 || + ctx->hash_mode == DRV_HASH_SHA384) + memcpy(state->digest_bytes_len, + cc_digest_len_sha512_init, + ctx->hash_len); + else + memcpy(state->digest_bytes_len, + cc_digest_len_init, + ctx->hash_len); + } + + if (ctx->hash_mode != DRV_HASH_NULL) { + dma_sync_single_for_cpu(dev, + ctx->opad_tmp_keys_dma_addr, + ctx->inter_digestsize, + DMA_BIDIRECTIONAL); + memcpy(state->opad_digest_buff, + ctx->opad_tmp_keys_buff, ctx->inter_digestsize); + } + } else { /*hash*/ + /* Copy the initial digests if hash flow. */ + const void *larval = cc_larval_digest(dev, ctx->hash_mode); + + memcpy(state->digest_buff, larval, ctx->inter_digestsize); + } +} + +static int cc_map_req(struct device *dev, struct ahash_req_ctx *state, + struct cc_hash_ctx *ctx) +{ + bool is_hmac = ctx->is_hmac; + + state->digest_buff_dma_addr = + dma_map_single(dev, state->digest_buff, + ctx->inter_digestsize, DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, state->digest_buff_dma_addr)) { + dev_err(dev, "Mapping digest len %d B at va=%p for DMA failed\n", + ctx->inter_digestsize, state->digest_buff); + return -EINVAL; + } + dev_dbg(dev, "Mapped digest %d B at va=%p to dma=%pad\n", + ctx->inter_digestsize, state->digest_buff, + &state->digest_buff_dma_addr); + + if (ctx->hw_mode != DRV_CIPHER_XCBC_MAC) { + state->digest_bytes_len_dma_addr = + dma_map_single(dev, state->digest_bytes_len, + HASH_MAX_LEN_SIZE, DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, state->digest_bytes_len_dma_addr)) { + dev_err(dev, "Mapping digest len %u B at va=%p for DMA failed\n", + HASH_MAX_LEN_SIZE, state->digest_bytes_len); + goto unmap_digest_buf; + } + dev_dbg(dev, "Mapped digest len %u B at va=%p to dma=%pad\n", + HASH_MAX_LEN_SIZE, state->digest_bytes_len, + &state->digest_bytes_len_dma_addr); + } + + if (is_hmac && ctx->hash_mode != DRV_HASH_NULL) { + state->opad_digest_dma_addr = + dma_map_single(dev, state->opad_digest_buff, + ctx->inter_digestsize, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, state->opad_digest_dma_addr)) { + dev_err(dev, "Mapping opad digest %d B at va=%p for DMA failed\n", + ctx->inter_digestsize, + state->opad_digest_buff); + goto unmap_digest_len; + } + dev_dbg(dev, "Mapped opad digest %d B at va=%p to dma=%pad\n", + ctx->inter_digestsize, state->opad_digest_buff, + &state->opad_digest_dma_addr); + } + + return 0; + +unmap_digest_len: + if (state->digest_bytes_len_dma_addr) { + dma_unmap_single(dev, state->digest_bytes_len_dma_addr, + HASH_MAX_LEN_SIZE, DMA_BIDIRECTIONAL); + state->digest_bytes_len_dma_addr = 0; + } +unmap_digest_buf: + if (state->digest_buff_dma_addr) { + dma_unmap_single(dev, state->digest_buff_dma_addr, + ctx->inter_digestsize, DMA_BIDIRECTIONAL); + state->digest_buff_dma_addr = 0; + } + + return -EINVAL; +} + +static void cc_unmap_req(struct device *dev, struct ahash_req_ctx *state, + struct cc_hash_ctx *ctx) +{ + if (state->digest_buff_dma_addr) { + dma_unmap_single(dev, state->digest_buff_dma_addr, + ctx->inter_digestsize, DMA_BIDIRECTIONAL); + dev_dbg(dev, "Unmapped digest-buffer: digest_buff_dma_addr=%pad\n", + &state->digest_buff_dma_addr); + state->digest_buff_dma_addr = 0; + } + if (state->digest_bytes_len_dma_addr) { + dma_unmap_single(dev, state->digest_bytes_len_dma_addr, + HASH_MAX_LEN_SIZE, DMA_BIDIRECTIONAL); + dev_dbg(dev, "Unmapped digest-bytes-len buffer: digest_bytes_len_dma_addr=%pad\n", + &state->digest_bytes_len_dma_addr); + state->digest_bytes_len_dma_addr = 0; + } + if (state->opad_digest_dma_addr) { + dma_unmap_single(dev, state->opad_digest_dma_addr, + ctx->inter_digestsize, DMA_BIDIRECTIONAL); + dev_dbg(dev, "Unmapped opad-digest: opad_digest_dma_addr=%pad\n", + &state->opad_digest_dma_addr); + state->opad_digest_dma_addr = 0; + } +} + +static void cc_unmap_result(struct device *dev, struct ahash_req_ctx *state, + unsigned int digestsize, u8 *result) +{ + if (state->digest_result_dma_addr) { + dma_unmap_single(dev, state->digest_result_dma_addr, digestsize, + DMA_BIDIRECTIONAL); + dev_dbg(dev, "unmpa digest result buffer va (%p) pa (%pad) len %u\n", + state->digest_result_buff, + &state->digest_result_dma_addr, digestsize); + memcpy(result, state->digest_result_buff, digestsize); + } + state->digest_result_dma_addr = 0; +} + +static void cc_update_complete(struct device *dev, void *cc_req, int err) +{ + struct ahash_request *req = (struct ahash_request *)cc_req; + struct ahash_req_ctx *state = ahash_request_ctx_dma(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); + + dev_dbg(dev, "req=%p\n", req); + + if (err != -EINPROGRESS) { + /* Not a BACKLOG notification */ + cc_unmap_hash_request(dev, state, req->src, false); + cc_unmap_req(dev, state, ctx); + } + + ahash_request_complete(req, err); +} + +static void cc_digest_complete(struct device *dev, void *cc_req, int err) +{ + struct ahash_request *req = (struct ahash_request *)cc_req; + struct ahash_req_ctx *state = ahash_request_ctx_dma(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); + u32 digestsize = crypto_ahash_digestsize(tfm); + + dev_dbg(dev, "req=%p\n", req); + + if (err != -EINPROGRESS) { + /* Not a BACKLOG notification */ + cc_unmap_hash_request(dev, state, req->src, false); + cc_unmap_result(dev, state, digestsize, req->result); + cc_unmap_req(dev, state, ctx); + } + + ahash_request_complete(req, err); +} + +static void cc_hash_complete(struct device *dev, void *cc_req, int err) +{ + struct ahash_request *req = (struct ahash_request *)cc_req; + struct ahash_req_ctx *state = ahash_request_ctx_dma(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); + u32 digestsize = crypto_ahash_digestsize(tfm); + + dev_dbg(dev, "req=%p\n", req); + + if (err != -EINPROGRESS) { + /* Not a BACKLOG notification */ + cc_unmap_hash_request(dev, state, req->src, false); + cc_unmap_result(dev, state, digestsize, req->result); + cc_unmap_req(dev, state, ctx); + } + + ahash_request_complete(req, err); +} + +static int cc_fin_result(struct cc_hw_desc *desc, struct ahash_request *req, + int idx) +{ + struct ahash_req_ctx *state = ahash_request_ctx_dma(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); + u32 digestsize = crypto_ahash_digestsize(tfm); + + /* Get final MAC result */ + hw_desc_init(&desc[idx]); + set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode); + set_dout_dlli(&desc[idx], state->digest_result_dma_addr, digestsize, + NS_BIT, 1); + set_queue_last_ind(ctx->drvdata, &desc[idx]); + set_flow_mode(&desc[idx], S_HASH_to_DOUT); + set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); + set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); + cc_set_endianity(ctx->hash_mode, &desc[idx]); + idx++; + + return idx; +} + +static int cc_fin_hmac(struct cc_hw_desc *desc, struct ahash_request *req, + int idx) +{ + struct ahash_req_ctx *state = ahash_request_ctx_dma(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); + u32 digestsize = crypto_ahash_digestsize(tfm); + + /* store the hash digest result in the context */ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], ctx->hw_mode); + set_dout_dlli(&desc[idx], state->digest_buff_dma_addr, digestsize, + NS_BIT, 0); + set_flow_mode(&desc[idx], S_HASH_to_DOUT); + cc_set_endianity(ctx->hash_mode, &desc[idx]); + set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); + idx++; + + /* Loading hash opad xor key state */ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], ctx->hw_mode); + set_din_type(&desc[idx], DMA_DLLI, state->opad_digest_dma_addr, + ctx->inter_digestsize, NS_BIT); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); + idx++; + + /* Load the hash current length */ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], ctx->hw_mode); + set_din_sram(&desc[idx], + cc_digest_len_addr(ctx->drvdata, ctx->hash_mode), + ctx->hash_len); + set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); + idx++; + + /* Memory Barrier: wait for IPAD/OPAD axi write to complete */ + hw_desc_init(&desc[idx]); + set_din_no_dma(&desc[idx], 0, 0xfffff0); + set_dout_no_dma(&desc[idx], 0, 0, 1); + idx++; + + /* Perform HASH update */ + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr, + digestsize, NS_BIT); + set_flow_mode(&desc[idx], DIN_HASH); + idx++; + + return idx; +} + +static int cc_hash_digest(struct ahash_request *req) +{ + struct ahash_req_ctx *state = ahash_request_ctx_dma(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); + u32 digestsize = crypto_ahash_digestsize(tfm); + struct scatterlist *src = req->src; + unsigned int nbytes = req->nbytes; + u8 *result = req->result; + struct device *dev = drvdata_to_dev(ctx->drvdata); + bool is_hmac = ctx->is_hmac; + struct cc_crypto_req cc_req = {}; + struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; + u32 larval_digest_addr; + int idx = 0; + int rc = 0; + gfp_t flags = cc_gfp_flags(&req->base); + + dev_dbg(dev, "===== %s-digest (%d) ====\n", is_hmac ? "hmac" : "hash", + nbytes); + + cc_init_req(dev, state, ctx); + + if (cc_map_req(dev, state, ctx)) { + dev_err(dev, "map_ahash_source() failed\n"); + return -ENOMEM; + } + + if (cc_map_result(dev, state, digestsize)) { + dev_err(dev, "map_ahash_digest() failed\n"); + cc_unmap_req(dev, state, ctx); + return -ENOMEM; + } + + if (cc_map_hash_request_final(ctx->drvdata, state, src, nbytes, 1, + flags)) { + dev_err(dev, "map_ahash_request_final() failed\n"); + cc_unmap_result(dev, state, digestsize, result); + cc_unmap_req(dev, state, ctx); + return -ENOMEM; + } + + /* Setup request structure */ + cc_req.user_cb = cc_digest_complete; + cc_req.user_arg = req; + + /* If HMAC then load hash IPAD xor key, if HASH then load initial + * digest + */ + hw_desc_init(&desc[idx]); + set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode); + if (is_hmac) { + set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr, + ctx->inter_digestsize, NS_BIT); + } else { + larval_digest_addr = cc_larval_digest_addr(ctx->drvdata, + ctx->hash_mode); + set_din_sram(&desc[idx], larval_digest_addr, + ctx->inter_digestsize); + } + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); + idx++; + + /* Load the hash current length */ + hw_desc_init(&desc[idx]); + set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode); + + if (is_hmac) { + set_din_type(&desc[idx], DMA_DLLI, + state->digest_bytes_len_dma_addr, + ctx->hash_len, NS_BIT); + } else { + set_din_const(&desc[idx], 0, ctx->hash_len); + if (nbytes) + set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); + else + set_cipher_do(&desc[idx], DO_PAD); + } + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); + idx++; + + cc_set_desc(state, ctx, DIN_HASH, desc, false, &idx); + + if (is_hmac) { + /* HW last hash block padding (aka. "DO_PAD") */ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], ctx->hw_mode); + set_dout_dlli(&desc[idx], state->digest_buff_dma_addr, + ctx->hash_len, NS_BIT, 0); + set_flow_mode(&desc[idx], S_HASH_to_DOUT); + set_setup_mode(&desc[idx], SETUP_WRITE_STATE1); + set_cipher_do(&desc[idx], DO_PAD); + idx++; + + idx = cc_fin_hmac(desc, req, idx); + } + + idx = cc_fin_result(desc, req, idx); + + rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); + if (rc != -EINPROGRESS && rc != -EBUSY) { + dev_err(dev, "send_request() failed (rc=%d)\n", rc); + cc_unmap_hash_request(dev, state, src, true); + cc_unmap_result(dev, state, digestsize, result); + cc_unmap_req(dev, state, ctx); + } + return rc; +} + +static int cc_restore_hash(struct cc_hw_desc *desc, struct cc_hash_ctx *ctx, + struct ahash_req_ctx *state, unsigned int idx) +{ + /* Restore hash digest */ + hw_desc_init(&desc[idx]); + set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode); + set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr, + ctx->inter_digestsize, NS_BIT); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); + idx++; + + /* Restore hash current length */ + hw_desc_init(&desc[idx]); + set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode); + set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); + set_din_type(&desc[idx], DMA_DLLI, state->digest_bytes_len_dma_addr, + ctx->hash_len, NS_BIT); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); + idx++; + + cc_set_desc(state, ctx, DIN_HASH, desc, false, &idx); + + return idx; +} + +static int cc_hash_update(struct ahash_request *req) +{ + struct ahash_req_ctx *state = ahash_request_ctx_dma(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); + unsigned int block_size = crypto_tfm_alg_blocksize(&tfm->base); + struct scatterlist *src = req->src; + unsigned int nbytes = req->nbytes; + struct device *dev = drvdata_to_dev(ctx->drvdata); + struct cc_crypto_req cc_req = {}; + struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; + u32 idx = 0; + int rc; + gfp_t flags = cc_gfp_flags(&req->base); + + dev_dbg(dev, "===== %s-update (%d) ====\n", ctx->is_hmac ? + "hmac" : "hash", nbytes); + + if (nbytes == 0) { + /* no real updates required */ + return 0; + } + + rc = cc_map_hash_request_update(ctx->drvdata, state, src, nbytes, + block_size, flags); + if (rc) { + if (rc == 1) { + dev_dbg(dev, " data size not require HW update %x\n", + nbytes); + /* No hardware updates are required */ + return 0; + } + dev_err(dev, "map_ahash_request_update() failed\n"); + return -ENOMEM; + } + + if (cc_map_req(dev, state, ctx)) { + dev_err(dev, "map_ahash_source() failed\n"); + cc_unmap_hash_request(dev, state, src, true); + return -EINVAL; + } + + /* Setup request structure */ + cc_req.user_cb = cc_update_complete; + cc_req.user_arg = req; + + idx = cc_restore_hash(desc, ctx, state, idx); + + /* store the hash digest result in context */ + hw_desc_init(&desc[idx]); + set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode); + set_dout_dlli(&desc[idx], state->digest_buff_dma_addr, + ctx->inter_digestsize, NS_BIT, 0); + set_flow_mode(&desc[idx], S_HASH_to_DOUT); + set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); + idx++; + + /* store current hash length in context */ + hw_desc_init(&desc[idx]); + set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode); + set_dout_dlli(&desc[idx], state->digest_bytes_len_dma_addr, + ctx->hash_len, NS_BIT, 1); + set_queue_last_ind(ctx->drvdata, &desc[idx]); + set_flow_mode(&desc[idx], S_HASH_to_DOUT); + set_setup_mode(&desc[idx], SETUP_WRITE_STATE1); + idx++; + + rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); + if (rc != -EINPROGRESS && rc != -EBUSY) { + dev_err(dev, "send_request() failed (rc=%d)\n", rc); + cc_unmap_hash_request(dev, state, src, true); + cc_unmap_req(dev, state, ctx); + } + return rc; +} + +static int cc_do_finup(struct ahash_request *req, bool update) +{ + struct ahash_req_ctx *state = ahash_request_ctx_dma(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); + u32 digestsize = crypto_ahash_digestsize(tfm); + struct scatterlist *src = req->src; + unsigned int nbytes = req->nbytes; + u8 *result = req->result; + struct device *dev = drvdata_to_dev(ctx->drvdata); + bool is_hmac = ctx->is_hmac; + struct cc_crypto_req cc_req = {}; + struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; + unsigned int idx = 0; + int rc; + gfp_t flags = cc_gfp_flags(&req->base); + + dev_dbg(dev, "===== %s-%s (%d) ====\n", is_hmac ? "hmac" : "hash", + update ? "finup" : "final", nbytes); + + if (cc_map_req(dev, state, ctx)) { + dev_err(dev, "map_ahash_source() failed\n"); + return -EINVAL; + } + + if (cc_map_hash_request_final(ctx->drvdata, state, src, nbytes, update, + flags)) { + dev_err(dev, "map_ahash_request_final() failed\n"); + cc_unmap_req(dev, state, ctx); + return -ENOMEM; + } + if (cc_map_result(dev, state, digestsize)) { + dev_err(dev, "map_ahash_digest() failed\n"); + cc_unmap_hash_request(dev, state, src, true); + cc_unmap_req(dev, state, ctx); + return -ENOMEM; + } + + /* Setup request structure */ + cc_req.user_cb = cc_hash_complete; + cc_req.user_arg = req; + + idx = cc_restore_hash(desc, ctx, state, idx); + + /* Pad the hash */ + hw_desc_init(&desc[idx]); + set_cipher_do(&desc[idx], DO_PAD); + set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode); + set_dout_dlli(&desc[idx], state->digest_bytes_len_dma_addr, + ctx->hash_len, NS_BIT, 0); + set_setup_mode(&desc[idx], SETUP_WRITE_STATE1); + set_flow_mode(&desc[idx], S_HASH_to_DOUT); + idx++; + + if (is_hmac) + idx = cc_fin_hmac(desc, req, idx); + + idx = cc_fin_result(desc, req, idx); + + rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); + if (rc != -EINPROGRESS && rc != -EBUSY) { + dev_err(dev, "send_request() failed (rc=%d)\n", rc); + cc_unmap_hash_request(dev, state, src, true); + cc_unmap_result(dev, state, digestsize, result); + cc_unmap_req(dev, state, ctx); + } + return rc; +} + +static int cc_hash_finup(struct ahash_request *req) +{ + return cc_do_finup(req, true); +} + + +static int cc_hash_final(struct ahash_request *req) +{ + return cc_do_finup(req, false); +} + +static int cc_hash_init(struct ahash_request *req) +{ + struct ahash_req_ctx *state = ahash_request_ctx_dma(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); + struct device *dev = drvdata_to_dev(ctx->drvdata); + + dev_dbg(dev, "===== init (%d) ====\n", req->nbytes); + + cc_init_req(dev, state, ctx); + + return 0; +} + +static int cc_hash_setkey(struct crypto_ahash *ahash, const u8 *key, + unsigned int keylen) +{ + unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST }; + struct cc_crypto_req cc_req = {}; + struct cc_hash_ctx *ctx = NULL; + int blocksize = 0; + int digestsize = 0; + int i, idx = 0, rc = 0; + struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; + u32 larval_addr; + struct device *dev; + + ctx = crypto_ahash_ctx_dma(ahash); + dev = drvdata_to_dev(ctx->drvdata); + dev_dbg(dev, "start keylen: %d", keylen); + + blocksize = crypto_tfm_alg_blocksize(&ahash->base); + digestsize = crypto_ahash_digestsize(ahash); + + larval_addr = cc_larval_digest_addr(ctx->drvdata, ctx->hash_mode); + + /* The keylen value distinguishes HASH in case keylen is ZERO bytes, + * any NON-ZERO value utilizes HMAC flow + */ + ctx->key_params.keylen = keylen; + ctx->key_params.key_dma_addr = 0; + ctx->is_hmac = true; + ctx->key_params.key = NULL; + + if (keylen) { + ctx->key_params.key = kmemdup(key, keylen, GFP_KERNEL); + if (!ctx->key_params.key) + return -ENOMEM; + + ctx->key_params.key_dma_addr = + dma_map_single(dev, ctx->key_params.key, keylen, + DMA_TO_DEVICE); + if (dma_mapping_error(dev, ctx->key_params.key_dma_addr)) { + dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n", + ctx->key_params.key, keylen); + kfree_sensitive(ctx->key_params.key); + return -ENOMEM; + } + dev_dbg(dev, "mapping key-buffer: key_dma_addr=%pad keylen=%u\n", + &ctx->key_params.key_dma_addr, ctx->key_params.keylen); + + if (keylen > blocksize) { + /* Load hash initial state */ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], ctx->hw_mode); + set_din_sram(&desc[idx], larval_addr, + ctx->inter_digestsize); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); + idx++; + + /* Load the hash current length*/ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], ctx->hw_mode); + set_din_const(&desc[idx], 0, ctx->hash_len); + set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); + idx++; + + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, + ctx->key_params.key_dma_addr, keylen, + NS_BIT); + set_flow_mode(&desc[idx], DIN_HASH); + idx++; + + /* Get hashed key */ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], ctx->hw_mode); + set_dout_dlli(&desc[idx], ctx->opad_tmp_keys_dma_addr, + digestsize, NS_BIT, 0); + set_flow_mode(&desc[idx], S_HASH_to_DOUT); + set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); + set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); + cc_set_endianity(ctx->hash_mode, &desc[idx]); + idx++; + + hw_desc_init(&desc[idx]); + set_din_const(&desc[idx], 0, (blocksize - digestsize)); + set_flow_mode(&desc[idx], BYPASS); + set_dout_dlli(&desc[idx], + (ctx->opad_tmp_keys_dma_addr + + digestsize), + (blocksize - digestsize), NS_BIT, 0); + idx++; + } else { + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, + ctx->key_params.key_dma_addr, keylen, + NS_BIT); + set_flow_mode(&desc[idx], BYPASS); + set_dout_dlli(&desc[idx], ctx->opad_tmp_keys_dma_addr, + keylen, NS_BIT, 0); + idx++; + + if ((blocksize - keylen)) { + hw_desc_init(&desc[idx]); + set_din_const(&desc[idx], 0, + (blocksize - keylen)); + set_flow_mode(&desc[idx], BYPASS); + set_dout_dlli(&desc[idx], + (ctx->opad_tmp_keys_dma_addr + + keylen), (blocksize - keylen), + NS_BIT, 0); + idx++; + } + } + } else { + hw_desc_init(&desc[idx]); + set_din_const(&desc[idx], 0, blocksize); + set_flow_mode(&desc[idx], BYPASS); + set_dout_dlli(&desc[idx], (ctx->opad_tmp_keys_dma_addr), + blocksize, NS_BIT, 0); + idx++; + } + + rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx); + if (rc) { + dev_err(dev, "send_request() failed (rc=%d)\n", rc); + goto out; + } + + /* calc derived HMAC key */ + for (idx = 0, i = 0; i < 2; i++) { + /* Load hash initial state */ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], ctx->hw_mode); + set_din_sram(&desc[idx], larval_addr, ctx->inter_digestsize); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); + idx++; + + /* Load the hash current length*/ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], ctx->hw_mode); + set_din_const(&desc[idx], 0, ctx->hash_len); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); + idx++; + + /* Prepare ipad key */ + hw_desc_init(&desc[idx]); + set_xor_val(&desc[idx], hmac_pad_const[i]); + set_cipher_mode(&desc[idx], ctx->hw_mode); + set_flow_mode(&desc[idx], S_DIN_to_HASH); + set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); + idx++; + + /* Perform HASH update */ + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, ctx->opad_tmp_keys_dma_addr, + blocksize, NS_BIT); + set_cipher_mode(&desc[idx], ctx->hw_mode); + set_xor_active(&desc[idx]); + set_flow_mode(&desc[idx], DIN_HASH); + idx++; + + /* Get the IPAD/OPAD xor key (Note, IPAD is the initial digest + * of the first HASH "update" state) + */ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], ctx->hw_mode); + if (i > 0) /* Not first iteration */ + set_dout_dlli(&desc[idx], ctx->opad_tmp_keys_dma_addr, + ctx->inter_digestsize, NS_BIT, 0); + else /* First iteration */ + set_dout_dlli(&desc[idx], ctx->digest_buff_dma_addr, + ctx->inter_digestsize, NS_BIT, 0); + set_flow_mode(&desc[idx], S_HASH_to_DOUT); + set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); + idx++; + } + + rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx); + +out: + if (ctx->key_params.key_dma_addr) { + dma_unmap_single(dev, ctx->key_params.key_dma_addr, + ctx->key_params.keylen, DMA_TO_DEVICE); + dev_dbg(dev, "Unmapped key-buffer: key_dma_addr=%pad keylen=%u\n", + &ctx->key_params.key_dma_addr, ctx->key_params.keylen); + } + + kfree_sensitive(ctx->key_params.key); + + return rc; +} + +static int cc_xcbc_setkey(struct crypto_ahash *ahash, + const u8 *key, unsigned int keylen) +{ + struct cc_crypto_req cc_req = {}; + struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + struct device *dev = drvdata_to_dev(ctx->drvdata); + int rc = 0; + unsigned int idx = 0; + struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; + + dev_dbg(dev, "===== setkey (%d) ====\n", keylen); + + switch (keylen) { + case AES_KEYSIZE_128: + case AES_KEYSIZE_192: + case AES_KEYSIZE_256: + break; + default: + return -EINVAL; + } + + ctx->key_params.keylen = keylen; + + ctx->key_params.key = kmemdup(key, keylen, GFP_KERNEL); + if (!ctx->key_params.key) + return -ENOMEM; + + ctx->key_params.key_dma_addr = + dma_map_single(dev, ctx->key_params.key, keylen, DMA_TO_DEVICE); + if (dma_mapping_error(dev, ctx->key_params.key_dma_addr)) { + dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n", + key, keylen); + kfree_sensitive(ctx->key_params.key); + return -ENOMEM; + } + dev_dbg(dev, "mapping key-buffer: key_dma_addr=%pad keylen=%u\n", + &ctx->key_params.key_dma_addr, ctx->key_params.keylen); + + ctx->is_hmac = true; + /* 1. Load the AES key */ + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, ctx->key_params.key_dma_addr, + keylen, NS_BIT); + set_cipher_mode(&desc[idx], DRV_CIPHER_ECB); + set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); + set_key_size_aes(&desc[idx], keylen); + set_flow_mode(&desc[idx], S_DIN_to_AES); + set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); + idx++; + + hw_desc_init(&desc[idx]); + set_din_const(&desc[idx], 0x01010101, CC_AES_128_BIT_KEY_SIZE); + set_flow_mode(&desc[idx], DIN_AES_DOUT); + set_dout_dlli(&desc[idx], + (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K1_OFFSET), + CC_AES_128_BIT_KEY_SIZE, NS_BIT, 0); + idx++; + + hw_desc_init(&desc[idx]); + set_din_const(&desc[idx], 0x02020202, CC_AES_128_BIT_KEY_SIZE); + set_flow_mode(&desc[idx], DIN_AES_DOUT); + set_dout_dlli(&desc[idx], + (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K2_OFFSET), + CC_AES_128_BIT_KEY_SIZE, NS_BIT, 0); + idx++; + + hw_desc_init(&desc[idx]); + set_din_const(&desc[idx], 0x03030303, CC_AES_128_BIT_KEY_SIZE); + set_flow_mode(&desc[idx], DIN_AES_DOUT); + set_dout_dlli(&desc[idx], + (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K3_OFFSET), + CC_AES_128_BIT_KEY_SIZE, NS_BIT, 0); + idx++; + + rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx); + + dma_unmap_single(dev, ctx->key_params.key_dma_addr, + ctx->key_params.keylen, DMA_TO_DEVICE); + dev_dbg(dev, "Unmapped key-buffer: key_dma_addr=%pad keylen=%u\n", + &ctx->key_params.key_dma_addr, ctx->key_params.keylen); + + kfree_sensitive(ctx->key_params.key); + + return rc; +} + +static int cc_cmac_setkey(struct crypto_ahash *ahash, + const u8 *key, unsigned int keylen) +{ + struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + struct device *dev = drvdata_to_dev(ctx->drvdata); + + dev_dbg(dev, "===== setkey (%d) ====\n", keylen); + + ctx->is_hmac = true; + + switch (keylen) { + case AES_KEYSIZE_128: + case AES_KEYSIZE_192: + case AES_KEYSIZE_256: + break; + default: + return -EINVAL; + } + + ctx->key_params.keylen = keylen; + + /* STAT_PHASE_1: Copy key to ctx */ + + dma_sync_single_for_cpu(dev, ctx->opad_tmp_keys_dma_addr, + keylen, DMA_TO_DEVICE); + + memcpy(ctx->opad_tmp_keys_buff, key, keylen); + if (keylen == 24) { + memset(ctx->opad_tmp_keys_buff + 24, 0, + CC_AES_KEY_SIZE_MAX - 24); + } + + dma_sync_single_for_device(dev, ctx->opad_tmp_keys_dma_addr, + keylen, DMA_TO_DEVICE); + + ctx->key_params.keylen = keylen; + + return 0; +} + +static void cc_free_ctx(struct cc_hash_ctx *ctx) +{ + struct device *dev = drvdata_to_dev(ctx->drvdata); + + if (ctx->digest_buff_dma_addr) { + dma_unmap_single(dev, ctx->digest_buff_dma_addr, + sizeof(ctx->digest_buff), DMA_BIDIRECTIONAL); + dev_dbg(dev, "Unmapped digest-buffer: digest_buff_dma_addr=%pad\n", + &ctx->digest_buff_dma_addr); + ctx->digest_buff_dma_addr = 0; + } + if (ctx->opad_tmp_keys_dma_addr) { + dma_unmap_single(dev, ctx->opad_tmp_keys_dma_addr, + sizeof(ctx->opad_tmp_keys_buff), + DMA_BIDIRECTIONAL); + dev_dbg(dev, "Unmapped opad-digest: opad_tmp_keys_dma_addr=%pad\n", + &ctx->opad_tmp_keys_dma_addr); + ctx->opad_tmp_keys_dma_addr = 0; + } + + ctx->key_params.keylen = 0; +} + +static int cc_alloc_ctx(struct cc_hash_ctx *ctx) +{ + struct device *dev = drvdata_to_dev(ctx->drvdata); + + ctx->key_params.keylen = 0; + + ctx->digest_buff_dma_addr = + dma_map_single(dev, ctx->digest_buff, sizeof(ctx->digest_buff), + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, ctx->digest_buff_dma_addr)) { + dev_err(dev, "Mapping digest len %zu B at va=%p for DMA failed\n", + sizeof(ctx->digest_buff), ctx->digest_buff); + goto fail; + } + dev_dbg(dev, "Mapped digest %zu B at va=%p to dma=%pad\n", + sizeof(ctx->digest_buff), ctx->digest_buff, + &ctx->digest_buff_dma_addr); + + ctx->opad_tmp_keys_dma_addr = + dma_map_single(dev, ctx->opad_tmp_keys_buff, + sizeof(ctx->opad_tmp_keys_buff), + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, ctx->opad_tmp_keys_dma_addr)) { + dev_err(dev, "Mapping opad digest %zu B at va=%p for DMA failed\n", + sizeof(ctx->opad_tmp_keys_buff), + ctx->opad_tmp_keys_buff); + goto fail; + } + dev_dbg(dev, "Mapped opad_tmp_keys %zu B at va=%p to dma=%pad\n", + sizeof(ctx->opad_tmp_keys_buff), ctx->opad_tmp_keys_buff, + &ctx->opad_tmp_keys_dma_addr); + + ctx->is_hmac = false; + return 0; + +fail: + cc_free_ctx(ctx); + return -ENOMEM; +} + +static int cc_get_hash_len(struct crypto_tfm *tfm) +{ + struct cc_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm); + + if (ctx->hash_mode == DRV_HASH_SM3) + return CC_SM3_HASH_LEN_SIZE; + else + return cc_get_default_hash_len(ctx->drvdata); +} + +static int cc_cra_init(struct crypto_tfm *tfm) +{ + struct cc_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm); + struct hash_alg_common *hash_alg_common = + container_of(tfm->__crt_alg, struct hash_alg_common, base); + struct ahash_alg *ahash_alg = + container_of(hash_alg_common, struct ahash_alg, halg); + struct cc_hash_alg *cc_alg = + container_of(ahash_alg, struct cc_hash_alg, ahash_alg); + + crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm), + sizeof(struct ahash_req_ctx)); + + ctx->hash_mode = cc_alg->hash_mode; + ctx->hw_mode = cc_alg->hw_mode; + ctx->inter_digestsize = cc_alg->inter_digestsize; + ctx->drvdata = cc_alg->drvdata; + ctx->hash_len = cc_get_hash_len(tfm); + return cc_alloc_ctx(ctx); +} + +static void cc_cra_exit(struct crypto_tfm *tfm) +{ + struct cc_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm); + struct device *dev = drvdata_to_dev(ctx->drvdata); + + dev_dbg(dev, "cc_cra_exit"); + cc_free_ctx(ctx); +} + +static int cc_mac_update(struct ahash_request *req) +{ + struct ahash_req_ctx *state = ahash_request_ctx_dma(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); + struct device *dev = drvdata_to_dev(ctx->drvdata); + unsigned int block_size = crypto_tfm_alg_blocksize(&tfm->base); + struct cc_crypto_req cc_req = {}; + struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; + int rc; + u32 idx = 0; + gfp_t flags = cc_gfp_flags(&req->base); + + if (req->nbytes == 0) { + /* no real updates required */ + return 0; + } + + state->xcbc_count++; + + rc = cc_map_hash_request_update(ctx->drvdata, state, req->src, + req->nbytes, block_size, flags); + if (rc) { + if (rc == 1) { + dev_dbg(dev, " data size not require HW update %x\n", + req->nbytes); + /* No hardware updates are required */ + return 0; + } + dev_err(dev, "map_ahash_request_update() failed\n"); + return -ENOMEM; + } + + if (cc_map_req(dev, state, ctx)) { + dev_err(dev, "map_ahash_source() failed\n"); + return -EINVAL; + } + + if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) + cc_setup_xcbc(req, desc, &idx); + else + cc_setup_cmac(req, desc, &idx); + + cc_set_desc(state, ctx, DIN_AES_DOUT, desc, true, &idx); + + /* store the hash digest result in context */ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], ctx->hw_mode); + set_dout_dlli(&desc[idx], state->digest_buff_dma_addr, + ctx->inter_digestsize, NS_BIT, 1); + set_queue_last_ind(ctx->drvdata, &desc[idx]); + set_flow_mode(&desc[idx], S_AES_to_DOUT); + set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); + idx++; + + /* Setup request structure */ + cc_req.user_cb = cc_update_complete; + cc_req.user_arg = req; + + rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); + if (rc != -EINPROGRESS && rc != -EBUSY) { + dev_err(dev, "send_request() failed (rc=%d)\n", rc); + cc_unmap_hash_request(dev, state, req->src, true); + cc_unmap_req(dev, state, ctx); + } + return rc; +} + +static int cc_mac_final(struct ahash_request *req) +{ + struct ahash_req_ctx *state = ahash_request_ctx_dma(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); + struct device *dev = drvdata_to_dev(ctx->drvdata); + struct cc_crypto_req cc_req = {}; + struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; + int idx = 0; + int rc = 0; + u32 key_size, key_len; + u32 digestsize = crypto_ahash_digestsize(tfm); + gfp_t flags = cc_gfp_flags(&req->base); + u32 rem_cnt = *cc_hash_buf_cnt(state); + + if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) { + key_size = CC_AES_128_BIT_KEY_SIZE; + key_len = CC_AES_128_BIT_KEY_SIZE; + } else { + key_size = (ctx->key_params.keylen == 24) ? AES_MAX_KEY_SIZE : + ctx->key_params.keylen; + key_len = ctx->key_params.keylen; + } + + dev_dbg(dev, "===== final xcbc reminder (%d) ====\n", rem_cnt); + + if (cc_map_req(dev, state, ctx)) { + dev_err(dev, "map_ahash_source() failed\n"); + return -EINVAL; + } + + if (cc_map_hash_request_final(ctx->drvdata, state, req->src, + req->nbytes, 0, flags)) { + dev_err(dev, "map_ahash_request_final() failed\n"); + cc_unmap_req(dev, state, ctx); + return -ENOMEM; + } + + if (cc_map_result(dev, state, digestsize)) { + dev_err(dev, "map_ahash_digest() failed\n"); + cc_unmap_hash_request(dev, state, req->src, true); + cc_unmap_req(dev, state, ctx); + return -ENOMEM; + } + + /* Setup request structure */ + cc_req.user_cb = cc_hash_complete; + cc_req.user_arg = req; + + if (state->xcbc_count && rem_cnt == 0) { + /* Load key for ECB decryption */ + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], DRV_CIPHER_ECB); + set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_DECRYPT); + set_din_type(&desc[idx], DMA_DLLI, + (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K1_OFFSET), + key_size, NS_BIT); + set_key_size_aes(&desc[idx], key_len); + set_flow_mode(&desc[idx], S_DIN_to_AES); + set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); + idx++; + + /* Initiate decryption of block state to previous + * block_state-XOR-M[n] + */ + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr, + CC_AES_BLOCK_SIZE, NS_BIT); + set_dout_dlli(&desc[idx], state->digest_buff_dma_addr, + CC_AES_BLOCK_SIZE, NS_BIT, 0); + set_flow_mode(&desc[idx], DIN_AES_DOUT); + idx++; + + /* Memory Barrier: wait for axi write to complete */ + hw_desc_init(&desc[idx]); + set_din_no_dma(&desc[idx], 0, 0xfffff0); + set_dout_no_dma(&desc[idx], 0, 0, 1); + idx++; + } + + if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) + cc_setup_xcbc(req, desc, &idx); + else + cc_setup_cmac(req, desc, &idx); + + if (state->xcbc_count == 0) { + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], ctx->hw_mode); + set_key_size_aes(&desc[idx], key_len); + set_cmac_size0_mode(&desc[idx]); + set_flow_mode(&desc[idx], S_DIN_to_AES); + idx++; + } else if (rem_cnt > 0) { + cc_set_desc(state, ctx, DIN_AES_DOUT, desc, false, &idx); + } else { + hw_desc_init(&desc[idx]); + set_din_const(&desc[idx], 0x00, CC_AES_BLOCK_SIZE); + set_flow_mode(&desc[idx], DIN_AES_DOUT); + idx++; + } + + /* Get final MAC result */ + hw_desc_init(&desc[idx]); + set_dout_dlli(&desc[idx], state->digest_result_dma_addr, + digestsize, NS_BIT, 1); + set_queue_last_ind(ctx->drvdata, &desc[idx]); + set_flow_mode(&desc[idx], S_AES_to_DOUT); + set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); + set_cipher_mode(&desc[idx], ctx->hw_mode); + idx++; + + rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); + if (rc != -EINPROGRESS && rc != -EBUSY) { + dev_err(dev, "send_request() failed (rc=%d)\n", rc); + cc_unmap_hash_request(dev, state, req->src, true); + cc_unmap_result(dev, state, digestsize, req->result); + cc_unmap_req(dev, state, ctx); + } + return rc; +} + +static int cc_mac_finup(struct ahash_request *req) +{ + struct ahash_req_ctx *state = ahash_request_ctx_dma(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); + struct device *dev = drvdata_to_dev(ctx->drvdata); + struct cc_crypto_req cc_req = {}; + struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; + int idx = 0; + int rc = 0; + u32 key_len = 0; + u32 digestsize = crypto_ahash_digestsize(tfm); + gfp_t flags = cc_gfp_flags(&req->base); + + dev_dbg(dev, "===== finup xcbc(%d) ====\n", req->nbytes); + if (state->xcbc_count > 0 && req->nbytes == 0) { + dev_dbg(dev, "No data to update. Call to fdx_mac_final\n"); + return cc_mac_final(req); + } + + if (cc_map_req(dev, state, ctx)) { + dev_err(dev, "map_ahash_source() failed\n"); + return -EINVAL; + } + + if (cc_map_hash_request_final(ctx->drvdata, state, req->src, + req->nbytes, 1, flags)) { + dev_err(dev, "map_ahash_request_final() failed\n"); + cc_unmap_req(dev, state, ctx); + return -ENOMEM; + } + if (cc_map_result(dev, state, digestsize)) { + dev_err(dev, "map_ahash_digest() failed\n"); + cc_unmap_hash_request(dev, state, req->src, true); + cc_unmap_req(dev, state, ctx); + return -ENOMEM; + } + + /* Setup request structure */ + cc_req.user_cb = cc_hash_complete; + cc_req.user_arg = req; + + if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) { + key_len = CC_AES_128_BIT_KEY_SIZE; + cc_setup_xcbc(req, desc, &idx); + } else { + key_len = ctx->key_params.keylen; + cc_setup_cmac(req, desc, &idx); + } + + if (req->nbytes == 0) { + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], ctx->hw_mode); + set_key_size_aes(&desc[idx], key_len); + set_cmac_size0_mode(&desc[idx]); + set_flow_mode(&desc[idx], S_DIN_to_AES); + idx++; + } else { + cc_set_desc(state, ctx, DIN_AES_DOUT, desc, false, &idx); + } + + /* Get final MAC result */ + hw_desc_init(&desc[idx]); + set_dout_dlli(&desc[idx], state->digest_result_dma_addr, + digestsize, NS_BIT, 1); + set_queue_last_ind(ctx->drvdata, &desc[idx]); + set_flow_mode(&desc[idx], S_AES_to_DOUT); + set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); + set_cipher_mode(&desc[idx], ctx->hw_mode); + idx++; + + rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); + if (rc != -EINPROGRESS && rc != -EBUSY) { + dev_err(dev, "send_request() failed (rc=%d)\n", rc); + cc_unmap_hash_request(dev, state, req->src, true); + cc_unmap_result(dev, state, digestsize, req->result); + cc_unmap_req(dev, state, ctx); + } + return rc; +} + +static int cc_mac_digest(struct ahash_request *req) +{ + struct ahash_req_ctx *state = ahash_request_ctx_dma(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); + struct device *dev = drvdata_to_dev(ctx->drvdata); + u32 digestsize = crypto_ahash_digestsize(tfm); + struct cc_crypto_req cc_req = {}; + struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; + u32 key_len; + unsigned int idx = 0; + int rc; + gfp_t flags = cc_gfp_flags(&req->base); + + dev_dbg(dev, "===== -digest mac (%d) ====\n", req->nbytes); + + cc_init_req(dev, state, ctx); + + if (cc_map_req(dev, state, ctx)) { + dev_err(dev, "map_ahash_source() failed\n"); + return -ENOMEM; + } + if (cc_map_result(dev, state, digestsize)) { + dev_err(dev, "map_ahash_digest() failed\n"); + cc_unmap_req(dev, state, ctx); + return -ENOMEM; + } + + if (cc_map_hash_request_final(ctx->drvdata, state, req->src, + req->nbytes, 1, flags)) { + dev_err(dev, "map_ahash_request_final() failed\n"); + cc_unmap_result(dev, state, digestsize, req->result); + cc_unmap_req(dev, state, ctx); + return -ENOMEM; + } + + /* Setup request structure */ + cc_req.user_cb = cc_digest_complete; + cc_req.user_arg = req; + + if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) { + key_len = CC_AES_128_BIT_KEY_SIZE; + cc_setup_xcbc(req, desc, &idx); + } else { + key_len = ctx->key_params.keylen; + cc_setup_cmac(req, desc, &idx); + } + + if (req->nbytes == 0) { + hw_desc_init(&desc[idx]); + set_cipher_mode(&desc[idx], ctx->hw_mode); + set_key_size_aes(&desc[idx], key_len); + set_cmac_size0_mode(&desc[idx]); + set_flow_mode(&desc[idx], S_DIN_to_AES); + idx++; + } else { + cc_set_desc(state, ctx, DIN_AES_DOUT, desc, false, &idx); + } + + /* Get final MAC result */ + hw_desc_init(&desc[idx]); + set_dout_dlli(&desc[idx], state->digest_result_dma_addr, + CC_AES_BLOCK_SIZE, NS_BIT, 1); + set_queue_last_ind(ctx->drvdata, &desc[idx]); + set_flow_mode(&desc[idx], S_AES_to_DOUT); + set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); + set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); + set_cipher_mode(&desc[idx], ctx->hw_mode); + idx++; + + rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); + if (rc != -EINPROGRESS && rc != -EBUSY) { + dev_err(dev, "send_request() failed (rc=%d)\n", rc); + cc_unmap_hash_request(dev, state, req->src, true); + cc_unmap_result(dev, state, digestsize, req->result); + cc_unmap_req(dev, state, ctx); + } + return rc; +} + +static int cc_hash_export(struct ahash_request *req, void *out) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + struct ahash_req_ctx *state = ahash_request_ctx_dma(req); + u8 *curr_buff = cc_hash_buf(state); + u32 curr_buff_cnt = *cc_hash_buf_cnt(state); + const u32 tmp = CC_EXPORT_MAGIC; + + memcpy(out, &tmp, sizeof(u32)); + out += sizeof(u32); + + memcpy(out, state->digest_buff, ctx->inter_digestsize); + out += ctx->inter_digestsize; + + memcpy(out, state->digest_bytes_len, ctx->hash_len); + out += ctx->hash_len; + + memcpy(out, &curr_buff_cnt, sizeof(u32)); + out += sizeof(u32); + + memcpy(out, curr_buff, curr_buff_cnt); + + return 0; +} + +static int cc_hash_import(struct ahash_request *req, const void *in) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); + struct device *dev = drvdata_to_dev(ctx->drvdata); + struct ahash_req_ctx *state = ahash_request_ctx_dma(req); + u32 tmp; + + memcpy(&tmp, in, sizeof(u32)); + if (tmp != CC_EXPORT_MAGIC) + return -EINVAL; + in += sizeof(u32); + + cc_init_req(dev, state, ctx); + + memcpy(state->digest_buff, in, ctx->inter_digestsize); + in += ctx->inter_digestsize; + + memcpy(state->digest_bytes_len, in, ctx->hash_len); + in += ctx->hash_len; + + /* Sanity check the data as much as possible */ + memcpy(&tmp, in, sizeof(u32)); + if (tmp > CC_MAX_HASH_BLCK_SIZE) + return -EINVAL; + in += sizeof(u32); + + state->buf_cnt[0] = tmp; + memcpy(state->buffers[0], in, tmp); + + return 0; +} + +struct cc_hash_template { + char name[CRYPTO_MAX_ALG_NAME]; + char driver_name[CRYPTO_MAX_ALG_NAME]; + char mac_name[CRYPTO_MAX_ALG_NAME]; + char mac_driver_name[CRYPTO_MAX_ALG_NAME]; + unsigned int blocksize; + bool is_mac; + bool synchronize; + struct ahash_alg template_ahash; + int hash_mode; + int hw_mode; + int inter_digestsize; + struct cc_drvdata *drvdata; + u32 min_hw_rev; + enum cc_std_body std_body; +}; + +#define CC_STATE_SIZE(_x) \ + ((_x) + HASH_MAX_LEN_SIZE + CC_MAX_HASH_BLCK_SIZE + (2 * sizeof(u32))) + +/* hash descriptors */ +static struct cc_hash_template driver_hash[] = { + //Asynchronous hash template + { + .name = "sha1", + .driver_name = "sha1-ccree", + .mac_name = "hmac(sha1)", + .mac_driver_name = "hmac-sha1-ccree", + .blocksize = SHA1_BLOCK_SIZE, + .is_mac = true, + .synchronize = false, + .template_ahash = { + .init = cc_hash_init, + .update = cc_hash_update, + .final = cc_hash_final, + .finup = cc_hash_finup, + .digest = cc_hash_digest, + .export = cc_hash_export, + .import = cc_hash_import, + .setkey = cc_hash_setkey, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = CC_STATE_SIZE(SHA1_DIGEST_SIZE), + }, + }, + .hash_mode = DRV_HASH_SHA1, + .hw_mode = DRV_HASH_HW_SHA1, + .inter_digestsize = SHA1_DIGEST_SIZE, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "sha256", + .driver_name = "sha256-ccree", + .mac_name = "hmac(sha256)", + .mac_driver_name = "hmac-sha256-ccree", + .blocksize = SHA256_BLOCK_SIZE, + .is_mac = true, + .template_ahash = { + .init = cc_hash_init, + .update = cc_hash_update, + .final = cc_hash_final, + .finup = cc_hash_finup, + .digest = cc_hash_digest, + .export = cc_hash_export, + .import = cc_hash_import, + .setkey = cc_hash_setkey, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = CC_STATE_SIZE(SHA256_DIGEST_SIZE) + }, + }, + .hash_mode = DRV_HASH_SHA256, + .hw_mode = DRV_HASH_HW_SHA256, + .inter_digestsize = SHA256_DIGEST_SIZE, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "sha224", + .driver_name = "sha224-ccree", + .mac_name = "hmac(sha224)", + .mac_driver_name = "hmac-sha224-ccree", + .blocksize = SHA224_BLOCK_SIZE, + .is_mac = true, + .template_ahash = { + .init = cc_hash_init, + .update = cc_hash_update, + .final = cc_hash_final, + .finup = cc_hash_finup, + .digest = cc_hash_digest, + .export = cc_hash_export, + .import = cc_hash_import, + .setkey = cc_hash_setkey, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = CC_STATE_SIZE(SHA256_DIGEST_SIZE), + }, + }, + .hash_mode = DRV_HASH_SHA224, + .hw_mode = DRV_HASH_HW_SHA256, + .inter_digestsize = SHA256_DIGEST_SIZE, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "sha384", + .driver_name = "sha384-ccree", + .mac_name = "hmac(sha384)", + .mac_driver_name = "hmac-sha384-ccree", + .blocksize = SHA384_BLOCK_SIZE, + .is_mac = true, + .template_ahash = { + .init = cc_hash_init, + .update = cc_hash_update, + .final = cc_hash_final, + .finup = cc_hash_finup, + .digest = cc_hash_digest, + .export = cc_hash_export, + .import = cc_hash_import, + .setkey = cc_hash_setkey, + .halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = CC_STATE_SIZE(SHA512_DIGEST_SIZE), + }, + }, + .hash_mode = DRV_HASH_SHA384, + .hw_mode = DRV_HASH_HW_SHA512, + .inter_digestsize = SHA512_DIGEST_SIZE, + .min_hw_rev = CC_HW_REV_712, + .std_body = CC_STD_NIST, + }, + { + .name = "sha512", + .driver_name = "sha512-ccree", + .mac_name = "hmac(sha512)", + .mac_driver_name = "hmac-sha512-ccree", + .blocksize = SHA512_BLOCK_SIZE, + .is_mac = true, + .template_ahash = { + .init = cc_hash_init, + .update = cc_hash_update, + .final = cc_hash_final, + .finup = cc_hash_finup, + .digest = cc_hash_digest, + .export = cc_hash_export, + .import = cc_hash_import, + .setkey = cc_hash_setkey, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = CC_STATE_SIZE(SHA512_DIGEST_SIZE), + }, + }, + .hash_mode = DRV_HASH_SHA512, + .hw_mode = DRV_HASH_HW_SHA512, + .inter_digestsize = SHA512_DIGEST_SIZE, + .min_hw_rev = CC_HW_REV_712, + .std_body = CC_STD_NIST, + }, + { + .name = "md5", + .driver_name = "md5-ccree", + .mac_name = "hmac(md5)", + .mac_driver_name = "hmac-md5-ccree", + .blocksize = MD5_HMAC_BLOCK_SIZE, + .is_mac = true, + .template_ahash = { + .init = cc_hash_init, + .update = cc_hash_update, + .final = cc_hash_final, + .finup = cc_hash_finup, + .digest = cc_hash_digest, + .export = cc_hash_export, + .import = cc_hash_import, + .setkey = cc_hash_setkey, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = CC_STATE_SIZE(MD5_DIGEST_SIZE), + }, + }, + .hash_mode = DRV_HASH_MD5, + .hw_mode = DRV_HASH_HW_MD5, + .inter_digestsize = MD5_DIGEST_SIZE, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .name = "sm3", + .driver_name = "sm3-ccree", + .blocksize = SM3_BLOCK_SIZE, + .is_mac = false, + .template_ahash = { + .init = cc_hash_init, + .update = cc_hash_update, + .final = cc_hash_final, + .finup = cc_hash_finup, + .digest = cc_hash_digest, + .export = cc_hash_export, + .import = cc_hash_import, + .setkey = cc_hash_setkey, + .halg = { + .digestsize = SM3_DIGEST_SIZE, + .statesize = CC_STATE_SIZE(SM3_DIGEST_SIZE), + }, + }, + .hash_mode = DRV_HASH_SM3, + .hw_mode = DRV_HASH_HW_SM3, + .inter_digestsize = SM3_DIGEST_SIZE, + .min_hw_rev = CC_HW_REV_713, + .std_body = CC_STD_OSCCA, + }, + { + .mac_name = "xcbc(aes)", + .mac_driver_name = "xcbc-aes-ccree", + .blocksize = AES_BLOCK_SIZE, + .is_mac = true, + .template_ahash = { + .init = cc_hash_init, + .update = cc_mac_update, + .final = cc_mac_final, + .finup = cc_mac_finup, + .digest = cc_mac_digest, + .setkey = cc_xcbc_setkey, + .export = cc_hash_export, + .import = cc_hash_import, + .halg = { + .digestsize = AES_BLOCK_SIZE, + .statesize = CC_STATE_SIZE(AES_BLOCK_SIZE), + }, + }, + .hash_mode = DRV_HASH_NULL, + .hw_mode = DRV_CIPHER_XCBC_MAC, + .inter_digestsize = AES_BLOCK_SIZE, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, + { + .mac_name = "cmac(aes)", + .mac_driver_name = "cmac-aes-ccree", + .blocksize = AES_BLOCK_SIZE, + .is_mac = true, + .template_ahash = { + .init = cc_hash_init, + .update = cc_mac_update, + .final = cc_mac_final, + .finup = cc_mac_finup, + .digest = cc_mac_digest, + .setkey = cc_cmac_setkey, + .export = cc_hash_export, + .import = cc_hash_import, + .halg = { + .digestsize = AES_BLOCK_SIZE, + .statesize = CC_STATE_SIZE(AES_BLOCK_SIZE), + }, + }, + .hash_mode = DRV_HASH_NULL, + .hw_mode = DRV_CIPHER_CMAC, + .inter_digestsize = AES_BLOCK_SIZE, + .min_hw_rev = CC_HW_REV_630, + .std_body = CC_STD_NIST, + }, +}; + +static struct cc_hash_alg *cc_alloc_hash_alg(struct cc_hash_template *template, + struct device *dev, bool keyed) +{ + struct cc_hash_alg *t_crypto_alg; + struct crypto_alg *alg; + struct ahash_alg *halg; + + t_crypto_alg = devm_kzalloc(dev, sizeof(*t_crypto_alg), GFP_KERNEL); + if (!t_crypto_alg) + return ERR_PTR(-ENOMEM); + + t_crypto_alg->ahash_alg = template->template_ahash; + halg = &t_crypto_alg->ahash_alg; + alg = &halg->halg.base; + + if (keyed) { + strscpy(alg->cra_name, template->mac_name); + strscpy(alg->cra_driver_name, template->mac_driver_name); + } else { + halg->setkey = NULL; + strscpy(alg->cra_name, template->name); + strscpy(alg->cra_driver_name, template->driver_name); + } + alg->cra_module = THIS_MODULE; + alg->cra_ctxsize = sizeof(struct cc_hash_ctx) + crypto_dma_padding(); + alg->cra_priority = CC_CRA_PRIO; + alg->cra_blocksize = template->blocksize; + alg->cra_alignmask = 0; + alg->cra_exit = cc_cra_exit; + + alg->cra_init = cc_cra_init; + alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; + + t_crypto_alg->hash_mode = template->hash_mode; + t_crypto_alg->hw_mode = template->hw_mode; + t_crypto_alg->inter_digestsize = template->inter_digestsize; + + return t_crypto_alg; +} + +static int cc_init_copy_sram(struct cc_drvdata *drvdata, const u32 *data, + unsigned int size, u32 *sram_buff_ofs) +{ + struct cc_hw_desc larval_seq[CC_DIGEST_SIZE_MAX / sizeof(u32)]; + unsigned int larval_seq_len = 0; + int rc; + + cc_set_sram_desc(data, *sram_buff_ofs, size / sizeof(*data), + larval_seq, &larval_seq_len); + rc = send_request_init(drvdata, larval_seq, larval_seq_len); + if (rc) + return rc; + + *sram_buff_ofs += size; + return 0; +} + +int cc_init_hash_sram(struct cc_drvdata *drvdata) +{ + struct cc_hash_handle *hash_handle = drvdata->hash_handle; + u32 sram_buff_ofs = hash_handle->digest_len_sram_addr; + bool large_sha_supported = (drvdata->hw_rev >= CC_HW_REV_712); + bool sm3_supported = (drvdata->hw_rev >= CC_HW_REV_713); + int rc = 0; + + /* Copy-to-sram digest-len */ + rc = cc_init_copy_sram(drvdata, cc_digest_len_init, + sizeof(cc_digest_len_init), &sram_buff_ofs); + if (rc) + goto init_digest_const_err; + + if (large_sha_supported) { + /* Copy-to-sram digest-len for sha384/512 */ + rc = cc_init_copy_sram(drvdata, cc_digest_len_sha512_init, + sizeof(cc_digest_len_sha512_init), + &sram_buff_ofs); + if (rc) + goto init_digest_const_err; + } + + /* The initial digests offset */ + hash_handle->larval_digest_sram_addr = sram_buff_ofs; + + /* Copy-to-sram initial SHA* digests */ + rc = cc_init_copy_sram(drvdata, cc_md5_init, sizeof(cc_md5_init), + &sram_buff_ofs); + if (rc) + goto init_digest_const_err; + + rc = cc_init_copy_sram(drvdata, cc_sha1_init, sizeof(cc_sha1_init), + &sram_buff_ofs); + if (rc) + goto init_digest_const_err; + + rc = cc_init_copy_sram(drvdata, cc_sha224_init, sizeof(cc_sha224_init), + &sram_buff_ofs); + if (rc) + goto init_digest_const_err; + + rc = cc_init_copy_sram(drvdata, cc_sha256_init, sizeof(cc_sha256_init), + &sram_buff_ofs); + if (rc) + goto init_digest_const_err; + + if (sm3_supported) { + rc = cc_init_copy_sram(drvdata, cc_sm3_init, + sizeof(cc_sm3_init), &sram_buff_ofs); + if (rc) + goto init_digest_const_err; + } + + if (large_sha_supported) { + rc = cc_init_copy_sram(drvdata, cc_sha384_init, + sizeof(cc_sha384_init), &sram_buff_ofs); + if (rc) + goto init_digest_const_err; + + rc = cc_init_copy_sram(drvdata, cc_sha512_init, + sizeof(cc_sha512_init), &sram_buff_ofs); + if (rc) + goto init_digest_const_err; + } + +init_digest_const_err: + return rc; +} + +int cc_hash_alloc(struct cc_drvdata *drvdata) +{ + struct cc_hash_handle *hash_handle; + u32 sram_buff; + u32 sram_size_to_alloc; + struct device *dev = drvdata_to_dev(drvdata); + int rc = 0; + int alg; + + hash_handle = devm_kzalloc(dev, sizeof(*hash_handle), GFP_KERNEL); + if (!hash_handle) + return -ENOMEM; + + INIT_LIST_HEAD(&hash_handle->hash_list); + drvdata->hash_handle = hash_handle; + + sram_size_to_alloc = sizeof(cc_digest_len_init) + + sizeof(cc_md5_init) + + sizeof(cc_sha1_init) + + sizeof(cc_sha224_init) + + sizeof(cc_sha256_init); + + if (drvdata->hw_rev >= CC_HW_REV_713) + sram_size_to_alloc += sizeof(cc_sm3_init); + + if (drvdata->hw_rev >= CC_HW_REV_712) + sram_size_to_alloc += sizeof(cc_digest_len_sha512_init) + + sizeof(cc_sha384_init) + sizeof(cc_sha512_init); + + sram_buff = cc_sram_alloc(drvdata, sram_size_to_alloc); + if (sram_buff == NULL_SRAM_ADDR) { + rc = -ENOMEM; + goto fail; + } + + /* The initial digest-len offset */ + hash_handle->digest_len_sram_addr = sram_buff; + + /*must be set before the alg registration as it is being used there*/ + rc = cc_init_hash_sram(drvdata); + if (rc) { + dev_err(dev, "Init digest CONST failed (rc=%d)\n", rc); + goto fail; + } + + /* ahash registration */ + for (alg = 0; alg < ARRAY_SIZE(driver_hash); alg++) { + struct cc_hash_alg *t_alg; + int hw_mode = driver_hash[alg].hw_mode; + + /* Check that the HW revision and variants are suitable */ + if ((driver_hash[alg].min_hw_rev > drvdata->hw_rev) || + !(drvdata->std_bodies & driver_hash[alg].std_body)) + continue; + + if (driver_hash[alg].is_mac) { + /* register hmac version */ + t_alg = cc_alloc_hash_alg(&driver_hash[alg], dev, true); + if (IS_ERR(t_alg)) { + rc = PTR_ERR(t_alg); + dev_err(dev, "%s alg allocation failed\n", + driver_hash[alg].driver_name); + goto fail; + } + t_alg->drvdata = drvdata; + + rc = crypto_register_ahash(&t_alg->ahash_alg); + if (rc) { + dev_err(dev, "%s alg registration failed\n", + driver_hash[alg].driver_name); + goto fail; + } + + list_add_tail(&t_alg->entry, &hash_handle->hash_list); + } + if (hw_mode == DRV_CIPHER_XCBC_MAC || + hw_mode == DRV_CIPHER_CMAC) + continue; + + /* register hash version */ + t_alg = cc_alloc_hash_alg(&driver_hash[alg], dev, false); + if (IS_ERR(t_alg)) { + rc = PTR_ERR(t_alg); + dev_err(dev, "%s alg allocation failed\n", + driver_hash[alg].driver_name); + goto fail; + } + t_alg->drvdata = drvdata; + + rc = crypto_register_ahash(&t_alg->ahash_alg); + if (rc) { + dev_err(dev, "%s alg registration failed\n", + driver_hash[alg].driver_name); + goto fail; + } + + list_add_tail(&t_alg->entry, &hash_handle->hash_list); + } + + return 0; + +fail: + cc_hash_free(drvdata); + return rc; +} + +int cc_hash_free(struct cc_drvdata *drvdata) +{ + struct cc_hash_alg *t_hash_alg, *hash_n; + struct cc_hash_handle *hash_handle = drvdata->hash_handle; + + list_for_each_entry_safe(t_hash_alg, hash_n, &hash_handle->hash_list, + entry) { + crypto_unregister_ahash(&t_hash_alg->ahash_alg); + list_del(&t_hash_alg->entry); + } + + return 0; +} + +static void cc_setup_xcbc(struct ahash_request *areq, struct cc_hw_desc desc[], + unsigned int *seq_size) +{ + unsigned int idx = *seq_size; + struct ahash_req_ctx *state = ahash_request_ctx_dma(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); + + /* Setup XCBC MAC K1 */ + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, (ctx->opad_tmp_keys_dma_addr + + XCBC_MAC_K1_OFFSET), + CC_AES_128_BIT_KEY_SIZE, NS_BIT); + set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); + set_hash_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC, ctx->hash_mode); + set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); + set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); + set_flow_mode(&desc[idx], S_DIN_to_AES); + idx++; + + /* Setup XCBC MAC K2 */ + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, + (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K2_OFFSET), + CC_AES_128_BIT_KEY_SIZE, NS_BIT); + set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); + set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); + set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); + set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); + set_flow_mode(&desc[idx], S_DIN_to_AES); + idx++; + + /* Setup XCBC MAC K3 */ + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, + (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K3_OFFSET), + CC_AES_128_BIT_KEY_SIZE, NS_BIT); + set_setup_mode(&desc[idx], SETUP_LOAD_STATE2); + set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); + set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); + set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); + set_flow_mode(&desc[idx], S_DIN_to_AES); + idx++; + + /* Loading MAC state */ + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr, + CC_AES_BLOCK_SIZE, NS_BIT); + set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); + set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); + set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); + set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); + set_flow_mode(&desc[idx], S_DIN_to_AES); + idx++; + *seq_size = idx; +} + +static void cc_setup_cmac(struct ahash_request *areq, struct cc_hw_desc desc[], + unsigned int *seq_size) +{ + unsigned int idx = *seq_size; + struct ahash_req_ctx *state = ahash_request_ctx_dma(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); + + /* Setup CMAC Key */ + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, ctx->opad_tmp_keys_dma_addr, + ((ctx->key_params.keylen == 24) ? AES_MAX_KEY_SIZE : + ctx->key_params.keylen), NS_BIT); + set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); + set_cipher_mode(&desc[idx], DRV_CIPHER_CMAC); + set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); + set_key_size_aes(&desc[idx], ctx->key_params.keylen); + set_flow_mode(&desc[idx], S_DIN_to_AES); + idx++; + + /* Load MAC state */ + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr, + CC_AES_BLOCK_SIZE, NS_BIT); + set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); + set_cipher_mode(&desc[idx], DRV_CIPHER_CMAC); + set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); + set_key_size_aes(&desc[idx], ctx->key_params.keylen); + set_flow_mode(&desc[idx], S_DIN_to_AES); + idx++; + *seq_size = idx; +} + +static void cc_set_desc(struct ahash_req_ctx *areq_ctx, + struct cc_hash_ctx *ctx, unsigned int flow_mode, + struct cc_hw_desc desc[], bool is_not_last_data, + unsigned int *seq_size) +{ + unsigned int idx = *seq_size; + struct device *dev = drvdata_to_dev(ctx->drvdata); + + if (areq_ctx->data_dma_buf_type == CC_DMA_BUF_DLLI) { + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, + sg_dma_address(areq_ctx->curr_sg), + areq_ctx->curr_sg->length, NS_BIT); + set_flow_mode(&desc[idx], flow_mode); + idx++; + } else { + if (areq_ctx->data_dma_buf_type == CC_DMA_BUF_NULL) { + dev_dbg(dev, " NULL mode\n"); + /* nothing to build */ + return; + } + /* bypass */ + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_DLLI, + areq_ctx->mlli_params.mlli_dma_addr, + areq_ctx->mlli_params.mlli_len, NS_BIT); + set_dout_sram(&desc[idx], ctx->drvdata->mlli_sram_addr, + areq_ctx->mlli_params.mlli_len); + set_flow_mode(&desc[idx], BYPASS); + idx++; + /* process */ + hw_desc_init(&desc[idx]); + set_din_type(&desc[idx], DMA_MLLI, + ctx->drvdata->mlli_sram_addr, + areq_ctx->mlli_nents, NS_BIT); + set_flow_mode(&desc[idx], flow_mode); + idx++; + } + if (is_not_last_data) + set_din_not_last_indication(&desc[(idx - 1)]); + /* return updated desc sequence size */ + *seq_size = idx; +} + +static const void *cc_larval_digest(struct device *dev, u32 mode) +{ + switch (mode) { + case DRV_HASH_MD5: + return cc_md5_init; + case DRV_HASH_SHA1: + return cc_sha1_init; + case DRV_HASH_SHA224: + return cc_sha224_init; + case DRV_HASH_SHA256: + return cc_sha256_init; + case DRV_HASH_SHA384: + return cc_sha384_init; + case DRV_HASH_SHA512: + return cc_sha512_init; + case DRV_HASH_SM3: + return cc_sm3_init; + default: + dev_err(dev, "Invalid hash mode (%d)\n", mode); + return cc_md5_init; + } +} + +/** + * cc_larval_digest_addr() - Get the address of the initial digest in SRAM + * according to the given hash mode + * + * @drvdata: Associated device driver context + * @mode: The Hash mode. Supported modes: MD5/SHA1/SHA224/SHA256 + * + * Return: + * The address of the initial digest in SRAM + */ +u32 cc_larval_digest_addr(void *drvdata, u32 mode) +{ + struct cc_drvdata *_drvdata = (struct cc_drvdata *)drvdata; + struct cc_hash_handle *hash_handle = _drvdata->hash_handle; + struct device *dev = drvdata_to_dev(_drvdata); + bool sm3_supported = (_drvdata->hw_rev >= CC_HW_REV_713); + u32 addr; + + switch (mode) { + case DRV_HASH_NULL: + break; /*Ignore*/ + case DRV_HASH_MD5: + return (hash_handle->larval_digest_sram_addr); + case DRV_HASH_SHA1: + return (hash_handle->larval_digest_sram_addr + + sizeof(cc_md5_init)); + case DRV_HASH_SHA224: + return (hash_handle->larval_digest_sram_addr + + sizeof(cc_md5_init) + + sizeof(cc_sha1_init)); + case DRV_HASH_SHA256: + return (hash_handle->larval_digest_sram_addr + + sizeof(cc_md5_init) + + sizeof(cc_sha1_init) + + sizeof(cc_sha224_init)); + case DRV_HASH_SM3: + return (hash_handle->larval_digest_sram_addr + + sizeof(cc_md5_init) + + sizeof(cc_sha1_init) + + sizeof(cc_sha224_init) + + sizeof(cc_sha256_init)); + case DRV_HASH_SHA384: + addr = (hash_handle->larval_digest_sram_addr + + sizeof(cc_md5_init) + + sizeof(cc_sha1_init) + + sizeof(cc_sha224_init) + + sizeof(cc_sha256_init)); + if (sm3_supported) + addr += sizeof(cc_sm3_init); + return addr; + case DRV_HASH_SHA512: + addr = (hash_handle->larval_digest_sram_addr + + sizeof(cc_md5_init) + + sizeof(cc_sha1_init) + + sizeof(cc_sha224_init) + + sizeof(cc_sha256_init) + + sizeof(cc_sha384_init)); + if (sm3_supported) + addr += sizeof(cc_sm3_init); + return addr; + default: + dev_err(dev, "Invalid hash mode (%d)\n", mode); + } + + /*This is valid wrong value to avoid kernel crash*/ + return hash_handle->larval_digest_sram_addr; +} + +u32 cc_digest_len_addr(void *drvdata, u32 mode) +{ + struct cc_drvdata *_drvdata = (struct cc_drvdata *)drvdata; + struct cc_hash_handle *hash_handle = _drvdata->hash_handle; + u32 digest_len_addr = hash_handle->digest_len_sram_addr; + + switch (mode) { + case DRV_HASH_SHA1: + case DRV_HASH_SHA224: + case DRV_HASH_SHA256: + case DRV_HASH_MD5: + return digest_len_addr; + case DRV_HASH_SHA384: + case DRV_HASH_SHA512: + return digest_len_addr + sizeof(cc_digest_len_init); + default: + return digest_len_addr; /*to avoid kernel crash*/ + } +} diff --git a/drivers/crypto/ccree/cc_hash.h b/drivers/crypto/ccree/cc_hash.h new file mode 100644 index 000000000..3d0f2179e --- /dev/null +++ b/drivers/crypto/ccree/cc_hash.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ + +/* \file cc_hash.h + * ARM CryptoCell Hash Crypto API + */ + +#ifndef __CC_HASH_H__ +#define __CC_HASH_H__ + +#include "cc_buffer_mgr.h" + +#define HMAC_IPAD_CONST 0x36363636 +#define HMAC_OPAD_CONST 0x5C5C5C5C +#define HASH_LEN_SIZE_712 16 +#define HASH_LEN_SIZE_630 8 +#define HASH_MAX_LEN_SIZE HASH_LEN_SIZE_712 +#define CC_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE +#define CC_MAX_HASH_BLCK_SIZE SHA512_BLOCK_SIZE + +#define XCBC_MAC_K1_OFFSET 0 +#define XCBC_MAC_K2_OFFSET 16 +#define XCBC_MAC_K3_OFFSET 32 + +#define CC_EXPORT_MAGIC 0xC2EE1070U + +/* this struct was taken from drivers/crypto/nx/nx-aes-xcbc.c and it is used + * for xcbc/cmac statesize + */ +struct aeshash_state { + u8 state[AES_BLOCK_SIZE]; + unsigned int count; + u8 buffer[AES_BLOCK_SIZE]; +}; + +/* ahash state */ +struct ahash_req_ctx { + u8 buffers[2][CC_MAX_HASH_BLCK_SIZE] ____cacheline_aligned; + u8 digest_result_buff[CC_MAX_HASH_DIGEST_SIZE] ____cacheline_aligned; + u8 digest_buff[CC_MAX_HASH_DIGEST_SIZE] ____cacheline_aligned; + u8 opad_digest_buff[CC_MAX_HASH_DIGEST_SIZE] ____cacheline_aligned; + u8 digest_bytes_len[HASH_MAX_LEN_SIZE] ____cacheline_aligned; + struct async_gen_req_ctx gen_ctx ____cacheline_aligned; + enum cc_req_dma_buf_type data_dma_buf_type; + dma_addr_t opad_digest_dma_addr; + dma_addr_t digest_buff_dma_addr; + dma_addr_t digest_bytes_len_dma_addr; + dma_addr_t digest_result_dma_addr; + u32 buf_cnt[2]; + u32 buff_index; + u32 xcbc_count; /* count xcbc update operatations */ + struct scatterlist buff_sg[2]; + struct scatterlist *curr_sg; + u32 in_nents; + u32 mlli_nents; + struct mlli_params mlli_params; +}; + +static inline u32 *cc_hash_buf_cnt(struct ahash_req_ctx *state) +{ + return &state->buf_cnt[state->buff_index]; +} + +static inline u8 *cc_hash_buf(struct ahash_req_ctx *state) +{ + return state->buffers[state->buff_index]; +} + +static inline u32 *cc_next_buf_cnt(struct ahash_req_ctx *state) +{ + return &state->buf_cnt[state->buff_index ^ 1]; +} + +static inline u8 *cc_next_buf(struct ahash_req_ctx *state) +{ + return state->buffers[state->buff_index ^ 1]; +} + +int cc_hash_alloc(struct cc_drvdata *drvdata); +int cc_init_hash_sram(struct cc_drvdata *drvdata); +int cc_hash_free(struct cc_drvdata *drvdata); + +/** + * cc_digest_len_addr() - Gets the initial digest length + * + * @drvdata: Associated device driver context + * @mode: The Hash mode. Supported modes: MD5/SHA1/SHA224/SHA256/SHA384/SHA512 + * + * Return: + * Returns the address of the initial digest length in SRAM + */ +u32 cc_digest_len_addr(void *drvdata, u32 mode); + +/** + * cc_larval_digest_addr() - Gets the address of the initial digest in SRAM + * according to the given hash mode + * + * @drvdata: Associated device driver context + * @mode: The Hash mode. Supported modes: MD5/SHA1/SHA224/SHA256/SHA384/SHA512 + * + * Return: + * The address of the initial digest in SRAM + */ +u32 cc_larval_digest_addr(void *drvdata, u32 mode); + +#endif /*__CC_HASH_H__*/ diff --git a/drivers/crypto/ccree/cc_host_regs.h b/drivers/crypto/ccree/cc_host_regs.h new file mode 100644 index 000000000..efe3e1d8b --- /dev/null +++ b/drivers/crypto/ccree/cc_host_regs.h @@ -0,0 +1,282 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2012-2019 ARM Limited or its affiliates. */ + +#ifndef __CC_HOST_H__ +#define __CC_HOST_H__ + +// -------------------------------------- +// BLOCK: HOST_P +// -------------------------------------- + + +/* IRR */ +#define CC_HOST_IRR_REG_OFFSET 0xA00UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_0_INT_BIT_SHIFT 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_0_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_DSCRPTR_COMPLETION_LOW_INT_BIT_SHIFT 0x2UL +#define CC_HOST_IRR_DSCRPTR_COMPLETION_LOW_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_1_INT_BIT_SHIFT 0x3UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_1_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_2_INT_BIT_SHIFT 0x4UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_2_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_3_INT_BIT_SHIFT 0x5UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_3_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_4_INT_BIT_SHIFT 0x6UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_4_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_5_INT_BIT_SHIFT 0x7UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_5_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_AXI_ERR_INT_BIT_SHIFT 0x8UL +#define CC_HOST_IRR_AXI_ERR_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_6_INT_BIT_SHIFT 0x9UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_6_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_7_INT_BIT_SHIFT 0xAUL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_7_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_GPR0_BIT_SHIFT 0xBUL +#define CC_HOST_IRR_GPR0_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_0_INT_BIT_SHIFT 0xCUL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_0_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_1_INT_BIT_SHIFT 0xDUL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_1_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_2_INT_BIT_SHIFT 0xEUL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_2_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_3_INT_BIT_SHIFT 0xFUL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_3_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_4_INT_BIT_SHIFT 0x10UL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_4_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_5_INT_BIT_SHIFT 0x11UL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_5_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_6_INT_BIT_SHIFT 0x12UL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_6_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_DSCRPTR_WATERMARK_INT_BIT_SHIFT 0x13UL +#define CC_HOST_IRR_DSCRPTR_WATERMARK_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_7_INT_BIT_SHIFT 0x14UL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_7_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_AXIM_COMP_INT_BIT_SHIFT 0x17UL +#define CC_HOST_IRR_AXIM_COMP_INT_BIT_SIZE 0x1UL +#define CC_HOST_SEP_SRAM_THRESHOLD_REG_OFFSET 0xA10UL +#define CC_HOST_SEP_SRAM_THRESHOLD_VALUE_BIT_SHIFT 0x0UL +#define CC_HOST_SEP_SRAM_THRESHOLD_VALUE_BIT_SIZE 0xCUL + +/* IMR */ +#define CC_HOST_IMR_REG_OFFSET 0x0A04UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_0_MASK_BIT_SHIFT 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_0_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_DSCRPTR_COMPLETION_MASK_BIT_SHIFT 0x2UL +#define CC_HOST_IMR_DSCRPTR_COMPLETION_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_1_MASK_BIT_SHIFT 0x3UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_1_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_2_MASK_BIT_SHIFT 0x4UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_2_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_3_MASK_BIT_SHIFT 0x5UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_3_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_4_MASK_BIT_SHIFT 0x6UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_4_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_5_MASK_BIT_SHIFT 0x7UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_5_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_AXI_ERR_MASK_BIT_SHIFT 0x8UL +#define CC_HOST_IMR_AXI_ERR_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_6_MASK_BIT_SHIFT 0x9UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_6_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_7_MASK_BIT_SHIFT 0xAUL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_7_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_GPR0_BIT_SHIFT 0xBUL +#define CC_HOST_IMR_GPR0_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_0_MASK_BIT_SHIFT 0xCUL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_0_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_1_MASK_BIT_SHIFT 0xDUL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_1_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_2_MASK_BIT_SHIFT 0xEUL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_2_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_3_MASK_BIT_SHIFT 0xFUL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_3_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_4_MASK_BIT_SHIFT 0x10UL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_4_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_5_MASK_BIT_SHIFT 0x11UL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_5_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_6_MASK_BIT_SHIFT 0x12UL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_6_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_DSCRPTR_WATERMARK_MASK0_BIT_SHIFT 0x13UL +#define CC_HOST_IMR_DSCRPTR_WATERMARK_MASK0_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_7_MASK_BIT_SHIFT 0x14UL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_7_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_AXIM_COMP_INT_MASK_BIT_SHIFT 0x17UL +#define CC_HOST_IMR_AXIM_COMP_INT_MASK_BIT_SIZE 0x1UL + +/* ICR */ +#define CC_HOST_ICR_REG_OFFSET 0xA08UL +#define CC_HOST_ICR_DSCRPTR_COMPLETION_BIT_SHIFT 0x2UL +#define CC_HOST_ICR_DSCRPTR_COMPLETION_BIT_SIZE 0x1UL +#define CC_HOST_ICR_AXI_ERR_CLEAR_BIT_SHIFT 0x8UL +#define CC_HOST_ICR_AXI_ERR_CLEAR_BIT_SIZE 0x1UL +#define CC_HOST_ICR_GPR_INT_CLEAR_BIT_SHIFT 0xBUL +#define CC_HOST_ICR_GPR_INT_CLEAR_BIT_SIZE 0x1UL +#define CC_HOST_ICR_DSCRPTR_WATERMARK_QUEUE0_CLEAR_BIT_SHIFT 0x13UL +#define CC_HOST_ICR_DSCRPTR_WATERMARK_QUEUE0_CLEAR_BIT_SIZE 0x1UL +#define CC_HOST_ICR_AXIM_COMP_INT_CLEAR_BIT_SHIFT 0x17UL +#define CC_HOST_ICR_AXIM_COMP_INT_CLEAR_BIT_SIZE 0x1UL +#define CC_NVM_IS_IDLE_REG_OFFSET 0x0A10UL +#define CC_NVM_IS_IDLE_VALUE_BIT_SHIFT 0x0UL +#define CC_NVM_IS_IDLE_VALUE_BIT_SIZE 0x1UL +#define CC_SECURITY_DISABLED_REG_OFFSET 0x0A1CUL +#define CC_SECURITY_DISABLED_VALUE_BIT_SHIFT 0x0UL +#define CC_SECURITY_DISABLED_VALUE_BIT_SIZE 0x1UL +#define CC_HOST_SIGNATURE_712_REG_OFFSET 0xA24UL +#define CC_HOST_SIGNATURE_630_REG_OFFSET 0xAC8UL +#define CC_HOST_SIGNATURE_VALUE_BIT_SHIFT 0x0UL +#define CC_HOST_SIGNATURE_VALUE_BIT_SIZE 0x20UL +#define CC_HOST_BOOT_REG_OFFSET 0xA28UL +#define CC_HOST_BOOT_SYNTHESIS_CONFIG_BIT_SHIFT 0x0UL +#define CC_HOST_BOOT_SYNTHESIS_CONFIG_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_LARGE_RKEK_LOCAL_BIT_SHIFT 0x1UL +#define CC_HOST_BOOT_LARGE_RKEK_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_HASH_IN_FUSES_LOCAL_BIT_SHIFT 0x2UL +#define CC_HOST_BOOT_HASH_IN_FUSES_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_EXT_MEM_SECURED_LOCAL_BIT_SHIFT 0x3UL +#define CC_HOST_BOOT_EXT_MEM_SECURED_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_RKEK_ECC_EXISTS_LOCAL_N_BIT_SHIFT 0x5UL +#define CC_HOST_BOOT_RKEK_ECC_EXISTS_LOCAL_N_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_SRAM_SIZE_LOCAL_BIT_SHIFT 0x6UL +#define CC_HOST_BOOT_SRAM_SIZE_LOCAL_BIT_SIZE 0x3UL +#define CC_HOST_BOOT_DSCRPTR_EXISTS_LOCAL_BIT_SHIFT 0x9UL +#define CC_HOST_BOOT_DSCRPTR_EXISTS_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_PAU_EXISTS_LOCAL_BIT_SHIFT 0xAUL +#define CC_HOST_BOOT_PAU_EXISTS_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_RNG_EXISTS_LOCAL_BIT_SHIFT 0xBUL +#define CC_HOST_BOOT_RNG_EXISTS_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_PKA_EXISTS_LOCAL_BIT_SHIFT 0xCUL +#define CC_HOST_BOOT_PKA_EXISTS_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_RC4_EXISTS_LOCAL_BIT_SHIFT 0xDUL +#define CC_HOST_BOOT_RC4_EXISTS_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_SHA_512_PRSNT_LOCAL_BIT_SHIFT 0xEUL +#define CC_HOST_BOOT_SHA_512_PRSNT_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_SHA_256_PRSNT_LOCAL_BIT_SHIFT 0xFUL +#define CC_HOST_BOOT_SHA_256_PRSNT_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_MD5_PRSNT_LOCAL_BIT_SHIFT 0x10UL +#define CC_HOST_BOOT_MD5_PRSNT_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_HASH_EXISTS_LOCAL_BIT_SHIFT 0x11UL +#define CC_HOST_BOOT_HASH_EXISTS_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_C2_EXISTS_LOCAL_BIT_SHIFT 0x12UL +#define CC_HOST_BOOT_C2_EXISTS_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_DES_EXISTS_LOCAL_BIT_SHIFT 0x13UL +#define CC_HOST_BOOT_DES_EXISTS_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_AES_XCBC_MAC_EXISTS_LOCAL_BIT_SHIFT 0x14UL +#define CC_HOST_BOOT_AES_XCBC_MAC_EXISTS_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_AES_CMAC_EXISTS_LOCAL_BIT_SHIFT 0x15UL +#define CC_HOST_BOOT_AES_CMAC_EXISTS_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_AES_CCM_EXISTS_LOCAL_BIT_SHIFT 0x16UL +#define CC_HOST_BOOT_AES_CCM_EXISTS_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_AES_XEX_HW_T_CALC_LOCAL_BIT_SHIFT 0x17UL +#define CC_HOST_BOOT_AES_XEX_HW_T_CALC_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_AES_XEX_EXISTS_LOCAL_BIT_SHIFT 0x18UL +#define CC_HOST_BOOT_AES_XEX_EXISTS_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_CTR_EXISTS_LOCAL_BIT_SHIFT 0x19UL +#define CC_HOST_BOOT_CTR_EXISTS_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_AES_DIN_BYTE_RESOLUTION_LOCAL_BIT_SHIFT 0x1AUL +#define CC_HOST_BOOT_AES_DIN_BYTE_RESOLUTION_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_TUNNELING_ENB_LOCAL_BIT_SHIFT 0x1BUL +#define CC_HOST_BOOT_TUNNELING_ENB_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_SUPPORT_256_192_KEY_LOCAL_BIT_SHIFT 0x1CUL +#define CC_HOST_BOOT_SUPPORT_256_192_KEY_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_ONLY_ENCRYPT_LOCAL_BIT_SHIFT 0x1DUL +#define CC_HOST_BOOT_ONLY_ENCRYPT_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_BOOT_AES_EXISTS_LOCAL_BIT_SHIFT 0x1EUL +#define CC_HOST_BOOT_AES_EXISTS_LOCAL_BIT_SIZE 0x1UL +#define CC_HOST_VERSION_712_REG_OFFSET 0xA40UL +#define CC_HOST_VERSION_630_REG_OFFSET 0xAD8UL +#define CC_HOST_VERSION_VALUE_BIT_SHIFT 0x0UL +#define CC_HOST_VERSION_VALUE_BIT_SIZE 0x20UL +#define CC_HOST_KFDE0_VALID_REG_OFFSET 0xA60UL +#define CC_HOST_KFDE0_VALID_VALUE_BIT_SHIFT 0x0UL +#define CC_HOST_KFDE0_VALID_VALUE_BIT_SIZE 0x1UL +#define CC_HOST_KFDE1_VALID_REG_OFFSET 0xA64UL +#define CC_HOST_KFDE1_VALID_VALUE_BIT_SHIFT 0x0UL +#define CC_HOST_KFDE1_VALID_VALUE_BIT_SIZE 0x1UL +#define CC_HOST_KFDE2_VALID_REG_OFFSET 0xA68UL +#define CC_HOST_KFDE2_VALID_VALUE_BIT_SHIFT 0x0UL +#define CC_HOST_KFDE2_VALID_VALUE_BIT_SIZE 0x1UL +#define CC_HOST_KFDE3_VALID_REG_OFFSET 0xA6CUL +#define CC_HOST_KFDE3_VALID_VALUE_BIT_SHIFT 0x0UL +#define CC_HOST_KFDE3_VALID_VALUE_BIT_SIZE 0x1UL +#define CC_HOST_GPR0_REG_OFFSET 0xA70UL +#define CC_HOST_GPR0_VALUE_BIT_SHIFT 0x0UL +#define CC_HOST_GPR0_VALUE_BIT_SIZE 0x20UL +#define CC_GPR_HOST_REG_OFFSET 0xA74UL +#define CC_GPR_HOST_VALUE_BIT_SHIFT 0x0UL +#define CC_GPR_HOST_VALUE_BIT_SIZE 0x20UL +#define CC_HOST_POWER_DOWN_EN_REG_OFFSET 0xA78UL +#define CC_HOST_POWER_DOWN_EN_VALUE_BIT_SHIFT 0x0UL +#define CC_HOST_POWER_DOWN_EN_VALUE_BIT_SIZE 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_REG_OFFSET 0x0A7CUL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_AES_ENGINE_BIT_SHIFT 0x0UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_AES_ENGINE_BIT_SIZE 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_AES_MAC_ENGINE_BIT_SHIFT 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_AES_MAC_ENGINE_BIT_SIZE 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_GHASH_ENGINE_BIT_SHIFT 0x2UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_GHASH_ENGINE_BIT_SIZE 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_DES_ENGINE_BIT_SHIFT 0x3UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_DES_ENGINE_BIT_SIZE 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_HASH_ENGINE_BIT_SHIFT 0x4UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_HASH_ENGINE_BIT_SIZE 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_SM3_ENGINE_BIT_SHIFT 0x5UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_SM3_ENGINE_BIT_SIZE 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_SM4_ENGINE_BIT_SHIFT 0x6UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_SM4_ENGINE_BIT_SIZE 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_OTP_DISCONNECTED_BIT_SHIFT 0x7UL +#define CC_HOST_REMOVE_INPUT_PINS_OTP_DISCONNECTED_BIT_SIZE 0x1UL +// -------------------------------------- +// BLOCK: ID_REGISTERS +// -------------------------------------- +#define CC_PERIPHERAL_ID_4_REG_OFFSET 0x0FD0UL +#define CC_PERIPHERAL_ID_4_VALUE_BIT_SHIFT 0x0UL +#define CC_PERIPHERAL_ID_4_VALUE_BIT_SIZE 0x4UL +#define CC_PIDRESERVED0_REG_OFFSET 0x0FD4UL +#define CC_PIDRESERVED1_REG_OFFSET 0x0FD8UL +#define CC_PIDRESERVED2_REG_OFFSET 0x0FDCUL +#define CC_PERIPHERAL_ID_0_REG_OFFSET 0x0FE0UL +#define CC_PERIPHERAL_ID_0_VALUE_BIT_SHIFT 0x0UL +#define CC_PERIPHERAL_ID_0_VALUE_BIT_SIZE 0x8UL +#define CC_PERIPHERAL_ID_1_REG_OFFSET 0x0FE4UL +#define CC_PERIPHERAL_ID_1_PART_1_BIT_SHIFT 0x0UL +#define CC_PERIPHERAL_ID_1_PART_1_BIT_SIZE 0x4UL +#define CC_PERIPHERAL_ID_1_DES_0_JEP106_BIT_SHIFT 0x4UL +#define CC_PERIPHERAL_ID_1_DES_0_JEP106_BIT_SIZE 0x4UL +#define CC_PERIPHERAL_ID_2_REG_OFFSET 0x0FE8UL +#define CC_PERIPHERAL_ID_2_DES_1_JEP106_BIT_SHIFT 0x0UL +#define CC_PERIPHERAL_ID_2_DES_1_JEP106_BIT_SIZE 0x3UL +#define CC_PERIPHERAL_ID_2_JEDEC_BIT_SHIFT 0x3UL +#define CC_PERIPHERAL_ID_2_JEDEC_BIT_SIZE 0x1UL +#define CC_PERIPHERAL_ID_2_REVISION_BIT_SHIFT 0x4UL +#define CC_PERIPHERAL_ID_2_REVISION_BIT_SIZE 0x4UL +#define CC_PERIPHERAL_ID_3_REG_OFFSET 0x0FECUL +#define CC_PERIPHERAL_ID_3_CMOD_BIT_SHIFT 0x0UL +#define CC_PERIPHERAL_ID_3_CMOD_BIT_SIZE 0x4UL +#define CC_PERIPHERAL_ID_3_REVAND_BIT_SHIFT 0x4UL +#define CC_PERIPHERAL_ID_3_REVAND_BIT_SIZE 0x4UL +#define CC_COMPONENT_ID_0_REG_OFFSET 0x0FF0UL +#define CC_COMPONENT_ID_0_VALUE_BIT_SHIFT 0x0UL +#define CC_COMPONENT_ID_0_VALUE_BIT_SIZE 0x8UL +#define CC_COMPONENT_ID_1_REG_OFFSET 0x0FF4UL +#define CC_COMPONENT_ID_1_PRMBL_1_BIT_SHIFT 0x0UL +#define CC_COMPONENT_ID_1_PRMBL_1_BIT_SIZE 0x4UL +#define CC_COMPONENT_ID_1_CLASS_BIT_SHIFT 0x4UL +#define CC_COMPONENT_ID_1_CLASS_BIT_SIZE 0x4UL +#define CC_COMPONENT_ID_2_REG_OFFSET 0x0FF8UL +#define CC_COMPONENT_ID_2_VALUE_BIT_SHIFT 0x0UL +#define CC_COMPONENT_ID_2_VALUE_BIT_SIZE 0x8UL +#define CC_COMPONENT_ID_3_REG_OFFSET 0x0FFCUL +#define CC_COMPONENT_ID_3_VALUE_BIT_SHIFT 0x0UL +#define CC_COMPONENT_ID_3_VALUE_BIT_SIZE 0x8UL +// -------------------------------------- +// BLOCK: HOST_SRAM +// -------------------------------------- +#define CC_SRAM_DATA_REG_OFFSET 0xF00UL +#define CC_SRAM_DATA_VALUE_BIT_SHIFT 0x0UL +#define CC_SRAM_DATA_VALUE_BIT_SIZE 0x20UL +#define CC_SRAM_ADDR_REG_OFFSET 0xF04UL +#define CC_SRAM_ADDR_VALUE_BIT_SHIFT 0x0UL +#define CC_SRAM_ADDR_VALUE_BIT_SIZE 0xFUL +#define CC_SRAM_DATA_READY_REG_OFFSET 0xF08UL +#define CC_SRAM_DATA_READY_VALUE_BIT_SHIFT 0x0UL +#define CC_SRAM_DATA_READY_VALUE_BIT_SIZE 0x1UL + +#endif //__CC_HOST_H__ diff --git a/drivers/crypto/ccree/cc_hw_queue_defs.h b/drivers/crypto/ccree/cc_hw_queue_defs.h new file mode 100644 index 000000000..15df58c66 --- /dev/null +++ b/drivers/crypto/ccree/cc_hw_queue_defs.h @@ -0,0 +1,633 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ + +#ifndef __CC_HW_QUEUE_DEFS_H__ +#define __CC_HW_QUEUE_DEFS_H__ + +#include + +#include "cc_kernel_regs.h" +#include + +/****************************************************************************** + * DEFINITIONS + ******************************************************************************/ + +#define HW_DESC_SIZE_WORDS 6 +/* Define max. available slots in HW queue */ +#define HW_QUEUE_SLOTS_MAX 15 + +#define CC_REG_LOW(name) (name ## _BIT_SHIFT) +#define CC_REG_HIGH(name) (CC_REG_LOW(name) + name ## _BIT_SIZE - 1) +#define CC_GENMASK(name) GENMASK(CC_REG_HIGH(name), CC_REG_LOW(name)) + +#define CC_HWQ_GENMASK(word, field) \ + CC_GENMASK(CC_DSCRPTR_QUEUE_WORD ## word ## _ ## field) + +#define WORD0_VALUE CC_HWQ_GENMASK(0, VALUE) +#define WORD0_CPP_CIPHER_MODE CC_HWQ_GENMASK(0, CPP_CIPHER_MODE) +#define WORD1_DIN_CONST_VALUE CC_HWQ_GENMASK(1, DIN_CONST_VALUE) +#define WORD1_DIN_DMA_MODE CC_HWQ_GENMASK(1, DIN_DMA_MODE) +#define WORD1_DIN_SIZE CC_HWQ_GENMASK(1, DIN_SIZE) +#define WORD1_NOT_LAST CC_HWQ_GENMASK(1, NOT_LAST) +#define WORD1_NS_BIT CC_HWQ_GENMASK(1, NS_BIT) +#define WORD1_LOCK_QUEUE CC_HWQ_GENMASK(1, LOCK_QUEUE) +#define WORD2_VALUE CC_HWQ_GENMASK(2, VALUE) +#define WORD3_DOUT_DMA_MODE CC_HWQ_GENMASK(3, DOUT_DMA_MODE) +#define WORD3_DOUT_LAST_IND CC_HWQ_GENMASK(3, DOUT_LAST_IND) +#define WORD3_DOUT_SIZE CC_HWQ_GENMASK(3, DOUT_SIZE) +#define WORD3_HASH_XOR_BIT CC_HWQ_GENMASK(3, HASH_XOR_BIT) +#define WORD3_NS_BIT CC_HWQ_GENMASK(3, NS_BIT) +#define WORD3_QUEUE_LAST_IND CC_HWQ_GENMASK(3, QUEUE_LAST_IND) +#define WORD4_ACK_NEEDED CC_HWQ_GENMASK(4, ACK_NEEDED) +#define WORD4_AES_SEL_N_HASH CC_HWQ_GENMASK(4, AES_SEL_N_HASH) +#define WORD4_AES_XOR_CRYPTO_KEY CC_HWQ_GENMASK(4, AES_XOR_CRYPTO_KEY) +#define WORD4_BYTES_SWAP CC_HWQ_GENMASK(4, BYTES_SWAP) +#define WORD4_CIPHER_CONF0 CC_HWQ_GENMASK(4, CIPHER_CONF0) +#define WORD4_CIPHER_CONF1 CC_HWQ_GENMASK(4, CIPHER_CONF1) +#define WORD4_CIPHER_CONF2 CC_HWQ_GENMASK(4, CIPHER_CONF2) +#define WORD4_CIPHER_DO CC_HWQ_GENMASK(4, CIPHER_DO) +#define WORD4_CIPHER_MODE CC_HWQ_GENMASK(4, CIPHER_MODE) +#define WORD4_CMAC_SIZE0 CC_HWQ_GENMASK(4, CMAC_SIZE0) +#define WORD4_DATA_FLOW_MODE CC_HWQ_GENMASK(4, DATA_FLOW_MODE) +#define WORD4_KEY_SIZE CC_HWQ_GENMASK(4, KEY_SIZE) +#define WORD4_SETUP_OPERATION CC_HWQ_GENMASK(4, SETUP_OPERATION) +#define WORD5_DIN_ADDR_HIGH CC_HWQ_GENMASK(5, DIN_ADDR_HIGH) +#define WORD5_DOUT_ADDR_HIGH CC_HWQ_GENMASK(5, DOUT_ADDR_HIGH) + +/****************************************************************************** + * TYPE DEFINITIONS + ******************************************************************************/ + +struct cc_hw_desc { + union { + u32 word[HW_DESC_SIZE_WORDS]; + u16 hword[HW_DESC_SIZE_WORDS * 2]; + }; +}; + +enum cc_axi_sec { + AXI_SECURE = 0, + AXI_NOT_SECURE = 1 +}; + +enum cc_desc_direction { + DESC_DIRECTION_ILLEGAL = -1, + DESC_DIRECTION_ENCRYPT_ENCRYPT = 0, + DESC_DIRECTION_DECRYPT_DECRYPT = 1, + DESC_DIRECTION_DECRYPT_ENCRYPT = 3, + DESC_DIRECTION_END = S32_MAX, +}; + +enum cc_dma_mode { + DMA_MODE_NULL = -1, + NO_DMA = 0, + DMA_SRAM = 1, + DMA_DLLI = 2, + DMA_MLLI = 3, + DMA_MODE_END = S32_MAX, +}; + +enum cc_flow_mode { + FLOW_MODE_NULL = -1, + /* data flows */ + BYPASS = 0, + DIN_AES_DOUT = 1, + AES_to_HASH = 2, + AES_and_HASH = 3, + DIN_DES_DOUT = 4, + DES_to_HASH = 5, + DES_and_HASH = 6, + DIN_HASH = 7, + DIN_HASH_and_BYPASS = 8, + AESMAC_and_BYPASS = 9, + AES_to_HASH_and_DOUT = 10, + DIN_RC4_DOUT = 11, + DES_to_HASH_and_DOUT = 12, + AES_to_AES_to_HASH_and_DOUT = 13, + AES_to_AES_to_HASH = 14, + AES_to_HASH_and_AES = 15, + DIN_SM4_DOUT = 16, + DIN_AES_AESMAC = 17, + HASH_to_DOUT = 18, + /* setup flows */ + S_DIN_to_AES = 32, + S_DIN_to_AES2 = 33, + S_DIN_to_DES = 34, + S_DIN_to_RC4 = 35, + S_DIN_to_SM4 = 36, + S_DIN_to_HASH = 37, + S_AES_to_DOUT = 38, + S_AES2_to_DOUT = 39, + S_SM4_to_DOUT = 40, + S_RC4_to_DOUT = 41, + S_DES_to_DOUT = 42, + S_HASH_to_DOUT = 43, + SET_FLOW_ID = 44, + FLOW_MODE_END = S32_MAX, +}; + +enum cc_setup_op { + SETUP_LOAD_NOP = 0, + SETUP_LOAD_STATE0 = 1, + SETUP_LOAD_STATE1 = 2, + SETUP_LOAD_STATE2 = 3, + SETUP_LOAD_KEY0 = 4, + SETUP_LOAD_XEX_KEY = 5, + SETUP_WRITE_STATE0 = 8, + SETUP_WRITE_STATE1 = 9, + SETUP_WRITE_STATE2 = 10, + SETUP_WRITE_STATE3 = 11, + SETUP_OP_END = S32_MAX, +}; + +enum cc_hash_conf_pad { + HASH_PADDING_DISABLED = 0, + HASH_PADDING_ENABLED = 1, + HASH_DIGEST_RESULT_LITTLE_ENDIAN = 2, + HASH_CONFIG1_PADDING_RESERVE32 = S32_MAX, +}; + +enum cc_aes_mac_selector { + AES_SK = 1, + AES_CMAC_INIT = 2, + AES_CMAC_SIZE0 = 3, + AES_MAC_END = S32_MAX, +}; + +#define HW_KEY_MASK_CIPHER_DO 0x3 +#define HW_KEY_SHIFT_CIPHER_CFG2 2 + +/* HwCryptoKey[1:0] is mapped to cipher_do[1:0] */ +/* HwCryptoKey[2:3] is mapped to cipher_config2[1:0] */ +enum cc_hw_crypto_key { + USER_KEY = 0, /* 0x0000 */ + ROOT_KEY = 1, /* 0x0001 */ + PROVISIONING_KEY = 2, /* 0x0010 */ /* ==KCP */ + SESSION_KEY = 3, /* 0x0011 */ + RESERVED_KEY = 4, /* NA */ + PLATFORM_KEY = 5, /* 0x0101 */ + CUSTOMER_KEY = 6, /* 0x0110 */ + KFDE0_KEY = 7, /* 0x0111 */ + KFDE1_KEY = 9, /* 0x1001 */ + KFDE2_KEY = 10, /* 0x1010 */ + KFDE3_KEY = 11, /* 0x1011 */ + END_OF_KEYS = S32_MAX, +}; + +#define CC_NUM_HW_KEY_SLOTS 4 +#define CC_FIRST_HW_KEY_SLOT 0 +#define CC_LAST_HW_KEY_SLOT (CC_FIRST_HW_KEY_SLOT + CC_NUM_HW_KEY_SLOTS - 1) + +#define CC_NUM_CPP_KEY_SLOTS 8 +#define CC_FIRST_CPP_KEY_SLOT 16 +#define CC_LAST_CPP_KEY_SLOT (CC_FIRST_CPP_KEY_SLOT + \ + CC_NUM_CPP_KEY_SLOTS - 1) + +enum cc_hw_aes_key_size { + AES_128_KEY = 0, + AES_192_KEY = 1, + AES_256_KEY = 2, + END_OF_AES_KEYS = S32_MAX, +}; + +enum cc_hash_cipher_pad { + DO_NOT_PAD = 0, + DO_PAD = 1, + HASH_CIPHER_DO_PADDING_RESERVE32 = S32_MAX, +}; + +#define CC_CPP_DIN_ADDR 0xFF00FF00UL +#define CC_CPP_DIN_SIZE 0xFF00FFUL + +/*****************************/ +/* Descriptor packing macros */ +/*****************************/ + +/** + * hw_desc_init() - Init a HW descriptor struct + * @pdesc: pointer to HW descriptor struct + */ +static inline void hw_desc_init(struct cc_hw_desc *pdesc) +{ + memset(pdesc, 0, sizeof(struct cc_hw_desc)); +} + +/** + * set_queue_last_ind_bit() - Indicate the end of current HW descriptors flow + * and release the HW engines. + * + * @pdesc: Pointer to HW descriptor struct + */ +static inline void set_queue_last_ind_bit(struct cc_hw_desc *pdesc) +{ + pdesc->word[3] |= FIELD_PREP(WORD3_QUEUE_LAST_IND, 1); +} + +/** + * set_din_type() - Set the DIN field of a HW descriptor + * + * @pdesc: Pointer to HW descriptor struct + * @dma_mode: The DMA mode: NO_DMA, SRAM, DLLI, MLLI, CONSTANT + * @addr: DIN address + * @size: Data size in bytes + * @axi_sec: AXI secure bit + */ +static inline void set_din_type(struct cc_hw_desc *pdesc, + enum cc_dma_mode dma_mode, dma_addr_t addr, + u32 size, enum cc_axi_sec axi_sec) +{ + pdesc->word[0] = lower_32_bits(addr); +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + pdesc->word[5] |= FIELD_PREP(WORD5_DIN_ADDR_HIGH, upper_32_bits(addr)); +#endif + pdesc->word[1] |= FIELD_PREP(WORD1_DIN_DMA_MODE, dma_mode) | + FIELD_PREP(WORD1_DIN_SIZE, size) | + FIELD_PREP(WORD1_NS_BIT, axi_sec); +} + +/** + * set_din_no_dma() - Set the DIN field of a HW descriptor to NO DMA mode. + * Used for NOP descriptor, register patches and other special modes. + * + * @pdesc: Pointer to HW descriptor struct + * @addr: DIN address + * @size: Data size in bytes + */ +static inline void set_din_no_dma(struct cc_hw_desc *pdesc, u32 addr, u32 size) +{ + pdesc->word[0] = addr; + pdesc->word[1] |= FIELD_PREP(WORD1_DIN_SIZE, size); +} + +/** + * set_cpp_crypto_key() - Setup the special CPP descriptor + * + * @pdesc: Pointer to HW descriptor struct + * @slot: Slot number + */ +static inline void set_cpp_crypto_key(struct cc_hw_desc *pdesc, u8 slot) +{ + pdesc->word[0] |= CC_CPP_DIN_ADDR; + + pdesc->word[1] |= FIELD_PREP(WORD1_DIN_SIZE, CC_CPP_DIN_SIZE); + pdesc->word[1] |= FIELD_PREP(WORD1_LOCK_QUEUE, 1); + + pdesc->word[4] |= FIELD_PREP(WORD4_SETUP_OPERATION, slot); +} + +/** + * set_din_sram() - Set the DIN field of a HW descriptor to SRAM mode. + * Note: No need to check SRAM alignment since host requests do not use SRAM and + * the adaptor will enforce alignment checks. + * + * @pdesc: Pointer to HW descriptor struct + * @addr: DIN address + * @size: Data size in bytes + */ +static inline void set_din_sram(struct cc_hw_desc *pdesc, u32 addr, u32 size) +{ + pdesc->word[0] = addr; + pdesc->word[1] |= FIELD_PREP(WORD1_DIN_SIZE, size) | + FIELD_PREP(WORD1_DIN_DMA_MODE, DMA_SRAM); +} + +/** + * set_din_const() - Set the DIN field of a HW descriptor to CONST mode + * + * @pdesc: Pointer to HW descriptor struct + * @val: DIN const value + * @size: Data size in bytes + */ +static inline void set_din_const(struct cc_hw_desc *pdesc, u32 val, u32 size) +{ + pdesc->word[0] = val; + pdesc->word[1] |= FIELD_PREP(WORD1_DIN_CONST_VALUE, 1) | + FIELD_PREP(WORD1_DIN_DMA_MODE, DMA_SRAM) | + FIELD_PREP(WORD1_DIN_SIZE, size); +} + +/** + * set_din_not_last_indication() - Set the DIN not last input data indicator + * + * @pdesc: Pointer to HW descriptor struct + */ +static inline void set_din_not_last_indication(struct cc_hw_desc *pdesc) +{ + pdesc->word[1] |= FIELD_PREP(WORD1_NOT_LAST, 1); +} + +/** + * set_dout_type() - Set the DOUT field of a HW descriptor + * + * @pdesc: Pointer to HW descriptor struct + * @dma_mode: The DMA mode: NO_DMA, SRAM, DLLI, MLLI, CONSTANT + * @addr: DOUT address + * @size: Data size in bytes + * @axi_sec: AXI secure bit + */ +static inline void set_dout_type(struct cc_hw_desc *pdesc, + enum cc_dma_mode dma_mode, dma_addr_t addr, + u32 size, enum cc_axi_sec axi_sec) +{ + pdesc->word[2] = lower_32_bits(addr); +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + pdesc->word[5] |= FIELD_PREP(WORD5_DOUT_ADDR_HIGH, upper_32_bits(addr)); +#endif + pdesc->word[3] |= FIELD_PREP(WORD3_DOUT_DMA_MODE, dma_mode) | + FIELD_PREP(WORD3_DOUT_SIZE, size) | + FIELD_PREP(WORD3_NS_BIT, axi_sec); +} + +/** + * set_dout_dlli() - Set the DOUT field of a HW descriptor to DLLI type + * The LAST INDICATION is provided by the user + * + * @pdesc: Pointer to HW descriptor struct + * @addr: DOUT address + * @size: Data size in bytes + * @axi_sec: AXI secure bit + * @last_ind: The last indication bit + */ +static inline void set_dout_dlli(struct cc_hw_desc *pdesc, dma_addr_t addr, + u32 size, enum cc_axi_sec axi_sec, + u32 last_ind) +{ + set_dout_type(pdesc, DMA_DLLI, addr, size, axi_sec); + pdesc->word[3] |= FIELD_PREP(WORD3_DOUT_LAST_IND, last_ind); +} + +/** + * set_dout_mlli() - Set the DOUT field of a HW descriptor to MLLI type + * The LAST INDICATION is provided by the user + * + * @pdesc: Pointer to HW descriptor struct + * @addr: DOUT address + * @size: Data size in bytes + * @axi_sec: AXI secure bit + * @last_ind: The last indication bit + */ +static inline void set_dout_mlli(struct cc_hw_desc *pdesc, u32 addr, u32 size, + enum cc_axi_sec axi_sec, bool last_ind) +{ + set_dout_type(pdesc, DMA_MLLI, addr, size, axi_sec); + pdesc->word[3] |= FIELD_PREP(WORD3_DOUT_LAST_IND, last_ind); +} + +/** + * set_dout_no_dma() - Set the DOUT field of a HW descriptor to NO DMA mode. + * Used for NOP descriptor, register patches and other special modes. + * + * @pdesc: pointer to HW descriptor struct + * @addr: DOUT address + * @size: Data size in bytes + * @write_enable: Enables a write operation to a register + */ +static inline void set_dout_no_dma(struct cc_hw_desc *pdesc, u32 addr, + u32 size, bool write_enable) +{ + pdesc->word[2] = addr; + pdesc->word[3] |= FIELD_PREP(WORD3_DOUT_SIZE, size) | + FIELD_PREP(WORD3_DOUT_LAST_IND, write_enable); +} + +/** + * set_xor_val() - Set the word for the XOR operation. + * + * @pdesc: Pointer to HW descriptor struct + * @val: XOR data value + */ +static inline void set_xor_val(struct cc_hw_desc *pdesc, u32 val) +{ + pdesc->word[2] = val; +} + +/** + * set_xor_active() - Set the XOR indicator bit in the descriptor + * + * @pdesc: Pointer to HW descriptor struct + */ +static inline void set_xor_active(struct cc_hw_desc *pdesc) +{ + pdesc->word[3] |= FIELD_PREP(WORD3_HASH_XOR_BIT, 1); +} + +/** + * set_aes_not_hash_mode() - Select the AES engine instead of HASH engine when + * setting up combined mode with AES XCBC MAC + * + * @pdesc: Pointer to HW descriptor struct + */ +static inline void set_aes_not_hash_mode(struct cc_hw_desc *pdesc) +{ + pdesc->word[4] |= FIELD_PREP(WORD4_AES_SEL_N_HASH, 1); +} + +/** + * set_aes_xor_crypto_key() - Set aes xor crypto key, which in some scenarios + * selects the SM3 engine + * + * @pdesc: Pointer to HW descriptor struct + */ +static inline void set_aes_xor_crypto_key(struct cc_hw_desc *pdesc) +{ + pdesc->word[4] |= FIELD_PREP(WORD4_AES_XOR_CRYPTO_KEY, 1); +} + +/** + * set_dout_sram() - Set the DOUT field of a HW descriptor to SRAM mode + * Note: No need to check SRAM alignment since host requests do not use SRAM and + * the adaptor will enforce alignment checks. + * + * @pdesc: Pointer to HW descriptor struct + * @addr: DOUT address + * @size: Data size in bytes + */ +static inline void set_dout_sram(struct cc_hw_desc *pdesc, u32 addr, u32 size) +{ + pdesc->word[2] = addr; + pdesc->word[3] |= FIELD_PREP(WORD3_DOUT_DMA_MODE, DMA_SRAM) | + FIELD_PREP(WORD3_DOUT_SIZE, size); +} + +/** + * set_xex_data_unit_size() - Set the data unit size for XEX mode in + * data_out_addr[15:0] + * + * @pdesc: Pointer to HW descriptor struct + * @size: Data unit size for XEX mode + */ +static inline void set_xex_data_unit_size(struct cc_hw_desc *pdesc, u32 size) +{ + pdesc->word[2] = size; +} + +/** + * set_multi2_num_rounds() - Set the number of rounds for Multi2 in + * data_out_addr[15:0] + * + * @pdesc: Pointer to HW descriptor struct + * @num: Number of rounds for Multi2 + */ +static inline void set_multi2_num_rounds(struct cc_hw_desc *pdesc, u32 num) +{ + pdesc->word[2] = num; +} + +/** + * set_flow_mode() - Set the flow mode. + * + * @pdesc: Pointer to HW descriptor struct + * @mode: Any one of the modes defined in [CC7x-DESC] + */ +static inline void set_flow_mode(struct cc_hw_desc *pdesc, + enum cc_flow_mode mode) +{ + pdesc->word[4] |= FIELD_PREP(WORD4_DATA_FLOW_MODE, mode); +} + +/** + * set_cipher_mode() - Set the cipher mode. + * + * @pdesc: Pointer to HW descriptor struct + * @mode: Any one of the modes defined in [CC7x-DESC] + */ +static inline void set_cipher_mode(struct cc_hw_desc *pdesc, int mode) +{ + pdesc->word[4] |= FIELD_PREP(WORD4_CIPHER_MODE, mode); +} + +/** + * set_hash_cipher_mode() - Set the cipher mode for hash algorithms. + * + * @pdesc: Pointer to HW descriptor struct + * @cipher_mode: Any one of the modes defined in [CC7x-DESC] + * @hash_mode: specifies which hash is being handled + */ +static inline void set_hash_cipher_mode(struct cc_hw_desc *pdesc, + enum drv_cipher_mode cipher_mode, + enum drv_hash_mode hash_mode) +{ + set_cipher_mode(pdesc, cipher_mode); + if (hash_mode == DRV_HASH_SM3) + set_aes_xor_crypto_key(pdesc); +} + +/** + * set_cipher_config0() - Set the cipher configuration fields. + * + * @pdesc: Pointer to HW descriptor struct + * @mode: Any one of the modes defined in [CC7x-DESC] + */ +static inline void set_cipher_config0(struct cc_hw_desc *pdesc, int mode) +{ + pdesc->word[4] |= FIELD_PREP(WORD4_CIPHER_CONF0, mode); +} + +/** + * set_cipher_config1() - Set the cipher configuration fields. + * + * @pdesc: Pointer to HW descriptor struct + * @config: Padding mode + */ +static inline void set_cipher_config1(struct cc_hw_desc *pdesc, + enum cc_hash_conf_pad config) +{ + pdesc->word[4] |= FIELD_PREP(WORD4_CIPHER_CONF1, config); +} + +/** + * set_hw_crypto_key() - Set HW key configuration fields. + * + * @pdesc: Pointer to HW descriptor struct + * @hw_key: The HW key slot asdefined in enum cc_hw_crypto_key + */ +static inline void set_hw_crypto_key(struct cc_hw_desc *pdesc, + enum cc_hw_crypto_key hw_key) +{ + pdesc->word[4] |= FIELD_PREP(WORD4_CIPHER_DO, + (hw_key & HW_KEY_MASK_CIPHER_DO)) | + FIELD_PREP(WORD4_CIPHER_CONF2, + (hw_key >> HW_KEY_SHIFT_CIPHER_CFG2)); +} + +/** + * set_bytes_swap() - Set byte order of all setup-finalize descriptors. + * + * @pdesc: Pointer to HW descriptor struct + * @config: True to enable byte swapping + */ +static inline void set_bytes_swap(struct cc_hw_desc *pdesc, bool config) +{ + pdesc->word[4] |= FIELD_PREP(WORD4_BYTES_SWAP, config); +} + +/** + * set_cmac_size0_mode() - Set CMAC_SIZE0 mode. + * + * @pdesc: Pointer to HW descriptor struct + */ +static inline void set_cmac_size0_mode(struct cc_hw_desc *pdesc) +{ + pdesc->word[4] |= FIELD_PREP(WORD4_CMAC_SIZE0, 1); +} + +/** + * set_key_size() - Set key size descriptor field. + * + * @pdesc: Pointer to HW descriptor struct + * @size: Key size in bytes (NOT size code) + */ +static inline void set_key_size(struct cc_hw_desc *pdesc, u32 size) +{ + pdesc->word[4] |= FIELD_PREP(WORD4_KEY_SIZE, size); +} + +/** + * set_key_size_aes() - Set AES key size. + * + * @pdesc: Pointer to HW descriptor struct + * @size: Key size in bytes (NOT size code) + */ +static inline void set_key_size_aes(struct cc_hw_desc *pdesc, u32 size) +{ + set_key_size(pdesc, ((size >> 3) - 2)); +} + +/** + * set_key_size_des() - Set DES key size. + * + * @pdesc: Pointer to HW descriptor struct + * @size: Key size in bytes (NOT size code) + */ +static inline void set_key_size_des(struct cc_hw_desc *pdesc, u32 size) +{ + set_key_size(pdesc, ((size >> 3) - 1)); +} + +/** + * set_setup_mode() - Set the descriptor setup mode + * + * @pdesc: Pointer to HW descriptor struct + * @mode: Any one of the setup modes defined in [CC7x-DESC] + */ +static inline void set_setup_mode(struct cc_hw_desc *pdesc, + enum cc_setup_op mode) +{ + pdesc->word[4] |= FIELD_PREP(WORD4_SETUP_OPERATION, mode); +} + +/** + * set_cipher_do() - Set the descriptor cipher DO + * + * @pdesc: Pointer to HW descriptor struct + * @config: Any one of the cipher do defined in [CC7x-DESC] + */ +static inline void set_cipher_do(struct cc_hw_desc *pdesc, + enum cc_hash_cipher_pad config) +{ + pdesc->word[4] |= FIELD_PREP(WORD4_CIPHER_DO, + (config & HW_KEY_MASK_CIPHER_DO)); +} + +#endif /*__CC_HW_QUEUE_DEFS_H__*/ diff --git a/drivers/crypto/ccree/cc_kernel_regs.h b/drivers/crypto/ccree/cc_kernel_regs.h new file mode 100644 index 000000000..582bae450 --- /dev/null +++ b/drivers/crypto/ccree/cc_kernel_regs.h @@ -0,0 +1,168 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ + +#ifndef __CC_CRYS_KERNEL_H__ +#define __CC_CRYS_KERNEL_H__ + +// -------------------------------------- +// BLOCK: DSCRPTR +// -------------------------------------- +#define CC_DSCRPTR_COMPLETION_COUNTER_REG_OFFSET 0xE00UL +#define CC_DSCRPTR_COMPLETION_COUNTER_COMPLETION_COUNTER_BIT_SHIFT 0x0UL +#define CC_DSCRPTR_COMPLETION_COUNTER_COMPLETION_COUNTER_BIT_SIZE 0x6UL +#define CC_DSCRPTR_COMPLETION_COUNTER_OVERFLOW_COUNTER_BIT_SHIFT 0x6UL +#define CC_DSCRPTR_COMPLETION_COUNTER_OVERFLOW_COUNTER_BIT_SIZE 0x1UL +#define CC_DSCRPTR_SW_RESET_REG_OFFSET 0xE40UL +#define CC_DSCRPTR_SW_RESET_VALUE_BIT_SHIFT 0x0UL +#define CC_DSCRPTR_SW_RESET_VALUE_BIT_SIZE 0x1UL +#define CC_DSCRPTR_QUEUE_SRAM_SIZE_REG_OFFSET 0xE60UL +#define CC_DSCRPTR_QUEUE_SRAM_SIZE_NUM_OF_DSCRPTR_BIT_SHIFT 0x0UL +#define CC_DSCRPTR_QUEUE_SRAM_SIZE_NUM_OF_DSCRPTR_BIT_SIZE 0xAUL +#define CC_DSCRPTR_QUEUE_SRAM_SIZE_DSCRPTR_SRAM_SIZE_BIT_SHIFT 0xAUL +#define CC_DSCRPTR_QUEUE_SRAM_SIZE_DSCRPTR_SRAM_SIZE_BIT_SIZE 0xCUL +#define CC_DSCRPTR_QUEUE_SRAM_SIZE_SRAM_SIZE_BIT_SHIFT 0x16UL +#define CC_DSCRPTR_QUEUE_SRAM_SIZE_SRAM_SIZE_BIT_SIZE 0x3UL +#define CC_DSCRPTR_SINGLE_ADDR_EN_REG_OFFSET 0xE64UL +#define CC_DSCRPTR_SINGLE_ADDR_EN_VALUE_BIT_SHIFT 0x0UL +#define CC_DSCRPTR_SINGLE_ADDR_EN_VALUE_BIT_SIZE 0x1UL +#define CC_DSCRPTR_MEASURE_CNTR_REG_OFFSET 0xE68UL +#define CC_DSCRPTR_MEASURE_CNTR_VALUE_BIT_SHIFT 0x0UL +#define CC_DSCRPTR_MEASURE_CNTR_VALUE_BIT_SIZE 0x20UL +#define CC_DSCRPTR_QUEUE_WORD0_REG_OFFSET 0xE80UL +#define CC_DSCRPTR_QUEUE_WORD0_VALUE_BIT_SHIFT 0x0UL +#define CC_DSCRPTR_QUEUE_WORD0_VALUE_BIT_SIZE 0x20UL +#define CC_DSCRPTR_QUEUE_WORD1_REG_OFFSET 0xE84UL +#define CC_DSCRPTR_QUEUE_WORD1_DIN_DMA_MODE_BIT_SHIFT 0x0UL +#define CC_DSCRPTR_QUEUE_WORD1_DIN_DMA_MODE_BIT_SIZE 0x2UL +#define CC_DSCRPTR_QUEUE_WORD1_DIN_SIZE_BIT_SHIFT 0x2UL +#define CC_DSCRPTR_QUEUE_WORD1_DIN_SIZE_BIT_SIZE 0x18UL +#define CC_DSCRPTR_QUEUE_WORD1_NS_BIT_BIT_SHIFT 0x1AUL +#define CC_DSCRPTR_QUEUE_WORD1_NS_BIT_BIT_SIZE 0x1UL +#define CC_DSCRPTR_QUEUE_WORD1_DIN_CONST_VALUE_BIT_SHIFT 0x1BUL +#define CC_DSCRPTR_QUEUE_WORD1_DIN_CONST_VALUE_BIT_SIZE 0x1UL +#define CC_DSCRPTR_QUEUE_WORD1_NOT_LAST_BIT_SHIFT 0x1CUL +#define CC_DSCRPTR_QUEUE_WORD1_NOT_LAST_BIT_SIZE 0x1UL +#define CC_DSCRPTR_QUEUE_WORD1_LOCK_QUEUE_BIT_SHIFT 0x1DUL +#define CC_DSCRPTR_QUEUE_WORD1_LOCK_QUEUE_BIT_SIZE 0x1UL +#define CC_DSCRPTR_QUEUE_WORD1_NOT_USED_BIT_SHIFT 0x1EUL +#define CC_DSCRPTR_QUEUE_WORD1_NOT_USED_BIT_SIZE 0x2UL +#define CC_DSCRPTR_QUEUE_WORD2_REG_OFFSET 0xE88UL +#define CC_DSCRPTR_QUEUE_WORD2_VALUE_BIT_SHIFT 0x0UL +#define CC_DSCRPTR_QUEUE_WORD2_VALUE_BIT_SIZE 0x20UL +#define CC_DSCRPTR_QUEUE_WORD3_REG_OFFSET 0xE8CUL +#define CC_DSCRPTR_QUEUE_WORD3_DOUT_DMA_MODE_BIT_SHIFT 0x0UL +#define CC_DSCRPTR_QUEUE_WORD3_DOUT_DMA_MODE_BIT_SIZE 0x2UL +#define CC_DSCRPTR_QUEUE_WORD3_DOUT_SIZE_BIT_SHIFT 0x2UL +#define CC_DSCRPTR_QUEUE_WORD3_DOUT_SIZE_BIT_SIZE 0x18UL +#define CC_DSCRPTR_QUEUE_WORD3_NS_BIT_BIT_SHIFT 0x1AUL +#define CC_DSCRPTR_QUEUE_WORD3_NS_BIT_BIT_SIZE 0x1UL +#define CC_DSCRPTR_QUEUE_WORD3_DOUT_LAST_IND_BIT_SHIFT 0x1BUL +#define CC_DSCRPTR_QUEUE_WORD3_DOUT_LAST_IND_BIT_SIZE 0x1UL +#define CC_DSCRPTR_QUEUE_WORD3_HASH_XOR_BIT_BIT_SHIFT 0x1DUL +#define CC_DSCRPTR_QUEUE_WORD3_HASH_XOR_BIT_BIT_SIZE 0x1UL +#define CC_DSCRPTR_QUEUE_WORD3_NOT_USED_BIT_SHIFT 0x1EUL +#define CC_DSCRPTR_QUEUE_WORD3_NOT_USED_BIT_SIZE 0x1UL +#define CC_DSCRPTR_QUEUE_WORD3_QUEUE_LAST_IND_BIT_SHIFT 0x1FUL +#define CC_DSCRPTR_QUEUE_WORD3_QUEUE_LAST_IND_BIT_SIZE 0x1UL +#define CC_DSCRPTR_QUEUE_WORD4_REG_OFFSET 0xE90UL +#define CC_DSCRPTR_QUEUE_WORD4_DATA_FLOW_MODE_BIT_SHIFT 0x0UL +#define CC_DSCRPTR_QUEUE_WORD4_DATA_FLOW_MODE_BIT_SIZE 0x6UL +#define CC_DSCRPTR_QUEUE_WORD4_AES_SEL_N_HASH_BIT_SHIFT 0x6UL +#define CC_DSCRPTR_QUEUE_WORD4_AES_SEL_N_HASH_BIT_SIZE 0x1UL +#define CC_DSCRPTR_QUEUE_WORD4_AES_XOR_CRYPTO_KEY_BIT_SHIFT 0x7UL +#define CC_DSCRPTR_QUEUE_WORD4_AES_XOR_CRYPTO_KEY_BIT_SIZE 0x1UL +#define CC_DSCRPTR_QUEUE_WORD4_ACK_NEEDED_BIT_SHIFT 0x8UL +#define CC_DSCRPTR_QUEUE_WORD4_ACK_NEEDED_BIT_SIZE 0x2UL +#define CC_DSCRPTR_QUEUE_WORD4_CIPHER_MODE_BIT_SHIFT 0xAUL +#define CC_DSCRPTR_QUEUE_WORD4_CIPHER_MODE_BIT_SIZE 0x4UL +#define CC_DSCRPTR_QUEUE_WORD4_CMAC_SIZE0_BIT_SHIFT 0xEUL +#define CC_DSCRPTR_QUEUE_WORD4_CMAC_SIZE0_BIT_SIZE 0x1UL +#define CC_DSCRPTR_QUEUE_WORD4_CIPHER_DO_BIT_SHIFT 0xFUL +#define CC_DSCRPTR_QUEUE_WORD4_CIPHER_DO_BIT_SIZE 0x2UL +#define CC_DSCRPTR_QUEUE_WORD4_CIPHER_CONF0_BIT_SHIFT 0x11UL +#define CC_DSCRPTR_QUEUE_WORD4_CIPHER_CONF0_BIT_SIZE 0x2UL +#define CC_DSCRPTR_QUEUE_WORD4_CIPHER_CONF1_BIT_SHIFT 0x13UL +#define CC_DSCRPTR_QUEUE_WORD4_CIPHER_CONF1_BIT_SIZE 0x1UL +#define CC_DSCRPTR_QUEUE_WORD4_CIPHER_CONF2_BIT_SHIFT 0x14UL +#define CC_DSCRPTR_QUEUE_WORD4_CIPHER_CONF2_BIT_SIZE 0x2UL +#define CC_DSCRPTR_QUEUE_WORD4_KEY_SIZE_BIT_SHIFT 0x16UL +#define CC_DSCRPTR_QUEUE_WORD4_KEY_SIZE_BIT_SIZE 0x2UL +#define CC_DSCRPTR_QUEUE_WORD4_SETUP_OPERATION_BIT_SHIFT 0x18UL +#define CC_DSCRPTR_QUEUE_WORD4_SETUP_OPERATION_BIT_SIZE 0x4UL +#define CC_DSCRPTR_QUEUE_WORD4_DIN_SRAM_ENDIANNESS_BIT_SHIFT 0x1CUL +#define CC_DSCRPTR_QUEUE_WORD4_DIN_SRAM_ENDIANNESS_BIT_SIZE 0x1UL +#define CC_DSCRPTR_QUEUE_WORD4_DOUT_SRAM_ENDIANNESS_BIT_SHIFT 0x1DUL +#define CC_DSCRPTR_QUEUE_WORD4_DOUT_SRAM_ENDIANNESS_BIT_SIZE 0x1UL +#define CC_DSCRPTR_QUEUE_WORD4_WORD_SWAP_BIT_SHIFT 0x1EUL +#define CC_DSCRPTR_QUEUE_WORD4_WORD_SWAP_BIT_SIZE 0x1UL +#define CC_DSCRPTR_QUEUE_WORD4_BYTES_SWAP_BIT_SHIFT 0x1FUL +#define CC_DSCRPTR_QUEUE_WORD4_BYTES_SWAP_BIT_SIZE 0x1UL +#define CC_DSCRPTR_QUEUE_WORD5_REG_OFFSET 0xE94UL +#define CC_DSCRPTR_QUEUE_WORD5_DIN_ADDR_HIGH_BIT_SHIFT 0x0UL +#define CC_DSCRPTR_QUEUE_WORD5_DIN_ADDR_HIGH_BIT_SIZE 0x10UL +#define CC_DSCRPTR_QUEUE_WORD5_DOUT_ADDR_HIGH_BIT_SHIFT 0x10UL +#define CC_DSCRPTR_QUEUE_WORD5_DOUT_ADDR_HIGH_BIT_SIZE 0x10UL +#define CC_DSCRPTR_QUEUE_WATERMARK_REG_OFFSET 0xE98UL +#define CC_DSCRPTR_QUEUE_WATERMARK_VALUE_BIT_SHIFT 0x0UL +#define CC_DSCRPTR_QUEUE_WATERMARK_VALUE_BIT_SIZE 0xAUL +#define CC_DSCRPTR_QUEUE_CONTENT_REG_OFFSET 0xE9CUL +#define CC_DSCRPTR_QUEUE_CONTENT_VALUE_BIT_SHIFT 0x0UL +#define CC_DSCRPTR_QUEUE_CONTENT_VALUE_BIT_SIZE 0xAUL +// -------------------------------------- +// BLOCK: AXI_P +// -------------------------------------- +#define CC_AXIM_MON_INFLIGHT_REG_OFFSET 0xB00UL +#define CC_AXIM_MON_INFLIGHT_VALUE_BIT_SHIFT 0x0UL +#define CC_AXIM_MON_INFLIGHT_VALUE_BIT_SIZE 0x8UL +#define CC_AXIM_MON_INFLIGHTLAST_REG_OFFSET 0xB40UL +#define CC_AXIM_MON_INFLIGHTLAST_VALUE_BIT_SHIFT 0x0UL +#define CC_AXIM_MON_INFLIGHTLAST_VALUE_BIT_SIZE 0x8UL +#define CC_AXIM_MON_COMP_REG_OFFSET 0xB80UL +#define CC_AXIM_MON_COMP8_REG_OFFSET 0xBA0UL +#define CC_AXIM_MON_COMP_VALUE_BIT_SHIFT 0x0UL +#define CC_AXIM_MON_COMP_VALUE_BIT_SIZE 0x10UL +#define CC_AXIM_MON_ERR_REG_OFFSET 0xBC4UL +#define CC_AXIM_MON_ERR_BRESP_BIT_SHIFT 0x0UL +#define CC_AXIM_MON_ERR_BRESP_BIT_SIZE 0x2UL +#define CC_AXIM_MON_ERR_BID_BIT_SHIFT 0x2UL +#define CC_AXIM_MON_ERR_BID_BIT_SIZE 0x4UL +#define CC_AXIM_MON_ERR_RRESP_BIT_SHIFT 0x10UL +#define CC_AXIM_MON_ERR_RRESP_BIT_SIZE 0x2UL +#define CC_AXIM_MON_ERR_RID_BIT_SHIFT 0x12UL +#define CC_AXIM_MON_ERR_RID_BIT_SIZE 0x4UL +#define CC_AXIM_CFG_REG_OFFSET 0xBE8UL +#define CC_AXIM_CFG_BRESPMASK_BIT_SHIFT 0x4UL +#define CC_AXIM_CFG_BRESPMASK_BIT_SIZE 0x1UL +#define CC_AXIM_CFG_RRESPMASK_BIT_SHIFT 0x5UL +#define CC_AXIM_CFG_RRESPMASK_BIT_SIZE 0x1UL +#define CC_AXIM_CFG_INFLTMASK_BIT_SHIFT 0x6UL +#define CC_AXIM_CFG_INFLTMASK_BIT_SIZE 0x1UL +#define CC_AXIM_CFG_COMPMASK_BIT_SHIFT 0x7UL +#define CC_AXIM_CFG_COMPMASK_BIT_SIZE 0x1UL +#define CC_AXIM_ACE_CONST_REG_OFFSET 0xBECUL +#define CC_AXIM_ACE_CONST_ARDOMAIN_BIT_SHIFT 0x0UL +#define CC_AXIM_ACE_CONST_ARDOMAIN_BIT_SIZE 0x2UL +#define CC_AXIM_ACE_CONST_AWDOMAIN_BIT_SHIFT 0x2UL +#define CC_AXIM_ACE_CONST_AWDOMAIN_BIT_SIZE 0x2UL +#define CC_AXIM_ACE_CONST_ARBAR_BIT_SHIFT 0x4UL +#define CC_AXIM_ACE_CONST_ARBAR_BIT_SIZE 0x2UL +#define CC_AXIM_ACE_CONST_AWBAR_BIT_SHIFT 0x6UL +#define CC_AXIM_ACE_CONST_AWBAR_BIT_SIZE 0x2UL +#define CC_AXIM_ACE_CONST_ARSNOOP_BIT_SHIFT 0x8UL +#define CC_AXIM_ACE_CONST_ARSNOOP_BIT_SIZE 0x4UL +#define CC_AXIM_ACE_CONST_AWSNOOP_NOT_ALIGNED_BIT_SHIFT 0xCUL +#define CC_AXIM_ACE_CONST_AWSNOOP_NOT_ALIGNED_BIT_SIZE 0x3UL +#define CC_AXIM_ACE_CONST_AWSNOOP_ALIGNED_BIT_SHIFT 0xFUL +#define CC_AXIM_ACE_CONST_AWSNOOP_ALIGNED_BIT_SIZE 0x3UL +#define CC_AXIM_ACE_CONST_AWADDR_NOT_MASKED_BIT_SHIFT 0x12UL +#define CC_AXIM_ACE_CONST_AWADDR_NOT_MASKED_BIT_SIZE 0x7UL +#define CC_AXIM_ACE_CONST_AWLEN_VAL_BIT_SHIFT 0x19UL +#define CC_AXIM_ACE_CONST_AWLEN_VAL_BIT_SIZE 0x4UL +#define CC_AXIM_CACHE_PARAMS_REG_OFFSET 0xBF0UL +#define CC_AXIM_CACHE_PARAMS_AWCACHE_LAST_BIT_SHIFT 0x0UL +#define CC_AXIM_CACHE_PARAMS_AWCACHE_LAST_BIT_SIZE 0x4UL +#define CC_AXIM_CACHE_PARAMS_AWCACHE_BIT_SHIFT 0x4UL +#define CC_AXIM_CACHE_PARAMS_AWCACHE_BIT_SIZE 0x4UL +#define CC_AXIM_CACHE_PARAMS_ARCACHE_BIT_SHIFT 0x8UL +#define CC_AXIM_CACHE_PARAMS_ARCACHE_BIT_SIZE 0x4UL +#endif // __CC_CRYS_KERNEL_H__ diff --git a/drivers/crypto/ccree/cc_lli_defs.h b/drivers/crypto/ccree/cc_lli_defs.h new file mode 100644 index 000000000..f891ab813 --- /dev/null +++ b/drivers/crypto/ccree/cc_lli_defs.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ + +#ifndef _CC_LLI_DEFS_H_ +#define _CC_LLI_DEFS_H_ + +#include + +/* Max DLLI size + * AKA CC_DSCRPTR_QUEUE_WORD1_DIN_SIZE_BIT_SIZE + */ +#define DLLI_SIZE_BIT_SIZE 0x18 + +#define CC_MAX_MLLI_ENTRY_SIZE 0xFFFF + +#define LLI_MAX_NUM_OF_DATA_ENTRIES 128 +#define LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES 8 +#define MLLI_TABLE_MIN_ALIGNMENT 4 /* 32 bit alignment */ +#define MAX_NUM_OF_BUFFERS_IN_MLLI 4 +#define MAX_NUM_OF_TOTAL_MLLI_ENTRIES \ + (2 * LLI_MAX_NUM_OF_DATA_ENTRIES + \ + LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES) + +/* Size of entry */ +#define LLI_ENTRY_WORD_SIZE 2 +#define LLI_ENTRY_BYTE_SIZE (LLI_ENTRY_WORD_SIZE * sizeof(u32)) + +/* Word0[31:0] = ADDR[31:0] */ +#define LLI_WORD0_OFFSET 0 +#define LLI_LADDR_BIT_OFFSET 0 +#define LLI_LADDR_BIT_SIZE 32 +/* Word1[31:16] = ADDR[47:32]; Word1[15:0] = SIZE */ +#define LLI_WORD1_OFFSET 1 +#define LLI_SIZE_BIT_OFFSET 0 +#define LLI_SIZE_BIT_SIZE 16 +#define LLI_HADDR_BIT_OFFSET 16 +#define LLI_HADDR_BIT_SIZE 16 + +#define LLI_SIZE_MASK GENMASK((LLI_SIZE_BIT_SIZE - 1), LLI_SIZE_BIT_OFFSET) +#define LLI_HADDR_MASK GENMASK( \ + (LLI_HADDR_BIT_OFFSET + LLI_HADDR_BIT_SIZE - 1),\ + LLI_HADDR_BIT_OFFSET) + +static inline void cc_lli_set_addr(u32 *lli_p, dma_addr_t addr) +{ + lli_p[LLI_WORD0_OFFSET] = (addr & U32_MAX); +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + lli_p[LLI_WORD1_OFFSET] &= ~LLI_HADDR_MASK; + lli_p[LLI_WORD1_OFFSET] |= FIELD_PREP(LLI_HADDR_MASK, (addr >> 32)); +#endif /* CONFIG_ARCH_DMA_ADDR_T_64BIT */ +} + +static inline void cc_lli_set_size(u32 *lli_p, u16 size) +{ + lli_p[LLI_WORD1_OFFSET] &= ~LLI_SIZE_MASK; + lli_p[LLI_WORD1_OFFSET] |= FIELD_PREP(LLI_SIZE_MASK, size); +} + +#endif /*_CC_LLI_DEFS_H_*/ diff --git a/drivers/crypto/ccree/cc_pm.c b/drivers/crypto/ccree/cc_pm.c new file mode 100644 index 000000000..bbd118f8d --- /dev/null +++ b/drivers/crypto/ccree/cc_pm.c @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ + +#include +#include +#include +#include "cc_driver.h" +#include "cc_buffer_mgr.h" +#include "cc_request_mgr.h" +#include "cc_sram_mgr.h" +#include "cc_hash.h" +#include "cc_pm.h" +#include "cc_fips.h" + +#define POWER_DOWN_ENABLE 0x01 +#define POWER_DOWN_DISABLE 0x00 + +static int cc_pm_suspend(struct device *dev) +{ + struct cc_drvdata *drvdata = dev_get_drvdata(dev); + + dev_dbg(dev, "set HOST_POWER_DOWN_EN\n"); + fini_cc_regs(drvdata); + cc_iowrite(drvdata, CC_REG(HOST_POWER_DOWN_EN), POWER_DOWN_ENABLE); + clk_disable_unprepare(drvdata->clk); + return 0; +} + +static int cc_pm_resume(struct device *dev) +{ + int rc; + struct cc_drvdata *drvdata = dev_get_drvdata(dev); + + dev_dbg(dev, "unset HOST_POWER_DOWN_EN\n"); + /* Enables the device source clk */ + rc = clk_prepare_enable(drvdata->clk); + if (rc) { + dev_err(dev, "failed getting clock back on. We're toast.\n"); + return rc; + } + /* wait for Cryptocell reset completion */ + if (!cc_wait_for_reset_completion(drvdata)) { + dev_err(dev, "Cryptocell reset not completed"); + clk_disable_unprepare(drvdata->clk); + return -EBUSY; + } + + cc_iowrite(drvdata, CC_REG(HOST_POWER_DOWN_EN), POWER_DOWN_DISABLE); + rc = init_cc_regs(drvdata); + if (rc) { + dev_err(dev, "init_cc_regs (%x)\n", rc); + clk_disable_unprepare(drvdata->clk); + return rc; + } + /* check if tee fips error occurred during power down */ + cc_tee_handle_fips_error(drvdata); + + cc_init_hash_sram(drvdata); + + return 0; +} + +const struct dev_pm_ops ccree_pm = { + SET_RUNTIME_PM_OPS(cc_pm_suspend, cc_pm_resume, NULL) +}; + +int cc_pm_get(struct device *dev) +{ + int rc = pm_runtime_get_sync(dev); + if (rc < 0) { + pm_runtime_put_noidle(dev); + return rc; + } + + return 0; +} + +void cc_pm_put_suspend(struct device *dev) +{ + pm_runtime_put_autosuspend(dev); +} diff --git a/drivers/crypto/ccree/cc_pm.h b/drivers/crypto/ccree/cc_pm.h new file mode 100644 index 000000000..50cac33de --- /dev/null +++ b/drivers/crypto/ccree/cc_pm.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ + +/* \file cc_pm.h + */ + +#ifndef __CC_POWER_MGR_H__ +#define __CC_POWER_MGR_H__ + +#include "cc_driver.h" + +#define CC_SUSPEND_TIMEOUT 3000 + +#if defined(CONFIG_PM) + +extern const struct dev_pm_ops ccree_pm; + +int cc_pm_get(struct device *dev); +void cc_pm_put_suspend(struct device *dev); + +#else + +static inline int cc_pm_get(struct device *dev) +{ + return 0; +} + +static inline void cc_pm_put_suspend(struct device *dev) {} + +#endif + +#endif /*__POWER_MGR_H__*/ diff --git a/drivers/crypto/ccree/cc_request_mgr.c b/drivers/crypto/ccree/cc_request_mgr.c new file mode 100644 index 000000000..aa0220212 --- /dev/null +++ b/drivers/crypto/ccree/cc_request_mgr.c @@ -0,0 +1,662 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ + +#include +#include +#include "cc_driver.h" +#include "cc_buffer_mgr.h" +#include "cc_request_mgr.h" +#include "cc_pm.h" + +#define CC_MAX_POLL_ITER 10 +/* The highest descriptor count in used */ +#define CC_MAX_DESC_SEQ_LEN 23 + +struct cc_req_mgr_handle { + /* Request manager resources */ + unsigned int hw_queue_size; /* HW capability */ + unsigned int min_free_hw_slots; + unsigned int max_used_sw_slots; + struct cc_crypto_req req_queue[MAX_REQUEST_QUEUE_SIZE]; + u32 req_queue_head; + u32 req_queue_tail; + u32 axi_completed; + u32 q_free_slots; + /* This lock protects access to HW register + * that must be single request at a time + */ + spinlock_t hw_lock; + struct cc_hw_desc compl_desc; + u8 *dummy_comp_buff; + dma_addr_t dummy_comp_buff_dma; + + /* backlog queue */ + struct list_head backlog; + unsigned int bl_len; + spinlock_t bl_lock; /* protect backlog queue */ + +#ifdef COMP_IN_WQ + struct workqueue_struct *workq; + struct delayed_work compwork; +#else + struct tasklet_struct comptask; +#endif +}; + +struct cc_bl_item { + struct cc_crypto_req creq; + struct cc_hw_desc desc[CC_MAX_DESC_SEQ_LEN]; + unsigned int len; + struct list_head list; + bool notif; +}; + +static const u32 cc_cpp_int_masks[CC_CPP_NUM_ALGS][CC_CPP_NUM_SLOTS] = { + { BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_0_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_1_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_2_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_3_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_4_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_5_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_6_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_7_INT_BIT_SHIFT) }, + { BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_0_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_1_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_2_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_3_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_4_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_5_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_6_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_7_INT_BIT_SHIFT) } +}; + +static void comp_handler(unsigned long devarg); +#ifdef COMP_IN_WQ +static void comp_work_handler(struct work_struct *work); +#endif + +static inline u32 cc_cpp_int_mask(enum cc_cpp_alg alg, int slot) +{ + alg = array_index_nospec(alg, CC_CPP_NUM_ALGS); + slot = array_index_nospec(slot, CC_CPP_NUM_SLOTS); + + return cc_cpp_int_masks[alg][slot]; +} + +void cc_req_mgr_fini(struct cc_drvdata *drvdata) +{ + struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle; + struct device *dev = drvdata_to_dev(drvdata); + + if (!req_mgr_h) + return; /* Not allocated */ + + if (req_mgr_h->dummy_comp_buff_dma) { + dma_free_coherent(dev, sizeof(u32), req_mgr_h->dummy_comp_buff, + req_mgr_h->dummy_comp_buff_dma); + } + + dev_dbg(dev, "max_used_hw_slots=%d\n", (req_mgr_h->hw_queue_size - + req_mgr_h->min_free_hw_slots)); + dev_dbg(dev, "max_used_sw_slots=%d\n", req_mgr_h->max_used_sw_slots); + +#ifdef COMP_IN_WQ + destroy_workqueue(req_mgr_h->workq); +#else + /* Kill tasklet */ + tasklet_kill(&req_mgr_h->comptask); +#endif + kfree_sensitive(req_mgr_h); + drvdata->request_mgr_handle = NULL; +} + +int cc_req_mgr_init(struct cc_drvdata *drvdata) +{ + struct cc_req_mgr_handle *req_mgr_h; + struct device *dev = drvdata_to_dev(drvdata); + int rc = 0; + + req_mgr_h = kzalloc_obj(*req_mgr_h); + if (!req_mgr_h) { + rc = -ENOMEM; + goto req_mgr_init_err; + } + + drvdata->request_mgr_handle = req_mgr_h; + + spin_lock_init(&req_mgr_h->hw_lock); + spin_lock_init(&req_mgr_h->bl_lock); + INIT_LIST_HEAD(&req_mgr_h->backlog); + +#ifdef COMP_IN_WQ + dev_dbg(dev, "Initializing completion workqueue\n"); + req_mgr_h->workq = create_singlethread_workqueue("ccree"); + if (!req_mgr_h->workq) { + dev_err(dev, "Failed creating work queue\n"); + rc = -ENOMEM; + goto req_mgr_init_err; + } + INIT_DELAYED_WORK(&req_mgr_h->compwork, comp_work_handler); +#else + dev_dbg(dev, "Initializing completion tasklet\n"); + tasklet_init(&req_mgr_h->comptask, comp_handler, + (unsigned long)drvdata); +#endif + req_mgr_h->hw_queue_size = cc_ioread(drvdata, + CC_REG(DSCRPTR_QUEUE_SRAM_SIZE)); + dev_dbg(dev, "hw_queue_size=0x%08X\n", req_mgr_h->hw_queue_size); + if (req_mgr_h->hw_queue_size < MIN_HW_QUEUE_SIZE) { + dev_err(dev, "Invalid HW queue size = %u (Min. required is %u)\n", + req_mgr_h->hw_queue_size, MIN_HW_QUEUE_SIZE); + rc = -ENOMEM; + goto req_mgr_init_err; + } + req_mgr_h->min_free_hw_slots = req_mgr_h->hw_queue_size; + req_mgr_h->max_used_sw_slots = 0; + + /* Allocate DMA word for "dummy" completion descriptor use */ + req_mgr_h->dummy_comp_buff = + dma_alloc_coherent(dev, sizeof(u32), + &req_mgr_h->dummy_comp_buff_dma, + GFP_KERNEL); + if (!req_mgr_h->dummy_comp_buff) { + dev_err(dev, "Not enough memory to allocate DMA (%zu) dropped buffer\n", + sizeof(u32)); + rc = -ENOMEM; + goto req_mgr_init_err; + } + + /* Init. "dummy" completion descriptor */ + hw_desc_init(&req_mgr_h->compl_desc); + set_din_const(&req_mgr_h->compl_desc, 0, sizeof(u32)); + set_dout_dlli(&req_mgr_h->compl_desc, req_mgr_h->dummy_comp_buff_dma, + sizeof(u32), NS_BIT, 1); + set_flow_mode(&req_mgr_h->compl_desc, BYPASS); + set_queue_last_ind(drvdata, &req_mgr_h->compl_desc); + + return 0; + +req_mgr_init_err: + cc_req_mgr_fini(drvdata); + return rc; +} + +static void enqueue_seq(struct cc_drvdata *drvdata, struct cc_hw_desc seq[], + unsigned int seq_len) +{ + int i, w; + void __iomem *reg = drvdata->cc_base + CC_REG(DSCRPTR_QUEUE_WORD0); + struct device *dev = drvdata_to_dev(drvdata); + + /* + * We do indeed write all 6 command words to the same + * register. The HW supports this. + */ + + for (i = 0; i < seq_len; i++) { + for (w = 0; w <= 5; w++) + writel_relaxed(seq[i].word[w], reg); + + if (cc_dump_desc) + dev_dbg(dev, "desc[%02d]: 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n", + i, seq[i].word[0], seq[i].word[1], + seq[i].word[2], seq[i].word[3], + seq[i].word[4], seq[i].word[5]); + } +} + +/** + * request_mgr_complete() - Completion will take place if and only if user + * requested completion by cc_send_sync_request(). + * + * @dev: Device pointer + * @dx_compl_h: The completion event to signal + * @dummy: unused error code + */ +static void request_mgr_complete(struct device *dev, void *dx_compl_h, + int dummy) +{ + struct completion *this_compl = dx_compl_h; + + complete(this_compl); +} + +static int cc_queues_status(struct cc_drvdata *drvdata, + struct cc_req_mgr_handle *req_mgr_h, + unsigned int total_seq_len) +{ + unsigned long poll_queue; + struct device *dev = drvdata_to_dev(drvdata); + + /* SW queue is checked only once as it will not + * be changed during the poll because the spinlock_bh + * is held by the thread + */ + if (((req_mgr_h->req_queue_head + 1) & (MAX_REQUEST_QUEUE_SIZE - 1)) == + req_mgr_h->req_queue_tail) { + dev_err(dev, "SW FIFO is full. req_queue_head=%d sw_fifo_len=%d\n", + req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE); + return -ENOSPC; + } + + if (req_mgr_h->q_free_slots >= total_seq_len) + return 0; + + /* Wait for space in HW queue. Poll constant num of iterations. */ + for (poll_queue = 0; poll_queue < CC_MAX_POLL_ITER ; poll_queue++) { + req_mgr_h->q_free_slots = + cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT)); + if (req_mgr_h->q_free_slots < req_mgr_h->min_free_hw_slots) + req_mgr_h->min_free_hw_slots = req_mgr_h->q_free_slots; + + if (req_mgr_h->q_free_slots >= total_seq_len) { + /* If there is enough place return */ + return 0; + } + + dev_dbg(dev, "HW FIFO is full. q_free_slots=%d total_seq_len=%d\n", + req_mgr_h->q_free_slots, total_seq_len); + } + /* No room in the HW queue try again later */ + dev_dbg(dev, "HW FIFO full, timeout. req_queue_head=%d sw_fifo_len=%d q_free_slots=%d total_seq_len=%d\n", + req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE, + req_mgr_h->q_free_slots, total_seq_len); + return -ENOSPC; +} + +/** + * cc_do_send_request() - Enqueue caller request to crypto hardware. + * Need to be called with HW lock held and PM running + * + * @drvdata: Associated device driver context + * @cc_req: The request to enqueue + * @desc: The crypto sequence + * @len: The crypto sequence length + * @add_comp: If "true": add an artificial dout DMA to mark completion + * + */ +static void cc_do_send_request(struct cc_drvdata *drvdata, + struct cc_crypto_req *cc_req, + struct cc_hw_desc *desc, unsigned int len, + bool add_comp) +{ + struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle; + unsigned int used_sw_slots; + unsigned int total_seq_len = len; /*initial sequence length*/ + struct device *dev = drvdata_to_dev(drvdata); + + used_sw_slots = ((req_mgr_h->req_queue_head - + req_mgr_h->req_queue_tail) & + (MAX_REQUEST_QUEUE_SIZE - 1)); + if (used_sw_slots > req_mgr_h->max_used_sw_slots) + req_mgr_h->max_used_sw_slots = used_sw_slots; + + /* Enqueue request - must be locked with HW lock*/ + req_mgr_h->req_queue[req_mgr_h->req_queue_head] = *cc_req; + req_mgr_h->req_queue_head = (req_mgr_h->req_queue_head + 1) & + (MAX_REQUEST_QUEUE_SIZE - 1); + + dev_dbg(dev, "Enqueue request head=%u\n", req_mgr_h->req_queue_head); + + /* + * We are about to push command to the HW via the command registers + * that may reference host memory. We need to issue a memory barrier + * to make sure there are no outstanding memory writes + */ + wmb(); + + /* STAT_PHASE_4: Push sequence */ + + enqueue_seq(drvdata, desc, len); + + if (add_comp) { + enqueue_seq(drvdata, &req_mgr_h->compl_desc, 1); + total_seq_len++; + } + + if (req_mgr_h->q_free_slots < total_seq_len) { + /* This situation should never occur. Maybe indicating problem + * with resuming power. Set the free slot count to 0 and hope + * for the best. + */ + dev_err(dev, "HW free slot count mismatch."); + req_mgr_h->q_free_slots = 0; + } else { + /* Update the free slots in HW queue */ + req_mgr_h->q_free_slots -= total_seq_len; + } +} + +static void cc_enqueue_backlog(struct cc_drvdata *drvdata, + struct cc_bl_item *bli) +{ + struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle; + struct device *dev = drvdata_to_dev(drvdata); + + spin_lock_bh(&mgr->bl_lock); + list_add_tail(&bli->list, &mgr->backlog); + ++mgr->bl_len; + dev_dbg(dev, "+++bl len: %d\n", mgr->bl_len); + spin_unlock_bh(&mgr->bl_lock); + tasklet_schedule(&mgr->comptask); +} + +static void cc_proc_backlog(struct cc_drvdata *drvdata) +{ + struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle; + struct cc_bl_item *bli; + struct cc_crypto_req *creq; + void *req; + struct device *dev = drvdata_to_dev(drvdata); + int rc; + + spin_lock(&mgr->bl_lock); + + while (mgr->bl_len) { + bli = list_first_entry(&mgr->backlog, struct cc_bl_item, list); + dev_dbg(dev, "---bl len: %d\n", mgr->bl_len); + + spin_unlock(&mgr->bl_lock); + + + creq = &bli->creq; + req = creq->user_arg; + + /* + * Notify the request we're moving out of the backlog + * but only if we haven't done so already. + */ + if (!bli->notif) { + creq->user_cb(dev, req, -EINPROGRESS); + bli->notif = true; + } + + spin_lock(&mgr->hw_lock); + + rc = cc_queues_status(drvdata, mgr, bli->len); + if (rc) { + /* + * There is still no room in the FIFO for + * this request. Bail out. We'll return here + * on the next completion irq. + */ + spin_unlock(&mgr->hw_lock); + return; + } + + cc_do_send_request(drvdata, &bli->creq, bli->desc, bli->len, + false); + spin_unlock(&mgr->hw_lock); + + /* Remove ourselves from the backlog list */ + spin_lock(&mgr->bl_lock); + list_del(&bli->list); + --mgr->bl_len; + kfree(bli); + } + + spin_unlock(&mgr->bl_lock); +} + +int cc_send_request(struct cc_drvdata *drvdata, struct cc_crypto_req *cc_req, + struct cc_hw_desc *desc, unsigned int len, + struct crypto_async_request *req) +{ + int rc; + struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle; + struct device *dev = drvdata_to_dev(drvdata); + bool backlog_ok = req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG; + gfp_t flags = cc_gfp_flags(req); + struct cc_bl_item *bli; + + rc = cc_pm_get(dev); + if (rc) { + dev_err(dev, "cc_pm_get returned %x\n", rc); + return rc; + } + + spin_lock_bh(&mgr->hw_lock); + rc = cc_queues_status(drvdata, mgr, len); + +#ifdef CC_DEBUG_FORCE_BACKLOG + if (backlog_ok) + rc = -ENOSPC; +#endif /* CC_DEBUG_FORCE_BACKLOG */ + + if (rc == -ENOSPC && backlog_ok) { + spin_unlock_bh(&mgr->hw_lock); + + bli = kmalloc_obj(*bli, flags); + if (!bli) { + cc_pm_put_suspend(dev); + return -ENOMEM; + } + + memcpy(&bli->creq, cc_req, sizeof(*cc_req)); + memcpy(&bli->desc, desc, len * sizeof(*desc)); + bli->len = len; + bli->notif = false; + cc_enqueue_backlog(drvdata, bli); + return -EBUSY; + } + + if (!rc) { + cc_do_send_request(drvdata, cc_req, desc, len, false); + rc = -EINPROGRESS; + } + + spin_unlock_bh(&mgr->hw_lock); + return rc; +} + +int cc_send_sync_request(struct cc_drvdata *drvdata, + struct cc_crypto_req *cc_req, struct cc_hw_desc *desc, + unsigned int len) +{ + int rc; + struct device *dev = drvdata_to_dev(drvdata); + struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle; + + init_completion(&cc_req->seq_compl); + cc_req->user_cb = request_mgr_complete; + cc_req->user_arg = &cc_req->seq_compl; + + rc = cc_pm_get(dev); + if (rc) { + dev_err(dev, "cc_pm_get returned %x\n", rc); + return rc; + } + + while (true) { + spin_lock_bh(&mgr->hw_lock); + rc = cc_queues_status(drvdata, mgr, len + 1); + + if (!rc) + break; + + spin_unlock_bh(&mgr->hw_lock); + wait_for_completion_interruptible(&drvdata->hw_queue_avail); + reinit_completion(&drvdata->hw_queue_avail); + } + + cc_do_send_request(drvdata, cc_req, desc, len, true); + spin_unlock_bh(&mgr->hw_lock); + wait_for_completion(&cc_req->seq_compl); + return 0; +} + +/** + * send_request_init() - Enqueue caller request to crypto hardware during init + * process. + * Assume this function is not called in the middle of a flow, + * since we set QUEUE_LAST_IND flag in the last descriptor. + * + * @drvdata: Associated device driver context + * @desc: The crypto sequence + * @len: The crypto sequence length + * + * Return: + * Returns "0" upon success + */ +int send_request_init(struct cc_drvdata *drvdata, struct cc_hw_desc *desc, + unsigned int len) +{ + struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle; + unsigned int total_seq_len = len; /*initial sequence length*/ + int rc = 0; + + /* Wait for space in HW and SW FIFO. Poll for as much as FIFO_TIMEOUT. + */ + rc = cc_queues_status(drvdata, req_mgr_h, total_seq_len); + if (rc) + return rc; + + set_queue_last_ind(drvdata, &desc[(len - 1)]); + + /* + * We are about to push command to the HW via the command registers + * that may reference host memory. We need to issue a memory barrier + * to make sure there are no outstanding memory writes + */ + wmb(); + enqueue_seq(drvdata, desc, len); + + /* Update the free slots in HW queue */ + req_mgr_h->q_free_slots = + cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT)); + + return 0; +} + +void complete_request(struct cc_drvdata *drvdata) +{ + struct cc_req_mgr_handle *request_mgr_handle = + drvdata->request_mgr_handle; + + complete(&drvdata->hw_queue_avail); +#ifdef COMP_IN_WQ + queue_delayed_work(request_mgr_handle->workq, + &request_mgr_handle->compwork, 0); +#else + tasklet_schedule(&request_mgr_handle->comptask); +#endif +} + +#ifdef COMP_IN_WQ +static void comp_work_handler(struct work_struct *work) +{ + struct cc_drvdata *drvdata = + container_of(work, struct cc_drvdata, compwork.work); + + comp_handler((unsigned long)drvdata); +} +#endif + +static void proc_completions(struct cc_drvdata *drvdata) +{ + struct cc_crypto_req *cc_req; + struct device *dev = drvdata_to_dev(drvdata); + struct cc_req_mgr_handle *request_mgr_handle = + drvdata->request_mgr_handle; + unsigned int *tail = &request_mgr_handle->req_queue_tail; + unsigned int *head = &request_mgr_handle->req_queue_head; + int rc; + u32 mask; + + while (request_mgr_handle->axi_completed) { + request_mgr_handle->axi_completed--; + + /* Dequeue request */ + if (*head == *tail) { + /* We are supposed to handle a completion but our + * queue is empty. This is not normal. Return and + * hope for the best. + */ + dev_err(dev, "Request queue is empty head == tail %u\n", + *head); + break; + } + + cc_req = &request_mgr_handle->req_queue[*tail]; + + if (cc_req->cpp.is_cpp) { + + dev_dbg(dev, "CPP request completion slot: %d alg:%d\n", + cc_req->cpp.slot, cc_req->cpp.alg); + mask = cc_cpp_int_mask(cc_req->cpp.alg, + cc_req->cpp.slot); + rc = (drvdata->irq & mask ? -EPERM : 0); + dev_dbg(dev, "Got mask: %x irq: %x rc: %d\n", mask, + drvdata->irq, rc); + } else { + dev_dbg(dev, "None CPP request completion\n"); + rc = 0; + } + + if (cc_req->user_cb) + cc_req->user_cb(dev, cc_req->user_arg, rc); + *tail = (*tail + 1) & (MAX_REQUEST_QUEUE_SIZE - 1); + dev_dbg(dev, "Dequeue request tail=%u\n", *tail); + dev_dbg(dev, "Request completed. axi_completed=%d\n", + request_mgr_handle->axi_completed); + cc_pm_put_suspend(dev); + } +} + +static inline u32 cc_axi_comp_count(struct cc_drvdata *drvdata) +{ + return FIELD_GET(AXIM_MON_COMP_VALUE, + cc_ioread(drvdata, drvdata->axim_mon_offset)); +} + +/* Deferred service handler, run as interrupt-fired tasklet */ +static void comp_handler(unsigned long devarg) +{ + struct cc_drvdata *drvdata = (struct cc_drvdata *)devarg; + struct cc_req_mgr_handle *request_mgr_handle = + drvdata->request_mgr_handle; + struct device *dev = drvdata_to_dev(drvdata); + u32 irq; + + dev_dbg(dev, "Completion handler called!\n"); + irq = (drvdata->irq & drvdata->comp_mask); + + /* To avoid the interrupt from firing as we unmask it, + * we clear it now + */ + cc_iowrite(drvdata, CC_REG(HOST_ICR), irq); + + /* Avoid race with above clear: Test completion counter once more */ + + request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata); + + dev_dbg(dev, "AXI completion after updated: %d\n", + request_mgr_handle->axi_completed); + + while (request_mgr_handle->axi_completed) { + do { + drvdata->irq |= cc_ioread(drvdata, CC_REG(HOST_IRR)); + irq = (drvdata->irq & drvdata->comp_mask); + proc_completions(drvdata); + + /* At this point (after proc_completions()), + * request_mgr_handle->axi_completed is 0. + */ + request_mgr_handle->axi_completed += + cc_axi_comp_count(drvdata); + } while (request_mgr_handle->axi_completed > 0); + + cc_iowrite(drvdata, CC_REG(HOST_ICR), irq); + + request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata); + } + + /* after verifying that there is nothing to do, + * unmask AXI completion interrupt + */ + cc_iowrite(drvdata, CC_REG(HOST_IMR), + cc_ioread(drvdata, CC_REG(HOST_IMR)) & ~drvdata->comp_mask); + + cc_proc_backlog(drvdata); + dev_dbg(dev, "Comp. handler done.\n"); +} diff --git a/drivers/crypto/ccree/cc_request_mgr.h b/drivers/crypto/ccree/cc_request_mgr.h new file mode 100644 index 000000000..ae25ca843 --- /dev/null +++ b/drivers/crypto/ccree/cc_request_mgr.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ + +/* \file cc_request_mgr.h + * Request Manager + */ + +#ifndef __REQUEST_MGR_H__ +#define __REQUEST_MGR_H__ + +#include "cc_hw_queue_defs.h" + +int cc_req_mgr_init(struct cc_drvdata *drvdata); + +/** + * cc_send_request() - Enqueue caller request to crypto hardware. + * + * @drvdata: Associated device driver context + * @cc_req: The request to enqueue + * @desc: The crypto sequence + * @len: The crypto sequence length + * @req: Asynchronous crypto request + * + * Return: + * Returns -EINPROGRESS or error + */ +int cc_send_request(struct cc_drvdata *drvdata, struct cc_crypto_req *cc_req, + struct cc_hw_desc *desc, unsigned int len, + struct crypto_async_request *req); + +int cc_send_sync_request(struct cc_drvdata *drvdata, + struct cc_crypto_req *cc_req, struct cc_hw_desc *desc, + unsigned int len); + +int send_request_init(struct cc_drvdata *drvdata, struct cc_hw_desc *desc, + unsigned int len); + +void complete_request(struct cc_drvdata *drvdata); + +void cc_req_mgr_fini(struct cc_drvdata *drvdata); + +#endif /*__REQUEST_MGR_H__*/ diff --git a/drivers/crypto/ccree/cc_sram_mgr.c b/drivers/crypto/ccree/cc_sram_mgr.c new file mode 100644 index 000000000..37a958563 --- /dev/null +++ b/drivers/crypto/ccree/cc_sram_mgr.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ + +#include "cc_driver.h" +#include "cc_sram_mgr.h" + +/** + * cc_sram_mgr_init() - Initializes SRAM pool. + * The pool starts right at the beginning of SRAM. + * Returns zero for success, negative value otherwise. + * + * @drvdata: Associated device driver context + * + * Return: + * 0 for success, negative error code for failure. + */ +int cc_sram_mgr_init(struct cc_drvdata *drvdata) +{ + u32 start = 0; + struct device *dev = drvdata_to_dev(drvdata); + + if (drvdata->hw_rev < CC_HW_REV_712) { + /* Pool starts after ROM bytes */ + start = cc_ioread(drvdata, CC_REG(HOST_SEP_SRAM_THRESHOLD)); + if ((start & 0x3) != 0) { + dev_err(dev, "Invalid SRAM offset 0x%x\n", start); + return -EINVAL; + } + } + + drvdata->sram_free_offset = start; + return 0; +} + +/** + * cc_sram_alloc() - Allocate buffer from SRAM pool. + * + * @drvdata: Associated device driver context + * @size: The requested numer of bytes to allocate + * + * Return: + * Address offset in SRAM or NULL_SRAM_ADDR for failure. + */ +u32 cc_sram_alloc(struct cc_drvdata *drvdata, u32 size) +{ + struct device *dev = drvdata_to_dev(drvdata); + u32 p; + + if ((size & 0x3)) { + dev_err(dev, "Requested buffer size (%u) is not multiple of 4", + size); + return NULL_SRAM_ADDR; + } + if (size > (CC_CC_SRAM_SIZE - drvdata->sram_free_offset)) { + dev_err(dev, "Not enough space to allocate %u B (at offset %u)\n", + size, drvdata->sram_free_offset); + return NULL_SRAM_ADDR; + } + + p = drvdata->sram_free_offset; + drvdata->sram_free_offset += size; + dev_dbg(dev, "Allocated %u B @ %u\n", size, p); + return p; +} + +/** + * cc_set_sram_desc() - Create const descriptors sequence to + * set values in given array into SRAM. + * Note: each const value can't exceed word size. + * + * @src: A pointer to array of words to set as consts. + * @dst: The target SRAM buffer to set into + * @nelement: The number of words in "src" array + * @seq: A pointer to the given IN/OUT descriptor sequence + * @seq_len: A pointer to the given IN/OUT sequence length + */ +void cc_set_sram_desc(const u32 *src, u32 dst, unsigned int nelement, + struct cc_hw_desc *seq, unsigned int *seq_len) +{ + u32 i; + unsigned int idx = *seq_len; + + for (i = 0; i < nelement; i++, idx++) { + hw_desc_init(&seq[idx]); + set_din_const(&seq[idx], src[i], sizeof(u32)); + set_dout_sram(&seq[idx], dst + (i * sizeof(u32)), sizeof(u32)); + set_flow_mode(&seq[idx], BYPASS); + } + + *seq_len = idx; +} diff --git a/drivers/crypto/ccree/cc_sram_mgr.h b/drivers/crypto/ccree/cc_sram_mgr.h new file mode 100644 index 000000000..1c965ef83 --- /dev/null +++ b/drivers/crypto/ccree/cc_sram_mgr.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ + +#ifndef __CC_SRAM_MGR_H__ +#define __CC_SRAM_MGR_H__ + +#ifndef CC_CC_SRAM_SIZE +#define CC_CC_SRAM_SIZE 4096 +#endif + +struct cc_drvdata; + +#define NULL_SRAM_ADDR ((u32)-1) + +/** + * cc_sram_mgr_init() - Initializes SRAM pool. + * The first X bytes of SRAM are reserved for ROM usage, hence, pool + * starts right after X bytes. + * + * @drvdata: Associated device driver context + * + * Return: + * Zero for success, negative value otherwise. + */ +int cc_sram_mgr_init(struct cc_drvdata *drvdata); + +/** + * cc_sram_alloc() - Allocate buffer from SRAM pool. + * + * @drvdata: Associated device driver context + * @size: The requested bytes to allocate + * + * Return: + * Address offset in SRAM or NULL_SRAM_ADDR for failure. + */ +u32 cc_sram_alloc(struct cc_drvdata *drvdata, u32 size); + +/** + * cc_set_sram_desc() - Create const descriptors sequence to + * set values in given array into SRAM. + * Note: each const value can't exceed word size. + * + * @src: A pointer to array of words to set as consts. + * @dst: The target SRAM buffer to set into + * @nelement: The number of words in "src" array + * @seq: A pointer to the given IN/OUT descriptor sequence + * @seq_len: A pointer to the given IN/OUT sequence length + */ +void cc_set_sram_desc(const u32 *src, u32 dst, unsigned int nelement, + struct cc_hw_desc *seq, unsigned int *seq_len); + +#endif /*__CC_SRAM_MGR_H__*/ diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig new file mode 100644 index 000000000..37294bb74 --- /dev/null +++ b/drivers/crypto/chelsio/Kconfig @@ -0,0 +1,23 @@ +# SPDX-License-Identifier: GPL-2.0-only +config CRYPTO_DEV_CHELSIO + tristate "Chelsio Crypto Co-processor Driver" + depends on CHELSIO_T4 + select CRYPTO_LIB_AES + select CRYPTO_LIB_GF128MUL + select CRYPTO_LIB_SHA1 + select CRYPTO_LIB_SHA256 + select CRYPTO_LIB_SHA512 + select CRYPTO_AUTHENC + help + The Chelsio Crypto Co-processor driver for T6 adapters. + + For general information about Chelsio and our products, visit + our website at . + + For customer support, please visit our customer support page at + . + + Please send feedback to . + + To compile this driver as a module, choose M here: the module + will be called chcr. diff --git a/drivers/crypto/chelsio/Makefile b/drivers/crypto/chelsio/Makefile new file mode 100644 index 000000000..2e5df484a --- /dev/null +++ b/drivers/crypto/chelsio/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0-only +ccflags-y := -I $(srctree)/drivers/net/ethernet/chelsio/cxgb4 + +obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chcr.o +chcr-objs := chcr_core.o chcr_algo.o diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c new file mode 100644 index 000000000..6dec42282 --- /dev/null +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -0,0 +1,4401 @@ +/* + * This file is part of the Chelsio T6 Crypto driver for Linux. + * + * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Written and Maintained by: + * Manoj Malviya (manojmalviya@chelsio.com) + * Atul Gupta (atul.gupta@chelsio.com) + * Jitendra Lulla (jlulla@chelsio.com) + * Yeshaswi M R Gowda (yeshaswi@chelsio.com) + * Harsh Jain (harsh@chelsio.com) + */ + +#define pr_fmt(fmt) "chcr:" fmt + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "t4fw_api.h" +#include "t4_msg.h" +#include "chcr_core.h" +#include "chcr_algo.h" +#include "chcr_crypto.h" + +#define IV AES_BLOCK_SIZE + +static unsigned int sgl_ent_len[] = { + 0, 0, 16, 24, 40, 48, 64, 72, 88, + 96, 112, 120, 136, 144, 160, 168, 184, + 192, 208, 216, 232, 240, 256, 264, 280, + 288, 304, 312, 328, 336, 352, 360, 376 +}; + +static unsigned int dsgl_ent_len[] = { + 0, 32, 32, 48, 48, 64, 64, 80, 80, + 112, 112, 128, 128, 144, 144, 160, 160, + 192, 192, 208, 208, 224, 224, 240, 240, + 272, 272, 288, 288, 304, 304, 320, 320 +}; + +static u32 round_constant[11] = { + 0x01000000, 0x02000000, 0x04000000, 0x08000000, + 0x10000000, 0x20000000, 0x40000000, 0x80000000, + 0x1B000000, 0x36000000, 0x6C000000 +}; + +static int chcr_handle_cipher_resp(struct skcipher_request *req, + unsigned char *input, int err); + +static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx) +{ + return &ctx->crypto_ctx->aeadctx; +} + +static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx) +{ + return &ctx->crypto_ctx->ablkctx; +} + +static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx) +{ + return &ctx->crypto_ctx->hmacctx; +} + +static inline struct chcr_gcm_ctx *GCM_CTX(struct chcr_aead_ctx *gctx) +{ + return gctx->ctx->gcm; +} + +static inline struct chcr_authenc_ctx *AUTHENC_CTX(struct chcr_aead_ctx *gctx) +{ + return gctx->ctx->authenc; +} + +static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx) +{ + return container_of(ctx->dev, struct uld_ctx, dev); +} + +static inline void chcr_init_hctx_per_wr(struct chcr_ahash_req_ctx *reqctx) +{ + memset(&reqctx->hctx_wr, 0, sizeof(struct chcr_hctx_per_wr)); +} + +static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen, + unsigned int entlen, + unsigned int skip) +{ + int nents = 0; + unsigned int less; + unsigned int skip_len = 0; + + while (sg && skip) { + if (sg_dma_len(sg) <= skip) { + skip -= sg_dma_len(sg); + skip_len = 0; + sg = sg_next(sg); + } else { + skip_len = skip; + skip = 0; + } + } + + while (sg && reqlen) { + less = min(reqlen, sg_dma_len(sg) - skip_len); + nents += DIV_ROUND_UP(less, entlen); + reqlen -= less; + skip_len = 0; + sg = sg_next(sg); + } + return nents; +} + +static inline int get_aead_subtype(struct crypto_aead *aead) +{ + struct aead_alg *alg = crypto_aead_alg(aead); + struct chcr_alg_template *chcr_crypto_alg = + container_of(alg, struct chcr_alg_template, alg.aead); + return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; +} + +void chcr_verify_tag(struct aead_request *req, u8 *input, int *err) +{ + u8 temp[SHA512_DIGEST_SIZE]; + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + int authsize = crypto_aead_authsize(tfm); + struct cpl_fw6_pld *fw6_pld; + int cmp = 0; + + fw6_pld = (struct cpl_fw6_pld *)input; + if ((get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) || + (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) { + cmp = crypto_memneq(&fw6_pld->data[2], (fw6_pld + 1), authsize); + } else { + + sg_pcopy_to_buffer(req->src, sg_nents(req->src), temp, + authsize, req->assoclen + + req->cryptlen - authsize); + cmp = crypto_memneq(temp, (fw6_pld + 1), authsize); + } + if (cmp) + *err = -EBADMSG; + else + *err = 0; +} + +static int chcr_inc_wrcount(struct chcr_dev *dev) +{ + if (dev->state == CHCR_DETACH) + return 1; + atomic_inc(&dev->inflight); + return 0; +} + +static inline void chcr_dec_wrcount(struct chcr_dev *dev) +{ + atomic_dec(&dev->inflight); +} + +static inline int chcr_handle_aead_resp(struct aead_request *req, + unsigned char *input, + int err) +{ + struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct chcr_dev *dev = a_ctx(tfm)->dev; + + chcr_aead_common_exit(req); + if (reqctx->verify == VERIFY_SW) { + chcr_verify_tag(req, input, &err); + reqctx->verify = VERIFY_HW; + } + chcr_dec_wrcount(dev); + aead_request_complete(req, err); + + return err; +} + +static void get_aes_decrypt_key(unsigned char *dec_key, + const unsigned char *key, + unsigned int keylength) +{ + u32 temp; + u32 w_ring[MAX_NK]; + int i, j, k; + u8 nr, nk; + + switch (keylength) { + case AES_KEYLENGTH_128BIT: + nk = KEYLENGTH_4BYTES; + nr = NUMBER_OF_ROUNDS_10; + break; + case AES_KEYLENGTH_192BIT: + nk = KEYLENGTH_6BYTES; + nr = NUMBER_OF_ROUNDS_12; + break; + case AES_KEYLENGTH_256BIT: + nk = KEYLENGTH_8BYTES; + nr = NUMBER_OF_ROUNDS_14; + break; + default: + return; + } + for (i = 0; i < nk; i++) + w_ring[i] = get_unaligned_be32(&key[i * 4]); + + i = 0; + temp = w_ring[nk - 1]; + while (i + nk < (nr + 1) * 4) { + if (!(i % nk)) { + /* RotWord(temp) */ + temp = (temp << 8) | (temp >> 24); + temp = aes_ks_subword(temp); + temp ^= round_constant[i / nk]; + } else if (nk == 8 && (i % 4 == 0)) { + temp = aes_ks_subword(temp); + } + w_ring[i % nk] ^= temp; + temp = w_ring[i % nk]; + i++; + } + i--; + for (k = 0, j = i % nk; k < nk; k++) { + put_unaligned_be32(w_ring[j], &dec_key[k * 4]); + j--; + if (j < 0) + j += nk; + } +} + +static int chcr_prepare_hmac_key(const u8 *raw_key, unsigned int raw_key_len, + int digestsize, void *istate, void *ostate) +{ + __be32 *istate32 = istate, *ostate32 = ostate; + __be64 *istate64 = istate, *ostate64 = ostate; + union { + struct hmac_sha1_key sha1; + struct hmac_sha224_key sha224; + struct hmac_sha256_key sha256; + struct hmac_sha384_key sha384; + struct hmac_sha512_key sha512; + } k; + + switch (digestsize) { + case SHA1_DIGEST_SIZE: + hmac_sha1_preparekey(&k.sha1, raw_key, raw_key_len); + for (int i = 0; i < ARRAY_SIZE(k.sha1.istate.h); i++) { + istate32[i] = cpu_to_be32(k.sha1.istate.h[i]); + ostate32[i] = cpu_to_be32(k.sha1.ostate.h[i]); + } + break; + case SHA224_DIGEST_SIZE: + hmac_sha224_preparekey(&k.sha224, raw_key, raw_key_len); + for (int i = 0; i < ARRAY_SIZE(k.sha224.key.istate.h); i++) { + istate32[i] = cpu_to_be32(k.sha224.key.istate.h[i]); + ostate32[i] = cpu_to_be32(k.sha224.key.ostate.h[i]); + } + break; + case SHA256_DIGEST_SIZE: + hmac_sha256_preparekey(&k.sha256, raw_key, raw_key_len); + for (int i = 0; i < ARRAY_SIZE(k.sha256.key.istate.h); i++) { + istate32[i] = cpu_to_be32(k.sha256.key.istate.h[i]); + ostate32[i] = cpu_to_be32(k.sha256.key.ostate.h[i]); + } + break; + case SHA384_DIGEST_SIZE: + hmac_sha384_preparekey(&k.sha384, raw_key, raw_key_len); + for (int i = 0; i < ARRAY_SIZE(k.sha384.key.istate.h); i++) { + istate64[i] = cpu_to_be64(k.sha384.key.istate.h[i]); + ostate64[i] = cpu_to_be64(k.sha384.key.ostate.h[i]); + } + break; + case SHA512_DIGEST_SIZE: + hmac_sha512_preparekey(&k.sha512, raw_key, raw_key_len); + for (int i = 0; i < ARRAY_SIZE(k.sha512.key.istate.h); i++) { + istate64[i] = cpu_to_be64(k.sha512.key.istate.h[i]); + ostate64[i] = cpu_to_be64(k.sha512.key.ostate.h[i]); + } + break; + default: + return -EINVAL; + } + memzero_explicit(&k, sizeof(k)); + return 0; +} + +static inline int is_hmac(struct crypto_tfm *tfm) +{ + struct crypto_alg *alg = tfm->__crt_alg; + struct chcr_alg_template *chcr_crypto_alg = + container_of(__crypto_ahash_alg(alg), struct chcr_alg_template, + alg.hash); + if (chcr_crypto_alg->type == CRYPTO_ALG_TYPE_HMAC) + return 1; + return 0; +} + +static inline void dsgl_walk_init(struct dsgl_walk *walk, + struct cpl_rx_phys_dsgl *dsgl) +{ + walk->dsgl = dsgl; + walk->nents = 0; + walk->to = (struct phys_sge_pairs *)(dsgl + 1); +} + +static inline void dsgl_walk_end(struct dsgl_walk *walk, unsigned short qid, + int pci_chan_id) +{ + struct cpl_rx_phys_dsgl *phys_cpl; + + phys_cpl = walk->dsgl; + + phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL) + | CPL_RX_PHYS_DSGL_ISRDMA_V(0)); + phys_cpl->pcirlxorder_to_noofsgentr = + htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) | + CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) | + CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) | + CPL_RX_PHYS_DSGL_PCITPHNT_V(0) | + CPL_RX_PHYS_DSGL_DCAID_V(0) | + CPL_RX_PHYS_DSGL_NOOFSGENTR_V(walk->nents)); + phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR; + phys_cpl->rss_hdr_int.qid = htons(qid); + phys_cpl->rss_hdr_int.hash_val = 0; + phys_cpl->rss_hdr_int.channel = pci_chan_id; +} + +static inline void dsgl_walk_add_page(struct dsgl_walk *walk, + size_t size, + dma_addr_t addr) +{ + int j; + + if (!size) + return; + j = walk->nents; + walk->to->len[j % 8] = htons(size); + walk->to->addr[j % 8] = cpu_to_be64(addr); + j++; + if ((j % 8) == 0) + walk->to++; + walk->nents = j; +} + +static void dsgl_walk_add_sg(struct dsgl_walk *walk, + struct scatterlist *sg, + unsigned int slen, + unsigned int skip) +{ + int skip_len = 0; + unsigned int left_size = slen, len = 0; + unsigned int j = walk->nents; + int offset, ent_len; + + if (!slen) + return; + while (sg && skip) { + if (sg_dma_len(sg) <= skip) { + skip -= sg_dma_len(sg); + skip_len = 0; + sg = sg_next(sg); + } else { + skip_len = skip; + skip = 0; + } + } + + while (left_size && sg) { + len = min_t(u32, left_size, sg_dma_len(sg) - skip_len); + offset = 0; + while (len) { + ent_len = min_t(u32, len, CHCR_DST_SG_SIZE); + walk->to->len[j % 8] = htons(ent_len); + walk->to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) + + offset + skip_len); + offset += ent_len; + len -= ent_len; + j++; + if ((j % 8) == 0) + walk->to++; + } + walk->last_sg = sg; + walk->last_sg_len = min_t(u32, left_size, sg_dma_len(sg) - + skip_len) + skip_len; + left_size -= min_t(u32, left_size, sg_dma_len(sg) - skip_len); + skip_len = 0; + sg = sg_next(sg); + } + walk->nents = j; +} + +static inline void ulptx_walk_init(struct ulptx_walk *walk, + struct ulptx_sgl *ulp) +{ + walk->sgl = ulp; + walk->nents = 0; + walk->pair_idx = 0; + walk->pair = ulp->sge; + walk->last_sg = NULL; + walk->last_sg_len = 0; +} + +static inline void ulptx_walk_end(struct ulptx_walk *walk) +{ + walk->sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) | + ULPTX_NSGE_V(walk->nents)); +} + + +static inline void ulptx_walk_add_page(struct ulptx_walk *walk, + size_t size, + dma_addr_t addr) +{ + if (!size) + return; + + if (walk->nents == 0) { + walk->sgl->len0 = cpu_to_be32(size); + walk->sgl->addr0 = cpu_to_be64(addr); + } else { + walk->pair->addr[walk->pair_idx] = cpu_to_be64(addr); + walk->pair->len[walk->pair_idx] = cpu_to_be32(size); + walk->pair_idx = !walk->pair_idx; + if (!walk->pair_idx) + walk->pair++; + } + walk->nents++; +} + +static void ulptx_walk_add_sg(struct ulptx_walk *walk, + struct scatterlist *sg, + unsigned int len, + unsigned int skip) +{ + int small; + int skip_len = 0; + unsigned int sgmin; + + if (!len) + return; + while (sg && skip) { + if (sg_dma_len(sg) <= skip) { + skip -= sg_dma_len(sg); + skip_len = 0; + sg = sg_next(sg); + } else { + skip_len = skip; + skip = 0; + } + } + WARN(!sg, "SG should not be null here\n"); + if (sg && (walk->nents == 0)) { + small = min_t(unsigned int, sg_dma_len(sg) - skip_len, len); + sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE); + walk->sgl->len0 = cpu_to_be32(sgmin); + walk->sgl->addr0 = cpu_to_be64(sg_dma_address(sg) + skip_len); + walk->nents++; + len -= sgmin; + walk->last_sg = sg; + walk->last_sg_len = sgmin + skip_len; + skip_len += sgmin; + if (sg_dma_len(sg) == skip_len) { + sg = sg_next(sg); + skip_len = 0; + } + } + + while (sg && len) { + small = min(sg_dma_len(sg) - skip_len, len); + sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE); + walk->pair->len[walk->pair_idx] = cpu_to_be32(sgmin); + walk->pair->addr[walk->pair_idx] = + cpu_to_be64(sg_dma_address(sg) + skip_len); + walk->pair_idx = !walk->pair_idx; + walk->nents++; + if (!walk->pair_idx) + walk->pair++; + len -= sgmin; + skip_len += sgmin; + walk->last_sg = sg; + walk->last_sg_len = skip_len; + if (sg_dma_len(sg) == skip_len) { + sg = sg_next(sg); + skip_len = 0; + } + } +} + +static inline int get_cryptoalg_subtype(struct crypto_skcipher *tfm) +{ + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct chcr_alg_template *chcr_crypto_alg = + container_of(alg, struct chcr_alg_template, alg.skcipher); + + return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; +} + +static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx) +{ + struct adapter *adap = netdev2adap(dev); + struct sge_uld_txq_info *txq_info = + adap->sge.uld_txq_info[CXGB4_TX_CRYPTO]; + struct sge_uld_txq *txq; + int ret = 0; + + local_bh_disable(); + txq = &txq_info->uldtxq[idx]; + spin_lock(&txq->sendq.lock); + if (txq->full) + ret = -1; + spin_unlock(&txq->sendq.lock); + local_bh_enable(); + return ret; +} + +static int generate_copy_rrkey(struct ablk_ctx *ablkctx, + struct _key_ctx *key_ctx) +{ + if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) { + memcpy(key_ctx->key, ablkctx->rrkey, ablkctx->enckey_len); + } else { + memcpy(key_ctx->key, + ablkctx->key + (ablkctx->enckey_len >> 1), + ablkctx->enckey_len >> 1); + memcpy(key_ctx->key + (ablkctx->enckey_len >> 1), + ablkctx->rrkey, ablkctx->enckey_len >> 1); + } + return 0; +} + +static int chcr_hash_ent_in_wr(struct scatterlist *src, + unsigned int minsg, + unsigned int space, + unsigned int srcskip) +{ + int srclen = 0; + int srcsg = minsg; + int soffset = 0, sless; + + if (sg_dma_len(src) == srcskip) { + src = sg_next(src); + srcskip = 0; + } + while (src && space > (sgl_ent_len[srcsg + 1])) { + sless = min_t(unsigned int, sg_dma_len(src) - soffset - srcskip, + CHCR_SRC_SG_SIZE); + srclen += sless; + soffset += sless; + srcsg++; + if (sg_dma_len(src) == (soffset + srcskip)) { + src = sg_next(src); + soffset = 0; + srcskip = 0; + } + } + return srclen; +} + +static int chcr_sg_ent_in_wr(struct scatterlist *src, + struct scatterlist *dst, + unsigned int minsg, + unsigned int space, + unsigned int srcskip, + unsigned int dstskip) +{ + int srclen = 0, dstlen = 0; + int srcsg = minsg, dstsg = minsg; + int offset = 0, soffset = 0, less, sless = 0; + + if (sg_dma_len(src) == srcskip) { + src = sg_next(src); + srcskip = 0; + } + if (sg_dma_len(dst) == dstskip) { + dst = sg_next(dst); + dstskip = 0; + } + + while (src && dst && + space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) { + sless = min_t(unsigned int, sg_dma_len(src) - srcskip - soffset, + CHCR_SRC_SG_SIZE); + srclen += sless; + srcsg++; + offset = 0; + while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) && + space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) { + if (srclen <= dstlen) + break; + less = min_t(unsigned int, sg_dma_len(dst) - offset - + dstskip, CHCR_DST_SG_SIZE); + dstlen += less; + offset += less; + if ((offset + dstskip) == sg_dma_len(dst)) { + dst = sg_next(dst); + offset = 0; + } + dstsg++; + dstskip = 0; + } + soffset += sless; + if ((soffset + srcskip) == sg_dma_len(src)) { + src = sg_next(src); + srcskip = 0; + soffset = 0; + } + + } + return min(srclen, dstlen); +} + +static int chcr_cipher_fallback(struct crypto_skcipher *cipher, + struct skcipher_request *req, + u8 *iv, + unsigned short op_type) +{ + struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); + int err; + + skcipher_request_set_tfm(&reqctx->fallback_req, cipher); + skcipher_request_set_callback(&reqctx->fallback_req, req->base.flags, + req->base.complete, req->base.data); + skcipher_request_set_crypt(&reqctx->fallback_req, req->src, req->dst, + req->cryptlen, iv); + + err = op_type ? crypto_skcipher_decrypt(&reqctx->fallback_req) : + crypto_skcipher_encrypt(&reqctx->fallback_req); + + return err; + +} + +static inline int get_qidxs(struct crypto_async_request *req, + unsigned int *txqidx, unsigned int *rxqidx) +{ + struct crypto_tfm *tfm = req->tfm; + int ret = 0; + + switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { + case CRYPTO_ALG_TYPE_AEAD: + { + struct aead_request *aead_req = + container_of(req, struct aead_request, base); + struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(aead_req); + *txqidx = reqctx->txqidx; + *rxqidx = reqctx->rxqidx; + break; + } + case CRYPTO_ALG_TYPE_SKCIPHER: + { + struct skcipher_request *sk_req = + container_of(req, struct skcipher_request, base); + struct chcr_skcipher_req_ctx *reqctx = + skcipher_request_ctx(sk_req); + *txqidx = reqctx->txqidx; + *rxqidx = reqctx->rxqidx; + break; + } + case CRYPTO_ALG_TYPE_AHASH: + { + struct ahash_request *ahash_req = + container_of(req, struct ahash_request, base); + struct chcr_ahash_req_ctx *reqctx = + ahash_request_ctx(ahash_req); + *txqidx = reqctx->txqidx; + *rxqidx = reqctx->rxqidx; + break; + } + default: + ret = -EINVAL; + /* should never get here */ + BUG(); + break; + } + return ret; +} + +static inline void create_wreq(struct chcr_context *ctx, + struct chcr_wr *chcr_req, + struct crypto_async_request *req, + unsigned int imm, + int hash_sz, + unsigned int len16, + unsigned int sc_len, + unsigned int lcb) +{ + struct uld_ctx *u_ctx = ULD_CTX(ctx); + unsigned int tx_channel_id, rx_channel_id; + unsigned int txqidx = 0, rxqidx = 0; + unsigned int qid, fid, portno; + + get_qidxs(req, &txqidx, &rxqidx); + qid = u_ctx->lldi.rxq_ids[rxqidx]; + fid = u_ctx->lldi.rxq_ids[0]; + portno = rxqidx / ctx->rxq_perchan; + tx_channel_id = txqidx / ctx->txq_perchan; + rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[portno]); + + + chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE; + chcr_req->wreq.pld_size_hash_size = + htonl(FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz)); + chcr_req->wreq.len16_pkd = + htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(len16, 16))); + chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req); + chcr_req->wreq.rx_chid_to_rx_q_id = FILL_WR_RX_Q_ID(rx_channel_id, qid, + !!lcb, txqidx); + + chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(tx_channel_id, fid); + chcr_req->ulptx.len = htonl((DIV_ROUND_UP(len16, 16) - + ((sizeof(chcr_req->wreq)) >> 4))); + chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(!imm); + chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) + + sizeof(chcr_req->key_ctx) + sc_len); +} + +/** + * create_cipher_wr - form the WR for cipher operations + * @wrparam: Container for create_cipher_wr()'s parameters + */ +static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(wrparam->req); + struct chcr_context *ctx = c_ctx(tfm); + struct uld_ctx *u_ctx = ULD_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct sk_buff *skb = NULL; + struct chcr_wr *chcr_req; + struct cpl_rx_phys_dsgl *phys_cpl; + struct ulptx_sgl *ulptx; + struct chcr_skcipher_req_ctx *reqctx = + skcipher_request_ctx(wrparam->req); + unsigned int temp = 0, transhdr_len, dst_size; + int error; + int nents; + unsigned int kctx_len; + gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + struct adapter *adap = padap(ctx->dev); + unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan; + + rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]); + nents = sg_nents_xlen(reqctx->dstsg, wrparam->bytes, CHCR_DST_SG_SIZE, + reqctx->dst_ofst); + dst_size = get_space_for_phys_dsgl(nents); + kctx_len = roundup(ablkctx->enckey_len, 16); + transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); + nents = sg_nents_xlen(reqctx->srcsg, wrparam->bytes, + CHCR_SRC_SG_SIZE, reqctx->src_ofst); + temp = reqctx->imm ? roundup(wrparam->bytes, 16) : + (sgl_len(nents) * 8); + transhdr_len += temp; + transhdr_len = roundup(transhdr_len, 16); + skb = alloc_skb(SGE_MAX_WR_LEN, flags); + if (!skb) { + error = -ENOMEM; + goto err; + } + chcr_req = __skb_put_zero(skb, transhdr_len); + chcr_req->sec_cpl.op_ivinsrtofst = + FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 1); + + chcr_req->sec_cpl.pldlen = htonl(IV + wrparam->bytes); + chcr_req->sec_cpl.aadstart_cipherstop_hi = + FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, IV + 1, 0); + + chcr_req->sec_cpl.cipherstop_lo_authinsert = + FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0); + chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0, + ablkctx->ciph_mode, + 0, 0, IV >> 1); + chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0, + 0, 1, dst_size); + + chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr; + if ((reqctx->op == CHCR_DECRYPT_OP) && + (!(get_cryptoalg_subtype(tfm) == + CRYPTO_ALG_SUB_TYPE_CTR)) && + (!(get_cryptoalg_subtype(tfm) == + CRYPTO_ALG_SUB_TYPE_CTR_RFC3686))) { + generate_copy_rrkey(ablkctx, &chcr_req->key_ctx); + } else { + if ((ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) || + (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CTR)) { + memcpy(chcr_req->key_ctx.key, ablkctx->key, + ablkctx->enckey_len); + } else { + memcpy(chcr_req->key_ctx.key, ablkctx->key + + (ablkctx->enckey_len >> 1), + ablkctx->enckey_len >> 1); + memcpy(chcr_req->key_ctx.key + + (ablkctx->enckey_len >> 1), + ablkctx->key, + ablkctx->enckey_len >> 1); + } + } + phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); + ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); + chcr_add_cipher_src_ent(wrparam->req, ulptx, wrparam); + chcr_add_cipher_dst_ent(wrparam->req, phys_cpl, wrparam, wrparam->qid); + + atomic_inc(&adap->chcr_stats.cipher_rqst); + temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len + IV + + (reqctx->imm ? (wrparam->bytes) : 0); + create_wreq(c_ctx(tfm), chcr_req, &(wrparam->req->base), reqctx->imm, 0, + transhdr_len, temp, + ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC); + reqctx->skb = skb; + + if (reqctx->op && (ablkctx->ciph_mode == + CHCR_SCMD_CIPHER_MODE_AES_CBC)) + sg_pcopy_to_buffer(wrparam->req->src, + sg_nents(wrparam->req->src), wrparam->req->iv, 16, + reqctx->processed + wrparam->bytes - AES_BLOCK_SIZE); + + return skb; +err: + return ERR_PTR(error); +} + +static inline int chcr_keyctx_ck_size(unsigned int keylen) +{ + int ck_size = 0; + + if (keylen == AES_KEYSIZE_128) + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; + else if (keylen == AES_KEYSIZE_192) + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; + else if (keylen == AES_KEYSIZE_256) + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; + else + ck_size = 0; + + return ck_size; +} +static int chcr_cipher_fallback_setkey(struct crypto_skcipher *cipher, + const u8 *key, + unsigned int keylen) +{ + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); + + crypto_skcipher_clear_flags(ablkctx->sw_cipher, + CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(ablkctx->sw_cipher, + cipher->base.crt_flags & CRYPTO_TFM_REQ_MASK); + return crypto_skcipher_setkey(ablkctx->sw_cipher, key, keylen); +} + +static int chcr_aes_cbc_setkey(struct crypto_skcipher *cipher, + const u8 *key, + unsigned int keylen) +{ + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); + unsigned int ck_size, context_size; + u16 alignment = 0; + int err; + + err = chcr_cipher_fallback_setkey(cipher, key, keylen); + if (err) + goto badkey_err; + + ck_size = chcr_keyctx_ck_size(keylen); + alignment = ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192 ? 8 : 0; + memcpy(ablkctx->key, key, keylen); + ablkctx->enckey_len = keylen; + get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, keylen << 3); + context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + + keylen + alignment) >> 4; + + ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, + 0, 0, context_size); + ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC; + return 0; +badkey_err: + ablkctx->enckey_len = 0; + + return err; +} + +static int chcr_aes_ctr_setkey(struct crypto_skcipher *cipher, + const u8 *key, + unsigned int keylen) +{ + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); + unsigned int ck_size, context_size; + u16 alignment = 0; + int err; + + err = chcr_cipher_fallback_setkey(cipher, key, keylen); + if (err) + goto badkey_err; + ck_size = chcr_keyctx_ck_size(keylen); + alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0; + memcpy(ablkctx->key, key, keylen); + ablkctx->enckey_len = keylen; + context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + + keylen + alignment) >> 4; + + ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, + 0, 0, context_size); + ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR; + + return 0; +badkey_err: + ablkctx->enckey_len = 0; + + return err; +} + +static int chcr_aes_rfc3686_setkey(struct crypto_skcipher *cipher, + const u8 *key, + unsigned int keylen) +{ + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); + unsigned int ck_size, context_size; + u16 alignment = 0; + int err; + + if (keylen < CTR_RFC3686_NONCE_SIZE) + return -EINVAL; + memcpy(ablkctx->nonce, key + (keylen - CTR_RFC3686_NONCE_SIZE), + CTR_RFC3686_NONCE_SIZE); + + keylen -= CTR_RFC3686_NONCE_SIZE; + err = chcr_cipher_fallback_setkey(cipher, key, keylen); + if (err) + goto badkey_err; + + ck_size = chcr_keyctx_ck_size(keylen); + alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0; + memcpy(ablkctx->key, key, keylen); + ablkctx->enckey_len = keylen; + context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + + keylen + alignment) >> 4; + + ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, + 0, 0, context_size); + ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR; + + return 0; +badkey_err: + ablkctx->enckey_len = 0; + + return err; +} +static void ctr_add_iv(u8 *dstiv, u8 *srciv, u32 add) +{ + unsigned int size = AES_BLOCK_SIZE; + __be32 *b = (__be32 *)(dstiv + size); + u32 c, prev; + + memcpy(dstiv, srciv, AES_BLOCK_SIZE); + for (; size >= 4; size -= 4) { + prev = be32_to_cpu(*--b); + c = prev + add; + *b = cpu_to_be32(c); + if (prev < c) + break; + add = 1; + } + +} + +static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes) +{ + __be32 *b = (__be32 *)(iv + AES_BLOCK_SIZE); + u64 c; + u32 temp = be32_to_cpu(*--b); + + temp = ~temp; + c = (u64)temp + 1; // No of block can processed without overflow + if ((bytes / AES_BLOCK_SIZE) >= c) + bytes = c * AES_BLOCK_SIZE; + return bytes; +} + +static int chcr_update_tweak(struct skcipher_request *req, u8 *iv, + u32 isfinal) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); + struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); + struct aes_key aes; + int ret, i; + u8 *key; + unsigned int keylen; + int round = reqctx->last_req_len / AES_BLOCK_SIZE; + int round8 = round / 8; + + memcpy(iv, reqctx->iv, AES_BLOCK_SIZE); + + keylen = ablkctx->enckey_len / 2; + key = ablkctx->key + keylen; + /* For a 192 bit key remove the padded zeroes which was + * added in chcr_xts_setkey + */ + if (KEY_CONTEXT_CK_SIZE_G(ntohl(ablkctx->key_ctx_hdr)) + == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) + ret = aes_preparekey(&aes, key, keylen - 8); + else + ret = aes_preparekey(&aes, key, keylen); + if (ret) + return ret; + aes_encrypt(&aes, iv, iv); + for (i = 0; i < round8; i++) + gf128mul_x8_ble((le128 *)iv, (le128 *)iv); + + for (i = 0; i < (round % 8); i++) + gf128mul_x_ble((le128 *)iv, (le128 *)iv); + + if (!isfinal) + aes_decrypt(&aes, iv, iv); + + memzero_explicit(&aes, sizeof(aes)); + return 0; +} + +static int chcr_update_cipher_iv(struct skcipher_request *req, + struct cpl_fw6_pld *fw6_pld, u8 *iv) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); + int subtype = get_cryptoalg_subtype(tfm); + int ret = 0; + + if (subtype == CRYPTO_ALG_SUB_TYPE_CTR) + ctr_add_iv(iv, req->iv, (reqctx->processed / + AES_BLOCK_SIZE)); + else if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) + *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE + + CTR_RFC3686_IV_SIZE) = cpu_to_be32((reqctx->processed / + AES_BLOCK_SIZE) + 1); + else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS) + ret = chcr_update_tweak(req, iv, 0); + else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) { + if (reqctx->op) + /*Updated before sending last WR*/ + memcpy(iv, req->iv, AES_BLOCK_SIZE); + else + memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE); + } + + return ret; + +} + +/* We need separate function for final iv because in rfc3686 Initial counter + * starts from 1 and buffer size of iv is 8 byte only which remains constant + * for subsequent update requests + */ + +static int chcr_final_cipher_iv(struct skcipher_request *req, + struct cpl_fw6_pld *fw6_pld, u8 *iv) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); + int subtype = get_cryptoalg_subtype(tfm); + int ret = 0; + + if (subtype == CRYPTO_ALG_SUB_TYPE_CTR) + ctr_add_iv(iv, req->iv, DIV_ROUND_UP(reqctx->processed, + AES_BLOCK_SIZE)); + else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS) { + if (!reqctx->partial_req) + memcpy(iv, reqctx->iv, AES_BLOCK_SIZE); + else + ret = chcr_update_tweak(req, iv, 1); + } + else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) { + /*Already updated for Decrypt*/ + if (!reqctx->op) + memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE); + + } + return ret; + +} + +static int chcr_handle_cipher_resp(struct skcipher_request *req, + unsigned char *input, int err) +{ + struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input; + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); + struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); + struct chcr_dev *dev = c_ctx(tfm)->dev; + struct chcr_context *ctx = c_ctx(tfm); + struct adapter *adap = padap(ctx->dev); + struct cipher_wr_param wrparam; + struct sk_buff *skb; + int bytes; + + if (err) + goto unmap; + if (req->cryptlen == reqctx->processed) { + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, + req); + err = chcr_final_cipher_iv(req, fw6_pld, req->iv); + goto complete; + } + + if (!reqctx->imm) { + bytes = chcr_sg_ent_in_wr(reqctx->srcsg, reqctx->dstsg, 0, + CIP_SPACE_LEFT(ablkctx->enckey_len), + reqctx->src_ofst, reqctx->dst_ofst); + if ((bytes + reqctx->processed) >= req->cryptlen) + bytes = req->cryptlen - reqctx->processed; + else + bytes = rounddown(bytes, 16); + } else { + /*CTR mode counter overflow*/ + bytes = req->cryptlen - reqctx->processed; + } + err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv); + if (err) + goto unmap; + + if (unlikely(bytes == 0)) { + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, + req); + memcpy(req->iv, reqctx->init_iv, IV); + atomic_inc(&adap->chcr_stats.fallback); + err = chcr_cipher_fallback(ablkctx->sw_cipher, req, req->iv, + reqctx->op); + goto complete; + } + + if (get_cryptoalg_subtype(tfm) == + CRYPTO_ALG_SUB_TYPE_CTR) + bytes = adjust_ctr_overflow(reqctx->iv, bytes); + wrparam.qid = u_ctx->lldi.rxq_ids[reqctx->rxqidx]; + wrparam.req = req; + wrparam.bytes = bytes; + skb = create_cipher_wr(&wrparam); + if (IS_ERR(skb)) { + pr_err("%s : Failed to form WR. No memory\n", __func__); + err = PTR_ERR(skb); + goto unmap; + } + skb->dev = u_ctx->lldi.ports[0]; + set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx); + chcr_send_wr(skb); + reqctx->last_req_len = bytes; + reqctx->processed += bytes; + if (get_cryptoalg_subtype(tfm) == + CRYPTO_ALG_SUB_TYPE_CBC && req->base.flags == + CRYPTO_TFM_REQ_MAY_SLEEP ) { + complete(&ctx->cbc_aes_aio_done); + } + return 0; +unmap: + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); +complete: + if (get_cryptoalg_subtype(tfm) == + CRYPTO_ALG_SUB_TYPE_CBC && req->base.flags == + CRYPTO_TFM_REQ_MAY_SLEEP ) { + complete(&ctx->cbc_aes_aio_done); + } + chcr_dec_wrcount(dev); + skcipher_request_complete(req, err); + return err; +} + +static int process_cipher(struct skcipher_request *req, + unsigned short qid, + struct sk_buff **skb, + unsigned short op_type) +{ + struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + unsigned int ivsize = crypto_skcipher_ivsize(tfm); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); + struct adapter *adap = padap(c_ctx(tfm)->dev); + struct cipher_wr_param wrparam; + int bytes, err = -EINVAL; + int subtype; + + reqctx->processed = 0; + reqctx->partial_req = 0; + if (!req->iv) + goto error; + subtype = get_cryptoalg_subtype(tfm); + if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) || + (req->cryptlen == 0) || + (req->cryptlen % crypto_skcipher_blocksize(tfm))) { + if (req->cryptlen == 0 && subtype != CRYPTO_ALG_SUB_TYPE_XTS) + goto fallback; + else if (req->cryptlen % crypto_skcipher_blocksize(tfm) && + subtype == CRYPTO_ALG_SUB_TYPE_XTS) + goto fallback; + pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n", + ablkctx->enckey_len, req->cryptlen, ivsize); + goto error; + } + + err = chcr_cipher_dma_map(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); + if (err) + goto error; + if (req->cryptlen < (SGE_MAX_WR_LEN - (sizeof(struct chcr_wr) + + AES_MIN_KEY_SIZE + + sizeof(struct cpl_rx_phys_dsgl) + + /*Min dsgl size*/ + 32))) { + /* Can be sent as Imm*/ + unsigned int dnents = 0, transhdr_len, phys_dsgl, kctx_len; + + dnents = sg_nents_xlen(req->dst, req->cryptlen, + CHCR_DST_SG_SIZE, 0); + phys_dsgl = get_space_for_phys_dsgl(dnents); + kctx_len = roundup(ablkctx->enckey_len, 16); + transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl); + reqctx->imm = (transhdr_len + IV + req->cryptlen) <= + SGE_MAX_WR_LEN; + bytes = IV + req->cryptlen; + + } else { + reqctx->imm = 0; + } + + if (!reqctx->imm) { + bytes = chcr_sg_ent_in_wr(req->src, req->dst, 0, + CIP_SPACE_LEFT(ablkctx->enckey_len), + 0, 0); + if ((bytes + reqctx->processed) >= req->cryptlen) + bytes = req->cryptlen - reqctx->processed; + else + bytes = rounddown(bytes, 16); + } else { + bytes = req->cryptlen; + } + if (subtype == CRYPTO_ALG_SUB_TYPE_CTR) { + bytes = adjust_ctr_overflow(req->iv, bytes); + } + if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) { + memcpy(reqctx->iv, ablkctx->nonce, CTR_RFC3686_NONCE_SIZE); + memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->iv, + CTR_RFC3686_IV_SIZE); + + /* initialize counter portion of counter block */ + *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE + + CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); + memcpy(reqctx->init_iv, reqctx->iv, IV); + + } else { + + memcpy(reqctx->iv, req->iv, IV); + memcpy(reqctx->init_iv, req->iv, IV); + } + if (unlikely(bytes == 0)) { + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, + req); +fallback: atomic_inc(&adap->chcr_stats.fallback); + err = chcr_cipher_fallback(ablkctx->sw_cipher, req, + subtype == + CRYPTO_ALG_SUB_TYPE_CTR_RFC3686 ? + reqctx->iv : req->iv, + op_type); + goto error; + } + reqctx->op = op_type; + reqctx->srcsg = req->src; + reqctx->dstsg = req->dst; + reqctx->src_ofst = 0; + reqctx->dst_ofst = 0; + wrparam.qid = qid; + wrparam.req = req; + wrparam.bytes = bytes; + *skb = create_cipher_wr(&wrparam); + if (IS_ERR(*skb)) { + err = PTR_ERR(*skb); + goto unmap; + } + reqctx->processed = bytes; + reqctx->last_req_len = bytes; + reqctx->partial_req = !!(req->cryptlen - reqctx->processed); + + return 0; +unmap: + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); +error: + return err; +} + +static int chcr_aes_encrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); + struct chcr_dev *dev = c_ctx(tfm)->dev; + struct sk_buff *skb = NULL; + int err; + struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); + struct chcr_context *ctx = c_ctx(tfm); + unsigned int cpu; + + cpu = get_cpu(); + reqctx->txqidx = cpu % ctx->ntxq; + reqctx->rxqidx = cpu % ctx->nrxq; + put_cpu(); + + err = chcr_inc_wrcount(dev); + if (err) + return -ENXIO; + if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], + reqctx->txqidx) && + (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) { + err = -ENOSPC; + goto error; + } + + err = process_cipher(req, u_ctx->lldi.rxq_ids[reqctx->rxqidx], + &skb, CHCR_ENCRYPT_OP); + if (err || !skb) + return err; + skb->dev = u_ctx->lldi.ports[0]; + set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx); + chcr_send_wr(skb); + if (get_cryptoalg_subtype(tfm) == + CRYPTO_ALG_SUB_TYPE_CBC && req->base.flags == + CRYPTO_TFM_REQ_MAY_SLEEP ) { + reqctx->partial_req = 1; + wait_for_completion(&ctx->cbc_aes_aio_done); + } + return -EINPROGRESS; +error: + chcr_dec_wrcount(dev); + return err; +} + +static int chcr_aes_decrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); + struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); + struct chcr_dev *dev = c_ctx(tfm)->dev; + struct sk_buff *skb = NULL; + int err; + struct chcr_context *ctx = c_ctx(tfm); + unsigned int cpu; + + cpu = get_cpu(); + reqctx->txqidx = cpu % ctx->ntxq; + reqctx->rxqidx = cpu % ctx->nrxq; + put_cpu(); + + err = chcr_inc_wrcount(dev); + if (err) + return -ENXIO; + + if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], + reqctx->txqidx) && + (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) + return -ENOSPC; + err = process_cipher(req, u_ctx->lldi.rxq_ids[reqctx->rxqidx], + &skb, CHCR_DECRYPT_OP); + if (err || !skb) + return err; + skb->dev = u_ctx->lldi.ports[0]; + set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx); + chcr_send_wr(skb); + return -EINPROGRESS; +} +static int chcr_device_init(struct chcr_context *ctx) +{ + struct uld_ctx *u_ctx = NULL; + int txq_perchan, ntxq; + int err = 0, rxq_perchan; + + if (!ctx->dev) { + u_ctx = assign_chcr_device(); + if (!u_ctx) { + err = -ENXIO; + pr_err("chcr device assignment fails\n"); + goto out; + } + ctx->dev = &u_ctx->dev; + ntxq = u_ctx->lldi.ntxq; + rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan; + txq_perchan = ntxq / u_ctx->lldi.nchan; + ctx->ntxq = ntxq; + ctx->nrxq = u_ctx->lldi.nrxq; + ctx->rxq_perchan = rxq_perchan; + ctx->txq_perchan = txq_perchan; + } +out: + return err; +} + +static int chcr_init_tfm(struct crypto_skcipher *tfm) +{ + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct chcr_context *ctx = crypto_skcipher_ctx(tfm); + struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + + ablkctx->sw_cipher = crypto_alloc_skcipher(alg->base.cra_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ablkctx->sw_cipher)) { + pr_err("failed to allocate fallback for %s\n", alg->base.cra_name); + return PTR_ERR(ablkctx->sw_cipher); + } + init_completion(&ctx->cbc_aes_aio_done); + crypto_skcipher_set_reqsize(tfm, sizeof(struct chcr_skcipher_req_ctx) + + crypto_skcipher_reqsize(ablkctx->sw_cipher)); + + return chcr_device_init(ctx); +} + +static int chcr_rfc3686_init(struct crypto_skcipher *tfm) +{ + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct chcr_context *ctx = crypto_skcipher_ctx(tfm); + struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + + /*RFC3686 initialises IV counter value to 1, rfc3686(ctr(aes)) + * cannot be used as fallback in chcr_handle_cipher_response + */ + ablkctx->sw_cipher = crypto_alloc_skcipher("ctr(aes)", 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ablkctx->sw_cipher)) { + pr_err("failed to allocate fallback for %s\n", alg->base.cra_name); + return PTR_ERR(ablkctx->sw_cipher); + } + crypto_skcipher_set_reqsize(tfm, sizeof(struct chcr_skcipher_req_ctx) + + crypto_skcipher_reqsize(ablkctx->sw_cipher)); + return chcr_device_init(ctx); +} + + +static void chcr_exit_tfm(struct crypto_skcipher *tfm) +{ + struct chcr_context *ctx = crypto_skcipher_ctx(tfm); + struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + + crypto_free_skcipher(ablkctx->sw_cipher); +} + +static int get_alg_config(struct algo_param *params, + unsigned int auth_size) +{ + switch (auth_size) { + case SHA1_DIGEST_SIZE: + params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160; + params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA1; + params->result_size = SHA1_DIGEST_SIZE; + break; + case SHA224_DIGEST_SIZE: + params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; + params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA224; + params->result_size = SHA256_DIGEST_SIZE; + break; + case SHA256_DIGEST_SIZE: + params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; + params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA256; + params->result_size = SHA256_DIGEST_SIZE; + break; + case SHA384_DIGEST_SIZE: + params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; + params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384; + params->result_size = SHA512_DIGEST_SIZE; + break; + case SHA512_DIGEST_SIZE: + params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; + params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512; + params->result_size = SHA512_DIGEST_SIZE; + break; + default: + pr_err("ERROR, unsupported digest size\n"); + return -EINVAL; + } + return 0; +} + +/** + * create_hash_wr - Create hash work request + * @req: Cipher req base + * @param: Container for create_hash_wr()'s parameters + */ +static struct sk_buff *create_hash_wr(struct ahash_request *req, + struct hash_wr_param *param) +{ + struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct chcr_context *ctx = h_ctx(tfm); + struct hmac_ctx *hmacctx = HMAC_CTX(ctx); + struct sk_buff *skb = NULL; + struct uld_ctx *u_ctx = ULD_CTX(ctx); + struct chcr_wr *chcr_req; + struct ulptx_sgl *ulptx; + unsigned int nents = 0, transhdr_len; + unsigned int temp = 0; + gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : + GFP_ATOMIC; + struct adapter *adap = padap(h_ctx(tfm)->dev); + int error = 0; + unsigned int rx_channel_id = req_ctx->rxqidx / ctx->rxq_perchan; + + rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]); + transhdr_len = HASH_TRANSHDR_SIZE(param->kctx_len); + req_ctx->hctx_wr.imm = (transhdr_len + param->bfr_len + + param->sg_len) <= SGE_MAX_WR_LEN; + nents = sg_nents_xlen(req_ctx->hctx_wr.srcsg, param->sg_len, + CHCR_SRC_SG_SIZE, req_ctx->hctx_wr.src_ofst); + nents += param->bfr_len ? 1 : 0; + transhdr_len += req_ctx->hctx_wr.imm ? roundup(param->bfr_len + + param->sg_len, 16) : (sgl_len(nents) * 8); + transhdr_len = roundup(transhdr_len, 16); + + skb = alloc_skb(transhdr_len, flags); + if (!skb) + return ERR_PTR(-ENOMEM); + chcr_req = __skb_put_zero(skb, transhdr_len); + + chcr_req->sec_cpl.op_ivinsrtofst = + FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 0); + + chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len); + + chcr_req->sec_cpl.aadstart_cipherstop_hi = + FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0); + chcr_req->sec_cpl.cipherstop_lo_authinsert = + FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0); + chcr_req->sec_cpl.seqno_numivs = + FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode, + param->opad_needed, 0); + + chcr_req->sec_cpl.ivgen_hdrlen = + FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0); + + memcpy(chcr_req->key_ctx.key, req_ctx->partial_hash, + param->alg_prm.result_size); + + if (param->opad_needed) + memcpy(chcr_req->key_ctx.key + + ((param->alg_prm.result_size <= 32) ? 32 : + CHCR_HASH_MAX_DIGEST_SIZE), + hmacctx->opad, param->alg_prm.result_size); + + chcr_req->key_ctx.ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY, + param->alg_prm.mk_size, 0, + param->opad_needed, + ((param->kctx_len + + sizeof(chcr_req->key_ctx)) >> 4)); + chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1); + ulptx = (struct ulptx_sgl *)((u8 *)(chcr_req + 1) + param->kctx_len + + DUMMY_BYTES); + if (param->bfr_len != 0) { + req_ctx->hctx_wr.dma_addr = + dma_map_single(&u_ctx->lldi.pdev->dev, req_ctx->reqbfr, + param->bfr_len, DMA_TO_DEVICE); + if (dma_mapping_error(&u_ctx->lldi.pdev->dev, + req_ctx->hctx_wr. dma_addr)) { + error = -ENOMEM; + goto err; + } + req_ctx->hctx_wr.dma_len = param->bfr_len; + } else { + req_ctx->hctx_wr.dma_addr = 0; + } + chcr_add_hash_src_ent(req, ulptx, param); + /* Request upto max wr size */ + temp = param->kctx_len + DUMMY_BYTES + (req_ctx->hctx_wr.imm ? + (param->sg_len + param->bfr_len) : 0); + atomic_inc(&adap->chcr_stats.digest_rqst); + create_wreq(h_ctx(tfm), chcr_req, &req->base, req_ctx->hctx_wr.imm, + param->hash_size, transhdr_len, + temp, 0); + req_ctx->hctx_wr.skb = skb; + return skb; +err: + kfree_skb(skb); + return ERR_PTR(error); +} + +static int chcr_ahash_update(struct ahash_request *req) +{ + struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); + struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); + struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm)); + struct chcr_context *ctx = h_ctx(rtfm); + struct chcr_dev *dev = h_ctx(rtfm)->dev; + struct sk_buff *skb; + u8 remainder = 0, bs; + unsigned int nbytes = req->nbytes; + struct hash_wr_param params; + int error; + unsigned int cpu; + + cpu = get_cpu(); + req_ctx->txqidx = cpu % ctx->ntxq; + req_ctx->rxqidx = cpu % ctx->nrxq; + put_cpu(); + + bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); + + if (nbytes + req_ctx->reqlen >= bs) { + remainder = (nbytes + req_ctx->reqlen) % bs; + nbytes = nbytes + req_ctx->reqlen - remainder; + } else { + sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->reqbfr + + req_ctx->reqlen, nbytes, 0); + req_ctx->reqlen += nbytes; + return 0; + } + error = chcr_inc_wrcount(dev); + if (error) + return -ENXIO; + /* Detach state for CHCR means lldi or padap is freed. Increasing + * inflight count for dev guarantees that lldi and padap is valid + */ + if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], + req_ctx->txqidx) && + (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) { + error = -ENOSPC; + goto err; + } + + chcr_init_hctx_per_wr(req_ctx); + error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); + if (error) { + error = -ENOMEM; + goto err; + } + get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); + params.kctx_len = roundup(params.alg_prm.result_size, 16); + params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen, + HASH_SPACE_LEFT(params.kctx_len), 0); + if (params.sg_len > req->nbytes) + params.sg_len = req->nbytes; + params.sg_len = rounddown(params.sg_len + req_ctx->reqlen, bs) - + req_ctx->reqlen; + params.opad_needed = 0; + params.more = 1; + params.last = 0; + params.bfr_len = req_ctx->reqlen; + params.scmd1 = 0; + req_ctx->hctx_wr.srcsg = req->src; + + params.hash_size = params.alg_prm.result_size; + req_ctx->data_len += params.sg_len + params.bfr_len; + skb = create_hash_wr(req, ¶ms); + if (IS_ERR(skb)) { + error = PTR_ERR(skb); + goto unmap; + } + + req_ctx->hctx_wr.processed += params.sg_len; + if (remainder) { + /* Swap buffers */ + swap(req_ctx->reqbfr, req_ctx->skbfr); + sg_pcopy_to_buffer(req->src, sg_nents(req->src), + req_ctx->reqbfr, remainder, req->nbytes - + remainder); + } + req_ctx->reqlen = remainder; + skb->dev = u_ctx->lldi.ports[0]; + set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx); + chcr_send_wr(skb); + return -EINPROGRESS; +unmap: + chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); +err: + chcr_dec_wrcount(dev); + return error; +} + +static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1) +{ + memset(bfr_ptr, 0, bs); + *bfr_ptr = 0x80; + if (bs == 64) + *(__be64 *)(bfr_ptr + 56) = cpu_to_be64(scmd1 << 3); + else + *(__be64 *)(bfr_ptr + 120) = cpu_to_be64(scmd1 << 3); +} + +static int chcr_ahash_final(struct ahash_request *req) +{ + struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); + struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); + struct chcr_dev *dev = h_ctx(rtfm)->dev; + struct hash_wr_param params; + struct sk_buff *skb; + struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm)); + struct chcr_context *ctx = h_ctx(rtfm); + u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); + int error; + unsigned int cpu; + + cpu = get_cpu(); + req_ctx->txqidx = cpu % ctx->ntxq; + req_ctx->rxqidx = cpu % ctx->nrxq; + put_cpu(); + + error = chcr_inc_wrcount(dev); + if (error) + return -ENXIO; + + chcr_init_hctx_per_wr(req_ctx); + if (is_hmac(crypto_ahash_tfm(rtfm))) + params.opad_needed = 1; + else + params.opad_needed = 0; + params.sg_len = 0; + req_ctx->hctx_wr.isfinal = 1; + get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); + params.kctx_len = roundup(params.alg_prm.result_size, 16); + if (is_hmac(crypto_ahash_tfm(rtfm))) { + params.opad_needed = 1; + params.kctx_len *= 2; + } else { + params.opad_needed = 0; + } + + req_ctx->hctx_wr.result = 1; + params.bfr_len = req_ctx->reqlen; + req_ctx->data_len += params.bfr_len + params.sg_len; + req_ctx->hctx_wr.srcsg = req->src; + if (req_ctx->reqlen == 0) { + create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len); + params.last = 0; + params.more = 1; + params.scmd1 = 0; + params.bfr_len = bs; + + } else { + params.scmd1 = req_ctx->data_len; + params.last = 1; + params.more = 0; + } + params.hash_size = crypto_ahash_digestsize(rtfm); + skb = create_hash_wr(req, ¶ms); + if (IS_ERR(skb)) { + error = PTR_ERR(skb); + goto err; + } + req_ctx->reqlen = 0; + skb->dev = u_ctx->lldi.ports[0]; + set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx); + chcr_send_wr(skb); + return -EINPROGRESS; +err: + chcr_dec_wrcount(dev); + return error; +} + +static int chcr_ahash_finup(struct ahash_request *req) +{ + struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); + struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); + struct chcr_dev *dev = h_ctx(rtfm)->dev; + struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm)); + struct chcr_context *ctx = h_ctx(rtfm); + struct sk_buff *skb; + struct hash_wr_param params; + u8 bs; + int error; + unsigned int cpu; + + cpu = get_cpu(); + req_ctx->txqidx = cpu % ctx->ntxq; + req_ctx->rxqidx = cpu % ctx->nrxq; + put_cpu(); + + bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); + error = chcr_inc_wrcount(dev); + if (error) + return -ENXIO; + + if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], + req_ctx->txqidx) && + (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) { + error = -ENOSPC; + goto err; + } + chcr_init_hctx_per_wr(req_ctx); + error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); + if (error) { + error = -ENOMEM; + goto err; + } + + get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); + params.kctx_len = roundup(params.alg_prm.result_size, 16); + if (is_hmac(crypto_ahash_tfm(rtfm))) { + params.kctx_len *= 2; + params.opad_needed = 1; + } else { + params.opad_needed = 0; + } + + params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen, + HASH_SPACE_LEFT(params.kctx_len), 0); + if (params.sg_len < req->nbytes) { + if (is_hmac(crypto_ahash_tfm(rtfm))) { + params.kctx_len /= 2; + params.opad_needed = 0; + } + params.last = 0; + params.more = 1; + params.sg_len = rounddown(params.sg_len + req_ctx->reqlen, bs) + - req_ctx->reqlen; + params.hash_size = params.alg_prm.result_size; + params.scmd1 = 0; + } else { + params.last = 1; + params.more = 0; + params.sg_len = req->nbytes; + params.hash_size = crypto_ahash_digestsize(rtfm); + params.scmd1 = req_ctx->data_len + req_ctx->reqlen + + params.sg_len; + } + params.bfr_len = req_ctx->reqlen; + req_ctx->data_len += params.bfr_len + params.sg_len; + req_ctx->hctx_wr.result = 1; + req_ctx->hctx_wr.srcsg = req->src; + if ((req_ctx->reqlen + req->nbytes) == 0) { + create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len); + params.last = 0; + params.more = 1; + params.scmd1 = 0; + params.bfr_len = bs; + } + skb = create_hash_wr(req, ¶ms); + if (IS_ERR(skb)) { + error = PTR_ERR(skb); + goto unmap; + } + req_ctx->reqlen = 0; + req_ctx->hctx_wr.processed += params.sg_len; + skb->dev = u_ctx->lldi.ports[0]; + set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx); + chcr_send_wr(skb); + return -EINPROGRESS; +unmap: + chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); +err: + chcr_dec_wrcount(dev); + return error; +} + +static int chcr_hmac_init(struct ahash_request *areq); +static int chcr_sha_init(struct ahash_request *areq); + +static int chcr_ahash_digest(struct ahash_request *req) +{ + struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); + struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); + struct chcr_dev *dev = h_ctx(rtfm)->dev; + struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm)); + struct chcr_context *ctx = h_ctx(rtfm); + struct sk_buff *skb; + struct hash_wr_param params; + u8 bs; + int error; + unsigned int cpu; + + cpu = get_cpu(); + req_ctx->txqidx = cpu % ctx->ntxq; + req_ctx->rxqidx = cpu % ctx->nrxq; + put_cpu(); + + if (is_hmac(crypto_ahash_tfm(rtfm))) + chcr_hmac_init(req); + else + chcr_sha_init(req); + + bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); + error = chcr_inc_wrcount(dev); + if (error) + return -ENXIO; + + if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], + req_ctx->txqidx) && + (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) { + error = -ENOSPC; + goto err; + } + + chcr_init_hctx_per_wr(req_ctx); + error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); + if (error) { + error = -ENOMEM; + goto err; + } + + get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); + params.kctx_len = roundup(params.alg_prm.result_size, 16); + if (is_hmac(crypto_ahash_tfm(rtfm))) { + params.kctx_len *= 2; + params.opad_needed = 1; + } else { + params.opad_needed = 0; + } + params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen, + HASH_SPACE_LEFT(params.kctx_len), 0); + if (params.sg_len < req->nbytes) { + if (is_hmac(crypto_ahash_tfm(rtfm))) { + params.kctx_len /= 2; + params.opad_needed = 0; + } + params.last = 0; + params.more = 1; + params.scmd1 = 0; + params.sg_len = rounddown(params.sg_len, bs); + params.hash_size = params.alg_prm.result_size; + } else { + params.sg_len = req->nbytes; + params.hash_size = crypto_ahash_digestsize(rtfm); + params.last = 1; + params.more = 0; + params.scmd1 = req->nbytes + req_ctx->data_len; + + } + params.bfr_len = 0; + req_ctx->hctx_wr.result = 1; + req_ctx->hctx_wr.srcsg = req->src; + req_ctx->data_len += params.bfr_len + params.sg_len; + + if (req->nbytes == 0) { + create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len); + params.more = 1; + params.bfr_len = bs; + } + + skb = create_hash_wr(req, ¶ms); + if (IS_ERR(skb)) { + error = PTR_ERR(skb); + goto unmap; + } + req_ctx->hctx_wr.processed += params.sg_len; + skb->dev = u_ctx->lldi.ports[0]; + set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx); + chcr_send_wr(skb); + return -EINPROGRESS; +unmap: + chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); +err: + chcr_dec_wrcount(dev); + return error; +} + +static int chcr_ahash_continue(struct ahash_request *req) +{ + struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); + struct chcr_hctx_per_wr *hctx_wr = &reqctx->hctx_wr; + struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); + struct chcr_context *ctx = h_ctx(rtfm); + struct uld_ctx *u_ctx = ULD_CTX(ctx); + struct sk_buff *skb; + struct hash_wr_param params; + u8 bs; + int error; + unsigned int cpu; + + cpu = get_cpu(); + reqctx->txqidx = cpu % ctx->ntxq; + reqctx->rxqidx = cpu % ctx->nrxq; + put_cpu(); + + bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); + get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); + params.kctx_len = roundup(params.alg_prm.result_size, 16); + if (is_hmac(crypto_ahash_tfm(rtfm))) { + params.kctx_len *= 2; + params.opad_needed = 1; + } else { + params.opad_needed = 0; + } + params.sg_len = chcr_hash_ent_in_wr(hctx_wr->srcsg, 0, + HASH_SPACE_LEFT(params.kctx_len), + hctx_wr->src_ofst); + if ((params.sg_len + hctx_wr->processed) > req->nbytes) + params.sg_len = req->nbytes - hctx_wr->processed; + if (!hctx_wr->result || + ((params.sg_len + hctx_wr->processed) < req->nbytes)) { + if (is_hmac(crypto_ahash_tfm(rtfm))) { + params.kctx_len /= 2; + params.opad_needed = 0; + } + params.last = 0; + params.more = 1; + params.sg_len = rounddown(params.sg_len, bs); + params.hash_size = params.alg_prm.result_size; + params.scmd1 = 0; + } else { + params.last = 1; + params.more = 0; + params.hash_size = crypto_ahash_digestsize(rtfm); + params.scmd1 = reqctx->data_len + params.sg_len; + } + params.bfr_len = 0; + reqctx->data_len += params.sg_len; + skb = create_hash_wr(req, ¶ms); + if (IS_ERR(skb)) { + error = PTR_ERR(skb); + goto err; + } + hctx_wr->processed += params.sg_len; + skb->dev = u_ctx->lldi.ports[0]; + set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx); + chcr_send_wr(skb); + return 0; +err: + return error; +} + +static inline void chcr_handle_ahash_resp(struct ahash_request *req, + unsigned char *input, + int err) +{ + struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); + struct chcr_hctx_per_wr *hctx_wr = &reqctx->hctx_wr; + int digestsize, updated_digestsize; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm)); + struct chcr_dev *dev = h_ctx(tfm)->dev; + + if (input == NULL) + goto out; + digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req)); + updated_digestsize = digestsize; + if (digestsize == SHA224_DIGEST_SIZE) + updated_digestsize = SHA256_DIGEST_SIZE; + else if (digestsize == SHA384_DIGEST_SIZE) + updated_digestsize = SHA512_DIGEST_SIZE; + + if (hctx_wr->dma_addr) { + dma_unmap_single(&u_ctx->lldi.pdev->dev, hctx_wr->dma_addr, + hctx_wr->dma_len, DMA_TO_DEVICE); + hctx_wr->dma_addr = 0; + } + if (hctx_wr->isfinal || ((hctx_wr->processed + reqctx->reqlen) == + req->nbytes)) { + if (hctx_wr->result == 1) { + hctx_wr->result = 0; + memcpy(req->result, input + sizeof(struct cpl_fw6_pld), + digestsize); + } else { + memcpy(reqctx->partial_hash, + input + sizeof(struct cpl_fw6_pld), + updated_digestsize); + + } + goto unmap; + } + memcpy(reqctx->partial_hash, input + sizeof(struct cpl_fw6_pld), + updated_digestsize); + + err = chcr_ahash_continue(req); + if (err) + goto unmap; + return; +unmap: + if (hctx_wr->is_sg_map) + chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); + + +out: + chcr_dec_wrcount(dev); + ahash_request_complete(req, err); +} + +/* + * chcr_handle_resp - Unmap the DMA buffers associated with the request + * @req: crypto request + */ +int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, + int err) +{ + struct crypto_tfm *tfm = req->tfm; + struct chcr_context *ctx = crypto_tfm_ctx(tfm); + struct adapter *adap = padap(ctx->dev); + + switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { + case CRYPTO_ALG_TYPE_AEAD: + err = chcr_handle_aead_resp(aead_request_cast(req), input, err); + break; + + case CRYPTO_ALG_TYPE_SKCIPHER: + chcr_handle_cipher_resp(skcipher_request_cast(req), + input, err); + break; + case CRYPTO_ALG_TYPE_AHASH: + chcr_handle_ahash_resp(ahash_request_cast(req), input, err); + } + atomic_inc(&adap->chcr_stats.complete); + return err; +} +static int chcr_ahash_export(struct ahash_request *areq, void *out) +{ + struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + struct chcr_ahash_req_ctx *state = out; + + state->reqlen = req_ctx->reqlen; + state->data_len = req_ctx->data_len; + memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen); + memcpy(state->partial_hash, req_ctx->partial_hash, + CHCR_HASH_MAX_DIGEST_SIZE); + chcr_init_hctx_per_wr(state); + return 0; +} + +static int chcr_ahash_import(struct ahash_request *areq, const void *in) +{ + struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in; + + req_ctx->reqlen = state->reqlen; + req_ctx->data_len = state->data_len; + req_ctx->reqbfr = req_ctx->bfr1; + req_ctx->skbfr = req_ctx->bfr2; + memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128); + memcpy(req_ctx->partial_hash, state->partial_hash, + CHCR_HASH_MAX_DIGEST_SIZE); + chcr_init_hctx_per_wr(req_ctx); + return 0; +} + +static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm)); + + /* use the key to calculate the ipad and opad. ipad will sent with the + * first request's data. opad will be sent with the final hash result + * ipad in hmacctx->ipad and opad in hmacctx->opad location + */ + return chcr_prepare_hmac_key(key, keylen, crypto_ahash_digestsize(tfm), + hmacctx->ipad, hmacctx->opad); +} + +static int chcr_aes_xts_setkey(struct crypto_skcipher *cipher, const u8 *key, + unsigned int key_len) +{ + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); + unsigned short context_size = 0; + int err; + + err = chcr_cipher_fallback_setkey(cipher, key, key_len); + if (err) + goto badkey_err; + + memcpy(ablkctx->key, key, key_len); + ablkctx->enckey_len = key_len; + get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, key_len << 2); + context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4; + /* Both keys for xts must be aligned to 16 byte boundary + * by padding with zeros. So for 24 byte keys padding 8 zeroes. + */ + if (key_len == 48) { + context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len + + 16) >> 4; + memmove(ablkctx->key + 32, ablkctx->key + 24, 24); + memset(ablkctx->key + 24, 0, 8); + memset(ablkctx->key + 56, 0, 8); + ablkctx->enckey_len = 64; + ablkctx->key_ctx_hdr = + FILL_KEY_CTX_HDR(CHCR_KEYCTX_CIPHER_KEY_SIZE_192, + CHCR_KEYCTX_NO_KEY, 1, + 0, context_size); + } else { + ablkctx->key_ctx_hdr = + FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ? + CHCR_KEYCTX_CIPHER_KEY_SIZE_128 : + CHCR_KEYCTX_CIPHER_KEY_SIZE_256, + CHCR_KEYCTX_NO_KEY, 1, + 0, context_size); + } + ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS; + return 0; +badkey_err: + ablkctx->enckey_len = 0; + + return err; +} + +static int chcr_sha_init(struct ahash_request *areq) +{ + struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + int digestsize = crypto_ahash_digestsize(tfm); + + req_ctx->data_len = 0; + req_ctx->reqlen = 0; + req_ctx->reqbfr = req_ctx->bfr1; + req_ctx->skbfr = req_ctx->bfr2; + copy_hash_init_values(req_ctx->partial_hash, digestsize); + + return 0; +} + +static int chcr_sha_cra_init(struct crypto_tfm *tfm) +{ + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct chcr_ahash_req_ctx)); + return chcr_device_init(crypto_tfm_ctx(tfm)); +} + +static int chcr_hmac_init(struct ahash_request *areq) +{ + struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq); + struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(rtfm)); + unsigned int digestsize = crypto_ahash_digestsize(rtfm); + unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); + + chcr_sha_init(areq); + req_ctx->data_len = bs; + if (is_hmac(crypto_ahash_tfm(rtfm))) { + if (digestsize == SHA224_DIGEST_SIZE) + memcpy(req_ctx->partial_hash, hmacctx->ipad, + SHA256_DIGEST_SIZE); + else if (digestsize == SHA384_DIGEST_SIZE) + memcpy(req_ctx->partial_hash, hmacctx->ipad, + SHA512_DIGEST_SIZE); + else + memcpy(req_ctx->partial_hash, hmacctx->ipad, + digestsize); + } + return 0; +} + +static int chcr_hmac_cra_init(struct crypto_tfm *tfm) +{ + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct chcr_ahash_req_ctx)); + return chcr_device_init(crypto_tfm_ctx(tfm)); +} + +inline void chcr_aead_common_exit(struct aead_request *req) +{ + struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm)); + + chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op); +} + +static int chcr_aead_common_init(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); + struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req); + unsigned int authsize = crypto_aead_authsize(tfm); + int error = -EINVAL; + + /* validate key size */ + if (aeadctx->enckey_len == 0) + goto err; + if (reqctx->op && req->cryptlen < authsize) + goto err; + if (reqctx->b0_len) + reqctx->scratch_pad = reqctx->iv + IV; + else + reqctx->scratch_pad = NULL; + + error = chcr_aead_dma_map(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, + reqctx->op); + if (error) { + error = -ENOMEM; + goto err; + } + + return 0; +err: + return error; +} + +static int chcr_aead_need_fallback(struct aead_request *req, int dst_nents, + int aadmax, int wrlen, + unsigned short op_type) +{ + unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req)); + + if (((req->cryptlen - (op_type ? authsize : 0)) == 0) || + dst_nents > MAX_DSGL_ENT || + (req->assoclen > aadmax) || + (wrlen > SGE_MAX_WR_LEN)) + return 1; + return 0; +} + +static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); + struct aead_request *subreq = aead_request_ctx_dma(req); + + aead_request_set_tfm(subreq, aeadctx->sw_cipher); + aead_request_set_callback(subreq, req->base.flags, + req->base.complete, req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, + req->iv); + aead_request_set_ad(subreq, req->assoclen); + return op_type ? crypto_aead_decrypt(subreq) : + crypto_aead_encrypt(subreq); +} + +static struct sk_buff *create_authenc_wr(struct aead_request *req, + unsigned short qid, + int size) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct chcr_context *ctx = a_ctx(tfm); + struct uld_ctx *u_ctx = ULD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); + struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req); + struct sk_buff *skb = NULL; + struct chcr_wr *chcr_req; + struct cpl_rx_phys_dsgl *phys_cpl; + struct ulptx_sgl *ulptx; + unsigned int transhdr_len; + unsigned int dst_size = 0, temp, subtype = get_aead_subtype(tfm); + unsigned int kctx_len = 0, dnents, snents; + unsigned int authsize = crypto_aead_authsize(tfm); + int error = -EINVAL; + u8 *ivptr; + int null = 0; + gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : + GFP_ATOMIC; + struct adapter *adap = padap(ctx->dev); + unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan; + + rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]); + if (req->cryptlen == 0) + return NULL; + + reqctx->b0_len = 0; + error = chcr_aead_common_init(req); + if (error) + return ERR_PTR(error); + + if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL || + subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) { + null = 1; + } + dnents = sg_nents_xlen(req->dst, req->assoclen + req->cryptlen + + (reqctx->op ? -authsize : authsize), CHCR_DST_SG_SIZE, 0); + dnents += MIN_AUTH_SG; // For IV + snents = sg_nents_xlen(req->src, req->assoclen + req->cryptlen, + CHCR_SRC_SG_SIZE, 0); + dst_size = get_space_for_phys_dsgl(dnents); + kctx_len = (KEY_CONTEXT_CTX_LEN_G(ntohl(aeadctx->key_ctx_hdr)) << 4) + - sizeof(chcr_req->key_ctx); + transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); + reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen) < + SGE_MAX_WR_LEN; + temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen, 16) + : (sgl_len(snents) * 8); + transhdr_len += temp; + transhdr_len = roundup(transhdr_len, 16); + + if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE, + transhdr_len, reqctx->op)) { + atomic_inc(&adap->chcr_stats.fallback); + chcr_aead_common_exit(req); + return ERR_PTR(chcr_aead_fallback(req, reqctx->op)); + } + skb = alloc_skb(transhdr_len, flags); + if (!skb) { + error = -ENOMEM; + goto err; + } + + chcr_req = __skb_put_zero(skb, transhdr_len); + + temp = (reqctx->op == CHCR_ENCRYPT_OP) ? 0 : authsize; + + /* + * Input order is AAD,IV and Payload. where IV should be included as + * the part of authdata. All other fields should be filled according + * to the hardware spec + */ + chcr_req->sec_cpl.op_ivinsrtofst = + FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 1); + chcr_req->sec_cpl.pldlen = htonl(req->assoclen + IV + req->cryptlen); + chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( + null ? 0 : 1 + IV, + null ? 0 : IV + req->assoclen, + req->assoclen + IV + 1, + (temp & 0x1F0) >> 4); + chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT( + temp & 0xF, + null ? 0 : req->assoclen + IV + 1, + temp, temp); + if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL || + subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA) + temp = CHCR_SCMD_CIPHER_MODE_AES_CTR; + else + temp = CHCR_SCMD_CIPHER_MODE_AES_CBC; + chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, + (reqctx->op == CHCR_ENCRYPT_OP) ? 1 : 0, + temp, + actx->auth_mode, aeadctx->hmac_ctrl, + IV >> 1); + chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, + 0, 0, dst_size); + + chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; + if (reqctx->op == CHCR_ENCRYPT_OP || + subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA || + subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) + memcpy(chcr_req->key_ctx.key, aeadctx->key, + aeadctx->enckey_len); + else + memcpy(chcr_req->key_ctx.key, actx->dec_rrkey, + aeadctx->enckey_len); + + memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16), + actx->h_iopad, kctx_len - roundup(aeadctx->enckey_len, 16)); + phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); + ivptr = (u8 *)(phys_cpl + 1) + dst_size; + ulptx = (struct ulptx_sgl *)(ivptr + IV); + if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA || + subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) { + memcpy(ivptr, aeadctx->nonce, CTR_RFC3686_NONCE_SIZE); + memcpy(ivptr + CTR_RFC3686_NONCE_SIZE, req->iv, + CTR_RFC3686_IV_SIZE); + *(__be32 *)(ivptr + CTR_RFC3686_NONCE_SIZE + + CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); + } else { + memcpy(ivptr, req->iv, IV); + } + chcr_add_aead_dst_ent(req, phys_cpl, qid); + chcr_add_aead_src_ent(req, ulptx); + atomic_inc(&adap->chcr_stats.cipher_rqst); + temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV + + kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen) : 0); + create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size, + transhdr_len, temp, 0); + reqctx->skb = skb; + + return skb; +err: + chcr_aead_common_exit(req); + + return ERR_PTR(error); +} + +int chcr_aead_dma_map(struct device *dev, + struct aead_request *req, + unsigned short op_type) +{ + int error; + struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + unsigned int authsize = crypto_aead_authsize(tfm); + int src_len, dst_len; + + /* calculate and handle src and dst sg length separately + * for inplace and out-of place operations + */ + if (req->src == req->dst) { + src_len = req->assoclen + req->cryptlen + (op_type ? + 0 : authsize); + dst_len = src_len; + } else { + src_len = req->assoclen + req->cryptlen; + dst_len = req->assoclen + req->cryptlen + (op_type ? + -authsize : authsize); + } + + if (!req->cryptlen || !src_len || !dst_len) + return 0; + reqctx->iv_dma = dma_map_single(dev, reqctx->iv, (IV + reqctx->b0_len), + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, reqctx->iv_dma)) + return -ENOMEM; + if (reqctx->b0_len) + reqctx->b0_dma = reqctx->iv_dma + IV; + else + reqctx->b0_dma = 0; + if (req->src == req->dst) { + error = dma_map_sg(dev, req->src, + sg_nents_for_len(req->src, src_len), + DMA_BIDIRECTIONAL); + if (!error) + goto err; + } else { + error = dma_map_sg(dev, req->src, + sg_nents_for_len(req->src, src_len), + DMA_TO_DEVICE); + if (!error) + goto err; + error = dma_map_sg(dev, req->dst, + sg_nents_for_len(req->dst, dst_len), + DMA_FROM_DEVICE); + if (!error) { + dma_unmap_sg(dev, req->src, + sg_nents_for_len(req->src, src_len), + DMA_TO_DEVICE); + goto err; + } + } + + return 0; +err: + dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); + return -ENOMEM; +} + +void chcr_aead_dma_unmap(struct device *dev, + struct aead_request *req, + unsigned short op_type) +{ + struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + unsigned int authsize = crypto_aead_authsize(tfm); + int src_len, dst_len; + + /* calculate and handle src and dst sg length separately + * for inplace and out-of place operations + */ + if (req->src == req->dst) { + src_len = req->assoclen + req->cryptlen + (op_type ? + 0 : authsize); + dst_len = src_len; + } else { + src_len = req->assoclen + req->cryptlen; + dst_len = req->assoclen + req->cryptlen + (op_type ? + -authsize : authsize); + } + + if (!req->cryptlen || !src_len || !dst_len) + return; + + dma_unmap_single(dev, reqctx->iv_dma, (IV + reqctx->b0_len), + DMA_BIDIRECTIONAL); + if (req->src == req->dst) { + dma_unmap_sg(dev, req->src, + sg_nents_for_len(req->src, src_len), + DMA_BIDIRECTIONAL); + } else { + dma_unmap_sg(dev, req->src, + sg_nents_for_len(req->src, src_len), + DMA_TO_DEVICE); + dma_unmap_sg(dev, req->dst, + sg_nents_for_len(req->dst, dst_len), + DMA_FROM_DEVICE); + } +} + +void chcr_add_aead_src_ent(struct aead_request *req, + struct ulptx_sgl *ulptx) +{ + struct ulptx_walk ulp_walk; + struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req); + + if (reqctx->imm) { + u8 *buf = (u8 *)ulptx; + + if (reqctx->b0_len) { + memcpy(buf, reqctx->scratch_pad, reqctx->b0_len); + buf += reqctx->b0_len; + } + sg_pcopy_to_buffer(req->src, sg_nents(req->src), + buf, req->cryptlen + req->assoclen, 0); + } else { + ulptx_walk_init(&ulp_walk, ulptx); + if (reqctx->b0_len) + ulptx_walk_add_page(&ulp_walk, reqctx->b0_len, + reqctx->b0_dma); + ulptx_walk_add_sg(&ulp_walk, req->src, req->cryptlen + + req->assoclen, 0); + ulptx_walk_end(&ulp_walk); + } +} + +void chcr_add_aead_dst_ent(struct aead_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + unsigned short qid) +{ + struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct dsgl_walk dsgl_walk; + unsigned int authsize = crypto_aead_authsize(tfm); + struct chcr_context *ctx = a_ctx(tfm); + struct uld_ctx *u_ctx = ULD_CTX(ctx); + u32 temp; + unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan; + + rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]); + dsgl_walk_init(&dsgl_walk, phys_cpl); + dsgl_walk_add_page(&dsgl_walk, IV + reqctx->b0_len, reqctx->iv_dma); + temp = req->assoclen + req->cryptlen + + (reqctx->op ? -authsize : authsize); + dsgl_walk_add_sg(&dsgl_walk, req->dst, temp, 0); + dsgl_walk_end(&dsgl_walk, qid, rx_channel_id); +} + +void chcr_add_cipher_src_ent(struct skcipher_request *req, + void *ulptx, + struct cipher_wr_param *wrparam) +{ + struct ulptx_walk ulp_walk; + struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); + u8 *buf = ulptx; + + memcpy(buf, reqctx->iv, IV); + buf += IV; + if (reqctx->imm) { + sg_pcopy_to_buffer(req->src, sg_nents(req->src), + buf, wrparam->bytes, reqctx->processed); + } else { + ulptx_walk_init(&ulp_walk, (struct ulptx_sgl *)buf); + ulptx_walk_add_sg(&ulp_walk, reqctx->srcsg, wrparam->bytes, + reqctx->src_ofst); + reqctx->srcsg = ulp_walk.last_sg; + reqctx->src_ofst = ulp_walk.last_sg_len; + ulptx_walk_end(&ulp_walk); + } +} + +void chcr_add_cipher_dst_ent(struct skcipher_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + struct cipher_wr_param *wrparam, + unsigned short qid) +{ + struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(wrparam->req); + struct chcr_context *ctx = c_ctx(tfm); + struct uld_ctx *u_ctx = ULD_CTX(ctx); + struct dsgl_walk dsgl_walk; + unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan; + + rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]); + dsgl_walk_init(&dsgl_walk, phys_cpl); + dsgl_walk_add_sg(&dsgl_walk, reqctx->dstsg, wrparam->bytes, + reqctx->dst_ofst); + reqctx->dstsg = dsgl_walk.last_sg; + reqctx->dst_ofst = dsgl_walk.last_sg_len; + dsgl_walk_end(&dsgl_walk, qid, rx_channel_id); +} + +void chcr_add_hash_src_ent(struct ahash_request *req, + struct ulptx_sgl *ulptx, + struct hash_wr_param *param) +{ + struct ulptx_walk ulp_walk; + struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); + + if (reqctx->hctx_wr.imm) { + u8 *buf = (u8 *)ulptx; + + if (param->bfr_len) { + memcpy(buf, reqctx->reqbfr, param->bfr_len); + buf += param->bfr_len; + } + + sg_pcopy_to_buffer(reqctx->hctx_wr.srcsg, + sg_nents(reqctx->hctx_wr.srcsg), buf, + param->sg_len, 0); + } else { + ulptx_walk_init(&ulp_walk, ulptx); + if (param->bfr_len) + ulptx_walk_add_page(&ulp_walk, param->bfr_len, + reqctx->hctx_wr.dma_addr); + ulptx_walk_add_sg(&ulp_walk, reqctx->hctx_wr.srcsg, + param->sg_len, reqctx->hctx_wr.src_ofst); + reqctx->hctx_wr.srcsg = ulp_walk.last_sg; + reqctx->hctx_wr.src_ofst = ulp_walk.last_sg_len; + ulptx_walk_end(&ulp_walk); + } +} + +int chcr_hash_dma_map(struct device *dev, + struct ahash_request *req) +{ + struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); + int error = 0; + + if (!req->nbytes) + return 0; + error = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + if (!error) + return -ENOMEM; + req_ctx->hctx_wr.is_sg_map = 1; + return 0; +} + +void chcr_hash_dma_unmap(struct device *dev, + struct ahash_request *req) +{ + struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); + + if (!req->nbytes) + return; + + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + req_ctx->hctx_wr.is_sg_map = 0; + +} + +int chcr_cipher_dma_map(struct device *dev, + struct skcipher_request *req) +{ + int error; + + if (req->src == req->dst) { + error = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_BIDIRECTIONAL); + if (!error) + goto err; + } else { + error = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + if (!error) + goto err; + error = dma_map_sg(dev, req->dst, sg_nents(req->dst), + DMA_FROM_DEVICE); + if (!error) { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + goto err; + } + } + + return 0; +err: + return -ENOMEM; +} + +void chcr_cipher_dma_unmap(struct device *dev, + struct skcipher_request *req) +{ + if (req->src == req->dst) { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_BIDIRECTIONAL); + } else { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + dma_unmap_sg(dev, req->dst, sg_nents(req->dst), + DMA_FROM_DEVICE); + } +} + +static int set_msg_len(u8 *block, unsigned int msglen, int csize) +{ + __be32 data; + + memset(block, 0, csize); + block += csize; + + if (csize >= 4) + csize = 4; + else if (msglen > (unsigned int)(1 << (8 * csize))) + return -EOVERFLOW; + + data = cpu_to_be32(msglen); + memcpy(block - csize, (u8 *)&data + 4 - csize, csize); + + return 0; +} + +static int generate_b0(struct aead_request *req, u8 *ivptr, + unsigned short op_type) +{ + unsigned int l, lp, m; + int rc; + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req); + u8 *b0 = reqctx->scratch_pad; + + m = crypto_aead_authsize(aead); + + memcpy(b0, ivptr, 16); + + lp = b0[0]; + l = lp + 1; + + /* set m, bits 3-5 */ + *b0 |= (8 * ((m - 2) / 2)); + + /* set adata, bit 6, if associated data is used */ + if (req->assoclen) + *b0 |= 64; + rc = set_msg_len(b0 + 16 - l, + (op_type == CHCR_DECRYPT_OP) ? + req->cryptlen - m : req->cryptlen, l); + + return rc; +} + +static inline int crypto_ccm_check_iv(const u8 *iv) +{ + /* 2 <= L <= 8, so 1 <= L' <= 7. */ + if (iv[0] < 1 || iv[0] > 7) + return -EINVAL; + + return 0; +} + +static int ccm_format_packet(struct aead_request *req, + u8 *ivptr, + unsigned int sub_type, + unsigned short op_type, + unsigned int assoclen) +{ + struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); + int rc = 0; + + if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) { + ivptr[0] = 3; + memcpy(ivptr + 1, &aeadctx->salt[0], 3); + memcpy(ivptr + 4, req->iv, 8); + memset(ivptr + 12, 0, 4); + } else { + memcpy(ivptr, req->iv, 16); + } + if (assoclen) + put_unaligned_be16(assoclen, &reqctx->scratch_pad[16]); + + rc = generate_b0(req, ivptr, op_type); + /* zero the ctr value */ + memset(ivptr + 15 - ivptr[0], 0, ivptr[0] + 1); + return rc; +} + +static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, + unsigned int dst_size, + struct aead_request *req, + unsigned short op_type) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct chcr_context *ctx = a_ctx(tfm); + struct uld_ctx *u_ctx = ULD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req); + unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM; + unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC; + unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan; + unsigned int ccm_xtra; + unsigned int tag_offset = 0, auth_offset = 0; + unsigned int assoclen; + + rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]); + + if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) + assoclen = req->assoclen - 8; + else + assoclen = req->assoclen; + ccm_xtra = CCM_B0_SIZE + + ((assoclen) ? CCM_AAD_FIELD_SIZE : 0); + + auth_offset = req->cryptlen ? + (req->assoclen + IV + 1 + ccm_xtra) : 0; + if (op_type == CHCR_DECRYPT_OP) { + if (crypto_aead_authsize(tfm) != req->cryptlen) + tag_offset = crypto_aead_authsize(tfm); + else + auth_offset = 0; + } + + sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 1); + sec_cpl->pldlen = + htonl(req->assoclen + IV + req->cryptlen + ccm_xtra); + /* For CCM there wil be b0 always. So AAD start will be 1 always */ + sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( + 1 + IV, IV + assoclen + ccm_xtra, + req->assoclen + IV + 1 + ccm_xtra, 0); + + sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0, + auth_offset, tag_offset, + (op_type == CHCR_ENCRYPT_OP) ? 0 : + crypto_aead_authsize(tfm)); + sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, + (op_type == CHCR_ENCRYPT_OP) ? 0 : 1, + cipher_mode, mac_mode, + aeadctx->hmac_ctrl, IV >> 1); + + sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0, + 0, dst_size); +} + +static int aead_ccm_validate_input(unsigned short op_type, + struct aead_request *req, + struct chcr_aead_ctx *aeadctx, + unsigned int sub_type) +{ + if (sub_type != CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) { + if (crypto_ccm_check_iv(req->iv)) { + pr_err("CCM: IV check fails\n"); + return -EINVAL; + } + } else { + if (req->assoclen != 16 && req->assoclen != 20) { + pr_err("RFC4309: Invalid AAD length %d\n", + req->assoclen); + return -EINVAL; + } + } + return 0; +} + +static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, + unsigned short qid, + int size) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); + struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req); + struct sk_buff *skb = NULL; + struct chcr_wr *chcr_req; + struct cpl_rx_phys_dsgl *phys_cpl; + struct ulptx_sgl *ulptx; + unsigned int transhdr_len; + unsigned int dst_size = 0, kctx_len, dnents, temp, snents; + unsigned int sub_type, assoclen = req->assoclen; + unsigned int authsize = crypto_aead_authsize(tfm); + int error = -EINVAL; + u8 *ivptr; + gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : + GFP_ATOMIC; + struct adapter *adap = padap(a_ctx(tfm)->dev); + + sub_type = get_aead_subtype(tfm); + if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) + assoclen -= 8; + reqctx->b0_len = CCM_B0_SIZE + (assoclen ? CCM_AAD_FIELD_SIZE : 0); + error = chcr_aead_common_init(req); + if (error) + return ERR_PTR(error); + + error = aead_ccm_validate_input(reqctx->op, req, aeadctx, sub_type); + if (error) + goto err; + dnents = sg_nents_xlen(req->dst, req->assoclen + req->cryptlen + + (reqctx->op ? -authsize : authsize), + CHCR_DST_SG_SIZE, 0); + dnents += MIN_CCM_SG; // For IV and B0 + dst_size = get_space_for_phys_dsgl(dnents); + snents = sg_nents_xlen(req->src, req->assoclen + req->cryptlen, + CHCR_SRC_SG_SIZE, 0); + snents += MIN_CCM_SG; //For B0 + kctx_len = roundup(aeadctx->enckey_len, 16) * 2; + transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); + reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen + + reqctx->b0_len) <= SGE_MAX_WR_LEN; + temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen + + reqctx->b0_len, 16) : + (sgl_len(snents) * 8); + transhdr_len += temp; + transhdr_len = roundup(transhdr_len, 16); + + if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE - + reqctx->b0_len, transhdr_len, reqctx->op)) { + atomic_inc(&adap->chcr_stats.fallback); + chcr_aead_common_exit(req); + return ERR_PTR(chcr_aead_fallback(req, reqctx->op)); + } + skb = alloc_skb(transhdr_len, flags); + + if (!skb) { + error = -ENOMEM; + goto err; + } + + chcr_req = __skb_put_zero(skb, transhdr_len); + + fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, reqctx->op); + + chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; + memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); + memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16), + aeadctx->key, aeadctx->enckey_len); + + phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); + ivptr = (u8 *)(phys_cpl + 1) + dst_size; + ulptx = (struct ulptx_sgl *)(ivptr + IV); + error = ccm_format_packet(req, ivptr, sub_type, reqctx->op, assoclen); + if (error) + goto dstmap_fail; + chcr_add_aead_dst_ent(req, phys_cpl, qid); + chcr_add_aead_src_ent(req, ulptx); + + atomic_inc(&adap->chcr_stats.aead_rqst); + temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV + + kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen + + reqctx->b0_len) : 0); + create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, 0, + transhdr_len, temp, 0); + reqctx->skb = skb; + + return skb; +dstmap_fail: + kfree_skb(skb); +err: + chcr_aead_common_exit(req); + return ERR_PTR(error); +} + +static struct sk_buff *create_gcm_wr(struct aead_request *req, + unsigned short qid, + int size) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct chcr_context *ctx = a_ctx(tfm); + struct uld_ctx *u_ctx = ULD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req); + struct sk_buff *skb = NULL; + struct chcr_wr *chcr_req; + struct cpl_rx_phys_dsgl *phys_cpl; + struct ulptx_sgl *ulptx; + unsigned int transhdr_len, dnents = 0, snents; + unsigned int dst_size = 0, temp = 0, kctx_len, assoclen = req->assoclen; + unsigned int authsize = crypto_aead_authsize(tfm); + int error = -EINVAL; + u8 *ivptr; + gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : + GFP_ATOMIC; + struct adapter *adap = padap(ctx->dev); + unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan; + + rx_channel_id = cxgb4_port_e2cchan(u_ctx->lldi.ports[rx_channel_id]); + if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) + assoclen = req->assoclen - 8; + + reqctx->b0_len = 0; + error = chcr_aead_common_init(req); + if (error) + return ERR_PTR(error); + dnents = sg_nents_xlen(req->dst, req->assoclen + req->cryptlen + + (reqctx->op ? -authsize : authsize), + CHCR_DST_SG_SIZE, 0); + snents = sg_nents_xlen(req->src, req->assoclen + req->cryptlen, + CHCR_SRC_SG_SIZE, 0); + dnents += MIN_GCM_SG; // For IV + dst_size = get_space_for_phys_dsgl(dnents); + kctx_len = roundup(aeadctx->enckey_len, 16) + AEAD_H_SIZE; + transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); + reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen) <= + SGE_MAX_WR_LEN; + temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen, 16) : + (sgl_len(snents) * 8); + transhdr_len += temp; + transhdr_len = roundup(transhdr_len, 16); + if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE, + transhdr_len, reqctx->op)) { + + atomic_inc(&adap->chcr_stats.fallback); + chcr_aead_common_exit(req); + return ERR_PTR(chcr_aead_fallback(req, reqctx->op)); + } + skb = alloc_skb(transhdr_len, flags); + if (!skb) { + error = -ENOMEM; + goto err; + } + + chcr_req = __skb_put_zero(skb, transhdr_len); + + //Offset of tag from end + temp = (reqctx->op == CHCR_ENCRYPT_OP) ? 0 : authsize; + chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR( + rx_channel_id, 2, 1); + chcr_req->sec_cpl.pldlen = + htonl(req->assoclen + IV + req->cryptlen); + chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( + assoclen ? 1 + IV : 0, + assoclen ? IV + assoclen : 0, + req->assoclen + IV + 1, 0); + chcr_req->sec_cpl.cipherstop_lo_authinsert = + FILL_SEC_CPL_AUTHINSERT(0, req->assoclen + IV + 1, + temp, temp); + chcr_req->sec_cpl.seqno_numivs = + FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, (reqctx->op == + CHCR_ENCRYPT_OP) ? 1 : 0, + CHCR_SCMD_CIPHER_MODE_AES_GCM, + CHCR_SCMD_AUTH_MODE_GHASH, + aeadctx->hmac_ctrl, IV >> 1); + chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, + 0, 0, dst_size); + chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; + memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); + memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16), + GCM_CTX(aeadctx)->ghash_h, AEAD_H_SIZE); + + phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); + ivptr = (u8 *)(phys_cpl + 1) + dst_size; + /* prepare a 16 byte iv */ + /* S A L T | IV | 0x00000001 */ + if (get_aead_subtype(tfm) == + CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) { + memcpy(ivptr, aeadctx->salt, 4); + memcpy(ivptr + 4, req->iv, GCM_RFC4106_IV_SIZE); + } else { + memcpy(ivptr, req->iv, GCM_AES_IV_SIZE); + } + put_unaligned_be32(0x01, &ivptr[12]); + ulptx = (struct ulptx_sgl *)(ivptr + 16); + + chcr_add_aead_dst_ent(req, phys_cpl, qid); + chcr_add_aead_src_ent(req, ulptx); + atomic_inc(&adap->chcr_stats.aead_rqst); + temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV + + kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen) : 0); + create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size, + transhdr_len, temp, reqctx->verify); + reqctx->skb = skb; + return skb; + +err: + chcr_aead_common_exit(req); + return ERR_PTR(error); +} + + + +static int chcr_aead_cra_init(struct crypto_aead *tfm) +{ + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); + struct aead_alg *alg = crypto_aead_alg(tfm); + + aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0, + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC); + if (IS_ERR(aeadctx->sw_cipher)) + return PTR_ERR(aeadctx->sw_cipher); + crypto_aead_set_reqsize_dma( + tfm, max(sizeof(struct chcr_aead_reqctx), + sizeof(struct aead_request) + + crypto_aead_reqsize(aeadctx->sw_cipher))); + return chcr_device_init(a_ctx(tfm)); +} + +static void chcr_aead_cra_exit(struct crypto_aead *tfm) +{ + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); + + crypto_free_aead(aeadctx->sw_cipher); +} + +static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); + + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP; + aeadctx->mayverify = VERIFY_HW; + return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); +} +static int chcr_authenc_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); + u32 maxauth = crypto_aead_maxauthsize(tfm); + + /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not + * true for sha1. authsize == 12 condition should be before + * authsize == (maxauth >> 1) + */ + if (authsize == ICV_4) { + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; + aeadctx->mayverify = VERIFY_HW; + } else if (authsize == ICV_6) { + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2; + aeadctx->mayverify = VERIFY_HW; + } else if (authsize == ICV_10) { + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366; + aeadctx->mayverify = VERIFY_HW; + } else if (authsize == ICV_12) { + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; + aeadctx->mayverify = VERIFY_HW; + } else if (authsize == ICV_14) { + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; + aeadctx->mayverify = VERIFY_HW; + } else if (authsize == (maxauth >> 1)) { + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; + aeadctx->mayverify = VERIFY_HW; + } else if (authsize == maxauth) { + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; + aeadctx->mayverify = VERIFY_HW; + } else { + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; + aeadctx->mayverify = VERIFY_SW; + } + return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); +} + + +static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) +{ + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); + + switch (authsize) { + case ICV_4: + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; + aeadctx->mayverify = VERIFY_HW; + break; + case ICV_8: + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; + aeadctx->mayverify = VERIFY_HW; + break; + case ICV_12: + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; + aeadctx->mayverify = VERIFY_HW; + break; + case ICV_14: + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; + aeadctx->mayverify = VERIFY_HW; + break; + case ICV_16: + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; + aeadctx->mayverify = VERIFY_HW; + break; + case ICV_13: + case ICV_15: + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; + aeadctx->mayverify = VERIFY_SW; + break; + default: + return -EINVAL; + } + return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); +} + +static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); + + switch (authsize) { + case ICV_8: + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; + aeadctx->mayverify = VERIFY_HW; + break; + case ICV_12: + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; + aeadctx->mayverify = VERIFY_HW; + break; + case ICV_16: + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; + aeadctx->mayverify = VERIFY_HW; + break; + default: + return -EINVAL; + } + return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); +} + +static int chcr_ccm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); + + switch (authsize) { + case ICV_4: + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; + aeadctx->mayverify = VERIFY_HW; + break; + case ICV_6: + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2; + aeadctx->mayverify = VERIFY_HW; + break; + case ICV_8: + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; + aeadctx->mayverify = VERIFY_HW; + break; + case ICV_10: + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366; + aeadctx->mayverify = VERIFY_HW; + break; + case ICV_12: + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; + aeadctx->mayverify = VERIFY_HW; + break; + case ICV_14: + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; + aeadctx->mayverify = VERIFY_HW; + break; + case ICV_16: + aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; + aeadctx->mayverify = VERIFY_HW; + break; + default: + return -EINVAL; + } + return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); +} + +static int chcr_ccm_common_setkey(struct crypto_aead *aead, + const u8 *key, + unsigned int keylen) +{ + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); + unsigned char ck_size, mk_size; + int key_ctx_size = 0; + + key_ctx_size = sizeof(struct _key_ctx) + roundup(keylen, 16) * 2; + if (keylen == AES_KEYSIZE_128) { + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; + mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128; + } else if (keylen == AES_KEYSIZE_192) { + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; + mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192; + } else if (keylen == AES_KEYSIZE_256) { + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; + mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; + } else { + aeadctx->enckey_len = 0; + return -EINVAL; + } + aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, mk_size, 0, 0, + key_ctx_size >> 4); + memcpy(aeadctx->key, key, keylen); + aeadctx->enckey_len = keylen; + + return 0; +} + +static int chcr_aead_ccm_setkey(struct crypto_aead *aead, + const u8 *key, + unsigned int keylen) +{ + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); + int error; + + crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) & + CRYPTO_TFM_REQ_MASK); + error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); + if (error) + return error; + return chcr_ccm_common_setkey(aead, key, keylen); +} + +static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); + int error; + + if (keylen < 3) { + aeadctx->enckey_len = 0; + return -EINVAL; + } + crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) & + CRYPTO_TFM_REQ_MASK); + error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); + if (error) + return error; + keylen -= 3; + memcpy(aeadctx->salt, key + keylen, 3); + return chcr_ccm_common_setkey(aead, key, keylen); +} + +static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); + struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx); + unsigned int ck_size; + int ret = 0, key_ctx_size = 0; + struct aes_enckey aes; + + aeadctx->enckey_len = 0; + crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) + & CRYPTO_TFM_REQ_MASK); + ret = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); + if (ret) + goto out; + + if (get_aead_subtype(aead) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 && + keylen > 3) { + keylen -= 4; /* nonce/salt is present in the last 4 bytes */ + memcpy(aeadctx->salt, key + keylen, 4); + } + if (keylen == AES_KEYSIZE_128) { + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; + } else if (keylen == AES_KEYSIZE_192) { + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; + } else if (keylen == AES_KEYSIZE_256) { + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; + } else { + pr_err("GCM: Invalid key length %d\n", keylen); + ret = -EINVAL; + goto out; + } + + memcpy(aeadctx->key, key, keylen); + aeadctx->enckey_len = keylen; + key_ctx_size = sizeof(struct _key_ctx) + roundup(keylen, 16) + + AEAD_H_SIZE; + aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, + CHCR_KEYCTX_MAC_KEY_SIZE_128, + 0, 0, + key_ctx_size >> 4); + /* Calculate the H = CIPH(K, 0 repeated 16 times). + * It will go in key context + */ + ret = aes_prepareenckey(&aes, key, keylen); + if (ret) { + aeadctx->enckey_len = 0; + goto out; + } + memset(gctx->ghash_h, 0, AEAD_H_SIZE); + aes_encrypt(&aes, gctx->ghash_h, gctx->ghash_h); + memzero_explicit(&aes, sizeof(aes)); + +out: + return ret; +} + +static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, + unsigned int keylen) +{ + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc)); + struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); + /* it contains auth and cipher key both*/ + struct crypto_authenc_keys keys; + unsigned int subtype; + unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize; + int err = 0, key_ctx_len = 0; + unsigned char ck_size = 0; + struct algo_param param; + int align; + + crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc) + & CRYPTO_TFM_REQ_MASK); + err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); + if (err) + goto out; + + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) + goto out; + + if (get_alg_config(¶m, max_authsize)) { + pr_err("Unsupported digest size\n"); + goto out; + } + subtype = get_aead_subtype(authenc); + if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA || + subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) { + if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE) + goto out; + memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen + - CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE); + keys.enckeylen -= CTR_RFC3686_NONCE_SIZE; + } + if (keys.enckeylen == AES_KEYSIZE_128) { + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; + } else if (keys.enckeylen == AES_KEYSIZE_192) { + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; + } else if (keys.enckeylen == AES_KEYSIZE_256) { + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; + } else { + pr_err("Unsupported cipher key\n"); + goto out; + } + + /* Copy only encryption key. We use authkey to generate h(ipad) and + * h(opad) so authkey is not needed again. authkeylen size have the + * size of the hash digest size. + */ + memcpy(aeadctx->key, keys.enckey, keys.enckeylen); + aeadctx->enckey_len = keys.enckeylen; + if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA || + subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) { + + get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key, + aeadctx->enckey_len << 3); + } + + align = KEYCTX_ALIGN_PAD(max_authsize); + err = chcr_prepare_hmac_key(keys.authkey, keys.authkeylen, max_authsize, + actx->h_iopad, + actx->h_iopad + param.result_size + align); + if (err) + goto out; + + key_ctx_len = sizeof(struct _key_ctx) + roundup(keys.enckeylen, 16) + + (param.result_size + align) * 2; + aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size, 0, 1, + key_ctx_len >> 4); + actx->auth_mode = param.auth_mode; + + memzero_explicit(&keys, sizeof(keys)); + return 0; + +out: + aeadctx->enckey_len = 0; + memzero_explicit(&keys, sizeof(keys)); + return -EINVAL; +} + +static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc, + const u8 *key, unsigned int keylen) +{ + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc)); + struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); + struct crypto_authenc_keys keys; + int err; + /* it contains auth and cipher key both*/ + unsigned int subtype; + int key_ctx_len = 0; + unsigned char ck_size = 0; + + crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc) + & CRYPTO_TFM_REQ_MASK); + err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); + if (err) + goto out; + + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) + goto out; + + subtype = get_aead_subtype(authenc); + if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA || + subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) { + if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE) + goto out; + memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen + - CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE); + keys.enckeylen -= CTR_RFC3686_NONCE_SIZE; + } + if (keys.enckeylen == AES_KEYSIZE_128) { + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; + } else if (keys.enckeylen == AES_KEYSIZE_192) { + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; + } else if (keys.enckeylen == AES_KEYSIZE_256) { + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; + } else { + pr_err("Unsupported cipher key %d\n", keys.enckeylen); + goto out; + } + memcpy(aeadctx->key, keys.enckey, keys.enckeylen); + aeadctx->enckey_len = keys.enckeylen; + if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA || + subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) { + get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key, + aeadctx->enckey_len << 3); + } + key_ctx_len = sizeof(struct _key_ctx) + roundup(keys.enckeylen, 16); + + aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 0, + 0, key_ctx_len >> 4); + actx->auth_mode = CHCR_SCMD_AUTH_MODE_NOP; + memzero_explicit(&keys, sizeof(keys)); + return 0; +out: + aeadctx->enckey_len = 0; + memzero_explicit(&keys, sizeof(keys)); + return -EINVAL; +} + +static int chcr_aead_op(struct aead_request *req, + int size, + create_wr_t create_wr_fn) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req); + struct chcr_context *ctx = a_ctx(tfm); + struct uld_ctx *u_ctx = ULD_CTX(ctx); + struct sk_buff *skb; + struct chcr_dev *cdev; + + cdev = a_ctx(tfm)->dev; + if (!cdev) { + pr_err("%s : No crypto device.\n", __func__); + return -ENXIO; + } + + if (chcr_inc_wrcount(cdev)) { + /* Detach state for CHCR means lldi or padap is freed. + * We cannot increment fallback here. + */ + return chcr_aead_fallback(req, reqctx->op); + } + + if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], + reqctx->txqidx) && + (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))) { + chcr_dec_wrcount(cdev); + return -ENOSPC; + } + + if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 && + crypto_ipsec_check_assoclen(req->assoclen) != 0) { + pr_err("RFC4106: Invalid value of assoclen %d\n", + req->assoclen); + return -EINVAL; + } + + /* Form a WR from req */ + skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[reqctx->rxqidx], size); + + if (IS_ERR_OR_NULL(skb)) { + chcr_dec_wrcount(cdev); + return PTR_ERR_OR_ZERO(skb); + } + + skb->dev = u_ctx->lldi.ports[0]; + set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx); + chcr_send_wr(skb); + return -EINPROGRESS; +} + +static int chcr_aead_encrypt(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req); + struct chcr_context *ctx = a_ctx(tfm); + unsigned int cpu; + + cpu = get_cpu(); + reqctx->txqidx = cpu % ctx->ntxq; + reqctx->rxqidx = cpu % ctx->nrxq; + put_cpu(); + + reqctx->verify = VERIFY_HW; + reqctx->op = CHCR_ENCRYPT_OP; + + switch (get_aead_subtype(tfm)) { + case CRYPTO_ALG_SUB_TYPE_CTR_SHA: + case CRYPTO_ALG_SUB_TYPE_CBC_SHA: + case CRYPTO_ALG_SUB_TYPE_CBC_NULL: + case CRYPTO_ALG_SUB_TYPE_CTR_NULL: + return chcr_aead_op(req, 0, create_authenc_wr); + case CRYPTO_ALG_SUB_TYPE_AEAD_CCM: + case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309: + return chcr_aead_op(req, 0, create_aead_ccm_wr); + default: + return chcr_aead_op(req, 0, create_gcm_wr); + } +} + +static int chcr_aead_decrypt(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct chcr_context *ctx = a_ctx(tfm); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_reqctx *reqctx = aead_request_ctx_dma(req); + int size; + unsigned int cpu; + + cpu = get_cpu(); + reqctx->txqidx = cpu % ctx->ntxq; + reqctx->rxqidx = cpu % ctx->nrxq; + put_cpu(); + + if (aeadctx->mayverify == VERIFY_SW) { + size = crypto_aead_maxauthsize(tfm); + reqctx->verify = VERIFY_SW; + } else { + size = 0; + reqctx->verify = VERIFY_HW; + } + reqctx->op = CHCR_DECRYPT_OP; + switch (get_aead_subtype(tfm)) { + case CRYPTO_ALG_SUB_TYPE_CBC_SHA: + case CRYPTO_ALG_SUB_TYPE_CTR_SHA: + case CRYPTO_ALG_SUB_TYPE_CBC_NULL: + case CRYPTO_ALG_SUB_TYPE_CTR_NULL: + return chcr_aead_op(req, size, create_authenc_wr); + case CRYPTO_ALG_SUB_TYPE_AEAD_CCM: + case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309: + return chcr_aead_op(req, size, create_aead_ccm_wr); + default: + return chcr_aead_op(req, size, create_gcm_wr); + } +} + +static struct chcr_alg_template driver_algs[] = { + /* AES-CBC */ + { + .type = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_SUB_TYPE_CBC, + .is_registered = 0, + .alg.skcipher = { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-chcr", + .base.cra_blocksize = AES_BLOCK_SIZE, + + .init = chcr_init_tfm, + .exit = chcr_exit_tfm, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = chcr_aes_cbc_setkey, + .encrypt = chcr_aes_encrypt, + .decrypt = chcr_aes_decrypt, + } + }, + { + .type = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_SUB_TYPE_XTS, + .is_registered = 0, + .alg.skcipher = { + .base.cra_name = "xts(aes)", + .base.cra_driver_name = "xts-aes-chcr", + .base.cra_blocksize = AES_BLOCK_SIZE, + + .init = chcr_init_tfm, + .exit = chcr_exit_tfm, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = chcr_aes_xts_setkey, + .encrypt = chcr_aes_encrypt, + .decrypt = chcr_aes_decrypt, + } + }, + { + .type = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_SUB_TYPE_CTR, + .is_registered = 0, + .alg.skcipher = { + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "ctr-aes-chcr", + .base.cra_blocksize = 1, + + .init = chcr_init_tfm, + .exit = chcr_exit_tfm, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = chcr_aes_ctr_setkey, + .encrypt = chcr_aes_encrypt, + .decrypt = chcr_aes_decrypt, + } + }, + { + .type = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_SUB_TYPE_CTR_RFC3686, + .is_registered = 0, + .alg.skcipher = { + .base.cra_name = "rfc3686(ctr(aes))", + .base.cra_driver_name = "rfc3686-ctr-aes-chcr", + .base.cra_blocksize = 1, + + .init = chcr_rfc3686_init, + .exit = chcr_exit_tfm, + .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + .setkey = chcr_aes_rfc3686_setkey, + .encrypt = chcr_aes_encrypt, + .decrypt = chcr_aes_decrypt, + } + }, + /* SHA */ + { + .type = CRYPTO_ALG_TYPE_AHASH, + .is_registered = 0, + .alg.hash = { + .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-chcr", + .cra_blocksize = SHA1_BLOCK_SIZE, + } + } + }, + { + .type = CRYPTO_ALG_TYPE_AHASH, + .is_registered = 0, + .alg.hash = { + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-chcr", + .cra_blocksize = SHA256_BLOCK_SIZE, + } + } + }, + { + .type = CRYPTO_ALG_TYPE_AHASH, + .is_registered = 0, + .alg.hash = { + .halg.digestsize = SHA224_DIGEST_SIZE, + .halg.base = { + .cra_name = "sha224", + .cra_driver_name = "sha224-chcr", + .cra_blocksize = SHA224_BLOCK_SIZE, + } + } + }, + { + .type = CRYPTO_ALG_TYPE_AHASH, + .is_registered = 0, + .alg.hash = { + .halg.digestsize = SHA384_DIGEST_SIZE, + .halg.base = { + .cra_name = "sha384", + .cra_driver_name = "sha384-chcr", + .cra_blocksize = SHA384_BLOCK_SIZE, + } + } + }, + { + .type = CRYPTO_ALG_TYPE_AHASH, + .is_registered = 0, + .alg.hash = { + .halg.digestsize = SHA512_DIGEST_SIZE, + .halg.base = { + .cra_name = "sha512", + .cra_driver_name = "sha512-chcr", + .cra_blocksize = SHA512_BLOCK_SIZE, + } + } + }, + /* HMAC */ + { + .type = CRYPTO_ALG_TYPE_HMAC, + .is_registered = 0, + .alg.hash = { + .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.base = { + .cra_name = "hmac(sha1)", + .cra_driver_name = "hmac-sha1-chcr", + .cra_blocksize = SHA1_BLOCK_SIZE, + } + } + }, + { + .type = CRYPTO_ALG_TYPE_HMAC, + .is_registered = 0, + .alg.hash = { + .halg.digestsize = SHA224_DIGEST_SIZE, + .halg.base = { + .cra_name = "hmac(sha224)", + .cra_driver_name = "hmac-sha224-chcr", + .cra_blocksize = SHA224_BLOCK_SIZE, + } + } + }, + { + .type = CRYPTO_ALG_TYPE_HMAC, + .is_registered = 0, + .alg.hash = { + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "hmac-sha256-chcr", + .cra_blocksize = SHA256_BLOCK_SIZE, + } + } + }, + { + .type = CRYPTO_ALG_TYPE_HMAC, + .is_registered = 0, + .alg.hash = { + .halg.digestsize = SHA384_DIGEST_SIZE, + .halg.base = { + .cra_name = "hmac(sha384)", + .cra_driver_name = "hmac-sha384-chcr", + .cra_blocksize = SHA384_BLOCK_SIZE, + } + } + }, + { + .type = CRYPTO_ALG_TYPE_HMAC, + .is_registered = 0, + .alg.hash = { + .halg.digestsize = SHA512_DIGEST_SIZE, + .halg.base = { + .cra_name = "hmac(sha512)", + .cra_driver_name = "hmac-sha512-chcr", + .cra_blocksize = SHA512_BLOCK_SIZE, + } + } + }, + /* Add AEAD Algorithms */ + { + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_GCM, + .is_registered = 0, + .alg.aead = { + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "gcm-aes-chcr", + .cra_blocksize = 1, + .cra_priority = CHCR_AEAD_PRIORITY, + .cra_ctxsize = sizeof(struct chcr_context) + + sizeof(struct chcr_aead_ctx) + + sizeof(struct chcr_gcm_ctx), + }, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = GHASH_DIGEST_SIZE, + .setkey = chcr_gcm_setkey, + .setauthsize = chcr_gcm_setauthsize, + } + }, + { + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106, + .is_registered = 0, + .alg.aead = { + .base = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "rfc4106-gcm-aes-chcr", + .cra_blocksize = 1, + .cra_priority = CHCR_AEAD_PRIORITY + 1, + .cra_ctxsize = sizeof(struct chcr_context) + + sizeof(struct chcr_aead_ctx) + + sizeof(struct chcr_gcm_ctx), + + }, + .ivsize = GCM_RFC4106_IV_SIZE, + .maxauthsize = GHASH_DIGEST_SIZE, + .setkey = chcr_gcm_setkey, + .setauthsize = chcr_4106_4309_setauthsize, + } + }, + { + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_CCM, + .is_registered = 0, + .alg.aead = { + .base = { + .cra_name = "ccm(aes)", + .cra_driver_name = "ccm-aes-chcr", + .cra_blocksize = 1, + .cra_priority = CHCR_AEAD_PRIORITY, + .cra_ctxsize = sizeof(struct chcr_context) + + sizeof(struct chcr_aead_ctx), + + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = GHASH_DIGEST_SIZE, + .setkey = chcr_aead_ccm_setkey, + .setauthsize = chcr_ccm_setauthsize, + } + }, + { + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309, + .is_registered = 0, + .alg.aead = { + .base = { + .cra_name = "rfc4309(ccm(aes))", + .cra_driver_name = "rfc4309-ccm-aes-chcr", + .cra_blocksize = 1, + .cra_priority = CHCR_AEAD_PRIORITY + 1, + .cra_ctxsize = sizeof(struct chcr_context) + + sizeof(struct chcr_aead_ctx), + + }, + .ivsize = 8, + .maxauthsize = GHASH_DIGEST_SIZE, + .setkey = chcr_aead_rfc4309_setkey, + .setauthsize = chcr_4106_4309_setauthsize, + } + }, + { + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, + .is_registered = 0, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = + "authenc-hmac-sha1-cbc-aes-chcr", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_priority = CHCR_AEAD_PRIORITY, + .cra_ctxsize = sizeof(struct chcr_context) + + sizeof(struct chcr_aead_ctx) + + sizeof(struct chcr_authenc_ctx), + + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + .setkey = chcr_authenc_setkey, + .setauthsize = chcr_authenc_setauthsize, + } + }, + { + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, + .is_registered = 0, + .alg.aead = { + .base = { + + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = + "authenc-hmac-sha256-cbc-aes-chcr", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_priority = CHCR_AEAD_PRIORITY, + .cra_ctxsize = sizeof(struct chcr_context) + + sizeof(struct chcr_aead_ctx) + + sizeof(struct chcr_authenc_ctx), + + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + .setkey = chcr_authenc_setkey, + .setauthsize = chcr_authenc_setauthsize, + } + }, + { + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, + .is_registered = 0, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(aes))", + .cra_driver_name = + "authenc-hmac-sha224-cbc-aes-chcr", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_priority = CHCR_AEAD_PRIORITY, + .cra_ctxsize = sizeof(struct chcr_context) + + sizeof(struct chcr_aead_ctx) + + sizeof(struct chcr_authenc_ctx), + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + .setkey = chcr_authenc_setkey, + .setauthsize = chcr_authenc_setauthsize, + } + }, + { + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, + .is_registered = 0, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(aes))", + .cra_driver_name = + "authenc-hmac-sha384-cbc-aes-chcr", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_priority = CHCR_AEAD_PRIORITY, + .cra_ctxsize = sizeof(struct chcr_context) + + sizeof(struct chcr_aead_ctx) + + sizeof(struct chcr_authenc_ctx), + + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + .setkey = chcr_authenc_setkey, + .setauthsize = chcr_authenc_setauthsize, + } + }, + { + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, + .is_registered = 0, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(aes))", + .cra_driver_name = + "authenc-hmac-sha512-cbc-aes-chcr", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_priority = CHCR_AEAD_PRIORITY, + .cra_ctxsize = sizeof(struct chcr_context) + + sizeof(struct chcr_aead_ctx) + + sizeof(struct chcr_authenc_ctx), + + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + .setkey = chcr_authenc_setkey, + .setauthsize = chcr_authenc_setauthsize, + } + }, + { + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_NULL, + .is_registered = 0, + .alg.aead = { + .base = { + .cra_name = "authenc(digest_null,cbc(aes))", + .cra_driver_name = + "authenc-digest_null-cbc-aes-chcr", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_priority = CHCR_AEAD_PRIORITY, + .cra_ctxsize = sizeof(struct chcr_context) + + sizeof(struct chcr_aead_ctx) + + sizeof(struct chcr_authenc_ctx), + + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = 0, + .setkey = chcr_aead_digest_null_setkey, + .setauthsize = chcr_authenc_null_setauthsize, + } + }, + { + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, + .is_registered = 0, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", + .cra_driver_name = + "authenc-hmac-sha1-rfc3686-ctr-aes-chcr", + .cra_blocksize = 1, + .cra_priority = CHCR_AEAD_PRIORITY, + .cra_ctxsize = sizeof(struct chcr_context) + + sizeof(struct chcr_aead_ctx) + + sizeof(struct chcr_authenc_ctx), + + }, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + .setkey = chcr_authenc_setkey, + .setauthsize = chcr_authenc_setauthsize, + } + }, + { + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, + .is_registered = 0, + .alg.aead = { + .base = { + + .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", + .cra_driver_name = + "authenc-hmac-sha256-rfc3686-ctr-aes-chcr", + .cra_blocksize = 1, + .cra_priority = CHCR_AEAD_PRIORITY, + .cra_ctxsize = sizeof(struct chcr_context) + + sizeof(struct chcr_aead_ctx) + + sizeof(struct chcr_authenc_ctx), + + }, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + .setkey = chcr_authenc_setkey, + .setauthsize = chcr_authenc_setauthsize, + } + }, + { + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, + .is_registered = 0, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))", + .cra_driver_name = + "authenc-hmac-sha224-rfc3686-ctr-aes-chcr", + .cra_blocksize = 1, + .cra_priority = CHCR_AEAD_PRIORITY, + .cra_ctxsize = sizeof(struct chcr_context) + + sizeof(struct chcr_aead_ctx) + + sizeof(struct chcr_authenc_ctx), + }, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + .setkey = chcr_authenc_setkey, + .setauthsize = chcr_authenc_setauthsize, + } + }, + { + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, + .is_registered = 0, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))", + .cra_driver_name = + "authenc-hmac-sha384-rfc3686-ctr-aes-chcr", + .cra_blocksize = 1, + .cra_priority = CHCR_AEAD_PRIORITY, + .cra_ctxsize = sizeof(struct chcr_context) + + sizeof(struct chcr_aead_ctx) + + sizeof(struct chcr_authenc_ctx), + + }, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + .setkey = chcr_authenc_setkey, + .setauthsize = chcr_authenc_setauthsize, + } + }, + { + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, + .is_registered = 0, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))", + .cra_driver_name = + "authenc-hmac-sha512-rfc3686-ctr-aes-chcr", + .cra_blocksize = 1, + .cra_priority = CHCR_AEAD_PRIORITY, + .cra_ctxsize = sizeof(struct chcr_context) + + sizeof(struct chcr_aead_ctx) + + sizeof(struct chcr_authenc_ctx), + + }, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + .setkey = chcr_authenc_setkey, + .setauthsize = chcr_authenc_setauthsize, + } + }, + { + .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_NULL, + .is_registered = 0, + .alg.aead = { + .base = { + .cra_name = "authenc(digest_null,rfc3686(ctr(aes)))", + .cra_driver_name = + "authenc-digest_null-rfc3686-ctr-aes-chcr", + .cra_blocksize = 1, + .cra_priority = CHCR_AEAD_PRIORITY, + .cra_ctxsize = sizeof(struct chcr_context) + + sizeof(struct chcr_aead_ctx) + + sizeof(struct chcr_authenc_ctx), + + }, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = 0, + .setkey = chcr_aead_digest_null_setkey, + .setauthsize = chcr_authenc_null_setauthsize, + } + }, +}; + +/* + * chcr_unregister_alg - Deregister crypto algorithms with + * kernel framework. + */ +static int chcr_unregister_alg(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { + switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) { + case CRYPTO_ALG_TYPE_SKCIPHER: + if (driver_algs[i].is_registered && refcount_read( + &driver_algs[i].alg.skcipher.base.cra_refcnt) + == 1) { + crypto_unregister_skcipher( + &driver_algs[i].alg.skcipher); + driver_algs[i].is_registered = 0; + } + break; + case CRYPTO_ALG_TYPE_AEAD: + if (driver_algs[i].is_registered && refcount_read( + &driver_algs[i].alg.aead.base.cra_refcnt) == 1) { + crypto_unregister_aead( + &driver_algs[i].alg.aead); + driver_algs[i].is_registered = 0; + } + break; + case CRYPTO_ALG_TYPE_AHASH: + if (driver_algs[i].is_registered && refcount_read( + &driver_algs[i].alg.hash.halg.base.cra_refcnt) + == 1) { + crypto_unregister_ahash( + &driver_algs[i].alg.hash); + driver_algs[i].is_registered = 0; + } + break; + } + } + return 0; +} + +#define SZ_AHASH_CTX sizeof(struct chcr_context) +#define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx)) +#define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx) + +/* + * chcr_register_alg - Register crypto algorithms with kernel framework. + */ +static int chcr_register_alg(void) +{ + struct crypto_alg ai; + struct ahash_alg *a_hash; + int err = 0, i; + char *name = NULL; + + for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { + if (driver_algs[i].is_registered) + continue; + switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) { + case CRYPTO_ALG_TYPE_SKCIPHER: + driver_algs[i].alg.skcipher.base.cra_priority = + CHCR_CRA_PRIORITY; + driver_algs[i].alg.skcipher.base.cra_module = THIS_MODULE; + driver_algs[i].alg.skcipher.base.cra_flags = + CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_NEED_FALLBACK; + driver_algs[i].alg.skcipher.base.cra_ctxsize = + sizeof(struct chcr_context) + + sizeof(struct ablk_ctx); + driver_algs[i].alg.skcipher.base.cra_alignmask = 0; + + err = crypto_register_skcipher(&driver_algs[i].alg.skcipher); + name = driver_algs[i].alg.skcipher.base.cra_driver_name; + break; + case CRYPTO_ALG_TYPE_AEAD: + driver_algs[i].alg.aead.base.cra_flags = + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ALLOCATES_MEMORY; + driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt; + driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt; + driver_algs[i].alg.aead.init = chcr_aead_cra_init; + driver_algs[i].alg.aead.exit = chcr_aead_cra_exit; + driver_algs[i].alg.aead.base.cra_module = THIS_MODULE; + err = crypto_register_aead(&driver_algs[i].alg.aead); + name = driver_algs[i].alg.aead.base.cra_driver_name; + break; + case CRYPTO_ALG_TYPE_AHASH: + a_hash = &driver_algs[i].alg.hash; + a_hash->update = chcr_ahash_update; + a_hash->final = chcr_ahash_final; + a_hash->finup = chcr_ahash_finup; + a_hash->digest = chcr_ahash_digest; + a_hash->export = chcr_ahash_export; + a_hash->import = chcr_ahash_import; + a_hash->halg.statesize = SZ_AHASH_REQ_CTX; + a_hash->halg.base.cra_priority = CHCR_CRA_PRIORITY; + a_hash->halg.base.cra_module = THIS_MODULE; + a_hash->halg.base.cra_flags = + CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY; + a_hash->halg.base.cra_alignmask = 0; + a_hash->halg.base.cra_exit = NULL; + + if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) { + a_hash->halg.base.cra_init = chcr_hmac_cra_init; + a_hash->init = chcr_hmac_init; + a_hash->setkey = chcr_ahash_setkey; + a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX; + } else { + a_hash->init = chcr_sha_init; + a_hash->halg.base.cra_ctxsize = SZ_AHASH_CTX; + a_hash->halg.base.cra_init = chcr_sha_cra_init; + } + err = crypto_register_ahash(&driver_algs[i].alg.hash); + ai = driver_algs[i].alg.hash.halg.base; + name = ai.cra_driver_name; + break; + } + if (err) { + pr_err("%s : Algorithm registration failed\n", name); + goto register_err; + } else { + driver_algs[i].is_registered = 1; + } + } + return 0; + +register_err: + chcr_unregister_alg(); + return err; +} + +/* + * start_crypto - Register the crypto algorithms. + * This should called once when the first device comesup. After this + * kernel will start calling driver APIs for crypto operations. + */ +int start_crypto(void) +{ + return chcr_register_alg(); +} + +/* + * stop_crypto - Deregister all the crypto algorithms with kernel. + * This should be called once when the last device goes down. After this + * kernel will not call the driver API for crypto operations. + */ +int stop_crypto(void) +{ + chcr_unregister_alg(); + return 0; +} diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h new file mode 100644 index 000000000..507aafe93 --- /dev/null +++ b/drivers/crypto/chelsio/chcr_algo.h @@ -0,0 +1,405 @@ +/* + * This file is part of the Chelsio T6 Crypto driver for Linux. + * + * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +#ifndef __CHCR_ALGO_H__ +#define __CHCR_ALGO_H__ + +/* Crypto key context */ +#define KEY_CONTEXT_CTX_LEN_S 24 +#define KEY_CONTEXT_CTX_LEN_M 0xff +#define KEY_CONTEXT_CTX_LEN_V(x) ((x) << KEY_CONTEXT_CTX_LEN_S) +#define KEY_CONTEXT_CTX_LEN_G(x) \ + (((x) >> KEY_CONTEXT_CTX_LEN_S) & KEY_CONTEXT_CTX_LEN_M) + +#define KEY_CONTEXT_DUAL_CK_S 12 +#define KEY_CONTEXT_DUAL_CK_M 0x1 +#define KEY_CONTEXT_DUAL_CK_V(x) ((x) << KEY_CONTEXT_DUAL_CK_S) +#define KEY_CONTEXT_DUAL_CK_G(x) \ +(((x) >> KEY_CONTEXT_DUAL_CK_S) & KEY_CONTEXT_DUAL_CK_M) +#define KEY_CONTEXT_DUAL_CK_F KEY_CONTEXT_DUAL_CK_V(1U) + +#define KEY_CONTEXT_SALT_PRESENT_S 10 +#define KEY_CONTEXT_SALT_PRESENT_M 0x1 +#define KEY_CONTEXT_SALT_PRESENT_V(x) ((x) << KEY_CONTEXT_SALT_PRESENT_S) +#define KEY_CONTEXT_SALT_PRESENT_G(x) \ + (((x) >> KEY_CONTEXT_SALT_PRESENT_S) & \ + KEY_CONTEXT_SALT_PRESENT_M) +#define KEY_CONTEXT_SALT_PRESENT_F KEY_CONTEXT_SALT_PRESENT_V(1U) + +#define KEY_CONTEXT_VALID_S 0 +#define KEY_CONTEXT_VALID_M 0x1 +#define KEY_CONTEXT_VALID_V(x) ((x) << KEY_CONTEXT_VALID_S) +#define KEY_CONTEXT_VALID_G(x) \ + (((x) >> KEY_CONTEXT_VALID_S) & \ + KEY_CONTEXT_VALID_M) +#define KEY_CONTEXT_VALID_F KEY_CONTEXT_VALID_V(1U) + +#define KEY_CONTEXT_CK_SIZE_S 6 +#define KEY_CONTEXT_CK_SIZE_M 0xf +#define KEY_CONTEXT_CK_SIZE_V(x) ((x) << KEY_CONTEXT_CK_SIZE_S) +#define KEY_CONTEXT_CK_SIZE_G(x) \ + (((x) >> KEY_CONTEXT_CK_SIZE_S) & KEY_CONTEXT_CK_SIZE_M) + +#define KEY_CONTEXT_MK_SIZE_S 2 +#define KEY_CONTEXT_MK_SIZE_M 0xf +#define KEY_CONTEXT_MK_SIZE_V(x) ((x) << KEY_CONTEXT_MK_SIZE_S) +#define KEY_CONTEXT_MK_SIZE_G(x) \ + (((x) >> KEY_CONTEXT_MK_SIZE_S) & KEY_CONTEXT_MK_SIZE_M) + +#define KEY_CONTEXT_OPAD_PRESENT_S 11 +#define KEY_CONTEXT_OPAD_PRESENT_M 0x1 +#define KEY_CONTEXT_OPAD_PRESENT_V(x) ((x) << KEY_CONTEXT_OPAD_PRESENT_S) +#define KEY_CONTEXT_OPAD_PRESENT_G(x) \ + (((x) >> KEY_CONTEXT_OPAD_PRESENT_S) & \ + KEY_CONTEXT_OPAD_PRESENT_M) +#define KEY_CONTEXT_OPAD_PRESENT_F KEY_CONTEXT_OPAD_PRESENT_V(1U) + +#define CHCR_HASH_MAX_DIGEST_SIZE 64 +#define CHCR_MAX_SHA_DIGEST_SIZE 64 + +#define IPSEC_TRUNCATED_ICV_SIZE 12 +#define TLS_TRUNCATED_HMAC_SIZE 10 +#define CBCMAC_DIGEST_SIZE 16 +#define MAX_HASH_NAME 20 + +#define SHA1_INIT_STATE_5X4B 5 +#define SHA256_INIT_STATE_8X4B 8 +#define SHA512_INIT_STATE_8X8B 8 +#define SHA1_INIT_STATE SHA1_INIT_STATE_5X4B +#define SHA224_INIT_STATE SHA256_INIT_STATE_8X4B +#define SHA256_INIT_STATE SHA256_INIT_STATE_8X4B +#define SHA384_INIT_STATE SHA512_INIT_STATE_8X8B +#define SHA512_INIT_STATE SHA512_INIT_STATE_8X8B + +#define DUMMY_BYTES 16 + +#define IPAD_DATA 0x36363636 +#define OPAD_DATA 0x5c5c5c5c + +#define TRANSHDR_SIZE(kctx_len)\ + (sizeof(struct chcr_wr) +\ + kctx_len) +#define CIPHER_TRANSHDR_SIZE(kctx_len, sge_pairs) \ + (TRANSHDR_SIZE((kctx_len)) + (sge_pairs) +\ + sizeof(struct cpl_rx_phys_dsgl) + AES_BLOCK_SIZE) +#define HASH_TRANSHDR_SIZE(kctx_len)\ + (TRANSHDR_SIZE(kctx_len) + DUMMY_BYTES) + + +#define FILL_SEC_CPL_OP_IVINSR(id, len, ofst) \ + htonl( \ + CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) | \ + CPL_TX_SEC_PDU_RXCHID_V((id)) | \ + CPL_TX_SEC_PDU_ACKFOLLOWS_V(0) | \ + CPL_TX_SEC_PDU_ULPTXLPBK_V(1) | \ + CPL_TX_SEC_PDU_CPLLEN_V((len)) | \ + CPL_TX_SEC_PDU_PLACEHOLDER_V(0) | \ + CPL_TX_SEC_PDU_IVINSRTOFST_V((ofst))) + +#define FILL_SEC_CPL_CIPHERSTOP_HI(a_start, a_stop, c_start, c_stop_hi) \ + htonl( \ + CPL_TX_SEC_PDU_AADSTART_V((a_start)) | \ + CPL_TX_SEC_PDU_AADSTOP_V((a_stop)) | \ + CPL_TX_SEC_PDU_CIPHERSTART_V((c_start)) | \ + CPL_TX_SEC_PDU_CIPHERSTOP_HI_V((c_stop_hi))) + +#define FILL_SEC_CPL_AUTHINSERT(c_stop_lo, a_start, a_stop, a_inst) \ + htonl( \ + CPL_TX_SEC_PDU_CIPHERSTOP_LO_V((c_stop_lo)) | \ + CPL_TX_SEC_PDU_AUTHSTART_V((a_start)) | \ + CPL_TX_SEC_PDU_AUTHSTOP_V((a_stop)) | \ + CPL_TX_SEC_PDU_AUTHINSERT_V((a_inst))) + +#define FILL_SEC_CPL_SCMD0_SEQNO(ctrl, seq, cmode, amode, opad, size) \ + htonl( \ + SCMD_SEQ_NO_CTRL_V(0) | \ + SCMD_STATUS_PRESENT_V(0) | \ + SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_GENERIC) | \ + SCMD_ENC_DEC_CTRL_V((ctrl)) | \ + SCMD_CIPH_AUTH_SEQ_CTRL_V((seq)) | \ + SCMD_CIPH_MODE_V((cmode)) | \ + SCMD_AUTH_MODE_V((amode)) | \ + SCMD_HMAC_CTRL_V((opad)) | \ + SCMD_IV_SIZE_V((size)) | \ + SCMD_NUM_IVS_V(0)) + +#define FILL_SEC_CPL_IVGEN_HDRLEN(last, more, ctx_in, mac, ivdrop, len) htonl( \ + SCMD_ENB_DBGID_V(0) | \ + SCMD_IV_GEN_CTRL_V(0) | \ + SCMD_LAST_FRAG_V((last)) | \ + SCMD_MORE_FRAGS_V((more)) | \ + SCMD_TLS_COMPPDU_V(0) | \ + SCMD_KEY_CTX_INLINE_V((ctx_in)) | \ + SCMD_TLS_FRAG_ENABLE_V(0) | \ + SCMD_MAC_ONLY_V((mac)) | \ + SCMD_AADIVDROP_V((ivdrop)) | \ + SCMD_HDR_LEN_V((len))) + +#define FILL_KEY_CTX_HDR(ck_size, mk_size, d_ck, opad, ctx_len) \ + htonl(KEY_CONTEXT_VALID_V(1) | \ + KEY_CONTEXT_CK_SIZE_V((ck_size)) | \ + KEY_CONTEXT_MK_SIZE_V(mk_size) | \ + KEY_CONTEXT_DUAL_CK_V((d_ck)) | \ + KEY_CONTEXT_OPAD_PRESENT_V((opad)) | \ + KEY_CONTEXT_SALT_PRESENT_V(1) | \ + KEY_CONTEXT_CTX_LEN_V((ctx_len))) + +#define FILL_KEY_CRX_HDR(ck_size, mk_size, d_ck, opad, ctx_len) \ + htonl(TLS_KEYCTX_RXMK_SIZE_V(mk_size) | \ + TLS_KEYCTX_RXCK_SIZE_V(ck_size) | \ + TLS_KEYCTX_RX_VALID_V(1) | \ + TLS_KEYCTX_RX_SEQCTR_V(3) | \ + TLS_KEYCTX_RXAUTH_MODE_V(4) | \ + TLS_KEYCTX_RXCIPH_MODE_V(2) | \ + TLS_KEYCTX_RXFLIT_CNT_V((ctx_len))) + +#define FILL_WR_OP_CCTX_SIZE \ + htonl( \ + FW_CRYPTO_LOOKASIDE_WR_OPCODE_V( \ + FW_CRYPTO_LOOKASIDE_WR) | \ + FW_CRYPTO_LOOKASIDE_WR_COMPL_V(0) | \ + FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_V((0)) | \ + FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_V(0) | \ + FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_V(0)) + +#define FILL_WR_RX_Q_ID(cid, qid, lcb, fid) \ + htonl( \ + FW_CRYPTO_LOOKASIDE_WR_RX_CHID_V((cid)) | \ + FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_V((qid)) | \ + FW_CRYPTO_LOOKASIDE_WR_LCB_V((lcb)) | \ + FW_CRYPTO_LOOKASIDE_WR_IV_V((IV_NOP)) | \ + FW_CRYPTO_LOOKASIDE_WR_FQIDX_V(fid)) + +#define FILL_ULPTX_CMD_DEST(cid, qid) \ + htonl(ULPTX_CMD_V(ULP_TX_PKT) | \ + ULP_TXPKT_DEST_V(0) | \ + ULP_TXPKT_DATAMODIFY_V(0) | \ + ULP_TXPKT_CHANNELID_V((cid)) | \ + ULP_TXPKT_RO_V(1) | \ + ULP_TXPKT_FID_V(qid)) + +#define KEYCTX_ALIGN_PAD(bs) ({unsigned int _bs = (bs);\ + _bs == SHA1_DIGEST_SIZE ? 12 : 0; }) + +#define FILL_PLD_SIZE_HASH_SIZE(payload_sgl_len, sgl_lengths, total_frags) \ + htonl(FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_V(payload_sgl_len ? \ + sgl_lengths[total_frags] : 0) |\ + FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(0)) + +#define FILL_LEN_PKD(calc_tx_flits_ofld, skb) \ + htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP((\ + calc_tx_flits_ofld(skb) * 8), 16))) + +#define FILL_CMD_MORE(immdatalen) htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) |\ + ULP_TX_SC_MORE_V((immdatalen))) +#define MAX_NK 8 +#define MAX_DSGL_ENT 32 +#define MIN_AUTH_SG 1 /* IV */ +#define MIN_GCM_SG 1 /* IV */ +#define MIN_DIGEST_SG 1 /*Partial Buffer*/ +#define MIN_CCM_SG 1 /*IV+B0*/ +#define CIP_SPACE_LEFT(len) \ + ((SGE_MAX_WR_LEN - CIP_WR_MIN_LEN - (len))) +#define HASH_SPACE_LEFT(len) \ + ((SGE_MAX_WR_LEN - HASH_WR_MIN_LEN - (len))) + +struct algo_param { + unsigned int auth_mode; + unsigned int mk_size; + unsigned int result_size; +}; + +struct hash_wr_param { + struct algo_param alg_prm; + unsigned int opad_needed; + unsigned int more; + unsigned int last; + unsigned int kctx_len; + unsigned int sg_len; + unsigned int bfr_len; + unsigned int hash_size; + u64 scmd1; +}; + +struct cipher_wr_param { + struct skcipher_request *req; + char *iv; + int bytes; + unsigned short qid; +}; +enum { + AES_KEYLENGTH_128BIT = 128, + AES_KEYLENGTH_192BIT = 192, + AES_KEYLENGTH_256BIT = 256 +}; + +enum { + KEYLENGTH_3BYTES = 3, + KEYLENGTH_4BYTES = 4, + KEYLENGTH_6BYTES = 6, + KEYLENGTH_8BYTES = 8 +}; + +enum { + NUMBER_OF_ROUNDS_10 = 10, + NUMBER_OF_ROUNDS_12 = 12, + NUMBER_OF_ROUNDS_14 = 14, +}; + +/* + * CCM defines values of 4, 6, 8, 10, 12, 14, and 16 octets, + * where they indicate the size of the integrity check value (ICV) + */ +enum { + ICV_4 = 4, + ICV_6 = 6, + ICV_8 = 8, + ICV_10 = 10, + ICV_12 = 12, + ICV_13 = 13, + ICV_14 = 14, + ICV_15 = 15, + ICV_16 = 16 +}; + +struct phys_sge_pairs { + __be16 len[8]; + __be64 addr[8]; +}; + + +static const u32 chcr_sha1_init[SHA1_DIGEST_SIZE / 4] = { + SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4, +}; + +static const u32 chcr_sha224_init[SHA256_DIGEST_SIZE / 4] = { + SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3, + SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7, +}; + +static const u32 chcr_sha256_init[SHA256_DIGEST_SIZE / 4] = { + SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, + SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7, +}; + +static const u64 chcr_sha384_init[SHA512_DIGEST_SIZE / 8] = { + SHA384_H0, SHA384_H1, SHA384_H2, SHA384_H3, + SHA384_H4, SHA384_H5, SHA384_H6, SHA384_H7, +}; + +static const u64 chcr_sha512_init[SHA512_DIGEST_SIZE / 8] = { + SHA512_H0, SHA512_H1, SHA512_H2, SHA512_H3, + SHA512_H4, SHA512_H5, SHA512_H6, SHA512_H7, +}; + +static inline void copy_hash_init_values(char *key, int digestsize) +{ + u8 i; + __be32 *dkey = (__be32 *)key; + u64 *ldkey = (u64 *)key; + __be64 *sha384 = (__be64 *)chcr_sha384_init; + __be64 *sha512 = (__be64 *)chcr_sha512_init; + + switch (digestsize) { + case SHA1_DIGEST_SIZE: + for (i = 0; i < SHA1_INIT_STATE; i++) + dkey[i] = cpu_to_be32(chcr_sha1_init[i]); + break; + case SHA224_DIGEST_SIZE: + for (i = 0; i < SHA224_INIT_STATE; i++) + dkey[i] = cpu_to_be32(chcr_sha224_init[i]); + break; + case SHA256_DIGEST_SIZE: + for (i = 0; i < SHA256_INIT_STATE; i++) + dkey[i] = cpu_to_be32(chcr_sha256_init[i]); + break; + case SHA384_DIGEST_SIZE: + for (i = 0; i < SHA384_INIT_STATE; i++) + ldkey[i] = be64_to_cpu(sha384[i]); + break; + case SHA512_DIGEST_SIZE: + for (i = 0; i < SHA512_INIT_STATE; i++) + ldkey[i] = be64_to_cpu(sha512[i]); + break; + } +} + +/* Number of len fields(8) * size of one addr field */ +#define PHYSDSGL_MAX_LEN_SIZE 16 + +static inline u16 get_space_for_phys_dsgl(unsigned int sgl_entr) +{ + /* len field size + addr field size */ + return ((sgl_entr >> 3) + ((sgl_entr % 8) ? + 1 : 0)) * PHYSDSGL_MAX_LEN_SIZE + + (sgl_entr << 3) + ((sgl_entr % 2 ? 1 : 0) << 3); +} + +/* The AES s-transform matrix (s-box). */ +static const u8 aes_sbox[256] = { + 99, 124, 119, 123, 242, 107, 111, 197, 48, 1, 103, 43, 254, 215, + 171, 118, 202, 130, 201, 125, 250, 89, 71, 240, 173, 212, 162, 175, + 156, 164, 114, 192, 183, 253, 147, 38, 54, 63, 247, 204, 52, 165, + 229, 241, 113, 216, 49, 21, 4, 199, 35, 195, 24, 150, 5, 154, 7, + 18, 128, 226, 235, 39, 178, 117, 9, 131, 44, 26, 27, 110, 90, + 160, 82, 59, 214, 179, 41, 227, 47, 132, 83, 209, 0, 237, 32, + 252, 177, 91, 106, 203, 190, 57, 74, 76, 88, 207, 208, 239, 170, + 251, 67, 77, 51, 133, 69, 249, 2, 127, 80, 60, 159, 168, 81, + 163, 64, 143, 146, 157, 56, 245, 188, 182, 218, 33, 16, 255, 243, + 210, 205, 12, 19, 236, 95, 151, 68, 23, 196, 167, 126, 61, 100, + 93, 25, 115, 96, 129, 79, 220, 34, 42, 144, 136, 70, 238, 184, + 20, 222, 94, 11, 219, 224, 50, 58, 10, 73, 6, 36, 92, 194, + 211, 172, 98, 145, 149, 228, 121, 231, 200, 55, 109, 141, 213, 78, + 169, 108, 86, 244, 234, 101, 122, 174, 8, 186, 120, 37, 46, 28, 166, + 180, 198, 232, 221, 116, 31, 75, 189, 139, 138, 112, 62, 181, 102, + 72, 3, 246, 14, 97, 53, 87, 185, 134, 193, 29, 158, 225, 248, + 152, 17, 105, 217, 142, 148, 155, 30, 135, 233, 206, 85, 40, 223, + 140, 161, 137, 13, 191, 230, 66, 104, 65, 153, 45, 15, 176, 84, + 187, 22 +}; + +static inline u32 aes_ks_subword(const u32 w) +{ + u8 bytes[4]; + + *(u32 *)(&bytes[0]) = w; + bytes[0] = aes_sbox[bytes[0]]; + bytes[1] = aes_sbox[bytes[1]]; + bytes[2] = aes_sbox[bytes[2]]; + bytes[3] = aes_sbox[bytes[3]]; + return *(u32 *)(&bytes[0]); +} + +#endif /* __CHCR_ALGO_H__ */ diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c new file mode 100644 index 000000000..2609a8c3c --- /dev/null +++ b/drivers/crypto/chelsio/chcr_core.c @@ -0,0 +1,311 @@ +/* + * This file is part of the Chelsio T4/T5/T6 Ethernet driver for Linux. + * + * Copyright (C) 2011-2016 Chelsio Communications. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation. + * + * Written and Maintained by: + * Manoj Malviya (manojmalviya@chelsio.com) + * Atul Gupta (atul.gupta@chelsio.com) + * Jitendra Lulla (jlulla@chelsio.com) + * Yeshaswi M R Gowda (yeshaswi@chelsio.com) + * Harsh Jain (harsh@chelsio.com) + */ + +#include +#include +#include + +#include +#include + +#include "t4_msg.h" +#include "chcr_core.h" +#include "cxgb4_uld.h" + +static struct chcr_driver_data drv_data; + +typedef int (*chcr_handler_func)(struct adapter *adap, unsigned char *input); +static int cpl_fw6_pld_handler(struct adapter *adap, unsigned char *input); +static void *chcr_uld_add(const struct cxgb4_lld_info *lld); +static int chcr_uld_state_change(void *handle, enum cxgb4_state state); + +static chcr_handler_func work_handlers[NUM_CPL_CMDS] = { + [CPL_FW6_PLD] = cpl_fw6_pld_handler, +}; + +static struct cxgb4_uld_info chcr_uld_info = { + .name = DRV_MODULE_NAME, + .nrxq = MAX_ULD_QSETS, + /* Max ntxq will be derived from fw config file*/ + .rxq_size = 1024, + .add = chcr_uld_add, + .state_change = chcr_uld_state_change, + .rx_handler = chcr_uld_rx_handler, +}; + +static void detach_work_fn(struct work_struct *work) +{ + struct chcr_dev *dev; + + dev = container_of(work, struct chcr_dev, detach_work.work); + + if (atomic_read(&dev->inflight)) { + dev->wqretry--; + if (dev->wqretry) { + pr_debug("Request Inflight Count %d\n", + atomic_read(&dev->inflight)); + + schedule_delayed_work(&dev->detach_work, WQ_DETACH_TM); + } else { + WARN(1, "CHCR:%d request Still Pending\n", + atomic_read(&dev->inflight)); + complete(&dev->detach_comp); + } + } else { + complete(&dev->detach_comp); + } +} + +struct uld_ctx *assign_chcr_device(void) +{ + struct uld_ctx *u_ctx = NULL; + + /* + * When multiple devices are present in system select + * device in round-robin fashion for crypto operations + * Although One session must use the same device to + * maintain request-response ordering. + */ + mutex_lock(&drv_data.drv_mutex); + if (!list_empty(&drv_data.act_dev)) { + u_ctx = drv_data.last_dev; + if (list_is_last(&drv_data.last_dev->entry, &drv_data.act_dev)) + drv_data.last_dev = list_first_entry(&drv_data.act_dev, + struct uld_ctx, entry); + else + drv_data.last_dev = + list_next_entry(drv_data.last_dev, entry); + } + mutex_unlock(&drv_data.drv_mutex); + return u_ctx; +} + +static void chcr_dev_add(struct uld_ctx *u_ctx) +{ + struct chcr_dev *dev; + + dev = &u_ctx->dev; + dev->state = CHCR_ATTACH; + atomic_set(&dev->inflight, 0); + mutex_lock(&drv_data.drv_mutex); + list_move(&u_ctx->entry, &drv_data.act_dev); + if (!drv_data.last_dev) + drv_data.last_dev = u_ctx; + mutex_unlock(&drv_data.drv_mutex); +} + +static void chcr_dev_init(struct uld_ctx *u_ctx) +{ + struct chcr_dev *dev; + + dev = &u_ctx->dev; + spin_lock_init(&dev->lock_chcr_dev); + INIT_DELAYED_WORK(&dev->detach_work, detach_work_fn); + init_completion(&dev->detach_comp); + dev->state = CHCR_INIT; + dev->wqretry = WQ_RETRY; + atomic_inc(&drv_data.dev_count); + atomic_set(&dev->inflight, 0); + mutex_lock(&drv_data.drv_mutex); + list_add_tail(&u_ctx->entry, &drv_data.inact_dev); + mutex_unlock(&drv_data.drv_mutex); +} + +static int chcr_dev_move(struct uld_ctx *u_ctx) +{ + mutex_lock(&drv_data.drv_mutex); + if (drv_data.last_dev == u_ctx) { + if (list_is_last(&drv_data.last_dev->entry, &drv_data.act_dev)) + drv_data.last_dev = list_first_entry(&drv_data.act_dev, + struct uld_ctx, entry); + else + drv_data.last_dev = + list_next_entry(drv_data.last_dev, entry); + } + list_move(&u_ctx->entry, &drv_data.inact_dev); + if (list_empty(&drv_data.act_dev)) + drv_data.last_dev = NULL; + atomic_dec(&drv_data.dev_count); + mutex_unlock(&drv_data.drv_mutex); + + return 0; +} + +static int cpl_fw6_pld_handler(struct adapter *adap, + unsigned char *input) +{ + struct crypto_async_request *req; + struct cpl_fw6_pld *fw6_pld; + u32 ack_err_status = 0; + int error_status = 0; + + fw6_pld = (struct cpl_fw6_pld *)input; + req = (struct crypto_async_request *)(uintptr_t)be64_to_cpu( + fw6_pld->data[1]); + + ack_err_status = + ntohl(*(__be32 *)((unsigned char *)&fw6_pld->data[0] + 4)); + if (CHK_MAC_ERR_BIT(ack_err_status) || CHK_PAD_ERR_BIT(ack_err_status)) + error_status = -EBADMSG; + /* call completion callback with failure status */ + if (req) { + error_status = chcr_handle_resp(req, input, error_status); + } else { + pr_err("Incorrect request address from the firmware\n"); + return -EFAULT; + } + if (error_status) + atomic_inc(&adap->chcr_stats.error); + + return 0; +} + +int chcr_send_wr(struct sk_buff *skb) +{ + return cxgb4_crypto_send(skb->dev, skb); +} + +static void *chcr_uld_add(const struct cxgb4_lld_info *lld) +{ + struct uld_ctx *u_ctx; + + /* Create the device and add it in the device list */ + pr_info_once("%s\n", DRV_DESC); + if (!(lld->ulp_crypto & ULP_CRYPTO_LOOKASIDE)) + return ERR_PTR(-EOPNOTSUPP); + + /* Create the device and add it in the device list */ + u_ctx = kzalloc_obj(*u_ctx); + if (!u_ctx) { + u_ctx = ERR_PTR(-ENOMEM); + goto out; + } + u_ctx->lldi = *lld; + chcr_dev_init(u_ctx); +out: + return u_ctx; +} + +int chcr_uld_rx_handler(void *handle, const __be64 *rsp, + const struct pkt_gl *pgl) +{ + struct uld_ctx *u_ctx = (struct uld_ctx *)handle; + struct chcr_dev *dev = &u_ctx->dev; + struct adapter *adap = padap(dev); + const struct cpl_fw6_pld *rpl = (struct cpl_fw6_pld *)rsp; + + if (!work_handlers[rpl->opcode]) { + pr_err("Unsupported opcode %d received\n", rpl->opcode); + return 0; + } + + if (!pgl) + work_handlers[rpl->opcode](adap, (unsigned char *)&rsp[1]); + else + work_handlers[rpl->opcode](adap, pgl->va); + return 0; +} + +static void chcr_detach_device(struct uld_ctx *u_ctx) +{ + struct chcr_dev *dev = &u_ctx->dev; + + if (dev->state == CHCR_DETACH) { + pr_debug("Detached Event received for already detach device\n"); + return; + } + dev->state = CHCR_DETACH; + if (atomic_read(&dev->inflight) != 0) { + schedule_delayed_work(&dev->detach_work, WQ_DETACH_TM); + wait_for_completion(&dev->detach_comp); + } + + // Move u_ctx to inactive_dev list + chcr_dev_move(u_ctx); +} + +static int chcr_uld_state_change(void *handle, enum cxgb4_state state) +{ + struct uld_ctx *u_ctx = handle; + int ret = 0; + + switch (state) { + case CXGB4_STATE_UP: + if (u_ctx->dev.state != CHCR_INIT) { + // ALready Initialised. + return 0; + } + chcr_dev_add(u_ctx); + ret = start_crypto(); + break; + + case CXGB4_STATE_DETACH: + chcr_detach_device(u_ctx); + if (!atomic_read(&drv_data.dev_count)) + stop_crypto(); + break; + + case CXGB4_STATE_START_RECOVERY: + case CXGB4_STATE_DOWN: + default: + break; + } + return ret; +} + +static int __init chcr_crypto_init(void) +{ + INIT_LIST_HEAD(&drv_data.act_dev); + INIT_LIST_HEAD(&drv_data.inact_dev); + atomic_set(&drv_data.dev_count, 0); + mutex_init(&drv_data.drv_mutex); + drv_data.last_dev = NULL; + cxgb4_register_uld(CXGB4_ULD_CRYPTO, &chcr_uld_info); + + return 0; +} + +static void __exit chcr_crypto_exit(void) +{ + struct uld_ctx *u_ctx, *tmp; + struct adapter *adap; + + stop_crypto(); + cxgb4_unregister_uld(CXGB4_ULD_CRYPTO); + /* Remove all devices from list */ + mutex_lock(&drv_data.drv_mutex); + list_for_each_entry_safe(u_ctx, tmp, &drv_data.act_dev, entry) { + adap = padap(&u_ctx->dev); + memset(&adap->chcr_stats, 0, sizeof(adap->chcr_stats)); + list_del(&u_ctx->entry); + kfree(u_ctx); + } + list_for_each_entry_safe(u_ctx, tmp, &drv_data.inact_dev, entry) { + adap = padap(&u_ctx->dev); + memset(&adap->chcr_stats, 0, sizeof(adap->chcr_stats)); + list_del(&u_ctx->entry); + kfree(u_ctx); + } + mutex_unlock(&drv_data.drv_mutex); +} + +module_init(chcr_crypto_init); +module_exit(chcr_crypto_exit); + +MODULE_DESCRIPTION("Crypto Co-processor for Chelsio Terminator cards."); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Chelsio Communications"); diff --git a/drivers/crypto/chelsio/chcr_core.h b/drivers/crypto/chelsio/chcr_core.h new file mode 100644 index 000000000..5e9d56813 --- /dev/null +++ b/drivers/crypto/chelsio/chcr_core.h @@ -0,0 +1,138 @@ +/* + * This file is part of the Chelsio T6 Crypto driver for Linux. + * + * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +#ifndef __CHCR_CORE_H__ +#define __CHCR_CORE_H__ + +#include +#include +#include "t4_hw.h" +#include "cxgb4.h" +#include "t4_msg.h" +#include "cxgb4_uld.h" + +#define DRV_MODULE_NAME "chcr" +#define DRV_DESC "Chelsio T6 Crypto Co-processor Driver" + +#define MAX_PENDING_REQ_TO_HW 20 +#define CHCR_TEST_RESPONSE_TIMEOUT 1000 +#define WQ_DETACH_TM (msecs_to_jiffies(50)) +#define PAD_ERROR_BIT 1 +#define CHK_PAD_ERR_BIT(x) (((x) >> PAD_ERROR_BIT) & 1) + +#define MAC_ERROR_BIT 0 +#define CHK_MAC_ERR_BIT(x) (((x) >> MAC_ERROR_BIT) & 1) +#define MAX_SALT 4 +#define CIP_WR_MIN_LEN (sizeof(struct chcr_wr) + \ + sizeof(struct cpl_rx_phys_dsgl) + \ + sizeof(struct ulptx_sgl) + 16) //IV + +#define HASH_WR_MIN_LEN (sizeof(struct chcr_wr) + \ + DUMMY_BYTES + \ + sizeof(struct ulptx_sgl)) +struct uld_ctx; + +struct _key_ctx { + __be32 ctx_hdr; + u8 salt[MAX_SALT]; + __be64 iv_to_auth; + unsigned char key[]; +}; + +#define WQ_RETRY 5 +struct chcr_driver_data { + struct list_head act_dev; + struct list_head inact_dev; + atomic_t dev_count; + struct mutex drv_mutex; + struct uld_ctx *last_dev; +}; + +enum chcr_state { + CHCR_INIT = 0, + CHCR_ATTACH, + CHCR_DETACH, +}; +struct chcr_wr { + struct fw_crypto_lookaside_wr wreq; + struct ulp_txpkt ulptx; + struct ulptx_idata sc_imm; + struct cpl_tx_sec_pdu sec_cpl; + struct _key_ctx key_ctx; +}; + +struct chcr_dev { + spinlock_t lock_chcr_dev; + enum chcr_state state; + atomic_t inflight; + int wqretry; + struct delayed_work detach_work; + struct completion detach_comp; +}; + +struct uld_ctx { + struct list_head entry; + struct cxgb4_lld_info lldi; + struct chcr_dev dev; +}; + +/* + * sgl_len - calculates the size of an SGL of the given capacity + * @n: the number of SGL entries + * Calculates the number of flits needed for a scatter/gather list that + * can hold the given number of entries. + */ +static inline unsigned int sgl_len(unsigned int n) +{ + n--; + return (3 * n) / 2 + (n & 1) + 2; +} + +static inline void *padap(struct chcr_dev *dev) +{ + struct uld_ctx *u_ctx = container_of(dev, struct uld_ctx, dev); + + return pci_get_drvdata(u_ctx->lldi.pdev); +} + +struct uld_ctx *assign_chcr_device(void); +int chcr_send_wr(struct sk_buff *skb); +int start_crypto(void); +int stop_crypto(void); +int chcr_uld_rx_handler(void *handle, const __be64 *rsp, + const struct pkt_gl *pgl); +int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, + int err); +#endif /* __CHCR_CORE_H__ */ diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h new file mode 100644 index 000000000..e1e79e5f0 --- /dev/null +++ b/drivers/crypto/chelsio/chcr_crypto.h @@ -0,0 +1,351 @@ +/* + * This file is part of the Chelsio T6 Crypto driver for Linux. + * + * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +#ifndef __CHCR_CRYPTO_H__ +#define __CHCR_CRYPTO_H__ + +#define GHASH_BLOCK_SIZE 16 +#define GHASH_DIGEST_SIZE 16 + +#define CCM_B0_SIZE 16 +#define CCM_AAD_FIELD_SIZE 2 +// 511 - 16(For IV) +#define T6_MAX_AAD_SIZE 495 + + +/* Define following if h/w is not dropping the AAD and IV data before + * giving the processed data + */ + +#define CHCR_CRA_PRIORITY 500 +#define CHCR_AEAD_PRIORITY 6000 +#define CHCR_AES_MAX_KEY_LEN (2 * (AES_MAX_KEY_SIZE)) /* consider xts */ +#define CHCR_MAX_CRYPTO_IV_LEN 16 /* AES IV len */ + +#define CHCR_MAX_AUTHENC_AES_KEY_LEN 32 /* max aes key length*/ +#define CHCR_MAX_AUTHENC_SHA_KEY_LEN 128 /* max sha key length*/ + +#define CHCR_GIVENCRYPT_OP 2 +/* CPL/SCMD parameters */ + +#define CHCR_ENCRYPT_OP 0 +#define CHCR_DECRYPT_OP 1 + +#define CHCR_SCMD_SEQ_NO_CTRL_32BIT 1 +#define CHCR_SCMD_SEQ_NO_CTRL_48BIT 2 +#define CHCR_SCMD_SEQ_NO_CTRL_64BIT 3 + +#define CHCR_SCMD_PROTO_VERSION_GENERIC 4 + +#define CHCR_SCMD_AUTH_CTRL_AUTH_CIPHER 0 +#define CHCR_SCMD_AUTH_CTRL_CIPHER_AUTH 1 + +#define CHCR_SCMD_CIPHER_MODE_NOP 0 +#define CHCR_SCMD_CIPHER_MODE_AES_CBC 1 +#define CHCR_SCMD_CIPHER_MODE_AES_GCM 2 +#define CHCR_SCMD_CIPHER_MODE_AES_CTR 3 +#define CHCR_SCMD_CIPHER_MODE_GENERIC_AES 4 +#define CHCR_SCMD_CIPHER_MODE_AES_XTS 6 +#define CHCR_SCMD_CIPHER_MODE_AES_CCM 7 + +#define CHCR_SCMD_AUTH_MODE_NOP 0 +#define CHCR_SCMD_AUTH_MODE_SHA1 1 +#define CHCR_SCMD_AUTH_MODE_SHA224 2 +#define CHCR_SCMD_AUTH_MODE_SHA256 3 +#define CHCR_SCMD_AUTH_MODE_GHASH 4 +#define CHCR_SCMD_AUTH_MODE_SHA512_224 5 +#define CHCR_SCMD_AUTH_MODE_SHA512_256 6 +#define CHCR_SCMD_AUTH_MODE_SHA512_384 7 +#define CHCR_SCMD_AUTH_MODE_SHA512_512 8 +#define CHCR_SCMD_AUTH_MODE_CBCMAC 9 +#define CHCR_SCMD_AUTH_MODE_CMAC 10 + +#define CHCR_SCMD_HMAC_CTRL_NOP 0 +#define CHCR_SCMD_HMAC_CTRL_NO_TRUNC 1 +#define CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366 2 +#define CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT 3 +#define CHCR_SCMD_HMAC_CTRL_PL1 4 +#define CHCR_SCMD_HMAC_CTRL_PL2 5 +#define CHCR_SCMD_HMAC_CTRL_PL3 6 +#define CHCR_SCMD_HMAC_CTRL_DIV2 7 +#define VERIFY_HW 0 +#define VERIFY_SW 1 + +#define CHCR_SCMD_IVGEN_CTRL_HW 0 +#define CHCR_SCMD_IVGEN_CTRL_SW 1 +/* This are not really mac key size. They are intermediate values + * of sha engine and its size + */ +#define CHCR_KEYCTX_MAC_KEY_SIZE_128 0 +#define CHCR_KEYCTX_MAC_KEY_SIZE_160 1 +#define CHCR_KEYCTX_MAC_KEY_SIZE_192 2 +#define CHCR_KEYCTX_MAC_KEY_SIZE_256 3 +#define CHCR_KEYCTX_MAC_KEY_SIZE_512 4 +#define CHCR_KEYCTX_CIPHER_KEY_SIZE_128 0 +#define CHCR_KEYCTX_CIPHER_KEY_SIZE_192 1 +#define CHCR_KEYCTX_CIPHER_KEY_SIZE_256 2 +#define CHCR_KEYCTX_NO_KEY 15 + +#define CHCR_CPL_FW4_PLD_IV_OFFSET (5 * 64) /* bytes. flt #5 and #6 */ +#define CHCR_CPL_FW4_PLD_HASH_RESULT_OFFSET (7 * 64) /* bytes. flt #7 */ +#define CHCR_CPL_FW4_PLD_DATA_SIZE (4 * 64) /* bytes. flt #4 to #7 */ + +#define KEY_CONTEXT_HDR_SALT_AND_PAD 16 +#define flits_to_bytes(x) (x * 8) + +#define IV_NOP 0 +#define IV_IMMEDIATE 1 +#define IV_DSGL 2 + +#define AEAD_H_SIZE 16 + +#define CRYPTO_ALG_SUB_TYPE_MASK 0x0f000000 +#define CRYPTO_ALG_SUB_TYPE_HASH_HMAC 0x01000000 +#define CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 0x02000000 +#define CRYPTO_ALG_SUB_TYPE_AEAD_GCM 0x03000000 +#define CRYPTO_ALG_SUB_TYPE_CBC_SHA 0x04000000 +#define CRYPTO_ALG_SUB_TYPE_AEAD_CCM 0x05000000 +#define CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309 0x06000000 +#define CRYPTO_ALG_SUB_TYPE_CBC_NULL 0x07000000 +#define CRYPTO_ALG_SUB_TYPE_CTR 0x08000000 +#define CRYPTO_ALG_SUB_TYPE_CTR_RFC3686 0x09000000 +#define CRYPTO_ALG_SUB_TYPE_XTS 0x0a000000 +#define CRYPTO_ALG_SUB_TYPE_CBC 0x0b000000 +#define CRYPTO_ALG_SUB_TYPE_CTR_SHA 0x0c000000 +#define CRYPTO_ALG_SUB_TYPE_CTR_NULL 0x0d000000 +#define CRYPTO_ALG_TYPE_HMAC (CRYPTO_ALG_TYPE_AHASH |\ + CRYPTO_ALG_SUB_TYPE_HASH_HMAC) + +#define MAX_SCRATCH_PAD_SIZE 32 + +#define CHCR_HASH_MAX_BLOCK_SIZE_64 64 +#define CHCR_HASH_MAX_BLOCK_SIZE_128 128 +#define CHCR_SRC_SG_SIZE (0x10000 - sizeof(int)) +#define CHCR_DST_SG_SIZE 2048 + +static inline struct chcr_context *a_ctx(struct crypto_aead *tfm) +{ + return crypto_aead_ctx(tfm); +} + +static inline struct chcr_context *c_ctx(struct crypto_skcipher *tfm) +{ + return crypto_skcipher_ctx(tfm); +} + +static inline struct chcr_context *h_ctx(struct crypto_ahash *tfm) +{ + return crypto_tfm_ctx(crypto_ahash_tfm(tfm)); +} + +struct ablk_ctx { + struct crypto_skcipher *sw_cipher; + __be32 key_ctx_hdr; + unsigned int enckey_len; + unsigned char ciph_mode; + u8 key[CHCR_AES_MAX_KEY_LEN]; + u8 nonce[4]; + u8 rrkey[AES_MAX_KEY_SIZE]; +}; +struct chcr_aead_reqctx { + struct sk_buff *skb; + dma_addr_t iv_dma; + dma_addr_t b0_dma; + unsigned int b0_len; + unsigned int op; + u16 imm; + u16 verify; + u16 txqidx; + u16 rxqidx; + u8 iv[CHCR_MAX_CRYPTO_IV_LEN + MAX_SCRATCH_PAD_SIZE]; + u8 *scratch_pad; +}; + +struct ulptx_walk { + struct ulptx_sgl *sgl; + unsigned int nents; + unsigned int pair_idx; + unsigned int last_sg_len; + struct scatterlist *last_sg; + struct ulptx_sge_pair *pair; + +}; + +struct dsgl_walk { + unsigned int nents; + unsigned int last_sg_len; + struct scatterlist *last_sg; + struct cpl_rx_phys_dsgl *dsgl; + struct phys_sge_pairs *to; +}; + +struct chcr_gcm_ctx { + u8 ghash_h[AEAD_H_SIZE]; +}; + +struct chcr_authenc_ctx { + u8 dec_rrkey[AES_MAX_KEY_SIZE]; + u8 h_iopad[2 * CHCR_HASH_MAX_DIGEST_SIZE]; + unsigned char auth_mode; +}; + +struct __aead_ctx { + union { + DECLARE_FLEX_ARRAY(struct chcr_gcm_ctx, gcm); + DECLARE_FLEX_ARRAY(struct chcr_authenc_ctx, authenc); + }; +}; + +struct chcr_aead_ctx { + __be32 key_ctx_hdr; + unsigned int enckey_len; + struct crypto_aead *sw_cipher; + u8 salt[MAX_SALT]; + u8 key[CHCR_AES_MAX_KEY_LEN]; + u8 nonce[4]; + u16 hmac_ctrl; + u16 mayverify; + struct __aead_ctx ctx[]; +}; + +struct hmac_ctx { + u8 ipad[CHCR_HASH_MAX_BLOCK_SIZE_128]; + u8 opad[CHCR_HASH_MAX_BLOCK_SIZE_128]; +}; + +struct __crypto_ctx { + union { + struct hmac_ctx hmacctx; + struct ablk_ctx ablkctx; + struct chcr_aead_ctx aeadctx; + }; +}; + +struct chcr_context { + struct chcr_dev *dev; + unsigned char rxq_perchan; + unsigned char txq_perchan; + unsigned int ntxq; + unsigned int nrxq; + struct completion cbc_aes_aio_done; + struct __crypto_ctx crypto_ctx[]; +}; + +struct chcr_hctx_per_wr { + struct scatterlist *srcsg; + struct sk_buff *skb; + dma_addr_t dma_addr; + u32 dma_len; + unsigned int src_ofst; + unsigned int processed; + u32 result; + u8 is_sg_map; + u8 imm; + /*Final callback called. Driver cannot rely on nbytes to decide + * final call + */ + u8 isfinal; +}; + +struct chcr_ahash_req_ctx { + struct chcr_hctx_per_wr hctx_wr; + u8 *reqbfr; + u8 *skbfr; + /* SKB which is being sent to the hardware for processing */ + u64 data_len; /* Data len till time */ + u16 txqidx; + u16 rxqidx; + u8 reqlen; + u8 partial_hash[CHCR_HASH_MAX_DIGEST_SIZE]; + u8 bfr1[CHCR_HASH_MAX_BLOCK_SIZE_128]; + u8 bfr2[CHCR_HASH_MAX_BLOCK_SIZE_128]; +}; + +struct chcr_skcipher_req_ctx { + struct sk_buff *skb; + struct scatterlist *dstsg; + unsigned int processed; + unsigned int last_req_len; + unsigned int partial_req; + struct scatterlist *srcsg; + unsigned int src_ofst; + unsigned int dst_ofst; + unsigned int op; + u16 imm; + u8 iv[CHCR_MAX_CRYPTO_IV_LEN]; + u8 init_iv[CHCR_MAX_CRYPTO_IV_LEN]; + u16 txqidx; + u16 rxqidx; + struct skcipher_request fallback_req; // keep at the end +}; + +struct chcr_alg_template { + u32 type; + u32 is_registered; + union { + struct skcipher_alg skcipher; + struct ahash_alg hash; + struct aead_alg aead; + } alg; +}; + +typedef struct sk_buff *(*create_wr_t)(struct aead_request *req, + unsigned short qid, + int size); + +void chcr_verify_tag(struct aead_request *req, u8 *input, int *err); +int chcr_aead_dma_map(struct device *dev, struct aead_request *req, + unsigned short op_type); +void chcr_aead_dma_unmap(struct device *dev, struct aead_request *req, + unsigned short op_type); +void chcr_add_aead_dst_ent(struct aead_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + unsigned short qid); +void chcr_add_aead_src_ent(struct aead_request *req, struct ulptx_sgl *ulptx); +void chcr_add_cipher_src_ent(struct skcipher_request *req, + void *ulptx, + struct cipher_wr_param *wrparam); +int chcr_cipher_dma_map(struct device *dev, struct skcipher_request *req); +void chcr_cipher_dma_unmap(struct device *dev, struct skcipher_request *req); +void chcr_add_cipher_dst_ent(struct skcipher_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + struct cipher_wr_param *wrparam, + unsigned short qid); +void chcr_add_hash_src_ent(struct ahash_request *req, struct ulptx_sgl *ulptx, + struct hash_wr_param *param); +int chcr_hash_dma_map(struct device *dev, struct ahash_request *req); +void chcr_hash_dma_unmap(struct device *dev, struct ahash_request *req); +void chcr_aead_common_exit(struct aead_request *req); +#endif /* __CHCR_CRYPTO_H__ */ diff --git a/drivers/crypto/gemini/Makefile b/drivers/crypto/gemini/Makefile new file mode 100644 index 000000000..c73c8b692 --- /dev/null +++ b/drivers/crypto/gemini/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_CRYPTO_DEV_SL3516) += sl3516-ce.o +sl3516-ce-y += sl3516-ce-core.o sl3516-ce-cipher.o sl3516-ce-rng.o diff --git a/drivers/crypto/gemini/sl3516-ce-cipher.c b/drivers/crypto/gemini/sl3516-ce-cipher.c new file mode 100644 index 000000000..02ec42823 --- /dev/null +++ b/drivers/crypto/gemini/sl3516-ce-cipher.c @@ -0,0 +1,381 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sl3516-ce-cipher.c - hardware cryptographic offloader for Storlink SL3516 SoC + * + * Copyright (C) 2021 Corentin LABBE + * + * This file adds support for AES cipher with 128,192,256 bits keysize in + * ECB mode. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "sl3516-ce.h" + +/* sl3516_ce_need_fallback - check if a request can be handled by the CE */ +static bool sl3516_ce_need_fallback(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sl3516_ce_dev *ce = op->ce; + struct scatterlist *in_sg; + struct scatterlist *out_sg; + struct scatterlist *sg; + + if (areq->cryptlen == 0 || areq->cryptlen % 16) { + ce->fallback_mod16++; + return true; + } + + /* + * check if we have enough descriptors for TX + * Note: TX need one control desc for each SG + */ + if (sg_nents(areq->src) > MAXDESC / 2) { + ce->fallback_sg_count_tx++; + return true; + } + /* check if we have enough descriptors for RX */ + if (sg_nents(areq->dst) > MAXDESC) { + ce->fallback_sg_count_rx++; + return true; + } + + sg = areq->src; + while (sg) { + if ((sg->length % 16) != 0) { + ce->fallback_mod16++; + return true; + } + if (!IS_ALIGNED(sg->offset, 16)) { + ce->fallback_align16++; + return true; + } + sg = sg_next(sg); + } + sg = areq->dst; + while (sg) { + if ((sg->length % 16) != 0) { + ce->fallback_mod16++; + return true; + } + if (!IS_ALIGNED(sg->offset, 16)) { + ce->fallback_align16++; + return true; + } + sg = sg_next(sg); + } + + /* need same numbers of SG (with same length) for source and destination */ + in_sg = areq->src; + out_sg = areq->dst; + while (in_sg && out_sg) { + if (in_sg->length != out_sg->length) { + ce->fallback_not_same_len++; + return true; + } + in_sg = sg_next(in_sg); + out_sg = sg_next(out_sg); + } + if (in_sg || out_sg) + return true; + + return false; +} + +static int sl3516_ce_cipher_fallback(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sl3516_ce_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct sl3516_ce_alg_template *algt; + int err; + + algt = container_of(alg, struct sl3516_ce_alg_template, alg.skcipher.base); + algt->stat_fb++; + + skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm); + skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags, + areq->base.complete, areq->base.data); + skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst, + areq->cryptlen, areq->iv); + if (rctx->op_dir == CE_DECRYPTION) + err = crypto_skcipher_decrypt(&rctx->fallback_req); + else + err = crypto_skcipher_encrypt(&rctx->fallback_req); + return err; +} + +static int sl3516_ce_cipher(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sl3516_ce_dev *ce = op->ce; + struct sl3516_ce_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct sl3516_ce_alg_template *algt; + struct scatterlist *sg; + unsigned int todo, len; + struct pkt_control_ecb *ecb; + int nr_sgs = 0; + int nr_sgd = 0; + int err = 0; + int i; + + algt = container_of(alg, struct sl3516_ce_alg_template, alg.skcipher.base); + + dev_dbg(ce->dev, "%s %s %u %x IV(%p %u) key=%u\n", __func__, + crypto_tfm_alg_name(areq->base.tfm), + areq->cryptlen, + rctx->op_dir, areq->iv, crypto_skcipher_ivsize(tfm), + op->keylen); + + algt->stat_req++; + + if (areq->src == areq->dst) { + nr_sgs = dma_map_sg(ce->dev, areq->src, sg_nents(areq->src), + DMA_BIDIRECTIONAL); + if (nr_sgs <= 0 || nr_sgs > MAXDESC / 2) { + dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); + err = -EINVAL; + goto theend; + } + nr_sgd = nr_sgs; + } else { + nr_sgs = dma_map_sg(ce->dev, areq->src, sg_nents(areq->src), + DMA_TO_DEVICE); + if (nr_sgs <= 0 || nr_sgs > MAXDESC / 2) { + dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); + err = -EINVAL; + goto theend; + } + nr_sgd = dma_map_sg(ce->dev, areq->dst, sg_nents(areq->dst), + DMA_FROM_DEVICE); + if (nr_sgd <= 0 || nr_sgd > MAXDESC) { + dev_err(ce->dev, "Invalid sg number %d\n", nr_sgd); + err = -EINVAL; + goto theend_sgs; + } + } + + len = areq->cryptlen; + i = 0; + sg = areq->src; + while (i < nr_sgs && sg && len) { + if (sg_dma_len(sg) == 0) + goto sgs_next; + rctx->t_src[i].addr = sg_dma_address(sg); + todo = min(len, sg_dma_len(sg)); + rctx->t_src[i].len = todo; + dev_dbg(ce->dev, "%s total=%u SGS(%d %u off=%d) todo=%u\n", __func__, + areq->cryptlen, i, rctx->t_src[i].len, sg->offset, todo); + len -= todo; + i++; +sgs_next: + sg = sg_next(sg); + } + if (len > 0) { + dev_err(ce->dev, "remaining len %d/%u nr_sgs=%d\n", len, areq->cryptlen, nr_sgs); + err = -EINVAL; + goto theend_sgs; + } + + len = areq->cryptlen; + i = 0; + sg = areq->dst; + while (i < nr_sgd && sg && len) { + if (sg_dma_len(sg) == 0) + goto sgd_next; + rctx->t_dst[i].addr = sg_dma_address(sg); + todo = min(len, sg_dma_len(sg)); + rctx->t_dst[i].len = todo; + dev_dbg(ce->dev, "%s total=%u SGD(%d %u off=%d) todo=%u\n", __func__, + areq->cryptlen, i, rctx->t_dst[i].len, sg->offset, todo); + len -= todo; + i++; + +sgd_next: + sg = sg_next(sg); + } + if (len > 0) { + dev_err(ce->dev, "remaining len %d\n", len); + err = -EINVAL; + goto theend_sgs; + } + + switch (algt->mode) { + case ECB_AES: + rctx->pctrllen = sizeof(struct pkt_control_ecb); + ecb = (struct pkt_control_ecb *)ce->pctrl; + + rctx->tqflag = TQ0_TYPE_CTRL; + rctx->tqflag |= TQ1_CIPHER; + ecb->control.op_mode = rctx->op_dir; + ecb->control.cipher_algorithm = ECB_AES; + ecb->cipher.header_len = 0; + ecb->cipher.algorithm_len = areq->cryptlen; + cpu_to_be32_array((__be32 *)ecb->key, (u32 *)op->key, op->keylen / 4); + rctx->h = &ecb->cipher; + + rctx->tqflag |= TQ4_KEY0; + rctx->tqflag |= TQ5_KEY4; + rctx->tqflag |= TQ6_KEY6; + ecb->control.aesnk = op->keylen / 4; + break; + } + + rctx->nr_sgs = nr_sgs; + rctx->nr_sgd = nr_sgd; + err = sl3516_ce_run_task(ce, rctx, crypto_tfm_alg_name(areq->base.tfm)); + +theend_sgs: + if (areq->src == areq->dst) { + dma_unmap_sg(ce->dev, areq->src, sg_nents(areq->src), + DMA_BIDIRECTIONAL); + } else { + dma_unmap_sg(ce->dev, areq->src, sg_nents(areq->src), + DMA_TO_DEVICE); + dma_unmap_sg(ce->dev, areq->dst, sg_nents(areq->dst), + DMA_FROM_DEVICE); + } + +theend: + + return err; +} + +int sl3516_ce_handle_cipher_request(struct crypto_engine *engine, void *areq) +{ + int err; + struct skcipher_request *breq = container_of(areq, struct skcipher_request, base); + + err = sl3516_ce_cipher(breq); + local_bh_disable(); + crypto_finalize_skcipher_request(engine, breq, err); + local_bh_enable(); + + return 0; +} + +int sl3516_ce_skdecrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sl3516_ce_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + struct crypto_engine *engine; + + memset(rctx, 0, sizeof(struct sl3516_ce_cipher_req_ctx)); + rctx->op_dir = CE_DECRYPTION; + + if (sl3516_ce_need_fallback(areq)) + return sl3516_ce_cipher_fallback(areq); + + engine = op->ce->engine; + + return crypto_transfer_skcipher_request_to_engine(engine, areq); +} + +int sl3516_ce_skencrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sl3516_ce_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + struct crypto_engine *engine; + + memset(rctx, 0, sizeof(struct sl3516_ce_cipher_req_ctx)); + rctx->op_dir = CE_ENCRYPTION; + + if (sl3516_ce_need_fallback(areq)) + return sl3516_ce_cipher_fallback(areq); + + engine = op->ce->engine; + + return crypto_transfer_skcipher_request_to_engine(engine, areq); +} + +int sl3516_ce_cipher_init(struct crypto_tfm *tfm) +{ + struct sl3516_ce_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm); + struct sl3516_ce_alg_template *algt; + const char *name = crypto_tfm_alg_name(tfm); + struct crypto_skcipher *sktfm = __crypto_skcipher_cast(tfm); + struct skcipher_alg *alg = crypto_skcipher_alg(sktfm); + int err; + + memset(op, 0, sizeof(struct sl3516_ce_cipher_tfm_ctx)); + + algt = container_of(alg, struct sl3516_ce_alg_template, alg.skcipher.base); + op->ce = algt->ce; + + op->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(op->fallback_tfm)) { + dev_err(op->ce->dev, "ERROR: Cannot allocate fallback for %s %ld\n", + name, PTR_ERR(op->fallback_tfm)); + return PTR_ERR(op->fallback_tfm); + } + + crypto_skcipher_set_reqsize(sktfm, sizeof(struct sl3516_ce_cipher_req_ctx) + + crypto_skcipher_reqsize(op->fallback_tfm)); + + dev_info(op->ce->dev, "Fallback for %s is %s\n", + crypto_tfm_alg_driver_name(&sktfm->base), + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(op->fallback_tfm))); + + err = pm_runtime_get_sync(op->ce->dev); + if (err < 0) + goto error_pm; + + return 0; +error_pm: + pm_runtime_put_noidle(op->ce->dev); + crypto_free_skcipher(op->fallback_tfm); + return err; +} + +void sl3516_ce_cipher_exit(struct crypto_tfm *tfm) +{ + struct sl3516_ce_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm); + + kfree_sensitive(op->key); + crypto_free_skcipher(op->fallback_tfm); + pm_runtime_put_sync_suspend(op->ce->dev); +} + +int sl3516_ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sl3516_ce_dev *ce = op->ce; + + switch (keylen) { + case 128 / 8: + break; + case 192 / 8: + break; + case 256 / 8: + break; + default: + dev_dbg(ce->dev, "ERROR: Invalid keylen %u\n", keylen); + return -EINVAL; + } + kfree_sensitive(op->key); + op->keylen = keylen; + op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); + if (!op->key) + return -ENOMEM; + + crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); + + return crypto_skcipher_setkey(op->fallback_tfm, key, keylen); +} diff --git a/drivers/crypto/gemini/sl3516-ce-core.c b/drivers/crypto/gemini/sl3516-ce-core.c new file mode 100644 index 000000000..c1df5f96e --- /dev/null +++ b/drivers/crypto/gemini/sl3516-ce-core.c @@ -0,0 +1,541 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sl3516-ce-core.c - hardware cryptographic offloader for Storlink SL3516 SoC + * + * Copyright (C) 2021 Corentin Labbe + * + * Core file which registers crypto algorithms supported by the CryptoEngine + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sl3516-ce.h" + +static int sl3516_ce_desc_init(struct sl3516_ce_dev *ce) +{ + const size_t sz = sizeof(struct descriptor) * MAXDESC; + int i; + + ce->tx = dma_alloc_coherent(ce->dev, sz, &ce->dtx, GFP_KERNEL); + if (!ce->tx) + return -ENOMEM; + ce->rx = dma_alloc_coherent(ce->dev, sz, &ce->drx, GFP_KERNEL); + if (!ce->rx) + goto err_rx; + + for (i = 0; i < MAXDESC; i++) { + ce->tx[i].frame_ctrl.bits.own = CE_CPU; + ce->tx[i].next_desc.next_descriptor = ce->dtx + (i + 1) * sizeof(struct descriptor); + } + ce->tx[MAXDESC - 1].next_desc.next_descriptor = ce->dtx; + + for (i = 0; i < MAXDESC; i++) { + ce->rx[i].frame_ctrl.bits.own = CE_CPU; + ce->rx[i].next_desc.next_descriptor = ce->drx + (i + 1) * sizeof(struct descriptor); + } + ce->rx[MAXDESC - 1].next_desc.next_descriptor = ce->drx; + + ce->pctrl = dma_alloc_coherent(ce->dev, sizeof(struct pkt_control_ecb), + &ce->dctrl, GFP_KERNEL); + if (!ce->pctrl) + goto err_pctrl; + + return 0; +err_pctrl: + dma_free_coherent(ce->dev, sz, ce->rx, ce->drx); +err_rx: + dma_free_coherent(ce->dev, sz, ce->tx, ce->dtx); + return -ENOMEM; +} + +static void sl3516_ce_free_descs(struct sl3516_ce_dev *ce) +{ + const size_t sz = sizeof(struct descriptor) * MAXDESC; + + dma_free_coherent(ce->dev, sz, ce->tx, ce->dtx); + dma_free_coherent(ce->dev, sz, ce->rx, ce->drx); + dma_free_coherent(ce->dev, sizeof(struct pkt_control_ecb), ce->pctrl, + ce->dctrl); +} + +static void start_dma_tx(struct sl3516_ce_dev *ce) +{ + u32 v; + + v = TXDMA_CTRL_START | TXDMA_CTRL_CHAIN_MODE | TXDMA_CTRL_CONTINUE | \ + TXDMA_CTRL_INT_FAIL | TXDMA_CTRL_INT_PERR | TXDMA_CTRL_BURST_UNK; + + writel(v, ce->base + IPSEC_TXDMA_CTRL); +} + +static void start_dma_rx(struct sl3516_ce_dev *ce) +{ + u32 v; + + v = RXDMA_CTRL_START | RXDMA_CTRL_CHAIN_MODE | RXDMA_CTRL_CONTINUE | \ + RXDMA_CTRL_BURST_UNK | RXDMA_CTRL_INT_FINISH | \ + RXDMA_CTRL_INT_FAIL | RXDMA_CTRL_INT_PERR | \ + RXDMA_CTRL_INT_EOD | RXDMA_CTRL_INT_EOF; + + writel(v, ce->base + IPSEC_RXDMA_CTRL); +} + +static struct descriptor *get_desc_tx(struct sl3516_ce_dev *ce) +{ + struct descriptor *dd; + + dd = &ce->tx[ce->ctx]; + ce->ctx++; + if (ce->ctx >= MAXDESC) + ce->ctx = 0; + return dd; +} + +static struct descriptor *get_desc_rx(struct sl3516_ce_dev *ce) +{ + struct descriptor *rdd; + + rdd = &ce->rx[ce->crx]; + ce->crx++; + if (ce->crx >= MAXDESC) + ce->crx = 0; + return rdd; +} + +int sl3516_ce_run_task(struct sl3516_ce_dev *ce, struct sl3516_ce_cipher_req_ctx *rctx, + const char *name) +{ + struct descriptor *dd, *rdd = NULL; + u32 v; + int i, err = 0; + + ce->stat_req++; + + reinit_completion(&ce->complete); + ce->status = 0; + + for (i = 0; i < rctx->nr_sgd; i++) { + dev_dbg(ce->dev, "%s handle DST SG %d/%d len=%d\n", __func__, + i, rctx->nr_sgd, rctx->t_dst[i].len); + rdd = get_desc_rx(ce); + rdd->buf_adr = rctx->t_dst[i].addr; + rdd->frame_ctrl.bits.buffer_size = rctx->t_dst[i].len; + rdd->frame_ctrl.bits.own = CE_DMA; + } + rdd->next_desc.bits.eofie = 1; + + for (i = 0; i < rctx->nr_sgs; i++) { + dev_dbg(ce->dev, "%s handle SRC SG %d/%d len=%d\n", __func__, + i, rctx->nr_sgs, rctx->t_src[i].len); + rctx->h->algorithm_len = rctx->t_src[i].len; + + dd = get_desc_tx(ce); + dd->frame_ctrl.raw = 0; + dd->flag_status.raw = 0; + dd->frame_ctrl.bits.buffer_size = rctx->pctrllen; + dd->buf_adr = ce->dctrl; + dd->flag_status.tx_flag.tqflag = rctx->tqflag; + dd->next_desc.bits.eofie = 0; + dd->next_desc.bits.dec = 0; + dd->next_desc.bits.sof_eof = DESC_FIRST | DESC_LAST; + dd->frame_ctrl.bits.own = CE_DMA; + + dd = get_desc_tx(ce); + dd->frame_ctrl.raw = 0; + dd->flag_status.raw = 0; + dd->frame_ctrl.bits.buffer_size = rctx->t_src[i].len; + dd->buf_adr = rctx->t_src[i].addr; + dd->flag_status.tx_flag.tqflag = 0; + dd->next_desc.bits.eofie = 0; + dd->next_desc.bits.dec = 0; + dd->next_desc.bits.sof_eof = DESC_FIRST | DESC_LAST; + dd->frame_ctrl.bits.own = CE_DMA; + start_dma_tx(ce); + start_dma_rx(ce); + } + wait_for_completion_interruptible_timeout(&ce->complete, + msecs_to_jiffies(5000)); + if (ce->status == 0) { + dev_err(ce->dev, "DMA timeout for %s\n", name); + err = -EFAULT; + } + v = readl(ce->base + IPSEC_STATUS_REG); + if (v & 0xFFF) { + dev_err(ce->dev, "IPSEC_STATUS_REG %x\n", v); + err = -EFAULT; + } + + return err; +} + +static irqreturn_t ce_irq_handler(int irq, void *data) +{ + struct sl3516_ce_dev *ce = (struct sl3516_ce_dev *)data; + u32 v; + + ce->stat_irq++; + + v = readl(ce->base + IPSEC_DMA_STATUS); + writel(v, ce->base + IPSEC_DMA_STATUS); + + if (v & DMA_STATUS_TS_DERR) + dev_err(ce->dev, "AHB bus Error While Tx !!!\n"); + if (v & DMA_STATUS_TS_PERR) + dev_err(ce->dev, "Tx Descriptor Protocol Error !!!\n"); + if (v & DMA_STATUS_RS_DERR) + dev_err(ce->dev, "AHB bus Error While Rx !!!\n"); + if (v & DMA_STATUS_RS_PERR) + dev_err(ce->dev, "Rx Descriptor Protocol Error !!!\n"); + + if (v & DMA_STATUS_TS_EOFI) + ce->stat_irq_tx++; + if (v & DMA_STATUS_RS_EOFI) { + ce->status = 1; + complete(&ce->complete); + ce->stat_irq_rx++; + return IRQ_HANDLED; + } + + return IRQ_HANDLED; +} + +static struct sl3516_ce_alg_template ce_algs[] = { +{ + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .mode = ECB_AES, + .alg.skcipher.base = { + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-sl3516", + .cra_priority = 400, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct sl3516_ce_cipher_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 0xf, + .cra_init = sl3516_ce_cipher_init, + .cra_exit = sl3516_ce_cipher_exit, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = sl3516_ce_aes_setkey, + .encrypt = sl3516_ce_skencrypt, + .decrypt = sl3516_ce_skdecrypt, + }, + .alg.skcipher.op = { + .do_one_request = sl3516_ce_handle_cipher_request, + }, +}, +}; + +static int sl3516_ce_debugfs_show(struct seq_file *seq, void *v) +{ + struct sl3516_ce_dev *ce = seq->private; + unsigned int i; + + seq_printf(seq, "HWRNG %lu %lu\n", + ce->hwrng_stat_req, ce->hwrng_stat_bytes); + seq_printf(seq, "IRQ %lu\n", ce->stat_irq); + seq_printf(seq, "IRQ TX %lu\n", ce->stat_irq_tx); + seq_printf(seq, "IRQ RX %lu\n", ce->stat_irq_rx); + seq_printf(seq, "nreq %lu\n", ce->stat_req); + seq_printf(seq, "fallback SG count TX %lu\n", ce->fallback_sg_count_tx); + seq_printf(seq, "fallback SG count RX %lu\n", ce->fallback_sg_count_rx); + seq_printf(seq, "fallback modulo16 %lu\n", ce->fallback_mod16); + seq_printf(seq, "fallback align16 %lu\n", ce->fallback_align16); + seq_printf(seq, "fallback not same len %lu\n", ce->fallback_not_same_len); + + for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { + if (!ce_algs[i].ce) + continue; + switch (ce_algs[i].type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n", + ce_algs[i].alg.skcipher.base.base.cra_driver_name, + ce_algs[i].alg.skcipher.base.base.cra_name, + ce_algs[i].stat_req, ce_algs[i].stat_fb); + break; + } + } + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(sl3516_ce_debugfs); + +static int sl3516_ce_register_algs(struct sl3516_ce_dev *ce) +{ + int err; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { + ce_algs[i].ce = ce; + switch (ce_algs[i].type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + dev_info(ce->dev, "DEBUG: Register %s\n", + ce_algs[i].alg.skcipher.base.base.cra_name); + err = crypto_engine_register_skcipher(&ce_algs[i].alg.skcipher); + if (err) { + dev_err(ce->dev, "Fail to register %s\n", + ce_algs[i].alg.skcipher.base.base.cra_name); + ce_algs[i].ce = NULL; + return err; + } + break; + default: + ce_algs[i].ce = NULL; + dev_err(ce->dev, "ERROR: tried to register an unknown algo\n"); + } + } + return 0; +} + +static void sl3516_ce_unregister_algs(struct sl3516_ce_dev *ce) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { + if (!ce_algs[i].ce) + continue; + switch (ce_algs[i].type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + dev_info(ce->dev, "Unregister %d %s\n", i, + ce_algs[i].alg.skcipher.base.base.cra_name); + crypto_engine_unregister_skcipher(&ce_algs[i].alg.skcipher); + break; + } + } +} + +static void sl3516_ce_start(struct sl3516_ce_dev *ce) +{ + ce->ctx = 0; + ce->crx = 0; + writel(ce->dtx, ce->base + IPSEC_TXDMA_CURR_DESC); + writel(ce->drx, ce->base + IPSEC_RXDMA_CURR_DESC); + writel(0, ce->base + IPSEC_DMA_STATUS); +} + +/* + * Power management strategy: The device is suspended unless a TFM exists for + * one of the algorithms proposed by this driver. + */ +static int sl3516_ce_pm_suspend(struct device *dev) +{ + struct sl3516_ce_dev *ce = dev_get_drvdata(dev); + + reset_control_assert(ce->reset); + clk_disable_unprepare(ce->clks); + return 0; +} + +static int sl3516_ce_pm_resume(struct device *dev) +{ + struct sl3516_ce_dev *ce = dev_get_drvdata(dev); + int err; + + err = clk_prepare_enable(ce->clks); + if (err) { + dev_err(ce->dev, "Cannot prepare_enable\n"); + goto error; + } + err = reset_control_deassert(ce->reset); + if (err) { + dev_err(ce->dev, "Cannot deassert reset control\n"); + goto error; + } + + sl3516_ce_start(ce); + + return 0; +error: + sl3516_ce_pm_suspend(dev); + return err; +} + +static const struct dev_pm_ops sl3516_ce_pm_ops = { + SET_RUNTIME_PM_OPS(sl3516_ce_pm_suspend, sl3516_ce_pm_resume, NULL) +}; + +static int sl3516_ce_pm_init(struct sl3516_ce_dev *ce) +{ + int err; + + pm_runtime_use_autosuspend(ce->dev); + pm_runtime_set_autosuspend_delay(ce->dev, 2000); + + err = pm_runtime_set_suspended(ce->dev); + if (err) + return err; + pm_runtime_enable(ce->dev); + return err; +} + +static void sl3516_ce_pm_exit(struct sl3516_ce_dev *ce) +{ + pm_runtime_disable(ce->dev); +} + +static int sl3516_ce_probe(struct platform_device *pdev) +{ + struct sl3516_ce_dev *ce; + int err, irq; + u32 v; + + ce = devm_kzalloc(&pdev->dev, sizeof(*ce), GFP_KERNEL); + if (!ce) + return -ENOMEM; + + ce->dev = &pdev->dev; + platform_set_drvdata(pdev, ce); + + ce->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(ce->base)) + return PTR_ERR(ce->base); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + err = devm_request_irq(&pdev->dev, irq, ce_irq_handler, 0, "crypto", ce); + if (err) + return err; + + ce->reset = devm_reset_control_get(&pdev->dev, NULL); + if (IS_ERR(ce->reset)) + return dev_err_probe(&pdev->dev, PTR_ERR(ce->reset), + "No reset control found\n"); + ce->clks = devm_clk_get(ce->dev, NULL); + if (IS_ERR(ce->clks)) { + err = PTR_ERR(ce->clks); + dev_err(ce->dev, "Cannot get clock err=%d\n", err); + return err; + } + + err = sl3516_ce_desc_init(ce); + if (err) + return err; + + err = sl3516_ce_pm_init(ce); + if (err) + goto error_pm; + + init_completion(&ce->complete); + + ce->engine = crypto_engine_alloc_init(ce->dev, true); + if (!ce->engine) { + dev_err(ce->dev, "Cannot allocate engine\n"); + err = -ENOMEM; + goto error_engine; + } + + err = crypto_engine_start(ce->engine); + if (err) { + dev_err(ce->dev, "Cannot start engine\n"); + goto error_engine; + } + + err = sl3516_ce_register_algs(ce); + if (err) + goto error_alg; + + err = sl3516_ce_rng_register(ce); + if (err) + goto error_rng; + + err = pm_runtime_resume_and_get(ce->dev); + if (err < 0) + goto error_pmuse; + + v = readl(ce->base + IPSEC_ID); + dev_info(ce->dev, "SL3516 dev %lx rev %lx\n", + v & GENMASK(31, 4), + v & GENMASK(3, 0)); + v = readl(ce->base + IPSEC_DMA_DEVICE_ID); + dev_info(ce->dev, "SL3516 DMA dev %lx rev %lx\n", + v & GENMASK(15, 4), + v & GENMASK(3, 0)); + + pm_runtime_put_sync(ce->dev); + + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SL3516_DEBUG)) { + struct dentry *dbgfs_dir __maybe_unused; + struct dentry *dbgfs_stats __maybe_unused; + + /* Ignore error of debugfs */ + dbgfs_dir = debugfs_create_dir("sl3516", NULL); + dbgfs_stats = debugfs_create_file("stats", 0444, + dbgfs_dir, ce, + &sl3516_ce_debugfs_fops); +#ifdef CONFIG_CRYPTO_DEV_SL3516_DEBUG + ce->dbgfs_dir = dbgfs_dir; + ce->dbgfs_stats = dbgfs_stats; +#endif + } + + return 0; +error_pmuse: + sl3516_ce_rng_unregister(ce); +error_rng: + sl3516_ce_unregister_algs(ce); +error_alg: + crypto_engine_exit(ce->engine); +error_engine: + sl3516_ce_pm_exit(ce); +error_pm: + sl3516_ce_free_descs(ce); + return err; +} + +static void sl3516_ce_remove(struct platform_device *pdev) +{ + struct sl3516_ce_dev *ce = platform_get_drvdata(pdev); + + sl3516_ce_rng_unregister(ce); + sl3516_ce_unregister_algs(ce); + crypto_engine_exit(ce->engine); + sl3516_ce_pm_exit(ce); + sl3516_ce_free_descs(ce); + +#ifdef CONFIG_CRYPTO_DEV_SL3516_DEBUG + debugfs_remove_recursive(ce->dbgfs_dir); +#endif +} + +static const struct of_device_id sl3516_ce_crypto_of_match_table[] = { + { .compatible = "cortina,sl3516-crypto"}, + {} +}; +MODULE_DEVICE_TABLE(of, sl3516_ce_crypto_of_match_table); + +static struct platform_driver sl3516_ce_driver = { + .probe = sl3516_ce_probe, + .remove = sl3516_ce_remove, + .driver = { + .name = "sl3516-crypto", + .pm = &sl3516_ce_pm_ops, + .of_match_table = sl3516_ce_crypto_of_match_table, + }, +}; + +module_platform_driver(sl3516_ce_driver); + +MODULE_DESCRIPTION("SL3516 cryptographic offloader"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Corentin Labbe "); diff --git a/drivers/crypto/gemini/sl3516-ce-rng.c b/drivers/crypto/gemini/sl3516-ce-rng.c new file mode 100644 index 000000000..76931ec1c --- /dev/null +++ b/drivers/crypto/gemini/sl3516-ce-rng.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sl3516-ce-rng.c - hardware cryptographic offloader for SL3516 SoC. + * + * Copyright (C) 2021 Corentin Labbe + * + * This file handle the RNG found in the SL3516 crypto engine + */ +#include "sl3516-ce.h" +#include +#include + +static int sl3516_ce_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) +{ + struct sl3516_ce_dev *ce; + u32 *data = buf; + size_t read = 0; + int err; + + ce = container_of(rng, struct sl3516_ce_dev, trng); + +#ifdef CONFIG_CRYPTO_DEV_SL3516_DEBUG + ce->hwrng_stat_req++; + ce->hwrng_stat_bytes += max; +#endif + + err = pm_runtime_get_sync(ce->dev); + if (err < 0) { + pm_runtime_put_noidle(ce->dev); + return err; + } + + while (read < max) { + *data = readl(ce->base + IPSEC_RAND_NUM_REG); + data++; + read += 4; + } + + pm_runtime_put(ce->dev); + + return read; +} + +int sl3516_ce_rng_register(struct sl3516_ce_dev *ce) +{ + int ret; + + ce->trng.name = "SL3516 Crypto Engine RNG"; + ce->trng.read = sl3516_ce_rng_read; + ce->trng.quality = 700; + + ret = hwrng_register(&ce->trng); + if (ret) + dev_err(ce->dev, "Fail to register the RNG\n"); + return ret; +} + +void sl3516_ce_rng_unregister(struct sl3516_ce_dev *ce) +{ + hwrng_unregister(&ce->trng); +} diff --git a/drivers/crypto/gemini/sl3516-ce.h b/drivers/crypto/gemini/sl3516-ce.h new file mode 100644 index 000000000..56b844d0c --- /dev/null +++ b/drivers/crypto/gemini/sl3516-ce.h @@ -0,0 +1,341 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * sl3516-ce.h - hardware cryptographic offloader for cortina/gemini SoC + * + * Copyright (C) 2021 Corentin LABBE + * + * General notes on this driver: + * Called either Crypto Acceleration Engine Module, Security Acceleration Engine + * or IPSEC module in the datasheet, it will be called Crypto Engine for short + * in this driver. + * The CE was designed to handle IPSEC and wifi(TKIP WEP) protocol. + * It can handle AES, DES, 3DES, MD5, WEP, TKIP, SHA1, HMAC(MD5), HMAC(SHA1), + * Michael cipher/digest suites. + * It acts the same as a network hw, with both RX and TX chained descriptors. + */ +#include +#include +#include +#include +#include +#include + +#define TQ0_TYPE_DATA 0 +#define TQ0_TYPE_CTRL BIT(0) +#define TQ1_CIPHER BIT(1) +#define TQ2_AUTH BIT(2) +#define TQ3_IV BIT(3) +#define TQ4_KEY0 BIT(4) +#define TQ5_KEY4 BIT(5) +#define TQ6_KEY6 BIT(6) +#define TQ7_AKEY0 BIT(7) +#define TQ8_AKEY2 BIT(8) +#define TQ9_AKEY2 BIT(9) + +#define ECB_AES 0x2 + +#define DESC_LAST 0x01 +#define DESC_FIRST 0x02 + +#define IPSEC_ID 0x0000 +#define IPSEC_STATUS_REG 0x00a8 +#define IPSEC_RAND_NUM_REG 0x00ac +#define IPSEC_DMA_DEVICE_ID 0xff00 +#define IPSEC_DMA_STATUS 0xff04 +#define IPSEC_TXDMA_CTRL 0xff08 +#define IPSEC_TXDMA_FIRST_DESC 0xff0c +#define IPSEC_TXDMA_CURR_DESC 0xff10 +#define IPSEC_RXDMA_CTRL 0xff14 +#define IPSEC_RXDMA_FIRST_DESC 0xff18 +#define IPSEC_RXDMA_CURR_DESC 0xff1c +#define IPSEC_TXDMA_BUF_ADDR 0xff28 +#define IPSEC_RXDMA_BUF_ADDR 0xff38 +#define IPSEC_RXDMA_BUF_SIZE 0xff30 + +#define CE_ENCRYPTION 0x01 +#define CE_DECRYPTION 0x03 + +#define MAXDESC 6 + +#define DMA_STATUS_RS_EOFI BIT(22) +#define DMA_STATUS_RS_PERR BIT(24) +#define DMA_STATUS_RS_DERR BIT(25) +#define DMA_STATUS_TS_EOFI BIT(27) +#define DMA_STATUS_TS_PERR BIT(29) +#define DMA_STATUS_TS_DERR BIT(30) + +#define TXDMA_CTRL_START BIT(31) +#define TXDMA_CTRL_CONTINUE BIT(30) +#define TXDMA_CTRL_CHAIN_MODE BIT(29) +/* the burst value is not documented in the datasheet */ +#define TXDMA_CTRL_BURST_UNK BIT(22) +#define TXDMA_CTRL_INT_FAIL BIT(17) +#define TXDMA_CTRL_INT_PERR BIT(16) + +#define RXDMA_CTRL_START BIT(31) +#define RXDMA_CTRL_CONTINUE BIT(30) +#define RXDMA_CTRL_CHAIN_MODE BIT(29) +/* the burst value is not documented in the datasheet */ +#define RXDMA_CTRL_BURST_UNK BIT(22) +#define RXDMA_CTRL_INT_FINISH BIT(18) +#define RXDMA_CTRL_INT_FAIL BIT(17) +#define RXDMA_CTRL_INT_PERR BIT(16) +#define RXDMA_CTRL_INT_EOD BIT(15) +#define RXDMA_CTRL_INT_EOF BIT(14) + +#define CE_CPU 0 +#define CE_DMA 1 + +/* + * struct sl3516_ce_descriptor - descriptor for CE operations + * @frame_ctrl: Information for the current descriptor + * @flag_status: For send packet, describe flag of operations. + * @buf_adr: pointer to a send/recv buffer for data packet + * @next_desc: control linking to other descriptors + */ +struct descriptor { + union { + u32 raw; + /* + * struct desc_frame_ctrl - Information for the current descriptor + * @buffer_size: the size of buffer at buf_adr + * @desc_count: Upon completion of a DMA operation, DMA + * write the number of descriptors used + * for the current frame + * @checksum: unknown + * @authcomp: unknown + * @perr: Protocol error during processing this descriptor + * @derr: Data error during processing this descriptor + * @own: 0 if owned by CPU, 1 for DMA + */ + struct desc_frame_ctrl { + u32 buffer_size :16; + u32 desc_count :6; + u32 checksum :6; + u32 authcomp :1; + u32 perr :1; + u32 derr :1; + u32 own :1; + } bits; + } frame_ctrl; + + union { + u32 raw; + /* + * struct desc_flag_status - flag for this descriptor + * @tqflag: list of flag describing the type of operation + * to be performed. + */ + struct desc_tx_flag_status { + u32 tqflag :10; + u32 unused :22; + } tx_flag; + } flag_status; + + u32 buf_adr; + + union { + u32 next_descriptor; + /* + * struct desc_next - describe chaining of descriptors + * @sof_eof: does the descriptor is first (0x11), + * the last (0x01), middle of a chan (0x00) + * or the only one (0x11) + * @dec: AHB bus address increase (0), decrease (1) + * @eofie: End of frame interrupt enable + * @ndar: Next descriptor address + */ + struct desc_next { + u32 sof_eof :2; + u32 dec :1; + u32 eofie :1; + u32 ndar :28; + } bits; + } next_desc; +}; + +/* + * struct control - The value of this register is used to set the + * operation mode of the IPSec Module. + * @process_id: Used to identify the process. The number will be copied + * to the descriptor status of the received packet. + * @auth_check_len: Number of 32-bit words to be checked or appended by the + * authentication module + * @auth_algorithm: + * @auth_mode: 0:append 1:Check Authentication Result + * @fcs_stream_copy: 0:enable 1:disable authentication stream copy + * @mix_key_sel: 0:use rCipherKey0-3 1:use Key Mixer + * @aesnk: AES Key Size + * @cipher_algorithm: choice of CBC/ECE and AES/DES/3DES + * @op_mode: Operation Mode for the IPSec Module + */ +struct pkt_control_header { + u32 process_id :8; + u32 auth_check_len :3; + u32 un1 :1; + u32 auth_algorithm :3; + u32 auth_mode :1; + u32 fcs_stream_copy :1; + u32 un2 :2; + u32 mix_key_sel :1; + u32 aesnk :4; + u32 cipher_algorithm :3; + u32 un3 :1; + u32 op_mode :4; +}; + +struct pkt_control_cipher { + u32 algorithm_len :16; + u32 header_len :16; +}; + +/* + * struct pkt_control_ecb - control packet for ECB + */ +struct pkt_control_ecb { + struct pkt_control_header control; + struct pkt_control_cipher cipher; + unsigned char key[AES_MAX_KEY_SIZE]; +}; + +/* + * struct sl3516_ce_dev - main container for all this driver information + * @base: base address + * @clks: clocks used + * @reset: pointer to reset controller + * @dev: the platform device + * @engine: ptr to the crypto/crypto_engine + * @complete: completion for the current task on this flow + * @status: set to 1 by interrupt if task is done + * @dtx: base DMA address for TX descriptors + * @tx base address of TX descriptors + * @drx: base DMA address for RX descriptors + * @rx base address of RX descriptors + * @ctx current used TX descriptor + * @crx current used RX descriptor + * @trng hw_random structure for RNG + * @hwrng_stat_req number of HWRNG requests + * @hwrng_stat_bytes total number of bytes generated by RNG + * @stat_irq number of IRQ handled by CE + * @stat_irq_tx number of TX IRQ handled by CE + * @stat_irq_rx number of RX IRQ handled by CE + * @stat_req number of requests handled by CE + * @fallbak_sg_count_tx number of fallback due to destination SG count + * @fallbak_sg_count_rx number of fallback due to source SG count + * @fallbak_not_same_len number of fallback due to difference in SG length + * @dbgfs_dir: Debugfs dentry for statistic directory + * @dbgfs_stats: Debugfs dentry for statistic counters + */ +struct sl3516_ce_dev { + void __iomem *base; + struct clk *clks; + struct reset_control *reset; + struct device *dev; + struct crypto_engine *engine; + struct completion complete; + int status; + dma_addr_t dtx; + struct descriptor *tx; + dma_addr_t drx; + struct descriptor *rx; + int ctx; + int crx; + struct hwrng trng; + unsigned long hwrng_stat_req; + unsigned long hwrng_stat_bytes; + unsigned long stat_irq; + unsigned long stat_irq_tx; + unsigned long stat_irq_rx; + unsigned long stat_req; + unsigned long fallback_sg_count_tx; + unsigned long fallback_sg_count_rx; + unsigned long fallback_not_same_len; + unsigned long fallback_mod16; + unsigned long fallback_align16; +#ifdef CONFIG_CRYPTO_DEV_SL3516_DEBUG + struct dentry *dbgfs_dir; + struct dentry *dbgfs_stats; +#endif + void *pctrl; + dma_addr_t dctrl; +}; + +struct sginfo { + u32 addr; + u32 len; +}; + +/* + * struct sl3516_ce_cipher_req_ctx - context for a skcipher request + * @t_src: list of mapped SGs with their size + * @t_dst: list of mapped SGs with their size + * @op_dir: direction (encrypt vs decrypt) for this request + * @pctrllen: the length of the ctrl packet + * @tqflag: the TQflag to set in data packet + * @h pointer to the pkt_control_cipher header + * @nr_sgs: number of source SG + * @nr_sgd: number of destination SG + * @fallback_req: request struct for invoking the fallback skcipher TFM + */ +struct sl3516_ce_cipher_req_ctx { + struct sginfo t_src[MAXDESC]; + struct sginfo t_dst[MAXDESC]; + u32 op_dir; + unsigned int pctrllen; + u32 tqflag; + struct pkt_control_cipher *h; + int nr_sgs; + int nr_sgd; + struct skcipher_request fallback_req; // keep at the end +}; + +/* + * struct sl3516_ce_cipher_tfm_ctx - context for a skcipher TFM + * @key: pointer to key data + * @keylen: len of the key + * @ce: pointer to the private data of driver handling this TFM + * @fallback_tfm: pointer to the fallback TFM + */ +struct sl3516_ce_cipher_tfm_ctx { + u32 *key; + u32 keylen; + struct sl3516_ce_dev *ce; + struct crypto_skcipher *fallback_tfm; +}; + +/* + * struct sl3516_ce_alg_template - crypto_alg template + * @type: the CRYPTO_ALG_TYPE for this template + * @mode: value to be used in control packet for this algorithm + * @ce: pointer to the sl3516_ce_dev structure associated with + * this template + * @alg: one of sub struct must be used + * @stat_req: number of request done on this template + * @stat_fb: number of request which has fallbacked + * @stat_bytes: total data size done by this template + */ +struct sl3516_ce_alg_template { + u32 type; + u32 mode; + struct sl3516_ce_dev *ce; + union { + struct skcipher_engine_alg skcipher; + } alg; + unsigned long stat_req; + unsigned long stat_fb; + unsigned long stat_bytes; +}; + +int sl3516_ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen); +int sl3516_ce_cipher_init(struct crypto_tfm *tfm); +void sl3516_ce_cipher_exit(struct crypto_tfm *tfm); +int sl3516_ce_skdecrypt(struct skcipher_request *areq); +int sl3516_ce_skencrypt(struct skcipher_request *areq); + +int sl3516_ce_run_task(struct sl3516_ce_dev *ce, + struct sl3516_ce_cipher_req_ctx *rctx, const char *name); + +int sl3516_ce_rng_register(struct sl3516_ce_dev *ce); +void sl3516_ce_rng_unregister(struct sl3516_ce_dev *ce); +int sl3516_ce_handle_cipher_request(struct crypto_engine *engine, void *areq); diff --git a/drivers/crypto/geode-aes.c b/drivers/crypto/geode-aes.c new file mode 100644 index 000000000..d933f26ae --- /dev/null +++ b/drivers/crypto/geode-aes.c @@ -0,0 +1,436 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + /* Copyright (C) 2004-2006, Advanced Micro Devices, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "geode-aes.h" + +/* Static structures */ + +static void __iomem *_iobase; +static DEFINE_SPINLOCK(lock); + +/* Write a 128 bit field (either a writable key or IV) */ +static inline void +_writefield(u32 offset, const void *value) +{ + int i; + + for (i = 0; i < 4; i++) + iowrite32(((const u32 *) value)[i], _iobase + offset + (i * 4)); +} + +/* Read a 128 bit field (either a writable key or IV) */ +static inline void +_readfield(u32 offset, void *value) +{ + int i; + + for (i = 0; i < 4; i++) + ((u32 *) value)[i] = ioread32(_iobase + offset + (i * 4)); +} + +static int +do_crypt(const void *src, void *dst, u32 len, u32 flags) +{ + u32 status; + u32 counter = AES_OP_TIMEOUT; + + iowrite32(virt_to_phys((void *)src), _iobase + AES_SOURCEA_REG); + iowrite32(virt_to_phys(dst), _iobase + AES_DSTA_REG); + iowrite32(len, _iobase + AES_LENA_REG); + + /* Start the operation */ + iowrite32(AES_CTRL_START | flags, _iobase + AES_CTRLA_REG); + + do { + status = ioread32(_iobase + AES_INTR_REG); + cpu_relax(); + } while (!(status & AES_INTRA_PENDING) && --counter); + + /* Clear the event */ + iowrite32((status & 0xFF) | AES_INTRA_PENDING, _iobase + AES_INTR_REG); + return counter ? 0 : 1; +} + +static void +geode_aes_crypt(const struct geode_aes_tfm_ctx *tctx, const void *src, + void *dst, u32 len, u8 *iv, int mode, int dir) +{ + u32 flags = 0; + unsigned long iflags; + int ret; + + /* If the source and destination is the same, then + * we need to turn on the coherent flags, otherwise + * we don't need to worry + */ + + flags |= (AES_CTRL_DCA | AES_CTRL_SCA); + + if (dir == AES_DIR_ENCRYPT) + flags |= AES_CTRL_ENCRYPT; + + /* Start the critical section */ + + spin_lock_irqsave(&lock, iflags); + + if (mode == AES_MODE_CBC) { + flags |= AES_CTRL_CBC; + _writefield(AES_WRITEIV0_REG, iv); + } + + flags |= AES_CTRL_WRKEY; + _writefield(AES_WRITEKEY0_REG, tctx->key); + + ret = do_crypt(src, dst, len, flags); + BUG_ON(ret); + + if (mode == AES_MODE_CBC) + _readfield(AES_WRITEIV0_REG, iv); + + spin_unlock_irqrestore(&lock, iflags); +} + +/* CRYPTO-API Functions */ + +static int geode_setkey_cip(struct crypto_tfm *tfm, const u8 *key, + unsigned int len) +{ + struct geode_aes_tfm_ctx *tctx = crypto_tfm_ctx(tfm); + + tctx->keylen = len; + + if (len == AES_KEYSIZE_128) { + memcpy(tctx->key, key, len); + return 0; + } + + if (len != AES_KEYSIZE_192 && len != AES_KEYSIZE_256) + /* not supported at all */ + return -EINVAL; + + /* + * The requested key size is not supported by HW, do a fallback + */ + tctx->fallback.cip->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; + tctx->fallback.cip->base.crt_flags |= + (tfm->crt_flags & CRYPTO_TFM_REQ_MASK); + + return crypto_cipher_setkey(tctx->fallback.cip, key, len); +} + +static int geode_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key, + unsigned int len) +{ + struct geode_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + + tctx->keylen = len; + + if (len == AES_KEYSIZE_128) { + memcpy(tctx->key, key, len); + return 0; + } + + if (len != AES_KEYSIZE_192 && len != AES_KEYSIZE_256) + /* not supported at all */ + return -EINVAL; + + /* + * The requested key size is not supported by HW, do a fallback + */ + crypto_skcipher_clear_flags(tctx->fallback.skcipher, + CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(tctx->fallback.skcipher, + crypto_skcipher_get_flags(tfm) & + CRYPTO_TFM_REQ_MASK); + return crypto_skcipher_setkey(tctx->fallback.skcipher, key, len); +} + +static void +geode_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) +{ + const struct geode_aes_tfm_ctx *tctx = crypto_tfm_ctx(tfm); + + if (unlikely(tctx->keylen != AES_KEYSIZE_128)) { + crypto_cipher_encrypt_one(tctx->fallback.cip, out, in); + return; + } + + geode_aes_crypt(tctx, in, out, AES_BLOCK_SIZE, NULL, + AES_MODE_ECB, AES_DIR_ENCRYPT); +} + + +static void +geode_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) +{ + const struct geode_aes_tfm_ctx *tctx = crypto_tfm_ctx(tfm); + + if (unlikely(tctx->keylen != AES_KEYSIZE_128)) { + crypto_cipher_decrypt_one(tctx->fallback.cip, out, in); + return; + } + + geode_aes_crypt(tctx, in, out, AES_BLOCK_SIZE, NULL, + AES_MODE_ECB, AES_DIR_DECRYPT); +} + +static int fallback_init_cip(struct crypto_tfm *tfm) +{ + const char *name = crypto_tfm_alg_name(tfm); + struct geode_aes_tfm_ctx *tctx = crypto_tfm_ctx(tfm); + + tctx->fallback.cip = crypto_alloc_cipher(name, 0, + CRYPTO_ALG_NEED_FALLBACK); + + if (IS_ERR(tctx->fallback.cip)) { + printk(KERN_ERR "Error allocating fallback algo %s\n", name); + return PTR_ERR(tctx->fallback.cip); + } + + return 0; +} + +static void fallback_exit_cip(struct crypto_tfm *tfm) +{ + struct geode_aes_tfm_ctx *tctx = crypto_tfm_ctx(tfm); + + crypto_free_cipher(tctx->fallback.cip); +} + +static struct crypto_alg geode_alg = { + .cra_name = "aes", + .cra_driver_name = "geode-aes", + .cra_priority = 300, + .cra_alignmask = 15, + .cra_flags = CRYPTO_ALG_TYPE_CIPHER | + CRYPTO_ALG_NEED_FALLBACK, + .cra_init = fallback_init_cip, + .cra_exit = fallback_exit_cip, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct geode_aes_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_u = { + .cipher = { + .cia_min_keysize = AES_MIN_KEY_SIZE, + .cia_max_keysize = AES_MAX_KEY_SIZE, + .cia_setkey = geode_setkey_cip, + .cia_encrypt = geode_encrypt, + .cia_decrypt = geode_decrypt + } + } +}; + +static int geode_init_skcipher(struct crypto_skcipher *tfm) +{ + const char *name = crypto_tfm_alg_name(&tfm->base); + struct geode_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + + tctx->fallback.skcipher = + crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC); + if (IS_ERR(tctx->fallback.skcipher)) { + printk(KERN_ERR "Error allocating fallback algo %s\n", name); + return PTR_ERR(tctx->fallback.skcipher); + } + + crypto_skcipher_set_reqsize(tfm, sizeof(struct skcipher_request) + + crypto_skcipher_reqsize(tctx->fallback.skcipher)); + return 0; +} + +static void geode_exit_skcipher(struct crypto_skcipher *tfm) +{ + struct geode_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + + crypto_free_skcipher(tctx->fallback.skcipher); +} + +static int geode_skcipher_crypt(struct skcipher_request *req, int mode, int dir) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct geode_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; + unsigned int nbytes; + int err; + + if (unlikely(tctx->keylen != AES_KEYSIZE_128)) { + struct skcipher_request *subreq = skcipher_request_ctx(req); + + *subreq = *req; + skcipher_request_set_tfm(subreq, tctx->fallback.skcipher); + if (dir == AES_DIR_DECRYPT) + return crypto_skcipher_decrypt(subreq); + else + return crypto_skcipher_encrypt(subreq); + } + + err = skcipher_walk_virt(&walk, req, false); + + while ((nbytes = walk.nbytes) != 0) { + geode_aes_crypt(tctx, walk.src.virt.addr, walk.dst.virt.addr, + round_down(nbytes, AES_BLOCK_SIZE), + walk.iv, mode, dir); + err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE); + } + + return err; +} + +static int geode_cbc_encrypt(struct skcipher_request *req) +{ + return geode_skcipher_crypt(req, AES_MODE_CBC, AES_DIR_ENCRYPT); +} + +static int geode_cbc_decrypt(struct skcipher_request *req) +{ + return geode_skcipher_crypt(req, AES_MODE_CBC, AES_DIR_DECRYPT); +} + +static int geode_ecb_encrypt(struct skcipher_request *req) +{ + return geode_skcipher_crypt(req, AES_MODE_ECB, AES_DIR_ENCRYPT); +} + +static int geode_ecb_decrypt(struct skcipher_request *req) +{ + return geode_skcipher_crypt(req, AES_MODE_ECB, AES_DIR_DECRYPT); +} + +static struct skcipher_alg geode_skcipher_algs[] = { + { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-geode", + .base.cra_priority = 400, + .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct geode_aes_tfm_ctx), + .base.cra_alignmask = 15, + .base.cra_module = THIS_MODULE, + .init = geode_init_skcipher, + .exit = geode_exit_skcipher, + .setkey = geode_setkey_skcipher, + .encrypt = geode_cbc_encrypt, + .decrypt = geode_cbc_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-geode", + .base.cra_priority = 400, + .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct geode_aes_tfm_ctx), + .base.cra_alignmask = 15, + .base.cra_module = THIS_MODULE, + .init = geode_init_skcipher, + .exit = geode_exit_skcipher, + .setkey = geode_setkey_skcipher, + .encrypt = geode_ecb_encrypt, + .decrypt = geode_ecb_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + }, +}; + +static void geode_aes_remove(struct pci_dev *dev) +{ + crypto_unregister_alg(&geode_alg); + crypto_unregister_skciphers(geode_skcipher_algs, + ARRAY_SIZE(geode_skcipher_algs)); + + pci_iounmap(dev, _iobase); + _iobase = NULL; + + pci_release_regions(dev); + pci_disable_device(dev); +} + + +static int geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id) +{ + int ret; + + ret = pci_enable_device(dev); + if (ret) + return ret; + + ret = pci_request_regions(dev, "geode-aes"); + if (ret) + goto eenable; + + _iobase = pci_iomap(dev, 0, 0); + + if (_iobase == NULL) { + ret = -ENOMEM; + goto erequest; + } + + /* Clear any pending activity */ + iowrite32(AES_INTR_PENDING | AES_INTR_MASK, _iobase + AES_INTR_REG); + + ret = crypto_register_alg(&geode_alg); + if (ret) + goto eiomap; + + ret = crypto_register_skciphers(geode_skcipher_algs, + ARRAY_SIZE(geode_skcipher_algs)); + if (ret) + goto ealg; + + dev_notice(&dev->dev, "GEODE AES engine enabled.\n"); + return 0; + + ealg: + crypto_unregister_alg(&geode_alg); + + eiomap: + pci_iounmap(dev, _iobase); + + erequest: + pci_release_regions(dev); + + eenable: + pci_disable_device(dev); + + dev_err(&dev->dev, "GEODE AES initialization failed.\n"); + return ret; +} + +static struct pci_device_id geode_aes_tbl[] = { + { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_LX_AES), }, + { 0, } +}; + +MODULE_DEVICE_TABLE(pci, geode_aes_tbl); + +static struct pci_driver geode_aes_driver = { + .name = "Geode LX AES", + .id_table = geode_aes_tbl, + .probe = geode_aes_probe, + .remove = geode_aes_remove, +}; + +module_pci_driver(geode_aes_driver); + +MODULE_AUTHOR("Advanced Micro Devices, Inc."); +MODULE_DESCRIPTION("Geode LX Hardware AES driver"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("CRYPTO_INTERNAL"); diff --git a/drivers/crypto/geode-aes.h b/drivers/crypto/geode-aes.h new file mode 100644 index 000000000..6d0a0cdc7 --- /dev/null +++ b/drivers/crypto/geode-aes.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 2003-2006, Advanced Micro Devices, Inc. + */ + +#ifndef _GEODE_AES_H_ +#define _GEODE_AES_H_ + +/* driver logic flags */ +#define AES_MODE_ECB 0 +#define AES_MODE_CBC 1 + +#define AES_DIR_DECRYPT 0 +#define AES_DIR_ENCRYPT 1 + +#define AES_FLAGS_HIDDENKEY (1 << 0) + +/* Register definitions */ + +#define AES_CTRLA_REG 0x0000 + +#define AES_CTRL_START 0x01 +#define AES_CTRL_DECRYPT 0x00 +#define AES_CTRL_ENCRYPT 0x02 +#define AES_CTRL_WRKEY 0x04 +#define AES_CTRL_DCA 0x08 +#define AES_CTRL_SCA 0x10 +#define AES_CTRL_CBC 0x20 + +#define AES_INTR_REG 0x0008 + +#define AES_INTRA_PENDING (1 << 16) +#define AES_INTRB_PENDING (1 << 17) + +#define AES_INTR_PENDING (AES_INTRA_PENDING | AES_INTRB_PENDING) +#define AES_INTR_MASK 0x07 + +#define AES_SOURCEA_REG 0x0010 +#define AES_DSTA_REG 0x0014 +#define AES_LENA_REG 0x0018 +#define AES_WRITEKEY0_REG 0x0030 +#define AES_WRITEIV0_REG 0x0040 + +/* A very large counter that is used to gracefully bail out of an + * operation in case of trouble + */ + +#define AES_OP_TIMEOUT 0x50000 + +struct geode_aes_tfm_ctx { + u8 key[AES_KEYSIZE_128]; + union { + struct crypto_skcipher *skcipher; + struct crypto_cipher *cip; + } fallback; + u32 keylen; +}; + +#endif diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c new file mode 100644 index 000000000..46bcef044 --- /dev/null +++ b/drivers/crypto/hifn_795x.c @@ -0,0 +1,2560 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * 2007+ Copyright (c) Evgeniy Polyakov + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +static char hifn_pll_ref[sizeof("extNNN")] = "ext"; +module_param_string(hifn_pll_ref, hifn_pll_ref, sizeof(hifn_pll_ref), 0444); +MODULE_PARM_DESC(hifn_pll_ref, + "PLL reference clock (pci[freq] or ext[freq], default ext)"); + +static atomic_t hifn_dev_number; + +#define ACRYPTO_OP_DECRYPT 0 +#define ACRYPTO_OP_ENCRYPT 1 +#define ACRYPTO_OP_HMAC 2 +#define ACRYPTO_OP_RNG 3 + +#define ACRYPTO_MODE_ECB 0 +#define ACRYPTO_MODE_CBC 1 +#define ACRYPTO_MODE_CFB 2 +#define ACRYPTO_MODE_OFB 3 + +#define ACRYPTO_TYPE_AES_128 0 +#define ACRYPTO_TYPE_AES_192 1 +#define ACRYPTO_TYPE_AES_256 2 +#define ACRYPTO_TYPE_3DES 3 +#define ACRYPTO_TYPE_DES 4 + +#define PCI_VENDOR_ID_HIFN 0x13A3 +#define PCI_DEVICE_ID_HIFN_7955 0x0020 +#define PCI_DEVICE_ID_HIFN_7956 0x001d + +/* I/O region sizes */ + +#define HIFN_BAR0_SIZE 0x1000 +#define HIFN_BAR1_SIZE 0x2000 +#define HIFN_BAR2_SIZE 0x8000 + +/* DMA registres */ + +#define HIFN_DMA_CRA 0x0C /* DMA Command Ring Address */ +#define HIFN_DMA_SDRA 0x1C /* DMA Source Data Ring Address */ +#define HIFN_DMA_RRA 0x2C /* DMA Result Ring Address */ +#define HIFN_DMA_DDRA 0x3C /* DMA Destination Data Ring Address */ +#define HIFN_DMA_STCTL 0x40 /* DMA Status and Control */ +#define HIFN_DMA_INTREN 0x44 /* DMA Interrupt Enable */ +#define HIFN_DMA_CFG1 0x48 /* DMA Configuration #1 */ +#define HIFN_DMA_CFG2 0x6C /* DMA Configuration #2 */ +#define HIFN_CHIP_ID 0x98 /* Chip ID */ + +/* + * Processing Unit Registers (offset from BASEREG0) + */ +#define HIFN_0_PUDATA 0x00 /* Processing Unit Data */ +#define HIFN_0_PUCTRL 0x04 /* Processing Unit Control */ +#define HIFN_0_PUISR 0x08 /* Processing Unit Interrupt Status */ +#define HIFN_0_PUCNFG 0x0c /* Processing Unit Configuration */ +#define HIFN_0_PUIER 0x10 /* Processing Unit Interrupt Enable */ +#define HIFN_0_PUSTAT 0x14 /* Processing Unit Status/Chip ID */ +#define HIFN_0_FIFOSTAT 0x18 /* FIFO Status */ +#define HIFN_0_FIFOCNFG 0x1c /* FIFO Configuration */ +#define HIFN_0_SPACESIZE 0x20 /* Register space size */ + +/* Processing Unit Control Register (HIFN_0_PUCTRL) */ +#define HIFN_PUCTRL_CLRSRCFIFO 0x0010 /* clear source fifo */ +#define HIFN_PUCTRL_STOP 0x0008 /* stop pu */ +#define HIFN_PUCTRL_LOCKRAM 0x0004 /* lock ram */ +#define HIFN_PUCTRL_DMAENA 0x0002 /* enable dma */ +#define HIFN_PUCTRL_RESET 0x0001 /* Reset processing unit */ + +/* Processing Unit Interrupt Status Register (HIFN_0_PUISR) */ +#define HIFN_PUISR_CMDINVAL 0x8000 /* Invalid command interrupt */ +#define HIFN_PUISR_DATAERR 0x4000 /* Data error interrupt */ +#define HIFN_PUISR_SRCFIFO 0x2000 /* Source FIFO ready interrupt */ +#define HIFN_PUISR_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */ +#define HIFN_PUISR_DSTOVER 0x0200 /* Destination overrun interrupt */ +#define HIFN_PUISR_SRCCMD 0x0080 /* Source command interrupt */ +#define HIFN_PUISR_SRCCTX 0x0040 /* Source context interrupt */ +#define HIFN_PUISR_SRCDATA 0x0020 /* Source data interrupt */ +#define HIFN_PUISR_DSTDATA 0x0010 /* Destination data interrupt */ +#define HIFN_PUISR_DSTRESULT 0x0004 /* Destination result interrupt */ + +/* Processing Unit Configuration Register (HIFN_0_PUCNFG) */ +#define HIFN_PUCNFG_DRAMMASK 0xe000 /* DRAM size mask */ +#define HIFN_PUCNFG_DSZ_256K 0x0000 /* 256k dram */ +#define HIFN_PUCNFG_DSZ_512K 0x2000 /* 512k dram */ +#define HIFN_PUCNFG_DSZ_1M 0x4000 /* 1m dram */ +#define HIFN_PUCNFG_DSZ_2M 0x6000 /* 2m dram */ +#define HIFN_PUCNFG_DSZ_4M 0x8000 /* 4m dram */ +#define HIFN_PUCNFG_DSZ_8M 0xa000 /* 8m dram */ +#define HIFN_PUNCFG_DSZ_16M 0xc000 /* 16m dram */ +#define HIFN_PUCNFG_DSZ_32M 0xe000 /* 32m dram */ +#define HIFN_PUCNFG_DRAMREFRESH 0x1800 /* DRAM refresh rate mask */ +#define HIFN_PUCNFG_DRFR_512 0x0000 /* 512 divisor of ECLK */ +#define HIFN_PUCNFG_DRFR_256 0x0800 /* 256 divisor of ECLK */ +#define HIFN_PUCNFG_DRFR_128 0x1000 /* 128 divisor of ECLK */ +#define HIFN_PUCNFG_TCALLPHASES 0x0200 /* your guess is as good as mine... */ +#define HIFN_PUCNFG_TCDRVTOTEM 0x0100 /* your guess is as good as mine... */ +#define HIFN_PUCNFG_BIGENDIAN 0x0080 /* DMA big endian mode */ +#define HIFN_PUCNFG_BUS32 0x0040 /* Bus width 32bits */ +#define HIFN_PUCNFG_BUS16 0x0000 /* Bus width 16 bits */ +#define HIFN_PUCNFG_CHIPID 0x0020 /* Allow chipid from PUSTAT */ +#define HIFN_PUCNFG_DRAM 0x0010 /* Context RAM is DRAM */ +#define HIFN_PUCNFG_SRAM 0x0000 /* Context RAM is SRAM */ +#define HIFN_PUCNFG_COMPSING 0x0004 /* Enable single compression context */ +#define HIFN_PUCNFG_ENCCNFG 0x0002 /* Encryption configuration */ + +/* Processing Unit Interrupt Enable Register (HIFN_0_PUIER) */ +#define HIFN_PUIER_CMDINVAL 0x8000 /* Invalid command interrupt */ +#define HIFN_PUIER_DATAERR 0x4000 /* Data error interrupt */ +#define HIFN_PUIER_SRCFIFO 0x2000 /* Source FIFO ready interrupt */ +#define HIFN_PUIER_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */ +#define HIFN_PUIER_DSTOVER 0x0200 /* Destination overrun interrupt */ +#define HIFN_PUIER_SRCCMD 0x0080 /* Source command interrupt */ +#define HIFN_PUIER_SRCCTX 0x0040 /* Source context interrupt */ +#define HIFN_PUIER_SRCDATA 0x0020 /* Source data interrupt */ +#define HIFN_PUIER_DSTDATA 0x0010 /* Destination data interrupt */ +#define HIFN_PUIER_DSTRESULT 0x0004 /* Destination result interrupt */ + +/* Processing Unit Status Register/Chip ID (HIFN_0_PUSTAT) */ +#define HIFN_PUSTAT_CMDINVAL 0x8000 /* Invalid command interrupt */ +#define HIFN_PUSTAT_DATAERR 0x4000 /* Data error interrupt */ +#define HIFN_PUSTAT_SRCFIFO 0x2000 /* Source FIFO ready interrupt */ +#define HIFN_PUSTAT_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */ +#define HIFN_PUSTAT_DSTOVER 0x0200 /* Destination overrun interrupt */ +#define HIFN_PUSTAT_SRCCMD 0x0080 /* Source command interrupt */ +#define HIFN_PUSTAT_SRCCTX 0x0040 /* Source context interrupt */ +#define HIFN_PUSTAT_SRCDATA 0x0020 /* Source data interrupt */ +#define HIFN_PUSTAT_DSTDATA 0x0010 /* Destination data interrupt */ +#define HIFN_PUSTAT_DSTRESULT 0x0004 /* Destination result interrupt */ +#define HIFN_PUSTAT_CHIPREV 0x00ff /* Chip revision mask */ +#define HIFN_PUSTAT_CHIPENA 0xff00 /* Chip enabled mask */ +#define HIFN_PUSTAT_ENA_2 0x1100 /* Level 2 enabled */ +#define HIFN_PUSTAT_ENA_1 0x1000 /* Level 1 enabled */ +#define HIFN_PUSTAT_ENA_0 0x3000 /* Level 0 enabled */ +#define HIFN_PUSTAT_REV_2 0x0020 /* 7751 PT6/2 */ +#define HIFN_PUSTAT_REV_3 0x0030 /* 7751 PT6/3 */ + +/* FIFO Status Register (HIFN_0_FIFOSTAT) */ +#define HIFN_FIFOSTAT_SRC 0x7f00 /* Source FIFO available */ +#define HIFN_FIFOSTAT_DST 0x007f /* Destination FIFO available */ + +/* FIFO Configuration Register (HIFN_0_FIFOCNFG) */ +#define HIFN_FIFOCNFG_THRESHOLD 0x0400 /* must be written as 1 */ + +/* + * DMA Interface Registers (offset from BASEREG1) + */ +#define HIFN_1_DMA_CRAR 0x0c /* DMA Command Ring Address */ +#define HIFN_1_DMA_SRAR 0x1c /* DMA Source Ring Address */ +#define HIFN_1_DMA_RRAR 0x2c /* DMA Result Ring Address */ +#define HIFN_1_DMA_DRAR 0x3c /* DMA Destination Ring Address */ +#define HIFN_1_DMA_CSR 0x40 /* DMA Status and Control */ +#define HIFN_1_DMA_IER 0x44 /* DMA Interrupt Enable */ +#define HIFN_1_DMA_CNFG 0x48 /* DMA Configuration */ +#define HIFN_1_PLL 0x4c /* 795x: PLL config */ +#define HIFN_1_7811_RNGENA 0x60 /* 7811: rng enable */ +#define HIFN_1_7811_RNGCFG 0x64 /* 7811: rng config */ +#define HIFN_1_7811_RNGDAT 0x68 /* 7811: rng data */ +#define HIFN_1_7811_RNGSTS 0x6c /* 7811: rng status */ +#define HIFN_1_7811_MIPSRST 0x94 /* 7811: MIPS reset */ +#define HIFN_1_REVID 0x98 /* Revision ID */ +#define HIFN_1_UNLOCK_SECRET1 0xf4 +#define HIFN_1_UNLOCK_SECRET2 0xfc +#define HIFN_1_PUB_RESET 0x204 /* Public/RNG Reset */ +#define HIFN_1_PUB_BASE 0x300 /* Public Base Address */ +#define HIFN_1_PUB_OPLEN 0x304 /* Public Operand Length */ +#define HIFN_1_PUB_OP 0x308 /* Public Operand */ +#define HIFN_1_PUB_STATUS 0x30c /* Public Status */ +#define HIFN_1_PUB_IEN 0x310 /* Public Interrupt enable */ +#define HIFN_1_RNG_CONFIG 0x314 /* RNG config */ +#define HIFN_1_RNG_DATA 0x318 /* RNG data */ +#define HIFN_1_PUB_MEM 0x400 /* start of Public key memory */ +#define HIFN_1_PUB_MEMEND 0xbff /* end of Public key memory */ + +/* DMA Status and Control Register (HIFN_1_DMA_CSR) */ +#define HIFN_DMACSR_D_CTRLMASK 0xc0000000 /* Destinition Ring Control */ +#define HIFN_DMACSR_D_CTRL_NOP 0x00000000 /* Dest. Control: no-op */ +#define HIFN_DMACSR_D_CTRL_DIS 0x40000000 /* Dest. Control: disable */ +#define HIFN_DMACSR_D_CTRL_ENA 0x80000000 /* Dest. Control: enable */ +#define HIFN_DMACSR_D_ABORT 0x20000000 /* Destinition Ring PCIAbort */ +#define HIFN_DMACSR_D_DONE 0x10000000 /* Destinition Ring Done */ +#define HIFN_DMACSR_D_LAST 0x08000000 /* Destinition Ring Last */ +#define HIFN_DMACSR_D_WAIT 0x04000000 /* Destinition Ring Waiting */ +#define HIFN_DMACSR_D_OVER 0x02000000 /* Destinition Ring Overflow */ +#define HIFN_DMACSR_R_CTRL 0x00c00000 /* Result Ring Control */ +#define HIFN_DMACSR_R_CTRL_NOP 0x00000000 /* Result Control: no-op */ +#define HIFN_DMACSR_R_CTRL_DIS 0x00400000 /* Result Control: disable */ +#define HIFN_DMACSR_R_CTRL_ENA 0x00800000 /* Result Control: enable */ +#define HIFN_DMACSR_R_ABORT 0x00200000 /* Result Ring PCI Abort */ +#define HIFN_DMACSR_R_DONE 0x00100000 /* Result Ring Done */ +#define HIFN_DMACSR_R_LAST 0x00080000 /* Result Ring Last */ +#define HIFN_DMACSR_R_WAIT 0x00040000 /* Result Ring Waiting */ +#define HIFN_DMACSR_R_OVER 0x00020000 /* Result Ring Overflow */ +#define HIFN_DMACSR_S_CTRL 0x0000c000 /* Source Ring Control */ +#define HIFN_DMACSR_S_CTRL_NOP 0x00000000 /* Source Control: no-op */ +#define HIFN_DMACSR_S_CTRL_DIS 0x00004000 /* Source Control: disable */ +#define HIFN_DMACSR_S_CTRL_ENA 0x00008000 /* Source Control: enable */ +#define HIFN_DMACSR_S_ABORT 0x00002000 /* Source Ring PCI Abort */ +#define HIFN_DMACSR_S_DONE 0x00001000 /* Source Ring Done */ +#define HIFN_DMACSR_S_LAST 0x00000800 /* Source Ring Last */ +#define HIFN_DMACSR_S_WAIT 0x00000400 /* Source Ring Waiting */ +#define HIFN_DMACSR_ILLW 0x00000200 /* Illegal write (7811 only) */ +#define HIFN_DMACSR_ILLR 0x00000100 /* Illegal read (7811 only) */ +#define HIFN_DMACSR_C_CTRL 0x000000c0 /* Command Ring Control */ +#define HIFN_DMACSR_C_CTRL_NOP 0x00000000 /* Command Control: no-op */ +#define HIFN_DMACSR_C_CTRL_DIS 0x00000040 /* Command Control: disable */ +#define HIFN_DMACSR_C_CTRL_ENA 0x00000080 /* Command Control: enable */ +#define HIFN_DMACSR_C_ABORT 0x00000020 /* Command Ring PCI Abort */ +#define HIFN_DMACSR_C_DONE 0x00000010 /* Command Ring Done */ +#define HIFN_DMACSR_C_LAST 0x00000008 /* Command Ring Last */ +#define HIFN_DMACSR_C_WAIT 0x00000004 /* Command Ring Waiting */ +#define HIFN_DMACSR_PUBDONE 0x00000002 /* Public op done (7951 only) */ +#define HIFN_DMACSR_ENGINE 0x00000001 /* Command Ring Engine IRQ */ + +/* DMA Interrupt Enable Register (HIFN_1_DMA_IER) */ +#define HIFN_DMAIER_D_ABORT 0x20000000 /* Destination Ring PCIAbort */ +#define HIFN_DMAIER_D_DONE 0x10000000 /* Destination Ring Done */ +#define HIFN_DMAIER_D_LAST 0x08000000 /* Destination Ring Last */ +#define HIFN_DMAIER_D_WAIT 0x04000000 /* Destination Ring Waiting */ +#define HIFN_DMAIER_D_OVER 0x02000000 /* Destination Ring Overflow */ +#define HIFN_DMAIER_R_ABORT 0x00200000 /* Result Ring PCI Abort */ +#define HIFN_DMAIER_R_DONE 0x00100000 /* Result Ring Done */ +#define HIFN_DMAIER_R_LAST 0x00080000 /* Result Ring Last */ +#define HIFN_DMAIER_R_WAIT 0x00040000 /* Result Ring Waiting */ +#define HIFN_DMAIER_R_OVER 0x00020000 /* Result Ring Overflow */ +#define HIFN_DMAIER_S_ABORT 0x00002000 /* Source Ring PCI Abort */ +#define HIFN_DMAIER_S_DONE 0x00001000 /* Source Ring Done */ +#define HIFN_DMAIER_S_LAST 0x00000800 /* Source Ring Last */ +#define HIFN_DMAIER_S_WAIT 0x00000400 /* Source Ring Waiting */ +#define HIFN_DMAIER_ILLW 0x00000200 /* Illegal write (7811 only) */ +#define HIFN_DMAIER_ILLR 0x00000100 /* Illegal read (7811 only) */ +#define HIFN_DMAIER_C_ABORT 0x00000020 /* Command Ring PCI Abort */ +#define HIFN_DMAIER_C_DONE 0x00000010 /* Command Ring Done */ +#define HIFN_DMAIER_C_LAST 0x00000008 /* Command Ring Last */ +#define HIFN_DMAIER_C_WAIT 0x00000004 /* Command Ring Waiting */ +#define HIFN_DMAIER_PUBDONE 0x00000002 /* public op done (7951 only) */ +#define HIFN_DMAIER_ENGINE 0x00000001 /* Engine IRQ */ + +/* DMA Configuration Register (HIFN_1_DMA_CNFG) */ +#define HIFN_DMACNFG_BIGENDIAN 0x10000000 /* big endian mode */ +#define HIFN_DMACNFG_POLLFREQ 0x00ff0000 /* Poll frequency mask */ +#define HIFN_DMACNFG_UNLOCK 0x00000800 +#define HIFN_DMACNFG_POLLINVAL 0x00000700 /* Invalid Poll Scalar */ +#define HIFN_DMACNFG_LAST 0x00000010 /* Host control LAST bit */ +#define HIFN_DMACNFG_MODE 0x00000004 /* DMA mode */ +#define HIFN_DMACNFG_DMARESET 0x00000002 /* DMA Reset # */ +#define HIFN_DMACNFG_MSTRESET 0x00000001 /* Master Reset # */ + +/* PLL configuration register */ +#define HIFN_PLL_REF_CLK_HBI 0x00000000 /* HBI reference clock */ +#define HIFN_PLL_REF_CLK_PLL 0x00000001 /* PLL reference clock */ +#define HIFN_PLL_BP 0x00000002 /* Reference clock bypass */ +#define HIFN_PLL_PK_CLK_HBI 0x00000000 /* PK engine HBI clock */ +#define HIFN_PLL_PK_CLK_PLL 0x00000008 /* PK engine PLL clock */ +#define HIFN_PLL_PE_CLK_HBI 0x00000000 /* PE engine HBI clock */ +#define HIFN_PLL_PE_CLK_PLL 0x00000010 /* PE engine PLL clock */ +#define HIFN_PLL_RESERVED_1 0x00000400 /* Reserved bit, must be 1 */ +#define HIFN_PLL_ND_SHIFT 11 /* Clock multiplier shift */ +#define HIFN_PLL_ND_MULT_2 0x00000000 /* PLL clock multiplier 2 */ +#define HIFN_PLL_ND_MULT_4 0x00000800 /* PLL clock multiplier 4 */ +#define HIFN_PLL_ND_MULT_6 0x00001000 /* PLL clock multiplier 6 */ +#define HIFN_PLL_ND_MULT_8 0x00001800 /* PLL clock multiplier 8 */ +#define HIFN_PLL_ND_MULT_10 0x00002000 /* PLL clock multiplier 10 */ +#define HIFN_PLL_ND_MULT_12 0x00002800 /* PLL clock multiplier 12 */ +#define HIFN_PLL_IS_1_8 0x00000000 /* charge pump (mult. 1-8) */ +#define HIFN_PLL_IS_9_12 0x00010000 /* charge pump (mult. 9-12) */ + +#define HIFN_PLL_FCK_MAX 266 /* Maximum PLL frequency */ + +/* Public key reset register (HIFN_1_PUB_RESET) */ +#define HIFN_PUBRST_RESET 0x00000001 /* reset public/rng unit */ + +/* Public base address register (HIFN_1_PUB_BASE) */ +#define HIFN_PUBBASE_ADDR 0x00003fff /* base address */ + +/* Public operand length register (HIFN_1_PUB_OPLEN) */ +#define HIFN_PUBOPLEN_MOD_M 0x0000007f /* modulus length mask */ +#define HIFN_PUBOPLEN_MOD_S 0 /* modulus length shift */ +#define HIFN_PUBOPLEN_EXP_M 0x0003ff80 /* exponent length mask */ +#define HIFN_PUBOPLEN_EXP_S 7 /* exponent length shift */ +#define HIFN_PUBOPLEN_RED_M 0x003c0000 /* reducend length mask */ +#define HIFN_PUBOPLEN_RED_S 18 /* reducend length shift */ + +/* Public operation register (HIFN_1_PUB_OP) */ +#define HIFN_PUBOP_AOFFSET_M 0x0000007f /* A offset mask */ +#define HIFN_PUBOP_AOFFSET_S 0 /* A offset shift */ +#define HIFN_PUBOP_BOFFSET_M 0x00000f80 /* B offset mask */ +#define HIFN_PUBOP_BOFFSET_S 7 /* B offset shift */ +#define HIFN_PUBOP_MOFFSET_M 0x0003f000 /* M offset mask */ +#define HIFN_PUBOP_MOFFSET_S 12 /* M offset shift */ +#define HIFN_PUBOP_OP_MASK 0x003c0000 /* Opcode: */ +#define HIFN_PUBOP_OP_NOP 0x00000000 /* NOP */ +#define HIFN_PUBOP_OP_ADD 0x00040000 /* ADD */ +#define HIFN_PUBOP_OP_ADDC 0x00080000 /* ADD w/carry */ +#define HIFN_PUBOP_OP_SUB 0x000c0000 /* SUB */ +#define HIFN_PUBOP_OP_SUBC 0x00100000 /* SUB w/carry */ +#define HIFN_PUBOP_OP_MODADD 0x00140000 /* Modular ADD */ +#define HIFN_PUBOP_OP_MODSUB 0x00180000 /* Modular SUB */ +#define HIFN_PUBOP_OP_INCA 0x001c0000 /* INC A */ +#define HIFN_PUBOP_OP_DECA 0x00200000 /* DEC A */ +#define HIFN_PUBOP_OP_MULT 0x00240000 /* MULT */ +#define HIFN_PUBOP_OP_MODMULT 0x00280000 /* Modular MULT */ +#define HIFN_PUBOP_OP_MODRED 0x002c0000 /* Modular RED */ +#define HIFN_PUBOP_OP_MODEXP 0x00300000 /* Modular EXP */ + +/* Public status register (HIFN_1_PUB_STATUS) */ +#define HIFN_PUBSTS_DONE 0x00000001 /* operation done */ +#define HIFN_PUBSTS_CARRY 0x00000002 /* carry */ + +/* Public interrupt enable register (HIFN_1_PUB_IEN) */ +#define HIFN_PUBIEN_DONE 0x00000001 /* operation done interrupt */ + +/* Random number generator config register (HIFN_1_RNG_CONFIG) */ +#define HIFN_RNGCFG_ENA 0x00000001 /* enable rng */ + +#define HIFN_NAMESIZE 32 +#define HIFN_MAX_RESULT_ORDER 5 + +#define HIFN_D_CMD_RSIZE (24 * 1) +#define HIFN_D_SRC_RSIZE (80 * 1) +#define HIFN_D_DST_RSIZE (80 * 1) +#define HIFN_D_RES_RSIZE (24 * 1) + +#define HIFN_D_DST_DALIGN 4 + +#define HIFN_QUEUE_LENGTH (HIFN_D_CMD_RSIZE - 1) + +#define AES_MIN_KEY_SIZE 16 +#define AES_MAX_KEY_SIZE 32 + +#define HIFN_DES_KEY_LENGTH 8 +#define HIFN_3DES_KEY_LENGTH 24 +#define HIFN_MAX_CRYPT_KEY_LENGTH AES_MAX_KEY_SIZE +#define HIFN_IV_LENGTH 8 +#define HIFN_AES_IV_LENGTH 16 +#define HIFN_MAX_IV_LENGTH HIFN_AES_IV_LENGTH + +#define HIFN_MAC_KEY_LENGTH 64 +#define HIFN_MD5_LENGTH 16 +#define HIFN_SHA1_LENGTH 20 +#define HIFN_MAC_TRUNC_LENGTH 12 + +#define HIFN_MAX_COMMAND (8 + 8 + 8 + 64 + 260) +#define HIFN_MAX_RESULT (8 + 4 + 4 + 20 + 4) +#define HIFN_USED_RESULT 12 + +struct hifn_desc { + volatile __le32 l; + volatile __le32 p; +}; + +struct hifn_dma { + struct hifn_desc cmdr[HIFN_D_CMD_RSIZE + 1]; + struct hifn_desc srcr[HIFN_D_SRC_RSIZE + 1]; + struct hifn_desc dstr[HIFN_D_DST_RSIZE + 1]; + struct hifn_desc resr[HIFN_D_RES_RSIZE + 1]; + + u8 command_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_COMMAND]; + u8 result_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_RESULT]; + + /* + * Our current positions for insertion and removal from the descriptor + * rings. + */ + volatile int cmdi, srci, dsti, resi; + volatile int cmdu, srcu, dstu, resu; + int cmdk, srck, dstk, resk; +}; + +#define HIFN_FLAG_CMD_BUSY (1 << 0) +#define HIFN_FLAG_SRC_BUSY (1 << 1) +#define HIFN_FLAG_DST_BUSY (1 << 2) +#define HIFN_FLAG_RES_BUSY (1 << 3) +#define HIFN_FLAG_OLD_KEY (1 << 4) + +#define HIFN_DEFAULT_ACTIVE_NUM 5 + +struct hifn_device { + char name[HIFN_NAMESIZE]; + + int irq; + + struct pci_dev *pdev; + void __iomem *bar[3]; + + void *desc_virt; + dma_addr_t desc_dma; + + u32 dmareg; + + void *sa[HIFN_D_RES_RSIZE]; + + spinlock_t lock; + + u32 flags; + int active, started; + struct delayed_work work; + unsigned long reset; + unsigned long success; + unsigned long prev_success; + + u8 snum; + + struct tasklet_struct tasklet; + + struct crypto_queue queue; + struct list_head alg_list; + + unsigned int pk_clk_freq; + +#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG + unsigned int rng_wait_time; + ktime_t rngtime; + struct hwrng rng; +#endif +}; + +#define HIFN_D_LENGTH 0x0000ffff +#define HIFN_D_NOINVALID 0x01000000 +#define HIFN_D_MASKDONEIRQ 0x02000000 +#define HIFN_D_DESTOVER 0x04000000 +#define HIFN_D_OVER 0x08000000 +#define HIFN_D_LAST 0x20000000 +#define HIFN_D_JUMP 0x40000000 +#define HIFN_D_VALID 0x80000000 + +struct hifn_base_command { + volatile __le16 masks; + volatile __le16 session_num; + volatile __le16 total_source_count; + volatile __le16 total_dest_count; +}; + +#define HIFN_BASE_CMD_COMP 0x0100 /* enable compression engine */ +#define HIFN_BASE_CMD_PAD 0x0200 /* enable padding engine */ +#define HIFN_BASE_CMD_MAC 0x0400 /* enable MAC engine */ +#define HIFN_BASE_CMD_CRYPT 0x0800 /* enable crypt engine */ +#define HIFN_BASE_CMD_DECODE 0x2000 +#define HIFN_BASE_CMD_SRCLEN_M 0xc000 +#define HIFN_BASE_CMD_SRCLEN_S 14 +#define HIFN_BASE_CMD_DSTLEN_M 0x3000 +#define HIFN_BASE_CMD_DSTLEN_S 12 +#define HIFN_BASE_CMD_LENMASK_HI 0x30000 +#define HIFN_BASE_CMD_LENMASK_LO 0x0ffff + +/* + * Structure to help build up the command data structure. + */ +struct hifn_crypt_command { + volatile __le16 masks; + volatile __le16 header_skip; + volatile __le16 source_count; + volatile __le16 reserved; +}; + +#define HIFN_CRYPT_CMD_ALG_MASK 0x0003 /* algorithm: */ +#define HIFN_CRYPT_CMD_ALG_DES 0x0000 /* DES */ +#define HIFN_CRYPT_CMD_ALG_3DES 0x0001 /* 3DES */ +#define HIFN_CRYPT_CMD_ALG_RC4 0x0002 /* RC4 */ +#define HIFN_CRYPT_CMD_ALG_AES 0x0003 /* AES */ +#define HIFN_CRYPT_CMD_MODE_MASK 0x0018 /* Encrypt mode: */ +#define HIFN_CRYPT_CMD_MODE_ECB 0x0000 /* ECB */ +#define HIFN_CRYPT_CMD_MODE_CBC 0x0008 /* CBC */ +#define HIFN_CRYPT_CMD_MODE_CFB 0x0010 /* CFB */ +#define HIFN_CRYPT_CMD_MODE_OFB 0x0018 /* OFB */ +#define HIFN_CRYPT_CMD_CLR_CTX 0x0040 /* clear context */ +#define HIFN_CRYPT_CMD_KSZ_MASK 0x0600 /* AES key size: */ +#define HIFN_CRYPT_CMD_KSZ_128 0x0000 /* 128 bit */ +#define HIFN_CRYPT_CMD_KSZ_192 0x0200 /* 192 bit */ +#define HIFN_CRYPT_CMD_KSZ_256 0x0400 /* 256 bit */ +#define HIFN_CRYPT_CMD_NEW_KEY 0x0800 /* expect new key */ +#define HIFN_CRYPT_CMD_NEW_IV 0x1000 /* expect new iv */ +#define HIFN_CRYPT_CMD_SRCLEN_M 0xc000 +#define HIFN_CRYPT_CMD_SRCLEN_S 14 + +#define HIFN_MAC_CMD_ALG_MASK 0x0001 +#define HIFN_MAC_CMD_ALG_SHA1 0x0000 +#define HIFN_MAC_CMD_ALG_MD5 0x0001 +#define HIFN_MAC_CMD_MODE_MASK 0x000c +#define HIFN_MAC_CMD_MODE_HMAC 0x0000 +#define HIFN_MAC_CMD_MODE_SSL_MAC 0x0004 +#define HIFN_MAC_CMD_MODE_HASH 0x0008 +#define HIFN_MAC_CMD_MODE_FULL 0x0004 +#define HIFN_MAC_CMD_TRUNC 0x0010 +#define HIFN_MAC_CMD_RESULT 0x0020 +#define HIFN_MAC_CMD_APPEND 0x0040 +#define HIFN_MAC_CMD_SRCLEN_M 0xc000 +#define HIFN_MAC_CMD_SRCLEN_S 14 + +/* + * MAC POS IPsec initiates authentication after encryption on encodes + * and before decryption on decodes. + */ +#define HIFN_MAC_CMD_POS_IPSEC 0x0200 +#define HIFN_MAC_CMD_NEW_KEY 0x0800 + +#define HIFN_COMP_CMD_SRCLEN_M 0xc000 +#define HIFN_COMP_CMD_SRCLEN_S 14 +#define HIFN_COMP_CMD_ONE 0x0100 /* must be one */ +#define HIFN_COMP_CMD_CLEARHIST 0x0010 /* clear history */ +#define HIFN_COMP_CMD_UPDATEHIST 0x0008 /* update history */ +#define HIFN_COMP_CMD_LZS_STRIP0 0x0004 /* LZS: strip zero */ +#define HIFN_COMP_CMD_MPPC_RESTART 0x0004 /* MPPC: restart */ +#define HIFN_COMP_CMD_ALG_MASK 0x0001 /* compression mode: */ +#define HIFN_COMP_CMD_ALG_MPPC 0x0001 /* MPPC */ +#define HIFN_COMP_CMD_ALG_LZS 0x0000 /* LZS */ + +struct hifn_base_result { + volatile __le16 flags; + volatile __le16 session; + volatile __le16 src_cnt; /* 15:0 of source count */ + volatile __le16 dst_cnt; /* 15:0 of dest count */ +}; + +#define HIFN_BASE_RES_DSTOVERRUN 0x0200 /* destination overrun */ +#define HIFN_BASE_RES_SRCLEN_M 0xc000 /* 17:16 of source count */ +#define HIFN_BASE_RES_SRCLEN_S 14 +#define HIFN_BASE_RES_DSTLEN_M 0x3000 /* 17:16 of dest count */ +#define HIFN_BASE_RES_DSTLEN_S 12 + +struct hifn_comp_result { + volatile __le16 flags; + volatile __le16 crc; +}; + +#define HIFN_COMP_RES_LCB_M 0xff00 /* longitudinal check byte */ +#define HIFN_COMP_RES_LCB_S 8 +#define HIFN_COMP_RES_RESTART 0x0004 /* MPPC: restart */ +#define HIFN_COMP_RES_ENDMARKER 0x0002 /* LZS: end marker seen */ +#define HIFN_COMP_RES_SRC_NOTZERO 0x0001 /* source expired */ + +struct hifn_mac_result { + volatile __le16 flags; + volatile __le16 reserved; + /* followed by 0, 6, 8, or 10 u16's of the MAC, then crypt */ +}; + +#define HIFN_MAC_RES_MISCOMPARE 0x0002 /* compare failed */ +#define HIFN_MAC_RES_SRC_NOTZERO 0x0001 /* source expired */ + +struct hifn_crypt_result { + volatile __le16 flags; + volatile __le16 reserved; +}; + +#define HIFN_CRYPT_RES_SRC_NOTZERO 0x0001 /* source expired */ + +#ifndef HIFN_POLL_FREQUENCY +#define HIFN_POLL_FREQUENCY 0x1 +#endif + +#ifndef HIFN_POLL_SCALAR +#define HIFN_POLL_SCALAR 0x0 +#endif + +#define HIFN_MAX_SEGLEN 0xffff /* maximum dma segment len */ +#define HIFN_MAX_DMALEN 0x3ffff /* maximum dma length */ + +struct hifn_crypto_alg { + struct list_head entry; + struct skcipher_alg alg; + struct hifn_device *dev; +}; + +#define ASYNC_SCATTERLIST_CACHE 16 + +#define ASYNC_FLAGS_MISALIGNED (1 << 0) + +struct hifn_cipher_walk { + struct scatterlist cache[ASYNC_SCATTERLIST_CACHE]; + u32 flags; + int num; +}; + +struct hifn_context { + u8 key[HIFN_MAX_CRYPT_KEY_LENGTH]; + struct hifn_device *dev; + unsigned int keysize; +}; + +struct hifn_request_context { + u8 *iv; + unsigned int ivsize; + u8 op, type, mode, unused; + struct hifn_cipher_walk walk; +}; + +#define crypto_alg_to_hifn(a) container_of(a, struct hifn_crypto_alg, alg) + +static inline u32 hifn_read_0(struct hifn_device *dev, u32 reg) +{ + return readl(dev->bar[0] + reg); +} + +static inline u32 hifn_read_1(struct hifn_device *dev, u32 reg) +{ + return readl(dev->bar[1] + reg); +} + +static inline void hifn_write_0(struct hifn_device *dev, u32 reg, u32 val) +{ + writel((__force u32)cpu_to_le32(val), dev->bar[0] + reg); +} + +static inline void hifn_write_1(struct hifn_device *dev, u32 reg, u32 val) +{ + writel((__force u32)cpu_to_le32(val), dev->bar[1] + reg); +} + +static void hifn_wait_puc(struct hifn_device *dev) +{ + int i; + u32 ret; + + for (i = 10000; i > 0; --i) { + ret = hifn_read_0(dev, HIFN_0_PUCTRL); + if (!(ret & HIFN_PUCTRL_RESET)) + break; + + udelay(1); + } + + if (!i) + dev_err(&dev->pdev->dev, "Failed to reset PUC unit.\n"); +} + +static void hifn_reset_puc(struct hifn_device *dev) +{ + hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA); + hifn_wait_puc(dev); +} + +static void hifn_stop_device(struct hifn_device *dev) +{ + hifn_write_1(dev, HIFN_1_DMA_CSR, + HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS | + HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS); + hifn_write_0(dev, HIFN_0_PUIER, 0); + hifn_write_1(dev, HIFN_1_DMA_IER, 0); +} + +static void hifn_reset_dma(struct hifn_device *dev, int full) +{ + hifn_stop_device(dev); + + /* + * Setting poll frequency and others to 0. + */ + hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | + HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); + mdelay(1); + + /* + * Reset DMA. + */ + if (full) { + hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE); + mdelay(1); + } else { + hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE | + HIFN_DMACNFG_MSTRESET); + hifn_reset_puc(dev); + } + + hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | + HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); + + hifn_reset_puc(dev); +} + +static u32 hifn_next_signature(u32 a, u_int cnt) +{ + int i; + u32 v; + + for (i = 0; i < cnt; i++) { + /* get the parity */ + v = a & 0x80080125; + v ^= v >> 16; + v ^= v >> 8; + v ^= v >> 4; + v ^= v >> 2; + v ^= v >> 1; + + a = (v & 1) ^ (a << 1); + } + + return a; +} + +static struct pci2id { + u_short pci_vendor; + u_short pci_prod; + char card_id[13]; +} pci2id[] = { + { + PCI_VENDOR_ID_HIFN, + PCI_DEVICE_ID_HIFN_7955, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00 } + }, + { + PCI_VENDOR_ID_HIFN, + PCI_DEVICE_ID_HIFN_7956, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00 } + } +}; + +#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG +static int hifn_rng_data_present(struct hwrng *rng, int wait) +{ + struct hifn_device *dev = (struct hifn_device *)rng->priv; + s64 nsec; + + nsec = ktime_to_ns(ktime_sub(ktime_get(), dev->rngtime)); + nsec -= dev->rng_wait_time; + if (nsec <= 0) + return 1; + if (!wait) + return 0; + ndelay(nsec); + return 1; +} + +static int hifn_rng_data_read(struct hwrng *rng, u32 *data) +{ + struct hifn_device *dev = (struct hifn_device *)rng->priv; + + *data = hifn_read_1(dev, HIFN_1_RNG_DATA); + dev->rngtime = ktime_get(); + return 4; +} + +static int hifn_register_rng(struct hifn_device *dev) +{ + /* + * We must wait at least 256 Pk_clk cycles between two reads of the rng. + */ + dev->rng_wait_time = DIV_ROUND_UP_ULL(NSEC_PER_SEC, + dev->pk_clk_freq) * 256; + + dev->rng.name = dev->name; + dev->rng.data_present = hifn_rng_data_present; + dev->rng.data_read = hifn_rng_data_read; + dev->rng.priv = (unsigned long)dev; + + return hwrng_register(&dev->rng); +} + +static void hifn_unregister_rng(struct hifn_device *dev) +{ + hwrng_unregister(&dev->rng); +} +#else +#define hifn_register_rng(dev) 0 +#define hifn_unregister_rng(dev) +#endif + +static int hifn_init_pubrng(struct hifn_device *dev) +{ + int i; + + hifn_write_1(dev, HIFN_1_PUB_RESET, hifn_read_1(dev, HIFN_1_PUB_RESET) | + HIFN_PUBRST_RESET); + + for (i = 100; i > 0; --i) { + mdelay(1); + + if ((hifn_read_1(dev, HIFN_1_PUB_RESET) & HIFN_PUBRST_RESET) == 0) + break; + } + + if (!i) { + dev_err(&dev->pdev->dev, "Failed to initialise public key engine.\n"); + } else { + hifn_write_1(dev, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE); + dev->dmareg |= HIFN_DMAIER_PUBDONE; + hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); + + dev_dbg(&dev->pdev->dev, "Public key engine has been successfully initialised.\n"); + } + + /* Enable RNG engine. */ + + hifn_write_1(dev, HIFN_1_RNG_CONFIG, + hifn_read_1(dev, HIFN_1_RNG_CONFIG) | HIFN_RNGCFG_ENA); + dev_dbg(&dev->pdev->dev, "RNG engine has been successfully initialised.\n"); + +#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG + /* First value must be discarded */ + hifn_read_1(dev, HIFN_1_RNG_DATA); + dev->rngtime = ktime_get(); +#endif + return 0; +} + +static int hifn_enable_crypto(struct hifn_device *dev) +{ + u32 dmacfg, addr; + char *offtbl = NULL; + int i; + + for (i = 0; i < ARRAY_SIZE(pci2id); i++) { + if (pci2id[i].pci_vendor == dev->pdev->vendor && + pci2id[i].pci_prod == dev->pdev->device) { + offtbl = pci2id[i].card_id; + break; + } + } + + if (!offtbl) { + dev_err(&dev->pdev->dev, "Unknown card!\n"); + return -ENODEV; + } + + dmacfg = hifn_read_1(dev, HIFN_1_DMA_CNFG); + + hifn_write_1(dev, HIFN_1_DMA_CNFG, + HIFN_DMACNFG_UNLOCK | HIFN_DMACNFG_MSTRESET | + HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); + mdelay(1); + addr = hifn_read_1(dev, HIFN_1_UNLOCK_SECRET1); + mdelay(1); + hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, 0); + mdelay(1); + + for (i = 0; i < 12; ++i) { + addr = hifn_next_signature(addr, offtbl[i] + 0x101); + hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, addr); + + mdelay(1); + } + hifn_write_1(dev, HIFN_1_DMA_CNFG, dmacfg); + + dev_dbg(&dev->pdev->dev, "%s %s.\n", dev->name, pci_name(dev->pdev)); + + return 0; +} + +static void hifn_init_dma(struct hifn_device *dev) +{ + struct hifn_dma *dma = dev->desc_virt; + u32 dptr = dev->desc_dma; + int i; + + for (i = 0; i < HIFN_D_CMD_RSIZE; ++i) + dma->cmdr[i].p = __cpu_to_le32(dptr + + offsetof(struct hifn_dma, command_bufs[i][0])); + for (i = 0; i < HIFN_D_RES_RSIZE; ++i) + dma->resr[i].p = __cpu_to_le32(dptr + + offsetof(struct hifn_dma, result_bufs[i][0])); + + /* Setup LAST descriptors. */ + dma->cmdr[HIFN_D_CMD_RSIZE].p = __cpu_to_le32(dptr + + offsetof(struct hifn_dma, cmdr[0])); + dma->srcr[HIFN_D_SRC_RSIZE].p = __cpu_to_le32(dptr + + offsetof(struct hifn_dma, srcr[0])); + dma->dstr[HIFN_D_DST_RSIZE].p = __cpu_to_le32(dptr + + offsetof(struct hifn_dma, dstr[0])); + dma->resr[HIFN_D_RES_RSIZE].p = __cpu_to_le32(dptr + + offsetof(struct hifn_dma, resr[0])); + + dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0; + dma->cmdi = dma->srci = dma->dsti = dma->resi = 0; + dma->cmdk = dma->srck = dma->dstk = dma->resk = 0; +} + +/* + * Initialize the PLL. We need to know the frequency of the reference clock + * to calculate the optimal multiplier. For PCI we assume 66MHz, since that + * allows us to operate without the risk of overclocking the chip. If it + * actually uses 33MHz, the chip will operate at half the speed, this can be + * overridden by specifying the frequency as module parameter (pci33). + * + * Unfortunately the PCI clock is not very suitable since the HIFN needs a + * stable clock and the PCI clock frequency may vary, so the default is the + * external clock. There is no way to find out its frequency, we default to + * 66MHz since according to Mike Ham of HiFn, almost every board in existence + * has an external crystal populated at 66MHz. + */ +static void hifn_init_pll(struct hifn_device *dev) +{ + unsigned int freq, m; + u32 pllcfg; + + pllcfg = HIFN_1_PLL | HIFN_PLL_RESERVED_1; + + if (strncmp(hifn_pll_ref, "ext", 3) == 0) + pllcfg |= HIFN_PLL_REF_CLK_PLL; + else + pllcfg |= HIFN_PLL_REF_CLK_HBI; + + if (hifn_pll_ref[3] == '\0' || + kstrtouint(hifn_pll_ref + 3, 10, &freq)) { + freq = 66; + dev_info(&dev->pdev->dev, "assuming %u MHz clock speed, override with hifn_pll_ref=%.3s\n", + freq, hifn_pll_ref); + } + + m = HIFN_PLL_FCK_MAX / freq; + + pllcfg |= (m / 2 - 1) << HIFN_PLL_ND_SHIFT; + if (m <= 8) + pllcfg |= HIFN_PLL_IS_1_8; + else + pllcfg |= HIFN_PLL_IS_9_12; + + /* Select clock source and enable clock bypass */ + hifn_write_1(dev, HIFN_1_PLL, pllcfg | + HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI | HIFN_PLL_BP); + + /* Let the chip lock to the input clock */ + mdelay(10); + + /* Disable clock bypass */ + hifn_write_1(dev, HIFN_1_PLL, pllcfg | + HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI); + + /* Switch the engines to the PLL */ + hifn_write_1(dev, HIFN_1_PLL, pllcfg | + HIFN_PLL_PK_CLK_PLL | HIFN_PLL_PE_CLK_PLL); + + /* + * The Fpk_clk runs at half the total speed. Its frequency is needed to + * calculate the minimum time between two reads of the rng. Since 33MHz + * is actually 33.333... we overestimate the frequency here, resulting + * in slightly larger intervals. + */ + dev->pk_clk_freq = 1000000 * (freq + 1) * m / 2; +} + +static void hifn_init_registers(struct hifn_device *dev) +{ + u32 dptr = dev->desc_dma; + + /* Initialization magic... */ + hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA); + hifn_write_0(dev, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD); + hifn_write_0(dev, HIFN_0_PUIER, HIFN_PUIER_DSTOVER); + + /* write all 4 ring address registers */ + hifn_write_1(dev, HIFN_1_DMA_CRAR, dptr + + offsetof(struct hifn_dma, cmdr[0])); + hifn_write_1(dev, HIFN_1_DMA_SRAR, dptr + + offsetof(struct hifn_dma, srcr[0])); + hifn_write_1(dev, HIFN_1_DMA_DRAR, dptr + + offsetof(struct hifn_dma, dstr[0])); + hifn_write_1(dev, HIFN_1_DMA_RRAR, dptr + + offsetof(struct hifn_dma, resr[0])); + + mdelay(2); +#if 0 + hifn_write_1(dev, HIFN_1_DMA_CSR, + HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS | + HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS | + HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST | + HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER | + HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST | + HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER | + HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST | + HIFN_DMACSR_S_WAIT | + HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST | + HIFN_DMACSR_C_WAIT | + HIFN_DMACSR_ENGINE | + HIFN_DMACSR_PUBDONE); +#else + hifn_write_1(dev, HIFN_1_DMA_CSR, + HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA | + HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA | + HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST | + HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER | + HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST | + HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER | + HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST | + HIFN_DMACSR_S_WAIT | + HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST | + HIFN_DMACSR_C_WAIT | + HIFN_DMACSR_ENGINE | + HIFN_DMACSR_PUBDONE); +#endif + hifn_read_1(dev, HIFN_1_DMA_CSR); + + dev->dmareg |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT | + HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER | + HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT | + HIFN_DMAIER_ENGINE; + dev->dmareg &= ~HIFN_DMAIER_C_WAIT; + + hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); + hifn_read_1(dev, HIFN_1_DMA_IER); +#if 0 + hifn_write_0(dev, HIFN_0_PUCNFG, HIFN_PUCNFG_ENCCNFG | + HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES | + HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 | + HIFN_PUCNFG_DRAM); +#else + hifn_write_0(dev, HIFN_0_PUCNFG, 0x10342); +#endif + hifn_init_pll(dev); + + hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER); + hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | + HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST | + ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) | + ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL)); +} + +static int hifn_setup_base_command(struct hifn_device *dev, u8 *buf, + unsigned dlen, unsigned slen, u16 mask, u8 snum) +{ + struct hifn_base_command *base_cmd; + u8 *buf_pos = buf; + + base_cmd = (struct hifn_base_command *)buf_pos; + base_cmd->masks = __cpu_to_le16(mask); + base_cmd->total_source_count = + __cpu_to_le16(slen & HIFN_BASE_CMD_LENMASK_LO); + base_cmd->total_dest_count = + __cpu_to_le16(dlen & HIFN_BASE_CMD_LENMASK_LO); + + dlen >>= 16; + slen >>= 16; + base_cmd->session_num = __cpu_to_le16(snum | + ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) | + ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M)); + + return sizeof(struct hifn_base_command); +} + +static int hifn_setup_crypto_command(struct hifn_device *dev, + u8 *buf, unsigned dlen, unsigned slen, + u8 *key, int keylen, u8 *iv, int ivsize, u16 mode) +{ + struct hifn_dma *dma = dev->desc_virt; + struct hifn_crypt_command *cry_cmd; + u8 *buf_pos = buf; + u16 cmd_len; + + cry_cmd = (struct hifn_crypt_command *)buf_pos; + + cry_cmd->source_count = __cpu_to_le16(dlen & 0xffff); + dlen >>= 16; + cry_cmd->masks = __cpu_to_le16(mode | + ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) & + HIFN_CRYPT_CMD_SRCLEN_M)); + cry_cmd->header_skip = 0; + cry_cmd->reserved = 0; + + buf_pos += sizeof(struct hifn_crypt_command); + + dma->cmdu++; + if (dma->cmdu > 1) { + dev->dmareg |= HIFN_DMAIER_C_WAIT; + hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); + } + + if (keylen) { + memcpy(buf_pos, key, keylen); + buf_pos += keylen; + } + if (ivsize) { + memcpy(buf_pos, iv, ivsize); + buf_pos += ivsize; + } + + cmd_len = buf_pos - buf; + + return cmd_len; +} + +static int hifn_setup_cmd_desc(struct hifn_device *dev, + struct hifn_context *ctx, struct hifn_request_context *rctx, + void *priv, unsigned int nbytes) +{ + struct hifn_dma *dma = dev->desc_virt; + int cmd_len, sa_idx; + u8 *buf, *buf_pos; + u16 mask; + + sa_idx = dma->cmdi; + buf_pos = buf = dma->command_bufs[dma->cmdi]; + + mask = 0; + switch (rctx->op) { + case ACRYPTO_OP_DECRYPT: + mask = HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE; + break; + case ACRYPTO_OP_ENCRYPT: + mask = HIFN_BASE_CMD_CRYPT; + break; + case ACRYPTO_OP_HMAC: + mask = HIFN_BASE_CMD_MAC; + break; + default: + goto err_out; + } + + buf_pos += hifn_setup_base_command(dev, buf_pos, nbytes, + nbytes, mask, dev->snum); + + if (rctx->op == ACRYPTO_OP_ENCRYPT || rctx->op == ACRYPTO_OP_DECRYPT) { + u16 md = 0; + + if (ctx->keysize) + md |= HIFN_CRYPT_CMD_NEW_KEY; + if (rctx->iv && rctx->mode != ACRYPTO_MODE_ECB) + md |= HIFN_CRYPT_CMD_NEW_IV; + + switch (rctx->mode) { + case ACRYPTO_MODE_ECB: + md |= HIFN_CRYPT_CMD_MODE_ECB; + break; + case ACRYPTO_MODE_CBC: + md |= HIFN_CRYPT_CMD_MODE_CBC; + break; + case ACRYPTO_MODE_CFB: + md |= HIFN_CRYPT_CMD_MODE_CFB; + break; + case ACRYPTO_MODE_OFB: + md |= HIFN_CRYPT_CMD_MODE_OFB; + break; + default: + goto err_out; + } + + switch (rctx->type) { + case ACRYPTO_TYPE_AES_128: + if (ctx->keysize != 16) + goto err_out; + md |= HIFN_CRYPT_CMD_KSZ_128 | + HIFN_CRYPT_CMD_ALG_AES; + break; + case ACRYPTO_TYPE_AES_192: + if (ctx->keysize != 24) + goto err_out; + md |= HIFN_CRYPT_CMD_KSZ_192 | + HIFN_CRYPT_CMD_ALG_AES; + break; + case ACRYPTO_TYPE_AES_256: + if (ctx->keysize != 32) + goto err_out; + md |= HIFN_CRYPT_CMD_KSZ_256 | + HIFN_CRYPT_CMD_ALG_AES; + break; + case ACRYPTO_TYPE_3DES: + if (ctx->keysize != 24) + goto err_out; + md |= HIFN_CRYPT_CMD_ALG_3DES; + break; + case ACRYPTO_TYPE_DES: + if (ctx->keysize != 8) + goto err_out; + md |= HIFN_CRYPT_CMD_ALG_DES; + break; + default: + goto err_out; + } + + buf_pos += hifn_setup_crypto_command(dev, buf_pos, + nbytes, nbytes, ctx->key, ctx->keysize, + rctx->iv, rctx->ivsize, md); + } + + dev->sa[sa_idx] = priv; + dev->started++; + + cmd_len = buf_pos - buf; + dma->cmdr[dma->cmdi].l = __cpu_to_le32(cmd_len | HIFN_D_VALID | + HIFN_D_LAST | HIFN_D_MASKDONEIRQ); + + if (++dma->cmdi == HIFN_D_CMD_RSIZE) { + dma->cmdr[dma->cmdi].l = __cpu_to_le32( + HIFN_D_VALID | HIFN_D_LAST | + HIFN_D_MASKDONEIRQ | HIFN_D_JUMP); + dma->cmdi = 0; + } else { + dma->cmdr[dma->cmdi - 1].l |= __cpu_to_le32(HIFN_D_VALID); + } + + if (!(dev->flags & HIFN_FLAG_CMD_BUSY)) { + hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA); + dev->flags |= HIFN_FLAG_CMD_BUSY; + } + return 0; + +err_out: + return -EINVAL; +} + +static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page, + unsigned int offset, unsigned int size, int last) +{ + struct hifn_dma *dma = dev->desc_virt; + int idx; + dma_addr_t addr; + + addr = dma_map_page(&dev->pdev->dev, page, offset, size, + DMA_TO_DEVICE); + + idx = dma->srci; + + dma->srcr[idx].p = __cpu_to_le32(addr); + dma->srcr[idx].l = __cpu_to_le32(size | HIFN_D_VALID | + HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0)); + + if (++idx == HIFN_D_SRC_RSIZE) { + dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID | + HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | + (last ? HIFN_D_LAST : 0)); + idx = 0; + } + + dma->srci = idx; + dma->srcu++; + + if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) { + hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA); + dev->flags |= HIFN_FLAG_SRC_BUSY; + } + + return size; +} + +static void hifn_setup_res_desc(struct hifn_device *dev) +{ + struct hifn_dma *dma = dev->desc_virt; + + dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT | + HIFN_D_VALID | HIFN_D_LAST); + /* + * dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID | + * HIFN_D_LAST); + */ + + if (++dma->resi == HIFN_D_RES_RSIZE) { + dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID | + HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | HIFN_D_LAST); + dma->resi = 0; + } + + dma->resu++; + + if (!(dev->flags & HIFN_FLAG_RES_BUSY)) { + hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA); + dev->flags |= HIFN_FLAG_RES_BUSY; + } +} + +static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page, + unsigned offset, unsigned size, int last) +{ + struct hifn_dma *dma = dev->desc_virt; + int idx; + dma_addr_t addr; + + addr = dma_map_page(&dev->pdev->dev, page, offset, size, + DMA_FROM_DEVICE); + + idx = dma->dsti; + dma->dstr[idx].p = __cpu_to_le32(addr); + dma->dstr[idx].l = __cpu_to_le32(size | HIFN_D_VALID | + HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0)); + + if (++idx == HIFN_D_DST_RSIZE) { + dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID | + HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | + (last ? HIFN_D_LAST : 0)); + idx = 0; + } + dma->dsti = idx; + dma->dstu++; + + if (!(dev->flags & HIFN_FLAG_DST_BUSY)) { + hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA); + dev->flags |= HIFN_FLAG_DST_BUSY; + } +} + +static int hifn_setup_dma(struct hifn_device *dev, + struct hifn_context *ctx, struct hifn_request_context *rctx, + struct scatterlist *src, struct scatterlist *dst, + unsigned int nbytes, void *priv) +{ + struct scatterlist *t; + struct page *spage, *dpage; + unsigned int soff, doff; + unsigned int n, len; + + n = nbytes; + while (n) { + spage = sg_page(src); + soff = src->offset; + len = min(src->length, n); + + hifn_setup_src_desc(dev, spage, soff, len, n - len == 0); + + src++; + n -= len; + } + + t = &rctx->walk.cache[0]; + n = nbytes; + while (n) { + if (t->length && rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { + BUG_ON(!sg_page(t)); + dpage = sg_page(t); + doff = 0; + len = t->length; + } else { + BUG_ON(!sg_page(dst)); + dpage = sg_page(dst); + doff = dst->offset; + len = dst->length; + } + len = min(len, n); + + hifn_setup_dst_desc(dev, dpage, doff, len, n - len == 0); + + dst++; + t++; + n -= len; + } + + hifn_setup_cmd_desc(dev, ctx, rctx, priv, nbytes); + hifn_setup_res_desc(dev); + return 0; +} + +static int hifn_cipher_walk_init(struct hifn_cipher_walk *w, + int num, gfp_t gfp_flags) +{ + int i; + + num = min(ASYNC_SCATTERLIST_CACHE, num); + sg_init_table(w->cache, num); + + w->num = 0; + for (i = 0; i < num; ++i) { + struct page *page = alloc_page(gfp_flags); + struct scatterlist *s; + + if (!page) + break; + + s = &w->cache[i]; + + sg_set_page(s, page, PAGE_SIZE, 0); + w->num++; + } + + return i; +} + +static void hifn_cipher_walk_exit(struct hifn_cipher_walk *w) +{ + int i; + + for (i = 0; i < w->num; ++i) { + struct scatterlist *s = &w->cache[i]; + + __free_page(sg_page(s)); + + s->length = 0; + } + + w->num = 0; +} + +static int skcipher_add(unsigned int *drestp, struct scatterlist *dst, + unsigned int size, unsigned int *nbytesp) +{ + unsigned int copy, drest = *drestp, nbytes = *nbytesp; + int idx = 0; + + if (drest < size || size > nbytes) + return -EINVAL; + + while (size) { + copy = min3(drest, size, dst->length); + + size -= copy; + drest -= copy; + nbytes -= copy; + + pr_debug("%s: copy: %u, size: %u, drest: %u, nbytes: %u.\n", + __func__, copy, size, drest, nbytes); + + dst++; + idx++; + } + + *nbytesp = nbytes; + *drestp = drest; + + return idx; +} + +static int hifn_cipher_walk(struct skcipher_request *req, + struct hifn_cipher_walk *w) +{ + struct scatterlist *dst, *t; + unsigned int nbytes = req->cryptlen, offset, copy, diff; + int idx, tidx, err; + + tidx = idx = 0; + offset = 0; + while (nbytes) { + if (idx >= w->num && (w->flags & ASYNC_FLAGS_MISALIGNED)) + return -EINVAL; + + dst = &req->dst[idx]; + + pr_debug("\n%s: dlen: %u, doff: %u, offset: %u, nbytes: %u.\n", + __func__, dst->length, dst->offset, offset, nbytes); + + if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) || + !IS_ALIGNED(dst->length, HIFN_D_DST_DALIGN) || + offset) { + unsigned slen = min(dst->length - offset, nbytes); + unsigned dlen = PAGE_SIZE; + + t = &w->cache[idx]; + + err = skcipher_add(&dlen, dst, slen, &nbytes); + if (err < 0) + return err; + + idx += err; + + copy = slen & ~(HIFN_D_DST_DALIGN - 1); + diff = slen & (HIFN_D_DST_DALIGN - 1); + + if (dlen < nbytes) { + /* + * Destination page does not have enough space + * to put there additional blocksized chunk, + * so we mark that page as containing only + * blocksize aligned chunks: + * t->length = (slen & ~(HIFN_D_DST_DALIGN - 1)); + * and increase number of bytes to be processed + * in next chunk: + * nbytes += diff; + */ + nbytes += diff; + + /* + * Temporary of course... + * Kick author if you will catch this one. + */ + pr_err("%s: dlen: %u, nbytes: %u, slen: %u, offset: %u.\n", + __func__, dlen, nbytes, slen, offset); + pr_err("%s: please contact author to fix this " + "issue, generally you should not catch " + "this path under any condition but who " + "knows how did you use crypto code.\n" + "Thank you.\n", __func__); + BUG(); + } else { + copy += diff + nbytes; + + dst = &req->dst[idx]; + + err = skcipher_add(&dlen, dst, nbytes, &nbytes); + if (err < 0) + return err; + + idx += err; + } + + t->length = copy; + t->offset = offset; + } else { + nbytes -= min(dst->length, nbytes); + idx++; + } + + tidx++; + } + + return tidx; +} + +static int hifn_setup_session(struct skcipher_request *req) +{ + struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); + struct hifn_request_context *rctx = skcipher_request_ctx(req); + struct hifn_device *dev = ctx->dev; + unsigned long dlen, flags; + unsigned int nbytes = req->cryptlen, idx = 0; + int err = -EINVAL, sg_num; + struct scatterlist *dst; + + if (rctx->iv && !rctx->ivsize && rctx->mode != ACRYPTO_MODE_ECB) + goto err_out_exit; + + rctx->walk.flags = 0; + + while (nbytes) { + dst = &req->dst[idx]; + dlen = min(dst->length, nbytes); + + if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) || + !IS_ALIGNED(dlen, HIFN_D_DST_DALIGN)) + rctx->walk.flags |= ASYNC_FLAGS_MISALIGNED; + + nbytes -= dlen; + idx++; + } + + if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { + err = hifn_cipher_walk_init(&rctx->walk, idx, GFP_ATOMIC); + if (err < 0) + return err; + } + + sg_num = hifn_cipher_walk(req, &rctx->walk); + if (sg_num < 0) { + err = sg_num; + goto err_out_exit; + } + + spin_lock_irqsave(&dev->lock, flags); + if (dev->started + sg_num > HIFN_QUEUE_LENGTH) { + err = -EAGAIN; + goto err_out; + } + + err = hifn_setup_dma(dev, ctx, rctx, req->src, req->dst, req->cryptlen, req); + if (err) + goto err_out; + + dev->snum++; + + dev->active = HIFN_DEFAULT_ACTIVE_NUM; + spin_unlock_irqrestore(&dev->lock, flags); + + return 0; + +err_out: + spin_unlock_irqrestore(&dev->lock, flags); +err_out_exit: + if (err) { + dev_info(&dev->pdev->dev, "iv: %p [%d], key: %p [%d], mode: %u, op: %u, " + "type: %u, err: %d.\n", + rctx->iv, rctx->ivsize, + ctx->key, ctx->keysize, + rctx->mode, rctx->op, rctx->type, err); + } + + return err; +} + +static int hifn_start_device(struct hifn_device *dev) +{ + int err; + + dev->started = dev->active = 0; + hifn_reset_dma(dev, 1); + + err = hifn_enable_crypto(dev); + if (err) + return err; + + hifn_reset_puc(dev); + + hifn_init_dma(dev); + + hifn_init_registers(dev); + + hifn_init_pubrng(dev); + + return 0; +} + +static int skcipher_get(void *saddr, unsigned int *srestp, unsigned int offset, + struct scatterlist *dst, unsigned int size, unsigned int *nbytesp) +{ + unsigned int srest = *srestp, nbytes = *nbytesp, copy; + void *daddr; + int idx = 0; + + if (srest < size || size > nbytes) + return -EINVAL; + + while (size) { + copy = min3(srest, dst->length, size); + + daddr = kmap_atomic(sg_page(dst)); + memcpy(daddr + dst->offset + offset, saddr, copy); + kunmap_atomic(daddr); + + nbytes -= copy; + size -= copy; + srest -= copy; + saddr += copy; + offset = 0; + + pr_debug("%s: copy: %u, size: %u, srest: %u, nbytes: %u.\n", + __func__, copy, size, srest, nbytes); + + dst++; + idx++; + } + + *nbytesp = nbytes; + *srestp = srest; + + return idx; +} + +static inline void hifn_complete_sa(struct hifn_device *dev, int i) +{ + unsigned long flags; + + spin_lock_irqsave(&dev->lock, flags); + dev->sa[i] = NULL; + dev->started--; + if (dev->started < 0) + dev_info(&dev->pdev->dev, "%s: started: %d.\n", __func__, + dev->started); + spin_unlock_irqrestore(&dev->lock, flags); + BUG_ON(dev->started < 0); +} + +static void hifn_process_ready(struct skcipher_request *req, int error) +{ + struct hifn_request_context *rctx = skcipher_request_ctx(req); + + if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { + unsigned int nbytes = req->cryptlen; + int idx = 0, err; + struct scatterlist *dst, *t; + void *saddr; + + while (nbytes) { + t = &rctx->walk.cache[idx]; + dst = &req->dst[idx]; + + pr_debug("\n%s: sg_page(t): %p, t->length: %u, " + "sg_page(dst): %p, dst->length: %u, " + "nbytes: %u.\n", + __func__, sg_page(t), t->length, + sg_page(dst), dst->length, nbytes); + + if (!t->length) { + nbytes -= min(dst->length, nbytes); + idx++; + continue; + } + + saddr = kmap_atomic(sg_page(t)); + + err = skcipher_get(saddr, &t->length, t->offset, + dst, nbytes, &nbytes); + if (err < 0) { + kunmap_atomic(saddr); + break; + } + + idx += err; + kunmap_atomic(saddr); + } + + hifn_cipher_walk_exit(&rctx->walk); + } + + skcipher_request_complete(req, error); +} + +static void hifn_clear_rings(struct hifn_device *dev, int error) +{ + struct hifn_dma *dma = dev->desc_virt; + int i, u; + + dev_dbg(&dev->pdev->dev, "ring cleanup 1: i: %d.%d.%d.%d, u: %d.%d.%d.%d, " + "k: %d.%d.%d.%d.\n", + dma->cmdi, dma->srci, dma->dsti, dma->resi, + dma->cmdu, dma->srcu, dma->dstu, dma->resu, + dma->cmdk, dma->srck, dma->dstk, dma->resk); + + i = dma->resk; u = dma->resu; + while (u != 0) { + if (dma->resr[i].l & __cpu_to_le32(HIFN_D_VALID)) + break; + + if (dev->sa[i]) { + dev->success++; + dev->reset = 0; + hifn_process_ready(dev->sa[i], error); + hifn_complete_sa(dev, i); + } + + if (++i == HIFN_D_RES_RSIZE) + i = 0; + u--; + } + dma->resk = i; dma->resu = u; + + i = dma->srck; u = dma->srcu; + while (u != 0) { + if (dma->srcr[i].l & __cpu_to_le32(HIFN_D_VALID)) + break; + if (++i == HIFN_D_SRC_RSIZE) + i = 0; + u--; + } + dma->srck = i; dma->srcu = u; + + i = dma->cmdk; u = dma->cmdu; + while (u != 0) { + if (dma->cmdr[i].l & __cpu_to_le32(HIFN_D_VALID)) + break; + if (++i == HIFN_D_CMD_RSIZE) + i = 0; + u--; + } + dma->cmdk = i; dma->cmdu = u; + + i = dma->dstk; u = dma->dstu; + while (u != 0) { + if (dma->dstr[i].l & __cpu_to_le32(HIFN_D_VALID)) + break; + if (++i == HIFN_D_DST_RSIZE) + i = 0; + u--; + } + dma->dstk = i; dma->dstu = u; + + dev_dbg(&dev->pdev->dev, "ring cleanup 2: i: %d.%d.%d.%d, u: %d.%d.%d.%d, " + "k: %d.%d.%d.%d.\n", + dma->cmdi, dma->srci, dma->dsti, dma->resi, + dma->cmdu, dma->srcu, dma->dstu, dma->resu, + dma->cmdk, dma->srck, dma->dstk, dma->resk); +} + +static void hifn_work(struct work_struct *work) +{ + struct delayed_work *dw = to_delayed_work(work); + struct hifn_device *dev = container_of(dw, struct hifn_device, work); + unsigned long flags; + int reset = 0; + u32 r = 0; + + spin_lock_irqsave(&dev->lock, flags); + if (dev->active == 0) { + struct hifn_dma *dma = dev->desc_virt; + + if (dma->cmdu == 0 && (dev->flags & HIFN_FLAG_CMD_BUSY)) { + dev->flags &= ~HIFN_FLAG_CMD_BUSY; + r |= HIFN_DMACSR_C_CTRL_DIS; + } + if (dma->srcu == 0 && (dev->flags & HIFN_FLAG_SRC_BUSY)) { + dev->flags &= ~HIFN_FLAG_SRC_BUSY; + r |= HIFN_DMACSR_S_CTRL_DIS; + } + if (dma->dstu == 0 && (dev->flags & HIFN_FLAG_DST_BUSY)) { + dev->flags &= ~HIFN_FLAG_DST_BUSY; + r |= HIFN_DMACSR_D_CTRL_DIS; + } + if (dma->resu == 0 && (dev->flags & HIFN_FLAG_RES_BUSY)) { + dev->flags &= ~HIFN_FLAG_RES_BUSY; + r |= HIFN_DMACSR_R_CTRL_DIS; + } + if (r) + hifn_write_1(dev, HIFN_1_DMA_CSR, r); + } else + dev->active--; + + if ((dev->prev_success == dev->success) && dev->started) + reset = 1; + dev->prev_success = dev->success; + spin_unlock_irqrestore(&dev->lock, flags); + + if (reset) { + if (++dev->reset >= 5) { + int i; + struct hifn_dma *dma = dev->desc_virt; + + dev_info(&dev->pdev->dev, + "r: %08x, active: %d, started: %d, " + "success: %lu: qlen: %u/%u, reset: %d.\n", + r, dev->active, dev->started, + dev->success, dev->queue.qlen, dev->queue.max_qlen, + reset); + + dev_info(&dev->pdev->dev, "%s: res: ", __func__); + for (i = 0; i < HIFN_D_RES_RSIZE; ++i) { + pr_info("%x.%p ", dma->resr[i].l, dev->sa[i]); + if (dev->sa[i]) { + hifn_process_ready(dev->sa[i], -ENODEV); + hifn_complete_sa(dev, i); + } + } + pr_info("\n"); + + hifn_reset_dma(dev, 1); + hifn_stop_device(dev); + hifn_start_device(dev); + dev->reset = 0; + } + + tasklet_schedule(&dev->tasklet); + } + + schedule_delayed_work(&dev->work, HZ); +} + +static irqreturn_t hifn_interrupt(int irq, void *data) +{ + struct hifn_device *dev = data; + struct hifn_dma *dma = dev->desc_virt; + u32 dmacsr, restart; + + dmacsr = hifn_read_1(dev, HIFN_1_DMA_CSR); + + dev_dbg(&dev->pdev->dev, "1 dmacsr: %08x, dmareg: %08x, res: %08x [%d], " + "i: %d.%d.%d.%d, u: %d.%d.%d.%d.\n", + dmacsr, dev->dmareg, dmacsr & dev->dmareg, dma->cmdi, + dma->cmdi, dma->srci, dma->dsti, dma->resi, + dma->cmdu, dma->srcu, dma->dstu, dma->resu); + + if ((dmacsr & dev->dmareg) == 0) + return IRQ_NONE; + + hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & dev->dmareg); + + if (dmacsr & HIFN_DMACSR_ENGINE) + hifn_write_0(dev, HIFN_0_PUISR, hifn_read_0(dev, HIFN_0_PUISR)); + if (dmacsr & HIFN_DMACSR_PUBDONE) + hifn_write_1(dev, HIFN_1_PUB_STATUS, + hifn_read_1(dev, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE); + + restart = dmacsr & (HIFN_DMACSR_R_OVER | HIFN_DMACSR_D_OVER); + if (restart) { + u32 puisr = hifn_read_0(dev, HIFN_0_PUISR); + + dev_warn(&dev->pdev->dev, "overflow: r: %d, d: %d, puisr: %08x, d: %u.\n", + !!(dmacsr & HIFN_DMACSR_R_OVER), + !!(dmacsr & HIFN_DMACSR_D_OVER), + puisr, !!(puisr & HIFN_PUISR_DSTOVER)); + if (!!(puisr & HIFN_PUISR_DSTOVER)) + hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER); + hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & (HIFN_DMACSR_R_OVER | + HIFN_DMACSR_D_OVER)); + } + + restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT | + HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT); + if (restart) { + dev_warn(&dev->pdev->dev, "abort: c: %d, s: %d, d: %d, r: %d.\n", + !!(dmacsr & HIFN_DMACSR_C_ABORT), + !!(dmacsr & HIFN_DMACSR_S_ABORT), + !!(dmacsr & HIFN_DMACSR_D_ABORT), + !!(dmacsr & HIFN_DMACSR_R_ABORT)); + hifn_reset_dma(dev, 1); + hifn_init_dma(dev); + hifn_init_registers(dev); + } + + if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->cmdu == 0)) { + dev_dbg(&dev->pdev->dev, "wait on command.\n"); + dev->dmareg &= ~(HIFN_DMAIER_C_WAIT); + hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); + } + + tasklet_schedule(&dev->tasklet); + + return IRQ_HANDLED; +} + +static void hifn_flush(struct hifn_device *dev) +{ + unsigned long flags; + struct crypto_async_request *async_req; + struct skcipher_request *req; + struct hifn_dma *dma = dev->desc_virt; + int i; + + for (i = 0; i < HIFN_D_RES_RSIZE; ++i) { + struct hifn_desc *d = &dma->resr[i]; + + if (dev->sa[i]) { + hifn_process_ready(dev->sa[i], + (d->l & __cpu_to_le32(HIFN_D_VALID)) ? -ENODEV : 0); + hifn_complete_sa(dev, i); + } + } + + spin_lock_irqsave(&dev->lock, flags); + while ((async_req = crypto_dequeue_request(&dev->queue))) { + req = skcipher_request_cast(async_req); + spin_unlock_irqrestore(&dev->lock, flags); + + hifn_process_ready(req, -ENODEV); + + spin_lock_irqsave(&dev->lock, flags); + } + spin_unlock_irqrestore(&dev->lock, flags); +} + +static int hifn_setkey(struct crypto_skcipher *cipher, const u8 *key, + unsigned int len) +{ + struct hifn_context *ctx = crypto_skcipher_ctx(cipher); + struct hifn_device *dev = ctx->dev; + int err; + + err = verify_skcipher_des_key(cipher, key); + if (err) + return err; + + dev->flags &= ~HIFN_FLAG_OLD_KEY; + + memcpy(ctx->key, key, len); + ctx->keysize = len; + + return 0; +} + +static int hifn_des3_setkey(struct crypto_skcipher *cipher, const u8 *key, + unsigned int len) +{ + struct hifn_context *ctx = crypto_skcipher_ctx(cipher); + struct hifn_device *dev = ctx->dev; + int err; + + err = verify_skcipher_des3_key(cipher, key); + if (err) + return err; + + dev->flags &= ~HIFN_FLAG_OLD_KEY; + + memcpy(ctx->key, key, len); + ctx->keysize = len; + + return 0; +} + +static int hifn_handle_req(struct skcipher_request *req) +{ + struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); + struct hifn_device *dev = ctx->dev; + int err = -EAGAIN; + + if (dev->started + DIV_ROUND_UP(req->cryptlen, PAGE_SIZE) <= HIFN_QUEUE_LENGTH) + err = hifn_setup_session(req); + + if (err == -EAGAIN) { + unsigned long flags; + + spin_lock_irqsave(&dev->lock, flags); + err = crypto_enqueue_request(&dev->queue, &req->base); + spin_unlock_irqrestore(&dev->lock, flags); + } + + return err; +} + +static int hifn_setup_crypto_req(struct skcipher_request *req, u8 op, + u8 type, u8 mode) +{ + struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); + struct hifn_request_context *rctx = skcipher_request_ctx(req); + unsigned ivsize; + + ivsize = crypto_skcipher_ivsize(crypto_skcipher_reqtfm(req)); + + if (req->iv && mode != ACRYPTO_MODE_ECB) { + if (type == ACRYPTO_TYPE_AES_128) + ivsize = HIFN_AES_IV_LENGTH; + else if (type == ACRYPTO_TYPE_DES) + ivsize = HIFN_DES_KEY_LENGTH; + else if (type == ACRYPTO_TYPE_3DES) + ivsize = HIFN_3DES_KEY_LENGTH; + } + + if (ctx->keysize != 16 && type == ACRYPTO_TYPE_AES_128) { + if (ctx->keysize == 24) + type = ACRYPTO_TYPE_AES_192; + else if (ctx->keysize == 32) + type = ACRYPTO_TYPE_AES_256; + } + + rctx->op = op; + rctx->mode = mode; + rctx->type = type; + rctx->iv = req->iv; + rctx->ivsize = ivsize; + + /* + * HEAVY TODO: needs to kick Herbert XU to write documentation. + * HEAVY TODO: needs to kick Herbert XU to write documentation. + * HEAVY TODO: needs to kick Herbert XU to write documentation. + */ + + return hifn_handle_req(req); +} + +static int hifn_process_queue(struct hifn_device *dev) +{ + struct crypto_async_request *async_req, *backlog; + struct skcipher_request *req; + unsigned long flags; + int err = 0; + + while (dev->started < HIFN_QUEUE_LENGTH) { + spin_lock_irqsave(&dev->lock, flags); + backlog = crypto_get_backlog(&dev->queue); + async_req = crypto_dequeue_request(&dev->queue); + spin_unlock_irqrestore(&dev->lock, flags); + + if (!async_req) + break; + + if (backlog) + crypto_request_complete(backlog, -EINPROGRESS); + + req = skcipher_request_cast(async_req); + + err = hifn_handle_req(req); + if (err) + break; + } + + return err; +} + +static int hifn_setup_crypto(struct skcipher_request *req, u8 op, + u8 type, u8 mode) +{ + int err; + struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); + struct hifn_device *dev = ctx->dev; + + err = hifn_setup_crypto_req(req, op, type, mode); + if (err) + return err; + + if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen) + hifn_process_queue(dev); + + return -EINPROGRESS; +} + +/* + * AES ecryption functions. + */ +static inline int hifn_encrypt_aes_ecb(struct skcipher_request *req) +{ + return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, + ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB); +} +static inline int hifn_encrypt_aes_cbc(struct skcipher_request *req) +{ + return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, + ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC); +} + +/* + * AES decryption functions. + */ +static inline int hifn_decrypt_aes_ecb(struct skcipher_request *req) +{ + return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, + ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB); +} +static inline int hifn_decrypt_aes_cbc(struct skcipher_request *req) +{ + return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, + ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC); +} + +/* + * DES ecryption functions. + */ +static inline int hifn_encrypt_des_ecb(struct skcipher_request *req) +{ + return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, + ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB); +} +static inline int hifn_encrypt_des_cbc(struct skcipher_request *req) +{ + return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, + ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC); +} + +/* + * DES decryption functions. + */ +static inline int hifn_decrypt_des_ecb(struct skcipher_request *req) +{ + return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, + ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB); +} +static inline int hifn_decrypt_des_cbc(struct skcipher_request *req) +{ + return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, + ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC); +} + +/* + * 3DES ecryption functions. + */ +static inline int hifn_encrypt_3des_ecb(struct skcipher_request *req) +{ + return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, + ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB); +} +static inline int hifn_encrypt_3des_cbc(struct skcipher_request *req) +{ + return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, + ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC); +} + +/* 3DES decryption functions. */ +static inline int hifn_decrypt_3des_ecb(struct skcipher_request *req) +{ + return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, + ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB); +} +static inline int hifn_decrypt_3des_cbc(struct skcipher_request *req) +{ + return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, + ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC); +} + +struct hifn_alg_template { + char name[CRYPTO_MAX_ALG_NAME]; + char drv_name[CRYPTO_MAX_ALG_NAME]; + unsigned int bsize; + struct skcipher_alg skcipher; +}; + +static const struct hifn_alg_template hifn_alg_templates[] = { + /* + * 3DES ECB and CBC modes. + */ + { + .name = "cbc(des3_ede)", .drv_name = "cbc-3des", .bsize = 8, + .skcipher = { + .ivsize = HIFN_IV_LENGTH, + .min_keysize = HIFN_3DES_KEY_LENGTH, + .max_keysize = HIFN_3DES_KEY_LENGTH, + .setkey = hifn_des3_setkey, + .encrypt = hifn_encrypt_3des_cbc, + .decrypt = hifn_decrypt_3des_cbc, + }, + }, + { + .name = "ecb(des3_ede)", .drv_name = "ecb-3des", .bsize = 8, + .skcipher = { + .min_keysize = HIFN_3DES_KEY_LENGTH, + .max_keysize = HIFN_3DES_KEY_LENGTH, + .setkey = hifn_des3_setkey, + .encrypt = hifn_encrypt_3des_ecb, + .decrypt = hifn_decrypt_3des_ecb, + }, + }, + + /* + * DES ECB and CBC modes. + */ + { + .name = "cbc(des)", .drv_name = "cbc-des", .bsize = 8, + .skcipher = { + .ivsize = HIFN_IV_LENGTH, + .min_keysize = HIFN_DES_KEY_LENGTH, + .max_keysize = HIFN_DES_KEY_LENGTH, + .setkey = hifn_setkey, + .encrypt = hifn_encrypt_des_cbc, + .decrypt = hifn_decrypt_des_cbc, + }, + }, + { + .name = "ecb(des)", .drv_name = "ecb-des", .bsize = 8, + .skcipher = { + .min_keysize = HIFN_DES_KEY_LENGTH, + .max_keysize = HIFN_DES_KEY_LENGTH, + .setkey = hifn_setkey, + .encrypt = hifn_encrypt_des_ecb, + .decrypt = hifn_decrypt_des_ecb, + }, + }, + + /* + * AES ECB and CBC modes. + */ + { + .name = "ecb(aes)", .drv_name = "ecb-aes", .bsize = 16, + .skcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = hifn_setkey, + .encrypt = hifn_encrypt_aes_ecb, + .decrypt = hifn_decrypt_aes_ecb, + }, + }, + { + .name = "cbc(aes)", .drv_name = "cbc-aes", .bsize = 16, + .skcipher = { + .ivsize = HIFN_AES_IV_LENGTH, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = hifn_setkey, + .encrypt = hifn_encrypt_aes_cbc, + .decrypt = hifn_decrypt_aes_cbc, + }, + }, +}; + +static int hifn_init_tfm(struct crypto_skcipher *tfm) +{ + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct hifn_crypto_alg *ha = crypto_alg_to_hifn(alg); + struct hifn_context *ctx = crypto_skcipher_ctx(tfm); + + ctx->dev = ha->dev; + crypto_skcipher_set_reqsize(tfm, sizeof(struct hifn_request_context)); + + return 0; +} + +static int hifn_alg_alloc(struct hifn_device *dev, const struct hifn_alg_template *t) +{ + struct hifn_crypto_alg *alg; + int err; + + alg = kzalloc_obj(*alg); + if (!alg) + return -ENOMEM; + + alg->alg = t->skcipher; + alg->alg.init = hifn_init_tfm; + + err = -EINVAL; + if (strscpy(alg->alg.base.cra_name, t->name) < 0) + goto out_free_alg; + if (snprintf(alg->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, + "%s-%s", t->drv_name, dev->name) >= CRYPTO_MAX_ALG_NAME) + goto out_free_alg; + + alg->alg.base.cra_priority = 300; + alg->alg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC; + alg->alg.base.cra_blocksize = t->bsize; + alg->alg.base.cra_ctxsize = sizeof(struct hifn_context); + alg->alg.base.cra_alignmask = 0; + alg->alg.base.cra_module = THIS_MODULE; + + alg->dev = dev; + + list_add_tail(&alg->entry, &dev->alg_list); + + err = crypto_register_skcipher(&alg->alg); + if (err) { + list_del(&alg->entry); +out_free_alg: + kfree(alg); + } + + return err; +} + +static void hifn_unregister_alg(struct hifn_device *dev) +{ + struct hifn_crypto_alg *a, *n; + + list_for_each_entry_safe(a, n, &dev->alg_list, entry) { + list_del(&a->entry); + crypto_unregister_skcipher(&a->alg); + kfree(a); + } +} + +static int hifn_register_alg(struct hifn_device *dev) +{ + int i, err; + + for (i = 0; i < ARRAY_SIZE(hifn_alg_templates); ++i) { + err = hifn_alg_alloc(dev, &hifn_alg_templates[i]); + if (err) + goto err_out_exit; + } + + return 0; + +err_out_exit: + hifn_unregister_alg(dev); + return err; +} + +static void hifn_tasklet_callback(unsigned long data) +{ + struct hifn_device *dev = (struct hifn_device *)data; + + /* + * This is ok to call this without lock being held, + * althogh it modifies some parameters used in parallel, + * (like dev->success), but they are used in process + * context or update is atomic (like setting dev->sa[i] to NULL). + */ + hifn_clear_rings(dev, 0); + + if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen) + hifn_process_queue(dev); +} + +static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + int err, i; + struct hifn_device *dev; + char name[8]; + + err = pci_enable_device(pdev); + if (err) + return err; + pci_set_master(pdev); + + err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); + if (err) + goto err_out_disable_pci_device; + + snprintf(name, sizeof(name), "hifn%d", + atomic_inc_return(&hifn_dev_number) - 1); + + err = pci_request_regions(pdev, name); + if (err) + goto err_out_disable_pci_device; + + if (pci_resource_len(pdev, 0) < HIFN_BAR0_SIZE || + pci_resource_len(pdev, 1) < HIFN_BAR1_SIZE || + pci_resource_len(pdev, 2) < HIFN_BAR2_SIZE) { + dev_err(&pdev->dev, "Broken hardware - I/O regions are too small.\n"); + err = -ENODEV; + goto err_out_free_regions; + } + + dev = kzalloc(sizeof(struct hifn_device) + sizeof(struct crypto_alg), + GFP_KERNEL); + if (!dev) { + err = -ENOMEM; + goto err_out_free_regions; + } + + INIT_LIST_HEAD(&dev->alg_list); + + strscpy(dev->name, name); + spin_lock_init(&dev->lock); + + for (i = 0; i < 3; ++i) { + unsigned long addr, size; + + addr = pci_resource_start(pdev, i); + size = pci_resource_len(pdev, i); + + dev->bar[i] = ioremap(addr, size); + if (!dev->bar[i]) { + err = -ENOMEM; + goto err_out_unmap_bars; + } + } + + dev->desc_virt = dma_alloc_coherent(&pdev->dev, + sizeof(struct hifn_dma), + &dev->desc_dma, GFP_KERNEL); + if (!dev->desc_virt) { + dev_err(&pdev->dev, "Failed to allocate descriptor rings.\n"); + err = -ENOMEM; + goto err_out_unmap_bars; + } + + dev->pdev = pdev; + dev->irq = pdev->irq; + + for (i = 0; i < HIFN_D_RES_RSIZE; ++i) + dev->sa[i] = NULL; + + pci_set_drvdata(pdev, dev); + + tasklet_init(&dev->tasklet, hifn_tasklet_callback, (unsigned long)dev); + + crypto_init_queue(&dev->queue, 1); + + err = request_irq(dev->irq, hifn_interrupt, IRQF_SHARED, dev->name, dev); + if (err) { + dev_err(&pdev->dev, "Failed to request IRQ%d: err: %d.\n", + dev->irq, err); + dev->irq = 0; + goto err_out_free_desc; + } + + err = hifn_start_device(dev); + if (err) + goto err_out_free_irq; + + err = hifn_register_rng(dev); + if (err) + goto err_out_stop_device; + + err = hifn_register_alg(dev); + if (err) + goto err_out_unregister_rng; + + INIT_DELAYED_WORK(&dev->work, hifn_work); + schedule_delayed_work(&dev->work, HZ); + + dev_dbg(&pdev->dev, "HIFN crypto accelerator card at %s has been " + "successfully registered as %s.\n", + pci_name(pdev), dev->name); + + return 0; + +err_out_unregister_rng: + hifn_unregister_rng(dev); +err_out_stop_device: + hifn_reset_dma(dev, 1); + hifn_stop_device(dev); +err_out_free_irq: + free_irq(dev->irq, dev); + tasklet_kill(&dev->tasklet); +err_out_free_desc: + dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma), dev->desc_virt, + dev->desc_dma); + +err_out_unmap_bars: + for (i = 0; i < 3; ++i) + if (dev->bar[i]) + iounmap(dev->bar[i]); + kfree(dev); + +err_out_free_regions: + pci_release_regions(pdev); + +err_out_disable_pci_device: + pci_disable_device(pdev); + + return err; +} + +static void hifn_remove(struct pci_dev *pdev) +{ + int i; + struct hifn_device *dev; + + dev = pci_get_drvdata(pdev); + + if (dev) { + cancel_delayed_work_sync(&dev->work); + + hifn_unregister_rng(dev); + hifn_unregister_alg(dev); + hifn_reset_dma(dev, 1); + hifn_stop_device(dev); + + free_irq(dev->irq, dev); + tasklet_kill(&dev->tasklet); + + hifn_flush(dev); + + dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma), + dev->desc_virt, dev->desc_dma); + for (i = 0; i < 3; ++i) + if (dev->bar[i]) + iounmap(dev->bar[i]); + + kfree(dev); + } + + pci_release_regions(pdev); + pci_disable_device(pdev); +} + +static struct pci_device_id hifn_pci_tbl[] = { + { PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7955) }, + { PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7956) }, + { 0 } +}; +MODULE_DEVICE_TABLE(pci, hifn_pci_tbl); + +static struct pci_driver hifn_pci_driver = { + .name = "hifn795x", + .id_table = hifn_pci_tbl, + .probe = hifn_probe, + .remove = hifn_remove, +}; + +static int __init hifn_init(void) +{ + unsigned int freq; + int err; + + if (strncmp(hifn_pll_ref, "ext", 3) && + strncmp(hifn_pll_ref, "pci", 3)) { + pr_err("hifn795x: invalid hifn_pll_ref clock, must be pci or ext"); + return -EINVAL; + } + + /* + * For the 7955/7956 the reference clock frequency must be in the + * range of 20MHz-100MHz. For the 7954 the upper bound is 66.67MHz, + * but this chip is currently not supported. + */ + if (hifn_pll_ref[3] != '\0') { + freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10); + if (freq < 20 || freq > 100) { + pr_err("hifn795x: invalid hifn_pll_ref frequency, must" + "be in the range of 20-100"); + return -EINVAL; + } + } + + err = pci_register_driver(&hifn_pci_driver); + if (err < 0) { + pr_err("Failed to register PCI driver for %s device.\n", + hifn_pci_driver.name); + return -ENODEV; + } + + pr_info("Driver for HIFN 795x crypto accelerator chip " + "has been successfully registered.\n"); + + return 0; +} + +static void __exit hifn_fini(void) +{ + pci_unregister_driver(&hifn_pci_driver); + + pr_info("Driver for HIFN 795x crypto accelerator chip " + "has been successfully unregistered.\n"); +} + +module_init(hifn_init); +module_exit(hifn_fini); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Evgeniy Polyakov "); +MODULE_DESCRIPTION("Driver for HIFN 795x crypto accelerator chip."); diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig new file mode 100644 index 000000000..aeff08ccb --- /dev/null +++ b/drivers/crypto/hisilicon/Kconfig @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: GPL-2.0 + +config CRYPTO_DEV_HISI_SEC + tristate "Support for Hisilicon SEC crypto block cipher accelerator" + select CRYPTO_SKCIPHER + select CRYPTO_ALGAPI + select CRYPTO_LIB_DES + select SG_SPLIT + depends on ARM64 || COMPILE_TEST + depends on HAS_IOMEM + help + Support for Hisilicon SEC Engine in Hip06 and Hip07 + + To compile this as a module, choose M here: the module + will be called hisi_sec. + +config CRYPTO_DEV_HISI_SEC2 + tristate "Support for HiSilicon SEC2 crypto block cipher accelerator" + select CRYPTO_SKCIPHER + select CRYPTO_ALGAPI + select CRYPTO_LIB_DES + select CRYPTO_DEV_HISI_QM + select CRYPTO_AEAD + select CRYPTO_AUTHENC + select CRYPTO_HMAC + select CRYPTO_SHA1 + select CRYPTO_SHA256 + select CRYPTO_SHA512 + select CRYPTO_SM4_GENERIC + depends on PCI_MSI + depends on UACCE || UACCE=n + depends on ARM64 || (COMPILE_TEST && 64BIT) + depends on ACPI + help + Support for HiSilicon SEC Engine of version 2 in crypto subsystem. + It provides AES, SM4, and 3DES algorithms with ECB + CBC, and XTS cipher mode, and AEAD algorithms. + + To compile this as a module, choose M here: the module + will be called hisi_sec2. + +config CRYPTO_DEV_HISI_QM + tristate + depends on ARM64 || COMPILE_TEST + depends on PCI_MSI + depends on UACCE || UACCE=n + depends on ACPI + help + HiSilicon accelerator engines use a common queue management + interface. Specific engine driver may use this module. + +config CRYPTO_DEV_HISI_ZIP + tristate "Support for HiSilicon ZIP accelerator" + depends on PCI_MSI + depends on ARM64 || (COMPILE_TEST && 64BIT) + depends on !CPU_BIG_ENDIAN || COMPILE_TEST + depends on UACCE || UACCE=n + depends on ACPI + select CRYPTO_DEV_HISI_QM + select CRYPTO_DEFLATE + select CRYPTO_LZ4 + help + Support for HiSilicon ZIP Driver + +config CRYPTO_DEV_HISI_HPRE + tristate "Support for HISI HPRE accelerator" + depends on PCI_MSI + depends on UACCE || UACCE=n + depends on ARM64 || (COMPILE_TEST && 64BIT) + depends on ACPI + select CRYPTO_DEV_HISI_QM + select CRYPTO_DH + select CRYPTO_RSA + select CRYPTO_ECDH + help + Support for HiSilicon HPRE(High Performance RSA Engine) + accelerator, which can accelerate RSA and DH algorithms. diff --git a/drivers/crypto/hisilicon/Makefile b/drivers/crypto/hisilicon/Makefile new file mode 100644 index 000000000..e1068ee9f --- /dev/null +++ b/drivers/crypto/hisilicon/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_CRYPTO_DEV_HISI_HPRE) += hpre/ +obj-$(CONFIG_CRYPTO_DEV_HISI_SEC) += sec/ +obj-$(CONFIG_CRYPTO_DEV_HISI_SEC2) += sec2/ +obj-$(CONFIG_CRYPTO_DEV_HISI_QM) += hisi_qm.o +hisi_qm-objs = qm.o sgl.o debugfs.o +obj-$(CONFIG_CRYPTO_DEV_HISI_ZIP) += zip/ diff --git a/drivers/crypto/hisilicon/debugfs.c b/drivers/crypto/hisilicon/debugfs.c new file mode 100644 index 000000000..3ee6de16e --- /dev/null +++ b/drivers/crypto/hisilicon/debugfs.c @@ -0,0 +1,1272 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2022 HiSilicon Limited. */ +#include +#include "qm_common.h" + +#define QM_DFX_BASE 0x0100000 +#define QM_DFX_STATE1 0x0104000 +#define QM_DFX_STATE2 0x01040C8 +#define QM_DFX_COMMON 0x0000 +#define QM_DFX_BASE_LEN 0x5A +#define QM_DFX_STATE1_LEN 0x2E +#define QM_DFX_STATE2_LEN 0x11 +#define QM_DFX_COMMON_LEN 0xC3 +#define QM_DFX_REGS_LEN 4UL +#define QM_DBG_TMP_BUF_LEN 22 +#define QM_XQC_ADDR_MASK GENMASK(31, 0) +#define CURRENT_FUN_MASK GENMASK(5, 0) +#define CURRENT_Q_MASK GENMASK(31, 16) +#define QM_SQE_ADDR_MASK GENMASK(7, 0) + +#define QM_DFX_MB_CNT_VF 0x104010 +#define QM_DFX_DB_CNT_VF 0x104020 +#define QM_DFX_SQE_CNT_VF_SQN 0x104030 +#define QM_DFX_CQE_CNT_VF_CQN 0x104040 +#define QM_DFX_QN_SHIFT 16 +#define QM_DFX_CNT_CLR_CE 0x100118 +#define QM_DBG_WRITE_LEN 1024 +#define QM_IN_IDLE_ST_REG 0x1040e4 +#define QM_IN_IDLE_STATE 0x1 + +static const char * const qm_debug_file_name[] = { + [CURRENT_QM] = "current_qm", + [CURRENT_Q] = "current_q", + [CLEAR_ENABLE] = "clear_enable", +}; + +static const char * const qm_s[] = { + "work", "stop", +}; + +struct qm_dfx_item { + const char *name; + u32 offset; +}; + +struct qm_cmd_dump_item { + const char *cmd; + const char *info_name; + int (*dump_fn)(struct hisi_qm *qm, char *cmd, const char *info_name); +}; + +static struct qm_dfx_item qm_dfx_files[] = { + {"err_irq", offsetof(struct qm_dfx, err_irq_cnt)}, + {"aeq_irq", offsetof(struct qm_dfx, aeq_irq_cnt)}, + {"abnormal_irq", offsetof(struct qm_dfx, abnormal_irq_cnt)}, + {"create_qp_err", offsetof(struct qm_dfx, create_qp_err_cnt)}, + {"mb_err", offsetof(struct qm_dfx, mb_err_cnt)}, +}; + +#define CNT_CYC_REGS_NUM 10 +static const struct debugfs_reg32 qm_dfx_regs[] = { + /* XXX_CNT are reading clear register */ + {"QM_ECC_1BIT_CNT ", 0x104000}, + {"QM_ECC_MBIT_CNT ", 0x104008}, + {"QM_DFX_MB_CNT ", 0x104018}, + {"QM_DFX_DB_CNT ", 0x104028}, + {"QM_DFX_SQE_CNT ", 0x104038}, + {"QM_DFX_CQE_CNT ", 0x104048}, + {"QM_DFX_SEND_SQE_TO_ACC_CNT ", 0x104050}, + {"QM_DFX_WB_SQE_FROM_ACC_CNT ", 0x104058}, + {"QM_DFX_ACC_FINISH_CNT ", 0x104060}, + {"QM_DFX_CQE_ERR_CNT ", 0x1040b4}, + {"QM_DFX_FUNS_ACTIVE_ST ", 0x200}, + {"QM_ECC_1BIT_INF ", 0x104004}, + {"QM_ECC_MBIT_INF ", 0x10400c}, + {"QM_DFX_ACC_RDY_VLD0 ", 0x1040a0}, + {"QM_DFX_ACC_RDY_VLD1 ", 0x1040a4}, + {"QM_DFX_AXI_RDY_VLD ", 0x1040a8}, + {"QM_DFX_FF_ST0 ", 0x1040c8}, + {"QM_DFX_FF_ST1 ", 0x1040cc}, + {"QM_DFX_FF_ST2 ", 0x1040d0}, + {"QM_DFX_FF_ST3 ", 0x1040d4}, + {"QM_DFX_FF_ST4 ", 0x1040d8}, + {"QM_DFX_FF_ST5 ", 0x1040dc}, + {"QM_DFX_FF_ST6 ", 0x1040e0}, + {"QM_IN_IDLE_ST ", 0x1040e4}, + {"QM_CACHE_CTL ", 0x100050}, + {"QM_TIMEOUT_CFG ", 0x100070}, + {"QM_DB_TIMEOUT_CFG ", 0x100074}, + {"QM_FLR_PENDING_TIME_CFG ", 0x100078}, + {"QM_ARUSR_MCFG1 ", 0x100088}, + {"QM_AWUSR_MCFG1 ", 0x100098}, + {"QM_AXI_M_CFG_ENABLE ", 0x1000B0}, + {"QM_RAS_CE_THRESHOLD ", 0x1000F8}, + {"QM_AXI_TIMEOUT_CTRL ", 0x100120}, + {"QM_AXI_TIMEOUT_STATUS ", 0x100124}, + {"QM_CQE_AGGR_TIMEOUT_CTRL ", 0x100144}, + {"ACC_RAS_MSI_INT_SEL ", 0x1040fc}, + {"QM_CQE_OUT ", 0x104100}, + {"QM_EQE_OUT ", 0x104104}, + {"QM_AEQE_OUT ", 0x104108}, + {"QM_DB_INFO0 ", 0x104180}, + {"QM_DB_INFO1 ", 0x104184}, + {"QM_AM_CTRL_GLOBAL ", 0x300000}, + {"QM_AM_CURR_PORT_STS ", 0x300100}, + {"QM_AM_CURR_TRANS_RETURN ", 0x300150}, + {"QM_AM_CURR_RD_MAX_TXID ", 0x300154}, + {"QM_AM_CURR_WR_MAX_TXID ", 0x300158}, + {"QM_AM_ALARM_RRESP ", 0x300180}, + {"QM_AM_ALARM_BRESP ", 0x300184}, +}; + +static const struct debugfs_reg32 qm_vf_dfx_regs[] = { + {"QM_DFX_FUNS_ACTIVE_ST ", 0x200}, +}; + +/* define the QM's dfx regs region and region length */ +static struct dfx_diff_registers qm_diff_regs[] = { + { + .reg_offset = QM_DFX_BASE, + .reg_len = QM_DFX_BASE_LEN, + }, { + .reg_offset = QM_DFX_STATE1, + .reg_len = QM_DFX_STATE1_LEN, + }, { + .reg_offset = QM_DFX_STATE2, + .reg_len = QM_DFX_STATE2_LEN, + }, { + .reg_offset = QM_DFX_COMMON, + .reg_len = QM_DFX_COMMON_LEN, + }, +}; + +static struct hisi_qm *file_to_qm(struct debugfs_file *file) +{ + struct qm_debug *debug = file->debug; + + return container_of(debug, struct hisi_qm, debug); +} + +static ssize_t qm_cmd_read(struct file *filp, char __user *buffer, + size_t count, loff_t *pos) +{ + char buf[QM_DBG_READ_LEN]; + int len; + + len = scnprintf(buf, QM_DBG_READ_LEN, "%s\n", + "Please echo help to cmd to get help information"); + + return simple_read_from_buffer(buffer, count, pos, buf, len); +} + +static void dump_show(struct hisi_qm *qm, void *info, + unsigned int info_size, const char *info_name) +{ + struct device *dev = &qm->pdev->dev; + u8 *info_curr = info; + u32 i; +#define BYTE_PER_DW 4 + + dev_info(dev, "%s DUMP\n", info_name); + for (i = 0; i < info_size; i += BYTE_PER_DW, info_curr += BYTE_PER_DW) { + pr_info("DW%u: %02X%02X %02X%02X\n", i / BYTE_PER_DW, + *(info_curr + 3), *(info_curr + 2), *(info_curr + 1), *(info_curr)); + } +} + +static int qm_sqc_dump(struct hisi_qm *qm, char *s, const char *name) +{ + struct device *dev = &qm->pdev->dev; + struct qm_sqc sqc; + u32 qp_id; + int ret; + + if (!s) + return -EINVAL; + + ret = kstrtou32(s, 0, &qp_id); + if (ret || qp_id >= qm->qp_num) { + dev_err(dev, "Please input qp num (0-%u)", qm->qp_num - 1); + return -EINVAL; + } + + ret = qm_set_and_get_xqc(qm, QM_MB_CMD_SQC, &sqc, qp_id, 1); + if (!ret) { + sqc.base_h = cpu_to_le32(QM_XQC_ADDR_MASK); + sqc.base_l = cpu_to_le32(QM_XQC_ADDR_MASK); + dump_show(qm, &sqc, sizeof(struct qm_sqc), name); + + return 0; + } + + down_read(&qm->qps_lock); + if (qm->sqc) { + memcpy(&sqc, qm->sqc + qp_id, sizeof(struct qm_sqc)); + sqc.base_h = cpu_to_le32(QM_XQC_ADDR_MASK); + sqc.base_l = cpu_to_le32(QM_XQC_ADDR_MASK); + dump_show(qm, &sqc, sizeof(struct qm_sqc), "SOFT SQC"); + } + up_read(&qm->qps_lock); + + return 0; +} + +static int qm_cqc_dump(struct hisi_qm *qm, char *s, const char *name) +{ + struct device *dev = &qm->pdev->dev; + struct qm_cqc cqc; + u32 qp_id; + int ret; + + if (!s) + return -EINVAL; + + ret = kstrtou32(s, 0, &qp_id); + if (ret || qp_id >= qm->qp_num) { + dev_err(dev, "Please input qp num (0-%u)", qm->qp_num - 1); + return -EINVAL; + } + + ret = qm_set_and_get_xqc(qm, QM_MB_CMD_CQC, &cqc, qp_id, 1); + if (!ret) { + cqc.base_h = cpu_to_le32(QM_XQC_ADDR_MASK); + cqc.base_l = cpu_to_le32(QM_XQC_ADDR_MASK); + dump_show(qm, &cqc, sizeof(struct qm_cqc), name); + + return 0; + } + + down_read(&qm->qps_lock); + if (qm->cqc) { + memcpy(&cqc, qm->cqc + qp_id, sizeof(struct qm_cqc)); + cqc.base_h = cpu_to_le32(QM_XQC_ADDR_MASK); + cqc.base_l = cpu_to_le32(QM_XQC_ADDR_MASK); + dump_show(qm, &cqc, sizeof(struct qm_cqc), "SOFT CQC"); + } + up_read(&qm->qps_lock); + + return 0; +} + +static int qm_eqc_aeqc_dump(struct hisi_qm *qm, char *s, const char *name) +{ + struct device *dev = &qm->pdev->dev; + struct qm_aeqc aeqc; + struct qm_eqc eqc; + size_t size; + void *xeqc; + int ret; + u8 cmd; + + if (strsep(&s, " ")) { + dev_err(dev, "Please do not input extra characters!\n"); + return -EINVAL; + } + + if (!strcmp(name, "EQC")) { + cmd = QM_MB_CMD_EQC; + size = sizeof(struct qm_eqc); + xeqc = &eqc; + } else { + cmd = QM_MB_CMD_AEQC; + size = sizeof(struct qm_aeqc); + xeqc = &aeqc; + } + + ret = qm_set_and_get_xqc(qm, cmd, xeqc, 0, 1); + if (ret) + return ret; + + aeqc.base_h = cpu_to_le32(QM_XQC_ADDR_MASK); + aeqc.base_l = cpu_to_le32(QM_XQC_ADDR_MASK); + eqc.base_h = cpu_to_le32(QM_XQC_ADDR_MASK); + eqc.base_l = cpu_to_le32(QM_XQC_ADDR_MASK); + dump_show(qm, xeqc, size, name); + + return ret; +} + +static int q_dump_param_parse(struct hisi_qm *qm, char *s, + u32 *e_id, u32 *q_id, u16 q_depth) +{ + struct device *dev = &qm->pdev->dev; + unsigned int qp_num = qm->qp_num; + char *presult; + int ret; + + presult = strsep(&s, " "); + if (!presult) { + dev_err(dev, "Please input qp number!\n"); + return -EINVAL; + } + + ret = kstrtou32(presult, 0, q_id); + if (ret || *q_id >= qp_num) { + dev_err(dev, "Please input qp num (0-%u)", qp_num - 1); + return -EINVAL; + } + + presult = strsep(&s, " "); + if (!presult) { + dev_err(dev, "Please input sqe number!\n"); + return -EINVAL; + } + + ret = kstrtou32(presult, 0, e_id); + if (ret || *e_id >= q_depth) { + dev_err(dev, "Please input sqe num (0-%d)", q_depth - 1); + return -EINVAL; + } + + if (strsep(&s, " ")) { + dev_err(dev, "Please do not input extra characters!\n"); + return -EINVAL; + } + + return 0; +} + +static int qm_sq_dump(struct hisi_qm *qm, char *s, const char *name) +{ + u16 sq_depth = qm->qp_array->sq_depth; + struct hisi_qp *qp; + u32 qp_id, sqe_id; + void *sqe; + int ret; + + ret = q_dump_param_parse(qm, s, &sqe_id, &qp_id, sq_depth); + if (ret) + return ret; + + sqe = kzalloc(qm->sqe_size, GFP_KERNEL); + if (!sqe) + return -ENOMEM; + + qp = &qm->qp_array[qp_id]; + memcpy(sqe, qp->sqe + sqe_id * qm->sqe_size, qm->sqe_size); + memset(sqe + qm->debug.sqe_mask_offset, QM_SQE_ADDR_MASK, + qm->debug.sqe_mask_len); + + dump_show(qm, sqe, qm->sqe_size, name); + + kfree(sqe); + + return 0; +} + +static int qm_cq_dump(struct hisi_qm *qm, char *s, const char *name) +{ + struct qm_cqe *cqe_curr; + struct hisi_qp *qp; + u32 qp_id, cqe_id; + int ret; + + ret = q_dump_param_parse(qm, s, &cqe_id, &qp_id, qm->qp_array->cq_depth); + if (ret) + return ret; + + qp = &qm->qp_array[qp_id]; + cqe_curr = qp->cqe + cqe_id; + dump_show(qm, cqe_curr, sizeof(struct qm_cqe), name); + + return 0; +} + +static int qm_eq_aeq_dump(struct hisi_qm *qm, char *s, const char *name) +{ + struct device *dev = &qm->pdev->dev; + u16 xeq_depth; + size_t size; + void *xeqe; + u32 xeqe_id; + int ret; + + if (!s) + return -EINVAL; + + ret = kstrtou32(s, 0, &xeqe_id); + if (ret) + return -EINVAL; + + if (!strcmp(name, "EQE")) { + xeq_depth = qm->eq_depth; + size = sizeof(struct qm_eqe); + } else { + xeq_depth = qm->aeq_depth; + size = sizeof(struct qm_aeqe); + } + + if (xeqe_id >= xeq_depth) { + dev_err(dev, "Please input eqe or aeqe num (0-%d)", xeq_depth - 1); + return -EINVAL; + } + + down_read(&qm->qps_lock); + + if (qm->eqe && !strcmp(name, "EQE")) { + xeqe = qm->eqe + xeqe_id; + } else if (qm->aeqe && !strcmp(name, "AEQE")) { + xeqe = qm->aeqe + xeqe_id; + } else { + ret = -EINVAL; + goto err_unlock; + } + + dump_show(qm, xeqe, size, name); + +err_unlock: + up_read(&qm->qps_lock); + return ret; +} + +static int qm_dbg_help(struct hisi_qm *qm, char *s) +{ + struct device *dev = &qm->pdev->dev; + + if (strsep(&s, " ")) { + dev_err(dev, "Please do not input extra characters!\n"); + return -EINVAL; + } + + dev_info(dev, "available commands:\n"); + dev_info(dev, "sqc \n"); + dev_info(dev, "cqc \n"); + dev_info(dev, "eqc\n"); + dev_info(dev, "aeqc\n"); + dev_info(dev, "sq \n"); + dev_info(dev, "cq \n"); + dev_info(dev, "eq \n"); + dev_info(dev, "aeq \n"); + + return 0; +} + +static const struct qm_cmd_dump_item qm_cmd_dump_table[] = { + { + .cmd = "sqc", + .info_name = "SQC", + .dump_fn = qm_sqc_dump, + }, { + .cmd = "cqc", + .info_name = "CQC", + .dump_fn = qm_cqc_dump, + }, { + .cmd = "eqc", + .info_name = "EQC", + .dump_fn = qm_eqc_aeqc_dump, + }, { + .cmd = "aeqc", + .info_name = "AEQC", + .dump_fn = qm_eqc_aeqc_dump, + }, { + .cmd = "sq", + .info_name = "SQE", + .dump_fn = qm_sq_dump, + }, { + .cmd = "cq", + .info_name = "CQE", + .dump_fn = qm_cq_dump, + }, { + .cmd = "eq", + .info_name = "EQE", + .dump_fn = qm_eq_aeq_dump, + }, { + .cmd = "aeq", + .info_name = "AEQE", + .dump_fn = qm_eq_aeq_dump, + }, +}; + +static int qm_cmd_write_dump(struct hisi_qm *qm, const char *cmd_buf) +{ + struct device *dev = &qm->pdev->dev; + char *presult, *s, *s_tmp; + int table_size, i, ret; + + s = kstrdup(cmd_buf, GFP_KERNEL); + if (!s) + return -ENOMEM; + + s_tmp = s; + presult = strsep(&s, " "); + if (!presult) { + ret = -EINVAL; + goto err_buffer_free; + } + + if (!strcmp(presult, "help")) { + ret = qm_dbg_help(qm, s); + goto err_buffer_free; + } + + table_size = ARRAY_SIZE(qm_cmd_dump_table); + for (i = 0; i < table_size; i++) { + if (!strcmp(presult, qm_cmd_dump_table[i].cmd)) { + ret = qm_cmd_dump_table[i].dump_fn(qm, s, + qm_cmd_dump_table[i].info_name); + break; + } + } + + if (i == table_size) { + dev_info(dev, "Please echo help\n"); + ret = -EINVAL; + } + +err_buffer_free: + kfree(s_tmp); + + return ret; +} + +static ssize_t qm_cmd_write(struct file *filp, const char __user *buffer, + size_t count, loff_t *pos) +{ + struct hisi_qm *qm = filp->private_data; + char *cmd_buf, *cmd_buf_tmp; + int ret; + + if (*pos) + return 0; + + ret = hisi_qm_get_dfx_access(qm); + if (ret) + return ret; + + /* Judge if the instance is being reset. */ + if (unlikely(atomic_read(&qm->status.flags) == QM_STOP)) { + ret = 0; + goto put_dfx_access; + } + + if (count > QM_DBG_WRITE_LEN) { + ret = -ENOSPC; + goto put_dfx_access; + } + + cmd_buf = memdup_user_nul(buffer, count); + if (IS_ERR(cmd_buf)) { + ret = PTR_ERR(cmd_buf); + goto put_dfx_access; + } + + cmd_buf_tmp = strchr(cmd_buf, '\n'); + if (cmd_buf_tmp) { + *cmd_buf_tmp = '\0'; + count = cmd_buf_tmp - cmd_buf + 1; + } + + ret = qm_cmd_write_dump(qm, cmd_buf); + if (ret) { + kfree(cmd_buf); + goto put_dfx_access; + } + + kfree(cmd_buf); + + ret = count; + +put_dfx_access: + hisi_qm_put_dfx_access(qm); + return ret; +} + +static const struct file_operations qm_cmd_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = qm_cmd_read, + .write = qm_cmd_write, +}; + +/** + * hisi_qm_regs_dump() - Dump registers's value. + * @s: debugfs file handle. + * @regset: accelerator registers information. + * + * Dump accelerator registers. + */ +void hisi_qm_regs_dump(struct seq_file *s, struct debugfs_regset32 *regset) +{ + struct pci_dev *pdev = to_pci_dev(regset->dev); + struct hisi_qm *qm = pci_get_drvdata(pdev); + const struct debugfs_reg32 *regs = regset->regs; + int regs_len = regset->nregs; + int i, ret; + u32 val; + + ret = hisi_qm_get_dfx_access(qm); + if (ret) + return; + + for (i = 0; i < regs_len; i++) { + val = readl(regset->base + regs[i].offset); + seq_printf(s, "%s= 0x%08x\n", regs[i].name, val); + } + + hisi_qm_put_dfx_access(qm); +} +EXPORT_SYMBOL_GPL(hisi_qm_regs_dump); + +static int qm_regs_show(struct seq_file *s, void *unused) +{ + struct hisi_qm *qm = s->private; + struct debugfs_regset32 regset; + + if (qm->fun_type == QM_HW_PF) { + regset.regs = qm_dfx_regs; + regset.nregs = ARRAY_SIZE(qm_dfx_regs); + } else { + regset.regs = qm_vf_dfx_regs; + regset.nregs = ARRAY_SIZE(qm_vf_dfx_regs); + } + + regset.base = qm->io_base; + regset.dev = &qm->pdev->dev; + + hisi_qm_regs_dump(s, ®set); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(qm_regs); + +static u32 current_q_read(struct hisi_qm *qm) +{ + return readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) >> QM_DFX_QN_SHIFT; +} + +static int current_q_write(struct hisi_qm *qm, u32 val) +{ + u32 tmp; + + if (val >= qm->debug.curr_qm_qp_num) + return -EINVAL; + + tmp = val << QM_DFX_QN_SHIFT | + (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_FUN_MASK); + writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN); + + tmp = val << QM_DFX_QN_SHIFT | + (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_FUN_MASK); + writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN); + + return 0; +} + +static u32 clear_enable_read(struct hisi_qm *qm) +{ + return readl(qm->io_base + QM_DFX_CNT_CLR_CE); +} + +/* rd_clr_ctrl 1 enable read clear, otherwise 0 disable it */ +static int clear_enable_write(struct hisi_qm *qm, u32 rd_clr_ctrl) +{ + if (rd_clr_ctrl > 1) + return -EINVAL; + + writel(rd_clr_ctrl, qm->io_base + QM_DFX_CNT_CLR_CE); + + return 0; +} + +static u32 current_qm_read(struct hisi_qm *qm) +{ + return readl(qm->io_base + QM_DFX_MB_CNT_VF); +} + +static int qm_get_vf_qp_num(struct hisi_qm *qm, u32 fun_num) +{ + u32 remain_q_num, vfq_num; + u32 num_vfs = qm->vfs_num; + + vfq_num = (qm->ctrl_qp_num - qm->qp_num) / num_vfs; + if (vfq_num >= qm->max_qp_num) + return qm->max_qp_num; + + remain_q_num = (qm->ctrl_qp_num - qm->qp_num) % num_vfs; + if (vfq_num + remain_q_num <= qm->max_qp_num) + return fun_num == num_vfs ? vfq_num + remain_q_num : vfq_num; + + /* + * if vfq_num + remain_q_num > max_qp_num, the last VFs, + * each with one more queue. + */ + return fun_num + remain_q_num > num_vfs ? vfq_num + 1 : vfq_num; +} + +static int current_qm_write(struct hisi_qm *qm, u32 val) +{ + u32 tmp; + + if (val > qm->vfs_num) + return -EINVAL; + + /* According PF or VF Dev ID to calculation curr_qm_qp_num and store */ + if (!val) + qm->debug.curr_qm_qp_num = qm->qp_num; + else + qm->debug.curr_qm_qp_num = qm_get_vf_qp_num(qm, val); + + writel(val, qm->io_base + QM_DFX_MB_CNT_VF); + writel(val, qm->io_base + QM_DFX_DB_CNT_VF); + + tmp = val | + (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_Q_MASK); + writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN); + + tmp = val | + (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_Q_MASK); + writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN); + + return 0; +} + +static ssize_t qm_debug_read(struct file *filp, char __user *buf, + size_t count, loff_t *pos) +{ + struct debugfs_file *file = filp->private_data; + enum qm_debug_file index = file->index; + struct hisi_qm *qm = file_to_qm(file); + char tbuf[QM_DBG_TMP_BUF_LEN]; + u32 val; + int ret; + + ret = hisi_qm_get_dfx_access(qm); + if (ret) + return ret; + + mutex_lock(&file->lock); + switch (index) { + case CURRENT_QM: + val = current_qm_read(qm); + break; + case CURRENT_Q: + val = current_q_read(qm); + break; + case CLEAR_ENABLE: + val = clear_enable_read(qm); + break; + default: + goto err_input; + } + mutex_unlock(&file->lock); + + hisi_qm_put_dfx_access(qm); + ret = scnprintf(tbuf, QM_DBG_TMP_BUF_LEN, "%u\n", val); + return simple_read_from_buffer(buf, count, pos, tbuf, ret); + +err_input: + mutex_unlock(&file->lock); + hisi_qm_put_dfx_access(qm); + return -EINVAL; +} + +static ssize_t qm_debug_write(struct file *filp, const char __user *buf, + size_t count, loff_t *pos) +{ + struct debugfs_file *file = filp->private_data; + enum qm_debug_file index = file->index; + struct hisi_qm *qm = file_to_qm(file); + unsigned long val; + char tbuf[QM_DBG_TMP_BUF_LEN]; + int len, ret; + + if (*pos != 0) + return 0; + + if (count >= QM_DBG_TMP_BUF_LEN) + return -ENOSPC; + + len = simple_write_to_buffer(tbuf, QM_DBG_TMP_BUF_LEN - 1, pos, buf, + count); + if (len < 0) + return len; + + tbuf[len] = '\0'; + if (kstrtoul(tbuf, 0, &val)) + return -EFAULT; + + ret = hisi_qm_get_dfx_access(qm); + if (ret) + return ret; + + mutex_lock(&file->lock); + switch (index) { + case CURRENT_QM: + ret = current_qm_write(qm, val); + break; + case CURRENT_Q: + ret = current_q_write(qm, val); + break; + case CLEAR_ENABLE: + ret = clear_enable_write(qm, val); + break; + default: + ret = -EINVAL; + } + mutex_unlock(&file->lock); + + hisi_qm_put_dfx_access(qm); + + if (ret) + return ret; + + return count; +} + +static const struct file_operations qm_debug_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = qm_debug_read, + .write = qm_debug_write, +}; + +static void dfx_regs_uninit(struct hisi_qm *qm, + struct dfx_diff_registers *dregs, int reg_len) +{ + int i; + + if (!dregs) + return; + + /* Setting the pointer is NULL to prevent double free */ + for (i = 0; i < reg_len; i++) { + if (!dregs[i].regs) + continue; + + kfree(dregs[i].regs); + dregs[i].regs = NULL; + } + kfree(dregs); +} + +static struct dfx_diff_registers *dfx_regs_init(struct hisi_qm *qm, + const struct dfx_diff_registers *cregs, u32 reg_len) +{ + struct dfx_diff_registers *diff_regs; + u32 j, base_offset; + int i; + + diff_regs = kzalloc_objs(*diff_regs, reg_len); + if (!diff_regs) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < reg_len; i++) { + if (!cregs[i].reg_len) + continue; + + diff_regs[i].reg_offset = cregs[i].reg_offset; + diff_regs[i].reg_len = cregs[i].reg_len; + diff_regs[i].regs = kcalloc(QM_DFX_REGS_LEN, cregs[i].reg_len, + GFP_KERNEL); + if (!diff_regs[i].regs) + goto alloc_error; + + for (j = 0; j < diff_regs[i].reg_len; j++) { + base_offset = diff_regs[i].reg_offset + + j * QM_DFX_REGS_LEN; + diff_regs[i].regs[j] = readl(qm->io_base + base_offset); + } + } + + return diff_regs; + +alloc_error: + while (i > 0) { + i--; + kfree(diff_regs[i].regs); + } + kfree(diff_regs); + return ERR_PTR(-ENOMEM); +} + +static int qm_diff_regs_init(struct hisi_qm *qm, + struct dfx_diff_registers *dregs, u32 reg_len) +{ + int ret; + + qm->debug.qm_diff_regs = dfx_regs_init(qm, qm_diff_regs, ARRAY_SIZE(qm_diff_regs)); + if (IS_ERR(qm->debug.qm_diff_regs)) { + ret = PTR_ERR(qm->debug.qm_diff_regs); + qm->debug.qm_diff_regs = NULL; + return ret; + } + + qm->debug.acc_diff_regs = dfx_regs_init(qm, dregs, reg_len); + if (IS_ERR(qm->debug.acc_diff_regs)) { + dfx_regs_uninit(qm, qm->debug.qm_diff_regs, ARRAY_SIZE(qm_diff_regs)); + ret = PTR_ERR(qm->debug.acc_diff_regs); + qm->debug.acc_diff_regs = NULL; + qm->debug.qm_diff_regs = NULL; + return ret; + } + + return 0; +} + +static void qm_last_regs_uninit(struct hisi_qm *qm) +{ + struct qm_debug *debug = &qm->debug; + + if (qm->fun_type == QM_HW_VF || !debug->qm_last_words) + return; + + kfree(debug->qm_last_words); + debug->qm_last_words = NULL; +} + +static int qm_last_regs_init(struct hisi_qm *qm) +{ + int dfx_regs_num = ARRAY_SIZE(qm_dfx_regs); + struct qm_debug *debug = &qm->debug; + int i; + + if (qm->fun_type == QM_HW_VF) + return 0; + + debug->qm_last_words = kcalloc(dfx_regs_num, sizeof(unsigned int), GFP_KERNEL); + if (!debug->qm_last_words) + return -ENOMEM; + + for (i = 0; i < dfx_regs_num; i++) { + debug->qm_last_words[i] = readl_relaxed(qm->io_base + + qm_dfx_regs[i].offset); + } + + return 0; +} + +static void qm_diff_regs_uninit(struct hisi_qm *qm, u32 reg_len) +{ + dfx_regs_uninit(qm, qm->debug.acc_diff_regs, reg_len); + qm->debug.acc_diff_regs = NULL; + dfx_regs_uninit(qm, qm->debug.qm_diff_regs, ARRAY_SIZE(qm_diff_regs)); + qm->debug.qm_diff_regs = NULL; +} + +/** + * hisi_qm_regs_debugfs_init() - Allocate memory for registers. + * @qm: device qm handle. + * @dregs: diff registers handle. + * @reg_len: diff registers region length. + */ +int hisi_qm_regs_debugfs_init(struct hisi_qm *qm, + struct dfx_diff_registers *dregs, u32 reg_len) +{ + int ret; + + if (!qm || !dregs) + return -EINVAL; + + if (qm->fun_type != QM_HW_PF) + return 0; + + ret = qm_last_regs_init(qm); + if (ret) { + dev_info(&qm->pdev->dev, "failed to init qm words memory!\n"); + return ret; + } + + ret = qm_diff_regs_init(qm, dregs, reg_len); + if (ret) { + qm_last_regs_uninit(qm); + return ret; + } + + return 0; +} +EXPORT_SYMBOL_GPL(hisi_qm_regs_debugfs_init); + +/** + * hisi_qm_regs_debugfs_uninit() - Free memory for registers. + * @qm: device qm handle. + * @reg_len: diff registers region length. + */ +void hisi_qm_regs_debugfs_uninit(struct hisi_qm *qm, u32 reg_len) +{ + if (!qm || qm->fun_type != QM_HW_PF) + return; + + qm_diff_regs_uninit(qm, reg_len); + qm_last_regs_uninit(qm); +} +EXPORT_SYMBOL_GPL(hisi_qm_regs_debugfs_uninit); + +/** + * hisi_qm_acc_diff_regs_dump() - Dump registers's value. + * @qm: device qm handle. + * @s: Debugfs file handle. + * @dregs: diff registers handle. + * @regs_len: diff registers region length. + */ +void hisi_qm_acc_diff_regs_dump(struct hisi_qm *qm, struct seq_file *s, + struct dfx_diff_registers *dregs, u32 regs_len) +{ + u32 j, val, base_offset; + int i, ret; + + if (!qm || !s || !dregs) + return; + + ret = hisi_qm_get_dfx_access(qm); + if (ret) + return; + + down_read(&qm->qps_lock); + for (i = 0; i < regs_len; i++) { + if (!dregs[i].reg_len) + continue; + + for (j = 0; j < dregs[i].reg_len; j++) { + base_offset = dregs[i].reg_offset + j * QM_DFX_REGS_LEN; + val = readl(qm->io_base + base_offset); + if (val != dregs[i].regs[j]) + seq_printf(s, "0x%08x = 0x%08x ---> 0x%08x\n", + base_offset, dregs[i].regs[j], val); + } + } + up_read(&qm->qps_lock); + + hisi_qm_put_dfx_access(qm); +} +EXPORT_SYMBOL_GPL(hisi_qm_acc_diff_regs_dump); + +void hisi_qm_show_last_dfx_regs(struct hisi_qm *qm) +{ + struct qm_debug *debug = &qm->debug; + struct pci_dev *pdev = qm->pdev; + u32 val; + int i; + + if (qm->fun_type == QM_HW_VF || !debug->qm_last_words) + return; + + for (i = 0; i < ARRAY_SIZE(qm_dfx_regs); i++) { + val = readl_relaxed(qm->io_base + qm_dfx_regs[i].offset); + if (debug->qm_last_words[i] != val) + pci_info(pdev, "%s \t= 0x%08x => 0x%08x\n", + qm_dfx_regs[i].name, debug->qm_last_words[i], val); + } +} + +static int qm_usage_percent(struct hisi_qm *qm, int chan_num) +{ + u32 val, used_bw, total_bw; + + val = readl(qm->io_base + QM_CHANNEL_USAGE_OFFSET + + chan_num * QM_CHANNEL_ADDR_INTRVL); + used_bw = lower_16_bits(val); + total_bw = upper_16_bits(val); + if (!total_bw) + return -EIO; + + if (total_bw <= used_bw) + return QM_MAX_DEV_USAGE; + + return (used_bw * QM_DEV_USAGE_RATE) / total_bw; +} + +static int qm_usage_show(struct seq_file *s, void *unused) +{ + struct hisi_qm *qm = s->private; + bool dev_is_active = true; + int i, ret; + + /* If device is in suspended, usage is 0. */ + ret = hisi_qm_get_dfx_access(qm); + if (ret == -EAGAIN) { + dev_is_active = false; + } else if (ret) { + dev_err(&qm->pdev->dev, "failed to get dfx access for usage_show!\n"); + return ret; + } + + ret = 0; + for (i = 0; i < qm->channel_data.channel_num; i++) { + if (dev_is_active) { + ret = qm_usage_percent(qm, i); + if (ret < 0) { + hisi_qm_put_dfx_access(qm); + return ret; + } + } + seq_printf(s, "%s: %d\n", qm->channel_data.channel_name[i], ret); + } + + if (dev_is_active) + hisi_qm_put_dfx_access(qm); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(qm_usage); + +static int qm_diff_regs_show(struct seq_file *s, void *unused) +{ + struct hisi_qm *qm = s->private; + + hisi_qm_acc_diff_regs_dump(qm, s, qm->debug.qm_diff_regs, + ARRAY_SIZE(qm_diff_regs)); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(qm_diff_regs); + +static int qm_state_show(struct seq_file *s, void *unused) +{ + struct hisi_qm *qm = s->private; + u32 val; + int ret; + + /* If device is in suspended, directly return the idle state. */ + ret = hisi_qm_get_dfx_access(qm); + if (!ret) { + val = readl(qm->io_base + QM_IN_IDLE_ST_REG); + hisi_qm_put_dfx_access(qm); + } else if (ret == -EAGAIN) { + val = QM_IN_IDLE_STATE; + } else { + return ret; + } + + seq_printf(s, "%u\n", val); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(qm_state); + +static ssize_t qm_status_read(struct file *filp, char __user *buffer, + size_t count, loff_t *pos) +{ + struct hisi_qm *qm = filp->private_data; + char buf[QM_DBG_READ_LEN]; + int val, len; + + val = atomic_read(&qm->status.flags); + len = scnprintf(buf, QM_DBG_READ_LEN, "%s\n", qm_s[val]); + + return simple_read_from_buffer(buffer, count, pos, buf, len); +} + +static const struct file_operations qm_status_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = qm_status_read, +}; + +static void qm_create_debugfs_file(struct hisi_qm *qm, struct dentry *dir, + enum qm_debug_file index) +{ + struct debugfs_file *file = qm->debug.files + index; + + file->index = index; + mutex_init(&file->lock); + file->debug = &qm->debug; + + debugfs_create_file(qm_debug_file_name[index], 0600, dir, file, + &qm_debug_fops); +} + +static int qm_debugfs_atomic64_set(void *data, u64 val) +{ + if (val) + return -EINVAL; + + atomic64_set((atomic64_t *)data, 0); + + return 0; +} + +static int qm_debugfs_atomic64_get(void *data, u64 *val) +{ + *val = atomic64_read((atomic64_t *)data); + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(qm_atomic64_ops, qm_debugfs_atomic64_get, + qm_debugfs_atomic64_set, "%llu\n"); + +/** + * hisi_qm_debug_init() - Initialize qm related debugfs files. + * @qm: The qm for which we want to add debugfs files. + * + * Create qm related debugfs files. + */ +void hisi_qm_debug_init(struct hisi_qm *qm) +{ + struct dfx_diff_registers *qm_regs = qm->debug.qm_diff_regs; + struct qm_dev_dfx *dev_dfx = &qm->debug.dev_dfx; + struct qm_dfx *dfx = &qm->debug.dfx; + struct dentry *qm_d; + void *data; + int i; + + qm_d = debugfs_create_dir("qm", qm->debug.debug_root); + qm->debug.qm_d = qm_d; + + /* only show this in PF */ + if (qm->fun_type == QM_HW_PF) { + debugfs_create_file("qm_state", 0444, qm->debug.qm_d, + qm, &qm_state_fops); + + qm_create_debugfs_file(qm, qm->debug.debug_root, CURRENT_QM); + for (i = CURRENT_Q; i < DEBUG_FILE_NUM; i++) + qm_create_debugfs_file(qm, qm->debug.qm_d, i); + } + + if (qm_regs) + debugfs_create_file("diff_regs", 0444, qm->debug.qm_d, + qm, &qm_diff_regs_fops); + + if (qm->ver >= QM_HW_V5) + debugfs_create_file("dev_usage", 0444, qm->debug.debug_root, qm, &qm_usage_fops); + + debugfs_create_file("regs", 0444, qm->debug.qm_d, qm, &qm_regs_fops); + + debugfs_create_file("cmd", 0600, qm->debug.qm_d, qm, &qm_cmd_fops); + + debugfs_create_file("status", 0444, qm->debug.qm_d, qm, + &qm_status_fops); + + debugfs_create_u32("dev_state", 0444, qm->debug.qm_d, &dev_dfx->dev_state); + debugfs_create_u32("dev_timeout", 0644, qm->debug.qm_d, &dev_dfx->dev_timeout); + + for (i = 0; i < ARRAY_SIZE(qm_dfx_files); i++) { + data = (atomic64_t *)((uintptr_t)dfx + qm_dfx_files[i].offset); + debugfs_create_file(qm_dfx_files[i].name, + 0644, + qm_d, + data, + &qm_atomic64_ops); + } + + if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) + hisi_qm_set_algqos_init(qm); +} +EXPORT_SYMBOL_GPL(hisi_qm_debug_init); + +/** + * hisi_qm_debug_regs_clear() - clear qm debug related registers. + * @qm: The qm for which we want to clear its debug registers. + */ +void hisi_qm_debug_regs_clear(struct hisi_qm *qm) +{ + const struct debugfs_reg32 *regs; + int i; + + /* clear current_qm */ + writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF); + writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF); + + /* clear current_q */ + writel(0x0, qm->io_base + QM_DFX_SQE_CNT_VF_SQN); + writel(0x0, qm->io_base + QM_DFX_CQE_CNT_VF_CQN); + + /* + * these registers are reading and clearing, so clear them after + * reading them. + */ + writel(0x1, qm->io_base + QM_DFX_CNT_CLR_CE); + + regs = qm_dfx_regs; + for (i = 0; i < CNT_CYC_REGS_NUM; i++) { + readl(qm->io_base + regs->offset); + regs++; + } + + /* clear clear_enable */ + writel(0x0, qm->io_base + QM_DFX_CNT_CLR_CE); +} +EXPORT_SYMBOL_GPL(hisi_qm_debug_regs_clear); diff --git a/drivers/crypto/hisilicon/hpre/Makefile b/drivers/crypto/hisilicon/hpre/Makefile new file mode 100644 index 000000000..4fd32b789 --- /dev/null +++ b/drivers/crypto/hisilicon/hpre/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_CRYPTO_DEV_HISI_HPRE) += hisi_hpre.o +hisi_hpre-objs = hpre_main.o hpre_crypto.o diff --git a/drivers/crypto/hisilicon/hpre/hpre.h b/drivers/crypto/hisilicon/hpre/hpre.h new file mode 100644 index 000000000..021dbd9a1 --- /dev/null +++ b/drivers/crypto/hisilicon/hpre/hpre.h @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2019 HiSilicon Limited. */ +#ifndef __HISI_HPRE_H +#define __HISI_HPRE_H + +#include +#include + +#define HPRE_SQE_SIZE sizeof(struct hpre_sqe) +#define HPRE_PF_DEF_Q_NUM 64 +#define HPRE_PF_DEF_Q_BASE 0 + +/* + * type used in qm sqc DW6. + * 0 - Algorithm which has been supported in V2, like RSA, DH and so on; + * 1 - ECC algorithm in V3. + */ +#define HPRE_V2_ALG_TYPE 0 +#define HPRE_V3_ECC_ALG_TYPE 1 + +enum { + HPRE_CLUSTER0, + HPRE_CLUSTER1, + HPRE_CLUSTER2, + HPRE_CLUSTER3, + HPRE_CLUSTERS_NUM_MAX +}; + +enum hpre_ctrl_dbgfs_file { + HPRE_CLEAR_ENABLE, + HPRE_CLUSTER_CTRL, + HPRE_DEBUG_FILE_NUM, +}; + +enum hpre_dfx_dbgfs_file { + HPRE_SEND_CNT, + HPRE_RECV_CNT, + HPRE_SEND_FAIL_CNT, + HPRE_SEND_BUSY_CNT, + HPRE_OVER_THRHLD_CNT, + HPRE_OVERTIME_THRHLD, + HPRE_INVALID_REQ_CNT, + HPRE_DFX_FILE_NUM +}; + +#define HPRE_DEBUGFS_FILE_NUM (HPRE_DEBUG_FILE_NUM + HPRE_CLUSTERS_NUM_MAX - 1) + +struct hpre_debugfs_file { + int index; + enum hpre_ctrl_dbgfs_file type; + spinlock_t lock; + struct hpre_debug *debug; +}; + +struct hpre_dfx { + atomic64_t value; + enum hpre_dfx_dbgfs_file type; +}; + +/* + * One HPRE controller has one PF and multiple VFs, some global configurations + * which PF has need this structure. + * Just relevant for PF. + */ +struct hpre_debug { + struct hpre_dfx dfx[HPRE_DFX_FILE_NUM]; + struct hpre_debugfs_file files[HPRE_DEBUGFS_FILE_NUM]; +}; + +struct hpre { + struct hisi_qm qm; + struct hpre_debug debug; + unsigned long status; +}; + +enum hpre_alg_type { + HPRE_ALG_NC_NCRT = 0x0, + HPRE_ALG_NC_CRT = 0x1, + HPRE_ALG_KG_STD = 0x2, + HPRE_ALG_KG_CRT = 0x3, + HPRE_ALG_DH_G2 = 0x4, + HPRE_ALG_DH = 0x5, + HPRE_ALG_ECC_MUL = 0xD, + /* shared by x25519 and x448, but x448 is not supported now */ + HPRE_ALG_CURVE25519_MUL = 0x10, +}; + +struct hpre_sqe { + __le32 dw0; + __u8 task_len1; + __u8 task_len2; + __u8 mrttest_num; + __u8 resv1; + __le64 key; + __le64 in; + __le64 out; + __le64 tag; +#define _HPRE_SQE_ALIGN_EXT 6 + __le32 rsvd1[_HPRE_SQE_ALIGN_EXT]; +}; + +enum hpre_cap_table_type { + QM_RAS_NFE_TYPE = 0x0, + QM_RAS_NFE_RESET, + QM_RAS_CE_TYPE, + HPRE_RAS_NFE_TYPE, + HPRE_RAS_NFE_RESET, + HPRE_RAS_CE_TYPE, + HPRE_CORE_INFO, + HPRE_CORE_EN, + HPRE_DRV_ALG_BITMAP, + HPRE_ALG_BITMAP, + HPRE_CORE1_BITMAP_CAP, + HPRE_CORE2_BITMAP_CAP, + HPRE_CORE3_BITMAP_CAP, + HPRE_CORE4_BITMAP_CAP, + HPRE_CORE5_BITMAP_CAP, + HPRE_CORE6_BITMAP_CAP, + HPRE_CORE7_BITMAP_CAP, + HPRE_CORE8_BITMAP_CAP, + HPRE_CORE9_BITMAP_CAP, + HPRE_CORE10_BITMAP_CAP, +}; + +struct hisi_qp *hpre_create_qp(u8 type); +int hpre_algs_register(struct hisi_qm *qm); +void hpre_algs_unregister(struct hisi_qm *qm); +bool hpre_check_alg_support(struct hisi_qm *qm, u32 alg); +#endif diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c new file mode 100644 index 000000000..09077abbf --- /dev/null +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -0,0 +1,1897 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2019 HiSilicon Limited. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "hpre.h" + +struct hpre_ctx; + +#define HPRE_CRYPTO_ALG_PRI 1000 +#define HPRE_ALIGN_SZ 64 +#define HPRE_BITS_2_BYTES_SHIFT 3 +#define HPRE_RSA_512BITS_KSZ 64 +#define HPRE_RSA_1536BITS_KSZ 192 +#define HPRE_CRT_PRMS 5 +#define HPRE_CRT_Q 2 +#define HPRE_CRT_P 3 +#define HPRE_CRT_INV 4 +#define HPRE_DH_G_FLAG 0x02 +#define HPRE_TRY_SEND_TIMES 100 +#define HPRE_INVLD_REQ_ID (-1) + +#define HPRE_SQE_ALG_BITS 5 +#define HPRE_SQE_DONE_SHIFT 30 +#define HPRE_DH_MAX_P_SZ 512 + +#define HPRE_DFX_SEC_TO_US 1000000 +#define HPRE_DFX_US_TO_NS 1000 + +#define HPRE_ENABLE_HPCORE_SHIFT 7 + +/* due to nist p521 */ +#define HPRE_ECC_MAX_KSZ 66 + +/* size in bytes of the n prime */ +#define HPRE_ECC_NIST_P192_N_SIZE 24 +#define HPRE_ECC_NIST_P256_N_SIZE 32 +#define HPRE_ECC_NIST_P384_N_SIZE 48 + +/* size in bytes */ +#define HPRE_ECC_HW256_KSZ_B 32 +#define HPRE_ECC_HW384_KSZ_B 48 + +/* capability register mask of driver */ +#define HPRE_DRV_RSA_MASK_CAP BIT(0) +#define HPRE_DRV_DH_MASK_CAP BIT(1) +#define HPRE_DRV_ECDH_MASK_CAP BIT(2) +#define HPRE_DRV_X25519_MASK_CAP BIT(5) + +static DEFINE_MUTEX(hpre_algs_lock); +static unsigned int hpre_available_devs; + +typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe); + +struct hpre_rsa_ctx { + /* low address: e--->n */ + char *pubkey; + dma_addr_t dma_pubkey; + + /* low address: d--->n */ + char *prikey; + dma_addr_t dma_prikey; + + /* low address: dq->dp->q->p->qinv */ + char *crt_prikey; + dma_addr_t dma_crt_prikey; + + struct crypto_akcipher *soft_tfm; +}; + +struct hpre_dh_ctx { + /* + * If base is g we compute the public key + * ya = g^xa mod p; [RFC2631 sec 2.1.1] + * else if base if the counterpart public key we + * compute the shared secret + * ZZ = yb^xa mod p; [RFC2631 sec 2.1.1] + * low address: d--->n, please refer to Hisilicon HPRE UM + */ + char *xa_p; + dma_addr_t dma_xa_p; + + char *g; /* m */ + dma_addr_t dma_g; + struct crypto_kpp *soft_tfm; +}; + +struct hpre_ecdh_ctx { + /* low address: p->a->k->b */ + unsigned char *p; + dma_addr_t dma_p; + + /* low address: x->y */ + unsigned char *g; + dma_addr_t dma_g; + struct crypto_kpp *soft_tfm; +}; + +struct hpre_ctx { + struct hisi_qp *qp; + struct device *dev; + struct hpre *hpre; + unsigned int key_sz; + bool crt_g2_mode; + union { + struct hpre_rsa_ctx rsa; + struct hpre_dh_ctx dh; + struct hpre_ecdh_ctx ecdh; + }; + /* for ecc algorithms */ + unsigned int curve_id; + /* for high performance core */ + u8 enable_hpcore; + bool fallback; +}; + +struct hpre_asym_request { + char *src; + char *dst; + struct hpre_sqe req; + struct hpre_ctx *ctx; + union { + struct akcipher_request *rsa; + struct kpp_request *dh; + struct kpp_request *ecdh; + } areq; + int err; + hpre_cb cb; + struct timespec64 req_time; +}; + +static inline unsigned int hpre_align_sz(void) +{ + return ((crypto_dma_align() - 1) | (HPRE_ALIGN_SZ - 1)) + 1; +} + +static inline unsigned int hpre_align_pd(void) +{ + return (hpre_align_sz() - 1) & ~(crypto_tfm_ctx_alignment() - 1); +} + +static void hpre_dfx_add_req_time(struct hpre_asym_request *hpre_req) +{ + struct hpre_ctx *ctx = hpre_req->ctx; + struct hpre_dfx *dfx = ctx->hpre->debug.dfx; + + if (atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value)) + ktime_get_ts64(&hpre_req->req_time); +} + +static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req, + struct scatterlist *data, unsigned int len, + int is_src, dma_addr_t *tmp) +{ + struct device *dev = hpre_req->ctx->dev; + enum dma_data_direction dma_dir; + + if (is_src) { + hpre_req->src = NULL; + dma_dir = DMA_TO_DEVICE; + } else { + hpre_req->dst = NULL; + dma_dir = DMA_FROM_DEVICE; + } + *tmp = dma_map_single(dev, sg_virt(data), len, dma_dir); + if (unlikely(dma_mapping_error(dev, *tmp))) { + dev_err(dev, "dma map data err!\n"); + return -ENOMEM; + } + + return 0; +} + +static int hpre_prepare_dma_buf(struct hpre_asym_request *hpre_req, + struct scatterlist *data, unsigned int len, + int is_src, dma_addr_t *tmp) +{ + struct hpre_ctx *ctx = hpre_req->ctx; + struct device *dev = ctx->dev; + void *ptr; + int shift; + + shift = ctx->key_sz - len; + if (unlikely(shift < 0)) + return -EINVAL; + + ptr = dma_alloc_coherent(dev, ctx->key_sz, tmp, GFP_ATOMIC); + if (unlikely(!ptr)) + return -ENOMEM; + + if (is_src) { + scatterwalk_map_and_copy(ptr + shift, data, 0, len, 0); + hpre_req->src = ptr; + } else { + hpre_req->dst = ptr; + } + + return 0; +} + +static int hpre_hw_data_init(struct hpre_asym_request *hpre_req, + struct scatterlist *data, unsigned int len, + int is_src, int is_dh) +{ + struct hpre_sqe *msg = &hpre_req->req; + struct hpre_ctx *ctx = hpre_req->ctx; + dma_addr_t tmp = 0; + int ret; + + /* when the data is dh's source, we should format it */ + if ((sg_is_last(data) && len == ctx->key_sz) && + ((is_dh && !is_src) || !is_dh)) + ret = hpre_get_data_dma_addr(hpre_req, data, len, is_src, &tmp); + else + ret = hpre_prepare_dma_buf(hpre_req, data, len, is_src, &tmp); + + if (unlikely(ret)) + return ret; + + if (is_src) + msg->in = cpu_to_le64(tmp); + else + msg->out = cpu_to_le64(tmp); + + return 0; +} + +static void hpre_hw_data_clr_all(struct hpre_ctx *ctx, + struct hpre_asym_request *req, + struct scatterlist *dst, + struct scatterlist *src) +{ + struct device *dev = ctx->dev; + struct hpre_sqe *sqe = &req->req; + dma_addr_t tmp; + + tmp = le64_to_cpu(sqe->in); + if (unlikely(dma_mapping_error(dev, tmp))) + return; + + if (src) { + if (req->src) + dma_free_coherent(dev, ctx->key_sz, req->src, tmp); + else + dma_unmap_single(dev, tmp, ctx->key_sz, DMA_TO_DEVICE); + } + + tmp = le64_to_cpu(sqe->out); + if (unlikely(dma_mapping_error(dev, tmp))) + return; + + if (req->dst) { + if (dst) + scatterwalk_map_and_copy(req->dst, dst, 0, + ctx->key_sz, 1); + dma_free_coherent(dev, ctx->key_sz, req->dst, tmp); + } else { + dma_unmap_single(dev, tmp, ctx->key_sz, DMA_FROM_DEVICE); + } +} + +static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe, + void **kreq) +{ + unsigned int err, done, alg; + +#define HPRE_NO_HW_ERR 0 +#define HPRE_HW_TASK_DONE 3 +#define HREE_HW_ERR_MASK GENMASK(10, 0) +#define HREE_SQE_DONE_MASK GENMASK(1, 0) +#define HREE_ALG_TYPE_MASK GENMASK(4, 0) + *kreq = (void *)le64_to_cpu(sqe->tag); + + err = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_ALG_BITS) & + HREE_HW_ERR_MASK; + done = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_DONE_SHIFT) & + HREE_SQE_DONE_MASK; + if (likely(err == HPRE_NO_HW_ERR && done == HPRE_HW_TASK_DONE)) + return 0; + + alg = le32_to_cpu(sqe->dw0) & HREE_ALG_TYPE_MASK; + dev_err_ratelimited(ctx->dev, "alg[0x%x] error: done[0x%x], etype[0x%x]\n", + alg, done, err); + + return -EINVAL; +} + +static void hpre_ctx_clear(struct hpre_ctx *ctx, bool is_clear_all) +{ + if (is_clear_all) + hisi_qm_free_qps(&ctx->qp, 1); + + ctx->crt_g2_mode = false; + ctx->key_sz = 0; +} + +static bool hpre_is_bd_timeout(struct hpre_asym_request *req, + u64 overtime_thrhld) +{ + struct timespec64 reply_time; + u64 time_use_us; + + ktime_get_ts64(&reply_time); + time_use_us = (reply_time.tv_sec - req->req_time.tv_sec) * + HPRE_DFX_SEC_TO_US + + (reply_time.tv_nsec - req->req_time.tv_nsec) / + HPRE_DFX_US_TO_NS; + + if (time_use_us <= overtime_thrhld) + return false; + + return true; +} + +static void hpre_dh_cb(struct hpre_ctx *ctx, void *resp) +{ + struct hpre_dfx *dfx = ctx->hpre->debug.dfx; + struct hpre_asym_request *req; + struct kpp_request *areq; + u64 overtime_thrhld; + int ret; + + ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req); + areq = req->areq.dh; + areq->dst_len = ctx->key_sz; + + overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value); + if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) + atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); + + hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src); + kpp_request_complete(areq, ret); + atomic64_inc(&dfx[HPRE_RECV_CNT].value); +} + +static void hpre_rsa_cb(struct hpre_ctx *ctx, void *resp) +{ + struct hpre_dfx *dfx = ctx->hpre->debug.dfx; + struct hpre_asym_request *req; + struct akcipher_request *areq; + u64 overtime_thrhld; + int ret; + + ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req); + + overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value); + if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) + atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); + + areq = req->areq.rsa; + areq->dst_len = ctx->key_sz; + hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src); + akcipher_request_complete(areq, ret); + atomic64_inc(&dfx[HPRE_RECV_CNT].value); +} + +static void hpre_alg_cb(struct hisi_qp *qp, void *resp) +{ + struct hpre_asym_request *h_req; + struct hpre_sqe *sqe = resp; + + h_req = (struct hpre_asym_request *)le64_to_cpu(sqe->tag); + if (unlikely(!h_req)) { + pr_err("Failed to get request, and qp_id is %u\n", qp->qp_id); + return; + } + + h_req->cb(h_req->ctx, resp); +} + +static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type) +{ + struct hisi_qp *qp; + struct hpre *hpre; + + qp = hpre_create_qp(type); + if (!qp) { + ctx->qp = NULL; + return -ENODEV; + } + + qp->req_cb = hpre_alg_cb; + ctx->qp = qp; + ctx->dev = &qp->qm->pdev->dev; + hpre = container_of(ctx->qp->qm, struct hpre, qm); + ctx->hpre = hpre; + ctx->key_sz = 0; + ctx->crt_g2_mode = false; + + return 0; +} + +static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa) +{ + struct hpre_asym_request *h_req; + struct hpre_sqe *msg; + void *tmp; + + if (is_rsa) { + struct akcipher_request *akreq = req; + + if (akreq->dst_len < ctx->key_sz) { + akreq->dst_len = ctx->key_sz; + return -EOVERFLOW; + } + + tmp = akcipher_request_ctx(akreq); + h_req = PTR_ALIGN(tmp, hpre_align_sz()); + h_req->cb = hpre_rsa_cb; + h_req->areq.rsa = akreq; + msg = &h_req->req; + memset(msg, 0, sizeof(*msg)); + } else { + struct kpp_request *kreq = req; + + if (kreq->dst_len < ctx->key_sz) { + kreq->dst_len = ctx->key_sz; + return -EOVERFLOW; + } + + tmp = kpp_request_ctx(kreq); + h_req = PTR_ALIGN(tmp, hpre_align_sz()); + h_req->cb = hpre_dh_cb; + h_req->areq.dh = kreq; + msg = &h_req->req; + memset(msg, 0, sizeof(*msg)); + msg->key = cpu_to_le64(ctx->dh.dma_xa_p); + } + + msg->in = cpu_to_le64(DMA_MAPPING_ERROR); + msg->out = cpu_to_le64(DMA_MAPPING_ERROR); + msg->dw0 |= cpu_to_le32(0x1 << HPRE_SQE_DONE_SHIFT); + msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; + h_req->ctx = ctx; + + hpre_dfx_add_req_time(h_req); + msg->tag = cpu_to_le64((uintptr_t)h_req); + + return 0; +} + +static int hpre_send(struct hpre_ctx *ctx, struct hpre_sqe *msg) +{ + struct hpre_dfx *dfx = ctx->hpre->debug.dfx; + int ctr = 0; + int ret; + + do { + atomic64_inc(&dfx[HPRE_SEND_CNT].value); + ret = hisi_qp_send(ctx->qp, msg); + if (ret != -EBUSY) + break; + atomic64_inc(&dfx[HPRE_SEND_BUSY_CNT].value); + } while (ctr++ < HPRE_TRY_SEND_TIMES); + + if (likely(!ret)) + return ret; + + if (ret != -EBUSY) + atomic64_inc(&dfx[HPRE_SEND_FAIL_CNT].value); + + return ret; +} + +static int hpre_dh_compute_value(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + void *tmp = kpp_request_ctx(req); + struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz()); + struct hpre_sqe *msg = &hpre_req->req; + int ret; + + ret = hpre_msg_request_set(ctx, req, false); + if (unlikely(ret)) + return ret; + + if (req->src) { + ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 1); + if (unlikely(ret)) + goto clear_all; + } else { + msg->in = cpu_to_le64(ctx->dh.dma_g); + } + + ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 1); + if (unlikely(ret)) + goto clear_all; + + if (ctx->crt_g2_mode && !req->src) + msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH_G2); + else + msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH); + + /* success */ + ret = hpre_send(ctx, msg); + if (likely(!ret)) + return -EINPROGRESS; + +clear_all: + hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); + + return ret; +} + +static struct kpp_request *hpre_dh_prepare_fb_req(struct kpp_request *req) +{ + struct kpp_request *fb_req = kpp_request_ctx(req); + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + + kpp_request_set_tfm(fb_req, ctx->dh.soft_tfm); + kpp_request_set_callback(fb_req, req->base.flags, req->base.complete, req->base.data); + kpp_request_set_input(fb_req, req->src, req->src_len); + kpp_request_set_output(fb_req, req->dst, req->dst_len); + + return fb_req; +} + +static int hpre_dh_generate_public_key(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + struct kpp_request *fb_req; + + if (ctx->fallback) { + fb_req = hpre_dh_prepare_fb_req(req); + return crypto_kpp_generate_public_key(fb_req); + } + + return hpre_dh_compute_value(req); +} + +static int hpre_dh_compute_shared_secret(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + struct kpp_request *fb_req; + + if (ctx->fallback) { + fb_req = hpre_dh_prepare_fb_req(req); + return crypto_kpp_compute_shared_secret(fb_req); + } + + return hpre_dh_compute_value(req); +} + +static int hpre_is_dh_params_length_valid(unsigned int key_sz) +{ +#define _HPRE_DH_GRP1 768 +#define _HPRE_DH_GRP2 1024 +#define _HPRE_DH_GRP5 1536 +#define _HPRE_DH_GRP14 2048 +#define _HPRE_DH_GRP15 3072 +#define _HPRE_DH_GRP16 4096 + switch (key_sz) { + case _HPRE_DH_GRP1: + case _HPRE_DH_GRP2: + case _HPRE_DH_GRP5: + case _HPRE_DH_GRP14: + case _HPRE_DH_GRP15: + case _HPRE_DH_GRP16: + return 0; + default: + return -EINVAL; + } +} + +static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params) +{ + struct device *dev = ctx->dev; + unsigned int sz; + + sz = ctx->key_sz = params->p_size; + ctx->dh.xa_p = dma_alloc_coherent(dev, sz << 1, + &ctx->dh.dma_xa_p, GFP_KERNEL); + if (!ctx->dh.xa_p) + return -ENOMEM; + + memcpy(ctx->dh.xa_p + sz, params->p, sz); + + /* If g equals 2 don't copy it */ + if (params->g_size == 1 && *(char *)params->g == HPRE_DH_G_FLAG) { + ctx->crt_g2_mode = true; + return 0; + } + + ctx->dh.g = dma_alloc_coherent(dev, sz, &ctx->dh.dma_g, GFP_KERNEL); + if (!ctx->dh.g) { + dma_free_coherent(dev, sz << 1, ctx->dh.xa_p, + ctx->dh.dma_xa_p); + ctx->dh.xa_p = NULL; + return -ENOMEM; + } + + memcpy(ctx->dh.g + (sz - params->g_size), params->g, params->g_size); + + return 0; +} + +static void hpre_dh_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) +{ + struct device *dev = ctx->dev; + unsigned int sz = ctx->key_sz; + + if (!ctx->qp) + return; + + if (ctx->dh.g) { + dma_free_coherent(dev, sz, ctx->dh.g, ctx->dh.dma_g); + ctx->dh.g = NULL; + } + + if (ctx->dh.xa_p) { + memzero_explicit(ctx->dh.xa_p, sz); + dma_free_coherent(dev, sz << 1, ctx->dh.xa_p, + ctx->dh.dma_xa_p); + ctx->dh.xa_p = NULL; + } + + hpre_ctx_clear(ctx, is_clear_all); +} + +static int hpre_dh_set_secret(struct crypto_kpp *tfm, const void *buf, + unsigned int len) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + struct dh params; + int ret; + + if (crypto_dh_decode_key(buf, len, ¶ms) < 0) + return -EINVAL; + + if (!ctx->qp) + goto set_soft_secret; + + if (hpre_is_dh_params_length_valid(params.p_size << + HPRE_BITS_2_BYTES_SHIFT)) + goto set_soft_secret; + + /* Free old secret if any */ + hpre_dh_clear_ctx(ctx, false); + + ret = hpre_dh_set_params(ctx, ¶ms); + if (ret < 0) + goto err_clear_ctx; + + memcpy(ctx->dh.xa_p + (ctx->key_sz - params.key_size), params.key, + params.key_size); + + ctx->fallback = false; + return 0; + +err_clear_ctx: + hpre_dh_clear_ctx(ctx, false); + return ret; +set_soft_secret: + ctx->fallback = true; + return crypto_kpp_set_secret(ctx->dh.soft_tfm, buf, len); +} + +static unsigned int hpre_dh_max_size(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + + if (ctx->fallback) + return crypto_kpp_maxsize(ctx->dh.soft_tfm); + + return ctx->key_sz; +} + +static int hpre_dh_init_tfm(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + const char *alg = kpp_alg_name(tfm); + unsigned int reqsize; + int ret; + + ctx->dh.soft_tfm = crypto_alloc_kpp(alg, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->dh.soft_tfm)) { + pr_err("Failed to alloc dh tfm!\n"); + return PTR_ERR(ctx->dh.soft_tfm); + } + + crypto_kpp_set_flags(ctx->dh.soft_tfm, crypto_kpp_get_flags(tfm)); + + reqsize = max(sizeof(struct hpre_asym_request) + hpre_align_pd(), + sizeof(struct kpp_request) + crypto_kpp_reqsize(ctx->dh.soft_tfm)); + kpp_set_reqsize(tfm, reqsize); + + ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); + if (ret && ret != -ENODEV) { + crypto_free_kpp(ctx->dh.soft_tfm); + return ret; + } else if (ret == -ENODEV) { + ctx->fallback = true; + } + + return 0; +} + +static void hpre_dh_exit_tfm(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + + hpre_dh_clear_ctx(ctx, true); + crypto_free_kpp(ctx->dh.soft_tfm); +} + +static void hpre_rsa_drop_leading_zeros(const char **ptr, size_t *len) +{ + while (!**ptr && *len) { + (*ptr)++; + (*len)--; + } +} + +static bool hpre_rsa_key_size_is_support(unsigned int len) +{ + unsigned int bits = len << HPRE_BITS_2_BYTES_SHIFT; + +#define _RSA_1024BITS_KEY_WDTH 1024 +#define _RSA_2048BITS_KEY_WDTH 2048 +#define _RSA_3072BITS_KEY_WDTH 3072 +#define _RSA_4096BITS_KEY_WDTH 4096 + + switch (bits) { + case _RSA_1024BITS_KEY_WDTH: + case _RSA_2048BITS_KEY_WDTH: + case _RSA_3072BITS_KEY_WDTH: + case _RSA_4096BITS_KEY_WDTH: + return true; + default: + return false; + } +} + +static int hpre_rsa_enc(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); + void *tmp = akcipher_request_ctx(req); + struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz()); + struct hpre_sqe *msg = &hpre_req->req; + int ret; + + /* For unsupported key size and unavailable devices, use soft tfm instead */ + if (ctx->fallback) { + akcipher_request_set_tfm(req, ctx->rsa.soft_tfm); + ret = crypto_akcipher_encrypt(req); + akcipher_request_set_tfm(req, tfm); + return ret; + } + + if (unlikely(!ctx->rsa.pubkey)) + return -EINVAL; + + ret = hpre_msg_request_set(ctx, req, true); + if (unlikely(ret)) + return ret; + + msg->dw0 |= cpu_to_le32(HPRE_ALG_NC_NCRT); + msg->key = cpu_to_le64(ctx->rsa.dma_pubkey); + + ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0); + if (unlikely(ret)) + goto clear_all; + + ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0); + if (unlikely(ret)) + goto clear_all; + + /* success */ + ret = hpre_send(ctx, msg); + if (likely(!ret)) + return -EINPROGRESS; + +clear_all: + hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); + + return ret; +} + +static int hpre_rsa_dec(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); + void *tmp = akcipher_request_ctx(req); + struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz()); + struct hpre_sqe *msg = &hpre_req->req; + int ret; + + /* For unsupported key size and unavailable devices, use soft tfm instead */ + if (ctx->fallback) { + akcipher_request_set_tfm(req, ctx->rsa.soft_tfm); + ret = crypto_akcipher_decrypt(req); + akcipher_request_set_tfm(req, tfm); + return ret; + } + + if (unlikely(!ctx->rsa.prikey)) + return -EINVAL; + + ret = hpre_msg_request_set(ctx, req, true); + if (unlikely(ret)) + return ret; + + if (ctx->crt_g2_mode) { + msg->key = cpu_to_le64(ctx->rsa.dma_crt_prikey); + msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | + HPRE_ALG_NC_CRT); + } else { + msg->key = cpu_to_le64(ctx->rsa.dma_prikey); + msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | + HPRE_ALG_NC_NCRT); + } + + ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0); + if (unlikely(ret)) + goto clear_all; + + ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0); + if (unlikely(ret)) + goto clear_all; + + /* success */ + ret = hpre_send(ctx, msg); + if (likely(!ret)) + return -EINPROGRESS; + +clear_all: + hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); + + return ret; +} + +static int hpre_rsa_set_n(struct hpre_ctx *ctx, const char *value, + size_t vlen, bool private) +{ + const char *ptr = value; + + hpre_rsa_drop_leading_zeros(&ptr, &vlen); + + ctx->key_sz = vlen; + + /* if invalid key size provided, we use software tfm */ + if (!hpre_rsa_key_size_is_support(ctx->key_sz)) { + ctx->fallback = true; + return 0; + } + + ctx->rsa.pubkey = dma_alloc_coherent(ctx->dev, vlen << 1, + &ctx->rsa.dma_pubkey, + GFP_KERNEL); + if (!ctx->rsa.pubkey) + return -ENOMEM; + + if (private) { + ctx->rsa.prikey = dma_alloc_coherent(ctx->dev, vlen << 1, + &ctx->rsa.dma_prikey, + GFP_KERNEL); + if (!ctx->rsa.prikey) { + dma_free_coherent(ctx->dev, vlen << 1, + ctx->rsa.pubkey, + ctx->rsa.dma_pubkey); + ctx->rsa.pubkey = NULL; + return -ENOMEM; + } + memcpy(ctx->rsa.prikey + vlen, ptr, vlen); + } + memcpy(ctx->rsa.pubkey + vlen, ptr, vlen); + + /* Using hardware HPRE to do RSA */ + return 1; +} + +static int hpre_rsa_set_e(struct hpre_ctx *ctx, const char *value, + size_t vlen) +{ + const char *ptr = value; + + hpre_rsa_drop_leading_zeros(&ptr, &vlen); + + if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) + return -EINVAL; + + memcpy(ctx->rsa.pubkey + ctx->key_sz - vlen, ptr, vlen); + + return 0; +} + +static int hpre_rsa_set_d(struct hpre_ctx *ctx, const char *value, + size_t vlen) +{ + const char *ptr = value; + + hpre_rsa_drop_leading_zeros(&ptr, &vlen); + + if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) + return -EINVAL; + + memcpy(ctx->rsa.prikey + ctx->key_sz - vlen, ptr, vlen); + + return 0; +} + +static int hpre_crt_para_get(char *para, size_t para_sz, + const char *raw, size_t raw_sz) +{ + const char *ptr = raw; + size_t len = raw_sz; + + hpre_rsa_drop_leading_zeros(&ptr, &len); + if (!len || len > para_sz) + return -EINVAL; + + memcpy(para + para_sz - len, ptr, len); + + return 0; +} + +static int hpre_rsa_setkey_crt(struct hpre_ctx *ctx, struct rsa_key *rsa_key) +{ + unsigned int hlf_ksz = ctx->key_sz >> 1; + struct device *dev = ctx->dev; + u64 offset; + int ret; + + ctx->rsa.crt_prikey = dma_alloc_coherent(dev, hlf_ksz * HPRE_CRT_PRMS, + &ctx->rsa.dma_crt_prikey, + GFP_KERNEL); + if (!ctx->rsa.crt_prikey) + return -ENOMEM; + + ret = hpre_crt_para_get(ctx->rsa.crt_prikey, hlf_ksz, + rsa_key->dq, rsa_key->dq_sz); + if (ret) + goto free_key; + + offset = hlf_ksz; + ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz, + rsa_key->dp, rsa_key->dp_sz); + if (ret) + goto free_key; + + offset = hlf_ksz * HPRE_CRT_Q; + ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz, + rsa_key->q, rsa_key->q_sz); + if (ret) + goto free_key; + + offset = hlf_ksz * HPRE_CRT_P; + ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz, + rsa_key->p, rsa_key->p_sz); + if (ret) + goto free_key; + + offset = hlf_ksz * HPRE_CRT_INV; + ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz, + rsa_key->qinv, rsa_key->qinv_sz); + if (ret) + goto free_key; + + ctx->crt_g2_mode = true; + + return 0; + +free_key: + offset = hlf_ksz * HPRE_CRT_PRMS; + memzero_explicit(ctx->rsa.crt_prikey, offset); + dma_free_coherent(dev, hlf_ksz * HPRE_CRT_PRMS, ctx->rsa.crt_prikey, + ctx->rsa.dma_crt_prikey); + ctx->rsa.crt_prikey = NULL; + ctx->crt_g2_mode = false; + + return ret; +} + +/* If it is clear all, all the resources of the QP will be cleaned. */ +static void hpre_rsa_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) +{ + unsigned int half_key_sz = ctx->key_sz >> 1; + struct device *dev = ctx->dev; + + if (!ctx->qp) + return; + + if (ctx->rsa.pubkey) { + dma_free_coherent(dev, ctx->key_sz << 1, + ctx->rsa.pubkey, ctx->rsa.dma_pubkey); + ctx->rsa.pubkey = NULL; + } + + if (ctx->rsa.crt_prikey) { + memzero_explicit(ctx->rsa.crt_prikey, + half_key_sz * HPRE_CRT_PRMS); + dma_free_coherent(dev, half_key_sz * HPRE_CRT_PRMS, + ctx->rsa.crt_prikey, ctx->rsa.dma_crt_prikey); + ctx->rsa.crt_prikey = NULL; + } + + if (ctx->rsa.prikey) { + memzero_explicit(ctx->rsa.prikey, ctx->key_sz); + dma_free_coherent(dev, ctx->key_sz << 1, ctx->rsa.prikey, + ctx->rsa.dma_prikey); + ctx->rsa.prikey = NULL; + } + + hpre_ctx_clear(ctx, is_clear_all); +} + +/* + * we should judge if it is CRT or not, + * CRT: return true, N-CRT: return false . + */ +static bool hpre_is_crt_key(struct rsa_key *key) +{ + u16 len = key->p_sz + key->q_sz + key->dp_sz + key->dq_sz + + key->qinv_sz; + +#define LEN_OF_NCRT_PARA 5 + + /* N-CRT less than 5 parameters */ + return len > LEN_OF_NCRT_PARA; +} + +static int hpre_rsa_setkey(struct hpre_ctx *ctx, const void *key, + unsigned int keylen, bool private) +{ + struct rsa_key rsa_key; + int ret; + + hpre_rsa_clear_ctx(ctx, false); + + if (private) + ret = rsa_parse_priv_key(&rsa_key, key, keylen); + else + ret = rsa_parse_pub_key(&rsa_key, key, keylen); + if (ret < 0) + return ret; + + ret = hpre_rsa_set_n(ctx, rsa_key.n, rsa_key.n_sz, private); + if (ret <= 0) + return ret; + + if (private) { + ret = hpre_rsa_set_d(ctx, rsa_key.d, rsa_key.d_sz); + if (ret < 0) + goto free; + + if (hpre_is_crt_key(&rsa_key)) { + ret = hpre_rsa_setkey_crt(ctx, &rsa_key); + if (ret < 0) + goto free; + } + } + + ret = hpre_rsa_set_e(ctx, rsa_key.e, rsa_key.e_sz); + if (ret < 0) + goto free; + + if ((private && !ctx->rsa.prikey) || !ctx->rsa.pubkey) { + ret = -EINVAL; + goto free; + } + + ctx->fallback = false; + return 0; + +free: + hpre_rsa_clear_ctx(ctx, false); + return ret; +} + +static int hpre_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); + int ret; + + ret = crypto_akcipher_set_pub_key(ctx->rsa.soft_tfm, key, keylen); + if (ret) + return ret; + + if (!ctx->qp) + return 0; + + return hpre_rsa_setkey(ctx, key, keylen, false); +} + +static int hpre_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); + int ret; + + ret = crypto_akcipher_set_priv_key(ctx->rsa.soft_tfm, key, keylen); + if (ret) + return ret; + + if (!ctx->qp) + return 0; + + return hpre_rsa_setkey(ctx, key, keylen, true); +} + +static unsigned int hpre_rsa_max_size(struct crypto_akcipher *tfm) +{ + struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); + + /* For unsupported key size and unavailable devices, use soft tfm instead */ + if (ctx->fallback) + return crypto_akcipher_maxsize(ctx->rsa.soft_tfm); + + return ctx->key_sz; +} + +static int hpre_rsa_init_tfm(struct crypto_akcipher *tfm) +{ + struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); + int ret; + + ctx->rsa.soft_tfm = crypto_alloc_akcipher("rsa-generic", 0, 0); + if (IS_ERR(ctx->rsa.soft_tfm)) { + pr_err("Can not alloc_akcipher!\n"); + return PTR_ERR(ctx->rsa.soft_tfm); + } + + akcipher_set_reqsize(tfm, sizeof(struct hpre_asym_request) + + hpre_align_pd()); + + ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); + if (ret && ret != -ENODEV) { + crypto_free_akcipher(ctx->rsa.soft_tfm); + return ret; + } else if (ret == -ENODEV) { + ctx->fallback = true; + } + + return 0; +} + +static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm) +{ + struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); + + hpre_rsa_clear_ctx(ctx, true); + crypto_free_akcipher(ctx->rsa.soft_tfm); +} + +static void hpre_key_to_big_end(u8 *data, int len) +{ + int i, j; + + for (i = 0; i < len / 2; i++) { + j = len - i - 1; + swap(data[j], data[i]); + } +} + +static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) +{ + struct device *dev = ctx->dev; + unsigned int sz = ctx->key_sz; + unsigned int shift = sz << 1; + + if (ctx->ecdh.p) { + /* ecdh: p->a->k->b */ + memzero_explicit(ctx->ecdh.p + shift, sz); + dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p); + ctx->ecdh.p = NULL; + } + + hpre_ctx_clear(ctx, is_clear_all); +} + +/* + * The bits of 192/224/256/384/521 are supported by HPRE, + * and convert the bits like: + * bits<=256, bits=256; 256key_sz << 1; + unsigned int shiftb = ctx->key_sz << 2; + void *p = ctx->ecdh.p + ctx->key_sz - cur_sz; + void *a = ctx->ecdh.p + shifta - cur_sz; + void *b = ctx->ecdh.p + shiftb - cur_sz; + void *x = ctx->ecdh.g + ctx->key_sz - cur_sz; + void *y = ctx->ecdh.g + shifta - cur_sz; + const struct ecc_curve *curve = ecc_get_curve(ctx->curve_id); + char *n; + + if (unlikely(!curve)) + return -EINVAL; + + n = kzalloc(ctx->key_sz, GFP_KERNEL); + if (!n) + return -ENOMEM; + + fill_curve_param(p, curve->p, cur_sz, curve->g.ndigits); + fill_curve_param(a, curve->a, cur_sz, curve->g.ndigits); + fill_curve_param(b, curve->b, cur_sz, curve->g.ndigits); + fill_curve_param(x, curve->g.x, cur_sz, curve->g.ndigits); + fill_curve_param(y, curve->g.y, cur_sz, curve->g.ndigits); + fill_curve_param(n, curve->n, cur_sz, curve->g.ndigits); + + if (params->key_size == cur_sz && memcmp(params->key, n, cur_sz) >= 0) { + kfree(n); + return -EINVAL; + } + + kfree(n); + return 0; +} + +static unsigned int hpre_ecdh_get_curvesz(unsigned short id) +{ + switch (id) { + case ECC_CURVE_NIST_P192: + return HPRE_ECC_NIST_P192_N_SIZE; + case ECC_CURVE_NIST_P256: + return HPRE_ECC_NIST_P256_N_SIZE; + case ECC_CURVE_NIST_P384: + return HPRE_ECC_NIST_P384_N_SIZE; + default: + break; + } + + return 0; +} + +static int hpre_ecdh_set_param(struct hpre_ctx *ctx, struct ecdh *params) +{ + struct device *dev = ctx->dev; + unsigned int sz, shift, curve_sz; + int ret; + + ctx->key_sz = hpre_ecdh_supported_curve(ctx->curve_id); + if (!ctx->key_sz) + return -EINVAL; + + curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id); + if (!curve_sz || params->key_size > curve_sz) + return -EINVAL; + + sz = ctx->key_sz; + + if (!ctx->ecdh.p) { + ctx->ecdh.p = dma_alloc_coherent(dev, sz << 3, &ctx->ecdh.dma_p, + GFP_KERNEL); + if (!ctx->ecdh.p) + return -ENOMEM; + } + + shift = sz << 2; + ctx->ecdh.g = ctx->ecdh.p + shift; + ctx->ecdh.dma_g = ctx->ecdh.dma_p + shift; + + ret = hpre_ecdh_fill_curve(ctx, params, curve_sz); + if (ret) { + dev_err(dev, "failed to fill curve_param, ret = %d!\n", ret); + dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p); + ctx->ecdh.p = NULL; + return ret; + } + + return 0; +} + +static bool hpre_key_is_zero(const char *key, unsigned short key_sz) +{ + int i; + + for (i = 0; i < key_sz; i++) + if (key[i]) + return false; + + return true; +} + +static int ecdh_gen_privkey(struct hpre_ctx *ctx, struct ecdh *params) +{ + struct device *dev = ctx->dev; + int ret; + + ret = crypto_stdrng_get_bytes(params->key, params->key_size); + if (ret) + dev_err(dev, "failed to get random bytes, ret = %d!\n", ret); + + return ret; +} + +static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, + unsigned int len) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + unsigned int sz, sz_shift, curve_sz; + struct device *dev = ctx->dev; + char key[HPRE_ECC_MAX_KSZ]; + struct ecdh params; + int ret; + + if (ctx->fallback) + return crypto_kpp_set_secret(ctx->ecdh.soft_tfm, buf, len); + + if (crypto_ecdh_decode_key(buf, len, ¶ms) < 0) { + dev_err(dev, "failed to decode ecdh key!\n"); + return -EINVAL; + } + + /* Use stdrng to generate private key */ + if (!params.key || !params.key_size) { + params.key = key; + curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id); + if (!curve_sz) { + dev_err(dev, "Invalid curve size!\n"); + return -EINVAL; + } + + params.key_size = curve_sz - 1; + ret = ecdh_gen_privkey(ctx, ¶ms); + if (ret) + return ret; + } + + if (hpre_key_is_zero(params.key, params.key_size)) { + dev_err(dev, "Invalid hpre key!\n"); + return -EINVAL; + } + + hpre_ecc_clear_ctx(ctx, false); + + ret = hpre_ecdh_set_param(ctx, ¶ms); + if (ret < 0) { + dev_err(dev, "failed to set hpre param, ret = %d!\n", ret); + return ret; + } + + sz = ctx->key_sz; + sz_shift = (sz << 1) + sz - params.key_size; + memcpy(ctx->ecdh.p + sz_shift, params.key, params.key_size); + + return 0; +} + +static void hpre_ecdh_hw_data_clr_all(struct hpre_ctx *ctx, + struct hpre_asym_request *req, + struct scatterlist *dst, + struct scatterlist *src) +{ + struct device *dev = ctx->dev; + struct hpre_sqe *sqe = &req->req; + dma_addr_t dma; + + dma = le64_to_cpu(sqe->in); + if (unlikely(dma_mapping_error(dev, dma))) + return; + + if (src && req->src) + dma_free_coherent(dev, ctx->key_sz << 2, req->src, dma); + + dma = le64_to_cpu(sqe->out); + if (unlikely(dma_mapping_error(dev, dma))) + return; + + if (req->dst) + dma_free_coherent(dev, ctx->key_sz << 1, req->dst, dma); + if (dst) + dma_unmap_single(dev, dma, ctx->key_sz << 1, DMA_FROM_DEVICE); +} + +static void hpre_ecdh_cb(struct hpre_ctx *ctx, void *resp) +{ + unsigned int curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id); + struct hpre_dfx *dfx = ctx->hpre->debug.dfx; + struct hpre_asym_request *req = NULL; + struct kpp_request *areq; + u64 overtime_thrhld; + char *p; + int ret; + + ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req); + areq = req->areq.ecdh; + areq->dst_len = ctx->key_sz << 1; + + overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value); + if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) + atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); + + /* Do unmap before data processing */ + hpre_ecdh_hw_data_clr_all(ctx, req, areq->dst, areq->src); + + p = sg_virt(areq->dst); + memmove(p, p + ctx->key_sz - curve_sz, curve_sz); + memmove(p + curve_sz, p + areq->dst_len - curve_sz, curve_sz); + + kpp_request_complete(areq, ret); + + atomic64_inc(&dfx[HPRE_RECV_CNT].value); +} + +static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx, + struct kpp_request *req) +{ + struct hpre_asym_request *h_req; + struct hpre_sqe *msg; + void *tmp; + + if (req->dst_len < ctx->key_sz << 1) { + req->dst_len = ctx->key_sz << 1; + return -EINVAL; + } + + tmp = kpp_request_ctx(req); + h_req = PTR_ALIGN(tmp, hpre_align_sz()); + h_req->cb = hpre_ecdh_cb; + h_req->areq.ecdh = req; + msg = &h_req->req; + memset(msg, 0, sizeof(*msg)); + msg->in = cpu_to_le64(DMA_MAPPING_ERROR); + msg->out = cpu_to_le64(DMA_MAPPING_ERROR); + msg->key = cpu_to_le64(ctx->ecdh.dma_p); + + msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT); + msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; + h_req->ctx = ctx; + + hpre_dfx_add_req_time(h_req); + msg->tag = cpu_to_le64((uintptr_t)h_req); + return 0; +} + +static int hpre_ecdh_src_data_init(struct hpre_asym_request *hpre_req, + struct scatterlist *data, unsigned int len) +{ + struct hpre_sqe *msg = &hpre_req->req; + struct hpre_ctx *ctx = hpre_req->ctx; + struct device *dev = ctx->dev; + unsigned int tmpshift; + dma_addr_t dma = 0; + void *ptr; + int shift; + + /* Src_data include gx and gy. */ + shift = ctx->key_sz - (len >> 1); + if (unlikely(shift < 0)) + return -EINVAL; + + ptr = dma_alloc_coherent(dev, ctx->key_sz << 2, &dma, GFP_KERNEL); + if (unlikely(!ptr)) + return -ENOMEM; + + tmpshift = ctx->key_sz << 1; + scatterwalk_map_and_copy(ptr + tmpshift, data, 0, len, 0); + memcpy(ptr + shift, ptr + tmpshift, len >> 1); + memcpy(ptr + ctx->key_sz + shift, ptr + tmpshift + (len >> 1), len >> 1); + + hpre_req->src = ptr; + msg->in = cpu_to_le64(dma); + return 0; +} + +static int hpre_ecdh_dst_data_init(struct hpre_asym_request *hpre_req, + struct scatterlist *data, unsigned int len) +{ + struct hpre_sqe *msg = &hpre_req->req; + struct hpre_ctx *ctx = hpre_req->ctx; + struct device *dev = ctx->dev; + dma_addr_t dma; + + if (unlikely(!data || !sg_is_last(data) || len != ctx->key_sz << 1)) { + dev_err(dev, "data or data length is illegal!\n"); + return -EINVAL; + } + + hpre_req->dst = NULL; + dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE); + if (unlikely(dma_mapping_error(dev, dma))) { + dev_err(dev, "dma map data err!\n"); + return -ENOMEM; + } + + msg->out = cpu_to_le64(dma); + return 0; +} + +static int hpre_ecdh_compute_value(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + struct device *dev = ctx->dev; + void *tmp = kpp_request_ctx(req); + struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz()); + struct hpre_sqe *msg = &hpre_req->req; + int ret; + + ret = hpre_ecdh_msg_request_set(ctx, req); + if (unlikely(ret)) { + dev_err(dev, "failed to set ecdh request, ret = %d!\n", ret); + return ret; + } + + if (req->src) { + ret = hpre_ecdh_src_data_init(hpre_req, req->src, req->src_len); + if (unlikely(ret)) { + dev_err(dev, "failed to init src data, ret = %d!\n", ret); + goto clear_all; + } + } else { + msg->in = cpu_to_le64(ctx->ecdh.dma_g); + } + + ret = hpre_ecdh_dst_data_init(hpre_req, req->dst, req->dst_len); + if (unlikely(ret)) { + dev_err(dev, "failed to init dst data, ret = %d!\n", ret); + goto clear_all; + } + + msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_ECC_MUL); + msg->resv1 = ctx->enable_hpcore << HPRE_ENABLE_HPCORE_SHIFT; + + ret = hpre_send(ctx, msg); + if (likely(!ret)) + return -EINPROGRESS; + +clear_all: + hpre_ecdh_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); + return ret; +} + +static int hpre_ecdh_generate_public_key(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + int ret; + + if (ctx->fallback) { + kpp_request_set_tfm(req, ctx->ecdh.soft_tfm); + ret = crypto_kpp_generate_public_key(req); + kpp_request_set_tfm(req, tfm); + return ret; + } + + return hpre_ecdh_compute_value(req); +} + +static int hpre_ecdh_compute_shared_secret(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + int ret; + + if (ctx->fallback) { + kpp_request_set_tfm(req, ctx->ecdh.soft_tfm); + ret = crypto_kpp_compute_shared_secret(req); + kpp_request_set_tfm(req, tfm); + return ret; + } + + return hpre_ecdh_compute_value(req); +} + +static unsigned int hpre_ecdh_max_size(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + + if (ctx->fallback) + return crypto_kpp_maxsize(ctx->ecdh.soft_tfm); + + /* max size is the pub_key_size, include x and y */ + return ctx->key_sz << 1; +} + +static int hpre_ecdh_init_tfm(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + const char *alg = kpp_alg_name(tfm); + int ret; + + ret = hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); + if (!ret) { + kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); + return 0; + } else if (ret && ret != -ENODEV) { + return ret; + } + + ctx->ecdh.soft_tfm = crypto_alloc_kpp(alg, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->ecdh.soft_tfm)) { + pr_err("Failed to alloc %s tfm!\n", alg); + return PTR_ERR(ctx->ecdh.soft_tfm); + } + + crypto_kpp_set_flags(ctx->ecdh.soft_tfm, crypto_kpp_get_flags(tfm)); + ctx->fallback = true; + + return 0; +} + +static int hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + + ctx->curve_id = ECC_CURVE_NIST_P192; + + return hpre_ecdh_init_tfm(tfm); +} + +static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + + ctx->curve_id = ECC_CURVE_NIST_P256; + ctx->enable_hpcore = 1; + + return hpre_ecdh_init_tfm(tfm); +} + +static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + + ctx->curve_id = ECC_CURVE_NIST_P384; + + return hpre_ecdh_init_tfm(tfm); +} + +static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + + if (ctx->fallback) { + crypto_free_kpp(ctx->ecdh.soft_tfm); + return; + } + + hpre_ecc_clear_ctx(ctx, true); +} + +static struct akcipher_alg rsa = { + .encrypt = hpre_rsa_enc, + .decrypt = hpre_rsa_dec, + .set_pub_key = hpre_rsa_setpubkey, + .set_priv_key = hpre_rsa_setprivkey, + .max_size = hpre_rsa_max_size, + .init = hpre_rsa_init_tfm, + .exit = hpre_rsa_exit_tfm, + .base = { + .cra_ctxsize = sizeof(struct hpre_ctx), + .cra_priority = HPRE_CRYPTO_ALG_PRI, + .cra_name = "rsa", + .cra_driver_name = "hpre-rsa", + .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, + }, +}; + +static struct kpp_alg dh = { + .set_secret = hpre_dh_set_secret, + .generate_public_key = hpre_dh_generate_public_key, + .compute_shared_secret = hpre_dh_compute_shared_secret, + .max_size = hpre_dh_max_size, + .init = hpre_dh_init_tfm, + .exit = hpre_dh_exit_tfm, + .base = { + .cra_ctxsize = sizeof(struct hpre_ctx), + .cra_priority = HPRE_CRYPTO_ALG_PRI, + .cra_name = "dh", + .cra_driver_name = "hpre-dh", + .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, + }, +}; + +static struct kpp_alg ecdh_curves[] = { + { + .set_secret = hpre_ecdh_set_secret, + .generate_public_key = hpre_ecdh_generate_public_key, + .compute_shared_secret = hpre_ecdh_compute_shared_secret, + .max_size = hpre_ecdh_max_size, + .init = hpre_ecdh_nist_p192_init_tfm, + .exit = hpre_ecdh_exit_tfm, + .base = { + .cra_ctxsize = sizeof(struct hpre_ctx), + .cra_priority = HPRE_CRYPTO_ALG_PRI, + .cra_name = "ecdh-nist-p192", + .cra_driver_name = "hpre-ecdh-nist-p192", + .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, + }, + }, { + .set_secret = hpre_ecdh_set_secret, + .generate_public_key = hpre_ecdh_generate_public_key, + .compute_shared_secret = hpre_ecdh_compute_shared_secret, + .max_size = hpre_ecdh_max_size, + .init = hpre_ecdh_nist_p256_init_tfm, + .exit = hpre_ecdh_exit_tfm, + .base = { + .cra_ctxsize = sizeof(struct hpre_ctx), + .cra_priority = HPRE_CRYPTO_ALG_PRI, + .cra_name = "ecdh-nist-p256", + .cra_driver_name = "hpre-ecdh-nist-p256", + .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, + }, + }, { + .set_secret = hpre_ecdh_set_secret, + .generate_public_key = hpre_ecdh_generate_public_key, + .compute_shared_secret = hpre_ecdh_compute_shared_secret, + .max_size = hpre_ecdh_max_size, + .init = hpre_ecdh_nist_p384_init_tfm, + .exit = hpre_ecdh_exit_tfm, + .base = { + .cra_ctxsize = sizeof(struct hpre_ctx), + .cra_priority = HPRE_CRYPTO_ALG_PRI, + .cra_name = "ecdh-nist-p384", + .cra_driver_name = "hpre-ecdh-nist-p384", + .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, + }, + } +}; + +static int hpre_register_rsa(struct hisi_qm *qm) +{ + int ret; + + if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP)) + return 0; + + rsa.base.cra_flags = 0; + ret = crypto_register_akcipher(&rsa); + if (ret) + dev_err(&qm->pdev->dev, "failed to register rsa (%d)!\n", ret); + + return ret; +} + +static void hpre_unregister_rsa(struct hisi_qm *qm) +{ + if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP)) + return; + + crypto_unregister_akcipher(&rsa); +} + +static int hpre_register_dh(struct hisi_qm *qm) +{ + int ret; + + if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP)) + return 0; + + ret = crypto_register_kpp(&dh); + if (ret) + dev_err(&qm->pdev->dev, "failed to register dh (%d)!\n", ret); + + return ret; +} + +static void hpre_unregister_dh(struct hisi_qm *qm) +{ + if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP)) + return; + + crypto_unregister_kpp(&dh); +} + +static int hpre_register_ecdh(struct hisi_qm *qm) +{ + int ret, i; + + if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP)) + return 0; + + for (i = 0; i < ARRAY_SIZE(ecdh_curves); i++) { + ret = crypto_register_kpp(&ecdh_curves[i]); + if (ret) { + dev_err(&qm->pdev->dev, "failed to register %s (%d)!\n", + ecdh_curves[i].base.cra_name, ret); + goto unreg_kpp; + } + } + + return 0; + +unreg_kpp: + for (--i; i >= 0; --i) + crypto_unregister_kpp(&ecdh_curves[i]); + + return ret; +} + +static void hpre_unregister_ecdh(struct hisi_qm *qm) +{ + int i; + + if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP)) + return; + + for (i = ARRAY_SIZE(ecdh_curves) - 1; i >= 0; --i) + crypto_unregister_kpp(&ecdh_curves[i]); +} + +int hpre_algs_register(struct hisi_qm *qm) +{ + int ret = 0; + + mutex_lock(&hpre_algs_lock); + if (hpre_available_devs) { + hpre_available_devs++; + goto unlock; + } + + ret = hpre_register_rsa(qm); + if (ret) + goto unlock; + + ret = hpre_register_dh(qm); + if (ret) + goto unreg_rsa; + + ret = hpre_register_ecdh(qm); + if (ret) + goto unreg_dh; + + hpre_available_devs++; + mutex_unlock(&hpre_algs_lock); + + return ret; + +unreg_dh: + hpre_unregister_dh(qm); +unreg_rsa: + hpre_unregister_rsa(qm); +unlock: + mutex_unlock(&hpre_algs_lock); + return ret; +} + +void hpre_algs_unregister(struct hisi_qm *qm) +{ + mutex_lock(&hpre_algs_lock); + if (--hpre_available_devs) + goto unlock; + + hpre_unregister_ecdh(qm); + hpre_unregister_dh(qm); + hpre_unregister_rsa(qm); + +unlock: + mutex_unlock(&hpre_algs_lock); +} diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c new file mode 100644 index 000000000..b6903fce6 --- /dev/null +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -0,0 +1,1754 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2018-2019 HiSilicon Limited. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "hpre.h" + +#define CAP_FILE_PERMISSION 0444 +#define HPRE_CTRL_CNT_CLR_CE_BIT BIT(0) +#define HPRE_CTRL_CNT_CLR_CE 0x301000 +#define HPRE_FSM_MAX_CNT 0x301008 +#define HPRE_VFG_AXQOS 0x30100c +#define HPRE_VFG_AXCACHE 0x301010 +#define HPRE_RDCHN_INI_CFG 0x301014 +#define HPRE_AWUSR_FP_CFG 0x301018 +#define HPRE_BD_ENDIAN 0x301020 +#define HPRE_ECC_BYPASS 0x301024 +#define HPRE_RAS_WIDTH_CFG 0x301028 +#define HPRE_POISON_BYPASS 0x30102c +#define HPRE_BD_ARUSR_CFG 0x301030 +#define HPRE_BD_AWUSR_CFG 0x301034 +#define HPRE_TYPES_ENB 0x301038 +#define HPRE_RSA_ENB BIT(0) +#define HPRE_ECC_ENB BIT(1) +#define HPRE_DATA_RUSER_CFG 0x30103c +#define HPRE_DATA_WUSER_CFG 0x301040 +#define HPRE_INT_MASK 0x301400 +#define HPRE_INT_STATUS 0x301800 +#define HPRE_HAC_INT_MSK 0x301400 +#define HPRE_HAC_RAS_CE_ENB 0x301410 +#define HPRE_HAC_RAS_NFE_ENB 0x301414 +#define HPRE_HAC_RAS_FE_ENB 0x301418 +#define HPRE_HAC_INT_SET 0x301500 +#define HPRE_AXI_ERROR_MASK GENMASK(21, 10) +#define HPRE_RNG_TIMEOUT_NUM 0x301A34 +#define HPRE_CORE_INT_ENABLE 0 +#define HPRE_RDCHN_INI_ST 0x301a00 +#define HPRE_CLSTR_BASE 0x302000 +#define HPRE_CORE_EN_OFFSET 0x04 +#define HPRE_CORE_INI_CFG_OFFSET 0x20 +#define HPRE_CORE_INI_STATUS_OFFSET 0x80 +#define HPRE_CORE_HTBT_WARN_OFFSET 0x8c +#define HPRE_CORE_IS_SCHD_OFFSET 0x90 + +#define HPRE_RAS_CE_ENB 0x301410 +#define HPRE_RAS_NFE_ENB 0x301414 +#define HPRE_RAS_FE_ENB 0x301418 +#define HPRE_OOO_SHUTDOWN_SEL 0x301a3c +#define HPRE_HAC_RAS_FE_ENABLE 0 +#define HPRE_RAS_MASK_ALL GENMASK(31, 0) +#define HPRE_RAS_CLEAR_ALL GENMASK(31, 0) + +#define HPRE_CORE_ENB (HPRE_CLSTR_BASE + HPRE_CORE_EN_OFFSET) +#define HPRE_CORE_INI_CFG (HPRE_CLSTR_BASE + HPRE_CORE_INI_CFG_OFFSET) +#define HPRE_CORE_INI_STATUS (HPRE_CLSTR_BASE + HPRE_CORE_INI_STATUS_OFFSET) +#define HPRE_HAC_ECC1_CNT 0x301a04 +#define HPRE_HAC_ECC2_CNT 0x301a08 +#define HPRE_HAC_SOURCE_INT 0x301600 +#define HPRE_CLSTR_ADDR_INTRVL 0x1000 +#define HPRE_CLUSTER_INQURY 0x100 +#define HPRE_CLSTR_ADDR_INQRY_RSLT 0x104 +#define HPRE_PASID_EN_BIT 9 +#define HPRE_REG_RD_INTVRL_US 10 +#define HPRE_REG_RD_TMOUT_US 1000 +#define HPRE_DBGFS_VAL_MAX_LEN 20 +#define PCI_DEVICE_ID_HUAWEI_HPRE_PF 0xa258 +#define HPRE_QM_USR_CFG_MASK GENMASK(31, 1) +#define HPRE_QM_AXI_CFG_MASK GENMASK(15, 0) +#define HPRE_QM_VFG_AX_MASK GENMASK(7, 0) +#define HPRE_BD_USR_MASK GENMASK(1, 0) +#define HPRE_PREFETCH_CFG 0x301130 +#define HPRE_SVA_PREFTCH_DFX 0x30115C +#define HPRE_PREFETCH_ENABLE (~(BIT(0) | BIT(30))) +#define HPRE_PREFETCH_DISABLE BIT(30) +#define HPRE_SVA_DISABLE_READY (BIT(4) | BIT(8)) +#define HPRE_SVA_PREFTCH_DFX4 0x301144 +#define HPRE_WAIT_SVA_READY 500000 +#define HPRE_READ_SVA_STATUS_TIMES 3 +#define HPRE_WAIT_US_MIN 10 +#define HPRE_WAIT_US_MAX 20 + +/* clock gate */ +#define HPRE_CLKGATE_CTL 0x301a10 +#define HPRE_PEH_CFG_AUTO_GATE 0x301a2c +#define HPRE_CLUSTER_DYN_CTL 0x302010 +#define HPRE_CORE_SHB_CFG 0x302088 +#define HPRE_CLKGATE_CTL_EN BIT(0) +#define HPRE_PEH_CFG_AUTO_GATE_EN BIT(0) +#define HPRE_CLUSTER_DYN_CTL_EN BIT(0) +#define HPRE_CORE_GATE_EN (BIT(30) | BIT(31)) + +#define HPRE_AM_OOO_SHUTDOWN_ENB 0x301044 +#define HPRE_AM_OOO_SHUTDOWN_ENABLE BIT(0) +#define HPRE_WR_MSI_PORT BIT(2) + +#define HPRE_CORE_ECC_2BIT_ERR BIT(1) +#define HPRE_OOO_ECC_2BIT_ERR BIT(5) + +#define HPRE_QM_BME_FLR BIT(7) +#define HPRE_QM_PM_FLR BIT(11) +#define HPRE_QM_SRIOV_FLR BIT(12) + +#define HPRE_SHAPER_TYPE_RATE 640 +#define HPRE_VIA_MSI_DSM 1 +#define HPRE_SQE_MASK_OFFSET 8 +#define HPRE_SQE_MASK_LEN 44 +#define HPRE_CTX_Q_NUM_DEF 1 + +#define HPRE_DFX_BASE 0x301000 +#define HPRE_DFX_COMMON1 0x301400 +#define HPRE_DFX_COMMON2 0x301A00 +#define HPRE_DFX_CORE 0x302000 +#define HPRE_DFX_BASE_LEN 0x55 +#define HPRE_DFX_COMMON1_LEN 0x41 +#define HPRE_DFX_COMMON2_LEN 0xE +#define HPRE_DFX_CORE_LEN 0x43 + +#define HPRE_MAX_CHANNEL_NUM 2 + +static const char hpre_name[] = "hisi_hpre"; +static struct dentry *hpre_debugfs_root; +static const struct pci_device_id hpre_dev_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_HPRE_PF) }, + { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_HPRE_VF) }, + { 0, } +}; + +MODULE_DEVICE_TABLE(pci, hpre_dev_ids); + +struct hpre_hw_error { + u32 int_msk; + const char *msg; +}; + +static const struct qm_dev_alg hpre_dev_algs[] = { + { + .alg_msk = BIT(0), + .alg = "rsa\n" + }, { + .alg_msk = BIT(1), + .alg = "dh\n" + }, { + .alg_msk = BIT(2), + .alg = "ecdh\n" + }, { + .alg_msk = BIT(3), + .alg = "ecdsa\n" + }, { + .alg_msk = BIT(4), + .alg = "sm2\n" + }, { + .alg_msk = BIT(5), + .alg = "x25519\n" + }, { + .alg_msk = BIT(6), + .alg = "x448\n" + }, { + /* sentinel */ + } +}; + +static struct hisi_qm_list hpre_devices = { + .register_to_crypto = hpre_algs_register, + .unregister_from_crypto = hpre_algs_unregister, +}; + +static const char * const hpre_debug_file_name[] = { + [HPRE_CLEAR_ENABLE] = "rdclr_en", + [HPRE_CLUSTER_CTRL] = "cluster_ctrl", +}; + +enum hpre_cap_type { + HPRE_QM_NFE_MASK_CAP, + HPRE_QM_RESET_MASK_CAP, + HPRE_QM_OOO_SHUTDOWN_MASK_CAP, + HPRE_QM_CE_MASK_CAP, + HPRE_NFE_MASK_CAP, + HPRE_RESET_MASK_CAP, + HPRE_OOO_SHUTDOWN_MASK_CAP, + HPRE_CE_MASK_CAP, + HPRE_CLUSTER_NUM_CAP, + HPRE_CORE_TYPE_NUM_CAP, + HPRE_CORE_NUM_CAP, + HPRE_CLUSTER_CORE_NUM_CAP, + HPRE_CORE_ENABLE_BITMAP_CAP, + HPRE_DRV_ALG_BITMAP_CAP, + HPRE_DEV_ALG_BITMAP_CAP, + HPRE_CORE1_ALG_BITMAP_CAP, + HPRE_CORE2_ALG_BITMAP_CAP, + HPRE_CORE3_ALG_BITMAP_CAP, + HPRE_CORE4_ALG_BITMAP_CAP, + HPRE_CORE5_ALG_BITMAP_CAP, + HPRE_CORE6_ALG_BITMAP_CAP, + HPRE_CORE7_ALG_BITMAP_CAP, + HPRE_CORE8_ALG_BITMAP_CAP, + HPRE_CORE9_ALG_BITMAP_CAP, + HPRE_CORE10_ALG_BITMAP_CAP +}; + +static const struct hisi_qm_cap_info hpre_basic_info[] = { + {HPRE_QM_NFE_MASK_CAP, 0x3124, 0, GENMASK(31, 0), 0x0, 0x1C37, 0x7C37}, + {HPRE_QM_RESET_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0xC37, 0x6C37}, + {HPRE_QM_OOO_SHUTDOWN_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0x4, 0x6C37}, + {HPRE_QM_CE_MASK_CAP, 0x312C, 0, GENMASK(31, 0), 0x0, 0x8, 0x8}, + {HPRE_NFE_MASK_CAP, 0x3130, 0, GENMASK(31, 0), 0x0, 0x3FFFFE, 0x1FFFC3E}, + {HPRE_RESET_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x3FFFFE, 0xBFFC3E}, + {HPRE_OOO_SHUTDOWN_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x22, 0xBFFC3E}, + {HPRE_CE_MASK_CAP, 0x3138, 0, GENMASK(31, 0), 0x0, 0x1, 0x1}, + {HPRE_CLUSTER_NUM_CAP, 0x313c, 20, GENMASK(3, 0), 0x0, 0x4, 0x1}, + {HPRE_CORE_TYPE_NUM_CAP, 0x313c, 16, GENMASK(3, 0), 0x0, 0x2, 0x2}, + {HPRE_CORE_NUM_CAP, 0x313c, 8, GENMASK(7, 0), 0x0, 0x8, 0xA}, + {HPRE_CLUSTER_CORE_NUM_CAP, 0x313c, 0, GENMASK(7, 0), 0x0, 0x2, 0xA}, + {HPRE_CORE_ENABLE_BITMAP_CAP, 0x3140, 0, GENMASK(31, 0), 0x0, 0xF, 0x3FF}, + {HPRE_DRV_ALG_BITMAP_CAP, 0x3144, 0, GENMASK(31, 0), 0x0, 0x03, 0x27}, + {HPRE_DEV_ALG_BITMAP_CAP, 0x3148, 0, GENMASK(31, 0), 0x0, 0x03, 0x7F}, + {HPRE_CORE1_ALG_BITMAP_CAP, 0x314c, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F}, + {HPRE_CORE2_ALG_BITMAP_CAP, 0x3150, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F}, + {HPRE_CORE3_ALG_BITMAP_CAP, 0x3154, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F}, + {HPRE_CORE4_ALG_BITMAP_CAP, 0x3158, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F}, + {HPRE_CORE5_ALG_BITMAP_CAP, 0x315c, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F}, + {HPRE_CORE6_ALG_BITMAP_CAP, 0x3160, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F}, + {HPRE_CORE7_ALG_BITMAP_CAP, 0x3164, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F}, + {HPRE_CORE8_ALG_BITMAP_CAP, 0x3168, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F}, + {HPRE_CORE9_ALG_BITMAP_CAP, 0x316c, 0, GENMASK(31, 0), 0x0, 0x10, 0x10}, + {HPRE_CORE10_ALG_BITMAP_CAP, 0x3170, 0, GENMASK(31, 0), 0x0, 0x10, 0x10} +}; + +static const struct hisi_qm_cap_query_info hpre_cap_query_info[] = { + {QM_RAS_NFE_TYPE, "QM_RAS_NFE_TYPE ", 0x3124, 0x0, 0x1C37, 0x7C37}, + {QM_RAS_NFE_RESET, "QM_RAS_NFE_RESET ", 0x3128, 0x0, 0xC77, 0x6C77}, + {QM_RAS_CE_TYPE, "QM_RAS_CE_TYPE ", 0x312C, 0x0, 0x8, 0x8}, + {HPRE_RAS_NFE_TYPE, "HPRE_RAS_NFE_TYPE ", 0x3130, 0x0, 0x3FFFFE, 0x1FFFC3E}, + {HPRE_RAS_NFE_RESET, "HPRE_RAS_NFE_RESET ", 0x3134, 0x0, 0x3FFFFE, 0xBFFC3E}, + {HPRE_RAS_CE_TYPE, "HPRE_RAS_CE_TYPE ", 0x3138, 0x0, 0x1, 0x1}, + {HPRE_CORE_INFO, "HPRE_CORE_INFO ", 0x313c, 0x0, 0x420802, 0x120A0A}, + {HPRE_CORE_EN, "HPRE_CORE_EN ", 0x3140, 0x0, 0xF, 0x3FF}, + {HPRE_DRV_ALG_BITMAP, "HPRE_DRV_ALG_BITMAP ", 0x3144, 0x0, 0x03, 0x27}, + {HPRE_ALG_BITMAP, "HPRE_ALG_BITMAP ", 0x3148, 0x0, 0x03, 0x7F}, + {HPRE_CORE1_BITMAP_CAP, "HPRE_CORE1_BITMAP_CAP ", 0x314c, 0x0, 0x7F, 0x7F}, + {HPRE_CORE2_BITMAP_CAP, "HPRE_CORE2_BITMAP_CAP ", 0x3150, 0x0, 0x7F, 0x7F}, + {HPRE_CORE3_BITMAP_CAP, "HPRE_CORE3_BITMAP_CAP ", 0x3154, 0x0, 0x7F, 0x7F}, + {HPRE_CORE4_BITMAP_CAP, "HPRE_CORE4_BITMAP_CAP ", 0x3158, 0x0, 0x7F, 0x7F}, + {HPRE_CORE5_BITMAP_CAP, "HPRE_CORE5_BITMAP_CAP ", 0x315c, 0x0, 0x7F, 0x7F}, + {HPRE_CORE6_BITMAP_CAP, "HPRE_CORE6_BITMAP_CAP ", 0x3160, 0x0, 0x7F, 0x7F}, + {HPRE_CORE7_BITMAP_CAP, "HPRE_CORE7_BITMAP_CAP ", 0x3164, 0x0, 0x7F, 0x7F}, + {HPRE_CORE8_BITMAP_CAP, "HPRE_CORE8_BITMAP_CAP ", 0x3168, 0x0, 0x7F, 0x7F}, + {HPRE_CORE9_BITMAP_CAP, "HPRE_CORE9_BITMAP_CAP ", 0x316c, 0x0, 0x10, 0x10}, + {HPRE_CORE10_BITMAP_CAP, "HPRE_CORE10_BITMAP_CAP ", 0x3170, 0x0, 0x10, 0x10}, +}; + +static const struct hpre_hw_error hpre_hw_errors[] = { + { + .int_msk = BIT(0), + .msg = "core_ecc_1bit_err_int_set" + }, { + .int_msk = BIT(1), + .msg = "core_ecc_2bit_err_int_set" + }, { + .int_msk = BIT(2), + .msg = "dat_wb_poison_int_set" + }, { + .int_msk = BIT(3), + .msg = "dat_rd_poison_int_set" + }, { + .int_msk = BIT(4), + .msg = "bd_rd_poison_int_set" + }, { + .int_msk = BIT(5), + .msg = "ooo_ecc_2bit_err_int_set" + }, { + .int_msk = BIT(6), + .msg = "cluster1_shb_timeout_int_set" + }, { + .int_msk = BIT(7), + .msg = "cluster2_shb_timeout_int_set" + }, { + .int_msk = BIT(8), + .msg = "cluster3_shb_timeout_int_set" + }, { + .int_msk = BIT(9), + .msg = "cluster4_shb_timeout_int_set" + }, { + .int_msk = GENMASK(15, 10), + .msg = "ooo_rdrsp_err_int_set" + }, { + .int_msk = GENMASK(21, 16), + .msg = "ooo_wrrsp_err_int_set" + }, { + .int_msk = BIT(22), + .msg = "pt_rng_timeout_int_set" + }, { + .int_msk = BIT(23), + .msg = "sva_fsm_timeout_int_set" + }, { + .int_msk = BIT(24), + .msg = "sva_int_set" + }, { + /* sentinel */ + } +}; + +static const u64 hpre_cluster_offsets[] = { + [HPRE_CLUSTER0] = + HPRE_CLSTR_BASE + HPRE_CLUSTER0 * HPRE_CLSTR_ADDR_INTRVL, + [HPRE_CLUSTER1] = + HPRE_CLSTR_BASE + HPRE_CLUSTER1 * HPRE_CLSTR_ADDR_INTRVL, + [HPRE_CLUSTER2] = + HPRE_CLSTR_BASE + HPRE_CLUSTER2 * HPRE_CLSTR_ADDR_INTRVL, + [HPRE_CLUSTER3] = + HPRE_CLSTR_BASE + HPRE_CLUSTER3 * HPRE_CLSTR_ADDR_INTRVL, +}; + +static const struct debugfs_reg32 hpre_cluster_dfx_regs[] = { + {"CORES_EN_STATUS ", HPRE_CORE_EN_OFFSET}, + {"CORES_INI_CFG ", HPRE_CORE_INI_CFG_OFFSET}, + {"CORES_INI_STATUS ", HPRE_CORE_INI_STATUS_OFFSET}, + {"CORES_HTBT_WARN ", HPRE_CORE_HTBT_WARN_OFFSET}, + {"CORES_IS_SCHD ", HPRE_CORE_IS_SCHD_OFFSET}, +}; + +static const struct debugfs_reg32 hpre_com_dfx_regs[] = { + {"READ_CLR_EN ", HPRE_CTRL_CNT_CLR_CE}, + {"AXQOS ", HPRE_VFG_AXQOS}, + {"AWUSR_CFG ", HPRE_AWUSR_FP_CFG}, + {"BD_ENDIAN ", HPRE_BD_ENDIAN}, + {"ECC_CHECK_CTRL ", HPRE_ECC_BYPASS}, + {"RAS_INT_WIDTH ", HPRE_RAS_WIDTH_CFG}, + {"POISON_BYPASS ", HPRE_POISON_BYPASS}, + {"BD_ARUSER ", HPRE_BD_ARUSR_CFG}, + {"BD_AWUSER ", HPRE_BD_AWUSR_CFG}, + {"DATA_ARUSER ", HPRE_DATA_RUSER_CFG}, + {"DATA_AWUSER ", HPRE_DATA_WUSER_CFG}, + {"INT_STATUS ", HPRE_INT_STATUS}, + {"INT_MASK ", HPRE_HAC_INT_MSK}, + {"RAS_CE_ENB ", HPRE_HAC_RAS_CE_ENB}, + {"RAS_NFE_ENB ", HPRE_HAC_RAS_NFE_ENB}, + {"RAS_FE_ENB ", HPRE_HAC_RAS_FE_ENB}, + {"INT_SET ", HPRE_HAC_INT_SET}, + {"RNG_TIMEOUT_NUM ", HPRE_RNG_TIMEOUT_NUM}, +}; + +static const char *hpre_dfx_files[HPRE_DFX_FILE_NUM] = { + "send_cnt", + "recv_cnt", + "send_fail_cnt", + "send_busy_cnt", + "over_thrhld_cnt", + "overtime_thrhld", + "invalid_req_cnt" +}; + +/* define the HPRE's dfx regs region and region length */ +static struct dfx_diff_registers hpre_diff_regs[] = { + { + .reg_offset = HPRE_DFX_BASE, + .reg_len = HPRE_DFX_BASE_LEN, + }, { + .reg_offset = HPRE_DFX_COMMON1, + .reg_len = HPRE_DFX_COMMON1_LEN, + }, { + .reg_offset = HPRE_DFX_COMMON2, + .reg_len = HPRE_DFX_COMMON2_LEN, + }, { + .reg_offset = HPRE_DFX_CORE, + .reg_len = HPRE_DFX_CORE_LEN, + }, +}; + +static const char *hpre_channel_name[HPRE_MAX_CHANNEL_NUM] = { + "RSA", + "ECC", +}; + +static const struct hisi_qm_err_ini hpre_err_ini; + +bool hpre_check_alg_support(struct hisi_qm *qm, u32 alg) +{ + u32 cap_val; + + cap_val = qm->cap_tables.dev_cap_table[HPRE_DRV_ALG_BITMAP].cap_val; + if (alg & cap_val) + return true; + + return false; +} + +static int hpre_diff_regs_show(struct seq_file *s, void *unused) +{ + struct hisi_qm *qm = s->private; + + hisi_qm_acc_diff_regs_dump(qm, s, qm->debug.acc_diff_regs, + ARRAY_SIZE(hpre_diff_regs)); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(hpre_diff_regs); + +static int hpre_com_regs_show(struct seq_file *s, void *unused) +{ + hisi_qm_regs_dump(s, s->private); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(hpre_com_regs); + +static int hpre_cluster_regs_show(struct seq_file *s, void *unused) +{ + hisi_qm_regs_dump(s, s->private); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(hpre_cluster_regs); + +static const struct kernel_param_ops hpre_uacce_mode_ops = { + .set = uacce_mode_set, + .get = param_get_int, +}; + +/* + * uacce_mode = 0 means hpre only register to crypto, + * uacce_mode = 1 means hpre both register to crypto and uacce. + */ +static u32 uacce_mode = UACCE_MODE_NOUACCE; +module_param_cb(uacce_mode, &hpre_uacce_mode_ops, &uacce_mode, 0444); +MODULE_PARM_DESC(uacce_mode, UACCE_MODE_DESC); + +static bool pf_q_num_flag; +static int pf_q_num_set(const char *val, const struct kernel_param *kp) +{ + pf_q_num_flag = true; + + return hisi_qm_q_num_set(val, kp, PCI_DEVICE_ID_HUAWEI_HPRE_PF); +} + +static const struct kernel_param_ops hpre_pf_q_num_ops = { + .set = pf_q_num_set, + .get = param_get_int, +}; + +static u32 pf_q_num = HPRE_PF_DEF_Q_NUM; +module_param_cb(pf_q_num, &hpre_pf_q_num_ops, &pf_q_num, 0444); +MODULE_PARM_DESC(pf_q_num, "Number of queues in PF of CS(2-1024)"); + +static const struct kernel_param_ops vfs_num_ops = { + .set = vfs_num_set, + .get = param_get_int, +}; + +static u32 vfs_num; +module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444); +MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)"); + +struct hisi_qp *hpre_create_qp(u8 type) +{ + int node = cpu_to_node(raw_smp_processor_id()); + struct hisi_qp *qp = NULL; + int ret; + + if (type != HPRE_V2_ALG_TYPE && type != HPRE_V3_ECC_ALG_TYPE) + return NULL; + + /* + * type: 0 - RSA/DH. algorithm supported in V2, + * 1 - ECC algorithm in V3. + */ + ret = hisi_qm_alloc_qps_node(&hpre_devices, 1, &type, node, &qp); + if (!ret) + return qp; + + return NULL; +} + +static int hpre_wait_sva_ready(struct hisi_qm *qm) +{ + u32 val, try_times = 0; + u8 count = 0; + + /* + * Read the register value every 10-20us. If the value is 0 for three + * consecutive times, the SVA module is ready. + */ + do { + val = readl(qm->io_base + HPRE_SVA_PREFTCH_DFX4); + if (val) + count = 0; + else if (++count == HPRE_READ_SVA_STATUS_TIMES) + break; + + usleep_range(HPRE_WAIT_US_MIN, HPRE_WAIT_US_MAX); + } while (++try_times < HPRE_WAIT_SVA_READY); + + if (try_times == HPRE_WAIT_SVA_READY) { + pci_err(qm->pdev, "failed to wait sva prefetch ready\n"); + return -ETIMEDOUT; + } + + return 0; +} + +static void hpre_config_pasid(struct hisi_qm *qm) +{ + u32 val1, val2; + + if (qm->ver >= QM_HW_V3) + return; + + val1 = readl_relaxed(qm->io_base + HPRE_DATA_RUSER_CFG); + val2 = readl_relaxed(qm->io_base + HPRE_DATA_WUSER_CFG); + if (qm->use_sva) { + val1 |= BIT(HPRE_PASID_EN_BIT); + val2 |= BIT(HPRE_PASID_EN_BIT); + } else { + val1 &= ~BIT(HPRE_PASID_EN_BIT); + val2 &= ~BIT(HPRE_PASID_EN_BIT); + } + writel_relaxed(val1, qm->io_base + HPRE_DATA_RUSER_CFG); + writel_relaxed(val2, qm->io_base + HPRE_DATA_WUSER_CFG); +} + +static int hpre_cfg_by_dsm(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + union acpi_object *obj; + guid_t guid; + + if (guid_parse("b06b81ab-0134-4a45-9b0c-483447b95fa7", &guid)) { + dev_err(dev, "Hpre GUID failed\n"); + return -EINVAL; + } + + /* Switch over to MSI handling due to non-standard PCI implementation */ + obj = acpi_evaluate_dsm(ACPI_HANDLE(dev), &guid, + 0, HPRE_VIA_MSI_DSM, NULL); + if (!obj) { + dev_err(dev, "ACPI handle failed!\n"); + return -EIO; + } + + ACPI_FREE(obj); + + return 0; +} + +static int hpre_set_cluster(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + u32 cluster_core_mask; + unsigned long offset; + u32 hpre_core_info; + u8 clusters_num; + u32 val = 0; + int ret, i; + + cluster_core_mask = qm->cap_tables.dev_cap_table[HPRE_CORE_EN].cap_val; + hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val; + clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) & + hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask; + for (i = 0; i < clusters_num; i++) { + offset = i * HPRE_CLSTR_ADDR_INTRVL; + + /* clusters initiating */ + writel(cluster_core_mask, + qm->io_base + offset + HPRE_CORE_ENB); + writel(0x1, qm->io_base + offset + HPRE_CORE_INI_CFG); + ret = readl_relaxed_poll_timeout(qm->io_base + offset + + HPRE_CORE_INI_STATUS, val, + ((val & cluster_core_mask) == + cluster_core_mask), + HPRE_REG_RD_INTVRL_US, + HPRE_REG_RD_TMOUT_US); + if (ret) { + dev_err(dev, + "cluster %d int st status timeout!\n", i); + return -ETIMEDOUT; + } + } + + return 0; +} + +/* + * For Kunpeng 920, we should disable FLR triggered by hardware (BME/PM/SRIOV). + * Or it may stay in D3 state when we bind and unbind hpre quickly, + * as it does FLR triggered by hardware. + */ +static void disable_flr_of_bme(struct hisi_qm *qm) +{ + u32 val; + + val = readl(qm->io_base + QM_PEH_AXUSER_CFG); + val &= ~(HPRE_QM_BME_FLR | HPRE_QM_SRIOV_FLR); + val |= HPRE_QM_PM_FLR; + writel(val, qm->io_base + QM_PEH_AXUSER_CFG); + writel(PEH_AXUSER_CFG_ENABLE, qm->io_base + QM_PEH_AXUSER_CFG_ENABLE); +} + +static void hpre_close_sva_prefetch(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) + return; + + val = readl_relaxed(qm->io_base + HPRE_PREFETCH_CFG); + val |= HPRE_PREFETCH_DISABLE; + writel(val, qm->io_base + HPRE_PREFETCH_CFG); + + ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_SVA_PREFTCH_DFX, + val, !(val & HPRE_SVA_DISABLE_READY), + HPRE_REG_RD_INTVRL_US, + HPRE_REG_RD_TMOUT_US); + if (ret) + pci_err(qm->pdev, "failed to close sva prefetch\n"); + + (void)hpre_wait_sva_ready(qm); +} + +static void hpre_open_sva_prefetch(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) + return; + + /* Enable prefetch */ + val = readl_relaxed(qm->io_base + HPRE_PREFETCH_CFG); + val &= HPRE_PREFETCH_ENABLE; + writel(val, qm->io_base + HPRE_PREFETCH_CFG); + + ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_PREFETCH_CFG, + val, !(val & HPRE_PREFETCH_DISABLE), + HPRE_REG_RD_INTVRL_US, + HPRE_REG_RD_TMOUT_US); + if (ret) { + pci_err(qm->pdev, "failed to open sva prefetch\n"); + hpre_close_sva_prefetch(qm); + return; + } + + ret = hpre_wait_sva_ready(qm); + if (ret) + hpre_close_sva_prefetch(qm); +} + +static void hpre_enable_clock_gate(struct hisi_qm *qm) +{ + unsigned long offset; + u8 clusters_num, i; + u32 hpre_core_info; + u32 val; + + if (qm->ver < QM_HW_V3) + return; + + val = readl(qm->io_base + HPRE_CLKGATE_CTL); + val |= HPRE_CLKGATE_CTL_EN; + writel(val, qm->io_base + HPRE_CLKGATE_CTL); + + val = readl(qm->io_base + HPRE_PEH_CFG_AUTO_GATE); + val |= HPRE_PEH_CFG_AUTO_GATE_EN; + writel(val, qm->io_base + HPRE_PEH_CFG_AUTO_GATE); + + hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val; + clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) & + hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask; + for (i = 0; i < clusters_num; i++) { + offset = (unsigned long)i * HPRE_CLSTR_ADDR_INTRVL; + val = readl(qm->io_base + offset + HPRE_CLUSTER_DYN_CTL); + val |= HPRE_CLUSTER_DYN_CTL_EN; + writel(val, qm->io_base + offset + HPRE_CLUSTER_DYN_CTL); + + val = readl(qm->io_base + offset + HPRE_CORE_SHB_CFG); + val |= HPRE_CORE_GATE_EN; + writel(val, qm->io_base + offset + HPRE_CORE_SHB_CFG); + } +} + +static void hpre_disable_clock_gate(struct hisi_qm *qm) +{ + unsigned long offset; + u8 clusters_num, i; + u32 hpre_core_info; + u32 val; + + if (qm->ver < QM_HW_V3) + return; + + val = readl(qm->io_base + HPRE_CLKGATE_CTL); + val &= ~HPRE_CLKGATE_CTL_EN; + writel(val, qm->io_base + HPRE_CLKGATE_CTL); + + val = readl(qm->io_base + HPRE_PEH_CFG_AUTO_GATE); + val &= ~HPRE_PEH_CFG_AUTO_GATE_EN; + writel(val, qm->io_base + HPRE_PEH_CFG_AUTO_GATE); + + hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val; + clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) & + hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask; + for (i = 0; i < clusters_num; i++) { + offset = (unsigned long)i * HPRE_CLSTR_ADDR_INTRVL; + val = readl(qm->io_base + offset + HPRE_CLUSTER_DYN_CTL); + val &= ~HPRE_CLUSTER_DYN_CTL_EN; + writel(val, qm->io_base + offset + HPRE_CLUSTER_DYN_CTL); + + val = readl(qm->io_base + offset + HPRE_CORE_SHB_CFG); + val &= ~HPRE_CORE_GATE_EN; + writel(val, qm->io_base + offset + HPRE_CORE_SHB_CFG); + } +} + +static int hpre_set_user_domain_and_cache(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + u32 val; + int ret; + + /* disabel dynamic clock gate before sram init */ + hpre_disable_clock_gate(qm); + + writel(HPRE_QM_USR_CFG_MASK, qm->io_base + QM_ARUSER_M_CFG_ENABLE); + writel(HPRE_QM_USR_CFG_MASK, qm->io_base + QM_AWUSER_M_CFG_ENABLE); + writel_relaxed(HPRE_QM_AXI_CFG_MASK, qm->io_base + QM_AXI_M_CFG); + + if (qm->ver >= QM_HW_V3) + writel(HPRE_RSA_ENB | HPRE_ECC_ENB, + qm->io_base + HPRE_TYPES_ENB); + else + writel(HPRE_RSA_ENB, qm->io_base + HPRE_TYPES_ENB); + + writel(HPRE_QM_VFG_AX_MASK, qm->io_base + HPRE_VFG_AXCACHE); + writel(0x0, qm->io_base + HPRE_BD_ENDIAN); + writel(0x0, qm->io_base + HPRE_POISON_BYPASS); + writel(0x0, qm->io_base + HPRE_ECC_BYPASS); + + writel(HPRE_BD_USR_MASK, qm->io_base + HPRE_BD_ARUSR_CFG); + writel(HPRE_BD_USR_MASK, qm->io_base + HPRE_BD_AWUSR_CFG); + writel(0x1, qm->io_base + HPRE_RDCHN_INI_CFG); + ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_RDCHN_INI_ST, val, + val & BIT(0), + HPRE_REG_RD_INTVRL_US, + HPRE_REG_RD_TMOUT_US); + if (ret) { + dev_err(dev, "read rd channel timeout fail!\n"); + return -ETIMEDOUT; + } + + ret = hpre_set_cluster(qm); + if (ret) + return -ETIMEDOUT; + + /* This setting is only needed by Kunpeng 920. */ + if (qm->ver == QM_HW_V2) { + ret = hpre_cfg_by_dsm(qm); + if (ret) + return ret; + + disable_flr_of_bme(qm); + } + + /* Config data buffer pasid needed by Kunpeng 920 */ + hpre_config_pasid(qm); + hpre_open_sva_prefetch(qm); + + hpre_enable_clock_gate(qm); + + return ret; +} + +static void hpre_cnt_regs_clear(struct hisi_qm *qm) +{ + unsigned long offset; + u32 hpre_core_info; + u8 clusters_num; + int i; + + /* clear clusterX/cluster_ctrl */ + hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val; + clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) & + hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask; + for (i = 0; i < clusters_num; i++) { + offset = HPRE_CLSTR_BASE + i * HPRE_CLSTR_ADDR_INTRVL; + writel(0x0, qm->io_base + offset + HPRE_CLUSTER_INQURY); + } + + /* clear rdclr_en */ + writel(0x0, qm->io_base + HPRE_CTRL_CNT_CLR_CE); + + hisi_qm_debug_regs_clear(qm); +} + +static void hpre_master_ooo_ctrl(struct hisi_qm *qm, bool enable) +{ + u32 val1, val2; + + val1 = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); + if (enable) { + val1 |= HPRE_AM_OOO_SHUTDOWN_ENABLE; + val2 = qm->err_info.dev_err.shutdown_mask; + } else { + val1 &= ~HPRE_AM_OOO_SHUTDOWN_ENABLE; + val2 = 0x0; + } + + if (qm->ver > QM_HW_V2) + writel(val2, qm->io_base + HPRE_OOO_SHUTDOWN_SEL); + + writel(val1, qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); +} + +static void hpre_hw_error_disable(struct hisi_qm *qm) +{ + /* disable hpre hw error interrupts */ + writel(HPRE_RAS_MASK_ALL, qm->io_base + HPRE_INT_MASK); + /* disable HPRE block master OOO when nfe occurs on Kunpeng930 */ + hpre_master_ooo_ctrl(qm, false); +} + +static void hpre_hw_error_enable(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; + + /* clear HPRE hw error source if having */ + writel(HPRE_RAS_CLEAR_ALL, qm->io_base + HPRE_HAC_SOURCE_INT); + + /* configure error type */ + writel(dev_err->ce, qm->io_base + HPRE_RAS_CE_ENB); + writel(dev_err->nfe, qm->io_base + HPRE_RAS_NFE_ENB); + writel(dev_err->fe, qm->io_base + HPRE_RAS_FE_ENB); + + /* enable HPRE block master OOO when nfe occurs on Kunpeng930 */ + hpre_master_ooo_ctrl(qm, true); + + /* enable hpre hw error interrupts */ + writel(~err_mask, qm->io_base + HPRE_INT_MASK); +} + +static inline struct hisi_qm *hpre_file_to_qm(struct hpre_debugfs_file *file) +{ + struct hpre *hpre = container_of(file->debug, struct hpre, debug); + + return &hpre->qm; +} + +static u32 hpre_clear_enable_read(struct hpre_debugfs_file *file) +{ + struct hisi_qm *qm = hpre_file_to_qm(file); + + return readl(qm->io_base + HPRE_CTRL_CNT_CLR_CE) & + HPRE_CTRL_CNT_CLR_CE_BIT; +} + +static int hpre_clear_enable_write(struct hpre_debugfs_file *file, u32 val) +{ + struct hisi_qm *qm = hpre_file_to_qm(file); + u32 tmp; + + if (val != 1 && val != 0) + return -EINVAL; + + tmp = (readl(qm->io_base + HPRE_CTRL_CNT_CLR_CE) & + ~HPRE_CTRL_CNT_CLR_CE_BIT) | val; + writel(tmp, qm->io_base + HPRE_CTRL_CNT_CLR_CE); + + return 0; +} + +static u32 hpre_cluster_inqry_read(struct hpre_debugfs_file *file) +{ + struct hisi_qm *qm = hpre_file_to_qm(file); + int cluster_index = file->index - HPRE_CLUSTER_CTRL; + unsigned long offset = HPRE_CLSTR_BASE + + cluster_index * HPRE_CLSTR_ADDR_INTRVL; + + return readl(qm->io_base + offset + HPRE_CLSTR_ADDR_INQRY_RSLT); +} + +static void hpre_cluster_inqry_write(struct hpre_debugfs_file *file, u32 val) +{ + struct hisi_qm *qm = hpre_file_to_qm(file); + int cluster_index = file->index - HPRE_CLUSTER_CTRL; + unsigned long offset = HPRE_CLSTR_BASE + cluster_index * + HPRE_CLSTR_ADDR_INTRVL; + + writel(val, qm->io_base + offset + HPRE_CLUSTER_INQURY); +} + +static ssize_t hpre_ctrl_debug_read(struct file *filp, char __user *buf, + size_t count, loff_t *pos) +{ + struct hpre_debugfs_file *file = filp->private_data; + struct hisi_qm *qm = hpre_file_to_qm(file); + char tbuf[HPRE_DBGFS_VAL_MAX_LEN]; + u32 val; + int ret; + + ret = hisi_qm_get_dfx_access(qm); + if (ret) + return ret; + + spin_lock_irq(&file->lock); + switch (file->type) { + case HPRE_CLEAR_ENABLE: + val = hpre_clear_enable_read(file); + break; + case HPRE_CLUSTER_CTRL: + val = hpre_cluster_inqry_read(file); + break; + default: + goto err_input; + } + spin_unlock_irq(&file->lock); + + hisi_qm_put_dfx_access(qm); + ret = snprintf(tbuf, HPRE_DBGFS_VAL_MAX_LEN, "%u\n", val); + return simple_read_from_buffer(buf, count, pos, tbuf, ret); + +err_input: + spin_unlock_irq(&file->lock); + hisi_qm_put_dfx_access(qm); + return -EINVAL; +} + +static ssize_t hpre_ctrl_debug_write(struct file *filp, const char __user *buf, + size_t count, loff_t *pos) +{ + struct hpre_debugfs_file *file = filp->private_data; + struct hisi_qm *qm = hpre_file_to_qm(file); + char tbuf[HPRE_DBGFS_VAL_MAX_LEN]; + unsigned long val; + int len, ret; + + if (*pos != 0) + return 0; + + if (count >= HPRE_DBGFS_VAL_MAX_LEN) + return -ENOSPC; + + len = simple_write_to_buffer(tbuf, HPRE_DBGFS_VAL_MAX_LEN - 1, + pos, buf, count); + if (len < 0) + return len; + + tbuf[len] = '\0'; + if (kstrtoul(tbuf, 0, &val)) + return -EFAULT; + + ret = hisi_qm_get_dfx_access(qm); + if (ret) + return ret; + + spin_lock_irq(&file->lock); + switch (file->type) { + case HPRE_CLEAR_ENABLE: + ret = hpre_clear_enable_write(file, val); + if (ret) + goto err_input; + break; + case HPRE_CLUSTER_CTRL: + hpre_cluster_inqry_write(file, val); + break; + default: + ret = -EINVAL; + goto err_input; + } + + ret = count; + +err_input: + spin_unlock_irq(&file->lock); + hisi_qm_put_dfx_access(qm); + return ret; +} + +static const struct file_operations hpre_ctrl_debug_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = hpre_ctrl_debug_read, + .write = hpre_ctrl_debug_write, +}; + +static int hpre_debugfs_atomic64_get(void *data, u64 *val) +{ + struct hpre_dfx *dfx_item = data; + + *val = atomic64_read(&dfx_item->value); + + return 0; +} + +static int hpre_debugfs_atomic64_set(void *data, u64 val) +{ + struct hpre_dfx *dfx_item = data; + struct hpre_dfx *hpre_dfx = NULL; + + if (dfx_item->type == HPRE_OVERTIME_THRHLD) { + hpre_dfx = dfx_item - HPRE_OVERTIME_THRHLD; + atomic64_set(&hpre_dfx[HPRE_OVER_THRHLD_CNT].value, 0); + } else if (val) { + return -EINVAL; + } + + atomic64_set(&dfx_item->value, val); + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(hpre_atomic64_ops, hpre_debugfs_atomic64_get, + hpre_debugfs_atomic64_set, "%llu\n"); + +static int hpre_create_debugfs_file(struct hisi_qm *qm, struct dentry *dir, + enum hpre_ctrl_dbgfs_file type, int indx) +{ + struct hpre *hpre = container_of(qm, struct hpre, qm); + struct hpre_debug *dbg = &hpre->debug; + struct dentry *file_dir; + + if (dir) + file_dir = dir; + else + file_dir = qm->debug.debug_root; + + if (type >= HPRE_DEBUG_FILE_NUM) + return -EINVAL; + + spin_lock_init(&dbg->files[indx].lock); + dbg->files[indx].debug = dbg; + dbg->files[indx].type = type; + dbg->files[indx].index = indx; + debugfs_create_file(hpre_debug_file_name[type], 0600, file_dir, + dbg->files + indx, &hpre_ctrl_debug_fops); + + return 0; +} + +static int hpre_pf_comm_regs_debugfs_init(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + struct debugfs_regset32 *regset; + + regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL); + if (!regset) + return -ENOMEM; + + regset->regs = hpre_com_dfx_regs; + regset->nregs = ARRAY_SIZE(hpre_com_dfx_regs); + regset->base = qm->io_base; + regset->dev = dev; + + debugfs_create_file("regs", 0444, qm->debug.debug_root, + regset, &hpre_com_regs_fops); + + return 0; +} + +static int hpre_cluster_debugfs_init(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + char buf[HPRE_DBGFS_VAL_MAX_LEN]; + struct debugfs_regset32 *regset; + struct dentry *tmp_d; + u32 hpre_core_info; + u8 clusters_num; + int i, ret; + + hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val; + clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) & + hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask; + for (i = 0; i < clusters_num; i++) { + ret = snprintf(buf, HPRE_DBGFS_VAL_MAX_LEN, "cluster%d", i); + if (ret >= HPRE_DBGFS_VAL_MAX_LEN) + return -EINVAL; + tmp_d = debugfs_create_dir(buf, qm->debug.debug_root); + + regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL); + if (!regset) + return -ENOMEM; + + regset->regs = hpre_cluster_dfx_regs; + regset->nregs = ARRAY_SIZE(hpre_cluster_dfx_regs); + regset->base = qm->io_base + hpre_cluster_offsets[i]; + regset->dev = dev; + + debugfs_create_file("regs", 0444, tmp_d, regset, + &hpre_cluster_regs_fops); + ret = hpre_create_debugfs_file(qm, tmp_d, HPRE_CLUSTER_CTRL, + i + HPRE_CLUSTER_CTRL); + if (ret) + return ret; + } + + return 0; +} + +static int hpre_ctrl_debug_init(struct hisi_qm *qm) +{ + int ret; + + ret = hpre_create_debugfs_file(qm, NULL, HPRE_CLEAR_ENABLE, + HPRE_CLEAR_ENABLE); + if (ret) + return ret; + + ret = hpre_pf_comm_regs_debugfs_init(qm); + if (ret) + return ret; + + return hpre_cluster_debugfs_init(qm); +} + +static int hpre_cap_regs_show(struct seq_file *s, void *unused) +{ + struct hisi_qm *qm = s->private; + u32 i, size; + + size = qm->cap_tables.qm_cap_size; + for (i = 0; i < size; i++) + seq_printf(s, "%s= 0x%08x\n", qm->cap_tables.qm_cap_table[i].name, + qm->cap_tables.qm_cap_table[i].cap_val); + + size = qm->cap_tables.dev_cap_size; + for (i = 0; i < size; i++) + seq_printf(s, "%s= 0x%08x\n", qm->cap_tables.dev_cap_table[i].name, + qm->cap_tables.dev_cap_table[i].cap_val); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(hpre_cap_regs); + +static void hpre_dfx_debug_init(struct hisi_qm *qm) +{ + struct dfx_diff_registers *hpre_regs = qm->debug.acc_diff_regs; + struct hpre *hpre = container_of(qm, struct hpre, qm); + struct hpre_dfx *dfx = hpre->debug.dfx; + struct dentry *parent; + int i; + + parent = debugfs_create_dir("hpre_dfx", qm->debug.debug_root); + for (i = 0; i < HPRE_DFX_FILE_NUM; i++) { + dfx[i].type = i; + debugfs_create_file(hpre_dfx_files[i], 0644, parent, &dfx[i], + &hpre_atomic64_ops); + } + + if (qm->fun_type == QM_HW_PF && hpre_regs) + debugfs_create_file("diff_regs", 0444, parent, + qm, &hpre_diff_regs_fops); + + debugfs_create_file("cap_regs", CAP_FILE_PERMISSION, + qm->debug.debug_root, qm, &hpre_cap_regs_fops); +} + +static int hpre_debugfs_init(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + int ret; + + ret = hisi_qm_regs_debugfs_init(qm, hpre_diff_regs, ARRAY_SIZE(hpre_diff_regs)); + if (ret) { + dev_warn(dev, "Failed to init HPRE diff regs!\n"); + return ret; + } + + qm->debug.debug_root = debugfs_create_dir(dev_name(dev), + hpre_debugfs_root); + qm->debug.sqe_mask_offset = HPRE_SQE_MASK_OFFSET; + qm->debug.sqe_mask_len = HPRE_SQE_MASK_LEN; + + hisi_qm_debug_init(qm); + + if (qm->pdev->device == PCI_DEVICE_ID_HUAWEI_HPRE_PF) { + ret = hpre_ctrl_debug_init(qm); + if (ret) + goto debugfs_remove; + } + + hpre_dfx_debug_init(qm); + + return 0; + +debugfs_remove: + debugfs_remove_recursive(qm->debug.debug_root); + hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(hpre_diff_regs)); + return ret; +} + +static void hpre_debugfs_exit(struct hisi_qm *qm) +{ + debugfs_remove_recursive(qm->debug.debug_root); + + hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(hpre_diff_regs)); +} + +static int hpre_pre_store_cap_reg(struct hisi_qm *qm) +{ + struct hisi_qm_cap_record *hpre_cap; + struct device *dev = &qm->pdev->dev; + u32 hpre_core_info; + u8 clusters_num; + size_t i, size; + + size = ARRAY_SIZE(hpre_cap_query_info); + hpre_cap = devm_kcalloc(dev, size, sizeof(*hpre_cap), GFP_KERNEL); + if (!hpre_cap) + return -ENOMEM; + + for (i = 0; i < size; i++) { + hpre_cap[i].type = hpre_cap_query_info[i].type; + hpre_cap[i].name = hpre_cap_query_info[i].name; + hpre_cap[i].cap_val = hisi_qm_get_cap_value(qm, hpre_cap_query_info, + i, qm->cap_ver); + } + + hpre_core_info = hpre_cap[HPRE_CORE_INFO].cap_val; + clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) & + hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask; + if (clusters_num > HPRE_CLUSTERS_NUM_MAX) { + dev_err(dev, "Device cluster num %u is out of range for driver supports %d!\n", + clusters_num, HPRE_CLUSTERS_NUM_MAX); + return -EINVAL; + } + + qm->cap_tables.dev_cap_table = hpre_cap; + qm->cap_tables.dev_cap_size = size; + + return 0; +} + +static void hpre_set_channels(struct hisi_qm *qm) +{ + struct qm_channel *channel_data = &qm->channel_data; + int i; + + channel_data->channel_num = HPRE_MAX_CHANNEL_NUM; + for (i = 0; i < HPRE_MAX_CHANNEL_NUM; i++) + channel_data->channel_name[i] = hpre_channel_name[i]; +} + +static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) +{ + u64 alg_msk; + int ret; + + if (pdev->revision == QM_HW_V1) { + pci_warn(pdev, "HPRE version 1 is not supported!\n"); + return -EINVAL; + } + + qm->mode = uacce_mode; + qm->pdev = pdev; + qm->sqe_size = HPRE_SQE_SIZE; + qm->dev_name = hpre_name; + + qm->fun_type = (pdev->device == PCI_DEVICE_ID_HUAWEI_HPRE_PF) ? + QM_HW_PF : QM_HW_VF; + if (qm->fun_type == QM_HW_PF) { + qm->qp_base = HPRE_PF_DEF_Q_BASE; + qm->qp_num = pf_q_num; + qm->debug.curr_qm_qp_num = pf_q_num; + qm->qm_list = &hpre_devices; + qm->err_ini = &hpre_err_ini; + if (pf_q_num_flag) + set_bit(QM_MODULE_PARAM, &qm->misc_ctl); + } + + ret = hisi_qm_init(qm); + if (ret) { + pci_err(pdev, "Failed to init hpre qm configures!\n"); + return ret; + } + + hpre_set_channels(qm); + /* Fetch and save the value of capability registers */ + ret = hpre_pre_store_cap_reg(qm); + if (ret) { + pci_err(pdev, "Failed to pre-store capability registers!\n"); + hisi_qm_uninit(qm); + return ret; + } + + alg_msk = qm->cap_tables.dev_cap_table[HPRE_ALG_BITMAP].cap_val; + ret = hisi_qm_set_algs(qm, alg_msk, hpre_dev_algs, ARRAY_SIZE(hpre_dev_algs)); + if (ret) { + pci_err(pdev, "Failed to set hpre algs!\n"); + hisi_qm_uninit(qm); + } + + return ret; +} + +static int hpre_show_last_regs_init(struct hisi_qm *qm) +{ + int cluster_dfx_regs_num = ARRAY_SIZE(hpre_cluster_dfx_regs); + int com_dfx_regs_num = ARRAY_SIZE(hpre_com_dfx_regs); + struct qm_debug *debug = &qm->debug; + void __iomem *io_base; + u32 hpre_core_info; + u8 clusters_num; + int i, j, idx; + + hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val; + clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) & + hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask; + debug->last_words = kcalloc(cluster_dfx_regs_num * clusters_num + + com_dfx_regs_num, sizeof(unsigned int), GFP_KERNEL); + if (!debug->last_words) + return -ENOMEM; + + for (i = 0; i < com_dfx_regs_num; i++) + debug->last_words[i] = readl_relaxed(qm->io_base + + hpre_com_dfx_regs[i].offset); + + for (i = 0; i < clusters_num; i++) { + io_base = qm->io_base + hpre_cluster_offsets[i]; + for (j = 0; j < cluster_dfx_regs_num; j++) { + idx = com_dfx_regs_num + i * cluster_dfx_regs_num + j; + debug->last_words[idx] = readl_relaxed( + io_base + hpre_cluster_dfx_regs[j].offset); + } + } + + return 0; +} + +static void hpre_show_last_regs_uninit(struct hisi_qm *qm) +{ + struct qm_debug *debug = &qm->debug; + + if (qm->fun_type == QM_HW_VF || !debug->last_words) + return; + + kfree(debug->last_words); + debug->last_words = NULL; +} + +static void hpre_show_last_dfx_regs(struct hisi_qm *qm) +{ + int cluster_dfx_regs_num = ARRAY_SIZE(hpre_cluster_dfx_regs); + int com_dfx_regs_num = ARRAY_SIZE(hpre_com_dfx_regs); + struct qm_debug *debug = &qm->debug; + struct pci_dev *pdev = qm->pdev; + void __iomem *io_base; + u32 hpre_core_info; + u8 clusters_num; + int i, j, idx; + u32 val; + + if (qm->fun_type == QM_HW_VF || !debug->last_words) + return; + + /* dumps last word of the debugging registers during controller reset */ + for (i = 0; i < com_dfx_regs_num; i++) { + val = readl_relaxed(qm->io_base + hpre_com_dfx_regs[i].offset); + if (debug->last_words[i] != val) + pci_info(pdev, "Common_core:%s \t= 0x%08x => 0x%08x\n", + hpre_com_dfx_regs[i].name, debug->last_words[i], val); + } + + hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val; + clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) & + hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask; + for (i = 0; i < clusters_num; i++) { + io_base = qm->io_base + hpre_cluster_offsets[i]; + for (j = 0; j < cluster_dfx_regs_num; j++) { + val = readl_relaxed(io_base + + hpre_cluster_dfx_regs[j].offset); + idx = com_dfx_regs_num + i * cluster_dfx_regs_num + j; + if (debug->last_words[idx] != val) + pci_info(pdev, "cluster-%d:%s \t= 0x%08x => 0x%08x\n", + i, hpre_cluster_dfx_regs[j].name, debug->last_words[idx], val); + } + } +} + +static void hpre_log_hw_error(struct hisi_qm *qm, u32 err_sts) +{ + const struct hpre_hw_error *err = hpre_hw_errors; + struct device *dev = &qm->pdev->dev; + + while (err->msg) { + if (err->int_msk & err_sts) + dev_warn(dev, "%s [error status=0x%x] found\n", + err->msg, err->int_msk); + err++; + } +} + +static u32 hpre_get_hw_err_status(struct hisi_qm *qm) +{ + return readl(qm->io_base + HPRE_INT_STATUS); +} + +static void hpre_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) +{ + writel(err_sts, qm->io_base + HPRE_HAC_SOURCE_INT); +} + +static void hpre_disable_error_report(struct hisi_qm *qm, u32 err_type) +{ + u32 nfe_mask = qm->err_info.dev_err.nfe; + + writel(nfe_mask & (~err_type), qm->io_base + HPRE_RAS_NFE_ENB); +} + +static void hpre_enable_error_report(struct hisi_qm *qm) +{ + u32 nfe_mask = qm->err_info.dev_err.nfe; + u32 ce_mask = qm->err_info.dev_err.ce; + + writel(nfe_mask, qm->io_base + HPRE_RAS_NFE_ENB); + writel(ce_mask, qm->io_base + HPRE_RAS_CE_ENB); +} + +static void hpre_open_axi_master_ooo(struct hisi_qm *qm) +{ + u32 value; + + value = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); + writel(value & ~HPRE_AM_OOO_SHUTDOWN_ENABLE, + qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); + writel(value | HPRE_AM_OOO_SHUTDOWN_ENABLE, + qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); +} + +static enum acc_err_result hpre_get_err_result(struct hisi_qm *qm) +{ + u32 err_status; + + err_status = hpre_get_hw_err_status(qm); + if (err_status) { + if (err_status & qm->err_info.dev_err.ecc_2bits_mask) + qm->err_status.is_dev_ecc_mbit = true; + hpre_log_hw_error(qm, err_status); + + if (err_status & qm->err_info.dev_err.reset_mask) { + /* Disable the same error reporting until device is recovered. */ + hpre_disable_error_report(qm, err_status); + return ACC_ERR_NEED_RESET; + } + hpre_clear_hw_err_status(qm, err_status); + /* Avoid firmware disable error report, re-enable. */ + hpre_enable_error_report(qm); + } + + return ACC_ERR_RECOVERED; +} + +static bool hpre_dev_is_abnormal(struct hisi_qm *qm) +{ + u32 err_status; + + err_status = hpre_get_hw_err_status(qm); + if (err_status & qm->err_info.dev_err.shutdown_mask) + return true; + + return false; +} + +static void hpre_disable_axi_error(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; + u32 val; + + val = ~(err_mask & (~HPRE_AXI_ERROR_MASK)); + writel(val, qm->io_base + HPRE_INT_MASK); + + if (qm->ver > QM_HW_V2) + writel(dev_err->shutdown_mask & (~HPRE_AXI_ERROR_MASK), + qm->io_base + HPRE_OOO_SHUTDOWN_SEL); +} + +static void hpre_enable_axi_error(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; + + /* clear axi error source */ + writel(HPRE_AXI_ERROR_MASK, qm->io_base + HPRE_HAC_SOURCE_INT); + + writel(~err_mask, qm->io_base + HPRE_INT_MASK); + + if (qm->ver > QM_HW_V2) + writel(dev_err->shutdown_mask, qm->io_base + HPRE_OOO_SHUTDOWN_SEL); +} + +static void hpre_err_info_init(struct hisi_qm *qm) +{ + struct hisi_qm_err_info *err_info = &qm->err_info; + struct hisi_qm_err_mask *qm_err = &err_info->qm_err; + struct hisi_qm_err_mask *dev_err = &err_info->dev_err; + + qm_err->fe = HPRE_HAC_RAS_FE_ENABLE; + qm_err->ce = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_QM_CE_MASK_CAP, qm->cap_ver); + qm_err->nfe = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_QM_NFE_MASK_CAP, qm->cap_ver); + qm_err->shutdown_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, + HPRE_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + qm_err->reset_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, + HPRE_QM_RESET_MASK_CAP, qm->cap_ver); + qm_err->ecc_2bits_mask = QM_ECC_MBIT; + + dev_err->fe = HPRE_HAC_RAS_FE_ENABLE; + dev_err->ce = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_CE_MASK_CAP, qm->cap_ver); + dev_err->nfe = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_NFE_MASK_CAP, qm->cap_ver); + dev_err->shutdown_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, + HPRE_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + dev_err->reset_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, + HPRE_RESET_MASK_CAP, qm->cap_ver); + dev_err->ecc_2bits_mask = HPRE_CORE_ECC_2BIT_ERR | HPRE_OOO_ECC_2BIT_ERR; + + err_info->msi_wr_port = HPRE_WR_MSI_PORT; + err_info->acpi_rst = "HRST"; +} + +static const struct hisi_qm_err_ini hpre_err_ini = { + .hw_init = hpre_set_user_domain_and_cache, + .hw_err_enable = hpre_hw_error_enable, + .hw_err_disable = hpre_hw_error_disable, + .get_dev_hw_err_status = hpre_get_hw_err_status, + .clear_dev_hw_err_status = hpre_clear_hw_err_status, + .open_axi_master_ooo = hpre_open_axi_master_ooo, + .open_sva_prefetch = hpre_open_sva_prefetch, + .close_sva_prefetch = hpre_close_sva_prefetch, + .show_last_dfx_regs = hpre_show_last_dfx_regs, + .err_info_init = hpre_err_info_init, + .get_err_result = hpre_get_err_result, + .dev_is_abnormal = hpre_dev_is_abnormal, + .disable_axi_error = hpre_disable_axi_error, + .enable_axi_error = hpre_enable_axi_error, +}; + +static int hpre_pf_probe_init(struct hpre *hpre) +{ + struct hisi_qm *qm = &hpre->qm; + int ret; + + ret = hpre_set_user_domain_and_cache(qm); + if (ret) + return ret; + + hisi_qm_dev_err_init(qm); + ret = hpre_show_last_regs_init(qm); + if (ret) + pci_err(qm->pdev, "Failed to init last word regs!\n"); + + return ret; +} + +static int hpre_probe_init(struct hpre *hpre) +{ + u32 type_rate = HPRE_SHAPER_TYPE_RATE; + struct hisi_qm *qm = &hpre->qm; + int ret; + + if (qm->fun_type == QM_HW_PF) { + ret = hpre_pf_probe_init(hpre); + if (ret) + return ret; + /* Enable shaper type 0 */ + if (qm->ver >= QM_HW_V3) { + type_rate |= QM_SHAPER_ENABLE; + qm->type_rate = type_rate; + } + } + + return 0; +} + +static void hpre_probe_uninit(struct hisi_qm *qm) +{ + if (qm->fun_type == QM_HW_VF) + return; + + hpre_cnt_regs_clear(qm); + qm->debug.curr_qm_qp_num = 0; + hpre_show_last_regs_uninit(qm); + hpre_close_sva_prefetch(qm); + hisi_qm_dev_err_uninit(qm); +} + +static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct hisi_qm *qm; + struct hpre *hpre; + int ret; + + hpre = devm_kzalloc(&pdev->dev, sizeof(*hpre), GFP_KERNEL); + if (!hpre) + return -ENOMEM; + + qm = &hpre->qm; + ret = hpre_qm_init(qm, pdev); + if (ret) { + pci_err(pdev, "Failed to init HPRE QM (%d)!\n", ret); + return ret; + } + + ret = hpre_probe_init(hpre); + if (ret) { + pci_err(pdev, "Failed to probe (%d)!\n", ret); + goto err_with_qm_init; + } + + ret = hisi_qm_start(qm); + if (ret) + goto err_with_probe_init; + + ret = hpre_debugfs_init(qm); + if (ret) + dev_warn(&pdev->dev, "init debugfs fail!\n"); + + hisi_qm_add_list(qm, &hpre_devices); + ret = hisi_qm_alg_register(qm, &hpre_devices, HPRE_CTX_Q_NUM_DEF); + if (ret < 0) { + pci_err(pdev, "fail to register algs to crypto!\n"); + goto err_qm_del_list; + } + + ret = hisi_qm_register_uacce(qm); + if (ret) { + pci_err(pdev, "failed to register uacce (%d)!\n", ret); + goto err_with_alg_register; + } + + if (qm->fun_type == QM_HW_PF && vfs_num) { + ret = hisi_qm_sriov_enable(pdev, vfs_num); + if (ret < 0) + goto err_with_alg_register; + } + + hisi_qm_pm_init(qm); + + return 0; + +err_with_alg_register: + hisi_qm_alg_unregister(qm, &hpre_devices, HPRE_CTX_Q_NUM_DEF); + +err_qm_del_list: + hisi_qm_del_list(qm, &hpre_devices); + hpre_debugfs_exit(qm); + hisi_qm_stop(qm, QM_NORMAL); + +err_with_probe_init: + hpre_probe_uninit(qm); + +err_with_qm_init: + hisi_qm_uninit(qm); + + return ret; +} + +static void hpre_remove(struct pci_dev *pdev) +{ + struct hisi_qm *qm = pci_get_drvdata(pdev); + + hisi_qm_pm_uninit(qm); + hisi_qm_wait_task_finish(qm, &hpre_devices); + hisi_qm_alg_unregister(qm, &hpre_devices, HPRE_CTX_Q_NUM_DEF); + hisi_qm_del_list(qm, &hpre_devices); + if (qm->fun_type == QM_HW_PF && qm->vfs_num) + hisi_qm_sriov_disable(pdev, true); + + hpre_debugfs_exit(qm); + hisi_qm_stop(qm, QM_NORMAL); + + hpre_probe_uninit(qm); + hisi_qm_uninit(qm); +} + +static const struct dev_pm_ops hpre_pm_ops = { + SET_RUNTIME_PM_OPS(hisi_qm_suspend, hisi_qm_resume, NULL) +}; + +static const struct pci_error_handlers hpre_err_handler = { + .error_detected = hisi_qm_dev_err_detected, + .slot_reset = hisi_qm_dev_slot_reset, + .reset_prepare = hisi_qm_reset_prepare, + .reset_done = hisi_qm_reset_done, +}; + +static struct pci_driver hpre_pci_driver = { + .name = hpre_name, + .id_table = hpre_dev_ids, + .probe = hpre_probe, + .remove = hpre_remove, + .sriov_configure = IS_ENABLED(CONFIG_PCI_IOV) ? + hisi_qm_sriov_configure : NULL, + .err_handler = &hpre_err_handler, + .shutdown = hisi_qm_dev_shutdown, + .driver.pm = &hpre_pm_ops, +}; + +struct pci_driver *hisi_hpre_get_pf_driver(void) +{ + return &hpre_pci_driver; +} +EXPORT_SYMBOL_GPL(hisi_hpre_get_pf_driver); + +static void hpre_register_debugfs(void) +{ + if (!debugfs_initialized()) + return; + + hpre_debugfs_root = debugfs_create_dir(hpre_name, NULL); +} + +static void hpre_unregister_debugfs(void) +{ + debugfs_remove_recursive(hpre_debugfs_root); +} + +static int __init hpre_init(void) +{ + int ret; + + hisi_qm_init_list(&hpre_devices); + hpre_register_debugfs(); + + ret = pci_register_driver(&hpre_pci_driver); + if (ret) { + hpre_unregister_debugfs(); + pr_err("hpre: can't register hisi hpre driver.\n"); + } + + return ret; +} + +static void __exit hpre_exit(void) +{ + pci_unregister_driver(&hpre_pci_driver); + hpre_unregister_debugfs(); +} + +module_init(hpre_init); +module_exit(hpre_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Zaibo Xu "); +MODULE_AUTHOR("Meng Yu "); +MODULE_DESCRIPTION("Driver for HiSilicon HPRE accelerator"); diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c new file mode 100644 index 000000000..c01966a4a --- /dev/null +++ b/drivers/crypto/hisilicon/qm.c @@ -0,0 +1,6362 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2019 HiSilicon Limited. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "qm_common.h" + +/* eq/aeq irq enable */ +#define QM_VF_AEQ_INT_SOURCE 0x0 +#define QM_VF_AEQ_INT_MASK 0x4 +#define QM_VF_EQ_INT_SOURCE 0x8 +#define QM_VF_EQ_INT_MASK 0xc + +#define QM_IRQ_VECTOR_MASK GENMASK(15, 0) +#define QM_IRQ_TYPE_MASK GENMASK(15, 0) +#define QM_IRQ_TYPE_SHIFT 16 +#define QM_ABN_IRQ_TYPE_MASK GENMASK(7, 0) + +/* mailbox */ +#define QM_MB_PING_ALL_VFS 0xffff +#define QM_MB_STATUS_MASK GENMASK(12, 9) +#define QM_MB_BUSY_MASK BIT(13) +#define QM_MB_MAX_WAIT_TIMEOUT USEC_PER_SEC +#define QM_MB_MAX_STOP_TIMEOUT (5 * USEC_PER_SEC) + +/* sqc shift */ +#define QM_SQ_HOP_NUM_SHIFT 0 +#define QM_SQ_PAGE_SIZE_SHIFT 4 +#define QM_SQ_BUF_SIZE_SHIFT 8 +#define QM_SQ_SQE_SIZE_SHIFT 12 +#define QM_SQ_PRIORITY_SHIFT 0 +#define QM_SQ_ORDERS_SHIFT 4 +#define QM_SQ_TYPE_SHIFT 8 +#define QM_QC_PASID_ENABLE 0x1 +#define QM_QC_PASID_ENABLE_SHIFT 7 + +#define QM_SQ_TYPE_MASK GENMASK(3, 0) +#define QM_SQ_TAIL_IDX(sqc) ((le16_to_cpu((sqc).w11) >> 6) & 0x1) +#define QM_SQC_DISABLE_QP (1U << 6) +#define QM_XQC_RANDOM_DATA 0xaaaa + +/* cqc shift */ +#define QM_CQ_HOP_NUM_SHIFT 0 +#define QM_CQ_PAGE_SIZE_SHIFT 4 +#define QM_CQ_BUF_SIZE_SHIFT 8 +#define QM_CQ_CQE_SIZE_SHIFT 12 +#define QM_CQ_PHASE_SHIFT 0 +#define QM_CQ_FLAG_SHIFT 1 + +#define QM_CQE_PHASE(cqe) (le16_to_cpu((cqe)->w7) & 0x1) +#define QM_QC_CQE_SIZE 4 +#define QM_CQ_TAIL_IDX(cqc) ((le16_to_cpu((cqc).w11) >> 6) & 0x1) + +/* eqc shift */ +#define QM_EQE_AEQE_SIZE (2UL << 12) +#define QM_EQC_PHASE_SHIFT 16 + +#define QM_EQE_PHASE(dw0) (((dw0) >> 16) & 0x1) +#define QM_EQE_CQN_MASK GENMASK(15, 0) + +#define QM_AEQE_PHASE(dw0) (((dw0) >> 16) & 0x1) +#define QM_AEQE_TYPE_SHIFT 17 +#define QM_AEQE_TYPE_MASK 0xf +#define QM_AEQE_CQN_MASK GENMASK(15, 0) +#define QM_CQ_OVERFLOW 0 +#define QM_EQ_OVERFLOW 1 +#define QM_CQE_ERROR 2 + +#define QM_XQ_DEPTH_SHIFT 16 +#define QM_XQ_DEPTH_MASK GENMASK(15, 0) + +#define QM_DOORBELL_CMD_SQ 0 +#define QM_DOORBELL_CMD_CQ 1 +#define QM_DOORBELL_CMD_EQ 2 +#define QM_DOORBELL_CMD_AEQ 3 + +#define QM_DOORBELL_BASE_V1 0x340 +#define QM_DB_CMD_SHIFT_V1 16 +#define QM_DB_INDEX_SHIFT_V1 32 +#define QM_DB_PRIORITY_SHIFT_V1 48 +#define QM_PAGE_SIZE 0x0034 +#define QM_QP_DB_INTERVAL 0x10000 +#define QM_DB_TIMEOUT_CFG 0x100074 +#define QM_DB_TIMEOUT_SET 0x1fffff + +#define QM_MEM_START_INIT 0x100040 +#define QM_MEM_INIT_DONE 0x100044 +#define QM_VFT_CFG_RDY 0x10006c +#define QM_VFT_CFG_OP_WR 0x100058 +#define QM_VFT_CFG_TYPE 0x10005c +#define QM_VFT_CFG 0x100060 +#define QM_VFT_CFG_OP_ENABLE 0x100054 +#define QM_PM_CTRL 0x100148 +#define QM_IDLE_DISABLE BIT(9) + +#define QM_SUB_VERSION_ID 0x210 + +#define QM_VFT_CFG_DATA_L 0x100064 +#define QM_VFT_CFG_DATA_H 0x100068 +#define QM_SQC_VFT_BUF_SIZE (7ULL << 8) +#define QM_SQC_VFT_SQC_SIZE (5ULL << 12) +#define QM_SQC_VFT_INDEX_NUMBER (1ULL << 16) +#define QM_SQC_VFT_START_SQN_SHIFT 28 +#define QM_SQC_VFT_VALID (1ULL << 44) +#define QM_SQC_VFT_SQN_SHIFT 45 +#define QM_CQC_VFT_BUF_SIZE (7ULL << 8) +#define QM_CQC_VFT_SQC_SIZE (5ULL << 12) +#define QM_CQC_VFT_INDEX_NUMBER (1ULL << 16) +#define QM_CQC_VFT_VALID (1ULL << 28) + +#define QM_SQC_VFT_BASE_SHIFT_V2 28 +#define QM_SQC_VFT_BASE_MASK_V2 GENMASK(15, 0) +#define QM_SQC_VFT_NUM_SHIFT_V2 45 +#define QM_SQC_VFT_NUM_MASK_V2 GENMASK(9, 0) +#define QM_MAX_QC_TYPE 2 + +#define QM_ABNORMAL_INT_SOURCE 0x100000 +#define QM_ABNORMAL_INT_MASK 0x100004 +#define QM_ABNORMAL_INT_STATUS 0x100008 +#define QM_ABNORMAL_INT_SET 0x10000c +#define QM_ABNORMAL_INF00 0x100010 +#define QM_FIFO_OVERFLOW_TYPE 0xc0 +#define QM_FIFO_OVERFLOW_TYPE_SHIFT 6 +#define QM_FIFO_OVERFLOW_VF 0x3f +#define QM_FIFO_OVERFLOW_QP_SHIFT 16 +#define QM_ABNORMAL_INF01 0x100014 +#define QM_DB_TIMEOUT_TYPE 0xc0 +#define QM_DB_TIMEOUT_TYPE_SHIFT 6 +#define QM_DB_TIMEOUT_VF 0x3f +#define QM_DB_TIMEOUT_QP_SHIFT 16 +#define QM_ABNORMAL_INF02 0x100018 +#define QM_AXI_POISON_ERR BIT(22) +#define QM_RAS_CE_ENABLE 0x1000ec +#define QM_RAS_FE_ENABLE 0x1000f0 +#define QM_RAS_NFE_ENABLE 0x1000f4 +#define QM_RAS_CE_THRESHOLD 0x1000f8 +#define QM_RAS_CE_TIMES_PER_IRQ 1 +#define QM_OOO_SHUTDOWN_SEL 0x1040f8 +#define QM_AXI_RRESP_ERR BIT(0) +#define QM_DB_TIMEOUT BIT(10) +#define QM_OF_FIFO_OF BIT(11) +#define QM_RAS_AXI_ERROR (BIT(0) | BIT(1) | BIT(12)) +#define QM_RAS_MASK_ALL GENMASK(31, 0) +#define QM_RAS_CLEAR_ALL GENMASK(31, 0) + +#define QM_RESET_WAIT_TIMEOUT 400 +#define QM_PEH_VENDOR_ID 0x1000d8 +#define ACC_VENDOR_ID_VALUE 0x5a5a +#define QM_PEH_DFX_INFO0 0x1000fc +#define QM_PEH_DFX_INFO1 0x100100 +#define QM_PEH_DFX_MASK (BIT(0) | BIT(2)) +#define QM_PEH_MSI_FINISH_MASK GENMASK(19, 16) +#define ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT 3 +#define ACC_PEH_MSI_DISABLE GENMASK(31, 0) +#define ACC_MASTER_GLOBAL_CTRL_SHUTDOWN 0x1 +#define ACC_MASTER_TRANS_RETURN_RW 3 +#define ACC_MASTER_TRANS_RETURN 0x300150 +#define ACC_MASTER_GLOBAL_CTRL 0x300000 +#define ACC_AM_CFG_PORT_WR_EN 0x30001c +#define ACC_AM_ROB_ECC_INT_STS 0x300104 +#define ACC_ROB_ECC_ERR_MULTPL BIT(1) +#define QM_MSI_CAP_ENABLE BIT(16) + +/* interfunction communication */ +#define QM_IFC_READY_STATUS 0x100128 +#define QM_IFC_INT_SET_P 0x100130 +#define QM_IFC_INT_CFG 0x100134 +#define QM_IFC_INT_SOURCE_P 0x100138 +#define QM_IFC_INT_SOURCE_V 0x0020 +#define QM_IFC_INT_MASK 0x0024 +#define QM_IFC_INT_STATUS 0x0028 +#define QM_IFC_INT_SET_V 0x002C +#define QM_PF2VF_PF_W 0x104700 +#define QM_VF2PF_PF_R 0x104800 +#define QM_VF2PF_VF_W 0x320 +#define QM_PF2VF_VF_R 0x380 +#define QM_IFC_SEND_ALL_VFS GENMASK(6, 0) +#define QM_IFC_INT_SOURCE_CLR GENMASK(63, 0) +#define QM_IFC_INT_SOURCE_MASK BIT(0) +#define QM_IFC_INT_DISABLE BIT(0) +#define QM_IFC_INT_STATUS_MASK BIT(0) +#define QM_IFC_INT_SET_MASK BIT(0) +#define QM_WAIT_DST_ACK 1000 +#define QM_MAX_PF_WAIT_COUNT 20 +#define QM_MAX_VF_WAIT_COUNT 40 +#define QM_VF_RESET_WAIT_US 20000 +#define QM_VF_RESET_WAIT_CNT 3000 +#define QM_VF2PF_REG_SIZE 4 +#define QM_IFC_CMD_MASK GENMASK(31, 0) +#define QM_IFC_DATA_SHIFT 32 +#define QM_VF_RESET_WAIT_TIMEOUT_US \ + (QM_VF_RESET_WAIT_US * QM_VF_RESET_WAIT_CNT) + +#define POLL_PERIOD 10 +#define POLL_TIMEOUT 1000 +#define WAIT_PERIOD_US_MAX 200 +#define WAIT_PERIOD_US_MIN 100 +#define MAX_WAIT_COUNTS 1000 +#define QM_CACHE_WB_START 0x204 +#define QM_CACHE_WB_DONE 0x208 +#define QM_FUNC_CAPS_REG 0x3100 +#define QM_CAPBILITY_VERSION GENMASK(7, 0) + +#define PCI_BAR_2 2 +#define PCI_BAR_4 4 +#define QMC_ALIGN(sz) ALIGN(sz, 32) + +#define QM_DBG_READ_LEN 256 +#define QM_PCI_COMMAND_INVALID ~0 +#define QM_RESET_STOP_TX_OFFSET 1 +#define QM_RESET_STOP_RX_OFFSET 2 + +#define WAIT_PERIOD 20 +#define REMOVE_WAIT_DELAY 10 + +#define QM_QOS_PARAM_NUM 2 +#define QM_QOS_MAX_VAL 1000 +#define QM_QOS_RATE 100 +#define QM_QOS_EXPAND_RATE 1000 +#define QM_SHAPER_CIR_B_MASK GENMASK(7, 0) +#define QM_SHAPER_CIR_U_MASK GENMASK(10, 8) +#define QM_SHAPER_CIR_S_MASK GENMASK(14, 11) +#define QM_SHAPER_FACTOR_CIR_U_SHIFT 8 +#define QM_SHAPER_FACTOR_CIR_S_SHIFT 11 +#define QM_SHAPER_FACTOR_CBS_B_SHIFT 15 +#define QM_SHAPER_FACTOR_CBS_S_SHIFT 19 +#define QM_SHAPER_CBS_B 1 +#define QM_SHAPER_VFT_OFFSET 6 +#define QM_QOS_MIN_ERROR_RATE 5 +#define QM_SHAPER_MIN_CBS_S 8 +#define QM_QOS_TICK 0x300U +#define QM_QOS_DIVISOR_CLK 0x1f40U +#define QM_QOS_MAX_CIR_B 200 +#define QM_QOS_MIN_CIR_B 100 +#define QM_QOS_MAX_CIR_U 6 +#define QM_AUTOSUSPEND_DELAY 3000 + +#define QM_DB_DROP_ALL_FUNC_ENABLE GENMASK(63, 0) +#define QM_DB_DROP_ALL_FUNC_DISABLE 0 +#define QM_DEV_DB_DROP 0x0100250 +#define QM_FUN_DB_DROP 0x0038 + +/* qm function err mask */ +#define QM_FUNC_AXI_ERR_ST0 0x100280 +#define QM_RAS_FUNC_ERROR (BIT(0) | BIT(1)) +#define QM_FUNC_RAS_CLEAR_ALL GENMASK(63, 0) + +/* qm isolation state mask */ +#define QM_ISOLATED_STATE BIT(31) +#define QM_ISOLATED_THRESHOLD_MASK GENMASK(15, 0) + + /* abnormal status value for stopping queue */ +#define QM_STOP_QUEUE_FAIL 1 +#define QM_DUMP_SQC_FAIL 3 +#define QM_DUMP_CQC_FAIL 4 +#define QM_FINISH_WAIT 5 + +#define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \ + (((hop_num) << QM_CQ_HOP_NUM_SHIFT) | \ + ((pg_sz) << QM_CQ_PAGE_SIZE_SHIFT) | \ + ((buf_sz) << QM_CQ_BUF_SIZE_SHIFT) | \ + ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT)) + +#define QM_MK_CQC_DW3_V2(cqe_sz, cq_depth) \ + ((((u32)cq_depth) - 1) | ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT)) + +#define QM_MK_SQC_W13(priority, orders, alg_type) \ + (((priority) << QM_SQ_PRIORITY_SHIFT) | \ + ((orders) << QM_SQ_ORDERS_SHIFT) | \ + (((alg_type) & QM_SQ_TYPE_MASK) << QM_SQ_TYPE_SHIFT)) + +#define QM_MK_SQC_DW3_V1(hop_num, pg_sz, buf_sz, sqe_sz) \ + (((hop_num) << QM_SQ_HOP_NUM_SHIFT) | \ + ((pg_sz) << QM_SQ_PAGE_SIZE_SHIFT) | \ + ((buf_sz) << QM_SQ_BUF_SIZE_SHIFT) | \ + ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT)) + +#define QM_MK_SQC_DW3_V2(sqe_sz, sq_depth) \ + ((((u32)sq_depth) - 1) | ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT)) + +enum vft_type { + SQC_VFT = 0, + CQC_VFT, + SHAPER_VFT, +}; + +enum qm_alg_type { + ALG_TYPE_0, + ALG_TYPE_1, +}; + +/* + * Message format for QM_VF_GET_ISOLATE and QM_PF_SET_ISOLATE commands + * + * These commands use a 32-bit command field (cmd) and 32-bit data field (data) + * + * Command behavior: + * - QM_VF_GET_ISOLATE: VF requests isolation status and threshold + * - QM_PF_SET_ISOLATE: PF sets isolation status and threshold + * + * Data field bit layout: + * - bit31 (MSB): Isolation status flag (1 = isolated, 0 = non-isolated) + * - bit15-0 (16 LSB): Isolation threshold value + * - bit30-16 (15 bits): Reserved + */ +enum qm_ifc_cmd { + QM_PF_FLR_PREPARE = 0x01, + QM_PF_SRST_PREPARE, + QM_PF_RESET_DONE, + QM_VF_PREPARE_DONE, + QM_VF_PREPARE_FAIL, + QM_VF_START_DONE, + QM_VF_START_FAIL, + QM_PF_SET_QOS, + QM_VF_GET_QOS, + QM_FUNCTION_RESET, + QM_VF_GET_ISOLATE, + QM_PF_SET_ISOLATE, +}; + +enum qm_basic_type { + QM_TOTAL_QP_NUM_CAP = 0x0, + QM_FUNC_MAX_QP_CAP, + QM_XEQ_DEPTH_CAP, + QM_QP_DEPTH_CAP, + QM_EQ_IRQ_TYPE_CAP, + QM_AEQ_IRQ_TYPE_CAP, + QM_ABN_IRQ_TYPE_CAP, + QM_PF2VF_IRQ_TYPE_CAP, + QM_PF_IRQ_NUM_CAP, + QM_VF_IRQ_NUM_CAP, +}; + +enum qm_cap_table_type { + QM_CAP_VF = 0x0, + QM_AEQE_NUM, + QM_SCQE_NUM, + QM_EQ_IRQ, + QM_AEQ_IRQ, + QM_ABNORMAL_IRQ, + QM_MB_IRQ, + MAX_IRQ_NUM, + EXT_BAR_INDEX, +}; + +static const struct hisi_qm_cap_query_info qm_cap_query_info[] = { + {QM_CAP_VF, "QM_CAP_VF ", 0x3100, 0x0, 0x0, 0x6F01}, + {QM_AEQE_NUM, "QM_AEQE_NUM ", 0x3104, 0x800, 0x4000800, 0x4000800}, + {QM_SCQE_NUM, "QM_SCQE_NUM ", + 0x3108, 0x4000400, 0x4000400, 0x4000400}, + {QM_EQ_IRQ, "QM_EQ_IRQ ", 0x310c, 0x10000, 0x10000, 0x10000}, + {QM_AEQ_IRQ, "QM_AEQ_IRQ ", 0x3110, 0x0, 0x10001, 0x10001}, + {QM_ABNORMAL_IRQ, "QM_ABNORMAL_IRQ ", 0x3114, 0x0, 0x10003, 0x10003}, + {QM_MB_IRQ, "QM_MB_IRQ ", 0x3118, 0x0, 0x0, 0x10002}, + {MAX_IRQ_NUM, "MAX_IRQ_NUM ", 0x311c, 0x10001, 0x40002, 0x40003}, + {EXT_BAR_INDEX, "EXT_BAR_INDEX ", 0x3120, 0x0, 0x0, 0x14}, +}; + +static const struct hisi_qm_cap_info qm_cap_info_comm[] = { + {QM_SUPPORT_DB_ISOLATION, 0x30, 0, BIT(0), 0x0, 0x0, 0x0}, + {QM_SUPPORT_FUNC_QOS, 0x3100, 0, BIT(8), 0x0, 0x0, 0x1}, + {QM_SUPPORT_STOP_QP, 0x3100, 0, BIT(9), 0x0, 0x0, 0x1}, + {QM_SUPPORT_STOP_FUNC, 0x3100, 0, BIT(10), 0x0, 0x0, 0x1}, + {QM_SUPPORT_MB_COMMAND, 0x3100, 0, BIT(11), 0x0, 0x0, 0x1}, + {QM_SUPPORT_SVA_PREFETCH, 0x3100, 0, BIT(14), 0x0, 0x0, 0x1}, + {QM_SUPPORT_DAE, 0x3100, 0, BIT(15), 0x0, 0x0, 0x0}, +}; + +static const struct hisi_qm_cap_info qm_cap_info_pf[] = { + {QM_SUPPORT_RPM, 0x3100, 0, BIT(13), 0x0, 0x0, 0x1}, +}; + +static const struct hisi_qm_cap_info qm_cap_info_vf[] = { + {QM_SUPPORT_RPM, 0x3100, 0, BIT(12), 0x0, 0x0, 0x0}, +}; + +static const struct hisi_qm_cap_info qm_basic_info[] = { + {QM_TOTAL_QP_NUM_CAP, 0x100158, 0, GENMASK(10, 0), 0x1000, 0x400, 0x400}, + {QM_FUNC_MAX_QP_CAP, 0x100158, 11, GENMASK(10, 0), 0x1000, 0x400, 0x400}, + {QM_XEQ_DEPTH_CAP, 0x3104, 0, GENMASK(31, 0), 0x800, 0x4000800, 0x4000800}, + {QM_QP_DEPTH_CAP, 0x3108, 0, GENMASK(31, 0), 0x4000400, 0x4000400, 0x4000400}, + {QM_EQ_IRQ_TYPE_CAP, 0x310c, 0, GENMASK(31, 0), 0x10000, 0x10000, 0x10000}, + {QM_AEQ_IRQ_TYPE_CAP, 0x3110, 0, GENMASK(31, 0), 0x0, 0x10001, 0x10001}, + {QM_ABN_IRQ_TYPE_CAP, 0x3114, 0, GENMASK(31, 0), 0x0, 0x10003, 0x10003}, + {QM_PF2VF_IRQ_TYPE_CAP, 0x3118, 0, GENMASK(31, 0), 0x0, 0x0, 0x10002}, + {QM_PF_IRQ_NUM_CAP, 0x311c, 16, GENMASK(15, 0), 0x1, 0x4, 0x4}, + {QM_VF_IRQ_NUM_CAP, 0x311c, 0, GENMASK(15, 0), 0x1, 0x2, 0x3}, +}; + +struct qm_mailbox { + __le16 w0; + __le16 queue_num; + __le32 base_l; + __le32 base_h; + __le32 rsvd; +}; + +struct qm_doorbell { + __le16 queue_num; + __le16 cmd; + __le16 index; + __le16 priority; +}; + +struct hisi_qm_resource { + struct hisi_qm *qm; + int distance; + struct list_head list; +}; + +/** + * struct qm_hw_err - Structure describing the device errors + * @list: hardware error list + * @timestamp: timestamp when the error occurred + */ +struct qm_hw_err { + struct list_head list; + unsigned long long timestamp; +}; + +struct hisi_qm_hw_ops { + int (*get_vft)(struct hisi_qm *qm, u32 *base, u32 *number); + void (*qm_db)(struct hisi_qm *qm, u16 qn, + u8 cmd, u16 index, u8 priority); + int (*debug_init)(struct hisi_qm *qm); + void (*hw_error_init)(struct hisi_qm *qm); + void (*hw_error_uninit)(struct hisi_qm *qm); + enum acc_err_result (*hw_error_handle)(struct hisi_qm *qm); + int (*set_msi)(struct hisi_qm *qm, bool set); + + /* (u64)msg = (u32)data << 32 | (enum qm_ifc_cmd)cmd */ + int (*set_ifc_begin)(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num); + void (*set_ifc_end)(struct hisi_qm *qm); + int (*get_ifc)(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num); +}; + +struct hisi_qm_hw_error { + u32 int_msk; + const char *msg; +}; + +static const struct hisi_qm_hw_error qm_hw_error[] = { + { .int_msk = BIT(0), .msg = "qm_axi_rresp" }, + { .int_msk = BIT(1), .msg = "qm_axi_bresp" }, + { .int_msk = BIT(2), .msg = "qm_ecc_mbit" }, + { .int_msk = BIT(3), .msg = "qm_ecc_1bit" }, + { .int_msk = BIT(4), .msg = "qm_acc_get_task_timeout" }, + { .int_msk = BIT(5), .msg = "qm_acc_do_task_timeout" }, + { .int_msk = BIT(6), .msg = "qm_acc_wb_not_ready_timeout" }, + { .int_msk = BIT(7), .msg = "qm_sq_cq_vf_invalid" }, + { .int_msk = BIT(8), .msg = "qm_cq_vf_invalid" }, + { .int_msk = BIT(9), .msg = "qm_sq_vf_invalid" }, + { .int_msk = BIT(10), .msg = "qm_db_timeout" }, + { .int_msk = BIT(11), .msg = "qm_of_fifo_of" }, + { .int_msk = BIT(12), .msg = "qm_db_random_invalid" }, + { .int_msk = BIT(13), .msg = "qm_mailbox_timeout" }, + { .int_msk = BIT(14), .msg = "qm_flr_timeout" }, +}; + +static const char * const qm_db_timeout[] = { + "sq", "cq", "eq", "aeq", +}; + +static const char * const qm_fifo_overflow[] = { + "cq", "eq", "aeq", +}; + +struct qm_typical_qos_table { + u32 start; + u32 end; + u32 val; +}; + +/* the qos step is 100 */ +static struct qm_typical_qos_table shaper_cir_s[] = { + {100, 100, 4}, + {200, 200, 3}, + {300, 500, 2}, + {600, 1000, 1}, + {1100, 100000, 0}, +}; + +static struct qm_typical_qos_table shaper_cbs_s[] = { + {100, 200, 9}, + {300, 500, 11}, + {600, 1000, 12}, + {1100, 10000, 16}, + {10100, 25000, 17}, + {25100, 50000, 18}, + {50100, 100000, 19} +}; + +static void qm_irqs_unregister(struct hisi_qm *qm); +static int qm_reset_device(struct hisi_qm *qm); +static void hisi_qm_stop_qp(struct hisi_qp *qp); +static int qm_restart(struct hisi_qm *qm); + +int hisi_qm_q_num_set(const char *val, const struct kernel_param *kp, + unsigned int device) +{ + struct pci_dev *pdev; + u32 n, q_num; + int ret; + + if (!val) + return -EINVAL; + + pdev = pci_get_device(PCI_VENDOR_ID_HUAWEI, device, NULL); + if (!pdev) { + q_num = min_t(u32, QM_QNUM_V1, QM_QNUM_V2); + pr_info("No device found currently, suppose queue number is %u\n", + q_num); + } else { + if (pdev->revision == QM_HW_V1) + q_num = QM_QNUM_V1; + else + q_num = QM_QNUM_V2; + + pci_dev_put(pdev); + } + + ret = kstrtou32(val, 10, &n); + if (ret || n < QM_MIN_QNUM || n > q_num) + return -EINVAL; + + return param_set_int(val, kp); +} +EXPORT_SYMBOL_GPL(hisi_qm_q_num_set); + +static u32 qm_get_hw_error_status(struct hisi_qm *qm) +{ + return readl(qm->io_base + QM_ABNORMAL_INT_STATUS); +} + +static u32 qm_get_dev_err_status(struct hisi_qm *qm) +{ + return qm->err_ini->get_dev_hw_err_status(qm); +} + +/* Check if the error causes the master ooo block */ +static bool qm_check_dev_error(struct hisi_qm *qm) +{ + struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); + u32 err_status; + + if (pf_qm->fun_type == QM_HW_VF) + return false; + + err_status = qm_get_hw_error_status(pf_qm); + if (err_status & pf_qm->err_info.qm_err.shutdown_mask) + return true; + + if (pf_qm->err_ini->dev_is_abnormal) + return pf_qm->err_ini->dev_is_abnormal(pf_qm); + + return false; +} + +static int qm_wait_reset_finish(struct hisi_qm *qm) +{ + int delay = 0; + + /* All reset requests need to be queued for processing */ + while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { + msleep(++delay); + if (delay > QM_RESET_WAIT_TIMEOUT) + return -EBUSY; + } + + return 0; +} + +static void qm_fun_db_ctrl(struct hisi_qm *qm, bool enable) +{ + u32 val; + + if (qm->ver >= QM_HW_V5) { + val = readl(qm->io_base + QM_FUN_DB_DROP); + val = enable ? (val | BIT(0)) : (val & ~BIT(0)); + + writel(val, qm->io_base + QM_FUN_DB_DROP); + } +} + +static void qm_dev_db_ctrl(struct hisi_qm *qm, bool enable) +{ + u64 val; + + if (qm->ver >= QM_HW_V5 && qm->fun_type == QM_HW_PF) { + val = enable ? QM_DB_DROP_ALL_FUNC_ENABLE : QM_DB_DROP_ALL_FUNC_DISABLE; + + writeq(val, qm->io_base + QM_DEV_DB_DROP); + } +} + +static int qm_reset_prepare_ready(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev)); + + /* + * PF and VF on host doesnot support resetting at the + * same time on Kunpeng920. + */ + if (qm->ver < QM_HW_V3) + return qm_wait_reset_finish(pf_qm); + + return qm_wait_reset_finish(qm); +} + +static void qm_reset_bit_clear(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev)); + + if (qm->ver < QM_HW_V3) + clear_bit(QM_RESETTING, &pf_qm->misc_ctl); + + clear_bit(QM_RESETTING, &qm->misc_ctl); +} + +static void qm_mb_pre_init(struct qm_mailbox *mailbox, u8 cmd, + u64 base, u16 queue, bool op) +{ + mailbox->w0 = cpu_to_le16((cmd) | + ((op) ? 0x1 << QM_MB_OP_SHIFT : 0) | + (0x1 << QM_MB_BUSY_SHIFT)); + mailbox->queue_num = cpu_to_le16(queue); + mailbox->base_l = cpu_to_le32(lower_32_bits(base)); + mailbox->base_h = cpu_to_le32(upper_32_bits(base)); + mailbox->rsvd = 0; +} + +/* + * The mailbox is 128 bits and requires a single read/write operation. + * Since there is no general 128-bit IO memory access API in the current + * ARM64 architecture, this needs to be implemented in the driver. + */ +static struct qm_mailbox qm_mb_read(struct hisi_qm *qm) +{ + struct qm_mailbox mailbox = {0}; + +#if IS_ENABLED(CONFIG_ARM64) + const void __iomem *fun_base = qm->io_base + QM_MB_CMD_SEND_BASE; + unsigned long tmp0, tmp1; + + asm volatile("ldp %0, %1, %3\n" + "stp %0, %1, %2\n" + : "=&r" (tmp0), + "=&r" (tmp1), + "+Q" (mailbox) + : "Q" (*((char __iomem *)fun_base)) + : "memory"); +#endif + + return mailbox; +} + +/* 128 bit should be written to hardware at one time to trigger a mailbox */ +static void qm_mb_write(struct hisi_qm *qm, const void *src) +{ +#if IS_ENABLED(CONFIG_ARM64) + void __iomem *fun_base = qm->io_base + QM_MB_CMD_SEND_BASE; + unsigned long tmp0, tmp1; + /* + * The dmb oshst instruction ensures that the data in the + * mailbox is written before it is sent to the hardware. + */ + asm volatile("ldp %0, %1, %3\n" + "dmb oshst\n" + "stp %0, %1, %2\n" + : "=&r" (tmp0), + "=&r" (tmp1), + "+Q" (*((char __iomem *)fun_base)) + : "Q" (*((char *)src)) + : "memory"); +#endif +} + +int hisi_qm_wait_mb_ready(struct hisi_qm *qm) +{ + struct qm_mailbox mailbox = {0}; + int ret; + + ret = read_poll_timeout(qm_mb_read, mailbox, + !(le16_to_cpu(mailbox.w0) & QM_MB_BUSY_MASK), + POLL_PERIOD, POLL_TIMEOUT, + true, qm); + if (ret) + dev_err(&qm->pdev->dev, "QM mailbox is busy to start!\n"); + + return ret; +} +EXPORT_SYMBOL_GPL(hisi_qm_wait_mb_ready); + +static int qm_wait_mb_finish(struct hisi_qm *qm, struct qm_mailbox *mailbox, u32 wait_timeout) +{ + struct device *dev = &qm->pdev->dev; + int ret; + + ret = read_poll_timeout(qm_mb_read, *mailbox, + !(le16_to_cpu(mailbox->w0) & QM_MB_BUSY_MASK), + POLL_PERIOD, wait_timeout, + true, qm); + if (ret) { + dev_err(dev, "QM mailbox operation timeout!\n"); + return ret; + } + + if (le16_to_cpu(mailbox->w0) & QM_MB_STATUS_MASK) { + dev_err(dev, "QM mailbox operation failed!\n"); + return -EIO; + } + + return 0; +} + +static int qm_mb_nolock(struct hisi_qm *qm, struct qm_mailbox *mailbox, u32 wait_timeout) +{ + int ret; + + ret = hisi_qm_wait_mb_ready(qm); + if (ret) + goto mb_err_cnt_increase; + + qm_mb_write(qm, mailbox); + + ret = qm_wait_mb_finish(qm, mailbox, wait_timeout); + if (ret) + goto mb_err_cnt_increase; + + return 0; + +mb_err_cnt_increase: + atomic64_inc(&qm->debug.dfx.mb_err_cnt); + return ret; +} + +int hisi_qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue, + bool op) +{ + struct qm_mailbox mailbox; + u32 wait_timeout; + int ret; + + if (cmd == QM_MB_CMD_STOP_QP || cmd == QM_MB_CMD_FLUSH_QM) + wait_timeout = QM_MB_MAX_STOP_TIMEOUT; + else + wait_timeout = QM_MB_MAX_WAIT_TIMEOUT; + + /* No need to judge if master OOO is blocked. */ + if (qm_check_dev_error(qm)) { + dev_err(&qm->pdev->dev, "QM mailbox operation failed since qm is stop!\n"); + return -EIO; + } + + qm_mb_pre_init(&mailbox, cmd, dma_addr, queue, op); + + mutex_lock(&qm->mailbox_lock); + ret = qm_mb_nolock(qm, &mailbox, wait_timeout); + mutex_unlock(&qm->mailbox_lock); + + return ret; +} +EXPORT_SYMBOL_GPL(hisi_qm_mb); + +int hisi_qm_mb_read(struct hisi_qm *qm, u64 *base, u8 cmd, u16 queue) +{ + struct qm_mailbox mailbox; + int ret; + + qm_mb_pre_init(&mailbox, cmd, 0, queue, 1); + mutex_lock(&qm->mailbox_lock); + ret = qm_mb_nolock(qm, &mailbox, QM_MB_MAX_WAIT_TIMEOUT); + mutex_unlock(&qm->mailbox_lock); + if (ret) + return ret; + + *base = le32_to_cpu(mailbox.base_l) | + ((u64)le32_to_cpu(mailbox.base_h) << 32); + + return 0; +} +EXPORT_SYMBOL_GPL(hisi_qm_mb_read); + +/* op 0: set xqc information to hardware, 1: get xqc information from hardware. */ +int qm_set_and_get_xqc(struct hisi_qm *qm, u8 cmd, void *xqc, u32 qp_id, bool op) +{ + struct qm_mailbox mailbox; + dma_addr_t xqc_dma; + void *tmp_xqc; + size_t size; + int ret; + + switch (cmd) { + case QM_MB_CMD_SQC: + size = sizeof(struct qm_sqc); + tmp_xqc = qm->xqc_buf.sqc; + xqc_dma = qm->xqc_buf.sqc_dma; + break; + case QM_MB_CMD_CQC: + size = sizeof(struct qm_cqc); + tmp_xqc = qm->xqc_buf.cqc; + xqc_dma = qm->xqc_buf.cqc_dma; + break; + case QM_MB_CMD_EQC: + size = sizeof(struct qm_eqc); + tmp_xqc = qm->xqc_buf.eqc; + xqc_dma = qm->xqc_buf.eqc_dma; + break; + case QM_MB_CMD_AEQC: + size = sizeof(struct qm_aeqc); + tmp_xqc = qm->xqc_buf.aeqc; + xqc_dma = qm->xqc_buf.aeqc_dma; + break; + default: + dev_err(&qm->pdev->dev, "unknown mailbox cmd %u\n", cmd); + return -EINVAL; + } + + /* Setting xqc will fail if master OOO is blocked. */ + if (qm_check_dev_error(qm)) { + dev_err(&qm->pdev->dev, "failed to send mailbox since qm is stop!\n"); + return -EIO; + } + + mutex_lock(&qm->mailbox_lock); + if (!op) + memcpy(tmp_xqc, xqc, size); + + qm_mb_pre_init(&mailbox, cmd, xqc_dma, qp_id, op); + ret = qm_mb_nolock(qm, &mailbox, QM_MB_MAX_WAIT_TIMEOUT); + if (!ret && op) + memcpy(xqc, tmp_xqc, size); + + mutex_unlock(&qm->mailbox_lock); + + return ret; +} + +static void qm_db_v1(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority) +{ + u64 doorbell; + + doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V1) | + ((u64)index << QM_DB_INDEX_SHIFT_V1) | + ((u64)priority << QM_DB_PRIORITY_SHIFT_V1); + + writeq(doorbell, qm->io_base + QM_DOORBELL_BASE_V1); +} + +static void qm_db_v2(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority) +{ + void __iomem *io_base = qm->io_base; + u16 randata = 0; + u64 doorbell; + + if (cmd == QM_DOORBELL_CMD_SQ || cmd == QM_DOORBELL_CMD_CQ) + io_base = qm->db_io_base + (u64)qn * qm->db_interval + + QM_DOORBELL_SQ_CQ_BASE_V2; + else + io_base += QM_DOORBELL_EQ_AEQ_BASE_V2; + + doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V2) | + ((u64)randata << QM_DB_RAND_SHIFT_V2) | + ((u64)index << QM_DB_INDEX_SHIFT_V2) | + ((u64)priority << QM_DB_PRIORITY_SHIFT_V2); + + writeq(doorbell, io_base); +} + +static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority) +{ + dev_dbg(&qm->pdev->dev, "QM doorbell request: qn=%u, cmd=%u, index=%u\n", + qn, cmd, index); + + qm->ops->qm_db(qm, qn, cmd, index, priority); +} + +static void qm_disable_clock_gate(struct hisi_qm *qm) +{ + u32 val; + + /* if qm enables clock gating in Kunpeng930, qos will be inaccurate. */ + if (qm->ver < QM_HW_V3) + return; + + val = readl(qm->io_base + QM_PM_CTRL); + val |= QM_IDLE_DISABLE; + writel(val, qm->io_base + QM_PM_CTRL); +} + +static int qm_dev_mem_reset(struct hisi_qm *qm) +{ + u32 val; + + writel(0x1, qm->io_base + QM_MEM_START_INIT); + return readl_relaxed_poll_timeout(qm->io_base + QM_MEM_INIT_DONE, val, + val & BIT(0), POLL_PERIOD, + POLL_TIMEOUT); +} + +/** + * hisi_qm_get_hw_info() - Get device information. + * @qm: The qm which want to get information. + * @info_table: Array for storing device information. + * @index: Index in info_table. + * @is_read: Whether read from reg, 0: not support read from reg. + * + * This function returns device information the caller needs. + */ +u32 hisi_qm_get_hw_info(struct hisi_qm *qm, + const struct hisi_qm_cap_info *info_table, + u32 index, bool is_read) +{ + u32 val; + + switch (qm->ver) { + case QM_HW_V1: + return info_table[index].v1_val; + case QM_HW_V2: + return info_table[index].v2_val; + default: + if (!is_read) + return info_table[index].v3_val; + + val = readl(qm->io_base + info_table[index].offset); + return (val >> info_table[index].shift) & info_table[index].mask; + } +} +EXPORT_SYMBOL_GPL(hisi_qm_get_hw_info); + +u32 hisi_qm_get_cap_value(struct hisi_qm *qm, + const struct hisi_qm_cap_query_info *info_table, + u32 index, bool is_read) +{ + u32 val; + + switch (qm->ver) { + case QM_HW_V1: + return info_table[index].v1_val; + case QM_HW_V2: + return info_table[index].v2_val; + default: + if (!is_read) + return info_table[index].v3_val; + + val = readl(qm->io_base + info_table[index].offset); + return val; + } +} +EXPORT_SYMBOL_GPL(hisi_qm_get_cap_value); + +static void qm_get_xqc_depth(struct hisi_qm *qm, u16 *low_bits, + u16 *high_bits, enum qm_basic_type type) +{ + u32 depth; + + depth = hisi_qm_get_hw_info(qm, qm_basic_info, type, qm->cap_ver); + *low_bits = depth & QM_XQ_DEPTH_MASK; + *high_bits = (depth >> QM_XQ_DEPTH_SHIFT) & QM_XQ_DEPTH_MASK; +} + +int hisi_qm_set_algs(struct hisi_qm *qm, u64 alg_msk, const struct qm_dev_alg *dev_algs, + u32 dev_algs_size) +{ + struct device *dev = &qm->pdev->dev; + char *algs, *ptr; + int i; + + if (!qm->uacce) + return 0; + + if (dev_algs_size >= QM_DEV_ALG_MAX_LEN) { + dev_err(dev, "algs size %u is equal or larger than %d.\n", + dev_algs_size, QM_DEV_ALG_MAX_LEN); + return -EINVAL; + } + + algs = devm_kzalloc(dev, QM_DEV_ALG_MAX_LEN, GFP_KERNEL); + if (!algs) + return -ENOMEM; + + for (i = 0; i < dev_algs_size; i++) + if (alg_msk & dev_algs[i].alg_msk) + strcat(algs, dev_algs[i].alg); + + ptr = strrchr(algs, '\n'); + if (ptr) + *ptr = '\0'; + + qm->uacce->algs = algs; + + return 0; +} +EXPORT_SYMBOL_GPL(hisi_qm_set_algs); + +static u32 qm_get_irq_num(struct hisi_qm *qm) +{ + if (qm->fun_type == QM_HW_PF) + return hisi_qm_get_hw_info(qm, qm_basic_info, QM_PF_IRQ_NUM_CAP, qm->cap_ver); + + return hisi_qm_get_hw_info(qm, qm_basic_info, QM_VF_IRQ_NUM_CAP, qm->cap_ver); +} + +static int qm_pm_get_sync(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + int ret; + + if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) + return 0; + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) { + dev_err(dev, "failed to get_sync(%d).\n", ret); + return ret; + } + + return 0; +} + +static void qm_pm_put_sync(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + + if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) + return; + + pm_runtime_put_autosuspend(dev); +} + +static void qm_cq_head_update(struct hisi_qp *qp) +{ + if (qp->qp_status.cq_head == qp->cq_depth - 1) { + qp->qp_status.cqc_phase = !qp->qp_status.cqc_phase; + qp->qp_status.cq_head = 0; + } else { + qp->qp_status.cq_head++; + } +} + +static void qm_poll_req_cb(struct hisi_qp *qp) +{ + struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head; + struct hisi_qm *qm = qp->qm; + + while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) { + dma_rmb(); + qp->req_cb(qp, qp->sqe + qm->sqe_size * + le16_to_cpu(cqe->sq_head)); + qm_cq_head_update(qp); + cqe = qp->cqe + qp->qp_status.cq_head; + qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, + qp->qp_status.cq_head, 0); + atomic_dec(&qp->qp_status.used); + + cond_resched(); + } + + /* set c_flag */ + qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, qp->qp_status.cq_head, 1); +} + +static void qm_work_process(struct work_struct *work) +{ + struct hisi_qm_poll_data *poll_data = + container_of(work, struct hisi_qm_poll_data, work); + struct hisi_qm *qm = poll_data->qm; + u16 eqe_num = poll_data->eqe_num; + struct hisi_qp *qp; + int i; + + for (i = eqe_num - 1; i >= 0; i--) { + qp = &qm->qp_array[poll_data->qp_finish_id[i]]; + if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP)) + continue; + + if (qp->event_cb) { + qp->event_cb(qp); + continue; + } + + if (likely(qp->req_cb)) + qm_poll_req_cb(qp); + } +} + +static void qm_get_complete_eqe_num(struct hisi_qm *qm) +{ + struct qm_eqe *eqe = qm->eqe + qm->status.eq_head; + struct hisi_qm_poll_data *poll_data = NULL; + u32 dw0 = le32_to_cpu(eqe->dw0); + u16 eq_depth = qm->eq_depth; + u16 cqn, eqe_num = 0; + + if (QM_EQE_PHASE(dw0) != qm->status.eqc_phase) { + atomic64_inc(&qm->debug.dfx.err_irq_cnt); + qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); + return; + } + + cqn = dw0 & QM_EQE_CQN_MASK; + if (unlikely(cqn >= qm->qp_num)) + return; + poll_data = &qm->poll_data[cqn]; + + do { + poll_data->qp_finish_id[eqe_num] = dw0 & QM_EQE_CQN_MASK; + eqe_num++; + + if (qm->status.eq_head == eq_depth - 1) { + qm->status.eqc_phase = !qm->status.eqc_phase; + eqe = qm->eqe; + qm->status.eq_head = 0; + } else { + eqe++; + qm->status.eq_head++; + } + + dw0 = le32_to_cpu(eqe->dw0); + if (QM_EQE_PHASE(dw0) != qm->status.eqc_phase) + break; + } while (eqe_num < (eq_depth >> 1) - 1); + + poll_data->eqe_num = eqe_num; + queue_work(qm->wq, &poll_data->work); + qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); +} + +static irqreturn_t qm_eq_irq(int irq, void *data) +{ + struct hisi_qm *qm = data; + + /* Get qp id of completed tasks and re-enable the interrupt */ + qm_get_complete_eqe_num(qm); + + return IRQ_HANDLED; +} + +static irqreturn_t qm_mb_cmd_irq(int irq, void *data) +{ + struct hisi_qm *qm = data; + u32 val; + + val = readl(qm->io_base + QM_IFC_INT_STATUS); + val &= QM_IFC_INT_STATUS_MASK; + if (!val) + return IRQ_NONE; + + if (test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl)) { + dev_warn(&qm->pdev->dev, "Driver is down, message cannot be processed!\n"); + return IRQ_HANDLED; + } + + schedule_work(&qm->cmd_process); + + return IRQ_HANDLED; +} + +static void qm_set_qp_disable(struct hisi_qp *qp, int offset) +{ + u32 *addr; + + if (qp->is_in_kernel) + return; + + addr = (u32 *)(qp->qdma.va + qp->qdma.size) - offset; + *addr = 1; + + /* make sure setup is completed */ + smp_wmb(); +} + +static void qm_disable_qp(struct hisi_qm *qm, u32 qp_id) +{ + struct hisi_qp *qp = &qm->qp_array[qp_id]; + + qm_set_qp_disable(qp, QM_RESET_STOP_TX_OFFSET); + hisi_qm_stop_qp(qp); + qm_set_qp_disable(qp, QM_RESET_STOP_RX_OFFSET); +} + +static void qm_reset_function(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + int ret; + + if (qm_check_dev_error(qm)) + return; + + ret = qm_reset_prepare_ready(qm); + if (ret) { + dev_err(dev, "reset function not ready\n"); + return; + } + + dev_info(dev, "function reset start...\n"); + ret = hisi_qm_stop(qm, QM_DOWN); + if (ret) { + dev_err(dev, "failed to stop qm when reset function\n"); + goto clear_bit; + } + + ret = qm_restart(qm); + if (ret) + dev_err(dev, "failed to start qm when reset function\n"); + +clear_bit: + qm_reset_bit_clear(qm); + dev_info(dev, "function reset end...\n"); +} + +static irqreturn_t qm_aeq_thread(int irq, void *data) +{ + struct hisi_qm *qm = data; + struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head; + u32 dw0 = le32_to_cpu(aeqe->dw0); + u16 aeq_depth = qm->aeq_depth; + u32 type, qp_id; + + atomic64_inc(&qm->debug.dfx.aeq_irq_cnt); + + if (qm_pm_get_sync(qm)) { + dev_err(&qm->pdev->dev, "failed to get runtime PM for aeq handle\n"); + return IRQ_HANDLED; + } + + while (QM_AEQE_PHASE(dw0) == qm->status.aeqc_phase) { + type = (dw0 >> QM_AEQE_TYPE_SHIFT) & QM_AEQE_TYPE_MASK; + qp_id = dw0 & QM_AEQE_CQN_MASK; + + switch (type) { + case QM_EQ_OVERFLOW: + dev_err(&qm->pdev->dev, "eq overflow, reset function\n"); + qm_reset_function(qm); + return IRQ_HANDLED; + case QM_CQ_OVERFLOW: + dev_err(&qm->pdev->dev, "cq overflow, stop qp(%u)\n", + qp_id); + fallthrough; + case QM_CQE_ERROR: + qm_disable_qp(qm, qp_id); + break; + default: + dev_err(&qm->pdev->dev, "unknown error type %u\n", + type); + break; + } + + if (qm->status.aeq_head == aeq_depth - 1) { + qm->status.aeqc_phase = !qm->status.aeqc_phase; + aeqe = qm->aeqe; + qm->status.aeq_head = 0; + } else { + aeqe++; + qm->status.aeq_head++; + } + dw0 = le32_to_cpu(aeqe->dw0); + } + + qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0); + + qm_pm_put_sync(qm); + + return IRQ_HANDLED; +} + +static void qm_init_qp_status(struct hisi_qp *qp) +{ + struct hisi_qp_status *qp_status = &qp->qp_status; + + qp_status->sq_tail = 0; + qp_status->cq_head = 0; + qp_status->cqc_phase = true; + atomic_set(&qp_status->used, 0); +} + +static void qm_init_prefetch(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + u32 page_type = 0x0; + + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) + return; + + switch (PAGE_SIZE) { + case SZ_4K: + page_type = 0x0; + break; + case SZ_16K: + page_type = 0x1; + break; + case SZ_64K: + page_type = 0x2; + break; + default: + dev_err(dev, "system page size is not support: %lu, default set to 4KB", + PAGE_SIZE); + } + + writel(page_type, qm->io_base + QM_PAGE_SIZE); +} + +/* + * acc_shaper_para_calc() Get the IR value by the qos formula, the return value + * is the expected qos calculated. + * the formula: + * IR = X Mbps if ir = 1 means IR = 100 Mbps, if ir = 10000 means = 10Gbps + * + * IR_b * (2 ^ IR_u) * 8000 + * IR(Mbps) = ------------------------- + * Tick * (2 ^ IR_s) + */ +static u32 acc_shaper_para_calc(u64 cir_b, u64 cir_u, u64 cir_s) +{ + return ((cir_b * QM_QOS_DIVISOR_CLK) * (1 << cir_u)) / + (QM_QOS_TICK * (1 << cir_s)); +} + +static u32 acc_shaper_calc_cbs_s(u32 ir) +{ + int table_size = ARRAY_SIZE(shaper_cbs_s); + int i; + + for (i = 0; i < table_size; i++) { + if (ir >= shaper_cbs_s[i].start && ir <= shaper_cbs_s[i].end) + return shaper_cbs_s[i].val; + } + + return QM_SHAPER_MIN_CBS_S; +} + +static u32 acc_shaper_calc_cir_s(u32 ir) +{ + int table_size = ARRAY_SIZE(shaper_cir_s); + int i; + + for (i = 0; i < table_size; i++) { + if (ir >= shaper_cir_s[i].start && ir <= shaper_cir_s[i].end) + return shaper_cir_s[i].val; + } + + return 0; +} + +static int qm_get_shaper_para(u32 ir, struct qm_shaper_factor *factor) +{ + u32 cir_b, cir_u, cir_s, ir_calc; + u32 error_rate; + + factor->cbs_s = acc_shaper_calc_cbs_s(ir); + cir_s = acc_shaper_calc_cir_s(ir); + + for (cir_b = QM_QOS_MIN_CIR_B; cir_b <= QM_QOS_MAX_CIR_B; cir_b++) { + for (cir_u = 0; cir_u <= QM_QOS_MAX_CIR_U; cir_u++) { + ir_calc = acc_shaper_para_calc(cir_b, cir_u, cir_s); + + error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir; + if (error_rate <= QM_QOS_MIN_ERROR_RATE) { + factor->cir_b = cir_b; + factor->cir_u = cir_u; + factor->cir_s = cir_s; + return 0; + } + } + } + + return -EINVAL; +} + +static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base, + u32 number, struct qm_shaper_factor *factor) +{ + u64 tmp = 0; + + if (number > 0) { + switch (type) { + case SQC_VFT: + if (qm->ver == QM_HW_V1) { + tmp = QM_SQC_VFT_BUF_SIZE | + QM_SQC_VFT_SQC_SIZE | + QM_SQC_VFT_INDEX_NUMBER | + QM_SQC_VFT_VALID | + (u64)base << QM_SQC_VFT_START_SQN_SHIFT; + } else { + tmp = (u64)base << QM_SQC_VFT_START_SQN_SHIFT | + QM_SQC_VFT_VALID | + (u64)(number - 1) << QM_SQC_VFT_SQN_SHIFT; + } + break; + case CQC_VFT: + if (qm->ver == QM_HW_V1) { + tmp = QM_CQC_VFT_BUF_SIZE | + QM_CQC_VFT_SQC_SIZE | + QM_CQC_VFT_INDEX_NUMBER | + QM_CQC_VFT_VALID; + } else { + tmp = QM_CQC_VFT_VALID; + } + break; + case SHAPER_VFT: + if (factor) { + tmp = factor->cir_b | + (factor->cir_u << QM_SHAPER_FACTOR_CIR_U_SHIFT) | + (factor->cir_s << QM_SHAPER_FACTOR_CIR_S_SHIFT) | + (QM_SHAPER_CBS_B << QM_SHAPER_FACTOR_CBS_B_SHIFT) | + (factor->cbs_s << QM_SHAPER_FACTOR_CBS_S_SHIFT); + } + break; + /* + * Note: The current logic only needs to handle the above three types + * If new types are added, they need to be supplemented here, + * otherwise undefined behavior may occur. + */ + default: + break; + } + } + + writel(lower_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_L); + writel(upper_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_H); +} + +static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type, + u32 fun_num, u32 base, u32 number) +{ + struct qm_shaper_factor *factor = NULL; + unsigned int val; + int ret; + + if (type == SHAPER_VFT && test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) + factor = &qm->factor[fun_num]; + + ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, + val & BIT(0), POLL_PERIOD, + POLL_TIMEOUT); + if (ret) + return ret; + + writel(0x0, qm->io_base + QM_VFT_CFG_OP_WR); + writel(type, qm->io_base + QM_VFT_CFG_TYPE); + if (type == SHAPER_VFT) + fun_num |= base << QM_SHAPER_VFT_OFFSET; + + writel(fun_num, qm->io_base + QM_VFT_CFG); + + qm_vft_data_cfg(qm, type, base, number, factor); + + writel(0x0, qm->io_base + QM_VFT_CFG_RDY); + writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE); + + return readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, + val & BIT(0), POLL_PERIOD, + POLL_TIMEOUT); +} + +static int qm_shaper_init_vft(struct hisi_qm *qm, u32 fun_num) +{ + u32 qos = qm->factor[fun_num].func_qos; + int ret, i; + + ret = qm_get_shaper_para(qos * QM_QOS_RATE, &qm->factor[fun_num]); + if (ret) { + dev_err(&qm->pdev->dev, "failed to calculate shaper parameter!\n"); + return ret; + } + writel(qm->type_rate, qm->io_base + QM_SHAPER_CFG); + for (i = ALG_TYPE_0; i <= ALG_TYPE_1; i++) { + /* The base number of queue reuse for different alg type */ + ret = qm_set_vft_common(qm, SHAPER_VFT, fun_num, i, 1); + if (ret) + return ret; + } + + return 0; +} + +/* The config should be conducted after qm_dev_mem_reset() */ +static int qm_set_sqc_cqc_vft(struct hisi_qm *qm, u32 fun_num, u32 base, + u32 number) +{ + int ret, i; + + for (i = SQC_VFT; i <= CQC_VFT; i++) { + ret = qm_set_vft_common(qm, i, fun_num, base, number); + if (ret) + return ret; + } + + /* init default shaper qos val */ + if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) { + ret = qm_shaper_init_vft(qm, fun_num); + if (ret) + goto back_sqc_cqc; + } + + return 0; +back_sqc_cqc: + for (i = SQC_VFT; i <= CQC_VFT; i++) + qm_set_vft_common(qm, i, fun_num, 0, 0); + + return ret; +} + +static int qm_get_vft_v2(struct hisi_qm *qm, u32 *base, u32 *number) +{ + u64 sqc_vft; + int ret; + + ret = hisi_qm_mb_read(qm, &sqc_vft, QM_MB_CMD_SQC_VFT_V2, 0); + if (ret) + return ret; + + *base = QM_SQC_VFT_BASE_MASK_V2 & (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2); + *number = (QM_SQC_VFT_NUM_MASK_V2 & + (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1; + + return 0; +} + +static void qm_hw_error_init_v1(struct hisi_qm *qm) +{ + writel(QM_RAS_MASK_ALL, qm->io_base + QM_ABNORMAL_INT_MASK); +} + +static void qm_hw_error_cfg(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *qm_err = &qm->err_info.qm_err; + + qm->error_mask = qm_err->nfe | qm_err->ce | qm_err->fe; + /* clear QM hw residual error source */ + writel(QM_RAS_CLEAR_ALL, qm->io_base + QM_ABNORMAL_INT_SOURCE); + if (qm->ver >= QM_HW_V5) + writeq(QM_FUNC_RAS_CLEAR_ALL, qm->io_base + QM_FUNC_AXI_ERR_ST0); + + /* configure error type */ + writel(qm_err->ce, qm->io_base + QM_RAS_CE_ENABLE); + writel(QM_RAS_CE_TIMES_PER_IRQ, qm->io_base + QM_RAS_CE_THRESHOLD); + writel(qm_err->nfe, qm->io_base + QM_RAS_NFE_ENABLE); + writel(qm_err->fe, qm->io_base + QM_RAS_FE_ENABLE); +} + +static void qm_hw_error_init_v2(struct hisi_qm *qm) +{ + qm_hw_error_cfg(qm); + + writel(~qm->error_mask, qm->io_base + QM_ABNORMAL_INT_MASK); +} + +static void qm_hw_error_uninit_v2(struct hisi_qm *qm) +{ + writel(QM_RAS_MASK_ALL, qm->io_base + QM_ABNORMAL_INT_MASK); +} + +static void qm_hw_error_init_v3(struct hisi_qm *qm) +{ + qm_hw_error_cfg(qm); + + /* enable close master ooo when hardware error happened */ + writel(qm->err_info.qm_err.shutdown_mask, qm->io_base + QM_OOO_SHUTDOWN_SEL); + writel(~qm->error_mask, qm->io_base + QM_ABNORMAL_INT_MASK); +} + +static void qm_hw_error_uninit_v3(struct hisi_qm *qm) +{ + writel(QM_RAS_MASK_ALL, qm->io_base + QM_ABNORMAL_INT_MASK); + + /* disable close master ooo when hardware error happened */ + writel(0x0, qm->io_base + QM_OOO_SHUTDOWN_SEL); +} + +static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status) +{ + const struct hisi_qm_hw_error *err; + struct device *dev = &qm->pdev->dev; + u32 reg_val, type, vf_num, qp_id; + int i; + + for (i = 0; i < ARRAY_SIZE(qm_hw_error); i++) { + err = &qm_hw_error[i]; + if (!(err->int_msk & error_status)) + continue; + + dev_err(dev, "%s [error status=0x%x] found\n", + err->msg, err->int_msk); + + if (err->int_msk & QM_DB_TIMEOUT) { + reg_val = readl(qm->io_base + QM_ABNORMAL_INF01); + type = (reg_val & QM_DB_TIMEOUT_TYPE) >> + QM_DB_TIMEOUT_TYPE_SHIFT; + vf_num = reg_val & QM_DB_TIMEOUT_VF; + qp_id = reg_val >> QM_DB_TIMEOUT_QP_SHIFT; + dev_err(dev, "qm %s doorbell timeout in function %u qp %u\n", + qm_db_timeout[type], vf_num, qp_id); + } else if (err->int_msk & QM_OF_FIFO_OF) { + reg_val = readl(qm->io_base + QM_ABNORMAL_INF00); + type = (reg_val & QM_FIFO_OVERFLOW_TYPE) >> + QM_FIFO_OVERFLOW_TYPE_SHIFT; + vf_num = reg_val & QM_FIFO_OVERFLOW_VF; + qp_id = reg_val >> QM_FIFO_OVERFLOW_QP_SHIFT; + if (type < ARRAY_SIZE(qm_fifo_overflow)) + dev_err(dev, "qm %s fifo overflow in function %u qp %u\n", + qm_fifo_overflow[type], vf_num, qp_id); + else + dev_err(dev, "unknown error type\n"); + } else if (err->int_msk & QM_AXI_RRESP_ERR) { + reg_val = readl(qm->io_base + QM_ABNORMAL_INF02); + if (reg_val & QM_AXI_POISON_ERR) + dev_err(dev, "qm axi poison error happened\n"); + } + } +} + +static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *qm_err = &qm->err_info.qm_err; + u32 error_status; + + error_status = qm_get_hw_error_status(qm); + if (error_status & qm->error_mask) { + if (error_status & QM_ECC_MBIT) + qm->err_status.is_qm_ecc_mbit = true; + + qm_log_hw_error(qm, error_status); + /* Trigger func reset only when error is detected in bit 0 or bit 1. */ + if ((qm->ver >= QM_HW_V5) && + (error_status & QM_RAS_FUNC_ERROR) && + (error_status & qm_err->reset_mask) == 0) { + writel(error_status, qm->io_base + QM_ABNORMAL_INT_SOURCE); + writel(qm_err->nfe, qm->io_base + QM_RAS_NFE_ENABLE); + return ACC_ERR_NEED_FUNC_RESET; + } + + if (error_status & qm_err->reset_mask) { + /* Disable the same error reporting until device is recovered. */ + writel(qm_err->nfe & (~error_status), qm->io_base + QM_RAS_NFE_ENABLE); + return ACC_ERR_NEED_RESET; + } + + /* Clear error source if not need reset. */ + writel(error_status, qm->io_base + QM_ABNORMAL_INT_SOURCE); + writel(qm_err->nfe, qm->io_base + QM_RAS_NFE_ENABLE); + writel(qm_err->ce, qm->io_base + QM_RAS_CE_ENABLE); + } + + return ACC_ERR_RECOVERED; +} + +static void qm_clear_cmd_interrupt(struct hisi_qm *qm, u64 vf_mask) +{ + u32 val; + + if (qm->fun_type == QM_HW_PF) + writeq(vf_mask, qm->io_base + QM_IFC_INT_SOURCE_P); + + val = readl(qm->io_base + QM_IFC_INT_SOURCE_V); + val |= QM_IFC_INT_SOURCE_MASK; + writel(val, qm->io_base + QM_IFC_INT_SOURCE_V); +} + +static void qm_handle_vf_msg(struct hisi_qm *qm, u32 vf_id) +{ + struct device *dev = &qm->pdev->dev; + enum qm_ifc_cmd cmd; + int ret; + + ret = qm->ops->get_ifc(qm, &cmd, NULL, vf_id); + if (ret) { + dev_err(dev, "failed to get command from VF(%u)!\n", vf_id); + return; + } + + switch (cmd) { + case QM_VF_PREPARE_FAIL: + dev_err(dev, "failed to stop VF(%u)!\n", vf_id); + break; + case QM_VF_START_FAIL: + dev_err(dev, "failed to start VF(%u)!\n", vf_id); + break; + case QM_VF_PREPARE_DONE: + case QM_VF_START_DONE: + break; + default: + dev_err(dev, "unsupported command(0x%x) sent by VF(%u)!\n", cmd, vf_id); + break; + } +} + +static int qm_wait_vf_prepare_finish(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + u32 vfs_num = qm->vfs_num; + int cnt = 0; + int ret = 0; + u64 val; + u32 i; + + if (!qm->vfs_num || !test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) + return 0; + + while (true) { + val = readq(qm->io_base + QM_IFC_INT_SOURCE_P); + /* All VFs send command to PF, break */ + if ((val & GENMASK(vfs_num, 1)) == GENMASK(vfs_num, 1)) + break; + + if (++cnt > QM_MAX_PF_WAIT_COUNT) { + ret = -EBUSY; + break; + } + + msleep(QM_WAIT_DST_ACK); + } + + /* PF check VFs msg */ + for (i = 1; i <= vfs_num; i++) { + if (val & BIT(i)) + qm_handle_vf_msg(qm, i); + else + dev_err(dev, "VF(%u) not ping PF!\n", i); + } + + /* PF clear interrupt to ack VFs */ + qm_clear_cmd_interrupt(qm, val); + + return ret; +} + +static void qm_trigger_vf_interrupt(struct hisi_qm *qm, u32 fun_num) +{ + u32 val; + + val = readl(qm->io_base + QM_IFC_INT_CFG); + val &= ~QM_IFC_SEND_ALL_VFS; + val |= fun_num; + writel(val, qm->io_base + QM_IFC_INT_CFG); + + val = readl(qm->io_base + QM_IFC_INT_SET_P); + val |= QM_IFC_INT_SET_MASK; + writel(val, qm->io_base + QM_IFC_INT_SET_P); +} + +static void qm_trigger_pf_interrupt(struct hisi_qm *qm) +{ + u32 val; + + val = readl(qm->io_base + QM_IFC_INT_SET_V); + val |= QM_IFC_INT_SET_MASK; + writel(val, qm->io_base + QM_IFC_INT_SET_V); +} + +static int qm_ping_single_vf(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num) +{ + struct device *dev = &qm->pdev->dev; + int cnt = 0; + u64 val; + int ret; + + ret = qm->ops->set_ifc_begin(qm, cmd, data, fun_num); + if (ret) { + dev_err(dev, "failed to send command to vf(%u)!\n", fun_num); + goto err_unlock; + } + + qm_trigger_vf_interrupt(qm, fun_num); + while (true) { + msleep(QM_WAIT_DST_ACK); + val = readq(qm->io_base + QM_IFC_READY_STATUS); + /* if VF respond, PF notifies VF successfully. */ + if (!(val & BIT(fun_num))) + goto err_unlock; + + if (++cnt > QM_MAX_PF_WAIT_COUNT) { + dev_err(dev, "failed to get response from VF(%u)!\n", fun_num); + ret = -ETIMEDOUT; + break; + } + } + +err_unlock: + qm->ops->set_ifc_end(qm); + return ret; +} + +static int qm_ping_all_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data) +{ + struct device *dev = &qm->pdev->dev; + u32 vfs_num = qm->vfs_num; + u64 val = 0; + int cnt = 0; + int ret; + u32 i; + + ret = qm->ops->set_ifc_begin(qm, cmd, data, QM_MB_PING_ALL_VFS); + if (ret) { + dev_err(dev, "failed to send command(0x%x) to all vfs!\n", cmd); + qm->ops->set_ifc_end(qm); + return ret; + } + + qm_trigger_vf_interrupt(qm, QM_IFC_SEND_ALL_VFS); + while (true) { + msleep(QM_WAIT_DST_ACK); + val = readq(qm->io_base + QM_IFC_READY_STATUS); + /* If all VFs acked, PF notifies VFs successfully. */ + if (!(val & GENMASK(vfs_num, 1))) { + qm->ops->set_ifc_end(qm); + return 0; + } + + if (++cnt > QM_MAX_PF_WAIT_COUNT) + break; + } + + qm->ops->set_ifc_end(qm); + + /* Check which vf respond timeout. */ + for (i = 1; i <= vfs_num; i++) { + if (val & BIT(i)) + dev_err(dev, "failed to get response from VF(%u)!\n", i); + } + + return -ETIMEDOUT; +} + +static int qm_ping_pf(struct hisi_qm *qm, enum qm_ifc_cmd cmd) +{ + int cnt = 0; + u32 val; + int ret; + + ret = qm->ops->set_ifc_begin(qm, cmd, 0, 0); + if (ret) { + dev_err(&qm->pdev->dev, "failed to send command(0x%x) to PF!\n", cmd); + goto unlock; + } + + qm_trigger_pf_interrupt(qm); + /* Waiting for PF response */ + while (true) { + msleep(QM_WAIT_DST_ACK); + val = readl(qm->io_base + QM_IFC_INT_SET_V); + if (!(val & QM_IFC_INT_STATUS_MASK)) + break; + + if (++cnt > QM_MAX_VF_WAIT_COUNT) { + ret = -ETIMEDOUT; + break; + } + } + +unlock: + qm->ops->set_ifc_end(qm); + + return ret; +} + +static int qm_drain_qm(struct hisi_qm *qm) +{ + return hisi_qm_mb(qm, QM_MB_CMD_FLUSH_QM, 0, 0, 0); +} + +static int qm_stop_qp(struct hisi_qp *qp) +{ + return hisi_qm_mb(qp->qm, QM_MB_CMD_STOP_QP, 0, qp->qp_id, 0); +} + +static int qm_set_msi(struct hisi_qm *qm, bool set) +{ + struct pci_dev *pdev = qm->pdev; + + if (set) { + pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64, + 0); + } else { + pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64, + ACC_PEH_MSI_DISABLE); + if (qm->err_status.is_qm_ecc_mbit || + qm->err_status.is_dev_ecc_mbit) + return 0; + + mdelay(1); + if (readl(qm->io_base + QM_PEH_DFX_INFO0)) + return -EFAULT; + } + + return 0; +} + +static void qm_wait_msi_finish(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 cmd = ~0; + int cnt = 0; + u32 val; + int ret; + + while (true) { + pci_read_config_dword(pdev, pdev->msi_cap + + PCI_MSI_PENDING_64, &cmd); + if (!cmd) + break; + + if (++cnt > MAX_WAIT_COUNTS) { + pci_warn(pdev, "failed to empty MSI PENDING!\n"); + break; + } + + udelay(1); + } + + ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_DFX_INFO0, + val, !(val & QM_PEH_DFX_MASK), + POLL_PERIOD, POLL_TIMEOUT); + if (ret) + pci_warn(pdev, "failed to empty PEH MSI!\n"); + + ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_DFX_INFO1, + val, !(val & QM_PEH_MSI_FINISH_MASK), + POLL_PERIOD, POLL_TIMEOUT); + if (ret) + pci_warn(pdev, "failed to finish MSI operation!\n"); +} + +static int qm_set_msi_v3(struct hisi_qm *qm, bool set) +{ + struct pci_dev *pdev = qm->pdev; + int ret = -ETIMEDOUT; + u32 cmd, i; + + pci_read_config_dword(pdev, pdev->msi_cap, &cmd); + if (set) + cmd |= QM_MSI_CAP_ENABLE; + else + cmd &= ~QM_MSI_CAP_ENABLE; + + pci_write_config_dword(pdev, pdev->msi_cap, cmd); + if (set) { + for (i = 0; i < MAX_WAIT_COUNTS; i++) { + pci_read_config_dword(pdev, pdev->msi_cap, &cmd); + if (cmd & QM_MSI_CAP_ENABLE) + return 0; + + udelay(1); + } + } else { + udelay(WAIT_PERIOD_US_MIN); + qm_wait_msi_finish(qm); + ret = 0; + } + + return ret; +} + +static int qm_set_ifc_begin_v3(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num) +{ + struct qm_mailbox mailbox; + u64 msg; + + msg = cmd | (u64)data << QM_IFC_DATA_SHIFT; + + qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, msg, fun_num, 0); + mutex_lock(&qm->mailbox_lock); + return qm_mb_nolock(qm, &mailbox, QM_MB_MAX_WAIT_TIMEOUT); +} + +static void qm_set_ifc_end_v3(struct hisi_qm *qm) +{ + mutex_unlock(&qm->mailbox_lock); +} + +static int qm_get_ifc_v3(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num) +{ + u64 msg; + int ret; + + ret = hisi_qm_mb_read(qm, &msg, QM_MB_CMD_DST, fun_num); + if (ret) + return ret; + + *cmd = msg & QM_IFC_CMD_MASK; + + if (data) + *data = msg >> QM_IFC_DATA_SHIFT; + + return 0; +} + +static int qm_set_ifc_begin_v4(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num) +{ + uintptr_t offset; + u64 msg; + + if (qm->fun_type == QM_HW_PF) + offset = QM_PF2VF_PF_W; + else + offset = QM_VF2PF_VF_W; + + msg = cmd | (u64)data << QM_IFC_DATA_SHIFT; + + mutex_lock(&qm->ifc_lock); + writeq(msg, qm->io_base + offset); + + return 0; +} + +static void qm_set_ifc_end_v4(struct hisi_qm *qm) +{ + mutex_unlock(&qm->ifc_lock); +} + +static u64 qm_get_ifc_pf(struct hisi_qm *qm, u32 fun_num) +{ + uintptr_t offset; + + offset = QM_VF2PF_PF_R + QM_VF2PF_REG_SIZE * fun_num; + + return (u64)readl(qm->io_base + offset); +} + +static u64 qm_get_ifc_vf(struct hisi_qm *qm) +{ + return readq(qm->io_base + QM_PF2VF_VF_R); +} + +static int qm_get_ifc_v4(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num) +{ + u64 msg; + + if (qm->fun_type == QM_HW_PF) + msg = qm_get_ifc_pf(qm, fun_num); + else + msg = qm_get_ifc_vf(qm); + + *cmd = msg & QM_IFC_CMD_MASK; + + if (data) + *data = msg >> QM_IFC_DATA_SHIFT; + + return 0; +} + +static const struct hisi_qm_hw_ops qm_hw_ops_v1 = { + .qm_db = qm_db_v1, + .hw_error_init = qm_hw_error_init_v1, + .set_msi = qm_set_msi, +}; + +static const struct hisi_qm_hw_ops qm_hw_ops_v2 = { + .get_vft = qm_get_vft_v2, + .qm_db = qm_db_v2, + .hw_error_init = qm_hw_error_init_v2, + .hw_error_uninit = qm_hw_error_uninit_v2, + .hw_error_handle = qm_hw_error_handle_v2, + .set_msi = qm_set_msi, +}; + +static const struct hisi_qm_hw_ops qm_hw_ops_v3 = { + .get_vft = qm_get_vft_v2, + .qm_db = qm_db_v2, + .hw_error_init = qm_hw_error_init_v3, + .hw_error_uninit = qm_hw_error_uninit_v3, + .hw_error_handle = qm_hw_error_handle_v2, + .set_msi = qm_set_msi_v3, + .set_ifc_begin = qm_set_ifc_begin_v3, + .set_ifc_end = qm_set_ifc_end_v3, + .get_ifc = qm_get_ifc_v3, +}; + +static const struct hisi_qm_hw_ops qm_hw_ops_v4 = { + .get_vft = qm_get_vft_v2, + .qm_db = qm_db_v2, + .hw_error_init = qm_hw_error_init_v3, + .hw_error_uninit = qm_hw_error_uninit_v3, + .hw_error_handle = qm_hw_error_handle_v2, + .set_msi = qm_set_msi_v3, + .set_ifc_begin = qm_set_ifc_begin_v4, + .set_ifc_end = qm_set_ifc_end_v4, + .get_ifc = qm_get_ifc_v4, +}; + +static void *qm_get_avail_sqe(struct hisi_qp *qp) +{ + struct hisi_qp_status *qp_status = &qp->qp_status; + u16 sq_tail = qp_status->sq_tail; + + if (unlikely(atomic_read(&qp->qp_status.used) == qp->sq_depth - 1)) + return NULL; + + return qp->sqe + sq_tail * qp->qm->sqe_size; +} + +static void hisi_qm_unset_hw_reset(struct hisi_qp *qp) +{ + u64 *addr; + + /* Use last 64 bits of DUS to reset status. */ + addr = (u64 *)(qp->qdma.va + qp->qdma.size) - QM_RESET_STOP_TX_OFFSET; + *addr = 0; +} + +static struct hisi_qp *find_shareable_qp(struct hisi_qm *qm, u8 alg_type, bool is_in_kernel) +{ + struct device *dev = &qm->pdev->dev; + struct hisi_qp *share_qp = NULL; + struct hisi_qp *qp; + u32 ref_count = ~0; + int i; + + if (!is_in_kernel) + goto queues_busy; + + for (i = 0; i < qm->qp_num; i++) { + qp = &qm->qp_array[i]; + if (qp->is_in_kernel && qp->alg_type == alg_type && qp->ref_count < ref_count) { + ref_count = qp->ref_count; + share_qp = qp; + } + } + + if (share_qp) { + share_qp->ref_count++; + return share_qp; + } + +queues_busy: + dev_info_ratelimited(dev, "All %u queues of QM are busy and no shareable queue\n", + qm->qp_num); + atomic64_inc(&qm->debug.dfx.create_qp_err_cnt); + return ERR_PTR(-EBUSY); +} + +static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type, bool is_in_kernel) +{ + struct device *dev = &qm->pdev->dev; + struct hisi_qp *qp; + int qp_id; + + if (atomic_read(&qm->status.flags) == QM_STOP) { + dev_info_ratelimited(dev, "failed to create qp as qm is stop!\n"); + return ERR_PTR(-EPERM); + } + + /* Try to find a shareable queue when all queues are busy */ + if (qm->qp_in_used == qm->qp_num) + return find_shareable_qp(qm, alg_type, is_in_kernel); + + qp_id = idr_alloc_cyclic(&qm->qp_idr, NULL, 0, qm->qp_num, GFP_ATOMIC); + if (qp_id < 0) { + dev_info_ratelimited(dev, "All %u queues of QM are busy!\n", + qm->qp_num); + atomic64_inc(&qm->debug.dfx.create_qp_err_cnt); + return ERR_PTR(-EBUSY); + } + + qp = &qm->qp_array[qp_id]; + hisi_qm_unset_hw_reset(qp); + memset(qp->cqe, 0, sizeof(struct qm_cqe) * qp->cq_depth); + + qp->event_cb = NULL; + qp->req_cb = NULL; + qp->alg_type = alg_type; + qp->is_in_kernel = is_in_kernel; + qm->qp_in_used++; + qp->ref_count = 1; + + return qp; +} + +/** + * hisi_qm_create_qp() - Create a queue pair from qm. + * @qm: The qm we create a qp from. + * @alg_type: Accelerator specific algorithm type in sqc. + * + * Return created qp, negative error code if failed. + */ +static struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type) +{ + struct hisi_qp *qp; + int ret; + + ret = qm_pm_get_sync(qm); + if (ret) + return ERR_PTR(ret); + + down_write(&qm->qps_lock); + qp = qm_create_qp_nolock(qm, alg_type, false); + up_write(&qm->qps_lock); + + if (IS_ERR(qp)) + qm_pm_put_sync(qm); + + return qp; +} + +/** + * hisi_qm_release_qp() - Release a qp back to its qm. + * @qp: The qp we want to release. + * + * This function releases the resource of a qp. + */ +static void hisi_qm_release_qp(struct hisi_qp *qp) +{ + struct hisi_qm *qm = qp->qm; + + down_write(&qm->qps_lock); + + qm->qp_in_used--; + idr_remove(&qm->qp_idr, qp->qp_id); + + up_write(&qm->qps_lock); + + qm_pm_put_sync(qm); +} + +static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) +{ + struct hisi_qm *qm = qp->qm; + enum qm_hw_ver ver = qm->ver; + struct qm_sqc sqc = {0}; + + if (ver == QM_HW_V1) { + sqc.dw3 = cpu_to_le32(QM_MK_SQC_DW3_V1(0, 0, 0, qm->sqe_size)); + sqc.w8 = cpu_to_le16(qp->sq_depth - 1); + } else { + sqc.dw3 = cpu_to_le32(QM_MK_SQC_DW3_V2(qm->sqe_size, qp->sq_depth)); + sqc.w8 = 0; /* rand_qc */ + } + sqc.w13 = cpu_to_le16(QM_MK_SQC_W13(0, 1, qp->alg_type)); + sqc.base_l = cpu_to_le32(lower_32_bits(qp->sqe_dma)); + sqc.base_h = cpu_to_le32(upper_32_bits(qp->sqe_dma)); + sqc.cq_num = cpu_to_le16(qp_id); + sqc.pasid = cpu_to_le16(pasid); + + if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel) + sqc.w11 = cpu_to_le16(QM_QC_PASID_ENABLE << + QM_QC_PASID_ENABLE_SHIFT); + + return qm_set_and_get_xqc(qm, QM_MB_CMD_SQC, &sqc, qp_id, 0); +} + +static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) +{ + struct hisi_qm *qm = qp->qm; + enum qm_hw_ver ver = qm->ver; + struct qm_cqc cqc = {0}; + + if (ver == QM_HW_V1) { + cqc.dw3 = cpu_to_le32(QM_MK_CQC_DW3_V1(0, 0, 0, QM_QC_CQE_SIZE)); + cqc.w8 = cpu_to_le16(qp->cq_depth - 1); + } else { + cqc.dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(QM_QC_CQE_SIZE, qp->cq_depth)); + cqc.w8 = 0; /* rand_qc */ + } + /* + * Enable request finishing interrupts defaultly. + * So, there will be some interrupts until disabling + * this. + */ + cqc.dw6 = cpu_to_le32(1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT); + cqc.base_l = cpu_to_le32(lower_32_bits(qp->cqe_dma)); + cqc.base_h = cpu_to_le32(upper_32_bits(qp->cqe_dma)); + cqc.pasid = cpu_to_le16(pasid); + + if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel) + cqc.w11 = cpu_to_le16(QM_QC_PASID_ENABLE); + + return qm_set_and_get_xqc(qm, QM_MB_CMD_CQC, &cqc, qp_id, 0); +} + +static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) +{ + int ret; + + qm_init_qp_status(qp); + + ret = qm_sq_ctx_cfg(qp, qp_id, pasid); + if (ret) + return ret; + + return qm_cq_ctx_cfg(qp, qp_id, pasid); +} + +static int qm_start_qp_nolock(struct hisi_qp *qp, unsigned long arg) +{ + struct hisi_qm *qm = qp->qm; + struct device *dev = &qm->pdev->dev; + int qp_id = qp->qp_id; + u32 pasid = arg; + int ret; + + if (atomic_read(&qm->status.flags) == QM_STOP) { + dev_info_ratelimited(dev, "failed to start qp as qm is stop!\n"); + return -EPERM; + } + + ret = qm_qp_ctx_cfg(qp, qp_id, pasid); + if (ret) + return ret; + + atomic_set(&qp->qp_status.flags, QP_START); + dev_dbg(dev, "queue %d started\n", qp_id); + + return 0; +} + +/** + * hisi_qm_start_qp() - Start a qp into running. + * @qp: The qp we want to start to run. + * @arg: Accelerator specific argument. + * + * After this function, qp can receive request from user. Return 0 if + * successful, negative error code if failed. + */ +static int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg) +{ + struct hisi_qm *qm = qp->qm; + int ret; + + down_write(&qm->qps_lock); + ret = qm_start_qp_nolock(qp, arg); + up_write(&qm->qps_lock); + + return ret; +} + +/** + * qp_stop_fail_cb() - call request cb. + * @qp: stopped failed qp. + * + * Callback function should be called whether task completed or not. + */ +static void qp_stop_fail_cb(struct hisi_qp *qp) +{ + int qp_used = atomic_read(&qp->qp_status.used); + u16 cur_tail = qp->qp_status.sq_tail; + u16 sq_depth = qp->sq_depth; + u16 cur_head = (cur_tail + sq_depth - qp_used) % sq_depth; + struct hisi_qm *qm = qp->qm; + u16 pos; + int i; + + for (i = 0; i < qp_used; i++) { + pos = (i + cur_head) % sq_depth; + qp->req_cb(qp, qp->sqe + (u32)(qm->sqe_size * pos)); + qm_cq_head_update(qp); + atomic_dec(&qp->qp_status.used); + } +} + +static int qm_wait_qp_empty(struct hisi_qm *qm, u32 *state, u32 qp_id) +{ + struct device *dev = &qm->pdev->dev; + struct qm_sqc sqc; + struct qm_cqc cqc; + int ret, i = 0; + + while (++i) { + ret = qm_set_and_get_xqc(qm, QM_MB_CMD_SQC, &sqc, qp_id, 1); + if (ret) { + dev_err_ratelimited(dev, "Failed to dump sqc!\n"); + *state = QM_DUMP_SQC_FAIL; + return ret; + } + + ret = qm_set_and_get_xqc(qm, QM_MB_CMD_CQC, &cqc, qp_id, 1); + if (ret) { + dev_err_ratelimited(dev, "Failed to dump cqc!\n"); + *state = QM_DUMP_CQC_FAIL; + return ret; + } + + if ((sqc.tail == cqc.tail) && + (QM_SQ_TAIL_IDX(sqc) == QM_CQ_TAIL_IDX(cqc))) + break; + + if (i == MAX_WAIT_COUNTS) { + dev_err(dev, "Fail to empty queue %u!\n", qp_id); + *state = QM_STOP_QUEUE_FAIL; + return -ETIMEDOUT; + } + + usleep_range(WAIT_PERIOD_US_MIN, WAIT_PERIOD_US_MAX); + } + + return 0; +} + +/** + * qm_drain_qp() - Drain a qp. + * @qp: The qp we want to drain. + * + * If the device does not support stopping queue by sending mailbox, + * determine whether the queue is cleared by judging the tail pointers of + * sq and cq. + */ +static int qm_drain_qp(struct hisi_qp *qp) +{ + struct hisi_qm *qm = qp->qm; + u32 state = 0; + int ret; + + /* No need to judge if master OOO is blocked. */ + if (qm_check_dev_error(qm)) + return 0; + + /* HW V3 supports drain qp by device */ + if (test_bit(QM_SUPPORT_STOP_QP, &qm->caps)) { + ret = qm_stop_qp(qp); + if (ret) { + dev_err(&qm->pdev->dev, "Failed to stop qp!\n"); + state = QM_STOP_QUEUE_FAIL; + goto set_dev_state; + } + return ret; + } + + ret = qm_wait_qp_empty(qm, &state, qp->qp_id); + if (ret) + goto set_dev_state; + + return 0; + +set_dev_state: + if (qm->debug.dev_dfx.dev_timeout) + qm->debug.dev_dfx.dev_state = state; + + return ret; +} + +static void qm_stop_qp_nolock(struct hisi_qp *qp) +{ + struct hisi_qm *qm = qp->qm; + struct device *dev = &qm->pdev->dev; + int ret; + + /* + * It is allowed to stop and release qp when reset, If the qp is + * stopped when reset but still want to be released then, the + * is_resetting flag should be set negative so that this qp will not + * be restarted after reset. + */ + if (atomic_read(&qp->qp_status.flags) != QP_START) { + qp->is_resetting = false; + return; + } + + atomic_set(&qp->qp_status.flags, QP_STOP); + + /* V3 supports direct stop function when FLR prepare */ + if (qm->ver < QM_HW_V3 || qm->status.stop_reason == QM_NORMAL) { + ret = qm_drain_qp(qp); + if (ret) + dev_err(dev, "Failed to drain out data for stopping qp(%u)!\n", qp->qp_id); + } + + flush_workqueue(qm->wq); + if (unlikely(qp->is_resetting && atomic_read(&qp->qp_status.used))) + qp_stop_fail_cb(qp); + + dev_dbg(dev, "stop queue %u!", qp->qp_id); +} + +/** + * hisi_qm_stop_qp() - Stop a qp in qm. + * @qp: The qp we want to stop. + * + * This function is reverse of hisi_qm_start_qp. + */ +static void hisi_qm_stop_qp(struct hisi_qp *qp) +{ + down_write(&qp->qm->qps_lock); + qm_stop_qp_nolock(qp); + up_write(&qp->qm->qps_lock); +} + +/** + * hisi_qp_send() - Queue up a task in the hardware queue. + * @qp: The qp in which to put the message. + * @msg: The message. + * + * This function will return -EBUSY if qp is currently full, and -EAGAIN + * if qp related qm is resetting. + * + * Note: This function may run with qm_irq_thread and ACC reset at same time. + * It has no race with qm_irq_thread. However, during hisi_qp_send, ACC + * reset may happen, we have no lock here considering performance. This + * causes current qm_db sending fail or can not receive sended sqe. QM + * sync/async receive function should handle the error sqe. ACC reset + * done function should clear used sqe to 0. + */ +int hisi_qp_send(struct hisi_qp *qp, const void *msg) +{ + struct hisi_qp_status *qp_status = &qp->qp_status; + u16 sq_tail, sq_tail_next; + void *sqe; + + spin_lock_bh(&qp->qp_lock); + if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP || + atomic_read(&qp->qm->status.flags) == QM_STOP || + qp->is_resetting)) { + spin_unlock_bh(&qp->qp_lock); + dev_info_ratelimited(&qp->qm->pdev->dev, "QP is stopped or resetting\n"); + return -EAGAIN; + } + + sqe = qm_get_avail_sqe(qp); + if (!sqe) { + spin_unlock_bh(&qp->qp_lock); + return -EBUSY; + } + + sq_tail = qp_status->sq_tail; + sq_tail_next = (sq_tail + 1) % qp->sq_depth; + memcpy(sqe, msg, qp->qm->sqe_size); + qp->msg[sq_tail] = msg; + + qm_db(qp->qm, qp->qp_id, QM_DOORBELL_CMD_SQ, sq_tail_next, 0); + atomic_inc(&qp->qp_status.used); + qp_status->sq_tail = sq_tail_next; + spin_unlock_bh(&qp->qp_lock); + + return 0; +} +EXPORT_SYMBOL_GPL(hisi_qp_send); + +static void hisi_qm_cache_wb(struct hisi_qm *qm) +{ + unsigned int val; + + if (qm->ver == QM_HW_V1) + return; + + writel(0x1, qm->io_base + QM_CACHE_WB_START); + if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE, + val, val & BIT(0), POLL_PERIOD, + POLL_TIMEOUT)) + dev_err(&qm->pdev->dev, "QM writeback sqc cache fail!\n"); +} + +static void qm_qp_event_notifier(struct hisi_qp *qp) +{ + wake_up_interruptible(&qp->uacce_q->wait); +} + + /* This function returns free number of qp in qm. */ +static int hisi_qm_get_available_instances(struct uacce_device *uacce) +{ + struct hisi_qm *qm = uacce->priv; + int ret; + + down_read(&qm->qps_lock); + ret = qm->qp_num - qm->qp_in_used; + up_read(&qm->qps_lock); + + return ret; +} + +static void hisi_qm_set_hw_reset(struct hisi_qm *qm, int offset) +{ + int i; + + for (i = 0; i < qm->qp_num; i++) + qm_set_qp_disable(&qm->qp_array[i], offset); +} + +static int hisi_qm_uacce_get_queue(struct uacce_device *uacce, + unsigned long arg, + struct uacce_queue *q) +{ + struct hisi_qm *qm = uacce->priv; + struct hisi_qp *qp; + u8 alg_type = 0; + + qp = hisi_qm_create_qp(qm, alg_type); + if (IS_ERR(qp)) + return PTR_ERR(qp); + + q->priv = qp; + q->uacce = uacce; + qp->uacce_q = q; + qp->event_cb = qm_qp_event_notifier; + qp->pasid = arg; + + return 0; +} + +static void hisi_qm_uacce_put_queue(struct uacce_queue *q) +{ + struct hisi_qp *qp = q->priv; + + hisi_qm_release_qp(qp); +} + +/* map sq/cq/doorbell to user space */ +static int hisi_qm_uacce_mmap(struct uacce_queue *q, + struct vm_area_struct *vma, + struct uacce_qfile_region *qfr) +{ + struct hisi_qp *qp = q->priv; + struct hisi_qm *qm = qp->qm; + resource_size_t phys_base = qm->db_phys_base + + qp->qp_id * qm->db_interval; + size_t sz = vma->vm_end - vma->vm_start; + struct pci_dev *pdev = qm->pdev; + struct device *dev = &pdev->dev; + unsigned long vm_pgoff; + int ret; + + switch (qfr->type) { + case UACCE_QFRT_MMIO: + if (qm->ver == QM_HW_V1) { + if (sz > PAGE_SIZE * QM_DOORBELL_PAGE_NR) + return -EINVAL; + } else if (!test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) { + if (sz > PAGE_SIZE * (QM_DOORBELL_PAGE_NR + + QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE)) + return -EINVAL; + } else { + if (sz > qm->db_interval) + return -EINVAL; + } + + vm_flags_set(vma, VM_IO); + + return remap_pfn_range(vma, vma->vm_start, + phys_base >> PAGE_SHIFT, + sz, pgprot_noncached(vma->vm_page_prot)); + case UACCE_QFRT_DUS: + if (sz != qp->qdma.size) + return -EINVAL; + + /* + * dma_mmap_coherent() requires vm_pgoff as 0 + * restore vm_pfoff to initial value for mmap() + */ + vm_pgoff = vma->vm_pgoff; + vma->vm_pgoff = 0; + ret = dma_mmap_coherent(dev, vma, qp->qdma.va, + qp->qdma.dma, sz); + vma->vm_pgoff = vm_pgoff; + return ret; + + default: + return -EINVAL; + } +} + +static int hisi_qm_uacce_start_queue(struct uacce_queue *q) +{ + struct hisi_qp *qp = q->priv; + + return hisi_qm_start_qp(qp, qp->pasid); +} + +static void hisi_qm_uacce_stop_queue(struct uacce_queue *q) +{ + struct hisi_qp *qp = q->priv; + struct hisi_qm *qm = qp->qm; + struct qm_dev_dfx *dev_dfx = &qm->debug.dev_dfx; + u32 i = 0; + + hisi_qm_stop_qp(qp); + + if (!dev_dfx->dev_timeout || !dev_dfx->dev_state) + return; + + /* + * After the queue fails to be stopped, + * wait for a period of time before releasing the queue. + */ + while (++i) { + msleep(WAIT_PERIOD); + + /* Since dev_timeout maybe modified, check i >= dev_timeout */ + if (i >= dev_dfx->dev_timeout) { + dev_err(&qm->pdev->dev, "Stop q %u timeout, state %u\n", + qp->qp_id, dev_dfx->dev_state); + dev_dfx->dev_state = QM_FINISH_WAIT; + break; + } + } +} + +static int hisi_qm_is_q_updated(struct uacce_queue *q) +{ + struct hisi_qp *qp = q->priv; + struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head; + int updated = 0; + + while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) { + /* make sure to read data from memory */ + dma_rmb(); + qm_cq_head_update(qp); + cqe = qp->cqe + qp->qp_status.cq_head; + updated = 1; + } + + return updated; +} + +static void qm_set_sqctype(struct uacce_queue *q, u16 type) +{ + struct hisi_qm *qm = q->uacce->priv; + struct hisi_qp *qp = q->priv; + + down_write(&qm->qps_lock); + qp->alg_type = type; + up_write(&qm->qps_lock); +} + +static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd, + unsigned long arg) +{ + struct hisi_qp *qp = q->priv; + struct hisi_qp_info qp_info; + struct hisi_qp_ctx qp_ctx; + + if (cmd == UACCE_CMD_QM_SET_QP_CTX) { + if (copy_from_user(&qp_ctx, (void __user *)arg, + sizeof(struct hisi_qp_ctx))) + return -EFAULT; + + if (qp_ctx.qc_type > QM_MAX_QC_TYPE) + return -EINVAL; + + qm_set_sqctype(q, qp_ctx.qc_type); + qp_ctx.id = qp->qp_id; + + if (copy_to_user((void __user *)arg, &qp_ctx, + sizeof(struct hisi_qp_ctx))) + return -EFAULT; + + return 0; + } else if (cmd == UACCE_CMD_QM_SET_QP_INFO) { + if (copy_from_user(&qp_info, (void __user *)arg, + sizeof(struct hisi_qp_info))) + return -EFAULT; + + qp_info.sqe_size = qp->qm->sqe_size; + qp_info.sq_depth = qp->sq_depth; + qp_info.cq_depth = qp->cq_depth; + + if (copy_to_user((void __user *)arg, &qp_info, + sizeof(struct hisi_qp_info))) + return -EFAULT; + + return 0; + } + + return -EINVAL; +} + +/** + * qm_hw_err_isolate() - Try to set the isolation status of the uacce device + * according to user's configuration of error threshold. + * @qm: the uacce device + */ +static int qm_hw_err_isolate(struct hisi_qm *qm) +{ + struct qm_hw_err *err, *tmp, *hw_err; + struct qm_err_isolate *isolate; + u32 count = 0; + + isolate = &qm->isolate_data; + +#define SECONDS_PER_HOUR 3600 + + /* All the hw errs are processed by PF driver */ + if (qm->uacce->is_vf || isolate->is_isolate || !isolate->err_threshold) + return 0; + + hw_err = kzalloc_obj(*hw_err); + if (!hw_err) + return -ENOMEM; + + /* + * Time-stamp every slot AER error. Then check the AER error log when the + * next device AER error occurred. if the device slot AER error count exceeds + * the setting error threshold in one hour, the isolated state will be set + * to true. And the AER error logs that exceed one hour will be cleared. + */ + mutex_lock(&isolate->isolate_lock); + hw_err->timestamp = jiffies; + list_for_each_entry_safe(err, tmp, &isolate->qm_hw_errs, list) { + if ((hw_err->timestamp - err->timestamp) / HZ > + SECONDS_PER_HOUR) { + list_del(&err->list); + kfree(err); + } else { + count++; + } + } + list_add(&hw_err->list, &isolate->qm_hw_errs); + + if (count >= isolate->err_threshold) + isolate->is_isolate = true; + mutex_unlock(&isolate->isolate_lock); + + return 0; +} + +static void qm_hw_err_destroy(struct hisi_qm *qm) +{ + struct qm_hw_err *err, *tmp; + + list_for_each_entry_safe(err, tmp, &qm->isolate_data.qm_hw_errs, list) { + list_del(&err->list); + kfree(err); + } +} + +static enum uacce_dev_state hisi_qm_get_isolate_state(struct uacce_device *uacce) +{ + struct hisi_qm *qm = uacce->priv; + struct hisi_qm *pf_qm; + + if (uacce->is_vf) + pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); + else + pf_qm = qm; + + return pf_qm->isolate_data.is_isolate ? + UACCE_DEV_ISOLATE : UACCE_DEV_NORMAL; +} + +static int hisi_qm_isolate_threshold_write(struct uacce_device *uacce, u32 num) +{ + struct hisi_qm *qm = uacce->priv; + int ret; + + /* Must be set by PF */ + if (uacce->is_vf) + return -EPERM; + + if (qm->isolate_data.is_isolate) + return -EPERM; + + mutex_lock(&qm->isolate_data.isolate_lock); + qm->isolate_data.err_threshold = num; + + /* After the policy is updated, need to reset the hardware err list */ + qm_hw_err_destroy(qm); + + if (!qm->vfs_num) { + mutex_unlock(&qm->isolate_data.isolate_lock); + return 0; + } + + /* Notify all VFs to update the isolation threshold. */ + if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) { + ret = qm_ping_all_vfs(qm, QM_PF_SET_ISOLATE, qm->isolate_data.err_threshold); + if (ret) + dev_err(&qm->pdev->dev, "failed to send command to all VFs set isolate!\n"); + } + mutex_unlock(&qm->isolate_data.isolate_lock); + + return 0; +} + +static u32 hisi_qm_isolate_threshold_read(struct uacce_device *uacce) +{ + struct hisi_qm *qm = uacce->priv; + struct hisi_qm *pf_qm; + + if (uacce->is_vf && !test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) { + pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); + return pf_qm->isolate_data.err_threshold; + } + + return qm->isolate_data.err_threshold; +} + +static const struct uacce_ops uacce_qm_ops = { + .get_available_instances = hisi_qm_get_available_instances, + .get_queue = hisi_qm_uacce_get_queue, + .put_queue = hisi_qm_uacce_put_queue, + .start_queue = hisi_qm_uacce_start_queue, + .stop_queue = hisi_qm_uacce_stop_queue, + .mmap = hisi_qm_uacce_mmap, + .ioctl = hisi_qm_uacce_ioctl, + .is_q_updated = hisi_qm_is_q_updated, + .get_isolate_state = hisi_qm_get_isolate_state, + .isolate_err_threshold_write = hisi_qm_isolate_threshold_write, + .isolate_err_threshold_read = hisi_qm_isolate_threshold_read, +}; + +static void qm_remove_uacce(struct hisi_qm *qm) +{ + struct uacce_device *uacce = qm->uacce; + + if (qm->use_sva) { + mutex_lock(&qm->isolate_data.isolate_lock); + qm_hw_err_destroy(qm); + mutex_unlock(&qm->isolate_data.isolate_lock); + + uacce_remove(uacce); + qm->uacce = NULL; + } +} + +static void qm_uacce_api_ver_init(struct hisi_qm *qm) +{ + struct uacce_device *uacce = qm->uacce; + + switch (qm->ver) { + case QM_HW_V1: + uacce->api_ver = HISI_QM_API_VER_BASE; + break; + case QM_HW_V2: + uacce->api_ver = HISI_QM_API_VER2_BASE; + break; + case QM_HW_V3: + case QM_HW_V4: + uacce->api_ver = HISI_QM_API_VER3_BASE; + break; + default: + uacce->api_ver = HISI_QM_API_VER5_BASE; + break; + } +} + +static int qm_alloc_uacce(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + struct uacce_device *uacce; + unsigned long mmio_page_nr; + unsigned long dus_page_nr; + u16 sq_depth, cq_depth; + struct uacce_interface interface = { + .flags = UACCE_DEV_SVA, + .ops = &uacce_qm_ops, + }; + + if (strscpy(interface.name, dev_driver_string(&pdev->dev)) < 0) + return -ENAMETOOLONG; + + uacce = uacce_alloc(&pdev->dev, &interface); + if (IS_ERR(uacce)) + return PTR_ERR(uacce); + + if (uacce->flags & UACCE_DEV_SVA) { + qm->use_sva = true; + } else { + /* only consider sva case */ + qm_remove_uacce(qm); + return -EINVAL; + } + + if (qm->fun_type == QM_HW_PF) + uacce->is_vf = false; + else + uacce->is_vf = true; + uacce->priv = qm; + + if (qm->ver == QM_HW_V1) + mmio_page_nr = QM_DOORBELL_PAGE_NR; + else if (!test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) + mmio_page_nr = QM_DOORBELL_PAGE_NR + + QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE; + else + mmio_page_nr = qm->db_interval / PAGE_SIZE; + + qm_get_xqc_depth(qm, &sq_depth, &cq_depth, QM_QP_DEPTH_CAP); + + /* Add one more page for device or qp status */ + dus_page_nr = (PAGE_SIZE - 1 + qm->sqe_size * sq_depth + + sizeof(struct qm_cqe) * cq_depth + PAGE_SIZE) >> + PAGE_SHIFT; + + uacce->qf_pg_num[UACCE_QFRT_MMIO] = mmio_page_nr; + uacce->qf_pg_num[UACCE_QFRT_DUS] = dus_page_nr; + + qm->uacce = uacce; + qm_uacce_api_ver_init(qm); + INIT_LIST_HEAD(&qm->isolate_data.qm_hw_errs); + mutex_init(&qm->isolate_data.isolate_lock); + + return 0; +} + +int hisi_qm_register_uacce(struct hisi_qm *qm) +{ + int ret; + + if (!qm->uacce) + return 0; + + dev_info(&qm->pdev->dev, "qm register to uacce\n"); + + if (qm->fun_type == QM_HW_VF && test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) { + ret = qm_ping_pf(qm, QM_VF_GET_ISOLATE); + if (ret) + dev_err(&qm->pdev->dev, "failed to send cmd to PF to get isolate!\n"); + } + + return uacce_register(qm->uacce); +} +EXPORT_SYMBOL_GPL(hisi_qm_register_uacce); + +/** + * qm_frozen() - Try to froze QM to cut continuous queue request. If + * there is user on the QM, return failure without doing anything. + * @qm: The qm needed to be fronzen. + * + * This function frozes QM, then we can do SRIOV disabling. + */ +static int qm_frozen(struct hisi_qm *qm) +{ + if (test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl)) + return 0; + + down_write(&qm->qps_lock); + + if (!qm->qp_in_used) { + qm->qp_in_used = qm->qp_num; + up_write(&qm->qps_lock); + set_bit(QM_DRIVER_REMOVING, &qm->misc_ctl); + return 0; + } + + up_write(&qm->qps_lock); + + return -EBUSY; +} + +static int qm_try_frozen_vfs(struct pci_dev *pdev, + struct hisi_qm_list *qm_list) +{ + struct hisi_qm *qm, *vf_qm; + struct pci_dev *dev; + int ret = 0; + + if (!qm_list || !pdev) + return -EINVAL; + + /* Try to frozen all the VFs as disable SRIOV */ + mutex_lock(&qm_list->lock); + list_for_each_entry(qm, &qm_list->list, list) { + dev = qm->pdev; + if (dev == pdev) + continue; + if (pci_physfn(dev) == pdev) { + vf_qm = pci_get_drvdata(dev); + ret = qm_frozen(vf_qm); + if (ret) + goto frozen_fail; + } + } + +frozen_fail: + mutex_unlock(&qm_list->lock); + + return ret; +} + +/** + * hisi_qm_wait_task_finish() - Wait until the task is finished + * when removing the driver. + * @qm: The qm needed to wait for the task to finish. + * @qm_list: The list of all available devices. + */ +void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list) +{ + while (qm_frozen(qm) || + ((qm->fun_type == QM_HW_PF) && + qm_try_frozen_vfs(qm->pdev, qm_list))) { + msleep(WAIT_PERIOD); + } + + /* Cancel possible RAS reset process during the uninstallation procedure. */ + if (qm->fun_type == QM_HW_PF) + cancel_work_sync(&qm->rst_work); + + if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) + flush_work(&qm->cmd_process); + + udelay(REMOVE_WAIT_DELAY); +} +EXPORT_SYMBOL_GPL(hisi_qm_wait_task_finish); + +static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num) +{ + struct device *dev = &qm->pdev->dev; + struct hisi_qp *qp; + int i; + + for (i = num - 1; i >= 0; i--) { + qp = &qm->qp_array[i]; + dma_free_coherent(dev, qp->qdma.size, qp->qdma.va, qp->qdma.dma); + kfree(qp->msg); + kfree(qm->poll_data[i].qp_finish_id); + } + + kfree(qm->poll_data); + kfree(qm->qp_array); +} + +static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id, + u16 sq_depth, u16 cq_depth) +{ + struct device *dev = &qm->pdev->dev; + size_t off = qm->sqe_size * sq_depth; + struct hisi_qp *qp; + int ret = -ENOMEM; + + qm->poll_data[id].qp_finish_id = kcalloc(qm->qp_num, sizeof(u16), + GFP_KERNEL); + if (!qm->poll_data[id].qp_finish_id) + return -ENOMEM; + + qp = &qm->qp_array[id]; + qp->msg = kmalloc_array(sq_depth, sizeof(void *), GFP_KERNEL); + if (!qp->msg) + goto err_free_qp_finish_id; + + qp->qdma.va = dma_alloc_coherent(dev, dma_size, &qp->qdma.dma, + GFP_KERNEL); + if (!qp->qdma.va) + goto err_free_qp_msg; + + qp->sqe = qp->qdma.va; + qp->sqe_dma = qp->qdma.dma; + qp->cqe = qp->qdma.va + off; + qp->cqe_dma = qp->qdma.dma + off; + qp->qdma.size = dma_size; + qp->sq_depth = sq_depth; + qp->cq_depth = cq_depth; + qp->qm = qm; + qp->qp_id = id; + + spin_lock_init(&qp->qp_lock); + spin_lock_init(&qp->backlog.lock); + INIT_LIST_HEAD(&qp->backlog.list); + + return 0; + +err_free_qp_msg: + kfree(qp->msg); +err_free_qp_finish_id: + kfree(qm->poll_data[id].qp_finish_id); + return ret; +} + +static void hisi_qm_pre_init(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + + if (qm->ver == QM_HW_V1) + qm->ops = &qm_hw_ops_v1; + else if (qm->ver == QM_HW_V2) + qm->ops = &qm_hw_ops_v2; + else if (qm->ver == QM_HW_V3) + qm->ops = &qm_hw_ops_v3; + else + qm->ops = &qm_hw_ops_v4; + + pci_set_drvdata(pdev, qm); + mutex_init(&qm->mailbox_lock); + mutex_init(&qm->ifc_lock); + init_rwsem(&qm->qps_lock); + qm->qp_in_used = 0; + if (test_bit(QM_SUPPORT_RPM, &qm->caps)) { + if (!acpi_device_power_manageable(ACPI_COMPANION(&pdev->dev))) + dev_info(&pdev->dev, "_PS0 and _PR0 are not defined"); + } +} + +static void qm_cmd_uninit(struct hisi_qm *qm) +{ + u32 val; + + if (!test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) + return; + + val = readl(qm->io_base + QM_IFC_INT_MASK); + val |= QM_IFC_INT_DISABLE; + writel(val, qm->io_base + QM_IFC_INT_MASK); +} + +static void qm_cmd_init(struct hisi_qm *qm) +{ + u32 val; + + if (!test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) + return; + + /* Clear communication interrupt source */ + qm_clear_cmd_interrupt(qm, QM_IFC_INT_SOURCE_CLR); + + /* Enable pf to vf communication reg. */ + val = readl(qm->io_base + QM_IFC_INT_MASK); + val &= ~QM_IFC_INT_DISABLE; + writel(val, qm->io_base + QM_IFC_INT_MASK); +} + +static void qm_put_pci_res(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + + if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) + iounmap(qm->db_io_base); + + iounmap(qm->io_base); + pci_release_mem_regions(pdev); +} + +static void hisi_mig_region_clear(struct hisi_qm *qm) +{ + u32 val; + + /* Clear migration region set of PF */ + if (qm->fun_type == QM_HW_PF && qm->ver > QM_HW_V3) { + val = readl(qm->io_base + QM_MIG_REGION_SEL); + val &= ~QM_MIG_REGION_EN; + writel(val, qm->io_base + QM_MIG_REGION_SEL); + } +} + +static void hisi_mig_region_enable(struct hisi_qm *qm) +{ + u32 val; + + /* Select migration region of PF */ + if (qm->fun_type == QM_HW_PF && qm->ver > QM_HW_V3) { + val = readl(qm->io_base + QM_MIG_REGION_SEL); + val |= QM_MIG_REGION_EN; + writel(val, qm->io_base + QM_MIG_REGION_SEL); + } +} + +static void hisi_qm_pci_uninit(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + + pci_free_irq_vectors(pdev); + hisi_mig_region_clear(qm); + qm_put_pci_res(qm); + pci_disable_device(pdev); +} + +static void hisi_qm_set_state(struct hisi_qm *qm, u8 state) +{ + if (qm->ver > QM_HW_V2 && qm->fun_type == QM_HW_VF) + writel(state, qm->io_base + QM_VF_STATE); +} + +static void hisi_qm_unint_work(struct hisi_qm *qm) +{ + destroy_workqueue(qm->wq); +} + +static void hisi_qm_free_rsv_buf(struct hisi_qm *qm) +{ + struct qm_dma *xqc_dma = &qm->xqc_buf.qcdma; + struct device *dev = &qm->pdev->dev; + + dma_free_coherent(dev, xqc_dma->size, xqc_dma->va, xqc_dma->dma); +} + +static void hisi_qm_memory_uninit(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + + hisi_qp_memory_uninit(qm, qm->qp_num); + hisi_qm_free_rsv_buf(qm); + if (qm->qdma.va) { + hisi_qm_cache_wb(qm); + dma_free_coherent(dev, qm->qdma.size, + qm->qdma.va, qm->qdma.dma); + } + + idr_destroy(&qm->qp_idr); + + if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) + kfree(qm->factor); +} + +/** + * hisi_qm_uninit() - Uninitialize qm. + * @qm: The qm needed uninit. + * + * This function uninits qm related device resources. + */ +void hisi_qm_uninit(struct hisi_qm *qm) +{ + qm_cmd_uninit(qm); + hisi_qm_unint_work(qm); + + down_write(&qm->qps_lock); + hisi_qm_memory_uninit(qm); + hisi_qm_set_state(qm, QM_NOT_READY); + up_write(&qm->qps_lock); + + qm_remove_uacce(qm); + qm_irqs_unregister(qm); + hisi_qm_pci_uninit(qm); +} +EXPORT_SYMBOL_GPL(hisi_qm_uninit); + +/** + * hisi_qm_get_vft() - Get vft from a qm. + * @qm: The qm we want to get its vft. + * @base: The base number of queue in vft. + * @number: The number of queues in vft. + * + * We can allocate multiple queues to a qm by configuring virtual function + * table. We get related configures by this function. Normally, we call this + * function in VF driver to get the queue information. + * + * qm hw v1 does not support this interface. + */ +static int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number) +{ + if (!base || !number) + return -EINVAL; + + if (!qm->ops->get_vft) { + dev_err(&qm->pdev->dev, "Don't support vft read!\n"); + return -EINVAL; + } + + return qm->ops->get_vft(qm, base, number); +} + +/** + * hisi_qm_set_vft() - Set vft to a qm. + * @qm: The qm we want to set its vft. + * @fun_num: The function number. + * @base: The base number of queue in vft. + * @number: The number of queues in vft. + * + * This function is alway called in PF driver, it is used to assign queues + * among PF and VFs. + * + * Assign queues A~B to PF: hisi_qm_set_vft(qm, 0, A, B - A + 1) + * Assign queues A~B to VF: hisi_qm_set_vft(qm, 2, A, B - A + 1) + * (VF function number 0x2) + */ +static int hisi_qm_set_vft(struct hisi_qm *qm, u32 fun_num, u32 base, + u32 number) +{ + u32 max_q_num = qm->ctrl_qp_num; + + if (base >= max_q_num || number > max_q_num || + (base + number) > max_q_num) + return -EINVAL; + + return qm_set_sqc_cqc_vft(qm, fun_num, base, number); +} + +static void qm_init_eq_aeq_status(struct hisi_qm *qm) +{ + struct hisi_qm_status *status = &qm->status; + + status->eq_head = 0; + status->aeq_head = 0; + status->eqc_phase = true; + status->aeqc_phase = true; +} + +static void qm_enable_eq_aeq_interrupts(struct hisi_qm *qm) +{ + /* Clear eq/aeq interrupt source */ + qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0); + qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); + + writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK); + writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK); +} + +static void qm_disable_eq_aeq_interrupts(struct hisi_qm *qm) +{ + writel(0x1, qm->io_base + QM_VF_EQ_INT_MASK); + writel(0x1, qm->io_base + QM_VF_AEQ_INT_MASK); +} + +static int qm_eq_ctx_cfg(struct hisi_qm *qm) +{ + struct qm_eqc eqc = {0}; + + eqc.base_l = cpu_to_le32(lower_32_bits(qm->eqe_dma)); + eqc.base_h = cpu_to_le32(upper_32_bits(qm->eqe_dma)); + if (qm->ver == QM_HW_V1) + eqc.dw3 = cpu_to_le32(QM_EQE_AEQE_SIZE); + eqc.dw6 = cpu_to_le32(((u32)qm->eq_depth - 1) | (1 << QM_EQC_PHASE_SHIFT)); + + return qm_set_and_get_xqc(qm, QM_MB_CMD_EQC, &eqc, 0, 0); +} + +static int qm_aeq_ctx_cfg(struct hisi_qm *qm) +{ + struct qm_aeqc aeqc = {0}; + + aeqc.base_l = cpu_to_le32(lower_32_bits(qm->aeqe_dma)); + aeqc.base_h = cpu_to_le32(upper_32_bits(qm->aeqe_dma)); + aeqc.dw6 = cpu_to_le32(((u32)qm->aeq_depth - 1) | (1 << QM_EQC_PHASE_SHIFT)); + + return qm_set_and_get_xqc(qm, QM_MB_CMD_AEQC, &aeqc, 0, 0); +} + +static int qm_eq_aeq_ctx_cfg(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + int ret; + + qm_init_eq_aeq_status(qm); + + /* Before starting the dev, clear the memory and then configure to device using. */ + memset(qm->qdma.va, 0, qm->qdma.size); + + ret = qm_eq_ctx_cfg(qm); + if (ret) { + dev_err(dev, "Set eqc failed!\n"); + return ret; + } + + return qm_aeq_ctx_cfg(qm); +} + +static int __hisi_qm_start(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + int ret; + + if (!qm->qdma.va) { + dev_err(dev, "qm qdma is NULL!\n"); + return -EINVAL; + } + + if (qm->fun_type == QM_HW_PF) { + ret = hisi_qm_set_vft(qm, 0, qm->qp_base, qm->qp_num); + if (ret) + return ret; + } + + ret = qm_eq_aeq_ctx_cfg(qm); + if (ret) + return ret; + + ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0); + if (ret) + return ret; + + ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, qm->cqc_dma, 0, 0); + if (ret) + return ret; + + /* Enables the doorbell function when the device is enabled. */ + qm_dev_db_ctrl(qm, false); + qm_fun_db_ctrl(qm, false); + qm_init_prefetch(qm); + qm_enable_eq_aeq_interrupts(qm); + + return 0; +} + +/** + * hisi_qm_start() - start qm + * @qm: The qm to be started. + * + * This function starts a qm, then we can allocate qp from this qm. + */ +int hisi_qm_start(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + int ret; + + down_write(&qm->qps_lock); + + dev_dbg(dev, "qm start with %u queue pairs\n", qm->qp_num); + + if (!qm->qp_num) { + dev_err(dev, "qp_num should not be 0\n"); + ret = -EINVAL; + goto err_unlock; + } + + ret = __hisi_qm_start(qm); + if (ret) + goto err_unlock; + + atomic_set(&qm->status.flags, QM_WORK); + hisi_qm_set_state(qm, QM_READY); + +err_unlock: + up_write(&qm->qps_lock); + return ret; +} +EXPORT_SYMBOL_GPL(hisi_qm_start); + +static int qm_restart(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + struct hisi_qp *qp; + int ret, i; + + ret = hisi_qm_start(qm); + if (ret < 0) + return ret; + + down_write(&qm->qps_lock); + for (i = 0; i < qm->qp_num; i++) { + qp = &qm->qp_array[i]; + if (atomic_read(&qp->qp_status.flags) == QP_STOP && + qp->is_resetting == true && qp->is_in_kernel == true) { + ret = qm_start_qp_nolock(qp, 0); + if (ret < 0) { + dev_err(dev, "Failed to start qp%d!\n", i); + + up_write(&qm->qps_lock); + return ret; + } + qp->is_resetting = false; + } + } + up_write(&qm->qps_lock); + + return 0; +} + +/* Stop started qps in reset flow */ +static void qm_stop_started_qp(struct hisi_qm *qm) +{ + struct hisi_qp *qp; + int i; + + for (i = 0; i < qm->qp_num; i++) { + qp = &qm->qp_array[i]; + if (atomic_read(&qp->qp_status.flags) == QP_START) { + qp->is_resetting = true; + qm_stop_qp_nolock(qp); + } + } +} + +/** + * qm_invalid_queues() - invalid all queues in use. + * @qm: The qm in which the queues will be invalidated. + * + * This function invalid all queues in use. If the doorbell command is sent + * to device in user space after the device is reset, the device discards + * the doorbell command. + */ +static void qm_invalid_queues(struct hisi_qm *qm) +{ + struct hisi_qp *qp; + struct qm_sqc *sqc; + struct qm_cqc *cqc; + int i; + + /* + * Normal stop queues is no longer used and does not need to be + * invalid queues. + */ + if (qm->status.stop_reason == QM_NORMAL) + return; + + if (qm->status.stop_reason == QM_DOWN || qm->status.stop_reason == QM_SHUTDOWN) + hisi_qm_cache_wb(qm); + + for (i = 0; i < qm->qp_num; i++) { + qp = &qm->qp_array[i]; + if (!qp->is_resetting) + continue; + + /* Modify random data and set sqc close bit to invalid queue. */ + sqc = qm->sqc + i; + cqc = qm->cqc + i; + sqc->w8 = cpu_to_le16(QM_XQC_RANDOM_DATA); + sqc->w13 = cpu_to_le16(QM_SQC_DISABLE_QP); + cqc->w8 = cpu_to_le16(QM_XQC_RANDOM_DATA); + if (qp->is_in_kernel) + memset(qp->qdma.va, 0, qp->qdma.size); + } +} + +/** + * hisi_qm_stop() - Stop a qm. + * @qm: The qm which will be stopped. + * @r: The reason to stop qm. + * + * This function stops qm and its qps, then qm can not accept request. + * Related resources are not released at this state, we can use hisi_qm_start + * to let qm start again. + */ +int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r) +{ + struct device *dev = &qm->pdev->dev; + int ret = 0; + + down_write(&qm->qps_lock); + + if (atomic_read(&qm->status.flags) == QM_STOP) + goto err_unlock; + + /* Stop all the request sending at first. */ + atomic_set(&qm->status.flags, QM_STOP); + qm->status.stop_reason = r; + + if (qm->status.stop_reason != QM_NORMAL) { + hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); + if (qm->status.stop_reason != QM_SHUTDOWN) + qm_fun_db_ctrl(qm, true); + /* + * When performing soft reset, the hardware will no longer + * do tasks, and the tasks in the device will be flushed + * out directly since the master ooo is closed. + */ + if (test_bit(QM_SUPPORT_STOP_FUNC, &qm->caps) && + r != QM_SOFT_RESET) { + ret = qm_drain_qm(qm); + if (ret) { + dev_err(dev, "failed to drain qm!\n"); + goto err_unlock; + } + } + + qm_stop_started_qp(qm); + + hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); + } + + qm_disable_eq_aeq_interrupts(qm); + if (qm->fun_type == QM_HW_PF) { + ret = hisi_qm_set_vft(qm, 0, 0, 0); + if (ret < 0) { + dev_err(dev, "Failed to set vft!\n"); + ret = -EBUSY; + goto err_unlock; + } + } + + qm_invalid_queues(qm); + qm->status.stop_reason = QM_NORMAL; + +err_unlock: + up_write(&qm->qps_lock); + return ret; +} +EXPORT_SYMBOL_GPL(hisi_qm_stop); + +static void qm_hw_error_init(struct hisi_qm *qm) +{ + if (!qm->ops->hw_error_init) { + dev_err(&qm->pdev->dev, "QM doesn't support hw error handling!\n"); + return; + } + + qm->ops->hw_error_init(qm); +} + +static void qm_hw_error_uninit(struct hisi_qm *qm) +{ + if (!qm->ops->hw_error_uninit) { + dev_err(&qm->pdev->dev, "Unexpected QM hw error uninit!\n"); + return; + } + + qm->ops->hw_error_uninit(qm); +} + +static enum acc_err_result qm_hw_error_handle(struct hisi_qm *qm) +{ + if (!qm->ops->hw_error_handle) { + dev_err(&qm->pdev->dev, "QM doesn't support hw error report!\n"); + return ACC_ERR_NONE; + } + + return qm->ops->hw_error_handle(qm); +} + +/** + * hisi_qm_dev_err_init() - Initialize device error configuration. + * @qm: The qm for which we want to do error initialization. + * + * Initialize QM and device error related configuration. + */ +void hisi_qm_dev_err_init(struct hisi_qm *qm) +{ + if (qm->fun_type == QM_HW_VF) + return; + + qm_hw_error_init(qm); + + if (!qm->err_ini->hw_err_enable) { + dev_err(&qm->pdev->dev, "Device doesn't support hw error init!\n"); + return; + } + qm->err_ini->hw_err_enable(qm); +} +EXPORT_SYMBOL_GPL(hisi_qm_dev_err_init); + +/** + * hisi_qm_dev_err_uninit() - Uninitialize device error configuration. + * @qm: The qm for which we want to do error uninitialization. + * + * Uninitialize QM and device error related configuration. + */ +void hisi_qm_dev_err_uninit(struct hisi_qm *qm) +{ + if (qm->fun_type == QM_HW_VF) + return; + + qm_hw_error_uninit(qm); + + if (!qm->err_ini->hw_err_disable) { + dev_err(&qm->pdev->dev, "Unexpected device hw error uninit!\n"); + return; + } + qm->err_ini->hw_err_disable(qm); +} +EXPORT_SYMBOL_GPL(hisi_qm_dev_err_uninit); + +static void qm_release_qp_nolock(struct hisi_qp *qp) +{ + struct hisi_qm *qm = qp->qm; + + if (--qp->ref_count) + return; + + qm->qp_in_used--; + idr_remove(&qm->qp_idr, qp->qp_id); +} + +/** + * hisi_qm_free_qps() - free multiple queue pairs. + * @qps: The queue pairs need to be freed. + * @qp_num: The num of queue pairs. + */ +void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num) +{ + int i; + + if (!qps || qp_num <= 0) + return; + + down_write(&qps[0]->qm->qps_lock); + + for (i = qp_num - 1; i >= 0; i--) { + if (qps[i]->ref_count == 1) + qm_stop_qp_nolock(qps[i]); + + qm_release_qp_nolock(qps[i]); + } + + up_write(&qps[0]->qm->qps_lock); + qm_pm_put_sync(qps[0]->qm); +} +EXPORT_SYMBOL_GPL(hisi_qm_free_qps); + +static void qm_insert_sorted(struct list_head *head, struct hisi_qm_resource *res) +{ + struct hisi_qm_resource *tmp; + struct list_head *n = head; + + list_for_each_entry(tmp, head, list) { + if (res->distance < tmp->distance) { + n = &tmp->list; + break; + } + } + list_add_tail(&res->list, n); +} + +static void free_list(struct list_head *head) +{ + struct hisi_qm_resource *res, *tmp; + + list_for_each_entry_safe(res, tmp, head, list) { + list_del(&res->list); + kfree(res); + } +} + +static int qm_get_and_start_qp(struct hisi_qm *qm, int qp_num, struct hisi_qp **qps, u8 *alg_type) +{ + int i, ret; + + ret = qm_pm_get_sync(qm); + if (ret) + return ret; + + down_write(&qm->qps_lock); + for (i = 0; i < qp_num; i++) { + qps[i] = qm_create_qp_nolock(qm, alg_type[i], true); + if (IS_ERR(qps[i])) { + ret = -ENODEV; + goto stop_and_free; + } + + if (qps[i]->ref_count != 1) + continue; + + ret = qm_start_qp_nolock(qps[i], 0); + if (ret) { + qm_release_qp_nolock(qps[i]); + goto stop_and_free; + } + } + up_write(&qm->qps_lock); + + return 0; + +stop_and_free: + for (i--; i >= 0; i--) { + if (qps[i]->ref_count == 1) + qm_stop_qp_nolock(qps[i]); + + qm_release_qp_nolock(qps[i]); + } + up_write(&qm->qps_lock); + qm_pm_put_sync(qm); + + return ret; +} + +static int hisi_qm_sort_devices(int node, struct list_head *head, + struct hisi_qm_list *qm_list) +{ + struct hisi_qm_resource *res; + struct hisi_qm *qm; + struct device *dev; + int dev_node; + LIST_HEAD(non_full_list); + LIST_HEAD(full_list); + + list_for_each_entry(qm, &qm_list->list, list) { + dev = &qm->pdev->dev; + + dev_node = dev_to_node(dev); + if (dev_node < 0) + dev_node = 0; + + res = kzalloc_obj(*res); + if (!res) + return -ENOMEM; + + res->qm = qm; + res->distance = node_distance(dev_node, node); + + if (qm->qp_in_used == qm->qp_num) + qm_insert_sorted(&full_list, res); + else + qm_insert_sorted(&non_full_list, res); + } + + list_splice_tail(&non_full_list, head); + list_splice_tail(&full_list, head); + + return 0; +} + +/** + * hisi_qm_alloc_qps_node() - Create multiple queue pairs. + * @qm_list: The list of all available devices. + * @qp_num: The number of queue pairs need created. + * @alg_type: The algorithm type. + * @node: The numa node. + * @qps: The queue pairs need created. + * + * This function will sort all available device according to numa distance. + * Then try to create all queue pairs from one device, if all devices do + * not meet the requirements will return error. + */ +int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, + u8 *alg_type, int node, struct hisi_qp **qps) +{ + struct hisi_qm_resource *tmp; + int ret = -ENODEV; + LIST_HEAD(head); + + if (!qps || !qm_list || qp_num <= 0) + return -EINVAL; + + mutex_lock(&qm_list->lock); + if (hisi_qm_sort_devices(node, &head, qm_list)) { + mutex_unlock(&qm_list->lock); + goto err; + } + + list_for_each_entry(tmp, &head, list) { + ret = qm_get_and_start_qp(tmp->qm, qp_num, qps, alg_type); + if (!ret) + break; + } + + mutex_unlock(&qm_list->lock); + if (ret) + pr_info("Failed to create qps, node[%d], qp[%d]!\n", + node, qp_num); + +err: + free_list(&head); + return ret; +} +EXPORT_SYMBOL_GPL(hisi_qm_alloc_qps_node); + +static int qm_vf_q_assign(struct hisi_qm *qm, u32 num_vfs) +{ + u32 remain_q_num, vfs_q_num, act_q_num, q_num, i, j; + u32 max_qp_num = qm->max_qp_num; + u32 q_base = qm->qp_num; + int ret; + + if (!num_vfs) + return -EINVAL; + + vfs_q_num = qm->ctrl_qp_num - qm->qp_num; + + /* If vfs_q_num is less than num_vfs, return error. */ + if (vfs_q_num < num_vfs) + return -EINVAL; + + q_num = vfs_q_num / num_vfs; + remain_q_num = vfs_q_num % num_vfs; + + for (i = num_vfs; i > 0; i--) { + /* + * if q_num + remain_q_num > max_qp_num in last vf, divide the + * remaining queues equally. + */ + if (i == num_vfs && q_num + remain_q_num <= max_qp_num) { + act_q_num = q_num + remain_q_num; + remain_q_num = 0; + } else if (remain_q_num > 0) { + act_q_num = q_num + 1; + remain_q_num--; + } else { + act_q_num = q_num; + } + + act_q_num = min(act_q_num, max_qp_num); + ret = hisi_qm_set_vft(qm, i, q_base, act_q_num); + if (ret) { + for (j = num_vfs; j > i; j--) + hisi_qm_set_vft(qm, j, 0, 0); + return ret; + } + q_base += act_q_num; + } + + return 0; +} + +static void qm_clear_vft_config(struct hisi_qm *qm) +{ + u32 i; + + /* + * When disabling SR-IOV, clear the configuration of each VF in the hardware + * sequentially. Failure to clear a single VF should not affect the clearing + * operation of other VFs. + */ + for (i = 1; i <= qm->vfs_num; i++) + (void)hisi_qm_set_vft(qm, i, 0, 0); + + qm->vfs_num = 0; +} + +static int qm_func_shaper_enable(struct hisi_qm *qm, u32 fun_index, u32 qos) +{ + struct device *dev = &qm->pdev->dev; + struct qm_shaper_factor t_factor; + u32 ir = qos * QM_QOS_RATE; + int ret, total_vfs, i; + + total_vfs = pci_sriov_get_totalvfs(qm->pdev); + if (fun_index > total_vfs) + return -EINVAL; + + memcpy(&t_factor, &qm->factor[fun_index], sizeof(t_factor)); + qm->factor[fun_index].func_qos = qos; + + ret = qm_get_shaper_para(ir, &qm->factor[fun_index]); + if (ret) { + dev_err(dev, "failed to calculate shaper parameter!\n"); + return -EINVAL; + } + + for (i = ALG_TYPE_0; i <= ALG_TYPE_1; i++) { + /* The base number of queue reuse for different alg type */ + ret = qm_set_vft_common(qm, SHAPER_VFT, fun_index, i, 1); + if (ret) { + dev_err(dev, "type: %d, failed to set shaper vft!\n", i); + goto back_func_qos; + } + } + + return 0; + +back_func_qos: + memcpy(&qm->factor[fun_index], &t_factor, sizeof(t_factor)); + for (i--; i >= ALG_TYPE_0; i--) { + ret = qm_set_vft_common(qm, SHAPER_VFT, fun_index, i, 1); + if (ret) + dev_err(dev, "failed to restore shaper vft during rollback!\n"); + } + + return -EINVAL; +} + +static u32 qm_get_shaper_vft_qos(struct hisi_qm *qm, u32 fun_index) +{ + u64 shaper_vft, ir_calc, ir; + u64 cir_u, cir_b, cir_s; + unsigned int val; + u32 error_rate; + int ret; + + ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, + val & BIT(0), POLL_PERIOD, + POLL_TIMEOUT); + if (ret) + return 0; + + writel(0x1, qm->io_base + QM_VFT_CFG_OP_WR); + writel(SHAPER_VFT, qm->io_base + QM_VFT_CFG_TYPE); + writel(fun_index, qm->io_base + QM_VFT_CFG); + + writel(0x0, qm->io_base + QM_VFT_CFG_RDY); + writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE); + + ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, + val & BIT(0), POLL_PERIOD, + POLL_TIMEOUT); + if (ret) + return 0; + + shaper_vft = readl(qm->io_base + QM_VFT_CFG_DATA_L) | + ((u64)readl(qm->io_base + QM_VFT_CFG_DATA_H) << 32); + + cir_b = shaper_vft & QM_SHAPER_CIR_B_MASK; + cir_u = shaper_vft & QM_SHAPER_CIR_U_MASK; + cir_u = cir_u >> QM_SHAPER_FACTOR_CIR_U_SHIFT; + + cir_s = shaper_vft & QM_SHAPER_CIR_S_MASK; + cir_s = cir_s >> QM_SHAPER_FACTOR_CIR_S_SHIFT; + + ir_calc = acc_shaper_para_calc(cir_b, cir_u, cir_s); + + ir = qm->factor[fun_index].func_qos * QM_QOS_RATE; + + error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir; + if (error_rate > QM_QOS_MIN_ERROR_RATE) { + pci_err(qm->pdev, "error_rate: %u, get function qos is error!\n", error_rate); + return 0; + } + + return ir; +} + +static void qm_vf_get_qos(struct hisi_qm *qm, u32 fun_num) +{ + struct device *dev = &qm->pdev->dev; + u32 qos; + int ret; + + qos = qm_get_shaper_vft_qos(qm, fun_num); + if (!qos) { + dev_err(dev, "function(%u) failed to get qos by PF!\n", fun_num); + return; + } + + ret = qm_ping_single_vf(qm, QM_PF_SET_QOS, qos, fun_num); + if (ret) + dev_err(dev, "failed to send command(0x%x) to VF(%u)!\n", QM_PF_SET_QOS, fun_num); +} + +static int qm_vf_read_qos(struct hisi_qm *qm) +{ + int cnt = 0; + int ret = -EINVAL; + + /* reset mailbox qos val */ + qm->mb_qos = 0; + + /* vf ping pf to get function qos */ + ret = qm_ping_pf(qm, QM_VF_GET_QOS); + if (ret) { + pci_err(qm->pdev, "failed to send cmd to PF to get qos!\n"); + return ret; + } + + while (true) { + msleep(QM_WAIT_DST_ACK); + if (qm->mb_qos) + break; + + if (++cnt > QM_MAX_VF_WAIT_COUNT) { + pci_err(qm->pdev, "PF ping VF timeout!\n"); + return -ETIMEDOUT; + } + } + + return ret; +} + +static ssize_t qm_algqos_read(struct file *filp, char __user *buf, + size_t count, loff_t *pos) +{ + struct hisi_qm *qm = filp->private_data; + char tbuf[QM_DBG_READ_LEN]; + u32 qos_val, ir; + int ret; + + ret = hisi_qm_get_dfx_access(qm); + if (ret) + return ret; + + /* Mailbox and reset cannot be operated at the same time */ + if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { + pci_err(qm->pdev, "dev resetting, read alg qos failed!\n"); + ret = -EAGAIN; + goto err_put_dfx_access; + } + + if (qm->fun_type == QM_HW_PF) { + ir = qm_get_shaper_vft_qos(qm, 0); + } else { + ret = qm_vf_read_qos(qm); + if (ret) + goto err_get_status; + ir = qm->mb_qos; + } + + qos_val = ir / QM_QOS_RATE; + ret = scnprintf(tbuf, QM_DBG_READ_LEN, "%u\n", qos_val); + + ret = simple_read_from_buffer(buf, count, pos, tbuf, ret); + +err_get_status: + clear_bit(QM_RESETTING, &qm->misc_ctl); +err_put_dfx_access: + hisi_qm_put_dfx_access(qm); + return ret; +} + +static ssize_t qm_get_qos_value(struct hisi_qm *qm, const char *buf, + unsigned long *val, + unsigned int *fun_index) +{ + const struct bus_type *bus_type = qm->pdev->dev.bus; + char tbuf_bdf[QM_DBG_READ_LEN] = {0}; + char val_buf[QM_DBG_READ_LEN] = {0}; + struct pci_dev *pdev; + struct device *dev; + int ret; + + ret = sscanf(buf, "%s %s", tbuf_bdf, val_buf); + if (ret != QM_QOS_PARAM_NUM) + return -EINVAL; + + ret = kstrtoul(val_buf, 10, val); + if (ret || *val == 0 || *val > QM_QOS_MAX_VAL) { + pci_err(qm->pdev, "input qos value is error, please set 1~1000!\n"); + return -EINVAL; + } + + dev = bus_find_device_by_name(bus_type, NULL, tbuf_bdf); + if (!dev) { + pci_err(qm->pdev, "input pci bdf number is error!\n"); + return -ENODEV; + } + + pdev = container_of(dev, struct pci_dev, dev); + if (pci_physfn(pdev) != qm->pdev) { + pci_err(qm->pdev, "the pdev input does not match the pf!\n"); + put_device(dev); + return -EINVAL; + } + + *fun_index = pdev->devfn; + put_device(dev); + + return 0; +} + +static ssize_t qm_algqos_write(struct file *filp, const char __user *buf, + size_t count, loff_t *pos) +{ + struct hisi_qm *qm = filp->private_data; + char tbuf[QM_DBG_READ_LEN]; + unsigned int fun_index; + unsigned long val; + int len, ret; + + if (*pos != 0) + return 0; + + if (count >= QM_DBG_READ_LEN) + return -ENOSPC; + + len = simple_write_to_buffer(tbuf, QM_DBG_READ_LEN - 1, pos, buf, count); + if (len < 0) + return len; + + tbuf[len] = '\0'; + ret = qm_get_qos_value(qm, tbuf, &val, &fun_index); + if (ret) + return ret; + + /* Mailbox and reset cannot be operated at the same time */ + if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { + pci_err(qm->pdev, "dev resetting, write alg qos failed!\n"); + return -EAGAIN; + } + + ret = qm_pm_get_sync(qm); + if (ret) { + ret = -EINVAL; + goto err_get_status; + } + + ret = qm_func_shaper_enable(qm, fun_index, val); + if (ret) { + pci_err(qm->pdev, "failed to enable function shaper!\n"); + ret = -EINVAL; + goto err_put_sync; + } + + pci_info(qm->pdev, "the qos value of function%u is set to %lu.\n", + fun_index, val); + ret = count; + +err_put_sync: + qm_pm_put_sync(qm); +err_get_status: + clear_bit(QM_RESETTING, &qm->misc_ctl); + return ret; +} + +static const struct file_operations qm_algqos_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = qm_algqos_read, + .write = qm_algqos_write, +}; + +/** + * hisi_qm_set_algqos_init() - Initialize function qos debugfs files. + * @qm: The qm for which we want to add debugfs files. + * + * Create function qos debugfs files, VF ping PF to get function qos. + */ +void hisi_qm_set_algqos_init(struct hisi_qm *qm) +{ + if (qm->fun_type == QM_HW_PF) + debugfs_create_file("alg_qos", 0644, qm->debug.debug_root, + qm, &qm_algqos_fops); + else if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) + debugfs_create_file("alg_qos", 0444, qm->debug.debug_root, + qm, &qm_algqos_fops); +} + +static void hisi_qm_init_vf_qos(struct hisi_qm *qm, int total_func) +{ + int i; + + for (i = 1; i <= total_func; i++) + qm->factor[i].func_qos = QM_QOS_MAX_VAL; +} + +/** + * hisi_qm_sriov_enable() - enable virtual functions + * @pdev: the PCIe device + * @max_vfs: the number of virtual functions to enable + * + * Returns the number of enabled VFs. If there are VFs enabled already or + * max_vfs is more than the total number of device can be enabled, returns + * failure. + */ +int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs) +{ + struct hisi_qm *qm = pci_get_drvdata(pdev); + int pre_existing_vfs, num_vfs, total_vfs, ret; + + ret = qm_pm_get_sync(qm); + if (ret) + return ret; + + total_vfs = pci_sriov_get_totalvfs(pdev); + pre_existing_vfs = pci_num_vf(pdev); + if (pre_existing_vfs) { + pci_err(pdev, "%d VFs already enabled. Please disable pre-enabled VFs!\n", + pre_existing_vfs); + goto err_put_sync; + } + + if (max_vfs > total_vfs) { + pci_err(pdev, "%d VFs is more than total VFs %d!\n", max_vfs, total_vfs); + ret = -ERANGE; + goto err_put_sync; + } + + num_vfs = max_vfs; + + if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) + hisi_qm_init_vf_qos(qm, num_vfs); + + ret = qm_vf_q_assign(qm, num_vfs); + if (ret) { + pci_err(pdev, "Can't assign queues for VF!\n"); + goto err_put_sync; + } + + qm->vfs_num = num_vfs; + ret = pci_enable_sriov(pdev, num_vfs); + if (ret) { + pci_err(pdev, "Can't enable VF!\n"); + qm_clear_vft_config(qm); + goto err_put_sync; + } + + pci_info(pdev, "VF enabled, vfs_num(=%d)!\n", num_vfs); + + return num_vfs; + +err_put_sync: + qm_pm_put_sync(qm); + return ret; +} +EXPORT_SYMBOL_GPL(hisi_qm_sriov_enable); + +/** + * hisi_qm_sriov_disable - disable virtual functions + * @pdev: the PCI device. + * @is_frozen: true when all the VFs are frozen. + * + * Return failure if there are VFs assigned already or VF is in used. + */ +int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen) +{ + struct hisi_qm *qm = pci_get_drvdata(pdev); + + if (pci_vfs_assigned(pdev)) { + pci_err(pdev, "Failed to disable VFs as VFs are assigned!\n"); + return -EPERM; + } + + /* While VF is in used, SRIOV cannot be disabled. */ + if (!is_frozen && qm_try_frozen_vfs(pdev, qm->qm_list)) { + pci_err(pdev, "Task is using its VF!\n"); + return -EBUSY; + } + + pci_disable_sriov(pdev); + qm_clear_vft_config(qm); + qm_pm_put_sync(qm); + + return 0; +} +EXPORT_SYMBOL_GPL(hisi_qm_sriov_disable); + +/** + * hisi_qm_sriov_configure - configure the number of VFs + * @pdev: The PCI device + * @num_vfs: The number of VFs need enabled + * + * Enable SR-IOV according to num_vfs, 0 means disable. + */ +int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs) +{ + if (num_vfs == 0) + return hisi_qm_sriov_disable(pdev, false); + + return hisi_qm_sriov_enable(pdev, num_vfs); +} +EXPORT_SYMBOL_GPL(hisi_qm_sriov_configure); + +static enum acc_err_result qm_dev_err_handle(struct hisi_qm *qm) +{ + if (!qm->err_ini->get_err_result) { + dev_err(&qm->pdev->dev, "Device doesn't support reset!\n"); + return ACC_ERR_NONE; + } + + return qm->err_ini->get_err_result(qm); +} + +static enum acc_err_result qm_process_dev_error(struct hisi_qm *qm) +{ + enum acc_err_result qm_ret, dev_ret; + + /* log qm error */ + qm_ret = qm_hw_error_handle(qm); + + /* log device error */ + dev_ret = qm_dev_err_handle(qm); + if (qm_ret == ACC_ERR_NEED_RESET || dev_ret == ACC_ERR_NEED_RESET) + return ACC_ERR_NEED_RESET; + + if (qm_ret == ACC_ERR_NEED_FUNC_RESET) + return ACC_ERR_NEED_FUNC_RESET; + + return ACC_ERR_RECOVERED; +} + +/** + * hisi_qm_dev_err_detected() - Get device and qm error status then log it. + * @pdev: The PCI device which need report error. + * @state: The connectivity between CPU and device. + * + * We register this function into PCIe AER handlers, It will report device or + * qm hardware error status when error occur. + */ +pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + struct hisi_qm *qm = pci_get_drvdata(pdev); + enum acc_err_result ret; + + if (pdev->is_virtfn) + return PCI_ERS_RESULT_NONE; + + pci_info(pdev, "PCI error detected, state(=%u)!!\n", state); + if (state == pci_channel_io_perm_failure) + return PCI_ERS_RESULT_DISCONNECT; + + ret = qm_process_dev_error(qm); + if (ret == ACC_ERR_NEED_RESET || ret == ACC_ERR_NEED_FUNC_RESET) + return PCI_ERS_RESULT_NEED_RESET; + + return PCI_ERS_RESULT_RECOVERED; +} +EXPORT_SYMBOL_GPL(hisi_qm_dev_err_detected); + +static int qm_check_req_recv(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + int ret; + u32 val; + + if (qm->ver >= QM_HW_V3) + return 0; + + writel(ACC_VENDOR_ID_VALUE, qm->io_base + QM_PEH_VENDOR_ID); + ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val, + (val == ACC_VENDOR_ID_VALUE), + POLL_PERIOD, POLL_TIMEOUT); + if (ret) { + dev_err(&pdev->dev, "Fails to read QM reg!\n"); + return ret; + } + + writel(PCI_VENDOR_ID_HUAWEI, qm->io_base + QM_PEH_VENDOR_ID); + ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val, + (val == PCI_VENDOR_ID_HUAWEI), + POLL_PERIOD, POLL_TIMEOUT); + if (ret) + dev_err(&pdev->dev, "Fails to read QM reg in the second time!\n"); + + return ret; +} + +static int qm_set_pf_mse(struct hisi_qm *qm, bool set) +{ + struct pci_dev *pdev = qm->pdev; + u16 cmd; + int i; + + pci_read_config_word(pdev, PCI_COMMAND, &cmd); + if (set) + cmd |= PCI_COMMAND_MEMORY; + else + cmd &= ~PCI_COMMAND_MEMORY; + + pci_write_config_word(pdev, PCI_COMMAND, cmd); + for (i = 0; i < MAX_WAIT_COUNTS; i++) { + pci_read_config_word(pdev, PCI_COMMAND, &cmd); + if (set == ((cmd & PCI_COMMAND_MEMORY) >> 1)) + return 0; + + udelay(1); + } + + return -ETIMEDOUT; +} + +static int qm_set_vf_mse(struct hisi_qm *qm, bool set) +{ + struct pci_dev *pdev = qm->pdev; + u16 sriov_ctrl; + int pos; + int i; + + /* + * Since function qm_set_vf_mse is called only after SRIOV is enabled, + * pci_find_ext_capability cannot return 0, pos does not need to be + * checked. + */ + pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV); + pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &sriov_ctrl); + if (set) + sriov_ctrl |= PCI_SRIOV_CTRL_MSE; + else + sriov_ctrl &= ~PCI_SRIOV_CTRL_MSE; + pci_write_config_word(pdev, pos + PCI_SRIOV_CTRL, sriov_ctrl); + + for (i = 0; i < MAX_WAIT_COUNTS; i++) { + pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &sriov_ctrl); + if (set == (sriov_ctrl & PCI_SRIOV_CTRL_MSE) >> + ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT) + return 0; + + udelay(1); + } + + return -ETIMEDOUT; +} + +static void qm_dev_ecc_mbit_handle(struct hisi_qm *qm) +{ + u32 nfe_enb = 0; + + /* Kunpeng930 hardware automatically close master ooo when NFE occurs */ + if (qm->ver >= QM_HW_V3) + return; + + if (!qm->err_status.is_dev_ecc_mbit && + qm->err_status.is_qm_ecc_mbit && + qm->err_ini->close_axi_master_ooo) { + qm->err_ini->close_axi_master_ooo(qm); + } else if (qm->err_status.is_dev_ecc_mbit && + !qm->err_status.is_qm_ecc_mbit && + !qm->err_ini->close_axi_master_ooo) { + nfe_enb = readl(qm->io_base + QM_RAS_NFE_ENABLE); + writel(nfe_enb & ~qm->err_info.qm_err.ecc_2bits_mask, + qm->io_base + QM_RAS_NFE_ENABLE); + writel(qm->err_info.qm_err.ecc_2bits_mask, qm->io_base + QM_ABNORMAL_INT_SET); + } +} + +static int qm_vf_reset_prepare(struct hisi_qm *qm, + enum qm_stop_reason stop_reason) +{ + struct hisi_qm_list *qm_list = qm->qm_list; + struct pci_dev *pdev = qm->pdev; + struct pci_dev *virtfn; + struct hisi_qm *vf_qm; + int ret = 0; + + mutex_lock(&qm_list->lock); + list_for_each_entry(vf_qm, &qm_list->list, list) { + virtfn = vf_qm->pdev; + if (virtfn == pdev) + continue; + + if (pci_physfn(virtfn) == pdev) { + /* save VFs PCIE BAR configuration */ + pci_save_state(virtfn); + + ret = hisi_qm_stop(vf_qm, stop_reason); + if (ret) + goto stop_fail; + } + } + +stop_fail: + mutex_unlock(&qm_list->lock); + return ret; +} + +static int qm_try_stop_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd, + enum qm_stop_reason stop_reason) +{ + struct pci_dev *pdev = qm->pdev; + int ret; + + if (!qm->vfs_num) + return 0; + + /* Kunpeng930 supports to notify VFs to stop before PF reset */ + if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) { + ret = qm_ping_all_vfs(qm, cmd, 0); + if (ret) + pci_err(pdev, "failed to send command to all VFs before PF reset!\n"); + } else { + ret = qm_vf_reset_prepare(qm, stop_reason); + if (ret) + pci_err(pdev, "failed to prepare reset, ret = %d.\n", ret); + } + + return ret; +} + +static int qm_controller_reset_prepare(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + int ret; + + if (qm->err_ini->set_priv_status) { + ret = qm->err_ini->set_priv_status(qm); + if (ret) + return ret; + } + + ret = qm_reset_prepare_ready(qm); + if (ret) { + pci_err(pdev, "Controller reset not ready!\n"); + return ret; + } + + qm_dev_ecc_mbit_handle(qm); + + /* PF obtains the information of VF by querying the register. */ + qm_cmd_uninit(qm); + + /* Whether VFs stop successfully, soft reset will continue. */ + ret = qm_try_stop_vfs(qm, QM_PF_SRST_PREPARE, QM_SOFT_RESET); + if (ret) + pci_err(pdev, "failed to stop vfs by pf in soft reset.\n"); + + ret = hisi_qm_stop(qm, QM_SOFT_RESET); + if (ret) { + pci_err(pdev, "Fails to stop QM!\n"); + qm_reset_bit_clear(qm); + return ret; + } + + if (qm->use_sva) { + ret = qm_hw_err_isolate(qm); + if (ret) + pci_err(pdev, "failed to isolate hw err!\n"); + } + + ret = qm_wait_vf_prepare_finish(qm); + if (ret) + pci_err(pdev, "failed to stop by vfs in soft reset!\n"); + + qm_dev_db_ctrl(qm, true); + + return 0; +} + +static int qm_master_ooo_check(struct hisi_qm *qm) +{ + u32 val; + int ret; + + /* Check the ooo register of the device before resetting the device. */ + writel(ACC_MASTER_GLOBAL_CTRL_SHUTDOWN, qm->io_base + ACC_MASTER_GLOBAL_CTRL); + ret = readl_relaxed_poll_timeout(qm->io_base + ACC_MASTER_TRANS_RETURN, + val, (val == ACC_MASTER_TRANS_RETURN_RW), + POLL_PERIOD, POLL_TIMEOUT); + if (ret) + pci_warn(qm->pdev, "Bus lock! Please reset system.\n"); + + return ret; +} + +static int qm_soft_reset_prepare(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + int ret; + + /* Ensure all doorbells and mailboxes received by QM */ + ret = qm_check_req_recv(qm); + if (ret) + return ret; + + if (qm->vfs_num) { + ret = qm_set_vf_mse(qm, false); + if (ret) { + pci_err(pdev, "Fails to disable vf MSE bit.\n"); + return ret; + } + } + + ret = qm->ops->set_msi(qm, false); + if (ret) { + pci_err(pdev, "Fails to disable PEH MSI bit.\n"); + return ret; + } + + ret = qm_master_ooo_check(qm); + if (ret) + return ret; + + if (qm->err_ini->close_sva_prefetch) + qm->err_ini->close_sva_prefetch(qm); + + ret = qm_set_pf_mse(qm, false); + if (ret) + pci_err(pdev, "Fails to disable pf MSE bit.\n"); + + return ret; +} + +static int qm_reset_device(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + + /* The reset related sub-control registers are not in PCI BAR */ + if (ACPI_HANDLE(&pdev->dev)) { + unsigned long long value = 0; + acpi_status s; + + s = acpi_evaluate_integer(ACPI_HANDLE(&pdev->dev), + qm->err_info.acpi_rst, + NULL, &value); + if (ACPI_FAILURE(s)) { + pci_err(pdev, "NO controller reset method!\n"); + return -EIO; + } + + if (value) { + pci_err(pdev, "Reset step %llu failed!\n", value); + return -EIO; + } + + return 0; + } + + pci_err(pdev, "No reset method!\n"); + return -EINVAL; +} + +static int qm_soft_reset(struct hisi_qm *qm) +{ + int ret; + + ret = qm_soft_reset_prepare(qm); + if (ret) + return ret; + + return qm_reset_device(qm); +} + +static int qm_vf_reset_done(struct hisi_qm *qm) +{ + struct hisi_qm_list *qm_list = qm->qm_list; + struct pci_dev *pdev = qm->pdev; + struct pci_dev *virtfn; + struct hisi_qm *vf_qm; + int ret = 0; + + mutex_lock(&qm_list->lock); + list_for_each_entry(vf_qm, &qm_list->list, list) { + virtfn = vf_qm->pdev; + if (virtfn == pdev) + continue; + + if (pci_physfn(virtfn) == pdev) { + /* enable VFs PCIE BAR configuration */ + pci_restore_state(virtfn); + + ret = qm_restart(vf_qm); + if (ret) + goto restart_fail; + } + } + +restart_fail: + mutex_unlock(&qm_list->lock); + return ret; +} + +static int qm_try_start_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd) +{ + struct pci_dev *pdev = qm->pdev; + u32 data; + int ret; + + if (!qm->vfs_num) + return 0; + + ret = qm_vf_q_assign(qm, qm->vfs_num); + if (ret) { + pci_err(pdev, "failed to assign VFs, ret = %d.\n", ret); + return ret; + } + + /* Kunpeng930 supports to notify VFs to start after PF reset. */ + if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) { + data = qm->isolate_data.err_threshold; + if (qm->isolate_data.is_isolate) + data |= QM_ISOLATED_STATE; + /* Broadcasting isolate info via RAS to all VFs. */ + ret = qm_ping_all_vfs(qm, cmd, data); + if (ret) + pci_warn(pdev, "failed to send cmd to all VFs after PF reset!\n"); + } else { + ret = qm_vf_reset_done(qm); + if (ret) + pci_warn(pdev, "failed to start vfs, ret = %d.\n", ret); + } + + return ret; +} + +static int qm_dev_hw_init(struct hisi_qm *qm) +{ + return qm->err_ini->hw_init(qm); +} + +static void qm_restart_prepare(struct hisi_qm *qm) +{ + u32 value; + + if (qm->ver >= QM_HW_V3) + return; + + if (!qm->err_status.is_qm_ecc_mbit && + !qm->err_status.is_dev_ecc_mbit) + return; + + /* temporarily close the OOO port used for PEH to write out MSI */ + value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN); + writel(value & ~qm->err_info.msi_wr_port, + qm->io_base + ACC_AM_CFG_PORT_WR_EN); + + /* clear dev ecc 2bit error source if having */ + value = qm_get_dev_err_status(qm) & qm->err_info.dev_err.ecc_2bits_mask; + if (value && qm->err_ini->clear_dev_hw_err_status) + qm->err_ini->clear_dev_hw_err_status(qm, value); + + /* clear QM ecc mbit error source */ + writel(qm->err_info.qm_err.ecc_2bits_mask, qm->io_base + QM_ABNORMAL_INT_SOURCE); + + /* clear AM Reorder Buffer ecc mbit source */ + writel(ACC_ROB_ECC_ERR_MULTPL, qm->io_base + ACC_AM_ROB_ECC_INT_STS); +} + +static void qm_restart_done(struct hisi_qm *qm) +{ + u32 value; + + if (qm->ver >= QM_HW_V3) + goto clear_flags; + + if (!qm->err_status.is_qm_ecc_mbit && + !qm->err_status.is_dev_ecc_mbit) + return; + + /* open the OOO port for PEH to write out MSI */ + value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN); + value |= qm->err_info.msi_wr_port; + writel(value, qm->io_base + ACC_AM_CFG_PORT_WR_EN); + +clear_flags: + qm->err_status.is_qm_ecc_mbit = false; + qm->err_status.is_dev_ecc_mbit = false; +} + +static void qm_disable_axi_error(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *qm_err = &qm->err_info.qm_err; + u32 val; + + val = ~(qm->error_mask & (~QM_RAS_AXI_ERROR)); + writel(val, qm->io_base + QM_ABNORMAL_INT_MASK); + if (qm->ver > QM_HW_V2) + writel(qm_err->shutdown_mask & (~QM_RAS_AXI_ERROR), + qm->io_base + QM_OOO_SHUTDOWN_SEL); + + if (qm->err_ini->disable_axi_error) + qm->err_ini->disable_axi_error(qm); +} + +static void qm_enable_axi_error(struct hisi_qm *qm) +{ + /* clear axi error source */ + writel(QM_RAS_AXI_ERROR, qm->io_base + QM_ABNORMAL_INT_SOURCE); + + writel(~qm->error_mask, qm->io_base + QM_ABNORMAL_INT_MASK); + if (qm->ver > QM_HW_V2) + writel(qm->err_info.qm_err.shutdown_mask, qm->io_base + QM_OOO_SHUTDOWN_SEL); + + if (qm->err_ini->enable_axi_error) + qm->err_ini->enable_axi_error(qm); +} + +static int qm_controller_reset_done(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + int ret; + + ret = qm->ops->set_msi(qm, true); + if (ret) { + pci_err(pdev, "Fails to enable PEH MSI bit!\n"); + return ret; + } + + ret = qm_set_pf_mse(qm, true); + if (ret) { + pci_err(pdev, "Fails to enable pf MSE bit!\n"); + return ret; + } + + if (qm->vfs_num) { + ret = qm_set_vf_mse(qm, true); + if (ret) { + pci_err(pdev, "Fails to enable vf MSE bit!\n"); + return ret; + } + } + + ret = qm_dev_hw_init(qm); + if (ret) { + pci_err(pdev, "Failed to init device\n"); + return ret; + } + + qm_restart_prepare(qm); + hisi_qm_dev_err_init(qm); + qm_disable_axi_error(qm); + if (qm->err_ini->open_axi_master_ooo) + qm->err_ini->open_axi_master_ooo(qm); + + ret = qm_dev_mem_reset(qm); + if (ret) { + pci_err(pdev, "failed to reset device memory\n"); + return ret; + } + + ret = qm_restart(qm); + if (ret) { + pci_err(pdev, "Failed to start QM!\n"); + return ret; + } + + ret = qm_try_start_vfs(qm, QM_PF_RESET_DONE); + if (ret) + pci_err(pdev, "failed to start vfs by pf in soft reset.\n"); + + ret = qm_wait_vf_prepare_finish(qm); + if (ret) + pci_err(pdev, "failed to start by vfs in soft reset!\n"); + qm_enable_axi_error(qm); + qm_cmd_init(qm); + qm_restart_done(qm); + + qm_reset_bit_clear(qm); + + return 0; +} + +static int qm_controller_reset(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + int ret; + + pci_info(pdev, "Controller resetting...\n"); + + ret = qm_controller_reset_prepare(qm); + if (ret) { + hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); + hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); + return ret; + } + + hisi_qm_show_last_dfx_regs(qm); + if (qm->err_ini->show_last_dfx_regs) + qm->err_ini->show_last_dfx_regs(qm); + + ret = qm_soft_reset(qm); + if (ret) + goto err_reset; + + ret = qm_controller_reset_done(qm); + if (ret) + goto err_reset; + + pci_info(pdev, "Controller reset complete\n"); + + return 0; + +err_reset: + pci_err(pdev, "Controller reset failed (%d)\n", ret); + qm_reset_bit_clear(qm); + + /* if resetting fails, isolate the device */ + if (qm->use_sva) + qm->isolate_data.is_isolate = true; + return ret; +} + +/** + * hisi_qm_dev_slot_reset() - slot reset + * @pdev: the PCIe device + * + * This function offers QM relate PCIe device reset interface. Drivers which + * use QM can use this function as slot_reset in its struct pci_error_handlers. + */ +pci_ers_result_t hisi_qm_dev_slot_reset(struct pci_dev *pdev) +{ + struct hisi_qm *qm = pci_get_drvdata(pdev); + int ret; + + if (pdev->is_virtfn) + return PCI_ERS_RESULT_RECOVERED; + + /* reset pcie device controller */ + ret = qm_controller_reset(qm); + if (ret) { + pci_err(pdev, "Controller reset failed (%d)\n", ret); + return PCI_ERS_RESULT_DISCONNECT; + } + + return PCI_ERS_RESULT_RECOVERED; +} +EXPORT_SYMBOL_GPL(hisi_qm_dev_slot_reset); + +void hisi_qm_reset_prepare(struct pci_dev *pdev) +{ + struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev)); + struct hisi_qm *qm = pci_get_drvdata(pdev); + u32 delay = 0; + int ret; + + /* + * Check whether there is an ECC mbit error, If it occurs, need to + * wait for soft reset to fix it. + */ + while (qm_check_dev_error(qm)) { + msleep(++delay); + if (delay > QM_RESET_WAIT_TIMEOUT) + return; + } + + ret = qm_reset_prepare_ready(qm); + if (ret) { + pci_err(pdev, "FLR not ready!\n"); + return; + } + + hisi_qm_dev_err_uninit(pf_qm); + + /* PF obtains the information of VF by querying the register. */ + if (qm->fun_type == QM_HW_PF) + qm_cmd_uninit(qm); + + ret = qm_try_stop_vfs(qm, QM_PF_FLR_PREPARE, QM_DOWN); + if (ret) + pci_err(pdev, "failed to stop vfs by pf in FLR.\n"); + + ret = hisi_qm_stop(qm, QM_DOWN); + if (ret) { + pci_err(pdev, "Failed to stop QM, ret = %d.\n", ret); + goto err_prepare; + } + + ret = qm_wait_vf_prepare_finish(qm); + if (ret) + pci_err(pdev, "failed to stop by vfs in FLR!\n"); + + qm_dev_db_ctrl(qm, true); + + pci_info(pdev, "FLR resetting...\n"); + + return; + +err_prepare: + pci_info(pdev, "FLR resetting prepare failed!\n"); + atomic_set(&qm->status.flags, QM_STOP); + hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); + hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); + qm_dev_db_ctrl(qm, true); +} +EXPORT_SYMBOL_GPL(hisi_qm_reset_prepare); + +static bool qm_flr_reset_complete(struct pci_dev *pdev) +{ + struct pci_dev *pf_pdev = pci_physfn(pdev); + struct hisi_qm *qm = pci_get_drvdata(pf_pdev); + u32 id; + + pci_read_config_dword(qm->pdev, PCI_COMMAND, &id); + if (id == QM_PCI_COMMAND_INVALID) { + pci_err(pdev, "Device can not be used!\n"); + return false; + } + + return true; +} + +void hisi_qm_reset_done(struct pci_dev *pdev) +{ + struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev)); + struct hisi_qm *qm = pci_get_drvdata(pdev); + int ret; + + if (qm->fun_type == QM_HW_PF) { + ret = qm_dev_hw_init(qm); + if (ret) { + pci_err(pdev, "Failed to init PF, ret = %d.\n", ret); + goto flr_done; + } + } + + hisi_qm_dev_err_init(pf_qm); + + ret = qm_restart(qm); + if (ret) { + pci_err(pdev, "Failed to start QM, ret = %d.\n", ret); + goto flr_done; + } + + ret = qm_try_start_vfs(qm, QM_PF_RESET_DONE); + if (ret) + pci_err(pdev, "failed to start vfs by pf in FLR.\n"); + + ret = qm_wait_vf_prepare_finish(qm); + if (ret) + pci_err(pdev, "failed to start by vfs in FLR!\n"); + +flr_done: + if (qm->fun_type == QM_HW_PF) + qm_cmd_init(qm); + + if (qm_flr_reset_complete(pdev)) + pci_info(pdev, "FLR reset complete\n"); + + qm_reset_bit_clear(qm); +} +EXPORT_SYMBOL_GPL(hisi_qm_reset_done); + +static irqreturn_t qm_rsvd_irq(int irq, void *data) +{ + struct hisi_qm *qm = data; + + dev_info(&qm->pdev->dev, "Reserved interrupt, ignore!\n"); + + return IRQ_HANDLED; +} + +static irqreturn_t qm_abnormal_irq(int irq, void *data) +{ + struct hisi_qm *qm = data; + + atomic64_inc(&qm->debug.dfx.abnormal_irq_cnt); + + if (!test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl)) + schedule_work(&qm->rst_work); + else + pci_warn(qm->pdev, "Driver is down, need to reload driver!\n"); + + return IRQ_HANDLED; +} + +/** + * hisi_qm_dev_shutdown() - Shutdown device. + * @pdev: The device will be shutdown. + * + * This function will stop qm when OS shutdown or rebooting. + */ +void hisi_qm_dev_shutdown(struct pci_dev *pdev) +{ + struct hisi_qm *qm = pci_get_drvdata(pdev); + int ret; + + ret = hisi_qm_stop(qm, QM_SHUTDOWN); + if (ret) + dev_err(&pdev->dev, "Fail to stop qm in shutdown!\n"); +} +EXPORT_SYMBOL_GPL(hisi_qm_dev_shutdown); + +static u64 qm_get_function_mask(struct hisi_qm *qm) +{ + return readq(qm->io_base + QM_FUNC_AXI_ERR_ST0); +} + +static void qm_clear_function_mask(struct hisi_qm *qm, u64 func_mask) +{ + /* Register write 1 clear */ + writeq(func_mask, qm->io_base + QM_FUNC_AXI_ERR_ST0); +} + +static void qm_function_reset(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + u64 func_mask; + u32 fun_num; + int ret; + + func_mask = qm_get_function_mask(qm); + if (!func_mask) { + dev_info(dev, "no function need reset!\n"); + return; + } + + for (fun_num = 1; fun_num <= qm->vfs_num; fun_num++) { + if (func_mask & BIT(fun_num)) { + ret = qm_ping_single_vf(qm, QM_FUNCTION_RESET, 0, fun_num); + /* When function ping fail, user decides the VF reset method. */ + if (ret) + dev_err(dev, "failed to send command(0x%x) to VF(%u)!\n", + (unsigned int)QM_FUNCTION_RESET, fun_num); + } + } + + if (func_mask & BIT(0)) { + dev_info(dev, "function reset start...\n"); + qm_reset_function(qm); + dev_info(dev, "function reset end!\n"); + } + + qm_clear_function_mask(qm, func_mask); +} + +static void hisi_qm_controller_reset(struct work_struct *rst_work) +{ + struct hisi_qm *qm = container_of(rst_work, struct hisi_qm, rst_work); + enum acc_err_result err_result; + int ret; + + ret = qm_pm_get_sync(qm); + if (ret) { + dev_err(&qm->pdev->dev, "failed to get runtime PM for controller\n"); + return; + } + + err_result = qm_process_dev_error(qm); + if (err_result == ACC_ERR_NEED_RESET) + (void)qm_controller_reset(qm); + else if (err_result == ACC_ERR_NEED_FUNC_RESET) + qm_function_reset(qm); + + qm_pm_put_sync(qm); +} + +static void qm_pf_reset_vf_prepare(struct hisi_qm *qm, + enum qm_stop_reason stop_reason) +{ + enum qm_ifc_cmd cmd = QM_VF_PREPARE_DONE; + struct pci_dev *pdev = qm->pdev; + int ret; + + ret = qm_reset_prepare_ready(qm); + if (ret) { + dev_err(&pdev->dev, "reset prepare not ready!\n"); + atomic_set(&qm->status.flags, QM_STOP); + cmd = QM_VF_PREPARE_FAIL; + goto err_prepare; + } + + ret = hisi_qm_stop(qm, stop_reason); + if (ret) { + dev_err(&pdev->dev, "failed to stop QM, ret = %d.\n", ret); + atomic_set(&qm->status.flags, QM_STOP); + cmd = QM_VF_PREPARE_FAIL; + goto err_prepare; + } else { + goto out; + } + +err_prepare: + hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); + hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); +out: + pci_save_state(pdev); + ret = qm_ping_pf(qm, cmd); + if (ret) + dev_warn(&pdev->dev, "PF responds timeout in reset prepare!\n"); +} + +static void qm_pf_reset_vf_done(struct hisi_qm *qm) +{ + enum qm_ifc_cmd cmd = QM_VF_START_DONE; + struct pci_dev *pdev = qm->pdev; + int ret; + + pci_restore_state(pdev); + ret = qm_restart(qm); + if (ret) { + dev_err(&pdev->dev, "failed to start QM, ret = %d.\n", ret); + cmd = QM_VF_START_FAIL; + } + + qm_cmd_init(qm); + ret = qm_ping_pf(qm, cmd); + if (ret) + dev_warn(&pdev->dev, "PF responds timeout in reset done!\n"); + + qm_reset_bit_clear(qm); +} + +static void qm_vf_update_isolate_info(struct hisi_qm *qm, u32 data) +{ + /* Updating the local isolation status. */ + mutex_lock(&qm->isolate_data.isolate_lock); + if (data & QM_ISOLATED_STATE) + qm->isolate_data.is_isolate = true; + else + qm->isolate_data.is_isolate = false; + qm->isolate_data.err_threshold = data & QM_ISOLATED_THRESHOLD_MASK; + mutex_unlock(&qm->isolate_data.isolate_lock); +} + +static int qm_wait_pf_reset_finish(struct hisi_qm *qm, enum qm_stop_reason stop_reason) +{ + struct device *dev = &qm->pdev->dev; + u32 val, cmd, data; + int ret; + + /* Wait for reset to finish */ + ret = readl_relaxed_poll_timeout(qm->io_base + QM_IFC_INT_SOURCE_V, val, + val == BIT(0), QM_VF_RESET_WAIT_US, + QM_VF_RESET_WAIT_TIMEOUT_US); + /* hardware completion status should be available by this time */ + if (ret) { + dev_err(dev, "couldn't get reset done status from PF, timeout!\n"); + return -ETIMEDOUT; + } + + /* + * Whether message is got successfully, + * VF needs to ack PF by clearing the interrupt. + */ + ret = qm->ops->get_ifc(qm, &cmd, &data, 0); + qm_clear_cmd_interrupt(qm, 0); + if (ret) { + dev_err(dev, "failed to get command from PF in reset done!\n"); + return ret; + } + + if (cmd != QM_PF_RESET_DONE) { + dev_err(dev, "the command(0x%x) is not reset done!\n", cmd); + return -EINVAL; + } + + /* The VF processes the device isolation information received from the RAS reset. */ + if (stop_reason == QM_SOFT_RESET) + qm_vf_update_isolate_info(qm, data); + + return 0; +} + +static void qm_pf_reset_vf_process(struct hisi_qm *qm, + enum qm_stop_reason stop_reason) +{ + struct device *dev = &qm->pdev->dev; + int ret; + + dev_info(dev, "device reset start...\n"); + + /* The message is obtained by querying the register during resetting */ + qm_cmd_uninit(qm); + qm_pf_reset_vf_prepare(qm, stop_reason); + + ret = qm_wait_pf_reset_finish(qm, stop_reason); + if (ret) + goto err_get_status; + + qm_pf_reset_vf_done(qm); + + dev_info(dev, "device reset done.\n"); + + return; + +err_get_status: + if (stop_reason == QM_SOFT_RESET) { + /* Update local isolation status on PF-VF reset failure. */ + mutex_lock(&qm->isolate_data.isolate_lock); + qm->isolate_data.is_isolate = true; + mutex_unlock(&qm->isolate_data.isolate_lock); + } + qm_cmd_init(qm); + qm_reset_bit_clear(qm); +} + +static void qm_vf_get_isolate_data(struct hisi_qm *qm, u32 fun_num) +{ + u32 data = qm->isolate_data.err_threshold; + struct device *dev = &qm->pdev->dev; + int ret; + + if (qm->isolate_data.is_isolate) + data |= QM_ISOLATED_STATE; + + ret = qm_ping_single_vf(qm, QM_PF_SET_ISOLATE, data, fun_num); + if (ret) + dev_err(dev, "failed to send command(0x%x) to VF(%u)!\n", + (unsigned int)QM_PF_SET_ISOLATE, fun_num); +} + +static void qm_handle_cmd_msg(struct hisi_qm *qm, u32 fun_num) +{ + struct device *dev = &qm->pdev->dev; + enum qm_ifc_cmd cmd; + u32 data; + int ret; + + /* + * Get the msg from source by sending mailbox. Whether message is got + * successfully, destination needs to ack source by clearing the interrupt. + */ + ret = qm->ops->get_ifc(qm, &cmd, &data, fun_num); + qm_clear_cmd_interrupt(qm, BIT(fun_num)); + if (ret) { + dev_err(dev, "failed to get command from source!\n"); + return; + } + + switch (cmd) { + case QM_PF_FLR_PREPARE: + qm_pf_reset_vf_process(qm, QM_DOWN); + break; + case QM_PF_SRST_PREPARE: + qm_pf_reset_vf_process(qm, QM_SOFT_RESET); + break; + case QM_VF_GET_QOS: + qm_vf_get_qos(qm, fun_num); + break; + case QM_PF_SET_QOS: + qm->mb_qos = data; + break; + case QM_FUNCTION_RESET: + dev_info(dev, "function reset start...\n"); + qm_reset_function(qm); + dev_info(dev, "function reset end!\n"); + break; + case QM_VF_GET_ISOLATE: + /* Read the isolation policy of the PF during VF initialization. */ + qm_vf_get_isolate_data(qm, fun_num); + break; + case QM_PF_SET_ISOLATE: + qm_vf_update_isolate_info(qm, data); + break; + default: + dev_err(dev, "unsupported command(0x%x) sent by function(%u)!\n", cmd, fun_num); + break; + } +} + +static void qm_cmd_process(struct work_struct *cmd_process) +{ + struct hisi_qm *qm = container_of(cmd_process, + struct hisi_qm, cmd_process); + u32 vfs_num = qm->vfs_num; + u64 val; + u32 i; + + if (qm->fun_type == QM_HW_PF) { + val = readq(qm->io_base + QM_IFC_INT_SOURCE_P); + if (!val) + return; + + for (i = 1; i <= vfs_num; i++) { + if (val & BIT(i)) + qm_handle_cmd_msg(qm, i); + } + + return; + } + + qm_handle_cmd_msg(qm, 0); +} + +/** + * hisi_qm_alg_register() - Register alg to crypto. + * @qm: The qm needs add. + * @qm_list: The qm list. + * @guard: Guard of qp_num. + * + * Register algorithm to crypto when the function is satisfy guard. + */ +int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list, int guard) +{ + struct device *dev = &qm->pdev->dev; + + if (qm->ver <= QM_HW_V2 && qm->use_sva) { + dev_info(dev, "HW V2 not both use uacce sva mode and hardware crypto algs.\n"); + return 0; + } + + if (qm->qp_num < guard) { + dev_info(dev, "qp_num is less than task need.\n"); + return 0; + } + + return qm_list->register_to_crypto(qm); +} +EXPORT_SYMBOL_GPL(hisi_qm_alg_register); + +/** + * hisi_qm_alg_unregister() - Unregister alg from crypto. + * @qm: The qm needs delete. + * @qm_list: The qm list. + * @guard: Guard of qp_num. + * + * Unregister algorithm from crypto when the last function is satisfy guard. + */ +void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list, int guard) +{ + if (qm->ver <= QM_HW_V2 && qm->use_sva) + return; + + if (qm->qp_num < guard) + return; + + qm_list->unregister_from_crypto(qm); +} +EXPORT_SYMBOL_GPL(hisi_qm_alg_unregister); + +static void qm_unregister_abnormal_irq(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 irq_vector, val; + + if (qm->fun_type == QM_HW_VF && qm->ver < QM_HW_V3) + return; + + val = qm->cap_tables.qm_cap_table[QM_ABNORMAL_IRQ].cap_val; + if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK)) + return; + + irq_vector = val & QM_IRQ_VECTOR_MASK; + free_irq(pci_irq_vector(pdev, irq_vector), qm); +} + +static int qm_register_abnormal_irq(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 irq_vector, val; + int ret; + + val = qm->cap_tables.qm_cap_table[QM_ABNORMAL_IRQ].cap_val; + if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK)) + return 0; + irq_vector = val & QM_IRQ_VECTOR_MASK; + + /* For VF, this is a reserved interrupt in V3 version. */ + if (qm->fun_type == QM_HW_VF) { + if (qm->ver < QM_HW_V3) + return 0; + + ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_rsvd_irq, + IRQF_NO_AUTOEN, qm->dev_name, qm); + if (ret) { + dev_err(&pdev->dev, "failed to request reserved irq, ret = %d!\n", ret); + return ret; + } + return 0; + } + + ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_abnormal_irq, 0, qm->dev_name, qm); + if (ret) + dev_err(&qm->pdev->dev, "failed to request abnormal irq, ret = %d!\n", ret); + + return ret; +} + +static void qm_unregister_mb_cmd_irq(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 irq_vector, val; + + val = qm->cap_tables.qm_cap_table[QM_MB_IRQ].cap_val; + if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) + return; + + irq_vector = val & QM_IRQ_VECTOR_MASK; + free_irq(pci_irq_vector(pdev, irq_vector), qm); +} + +static int qm_register_mb_cmd_irq(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 irq_vector, val; + int ret; + + val = qm->cap_tables.qm_cap_table[QM_MB_IRQ].cap_val; + if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) + return 0; + + irq_vector = val & QM_IRQ_VECTOR_MASK; + ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_mb_cmd_irq, 0, qm->dev_name, qm); + if (ret) + dev_err(&pdev->dev, "failed to request function communication irq, ret = %d", ret); + + return ret; +} + +static void qm_unregister_aeq_irq(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 irq_vector, val; + + val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ].cap_val; + if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) + return; + + irq_vector = val & QM_IRQ_VECTOR_MASK; + free_irq(pci_irq_vector(pdev, irq_vector), qm); +} + +static int qm_register_aeq_irq(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 irq_vector, val; + int ret; + + val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ].cap_val; + if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) + return 0; + + irq_vector = val & QM_IRQ_VECTOR_MASK; + ret = request_threaded_irq(pci_irq_vector(pdev, irq_vector), NULL, + qm_aeq_thread, IRQF_ONESHOT, qm->dev_name, qm); + if (ret) + dev_err(&pdev->dev, "failed to request eq irq, ret = %d", ret); + + return ret; +} + +static void qm_unregister_eq_irq(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 irq_vector, val; + + val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ].cap_val; + if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) + return; + + irq_vector = val & QM_IRQ_VECTOR_MASK; + free_irq(pci_irq_vector(pdev, irq_vector), qm); +} + +static int qm_register_eq_irq(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 irq_vector, val; + int ret; + + val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ].cap_val; + if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) + return 0; + + irq_vector = val & QM_IRQ_VECTOR_MASK; + ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_eq_irq, 0, qm->dev_name, qm); + if (ret) + dev_err(&pdev->dev, "failed to request eq irq, ret = %d", ret); + + return ret; +} + +static void qm_irqs_unregister(struct hisi_qm *qm) +{ + qm_unregister_mb_cmd_irq(qm); + qm_unregister_abnormal_irq(qm); + qm_unregister_aeq_irq(qm); + qm_unregister_eq_irq(qm); +} + +static int qm_irqs_register(struct hisi_qm *qm) +{ + int ret; + + ret = qm_register_eq_irq(qm); + if (ret) + return ret; + + ret = qm_register_aeq_irq(qm); + if (ret) + goto free_eq_irq; + + ret = qm_register_abnormal_irq(qm); + if (ret) + goto free_aeq_irq; + + ret = qm_register_mb_cmd_irq(qm); + if (ret) + goto free_abnormal_irq; + + return 0; + +free_abnormal_irq: + qm_unregister_abnormal_irq(qm); +free_aeq_irq: + qm_unregister_aeq_irq(qm); +free_eq_irq: + qm_unregister_eq_irq(qm); + return ret; +} + +static int qm_get_qp_num(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + bool is_db_isolation; + + /* VF's qp_num assigned by PF in v2, and VF can get qp_num by vft. */ + if (qm->fun_type == QM_HW_VF) { + if (qm->ver != QM_HW_V1) + /* v2 starts to support get vft by mailbox */ + return hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num); + + return 0; + } + + is_db_isolation = test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps); + qm->ctrl_qp_num = hisi_qm_get_hw_info(qm, qm_basic_info, QM_TOTAL_QP_NUM_CAP, true); + qm->max_qp_num = hisi_qm_get_hw_info(qm, qm_basic_info, + QM_FUNC_MAX_QP_CAP, is_db_isolation); + + if (qm->qp_num <= qm->max_qp_num) + return 0; + + if (test_bit(QM_MODULE_PARAM, &qm->misc_ctl)) { + /* Check whether the set qp number is valid */ + dev_err(dev, "qp num(%u) is more than max qp num(%u)!\n", + qm->qp_num, qm->max_qp_num); + return -EINVAL; + } + + dev_info(dev, "Default qp num(%u) is too big, reset it to Function's max qp num(%u)!\n", + qm->qp_num, qm->max_qp_num); + qm->qp_num = qm->max_qp_num; + qm->debug.curr_qm_qp_num = qm->qp_num; + + return 0; +} + +static int qm_pre_store_caps(struct hisi_qm *qm) +{ + struct hisi_qm_cap_record *qm_cap; + struct pci_dev *pdev = qm->pdev; + size_t i, size; + + size = ARRAY_SIZE(qm_cap_query_info); + qm_cap = devm_kcalloc(&pdev->dev, sizeof(*qm_cap), size, GFP_KERNEL); + if (!qm_cap) + return -ENOMEM; + + for (i = 0; i < size; i++) { + qm_cap[i].type = qm_cap_query_info[i].type; + qm_cap[i].name = qm_cap_query_info[i].name; + qm_cap[i].cap_val = hisi_qm_get_cap_value(qm, qm_cap_query_info, + i, qm->cap_ver); + } + + qm->cap_tables.qm_cap_table = qm_cap; + qm->cap_tables.qm_cap_size = size; + + return 0; +} + +static int qm_get_hw_caps(struct hisi_qm *qm) +{ + const struct hisi_qm_cap_info *cap_info = qm->fun_type == QM_HW_PF ? + qm_cap_info_pf : qm_cap_info_vf; + u32 size = qm->fun_type == QM_HW_PF ? ARRAY_SIZE(qm_cap_info_pf) : + ARRAY_SIZE(qm_cap_info_vf); + u32 val, i; + + /* Doorbell isolate register is a independent register. */ + val = hisi_qm_get_hw_info(qm, qm_cap_info_comm, QM_SUPPORT_DB_ISOLATION, true); + if (val) + set_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps); + + if (qm->ver >= QM_HW_V3) { + val = readl(qm->io_base + QM_FUNC_CAPS_REG); + qm->cap_ver = val & QM_CAPBILITY_VERSION; + } + + /* Get PF/VF common capbility */ + for (i = 1; i < ARRAY_SIZE(qm_cap_info_comm); i++) { + val = hisi_qm_get_hw_info(qm, qm_cap_info_comm, i, qm->cap_ver); + if (val) + set_bit(qm_cap_info_comm[i].type, &qm->caps); + } + + /* Get PF/VF different capbility */ + for (i = 0; i < size; i++) { + val = hisi_qm_get_hw_info(qm, cap_info, i, qm->cap_ver); + if (val) + set_bit(cap_info[i].type, &qm->caps); + } + + /* Fetch and save the value of qm capability registers */ + return qm_pre_store_caps(qm); +} + +static void qm_get_version(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 sub_version_id; + + qm->ver = pdev->revision; + + if (pdev->revision == QM_HW_V3) { + sub_version_id = readl(qm->io_base + QM_SUB_VERSION_ID); + if (sub_version_id) + qm->ver = sub_version_id; + } +} + +static int qm_get_pci_res(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + struct device *dev = &pdev->dev; + int ret; + + ret = pci_request_mem_regions(pdev, qm->dev_name); + if (ret < 0) { + dev_err(dev, "Failed to request mem regions!\n"); + return ret; + } + + qm->phys_base = pci_resource_start(pdev, PCI_BAR_2); + qm->io_base = ioremap(qm->phys_base, pci_resource_len(pdev, PCI_BAR_2)); + if (!qm->io_base) { + ret = -EIO; + goto err_request_mem_regions; + } + + qm_get_version(qm); + + ret = qm_get_hw_caps(qm); + if (ret) + goto err_ioremap; + + if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) { + qm->db_interval = QM_QP_DB_INTERVAL; + qm->db_phys_base = pci_resource_start(pdev, PCI_BAR_4); + qm->db_io_base = ioremap(qm->db_phys_base, + pci_resource_len(pdev, PCI_BAR_4)); + if (!qm->db_io_base) { + ret = -EIO; + goto err_ioremap; + } + } else { + qm->db_phys_base = qm->phys_base; + qm->db_io_base = qm->io_base; + qm->db_interval = 0; + } + + hisi_qm_pre_init(qm); + ret = qm_get_qp_num(qm); + if (ret) + goto err_db_ioremap; + + return 0; + +err_db_ioremap: + if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) + iounmap(qm->db_io_base); +err_ioremap: + iounmap(qm->io_base); +err_request_mem_regions: + pci_release_mem_regions(pdev); + return ret; +} + +static int qm_clear_device(struct hisi_qm *qm) +{ + acpi_handle handle = ACPI_HANDLE(&qm->pdev->dev); + int ret; + + if (qm->fun_type == QM_HW_VF) + return 0; + + /* Device does not support reset, return */ + if (!qm->err_ini->err_info_init) + return 0; + qm->err_ini->err_info_init(qm); + + if (!handle) + return 0; + + /* No reset method, return */ + if (!acpi_has_method(handle, qm->err_info.acpi_rst)) + return 0; + + ret = qm_master_ooo_check(qm); + if (ret) { + writel(0x0, qm->io_base + ACC_MASTER_GLOBAL_CTRL); + return ret; + } + + if (qm->err_ini->set_priv_status) { + ret = qm->err_ini->set_priv_status(qm); + if (ret) { + writel(0x0, qm->io_base + ACC_MASTER_GLOBAL_CTRL); + return ret; + } + } + + return qm_reset_device(qm); +} + +static int hisi_qm_pci_init(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + struct device *dev = &pdev->dev; + unsigned int num_vec; + int ret; + + ret = pci_enable_device_mem(pdev); + if (ret < 0) { + dev_err(dev, "Failed to enable device mem!\n"); + return ret; + } + + ret = qm_get_pci_res(qm); + if (ret) + goto err_disable_pcidev; + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); + if (ret < 0) + goto err_get_pci_res; + pci_set_master(pdev); + + num_vec = qm_get_irq_num(qm); + if (!num_vec) { + dev_err(dev, "Device irq num is zero!\n"); + ret = -EINVAL; + goto err_get_pci_res; + } + num_vec = roundup_pow_of_two(num_vec); + ret = pci_alloc_irq_vectors(pdev, num_vec, num_vec, PCI_IRQ_MSI); + if (ret < 0) { + dev_err(dev, "Failed to enable MSI vectors!\n"); + goto err_get_pci_res; + } + + ret = qm_clear_device(qm); + if (ret) + goto err_free_vectors; + + return 0; + +err_free_vectors: + pci_free_irq_vectors(pdev); +err_get_pci_res: + qm_put_pci_res(qm); +err_disable_pcidev: + pci_disable_device(pdev); + return ret; +} + +static int hisi_qm_init_work(struct hisi_qm *qm) +{ + int i; + + for (i = 0; i < qm->qp_num; i++) + INIT_WORK(&qm->poll_data[i].work, qm_work_process); + + if (qm->fun_type == QM_HW_PF) + INIT_WORK(&qm->rst_work, hisi_qm_controller_reset); + + if (qm->ver > QM_HW_V2) + INIT_WORK(&qm->cmd_process, qm_cmd_process); + + qm->wq = alloc_workqueue("%s", WQ_HIGHPRI | WQ_MEM_RECLAIM | + WQ_UNBOUND, num_online_cpus(), + pci_name(qm->pdev)); + if (!qm->wq) { + pci_err(qm->pdev, "failed to alloc workqueue!\n"); + return -ENOMEM; + } + + return 0; +} + +static int hisi_qp_alloc_memory(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + u16 sq_depth, cq_depth; + size_t qp_dma_size; + int i, ret; + + qm->qp_array = kzalloc_objs(struct hisi_qp, qm->qp_num); + if (!qm->qp_array) + return -ENOMEM; + + qm->poll_data = kzalloc_objs(struct hisi_qm_poll_data, qm->qp_num); + if (!qm->poll_data) { + kfree(qm->qp_array); + return -ENOMEM; + } + + qm_get_xqc_depth(qm, &sq_depth, &cq_depth, QM_QP_DEPTH_CAP); + + /* one more page for device or qp statuses */ + qp_dma_size = qm->sqe_size * sq_depth + sizeof(struct qm_cqe) * cq_depth; + qp_dma_size = PAGE_ALIGN(qp_dma_size) + PAGE_SIZE; + for (i = 0; i < qm->qp_num; i++) { + qm->poll_data[i].qm = qm; + ret = hisi_qp_memory_init(qm, qp_dma_size, i, sq_depth, cq_depth); + if (ret) + goto err_init_qp_mem; + + dev_dbg(dev, "allocate qp dma buf size=%zx)\n", qp_dma_size); + } + + return 0; +err_init_qp_mem: + hisi_qp_memory_uninit(qm, i); + + return ret; +} + +static int hisi_qm_alloc_rsv_buf(struct hisi_qm *qm) +{ + struct qm_rsv_buf *xqc_buf = &qm->xqc_buf; + struct qm_dma *xqc_dma = &xqc_buf->qcdma; + struct device *dev = &qm->pdev->dev; + size_t off = 0; + +#define QM_XQC_BUF_INIT(xqc_buf, type) do { \ + (xqc_buf)->type = ((xqc_buf)->qcdma.va + (off)); \ + (xqc_buf)->type##_dma = (xqc_buf)->qcdma.dma + (off); \ + off += QMC_ALIGN(sizeof(struct qm_##type)); \ +} while (0) + + xqc_dma->size = QMC_ALIGN(sizeof(struct qm_eqc)) + + QMC_ALIGN(sizeof(struct qm_aeqc)) + + QMC_ALIGN(sizeof(struct qm_sqc)) + + QMC_ALIGN(sizeof(struct qm_cqc)); + xqc_dma->va = dma_alloc_coherent(dev, xqc_dma->size, + &xqc_dma->dma, GFP_KERNEL); + if (!xqc_dma->va) + return -ENOMEM; + + QM_XQC_BUF_INIT(xqc_buf, eqc); + QM_XQC_BUF_INIT(xqc_buf, aeqc); + QM_XQC_BUF_INIT(xqc_buf, sqc); + QM_XQC_BUF_INIT(xqc_buf, cqc); + + return 0; +} + +static int hisi_qm_memory_init(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + int ret, total_func; + size_t off = 0; + + if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) { + total_func = pci_sriov_get_totalvfs(qm->pdev) + 1; + qm->factor = kzalloc_objs(struct qm_shaper_factor, total_func); + if (!qm->factor) + return -ENOMEM; + + /* Only the PF value needs to be initialized */ + qm->factor[0].func_qos = QM_QOS_MAX_VAL; + } + +#define QM_INIT_BUF(qm, type, num) do { \ + (qm)->type = ((qm)->qdma.va + (off)); \ + (qm)->type##_dma = (qm)->qdma.dma + (off); \ + off += QMC_ALIGN(sizeof(struct qm_##type) * (num)); \ +} while (0) + + idr_init(&qm->qp_idr); + qm_get_xqc_depth(qm, &qm->eq_depth, &qm->aeq_depth, QM_XEQ_DEPTH_CAP); + qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * qm->eq_depth) + + QMC_ALIGN(sizeof(struct qm_aeqe) * qm->aeq_depth) + + QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) + + QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num); + qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size, &qm->qdma.dma, + GFP_ATOMIC); + dev_dbg(dev, "allocate qm dma buf size=%zx)\n", qm->qdma.size); + if (!qm->qdma.va) { + ret = -ENOMEM; + goto err_destroy_idr; + } + + QM_INIT_BUF(qm, eqe, qm->eq_depth); + QM_INIT_BUF(qm, aeqe, qm->aeq_depth); + QM_INIT_BUF(qm, sqc, qm->qp_num); + QM_INIT_BUF(qm, cqc, qm->qp_num); + + ret = hisi_qm_alloc_rsv_buf(qm); + if (ret) + goto err_free_qdma; + + ret = hisi_qp_alloc_memory(qm); + if (ret) + goto err_free_reserve_buf; + + return 0; + +err_free_reserve_buf: + hisi_qm_free_rsv_buf(qm); +err_free_qdma: + dma_free_coherent(dev, qm->qdma.size, qm->qdma.va, qm->qdma.dma); +err_destroy_idr: + idr_destroy(&qm->qp_idr); + if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) + kfree(qm->factor); + + return ret; +} + +/** + * hisi_qm_init() - Initialize configures about qm. + * @qm: The qm needing init. + * + * This function init qm, then we can call hisi_qm_start to put qm into work. + */ +int hisi_qm_init(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + struct device *dev = &pdev->dev; + int ret; + + ret = hisi_qm_pci_init(qm); + if (ret) + return ret; + + ret = qm_irqs_register(qm); + if (ret) + goto err_pci_init; + + if (qm->fun_type == QM_HW_PF) { + /* Set the doorbell timeout to QM_DB_TIMEOUT_CFG ns. */ + writel(QM_DB_TIMEOUT_SET, qm->io_base + QM_DB_TIMEOUT_CFG); + qm_disable_clock_gate(qm); + ret = qm_dev_mem_reset(qm); + if (ret) { + dev_err(dev, "failed to reset device memory\n"); + goto err_irq_register; + } + } + + if (qm->mode == UACCE_MODE_SVA) { + ret = qm_alloc_uacce(qm); + if (ret < 0) + dev_warn(dev, "fail to alloc uacce (%d)\n", ret); + } + + ret = hisi_qm_memory_init(qm); + if (ret) + goto err_alloc_uacce; + + ret = hisi_qm_init_work(qm); + if (ret) + goto err_free_qm_memory; + + qm_cmd_init(qm); + hisi_mig_region_enable(qm); + + return 0; + +err_free_qm_memory: + hisi_qm_memory_uninit(qm); +err_alloc_uacce: + qm_remove_uacce(qm); +err_irq_register: + qm_irqs_unregister(qm); +err_pci_init: + hisi_qm_pci_uninit(qm); + return ret; +} +EXPORT_SYMBOL_GPL(hisi_qm_init); + +/** + * hisi_qm_get_dfx_access() - Try to get dfx access. + * @qm: pointer to accelerator device. + * + * Try to get dfx access, then user can get message. + * + * If device is in suspended, return failure, otherwise + * bump up the runtime PM usage counter. + */ +int hisi_qm_get_dfx_access(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + + if (pm_runtime_suspended(dev)) { + dev_info(dev, "can not read/write - device in suspended.\n"); + return -EAGAIN; + } + + return qm_pm_get_sync(qm); +} +EXPORT_SYMBOL_GPL(hisi_qm_get_dfx_access); + +/** + * hisi_qm_put_dfx_access() - Put dfx access. + * @qm: pointer to accelerator device. + * + * Put dfx access, drop runtime PM usage counter. + */ +void hisi_qm_put_dfx_access(struct hisi_qm *qm) +{ + qm_pm_put_sync(qm); +} +EXPORT_SYMBOL_GPL(hisi_qm_put_dfx_access); + +/** + * hisi_qm_pm_init() - Initialize qm runtime PM. + * @qm: pointer to accelerator device. + * + * Function that initialize qm runtime PM. + */ +void hisi_qm_pm_init(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + + if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) + return; + + pm_runtime_set_autosuspend_delay(dev, QM_AUTOSUSPEND_DELAY); + pm_runtime_use_autosuspend(dev); + pm_runtime_put_noidle(dev); +} +EXPORT_SYMBOL_GPL(hisi_qm_pm_init); + +/** + * hisi_qm_pm_uninit() - Uninitialize qm runtime PM. + * @qm: pointer to accelerator device. + * + * Function that uninitialize qm runtime PM. + */ +void hisi_qm_pm_uninit(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + + if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) + return; + + pm_runtime_get_noresume(dev); + pm_runtime_dont_use_autosuspend(dev); +} +EXPORT_SYMBOL_GPL(hisi_qm_pm_uninit); + +static int qm_prepare_for_suspend(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + int ret; + + ret = qm->ops->set_msi(qm, false); + if (ret) { + pci_err(pdev, "failed to disable MSI before suspending!\n"); + return ret; + } + + ret = qm_master_ooo_check(qm); + if (ret) + return ret; + + if (qm->err_ini->set_priv_status) { + ret = qm->err_ini->set_priv_status(qm); + if (ret) + return ret; + } + + ret = qm_set_pf_mse(qm, false); + if (ret) + pci_err(pdev, "failed to disable MSE before suspending!\n"); + + return ret; +} + +static int qm_rebuild_for_resume(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + int ret; + + ret = qm_set_pf_mse(qm, true); + if (ret) { + pci_err(pdev, "failed to enable MSE after resuming!\n"); + return ret; + } + + ret = qm->ops->set_msi(qm, true); + if (ret) { + pci_err(pdev, "failed to enable MSI after resuming!\n"); + return ret; + } + + ret = qm_dev_hw_init(qm); + if (ret) { + pci_err(pdev, "failed to init device after resuming\n"); + return ret; + } + + qm_cmd_init(qm); + hisi_mig_region_enable(qm); + hisi_qm_dev_err_init(qm); + /* Set the doorbell timeout to QM_DB_TIMEOUT_CFG ns. */ + writel(QM_DB_TIMEOUT_SET, qm->io_base + QM_DB_TIMEOUT_CFG); + qm_disable_clock_gate(qm); + ret = qm_dev_mem_reset(qm); + if (ret) + pci_err(pdev, "failed to reset device memory\n"); + + return ret; +} + +/** + * hisi_qm_suspend() - Runtime suspend of given device. + * @dev: device to suspend. + * + * Function that suspend the device. + */ +int hisi_qm_suspend(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct hisi_qm *qm = pci_get_drvdata(pdev); + int ret; + + pci_info(pdev, "entering suspended state\n"); + + ret = hisi_qm_stop(qm, QM_NORMAL); + if (ret) { + pci_err(pdev, "failed to stop qm(%d)\n", ret); + return ret; + } + + ret = qm_prepare_for_suspend(qm); + if (ret) + pci_err(pdev, "failed to prepare suspended(%d)\n", ret); + + return ret; +} +EXPORT_SYMBOL_GPL(hisi_qm_suspend); + +/** + * hisi_qm_resume() - Runtime resume of given device. + * @dev: device to resume. + * + * Function that resume the device. + */ +int hisi_qm_resume(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct hisi_qm *qm = pci_get_drvdata(pdev); + int ret; + + pci_info(pdev, "resuming from suspend state\n"); + + ret = qm_rebuild_for_resume(qm); + if (ret) { + pci_err(pdev, "failed to rebuild resume(%d)\n", ret); + return ret; + } + + ret = hisi_qm_start(qm); + if (ret) { + if (qm_check_dev_error(qm)) { + pci_info(pdev, "failed to start qm due to device error, device will be reset!\n"); + return 0; + } + + pci_err(pdev, "failed to start qm(%d)!\n", ret); + } + + return ret; +} +EXPORT_SYMBOL_GPL(hisi_qm_resume); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Zhou Wang "); +MODULE_DESCRIPTION("HiSilicon Accelerator queue manager driver"); diff --git a/drivers/crypto/hisilicon/qm_common.h b/drivers/crypto/hisilicon/qm_common.h new file mode 100644 index 000000000..0760bf55f --- /dev/null +++ b/drivers/crypto/hisilicon/qm_common.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2022 HiSilicon Limited. */ +#ifndef QM_COMMON_H +#define QM_COMMON_H + +#define QM_DBG_READ_LEN 256 + +struct qm_cqe { + __le32 rsvd0; + __le16 cmd_id; + __le16 rsvd1; + __le16 sq_head; + __le16 sq_num; + __le16 rsvd2; + __le16 w7; +}; + +struct qm_eqe { + __le32 dw0; +}; + +struct qm_aeqe { + __le32 dw0; +}; + +struct qm_sqc { + __le16 head; + __le16 tail; + __le32 base_l; + __le32 base_h; + __le32 dw3; + __le16 w8; + __le16 rsvd0; + __le16 pasid; + __le16 w11; + __le16 cq_num; + __le16 w13; + __le32 rsvd1; +}; + +struct qm_cqc { + __le16 head; + __le16 tail; + __le32 base_l; + __le32 base_h; + __le32 dw3; + __le16 w8; + __le16 rsvd0; + __le16 pasid; + __le16 w11; + __le32 dw6; + __le32 rsvd1; +}; + +struct qm_eqc { + __le16 head; + __le16 tail; + __le32 base_l; + __le32 base_h; + __le32 dw3; + __le32 rsvd[2]; + __le32 dw6; +}; + +struct qm_aeqc { + __le16 head; + __le16 tail; + __le32 base_l; + __le32 base_h; + __le32 dw3; + __le32 rsvd[2]; + __le32 dw6; +}; + +int qm_set_and_get_xqc(struct hisi_qm *qm, u8 cmd, void *xqc, u32 qp_id, bool op); +void hisi_qm_show_last_dfx_regs(struct hisi_qm *qm); +void hisi_qm_set_algqos_init(struct hisi_qm *qm); + +#endif diff --git a/drivers/crypto/hisilicon/sec/Makefile b/drivers/crypto/hisilicon/sec/Makefile new file mode 100644 index 000000000..a55b698e0 --- /dev/null +++ b/drivers/crypto/hisilicon/sec/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_CRYPTO_DEV_HISI_SEC) += hisi_sec.o +hisi_sec-y = sec_algs.o sec_drv.o diff --git a/drivers/crypto/hisilicon/sec/sec_algs.c b/drivers/crypto/hisilicon/sec/sec_algs.c new file mode 100644 index 000000000..85eecbb40 --- /dev/null +++ b/drivers/crypto/hisilicon/sec/sec_algs.c @@ -0,0 +1,1122 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2016-2017 HiSilicon Limited. */ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "sec_drv.h" + +#define SEC_MAX_CIPHER_KEY 64 +#define SEC_REQ_LIMIT SZ_32M + +struct sec_c_alg_cfg { + unsigned c_alg : 3; + unsigned c_mode : 3; + unsigned key_len : 2; + unsigned c_width : 2; +}; + +static const struct sec_c_alg_cfg sec_c_alg_cfgs[] = { + [SEC_C_DES_ECB_64] = { + .c_alg = SEC_C_ALG_DES, + .c_mode = SEC_C_MODE_ECB, + .key_len = SEC_KEY_LEN_DES, + }, + [SEC_C_DES_CBC_64] = { + .c_alg = SEC_C_ALG_DES, + .c_mode = SEC_C_MODE_CBC, + .key_len = SEC_KEY_LEN_DES, + }, + [SEC_C_3DES_ECB_192_3KEY] = { + .c_alg = SEC_C_ALG_3DES, + .c_mode = SEC_C_MODE_ECB, + .key_len = SEC_KEY_LEN_3DES_3_KEY, + }, + [SEC_C_3DES_ECB_192_2KEY] = { + .c_alg = SEC_C_ALG_3DES, + .c_mode = SEC_C_MODE_ECB, + .key_len = SEC_KEY_LEN_3DES_2_KEY, + }, + [SEC_C_3DES_CBC_192_3KEY] = { + .c_alg = SEC_C_ALG_3DES, + .c_mode = SEC_C_MODE_CBC, + .key_len = SEC_KEY_LEN_3DES_3_KEY, + }, + [SEC_C_3DES_CBC_192_2KEY] = { + .c_alg = SEC_C_ALG_3DES, + .c_mode = SEC_C_MODE_CBC, + .key_len = SEC_KEY_LEN_3DES_2_KEY, + }, + [SEC_C_AES_ECB_128] = { + .c_alg = SEC_C_ALG_AES, + .c_mode = SEC_C_MODE_ECB, + .key_len = SEC_KEY_LEN_AES_128, + }, + [SEC_C_AES_ECB_192] = { + .c_alg = SEC_C_ALG_AES, + .c_mode = SEC_C_MODE_ECB, + .key_len = SEC_KEY_LEN_AES_192, + }, + [SEC_C_AES_ECB_256] = { + .c_alg = SEC_C_ALG_AES, + .c_mode = SEC_C_MODE_ECB, + .key_len = SEC_KEY_LEN_AES_256, + }, + [SEC_C_AES_CBC_128] = { + .c_alg = SEC_C_ALG_AES, + .c_mode = SEC_C_MODE_CBC, + .key_len = SEC_KEY_LEN_AES_128, + }, + [SEC_C_AES_CBC_192] = { + .c_alg = SEC_C_ALG_AES, + .c_mode = SEC_C_MODE_CBC, + .key_len = SEC_KEY_LEN_AES_192, + }, + [SEC_C_AES_CBC_256] = { + .c_alg = SEC_C_ALG_AES, + .c_mode = SEC_C_MODE_CBC, + .key_len = SEC_KEY_LEN_AES_256, + }, + [SEC_C_AES_CTR_128] = { + .c_alg = SEC_C_ALG_AES, + .c_mode = SEC_C_MODE_CTR, + .key_len = SEC_KEY_LEN_AES_128, + }, + [SEC_C_AES_CTR_192] = { + .c_alg = SEC_C_ALG_AES, + .c_mode = SEC_C_MODE_CTR, + .key_len = SEC_KEY_LEN_AES_192, + }, + [SEC_C_AES_CTR_256] = { + .c_alg = SEC_C_ALG_AES, + .c_mode = SEC_C_MODE_CTR, + .key_len = SEC_KEY_LEN_AES_256, + }, + [SEC_C_AES_XTS_128] = { + .c_alg = SEC_C_ALG_AES, + .c_mode = SEC_C_MODE_XTS, + .key_len = SEC_KEY_LEN_AES_128, + }, + [SEC_C_AES_XTS_256] = { + .c_alg = SEC_C_ALG_AES, + .c_mode = SEC_C_MODE_XTS, + .key_len = SEC_KEY_LEN_AES_256, + }, + [SEC_C_NULL] = { + }, +}; + +/* + * Mutex used to ensure safe operation of reference count of + * alg providers + */ +static DEFINE_MUTEX(algs_lock); +static unsigned int active_devs; + +static void sec_alg_skcipher_init_template(struct sec_alg_tfm_ctx *ctx, + struct sec_bd_info *req, + enum sec_cipher_alg alg) +{ + const struct sec_c_alg_cfg *cfg = &sec_c_alg_cfgs[alg]; + + memset(req, 0, sizeof(*req)); + req->w0 |= cfg->c_mode << SEC_BD_W0_C_MODE_S; + req->w1 |= cfg->c_alg << SEC_BD_W1_C_ALG_S; + req->w3 |= cfg->key_len << SEC_BD_W3_C_KEY_LEN_S; + req->w0 |= cfg->c_width << SEC_BD_W0_C_WIDTH_S; + + req->cipher_key_addr_lo = lower_32_bits(ctx->pkey); + req->cipher_key_addr_hi = upper_32_bits(ctx->pkey); +} + +static void sec_alg_skcipher_init_context(struct crypto_skcipher *atfm, + const u8 *key, + unsigned int keylen, + enum sec_cipher_alg alg) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(atfm); + struct sec_alg_tfm_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->cipher_alg = alg; + memcpy(ctx->key, key, keylen); + sec_alg_skcipher_init_template(ctx, &ctx->req_template, + ctx->cipher_alg); +} + +static void sec_free_hw_sgl(struct sec_hw_sgl *hw_sgl, + dma_addr_t psec_sgl, struct sec_dev_info *info) +{ + struct sec_hw_sgl *sgl_current, *sgl_next; + dma_addr_t sgl_next_dma; + + sgl_current = hw_sgl; + while (sgl_current) { + sgl_next = sgl_current->next; + sgl_next_dma = sgl_current->next_sgl; + + dma_pool_free(info->hw_sgl_pool, sgl_current, psec_sgl); + + sgl_current = sgl_next; + psec_sgl = sgl_next_dma; + } +} + +static int sec_alloc_and_fill_hw_sgl(struct sec_hw_sgl **sec_sgl, + dma_addr_t *psec_sgl, + struct scatterlist *sgl, + int count, + struct sec_dev_info *info, + gfp_t gfp) +{ + struct sec_hw_sgl *sgl_current = NULL; + struct sec_hw_sgl *sgl_next; + dma_addr_t sgl_next_dma; + struct scatterlist *sg; + int ret, sge_index, i; + + if (!count) + return -EINVAL; + + for_each_sg(sgl, sg, count, i) { + sge_index = i % SEC_MAX_SGE_NUM; + if (sge_index == 0) { + sgl_next = dma_pool_zalloc(info->hw_sgl_pool, + gfp, &sgl_next_dma); + if (!sgl_next) { + ret = -ENOMEM; + goto err_free_hw_sgls; + } + + if (!sgl_current) { /* First one */ + *psec_sgl = sgl_next_dma; + *sec_sgl = sgl_next; + } else { /* Chained */ + sgl_current->entry_sum_in_sgl = SEC_MAX_SGE_NUM; + sgl_current->next_sgl = sgl_next_dma; + sgl_current->next = sgl_next; + } + sgl_current = sgl_next; + } + sgl_current->sge_entries[sge_index].buf = sg_dma_address(sg); + sgl_current->sge_entries[sge_index].len = sg_dma_len(sg); + sgl_current->data_bytes_in_sgl += sg_dma_len(sg); + } + sgl_current->entry_sum_in_sgl = count % SEC_MAX_SGE_NUM; + sgl_current->next_sgl = 0; + (*sec_sgl)->entry_sum_in_chain = count; + + return 0; + +err_free_hw_sgls: + sec_free_hw_sgl(*sec_sgl, *psec_sgl, info); + *psec_sgl = 0; + + return ret; +} + +static int sec_alg_skcipher_setkey(struct crypto_skcipher *tfm, + const u8 *key, unsigned int keylen, + enum sec_cipher_alg alg) +{ + struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + struct device *dev = ctx->queue->dev_info->dev; + + mutex_lock(&ctx->lock); + if (ctx->key) { + /* rekeying */ + memset(ctx->key, 0, SEC_MAX_CIPHER_KEY); + } else { + /* new key */ + ctx->key = dma_alloc_coherent(dev, SEC_MAX_CIPHER_KEY, + &ctx->pkey, GFP_KERNEL); + if (!ctx->key) { + mutex_unlock(&ctx->lock); + return -ENOMEM; + } + } + mutex_unlock(&ctx->lock); + sec_alg_skcipher_init_context(tfm, key, keylen, alg); + + return 0; +} + +static int sec_alg_skcipher_setkey_aes_ecb(struct crypto_skcipher *tfm, + const u8 *key, unsigned int keylen) +{ + enum sec_cipher_alg alg; + + switch (keylen) { + case AES_KEYSIZE_128: + alg = SEC_C_AES_ECB_128; + break; + case AES_KEYSIZE_192: + alg = SEC_C_AES_ECB_192; + break; + case AES_KEYSIZE_256: + alg = SEC_C_AES_ECB_256; + break; + default: + return -EINVAL; + } + + return sec_alg_skcipher_setkey(tfm, key, keylen, alg); +} + +static int sec_alg_skcipher_setkey_aes_cbc(struct crypto_skcipher *tfm, + const u8 *key, unsigned int keylen) +{ + enum sec_cipher_alg alg; + + switch (keylen) { + case AES_KEYSIZE_128: + alg = SEC_C_AES_CBC_128; + break; + case AES_KEYSIZE_192: + alg = SEC_C_AES_CBC_192; + break; + case AES_KEYSIZE_256: + alg = SEC_C_AES_CBC_256; + break; + default: + return -EINVAL; + } + + return sec_alg_skcipher_setkey(tfm, key, keylen, alg); +} + +static int sec_alg_skcipher_setkey_aes_ctr(struct crypto_skcipher *tfm, + const u8 *key, unsigned int keylen) +{ + enum sec_cipher_alg alg; + + switch (keylen) { + case AES_KEYSIZE_128: + alg = SEC_C_AES_CTR_128; + break; + case AES_KEYSIZE_192: + alg = SEC_C_AES_CTR_192; + break; + case AES_KEYSIZE_256: + alg = SEC_C_AES_CTR_256; + break; + default: + return -EINVAL; + } + + return sec_alg_skcipher_setkey(tfm, key, keylen, alg); +} + +static int sec_alg_skcipher_setkey_aes_xts(struct crypto_skcipher *tfm, + const u8 *key, unsigned int keylen) +{ + enum sec_cipher_alg alg; + int ret; + + ret = xts_verify_key(tfm, key, keylen); + if (ret) + return ret; + + switch (keylen) { + case AES_KEYSIZE_128 * 2: + alg = SEC_C_AES_XTS_128; + break; + case AES_KEYSIZE_256 * 2: + alg = SEC_C_AES_XTS_256; + break; + default: + return -EINVAL; + } + + return sec_alg_skcipher_setkey(tfm, key, keylen, alg); +} + +static int sec_alg_skcipher_setkey_des_ecb(struct crypto_skcipher *tfm, + const u8 *key, unsigned int keylen) +{ + return verify_skcipher_des_key(tfm, key) ?: + sec_alg_skcipher_setkey(tfm, key, keylen, SEC_C_DES_ECB_64); +} + +static int sec_alg_skcipher_setkey_des_cbc(struct crypto_skcipher *tfm, + const u8 *key, unsigned int keylen) +{ + return verify_skcipher_des_key(tfm, key) ?: + sec_alg_skcipher_setkey(tfm, key, keylen, SEC_C_DES_CBC_64); +} + +static int sec_alg_skcipher_setkey_3des_ecb(struct crypto_skcipher *tfm, + const u8 *key, unsigned int keylen) +{ + return verify_skcipher_des3_key(tfm, key) ?: + sec_alg_skcipher_setkey(tfm, key, keylen, + SEC_C_3DES_ECB_192_3KEY); +} + +static int sec_alg_skcipher_setkey_3des_cbc(struct crypto_skcipher *tfm, + const u8 *key, unsigned int keylen) +{ + return verify_skcipher_des3_key(tfm, key) ?: + sec_alg_skcipher_setkey(tfm, key, keylen, + SEC_C_3DES_CBC_192_3KEY); +} + +static void sec_alg_free_el(struct sec_request_el *el, + struct sec_dev_info *info) +{ + sec_free_hw_sgl(el->out, el->dma_out, info); + sec_free_hw_sgl(el->in, el->dma_in, info); + kfree(el->sgl_in); + kfree(el->sgl_out); + kfree(el); +} + +/* queuelock must be held */ +static int sec_send_request(struct sec_request *sec_req, struct sec_queue *queue) +{ + struct sec_request_el *el, *temp; + int ret = 0; + + mutex_lock(&sec_req->lock); + list_for_each_entry_safe(el, temp, &sec_req->elements, head) { + /* + * Add to hardware queue only under following circumstances + * 1) Software and hardware queue empty so no chain dependencies + * 2) No dependencies as new IV - (check software queue empty + * to maintain order) + * 3) No dependencies because the mode does no chaining. + * + * In other cases first insert onto the software queue which + * is then emptied as requests complete + */ + if (!queue->havesoftqueue || + (kfifo_is_empty(&queue->softqueue) && + sec_queue_empty(queue))) { + ret = sec_queue_send(queue, &el->req, sec_req); + if (ret == -EAGAIN) { + /* Wait unti we can send then try again */ + /* DEAD if here - should not happen */ + ret = -EBUSY; + goto err_unlock; + } + } else { + kfifo_put(&queue->softqueue, el); + } + } +err_unlock: + mutex_unlock(&sec_req->lock); + + return ret; +} + +static void sec_skcipher_alg_callback(struct sec_bd_info *sec_resp, + struct crypto_async_request *req_base) +{ + struct skcipher_request *skreq = container_of(req_base, + struct skcipher_request, + base); + struct sec_request *sec_req = skcipher_request_ctx(skreq); + struct sec_request *backlog_req; + struct sec_request_el *sec_req_el, *nextrequest; + struct sec_alg_tfm_ctx *ctx = sec_req->tfm_ctx; + struct crypto_skcipher *atfm = crypto_skcipher_reqtfm(skreq); + struct device *dev = ctx->queue->dev_info->dev; + int icv_or_skey_en, ret; + bool done; + + sec_req_el = list_first_entry(&sec_req->elements, struct sec_request_el, + head); + icv_or_skey_en = (sec_resp->w0 & SEC_BD_W0_ICV_OR_SKEY_EN_M) >> + SEC_BD_W0_ICV_OR_SKEY_EN_S; + if (sec_resp->w1 & SEC_BD_W1_BD_INVALID || icv_or_skey_en == 3) { + dev_err(dev, "Got an invalid answer %lu %d\n", + sec_resp->w1 & SEC_BD_W1_BD_INVALID, + icv_or_skey_en); + sec_req->err = -EINVAL; + /* + * We need to muddle on to avoid getting stuck with elements + * on the queue. Error will be reported so requester so + * it should be able to handle appropriately. + */ + } + + spin_lock_bh(&ctx->queue->queuelock); + /* Put the IV in place for chained cases */ + switch (ctx->cipher_alg) { + case SEC_C_AES_CBC_128: + case SEC_C_AES_CBC_192: + case SEC_C_AES_CBC_256: + if (sec_req_el->req.w0 & SEC_BD_W0_DE) + sg_pcopy_to_buffer(sec_req_el->sgl_out, + sg_nents(sec_req_el->sgl_out), + skreq->iv, + crypto_skcipher_ivsize(atfm), + sec_req_el->el_length - + crypto_skcipher_ivsize(atfm)); + else + sg_pcopy_to_buffer(sec_req_el->sgl_in, + sg_nents(sec_req_el->sgl_in), + skreq->iv, + crypto_skcipher_ivsize(atfm), + sec_req_el->el_length - + crypto_skcipher_ivsize(atfm)); + /* No need to sync to the device as coherent DMA */ + break; + case SEC_C_AES_CTR_128: + case SEC_C_AES_CTR_192: + case SEC_C_AES_CTR_256: + crypto_inc(skreq->iv, 16); + break; + default: + /* Do not update */ + break; + } + + if (ctx->queue->havesoftqueue && + !kfifo_is_empty(&ctx->queue->softqueue) && + sec_queue_empty(ctx->queue)) { + ret = kfifo_get(&ctx->queue->softqueue, &nextrequest); + if (ret <= 0) + dev_err(dev, + "Error getting next element from kfifo %d\n", + ret); + else + /* We know there is space so this cannot fail */ + sec_queue_send(ctx->queue, &nextrequest->req, + nextrequest->sec_req); + } else if (!list_empty(&ctx->backlog)) { + /* Need to verify there is room first */ + backlog_req = list_first_entry(&ctx->backlog, + typeof(*backlog_req), + backlog_head); + if (sec_queue_can_enqueue(ctx->queue, + backlog_req->num_elements) || + (ctx->queue->havesoftqueue && + kfifo_avail(&ctx->queue->softqueue) > + backlog_req->num_elements)) { + sec_send_request(backlog_req, ctx->queue); + crypto_request_complete(backlog_req->req_base, + -EINPROGRESS); + list_del(&backlog_req->backlog_head); + } + } + spin_unlock_bh(&ctx->queue->queuelock); + + mutex_lock(&sec_req->lock); + list_del(&sec_req_el->head); + mutex_unlock(&sec_req->lock); + sec_alg_free_el(sec_req_el, ctx->queue->dev_info); + + /* + * Request is done. + * The dance is needed as the lock is freed in the completion + */ + mutex_lock(&sec_req->lock); + done = list_empty(&sec_req->elements); + mutex_unlock(&sec_req->lock); + if (done) { + if (crypto_skcipher_ivsize(atfm)) { + dma_unmap_single(dev, sec_req->dma_iv, + crypto_skcipher_ivsize(atfm), + DMA_TO_DEVICE); + } + dma_unmap_sg(dev, skreq->src, sec_req->len_in, + DMA_BIDIRECTIONAL); + if (skreq->src != skreq->dst) + dma_unmap_sg(dev, skreq->dst, sec_req->len_out, + DMA_BIDIRECTIONAL); + skcipher_request_complete(skreq, sec_req->err); + } +} + +void sec_alg_callback(struct sec_bd_info *resp, void *shadow) +{ + struct sec_request *sec_req = shadow; + + sec_req->cb(resp, sec_req->req_base); +} + +static int sec_alg_alloc_and_calc_split_sizes(int length, size_t **split_sizes, + int *steps, gfp_t gfp) +{ + size_t *sizes; + int i; + + /* Split into suitable sized blocks */ + *steps = roundup(length, SEC_REQ_LIMIT) / SEC_REQ_LIMIT; + sizes = kzalloc_objs(*sizes, *steps, gfp); + if (!sizes) + return -ENOMEM; + + for (i = 0; i < *steps - 1; i++) + sizes[i] = SEC_REQ_LIMIT; + sizes[*steps - 1] = length - SEC_REQ_LIMIT * (*steps - 1); + *split_sizes = sizes; + + return 0; +} + +static int sec_map_and_split_sg(struct scatterlist *sgl, size_t *split_sizes, + int steps, struct scatterlist ***splits, + int **splits_nents, + int sgl_len_in, + struct device *dev, gfp_t gfp) +{ + int ret, count; + + count = dma_map_sg(dev, sgl, sgl_len_in, DMA_BIDIRECTIONAL); + if (!count) + return -EINVAL; + + *splits = kzalloc_objs(struct scatterlist *, steps, gfp); + if (!*splits) { + ret = -ENOMEM; + goto err_unmap_sg; + } + *splits_nents = kzalloc_objs(int, steps, gfp); + if (!*splits_nents) { + ret = -ENOMEM; + goto err_free_splits; + } + + /* output the scatter list before and after this */ + ret = sg_split(sgl, count, 0, steps, split_sizes, + *splits, *splits_nents, gfp); + if (ret) { + ret = -ENOMEM; + goto err_free_splits_nents; + } + + return 0; + +err_free_splits_nents: + kfree(*splits_nents); +err_free_splits: + kfree(*splits); +err_unmap_sg: + dma_unmap_sg(dev, sgl, sgl_len_in, DMA_BIDIRECTIONAL); + + return ret; +} + +/* + * Reverses the sec_map_and_split_sg call for messages not yet added to + * the queues. + */ +static void sec_unmap_sg_on_err(struct scatterlist *sgl, int steps, + struct scatterlist **splits, int *splits_nents, + int sgl_len_in, struct device *dev) +{ + int i; + + for (i = 0; i < steps; i++) + kfree(splits[i]); + kfree(splits_nents); + kfree(splits); + + dma_unmap_sg(dev, sgl, sgl_len_in, DMA_BIDIRECTIONAL); +} + +static struct sec_request_el +*sec_alg_alloc_and_fill_el(struct sec_bd_info *template, int encrypt, + int el_size, bool different_dest, + struct scatterlist *sgl_in, int n_ents_in, + struct scatterlist *sgl_out, int n_ents_out, + struct sec_dev_info *info, gfp_t gfp) +{ + struct sec_request_el *el; + struct sec_bd_info *req; + int ret; + + el = kzalloc_obj(*el, gfp); + if (!el) + return ERR_PTR(-ENOMEM); + el->el_length = el_size; + req = &el->req; + memcpy(req, template, sizeof(*req)); + + req->w0 &= ~SEC_BD_W0_CIPHER_M; + if (encrypt) + req->w0 |= SEC_CIPHER_ENCRYPT << SEC_BD_W0_CIPHER_S; + else + req->w0 |= SEC_CIPHER_DECRYPT << SEC_BD_W0_CIPHER_S; + + req->w0 &= ~SEC_BD_W0_C_GRAN_SIZE_19_16_M; + req->w0 |= ((el_size >> 16) << SEC_BD_W0_C_GRAN_SIZE_19_16_S) & + SEC_BD_W0_C_GRAN_SIZE_19_16_M; + + req->w0 &= ~SEC_BD_W0_C_GRAN_SIZE_21_20_M; + req->w0 |= ((el_size >> 20) << SEC_BD_W0_C_GRAN_SIZE_21_20_S) & + SEC_BD_W0_C_GRAN_SIZE_21_20_M; + + /* Writing whole u32 so no need to take care of masking */ + req->w2 = ((1 << SEC_BD_W2_GRAN_NUM_S) & SEC_BD_W2_GRAN_NUM_M) | + ((el_size << SEC_BD_W2_C_GRAN_SIZE_15_0_S) & + SEC_BD_W2_C_GRAN_SIZE_15_0_M); + + req->w3 &= ~SEC_BD_W3_CIPHER_LEN_OFFSET_M; + req->w1 |= SEC_BD_W1_ADDR_TYPE; + + el->sgl_in = sgl_in; + + ret = sec_alloc_and_fill_hw_sgl(&el->in, &el->dma_in, el->sgl_in, + n_ents_in, info, gfp); + if (ret) + goto err_free_el; + + req->data_addr_lo = lower_32_bits(el->dma_in); + req->data_addr_hi = upper_32_bits(el->dma_in); + + if (different_dest) { + el->sgl_out = sgl_out; + ret = sec_alloc_and_fill_hw_sgl(&el->out, &el->dma_out, + el->sgl_out, + n_ents_out, info, gfp); + if (ret) + goto err_free_hw_sgl_in; + + req->w0 |= SEC_BD_W0_DE; + req->cipher_destin_addr_lo = lower_32_bits(el->dma_out); + req->cipher_destin_addr_hi = upper_32_bits(el->dma_out); + + } else { + req->w0 &= ~SEC_BD_W0_DE; + req->cipher_destin_addr_lo = lower_32_bits(el->dma_in); + req->cipher_destin_addr_hi = upper_32_bits(el->dma_in); + } + + return el; + +err_free_hw_sgl_in: + sec_free_hw_sgl(el->in, el->dma_in, info); +err_free_el: + kfree(el); + + return ERR_PTR(ret); +} + +static int sec_alg_skcipher_crypto(struct skcipher_request *skreq, + bool encrypt) +{ + struct crypto_skcipher *atfm = crypto_skcipher_reqtfm(skreq); + struct crypto_tfm *tfm = crypto_skcipher_tfm(atfm); + struct sec_alg_tfm_ctx *ctx = crypto_tfm_ctx(tfm); + struct sec_queue *queue = ctx->queue; + struct sec_request *sec_req = skcipher_request_ctx(skreq); + struct sec_dev_info *info = queue->dev_info; + int i, ret, steps; + size_t *split_sizes; + struct scatterlist **splits_in; + struct scatterlist **splits_out = NULL; + int *splits_in_nents; + int *splits_out_nents = NULL; + struct sec_request_el *el, *temp; + bool split = skreq->src != skreq->dst; + gfp_t gfp = skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; + + mutex_init(&sec_req->lock); + sec_req->req_base = &skreq->base; + sec_req->err = 0; + /* SGL mapping out here to allow us to break it up as necessary */ + sec_req->len_in = sg_nents(skreq->src); + + ret = sec_alg_alloc_and_calc_split_sizes(skreq->cryptlen, &split_sizes, + &steps, gfp); + if (ret) + return ret; + sec_req->num_elements = steps; + ret = sec_map_and_split_sg(skreq->src, split_sizes, steps, &splits_in, + &splits_in_nents, sec_req->len_in, + info->dev, gfp); + if (ret) + goto err_free_split_sizes; + + if (split) { + sec_req->len_out = sg_nents(skreq->dst); + ret = sec_map_and_split_sg(skreq->dst, split_sizes, steps, + &splits_out, &splits_out_nents, + sec_req->len_out, info->dev, gfp); + if (ret) + goto err_unmap_in_sg; + } + /* Shared info stored in seq_req - applies to all BDs */ + sec_req->tfm_ctx = ctx; + sec_req->cb = sec_skcipher_alg_callback; + INIT_LIST_HEAD(&sec_req->elements); + + /* + * Future optimization. + * In the chaining case we can't use a dma pool bounce buffer + * but in the case where we know there is no chaining we can + */ + if (crypto_skcipher_ivsize(atfm)) { + sec_req->dma_iv = dma_map_single(info->dev, skreq->iv, + crypto_skcipher_ivsize(atfm), + DMA_TO_DEVICE); + if (dma_mapping_error(info->dev, sec_req->dma_iv)) { + ret = -ENOMEM; + goto err_unmap_out_sg; + } + } + + /* Set them all up then queue - cleaner error handling. */ + for (i = 0; i < steps; i++) { + el = sec_alg_alloc_and_fill_el(&ctx->req_template, + encrypt ? 1 : 0, + split_sizes[i], + skreq->src != skreq->dst, + splits_in[i], splits_in_nents[i], + split ? splits_out[i] : NULL, + split ? splits_out_nents[i] : 0, + info, gfp); + if (IS_ERR(el)) { + ret = PTR_ERR(el); + goto err_free_elements; + } + el->req.cipher_iv_addr_lo = lower_32_bits(sec_req->dma_iv); + el->req.cipher_iv_addr_hi = upper_32_bits(sec_req->dma_iv); + el->sec_req = sec_req; + list_add_tail(&el->head, &sec_req->elements); + } + + /* + * Only attempt to queue if the whole lot can fit in the queue - + * we can't successfully cleanup after a partial queing so this + * must succeed or fail atomically. + * + * Big hammer test of both software and hardware queues - could be + * more refined but this is unlikely to happen so no need. + */ + + /* Grab a big lock for a long time to avoid concurrency issues */ + spin_lock_bh(&queue->queuelock); + + /* + * Can go on to queue if we have space in either: + * 1) The hardware queue and no software queue + * 2) The software queue + * AND there is nothing in the backlog. If there is backlog we + * have to only queue to the backlog queue and return busy. + */ + if ((!sec_queue_can_enqueue(queue, steps) && + (!queue->havesoftqueue || + kfifo_avail(&queue->softqueue) > steps)) || + !list_empty(&ctx->backlog)) { + ret = -EBUSY; + if ((skreq->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { + list_add_tail(&sec_req->backlog_head, &ctx->backlog); + spin_unlock_bh(&queue->queuelock); + goto out; + } + + spin_unlock_bh(&queue->queuelock); + goto err_free_elements; + } + ret = sec_send_request(sec_req, queue); + spin_unlock_bh(&queue->queuelock); + if (ret) + goto err_free_elements; + + ret = -EINPROGRESS; +out: + /* Cleanup - all elements in pointer arrays have been copied */ + kfree(splits_in_nents); + kfree(splits_in); + kfree(splits_out_nents); + kfree(splits_out); + kfree(split_sizes); + return ret; + +err_free_elements: + list_for_each_entry_safe(el, temp, &sec_req->elements, head) { + list_del(&el->head); + sec_alg_free_el(el, info); + } + if (crypto_skcipher_ivsize(atfm)) + dma_unmap_single(info->dev, sec_req->dma_iv, + crypto_skcipher_ivsize(atfm), + DMA_TO_DEVICE); +err_unmap_out_sg: + if (split) + sec_unmap_sg_on_err(skreq->dst, steps, splits_out, + splits_out_nents, sec_req->len_out, + info->dev); +err_unmap_in_sg: + sec_unmap_sg_on_err(skreq->src, steps, splits_in, splits_in_nents, + sec_req->len_in, info->dev); +err_free_split_sizes: + kfree(split_sizes); + + return ret; +} + +static int sec_alg_skcipher_encrypt(struct skcipher_request *req) +{ + return sec_alg_skcipher_crypto(req, true); +} + +static int sec_alg_skcipher_decrypt(struct skcipher_request *req) +{ + return sec_alg_skcipher_crypto(req, false); +} + +static int sec_alg_skcipher_init(struct crypto_skcipher *tfm) +{ + struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + + mutex_init(&ctx->lock); + INIT_LIST_HEAD(&ctx->backlog); + crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_request)); + + ctx->queue = sec_queue_alloc_start_safe(); + if (IS_ERR(ctx->queue)) + return PTR_ERR(ctx->queue); + + spin_lock_init(&ctx->queue->queuelock); + ctx->queue->havesoftqueue = false; + + return 0; +} + +static void sec_alg_skcipher_exit(struct crypto_skcipher *tfm) +{ + struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + struct device *dev = ctx->queue->dev_info->dev; + + if (ctx->key) { + memzero_explicit(ctx->key, SEC_MAX_CIPHER_KEY); + dma_free_coherent(dev, SEC_MAX_CIPHER_KEY, ctx->key, + ctx->pkey); + } + sec_queue_stop_release(ctx->queue); +} + +static int sec_alg_skcipher_init_with_queue(struct crypto_skcipher *tfm) +{ + struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + int ret; + + ret = sec_alg_skcipher_init(tfm); + if (ret) + return ret; + + INIT_KFIFO(ctx->queue->softqueue); + ret = kfifo_alloc(&ctx->queue->softqueue, 512, GFP_KERNEL); + if (ret) { + sec_alg_skcipher_exit(tfm); + return ret; + } + ctx->queue->havesoftqueue = true; + + return 0; +} + +static void sec_alg_skcipher_exit_with_queue(struct crypto_skcipher *tfm) +{ + struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + + kfifo_free(&ctx->queue->softqueue); + sec_alg_skcipher_exit(tfm); +} + +static struct skcipher_alg sec_algs[] = { + { + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "hisi_sec_aes_ecb", + .cra_priority = 4001, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sec_alg_tfm_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = sec_alg_skcipher_init, + .exit = sec_alg_skcipher_exit, + .setkey = sec_alg_skcipher_setkey_aes_ecb, + .decrypt = sec_alg_skcipher_decrypt, + .encrypt = sec_alg_skcipher_encrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = 0, + }, { + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "hisi_sec_aes_cbc", + .cra_priority = 4001, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sec_alg_tfm_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = sec_alg_skcipher_init_with_queue, + .exit = sec_alg_skcipher_exit_with_queue, + .setkey = sec_alg_skcipher_setkey_aes_cbc, + .decrypt = sec_alg_skcipher_decrypt, + .encrypt = sec_alg_skcipher_encrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, { + .base = { + .cra_name = "ctr(aes)", + .cra_driver_name = "hisi_sec_aes_ctr", + .cra_priority = 4001, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sec_alg_tfm_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = sec_alg_skcipher_init_with_queue, + .exit = sec_alg_skcipher_exit_with_queue, + .setkey = sec_alg_skcipher_setkey_aes_ctr, + .decrypt = sec_alg_skcipher_decrypt, + .encrypt = sec_alg_skcipher_encrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, { + .base = { + .cra_name = "xts(aes)", + .cra_driver_name = "hisi_sec_aes_xts", + .cra_priority = 4001, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sec_alg_tfm_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = sec_alg_skcipher_init, + .exit = sec_alg_skcipher_exit, + .setkey = sec_alg_skcipher_setkey_aes_xts, + .decrypt = sec_alg_skcipher_decrypt, + .encrypt = sec_alg_skcipher_encrypt, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, { + /* Unable to find any test vectors so untested */ + .base = { + .cra_name = "ecb(des)", + .cra_driver_name = "hisi_sec_des_ecb", + .cra_priority = 4001, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sec_alg_tfm_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = sec_alg_skcipher_init, + .exit = sec_alg_skcipher_exit, + .setkey = sec_alg_skcipher_setkey_des_ecb, + .decrypt = sec_alg_skcipher_decrypt, + .encrypt = sec_alg_skcipher_encrypt, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = 0, + }, { + .base = { + .cra_name = "cbc(des)", + .cra_driver_name = "hisi_sec_des_cbc", + .cra_priority = 4001, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sec_alg_tfm_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = sec_alg_skcipher_init_with_queue, + .exit = sec_alg_skcipher_exit_with_queue, + .setkey = sec_alg_skcipher_setkey_des_cbc, + .decrypt = sec_alg_skcipher_decrypt, + .encrypt = sec_alg_skcipher_encrypt, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + }, { + .base = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "hisi_sec_3des_cbc", + .cra_priority = 4001, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sec_alg_tfm_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = sec_alg_skcipher_init_with_queue, + .exit = sec_alg_skcipher_exit_with_queue, + .setkey = sec_alg_skcipher_setkey_3des_cbc, + .decrypt = sec_alg_skcipher_decrypt, + .encrypt = sec_alg_skcipher_encrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + }, { + .base = { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "hisi_sec_3des_ecb", + .cra_priority = 4001, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sec_alg_tfm_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = sec_alg_skcipher_init, + .exit = sec_alg_skcipher_exit, + .setkey = sec_alg_skcipher_setkey_3des_ecb, + .decrypt = sec_alg_skcipher_decrypt, + .encrypt = sec_alg_skcipher_encrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = 0, + } +}; + +int sec_algs_register(void) +{ + int ret = 0; + + mutex_lock(&algs_lock); + if (++active_devs != 1) + goto unlock; + + ret = crypto_register_skciphers(sec_algs, ARRAY_SIZE(sec_algs)); + if (ret) + --active_devs; +unlock: + mutex_unlock(&algs_lock); + + return ret; +} + +void sec_algs_unregister(void) +{ + mutex_lock(&algs_lock); + if (--active_devs != 0) + goto unlock; + crypto_unregister_skciphers(sec_algs, ARRAY_SIZE(sec_algs)); + +unlock: + mutex_unlock(&algs_lock); +} diff --git a/drivers/crypto/hisilicon/sec/sec_drv.c b/drivers/crypto/hisilicon/sec/sec_drv.c new file mode 100644 index 000000000..2514a5e1f --- /dev/null +++ b/drivers/crypto/hisilicon/sec/sec_drv.c @@ -0,0 +1,1307 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Driver for the HiSilicon SEC units found on Hip06 Hip07 + * + * Copyright (c) 2016-2017 HiSilicon Limited. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sec_drv.h" + +#define SEC_QUEUE_AR_FROCE_ALLOC 0 +#define SEC_QUEUE_AR_FROCE_NOALLOC 1 +#define SEC_QUEUE_AR_FROCE_DIS 2 + +#define SEC_QUEUE_AW_FROCE_ALLOC 0 +#define SEC_QUEUE_AW_FROCE_NOALLOC 1 +#define SEC_QUEUE_AW_FROCE_DIS 2 + +/* SEC_ALGSUB registers */ +#define SEC_ALGSUB_CLK_EN_REG 0x03b8 +#define SEC_ALGSUB_CLK_DIS_REG 0x03bc +#define SEC_ALGSUB_CLK_ST_REG 0x535c +#define SEC_ALGSUB_RST_REQ_REG 0x0aa8 +#define SEC_ALGSUB_RST_DREQ_REG 0x0aac +#define SEC_ALGSUB_RST_ST_REG 0x5a54 +#define SEC_ALGSUB_RST_ST_IS_RST BIT(0) + +#define SEC_ALGSUB_BUILD_RST_REQ_REG 0x0ab8 +#define SEC_ALGSUB_BUILD_RST_DREQ_REG 0x0abc +#define SEC_ALGSUB_BUILD_RST_ST_REG 0x5a5c +#define SEC_ALGSUB_BUILD_RST_ST_IS_RST BIT(0) + +#define SEC_SAA_BASE 0x00001000UL + +/* SEC_SAA registers */ +#define SEC_SAA_CTRL_REG(x) ((x) * SEC_SAA_ADDR_SIZE) +#define SEC_SAA_CTRL_GET_QM_EN BIT(0) + +#define SEC_ST_INTMSK1_REG 0x0200 +#define SEC_ST_RINT1_REG 0x0400 +#define SEC_ST_INTSTS1_REG 0x0600 +#define SEC_BD_MNG_STAT_REG 0x0800 +#define SEC_PARSING_STAT_REG 0x0804 +#define SEC_LOAD_TIME_OUT_CNT_REG 0x0808 +#define SEC_CORE_WORK_TIME_OUT_CNT_REG 0x080c +#define SEC_BACK_TIME_OUT_CNT_REG 0x0810 +#define SEC_BD1_PARSING_RD_TIME_OUT_CNT_REG 0x0814 +#define SEC_BD1_PARSING_WR_TIME_OUT_CNT_REG 0x0818 +#define SEC_BD2_PARSING_RD_TIME_OUT_CNT_REG 0x081c +#define SEC_BD2_PARSING_WR_TIME_OUT_CNT_REG 0x0820 +#define SEC_SAA_ACC_REG 0x083c +#define SEC_BD_NUM_CNT_IN_SEC_REG 0x0858 +#define SEC_LOAD_WORK_TIME_CNT_REG 0x0860 +#define SEC_CORE_WORK_WORK_TIME_CNT_REG 0x0864 +#define SEC_BACK_WORK_TIME_CNT_REG 0x0868 +#define SEC_SAA_IDLE_TIME_CNT_REG 0x086c +#define SEC_SAA_CLK_CNT_REG 0x0870 + +/* SEC_COMMON registers */ +#define SEC_CLK_EN_REG 0x0000 +#define SEC_CTRL_REG 0x0004 + +#define SEC_COMMON_CNT_CLR_CE_REG 0x0008 +#define SEC_COMMON_CNT_CLR_CE_CLEAR BIT(0) +#define SEC_COMMON_CNT_CLR_CE_SNAP_EN BIT(1) + +#define SEC_SECURE_CTRL_REG 0x000c +#define SEC_AXI_CACHE_CFG_REG 0x0010 +#define SEC_AXI_QOS_CFG_REG 0x0014 +#define SEC_IPV4_MASK_TABLE_REG 0x0020 +#define SEC_IPV6_MASK_TABLE_X_REG(x) (0x0024 + (x) * 4) +#define SEC_FSM_MAX_CNT_REG 0x0064 + +#define SEC_CTRL2_REG 0x0068 +#define SEC_CTRL2_DATA_AXI_RD_OTSD_CFG_M GENMASK(3, 0) +#define SEC_CTRL2_DATA_AXI_RD_OTSD_CFG_S 0 +#define SEC_CTRL2_DATA_AXI_WR_OTSD_CFG_M GENMASK(6, 4) +#define SEC_CTRL2_DATA_AXI_WR_OTSD_CFG_S 4 +#define SEC_CTRL2_CLK_GATE_EN BIT(7) +#define SEC_CTRL2_ENDIAN_BD BIT(8) +#define SEC_CTRL2_ENDIAN_BD_TYPE BIT(9) + +#define SEC_CNT_PRECISION_CFG_REG 0x006c +#define SEC_DEBUG_BD_CFG_REG 0x0070 +#define SEC_DEBUG_BD_CFG_WB_NORMAL BIT(0) +#define SEC_DEBUG_BD_CFG_WB_EN BIT(1) + +#define SEC_Q_SIGHT_SEL 0x0074 +#define SEC_Q_SIGHT_HIS_CLR 0x0078 +#define SEC_Q_VMID_CFG_REG(q) (0x0100 + (q) * 4) +#define SEC_Q_WEIGHT_CFG_REG(q) (0x200 + (q) * 4) +#define SEC_STAT_CLR_REG 0x0a00 +#define SEC_SAA_IDLE_CNT_CLR_REG 0x0a04 +#define SEC_QM_CPL_Q_IDBUF_DFX_CFG_REG 0x0b00 +#define SEC_QM_CPL_Q_IDBUF_DFX_RESULT_REG 0x0b04 +#define SEC_QM_BD_DFX_CFG_REG 0x0b08 +#define SEC_QM_BD_DFX_RESULT_REG 0x0b0c +#define SEC_QM_BDID_DFX_RESULT_REG 0x0b10 +#define SEC_QM_BD_DFIFO_STATUS_REG 0x0b14 +#define SEC_QM_BD_DFX_CFG2_REG 0x0b1c +#define SEC_QM_BD_DFX_RESULT2_REG 0x0b20 +#define SEC_QM_BD_IDFIFO_STATUS_REG 0x0b18 +#define SEC_QM_BD_DFIFO_STATUS2_REG 0x0b28 +#define SEC_QM_BD_IDFIFO_STATUS2_REG 0x0b2c + +#define SEC_HASH_IPV4_MASK 0xfff00000 +#define SEC_MAX_SAA_NUM 0xa +#define SEC_SAA_ADDR_SIZE 0x1000 + +#define SEC_Q_INIT_REG 0x0 +#define SEC_Q_INIT_WO_STAT_CLEAR 0x2 +#define SEC_Q_INIT_AND_STAT_CLEAR 0x3 + +#define SEC_Q_CFG_REG 0x8 +#define SEC_Q_CFG_REORDER BIT(0) + +#define SEC_Q_PROC_NUM_CFG_REG 0x10 +#define SEC_QUEUE_ENB_REG 0x18 + +#define SEC_Q_DEPTH_CFG_REG 0x50 +#define SEC_Q_DEPTH_CFG_DEPTH_M GENMASK(11, 0) +#define SEC_Q_DEPTH_CFG_DEPTH_S 0 + +#define SEC_Q_BASE_HADDR_REG 0x54 +#define SEC_Q_BASE_LADDR_REG 0x58 +#define SEC_Q_WR_PTR_REG 0x5c +#define SEC_Q_OUTORDER_BASE_HADDR_REG 0x60 +#define SEC_Q_OUTORDER_BASE_LADDR_REG 0x64 +#define SEC_Q_OUTORDER_RD_PTR_REG 0x68 +#define SEC_Q_OT_TH_REG 0x6c + +#define SEC_Q_ARUSER_CFG_REG 0x70 +#define SEC_Q_ARUSER_CFG_FA BIT(0) +#define SEC_Q_ARUSER_CFG_FNA BIT(1) +#define SEC_Q_ARUSER_CFG_RINVLD BIT(2) +#define SEC_Q_ARUSER_CFG_PKG BIT(3) + +#define SEC_Q_AWUSER_CFG_REG 0x74 +#define SEC_Q_AWUSER_CFG_FA BIT(0) +#define SEC_Q_AWUSER_CFG_FNA BIT(1) +#define SEC_Q_AWUSER_CFG_PKG BIT(2) + +#define SEC_Q_ERR_BASE_HADDR_REG 0x7c +#define SEC_Q_ERR_BASE_LADDR_REG 0x80 +#define SEC_Q_CFG_VF_NUM_REG 0x84 +#define SEC_Q_SOFT_PROC_PTR_REG 0x88 +#define SEC_Q_FAIL_INT_MSK_REG 0x300 +#define SEC_Q_FLOW_INT_MKS_REG 0x304 +#define SEC_Q_FAIL_RINT_REG 0x400 +#define SEC_Q_FLOW_RINT_REG 0x404 +#define SEC_Q_FAIL_INT_STATUS_REG 0x500 +#define SEC_Q_FLOW_INT_STATUS_REG 0x504 +#define SEC_Q_STATUS_REG 0x600 +#define SEC_Q_RD_PTR_REG 0x604 +#define SEC_Q_PRO_PTR_REG 0x608 +#define SEC_Q_OUTORDER_WR_PTR_REG 0x60c +#define SEC_Q_OT_CNT_STATUS_REG 0x610 +#define SEC_Q_INORDER_BD_NUM_ST_REG 0x650 +#define SEC_Q_INORDER_GET_FLAG_ST_REG 0x654 +#define SEC_Q_INORDER_ADD_FLAG_ST_REG 0x658 +#define SEC_Q_INORDER_TASK_INT_NUM_LEFT_ST_REG 0x65c +#define SEC_Q_RD_DONE_PTR_REG 0x660 +#define SEC_Q_CPL_Q_BD_NUM_ST_REG 0x700 +#define SEC_Q_CPL_Q_PTR_ST_REG 0x704 +#define SEC_Q_CPL_Q_H_ADDR_ST_REG 0x708 +#define SEC_Q_CPL_Q_L_ADDR_ST_REG 0x70c +#define SEC_Q_CPL_TASK_INT_NUM_LEFT_ST_REG 0x710 +#define SEC_Q_WRR_ID_CHECK_REG 0x714 +#define SEC_Q_CPLQ_FULL_CHECK_REG 0x718 +#define SEC_Q_SUCCESS_BD_CNT_REG 0x800 +#define SEC_Q_FAIL_BD_CNT_REG 0x804 +#define SEC_Q_GET_BD_CNT_REG 0x808 +#define SEC_Q_IVLD_CNT_REG 0x80c +#define SEC_Q_BD_PROC_GET_CNT_REG 0x810 +#define SEC_Q_BD_PROC_DONE_CNT_REG 0x814 +#define SEC_Q_LAT_CLR_REG 0x850 +#define SEC_Q_PKT_LAT_MAX_REG 0x854 +#define SEC_Q_PKT_LAT_AVG_REG 0x858 +#define SEC_Q_PKT_LAT_MIN_REG 0x85c +#define SEC_Q_ID_CLR_CFG_REG 0x900 +#define SEC_Q_1ST_BD_ERR_ID_REG 0x904 +#define SEC_Q_1ST_AUTH_FAIL_ID_REG 0x908 +#define SEC_Q_1ST_RD_ERR_ID_REG 0x90c +#define SEC_Q_1ST_ECC2_ERR_ID_REG 0x910 +#define SEC_Q_1ST_IVLD_ID_REG 0x914 +#define SEC_Q_1ST_BD_WR_ERR_ID_REG 0x918 +#define SEC_Q_1ST_ERR_BD_WR_ERR_ID_REG 0x91c +#define SEC_Q_1ST_BD_MAC_WR_ERR_ID_REG 0x920 + +struct sec_debug_bd_info { +#define SEC_DEBUG_BD_INFO_SOFT_ERR_CHECK_M GENMASK(22, 0) + u32 soft_err_check; +#define SEC_DEBUG_BD_INFO_HARD_ERR_CHECK_M GENMASK(9, 0) + u32 hard_err_check; + u32 icv_mac1st_word; +#define SEC_DEBUG_BD_INFO_GET_ID_M GENMASK(19, 0) + u32 sec_get_id; + /* W4---W15 */ + u32 reserv_left[12]; +}; + +struct sec_out_bd_info { +#define SEC_OUT_BD_INFO_Q_ID_M GENMASK(11, 0) +#define SEC_OUT_BD_INFO_ECC_2BIT_ERR BIT(14) + u16 data; +}; + +#define SEC_MAX_DEVICES 8 +static struct sec_dev_info *sec_devices[SEC_MAX_DEVICES]; +static DEFINE_MUTEX(sec_id_lock); + +static int sec_queue_map_io(struct sec_queue *queue) +{ + struct device *dev = queue->dev_info->dev; + struct resource *res; + + res = platform_get_resource(to_platform_device(dev), + IORESOURCE_MEM, + 2 + queue->queue_id); + if (!res) { + dev_err(dev, "Failed to get queue %u memory resource\n", + queue->queue_id); + return -ENOMEM; + } + queue->regs = ioremap(res->start, resource_size(res)); + if (!queue->regs) + return -ENOMEM; + + return 0; +} + +static void sec_queue_unmap_io(struct sec_queue *queue) +{ + iounmap(queue->regs); +} + +static int sec_queue_ar_pkgattr(struct sec_queue *queue, u32 ar_pkg) +{ + void __iomem *addr = queue->regs + SEC_Q_ARUSER_CFG_REG; + u32 regval; + + regval = readl_relaxed(addr); + if (ar_pkg) + regval |= SEC_Q_ARUSER_CFG_PKG; + else + regval &= ~SEC_Q_ARUSER_CFG_PKG; + writel_relaxed(regval, addr); + + return 0; +} + +static int sec_queue_aw_pkgattr(struct sec_queue *queue, u32 aw_pkg) +{ + void __iomem *addr = queue->regs + SEC_Q_AWUSER_CFG_REG; + u32 regval; + + regval = readl_relaxed(addr); + regval |= SEC_Q_AWUSER_CFG_PKG; + writel_relaxed(regval, addr); + + return 0; +} + +static int sec_clk_en(struct sec_dev_info *info) +{ + void __iomem *base = info->regs[SEC_COMMON]; + u32 i = 0; + + writel_relaxed(0x7, base + SEC_ALGSUB_CLK_EN_REG); + do { + usleep_range(1000, 10000); + if ((readl_relaxed(base + SEC_ALGSUB_CLK_ST_REG) & 0x7) == 0x7) + return 0; + i++; + } while (i < 10); + dev_err(info->dev, "sec clock enable fail!\n"); + + return -EIO; +} + +static int sec_clk_dis(struct sec_dev_info *info) +{ + void __iomem *base = info->regs[SEC_COMMON]; + u32 i = 0; + + writel_relaxed(0x7, base + SEC_ALGSUB_CLK_DIS_REG); + do { + usleep_range(1000, 10000); + if ((readl_relaxed(base + SEC_ALGSUB_CLK_ST_REG) & 0x7) == 0) + return 0; + i++; + } while (i < 10); + dev_err(info->dev, "sec clock disable fail!\n"); + + return -EIO; +} + +static int sec_reset_whole_module(struct sec_dev_info *info) +{ + void __iomem *base = info->regs[SEC_COMMON]; + bool is_reset, b_is_reset; + u32 i = 0; + + writel_relaxed(1, base + SEC_ALGSUB_RST_REQ_REG); + writel_relaxed(1, base + SEC_ALGSUB_BUILD_RST_REQ_REG); + while (1) { + usleep_range(1000, 10000); + is_reset = readl_relaxed(base + SEC_ALGSUB_RST_ST_REG) & + SEC_ALGSUB_RST_ST_IS_RST; + b_is_reset = readl_relaxed(base + SEC_ALGSUB_BUILD_RST_ST_REG) & + SEC_ALGSUB_BUILD_RST_ST_IS_RST; + if (is_reset && b_is_reset) + break; + i++; + if (i > 10) { + dev_err(info->dev, "Reset req failed\n"); + return -EIO; + } + } + + i = 0; + writel_relaxed(1, base + SEC_ALGSUB_RST_DREQ_REG); + writel_relaxed(1, base + SEC_ALGSUB_BUILD_RST_DREQ_REG); + while (1) { + usleep_range(1000, 10000); + is_reset = readl_relaxed(base + SEC_ALGSUB_RST_ST_REG) & + SEC_ALGSUB_RST_ST_IS_RST; + b_is_reset = readl_relaxed(base + SEC_ALGSUB_BUILD_RST_ST_REG) & + SEC_ALGSUB_BUILD_RST_ST_IS_RST; + if (!is_reset && !b_is_reset) + break; + + i++; + if (i > 10) { + dev_err(info->dev, "Reset dreq failed\n"); + return -EIO; + } + } + + return 0; +} + +static void sec_bd_endian_little(struct sec_dev_info *info) +{ + void __iomem *addr = info->regs[SEC_SAA] + SEC_CTRL2_REG; + u32 regval; + + regval = readl_relaxed(addr); + regval &= ~(SEC_CTRL2_ENDIAN_BD | SEC_CTRL2_ENDIAN_BD_TYPE); + writel_relaxed(regval, addr); +} + +/* + * sec_cache_config - configure optimum cache placement + */ +static void sec_cache_config(struct sec_dev_info *info) +{ + struct iommu_domain *domain; + void __iomem *addr = info->regs[SEC_SAA] + SEC_CTRL_REG; + + domain = iommu_get_domain_for_dev(info->dev); + + /* Check that translation is occurring */ + if (domain && (domain->type & __IOMMU_DOMAIN_PAGING)) + writel_relaxed(0x44cf9e, addr); + else + writel_relaxed(0x4cfd9, addr); +} + +static void sec_data_axiwr_otsd_cfg(struct sec_dev_info *info, u32 cfg) +{ + void __iomem *addr = info->regs[SEC_SAA] + SEC_CTRL2_REG; + u32 regval; + + regval = readl_relaxed(addr); + regval &= ~SEC_CTRL2_DATA_AXI_WR_OTSD_CFG_M; + regval |= (cfg << SEC_CTRL2_DATA_AXI_WR_OTSD_CFG_S) & + SEC_CTRL2_DATA_AXI_WR_OTSD_CFG_M; + writel_relaxed(regval, addr); +} + +static void sec_data_axird_otsd_cfg(struct sec_dev_info *info, u32 cfg) +{ + void __iomem *addr = info->regs[SEC_SAA] + SEC_CTRL2_REG; + u32 regval; + + regval = readl_relaxed(addr); + regval &= ~SEC_CTRL2_DATA_AXI_RD_OTSD_CFG_M; + regval |= (cfg << SEC_CTRL2_DATA_AXI_RD_OTSD_CFG_S) & + SEC_CTRL2_DATA_AXI_RD_OTSD_CFG_M; + writel_relaxed(regval, addr); +} + +static void sec_clk_gate_en(struct sec_dev_info *info, bool clkgate) +{ + void __iomem *addr = info->regs[SEC_SAA] + SEC_CTRL2_REG; + u32 regval; + + regval = readl_relaxed(addr); + if (clkgate) + regval |= SEC_CTRL2_CLK_GATE_EN; + else + regval &= ~SEC_CTRL2_CLK_GATE_EN; + writel_relaxed(regval, addr); +} + +static void sec_comm_cnt_cfg(struct sec_dev_info *info, bool clr_ce) +{ + void __iomem *addr = info->regs[SEC_SAA] + SEC_COMMON_CNT_CLR_CE_REG; + u32 regval; + + regval = readl_relaxed(addr); + if (clr_ce) + regval |= SEC_COMMON_CNT_CLR_CE_CLEAR; + else + regval &= ~SEC_COMMON_CNT_CLR_CE_CLEAR; + writel_relaxed(regval, addr); +} + +static void sec_commsnap_en(struct sec_dev_info *info, bool snap_en) +{ + void __iomem *addr = info->regs[SEC_SAA] + SEC_COMMON_CNT_CLR_CE_REG; + u32 regval; + + regval = readl_relaxed(addr); + if (snap_en) + regval |= SEC_COMMON_CNT_CLR_CE_SNAP_EN; + else + regval &= ~SEC_COMMON_CNT_CLR_CE_SNAP_EN; + writel_relaxed(regval, addr); +} + +static void sec_ipv6_hashmask(struct sec_dev_info *info, u32 hash_mask[]) +{ + void __iomem *base = info->regs[SEC_SAA]; + int i; + + for (i = 0; i < 10; i++) + writel_relaxed(hash_mask[0], + base + SEC_IPV6_MASK_TABLE_X_REG(i)); +} + +static int sec_ipv4_hashmask(struct sec_dev_info *info, u32 hash_mask) +{ + if (hash_mask & SEC_HASH_IPV4_MASK) { + dev_err(info->dev, "Sec Ipv4 Hash Mask Input Error!\n"); + return -EINVAL; + } + + writel_relaxed(hash_mask, + info->regs[SEC_SAA] + SEC_IPV4_MASK_TABLE_REG); + + return 0; +} + +static void sec_set_dbg_bd_cfg(struct sec_dev_info *info, u32 cfg) +{ + void __iomem *addr = info->regs[SEC_SAA] + SEC_DEBUG_BD_CFG_REG; + u32 regval; + + regval = readl_relaxed(addr); + /* Always disable write back of normal bd */ + regval &= ~SEC_DEBUG_BD_CFG_WB_NORMAL; + + if (cfg) + regval &= ~SEC_DEBUG_BD_CFG_WB_EN; + else + regval |= SEC_DEBUG_BD_CFG_WB_EN; + + writel_relaxed(regval, addr); +} + +static void sec_saa_getqm_en(struct sec_dev_info *info, u32 saa_indx, u32 en) +{ + void __iomem *addr = info->regs[SEC_SAA] + SEC_SAA_BASE + + SEC_SAA_CTRL_REG(saa_indx); + u32 regval; + + regval = readl_relaxed(addr); + if (en) + regval |= SEC_SAA_CTRL_GET_QM_EN; + else + regval &= ~SEC_SAA_CTRL_GET_QM_EN; + writel_relaxed(regval, addr); +} + +static void sec_saa_int_mask(struct sec_dev_info *info, u32 saa_indx, + u32 saa_int_mask) +{ + writel_relaxed(saa_int_mask, + info->regs[SEC_SAA] + SEC_SAA_BASE + SEC_ST_INTMSK1_REG + + saa_indx * SEC_SAA_ADDR_SIZE); +} + +static void sec_streamid(struct sec_dev_info *info, int i) +{ + #define SEC_SID 0x600 + #define SEC_VMID 0 + + writel_relaxed((SEC_VMID | ((SEC_SID & 0xffff) << 8)), + info->regs[SEC_SAA] + SEC_Q_VMID_CFG_REG(i)); +} + +static void sec_queue_ar_alloc(struct sec_queue *queue, u32 alloc) +{ + void __iomem *addr = queue->regs + SEC_Q_ARUSER_CFG_REG; + u32 regval; + + regval = readl_relaxed(addr); + if (alloc == SEC_QUEUE_AR_FROCE_ALLOC) { + regval |= SEC_Q_ARUSER_CFG_FA; + regval &= ~SEC_Q_ARUSER_CFG_FNA; + } else { + regval &= ~SEC_Q_ARUSER_CFG_FA; + regval |= SEC_Q_ARUSER_CFG_FNA; + } + + writel_relaxed(regval, addr); +} + +static void sec_queue_aw_alloc(struct sec_queue *queue, u32 alloc) +{ + void __iomem *addr = queue->regs + SEC_Q_AWUSER_CFG_REG; + u32 regval; + + regval = readl_relaxed(addr); + if (alloc == SEC_QUEUE_AW_FROCE_ALLOC) { + regval |= SEC_Q_AWUSER_CFG_FA; + regval &= ~SEC_Q_AWUSER_CFG_FNA; + } else { + regval &= ~SEC_Q_AWUSER_CFG_FA; + regval |= SEC_Q_AWUSER_CFG_FNA; + } + + writel_relaxed(regval, addr); +} + +static void sec_queue_reorder(struct sec_queue *queue, bool reorder) +{ + void __iomem *base = queue->regs; + u32 regval; + + regval = readl_relaxed(base + SEC_Q_CFG_REG); + if (reorder) + regval |= SEC_Q_CFG_REORDER; + else + regval &= ~SEC_Q_CFG_REORDER; + writel_relaxed(regval, base + SEC_Q_CFG_REG); +} + +static void sec_queue_depth(struct sec_queue *queue, u32 depth) +{ + void __iomem *addr = queue->regs + SEC_Q_DEPTH_CFG_REG; + u32 regval; + + regval = readl_relaxed(addr); + regval &= ~SEC_Q_DEPTH_CFG_DEPTH_M; + regval |= (depth << SEC_Q_DEPTH_CFG_DEPTH_S) & SEC_Q_DEPTH_CFG_DEPTH_M; + + writel_relaxed(regval, addr); +} + +static void sec_queue_cmdbase_addr(struct sec_queue *queue, u64 addr) +{ + writel_relaxed(upper_32_bits(addr), queue->regs + SEC_Q_BASE_HADDR_REG); + writel_relaxed(lower_32_bits(addr), queue->regs + SEC_Q_BASE_LADDR_REG); +} + +static void sec_queue_outorder_addr(struct sec_queue *queue, u64 addr) +{ + writel_relaxed(upper_32_bits(addr), + queue->regs + SEC_Q_OUTORDER_BASE_HADDR_REG); + writel_relaxed(lower_32_bits(addr), + queue->regs + SEC_Q_OUTORDER_BASE_LADDR_REG); +} + +static void sec_queue_errbase_addr(struct sec_queue *queue, u64 addr) +{ + writel_relaxed(upper_32_bits(addr), + queue->regs + SEC_Q_ERR_BASE_HADDR_REG); + writel_relaxed(lower_32_bits(addr), + queue->regs + SEC_Q_ERR_BASE_LADDR_REG); +} + +static void sec_queue_irq_disable(struct sec_queue *queue) +{ + writel_relaxed((u32)~0, queue->regs + SEC_Q_FLOW_INT_MKS_REG); +} + +static void sec_queue_irq_enable(struct sec_queue *queue) +{ + writel_relaxed(0, queue->regs + SEC_Q_FLOW_INT_MKS_REG); +} + +static void sec_queue_abn_irq_disable(struct sec_queue *queue) +{ + writel_relaxed((u32)~0, queue->regs + SEC_Q_FAIL_INT_MSK_REG); +} + +static void sec_queue_stop(struct sec_queue *queue) +{ + disable_irq(queue->task_irq); + sec_queue_irq_disable(queue); + writel_relaxed(0x0, queue->regs + SEC_QUEUE_ENB_REG); +} + +static void sec_queue_start(struct sec_queue *queue) +{ + sec_queue_irq_enable(queue); + enable_irq(queue->task_irq); + queue->expected = 0; + writel_relaxed(SEC_Q_INIT_AND_STAT_CLEAR, queue->regs + SEC_Q_INIT_REG); + writel_relaxed(0x1, queue->regs + SEC_QUEUE_ENB_REG); +} + +static struct sec_queue *sec_alloc_queue(struct sec_dev_info *info) +{ + int i; + + mutex_lock(&info->dev_lock); + + /* Get the first idle queue in SEC device */ + for (i = 0; i < SEC_Q_NUM; i++) + if (!info->queues[i].in_use) { + info->queues[i].in_use = true; + info->queues_in_use++; + mutex_unlock(&info->dev_lock); + + return &info->queues[i]; + } + mutex_unlock(&info->dev_lock); + + return ERR_PTR(-ENODEV); +} + +static int sec_queue_free(struct sec_queue *queue) +{ + struct sec_dev_info *info = queue->dev_info; + + if (queue->queue_id >= SEC_Q_NUM) { + dev_err(info->dev, "No queue %u\n", queue->queue_id); + return -ENODEV; + } + + if (!queue->in_use) { + dev_err(info->dev, "Queue %u is idle\n", queue->queue_id); + return -ENODEV; + } + + mutex_lock(&info->dev_lock); + queue->in_use = false; + info->queues_in_use--; + mutex_unlock(&info->dev_lock); + + return 0; +} + +static irqreturn_t sec_isr_handle_th(int irq, void *q) +{ + sec_queue_irq_disable(q); + return IRQ_WAKE_THREAD; +} + +static irqreturn_t sec_isr_handle(int irq, void *q) +{ + struct sec_queue *queue = q; + struct sec_queue_ring_cmd *msg_ring = &queue->ring_cmd; + struct sec_queue_ring_cq *cq_ring = &queue->ring_cq; + struct sec_out_bd_info *outorder_msg; + struct sec_bd_info *msg; + u32 ooo_read, ooo_write; + void __iomem *base = queue->regs; + int q_id; + + ooo_read = readl(base + SEC_Q_OUTORDER_RD_PTR_REG); + ooo_write = readl(base + SEC_Q_OUTORDER_WR_PTR_REG); + outorder_msg = cq_ring->vaddr + ooo_read; + q_id = outorder_msg->data & SEC_OUT_BD_INFO_Q_ID_M; + msg = msg_ring->vaddr + q_id; + + while ((ooo_write != ooo_read) && msg->w0 & SEC_BD_W0_DONE) { + /* + * Must be before callback otherwise blocks adding other chained + * elements + */ + set_bit(q_id, queue->unprocessed); + if (q_id == queue->expected) + while (test_bit(queue->expected, queue->unprocessed)) { + clear_bit(queue->expected, queue->unprocessed); + msg = msg_ring->vaddr + queue->expected; + msg->w0 &= ~SEC_BD_W0_DONE; + msg_ring->callback(msg, + queue->shadow[queue->expected]); + queue->shadow[queue->expected] = NULL; + queue->expected = (queue->expected + 1) % + SEC_QUEUE_LEN; + atomic_dec(&msg_ring->used); + } + + ooo_read = (ooo_read + 1) % SEC_QUEUE_LEN; + writel(ooo_read, base + SEC_Q_OUTORDER_RD_PTR_REG); + ooo_write = readl(base + SEC_Q_OUTORDER_WR_PTR_REG); + outorder_msg = cq_ring->vaddr + ooo_read; + q_id = outorder_msg->data & SEC_OUT_BD_INFO_Q_ID_M; + msg = msg_ring->vaddr + q_id; + } + + sec_queue_irq_enable(queue); + + return IRQ_HANDLED; +} + +static int sec_queue_irq_init(struct sec_queue *queue) +{ + struct sec_dev_info *info = queue->dev_info; + int irq = queue->task_irq; + int ret; + + ret = request_threaded_irq(irq, sec_isr_handle_th, sec_isr_handle, + IRQF_TRIGGER_RISING, queue->name, queue); + if (ret) { + dev_err(info->dev, "request irq(%d) failed %d\n", irq, ret); + return ret; + } + disable_irq(irq); + + return 0; +} + +static int sec_queue_irq_uninit(struct sec_queue *queue) +{ + free_irq(queue->task_irq, queue); + + return 0; +} + +static struct sec_dev_info *sec_device_get(void) +{ + struct sec_dev_info *sec_dev = NULL; + struct sec_dev_info *this_sec_dev; + int least_busy_n = SEC_Q_NUM + 1; + int i; + + /* Find which one is least busy and use that first */ + for (i = 0; i < SEC_MAX_DEVICES; i++) { + this_sec_dev = sec_devices[i]; + if (this_sec_dev && + this_sec_dev->queues_in_use < least_busy_n) { + least_busy_n = this_sec_dev->queues_in_use; + sec_dev = this_sec_dev; + } + } + + return sec_dev; +} + +static struct sec_queue *sec_queue_alloc_start(struct sec_dev_info *info) +{ + struct sec_queue *queue; + + queue = sec_alloc_queue(info); + if (IS_ERR(queue)) { + dev_err(info->dev, "alloc sec queue failed! %ld\n", + PTR_ERR(queue)); + return queue; + } + + sec_queue_start(queue); + + return queue; +} + +/** + * sec_queue_alloc_start_safe - get a hw queue from appropriate instance + * + * This function does extremely simplistic load balancing. It does not take into + * account NUMA locality of the accelerator, or which cpu has requested the + * queue. Future work may focus on optimizing this in order to improve full + * machine throughput. + */ +struct sec_queue *sec_queue_alloc_start_safe(void) +{ + struct sec_dev_info *info; + struct sec_queue *queue = ERR_PTR(-ENODEV); + + mutex_lock(&sec_id_lock); + info = sec_device_get(); + if (!info) + goto unlock; + + queue = sec_queue_alloc_start(info); + +unlock: + mutex_unlock(&sec_id_lock); + + return queue; +} + +/** + * sec_queue_stop_release() - free up a hw queue for reuse + * @queue: The queue we are done with. + * + * This will stop the current queue, terminanting any transactions + * that are inflight an return it to the pool of available hw queuess + */ +int sec_queue_stop_release(struct sec_queue *queue) +{ + struct device *dev = queue->dev_info->dev; + int ret; + + sec_queue_stop(queue); + + ret = sec_queue_free(queue); + if (ret) + dev_err(dev, "Releasing queue failed %d\n", ret); + + return ret; +} + +/** + * sec_queue_empty() - Is this hardware queue currently empty. + * @queue: The queue to test + * + * We need to know if we have an empty queue for some of the chaining modes + * as if it is not empty we may need to hold the message in a software queue + * until the hw queue is drained. + */ +bool sec_queue_empty(struct sec_queue *queue) +{ + struct sec_queue_ring_cmd *msg_ring = &queue->ring_cmd; + + return !atomic_read(&msg_ring->used); +} + +/** + * sec_queue_send() - queue up a single operation in the hw queue + * @queue: The queue in which to put the message + * @msg: The message + * @ctx: Context to be put in the shadow array and passed back to cb on result. + * + * This function will return -EAGAIN if the queue is currently full. + */ +int sec_queue_send(struct sec_queue *queue, struct sec_bd_info *msg, void *ctx) +{ + struct sec_queue_ring_cmd *msg_ring = &queue->ring_cmd; + void __iomem *base = queue->regs; + u32 write, read; + + mutex_lock(&msg_ring->lock); + read = readl(base + SEC_Q_RD_PTR_REG); + write = readl(base + SEC_Q_WR_PTR_REG); + if (write == read && atomic_read(&msg_ring->used) == SEC_QUEUE_LEN) { + mutex_unlock(&msg_ring->lock); + return -EAGAIN; + } + memcpy(msg_ring->vaddr + write, msg, sizeof(*msg)); + queue->shadow[write] = ctx; + write = (write + 1) % SEC_QUEUE_LEN; + + /* Ensure content updated before queue advance */ + wmb(); + writel(write, base + SEC_Q_WR_PTR_REG); + + atomic_inc(&msg_ring->used); + mutex_unlock(&msg_ring->lock); + + return 0; +} + +bool sec_queue_can_enqueue(struct sec_queue *queue, int num) +{ + struct sec_queue_ring_cmd *msg_ring = &queue->ring_cmd; + + return SEC_QUEUE_LEN - atomic_read(&msg_ring->used) >= num; +} + +static void sec_queue_hw_init(struct sec_queue *queue) +{ + sec_queue_ar_alloc(queue, SEC_QUEUE_AR_FROCE_NOALLOC); + sec_queue_aw_alloc(queue, SEC_QUEUE_AW_FROCE_NOALLOC); + sec_queue_ar_pkgattr(queue, 1); + sec_queue_aw_pkgattr(queue, 1); + + /* Enable out of order queue */ + sec_queue_reorder(queue, true); + + /* Interrupt after a single complete element */ + writel_relaxed(1, queue->regs + SEC_Q_PROC_NUM_CFG_REG); + + sec_queue_depth(queue, SEC_QUEUE_LEN - 1); + + sec_queue_cmdbase_addr(queue, queue->ring_cmd.paddr); + + sec_queue_outorder_addr(queue, queue->ring_cq.paddr); + + sec_queue_errbase_addr(queue, queue->ring_db.paddr); + + writel_relaxed(0x100, queue->regs + SEC_Q_OT_TH_REG); + + sec_queue_abn_irq_disable(queue); + sec_queue_irq_disable(queue); + writel_relaxed(SEC_Q_INIT_AND_STAT_CLEAR, queue->regs + SEC_Q_INIT_REG); +} + +static int sec_hw_init(struct sec_dev_info *info) +{ + struct iommu_domain *domain; + u32 sec_ipv4_mask = 0; + u32 sec_ipv6_mask[10] = {}; + int ret; + u32 i; + + domain = iommu_get_domain_for_dev(info->dev); + + /* + * Enable all available processing unit clocks. + * Only the first cluster is usable with translations. + */ + if (domain && (domain->type & __IOMMU_DOMAIN_PAGING)) + info->num_saas = 5; + + else + info->num_saas = 10; + + writel_relaxed(GENMASK(info->num_saas - 1, 0), + info->regs[SEC_SAA] + SEC_CLK_EN_REG); + + /* 32 bit little endian */ + sec_bd_endian_little(info); + + sec_cache_config(info); + + /* Data axi port write and read outstanding config as per datasheet */ + sec_data_axiwr_otsd_cfg(info, 0x7); + sec_data_axird_otsd_cfg(info, 0x7); + + /* Enable clock gating */ + sec_clk_gate_en(info, true); + + /* Set CNT_CYC register not read clear */ + sec_comm_cnt_cfg(info, false); + + /* Enable CNT_CYC */ + sec_commsnap_en(info, false); + + writel_relaxed((u32)~0, info->regs[SEC_SAA] + SEC_FSM_MAX_CNT_REG); + + ret = sec_ipv4_hashmask(info, sec_ipv4_mask); + if (ret) { + dev_err(info->dev, "Failed to set ipv4 hashmask %d\n", ret); + return -EIO; + } + + sec_ipv6_hashmask(info, sec_ipv6_mask); + + /* do not use debug bd */ + sec_set_dbg_bd_cfg(info, 0); + + if (domain && (domain->type & __IOMMU_DOMAIN_PAGING)) { + for (i = 0; i < SEC_Q_NUM; i++) { + sec_streamid(info, i); + /* Same QoS for all queues */ + writel_relaxed(0x3f, + info->regs[SEC_SAA] + + SEC_Q_WEIGHT_CFG_REG(i)); + } + } + + for (i = 0; i < info->num_saas; i++) { + sec_saa_getqm_en(info, i, 1); + sec_saa_int_mask(info, i, 0); + } + + return 0; +} + +static void sec_hw_exit(struct sec_dev_info *info) +{ + int i; + + for (i = 0; i < SEC_MAX_SAA_NUM; i++) { + sec_saa_int_mask(info, i, (u32)~0); + sec_saa_getqm_en(info, i, 0); + } +} + +static void sec_queue_base_init(struct sec_dev_info *info, + struct sec_queue *queue, int queue_id) +{ + queue->dev_info = info; + queue->queue_id = queue_id; + snprintf(queue->name, sizeof(queue->name), + "%s_%d", dev_name(info->dev), queue->queue_id); +} + +static int sec_map_io(struct sec_dev_info *info, struct platform_device *pdev) +{ + int i; + + for (i = 0; i < SEC_NUM_ADDR_REGIONS; i++) { + info->regs[i] = devm_platform_ioremap_resource(pdev, i); + if (IS_ERR(info->regs[i])) + return PTR_ERR(info->regs[i]); + } + + return 0; +} + +static int sec_base_init(struct sec_dev_info *info, + struct platform_device *pdev) +{ + int ret; + + ret = sec_map_io(info, pdev); + if (ret) + return ret; + + ret = sec_clk_en(info); + if (ret) + return ret; + + ret = sec_reset_whole_module(info); + if (ret) + goto sec_clk_disable; + + ret = sec_hw_init(info); + if (ret) + goto sec_clk_disable; + + return 0; + +sec_clk_disable: + sec_clk_dis(info); + + return ret; +} + +static void sec_base_exit(struct sec_dev_info *info) +{ + sec_hw_exit(info); + sec_clk_dis(info); +} + +#define SEC_Q_CMD_SIZE \ + round_up(SEC_QUEUE_LEN * sizeof(struct sec_bd_info), PAGE_SIZE) +#define SEC_Q_CQ_SIZE \ + round_up(SEC_QUEUE_LEN * sizeof(struct sec_out_bd_info), PAGE_SIZE) +#define SEC_Q_DB_SIZE \ + round_up(SEC_QUEUE_LEN * sizeof(struct sec_debug_bd_info), PAGE_SIZE) + +static int sec_queue_res_cfg(struct sec_queue *queue) +{ + struct device *dev = queue->dev_info->dev; + struct sec_queue_ring_cmd *ring_cmd = &queue->ring_cmd; + struct sec_queue_ring_cq *ring_cq = &queue->ring_cq; + struct sec_queue_ring_db *ring_db = &queue->ring_db; + int ret; + + ring_cmd->vaddr = dma_alloc_coherent(dev, SEC_Q_CMD_SIZE, + &ring_cmd->paddr, GFP_KERNEL); + if (!ring_cmd->vaddr) + return -ENOMEM; + + atomic_set(&ring_cmd->used, 0); + mutex_init(&ring_cmd->lock); + ring_cmd->callback = sec_alg_callback; + + ring_cq->vaddr = dma_alloc_coherent(dev, SEC_Q_CQ_SIZE, + &ring_cq->paddr, GFP_KERNEL); + if (!ring_cq->vaddr) { + ret = -ENOMEM; + goto err_free_ring_cmd; + } + + ring_db->vaddr = dma_alloc_coherent(dev, SEC_Q_DB_SIZE, + &ring_db->paddr, GFP_KERNEL); + if (!ring_db->vaddr) { + ret = -ENOMEM; + goto err_free_ring_cq; + } + queue->task_irq = platform_get_irq(to_platform_device(dev), + queue->queue_id * 2 + 1); + if (queue->task_irq < 0) { + ret = queue->task_irq; + goto err_free_ring_db; + } + + return 0; + +err_free_ring_db: + dma_free_coherent(dev, SEC_Q_DB_SIZE, queue->ring_db.vaddr, + queue->ring_db.paddr); +err_free_ring_cq: + dma_free_coherent(dev, SEC_Q_CQ_SIZE, queue->ring_cq.vaddr, + queue->ring_cq.paddr); +err_free_ring_cmd: + dma_free_coherent(dev, SEC_Q_CMD_SIZE, queue->ring_cmd.vaddr, + queue->ring_cmd.paddr); + + return ret; +} + +static void sec_queue_free_ring_pages(struct sec_queue *queue) +{ + struct device *dev = queue->dev_info->dev; + + dma_free_coherent(dev, SEC_Q_DB_SIZE, queue->ring_db.vaddr, + queue->ring_db.paddr); + dma_free_coherent(dev, SEC_Q_CQ_SIZE, queue->ring_cq.vaddr, + queue->ring_cq.paddr); + dma_free_coherent(dev, SEC_Q_CMD_SIZE, queue->ring_cmd.vaddr, + queue->ring_cmd.paddr); +} + +static int sec_queue_config(struct sec_dev_info *info, struct sec_queue *queue, + int queue_id) +{ + int ret; + + sec_queue_base_init(info, queue, queue_id); + + ret = sec_queue_res_cfg(queue); + if (ret) + return ret; + + ret = sec_queue_map_io(queue); + if (ret) { + dev_err(info->dev, "Queue map failed %d\n", ret); + sec_queue_free_ring_pages(queue); + return ret; + } + + sec_queue_hw_init(queue); + + return 0; +} + +static void sec_queue_unconfig(struct sec_dev_info *info, + struct sec_queue *queue) +{ + sec_queue_unmap_io(queue); + sec_queue_free_ring_pages(queue); +} + +static int sec_id_alloc(struct sec_dev_info *info) +{ + int ret = 0; + int i; + + mutex_lock(&sec_id_lock); + + for (i = 0; i < SEC_MAX_DEVICES; i++) + if (!sec_devices[i]) + break; + if (i == SEC_MAX_DEVICES) { + ret = -ENOMEM; + goto unlock; + } + info->sec_id = i; + sec_devices[info->sec_id] = info; + +unlock: + mutex_unlock(&sec_id_lock); + + return ret; +} + +static void sec_id_free(struct sec_dev_info *info) +{ + mutex_lock(&sec_id_lock); + sec_devices[info->sec_id] = NULL; + mutex_unlock(&sec_id_lock); +} + +static int sec_probe(struct platform_device *pdev) +{ + struct sec_dev_info *info; + struct device *dev = &pdev->dev; + int i, j; + int ret; + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); + if (ret) { + dev_err(dev, "Failed to set 64 bit dma mask %d", ret); + return -ENODEV; + } + + info = devm_kzalloc(dev, (sizeof(*info)), GFP_KERNEL); + if (!info) + return -ENOMEM; + + info->dev = dev; + mutex_init(&info->dev_lock); + + info->hw_sgl_pool = dmam_pool_create("sgl", dev, + sizeof(struct sec_hw_sgl), 64, 0); + if (!info->hw_sgl_pool) { + dev_err(dev, "Failed to create sec sgl dma pool\n"); + return -ENOMEM; + } + + ret = sec_base_init(info, pdev); + if (ret) { + dev_err(dev, "Base initialization fail! %d\n", ret); + return ret; + } + + for (i = 0; i < SEC_Q_NUM; i++) { + ret = sec_queue_config(info, &info->queues[i], i); + if (ret) + goto queues_unconfig; + + ret = sec_queue_irq_init(&info->queues[i]); + if (ret) { + sec_queue_unconfig(info, &info->queues[i]); + goto queues_unconfig; + } + } + + ret = sec_algs_register(); + if (ret) { + dev_err(dev, "Failed to register algorithms with crypto %d\n", + ret); + goto queues_unconfig; + } + + platform_set_drvdata(pdev, info); + + ret = sec_id_alloc(info); + if (ret) + goto algs_unregister; + + return 0; + +algs_unregister: + sec_algs_unregister(); +queues_unconfig: + for (j = i - 1; j >= 0; j--) { + sec_queue_irq_uninit(&info->queues[j]); + sec_queue_unconfig(info, &info->queues[j]); + } + sec_base_exit(info); + + return ret; +} + +static void sec_remove(struct platform_device *pdev) +{ + struct sec_dev_info *info = platform_get_drvdata(pdev); + int i; + + /* Unexpose as soon as possible, reuse during remove is fine */ + sec_id_free(info); + + sec_algs_unregister(); + + for (i = 0; i < SEC_Q_NUM; i++) { + sec_queue_irq_uninit(&info->queues[i]); + sec_queue_unconfig(info, &info->queues[i]); + } + + sec_base_exit(info); +} + +static const __maybe_unused struct of_device_id sec_match[] = { + { .compatible = "hisilicon,hip06-sec" }, + { .compatible = "hisilicon,hip07-sec" }, + {} +}; +MODULE_DEVICE_TABLE(of, sec_match); + +static const __maybe_unused struct acpi_device_id sec_acpi_match[] = { + { "HISI02C1", 0 }, + { } +}; +MODULE_DEVICE_TABLE(acpi, sec_acpi_match); + +static struct platform_driver sec_driver = { + .probe = sec_probe, + .remove = sec_remove, + .driver = { + .name = "hisi_sec_platform_driver", + .of_match_table = sec_match, + .acpi_match_table = ACPI_PTR(sec_acpi_match), + }, +}; +module_platform_driver(sec_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("HiSilicon Security Accelerators"); +MODULE_AUTHOR("Zaibo Xu "); diff --git a/drivers/crypto/hisilicon/sec/sec_drv.h b/drivers/crypto/hisilicon/sec/sec_drv.h new file mode 100644 index 000000000..e2a50bf22 --- /dev/null +++ b/drivers/crypto/hisilicon/sec/sec_drv.h @@ -0,0 +1,428 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2016-2017 HiSilicon Limited. */ + +#ifndef _SEC_DRV_H_ +#define _SEC_DRV_H_ + +#include +#include + +#define SEC_MAX_SGE_NUM 64 +#define SEC_HW_RING_NUM 3 + +#define SEC_CMD_RING 0 +#define SEC_OUTORDER_RING 1 +#define SEC_DBG_RING 2 + +/* A reasonable length to balance memory use against flexibility */ +#define SEC_QUEUE_LEN 512 + +#define SEC_MAX_SGE_NUM 64 + +struct sec_bd_info { +#define SEC_BD_W0_T_LEN_M GENMASK(4, 0) +#define SEC_BD_W0_T_LEN_S 0 + +#define SEC_BD_W0_C_WIDTH_M GENMASK(6, 5) +#define SEC_BD_W0_C_WIDTH_S 5 +#define SEC_C_WIDTH_AES_128BIT 0 +#define SEC_C_WIDTH_AES_8BIT 1 +#define SEC_C_WIDTH_AES_1BIT 2 +#define SEC_C_WIDTH_DES_64BIT 0 +#define SEC_C_WIDTH_DES_8BIT 1 +#define SEC_C_WIDTH_DES_1BIT 2 + +#define SEC_BD_W0_C_MODE_M GENMASK(9, 7) +#define SEC_BD_W0_C_MODE_S 7 +#define SEC_C_MODE_ECB 0 +#define SEC_C_MODE_CBC 1 +#define SEC_C_MODE_CTR 4 +#define SEC_C_MODE_CCM 5 +#define SEC_C_MODE_GCM 6 +#define SEC_C_MODE_XTS 7 + +#define SEC_BD_W0_SEQ BIT(10) +#define SEC_BD_W0_DE BIT(11) +#define SEC_BD_W0_DAT_SKIP_M GENMASK(13, 12) +#define SEC_BD_W0_DAT_SKIP_S 12 +#define SEC_BD_W0_C_GRAN_SIZE_19_16_M GENMASK(17, 14) +#define SEC_BD_W0_C_GRAN_SIZE_19_16_S 14 + +#define SEC_BD_W0_CIPHER_M GENMASK(19, 18) +#define SEC_BD_W0_CIPHER_S 18 +#define SEC_CIPHER_NULL 0 +#define SEC_CIPHER_ENCRYPT 1 +#define SEC_CIPHER_DECRYPT 2 + +#define SEC_BD_W0_AUTH_M GENMASK(21, 20) +#define SEC_BD_W0_AUTH_S 20 +#define SEC_AUTH_NULL 0 +#define SEC_AUTH_MAC 1 +#define SEC_AUTH_VERIF 2 + +#define SEC_BD_W0_AI_GEN BIT(22) +#define SEC_BD_W0_CI_GEN BIT(23) +#define SEC_BD_W0_NO_HPAD BIT(24) +#define SEC_BD_W0_HM_M GENMASK(26, 25) +#define SEC_BD_W0_HM_S 25 +#define SEC_BD_W0_ICV_OR_SKEY_EN_M GENMASK(28, 27) +#define SEC_BD_W0_ICV_OR_SKEY_EN_S 27 + +/* Multi purpose field - gran size bits for send, flag for recv */ +#define SEC_BD_W0_FLAG_M GENMASK(30, 29) +#define SEC_BD_W0_C_GRAN_SIZE_21_20_M GENMASK(30, 29) +#define SEC_BD_W0_FLAG_S 29 +#define SEC_BD_W0_C_GRAN_SIZE_21_20_S 29 + +#define SEC_BD_W0_DONE BIT(31) + u32 w0; + +#define SEC_BD_W1_AUTH_GRAN_SIZE_M GENMASK(21, 0) +#define SEC_BD_W1_AUTH_GRAN_SIZE_S 0 +#define SEC_BD_W1_M_KEY_EN BIT(22) +#define SEC_BD_W1_BD_INVALID BIT(23) +#define SEC_BD_W1_ADDR_TYPE BIT(24) + +#define SEC_BD_W1_A_ALG_M GENMASK(28, 25) +#define SEC_BD_W1_A_ALG_S 25 +#define SEC_A_ALG_SHA1 0 +#define SEC_A_ALG_SHA256 1 +#define SEC_A_ALG_MD5 2 +#define SEC_A_ALG_SHA224 3 +#define SEC_A_ALG_HMAC_SHA1 8 +#define SEC_A_ALG_HMAC_SHA224 10 +#define SEC_A_ALG_HMAC_SHA256 11 +#define SEC_A_ALG_HMAC_MD5 12 +#define SEC_A_ALG_AES_XCBC 13 +#define SEC_A_ALG_AES_CMAC 14 + +#define SEC_BD_W1_C_ALG_M GENMASK(31, 29) +#define SEC_BD_W1_C_ALG_S 29 +#define SEC_C_ALG_DES 0 +#define SEC_C_ALG_3DES 1 +#define SEC_C_ALG_AES 2 + + u32 w1; + +#define SEC_BD_W2_C_GRAN_SIZE_15_0_M GENMASK(15, 0) +#define SEC_BD_W2_C_GRAN_SIZE_15_0_S 0 +#define SEC_BD_W2_GRAN_NUM_M GENMASK(31, 16) +#define SEC_BD_W2_GRAN_NUM_S 16 + u32 w2; + +#define SEC_BD_W3_AUTH_LEN_OFFSET_M GENMASK(9, 0) +#define SEC_BD_W3_AUTH_LEN_OFFSET_S 0 +#define SEC_BD_W3_CIPHER_LEN_OFFSET_M GENMASK(19, 10) +#define SEC_BD_W3_CIPHER_LEN_OFFSET_S 10 +#define SEC_BD_W3_MAC_LEN_M GENMASK(24, 20) +#define SEC_BD_W3_MAC_LEN_S 20 +#define SEC_BD_W3_A_KEY_LEN_M GENMASK(29, 25) +#define SEC_BD_W3_A_KEY_LEN_S 25 +#define SEC_BD_W3_C_KEY_LEN_M GENMASK(31, 30) +#define SEC_BD_W3_C_KEY_LEN_S 30 +#define SEC_KEY_LEN_AES_128 0 +#define SEC_KEY_LEN_AES_192 1 +#define SEC_KEY_LEN_AES_256 2 +#define SEC_KEY_LEN_DES 1 +#define SEC_KEY_LEN_3DES_3_KEY 1 +#define SEC_KEY_LEN_3DES_2_KEY 3 + u32 w3; + + /* W4,5 */ + union { + u32 authkey_addr_lo; + u32 authiv_addr_lo; + }; + union { + u32 authkey_addr_hi; + u32 authiv_addr_hi; + }; + + /* W6,7 */ + u32 cipher_key_addr_lo; + u32 cipher_key_addr_hi; + + /* W8,9 */ + u32 cipher_iv_addr_lo; + u32 cipher_iv_addr_hi; + + /* W10,11 */ + u32 data_addr_lo; + u32 data_addr_hi; + + /* W12,13 */ + u32 mac_addr_lo; + u32 mac_addr_hi; + + /* W14,15 */ + u32 cipher_destin_addr_lo; + u32 cipher_destin_addr_hi; +}; + +enum sec_mem_region { + SEC_COMMON = 0, + SEC_SAA, + SEC_NUM_ADDR_REGIONS +}; + +#define SEC_NAME_SIZE 64 +#define SEC_Q_NUM 16 + + +/** + * struct sec_queue_ring_cmd - store information about a SEC HW cmd ring + * @used: Local counter used to cheaply establish if the ring is empty. + * @lock: Protect against simultaneous adjusting of the read and write pointers. + * @vaddr: Virtual address for the ram pages used for the ring. + * @paddr: Physical address of the dma mapped region of ram used for the ring. + * @callback: Callback function called on a ring element completing. + */ +struct sec_queue_ring_cmd { + atomic_t used; + struct mutex lock; + struct sec_bd_info *vaddr; + dma_addr_t paddr; + void (*callback)(struct sec_bd_info *resp, void *ctx); +}; + +struct sec_debug_bd_info; +struct sec_queue_ring_db { + struct sec_debug_bd_info *vaddr; + dma_addr_t paddr; +}; + +struct sec_out_bd_info; +struct sec_queue_ring_cq { + struct sec_out_bd_info *vaddr; + dma_addr_t paddr; +}; + +struct sec_dev_info; + +enum sec_cipher_alg { + SEC_C_DES_ECB_64, + SEC_C_DES_CBC_64, + + SEC_C_3DES_ECB_192_3KEY, + SEC_C_3DES_ECB_192_2KEY, + + SEC_C_3DES_CBC_192_3KEY, + SEC_C_3DES_CBC_192_2KEY, + + SEC_C_AES_ECB_128, + SEC_C_AES_ECB_192, + SEC_C_AES_ECB_256, + + SEC_C_AES_CBC_128, + SEC_C_AES_CBC_192, + SEC_C_AES_CBC_256, + + SEC_C_AES_CTR_128, + SEC_C_AES_CTR_192, + SEC_C_AES_CTR_256, + + SEC_C_AES_XTS_128, + SEC_C_AES_XTS_256, + + SEC_C_NULL, +}; + +/** + * struct sec_alg_tfm_ctx - hardware specific tranformation context + * @cipher_alg: Cipher algorithm enabled include encryption mode. + * @key: Key storage if required. + * @pkey: DMA address for the key storage. + * @req_template: Request template to save time on setup. + * @queue: The hardware queue associated with this tfm context. + * @lock: Protect key and pkey to ensure they are consistent + * @auth_buf: Current context buffer for auth operations. + * @backlog: The backlog queue used for cases where our buffers aren't + * large enough. + */ +struct sec_alg_tfm_ctx { + enum sec_cipher_alg cipher_alg; + u8 *key; + dma_addr_t pkey; + struct sec_bd_info req_template; + struct sec_queue *queue; + struct mutex lock; + u8 *auth_buf; + struct list_head backlog; +}; + +/** + * struct sec_request - data associate with a single crypto request + * @elements: List of subparts of this request (hardware size restriction) + * @num_elements: The number of subparts (used as an optimization) + * @lock: Protect elements of this structure against concurrent change. + * @tfm_ctx: hardware specific context. + * @len_in: length of in sgl from upper layers + * @len_out: length of out sgl from upper layers + * @dma_iv: initialization vector - phsyical address + * @err: store used to track errors across subelements of this request. + * @req_base: pointer to base element of associate crypto context. + * This is needed to allow shared handling skcipher, ahash etc. + * @cb: completion callback. + * @backlog_head: list head to allow backlog maintenance. + * + * The hardware is limited in the maximum size of data that it can + * process from a single BD. Typically this is fairly large (32MB) + * but still requires the complexity of splitting the incoming + * skreq up into a number of elements complete with appropriate + * iv chaining. + */ +struct sec_request { + struct list_head elements; + int num_elements; + struct mutex lock; + struct sec_alg_tfm_ctx *tfm_ctx; + int len_in; + int len_out; + dma_addr_t dma_iv; + int err; + struct crypto_async_request *req_base; + void (*cb)(struct sec_bd_info *resp, struct crypto_async_request *req); + struct list_head backlog_head; +}; + +/** + * struct sec_request_el - A subpart of a request. + * @head: allow us to attach this to the list in the sec_request + * @req: hardware block descriptor corresponding to this request subpart + * @in: hardware sgl for input - virtual address + * @dma_in: hardware sgl for input - physical address + * @sgl_in: scatterlist for this request subpart + * @out: hardware sgl for output - virtual address + * @dma_out: hardware sgl for output - physical address + * @sgl_out: scatterlist for this request subpart + * @sec_req: The request which this subpart forms a part of + * @el_length: Number of bytes in this subpart. Needed to locate + * last ivsize chunk for iv chaining. + */ +struct sec_request_el { + struct list_head head; + struct sec_bd_info req; + struct sec_hw_sgl *in; + dma_addr_t dma_in; + struct scatterlist *sgl_in; + struct sec_hw_sgl *out; + dma_addr_t dma_out; + struct scatterlist *sgl_out; + struct sec_request *sec_req; + size_t el_length; +}; + +/** + * struct sec_queue - All the information about a HW queue + * @dev_info: The parent SEC device to which this queue belongs. + * @task_irq: Completion interrupt for the queue. + * @name: Human readable queue description also used as irq name. + * @ring: The several HW rings associated with one queue. + * @regs: The iomapped device registers + * @queue_id: Index of the queue used for naming and resource selection. + * @in_use: Flag to say if the queue is in use. + * @expected: The next expected element to finish assuming we were in order. + * @uprocessed: A bitmap to track which OoO elements are done but not handled. + * @softqueue: A software queue used when chaining requirements prevent direct + * use of the hardware queues. + * @havesoftqueue: A flag to say we have a queues - as we may need one for the + * current mode. + * @queuelock: Protect the soft queue from concurrent changes to avoid some + * potential loss of data races. + * @shadow: Pointers back to the shadow copy of the hardware ring element + * need because we can't store any context reference in the bd element. + */ +struct sec_queue { + struct sec_dev_info *dev_info; + int task_irq; + char name[SEC_NAME_SIZE]; + struct sec_queue_ring_cmd ring_cmd; + struct sec_queue_ring_cq ring_cq; + struct sec_queue_ring_db ring_db; + void __iomem *regs; + u32 queue_id; + bool in_use; + int expected; + + DECLARE_BITMAP(unprocessed, SEC_QUEUE_LEN); + DECLARE_KFIFO_PTR(softqueue, typeof(struct sec_request_el *)); + bool havesoftqueue; + spinlock_t queuelock; + void *shadow[SEC_QUEUE_LEN]; +}; + +/** + * struct sec_hw_sge: Track each of the 64 element SEC HW SGL entries + * @buf: The IOV dma address for this entry. + * @len: Length of this IOV. + * @pad: Reserved space. + */ +struct sec_hw_sge { + dma_addr_t buf; + unsigned int len; + unsigned int pad; +}; + +/** + * struct sec_hw_sgl: One hardware SGL entry. + * @next_sgl: The next entry if we need to chain dma address. Null if last. + * @entry_sum_in_chain: The full count of SGEs - only matters for first SGL. + * @entry_sum_in_sgl: The number of SGEs in this SGL element. + * @flag: Unused in skciphers. + * @serial_num: Unsued in skciphers. + * @cpuid: Currently unused. + * @data_bytes_in_sgl: Count of bytes from all SGEs in this SGL. + * @next: Virtual address used to stash the next sgl - useful in completion. + * @reserved: A reserved field not currently used. + * @sge_entries: The (up to) 64 Scatter Gather Entries, representing IOVs. + * @node: Currently unused. + */ +struct sec_hw_sgl { + dma_addr_t next_sgl; + u16 entry_sum_in_chain; + u16 entry_sum_in_sgl; + u32 flag; + u64 serial_num; + u32 cpuid; + u32 data_bytes_in_sgl; + struct sec_hw_sgl *next; + u64 reserved; + struct sec_hw_sge sge_entries[SEC_MAX_SGE_NUM]; + u8 node[16]; +}; + +struct dma_pool; + +/** + * struct sec_dev_info: The full SEC unit comprising queues and processors. + * @sec_id: Index used to track which SEC this is when more than one is present. + * @num_saas: The number of backed processors enabled. + * @regs: iomapped register regions shared by whole SEC unit. + * @dev_lock: Protects concurrent queue allocation / freeing for the SEC. + * @queues: The 16 queues that this SEC instance provides. + * @dev: Device pointer. + * @hw_sgl_pool: DMA pool used to mimise mapping for the scatter gather lists. + */ +struct sec_dev_info { + int sec_id; + int num_saas; + void __iomem *regs[SEC_NUM_ADDR_REGIONS]; + struct mutex dev_lock; + int queues_in_use; + struct sec_queue queues[SEC_Q_NUM]; + struct device *dev; + struct dma_pool *hw_sgl_pool; +}; + +int sec_queue_send(struct sec_queue *queue, struct sec_bd_info *msg, void *ctx); +bool sec_queue_can_enqueue(struct sec_queue *queue, int num); +int sec_queue_stop_release(struct sec_queue *queue); +struct sec_queue *sec_queue_alloc_start_safe(void); +bool sec_queue_empty(struct sec_queue *queue); + +/* Algorithm specific elements from sec_algs.c */ +void sec_alg_callback(struct sec_bd_info *resp, void *ctx); +int sec_algs_register(void); +void sec_algs_unregister(void); + +#endif /* _SEC_DRV_H_ */ diff --git a/drivers/crypto/hisilicon/sec2/Makefile b/drivers/crypto/hisilicon/sec2/Makefile new file mode 100644 index 000000000..b4f6cf14b --- /dev/null +++ b/drivers/crypto/hisilicon/sec2/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_CRYPTO_DEV_HISI_SEC2) += hisi_sec2.o +hisi_sec2-objs = sec_main.o sec_crypto.o diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h new file mode 100644 index 000000000..c12a39a8a --- /dev/null +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -0,0 +1,286 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2019 HiSilicon Limited. */ + +#ifndef __HISI_SEC_V2_H +#define __HISI_SEC_V2_H + +#include +#include "sec_crypto.h" + +#define SEC_PBUF_SZ 512 +#define SEC_MAX_MAC_LEN 64 +#define SEC_IV_SIZE 24 +#define SEC_SGE_NR_NUM 4 +#define SEC_SGL_ALIGN_SIZE 64 + +/* Algorithm resource per hardware SEC queue */ +struct sec_alg_res { + u8 *pbuf; + dma_addr_t pbuf_dma; + u8 *c_ivin; + dma_addr_t c_ivin_dma; + u8 *a_ivin; + dma_addr_t a_ivin_dma; + u8 *out_mac; + dma_addr_t out_mac_dma; + u16 depth; +}; + +struct sec_hw_sge { + dma_addr_t buf; + void *page_ctrl; + __le32 len; + __le32 pad; + __le32 pad0; + __le32 pad1; +}; + +struct sec_hw_sgl { + dma_addr_t next_dma; + __le16 entry_sum_in_chain; + __le16 entry_sum_in_sgl; + __le16 entry_length_in_sgl; + __le16 pad0; + __le64 pad1[5]; + struct sec_hw_sgl *next; + struct sec_hw_sge sge_entries[SEC_SGE_NR_NUM]; +} __aligned(SEC_SGL_ALIGN_SIZE); + +struct sec_src_dst_buf { + struct sec_hw_sgl in; + struct sec_hw_sgl out; +}; + +struct sec_request_buf { + union { + struct sec_src_dst_buf data_buf; + __u8 pbuf[SEC_PBUF_SZ]; + }; + dma_addr_t in_dma; + dma_addr_t out_dma; +}; + +/* Cipher request of SEC private */ +struct sec_cipher_req { + struct hisi_acc_hw_sgl *c_out; + dma_addr_t c_out_dma; + u8 *c_ivin; + dma_addr_t c_ivin_dma; + struct skcipher_request *sk_req; + u32 c_len; + bool encrypt; + __u8 c_ivin_buf[SEC_IV_SIZE]; +}; + +struct sec_aead_req { + u8 *out_mac; + dma_addr_t out_mac_dma; + u8 *a_ivin; + dma_addr_t a_ivin_dma; + struct aead_request *aead_req; + __u8 a_ivin_buf[SEC_IV_SIZE]; + __u8 out_mac_buf[SEC_MAX_MAC_LEN]; +}; + +/* SEC request of Crypto */ +struct sec_req { + union { + struct sec_sqe sec_sqe; + struct sec_sqe3 sec_sqe3; + }; + struct sec_ctx *ctx; + struct sec_qp_ctx *qp_ctx; + + /** + * Common parameter of the SEC request. + */ + struct hisi_acc_hw_sgl *in; + dma_addr_t in_dma; + struct sec_cipher_req c_req; + struct sec_aead_req aead_req; + struct crypto_async_request *base; + + int err_type; + int req_id; + u32 flag; + + bool use_pbuf; + + struct list_head list; + struct sec_request_buf buf; +}; + +/** + * struct sec_req_op - Operations for SEC request + * @buf_map: DMA map the SGL buffers of the request + * @buf_unmap: DMA unmap the SGL buffers of the request + * @bd_fill: Fill the SEC queue BD + * @bd_send: Send the SEC BD into the hardware queue + * @callback: Call back for the request + * @process: Main processing logic of Skcipher + */ +struct sec_req_op { + int (*buf_map)(struct sec_ctx *ctx, struct sec_req *req); + void (*buf_unmap)(struct sec_ctx *ctx, struct sec_req *req); + void (*do_transfer)(struct sec_ctx *ctx, struct sec_req *req); + int (*bd_fill)(struct sec_ctx *ctx, struct sec_req *req); + int (*bd_send)(struct sec_ctx *ctx, struct sec_req *req); + void (*callback)(struct sec_ctx *ctx, struct sec_req *req, int err); + int (*process)(struct sec_ctx *ctx, struct sec_req *req); +}; + +/* SEC auth context */ +struct sec_auth_ctx { + dma_addr_t a_key_dma; + u8 *a_key; + u8 a_key_len; + u8 a_alg; + struct crypto_shash *hash_tfm; + struct crypto_aead *fallback_aead_tfm; +}; + +/* SEC cipher context which cipher's relatives */ +struct sec_cipher_ctx { + u8 *c_key; + dma_addr_t c_key_dma; + sector_t iv_offset; + u32 c_gran_size; + u32 ivsize; + u8 c_mode; + u8 c_alg; + u8 c_key_len; + + /* add software support */ + bool fallback; + struct crypto_sync_skcipher *fbtfm; +}; + +/* SEC queue context which defines queue's relatives */ +struct sec_qp_ctx { + struct hisi_qp *qp; + struct sec_req **req_list; + struct idr req_idr; + struct sec_alg_res *res; + struct sec_ctx *ctx; + spinlock_t req_lock; + spinlock_t id_lock; + struct hisi_acc_sgl_pool *c_in_pool; + struct hisi_acc_sgl_pool *c_out_pool; + u16 send_head; +}; + +enum sec_alg_type { + SEC_SKCIPHER, + SEC_AEAD +}; + +/* SEC Crypto TFM context which defines queue and cipher .etc relatives */ +struct sec_ctx { + struct sec_qp_ctx *qp_ctx; + struct sec_dev *sec; + const struct sec_req_op *req_op; + struct hisi_qp **qps; + + /* Current cyclic index to select a queue for encipher */ + atomic_t enc_qcyclic; + + /* Current cyclic index to select a queue for decipher */ + atomic_t dec_qcyclic; + + enum sec_alg_type alg_type; + bool pbuf_supported; + struct sec_cipher_ctx c_ctx; + struct sec_auth_ctx a_ctx; + u8 type_supported; + struct device *dev; +}; + + +enum sec_debug_file_index { + SEC_CLEAR_ENABLE, + SEC_DEBUG_FILE_NUM, +}; + +struct sec_debug_file { + enum sec_debug_file_index index; + spinlock_t lock; + struct hisi_qm *qm; +}; + +struct sec_dfx { + atomic64_t send_cnt; + atomic64_t recv_cnt; + atomic64_t send_busy_cnt; + atomic64_t recv_busy_cnt; + atomic64_t err_bd_cnt; + atomic64_t invalid_req_cnt; + atomic64_t done_flag_cnt; +}; + +struct sec_debug { + struct sec_dfx dfx; + struct sec_debug_file files[SEC_DEBUG_FILE_NUM]; +}; + +struct sec_dev { + struct hisi_qm qm; + struct sec_debug debug; + u32 ctx_q_num; + bool iommu_used; +}; + +enum sec_cap_type { + SEC_QM_NFE_MASK_CAP = 0x0, + SEC_QM_RESET_MASK_CAP, + SEC_QM_OOO_SHUTDOWN_MASK_CAP, + SEC_QM_CE_MASK_CAP, + SEC_NFE_MASK_CAP, + SEC_RESET_MASK_CAP, + SEC_OOO_SHUTDOWN_MASK_CAP, + SEC_CE_MASK_CAP, + SEC_CLUSTER_NUM_CAP, + SEC_CORE_TYPE_NUM_CAP, + SEC_CORE_NUM_CAP, + SEC_CORES_PER_CLUSTER_NUM_CAP, + SEC_CORE_ENABLE_BITMAP, + SEC_DRV_ALG_BITMAP_LOW, + SEC_DRV_ALG_BITMAP_HIGH, + SEC_DEV_ALG_BITMAP_LOW, + SEC_DEV_ALG_BITMAP_HIGH, + SEC_CORE1_ALG_BITMAP_LOW, + SEC_CORE1_ALG_BITMAP_HIGH, + SEC_CORE2_ALG_BITMAP_LOW, + SEC_CORE2_ALG_BITMAP_HIGH, + SEC_CORE3_ALG_BITMAP_LOW, + SEC_CORE3_ALG_BITMAP_HIGH, + SEC_CORE4_ALG_BITMAP_LOW, + SEC_CORE4_ALG_BITMAP_HIGH, +}; + +enum sec_cap_table_type { + QM_RAS_NFE_TYPE = 0x0, + QM_RAS_NFE_RESET, + QM_RAS_CE_TYPE, + SEC_RAS_NFE_TYPE, + SEC_RAS_NFE_RESET, + SEC_RAS_CE_TYPE, + SEC_CORE_INFO, + SEC_CORE_EN, + SEC_DRV_ALG_BITMAP_LOW_TB, + SEC_DRV_ALG_BITMAP_HIGH_TB, + SEC_ALG_BITMAP_LOW, + SEC_ALG_BITMAP_HIGH, + SEC_CORE1_BITMAP_LOW, + SEC_CORE1_BITMAP_HIGH, + SEC_CORE2_BITMAP_LOW, + SEC_CORE2_BITMAP_HIGH, + SEC_CORE3_BITMAP_LOW, + SEC_CORE3_BITMAP_HIGH, + SEC_CORE4_BITMAP_LOW, + SEC_CORE4_BITMAP_HIGH, +}; + +void sec_destroy_qps(struct hisi_qp **qps, int qp_num); +struct hisi_qp **sec_create_qps(void); +u64 sec_get_alg_bitmap(struct hisi_qm *qm, u32 high, u32 low); +#endif diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c new file mode 100644 index 000000000..0a2f7c8b4 --- /dev/null +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -0,0 +1,2780 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2019 HiSilicon Limited. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sec.h" +#include "sec_crypto.h" + +#define SEC_PRIORITY 80 +#define SEC_XTS_MIN_KEY_SIZE (2 * AES_MIN_KEY_SIZE) +#define SEC_XTS_MID_KEY_SIZE (3 * AES_MIN_KEY_SIZE) +#define SEC_XTS_MAX_KEY_SIZE (2 * AES_MAX_KEY_SIZE) +#define SEC_DES3_2KEY_SIZE (2 * DES_KEY_SIZE) +#define SEC_DES3_3KEY_SIZE (3 * DES_KEY_SIZE) + +/* SEC sqe(bd) bit operational relative MACRO */ +#define SEC_DE_OFFSET 1 +#define SEC_CIPHER_OFFSET 4 +#define SEC_SCENE_OFFSET 3 +#define SEC_DST_SGL_OFFSET 2 +#define SEC_SRC_SGL_OFFSET 7 +#define SEC_CKEY_OFFSET 9 +#define SEC_CMODE_OFFSET 12 +#define SEC_AKEY_OFFSET 5 +#define SEC_AEAD_ALG_OFFSET 11 +#define SEC_AUTH_OFFSET 6 + +#define SEC_DE_OFFSET_V3 9 +#define SEC_SCENE_OFFSET_V3 5 +#define SEC_CKEY_OFFSET_V3 13 +#define SEC_CTR_CNT_OFFSET 25 +#define SEC_CTR_CNT_ROLLOVER 2 +#define SEC_SRC_SGL_OFFSET_V3 11 +#define SEC_DST_SGL_OFFSET_V3 14 +#define SEC_CALG_OFFSET_V3 4 +#define SEC_AKEY_OFFSET_V3 9 +#define SEC_MAC_OFFSET_V3 4 +#define SEC_AUTH_ALG_OFFSET_V3 15 +#define SEC_CIPHER_AUTH_V3 0xbf +#define SEC_AUTH_CIPHER_V3 0x40 +#define SEC_FLAG_OFFSET 7 +#define SEC_FLAG_MASK 0x0780 +#define SEC_DONE_MASK 0x0001 +#define SEC_ICV_MASK 0x000E + +#define SEC_TOTAL_IV_SZ(depth) (SEC_IV_SIZE * (depth)) +#define SEC_SGL_SGE_NR 128 +#define SEC_CIPHER_AUTH 0xfe +#define SEC_AUTH_CIPHER 0x1 +#define SEC_MAX_MAC_LEN 64 +#define SEC_MAX_AAD_LEN 65535 +#define SEC_MAX_CCM_AAD_LEN 65279 +#define SEC_TOTAL_MAC_SZ(depth) (SEC_MAX_MAC_LEN * (depth)) + +#define SEC_PBUF_IV_OFFSET SEC_PBUF_SZ +#define SEC_PBUF_MAC_OFFSET (SEC_PBUF_SZ + SEC_IV_SIZE) +#define SEC_PBUF_PKG (SEC_PBUF_SZ + SEC_IV_SIZE + \ + SEC_MAX_MAC_LEN * 2) +#define SEC_PBUF_NUM (PAGE_SIZE / SEC_PBUF_PKG) +#define SEC_PBUF_PAGE_NUM(depth) ((depth) / SEC_PBUF_NUM) +#define SEC_PBUF_LEFT_SZ(depth) (SEC_PBUF_PKG * ((depth) - \ + SEC_PBUF_PAGE_NUM(depth) * SEC_PBUF_NUM)) +#define SEC_TOTAL_PBUF_SZ(depth) (PAGE_SIZE * SEC_PBUF_PAGE_NUM(depth) + \ + SEC_PBUF_LEFT_SZ(depth)) + +#define SEC_SQE_CFLAG 2 +#define SEC_SQE_AEAD_FLAG 3 +#define SEC_SQE_DONE 0x1 +#define SEC_ICV_ERR 0x2 +#define MAC_LEN_MASK 0x1U +#define MAX_INPUT_DATA_LEN 0xFFFE00 +#define BITS_MASK 0xFF +#define WORD_MASK 0x3 +#define BYTE_BITS 0x8 +#define BYTES_TO_WORDS(bcount) ((bcount) >> 2) +#define SEC_XTS_NAME_SZ 0x3 +#define IV_CM_CAL_NUM 2 +#define IV_CL_MASK 0x7 +#define IV_CL_MIN 2 +#define IV_CL_MID 4 +#define IV_CL_MAX 8 +#define IV_FLAGS_OFFSET 0x6 +#define IV_CM_OFFSET 0x3 +#define IV_LAST_BYTE1 1 +#define IV_LAST_BYTE_MASK 0xFF +#define IV_CTR_INIT 0x1 +#define IV_BYTE_OFFSET 0x8 +#define SEC_GCM_MIN_AUTH_SZ 0x8 +#define SEC_RETRY_MAX_CNT 5U + +static DEFINE_MUTEX(sec_algs_lock); +static unsigned int sec_available_devs; + +struct sec_skcipher { + u64 alg_msk; + struct skcipher_alg alg; +}; + +struct sec_aead { + u64 alg_msk; + struct aead_alg alg; +}; + +static int sec_aead_soft_crypto(struct sec_ctx *ctx, + struct aead_request *aead_req, + bool encrypt); +static int sec_skcipher_soft_crypto(struct sec_ctx *ctx, + struct skcipher_request *sreq, bool encrypt); + +static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx) +{ + int req_id; + + spin_lock_bh(&qp_ctx->id_lock); + req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL, 0, qp_ctx->qp->sq_depth, GFP_ATOMIC); + spin_unlock_bh(&qp_ctx->id_lock); + return req_id; +} + +static void sec_free_req_id(struct sec_req *req) +{ + struct sec_qp_ctx *qp_ctx = req->qp_ctx; + int req_id = req->req_id; + + if (unlikely(req_id < 0 || req_id >= qp_ctx->qp->sq_depth)) { + dev_err(req->ctx->dev, "free request id invalid!\n"); + return; + } + + spin_lock_bh(&qp_ctx->id_lock); + idr_remove(&qp_ctx->req_idr, req_id); + spin_unlock_bh(&qp_ctx->id_lock); +} + +static void pre_parse_finished_bd(struct bd_status *status, void *resp) +{ + struct sec_sqe *bd = resp; + + status->done = le16_to_cpu(bd->type2.done_flag) & SEC_DONE_MASK; + status->icv = (le16_to_cpu(bd->type2.done_flag) & SEC_ICV_MASK) >> 1; + status->flag = (le16_to_cpu(bd->type2.done_flag) & + SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; + status->tag = le16_to_cpu(bd->type2.tag); + status->err_type = bd->type2.error_type; +} + +static void pre_parse_finished_bd3(struct bd_status *status, void *resp) +{ + struct sec_sqe3 *bd3 = resp; + + status->done = le16_to_cpu(bd3->done_flag) & SEC_DONE_MASK; + status->icv = (le16_to_cpu(bd3->done_flag) & SEC_ICV_MASK) >> 1; + status->flag = (le16_to_cpu(bd3->done_flag) & + SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; + status->tag = le64_to_cpu(bd3->tag); + status->err_type = bd3->error_type; +} + +static int sec_cb_status_check(struct sec_req *req, + struct bd_status *status) +{ + struct sec_ctx *ctx = req->ctx; + + if (unlikely(req->err_type || status->done != SEC_SQE_DONE)) { + dev_err_ratelimited(ctx->dev, "err_type[%d], done[%u]\n", + req->err_type, status->done); + return -EIO; + } + + if (unlikely(ctx->alg_type == SEC_SKCIPHER)) { + if (unlikely(status->flag != SEC_SQE_CFLAG)) { + dev_err_ratelimited(ctx->dev, "flag[%u]\n", + status->flag); + return -EIO; + } + } else if (unlikely(ctx->alg_type == SEC_AEAD)) { + if (unlikely(status->flag != SEC_SQE_AEAD_FLAG || + status->icv == SEC_ICV_ERR)) { + dev_err_ratelimited(ctx->dev, + "flag[%u], icv[%u]\n", + status->flag, status->icv); + return -EBADMSG; + } + } + + return 0; +} + +static int qp_send_message(struct sec_req *req) +{ + struct sec_qp_ctx *qp_ctx = req->qp_ctx; + int ret; + + if (atomic_read(&qp_ctx->qp->qp_status.used) == qp_ctx->qp->sq_depth - 1) + return -EBUSY; + + spin_lock_bh(&qp_ctx->req_lock); + if (atomic_read(&qp_ctx->qp->qp_status.used) == qp_ctx->qp->sq_depth - 1) { + spin_unlock_bh(&qp_ctx->req_lock); + return -EBUSY; + } + + if (qp_ctx->ctx->type_supported == SEC_BD_TYPE2) { + req->sec_sqe.type2.tag = cpu_to_le16((u16)qp_ctx->send_head); + qp_ctx->req_list[qp_ctx->send_head] = req; + } + + ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe); + if (ret) { + spin_unlock_bh(&qp_ctx->req_lock); + return ret; + } + if (qp_ctx->ctx->type_supported == SEC_BD_TYPE2) + qp_ctx->send_head = (qp_ctx->send_head + 1) % qp_ctx->qp->sq_depth; + + spin_unlock_bh(&qp_ctx->req_lock); + + atomic64_inc(&qp_ctx->ctx->sec->debug.dfx.send_cnt); + return -EINPROGRESS; +} + +static void sec_alg_send_backlog_soft(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) +{ + struct sec_req *req, *tmp; + int ret; + + list_for_each_entry_safe(req, tmp, &qp_ctx->qp->backlog.list, list) { + list_del(&req->list); + ctx->req_op->buf_unmap(ctx, req); + if (req->req_id >= 0) + sec_free_req_id(req); + + if (ctx->alg_type == SEC_AEAD) + ret = sec_aead_soft_crypto(ctx, req->aead_req.aead_req, + req->c_req.encrypt); + else + ret = sec_skcipher_soft_crypto(ctx, req->c_req.sk_req, + req->c_req.encrypt); + + /* Wake up the busy thread first, then return the errno. */ + crypto_request_complete(req->base, -EINPROGRESS); + crypto_request_complete(req->base, ret); + } +} + +static void sec_alg_send_backlog(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) +{ + struct hisi_qp *qp = qp_ctx->qp; + struct sec_req *req, *tmp; + int ret; + + spin_lock_bh(&qp->backlog.lock); + list_for_each_entry_safe(req, tmp, &qp->backlog.list, list) { + ret = qp_send_message(req); + switch (ret) { + case -EINPROGRESS: + list_del(&req->list); + crypto_request_complete(req->base, -EINPROGRESS); + break; + case -EBUSY: + /* Device is busy and stop send any request. */ + goto unlock; + default: + /* Release memory resources and send all requests through software. */ + sec_alg_send_backlog_soft(ctx, qp_ctx); + goto unlock; + } + } + +unlock: + spin_unlock_bh(&qp->backlog.lock); +} + +static void sec_req_cb(struct hisi_qp *qp, void *resp) +{ + const struct sec_sqe *sqe = qp->msg[qp->qp_status.cq_head]; + struct sec_req *req = container_of(sqe, struct sec_req, sec_sqe); + struct sec_ctx *ctx = req->ctx; + struct sec_dfx *dfx = &ctx->sec->debug.dfx; + struct bd_status status; + int err; + + pre_parse_finished_bd(&status, resp); + + req->err_type = status.err_type; + err = sec_cb_status_check(req, &status); + if (err) + atomic64_inc(&dfx->done_flag_cnt); + + atomic64_inc(&dfx->recv_cnt); + + ctx->req_op->buf_unmap(ctx, req); + ctx->req_op->callback(ctx, req, err); +} + +static void sec_req_cb3(struct hisi_qp *qp, void *resp) +{ + struct bd_status status; + struct sec_ctx *ctx; + struct sec_dfx *dfx; + struct sec_req *req; + int err; + + pre_parse_finished_bd3(&status, resp); + + req = (void *)(uintptr_t)status.tag; + req->err_type = status.err_type; + ctx = req->ctx; + dfx = &ctx->sec->debug.dfx; + + err = sec_cb_status_check(req, &status); + if (err) + atomic64_inc(&dfx->done_flag_cnt); + + atomic64_inc(&dfx->recv_cnt); + + ctx->req_op->buf_unmap(ctx, req); + ctx->req_op->callback(ctx, req, err); +} + +static int sec_alg_send_message_retry(struct sec_req *req) +{ + int ctr = 0; + int ret; + + do { + ret = qp_send_message(req); + } while (ret == -EBUSY && ctr++ < SEC_RETRY_MAX_CNT); + + return ret; +} + +static int sec_alg_try_enqueue(struct sec_req *req) +{ + struct hisi_qp *qp = req->qp_ctx->qp; + + /* Check if any request is already backlogged */ + if (!list_empty(&qp->backlog.list)) + return -EBUSY; + + /* Try to enqueue to HW ring */ + return qp_send_message(req); +} + + +static int sec_alg_send_message_maybacklog(struct sec_req *req) +{ + struct hisi_qp *qp = req->qp_ctx->qp; + int ret; + + ret = sec_alg_try_enqueue(req); + if (ret != -EBUSY) + return ret; + + spin_lock_bh(&qp->backlog.lock); + ret = sec_alg_try_enqueue(req); + if (ret == -EBUSY) + list_add_tail(&req->list, &qp->backlog.list); + spin_unlock_bh(&qp->backlog.lock); + + return ret; +} + +static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req) +{ + if (req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG) + return sec_alg_send_message_maybacklog(req); + + return sec_alg_send_message_retry(req); +} + +static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res) +{ + u16 q_depth = res->depth; + int i; + + res->c_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ(q_depth), + &res->c_ivin_dma, GFP_KERNEL); + if (!res->c_ivin) + return -ENOMEM; + + for (i = 1; i < q_depth; i++) { + res[i].c_ivin_dma = res->c_ivin_dma + i * SEC_IV_SIZE; + res[i].c_ivin = res->c_ivin + i * SEC_IV_SIZE; + } + + return 0; +} + +static void sec_free_civ_resource(struct device *dev, struct sec_alg_res *res) +{ + if (res->c_ivin) + dma_free_coherent(dev, SEC_TOTAL_IV_SZ(res->depth), + res->c_ivin, res->c_ivin_dma); +} + +static int sec_alloc_aiv_resource(struct device *dev, struct sec_alg_res *res) +{ + u16 q_depth = res->depth; + int i; + + res->a_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ(q_depth), + &res->a_ivin_dma, GFP_KERNEL); + if (!res->a_ivin) + return -ENOMEM; + + for (i = 1; i < q_depth; i++) { + res[i].a_ivin_dma = res->a_ivin_dma + i * SEC_IV_SIZE; + res[i].a_ivin = res->a_ivin + i * SEC_IV_SIZE; + } + + return 0; +} + +static void sec_free_aiv_resource(struct device *dev, struct sec_alg_res *res) +{ + if (res->a_ivin) + dma_free_coherent(dev, SEC_TOTAL_IV_SZ(res->depth), + res->a_ivin, res->a_ivin_dma); +} + +static int sec_alloc_mac_resource(struct device *dev, struct sec_alg_res *res) +{ + u16 q_depth = res->depth; + int i; + + res->out_mac = dma_alloc_coherent(dev, SEC_TOTAL_MAC_SZ(q_depth) << 1, + &res->out_mac_dma, GFP_KERNEL); + if (!res->out_mac) + return -ENOMEM; + + for (i = 1; i < q_depth; i++) { + res[i].out_mac_dma = res->out_mac_dma + + i * (SEC_MAX_MAC_LEN << 1); + res[i].out_mac = res->out_mac + i * (SEC_MAX_MAC_LEN << 1); + } + + return 0; +} + +static void sec_free_mac_resource(struct device *dev, struct sec_alg_res *res) +{ + if (res->out_mac) + dma_free_coherent(dev, SEC_TOTAL_MAC_SZ(res->depth) << 1, + res->out_mac, res->out_mac_dma); +} + +static void sec_free_pbuf_resource(struct device *dev, struct sec_alg_res *res) +{ + if (res->pbuf) + dma_free_coherent(dev, SEC_TOTAL_PBUF_SZ(res->depth), + res->pbuf, res->pbuf_dma); +} + +/* + * To improve performance, pbuffer is used for + * small packets (< 512Bytes) as IOMMU translation using. + */ +static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res) +{ + u16 q_depth = res->depth; + int size = SEC_PBUF_PAGE_NUM(q_depth); + int pbuf_page_offset; + int i, j, k; + + res->pbuf = dma_alloc_coherent(dev, SEC_TOTAL_PBUF_SZ(q_depth), + &res->pbuf_dma, GFP_KERNEL); + if (!res->pbuf) + return -ENOMEM; + + /* + * SEC_PBUF_PKG contains data pbuf, iv and + * out_mac : + * Every PAGE contains six SEC_PBUF_PKG + * The sec_qp_ctx contains QM_Q_DEPTH numbers of SEC_PBUF_PKG + * So we need SEC_PBUF_PAGE_NUM numbers of PAGE + * for the SEC_TOTAL_PBUF_SZ + */ + for (i = 0; i <= size; i++) { + pbuf_page_offset = PAGE_SIZE * i; + for (j = 0; j < SEC_PBUF_NUM; j++) { + k = i * SEC_PBUF_NUM + j; + if (k == q_depth) + break; + res[k].pbuf = res->pbuf + + j * SEC_PBUF_PKG + pbuf_page_offset; + res[k].pbuf_dma = res->pbuf_dma + + j * SEC_PBUF_PKG + pbuf_page_offset; + } + } + + return 0; +} + +static int sec_alg_resource_alloc(struct sec_ctx *ctx, + struct sec_qp_ctx *qp_ctx) +{ + struct sec_alg_res *res = qp_ctx->res; + struct device *dev = ctx->dev; + int ret; + + ret = sec_alloc_civ_resource(dev, res); + if (ret) + return ret; + + if (ctx->alg_type == SEC_AEAD) { + ret = sec_alloc_aiv_resource(dev, res); + if (ret) + goto alloc_aiv_fail; + + ret = sec_alloc_mac_resource(dev, res); + if (ret) + goto alloc_mac_fail; + } + if (ctx->pbuf_supported) { + ret = sec_alloc_pbuf_resource(dev, res); + if (ret) { + dev_err(dev, "fail to alloc pbuf dma resource!\n"); + goto alloc_pbuf_fail; + } + } + + return 0; + +alloc_pbuf_fail: + if (ctx->alg_type == SEC_AEAD) + sec_free_mac_resource(dev, qp_ctx->res); +alloc_mac_fail: + if (ctx->alg_type == SEC_AEAD) + sec_free_aiv_resource(dev, res); +alloc_aiv_fail: + sec_free_civ_resource(dev, res); + return ret; +} + +static void sec_alg_resource_free(struct sec_ctx *ctx, + struct sec_qp_ctx *qp_ctx) +{ + struct device *dev = ctx->dev; + + sec_free_civ_resource(dev, qp_ctx->res); + + if (ctx->pbuf_supported) + sec_free_pbuf_resource(dev, qp_ctx->res); + if (ctx->alg_type == SEC_AEAD) { + sec_free_mac_resource(dev, qp_ctx->res); + sec_free_aiv_resource(dev, qp_ctx->res); + } +} + +static int sec_alloc_qp_ctx_resource(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) +{ + u16 q_depth = qp_ctx->qp->sq_depth; + struct device *dev = ctx->dev; + int ret = -ENOMEM; + + qp_ctx->req_list = kzalloc_objs(struct sec_req *, q_depth); + if (!qp_ctx->req_list) + return ret; + + qp_ctx->res = kzalloc_objs(struct sec_alg_res, q_depth); + if (!qp_ctx->res) + goto err_free_req_list; + qp_ctx->res->depth = q_depth; + + qp_ctx->c_in_pool = hisi_acc_create_sgl_pool(dev, q_depth, SEC_SGL_SGE_NR); + if (IS_ERR(qp_ctx->c_in_pool)) { + dev_err(dev, "fail to create sgl pool for input!\n"); + goto err_free_res; + } + + qp_ctx->c_out_pool = hisi_acc_create_sgl_pool(dev, q_depth, SEC_SGL_SGE_NR); + if (IS_ERR(qp_ctx->c_out_pool)) { + dev_err(dev, "fail to create sgl pool for output!\n"); + goto err_free_c_in_pool; + } + + ret = sec_alg_resource_alloc(ctx, qp_ctx); + if (ret) + goto err_free_c_out_pool; + + return 0; + +err_free_c_out_pool: + hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool); +err_free_c_in_pool: + hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); +err_free_res: + kfree(qp_ctx->res); +err_free_req_list: + kfree(qp_ctx->req_list); + return ret; +} + +static void sec_free_qp_ctx_resource(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) +{ + struct device *dev = ctx->dev; + + sec_alg_resource_free(ctx, qp_ctx); + hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool); + hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); + kfree(qp_ctx->res); + kfree(qp_ctx->req_list); +} + +static int sec_create_qp_ctx(struct sec_ctx *ctx, int qp_ctx_id) +{ + struct sec_qp_ctx *qp_ctx; + struct hisi_qp *qp; + int ret; + + qp_ctx = &ctx->qp_ctx[qp_ctx_id]; + qp = ctx->qps[qp_ctx_id]; + qp_ctx->qp = qp; + qp_ctx->ctx = ctx; + + if (ctx->type_supported == SEC_BD_TYPE3) + qp->req_cb = sec_req_cb3; + else + qp->req_cb = sec_req_cb; + + spin_lock_init(&qp_ctx->req_lock); + idr_init(&qp_ctx->req_idr); + spin_lock_init(&qp_ctx->id_lock); + qp_ctx->send_head = 0; + + ret = sec_alloc_qp_ctx_resource(ctx, qp_ctx); + if (ret) + goto err_destroy_idr; + + return 0; + +err_destroy_idr: + idr_destroy(&qp_ctx->req_idr); + return ret; +} + +static void sec_release_qp_ctx(struct sec_ctx *ctx, + struct sec_qp_ctx *qp_ctx) +{ + sec_free_qp_ctx_resource(ctx, qp_ctx); + idr_destroy(&qp_ctx->req_idr); +} + +static int sec_ctx_base_init(struct sec_ctx *ctx) +{ + struct sec_dev *sec; + int i, ret; + + ctx->qps = sec_create_qps(); + if (!ctx->qps) + return -ENODEV; + + sec = container_of(ctx->qps[0]->qm, struct sec_dev, qm); + ctx->sec = sec; + ctx->dev = &sec->qm.pdev->dev; + + ctx->pbuf_supported = ctx->sec->iommu_used; + if (sec->qm.ver < QM_HW_V3) + ctx->type_supported = SEC_BD_TYPE2; + else + ctx->type_supported = SEC_BD_TYPE3; + + ctx->qp_ctx = kzalloc_objs(struct sec_qp_ctx, sec->ctx_q_num); + if (!ctx->qp_ctx) { + ret = -ENOMEM; + goto err_destroy_qps; + } + + for (i = 0; i < sec->ctx_q_num; i++) { + ret = sec_create_qp_ctx(ctx, i); + if (ret) + goto err_sec_release_qp_ctx; + } + + return 0; + +err_sec_release_qp_ctx: + for (i = i - 1; i >= 0; i--) + sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); + kfree(ctx->qp_ctx); +err_destroy_qps: + sec_destroy_qps(ctx->qps, sec->ctx_q_num); + return ret; +} + +static void sec_ctx_base_uninit(struct sec_ctx *ctx) +{ + int i; + + if (!ctx->qps) + return; + + for (i = 0; i < ctx->sec->ctx_q_num; i++) + sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); + + sec_destroy_qps(ctx->qps, ctx->sec->ctx_q_num); + kfree(ctx->qp_ctx); +} + +static int sec_cipher_init(struct sec_ctx *ctx) +{ + struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + + if (!ctx->qps) + return 0; + + c_ctx->c_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE, + &c_ctx->c_key_dma, GFP_KERNEL); + if (!c_ctx->c_key) + return -ENOMEM; + + return 0; +} + +static void sec_cipher_uninit(struct sec_ctx *ctx) +{ + struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + + if (!ctx->qps) + return; + + memzero_explicit(c_ctx->c_key, SEC_MAX_KEY_SIZE); + dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE, + c_ctx->c_key, c_ctx->c_key_dma); +} + +static int sec_auth_init(struct sec_ctx *ctx) +{ + struct sec_auth_ctx *a_ctx = &ctx->a_ctx; + + a_ctx->a_key = dma_alloc_coherent(ctx->dev, SEC_MAX_AKEY_SIZE, + &a_ctx->a_key_dma, GFP_KERNEL); + if (!a_ctx->a_key) + return -ENOMEM; + + return 0; +} + +static void sec_auth_uninit(struct sec_ctx *ctx) +{ + struct sec_auth_ctx *a_ctx = &ctx->a_ctx; + + if (!ctx->qps) + return; + + memzero_explicit(a_ctx->a_key, SEC_MAX_AKEY_SIZE); + dma_free_coherent(ctx->dev, SEC_MAX_AKEY_SIZE, + a_ctx->a_key, a_ctx->a_key_dma); +} + +static int sec_skcipher_fbtfm_init(struct crypto_skcipher *tfm) +{ + const char *alg = crypto_tfm_alg_name(&tfm->base); + struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); + struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + + c_ctx->fallback = false; + + c_ctx->fbtfm = crypto_alloc_sync_skcipher(alg, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(c_ctx->fbtfm)) { + pr_err("failed to alloc fallback tfm for %s!\n", alg); + return PTR_ERR(c_ctx->fbtfm); + } + + return 0; +} + +static int sec_skcipher_init(struct crypto_skcipher *tfm) +{ + struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); + int ret; + + ctx->alg_type = SEC_SKCIPHER; + crypto_skcipher_set_reqsize_dma(tfm, sizeof(struct sec_req)); + ctx->c_ctx.ivsize = crypto_skcipher_ivsize(tfm); + if (ctx->c_ctx.ivsize > SEC_IV_SIZE) { + pr_err("get error skcipher iv size!\n"); + return -EINVAL; + } + + ret = sec_ctx_base_init(ctx); + if (ret && ret != -ENODEV) + return ret; + + ret = sec_cipher_init(ctx); + if (ret) + goto err_cipher_init; + + ret = sec_skcipher_fbtfm_init(tfm); + if (ret) + goto err_fbtfm_init; + + return 0; + +err_fbtfm_init: + sec_cipher_uninit(ctx); +err_cipher_init: + sec_ctx_base_uninit(ctx); + return ret; +} + +static void sec_skcipher_uninit(struct crypto_skcipher *tfm) +{ + struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (ctx->c_ctx.fbtfm) + crypto_free_sync_skcipher(ctx->c_ctx.fbtfm); + + sec_cipher_uninit(ctx); + sec_ctx_base_uninit(ctx); +} + +static int sec_skcipher_3des_setkey(struct crypto_skcipher *tfm, const u8 *key, const u32 keylen) +{ + struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); + struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + int ret; + + ret = verify_skcipher_des3_key(tfm, key); + if (ret) + return ret; + + switch (keylen) { + case SEC_DES3_2KEY_SIZE: + c_ctx->c_key_len = SEC_CKEY_3DES_2KEY; + break; + case SEC_DES3_3KEY_SIZE: + c_ctx->c_key_len = SEC_CKEY_3DES_3KEY; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx *c_ctx, + const u32 keylen, + const enum sec_cmode c_mode) +{ + if (c_mode == SEC_CMODE_XTS) { + switch (keylen) { + case SEC_XTS_MIN_KEY_SIZE: + c_ctx->c_key_len = SEC_CKEY_128BIT; + break; + case SEC_XTS_MID_KEY_SIZE: + c_ctx->fallback = true; + break; + case SEC_XTS_MAX_KEY_SIZE: + c_ctx->c_key_len = SEC_CKEY_256BIT; + break; + default: + pr_err("hisi_sec2: xts mode key error!\n"); + return -EINVAL; + } + } else { + if (c_ctx->c_alg == SEC_CALG_SM4 && + keylen != AES_KEYSIZE_128) { + pr_err("hisi_sec2: sm4 key error!\n"); + return -EINVAL; + } else { + switch (keylen) { + case AES_KEYSIZE_128: + c_ctx->c_key_len = SEC_CKEY_128BIT; + break; + case AES_KEYSIZE_192: + c_ctx->c_key_len = SEC_CKEY_192BIT; + break; + case AES_KEYSIZE_256: + c_ctx->c_key_len = SEC_CKEY_256BIT; + break; + default: + pr_err("hisi_sec2: aes key error!\n"); + return -EINVAL; + } + } + } + + return 0; +} + +static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, + const u32 keylen, const enum sec_calg c_alg, + const enum sec_cmode c_mode) +{ + struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); + struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + struct device *dev = ctx->dev; + int ret; + + if (!ctx->qps) + goto set_soft_key; + + if (c_mode == SEC_CMODE_XTS) { + ret = xts_verify_key(tfm, key, keylen); + if (ret) { + dev_err(dev, "xts mode key err!\n"); + return ret; + } + } + + c_ctx->c_alg = c_alg; + c_ctx->c_mode = c_mode; + + switch (c_alg) { + case SEC_CALG_3DES: + ret = sec_skcipher_3des_setkey(tfm, key, keylen); + break; + case SEC_CALG_AES: + case SEC_CALG_SM4: + ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode); + break; + default: + dev_err(dev, "sec c_alg err!\n"); + return -EINVAL; + } + + if (ret) { + dev_err(dev, "set sec key err!\n"); + return ret; + } + + memcpy(c_ctx->c_key, key, keylen); + +set_soft_key: + ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen); + if (ret) { + dev_err(dev, "failed to set fallback skcipher key!\n"); + return ret; + } + + return 0; +} + +#define GEN_SEC_SETKEY_FUNC(name, c_alg, c_mode) \ +static int sec_setkey_##name(struct crypto_skcipher *tfm, const u8 *key,\ + u32 keylen) \ +{ \ + return sec_skcipher_setkey(tfm, key, keylen, c_alg, c_mode); \ +} + +GEN_SEC_SETKEY_FUNC(aes_ecb, SEC_CALG_AES, SEC_CMODE_ECB) +GEN_SEC_SETKEY_FUNC(aes_cbc, SEC_CALG_AES, SEC_CMODE_CBC) +GEN_SEC_SETKEY_FUNC(aes_xts, SEC_CALG_AES, SEC_CMODE_XTS) +GEN_SEC_SETKEY_FUNC(aes_ctr, SEC_CALG_AES, SEC_CMODE_CTR) +GEN_SEC_SETKEY_FUNC(3des_ecb, SEC_CALG_3DES, SEC_CMODE_ECB) +GEN_SEC_SETKEY_FUNC(3des_cbc, SEC_CALG_3DES, SEC_CMODE_CBC) +GEN_SEC_SETKEY_FUNC(sm4_xts, SEC_CALG_SM4, SEC_CMODE_XTS) +GEN_SEC_SETKEY_FUNC(sm4_cbc, SEC_CALG_SM4, SEC_CMODE_CBC) +GEN_SEC_SETKEY_FUNC(sm4_ctr, SEC_CALG_SM4, SEC_CMODE_CTR) + +static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req, + struct scatterlist *src) +{ + struct aead_request *aead_req = req->aead_req.aead_req; + struct sec_cipher_req *c_req = &req->c_req; + struct sec_qp_ctx *qp_ctx = req->qp_ctx; + struct sec_request_buf *buf = &req->buf; + struct device *dev = ctx->dev; + int copy_size, pbuf_length; + int req_id = req->req_id; + struct crypto_aead *tfm; + u8 *mac_offset, *pbuf; + size_t authsize; + + if (ctx->alg_type == SEC_AEAD) + copy_size = aead_req->cryptlen + aead_req->assoclen; + else + copy_size = c_req->c_len; + + + pbuf = req->req_id < 0 ? buf->pbuf : qp_ctx->res[req_id].pbuf; + pbuf_length = sg_copy_to_buffer(src, sg_nents(src), pbuf, copy_size); + if (unlikely(pbuf_length != copy_size)) { + dev_err(dev, "copy src data to pbuf error!\n"); + return -EINVAL; + } + if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) { + tfm = crypto_aead_reqtfm(aead_req); + authsize = crypto_aead_authsize(tfm); + mac_offset = pbuf + copy_size - authsize; + memcpy(req->aead_req.out_mac, mac_offset, authsize); + } + + if (req->req_id < 0) { + buf->in_dma = dma_map_single(dev, buf->pbuf, SEC_PBUF_SZ, DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(dev, buf->in_dma))) + return -ENOMEM; + + buf->out_dma = buf->in_dma; + return 0; + } + + req->in_dma = qp_ctx->res[req_id].pbuf_dma; + c_req->c_out_dma = req->in_dma; + + return 0; +} + +static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req, + struct scatterlist *dst) +{ + struct aead_request *aead_req = req->aead_req.aead_req; + struct sec_cipher_req *c_req = &req->c_req; + struct sec_qp_ctx *qp_ctx = req->qp_ctx; + struct sec_request_buf *buf = &req->buf; + int copy_size, pbuf_length; + int req_id = req->req_id; + + if (ctx->alg_type == SEC_AEAD) + copy_size = c_req->c_len + aead_req->assoclen; + else + copy_size = c_req->c_len; + + if (req->req_id < 0) + pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), buf->pbuf, copy_size); + else + pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), qp_ctx->res[req_id].pbuf, + copy_size); + if (unlikely(pbuf_length != copy_size)) + dev_err(ctx->dev, "copy pbuf data to dst error!\n"); + + if (req->req_id < 0) + dma_unmap_single(ctx->dev, buf->in_dma, SEC_PBUF_SZ, DMA_BIDIRECTIONAL); +} + +static int sec_aead_mac_init(struct sec_aead_req *req) +{ + struct aead_request *aead_req = req->aead_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); + size_t authsize = crypto_aead_authsize(tfm); + struct scatterlist *sgl = aead_req->src; + u8 *mac_out = req->out_mac; + size_t copy_size; + off_t skip_size; + + /* Copy input mac */ + skip_size = aead_req->assoclen + aead_req->cryptlen - authsize; + copy_size = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac_out, authsize, skip_size); + if (unlikely(copy_size != authsize)) + return -EINVAL; + + return 0; +} + +static void fill_sg_to_hw_sge(struct scatterlist *sgl, struct sec_hw_sge *hw_sge) +{ + hw_sge->buf = sg_dma_address(sgl); + hw_sge->len = cpu_to_le32(sg_dma_len(sgl)); + hw_sge->page_ctrl = sg_virt(sgl); +} + +static int sec_cipher_to_hw_sgl(struct device *dev, struct scatterlist *src, + struct sec_hw_sgl *src_in, dma_addr_t *hw_sgl_dma, + int dma_dir) +{ + struct sec_hw_sge *curr_hw_sge = src_in->sge_entries; + u32 i, sg_n, sg_n_mapped; + struct scatterlist *sg; + u32 sge_var = 0; + + sg_n = sg_nents(src); + sg_n_mapped = dma_map_sg(dev, src, sg_n, dma_dir); + if (unlikely(!sg_n_mapped)) { + dev_err(dev, "dma mapping for SG error!\n"); + return -EINVAL; + } else if (unlikely(sg_n_mapped > SEC_SGE_NR_NUM)) { + dev_err(dev, "the number of entries in input scatterlist error!\n"); + dma_unmap_sg(dev, src, sg_n, dma_dir); + return -EINVAL; + } + + for_each_sg(src, sg, sg_n_mapped, i) { + fill_sg_to_hw_sge(sg, curr_hw_sge); + curr_hw_sge++; + sge_var++; + } + + src_in->entry_sum_in_sgl = cpu_to_le16(sge_var); + src_in->entry_sum_in_chain = cpu_to_le16(SEC_SGE_NR_NUM); + src_in->entry_length_in_sgl = cpu_to_le16(SEC_SGE_NR_NUM); + *hw_sgl_dma = dma_map_single(dev, src_in, sizeof(struct sec_hw_sgl), dma_dir); + if (unlikely(dma_mapping_error(dev, *hw_sgl_dma))) { + dma_unmap_sg(dev, src, sg_n, dma_dir); + return -ENOMEM; + } + + return 0; +} + +static void sec_cipher_put_hw_sgl(struct device *dev, struct scatterlist *src, + dma_addr_t src_in, int dma_dir) +{ + dma_unmap_single(dev, src_in, sizeof(struct sec_hw_sgl), dma_dir); + dma_unmap_sg(dev, src, sg_nents(src), dma_dir); +} + +static int sec_cipher_map_sgl(struct device *dev, struct sec_req *req, + struct scatterlist *src, struct scatterlist *dst) +{ + struct sec_hw_sgl *src_in = &req->buf.data_buf.in; + struct sec_hw_sgl *dst_out = &req->buf.data_buf.out; + int ret; + + if (dst == src) { + ret = sec_cipher_to_hw_sgl(dev, src, src_in, &req->buf.in_dma, + DMA_BIDIRECTIONAL); + req->buf.out_dma = req->buf.in_dma; + return ret; + } + + ret = sec_cipher_to_hw_sgl(dev, src, src_in, &req->buf.in_dma, DMA_TO_DEVICE); + if (unlikely(ret)) + return ret; + + ret = sec_cipher_to_hw_sgl(dev, dst, dst_out, &req->buf.out_dma, + DMA_FROM_DEVICE); + if (unlikely(ret)) { + sec_cipher_put_hw_sgl(dev, src, req->buf.in_dma, DMA_TO_DEVICE); + return ret; + } + + return 0; +} + +static int sec_cipher_map_inner(struct sec_ctx *ctx, struct sec_req *req, + struct scatterlist *src, struct scatterlist *dst) +{ + struct sec_cipher_req *c_req = &req->c_req; + struct sec_aead_req *a_req = &req->aead_req; + struct sec_qp_ctx *qp_ctx = req->qp_ctx; + struct sec_alg_res *res = &qp_ctx->res[req->req_id]; + struct device *dev = ctx->dev; + enum dma_data_direction src_direction; + int ret; + + if (req->use_pbuf) { + c_req->c_ivin = res->pbuf + SEC_PBUF_IV_OFFSET; + c_req->c_ivin_dma = res->pbuf_dma + SEC_PBUF_IV_OFFSET; + if (ctx->alg_type == SEC_AEAD) { + a_req->a_ivin = res->a_ivin; + a_req->a_ivin_dma = res->a_ivin_dma; + a_req->out_mac = res->pbuf + SEC_PBUF_MAC_OFFSET; + a_req->out_mac_dma = res->pbuf_dma + + SEC_PBUF_MAC_OFFSET; + } + return sec_cipher_pbuf_map(ctx, req, src); + } + + c_req->c_ivin = res->c_ivin; + c_req->c_ivin_dma = res->c_ivin_dma; + if (ctx->alg_type == SEC_AEAD) { + a_req->a_ivin = res->a_ivin; + a_req->a_ivin_dma = res->a_ivin_dma; + a_req->out_mac = res->out_mac; + a_req->out_mac_dma = res->out_mac_dma; + } + + src_direction = dst == src ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; + req->in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src, + qp_ctx->c_in_pool, + req->req_id, + &req->in_dma, src_direction); + if (IS_ERR(req->in)) { + dev_err(dev, "fail to dma map input sgl buffers!\n"); + return PTR_ERR(req->in); + } + + if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) { + ret = sec_aead_mac_init(a_req); + if (unlikely(ret)) { + dev_err(dev, "fail to init mac data for ICV!\n"); + hisi_acc_sg_buf_unmap(dev, src, req->in, src_direction); + return ret; + } + } + + if (dst == src) { + c_req->c_out = req->in; + c_req->c_out_dma = req->in_dma; + } else { + c_req->c_out = hisi_acc_sg_buf_map_to_hw_sgl(dev, dst, + qp_ctx->c_out_pool, + req->req_id, + &c_req->c_out_dma, + DMA_FROM_DEVICE); + + if (IS_ERR(c_req->c_out)) { + dev_err(dev, "fail to dma map output sgl buffers!\n"); + hisi_acc_sg_buf_unmap(dev, src, req->in, src_direction); + return PTR_ERR(c_req->c_out); + } + } + + return 0; +} + +static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, + struct scatterlist *src, struct scatterlist *dst) +{ + struct sec_aead_req *a_req = &req->aead_req; + struct sec_cipher_req *c_req = &req->c_req; + bool is_aead = (ctx->alg_type == SEC_AEAD); + struct device *dev = ctx->dev; + int ret = -ENOMEM; + + if (req->req_id >= 0) + return sec_cipher_map_inner(ctx, req, src, dst); + + c_req->c_ivin = c_req->c_ivin_buf; + c_req->c_ivin_dma = dma_map_single(dev, c_req->c_ivin, + SEC_IV_SIZE, DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, c_req->c_ivin_dma))) + return -ENOMEM; + + if (is_aead) { + a_req->a_ivin = a_req->a_ivin_buf; + a_req->out_mac = a_req->out_mac_buf; + a_req->a_ivin_dma = dma_map_single(dev, a_req->a_ivin, + SEC_IV_SIZE, DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, a_req->a_ivin_dma))) + goto free_c_ivin_dma; + + a_req->out_mac_dma = dma_map_single(dev, a_req->out_mac, + SEC_MAX_MAC_LEN, DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(dev, a_req->out_mac_dma))) + goto free_a_ivin_dma; + } + if (req->use_pbuf) { + ret = sec_cipher_pbuf_map(ctx, req, src); + if (unlikely(ret)) + goto free_out_mac_dma; + + return 0; + } + + if (!c_req->encrypt && is_aead) { + ret = sec_aead_mac_init(a_req); + if (unlikely(ret)) { + dev_err(dev, "fail to init mac data for ICV!\n"); + goto free_out_mac_dma; + } + } + + ret = sec_cipher_map_sgl(dev, req, src, dst); + if (unlikely(ret)) { + dev_err(dev, "fail to dma map input sgl buffers!\n"); + goto free_out_mac_dma; + } + + return 0; + +free_out_mac_dma: + if (is_aead) + dma_unmap_single(dev, a_req->out_mac_dma, SEC_MAX_MAC_LEN, DMA_BIDIRECTIONAL); +free_a_ivin_dma: + if (is_aead) + dma_unmap_single(dev, a_req->a_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE); +free_c_ivin_dma: + dma_unmap_single(dev, c_req->c_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE); + return ret; +} + +static void sec_cipher_unmap(struct sec_ctx *ctx, struct sec_req *req, + struct scatterlist *src, struct scatterlist *dst) +{ + struct sec_aead_req *a_req = &req->aead_req; + struct sec_cipher_req *c_req = &req->c_req; + struct device *dev = ctx->dev; + + if (req->req_id >= 0) { + if (req->use_pbuf) { + sec_cipher_pbuf_unmap(ctx, req, dst); + } else { + if (dst != src) { + hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out, DMA_FROM_DEVICE); + hisi_acc_sg_buf_unmap(dev, src, req->in, DMA_TO_DEVICE); + } else { + hisi_acc_sg_buf_unmap(dev, src, req->in, DMA_BIDIRECTIONAL); + } + } + return; + } + + if (req->use_pbuf) { + sec_cipher_pbuf_unmap(ctx, req, dst); + } else { + if (dst != src) { + sec_cipher_put_hw_sgl(dev, dst, req->buf.out_dma, DMA_FROM_DEVICE); + sec_cipher_put_hw_sgl(dev, src, req->buf.in_dma, DMA_TO_DEVICE); + } else { + sec_cipher_put_hw_sgl(dev, src, req->buf.in_dma, DMA_BIDIRECTIONAL); + } + } + + dma_unmap_single(dev, c_req->c_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE); + if (ctx->alg_type == SEC_AEAD) { + dma_unmap_single(dev, a_req->a_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE); + dma_unmap_single(dev, a_req->out_mac_dma, SEC_MAX_MAC_LEN, DMA_BIDIRECTIONAL); + } +} + +static int sec_skcipher_sgl_map(struct sec_ctx *ctx, struct sec_req *req) +{ + struct skcipher_request *sq = req->c_req.sk_req; + + return sec_cipher_map(ctx, req, sq->src, sq->dst); +} + +static void sec_skcipher_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req) +{ + struct skcipher_request *sq = req->c_req.sk_req; + + sec_cipher_unmap(ctx, req, sq->src, sq->dst); +} + +static int sec_aead_aes_set_key(struct sec_cipher_ctx *c_ctx, + struct crypto_authenc_keys *keys) +{ + switch (keys->enckeylen) { + case AES_KEYSIZE_128: + c_ctx->c_key_len = SEC_CKEY_128BIT; + break; + case AES_KEYSIZE_192: + c_ctx->c_key_len = SEC_CKEY_192BIT; + break; + case AES_KEYSIZE_256: + c_ctx->c_key_len = SEC_CKEY_256BIT; + break; + default: + pr_err("hisi_sec2: aead aes key error!\n"); + return -EINVAL; + } + memcpy(c_ctx->c_key, keys->enckey, keys->enckeylen); + + return 0; +} + +static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, + struct crypto_authenc_keys *keys) +{ + struct crypto_shash *hash_tfm = ctx->hash_tfm; + int blocksize, digestsize, ret; + + blocksize = crypto_shash_blocksize(hash_tfm); + digestsize = crypto_shash_digestsize(hash_tfm); + if (keys->authkeylen > blocksize) { + ret = crypto_shash_tfm_digest(hash_tfm, keys->authkey, + keys->authkeylen, ctx->a_key); + if (ret) { + pr_err("hisi_sec2: aead auth digest error!\n"); + return -EINVAL; + } + ctx->a_key_len = digestsize; + } else { + if (keys->authkeylen) + memcpy(ctx->a_key, keys->authkey, keys->authkeylen); + ctx->a_key_len = keys->authkeylen; + } + + return 0; +} + +static int sec_aead_setauthsize(struct crypto_aead *aead, unsigned int authsize) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(aead); + struct sec_ctx *ctx = crypto_tfm_ctx(tfm); + struct sec_auth_ctx *a_ctx = &ctx->a_ctx; + + return crypto_aead_setauthsize(a_ctx->fallback_aead_tfm, authsize); +} + +static int sec_aead_fallback_setkey(struct sec_auth_ctx *a_ctx, + struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + crypto_aead_clear_flags(a_ctx->fallback_aead_tfm, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(a_ctx->fallback_aead_tfm, + crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK); + return crypto_aead_setkey(a_ctx->fallback_aead_tfm, key, keylen); +} + +static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, + const u32 keylen, const enum sec_hash_alg a_alg, + const enum sec_calg c_alg, + const enum sec_cmode c_mode) +{ + struct sec_ctx *ctx = crypto_aead_ctx(tfm); + struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + struct sec_auth_ctx *a_ctx = &ctx->a_ctx; + struct device *dev = ctx->dev; + struct crypto_authenc_keys keys; + int ret; + + if (!ctx->qps) + return sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); + + ctx->a_ctx.a_alg = a_alg; + ctx->c_ctx.c_alg = c_alg; + c_ctx->c_mode = c_mode; + + if (c_mode == SEC_CMODE_CCM || c_mode == SEC_CMODE_GCM) { + ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode); + if (ret) { + dev_err(dev, "set sec aes ccm cipher key err!\n"); + return ret; + } + memcpy(c_ctx->c_key, key, keylen); + + return sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); + } + + ret = crypto_authenc_extractkeys(&keys, key, keylen); + if (ret) { + dev_err(dev, "sec extract aead keys err!\n"); + goto bad_key; + } + + ret = sec_aead_aes_set_key(c_ctx, &keys); + if (ret) { + dev_err(dev, "set sec cipher key err!\n"); + goto bad_key; + } + + ret = sec_aead_auth_set_key(&ctx->a_ctx, &keys); + if (ret) { + dev_err(dev, "set sec auth key err!\n"); + goto bad_key; + } + + ret = sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); + if (ret) { + dev_err(dev, "set sec fallback key err!\n"); + goto bad_key; + } + + return 0; + +bad_key: + memzero_explicit(&keys, sizeof(struct crypto_authenc_keys)); + return ret; +} + + +#define GEN_SEC_AEAD_SETKEY_FUNC(name, aalg, calg, cmode) \ +static int sec_setkey_##name(struct crypto_aead *tfm, const u8 *key, u32 keylen) \ +{ \ + return sec_aead_setkey(tfm, key, keylen, aalg, calg, cmode); \ +} + +GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1, SEC_A_HMAC_SHA1, SEC_CALG_AES, SEC_CMODE_CBC) +GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha256, SEC_A_HMAC_SHA256, SEC_CALG_AES, SEC_CMODE_CBC) +GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha512, SEC_A_HMAC_SHA512, SEC_CALG_AES, SEC_CMODE_CBC) +GEN_SEC_AEAD_SETKEY_FUNC(aes_ccm, 0, SEC_CALG_AES, SEC_CMODE_CCM) +GEN_SEC_AEAD_SETKEY_FUNC(aes_gcm, 0, SEC_CALG_AES, SEC_CMODE_GCM) +GEN_SEC_AEAD_SETKEY_FUNC(sm4_ccm, 0, SEC_CALG_SM4, SEC_CMODE_CCM) +GEN_SEC_AEAD_SETKEY_FUNC(sm4_gcm, 0, SEC_CALG_SM4, SEC_CMODE_GCM) + +static int sec_aead_sgl_map(struct sec_ctx *ctx, struct sec_req *req) +{ + struct aead_request *aq = req->aead_req.aead_req; + + return sec_cipher_map(ctx, req, aq->src, aq->dst); +} + +static void sec_aead_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req) +{ + struct aead_request *aq = req->aead_req.aead_req; + + sec_cipher_unmap(ctx, req, aq->src, aq->dst); +} + +static int sec_request_transfer(struct sec_ctx *ctx, struct sec_req *req) +{ + int ret; + + ret = ctx->req_op->buf_map(ctx, req); + if (unlikely(ret)) + return ret; + + ctx->req_op->do_transfer(ctx, req); + + ret = ctx->req_op->bd_fill(ctx, req); + if (unlikely(ret)) + goto unmap_req_buf; + + return ret; + +unmap_req_buf: + ctx->req_op->buf_unmap(ctx, req); + return ret; +} + +static void sec_request_untransfer(struct sec_ctx *ctx, struct sec_req *req) +{ + ctx->req_op->buf_unmap(ctx, req); +} + +static void sec_skcipher_copy_iv(struct sec_ctx *ctx, struct sec_req *req) +{ + struct skcipher_request *sk_req = req->c_req.sk_req; + struct sec_cipher_req *c_req = &req->c_req; + + memcpy(c_req->c_ivin, sk_req->iv, ctx->c_ctx.ivsize); +} + +static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req) +{ + struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + struct sec_cipher_req *c_req = &req->c_req; + struct sec_sqe *sec_sqe = &req->sec_sqe; + u8 scene, sa_type, da_type; + u8 bd_type, cipher; + u8 de = 0; + + memset(sec_sqe, 0, sizeof(struct sec_sqe)); + + sec_sqe->type2.c_key_addr = cpu_to_le64(c_ctx->c_key_dma); + sec_sqe->type2.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma); + if (req->req_id < 0) { + sec_sqe->type2.data_src_addr = cpu_to_le64(req->buf.in_dma); + sec_sqe->type2.data_dst_addr = cpu_to_le64(req->buf.out_dma); + } else { + sec_sqe->type2.data_src_addr = cpu_to_le64(req->in_dma); + sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma); + } + if (sec_sqe->type2.data_src_addr != sec_sqe->type2.data_dst_addr) + de = 0x1 << SEC_DE_OFFSET; + + sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_mode) << + SEC_CMODE_OFFSET); + sec_sqe->type2.c_alg = c_ctx->c_alg; + sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_key_len) << + SEC_CKEY_OFFSET); + + bd_type = SEC_BD_TYPE2; + if (c_req->encrypt) + cipher = SEC_CIPHER_ENC << SEC_CIPHER_OFFSET; + else + cipher = SEC_CIPHER_DEC << SEC_CIPHER_OFFSET; + sec_sqe->type_cipher_auth = bd_type | cipher; + + /* Set destination and source address type */ + if (req->use_pbuf) { + sa_type = SEC_PBUF << SEC_SRC_SGL_OFFSET; + da_type = SEC_PBUF << SEC_DST_SGL_OFFSET; + } else { + sa_type = SEC_SGL << SEC_SRC_SGL_OFFSET; + da_type = SEC_SGL << SEC_DST_SGL_OFFSET; + } + + sec_sqe->sdm_addr_type |= da_type; + scene = SEC_COMM_SCENE << SEC_SCENE_OFFSET; + + sec_sqe->sds_sa_type = (de | scene | sa_type); + + sec_sqe->type2.clen_ivhlen |= cpu_to_le32(c_req->c_len); + + return 0; +} + +static int sec_skcipher_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req) +{ + struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3; + struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + struct sec_cipher_req *c_req = &req->c_req; + u32 bd_param = 0; + u16 cipher; + + memset(sec_sqe3, 0, sizeof(struct sec_sqe3)); + + sec_sqe3->c_key_addr = cpu_to_le64(c_ctx->c_key_dma); + sec_sqe3->no_scene.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma); + if (req->req_id < 0) { + sec_sqe3->data_src_addr = cpu_to_le64(req->buf.in_dma); + sec_sqe3->data_dst_addr = cpu_to_le64(req->buf.out_dma); + } else { + sec_sqe3->data_src_addr = cpu_to_le64(req->in_dma); + sec_sqe3->data_dst_addr = cpu_to_le64(c_req->c_out_dma); + } + if (sec_sqe3->data_src_addr != sec_sqe3->data_dst_addr) + bd_param |= 0x1 << SEC_DE_OFFSET_V3; + + sec_sqe3->c_mode_alg = ((u8)c_ctx->c_alg << SEC_CALG_OFFSET_V3) | + c_ctx->c_mode; + sec_sqe3->c_icv_key |= cpu_to_le16(((u16)c_ctx->c_key_len) << + SEC_CKEY_OFFSET_V3); + + if (c_req->encrypt) + cipher = SEC_CIPHER_ENC; + else + cipher = SEC_CIPHER_DEC; + sec_sqe3->c_icv_key |= cpu_to_le16(cipher); + + /* Set the CTR counter mode is 128bit rollover */ + sec_sqe3->auth_mac_key = cpu_to_le32((u32)SEC_CTR_CNT_ROLLOVER << + SEC_CTR_CNT_OFFSET); + + if (req->use_pbuf) { + bd_param |= SEC_PBUF << SEC_SRC_SGL_OFFSET_V3; + bd_param |= SEC_PBUF << SEC_DST_SGL_OFFSET_V3; + } else { + bd_param |= SEC_SGL << SEC_SRC_SGL_OFFSET_V3; + bd_param |= SEC_SGL << SEC_DST_SGL_OFFSET_V3; + } + + bd_param |= SEC_COMM_SCENE << SEC_SCENE_OFFSET_V3; + + bd_param |= SEC_BD_TYPE3; + sec_sqe3->bd_param = cpu_to_le32(bd_param); + + sec_sqe3->c_len_ivin |= cpu_to_le32(c_req->c_len); + sec_sqe3->tag = cpu_to_le64((unsigned long)req); + + return 0; +} + +/* increment counter (128-bit int) */ +static void ctr_iv_inc(__u8 *counter, __u8 bits, __u32 nums) +{ + do { + --bits; + nums += counter[bits]; + counter[bits] = nums & BITS_MASK; + nums >>= BYTE_BITS; + } while (bits && nums); +} + +static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type) +{ + struct aead_request *aead_req = req->aead_req.aead_req; + struct skcipher_request *sk_req = req->c_req.sk_req; + u32 iv_size = req->ctx->c_ctx.ivsize; + struct scatterlist *sgl; + unsigned int cryptlen; + size_t sz; + u8 *iv; + + if (alg_type == SEC_SKCIPHER) { + sgl = req->c_req.encrypt ? sk_req->dst : sk_req->src; + iv = sk_req->iv; + cryptlen = sk_req->cryptlen; + } else { + sgl = req->c_req.encrypt ? aead_req->dst : aead_req->src; + iv = aead_req->iv; + cryptlen = aead_req->cryptlen; + } + + if (req->ctx->c_ctx.c_mode == SEC_CMODE_CBC) { + sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), iv, iv_size, + cryptlen - iv_size); + if (unlikely(sz != iv_size)) + dev_err(req->ctx->dev, "copy output iv error!\n"); + } else { + sz = (cryptlen + iv_size - 1) / iv_size; + ctr_iv_inc(iv, iv_size, sz); + } +} + +static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req, + int err) +{ + struct sec_qp_ctx *qp_ctx = req->qp_ctx; + + if (req->req_id >= 0) + sec_free_req_id(req); + + /* IV output at encrypto of CBC/CTR mode */ + if (!err && (ctx->c_ctx.c_mode == SEC_CMODE_CBC || + ctx->c_ctx.c_mode == SEC_CMODE_CTR) && req->c_req.encrypt) + sec_update_iv(req, SEC_SKCIPHER); + + crypto_request_complete(req->base, err); + sec_alg_send_backlog(ctx, qp_ctx); +} + +static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req) +{ + struct aead_request *aead_req = req->aead_req.aead_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); + size_t authsize = crypto_aead_authsize(tfm); + struct sec_aead_req *a_req = &req->aead_req; + struct sec_cipher_req *c_req = &req->c_req; + u32 data_size = aead_req->cryptlen; + u8 flage = 0; + u8 cm, cl, i; + + /* the specification has been checked in aead_iv_demension_check() */ + cl = c_req->c_ivin[0] + 1; + c_req->c_ivin[ctx->c_ctx.ivsize - cl] = 0x00; + memset(&c_req->c_ivin[ctx->c_ctx.ivsize - cl], 0, cl); + c_req->c_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] = IV_CTR_INIT; + + /* the last 3bit is L' */ + flage |= c_req->c_ivin[0] & IV_CL_MASK; + + /* the M' is bit3~bit5, the Flags is bit6 */ + cm = (authsize - IV_CM_CAL_NUM) / IV_CM_CAL_NUM; + flage |= cm << IV_CM_OFFSET; + if (aead_req->assoclen) + flage |= 0x01 << IV_FLAGS_OFFSET; + + memcpy(a_req->a_ivin, c_req->c_ivin, ctx->c_ctx.ivsize); + a_req->a_ivin[0] = flage; + + /* + * the last 32bit is counter's initial number, + * but the nonce uses the first 16bit + * the tail 16bit fill with the cipher length + * When CL is 3, the tail 24bit fill with the cipher length. + */ + if (!c_req->encrypt) + data_size = aead_req->cryptlen - authsize; + + for (i = 1; i <= cl; i++) { + a_req->a_ivin[ctx->c_ctx.ivsize - i] = + data_size & IV_LAST_BYTE_MASK; + data_size >>= IV_BYTE_OFFSET; + } +} + +static void sec_aead_set_iv(struct sec_ctx *ctx, struct sec_req *req) +{ + struct aead_request *aead_req = req->aead_req.aead_req; + struct sec_aead_req *a_req = &req->aead_req; + struct sec_cipher_req *c_req = &req->c_req; + + memcpy(c_req->c_ivin, aead_req->iv, ctx->c_ctx.ivsize); + + if (ctx->c_ctx.c_mode == SEC_CMODE_CCM) { + /* + * CCM 16Byte Cipher_IV: {1B_Flage,13B_IV,2B_counter}, + * the counter must set to 0x01 + * CCM 16Byte Auth_IV: {1B_AFlage,13B_IV,2B_Ptext_length} + */ + set_aead_auth_iv(ctx, req); + } else if (ctx->c_ctx.c_mode == SEC_CMODE_GCM) { + /* GCM 12Byte Cipher_IV == Auth_IV */ + memcpy(a_req->a_ivin, c_req->c_ivin, SEC_AIV_SIZE); + } +} + +static void sec_auth_bd_fill_xcm(struct sec_auth_ctx *ctx, int dir, + struct sec_req *req, struct sec_sqe *sec_sqe) +{ + struct sec_aead_req *a_req = &req->aead_req; + struct aead_request *aq = a_req->aead_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(aq); + size_t authsize = crypto_aead_authsize(tfm); + + /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */ + sec_sqe->type2.icvw_kmode |= cpu_to_le16((u16)authsize); + + /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */ + sec_sqe->type2.a_key_addr = sec_sqe->type2.c_key_addr; + sec_sqe->type2.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma); + sec_sqe->type_cipher_auth |= SEC_NO_AUTH << SEC_AUTH_OFFSET; + + if (dir) + sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH; + else + sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER; + + sec_sqe->type2.alen_ivllen = cpu_to_le32(aq->assoclen); + sec_sqe->type2.auth_src_offset = cpu_to_le16(0x0); + sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen); + + sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma); +} + +static void sec_auth_bd_fill_xcm_v3(struct sec_auth_ctx *ctx, int dir, + struct sec_req *req, struct sec_sqe3 *sqe3) +{ + struct sec_aead_req *a_req = &req->aead_req; + struct aead_request *aq = a_req->aead_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(aq); + size_t authsize = crypto_aead_authsize(tfm); + + /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */ + sqe3->c_icv_key |= cpu_to_le16((u16)authsize << SEC_MAC_OFFSET_V3); + + /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */ + sqe3->a_key_addr = sqe3->c_key_addr; + sqe3->auth_ivin.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma); + sqe3->auth_mac_key |= SEC_NO_AUTH; + + if (dir) + sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3; + else + sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3; + + sqe3->a_len_key = cpu_to_le32(aq->assoclen); + sqe3->auth_src_offset = cpu_to_le16(0x0); + sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen); + sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma); +} + +static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir, + struct sec_req *req, struct sec_sqe *sec_sqe) +{ + struct sec_aead_req *a_req = &req->aead_req; + struct sec_cipher_req *c_req = &req->c_req; + struct aead_request *aq = a_req->aead_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(aq); + size_t authsize = crypto_aead_authsize(tfm); + + sec_sqe->type2.a_key_addr = cpu_to_le64(ctx->a_key_dma); + + sec_sqe->type2.mac_key_alg = cpu_to_le32(BYTES_TO_WORDS(authsize)); + + sec_sqe->type2.mac_key_alg |= + cpu_to_le32((u32)BYTES_TO_WORDS(ctx->a_key_len) << SEC_AKEY_OFFSET); + + sec_sqe->type2.mac_key_alg |= + cpu_to_le32((u32)(ctx->a_alg) << SEC_AEAD_ALG_OFFSET); + + if (dir) { + sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE1 << SEC_AUTH_OFFSET; + sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH; + } else { + sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE2 << SEC_AUTH_OFFSET; + sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER; + } + sec_sqe->type2.alen_ivllen = cpu_to_le32(c_req->c_len + aq->assoclen); + + sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen); + + sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma); +} + +static int sec_aead_bd_fill(struct sec_ctx *ctx, struct sec_req *req) +{ + struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; + struct sec_sqe *sec_sqe = &req->sec_sqe; + int ret; + + ret = sec_skcipher_bd_fill(ctx, req); + if (unlikely(ret)) { + dev_err(ctx->dev, "skcipher bd fill is error!\n"); + return ret; + } + + if (ctx->c_ctx.c_mode == SEC_CMODE_CCM || + ctx->c_ctx.c_mode == SEC_CMODE_GCM) + sec_auth_bd_fill_xcm(auth_ctx, req->c_req.encrypt, req, sec_sqe); + else + sec_auth_bd_fill_ex(auth_ctx, req->c_req.encrypt, req, sec_sqe); + + return 0; +} + +static void sec_auth_bd_fill_ex_v3(struct sec_auth_ctx *ctx, int dir, + struct sec_req *req, struct sec_sqe3 *sqe3) +{ + struct sec_aead_req *a_req = &req->aead_req; + struct sec_cipher_req *c_req = &req->c_req; + struct aead_request *aq = a_req->aead_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(aq); + size_t authsize = crypto_aead_authsize(tfm); + + sqe3->a_key_addr = cpu_to_le64(ctx->a_key_dma); + + sqe3->auth_mac_key |= + cpu_to_le32(BYTES_TO_WORDS(authsize) << SEC_MAC_OFFSET_V3); + + sqe3->auth_mac_key |= + cpu_to_le32((u32)BYTES_TO_WORDS(ctx->a_key_len) << SEC_AKEY_OFFSET_V3); + + sqe3->auth_mac_key |= + cpu_to_le32((u32)(ctx->a_alg) << SEC_AUTH_ALG_OFFSET_V3); + + if (dir) { + sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE1); + sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3; + } else { + sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE2); + sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3; + } + sqe3->a_len_key = cpu_to_le32(c_req->c_len + aq->assoclen); + + sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen); + + sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma); +} + +static int sec_aead_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req) +{ + struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; + struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3; + int ret; + + ret = sec_skcipher_bd_fill_v3(ctx, req); + if (unlikely(ret)) { + dev_err(ctx->dev, "skcipher bd3 fill is error!\n"); + return ret; + } + + if (ctx->c_ctx.c_mode == SEC_CMODE_CCM || + ctx->c_ctx.c_mode == SEC_CMODE_GCM) + sec_auth_bd_fill_xcm_v3(auth_ctx, req->c_req.encrypt, + req, sec_sqe3); + else + sec_auth_bd_fill_ex_v3(auth_ctx, req->c_req.encrypt, + req, sec_sqe3); + + return 0; +} + +static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err) +{ + struct aead_request *a_req = req->aead_req.aead_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); + size_t authsize = crypto_aead_authsize(tfm); + struct sec_qp_ctx *qp_ctx = req->qp_ctx; + size_t sz; + + if (!err && req->c_req.encrypt) { + if (c->c_ctx.c_mode == SEC_CMODE_CBC) + sec_update_iv(req, SEC_AEAD); + + sz = sg_pcopy_from_buffer(a_req->dst, sg_nents(a_req->dst), req->aead_req.out_mac, + authsize, a_req->cryptlen + a_req->assoclen); + if (unlikely(sz != authsize)) { + dev_err(c->dev, "copy out mac err!\n"); + err = -EINVAL; + } + } + + if (req->req_id >= 0) + sec_free_req_id(req); + + crypto_request_complete(req->base, err); + sec_alg_send_backlog(c, qp_ctx); +} + +static void sec_request_uninit(struct sec_req *req) +{ + if (req->req_id >= 0) + sec_free_req_id(req); +} + +static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req) +{ + struct sec_qp_ctx *qp_ctx; + int i = 0; + + do { + qp_ctx = &ctx->qp_ctx[i]; + req->req_id = sec_alloc_req_id(req, qp_ctx); + } while (req->req_id < 0 && ++i < ctx->sec->ctx_q_num); + + req->qp_ctx = qp_ctx; + + return 0; +} + +static int sec_process(struct sec_ctx *ctx, struct sec_req *req) +{ + int ret; + + ret = sec_request_init(ctx, req); + if (unlikely(ret)) + return ret; + + ret = sec_request_transfer(ctx, req); + if (unlikely(ret)) + goto err_uninit_req; + + /* Output IV as decrypto */ + if (!req->c_req.encrypt && (ctx->c_ctx.c_mode == SEC_CMODE_CBC || + ctx->c_ctx.c_mode == SEC_CMODE_CTR)) + sec_update_iv(req, ctx->alg_type); + + ret = ctx->req_op->bd_send(ctx, req); + if (unlikely((ret != -EBUSY && ret != -EINPROGRESS))) { + dev_err_ratelimited(ctx->dev, "send sec request failed!\n"); + goto err_send_req; + } + + return ret; + +err_send_req: + /* As failing, restore the IV from user */ + if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) { + if (ctx->alg_type == SEC_SKCIPHER) + memcpy(req->c_req.sk_req->iv, req->c_req.c_ivin, + ctx->c_ctx.ivsize); + else + memcpy(req->aead_req.aead_req->iv, req->c_req.c_ivin, + ctx->c_ctx.ivsize); + } + + sec_request_untransfer(ctx, req); + +err_uninit_req: + sec_request_uninit(req); + if (ctx->alg_type == SEC_AEAD) + ret = sec_aead_soft_crypto(ctx, req->aead_req.aead_req, + req->c_req.encrypt); + else + ret = sec_skcipher_soft_crypto(ctx, req->c_req.sk_req, + req->c_req.encrypt); + return ret; +} + +static const struct sec_req_op sec_skcipher_req_ops = { + .buf_map = sec_skcipher_sgl_map, + .buf_unmap = sec_skcipher_sgl_unmap, + .do_transfer = sec_skcipher_copy_iv, + .bd_fill = sec_skcipher_bd_fill, + .bd_send = sec_bd_send, + .callback = sec_skcipher_callback, + .process = sec_process, +}; + +static const struct sec_req_op sec_aead_req_ops = { + .buf_map = sec_aead_sgl_map, + .buf_unmap = sec_aead_sgl_unmap, + .do_transfer = sec_aead_set_iv, + .bd_fill = sec_aead_bd_fill, + .bd_send = sec_bd_send, + .callback = sec_aead_callback, + .process = sec_process, +}; + +static const struct sec_req_op sec_skcipher_req_ops_v3 = { + .buf_map = sec_skcipher_sgl_map, + .buf_unmap = sec_skcipher_sgl_unmap, + .do_transfer = sec_skcipher_copy_iv, + .bd_fill = sec_skcipher_bd_fill_v3, + .bd_send = sec_bd_send, + .callback = sec_skcipher_callback, + .process = sec_process, +}; + +static const struct sec_req_op sec_aead_req_ops_v3 = { + .buf_map = sec_aead_sgl_map, + .buf_unmap = sec_aead_sgl_unmap, + .do_transfer = sec_aead_set_iv, + .bd_fill = sec_aead_bd_fill_v3, + .bd_send = sec_bd_send, + .callback = sec_aead_callback, + .process = sec_process, +}; + +static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm) +{ + struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); + int ret; + + ret = sec_skcipher_init(tfm); + if (ret) + return ret; + + if (!ctx->qps) + return 0; + + if (ctx->type_supported == SEC_BD_TYPE3) + ctx->req_op = &sec_skcipher_req_ops_v3; + else + ctx->req_op = &sec_skcipher_req_ops; + + return 0; +} + +static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm) +{ + sec_skcipher_uninit(tfm); +} + +static int sec_aead_init(struct crypto_aead *tfm) +{ + struct sec_ctx *ctx = crypto_aead_ctx(tfm); + int ret; + + crypto_aead_set_reqsize_dma(tfm, sizeof(struct sec_req)); + ctx->alg_type = SEC_AEAD; + ctx->c_ctx.ivsize = crypto_aead_ivsize(tfm); + if (ctx->c_ctx.ivsize < SEC_AIV_SIZE || + ctx->c_ctx.ivsize > SEC_IV_SIZE) { + pr_err("get error aead iv size!\n"); + return -EINVAL; + } + + ret = sec_ctx_base_init(ctx); + if (ret) + return ret; + + if (ctx->type_supported == SEC_BD_TYPE3) + ctx->req_op = &sec_aead_req_ops_v3; + else + ctx->req_op = &sec_aead_req_ops; + + ret = sec_auth_init(ctx); + if (ret) + goto err_auth_init; + + ret = sec_cipher_init(ctx); + if (ret) + goto err_cipher_init; + + return ret; + +err_cipher_init: + sec_auth_uninit(ctx); +err_auth_init: + sec_ctx_base_uninit(ctx); + return ret; +} + +static void sec_aead_exit(struct crypto_aead *tfm) +{ + struct sec_ctx *ctx = crypto_aead_ctx(tfm); + + sec_cipher_uninit(ctx); + sec_auth_uninit(ctx); + sec_ctx_base_uninit(ctx); +} + +static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name) +{ + struct aead_alg *alg = crypto_aead_alg(tfm); + struct sec_ctx *ctx = crypto_aead_ctx(tfm); + struct sec_auth_ctx *a_ctx = &ctx->a_ctx; + const char *aead_name = alg->base.cra_name; + int ret; + + ret = sec_aead_init(tfm); + if (ret && ret != -ENODEV) { + pr_err("hisi_sec2: aead init error!\n"); + return ret; + } + + a_ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); + if (IS_ERR(a_ctx->hash_tfm)) { + dev_err(ctx->dev, "aead alloc shash error!\n"); + sec_aead_exit(tfm); + return PTR_ERR(a_ctx->hash_tfm); + } + + a_ctx->fallback_aead_tfm = crypto_alloc_aead(aead_name, 0, + CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC); + if (IS_ERR(a_ctx->fallback_aead_tfm)) { + dev_err(ctx->dev, "aead driver alloc fallback tfm error!\n"); + crypto_free_shash(ctx->a_ctx.hash_tfm); + sec_aead_exit(tfm); + return PTR_ERR(a_ctx->fallback_aead_tfm); + } + + return 0; +} + +static void sec_aead_ctx_exit(struct crypto_aead *tfm) +{ + struct sec_ctx *ctx = crypto_aead_ctx(tfm); + + crypto_free_aead(ctx->a_ctx.fallback_aead_tfm); + crypto_free_shash(ctx->a_ctx.hash_tfm); + sec_aead_exit(tfm); +} + +static int sec_aead_xcm_ctx_init(struct crypto_aead *tfm) +{ + struct aead_alg *alg = crypto_aead_alg(tfm); + struct sec_ctx *ctx = crypto_aead_ctx(tfm); + struct sec_auth_ctx *a_ctx = &ctx->a_ctx; + const char *aead_name = alg->base.cra_name; + int ret; + + ret = sec_aead_init(tfm); + if (ret && ret != -ENODEV) { + dev_err(ctx->dev, "hisi_sec2: aead xcm init error!\n"); + return ret; + } + + a_ctx->fallback_aead_tfm = crypto_alloc_aead(aead_name, 0, + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC); + if (IS_ERR(a_ctx->fallback_aead_tfm)) { + dev_err(ctx->dev, "aead driver alloc fallback tfm error!\n"); + sec_aead_exit(tfm); + return PTR_ERR(a_ctx->fallback_aead_tfm); + } + + return 0; +} + +static void sec_aead_xcm_ctx_exit(struct crypto_aead *tfm) +{ + struct sec_ctx *ctx = crypto_aead_ctx(tfm); + + crypto_free_aead(ctx->a_ctx.fallback_aead_tfm); + sec_aead_exit(tfm); +} + +static int sec_aead_sha1_ctx_init(struct crypto_aead *tfm) +{ + return sec_aead_ctx_init(tfm, "sha1"); +} + +static int sec_aead_sha256_ctx_init(struct crypto_aead *tfm) +{ + return sec_aead_ctx_init(tfm, "sha256"); +} + +static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm) +{ + return sec_aead_ctx_init(tfm, "sha512"); +} + +static int sec_skcipher_cryptlen_check(struct sec_ctx *ctx, struct sec_req *sreq) +{ + u32 cryptlen = sreq->c_req.sk_req->cryptlen; + struct device *dev = ctx->dev; + u8 c_mode = ctx->c_ctx.c_mode; + int ret = 0; + + switch (c_mode) { + case SEC_CMODE_XTS: + if (unlikely(cryptlen < AES_BLOCK_SIZE)) { + dev_err(dev, "skcipher XTS mode input length error!\n"); + ret = -EINVAL; + } + break; + case SEC_CMODE_ECB: + case SEC_CMODE_CBC: + if (unlikely(cryptlen & (AES_BLOCK_SIZE - 1))) { + dev_err(dev, "skcipher AES input length error!\n"); + ret = -EINVAL; + } + break; + case SEC_CMODE_CTR: + break; + default: + ret = -EINVAL; + } + + return ret; +} + +static int sec_skcipher_param_check(struct sec_ctx *ctx, + struct sec_req *sreq, bool *need_fallback) +{ + struct skcipher_request *sk_req = sreq->c_req.sk_req; + struct device *dev = ctx->dev; + u8 c_alg = ctx->c_ctx.c_alg; + + if (unlikely(!sk_req->src || !sk_req->dst)) { + dev_err(dev, "skcipher input param error!\n"); + return -EINVAL; + } + + if (sk_req->cryptlen > MAX_INPUT_DATA_LEN) + *need_fallback = true; + + sreq->c_req.c_len = sk_req->cryptlen; + + if (ctx->pbuf_supported && sk_req->cryptlen <= SEC_PBUF_SZ) + sreq->use_pbuf = true; + else + sreq->use_pbuf = false; + + if (c_alg == SEC_CALG_3DES) { + if (unlikely(sk_req->cryptlen & (DES3_EDE_BLOCK_SIZE - 1))) { + dev_err(dev, "skcipher 3des input length error!\n"); + return -EINVAL; + } + return 0; + } else if (c_alg == SEC_CALG_AES || c_alg == SEC_CALG_SM4) { + return sec_skcipher_cryptlen_check(ctx, sreq); + } + + dev_err(dev, "skcipher algorithm error!\n"); + + return -EINVAL; +} + +static int sec_skcipher_soft_crypto(struct sec_ctx *ctx, + struct skcipher_request *sreq, bool encrypt) +{ + struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, c_ctx->fbtfm); + struct device *dev = ctx->dev; + int ret; + + if (!c_ctx->fbtfm) { + dev_err_ratelimited(dev, "the soft tfm isn't supported in the current system.\n"); + return -EINVAL; + } + + skcipher_request_set_sync_tfm(subreq, c_ctx->fbtfm); + + /* software need sync mode to do crypto */ + skcipher_request_set_callback(subreq, sreq->base.flags, + NULL, NULL); + skcipher_request_set_crypt(subreq, sreq->src, sreq->dst, + sreq->cryptlen, sreq->iv); + if (encrypt) + ret = crypto_skcipher_encrypt(subreq); + else + ret = crypto_skcipher_decrypt(subreq); + + skcipher_request_zero(subreq); + + return ret; +} + +static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(sk_req); + struct sec_req *req = skcipher_request_ctx_dma(sk_req); + struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); + bool need_fallback = false; + int ret; + + if (!ctx->qps) + goto soft_crypto; + + if (!sk_req->cryptlen) { + if (ctx->c_ctx.c_mode == SEC_CMODE_XTS) + return -EINVAL; + return 0; + } + + req->flag = sk_req->base.flags; + req->c_req.sk_req = sk_req; + req->c_req.encrypt = encrypt; + req->ctx = ctx; + req->base = &sk_req->base; + + ret = sec_skcipher_param_check(ctx, req, &need_fallback); + if (unlikely(ret)) + return -EINVAL; + + if (unlikely(ctx->c_ctx.fallback || need_fallback)) + goto soft_crypto; + + return ctx->req_op->process(ctx, req); + +soft_crypto: + return sec_skcipher_soft_crypto(ctx, sk_req, encrypt); +} + +static int sec_skcipher_encrypt(struct skcipher_request *sk_req) +{ + return sec_skcipher_crypto(sk_req, true); +} + +static int sec_skcipher_decrypt(struct skcipher_request *sk_req) +{ + return sec_skcipher_crypto(sk_req, false); +} + +#define SEC_SKCIPHER_ALG(sec_cra_name, sec_set_key, \ + sec_min_key_size, sec_max_key_size, blk_size, iv_size)\ +{\ + .base = {\ + .cra_name = sec_cra_name,\ + .cra_driver_name = "hisi_sec_"sec_cra_name,\ + .cra_priority = SEC_PRIORITY,\ + .cra_flags = CRYPTO_ALG_ASYNC |\ + CRYPTO_ALG_NEED_FALLBACK,\ + .cra_blocksize = blk_size,\ + .cra_ctxsize = sizeof(struct sec_ctx),\ + .cra_module = THIS_MODULE,\ + },\ + .init = sec_skcipher_ctx_init,\ + .exit = sec_skcipher_ctx_exit,\ + .setkey = sec_set_key,\ + .decrypt = sec_skcipher_decrypt,\ + .encrypt = sec_skcipher_encrypt,\ + .min_keysize = sec_min_key_size,\ + .max_keysize = sec_max_key_size,\ + .ivsize = iv_size,\ +} + +static struct sec_skcipher sec_skciphers[] = { + { + .alg_msk = BIT(0), + .alg = SEC_SKCIPHER_ALG("ecb(aes)", sec_setkey_aes_ecb, AES_MIN_KEY_SIZE, + AES_MAX_KEY_SIZE, AES_BLOCK_SIZE, 0), + }, + { + .alg_msk = BIT(1), + .alg = SEC_SKCIPHER_ALG("cbc(aes)", sec_setkey_aes_cbc, AES_MIN_KEY_SIZE, + AES_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(2), + .alg = SEC_SKCIPHER_ALG("ctr(aes)", sec_setkey_aes_ctr, AES_MIN_KEY_SIZE, + AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(3), + .alg = SEC_SKCIPHER_ALG("xts(aes)", sec_setkey_aes_xts, SEC_XTS_MIN_KEY_SIZE, + SEC_XTS_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(12), + .alg = SEC_SKCIPHER_ALG("cbc(sm4)", sec_setkey_sm4_cbc, AES_MIN_KEY_SIZE, + AES_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(13), + .alg = SEC_SKCIPHER_ALG("ctr(sm4)", sec_setkey_sm4_ctr, AES_MIN_KEY_SIZE, + AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(14), + .alg = SEC_SKCIPHER_ALG("xts(sm4)", sec_setkey_sm4_xts, SEC_XTS_MIN_KEY_SIZE, + SEC_XTS_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(23), + .alg = SEC_SKCIPHER_ALG("ecb(des3_ede)", sec_setkey_3des_ecb, SEC_DES3_3KEY_SIZE, + SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE, 0), + }, + { + .alg_msk = BIT(24), + .alg = SEC_SKCIPHER_ALG("cbc(des3_ede)", sec_setkey_3des_cbc, SEC_DES3_3KEY_SIZE, + SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE, + DES3_EDE_BLOCK_SIZE), + }, +}; + +static int aead_iv_demension_check(struct aead_request *aead_req) +{ + u8 cl; + + cl = aead_req->iv[0] + 1; + if (cl < IV_CL_MIN || cl > IV_CL_MAX) + return -EINVAL; + + if (cl < IV_CL_MID && aead_req->cryptlen >> (BYTE_BITS * cl)) + return -EOVERFLOW; + + return 0; +} + +static int sec_aead_spec_check(struct sec_ctx *ctx, struct sec_req *sreq) +{ + struct aead_request *req = sreq->aead_req.aead_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + size_t sz = crypto_aead_authsize(tfm); + u8 c_mode = ctx->c_ctx.c_mode; + int ret; + + if (unlikely(ctx->sec->qm.ver == QM_HW_V2 && !sreq->c_req.c_len)) + return -EINVAL; + + if (unlikely(req->cryptlen + req->assoclen > MAX_INPUT_DATA_LEN || + req->assoclen > SEC_MAX_AAD_LEN)) + return -EINVAL; + + if (c_mode == SEC_CMODE_CCM) { + if (unlikely(req->assoclen > SEC_MAX_CCM_AAD_LEN)) + return -EINVAL; + + ret = aead_iv_demension_check(req); + if (unlikely(ret)) + return -EINVAL; + } else if (c_mode == SEC_CMODE_CBC) { + if (unlikely(sz & WORD_MASK)) + return -EINVAL; + if (unlikely(ctx->a_ctx.a_key_len & WORD_MASK)) + return -EINVAL; + } else if (c_mode == SEC_CMODE_GCM) { + if (unlikely(sz < SEC_GCM_MIN_AUTH_SZ)) + return -EINVAL; + } + + return 0; +} + +static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq, bool *need_fallback) +{ + struct aead_request *req = sreq->aead_req.aead_req; + struct device *dev = ctx->dev; + u8 c_alg = ctx->c_ctx.c_alg; + + if (unlikely(!req->src || !req->dst)) { + dev_err(dev, "aead input param error!\n"); + return -EINVAL; + } + + if (unlikely(ctx->c_ctx.c_mode == SEC_CMODE_CBC && + sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) { + dev_err(dev, "aead cbc mode input data length error!\n"); + return -EINVAL; + } + + /* Support AES or SM4 */ + if (unlikely(c_alg != SEC_CALG_AES && c_alg != SEC_CALG_SM4)) { + dev_err(dev, "aead crypto alg error!\n"); + return -EINVAL; + } + + if (unlikely(sec_aead_spec_check(ctx, sreq))) { + *need_fallback = true; + return -EINVAL; + } + + if (ctx->pbuf_supported && (req->cryptlen + req->assoclen) <= + SEC_PBUF_SZ) + sreq->use_pbuf = true; + else + sreq->use_pbuf = false; + + return 0; +} + +static int sec_aead_soft_crypto(struct sec_ctx *ctx, + struct aead_request *aead_req, + bool encrypt) +{ + struct sec_auth_ctx *a_ctx = &ctx->a_ctx; + struct aead_request *subreq; + int ret; + + subreq = aead_request_alloc(a_ctx->fallback_aead_tfm, GFP_KERNEL); + if (!subreq) + return -ENOMEM; + + aead_request_set_tfm(subreq, a_ctx->fallback_aead_tfm); + aead_request_set_callback(subreq, aead_req->base.flags, + aead_req->base.complete, aead_req->base.data); + aead_request_set_crypt(subreq, aead_req->src, aead_req->dst, + aead_req->cryptlen, aead_req->iv); + aead_request_set_ad(subreq, aead_req->assoclen); + + if (encrypt) + ret = crypto_aead_encrypt(subreq); + else + ret = crypto_aead_decrypt(subreq); + aead_request_free(subreq); + + return ret; +} + +static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); + struct sec_req *req = aead_request_ctx_dma(a_req); + struct sec_ctx *ctx = crypto_aead_ctx(tfm); + size_t sz = crypto_aead_authsize(tfm); + bool need_fallback = false; + int ret; + + if (!ctx->qps) + goto soft_crypto; + + req->flag = a_req->base.flags; + req->aead_req.aead_req = a_req; + req->c_req.encrypt = encrypt; + req->ctx = ctx; + req->base = &a_req->base; + req->c_req.c_len = a_req->cryptlen - (req->c_req.encrypt ? 0 : sz); + + ret = sec_aead_param_check(ctx, req, &need_fallback); + if (unlikely(ret)) { + if (need_fallback) + goto soft_crypto; + return -EINVAL; + } + + return ctx->req_op->process(ctx, req); + +soft_crypto: + return sec_aead_soft_crypto(ctx, a_req, encrypt); +} + +static int sec_aead_encrypt(struct aead_request *a_req) +{ + return sec_aead_crypto(a_req, true); +} + +static int sec_aead_decrypt(struct aead_request *a_req) +{ + return sec_aead_crypto(a_req, false); +} + +#define SEC_AEAD_ALG(sec_cra_name, sec_set_key, ctx_init,\ + ctx_exit, blk_size, iv_size, max_authsize)\ +{\ + .base = {\ + .cra_name = sec_cra_name,\ + .cra_driver_name = "hisi_sec_"sec_cra_name,\ + .cra_priority = SEC_PRIORITY,\ + .cra_flags = CRYPTO_ALG_ASYNC |\ + CRYPTO_ALG_NEED_FALLBACK,\ + .cra_blocksize = blk_size,\ + .cra_ctxsize = sizeof(struct sec_ctx),\ + .cra_module = THIS_MODULE,\ + },\ + .init = ctx_init,\ + .exit = ctx_exit,\ + .setkey = sec_set_key,\ + .setauthsize = sec_aead_setauthsize,\ + .decrypt = sec_aead_decrypt,\ + .encrypt = sec_aead_encrypt,\ + .ivsize = iv_size,\ + .maxauthsize = max_authsize,\ +} + +static struct sec_aead sec_aeads[] = { + { + .alg_msk = BIT(6), + .alg = SEC_AEAD_ALG("ccm(aes)", sec_setkey_aes_ccm, sec_aead_xcm_ctx_init, + sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE, + AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(7), + .alg = SEC_AEAD_ALG("gcm(aes)", sec_setkey_aes_gcm, sec_aead_xcm_ctx_init, + sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, SEC_AIV_SIZE, + AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(17), + .alg = SEC_AEAD_ALG("ccm(sm4)", sec_setkey_sm4_ccm, sec_aead_xcm_ctx_init, + sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE, + AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(18), + .alg = SEC_AEAD_ALG("gcm(sm4)", sec_setkey_sm4_gcm, sec_aead_xcm_ctx_init, + sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, SEC_AIV_SIZE, + AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(43), + .alg = SEC_AEAD_ALG("authenc(hmac(sha1),cbc(aes))", sec_setkey_aes_cbc_sha1, + sec_aead_sha1_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE, + AES_BLOCK_SIZE, SHA1_DIGEST_SIZE), + }, + { + .alg_msk = BIT(44), + .alg = SEC_AEAD_ALG("authenc(hmac(sha256),cbc(aes))", sec_setkey_aes_cbc_sha256, + sec_aead_sha256_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE, + AES_BLOCK_SIZE, SHA256_DIGEST_SIZE), + }, + { + .alg_msk = BIT(45), + .alg = SEC_AEAD_ALG("authenc(hmac(sha512),cbc(aes))", sec_setkey_aes_cbc_sha512, + sec_aead_sha512_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE, + AES_BLOCK_SIZE, SHA512_DIGEST_SIZE), + }, +}; + +static void sec_unregister_skcipher(u64 alg_mask, int end) +{ + int i; + + for (i = 0; i < end; i++) + if (sec_skciphers[i].alg_msk & alg_mask) + crypto_unregister_skcipher(&sec_skciphers[i].alg); +} + +static int sec_register_skcipher(u64 alg_mask) +{ + int i, ret, count; + + count = ARRAY_SIZE(sec_skciphers); + + for (i = 0; i < count; i++) { + if (!(sec_skciphers[i].alg_msk & alg_mask)) + continue; + + ret = crypto_register_skcipher(&sec_skciphers[i].alg); + if (ret) + goto err; + } + + return 0; + +err: + sec_unregister_skcipher(alg_mask, i); + + return ret; +} + +static void sec_unregister_aead(u64 alg_mask, int end) +{ + int i; + + for (i = 0; i < end; i++) + if (sec_aeads[i].alg_msk & alg_mask) + crypto_unregister_aead(&sec_aeads[i].alg); +} + +static int sec_register_aead(u64 alg_mask) +{ + int i, ret, count; + + count = ARRAY_SIZE(sec_aeads); + + for (i = 0; i < count; i++) { + if (!(sec_aeads[i].alg_msk & alg_mask)) + continue; + + ret = crypto_register_aead(&sec_aeads[i].alg); + if (ret) + goto err; + } + + return 0; + +err: + sec_unregister_aead(alg_mask, i); + + return ret; +} + +int sec_register_to_crypto(struct hisi_qm *qm) +{ + u64 alg_mask; + int ret = 0; + + alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_TB, + SEC_DRV_ALG_BITMAP_LOW_TB); + + mutex_lock(&sec_algs_lock); + if (sec_available_devs) { + sec_available_devs++; + goto unlock; + } + + ret = sec_register_skcipher(alg_mask); + if (ret) + goto unlock; + + ret = sec_register_aead(alg_mask); + if (ret) + goto unreg_skcipher; + + sec_available_devs++; + mutex_unlock(&sec_algs_lock); + + return 0; + +unreg_skcipher: + sec_unregister_skcipher(alg_mask, ARRAY_SIZE(sec_skciphers)); +unlock: + mutex_unlock(&sec_algs_lock); + return ret; +} + +void sec_unregister_from_crypto(struct hisi_qm *qm) +{ + u64 alg_mask; + + alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_TB, + SEC_DRV_ALG_BITMAP_LOW_TB); + + mutex_lock(&sec_algs_lock); + if (--sec_available_devs) + goto unlock; + + sec_unregister_aead(alg_mask, ARRAY_SIZE(sec_aeads)); + sec_unregister_skcipher(alg_mask, ARRAY_SIZE(sec_skciphers)); + +unlock: + mutex_unlock(&sec_algs_lock); +} diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.h b/drivers/crypto/hisilicon/sec2/sec_crypto.h new file mode 100644 index 000000000..04725b514 --- /dev/null +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.h @@ -0,0 +1,397 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2019 HiSilicon Limited. */ + +#ifndef __HISI_SEC_V2_CRYPTO_H +#define __HISI_SEC_V2_CRYPTO_H + +#define SEC_AIV_SIZE 12 +#define SEC_IV_SIZE 24 +#define SEC_MAX_KEY_SIZE 64 +#define SEC_MAX_AKEY_SIZE 128 +#define SEC_COMM_SCENE 0 +#define SEC_MIN_BLOCK_SZ 1 + +enum sec_calg { + SEC_CALG_3DES = 0x1, + SEC_CALG_AES = 0x2, + SEC_CALG_SM4 = 0x3, +}; + +enum sec_hash_alg { + SEC_A_HMAC_SHA1 = 0x10, + SEC_A_HMAC_SHA256 = 0x11, + SEC_A_HMAC_SHA512 = 0x15, +}; + +enum sec_cmode { + SEC_CMODE_ECB = 0x0, + SEC_CMODE_CBC = 0x1, + SEC_CMODE_CTR = 0x4, + SEC_CMODE_CCM = 0x5, + SEC_CMODE_GCM = 0x6, + SEC_CMODE_XTS = 0x7, +}; + +enum sec_ckey_type { + SEC_CKEY_128BIT = 0x0, + SEC_CKEY_192BIT = 0x1, + SEC_CKEY_256BIT = 0x2, + SEC_CKEY_3DES_3KEY = 0x1, + SEC_CKEY_3DES_2KEY = 0x3, +}; + +enum sec_bd_type { + SEC_BD_TYPE1 = 0x1, + SEC_BD_TYPE2 = 0x2, + SEC_BD_TYPE3 = 0x3, +}; + +enum sec_auth { + SEC_NO_AUTH = 0x0, + SEC_AUTH_TYPE1 = 0x1, + SEC_AUTH_TYPE2 = 0x2, +}; + +enum sec_cipher_dir { + SEC_CIPHER_ENC = 0x1, + SEC_CIPHER_DEC = 0x2, +}; + +enum sec_addr_type { + SEC_PBUF = 0x0, + SEC_SGL = 0x1, + SEC_PRP = 0x2, +}; + +struct bd_status { + u64 tag; + u8 done; + u8 err_type; + u16 flag; + u16 icv; +}; + +enum { + AUTHPAD_PAD, + AUTHPAD_NOPAD, +}; + +enum { + AIGEN_GEN, + AIGEN_NOGEN, +}; + +struct sec_sqe_type2 { + /* + * mac_len: 0~4 bits + * a_key_len: 5~10 bits + * a_alg: 11~16 bits + */ + __le32 mac_key_alg; + + /* + * c_icv_len: 0~5 bits + * c_width: 6~8 bits + * c_key_len: 9~11 bits + * c_mode: 12~15 bits + */ + __le16 icvw_kmode; + + /* c_alg: 0~3 bits */ + __u8 c_alg; + __u8 rsvd4; + + /* + * a_len: 0~23 bits + * iv_offset_l: 24~31 bits + */ + __le32 alen_ivllen; + + /* + * c_len: 0~23 bits + * iv_offset_h: 24~31 bits + */ + __le32 clen_ivhlen; + + __le16 auth_src_offset; + __le16 cipher_src_offset; + __le16 cs_ip_header_offset; + __le16 cs_udp_header_offset; + __le16 pass_word_len; + __le16 dk_len; + __u8 salt3; + __u8 salt2; + __u8 salt1; + __u8 salt0; + + __le16 tag; + __le16 rsvd5; + + /* + * c_pad_type: 0~3 bits + * c_pad_len: 4~11 bits + * c_pad_data_type: 12~15 bits + */ + __le16 cph_pad; + + /* c_pad_len_field: 0~1 bits */ + __le16 c_pad_len_field; + + __le64 long_a_data_len; + __le64 a_ivin_addr; + __le64 a_key_addr; + __le64 mac_addr; + __le64 c_ivin_addr; + __le64 c_key_addr; + + __le64 data_src_addr; + __le64 data_dst_addr; + + /* + * done: 0 bit + * icv: 1~3 bits + * csc: 4~6 bits + * flag: 7-10 bits + * dif_check: 11~13 bits + */ + __le16 done_flag; + + __u8 error_type; + __u8 warning_type; + __u8 mac_i3; + __u8 mac_i2; + __u8 mac_i1; + __u8 mac_i0; + __le16 check_sum_i; + __u8 tls_pad_len_i; + __u8 rsvd12; + __le32 counter; +}; + +struct sec_sqe { + /* + * type: 0~3 bits + * cipher: 4~5 bits + * auth: 6~7 bit s + */ + __u8 type_cipher_auth; + + /* + * seq: 0 bit + * de: 1~2 bits + * scene: 3~6 bits + * src_addr_type: ~7 bit, with sdm_addr_type 0-1 bits + */ + __u8 sds_sa_type; + + /* + * src_addr_type: 0~1 bits, not used now, + * if support PRP, set this field, or set zero. + * dst_addr_type: 2~4 bits + * mac_addr_type: 5~7 bits + */ + __u8 sdm_addr_type; + __u8 rsvd0; + + /* + * nonce_len(type2): 0~3 bits + * huk(type2): 4 bit + * key_s(type2): 5 bit + * ci_gen: 6~7 bits + */ + __u8 huk_key_ci; + + /* + * ai_gen: 0~1 bits + * a_pad(type2): 2~3 bits + * c_s(type2): 4~5 bits + */ + __u8 ai_apd_cs; + + /* + * rhf(type2): 0 bit + * c_key_type: 1~2 bits + * a_key_type: 3~4 bits + * write_frame_len(type2): 5~7 bits + */ + __u8 rca_key_frm; + + /* + * cal_iv_addr_en(type2): 0 bit + * tls_up(type2): 1 bit + * inveld: 7 bit + */ + __u8 iv_tls_ld; + + /* Just using type2 BD now */ + struct sec_sqe_type2 type2; +}; + +struct bd3_auth_ivin { + __le64 a_ivin_addr; + __le32 rsvd0; + __le32 rsvd1; +} __packed __aligned(4); + +struct bd3_skip_data { + __le32 rsvd0; + + /* + * gran_num: 0~15 bits + * reserved: 16~31 bits + */ + __le32 gran_num; + + /* + * src_skip_data_len: 0~24 bits + * reserved: 25~31 bits + */ + __le32 src_skip_data_len; + + /* + * dst_skip_data_len: 0~24 bits + * reserved: 25~31 bits + */ + __le32 dst_skip_data_len; +}; + +struct bd3_stream_scene { + __le64 c_ivin_addr; + __le64 long_a_data_len; + + /* + * auth_pad: 0~1 bits + * stream_protocol: 2~4 bits + * reserved: 5~7 bits + */ + __u8 stream_auth_pad; + __u8 plaintext_type; + __le16 pad_len_1p3; +} __packed __aligned(4); + +struct bd3_no_scene { + __le64 c_ivin_addr; + __le32 rsvd0; + __le32 rsvd1; + __le32 rsvd2; +} __packed __aligned(4); + +struct bd3_check_sum { + __u8 rsvd0; + __u8 hac_sva_status; + __le16 check_sum_i; +}; + +struct bd3_tls_type_back { + __u8 tls_1p3_type_back; + __u8 hac_sva_status; + __le16 pad_len_1p3_back; +}; + +struct sec_sqe3 { + /* + * type: 0~3 bit + * bd_invalid: 4 bit + * scene: 5~8 bit + * de: 9~10 bit + * src_addr_type: 11~13 bit + * dst_addr_type: 14~16 bit + * mac_addr_type: 17~19 bit + * reserved: 20~31 bits + */ + __le32 bd_param; + + /* + * cipher: 0~1 bits + * ci_gen: 2~3 bit + * c_icv_len: 4~9 bit + * c_width: 10~12 bits + * c_key_len: 13~15 bits + */ + __le16 c_icv_key; + + /* + * c_mode : 0~3 bits + * c_alg : 4~7 bits + */ + __u8 c_mode_alg; + + /* + * nonce_len : 0~3 bits + * huk : 4 bits + * cal_iv_addr_en : 5 bits + * seq : 6 bits + * reserved : 7 bits + */ + __u8 huk_iv_seq; + + __le64 tag; + __le64 data_src_addr; + __le64 a_key_addr; + union { + struct bd3_auth_ivin auth_ivin; + struct bd3_skip_data skip_data; + }; + + __le64 c_key_addr; + + /* + * auth: 0~1 bits + * ai_gen: 2~3 bits + * mac_len: 4~8 bits + * akey_len: 9~14 bits + * a_alg: 15~20 bits + * key_sel: 21~24 bits + * ctr_count_mode/sm4_xts: 25~26 bits + * sva_prefetch: 27 bits + * key_wrap_num: 28~30 bits + * update_key: 31 bits + */ + __le32 auth_mac_key; + __le32 salt; + __le16 auth_src_offset; + __le16 cipher_src_offset; + + /* + * auth_len: 0~23 bit + * auth_key_offset: 24~31 bits + */ + __le32 a_len_key; + + /* + * cipher_len: 0~23 bit + * auth_ivin_offset: 24~31 bits + */ + __le32 c_len_ivin; + __le64 data_dst_addr; + __le64 mac_addr; + union { + struct bd3_stream_scene stream_scene; + struct bd3_no_scene no_scene; + }; + + /* + * done: 0 bit + * icv: 1~3 bit + * csc: 4~6 bit + * flag: 7~10 bit + * reserved: 11~15 bit + */ + __le16 done_flag; + __u8 error_type; + __u8 warning_type; + union { + __le32 mac_i; + __le32 kek_key_addr_l; + }; + union { + __le32 kek_key_addr_h; + struct bd3_check_sum check_sum; + struct bd3_tls_type_back tls_type_back; + }; + __le32 counter; +} __packed __aligned(4); + +int sec_register_to_crypto(struct hisi_qm *qm); +void sec_unregister_from_crypto(struct hisi_qm *qm); +#endif diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c new file mode 100644 index 000000000..752565200 --- /dev/null +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -0,0 +1,1575 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2019 HiSilicon Limited. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "sec.h" + +#define CAP_FILE_PERMISSION 0444 +#define SEC_VF_NUM 63 +#define SEC_QUEUE_NUM_V1 4096 +#define PCI_DEVICE_ID_HUAWEI_SEC_PF 0xa255 + +#define SEC_BD_ERR_CHK_EN0 0xEFFFFFFF +#define SEC_BD_ERR_CHK_EN1 0x7ffff7fd +#define SEC_BD_ERR_CHK_EN3 0xffffbfff + +#define SEC_SQE_SIZE 128 +#define SEC_PF_DEF_Q_NUM 256 +#define SEC_PF_DEF_Q_BASE 0 +#define SEC_CTX_Q_NUM_DEF 2 +#define SEC_CTX_Q_NUM_MAX 32 + +#define SEC_CTRL_CNT_CLR_CE 0x301120 +#define SEC_CTRL_CNT_CLR_CE_BIT BIT(0) +#define SEC_CORE_INT_SOURCE 0x301010 +#define SEC_CORE_INT_MASK 0x301000 +#define SEC_CORE_INT_STATUS 0x301008 +#define SEC_CORE_SRAM_ECC_ERR_INFO 0x301C14 +#define SEC_ECC_NUM 16 +#define SEC_ECC_MASH 0xFF +#define SEC_CORE_INT_DISABLE 0x0 + +#define SEC_RAS_CE_REG 0x301050 +#define SEC_RAS_FE_REG 0x301054 +#define SEC_RAS_NFE_REG 0x301058 +#define SEC_RAS_FE_ENB_MSK 0x0 +#define SEC_OOO_SHUTDOWN_SEL 0x301014 +#define SEC_RAS_DISABLE 0x0 +#define SEC_AXI_ERROR_MASK (BIT(0) | BIT(1)) +#define SEC_RAS_CLEAR_ALL GENMASK(31, 0) + +#define SEC_MEM_START_INIT_REG 0x301100 +#define SEC_MEM_INIT_DONE_REG 0x301104 + +/* clock gating */ +#define SEC_CONTROL_REG 0x301200 +#define SEC_DYNAMIC_GATE_REG 0x30121c +#define SEC_CORE_AUTO_GATE 0x30212c +#define SEC_DYNAMIC_GATE_EN 0x7fff +#define SEC_CORE_AUTO_GATE_EN GENMASK(3, 0) +#define SEC_CLK_GATE_ENABLE BIT(3) +#define SEC_CLK_GATE_DISABLE (~BIT(3)) + +#define SEC_TRNG_EN_SHIFT 8 +#define SEC_AXI_SHUTDOWN_ENABLE BIT(12) +#define SEC_AXI_SHUTDOWN_DISABLE 0xFFFFEFFF + +#define SEC_INTERFACE_USER_CTRL0_REG 0x301220 +#define SEC_INTERFACE_USER_CTRL1_REG 0x301224 +#define SEC_SAA_EN_REG 0x301270 +#define SEC_BD_ERR_CHK_EN_REG0 0x301380 +#define SEC_BD_ERR_CHK_EN_REG1 0x301384 +#define SEC_BD_ERR_CHK_EN_REG3 0x30138c + +#define SEC_USER0_SMMU_NORMAL (BIT(23) | BIT(15)) +#define SEC_USER1_SMMU_NORMAL (BIT(31) | BIT(23) | BIT(15) | BIT(7)) +#define SEC_USER1_ENABLE_CONTEXT_SSV BIT(24) +#define SEC_USER1_ENABLE_DATA_SSV BIT(16) +#define SEC_USER1_WB_CONTEXT_SSV BIT(8) +#define SEC_USER1_WB_DATA_SSV BIT(0) +#define SEC_USER1_SVA_SET (SEC_USER1_ENABLE_CONTEXT_SSV | \ + SEC_USER1_ENABLE_DATA_SSV | \ + SEC_USER1_WB_CONTEXT_SSV | \ + SEC_USER1_WB_DATA_SSV) +#define SEC_USER1_SMMU_SVA (SEC_USER1_SMMU_NORMAL | SEC_USER1_SVA_SET) +#define SEC_USER1_SMMU_MASK (~SEC_USER1_SVA_SET) +#define SEC_INTERFACE_USER_CTRL0_REG_V3 0x302220 +#define SEC_INTERFACE_USER_CTRL1_REG_V3 0x302224 +#define SEC_USER1_SMMU_NORMAL_V3 (BIT(23) | BIT(17) | BIT(11) | BIT(5)) +#define SEC_USER1_SMMU_MASK_V3 0xFF79E79E +#define SEC_CORE_INT_STATUS_M_ECC BIT(2) + +#define SEC_PREFETCH_CFG 0x301130 +#define SEC_SVA_TRANS 0x301EC4 +#define SEC_PREFETCH_ENABLE (~(BIT(0) | BIT(1) | BIT(11))) +#define SEC_PREFETCH_DISABLE BIT(1) +#define SEC_SVA_DISABLE_READY (BIT(7) | BIT(11)) +#define SEC_SVA_PREFETCH_INFO 0x301ED4 +#define SEC_SVA_STALL_NUM GENMASK(23, 8) +#define SEC_SVA_PREFETCH_NUM GENMASK(2, 0) +#define SEC_WAIT_SVA_READY 500000 +#define SEC_READ_SVA_STATUS_TIMES 3 +#define SEC_WAIT_US_MIN 10 +#define SEC_WAIT_US_MAX 20 +#define SEC_WAIT_QP_US_MIN 1000 +#define SEC_WAIT_QP_US_MAX 2000 +#define SEC_MAX_WAIT_TIMES 2000 + +#define SEC_DELAY_10_US 10 +#define SEC_POLL_TIMEOUT_US 1000 +#define SEC_DBGFS_VAL_MAX_LEN 20 +#define SEC_SINGLE_PORT_MAX_TRANS 0x2060 + +#define SEC_SQE_MASK_OFFSET 16 +#define SEC_SQE_MASK_LEN 108 +#define SEC_SHAPER_TYPE_RATE 400 + +#define SEC_DFX_BASE 0x301000 +#define SEC_DFX_CORE 0x302100 +#define SEC_DFX_COMMON1 0x301600 +#define SEC_DFX_COMMON2 0x301C00 +#define SEC_DFX_BASE_LEN 0x9D +#define SEC_DFX_CORE_LEN 0x32B +#define SEC_DFX_COMMON1_LEN 0x45 +#define SEC_DFX_COMMON2_LEN 0xBA + +#define SEC_ALG_BITMAP_SHIFT 32 + +#define SEC_CIPHER_BITMAP (GENMASK_ULL(5, 0) | GENMASK_ULL(16, 12) | \ + GENMASK(24, 21)) +#define SEC_DIGEST_BITMAP (GENMASK_ULL(11, 8) | GENMASK_ULL(20, 19) | \ + GENMASK_ULL(42, 25)) +#define SEC_AEAD_BITMAP (GENMASK_ULL(7, 6) | GENMASK_ULL(18, 17) | \ + GENMASK_ULL(45, 43)) + +#define SEC_MAX_CHANNEL_NUM 1 + +struct sec_hw_error { + u32 int_msk; + const char *msg; +}; + +struct sec_dfx_item { + const char *name; + u32 offset; +}; + +static const char sec_name[] = "hisi_sec2"; +static struct dentry *sec_debugfs_root; + +static struct hisi_qm_list sec_devices = { + .register_to_crypto = sec_register_to_crypto, + .unregister_from_crypto = sec_unregister_from_crypto, +}; + +static const struct hisi_qm_cap_info sec_basic_info[] = { + {SEC_QM_NFE_MASK_CAP, 0x3124, 0, GENMASK(31, 0), 0x0, 0x1C77, 0x7C77}, + {SEC_QM_RESET_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0xC77, 0x6C77}, + {SEC_QM_OOO_SHUTDOWN_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0x4, 0x6C77}, + {SEC_QM_CE_MASK_CAP, 0x312C, 0, GENMASK(31, 0), 0x0, 0x8, 0x8}, + {SEC_NFE_MASK_CAP, 0x3130, 0, GENMASK(31, 0), 0x0, 0x177, 0x60177}, + {SEC_RESET_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x177, 0x177}, + {SEC_OOO_SHUTDOWN_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x4, 0x177}, + {SEC_CE_MASK_CAP, 0x3138, 0, GENMASK(31, 0), 0x0, 0x88, 0xC088}, + {SEC_CLUSTER_NUM_CAP, 0x313c, 20, GENMASK(3, 0), 0x1, 0x1, 0x1}, + {SEC_CORE_TYPE_NUM_CAP, 0x313c, 16, GENMASK(3, 0), 0x1, 0x1, 0x1}, + {SEC_CORE_NUM_CAP, 0x313c, 8, GENMASK(7, 0), 0x4, 0x4, 0x4}, + {SEC_CORES_PER_CLUSTER_NUM_CAP, 0x313c, 0, GENMASK(7, 0), 0x4, 0x4, 0x4}, + {SEC_CORE_ENABLE_BITMAP, 0x3140, 0, GENMASK(31, 0), 0x17F, 0x17F, 0xF}, + {SEC_DRV_ALG_BITMAP_LOW, 0x3144, 0, GENMASK(31, 0), 0x18050CB, 0x18050CB, 0x18670CF}, + {SEC_DRV_ALG_BITMAP_HIGH, 0x3148, 0, GENMASK(31, 0), 0x395C, 0x395C, 0x395C}, + {SEC_DEV_ALG_BITMAP_LOW, 0x314c, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_DEV_ALG_BITMAP_HIGH, 0x3150, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, + {SEC_CORE1_ALG_BITMAP_LOW, 0x3154, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_CORE1_ALG_BITMAP_HIGH, 0x3158, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, + {SEC_CORE2_ALG_BITMAP_LOW, 0x315c, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_CORE2_ALG_BITMAP_HIGH, 0x3160, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, + {SEC_CORE3_ALG_BITMAP_LOW, 0x3164, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_CORE3_ALG_BITMAP_HIGH, 0x3168, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, + {SEC_CORE4_ALG_BITMAP_LOW, 0x316c, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_CORE4_ALG_BITMAP_HIGH, 0x3170, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, +}; + +static const struct hisi_qm_cap_query_info sec_cap_query_info[] = { + {QM_RAS_NFE_TYPE, "QM_RAS_NFE_TYPE ", 0x3124, 0x0, 0x1C77, 0x7C77}, + {QM_RAS_NFE_RESET, "QM_RAS_NFE_RESET ", 0x3128, 0x0, 0xC77, 0x6C77}, + {QM_RAS_CE_TYPE, "QM_RAS_CE_TYPE ", 0x312C, 0x0, 0x8, 0x8}, + {SEC_RAS_NFE_TYPE, "SEC_RAS_NFE_TYPE ", 0x3130, 0x0, 0x177, 0x60177}, + {SEC_RAS_NFE_RESET, "SEC_RAS_NFE_RESET ", 0x3134, 0x0, 0x177, 0x177}, + {SEC_RAS_CE_TYPE, "SEC_RAS_CE_TYPE ", 0x3138, 0x0, 0x88, 0xC088}, + {SEC_CORE_INFO, "SEC_CORE_INFO ", 0x313c, 0x110404, 0x110404, 0x110404}, + {SEC_CORE_EN, "SEC_CORE_EN ", 0x3140, 0x17F, 0x17F, 0xF}, + {SEC_DRV_ALG_BITMAP_LOW_TB, "SEC_DRV_ALG_BITMAP_LOW ", + 0x3144, 0x18050CB, 0x18050CB, 0x18670CF}, + {SEC_DRV_ALG_BITMAP_HIGH_TB, "SEC_DRV_ALG_BITMAP_HIGH ", + 0x3148, 0x395C, 0x395C, 0x395C}, + {SEC_ALG_BITMAP_LOW, "SEC_ALG_BITMAP_LOW ", + 0x314c, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_ALG_BITMAP_HIGH, "SEC_ALG_BITMAP_HIGH ", 0x3150, 0x3FFF, 0x3FFF, 0x3FFF}, + {SEC_CORE1_BITMAP_LOW, "SEC_CORE1_BITMAP_LOW ", + 0x3154, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_CORE1_BITMAP_HIGH, "SEC_CORE1_BITMAP_HIGH ", 0x3158, 0x3FFF, 0x3FFF, 0x3FFF}, + {SEC_CORE2_BITMAP_LOW, "SEC_CORE2_BITMAP_LOW ", + 0x315c, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_CORE2_BITMAP_HIGH, "SEC_CORE2_BITMAP_HIGH ", 0x3160, 0x3FFF, 0x3FFF, 0x3FFF}, + {SEC_CORE3_BITMAP_LOW, "SEC_CORE3_BITMAP_LOW ", + 0x3164, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_CORE3_BITMAP_HIGH, "SEC_CORE3_BITMAP_HIGH ", 0x3168, 0x3FFF, 0x3FFF, 0x3FFF}, + {SEC_CORE4_BITMAP_LOW, "SEC_CORE4_BITMAP_LOW ", + 0x316c, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_CORE4_BITMAP_HIGH, "SEC_CORE4_BITMAP_HIGH ", 0x3170, 0x3FFF, 0x3FFF, 0x3FFF}, +}; + +static const struct qm_dev_alg sec_dev_algs[] = { { + .alg_msk = SEC_CIPHER_BITMAP, + .alg = "cipher\n", + }, { + .alg_msk = SEC_DIGEST_BITMAP, + .alg = "digest\n", + }, { + .alg_msk = SEC_AEAD_BITMAP, + .alg = "aead\n", + }, +}; + +static const struct sec_hw_error sec_hw_errors[] = { + { + .int_msk = BIT(0), + .msg = "sec_axi_rresp_err_rint" + }, + { + .int_msk = BIT(1), + .msg = "sec_axi_bresp_err_rint" + }, + { + .int_msk = BIT(2), + .msg = "sec_ecc_2bit_err_rint" + }, + { + .int_msk = BIT(3), + .msg = "sec_ecc_1bit_err_rint" + }, + { + .int_msk = BIT(4), + .msg = "sec_req_trng_timeout_rint" + }, + { + .int_msk = BIT(5), + .msg = "sec_fsm_hbeat_rint" + }, + { + .int_msk = BIT(6), + .msg = "sec_channel_req_rng_timeout_rint" + }, + { + .int_msk = BIT(7), + .msg = "sec_bd_err_rint" + }, + { + .int_msk = BIT(8), + .msg = "sec_chain_buff_err_rint" + }, + { + .int_msk = BIT(14), + .msg = "sec_no_secure_access" + }, + { + .int_msk = BIT(15), + .msg = "sec_wrapping_key_auth_err" + }, + { + .int_msk = BIT(16), + .msg = "sec_km_key_crc_fail" + }, + { + .int_msk = BIT(17), + .msg = "sec_axi_poison_err" + }, + { + .int_msk = BIT(18), + .msg = "sec_sva_err" + }, + {} +}; + +static const char * const sec_dbg_file_name[] = { + [SEC_CLEAR_ENABLE] = "clear_enable", +}; + +static struct sec_dfx_item sec_dfx_labels[] = { + {"send_cnt", offsetof(struct sec_dfx, send_cnt)}, + {"recv_cnt", offsetof(struct sec_dfx, recv_cnt)}, + {"send_busy_cnt", offsetof(struct sec_dfx, send_busy_cnt)}, + {"recv_busy_cnt", offsetof(struct sec_dfx, recv_busy_cnt)}, + {"err_bd_cnt", offsetof(struct sec_dfx, err_bd_cnt)}, + {"invalid_req_cnt", offsetof(struct sec_dfx, invalid_req_cnt)}, + {"done_flag_cnt", offsetof(struct sec_dfx, done_flag_cnt)}, +}; + +static const struct debugfs_reg32 sec_dfx_regs[] = { + {"SEC_PF_ABNORMAL_INT_SOURCE ", 0x301010}, + {"SEC_SAA_EN ", 0x301270}, + {"SEC_BD_LATENCY_MIN ", 0x301600}, + {"SEC_BD_LATENCY_MAX ", 0x301608}, + {"SEC_BD_LATENCY_AVG ", 0x30160C}, + {"SEC_BD_NUM_IN_SAA0 ", 0x301670}, + {"SEC_BD_NUM_IN_SAA1 ", 0x301674}, + {"SEC_BD_NUM_IN_SEC ", 0x301680}, + {"SEC_ECC_1BIT_CNT ", 0x301C00}, + {"SEC_ECC_1BIT_INFO ", 0x301C04}, + {"SEC_ECC_2BIT_CNT ", 0x301C10}, + {"SEC_ECC_2BIT_INFO ", 0x301C14}, + {"SEC_BD_SAA0 ", 0x301C20}, + {"SEC_BD_SAA1 ", 0x301C24}, + {"SEC_BD_SAA2 ", 0x301C28}, + {"SEC_BD_SAA3 ", 0x301C2C}, + {"SEC_BD_SAA4 ", 0x301C30}, + {"SEC_BD_SAA5 ", 0x301C34}, + {"SEC_BD_SAA6 ", 0x301C38}, + {"SEC_BD_SAA7 ", 0x301C3C}, + {"SEC_BD_SAA8 ", 0x301C40}, + {"SEC_RAS_CE_ENABLE ", 0x301050}, + {"SEC_RAS_FE_ENABLE ", 0x301054}, + {"SEC_RAS_NFE_ENABLE ", 0x301058}, + {"SEC_REQ_TRNG_TIME_TH ", 0x30112C}, + {"SEC_CHANNEL_RNG_REQ_THLD ", 0x302110}, +}; + +/* define the SEC's dfx regs region and region length */ +static struct dfx_diff_registers sec_diff_regs[] = { + { + .reg_offset = SEC_DFX_BASE, + .reg_len = SEC_DFX_BASE_LEN, + }, { + .reg_offset = SEC_DFX_COMMON1, + .reg_len = SEC_DFX_COMMON1_LEN, + }, { + .reg_offset = SEC_DFX_COMMON2, + .reg_len = SEC_DFX_COMMON2_LEN, + }, { + .reg_offset = SEC_DFX_CORE, + .reg_len = SEC_DFX_CORE_LEN, + }, +}; + +static int sec_diff_regs_show(struct seq_file *s, void *unused) +{ + struct hisi_qm *qm = s->private; + + hisi_qm_acc_diff_regs_dump(qm, s, qm->debug.acc_diff_regs, + ARRAY_SIZE(sec_diff_regs)); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(sec_diff_regs); + +static bool pf_q_num_flag; +static int sec_pf_q_num_set(const char *val, const struct kernel_param *kp) +{ + pf_q_num_flag = true; + + return hisi_qm_q_num_set(val, kp, PCI_DEVICE_ID_HUAWEI_SEC_PF); +} + +static const struct kernel_param_ops sec_pf_q_num_ops = { + .set = sec_pf_q_num_set, + .get = param_get_int, +}; + +static u32 pf_q_num = SEC_PF_DEF_Q_NUM; +module_param_cb(pf_q_num, &sec_pf_q_num_ops, &pf_q_num, 0444); +MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 2-4096, v2 2-1024)"); + +static int sec_ctx_q_num_set(const char *val, const struct kernel_param *kp) +{ + u32 ctx_q_num; + int ret; + + if (!val) + return -EINVAL; + + ret = kstrtou32(val, 10, &ctx_q_num); + if (ret) + return -EINVAL; + + if (!ctx_q_num || ctx_q_num > SEC_CTX_Q_NUM_MAX || ctx_q_num & 0x1) { + pr_err("ctx queue num[%u] is invalid!\n", ctx_q_num); + return -EINVAL; + } + + return param_set_int(val, kp); +} + +static const struct kernel_param_ops sec_ctx_q_num_ops = { + .set = sec_ctx_q_num_set, + .get = param_get_int, +}; +static u32 ctx_q_num = SEC_CTX_Q_NUM_DEF; +module_param_cb(ctx_q_num, &sec_ctx_q_num_ops, &ctx_q_num, 0444); +MODULE_PARM_DESC(ctx_q_num, "Queue num in ctx (2 default, 2, 4, ..., 32)"); + +static const struct kernel_param_ops vfs_num_ops = { + .set = vfs_num_set, + .get = param_get_int, +}; + +static u32 vfs_num; +module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444); +MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)"); + +void sec_destroy_qps(struct hisi_qp **qps, int qp_num) +{ + hisi_qm_free_qps(qps, qp_num); + kfree(qps); +} + +struct hisi_qp **sec_create_qps(void) +{ + int node = cpu_to_node(raw_smp_processor_id()); + u32 ctx_num = ctx_q_num; + struct hisi_qp **qps; + u8 *type; + int ret; + + qps = kzalloc_objs(struct hisi_qp *, ctx_num); + if (!qps) + return NULL; + + /* The type of SEC is all 0, so just allocated by kcalloc */ + type = kcalloc(ctx_num, sizeof(u8), GFP_KERNEL); + if (!type) { + kfree(qps); + return NULL; + } + + ret = hisi_qm_alloc_qps_node(&sec_devices, ctx_num, type, node, qps); + if (ret) { + kfree(type); + kfree(qps); + return NULL; + } + + kfree(type); + return qps; +} + +u64 sec_get_alg_bitmap(struct hisi_qm *qm, u32 high, u32 low) +{ + u32 cap_val_h, cap_val_l; + + cap_val_h = qm->cap_tables.dev_cap_table[high].cap_val; + cap_val_l = qm->cap_tables.dev_cap_table[low].cap_val; + + return ((u64)cap_val_h << SEC_ALG_BITMAP_SHIFT) | (u64)cap_val_l; +} + +static const struct kernel_param_ops sec_uacce_mode_ops = { + .set = uacce_mode_set, + .get = param_get_int, +}; + +/* + * uacce_mode = 0 means sec only register to crypto, + * uacce_mode = 1 means sec both register to crypto and uacce. + */ +static u32 uacce_mode = UACCE_MODE_NOUACCE; +module_param_cb(uacce_mode, &sec_uacce_mode_ops, &uacce_mode, 0444); +MODULE_PARM_DESC(uacce_mode, UACCE_MODE_DESC); + +static const struct pci_device_id sec_dev_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_SEC_PF) }, + { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_SEC_VF) }, + { 0, } +}; +MODULE_DEVICE_TABLE(pci, sec_dev_ids); + +static void sec_set_endian(struct hisi_qm *qm) +{ + u32 reg; + + reg = readl_relaxed(qm->io_base + SEC_CONTROL_REG); + reg &= ~(BIT(1) | BIT(0)); + if (!IS_ENABLED(CONFIG_64BIT)) + reg |= BIT(1); + + if (!IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN)) + reg |= BIT(0); + + writel_relaxed(reg, qm->io_base + SEC_CONTROL_REG); +} + +static int sec_wait_sva_ready(struct hisi_qm *qm, __u32 offset, __u32 mask) +{ + u32 val, try_times = 0; + u8 count = 0; + + /* + * Read the register value every 10-20us. If the value is 0 for three + * consecutive times, the SVA module is ready. + */ + do { + val = readl(qm->io_base + offset); + if (val & mask) + count = 0; + else if (++count == SEC_READ_SVA_STATUS_TIMES) + break; + + usleep_range(SEC_WAIT_US_MIN, SEC_WAIT_US_MAX); + } while (++try_times < SEC_WAIT_SVA_READY); + + if (try_times == SEC_WAIT_SVA_READY) { + pci_err(qm->pdev, "failed to wait sva prefetch ready\n"); + return -ETIMEDOUT; + } + + return 0; +} + +static void sec_close_sva_prefetch(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) + return; + + val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG); + val |= SEC_PREFETCH_DISABLE; + writel(val, qm->io_base + SEC_PREFETCH_CFG); + + ret = readl_relaxed_poll_timeout(qm->io_base + SEC_SVA_TRANS, + val, !(val & SEC_SVA_DISABLE_READY), + SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US); + if (ret) + pci_err(qm->pdev, "failed to close sva prefetch\n"); + + (void)sec_wait_sva_ready(qm, SEC_SVA_PREFETCH_INFO, SEC_SVA_STALL_NUM); +} + +static void sec_open_sva_prefetch(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) + return; + + /* Enable prefetch */ + val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG); + val &= SEC_PREFETCH_ENABLE; + writel(val, qm->io_base + SEC_PREFETCH_CFG); + + ret = readl_relaxed_poll_timeout(qm->io_base + SEC_PREFETCH_CFG, + val, !(val & SEC_PREFETCH_DISABLE), + SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US); + if (ret) { + pci_err(qm->pdev, "failed to open sva prefetch\n"); + sec_close_sva_prefetch(qm); + return; + } + + ret = sec_wait_sva_ready(qm, SEC_SVA_TRANS, SEC_SVA_PREFETCH_NUM); + if (ret) + sec_close_sva_prefetch(qm); +} + +static void sec_engine_sva_config(struct hisi_qm *qm) +{ + u32 reg; + + if (qm->ver > QM_HW_V2) { + reg = readl_relaxed(qm->io_base + + SEC_INTERFACE_USER_CTRL0_REG_V3); + reg |= SEC_USER0_SMMU_NORMAL; + writel_relaxed(reg, qm->io_base + + SEC_INTERFACE_USER_CTRL0_REG_V3); + + reg = readl_relaxed(qm->io_base + + SEC_INTERFACE_USER_CTRL1_REG_V3); + reg &= SEC_USER1_SMMU_MASK_V3; + reg |= SEC_USER1_SMMU_NORMAL_V3; + writel_relaxed(reg, qm->io_base + + SEC_INTERFACE_USER_CTRL1_REG_V3); + } else { + reg = readl_relaxed(qm->io_base + + SEC_INTERFACE_USER_CTRL0_REG); + reg |= SEC_USER0_SMMU_NORMAL; + writel_relaxed(reg, qm->io_base + + SEC_INTERFACE_USER_CTRL0_REG); + reg = readl_relaxed(qm->io_base + + SEC_INTERFACE_USER_CTRL1_REG); + reg &= SEC_USER1_SMMU_MASK; + if (qm->use_sva) + reg |= SEC_USER1_SMMU_SVA; + else + reg |= SEC_USER1_SMMU_NORMAL; + writel_relaxed(reg, qm->io_base + + SEC_INTERFACE_USER_CTRL1_REG); + } + sec_open_sva_prefetch(qm); +} + +static void sec_enable_clock_gate(struct hisi_qm *qm) +{ + u32 val; + + if (qm->ver < QM_HW_V3) + return; + + val = readl_relaxed(qm->io_base + SEC_CONTROL_REG); + val |= SEC_CLK_GATE_ENABLE; + writel_relaxed(val, qm->io_base + SEC_CONTROL_REG); + + val = readl(qm->io_base + SEC_DYNAMIC_GATE_REG); + val |= SEC_DYNAMIC_GATE_EN; + writel(val, qm->io_base + SEC_DYNAMIC_GATE_REG); + + val = readl(qm->io_base + SEC_CORE_AUTO_GATE); + val |= SEC_CORE_AUTO_GATE_EN; + writel(val, qm->io_base + SEC_CORE_AUTO_GATE); +} + +static void sec_disable_clock_gate(struct hisi_qm *qm) +{ + u32 val; + + /* Kunpeng920 needs to close clock gating */ + val = readl_relaxed(qm->io_base + SEC_CONTROL_REG); + val &= SEC_CLK_GATE_DISABLE; + writel_relaxed(val, qm->io_base + SEC_CONTROL_REG); +} + +static int sec_engine_init(struct hisi_qm *qm) +{ + int ret; + u32 reg; + + /* disable clock gate control before mem init */ + sec_disable_clock_gate(qm); + + writel_relaxed(0x1, qm->io_base + SEC_MEM_START_INIT_REG); + + ret = readl_relaxed_poll_timeout(qm->io_base + SEC_MEM_INIT_DONE_REG, + reg, reg & 0x1, SEC_DELAY_10_US, + SEC_POLL_TIMEOUT_US); + if (ret) { + pci_err(qm->pdev, "fail to init sec mem\n"); + return ret; + } + + reg = readl_relaxed(qm->io_base + SEC_CONTROL_REG); + reg |= (0x1 << SEC_TRNG_EN_SHIFT); + writel_relaxed(reg, qm->io_base + SEC_CONTROL_REG); + + sec_engine_sva_config(qm); + + writel(SEC_SINGLE_PORT_MAX_TRANS, + qm->io_base + AM_CFG_SINGLE_PORT_MAX_TRANS); + + reg = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_CORE_ENABLE_BITMAP, qm->cap_ver); + writel(reg, qm->io_base + SEC_SAA_EN_REG); + + if (qm->ver < QM_HW_V3) { + /* HW V2 enable sm4 extra mode, as ctr/ecb */ + writel_relaxed(SEC_BD_ERR_CHK_EN0, + qm->io_base + SEC_BD_ERR_CHK_EN_REG0); + + /* HW V2 enable sm4 xts mode multiple iv */ + writel_relaxed(SEC_BD_ERR_CHK_EN1, + qm->io_base + SEC_BD_ERR_CHK_EN_REG1); + writel_relaxed(SEC_BD_ERR_CHK_EN3, + qm->io_base + SEC_BD_ERR_CHK_EN_REG3); + } + + /* config endian */ + sec_set_endian(qm); + + sec_enable_clock_gate(qm); + + return 0; +} + +static int sec_set_user_domain_and_cache(struct hisi_qm *qm) +{ + /* qm user domain */ + writel(AXUSER_BASE, qm->io_base + QM_ARUSER_M_CFG_1); + writel(ARUSER_M_CFG_ENABLE, qm->io_base + QM_ARUSER_M_CFG_ENABLE); + writel(AXUSER_BASE, qm->io_base + QM_AWUSER_M_CFG_1); + writel(AWUSER_M_CFG_ENABLE, qm->io_base + QM_AWUSER_M_CFG_ENABLE); + writel(WUSER_M_CFG_ENABLE, qm->io_base + QM_WUSER_M_CFG_ENABLE); + + /* qm cache */ + writel(AXI_M_CFG, qm->io_base + QM_AXI_M_CFG); + writel(AXI_M_CFG_ENABLE, qm->io_base + QM_AXI_M_CFG_ENABLE); + + /* disable FLR triggered by BME(bus master enable) */ + writel(PEH_AXUSER_CFG, qm->io_base + QM_PEH_AXUSER_CFG); + writel(PEH_AXUSER_CFG_ENABLE, qm->io_base + QM_PEH_AXUSER_CFG_ENABLE); + + /* enable sqc,cqc writeback */ + writel(SQC_CACHE_ENABLE | CQC_CACHE_ENABLE | SQC_CACHE_WB_ENABLE | + CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) | + FIELD_PREP(CQC_CACHE_WB_THRD, 1), qm->io_base + QM_CACHE_CTL); + + return sec_engine_init(qm); +} + +/* sec_debug_regs_clear() - clear the sec debug regs */ +static void sec_debug_regs_clear(struct hisi_qm *qm) +{ + int i; + + /* clear sec dfx regs */ + writel(0x1, qm->io_base + SEC_CTRL_CNT_CLR_CE); + for (i = 0; i < ARRAY_SIZE(sec_dfx_regs); i++) + readl(qm->io_base + sec_dfx_regs[i].offset); + + /* clear rdclr_en */ + writel(0x0, qm->io_base + SEC_CTRL_CNT_CLR_CE); + + hisi_qm_debug_regs_clear(qm); +} + +static void sec_master_ooo_ctrl(struct hisi_qm *qm, bool enable) +{ + u32 val1, val2; + + val1 = readl(qm->io_base + SEC_CONTROL_REG); + if (enable) { + val1 |= SEC_AXI_SHUTDOWN_ENABLE; + val2 = qm->err_info.dev_err.shutdown_mask; + } else { + val1 &= SEC_AXI_SHUTDOWN_DISABLE; + val2 = 0x0; + } + + if (qm->ver > QM_HW_V2) + writel(val2, qm->io_base + SEC_OOO_SHUTDOWN_SEL); + + writel(val1, qm->io_base + SEC_CONTROL_REG); +} + +static void sec_hw_error_enable(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; + + if (qm->ver == QM_HW_V1) { + writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK); + pci_info(qm->pdev, "V1 not support hw error handle\n"); + return; + } + + /* clear SEC hw error source if having */ + writel(SEC_RAS_CLEAR_ALL, qm->io_base + SEC_CORE_INT_SOURCE); + + /* enable RAS int */ + writel(dev_err->ce, qm->io_base + SEC_RAS_CE_REG); + writel(dev_err->fe, qm->io_base + SEC_RAS_FE_REG); + writel(dev_err->nfe, qm->io_base + SEC_RAS_NFE_REG); + + /* enable SEC block master OOO when nfe occurs on Kunpeng930 */ + sec_master_ooo_ctrl(qm, true); + + /* enable SEC hw error interrupts */ + writel(err_mask, qm->io_base + SEC_CORE_INT_MASK); +} + +static void sec_hw_error_disable(struct hisi_qm *qm) +{ + /* disable SEC hw error interrupts */ + writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK); + + /* disable SEC block master OOO when nfe occurs on Kunpeng930 */ + sec_master_ooo_ctrl(qm, false); + + /* disable RAS int */ + writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_CE_REG); + writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_FE_REG); + writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_NFE_REG); +} + +static u32 sec_clear_enable_read(struct hisi_qm *qm) +{ + return readl(qm->io_base + SEC_CTRL_CNT_CLR_CE) & + SEC_CTRL_CNT_CLR_CE_BIT; +} + +static int sec_clear_enable_write(struct hisi_qm *qm, u32 val) +{ + u32 tmp; + + if (val != 1 && val) + return -EINVAL; + + tmp = (readl(qm->io_base + SEC_CTRL_CNT_CLR_CE) & + ~SEC_CTRL_CNT_CLR_CE_BIT) | val; + writel(tmp, qm->io_base + SEC_CTRL_CNT_CLR_CE); + + return 0; +} + +static ssize_t sec_debug_read(struct file *filp, char __user *buf, + size_t count, loff_t *pos) +{ + struct sec_debug_file *file = filp->private_data; + char tbuf[SEC_DBGFS_VAL_MAX_LEN]; + struct hisi_qm *qm = file->qm; + u32 val; + int ret; + + ret = hisi_qm_get_dfx_access(qm); + if (ret) + return ret; + + spin_lock_irq(&file->lock); + + switch (file->index) { + case SEC_CLEAR_ENABLE: + val = sec_clear_enable_read(qm); + break; + default: + goto err_input; + } + + spin_unlock_irq(&file->lock); + + hisi_qm_put_dfx_access(qm); + ret = snprintf(tbuf, SEC_DBGFS_VAL_MAX_LEN, "%u\n", val); + return simple_read_from_buffer(buf, count, pos, tbuf, ret); + +err_input: + spin_unlock_irq(&file->lock); + hisi_qm_put_dfx_access(qm); + return -EINVAL; +} + +static ssize_t sec_debug_write(struct file *filp, const char __user *buf, + size_t count, loff_t *pos) +{ + struct sec_debug_file *file = filp->private_data; + char tbuf[SEC_DBGFS_VAL_MAX_LEN]; + struct hisi_qm *qm = file->qm; + unsigned long val; + int len, ret; + + if (*pos != 0) + return 0; + + if (count >= SEC_DBGFS_VAL_MAX_LEN) + return -ENOSPC; + + len = simple_write_to_buffer(tbuf, SEC_DBGFS_VAL_MAX_LEN - 1, + pos, buf, count); + if (len < 0) + return len; + + tbuf[len] = '\0'; + if (kstrtoul(tbuf, 0, &val)) + return -EFAULT; + + ret = hisi_qm_get_dfx_access(qm); + if (ret) + return ret; + + spin_lock_irq(&file->lock); + + switch (file->index) { + case SEC_CLEAR_ENABLE: + ret = sec_clear_enable_write(qm, val); + if (ret) + goto err_input; + break; + default: + ret = -EINVAL; + goto err_input; + } + + ret = count; + + err_input: + spin_unlock_irq(&file->lock); + hisi_qm_put_dfx_access(qm); + return ret; +} + +static const struct file_operations sec_dbg_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = sec_debug_read, + .write = sec_debug_write, +}; + +static int sec_debugfs_atomic64_get(void *data, u64 *val) +{ + *val = atomic64_read((atomic64_t *)data); + + return 0; +} + +static int sec_debugfs_atomic64_set(void *data, u64 val) +{ + if (val) + return -EINVAL; + + atomic64_set((atomic64_t *)data, 0); + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(sec_atomic64_ops, sec_debugfs_atomic64_get, + sec_debugfs_atomic64_set, "%llu\n"); + +static int sec_regs_show(struct seq_file *s, void *unused) +{ + hisi_qm_regs_dump(s, s->private); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(sec_regs); + +static int sec_cap_regs_show(struct seq_file *s, void *unused) +{ + struct hisi_qm *qm = s->private; + u32 i, size; + + size = qm->cap_tables.qm_cap_size; + for (i = 0; i < size; i++) + seq_printf(s, "%s= 0x%08x\n", qm->cap_tables.qm_cap_table[i].name, + qm->cap_tables.qm_cap_table[i].cap_val); + + size = qm->cap_tables.dev_cap_size; + for (i = 0; i < size; i++) + seq_printf(s, "%s= 0x%08x\n", qm->cap_tables.dev_cap_table[i].name, + qm->cap_tables.dev_cap_table[i].cap_val); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(sec_cap_regs); + +static int sec_core_debug_init(struct hisi_qm *qm) +{ + struct dfx_diff_registers *sec_regs = qm->debug.acc_diff_regs; + struct sec_dev *sec = container_of(qm, struct sec_dev, qm); + struct device *dev = &qm->pdev->dev; + struct sec_dfx *dfx = &sec->debug.dfx; + struct debugfs_regset32 *regset; + struct dentry *tmp_d; + int i; + + tmp_d = debugfs_create_dir("sec_dfx", qm->debug.debug_root); + + regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL); + if (!regset) + return -ENOMEM; + + regset->regs = sec_dfx_regs; + regset->nregs = ARRAY_SIZE(sec_dfx_regs); + regset->base = qm->io_base; + regset->dev = dev; + + if (qm->pdev->device == PCI_DEVICE_ID_HUAWEI_SEC_PF) + debugfs_create_file("regs", 0444, tmp_d, regset, &sec_regs_fops); + if (qm->fun_type == QM_HW_PF && sec_regs) + debugfs_create_file("diff_regs", 0444, tmp_d, + qm, &sec_diff_regs_fops); + + for (i = 0; i < ARRAY_SIZE(sec_dfx_labels); i++) { + atomic64_t *data = (atomic64_t *)((uintptr_t)dfx + + sec_dfx_labels[i].offset); + debugfs_create_file(sec_dfx_labels[i].name, 0644, + tmp_d, data, &sec_atomic64_ops); + } + + debugfs_create_file("cap_regs", CAP_FILE_PERMISSION, + qm->debug.debug_root, qm, &sec_cap_regs_fops); + + return 0; +} + +static int sec_debug_init(struct hisi_qm *qm) +{ + struct sec_dev *sec = container_of(qm, struct sec_dev, qm); + int i; + + if (qm->pdev->device == PCI_DEVICE_ID_HUAWEI_SEC_PF) { + for (i = SEC_CLEAR_ENABLE; i < SEC_DEBUG_FILE_NUM; i++) { + spin_lock_init(&sec->debug.files[i].lock); + sec->debug.files[i].index = i; + sec->debug.files[i].qm = qm; + + debugfs_create_file(sec_dbg_file_name[i], 0600, + qm->debug.debug_root, + sec->debug.files + i, + &sec_dbg_fops); + } + } + + return sec_core_debug_init(qm); +} + +static int sec_debugfs_init(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + int ret; + + ret = hisi_qm_regs_debugfs_init(qm, sec_diff_regs, ARRAY_SIZE(sec_diff_regs)); + if (ret) { + dev_warn(dev, "Failed to init SEC diff regs!\n"); + return ret; + } + + qm->debug.debug_root = debugfs_create_dir(dev_name(dev), + sec_debugfs_root); + qm->debug.sqe_mask_offset = SEC_SQE_MASK_OFFSET; + qm->debug.sqe_mask_len = SEC_SQE_MASK_LEN; + + hisi_qm_debug_init(qm); + + ret = sec_debug_init(qm); + if (ret) + goto debugfs_remove; + + return 0; + +debugfs_remove: + debugfs_remove_recursive(qm->debug.debug_root); + hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(sec_diff_regs)); + return ret; +} + +static void sec_debugfs_exit(struct hisi_qm *qm) +{ + debugfs_remove_recursive(qm->debug.debug_root); + + hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(sec_diff_regs)); +} + +static int sec_show_last_regs_init(struct hisi_qm *qm) +{ + struct qm_debug *debug = &qm->debug; + int i; + + debug->last_words = kcalloc(ARRAY_SIZE(sec_dfx_regs), + sizeof(unsigned int), GFP_KERNEL); + if (!debug->last_words) + return -ENOMEM; + + for (i = 0; i < ARRAY_SIZE(sec_dfx_regs); i++) + debug->last_words[i] = readl_relaxed(qm->io_base + + sec_dfx_regs[i].offset); + + return 0; +} + +static void sec_show_last_regs_uninit(struct hisi_qm *qm) +{ + struct qm_debug *debug = &qm->debug; + + if (qm->fun_type == QM_HW_VF || !debug->last_words) + return; + + kfree(debug->last_words); + debug->last_words = NULL; +} + +static void sec_show_last_dfx_regs(struct hisi_qm *qm) +{ + struct qm_debug *debug = &qm->debug; + struct pci_dev *pdev = qm->pdev; + u32 val; + int i; + + if (qm->fun_type == QM_HW_VF || !debug->last_words) + return; + + /* dumps last word of the debugging registers during controller reset */ + for (i = 0; i < ARRAY_SIZE(sec_dfx_regs); i++) { + val = readl_relaxed(qm->io_base + sec_dfx_regs[i].offset); + if (val != debug->last_words[i]) + pci_info(pdev, "%s \t= 0x%08x => 0x%08x\n", + sec_dfx_regs[i].name, debug->last_words[i], val); + } +} + +static void sec_log_hw_error(struct hisi_qm *qm, u32 err_sts) +{ + const struct sec_hw_error *errs = sec_hw_errors; + struct device *dev = &qm->pdev->dev; + u32 err_val; + + while (errs->msg) { + if (errs->int_msk & err_sts) { + dev_err(dev, "%s [error status=0x%x] found\n", + errs->msg, errs->int_msk); + + if (SEC_CORE_INT_STATUS_M_ECC & errs->int_msk) { + err_val = readl(qm->io_base + + SEC_CORE_SRAM_ECC_ERR_INFO); + dev_err(dev, "multi ecc sram num=0x%x\n", + ((err_val) >> SEC_ECC_NUM) & + SEC_ECC_MASH); + } + } + errs++; + } +} + +static u32 sec_get_hw_err_status(struct hisi_qm *qm) +{ + return readl(qm->io_base + SEC_CORE_INT_STATUS); +} + +static void sec_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) +{ + writel(err_sts, qm->io_base + SEC_CORE_INT_SOURCE); +} + +static void sec_disable_error_report(struct hisi_qm *qm, u32 err_type) +{ + u32 nfe_mask = qm->err_info.dev_err.nfe; + + writel(nfe_mask & (~err_type), qm->io_base + SEC_RAS_NFE_REG); +} + +static void sec_enable_error_report(struct hisi_qm *qm) +{ + u32 nfe_mask = qm->err_info.dev_err.nfe; + u32 ce_mask = qm->err_info.dev_err.ce; + + writel(nfe_mask, qm->io_base + SEC_RAS_NFE_REG); + writel(ce_mask, qm->io_base + SEC_RAS_CE_REG); +} + +static void sec_open_axi_master_ooo(struct hisi_qm *qm) +{ + u32 val; + + val = readl(qm->io_base + SEC_CONTROL_REG); + writel(val & SEC_AXI_SHUTDOWN_DISABLE, qm->io_base + SEC_CONTROL_REG); + writel(val | SEC_AXI_SHUTDOWN_ENABLE, qm->io_base + SEC_CONTROL_REG); +} + +static enum acc_err_result sec_get_err_result(struct hisi_qm *qm) +{ + u32 err_status; + + err_status = sec_get_hw_err_status(qm); + if (err_status) { + if (err_status & qm->err_info.dev_err.ecc_2bits_mask) + qm->err_status.is_dev_ecc_mbit = true; + sec_log_hw_error(qm, err_status); + + if (err_status & qm->err_info.dev_err.reset_mask) { + /* Disable the same error reporting until device is recovered. */ + sec_disable_error_report(qm, err_status); + return ACC_ERR_NEED_RESET; + } + sec_clear_hw_err_status(qm, err_status); + /* Avoid firmware disable error report, re-enable. */ + sec_enable_error_report(qm); + } + + return ACC_ERR_RECOVERED; +} + +static bool sec_dev_is_abnormal(struct hisi_qm *qm) +{ + u32 err_status; + + err_status = sec_get_hw_err_status(qm); + if (err_status & qm->err_info.dev_err.shutdown_mask) + return true; + + return false; +} + +static void sec_disable_axi_error(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; + + writel(err_mask & ~SEC_AXI_ERROR_MASK, qm->io_base + SEC_CORE_INT_MASK); + + if (qm->ver > QM_HW_V2) + writel(dev_err->shutdown_mask & (~SEC_AXI_ERROR_MASK), + qm->io_base + SEC_OOO_SHUTDOWN_SEL); +} + +static void sec_enable_axi_error(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; + + /* clear axi error source */ + writel(SEC_AXI_ERROR_MASK, qm->io_base + SEC_CORE_INT_SOURCE); + + writel(err_mask, qm->io_base + SEC_CORE_INT_MASK); + + if (qm->ver > QM_HW_V2) + writel(dev_err->shutdown_mask, qm->io_base + SEC_OOO_SHUTDOWN_SEL); +} + +static void sec_err_info_init(struct hisi_qm *qm) +{ + struct hisi_qm_err_info *err_info = &qm->err_info; + struct hisi_qm_err_mask *qm_err = &err_info->qm_err; + struct hisi_qm_err_mask *dev_err = &err_info->dev_err; + + qm_err->fe = SEC_RAS_FE_ENB_MSK; + qm_err->ce = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_CE_MASK_CAP, qm->cap_ver); + qm_err->nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_NFE_MASK_CAP, qm->cap_ver); + qm_err->shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info, + SEC_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + qm_err->reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info, + SEC_QM_RESET_MASK_CAP, qm->cap_ver); + qm_err->ecc_2bits_mask = QM_ECC_MBIT; + + dev_err->fe = SEC_RAS_FE_ENB_MSK; + dev_err->ce = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_CE_MASK_CAP, qm->cap_ver); + dev_err->nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_NFE_MASK_CAP, qm->cap_ver); + dev_err->shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info, + SEC_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + dev_err->reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info, + SEC_RESET_MASK_CAP, qm->cap_ver); + dev_err->ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC; + + err_info->msi_wr_port = BIT(0); + err_info->acpi_rst = "SRST"; +} + +static const struct hisi_qm_err_ini sec_err_ini = { + .hw_init = sec_set_user_domain_and_cache, + .hw_err_enable = sec_hw_error_enable, + .hw_err_disable = sec_hw_error_disable, + .get_dev_hw_err_status = sec_get_hw_err_status, + .clear_dev_hw_err_status = sec_clear_hw_err_status, + .open_axi_master_ooo = sec_open_axi_master_ooo, + .open_sva_prefetch = sec_open_sva_prefetch, + .close_sva_prefetch = sec_close_sva_prefetch, + .show_last_dfx_regs = sec_show_last_dfx_regs, + .err_info_init = sec_err_info_init, + .get_err_result = sec_get_err_result, + .dev_is_abnormal = sec_dev_is_abnormal, + .disable_axi_error = sec_disable_axi_error, + .enable_axi_error = sec_enable_axi_error, +}; + +static int sec_pf_probe_init(struct sec_dev *sec) +{ + struct hisi_qm *qm = &sec->qm; + int ret; + + ret = sec_set_user_domain_and_cache(qm); + if (ret) + return ret; + + hisi_qm_dev_err_init(qm); + sec_debug_regs_clear(qm); + ret = sec_show_last_regs_init(qm); + if (ret) + pci_err(qm->pdev, "Failed to init last word regs!\n"); + + return ret; +} + +static int sec_pre_store_cap_reg(struct hisi_qm *qm) +{ + struct hisi_qm_cap_record *sec_cap; + struct pci_dev *pdev = qm->pdev; + size_t i, size; + + size = ARRAY_SIZE(sec_cap_query_info); + sec_cap = devm_kcalloc(&pdev->dev, size, sizeof(*sec_cap), GFP_KERNEL); + if (!sec_cap) + return -ENOMEM; + + for (i = 0; i < size; i++) { + sec_cap[i].type = sec_cap_query_info[i].type; + sec_cap[i].name = sec_cap_query_info[i].name; + sec_cap[i].cap_val = hisi_qm_get_cap_value(qm, sec_cap_query_info, + i, qm->cap_ver); + } + + qm->cap_tables.dev_cap_table = sec_cap; + qm->cap_tables.dev_cap_size = size; + + return 0; +} + +static void sec_set_channels(struct hisi_qm *qm) +{ + struct qm_channel *channel_data = &qm->channel_data; + + channel_data->channel_num = SEC_MAX_CHANNEL_NUM; + channel_data->channel_name[0] = "SEC"; +} + +static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) +{ + u64 alg_msk; + int ret; + + qm->pdev = pdev; + qm->mode = uacce_mode; + qm->sqe_size = SEC_SQE_SIZE; + qm->dev_name = sec_name; + + qm->fun_type = (pdev->device == PCI_DEVICE_ID_HUAWEI_SEC_PF) ? + QM_HW_PF : QM_HW_VF; + if (qm->fun_type == QM_HW_PF) { + qm->qp_base = SEC_PF_DEF_Q_BASE; + qm->qp_num = pf_q_num; + qm->debug.curr_qm_qp_num = pf_q_num; + qm->qm_list = &sec_devices; + qm->err_ini = &sec_err_ini; + if (pf_q_num_flag) + set_bit(QM_MODULE_PARAM, &qm->misc_ctl); + } else if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V1) { + /* + * have no way to get qm configure in VM in v1 hardware, + * so currently force PF to uses SEC_PF_DEF_Q_NUM, and force + * to trigger only one VF in v1 hardware. + * v2 hardware has no such problem. + */ + qm->qp_base = SEC_PF_DEF_Q_NUM; + qm->qp_num = SEC_QUEUE_NUM_V1 - SEC_PF_DEF_Q_NUM; + } + + ret = hisi_qm_init(qm); + if (ret) { + pci_err(qm->pdev, "Failed to init sec qm configures!\n"); + return ret; + } + + sec_set_channels(qm); + /* Fetch and save the value of capability registers */ + ret = sec_pre_store_cap_reg(qm); + if (ret) { + pci_err(qm->pdev, "Failed to pre-store capability registers!\n"); + hisi_qm_uninit(qm); + return ret; + } + alg_msk = sec_get_alg_bitmap(qm, SEC_ALG_BITMAP_HIGH, SEC_ALG_BITMAP_LOW); + ret = hisi_qm_set_algs(qm, alg_msk, sec_dev_algs, ARRAY_SIZE(sec_dev_algs)); + if (ret) { + pci_err(qm->pdev, "Failed to set sec algs!\n"); + hisi_qm_uninit(qm); + } + + return ret; +} + +static void sec_qm_uninit(struct hisi_qm *qm) +{ + hisi_qm_uninit(qm); +} + +static int sec_probe_init(struct sec_dev *sec) +{ + u32 type_rate = SEC_SHAPER_TYPE_RATE; + struct hisi_qm *qm = &sec->qm; + int ret; + + if (qm->fun_type == QM_HW_PF) { + ret = sec_pf_probe_init(sec); + if (ret) + return ret; + /* enable shaper type 0 */ + if (qm->ver >= QM_HW_V3) { + type_rate |= QM_SHAPER_ENABLE; + qm->type_rate = type_rate; + } + } + + return 0; +} + +static void sec_probe_uninit(struct hisi_qm *qm) +{ + if (qm->fun_type == QM_HW_VF) + return; + + sec_debug_regs_clear(qm); + sec_show_last_regs_uninit(qm); + sec_close_sva_prefetch(qm); + hisi_qm_dev_err_uninit(qm); +} + +static void sec_iommu_used_check(struct sec_dev *sec) +{ + struct iommu_domain *domain; + struct device *dev = &sec->qm.pdev->dev; + + domain = iommu_get_domain_for_dev(dev); + + /* Check if iommu is used */ + sec->iommu_used = false; + if (domain) { + if (domain->type & __IOMMU_DOMAIN_PAGING) + sec->iommu_used = true; + dev_info(dev, "SMMU Opened, the iommu type = %u\n", + domain->type); + } +} + +static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct sec_dev *sec; + struct hisi_qm *qm; + int ret; + + sec = devm_kzalloc(&pdev->dev, sizeof(*sec), GFP_KERNEL); + if (!sec) + return -ENOMEM; + + qm = &sec->qm; + ret = sec_qm_init(qm, pdev); + if (ret) { + pci_err(pdev, "Failed to init SEC QM (%d)!\n", ret); + return ret; + } + + sec->ctx_q_num = ctx_q_num; + sec_iommu_used_check(sec); + + ret = sec_probe_init(sec); + if (ret) { + pci_err(pdev, "Failed to probe!\n"); + goto err_qm_uninit; + } + + ret = hisi_qm_start(qm); + if (ret) { + pci_err(pdev, "Failed to start sec qm!\n"); + goto err_probe_uninit; + } + + ret = sec_debugfs_init(qm); + if (ret) + pci_warn(pdev, "Failed to init debugfs!\n"); + + hisi_qm_add_list(qm, &sec_devices); + ret = hisi_qm_alg_register(qm, &sec_devices, ctx_q_num); + if (ret < 0) { + pr_err("Failed to register driver to crypto.\n"); + goto err_qm_del_list; + } + + ret = hisi_qm_register_uacce(qm); + if (ret) { + pci_err(pdev, "failed to register uacce (%d)!\n", ret); + goto err_alg_unregister; + } + + if (qm->fun_type == QM_HW_PF && vfs_num) { + ret = hisi_qm_sriov_enable(pdev, vfs_num); + if (ret < 0) + goto err_alg_unregister; + } + + hisi_qm_pm_init(qm); + + return 0; + +err_alg_unregister: + hisi_qm_alg_unregister(qm, &sec_devices, ctx_q_num); +err_qm_del_list: + hisi_qm_del_list(qm, &sec_devices); + sec_debugfs_exit(qm); + hisi_qm_stop(qm, QM_NORMAL); +err_probe_uninit: + sec_probe_uninit(qm); +err_qm_uninit: + sec_qm_uninit(qm); + return ret; +} + +static void sec_remove(struct pci_dev *pdev) +{ + struct hisi_qm *qm = pci_get_drvdata(pdev); + + hisi_qm_pm_uninit(qm); + hisi_qm_wait_task_finish(qm, &sec_devices); + hisi_qm_alg_unregister(qm, &sec_devices, ctx_q_num); + hisi_qm_del_list(qm, &sec_devices); + + if (qm->fun_type == QM_HW_PF && qm->vfs_num) + hisi_qm_sriov_disable(pdev, true); + + sec_debugfs_exit(qm); + + (void)hisi_qm_stop(qm, QM_NORMAL); + sec_probe_uninit(qm); + + sec_qm_uninit(qm); +} + +static const struct dev_pm_ops sec_pm_ops = { + SET_RUNTIME_PM_OPS(hisi_qm_suspend, hisi_qm_resume, NULL) +}; + +static const struct pci_error_handlers sec_err_handler = { + .error_detected = hisi_qm_dev_err_detected, + .slot_reset = hisi_qm_dev_slot_reset, + .reset_prepare = hisi_qm_reset_prepare, + .reset_done = hisi_qm_reset_done, +}; + +static struct pci_driver sec_pci_driver = { + .name = "hisi_sec2", + .id_table = sec_dev_ids, + .probe = sec_probe, + .remove = sec_remove, + .err_handler = &sec_err_handler, + .sriov_configure = IS_ENABLED(CONFIG_PCI_IOV) ? + hisi_qm_sriov_configure : NULL, + .shutdown = hisi_qm_dev_shutdown, + .driver.pm = &sec_pm_ops, +}; + +struct pci_driver *hisi_sec_get_pf_driver(void) +{ + return &sec_pci_driver; +} +EXPORT_SYMBOL_GPL(hisi_sec_get_pf_driver); + +static void sec_register_debugfs(void) +{ + if (!debugfs_initialized()) + return; + + sec_debugfs_root = debugfs_create_dir("hisi_sec2", NULL); +} + +static void sec_unregister_debugfs(void) +{ + debugfs_remove_recursive(sec_debugfs_root); +} + +static int __init sec_init(void) +{ + int ret; + + hisi_qm_init_list(&sec_devices); + sec_register_debugfs(); + + ret = pci_register_driver(&sec_pci_driver); + if (ret < 0) { + sec_unregister_debugfs(); + pr_err("Failed to register pci driver.\n"); + return ret; + } + + return 0; +} + +static void __exit sec_exit(void) +{ + pci_unregister_driver(&sec_pci_driver); + sec_unregister_debugfs(); +} + +module_init(sec_init); +module_exit(sec_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Zaibo Xu "); +MODULE_AUTHOR("Longfang Liu "); +MODULE_AUTHOR("Kai Ye "); +MODULE_AUTHOR("Wei Zhang "); +MODULE_DESCRIPTION("Driver for HiSilicon SEC accelerator"); diff --git a/drivers/crypto/hisilicon/sgl.c b/drivers/crypto/hisilicon/sgl.c new file mode 100644 index 000000000..1b2607797 --- /dev/null +++ b/drivers/crypto/hisilicon/sgl.c @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2019 HiSilicon Limited. */ +#include +#include +#include +#include +#include + +#define HISI_ACC_SGL_SGE_NR_MIN 1 +#define HISI_ACC_SGL_NR_MAX 256 +#define HISI_ACC_SGL_ALIGN_SIZE 64 +#define HISI_ACC_MEM_BLOCK_NR 5 + +struct acc_hw_sge { + dma_addr_t buf; + void *page_ctrl; + __le32 len; + __le32 pad; + __le32 pad0; + __le32 pad1; +}; + +/* use default sgl head size 64B */ +struct hisi_acc_hw_sgl { + dma_addr_t next_dma; + __le16 entry_sum_in_chain; + __le16 entry_sum_in_sgl; + __le16 entry_length_in_sgl; + __le16 pad0; + __le64 pad1[5]; + struct hisi_acc_hw_sgl *next; + struct acc_hw_sge sge_entries[]; +} __aligned(1); + +struct hisi_acc_sgl_pool { + struct mem_block { + struct hisi_acc_hw_sgl *sgl; + dma_addr_t sgl_dma; + size_t size; + } mem_block[HISI_ACC_MEM_BLOCK_NR]; + u32 sgl_num_per_block; + u32 block_num; + u32 count; + u32 sge_nr; + size_t sgl_size; +}; + +/** + * hisi_acc_create_sgl_pool() - Create a hw sgl pool. + * @dev: The device which hw sgl pool belongs to. + * @count: Count of hisi_acc_hw_sgl in pool. + * @sge_nr: The count of sge in hw_sgl + * + * This function creates a hw sgl pool, after this user can get hw sgl memory + * from it. + */ +struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev, + u32 count, u32 sge_nr) +{ + u32 sgl_size, block_size, sgl_num_per_block, block_num, remain_sgl; + struct hisi_acc_sgl_pool *pool; + struct mem_block *block; + u32 i, j; + + if (!dev || !count || !sge_nr || sge_nr > HISI_ACC_SGL_SGE_NR_MAX) + return ERR_PTR(-EINVAL); + + sgl_size = ALIGN(sizeof(struct acc_hw_sge) * sge_nr + + sizeof(struct hisi_acc_hw_sgl), + HISI_ACC_SGL_ALIGN_SIZE); + + /* + * the pool may allocate a block of memory of size PAGE_SIZE * 2^MAX_PAGE_ORDER, + * block size may exceed 2^31 on ia64, so the max of block size is 2^31 + */ + block_size = 1 << (PAGE_SHIFT + MAX_PAGE_ORDER < 32 ? + PAGE_SHIFT + MAX_PAGE_ORDER : 31); + sgl_num_per_block = block_size / sgl_size; + block_num = count / sgl_num_per_block; + remain_sgl = count % sgl_num_per_block; + + if ((!remain_sgl && block_num > HISI_ACC_MEM_BLOCK_NR) || + (remain_sgl > 0 && block_num > HISI_ACC_MEM_BLOCK_NR - 1)) + return ERR_PTR(-EINVAL); + + pool = kzalloc_obj(*pool); + if (!pool) + return ERR_PTR(-ENOMEM); + block = pool->mem_block; + + for (i = 0; i < block_num; i++) { + block[i].sgl = dma_alloc_coherent(dev, block_size, + &block[i].sgl_dma, + GFP_KERNEL); + if (!block[i].sgl) { + dev_err(dev, "Fail to allocate hw SG buffer!\n"); + goto err_free_mem; + } + + block[i].size = block_size; + } + + if (remain_sgl > 0) { + block[i].sgl = dma_alloc_coherent(dev, remain_sgl * sgl_size, + &block[i].sgl_dma, + GFP_KERNEL); + if (!block[i].sgl) { + dev_err(dev, "Fail to allocate remained hw SG buffer!\n"); + goto err_free_mem; + } + + block[i].size = remain_sgl * sgl_size; + } + + pool->sgl_num_per_block = sgl_num_per_block; + pool->block_num = remain_sgl ? block_num + 1 : block_num; + pool->count = count; + pool->sgl_size = sgl_size; + pool->sge_nr = sge_nr; + + return pool; + +err_free_mem: + for (j = 0; j < i; j++) + dma_free_coherent(dev, block_size, block[j].sgl, + block[j].sgl_dma); + + kfree_sensitive(pool); + return ERR_PTR(-ENOMEM); +} +EXPORT_SYMBOL_GPL(hisi_acc_create_sgl_pool); + +/** + * hisi_acc_free_sgl_pool() - Free a hw sgl pool. + * @dev: The device which hw sgl pool belongs to. + * @pool: Pointer of pool. + * + * This function frees memory of a hw sgl pool. + */ +void hisi_acc_free_sgl_pool(struct device *dev, struct hisi_acc_sgl_pool *pool) +{ + struct mem_block *block; + u32 i; + + if (!dev || !pool) + return; + + block = pool->mem_block; + + for (i = 0; i < pool->block_num; i++) + dma_free_coherent(dev, block[i].size, block[i].sgl, + block[i].sgl_dma); + + kfree(pool); +} +EXPORT_SYMBOL_GPL(hisi_acc_free_sgl_pool); + +static struct hisi_acc_hw_sgl *acc_get_sgl(struct hisi_acc_sgl_pool *pool, + u32 index, dma_addr_t *hw_sgl_dma) +{ + struct mem_block *block; + u32 block_index, offset; + + block = pool->mem_block; + block_index = index / pool->sgl_num_per_block; + offset = index % pool->sgl_num_per_block; + + *hw_sgl_dma = block[block_index].sgl_dma + pool->sgl_size * offset; + return (void *)block[block_index].sgl + pool->sgl_size * offset; +} + +static void sg_map_to_hw_sg(struct scatterlist *sgl, + struct acc_hw_sge *hw_sge) +{ + hw_sge->buf = sg_dma_address(sgl); + hw_sge->len = cpu_to_le32(sg_dma_len(sgl)); + hw_sge->page_ctrl = sg_virt(sgl); +} + +static void inc_hw_sgl_sge(struct hisi_acc_hw_sgl *hw_sgl) +{ + u16 var = le16_to_cpu(hw_sgl->entry_sum_in_sgl); + + var++; + hw_sgl->entry_sum_in_sgl = cpu_to_le16(var); +} + +static void update_hw_sgl_sum_sge(struct hisi_acc_hw_sgl *hw_sgl, u16 sum) +{ + hw_sgl->entry_sum_in_chain = cpu_to_le16(sum); +} + +static void clear_hw_sgl_sge(struct hisi_acc_hw_sgl *hw_sgl) +{ + struct acc_hw_sge *hw_sge = hw_sgl->sge_entries; + u16 entry_sum = le16_to_cpu(hw_sgl->entry_sum_in_sgl); + int i; + + for (i = 0; i < entry_sum; i++) { + hw_sge[i].page_ctrl = NULL; + hw_sge[i].buf = 0; + hw_sge[i].len = 0; + } +} + +/** + * hisi_acc_sg_buf_map_to_hw_sgl - Map a scatterlist to a hw sgl. + * @dev: The device which hw sgl belongs to. + * @sgl: Scatterlist which will be mapped to hw sgl. + * @pool: Pool which hw sgl memory will be allocated in. + * @index: Index of hisi_acc_hw_sgl in pool. + * @hw_sgl_dma: The dma address of allocated hw sgl. + * @dir: DMA direction. + * + * This function builds hw sgl according input sgl, user can use hw_sgl_dma + * as src/dst in its BD. Only support single hw sgl currently. + */ +struct hisi_acc_hw_sgl * +hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev, struct scatterlist *sgl, + struct hisi_acc_sgl_pool *pool, u32 index, + dma_addr_t *hw_sgl_dma, enum dma_data_direction dir) +{ + struct hisi_acc_hw_sgl *curr_hw_sgl; + unsigned int i, sg_n_mapped; + dma_addr_t curr_sgl_dma = 0; + struct acc_hw_sge *curr_hw_sge; + struct scatterlist *sg; + int sg_n, ret; + + if (!dev || !sgl || !pool || !hw_sgl_dma || index >= pool->count) + return ERR_PTR(-EINVAL); + + sg_n = sg_nents(sgl); + + sg_n_mapped = dma_map_sg(dev, sgl, sg_n, dir); + if (!sg_n_mapped) { + dev_err(dev, "DMA mapping for SG error!\n"); + return ERR_PTR(-EINVAL); + } + + if (sg_n_mapped > pool->sge_nr) { + dev_err(dev, "the number of entries in input scatterlist is bigger than SGL pool setting.\n"); + ret = -EINVAL; + goto err_unmap; + } + + curr_hw_sgl = acc_get_sgl(pool, index, &curr_sgl_dma); + curr_hw_sgl->entry_length_in_sgl = cpu_to_le16(pool->sge_nr); + curr_hw_sge = curr_hw_sgl->sge_entries; + + for_each_sg(sgl, sg, sg_n_mapped, i) { + sg_map_to_hw_sg(sg, curr_hw_sge); + inc_hw_sgl_sge(curr_hw_sgl); + curr_hw_sge++; + } + + update_hw_sgl_sum_sge(curr_hw_sgl, pool->sge_nr); + *hw_sgl_dma = curr_sgl_dma; + + return curr_hw_sgl; + +err_unmap: + dma_unmap_sg(dev, sgl, sg_n, dir); + + return ERR_PTR(ret); +} +EXPORT_SYMBOL_GPL(hisi_acc_sg_buf_map_to_hw_sgl); + +/** + * hisi_acc_sg_buf_unmap() - Unmap allocated hw sgl. + * @dev: The device which hw sgl belongs to. + * @sgl: Related scatterlist. + * @hw_sgl: Virtual address of hw sgl. + * @dir: DMA direction. + * + * This function unmaps allocated hw sgl. + */ +void hisi_acc_sg_buf_unmap(struct device *dev, struct scatterlist *sgl, + struct hisi_acc_hw_sgl *hw_sgl, enum dma_data_direction dir) +{ + if (!dev || !sgl || !hw_sgl) + return; + + dma_unmap_sg(dev, sgl, sg_nents(sgl), dir); + clear_hw_sgl_sge(hw_sgl); + hw_sgl->entry_sum_in_chain = 0; + hw_sgl->entry_sum_in_sgl = 0; + hw_sgl->entry_length_in_sgl = 0; +} +EXPORT_SYMBOL_GPL(hisi_acc_sg_buf_unmap); diff --git a/drivers/crypto/hisilicon/zip/Makefile b/drivers/crypto/hisilicon/zip/Makefile new file mode 100644 index 000000000..13de020b7 --- /dev/null +++ b/drivers/crypto/hisilicon/zip/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_CRYPTO_DEV_HISI_ZIP) += hisi_zip.o +hisi_zip-objs = zip_main.o zip_crypto.o dae_main.o diff --git a/drivers/crypto/hisilicon/zip/dae_main.c b/drivers/crypto/hisilicon/zip/dae_main.c new file mode 100644 index 000000000..68aebd02f --- /dev/null +++ b/drivers/crypto/hisilicon/zip/dae_main.c @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2024 HiSilicon Limited. */ + +#include +#include +#include +#include "zip.h" + +/* memory */ +#define DAE_MEM_START_OFFSET 0x331040 +#define DAE_MEM_DONE_OFFSET 0x331044 +#define DAE_MEM_START_MASK 0x1 +#define DAE_MEM_DONE_MASK 0x1 +#define DAE_REG_RD_INTVRL_US 10 +#define DAE_REG_RD_TMOUT_US USEC_PER_SEC + +#define DAE_ALG_NAME "hashagg" +#define DAE_V5_ALG_NAME "hashagg\nudma\nhashjoin\ngather" + +/* error */ +#define DAE_AXI_CFG_OFFSET 0x331000 +#define DAE_AXI_SHUTDOWN_MASK (BIT(0) | BIT(5)) +#define DAE_ERR_SOURCE_OFFSET 0x331C84 +#define DAE_ERR_STATUS_OFFSET 0x331C88 +#define DAE_ERR_CE_OFFSET 0x331CA0 +#define DAE_ERR_CE_MASK BIT(3) +#define DAE_ERR_NFE_OFFSET 0x331CA4 +#define DAE_ERR_NFE_MASK 0x17 +#define DAE_ERR_FE_OFFSET 0x331CA8 +#define DAE_ERR_FE_MASK 0 +#define DAE_ECC_MBIT_MASK BIT(2) +#define DAE_ECC_INFO_OFFSET 0x33400C +#define DAE_ERR_SHUTDOWN_OFFSET 0x331CAC +#define DAE_ERR_SHUTDOWN_MASK 0x17 +#define DAE_ERR_ENABLE_OFFSET 0x331C80 +#define DAE_ERR_ENABLE_MASK (DAE_ERR_FE_MASK | DAE_ERR_NFE_MASK | DAE_ERR_CE_MASK) +#define DAE_AM_CTRL_GLOBAL_OFFSET 0x330000 +#define DAE_AM_RETURN_OFFSET 0x330150 +#define DAE_AM_RETURN_MASK 0x3 +#define DAE_AXI_CFG_OFFSET 0x331000 +#define DAE_AXI_SHUTDOWN_EN_MASK (BIT(0) | BIT(5)) + +struct hisi_dae_hw_error { + u32 int_msk; + const char *msg; +}; + +static const struct hisi_dae_hw_error dae_hw_error[] = { + { .int_msk = BIT(0), .msg = "dae_axi_bus_err" }, + { .int_msk = BIT(1), .msg = "dae_axi_poison_err" }, + { .int_msk = BIT(2), .msg = "dae_ecc_2bit_err" }, + { .int_msk = BIT(3), .msg = "dae_ecc_1bit_err" }, + { .int_msk = BIT(4), .msg = "dae_fsm_hbeat_err" }, +}; + +static inline bool dae_is_support(struct hisi_qm *qm) +{ + if (test_bit(QM_SUPPORT_DAE, &qm->caps)) + return true; + + return false; +} + +int hisi_dae_set_user_domain(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (!dae_is_support(qm)) + return 0; + + val = readl(qm->io_base + DAE_MEM_START_OFFSET); + val |= DAE_MEM_START_MASK; + writel(val, qm->io_base + DAE_MEM_START_OFFSET); + ret = readl_relaxed_poll_timeout(qm->io_base + DAE_MEM_DONE_OFFSET, val, + val & DAE_MEM_DONE_MASK, + DAE_REG_RD_INTVRL_US, DAE_REG_RD_TMOUT_US); + if (ret) + pci_err(qm->pdev, "failed to init dae memory!\n"); + + return ret; +} + +int hisi_dae_set_alg(struct hisi_qm *qm) +{ + const char *alg_name; + size_t len; + + if (!dae_is_support(qm)) + return 0; + + if (!qm->uacce) + return 0; + + if (qm->ver >= QM_HW_V5) + alg_name = DAE_V5_ALG_NAME; + else + alg_name = DAE_ALG_NAME; + + len = strlen(qm->uacce->algs); + /* A line break may be required */ + if (len + strlen(alg_name) + 1 >= QM_DEV_ALG_MAX_LEN) { + pci_err(qm->pdev, "algorithm name is too long!\n"); + return -EINVAL; + } + + if (len) + strcat((char *)qm->uacce->algs, "\n"); + + strcat((char *)qm->uacce->algs, alg_name); + + return 0; +} + +static void hisi_dae_master_ooo_ctrl(struct hisi_qm *qm, bool enable) +{ + u32 axi_val, err_val; + + axi_val = readl(qm->io_base + DAE_AXI_CFG_OFFSET); + if (enable) { + axi_val |= DAE_AXI_SHUTDOWN_MASK; + err_val = DAE_ERR_SHUTDOWN_MASK; + } else { + axi_val &= ~DAE_AXI_SHUTDOWN_MASK; + err_val = 0; + } + + writel(axi_val, qm->io_base + DAE_AXI_CFG_OFFSET); + writel(err_val, qm->io_base + DAE_ERR_SHUTDOWN_OFFSET); +} + +void hisi_dae_hw_error_enable(struct hisi_qm *qm) +{ + if (!dae_is_support(qm)) + return; + + /* clear dae hw error source if having */ + writel(DAE_ERR_ENABLE_MASK, qm->io_base + DAE_ERR_SOURCE_OFFSET); + + /* configure error type */ + writel(DAE_ERR_CE_MASK, qm->io_base + DAE_ERR_CE_OFFSET); + writel(DAE_ERR_NFE_MASK, qm->io_base + DAE_ERR_NFE_OFFSET); + writel(DAE_ERR_FE_MASK, qm->io_base + DAE_ERR_FE_OFFSET); + + hisi_dae_master_ooo_ctrl(qm, true); + + /* enable dae hw error interrupts */ + writel(DAE_ERR_ENABLE_MASK, qm->io_base + DAE_ERR_ENABLE_OFFSET); +} + +void hisi_dae_hw_error_disable(struct hisi_qm *qm) +{ + if (!dae_is_support(qm)) + return; + + writel(0, qm->io_base + DAE_ERR_ENABLE_OFFSET); + hisi_dae_master_ooo_ctrl(qm, false); +} + +static u32 hisi_dae_get_hw_err_status(struct hisi_qm *qm) +{ + return readl(qm->io_base + DAE_ERR_STATUS_OFFSET); +} + +static void hisi_dae_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) +{ + if (!dae_is_support(qm)) + return; + + writel(err_sts, qm->io_base + DAE_ERR_SOURCE_OFFSET); +} + +static void hisi_dae_disable_error_report(struct hisi_qm *qm, u32 err_type) +{ + writel(DAE_ERR_NFE_MASK & (~err_type), qm->io_base + DAE_ERR_NFE_OFFSET); +} + +static void hisi_dae_enable_error_report(struct hisi_qm *qm) +{ + writel(DAE_ERR_CE_MASK, qm->io_base + DAE_ERR_CE_OFFSET); + writel(DAE_ERR_NFE_MASK, qm->io_base + DAE_ERR_NFE_OFFSET); +} + +static void hisi_dae_log_hw_error(struct hisi_qm *qm, u32 err_type) +{ + const struct hisi_dae_hw_error *err = dae_hw_error; + struct device *dev = &qm->pdev->dev; + u32 ecc_info; + size_t i; + + for (i = 0; i < ARRAY_SIZE(dae_hw_error); i++) { + err = &dae_hw_error[i]; + if (!(err->int_msk & err_type)) + continue; + + dev_err(dev, "%s [error status=0x%x] found\n", + err->msg, err->int_msk); + + if (err->int_msk & DAE_ECC_MBIT_MASK) { + ecc_info = readl(qm->io_base + DAE_ECC_INFO_OFFSET); + dev_err(dev, "dae multi ecc sram info 0x%x\n", ecc_info); + } + } +} + +enum acc_err_result hisi_dae_get_err_result(struct hisi_qm *qm) +{ + u32 err_status; + + if (!dae_is_support(qm)) + return ACC_ERR_NONE; + + err_status = hisi_dae_get_hw_err_status(qm); + if (!err_status) + return ACC_ERR_NONE; + + hisi_dae_log_hw_error(qm, err_status); + + if (err_status & DAE_ERR_NFE_MASK) { + /* Disable the same error reporting until device is recovered. */ + hisi_dae_disable_error_report(qm, err_status); + return ACC_ERR_NEED_RESET; + } + hisi_dae_clear_hw_err_status(qm, err_status); + /* Avoid firmware disable error report, re-enable. */ + hisi_dae_enable_error_report(qm); + + return ACC_ERR_RECOVERED; +} + +bool hisi_dae_dev_is_abnormal(struct hisi_qm *qm) +{ + u32 err_status; + + if (!dae_is_support(qm)) + return false; + + err_status = hisi_dae_get_hw_err_status(qm); + if (err_status & DAE_ERR_NFE_MASK) + return true; + + return false; +} + +int hisi_dae_close_axi_master_ooo(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (!dae_is_support(qm)) + return 0; + + val = readl(qm->io_base + DAE_AM_CTRL_GLOBAL_OFFSET); + val |= BIT(0); + writel(val, qm->io_base + DAE_AM_CTRL_GLOBAL_OFFSET); + + ret = readl_relaxed_poll_timeout(qm->io_base + DAE_AM_RETURN_OFFSET, + val, (val == DAE_AM_RETURN_MASK), + DAE_REG_RD_INTVRL_US, DAE_REG_RD_TMOUT_US); + if (ret) + dev_err(&qm->pdev->dev, "failed to close dae axi ooo!\n"); + + return ret; +} + +void hisi_dae_open_axi_master_ooo(struct hisi_qm *qm) +{ + u32 val; + + if (!dae_is_support(qm)) + return; + + val = readl(qm->io_base + DAE_AXI_CFG_OFFSET); + + writel(val & ~DAE_AXI_SHUTDOWN_EN_MASK, qm->io_base + DAE_AXI_CFG_OFFSET); + writel(val | DAE_AXI_SHUTDOWN_EN_MASK, qm->io_base + DAE_AXI_CFG_OFFSET); +} diff --git a/drivers/crypto/hisilicon/zip/zip.h b/drivers/crypto/hisilicon/zip/zip.h new file mode 100644 index 000000000..b83f22828 --- /dev/null +++ b/drivers/crypto/hisilicon/zip/zip.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2019 HiSilicon Limited. */ +#ifndef HISI_ZIP_H +#define HISI_ZIP_H + +#undef pr_fmt +#define pr_fmt(fmt) "hisi_zip: " fmt + +#include +#include + +enum hisi_zip_error_type { + /* negative compression */ + HZIP_NC_ERR = 0x0d, +}; + +struct hisi_zip_dfx { + atomic64_t send_cnt; + atomic64_t recv_cnt; + atomic64_t send_busy_cnt; + atomic64_t err_bd_cnt; +}; + +struct hisi_zip_ctrl; + +struct hisi_zip { + struct hisi_qm qm; + struct hisi_zip_ctrl *ctrl; + struct hisi_zip_dfx dfx; +}; + +struct hisi_zip_sqe { + u32 consumed; + u32 produced; + u32 comp_data_length; + /* + * status: 0~7 bits + * rsvd: 8~31 bits + */ + u32 dw3; + u32 input_data_length; + u32 dw5; + u32 dw6; + /* + * in_sge_data_offset: 0~23 bits + * rsvd: 24~27 bits + * sqe_type: 29~31 bits + */ + u32 dw7; + /* + * out_sge_data_offset: 0~23 bits + * rsvd: 24~31 bits + */ + u32 dw8; + /* + * request_type: 0~7 bits + * buffer_type: 8~11 bits + * rsvd: 13~31 bits + */ + u32 dw9; + u32 dw10; + u32 dw11; + u32 dw12; + /* tag: in sqe type 0 */ + u32 dw13; + u32 dest_avail_out; + u32 dw15; + u32 dw16; + u32 dw17; + u32 source_addr_l; + u32 source_addr_h; + u32 dest_addr_l; + u32 dest_addr_h; + u32 dw22; + u32 dw23; + u32 dw24; + u32 dw25; + /* tag: in sqe type 3 */ + u32 dw26; + u32 dw27; + u32 rsvd1[4]; +}; + +enum zip_cap_table_type { + QM_RAS_NFE_TYPE, + QM_RAS_NFE_RESET, + QM_RAS_CE_TYPE, + ZIP_RAS_NFE_TYPE, + ZIP_RAS_NFE_RESET, + ZIP_RAS_CE_TYPE, + ZIP_CORE_INFO, + ZIP_CORE_EN, + ZIP_DRV_ALG_BITMAP_TB, + ZIP_ALG_BITMAP, + ZIP_CORE1_BITMAP, + ZIP_CORE2_BITMAP, + ZIP_CORE3_BITMAP, + ZIP_CORE4_BITMAP, + ZIP_CORE5_BITMAP, +}; + +int zip_create_qps(struct hisi_qp **qps, int qp_num, int node, u8 *alg_type); +int hisi_zip_register_to_crypto(struct hisi_qm *qm); +void hisi_zip_unregister_from_crypto(struct hisi_qm *qm); +bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg); +int hisi_dae_set_user_domain(struct hisi_qm *qm); +int hisi_dae_set_alg(struct hisi_qm *qm); +void hisi_dae_hw_error_disable(struct hisi_qm *qm); +void hisi_dae_hw_error_enable(struct hisi_qm *qm); +void hisi_dae_open_axi_master_ooo(struct hisi_qm *qm); +int hisi_dae_close_axi_master_ooo(struct hisi_qm *qm); +bool hisi_dae_dev_is_abnormal(struct hisi_qm *qm); +enum acc_err_result hisi_dae_get_err_result(struct hisi_qm *qm); +#endif diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c new file mode 100644 index 000000000..70adde049 --- /dev/null +++ b/drivers/crypto/hisilicon/zip/zip_crypto.c @@ -0,0 +1,727 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2019 HiSilicon Limited. */ +#include +#include +#include +#include +#include +#include "zip.h" + +/* hisi_zip_sqe dw3 */ +#define HZIP_BD_STATUS_M GENMASK(7, 0) +/* hisi_zip_sqe dw7 */ +#define HZIP_IN_SGE_DATA_OFFSET_M GENMASK(23, 0) +#define HZIP_SQE_TYPE_M GENMASK(31, 28) +/* hisi_zip_sqe dw8 */ +#define HZIP_OUT_SGE_DATA_OFFSET_M GENMASK(23, 0) +/* hisi_zip_sqe dw9 */ +#define HZIP_REQ_TYPE_M GENMASK(7, 0) +#define HZIP_ALG_TYPE_DEFLATE 0x01 +#define HZIP_ALG_TYPE_LZ4 0x04 +#define HZIP_BUF_TYPE_M GENMASK(11, 8) +#define HZIP_SGL 0x1 +#define HZIP_WIN_SIZE_M GENMASK(15, 12) +#define HZIP_16K_WINSZ 0x2 + +#define HZIP_ALG_PRIORITY 300 +#define HZIP_SGL_SGE_NR 10 + +#define HZIP_ALG_DEFLATE GENMASK(5, 4) +#define HZIP_ALG_LZ4 BIT(8) + +static DEFINE_MUTEX(zip_algs_lock); +static unsigned int zip_available_devs; + +enum hisi_zip_alg_type { + HZIP_ALG_TYPE_COMP = 0, + HZIP_ALG_TYPE_DECOMP = 1, +}; + +enum { + HZIP_QPC_COMP, + HZIP_QPC_DECOMP, + HZIP_CTX_Q_NUM +}; + +#define GET_REQ_FROM_SQE(sqe) ((u64)(sqe)->dw26 | (u64)(sqe)->dw27 << 32) +#define COMP_NAME_TO_TYPE(alg_name) \ + (!strcmp((alg_name), "deflate") ? HZIP_ALG_TYPE_DEFLATE : \ + (!strcmp((alg_name), "lz4") ? HZIP_ALG_TYPE_LZ4 : 0)) + +struct hisi_zip_req { + struct acomp_req *req; + struct hisi_acc_hw_sgl *hw_src; + struct hisi_acc_hw_sgl *hw_dst; + dma_addr_t dma_src; + dma_addr_t dma_dst; + struct hisi_zip_qp_ctx *qp_ctx; + u16 req_id; +}; + +struct hisi_zip_req_q { + struct hisi_zip_req *q; + unsigned long *req_bitmap; + spinlock_t req_lock; + u16 size; +}; + +struct hisi_zip_qp_ctx { + struct hisi_qp *qp; + struct hisi_zip_req_q req_q; + struct hisi_acc_sgl_pool *sgl_pool; + struct hisi_zip *zip_dev; + struct hisi_zip_ctx *ctx; + u8 req_type; +}; + +struct hisi_zip_sqe_ops { + u8 sqe_type; + void (*fill_addr)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req); + void (*fill_buf_size)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req); + void (*fill_buf_type)(struct hisi_zip_sqe *sqe, u8 buf_type); + void (*fill_req_type)(struct hisi_zip_sqe *sqe, u8 req_type); + void (*fill_win_size)(struct hisi_zip_sqe *sqe, u8 win_size); + void (*fill_tag)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req); + void (*fill_sqe_type)(struct hisi_zip_sqe *sqe, u8 sqe_type); + u32 (*get_status)(struct hisi_zip_sqe *sqe); + u32 (*get_dstlen)(struct hisi_zip_sqe *sqe); +}; + +struct hisi_zip_ctx { + struct hisi_zip_qp_ctx qp_ctx[HZIP_CTX_Q_NUM]; + const struct hisi_zip_sqe_ops *ops; + bool fallback; +}; + +static int sgl_sge_nr_set(const char *val, const struct kernel_param *kp) +{ + int ret; + u16 n; + + if (!val) + return -EINVAL; + + ret = kstrtou16(val, 10, &n); + if (ret || n == 0 || n > HISI_ACC_SGL_SGE_NR_MAX) + return -EINVAL; + + return param_set_ushort(val, kp); +} + +static const struct kernel_param_ops sgl_sge_nr_ops = { + .set = sgl_sge_nr_set, + .get = param_get_ushort, +}; + +static u16 sgl_sge_nr = HZIP_SGL_SGE_NR; +module_param_cb(sgl_sge_nr, &sgl_sge_nr_ops, &sgl_sge_nr, 0444); +MODULE_PARM_DESC(sgl_sge_nr, "Number of sge in sgl(1-255)"); + +static int hisi_zip_fallback_do_work(struct acomp_req *acomp_req, bool is_decompress) +{ + ACOMP_FBREQ_ON_STACK(fbreq, acomp_req); + int ret; + + if (!is_decompress) + ret = crypto_acomp_compress(fbreq); + else + ret = crypto_acomp_decompress(fbreq); + if (ret) { + pr_err("failed to do fallback work, ret=%d\n", ret); + return ret; + } + + acomp_req->dlen = fbreq->dlen; + return ret; +} + +static struct hisi_zip_req *hisi_zip_create_req(struct hisi_zip_qp_ctx *qp_ctx, + struct acomp_req *req) +{ + struct hisi_zip_req_q *req_q = &qp_ctx->req_q; + struct hisi_zip_req *q = req_q->q; + struct hisi_zip_req *req_cache; + int req_id; + + spin_lock(&req_q->req_lock); + + req_id = find_first_zero_bit(req_q->req_bitmap, req_q->size); + if (req_id >= req_q->size) { + spin_unlock(&req_q->req_lock); + dev_dbg(&qp_ctx->qp->qm->pdev->dev, "req cache is full!\n"); + return ERR_PTR(-EAGAIN); + } + set_bit(req_id, req_q->req_bitmap); + + spin_unlock(&req_q->req_lock); + + req_cache = q + req_id; + req_cache->req_id = req_id; + req_cache->req = req; + req_cache->qp_ctx = qp_ctx; + + return req_cache; +} + +static void hisi_zip_remove_req(struct hisi_zip_qp_ctx *qp_ctx, + struct hisi_zip_req *req) +{ + struct hisi_zip_req_q *req_q = &qp_ctx->req_q; + + spin_lock(&req_q->req_lock); + clear_bit(req->req_id, req_q->req_bitmap); + spin_unlock(&req_q->req_lock); +} + +static void hisi_zip_fill_addr(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) +{ + sqe->source_addr_l = lower_32_bits(req->dma_src); + sqe->source_addr_h = upper_32_bits(req->dma_src); + sqe->dest_addr_l = lower_32_bits(req->dma_dst); + sqe->dest_addr_h = upper_32_bits(req->dma_dst); +} + +static void hisi_zip_fill_buf_size(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) +{ + struct acomp_req *a_req = req->req; + + sqe->input_data_length = a_req->slen; + sqe->dest_avail_out = a_req->dlen; +} + +static void hisi_zip_fill_buf_type(struct hisi_zip_sqe *sqe, u8 buf_type) +{ + u32 val; + + val = sqe->dw9 & ~HZIP_BUF_TYPE_M; + val |= FIELD_PREP(HZIP_BUF_TYPE_M, buf_type); + sqe->dw9 = val; +} + +static void hisi_zip_fill_req_type(struct hisi_zip_sqe *sqe, u8 req_type) +{ + u32 val; + + val = sqe->dw9 & ~HZIP_REQ_TYPE_M; + val |= FIELD_PREP(HZIP_REQ_TYPE_M, req_type); + sqe->dw9 = val; +} + +static void hisi_zip_fill_win_size(struct hisi_zip_sqe *sqe, u8 win_size) +{ + u32 val; + + val = sqe->dw9 & ~HZIP_WIN_SIZE_M; + val |= FIELD_PREP(HZIP_WIN_SIZE_M, win_size); + sqe->dw9 = val; +} + +static void hisi_zip_fill_tag(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) +{ + sqe->dw26 = lower_32_bits((u64)req); + sqe->dw27 = upper_32_bits((u64)req); +} + +static void hisi_zip_fill_sqe_type(struct hisi_zip_sqe *sqe, u8 sqe_type) +{ + u32 val; + + val = sqe->dw7 & ~HZIP_SQE_TYPE_M; + val |= FIELD_PREP(HZIP_SQE_TYPE_M, sqe_type); + sqe->dw7 = val; +} + +static void hisi_zip_fill_sqe(struct hisi_zip_ctx *ctx, struct hisi_zip_sqe *sqe, + u8 req_type, struct hisi_zip_req *req) +{ + const struct hisi_zip_sqe_ops *ops = ctx->ops; + + memset(sqe, 0, sizeof(struct hisi_zip_sqe)); + + ops->fill_addr(sqe, req); + ops->fill_buf_size(sqe, req); + ops->fill_buf_type(sqe, HZIP_SGL); + ops->fill_req_type(sqe, req_type); + ops->fill_win_size(sqe, HZIP_16K_WINSZ); + ops->fill_tag(sqe, req); + ops->fill_sqe_type(sqe, ops->sqe_type); +} + +static int hisi_zip_do_work(struct hisi_zip_qp_ctx *qp_ctx, + struct hisi_zip_req *req) +{ + struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool; + struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; + struct acomp_req *a_req = req->req; + struct hisi_qp *qp = qp_ctx->qp; + struct device *dev = &qp->qm->pdev->dev; + struct hisi_zip_sqe zip_sqe; + int ret; + + if (unlikely(!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen)) + return -EINVAL; + + req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool, + req->req_id << 1, &req->dma_src, + DMA_TO_DEVICE); + if (IS_ERR(req->hw_src)) { + dev_err(dev, "failed to map the src buffer to hw sgl (%ld)!\n", + PTR_ERR(req->hw_src)); + return PTR_ERR(req->hw_src); + } + + req->hw_dst = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->dst, pool, + (req->req_id << 1) + 1, + &req->dma_dst, DMA_FROM_DEVICE); + if (IS_ERR(req->hw_dst)) { + ret = PTR_ERR(req->hw_dst); + dev_err(dev, "failed to map the dst buffer to hw sgl (%d)!\n", + ret); + goto err_unmap_input; + } + + hisi_zip_fill_sqe(qp_ctx->ctx, &zip_sqe, qp_ctx->req_type, req); + + /* send command to start a task */ + atomic64_inc(&dfx->send_cnt); + ret = hisi_qp_send(qp, &zip_sqe); + if (unlikely(ret < 0)) { + atomic64_inc(&dfx->send_busy_cnt); + ret = -EAGAIN; + dev_dbg_ratelimited(dev, "failed to send request!\n"); + goto err_unmap_output; + } + + return -EINPROGRESS; + +err_unmap_output: + hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst, DMA_FROM_DEVICE); +err_unmap_input: + hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src, DMA_TO_DEVICE); + return ret; +} + +static u32 hisi_zip_get_status(struct hisi_zip_sqe *sqe) +{ + return sqe->dw3 & HZIP_BD_STATUS_M; +} + +static u32 hisi_zip_get_dstlen(struct hisi_zip_sqe *sqe) +{ + return sqe->produced; +} + +static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data) +{ + struct hisi_zip_sqe *sqe = data; + struct hisi_zip_req *req = (struct hisi_zip_req *)GET_REQ_FROM_SQE(sqe); + struct hisi_zip_qp_ctx *qp_ctx = req->qp_ctx; + const struct hisi_zip_sqe_ops *ops = qp_ctx->ctx->ops; + struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; + struct device *dev = &qp->qm->pdev->dev; + struct acomp_req *acomp_req = req->req; + int err = 0; + u32 status; + + atomic64_inc(&dfx->recv_cnt); + status = ops->get_status(sqe); + if (unlikely(status != 0 && status != HZIP_NC_ERR)) { + dev_err(dev, "%scompress fail in qp%u: %u, output: %u\n", + (qp->alg_type == 0) ? "" : "de", qp->qp_id, status, + sqe->produced); + atomic64_inc(&dfx->err_bd_cnt); + err = -EIO; + } + + hisi_acc_sg_buf_unmap(dev, acomp_req->dst, req->hw_dst, DMA_FROM_DEVICE); + hisi_acc_sg_buf_unmap(dev, acomp_req->src, req->hw_src, DMA_TO_DEVICE); + + acomp_req->dlen = ops->get_dstlen(sqe); + + if (acomp_req->base.complete) + acomp_request_complete(acomp_req, err); + + hisi_zip_remove_req(qp_ctx, req); +} + +static int hisi_zip_acompress(struct acomp_req *acomp_req) +{ + struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); + struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_COMP]; + struct hisi_zip_req *req; + struct device *dev; + int ret; + + if (ctx->fallback) + return hisi_zip_fallback_do_work(acomp_req, 0); + + dev = &qp_ctx->qp->qm->pdev->dev; + + req = hisi_zip_create_req(qp_ctx, acomp_req); + if (IS_ERR(req)) + return PTR_ERR(req); + + ret = hisi_zip_do_work(qp_ctx, req); + if (unlikely(ret != -EINPROGRESS)) { + dev_info_ratelimited(dev, "failed to do compress (%d)!\n", ret); + hisi_zip_remove_req(qp_ctx, req); + } + + return ret; +} + +static int hisi_zip_adecompress(struct acomp_req *acomp_req) +{ + struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); + struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_DECOMP]; + struct hisi_zip_req *req; + struct device *dev; + int ret; + + if (ctx->fallback) + return hisi_zip_fallback_do_work(acomp_req, 1); + + dev = &qp_ctx->qp->qm->pdev->dev; + + req = hisi_zip_create_req(qp_ctx, acomp_req); + if (IS_ERR(req)) + return PTR_ERR(req); + + ret = hisi_zip_do_work(qp_ctx, req); + if (unlikely(ret != -EINPROGRESS)) { + dev_info_ratelimited(dev, "failed to do decompress (%d)!\n", + ret); + hisi_zip_remove_req(qp_ctx, req); + } + + return ret; +} + +static int hisi_zip_decompress(struct acomp_req *acomp_req) +{ + return hisi_zip_fallback_do_work(acomp_req, 1); +} + +static const struct hisi_zip_sqe_ops hisi_zip_ops = { + .sqe_type = 0x3, + .fill_addr = hisi_zip_fill_addr, + .fill_buf_size = hisi_zip_fill_buf_size, + .fill_buf_type = hisi_zip_fill_buf_type, + .fill_req_type = hisi_zip_fill_req_type, + .fill_win_size = hisi_zip_fill_win_size, + .fill_tag = hisi_zip_fill_tag, + .fill_sqe_type = hisi_zip_fill_sqe_type, + .get_status = hisi_zip_get_status, + .get_dstlen = hisi_zip_get_dstlen, +}; + +static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int node) +{ + struct hisi_qp *qps[HZIP_CTX_Q_NUM] = { NULL }; + struct hisi_zip_qp_ctx *qp_ctx; + u8 alg_type[HZIP_CTX_Q_NUM]; + struct hisi_zip *hisi_zip; + int ret, i; + + /* alg_type = 0 for compress, 1 for decompress in hw sqe */ + for (i = 0; i < HZIP_CTX_Q_NUM; i++) + alg_type[i] = i; + + ret = zip_create_qps(qps, HZIP_CTX_Q_NUM, node, alg_type); + if (ret) { + pr_err("failed to create zip qps (%d)!\n", ret); + return -ENODEV; + } + + hisi_zip = container_of(qps[0]->qm, struct hisi_zip, qm); + + for (i = 0; i < HZIP_CTX_Q_NUM; i++) { + qp_ctx = &hisi_zip_ctx->qp_ctx[i]; + qp_ctx->ctx = hisi_zip_ctx; + qp_ctx->zip_dev = hisi_zip; + qp_ctx->req_type = req_type; + qp_ctx->qp = qps[i]; + } + + hisi_zip_ctx->ops = &hisi_zip_ops; + + return 0; +} + +static void hisi_zip_ctx_exit(struct hisi_zip_ctx *hisi_zip_ctx) +{ + struct hisi_qp *qps[HZIP_CTX_Q_NUM] = { NULL }; + int i; + + for (i = 0; i < HZIP_CTX_Q_NUM; i++) + qps[i] = hisi_zip_ctx->qp_ctx[i].qp; + + hisi_qm_free_qps(qps, HZIP_CTX_Q_NUM); +} + +static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx) +{ + u16 q_depth = ctx->qp_ctx[0].qp->sq_depth; + struct hisi_zip_req_q *req_q; + int i, ret; + + for (i = 0; i < HZIP_CTX_Q_NUM; i++) { + req_q = &ctx->qp_ctx[i].req_q; + req_q->size = q_depth; + + req_q->req_bitmap = bitmap_zalloc(req_q->size, GFP_KERNEL); + if (!req_q->req_bitmap) { + ret = -ENOMEM; + if (i == 0) + return ret; + + goto err_free_comp_q; + } + spin_lock_init(&req_q->req_lock); + + req_q->q = kzalloc_objs(struct hisi_zip_req, req_q->size); + if (!req_q->q) { + ret = -ENOMEM; + if (i == 0) + goto err_free_comp_bitmap; + else + goto err_free_decomp_bitmap; + } + } + + return 0; + +err_free_decomp_bitmap: + bitmap_free(ctx->qp_ctx[HZIP_QPC_DECOMP].req_q.req_bitmap); +err_free_comp_q: + kfree(ctx->qp_ctx[HZIP_QPC_COMP].req_q.q); +err_free_comp_bitmap: + bitmap_free(ctx->qp_ctx[HZIP_QPC_COMP].req_q.req_bitmap); + return ret; +} + +static void hisi_zip_release_req_q(struct hisi_zip_ctx *ctx) +{ + int i; + + for (i = 0; i < HZIP_CTX_Q_NUM; i++) { + kfree(ctx->qp_ctx[i].req_q.q); + bitmap_free(ctx->qp_ctx[i].req_q.req_bitmap); + } +} + +static int hisi_zip_create_sgl_pool(struct hisi_zip_ctx *ctx) +{ + u16 q_depth = ctx->qp_ctx[0].qp->sq_depth; + struct hisi_zip_qp_ctx *tmp; + struct device *dev; + int i; + + for (i = 0; i < HZIP_CTX_Q_NUM; i++) { + tmp = &ctx->qp_ctx[i]; + dev = &tmp->qp->qm->pdev->dev; + tmp->sgl_pool = hisi_acc_create_sgl_pool(dev, q_depth << 1, + sgl_sge_nr); + if (IS_ERR(tmp->sgl_pool)) { + if (i == 1) + goto err_free_sgl_pool0; + return -ENOMEM; + } + } + + return 0; + +err_free_sgl_pool0: + hisi_acc_free_sgl_pool(&ctx->qp_ctx[HZIP_QPC_COMP].qp->qm->pdev->dev, + ctx->qp_ctx[HZIP_QPC_COMP].sgl_pool); + return -ENOMEM; +} + +static void hisi_zip_release_sgl_pool(struct hisi_zip_ctx *ctx) +{ + int i; + + for (i = 0; i < HZIP_CTX_Q_NUM; i++) + hisi_acc_free_sgl_pool(&ctx->qp_ctx[i].qp->qm->pdev->dev, + ctx->qp_ctx[i].sgl_pool); +} + +static void hisi_zip_set_acomp_cb(struct hisi_zip_ctx *ctx, + void (*fn)(struct hisi_qp *, void *)) +{ + int i; + + for (i = 0; i < HZIP_CTX_Q_NUM; i++) + ctx->qp_ctx[i].qp->req_cb = fn; +} + +static int hisi_zip_acomp_init(struct crypto_acomp *tfm) +{ + const char *alg_name = crypto_tfm_alg_name(&tfm->base); + struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base); + struct device *dev; + int ret; + + ret = hisi_zip_ctx_init(ctx, COMP_NAME_TO_TYPE(alg_name), tfm->base.node); + if (ret) { + pr_err("failed to init ctx (%d)!\n", ret); + goto switch_to_soft; + } + + dev = &ctx->qp_ctx[0].qp->qm->pdev->dev; + + ret = hisi_zip_create_req_q(ctx); + if (ret) { + dev_err(dev, "failed to create request queue (%d)!\n", ret); + goto err_ctx_exit; + } + + ret = hisi_zip_create_sgl_pool(ctx); + if (ret) { + dev_err(dev, "failed to create sgl pool (%d)!\n", ret); + goto err_release_req_q; + } + + hisi_zip_set_acomp_cb(ctx, hisi_zip_acomp_cb); + + return 0; + +err_release_req_q: + hisi_zip_release_req_q(ctx); +err_ctx_exit: + hisi_zip_ctx_exit(ctx); +switch_to_soft: + ctx->fallback = true; + return 0; +} + +static void hisi_zip_acomp_exit(struct crypto_acomp *tfm) +{ + struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base); + + if (ctx->fallback) + return; + + hisi_zip_release_sgl_pool(ctx); + hisi_zip_release_req_q(ctx); + hisi_zip_ctx_exit(ctx); +} + +static struct acomp_alg hisi_zip_acomp_deflate = { + .init = hisi_zip_acomp_init, + .exit = hisi_zip_acomp_exit, + .compress = hisi_zip_acompress, + .decompress = hisi_zip_adecompress, + .base = { + .cra_name = "deflate", + .cra_driver_name = "hisi-deflate-acomp", + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_module = THIS_MODULE, + .cra_priority = HZIP_ALG_PRIORITY, + .cra_ctxsize = sizeof(struct hisi_zip_ctx), + } +}; + +static int hisi_zip_register_deflate(struct hisi_qm *qm) +{ + int ret; + + if (!hisi_zip_alg_support(qm, HZIP_ALG_DEFLATE)) + return 0; + + ret = crypto_register_acomp(&hisi_zip_acomp_deflate); + if (ret) + dev_err(&qm->pdev->dev, "failed to register to deflate (%d)!\n", ret); + + return ret; +} + +static void hisi_zip_unregister_deflate(struct hisi_qm *qm) +{ + if (!hisi_zip_alg_support(qm, HZIP_ALG_DEFLATE)) + return; + + crypto_unregister_acomp(&hisi_zip_acomp_deflate); +} + +static struct acomp_alg hisi_zip_acomp_lz4 = { + .init = hisi_zip_acomp_init, + .exit = hisi_zip_acomp_exit, + .compress = hisi_zip_acompress, + .decompress = hisi_zip_decompress, + .base = { + .cra_name = "lz4", + .cra_driver_name = "hisi-lz4-acomp", + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_module = THIS_MODULE, + .cra_priority = HZIP_ALG_PRIORITY, + .cra_ctxsize = sizeof(struct hisi_zip_ctx), + } +}; + +static int hisi_zip_register_lz4(struct hisi_qm *qm) +{ + int ret; + + if (!hisi_zip_alg_support(qm, HZIP_ALG_LZ4)) + return 0; + + ret = crypto_register_acomp(&hisi_zip_acomp_lz4); + if (ret) + dev_err(&qm->pdev->dev, "failed to register to LZ4 (%d)!\n", ret); + + return ret; +} + +static void hisi_zip_unregister_lz4(struct hisi_qm *qm) +{ + if (!hisi_zip_alg_support(qm, HZIP_ALG_LZ4)) + return; + + crypto_unregister_acomp(&hisi_zip_acomp_lz4); +} + +int hisi_zip_register_to_crypto(struct hisi_qm *qm) +{ + int ret = 0; + + mutex_lock(&zip_algs_lock); + if (zip_available_devs) { + zip_available_devs++; + goto unlock; + } + + ret = hisi_zip_register_deflate(qm); + if (ret) + goto unlock; + + ret = hisi_zip_register_lz4(qm); + if (ret) + goto unreg_deflate; + + zip_available_devs++; + mutex_unlock(&zip_algs_lock); + + return 0; + +unreg_deflate: + hisi_zip_unregister_deflate(qm); +unlock: + mutex_unlock(&zip_algs_lock); + return ret; +} + +void hisi_zip_unregister_from_crypto(struct hisi_qm *qm) +{ + mutex_lock(&zip_algs_lock); + if (--zip_available_devs) + goto unlock; + + hisi_zip_unregister_deflate(qm); + hisi_zip_unregister_lz4(qm); + +unlock: + mutex_unlock(&zip_algs_lock); +} diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c new file mode 100644 index 000000000..706a73656 --- /dev/null +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -0,0 +1,1680 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2019 HiSilicon Limited. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "zip.h" + +#define CAP_FILE_PERMISSION 0444 +#define PCI_DEVICE_ID_HUAWEI_ZIP_PF 0xa250 + +#define HZIP_QUEUE_NUM_V1 4096 + +#define HZIP_CLOCK_GATE_CTRL 0x301004 +#define HZIP_DECOMP_CHECK_ENABLE BIT(16) +#define HZIP_FSM_MAX_CNT 0x301008 + +#define HZIP_PORT_ARCA_CHE_0 0x301040 +#define HZIP_PORT_ARCA_CHE_1 0x301044 +#define HZIP_PORT_AWCA_CHE_0 0x301060 +#define HZIP_PORT_AWCA_CHE_1 0x301064 +#define HZIP_CACHE_ALL_EN 0xffffffff + +#define HZIP_BD_RUSER_32_63 0x301110 +#define HZIP_SGL_RUSER_32_63 0x30111c +#define HZIP_DATA_RUSER_32_63 0x301128 +#define HZIP_DATA_WUSER_32_63 0x301134 +#define HZIP_BD_WUSER_32_63 0x301140 + +#define HZIP_QM_IDEL_STATUS 0x3040e4 + +#define HZIP_CORE_DFX_BASE 0x301000 +#define HZIP_CORE_DFX_DECOMP_BASE 0x304000 +#define HZIP_CORE_DFX_COMP_0 0x302000 +#define HZIP_CORE_DFX_COMP_1 0x303000 +#define HZIP_CORE_DFX_DECOMP_0 0x304000 +#define HZIP_CORE_DFX_DECOMP_1 0x305000 +#define HZIP_CORE_DFX_DECOMP_2 0x306000 +#define HZIP_CORE_DFX_DECOMP_3 0x307000 +#define HZIP_CORE_DFX_DECOMP_4 0x308000 +#define HZIP_CORE_DFX_DECOMP_5 0x309000 +#define HZIP_CORE_REGS_BASE_LEN 0xB0 +#define HZIP_CORE_REGS_DFX_LEN 0x28 +#define HZIP_CORE_ADDR_INTRVL 0x1000 + +#define HZIP_CORE_INT_SOURCE 0x3010A0 +#define HZIP_CORE_INT_MASK_REG 0x3010A4 +#define HZIP_CORE_INT_SET 0x3010A8 +#define HZIP_CORE_INT_STATUS 0x3010AC +#define HZIP_CORE_INT_STATUS_M_ECC BIT(1) +#define HZIP_CORE_SRAM_ECC_ERR_INFO 0x301148 +#define HZIP_CORE_INT_RAS_CE_ENB 0x301160 +#define HZIP_CORE_INT_RAS_NFE_ENB 0x301164 +#define HZIP_CORE_INT_RAS_FE_ENB 0x301168 +#define HZIP_CORE_INT_RAS_FE_ENB_MASK 0x0 +#define HZIP_OOO_SHUTDOWN_SEL 0x30120C +#define HZIP_SRAM_ECC_ERR_NUM_SHIFT 16 +#define HZIP_SRAM_ECC_ERR_ADDR_SHIFT 24 +#define HZIP_CORE_INT_MASK_ALL GENMASK(31, 0) +#define HZIP_CORE_RAS_CLEAR_ALL GENMASK(31, 0) +#define HZIP_AXI_ERROR_MASK (BIT(2) | BIT(3)) +#define HZIP_SQE_SIZE 128 +#define HZIP_PF_DEF_Q_NUM 64 +#define HZIP_PF_DEF_Q_BASE 0 +#define HZIP_CTX_Q_NUM_DEF 2 + +#define HZIP_SOFT_CTRL_CNT_CLR_CE 0x301000 +#define HZIP_SOFT_CTRL_CNT_CLR_CE_BIT BIT(0) +#define HZIP_SOFT_CTRL_ZIP_CONTROL 0x30100C +#define HZIP_AXI_SHUTDOWN_ENABLE BIT(14) +#define HZIP_WR_PORT BIT(11) + +#define HZIP_ALG_ZLIB_BIT GENMASK(1, 0) +#define HZIP_ALG_GZIP_BIT GENMASK(3, 2) +#define HZIP_ALG_DEFLATE_BIT GENMASK(5, 4) +#define HZIP_ALG_LZ77_BIT GENMASK(7, 6) +#define HZIP_ALG_LZ4_BIT GENMASK(9, 8) + +#define HZIP_BUF_SIZE 22 +#define HZIP_SQE_MASK_OFFSET 64 +#define HZIP_SQE_MASK_LEN 48 + +#define HZIP_CNT_CLR_CE_EN BIT(0) +#define HZIP_RO_CNT_CLR_CE_EN BIT(2) +#define HZIP_RD_CNT_CLR_CE_EN (HZIP_CNT_CLR_CE_EN | \ + HZIP_RO_CNT_CLR_CE_EN) + +#define HZIP_PREFETCH_CFG 0x3011B0 +#define HZIP_SVA_TRANS 0x3011C4 +#define HZIP_PREFETCH_ENABLE (~(BIT(26) | BIT(17) | BIT(0))) +#define HZIP_SVA_PREFETCH_DISABLE BIT(26) +#define HZIP_SVA_DISABLE_READY (BIT(26) | BIT(30)) +#define HZIP_SVA_PREFETCH_NUM GENMASK(18, 16) +#define HZIP_SVA_STALL_NUM GENMASK(15, 0) +#define HZIP_SHAPER_RATE_COMPRESS 750 +#define HZIP_SHAPER_RATE_DECOMPRESS 140 +#define HZIP_DELAY_1_US 1 +#define HZIP_POLL_TIMEOUT_US 1000 +#define HZIP_WAIT_SVA_READY 500000 +#define HZIP_READ_SVA_STATUS_TIMES 3 +#define HZIP_WAIT_US_MIN 10 +#define HZIP_WAIT_US_MAX 20 + +/* clock gating */ +#define HZIP_PEH_CFG_AUTO_GATE 0x3011A8 +#define HZIP_PEH_CFG_AUTO_GATE_EN BIT(0) +#define HZIP_CORE_GATED_EN GENMASK(15, 8) +#define HZIP_CORE_GATED_OOO_EN BIT(29) +#define HZIP_CLOCK_GATED_EN (HZIP_CORE_GATED_EN | \ + HZIP_CORE_GATED_OOO_EN) + +/* zip comp high performance */ +#define HZIP_HIGH_PERF_OFFSET 0x301208 + +#define HZIP_LIT_LEN_EN_OFFSET 0x301204 +#define HZIP_LIT_LEN_EN_EN BIT(4) + +#define HZIP_MAX_CHANNEL_NUM 3 + +enum { + HZIP_HIGH_COMP_RATE, + HZIP_HIGH_COMP_PERF, +}; + +static const char hisi_zip_name[] = "hisi_zip"; +static struct dentry *hzip_debugfs_root; + +struct hisi_zip_hw_error { + u32 int_msk; + const char *msg; +}; + +struct zip_dfx_item { + const char *name; + u32 offset; +}; + +static const struct qm_dev_alg zip_dev_algs[] = { { + .alg_msk = HZIP_ALG_ZLIB_BIT, + .alg = "zlib\n", + }, { + .alg_msk = HZIP_ALG_GZIP_BIT, + .alg = "gzip\n", + }, { + .alg_msk = HZIP_ALG_DEFLATE_BIT, + .alg = "deflate\n", + }, { + .alg_msk = HZIP_ALG_LZ77_BIT, + .alg = "lz77_zstd\n", + }, { + .alg_msk = HZIP_ALG_LZ77_BIT, + .alg = "lz77_only\n", + }, { + .alg_msk = HZIP_ALG_LZ4_BIT, + .alg = "lz4\n", + }, +}; + +static struct hisi_qm_list zip_devices = { + .register_to_crypto = hisi_zip_register_to_crypto, + .unregister_from_crypto = hisi_zip_unregister_from_crypto, +}; + +static struct zip_dfx_item zip_dfx_files[] = { + {"send_cnt", offsetof(struct hisi_zip_dfx, send_cnt)}, + {"recv_cnt", offsetof(struct hisi_zip_dfx, recv_cnt)}, + {"send_busy_cnt", offsetof(struct hisi_zip_dfx, send_busy_cnt)}, + {"err_bd_cnt", offsetof(struct hisi_zip_dfx, err_bd_cnt)}, +}; + +static const struct hisi_zip_hw_error zip_hw_error[] = { + { .int_msk = BIT(0), .msg = "zip_ecc_1bitt_err" }, + { .int_msk = BIT(1), .msg = "zip_ecc_2bit_err" }, + { .int_msk = BIT(2), .msg = "zip_axi_rresp_err" }, + { .int_msk = BIT(3), .msg = "zip_axi_bresp_err" }, + { .int_msk = BIT(4), .msg = "zip_src_addr_parse_err" }, + { .int_msk = BIT(5), .msg = "zip_dst_addr_parse_err" }, + { .int_msk = BIT(6), .msg = "zip_pre_in_addr_err" }, + { .int_msk = BIT(7), .msg = "zip_pre_in_data_err" }, + { .int_msk = BIT(8), .msg = "zip_com_inf_err" }, + { .int_msk = BIT(9), .msg = "zip_enc_inf_err" }, + { .int_msk = BIT(10), .msg = "zip_pre_out_err" }, + { .int_msk = BIT(11), .msg = "zip_axi_poison_err" }, + { .int_msk = BIT(12), .msg = "zip_sva_err" }, + { /* sentinel */ } +}; + +enum ctrl_debug_file_index { + HZIP_CLEAR_ENABLE, + HZIP_DEBUG_FILE_NUM, +}; + +static const char * const ctrl_debug_file_name[] = { + [HZIP_CLEAR_ENABLE] = "clear_enable", +}; + +struct ctrl_debug_file { + enum ctrl_debug_file_index index; + spinlock_t lock; + struct hisi_zip_ctrl *ctrl; +}; + +/* + * One ZIP controller has one PF and multiple VFs, some global configurations + * which PF has need this structure. + * + * Just relevant for PF. + */ +struct hisi_zip_ctrl { + struct hisi_zip *hisi_zip; + struct ctrl_debug_file files[HZIP_DEBUG_FILE_NUM]; +}; + +enum zip_cap_type { + ZIP_QM_NFE_MASK_CAP = 0x0, + ZIP_QM_RESET_MASK_CAP, + ZIP_QM_OOO_SHUTDOWN_MASK_CAP, + ZIP_QM_CE_MASK_CAP, + ZIP_NFE_MASK_CAP, + ZIP_RESET_MASK_CAP, + ZIP_OOO_SHUTDOWN_MASK_CAP, + ZIP_CE_MASK_CAP, + ZIP_CLUSTER_NUM_CAP, + ZIP_CORE_TYPE_NUM_CAP, + ZIP_CORE_NUM_CAP, + ZIP_CLUSTER_COMP_NUM_CAP, + ZIP_CLUSTER_DECOMP_NUM_CAP, + ZIP_DECOMP_ENABLE_BITMAP, + ZIP_COMP_ENABLE_BITMAP, + ZIP_DRV_ALG_BITMAP, + ZIP_DEV_ALG_BITMAP, + ZIP_CORE1_ALG_BITMAP, + ZIP_CORE2_ALG_BITMAP, + ZIP_CORE3_ALG_BITMAP, + ZIP_CORE4_ALG_BITMAP, + ZIP_CORE5_ALG_BITMAP, + ZIP_CAP_MAX +}; + +static struct hisi_qm_cap_info zip_basic_cap_info[] = { + {ZIP_QM_NFE_MASK_CAP, 0x3124, 0, GENMASK(31, 0), 0x0, 0x1C57, 0x7C77}, + {ZIP_QM_RESET_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0xC57, 0x6C77}, + {ZIP_QM_OOO_SHUTDOWN_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0x4, 0x6C77}, + {ZIP_QM_CE_MASK_CAP, 0x312C, 0, GENMASK(31, 0), 0x0, 0x8, 0x8}, + {ZIP_NFE_MASK_CAP, 0x3130, 0, GENMASK(31, 0), 0x0, 0x7FE, 0x1FFE}, + {ZIP_RESET_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x7FE, 0x7FE}, + {ZIP_OOO_SHUTDOWN_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x2, 0x7FE}, + {ZIP_CE_MASK_CAP, 0x3138, 0, GENMASK(31, 0), 0x0, 0x1, 0x1}, + {ZIP_CLUSTER_NUM_CAP, 0x313C, 28, GENMASK(3, 0), 0x1, 0x1, 0x1}, + {ZIP_CORE_TYPE_NUM_CAP, 0x313C, 24, GENMASK(3, 0), 0x2, 0x2, 0x2}, + {ZIP_CORE_NUM_CAP, 0x313C, 16, GENMASK(7, 0), 0x8, 0x8, 0x5}, + {ZIP_CLUSTER_COMP_NUM_CAP, 0x313C, 8, GENMASK(7, 0), 0x2, 0x2, 0x2}, + {ZIP_CLUSTER_DECOMP_NUM_CAP, 0x313C, 0, GENMASK(7, 0), 0x6, 0x6, 0x3}, + {ZIP_DECOMP_ENABLE_BITMAP, 0x3140, 16, GENMASK(15, 0), 0xFC, 0xFC, 0x1C}, + {ZIP_COMP_ENABLE_BITMAP, 0x3140, 0, GENMASK(15, 0), 0x3, 0x3, 0x3}, + {ZIP_DRV_ALG_BITMAP, 0x3144, 0, GENMASK(31, 0), 0x0, 0x0, 0x30}, + {ZIP_DEV_ALG_BITMAP, 0x3148, 0, GENMASK(31, 0), 0xF, 0xF, 0x3F}, + {ZIP_CORE1_ALG_BITMAP, 0x314C, 0, GENMASK(31, 0), 0x5, 0x5, 0xD5}, + {ZIP_CORE2_ALG_BITMAP, 0x3150, 0, GENMASK(31, 0), 0x5, 0x5, 0xD5}, + {ZIP_CORE3_ALG_BITMAP, 0x3154, 0, GENMASK(31, 0), 0xA, 0xA, 0x2A}, + {ZIP_CORE4_ALG_BITMAP, 0x3158, 0, GENMASK(31, 0), 0xA, 0xA, 0x2A}, + {ZIP_CORE5_ALG_BITMAP, 0x315C, 0, GENMASK(31, 0), 0xA, 0xA, 0x2A}, + {ZIP_CAP_MAX, 0x317c, 0, GENMASK(0, 0), 0x0, 0x0, 0x0} +}; + +static const struct hisi_qm_cap_query_info zip_cap_query_info[] = { + {QM_RAS_NFE_TYPE, "QM_RAS_NFE_TYPE ", 0x3124, 0x0, 0x1C57, 0x7C77}, + {QM_RAS_NFE_RESET, "QM_RAS_NFE_RESET ", 0x3128, 0x0, 0xC57, 0x6C77}, + {QM_RAS_CE_TYPE, "QM_RAS_CE_TYPE ", 0x312C, 0x0, 0x8, 0x8}, + {ZIP_RAS_NFE_TYPE, "ZIP_RAS_NFE_TYPE ", 0x3130, 0x0, 0x7FE, 0x1FFE}, + {ZIP_RAS_NFE_RESET, "ZIP_RAS_NFE_RESET ", 0x3134, 0x0, 0x7FE, 0x7FE}, + {ZIP_RAS_CE_TYPE, "ZIP_RAS_CE_TYPE ", 0x3138, 0x0, 0x1, 0x1}, + {ZIP_CORE_INFO, "ZIP_CORE_INFO ", 0x313C, 0x12080206, 0x12080206, 0x12050203}, + {ZIP_CORE_EN, "ZIP_CORE_EN ", 0x3140, 0xFC0003, 0xFC0003, 0x1C0003}, + {ZIP_DRV_ALG_BITMAP_TB, "ZIP_DRV_ALG_BITMAP ", 0x3144, 0x0, 0x0, 0x30}, + {ZIP_ALG_BITMAP, "ZIP_ALG_BITMAP ", 0x3148, 0xF, 0xF, 0x3F}, + {ZIP_CORE1_BITMAP, "ZIP_CORE1_BITMAP ", 0x314C, 0x5, 0x5, 0xD5}, + {ZIP_CORE2_BITMAP, "ZIP_CORE2_BITMAP ", 0x3150, 0x5, 0x5, 0xD5}, + {ZIP_CORE3_BITMAP, "ZIP_CORE3_BITMAP ", 0x3154, 0xA, 0xA, 0x2A}, + {ZIP_CORE4_BITMAP, "ZIP_CORE4_BITMAP ", 0x3158, 0xA, 0xA, 0x2A}, + {ZIP_CORE5_BITMAP, "ZIP_CORE5_BITMAP ", 0x315C, 0xA, 0xA, 0x2A}, +}; + +static const struct debugfs_reg32 hzip_dfx_regs[] = { + {"HZIP_GET_BD_NUM ", 0x00}, + {"HZIP_GET_RIGHT_BD ", 0x04}, + {"HZIP_GET_ERROR_BD ", 0x08}, + {"HZIP_DONE_BD_NUM ", 0x0c}, + {"HZIP_WORK_CYCLE ", 0x10}, + {"HZIP_IDLE_CYCLE ", 0x18}, + {"HZIP_MAX_DELAY ", 0x20}, + {"HZIP_MIN_DELAY ", 0x24}, + {"HZIP_AVG_DELAY ", 0x28}, + {"HZIP_MEM_VISIBLE_DATA ", 0x30}, + {"HZIP_MEM_VISIBLE_ADDR ", 0x34}, + {"HZIP_CONSUMED_BYTE ", 0x38}, + {"HZIP_PRODUCED_BYTE ", 0x40}, + {"HZIP_COMP_INF ", 0x70}, + {"HZIP_PRE_OUT ", 0x78}, + {"HZIP_BD_RD ", 0x7c}, + {"HZIP_BD_WR ", 0x80}, + {"HZIP_GET_BD_AXI_ERR_NUM ", 0x84}, + {"HZIP_GET_BD_PARSE_ERR_NUM ", 0x88}, + {"HZIP_ADD_BD_AXI_ERR_NUM ", 0x8c}, + {"HZIP_DECOMP_STF_RELOAD_CURR_ST ", 0x94}, + {"HZIP_DECOMP_LZ77_CURR_ST ", 0x9c}, +}; + +static const struct debugfs_reg32 hzip_com_dfx_regs[] = { + {"HZIP_CLOCK_GATE_CTRL ", 0x301004}, + {"HZIP_CORE_INT_RAS_CE_ENB ", 0x301160}, + {"HZIP_CORE_INT_RAS_NFE_ENB ", 0x301164}, + {"HZIP_CORE_INT_RAS_FE_ENB ", 0x301168}, + {"HZIP_UNCOM_ERR_RAS_CTRL ", 0x30116C}, +}; + +static const struct debugfs_reg32 hzip_dump_dfx_regs[] = { + {"HZIP_GET_BD_NUM ", 0x00}, + {"HZIP_GET_RIGHT_BD ", 0x04}, + {"HZIP_GET_ERROR_BD ", 0x08}, + {"HZIP_DONE_BD_NUM ", 0x0c}, + {"HZIP_MAX_DELAY ", 0x20}, +}; + +/* define the ZIP's dfx regs region and region length */ +static struct dfx_diff_registers hzip_diff_regs[] = { + { + .reg_offset = HZIP_CORE_DFX_BASE, + .reg_len = HZIP_CORE_REGS_BASE_LEN, + }, { + .reg_offset = HZIP_CORE_DFX_COMP_0, + .reg_len = HZIP_CORE_REGS_DFX_LEN, + }, { + .reg_offset = HZIP_CORE_DFX_COMP_1, + .reg_len = HZIP_CORE_REGS_DFX_LEN, + }, { + .reg_offset = HZIP_CORE_DFX_DECOMP_0, + .reg_len = HZIP_CORE_REGS_DFX_LEN, + }, { + .reg_offset = HZIP_CORE_DFX_DECOMP_1, + .reg_len = HZIP_CORE_REGS_DFX_LEN, + }, { + .reg_offset = HZIP_CORE_DFX_DECOMP_2, + .reg_len = HZIP_CORE_REGS_DFX_LEN, + }, { + .reg_offset = HZIP_CORE_DFX_DECOMP_3, + .reg_len = HZIP_CORE_REGS_DFX_LEN, + }, { + .reg_offset = HZIP_CORE_DFX_DECOMP_4, + .reg_len = HZIP_CORE_REGS_DFX_LEN, + }, { + .reg_offset = HZIP_CORE_DFX_DECOMP_5, + .reg_len = HZIP_CORE_REGS_DFX_LEN, + }, +}; + +static const char *zip_channel_name[HZIP_MAX_CHANNEL_NUM] = { + "COMPRESS", + "DECOMPRESS", + "DAE" +}; + +static int hzip_diff_regs_show(struct seq_file *s, void *unused) +{ + struct hisi_qm *qm = s->private; + + hisi_qm_acc_diff_regs_dump(qm, s, qm->debug.acc_diff_regs, + ARRAY_SIZE(hzip_diff_regs)); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(hzip_diff_regs); + +static int perf_mode_set(const char *val, const struct kernel_param *kp) +{ + int ret; + u32 n; + + if (!val) + return -EINVAL; + + ret = kstrtou32(val, 10, &n); + if (ret != 0 || (n != HZIP_HIGH_COMP_PERF && + n != HZIP_HIGH_COMP_RATE)) + return -EINVAL; + + return param_set_int(val, kp); +} + +static const struct kernel_param_ops zip_com_perf_ops = { + .set = perf_mode_set, + .get = param_get_int, +}; + +/* + * perf_mode = 0 means enable high compression rate mode, + * perf_mode = 1 means enable high compression performance mode. + * These two modes only apply to the compression direction. + */ +static u32 perf_mode = HZIP_HIGH_COMP_RATE; +module_param_cb(perf_mode, &zip_com_perf_ops, &perf_mode, 0444); +MODULE_PARM_DESC(perf_mode, "ZIP high perf mode 0(default), 1(enable)"); + +static const struct kernel_param_ops zip_uacce_mode_ops = { + .set = uacce_mode_set, + .get = param_get_int, +}; + +/* + * uacce_mode = 0 means zip only register to crypto, + * uacce_mode = 1 means zip both register to crypto and uacce. + */ +static u32 uacce_mode = UACCE_MODE_NOUACCE; +module_param_cb(uacce_mode, &zip_uacce_mode_ops, &uacce_mode, 0444); +MODULE_PARM_DESC(uacce_mode, UACCE_MODE_DESC); + +static bool pf_q_num_flag; +static int pf_q_num_set(const char *val, const struct kernel_param *kp) +{ + pf_q_num_flag = true; + + return hisi_qm_q_num_set(val, kp, PCI_DEVICE_ID_HUAWEI_ZIP_PF); +} + +static const struct kernel_param_ops pf_q_num_ops = { + .set = pf_q_num_set, + .get = param_get_int, +}; + +static u32 pf_q_num = HZIP_PF_DEF_Q_NUM; +module_param_cb(pf_q_num, &pf_q_num_ops, &pf_q_num, 0444); +MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 2-4096, v2 2-1024)"); + +static const struct kernel_param_ops vfs_num_ops = { + .set = vfs_num_set, + .get = param_get_int, +}; + +static u32 vfs_num; +module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444); +MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)"); + +static const struct pci_device_id hisi_zip_dev_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_ZIP_PF) }, + { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_ZIP_VF) }, + { 0, } +}; +MODULE_DEVICE_TABLE(pci, hisi_zip_dev_ids); + +int zip_create_qps(struct hisi_qp **qps, int qp_num, int node, u8 *alg_type) +{ + if (node == NUMA_NO_NODE) + node = cpu_to_node(raw_smp_processor_id()); + + return hisi_qm_alloc_qps_node(&zip_devices, qp_num, alg_type, node, qps); +} + +bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg) +{ + u32 cap_val; + + cap_val = qm->cap_tables.dev_cap_table[ZIP_DRV_ALG_BITMAP_TB].cap_val; + if ((alg & cap_val) == alg) + return true; + + return false; +} + +static void hisi_zip_literal_set(struct hisi_qm *qm) +{ + u32 val; + + if (qm->ver < QM_HW_V3) + return; + + val = readl_relaxed(qm->io_base + HZIP_LIT_LEN_EN_OFFSET); + val &= ~HZIP_LIT_LEN_EN_EN; + + /* enable literal length in stream mode compression */ + writel(val, qm->io_base + HZIP_LIT_LEN_EN_OFFSET); +} + +static void hisi_zip_set_high_perf(struct hisi_qm *qm) +{ + u32 val; + + val = readl_relaxed(qm->io_base + HZIP_HIGH_PERF_OFFSET); + if (perf_mode == HZIP_HIGH_COMP_PERF) + val |= HZIP_HIGH_COMP_PERF; + else + val &= ~HZIP_HIGH_COMP_PERF; + + /* Set perf mode */ + writel(val, qm->io_base + HZIP_HIGH_PERF_OFFSET); +} + +static int hisi_zip_wait_sva_ready(struct hisi_qm *qm, __u32 offset, __u32 mask) +{ + u32 val, try_times = 0; + u8 count = 0; + + /* + * Read the register value every 10-20us. If the value is 0 for three + * consecutive times, the SVA module is ready. + */ + do { + val = readl(qm->io_base + offset); + if (val & mask) + count = 0; + else if (++count == HZIP_READ_SVA_STATUS_TIMES) + break; + + usleep_range(HZIP_WAIT_US_MIN, HZIP_WAIT_US_MAX); + } while (++try_times < HZIP_WAIT_SVA_READY); + + if (try_times == HZIP_WAIT_SVA_READY) { + pci_err(qm->pdev, "failed to wait sva prefetch ready\n"); + return -ETIMEDOUT; + } + + return 0; +} + +static void hisi_zip_close_sva_prefetch(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) + return; + + val = readl_relaxed(qm->io_base + HZIP_PREFETCH_CFG); + val |= HZIP_SVA_PREFETCH_DISABLE; + writel(val, qm->io_base + HZIP_PREFETCH_CFG); + + ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_SVA_TRANS, + val, !(val & HZIP_SVA_DISABLE_READY), + HZIP_DELAY_1_US, HZIP_POLL_TIMEOUT_US); + if (ret) + pci_err(qm->pdev, "failed to close sva prefetch\n"); + + (void)hisi_zip_wait_sva_ready(qm, HZIP_SVA_TRANS, HZIP_SVA_STALL_NUM); +} + +static void hisi_zip_open_sva_prefetch(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) + return; + + /* Enable prefetch */ + val = readl_relaxed(qm->io_base + HZIP_PREFETCH_CFG); + val &= HZIP_PREFETCH_ENABLE; + writel(val, qm->io_base + HZIP_PREFETCH_CFG); + + ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_PREFETCH_CFG, + val, !(val & HZIP_SVA_PREFETCH_DISABLE), + HZIP_DELAY_1_US, HZIP_POLL_TIMEOUT_US); + if (ret) { + pci_err(qm->pdev, "failed to open sva prefetch\n"); + hisi_zip_close_sva_prefetch(qm); + return; + } + + ret = hisi_zip_wait_sva_ready(qm, HZIP_SVA_TRANS, HZIP_SVA_PREFETCH_NUM); + if (ret) + hisi_zip_close_sva_prefetch(qm); +} + +static void hisi_zip_enable_clock_gate(struct hisi_qm *qm) +{ + u32 val; + + if (qm->ver < QM_HW_V3) + return; + + val = readl(qm->io_base + HZIP_CLOCK_GATE_CTRL); + val |= HZIP_CLOCK_GATED_EN; + writel(val, qm->io_base + HZIP_CLOCK_GATE_CTRL); + + val = readl(qm->io_base + HZIP_PEH_CFG_AUTO_GATE); + val |= HZIP_PEH_CFG_AUTO_GATE_EN; + writel(val, qm->io_base + HZIP_PEH_CFG_AUTO_GATE); +} + +static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) +{ + void __iomem *base = qm->io_base; + u32 dcomp_bm, comp_bm; + u32 zip_core_en; + int ret; + + /* qm user domain */ + writel(AXUSER_BASE, base + QM_ARUSER_M_CFG_1); + writel(ARUSER_M_CFG_ENABLE, base + QM_ARUSER_M_CFG_ENABLE); + writel(AXUSER_BASE, base + QM_AWUSER_M_CFG_1); + writel(AWUSER_M_CFG_ENABLE, base + QM_AWUSER_M_CFG_ENABLE); + writel(WUSER_M_CFG_ENABLE, base + QM_WUSER_M_CFG_ENABLE); + + /* qm cache */ + writel(AXI_M_CFG, base + QM_AXI_M_CFG); + writel(AXI_M_CFG_ENABLE, base + QM_AXI_M_CFG_ENABLE); + + /* disable FLR triggered by BME(bus master enable) */ + writel(PEH_AXUSER_CFG, base + QM_PEH_AXUSER_CFG); + writel(PEH_AXUSER_CFG_ENABLE, base + QM_PEH_AXUSER_CFG_ENABLE); + + /* cache */ + writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_0); + writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_1); + writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_0); + writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_1); + + /* user domain configurations */ + writel(AXUSER_BASE, base + HZIP_BD_RUSER_32_63); + writel(AXUSER_BASE, base + HZIP_BD_WUSER_32_63); + + if (qm->use_sva && qm->ver == QM_HW_V2) { + writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_RUSER_32_63); + writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_WUSER_32_63); + writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_SGL_RUSER_32_63); + } else { + writel(AXUSER_BASE, base + HZIP_DATA_RUSER_32_63); + writel(AXUSER_BASE, base + HZIP_DATA_WUSER_32_63); + writel(AXUSER_BASE, base + HZIP_SGL_RUSER_32_63); + } + hisi_zip_open_sva_prefetch(qm); + + /* let's open all compression/decompression cores */ + + zip_core_en = qm->cap_tables.dev_cap_table[ZIP_CORE_EN].cap_val; + dcomp_bm = (zip_core_en >> zip_basic_cap_info[ZIP_DECOMP_ENABLE_BITMAP].shift) & + zip_basic_cap_info[ZIP_DECOMP_ENABLE_BITMAP].mask; + comp_bm = (zip_core_en >> zip_basic_cap_info[ZIP_COMP_ENABLE_BITMAP].shift) & + zip_basic_cap_info[ZIP_COMP_ENABLE_BITMAP].mask; + writel(HZIP_DECOMP_CHECK_ENABLE | dcomp_bm | comp_bm, base + HZIP_CLOCK_GATE_CTRL); + + /* enable sqc,cqc writeback */ + writel(SQC_CACHE_ENABLE | CQC_CACHE_ENABLE | SQC_CACHE_WB_ENABLE | + CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) | + FIELD_PREP(CQC_CACHE_WB_THRD, 1), base + QM_CACHE_CTL); + + hisi_zip_set_high_perf(qm); + hisi_zip_literal_set(qm); + hisi_zip_enable_clock_gate(qm); + + ret = hisi_dae_set_user_domain(qm); + if (ret) + goto close_sva_prefetch; + + return 0; + +close_sva_prefetch: + hisi_zip_close_sva_prefetch(qm); + return ret; +} + +static void hisi_zip_master_ooo_ctrl(struct hisi_qm *qm, bool enable) +{ + u32 val1, val2; + + val1 = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); + if (enable) { + val1 |= HZIP_AXI_SHUTDOWN_ENABLE; + val2 = qm->err_info.dev_err.shutdown_mask; + } else { + val1 &= ~HZIP_AXI_SHUTDOWN_ENABLE; + val2 = 0x0; + } + + if (qm->ver > QM_HW_V2) + writel(val2, qm->io_base + HZIP_OOO_SHUTDOWN_SEL); + + writel(val1, qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); +} + +static void hisi_zip_hw_error_enable(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; + + if (qm->ver == QM_HW_V1) { + writel(HZIP_CORE_INT_MASK_ALL, + qm->io_base + HZIP_CORE_INT_MASK_REG); + dev_info(&qm->pdev->dev, "Does not support hw error handle\n"); + return; + } + + /* clear ZIP hw error source if having */ + writel(HZIP_CORE_RAS_CLEAR_ALL, qm->io_base + HZIP_CORE_INT_SOURCE); + + /* configure error type */ + writel(dev_err->ce, qm->io_base + HZIP_CORE_INT_RAS_CE_ENB); + writel(dev_err->fe, qm->io_base + HZIP_CORE_INT_RAS_FE_ENB); + writel(dev_err->nfe, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); + + hisi_zip_master_ooo_ctrl(qm, true); + + /* enable ZIP hw error interrupts */ + writel(~err_mask, qm->io_base + HZIP_CORE_INT_MASK_REG); + + hisi_dae_hw_error_enable(qm); +} + +static void hisi_zip_hw_error_disable(struct hisi_qm *qm) +{ + /* disable ZIP hw error interrupts */ + writel(HZIP_CORE_INT_MASK_ALL, qm->io_base + HZIP_CORE_INT_MASK_REG); + + hisi_zip_master_ooo_ctrl(qm, false); + + hisi_dae_hw_error_disable(qm); +} + +static inline struct hisi_qm *file_to_qm(struct ctrl_debug_file *file) +{ + struct hisi_zip *hisi_zip = file->ctrl->hisi_zip; + + return &hisi_zip->qm; +} + +static u32 clear_enable_read(struct hisi_qm *qm) +{ + return readl(qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE) & + HZIP_SOFT_CTRL_CNT_CLR_CE_BIT; +} + +static int clear_enable_write(struct hisi_qm *qm, u32 val) +{ + u32 tmp; + + if (val != 1 && val != 0) + return -EINVAL; + + tmp = (readl(qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE) & + ~HZIP_SOFT_CTRL_CNT_CLR_CE_BIT) | val; + writel(tmp, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE); + + return 0; +} + +static ssize_t hisi_zip_ctrl_debug_read(struct file *filp, char __user *buf, + size_t count, loff_t *pos) +{ + struct ctrl_debug_file *file = filp->private_data; + struct hisi_qm *qm = file_to_qm(file); + char tbuf[HZIP_BUF_SIZE]; + u32 val; + int ret; + + ret = hisi_qm_get_dfx_access(qm); + if (ret) + return ret; + + spin_lock_irq(&file->lock); + switch (file->index) { + case HZIP_CLEAR_ENABLE: + val = clear_enable_read(qm); + break; + default: + goto err_input; + } + spin_unlock_irq(&file->lock); + + hisi_qm_put_dfx_access(qm); + ret = scnprintf(tbuf, sizeof(tbuf), "%u\n", val); + return simple_read_from_buffer(buf, count, pos, tbuf, ret); + +err_input: + spin_unlock_irq(&file->lock); + hisi_qm_put_dfx_access(qm); + return -EINVAL; +} + +static ssize_t hisi_zip_ctrl_debug_write(struct file *filp, + const char __user *buf, + size_t count, loff_t *pos) +{ + struct ctrl_debug_file *file = filp->private_data; + struct hisi_qm *qm = file_to_qm(file); + char tbuf[HZIP_BUF_SIZE]; + unsigned long val; + int len, ret; + + if (*pos != 0) + return 0; + + if (count >= HZIP_BUF_SIZE) + return -ENOSPC; + + len = simple_write_to_buffer(tbuf, HZIP_BUF_SIZE - 1, pos, buf, count); + if (len < 0) + return len; + + tbuf[len] = '\0'; + ret = kstrtoul(tbuf, 0, &val); + if (ret) + return ret; + + ret = hisi_qm_get_dfx_access(qm); + if (ret) + return ret; + + spin_lock_irq(&file->lock); + switch (file->index) { + case HZIP_CLEAR_ENABLE: + ret = clear_enable_write(qm, val); + if (ret) + goto err_input; + break; + default: + ret = -EINVAL; + goto err_input; + } + + ret = count; + +err_input: + spin_unlock_irq(&file->lock); + hisi_qm_put_dfx_access(qm); + return ret; +} + +static const struct file_operations ctrl_debug_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = hisi_zip_ctrl_debug_read, + .write = hisi_zip_ctrl_debug_write, +}; + +static int zip_debugfs_atomic64_set(void *data, u64 val) +{ + if (val) + return -EINVAL; + + atomic64_set((atomic64_t *)data, 0); + + return 0; +} + +static int zip_debugfs_atomic64_get(void *data, u64 *val) +{ + *val = atomic64_read((atomic64_t *)data); + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(zip_atomic64_ops, zip_debugfs_atomic64_get, + zip_debugfs_atomic64_set, "%llu\n"); + +static int hisi_zip_regs_show(struct seq_file *s, void *unused) +{ + hisi_qm_regs_dump(s, s->private); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(hisi_zip_regs); + +static void __iomem *get_zip_core_addr(struct hisi_qm *qm, int core_num) +{ + u8 zip_comp_core_num; + u32 zip_core_info; + + zip_core_info = qm->cap_tables.dev_cap_table[ZIP_CORE_INFO].cap_val; + zip_comp_core_num = (zip_core_info >> zip_basic_cap_info[ZIP_CLUSTER_COMP_NUM_CAP].shift) & + zip_basic_cap_info[ZIP_CLUSTER_COMP_NUM_CAP].mask; + + if (core_num < zip_comp_core_num) + return qm->io_base + HZIP_CORE_DFX_BASE + + (core_num + 1) * HZIP_CORE_ADDR_INTRVL; + + return qm->io_base + HZIP_CORE_DFX_DECOMP_BASE + + (core_num - zip_comp_core_num) * HZIP_CORE_ADDR_INTRVL; +} + +static int hisi_zip_core_debug_init(struct hisi_qm *qm) +{ + u32 zip_core_num, zip_comp_core_num; + struct device *dev = &qm->pdev->dev; + struct debugfs_regset32 *regset; + u32 zip_core_info; + struct dentry *tmp_d; + char buf[HZIP_BUF_SIZE]; + int i; + + zip_core_info = qm->cap_tables.dev_cap_table[ZIP_CORE_INFO].cap_val; + zip_core_num = (zip_core_info >> zip_basic_cap_info[ZIP_CORE_NUM_CAP].shift) & + zip_basic_cap_info[ZIP_CORE_NUM_CAP].mask; + zip_comp_core_num = (zip_core_info >> zip_basic_cap_info[ZIP_CLUSTER_COMP_NUM_CAP].shift) & + zip_basic_cap_info[ZIP_CLUSTER_COMP_NUM_CAP].mask; + + for (i = 0; i < zip_core_num; i++) { + if (i < zip_comp_core_num) + scnprintf(buf, sizeof(buf), "comp_core%d", i); + else + scnprintf(buf, sizeof(buf), "decomp_core%d", + i - zip_comp_core_num); + + regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL); + if (!regset) + return -ENOENT; + + regset->regs = hzip_dfx_regs; + regset->nregs = ARRAY_SIZE(hzip_dfx_regs); + regset->base = get_zip_core_addr(qm, i); + regset->dev = dev; + + tmp_d = debugfs_create_dir(buf, qm->debug.debug_root); + debugfs_create_file("regs", 0444, tmp_d, regset, + &hisi_zip_regs_fops); + } + + return 0; +} + +static int zip_cap_regs_show(struct seq_file *s, void *unused) +{ + struct hisi_qm *qm = s->private; + u32 i, size; + + size = qm->cap_tables.qm_cap_size; + for (i = 0; i < size; i++) + seq_printf(s, "%s= 0x%08x\n", qm->cap_tables.qm_cap_table[i].name, + qm->cap_tables.qm_cap_table[i].cap_val); + + size = qm->cap_tables.dev_cap_size; + for (i = 0; i < size; i++) + seq_printf(s, "%s= 0x%08x\n", qm->cap_tables.dev_cap_table[i].name, + qm->cap_tables.dev_cap_table[i].cap_val); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(zip_cap_regs); + +static void hisi_zip_dfx_debug_init(struct hisi_qm *qm) +{ + struct dfx_diff_registers *hzip_regs = qm->debug.acc_diff_regs; + struct hisi_zip *zip = container_of(qm, struct hisi_zip, qm); + struct hisi_zip_dfx *dfx = &zip->dfx; + struct dentry *tmp_dir; + void *data; + int i; + + tmp_dir = debugfs_create_dir("zip_dfx", qm->debug.debug_root); + for (i = 0; i < ARRAY_SIZE(zip_dfx_files); i++) { + data = (atomic64_t *)((uintptr_t)dfx + zip_dfx_files[i].offset); + debugfs_create_file(zip_dfx_files[i].name, + 0644, tmp_dir, data, + &zip_atomic64_ops); + } + + if (qm->fun_type == QM_HW_PF && hzip_regs) + debugfs_create_file("diff_regs", 0444, tmp_dir, + qm, &hzip_diff_regs_fops); + + debugfs_create_file("cap_regs", CAP_FILE_PERMISSION, + qm->debug.debug_root, qm, &zip_cap_regs_fops); +} + +static int hisi_zip_ctrl_debug_init(struct hisi_qm *qm) +{ + struct hisi_zip *zip = container_of(qm, struct hisi_zip, qm); + int i; + + for (i = HZIP_CLEAR_ENABLE; i < HZIP_DEBUG_FILE_NUM; i++) { + spin_lock_init(&zip->ctrl->files[i].lock); + zip->ctrl->files[i].ctrl = zip->ctrl; + zip->ctrl->files[i].index = i; + + debugfs_create_file(ctrl_debug_file_name[i], 0600, + qm->debug.debug_root, + zip->ctrl->files + i, + &ctrl_debug_fops); + } + + return hisi_zip_core_debug_init(qm); +} + +static int hisi_zip_debugfs_init(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + int ret; + + ret = hisi_qm_regs_debugfs_init(qm, hzip_diff_regs, ARRAY_SIZE(hzip_diff_regs)); + if (ret) { + dev_warn(dev, "Failed to init ZIP diff regs!\n"); + return ret; + } + + qm->debug.sqe_mask_offset = HZIP_SQE_MASK_OFFSET; + qm->debug.sqe_mask_len = HZIP_SQE_MASK_LEN; + qm->debug.debug_root = debugfs_create_dir(dev_name(dev), + hzip_debugfs_root); + + hisi_qm_debug_init(qm); + + if (qm->fun_type == QM_HW_PF) { + ret = hisi_zip_ctrl_debug_init(qm); + if (ret) + goto debugfs_remove; + } + + hisi_zip_dfx_debug_init(qm); + + return 0; + +debugfs_remove: + debugfs_remove_recursive(qm->debug.debug_root); + hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(hzip_diff_regs)); + return ret; +} + +/* hisi_zip_debug_regs_clear() - clear the zip debug regs */ +static void hisi_zip_debug_regs_clear(struct hisi_qm *qm) +{ + u32 zip_core_info; + u8 zip_core_num; + int i, j; + + zip_core_info = qm->cap_tables.dev_cap_table[ZIP_CORE_INFO].cap_val; + zip_core_num = (zip_core_info >> zip_basic_cap_info[ZIP_CORE_NUM_CAP].shift) & + zip_basic_cap_info[ZIP_CORE_NUM_CAP].mask; + + /* enable register read_clear bit */ + writel(HZIP_RD_CNT_CLR_CE_EN, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE); + for (i = 0; i < zip_core_num; i++) + for (j = 0; j < ARRAY_SIZE(hzip_dfx_regs); j++) + readl(get_zip_core_addr(qm, i) + + hzip_dfx_regs[j].offset); + + /* disable register read_clear bit */ + writel(0x0, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE); + + hisi_qm_debug_regs_clear(qm); +} + +static void hisi_zip_debugfs_exit(struct hisi_qm *qm) +{ + debugfs_remove_recursive(qm->debug.debug_root); + + hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(hzip_diff_regs)); + + if (qm->fun_type == QM_HW_PF) { + hisi_zip_debug_regs_clear(qm); + qm->debug.curr_qm_qp_num = 0; + } +} + +static int hisi_zip_show_last_regs_init(struct hisi_qm *qm) +{ + int core_dfx_regs_num = ARRAY_SIZE(hzip_dump_dfx_regs); + int com_dfx_regs_num = ARRAY_SIZE(hzip_com_dfx_regs); + struct qm_debug *debug = &qm->debug; + void __iomem *io_base; + u32 zip_core_info; + u32 zip_core_num; + int i, j, idx; + + zip_core_info = qm->cap_tables.dev_cap_table[ZIP_CORE_INFO].cap_val; + zip_core_num = (zip_core_info >> zip_basic_cap_info[ZIP_CORE_NUM_CAP].shift) & + zip_basic_cap_info[ZIP_CORE_NUM_CAP].mask; + + debug->last_words = kcalloc(core_dfx_regs_num * zip_core_num + com_dfx_regs_num, + sizeof(unsigned int), GFP_KERNEL); + if (!debug->last_words) + return -ENOMEM; + + for (i = 0; i < com_dfx_regs_num; i++) { + io_base = qm->io_base + hzip_com_dfx_regs[i].offset; + debug->last_words[i] = readl_relaxed(io_base); + } + + for (i = 0; i < zip_core_num; i++) { + io_base = get_zip_core_addr(qm, i); + for (j = 0; j < core_dfx_regs_num; j++) { + idx = com_dfx_regs_num + i * core_dfx_regs_num + j; + debug->last_words[idx] = readl_relaxed( + io_base + hzip_dump_dfx_regs[j].offset); + } + } + + return 0; +} + +static void hisi_zip_show_last_regs_uninit(struct hisi_qm *qm) +{ + struct qm_debug *debug = &qm->debug; + + if (qm->fun_type == QM_HW_VF || !debug->last_words) + return; + + kfree(debug->last_words); + debug->last_words = NULL; +} + +static void hisi_zip_show_last_dfx_regs(struct hisi_qm *qm) +{ + int core_dfx_regs_num = ARRAY_SIZE(hzip_dump_dfx_regs); + int com_dfx_regs_num = ARRAY_SIZE(hzip_com_dfx_regs); + u32 zip_core_num, zip_comp_core_num; + struct qm_debug *debug = &qm->debug; + char buf[HZIP_BUF_SIZE]; + u32 zip_core_info; + void __iomem *base; + int i, j, idx; + u32 val; + + if (qm->fun_type == QM_HW_VF || !debug->last_words) + return; + + for (i = 0; i < com_dfx_regs_num; i++) { + val = readl_relaxed(qm->io_base + hzip_com_dfx_regs[i].offset); + if (debug->last_words[i] != val) + pci_info(qm->pdev, "com_dfx: %s \t= 0x%08x => 0x%08x\n", + hzip_com_dfx_regs[i].name, debug->last_words[i], val); + } + + zip_core_info = qm->cap_tables.dev_cap_table[ZIP_CORE_INFO].cap_val; + zip_core_num = (zip_core_info >> zip_basic_cap_info[ZIP_CORE_NUM_CAP].shift) & + zip_basic_cap_info[ZIP_CORE_NUM_CAP].mask; + zip_comp_core_num = (zip_core_info >> zip_basic_cap_info[ZIP_CLUSTER_COMP_NUM_CAP].shift) & + zip_basic_cap_info[ZIP_CLUSTER_COMP_NUM_CAP].mask; + + for (i = 0; i < zip_core_num; i++) { + if (i < zip_comp_core_num) + scnprintf(buf, sizeof(buf), "Comp_core-%d", i); + else + scnprintf(buf, sizeof(buf), "Decomp_core-%d", + i - zip_comp_core_num); + base = get_zip_core_addr(qm, i); + + pci_info(qm->pdev, "==>%s:\n", buf); + /* dump last word for dfx regs during control resetting */ + for (j = 0; j < core_dfx_regs_num; j++) { + idx = com_dfx_regs_num + i * core_dfx_regs_num + j; + val = readl_relaxed(base + hzip_dump_dfx_regs[j].offset); + if (debug->last_words[idx] != val) + pci_info(qm->pdev, "%s \t= 0x%08x => 0x%08x\n", + hzip_dump_dfx_regs[j].name, + debug->last_words[idx], val); + } + } +} + +static void hisi_zip_log_hw_error(struct hisi_qm *qm, u32 err_sts) +{ + const struct hisi_zip_hw_error *err = zip_hw_error; + struct device *dev = &qm->pdev->dev; + u32 err_val; + + while (err->msg) { + if (err->int_msk & err_sts) { + dev_err(dev, "%s [error status=0x%x] found\n", + err->msg, err->int_msk); + + if (err->int_msk & HZIP_CORE_INT_STATUS_M_ECC) { + err_val = readl(qm->io_base + + HZIP_CORE_SRAM_ECC_ERR_INFO); + dev_err(dev, "hisi-zip multi ecc sram num=0x%x\n", + ((err_val >> + HZIP_SRAM_ECC_ERR_NUM_SHIFT) & 0xFF)); + } + } + err++; + } +} + +static u32 hisi_zip_get_hw_err_status(struct hisi_qm *qm) +{ + return readl(qm->io_base + HZIP_CORE_INT_STATUS); +} + +static void hisi_zip_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) +{ + writel(err_sts, qm->io_base + HZIP_CORE_INT_SOURCE); +} + +static void hisi_zip_disable_error_report(struct hisi_qm *qm, u32 err_type) +{ + u32 nfe_mask = qm->err_info.dev_err.nfe; + + writel(nfe_mask & (~err_type), qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); +} + +static void hisi_zip_enable_error_report(struct hisi_qm *qm) +{ + u32 nfe_mask = qm->err_info.dev_err.nfe; + u32 ce_mask = qm->err_info.dev_err.ce; + + writel(nfe_mask, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); + writel(ce_mask, qm->io_base + HZIP_CORE_INT_RAS_CE_ENB); +} + +static void hisi_zip_open_axi_master_ooo(struct hisi_qm *qm) +{ + u32 val; + + val = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); + + writel(val & ~HZIP_AXI_SHUTDOWN_ENABLE, + qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); + + writel(val | HZIP_AXI_SHUTDOWN_ENABLE, + qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); + + hisi_dae_open_axi_master_ooo(qm); +} + +static void hisi_zip_close_axi_master_ooo(struct hisi_qm *qm) +{ + u32 nfe_enb; + + /* Disable ECC Mbit error report. */ + nfe_enb = readl(qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); + writel(nfe_enb & ~HZIP_CORE_INT_STATUS_M_ECC, + qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); + + /* Inject zip ECC Mbit error to block master ooo. */ + writel(HZIP_CORE_INT_STATUS_M_ECC, + qm->io_base + HZIP_CORE_INT_SET); +} + +static enum acc_err_result hisi_zip_get_err_result(struct hisi_qm *qm) +{ + enum acc_err_result zip_result = ACC_ERR_NONE; + enum acc_err_result dae_result; + u32 err_status; + + /* Get device hardware new error status */ + err_status = hisi_zip_get_hw_err_status(qm); + if (err_status) { + if (err_status & qm->err_info.dev_err.ecc_2bits_mask) + qm->err_status.is_dev_ecc_mbit = true; + hisi_zip_log_hw_error(qm, err_status); + + if (err_status & qm->err_info.dev_err.reset_mask) { + /* Disable the same error reporting until device is recovered. */ + hisi_zip_disable_error_report(qm, err_status); + zip_result = ACC_ERR_NEED_RESET; + } else { + hisi_zip_clear_hw_err_status(qm, err_status); + /* Avoid firmware disable error report, re-enable. */ + hisi_zip_enable_error_report(qm); + } + } + + dae_result = hisi_dae_get_err_result(qm); + + return (zip_result == ACC_ERR_NEED_RESET || + dae_result == ACC_ERR_NEED_RESET) ? + ACC_ERR_NEED_RESET : ACC_ERR_RECOVERED; +} + +static bool hisi_zip_dev_is_abnormal(struct hisi_qm *qm) +{ + u32 err_status; + + err_status = hisi_zip_get_hw_err_status(qm); + if (err_status & qm->err_info.dev_err.shutdown_mask) + return true; + + return hisi_dae_dev_is_abnormal(qm); +} + +static int hisi_zip_set_priv_status(struct hisi_qm *qm) +{ + return hisi_dae_close_axi_master_ooo(qm); +} + +static void hisi_zip_disable_axi_error(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; + u32 val; + + val = ~(err_mask & (~HZIP_AXI_ERROR_MASK)); + writel(val, qm->io_base + HZIP_CORE_INT_MASK_REG); + + if (qm->ver > QM_HW_V2) + writel(dev_err->shutdown_mask & (~HZIP_AXI_ERROR_MASK), + qm->io_base + HZIP_OOO_SHUTDOWN_SEL); +} + +static void hisi_zip_enable_axi_error(struct hisi_qm *qm) +{ + struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; + u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; + + /* clear axi error source */ + writel(HZIP_AXI_ERROR_MASK, qm->io_base + HZIP_CORE_INT_SOURCE); + + writel(~err_mask, qm->io_base + HZIP_CORE_INT_MASK_REG); + + if (qm->ver > QM_HW_V2) + writel(dev_err->shutdown_mask, qm->io_base + HZIP_OOO_SHUTDOWN_SEL); +} + +static void hisi_zip_err_info_init(struct hisi_qm *qm) +{ + struct hisi_qm_err_info *err_info = &qm->err_info; + struct hisi_qm_err_mask *qm_err = &err_info->qm_err; + struct hisi_qm_err_mask *dev_err = &err_info->dev_err; + + qm_err->fe = HZIP_CORE_INT_RAS_FE_ENB_MASK; + qm_err->ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_QM_CE_MASK_CAP, qm->cap_ver); + qm_err->nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_QM_NFE_MASK_CAP, qm->cap_ver); + qm_err->ecc_2bits_mask = QM_ECC_MBIT; + qm_err->reset_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_QM_RESET_MASK_CAP, qm->cap_ver); + qm_err->shutdown_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + + dev_err->fe = HZIP_CORE_INT_RAS_FE_ENB_MASK; + dev_err->ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CE_MASK_CAP, qm->cap_ver); + dev_err->nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver); + dev_err->ecc_2bits_mask = HZIP_CORE_INT_STATUS_M_ECC; + dev_err->shutdown_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + dev_err->reset_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_RESET_MASK_CAP, qm->cap_ver); + + err_info->msi_wr_port = HZIP_WR_PORT; + err_info->acpi_rst = "ZRST"; +} + +static const struct hisi_qm_err_ini hisi_zip_err_ini = { + .hw_init = hisi_zip_set_user_domain_and_cache, + .hw_err_enable = hisi_zip_hw_error_enable, + .hw_err_disable = hisi_zip_hw_error_disable, + .get_dev_hw_err_status = hisi_zip_get_hw_err_status, + .clear_dev_hw_err_status = hisi_zip_clear_hw_err_status, + .open_axi_master_ooo = hisi_zip_open_axi_master_ooo, + .close_axi_master_ooo = hisi_zip_close_axi_master_ooo, + .open_sva_prefetch = hisi_zip_open_sva_prefetch, + .close_sva_prefetch = hisi_zip_close_sva_prefetch, + .show_last_dfx_regs = hisi_zip_show_last_dfx_regs, + .err_info_init = hisi_zip_err_info_init, + .get_err_result = hisi_zip_get_err_result, + .set_priv_status = hisi_zip_set_priv_status, + .dev_is_abnormal = hisi_zip_dev_is_abnormal, + .disable_axi_error = hisi_zip_disable_axi_error, + .enable_axi_error = hisi_zip_enable_axi_error, +}; + +static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) +{ + struct hisi_qm *qm = &hisi_zip->qm; + struct hisi_zip_ctrl *ctrl; + int ret; + + ctrl = devm_kzalloc(&qm->pdev->dev, sizeof(*ctrl), GFP_KERNEL); + if (!ctrl) + return -ENOMEM; + + hisi_zip->ctrl = ctrl; + ctrl->hisi_zip = hisi_zip; + + ret = hisi_zip_set_user_domain_and_cache(qm); + if (ret) + return ret; + + hisi_qm_dev_err_init(qm); + hisi_zip_debug_regs_clear(qm); + + ret = hisi_zip_show_last_regs_init(qm); + if (ret) + pci_err(qm->pdev, "Failed to init last word regs!\n"); + + return ret; +} + +static int zip_pre_store_cap_reg(struct hisi_qm *qm) +{ + struct hisi_qm_cap_record *zip_cap; + struct pci_dev *pdev = qm->pdev; + size_t i, size; + + size = ARRAY_SIZE(zip_cap_query_info); + zip_cap = devm_kcalloc(&pdev->dev, size, sizeof(*zip_cap), GFP_KERNEL); + if (!zip_cap) + return -ENOMEM; + + for (i = 0; i < size; i++) { + zip_cap[i].type = zip_cap_query_info[i].type; + zip_cap[i].name = zip_cap_query_info[i].name; + zip_cap[i].cap_val = hisi_qm_get_cap_value(qm, zip_cap_query_info, + i, qm->cap_ver); + } + + qm->cap_tables.dev_cap_table = zip_cap; + qm->cap_tables.dev_cap_size = size; + + return 0; +} + +static void zip_set_channels(struct hisi_qm *qm) +{ + struct qm_channel *channel_data = &qm->channel_data; + int i; + + channel_data->channel_num = HZIP_MAX_CHANNEL_NUM; + for (i = 0; i < HZIP_MAX_CHANNEL_NUM; i++) + channel_data->channel_name[i] = zip_channel_name[i]; +} + +static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) +{ + u64 alg_msk; + int ret; + + qm->pdev = pdev; + qm->mode = uacce_mode; + qm->sqe_size = HZIP_SQE_SIZE; + qm->dev_name = hisi_zip_name; + + qm->fun_type = (pdev->device == PCI_DEVICE_ID_HUAWEI_ZIP_PF) ? + QM_HW_PF : QM_HW_VF; + if (qm->fun_type == QM_HW_PF) { + qm->qp_base = HZIP_PF_DEF_Q_BASE; + qm->qp_num = pf_q_num; + qm->debug.curr_qm_qp_num = pf_q_num; + qm->qm_list = &zip_devices; + qm->err_ini = &hisi_zip_err_ini; + if (pf_q_num_flag) + set_bit(QM_MODULE_PARAM, &qm->misc_ctl); + } else if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V1) { + /* + * have no way to get qm configure in VM in v1 hardware, + * so currently force PF to uses HZIP_PF_DEF_Q_NUM, and force + * to trigger only one VF in v1 hardware. + * + * v2 hardware has no such problem. + */ + qm->qp_base = HZIP_PF_DEF_Q_NUM; + qm->qp_num = HZIP_QUEUE_NUM_V1 - HZIP_PF_DEF_Q_NUM; + } + + ret = hisi_qm_init(qm); + if (ret) { + pci_err(qm->pdev, "Failed to init zip qm configures!\n"); + return ret; + } + + zip_set_channels(qm); + /* Fetch and save the value of capability registers */ + ret = zip_pre_store_cap_reg(qm); + if (ret) { + pci_err(qm->pdev, "Failed to pre-store capability registers!\n"); + goto err_qm_uninit; + } + + alg_msk = qm->cap_tables.dev_cap_table[ZIP_ALG_BITMAP].cap_val; + ret = hisi_qm_set_algs(qm, alg_msk, zip_dev_algs, ARRAY_SIZE(zip_dev_algs)); + if (ret) { + pci_err(qm->pdev, "Failed to set zip algs!\n"); + goto err_qm_uninit; + } + + ret = hisi_dae_set_alg(qm); + if (ret) + goto err_qm_uninit; + + return 0; + +err_qm_uninit: + hisi_qm_uninit(qm); + return ret; +} + +static void hisi_zip_qm_uninit(struct hisi_qm *qm) +{ + hisi_qm_uninit(qm); +} + +static int hisi_zip_probe_init(struct hisi_zip *hisi_zip) +{ + u32 type_rate = HZIP_SHAPER_RATE_COMPRESS; + struct hisi_qm *qm = &hisi_zip->qm; + int ret; + + if (qm->fun_type == QM_HW_PF) { + ret = hisi_zip_pf_probe_init(hisi_zip); + if (ret) + return ret; + /* enable shaper type 0 */ + if (qm->ver >= QM_HW_V3) { + type_rate |= QM_SHAPER_ENABLE; + + /* ZIP need to enable shaper type 1 */ + type_rate |= HZIP_SHAPER_RATE_DECOMPRESS << QM_SHAPER_TYPE1_OFFSET; + qm->type_rate = type_rate; + } + } + + return 0; +} + +static void hisi_zip_probe_uninit(struct hisi_qm *qm) +{ + if (qm->fun_type == QM_HW_VF) + return; + + hisi_zip_show_last_regs_uninit(qm); + hisi_zip_close_sva_prefetch(qm); + hisi_qm_dev_err_uninit(qm); +} + +static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct hisi_zip *hisi_zip; + struct hisi_qm *qm; + int ret; + + hisi_zip = devm_kzalloc(&pdev->dev, sizeof(*hisi_zip), GFP_KERNEL); + if (!hisi_zip) + return -ENOMEM; + + qm = &hisi_zip->qm; + + ret = hisi_zip_qm_init(qm, pdev); + if (ret) { + pci_err(pdev, "Failed to init ZIP QM (%d)!\n", ret); + return ret; + } + + ret = hisi_zip_probe_init(hisi_zip); + if (ret) { + pci_err(pdev, "Failed to probe (%d)!\n", ret); + goto err_qm_uninit; + } + + ret = hisi_qm_start(qm); + if (ret) + goto err_probe_uninit; + + ret = hisi_zip_debugfs_init(qm); + if (ret) + pci_err(pdev, "failed to init debugfs (%d)!\n", ret); + + hisi_qm_add_list(qm, &zip_devices); + ret = hisi_qm_alg_register(qm, &zip_devices, HZIP_CTX_Q_NUM_DEF); + if (ret < 0) { + pci_err(pdev, "failed to register driver to crypto!\n"); + goto err_qm_del_list; + } + + ret = hisi_qm_register_uacce(qm); + if (ret) { + pci_err(pdev, "failed to register uacce (%d)!\n", ret); + goto err_qm_alg_unregister; + } + + if (qm->fun_type == QM_HW_PF && vfs_num > 0) { + ret = hisi_qm_sriov_enable(pdev, vfs_num); + if (ret < 0) + goto err_qm_alg_unregister; + } + + hisi_qm_pm_init(qm); + + return 0; + +err_qm_alg_unregister: + hisi_qm_alg_unregister(qm, &zip_devices, HZIP_CTX_Q_NUM_DEF); + +err_qm_del_list: + hisi_qm_del_list(qm, &zip_devices); + hisi_zip_debugfs_exit(qm); + hisi_qm_stop(qm, QM_NORMAL); + +err_probe_uninit: + hisi_zip_probe_uninit(qm); + +err_qm_uninit: + hisi_zip_qm_uninit(qm); + + return ret; +} + +static void hisi_zip_remove(struct pci_dev *pdev) +{ + struct hisi_qm *qm = pci_get_drvdata(pdev); + + hisi_qm_pm_uninit(qm); + hisi_qm_wait_task_finish(qm, &zip_devices); + hisi_qm_alg_unregister(qm, &zip_devices, HZIP_CTX_Q_NUM_DEF); + hisi_qm_del_list(qm, &zip_devices); + + if (qm->fun_type == QM_HW_PF && qm->vfs_num) + hisi_qm_sriov_disable(pdev, true); + + hisi_zip_debugfs_exit(qm); + hisi_qm_stop(qm, QM_NORMAL); + hisi_zip_probe_uninit(qm); + hisi_zip_qm_uninit(qm); +} + +static const struct dev_pm_ops hisi_zip_pm_ops = { + SET_RUNTIME_PM_OPS(hisi_qm_suspend, hisi_qm_resume, NULL) +}; + +static const struct pci_error_handlers hisi_zip_err_handler = { + .error_detected = hisi_qm_dev_err_detected, + .slot_reset = hisi_qm_dev_slot_reset, + .reset_prepare = hisi_qm_reset_prepare, + .reset_done = hisi_qm_reset_done, +}; + +static struct pci_driver hisi_zip_pci_driver = { + .name = "hisi_zip", + .id_table = hisi_zip_dev_ids, + .probe = hisi_zip_probe, + .remove = hisi_zip_remove, + .sriov_configure = IS_ENABLED(CONFIG_PCI_IOV) ? + hisi_qm_sriov_configure : NULL, + .err_handler = &hisi_zip_err_handler, + .shutdown = hisi_qm_dev_shutdown, + .driver.pm = &hisi_zip_pm_ops, +}; + +struct pci_driver *hisi_zip_get_pf_driver(void) +{ + return &hisi_zip_pci_driver; +} +EXPORT_SYMBOL_GPL(hisi_zip_get_pf_driver); + +static void hisi_zip_register_debugfs(void) +{ + if (!debugfs_initialized()) + return; + + hzip_debugfs_root = debugfs_create_dir("hisi_zip", NULL); +} + +static void hisi_zip_unregister_debugfs(void) +{ + debugfs_remove_recursive(hzip_debugfs_root); +} + +static int __init hisi_zip_init(void) +{ + int ret; + + hisi_qm_init_list(&zip_devices); + hisi_zip_register_debugfs(); + + ret = pci_register_driver(&hisi_zip_pci_driver); + if (ret < 0) { + hisi_zip_unregister_debugfs(); + pr_err("Failed to register pci driver.\n"); + } + + return ret; +} + +static void __exit hisi_zip_exit(void) +{ + pci_unregister_driver(&hisi_zip_pci_driver); + hisi_zip_unregister_debugfs(); +} + +module_init(hisi_zip_init); +module_exit(hisi_zip_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Zhou Wang "); +MODULE_DESCRIPTION("Driver for HiSilicon ZIP accelerator"); diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c new file mode 100644 index 000000000..1ccfc0ecc --- /dev/null +++ b/drivers/crypto/img-hash.c @@ -0,0 +1,1082 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2014 Imagination Technologies + * Authors: Will Thomas, James Hartley + * + * Interface structure taken from omap-sham driver + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#define CR_RESET 0 +#define CR_RESET_SET 1 +#define CR_RESET_UNSET 0 + +#define CR_MESSAGE_LENGTH_H 0x4 +#define CR_MESSAGE_LENGTH_L 0x8 + +#define CR_CONTROL 0xc +#define CR_CONTROL_BYTE_ORDER_3210 0 +#define CR_CONTROL_BYTE_ORDER_0123 1 +#define CR_CONTROL_BYTE_ORDER_2310 2 +#define CR_CONTROL_BYTE_ORDER_1032 3 +#define CR_CONTROL_BYTE_ORDER_SHIFT 8 +#define CR_CONTROL_ALGO_MD5 0 +#define CR_CONTROL_ALGO_SHA1 1 +#define CR_CONTROL_ALGO_SHA224 2 +#define CR_CONTROL_ALGO_SHA256 3 + +#define CR_INTSTAT 0x10 +#define CR_INTENAB 0x14 +#define CR_INTCLEAR 0x18 +#define CR_INT_RESULTS_AVAILABLE BIT(0) +#define CR_INT_NEW_RESULTS_SET BIT(1) +#define CR_INT_RESULT_READ_ERR BIT(2) +#define CR_INT_MESSAGE_WRITE_ERROR BIT(3) +#define CR_INT_STATUS BIT(8) + +#define CR_RESULT_QUEUE 0x1c +#define CR_RSD0 0x40 +#define CR_CORE_REV 0x50 +#define CR_CORE_DES1 0x60 +#define CR_CORE_DES2 0x70 + +#define DRIVER_FLAGS_BUSY BIT(0) +#define DRIVER_FLAGS_FINAL BIT(1) +#define DRIVER_FLAGS_DMA_ACTIVE BIT(2) +#define DRIVER_FLAGS_OUTPUT_READY BIT(3) +#define DRIVER_FLAGS_INIT BIT(4) +#define DRIVER_FLAGS_CPU BIT(5) +#define DRIVER_FLAGS_DMA_READY BIT(6) +#define DRIVER_FLAGS_ERROR BIT(7) +#define DRIVER_FLAGS_SG BIT(8) +#define DRIVER_FLAGS_SHA1 BIT(18) +#define DRIVER_FLAGS_SHA224 BIT(19) +#define DRIVER_FLAGS_SHA256 BIT(20) +#define DRIVER_FLAGS_MD5 BIT(21) + +#define IMG_HASH_QUEUE_LENGTH 20 +#define IMG_HASH_DMA_BURST 4 +#define IMG_HASH_DMA_THRESHOLD 64 + +#ifdef __LITTLE_ENDIAN +#define IMG_HASH_BYTE_ORDER CR_CONTROL_BYTE_ORDER_3210 +#else +#define IMG_HASH_BYTE_ORDER CR_CONTROL_BYTE_ORDER_0123 +#endif + +struct img_hash_dev; + +struct img_hash_request_ctx { + struct img_hash_dev *hdev; + u8 digest[SHA256_DIGEST_SIZE] __aligned(sizeof(u32)); + unsigned long flags; + size_t digsize; + + dma_addr_t dma_addr; + size_t dma_ct; + + /* sg root */ + struct scatterlist *sgfirst; + /* walk state */ + struct scatterlist *sg; + size_t nents; + size_t offset; + unsigned int total; + size_t sent; + + unsigned long op; + + size_t bufcnt; + struct ahash_request fallback_req; + + /* Zero length buffer must remain last member of struct */ + u8 buffer[] __aligned(sizeof(u32)); +}; + +struct img_hash_ctx { + struct img_hash_dev *hdev; + unsigned long flags; + struct crypto_ahash *fallback; +}; + +struct img_hash_dev { + struct list_head list; + struct device *dev; + struct clk *hash_clk; + struct clk *sys_clk; + void __iomem *io_base; + + phys_addr_t bus_addr; + void __iomem *cpu_addr; + + spinlock_t lock; + int err; + struct tasklet_struct done_task; + struct tasklet_struct dma_task; + + unsigned long flags; + struct crypto_queue queue; + struct ahash_request *req; + + struct dma_chan *dma_lch; +}; + +struct img_hash_drv { + struct list_head dev_list; + spinlock_t lock; +}; + +static struct img_hash_drv img_hash = { + .dev_list = LIST_HEAD_INIT(img_hash.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(img_hash.lock), +}; + +static inline u32 img_hash_read(struct img_hash_dev *hdev, u32 offset) +{ + return readl_relaxed(hdev->io_base + offset); +} + +static inline void img_hash_write(struct img_hash_dev *hdev, + u32 offset, u32 value) +{ + writel_relaxed(value, hdev->io_base + offset); +} + +static inline __be32 img_hash_read_result_queue(struct img_hash_dev *hdev) +{ + return cpu_to_be32(img_hash_read(hdev, CR_RESULT_QUEUE)); +} + +static void img_hash_start(struct img_hash_dev *hdev, bool dma) +{ + struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); + u32 cr = IMG_HASH_BYTE_ORDER << CR_CONTROL_BYTE_ORDER_SHIFT; + + if (ctx->flags & DRIVER_FLAGS_MD5) + cr |= CR_CONTROL_ALGO_MD5; + else if (ctx->flags & DRIVER_FLAGS_SHA1) + cr |= CR_CONTROL_ALGO_SHA1; + else if (ctx->flags & DRIVER_FLAGS_SHA224) + cr |= CR_CONTROL_ALGO_SHA224; + else if (ctx->flags & DRIVER_FLAGS_SHA256) + cr |= CR_CONTROL_ALGO_SHA256; + dev_dbg(hdev->dev, "Starting hash process\n"); + img_hash_write(hdev, CR_CONTROL, cr); + + /* + * The hardware block requires two cycles between writing the control + * register and writing the first word of data in non DMA mode, to + * ensure the first data write is not grouped in burst with the control + * register write a read is issued to 'flush' the bus. + */ + if (!dma) + img_hash_read(hdev, CR_CONTROL); +} + +static int img_hash_xmit_cpu(struct img_hash_dev *hdev, const u8 *buf, + size_t length, int final) +{ + u32 count, len32; + const u32 *buffer = (const u32 *)buf; + + dev_dbg(hdev->dev, "xmit_cpu: length: %zu bytes\n", length); + + if (final) + hdev->flags |= DRIVER_FLAGS_FINAL; + + len32 = DIV_ROUND_UP(length, sizeof(u32)); + + for (count = 0; count < len32; count++) + writel_relaxed(buffer[count], hdev->cpu_addr); + + return -EINPROGRESS; +} + +static void img_hash_dma_callback(void *data) +{ + struct img_hash_dev *hdev = data; + struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); + + if (ctx->bufcnt) { + img_hash_xmit_cpu(hdev, ctx->buffer, ctx->bufcnt, 0); + ctx->bufcnt = 0; + } + if (ctx->sg) + tasklet_schedule(&hdev->dma_task); +} + +static int img_hash_xmit_dma(struct img_hash_dev *hdev, struct scatterlist *sg) +{ + struct dma_async_tx_descriptor *desc; + struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); + + ctx->dma_ct = dma_map_sg(hdev->dev, sg, 1, DMA_TO_DEVICE); + if (ctx->dma_ct == 0) { + dev_err(hdev->dev, "Invalid DMA sg\n"); + hdev->err = -EINVAL; + return -EINVAL; + } + + desc = dmaengine_prep_slave_sg(hdev->dma_lch, + sg, + ctx->dma_ct, + DMA_MEM_TO_DEV, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!desc) { + dev_err(hdev->dev, "Null DMA descriptor\n"); + hdev->err = -EINVAL; + dma_unmap_sg(hdev->dev, sg, 1, DMA_TO_DEVICE); + return -EINVAL; + } + desc->callback = img_hash_dma_callback; + desc->callback_param = hdev; + dmaengine_submit(desc); + dma_async_issue_pending(hdev->dma_lch); + + return 0; +} + +static int img_hash_write_via_cpu(struct img_hash_dev *hdev) +{ + struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); + + ctx->bufcnt = sg_copy_to_buffer(hdev->req->src, sg_nents(ctx->sg), + ctx->buffer, hdev->req->nbytes); + + ctx->total = hdev->req->nbytes; + ctx->bufcnt = 0; + + hdev->flags |= (DRIVER_FLAGS_CPU | DRIVER_FLAGS_FINAL); + + img_hash_start(hdev, false); + + return img_hash_xmit_cpu(hdev, ctx->buffer, ctx->total, 1); +} + +static int img_hash_finish(struct ahash_request *req) +{ + struct img_hash_request_ctx *ctx = ahash_request_ctx(req); + + if (!req->result) + return -EINVAL; + + memcpy(req->result, ctx->digest, ctx->digsize); + + return 0; +} + +static void img_hash_copy_hash(struct ahash_request *req) +{ + struct img_hash_request_ctx *ctx = ahash_request_ctx(req); + __be32 *hash = (__be32 *)ctx->digest; + int i; + + for (i = (ctx->digsize / sizeof(*hash)) - 1; i >= 0; i--) + hash[i] = img_hash_read_result_queue(ctx->hdev); +} + +static void img_hash_finish_req(struct ahash_request *req, int err) +{ + struct img_hash_request_ctx *ctx = ahash_request_ctx(req); + struct img_hash_dev *hdev = ctx->hdev; + + if (!err) { + img_hash_copy_hash(req); + if (DRIVER_FLAGS_FINAL & hdev->flags) + err = img_hash_finish(req); + } else { + dev_warn(hdev->dev, "Hash failed with error %d\n", err); + ctx->flags |= DRIVER_FLAGS_ERROR; + } + + hdev->flags &= ~(DRIVER_FLAGS_DMA_READY | DRIVER_FLAGS_OUTPUT_READY | + DRIVER_FLAGS_CPU | DRIVER_FLAGS_BUSY | DRIVER_FLAGS_FINAL); + + if (req->base.complete) + ahash_request_complete(req, err); +} + +static int img_hash_write_via_dma(struct img_hash_dev *hdev) +{ + struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); + + img_hash_start(hdev, true); + + dev_dbg(hdev->dev, "xmit dma size: %d\n", ctx->total); + + if (!ctx->total) + hdev->flags |= DRIVER_FLAGS_FINAL; + + hdev->flags |= DRIVER_FLAGS_DMA_ACTIVE | DRIVER_FLAGS_FINAL; + + tasklet_schedule(&hdev->dma_task); + + return -EINPROGRESS; +} + +static int img_hash_dma_init(struct img_hash_dev *hdev) +{ + struct dma_slave_config dma_conf; + int err; + + hdev->dma_lch = dma_request_chan(hdev->dev, "tx"); + if (IS_ERR(hdev->dma_lch)) { + dev_err(hdev->dev, "Couldn't acquire a slave DMA channel.\n"); + return PTR_ERR(hdev->dma_lch); + } + dma_conf.direction = DMA_MEM_TO_DEV; + dma_conf.dst_addr = hdev->bus_addr; + dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + dma_conf.dst_maxburst = IMG_HASH_DMA_BURST; + dma_conf.device_fc = false; + + err = dmaengine_slave_config(hdev->dma_lch, &dma_conf); + if (err) { + dev_err(hdev->dev, "Couldn't configure DMA slave.\n"); + dma_release_channel(hdev->dma_lch); + return err; + } + + return 0; +} + +static void img_hash_dma_task(unsigned long d) +{ + struct img_hash_dev *hdev = (struct img_hash_dev *)d; + struct img_hash_request_ctx *ctx; + u8 *addr; + size_t nbytes, bleft, wsend, len, tbc; + struct scatterlist tsg; + + if (!hdev->req) + return; + + ctx = ahash_request_ctx(hdev->req); + if (!ctx->sg) + return; + + addr = sg_virt(ctx->sg); + nbytes = ctx->sg->length - ctx->offset; + + /* + * The hash accelerator does not support a data valid mask. This means + * that if each dma (i.e. per page) is not a multiple of 4 bytes, the + * padding bytes in the last word written by that dma would erroneously + * be included in the hash. To avoid this we round down the transfer, + * and add the excess to the start of the next dma. It does not matter + * that the final dma may not be a multiple of 4 bytes as the hashing + * block is programmed to accept the correct number of bytes. + */ + + bleft = nbytes % 4; + wsend = (nbytes / 4); + + if (wsend) { + sg_init_one(&tsg, addr + ctx->offset, wsend * 4); + if (img_hash_xmit_dma(hdev, &tsg)) { + dev_err(hdev->dev, "DMA failed, falling back to CPU"); + ctx->flags |= DRIVER_FLAGS_CPU; + hdev->err = 0; + img_hash_xmit_cpu(hdev, addr + ctx->offset, + wsend * 4, 0); + ctx->sent += wsend * 4; + wsend = 0; + } else { + ctx->sent += wsend * 4; + } + } + + if (bleft) { + ctx->bufcnt = sg_pcopy_to_buffer(ctx->sgfirst, ctx->nents, + ctx->buffer, bleft, ctx->sent); + tbc = 0; + ctx->sg = sg_next(ctx->sg); + while (ctx->sg && (ctx->bufcnt < 4)) { + len = ctx->sg->length; + if (likely(len > (4 - ctx->bufcnt))) + len = 4 - ctx->bufcnt; + tbc = sg_pcopy_to_buffer(ctx->sgfirst, ctx->nents, + ctx->buffer + ctx->bufcnt, len, + ctx->sent + ctx->bufcnt); + ctx->bufcnt += tbc; + if (tbc >= ctx->sg->length) { + ctx->sg = sg_next(ctx->sg); + tbc = 0; + } + } + + ctx->sent += ctx->bufcnt; + ctx->offset = tbc; + + if (!wsend) + img_hash_dma_callback(hdev); + } else { + ctx->offset = 0; + ctx->sg = sg_next(ctx->sg); + } +} + +static int img_hash_write_via_dma_stop(struct img_hash_dev *hdev) +{ + struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); + + if (ctx->flags & DRIVER_FLAGS_SG) + dma_unmap_sg(hdev->dev, ctx->sg, 1, DMA_TO_DEVICE); + + return 0; +} + +static int img_hash_process_data(struct img_hash_dev *hdev) +{ + struct ahash_request *req = hdev->req; + struct img_hash_request_ctx *ctx = ahash_request_ctx(req); + int err = 0; + + ctx->bufcnt = 0; + + if (req->nbytes >= IMG_HASH_DMA_THRESHOLD) { + dev_dbg(hdev->dev, "process data request(%d bytes) using DMA\n", + req->nbytes); + err = img_hash_write_via_dma(hdev); + } else { + dev_dbg(hdev->dev, "process data request(%d bytes) using CPU\n", + req->nbytes); + err = img_hash_write_via_cpu(hdev); + } + return err; +} + +static int img_hash_hw_init(struct img_hash_dev *hdev) +{ + unsigned long long nbits; + u32 u, l; + + img_hash_write(hdev, CR_RESET, CR_RESET_SET); + img_hash_write(hdev, CR_RESET, CR_RESET_UNSET); + img_hash_write(hdev, CR_INTENAB, CR_INT_NEW_RESULTS_SET); + + nbits = (u64)hdev->req->nbytes << 3; + u = nbits >> 32; + l = nbits; + img_hash_write(hdev, CR_MESSAGE_LENGTH_H, u); + img_hash_write(hdev, CR_MESSAGE_LENGTH_L, l); + + if (!(DRIVER_FLAGS_INIT & hdev->flags)) { + hdev->flags |= DRIVER_FLAGS_INIT; + hdev->err = 0; + } + dev_dbg(hdev->dev, "hw initialized, nbits: %llx\n", nbits); + return 0; +} + +static int img_hash_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct img_hash_request_ctx *rctx = ahash_request_ctx(req); + struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + + return crypto_ahash_init(&rctx->fallback_req); +} + +static int img_hash_handle_queue(struct img_hash_dev *hdev, + struct ahash_request *req) +{ + struct crypto_async_request *async_req, *backlog; + struct img_hash_request_ctx *ctx; + unsigned long flags; + int err = 0, res = 0; + + spin_lock_irqsave(&hdev->lock, flags); + + if (req) + res = ahash_enqueue_request(&hdev->queue, req); + + if (DRIVER_FLAGS_BUSY & hdev->flags) { + spin_unlock_irqrestore(&hdev->lock, flags); + return res; + } + + backlog = crypto_get_backlog(&hdev->queue); + async_req = crypto_dequeue_request(&hdev->queue); + if (async_req) + hdev->flags |= DRIVER_FLAGS_BUSY; + + spin_unlock_irqrestore(&hdev->lock, flags); + + if (!async_req) + return res; + + if (backlog) + crypto_request_complete(backlog, -EINPROGRESS); + + req = ahash_request_cast(async_req); + hdev->req = req; + + ctx = ahash_request_ctx(req); + + dev_info(hdev->dev, "processing req, op: %lu, bytes: %d\n", + ctx->op, req->nbytes); + + err = img_hash_hw_init(hdev); + + if (!err) + err = img_hash_process_data(hdev); + + if (err != -EINPROGRESS) { + /* done_task will not finish so do it here */ + img_hash_finish_req(req, err); + } + return res; +} + +static int img_hash_update(struct ahash_request *req) +{ + struct img_hash_request_ctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, NULL, req->nbytes); + + return crypto_ahash_update(&rctx->fallback_req); +} + +static int img_hash_final(struct ahash_request *req) +{ + struct img_hash_request_ctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, NULL, req->result, 0); + + return crypto_ahash_final(&rctx->fallback_req); +} + +static int img_hash_finup(struct ahash_request *req) +{ + struct img_hash_request_ctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result, + req->nbytes); + + + return crypto_ahash_finup(&rctx->fallback_req); +} + +static int img_hash_import(struct ahash_request *req, const void *in) +{ + struct img_hash_request_ctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + + return crypto_ahash_import(&rctx->fallback_req, in); +} + +static int img_hash_export(struct ahash_request *req, void *out) +{ + struct img_hash_request_ctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + + return crypto_ahash_export(&rctx->fallback_req, out); +} + +static int img_hash_digest(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct img_hash_ctx *tctx = crypto_ahash_ctx(tfm); + struct img_hash_request_ctx *ctx = ahash_request_ctx(req); + + spin_lock(&img_hash.lock); + if (!tctx->hdev) + tctx->hdev = list_first_entry_or_null(&img_hash.dev_list, + struct img_hash_dev, list); + ctx->hdev = tctx->hdev; + spin_unlock(&img_hash.lock); + + ctx->flags = 0; + ctx->digsize = crypto_ahash_digestsize(tfm); + + switch (ctx->digsize) { + case SHA1_DIGEST_SIZE: + ctx->flags |= DRIVER_FLAGS_SHA1; + break; + case SHA256_DIGEST_SIZE: + ctx->flags |= DRIVER_FLAGS_SHA256; + break; + case SHA224_DIGEST_SIZE: + ctx->flags |= DRIVER_FLAGS_SHA224; + break; + case MD5_DIGEST_SIZE: + ctx->flags |= DRIVER_FLAGS_MD5; + break; + default: + return -EINVAL; + } + + ctx->bufcnt = 0; + ctx->offset = 0; + ctx->sent = 0; + ctx->total = req->nbytes; + ctx->sg = req->src; + ctx->sgfirst = req->src; + ctx->nents = sg_nents(ctx->sg); + + return img_hash_handle_queue(ctx->hdev, req); +} + +static int img_hash_cra_init(struct crypto_tfm *tfm, const char *alg_name) +{ + struct img_hash_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->fallback = crypto_alloc_ahash(alg_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fallback)) { + pr_err("img_hash: Could not load fallback driver.\n"); + return PTR_ERR(ctx->fallback); + } + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct img_hash_request_ctx) + + crypto_ahash_reqsize(ctx->fallback) + + IMG_HASH_DMA_THRESHOLD); + + return 0; +} + +static int img_hash_cra_md5_init(struct crypto_tfm *tfm) +{ + return img_hash_cra_init(tfm, "md5-lib"); +} + +static int img_hash_cra_sha1_init(struct crypto_tfm *tfm) +{ + return img_hash_cra_init(tfm, "sha1-lib"); +} + +static int img_hash_cra_sha224_init(struct crypto_tfm *tfm) +{ + return img_hash_cra_init(tfm, "sha224-lib"); +} + +static int img_hash_cra_sha256_init(struct crypto_tfm *tfm) +{ + return img_hash_cra_init(tfm, "sha256-lib"); +} + +static void img_hash_cra_exit(struct crypto_tfm *tfm) +{ + struct img_hash_ctx *tctx = crypto_tfm_ctx(tfm); + + crypto_free_ahash(tctx->fallback); +} + +static irqreturn_t img_irq_handler(int irq, void *dev_id) +{ + struct img_hash_dev *hdev = dev_id; + u32 reg; + + reg = img_hash_read(hdev, CR_INTSTAT); + img_hash_write(hdev, CR_INTCLEAR, reg); + + if (reg & CR_INT_NEW_RESULTS_SET) { + dev_dbg(hdev->dev, "IRQ CR_INT_NEW_RESULTS_SET\n"); + if (DRIVER_FLAGS_BUSY & hdev->flags) { + hdev->flags |= DRIVER_FLAGS_OUTPUT_READY; + if (!(DRIVER_FLAGS_CPU & hdev->flags)) + hdev->flags |= DRIVER_FLAGS_DMA_READY; + tasklet_schedule(&hdev->done_task); + } else { + dev_warn(hdev->dev, + "HASH interrupt when no active requests.\n"); + } + } else if (reg & CR_INT_RESULTS_AVAILABLE) { + dev_warn(hdev->dev, + "IRQ triggered before the hash had completed\n"); + } else if (reg & CR_INT_RESULT_READ_ERR) { + dev_warn(hdev->dev, + "Attempt to read from an empty result queue\n"); + } else if (reg & CR_INT_MESSAGE_WRITE_ERROR) { + dev_warn(hdev->dev, + "Data written before the hardware was configured\n"); + } + return IRQ_HANDLED; +} + +static struct ahash_alg img_algs[] = { + { + .init = img_hash_init, + .update = img_hash_update, + .final = img_hash_final, + .finup = img_hash_finup, + .export = img_hash_export, + .import = img_hash_import, + .digest = img_hash_digest, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct md5_state), + .base = { + .cra_name = "md5", + .cra_driver_name = "img-md5", + .cra_priority = 300, + .cra_flags = + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct img_hash_ctx), + .cra_init = img_hash_cra_md5_init, + .cra_exit = img_hash_cra_exit, + .cra_module = THIS_MODULE, + } + } + }, + { + .init = img_hash_init, + .update = img_hash_update, + .final = img_hash_final, + .finup = img_hash_finup, + .export = img_hash_export, + .import = img_hash_import, + .digest = img_hash_digest, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct sha1_state), + .base = { + .cra_name = "sha1", + .cra_driver_name = "img-sha1", + .cra_priority = 300, + .cra_flags = + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct img_hash_ctx), + .cra_init = img_hash_cra_sha1_init, + .cra_exit = img_hash_cra_exit, + .cra_module = THIS_MODULE, + } + } + }, + { + .init = img_hash_init, + .update = img_hash_update, + .final = img_hash_final, + .finup = img_hash_finup, + .export = img_hash_export, + .import = img_hash_import, + .digest = img_hash_digest, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha224", + .cra_driver_name = "img-sha224", + .cra_priority = 300, + .cra_flags = + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct img_hash_ctx), + .cra_init = img_hash_cra_sha224_init, + .cra_exit = img_hash_cra_exit, + .cra_module = THIS_MODULE, + } + } + }, + { + .init = img_hash_init, + .update = img_hash_update, + .final = img_hash_final, + .finup = img_hash_finup, + .export = img_hash_export, + .import = img_hash_import, + .digest = img_hash_digest, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha256", + .cra_driver_name = "img-sha256", + .cra_priority = 300, + .cra_flags = + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct img_hash_ctx), + .cra_init = img_hash_cra_sha256_init, + .cra_exit = img_hash_cra_exit, + .cra_module = THIS_MODULE, + } + } + } +}; + +static int img_register_algs(struct img_hash_dev *hdev) +{ + int i, err; + + for (i = 0; i < ARRAY_SIZE(img_algs); i++) { + err = crypto_register_ahash(&img_algs[i]); + if (err) { + crypto_unregister_ahashes(img_algs, i); + return err; + } + } + + return 0; +} + +static void img_unregister_algs(struct img_hash_dev *hdev) +{ + crypto_unregister_ahashes(img_algs, ARRAY_SIZE(img_algs)); +} + +static void img_hash_done_task(unsigned long data) +{ + struct img_hash_dev *hdev = (struct img_hash_dev *)data; + int err = 0; + + if (hdev->err == -EINVAL) { + err = hdev->err; + goto finish; + } + + if (!(DRIVER_FLAGS_BUSY & hdev->flags)) { + img_hash_handle_queue(hdev, NULL); + return; + } + + if (DRIVER_FLAGS_CPU & hdev->flags) { + if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) { + hdev->flags &= ~DRIVER_FLAGS_OUTPUT_READY; + goto finish; + } + } else if (DRIVER_FLAGS_DMA_READY & hdev->flags) { + if (DRIVER_FLAGS_DMA_ACTIVE & hdev->flags) { + hdev->flags &= ~DRIVER_FLAGS_DMA_ACTIVE; + img_hash_write_via_dma_stop(hdev); + if (hdev->err) { + err = hdev->err; + goto finish; + } + } + if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) { + hdev->flags &= ~(DRIVER_FLAGS_DMA_READY | + DRIVER_FLAGS_OUTPUT_READY); + goto finish; + } + } + return; + +finish: + img_hash_finish_req(hdev->req, err); +} + +static const struct of_device_id img_hash_match[] __maybe_unused = { + { .compatible = "img,hash-accelerator" }, + {} +}; +MODULE_DEVICE_TABLE(of, img_hash_match); + +static int img_hash_probe(struct platform_device *pdev) +{ + struct img_hash_dev *hdev; + struct device *dev = &pdev->dev; + struct resource *hash_res; + int irq; + int err; + + hdev = devm_kzalloc(dev, sizeof(*hdev), GFP_KERNEL); + if (hdev == NULL) + return -ENOMEM; + + spin_lock_init(&hdev->lock); + + hdev->dev = dev; + + platform_set_drvdata(pdev, hdev); + + INIT_LIST_HEAD(&hdev->list); + + tasklet_init(&hdev->done_task, img_hash_done_task, (unsigned long)hdev); + tasklet_init(&hdev->dma_task, img_hash_dma_task, (unsigned long)hdev); + + crypto_init_queue(&hdev->queue, IMG_HASH_QUEUE_LENGTH); + + /* Register bank */ + hdev->io_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(hdev->io_base)) { + err = PTR_ERR(hdev->io_base); + goto res_err; + } + + /* Write port (DMA or CPU) */ + hdev->cpu_addr = devm_platform_get_and_ioremap_resource(pdev, 1, &hash_res); + if (IS_ERR(hdev->cpu_addr)) { + err = PTR_ERR(hdev->cpu_addr); + goto res_err; + } + hdev->bus_addr = hash_res->start; + + irq = platform_get_irq(pdev, 0); + if (irq < 0) { + err = irq; + goto res_err; + } + + err = devm_request_irq(dev, irq, img_irq_handler, 0, + dev_name(dev), hdev); + if (err) + goto res_err; + dev_dbg(dev, "using IRQ channel %d\n", irq); + + hdev->hash_clk = devm_clk_get_enabled(&pdev->dev, "hash"); + if (IS_ERR(hdev->hash_clk)) { + dev_err(dev, "clock initialization failed.\n"); + err = PTR_ERR(hdev->hash_clk); + goto res_err; + } + + hdev->sys_clk = devm_clk_get_enabled(&pdev->dev, "sys"); + if (IS_ERR(hdev->sys_clk)) { + dev_err(dev, "clock initialization failed.\n"); + err = PTR_ERR(hdev->sys_clk); + goto res_err; + } + + err = img_hash_dma_init(hdev); + if (err) + goto res_err; + + dev_dbg(dev, "using %s for DMA transfers\n", + dma_chan_name(hdev->dma_lch)); + + spin_lock(&img_hash.lock); + list_add_tail(&hdev->list, &img_hash.dev_list); + spin_unlock(&img_hash.lock); + + err = img_register_algs(hdev); + if (err) + goto err_algs; + dev_info(dev, "Img MD5/SHA1/SHA224/SHA256 Hardware accelerator initialized\n"); + + return 0; + +err_algs: + spin_lock(&img_hash.lock); + list_del(&hdev->list); + spin_unlock(&img_hash.lock); + dma_release_channel(hdev->dma_lch); +res_err: + tasklet_kill(&hdev->done_task); + tasklet_kill(&hdev->dma_task); + + return err; +} + +static void img_hash_remove(struct platform_device *pdev) +{ + struct img_hash_dev *hdev; + + hdev = platform_get_drvdata(pdev); + spin_lock(&img_hash.lock); + list_del(&hdev->list); + spin_unlock(&img_hash.lock); + + img_unregister_algs(hdev); + + tasklet_kill(&hdev->done_task); + tasklet_kill(&hdev->dma_task); + + dma_release_channel(hdev->dma_lch); +} + +#ifdef CONFIG_PM_SLEEP +static int img_hash_suspend(struct device *dev) +{ + struct img_hash_dev *hdev = dev_get_drvdata(dev); + + clk_disable_unprepare(hdev->hash_clk); + clk_disable_unprepare(hdev->sys_clk); + + return 0; +} + +static int img_hash_resume(struct device *dev) +{ + struct img_hash_dev *hdev = dev_get_drvdata(dev); + int ret; + + ret = clk_prepare_enable(hdev->hash_clk); + if (ret) + return ret; + + ret = clk_prepare_enable(hdev->sys_clk); + if (ret) { + clk_disable_unprepare(hdev->hash_clk); + return ret; + } + + return 0; +} +#endif /* CONFIG_PM_SLEEP */ + +static const struct dev_pm_ops img_hash_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(img_hash_suspend, img_hash_resume) +}; + +static struct platform_driver img_hash_driver = { + .probe = img_hash_probe, + .remove = img_hash_remove, + .driver = { + .name = "img-hash-accelerator", + .pm = &img_hash_pm_ops, + .of_match_table = img_hash_match, + } +}; +module_platform_driver(img_hash_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Imgtec SHA1/224/256 & MD5 hw accelerator driver"); +MODULE_AUTHOR("Will Thomas."); +MODULE_AUTHOR("James Hartley "); diff --git a/drivers/crypto/inside-secure/Makefile b/drivers/crypto/inside-secure/Makefile new file mode 100644 index 000000000..30d13fd5d --- /dev/null +++ b/drivers/crypto/inside-secure/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_SAFEXCEL) += crypto_safexcel.o +crypto_safexcel-objs := safexcel.o safexcel_ring.o safexcel_cipher.o safexcel_hash.o +obj-y += eip93/ diff --git a/drivers/crypto/inside-secure/eip93/Kconfig b/drivers/crypto/inside-secure/eip93/Kconfig new file mode 100644 index 000000000..29523f692 --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/Kconfig @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: GPL-2.0 +config CRYPTO_DEV_EIP93 + tristate "Support for EIP93 crypto HW accelerators" + depends on SOC_MT7621 || ARCH_AIROHA || ECONET || COMPILE_TEST + select CRYPTO_LIB_AES + select CRYPTO_LIB_DES + select CRYPTO_SKCIPHER + select CRYPTO_AEAD + select CRYPTO_AUTHENC + select CRYPTO_MD5 + select CRYPTO_SHA1 + select CRYPTO_SHA256 + help + EIP93 have various crypto HW accelerators. Select this if + you want to use the EIP93 modules for any of the crypto algorithms. + + If the IP supports it, this provide offload for AES - ECB, CBC and + CTR crypto. Also provide DES and 3DES ECB and CBC. + + Also provide AEAD authenc(hmac(x), cipher(y)) for supported algo. diff --git a/drivers/crypto/inside-secure/eip93/Makefile b/drivers/crypto/inside-secure/eip93/Makefile new file mode 100644 index 000000000..a3d3d3677 --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/Makefile @@ -0,0 +1,5 @@ +obj-$(CONFIG_CRYPTO_DEV_EIP93) += crypto-hw-eip93.o + +crypto-hw-eip93-y += eip93-main.o eip93-common.o +crypto-hw-eip93-y += eip93-cipher.o eip93-aead.o +crypto-hw-eip93-y += eip93-hash.o diff --git a/drivers/crypto/inside-secure/eip93/eip93-aead.c b/drivers/crypto/inside-secure/eip93/eip93-aead.c new file mode 100644 index 000000000..2bbd0af7b --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-aead.c @@ -0,0 +1,711 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2019 - 2021 + * + * Richard van Schagen + * Christian Marangi + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include "eip93-aead.h" +#include "eip93-cipher.h" +#include "eip93-common.h" +#include "eip93-regs.h" + +void eip93_aead_handle_result(struct crypto_async_request *async, int err) +{ + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(async->tfm); + struct eip93_device *eip93 = ctx->eip93; + struct aead_request *req = aead_request_cast(async); + struct eip93_cipher_reqctx *rctx = aead_request_ctx(req); + + eip93_unmap_dma(eip93, rctx, req->src, req->dst); + eip93_handle_result(eip93, rctx, req->iv); + + aead_request_complete(req, err); +} + +static int eip93_aead_send_req(struct crypto_async_request *async) +{ + struct aead_request *req = aead_request_cast(async); + struct eip93_cipher_reqctx *rctx = aead_request_ctx(req); + int err; + + err = check_valid_request(rctx); + if (err) { + aead_request_complete(req, err); + return err; + } + + return eip93_send_req(async, req->iv, rctx); +} + +/* Crypto aead API functions */ +static int eip93_aead_cra_init(struct crypto_tfm *tfm) +{ + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + struct eip93_alg_template *tmpl = container_of(tfm->__crt_alg, + struct eip93_alg_template, alg.aead.base); + + crypto_aead_set_reqsize(__crypto_aead_cast(tfm), + sizeof(struct eip93_cipher_reqctx)); + + ctx->eip93 = tmpl->eip93; + ctx->flags = tmpl->flags; + ctx->type = tmpl->type; + ctx->set_assoc = true; + + ctx->sa_record = kzalloc_obj(*ctx->sa_record); + if (!ctx->sa_record) + return -ENOMEM; + + return 0; +} + +static void eip93_aead_cra_exit(struct crypto_tfm *tfm) +{ + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + + dma_unmap_single(ctx->eip93->dev, ctx->sa_record_base, + sizeof(*ctx->sa_record), DMA_TO_DEVICE); + kfree(ctx->sa_record); +} + +static int eip93_aead_setkey(struct crypto_aead *ctfm, const u8 *key, + unsigned int len) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_authenc_keys keys; + struct crypto_aes_ctx aes; + struct sa_record *sa_record = ctx->sa_record; + u32 nonce = 0; + int ret; + + if (crypto_authenc_extractkeys(&keys, key, len)) + return -EINVAL; + + if (IS_RFC3686(ctx->flags)) { + if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE) + return -EINVAL; + + keys.enckeylen -= CTR_RFC3686_NONCE_SIZE; + memcpy(&nonce, keys.enckey + keys.enckeylen, + CTR_RFC3686_NONCE_SIZE); + } + + switch ((ctx->flags & EIP93_ALG_MASK)) { + case EIP93_ALG_DES: + ret = verify_aead_des_key(ctfm, keys.enckey, keys.enckeylen); + if (ret) + return ret; + + break; + case EIP93_ALG_3DES: + if (keys.enckeylen != DES3_EDE_KEY_SIZE) + return -EINVAL; + + ret = verify_aead_des3_key(ctfm, keys.enckey, keys.enckeylen); + if (ret) + return ret; + + break; + case EIP93_ALG_AES: + ret = aes_expandkey(&aes, keys.enckey, keys.enckeylen); + if (ret) + return ret; + + break; + } + + ctx->blksize = crypto_aead_blocksize(ctfm); + /* Encryption key */ + eip93_set_sa_record(sa_record, keys.enckeylen, ctx->flags); + sa_record->sa_cmd0_word &= ~EIP93_SA_CMD_OPCODE; + sa_record->sa_cmd0_word |= FIELD_PREP(EIP93_SA_CMD_OPCODE, + EIP93_SA_CMD_OPCODE_BASIC_OUT_ENC_HASH); + sa_record->sa_cmd0_word &= ~EIP93_SA_CMD_DIGEST_LENGTH; + sa_record->sa_cmd0_word |= FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, + ctx->authsize / sizeof(u32)); + + memcpy(sa_record->sa_key, keys.enckey, keys.enckeylen); + ctx->sa_nonce = nonce; + sa_record->sa_nonce = nonce; + + /* authentication key */ + ret = eip93_hmac_setkey(ctx->flags, keys.authkey, keys.authkeylen, + ctx->authsize, sa_record->sa_i_digest, + sa_record->sa_o_digest, false); + + ctx->set_assoc = true; + + return ret; +} + +static int eip93_aead_setauthsize(struct crypto_aead *ctfm, + unsigned int authsize) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->authsize = authsize; + ctx->sa_record->sa_cmd0_word &= ~EIP93_SA_CMD_DIGEST_LENGTH; + ctx->sa_record->sa_cmd0_word |= FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, + ctx->authsize / sizeof(u32)); + + return 0; +} + +static void eip93_aead_setassoc(struct eip93_crypto_ctx *ctx, + struct aead_request *req) +{ + struct sa_record *sa_record = ctx->sa_record; + + sa_record->sa_cmd1_word &= ~EIP93_SA_CMD_HASH_CRYPT_OFFSET; + sa_record->sa_cmd1_word |= FIELD_PREP(EIP93_SA_CMD_HASH_CRYPT_OFFSET, + req->assoclen / sizeof(u32)); + + ctx->assoclen = req->assoclen; +} + +static int eip93_aead_crypt(struct aead_request *req) +{ + struct eip93_cipher_reqctx *rctx = aead_request_ctx(req); + struct crypto_async_request *async = &req->base; + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + int ret; + + ctx->sa_record_base = dma_map_single(ctx->eip93->dev, ctx->sa_record, + sizeof(*ctx->sa_record), DMA_TO_DEVICE); + ret = dma_mapping_error(ctx->eip93->dev, ctx->sa_record_base); + if (ret) + return ret; + + rctx->textsize = req->cryptlen; + rctx->blksize = ctx->blksize; + rctx->assoclen = req->assoclen; + rctx->authsize = ctx->authsize; + rctx->sg_src = req->src; + rctx->sg_dst = req->dst; + rctx->ivsize = crypto_aead_ivsize(aead); + rctx->desc_flags = EIP93_DESC_AEAD; + rctx->sa_record_base = ctx->sa_record_base; + + if (IS_DECRYPT(rctx->flags)) + rctx->textsize -= rctx->authsize; + + return eip93_aead_send_req(async); +} + +static int eip93_aead_encrypt(struct aead_request *req) +{ + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct eip93_cipher_reqctx *rctx = aead_request_ctx(req); + + rctx->flags = ctx->flags; + rctx->flags |= EIP93_ENCRYPT; + if (ctx->set_assoc) { + eip93_aead_setassoc(ctx, req); + ctx->set_assoc = false; + } + + if (req->assoclen != ctx->assoclen) { + dev_err(ctx->eip93->dev, "Request AAD length error\n"); + return -EINVAL; + } + + return eip93_aead_crypt(req); +} + +static int eip93_aead_decrypt(struct aead_request *req) +{ + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct eip93_cipher_reqctx *rctx = aead_request_ctx(req); + + ctx->sa_record->sa_cmd0_word |= EIP93_SA_CMD_DIRECTION_IN; + ctx->sa_record->sa_cmd1_word &= ~(EIP93_SA_CMD_COPY_PAD | + EIP93_SA_CMD_COPY_DIGEST); + + rctx->flags = ctx->flags; + rctx->flags |= EIP93_DECRYPT; + if (ctx->set_assoc) { + eip93_aead_setassoc(ctx, req); + ctx->set_assoc = false; + } + + if (req->assoclen != ctx->assoclen) { + dev_err(ctx->eip93->dev, "Request AAD length error\n"); + return -EINVAL; + } + + return eip93_aead_crypt(req); +} + +/* Available authenc algorithms in this module */ +struct eip93_alg_template eip93_alg_authenc_hmac_md5_cbc_aes = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_MD5 | EIP93_MODE_CBC | EIP93_ALG_AES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = MD5_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(md5),cbc(aes))", + .cra_driver_name = + "authenc(hmac(md5-eip93), cbc(aes-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_sha1_cbc_aes = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA1 | EIP93_MODE_CBC | EIP93_ALG_AES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = + "authenc(hmac(sha1-eip93),cbc(aes-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_sha224_cbc_aes = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA224 | EIP93_MODE_CBC | EIP93_ALG_AES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = SHA224_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha224),cbc(aes))", + .cra_driver_name = + "authenc(hmac(sha224-eip93),cbc(aes-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_sha256_cbc_aes = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA256 | EIP93_MODE_CBC | EIP93_ALG_AES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = SHA256_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = + "authenc(hmac(sha256-eip93),cbc(aes-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_md5_rfc3686_aes = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_MD5 | + EIP93_MODE_CTR | EIP93_MODE_RFC3686 | EIP93_ALG_AES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = MD5_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(md5),rfc3686(ctr(aes)))", + .cra_driver_name = + "authenc(hmac(md5-eip93),rfc3686(ctr(aes-eip93)))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_sha1_rfc3686_aes = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA1 | + EIP93_MODE_CTR | EIP93_MODE_RFC3686 | EIP93_ALG_AES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", + .cra_driver_name = + "authenc(hmac(sha1-eip93),rfc3686(ctr(aes-eip93)))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_sha224_rfc3686_aes = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA224 | + EIP93_MODE_CTR | EIP93_MODE_RFC3686 | EIP93_ALG_AES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = SHA224_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))", + .cra_driver_name = + "authenc(hmac(sha224-eip93),rfc3686(ctr(aes-eip93)))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_sha256_rfc3686_aes = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA256 | + EIP93_MODE_CTR | EIP93_MODE_RFC3686 | EIP93_ALG_AES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = SHA256_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", + .cra_driver_name = + "authenc(hmac(sha256-eip93),rfc3686(ctr(aes-eip93)))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_md5_cbc_des = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_MD5 | EIP93_MODE_CBC | EIP93_ALG_DES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = MD5_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(md5),cbc(des))", + .cra_driver_name = + "authenc(hmac(md5-eip93),cbc(des-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_sha1_cbc_des = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA1 | EIP93_MODE_CBC | EIP93_ALG_DES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des))", + .cra_driver_name = + "authenc(hmac(sha1-eip93),cbc(des-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_sha224_cbc_des = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA224 | EIP93_MODE_CBC | EIP93_ALG_DES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = SHA224_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha224),cbc(des))", + .cra_driver_name = + "authenc(hmac(sha224-eip93),cbc(des-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_sha256_cbc_des = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA256 | EIP93_MODE_CBC | EIP93_ALG_DES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = SHA256_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha256),cbc(des))", + .cra_driver_name = + "authenc(hmac(sha256-eip93),cbc(des-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_md5_cbc_des3_ede = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_MD5 | EIP93_MODE_CBC | EIP93_ALG_3DES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = MD5_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(md5),cbc(des3_ede))", + .cra_driver_name = + "authenc(hmac(md5-eip93),cbc(des3_ede-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0x0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_sha1_cbc_des3_ede = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA1 | EIP93_MODE_CBC | EIP93_ALG_3DES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", + .cra_driver_name = + "authenc(hmac(sha1-eip93),cbc(des3_ede-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0x0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_sha224_cbc_des3_ede = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA224 | EIP93_MODE_CBC | EIP93_ALG_3DES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = SHA224_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha224),cbc(des3_ede))", + .cra_driver_name = + "authenc(hmac(sha224-eip93),cbc(des3_ede-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0x0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_authenc_hmac_sha256_cbc_des3_ede = { + .type = EIP93_ALG_TYPE_AEAD, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA256 | EIP93_MODE_CBC | EIP93_ALG_3DES, + .alg.aead = { + .setkey = eip93_aead_setkey, + .encrypt = eip93_aead_encrypt, + .decrypt = eip93_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setauthsize = eip93_aead_setauthsize, + .maxauthsize = SHA256_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha256),cbc(des3_ede))", + .cra_driver_name = + "authenc(hmac(sha256-eip93),cbc(des3_ede-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0x0, + .cra_init = eip93_aead_cra_init, + .cra_exit = eip93_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; diff --git a/drivers/crypto/inside-secure/eip93/eip93-aead.h b/drivers/crypto/inside-secure/eip93/eip93-aead.h new file mode 100644 index 000000000..d933a8fbd --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-aead.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2019 - 2021 + * + * Richard van Schagen + * Christian Marangi + */ +#ifndef _EIP93_AEAD_H_ +#define _EIP93_AEAD_H_ + +extern struct eip93_alg_template eip93_alg_authenc_hmac_md5_cbc_aes; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha1_cbc_aes; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha224_cbc_aes; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha256_cbc_aes; +extern struct eip93_alg_template eip93_alg_authenc_hmac_md5_ctr_aes; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha1_ctr_aes; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha224_ctr_aes; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha256_ctr_aes; +extern struct eip93_alg_template eip93_alg_authenc_hmac_md5_rfc3686_aes; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha1_rfc3686_aes; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha224_rfc3686_aes; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha256_rfc3686_aes; +extern struct eip93_alg_template eip93_alg_authenc_hmac_md5_cbc_des; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha1_cbc_des; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha224_cbc_des; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha256_cbc_des; +extern struct eip93_alg_template eip93_alg_authenc_hmac_md5_cbc_des3_ede; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha1_cbc_des3_ede; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha224_cbc_des3_ede; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha256_cbc_des3_ede; +extern struct eip93_alg_template eip93_alg_authenc_hmac_md5_ecb_null; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha1_ecb_null; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha224_ecb_null; +extern struct eip93_alg_template eip93_alg_authenc_hmac_sha256_ecb_null; + +void eip93_aead_handle_result(struct crypto_async_request *async, int err); + +#endif /* _EIP93_AEAD_H_ */ diff --git a/drivers/crypto/inside-secure/eip93/eip93-aes.h b/drivers/crypto/inside-secure/eip93/eip93-aes.h new file mode 100644 index 000000000..82064cc8f --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-aes.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2019 - 2021 + * + * Richard van Schagen + * Christian Marangi + */ +#ifndef _EIP93_AES_H_ +#define _EIP93_AES_H_ + +extern struct eip93_alg_template eip93_alg_ecb_aes; +extern struct eip93_alg_template eip93_alg_cbc_aes; +extern struct eip93_alg_template eip93_alg_ctr_aes; +extern struct eip93_alg_template eip93_alg_rfc3686_aes; + +#endif /* _EIP93_AES_H_ */ diff --git a/drivers/crypto/inside-secure/eip93/eip93-cipher.c b/drivers/crypto/inside-secure/eip93/eip93-cipher.c new file mode 100644 index 000000000..4dd7ab750 --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-cipher.c @@ -0,0 +1,413 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2019 - 2021 + * + * Richard van Schagen + * Christian Marangi + */ + +#include +#include +#include +#include + +#include "eip93-aes.h" +#include "eip93-cipher.h" +#include "eip93-common.h" +#include "eip93-des.h" +#include "eip93-regs.h" + +void eip93_skcipher_handle_result(struct crypto_async_request *async, int err) +{ + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(async->tfm); + struct eip93_device *eip93 = ctx->eip93; + struct skcipher_request *req = skcipher_request_cast(async); + struct eip93_cipher_reqctx *rctx = skcipher_request_ctx(req); + + eip93_unmap_dma(eip93, rctx, req->src, req->dst); + eip93_handle_result(eip93, rctx, req->iv); + + skcipher_request_complete(req, err); +} + +static int eip93_skcipher_send_req(struct crypto_async_request *async) +{ + struct skcipher_request *req = skcipher_request_cast(async); + struct eip93_cipher_reqctx *rctx = skcipher_request_ctx(req); + int err; + + err = check_valid_request(rctx); + + if (err) { + skcipher_request_complete(req, err); + return err; + } + + return eip93_send_req(async, req->iv, rctx); +} + +/* Crypto skcipher API functions */ +static int eip93_skcipher_cra_init(struct crypto_tfm *tfm) +{ + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + struct eip93_alg_template *tmpl = container_of(tfm->__crt_alg, + struct eip93_alg_template, alg.skcipher.base); + + crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm), + sizeof(struct eip93_cipher_reqctx)); + + memset(ctx, 0, sizeof(*ctx)); + + ctx->eip93 = tmpl->eip93; + ctx->type = tmpl->type; + + ctx->sa_record = kzalloc_obj(*ctx->sa_record); + if (!ctx->sa_record) + return -ENOMEM; + + return 0; +} + +static void eip93_skcipher_cra_exit(struct crypto_tfm *tfm) +{ + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + + dma_unmap_single(ctx->eip93->dev, ctx->sa_record_base, + sizeof(*ctx->sa_record), DMA_TO_DEVICE); + kfree(ctx->sa_record); +} + +static int eip93_skcipher_setkey(struct crypto_skcipher *ctfm, const u8 *key, + unsigned int len) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + struct eip93_alg_template *tmpl = container_of(tfm->__crt_alg, + struct eip93_alg_template, + alg.skcipher.base); + struct sa_record *sa_record = ctx->sa_record; + unsigned int keylen = len; + u32 flags = tmpl->flags; + u32 nonce = 0; + int ret; + + if (!key || !keylen) + return -EINVAL; + + if (IS_RFC3686(flags)) { + if (len < CTR_RFC3686_NONCE_SIZE) + return -EINVAL; + + keylen = len - CTR_RFC3686_NONCE_SIZE; + memcpy(&nonce, key + keylen, CTR_RFC3686_NONCE_SIZE); + } + + if (flags & EIP93_ALG_DES) { + ctx->blksize = DES_BLOCK_SIZE; + ret = verify_skcipher_des_key(ctfm, key); + if (ret) + return ret; + } + if (flags & EIP93_ALG_3DES) { + ctx->blksize = DES3_EDE_BLOCK_SIZE; + ret = verify_skcipher_des3_key(ctfm, key); + if (ret) + return ret; + } + + if (flags & EIP93_ALG_AES) { + struct crypto_aes_ctx aes; + + ctx->blksize = AES_BLOCK_SIZE; + ret = aes_expandkey(&aes, key, keylen); + if (ret) + return ret; + } + + eip93_set_sa_record(sa_record, keylen, flags); + + memcpy(sa_record->sa_key, key, keylen); + ctx->sa_nonce = nonce; + sa_record->sa_nonce = nonce; + + return 0; +} + +static int eip93_skcipher_crypt(struct skcipher_request *req) +{ + struct eip93_cipher_reqctx *rctx = skcipher_request_ctx(req); + struct crypto_async_request *async = &req->base; + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + int ret; + + if (!req->cryptlen) + return 0; + + /* + * ECB and CBC algorithms require message lengths to be + * multiples of block size. + */ + if (IS_ECB(rctx->flags) || IS_CBC(rctx->flags)) + if (!IS_ALIGNED(req->cryptlen, + crypto_skcipher_blocksize(skcipher))) + return -EINVAL; + + ctx->sa_record_base = dma_map_single(ctx->eip93->dev, ctx->sa_record, + sizeof(*ctx->sa_record), DMA_TO_DEVICE); + ret = dma_mapping_error(ctx->eip93->dev, ctx->sa_record_base); + if (ret) + return ret; + + rctx->assoclen = 0; + rctx->textsize = req->cryptlen; + rctx->authsize = 0; + rctx->sg_src = req->src; + rctx->sg_dst = req->dst; + rctx->ivsize = crypto_skcipher_ivsize(skcipher); + rctx->blksize = ctx->blksize; + rctx->desc_flags = EIP93_DESC_SKCIPHER; + rctx->sa_record_base = ctx->sa_record_base; + + return eip93_skcipher_send_req(async); +} + +static int eip93_skcipher_encrypt(struct skcipher_request *req) +{ + struct eip93_cipher_reqctx *rctx = skcipher_request_ctx(req); + struct eip93_alg_template *tmpl = container_of(req->base.tfm->__crt_alg, + struct eip93_alg_template, alg.skcipher.base); + + rctx->flags = tmpl->flags; + rctx->flags |= EIP93_ENCRYPT; + + return eip93_skcipher_crypt(req); +} + +static int eip93_skcipher_decrypt(struct skcipher_request *req) +{ + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct eip93_cipher_reqctx *rctx = skcipher_request_ctx(req); + struct eip93_alg_template *tmpl = container_of(req->base.tfm->__crt_alg, + struct eip93_alg_template, alg.skcipher.base); + + ctx->sa_record->sa_cmd0_word |= EIP93_SA_CMD_DIRECTION_IN; + + rctx->flags = tmpl->flags; + rctx->flags |= EIP93_DECRYPT; + + return eip93_skcipher_crypt(req); +} + +/* Available algorithms in this module */ +struct eip93_alg_template eip93_alg_ecb_aes = { + .type = EIP93_ALG_TYPE_SKCIPHER, + .flags = EIP93_MODE_ECB | EIP93_ALG_AES, + .alg.skcipher = { + .setkey = eip93_skcipher_setkey, + .encrypt = eip93_skcipher_encrypt, + .decrypt = eip93_skcipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = 0, + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb(aes-eip93)", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0xf, + .cra_init = eip93_skcipher_cra_init, + .cra_exit = eip93_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_cbc_aes = { + .type = EIP93_ALG_TYPE_SKCIPHER, + .flags = EIP93_MODE_CBC | EIP93_ALG_AES, + .alg.skcipher = { + .setkey = eip93_skcipher_setkey, + .encrypt = eip93_skcipher_encrypt, + .decrypt = eip93_skcipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc(aes-eip93)", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0xf, + .cra_init = eip93_skcipher_cra_init, + .cra_exit = eip93_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_ctr_aes = { + .type = EIP93_ALG_TYPE_SKCIPHER, + .flags = EIP93_MODE_CTR | EIP93_ALG_AES, + .alg.skcipher = { + .setkey = eip93_skcipher_setkey, + .encrypt = eip93_skcipher_encrypt, + .decrypt = eip93_skcipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .base = { + .cra_name = "ctr(aes)", + .cra_driver_name = "ctr(aes-eip93)", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0xf, + .cra_init = eip93_skcipher_cra_init, + .cra_exit = eip93_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_rfc3686_aes = { + .type = EIP93_ALG_TYPE_SKCIPHER, + .flags = EIP93_MODE_CTR | EIP93_MODE_RFC3686 | EIP93_ALG_AES, + .alg.skcipher = { + .setkey = eip93_skcipher_setkey, + .encrypt = eip93_skcipher_encrypt, + .decrypt = eip93_skcipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + .base = { + .cra_name = "rfc3686(ctr(aes))", + .cra_driver_name = "rfc3686(ctr(aes-eip93))", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0xf, + .cra_init = eip93_skcipher_cra_init, + .cra_exit = eip93_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_ecb_des = { + .type = EIP93_ALG_TYPE_SKCIPHER, + .flags = EIP93_MODE_ECB | EIP93_ALG_DES, + .alg.skcipher = { + .setkey = eip93_skcipher_setkey, + .encrypt = eip93_skcipher_encrypt, + .decrypt = eip93_skcipher_decrypt, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = 0, + .base = { + .cra_name = "ecb(des)", + .cra_driver_name = "ecb(des-eip93)", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_skcipher_cra_init, + .cra_exit = eip93_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_cbc_des = { + .type = EIP93_ALG_TYPE_SKCIPHER, + .flags = EIP93_MODE_CBC | EIP93_ALG_DES, + .alg.skcipher = { + .setkey = eip93_skcipher_setkey, + .encrypt = eip93_skcipher_encrypt, + .decrypt = eip93_skcipher_decrypt, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .base = { + .cra_name = "cbc(des)", + .cra_driver_name = "cbc(des-eip93)", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_skcipher_cra_init, + .cra_exit = eip93_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_ecb_des3_ede = { + .type = EIP93_ALG_TYPE_SKCIPHER, + .flags = EIP93_MODE_ECB | EIP93_ALG_3DES, + .alg.skcipher = { + .setkey = eip93_skcipher_setkey, + .encrypt = eip93_skcipher_encrypt, + .decrypt = eip93_skcipher_decrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = 0, + .base = { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "ecb(des3_ede-eip93)", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_skcipher_cra_init, + .cra_exit = eip93_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +struct eip93_alg_template eip93_alg_cbc_des3_ede = { + .type = EIP93_ALG_TYPE_SKCIPHER, + .flags = EIP93_MODE_CBC | EIP93_ALG_3DES, + .alg.skcipher = { + .setkey = eip93_skcipher_setkey, + .encrypt = eip93_skcipher_encrypt, + .decrypt = eip93_skcipher_decrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .base = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "cbc(des3_ede-eip93)", + .cra_priority = EIP93_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_crypto_ctx), + .cra_alignmask = 0, + .cra_init = eip93_skcipher_cra_init, + .cra_exit = eip93_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; diff --git a/drivers/crypto/inside-secure/eip93/eip93-cipher.h b/drivers/crypto/inside-secure/eip93/eip93-cipher.h new file mode 100644 index 000000000..47e4e84ff --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-cipher.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2019 - 2021 + * + * Richard van Schagen + * Christian Marangi + */ +#ifndef _EIP93_CIPHER_H_ +#define _EIP93_CIPHER_H_ + +#include "eip93-main.h" + +struct eip93_crypto_ctx { + struct eip93_device *eip93; + u32 flags; + struct sa_record *sa_record; + u32 sa_nonce; + int blksize; + dma_addr_t sa_record_base; + /* AEAD specific */ + unsigned int authsize; + unsigned int assoclen; + bool set_assoc; + enum eip93_alg_type type; +}; + +struct eip93_cipher_reqctx { + u16 desc_flags; + u16 flags; + unsigned int blksize; + unsigned int ivsize; + unsigned int textsize; + unsigned int assoclen; + unsigned int authsize; + dma_addr_t sa_record_base; + struct sa_state *sa_state; + dma_addr_t sa_state_base; + struct eip93_descriptor *cdesc; + struct scatterlist *sg_src; + struct scatterlist *sg_dst; + int src_nents; + int dst_nents; + struct sa_state *sa_state_ctr; + dma_addr_t sa_state_ctr_base; +}; + +int check_valid_request(struct eip93_cipher_reqctx *rctx); + +void eip93_unmap_dma(struct eip93_device *eip93, struct eip93_cipher_reqctx *rctx, + struct scatterlist *reqsrc, struct scatterlist *reqdst); + +void eip93_skcipher_handle_result(struct crypto_async_request *async, int err); + +int eip93_send_req(struct crypto_async_request *async, + const u8 *reqiv, struct eip93_cipher_reqctx *rctx); + +void eip93_handle_result(struct eip93_device *eip93, struct eip93_cipher_reqctx *rctx, + u8 *reqiv); + +#endif /* _EIP93_CIPHER_H_ */ diff --git a/drivers/crypto/inside-secure/eip93/eip93-common.c b/drivers/crypto/inside-secure/eip93/eip93-common.c new file mode 100644 index 000000000..dacf586b2 --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-common.c @@ -0,0 +1,821 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2019 - 2021 + * + * Richard van Schagen + * Christian Marangi + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "eip93-cipher.h" +#include "eip93-hash.h" +#include "eip93-common.h" +#include "eip93-main.h" +#include "eip93-regs.h" + +int eip93_parse_ctrl_stat_err(struct eip93_device *eip93, int err) +{ + u32 ext_err; + + if (!err) + return 0; + + switch (err & ~EIP93_PE_CTRL_PE_EXT_ERR_CODE) { + case EIP93_PE_CTRL_PE_AUTH_ERR: + case EIP93_PE_CTRL_PE_PAD_ERR: + return -EBADMSG; + /* let software handle anti-replay errors */ + case EIP93_PE_CTRL_PE_SEQNUM_ERR: + return 0; + case EIP93_PE_CTRL_PE_EXT_ERR: + break; + default: + dev_err(eip93->dev, "Unhandled error 0x%08x\n", err); + return -EINVAL; + } + + /* Parse additional ext errors */ + ext_err = FIELD_GET(EIP93_PE_CTRL_PE_EXT_ERR_CODE, err); + switch (ext_err) { + case EIP93_PE_CTRL_PE_EXT_ERR_BUS: + case EIP93_PE_CTRL_PE_EXT_ERR_PROCESSING: + return -EIO; + case EIP93_PE_CTRL_PE_EXT_ERR_DESC_OWNER: + return -EACCES; + case EIP93_PE_CTRL_PE_EXT_ERR_INVALID_CRYPTO_OP: + case EIP93_PE_CTRL_PE_EXT_ERR_INVALID_CRYPTO_ALGO: + case EIP93_PE_CTRL_PE_EXT_ERR_SPI: + return -EINVAL; + case EIP93_PE_CTRL_PE_EXT_ERR_ZERO_LENGTH: + case EIP93_PE_CTRL_PE_EXT_ERR_INVALID_PK_LENGTH: + case EIP93_PE_CTRL_PE_EXT_ERR_BLOCK_SIZE_ERR: + return -EBADMSG; + default: + dev_err(eip93->dev, "Unhandled ext error 0x%08x\n", ext_err); + return -EINVAL; + } +} + +static void *eip93_ring_next_wptr(struct eip93_device *eip93, + struct eip93_desc_ring *ring) +{ + void *ptr = ring->write; + + if ((ring->write == ring->read - ring->offset) || + (ring->read == ring->base && ring->write == ring->base_end)) + return ERR_PTR(-ENOMEM); + + if (ring->write == ring->base_end) + ring->write = ring->base; + else + ring->write += ring->offset; + + return ptr; +} + +static void *eip93_ring_next_rptr(struct eip93_device *eip93, + struct eip93_desc_ring *ring) +{ + void *ptr = ring->read; + + if (ring->write == ring->read) + return ERR_PTR(-ENOENT); + + if (ring->read == ring->base_end) + ring->read = ring->base; + else + ring->read += ring->offset; + + return ptr; +} + +int eip93_put_descriptor(struct eip93_device *eip93, + struct eip93_descriptor *desc) +{ + struct eip93_descriptor *cdesc; + struct eip93_descriptor *rdesc; + + rdesc = eip93_ring_next_wptr(eip93, &eip93->ring->rdr); + if (IS_ERR(rdesc)) + return -ENOENT; + + cdesc = eip93_ring_next_wptr(eip93, &eip93->ring->cdr); + if (IS_ERR(cdesc)) + return -ENOENT; + + memset(rdesc, 0, sizeof(struct eip93_descriptor)); + + memcpy(cdesc, desc, sizeof(struct eip93_descriptor)); + + return 0; +} + +void *eip93_get_descriptor(struct eip93_device *eip93) +{ + struct eip93_descriptor *cdesc; + void *ptr; + + cdesc = eip93_ring_next_rptr(eip93, &eip93->ring->cdr); + if (IS_ERR(cdesc)) + return ERR_PTR(-ENOENT); + + memset(cdesc, 0, sizeof(struct eip93_descriptor)); + + ptr = eip93_ring_next_rptr(eip93, &eip93->ring->rdr); + if (IS_ERR(ptr)) + return ERR_PTR(-ENOENT); + + return ptr; +} + +static void eip93_free_sg_copy(const int len, struct scatterlist **sg) +{ + if (!*sg || !len) + return; + + free_pages((unsigned long)sg_virt(*sg), get_order(len)); + kfree(*sg); + *sg = NULL; +} + +static int eip93_make_sg_copy(struct scatterlist *src, struct scatterlist **dst, + const u32 len, const bool copy) +{ + void *pages; + + *dst = kmalloc_obj(**dst); + if (!*dst) + return -ENOMEM; + + pages = (void *)__get_free_pages(GFP_KERNEL | GFP_DMA, + get_order(len)); + if (!pages) { + kfree(*dst); + *dst = NULL; + return -ENOMEM; + } + + sg_init_table(*dst, 1); + sg_set_buf(*dst, pages, len); + + /* copy only as requested */ + if (copy) + sg_copy_to_buffer(src, sg_nents(src), pages, len); + + return 0; +} + +static bool eip93_is_sg_aligned(struct scatterlist *sg, u32 len, + const int blksize) +{ + int nents; + + for (nents = 0; sg; sg = sg_next(sg), ++nents) { + if (!IS_ALIGNED(sg->offset, 4)) + return false; + + if (len <= sg->length) { + if (!IS_ALIGNED(len, blksize)) + return false; + + return true; + } + + if (!IS_ALIGNED(sg->length, blksize)) + return false; + + len -= sg->length; + } + return false; +} + +int check_valid_request(struct eip93_cipher_reqctx *rctx) +{ + struct scatterlist *src = rctx->sg_src; + struct scatterlist *dst = rctx->sg_dst; + u32 textsize = rctx->textsize; + u32 authsize = rctx->authsize; + u32 blksize = rctx->blksize; + u32 totlen_src = rctx->assoclen + rctx->textsize; + u32 totlen_dst = rctx->assoclen + rctx->textsize; + u32 copy_len; + bool src_align, dst_align; + int src_nents, dst_nents; + int err = -EINVAL; + + if (!IS_CTR(rctx->flags)) { + if (!IS_ALIGNED(textsize, blksize)) + return err; + } + + if (authsize) { + if (IS_ENCRYPT(rctx->flags)) + totlen_dst += authsize; + else + totlen_src += authsize; + } + + src_nents = sg_nents_for_len(src, totlen_src); + if (src_nents < 0) + return src_nents; + + dst_nents = sg_nents_for_len(dst, totlen_dst); + if (dst_nents < 0) + return dst_nents; + + if (src == dst) { + src_nents = max(src_nents, dst_nents); + dst_nents = src_nents; + if (unlikely((totlen_src || totlen_dst) && !src_nents)) + return err; + + } else { + if (unlikely(totlen_src && !src_nents)) + return err; + + if (unlikely(totlen_dst && !dst_nents)) + return err; + } + + if (authsize) { + if (dst_nents == 1 && src_nents == 1) { + src_align = eip93_is_sg_aligned(src, totlen_src, blksize); + if (src == dst) + dst_align = src_align; + else + dst_align = eip93_is_sg_aligned(dst, totlen_dst, blksize); + } else { + src_align = false; + dst_align = false; + } + } else { + src_align = eip93_is_sg_aligned(src, totlen_src, blksize); + if (src == dst) + dst_align = src_align; + else + dst_align = eip93_is_sg_aligned(dst, totlen_dst, blksize); + } + + copy_len = max(totlen_src, totlen_dst); + if (!src_align) { + err = eip93_make_sg_copy(src, &rctx->sg_src, copy_len, true); + if (err) + return err; + } + + if (!dst_align) { + err = eip93_make_sg_copy(dst, &rctx->sg_dst, copy_len, false); + if (err) + return err; + } + + src_nents = sg_nents_for_len(rctx->sg_src, totlen_src); + if (src_nents < 0) + return src_nents; + + dst_nents = sg_nents_for_len(rctx->sg_dst, totlen_dst); + if (dst_nents < 0) + return dst_nents; + + rctx->src_nents = src_nents; + rctx->dst_nents = dst_nents; + + return 0; +} + +/* + * Set sa_record function: + * Even sa_record is set to "0", keep " = 0" for readability. + */ +void eip93_set_sa_record(struct sa_record *sa_record, const unsigned int keylen, + const u32 flags) +{ + /* Reset cmd word */ + sa_record->sa_cmd0_word = 0; + sa_record->sa_cmd1_word = 0; + + sa_record->sa_cmd0_word |= EIP93_SA_CMD_IV_FROM_STATE; + if (!IS_ECB(flags)) + sa_record->sa_cmd0_word |= EIP93_SA_CMD_SAVE_IV; + + sa_record->sa_cmd0_word |= EIP93_SA_CMD_OP_BASIC; + + switch ((flags & EIP93_ALG_MASK)) { + case EIP93_ALG_AES: + sa_record->sa_cmd0_word |= EIP93_SA_CMD_CIPHER_AES; + sa_record->sa_cmd1_word |= FIELD_PREP(EIP93_SA_CMD_AES_KEY_LENGTH, + keylen >> 3); + break; + case EIP93_ALG_3DES: + sa_record->sa_cmd0_word |= EIP93_SA_CMD_CIPHER_3DES; + break; + case EIP93_ALG_DES: + sa_record->sa_cmd0_word |= EIP93_SA_CMD_CIPHER_DES; + break; + default: + sa_record->sa_cmd0_word |= EIP93_SA_CMD_CIPHER_NULL; + } + + switch ((flags & EIP93_HASH_MASK)) { + case EIP93_HASH_SHA256: + sa_record->sa_cmd0_word |= EIP93_SA_CMD_HASH_SHA256; + break; + case EIP93_HASH_SHA224: + sa_record->sa_cmd0_word |= EIP93_SA_CMD_HASH_SHA224; + break; + case EIP93_HASH_SHA1: + sa_record->sa_cmd0_word |= EIP93_SA_CMD_HASH_SHA1; + break; + case EIP93_HASH_MD5: + sa_record->sa_cmd0_word |= EIP93_SA_CMD_HASH_MD5; + break; + default: + sa_record->sa_cmd0_word |= EIP93_SA_CMD_HASH_NULL; + } + + sa_record->sa_cmd0_word |= EIP93_SA_CMD_PAD_ZERO; + + switch ((flags & EIP93_MODE_MASK)) { + case EIP93_MODE_CBC: + sa_record->sa_cmd1_word |= EIP93_SA_CMD_CHIPER_MODE_CBC; + break; + case EIP93_MODE_CTR: + sa_record->sa_cmd1_word |= EIP93_SA_CMD_CHIPER_MODE_CTR; + break; + case EIP93_MODE_ECB: + sa_record->sa_cmd1_word |= EIP93_SA_CMD_CHIPER_MODE_ECB; + break; + } + + sa_record->sa_cmd0_word |= EIP93_SA_CMD_DIGEST_3WORD; + if (IS_HASH(flags)) { + sa_record->sa_cmd1_word |= EIP93_SA_CMD_COPY_PAD; + sa_record->sa_cmd1_word |= EIP93_SA_CMD_COPY_DIGEST; + } + + if (IS_HMAC(flags)) { + sa_record->sa_cmd1_word |= EIP93_SA_CMD_HMAC; + sa_record->sa_cmd1_word |= EIP93_SA_CMD_COPY_HEADER; + } + + sa_record->sa_spi = 0x0; + sa_record->sa_seqmum_mask[0] = 0xFFFFFFFF; + sa_record->sa_seqmum_mask[1] = 0x0; +} + +/* + * Poor mans Scatter/gather function: + * Create a Descriptor for every segment to avoid copying buffers. + * For performance better to wait for hardware to perform multiple DMA + */ +static int eip93_scatter_combine(struct eip93_device *eip93, + struct eip93_cipher_reqctx *rctx, + u32 datalen, u32 split, int offsetin) +{ + struct eip93_descriptor *cdesc = rctx->cdesc; + struct scatterlist *sgsrc = rctx->sg_src; + struct scatterlist *sgdst = rctx->sg_dst; + unsigned int remainin = sg_dma_len(sgsrc); + unsigned int remainout = sg_dma_len(sgdst); + dma_addr_t saddr = sg_dma_address(sgsrc); + dma_addr_t daddr = sg_dma_address(sgdst); + dma_addr_t state_addr; + u32 src_addr, dst_addr, len, n; + bool nextin = false; + bool nextout = false; + int offsetout = 0; + int err; + + if (IS_ECB(rctx->flags)) + rctx->sa_state_base = 0; + + if (split < datalen) { + state_addr = rctx->sa_state_ctr_base; + n = split; + } else { + state_addr = rctx->sa_state_base; + n = datalen; + } + + do { + if (nextin) { + sgsrc = sg_next(sgsrc); + remainin = sg_dma_len(sgsrc); + if (remainin == 0) + continue; + + saddr = sg_dma_address(sgsrc); + offsetin = 0; + nextin = false; + } + + if (nextout) { + sgdst = sg_next(sgdst); + remainout = sg_dma_len(sgdst); + if (remainout == 0) + continue; + + daddr = sg_dma_address(sgdst); + offsetout = 0; + nextout = false; + } + src_addr = saddr + offsetin; + dst_addr = daddr + offsetout; + + if (remainin == remainout) { + len = remainin; + if (len > n) { + len = n; + remainin -= n; + remainout -= n; + offsetin += n; + offsetout += n; + } else { + nextin = true; + nextout = true; + } + } else if (remainin < remainout) { + len = remainin; + if (len > n) { + len = n; + remainin -= n; + remainout -= n; + offsetin += n; + offsetout += n; + } else { + offsetout += len; + remainout -= len; + nextin = true; + } + } else { + len = remainout; + if (len > n) { + len = n; + remainin -= n; + remainout -= n; + offsetin += n; + offsetout += n; + } else { + offsetin += len; + remainin -= len; + nextout = true; + } + } + n -= len; + + cdesc->src_addr = src_addr; + cdesc->dst_addr = dst_addr; + cdesc->state_addr = state_addr; + cdesc->pe_length_word = FIELD_PREP(EIP93_PE_LENGTH_HOST_PE_READY, + EIP93_PE_LENGTH_HOST_READY); + cdesc->pe_length_word |= FIELD_PREP(EIP93_PE_LENGTH_LENGTH, len); + + if (n == 0) { + n = datalen - split; + split = datalen; + state_addr = rctx->sa_state_base; + } + + if (n == 0) + cdesc->user_id |= FIELD_PREP(EIP93_PE_USER_ID_DESC_FLAGS, + EIP93_DESC_LAST); + + /* + * Loop - Delay - No need to rollback + * Maybe refine by slowing down at EIP93_RING_BUSY + */ +again: + scoped_guard(spinlock_irqsave, &eip93->ring->write_lock) + err = eip93_put_descriptor(eip93, cdesc); + if (err) { + usleep_range(EIP93_RING_BUSY_DELAY, + EIP93_RING_BUSY_DELAY * 2); + goto again; + } + /* Writing new descriptor count starts DMA action */ + writel(1, eip93->base + EIP93_REG_PE_CD_COUNT); + } while (n); + + return -EINPROGRESS; +} + +int eip93_send_req(struct crypto_async_request *async, + const u8 *reqiv, struct eip93_cipher_reqctx *rctx) +{ + struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(async->tfm); + struct eip93_device *eip93 = ctx->eip93; + struct scatterlist *src = rctx->sg_src; + struct scatterlist *dst = rctx->sg_dst; + struct sa_state *sa_state; + struct eip93_descriptor cdesc; + u32 flags = rctx->flags; + int offsetin = 0, err; + u32 datalen = rctx->assoclen + rctx->textsize; + u32 split = datalen; + u32 start, end, ctr, blocks; + u32 iv[AES_BLOCK_SIZE / sizeof(u32)]; + int crypto_async_idr; + + rctx->sa_state_ctr = NULL; + rctx->sa_state = NULL; + + if (IS_ECB(flags)) + goto skip_iv; + + memcpy(iv, reqiv, rctx->ivsize); + + rctx->sa_state = kzalloc_obj(*rctx->sa_state); + if (!rctx->sa_state) + return -ENOMEM; + + sa_state = rctx->sa_state; + + memcpy(sa_state->state_iv, iv, rctx->ivsize); + if (IS_RFC3686(flags)) { + sa_state->state_iv[0] = ctx->sa_nonce; + sa_state->state_iv[1] = iv[0]; + sa_state->state_iv[2] = iv[1]; + sa_state->state_iv[3] = (u32 __force)cpu_to_be32(0x1); + } else if (!IS_HMAC(flags) && IS_CTR(flags)) { + /* Compute data length. */ + blocks = DIV_ROUND_UP(rctx->textsize, AES_BLOCK_SIZE); + ctr = be32_to_cpu((__be32 __force)iv[3]); + /* Check 32bit counter overflow. */ + start = ctr; + end = start + blocks - 1; + if (end < start) { + split = AES_BLOCK_SIZE * -start; + /* + * Increment the counter manually to cope with + * the hardware counter overflow. + */ + iv[3] = 0xffffffff; + crypto_inc((u8 *)iv, AES_BLOCK_SIZE); + + rctx->sa_state_ctr = kzalloc_obj(*rctx->sa_state_ctr); + if (!rctx->sa_state_ctr) { + err = -ENOMEM; + goto free_sa_state; + } + + memcpy(rctx->sa_state_ctr->state_iv, reqiv, rctx->ivsize); + memcpy(sa_state->state_iv, iv, rctx->ivsize); + + rctx->sa_state_ctr_base = dma_map_single(eip93->dev, rctx->sa_state_ctr, + sizeof(*rctx->sa_state_ctr), + DMA_TO_DEVICE); + err = dma_mapping_error(eip93->dev, rctx->sa_state_ctr_base); + if (err) + goto free_sa_state_ctr; + } + } + + rctx->sa_state_base = dma_map_single(eip93->dev, rctx->sa_state, + sizeof(*rctx->sa_state), DMA_TO_DEVICE); + err = dma_mapping_error(eip93->dev, rctx->sa_state_base); + if (err) + goto free_sa_state_ctr_dma; + +skip_iv: + + cdesc.pe_ctrl_stat_word = FIELD_PREP(EIP93_PE_CTRL_PE_READY_DES_TRING_OWN, + EIP93_PE_CTRL_HOST_READY); + cdesc.sa_addr = rctx->sa_record_base; + cdesc.arc4_addr = 0; + + scoped_guard(spinlock_bh, &eip93->ring->idr_lock) + crypto_async_idr = idr_alloc(&eip93->ring->crypto_async_idr, async, 0, + EIP93_RING_NUM - 1, GFP_ATOMIC); + + cdesc.user_id = FIELD_PREP(EIP93_PE_USER_ID_CRYPTO_IDR, (u16)crypto_async_idr) | + FIELD_PREP(EIP93_PE_USER_ID_DESC_FLAGS, rctx->desc_flags); + + rctx->cdesc = &cdesc; + + /* map DMA_BIDIRECTIONAL to invalidate cache on destination + * implies __dma_cache_wback_inv + */ + if (!dma_map_sg(eip93->dev, dst, rctx->dst_nents, DMA_BIDIRECTIONAL)) { + err = -ENOMEM; + goto free_sa_state_ctr_dma; + } + + if (src != dst && + !dma_map_sg(eip93->dev, src, rctx->src_nents, DMA_TO_DEVICE)) { + err = -ENOMEM; + goto free_sg_dma; + } + + return eip93_scatter_combine(eip93, rctx, datalen, split, offsetin); + +free_sg_dma: + dma_unmap_sg(eip93->dev, dst, rctx->dst_nents, DMA_BIDIRECTIONAL); +free_sa_state_ctr_dma: + if (rctx->sa_state_ctr) + dma_unmap_single(eip93->dev, rctx->sa_state_ctr_base, + sizeof(*rctx->sa_state_ctr), + DMA_TO_DEVICE); +free_sa_state_ctr: + kfree(rctx->sa_state_ctr); + if (rctx->sa_state) + dma_unmap_single(eip93->dev, rctx->sa_state_base, + sizeof(*rctx->sa_state), + DMA_TO_DEVICE); +free_sa_state: + kfree(rctx->sa_state); + + return err; +} + +void eip93_unmap_dma(struct eip93_device *eip93, struct eip93_cipher_reqctx *rctx, + struct scatterlist *reqsrc, struct scatterlist *reqdst) +{ + u32 len = rctx->assoclen + rctx->textsize; + u32 authsize = rctx->authsize; + u32 flags = rctx->flags; + u32 *otag; + int i; + + if (rctx->sg_src == rctx->sg_dst) { + dma_unmap_sg(eip93->dev, rctx->sg_dst, rctx->dst_nents, + DMA_BIDIRECTIONAL); + goto process_tag; + } + + dma_unmap_sg(eip93->dev, rctx->sg_src, rctx->src_nents, + DMA_TO_DEVICE); + + if (rctx->sg_src != reqsrc) + eip93_free_sg_copy(len + rctx->authsize, &rctx->sg_src); + + dma_unmap_sg(eip93->dev, rctx->sg_dst, rctx->dst_nents, + DMA_BIDIRECTIONAL); + + /* SHA tags need conversion from net-to-host */ +process_tag: + if (IS_DECRYPT(flags)) + authsize = 0; + + if (authsize) { + if (!IS_HASH_MD5(flags)) { + otag = sg_virt(rctx->sg_dst) + len; + for (i = 0; i < (authsize / 4); i++) + otag[i] = be32_to_cpu((__be32 __force)otag[i]); + } + } + + if (rctx->sg_dst != reqdst) { + sg_copy_from_buffer(reqdst, sg_nents(reqdst), + sg_virt(rctx->sg_dst), len + authsize); + eip93_free_sg_copy(len + rctx->authsize, &rctx->sg_dst); + } +} + +void eip93_handle_result(struct eip93_device *eip93, struct eip93_cipher_reqctx *rctx, + u8 *reqiv) +{ + if (rctx->sa_state_ctr) + dma_unmap_single(eip93->dev, rctx->sa_state_ctr_base, + sizeof(*rctx->sa_state_ctr), + DMA_FROM_DEVICE); + + if (rctx->sa_state) + dma_unmap_single(eip93->dev, rctx->sa_state_base, + sizeof(*rctx->sa_state), + DMA_FROM_DEVICE); + + if (!IS_ECB(rctx->flags)) + memcpy(reqiv, rctx->sa_state->state_iv, rctx->ivsize); + + kfree(rctx->sa_state_ctr); + kfree(rctx->sa_state); +} + +int eip93_hmac_setkey(u32 ctx_flags, const u8 *key, unsigned int keylen, + unsigned int hashlen, u8 *dest_ipad, u8 *dest_opad, + bool skip_ipad) +{ + u8 ipad[SHA256_BLOCK_SIZE], opad[SHA256_BLOCK_SIZE]; + struct crypto_ahash *ahash_tfm; + struct eip93_hash_reqctx *rctx; + struct ahash_request *req; + DECLARE_CRYPTO_WAIT(wait); + struct scatterlist sg[1]; + const char *alg_name; + int i, ret; + + switch (ctx_flags & EIP93_HASH_MASK) { + case EIP93_HASH_SHA256: + alg_name = "sha256-eip93"; + break; + case EIP93_HASH_SHA224: + alg_name = "sha224-eip93"; + break; + case EIP93_HASH_SHA1: + alg_name = "sha1-eip93"; + break; + case EIP93_HASH_MD5: + alg_name = "md5-eip93"; + break; + default: /* Impossible */ + return -EINVAL; + } + + ahash_tfm = crypto_alloc_ahash(alg_name, 0, 0); + if (IS_ERR(ahash_tfm)) + return PTR_ERR(ahash_tfm); + + req = ahash_request_alloc(ahash_tfm, GFP_ATOMIC); + if (!req) { + ret = -ENOMEM; + goto err_ahash; + } + + rctx = ahash_request_ctx_dma(req); + crypto_init_wait(&wait); + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + + /* Hash the key if > SHA256_BLOCK_SIZE */ + if (keylen > SHA256_BLOCK_SIZE) { + sg_init_one(&sg[0], key, keylen); + + ahash_request_set_crypt(req, sg, ipad, keylen); + ret = crypto_wait_req(crypto_ahash_digest(req), &wait); + if (ret) + goto err_req; + + keylen = hashlen; + } else { + memcpy(ipad, key, keylen); + } + + /* Copy to opad */ + memset(ipad + keylen, 0, SHA256_BLOCK_SIZE - keylen); + memcpy(opad, ipad, SHA256_BLOCK_SIZE); + + /* Pad with HMAC constants */ + for (i = 0; i < SHA256_BLOCK_SIZE; i++) { + ipad[i] ^= HMAC_IPAD_VALUE; + opad[i] ^= HMAC_OPAD_VALUE; + } + + if (skip_ipad) { + memcpy(dest_ipad, ipad, SHA256_BLOCK_SIZE); + } else { + /* Hash ipad */ + sg_init_one(&sg[0], ipad, SHA256_BLOCK_SIZE); + ahash_request_set_crypt(req, sg, dest_ipad, SHA256_BLOCK_SIZE); + ret = crypto_ahash_init(req); + if (ret) + goto err_req; + + /* Disable HASH_FINALIZE for ipad hash */ + rctx->partial_hash = true; + + ret = crypto_wait_req(crypto_ahash_finup(req), &wait); + if (ret) + goto err_req; + } + + /* Hash opad */ + sg_init_one(&sg[0], opad, SHA256_BLOCK_SIZE); + ahash_request_set_crypt(req, sg, dest_opad, SHA256_BLOCK_SIZE); + ret = crypto_ahash_init(req); + if (ret) + goto err_req; + + /* Disable HASH_FINALIZE for opad hash */ + rctx->partial_hash = true; + + ret = crypto_wait_req(crypto_ahash_finup(req), &wait); + if (ret) + goto err_req; + + if (!IS_HASH_MD5(ctx_flags)) { + for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(u32); i++) { + u32 *ipad_hash = (u32 *)dest_ipad; + u32 *opad_hash = (u32 *)dest_opad; + + if (!skip_ipad) + ipad_hash[i] = (u32 __force)cpu_to_be32(ipad_hash[i]); + opad_hash[i] = (u32 __force)cpu_to_be32(opad_hash[i]); + } + } + +err_req: + ahash_request_free(req); +err_ahash: + crypto_free_ahash(ahash_tfm); + + return ret; +} diff --git a/drivers/crypto/inside-secure/eip93/eip93-common.h b/drivers/crypto/inside-secure/eip93/eip93-common.h new file mode 100644 index 000000000..41c43782e --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-common.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2019 - 2021 + * + * Richard van Schagen + * Christian Marangi + */ + +#ifndef _EIP93_COMMON_H_ +#define _EIP93_COMMON_H_ + +void *eip93_get_descriptor(struct eip93_device *eip93); +int eip93_put_descriptor(struct eip93_device *eip93, struct eip93_descriptor *desc); + +void eip93_set_sa_record(struct sa_record *sa_record, const unsigned int keylen, + const u32 flags); + +int eip93_parse_ctrl_stat_err(struct eip93_device *eip93, int err); + +int eip93_hmac_setkey(u32 ctx_flags, const u8 *key, unsigned int keylen, + unsigned int hashlen, u8 *ipad, u8 *opad, + bool skip_ipad); + +#endif /* _EIP93_COMMON_H_ */ diff --git a/drivers/crypto/inside-secure/eip93/eip93-des.h b/drivers/crypto/inside-secure/eip93/eip93-des.h new file mode 100644 index 000000000..53ffe0f34 --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-des.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2019 - 2021 + * + * Richard van Schagen + * Christian Marangi + */ +#ifndef _EIP93_DES_H_ +#define _EIP93_DES_H_ + +extern struct eip93_alg_template eip93_alg_ecb_des; +extern struct eip93_alg_template eip93_alg_cbc_des; +extern struct eip93_alg_template eip93_alg_ecb_des3_ede; +extern struct eip93_alg_template eip93_alg_cbc_des3_ede; + +#endif /* _EIP93_DES_H_ */ diff --git a/drivers/crypto/inside-secure/eip93/eip93-hash.c b/drivers/crypto/inside-secure/eip93/eip93-hash.c new file mode 100644 index 000000000..84d3ff2d3 --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-hash.c @@ -0,0 +1,875 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2024 + * + * Christian Marangi + */ + +#include +#include +#include +#include +#include +#include + +#include "eip93-cipher.h" +#include "eip93-hash.h" +#include "eip93-main.h" +#include "eip93-common.h" +#include "eip93-regs.h" + +static void eip93_hash_free_data_blocks(struct ahash_request *req) +{ + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct eip93_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct eip93_device *eip93 = ctx->eip93; + struct mkt_hash_block *block, *tmp; + + list_for_each_entry_safe(block, tmp, &rctx->blocks, list) { + dma_unmap_single(eip93->dev, block->data_dma, + SHA256_BLOCK_SIZE, DMA_TO_DEVICE); + kfree(block); + } + if (!list_empty(&rctx->blocks)) + INIT_LIST_HEAD(&rctx->blocks); + + if (rctx->finalize) + dma_unmap_single(eip93->dev, rctx->data_dma, + rctx->data_used, + DMA_TO_DEVICE); +} + +static void eip93_hash_free_sa_record(struct ahash_request *req) +{ + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct eip93_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct eip93_device *eip93 = ctx->eip93; + + if (IS_HMAC(ctx->flags)) + dma_unmap_single(eip93->dev, rctx->sa_record_hmac_base, + sizeof(rctx->sa_record_hmac), DMA_TO_DEVICE); + + dma_unmap_single(eip93->dev, rctx->sa_record_base, + sizeof(rctx->sa_record), DMA_TO_DEVICE); +} + +void eip93_hash_handle_result(struct crypto_async_request *async, int err) +{ + struct ahash_request *req = ahash_request_cast(async); + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct eip93_hash_ctx *ctx = crypto_ahash_ctx(ahash); + int digestsize = crypto_ahash_digestsize(ahash); + struct sa_state *sa_state = &rctx->sa_state; + struct eip93_device *eip93 = ctx->eip93; + int i; + + dma_unmap_single(eip93->dev, rctx->sa_state_base, + sizeof(*sa_state), DMA_FROM_DEVICE); + + /* + * With partial_hash assume SHA256_DIGEST_SIZE buffer is passed. + * This is to handle SHA224 that have a 32 byte intermediate digest. + */ + if (rctx->partial_hash) + digestsize = SHA256_DIGEST_SIZE; + + if (rctx->finalize || rctx->partial_hash) { + /* bytes needs to be swapped for req->result */ + if (!IS_HASH_MD5(ctx->flags)) { + for (i = 0; i < digestsize / sizeof(u32); i++) { + u32 *digest = (u32 *)sa_state->state_i_digest; + + digest[i] = be32_to_cpu((__be32 __force)digest[i]); + } + } + + memcpy(req->result, sa_state->state_i_digest, digestsize); + } + + eip93_hash_free_sa_record(req); + eip93_hash_free_data_blocks(req); + + ahash_request_complete(req, err); +} + +static void eip93_hash_init_sa_state_digest(u32 hash, u8 *digest) +{ + static const u32 sha256_init[] = { + SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, + SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7 + }; + static const u32 sha224_init[] = { + SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3, + SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7 + }; + static const u32 sha1_init[] = { + SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 + }; + static const u32 md5_init[] = { + MD5_H0, MD5_H1, MD5_H2, MD5_H3 + }; + + /* Init HASH constant */ + switch (hash) { + case EIP93_HASH_SHA256: + memcpy(digest, sha256_init, sizeof(sha256_init)); + return; + case EIP93_HASH_SHA224: + memcpy(digest, sha224_init, sizeof(sha224_init)); + return; + case EIP93_HASH_SHA1: + memcpy(digest, sha1_init, sizeof(sha1_init)); + return; + case EIP93_HASH_MD5: + memcpy(digest, md5_init, sizeof(md5_init)); + return; + default: /* Impossible */ + return; + } +} + +static void eip93_hash_export_sa_state(struct ahash_request *req, + struct eip93_hash_export_state *state) +{ + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + struct sa_state *sa_state = &rctx->sa_state; + + /* + * EIP93 have special handling for state_byte_cnt in sa_state. + * Even if a zero packet is passed (and a BADMSG is returned), + * state_byte_cnt is incremented to the digest handled (with the hash + * primitive). This is problematic with export/import as EIP93 + * expect 0 state_byte_cnt for the very first iteration. + */ + if (!rctx->len) + memset(state->state_len, 0, sizeof(u32) * 2); + else + memcpy(state->state_len, sa_state->state_byte_cnt, + sizeof(u32) * 2); + memcpy(state->state_hash, sa_state->state_i_digest, + SHA256_DIGEST_SIZE); + state->len = rctx->len; + state->data_used = rctx->data_used; +} + +static void __eip93_hash_init(struct ahash_request *req) +{ + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct eip93_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct sa_record *sa_record = &rctx->sa_record; + int digestsize; + + digestsize = crypto_ahash_digestsize(ahash); + + eip93_set_sa_record(sa_record, 0, ctx->flags); + sa_record->sa_cmd0_word |= EIP93_SA_CMD_HASH_FROM_STATE; + sa_record->sa_cmd0_word |= EIP93_SA_CMD_SAVE_HASH; + sa_record->sa_cmd0_word &= ~EIP93_SA_CMD_OPCODE; + sa_record->sa_cmd0_word |= FIELD_PREP(EIP93_SA_CMD_OPCODE, + EIP93_SA_CMD_OPCODE_BASIC_OUT_HASH); + sa_record->sa_cmd0_word &= ~EIP93_SA_CMD_DIGEST_LENGTH; + sa_record->sa_cmd0_word |= FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, + digestsize / sizeof(u32)); + + /* + * HMAC special handling + * Enabling CMD_HMAC force the inner hash to be always finalized. + * This cause problems on handling message > 64 byte as we + * need to produce intermediate inner hash on sending intermediate + * 64 bytes blocks. + * + * To handle this, enable CMD_HMAC only on the last block. + * We make a duplicate of sa_record and on the last descriptor, + * we pass a dedicated sa_record with CMD_HMAC enabled to make + * EIP93 apply the outer hash. + */ + if (IS_HMAC(ctx->flags)) { + struct sa_record *sa_record_hmac = &rctx->sa_record_hmac; + + memcpy(sa_record_hmac, sa_record, sizeof(*sa_record)); + /* Copy pre-hashed opad for HMAC */ + memcpy(sa_record_hmac->sa_o_digest, ctx->opad, SHA256_DIGEST_SIZE); + + /* Disable HMAC for hash normal sa_record */ + sa_record->sa_cmd1_word &= ~EIP93_SA_CMD_HMAC; + } + + rctx->len = 0; + rctx->data_used = 0; + rctx->partial_hash = false; + rctx->finalize = false; + INIT_LIST_HEAD(&rctx->blocks); +} + +static int eip93_send_hash_req(struct crypto_async_request *async, u8 *data, + dma_addr_t *data_dma, u32 len, bool last) +{ + struct ahash_request *req = ahash_request_cast(async); + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct eip93_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct eip93_device *eip93 = ctx->eip93; + struct eip93_descriptor cdesc = { }; + dma_addr_t src_addr; + int ret; + + /* Map block data to DMA */ + src_addr = dma_map_single(eip93->dev, data, len, DMA_TO_DEVICE); + ret = dma_mapping_error(eip93->dev, src_addr); + if (ret) + return ret; + + cdesc.pe_ctrl_stat_word = FIELD_PREP(EIP93_PE_CTRL_PE_READY_DES_TRING_OWN, + EIP93_PE_CTRL_HOST_READY); + cdesc.sa_addr = rctx->sa_record_base; + cdesc.arc4_addr = 0; + + cdesc.state_addr = rctx->sa_state_base; + cdesc.src_addr = src_addr; + cdesc.pe_length_word = FIELD_PREP(EIP93_PE_LENGTH_HOST_PE_READY, + EIP93_PE_LENGTH_HOST_READY); + cdesc.pe_length_word |= FIELD_PREP(EIP93_PE_LENGTH_LENGTH, + len); + + cdesc.user_id |= FIELD_PREP(EIP93_PE_USER_ID_DESC_FLAGS, EIP93_DESC_HASH); + + if (last) { + int crypto_async_idr; + + if (rctx->finalize && !rctx->partial_hash) { + /* For last block, pass sa_record with CMD_HMAC enabled */ + if (IS_HMAC(ctx->flags)) { + struct sa_record *sa_record_hmac = &rctx->sa_record_hmac; + + rctx->sa_record_hmac_base = dma_map_single(eip93->dev, + sa_record_hmac, + sizeof(*sa_record_hmac), + DMA_TO_DEVICE); + ret = dma_mapping_error(eip93->dev, rctx->sa_record_hmac_base); + if (ret) + return ret; + + cdesc.sa_addr = rctx->sa_record_hmac_base; + } + + cdesc.pe_ctrl_stat_word |= EIP93_PE_CTRL_PE_HASH_FINAL; + } + + scoped_guard(spinlock_bh, &eip93->ring->idr_lock) + crypto_async_idr = idr_alloc(&eip93->ring->crypto_async_idr, async, 0, + EIP93_RING_NUM - 1, GFP_ATOMIC); + + cdesc.user_id |= FIELD_PREP(EIP93_PE_USER_ID_CRYPTO_IDR, (u16)crypto_async_idr) | + FIELD_PREP(EIP93_PE_USER_ID_DESC_FLAGS, EIP93_DESC_LAST); + } + +again: + scoped_guard(spinlock_irqsave, &eip93->ring->write_lock) + ret = eip93_put_descriptor(eip93, &cdesc); + if (ret) { + usleep_range(EIP93_RING_BUSY_DELAY, + EIP93_RING_BUSY_DELAY * 2); + goto again; + } + + /* Writing new descriptor count starts DMA action */ + writel(1, eip93->base + EIP93_REG_PE_CD_COUNT); + + *data_dma = src_addr; + return 0; +} + +static int eip93_hash_init(struct ahash_request *req) +{ + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct eip93_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct sa_state *sa_state = &rctx->sa_state; + + memset(sa_state->state_byte_cnt, 0, sizeof(u32) * 2); + eip93_hash_init_sa_state_digest(ctx->flags & EIP93_HASH_MASK, + sa_state->state_i_digest); + + __eip93_hash_init(req); + + /* For HMAC setup the initial block for ipad */ + if (IS_HMAC(ctx->flags)) { + memcpy(rctx->data, ctx->ipad, SHA256_BLOCK_SIZE); + + rctx->data_used = SHA256_BLOCK_SIZE; + rctx->len += SHA256_BLOCK_SIZE; + } + + return 0; +} + +/* + * With complete_req true, we wait for the engine to consume all the block in list, + * else we just queue the block to the engine as final() will wait. This is useful + * for finup(). + */ +static int __eip93_hash_update(struct ahash_request *req, bool complete_req) +{ + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + struct crypto_async_request *async = &req->base; + unsigned int read, to_consume = req->nbytes; + unsigned int max_read, consumed = 0; + struct mkt_hash_block *block; + bool wait_req = false; + int offset; + int ret; + + /* Get the offset and available space to fill req data */ + offset = rctx->data_used; + max_read = SHA256_BLOCK_SIZE - offset; + + /* Consume req in block of SHA256_BLOCK_SIZE. + * to_read is initially set to space available in the req data + * and then reset to SHA256_BLOCK_SIZE. + */ + while (to_consume > max_read) { + block = kzalloc_obj(*block, GFP_ATOMIC); + if (!block) { + ret = -ENOMEM; + goto free_blocks; + } + + read = sg_pcopy_to_buffer(req->src, sg_nents(req->src), + block->data + offset, + max_read, consumed); + + /* + * For first iteration only, copy req data to block + * and reset offset and max_read for next iteration. + */ + if (offset > 0) { + memcpy(block->data, rctx->data, offset); + offset = 0; + max_read = SHA256_BLOCK_SIZE; + } + + list_add(&block->list, &rctx->blocks); + to_consume -= read; + consumed += read; + } + + /* Write the remaining data to req data */ + read = sg_pcopy_to_buffer(req->src, sg_nents(req->src), + rctx->data + offset, to_consume, + consumed); + rctx->data_used = offset + read; + + /* Update counter with processed bytes */ + rctx->len += read + consumed; + + /* Consume all the block added to list */ + list_for_each_entry_reverse(block, &rctx->blocks, list) { + wait_req = complete_req && + list_is_first(&block->list, &rctx->blocks); + + ret = eip93_send_hash_req(async, block->data, + &block->data_dma, + SHA256_BLOCK_SIZE, wait_req); + if (ret) + goto free_blocks; + } + + return wait_req ? -EINPROGRESS : 0; + +free_blocks: + eip93_hash_free_data_blocks(req); + + return ret; +} + +static int eip93_hash_update(struct ahash_request *req) +{ + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct eip93_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct sa_record *sa_record = &rctx->sa_record; + struct sa_state *sa_state = &rctx->sa_state; + struct eip93_device *eip93 = ctx->eip93; + int ret; + + if (!req->nbytes) + return 0; + + rctx->sa_state_base = dma_map_single(eip93->dev, sa_state, + sizeof(*sa_state), + DMA_TO_DEVICE); + ret = dma_mapping_error(eip93->dev, rctx->sa_state_base); + if (ret) + return ret; + + rctx->sa_record_base = dma_map_single(eip93->dev, sa_record, + sizeof(*sa_record), + DMA_TO_DEVICE); + ret = dma_mapping_error(eip93->dev, rctx->sa_record_base); + if (ret) + goto free_sa_state; + + ret = __eip93_hash_update(req, true); + if (ret && ret != -EINPROGRESS) + goto free_sa_record; + + return ret; + +free_sa_record: + dma_unmap_single(eip93->dev, rctx->sa_record_base, + sizeof(*sa_record), DMA_TO_DEVICE); + +free_sa_state: + dma_unmap_single(eip93->dev, rctx->sa_state_base, + sizeof(*sa_state), DMA_TO_DEVICE); + + return ret; +} + +/* + * With map_data true, we map the sa_record and sa_state. This is needed + * for finup() as the they are mapped before calling update() + */ +static int __eip93_hash_final(struct ahash_request *req, bool map_dma) +{ + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct eip93_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct crypto_async_request *async = &req->base; + struct sa_record *sa_record = &rctx->sa_record; + struct sa_state *sa_state = &rctx->sa_state; + struct eip93_device *eip93 = ctx->eip93; + int ret; + + /* EIP93 can't handle zero bytes hash */ + if (!rctx->len && !IS_HMAC(ctx->flags)) { + switch ((ctx->flags & EIP93_HASH_MASK)) { + case EIP93_HASH_SHA256: + memcpy(req->result, sha256_zero_message_hash, + SHA256_DIGEST_SIZE); + break; + case EIP93_HASH_SHA224: + memcpy(req->result, sha224_zero_message_hash, + SHA224_DIGEST_SIZE); + break; + case EIP93_HASH_SHA1: + memcpy(req->result, sha1_zero_message_hash, + SHA1_DIGEST_SIZE); + break; + case EIP93_HASH_MD5: + memcpy(req->result, md5_zero_message_hash, + MD5_DIGEST_SIZE); + break; + default: /* Impossible */ + return -EINVAL; + } + + return 0; + } + + /* Signal interrupt from engine is for last block */ + rctx->finalize = true; + + if (map_dma) { + rctx->sa_state_base = dma_map_single(eip93->dev, sa_state, + sizeof(*sa_state), + DMA_TO_DEVICE); + ret = dma_mapping_error(eip93->dev, rctx->sa_state_base); + if (ret) + return ret; + + rctx->sa_record_base = dma_map_single(eip93->dev, sa_record, + sizeof(*sa_record), + DMA_TO_DEVICE); + ret = dma_mapping_error(eip93->dev, rctx->sa_record_base); + if (ret) + goto free_sa_state; + } + + /* Send last block */ + ret = eip93_send_hash_req(async, rctx->data, &rctx->data_dma, + rctx->data_used, true); + if (ret) + goto free_blocks; + + return -EINPROGRESS; + +free_blocks: + eip93_hash_free_data_blocks(req); + + dma_unmap_single(eip93->dev, rctx->sa_record_base, + sizeof(*sa_record), DMA_TO_DEVICE); + +free_sa_state: + dma_unmap_single(eip93->dev, rctx->sa_state_base, + sizeof(*sa_state), DMA_TO_DEVICE); + + return ret; +} + +static int eip93_hash_final(struct ahash_request *req) +{ + return __eip93_hash_final(req, true); +} + +static int eip93_hash_finup(struct ahash_request *req) +{ + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct eip93_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct sa_record *sa_record = &rctx->sa_record; + struct sa_state *sa_state = &rctx->sa_state; + struct eip93_device *eip93 = ctx->eip93; + int ret; + + if (rctx->len + req->nbytes || IS_HMAC(ctx->flags)) { + rctx->sa_state_base = dma_map_single(eip93->dev, sa_state, + sizeof(*sa_state), + DMA_TO_DEVICE); + ret = dma_mapping_error(eip93->dev, rctx->sa_state_base); + if (ret) + return ret; + + rctx->sa_record_base = dma_map_single(eip93->dev, sa_record, + sizeof(*sa_record), + DMA_TO_DEVICE); + ret = dma_mapping_error(eip93->dev, rctx->sa_record_base); + if (ret) + goto free_sa_state; + + ret = __eip93_hash_update(req, false); + if (ret) + goto free_sa_record; + } + + return __eip93_hash_final(req, false); + +free_sa_record: + dma_unmap_single(eip93->dev, rctx->sa_record_base, + sizeof(*sa_record), DMA_TO_DEVICE); +free_sa_state: + dma_unmap_single(eip93->dev, rctx->sa_state_base, + sizeof(*sa_state), DMA_TO_DEVICE); + + return ret; +} + +static int eip93_hash_hmac_setkey(struct crypto_ahash *ahash, const u8 *key, + u32 keylen) +{ + unsigned int digestsize = crypto_ahash_digestsize(ahash); + struct crypto_tfm *tfm = crypto_ahash_tfm(ahash); + struct eip93_hash_ctx *ctx = crypto_tfm_ctx(tfm); + + return eip93_hmac_setkey(ctx->flags, key, keylen, digestsize, + ctx->ipad, ctx->opad, true); +} + +static int eip93_hash_cra_init(struct crypto_tfm *tfm) +{ + struct eip93_hash_ctx *ctx = crypto_tfm_ctx(tfm); + struct eip93_alg_template *tmpl = container_of(tfm->__crt_alg, + struct eip93_alg_template, alg.ahash.halg.base); + + crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm), + sizeof(struct eip93_hash_reqctx)); + + ctx->eip93 = tmpl->eip93; + ctx->flags = tmpl->flags; + + return 0; +} + +static int eip93_hash_digest(struct ahash_request *req) +{ + int ret; + + ret = eip93_hash_init(req); + if (ret) + return ret; + + return eip93_hash_finup(req); +} + +static int eip93_hash_import(struct ahash_request *req, const void *in) +{ + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + const struct eip93_hash_export_state *state = in; + struct sa_state *sa_state = &rctx->sa_state; + + memcpy(sa_state->state_byte_cnt, state->state_len, sizeof(u32) * 2); + memcpy(sa_state->state_i_digest, state->state_hash, SHA256_DIGEST_SIZE); + + __eip93_hash_init(req); + + rctx->len = state->len; + rctx->data_used = state->data_used; + + /* Skip copying data if we have nothing to copy */ + if (rctx->len) + memcpy(rctx->data, state->data, rctx->data_used); + + return 0; +} + +static int eip93_hash_export(struct ahash_request *req, void *out) +{ + struct eip93_hash_reqctx *rctx = ahash_request_ctx_dma(req); + struct eip93_hash_export_state *state = out; + + /* Save the first block in state data */ + if (rctx->len) + memcpy(state->data, rctx->data, rctx->data_used); + + eip93_hash_export_sa_state(req, state); + + return 0; +} + +struct eip93_alg_template eip93_alg_md5 = { + .type = EIP93_ALG_TYPE_HASH, + .flags = EIP93_HASH_MD5, + .alg.ahash = { + .init = eip93_hash_init, + .update = eip93_hash_update, + .final = eip93_hash_final, + .finup = eip93_hash_finup, + .digest = eip93_hash_digest, + .export = eip93_hash_export, + .import = eip93_hash_import, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct eip93_hash_export_state), + .base = { + .cra_name = "md5", + .cra_driver_name = "md5-eip93", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_hash_ctx), + .cra_init = eip93_hash_cra_init, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +struct eip93_alg_template eip93_alg_sha1 = { + .type = EIP93_ALG_TYPE_HASH, + .flags = EIP93_HASH_SHA1, + .alg.ahash = { + .init = eip93_hash_init, + .update = eip93_hash_update, + .final = eip93_hash_final, + .finup = eip93_hash_finup, + .digest = eip93_hash_digest, + .export = eip93_hash_export, + .import = eip93_hash_import, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct eip93_hash_export_state), + .base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-eip93", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_hash_ctx), + .cra_init = eip93_hash_cra_init, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +struct eip93_alg_template eip93_alg_sha224 = { + .type = EIP93_ALG_TYPE_HASH, + .flags = EIP93_HASH_SHA224, + .alg.ahash = { + .init = eip93_hash_init, + .update = eip93_hash_update, + .final = eip93_hash_final, + .finup = eip93_hash_finup, + .digest = eip93_hash_digest, + .export = eip93_hash_export, + .import = eip93_hash_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct eip93_hash_export_state), + .base = { + .cra_name = "sha224", + .cra_driver_name = "sha224-eip93", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_hash_ctx), + .cra_init = eip93_hash_cra_init, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +struct eip93_alg_template eip93_alg_sha256 = { + .type = EIP93_ALG_TYPE_HASH, + .flags = EIP93_HASH_SHA256, + .alg.ahash = { + .init = eip93_hash_init, + .update = eip93_hash_update, + .final = eip93_hash_final, + .finup = eip93_hash_finup, + .digest = eip93_hash_digest, + .export = eip93_hash_export, + .import = eip93_hash_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct eip93_hash_export_state), + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-eip93", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_hash_ctx), + .cra_init = eip93_hash_cra_init, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +struct eip93_alg_template eip93_alg_hmac_md5 = { + .type = EIP93_ALG_TYPE_HASH, + .flags = EIP93_HASH_HMAC | EIP93_HASH_MD5, + .alg.ahash = { + .init = eip93_hash_init, + .update = eip93_hash_update, + .final = eip93_hash_final, + .finup = eip93_hash_finup, + .digest = eip93_hash_digest, + .setkey = eip93_hash_hmac_setkey, + .export = eip93_hash_export, + .import = eip93_hash_import, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct eip93_hash_export_state), + .base = { + .cra_name = "hmac(md5)", + .cra_driver_name = "hmac(md5-eip93)", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_hash_ctx), + .cra_init = eip93_hash_cra_init, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +struct eip93_alg_template eip93_alg_hmac_sha1 = { + .type = EIP93_ALG_TYPE_HASH, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA1, + .alg.ahash = { + .init = eip93_hash_init, + .update = eip93_hash_update, + .final = eip93_hash_final, + .finup = eip93_hash_finup, + .digest = eip93_hash_digest, + .setkey = eip93_hash_hmac_setkey, + .export = eip93_hash_export, + .import = eip93_hash_import, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct eip93_hash_export_state), + .base = { + .cra_name = "hmac(sha1)", + .cra_driver_name = "hmac(sha1-eip93)", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_hash_ctx), + .cra_init = eip93_hash_cra_init, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +struct eip93_alg_template eip93_alg_hmac_sha224 = { + .type = EIP93_ALG_TYPE_HASH, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA224, + .alg.ahash = { + .init = eip93_hash_init, + .update = eip93_hash_update, + .final = eip93_hash_final, + .finup = eip93_hash_finup, + .digest = eip93_hash_digest, + .setkey = eip93_hash_hmac_setkey, + .export = eip93_hash_export, + .import = eip93_hash_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct eip93_hash_export_state), + .base = { + .cra_name = "hmac(sha224)", + .cra_driver_name = "hmac(sha224-eip93)", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_hash_ctx), + .cra_init = eip93_hash_cra_init, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +struct eip93_alg_template eip93_alg_hmac_sha256 = { + .type = EIP93_ALG_TYPE_HASH, + .flags = EIP93_HASH_HMAC | EIP93_HASH_SHA256, + .alg.ahash = { + .init = eip93_hash_init, + .update = eip93_hash_update, + .final = eip93_hash_final, + .finup = eip93_hash_finup, + .digest = eip93_hash_digest, + .setkey = eip93_hash_hmac_setkey, + .export = eip93_hash_export, + .import = eip93_hash_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct eip93_hash_export_state), + .base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "hmac(sha256-eip93)", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct eip93_hash_ctx), + .cra_init = eip93_hash_cra_init, + .cra_module = THIS_MODULE, + }, + }, + }, +}; diff --git a/drivers/crypto/inside-secure/eip93/eip93-hash.h b/drivers/crypto/inside-secure/eip93/eip93-hash.h new file mode 100644 index 000000000..29da18d78 --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-hash.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2019 - 2021 + * + * Richard van Schagen + * Christian Marangi + */ +#ifndef _EIP93_HASH_H_ +#define _EIP93_HASH_H_ + +#include + +#include "eip93-main.h" +#include "eip93-regs.h" + +struct eip93_hash_ctx { + struct eip93_device *eip93; + u32 flags; + + u8 ipad[SHA256_BLOCK_SIZE] __aligned(sizeof(u32)); + u8 opad[SHA256_DIGEST_SIZE] __aligned(sizeof(u32)); +}; + +struct eip93_hash_reqctx { + /* Placement is important for DMA align */ + struct { + struct sa_record sa_record; + struct sa_record sa_record_hmac; + struct sa_state sa_state; + } __aligned(CRYPTO_DMA_ALIGN); + + dma_addr_t sa_record_base; + dma_addr_t sa_record_hmac_base; + dma_addr_t sa_state_base; + + /* Don't enable HASH_FINALIZE when last block is sent */ + bool partial_hash; + + /* Set to signal interrupt is for final packet */ + bool finalize; + + /* + * EIP93 requires data to be accumulated in block of 64 bytes + * for intermediate hash calculation. + */ + u64 len; + u32 data_used; + + u8 data[SHA256_BLOCK_SIZE] __aligned(sizeof(u32)); + dma_addr_t data_dma; + + struct list_head blocks; +}; + +struct mkt_hash_block { + struct list_head list; + u8 data[SHA256_BLOCK_SIZE] __aligned(sizeof(u32)); + dma_addr_t data_dma; +}; + +struct eip93_hash_export_state { + u64 len; + u32 data_used; + + u32 state_len[2]; + u8 state_hash[SHA256_DIGEST_SIZE] __aligned(sizeof(u32)); + + u8 data[SHA256_BLOCK_SIZE] __aligned(sizeof(u32)); +}; + +void eip93_hash_handle_result(struct crypto_async_request *async, int err); + +extern struct eip93_alg_template eip93_alg_md5; +extern struct eip93_alg_template eip93_alg_sha1; +extern struct eip93_alg_template eip93_alg_sha224; +extern struct eip93_alg_template eip93_alg_sha256; +extern struct eip93_alg_template eip93_alg_hmac_md5; +extern struct eip93_alg_template eip93_alg_hmac_sha1; +extern struct eip93_alg_template eip93_alg_hmac_sha224; +extern struct eip93_alg_template eip93_alg_hmac_sha256; + +#endif /* _EIP93_HASH_H_ */ diff --git a/drivers/crypto/inside-secure/eip93/eip93-main.c b/drivers/crypto/inside-secure/eip93/eip93-main.c new file mode 100644 index 000000000..e62785952 --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-main.c @@ -0,0 +1,512 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2019 - 2021 + * + * Richard van Schagen + * Christian Marangi + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "eip93-main.h" +#include "eip93-regs.h" +#include "eip93-common.h" +#include "eip93-cipher.h" +#include "eip93-aes.h" +#include "eip93-des.h" +#include "eip93-aead.h" +#include "eip93-hash.h" + +static struct eip93_alg_template *eip93_algs[] = { + &eip93_alg_ecb_des, + &eip93_alg_cbc_des, + &eip93_alg_ecb_des3_ede, + &eip93_alg_cbc_des3_ede, + &eip93_alg_ecb_aes, + &eip93_alg_cbc_aes, + &eip93_alg_ctr_aes, + &eip93_alg_rfc3686_aes, + &eip93_alg_md5, + &eip93_alg_sha1, + &eip93_alg_sha224, + &eip93_alg_sha256, + &eip93_alg_hmac_md5, + &eip93_alg_hmac_sha1, + &eip93_alg_hmac_sha224, + &eip93_alg_hmac_sha256, + &eip93_alg_authenc_hmac_md5_cbc_des, + &eip93_alg_authenc_hmac_sha1_cbc_des, + &eip93_alg_authenc_hmac_sha224_cbc_des, + &eip93_alg_authenc_hmac_sha256_cbc_des, + &eip93_alg_authenc_hmac_md5_cbc_des3_ede, + &eip93_alg_authenc_hmac_sha1_cbc_des3_ede, + &eip93_alg_authenc_hmac_sha224_cbc_des3_ede, + &eip93_alg_authenc_hmac_sha256_cbc_des3_ede, + &eip93_alg_authenc_hmac_md5_cbc_aes, + &eip93_alg_authenc_hmac_sha1_cbc_aes, + &eip93_alg_authenc_hmac_sha224_cbc_aes, + &eip93_alg_authenc_hmac_sha256_cbc_aes, + &eip93_alg_authenc_hmac_md5_rfc3686_aes, + &eip93_alg_authenc_hmac_sha1_rfc3686_aes, + &eip93_alg_authenc_hmac_sha224_rfc3686_aes, + &eip93_alg_authenc_hmac_sha256_rfc3686_aes, +}; + +inline void eip93_irq_disable(struct eip93_device *eip93, u32 mask) +{ + __raw_writel(mask, eip93->base + EIP93_REG_MASK_DISABLE); +} + +inline void eip93_irq_enable(struct eip93_device *eip93, u32 mask) +{ + __raw_writel(mask, eip93->base + EIP93_REG_MASK_ENABLE); +} + +inline void eip93_irq_clear(struct eip93_device *eip93, u32 mask) +{ + __raw_writel(mask, eip93->base + EIP93_REG_INT_CLR); +} + +static int eip93_algo_is_supported(u32 alg_flags, u32 supported_algo_flags) +{ + if ((IS_DES(alg_flags) || IS_3DES(alg_flags)) && + !(supported_algo_flags & EIP93_PE_OPTION_TDES)) + return 0; + + if (IS_AES(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_AES)) + return 0; + + if (IS_HASH_MD5(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_MD5)) + return 0; + + if (IS_HASH_SHA1(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_SHA_1)) + return 0; + + if (IS_HASH_SHA224(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_SHA_224)) + return 0; + + if (IS_HASH_SHA256(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_SHA_256)) + return 0; + + return 1; +} + +static void eip93_unregister_algs(u32 supported_algo_flags, unsigned int i) +{ + unsigned int j; + + for (j = 0; j < i; j++) { + if (!eip93_algo_is_supported(eip93_algs[j]->flags, + supported_algo_flags)) + continue; + + switch (eip93_algs[j]->type) { + case EIP93_ALG_TYPE_SKCIPHER: + crypto_unregister_skcipher(&eip93_algs[j]->alg.skcipher); + break; + case EIP93_ALG_TYPE_AEAD: + crypto_unregister_aead(&eip93_algs[j]->alg.aead); + break; + case EIP93_ALG_TYPE_HASH: + crypto_unregister_ahash(&eip93_algs[j]->alg.ahash); + break; + } + } +} + +static int eip93_register_algs(struct eip93_device *eip93, u32 supported_algo_flags) +{ + unsigned int i; + int ret = 0; + + for (i = 0; i < ARRAY_SIZE(eip93_algs); i++) { + u32 alg_flags = eip93_algs[i]->flags; + + eip93_algs[i]->eip93 = eip93; + + if (!eip93_algo_is_supported(alg_flags, supported_algo_flags)) + continue; + + if (IS_AES(alg_flags) && !IS_HMAC(alg_flags)) { + if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY128) + eip93_algs[i]->alg.skcipher.max_keysize = + AES_KEYSIZE_128; + + if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY192) + eip93_algs[i]->alg.skcipher.max_keysize = + AES_KEYSIZE_192; + + if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY256) + eip93_algs[i]->alg.skcipher.max_keysize = + AES_KEYSIZE_256; + + if (IS_RFC3686(alg_flags)) + eip93_algs[i]->alg.skcipher.max_keysize += + CTR_RFC3686_NONCE_SIZE; + } + + switch (eip93_algs[i]->type) { + case EIP93_ALG_TYPE_SKCIPHER: + ret = crypto_register_skcipher(&eip93_algs[i]->alg.skcipher); + break; + case EIP93_ALG_TYPE_AEAD: + ret = crypto_register_aead(&eip93_algs[i]->alg.aead); + break; + case EIP93_ALG_TYPE_HASH: + ret = crypto_register_ahash(&eip93_algs[i]->alg.ahash); + break; + } + if (ret) + goto fail; + } + + return 0; + +fail: + eip93_unregister_algs(supported_algo_flags, i); + + return ret; +} + +static void eip93_handle_result_descriptor(struct eip93_device *eip93) +{ + struct crypto_async_request *async; + struct eip93_descriptor *rdesc; + u16 desc_flags, crypto_idr; + bool last_entry; + int handled, left, err; + u32 pe_ctrl_stat; + u32 pe_length; + +get_more: + handled = 0; + + left = readl(eip93->base + EIP93_REG_PE_RD_COUNT) & EIP93_PE_RD_COUNT; + + if (!left) { + eip93_irq_clear(eip93, EIP93_INT_RDR_THRESH); + eip93_irq_enable(eip93, EIP93_INT_RDR_THRESH); + return; + } + + last_entry = false; + + while (left) { + scoped_guard(spinlock_irqsave, &eip93->ring->read_lock) + rdesc = eip93_get_descriptor(eip93); + if (IS_ERR(rdesc)) { + dev_err(eip93->dev, "Ndesc: %d nreq: %d\n", + handled, left); + err = -EIO; + break; + } + /* make sure DMA is finished writing */ + do { + pe_ctrl_stat = READ_ONCE(rdesc->pe_ctrl_stat_word); + pe_length = READ_ONCE(rdesc->pe_length_word); + } while (FIELD_GET(EIP93_PE_CTRL_PE_READY_DES_TRING_OWN, pe_ctrl_stat) != + EIP93_PE_CTRL_PE_READY || + FIELD_GET(EIP93_PE_LENGTH_HOST_PE_READY, pe_length) != + EIP93_PE_LENGTH_PE_READY); + + err = rdesc->pe_ctrl_stat_word & (EIP93_PE_CTRL_PE_EXT_ERR_CODE | + EIP93_PE_CTRL_PE_EXT_ERR | + EIP93_PE_CTRL_PE_SEQNUM_ERR | + EIP93_PE_CTRL_PE_PAD_ERR | + EIP93_PE_CTRL_PE_AUTH_ERR); + + desc_flags = FIELD_GET(EIP93_PE_USER_ID_DESC_FLAGS, rdesc->user_id); + crypto_idr = FIELD_GET(EIP93_PE_USER_ID_CRYPTO_IDR, rdesc->user_id); + + writel(1, eip93->base + EIP93_REG_PE_RD_COUNT); + eip93_irq_clear(eip93, EIP93_INT_RDR_THRESH); + + handled++; + left--; + + if (desc_flags & EIP93_DESC_LAST) { + last_entry = true; + break; + } + } + + if (!last_entry) + goto get_more; + + /* Get crypto async ref only for last descriptor */ + scoped_guard(spinlock_bh, &eip93->ring->idr_lock) { + async = idr_find(&eip93->ring->crypto_async_idr, crypto_idr); + idr_remove(&eip93->ring->crypto_async_idr, crypto_idr); + } + + /* Parse error in ctrl stat word */ + err = eip93_parse_ctrl_stat_err(eip93, err); + + if (desc_flags & EIP93_DESC_SKCIPHER) + eip93_skcipher_handle_result(async, err); + + if (desc_flags & EIP93_DESC_AEAD) + eip93_aead_handle_result(async, err); + + if (desc_flags & EIP93_DESC_HASH) + eip93_hash_handle_result(async, err); + + goto get_more; +} + +static void eip93_done_task(unsigned long data) +{ + struct eip93_device *eip93 = (struct eip93_device *)data; + + eip93_handle_result_descriptor(eip93); +} + +static irqreturn_t eip93_irq_handler(int irq, void *data) +{ + struct eip93_device *eip93 = data; + u32 irq_status; + + irq_status = readl(eip93->base + EIP93_REG_INT_MASK_STAT); + if (FIELD_GET(EIP93_INT_RDR_THRESH, irq_status)) { + eip93_irq_disable(eip93, EIP93_INT_RDR_THRESH); + tasklet_schedule(&eip93->ring->done_task); + return IRQ_HANDLED; + } + + /* Ignore errors in AUTO mode, handled by the RDR */ + eip93_irq_clear(eip93, irq_status); + if (irq_status) + eip93_irq_disable(eip93, irq_status); + + return IRQ_NONE; +} + +static void eip93_initialize(struct eip93_device *eip93, u32 supported_algo_flags) +{ + u32 val; + + /* Reset PE and rings */ + val = EIP93_PE_CONFIG_RST_PE | EIP93_PE_CONFIG_RST_RING; + val |= EIP93_PE_TARGET_AUTO_RING_MODE; + /* For Auto more, update the CDR ring owner after processing */ + val |= EIP93_PE_CONFIG_EN_CDR_UPDATE; + writel(val, eip93->base + EIP93_REG_PE_CONFIG); + + /* Wait for PE and ring to reset */ + usleep_range(10, 20); + + /* Release PE and ring reset */ + val = readl(eip93->base + EIP93_REG_PE_CONFIG); + val &= ~(EIP93_PE_CONFIG_RST_PE | EIP93_PE_CONFIG_RST_RING); + writel(val, eip93->base + EIP93_REG_PE_CONFIG); + + /* Config Clocks */ + val = EIP93_PE_CLOCK_EN_PE_CLK; + if (supported_algo_flags & EIP93_PE_OPTION_TDES) + val |= EIP93_PE_CLOCK_EN_DES_CLK; + if (supported_algo_flags & EIP93_PE_OPTION_AES) + val |= EIP93_PE_CLOCK_EN_AES_CLK; + if (supported_algo_flags & + (EIP93_PE_OPTION_MD5 | EIP93_PE_OPTION_SHA_1 | EIP93_PE_OPTION_SHA_224 | + EIP93_PE_OPTION_SHA_256)) + val |= EIP93_PE_CLOCK_EN_HASH_CLK; + writel(val, eip93->base + EIP93_REG_PE_CLOCK_CTRL); + + /* Config DMA thresholds */ + val = FIELD_PREP(EIP93_PE_OUTBUF_THRESH, 128) | + FIELD_PREP(EIP93_PE_INBUF_THRESH, 128); + writel(val, eip93->base + EIP93_REG_PE_BUF_THRESH); + + /* Clear/ack all interrupts before disable all */ + eip93_irq_clear(eip93, EIP93_INT_ALL); + eip93_irq_disable(eip93, EIP93_INT_ALL); + + /* Setup CRD threshold to trigger interrupt */ + val = FIELD_PREP(EIPR93_PE_CDR_THRESH, EIP93_RING_NUM - EIP93_RING_BUSY); + /* + * Configure RDR interrupt to be triggered if RD counter is not 0 + * for more than 2^(N+10) system clocks. + */ + val |= FIELD_PREP(EIPR93_PE_RD_TIMEOUT, 5) | EIPR93_PE_TIMEROUT_EN; + writel(val, eip93->base + EIP93_REG_PE_RING_THRESH); +} + +static void eip93_desc_free(struct eip93_device *eip93) +{ + writel(0, eip93->base + EIP93_REG_PE_RING_CONFIG); + writel(0, eip93->base + EIP93_REG_PE_CDR_BASE); + writel(0, eip93->base + EIP93_REG_PE_RDR_BASE); +} + +static int eip93_set_ring(struct eip93_device *eip93, struct eip93_desc_ring *ring) +{ + ring->offset = sizeof(struct eip93_descriptor); + ring->base = dmam_alloc_coherent(eip93->dev, + sizeof(struct eip93_descriptor) * EIP93_RING_NUM, + &ring->base_dma, GFP_KERNEL); + if (!ring->base) + return -ENOMEM; + + ring->write = ring->base; + ring->base_end = ring->base + sizeof(struct eip93_descriptor) * (EIP93_RING_NUM - 1); + ring->read = ring->base; + + return 0; +} + +static int eip93_desc_init(struct eip93_device *eip93) +{ + struct eip93_desc_ring *cdr = &eip93->ring->cdr; + struct eip93_desc_ring *rdr = &eip93->ring->rdr; + int ret; + u32 val; + + ret = eip93_set_ring(eip93, cdr); + if (ret) + return ret; + + ret = eip93_set_ring(eip93, rdr); + if (ret) + return ret; + + writel((u32 __force)cdr->base_dma, eip93->base + EIP93_REG_PE_CDR_BASE); + writel((u32 __force)rdr->base_dma, eip93->base + EIP93_REG_PE_RDR_BASE); + + val = FIELD_PREP(EIP93_PE_RING_SIZE, EIP93_RING_NUM - 1); + writel(val, eip93->base + EIP93_REG_PE_RING_CONFIG); + + return 0; +} + +static void eip93_cleanup(struct eip93_device *eip93) +{ + tasklet_kill(&eip93->ring->done_task); + + /* Clear/ack all interrupts before disable all */ + eip93_irq_clear(eip93, EIP93_INT_ALL); + eip93_irq_disable(eip93, EIP93_INT_ALL); + + writel(0, eip93->base + EIP93_REG_PE_CLOCK_CTRL); + + eip93_desc_free(eip93); + + idr_destroy(&eip93->ring->crypto_async_idr); +} + +static int eip93_crypto_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct eip93_device *eip93; + u32 ver, algo_flags; + int ret; + + eip93 = devm_kzalloc(dev, struct_size(eip93, ring, 1), GFP_KERNEL); + if (!eip93) + return -ENOMEM; + + eip93->dev = dev; + platform_set_drvdata(pdev, eip93); + + eip93->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(eip93->base)) + return PTR_ERR(eip93->base); + + eip93->irq = platform_get_irq(pdev, 0); + if (eip93->irq < 0) + return eip93->irq; + + ret = devm_request_threaded_irq(eip93->dev, eip93->irq, eip93_irq_handler, + NULL, IRQF_ONESHOT, + dev_name(eip93->dev), eip93); + if (ret) + return ret; + + ret = eip93_desc_init(eip93); + if (ret) + return ret; + + tasklet_init(&eip93->ring->done_task, eip93_done_task, (unsigned long)eip93); + + spin_lock_init(&eip93->ring->read_lock); + spin_lock_init(&eip93->ring->write_lock); + + spin_lock_init(&eip93->ring->idr_lock); + idr_init(&eip93->ring->crypto_async_idr); + + algo_flags = readl(eip93->base + EIP93_REG_PE_OPTION_1); + + eip93_initialize(eip93, algo_flags); + + /* Init finished, enable RDR interrupt */ + eip93_irq_enable(eip93, EIP93_INT_RDR_THRESH); + + ret = eip93_register_algs(eip93, algo_flags); + if (ret) { + eip93_cleanup(eip93); + return ret; + } + + ver = readl(eip93->base + EIP93_REG_PE_REVISION); + /* EIP_EIP_NO:MAJOR_HW_REV:MINOR_HW_REV:HW_PATCH,PE(ALGO_FLAGS) */ + dev_info(eip93->dev, "EIP%lu:%lx:%lx:%lx,PE(0x%x:0x%x)\n", + FIELD_GET(EIP93_PE_REVISION_EIP_NO, ver), + FIELD_GET(EIP93_PE_REVISION_MAJ_HW_REV, ver), + FIELD_GET(EIP93_PE_REVISION_MIN_HW_REV, ver), + FIELD_GET(EIP93_PE_REVISION_HW_PATCH, ver), + algo_flags, + readl(eip93->base + EIP93_REG_PE_OPTION_0)); + + return 0; +} + +static void eip93_crypto_remove(struct platform_device *pdev) +{ + struct eip93_device *eip93 = platform_get_drvdata(pdev); + u32 algo_flags; + + algo_flags = readl(eip93->base + EIP93_REG_PE_OPTION_1); + + eip93_unregister_algs(algo_flags, ARRAY_SIZE(eip93_algs)); + eip93_cleanup(eip93); +} + +static const struct of_device_id eip93_crypto_of_match[] = { + { .compatible = "inside-secure,safexcel-eip93i", }, + { .compatible = "inside-secure,safexcel-eip93ie", }, + { .compatible = "inside-secure,safexcel-eip93is", }, + { .compatible = "inside-secure,safexcel-eip93ies", }, + /* IW not supported currently, missing AES-XCB-MAC/AES-CCM */ + /* { .compatible = "inside-secure,safexcel-eip93iw", }, */ + {} +}; +MODULE_DEVICE_TABLE(of, eip93_crypto_of_match); + +static struct platform_driver eip93_crypto_driver = { + .probe = eip93_crypto_probe, + .remove = eip93_crypto_remove, + .driver = { + .name = "inside-secure-eip93", + .of_match_table = eip93_crypto_of_match, + }, +}; +module_platform_driver(eip93_crypto_driver); + +MODULE_AUTHOR("Richard van Schagen "); +MODULE_AUTHOR("Christian Marangi "); +MODULE_DESCRIPTION("Mediatek EIP-93 crypto engine driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/inside-secure/eip93/eip93-main.h b/drivers/crypto/inside-secure/eip93/eip93-main.h new file mode 100644 index 000000000..5f0f51081 --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-main.h @@ -0,0 +1,151 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2019 - 2021 + * + * Richard van Schagen + * Christian Marangi + */ +#ifndef _EIP93_MAIN_H_ +#define _EIP93_MAIN_H_ + +#include +#include +#include +#include +#include + +#define EIP93_RING_BUSY_DELAY 500 + +#define EIP93_RING_NUM 512 +#define EIP93_RING_BUSY 32 +#define EIP93_CRA_PRIORITY 1500 + +#define EIP93_RING_SA_STATE_ADDR(base, idx) ((base) + (idx)) +#define EIP93_RING_SA_STATE_DMA(dma_base, idx) ((u32 __force)(dma_base) + \ + ((idx) * sizeof(struct sa_state))) + +/* cipher algorithms */ +#define EIP93_ALG_DES BIT(0) +#define EIP93_ALG_3DES BIT(1) +#define EIP93_ALG_AES BIT(2) +#define EIP93_ALG_MASK GENMASK(2, 0) +/* hash and hmac algorithms */ +#define EIP93_HASH_MD5 BIT(3) +#define EIP93_HASH_SHA1 BIT(4) +#define EIP93_HASH_SHA224 BIT(5) +#define EIP93_HASH_SHA256 BIT(6) +#define EIP93_HASH_HMAC BIT(7) +#define EIP93_HASH_MASK GENMASK(6, 3) +/* cipher modes */ +#define EIP93_MODE_CBC BIT(8) +#define EIP93_MODE_ECB BIT(9) +#define EIP93_MODE_CTR BIT(10) +#define EIP93_MODE_RFC3686 BIT(11) +#define EIP93_MODE_MASK GENMASK(10, 8) + +/* cipher encryption/decryption operations */ +#define EIP93_ENCRYPT BIT(12) +#define EIP93_DECRYPT BIT(13) + +#define EIP93_BUSY BIT(14) + +/* descriptor flags */ +#define EIP93_DESC_DMA_IV BIT(0) +#define EIP93_DESC_IPSEC BIT(1) +#define EIP93_DESC_FINISH BIT(2) +#define EIP93_DESC_LAST BIT(3) +#define EIP93_DESC_FAKE_HMAC BIT(4) +#define EIP93_DESC_PRNG BIT(5) +#define EIP93_DESC_HASH BIT(6) +#define EIP93_DESC_AEAD BIT(7) +#define EIP93_DESC_SKCIPHER BIT(8) +#define EIP93_DESC_ASYNC BIT(9) + +#define IS_DMA_IV(desc_flags) ((desc_flags) & EIP93_DESC_DMA_IV) + +#define IS_DES(flags) ((flags) & EIP93_ALG_DES) +#define IS_3DES(flags) ((flags) & EIP93_ALG_3DES) +#define IS_AES(flags) ((flags) & EIP93_ALG_AES) + +#define IS_HASH_MD5(flags) ((flags) & EIP93_HASH_MD5) +#define IS_HASH_SHA1(flags) ((flags) & EIP93_HASH_SHA1) +#define IS_HASH_SHA224(flags) ((flags) & EIP93_HASH_SHA224) +#define IS_HASH_SHA256(flags) ((flags) & EIP93_HASH_SHA256) +#define IS_HMAC(flags) ((flags) & EIP93_HASH_HMAC) + +#define IS_CBC(mode) ((mode) & EIP93_MODE_CBC) +#define IS_ECB(mode) ((mode) & EIP93_MODE_ECB) +#define IS_CTR(mode) ((mode) & EIP93_MODE_CTR) +#define IS_RFC3686(mode) ((mode) & EIP93_MODE_RFC3686) + +#define IS_BUSY(flags) ((flags) & EIP93_BUSY) + +#define IS_ENCRYPT(dir) ((dir) & EIP93_ENCRYPT) +#define IS_DECRYPT(dir) ((dir) & EIP93_DECRYPT) + +#define IS_CIPHER(flags) ((flags) & (EIP93_ALG_DES | \ + EIP93_ALG_3DES | \ + EIP93_ALG_AES)) + +#define IS_HASH(flags) ((flags) & (EIP93_HASH_MD5 | \ + EIP93_HASH_SHA1 | \ + EIP93_HASH_SHA224 | \ + EIP93_HASH_SHA256)) + +struct eip93_desc_ring { + void *base; + void *base_end; + dma_addr_t base_dma; + /* write and read pointers */ + void *read; + void *write; + /* descriptor element offset */ + u32 offset; +}; + +struct eip93_state_pool { + void *base; + dma_addr_t base_dma; +}; + +struct eip93_ring { + struct tasklet_struct done_task; + /* command/result rings */ + struct eip93_desc_ring cdr; + struct eip93_desc_ring rdr; + spinlock_t write_lock; + spinlock_t read_lock; + /* aync idr */ + spinlock_t idr_lock; + struct idr crypto_async_idr; +}; + +/** + * struct eip93_device - crypto engine device structure + */ +struct eip93_device { + void __iomem *base; + struct device *dev; + struct clk *clk; + int irq; + struct eip93_ring ring[]; +}; + +enum eip93_alg_type { + EIP93_ALG_TYPE_AEAD, + EIP93_ALG_TYPE_SKCIPHER, + EIP93_ALG_TYPE_HASH, +}; + +struct eip93_alg_template { + struct eip93_device *eip93; + enum eip93_alg_type type; + u32 flags; + union { + struct aead_alg aead; + struct skcipher_alg skcipher; + struct ahash_alg ahash; + } alg; +}; + +#endif /* _EIP93_MAIN_H_ */ diff --git a/drivers/crypto/inside-secure/eip93/eip93-regs.h b/drivers/crypto/inside-secure/eip93/eip93-regs.h new file mode 100644 index 000000000..96d28c665 --- /dev/null +++ b/drivers/crypto/inside-secure/eip93/eip93-regs.h @@ -0,0 +1,335 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2019 - 2021 + * + * Richard van Schagen + * Christian Marangi + */ +#ifndef REG_EIP93_H +#define REG_EIP93_H + +#define EIP93_REG_PE_CTRL_STAT 0x0 +#define EIP93_PE_CTRL_PE_PAD_CTRL_STAT GENMASK(31, 24) +#define EIP93_PE_CTRL_PE_EXT_ERR_CODE GENMASK(23, 20) +#define EIP93_PE_CTRL_PE_EXT_ERR_PROCESSING 0x8 +#define EIP93_PE_CTRL_PE_EXT_ERR_BLOCK_SIZE_ERR 0x7 +#define EIP93_PE_CTRL_PE_EXT_ERR_INVALID_PK_LENGTH 0x6 +#define EIP93_PE_CTRL_PE_EXT_ERR_ZERO_LENGTH 0x5 +#define EIP93_PE_CTRL_PE_EXT_ERR_SPI 0x4 +#define EIP93_PE_CTRL_PE_EXT_ERR_INVALID_CRYPTO_ALGO 0x3 +#define EIP93_PE_CTRL_PE_EXT_ERR_INVALID_CRYPTO_OP 0x2 +#define EIP93_PE_CTRL_PE_EXT_ERR_DESC_OWNER 0x1 +#define EIP93_PE_CTRL_PE_EXT_ERR_BUS 0x0 +#define EIP93_PE_CTRL_PE_EXT_ERR BIT(19) +#define EIP93_PE_CTRL_PE_SEQNUM_ERR BIT(18) +#define EIP93_PE_CTRL_PE_PAD_ERR BIT(17) +#define EIP93_PE_CTRL_PE_AUTH_ERR BIT(16) +#define EIP93_PE_CTRL_PE_PAD_VALUE GENMASK(15, 8) +#define EIP93_PE_CTRL_PE_PRNG_MODE GENMASK(7, 6) +#define EIP93_PE_CTRL_PE_HASH_FINAL BIT(4) +#define EIP93_PE_CTRL_PE_INIT_ARC4 BIT(3) +#define EIP93_PE_CTRL_PE_READY_DES_TRING_OWN GENMASK(1, 0) +#define EIP93_PE_CTRL_PE_READY 0x2 +#define EIP93_PE_CTRL_HOST_READY 0x1 +#define EIP93_REG_PE_SOURCE_ADDR 0x4 +#define EIP93_REG_PE_DEST_ADDR 0x8 +#define EIP93_REG_PE_SA_ADDR 0xc +#define EIP93_REG_PE_ADDR 0x10 /* STATE_ADDR */ +/* + * Special implementation for user ID + * user_id in eip93_descriptor is used to identify the + * descriptor and is opaque and can be used by the driver + * in custom way. + * + * The usage of this should be to put an address to the crypto + * request struct from the kernel but this can't work in 64bit + * world. + * + * Also it's required to put some flags to identify the last + * descriptor. + * + * To handle this, split the u32 in 2 part: + * - 31:16 descriptor flags + * - 15:0 IDR to connect the crypto request address + */ +#define EIP93_REG_PE_USER_ID 0x18 +#define EIP93_PE_USER_ID_DESC_FLAGS GENMASK(31, 16) +#define EIP93_PE_USER_ID_CRYPTO_IDR GENMASK(15, 0) +#define EIP93_REG_PE_LENGTH 0x1c +#define EIP93_PE_LENGTH_BYPASS GENMASK(31, 24) +#define EIP93_PE_LENGTH_HOST_PE_READY GENMASK(23, 22) +#define EIP93_PE_LENGTH_PE_READY 0x2 +#define EIP93_PE_LENGTH_HOST_READY 0x1 +#define EIP93_PE_LENGTH_LENGTH GENMASK(19, 0) + +/* PACKET ENGINE RING configuration registers */ +#define EIP93_REG_PE_CDR_BASE 0x80 +#define EIP93_REG_PE_RDR_BASE 0x84 +#define EIP93_REG_PE_RING_CONFIG 0x88 +#define EIP93_PE_EN_EXT_TRIG BIT(31) +/* Absent in later revision of eip93 */ +/* #define EIP93_PE_RING_OFFSET GENMASK(23, 15) */ +#define EIP93_PE_RING_SIZE GENMASK(9, 0) +#define EIP93_REG_PE_RING_THRESH 0x8c +#define EIPR93_PE_TIMEROUT_EN BIT(31) +#define EIPR93_PE_RD_TIMEOUT GENMASK(29, 26) +#define EIPR93_PE_RDR_THRESH GENMASK(25, 16) +#define EIPR93_PE_CDR_THRESH GENMASK(9, 0) +#define EIP93_REG_PE_CD_COUNT 0x90 +#define EIP93_PE_CD_COUNT GENMASK(10, 0) +/* + * In the same register, writing a value in GENMASK(7, 0) will + * increment the descriptor count and start DMA action. + */ +#define EIP93_PE_CD_COUNT_INCR GENMASK(7, 0) +#define EIP93_REG_PE_RD_COUNT 0x94 +#define EIP93_PE_RD_COUNT GENMASK(10, 0) +/* + * In the same register, writing a value in GENMASK(7, 0) will + * increment the descriptor count and start DMA action. + */ +#define EIP93_PE_RD_COUNT_INCR GENMASK(7, 0) +#define EIP93_REG_PE_RING_RW_PNTR 0x98 /* RING_PNTR */ + +/* PACKET ENGINE configuration registers */ +#define EIP93_REG_PE_CONFIG 0x100 +#define EIP93_PE_CONFIG_SWAP_TARGET BIT(20) +#define EIP93_PE_CONFIG_SWAP_DATA BIT(18) +#define EIP93_PE_CONFIG_SWAP_SA BIT(17) +#define EIP93_PE_CONFIG_SWAP_CDRD BIT(16) +#define EIP93_PE_CONFIG_EN_CDR_UPDATE BIT(10) +#define EIP93_PE_CONFIG_PE_MODE GENMASK(9, 8) +#define EIP93_PE_TARGET_AUTO_RING_MODE FIELD_PREP(EIP93_PE_CONFIG_PE_MODE, 0x3) +#define EIP93_PE_TARGET_COMMAND_NO_RDR_MODE FIELD_PREP(EIP93_PE_CONFIG_PE_MODE, 0x2) +#define EIP93_PE_TARGET_COMMAND_WITH_RDR_MODE FIELD_PREP(EIP93_PE_CONFIG_PE_MODE, 0x1) +#define EIP93_PE_DIRECT_HOST_MODE FIELD_PREP(EIP93_PE_CONFIG_PE_MODE, 0x0) +#define EIP93_PE_CONFIG_RST_RING BIT(1) +#define EIP93_PE_CONFIG_RST_PE BIT(0) +#define EIP93_REG_PE_STATUS 0x104 +#define EIP93_REG_PE_BUF_THRESH 0x10c +#define EIP93_PE_OUTBUF_THRESH GENMASK(23, 16) +#define EIP93_PE_INBUF_THRESH GENMASK(7, 0) +#define EIP93_REG_PE_INBUF_COUNT 0x110 +#define EIP93_REG_PE_OUTBUF_COUNT 0x114 +#define EIP93_REG_PE_BUF_RW_PNTR 0x118 /* BUF_PNTR */ + +/* PACKET ENGINE endian config */ +#define EIP93_REG_PE_ENDIAN_CONFIG 0x1cc +#define EIP93_AIROHA_REG_PE_ENDIAN_CONFIG 0x1d0 +#define EIP93_PE_ENDIAN_TARGET_BYTE_SWAP GENMASK(23, 16) +#define EIP93_PE_ENDIAN_MASTER_BYTE_SWAP GENMASK(7, 0) +/* + * Byte goes 2 and 2 and are referenced by ID + * Split GENMASK(7, 0) in 4 part, one for each byte. + * Example LITTLE ENDIAN: Example BIG ENDIAN + * GENMASK(7, 6) 0x3 GENMASK(7, 6) 0x0 + * GENMASK(5, 4) 0x2 GENMASK(7, 6) 0x1 + * GENMASK(3, 2) 0x1 GENMASK(3, 2) 0x2 + * GENMASK(1, 0) 0x0 GENMASK(1, 0) 0x3 + */ +#define EIP93_PE_ENDIAN_BYTE0 0x0 +#define EIP93_PE_ENDIAN_BYTE1 0x1 +#define EIP93_PE_ENDIAN_BYTE2 0x2 +#define EIP93_PE_ENDIAN_BYTE3 0x3 + +/* EIP93 CLOCK control registers */ +#define EIP93_REG_PE_CLOCK_CTRL 0x1e8 +#define EIP93_PE_CLOCK_EN_HASH_CLK BIT(4) +#define EIP93_PE_CLOCK_EN_ARC4_CLK BIT(3) +#define EIP93_PE_CLOCK_EN_AES_CLK BIT(2) +#define EIP93_PE_CLOCK_EN_DES_CLK BIT(1) +#define EIP93_PE_CLOCK_EN_PE_CLK BIT(0) + +/* EIP93 Device Option and Revision Register */ +#define EIP93_REG_PE_OPTION_1 0x1f4 +#define EIP93_PE_OPTION_MAC_KEY256 BIT(31) +#define EIP93_PE_OPTION_MAC_KEY192 BIT(30) +#define EIP93_PE_OPTION_MAC_KEY128 BIT(29) +#define EIP93_PE_OPTION_AES_CBC_MAC BIT(28) +#define EIP93_PE_OPTION_AES_XCBX BIT(23) +#define EIP93_PE_OPTION_SHA_256 BIT(19) +#define EIP93_PE_OPTION_SHA_224 BIT(18) +#define EIP93_PE_OPTION_SHA_1 BIT(17) +#define EIP93_PE_OPTION_MD5 BIT(16) +#define EIP93_PE_OPTION_AES_KEY256 BIT(15) +#define EIP93_PE_OPTION_AES_KEY192 BIT(14) +#define EIP93_PE_OPTION_AES_KEY128 BIT(13) +#define EIP93_PE_OPTION_AES BIT(2) +#define EIP93_PE_OPTION_ARC4 BIT(1) +#define EIP93_PE_OPTION_TDES BIT(0) /* DES and TDES */ +#define EIP93_REG_PE_OPTION_0 0x1f8 +#define EIP93_REG_PE_REVISION 0x1fc +#define EIP93_PE_REVISION_MAJ_HW_REV GENMASK(27, 24) +#define EIP93_PE_REVISION_MIN_HW_REV GENMASK(23, 20) +#define EIP93_PE_REVISION_HW_PATCH GENMASK(19, 16) +#define EIP93_PE_REVISION_EIP_NO GENMASK(7, 0) + +/* EIP93 Interrupt Control Register */ +#define EIP93_REG_INT_UNMASK_STAT 0x200 +#define EIP93_REG_INT_MASK_STAT 0x204 +#define EIP93_REG_INT_CLR 0x204 +#define EIP93_REG_INT_MASK 0x208 /* INT_EN */ +/* Each int reg have the same bitmap */ +#define EIP93_INT_INTERFACE_ERR BIT(18) +#define EIP93_INT_RPOC_ERR BIT(17) +#define EIP93_INT_PE_RING_ERR BIT(16) +#define EIP93_INT_HALT BIT(15) +#define EIP93_INT_OUTBUF_THRESH BIT(11) +#define EIP93_INT_INBUF_THRESH BIT(10) +#define EIP93_INT_OPERATION_DONE BIT(9) +#define EIP93_INT_RDR_THRESH BIT(1) +#define EIP93_INT_CDR_THRESH BIT(0) +#define EIP93_INT_ALL (EIP93_INT_INTERFACE_ERR | \ + EIP93_INT_RPOC_ERR | \ + EIP93_INT_PE_RING_ERR | \ + EIP93_INT_HALT | \ + EIP93_INT_OUTBUF_THRESH | \ + EIP93_INT_INBUF_THRESH | \ + EIP93_INT_OPERATION_DONE | \ + EIP93_INT_RDR_THRESH | \ + EIP93_INT_CDR_THRESH) + +#define EIP93_REG_INT_CFG 0x20c +#define EIP93_INT_TYPE_PULSE BIT(0) +#define EIP93_REG_MASK_ENABLE 0x210 +#define EIP93_REG_MASK_DISABLE 0x214 + +/* EIP93 SA Record register */ +#define EIP93_REG_SA_CMD_0 0x400 +#define EIP93_SA_CMD_SAVE_HASH BIT(29) +#define EIP93_SA_CMD_SAVE_IV BIT(28) +#define EIP93_SA_CMD_HASH_SOURCE GENMASK(27, 26) +#define EIP93_SA_CMD_HASH_NO_LOAD FIELD_PREP(EIP93_SA_CMD_HASH_SOURCE, 0x3) +#define EIP93_SA_CMD_HASH_FROM_STATE FIELD_PREP(EIP93_SA_CMD_HASH_SOURCE, 0x2) +#define EIP93_SA_CMD_HASH_FROM_SA FIELD_PREP(EIP93_SA_CMD_HASH_SOURCE, 0x0) +#define EIP93_SA_CMD_IV_SOURCE GENMASK(25, 24) +#define EIP93_SA_CMD_IV_FROM_PRNG FIELD_PREP(EIP93_SA_CMD_IV_SOURCE, 0x3) +#define EIP93_SA_CMD_IV_FROM_STATE FIELD_PREP(EIP93_SA_CMD_IV_SOURCE, 0x2) +#define EIP93_SA_CMD_IV_FROM_INPUT FIELD_PREP(EIP93_SA_CMD_IV_SOURCE, 0x1) +#define EIP93_SA_CMD_IV_NO_LOAD FIELD_PREP(EIP93_SA_CMD_IV_SOURCE, 0x0) +#define EIP93_SA_CMD_DIGEST_LENGTH GENMASK(23, 20) +#define EIP93_SA_CMD_DIGEST_10WORD FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, 0xa) /* SRTP and TLS */ +#define EIP93_SA_CMD_DIGEST_8WORD FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, 0x8) /* SHA-256 */ +#define EIP93_SA_CMD_DIGEST_7WORD FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, 0x7) /* SHA-224 */ +#define EIP93_SA_CMD_DIGEST_6WORD FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, 0x6) +#define EIP93_SA_CMD_DIGEST_5WORD FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, 0x5) /* SHA1 */ +#define EIP93_SA_CMD_DIGEST_4WORD FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, 0x4) /* MD5 and AES-based */ +#define EIP93_SA_CMD_DIGEST_3WORD_IPSEC FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, 0x3) /* IPSEC */ +#define EIP93_SA_CMD_DIGEST_2WORD FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, 0x2) +#define EIP93_SA_CMD_DIGEST_1WORD FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, 0x1) +#define EIP93_SA_CMD_DIGEST_3WORD FIELD_PREP(EIP93_SA_CMD_DIGEST_LENGTH, 0x0) /* 96bit output */ +#define EIP93_SA_CMD_HDR_PROC BIT(19) +#define EIP93_SA_CMD_EXT_PAD BIT(18) +#define EIP93_SA_CMD_SCPAD BIT(17) +#define EIP93_SA_CMD_HASH GENMASK(15, 12) +#define EIP93_SA_CMD_HASH_NULL FIELD_PREP(EIP93_SA_CMD_HASH, 0xf) +#define EIP93_SA_CMD_HASH_SHA256 FIELD_PREP(EIP93_SA_CMD_HASH, 0x3) +#define EIP93_SA_CMD_HASH_SHA224 FIELD_PREP(EIP93_SA_CMD_HASH, 0x2) +#define EIP93_SA_CMD_HASH_SHA1 FIELD_PREP(EIP93_SA_CMD_HASH, 0x1) +#define EIP93_SA_CMD_HASH_MD5 FIELD_PREP(EIP93_SA_CMD_HASH, 0x0) +#define EIP93_SA_CMD_CIPHER GENMASK(11, 8) +#define EIP93_SA_CMD_CIPHER_NULL FIELD_PREP(EIP93_SA_CMD_CIPHER, 0xf) +#define EIP93_SA_CMD_CIPHER_AES FIELD_PREP(EIP93_SA_CMD_CIPHER, 0x3) +#define EIP93_SA_CMD_CIPHER_ARC4 FIELD_PREP(EIP93_SA_CMD_CIPHER, 0x2) +#define EIP93_SA_CMD_CIPHER_3DES FIELD_PREP(EIP93_SA_CMD_CIPHER, 0x1) +#define EIP93_SA_CMD_CIPHER_DES FIELD_PREP(EIP93_SA_CMD_CIPHER, 0x0) +#define EIP93_SA_CMD_PAD_TYPE GENMASK(7, 6) +#define EIP93_SA_CMD_PAD_CONST_SSL FIELD_PREP(EIP93_SA_CMD_PAD_TYPE, 0x6) +#define EIP93_SA_CMD_PAD_TLS_DTLS FIELD_PREP(EIP93_SA_CMD_PAD_TYPE, 0x5) +#define EIP93_SA_CMD_PAD_ZERO FIELD_PREP(EIP93_SA_CMD_PAD_TYPE, 0x3) +#define EIP93_SA_CMD_PAD_CONST FIELD_PREP(EIP93_SA_CMD_PAD_TYPE, 0x2) +#define EIP93_SA_CMD_PAD_PKCS7 FIELD_PREP(EIP93_SA_CMD_PAD_TYPE, 0x1) +#define EIP93_SA_CMD_PAD_IPSEC FIELD_PREP(EIP93_SA_CMD_PAD_TYPE, 0x0) +#define EIP93_SA_CMD_OPGROUP GENMASK(5, 4) +#define EIP93_SA_CMD_OP_EXT FIELD_PREP(EIP93_SA_CMD_OPGROUP, 0x2) +#define EIP93_SA_CMD_OP_PROTOCOL FIELD_PREP(EIP93_SA_CMD_OPGROUP, 0x1) +#define EIP93_SA_CMD_OP_BASIC FIELD_PREP(EIP93_SA_CMD_OPGROUP, 0x0) +#define EIP93_SA_CMD_DIRECTION_IN BIT(3) /* 0: outbount 1: inbound */ +#define EIP93_SA_CMD_OPCODE GENMASK(2, 0) +#define EIP93_SA_CMD_OPCODE_BASIC_OUT_PRNG 0x7 +#define EIP93_SA_CMD_OPCODE_BASIC_OUT_HASH 0x3 +#define EIP93_SA_CMD_OPCODE_BASIC_OUT_ENC_HASH 0x1 +#define EIP93_SA_CMD_OPCODE_BASIC_OUT_ENC 0x0 +#define EIP93_SA_CMD_OPCODE_BASIC_IN_HASH 0x3 +#define EIP93_SA_CMD_OPCODE_BASIC_IN_HASH_DEC 0x1 +#define EIP93_SA_CMD_OPCODE_BASIC_IN_DEC 0x0 +#define EIP93_SA_CMD_OPCODE_PROTOCOL_OUT_ESP 0x0 +#define EIP93_SA_CMD_OPCODE_PROTOCOL_OUT_SSL 0x4 +#define EIP93_SA_CMD_OPCODE_PROTOCOL_OUT_TLS 0x5 +#define EIP93_SA_CMD_OPCODE_PROTOCOL_OUT_SRTP 0x7 +#define EIP93_SA_CMD_OPCODE_PROTOCOL_IN_ESP 0x0 +#define EIP93_SA_CMD_OPCODE_PROTOCOL_IN_SSL 0x2 +#define EIP93_SA_CMD_OPCODE_PROTOCOL_IN_TLS 0x3 +#define EIP93_SA_CMD_OPCODE_PROTOCOL_IN_SRTP 0x7 +#define EIP93_SA_CMD_OPCODE_EXT_OUT_DTSL 0x1 +#define EIP93_SA_CMD_OPCODE_EXT_OUT_SSL 0x4 +#define EIP93_SA_CMD_OPCODE_EXT_OUT_TLSV10 0x5 +#define EIP93_SA_CMD_OPCODE_EXT_OUT_TLSV11 0x6 +#define EIP93_SA_CMD_OPCODE_EXT_IN_DTSL 0x1 +#define EIP93_SA_CMD_OPCODE_EXT_IN_SSL 0x4 +#define EIP93_SA_CMD_OPCODE_EXT_IN_TLSV10 0x5 +#define EIP93_SA_CMD_OPCODE_EXT_IN_TLSV11 0x6 +#define EIP93_REG_SA_CMD_1 0x404 +#define EIP93_SA_CMD_EN_SEQNUM_CHK BIT(29) +/* This mask can be either used for ARC4 or AES */ +#define EIP93_SA_CMD_ARC4_KEY_LENGHT GENMASK(28, 24) +#define EIP93_SA_CMD_AES_DEC_KEY BIT(28) /* 0: encrypt key 1: decrypt key */ +#define EIP93_SA_CMD_AES_KEY_LENGTH GENMASK(26, 24) +#define EIP93_SA_CMD_AES_KEY_256BIT FIELD_PREP(EIP93_SA_CMD_AES_KEY_LENGTH, 0x4) +#define EIP93_SA_CMD_AES_KEY_192BIT FIELD_PREP(EIP93_SA_CMD_AES_KEY_LENGTH, 0x3) +#define EIP93_SA_CMD_AES_KEY_128BIT FIELD_PREP(EIP93_SA_CMD_AES_KEY_LENGTH, 0x2) +#define EIP93_SA_CMD_HASH_CRYPT_OFFSET GENMASK(23, 16) +#define EIP93_SA_CMD_BYTE_OFFSET BIT(13) /* 0: CRYPT_OFFSET in 32bit word 1: CRYPT_OFFSET in 8bit bytes */ +#define EIP93_SA_CMD_HMAC BIT(12) +#define EIP93_SA_CMD_SSL_MAC BIT(12) +/* This mask can be either used for ARC4 or AES */ +#define EIP93_SA_CMD_CHIPER_MODE GENMASK(9, 8) +/* AES or DES operations */ +#define EIP93_SA_CMD_CHIPER_MODE_ICM FIELD_PREP(EIP93_SA_CMD_CHIPER_MODE, 0x3) +#define EIP93_SA_CMD_CHIPER_MODE_CTR FIELD_PREP(EIP93_SA_CMD_CHIPER_MODE, 0x2) +#define EIP93_SA_CMD_CHIPER_MODE_CBC FIELD_PREP(EIP93_SA_CMD_CHIPER_MODE, 0x1) +#define EIP93_SA_CMD_CHIPER_MODE_ECB FIELD_PREP(EIP93_SA_CMD_CHIPER_MODE, 0x0) +/* ARC4 operations */ +#define EIP93_SA_CMD_CHIPER_MODE_STATEFULL FIELD_PREP(EIP93_SA_CMD_CHIPER_MODE, 0x1) +#define EIP93_SA_CMD_CHIPER_MODE_STATELESS FIELD_PREP(EIP93_SA_CMD_CHIPER_MODE, 0x0) +#define EIP93_SA_CMD_COPY_PAD BIT(3) +#define EIP93_SA_CMD_COPY_PAYLOAD BIT(2) +#define EIP93_SA_CMD_COPY_HEADER BIT(1) +#define EIP93_SA_CMD_COPY_DIGEST BIT(0) /* With this enabled, COPY_PAD is required */ + +/* State save register */ +#define EIP93_REG_STATE_IV_0 0x500 +#define EIP93_REG_STATE_IV_1 0x504 + +#define EIP93_REG_PE_ARC4STATE 0x700 + +struct sa_record { + u32 sa_cmd0_word; + u32 sa_cmd1_word; + u32 sa_key[8]; + u8 sa_i_digest[32]; + u8 sa_o_digest[32]; + u32 sa_spi; + u32 sa_seqnum[2]; + u32 sa_seqmum_mask[2]; + u32 sa_nonce; +} __packed; + +struct sa_state { + u32 state_iv[4]; + u32 state_byte_cnt[2]; + u8 state_i_digest[32]; +} __packed; + +struct eip93_descriptor { + u32 pe_ctrl_stat_word; + u32 src_addr; + u32 dst_addr; + u32 sa_addr; + u32 state_addr; + u32 arc4_addr; + u32 user_id; + u32 pe_length_word; +} __packed; + +#endif diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c new file mode 100644 index 000000000..71b920e17 --- /dev/null +++ b/drivers/crypto/inside-secure/safexcel.c @@ -0,0 +1,2044 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2017 Marvell + * + * Antoine Tenart + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "safexcel.h" + +static u32 max_rings = EIP197_MAX_RINGS; +module_param(max_rings, uint, 0644); +MODULE_PARM_DESC(max_rings, "Maximum number of rings to use."); + +static void eip197_trc_cache_setupvirt(struct safexcel_crypto_priv *priv) +{ + int i; + + /* + * Map all interfaces/rings to register index 0 + * so they can share contexts. Without this, the EIP197 will + * assume each interface/ring to be in its own memory domain + * i.e. have its own subset of UNIQUE memory addresses. + * Which would cause records with the SAME memory address to + * use DIFFERENT cache buffers, causing both poor cache utilization + * AND serious coherence/invalidation issues. + */ + for (i = 0; i < 4; i++) + writel(0, priv->base + EIP197_FLUE_IFC_LUT(i)); + + /* + * Initialize other virtualization regs for cache + * These may not be in their reset state ... + */ + for (i = 0; i < priv->config.rings; i++) { + writel(0, priv->base + EIP197_FLUE_CACHEBASE_LO(i)); + writel(0, priv->base + EIP197_FLUE_CACHEBASE_HI(i)); + writel(EIP197_FLUE_CONFIG_MAGIC, + priv->base + EIP197_FLUE_CONFIG(i)); + } + writel(0, priv->base + EIP197_FLUE_OFFSETS); + writel(0, priv->base + EIP197_FLUE_ARC4_OFFSET); +} + +static void eip197_trc_cache_banksel(struct safexcel_crypto_priv *priv, + u32 addrmid, int *actbank) +{ + u32 val; + int curbank; + + curbank = addrmid >> 16; + if (curbank != *actbank) { + val = readl(priv->base + EIP197_CS_RAM_CTRL); + val = (val & ~EIP197_CS_BANKSEL_MASK) | + (curbank << EIP197_CS_BANKSEL_OFS); + writel(val, priv->base + EIP197_CS_RAM_CTRL); + *actbank = curbank; + } +} + +static u32 eip197_trc_cache_probe(struct safexcel_crypto_priv *priv, + int maxbanks, u32 probemask, u32 stride) +{ + u32 val, addrhi, addrlo, addrmid, addralias, delta, marker; + int actbank; + + /* + * And probe the actual size of the physically attached cache data RAM + * Using a binary subdivision algorithm downto 32 byte cache lines. + */ + addrhi = 1 << (16 + maxbanks); + addrlo = 0; + actbank = min(maxbanks - 1, 0); + while ((addrhi - addrlo) > stride) { + /* write marker to lowest address in top half */ + addrmid = (addrhi + addrlo) >> 1; + marker = (addrmid ^ 0xabadbabe) & probemask; /* Unique */ + eip197_trc_cache_banksel(priv, addrmid, &actbank); + writel(marker, + priv->base + EIP197_CLASSIFICATION_RAMS + + (addrmid & 0xffff)); + + /* write invalid markers to possible aliases */ + delta = 1 << __fls(addrmid); + while (delta >= stride) { + addralias = addrmid - delta; + eip197_trc_cache_banksel(priv, addralias, &actbank); + writel(~marker, + priv->base + EIP197_CLASSIFICATION_RAMS + + (addralias & 0xffff)); + delta >>= 1; + } + + /* read back marker from top half */ + eip197_trc_cache_banksel(priv, addrmid, &actbank); + val = readl(priv->base + EIP197_CLASSIFICATION_RAMS + + (addrmid & 0xffff)); + + if ((val & probemask) == marker) + /* read back correct, continue with top half */ + addrlo = addrmid; + else + /* not read back correct, continue with bottom half */ + addrhi = addrmid; + } + return addrhi; +} + +static void eip197_trc_cache_clear(struct safexcel_crypto_priv *priv, + int cs_rc_max, int cs_ht_wc) +{ + int i; + u32 htable_offset, val, offset; + + /* Clear all records in administration RAM */ + for (i = 0; i < cs_rc_max; i++) { + offset = EIP197_CLASSIFICATION_RAMS + i * EIP197_CS_RC_SIZE; + + writel(EIP197_CS_RC_NEXT(EIP197_RC_NULL) | + EIP197_CS_RC_PREV(EIP197_RC_NULL), + priv->base + offset); + + val = EIP197_CS_RC_NEXT(i + 1) | EIP197_CS_RC_PREV(i - 1); + if (i == 0) + val |= EIP197_CS_RC_PREV(EIP197_RC_NULL); + else if (i == cs_rc_max - 1) + val |= EIP197_CS_RC_NEXT(EIP197_RC_NULL); + writel(val, priv->base + offset + 4); + /* must also initialize the address key due to ECC! */ + writel(0, priv->base + offset + 8); + writel(0, priv->base + offset + 12); + } + + /* Clear the hash table entries */ + htable_offset = cs_rc_max * EIP197_CS_RC_SIZE; + for (i = 0; i < cs_ht_wc; i++) + writel(GENMASK(29, 0), + priv->base + EIP197_CLASSIFICATION_RAMS + + htable_offset + i * sizeof(u32)); +} + +static int eip197_trc_cache_init(struct safexcel_crypto_priv *priv) +{ + u32 val, dsize, asize; + int cs_rc_max, cs_ht_wc, cs_trc_rec_wc, cs_trc_lg_rec_wc; + int cs_rc_abs_max, cs_ht_sz; + int maxbanks; + + /* Setup (dummy) virtualization for cache */ + eip197_trc_cache_setupvirt(priv); + + /* + * Enable the record cache memory access and + * probe the bank select width + */ + val = readl(priv->base + EIP197_CS_RAM_CTRL); + val &= ~EIP197_TRC_ENABLE_MASK; + val |= EIP197_TRC_ENABLE_0 | EIP197_CS_BANKSEL_MASK; + writel(val, priv->base + EIP197_CS_RAM_CTRL); + val = readl(priv->base + EIP197_CS_RAM_CTRL); + maxbanks = ((val&EIP197_CS_BANKSEL_MASK)>>EIP197_CS_BANKSEL_OFS) + 1; + + /* Clear all ECC errors */ + writel(0, priv->base + EIP197_TRC_ECCCTRL); + + /* + * Make sure the cache memory is accessible by taking record cache into + * reset. Need data memory access here, not admin access. + */ + val = readl(priv->base + EIP197_TRC_PARAMS); + val |= EIP197_TRC_PARAMS_SW_RESET | EIP197_TRC_PARAMS_DATA_ACCESS; + writel(val, priv->base + EIP197_TRC_PARAMS); + + /* Probed data RAM size in bytes */ + dsize = eip197_trc_cache_probe(priv, maxbanks, 0xffffffff, 32); + + /* + * Now probe the administration RAM size pretty much the same way + * Except that only the lower 30 bits are writable and we don't need + * bank selects + */ + val = readl(priv->base + EIP197_TRC_PARAMS); + /* admin access now */ + val &= ~(EIP197_TRC_PARAMS_DATA_ACCESS | EIP197_CS_BANKSEL_MASK); + writel(val, priv->base + EIP197_TRC_PARAMS); + + /* Probed admin RAM size in admin words */ + asize = eip197_trc_cache_probe(priv, 0, 0x3fffffff, 16) >> 4; + + /* Clear any ECC errors detected while probing! */ + writel(0, priv->base + EIP197_TRC_ECCCTRL); + + /* Sanity check probing results */ + if (dsize < EIP197_MIN_DSIZE || asize < EIP197_MIN_ASIZE) { + dev_err(priv->dev, "Record cache probing failed (%d,%d).", + dsize, asize); + return -ENODEV; + } + + /* + * Determine optimal configuration from RAM sizes + * Note that we assume that the physical RAM configuration is sane + * Therefore, we don't do any parameter error checking here ... + */ + + /* For now, just use a single record format covering everything */ + cs_trc_rec_wc = EIP197_CS_TRC_REC_WC; + cs_trc_lg_rec_wc = EIP197_CS_TRC_REC_WC; + + /* + * Step #1: How many records will physically fit? + * Hard upper limit is 1023! + */ + cs_rc_abs_max = min_t(uint, ((dsize >> 2) / cs_trc_lg_rec_wc), 1023); + /* Step #2: Need at least 2 words in the admin RAM per record */ + cs_rc_max = min_t(uint, cs_rc_abs_max, (asize >> 1)); + /* Step #3: Determine log2 of hash table size */ + cs_ht_sz = __fls(asize - cs_rc_max) - 2; + /* Step #4: determine current size of hash table in dwords */ + cs_ht_wc = 16 << cs_ht_sz; /* dwords, not admin words */ + /* Step #5: add back excess words and see if we can fit more records */ + cs_rc_max = min_t(uint, cs_rc_abs_max, asize - (cs_ht_wc >> 2)); + + /* Clear the cache RAMs */ + eip197_trc_cache_clear(priv, cs_rc_max, cs_ht_wc); + + /* Disable the record cache memory access */ + val = readl(priv->base + EIP197_CS_RAM_CTRL); + val &= ~EIP197_TRC_ENABLE_MASK; + writel(val, priv->base + EIP197_CS_RAM_CTRL); + + /* Write head and tail pointers of the record free chain */ + val = EIP197_TRC_FREECHAIN_HEAD_PTR(0) | + EIP197_TRC_FREECHAIN_TAIL_PTR(cs_rc_max - 1); + writel(val, priv->base + EIP197_TRC_FREECHAIN); + + /* Configure the record cache #1 */ + val = EIP197_TRC_PARAMS2_RC_SZ_SMALL(cs_trc_rec_wc) | + EIP197_TRC_PARAMS2_HTABLE_PTR(cs_rc_max); + writel(val, priv->base + EIP197_TRC_PARAMS2); + + /* Configure the record cache #2 */ + val = EIP197_TRC_PARAMS_RC_SZ_LARGE(cs_trc_lg_rec_wc) | + EIP197_TRC_PARAMS_BLK_TIMER_SPEED(1) | + EIP197_TRC_PARAMS_HTABLE_SZ(cs_ht_sz); + writel(val, priv->base + EIP197_TRC_PARAMS); + + dev_info(priv->dev, "TRC init: %dd,%da (%dr,%dh)\n", + dsize, asize, cs_rc_max, cs_ht_wc + cs_ht_wc); + return 0; +} + +static void eip197_init_firmware(struct safexcel_crypto_priv *priv) +{ + int pe, i; + u32 val; + + for (pe = 0; pe < priv->config.pes; pe++) { + /* Configure the token FIFO's */ + writel(3, EIP197_PE(priv) + EIP197_PE_ICE_PUTF_CTRL(pe)); + writel(0, EIP197_PE(priv) + EIP197_PE_ICE_PPTF_CTRL(pe)); + + /* Clear the ICE scratchpad memory */ + val = readl(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe)); + val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER | + EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN | + EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS | + EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS; + writel(val, EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe)); + + /* clear the scratchpad RAM using 32 bit writes only */ + for (i = 0; i < EIP197_NUM_OF_SCRATCH_BLOCKS; i++) + writel(0, EIP197_PE(priv) + + EIP197_PE_ICE_SCRATCH_RAM(pe) + (i << 2)); + + /* Reset the IFPP engine to make its program mem accessible */ + writel(EIP197_PE_ICE_x_CTRL_SW_RESET | + EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR | + EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR, + EIP197_PE(priv) + EIP197_PE_ICE_FPP_CTRL(pe)); + + /* Reset the IPUE engine to make its program mem accessible */ + writel(EIP197_PE_ICE_x_CTRL_SW_RESET | + EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR | + EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR, + EIP197_PE(priv) + EIP197_PE_ICE_PUE_CTRL(pe)); + + /* Enable access to all IFPP program memories */ + writel(EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN, + EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe)); + + /* bypass the OCE, if present */ + if (priv->flags & EIP197_OCE) + writel(EIP197_DEBUG_OCE_BYPASS, EIP197_PE(priv) + + EIP197_PE_DEBUG(pe)); + } + +} + +static int eip197_write_firmware(struct safexcel_crypto_priv *priv, + const struct firmware *fw) +{ + u32 val; + int i; + + /* Write the firmware */ + for (i = 0; i < fw->size / sizeof(u32); i++) { + if (priv->data->fw_little_endian) + val = le32_to_cpu(((const __le32 *)fw->data)[i]); + else + val = be32_to_cpu(((const __be32 *)fw->data)[i]); + + writel(val, + priv->base + EIP197_CLASSIFICATION_RAMS + + i * sizeof(val)); + } + + /* Exclude final 2 NOPs from size */ + return i - EIP197_FW_TERMINAL_NOPS; +} + +/* + * If FW is actual production firmware, then poll for its initialization + * to complete and check if it is good for the HW, otherwise just return OK. + */ +static bool poll_fw_ready(struct safexcel_crypto_priv *priv, int fpp) +{ + int pe, pollcnt; + u32 base, pollofs; + + if (fpp) + pollofs = EIP197_FW_FPP_READY; + else + pollofs = EIP197_FW_PUE_READY; + + for (pe = 0; pe < priv->config.pes; pe++) { + base = EIP197_PE_ICE_SCRATCH_RAM(pe); + pollcnt = EIP197_FW_START_POLLCNT; + while (pollcnt && + (readl_relaxed(EIP197_PE(priv) + base + + pollofs) != 1)) { + pollcnt--; + } + if (!pollcnt) { + dev_err(priv->dev, "FW(%d) for PE %d failed to start\n", + fpp, pe); + return false; + } + } + return true; +} + +static bool eip197_start_firmware(struct safexcel_crypto_priv *priv, + int ipuesz, int ifppsz, int minifw) +{ + int pe; + u32 val; + + for (pe = 0; pe < priv->config.pes; pe++) { + /* Disable access to all program memory */ + writel(0, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe)); + + /* Start IFPP microengines */ + if (minifw) + val = 0; + else + val = EIP197_PE_ICE_UENG_START_OFFSET((ifppsz - 1) & + EIP197_PE_ICE_UENG_INIT_ALIGN_MASK) | + EIP197_PE_ICE_UENG_DEBUG_RESET; + writel(val, EIP197_PE(priv) + EIP197_PE_ICE_FPP_CTRL(pe)); + + /* Start IPUE microengines */ + if (minifw) + val = 0; + else + val = EIP197_PE_ICE_UENG_START_OFFSET((ipuesz - 1) & + EIP197_PE_ICE_UENG_INIT_ALIGN_MASK) | + EIP197_PE_ICE_UENG_DEBUG_RESET; + writel(val, EIP197_PE(priv) + EIP197_PE_ICE_PUE_CTRL(pe)); + } + + /* For miniFW startup, there is no initialization, so always succeed */ + if (minifw) + return true; + + /* Wait until all the firmwares have properly started up */ + if (!poll_fw_ready(priv, 1)) + return false; + if (!poll_fw_ready(priv, 0)) + return false; + + return true; +} + +static int eip197_load_firmwares(struct safexcel_crypto_priv *priv) +{ + const char *fw_name[] = {"ifpp.bin", "ipue.bin"}; + const struct firmware *fw[FW_NB]; + char fw_path[37], *dir = NULL; + int i, j, ret = 0, pe; + int ipuesz, ifppsz, minifw = 0; + + if (priv->data->version == EIP197D_MRVL) + dir = "eip197d"; + else if (priv->data->version == EIP197B_MRVL || + priv->data->version == EIP197_DEVBRD) + dir = "eip197b"; + else if (priv->data->version == EIP197C_MXL) + dir = "eip197c"; + else + return -ENODEV; + +retry_fw: + for (i = 0; i < FW_NB; i++) { + snprintf(fw_path, 37, "inside-secure/%s/%s", dir, fw_name[i]); + ret = firmware_request_nowarn(&fw[i], fw_path, priv->dev); + if (ret) { + if (minifw || priv->data->version != EIP197B_MRVL) + goto release_fw; + + /* Fallback to the old firmware location for the + * EIP197b. + */ + ret = firmware_request_nowarn(&fw[i], fw_name[i], + priv->dev); + if (ret) + goto release_fw; + } + } + + eip197_init_firmware(priv); + + ifppsz = eip197_write_firmware(priv, fw[FW_IFPP]); + + /* Enable access to IPUE program memories */ + for (pe = 0; pe < priv->config.pes; pe++) + writel(EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN, + EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe)); + + ipuesz = eip197_write_firmware(priv, fw[FW_IPUE]); + + if (eip197_start_firmware(priv, ipuesz, ifppsz, minifw)) { + dev_dbg(priv->dev, "Firmware loaded successfully\n"); + return 0; + } + + ret = -ENODEV; + +release_fw: + for (j = 0; j < i; j++) + release_firmware(fw[j]); + + if (!minifw) { + /* Retry with minifw path */ + dev_dbg(priv->dev, "Firmware set not (fully) present or init failed, falling back to BCLA mode\n"); + dir = "eip197_minifw"; + minifw = 1; + goto retry_fw; + } + + dev_err(priv->dev, "Firmware load failed.\n"); + + return ret; +} + +static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv) +{ + u32 cd_size_rnd, val; + int i, cd_fetch_cnt; + + cd_size_rnd = (priv->config.cd_size + + (BIT(priv->hwconfig.hwdataw) - 1)) >> + priv->hwconfig.hwdataw; + /* determine number of CD's we can fetch into the CD FIFO as 1 block */ + if (priv->flags & SAFEXCEL_HW_EIP197) { + /* EIP197: try to fetch enough in 1 go to keep all pipes busy */ + cd_fetch_cnt = (1 << priv->hwconfig.hwcfsize) / cd_size_rnd; + cd_fetch_cnt = min_t(uint, cd_fetch_cnt, + (priv->config.pes * EIP197_FETCH_DEPTH)); + } else { + /* for the EIP97, just fetch all that fits minus 1 */ + cd_fetch_cnt = ((1 << priv->hwconfig.hwcfsize) / + cd_size_rnd) - 1; + } + /* + * Since we're using command desc's way larger than formally specified, + * we need to check whether we can fit even 1 for low-end EIP196's! + */ + if (!cd_fetch_cnt) { + dev_err(priv->dev, "Unable to fit even 1 command desc!\n"); + return -ENODEV; + } + + for (i = 0; i < priv->config.rings; i++) { + /* ring base address */ + writel(lower_32_bits(priv->ring[i].cdr.base_dma), + EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); + writel(upper_32_bits(priv->ring[i].cdr.base_dma), + EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); + + writel(EIP197_xDR_DESC_MODE_64BIT | EIP197_CDR_DESC_MODE_ADCP | + (priv->config.cd_offset << 14) | priv->config.cd_size, + EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE); + writel(((cd_fetch_cnt * + (cd_size_rnd << priv->hwconfig.hwdataw)) << 16) | + (cd_fetch_cnt * (priv->config.cd_offset / sizeof(u32))), + EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG); + + /* Configure DMA tx control */ + val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS); + val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS); + writel(val, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DMA_CFG); + + /* clear any pending interrupt */ + writel(GENMASK(5, 0), + EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT); + } + + return 0; +} + +static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv) +{ + u32 rd_size_rnd, val; + int i, rd_fetch_cnt; + + /* determine number of RD's we can fetch into the FIFO as one block */ + rd_size_rnd = (EIP197_RD64_FETCH_SIZE + + (BIT(priv->hwconfig.hwdataw) - 1)) >> + priv->hwconfig.hwdataw; + if (priv->flags & SAFEXCEL_HW_EIP197) { + /* EIP197: try to fetch enough in 1 go to keep all pipes busy */ + rd_fetch_cnt = (1 << priv->hwconfig.hwrfsize) / rd_size_rnd; + rd_fetch_cnt = min_t(uint, rd_fetch_cnt, + (priv->config.pes * EIP197_FETCH_DEPTH)); + } else { + /* for the EIP97, just fetch all that fits minus 1 */ + rd_fetch_cnt = ((1 << priv->hwconfig.hwrfsize) / + rd_size_rnd) - 1; + } + + for (i = 0; i < priv->config.rings; i++) { + /* ring base address */ + writel(lower_32_bits(priv->ring[i].rdr.base_dma), + EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); + writel(upper_32_bits(priv->ring[i].rdr.base_dma), + EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); + + writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.rd_offset << 14) | + priv->config.rd_size, + EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE); + + writel(((rd_fetch_cnt * + (rd_size_rnd << priv->hwconfig.hwdataw)) << 16) | + (rd_fetch_cnt * (priv->config.rd_offset / sizeof(u32))), + EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG); + + /* Configure DMA tx control */ + val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS); + val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS); + val |= EIP197_HIA_xDR_WR_RES_BUF | EIP197_HIA_xDR_WR_CTRL_BUF; + writel(val, + EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DMA_CFG); + + /* clear any pending interrupt */ + writel(GENMASK(7, 0), + EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT); + + /* enable ring interrupt */ + val = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i)); + val |= EIP197_RDR_IRQ(i); + writel(val, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i)); + } + + return 0; +} + +static int safexcel_hw_init(struct safexcel_crypto_priv *priv) +{ + u32 val; + int i, ret, pe, opbuflo, opbufhi; + + dev_dbg(priv->dev, "HW init: using %d pipe(s) and %d ring(s)\n", + priv->config.pes, priv->config.rings); + + /* + * For EIP197's only set maximum number of TX commands to 2^5 = 32 + * Skip for the EIP97 as it does not have this field. + */ + if (priv->flags & SAFEXCEL_HW_EIP197) { + val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL); + val |= EIP197_MST_CTRL_TX_MAX_CMD(5); + writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL); + } + + /* Configure wr/rd cache values */ + writel(EIP197_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) | + EIP197_MST_CTRL_WD_CACHE(WR_CACHE_4BITS), + EIP197_HIA_GEN_CFG(priv) + EIP197_MST_CTRL); + + /* Interrupts reset */ + + /* Disable all global interrupts */ + writel(0, EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ENABLE_CTRL); + + /* Clear any pending interrupt */ + writel(GENMASK(31, 0), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK); + + /* Processing Engine configuration */ + for (pe = 0; pe < priv->config.pes; pe++) { + /* Data Fetch Engine configuration */ + + /* Reset all DFE threads */ + writel(EIP197_DxE_THR_CTRL_RESET_PE, + EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe)); + + if (priv->flags & EIP197_PE_ARB) + /* Reset HIA input interface arbiter (if present) */ + writel(EIP197_HIA_RA_PE_CTRL_RESET, + EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe)); + + /* DMA transfer size to use */ + val = EIP197_HIA_DFE_CFG_DIS_DEBUG; + val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(6) | + EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(9); + val |= EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(6) | + EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(7); + val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS); + val |= EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS); + writel(val, EIP197_HIA_DFE(priv) + EIP197_HIA_DFE_CFG(pe)); + + /* Leave the DFE threads reset state */ + writel(0, EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe)); + + /* Configure the processing engine thresholds */ + writel(EIP197_PE_IN_xBUF_THRES_MIN(6) | + EIP197_PE_IN_xBUF_THRES_MAX(9), + EIP197_PE(priv) + EIP197_PE_IN_DBUF_THRES(pe)); + writel(EIP197_PE_IN_xBUF_THRES_MIN(6) | + EIP197_PE_IN_xBUF_THRES_MAX(7), + EIP197_PE(priv) + EIP197_PE_IN_TBUF_THRES(pe)); + + if (priv->flags & SAFEXCEL_HW_EIP197) + /* enable HIA input interface arbiter and rings */ + writel(EIP197_HIA_RA_PE_CTRL_EN | + GENMASK(priv->config.rings - 1, 0), + EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe)); + + /* Data Store Engine configuration */ + + /* Reset all DSE threads */ + writel(EIP197_DxE_THR_CTRL_RESET_PE, + EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe)); + + /* Wait for all DSE threads to complete */ + while ((readl(EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_STAT(pe)) & + GENMASK(15, 12)) != GENMASK(15, 12)) + ; + + /* DMA transfer size to use */ + if (priv->hwconfig.hwnumpes > 4) { + opbuflo = 9; + opbufhi = 10; + } else { + opbuflo = 7; + opbufhi = 8; + } + val = EIP197_HIA_DSE_CFG_DIS_DEBUG; + val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(opbuflo) | + EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(opbufhi); + val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS); + val |= EIP197_HIA_DSE_CFG_ALWAYS_BUFFERABLE; + /* FIXME: instability issues can occur for EIP97 but disabling + * it impacts performance. + */ + if (priv->flags & SAFEXCEL_HW_EIP197) + val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR; + writel(val, EIP197_HIA_DSE(priv) + EIP197_HIA_DSE_CFG(pe)); + + /* Leave the DSE threads reset state */ + writel(0, EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe)); + + /* Configure the processing engine thresholds */ + writel(EIP197_PE_OUT_DBUF_THRES_MIN(opbuflo) | + EIP197_PE_OUT_DBUF_THRES_MAX(opbufhi), + EIP197_PE(priv) + EIP197_PE_OUT_DBUF_THRES(pe)); + + /* Processing Engine configuration */ + + /* Token & context configuration */ + val = EIP197_PE_EIP96_TOKEN_CTRL_CTX_UPDATES | + EIP197_PE_EIP96_TOKEN_CTRL_NO_TOKEN_WAIT | + EIP197_PE_EIP96_TOKEN_CTRL_ENABLE_TIMEOUT; + writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_TOKEN_CTRL(pe)); + + /* H/W capabilities selection: just enable everything */ + writel(EIP197_FUNCTION_ALL, + EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION_EN(pe)); + writel(EIP197_FUNCTION_ALL, + EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION2_EN(pe)); + } + + /* Command Descriptor Rings prepare */ + for (i = 0; i < priv->config.rings; i++) { + /* Clear interrupts for this ring */ + writel(GENMASK(31, 0), + EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CLR(i)); + + /* Disable external triggering */ + writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG); + + /* Clear the pending prepared counter */ + writel(EIP197_xDR_PREP_CLR_COUNT, + EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT); + + /* Clear the pending processed counter */ + writel(EIP197_xDR_PROC_CLR_COUNT, + EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT); + + writel(0, + EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR); + writel(0, + EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR); + + writel((EIP197_DEFAULT_RING_SIZE * priv->config.cd_offset), + EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_SIZE); + } + + /* Result Descriptor Ring prepare */ + for (i = 0; i < priv->config.rings; i++) { + /* Disable external triggering*/ + writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG); + + /* Clear the pending prepared counter */ + writel(EIP197_xDR_PREP_CLR_COUNT, + EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT); + + /* Clear the pending processed counter */ + writel(EIP197_xDR_PROC_CLR_COUNT, + EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT); + + writel(0, + EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR); + writel(0, + EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR); + + /* Ring size */ + writel((EIP197_DEFAULT_RING_SIZE * priv->config.rd_offset), + EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_SIZE); + } + + for (pe = 0; pe < priv->config.pes; pe++) { + /* Enable command descriptor rings */ + writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0), + EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe)); + + /* Enable result descriptor rings */ + writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0), + EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe)); + } + + /* Clear any HIA interrupt */ + writel(GENMASK(30, 20), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK); + + if (priv->flags & EIP197_SIMPLE_TRC) { + writel(EIP197_STRC_CONFIG_INIT | + EIP197_STRC_CONFIG_LARGE_REC(EIP197_CS_TRC_REC_WC) | + EIP197_STRC_CONFIG_SMALL_REC(EIP197_CS_TRC_REC_WC), + priv->base + EIP197_STRC_CONFIG); + writel(EIP197_PE_EIP96_TOKEN_CTRL2_CTX_DONE, + EIP197_PE(priv) + EIP197_PE_EIP96_TOKEN_CTRL2(0)); + } else if (priv->flags & SAFEXCEL_HW_EIP197) { + ret = eip197_trc_cache_init(priv); + if (ret) + return ret; + } + + if (priv->flags & EIP197_ICE) { + ret = eip197_load_firmwares(priv); + if (ret) + return ret; + } + + return safexcel_hw_setup_cdesc_rings(priv) ?: + safexcel_hw_setup_rdesc_rings(priv) ?: + 0; +} + +/* Called with ring's lock taken */ +static void safexcel_try_push_requests(struct safexcel_crypto_priv *priv, + int ring) +{ + int coal = min_t(int, priv->ring[ring].requests, EIP197_MAX_BATCH_SZ); + + if (!coal) + return; + + /* Configure when we want an interrupt */ + writel(EIP197_HIA_RDR_THRESH_PKT_MODE | + EIP197_HIA_RDR_THRESH_PROC_PKT(coal), + EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_THRESH); +} + +void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring) +{ + struct crypto_async_request *req, *backlog; + struct safexcel_context *ctx; + int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results; + + /* If a request wasn't properly dequeued because of a lack of resources, + * proceeded it first, + */ + req = priv->ring[ring].req; + backlog = priv->ring[ring].backlog; + if (req) + goto handle_req; + + while (true) { + spin_lock_bh(&priv->ring[ring].queue_lock); + backlog = crypto_get_backlog(&priv->ring[ring].queue); + req = crypto_dequeue_request(&priv->ring[ring].queue); + spin_unlock_bh(&priv->ring[ring].queue_lock); + + if (!req) { + priv->ring[ring].req = NULL; + priv->ring[ring].backlog = NULL; + goto finalize; + } + +handle_req: + ctx = crypto_tfm_ctx(req->tfm); + ret = ctx->send(req, ring, &commands, &results); + if (ret) + goto request_failed; + + if (backlog) + crypto_request_complete(backlog, -EINPROGRESS); + + /* In case the send() helper did not issue any command to push + * to the engine because the input data was cached, continue to + * dequeue other requests as this is valid and not an error. + */ + if (!commands && !results) + continue; + + cdesc += commands; + rdesc += results; + nreq++; + } + +request_failed: + /* Not enough resources to handle all the requests. Bail out and save + * the request and the backlog for the next dequeue call (per-ring). + */ + priv->ring[ring].req = req; + priv->ring[ring].backlog = backlog; + +finalize: + if (!nreq) + return; + + spin_lock_bh(&priv->ring[ring].lock); + + priv->ring[ring].requests += nreq; + + if (!priv->ring[ring].busy) { + safexcel_try_push_requests(priv, ring); + priv->ring[ring].busy = true; + } + + spin_unlock_bh(&priv->ring[ring].lock); + + /* let the RDR know we have pending descriptors */ + writel((rdesc * priv->config.rd_offset), + EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT); + + /* let the CDR know we have pending descriptors */ + writel((cdesc * priv->config.cd_offset), + EIP197_HIA_CDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT); +} + +inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv, + void *rdp) +{ + struct safexcel_result_desc *rdesc = rdp; + struct result_data_desc *result_data = rdp + priv->config.res_offset; + + if (likely((!rdesc->last_seg) || /* Rest only valid if last seg! */ + ((!rdesc->descriptor_overflow) && + (!rdesc->buffer_overflow) && + (!result_data->error_code)))) + return 0; + + if (rdesc->descriptor_overflow) + dev_err(priv->dev, "Descriptor overflow detected"); + + if (rdesc->buffer_overflow) + dev_err(priv->dev, "Buffer overflow detected"); + + if (result_data->error_code & 0x4066) { + /* Fatal error (bits 1,2,5,6 & 14) */ + dev_err(priv->dev, + "result descriptor error (%x)", + result_data->error_code); + + return -EIO; + } else if (result_data->error_code & + (BIT(7) | BIT(4) | BIT(3) | BIT(0))) { + /* + * Give priority over authentication fails: + * Blocksize, length & overflow errors, + * something wrong with the input! + */ + return -EINVAL; + } else if (result_data->error_code & BIT(9)) { + /* Authentication failed */ + return -EBADMSG; + } + + /* All other non-fatal errors */ + return -EINVAL; +} + +inline void safexcel_rdr_req_set(struct safexcel_crypto_priv *priv, + int ring, + struct safexcel_result_desc *rdesc, + struct crypto_async_request *req) +{ + int i = safexcel_ring_rdr_rdesc_index(priv, ring, rdesc); + + priv->ring[ring].rdr_req[i] = req; +} + +inline struct crypto_async_request * +safexcel_rdr_req_get(struct safexcel_crypto_priv *priv, int ring) +{ + int i = safexcel_ring_first_rdr_index(priv, ring); + + return priv->ring[ring].rdr_req[i]; +} + +void safexcel_complete(struct safexcel_crypto_priv *priv, int ring) +{ + struct safexcel_command_desc *cdesc; + + /* Acknowledge the command descriptors */ + do { + cdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].cdr); + if (IS_ERR(cdesc)) { + dev_err(priv->dev, + "Could not retrieve the command descriptor\n"); + return; + } + } while (!cdesc->last_seg); +} + +int safexcel_invalidate_cache(struct crypto_async_request *async, + struct safexcel_crypto_priv *priv, + dma_addr_t ctxr_dma, int ring) +{ + struct safexcel_command_desc *cdesc; + struct safexcel_result_desc *rdesc; + struct safexcel_token *dmmy; + int ret = 0; + + /* Prepare command descriptor */ + cdesc = safexcel_add_cdesc(priv, ring, true, true, 0, 0, 0, ctxr_dma, + &dmmy); + if (IS_ERR(cdesc)) + return PTR_ERR(cdesc); + + cdesc->control_data.type = EIP197_TYPE_EXTENDED; + cdesc->control_data.options = 0; + cdesc->control_data.context_lo &= ~EIP197_CONTEXT_SIZE_MASK; + cdesc->control_data.control0 = CONTEXT_CONTROL_INV_TR; + + /* Prepare result descriptor */ + rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0); + + if (IS_ERR(rdesc)) { + ret = PTR_ERR(rdesc); + goto cdesc_rollback; + } + + safexcel_rdr_req_set(priv, ring, rdesc, async); + + return ret; + +cdesc_rollback: + safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); + + return ret; +} + +static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv *priv, + int ring) +{ + struct crypto_async_request *req; + struct safexcel_context *ctx; + int ret, i, nreq, ndesc, tot_descs, handled = 0; + bool should_complete; + +handle_results: + tot_descs = 0; + + nreq = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT); + nreq >>= EIP197_xDR_PROC_xD_PKT_OFFSET; + nreq &= EIP197_xDR_PROC_xD_PKT_MASK; + if (!nreq) + goto requests_left; + + for (i = 0; i < nreq; i++) { + req = safexcel_rdr_req_get(priv, ring); + + ctx = crypto_tfm_ctx(req->tfm); + ndesc = ctx->handle_result(priv, ring, req, + &should_complete, &ret); + if (ndesc < 0) { + dev_err(priv->dev, "failed to handle result (%d)\n", + ndesc); + goto acknowledge; + } + + if (should_complete) { + local_bh_disable(); + crypto_request_complete(req, ret); + local_bh_enable(); + } + + tot_descs += ndesc; + handled++; + } + +acknowledge: + if (i) + writel(EIP197_xDR_PROC_xD_PKT(i) | + (tot_descs * priv->config.rd_offset), + EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT); + + /* If the number of requests overflowed the counter, try to proceed more + * requests. + */ + if (nreq == EIP197_xDR_PROC_xD_PKT_MASK) + goto handle_results; + +requests_left: + spin_lock_bh(&priv->ring[ring].lock); + + priv->ring[ring].requests -= handled; + safexcel_try_push_requests(priv, ring); + + if (!priv->ring[ring].requests) + priv->ring[ring].busy = false; + + spin_unlock_bh(&priv->ring[ring].lock); +} + +static void safexcel_dequeue_work(struct work_struct *work) +{ + struct safexcel_work_data *data = + container_of(work, struct safexcel_work_data, work); + + safexcel_dequeue(data->priv, data->ring); +} + +struct safexcel_ring_irq_data { + struct safexcel_crypto_priv *priv; + int ring; +}; + +static irqreturn_t safexcel_irq_ring(int irq, void *data) +{ + struct safexcel_ring_irq_data *irq_data = data; + struct safexcel_crypto_priv *priv = irq_data->priv; + int ring = irq_data->ring, rc = IRQ_NONE; + u32 status, stat; + + status = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLED_STAT(ring)); + if (!status) + return rc; + + /* RDR interrupts */ + if (status & EIP197_RDR_IRQ(ring)) { + stat = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT); + + if (unlikely(stat & EIP197_xDR_ERR)) { + /* + * Fatal error, the RDR is unusable and must be + * reinitialized. This should not happen under + * normal circumstances. + */ + dev_err(priv->dev, "RDR: fatal error.\n"); + } else if (likely(stat & EIP197_xDR_THRESH)) { + rc = IRQ_WAKE_THREAD; + } + + /* ACK the interrupts */ + writel(stat & 0xff, + EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT); + } + + /* ACK the interrupts */ + writel(status, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ACK(ring)); + + return rc; +} + +static irqreturn_t safexcel_irq_ring_thread(int irq, void *data) +{ + struct safexcel_ring_irq_data *irq_data = data; + struct safexcel_crypto_priv *priv = irq_data->priv; + int ring = irq_data->ring; + + safexcel_handle_result_descriptor(priv, ring); + + queue_work(priv->ring[ring].workqueue, + &priv->ring[ring].work_data.work); + + return IRQ_HANDLED; +} + +static int safexcel_request_ring_irq(void *pdev, int irqid, + int is_pci_dev, + int ring_id, + irq_handler_t handler, + irq_handler_t threaded_handler, + struct safexcel_ring_irq_data *ring_irq_priv) +{ + int ret, irq, cpu; + struct device *dev; + + if (IS_ENABLED(CONFIG_PCI) && is_pci_dev) { + struct pci_dev *pci_pdev = pdev; + + dev = &pci_pdev->dev; + irq = pci_irq_vector(pci_pdev, irqid); + if (irq < 0) { + dev_err(dev, "unable to get device MSI IRQ %d (err %d)\n", + irqid, irq); + return irq; + } + } else if (IS_ENABLED(CONFIG_OF)) { + struct platform_device *plf_pdev = pdev; + char irq_name[6] = {0}; /* "ringX\0" */ + + snprintf(irq_name, 6, "ring%d", irqid); + dev = &plf_pdev->dev; + irq = platform_get_irq_byname(plf_pdev, irq_name); + + if (irq < 0) + return irq; + } else { + return -ENXIO; + } + + ret = devm_request_threaded_irq(dev, irq, handler, + threaded_handler, IRQF_ONESHOT, + dev_name(dev), ring_irq_priv); + if (ret) + return ret; + + /* Set affinity */ + cpu = cpumask_local_spread(ring_id, NUMA_NO_NODE); + irq_set_affinity_hint(irq, get_cpu_mask(cpu)); + + return irq; +} + +static struct safexcel_alg_template *safexcel_algs[] = { + &safexcel_alg_ecb_des, + &safexcel_alg_cbc_des, + &safexcel_alg_ecb_des3_ede, + &safexcel_alg_cbc_des3_ede, + &safexcel_alg_ecb_aes, + &safexcel_alg_cbc_aes, + &safexcel_alg_ctr_aes, + &safexcel_alg_md5, + &safexcel_alg_sha1, + &safexcel_alg_sha224, + &safexcel_alg_sha256, + &safexcel_alg_sha384, + &safexcel_alg_sha512, + &safexcel_alg_hmac_md5, + &safexcel_alg_hmac_sha1, + &safexcel_alg_hmac_sha224, + &safexcel_alg_hmac_sha256, + &safexcel_alg_hmac_sha384, + &safexcel_alg_hmac_sha512, + &safexcel_alg_authenc_hmac_md5_cbc_aes, + &safexcel_alg_authenc_hmac_sha1_cbc_aes, + &safexcel_alg_authenc_hmac_sha224_cbc_aes, + &safexcel_alg_authenc_hmac_sha256_cbc_aes, + &safexcel_alg_authenc_hmac_sha384_cbc_aes, + &safexcel_alg_authenc_hmac_sha512_cbc_aes, + &safexcel_alg_authenc_hmac_md5_ctr_aes, + &safexcel_alg_authenc_hmac_sha1_ctr_aes, + &safexcel_alg_authenc_hmac_sha224_ctr_aes, + &safexcel_alg_authenc_hmac_sha256_ctr_aes, + &safexcel_alg_authenc_hmac_sha384_ctr_aes, + &safexcel_alg_authenc_hmac_sha512_ctr_aes, + &safexcel_alg_xts_aes, + &safexcel_alg_gcm, + &safexcel_alg_ccm, + &safexcel_alg_cbcmac, + &safexcel_alg_xcbcmac, + &safexcel_alg_cmac, + &safexcel_alg_chacha20, + &safexcel_alg_chachapoly, + &safexcel_alg_chachapoly_esp, + &safexcel_alg_sm3, + &safexcel_alg_hmac_sm3, + &safexcel_alg_ecb_sm4, + &safexcel_alg_cbc_sm4, + &safexcel_alg_ctr_sm4, + &safexcel_alg_authenc_hmac_sha1_cbc_sm4, + &safexcel_alg_authenc_hmac_sm3_cbc_sm4, + &safexcel_alg_authenc_hmac_sha1_ctr_sm4, + &safexcel_alg_authenc_hmac_sm3_ctr_sm4, + &safexcel_alg_sha3_224, + &safexcel_alg_sha3_256, + &safexcel_alg_sha3_384, + &safexcel_alg_sha3_512, + &safexcel_alg_hmac_sha3_224, + &safexcel_alg_hmac_sha3_256, + &safexcel_alg_hmac_sha3_384, + &safexcel_alg_hmac_sha3_512, + &safexcel_alg_authenc_hmac_md5_cbc_des3_ede, + &safexcel_alg_authenc_hmac_sha1_cbc_des3_ede, + &safexcel_alg_authenc_hmac_sha256_cbc_des3_ede, + &safexcel_alg_authenc_hmac_sha224_cbc_des3_ede, + &safexcel_alg_authenc_hmac_sha512_cbc_des3_ede, + &safexcel_alg_authenc_hmac_sha384_cbc_des3_ede, + &safexcel_alg_authenc_hmac_md5_cbc_des, + &safexcel_alg_authenc_hmac_sha1_cbc_des, + &safexcel_alg_authenc_hmac_sha256_cbc_des, + &safexcel_alg_authenc_hmac_sha224_cbc_des, + &safexcel_alg_authenc_hmac_sha512_cbc_des, + &safexcel_alg_authenc_hmac_sha384_cbc_des, + &safexcel_alg_rfc4106_gcm, + &safexcel_alg_rfc4543_gcm, + &safexcel_alg_rfc4309_ccm, +}; + +static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) +{ + int i, j, ret = 0; + + for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) { + safexcel_algs[i]->priv = priv; + + /* Do we have all required base algorithms available? */ + if ((safexcel_algs[i]->algo_mask & priv->hwconfig.algo_flags) != + safexcel_algs[i]->algo_mask) + /* No, so don't register this ciphersuite */ + continue; + + if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER) + ret = crypto_register_skcipher(&safexcel_algs[i]->alg.skcipher); + else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD) + ret = crypto_register_aead(&safexcel_algs[i]->alg.aead); + else + ret = crypto_register_ahash(&safexcel_algs[i]->alg.ahash); + + if (ret) + goto fail; + } + + return 0; + +fail: + for (j = 0; j < i; j++) { + /* Do we have all required base algorithms available? */ + if ((safexcel_algs[j]->algo_mask & priv->hwconfig.algo_flags) != + safexcel_algs[j]->algo_mask) + /* No, so don't unregister this ciphersuite */ + continue; + + if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_SKCIPHER) + crypto_unregister_skcipher(&safexcel_algs[j]->alg.skcipher); + else if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_AEAD) + crypto_unregister_aead(&safexcel_algs[j]->alg.aead); + else + crypto_unregister_ahash(&safexcel_algs[j]->alg.ahash); + } + + return ret; +} + +static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) { + /* Do we have all required base algorithms available? */ + if ((safexcel_algs[i]->algo_mask & priv->hwconfig.algo_flags) != + safexcel_algs[i]->algo_mask) + /* No, so don't unregister this ciphersuite */ + continue; + + if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER) + crypto_unregister_skcipher(&safexcel_algs[i]->alg.skcipher); + else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD) + crypto_unregister_aead(&safexcel_algs[i]->alg.aead); + else + crypto_unregister_ahash(&safexcel_algs[i]->alg.ahash); + } +} + +static void safexcel_configure(struct safexcel_crypto_priv *priv) +{ + u32 mask = BIT(priv->hwconfig.hwdataw) - 1; + + priv->config.pes = priv->hwconfig.hwnumpes; + priv->config.rings = min_t(u32, priv->hwconfig.hwnumrings, max_rings); + /* Cannot currently support more rings than we have ring AICs! */ + priv->config.rings = min_t(u32, priv->config.rings, + priv->hwconfig.hwnumraic); + + priv->config.cd_size = EIP197_CD64_FETCH_SIZE; + priv->config.cd_offset = (priv->config.cd_size + mask) & ~mask; + priv->config.cdsh_offset = (EIP197_MAX_TOKENS + mask) & ~mask; + + /* res token is behind the descr, but ofs must be rounded to buswdth */ + priv->config.res_offset = (EIP197_RD64_FETCH_SIZE + mask) & ~mask; + /* now the size of the descr is this 1st part plus the result struct */ + priv->config.rd_size = priv->config.res_offset + + EIP197_RD64_RESULT_SIZE; + priv->config.rd_offset = (priv->config.rd_size + mask) & ~mask; + + /* convert dwords to bytes */ + priv->config.cd_offset *= sizeof(u32); + priv->config.cdsh_offset *= sizeof(u32); + priv->config.rd_offset *= sizeof(u32); + priv->config.res_offset *= sizeof(u32); +} + +static void safexcel_init_register_offsets(struct safexcel_crypto_priv *priv) +{ + struct safexcel_register_offsets *offsets = &priv->offsets; + + if (priv->flags & SAFEXCEL_HW_EIP197) { + offsets->hia_aic = EIP197_HIA_AIC_BASE; + offsets->hia_aic_g = EIP197_HIA_AIC_G_BASE; + offsets->hia_aic_r = EIP197_HIA_AIC_R_BASE; + offsets->hia_aic_xdr = EIP197_HIA_AIC_xDR_BASE; + offsets->hia_dfe = EIP197_HIA_DFE_BASE; + offsets->hia_dfe_thr = EIP197_HIA_DFE_THR_BASE; + offsets->hia_dse = EIP197_HIA_DSE_BASE; + offsets->hia_dse_thr = EIP197_HIA_DSE_THR_BASE; + offsets->hia_gen_cfg = EIP197_HIA_GEN_CFG_BASE; + offsets->pe = EIP197_PE_BASE; + offsets->global = EIP197_GLOBAL_BASE; + } else { + offsets->hia_aic = EIP97_HIA_AIC_BASE; + offsets->hia_aic_g = EIP97_HIA_AIC_G_BASE; + offsets->hia_aic_r = EIP97_HIA_AIC_R_BASE; + offsets->hia_aic_xdr = EIP97_HIA_AIC_xDR_BASE; + offsets->hia_dfe = EIP97_HIA_DFE_BASE; + offsets->hia_dfe_thr = EIP97_HIA_DFE_THR_BASE; + offsets->hia_dse = EIP97_HIA_DSE_BASE; + offsets->hia_dse_thr = EIP97_HIA_DSE_THR_BASE; + offsets->hia_gen_cfg = EIP97_HIA_GEN_CFG_BASE; + offsets->pe = EIP97_PE_BASE; + offsets->global = EIP97_GLOBAL_BASE; + } +} + +/* + * Generic part of probe routine, shared by platform and PCI driver + * + * Assumes IO resources have been mapped, private data mem has been allocated, + * clocks have been enabled, device pointer has been assigned etc. + * + */ +static int safexcel_probe_generic(void *pdev, + struct safexcel_crypto_priv *priv, + int is_pci_dev) +{ + struct device *dev = priv->dev; + u32 peid, version, mask, val, hiaopt, hwopt, peopt; + int i, ret, hwctg; + + priv->context_pool = dmam_pool_create("safexcel-context", dev, + sizeof(struct safexcel_context_record), + 1, 0); + if (!priv->context_pool) + return -ENOMEM; + + /* + * First try the EIP97 HIA version regs + * For the EIP197, this is guaranteed to NOT return any of the test + * values + */ + version = readl(priv->base + EIP97_HIA_AIC_BASE + EIP197_HIA_VERSION); + + mask = 0; /* do not swap */ + if (EIP197_REG_LO16(version) == EIP197_HIA_VERSION_LE) { + priv->hwconfig.hiaver = EIP197_VERSION_MASK(version); + } else if (EIP197_REG_HI16(version) == EIP197_HIA_VERSION_BE) { + /* read back byte-swapped, so complement byte swap bits */ + mask = EIP197_MST_CTRL_BYTE_SWAP_BITS; + priv->hwconfig.hiaver = EIP197_VERSION_SWAP(version); + } else { + /* So it wasn't an EIP97 ... maybe it's an EIP197? */ + version = readl(priv->base + EIP197_HIA_AIC_BASE + + EIP197_HIA_VERSION); + if (EIP197_REG_LO16(version) == EIP197_HIA_VERSION_LE) { + priv->hwconfig.hiaver = EIP197_VERSION_MASK(version); + priv->flags |= SAFEXCEL_HW_EIP197; + } else if (EIP197_REG_HI16(version) == + EIP197_HIA_VERSION_BE) { + /* read back byte-swapped, so complement swap bits */ + mask = EIP197_MST_CTRL_BYTE_SWAP_BITS; + priv->hwconfig.hiaver = EIP197_VERSION_SWAP(version); + priv->flags |= SAFEXCEL_HW_EIP197; + } else { + return -ENODEV; + } + } + + /* Now initialize the reg offsets based on the probing info so far */ + safexcel_init_register_offsets(priv); + + /* + * If the version was read byte-swapped, we need to flip the device + * swapping Keep in mind here, though, that what we write will also be + * byte-swapped ... + */ + if (mask) { + val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL); + val = val ^ (mask >> 24); /* toggle byte swap bits */ + writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL); + } + + /* + * We're not done probing yet! We may fall through to here if no HIA + * was found at all. So, with the endianness presumably correct now and + * the offsets setup, *really* probe for the EIP97/EIP197. + */ + version = readl(EIP197_GLOBAL(priv) + EIP197_VERSION); + if (((priv->flags & SAFEXCEL_HW_EIP197) && + (EIP197_REG_LO16(version) != EIP197_VERSION_LE) && + (EIP197_REG_LO16(version) != EIP196_VERSION_LE)) || + ((!(priv->flags & SAFEXCEL_HW_EIP197) && + (EIP197_REG_LO16(version) != EIP97_VERSION_LE)))) { + /* + * We did not find the device that matched our initial probing + * (or our initial probing failed) Report appropriate error. + */ + dev_err(priv->dev, "Probing for EIP97/EIP19x failed - no such device (read %08x)\n", + version); + return -ENODEV; + } + + priv->hwconfig.hwver = EIP197_VERSION_MASK(version); + hwctg = version >> 28; + peid = version & 255; + + /* Detect EIP206 processing pipe */ + version = readl(EIP197_PE(priv) + EIP197_PE_VERSION(0)); + if (EIP197_REG_LO16(version) != EIP206_VERSION_LE) { + dev_err(priv->dev, "EIP%d: EIP206 not detected\n", peid); + return -ENODEV; + } + priv->hwconfig.ppver = EIP197_VERSION_MASK(version); + + /* Detect EIP96 packet engine and version */ + version = readl(EIP197_PE(priv) + EIP197_PE_EIP96_VERSION(0)); + if (EIP197_REG_LO16(version) != EIP96_VERSION_LE) { + dev_err(dev, "EIP%d: EIP96 not detected.\n", peid); + return -ENODEV; + } + priv->hwconfig.pever = EIP197_VERSION_MASK(version); + + hwopt = readl(EIP197_GLOBAL(priv) + EIP197_OPTIONS); + hiaopt = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_OPTIONS); + + priv->hwconfig.icever = 0; + priv->hwconfig.ocever = 0; + priv->hwconfig.psever = 0; + if (priv->flags & SAFEXCEL_HW_EIP197) { + /* EIP197 */ + peopt = readl(EIP197_PE(priv) + EIP197_PE_OPTIONS(0)); + + priv->hwconfig.hwdataw = (hiaopt >> EIP197_HWDATAW_OFFSET) & + EIP197_HWDATAW_MASK; + priv->hwconfig.hwcfsize = ((hiaopt >> EIP197_CFSIZE_OFFSET) & + EIP197_CFSIZE_MASK) + + EIP197_CFSIZE_ADJUST; + priv->hwconfig.hwrfsize = ((hiaopt >> EIP197_RFSIZE_OFFSET) & + EIP197_RFSIZE_MASK) + + EIP197_RFSIZE_ADJUST; + priv->hwconfig.hwnumpes = (hiaopt >> EIP197_N_PES_OFFSET) & + EIP197_N_PES_MASK; + priv->hwconfig.hwnumrings = (hiaopt >> EIP197_N_RINGS_OFFSET) & + EIP197_N_RINGS_MASK; + if (hiaopt & EIP197_HIA_OPT_HAS_PE_ARB) + priv->flags |= EIP197_PE_ARB; + if (EIP206_OPT_ICE_TYPE(peopt) == 1) { + priv->flags |= EIP197_ICE; + /* Detect ICE EIP207 class. engine and version */ + version = readl(EIP197_PE(priv) + + EIP197_PE_ICE_VERSION(0)); + if (EIP197_REG_LO16(version) != EIP207_VERSION_LE) { + dev_err(dev, "EIP%d: ICE EIP207 not detected.\n", + peid); + return -ENODEV; + } + priv->hwconfig.icever = EIP197_VERSION_MASK(version); + } + if (EIP206_OPT_OCE_TYPE(peopt) == 1) { + priv->flags |= EIP197_OCE; + /* Detect EIP96PP packet stream editor and version */ + version = readl(EIP197_PE(priv) + EIP197_PE_PSE_VERSION(0)); + if (EIP197_REG_LO16(version) != EIP96_VERSION_LE) { + dev_err(dev, "EIP%d: EIP96PP not detected.\n", peid); + return -ENODEV; + } + priv->hwconfig.psever = EIP197_VERSION_MASK(version); + /* Detect OCE EIP207 class. engine and version */ + version = readl(EIP197_PE(priv) + + EIP197_PE_ICE_VERSION(0)); + if (EIP197_REG_LO16(version) != EIP207_VERSION_LE) { + dev_err(dev, "EIP%d: OCE EIP207 not detected.\n", + peid); + return -ENODEV; + } + priv->hwconfig.ocever = EIP197_VERSION_MASK(version); + } + /* If not a full TRC, then assume simple TRC */ + if (!(hwopt & EIP197_OPT_HAS_TRC)) + priv->flags |= EIP197_SIMPLE_TRC; + /* EIP197 always has SOME form of TRC */ + priv->flags |= EIP197_TRC_CACHE; + } else { + /* EIP97 */ + priv->hwconfig.hwdataw = (hiaopt >> EIP197_HWDATAW_OFFSET) & + EIP97_HWDATAW_MASK; + priv->hwconfig.hwcfsize = (hiaopt >> EIP97_CFSIZE_OFFSET) & + EIP97_CFSIZE_MASK; + priv->hwconfig.hwrfsize = (hiaopt >> EIP97_RFSIZE_OFFSET) & + EIP97_RFSIZE_MASK; + priv->hwconfig.hwnumpes = 1; /* by definition */ + priv->hwconfig.hwnumrings = (hiaopt >> EIP197_N_RINGS_OFFSET) & + EIP197_N_RINGS_MASK; + } + + /* Scan for ring AIC's */ + for (i = 0; i < EIP197_MAX_RING_AIC; i++) { + version = readl(EIP197_HIA_AIC_R(priv) + + EIP197_HIA_AIC_R_VERSION(i)); + if (EIP197_REG_LO16(version) != EIP201_VERSION_LE) + break; + } + priv->hwconfig.hwnumraic = i; + /* Low-end EIP196 may not have any ring AIC's ... */ + if (!priv->hwconfig.hwnumraic) { + dev_err(priv->dev, "No ring interrupt controller present!\n"); + return -ENODEV; + } + + /* Get supported algorithms from EIP96 transform engine */ + priv->hwconfig.algo_flags = readl(EIP197_PE(priv) + + EIP197_PE_EIP96_OPTIONS(0)); + + /* Print single info line describing what we just detected */ + dev_info(priv->dev, "EIP%d:%x(%d,%d,%d,%d)-HIA:%x(%d,%d,%d),PE:%x/%x(alg:%08x)/%x/%x/%x\n", + peid, priv->hwconfig.hwver, hwctg, priv->hwconfig.hwnumpes, + priv->hwconfig.hwnumrings, priv->hwconfig.hwnumraic, + priv->hwconfig.hiaver, priv->hwconfig.hwdataw, + priv->hwconfig.hwcfsize, priv->hwconfig.hwrfsize, + priv->hwconfig.ppver, priv->hwconfig.pever, + priv->hwconfig.algo_flags, priv->hwconfig.icever, + priv->hwconfig.ocever, priv->hwconfig.psever); + + safexcel_configure(priv); + + if (IS_ENABLED(CONFIG_PCI) && priv->data->version == EIP197_DEVBRD) { + /* + * Request MSI vectors for global + 1 per ring - + * or just 1 for older dev images + */ + struct pci_dev *pci_pdev = pdev; + + ret = pci_alloc_irq_vectors(pci_pdev, + priv->config.rings + 1, + priv->config.rings + 1, + PCI_IRQ_MSI | PCI_IRQ_MSIX); + if (ret < 0) { + dev_err(dev, "Failed to allocate PCI MSI interrupts\n"); + return ret; + } + } + + /* Register the ring IRQ handlers and configure the rings */ + priv->ring = devm_kcalloc(dev, priv->config.rings, + sizeof(*priv->ring), + GFP_KERNEL); + if (!priv->ring) + return -ENOMEM; + + for (i = 0; i < priv->config.rings; i++) { + char wq_name[9] = {0}; + int irq; + struct safexcel_ring_irq_data *ring_irq; + + ret = safexcel_init_ring_descriptors(priv, + &priv->ring[i].cdr, + &priv->ring[i].rdr); + if (ret) { + dev_err(dev, "Failed to initialize rings\n"); + goto err_cleanup_rings; + } + + priv->ring[i].rdr_req = devm_kcalloc(dev, + EIP197_DEFAULT_RING_SIZE, + sizeof(*priv->ring[i].rdr_req), + GFP_KERNEL); + if (!priv->ring[i].rdr_req) { + ret = -ENOMEM; + goto err_cleanup_rings; + } + + ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL); + if (!ring_irq) { + ret = -ENOMEM; + goto err_cleanup_rings; + } + + ring_irq->priv = priv; + ring_irq->ring = i; + + irq = safexcel_request_ring_irq(pdev, + EIP197_IRQ_NUMBER(i, is_pci_dev), + is_pci_dev, + i, + safexcel_irq_ring, + safexcel_irq_ring_thread, + ring_irq); + if (irq < 0) { + dev_err(dev, "Failed to get IRQ ID for ring %d\n", i); + ret = irq; + goto err_cleanup_rings; + } + + priv->ring[i].irq = irq; + priv->ring[i].work_data.priv = priv; + priv->ring[i].work_data.ring = i; + INIT_WORK(&priv->ring[i].work_data.work, + safexcel_dequeue_work); + + snprintf(wq_name, 9, "wq_ring%d", i); + priv->ring[i].workqueue = + create_singlethread_workqueue(wq_name); + if (!priv->ring[i].workqueue) { + ret = -ENOMEM; + goto err_cleanup_rings; + } + + priv->ring[i].requests = 0; + priv->ring[i].busy = false; + + crypto_init_queue(&priv->ring[i].queue, + EIP197_DEFAULT_RING_SIZE); + + spin_lock_init(&priv->ring[i].lock); + spin_lock_init(&priv->ring[i].queue_lock); + } + + atomic_set(&priv->ring_used, 0); + + ret = safexcel_hw_init(priv); + if (ret) { + dev_err(dev, "HW init failed (%d)\n", ret); + goto err_cleanup_rings; + } + + ret = safexcel_register_algorithms(priv); + if (ret) { + dev_err(dev, "Failed to register algorithms (%d)\n", ret); + goto err_cleanup_rings; + } + + return 0; + +err_cleanup_rings: + for (i = 0; i < priv->config.rings; i++) { + if (priv->ring[i].irq) + irq_set_affinity_hint(priv->ring[i].irq, NULL); + if (priv->ring[i].workqueue) + destroy_workqueue(priv->ring[i].workqueue); + } + + return ret; +} + +static void safexcel_hw_reset_rings(struct safexcel_crypto_priv *priv) +{ + int i; + + for (i = 0; i < priv->config.rings; i++) { + /* clear any pending interrupt */ + writel(GENMASK(5, 0), EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT); + writel(GENMASK(7, 0), EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT); + + /* Reset the CDR base address */ + writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); + writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); + + /* Reset the RDR base address */ + writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); + writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); + } +} + +/* for Device Tree platform driver */ + +static int safexcel_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct safexcel_crypto_priv *priv; + int ret; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->dev = dev; + priv->data = (struct safexcel_priv_data *)of_device_get_match_data(dev); + + platform_set_drvdata(pdev, priv); + + priv->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(priv->base)) { + dev_err(dev, "failed to get resource\n"); + return PTR_ERR(priv->base); + } + + priv->clk = devm_clk_get(&pdev->dev, NULL); + ret = PTR_ERR_OR_ZERO(priv->clk); + /* The clock isn't mandatory */ + if (ret != -ENOENT) { + if (ret) + return ret; + + ret = clk_prepare_enable(priv->clk); + if (ret) { + dev_err(dev, "unable to enable clk (%d)\n", ret); + return ret; + } + } + + priv->reg_clk = devm_clk_get(&pdev->dev, "reg"); + ret = PTR_ERR_OR_ZERO(priv->reg_clk); + /* The clock isn't mandatory */ + if (ret != -ENOENT) { + if (ret) + goto err_core_clk; + + ret = clk_prepare_enable(priv->reg_clk); + if (ret) { + dev_err(dev, "unable to enable reg clk (%d)\n", ret); + goto err_core_clk; + } + } + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); + if (ret) + goto err_reg_clk; + + /* Generic EIP97/EIP197 device probing */ + ret = safexcel_probe_generic(pdev, priv, 0); + if (ret) + goto err_reg_clk; + + return 0; + +err_reg_clk: + clk_disable_unprepare(priv->reg_clk); +err_core_clk: + clk_disable_unprepare(priv->clk); + return ret; +} + +static void safexcel_remove(struct platform_device *pdev) +{ + struct safexcel_crypto_priv *priv = platform_get_drvdata(pdev); + int i; + + safexcel_unregister_algorithms(priv); + safexcel_hw_reset_rings(priv); + + clk_disable_unprepare(priv->reg_clk); + clk_disable_unprepare(priv->clk); + + for (i = 0; i < priv->config.rings; i++) { + irq_set_affinity_hint(priv->ring[i].irq, NULL); + destroy_workqueue(priv->ring[i].workqueue); + } +} + +static const struct safexcel_priv_data eip97ies_mrvl_data = { + .version = EIP97IES_MRVL, +}; + +static const struct safexcel_priv_data eip197b_mrvl_data = { + .version = EIP197B_MRVL, +}; + +static const struct safexcel_priv_data eip197d_mrvl_data = { + .version = EIP197D_MRVL, +}; + +static const struct safexcel_priv_data eip197_devbrd_data = { + .version = EIP197_DEVBRD, +}; + +static const struct safexcel_priv_data eip197c_mxl_data = { + .version = EIP197C_MXL, + .fw_little_endian = true, +}; + +static const struct of_device_id safexcel_of_match_table[] = { + { + .compatible = "inside-secure,safexcel-eip97ies", + .data = &eip97ies_mrvl_data, + }, + { + .compatible = "inside-secure,safexcel-eip197b", + .data = &eip197b_mrvl_data, + }, + { + .compatible = "inside-secure,safexcel-eip197d", + .data = &eip197d_mrvl_data, + }, + { + .compatible = "inside-secure,safexcel-eip197c-mxl", + .data = &eip197c_mxl_data, + }, + /* For backward compatibility and intended for generic use */ + { + .compatible = "inside-secure,safexcel-eip97", + .data = &eip97ies_mrvl_data, + }, + { + .compatible = "inside-secure,safexcel-eip197", + .data = &eip197b_mrvl_data, + }, + {}, +}; + +MODULE_DEVICE_TABLE(of, safexcel_of_match_table); + +static struct platform_driver crypto_safexcel = { + .probe = safexcel_probe, + .remove = safexcel_remove, + .driver = { + .name = "crypto-safexcel", + .of_match_table = safexcel_of_match_table, + }, +}; + +/* PCIE devices - i.e. Inside Secure development boards */ + +static int safexcel_pci_probe(struct pci_dev *pdev, + const struct pci_device_id *ent) +{ + struct device *dev = &pdev->dev; + struct safexcel_crypto_priv *priv; + void __iomem *pciebase; + int rc; + u32 val; + + dev_dbg(dev, "Probing PCIE device: vendor %04x, device %04x, subv %04x, subdev %04x, ctxt %lx\n", + ent->vendor, ent->device, ent->subvendor, + ent->subdevice, ent->driver_data); + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->dev = dev; + priv->data = (struct safexcel_priv_data *)ent->driver_data; + + pci_set_drvdata(pdev, priv); + + /* enable the device */ + rc = pcim_enable_device(pdev); + if (rc) { + dev_err(dev, "Failed to enable PCI device\n"); + return rc; + } + + /* take ownership of PCI BAR0 */ + rc = pcim_iomap_regions(pdev, 1, "crypto_safexcel"); + if (rc) { + dev_err(dev, "Failed to map IO region for BAR0\n"); + return rc; + } + priv->base = pcim_iomap_table(pdev)[0]; + + if (priv->data->version == EIP197_DEVBRD) { + dev_dbg(dev, "Device identified as FPGA based development board - applying HW reset\n"); + + rc = pcim_iomap_regions(pdev, 4, "crypto_safexcel"); + if (rc) { + dev_err(dev, "Failed to map IO region for BAR4\n"); + return rc; + } + + pciebase = pcim_iomap_table(pdev)[2]; + val = readl(pciebase + EIP197_XLX_IRQ_BLOCK_ID_ADDR); + if ((val >> 16) == EIP197_XLX_IRQ_BLOCK_ID_VALUE) { + dev_dbg(dev, "Detected Xilinx PCIE IRQ block version %d, multiple MSI support enabled\n", + (val & 0xff)); + + /* Setup MSI identity map mapping */ + writel(EIP197_XLX_USER_VECT_LUT0_IDENT, + pciebase + EIP197_XLX_USER_VECT_LUT0_ADDR); + writel(EIP197_XLX_USER_VECT_LUT1_IDENT, + pciebase + EIP197_XLX_USER_VECT_LUT1_ADDR); + writel(EIP197_XLX_USER_VECT_LUT2_IDENT, + pciebase + EIP197_XLX_USER_VECT_LUT2_ADDR); + writel(EIP197_XLX_USER_VECT_LUT3_IDENT, + pciebase + EIP197_XLX_USER_VECT_LUT3_ADDR); + + /* Enable all device interrupts */ + writel(GENMASK(31, 0), + pciebase + EIP197_XLX_USER_INT_ENB_MSK); + } else { + dev_err(dev, "Unrecognised IRQ block identifier %x\n", + val); + return -ENODEV; + } + + /* HW reset FPGA dev board */ + /* assert reset */ + writel(1, priv->base + EIP197_XLX_GPIO_BASE); + wmb(); /* maintain strict ordering for accesses here */ + /* deassert reset */ + writel(0, priv->base + EIP197_XLX_GPIO_BASE); + wmb(); /* maintain strict ordering for accesses here */ + } + + /* enable bus mastering */ + pci_set_master(pdev); + + /* Generic EIP97/EIP197 device probing */ + rc = safexcel_probe_generic(pdev, priv, 1); + return rc; +} + +static void safexcel_pci_remove(struct pci_dev *pdev) +{ + struct safexcel_crypto_priv *priv = pci_get_drvdata(pdev); + int i; + + safexcel_unregister_algorithms(priv); + + for (i = 0; i < priv->config.rings; i++) + destroy_workqueue(priv->ring[i].workqueue); + + safexcel_hw_reset_rings(priv); +} + +static const struct pci_device_id safexcel_pci_ids[] = { + { + PCI_DEVICE_SUB(PCI_VENDOR_ID_XILINX, 0x9038, + 0x16ae, 0xc522), + .driver_data = (kernel_ulong_t)&eip197_devbrd_data, + }, + {}, +}; + +MODULE_DEVICE_TABLE(pci, safexcel_pci_ids); + +static struct pci_driver safexcel_pci_driver = { + .name = "crypto-safexcel", + .id_table = safexcel_pci_ids, + .probe = safexcel_pci_probe, + .remove = safexcel_pci_remove, +}; + +static int __init safexcel_init(void) +{ + int ret; + + /* Register PCI driver */ + ret = pci_register_driver(&safexcel_pci_driver); + + /* Register platform driver */ + if (IS_ENABLED(CONFIG_OF) && !ret) { + ret = platform_driver_register(&crypto_safexcel); + if (ret) + pci_unregister_driver(&safexcel_pci_driver); + } + + return ret; +} + +static void __exit safexcel_exit(void) +{ + /* Unregister platform driver */ + if (IS_ENABLED(CONFIG_OF)) + platform_driver_unregister(&crypto_safexcel); + + /* Unregister PCI driver if successfully registered before */ + pci_unregister_driver(&safexcel_pci_driver); +} + +module_init(safexcel_init); +module_exit(safexcel_exit); + +MODULE_AUTHOR("Antoine Tenart "); +MODULE_AUTHOR("Ofer Heifetz "); +MODULE_AUTHOR("Igal Liberman "); +MODULE_DESCRIPTION("Support for SafeXcel cryptographic engines: EIP97 & EIP197"); +MODULE_LICENSE("GPL v2"); +MODULE_IMPORT_NS("CRYPTO_INTERNAL"); + +MODULE_FIRMWARE("ifpp.bin"); +MODULE_FIRMWARE("ipue.bin"); +MODULE_FIRMWARE("inside-secure/eip197b/ifpp.bin"); +MODULE_FIRMWARE("inside-secure/eip197b/ipue.bin"); +MODULE_FIRMWARE("inside-secure/eip197d/ifpp.bin"); +MODULE_FIRMWARE("inside-secure/eip197d/ipue.bin"); +MODULE_FIRMWARE("inside-secure/eip197_minifw/ifpp.bin"); +MODULE_FIRMWARE("inside-secure/eip197_minifw/ipue.bin"); diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h new file mode 100644 index 000000000..52fd460c0 --- /dev/null +++ b/drivers/crypto/inside-secure/safexcel.h @@ -0,0 +1,1002 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2017 Marvell + * + * Antoine Tenart + */ + +#ifndef __SAFEXCEL_H__ +#define __SAFEXCEL_H__ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define EIP197_HIA_VERSION_BE 0xca35 +#define EIP197_HIA_VERSION_LE 0x35ca +#define EIP97_VERSION_LE 0x9e61 +#define EIP196_VERSION_LE 0x3bc4 +#define EIP197_VERSION_LE 0x3ac5 +#define EIP96_VERSION_LE 0x9f60 +#define EIP201_VERSION_LE 0x36c9 +#define EIP206_VERSION_LE 0x31ce +#define EIP207_VERSION_LE 0x30cf +#define EIP197_REG_LO16(reg) (reg & 0xffff) +#define EIP197_REG_HI16(reg) ((reg >> 16) & 0xffff) +#define EIP197_VERSION_MASK(reg) ((reg >> 16) & 0xfff) +#define EIP197_VERSION_SWAP(reg) (((reg & 0xf0) << 4) | \ + ((reg >> 4) & 0xf0) | \ + ((reg >> 12) & 0xf)) + +/* EIP197 HIA OPTIONS ENCODING */ +#define EIP197_HIA_OPT_HAS_PE_ARB BIT(29) + +/* EIP206 OPTIONS ENCODING */ +#define EIP206_OPT_ICE_TYPE(n) ((n>>8)&3) +#define EIP206_OPT_OCE_TYPE(n) ((n>>10)&3) + +/* EIP197 OPTIONS ENCODING */ +#define EIP197_OPT_HAS_TRC BIT(31) + +/* Static configuration */ +#define EIP197_DEFAULT_RING_SIZE 400 +#define EIP197_EMB_TOKENS 4 /* Pad CD to 16 dwords */ +#define EIP197_MAX_TOKENS 16 +#define EIP197_MAX_RINGS 4 +#define EIP197_FETCH_DEPTH 2 +#define EIP197_MAX_BATCH_SZ 64 +#define EIP197_MAX_RING_AIC 14 + +#define EIP197_GFP_FLAGS(base) ((base).flags & CRYPTO_TFM_REQ_MAY_SLEEP ? \ + GFP_KERNEL : GFP_ATOMIC) + +/* Custom on-stack requests (for invalidation) */ +#define EIP197_SKCIPHER_REQ_SIZE sizeof(struct skcipher_request) + \ + sizeof(struct safexcel_cipher_req) +#define EIP197_AHASH_REQ_SIZE sizeof(struct ahash_request) + \ + sizeof(struct safexcel_ahash_req) +#define EIP197_AEAD_REQ_SIZE sizeof(struct aead_request) + \ + sizeof(struct safexcel_cipher_req) +#define EIP197_REQUEST_ON_STACK(name, type, size) \ + char __##name##_desc[size] CRYPTO_MINALIGN_ATTR; \ + struct type##_request *name = (void *)__##name##_desc + +/* Xilinx dev board base offsets */ +#define EIP197_XLX_GPIO_BASE 0x200000 +#define EIP197_XLX_IRQ_BLOCK_ID_ADDR 0x2000 +#define EIP197_XLX_IRQ_BLOCK_ID_VALUE 0x1fc2 +#define EIP197_XLX_USER_INT_ENB_MSK 0x2004 +#define EIP197_XLX_USER_INT_ENB_SET 0x2008 +#define EIP197_XLX_USER_INT_ENB_CLEAR 0x200c +#define EIP197_XLX_USER_INT_BLOCK 0x2040 +#define EIP197_XLX_USER_INT_PEND 0x2048 +#define EIP197_XLX_USER_VECT_LUT0_ADDR 0x2080 +#define EIP197_XLX_USER_VECT_LUT0_IDENT 0x03020100 +#define EIP197_XLX_USER_VECT_LUT1_ADDR 0x2084 +#define EIP197_XLX_USER_VECT_LUT1_IDENT 0x07060504 +#define EIP197_XLX_USER_VECT_LUT2_ADDR 0x2088 +#define EIP197_XLX_USER_VECT_LUT2_IDENT 0x0b0a0908 +#define EIP197_XLX_USER_VECT_LUT3_ADDR 0x208c +#define EIP197_XLX_USER_VECT_LUT3_IDENT 0x0f0e0d0c + +/* Helper defines for probe function */ +#define EIP197_IRQ_NUMBER(i, is_pci) (i + is_pci) + +/* Register base offsets */ +#define EIP197_HIA_AIC(priv) ((priv)->base + (priv)->offsets.hia_aic) +#define EIP197_HIA_AIC_G(priv) ((priv)->base + (priv)->offsets.hia_aic_g) +#define EIP197_HIA_AIC_R(priv) ((priv)->base + (priv)->offsets.hia_aic_r) +#define EIP197_HIA_AIC_xDR(priv) ((priv)->base + (priv)->offsets.hia_aic_xdr) +#define EIP197_HIA_DFE(priv) ((priv)->base + (priv)->offsets.hia_dfe) +#define EIP197_HIA_DFE_THR(priv) ((priv)->base + (priv)->offsets.hia_dfe_thr) +#define EIP197_HIA_DSE(priv) ((priv)->base + (priv)->offsets.hia_dse) +#define EIP197_HIA_DSE_THR(priv) ((priv)->base + (priv)->offsets.hia_dse_thr) +#define EIP197_HIA_GEN_CFG(priv) ((priv)->base + (priv)->offsets.hia_gen_cfg) +#define EIP197_PE(priv) ((priv)->base + (priv)->offsets.pe) +#define EIP197_GLOBAL(priv) ((priv)->base + (priv)->offsets.global) + +/* EIP197 base offsets */ +#define EIP197_HIA_AIC_BASE 0x90000 +#define EIP197_HIA_AIC_G_BASE 0x90000 +#define EIP197_HIA_AIC_R_BASE 0x90800 +#define EIP197_HIA_AIC_xDR_BASE 0x80000 +#define EIP197_HIA_DFE_BASE 0x8c000 +#define EIP197_HIA_DFE_THR_BASE 0x8c040 +#define EIP197_HIA_DSE_BASE 0x8d000 +#define EIP197_HIA_DSE_THR_BASE 0x8d040 +#define EIP197_HIA_GEN_CFG_BASE 0xf0000 +#define EIP197_PE_BASE 0xa0000 +#define EIP197_GLOBAL_BASE 0xf0000 + +/* EIP97 base offsets */ +#define EIP97_HIA_AIC_BASE 0x0 +#define EIP97_HIA_AIC_G_BASE 0x0 +#define EIP97_HIA_AIC_R_BASE 0x0 +#define EIP97_HIA_AIC_xDR_BASE 0x0 +#define EIP97_HIA_DFE_BASE 0xf000 +#define EIP97_HIA_DFE_THR_BASE 0xf200 +#define EIP97_HIA_DSE_BASE 0xf400 +#define EIP97_HIA_DSE_THR_BASE 0xf600 +#define EIP97_HIA_GEN_CFG_BASE 0x10000 +#define EIP97_PE_BASE 0x10000 +#define EIP97_GLOBAL_BASE 0x10000 + +/* CDR/RDR register offsets */ +#define EIP197_HIA_xDR_OFF(priv, r) (EIP197_HIA_AIC_xDR(priv) + (r) * 0x1000) +#define EIP197_HIA_CDR(priv, r) (EIP197_HIA_xDR_OFF(priv, r)) +#define EIP197_HIA_RDR(priv, r) (EIP197_HIA_xDR_OFF(priv, r) + 0x800) +#define EIP197_HIA_xDR_RING_BASE_ADDR_LO 0x0000 +#define EIP197_HIA_xDR_RING_BASE_ADDR_HI 0x0004 +#define EIP197_HIA_xDR_RING_SIZE 0x0018 +#define EIP197_HIA_xDR_DESC_SIZE 0x001c +#define EIP197_HIA_xDR_CFG 0x0020 +#define EIP197_HIA_xDR_DMA_CFG 0x0024 +#define EIP197_HIA_xDR_THRESH 0x0028 +#define EIP197_HIA_xDR_PREP_COUNT 0x002c +#define EIP197_HIA_xDR_PROC_COUNT 0x0030 +#define EIP197_HIA_xDR_PREP_PNTR 0x0034 +#define EIP197_HIA_xDR_PROC_PNTR 0x0038 +#define EIP197_HIA_xDR_STAT 0x003c + +/* register offsets */ +#define EIP197_HIA_DFE_CFG(n) (0x0000 + (128 * (n))) +#define EIP197_HIA_DFE_THR_CTRL(n) (0x0000 + (128 * (n))) +#define EIP197_HIA_DFE_THR_STAT(n) (0x0004 + (128 * (n))) +#define EIP197_HIA_DSE_CFG(n) (0x0000 + (128 * (n))) +#define EIP197_HIA_DSE_THR_CTRL(n) (0x0000 + (128 * (n))) +#define EIP197_HIA_DSE_THR_STAT(n) (0x0004 + (128 * (n))) +#define EIP197_HIA_RA_PE_CTRL(n) (0x0010 + (8 * (n))) +#define EIP197_HIA_RA_PE_STAT 0x0014 +#define EIP197_HIA_AIC_R_OFF(r) ((r) * 0x1000) +#define EIP197_HIA_AIC_R_ENABLE_CTRL(r) (0xe008 - EIP197_HIA_AIC_R_OFF(r)) +#define EIP197_HIA_AIC_R_ENABLED_STAT(r) (0xe010 - EIP197_HIA_AIC_R_OFF(r)) +#define EIP197_HIA_AIC_R_ACK(r) (0xe010 - EIP197_HIA_AIC_R_OFF(r)) +#define EIP197_HIA_AIC_R_ENABLE_CLR(r) (0xe014 - EIP197_HIA_AIC_R_OFF(r)) +#define EIP197_HIA_AIC_R_VERSION(r) (0xe01c - EIP197_HIA_AIC_R_OFF(r)) +#define EIP197_HIA_AIC_G_ENABLE_CTRL 0xf808 +#define EIP197_HIA_AIC_G_ENABLED_STAT 0xf810 +#define EIP197_HIA_AIC_G_ACK 0xf810 +#define EIP197_HIA_MST_CTRL 0xfff4 +#define EIP197_HIA_OPTIONS 0xfff8 +#define EIP197_HIA_VERSION 0xfffc +#define EIP197_PE_IN_DBUF_THRES(n) (0x0000 + (0x2000 * (n))) +#define EIP197_PE_IN_TBUF_THRES(n) (0x0100 + (0x2000 * (n))) +#define EIP197_PE_ICE_SCRATCH_RAM(n) (0x0800 + (0x2000 * (n))) +#define EIP197_PE_ICE_PUE_CTRL(n) (0x0c80 + (0x2000 * (n))) +#define EIP197_PE_ICE_PUTF_CTRL(n) (0x0d00 + (0x2000 * (n))) +#define EIP197_PE_ICE_SCRATCH_CTRL(n) (0x0d04 + (0x2000 * (n))) +#define EIP197_PE_ICE_FPP_CTRL(n) (0x0d80 + (0x2000 * (n))) +#define EIP197_PE_ICE_PPTF_CTRL(n) (0x0e00 + (0x2000 * (n))) +#define EIP197_PE_ICE_RAM_CTRL(n) (0x0ff0 + (0x2000 * (n))) +#define EIP197_PE_ICE_VERSION(n) (0x0ffc + (0x2000 * (n))) +#define EIP197_PE_EIP96_TOKEN_CTRL(n) (0x1000 + (0x2000 * (n))) +#define EIP197_PE_EIP96_FUNCTION_EN(n) (0x1004 + (0x2000 * (n))) +#define EIP197_PE_EIP96_CONTEXT_CTRL(n) (0x1008 + (0x2000 * (n))) +#define EIP197_PE_EIP96_CONTEXT_STAT(n) (0x100c + (0x2000 * (n))) +#define EIP197_PE_EIP96_TOKEN_CTRL2(n) (0x102c + (0x2000 * (n))) +#define EIP197_PE_EIP96_FUNCTION2_EN(n) (0x1030 + (0x2000 * (n))) +#define EIP197_PE_EIP96_OPTIONS(n) (0x13f8 + (0x2000 * (n))) +#define EIP197_PE_EIP96_VERSION(n) (0x13fc + (0x2000 * (n))) +#define EIP197_PE_OCE_VERSION(n) (0x1bfc + (0x2000 * (n))) +#define EIP197_PE_OUT_DBUF_THRES(n) (0x1c00 + (0x2000 * (n))) +#define EIP197_PE_OUT_TBUF_THRES(n) (0x1d00 + (0x2000 * (n))) +#define EIP197_PE_PSE_VERSION(n) (0x1efc + (0x2000 * (n))) +#define EIP197_PE_DEBUG(n) (0x1ff4 + (0x2000 * (n))) +#define EIP197_PE_OPTIONS(n) (0x1ff8 + (0x2000 * (n))) +#define EIP197_PE_VERSION(n) (0x1ffc + (0x2000 * (n))) +#define EIP197_MST_CTRL 0xfff4 +#define EIP197_OPTIONS 0xfff8 +#define EIP197_VERSION 0xfffc + +/* EIP197-specific registers, no indirection */ +#define EIP197_CLASSIFICATION_RAMS 0xe0000 +#define EIP197_TRC_CTRL 0xf0800 +#define EIP197_TRC_LASTRES 0xf0804 +#define EIP197_TRC_REGINDEX 0xf0808 +#define EIP197_TRC_PARAMS 0xf0820 +#define EIP197_TRC_FREECHAIN 0xf0824 +#define EIP197_TRC_PARAMS2 0xf0828 +#define EIP197_TRC_ECCCTRL 0xf0830 +#define EIP197_TRC_ECCSTAT 0xf0834 +#define EIP197_TRC_ECCADMINSTAT 0xf0838 +#define EIP197_TRC_ECCDATASTAT 0xf083c +#define EIP197_TRC_ECCDATA 0xf0840 +#define EIP197_STRC_CONFIG 0xf43f0 +#define EIP197_FLUE_CACHEBASE_LO(n) (0xf6000 + (32 * (n))) +#define EIP197_FLUE_CACHEBASE_HI(n) (0xf6004 + (32 * (n))) +#define EIP197_FLUE_CONFIG(n) (0xf6010 + (32 * (n))) +#define EIP197_FLUE_OFFSETS 0xf6808 +#define EIP197_FLUE_ARC4_OFFSET 0xf680c +#define EIP197_FLUE_IFC_LUT(n) (0xf6820 + (4 * (n))) +#define EIP197_CS_RAM_CTRL 0xf7ff0 + +/* EIP197_HIA_xDR_DESC_SIZE */ +#define EIP197_xDR_DESC_MODE_64BIT BIT(31) +#define EIP197_CDR_DESC_MODE_ADCP BIT(30) + +/* EIP197_HIA_xDR_DMA_CFG */ +#define EIP197_HIA_xDR_WR_RES_BUF BIT(22) +#define EIP197_HIA_xDR_WR_CTRL_BUF BIT(23) +#define EIP197_HIA_xDR_WR_OWN_BUF BIT(24) +#define EIP197_HIA_xDR_CFG_WR_CACHE(n) (((n) & 0x7) << 25) +#define EIP197_HIA_xDR_CFG_RD_CACHE(n) (((n) & 0x7) << 29) + +/* EIP197_HIA_CDR_THRESH */ +#define EIP197_HIA_CDR_THRESH_PROC_PKT(n) (n) +#define EIP197_HIA_CDR_THRESH_PROC_MODE BIT(22) +#define EIP197_HIA_CDR_THRESH_PKT_MODE BIT(23) +#define EIP197_HIA_CDR_THRESH_TIMEOUT(n) ((n) << 24) /* x256 clk cycles */ + +/* EIP197_HIA_RDR_THRESH */ +#define EIP197_HIA_RDR_THRESH_PROC_PKT(n) (n) +#define EIP197_HIA_RDR_THRESH_PKT_MODE BIT(23) +#define EIP197_HIA_RDR_THRESH_TIMEOUT(n) ((n) << 24) /* x256 clk cycles */ + +/* EIP197_HIA_xDR_PREP_COUNT */ +#define EIP197_xDR_PREP_CLR_COUNT BIT(31) + +/* EIP197_HIA_xDR_PROC_COUNT */ +#define EIP197_xDR_PROC_xD_PKT_OFFSET 24 +#define EIP197_xDR_PROC_xD_PKT_MASK GENMASK(6, 0) +#define EIP197_xDR_PROC_xD_PKT(n) ((n) << 24) +#define EIP197_xDR_PROC_CLR_COUNT BIT(31) + +/* EIP197_HIA_xDR_STAT */ +#define EIP197_xDR_DMA_ERR BIT(0) +#define EIP197_xDR_PREP_CMD_THRES BIT(1) +#define EIP197_xDR_ERR BIT(2) +#define EIP197_xDR_THRESH BIT(4) +#define EIP197_xDR_TIMEOUT BIT(5) + +#define EIP197_HIA_RA_PE_CTRL_RESET BIT(31) +#define EIP197_HIA_RA_PE_CTRL_EN BIT(30) + +/* EIP197_HIA_OPTIONS */ +#define EIP197_N_RINGS_OFFSET 0 +#define EIP197_N_RINGS_MASK GENMASK(3, 0) +#define EIP197_N_PES_OFFSET 4 +#define EIP197_N_PES_MASK GENMASK(4, 0) +#define EIP97_N_PES_MASK GENMASK(2, 0) +#define EIP197_HWDATAW_OFFSET 25 +#define EIP197_HWDATAW_MASK GENMASK(3, 0) +#define EIP97_HWDATAW_MASK GENMASK(2, 0) +#define EIP197_CFSIZE_OFFSET 9 +#define EIP197_CFSIZE_ADJUST 4 +#define EIP97_CFSIZE_OFFSET 8 +#define EIP197_CFSIZE_MASK GENMASK(2, 0) +#define EIP97_CFSIZE_MASK GENMASK(3, 0) +#define EIP197_RFSIZE_OFFSET 12 +#define EIP197_RFSIZE_ADJUST 4 +#define EIP97_RFSIZE_OFFSET 12 +#define EIP197_RFSIZE_MASK GENMASK(2, 0) +#define EIP97_RFSIZE_MASK GENMASK(3, 0) + +/* EIP197_HIA_AIC_R_ENABLE_CTRL */ +#define EIP197_CDR_IRQ(n) BIT((n) * 2) +#define EIP197_RDR_IRQ(n) BIT((n) * 2 + 1) + +/* EIP197_HIA_DFE/DSE_CFG */ +#define EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(n) ((n) << 0) +#define EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(n) (((n) & 0x7) << 4) +#define EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(n) ((n) << 8) +#define EIP197_HIA_DSE_CFG_ALWAYS_BUFFERABLE GENMASK(15, 14) +#define EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(n) ((n) << 16) +#define EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(n) (((n) & 0x7) << 20) +#define EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(n) ((n) << 24) +#define EIP197_HIA_DFE_CFG_DIS_DEBUG GENMASK(31, 29) +#define EIP197_HIA_DSE_CFG_EN_SINGLE_WR BIT(29) +#define EIP197_HIA_DSE_CFG_DIS_DEBUG GENMASK(31, 30) + +/* EIP197_HIA_DFE/DSE_THR_CTRL */ +#define EIP197_DxE_THR_CTRL_EN BIT(30) +#define EIP197_DxE_THR_CTRL_RESET_PE BIT(31) + +/* EIP197_PE_ICE_PUE/FPP_CTRL */ +#define EIP197_PE_ICE_UENG_START_OFFSET(n) ((n) << 16) +#define EIP197_PE_ICE_UENG_INIT_ALIGN_MASK 0x7ff0 +#define EIP197_PE_ICE_UENG_DEBUG_RESET BIT(3) + +/* EIP197_HIA_AIC_G_ENABLED_STAT */ +#define EIP197_G_IRQ_DFE(n) BIT((n) << 1) +#define EIP197_G_IRQ_DSE(n) BIT(((n) << 1) + 1) +#define EIP197_G_IRQ_RING BIT(16) +#define EIP197_G_IRQ_PE(n) BIT((n) + 20) + +/* EIP197_HIA_MST_CTRL */ +#define RD_CACHE_3BITS 0x5 +#define WR_CACHE_3BITS 0x3 +#define RD_CACHE_4BITS (RD_CACHE_3BITS << 1 | BIT(0)) +#define WR_CACHE_4BITS (WR_CACHE_3BITS << 1 | BIT(0)) +#define EIP197_MST_CTRL_RD_CACHE(n) (((n) & 0xf) << 0) +#define EIP197_MST_CTRL_WD_CACHE(n) (((n) & 0xf) << 4) +#define EIP197_MST_CTRL_TX_MAX_CMD(n) (((n) & 0xf) << 20) +#define EIP197_MST_CTRL_BYTE_SWAP BIT(24) +#define EIP197_MST_CTRL_NO_BYTE_SWAP BIT(25) +#define EIP197_MST_CTRL_BYTE_SWAP_BITS GENMASK(25, 24) + +/* EIP197_PE_IN_DBUF/TBUF_THRES */ +#define EIP197_PE_IN_xBUF_THRES_MIN(n) ((n) << 8) +#define EIP197_PE_IN_xBUF_THRES_MAX(n) ((n) << 12) + +/* EIP197_PE_OUT_DBUF_THRES */ +#define EIP197_PE_OUT_DBUF_THRES_MIN(n) ((n) << 0) +#define EIP197_PE_OUT_DBUF_THRES_MAX(n) ((n) << 4) + +/* EIP197_PE_ICE_SCRATCH_CTRL */ +#define EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER BIT(2) +#define EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN BIT(3) +#define EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS BIT(24) +#define EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS BIT(25) + +/* EIP197_PE_ICE_SCRATCH_RAM */ +#define EIP197_NUM_OF_SCRATCH_BLOCKS 32 + +/* EIP197_PE_ICE_PUE/FPP_CTRL */ +#define EIP197_PE_ICE_x_CTRL_SW_RESET BIT(0) +#define EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR BIT(14) +#define EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR BIT(15) + +/* EIP197_PE_ICE_RAM_CTRL */ +#define EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN BIT(0) +#define EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN BIT(1) + +/* EIP197_PE_EIP96_TOKEN_CTRL */ +#define EIP197_PE_EIP96_TOKEN_CTRL_CTX_UPDATES BIT(16) +#define EIP197_PE_EIP96_TOKEN_CTRL_NO_TOKEN_WAIT BIT(17) +#define EIP197_PE_EIP96_TOKEN_CTRL_ENABLE_TIMEOUT BIT(22) + +/* EIP197_PE_EIP96_FUNCTION_EN */ +#define EIP197_FUNCTION_ALL 0xffffffff + +/* EIP197_PE_EIP96_CONTEXT_CTRL */ +#define EIP197_CONTEXT_SIZE(n) (n) +#define EIP197_ADDRESS_MODE BIT(8) +#define EIP197_CONTROL_MODE BIT(9) + +/* EIP197_PE_EIP96_TOKEN_CTRL2 */ +#define EIP197_PE_EIP96_TOKEN_CTRL2_CTX_DONE BIT(3) + +/* EIP197_PE_DEBUG */ +#define EIP197_DEBUG_OCE_BYPASS BIT(1) + +/* EIP197_STRC_CONFIG */ +#define EIP197_STRC_CONFIG_INIT BIT(31) +#define EIP197_STRC_CONFIG_LARGE_REC(s) (s<<8) +#define EIP197_STRC_CONFIG_SMALL_REC(s) (s<<0) + +/* EIP197_FLUE_CONFIG */ +#define EIP197_FLUE_CONFIG_MAGIC 0xc7000004 + +/* Context Control */ +struct safexcel_context_record { + __le32 control0; + __le32 control1; + + __le32 data[40]; +} __packed; + +/* control0 */ +#define CONTEXT_CONTROL_TYPE_NULL_OUT 0x0 +#define CONTEXT_CONTROL_TYPE_NULL_IN 0x1 +#define CONTEXT_CONTROL_TYPE_HASH_OUT 0x2 +#define CONTEXT_CONTROL_TYPE_HASH_IN 0x3 +#define CONTEXT_CONTROL_TYPE_CRYPTO_OUT 0x4 +#define CONTEXT_CONTROL_TYPE_CRYPTO_IN 0x5 +#define CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT 0x6 +#define CONTEXT_CONTROL_TYPE_DECRYPT_HASH_IN 0x7 +#define CONTEXT_CONTROL_TYPE_HASH_ENCRYPT_OUT 0xe +#define CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN 0xf +#define CONTEXT_CONTROL_RESTART_HASH BIT(4) +#define CONTEXT_CONTROL_NO_FINISH_HASH BIT(5) +#define CONTEXT_CONTROL_SIZE(n) ((n) << 8) +#define CONTEXT_CONTROL_KEY_EN BIT(16) +#define CONTEXT_CONTROL_CRYPTO_ALG_DES (0x0 << 17) +#define CONTEXT_CONTROL_CRYPTO_ALG_3DES (0x2 << 17) +#define CONTEXT_CONTROL_CRYPTO_ALG_AES128 (0x5 << 17) +#define CONTEXT_CONTROL_CRYPTO_ALG_AES192 (0x6 << 17) +#define CONTEXT_CONTROL_CRYPTO_ALG_AES256 (0x7 << 17) +#define CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20 (0x8 << 17) +#define CONTEXT_CONTROL_CRYPTO_ALG_SM4 (0xd << 17) +#define CONTEXT_CONTROL_DIGEST_INITIAL (0x0 << 21) +#define CONTEXT_CONTROL_DIGEST_PRECOMPUTED (0x1 << 21) +#define CONTEXT_CONTROL_DIGEST_XCM (0x2 << 21) +#define CONTEXT_CONTROL_DIGEST_HMAC (0x3 << 21) +#define CONTEXT_CONTROL_CRYPTO_ALG_MD5 (0x0 << 23) +#define CONTEXT_CONTROL_CRYPTO_ALG_CRC32 (0x0 << 23) +#define CONTEXT_CONTROL_CRYPTO_ALG_SHA1 (0x2 << 23) +#define CONTEXT_CONTROL_CRYPTO_ALG_SHA224 (0x4 << 23) +#define CONTEXT_CONTROL_CRYPTO_ALG_SHA256 (0x3 << 23) +#define CONTEXT_CONTROL_CRYPTO_ALG_SHA384 (0x6 << 23) +#define CONTEXT_CONTROL_CRYPTO_ALG_SHA512 (0x5 << 23) +#define CONTEXT_CONTROL_CRYPTO_ALG_GHASH (0x4 << 23) +#define CONTEXT_CONTROL_CRYPTO_ALG_XCBC128 (0x1 << 23) +#define CONTEXT_CONTROL_CRYPTO_ALG_XCBC192 (0x2 << 23) +#define CONTEXT_CONTROL_CRYPTO_ALG_XCBC256 (0x3 << 23) +#define CONTEXT_CONTROL_CRYPTO_ALG_SM3 (0x7 << 23) +#define CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256 (0xb << 23) +#define CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224 (0xc << 23) +#define CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512 (0xd << 23) +#define CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384 (0xe << 23) +#define CONTEXT_CONTROL_CRYPTO_ALG_POLY1305 (0xf << 23) +#define CONTEXT_CONTROL_INV_FR (0x5 << 24) +#define CONTEXT_CONTROL_INV_TR (0x6 << 24) + +/* control1 */ +#define CONTEXT_CONTROL_CRYPTO_MODE_ECB (0 << 0) +#define CONTEXT_CONTROL_CRYPTO_MODE_CBC (1 << 0) +#define CONTEXT_CONTROL_CHACHA20_MODE_256_32 (2 << 0) +#define CONTEXT_CONTROL_CRYPTO_MODE_OFB (4 << 0) +#define CONTEXT_CONTROL_CRYPTO_MODE_CFB (5 << 0) +#define CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD (6 << 0) +#define CONTEXT_CONTROL_CRYPTO_MODE_XTS (7 << 0) +#define CONTEXT_CONTROL_CRYPTO_MODE_XCM ((6 << 0) | BIT(17)) +#define CONTEXT_CONTROL_CHACHA20_MODE_CALC_OTK (12 << 0) +#define CONTEXT_CONTROL_IV0 BIT(5) +#define CONTEXT_CONTROL_IV1 BIT(6) +#define CONTEXT_CONTROL_IV2 BIT(7) +#define CONTEXT_CONTROL_IV3 BIT(8) +#define CONTEXT_CONTROL_DIGEST_CNT BIT(9) +#define CONTEXT_CONTROL_COUNTER_MODE BIT(10) +#define CONTEXT_CONTROL_CRYPTO_STORE BIT(12) +#define CONTEXT_CONTROL_HASH_STORE BIT(19) + +#define EIP197_XCM_MODE_GCM 1 +#define EIP197_XCM_MODE_CCM 2 + +#define EIP197_AEAD_TYPE_IPSEC_ESP 2 +#define EIP197_AEAD_TYPE_IPSEC_ESP_GMAC 3 +#define EIP197_AEAD_IPSEC_IV_SIZE 8 +#define EIP197_AEAD_IPSEC_NONCE_SIZE 4 +#define EIP197_AEAD_IPSEC_COUNTER_SIZE 4 +#define EIP197_AEAD_IPSEC_CCM_NONCE_SIZE 3 + +/* The hash counter given to the engine in the context has a granularity of + * 64 bits. + */ +#define EIP197_COUNTER_BLOCK_SIZE 64 + +/* EIP197_CS_RAM_CTRL */ +#define EIP197_TRC_ENABLE_0 BIT(4) +#define EIP197_TRC_ENABLE_1 BIT(5) +#define EIP197_TRC_ENABLE_2 BIT(6) +#define EIP197_TRC_ENABLE_MASK GENMASK(6, 4) +#define EIP197_CS_BANKSEL_MASK GENMASK(14, 12) +#define EIP197_CS_BANKSEL_OFS 12 + +/* EIP197_TRC_PARAMS */ +#define EIP197_TRC_PARAMS_SW_RESET BIT(0) +#define EIP197_TRC_PARAMS_DATA_ACCESS BIT(2) +#define EIP197_TRC_PARAMS_HTABLE_SZ(x) ((x) << 4) +#define EIP197_TRC_PARAMS_BLK_TIMER_SPEED(x) ((x) << 10) +#define EIP197_TRC_PARAMS_RC_SZ_LARGE(n) ((n) << 18) + +/* EIP197_TRC_FREECHAIN */ +#define EIP197_TRC_FREECHAIN_HEAD_PTR(p) (p) +#define EIP197_TRC_FREECHAIN_TAIL_PTR(p) ((p) << 16) + +/* EIP197_TRC_PARAMS2 */ +#define EIP197_TRC_PARAMS2_HTABLE_PTR(p) (p) +#define EIP197_TRC_PARAMS2_RC_SZ_SMALL(n) ((n) << 18) + +/* Cache helpers */ +#define EIP197_MIN_DSIZE 1024 +#define EIP197_MIN_ASIZE 8 +#define EIP197_CS_TRC_REC_WC 64 +#define EIP197_CS_RC_SIZE (4 * sizeof(u32)) +#define EIP197_CS_RC_NEXT(x) (x) +#define EIP197_CS_RC_PREV(x) ((x) << 10) +#define EIP197_RC_NULL 0x3ff + +/* Result data */ +struct result_data_desc { + u32 packet_length:17; + u32 error_code:15; + + u32 bypass_length:4; + u32 e15:1; + u32 rsvd0:16; + u32 hash_bytes:1; + u32 hash_length:6; + u32 generic_bytes:1; + u32 checksum:1; + u32 next_header:1; + u32 length:1; + + u16 application_id; + u16 rsvd1; + + u32 rsvd2[5]; +} __packed; + + +/* Basic Result Descriptor format */ +struct safexcel_result_desc { + u32 particle_size:17; + u8 rsvd0:3; + u8 descriptor_overflow:1; + u8 buffer_overflow:1; + u8 last_seg:1; + u8 first_seg:1; + u16 result_size:8; + + u32 rsvd1; + + u32 data_lo; + u32 data_hi; +} __packed; + +/* + * The EIP(1)97 only needs to fetch the descriptor part of + * the result descriptor, not the result token part! + */ +#define EIP197_RD64_FETCH_SIZE (sizeof(struct safexcel_result_desc) /\ + sizeof(u32)) +#define EIP197_RD64_RESULT_SIZE (sizeof(struct result_data_desc) /\ + sizeof(u32)) + +struct safexcel_token { + u32 packet_length:17; + u8 stat:2; + u16 instructions:9; + u8 opcode:4; +} __packed; + +#define EIP197_TOKEN_HASH_RESULT_VERIFY BIT(16) + +#define EIP197_TOKEN_CTX_OFFSET(x) (x) +#define EIP197_TOKEN_DIRECTION_EXTERNAL BIT(11) +#define EIP197_TOKEN_EXEC_IF_SUCCESSFUL (0x1 << 12) + +#define EIP197_TOKEN_STAT_LAST_HASH BIT(0) +#define EIP197_TOKEN_STAT_LAST_PACKET BIT(1) +#define EIP197_TOKEN_OPCODE_DIRECTION 0x0 +#define EIP197_TOKEN_OPCODE_INSERT 0x2 +#define EIP197_TOKEN_OPCODE_NOOP EIP197_TOKEN_OPCODE_INSERT +#define EIP197_TOKEN_OPCODE_RETRIEVE 0x4 +#define EIP197_TOKEN_OPCODE_INSERT_REMRES 0xa +#define EIP197_TOKEN_OPCODE_VERIFY 0xd +#define EIP197_TOKEN_OPCODE_CTX_ACCESS 0xe +#define EIP197_TOKEN_OPCODE_BYPASS GENMASK(3, 0) + +static inline void eip197_noop_token(struct safexcel_token *token) +{ + token->opcode = EIP197_TOKEN_OPCODE_NOOP; + token->packet_length = BIT(2); + token->stat = 0; + token->instructions = 0; +} + +/* Instructions */ +#define EIP197_TOKEN_INS_INSERT_HASH_DIGEST 0x1c +#define EIP197_TOKEN_INS_ORIGIN_IV0 0x14 +#define EIP197_TOKEN_INS_ORIGIN_TOKEN 0x1b +#define EIP197_TOKEN_INS_ORIGIN_LEN(x) ((x) << 5) +#define EIP197_TOKEN_INS_TYPE_OUTPUT BIT(5) +#define EIP197_TOKEN_INS_TYPE_HASH BIT(6) +#define EIP197_TOKEN_INS_TYPE_CRYPTO BIT(7) +#define EIP197_TOKEN_INS_LAST BIT(8) + +/* Processing Engine Control Data */ +struct safexcel_control_data_desc { + u32 packet_length:17; + u16 options:13; + u8 type:2; + + u16 application_id; + u16 rsvd; + + u32 context_lo; + u32 context_hi; + + u32 control0; + u32 control1; + + u32 token[EIP197_EMB_TOKENS]; +} __packed; + +#define EIP197_OPTION_MAGIC_VALUE BIT(0) +#define EIP197_OPTION_64BIT_CTX BIT(1) +#define EIP197_OPTION_RC_AUTO (0x2 << 3) +#define EIP197_OPTION_CTX_CTRL_IN_CMD BIT(8) +#define EIP197_OPTION_2_TOKEN_IV_CMD GENMASK(11, 10) +#define EIP197_OPTION_4_TOKEN_IV_CMD GENMASK(11, 9) + +#define EIP197_TYPE_BCLA 0x0 +#define EIP197_TYPE_EXTENDED 0x3 +#define EIP197_CONTEXT_SMALL 0x2 +#define EIP197_CONTEXT_SIZE_MASK 0x3 + +/* Basic Command Descriptor format */ +struct safexcel_command_desc { + u32 particle_size:17; + u8 rsvd0:5; + u8 last_seg:1; + u8 first_seg:1; + u8 additional_cdata_size:8; + + u32 rsvd1; + + u32 data_lo; + u32 data_hi; + + u32 atok_lo; + u32 atok_hi; + + struct safexcel_control_data_desc control_data; +} __packed; + +#define EIP197_CD64_FETCH_SIZE (sizeof(struct safexcel_command_desc) /\ + sizeof(u32)) + +/* + * Internal structures & functions + */ + +#define EIP197_FW_TERMINAL_NOPS 2 +#define EIP197_FW_START_POLLCNT 16 +#define EIP197_FW_PUE_READY 0x14 +#define EIP197_FW_FPP_READY 0x18 + +enum eip197_fw { + FW_IFPP = 0, + FW_IPUE, + FW_NB +}; + +struct safexcel_desc_ring { + void *base; + void *shbase; + void *base_end; + void *shbase_end; + dma_addr_t base_dma; + dma_addr_t shbase_dma; + + /* write and read pointers */ + void *write; + void *shwrite; + void *read; + + /* descriptor element offset */ + unsigned int offset; + unsigned int shoffset; +}; + +enum safexcel_alg_type { + SAFEXCEL_ALG_TYPE_SKCIPHER, + SAFEXCEL_ALG_TYPE_AEAD, + SAFEXCEL_ALG_TYPE_AHASH, +}; + +struct safexcel_config { + u32 pes; + u32 rings; + + u32 cd_size; + u32 cd_offset; + u32 cdsh_offset; + + u32 rd_size; + u32 rd_offset; + u32 res_offset; +}; + +struct safexcel_work_data { + struct work_struct work; + struct safexcel_crypto_priv *priv; + int ring; +}; + +struct safexcel_ring { + spinlock_t lock; + + struct workqueue_struct *workqueue; + struct safexcel_work_data work_data; + + /* command/result rings */ + struct safexcel_desc_ring cdr; + struct safexcel_desc_ring rdr; + + /* result ring crypto API request */ + struct crypto_async_request **rdr_req; + + /* queue */ + struct crypto_queue queue; + spinlock_t queue_lock; + + /* Number of requests in the engine. */ + int requests; + + /* The ring is currently handling at least one request */ + bool busy; + + /* Store for current requests when bailing out of the dequeueing + * function when no enough resources are available. + */ + struct crypto_async_request *req; + struct crypto_async_request *backlog; + + /* irq of this ring */ + int irq; +}; + +/* EIP integration context flags */ +enum safexcel_eip_version { + /* Platform (EIP integration context) specifier */ + EIP97IES_MRVL, + EIP197B_MRVL, + EIP197D_MRVL, + EIP197_DEVBRD, + EIP197C_MXL, +}; + +struct safexcel_priv_data { + enum safexcel_eip_version version; + bool fw_little_endian; +}; + +/* Priority we use for advertising our algorithms */ +#define SAFEXCEL_CRA_PRIORITY 300 + +/* SM3 digest result for zero length message */ +#define EIP197_SM3_ZEROM_HASH "\x1A\xB2\x1D\x83\x55\xCF\xA1\x7F" \ + "\x8E\x61\x19\x48\x31\xE8\x1A\x8F" \ + "\x22\xBE\xC8\xC7\x28\xFE\xFB\x74" \ + "\x7E\xD0\x35\xEB\x50\x82\xAA\x2B" + +/* EIP algorithm presence flags */ +enum safexcel_eip_algorithms { + SAFEXCEL_ALG_BC0 = BIT(5), + SAFEXCEL_ALG_SM4 = BIT(6), + SAFEXCEL_ALG_SM3 = BIT(7), + SAFEXCEL_ALG_CHACHA20 = BIT(8), + SAFEXCEL_ALG_POLY1305 = BIT(9), + SAFEXCEL_SEQMASK_256 = BIT(10), + SAFEXCEL_SEQMASK_384 = BIT(11), + SAFEXCEL_ALG_AES = BIT(12), + SAFEXCEL_ALG_AES_XFB = BIT(13), + SAFEXCEL_ALG_DES = BIT(15), + SAFEXCEL_ALG_DES_XFB = BIT(16), + SAFEXCEL_ALG_ARC4 = BIT(18), + SAFEXCEL_ALG_AES_XTS = BIT(20), + SAFEXCEL_ALG_WIRELESS = BIT(21), + SAFEXCEL_ALG_MD5 = BIT(22), + SAFEXCEL_ALG_SHA1 = BIT(23), + SAFEXCEL_ALG_SHA2_256 = BIT(25), + SAFEXCEL_ALG_SHA2_512 = BIT(26), + SAFEXCEL_ALG_XCBC_MAC = BIT(27), + SAFEXCEL_ALG_CBC_MAC_ALL = BIT(29), + SAFEXCEL_ALG_GHASH = BIT(30), + SAFEXCEL_ALG_SHA3 = BIT(31), +}; + +struct safexcel_register_offsets { + u32 hia_aic; + u32 hia_aic_g; + u32 hia_aic_r; + u32 hia_aic_xdr; + u32 hia_dfe; + u32 hia_dfe_thr; + u32 hia_dse; + u32 hia_dse_thr; + u32 hia_gen_cfg; + u32 pe; + u32 global; +}; + +enum safexcel_flags { + EIP197_TRC_CACHE = BIT(0), + SAFEXCEL_HW_EIP197 = BIT(1), + EIP197_PE_ARB = BIT(2), + EIP197_ICE = BIT(3), + EIP197_SIMPLE_TRC = BIT(4), + EIP197_OCE = BIT(5), +}; + +struct safexcel_hwconfig { + enum safexcel_eip_algorithms algo_flags; + int hwver; + int hiaver; + int ppver; + int icever; + int pever; + int ocever; + int psever; + int hwdataw; + int hwcfsize; + int hwrfsize; + int hwnumpes; + int hwnumrings; + int hwnumraic; +}; + +struct safexcel_crypto_priv { + void __iomem *base; + struct device *dev; + struct clk *clk; + struct clk *reg_clk; + struct safexcel_config config; + + struct safexcel_priv_data *data; + struct safexcel_register_offsets offsets; + struct safexcel_hwconfig hwconfig; + u32 flags; + + /* context DMA pool */ + struct dma_pool *context_pool; + + atomic_t ring_used; + + struct safexcel_ring *ring; +}; + +struct safexcel_context { + int (*send)(struct crypto_async_request *req, int ring, + int *commands, int *results); + int (*handle_result)(struct safexcel_crypto_priv *priv, int ring, + struct crypto_async_request *req, bool *complete, + int *ret); + struct safexcel_context_record *ctxr; + struct safexcel_crypto_priv *priv; + dma_addr_t ctxr_dma; + + union { + __le32 le[SHA3_512_BLOCK_SIZE / 4]; + __be32 be[SHA3_512_BLOCK_SIZE / 4]; + u32 word[SHA3_512_BLOCK_SIZE / 4]; + u8 byte[SHA3_512_BLOCK_SIZE]; + } ipad, opad; + + int ring; + bool needs_inv; + bool exit_inv; +}; + +#define HASH_CACHE_SIZE SHA512_BLOCK_SIZE + +struct safexcel_ahash_export_state { + u64 len; + u64 processed; + + u32 digest; + + u32 state[SHA512_DIGEST_SIZE / sizeof(u32)]; + u8 cache[HASH_CACHE_SIZE]; +}; + +/* + * Template structure to describe the algorithms in order to register them. + * It also has the purpose to contain our private structure and is actually + * the only way I know in this framework to avoid having global pointers... + */ +struct safexcel_alg_template { + struct safexcel_crypto_priv *priv; + enum safexcel_alg_type type; + enum safexcel_eip_algorithms algo_mask; + union { + struct skcipher_alg skcipher; + struct aead_alg aead; + struct ahash_alg ahash; + } alg; +}; + +void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring); +int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv, + void *rdp); +void safexcel_complete(struct safexcel_crypto_priv *priv, int ring); +int safexcel_invalidate_cache(struct crypto_async_request *async, + struct safexcel_crypto_priv *priv, + dma_addr_t ctxr_dma, int ring); +int safexcel_init_ring_descriptors(struct safexcel_crypto_priv *priv, + struct safexcel_desc_ring *cdr, + struct safexcel_desc_ring *rdr); +int safexcel_select_ring(struct safexcel_crypto_priv *priv); +void *safexcel_ring_next_rptr(struct safexcel_crypto_priv *priv, + struct safexcel_desc_ring *ring); +void safexcel_ring_rollback_wptr(struct safexcel_crypto_priv *priv, + struct safexcel_desc_ring *ring); +struct safexcel_command_desc *safexcel_add_cdesc(struct safexcel_crypto_priv *priv, + int ring_id, + bool first, bool last, + dma_addr_t data, u32 len, + u32 full_data_len, + dma_addr_t context, + struct safexcel_token **atoken); +struct safexcel_result_desc *safexcel_add_rdesc(struct safexcel_crypto_priv *priv, + int ring_id, + bool first, bool last, + dma_addr_t data, u32 len); +int safexcel_ring_first_rdr_index(struct safexcel_crypto_priv *priv, + int ring); +int safexcel_ring_rdr_rdesc_index(struct safexcel_crypto_priv *priv, + int ring, + struct safexcel_result_desc *rdesc); +void safexcel_rdr_req_set(struct safexcel_crypto_priv *priv, + int ring, + struct safexcel_result_desc *rdesc, + struct crypto_async_request *req); +inline struct crypto_async_request * +safexcel_rdr_req_get(struct safexcel_crypto_priv *priv, int ring); +int safexcel_hmac_setkey(struct safexcel_context *base, const u8 *key, + unsigned int keylen, const char *alg, + unsigned int state_sz); + +/* available algorithms */ +extern struct safexcel_alg_template safexcel_alg_ecb_des; +extern struct safexcel_alg_template safexcel_alg_cbc_des; +extern struct safexcel_alg_template safexcel_alg_ecb_des3_ede; +extern struct safexcel_alg_template safexcel_alg_cbc_des3_ede; +extern struct safexcel_alg_template safexcel_alg_ecb_aes; +extern struct safexcel_alg_template safexcel_alg_cbc_aes; +extern struct safexcel_alg_template safexcel_alg_ctr_aes; +extern struct safexcel_alg_template safexcel_alg_md5; +extern struct safexcel_alg_template safexcel_alg_sha1; +extern struct safexcel_alg_template safexcel_alg_sha224; +extern struct safexcel_alg_template safexcel_alg_sha256; +extern struct safexcel_alg_template safexcel_alg_sha384; +extern struct safexcel_alg_template safexcel_alg_sha512; +extern struct safexcel_alg_template safexcel_alg_hmac_md5; +extern struct safexcel_alg_template safexcel_alg_hmac_sha1; +extern struct safexcel_alg_template safexcel_alg_hmac_sha224; +extern struct safexcel_alg_template safexcel_alg_hmac_sha256; +extern struct safexcel_alg_template safexcel_alg_hmac_sha384; +extern struct safexcel_alg_template safexcel_alg_hmac_sha512; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_md5_cbc_aes; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_aes; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_aes; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_aes; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_md5_ctr_aes; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_aes; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_ctr_aes; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_ctr_aes; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_ctr_aes; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_ctr_aes; +extern struct safexcel_alg_template safexcel_alg_xts_aes; +extern struct safexcel_alg_template safexcel_alg_gcm; +extern struct safexcel_alg_template safexcel_alg_ccm; +extern struct safexcel_alg_template safexcel_alg_cbcmac; +extern struct safexcel_alg_template safexcel_alg_xcbcmac; +extern struct safexcel_alg_template safexcel_alg_cmac; +extern struct safexcel_alg_template safexcel_alg_chacha20; +extern struct safexcel_alg_template safexcel_alg_chachapoly; +extern struct safexcel_alg_template safexcel_alg_chachapoly_esp; +extern struct safexcel_alg_template safexcel_alg_sm3; +extern struct safexcel_alg_template safexcel_alg_hmac_sm3; +extern struct safexcel_alg_template safexcel_alg_ecb_sm4; +extern struct safexcel_alg_template safexcel_alg_cbc_sm4; +extern struct safexcel_alg_template safexcel_alg_ctr_sm4; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_sm4; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_cbc_sm4; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_sm4; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_ctr_sm4; +extern struct safexcel_alg_template safexcel_alg_sha3_224; +extern struct safexcel_alg_template safexcel_alg_sha3_256; +extern struct safexcel_alg_template safexcel_alg_sha3_384; +extern struct safexcel_alg_template safexcel_alg_sha3_512; +extern struct safexcel_alg_template safexcel_alg_hmac_sha3_224; +extern struct safexcel_alg_template safexcel_alg_hmac_sha3_256; +extern struct safexcel_alg_template safexcel_alg_hmac_sha3_384; +extern struct safexcel_alg_template safexcel_alg_hmac_sha3_512; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_md5_cbc_des3_ede; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des3_ede; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des3_ede; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des3_ede; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des3_ede; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des3_ede; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_md5_cbc_des; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des; +extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des; +extern struct safexcel_alg_template safexcel_alg_rfc4106_gcm; +extern struct safexcel_alg_template safexcel_alg_rfc4543_gcm; +extern struct safexcel_alg_template safexcel_alg_rfc4309_ccm; + +#endif diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c new file mode 100644 index 000000000..a8349b684 --- /dev/null +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -0,0 +1,3758 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2017 Marvell + * + * Antoine Tenart + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "safexcel.h" + +enum safexcel_cipher_direction { + SAFEXCEL_ENCRYPT, + SAFEXCEL_DECRYPT, +}; + +enum safexcel_cipher_alg { + SAFEXCEL_DES, + SAFEXCEL_3DES, + SAFEXCEL_AES, + SAFEXCEL_CHACHA20, + SAFEXCEL_SM4, +}; + +struct safexcel_cipher_ctx { + struct safexcel_context base; + struct safexcel_crypto_priv *priv; + + u32 mode; + enum safexcel_cipher_alg alg; + u8 aead; /* !=0=AEAD, 2=IPSec ESP AEAD, 3=IPsec ESP GMAC */ + u8 xcm; /* 0=authenc, 1=GCM, 2 reserved for CCM */ + u8 aadskip; + u8 blocksz; + u32 ivmask; + u32 ctrinit; + + __le32 key[16]; + u32 nonce; + unsigned int key_len, xts; + + /* All the below is AEAD specific */ + u32 hash_alg; + u32 state_sz; + + struct crypto_aead *fback; +}; + +struct safexcel_cipher_req { + enum safexcel_cipher_direction direction; + /* Number of result descriptors associated to the request */ + unsigned int rdescs; + bool needs_inv; + int nr_src, nr_dst; +}; + +static int safexcel_skcipher_iv(struct safexcel_cipher_ctx *ctx, u8 *iv, + struct safexcel_command_desc *cdesc) +{ + if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) { + cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; + /* 32 bit nonce */ + cdesc->control_data.token[0] = ctx->nonce; + /* 64 bit IV part */ + memcpy(&cdesc->control_data.token[1], iv, 8); + /* 32 bit counter, start at 0 or 1 (big endian!) */ + cdesc->control_data.token[3] = + (__force u32)cpu_to_be32(ctx->ctrinit); + return 4; + } + if (ctx->alg == SAFEXCEL_CHACHA20) { + cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; + /* 96 bit nonce part */ + memcpy(&cdesc->control_data.token[0], &iv[4], 12); + /* 32 bit counter */ + cdesc->control_data.token[3] = *(u32 *)iv; + return 4; + } + + cdesc->control_data.options |= ctx->ivmask; + memcpy(cdesc->control_data.token, iv, ctx->blocksz); + return ctx->blocksz / sizeof(u32); +} + +static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, + struct safexcel_command_desc *cdesc, + struct safexcel_token *atoken, + u32 length) +{ + struct safexcel_token *token; + int ivlen; + + ivlen = safexcel_skcipher_iv(ctx, iv, cdesc); + if (ivlen == 4) { + /* No space in cdesc, instruction moves to atoken */ + cdesc->additional_cdata_size = 1; + token = atoken; + } else { + /* Everything fits in cdesc */ + token = (struct safexcel_token *)(cdesc->control_data.token + 2); + /* Need to pad with NOP */ + eip197_noop_token(&token[1]); + } + + token->opcode = EIP197_TOKEN_OPCODE_DIRECTION; + token->packet_length = length; + token->stat = EIP197_TOKEN_STAT_LAST_PACKET | + EIP197_TOKEN_STAT_LAST_HASH; + token->instructions = EIP197_TOKEN_INS_LAST | + EIP197_TOKEN_INS_TYPE_CRYPTO | + EIP197_TOKEN_INS_TYPE_OUTPUT; +} + +static void safexcel_aead_iv(struct safexcel_cipher_ctx *ctx, u8 *iv, + struct safexcel_command_desc *cdesc) +{ + if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD || + ctx->aead & EIP197_AEAD_TYPE_IPSEC_ESP) { /* _ESP and _ESP_GMAC */ + /* 32 bit nonce */ + cdesc->control_data.token[0] = ctx->nonce; + /* 64 bit IV part */ + memcpy(&cdesc->control_data.token[1], iv, 8); + /* 32 bit counter, start at 0 or 1 (big endian!) */ + cdesc->control_data.token[3] = + (__force u32)cpu_to_be32(ctx->ctrinit); + return; + } + if (ctx->xcm == EIP197_XCM_MODE_GCM || ctx->alg == SAFEXCEL_CHACHA20) { + /* 96 bit IV part */ + memcpy(&cdesc->control_data.token[0], iv, 12); + /* 32 bit counter, start at 0 or 1 (big endian!) */ + cdesc->control_data.token[3] = + (__force u32)cpu_to_be32(ctx->ctrinit); + return; + } + /* CBC */ + memcpy(cdesc->control_data.token, iv, ctx->blocksz); +} + +static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, + struct safexcel_command_desc *cdesc, + struct safexcel_token *atoken, + enum safexcel_cipher_direction direction, + u32 cryptlen, u32 assoclen, u32 digestsize) +{ + struct safexcel_token *aadref; + int atoksize = 2; /* Start with minimum size */ + int assocadj = assoclen - ctx->aadskip, aadalign; + + /* Always 4 dwords of embedded IV for AEAD modes */ + cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; + + if (direction == SAFEXCEL_DECRYPT) + cryptlen -= digestsize; + + if (unlikely(ctx->xcm == EIP197_XCM_MODE_CCM)) { + /* Construct IV block B0 for the CBC-MAC */ + u8 *final_iv = (u8 *)cdesc->control_data.token; + u8 *cbcmaciv = (u8 *)&atoken[1]; + __le32 *aadlen = (__le32 *)&atoken[5]; + + if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { + /* Length + nonce */ + cdesc->control_data.token[0] = ctx->nonce; + /* Fixup flags byte */ + *(__le32 *)cbcmaciv = + cpu_to_le32(ctx->nonce | + ((assocadj > 0) << 6) | + ((digestsize - 2) << 2)); + /* 64 bit IV part */ + memcpy(&cdesc->control_data.token[1], iv, 8); + memcpy(cbcmaciv + 4, iv, 8); + /* Start counter at 0 */ + cdesc->control_data.token[3] = 0; + /* Message length */ + *(__be32 *)(cbcmaciv + 12) = cpu_to_be32(cryptlen); + } else { + /* Variable length IV part */ + memcpy(final_iv, iv, 15 - iv[0]); + memcpy(cbcmaciv, iv, 15 - iv[0]); + /* Start variable length counter at 0 */ + memset(final_iv + 15 - iv[0], 0, iv[0] + 1); + memset(cbcmaciv + 15 - iv[0], 0, iv[0] - 1); + /* fixup flags byte */ + cbcmaciv[0] |= ((assocadj > 0) << 6) | + ((digestsize - 2) << 2); + /* insert lower 2 bytes of message length */ + cbcmaciv[14] = cryptlen >> 8; + cbcmaciv[15] = cryptlen & 255; + } + + atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; + atoken->packet_length = AES_BLOCK_SIZE + + ((assocadj > 0) << 1); + atoken->stat = 0; + atoken->instructions = EIP197_TOKEN_INS_ORIGIN_TOKEN | + EIP197_TOKEN_INS_TYPE_HASH; + + if (likely(assocadj)) { + *aadlen = cpu_to_le32((assocadj >> 8) | + (assocadj & 255) << 8); + atoken += 6; + atoksize += 7; + } else { + atoken += 5; + atoksize += 6; + } + + /* Process AAD data */ + aadref = atoken; + atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; + atoken->packet_length = assocadj; + atoken->stat = 0; + atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH; + atoken++; + + /* For CCM only, align AAD data towards hash engine */ + atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; + aadalign = (assocadj + 2) & 15; + atoken->packet_length = assocadj && aadalign ? + 16 - aadalign : + 0; + if (likely(cryptlen)) { + atoken->stat = 0; + atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH; + } else { + atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; + atoken->instructions = EIP197_TOKEN_INS_LAST | + EIP197_TOKEN_INS_TYPE_HASH; + } + } else { + safexcel_aead_iv(ctx, iv, cdesc); + + /* Process AAD data */ + aadref = atoken; + atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; + atoken->packet_length = assocadj; + atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; + atoken->instructions = EIP197_TOKEN_INS_LAST | + EIP197_TOKEN_INS_TYPE_HASH; + } + atoken++; + + if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { + /* For ESP mode (and not GMAC), skip over the IV */ + atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; + atoken->packet_length = EIP197_AEAD_IPSEC_IV_SIZE; + atoken->stat = 0; + atoken->instructions = 0; + atoken++; + atoksize++; + } else if (unlikely(ctx->alg == SAFEXCEL_CHACHA20 && + direction == SAFEXCEL_DECRYPT)) { + /* Poly-chacha decryption needs a dummy NOP here ... */ + atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; + atoken->packet_length = 16; /* According to Op Manual */ + atoken->stat = 0; + atoken->instructions = 0; + atoken++; + atoksize++; + } + + if (ctx->xcm) { + /* For GCM and CCM, obtain enc(Y0) */ + atoken->opcode = EIP197_TOKEN_OPCODE_INSERT_REMRES; + atoken->packet_length = 0; + atoken->stat = 0; + atoken->instructions = AES_BLOCK_SIZE; + atoken++; + + atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; + atoken->packet_length = AES_BLOCK_SIZE; + atoken->stat = 0; + atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | + EIP197_TOKEN_INS_TYPE_CRYPTO; + atoken++; + atoksize += 2; + } + + if (likely(cryptlen || ctx->alg == SAFEXCEL_CHACHA20)) { + /* Fixup stat field for AAD direction instruction */ + aadref->stat = 0; + + /* Process crypto data */ + atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; + atoken->packet_length = cryptlen; + + if (unlikely(ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC)) { + /* Fixup instruction field for AAD dir instruction */ + aadref->instructions = EIP197_TOKEN_INS_TYPE_HASH; + + /* Do not send to crypt engine in case of GMAC */ + atoken->instructions = EIP197_TOKEN_INS_LAST | + EIP197_TOKEN_INS_TYPE_HASH | + EIP197_TOKEN_INS_TYPE_OUTPUT; + } else { + atoken->instructions = EIP197_TOKEN_INS_LAST | + EIP197_TOKEN_INS_TYPE_CRYPTO | + EIP197_TOKEN_INS_TYPE_HASH | + EIP197_TOKEN_INS_TYPE_OUTPUT; + } + + cryptlen &= 15; + if (unlikely(ctx->xcm == EIP197_XCM_MODE_CCM && cryptlen)) { + atoken->stat = 0; + /* For CCM only, pad crypto data to the hash engine */ + atoken++; + atoksize++; + atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; + atoken->packet_length = 16 - cryptlen; + atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; + atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH; + } else { + atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; + } + atoken++; + atoksize++; + } + + if (direction == SAFEXCEL_ENCRYPT) { + /* Append ICV */ + atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; + atoken->packet_length = digestsize; + atoken->stat = EIP197_TOKEN_STAT_LAST_HASH | + EIP197_TOKEN_STAT_LAST_PACKET; + atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | + EIP197_TOKEN_INS_INSERT_HASH_DIGEST; + } else { + /* Extract ICV */ + atoken->opcode = EIP197_TOKEN_OPCODE_RETRIEVE; + atoken->packet_length = digestsize; + atoken->stat = EIP197_TOKEN_STAT_LAST_HASH | + EIP197_TOKEN_STAT_LAST_PACKET; + atoken->instructions = EIP197_TOKEN_INS_INSERT_HASH_DIGEST; + atoken++; + atoksize++; + + /* Verify ICV */ + atoken->opcode = EIP197_TOKEN_OPCODE_VERIFY; + atoken->packet_length = digestsize | + EIP197_TOKEN_HASH_RESULT_VERIFY; + atoken->stat = EIP197_TOKEN_STAT_LAST_HASH | + EIP197_TOKEN_STAT_LAST_PACKET; + atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT; + } + + /* Fixup length of the token in the command descriptor */ + cdesc->additional_cdata_size = atoksize; +} + +static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm, + const u8 *key, unsigned int len) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_crypto_priv *priv = ctx->base.priv; + struct crypto_aes_ctx aes; + int ret, i; + + ret = aes_expandkey(&aes, key, len); + if (ret) + return ret; + + if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { + for (i = 0; i < len / sizeof(u32); i++) { + if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { + ctx->base.needs_inv = true; + break; + } + } + } + + for (i = 0; i < len / sizeof(u32); i++) + ctx->key[i] = cpu_to_le32(aes.key_enc[i]); + + ctx->key_len = len; + + memzero_explicit(&aes, sizeof(aes)); + return 0; +} + +static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key, + unsigned int len) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_crypto_priv *priv = ctx->base.priv; + struct crypto_authenc_keys keys; + struct crypto_aes_ctx aes; + int err = -EINVAL, i; + const char *alg; + + if (unlikely(crypto_authenc_extractkeys(&keys, key, len))) + goto badkey; + + if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) { + /* Must have at least space for the nonce here */ + if (unlikely(keys.enckeylen < CTR_RFC3686_NONCE_SIZE)) + goto badkey; + /* last 4 bytes of key are the nonce! */ + ctx->nonce = *(u32 *)(keys.enckey + keys.enckeylen - + CTR_RFC3686_NONCE_SIZE); + /* exclude the nonce here */ + keys.enckeylen -= CTR_RFC3686_NONCE_SIZE; + } + + /* Encryption key */ + switch (ctx->alg) { + case SAFEXCEL_DES: + err = verify_aead_des_key(ctfm, keys.enckey, keys.enckeylen); + if (unlikely(err)) + goto badkey; + break; + case SAFEXCEL_3DES: + err = verify_aead_des3_key(ctfm, keys.enckey, keys.enckeylen); + if (unlikely(err)) + goto badkey; + break; + case SAFEXCEL_AES: + err = aes_expandkey(&aes, keys.enckey, keys.enckeylen); + if (unlikely(err)) + goto badkey; + break; + case SAFEXCEL_SM4: + if (unlikely(keys.enckeylen != SM4_KEY_SIZE)) + goto badkey; + break; + default: + dev_err(priv->dev, "aead: unsupported cipher algorithm\n"); + goto badkey; + } + + if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { + for (i = 0; i < keys.enckeylen / sizeof(u32); i++) { + if (le32_to_cpu(ctx->key[i]) != + ((u32 *)keys.enckey)[i]) { + ctx->base.needs_inv = true; + break; + } + } + } + + /* Auth key */ + switch (ctx->hash_alg) { + case CONTEXT_CONTROL_CRYPTO_ALG_MD5: + alg = "safexcel-md5"; + break; + case CONTEXT_CONTROL_CRYPTO_ALG_SHA1: + alg = "safexcel-sha1"; + break; + case CONTEXT_CONTROL_CRYPTO_ALG_SHA224: + alg = "safexcel-sha224"; + break; + case CONTEXT_CONTROL_CRYPTO_ALG_SHA256: + alg = "safexcel-sha256"; + break; + case CONTEXT_CONTROL_CRYPTO_ALG_SHA384: + alg = "safexcel-sha384"; + break; + case CONTEXT_CONTROL_CRYPTO_ALG_SHA512: + alg = "safexcel-sha512"; + break; + case CONTEXT_CONTROL_CRYPTO_ALG_SM3: + alg = "safexcel-sm3"; + break; + default: + dev_err(priv->dev, "aead: unsupported hash algorithm\n"); + goto badkey; + } + + if (safexcel_hmac_setkey(&ctx->base, keys.authkey, keys.authkeylen, + alg, ctx->state_sz)) + goto badkey; + + /* Now copy the keys into the context */ + for (i = 0; i < keys.enckeylen / sizeof(u32); i++) + ctx->key[i] = cpu_to_le32(((u32 *)keys.enckey)[i]); + ctx->key_len = keys.enckeylen; + + memzero_explicit(&keys, sizeof(keys)); + return 0; + +badkey: + memzero_explicit(&keys, sizeof(keys)); + return err; +} + +static int safexcel_context_control(struct safexcel_cipher_ctx *ctx, + struct crypto_async_request *async, + struct safexcel_cipher_req *sreq, + struct safexcel_command_desc *cdesc) +{ + struct safexcel_crypto_priv *priv = ctx->base.priv; + int ctrl_size = ctx->key_len / sizeof(u32); + + cdesc->control_data.control1 = ctx->mode; + + if (ctx->aead) { + /* Take in account the ipad+opad digests */ + if (ctx->xcm) { + ctrl_size += ctx->state_sz / sizeof(u32); + cdesc->control_data.control0 = + CONTEXT_CONTROL_KEY_EN | + CONTEXT_CONTROL_DIGEST_XCM | + ctx->hash_alg | + CONTEXT_CONTROL_SIZE(ctrl_size); + } else if (ctx->alg == SAFEXCEL_CHACHA20) { + /* Chacha20-Poly1305 */ + cdesc->control_data.control0 = + CONTEXT_CONTROL_KEY_EN | + CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20 | + (sreq->direction == SAFEXCEL_ENCRYPT ? + CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT : + CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN) | + ctx->hash_alg | + CONTEXT_CONTROL_SIZE(ctrl_size); + return 0; + } else { + ctrl_size += ctx->state_sz / sizeof(u32) * 2; + cdesc->control_data.control0 = + CONTEXT_CONTROL_KEY_EN | + CONTEXT_CONTROL_DIGEST_HMAC | + ctx->hash_alg | + CONTEXT_CONTROL_SIZE(ctrl_size); + } + + if (sreq->direction == SAFEXCEL_ENCRYPT && + (ctx->xcm == EIP197_XCM_MODE_CCM || + ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC)) + cdesc->control_data.control0 |= + CONTEXT_CONTROL_TYPE_HASH_ENCRYPT_OUT; + else if (sreq->direction == SAFEXCEL_ENCRYPT) + cdesc->control_data.control0 |= + CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT; + else if (ctx->xcm == EIP197_XCM_MODE_CCM) + cdesc->control_data.control0 |= + CONTEXT_CONTROL_TYPE_DECRYPT_HASH_IN; + else + cdesc->control_data.control0 |= + CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN; + } else { + if (sreq->direction == SAFEXCEL_ENCRYPT) + cdesc->control_data.control0 = + CONTEXT_CONTROL_TYPE_CRYPTO_OUT | + CONTEXT_CONTROL_KEY_EN | + CONTEXT_CONTROL_SIZE(ctrl_size); + else + cdesc->control_data.control0 = + CONTEXT_CONTROL_TYPE_CRYPTO_IN | + CONTEXT_CONTROL_KEY_EN | + CONTEXT_CONTROL_SIZE(ctrl_size); + } + + if (ctx->alg == SAFEXCEL_DES) { + cdesc->control_data.control0 |= + CONTEXT_CONTROL_CRYPTO_ALG_DES; + } else if (ctx->alg == SAFEXCEL_3DES) { + cdesc->control_data.control0 |= + CONTEXT_CONTROL_CRYPTO_ALG_3DES; + } else if (ctx->alg == SAFEXCEL_AES) { + switch (ctx->key_len >> ctx->xts) { + case AES_KEYSIZE_128: + cdesc->control_data.control0 |= + CONTEXT_CONTROL_CRYPTO_ALG_AES128; + break; + case AES_KEYSIZE_192: + cdesc->control_data.control0 |= + CONTEXT_CONTROL_CRYPTO_ALG_AES192; + break; + case AES_KEYSIZE_256: + cdesc->control_data.control0 |= + CONTEXT_CONTROL_CRYPTO_ALG_AES256; + break; + default: + dev_err(priv->dev, "aes keysize not supported: %u\n", + ctx->key_len >> ctx->xts); + return -EINVAL; + } + } else if (ctx->alg == SAFEXCEL_CHACHA20) { + cdesc->control_data.control0 |= + CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20; + } else if (ctx->alg == SAFEXCEL_SM4) { + cdesc->control_data.control0 |= + CONTEXT_CONTROL_CRYPTO_ALG_SM4; + } + + return 0; +} + +static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int ring, + struct crypto_async_request *async, + struct scatterlist *src, + struct scatterlist *dst, + unsigned int cryptlen, + struct safexcel_cipher_req *sreq, + bool *should_complete, int *ret) +{ + struct skcipher_request *areq = skcipher_request_cast(async); + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq); + struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(skcipher); + struct safexcel_result_desc *rdesc; + int ndesc = 0; + + *ret = 0; + + if (unlikely(!sreq->rdescs)) + return 0; + + while (sreq->rdescs--) { + rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); + if (IS_ERR(rdesc)) { + dev_err(priv->dev, + "cipher: result: could not retrieve the result descriptor\n"); + *ret = PTR_ERR(rdesc); + break; + } + + if (likely(!*ret)) + *ret = safexcel_rdesc_check_errors(priv, rdesc); + + ndesc++; + } + + safexcel_complete(priv, ring); + + if (src == dst) { + if (sreq->nr_src > 0) + dma_unmap_sg(priv->dev, src, sreq->nr_src, + DMA_BIDIRECTIONAL); + } else { + if (sreq->nr_src > 0) + dma_unmap_sg(priv->dev, src, sreq->nr_src, + DMA_TO_DEVICE); + if (sreq->nr_dst > 0) + dma_unmap_sg(priv->dev, dst, sreq->nr_dst, + DMA_FROM_DEVICE); + } + + /* + * Update IV in req from last crypto output word for CBC modes + */ + if ((!ctx->aead) && (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) && + (sreq->direction == SAFEXCEL_ENCRYPT)) { + /* For encrypt take the last output word */ + sg_pcopy_to_buffer(dst, sreq->nr_dst, areq->iv, + crypto_skcipher_ivsize(skcipher), + (cryptlen - + crypto_skcipher_ivsize(skcipher))); + } + + *should_complete = true; + + return ndesc; +} + +static int safexcel_send_req(struct crypto_async_request *base, int ring, + struct safexcel_cipher_req *sreq, + struct scatterlist *src, struct scatterlist *dst, + unsigned int cryptlen, unsigned int assoclen, + unsigned int digestsize, u8 *iv, int *commands, + int *results) +{ + struct skcipher_request *areq = skcipher_request_cast(base); + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); + struct safexcel_crypto_priv *priv = ctx->base.priv; + struct safexcel_command_desc *cdesc; + struct safexcel_command_desc *first_cdesc = NULL; + struct safexcel_result_desc *rdesc, *first_rdesc = NULL; + struct scatterlist *sg; + unsigned int totlen; + unsigned int totlen_src = cryptlen + assoclen; + unsigned int totlen_dst = totlen_src; + struct safexcel_token *atoken; + int n_cdesc = 0, n_rdesc = 0; + int queued, i, ret = 0; + bool first = true; + + sreq->nr_src = sg_nents_for_len(src, totlen_src); + + if (ctx->aead) { + /* + * AEAD has auth tag appended to output for encrypt and + * removed from the output for decrypt! + */ + if (sreq->direction == SAFEXCEL_DECRYPT) + totlen_dst -= digestsize; + else + totlen_dst += digestsize; + + memcpy(ctx->base.ctxr->data + ctx->key_len / sizeof(u32), + &ctx->base.ipad, ctx->state_sz); + if (!ctx->xcm) + memcpy(ctx->base.ctxr->data + (ctx->key_len + + ctx->state_sz) / sizeof(u32), &ctx->base.opad, + ctx->state_sz); + } else if ((ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) && + (sreq->direction == SAFEXCEL_DECRYPT)) { + /* + * Save IV from last crypto input word for CBC modes in decrypt + * direction. Need to do this first in case of inplace operation + * as it will be overwritten. + */ + sg_pcopy_to_buffer(src, sreq->nr_src, areq->iv, + crypto_skcipher_ivsize(skcipher), + (totlen_src - + crypto_skcipher_ivsize(skcipher))); + } + + sreq->nr_dst = sg_nents_for_len(dst, totlen_dst); + + /* + * Remember actual input length, source buffer length may be + * updated in case of inline operation below. + */ + totlen = totlen_src; + queued = totlen_src; + + if (src == dst) { + sreq->nr_src = max(sreq->nr_src, sreq->nr_dst); + sreq->nr_dst = sreq->nr_src; + if (unlikely((totlen_src || totlen_dst) && + (sreq->nr_src <= 0))) { + dev_err(priv->dev, "In-place buffer not large enough (need %d bytes)!", + max(totlen_src, totlen_dst)); + return -EINVAL; + } + if (sreq->nr_src > 0 && + !dma_map_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL)) + return -EIO; + } else { + if (unlikely(totlen_src && (sreq->nr_src <= 0))) { + dev_err(priv->dev, "Source buffer not large enough (need %d bytes)!", + totlen_src); + return -EINVAL; + } + + if (sreq->nr_src > 0 && + !dma_map_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE)) + return -EIO; + + if (unlikely(totlen_dst && (sreq->nr_dst <= 0))) { + dev_err(priv->dev, "Dest buffer not large enough (need %d bytes)!", + totlen_dst); + ret = -EINVAL; + goto unmap; + } + + if (sreq->nr_dst > 0 && + !dma_map_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE)) { + ret = -EIO; + goto unmap; + } + } + + memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len); + + if (!totlen) { + /* + * The EIP97 cannot deal with zero length input packets! + * So stuff a dummy command descriptor indicating a 1 byte + * (dummy) input packet, using the context record as source. + */ + first_cdesc = safexcel_add_cdesc(priv, ring, + 1, 1, ctx->base.ctxr_dma, + 1, 1, ctx->base.ctxr_dma, + &atoken); + if (IS_ERR(first_cdesc)) { + /* No space left in the command descriptor ring */ + ret = PTR_ERR(first_cdesc); + goto cdesc_rollback; + } + n_cdesc = 1; + goto skip_cdesc; + } + + /* command descriptors */ + for_each_sg(src, sg, sreq->nr_src, i) { + int len = sg_dma_len(sg); + + /* Do not overflow the request */ + if (queued < len) + len = queued; + + cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, + !(queued - len), + sg_dma_address(sg), len, totlen, + ctx->base.ctxr_dma, &atoken); + if (IS_ERR(cdesc)) { + /* No space left in the command descriptor ring */ + ret = PTR_ERR(cdesc); + goto cdesc_rollback; + } + + if (!n_cdesc) + first_cdesc = cdesc; + + n_cdesc++; + queued -= len; + if (!queued) + break; + } +skip_cdesc: + /* Add context control words and token to first command descriptor */ + safexcel_context_control(ctx, base, sreq, first_cdesc); + if (ctx->aead) + safexcel_aead_token(ctx, iv, first_cdesc, atoken, + sreq->direction, cryptlen, + assoclen, digestsize); + else + safexcel_skcipher_token(ctx, iv, first_cdesc, atoken, + cryptlen); + + /* result descriptors */ + for_each_sg(dst, sg, sreq->nr_dst, i) { + bool last = (i == sreq->nr_dst - 1); + u32 len = sg_dma_len(sg); + + /* only allow the part of the buffer we know we need */ + if (len > totlen_dst) + len = totlen_dst; + if (unlikely(!len)) + break; + totlen_dst -= len; + + /* skip over AAD space in buffer - not written */ + if (assoclen) { + if (assoclen >= len) { + assoclen -= len; + continue; + } + rdesc = safexcel_add_rdesc(priv, ring, first, last, + sg_dma_address(sg) + + assoclen, + len - assoclen); + assoclen = 0; + } else { + rdesc = safexcel_add_rdesc(priv, ring, first, last, + sg_dma_address(sg), + len); + } + if (IS_ERR(rdesc)) { + /* No space left in the result descriptor ring */ + ret = PTR_ERR(rdesc); + goto rdesc_rollback; + } + if (first) { + first_rdesc = rdesc; + first = false; + } + n_rdesc++; + } + + if (unlikely(first)) { + /* + * Special case: AEAD decrypt with only AAD data. + * In this case there is NO output data from the engine, + * but the engine still needs a result descriptor! + * Create a dummy one just for catching the result token. + */ + rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0); + if (IS_ERR(rdesc)) { + /* No space left in the result descriptor ring */ + ret = PTR_ERR(rdesc); + goto rdesc_rollback; + } + first_rdesc = rdesc; + n_rdesc = 1; + } + + safexcel_rdr_req_set(priv, ring, first_rdesc, base); + + *commands = n_cdesc; + *results = n_rdesc; + return 0; + +rdesc_rollback: + for (i = 0; i < n_rdesc; i++) + safexcel_ring_rollback_wptr(priv, &priv->ring[ring].rdr); +cdesc_rollback: + for (i = 0; i < n_cdesc; i++) + safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); +unmap: + if (src == dst) { + if (sreq->nr_src > 0) + dma_unmap_sg(priv->dev, src, sreq->nr_src, + DMA_BIDIRECTIONAL); + } else { + if (sreq->nr_src > 0) + dma_unmap_sg(priv->dev, src, sreq->nr_src, + DMA_TO_DEVICE); + if (sreq->nr_dst > 0) + dma_unmap_sg(priv->dev, dst, sreq->nr_dst, + DMA_FROM_DEVICE); + } + + return ret; +} + +static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, + int ring, + struct crypto_async_request *base, + struct safexcel_cipher_req *sreq, + bool *should_complete, int *ret) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); + struct safexcel_result_desc *rdesc; + int ndesc = 0, enq_ret; + + *ret = 0; + + if (unlikely(!sreq->rdescs)) + return 0; + + while (sreq->rdescs--) { + rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); + if (IS_ERR(rdesc)) { + dev_err(priv->dev, + "cipher: invalidate: could not retrieve the result descriptor\n"); + *ret = PTR_ERR(rdesc); + break; + } + + if (likely(!*ret)) + *ret = safexcel_rdesc_check_errors(priv, rdesc); + + ndesc++; + } + + safexcel_complete(priv, ring); + + if (ctx->base.exit_inv) { + dma_pool_free(priv->context_pool, ctx->base.ctxr, + ctx->base.ctxr_dma); + + *should_complete = true; + + return ndesc; + } + + ring = safexcel_select_ring(priv); + ctx->base.ring = ring; + + spin_lock_bh(&priv->ring[ring].queue_lock); + enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, base); + spin_unlock_bh(&priv->ring[ring].queue_lock); + + if (enq_ret != -EINPROGRESS) + *ret = enq_ret; + + queue_work(priv->ring[ring].workqueue, + &priv->ring[ring].work_data.work); + + *should_complete = false; + + return ndesc; +} + +static int safexcel_skcipher_handle_result(struct safexcel_crypto_priv *priv, + int ring, + struct crypto_async_request *async, + bool *should_complete, int *ret) +{ + struct skcipher_request *req = skcipher_request_cast(async); + struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); + int err; + + if (sreq->needs_inv) { + sreq->needs_inv = false; + err = safexcel_handle_inv_result(priv, ring, async, sreq, + should_complete, ret); + } else { + err = safexcel_handle_req_result(priv, ring, async, req->src, + req->dst, req->cryptlen, sreq, + should_complete, ret); + } + + return err; +} + +static int safexcel_aead_handle_result(struct safexcel_crypto_priv *priv, + int ring, + struct crypto_async_request *async, + bool *should_complete, int *ret) +{ + struct aead_request *req = aead_request_cast(async); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct safexcel_cipher_req *sreq = aead_request_ctx(req); + int err; + + if (sreq->needs_inv) { + sreq->needs_inv = false; + err = safexcel_handle_inv_result(priv, ring, async, sreq, + should_complete, ret); + } else { + err = safexcel_handle_req_result(priv, ring, async, req->src, + req->dst, + req->cryptlen + crypto_aead_authsize(tfm), + sreq, should_complete, ret); + } + + return err; +} + +static int safexcel_cipher_send_inv(struct crypto_async_request *base, + int ring, int *commands, int *results) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); + struct safexcel_crypto_priv *priv = ctx->base.priv; + int ret; + + ret = safexcel_invalidate_cache(base, priv, ctx->base.ctxr_dma, ring); + if (unlikely(ret)) + return ret; + + *commands = 1; + *results = 1; + + return 0; +} + +static int safexcel_skcipher_send(struct crypto_async_request *async, int ring, + int *commands, int *results) +{ + struct skcipher_request *req = skcipher_request_cast(async); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); + struct safexcel_crypto_priv *priv = ctx->base.priv; + int ret; + + BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv); + + if (sreq->needs_inv) { + ret = safexcel_cipher_send_inv(async, ring, commands, results); + } else { + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + u8 input_iv[AES_BLOCK_SIZE]; + + /* + * Save input IV in case of CBC decrypt mode + * Will be overwritten with output IV prior to use! + */ + memcpy(input_iv, req->iv, crypto_skcipher_ivsize(skcipher)); + + ret = safexcel_send_req(async, ring, sreq, req->src, + req->dst, req->cryptlen, 0, 0, input_iv, + commands, results); + } + + sreq->rdescs = *results; + return ret; +} + +static int safexcel_aead_send(struct crypto_async_request *async, int ring, + int *commands, int *results) +{ + struct aead_request *req = aead_request_cast(async); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct safexcel_cipher_req *sreq = aead_request_ctx(req); + struct safexcel_crypto_priv *priv = ctx->base.priv; + int ret; + + BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv); + + if (sreq->needs_inv) + ret = safexcel_cipher_send_inv(async, ring, commands, results); + else + ret = safexcel_send_req(async, ring, sreq, req->src, req->dst, + req->cryptlen, req->assoclen, + crypto_aead_authsize(tfm), req->iv, + commands, results); + sreq->rdescs = *results; + return ret; +} + +static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm, + struct crypto_async_request *base, + struct safexcel_cipher_req *sreq, + struct crypto_wait *result) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_crypto_priv *priv = ctx->base.priv; + int ring = ctx->base.ring; + int err; + + ctx = crypto_tfm_ctx(base->tfm); + ctx->base.exit_inv = true; + sreq->needs_inv = true; + + spin_lock_bh(&priv->ring[ring].queue_lock); + crypto_enqueue_request(&priv->ring[ring].queue, base); + spin_unlock_bh(&priv->ring[ring].queue_lock); + + queue_work(priv->ring[ring].workqueue, + &priv->ring[ring].work_data.work); + + err = crypto_wait_req(-EINPROGRESS, result); + + if (err) { + dev_warn(priv->dev, + "cipher: sync: invalidate: completion error %d\n", + err); + return err; + } + + return 0; +} + +static int safexcel_skcipher_exit_inv(struct crypto_tfm *tfm) +{ + EIP197_REQUEST_ON_STACK(req, skcipher, EIP197_SKCIPHER_REQ_SIZE); + struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); + DECLARE_CRYPTO_WAIT(result); + + memset(req, 0, sizeof(struct skcipher_request)); + + skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &result); + skcipher_request_set_tfm(req, __crypto_skcipher_cast(tfm)); + + return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result); +} + +static int safexcel_aead_exit_inv(struct crypto_tfm *tfm) +{ + EIP197_REQUEST_ON_STACK(req, aead, EIP197_AEAD_REQ_SIZE); + struct safexcel_cipher_req *sreq = aead_request_ctx(req); + DECLARE_CRYPTO_WAIT(result); + + memset(req, 0, sizeof(struct aead_request)); + + aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &result); + aead_request_set_tfm(req, __crypto_aead_cast(tfm)); + + return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result); +} + +static int safexcel_queue_req(struct crypto_async_request *base, + struct safexcel_cipher_req *sreq, + enum safexcel_cipher_direction dir) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); + struct safexcel_crypto_priv *priv = ctx->base.priv; + int ret, ring; + + sreq->needs_inv = false; + sreq->direction = dir; + + if (ctx->base.ctxr) { + if (priv->flags & EIP197_TRC_CACHE && ctx->base.needs_inv) { + sreq->needs_inv = true; + ctx->base.needs_inv = false; + } + } else { + ctx->base.ring = safexcel_select_ring(priv); + ctx->base.ctxr = dma_pool_zalloc(priv->context_pool, + EIP197_GFP_FLAGS(*base), + &ctx->base.ctxr_dma); + if (!ctx->base.ctxr) + return -ENOMEM; + } + + ring = ctx->base.ring; + + spin_lock_bh(&priv->ring[ring].queue_lock); + ret = crypto_enqueue_request(&priv->ring[ring].queue, base); + spin_unlock_bh(&priv->ring[ring].queue_lock); + + queue_work(priv->ring[ring].workqueue, + &priv->ring[ring].work_data.work); + + return ret; +} + +static int safexcel_encrypt(struct skcipher_request *req) +{ + return safexcel_queue_req(&req->base, skcipher_request_ctx(req), + SAFEXCEL_ENCRYPT); +} + +static int safexcel_decrypt(struct skcipher_request *req) +{ + return safexcel_queue_req(&req->base, skcipher_request_ctx(req), + SAFEXCEL_DECRYPT); +} + +static int safexcel_skcipher_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_alg_template *tmpl = + container_of(tfm->__crt_alg, struct safexcel_alg_template, + alg.skcipher.base); + + crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm), + sizeof(struct safexcel_cipher_req)); + + ctx->base.priv = tmpl->priv; + + ctx->base.send = safexcel_skcipher_send; + ctx->base.handle_result = safexcel_skcipher_handle_result; + ctx->ivmask = EIP197_OPTION_4_TOKEN_IV_CMD; + ctx->ctrinit = 1; + return 0; +} + +static int safexcel_cipher_cra_exit(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + memzero_explicit(ctx->key, sizeof(ctx->key)); + + /* context not allocated, skip invalidation */ + if (!ctx->base.ctxr) + return -ENOMEM; + + memzero_explicit(ctx->base.ctxr->data, sizeof(ctx->base.ctxr->data)); + return 0; +} + +static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_crypto_priv *priv = ctx->base.priv; + int ret; + + if (safexcel_cipher_cra_exit(tfm)) + return; + + if (priv->flags & EIP197_TRC_CACHE) { + ret = safexcel_skcipher_exit_inv(tfm); + if (ret) + dev_warn(priv->dev, "skcipher: invalidation error %d\n", + ret); + } else { + dma_pool_free(priv->context_pool, ctx->base.ctxr, + ctx->base.ctxr_dma); + } +} + +static void safexcel_aead_cra_exit(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_crypto_priv *priv = ctx->base.priv; + int ret; + + if (safexcel_cipher_cra_exit(tfm)) + return; + + if (priv->flags & EIP197_TRC_CACHE) { + ret = safexcel_aead_exit_inv(tfm); + if (ret) + dev_warn(priv->dev, "aead: invalidation error %d\n", + ret); + } else { + dma_pool_free(priv->context_pool, ctx->base.ctxr, + ctx->base.ctxr_dma); + } +} + +static int safexcel_skcipher_aes_ecb_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_skcipher_cra_init(tfm); + ctx->alg = SAFEXCEL_AES; + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; + ctx->blocksz = 0; + ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_ecb_aes = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .algo_mask = SAFEXCEL_ALG_AES, + .alg.skcipher = { + .setkey = safexcel_skcipher_aes_setkey, + .encrypt = safexcel_encrypt, + .decrypt = safexcel_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "safexcel-ecb-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_aes_ecb_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_skcipher_aes_cbc_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_skcipher_cra_init(tfm); + ctx->alg = SAFEXCEL_AES; + ctx->blocksz = AES_BLOCK_SIZE; + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; + return 0; +} + +struct safexcel_alg_template safexcel_alg_cbc_aes = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .algo_mask = SAFEXCEL_ALG_AES, + .alg.skcipher = { + .setkey = safexcel_skcipher_aes_setkey, + .encrypt = safexcel_encrypt, + .decrypt = safexcel_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "safexcel-cbc-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_aes_cbc_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_skcipher_aesctr_setkey(struct crypto_skcipher *ctfm, + const u8 *key, unsigned int len) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_crypto_priv *priv = ctx->base.priv; + struct crypto_aes_ctx aes; + int ret, i; + unsigned int keylen; + + /* last 4 bytes of key are the nonce! */ + ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); + /* exclude the nonce here */ + keylen = len - CTR_RFC3686_NONCE_SIZE; + ret = aes_expandkey(&aes, key, keylen); + if (ret) + return ret; + + if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { + for (i = 0; i < keylen / sizeof(u32); i++) { + if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { + ctx->base.needs_inv = true; + break; + } + } + } + + for (i = 0; i < keylen / sizeof(u32); i++) + ctx->key[i] = cpu_to_le32(aes.key_enc[i]); + + ctx->key_len = keylen; + + memzero_explicit(&aes, sizeof(aes)); + return 0; +} + +static int safexcel_skcipher_aes_ctr_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_skcipher_cra_init(tfm); + ctx->alg = SAFEXCEL_AES; + ctx->blocksz = AES_BLOCK_SIZE; + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_ctr_aes = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .algo_mask = SAFEXCEL_ALG_AES, + .alg.skcipher = { + .setkey = safexcel_skcipher_aesctr_setkey, + .encrypt = safexcel_encrypt, + .decrypt = safexcel_decrypt, + /* Add nonce size */ + .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + .base = { + .cra_name = "rfc3686(ctr(aes))", + .cra_driver_name = "safexcel-ctr-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_aes_ctr_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_des_setkey(struct crypto_skcipher *ctfm, const u8 *key, + unsigned int len) +{ + struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); + struct safexcel_crypto_priv *priv = ctx->base.priv; + int ret; + + ret = verify_skcipher_des_key(ctfm, key); + if (ret) + return ret; + + /* if context exits and key changed, need to invalidate it */ + if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) + if (memcmp(ctx->key, key, len)) + ctx->base.needs_inv = true; + + memcpy(ctx->key, key, len); + ctx->key_len = len; + + return 0; +} + +static int safexcel_skcipher_des_cbc_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_skcipher_cra_init(tfm); + ctx->alg = SAFEXCEL_DES; + ctx->blocksz = DES_BLOCK_SIZE; + ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; + return 0; +} + +struct safexcel_alg_template safexcel_alg_cbc_des = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .algo_mask = SAFEXCEL_ALG_DES, + .alg.skcipher = { + .setkey = safexcel_des_setkey, + .encrypt = safexcel_encrypt, + .decrypt = safexcel_decrypt, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .base = { + .cra_name = "cbc(des)", + .cra_driver_name = "safexcel-cbc-des", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_des_cbc_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_skcipher_des_ecb_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_skcipher_cra_init(tfm); + ctx->alg = SAFEXCEL_DES; + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; + ctx->blocksz = 0; + ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_ecb_des = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .algo_mask = SAFEXCEL_ALG_DES, + .alg.skcipher = { + .setkey = safexcel_des_setkey, + .encrypt = safexcel_encrypt, + .decrypt = safexcel_decrypt, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .base = { + .cra_name = "ecb(des)", + .cra_driver_name = "safexcel-ecb-des", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_des_ecb_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_des3_ede_setkey(struct crypto_skcipher *ctfm, + const u8 *key, unsigned int len) +{ + struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); + struct safexcel_crypto_priv *priv = ctx->base.priv; + int err; + + err = verify_skcipher_des3_key(ctfm, key); + if (err) + return err; + + /* if context exits and key changed, need to invalidate it */ + if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) + if (memcmp(ctx->key, key, len)) + ctx->base.needs_inv = true; + + memcpy(ctx->key, key, len); + ctx->key_len = len; + + return 0; +} + +static int safexcel_skcipher_des3_cbc_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_skcipher_cra_init(tfm); + ctx->alg = SAFEXCEL_3DES; + ctx->blocksz = DES3_EDE_BLOCK_SIZE; + ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; + return 0; +} + +struct safexcel_alg_template safexcel_alg_cbc_des3_ede = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .algo_mask = SAFEXCEL_ALG_DES, + .alg.skcipher = { + .setkey = safexcel_des3_ede_setkey, + .encrypt = safexcel_encrypt, + .decrypt = safexcel_decrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .base = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "safexcel-cbc-des3_ede", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_des3_cbc_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_skcipher_des3_ecb_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_skcipher_cra_init(tfm); + ctx->alg = SAFEXCEL_3DES; + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; + ctx->blocksz = 0; + ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_ecb_des3_ede = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .algo_mask = SAFEXCEL_ALG_DES, + .alg.skcipher = { + .setkey = safexcel_des3_ede_setkey, + .encrypt = safexcel_encrypt, + .decrypt = safexcel_decrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .base = { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "safexcel-ecb-des3_ede", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_des3_ecb_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_encrypt(struct aead_request *req) +{ + struct safexcel_cipher_req *creq = aead_request_ctx(req); + + return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); +} + +static int safexcel_aead_decrypt(struct aead_request *req) +{ + struct safexcel_cipher_req *creq = aead_request_ctx(req); + + return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); +} + +static int safexcel_aead_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_alg_template *tmpl = + container_of(tfm->__crt_alg, struct safexcel_alg_template, + alg.aead.base); + + crypto_aead_set_reqsize(__crypto_aead_cast(tfm), + sizeof(struct safexcel_cipher_req)); + + ctx->base.priv = tmpl->priv; + + ctx->alg = SAFEXCEL_AES; /* default */ + ctx->blocksz = AES_BLOCK_SIZE; + ctx->ivmask = EIP197_OPTION_4_TOKEN_IV_CMD; + ctx->ctrinit = 1; + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; /* default */ + ctx->aead = true; + ctx->base.send = safexcel_aead_send; + ctx->base.handle_result = safexcel_aead_handle_result; + return 0; +} + +static int safexcel_aead_md5_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_cra_init(tfm); + ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5; + ctx->state_sz = MD5_DIGEST_SIZE; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_md5_cbc_aes = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_MD5, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(md5),cbc(aes))", + .cra_driver_name = "safexcel-authenc-hmac-md5-cbc-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_md5_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha1_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_cra_init(tfm); + ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; + ctx->state_sz = SHA1_DIGEST_SIZE; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_aes = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha1_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha256_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_cra_init(tfm); + ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256; + ctx->state_sz = SHA256_DIGEST_SIZE; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha256_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha224_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_cra_init(tfm); + ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224; + ctx->state_sz = SHA256_DIGEST_SIZE; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha224),cbc(aes))", + .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha224_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha512_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_cra_init(tfm); + ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512; + ctx->state_sz = SHA512_DIGEST_SIZE; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_aes = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha512),cbc(aes))", + .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha512_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha384_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_cra_init(tfm); + ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384; + ctx->state_sz = SHA512_DIGEST_SIZE; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_aes = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha384),cbc(aes))", + .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha384_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_md5_des3_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_md5_cra_init(tfm); + ctx->alg = SAFEXCEL_3DES; /* override default */ + ctx->blocksz = DES3_EDE_BLOCK_SIZE; + ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_md5_cbc_des3_ede = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_MD5, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(md5),cbc(des3_ede))", + .cra_driver_name = "safexcel-authenc-hmac-md5-cbc-des3_ede", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_md5_des3_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha1_des3_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha1_cra_init(tfm); + ctx->alg = SAFEXCEL_3DES; /* override default */ + ctx->blocksz = DES3_EDE_BLOCK_SIZE; + ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des3_ede = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", + .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des3_ede", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha1_des3_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha256_des3_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha256_cra_init(tfm); + ctx->alg = SAFEXCEL_3DES; /* override default */ + ctx->blocksz = DES3_EDE_BLOCK_SIZE; + ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des3_ede = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha256),cbc(des3_ede))", + .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des3_ede", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha256_des3_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha224_des3_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha224_cra_init(tfm); + ctx->alg = SAFEXCEL_3DES; /* override default */ + ctx->blocksz = DES3_EDE_BLOCK_SIZE; + ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des3_ede = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha224),cbc(des3_ede))", + .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des3_ede", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha224_des3_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha512_des3_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha512_cra_init(tfm); + ctx->alg = SAFEXCEL_3DES; /* override default */ + ctx->blocksz = DES3_EDE_BLOCK_SIZE; + ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des3_ede = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha512),cbc(des3_ede))", + .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des3_ede", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha512_des3_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha384_des3_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha384_cra_init(tfm); + ctx->alg = SAFEXCEL_3DES; /* override default */ + ctx->blocksz = DES3_EDE_BLOCK_SIZE; + ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des3_ede = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha384),cbc(des3_ede))", + .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des3_ede", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha384_des3_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_md5_des_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_md5_cra_init(tfm); + ctx->alg = SAFEXCEL_DES; /* override default */ + ctx->blocksz = DES_BLOCK_SIZE; + ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_md5_cbc_des = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_MD5, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(md5),cbc(des))", + .cra_driver_name = "safexcel-authenc-hmac-md5-cbc-des", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_md5_des_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha1_des_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha1_cra_init(tfm); + ctx->alg = SAFEXCEL_DES; /* override default */ + ctx->blocksz = DES_BLOCK_SIZE; + ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des))", + .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha1_des_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha256_des_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha256_cra_init(tfm); + ctx->alg = SAFEXCEL_DES; /* override default */ + ctx->blocksz = DES_BLOCK_SIZE; + ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha256),cbc(des))", + .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha256_des_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha224_des_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha224_cra_init(tfm); + ctx->alg = SAFEXCEL_DES; /* override default */ + ctx->blocksz = DES_BLOCK_SIZE; + ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha224),cbc(des))", + .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha224_des_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha512_des_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha512_cra_init(tfm); + ctx->alg = SAFEXCEL_DES; /* override default */ + ctx->blocksz = DES_BLOCK_SIZE; + ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha512),cbc(des))", + .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha512_des_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha384_des_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha384_cra_init(tfm); + ctx->alg = SAFEXCEL_DES; /* override default */ + ctx->blocksz = DES_BLOCK_SIZE; + ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha384),cbc(des))", + .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha384_des_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_md5_ctr_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_md5_cra_init(tfm); + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_md5_ctr_aes = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_MD5, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(md5),rfc3686(ctr(aes)))", + .cra_driver_name = "safexcel-authenc-hmac-md5-ctr-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_md5_ctr_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha1_ctr_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha1_cra_init(tfm); + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_aes = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", + .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha1_ctr_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha256_ctr_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha256_cra_init(tfm); + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_ctr_aes = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", + .cra_driver_name = "safexcel-authenc-hmac-sha256-ctr-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha256_ctr_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha224_ctr_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha224_cra_init(tfm); + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_ctr_aes = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))", + .cra_driver_name = "safexcel-authenc-hmac-sha224-ctr-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha224_ctr_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha512_ctr_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha512_cra_init(tfm); + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_ctr_aes = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))", + .cra_driver_name = "safexcel-authenc-hmac-sha512-ctr-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha512_ctr_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sha384_ctr_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sha384_cra_init(tfm); + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_ctr_aes = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))", + .cra_driver_name = "safexcel-authenc-hmac-sha384-ctr-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sha384_ctr_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_skcipher_aesxts_setkey(struct crypto_skcipher *ctfm, + const u8 *key, unsigned int len) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_crypto_priv *priv = ctx->base.priv; + struct crypto_aes_ctx aes; + int ret, i; + unsigned int keylen; + + /* Check for illegal XTS keys */ + ret = xts_verify_key(ctfm, key, len); + if (ret) + return ret; + + /* Only half of the key data is cipher key */ + keylen = (len >> 1); + ret = aes_expandkey(&aes, key, keylen); + if (ret) + return ret; + + if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { + for (i = 0; i < keylen / sizeof(u32); i++) { + if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { + ctx->base.needs_inv = true; + break; + } + } + } + + for (i = 0; i < keylen / sizeof(u32); i++) + ctx->key[i] = cpu_to_le32(aes.key_enc[i]); + + /* The other half is the tweak key */ + ret = aes_expandkey(&aes, (u8 *)(key + keylen), keylen); + if (ret) + return ret; + + if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { + for (i = 0; i < keylen / sizeof(u32); i++) { + if (le32_to_cpu(ctx->key[i + keylen / sizeof(u32)]) != + aes.key_enc[i]) { + ctx->base.needs_inv = true; + break; + } + } + } + + for (i = 0; i < keylen / sizeof(u32); i++) + ctx->key[i + keylen / sizeof(u32)] = + cpu_to_le32(aes.key_enc[i]); + + ctx->key_len = keylen << 1; + + memzero_explicit(&aes, sizeof(aes)); + return 0; +} + +static int safexcel_skcipher_aes_xts_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_skcipher_cra_init(tfm); + ctx->alg = SAFEXCEL_AES; + ctx->blocksz = AES_BLOCK_SIZE; + ctx->xts = 1; + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XTS; + return 0; +} + +static int safexcel_encrypt_xts(struct skcipher_request *req) +{ + if (req->cryptlen < XTS_BLOCK_SIZE) + return -EINVAL; + return safexcel_queue_req(&req->base, skcipher_request_ctx(req), + SAFEXCEL_ENCRYPT); +} + +static int safexcel_decrypt_xts(struct skcipher_request *req) +{ + if (req->cryptlen < XTS_BLOCK_SIZE) + return -EINVAL; + return safexcel_queue_req(&req->base, skcipher_request_ctx(req), + SAFEXCEL_DECRYPT); +} + +struct safexcel_alg_template safexcel_alg_xts_aes = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XTS, + .alg.skcipher = { + .setkey = safexcel_skcipher_aesxts_setkey, + .encrypt = safexcel_encrypt_xts, + .decrypt = safexcel_decrypt_xts, + /* XTS actually uses 2 AES keys glued together */ + .min_keysize = AES_MIN_KEY_SIZE * 2, + .max_keysize = AES_MAX_KEY_SIZE * 2, + .ivsize = XTS_BLOCK_SIZE, + .base = { + .cra_name = "xts(aes)", + .cra_driver_name = "safexcel-xts-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = XTS_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_aes_xts_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, + unsigned int len) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_crypto_priv *priv = ctx->base.priv; + struct aes_enckey aes; + u32 hashkey[AES_BLOCK_SIZE >> 2]; + int ret, i; + + ret = aes_prepareenckey(&aes, key, len); + if (ret) + return ret; + + if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { + for (i = 0; i < len / sizeof(u32); i++) { + if (ctx->key[i] != get_unaligned((__le32 *)key + i)) { + ctx->base.needs_inv = true; + break; + } + } + } + + for (i = 0; i < len / sizeof(u32); i++) + ctx->key[i] = get_unaligned((__le32 *)key + i); + + ctx->key_len = len; + + /* Compute hash key by encrypting zeroes with cipher key */ + memset(hashkey, 0, AES_BLOCK_SIZE); + aes_encrypt(&aes, (u8 *)hashkey, (u8 *)hashkey); + + if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { + for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) { + if (be32_to_cpu(ctx->base.ipad.be[i]) != hashkey[i]) { + ctx->base.needs_inv = true; + break; + } + } + } + + for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) + ctx->base.ipad.be[i] = cpu_to_be32(hashkey[i]); + + memzero_explicit(hashkey, AES_BLOCK_SIZE); + memzero_explicit(&aes, sizeof(aes)); + return 0; +} + +static int safexcel_aead_gcm_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_cra_init(tfm); + ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_GHASH; + ctx->state_sz = GHASH_BLOCK_SIZE; + ctx->xcm = EIP197_XCM_MODE_GCM; + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */ + + return 0; +} + +static void safexcel_aead_gcm_cra_exit(struct crypto_tfm *tfm) +{ + safexcel_aead_cra_exit(tfm); +} + +static int safexcel_aead_gcm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + return crypto_gcm_check_authsize(authsize); +} + +struct safexcel_alg_template safexcel_alg_gcm = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, + .alg.aead = { + .setkey = safexcel_aead_gcm_setkey, + .setauthsize = safexcel_aead_gcm_setauthsize, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = GHASH_DIGEST_SIZE, + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "safexcel-gcm-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_gcm_cra_init, + .cra_exit = safexcel_aead_gcm_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_ccm_setkey(struct crypto_aead *ctfm, const u8 *key, + unsigned int len) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_crypto_priv *priv = ctx->base.priv; + struct crypto_aes_ctx aes; + int ret, i; + + ret = aes_expandkey(&aes, key, len); + if (ret) { + memzero_explicit(&aes, sizeof(aes)); + return ret; + } + + if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { + for (i = 0; i < len / sizeof(u32); i++) { + if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { + ctx->base.needs_inv = true; + break; + } + } + } + + for (i = 0; i < len / sizeof(u32); i++) { + ctx->key[i] = cpu_to_le32(aes.key_enc[i]); + ctx->base.ipad.be[i + 2 * AES_BLOCK_SIZE / sizeof(u32)] = + cpu_to_be32(aes.key_enc[i]); + } + + ctx->key_len = len; + ctx->state_sz = 2 * AES_BLOCK_SIZE + len; + + if (len == AES_KEYSIZE_192) + ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192; + else if (len == AES_KEYSIZE_256) + ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256; + else + ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; + + memzero_explicit(&aes, sizeof(aes)); + return 0; +} + +static int safexcel_aead_ccm_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_cra_init(tfm); + ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; + ctx->state_sz = 3 * AES_BLOCK_SIZE; + ctx->xcm = EIP197_XCM_MODE_CCM; + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */ + ctx->ctrinit = 0; + return 0; +} + +static int safexcel_aead_ccm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + /* Borrowed from crypto/ccm.c */ + switch (authsize) { + case 4: + case 6: + case 8: + case 10: + case 12: + case 14: + case 16: + break; + default: + return -EINVAL; + } + + return 0; +} + +static int safexcel_ccm_encrypt(struct aead_request *req) +{ + struct safexcel_cipher_req *creq = aead_request_ctx(req); + + if (req->iv[0] < 1 || req->iv[0] > 7) + return -EINVAL; + + return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); +} + +static int safexcel_ccm_decrypt(struct aead_request *req) +{ + struct safexcel_cipher_req *creq = aead_request_ctx(req); + + if (req->iv[0] < 1 || req->iv[0] > 7) + return -EINVAL; + + return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); +} + +struct safexcel_alg_template safexcel_alg_ccm = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL, + .alg.aead = { + .setkey = safexcel_aead_ccm_setkey, + .setauthsize = safexcel_aead_ccm_setauthsize, + .encrypt = safexcel_ccm_encrypt, + .decrypt = safexcel_ccm_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .base = { + .cra_name = "ccm(aes)", + .cra_driver_name = "safexcel-ccm-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_ccm_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static void safexcel_chacha20_setkey(struct safexcel_cipher_ctx *ctx, + const u8 *key) +{ + struct safexcel_crypto_priv *priv = ctx->base.priv; + + if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) + if (memcmp(ctx->key, key, CHACHA_KEY_SIZE)) + ctx->base.needs_inv = true; + + memcpy(ctx->key, key, CHACHA_KEY_SIZE); + ctx->key_len = CHACHA_KEY_SIZE; +} + +static int safexcel_skcipher_chacha20_setkey(struct crypto_skcipher *ctfm, + const u8 *key, unsigned int len) +{ + struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); + + if (len != CHACHA_KEY_SIZE) + return -EINVAL; + + safexcel_chacha20_setkey(ctx, key); + + return 0; +} + +static int safexcel_skcipher_chacha20_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_skcipher_cra_init(tfm); + ctx->alg = SAFEXCEL_CHACHA20; + ctx->ctrinit = 0; + ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32; + return 0; +} + +struct safexcel_alg_template safexcel_alg_chacha20 = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .algo_mask = SAFEXCEL_ALG_CHACHA20, + .alg.skcipher = { + .setkey = safexcel_skcipher_chacha20_setkey, + .encrypt = safexcel_encrypt, + .decrypt = safexcel_decrypt, + .min_keysize = CHACHA_KEY_SIZE, + .max_keysize = CHACHA_KEY_SIZE, + .ivsize = CHACHA_IV_SIZE, + .base = { + .cra_name = "chacha20", + .cra_driver_name = "safexcel-chacha20", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_chacha20_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_chachapoly_setkey(struct crypto_aead *ctfm, + const u8 *key, unsigned int len) +{ + struct safexcel_cipher_ctx *ctx = crypto_aead_ctx(ctfm); + + if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP && + len > EIP197_AEAD_IPSEC_NONCE_SIZE) { + /* ESP variant has nonce appended to key */ + len -= EIP197_AEAD_IPSEC_NONCE_SIZE; + ctx->nonce = *(u32 *)(key + len); + } + if (len != CHACHA_KEY_SIZE) + return -EINVAL; + + safexcel_chacha20_setkey(ctx, key); + + return 0; +} + +static int safexcel_aead_chachapoly_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + if (authsize != POLY1305_DIGEST_SIZE) + return -EINVAL; + return 0; +} + +static int safexcel_aead_chachapoly_crypt(struct aead_request *req, + enum safexcel_cipher_direction dir) +{ + struct safexcel_cipher_req *creq = aead_request_ctx(req); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct crypto_tfm *tfm = crypto_aead_tfm(aead); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct aead_request *subreq = aead_request_ctx(req); + u32 key[CHACHA_KEY_SIZE / sizeof(u32) + 1]; + int ret = 0; + + /* + * Instead of wasting time detecting umpteen silly corner cases, + * just dump all "small" requests to the fallback implementation. + * HW would not be faster on such small requests anyway. + */ + if (likely((ctx->aead != EIP197_AEAD_TYPE_IPSEC_ESP || + req->assoclen >= EIP197_AEAD_IPSEC_IV_SIZE) && + req->cryptlen > POLY1305_DIGEST_SIZE)) { + return safexcel_queue_req(&req->base, creq, dir); + } + + /* HW cannot do full (AAD+payload) zero length, use fallback */ + memcpy(key, ctx->key, CHACHA_KEY_SIZE); + if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { + /* ESP variant has nonce appended to the key */ + key[CHACHA_KEY_SIZE / sizeof(u32)] = ctx->nonce; + ret = crypto_aead_setkey(ctx->fback, (u8 *)key, + CHACHA_KEY_SIZE + + EIP197_AEAD_IPSEC_NONCE_SIZE); + } else { + ret = crypto_aead_setkey(ctx->fback, (u8 *)key, + CHACHA_KEY_SIZE); + } + if (ret) { + crypto_aead_clear_flags(aead, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(aead, crypto_aead_get_flags(ctx->fback) & + CRYPTO_TFM_REQ_MASK); + return ret; + } + + aead_request_set_tfm(subreq, ctx->fback); + aead_request_set_callback(subreq, req->base.flags, req->base.complete, + req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, + req->iv); + aead_request_set_ad(subreq, req->assoclen); + + return (dir == SAFEXCEL_ENCRYPT) ? + crypto_aead_encrypt(subreq) : + crypto_aead_decrypt(subreq); +} + +static int safexcel_aead_chachapoly_encrypt(struct aead_request *req) +{ + return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_ENCRYPT); +} + +static int safexcel_aead_chachapoly_decrypt(struct aead_request *req) +{ + return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_DECRYPT); +} + +static int safexcel_aead_fallback_cra_init(struct crypto_tfm *tfm) +{ + struct crypto_aead *aead = __crypto_aead_cast(tfm); + struct aead_alg *alg = crypto_aead_alg(aead); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_cra_init(tfm); + + /* Allocate fallback implementation */ + ctx->fback = crypto_alloc_aead(alg->base.cra_name, 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fback)) + return PTR_ERR(ctx->fback); + + crypto_aead_set_reqsize(aead, max(sizeof(struct safexcel_cipher_req), + sizeof(struct aead_request) + + crypto_aead_reqsize(ctx->fback))); + + return 0; +} + +static int safexcel_aead_chachapoly_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_fallback_cra_init(tfm); + ctx->alg = SAFEXCEL_CHACHA20; + ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32 | + CONTEXT_CONTROL_CHACHA20_MODE_CALC_OTK; + ctx->ctrinit = 0; + ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_POLY1305; + ctx->state_sz = 0; /* Precomputed by HW */ + return 0; +} + +static void safexcel_aead_fallback_cra_exit(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + crypto_free_aead(ctx->fback); + safexcel_aead_cra_exit(tfm); +} + +struct safexcel_alg_template safexcel_alg_chachapoly = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305, + .alg.aead = { + .setkey = safexcel_aead_chachapoly_setkey, + .setauthsize = safexcel_aead_chachapoly_setauthsize, + .encrypt = safexcel_aead_chachapoly_encrypt, + .decrypt = safexcel_aead_chachapoly_decrypt, + .ivsize = CHACHAPOLY_IV_SIZE, + .maxauthsize = POLY1305_DIGEST_SIZE, + .base = { + .cra_name = "rfc7539(chacha20,poly1305)", + .cra_driver_name = "safexcel-chacha20-poly1305", + /* +1 to put it above HW chacha + SW poly */ + .cra_priority = SAFEXCEL_CRA_PRIORITY + 1, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_chachapoly_cra_init, + .cra_exit = safexcel_aead_fallback_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_chachapolyesp_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + int ret; + + ret = safexcel_aead_chachapoly_cra_init(tfm); + ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; + ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE; + return ret; +} + +struct safexcel_alg_template safexcel_alg_chachapoly_esp = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305, + .alg.aead = { + .setkey = safexcel_aead_chachapoly_setkey, + .setauthsize = safexcel_aead_chachapoly_setauthsize, + .encrypt = safexcel_aead_chachapoly_encrypt, + .decrypt = safexcel_aead_chachapoly_decrypt, + .ivsize = CHACHAPOLY_IV_SIZE - EIP197_AEAD_IPSEC_NONCE_SIZE, + .maxauthsize = POLY1305_DIGEST_SIZE, + .base = { + .cra_name = "rfc7539esp(chacha20,poly1305)", + .cra_driver_name = "safexcel-chacha20-poly1305-esp", + /* +1 to put it above HW chacha + SW poly */ + .cra_priority = SAFEXCEL_CRA_PRIORITY + 1, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_chachapolyesp_cra_init, + .cra_exit = safexcel_aead_fallback_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_skcipher_sm4_setkey(struct crypto_skcipher *ctfm, + const u8 *key, unsigned int len) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_crypto_priv *priv = ctx->base.priv; + + if (len != SM4_KEY_SIZE) + return -EINVAL; + + if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) + if (memcmp(ctx->key, key, SM4_KEY_SIZE)) + ctx->base.needs_inv = true; + + memcpy(ctx->key, key, SM4_KEY_SIZE); + ctx->key_len = SM4_KEY_SIZE; + + return 0; +} + +static int safexcel_sm4_blk_encrypt(struct skcipher_request *req) +{ + /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ + if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) + return -EINVAL; + else + return safexcel_queue_req(&req->base, skcipher_request_ctx(req), + SAFEXCEL_ENCRYPT); +} + +static int safexcel_sm4_blk_decrypt(struct skcipher_request *req) +{ + /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ + if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) + return -EINVAL; + else + return safexcel_queue_req(&req->base, skcipher_request_ctx(req), + SAFEXCEL_DECRYPT); +} + +static int safexcel_skcipher_sm4_ecb_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_skcipher_cra_init(tfm); + ctx->alg = SAFEXCEL_SM4; + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; + ctx->blocksz = 0; + ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_ecb_sm4 = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .algo_mask = SAFEXCEL_ALG_SM4, + .alg.skcipher = { + .setkey = safexcel_skcipher_sm4_setkey, + .encrypt = safexcel_sm4_blk_encrypt, + .decrypt = safexcel_sm4_blk_decrypt, + .min_keysize = SM4_KEY_SIZE, + .max_keysize = SM4_KEY_SIZE, + .base = { + .cra_name = "ecb(sm4)", + .cra_driver_name = "safexcel-ecb-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SM4_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_sm4_ecb_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_skcipher_sm4_cbc_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_skcipher_cra_init(tfm); + ctx->alg = SAFEXCEL_SM4; + ctx->blocksz = SM4_BLOCK_SIZE; + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; + return 0; +} + +struct safexcel_alg_template safexcel_alg_cbc_sm4 = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .algo_mask = SAFEXCEL_ALG_SM4, + .alg.skcipher = { + .setkey = safexcel_skcipher_sm4_setkey, + .encrypt = safexcel_sm4_blk_encrypt, + .decrypt = safexcel_sm4_blk_decrypt, + .min_keysize = SM4_KEY_SIZE, + .max_keysize = SM4_KEY_SIZE, + .ivsize = SM4_BLOCK_SIZE, + .base = { + .cra_name = "cbc(sm4)", + .cra_driver_name = "safexcel-cbc-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SM4_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_sm4_cbc_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_skcipher_sm4ctr_setkey(struct crypto_skcipher *ctfm, + const u8 *key, unsigned int len) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + /* last 4 bytes of key are the nonce! */ + ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); + /* exclude the nonce here */ + len -= CTR_RFC3686_NONCE_SIZE; + + return safexcel_skcipher_sm4_setkey(ctfm, key, len); +} + +static int safexcel_skcipher_sm4_ctr_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_skcipher_cra_init(tfm); + ctx->alg = SAFEXCEL_SM4; + ctx->blocksz = SM4_BLOCK_SIZE; + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_ctr_sm4 = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .algo_mask = SAFEXCEL_ALG_SM4, + .alg.skcipher = { + .setkey = safexcel_skcipher_sm4ctr_setkey, + .encrypt = safexcel_encrypt, + .decrypt = safexcel_decrypt, + /* Add nonce size */ + .min_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .max_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + .base = { + .cra_name = "rfc3686(ctr(sm4))", + .cra_driver_name = "safexcel-ctr-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_sm4_ctr_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sm4_blk_encrypt(struct aead_request *req) +{ + /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ + if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) + return -EINVAL; + + return safexcel_queue_req(&req->base, aead_request_ctx(req), + SAFEXCEL_ENCRYPT); +} + +static int safexcel_aead_sm4_blk_decrypt(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + + /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ + if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1)) + return -EINVAL; + + return safexcel_queue_req(&req->base, aead_request_ctx(req), + SAFEXCEL_DECRYPT); +} + +static int safexcel_aead_sm4cbc_sha1_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_cra_init(tfm); + ctx->alg = SAFEXCEL_SM4; + ctx->blocksz = SM4_BLOCK_SIZE; + ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; + ctx->state_sz = SHA1_DIGEST_SIZE; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_sm4 = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_sm4_blk_encrypt, + .decrypt = safexcel_aead_sm4_blk_decrypt, + .ivsize = SM4_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha1),cbc(sm4))", + .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SM4_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sm4cbc_sha1_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_fallback_setkey(struct crypto_aead *ctfm, + const u8 *key, unsigned int len) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + /* Keep fallback cipher synchronized */ + return crypto_aead_setkey(ctx->fback, (u8 *)key, len) ?: + safexcel_aead_setkey(ctfm, key, len); +} + +static int safexcel_aead_fallback_setauthsize(struct crypto_aead *ctfm, + unsigned int authsize) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + /* Keep fallback cipher synchronized */ + return crypto_aead_setauthsize(ctx->fback, authsize); +} + +static int safexcel_aead_fallback_crypt(struct aead_request *req, + enum safexcel_cipher_direction dir) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct crypto_tfm *tfm = crypto_aead_tfm(aead); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct aead_request *subreq = aead_request_ctx(req); + + aead_request_set_tfm(subreq, ctx->fback); + aead_request_set_callback(subreq, req->base.flags, req->base.complete, + req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, + req->iv); + aead_request_set_ad(subreq, req->assoclen); + + return (dir == SAFEXCEL_ENCRYPT) ? + crypto_aead_encrypt(subreq) : + crypto_aead_decrypt(subreq); +} + +static int safexcel_aead_sm4cbc_sm3_encrypt(struct aead_request *req) +{ + struct safexcel_cipher_req *creq = aead_request_ctx(req); + + /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ + if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) + return -EINVAL; + else if (req->cryptlen || req->assoclen) /* If input length > 0 only */ + return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); + + /* HW cannot do full (AAD+payload) zero length, use fallback */ + return safexcel_aead_fallback_crypt(req, SAFEXCEL_ENCRYPT); +} + +static int safexcel_aead_sm4cbc_sm3_decrypt(struct aead_request *req) +{ + struct safexcel_cipher_req *creq = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + + /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ + if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1)) + return -EINVAL; + else if (req->cryptlen > crypto_aead_authsize(tfm) || req->assoclen) + /* If input length > 0 only */ + return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); + + /* HW cannot do full (AAD+payload) zero length, use fallback */ + return safexcel_aead_fallback_crypt(req, SAFEXCEL_DECRYPT); +} + +static int safexcel_aead_sm4cbc_sm3_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_fallback_cra_init(tfm); + ctx->alg = SAFEXCEL_SM4; + ctx->blocksz = SM4_BLOCK_SIZE; + ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3; + ctx->state_sz = SM3_DIGEST_SIZE; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_cbc_sm4 = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3, + .alg.aead = { + .setkey = safexcel_aead_fallback_setkey, + .setauthsize = safexcel_aead_fallback_setauthsize, + .encrypt = safexcel_aead_sm4cbc_sm3_encrypt, + .decrypt = safexcel_aead_sm4cbc_sm3_decrypt, + .ivsize = SM4_BLOCK_SIZE, + .maxauthsize = SM3_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sm3),cbc(sm4))", + .cra_driver_name = "safexcel-authenc-hmac-sm3-cbc-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SM4_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sm4cbc_sm3_cra_init, + .cra_exit = safexcel_aead_fallback_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sm4ctr_sha1_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sm4cbc_sha1_cra_init(tfm); + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_sm4 = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha1),rfc3686(ctr(sm4)))", + .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sm4ctr_sha1_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_aead_sm4ctr_sm3_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_aead_sm4cbc_sm3_cra_init(tfm); + ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; + return 0; +} + +struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_ctr_sm4 = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3, + .alg.aead = { + .setkey = safexcel_aead_setkey, + .encrypt = safexcel_aead_encrypt, + .decrypt = safexcel_aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SM3_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sm3),rfc3686(ctr(sm4)))", + .cra_driver_name = "safexcel-authenc-hmac-sm3-ctr-sm4", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_aead_sm4ctr_sm3_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_rfc4106_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, + unsigned int len) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + /* last 4 bytes of key are the nonce! */ + ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); + + len -= CTR_RFC3686_NONCE_SIZE; + return safexcel_aead_gcm_setkey(ctfm, key, len); +} + +static int safexcel_rfc4106_gcm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + return crypto_rfc4106_check_authsize(authsize); +} + +static int safexcel_rfc4106_encrypt(struct aead_request *req) +{ + return crypto_ipsec_check_assoclen(req->assoclen) ?: + safexcel_aead_encrypt(req); +} + +static int safexcel_rfc4106_decrypt(struct aead_request *req) +{ + return crypto_ipsec_check_assoclen(req->assoclen) ?: + safexcel_aead_decrypt(req); +} + +static int safexcel_rfc4106_gcm_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + int ret; + + ret = safexcel_aead_gcm_cra_init(tfm); + ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; + ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE; + return ret; +} + +struct safexcel_alg_template safexcel_alg_rfc4106_gcm = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, + .alg.aead = { + .setkey = safexcel_rfc4106_gcm_setkey, + .setauthsize = safexcel_rfc4106_gcm_setauthsize, + .encrypt = safexcel_rfc4106_encrypt, + .decrypt = safexcel_rfc4106_decrypt, + .ivsize = GCM_RFC4106_IV_SIZE, + .maxauthsize = GHASH_DIGEST_SIZE, + .base = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "safexcel-rfc4106-gcm-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_rfc4106_gcm_cra_init, + .cra_exit = safexcel_aead_gcm_cra_exit, + }, + }, +}; + +static int safexcel_rfc4543_gcm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + if (authsize != GHASH_DIGEST_SIZE) + return -EINVAL; + + return 0; +} + +static int safexcel_rfc4543_gcm_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + int ret; + + ret = safexcel_aead_gcm_cra_init(tfm); + ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP_GMAC; + return ret; +} + +struct safexcel_alg_template safexcel_alg_rfc4543_gcm = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, + .alg.aead = { + .setkey = safexcel_rfc4106_gcm_setkey, + .setauthsize = safexcel_rfc4543_gcm_setauthsize, + .encrypt = safexcel_rfc4106_encrypt, + .decrypt = safexcel_rfc4106_decrypt, + .ivsize = GCM_RFC4543_IV_SIZE, + .maxauthsize = GHASH_DIGEST_SIZE, + .base = { + .cra_name = "rfc4543(gcm(aes))", + .cra_driver_name = "safexcel-rfc4543-gcm-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_rfc4543_gcm_cra_init, + .cra_exit = safexcel_aead_gcm_cra_exit, + }, + }, +}; + +static int safexcel_rfc4309_ccm_setkey(struct crypto_aead *ctfm, const u8 *key, + unsigned int len) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + /* First byte of the nonce = L = always 3 for RFC4309 (4 byte ctr) */ + *(u8 *)&ctx->nonce = EIP197_AEAD_IPSEC_COUNTER_SIZE - 1; + /* last 3 bytes of key are the nonce! */ + memcpy((u8 *)&ctx->nonce + 1, key + len - + EIP197_AEAD_IPSEC_CCM_NONCE_SIZE, + EIP197_AEAD_IPSEC_CCM_NONCE_SIZE); + + len -= EIP197_AEAD_IPSEC_CCM_NONCE_SIZE; + return safexcel_aead_ccm_setkey(ctfm, key, len); +} + +static int safexcel_rfc4309_ccm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + /* Borrowed from crypto/ccm.c */ + switch (authsize) { + case 8: + case 12: + case 16: + break; + default: + return -EINVAL; + } + + return 0; +} + +static int safexcel_rfc4309_ccm_encrypt(struct aead_request *req) +{ + struct safexcel_cipher_req *creq = aead_request_ctx(req); + + /* Borrowed from crypto/ccm.c */ + if (req->assoclen != 16 && req->assoclen != 20) + return -EINVAL; + + return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); +} + +static int safexcel_rfc4309_ccm_decrypt(struct aead_request *req) +{ + struct safexcel_cipher_req *creq = aead_request_ctx(req); + + /* Borrowed from crypto/ccm.c */ + if (req->assoclen != 16 && req->assoclen != 20) + return -EINVAL; + + return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); +} + +static int safexcel_rfc4309_ccm_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + int ret; + + ret = safexcel_aead_ccm_cra_init(tfm); + ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; + ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE; + return ret; +} + +struct safexcel_alg_template safexcel_alg_rfc4309_ccm = { + .type = SAFEXCEL_ALG_TYPE_AEAD, + .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL, + .alg.aead = { + .setkey = safexcel_rfc4309_ccm_setkey, + .setauthsize = safexcel_rfc4309_ccm_setauthsize, + .encrypt = safexcel_rfc4309_ccm_encrypt, + .decrypt = safexcel_rfc4309_ccm_decrypt, + .ivsize = EIP197_AEAD_IPSEC_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .base = { + .cra_name = "rfc4309(ccm(aes))", + .cra_driver_name = "safexcel-rfc4309-ccm-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_rfc4309_ccm_cra_init, + .cra_exit = safexcel_aead_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c new file mode 100644 index 000000000..3402e570d --- /dev/null +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -0,0 +1,3012 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2017 Marvell + * + * Antoine Tenart + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "safexcel.h" + +struct safexcel_ahash_ctx { + struct safexcel_context base; + + u32 alg; + u8 key_sz; + bool cbcmac; + bool do_fallback; + bool fb_init_done; + bool fb_do_setkey; + + struct aes_enckey *aes; + struct crypto_ahash *fback; + struct crypto_shash *shpre; + struct shash_desc *shdesc; +}; + +struct safexcel_ahash_req { + bool last_req; + bool finish; + bool hmac; + bool needs_inv; + bool hmac_zlen; + bool len_is_le; + bool not_first; + bool xcbcmac; + + int nents; + dma_addr_t result_dma; + + u32 digest; + + u8 state_sz; /* expected state size, only set once */ + u8 block_sz; /* block size, only set once */ + u8 digest_sz; /* output digest size, only set once */ + __le32 state[SHA3_512_BLOCK_SIZE / + sizeof(__le32)] __aligned(sizeof(__le32)); + + u64 len; + u64 processed; + + u8 cache[HASH_CACHE_SIZE] __aligned(sizeof(u32)); + dma_addr_t cache_dma; + unsigned int cache_sz; + + u8 cache_next[HASH_CACHE_SIZE] __aligned(sizeof(u32)); +}; + +static inline u64 safexcel_queued_len(struct safexcel_ahash_req *req) +{ + return req->len - req->processed; +} + +static void safexcel_hash_token(struct safexcel_command_desc *cdesc, + u32 input_length, u32 result_length, + bool cbcmac) +{ + struct safexcel_token *token = + (struct safexcel_token *)cdesc->control_data.token; + + token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION; + token[0].packet_length = input_length; + token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH; + + input_length &= 15; + if (unlikely(cbcmac && input_length)) { + token[0].stat = 0; + token[1].opcode = EIP197_TOKEN_OPCODE_INSERT; + token[1].packet_length = 16 - input_length; + token[1].stat = EIP197_TOKEN_STAT_LAST_HASH; + token[1].instructions = EIP197_TOKEN_INS_TYPE_HASH; + } else { + token[0].stat = EIP197_TOKEN_STAT_LAST_HASH; + eip197_noop_token(&token[1]); + } + + token[2].opcode = EIP197_TOKEN_OPCODE_INSERT; + token[2].stat = EIP197_TOKEN_STAT_LAST_HASH | + EIP197_TOKEN_STAT_LAST_PACKET; + token[2].packet_length = result_length; + token[2].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | + EIP197_TOKEN_INS_INSERT_HASH_DIGEST; + + eip197_noop_token(&token[3]); +} + +static void safexcel_context_control(struct safexcel_ahash_ctx *ctx, + struct safexcel_ahash_req *req, + struct safexcel_command_desc *cdesc) +{ + struct safexcel_crypto_priv *priv = ctx->base.priv; + u64 count = 0; + + cdesc->control_data.control0 = ctx->alg; + cdesc->control_data.control1 = 0; + + /* + * Copy the input digest if needed, and setup the context + * fields. Do this now as we need it to setup the first command + * descriptor. + */ + if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM)) { + if (req->xcbcmac) + memcpy(ctx->base.ctxr->data, &ctx->base.ipad, ctx->key_sz); + else + memcpy(ctx->base.ctxr->data, req->state, req->state_sz); + + if (!req->finish && req->xcbcmac) + cdesc->control_data.control0 |= + CONTEXT_CONTROL_DIGEST_XCM | + CONTEXT_CONTROL_TYPE_HASH_OUT | + CONTEXT_CONTROL_NO_FINISH_HASH | + CONTEXT_CONTROL_SIZE(req->state_sz / + sizeof(u32)); + else + cdesc->control_data.control0 |= + CONTEXT_CONTROL_DIGEST_XCM | + CONTEXT_CONTROL_TYPE_HASH_OUT | + CONTEXT_CONTROL_SIZE(req->state_sz / + sizeof(u32)); + return; + } else if (!req->processed) { + /* First - and possibly only - block of basic hash only */ + if (req->finish) + cdesc->control_data.control0 |= req->digest | + CONTEXT_CONTROL_TYPE_HASH_OUT | + CONTEXT_CONTROL_RESTART_HASH | + /* ensure its not 0! */ + CONTEXT_CONTROL_SIZE(1); + else + cdesc->control_data.control0 |= req->digest | + CONTEXT_CONTROL_TYPE_HASH_OUT | + CONTEXT_CONTROL_RESTART_HASH | + CONTEXT_CONTROL_NO_FINISH_HASH | + /* ensure its not 0! */ + CONTEXT_CONTROL_SIZE(1); + return; + } + + /* Hash continuation or HMAC, setup (inner) digest from state */ + memcpy(ctx->base.ctxr->data, req->state, req->state_sz); + + if (req->finish) { + /* Compute digest count for hash/HMAC finish operations */ + if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) || + req->hmac_zlen || (req->processed != req->block_sz)) { + count = req->processed / EIP197_COUNTER_BLOCK_SIZE; + + /* This is a hardware limitation, as the + * counter must fit into an u32. This represents + * a fairly big amount of input data, so we + * shouldn't see this. + */ + if (unlikely(count & 0xffffffff00000000ULL)) { + dev_warn(priv->dev, + "Input data is too big\n"); + return; + } + } + + if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) || + /* Special case: zero length HMAC */ + req->hmac_zlen || + /* PE HW < 4.4 cannot do HMAC continue, fake using hash */ + (req->processed != req->block_sz)) { + /* Basic hash continue operation, need digest + cnt */ + cdesc->control_data.control0 |= + CONTEXT_CONTROL_SIZE((req->state_sz >> 2) + 1) | + CONTEXT_CONTROL_TYPE_HASH_OUT | + CONTEXT_CONTROL_DIGEST_PRECOMPUTED; + /* For zero-len HMAC, don't finalize, already padded! */ + if (req->hmac_zlen) + cdesc->control_data.control0 |= + CONTEXT_CONTROL_NO_FINISH_HASH; + cdesc->control_data.control1 |= + CONTEXT_CONTROL_DIGEST_CNT; + ctx->base.ctxr->data[req->state_sz >> 2] = + cpu_to_le32(count); + req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; + + /* Clear zero-length HMAC flag for next operation! */ + req->hmac_zlen = false; + } else { /* HMAC */ + /* Need outer digest for HMAC finalization */ + memcpy(ctx->base.ctxr->data + (req->state_sz >> 2), + &ctx->base.opad, req->state_sz); + + /* Single pass HMAC - no digest count */ + cdesc->control_data.control0 |= + CONTEXT_CONTROL_SIZE(req->state_sz >> 1) | + CONTEXT_CONTROL_TYPE_HASH_OUT | + CONTEXT_CONTROL_DIGEST_HMAC; + } + } else { /* Hash continuation, do not finish yet */ + cdesc->control_data.control0 |= + CONTEXT_CONTROL_SIZE(req->state_sz >> 2) | + CONTEXT_CONTROL_DIGEST_PRECOMPUTED | + CONTEXT_CONTROL_TYPE_HASH_OUT | + CONTEXT_CONTROL_NO_FINISH_HASH; + } +} + +static int safexcel_ahash_enqueue(struct ahash_request *areq); + +static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, + int ring, + struct crypto_async_request *async, + bool *should_complete, int *ret) +{ + struct safexcel_result_desc *rdesc; + struct ahash_request *areq = ahash_request_cast(async); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); + struct safexcel_ahash_req *sreq = ahash_request_ctx_dma(areq); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash); + u64 cache_len; + + *ret = 0; + + rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); + if (IS_ERR(rdesc)) { + dev_err(priv->dev, + "hash: result: could not retrieve the result descriptor\n"); + *ret = PTR_ERR(rdesc); + } else { + *ret = safexcel_rdesc_check_errors(priv, rdesc); + } + + safexcel_complete(priv, ring); + + if (sreq->nents) { + dma_unmap_sg(priv->dev, areq->src, + sg_nents_for_len(areq->src, areq->nbytes), + DMA_TO_DEVICE); + sreq->nents = 0; + } + + if (sreq->result_dma) { + dma_unmap_single(priv->dev, sreq->result_dma, sreq->digest_sz, + DMA_FROM_DEVICE); + sreq->result_dma = 0; + } + + if (sreq->cache_dma) { + dma_unmap_single(priv->dev, sreq->cache_dma, sreq->cache_sz, + DMA_TO_DEVICE); + sreq->cache_dma = 0; + sreq->cache_sz = 0; + } + + if (sreq->finish) { + if (sreq->hmac && + (sreq->digest != CONTEXT_CONTROL_DIGEST_HMAC)) { + /* Faking HMAC using hash - need to do outer hash */ + memcpy(sreq->cache, sreq->state, + crypto_ahash_digestsize(ahash)); + + memcpy(sreq->state, &ctx->base.opad, sreq->digest_sz); + + sreq->len = sreq->block_sz + + crypto_ahash_digestsize(ahash); + sreq->processed = sreq->block_sz; + sreq->hmac = 0; + + if (priv->flags & EIP197_TRC_CACHE) + ctx->base.needs_inv = true; + areq->nbytes = 0; + safexcel_ahash_enqueue(areq); + + *should_complete = false; /* Not done yet */ + return 1; + } + + memcpy(areq->result, sreq->state, + crypto_ahash_digestsize(ahash)); + } + + cache_len = safexcel_queued_len(sreq); + if (cache_len) + memcpy(sreq->cache, sreq->cache_next, cache_len); + + *should_complete = true; + + return 1; +} + +static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, + int *commands, int *results) +{ + struct ahash_request *areq = ahash_request_cast(async); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_crypto_priv *priv = ctx->base.priv; + struct safexcel_command_desc *cdesc, *first_cdesc = NULL; + struct safexcel_result_desc *rdesc; + struct scatterlist *sg; + struct safexcel_token *dmmy; + int i, extra = 0, n_cdesc = 0, ret = 0, cache_len, skip = 0; + u64 queued, len; + + queued = safexcel_queued_len(req); + if (queued <= HASH_CACHE_SIZE) + cache_len = queued; + else + cache_len = queued - areq->nbytes; + + if (!req->finish && !req->last_req) { + /* If this is not the last request and the queued data does not + * fit into full cache blocks, cache it for the next send call. + */ + extra = queued & (HASH_CACHE_SIZE - 1); + + /* If this is not the last request and the queued data + * is a multiple of a block, cache the last one for now. + */ + if (!extra) + extra = HASH_CACHE_SIZE; + + sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), + req->cache_next, extra, + areq->nbytes - extra); + + queued -= extra; + + if (!queued) { + *commands = 0; + *results = 0; + return 0; + } + + extra = 0; + } + + if (unlikely(req->xcbcmac && req->processed > AES_BLOCK_SIZE)) { + if (unlikely(cache_len < AES_BLOCK_SIZE)) { + /* + * Cache contains less than 1 full block, complete. + */ + extra = AES_BLOCK_SIZE - cache_len; + if (queued > cache_len) { + /* More data follows: borrow bytes */ + u64 tmp = queued - cache_len; + + skip = min_t(u64, tmp, extra); + sg_pcopy_to_buffer(areq->src, + sg_nents(areq->src), + req->cache + cache_len, + skip, 0); + } + extra -= skip; + memset(req->cache + cache_len + skip, 0, extra); + if (!ctx->cbcmac && extra) { + // 10- padding for XCBCMAC & CMAC + req->cache[cache_len + skip] = 0x80; + // HW will use K2 iso K3 - compensate! + for (i = 0; i < AES_BLOCK_SIZE / 4; i++) { + u32 *cache = (void *)req->cache; + u32 *ipad = ctx->base.ipad.word; + u32 x; + + x = ipad[i] ^ ipad[i + 4]; + cache[i] ^= swab32(x); + } + } + cache_len = AES_BLOCK_SIZE; + queued = queued + extra; + } + + /* XCBC continue: XOR previous result into 1st word */ + crypto_xor(req->cache, (const u8 *)req->state, AES_BLOCK_SIZE); + } + + len = queued; + /* Add a command descriptor for the cached data, if any */ + if (cache_len) { + req->cache_dma = dma_map_single(priv->dev, req->cache, + cache_len, DMA_TO_DEVICE); + if (dma_mapping_error(priv->dev, req->cache_dma)) + return -EINVAL; + + req->cache_sz = cache_len; + first_cdesc = safexcel_add_cdesc(priv, ring, 1, + (cache_len == len), + req->cache_dma, cache_len, + len, ctx->base.ctxr_dma, + &dmmy); + if (IS_ERR(first_cdesc)) { + ret = PTR_ERR(first_cdesc); + goto unmap_cache; + } + n_cdesc++; + + queued -= cache_len; + if (!queued) + goto send_command; + } + + /* Now handle the current ahash request buffer(s) */ + req->nents = dma_map_sg(priv->dev, areq->src, + sg_nents_for_len(areq->src, + areq->nbytes), + DMA_TO_DEVICE); + if (!req->nents) { + ret = -ENOMEM; + goto cdesc_rollback; + } + + for_each_sg(areq->src, sg, req->nents, i) { + int sglen = sg_dma_len(sg); + + if (unlikely(sglen <= skip)) { + skip -= sglen; + continue; + } + + /* Do not overflow the request */ + if ((queued + skip) <= sglen) + sglen = queued; + else + sglen -= skip; + + cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, + !(queued - sglen), + sg_dma_address(sg) + skip, sglen, + len, ctx->base.ctxr_dma, &dmmy); + if (IS_ERR(cdesc)) { + ret = PTR_ERR(cdesc); + goto unmap_sg; + } + + if (!n_cdesc) + first_cdesc = cdesc; + n_cdesc++; + + queued -= sglen; + if (!queued) + break; + skip = 0; + } + +send_command: + /* Setup the context options */ + safexcel_context_control(ctx, req, first_cdesc); + + /* Add the token */ + safexcel_hash_token(first_cdesc, len, req->digest_sz, ctx->cbcmac); + + req->result_dma = dma_map_single(priv->dev, req->state, req->digest_sz, + DMA_FROM_DEVICE); + if (dma_mapping_error(priv->dev, req->result_dma)) { + ret = -EINVAL; + goto unmap_sg; + } + + /* Add a result descriptor */ + rdesc = safexcel_add_rdesc(priv, ring, 1, 1, req->result_dma, + req->digest_sz); + if (IS_ERR(rdesc)) { + ret = PTR_ERR(rdesc); + goto unmap_result; + } + + safexcel_rdr_req_set(priv, ring, rdesc, &areq->base); + + req->processed += len - extra; + + *commands = n_cdesc; + *results = 1; + return 0; + +unmap_result: + dma_unmap_single(priv->dev, req->result_dma, req->digest_sz, + DMA_FROM_DEVICE); +unmap_sg: + if (req->nents) { + dma_unmap_sg(priv->dev, areq->src, + sg_nents_for_len(areq->src, areq->nbytes), + DMA_TO_DEVICE); + req->nents = 0; + } +cdesc_rollback: + for (i = 0; i < n_cdesc; i++) + safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); +unmap_cache: + if (req->cache_dma) { + dma_unmap_single(priv->dev, req->cache_dma, req->cache_sz, + DMA_TO_DEVICE); + req->cache_dma = 0; + req->cache_sz = 0; + } + + return ret; +} + +static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, + int ring, + struct crypto_async_request *async, + bool *should_complete, int *ret) +{ + struct safexcel_result_desc *rdesc; + struct ahash_request *areq = ahash_request_cast(async); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash); + int enq_ret; + + *ret = 0; + + rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); + if (IS_ERR(rdesc)) { + dev_err(priv->dev, + "hash: invalidate: could not retrieve the result descriptor\n"); + *ret = PTR_ERR(rdesc); + } else { + *ret = safexcel_rdesc_check_errors(priv, rdesc); + } + + safexcel_complete(priv, ring); + + if (ctx->base.exit_inv) { + dma_pool_free(priv->context_pool, ctx->base.ctxr, + ctx->base.ctxr_dma); + + *should_complete = true; + return 1; + } + + ring = safexcel_select_ring(priv); + ctx->base.ring = ring; + + spin_lock_bh(&priv->ring[ring].queue_lock); + enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, async); + spin_unlock_bh(&priv->ring[ring].queue_lock); + + if (enq_ret != -EINPROGRESS) + *ret = enq_ret; + + queue_work(priv->ring[ring].workqueue, + &priv->ring[ring].work_data.work); + + *should_complete = false; + + return 1; +} + +static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring, + struct crypto_async_request *async, + bool *should_complete, int *ret) +{ + struct ahash_request *areq = ahash_request_cast(async); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + int err; + + BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && req->needs_inv); + + if (req->needs_inv) { + req->needs_inv = false; + err = safexcel_handle_inv_result(priv, ring, async, + should_complete, ret); + } else { + err = safexcel_handle_req_result(priv, ring, async, + should_complete, ret); + } + + return err; +} + +static int safexcel_ahash_send_inv(struct crypto_async_request *async, + int ring, int *commands, int *results) +{ + struct ahash_request *areq = ahash_request_cast(async); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + int ret; + + ret = safexcel_invalidate_cache(async, ctx->base.priv, + ctx->base.ctxr_dma, ring); + if (unlikely(ret)) + return ret; + + *commands = 1; + *results = 1; + + return 0; +} + +static int safexcel_ahash_send(struct crypto_async_request *async, + int ring, int *commands, int *results) +{ + struct ahash_request *areq = ahash_request_cast(async); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + int ret; + + if (req->needs_inv) + ret = safexcel_ahash_send_inv(async, ring, commands, results); + else + ret = safexcel_ahash_send_req(async, ring, commands, results); + + return ret; +} + +static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm) +{ + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_crypto_priv *priv = ctx->base.priv; + EIP197_REQUEST_ON_STACK(req, ahash, EIP197_AHASH_REQ_SIZE); + struct safexcel_ahash_req *rctx = ahash_request_ctx_dma(req); + DECLARE_CRYPTO_WAIT(result); + int ring = ctx->base.ring; + int err; + + memset(req, 0, EIP197_AHASH_REQ_SIZE); + + /* create invalidation request */ + init_completion(&result.completion); + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &result); + + ahash_request_set_tfm(req, __crypto_ahash_cast(tfm)); + ctx = crypto_tfm_ctx(req->base.tfm); + ctx->base.exit_inv = true; + rctx->needs_inv = true; + + spin_lock_bh(&priv->ring[ring].queue_lock); + crypto_enqueue_request(&priv->ring[ring].queue, &req->base); + spin_unlock_bh(&priv->ring[ring].queue_lock); + + queue_work(priv->ring[ring].workqueue, + &priv->ring[ring].work_data.work); + + err = crypto_wait_req(-EINPROGRESS, &result); + + if (err) { + dev_warn(priv->dev, "hash: completion error (%d)\n", err); + return err; + } + + return 0; +} + +/* safexcel_ahash_cache: cache data until at least one request can be sent to + * the engine, aka. when there is at least 1 block size in the pipe. + */ +static int safexcel_ahash_cache(struct ahash_request *areq) +{ + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + u64 cache_len; + + /* cache_len: everything accepted by the driver but not sent yet, + * tot sz handled by update() - last req sz - tot sz handled by send() + */ + cache_len = safexcel_queued_len(req); + + /* + * In case there isn't enough bytes to proceed (less than a + * block size), cache the data until we have enough. + */ + if (cache_len + areq->nbytes <= HASH_CACHE_SIZE) { + sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), + req->cache + cache_len, + areq->nbytes, 0); + return 0; + } + + /* We couldn't cache all the data */ + return -E2BIG; +} + +static int safexcel_ahash_enqueue(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + struct safexcel_crypto_priv *priv = ctx->base.priv; + int ret, ring; + + req->needs_inv = false; + + if (ctx->base.ctxr) { + if (priv->flags & EIP197_TRC_CACHE && !ctx->base.needs_inv && + /* invalidate for *any* non-XCBC continuation */ + ((req->not_first && !req->xcbcmac) || + /* invalidate if (i)digest changed */ + memcmp(ctx->base.ctxr->data, req->state, req->state_sz) || + /* invalidate for HMAC finish with odigest changed */ + (req->finish && req->hmac && + memcmp(ctx->base.ctxr->data + (req->state_sz>>2), + &ctx->base.opad, req->state_sz)))) + /* + * We're still setting needs_inv here, even though it is + * cleared right away, because the needs_inv flag can be + * set in other functions and we want to keep the same + * logic. + */ + ctx->base.needs_inv = true; + + if (ctx->base.needs_inv) { + ctx->base.needs_inv = false; + req->needs_inv = true; + } + } else { + ctx->base.ring = safexcel_select_ring(priv); + ctx->base.ctxr = dma_pool_zalloc(priv->context_pool, + EIP197_GFP_FLAGS(areq->base), + &ctx->base.ctxr_dma); + if (!ctx->base.ctxr) + return -ENOMEM; + } + req->not_first = true; + + ring = ctx->base.ring; + + spin_lock_bh(&priv->ring[ring].queue_lock); + ret = crypto_enqueue_request(&priv->ring[ring].queue, &areq->base); + spin_unlock_bh(&priv->ring[ring].queue_lock); + + queue_work(priv->ring[ring].workqueue, + &priv->ring[ring].work_data.work); + + return ret; +} + +static int safexcel_ahash_update(struct ahash_request *areq) +{ + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + int ret; + + /* If the request is 0 length, do nothing */ + if (!areq->nbytes) + return 0; + + /* Add request to the cache if it fits */ + ret = safexcel_ahash_cache(areq); + + /* Update total request length */ + req->len += areq->nbytes; + + /* If not all data could fit into the cache, go process the excess. + * Also go process immediately for an HMAC IV precompute, which + * will never be finished at all, but needs to be processed anyway. + */ + if ((ret && !req->finish) || req->last_req) + return safexcel_ahash_enqueue(areq); + + return 0; +} + +static int safexcel_ahash_final(struct ahash_request *areq) +{ + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + + req->finish = true; + + if (unlikely(!req->len && !areq->nbytes)) { + /* + * If we have an overall 0 length *hash* request: + * The HW cannot do 0 length hash, so we provide the correct + * result directly here. + */ + if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5) + memcpy(areq->result, md5_zero_message_hash, + MD5_DIGEST_SIZE); + else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1) + memcpy(areq->result, sha1_zero_message_hash, + SHA1_DIGEST_SIZE); + else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224) + memcpy(areq->result, sha224_zero_message_hash, + SHA224_DIGEST_SIZE); + else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256) + memcpy(areq->result, sha256_zero_message_hash, + SHA256_DIGEST_SIZE); + else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA384) + memcpy(areq->result, sha384_zero_message_hash, + SHA384_DIGEST_SIZE); + else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA512) + memcpy(areq->result, sha512_zero_message_hash, + SHA512_DIGEST_SIZE); + else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SM3) { + memcpy(areq->result, + EIP197_SM3_ZEROM_HASH, SM3_DIGEST_SIZE); + } + + return 0; + } else if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM && + ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5 && + req->len == sizeof(u32) && !areq->nbytes)) { + /* Zero length CRC32 */ + memcpy(areq->result, &ctx->base.ipad, sizeof(u32)); + return 0; + } else if (unlikely(ctx->cbcmac && req->len == AES_BLOCK_SIZE && + !areq->nbytes)) { + /* Zero length CBC MAC */ + memset(areq->result, 0, AES_BLOCK_SIZE); + return 0; + } else if (unlikely(req->xcbcmac && req->len == AES_BLOCK_SIZE && + !areq->nbytes)) { + /* Zero length (X)CBC/CMAC */ + int i; + + for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) { + u32 *result = (void *)areq->result; + + /* K3 */ + result[i] = swab32(ctx->base.ipad.word[i + 4]); + } + areq->result[0] ^= 0x80; // 10- padding + aes_encrypt(ctx->aes, areq->result, areq->result); + return 0; + } else if (unlikely(req->hmac && + (req->len == req->block_sz) && + !areq->nbytes)) { + /* + * If we have an overall 0 length *HMAC* request: + * For HMAC, we need to finalize the inner digest + * and then perform the outer hash. + */ + + /* generate pad block in the cache */ + /* start with a hash block of all zeroes */ + memset(req->cache, 0, req->block_sz); + /* set the first byte to 0x80 to 'append a 1 bit' */ + req->cache[0] = 0x80; + /* add the length in bits in the last 2 bytes */ + if (req->len_is_le) { + /* Little endian length word (e.g. MD5) */ + req->cache[req->block_sz-8] = (req->block_sz << 3) & + 255; + req->cache[req->block_sz-7] = (req->block_sz >> 5); + } else { + /* Big endian length word (e.g. any SHA) */ + req->cache[req->block_sz-2] = (req->block_sz >> 5); + req->cache[req->block_sz-1] = (req->block_sz << 3) & + 255; + } + + req->len += req->block_sz; /* plus 1 hash block */ + + /* Set special zero-length HMAC flag */ + req->hmac_zlen = true; + + /* Finalize HMAC */ + req->digest = CONTEXT_CONTROL_DIGEST_HMAC; + } else if (req->hmac) { + /* Finalize HMAC */ + req->digest = CONTEXT_CONTROL_DIGEST_HMAC; + } + + return safexcel_ahash_enqueue(areq); +} + +static int safexcel_ahash_finup(struct ahash_request *areq) +{ + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + + req->finish = true; + + safexcel_ahash_update(areq); + return safexcel_ahash_final(areq); +} + +static int safexcel_ahash_export(struct ahash_request *areq, void *out) +{ + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + struct safexcel_ahash_export_state *export = out; + + export->len = req->len; + export->processed = req->processed; + + export->digest = req->digest; + + memcpy(export->state, req->state, req->state_sz); + memcpy(export->cache, req->cache, HASH_CACHE_SIZE); + + return 0; +} + +static int safexcel_ahash_import(struct ahash_request *areq, const void *in) +{ + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + const struct safexcel_ahash_export_state *export = in; + int ret; + + ret = crypto_ahash_init(areq); + if (ret) + return ret; + + req->len = export->len; + req->processed = export->processed; + + req->digest = export->digest; + + memcpy(req->cache, export->cache, HASH_CACHE_SIZE); + memcpy(req->state, export->state, req->state_sz); + + return 0; +} + +static int safexcel_ahash_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_alg_template *tmpl = + container_of(__crypto_ahash_alg(tfm->__crt_alg), + struct safexcel_alg_template, alg.ahash); + + ctx->base.priv = tmpl->priv; + ctx->base.send = safexcel_ahash_send; + ctx->base.handle_result = safexcel_handle_result; + ctx->fb_do_setkey = false; + + crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm), + sizeof(struct safexcel_ahash_req)); + return 0; +} + +static int safexcel_sha1_init(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + + memset(req, 0, sizeof(*req)); + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; + req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; + req->state_sz = SHA1_DIGEST_SIZE; + req->digest_sz = SHA1_DIGEST_SIZE; + req->block_sz = SHA1_BLOCK_SIZE; + + return 0; +} + +static int safexcel_sha1_digest(struct ahash_request *areq) +{ + int ret = safexcel_sha1_init(areq); + + if (ret) + return ret; + + return safexcel_ahash_finup(areq); +} + +static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm) +{ + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_crypto_priv *priv = ctx->base.priv; + int ret; + + /* context not allocated, skip invalidation */ + if (!ctx->base.ctxr) + return; + + if (priv->flags & EIP197_TRC_CACHE) { + ret = safexcel_ahash_exit_inv(tfm); + if (ret) + dev_warn(priv->dev, "hash: invalidation error %d\n", ret); + } else { + dma_pool_free(priv->context_pool, ctx->base.ctxr, + ctx->base.ctxr_dma); + } +} + +struct safexcel_alg_template safexcel_alg_sha1 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA1, + .alg.ahash = { + .init = safexcel_sha1_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_sha1_digest, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "sha1", + .cra_driver_name = "safexcel-sha1", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_ahash_cra_init, + .cra_exit = safexcel_ahash_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_hmac_sha1_init(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + + memset(req, 0, sizeof(*req)); + + /* Start from ipad precompute */ + memcpy(req->state, &ctx->base.ipad, SHA1_DIGEST_SIZE); + /* Already processed the key^ipad part now! */ + req->len = SHA1_BLOCK_SIZE; + req->processed = SHA1_BLOCK_SIZE; + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; + req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; + req->state_sz = SHA1_DIGEST_SIZE; + req->digest_sz = SHA1_DIGEST_SIZE; + req->block_sz = SHA1_BLOCK_SIZE; + req->hmac = true; + + return 0; +} + +static int safexcel_hmac_sha1_digest(struct ahash_request *areq) +{ + int ret = safexcel_hmac_sha1_init(areq); + + if (ret) + return ret; + + return safexcel_ahash_finup(areq); +} + +static int safexcel_hmac_init_pad(struct ahash_request *areq, + unsigned int blocksize, const u8 *key, + unsigned int keylen, u8 *ipad, u8 *opad) +{ + DECLARE_CRYPTO_WAIT(result); + struct scatterlist sg; + int ret, i; + u8 *keydup; + + if (keylen <= blocksize) { + memcpy(ipad, key, keylen); + } else { + keydup = kmemdup(key, keylen, GFP_KERNEL); + if (!keydup) + return -ENOMEM; + + ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &result); + sg_init_one(&sg, keydup, keylen); + ahash_request_set_crypt(areq, &sg, ipad, keylen); + + ret = crypto_ahash_digest(areq); + ret = crypto_wait_req(ret, &result); + + /* Avoid leaking */ + kfree_sensitive(keydup); + + if (ret) + return ret; + + keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq)); + } + + memset(ipad + keylen, 0, blocksize - keylen); + memcpy(opad, ipad, blocksize); + + for (i = 0; i < blocksize; i++) { + ipad[i] ^= HMAC_IPAD_VALUE; + opad[i] ^= HMAC_OPAD_VALUE; + } + + return 0; +} + +static int safexcel_hmac_init_iv(struct ahash_request *areq, + unsigned int blocksize, u8 *pad, void *state) +{ + struct safexcel_ahash_req *req; + DECLARE_CRYPTO_WAIT(result); + struct scatterlist sg; + int ret; + + ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &result); + sg_init_one(&sg, pad, blocksize); + ahash_request_set_crypt(areq, &sg, pad, blocksize); + + ret = crypto_ahash_init(areq); + if (ret) + return ret; + + req = ahash_request_ctx_dma(areq); + req->hmac = true; + req->last_req = true; + + ret = crypto_ahash_update(areq); + ret = crypto_wait_req(ret, &result); + + return ret ?: crypto_ahash_export(areq, state); +} + +static int __safexcel_hmac_setkey(const char *alg, const u8 *key, + unsigned int keylen, + void *istate, void *ostate) +{ + struct ahash_request *areq; + struct crypto_ahash *tfm; + unsigned int blocksize; + u8 *ipad, *opad; + int ret; + + tfm = crypto_alloc_ahash(alg, 0, 0); + if (IS_ERR(tfm)) + return PTR_ERR(tfm); + + areq = ahash_request_alloc(tfm, GFP_KERNEL); + if (!areq) { + ret = -ENOMEM; + goto free_ahash; + } + + crypto_ahash_clear_flags(tfm, ~0); + blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + + ipad = kcalloc(2, blocksize, GFP_KERNEL); + if (!ipad) { + ret = -ENOMEM; + goto free_request; + } + + opad = ipad + blocksize; + + ret = safexcel_hmac_init_pad(areq, blocksize, key, keylen, ipad, opad); + if (ret) + goto free_ipad; + + ret = safexcel_hmac_init_iv(areq, blocksize, ipad, istate); + if (ret) + goto free_ipad; + + ret = safexcel_hmac_init_iv(areq, blocksize, opad, ostate); + +free_ipad: + kfree(ipad); +free_request: + ahash_request_free(areq); +free_ahash: + crypto_free_ahash(tfm); + + return ret; +} + +int safexcel_hmac_setkey(struct safexcel_context *base, const u8 *key, + unsigned int keylen, const char *alg, + unsigned int state_sz) +{ + struct safexcel_crypto_priv *priv = base->priv; + struct safexcel_ahash_export_state istate, ostate; + int ret; + + ret = __safexcel_hmac_setkey(alg, key, keylen, &istate, &ostate); + if (ret) + return ret; + + if (priv->flags & EIP197_TRC_CACHE && base->ctxr && + (memcmp(&base->ipad, istate.state, state_sz) || + memcmp(&base->opad, ostate.state, state_sz))) + base->needs_inv = true; + + memcpy(&base->ipad, &istate.state, state_sz); + memcpy(&base->opad, &ostate.state, state_sz); + + return 0; +} + +static int safexcel_hmac_alg_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen, const char *alg, + unsigned int state_sz) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + + return safexcel_hmac_setkey(&ctx->base, key, keylen, alg, state_sz); +} + +static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha1", + SHA1_DIGEST_SIZE); +} + +struct safexcel_alg_template safexcel_alg_hmac_sha1 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA1, + .alg.ahash = { + .init = safexcel_hmac_sha1_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_hmac_sha1_digest, + .setkey = safexcel_hmac_sha1_setkey, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "hmac(sha1)", + .cra_driver_name = "safexcel-hmac-sha1", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_ahash_cra_init, + .cra_exit = safexcel_ahash_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_sha256_init(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + + memset(req, 0, sizeof(*req)); + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256; + req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; + req->state_sz = SHA256_DIGEST_SIZE; + req->digest_sz = SHA256_DIGEST_SIZE; + req->block_sz = SHA256_BLOCK_SIZE; + + return 0; +} + +static int safexcel_sha256_digest(struct ahash_request *areq) +{ + int ret = safexcel_sha256_init(areq); + + if (ret) + return ret; + + return safexcel_ahash_finup(areq); +} + +struct safexcel_alg_template safexcel_alg_sha256 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA2_256, + .alg.ahash = { + .init = safexcel_sha256_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_sha256_digest, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "sha256", + .cra_driver_name = "safexcel-sha256", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_ahash_cra_init, + .cra_exit = safexcel_ahash_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_sha224_init(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + + memset(req, 0, sizeof(*req)); + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224; + req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; + req->state_sz = SHA256_DIGEST_SIZE; + req->digest_sz = SHA256_DIGEST_SIZE; + req->block_sz = SHA256_BLOCK_SIZE; + + return 0; +} + +static int safexcel_sha224_digest(struct ahash_request *areq) +{ + int ret = safexcel_sha224_init(areq); + + if (ret) + return ret; + + return safexcel_ahash_finup(areq); +} + +struct safexcel_alg_template safexcel_alg_sha224 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA2_256, + .alg.ahash = { + .init = safexcel_sha224_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_sha224_digest, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "sha224", + .cra_driver_name = "safexcel-sha224", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_ahash_cra_init, + .cra_exit = safexcel_ahash_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_hmac_sha224_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha224", + SHA256_DIGEST_SIZE); +} + +static int safexcel_hmac_sha224_init(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + + memset(req, 0, sizeof(*req)); + + /* Start from ipad precompute */ + memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE); + /* Already processed the key^ipad part now! */ + req->len = SHA256_BLOCK_SIZE; + req->processed = SHA256_BLOCK_SIZE; + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224; + req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; + req->state_sz = SHA256_DIGEST_SIZE; + req->digest_sz = SHA256_DIGEST_SIZE; + req->block_sz = SHA256_BLOCK_SIZE; + req->hmac = true; + + return 0; +} + +static int safexcel_hmac_sha224_digest(struct ahash_request *areq) +{ + int ret = safexcel_hmac_sha224_init(areq); + + if (ret) + return ret; + + return safexcel_ahash_finup(areq); +} + +struct safexcel_alg_template safexcel_alg_hmac_sha224 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA2_256, + .alg.ahash = { + .init = safexcel_hmac_sha224_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_hmac_sha224_digest, + .setkey = safexcel_hmac_sha224_setkey, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "hmac(sha224)", + .cra_driver_name = "safexcel-hmac-sha224", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_ahash_cra_init, + .cra_exit = safexcel_ahash_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_hmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha256", + SHA256_DIGEST_SIZE); +} + +static int safexcel_hmac_sha256_init(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + + memset(req, 0, sizeof(*req)); + + /* Start from ipad precompute */ + memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE); + /* Already processed the key^ipad part now! */ + req->len = SHA256_BLOCK_SIZE; + req->processed = SHA256_BLOCK_SIZE; + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256; + req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; + req->state_sz = SHA256_DIGEST_SIZE; + req->digest_sz = SHA256_DIGEST_SIZE; + req->block_sz = SHA256_BLOCK_SIZE; + req->hmac = true; + + return 0; +} + +static int safexcel_hmac_sha256_digest(struct ahash_request *areq) +{ + int ret = safexcel_hmac_sha256_init(areq); + + if (ret) + return ret; + + return safexcel_ahash_finup(areq); +} + +struct safexcel_alg_template safexcel_alg_hmac_sha256 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA2_256, + .alg.ahash = { + .init = safexcel_hmac_sha256_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_hmac_sha256_digest, + .setkey = safexcel_hmac_sha256_setkey, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "safexcel-hmac-sha256", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_ahash_cra_init, + .cra_exit = safexcel_ahash_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_sha512_init(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + + memset(req, 0, sizeof(*req)); + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512; + req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; + req->state_sz = SHA512_DIGEST_SIZE; + req->digest_sz = SHA512_DIGEST_SIZE; + req->block_sz = SHA512_BLOCK_SIZE; + + return 0; +} + +static int safexcel_sha512_digest(struct ahash_request *areq) +{ + int ret = safexcel_sha512_init(areq); + + if (ret) + return ret; + + return safexcel_ahash_finup(areq); +} + +struct safexcel_alg_template safexcel_alg_sha512 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA2_512, + .alg.ahash = { + .init = safexcel_sha512_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_sha512_digest, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "sha512", + .cra_driver_name = "safexcel-sha512", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_ahash_cra_init, + .cra_exit = safexcel_ahash_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_sha384_init(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + + memset(req, 0, sizeof(*req)); + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384; + req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; + req->state_sz = SHA512_DIGEST_SIZE; + req->digest_sz = SHA512_DIGEST_SIZE; + req->block_sz = SHA512_BLOCK_SIZE; + + return 0; +} + +static int safexcel_sha384_digest(struct ahash_request *areq) +{ + int ret = safexcel_sha384_init(areq); + + if (ret) + return ret; + + return safexcel_ahash_finup(areq); +} + +struct safexcel_alg_template safexcel_alg_sha384 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA2_512, + .alg.ahash = { + .init = safexcel_sha384_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_sha384_digest, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "sha384", + .cra_driver_name = "safexcel-sha384", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_ahash_cra_init, + .cra_exit = safexcel_ahash_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_hmac_sha512_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha512", + SHA512_DIGEST_SIZE); +} + +static int safexcel_hmac_sha512_init(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + + memset(req, 0, sizeof(*req)); + + /* Start from ipad precompute */ + memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE); + /* Already processed the key^ipad part now! */ + req->len = SHA512_BLOCK_SIZE; + req->processed = SHA512_BLOCK_SIZE; + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512; + req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; + req->state_sz = SHA512_DIGEST_SIZE; + req->digest_sz = SHA512_DIGEST_SIZE; + req->block_sz = SHA512_BLOCK_SIZE; + req->hmac = true; + + return 0; +} + +static int safexcel_hmac_sha512_digest(struct ahash_request *areq) +{ + int ret = safexcel_hmac_sha512_init(areq); + + if (ret) + return ret; + + return safexcel_ahash_finup(areq); +} + +struct safexcel_alg_template safexcel_alg_hmac_sha512 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA2_512, + .alg.ahash = { + .init = safexcel_hmac_sha512_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_hmac_sha512_digest, + .setkey = safexcel_hmac_sha512_setkey, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "hmac(sha512)", + .cra_driver_name = "safexcel-hmac-sha512", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_ahash_cra_init, + .cra_exit = safexcel_ahash_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_hmac_sha384_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha384", + SHA512_DIGEST_SIZE); +} + +static int safexcel_hmac_sha384_init(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + + memset(req, 0, sizeof(*req)); + + /* Start from ipad precompute */ + memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE); + /* Already processed the key^ipad part now! */ + req->len = SHA512_BLOCK_SIZE; + req->processed = SHA512_BLOCK_SIZE; + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384; + req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; + req->state_sz = SHA512_DIGEST_SIZE; + req->digest_sz = SHA512_DIGEST_SIZE; + req->block_sz = SHA512_BLOCK_SIZE; + req->hmac = true; + + return 0; +} + +static int safexcel_hmac_sha384_digest(struct ahash_request *areq) +{ + int ret = safexcel_hmac_sha384_init(areq); + + if (ret) + return ret; + + return safexcel_ahash_finup(areq); +} + +struct safexcel_alg_template safexcel_alg_hmac_sha384 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA2_512, + .alg.ahash = { + .init = safexcel_hmac_sha384_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_hmac_sha384_digest, + .setkey = safexcel_hmac_sha384_setkey, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "hmac(sha384)", + .cra_driver_name = "safexcel-hmac-sha384", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_ahash_cra_init, + .cra_exit = safexcel_ahash_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_md5_init(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + + memset(req, 0, sizeof(*req)); + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5; + req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; + req->state_sz = MD5_DIGEST_SIZE; + req->digest_sz = MD5_DIGEST_SIZE; + req->block_sz = MD5_HMAC_BLOCK_SIZE; + + return 0; +} + +static int safexcel_md5_digest(struct ahash_request *areq) +{ + int ret = safexcel_md5_init(areq); + + if (ret) + return ret; + + return safexcel_ahash_finup(areq); +} + +struct safexcel_alg_template safexcel_alg_md5 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_MD5, + .alg.ahash = { + .init = safexcel_md5_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_md5_digest, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "md5", + .cra_driver_name = "safexcel-md5", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_ahash_cra_init, + .cra_exit = safexcel_ahash_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_hmac_md5_init(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + + memset(req, 0, sizeof(*req)); + + /* Start from ipad precompute */ + memcpy(req->state, &ctx->base.ipad, MD5_DIGEST_SIZE); + /* Already processed the key^ipad part now! */ + req->len = MD5_HMAC_BLOCK_SIZE; + req->processed = MD5_HMAC_BLOCK_SIZE; + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5; + req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; + req->state_sz = MD5_DIGEST_SIZE; + req->digest_sz = MD5_DIGEST_SIZE; + req->block_sz = MD5_HMAC_BLOCK_SIZE; + req->len_is_le = true; /* MD5 is little endian! ... */ + req->hmac = true; + + return 0; +} + +static int safexcel_hmac_md5_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-md5", + MD5_DIGEST_SIZE); +} + +static int safexcel_hmac_md5_digest(struct ahash_request *areq) +{ + int ret = safexcel_hmac_md5_init(areq); + + if (ret) + return ret; + + return safexcel_ahash_finup(areq); +} + +struct safexcel_alg_template safexcel_alg_hmac_md5 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_MD5, + .alg.ahash = { + .init = safexcel_hmac_md5_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_hmac_md5_digest, + .setkey = safexcel_hmac_md5_setkey, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "hmac(md5)", + .cra_driver_name = "safexcel-hmac-md5", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_ahash_cra_init, + .cra_exit = safexcel_ahash_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_cbcmac_init(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + + memset(req, 0, sizeof(*req)); + + /* Start from loaded keys */ + memcpy(req->state, &ctx->base.ipad, ctx->key_sz); + /* Set processed to non-zero to enable invalidation detection */ + req->len = AES_BLOCK_SIZE; + req->processed = AES_BLOCK_SIZE; + + req->digest = CONTEXT_CONTROL_DIGEST_XCM; + req->state_sz = ctx->key_sz; + req->digest_sz = AES_BLOCK_SIZE; + req->block_sz = AES_BLOCK_SIZE; + req->xcbcmac = true; + + return 0; +} + +static int safexcel_cbcmac_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int len) +{ + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); + struct crypto_aes_ctx aes; + int ret, i; + + ret = aes_expandkey(&aes, key, len); + if (ret) + return ret; + + memset(&ctx->base.ipad, 0, 2 * AES_BLOCK_SIZE); + for (i = 0; i < len / sizeof(u32); i++) + ctx->base.ipad.be[i + 8] = cpu_to_be32(aes.key_enc[i]); + + if (len == AES_KEYSIZE_192) { + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192; + ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE; + } else if (len == AES_KEYSIZE_256) { + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256; + ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE; + } else { + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; + ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE; + } + ctx->cbcmac = true; + + memzero_explicit(&aes, sizeof(aes)); + return 0; +} + +static int safexcel_cbcmac_digest(struct ahash_request *areq) +{ + return safexcel_cbcmac_init(areq) ?: safexcel_ahash_finup(areq); +} + +struct safexcel_alg_template safexcel_alg_cbcmac = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = 0, + .alg.ahash = { + .init = safexcel_cbcmac_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_cbcmac_digest, + .setkey = safexcel_cbcmac_setkey, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = AES_BLOCK_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "cbcmac(aes)", + .cra_driver_name = "safexcel-cbcmac-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_ahash_cra_init, + .cra_exit = safexcel_ahash_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_xcbcmac_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int len) +{ + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); + u32 key_tmp[3 * AES_BLOCK_SIZE / sizeof(u32)]; + int ret, i; + + ret = aes_prepareenckey(ctx->aes, key, len); + if (ret) + return ret; + + /* precompute the XCBC key material */ + aes_encrypt(ctx->aes, (u8 *)key_tmp + 2 * AES_BLOCK_SIZE, + "\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); + aes_encrypt(ctx->aes, (u8 *)key_tmp, + "\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2"); + aes_encrypt(ctx->aes, (u8 *)key_tmp + AES_BLOCK_SIZE, + "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3"); + for (i = 0; i < 3 * AES_BLOCK_SIZE / sizeof(u32); i++) + ctx->base.ipad.word[i] = swab32(key_tmp[i]); + + ret = aes_prepareenckey(ctx->aes, + (u8 *)key_tmp + 2 * AES_BLOCK_SIZE, + AES_MIN_KEY_SIZE); + if (ret) + return ret; + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; + ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE; + ctx->cbcmac = false; + + return 0; +} + +static int safexcel_xcbcmac_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_ahash_cra_init(tfm); + ctx->aes = kmalloc_obj(*ctx->aes); + return ctx->aes == NULL ? -ENOMEM : 0; +} + +static void safexcel_xcbcmac_cra_exit(struct crypto_tfm *tfm) +{ + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); + + kfree(ctx->aes); + safexcel_ahash_cra_exit(tfm); +} + +struct safexcel_alg_template safexcel_alg_xcbcmac = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = 0, + .alg.ahash = { + .init = safexcel_cbcmac_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_cbcmac_digest, + .setkey = safexcel_xcbcmac_setkey, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = AES_BLOCK_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "xcbc(aes)", + .cra_driver_name = "safexcel-xcbc-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_xcbcmac_cra_init, + .cra_exit = safexcel_xcbcmac_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int len) +{ + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); + __be64 consts[4]; + u64 _const[2]; + u8 msb_mask, gfmask; + int ret, i; + + /* precompute the CMAC key material */ + ret = aes_prepareenckey(ctx->aes, key, len); + if (ret) + return ret; + + for (i = 0; i < len / sizeof(u32); i++) + ctx->base.ipad.word[i + 8] = get_unaligned_be32(&key[4 * i]); + + /* code below borrowed from crypto/cmac.c */ + /* encrypt the zero block */ + memset(consts, 0, AES_BLOCK_SIZE); + aes_encrypt(ctx->aes, (u8 *)consts, (u8 *)consts); + + gfmask = 0x87; + _const[0] = be64_to_cpu(consts[1]); + _const[1] = be64_to_cpu(consts[0]); + + /* gf(2^128) multiply zero-ciphertext with u and u^2 */ + for (i = 0; i < 4; i += 2) { + msb_mask = ((s64)_const[1] >> 63) & gfmask; + _const[1] = (_const[1] << 1) | (_const[0] >> 63); + _const[0] = (_const[0] << 1) ^ msb_mask; + + consts[i + 0] = cpu_to_be64(_const[1]); + consts[i + 1] = cpu_to_be64(_const[0]); + } + /* end of code borrowed from crypto/cmac.c */ + + for (i = 0; i < 2 * AES_BLOCK_SIZE / sizeof(u32); i++) + ctx->base.ipad.be[i] = cpu_to_be32(((u32 *)consts)[i]); + + if (len == AES_KEYSIZE_192) { + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192; + ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE; + } else if (len == AES_KEYSIZE_256) { + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256; + ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE; + } else { + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; + ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE; + } + ctx->cbcmac = false; + + return 0; +} + +struct safexcel_alg_template safexcel_alg_cmac = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = 0, + .alg.ahash = { + .init = safexcel_cbcmac_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_cbcmac_digest, + .setkey = safexcel_cmac_setkey, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = AES_BLOCK_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "cmac(aes)", + .cra_driver_name = "safexcel-cmac-aes", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_xcbcmac_cra_init, + .cra_exit = safexcel_xcbcmac_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_sm3_init(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + + memset(req, 0, sizeof(*req)); + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3; + req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; + req->state_sz = SM3_DIGEST_SIZE; + req->digest_sz = SM3_DIGEST_SIZE; + req->block_sz = SM3_BLOCK_SIZE; + + return 0; +} + +static int safexcel_sm3_digest(struct ahash_request *areq) +{ + int ret = safexcel_sm3_init(areq); + + if (ret) + return ret; + + return safexcel_ahash_finup(areq); +} + +struct safexcel_alg_template safexcel_alg_sm3 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SM3, + .alg.ahash = { + .init = safexcel_sm3_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_sm3_digest, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = SM3_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "sm3", + .cra_driver_name = "safexcel-sm3", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SM3_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_ahash_cra_init, + .cra_exit = safexcel_ahash_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_hmac_sm3_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sm3", + SM3_DIGEST_SIZE); +} + +static int safexcel_hmac_sm3_init(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + + memset(req, 0, sizeof(*req)); + + /* Start from ipad precompute */ + memcpy(req->state, &ctx->base.ipad, SM3_DIGEST_SIZE); + /* Already processed the key^ipad part now! */ + req->len = SM3_BLOCK_SIZE; + req->processed = SM3_BLOCK_SIZE; + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3; + req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; + req->state_sz = SM3_DIGEST_SIZE; + req->digest_sz = SM3_DIGEST_SIZE; + req->block_sz = SM3_BLOCK_SIZE; + req->hmac = true; + + return 0; +} + +static int safexcel_hmac_sm3_digest(struct ahash_request *areq) +{ + int ret = safexcel_hmac_sm3_init(areq); + + if (ret) + return ret; + + return safexcel_ahash_finup(areq); +} + +struct safexcel_alg_template safexcel_alg_hmac_sm3 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SM3, + .alg.ahash = { + .init = safexcel_hmac_sm3_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_hmac_sm3_digest, + .setkey = safexcel_hmac_sm3_setkey, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = SM3_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "hmac(sm3)", + .cra_driver_name = "safexcel-hmac-sm3", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SM3_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_ahash_cra_init, + .cra_exit = safexcel_ahash_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_sha3_224_init(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + + memset(req, 0, sizeof(*req)); + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224; + req->digest = CONTEXT_CONTROL_DIGEST_INITIAL; + req->state_sz = SHA3_224_DIGEST_SIZE; + req->digest_sz = SHA3_224_DIGEST_SIZE; + req->block_sz = SHA3_224_BLOCK_SIZE; + ctx->do_fallback = false; + ctx->fb_init_done = false; + return 0; +} + +static int safexcel_sha3_fbcheck(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct ahash_request *subreq = ahash_request_ctx_dma(req); + int ret = 0; + + if (ctx->do_fallback) { + ahash_request_set_tfm(subreq, ctx->fback); + ahash_request_set_callback(subreq, req->base.flags, + req->base.complete, req->base.data); + ahash_request_set_crypt(subreq, req->src, req->result, + req->nbytes); + if (!ctx->fb_init_done) { + if (ctx->fb_do_setkey) { + /* Set fallback cipher HMAC key */ + u8 key[SHA3_224_BLOCK_SIZE]; + + memcpy(key, &ctx->base.ipad, + crypto_ahash_blocksize(ctx->fback) / 2); + memcpy(key + + crypto_ahash_blocksize(ctx->fback) / 2, + &ctx->base.opad, + crypto_ahash_blocksize(ctx->fback) / 2); + ret = crypto_ahash_setkey(ctx->fback, key, + crypto_ahash_blocksize(ctx->fback)); + memzero_explicit(key, + crypto_ahash_blocksize(ctx->fback)); + ctx->fb_do_setkey = false; + } + ret = ret ?: crypto_ahash_init(subreq); + ctx->fb_init_done = true; + } + } + return ret; +} + +static int safexcel_sha3_update(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct ahash_request *subreq = ahash_request_ctx_dma(req); + + ctx->do_fallback = true; + return safexcel_sha3_fbcheck(req) ?: crypto_ahash_update(subreq); +} + +static int safexcel_sha3_final(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct ahash_request *subreq = ahash_request_ctx_dma(req); + + ctx->do_fallback = true; + return safexcel_sha3_fbcheck(req) ?: crypto_ahash_final(subreq); +} + +static int safexcel_sha3_finup(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct ahash_request *subreq = ahash_request_ctx_dma(req); + + ctx->do_fallback |= !req->nbytes; + if (ctx->do_fallback) + /* Update or ex/import happened or len 0, cannot use the HW */ + return safexcel_sha3_fbcheck(req) ?: + crypto_ahash_finup(subreq); + else + return safexcel_ahash_finup(req); +} + +static int safexcel_sha3_digest_fallback(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct ahash_request *subreq = ahash_request_ctx_dma(req); + + ctx->do_fallback = true; + ctx->fb_init_done = false; + return safexcel_sha3_fbcheck(req) ?: crypto_ahash_finup(subreq); +} + +static int safexcel_sha3_224_digest(struct ahash_request *req) +{ + if (req->nbytes) + return safexcel_sha3_224_init(req) ?: safexcel_ahash_finup(req); + + /* HW cannot do zero length hash, use fallback instead */ + return safexcel_sha3_digest_fallback(req); +} + +static int safexcel_sha3_export(struct ahash_request *req, void *out) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct ahash_request *subreq = ahash_request_ctx_dma(req); + + ctx->do_fallback = true; + return safexcel_sha3_fbcheck(req) ?: crypto_ahash_export(subreq, out); +} + +static int safexcel_sha3_import(struct ahash_request *req, const void *in) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct ahash_request *subreq = ahash_request_ctx_dma(req); + + ctx->do_fallback = true; + return safexcel_sha3_fbcheck(req) ?: crypto_ahash_import(subreq, in); + // return safexcel_ahash_import(req, in); +} + +static int safexcel_sha3_cra_init(struct crypto_tfm *tfm) +{ + struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); + + safexcel_ahash_cra_init(tfm); + + /* Allocate fallback implementation */ + ctx->fback = crypto_alloc_ahash(crypto_tfm_alg_name(tfm), 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fback)) + return PTR_ERR(ctx->fback); + + /* Update statesize from fallback algorithm! */ + crypto_hash_alg_common(ahash)->statesize = + crypto_ahash_statesize(ctx->fback); + crypto_ahash_set_reqsize_dma( + ahash, max(sizeof(struct safexcel_ahash_req), + sizeof(struct ahash_request) + + crypto_ahash_reqsize(ctx->fback))); + return 0; +} + +static void safexcel_sha3_cra_exit(struct crypto_tfm *tfm) +{ + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); + + crypto_free_ahash(ctx->fback); + safexcel_ahash_cra_exit(tfm); +} + +struct safexcel_alg_template safexcel_alg_sha3_224 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA3, + .alg.ahash = { + .init = safexcel_sha3_224_init, + .update = safexcel_sha3_update, + .final = safexcel_sha3_final, + .finup = safexcel_sha3_finup, + .digest = safexcel_sha3_224_digest, + .export = safexcel_sha3_export, + .import = safexcel_sha3_import, + .halg = { + .digestsize = SHA3_224_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "sha3-224", + .cra_driver_name = "safexcel-sha3-224", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA3_224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_sha3_cra_init, + .cra_exit = safexcel_sha3_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_sha3_256_init(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + + memset(req, 0, sizeof(*req)); + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256; + req->digest = CONTEXT_CONTROL_DIGEST_INITIAL; + req->state_sz = SHA3_256_DIGEST_SIZE; + req->digest_sz = SHA3_256_DIGEST_SIZE; + req->block_sz = SHA3_256_BLOCK_SIZE; + ctx->do_fallback = false; + ctx->fb_init_done = false; + return 0; +} + +static int safexcel_sha3_256_digest(struct ahash_request *req) +{ + if (req->nbytes) + return safexcel_sha3_256_init(req) ?: safexcel_ahash_finup(req); + + /* HW cannot do zero length hash, use fallback instead */ + return safexcel_sha3_digest_fallback(req); +} + +struct safexcel_alg_template safexcel_alg_sha3_256 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA3, + .alg.ahash = { + .init = safexcel_sha3_256_init, + .update = safexcel_sha3_update, + .final = safexcel_sha3_final, + .finup = safexcel_sha3_finup, + .digest = safexcel_sha3_256_digest, + .export = safexcel_sha3_export, + .import = safexcel_sha3_import, + .halg = { + .digestsize = SHA3_256_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "sha3-256", + .cra_driver_name = "safexcel-sha3-256", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA3_256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_sha3_cra_init, + .cra_exit = safexcel_sha3_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_sha3_384_init(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + + memset(req, 0, sizeof(*req)); + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384; + req->digest = CONTEXT_CONTROL_DIGEST_INITIAL; + req->state_sz = SHA3_384_DIGEST_SIZE; + req->digest_sz = SHA3_384_DIGEST_SIZE; + req->block_sz = SHA3_384_BLOCK_SIZE; + ctx->do_fallback = false; + ctx->fb_init_done = false; + return 0; +} + +static int safexcel_sha3_384_digest(struct ahash_request *req) +{ + if (req->nbytes) + return safexcel_sha3_384_init(req) ?: safexcel_ahash_finup(req); + + /* HW cannot do zero length hash, use fallback instead */ + return safexcel_sha3_digest_fallback(req); +} + +struct safexcel_alg_template safexcel_alg_sha3_384 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA3, + .alg.ahash = { + .init = safexcel_sha3_384_init, + .update = safexcel_sha3_update, + .final = safexcel_sha3_final, + .finup = safexcel_sha3_finup, + .digest = safexcel_sha3_384_digest, + .export = safexcel_sha3_export, + .import = safexcel_sha3_import, + .halg = { + .digestsize = SHA3_384_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "sha3-384", + .cra_driver_name = "safexcel-sha3-384", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA3_384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_sha3_cra_init, + .cra_exit = safexcel_sha3_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_sha3_512_init(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + + memset(req, 0, sizeof(*req)); + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512; + req->digest = CONTEXT_CONTROL_DIGEST_INITIAL; + req->state_sz = SHA3_512_DIGEST_SIZE; + req->digest_sz = SHA3_512_DIGEST_SIZE; + req->block_sz = SHA3_512_BLOCK_SIZE; + ctx->do_fallback = false; + ctx->fb_init_done = false; + return 0; +} + +static int safexcel_sha3_512_digest(struct ahash_request *req) +{ + if (req->nbytes) + return safexcel_sha3_512_init(req) ?: safexcel_ahash_finup(req); + + /* HW cannot do zero length hash, use fallback instead */ + return safexcel_sha3_digest_fallback(req); +} + +struct safexcel_alg_template safexcel_alg_sha3_512 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA3, + .alg.ahash = { + .init = safexcel_sha3_512_init, + .update = safexcel_sha3_update, + .final = safexcel_sha3_final, + .finup = safexcel_sha3_finup, + .digest = safexcel_sha3_512_digest, + .export = safexcel_sha3_export, + .import = safexcel_sha3_import, + .halg = { + .digestsize = SHA3_512_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "sha3-512", + .cra_driver_name = "safexcel-sha3-512", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA3_512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_sha3_cra_init, + .cra_exit = safexcel_sha3_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_hmac_sha3_cra_init(struct crypto_tfm *tfm, const char *alg) +{ + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); + int ret; + + ret = safexcel_sha3_cra_init(tfm); + if (ret) + return ret; + + /* Allocate precalc basic digest implementation */ + ctx->shpre = crypto_alloc_shash(alg, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->shpre)) + return PTR_ERR(ctx->shpre); + + ctx->shdesc = kmalloc(sizeof(*ctx->shdesc) + + crypto_shash_descsize(ctx->shpre), GFP_KERNEL); + if (!ctx->shdesc) { + crypto_free_shash(ctx->shpre); + return -ENOMEM; + } + ctx->shdesc->tfm = ctx->shpre; + return 0; +} + +static void safexcel_hmac_sha3_cra_exit(struct crypto_tfm *tfm) +{ + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); + + crypto_free_ahash(ctx->fback); + crypto_free_shash(ctx->shpre); + kfree(ctx->shdesc); + safexcel_ahash_cra_exit(tfm); +} + +static int safexcel_hmac_sha3_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + int ret = 0; + + if (keylen > crypto_ahash_blocksize(tfm)) { + /* + * If the key is larger than the blocksize, then hash it + * first using our fallback cipher + */ + ret = crypto_shash_digest(ctx->shdesc, key, keylen, + ctx->base.ipad.byte); + keylen = crypto_shash_digestsize(ctx->shpre); + + /* + * If the digest is larger than half the blocksize, we need to + * move the rest to opad due to the way our HMAC infra works. + */ + if (keylen > crypto_ahash_blocksize(tfm) / 2) + /* Buffers overlap, need to use memmove iso memcpy! */ + memmove(&ctx->base.opad, + ctx->base.ipad.byte + + crypto_ahash_blocksize(tfm) / 2, + keylen - crypto_ahash_blocksize(tfm) / 2); + } else { + /* + * Copy the key to our ipad & opad buffers + * Note that ipad and opad each contain one half of the key, + * to match the existing HMAC driver infrastructure. + */ + if (keylen <= crypto_ahash_blocksize(tfm) / 2) { + memcpy(&ctx->base.ipad, key, keylen); + } else { + memcpy(&ctx->base.ipad, key, + crypto_ahash_blocksize(tfm) / 2); + memcpy(&ctx->base.opad, + key + crypto_ahash_blocksize(tfm) / 2, + keylen - crypto_ahash_blocksize(tfm) / 2); + } + } + + /* Pad key with zeroes */ + if (keylen <= crypto_ahash_blocksize(tfm) / 2) { + memset(ctx->base.ipad.byte + keylen, 0, + crypto_ahash_blocksize(tfm) / 2 - keylen); + memset(&ctx->base.opad, 0, crypto_ahash_blocksize(tfm) / 2); + } else { + memset(ctx->base.opad.byte + keylen - + crypto_ahash_blocksize(tfm) / 2, 0, + crypto_ahash_blocksize(tfm) - keylen); + } + + /* If doing fallback, still need to set the new key! */ + ctx->fb_do_setkey = true; + return ret; +} + +static int safexcel_hmac_sha3_224_init(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + + memset(req, 0, sizeof(*req)); + + /* Copy (half of) the key */ + memcpy(req->state, &ctx->base.ipad, SHA3_224_BLOCK_SIZE / 2); + /* Start of HMAC should have len == processed == blocksize */ + req->len = SHA3_224_BLOCK_SIZE; + req->processed = SHA3_224_BLOCK_SIZE; + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224; + req->digest = CONTEXT_CONTROL_DIGEST_HMAC; + req->state_sz = SHA3_224_BLOCK_SIZE / 2; + req->digest_sz = SHA3_224_DIGEST_SIZE; + req->block_sz = SHA3_224_BLOCK_SIZE; + req->hmac = true; + ctx->do_fallback = false; + ctx->fb_init_done = false; + return 0; +} + +static int safexcel_hmac_sha3_224_digest(struct ahash_request *req) +{ + if (req->nbytes) + return safexcel_hmac_sha3_224_init(req) ?: + safexcel_ahash_finup(req); + + /* HW cannot do zero length HMAC, use fallback instead */ + return safexcel_sha3_digest_fallback(req); +} + +static int safexcel_hmac_sha3_224_cra_init(struct crypto_tfm *tfm) +{ + return safexcel_hmac_sha3_cra_init(tfm, "sha3-224"); +} + +struct safexcel_alg_template safexcel_alg_hmac_sha3_224 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA3, + .alg.ahash = { + .init = safexcel_hmac_sha3_224_init, + .update = safexcel_sha3_update, + .final = safexcel_sha3_final, + .finup = safexcel_sha3_finup, + .digest = safexcel_hmac_sha3_224_digest, + .setkey = safexcel_hmac_sha3_setkey, + .export = safexcel_sha3_export, + .import = safexcel_sha3_import, + .halg = { + .digestsize = SHA3_224_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "hmac(sha3-224)", + .cra_driver_name = "safexcel-hmac-sha3-224", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA3_224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_hmac_sha3_224_cra_init, + .cra_exit = safexcel_hmac_sha3_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_hmac_sha3_256_init(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + + memset(req, 0, sizeof(*req)); + + /* Copy (half of) the key */ + memcpy(req->state, &ctx->base.ipad, SHA3_256_BLOCK_SIZE / 2); + /* Start of HMAC should have len == processed == blocksize */ + req->len = SHA3_256_BLOCK_SIZE; + req->processed = SHA3_256_BLOCK_SIZE; + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256; + req->digest = CONTEXT_CONTROL_DIGEST_HMAC; + req->state_sz = SHA3_256_BLOCK_SIZE / 2; + req->digest_sz = SHA3_256_DIGEST_SIZE; + req->block_sz = SHA3_256_BLOCK_SIZE; + req->hmac = true; + ctx->do_fallback = false; + ctx->fb_init_done = false; + return 0; +} + +static int safexcel_hmac_sha3_256_digest(struct ahash_request *req) +{ + if (req->nbytes) + return safexcel_hmac_sha3_256_init(req) ?: + safexcel_ahash_finup(req); + + /* HW cannot do zero length HMAC, use fallback instead */ + return safexcel_sha3_digest_fallback(req); +} + +static int safexcel_hmac_sha3_256_cra_init(struct crypto_tfm *tfm) +{ + return safexcel_hmac_sha3_cra_init(tfm, "sha3-256"); +} + +struct safexcel_alg_template safexcel_alg_hmac_sha3_256 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA3, + .alg.ahash = { + .init = safexcel_hmac_sha3_256_init, + .update = safexcel_sha3_update, + .final = safexcel_sha3_final, + .finup = safexcel_sha3_finup, + .digest = safexcel_hmac_sha3_256_digest, + .setkey = safexcel_hmac_sha3_setkey, + .export = safexcel_sha3_export, + .import = safexcel_sha3_import, + .halg = { + .digestsize = SHA3_256_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "hmac(sha3-256)", + .cra_driver_name = "safexcel-hmac-sha3-256", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA3_256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_hmac_sha3_256_cra_init, + .cra_exit = safexcel_hmac_sha3_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_hmac_sha3_384_init(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + + memset(req, 0, sizeof(*req)); + + /* Copy (half of) the key */ + memcpy(req->state, &ctx->base.ipad, SHA3_384_BLOCK_SIZE / 2); + /* Start of HMAC should have len == processed == blocksize */ + req->len = SHA3_384_BLOCK_SIZE; + req->processed = SHA3_384_BLOCK_SIZE; + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384; + req->digest = CONTEXT_CONTROL_DIGEST_HMAC; + req->state_sz = SHA3_384_BLOCK_SIZE / 2; + req->digest_sz = SHA3_384_DIGEST_SIZE; + req->block_sz = SHA3_384_BLOCK_SIZE; + req->hmac = true; + ctx->do_fallback = false; + ctx->fb_init_done = false; + return 0; +} + +static int safexcel_hmac_sha3_384_digest(struct ahash_request *req) +{ + if (req->nbytes) + return safexcel_hmac_sha3_384_init(req) ?: + safexcel_ahash_finup(req); + + /* HW cannot do zero length HMAC, use fallback instead */ + return safexcel_sha3_digest_fallback(req); +} + +static int safexcel_hmac_sha3_384_cra_init(struct crypto_tfm *tfm) +{ + return safexcel_hmac_sha3_cra_init(tfm, "sha3-384"); +} + +struct safexcel_alg_template safexcel_alg_hmac_sha3_384 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA3, + .alg.ahash = { + .init = safexcel_hmac_sha3_384_init, + .update = safexcel_sha3_update, + .final = safexcel_sha3_final, + .finup = safexcel_sha3_finup, + .digest = safexcel_hmac_sha3_384_digest, + .setkey = safexcel_hmac_sha3_setkey, + .export = safexcel_sha3_export, + .import = safexcel_sha3_import, + .halg = { + .digestsize = SHA3_384_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "hmac(sha3-384)", + .cra_driver_name = "safexcel-hmac-sha3-384", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA3_384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_hmac_sha3_384_cra_init, + .cra_exit = safexcel_hmac_sha3_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_hmac_sha3_512_init(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); + + memset(req, 0, sizeof(*req)); + + /* Copy (half of) the key */ + memcpy(req->state, &ctx->base.ipad, SHA3_512_BLOCK_SIZE / 2); + /* Start of HMAC should have len == processed == blocksize */ + req->len = SHA3_512_BLOCK_SIZE; + req->processed = SHA3_512_BLOCK_SIZE; + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512; + req->digest = CONTEXT_CONTROL_DIGEST_HMAC; + req->state_sz = SHA3_512_BLOCK_SIZE / 2; + req->digest_sz = SHA3_512_DIGEST_SIZE; + req->block_sz = SHA3_512_BLOCK_SIZE; + req->hmac = true; + ctx->do_fallback = false; + ctx->fb_init_done = false; + return 0; +} + +static int safexcel_hmac_sha3_512_digest(struct ahash_request *req) +{ + if (req->nbytes) + return safexcel_hmac_sha3_512_init(req) ?: + safexcel_ahash_finup(req); + + /* HW cannot do zero length HMAC, use fallback instead */ + return safexcel_sha3_digest_fallback(req); +} + +static int safexcel_hmac_sha3_512_cra_init(struct crypto_tfm *tfm) +{ + return safexcel_hmac_sha3_cra_init(tfm, "sha3-512"); +} +struct safexcel_alg_template safexcel_alg_hmac_sha3_512 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .algo_mask = SAFEXCEL_ALG_SHA3, + .alg.ahash = { + .init = safexcel_hmac_sha3_512_init, + .update = safexcel_sha3_update, + .final = safexcel_sha3_final, + .finup = safexcel_sha3_finup, + .digest = safexcel_hmac_sha3_512_digest, + .setkey = safexcel_hmac_sha3_setkey, + .export = safexcel_sha3_export, + .import = safexcel_sha3_import, + .halg = { + .digestsize = SHA3_512_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "hmac(sha3-512)", + .cra_driver_name = "safexcel-hmac-sha3-512", + .cra_priority = SAFEXCEL_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA3_512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_hmac_sha3_512_cra_init, + .cra_exit = safexcel_hmac_sha3_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; diff --git a/drivers/crypto/inside-secure/safexcel_ring.c b/drivers/crypto/inside-secure/safexcel_ring.c new file mode 100644 index 000000000..90f15032c --- /dev/null +++ b/drivers/crypto/inside-secure/safexcel_ring.c @@ -0,0 +1,254 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2017 Marvell + * + * Antoine Tenart + */ + +#include +#include + +#include "safexcel.h" + +int safexcel_init_ring_descriptors(struct safexcel_crypto_priv *priv, + struct safexcel_desc_ring *cdr, + struct safexcel_desc_ring *rdr) +{ + int i; + struct safexcel_command_desc *cdesc; + dma_addr_t atok; + + /* Actual command descriptor ring */ + cdr->offset = priv->config.cd_offset; + cdr->base = dmam_alloc_coherent(priv->dev, + cdr->offset * EIP197_DEFAULT_RING_SIZE, + &cdr->base_dma, GFP_KERNEL); + if (!cdr->base) + return -ENOMEM; + cdr->write = cdr->base; + cdr->base_end = cdr->base + cdr->offset * (EIP197_DEFAULT_RING_SIZE - 1); + cdr->read = cdr->base; + + /* Command descriptor shadow ring for storing additional token data */ + cdr->shoffset = priv->config.cdsh_offset; + cdr->shbase = dmam_alloc_coherent(priv->dev, + cdr->shoffset * + EIP197_DEFAULT_RING_SIZE, + &cdr->shbase_dma, GFP_KERNEL); + if (!cdr->shbase) + return -ENOMEM; + cdr->shwrite = cdr->shbase; + cdr->shbase_end = cdr->shbase + cdr->shoffset * + (EIP197_DEFAULT_RING_SIZE - 1); + + /* + * Populate command descriptors with physical pointers to shadow descs. + * Note that we only need to do this once if we don't overwrite them. + */ + cdesc = cdr->base; + atok = cdr->shbase_dma; + for (i = 0; i < EIP197_DEFAULT_RING_SIZE; i++) { + cdesc->atok_lo = lower_32_bits(atok); + cdesc->atok_hi = upper_32_bits(atok); + cdesc = (void *)cdesc + cdr->offset; + atok += cdr->shoffset; + } + + rdr->offset = priv->config.rd_offset; + /* Use shoffset for result token offset here */ + rdr->shoffset = priv->config.res_offset; + rdr->base = dmam_alloc_coherent(priv->dev, + rdr->offset * EIP197_DEFAULT_RING_SIZE, + &rdr->base_dma, GFP_KERNEL); + if (!rdr->base) + return -ENOMEM; + rdr->write = rdr->base; + rdr->base_end = rdr->base + rdr->offset * (EIP197_DEFAULT_RING_SIZE - 1); + rdr->read = rdr->base; + + return 0; +} + +inline int safexcel_select_ring(struct safexcel_crypto_priv *priv) +{ + return (atomic_inc_return(&priv->ring_used) % priv->config.rings); +} + +static void *safexcel_ring_next_cwptr(struct safexcel_crypto_priv *priv, + struct safexcel_desc_ring *ring, + bool first, + struct safexcel_token **atoken) +{ + void *ptr = ring->write; + + if (first) + *atoken = ring->shwrite; + + if ((ring->write == ring->read - ring->offset) || + (ring->read == ring->base && ring->write == ring->base_end)) + return ERR_PTR(-ENOMEM); + + if (ring->write == ring->base_end) { + ring->write = ring->base; + ring->shwrite = ring->shbase; + } else { + ring->write += ring->offset; + ring->shwrite += ring->shoffset; + } + + return ptr; +} + +static void *safexcel_ring_next_rwptr(struct safexcel_crypto_priv *priv, + struct safexcel_desc_ring *ring, + struct result_data_desc **rtoken) +{ + void *ptr = ring->write; + + /* Result token at relative offset shoffset */ + *rtoken = ring->write + ring->shoffset; + + if ((ring->write == ring->read - ring->offset) || + (ring->read == ring->base && ring->write == ring->base_end)) + return ERR_PTR(-ENOMEM); + + if (ring->write == ring->base_end) + ring->write = ring->base; + else + ring->write += ring->offset; + + return ptr; +} + +void *safexcel_ring_next_rptr(struct safexcel_crypto_priv *priv, + struct safexcel_desc_ring *ring) +{ + void *ptr = ring->read; + + if (ring->write == ring->read) + return ERR_PTR(-ENOENT); + + if (ring->read == ring->base_end) + ring->read = ring->base; + else + ring->read += ring->offset; + + return ptr; +} + +inline void *safexcel_ring_curr_rptr(struct safexcel_crypto_priv *priv, + int ring) +{ + struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr; + + return rdr->read; +} + +inline int safexcel_ring_first_rdr_index(struct safexcel_crypto_priv *priv, + int ring) +{ + struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr; + + return (rdr->read - rdr->base) / rdr->offset; +} + +inline int safexcel_ring_rdr_rdesc_index(struct safexcel_crypto_priv *priv, + int ring, + struct safexcel_result_desc *rdesc) +{ + struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr; + + return ((void *)rdesc - rdr->base) / rdr->offset; +} + +void safexcel_ring_rollback_wptr(struct safexcel_crypto_priv *priv, + struct safexcel_desc_ring *ring) +{ + if (ring->write == ring->read) + return; + + if (ring->write == ring->base) { + ring->write = ring->base_end; + ring->shwrite = ring->shbase_end; + } else { + ring->write -= ring->offset; + ring->shwrite -= ring->shoffset; + } +} + +struct safexcel_command_desc *safexcel_add_cdesc(struct safexcel_crypto_priv *priv, + int ring_id, + bool first, bool last, + dma_addr_t data, u32 data_len, + u32 full_data_len, + dma_addr_t context, + struct safexcel_token **atoken) +{ + struct safexcel_command_desc *cdesc; + + cdesc = safexcel_ring_next_cwptr(priv, &priv->ring[ring_id].cdr, + first, atoken); + if (IS_ERR(cdesc)) + return cdesc; + + cdesc->particle_size = data_len; + cdesc->rsvd0 = 0; + cdesc->last_seg = last; + cdesc->first_seg = first; + cdesc->additional_cdata_size = 0; + cdesc->rsvd1 = 0; + cdesc->data_lo = lower_32_bits(data); + cdesc->data_hi = upper_32_bits(data); + + if (first) { + /* + * Note that the length here MUST be >0 or else the EIP(1)97 + * may hang. Newer EIP197 firmware actually incorporates this + * fix already, but that doesn't help the EIP97 and we may + * also be running older firmware. + */ + cdesc->control_data.packet_length = full_data_len ?: 1; + cdesc->control_data.options = EIP197_OPTION_MAGIC_VALUE | + EIP197_OPTION_64BIT_CTX | + EIP197_OPTION_CTX_CTRL_IN_CMD | + EIP197_OPTION_RC_AUTO; + cdesc->control_data.type = EIP197_TYPE_BCLA; + cdesc->control_data.context_lo = lower_32_bits(context) | + EIP197_CONTEXT_SMALL; + cdesc->control_data.context_hi = upper_32_bits(context); + } + + return cdesc; +} + +struct safexcel_result_desc *safexcel_add_rdesc(struct safexcel_crypto_priv *priv, + int ring_id, + bool first, bool last, + dma_addr_t data, u32 len) +{ + struct safexcel_result_desc *rdesc; + struct result_data_desc *rtoken; + + rdesc = safexcel_ring_next_rwptr(priv, &priv->ring[ring_id].rdr, + &rtoken); + if (IS_ERR(rdesc)) + return rdesc; + + rdesc->particle_size = len; + rdesc->rsvd0 = 0; + rdesc->descriptor_overflow = 1; /* assume error */ + rdesc->buffer_overflow = 1; /* assume error */ + rdesc->last_seg = last; + rdesc->first_seg = first; + rdesc->result_size = EIP197_RD64_RESULT_SIZE; + rdesc->rsvd1 = 0; + rdesc->data_lo = lower_32_bits(data); + rdesc->data_hi = upper_32_bits(data); + + /* Clear length in result token */ + rtoken->packet_length = 0; + /* Assume errors - HW will clear if not the case */ + rtoken->error_code = 0x7fff; + + return rdesc; +} diff --git a/drivers/crypto/intel/Kconfig b/drivers/crypto/intel/Kconfig new file mode 100644 index 000000000..f38cd62a3 --- /dev/null +++ b/drivers/crypto/intel/Kconfig @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 + +source "drivers/crypto/intel/keembay/Kconfig" +source "drivers/crypto/intel/ixp4xx/Kconfig" +source "drivers/crypto/intel/qat/Kconfig" +source "drivers/crypto/intel/iaa/Kconfig" diff --git a/drivers/crypto/intel/Makefile b/drivers/crypto/intel/Makefile new file mode 100644 index 000000000..2f56f6d34 --- /dev/null +++ b/drivers/crypto/intel/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-y += keembay/ +obj-y += ixp4xx/ +obj-$(CONFIG_CRYPTO_DEV_QAT) += qat/ +obj-$(CONFIG_CRYPTO_DEV_IAA_CRYPTO) += iaa/ diff --git a/drivers/crypto/intel/iaa/Kconfig b/drivers/crypto/intel/iaa/Kconfig new file mode 100644 index 000000000..d53f4b1d4 --- /dev/null +++ b/drivers/crypto/intel/iaa/Kconfig @@ -0,0 +1,19 @@ +config CRYPTO_DEV_IAA_CRYPTO + tristate "Support for Intel(R) IAA Compression Accelerator" + depends on CRYPTO_DEFLATE + depends on INTEL_IDXD + default n + help + This driver supports acceleration for compression and + decompression with the Intel Analytics Accelerator (IAA) + hardware using the cryptographic API. If you choose 'M' + here, the module will be called iaa_crypto. + +config CRYPTO_DEV_IAA_CRYPTO_STATS + bool "Enable Intel(R) IAA Compression Accelerator Statistics" + depends on CRYPTO_DEV_IAA_CRYPTO + default n + help + Enable statistics for the IAA compression accelerator. + These include per-device and per-workqueue statistics in + addition to global driver statistics. diff --git a/drivers/crypto/intel/iaa/Makefile b/drivers/crypto/intel/iaa/Makefile new file mode 100644 index 000000000..55bda7770 --- /dev/null +++ b/drivers/crypto/intel/iaa/Makefile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for IAA crypto device drivers +# + +ccflags-y += -I $(srctree)/drivers/dma/idxd -DDEFAULT_SYMBOL_NAMESPACE='"IDXD"' + +obj-$(CONFIG_CRYPTO_DEV_IAA_CRYPTO) := iaa_crypto.o + +iaa_crypto-y := iaa_crypto_main.o iaa_crypto_comp_fixed.o + +iaa_crypto-$(CONFIG_CRYPTO_DEV_IAA_CRYPTO_STATS) += iaa_crypto_stats.o diff --git a/drivers/crypto/intel/iaa/iaa_crypto.h b/drivers/crypto/intel/iaa/iaa_crypto.h new file mode 100644 index 000000000..56985e395 --- /dev/null +++ b/drivers/crypto/intel/iaa/iaa_crypto.h @@ -0,0 +1,148 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2021 Intel Corporation. All rights rsvd. */ + +#ifndef __IAA_CRYPTO_H__ +#define __IAA_CRYPTO_H__ + +#include +#include +#include + +#define IDXD_SUBDRIVER_NAME "crypto" + +#define IAA_DECOMP_ENABLE BIT(0) +#define IAA_DECOMP_FLUSH_OUTPUT BIT(1) +#define IAA_DECOMP_CHECK_FOR_EOB BIT(2) +#define IAA_DECOMP_STOP_ON_EOB BIT(3) +#define IAA_DECOMP_SUPPRESS_OUTPUT BIT(9) + +#define IAA_COMP_FLUSH_OUTPUT BIT(1) +#define IAA_COMP_APPEND_EOB BIT(2) + +#define IAA_COMPLETION_TIMEOUT 1000000 + +#define IAA_ANALYTICS_ERROR 0x0a +#define IAA_ERROR_DECOMP_BUF_OVERFLOW 0x0b +#define IAA_ERROR_COMP_BUF_OVERFLOW 0x19 +#define IAA_ERROR_WATCHDOG_EXPIRED 0x24 + +#define IAA_COMP_MODES_MAX 2 + +#define FIXED_HDR 0x2 +#define FIXED_HDR_SIZE 3 + +#define IAA_COMP_FLAGS (IAA_COMP_FLUSH_OUTPUT | \ + IAA_COMP_APPEND_EOB) + +#define IAA_DECOMP_FLAGS (IAA_DECOMP_ENABLE | \ + IAA_DECOMP_FLUSH_OUTPUT | \ + IAA_DECOMP_CHECK_FOR_EOB | \ + IAA_DECOMP_STOP_ON_EOB) + +/* Representation of IAA workqueue */ +struct iaa_wq { + struct list_head list; + + struct idxd_wq *wq; + int ref; + bool remove; + + struct iaa_device *iaa_device; + + atomic64_t comp_calls; + atomic64_t comp_bytes; + atomic64_t decomp_calls; + atomic64_t decomp_bytes; +}; + +struct iaa_device_compression_mode { + const char *name; + + struct aecs_comp_table_record *aecs_comp_table; + + dma_addr_t aecs_comp_table_dma_addr; +}; + +/* Representation of IAA device with wqs, populated by probe */ +struct iaa_device { + struct list_head list; + struct idxd_device *idxd; + + struct iaa_device_compression_mode *compression_modes[IAA_COMP_MODES_MAX]; + + int n_wq; + struct list_head wqs; + + atomic64_t comp_calls; + atomic64_t comp_bytes; + atomic64_t decomp_calls; + atomic64_t decomp_bytes; +}; + +struct wq_table_entry { + struct idxd_wq **wqs; + int max_wqs; + int n_wqs; + int cur_wq; +}; + +#define IAA_AECS_ALIGN 32 + +/* + * Analytics Engine Configuration and State (AECS) contains parameters and + * internal state of the analytics engine. + */ +struct aecs_comp_table_record { + u32 crc; + u32 xor_checksum; + u32 reserved0[5]; + u32 num_output_accum_bits; + u8 output_accum[256]; + u32 ll_sym[286]; + u32 reserved1; + u32 reserved2; + u32 d_sym[30]; + u32 reserved_padding[2]; +} __packed; + +int iaa_aecs_init_fixed(void); +void iaa_aecs_cleanup_fixed(void); + +typedef int (*iaa_dev_comp_init_fn_t) (struct iaa_device_compression_mode *mode); +typedef int (*iaa_dev_comp_free_fn_t) (struct iaa_device_compression_mode *mode); + +struct iaa_compression_mode { + const char *name; + u32 *ll_table; + int ll_table_size; + u32 *d_table; + int d_table_size; + iaa_dev_comp_init_fn_t init; + iaa_dev_comp_free_fn_t free; +}; + +int add_iaa_compression_mode(const char *name, + const u32 *ll_table, + int ll_table_size, + const u32 *d_table, + int d_table_size, + iaa_dev_comp_init_fn_t init, + iaa_dev_comp_free_fn_t free); + +void remove_iaa_compression_mode(const char *name); + +enum iaa_mode { + IAA_MODE_FIXED, +}; + +struct iaa_compression_ctx { + enum iaa_mode mode; + bool verify_compress; + bool async_mode; + bool use_irq; +}; + +extern struct list_head iaa_devices; +extern struct mutex iaa_devices_lock; + +#endif diff --git a/drivers/crypto/intel/iaa/iaa_crypto_comp_fixed.c b/drivers/crypto/intel/iaa/iaa_crypto_comp_fixed.c new file mode 100644 index 000000000..19d9a333a --- /dev/null +++ b/drivers/crypto/intel/iaa/iaa_crypto_comp_fixed.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2021 Intel Corporation. All rights rsvd. */ + +#include "idxd.h" +#include "iaa_crypto.h" + +/* + * Fixed Huffman tables the IAA hardware requires to implement RFC-1951. + */ +static const u32 fixed_ll_sym[286] = { + 0x40030, 0x40031, 0x40032, 0x40033, 0x40034, 0x40035, 0x40036, 0x40037, + 0x40038, 0x40039, 0x4003A, 0x4003B, 0x4003C, 0x4003D, 0x4003E, 0x4003F, + 0x40040, 0x40041, 0x40042, 0x40043, 0x40044, 0x40045, 0x40046, 0x40047, + 0x40048, 0x40049, 0x4004A, 0x4004B, 0x4004C, 0x4004D, 0x4004E, 0x4004F, + 0x40050, 0x40051, 0x40052, 0x40053, 0x40054, 0x40055, 0x40056, 0x40057, + 0x40058, 0x40059, 0x4005A, 0x4005B, 0x4005C, 0x4005D, 0x4005E, 0x4005F, + 0x40060, 0x40061, 0x40062, 0x40063, 0x40064, 0x40065, 0x40066, 0x40067, + 0x40068, 0x40069, 0x4006A, 0x4006B, 0x4006C, 0x4006D, 0x4006E, 0x4006F, + 0x40070, 0x40071, 0x40072, 0x40073, 0x40074, 0x40075, 0x40076, 0x40077, + 0x40078, 0x40079, 0x4007A, 0x4007B, 0x4007C, 0x4007D, 0x4007E, 0x4007F, + 0x40080, 0x40081, 0x40082, 0x40083, 0x40084, 0x40085, 0x40086, 0x40087, + 0x40088, 0x40089, 0x4008A, 0x4008B, 0x4008C, 0x4008D, 0x4008E, 0x4008F, + 0x40090, 0x40091, 0x40092, 0x40093, 0x40094, 0x40095, 0x40096, 0x40097, + 0x40098, 0x40099, 0x4009A, 0x4009B, 0x4009C, 0x4009D, 0x4009E, 0x4009F, + 0x400A0, 0x400A1, 0x400A2, 0x400A3, 0x400A4, 0x400A5, 0x400A6, 0x400A7, + 0x400A8, 0x400A9, 0x400AA, 0x400AB, 0x400AC, 0x400AD, 0x400AE, 0x400AF, + 0x400B0, 0x400B1, 0x400B2, 0x400B3, 0x400B4, 0x400B5, 0x400B6, 0x400B7, + 0x400B8, 0x400B9, 0x400BA, 0x400BB, 0x400BC, 0x400BD, 0x400BE, 0x400BF, + 0x48190, 0x48191, 0x48192, 0x48193, 0x48194, 0x48195, 0x48196, 0x48197, + 0x48198, 0x48199, 0x4819A, 0x4819B, 0x4819C, 0x4819D, 0x4819E, 0x4819F, + 0x481A0, 0x481A1, 0x481A2, 0x481A3, 0x481A4, 0x481A5, 0x481A6, 0x481A7, + 0x481A8, 0x481A9, 0x481AA, 0x481AB, 0x481AC, 0x481AD, 0x481AE, 0x481AF, + 0x481B0, 0x481B1, 0x481B2, 0x481B3, 0x481B4, 0x481B5, 0x481B6, 0x481B7, + 0x481B8, 0x481B9, 0x481BA, 0x481BB, 0x481BC, 0x481BD, 0x481BE, 0x481BF, + 0x481C0, 0x481C1, 0x481C2, 0x481C3, 0x481C4, 0x481C5, 0x481C6, 0x481C7, + 0x481C8, 0x481C9, 0x481CA, 0x481CB, 0x481CC, 0x481CD, 0x481CE, 0x481CF, + 0x481D0, 0x481D1, 0x481D2, 0x481D3, 0x481D4, 0x481D5, 0x481D6, 0x481D7, + 0x481D8, 0x481D9, 0x481DA, 0x481DB, 0x481DC, 0x481DD, 0x481DE, 0x481DF, + 0x481E0, 0x481E1, 0x481E2, 0x481E3, 0x481E4, 0x481E5, 0x481E6, 0x481E7, + 0x481E8, 0x481E9, 0x481EA, 0x481EB, 0x481EC, 0x481ED, 0x481EE, 0x481EF, + 0x481F0, 0x481F1, 0x481F2, 0x481F3, 0x481F4, 0x481F5, 0x481F6, 0x481F7, + 0x481F8, 0x481F9, 0x481FA, 0x481FB, 0x481FC, 0x481FD, 0x481FE, 0x481FF, + 0x38000, 0x38001, 0x38002, 0x38003, 0x38004, 0x38005, 0x38006, 0x38007, + 0x38008, 0x38009, 0x3800A, 0x3800B, 0x3800C, 0x3800D, 0x3800E, 0x3800F, + 0x38010, 0x38011, 0x38012, 0x38013, 0x38014, 0x38015, 0x38016, 0x38017, + 0x400C0, 0x400C1, 0x400C2, 0x400C3, 0x400C4, 0x400C5 +}; + +static const u32 fixed_d_sym[30] = { + 0x28000, 0x28001, 0x28002, 0x28003, 0x28004, 0x28005, 0x28006, 0x28007, + 0x28008, 0x28009, 0x2800A, 0x2800B, 0x2800C, 0x2800D, 0x2800E, 0x2800F, + 0x28010, 0x28011, 0x28012, 0x28013, 0x28014, 0x28015, 0x28016, 0x28017, + 0x28018, 0x28019, 0x2801A, 0x2801B, 0x2801C, 0x2801D +}; + +static int init_fixed_mode(struct iaa_device_compression_mode *mode) +{ + struct aecs_comp_table_record *comp_table = mode->aecs_comp_table; + u32 bfinal = 1; + u32 offset; + + /* Configure aecs table using fixed Huffman table */ + comp_table->crc = 0; + comp_table->xor_checksum = 0; + offset = comp_table->num_output_accum_bits / 8; + comp_table->output_accum[offset] = FIXED_HDR | bfinal; + comp_table->num_output_accum_bits = FIXED_HDR_SIZE; + + return 0; +} + +int iaa_aecs_init_fixed(void) +{ + int ret; + + ret = add_iaa_compression_mode("fixed", + fixed_ll_sym, + sizeof(fixed_ll_sym), + fixed_d_sym, + sizeof(fixed_d_sym), + init_fixed_mode, NULL); + if (!ret) + pr_debug("IAA fixed compression mode initialized\n"); + + return ret; +} + +void iaa_aecs_cleanup_fixed(void) +{ + remove_iaa_compression_mode("fixed"); +} diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c new file mode 100644 index 000000000..c9ab4b83a --- /dev/null +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -0,0 +1,2035 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2021 Intel Corporation. All rights rsvd. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "idxd.h" +#include "iaa_crypto.h" +#include "iaa_crypto_stats.h" + +#ifdef pr_fmt +#undef pr_fmt +#endif + +#define pr_fmt(fmt) "idxd: " IDXD_SUBDRIVER_NAME ": " fmt + +#define IAA_ALG_PRIORITY 300 + +/* number of iaa instances probed */ +static unsigned int nr_iaa; +static unsigned int nr_cpus; +static unsigned int nr_nodes; +static unsigned int nr_cpus_per_node; + +/* Number of physical cpus sharing each iaa instance */ +static unsigned int cpus_per_iaa; + +/* Per-cpu lookup table for balanced wqs */ +static struct wq_table_entry __percpu *wq_table; + +static struct idxd_wq *wq_table_next_wq(int cpu) +{ + struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu); + + if (++entry->cur_wq >= entry->n_wqs) + entry->cur_wq = 0; + + if (!entry->wqs[entry->cur_wq]) + return NULL; + + pr_debug("%s: returning wq at idx %d (iaa wq %d.%d) from cpu %d\n", __func__, + entry->cur_wq, entry->wqs[entry->cur_wq]->idxd->id, + entry->wqs[entry->cur_wq]->id, cpu); + + return entry->wqs[entry->cur_wq]; +} + +static void wq_table_add(int cpu, struct idxd_wq *wq) +{ + struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu); + + if (WARN_ON(entry->n_wqs == entry->max_wqs)) + return; + + entry->wqs[entry->n_wqs++] = wq; + + pr_debug("%s: added iaa wq %d.%d to idx %d of cpu %d\n", __func__, + entry->wqs[entry->n_wqs - 1]->idxd->id, + entry->wqs[entry->n_wqs - 1]->id, entry->n_wqs - 1, cpu); +} + +static void wq_table_free_entry(int cpu) +{ + struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu); + + kfree(entry->wqs); + memset(entry, 0, sizeof(*entry)); +} + +static void wq_table_clear_entry(int cpu) +{ + struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu); + + entry->n_wqs = 0; + entry->cur_wq = 0; + memset(entry->wqs, 0, entry->max_wqs * sizeof(struct idxd_wq *)); +} + +LIST_HEAD(iaa_devices); +DEFINE_MUTEX(iaa_devices_lock); + +/* If enabled, IAA hw crypto algos are registered, unavailable otherwise */ +static bool iaa_crypto_enabled; +static bool iaa_crypto_registered; + +/* Verify results of IAA compress or not */ +static bool iaa_verify_compress = true; + +static ssize_t verify_compress_show(struct device_driver *driver, char *buf) +{ + return sysfs_emit(buf, "%d\n", iaa_verify_compress); +} + +static ssize_t verify_compress_store(struct device_driver *driver, + const char *buf, size_t count) +{ + int ret = -EBUSY; + + mutex_lock(&iaa_devices_lock); + + if (iaa_crypto_enabled) + goto out; + + ret = kstrtobool(buf, &iaa_verify_compress); + if (ret) + goto out; + + ret = count; +out: + mutex_unlock(&iaa_devices_lock); + + return ret; +} +static DRIVER_ATTR_RW(verify_compress); + +/* + * The iaa crypto driver supports three 'sync' methods determining how + * compressions and decompressions are performed: + * + * - sync: the compression or decompression completes before + * returning. This is the mode used by the async crypto + * interface when the sync mode is set to 'sync' and by + * the sync crypto interface regardless of setting. + * + * - async: the compression or decompression is submitted and returns + * immediately. Completion interrupts are not used so + * the caller is responsible for polling the descriptor + * for completion. This mode is applicable to only the + * async crypto interface and is ignored for anything + * else. + * + * - async_irq: the compression or decompression is submitted and + * returns immediately. Completion interrupts are + * enabled so the caller can wait for the completion and + * yield to other threads. When the compression or + * decompression completes, the completion is signaled + * and the caller awakened. This mode is applicable to + * only the async crypto interface and is ignored for + * anything else. + * + * These modes can be set using the iaa_crypto sync_mode driver + * attribute. + */ + +/* Use async mode */ +static bool async_mode; +/* Use interrupts */ +static bool use_irq; + +struct iaa_req_ctx { + u32 compression_crc; + struct page *bounce_src; + dma_addr_t bounce_src_dma; + unsigned int bounce_src_len; +}; + +static mempool_t *iaa_bounce_pool; +#define IAA_BOUNCE_POOL_SIZE 128 + +/** + * set_iaa_sync_mode - Set IAA sync mode + * @name: The name of the sync mode + * + * Make the IAA sync mode named @name the current sync mode used by + * compression/decompression. + */ + +static int set_iaa_sync_mode(const char *name) +{ + int ret = 0; + + if (sysfs_streq(name, "sync")) { + async_mode = false; + use_irq = false; + } else if (sysfs_streq(name, "async")) { + async_mode = false; + use_irq = false; + } else if (sysfs_streq(name, "async_irq")) { + async_mode = true; + use_irq = true; + } else { + ret = -EINVAL; + } + + return ret; +} + +static ssize_t sync_mode_show(struct device_driver *driver, char *buf) +{ + int ret = 0; + + if (!async_mode && !use_irq) + ret = sysfs_emit(buf, "%s\n", "sync"); + else if (async_mode && !use_irq) + ret = sysfs_emit(buf, "%s\n", "async"); + else if (async_mode && use_irq) + ret = sysfs_emit(buf, "%s\n", "async_irq"); + + return ret; +} + +static ssize_t sync_mode_store(struct device_driver *driver, + const char *buf, size_t count) +{ + int ret = -EBUSY; + + mutex_lock(&iaa_devices_lock); + + if (iaa_crypto_enabled) + goto out; + + ret = set_iaa_sync_mode(buf); + if (ret == 0) + ret = count; +out: + mutex_unlock(&iaa_devices_lock); + + return ret; +} +static DRIVER_ATTR_RW(sync_mode); + +static struct iaa_compression_mode *iaa_compression_modes[IAA_COMP_MODES_MAX]; + +static int find_empty_iaa_compression_mode(void) +{ + int i; + + for (i = 0; i < IAA_COMP_MODES_MAX; i++) + if (!iaa_compression_modes[i]) + return i; + + return -EINVAL; +} + +static struct iaa_compression_mode *find_iaa_compression_mode(const char *name, int *idx) +{ + struct iaa_compression_mode *mode; + int i; + + for (i = 0; i < IAA_COMP_MODES_MAX; i++) { + mode = iaa_compression_modes[i]; + if (!mode) + continue; + + if (!strcmp(mode->name, name)) { + *idx = i; + return iaa_compression_modes[i]; + } + } + + return NULL; +} + +static void free_iaa_compression_mode(struct iaa_compression_mode *mode) +{ + kfree(mode->name); + kfree(mode->ll_table); + kfree(mode->d_table); + + kfree(mode); +} + +/* + * IAA Compression modes are defined by an ll_table and a d_table. + * These tables are typically generated and captured using statistics + * collected from running actual compress/decompress workloads. + * + * A module or other kernel code can add and remove compression modes + * with a given name using the exported @add_iaa_compression_mode() + * and @remove_iaa_compression_mode functions. + * + * When a new compression mode is added, the tables are saved in a + * global compression mode list. When IAA devices are added, a + * per-IAA device dma mapping is created for each IAA device, for each + * compression mode. These are the tables used to do the actual + * compression/deccompression and are unmapped if/when the devices are + * removed. Currently, compression modes must be added before any + * device is added, and removed after all devices have been removed. + */ + +/** + * remove_iaa_compression_mode - Remove an IAA compression mode + * @name: The name the compression mode will be known as + * + * Remove the IAA compression mode named @name. + */ +void remove_iaa_compression_mode(const char *name) +{ + struct iaa_compression_mode *mode; + int idx; + + mutex_lock(&iaa_devices_lock); + + if (!list_empty(&iaa_devices)) + goto out; + + mode = find_iaa_compression_mode(name, &idx); + if (mode) { + free_iaa_compression_mode(mode); + iaa_compression_modes[idx] = NULL; + } +out: + mutex_unlock(&iaa_devices_lock); +} +EXPORT_SYMBOL_GPL(remove_iaa_compression_mode); + +/** + * add_iaa_compression_mode - Add an IAA compression mode + * @name: The name the compression mode will be known as + * @ll_table: The ll table + * @ll_table_size: The ll table size in bytes + * @d_table: The d table + * @d_table_size: The d table size in bytes + * @init: Optional callback function to init the compression mode data + * @free: Optional callback function to free the compression mode data + * + * Add a new IAA compression mode named @name. + * + * Returns 0 if successful, errcode otherwise. + */ +int add_iaa_compression_mode(const char *name, + const u32 *ll_table, + int ll_table_size, + const u32 *d_table, + int d_table_size, + iaa_dev_comp_init_fn_t init, + iaa_dev_comp_free_fn_t free) +{ + struct iaa_compression_mode *mode; + int idx, ret = -ENOMEM; + + mutex_lock(&iaa_devices_lock); + + if (!list_empty(&iaa_devices)) { + ret = -EBUSY; + goto out; + } + + mode = kzalloc_obj(*mode); + if (!mode) + goto out; + + mode->name = kstrdup(name, GFP_KERNEL); + if (!mode->name) + goto free; + + if (ll_table) { + mode->ll_table = kmemdup(ll_table, ll_table_size, GFP_KERNEL); + if (!mode->ll_table) + goto free; + mode->ll_table_size = ll_table_size; + } + + if (d_table) { + mode->d_table = kmemdup(d_table, d_table_size, GFP_KERNEL); + if (!mode->d_table) + goto free; + mode->d_table_size = d_table_size; + } + + mode->init = init; + mode->free = free; + + idx = find_empty_iaa_compression_mode(); + if (idx < 0) + goto free; + + pr_debug("IAA compression mode %s added at idx %d\n", + mode->name, idx); + + iaa_compression_modes[idx] = mode; + + ret = 0; +out: + mutex_unlock(&iaa_devices_lock); + + return ret; +free: + free_iaa_compression_mode(mode); + goto out; +} +EXPORT_SYMBOL_GPL(add_iaa_compression_mode); + +static struct iaa_device_compression_mode * +get_iaa_device_compression_mode(struct iaa_device *iaa_device, int idx) +{ + return iaa_device->compression_modes[idx]; +} + +static void free_device_compression_mode(struct iaa_device *iaa_device, + struct iaa_device_compression_mode *device_mode) +{ + size_t size = sizeof(struct aecs_comp_table_record) + IAA_AECS_ALIGN; + struct device *dev = &iaa_device->idxd->pdev->dev; + + kfree(device_mode->name); + + if (device_mode->aecs_comp_table) + dma_free_coherent(dev, size, device_mode->aecs_comp_table, + device_mode->aecs_comp_table_dma_addr); + kfree(device_mode); +} + +#define IDXD_OP_FLAG_AECS_RW_TGLS 0x400000 +#define IAX_AECS_DEFAULT_FLAG (IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC) +#define IAX_AECS_COMPRESS_FLAG (IAX_AECS_DEFAULT_FLAG | IDXD_OP_FLAG_RD_SRC2_AECS) +#define IAX_AECS_DECOMPRESS_FLAG (IAX_AECS_DEFAULT_FLAG | IDXD_OP_FLAG_RD_SRC2_AECS) +#define IAX_AECS_GEN_FLAG (IAX_AECS_DEFAULT_FLAG | \ + IDXD_OP_FLAG_WR_SRC2_AECS_COMP | \ + IDXD_OP_FLAG_AECS_RW_TGLS) + +static int check_completion(struct device *dev, + struct iax_completion_record *comp, + bool compress, + bool only_once); + +static int init_device_compression_mode(struct iaa_device *iaa_device, + struct iaa_compression_mode *mode, + int idx, struct idxd_wq *wq) +{ + size_t size = sizeof(struct aecs_comp_table_record) + IAA_AECS_ALIGN; + struct device *dev = &iaa_device->idxd->pdev->dev; + struct iaa_device_compression_mode *device_mode; + int ret = -ENOMEM; + + device_mode = kzalloc_obj(*device_mode); + if (!device_mode) + return -ENOMEM; + + device_mode->name = kstrdup(mode->name, GFP_KERNEL); + if (!device_mode->name) + goto free; + + device_mode->aecs_comp_table = dma_alloc_coherent(dev, size, + &device_mode->aecs_comp_table_dma_addr, GFP_KERNEL); + if (!device_mode->aecs_comp_table) + goto free; + + /* Add Huffman table to aecs */ + memset(device_mode->aecs_comp_table, 0, sizeof(*device_mode->aecs_comp_table)); + memcpy(device_mode->aecs_comp_table->ll_sym, mode->ll_table, mode->ll_table_size); + memcpy(device_mode->aecs_comp_table->d_sym, mode->d_table, mode->d_table_size); + + if (mode->init) { + ret = mode->init(device_mode); + if (ret) + goto free; + } + + /* mode index should match iaa_compression_modes idx */ + iaa_device->compression_modes[idx] = device_mode; + + pr_debug("IAA %s compression mode initialized for iaa device %d\n", + mode->name, iaa_device->idxd->id); + + ret = 0; +out: + return ret; +free: + pr_debug("IAA %s compression mode initialization failed for iaa device %d\n", + mode->name, iaa_device->idxd->id); + + free_device_compression_mode(iaa_device, device_mode); + goto out; +} + +static int init_device_compression_modes(struct iaa_device *iaa_device, + struct idxd_wq *wq) +{ + struct iaa_compression_mode *mode; + int i, ret = 0; + + for (i = 0; i < IAA_COMP_MODES_MAX; i++) { + mode = iaa_compression_modes[i]; + if (!mode) + continue; + + ret = init_device_compression_mode(iaa_device, mode, i, wq); + if (ret) + break; + } + + return ret; +} + +static void remove_device_compression_modes(struct iaa_device *iaa_device) +{ + struct iaa_device_compression_mode *device_mode; + int i; + + for (i = 0; i < IAA_COMP_MODES_MAX; i++) { + device_mode = iaa_device->compression_modes[i]; + if (!device_mode) + continue; + + if (iaa_compression_modes[i]->free) + iaa_compression_modes[i]->free(device_mode); + free_device_compression_mode(iaa_device, device_mode); + iaa_device->compression_modes[i] = NULL; + } +} + +static struct iaa_device *iaa_device_alloc(void) +{ + struct iaa_device *iaa_device; + + iaa_device = kzalloc_obj(*iaa_device); + if (!iaa_device) + return NULL; + + INIT_LIST_HEAD(&iaa_device->wqs); + + return iaa_device; +} + +static bool iaa_has_wq(struct iaa_device *iaa_device, struct idxd_wq *wq) +{ + struct iaa_wq *iaa_wq; + + list_for_each_entry(iaa_wq, &iaa_device->wqs, list) { + if (iaa_wq->wq == wq) + return true; + } + + return false; +} + +static struct iaa_device *add_iaa_device(struct idxd_device *idxd) +{ + struct iaa_device *iaa_device; + + iaa_device = iaa_device_alloc(); + if (!iaa_device) + return NULL; + + iaa_device->idxd = idxd; + + list_add_tail(&iaa_device->list, &iaa_devices); + + nr_iaa++; + + return iaa_device; +} + +static int init_iaa_device(struct iaa_device *iaa_device, struct iaa_wq *iaa_wq) +{ + return init_device_compression_modes(iaa_device, iaa_wq->wq); +} + +static void del_iaa_device(struct iaa_device *iaa_device) +{ + list_del(&iaa_device->list); + + nr_iaa--; +} + +static int add_iaa_wq(struct iaa_device *iaa_device, struct idxd_wq *wq, + struct iaa_wq **new_wq) +{ + struct idxd_device *idxd = iaa_device->idxd; + struct pci_dev *pdev = idxd->pdev; + struct device *dev = &pdev->dev; + struct iaa_wq *iaa_wq; + + iaa_wq = kzalloc_obj(*iaa_wq); + if (!iaa_wq) + return -ENOMEM; + + iaa_wq->wq = wq; + iaa_wq->iaa_device = iaa_device; + idxd_wq_set_private(wq, iaa_wq); + + list_add_tail(&iaa_wq->list, &iaa_device->wqs); + + iaa_device->n_wq++; + + if (new_wq) + *new_wq = iaa_wq; + + dev_dbg(dev, "added wq %d to iaa device %d, n_wq %d\n", + wq->id, iaa_device->idxd->id, iaa_device->n_wq); + + return 0; +} + +static void del_iaa_wq(struct iaa_device *iaa_device, struct idxd_wq *wq) +{ + struct idxd_device *idxd = iaa_device->idxd; + struct pci_dev *pdev = idxd->pdev; + struct device *dev = &pdev->dev; + struct iaa_wq *iaa_wq; + + list_for_each_entry(iaa_wq, &iaa_device->wqs, list) { + if (iaa_wq->wq == wq) { + list_del(&iaa_wq->list); + iaa_device->n_wq--; + + dev_dbg(dev, "removed wq %d from iaa_device %d, n_wq %d, nr_iaa %d\n", + wq->id, iaa_device->idxd->id, + iaa_device->n_wq, nr_iaa); + + if (iaa_device->n_wq == 0) + del_iaa_device(iaa_device); + break; + } + } +} + +static void clear_wq_table(void) +{ + int cpu; + + for (cpu = 0; cpu < nr_cpus; cpu++) + wq_table_clear_entry(cpu); + + pr_debug("cleared wq table\n"); +} + +static void free_iaa_device(struct iaa_device *iaa_device) +{ + if (!iaa_device) + return; + + remove_device_compression_modes(iaa_device); + kfree(iaa_device); +} + +static void __free_iaa_wq(struct iaa_wq *iaa_wq) +{ + struct iaa_device *iaa_device; + + if (!iaa_wq) + return; + + iaa_device = iaa_wq->iaa_device; + if (iaa_device->n_wq == 0) + free_iaa_device(iaa_wq->iaa_device); +} + +static void free_iaa_wq(struct iaa_wq *iaa_wq) +{ + struct idxd_wq *wq; + + __free_iaa_wq(iaa_wq); + + wq = iaa_wq->wq; + + kfree(iaa_wq); + idxd_wq_set_private(wq, NULL); +} + +static int iaa_wq_get(struct idxd_wq *wq) +{ + struct idxd_device *idxd = wq->idxd; + struct iaa_wq *iaa_wq; + int ret = 0; + + spin_lock(&idxd->dev_lock); + iaa_wq = idxd_wq_get_private(wq); + if (iaa_wq && !iaa_wq->remove) { + iaa_wq->ref++; + idxd_wq_get(wq); + } else { + ret = -ENODEV; + } + spin_unlock(&idxd->dev_lock); + + return ret; +} + +static int iaa_wq_put(struct idxd_wq *wq) +{ + struct idxd_device *idxd = wq->idxd; + struct iaa_wq *iaa_wq; + bool free = false; + int ret = 0; + + spin_lock(&idxd->dev_lock); + iaa_wq = idxd_wq_get_private(wq); + if (iaa_wq) { + iaa_wq->ref--; + if (iaa_wq->ref == 0 && iaa_wq->remove) { + idxd_wq_set_private(wq, NULL); + free = true; + } + idxd_wq_put(wq); + } else { + ret = -ENODEV; + } + spin_unlock(&idxd->dev_lock); + if (free) { + __free_iaa_wq(iaa_wq); + kfree(iaa_wq); + } + + return ret; +} + +static void free_wq_table(void) +{ + int cpu; + + for (cpu = 0; cpu < nr_cpus; cpu++) + wq_table_free_entry(cpu); + + free_percpu(wq_table); + + pr_debug("freed wq table\n"); +} + +static int alloc_wq_table(int max_wqs) +{ + struct wq_table_entry *entry; + int cpu; + + wq_table = alloc_percpu(struct wq_table_entry); + if (!wq_table) + return -ENOMEM; + + for (cpu = 0; cpu < nr_cpus; cpu++) { + entry = per_cpu_ptr(wq_table, cpu); + entry->wqs = kzalloc_objs(*entry->wqs, max_wqs); + if (!entry->wqs) { + free_wq_table(); + return -ENOMEM; + } + + entry->max_wqs = max_wqs; + } + + pr_debug("initialized wq table\n"); + + return 0; +} + +static int save_iaa_wq(struct idxd_wq *wq) +{ + struct iaa_device *iaa_device, *found = NULL; + struct idxd_device *idxd; + struct pci_dev *pdev; + struct device *dev; + int ret = 0; + + list_for_each_entry(iaa_device, &iaa_devices, list) { + if (iaa_device->idxd == wq->idxd) { + idxd = iaa_device->idxd; + pdev = idxd->pdev; + dev = &pdev->dev; + /* + * Check to see that we don't already have this wq. + * Shouldn't happen but we don't control probing. + */ + if (iaa_has_wq(iaa_device, wq)) { + dev_dbg(dev, "same wq probed multiple times for iaa_device %p\n", + iaa_device); + goto out; + } + + found = iaa_device; + + ret = add_iaa_wq(iaa_device, wq, NULL); + if (ret) + goto out; + + break; + } + } + + if (!found) { + struct iaa_device *new_device; + struct iaa_wq *new_wq; + + new_device = add_iaa_device(wq->idxd); + if (!new_device) { + ret = -ENOMEM; + goto out; + } + + ret = add_iaa_wq(new_device, wq, &new_wq); + if (ret) { + del_iaa_device(new_device); + free_iaa_device(new_device); + goto out; + } + + ret = init_iaa_device(new_device, new_wq); + if (ret) { + del_iaa_wq(new_device, new_wq->wq); + del_iaa_device(new_device); + free_iaa_wq(new_wq); + goto out; + } + } + + if (WARN_ON(nr_iaa == 0)) + return -EINVAL; + + cpus_per_iaa = (nr_nodes * nr_cpus_per_node) / nr_iaa; + if (!cpus_per_iaa) + cpus_per_iaa = 1; +out: + return ret; +} + +static void remove_iaa_wq(struct idxd_wq *wq) +{ + struct iaa_device *iaa_device; + + list_for_each_entry(iaa_device, &iaa_devices, list) { + if (iaa_has_wq(iaa_device, wq)) { + del_iaa_wq(iaa_device, wq); + break; + } + } + + if (nr_iaa) { + cpus_per_iaa = (nr_nodes * nr_cpus_per_node) / nr_iaa; + if (!cpus_per_iaa) + cpus_per_iaa = 1; + } else + cpus_per_iaa = 1; +} + +static int wq_table_add_wqs(int iaa, int cpu) +{ + struct iaa_device *iaa_device, *found_device = NULL; + int ret = 0, cur_iaa = 0, n_wqs_added = 0; + struct idxd_device *idxd; + struct iaa_wq *iaa_wq; + struct pci_dev *pdev; + struct device *dev; + + list_for_each_entry(iaa_device, &iaa_devices, list) { + idxd = iaa_device->idxd; + pdev = idxd->pdev; + dev = &pdev->dev; + + if (cur_iaa != iaa) { + cur_iaa++; + continue; + } + + found_device = iaa_device; + dev_dbg(dev, "getting wq from iaa_device %d, cur_iaa %d\n", + found_device->idxd->id, cur_iaa); + break; + } + + if (!found_device) { + found_device = list_first_entry_or_null(&iaa_devices, + struct iaa_device, list); + if (!found_device) { + pr_debug("couldn't find any iaa devices with wqs!\n"); + ret = -EINVAL; + goto out; + } + cur_iaa = 0; + + idxd = found_device->idxd; + pdev = idxd->pdev; + dev = &pdev->dev; + dev_dbg(dev, "getting wq from only iaa_device %d, cur_iaa %d\n", + found_device->idxd->id, cur_iaa); + } + + list_for_each_entry(iaa_wq, &found_device->wqs, list) { + wq_table_add(cpu, iaa_wq->wq); + pr_debug("rebalance: added wq for cpu=%d: iaa wq %d.%d\n", + cpu, iaa_wq->wq->idxd->id, iaa_wq->wq->id); + n_wqs_added++; + } + + if (!n_wqs_added) { + pr_debug("couldn't find any iaa wqs!\n"); + ret = -EINVAL; + goto out; + } +out: + return ret; +} + +/* + * Rebalance the wq table so that given a cpu, it's easy to find the + * closest IAA instance. The idea is to try to choose the most + * appropriate IAA instance for a caller and spread available + * workqueues around to clients. + */ +static void rebalance_wq_table(void) +{ + const struct cpumask *node_cpus; + int node_cpu, node, cpu, iaa = 0; + + if (nr_iaa == 0) + return; + + pr_debug("rebalance: nr_nodes=%d, nr_cpus %d, nr_iaa %d, cpus_per_iaa %d\n", + nr_nodes, nr_cpus, nr_iaa, cpus_per_iaa); + + clear_wq_table(); + + if (nr_iaa == 1) { + for_each_possible_cpu(cpu) { + if (WARN_ON(wq_table_add_wqs(0, cpu))) + goto err; + } + + return; + } + + cpu = 0; + for_each_node_with_cpus(node) { + node_cpus = cpumask_of_node(node); + + for_each_cpu(node_cpu, node_cpus) { + iaa = cpu / cpus_per_iaa; + if (WARN_ON(wq_table_add_wqs(iaa, node_cpu))) + goto err; + cpu++; + } + } + + return; +err: + pr_debug("could not add any wqs for iaa %d to cpu %d!\n", iaa, cpu); +} + +static inline int check_completion(struct device *dev, + struct iax_completion_record *comp, + bool compress, + bool only_once) +{ + char *op_str = compress ? "compress" : "decompress"; + int status_checks = 0; + int ret = 0; + + while (!comp->status) { + if (only_once) + return -EAGAIN; + cpu_relax(); + if (status_checks++ >= IAA_COMPLETION_TIMEOUT) { + /* Something is wrong with the hw, disable it. */ + dev_err(dev, "%s completion timed out - " + "assuming broken hw, iaa_crypto now DISABLED\n", + op_str); + iaa_crypto_enabled = false; + ret = -ETIMEDOUT; + goto out; + } + } + + if (comp->status != IAX_COMP_SUCCESS) { + if (comp->status == IAA_ERROR_WATCHDOG_EXPIRED) { + ret = -ETIMEDOUT; + dev_dbg(dev, "%s timed out, size=0x%x\n", + op_str, comp->output_size); + update_completion_timeout_errs(); + goto out; + } + + if (comp->status == IAA_ANALYTICS_ERROR && + comp->error_code == IAA_ERROR_COMP_BUF_OVERFLOW && compress) { + ret = -E2BIG; + dev_dbg(dev, "compressed > uncompressed size," + " not compressing, size=0x%x\n", + comp->output_size); + update_completion_comp_buf_overflow_errs(); + goto out; + } + + if (comp->status == IAA_ERROR_DECOMP_BUF_OVERFLOW) { + ret = -EOVERFLOW; + goto out; + } + + ret = -EINVAL; + dev_dbg(dev, "iaa %s status=0x%x, error=0x%x, size=0x%x\n", + op_str, comp->status, comp->error_code, comp->output_size); + print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET, 8, 1, comp, 64, 0); + update_completion_einval_errs(); + + goto out; + } +out: + return ret; +} + +static bool iaa_error_should_retry(struct idxd_desc *idxd_desc) +{ + return idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR; +} + +static void iaa_unmap_src(struct device *dev, struct acomp_req *req) +{ + struct iaa_req_ctx *req_ctx = acomp_request_ctx(req); + + if (req_ctx->bounce_src) { + dma_unmap_page(dev, req_ctx->bounce_src_dma, + req_ctx->bounce_src_len, DMA_TO_DEVICE); + mempool_free(req_ctx->bounce_src, iaa_bounce_pool); + req_ctx->bounce_src = NULL; + req_ctx->bounce_src_dma = 0; + req_ctx->bounce_src_len = 0; + return; + } + + dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE); +} + +static int deflate_generic_decompress(struct acomp_req *req) +{ + ACOMP_FBREQ_ON_STACK(fbreq, req); + int ret; + + ret = crypto_acomp_decompress(fbreq); + req->dlen = fbreq->dlen; + + update_total_sw_decomp_calls(); + + return ret; +} + +static int deflate_generic_compress(struct acomp_req *req) +{ + ACOMP_FBREQ_ON_STACK(fbreq, req); + int ret; + + ret = crypto_acomp_compress(fbreq); + req->dlen = fbreq->dlen; + + update_total_sw_comp_calls(); + + return ret; +} + +static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq, + struct acomp_req *req, + dma_addr_t *src_addr, dma_addr_t *dst_addr); + +static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req, + struct idxd_wq *wq, + dma_addr_t src_addr, unsigned int slen, + dma_addr_t dst_addr, unsigned int *dlen); + +static void iaa_desc_complete(struct idxd_desc *idxd_desc, + enum idxd_complete_type comp_type, + bool free_desc, void *__ctx, + u32 *status) +{ + struct iaa_device_compression_mode *active_compression_mode; + struct iaa_compression_ctx *compression_ctx; + struct crypto_ctx *ctx = __ctx; + struct iaa_req_ctx *req_ctx = acomp_request_ctx(ctx->req); + struct iaa_device *iaa_device; + struct idxd_device *idxd; + struct iaa_wq *iaa_wq; + struct pci_dev *pdev; + struct device *dev; + int ret, err = 0; + + compression_ctx = crypto_tfm_ctx(ctx->tfm); + + iaa_wq = idxd_wq_get_private(idxd_desc->wq); + iaa_device = iaa_wq->iaa_device; + idxd = iaa_device->idxd; + pdev = idxd->pdev; + dev = &pdev->dev; + + active_compression_mode = get_iaa_device_compression_mode(iaa_device, + compression_ctx->mode); + dev_dbg(dev, "%s: compression mode %s," + " ctx->src_addr %llx, ctx->dst_addr %llx\n", __func__, + active_compression_mode->name, + ctx->src_addr, ctx->dst_addr); + + ret = check_completion(dev, idxd_desc->iax_completion, + ctx->compress, false); + if (ret) { + dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret); + if (!ctx->compress && iaa_error_should_retry(idxd_desc)) { + pr_warn("%s: falling back to deflate-generic decompress, " + "analytics error code %x\n", __func__, + idxd_desc->iax_completion->error_code); + dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), + DMA_FROM_DEVICE); + iaa_unmap_src(dev, ctx->req); + + ret = deflate_generic_decompress(ctx->req); + if (ret) { + dev_dbg(dev, "%s: deflate-generic failed ret=%d\n", + __func__, ret); + err = -EIO; + } + goto out; + } else { + err = -EIO; + goto err; + } + } else { + ctx->req->dlen = idxd_desc->iax_completion->output_size; + + if (!ctx->compress) { + update_total_decomp_bytes_in(ctx->req->slen); + update_wq_decomp_bytes(iaa_wq->wq, ctx->req->slen); + } + } + + /* Update stats */ + if (ctx->compress) { + update_total_comp_bytes_out(ctx->req->dlen); + update_wq_comp_bytes(iaa_wq->wq, ctx->req->dlen); + } + + if (ctx->compress && compression_ctx->verify_compress) { + dma_addr_t src_addr, dst_addr; + + req_ctx->compression_crc = idxd_desc->iax_completion->crc; + + ret = iaa_remap_for_verify(dev, iaa_wq, ctx->req, &src_addr, &dst_addr); + if (ret) { + dev_dbg(dev, "%s: compress verify remap failed ret=%d\n", __func__, ret); + err = -EIO; + goto out; + } + + ret = iaa_compress_verify(ctx->tfm, ctx->req, iaa_wq->wq, src_addr, + ctx->req->slen, dst_addr, &ctx->req->dlen); + if (ret) { + dev_dbg(dev, "%s: compress verify failed ret=%d\n", __func__, ret); + err = -EIO; + } + + dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_TO_DEVICE); + dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_FROM_DEVICE); + + goto out; + } +err: + dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_FROM_DEVICE); + iaa_unmap_src(dev, ctx->req); +out: + if (ret != 0) + dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret); + + if (ctx->req->base.complete) + acomp_request_complete(ctx->req, err); + + if (free_desc) + idxd_free_desc(idxd_desc->wq, idxd_desc); + iaa_wq_put(idxd_desc->wq); +} + +static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, + struct idxd_wq *wq, + dma_addr_t src_addr, unsigned int slen, + dma_addr_t dst_addr, unsigned int *dlen) +{ + struct iaa_device_compression_mode *active_compression_mode; + struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); + struct iaa_req_ctx *req_ctx = acomp_request_ctx(req); + struct iaa_device *iaa_device; + struct idxd_desc *idxd_desc; + struct iax_hw_desc *desc; + struct idxd_device *idxd; + struct iaa_wq *iaa_wq; + struct pci_dev *pdev; + struct device *dev; + int ret = 0; + + iaa_wq = idxd_wq_get_private(wq); + iaa_device = iaa_wq->iaa_device; + idxd = iaa_device->idxd; + pdev = idxd->pdev; + dev = &pdev->dev; + + active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode); + + idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK); + if (IS_ERR(idxd_desc)) { + dev_dbg(dev, "idxd descriptor allocation failed\n"); + dev_dbg(dev, "iaa compress failed: ret=%ld\n", PTR_ERR(idxd_desc)); + return PTR_ERR(idxd_desc); + } + desc = idxd_desc->iax_hw; + + desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | + IDXD_OP_FLAG_RD_SRC2_AECS | IDXD_OP_FLAG_CC; + desc->opcode = IAX_OPCODE_COMPRESS; + desc->compr_flags = IAA_COMP_FLAGS; + desc->priv = 0; + + desc->src1_addr = (u64)src_addr; + desc->src1_size = slen; + desc->dst_addr = (u64)dst_addr; + desc->max_dst_size = *dlen; + desc->src2_addr = active_compression_mode->aecs_comp_table_dma_addr; + desc->src2_size = sizeof(struct aecs_comp_table_record); + desc->completion_addr = idxd_desc->compl_dma; + + if (ctx->use_irq) { + desc->flags |= IDXD_OP_FLAG_RCI; + + idxd_desc->crypto.req = req; + idxd_desc->crypto.tfm = tfm; + idxd_desc->crypto.src_addr = src_addr; + idxd_desc->crypto.dst_addr = dst_addr; + idxd_desc->crypto.compress = true; + + dev_dbg(dev, "%s use_async_irq: compression mode %s," + " src_addr %llx, dst_addr %llx\n", __func__, + active_compression_mode->name, + src_addr, dst_addr); + } + + dev_dbg(dev, "%s: compression mode %s," + " desc->src1_addr %llx, desc->src1_size %d," + " desc->dst_addr %llx, desc->max_dst_size %d," + " desc->src2_addr %llx, desc->src2_size %d\n", __func__, + active_compression_mode->name, + desc->src1_addr, desc->src1_size, desc->dst_addr, + desc->max_dst_size, desc->src2_addr, desc->src2_size); + + ret = idxd_submit_desc(wq, idxd_desc); + if (ret) { + dev_dbg(dev, "submit_desc failed ret=%d\n", ret); + goto err; + } + + /* Update stats */ + update_total_comp_calls(); + update_wq_comp_calls(wq); + + if (ctx->async_mode) { + ret = -EINPROGRESS; + dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__); + goto out; + } + + ret = check_completion(dev, idxd_desc->iax_completion, true, false); + if (ret) { + dev_dbg(dev, "check_completion failed ret=%d\n", ret); + goto err; + } + + *dlen = idxd_desc->iax_completion->output_size; + + /* Update stats */ + update_total_comp_bytes_out(*dlen); + update_wq_comp_bytes(wq, *dlen); + + req_ctx->compression_crc = idxd_desc->iax_completion->crc; + + if (!ctx->async_mode) + idxd_free_desc(wq, idxd_desc); +out: + return ret; +err: + idxd_free_desc(wq, idxd_desc); + dev_dbg(dev, "iaa compress failed: ret=%d\n", ret); + + goto out; +} + +static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq, + struct acomp_req *req, + dma_addr_t *src_addr, dma_addr_t *dst_addr) +{ + int ret = 0; + int nr_sgs; + + dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); + dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); + + nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE); + if (nr_sgs <= 0 || nr_sgs > 1) { + dev_dbg(dev, "verify: couldn't map src sg for iaa device %d," + " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, + iaa_wq->wq->id, ret); + ret = -EIO; + goto out; + } + *src_addr = sg_dma_address(req->src); + dev_dbg(dev, "verify: dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p," + " req->slen %d, sg_dma_len(sg) %d\n", *src_addr, nr_sgs, + req->src, req->slen, sg_dma_len(req->src)); + + nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_TO_DEVICE); + if (nr_sgs <= 0 || nr_sgs > 1) { + dev_dbg(dev, "verify: couldn't map dst sg for iaa device %d," + " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, + iaa_wq->wq->id, ret); + ret = -EIO; + dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE); + goto out; + } + *dst_addr = sg_dma_address(req->dst); + dev_dbg(dev, "verify: dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p," + " req->dlen %d, sg_dma_len(sg) %d\n", *dst_addr, nr_sgs, + req->dst, req->dlen, sg_dma_len(req->dst)); +out: + return ret; +} + +static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req, + struct idxd_wq *wq, + dma_addr_t src_addr, unsigned int slen, + dma_addr_t dst_addr, unsigned int *dlen) +{ + struct iaa_device_compression_mode *active_compression_mode; + struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); + struct iaa_req_ctx *req_ctx = acomp_request_ctx(req); + struct iaa_device *iaa_device; + struct idxd_desc *idxd_desc; + struct iax_hw_desc *desc; + struct idxd_device *idxd; + struct iaa_wq *iaa_wq; + struct pci_dev *pdev; + struct device *dev; + int ret = 0; + + iaa_wq = idxd_wq_get_private(wq); + iaa_device = iaa_wq->iaa_device; + idxd = iaa_device->idxd; + pdev = idxd->pdev; + dev = &pdev->dev; + + active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode); + + idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK); + if (IS_ERR(idxd_desc)) { + dev_dbg(dev, "idxd descriptor allocation failed\n"); + dev_dbg(dev, "iaa compress failed: ret=%ld\n", + PTR_ERR(idxd_desc)); + return PTR_ERR(idxd_desc); + } + desc = idxd_desc->iax_hw; + + /* Verify (optional) - decompress and check crc, suppress dest write */ + + desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC; + desc->opcode = IAX_OPCODE_DECOMPRESS; + desc->decompr_flags = IAA_DECOMP_FLAGS | IAA_DECOMP_SUPPRESS_OUTPUT; + desc->priv = 0; + + desc->src1_addr = (u64)dst_addr; + desc->src1_size = *dlen; + desc->dst_addr = (u64)src_addr; + desc->max_dst_size = slen; + desc->completion_addr = idxd_desc->compl_dma; + + dev_dbg(dev, "(verify) compression mode %s," + " desc->src1_addr %llx, desc->src1_size %d," + " desc->dst_addr %llx, desc->max_dst_size %d," + " desc->src2_addr %llx, desc->src2_size %d\n", + active_compression_mode->name, + desc->src1_addr, desc->src1_size, desc->dst_addr, + desc->max_dst_size, desc->src2_addr, desc->src2_size); + + ret = idxd_submit_desc(wq, idxd_desc); + if (ret) { + dev_dbg(dev, "submit_desc (verify) failed ret=%d\n", ret); + goto err; + } + + ret = check_completion(dev, idxd_desc->iax_completion, false, false); + if (ret) { + dev_dbg(dev, "(verify) check_completion failed ret=%d\n", ret); + goto err; + } + + if (req_ctx->compression_crc != idxd_desc->iax_completion->crc) { + ret = -EINVAL; + dev_dbg(dev, "(verify) iaa comp/decomp crc mismatch: comp=0x%x, decomp=0x%x\n", + req_ctx->compression_crc, + idxd_desc->iax_completion->crc); + print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET, + 8, 1, idxd_desc->iax_completion, 64, 0); + goto err; + } + + idxd_free_desc(wq, idxd_desc); +out: + return ret; +err: + idxd_free_desc(wq, idxd_desc); + dev_dbg(dev, "iaa compress failed: ret=%d\n", ret); + + goto out; +} + +static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, + struct idxd_wq *wq, + dma_addr_t src_addr, unsigned int slen, + dma_addr_t dst_addr, unsigned int *dlen) +{ + struct iaa_device_compression_mode *active_compression_mode; + struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); + struct iaa_device *iaa_device; + struct idxd_desc *idxd_desc; + struct iax_hw_desc *desc; + struct idxd_device *idxd; + struct iaa_wq *iaa_wq; + struct pci_dev *pdev; + struct device *dev; + int ret = 0; + + iaa_wq = idxd_wq_get_private(wq); + iaa_device = iaa_wq->iaa_device; + idxd = iaa_device->idxd; + pdev = idxd->pdev; + dev = &pdev->dev; + + active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode); + + idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK); + if (IS_ERR(idxd_desc)) { + dev_dbg(dev, "idxd descriptor allocation failed\n"); + dev_dbg(dev, "iaa decompress failed: ret=%ld\n", + PTR_ERR(idxd_desc)); + return PTR_ERR(idxd_desc); + } + desc = idxd_desc->iax_hw; + + desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC; + desc->opcode = IAX_OPCODE_DECOMPRESS; + desc->max_dst_size = PAGE_SIZE; + desc->decompr_flags = IAA_DECOMP_FLAGS; + desc->priv = 0; + + desc->src1_addr = (u64)src_addr; + desc->dst_addr = (u64)dst_addr; + desc->max_dst_size = *dlen; + desc->src1_size = slen; + desc->completion_addr = idxd_desc->compl_dma; + + if (ctx->use_irq) { + desc->flags |= IDXD_OP_FLAG_RCI; + + idxd_desc->crypto.req = req; + idxd_desc->crypto.tfm = tfm; + idxd_desc->crypto.src_addr = src_addr; + idxd_desc->crypto.dst_addr = dst_addr; + idxd_desc->crypto.compress = false; + + dev_dbg(dev, "%s: use_async_irq compression mode %s," + " src_addr %llx, dst_addr %llx\n", __func__, + active_compression_mode->name, + src_addr, dst_addr); + } + + dev_dbg(dev, "%s: decompression mode %s," + " desc->src1_addr %llx, desc->src1_size %d," + " desc->dst_addr %llx, desc->max_dst_size %d," + " desc->src2_addr %llx, desc->src2_size %d\n", __func__, + active_compression_mode->name, + desc->src1_addr, desc->src1_size, desc->dst_addr, + desc->max_dst_size, desc->src2_addr, desc->src2_size); + + ret = idxd_submit_desc(wq, idxd_desc); + if (ret) { + dev_dbg(dev, "submit_desc failed ret=%d\n", ret); + goto err; + } + + /* Update stats */ + update_total_decomp_calls(); + update_wq_decomp_calls(wq); + + if (ctx->async_mode) { + ret = -EINPROGRESS; + dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__); + goto out; + } + + ret = check_completion(dev, idxd_desc->iax_completion, false, false); + if (ret) { + dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret); + if (iaa_error_should_retry(idxd_desc)) + ret = -EAGAIN; + goto err; + } else { + req->dlen = idxd_desc->iax_completion->output_size; + + /* Update stats */ + update_total_decomp_bytes_in(slen); + update_wq_decomp_bytes(wq, slen); + } + + *dlen = req->dlen; + + if (!ctx->async_mode) + idxd_free_desc(wq, idxd_desc); +out: + return ret; +err: + idxd_free_desc(wq, idxd_desc); + dev_dbg(dev, "iaa decompress failed: ret=%d\n", ret); + + goto out; +} + +static int iaa_comp_acompress(struct acomp_req *req) +{ + struct iaa_req_ctx *req_ctx = acomp_request_ctx(req); + struct iaa_compression_ctx *compression_ctx; + struct crypto_tfm *tfm = req->base.tfm; + dma_addr_t src_addr, dst_addr; + int cpu, ret = 0; + struct iaa_wq *iaa_wq; + struct idxd_wq *wq; + struct device *dev; + + req_ctx->bounce_src = NULL; + req_ctx->bounce_src_dma = 0; + req_ctx->bounce_src_len = 0; + + compression_ctx = crypto_tfm_ctx(tfm); + + if (!iaa_crypto_enabled) { + pr_debug("iaa_crypto disabled, not compressing\n"); + return -ENODEV; + } + + if (!req->src || !req->slen || !req->dst) { + pr_debug("invalid req, not compressing\n"); + return -EINVAL; + } + + /* Fall back to software if src or dst has multiple sg entries */ + if (sg_nents(req->src) > 1 || sg_nents(req->dst) > 1) + return deflate_generic_compress(req); + + cpu = get_cpu(); + wq = wq_table_next_wq(cpu); + put_cpu(); + if (!wq) { + pr_debug("no wq configured for cpu=%d\n", cpu); + return -ENODEV; + } + + ret = iaa_wq_get(wq); + if (ret) { + pr_debug("no wq available for cpu=%d\n", cpu); + return -ENODEV; + } + + iaa_wq = idxd_wq_get_private(wq); + + dev = &wq->idxd->pdev->dev; + + if (!dma_map_sg(dev, req->src, 1, DMA_TO_DEVICE)) { + dev_dbg(dev, "couldn't map src sg for iaa device %d, wq %d\n", + iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id); + iaa_wq_put(wq); + return deflate_generic_compress(req); + } + src_addr = sg_dma_address(req->src); + dev_dbg(dev, "map src %llx req->src %p slen %d sg_len %d\n", src_addr, + req->src, req->slen, sg_dma_len(req->src)); + + if (!dma_map_sg(dev, req->dst, 1, DMA_FROM_DEVICE)) { + dev_dbg(dev, "couldn't map dst sg for iaa device %d, wq %d\n", + iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id); + dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE); + iaa_wq_put(wq); + return deflate_generic_compress(req); + } + dst_addr = sg_dma_address(req->dst); + dev_dbg(dev, "map dst %llx req->dst %p dlen %d sg_len %d\n", dst_addr, + req->dst, req->dlen, sg_dma_len(req->dst)); + + ret = iaa_compress(tfm, req, wq, src_addr, req->slen, dst_addr, + &req->dlen); + if (ret == -EINPROGRESS) + return ret; + + if (!ret && compression_ctx->verify_compress) { + ret = iaa_remap_for_verify(dev, iaa_wq, req, &src_addr, &dst_addr); + if (ret) { + dev_dbg(dev, "%s: compress verify remap failed ret=%d\n", __func__, ret); + goto out; + } + + ret = iaa_compress_verify(tfm, req, wq, src_addr, req->slen, + dst_addr, &req->dlen); + if (ret) + dev_dbg(dev, "asynchronous compress verification failed ret=%d\n", ret); + + dma_unmap_sg(dev, req->dst, 1, DMA_TO_DEVICE); + dma_unmap_sg(dev, req->src, 1, DMA_FROM_DEVICE); + + goto out; + } + + if (ret) + dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret); + + dma_unmap_sg(dev, req->dst, 1, DMA_FROM_DEVICE); + dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE); +out: + iaa_wq_put(wq); + + return ret; +} + +static int iaa_comp_adecompress(struct acomp_req *req) +{ + struct iaa_req_ctx *req_ctx = acomp_request_ctx(req); + struct crypto_tfm *tfm = req->base.tfm; + dma_addr_t src_addr, dst_addr; + bool use_bounce_src = false; + int cpu, ret = 0; + struct iaa_wq *iaa_wq; + struct device *dev; + struct idxd_wq *wq; + struct page *page; + + req_ctx->bounce_src = NULL; + req_ctx->bounce_src_dma = 0; + req_ctx->bounce_src_len = 0; + + if (!iaa_crypto_enabled) { + pr_debug("iaa_crypto disabled, not decompressing\n"); + return -ENODEV; + } + + if (!req->src || !req->slen || !req->dst) { + pr_debug("invalid req, not decompressing\n"); + return -EINVAL; + } + + /* Fall back to software if dst has multiple sg entries */ + if (sg_nents(req->dst) > 1) + return deflate_generic_decompress(req); + + if (sg_nents(req->src) > 1) { + if (req->slen > PAGE_SIZE) + return deflate_generic_decompress(req); + use_bounce_src = true; + } + + cpu = get_cpu(); + wq = wq_table_next_wq(cpu); + put_cpu(); + if (!wq) { + pr_debug("no wq configured for cpu=%d\n", cpu); + return -ENODEV; + } + + ret = iaa_wq_get(wq); + if (ret) { + pr_debug("no wq available for cpu=%d\n", cpu); + return -ENODEV; + } + + iaa_wq = idxd_wq_get_private(wq); + + dev = &wq->idxd->pdev->dev; + + if (unlikely(use_bounce_src)) { + page = mempool_alloc(iaa_bounce_pool, GFP_ATOMIC); + if (!page) { + iaa_wq_put(wq); + return deflate_generic_decompress(req); + } + + if (sg_copy_to_buffer(req->src, sg_nents(req->src), + page_address(page), req->slen) != req->slen) { + mempool_free(page, iaa_bounce_pool); + iaa_wq_put(wq); + return deflate_generic_decompress(req); + } + + src_addr = dma_map_page(dev, page, 0, req->slen, DMA_TO_DEVICE); + if (dma_mapping_error(dev, src_addr)) { + mempool_free(page, iaa_bounce_pool); + iaa_wq_put(wq); + return deflate_generic_decompress(req); + } + + req_ctx->bounce_src = page; + req_ctx->bounce_src_dma = src_addr; + req_ctx->bounce_src_len = req->slen; + } else { + if (!dma_map_sg(dev, req->src, 1, DMA_TO_DEVICE)) { + dev_dbg(dev, "couldn't map src sg for iaa device %d, wq %d\n", + iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id); + iaa_wq_put(wq); + return deflate_generic_decompress(req); + } + + src_addr = sg_dma_address(req->src); + dev_dbg(dev, "map src %llx req->src %p slen %d sg_len %d\n", src_addr, + req->src, req->slen, sg_dma_len(req->src)); + } + + if (!dma_map_sg(dev, req->dst, 1, DMA_FROM_DEVICE)) { + dev_dbg(dev, "couldn't map dst sg for iaa device %d, wq %d\n", + iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id); + iaa_unmap_src(dev, req); + iaa_wq_put(wq); + return deflate_generic_decompress(req); + } + dst_addr = sg_dma_address(req->dst); + dev_dbg(dev, "map dst %llx req->dst %p dlen %d sg_len %d\n", dst_addr, + req->dst, req->dlen, sg_dma_len(req->dst)); + + ret = iaa_decompress(tfm, req, wq, src_addr, req->slen, + dst_addr, &req->dlen); + if (ret == -EINPROGRESS) + return ret; + + if (ret != 0 && ret != -EAGAIN) + dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret); + + dma_unmap_sg(dev, req->dst, 1, DMA_FROM_DEVICE); + iaa_unmap_src(dev, req); + iaa_wq_put(wq); + + if (ret == -EAGAIN) + ret = deflate_generic_decompress(req); + + return ret; +} + +static void compression_ctx_init(struct iaa_compression_ctx *ctx) +{ + ctx->verify_compress = iaa_verify_compress; + ctx->async_mode = async_mode; + ctx->use_irq = use_irq; +} + +static int iaa_comp_init_fixed(struct crypto_acomp *acomp_tfm) +{ + struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm); + struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); + + compression_ctx_init(ctx); + + ctx->mode = IAA_MODE_FIXED; + + return 0; +} + +static struct acomp_alg iaa_acomp_fixed_deflate = { + .init = iaa_comp_init_fixed, + .compress = iaa_comp_acompress, + .decompress = iaa_comp_adecompress, + .base = { + .cra_name = "deflate", + .cra_driver_name = "deflate-iaa", + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_ctxsize = sizeof(struct iaa_compression_ctx), + .cra_reqsize = sizeof(struct iaa_req_ctx), + .cra_module = THIS_MODULE, + .cra_priority = IAA_ALG_PRIORITY, + } +}; + +static int iaa_register_compression_device(void) +{ + int ret; + + ret = crypto_register_acomp(&iaa_acomp_fixed_deflate); + if (ret) { + pr_err("deflate algorithm acomp fixed registration failed (%d)\n", ret); + goto out; + } + + iaa_crypto_registered = true; +out: + return ret; +} + +static void iaa_unregister_compression_device(void) +{ + if (iaa_crypto_registered) + crypto_unregister_acomp(&iaa_acomp_fixed_deflate); +} + +static int iaa_crypto_probe(struct idxd_dev *idxd_dev) +{ + struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); + struct idxd_device *idxd = wq->idxd; + struct idxd_driver_data *data = idxd->data; + struct device *dev = &idxd_dev->conf_dev; + bool first_wq = false; + int ret = 0; + + if (idxd->state != IDXD_DEV_ENABLED) + return -ENXIO; + + if (data->type != IDXD_TYPE_IAX) + return -ENODEV; + + mutex_lock(&wq->wq_lock); + + if (idxd_wq_get_private(wq)) { + mutex_unlock(&wq->wq_lock); + return -EBUSY; + } + + if (!idxd_wq_driver_name_match(wq, dev)) { + dev_dbg(dev, "wq %d.%d driver_name match failed: wq driver_name %s, dev driver name %s\n", + idxd->id, wq->id, wq->driver_name, dev->driver->name); + idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME; + ret = -ENODEV; + goto err; + } + + wq->type = IDXD_WQT_KERNEL; + + ret = idxd_drv_enable_wq(wq); + if (ret < 0) { + dev_dbg(dev, "enable wq %d.%d failed: %d\n", + idxd->id, wq->id, ret); + ret = -ENXIO; + goto err; + } + + mutex_lock(&iaa_devices_lock); + + if (list_empty(&iaa_devices)) { + ret = alloc_wq_table(wq->idxd->max_wqs); + if (ret) + goto err_alloc; + first_wq = true; + } + + ret = save_iaa_wq(wq); + if (ret) + goto err_save; + + rebalance_wq_table(); + + if (first_wq) { + iaa_crypto_enabled = true; + ret = iaa_register_compression_device(); + if (ret != 0) { + iaa_crypto_enabled = false; + dev_dbg(dev, "IAA compression device registration failed\n"); + goto err_register; + } + try_module_get(THIS_MODULE); + + pr_info("iaa_crypto now ENABLED\n"); + } + + mutex_unlock(&iaa_devices_lock); +out: + mutex_unlock(&wq->wq_lock); + + return ret; + +err_register: + remove_iaa_wq(wq); + free_iaa_wq(idxd_wq_get_private(wq)); +err_save: + if (first_wq) + free_wq_table(); +err_alloc: + mutex_unlock(&iaa_devices_lock); + idxd_drv_disable_wq(wq); +err: + wq->type = IDXD_WQT_NONE; + + goto out; +} + +static void iaa_crypto_remove(struct idxd_dev *idxd_dev) +{ + struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); + struct idxd_device *idxd = wq->idxd; + struct iaa_wq *iaa_wq; + bool free = false; + + idxd_wq_quiesce(wq); + + mutex_lock(&wq->wq_lock); + mutex_lock(&iaa_devices_lock); + + remove_iaa_wq(wq); + + spin_lock(&idxd->dev_lock); + iaa_wq = idxd_wq_get_private(wq); + if (!iaa_wq) { + spin_unlock(&idxd->dev_lock); + pr_err("%s: no iaa_wq available to remove\n", __func__); + goto out; + } + + if (iaa_wq->ref) { + iaa_wq->remove = true; + } else { + wq = iaa_wq->wq; + idxd_wq_set_private(wq, NULL); + free = true; + } + spin_unlock(&idxd->dev_lock); + if (free) { + __free_iaa_wq(iaa_wq); + kfree(iaa_wq); + } + + idxd_drv_disable_wq(wq); + rebalance_wq_table(); + + if (nr_iaa == 0) { + iaa_crypto_enabled = false; + free_wq_table(); + module_put(THIS_MODULE); + + pr_info("iaa_crypto now DISABLED\n"); + } +out: + mutex_unlock(&iaa_devices_lock); + mutex_unlock(&wq->wq_lock); +} + +static enum idxd_dev_type dev_types[] = { + IDXD_DEV_WQ, + IDXD_DEV_NONE, +}; + +static struct idxd_device_driver iaa_crypto_driver = { + .probe = iaa_crypto_probe, + .remove = iaa_crypto_remove, + .name = IDXD_SUBDRIVER_NAME, + .type = dev_types, + .desc_complete = iaa_desc_complete, +}; + +static int __init iaa_crypto_init_module(void) +{ + int ret = 0; + int node; + + nr_cpus = num_possible_cpus(); + for_each_node_with_cpus(node) + nr_nodes++; + if (!nr_nodes) { + pr_err("IAA couldn't find any nodes with cpus\n"); + return -ENODEV; + } + nr_cpus_per_node = nr_cpus / nr_nodes; + + ret = iaa_aecs_init_fixed(); + if (ret < 0) { + pr_debug("IAA fixed compression mode init failed\n"); + goto err_aecs_init; + } + + iaa_bounce_pool = mempool_create_page_pool(IAA_BOUNCE_POOL_SIZE, 0); + if (!iaa_bounce_pool) { + ret = -ENOMEM; + goto err_bounce_pool; + } + + ret = idxd_driver_register(&iaa_crypto_driver); + if (ret) { + pr_debug("IAA wq sub-driver registration failed\n"); + goto err_driver_reg; + } + + ret = driver_create_file(&iaa_crypto_driver.drv, + &driver_attr_verify_compress); + if (ret) { + pr_debug("IAA verify_compress attr creation failed\n"); + goto err_verify_attr_create; + } + + ret = driver_create_file(&iaa_crypto_driver.drv, + &driver_attr_sync_mode); + if (ret) { + pr_debug("IAA sync mode attr creation failed\n"); + goto err_sync_attr_create; + } + + if (iaa_crypto_debugfs_init()) + pr_warn("debugfs init failed, stats not available\n"); + + pr_debug("initialized\n"); +out: + return ret; + +err_sync_attr_create: + driver_remove_file(&iaa_crypto_driver.drv, + &driver_attr_verify_compress); +err_verify_attr_create: + idxd_driver_unregister(&iaa_crypto_driver); +err_driver_reg: + mempool_destroy(iaa_bounce_pool); + iaa_bounce_pool = NULL; +err_bounce_pool: + iaa_aecs_cleanup_fixed(); +err_aecs_init: + + goto out; +} + +static void __exit iaa_crypto_cleanup_module(void) +{ + iaa_unregister_compression_device(); + + iaa_crypto_debugfs_cleanup(); + driver_remove_file(&iaa_crypto_driver.drv, + &driver_attr_sync_mode); + driver_remove_file(&iaa_crypto_driver.drv, + &driver_attr_verify_compress); + idxd_driver_unregister(&iaa_crypto_driver); + mempool_destroy(iaa_bounce_pool); + iaa_bounce_pool = NULL; + iaa_aecs_cleanup_fixed(); + + pr_debug("cleaned up\n"); +} + +MODULE_IMPORT_NS("IDXD"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS_IDXD_DEVICE(0); +MODULE_AUTHOR("Intel Corporation"); +MODULE_DESCRIPTION("IAA Compression Accelerator Crypto Driver"); + +module_init(iaa_crypto_init_module); +module_exit(iaa_crypto_cleanup_module); diff --git a/drivers/crypto/intel/iaa/iaa_crypto_stats.c b/drivers/crypto/intel/iaa/iaa_crypto_stats.c new file mode 100644 index 000000000..2f2ed88c8 --- /dev/null +++ b/drivers/crypto/intel/iaa/iaa_crypto_stats.c @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2021 Intel Corporation. All rights rsvd. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../dma/idxd/idxd.h" +#include +#include +#include "iaa_crypto.h" +#include "iaa_crypto_stats.h" + +static atomic64_t total_comp_calls; +static atomic64_t total_decomp_calls; +static atomic64_t total_sw_comp_calls; +static atomic64_t total_sw_decomp_calls; +static atomic64_t total_comp_bytes_out; +static atomic64_t total_decomp_bytes_in; +static atomic64_t total_completion_einval_errors; +static atomic64_t total_completion_timeout_errors; +static atomic64_t total_completion_comp_buf_overflow_errors; + +static struct dentry *iaa_crypto_debugfs_root; + +void update_total_comp_calls(void) +{ + atomic64_inc(&total_comp_calls); +} + +void update_total_comp_bytes_out(int n) +{ + atomic64_add(n, &total_comp_bytes_out); +} + +void update_total_decomp_calls(void) +{ + atomic64_inc(&total_decomp_calls); +} + +void update_total_sw_comp_calls(void) +{ + atomic64_inc(&total_sw_comp_calls); +} + +void update_total_sw_decomp_calls(void) +{ + atomic64_inc(&total_sw_decomp_calls); +} + +void update_total_decomp_bytes_in(int n) +{ + atomic64_add(n, &total_decomp_bytes_in); +} + +void update_completion_einval_errs(void) +{ + atomic64_inc(&total_completion_einval_errors); +} + +void update_completion_timeout_errs(void) +{ + atomic64_inc(&total_completion_timeout_errors); +} + +void update_completion_comp_buf_overflow_errs(void) +{ + atomic64_inc(&total_completion_comp_buf_overflow_errors); +} + +void update_wq_comp_calls(struct idxd_wq *idxd_wq) +{ + struct iaa_wq *wq = idxd_wq_get_private(idxd_wq); + + atomic64_inc(&wq->comp_calls); + atomic64_inc(&wq->iaa_device->comp_calls); +} + +void update_wq_comp_bytes(struct idxd_wq *idxd_wq, int n) +{ + struct iaa_wq *wq = idxd_wq_get_private(idxd_wq); + + atomic64_add(n, &wq->comp_bytes); + atomic64_add(n, &wq->iaa_device->comp_bytes); +} + +void update_wq_decomp_calls(struct idxd_wq *idxd_wq) +{ + struct iaa_wq *wq = idxd_wq_get_private(idxd_wq); + + atomic64_inc(&wq->decomp_calls); + atomic64_inc(&wq->iaa_device->decomp_calls); +} + +void update_wq_decomp_bytes(struct idxd_wq *idxd_wq, int n) +{ + struct iaa_wq *wq = idxd_wq_get_private(idxd_wq); + + atomic64_add(n, &wq->decomp_bytes); + atomic64_add(n, &wq->iaa_device->decomp_bytes); +} + +static void reset_iaa_crypto_stats(void) +{ + atomic64_set(&total_comp_calls, 0); + atomic64_set(&total_decomp_calls, 0); + atomic64_set(&total_sw_comp_calls, 0); + atomic64_set(&total_sw_decomp_calls, 0); + atomic64_set(&total_comp_bytes_out, 0); + atomic64_set(&total_decomp_bytes_in, 0); + atomic64_set(&total_completion_einval_errors, 0); + atomic64_set(&total_completion_timeout_errors, 0); + atomic64_set(&total_completion_comp_buf_overflow_errors, 0); +} + +static void reset_wq_stats(struct iaa_wq *wq) +{ + atomic64_set(&wq->comp_calls, 0); + atomic64_set(&wq->comp_bytes, 0); + atomic64_set(&wq->decomp_calls, 0); + atomic64_set(&wq->decomp_bytes, 0); +} + +static void reset_device_stats(struct iaa_device *iaa_device) +{ + struct iaa_wq *iaa_wq; + + atomic64_set(&iaa_device->comp_calls, 0); + atomic64_set(&iaa_device->comp_bytes, 0); + atomic64_set(&iaa_device->decomp_calls, 0); + atomic64_set(&iaa_device->decomp_bytes, 0); + + list_for_each_entry(iaa_wq, &iaa_device->wqs, list) + reset_wq_stats(iaa_wq); +} + +static void wq_show(struct seq_file *m, struct iaa_wq *iaa_wq) +{ + seq_printf(m, " name: %s\n", iaa_wq->wq->name); + seq_printf(m, " comp_calls: %llu\n", + atomic64_read(&iaa_wq->comp_calls)); + seq_printf(m, " comp_bytes: %llu\n", + atomic64_read(&iaa_wq->comp_bytes)); + seq_printf(m, " decomp_calls: %llu\n", + atomic64_read(&iaa_wq->decomp_calls)); + seq_printf(m, " decomp_bytes: %llu\n\n", + atomic64_read(&iaa_wq->decomp_bytes)); +} + +static void device_stats_show(struct seq_file *m, struct iaa_device *iaa_device) +{ + struct iaa_wq *iaa_wq; + + seq_puts(m, "iaa device:\n"); + seq_printf(m, " id: %d\n", iaa_device->idxd->id); + seq_printf(m, " n_wqs: %d\n", iaa_device->n_wq); + seq_printf(m, " comp_calls: %llu\n", + atomic64_read(&iaa_device->comp_calls)); + seq_printf(m, " comp_bytes: %llu\n", + atomic64_read(&iaa_device->comp_bytes)); + seq_printf(m, " decomp_calls: %llu\n", + atomic64_read(&iaa_device->decomp_calls)); + seq_printf(m, " decomp_bytes: %llu\n", + atomic64_read(&iaa_device->decomp_bytes)); + seq_puts(m, " wqs:\n"); + + list_for_each_entry(iaa_wq, &iaa_device->wqs, list) + wq_show(m, iaa_wq); +} + +static int global_stats_show(struct seq_file *m, void *v) +{ + seq_puts(m, "global stats:\n"); + seq_printf(m, " total_comp_calls: %llu\n", + atomic64_read(&total_comp_calls)); + seq_printf(m, " total_decomp_calls: %llu\n", + atomic64_read(&total_decomp_calls)); + seq_printf(m, " total_sw_comp_calls: %llu\n", + atomic64_read(&total_sw_comp_calls)); + seq_printf(m, " total_sw_decomp_calls: %llu\n", + atomic64_read(&total_sw_decomp_calls)); + seq_printf(m, " total_comp_bytes_out: %llu\n", + atomic64_read(&total_comp_bytes_out)); + seq_printf(m, " total_decomp_bytes_in: %llu\n", + atomic64_read(&total_decomp_bytes_in)); + seq_printf(m, " total_completion_einval_errors: %llu\n", + atomic64_read(&total_completion_einval_errors)); + seq_printf(m, " total_completion_timeout_errors: %llu\n", + atomic64_read(&total_completion_timeout_errors)); + seq_printf(m, " total_completion_comp_buf_overflow_errors: %llu\n\n", + atomic64_read(&total_completion_comp_buf_overflow_errors)); + + return 0; +} + +static int wq_stats_show(struct seq_file *m, void *v) +{ + struct iaa_device *iaa_device; + + mutex_lock(&iaa_devices_lock); + + list_for_each_entry(iaa_device, &iaa_devices, list) + device_stats_show(m, iaa_device); + + mutex_unlock(&iaa_devices_lock); + + return 0; +} + +static int iaa_crypto_stats_reset(void *data, u64 value) +{ + struct iaa_device *iaa_device; + + reset_iaa_crypto_stats(); + + mutex_lock(&iaa_devices_lock); + + list_for_each_entry(iaa_device, &iaa_devices, list) + reset_device_stats(iaa_device); + + mutex_unlock(&iaa_devices_lock); + + return 0; +} + +static int wq_stats_open(struct inode *inode, struct file *file) +{ + return single_open(file, wq_stats_show, file); +} + +static const struct file_operations wq_stats_fops = { + .open = wq_stats_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int global_stats_open(struct inode *inode, struct file *file) +{ + return single_open(file, global_stats_show, file); +} + +static const struct file_operations global_stats_fops = { + .open = global_stats_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +DEFINE_DEBUGFS_ATTRIBUTE(wq_stats_reset_fops, NULL, iaa_crypto_stats_reset, "%llu\n"); + +int __init iaa_crypto_debugfs_init(void) +{ + if (!debugfs_initialized()) + return -ENODEV; + + iaa_crypto_debugfs_root = debugfs_create_dir("iaa_crypto", NULL); + + debugfs_create_file("global_stats", 0644, iaa_crypto_debugfs_root, NULL, + &global_stats_fops); + debugfs_create_file("wq_stats", 0644, iaa_crypto_debugfs_root, NULL, + &wq_stats_fops); + debugfs_create_file("stats_reset", 0644, iaa_crypto_debugfs_root, NULL, + &wq_stats_reset_fops); + + return 0; +} + +void __exit iaa_crypto_debugfs_cleanup(void) +{ + debugfs_remove_recursive(iaa_crypto_debugfs_root); +} + +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/intel/iaa/iaa_crypto_stats.h b/drivers/crypto/intel/iaa/iaa_crypto_stats.h new file mode 100644 index 000000000..6e0c6f993 --- /dev/null +++ b/drivers/crypto/intel/iaa/iaa_crypto_stats.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2021 Intel Corporation. All rights rsvd. */ + +#ifndef __CRYPTO_DEV_IAA_CRYPTO_STATS_H__ +#define __CRYPTO_DEV_IAA_CRYPTO_STATS_H__ + +#if defined(CONFIG_CRYPTO_DEV_IAA_CRYPTO_STATS) +int iaa_crypto_debugfs_init(void); +void iaa_crypto_debugfs_cleanup(void); + +void update_total_comp_calls(void); +void update_total_comp_bytes_out(int n); +void update_total_decomp_calls(void); +void update_total_sw_comp_calls(void); +void update_total_sw_decomp_calls(void); +void update_total_decomp_bytes_in(int n); +void update_completion_einval_errs(void); +void update_completion_timeout_errs(void); +void update_completion_comp_buf_overflow_errs(void); + +void update_wq_comp_calls(struct idxd_wq *idxd_wq); +void update_wq_comp_bytes(struct idxd_wq *idxd_wq, int n); +void update_wq_decomp_calls(struct idxd_wq *idxd_wq); +void update_wq_decomp_bytes(struct idxd_wq *idxd_wq, int n); + +#else +static inline int iaa_crypto_debugfs_init(void) { return 0; } +static inline void iaa_crypto_debugfs_cleanup(void) {} + +static inline void update_total_comp_calls(void) {} +static inline void update_total_comp_bytes_out(int n) {} +static inline void update_total_decomp_calls(void) {} +static inline void update_total_sw_comp_calls(void) {} +static inline void update_total_sw_decomp_calls(void) {} +static inline void update_total_decomp_bytes_in(int n) {} +static inline void update_completion_einval_errs(void) {} +static inline void update_completion_timeout_errs(void) {} +static inline void update_completion_comp_buf_overflow_errs(void) {} + +static inline void update_wq_comp_calls(struct idxd_wq *idxd_wq) {} +static inline void update_wq_comp_bytes(struct idxd_wq *idxd_wq, int n) {} +static inline void update_wq_decomp_calls(struct idxd_wq *idxd_wq) {} +static inline void update_wq_decomp_bytes(struct idxd_wq *idxd_wq, int n) {} + +#endif // CONFIG_CRYPTO_DEV_IAA_CRYPTO_STATS + +#endif diff --git a/drivers/crypto/intel/ixp4xx/Kconfig b/drivers/crypto/intel/ixp4xx/Kconfig new file mode 100644 index 000000000..af3cc5688 --- /dev/null +++ b/drivers/crypto/intel/ixp4xx/Kconfig @@ -0,0 +1,14 @@ +config CRYPTO_DEV_IXP4XX + tristate "Driver for IXP4xx crypto hardware acceleration" + depends on (ARCH_IXP4XX || COMPILE_TEST) && IXP4XX_QMGR && IXP4XX_NPE + select CRYPTO_AES + select CRYPTO_DES + select CRYPTO_ECB + select CRYPTO_CBC + select CRYPTO_CTR + select CRYPTO_LIB_DES + select CRYPTO_AEAD + select CRYPTO_AUTHENC + select CRYPTO_SKCIPHER + help + Driver for the IXP4xx NPE crypto engine. diff --git a/drivers/crypto/intel/ixp4xx/Makefile b/drivers/crypto/intel/ixp4xx/Makefile new file mode 100644 index 000000000..74ebefd93 --- /dev/null +++ b/drivers/crypto/intel/ixp4xx/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o diff --git a/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c b/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c new file mode 100644 index 000000000..fdda04672 --- /dev/null +++ b/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c @@ -0,0 +1,1609 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel IXP4xx NPE-C crypto driver + * + * Copyright (C) 2008 Christian Hohnstaedt + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +/* Intermittent includes, delete this after v5.14-rc1 */ +#include + +#define MAX_KEYLEN 32 + +/* hash: cfgword + 2 * digestlen; crypt: keylen + cfgword */ +#define NPE_CTX_LEN 80 +#define AES_BLOCK128 16 + +#define NPE_OP_HASH_VERIFY 0x01 +#define NPE_OP_CCM_ENABLE 0x04 +#define NPE_OP_CRYPT_ENABLE 0x08 +#define NPE_OP_HASH_ENABLE 0x10 +#define NPE_OP_NOT_IN_PLACE 0x20 +#define NPE_OP_HMAC_DISABLE 0x40 +#define NPE_OP_CRYPT_ENCRYPT 0x80 + +#define NPE_OP_CCM_GEN_MIC 0xcc +#define NPE_OP_HASH_GEN_ICV 0x50 +#define NPE_OP_ENC_GEN_KEY 0xc9 + +#define MOD_ECB 0x0000 +#define MOD_CTR 0x1000 +#define MOD_CBC_ENC 0x2000 +#define MOD_CBC_DEC 0x3000 +#define MOD_CCM_ENC 0x4000 +#define MOD_CCM_DEC 0x5000 + +#define KEYLEN_128 4 +#define KEYLEN_192 6 +#define KEYLEN_256 8 + +#define CIPH_DECR 0x0000 +#define CIPH_ENCR 0x0400 + +#define MOD_DES 0x0000 +#define MOD_TDEA2 0x0100 +#define MOD_3DES 0x0200 +#define MOD_AES 0x0800 +#define MOD_AES128 (0x0800 | KEYLEN_128) +#define MOD_AES192 (0x0900 | KEYLEN_192) +#define MOD_AES256 (0x0a00 | KEYLEN_256) + +#define MAX_IVLEN 16 +#define NPE_QLEN 16 +/* Space for registering when the first + * NPE_QLEN crypt_ctl are busy */ +#define NPE_QLEN_TOTAL 64 + +#define CTL_FLAG_UNUSED 0x0000 +#define CTL_FLAG_USED 0x1000 +#define CTL_FLAG_PERFORM_ABLK 0x0001 +#define CTL_FLAG_GEN_ICV 0x0002 +#define CTL_FLAG_GEN_REVAES 0x0004 +#define CTL_FLAG_PERFORM_AEAD 0x0008 +#define CTL_FLAG_MASK 0x000f + +#define HMAC_PAD_BLOCKLEN SHA1_BLOCK_SIZE + +#define MD5_DIGEST_SIZE 16 + +struct buffer_desc { + u32 phys_next; +#ifdef __ARMEB__ + u16 buf_len; + u16 pkt_len; +#else + u16 pkt_len; + u16 buf_len; +#endif + dma_addr_t phys_addr; + u32 __reserved[4]; + struct buffer_desc *next; + enum dma_data_direction dir; +}; + +struct crypt_ctl { +#ifdef __ARMEB__ + u8 mode; /* NPE_OP_* operation mode */ + u8 init_len; + u16 reserved; +#else + u16 reserved; + u8 init_len; + u8 mode; /* NPE_OP_* operation mode */ +#endif + u8 iv[MAX_IVLEN]; /* IV for CBC mode or CTR IV for CTR mode */ + u32 icv_rev_aes; /* icv or rev aes */ + u32 src_buf; + u32 dst_buf; +#ifdef __ARMEB__ + u16 auth_offs; /* Authentication start offset */ + u16 auth_len; /* Authentication data length */ + u16 crypt_offs; /* Cryption start offset */ + u16 crypt_len; /* Cryption data length */ +#else + u16 auth_len; /* Authentication data length */ + u16 auth_offs; /* Authentication start offset */ + u16 crypt_len; /* Cryption data length */ + u16 crypt_offs; /* Cryption start offset */ +#endif + u32 aadAddr; /* Additional Auth Data Addr for CCM mode */ + u32 crypto_ctx; /* NPE Crypto Param structure address */ + + /* Used by Host: 4*4 bytes*/ + unsigned int ctl_flags; + union { + struct skcipher_request *ablk_req; + struct aead_request *aead_req; + struct crypto_tfm *tfm; + } data; + struct buffer_desc *regist_buf; + u8 *regist_ptr; +}; + +struct ablk_ctx { + struct buffer_desc *src; + struct buffer_desc *dst; + u8 iv[MAX_IVLEN]; + bool encrypt; + struct skcipher_request fallback_req; // keep at the end +}; + +struct aead_ctx { + struct buffer_desc *src; + struct buffer_desc *dst; + struct scatterlist ivlist; + /* used when the hmac is not on one sg entry */ + u8 *hmac_virt; + int encrypt; +}; + +struct ix_hash_algo { + u32 cfgword; + unsigned char *icv; +}; + +struct ix_sa_dir { + unsigned char *npe_ctx; + dma_addr_t npe_ctx_phys; + int npe_ctx_idx; + u8 npe_mode; +}; + +struct ixp_ctx { + struct ix_sa_dir encrypt; + struct ix_sa_dir decrypt; + int authkey_len; + u8 authkey[MAX_KEYLEN]; + int enckey_len; + u8 enckey[MAX_KEYLEN]; + u8 salt[MAX_IVLEN]; + u8 nonce[CTR_RFC3686_NONCE_SIZE]; + unsigned int salted; + atomic_t configuring; + struct completion completion; + struct crypto_skcipher *fallback_tfm; +}; + +struct ixp_alg { + struct skcipher_alg crypto; + const struct ix_hash_algo *hash; + u32 cfg_enc; + u32 cfg_dec; + + int registered; +}; + +struct ixp_aead_alg { + struct aead_alg crypto; + const struct ix_hash_algo *hash; + u32 cfg_enc; + u32 cfg_dec; + + int registered; +}; + +static const struct ix_hash_algo hash_alg_md5 = { + .cfgword = 0xAA010004, + .icv = "\x01\x23\x45\x67\x89\xAB\xCD\xEF" + "\xFE\xDC\xBA\x98\x76\x54\x32\x10", +}; + +static const struct ix_hash_algo hash_alg_sha1 = { + .cfgword = 0x00000005, + .icv = "\x67\x45\x23\x01\xEF\xCD\xAB\x89\x98\xBA" + "\xDC\xFE\x10\x32\x54\x76\xC3\xD2\xE1\xF0", +}; + +static struct npe *npe_c; + +static unsigned int send_qid; +static unsigned int recv_qid; +static struct dma_pool *buffer_pool; +static struct dma_pool *ctx_pool; + +static struct crypt_ctl *crypt_virt; +static dma_addr_t crypt_phys; + +static int support_aes = 1; + +static struct platform_device *pdev; + +static inline dma_addr_t crypt_virt2phys(struct crypt_ctl *virt) +{ + return crypt_phys + (virt - crypt_virt) * sizeof(struct crypt_ctl); +} + +static inline struct crypt_ctl *crypt_phys2virt(dma_addr_t phys) +{ + return crypt_virt + (phys - crypt_phys) / sizeof(struct crypt_ctl); +} + +static inline u32 cipher_cfg_enc(struct crypto_tfm *tfm) +{ + return container_of(tfm->__crt_alg, struct ixp_alg, crypto.base)->cfg_enc; +} + +static inline u32 cipher_cfg_dec(struct crypto_tfm *tfm) +{ + return container_of(tfm->__crt_alg, struct ixp_alg, crypto.base)->cfg_dec; +} + +static inline const struct ix_hash_algo *ix_hash(struct crypto_tfm *tfm) +{ + return container_of(tfm->__crt_alg, struct ixp_alg, crypto.base)->hash; +} + +static int setup_crypt_desc(void) +{ + struct device *dev = &pdev->dev; + + BUILD_BUG_ON(!(IS_ENABLED(CONFIG_COMPILE_TEST) && + IS_ENABLED(CONFIG_64BIT)) && + sizeof(struct crypt_ctl) != 64); + crypt_virt = dma_alloc_coherent(dev, + NPE_QLEN * sizeof(struct crypt_ctl), + &crypt_phys, GFP_ATOMIC); + if (!crypt_virt) + return -ENOMEM; + return 0; +} + +static DEFINE_SPINLOCK(desc_lock); +static struct crypt_ctl *get_crypt_desc(void) +{ + int i; + static int idx; + unsigned long flags; + + spin_lock_irqsave(&desc_lock, flags); + + if (unlikely(!crypt_virt)) + setup_crypt_desc(); + if (unlikely(!crypt_virt)) { + spin_unlock_irqrestore(&desc_lock, flags); + return NULL; + } + i = idx; + if (crypt_virt[i].ctl_flags == CTL_FLAG_UNUSED) { + if (++idx >= NPE_QLEN) + idx = 0; + crypt_virt[i].ctl_flags = CTL_FLAG_USED; + spin_unlock_irqrestore(&desc_lock, flags); + return crypt_virt + i; + } else { + spin_unlock_irqrestore(&desc_lock, flags); + return NULL; + } +} + +static DEFINE_SPINLOCK(emerg_lock); +static struct crypt_ctl *get_crypt_desc_emerg(void) +{ + int i; + static int idx = NPE_QLEN; + struct crypt_ctl *desc; + unsigned long flags; + + desc = get_crypt_desc(); + if (desc) + return desc; + if (unlikely(!crypt_virt)) + return NULL; + + spin_lock_irqsave(&emerg_lock, flags); + i = idx; + if (crypt_virt[i].ctl_flags == CTL_FLAG_UNUSED) { + if (++idx >= NPE_QLEN_TOTAL) + idx = NPE_QLEN; + crypt_virt[i].ctl_flags = CTL_FLAG_USED; + spin_unlock_irqrestore(&emerg_lock, flags); + return crypt_virt + i; + } else { + spin_unlock_irqrestore(&emerg_lock, flags); + return NULL; + } +} + +static void free_buf_chain(struct device *dev, struct buffer_desc *buf, + dma_addr_t phys) +{ + while (buf) { + struct buffer_desc *buf1; + u32 phys1; + + buf1 = buf->next; + phys1 = buf->phys_next; + dma_unmap_single(dev, buf->phys_addr, buf->buf_len, buf->dir); + dma_pool_free(buffer_pool, buf, phys); + buf = buf1; + phys = phys1; + } +} + +static struct tasklet_struct crypto_done_tasklet; + +static void finish_scattered_hmac(struct crypt_ctl *crypt) +{ + struct aead_request *req = crypt->data.aead_req; + struct aead_ctx *req_ctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + int authsize = crypto_aead_authsize(tfm); + int decryptlen = req->assoclen + req->cryptlen - authsize; + + if (req_ctx->encrypt) { + scatterwalk_map_and_copy(req_ctx->hmac_virt, req->dst, + decryptlen, authsize, 1); + } + dma_pool_free(buffer_pool, req_ctx->hmac_virt, crypt->icv_rev_aes); +} + +static void one_packet(dma_addr_t phys) +{ + struct device *dev = &pdev->dev; + struct crypt_ctl *crypt; + struct ixp_ctx *ctx; + int failed; + + failed = phys & 0x1 ? -EBADMSG : 0; + phys &= ~0x3; + crypt = crypt_phys2virt(phys); + + switch (crypt->ctl_flags & CTL_FLAG_MASK) { + case CTL_FLAG_PERFORM_AEAD: { + struct aead_request *req = crypt->data.aead_req; + struct aead_ctx *req_ctx = aead_request_ctx(req); + + free_buf_chain(dev, req_ctx->src, crypt->src_buf); + free_buf_chain(dev, req_ctx->dst, crypt->dst_buf); + if (req_ctx->hmac_virt) + finish_scattered_hmac(crypt); + + aead_request_complete(req, failed); + break; + } + case CTL_FLAG_PERFORM_ABLK: { + struct skcipher_request *req = crypt->data.ablk_req; + struct ablk_ctx *req_ctx = skcipher_request_ctx(req); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + unsigned int ivsize = crypto_skcipher_ivsize(tfm); + unsigned int offset; + + if (ivsize > 0) { + offset = req->cryptlen - ivsize; + if (req_ctx->encrypt) { + scatterwalk_map_and_copy(req->iv, req->dst, + offset, ivsize, 0); + } else { + memcpy(req->iv, req_ctx->iv, ivsize); + memzero_explicit(req_ctx->iv, ivsize); + } + } + + if (req_ctx->dst) + free_buf_chain(dev, req_ctx->dst, crypt->dst_buf); + + free_buf_chain(dev, req_ctx->src, crypt->src_buf); + skcipher_request_complete(req, failed); + break; + } + case CTL_FLAG_GEN_ICV: + ctx = crypto_tfm_ctx(crypt->data.tfm); + dma_pool_free(ctx_pool, crypt->regist_ptr, + crypt->regist_buf->phys_addr); + dma_pool_free(buffer_pool, crypt->regist_buf, crypt->src_buf); + if (atomic_dec_and_test(&ctx->configuring)) + complete(&ctx->completion); + break; + case CTL_FLAG_GEN_REVAES: + ctx = crypto_tfm_ctx(crypt->data.tfm); + *(__be32 *)ctx->decrypt.npe_ctx &= cpu_to_be32(~CIPH_ENCR); + if (atomic_dec_and_test(&ctx->configuring)) + complete(&ctx->completion); + break; + default: + BUG(); + } + crypt->ctl_flags = CTL_FLAG_UNUSED; +} + +static void irqhandler(void *_unused) +{ + tasklet_schedule(&crypto_done_tasklet); +} + +static void crypto_done_action(unsigned long arg) +{ + int i; + + for (i = 0; i < 4; i++) { + dma_addr_t phys = qmgr_get_entry(recv_qid); + if (!phys) + return; + one_packet(phys); + } + tasklet_schedule(&crypto_done_tasklet); +} + +static int init_ixp_crypto(struct device *dev) +{ + struct device_node *np = dev->of_node; + u32 msg[2] = { 0, 0 }; + int ret = -ENODEV; + u32 npe_id; + + dev_info(dev, "probing...\n"); + + /* Locate the NPE and queue manager to use from device tree */ + if (IS_ENABLED(CONFIG_OF) && np) { + struct of_phandle_args queue_spec; + struct of_phandle_args npe_spec; + + ret = of_parse_phandle_with_fixed_args(np, "intel,npe-handle", + 1, 0, &npe_spec); + if (ret) { + dev_err(dev, "no NPE engine specified\n"); + return -ENODEV; + } + npe_id = npe_spec.args[0]; + of_node_put(npe_spec.np); + + ret = of_parse_phandle_with_fixed_args(np, "queue-rx", 1, 0, + &queue_spec); + if (ret) { + dev_err(dev, "no rx queue phandle\n"); + return -ENODEV; + } + recv_qid = queue_spec.args[0]; + of_node_put(queue_spec.np); + + ret = of_parse_phandle_with_fixed_args(np, "queue-txready", 1, 0, + &queue_spec); + if (ret) { + dev_err(dev, "no txready queue phandle\n"); + return -ENODEV; + } + send_qid = queue_spec.args[0]; + of_node_put(queue_spec.np); + } else { + /* + * Hardcoded engine when using platform data, this goes away + * when we switch to using DT only. + */ + npe_id = 2; + send_qid = 29; + recv_qid = 30; + } + + npe_c = npe_request(npe_id); + if (!npe_c) + return ret; + + if (!npe_running(npe_c)) { + ret = npe_load_firmware(npe_c, npe_name(npe_c), dev); + if (ret) + goto npe_release; + if (npe_recv_message(npe_c, msg, "STATUS_MSG")) + goto npe_error; + } else { + if (npe_send_message(npe_c, msg, "STATUS_MSG")) + goto npe_error; + + if (npe_recv_message(npe_c, msg, "STATUS_MSG")) + goto npe_error; + } + + switch ((msg[1] >> 16) & 0xff) { + case 3: + dev_warn(dev, "Firmware of %s lacks AES support\n", npe_name(npe_c)); + support_aes = 0; + break; + case 4: + case 5: + support_aes = 1; + break; + default: + dev_err(dev, "Firmware of %s lacks crypto support\n", npe_name(npe_c)); + ret = -ENODEV; + goto npe_release; + } + /* buffer_pool will also be used to sometimes store the hmac, + * so assure it is large enough + */ + BUILD_BUG_ON(SHA1_DIGEST_SIZE > sizeof(struct buffer_desc)); + buffer_pool = dma_pool_create("buffer", dev, sizeof(struct buffer_desc), + 32, 0); + ret = -ENOMEM; + if (!buffer_pool) + goto err; + + ctx_pool = dma_pool_create("context", dev, NPE_CTX_LEN, 16, 0); + if (!ctx_pool) + goto err; + + ret = qmgr_request_queue(send_qid, NPE_QLEN_TOTAL, 0, 0, + "ixp_crypto:out", NULL); + if (ret) + goto err; + ret = qmgr_request_queue(recv_qid, NPE_QLEN, 0, 0, + "ixp_crypto:in", NULL); + if (ret) { + qmgr_release_queue(send_qid); + goto err; + } + qmgr_set_irq(recv_qid, QUEUE_IRQ_SRC_NOT_EMPTY, irqhandler, NULL); + tasklet_init(&crypto_done_tasklet, crypto_done_action, 0); + + qmgr_enable_irq(recv_qid); + return 0; + +npe_error: + dev_err(dev, "%s not responding\n", npe_name(npe_c)); + ret = -EIO; +err: + dma_pool_destroy(ctx_pool); + dma_pool_destroy(buffer_pool); +npe_release: + npe_release(npe_c); + return ret; +} + +static void release_ixp_crypto(struct device *dev) +{ + qmgr_disable_irq(recv_qid); + tasklet_kill(&crypto_done_tasklet); + + qmgr_release_queue(send_qid); + qmgr_release_queue(recv_qid); + + dma_pool_destroy(ctx_pool); + dma_pool_destroy(buffer_pool); + + npe_release(npe_c); + + if (crypt_virt) + dma_free_coherent(dev, NPE_QLEN * sizeof(struct crypt_ctl), + crypt_virt, crypt_phys); +} + +static void reset_sa_dir(struct ix_sa_dir *dir) +{ + memset(dir->npe_ctx, 0, NPE_CTX_LEN); + dir->npe_ctx_idx = 0; + dir->npe_mode = 0; +} + +static int init_sa_dir(struct ix_sa_dir *dir) +{ + dir->npe_ctx = dma_pool_alloc(ctx_pool, GFP_KERNEL, &dir->npe_ctx_phys); + if (!dir->npe_ctx) + return -ENOMEM; + + reset_sa_dir(dir); + return 0; +} + +static void free_sa_dir(struct ix_sa_dir *dir) +{ + memset(dir->npe_ctx, 0, NPE_CTX_LEN); + dma_pool_free(ctx_pool, dir->npe_ctx, dir->npe_ctx_phys); +} + +static int init_tfm(struct crypto_tfm *tfm) +{ + struct ixp_ctx *ctx = crypto_tfm_ctx(tfm); + int ret; + + atomic_set(&ctx->configuring, 0); + ret = init_sa_dir(&ctx->encrypt); + if (ret) + return ret; + ret = init_sa_dir(&ctx->decrypt); + if (ret) + free_sa_dir(&ctx->encrypt); + + return ret; +} + +static int init_tfm_ablk(struct crypto_skcipher *tfm) +{ + struct crypto_tfm *ctfm = crypto_skcipher_tfm(tfm); + struct ixp_ctx *ctx = crypto_tfm_ctx(ctfm); + const char *name = crypto_tfm_alg_name(ctfm); + + ctx->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fallback_tfm)) { + pr_err("ERROR: Cannot allocate fallback for %s %ld\n", + name, PTR_ERR(ctx->fallback_tfm)); + return PTR_ERR(ctx->fallback_tfm); + } + + pr_info("Fallback for %s is %s\n", + crypto_tfm_alg_driver_name(&tfm->base), + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(ctx->fallback_tfm)) + ); + + crypto_skcipher_set_reqsize(tfm, sizeof(struct ablk_ctx) + crypto_skcipher_reqsize(ctx->fallback_tfm)); + return init_tfm(crypto_skcipher_tfm(tfm)); +} + +static int init_tfm_aead(struct crypto_aead *tfm) +{ + crypto_aead_set_reqsize(tfm, sizeof(struct aead_ctx)); + return init_tfm(crypto_aead_tfm(tfm)); +} + +static void exit_tfm(struct crypto_tfm *tfm) +{ + struct ixp_ctx *ctx = crypto_tfm_ctx(tfm); + + free_sa_dir(&ctx->encrypt); + free_sa_dir(&ctx->decrypt); +} + +static void exit_tfm_ablk(struct crypto_skcipher *tfm) +{ + struct crypto_tfm *ctfm = crypto_skcipher_tfm(tfm); + struct ixp_ctx *ctx = crypto_tfm_ctx(ctfm); + + crypto_free_skcipher(ctx->fallback_tfm); + exit_tfm(crypto_skcipher_tfm(tfm)); +} + +static void exit_tfm_aead(struct crypto_aead *tfm) +{ + exit_tfm(crypto_aead_tfm(tfm)); +} + +static int register_chain_var(struct crypto_tfm *tfm, u8 xpad, u32 target, + int init_len, u32 ctx_addr, const u8 *key, + int key_len) +{ + struct ixp_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypt_ctl *crypt; + struct buffer_desc *buf; + int i; + u8 *pad; + dma_addr_t pad_phys, buf_phys; + + BUILD_BUG_ON(NPE_CTX_LEN < HMAC_PAD_BLOCKLEN); + pad = dma_pool_alloc(ctx_pool, GFP_KERNEL, &pad_phys); + if (!pad) + return -ENOMEM; + buf = dma_pool_alloc(buffer_pool, GFP_KERNEL, &buf_phys); + if (!buf) { + dma_pool_free(ctx_pool, pad, pad_phys); + return -ENOMEM; + } + crypt = get_crypt_desc_emerg(); + if (!crypt) { + dma_pool_free(ctx_pool, pad, pad_phys); + dma_pool_free(buffer_pool, buf, buf_phys); + return -EAGAIN; + } + + memcpy(pad, key, key_len); + memset(pad + key_len, 0, HMAC_PAD_BLOCKLEN - key_len); + for (i = 0; i < HMAC_PAD_BLOCKLEN; i++) + pad[i] ^= xpad; + + crypt->data.tfm = tfm; + crypt->regist_ptr = pad; + crypt->regist_buf = buf; + + crypt->auth_offs = 0; + crypt->auth_len = HMAC_PAD_BLOCKLEN; + crypt->crypto_ctx = ctx_addr; + crypt->src_buf = buf_phys; + crypt->icv_rev_aes = target; + crypt->mode = NPE_OP_HASH_GEN_ICV; + crypt->init_len = init_len; + crypt->ctl_flags |= CTL_FLAG_GEN_ICV; + + buf->next = NULL; + buf->buf_len = HMAC_PAD_BLOCKLEN; + buf->pkt_len = 0; + buf->phys_addr = pad_phys; + + atomic_inc(&ctx->configuring); + qmgr_put_entry(send_qid, crypt_virt2phys(crypt)); + BUG_ON(qmgr_stat_overflow(send_qid)); + return 0; +} + +static int setup_auth(struct crypto_tfm *tfm, int encrypt, unsigned int authsize, + const u8 *key, int key_len, unsigned int digest_len) +{ + u32 itarget, otarget, npe_ctx_addr; + unsigned char *cinfo; + int init_len, ret = 0; + u32 cfgword; + struct ix_sa_dir *dir; + struct ixp_ctx *ctx = crypto_tfm_ctx(tfm); + const struct ix_hash_algo *algo; + + dir = encrypt ? &ctx->encrypt : &ctx->decrypt; + cinfo = dir->npe_ctx + dir->npe_ctx_idx; + algo = ix_hash(tfm); + + /* write cfg word to cryptinfo */ + cfgword = algo->cfgword | (authsize << 6); /* (authsize/4) << 8 */ +#ifndef __ARMEB__ + cfgword ^= 0xAA000000; /* change the "byte swap" flags */ +#endif + *(__be32 *)cinfo = cpu_to_be32(cfgword); + cinfo += sizeof(cfgword); + + /* write ICV to cryptinfo */ + memcpy(cinfo, algo->icv, digest_len); + cinfo += digest_len; + + itarget = dir->npe_ctx_phys + dir->npe_ctx_idx + + sizeof(algo->cfgword); + otarget = itarget + digest_len; + init_len = cinfo - (dir->npe_ctx + dir->npe_ctx_idx); + npe_ctx_addr = dir->npe_ctx_phys + dir->npe_ctx_idx; + + dir->npe_ctx_idx += init_len; + dir->npe_mode |= NPE_OP_HASH_ENABLE; + + if (!encrypt) + dir->npe_mode |= NPE_OP_HASH_VERIFY; + + ret = register_chain_var(tfm, HMAC_OPAD_VALUE, otarget, + init_len, npe_ctx_addr, key, key_len); + if (ret) + return ret; + return register_chain_var(tfm, HMAC_IPAD_VALUE, itarget, + init_len, npe_ctx_addr, key, key_len); +} + +static int gen_rev_aes_key(struct crypto_tfm *tfm) +{ + struct crypt_ctl *crypt; + struct ixp_ctx *ctx = crypto_tfm_ctx(tfm); + struct ix_sa_dir *dir = &ctx->decrypt; + + crypt = get_crypt_desc_emerg(); + if (!crypt) + return -EAGAIN; + + *(__be32 *)dir->npe_ctx |= cpu_to_be32(CIPH_ENCR); + + crypt->data.tfm = tfm; + crypt->crypt_offs = 0; + crypt->crypt_len = AES_BLOCK128; + crypt->src_buf = 0; + crypt->crypto_ctx = dir->npe_ctx_phys; + crypt->icv_rev_aes = dir->npe_ctx_phys + sizeof(u32); + crypt->mode = NPE_OP_ENC_GEN_KEY; + crypt->init_len = dir->npe_ctx_idx; + crypt->ctl_flags |= CTL_FLAG_GEN_REVAES; + + atomic_inc(&ctx->configuring); + qmgr_put_entry(send_qid, crypt_virt2phys(crypt)); + BUG_ON(qmgr_stat_overflow(send_qid)); + return 0; +} + +static int setup_cipher(struct crypto_tfm *tfm, int encrypt, const u8 *key, + int key_len) +{ + u8 *cinfo; + u32 cipher_cfg; + u32 keylen_cfg = 0; + struct ix_sa_dir *dir; + struct ixp_ctx *ctx = crypto_tfm_ctx(tfm); + int err; + + dir = encrypt ? &ctx->encrypt : &ctx->decrypt; + cinfo = dir->npe_ctx; + + if (encrypt) { + cipher_cfg = cipher_cfg_enc(tfm); + dir->npe_mode |= NPE_OP_CRYPT_ENCRYPT; + } else { + cipher_cfg = cipher_cfg_dec(tfm); + } + if (cipher_cfg & MOD_AES) { + switch (key_len) { + case 16: + keylen_cfg = MOD_AES128; + break; + case 24: + keylen_cfg = MOD_AES192; + break; + case 32: + keylen_cfg = MOD_AES256; + break; + default: + return -EINVAL; + } + cipher_cfg |= keylen_cfg; + } else { + err = crypto_des_verify_key(tfm, key); + if (err) + return err; + } + /* write cfg word to cryptinfo */ + *(__be32 *)cinfo = cpu_to_be32(cipher_cfg); + cinfo += sizeof(cipher_cfg); + + /* write cipher key to cryptinfo */ + memcpy(cinfo, key, key_len); + /* NPE wants keylen set to DES3_EDE_KEY_SIZE even for single DES */ + if (key_len < DES3_EDE_KEY_SIZE && !(cipher_cfg & MOD_AES)) { + memset(cinfo + key_len, 0, DES3_EDE_KEY_SIZE - key_len); + key_len = DES3_EDE_KEY_SIZE; + } + dir->npe_ctx_idx = sizeof(cipher_cfg) + key_len; + dir->npe_mode |= NPE_OP_CRYPT_ENABLE; + if ((cipher_cfg & MOD_AES) && !encrypt) + return gen_rev_aes_key(tfm); + + return 0; +} + +static struct buffer_desc *chainup_buffers(struct device *dev, + struct scatterlist *sg, unsigned int nbytes, + struct buffer_desc *buf, gfp_t flags, + enum dma_data_direction dir) +{ + for (; nbytes > 0; sg = sg_next(sg)) { + unsigned int len = min(nbytes, sg->length); + struct buffer_desc *next_buf; + dma_addr_t next_buf_phys; + void *ptr; + + nbytes -= len; + ptr = sg_virt(sg); + next_buf = dma_pool_alloc(buffer_pool, flags, &next_buf_phys); + if (!next_buf) { + buf->next = NULL; + buf->phys_next = 0; + return NULL; + } + sg_dma_address(sg) = dma_map_single(dev, ptr, len, dir); + buf->next = next_buf; + buf->phys_next = next_buf_phys; + buf = next_buf; + + buf->phys_addr = sg_dma_address(sg); + buf->buf_len = len; + buf->dir = dir; + } + buf->next = NULL; + buf->phys_next = 0; + return buf; +} + +static int ablk_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int key_len) +{ + struct ixp_ctx *ctx = crypto_skcipher_ctx(tfm); + int ret; + + init_completion(&ctx->completion); + atomic_inc(&ctx->configuring); + + reset_sa_dir(&ctx->encrypt); + reset_sa_dir(&ctx->decrypt); + + ctx->encrypt.npe_mode = NPE_OP_HMAC_DISABLE; + ctx->decrypt.npe_mode = NPE_OP_HMAC_DISABLE; + + ret = setup_cipher(&tfm->base, 0, key, key_len); + if (ret) + goto out; + ret = setup_cipher(&tfm->base, 1, key, key_len); +out: + if (!atomic_dec_and_test(&ctx->configuring)) + wait_for_completion(&ctx->completion); + if (ret) + return ret; + crypto_skcipher_clear_flags(ctx->fallback_tfm, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(ctx->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); + + return crypto_skcipher_setkey(ctx->fallback_tfm, key, key_len); +} + +static int ablk_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int key_len) +{ + return verify_skcipher_des3_key(tfm, key) ?: + ablk_setkey(tfm, key, key_len); +} + +static int ablk_rfc3686_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int key_len) +{ + struct ixp_ctx *ctx = crypto_skcipher_ctx(tfm); + + /* the nonce is stored in bytes at end of key */ + if (key_len < CTR_RFC3686_NONCE_SIZE) + return -EINVAL; + + memcpy(ctx->nonce, key + (key_len - CTR_RFC3686_NONCE_SIZE), + CTR_RFC3686_NONCE_SIZE); + + key_len -= CTR_RFC3686_NONCE_SIZE; + return ablk_setkey(tfm, key, key_len); +} + +static int ixp4xx_cipher_fallback(struct skcipher_request *areq, int encrypt) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct ixp_ctx *op = crypto_skcipher_ctx(tfm); + struct ablk_ctx *rctx = skcipher_request_ctx(areq); + int err; + + skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm); + skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags, + areq->base.complete, areq->base.data); + skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst, + areq->cryptlen, areq->iv); + if (encrypt) + err = crypto_skcipher_encrypt(&rctx->fallback_req); + else + err = crypto_skcipher_decrypt(&rctx->fallback_req); + return err; +} + +static int ablk_perform(struct skcipher_request *req, int encrypt) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ixp_ctx *ctx = crypto_skcipher_ctx(tfm); + unsigned int ivsize = crypto_skcipher_ivsize(tfm); + struct ix_sa_dir *dir; + struct crypt_ctl *crypt; + unsigned int nbytes = req->cryptlen; + enum dma_data_direction src_direction = DMA_BIDIRECTIONAL; + struct ablk_ctx *req_ctx = skcipher_request_ctx(req); + struct buffer_desc *buf, src_hook; + struct device *dev = &pdev->dev; + unsigned int offset; + gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + if (sg_nents(req->src) > 1 || sg_nents(req->dst) > 1) + return ixp4xx_cipher_fallback(req, encrypt); + + if (qmgr_stat_full(send_qid)) + return -EAGAIN; + if (atomic_read(&ctx->configuring)) + return -EAGAIN; + + dir = encrypt ? &ctx->encrypt : &ctx->decrypt; + req_ctx->encrypt = encrypt; + + crypt = get_crypt_desc(); + if (!crypt) + return -ENOMEM; + + crypt->data.ablk_req = req; + crypt->crypto_ctx = dir->npe_ctx_phys; + crypt->mode = dir->npe_mode; + crypt->init_len = dir->npe_ctx_idx; + + crypt->crypt_offs = 0; + crypt->crypt_len = nbytes; + + BUG_ON(ivsize && !req->iv); + memcpy(crypt->iv, req->iv, ivsize); + if (ivsize > 0 && !encrypt) { + offset = req->cryptlen - ivsize; + scatterwalk_map_and_copy(req_ctx->iv, req->src, offset, ivsize, 0); + } + if (req->src != req->dst) { + struct buffer_desc dst_hook; + + crypt->mode |= NPE_OP_NOT_IN_PLACE; + /* This was never tested by Intel + * for more than one dst buffer, I think. */ + req_ctx->dst = NULL; + buf = chainup_buffers(dev, req->dst, nbytes, &dst_hook, + flags, DMA_FROM_DEVICE); + req_ctx->dst = dst_hook.next; + crypt->dst_buf = dst_hook.phys_next; + if (!buf) + goto free_buf_dest; + src_direction = DMA_TO_DEVICE; + } else { + req_ctx->dst = NULL; + } + req_ctx->src = NULL; + buf = chainup_buffers(dev, req->src, nbytes, &src_hook, flags, + src_direction); + req_ctx->src = src_hook.next; + crypt->src_buf = src_hook.phys_next; + if (!buf) + goto free_buf_src; + + crypt->ctl_flags |= CTL_FLAG_PERFORM_ABLK; + qmgr_put_entry(send_qid, crypt_virt2phys(crypt)); + BUG_ON(qmgr_stat_overflow(send_qid)); + return -EINPROGRESS; + +free_buf_src: + free_buf_chain(dev, req_ctx->src, crypt->src_buf); +free_buf_dest: + if (req->src != req->dst) + free_buf_chain(dev, req_ctx->dst, crypt->dst_buf); + + crypt->ctl_flags = CTL_FLAG_UNUSED; + return -ENOMEM; +} + +static int ablk_encrypt(struct skcipher_request *req) +{ + return ablk_perform(req, 1); +} + +static int ablk_decrypt(struct skcipher_request *req) +{ + return ablk_perform(req, 0); +} + +static int ablk_rfc3686_crypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ixp_ctx *ctx = crypto_skcipher_ctx(tfm); + u8 iv[CTR_RFC3686_BLOCK_SIZE]; + u8 *info = req->iv; + int ret; + + /* set up counter block */ + memcpy(iv, ctx->nonce, CTR_RFC3686_NONCE_SIZE); + memcpy(iv + CTR_RFC3686_NONCE_SIZE, info, CTR_RFC3686_IV_SIZE); + + /* initialize counter portion of counter block */ + *(__be32 *)(iv + CTR_RFC3686_NONCE_SIZE + CTR_RFC3686_IV_SIZE) = + cpu_to_be32(1); + + req->iv = iv; + ret = ablk_perform(req, 1); + req->iv = info; + return ret; +} + +static int aead_perform(struct aead_request *req, int encrypt, + int cryptoffset, int eff_cryptlen, u8 *iv) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct ixp_ctx *ctx = crypto_aead_ctx(tfm); + unsigned int ivsize = crypto_aead_ivsize(tfm); + unsigned int authsize = crypto_aead_authsize(tfm); + struct ix_sa_dir *dir; + struct crypt_ctl *crypt; + unsigned int cryptlen; + struct buffer_desc *buf, src_hook; + struct aead_ctx *req_ctx = aead_request_ctx(req); + struct device *dev = &pdev->dev; + gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + enum dma_data_direction src_direction = DMA_BIDIRECTIONAL; + unsigned int lastlen; + + if (qmgr_stat_full(send_qid)) + return -EAGAIN; + if (atomic_read(&ctx->configuring)) + return -EAGAIN; + + if (encrypt) { + dir = &ctx->encrypt; + cryptlen = req->cryptlen; + } else { + dir = &ctx->decrypt; + /* req->cryptlen includes the authsize when decrypting */ + cryptlen = req->cryptlen - authsize; + eff_cryptlen -= authsize; + } + crypt = get_crypt_desc(); + if (!crypt) + return -ENOMEM; + + crypt->data.aead_req = req; + crypt->crypto_ctx = dir->npe_ctx_phys; + crypt->mode = dir->npe_mode; + crypt->init_len = dir->npe_ctx_idx; + + crypt->crypt_offs = cryptoffset; + crypt->crypt_len = eff_cryptlen; + + crypt->auth_offs = 0; + crypt->auth_len = req->assoclen + cryptlen; + BUG_ON(ivsize && !req->iv); + memcpy(crypt->iv, req->iv, ivsize); + + buf = chainup_buffers(dev, req->src, crypt->auth_len, + &src_hook, flags, src_direction); + req_ctx->src = src_hook.next; + crypt->src_buf = src_hook.phys_next; + if (!buf) + goto free_buf_src; + + lastlen = buf->buf_len; + if (lastlen >= authsize) + crypt->icv_rev_aes = buf->phys_addr + + buf->buf_len - authsize; + + req_ctx->dst = NULL; + + if (req->src != req->dst) { + struct buffer_desc dst_hook; + + crypt->mode |= NPE_OP_NOT_IN_PLACE; + src_direction = DMA_TO_DEVICE; + + buf = chainup_buffers(dev, req->dst, crypt->auth_len, + &dst_hook, flags, DMA_FROM_DEVICE); + req_ctx->dst = dst_hook.next; + crypt->dst_buf = dst_hook.phys_next; + + if (!buf) + goto free_buf_dst; + + if (encrypt) { + lastlen = buf->buf_len; + if (lastlen >= authsize) + crypt->icv_rev_aes = buf->phys_addr + + buf->buf_len - authsize; + } + } + + if (unlikely(lastlen < authsize)) { + dma_addr_t dma; + /* The 12 hmac bytes are scattered, + * we need to copy them into a safe buffer */ + req_ctx->hmac_virt = dma_pool_alloc(buffer_pool, flags, &dma); + if (unlikely(!req_ctx->hmac_virt)) + goto free_buf_dst; + crypt->icv_rev_aes = dma; + if (!encrypt) { + scatterwalk_map_and_copy(req_ctx->hmac_virt, + req->src, cryptlen, authsize, 0); + } + req_ctx->encrypt = encrypt; + } else { + req_ctx->hmac_virt = NULL; + } + + crypt->ctl_flags |= CTL_FLAG_PERFORM_AEAD; + qmgr_put_entry(send_qid, crypt_virt2phys(crypt)); + BUG_ON(qmgr_stat_overflow(send_qid)); + return -EINPROGRESS; + +free_buf_dst: + free_buf_chain(dev, req_ctx->dst, crypt->dst_buf); +free_buf_src: + free_buf_chain(dev, req_ctx->src, crypt->src_buf); + crypt->ctl_flags = CTL_FLAG_UNUSED; + return -ENOMEM; +} + +static int aead_setup(struct crypto_aead *tfm, unsigned int authsize) +{ + struct ixp_ctx *ctx = crypto_aead_ctx(tfm); + unsigned int digest_len = crypto_aead_maxauthsize(tfm); + int ret; + + if (!ctx->enckey_len && !ctx->authkey_len) + return 0; + init_completion(&ctx->completion); + atomic_inc(&ctx->configuring); + + reset_sa_dir(&ctx->encrypt); + reset_sa_dir(&ctx->decrypt); + + ret = setup_cipher(&tfm->base, 0, ctx->enckey, ctx->enckey_len); + if (ret) + goto out; + ret = setup_cipher(&tfm->base, 1, ctx->enckey, ctx->enckey_len); + if (ret) + goto out; + ret = setup_auth(&tfm->base, 0, authsize, ctx->authkey, + ctx->authkey_len, digest_len); + if (ret) + goto out; + ret = setup_auth(&tfm->base, 1, authsize, ctx->authkey, + ctx->authkey_len, digest_len); +out: + if (!atomic_dec_and_test(&ctx->configuring)) + wait_for_completion(&ctx->completion); + return ret; +} + +static int aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) +{ + int max = crypto_aead_maxauthsize(tfm) >> 2; + + if ((authsize >> 2) < 1 || (authsize >> 2) > max || (authsize & 3)) + return -EINVAL; + return aead_setup(tfm, authsize); +} + +static int aead_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct ixp_ctx *ctx = crypto_aead_ctx(tfm); + struct crypto_authenc_keys keys; + + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) + goto badkey; + + if (keys.authkeylen > sizeof(ctx->authkey)) + goto badkey; + + if (keys.enckeylen > sizeof(ctx->enckey)) + goto badkey; + + memcpy(ctx->authkey, keys.authkey, keys.authkeylen); + memcpy(ctx->enckey, keys.enckey, keys.enckeylen); + ctx->authkey_len = keys.authkeylen; + ctx->enckey_len = keys.enckeylen; + + memzero_explicit(&keys, sizeof(keys)); + return aead_setup(tfm, crypto_aead_authsize(tfm)); +badkey: + memzero_explicit(&keys, sizeof(keys)); + return -EINVAL; +} + +static int des3_aead_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct ixp_ctx *ctx = crypto_aead_ctx(tfm); + struct crypto_authenc_keys keys; + int err; + + err = crypto_authenc_extractkeys(&keys, key, keylen); + if (unlikely(err)) + goto badkey; + + err = -EINVAL; + if (keys.authkeylen > sizeof(ctx->authkey)) + goto badkey; + + err = verify_aead_des3_key(tfm, keys.enckey, keys.enckeylen); + if (err) + goto badkey; + + memcpy(ctx->authkey, keys.authkey, keys.authkeylen); + memcpy(ctx->enckey, keys.enckey, keys.enckeylen); + ctx->authkey_len = keys.authkeylen; + ctx->enckey_len = keys.enckeylen; + + memzero_explicit(&keys, sizeof(keys)); + return aead_setup(tfm, crypto_aead_authsize(tfm)); +badkey: + memzero_explicit(&keys, sizeof(keys)); + return err; +} + +static int aead_encrypt(struct aead_request *req) +{ + return aead_perform(req, 1, req->assoclen, req->cryptlen, req->iv); +} + +static int aead_decrypt(struct aead_request *req) +{ + return aead_perform(req, 0, req->assoclen, req->cryptlen, req->iv); +} + +static struct ixp_alg ixp4xx_algos[] = { +{ + .crypto = { + .base.cra_name = "cbc(des)", + .base.cra_blocksize = DES_BLOCK_SIZE, + + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + }, + .cfg_enc = CIPH_ENCR | MOD_DES | MOD_CBC_ENC | KEYLEN_192, + .cfg_dec = CIPH_DECR | MOD_DES | MOD_CBC_DEC | KEYLEN_192, + +}, { + .crypto = { + .base.cra_name = "ecb(des)", + .base.cra_blocksize = DES_BLOCK_SIZE, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + }, + .cfg_enc = CIPH_ENCR | MOD_DES | MOD_ECB | KEYLEN_192, + .cfg_dec = CIPH_DECR | MOD_DES | MOD_ECB | KEYLEN_192, +}, { + .crypto = { + .base.cra_name = "cbc(des3_ede)", + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setkey = ablk_des3_setkey, + }, + .cfg_enc = CIPH_ENCR | MOD_3DES | MOD_CBC_ENC | KEYLEN_192, + .cfg_dec = CIPH_DECR | MOD_3DES | MOD_CBC_DEC | KEYLEN_192, +}, { + .crypto = { + .base.cra_name = "ecb(des3_ede)", + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = ablk_des3_setkey, + }, + .cfg_enc = CIPH_ENCR | MOD_3DES | MOD_ECB | KEYLEN_192, + .cfg_dec = CIPH_DECR | MOD_3DES | MOD_ECB | KEYLEN_192, +}, { + .crypto = { + .base.cra_name = "cbc(aes)", + .base.cra_blocksize = AES_BLOCK_SIZE, + + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CBC_ENC, + .cfg_dec = CIPH_DECR | MOD_AES | MOD_CBC_DEC, +}, { + .crypto = { + .base.cra_name = "ecb(aes)", + .base.cra_blocksize = AES_BLOCK_SIZE, + + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + }, + .cfg_enc = CIPH_ENCR | MOD_AES | MOD_ECB, + .cfg_dec = CIPH_DECR | MOD_AES | MOD_ECB, +}, { + .crypto = { + .base.cra_name = "ctr(aes)", + .base.cra_blocksize = 1, + + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CTR, + .cfg_dec = CIPH_ENCR | MOD_AES | MOD_CTR, +}, { + .crypto = { + .base.cra_name = "rfc3686(ctr(aes))", + .base.cra_blocksize = 1, + + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = ablk_rfc3686_setkey, + .encrypt = ablk_rfc3686_crypt, + .decrypt = ablk_rfc3686_crypt, + }, + .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CTR, + .cfg_dec = CIPH_ENCR | MOD_AES | MOD_CTR, +} }; + +static struct ixp_aead_alg ixp4xx_aeads[] = { +{ + .crypto = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des))", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .hash = &hash_alg_md5, + .cfg_enc = CIPH_ENCR | MOD_DES | MOD_CBC_ENC | KEYLEN_192, + .cfg_dec = CIPH_DECR | MOD_DES | MOD_CBC_DEC | KEYLEN_192, +}, { + .crypto = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des3_ede))", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + .setkey = des3_aead_setkey, + }, + .hash = &hash_alg_md5, + .cfg_enc = CIPH_ENCR | MOD_3DES | MOD_CBC_ENC | KEYLEN_192, + .cfg_dec = CIPH_DECR | MOD_3DES | MOD_CBC_DEC | KEYLEN_192, +}, { + .crypto = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des))", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .hash = &hash_alg_sha1, + .cfg_enc = CIPH_ENCR | MOD_DES | MOD_CBC_ENC | KEYLEN_192, + .cfg_dec = CIPH_DECR | MOD_DES | MOD_CBC_DEC | KEYLEN_192, +}, { + .crypto = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + .setkey = des3_aead_setkey, + }, + .hash = &hash_alg_sha1, + .cfg_enc = CIPH_ENCR | MOD_3DES | MOD_CBC_ENC | KEYLEN_192, + .cfg_dec = CIPH_DECR | MOD_3DES | MOD_CBC_DEC | KEYLEN_192, +}, { + .crypto = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(aes))", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .hash = &hash_alg_md5, + .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CBC_ENC, + .cfg_dec = CIPH_DECR | MOD_AES | MOD_CBC_DEC, +}, { + .crypto = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .hash = &hash_alg_sha1, + .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CBC_ENC, + .cfg_dec = CIPH_DECR | MOD_AES | MOD_CBC_DEC, +} }; + +#define IXP_POSTFIX "-ixp4xx" + +static int ixp_crypto_probe(struct platform_device *_pdev) +{ + struct device *dev = &_pdev->dev; + int num = ARRAY_SIZE(ixp4xx_algos); + int i, err; + + pdev = _pdev; + + err = init_ixp_crypto(dev); + if (err) + return err; + + for (i = 0; i < num; i++) { + struct skcipher_alg *cra = &ixp4xx_algos[i].crypto; + + if (snprintf(cra->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, + "%s"IXP_POSTFIX, cra->base.cra_name) >= + CRYPTO_MAX_ALG_NAME) + continue; + if (!support_aes && (ixp4xx_algos[i].cfg_enc & MOD_AES)) + continue; + + /* block ciphers */ + cra->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_NEED_FALLBACK; + if (!cra->setkey) + cra->setkey = ablk_setkey; + if (!cra->encrypt) + cra->encrypt = ablk_encrypt; + if (!cra->decrypt) + cra->decrypt = ablk_decrypt; + cra->init = init_tfm_ablk; + cra->exit = exit_tfm_ablk; + + cra->base.cra_ctxsize = sizeof(struct ixp_ctx); + cra->base.cra_module = THIS_MODULE; + cra->base.cra_alignmask = 3; + cra->base.cra_priority = 300; + if (crypto_register_skcipher(cra)) + dev_err(&pdev->dev, "Failed to register '%s'\n", + cra->base.cra_name); + else + ixp4xx_algos[i].registered = 1; + } + + for (i = 0; i < ARRAY_SIZE(ixp4xx_aeads); i++) { + struct aead_alg *cra = &ixp4xx_aeads[i].crypto; + + if (snprintf(cra->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, + "%s"IXP_POSTFIX, cra->base.cra_name) >= + CRYPTO_MAX_ALG_NAME) + continue; + if (!support_aes && (ixp4xx_algos[i].cfg_enc & MOD_AES)) + continue; + + /* authenc */ + cra->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY; + cra->setkey = cra->setkey ?: aead_setkey; + cra->setauthsize = aead_setauthsize; + cra->encrypt = aead_encrypt; + cra->decrypt = aead_decrypt; + cra->init = init_tfm_aead; + cra->exit = exit_tfm_aead; + + cra->base.cra_ctxsize = sizeof(struct ixp_ctx); + cra->base.cra_module = THIS_MODULE; + cra->base.cra_alignmask = 3; + cra->base.cra_priority = 300; + + if (crypto_register_aead(cra)) + dev_err(&pdev->dev, "Failed to register '%s'\n", + cra->base.cra_driver_name); + else + ixp4xx_aeads[i].registered = 1; + } + return 0; +} + +static void ixp_crypto_remove(struct platform_device *pdev) +{ + int num = ARRAY_SIZE(ixp4xx_algos); + int i; + + for (i = 0; i < ARRAY_SIZE(ixp4xx_aeads); i++) { + if (ixp4xx_aeads[i].registered) + crypto_unregister_aead(&ixp4xx_aeads[i].crypto); + } + + for (i = 0; i < num; i++) { + if (ixp4xx_algos[i].registered) + crypto_unregister_skcipher(&ixp4xx_algos[i].crypto); + } + release_ixp_crypto(&pdev->dev); +} +static const struct of_device_id ixp4xx_crypto_of_match[] = { + { + .compatible = "intel,ixp4xx-crypto", + }, + {}, +}; +MODULE_DEVICE_TABLE(of, ixp4xx_crypto_of_match); + +static struct platform_driver ixp_crypto_driver = { + .probe = ixp_crypto_probe, + .remove = ixp_crypto_remove, + .driver = { + .name = "ixp4xx_crypto", + .of_match_table = ixp4xx_crypto_of_match, + }, +}; +module_platform_driver(ixp_crypto_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Christian Hohnstaedt "); +MODULE_DESCRIPTION("IXP4xx hardware crypto"); + diff --git a/drivers/crypto/intel/keembay/Kconfig b/drivers/crypto/intel/keembay/Kconfig new file mode 100644 index 000000000..1cd62f9c3 --- /dev/null +++ b/drivers/crypto/intel/keembay/Kconfig @@ -0,0 +1,90 @@ +config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4 + tristate "Support for Intel Keem Bay OCS AES/SM4 HW acceleration" + depends on HAS_IOMEM + depends on ARCH_KEEMBAY || COMPILE_TEST + select CRYPTO_SKCIPHER + select CRYPTO_AEAD + select CRYPTO_ENGINE + help + Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) AES and + SM4 cipher hardware acceleration for use with Crypto API. + + Provides HW acceleration for the following transformations: + cbc(aes), ctr(aes), ccm(aes), gcm(aes), cbc(sm4), ctr(sm4), ccm(sm4) + and gcm(sm4). + + Optionally, support for the following transformations can also be + enabled: ecb(aes), cts(cbc(aes)), ecb(sm4) and cts(cbc(sm4)). + +config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB + bool "Support for Intel Keem Bay OCS AES/SM4 ECB HW acceleration" + depends on CRYPTO_DEV_KEEMBAY_OCS_AES_SM4 + help + Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) + AES/SM4 ECB mode hardware acceleration for use with Crypto API. + + Provides OCS version of ecb(aes) and ecb(sm4) + + Intel does not recommend use of ECB mode with AES/SM4. + +config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS + bool "Support for Intel Keem Bay OCS AES/SM4 CTS HW acceleration" + depends on CRYPTO_DEV_KEEMBAY_OCS_AES_SM4 + help + Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) + AES/SM4 CBC with CTS mode hardware acceleration for use with + Crypto API. + + Provides OCS version of cts(cbc(aes)) and cts(cbc(sm4)). + + Intel does not recommend use of CTS mode with AES/SM4. + +config CRYPTO_DEV_KEEMBAY_OCS_ECC + tristate "Support for Intel Keem Bay OCS ECC HW acceleration" + depends on ARCH_KEEMBAY || COMPILE_TEST + depends on OF + depends on HAS_IOMEM + select CRYPTO_ECDH + select CRYPTO_ENGINE + help + Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) + Elliptic Curve Cryptography (ECC) hardware acceleration for use with + Crypto API. + + Provides OCS acceleration for ECDH-256 and ECDH-384. + + Say Y or M if you are compiling for the Intel Keem Bay SoC. The + module will be called keembay-ocs-ecc. + + If unsure, say N. + +config CRYPTO_DEV_KEEMBAY_OCS_HCU + tristate "Support for Intel Keem Bay OCS HCU HW acceleration" + select CRYPTO_HASH + select CRYPTO_ENGINE + depends on HAS_IOMEM + depends on ARCH_KEEMBAY || COMPILE_TEST + depends on OF + help + Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) Hash + Control Unit (HCU) hardware acceleration for use with Crypto API. + + Provides OCS HCU hardware acceleration of sha256, sha384, sha512, and + sm3, as well as the HMAC variant of these algorithms. + + Say Y or M if you're building for the Intel Keem Bay SoC. If compiled + as a module, the module will be called keembay-ocs-hcu. + + If unsure, say N. + +config CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 + bool "Enable sha224 and hmac(sha224) support in Intel Keem Bay OCS HCU" + depends on CRYPTO_DEV_KEEMBAY_OCS_HCU + help + Enables support for sha224 and hmac(sha224) algorithms in the Intel + Keem Bay OCS HCU driver. Intel recommends not to use these + algorithms. + + Provides OCS HCU hardware acceleration of sha224 and hmac(224). + + If unsure, say N. diff --git a/drivers/crypto/intel/keembay/Makefile b/drivers/crypto/intel/keembay/Makefile new file mode 100644 index 000000000..7c12c3c13 --- /dev/null +++ b/drivers/crypto/intel/keembay/Makefile @@ -0,0 +1,10 @@ +# +# Makefile for Intel Keem Bay OCS Crypto API Linux drivers +# +obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4) += keembay-ocs-aes.o +keembay-ocs-aes-objs := keembay-ocs-aes-core.o ocs-aes.o + +obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_ECC) += keembay-ocs-ecc.o + +obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU) += keembay-ocs-hcu.o +keembay-ocs-hcu-objs := keembay-ocs-hcu-core.o ocs-hcu.o diff --git a/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c b/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c new file mode 100644 index 000000000..660830c36 --- /dev/null +++ b/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c @@ -0,0 +1,1689 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel Keem Bay OCS AES Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ocs-aes.h" + +#define KMB_OCS_PRIORITY 350 +#define DRV_NAME "keembay-ocs-aes" + +#define OCS_AES_MIN_KEY_SIZE 16 +#define OCS_AES_MAX_KEY_SIZE 32 +#define OCS_AES_KEYSIZE_128 16 +#define OCS_AES_KEYSIZE_192 24 +#define OCS_AES_KEYSIZE_256 32 +#define OCS_SM4_KEY_SIZE 16 + +/** + * struct ocs_aes_tctx - OCS AES Transform context + * @aes_dev: The OCS AES device. + * @key: AES/SM4 key. + * @key_len: The length (in bytes) of @key. + * @cipher: OCS cipher to use (either AES or SM4). + * @sw_cipher: The cipher to use as fallback. + * @use_fallback: Whether or not fallback cipher should be used. + */ +struct ocs_aes_tctx { + struct ocs_aes_dev *aes_dev; + u8 key[OCS_AES_KEYSIZE_256]; + unsigned int key_len; + enum ocs_cipher cipher; + union { + struct crypto_sync_skcipher *sk; + struct crypto_aead *aead; + } sw_cipher; + bool use_fallback; +}; + +/** + * struct ocs_aes_rctx - OCS AES Request context. + * @instruction: Instruction to be executed (encrypt / decrypt). + * @mode: Mode to use (ECB, CBC, CTR, CCm, GCM, CTS) + * @src_nents: Number of source SG entries. + * @dst_nents: Number of destination SG entries. + * @src_dma_count: The number of DMA-mapped entries of the source SG. + * @dst_dma_count: The number of DMA-mapped entries of the destination SG. + * @in_place: Whether or not this is an in place request, i.e., + * src_sg == dst_sg. + * @src_dll: OCS DMA linked list for input data. + * @dst_dll: OCS DMA linked list for output data. + * @last_ct_blk: Buffer to hold last cipher text block (only used in CBC + * mode). + * @cts_swap: Whether or not CTS swap must be performed. + * @aad_src_dll: OCS DMA linked list for input AAD data. + * @aad_dst_dll: OCS DMA linked list for output AAD data. + * @in_tag: Buffer to hold input encrypted tag (only used for + * CCM/GCM decrypt). + * @out_tag: Buffer to hold output encrypted / decrypted tag (only + * used for GCM encrypt / decrypt). + */ +struct ocs_aes_rctx { + /* Fields common across all modes. */ + enum ocs_instruction instruction; + enum ocs_mode mode; + int src_nents; + int dst_nents; + int src_dma_count; + int dst_dma_count; + bool in_place; + struct ocs_dll_desc src_dll; + struct ocs_dll_desc dst_dll; + + /* CBC specific */ + u8 last_ct_blk[AES_BLOCK_SIZE]; + + /* CTS specific */ + int cts_swap; + + /* CCM/GCM specific */ + struct ocs_dll_desc aad_src_dll; + struct ocs_dll_desc aad_dst_dll; + u8 in_tag[AES_BLOCK_SIZE]; + + /* GCM specific */ + u8 out_tag[AES_BLOCK_SIZE]; +}; + +/* Driver data. */ +struct ocs_aes_drv { + struct list_head dev_list; + spinlock_t lock; /* Protects dev_list. */ +}; + +static struct ocs_aes_drv ocs_aes = { + .dev_list = LIST_HEAD_INIT(ocs_aes.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(ocs_aes.lock), +}; + +static struct ocs_aes_dev *kmb_ocs_aes_find_dev(struct ocs_aes_tctx *tctx) +{ + struct ocs_aes_dev *aes_dev; + + spin_lock(&ocs_aes.lock); + + if (tctx->aes_dev) { + aes_dev = tctx->aes_dev; + goto exit; + } + + /* Only a single OCS device available */ + aes_dev = list_first_entry(&ocs_aes.dev_list, struct ocs_aes_dev, list); + tctx->aes_dev = aes_dev; + +exit: + spin_unlock(&ocs_aes.lock); + + return aes_dev; +} + +/* + * Ensure key is 128-bit or 256-bit for AES or 128-bit for SM4 and an actual + * key is being passed in. + * + * Return: 0 if key is valid, -EINVAL otherwise. + */ +static int check_key(const u8 *in_key, size_t key_len, enum ocs_cipher cipher) +{ + if (!in_key) + return -EINVAL; + + /* For AES, only 128-byte or 256-byte keys are supported. */ + if (cipher == OCS_AES && (key_len == OCS_AES_KEYSIZE_128 || + key_len == OCS_AES_KEYSIZE_256)) + return 0; + + /* For SM4, only 128-byte keys are supported. */ + if (cipher == OCS_SM4 && key_len == OCS_AES_KEYSIZE_128) + return 0; + + /* Everything else is unsupported. */ + return -EINVAL; +} + +/* Save key into transformation context. */ +static int save_key(struct ocs_aes_tctx *tctx, const u8 *in_key, size_t key_len, + enum ocs_cipher cipher) +{ + int ret; + + ret = check_key(in_key, key_len, cipher); + if (ret) + return ret; + + memcpy(tctx->key, in_key, key_len); + tctx->key_len = key_len; + tctx->cipher = cipher; + + return 0; +} + +/* Set key for symmetric cypher. */ +static int kmb_ocs_sk_set_key(struct crypto_skcipher *tfm, const u8 *in_key, + size_t key_len, enum ocs_cipher cipher) +{ + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + + /* Fallback is used for AES with 192-bit key. */ + tctx->use_fallback = (cipher == OCS_AES && + key_len == OCS_AES_KEYSIZE_192); + + if (!tctx->use_fallback) + return save_key(tctx, in_key, key_len, cipher); + + crypto_sync_skcipher_clear_flags(tctx->sw_cipher.sk, + CRYPTO_TFM_REQ_MASK); + crypto_sync_skcipher_set_flags(tctx->sw_cipher.sk, + tfm->base.crt_flags & + CRYPTO_TFM_REQ_MASK); + + return crypto_sync_skcipher_setkey(tctx->sw_cipher.sk, in_key, key_len); +} + +/* Set key for AEAD cipher. */ +static int kmb_ocs_aead_set_key(struct crypto_aead *tfm, const u8 *in_key, + size_t key_len, enum ocs_cipher cipher) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); + + /* Fallback is used for AES with 192-bit key. */ + tctx->use_fallback = (cipher == OCS_AES && + key_len == OCS_AES_KEYSIZE_192); + + if (!tctx->use_fallback) + return save_key(tctx, in_key, key_len, cipher); + + crypto_aead_clear_flags(tctx->sw_cipher.aead, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(tctx->sw_cipher.aead, + crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK); + + return crypto_aead_setkey(tctx->sw_cipher.aead, in_key, key_len); +} + +/* Swap two AES blocks in SG lists. */ +static void sg_swap_blocks(struct scatterlist *sgl, unsigned int nents, + off_t blk1_offset, off_t blk2_offset) +{ + u8 tmp_buf1[AES_BLOCK_SIZE], tmp_buf2[AES_BLOCK_SIZE]; + + /* + * No easy way to copy within sg list, so copy both blocks to temporary + * buffers first. + */ + sg_pcopy_to_buffer(sgl, nents, tmp_buf1, AES_BLOCK_SIZE, blk1_offset); + sg_pcopy_to_buffer(sgl, nents, tmp_buf2, AES_BLOCK_SIZE, blk2_offset); + sg_pcopy_from_buffer(sgl, nents, tmp_buf1, AES_BLOCK_SIZE, blk2_offset); + sg_pcopy_from_buffer(sgl, nents, tmp_buf2, AES_BLOCK_SIZE, blk1_offset); +} + +/* Initialize request context to default values. */ +static void ocs_aes_init_rctx(struct ocs_aes_rctx *rctx) +{ + /* Zero everything. */ + memset(rctx, 0, sizeof(*rctx)); + + /* Set initial value for DMA addresses. */ + rctx->src_dll.dma_addr = DMA_MAPPING_ERROR; + rctx->dst_dll.dma_addr = DMA_MAPPING_ERROR; + rctx->aad_src_dll.dma_addr = DMA_MAPPING_ERROR; + rctx->aad_dst_dll.dma_addr = DMA_MAPPING_ERROR; +} + +static int kmb_ocs_sk_validate_input(struct skcipher_request *req, + enum ocs_mode mode) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + int iv_size = crypto_skcipher_ivsize(tfm); + + switch (mode) { + case OCS_MODE_ECB: + /* Ensure input length is multiple of block size */ + if (req->cryptlen % AES_BLOCK_SIZE != 0) + return -EINVAL; + + return 0; + + case OCS_MODE_CBC: + /* Ensure input length is multiple of block size */ + if (req->cryptlen % AES_BLOCK_SIZE != 0) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!req->iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + /* + * NOTE: Since req->cryptlen == 0 case was already handled in + * kmb_ocs_sk_common(), the above two conditions also guarantee + * that: cryptlen >= iv_size + */ + return 0; + + case OCS_MODE_CTR: + /* Ensure IV is present and block size in length */ + if (!req->iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + return 0; + + case OCS_MODE_CTS: + /* Ensure input length >= block size */ + if (req->cryptlen < AES_BLOCK_SIZE) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!req->iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + default: + return -EINVAL; + } +} + +/* + * Called by encrypt() / decrypt() skcipher functions. + * + * Use fallback if needed, otherwise initialize context and enqueue request + * into engine. + */ +static int kmb_ocs_sk_common(struct skcipher_request *req, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + enum ocs_mode mode) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + struct ocs_aes_dev *aes_dev; + int rc; + + if (tctx->use_fallback) { + SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, tctx->sw_cipher.sk); + + skcipher_request_set_sync_tfm(subreq, tctx->sw_cipher.sk); + skcipher_request_set_callback(subreq, req->base.flags, NULL, + NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + + if (instruction == OCS_ENCRYPT) + rc = crypto_skcipher_encrypt(subreq); + else + rc = crypto_skcipher_decrypt(subreq); + + skcipher_request_zero(subreq); + + return rc; + } + + /* + * If cryptlen == 0, no processing needed for ECB, CBC and CTR. + * + * For CTS continue: kmb_ocs_sk_validate_input() will return -EINVAL. + */ + if (!req->cryptlen && mode != OCS_MODE_CTS) + return 0; + + rc = kmb_ocs_sk_validate_input(req, mode); + if (rc) + return rc; + + aes_dev = kmb_ocs_aes_find_dev(tctx); + if (!aes_dev) + return -ENODEV; + + if (cipher != tctx->cipher) + return -EINVAL; + + ocs_aes_init_rctx(rctx); + rctx->instruction = instruction; + rctx->mode = mode; + + return crypto_transfer_skcipher_request_to_engine(aes_dev->engine, req); +} + +static void cleanup_ocs_dma_linked_list(struct device *dev, + struct ocs_dll_desc *dll) +{ + if (dll->vaddr) + dma_free_coherent(dev, dll->size, dll->vaddr, dll->dma_addr); + dll->vaddr = NULL; + dll->size = 0; + dll->dma_addr = DMA_MAPPING_ERROR; +} + +static void kmb_ocs_sk_dma_cleanup(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + struct device *dev = tctx->aes_dev->dev; + + if (rctx->src_dma_count) { + dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE); + rctx->src_dma_count = 0; + } + + if (rctx->dst_dma_count) { + dma_unmap_sg(dev, req->dst, rctx->dst_nents, rctx->in_place ? + DMA_BIDIRECTIONAL : + DMA_FROM_DEVICE); + rctx->dst_dma_count = 0; + } + + /* Clean up OCS DMA linked lists */ + cleanup_ocs_dma_linked_list(dev, &rctx->src_dll); + cleanup_ocs_dma_linked_list(dev, &rctx->dst_dll); +} + +static int kmb_ocs_sk_prepare_inplace(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + int iv_size = crypto_skcipher_ivsize(tfm); + int rc; + + /* + * For CBC decrypt, save last block (iv) to last_ct_blk buffer. + * + * Note: if we are here, we already checked that cryptlen >= iv_size + * and iv_size == AES_BLOCK_SIZE (i.e., the size of last_ct_blk); see + * kmb_ocs_sk_validate_input(). + */ + if (rctx->mode == OCS_MODE_CBC && rctx->instruction == OCS_DECRYPT) + scatterwalk_map_and_copy(rctx->last_ct_blk, req->src, + req->cryptlen - iv_size, iv_size, 0); + + /* For CTS decrypt, swap last two blocks, if needed. */ + if (rctx->cts_swap && rctx->instruction == OCS_DECRYPT) + sg_swap_blocks(req->dst, rctx->dst_nents, + req->cryptlen - AES_BLOCK_SIZE, + req->cryptlen - (2 * AES_BLOCK_SIZE)); + + /* src and dst buffers are the same, use bidirectional DMA mapping. */ + rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst, + rctx->dst_nents, DMA_BIDIRECTIONAL); + if (rctx->dst_dma_count == 0) { + dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n"); + return -ENOMEM; + } + + /* Create DST linked list */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, &rctx->dst_dll, + req->cryptlen, 0); + if (rc) + return rc; + /* + * If descriptor creation was successful, set the src_dll.dma_addr to + * the value of dst_dll.dma_addr, as we do in-place AES operation on + * the src. + */ + rctx->src_dll.dma_addr = rctx->dst_dll.dma_addr; + + return 0; +} + +static int kmb_ocs_sk_prepare_notinplace(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + int rc; + + rctx->src_nents = sg_nents_for_len(req->src, req->cryptlen); + if (rctx->src_nents < 0) + return -EBADMSG; + + /* Map SRC SG. */ + rctx->src_dma_count = dma_map_sg(tctx->aes_dev->dev, req->src, + rctx->src_nents, DMA_TO_DEVICE); + if (rctx->src_dma_count == 0) { + dev_err(tctx->aes_dev->dev, "Failed to map source sg\n"); + return -ENOMEM; + } + + /* Create SRC linked list */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src, + rctx->src_dma_count, &rctx->src_dll, + req->cryptlen, 0); + if (rc) + return rc; + + /* Map DST SG. */ + rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst, + rctx->dst_nents, DMA_FROM_DEVICE); + if (rctx->dst_dma_count == 0) { + dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n"); + return -ENOMEM; + } + + /* Create DST linked list */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, &rctx->dst_dll, + req->cryptlen, 0); + if (rc) + return rc; + + /* If this is not a CTS decrypt operation with swapping, we are done. */ + if (!(rctx->cts_swap && rctx->instruction == OCS_DECRYPT)) + return 0; + + /* + * Otherwise, we have to copy src to dst (as we cannot modify src). + * Use OCS AES bypass mode to copy src to dst via DMA. + * + * NOTE: for anything other than small data sizes this is rather + * inefficient. + */ + rc = ocs_aes_bypass_op(tctx->aes_dev, rctx->dst_dll.dma_addr, + rctx->src_dll.dma_addr, req->cryptlen); + if (rc) + return rc; + + /* + * Now dst == src, so clean up what we did so far and use in_place + * logic. + */ + kmb_ocs_sk_dma_cleanup(req); + rctx->in_place = true; + + return kmb_ocs_sk_prepare_inplace(req); +} + +static int kmb_ocs_sk_run(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + struct ocs_aes_dev *aes_dev = tctx->aes_dev; + int iv_size = crypto_skcipher_ivsize(tfm); + int rc; + + rctx->dst_nents = sg_nents_for_len(req->dst, req->cryptlen); + if (rctx->dst_nents < 0) + return -EBADMSG; + + /* + * If 2 blocks or greater, and multiple of block size swap last two + * blocks to be compatible with other crypto API CTS implementations: + * OCS mode uses CBC-CS2, whereas other crypto API implementations use + * CBC-CS3. + * CBC-CS2 and CBC-CS3 defined by: + * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38a-add.pdf + */ + rctx->cts_swap = (rctx->mode == OCS_MODE_CTS && + req->cryptlen > AES_BLOCK_SIZE && + req->cryptlen % AES_BLOCK_SIZE == 0); + + rctx->in_place = (req->src == req->dst); + + if (rctx->in_place) + rc = kmb_ocs_sk_prepare_inplace(req); + else + rc = kmb_ocs_sk_prepare_notinplace(req); + + if (rc) + goto error; + + rc = ocs_aes_op(aes_dev, rctx->mode, tctx->cipher, rctx->instruction, + rctx->dst_dll.dma_addr, rctx->src_dll.dma_addr, + req->cryptlen, req->iv, iv_size); + if (rc) + goto error; + + /* Clean-up DMA before further processing output. */ + kmb_ocs_sk_dma_cleanup(req); + + /* For CTS Encrypt, swap last 2 blocks, if needed. */ + if (rctx->cts_swap && rctx->instruction == OCS_ENCRYPT) { + sg_swap_blocks(req->dst, rctx->dst_nents, + req->cryptlen - AES_BLOCK_SIZE, + req->cryptlen - (2 * AES_BLOCK_SIZE)); + return 0; + } + + /* For CBC copy IV to req->IV. */ + if (rctx->mode == OCS_MODE_CBC) { + /* CBC encrypt case. */ + if (rctx->instruction == OCS_ENCRYPT) { + scatterwalk_map_and_copy(req->iv, req->dst, + req->cryptlen - iv_size, + iv_size, 0); + return 0; + } + /* CBC decrypt case. */ + if (rctx->in_place) + memcpy(req->iv, rctx->last_ct_blk, iv_size); + else + scatterwalk_map_and_copy(req->iv, req->src, + req->cryptlen - iv_size, + iv_size, 0); + return 0; + } + /* For all other modes there's nothing to do. */ + + return 0; + +error: + kmb_ocs_sk_dma_cleanup(req); + + return rc; +} + +static int kmb_ocs_aead_validate_input(struct aead_request *req, + enum ocs_instruction instruction, + enum ocs_mode mode) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + int tag_size = crypto_aead_authsize(tfm); + int iv_size = crypto_aead_ivsize(tfm); + + /* For decrypt crytplen == len(PT) + len(tag). */ + if (instruction == OCS_DECRYPT && req->cryptlen < tag_size) + return -EINVAL; + + /* IV is mandatory. */ + if (!req->iv) + return -EINVAL; + + switch (mode) { + case OCS_MODE_GCM: + if (iv_size != GCM_AES_IV_SIZE) + return -EINVAL; + + return 0; + + case OCS_MODE_CCM: + /* Ensure IV is present and block size in length */ + if (iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + + default: + return -EINVAL; + } +} + +/* + * Called by encrypt() / decrypt() aead functions. + * + * Use fallback if needed, otherwise initialize context and enqueue request + * into engine. + */ +static int kmb_ocs_aead_common(struct aead_request *req, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + enum ocs_mode mode) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct ocs_aes_rctx *rctx = aead_request_ctx(req); + struct ocs_aes_dev *dd; + int rc; + + if (tctx->use_fallback) { + struct aead_request *subreq = aead_request_ctx(req); + + aead_request_set_tfm(subreq, tctx->sw_cipher.aead); + aead_request_set_callback(subreq, req->base.flags, + req->base.complete, req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + aead_request_set_ad(subreq, req->assoclen); + rc = crypto_aead_setauthsize(tctx->sw_cipher.aead, + crypto_aead_authsize(crypto_aead_reqtfm(req))); + if (rc) + return rc; + + return (instruction == OCS_ENCRYPT) ? + crypto_aead_encrypt(subreq) : + crypto_aead_decrypt(subreq); + } + + rc = kmb_ocs_aead_validate_input(req, instruction, mode); + if (rc) + return rc; + + dd = kmb_ocs_aes_find_dev(tctx); + if (!dd) + return -ENODEV; + + if (cipher != tctx->cipher) + return -EINVAL; + + ocs_aes_init_rctx(rctx); + rctx->instruction = instruction; + rctx->mode = mode; + + return crypto_transfer_aead_request_to_engine(dd->engine, req); +} + +static void kmb_ocs_aead_dma_cleanup(struct aead_request *req) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct ocs_aes_rctx *rctx = aead_request_ctx(req); + struct device *dev = tctx->aes_dev->dev; + + if (rctx->src_dma_count) { + dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE); + rctx->src_dma_count = 0; + } + + if (rctx->dst_dma_count) { + dma_unmap_sg(dev, req->dst, rctx->dst_nents, rctx->in_place ? + DMA_BIDIRECTIONAL : + DMA_FROM_DEVICE); + rctx->dst_dma_count = 0; + } + /* Clean up OCS DMA linked lists */ + cleanup_ocs_dma_linked_list(dev, &rctx->src_dll); + cleanup_ocs_dma_linked_list(dev, &rctx->dst_dll); + cleanup_ocs_dma_linked_list(dev, &rctx->aad_src_dll); + cleanup_ocs_dma_linked_list(dev, &rctx->aad_dst_dll); +} + +/** + * kmb_ocs_aead_dma_prepare() - Do DMA mapping for AEAD processing. + * @req: The AEAD request being processed. + * @src_dll_size: Where to store the length of the data mapped into the + * src_dll OCS DMA list. + * + * Do the following: + * - DMA map req->src and req->dst + * - Initialize the following OCS DMA linked lists: rctx->src_dll, + * rctx->dst_dll, rctx->aad_src_dll and rxtc->aad_dst_dll. + * + * Return: 0 on success, negative error code otherwise. + */ +static int kmb_ocs_aead_dma_prepare(struct aead_request *req, u32 *src_dll_size) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + const int tag_size = crypto_aead_authsize(crypto_aead_reqtfm(req)); + struct ocs_aes_rctx *rctx = aead_request_ctx(req); + u32 in_size; /* The length of the data to be mapped by src_dll. */ + u32 out_size; /* The length of the data to be mapped by dst_dll. */ + u32 dst_size; /* The length of the data in dst_sg. */ + int rc; + + /* Get number of entries in input data SG list. */ + rctx->src_nents = sg_nents_for_len(req->src, + req->assoclen + req->cryptlen); + if (rctx->src_nents < 0) + return -EBADMSG; + + if (rctx->instruction == OCS_DECRYPT) { + /* + * For decrypt: + * - src sg list is: AAD|CT|tag + * - dst sg list expects: AAD|PT + * + * in_size == len(CT); out_size == len(PT) + */ + + /* req->cryptlen includes both CT and tag. */ + in_size = req->cryptlen - tag_size; + + /* out_size = PT size == CT size */ + out_size = in_size; + + /* len(dst_sg) == len(AAD) + len(PT) */ + dst_size = req->assoclen + out_size; + + /* + * Copy tag from source SG list to 'in_tag' buffer. + * + * Note: this needs to be done here, before DMA mapping src_sg. + */ + sg_pcopy_to_buffer(req->src, rctx->src_nents, rctx->in_tag, + tag_size, req->assoclen + in_size); + + } else { /* OCS_ENCRYPT */ + /* + * For encrypt: + * src sg list is: AAD|PT + * dst sg list expects: AAD|CT|tag + */ + /* in_size == len(PT) */ + in_size = req->cryptlen; + + /* + * In CCM mode the OCS engine appends the tag to the ciphertext, + * but in GCM mode the tag must be read from the tag registers + * and appended manually below + */ + out_size = (rctx->mode == OCS_MODE_CCM) ? in_size + tag_size : + in_size; + /* len(dst_sg) == len(AAD) + len(CT) + len(tag) */ + dst_size = req->assoclen + in_size + tag_size; + } + *src_dll_size = in_size; + + /* Get number of entries in output data SG list. */ + rctx->dst_nents = sg_nents_for_len(req->dst, dst_size); + if (rctx->dst_nents < 0) + return -EBADMSG; + + rctx->in_place = (req->src == req->dst) ? 1 : 0; + + /* Map destination; use bidirectional mapping for in-place case. */ + rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst, + rctx->dst_nents, + rctx->in_place ? DMA_BIDIRECTIONAL : + DMA_FROM_DEVICE); + if (rctx->dst_dma_count == 0 && rctx->dst_nents != 0) { + dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n"); + return -ENOMEM; + } + + /* Create AAD DST list: maps dst[0:AAD_SIZE-1]. */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, + &rctx->aad_dst_dll, req->assoclen, + 0); + if (rc) + return rc; + + /* Create DST list: maps dst[AAD_SIZE:out_size] */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, &rctx->dst_dll, + out_size, req->assoclen); + if (rc) + return rc; + + if (rctx->in_place) { + /* If this is not CCM encrypt, we are done. */ + if (!(rctx->mode == OCS_MODE_CCM && + rctx->instruction == OCS_ENCRYPT)) { + /* + * SRC and DST are the same, so re-use the same DMA + * addresses (to avoid allocating new DMA lists + * identical to the dst ones). + */ + rctx->src_dll.dma_addr = rctx->dst_dll.dma_addr; + rctx->aad_src_dll.dma_addr = rctx->aad_dst_dll.dma_addr; + + return 0; + } + /* + * For CCM encrypt the input and output linked lists contain + * different amounts of data, so, we need to create different + * SRC and AAD SRC lists, even for the in-place case. + */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, + &rctx->aad_src_dll, + req->assoclen, 0); + if (rc) + return rc; + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, + &rctx->src_dll, in_size, + req->assoclen); + if (rc) + return rc; + + return 0; + } + /* Not in-place case. */ + + /* Map source SG. */ + rctx->src_dma_count = dma_map_sg(tctx->aes_dev->dev, req->src, + rctx->src_nents, DMA_TO_DEVICE); + if (rctx->src_dma_count == 0 && rctx->src_nents != 0) { + dev_err(tctx->aes_dev->dev, "Failed to map source sg\n"); + return -ENOMEM; + } + + /* Create AAD SRC list. */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src, + rctx->src_dma_count, + &rctx->aad_src_dll, + req->assoclen, 0); + if (rc) + return rc; + + /* Create SRC list. */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src, + rctx->src_dma_count, + &rctx->src_dll, in_size, + req->assoclen); + if (rc) + return rc; + + if (req->assoclen == 0) + return 0; + + /* Copy AAD from src sg to dst sg using OCS DMA. */ + rc = ocs_aes_bypass_op(tctx->aes_dev, rctx->aad_dst_dll.dma_addr, + rctx->aad_src_dll.dma_addr, req->cryptlen); + if (rc) + dev_err(tctx->aes_dev->dev, + "Failed to copy source AAD to destination AAD\n"); + + return rc; +} + +static int kmb_ocs_aead_run(struct aead_request *req) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + const int tag_size = crypto_aead_authsize(crypto_aead_reqtfm(req)); + struct ocs_aes_rctx *rctx = aead_request_ctx(req); + u32 in_size; /* The length of the data mapped by src_dll. */ + int rc; + + rc = kmb_ocs_aead_dma_prepare(req, &in_size); + if (rc) + goto exit; + + /* For CCM, we just call the OCS processing and we are done. */ + if (rctx->mode == OCS_MODE_CCM) { + rc = ocs_aes_ccm_op(tctx->aes_dev, tctx->cipher, + rctx->instruction, rctx->dst_dll.dma_addr, + rctx->src_dll.dma_addr, in_size, + req->iv, + rctx->aad_src_dll.dma_addr, req->assoclen, + rctx->in_tag, tag_size); + goto exit; + } + /* GCM case; invoke OCS processing. */ + rc = ocs_aes_gcm_op(tctx->aes_dev, tctx->cipher, + rctx->instruction, + rctx->dst_dll.dma_addr, + rctx->src_dll.dma_addr, in_size, + req->iv, + rctx->aad_src_dll.dma_addr, req->assoclen, + rctx->out_tag, tag_size); + if (rc) + goto exit; + + /* For GCM decrypt, we have to compare in_tag with out_tag. */ + if (rctx->instruction == OCS_DECRYPT) { + rc = crypto_memneq(rctx->in_tag, rctx->out_tag, tag_size) ? + -EBADMSG : 0; + goto exit; + } + + /* For GCM encrypt, we must manually copy out_tag to DST sg. */ + + /* Clean-up must be called before the sg_pcopy_from_buffer() below. */ + kmb_ocs_aead_dma_cleanup(req); + + /* Copy tag to destination sg after AAD and CT. */ + sg_pcopy_from_buffer(req->dst, rctx->dst_nents, rctx->out_tag, + tag_size, req->assoclen + req->cryptlen); + + /* Return directly as DMA cleanup already done. */ + return 0; + +exit: + kmb_ocs_aead_dma_cleanup(req); + + return rc; +} + +static int kmb_ocs_aes_sk_do_one_request(struct crypto_engine *engine, + void *areq) +{ + struct skcipher_request *req = + container_of(areq, struct skcipher_request, base); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + int err; + + if (!tctx->aes_dev) { + err = -ENODEV; + goto exit; + } + + err = ocs_aes_set_key(tctx->aes_dev, tctx->key_len, tctx->key, + tctx->cipher); + if (err) + goto exit; + + err = kmb_ocs_sk_run(req); + +exit: + crypto_finalize_skcipher_request(engine, req, err); + + return 0; +} + +static int kmb_ocs_aes_aead_do_one_request(struct crypto_engine *engine, + void *areq) +{ + struct aead_request *req = container_of(areq, + struct aead_request, base); + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + int err; + + if (!tctx->aes_dev) + return -ENODEV; + + err = ocs_aes_set_key(tctx->aes_dev, tctx->key_len, tctx->key, + tctx->cipher); + if (err) + goto exit; + + err = kmb_ocs_aead_run(req); + +exit: + crypto_finalize_aead_request(tctx->aes_dev->engine, req, err); + + return 0; +} + +static int kmb_ocs_aes_set_key(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) +{ + return kmb_ocs_sk_set_key(tfm, in_key, key_len, OCS_AES); +} + +static int kmb_ocs_aes_aead_set_key(struct crypto_aead *tfm, const u8 *in_key, + unsigned int key_len) +{ + return kmb_ocs_aead_set_key(tfm, in_key, key_len, OCS_AES); +} + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB +static int kmb_ocs_aes_ecb_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_ECB); +} + +static int kmb_ocs_aes_ecb_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_ECB); +} +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + +static int kmb_ocs_aes_cbc_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CBC); +} + +static int kmb_ocs_aes_cbc_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CBC); +} + +static int kmb_ocs_aes_ctr_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CTR); +} + +static int kmb_ocs_aes_ctr_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CTR); +} + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS +static int kmb_ocs_aes_cts_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CTS); +} + +static int kmb_ocs_aes_cts_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CTS); +} +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ + +static int kmb_ocs_aes_gcm_encrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_GCM); +} + +static int kmb_ocs_aes_gcm_decrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_GCM); +} + +static int kmb_ocs_aes_ccm_encrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CCM); +} + +static int kmb_ocs_aes_ccm_decrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CCM); +} + +static int kmb_ocs_sm4_set_key(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) +{ + return kmb_ocs_sk_set_key(tfm, in_key, key_len, OCS_SM4); +} + +static int kmb_ocs_sm4_aead_set_key(struct crypto_aead *tfm, const u8 *in_key, + unsigned int key_len) +{ + return kmb_ocs_aead_set_key(tfm, in_key, key_len, OCS_SM4); +} + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB +static int kmb_ocs_sm4_ecb_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_ECB); +} + +static int kmb_ocs_sm4_ecb_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_ECB); +} +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + +static int kmb_ocs_sm4_cbc_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CBC); +} + +static int kmb_ocs_sm4_cbc_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CBC); +} + +static int kmb_ocs_sm4_ctr_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CTR); +} + +static int kmb_ocs_sm4_ctr_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CTR); +} + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS +static int kmb_ocs_sm4_cts_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CTS); +} + +static int kmb_ocs_sm4_cts_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CTS); +} +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ + +static int kmb_ocs_sm4_gcm_encrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_GCM); +} + +static int kmb_ocs_sm4_gcm_decrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_GCM); +} + +static int kmb_ocs_sm4_ccm_encrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CCM); +} + +static int kmb_ocs_sm4_ccm_decrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CCM); +} + +static int ocs_aes_init_tfm(struct crypto_skcipher *tfm) +{ + const char *alg_name = crypto_tfm_alg_name(&tfm->base); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + struct crypto_sync_skcipher *blk; + + /* set fallback cipher in case it will be needed */ + blk = crypto_alloc_sync_skcipher(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(blk)) + return PTR_ERR(blk); + + tctx->sw_cipher.sk = blk; + + crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); + + return 0; +} + +static int ocs_sm4_init_tfm(struct crypto_skcipher *tfm) +{ + crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); + + return 0; +} + +static inline void clear_key(struct ocs_aes_tctx *tctx) +{ + memzero_explicit(tctx->key, OCS_AES_KEYSIZE_256); + + /* Zero key registers if set */ + if (tctx->aes_dev) + ocs_aes_set_key(tctx->aes_dev, OCS_AES_KEYSIZE_256, + tctx->key, OCS_AES); +} + +static void ocs_exit_tfm(struct crypto_skcipher *tfm) +{ + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + + clear_key(tctx); + + if (tctx->sw_cipher.sk) { + crypto_free_sync_skcipher(tctx->sw_cipher.sk); + tctx->sw_cipher.sk = NULL; + } +} + +static int ocs_aes_aead_cra_init(struct crypto_aead *tfm) +{ + const char *alg_name = crypto_tfm_alg_name(&tfm->base); + struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); + struct crypto_aead *blk; + + /* Set fallback cipher in case it will be needed */ + blk = crypto_alloc_aead(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(blk)) + return PTR_ERR(blk); + + tctx->sw_cipher.aead = blk; + + crypto_aead_set_reqsize(tfm, + max(sizeof(struct ocs_aes_rctx), + (sizeof(struct aead_request) + + crypto_aead_reqsize(tctx->sw_cipher.aead)))); + + return 0; +} + +static int kmb_ocs_aead_ccm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + switch (authsize) { + case 4: + case 6: + case 8: + case 10: + case 12: + case 14: + case 16: + return 0; + default: + return -EINVAL; + } +} + +static int kmb_ocs_aead_gcm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + return crypto_gcm_check_authsize(authsize); +} + +static int ocs_sm4_aead_cra_init(struct crypto_aead *tfm) +{ + crypto_aead_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); + + return 0; +} + +static void ocs_aead_cra_exit(struct crypto_aead *tfm) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); + + clear_key(tctx); + + if (tctx->sw_cipher.aead) { + crypto_free_aead(tctx->sw_cipher.aead); + tctx->sw_cipher.aead = NULL; + } +} + +static struct skcipher_engine_alg algs[] = { +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB + { + .base.base.cra_name = "ecb(aes)", + .base.base.cra_driver_name = "ecb-aes-keembay-ocs", + .base.base.cra_priority = KMB_OCS_PRIORITY, + .base.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.base.cra_blocksize = AES_BLOCK_SIZE, + .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.base.cra_module = THIS_MODULE, + .base.base.cra_alignmask = 0, + + .base.min_keysize = OCS_AES_MIN_KEY_SIZE, + .base.max_keysize = OCS_AES_MAX_KEY_SIZE, + .base.setkey = kmb_ocs_aes_set_key, + .base.encrypt = kmb_ocs_aes_ecb_encrypt, + .base.decrypt = kmb_ocs_aes_ecb_decrypt, + .base.init = ocs_aes_init_tfm, + .base.exit = ocs_exit_tfm, + .op.do_one_request = kmb_ocs_aes_sk_do_one_request, + }, +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + { + .base.base.cra_name = "cbc(aes)", + .base.base.cra_driver_name = "cbc-aes-keembay-ocs", + .base.base.cra_priority = KMB_OCS_PRIORITY, + .base.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.base.cra_blocksize = AES_BLOCK_SIZE, + .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.base.cra_module = THIS_MODULE, + .base.base.cra_alignmask = 0, + + .base.min_keysize = OCS_AES_MIN_KEY_SIZE, + .base.max_keysize = OCS_AES_MAX_KEY_SIZE, + .base.ivsize = AES_BLOCK_SIZE, + .base.setkey = kmb_ocs_aes_set_key, + .base.encrypt = kmb_ocs_aes_cbc_encrypt, + .base.decrypt = kmb_ocs_aes_cbc_decrypt, + .base.init = ocs_aes_init_tfm, + .base.exit = ocs_exit_tfm, + .op.do_one_request = kmb_ocs_aes_sk_do_one_request, + }, + { + .base.base.cra_name = "ctr(aes)", + .base.base.cra_driver_name = "ctr-aes-keembay-ocs", + .base.base.cra_priority = KMB_OCS_PRIORITY, + .base.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.base.cra_blocksize = 1, + .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.base.cra_module = THIS_MODULE, + .base.base.cra_alignmask = 0, + + .base.min_keysize = OCS_AES_MIN_KEY_SIZE, + .base.max_keysize = OCS_AES_MAX_KEY_SIZE, + .base.ivsize = AES_BLOCK_SIZE, + .base.setkey = kmb_ocs_aes_set_key, + .base.encrypt = kmb_ocs_aes_ctr_encrypt, + .base.decrypt = kmb_ocs_aes_ctr_decrypt, + .base.init = ocs_aes_init_tfm, + .base.exit = ocs_exit_tfm, + .op.do_one_request = kmb_ocs_aes_sk_do_one_request, + }, +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS + { + .base.base.cra_name = "cts(cbc(aes))", + .base.base.cra_driver_name = "cts-aes-keembay-ocs", + .base.base.cra_priority = KMB_OCS_PRIORITY, + .base.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.base.cra_blocksize = AES_BLOCK_SIZE, + .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.base.cra_module = THIS_MODULE, + .base.base.cra_alignmask = 0, + + .base.min_keysize = OCS_AES_MIN_KEY_SIZE, + .base.max_keysize = OCS_AES_MAX_KEY_SIZE, + .base.ivsize = AES_BLOCK_SIZE, + .base.setkey = kmb_ocs_aes_set_key, + .base.encrypt = kmb_ocs_aes_cts_encrypt, + .base.decrypt = kmb_ocs_aes_cts_decrypt, + .base.init = ocs_aes_init_tfm, + .base.exit = ocs_exit_tfm, + .op.do_one_request = kmb_ocs_aes_sk_do_one_request, + }, +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB + { + .base.base.cra_name = "ecb(sm4)", + .base.base.cra_driver_name = "ecb-sm4-keembay-ocs", + .base.base.cra_priority = KMB_OCS_PRIORITY, + .base.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.base.cra_blocksize = AES_BLOCK_SIZE, + .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.base.cra_module = THIS_MODULE, + .base.base.cra_alignmask = 0, + + .base.min_keysize = OCS_SM4_KEY_SIZE, + .base.max_keysize = OCS_SM4_KEY_SIZE, + .base.setkey = kmb_ocs_sm4_set_key, + .base.encrypt = kmb_ocs_sm4_ecb_encrypt, + .base.decrypt = kmb_ocs_sm4_ecb_decrypt, + .base.init = ocs_sm4_init_tfm, + .base.exit = ocs_exit_tfm, + .op.do_one_request = kmb_ocs_aes_sk_do_one_request, + }, +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + { + .base.base.cra_name = "cbc(sm4)", + .base.base.cra_driver_name = "cbc-sm4-keembay-ocs", + .base.base.cra_priority = KMB_OCS_PRIORITY, + .base.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.base.cra_blocksize = AES_BLOCK_SIZE, + .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.base.cra_module = THIS_MODULE, + .base.base.cra_alignmask = 0, + + .base.min_keysize = OCS_SM4_KEY_SIZE, + .base.max_keysize = OCS_SM4_KEY_SIZE, + .base.ivsize = AES_BLOCK_SIZE, + .base.setkey = kmb_ocs_sm4_set_key, + .base.encrypt = kmb_ocs_sm4_cbc_encrypt, + .base.decrypt = kmb_ocs_sm4_cbc_decrypt, + .base.init = ocs_sm4_init_tfm, + .base.exit = ocs_exit_tfm, + .op.do_one_request = kmb_ocs_aes_sk_do_one_request, + }, + { + .base.base.cra_name = "ctr(sm4)", + .base.base.cra_driver_name = "ctr-sm4-keembay-ocs", + .base.base.cra_priority = KMB_OCS_PRIORITY, + .base.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.base.cra_blocksize = 1, + .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.base.cra_module = THIS_MODULE, + .base.base.cra_alignmask = 0, + + .base.min_keysize = OCS_SM4_KEY_SIZE, + .base.max_keysize = OCS_SM4_KEY_SIZE, + .base.ivsize = AES_BLOCK_SIZE, + .base.setkey = kmb_ocs_sm4_set_key, + .base.encrypt = kmb_ocs_sm4_ctr_encrypt, + .base.decrypt = kmb_ocs_sm4_ctr_decrypt, + .base.init = ocs_sm4_init_tfm, + .base.exit = ocs_exit_tfm, + .op.do_one_request = kmb_ocs_aes_sk_do_one_request, + }, +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS + { + .base.base.cra_name = "cts(cbc(sm4))", + .base.base.cra_driver_name = "cts-sm4-keembay-ocs", + .base.base.cra_priority = KMB_OCS_PRIORITY, + .base.base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.base.cra_blocksize = AES_BLOCK_SIZE, + .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.base.cra_module = THIS_MODULE, + .base.base.cra_alignmask = 0, + + .base.min_keysize = OCS_SM4_KEY_SIZE, + .base.max_keysize = OCS_SM4_KEY_SIZE, + .base.ivsize = AES_BLOCK_SIZE, + .base.setkey = kmb_ocs_sm4_set_key, + .base.encrypt = kmb_ocs_sm4_cts_encrypt, + .base.decrypt = kmb_ocs_sm4_cts_decrypt, + .base.init = ocs_sm4_init_tfm, + .base.exit = ocs_exit_tfm, + .op.do_one_request = kmb_ocs_aes_sk_do_one_request, + } +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ +}; + +static struct aead_engine_alg algs_aead[] = { + { + .base.base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "gcm-aes-keembay-ocs", + .cra_priority = KMB_OCS_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct ocs_aes_tctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .base.init = ocs_aes_aead_cra_init, + .base.exit = ocs_aead_cra_exit, + .base.ivsize = GCM_AES_IV_SIZE, + .base.maxauthsize = AES_BLOCK_SIZE, + .base.setauthsize = kmb_ocs_aead_gcm_setauthsize, + .base.setkey = kmb_ocs_aes_aead_set_key, + .base.encrypt = kmb_ocs_aes_gcm_encrypt, + .base.decrypt = kmb_ocs_aes_gcm_decrypt, + .op.do_one_request = kmb_ocs_aes_aead_do_one_request, + }, + { + .base.base = { + .cra_name = "ccm(aes)", + .cra_driver_name = "ccm-aes-keembay-ocs", + .cra_priority = KMB_OCS_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct ocs_aes_tctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .base.init = ocs_aes_aead_cra_init, + .base.exit = ocs_aead_cra_exit, + .base.ivsize = AES_BLOCK_SIZE, + .base.maxauthsize = AES_BLOCK_SIZE, + .base.setauthsize = kmb_ocs_aead_ccm_setauthsize, + .base.setkey = kmb_ocs_aes_aead_set_key, + .base.encrypt = kmb_ocs_aes_ccm_encrypt, + .base.decrypt = kmb_ocs_aes_ccm_decrypt, + .op.do_one_request = kmb_ocs_aes_aead_do_one_request, + }, + { + .base.base = { + .cra_name = "gcm(sm4)", + .cra_driver_name = "gcm-sm4-keembay-ocs", + .cra_priority = KMB_OCS_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct ocs_aes_tctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .base.init = ocs_sm4_aead_cra_init, + .base.exit = ocs_aead_cra_exit, + .base.ivsize = GCM_AES_IV_SIZE, + .base.maxauthsize = AES_BLOCK_SIZE, + .base.setauthsize = kmb_ocs_aead_gcm_setauthsize, + .base.setkey = kmb_ocs_sm4_aead_set_key, + .base.encrypt = kmb_ocs_sm4_gcm_encrypt, + .base.decrypt = kmb_ocs_sm4_gcm_decrypt, + .op.do_one_request = kmb_ocs_aes_aead_do_one_request, + }, + { + .base.base = { + .cra_name = "ccm(sm4)", + .cra_driver_name = "ccm-sm4-keembay-ocs", + .cra_priority = KMB_OCS_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct ocs_aes_tctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .base.init = ocs_sm4_aead_cra_init, + .base.exit = ocs_aead_cra_exit, + .base.ivsize = AES_BLOCK_SIZE, + .base.maxauthsize = AES_BLOCK_SIZE, + .base.setauthsize = kmb_ocs_aead_ccm_setauthsize, + .base.setkey = kmb_ocs_sm4_aead_set_key, + .base.encrypt = kmb_ocs_sm4_ccm_encrypt, + .base.decrypt = kmb_ocs_sm4_ccm_decrypt, + .op.do_one_request = kmb_ocs_aes_aead_do_one_request, + } +}; + +static void unregister_aes_algs(struct ocs_aes_dev *aes_dev) +{ + crypto_engine_unregister_aeads(algs_aead, ARRAY_SIZE(algs_aead)); + crypto_engine_unregister_skciphers(algs, ARRAY_SIZE(algs)); +} + +static int register_aes_algs(struct ocs_aes_dev *aes_dev) +{ + int ret; + + /* + * If any algorithm fails to register, all preceding algorithms that + * were registered in the same call are automatically unregistered. + */ + ret = crypto_engine_register_aeads(algs_aead, ARRAY_SIZE(algs_aead)); + if (ret) + return ret; + + ret = crypto_engine_register_skciphers(algs, ARRAY_SIZE(algs)); + if (ret) + crypto_engine_unregister_aeads(algs_aead, ARRAY_SIZE(algs_aead)); + + return ret; +} + +/* Device tree driver match. */ +static const struct of_device_id kmb_ocs_aes_of_match[] = { + { + .compatible = "intel,keembay-ocs-aes", + }, + {} +}; +MODULE_DEVICE_TABLE(of, kmb_ocs_aes_of_match); + +static void kmb_ocs_aes_remove(struct platform_device *pdev) +{ + struct ocs_aes_dev *aes_dev; + + aes_dev = platform_get_drvdata(pdev); + + unregister_aes_algs(aes_dev); + + spin_lock(&ocs_aes.lock); + list_del(&aes_dev->list); + spin_unlock(&ocs_aes.lock); + + crypto_engine_exit(aes_dev->engine); +} + +static int kmb_ocs_aes_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct ocs_aes_dev *aes_dev; + int rc; + + aes_dev = devm_kzalloc(dev, sizeof(*aes_dev), GFP_KERNEL); + if (!aes_dev) + return -ENOMEM; + + aes_dev->dev = dev; + + platform_set_drvdata(pdev, aes_dev); + + rc = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + if (rc) { + dev_err(dev, "Failed to set 32 bit dma mask %d\n", rc); + return rc; + } + + /* Get base register address. */ + aes_dev->base_reg = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(aes_dev->base_reg)) + return PTR_ERR(aes_dev->base_reg); + + init_completion(&aes_dev->irq_completion); + + /* Get and request IRQ */ + aes_dev->irq = platform_get_irq(pdev, 0); + if (aes_dev->irq < 0) + return aes_dev->irq; + + rc = devm_request_threaded_irq(dev, aes_dev->irq, ocs_aes_irq_handler, + NULL, 0, "keembay-ocs-aes", aes_dev); + if (rc < 0) + return rc; + + INIT_LIST_HEAD(&aes_dev->list); + spin_lock(&ocs_aes.lock); + list_add_tail(&aes_dev->list, &ocs_aes.dev_list); + spin_unlock(&ocs_aes.lock); + + /* Initialize crypto engine */ + aes_dev->engine = crypto_engine_alloc_init(dev, true); + if (!aes_dev->engine) { + rc = -ENOMEM; + goto list_del; + } + + rc = crypto_engine_start(aes_dev->engine); + if (rc) { + dev_err(dev, "Could not start crypto engine\n"); + goto cleanup; + } + + rc = register_aes_algs(aes_dev); + if (rc) { + dev_err(dev, + "Could not register OCS algorithms with Crypto API\n"); + goto cleanup; + } + + return 0; + +cleanup: + crypto_engine_exit(aes_dev->engine); +list_del: + spin_lock(&ocs_aes.lock); + list_del(&aes_dev->list); + spin_unlock(&ocs_aes.lock); + + return rc; +} + +/* The OCS driver is a platform device. */ +static struct platform_driver kmb_ocs_aes_driver = { + .probe = kmb_ocs_aes_probe, + .remove = kmb_ocs_aes_remove, + .driver = { + .name = DRV_NAME, + .of_match_table = kmb_ocs_aes_of_match, + }, +}; + +module_platform_driver(kmb_ocs_aes_driver); + +MODULE_DESCRIPTION("Intel Keem Bay Offload and Crypto Subsystem (OCS) AES/SM4 Driver"); +MODULE_LICENSE("GPL"); + +MODULE_ALIAS_CRYPTO("cbc-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ctr-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("gcm-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ccm-aes-keembay-ocs"); + +MODULE_ALIAS_CRYPTO("cbc-sm4-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ctr-sm4-keembay-ocs"); +MODULE_ALIAS_CRYPTO("gcm-sm4-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ccm-sm4-keembay-ocs"); + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB +MODULE_ALIAS_CRYPTO("ecb-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ecb-sm4-keembay-ocs"); +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS +MODULE_ALIAS_CRYPTO("cts-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("cts-sm4-keembay-ocs"); +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ diff --git a/drivers/crypto/intel/keembay/keembay-ocs-ecc.c b/drivers/crypto/intel/keembay/keembay-ocs-ecc.c new file mode 100644 index 000000000..9bdee39dc --- /dev/null +++ b/drivers/crypto/intel/keembay/keembay-ocs-ecc.c @@ -0,0 +1,997 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel Keem Bay OCS ECC Crypto Driver. + * + * Copyright (C) 2019-2021 Intel Corporation + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DRV_NAME "keembay-ocs-ecc" + +#define KMB_OCS_ECC_PRIORITY 350 + +#define HW_OFFS_OCS_ECC_COMMAND 0x00000000 +#define HW_OFFS_OCS_ECC_STATUS 0x00000004 +#define HW_OFFS_OCS_ECC_DATA_IN 0x00000080 +#define HW_OFFS_OCS_ECC_CX_DATA_OUT 0x00000100 +#define HW_OFFS_OCS_ECC_CY_DATA_OUT 0x00000180 +#define HW_OFFS_OCS_ECC_ISR 0x00000400 +#define HW_OFFS_OCS_ECC_IER 0x00000404 + +#define HW_OCS_ECC_ISR_INT_STATUS_DONE BIT(0) +#define HW_OCS_ECC_COMMAND_INS_BP BIT(0) + +#define HW_OCS_ECC_COMMAND_START_VAL BIT(0) + +#define OCS_ECC_OP_SIZE_384 BIT(8) +#define OCS_ECC_OP_SIZE_256 0 + +/* ECC Instruction : for ECC_COMMAND */ +#define OCS_ECC_INST_WRITE_AX (0x1 << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_WRITE_AY (0x2 << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_WRITE_BX_D (0x3 << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_WRITE_BY_L (0x4 << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_WRITE_P (0x5 << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_WRITE_A (0x6 << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_CALC_D_IDX_A (0x8 << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_CALC_A_POW_B_MODP (0xB << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_CALC_A_MUL_B_MODP (0xC << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_CALC_A_ADD_B_MODP (0xD << HW_OCS_ECC_COMMAND_INS_BP) + +#define ECC_ENABLE_INTR 1 + +#define POLL_USEC 100 +#define TIMEOUT_USEC 10000 + +#define KMB_ECC_VLI_MAX_DIGITS ECC_CURVE_NIST_P384_DIGITS +#define KMB_ECC_VLI_MAX_BYTES (KMB_ECC_VLI_MAX_DIGITS \ + << ECC_DIGITS_TO_BYTES_SHIFT) + +#define POW_CUBE 3 + +/** + * struct ocs_ecc_dev - ECC device context + * @list: List of device contexts + * @dev: OCS ECC device + * @base_reg: IO base address of OCS ECC + * @engine: Crypto engine for the device + * @irq_done: IRQ done completion. + * @irq: IRQ number + */ +struct ocs_ecc_dev { + struct list_head list; + struct device *dev; + void __iomem *base_reg; + struct crypto_engine *engine; + struct completion irq_done; + int irq; +}; + +/** + * struct ocs_ecc_ctx - Transformation context. + * @ecc_dev: The ECC driver associated with this context. + * @curve: The elliptic curve used by this transformation. + * @private_key: The private key. + */ +struct ocs_ecc_ctx { + struct ocs_ecc_dev *ecc_dev; + const struct ecc_curve *curve; + u64 private_key[KMB_ECC_VLI_MAX_DIGITS]; +}; + +/* Driver data. */ +struct ocs_ecc_drv { + struct list_head dev_list; + spinlock_t lock; /* Protects dev_list. */ +}; + +/* Global variable holding the list of OCS ECC devices (only one expected). */ +static struct ocs_ecc_drv ocs_ecc = { + .dev_list = LIST_HEAD_INIT(ocs_ecc.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(ocs_ecc.lock), +}; + +/* Get OCS ECC tfm context from kpp_request. */ +static inline struct ocs_ecc_ctx *kmb_ocs_ecc_tctx(struct kpp_request *req) +{ + return kpp_tfm_ctx(crypto_kpp_reqtfm(req)); +} + +/* Converts number of digits to number of bytes. */ +static inline unsigned int digits_to_bytes(unsigned int n) +{ + return n << ECC_DIGITS_TO_BYTES_SHIFT; +} + +/* + * Wait for ECC idle i.e when an operation (other than write operations) + * is done. + */ +static inline int ocs_ecc_wait_idle(struct ocs_ecc_dev *dev) +{ + u32 value; + + return readl_poll_timeout((dev->base_reg + HW_OFFS_OCS_ECC_STATUS), + value, + !(value & HW_OCS_ECC_ISR_INT_STATUS_DONE), + POLL_USEC, TIMEOUT_USEC); +} + +static void ocs_ecc_cmd_start(struct ocs_ecc_dev *ecc_dev, u32 op_size) +{ + iowrite32(op_size | HW_OCS_ECC_COMMAND_START_VAL, + ecc_dev->base_reg + HW_OFFS_OCS_ECC_COMMAND); +} + +/* Direct write of u32 buffer to ECC engine with associated instruction. */ +static void ocs_ecc_write_cmd_and_data(struct ocs_ecc_dev *dev, + u32 op_size, + u32 inst, + const void *data_in, + size_t data_size) +{ + iowrite32(op_size | inst, dev->base_reg + HW_OFFS_OCS_ECC_COMMAND); + + /* MMIO Write src uint32 to dst. */ + memcpy_toio(dev->base_reg + HW_OFFS_OCS_ECC_DATA_IN, data_in, + data_size); +} + +/* Start OCS ECC operation and wait for its completion. */ +static int ocs_ecc_trigger_op(struct ocs_ecc_dev *ecc_dev, u32 op_size, + u32 inst) +{ + reinit_completion(&ecc_dev->irq_done); + + iowrite32(ECC_ENABLE_INTR, ecc_dev->base_reg + HW_OFFS_OCS_ECC_IER); + iowrite32(op_size | inst, ecc_dev->base_reg + HW_OFFS_OCS_ECC_COMMAND); + + return wait_for_completion_interruptible(&ecc_dev->irq_done); +} + +/** + * ocs_ecc_read_cx_out() - Read the CX data output buffer. + * @dev: The OCS ECC device to read from. + * @cx_out: The buffer where to store the CX value. Must be at least + * @byte_count byte long. + * @byte_count: The amount of data to read. + */ +static inline void ocs_ecc_read_cx_out(struct ocs_ecc_dev *dev, void *cx_out, + size_t byte_count) +{ + memcpy_fromio(cx_out, dev->base_reg + HW_OFFS_OCS_ECC_CX_DATA_OUT, + byte_count); +} + +/** + * ocs_ecc_read_cy_out() - Read the CX data output buffer. + * @dev: The OCS ECC device to read from. + * @cy_out: The buffer where to store the CY value. Must be at least + * @byte_count byte long. + * @byte_count: The amount of data to read. + */ +static inline void ocs_ecc_read_cy_out(struct ocs_ecc_dev *dev, void *cy_out, + size_t byte_count) +{ + memcpy_fromio(cy_out, dev->base_reg + HW_OFFS_OCS_ECC_CY_DATA_OUT, + byte_count); +} + +static struct ocs_ecc_dev *kmb_ocs_ecc_find_dev(struct ocs_ecc_ctx *tctx) +{ + if (tctx->ecc_dev) + return tctx->ecc_dev; + + spin_lock(&ocs_ecc.lock); + + /* Only a single OCS device available. */ + tctx->ecc_dev = list_first_entry(&ocs_ecc.dev_list, struct ocs_ecc_dev, + list); + + spin_unlock(&ocs_ecc.lock); + + return tctx->ecc_dev; +} + +/* Do point multiplication using OCS ECC HW. */ +static int kmb_ecc_point_mult(struct ocs_ecc_dev *ecc_dev, + struct ecc_point *result, + const struct ecc_point *point, + u64 *scalar, + const struct ecc_curve *curve) +{ + u8 sca[KMB_ECC_VLI_MAX_BYTES]; /* Use the maximum data size. */ + u32 op_size = (curve->g.ndigits > ECC_CURVE_NIST_P256_DIGITS) ? + OCS_ECC_OP_SIZE_384 : OCS_ECC_OP_SIZE_256; + size_t nbytes = digits_to_bytes(curve->g.ndigits); + int rc = 0; + + /* Generate random nbytes for Simple and Differential SCA protection. */ + rc = crypto_stdrng_get_bytes(sca, nbytes); + if (rc) + return rc; + + /* Wait engine to be idle before starting new operation. */ + rc = ocs_ecc_wait_idle(ecc_dev); + if (rc) + return rc; + + /* Send ecc_start pulse as well as indicating operation size. */ + ocs_ecc_cmd_start(ecc_dev, op_size); + + /* Write ax param; Base point (Gx). */ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AX, + point->x, nbytes); + + /* Write ay param; Base point (Gy). */ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AY, + point->y, nbytes); + + /* + * Write the private key into DATA_IN reg. + * + * Since DATA_IN register is used to write different values during the + * computation private Key value is overwritten with + * side-channel-resistance value. + */ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_BX_D, + scalar, nbytes); + + /* Write operand by/l. */ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_BY_L, + sca, nbytes); + memzero_explicit(sca, sizeof(sca)); + + /* Write p = curve prime(GF modulus). */ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_P, + curve->p, nbytes); + + /* Write a = curve coefficient. */ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_A, + curve->a, nbytes); + + /* Make hardware perform the multiplication. */ + rc = ocs_ecc_trigger_op(ecc_dev, op_size, OCS_ECC_INST_CALC_D_IDX_A); + if (rc) + return rc; + + /* Read result. */ + ocs_ecc_read_cx_out(ecc_dev, result->x, nbytes); + ocs_ecc_read_cy_out(ecc_dev, result->y, nbytes); + + return 0; +} + +/** + * kmb_ecc_do_scalar_op() - Perform Scalar operation using OCS ECC HW. + * @ecc_dev: The OCS ECC device to use. + * @scalar_out: Where to store the output scalar. + * @scalar_a: Input scalar operand 'a'. + * @scalar_b: Input scalar operand 'b' + * @curve: The curve on which the operation is performed. + * @ndigits: The size of the operands (in digits). + * @inst: The operation to perform (as an OCS ECC instruction). + * + * Return: 0 on success, negative error code otherwise. + */ +static int kmb_ecc_do_scalar_op(struct ocs_ecc_dev *ecc_dev, u64 *scalar_out, + const u64 *scalar_a, const u64 *scalar_b, + const struct ecc_curve *curve, + unsigned int ndigits, const u32 inst) +{ + u32 op_size = (ndigits > ECC_CURVE_NIST_P256_DIGITS) ? + OCS_ECC_OP_SIZE_384 : OCS_ECC_OP_SIZE_256; + size_t nbytes = digits_to_bytes(ndigits); + int rc; + + /* Wait engine to be idle before starting new operation. */ + rc = ocs_ecc_wait_idle(ecc_dev); + if (rc) + return rc; + + /* Send ecc_start pulse as well as indicating operation size. */ + ocs_ecc_cmd_start(ecc_dev, op_size); + + /* Write ax param (Base point (Gx).*/ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AX, + scalar_a, nbytes); + + /* Write ay param Base point (Gy).*/ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AY, + scalar_b, nbytes); + + /* Write p = curve prime(GF modulus).*/ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_P, + curve->p, nbytes); + + /* Give instruction A.B or A+B to ECC engine. */ + rc = ocs_ecc_trigger_op(ecc_dev, op_size, inst); + if (rc) + return rc; + + ocs_ecc_read_cx_out(ecc_dev, scalar_out, nbytes); + + if (vli_is_zero(scalar_out, ndigits)) + return -EINVAL; + + return 0; +} + +/* SP800-56A section 5.6.2.3.4 partial verification: ephemeral keys only */ +static int kmb_ocs_ecc_is_pubkey_valid_partial(struct ocs_ecc_dev *ecc_dev, + const struct ecc_curve *curve, + struct ecc_point *pk) +{ + u64 xxx[KMB_ECC_VLI_MAX_DIGITS] = { 0 }; + u64 yy[KMB_ECC_VLI_MAX_DIGITS] = { 0 }; + u64 w[KMB_ECC_VLI_MAX_DIGITS] = { 0 }; + int rc; + + if (WARN_ON(pk->ndigits != curve->g.ndigits)) + return -EINVAL; + + /* Check 1: Verify key is not the zero point. */ + if (ecc_point_is_zero(pk)) + return -EINVAL; + + /* Check 2: Verify key is in the range [0, p-1]. */ + if (vli_cmp(curve->p, pk->x, pk->ndigits) != 1) + return -EINVAL; + + if (vli_cmp(curve->p, pk->y, pk->ndigits) != 1) + return -EINVAL; + + /* Check 3: Verify that y^2 == (x^3 + a·x + b) mod p */ + + /* y^2 */ + /* Compute y^2 -> store in yy */ + rc = kmb_ecc_do_scalar_op(ecc_dev, yy, pk->y, pk->y, curve, pk->ndigits, + OCS_ECC_INST_CALC_A_MUL_B_MODP); + if (rc) + goto exit; + + /* x^3 */ + /* Assigning w = 3, used for calculating x^3. */ + w[0] = POW_CUBE; + /* Load the next stage.*/ + rc = kmb_ecc_do_scalar_op(ecc_dev, xxx, pk->x, w, curve, pk->ndigits, + OCS_ECC_INST_CALC_A_POW_B_MODP); + if (rc) + goto exit; + + /* Do a*x -> store in w. */ + rc = kmb_ecc_do_scalar_op(ecc_dev, w, curve->a, pk->x, curve, + pk->ndigits, + OCS_ECC_INST_CALC_A_MUL_B_MODP); + if (rc) + goto exit; + + /* Do ax + b == w + b; store in w. */ + rc = kmb_ecc_do_scalar_op(ecc_dev, w, w, curve->b, curve, + pk->ndigits, + OCS_ECC_INST_CALC_A_ADD_B_MODP); + if (rc) + goto exit; + + /* x^3 + ax + b == x^3 + w -> store in w. */ + rc = kmb_ecc_do_scalar_op(ecc_dev, w, xxx, w, curve, pk->ndigits, + OCS_ECC_INST_CALC_A_ADD_B_MODP); + if (rc) + goto exit; + + /* Compare y^2 == x^3 + a·x + b. */ + rc = vli_cmp(yy, w, pk->ndigits); + if (rc) + rc = -EINVAL; + +exit: + memzero_explicit(xxx, sizeof(xxx)); + memzero_explicit(yy, sizeof(yy)); + memzero_explicit(w, sizeof(w)); + + return rc; +} + +/* SP800-56A section 5.6.2.3.3 full verification */ +static int kmb_ocs_ecc_is_pubkey_valid_full(struct ocs_ecc_dev *ecc_dev, + const struct ecc_curve *curve, + struct ecc_point *pk) +{ + struct ecc_point *nQ; + int rc; + + /* Checks 1 through 3 */ + rc = kmb_ocs_ecc_is_pubkey_valid_partial(ecc_dev, curve, pk); + if (rc) + return rc; + + /* Check 4: Verify that nQ is the zero point. */ + nQ = ecc_alloc_point(pk->ndigits); + if (!nQ) + return -ENOMEM; + + rc = kmb_ecc_point_mult(ecc_dev, nQ, pk, curve->n, curve); + if (rc) + goto exit; + + if (!ecc_point_is_zero(nQ)) + rc = -EINVAL; + +exit: + ecc_free_point(nQ); + + return rc; +} + +static int kmb_ecc_is_key_valid(const struct ecc_curve *curve, + const u64 *private_key, size_t private_key_len) +{ + size_t ndigits = curve->g.ndigits; + u64 one[KMB_ECC_VLI_MAX_DIGITS] = {1}; + u64 res[KMB_ECC_VLI_MAX_DIGITS]; + + if (private_key_len != digits_to_bytes(ndigits)) + return -EINVAL; + + if (!private_key) + return -EINVAL; + + /* Make sure the private key is in the range [2, n-3]. */ + if (vli_cmp(one, private_key, ndigits) != -1) + return -EINVAL; + + vli_sub(res, curve->n, one, ndigits); + vli_sub(res, res, one, ndigits); + if (vli_cmp(res, private_key, ndigits) != 1) + return -EINVAL; + + return 0; +} + +/* + * ECC private keys are generated using the method of extra random bits, + * equivalent to that described in FIPS 186-4, Appendix B.4.1. + * + * d = (c mod(n–1)) + 1 where c is a string of random bits, 64 bits longer + * than requested + * 0 <= c mod(n-1) <= n-2 and implies that + * 1 <= d <= n-1 + * + * This method generates a private key uniformly distributed in the range + * [1, n-1]. + */ +static int kmb_ecc_gen_privkey(const struct ecc_curve *curve, u64 *privkey) +{ + size_t nbytes = digits_to_bytes(curve->g.ndigits); + u64 priv[KMB_ECC_VLI_MAX_DIGITS]; + size_t nbits; + int rc; + + nbits = vli_num_bits(curve->n, curve->g.ndigits); + + /* Check that N is included in Table 1 of FIPS 186-4, section 6.1.1 */ + if (nbits < 160 || curve->g.ndigits > ARRAY_SIZE(priv)) + return -EINVAL; + + /* + * FIPS 186-4 recommends that the private key should be obtained from a + * RBG with a security strength equal to or greater than the security + * strength associated with N. + * + * The maximum security strength identified by NIST SP800-57pt1r4 for + * ECC is 256 (N >= 512). + * + * This condition is met by stdrng because it selects a favored DRBG + * with a security strength of 256. + */ + rc = crypto_stdrng_get_bytes(priv, nbytes); + if (rc) + goto cleanup; + + rc = kmb_ecc_is_key_valid(curve, priv, nbytes); + if (rc) + goto cleanup; + + ecc_swap_digits(priv, privkey, curve->g.ndigits); + +cleanup: + memzero_explicit(&priv, sizeof(priv)); + + return rc; +} + +static int kmb_ocs_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, + unsigned int len) +{ + struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); + struct ecdh params; + int rc = 0; + + rc = crypto_ecdh_decode_key(buf, len, ¶ms); + if (rc) + goto cleanup; + + /* Ensure key size is not bigger then expected. */ + if (params.key_size > digits_to_bytes(tctx->curve->g.ndigits)) { + rc = -EINVAL; + goto cleanup; + } + + /* Auto-generate private key is not provided. */ + if (!params.key || !params.key_size) { + rc = kmb_ecc_gen_privkey(tctx->curve, tctx->private_key); + goto cleanup; + } + + rc = kmb_ecc_is_key_valid(tctx->curve, (const u64 *)params.key, + params.key_size); + if (rc) + goto cleanup; + + ecc_swap_digits((const u64 *)params.key, tctx->private_key, + tctx->curve->g.ndigits); +cleanup: + memzero_explicit(¶ms, sizeof(params)); + + if (rc) + tctx->curve = NULL; + + return rc; +} + +/* Compute shared secret. */ +static int kmb_ecc_do_shared_secret(struct ocs_ecc_ctx *tctx, + struct kpp_request *req) +{ + struct ocs_ecc_dev *ecc_dev = tctx->ecc_dev; + const struct ecc_curve *curve = tctx->curve; + u64 shared_secret[KMB_ECC_VLI_MAX_DIGITS]; + u64 pubk_buf[KMB_ECC_VLI_MAX_DIGITS * 2]; + size_t copied, nbytes, pubk_len; + struct ecc_point *pk, *result; + int rc; + + nbytes = digits_to_bytes(curve->g.ndigits); + + /* Public key is a point, thus it has two coordinates */ + pubk_len = 2 * nbytes; + + /* Copy public key from SG list to pubk_buf. */ + copied = sg_copy_to_buffer(req->src, + sg_nents_for_len(req->src, pubk_len), + pubk_buf, pubk_len); + if (copied != pubk_len) + return -EINVAL; + + /* Allocate and initialize public key point. */ + pk = ecc_alloc_point(curve->g.ndigits); + if (!pk) + return -ENOMEM; + + ecc_swap_digits(pubk_buf, pk->x, curve->g.ndigits); + ecc_swap_digits(&pubk_buf[curve->g.ndigits], pk->y, curve->g.ndigits); + + /* + * Check the public key for following + * Check 1: Verify key is not the zero point. + * Check 2: Verify key is in the range [1, p-1]. + * Check 3: Verify that y^2 == (x^3 + a·x + b) mod p + */ + rc = kmb_ocs_ecc_is_pubkey_valid_partial(ecc_dev, curve, pk); + if (rc) + goto exit_free_pk; + + /* Allocate point for storing computed shared secret. */ + result = ecc_alloc_point(pk->ndigits); + if (!result) { + rc = -ENOMEM; + goto exit_free_pk; + } + + /* Calculate the shared secret.*/ + rc = kmb_ecc_point_mult(ecc_dev, result, pk, tctx->private_key, curve); + if (rc) + goto exit_free_result; + + if (ecc_point_is_zero(result)) { + rc = -EFAULT; + goto exit_free_result; + } + + /* Copy shared secret from point to buffer. */ + ecc_swap_digits(result->x, shared_secret, result->ndigits); + + /* Request might ask for less bytes than what we have. */ + nbytes = min_t(size_t, nbytes, req->dst_len); + + copied = sg_copy_from_buffer(req->dst, + sg_nents_for_len(req->dst, nbytes), + shared_secret, nbytes); + + if (copied != nbytes) + rc = -EINVAL; + + memzero_explicit(shared_secret, sizeof(shared_secret)); + +exit_free_result: + ecc_free_point(result); + +exit_free_pk: + ecc_free_point(pk); + + return rc; +} + +/* Compute public key. */ +static int kmb_ecc_do_public_key(struct ocs_ecc_ctx *tctx, + struct kpp_request *req) +{ + const struct ecc_curve *curve = tctx->curve; + u64 pubk_buf[KMB_ECC_VLI_MAX_DIGITS * 2]; + struct ecc_point *pk; + size_t pubk_len; + size_t copied; + int rc; + + /* Public key is a point, so it has double the digits. */ + pubk_len = 2 * digits_to_bytes(curve->g.ndigits); + + pk = ecc_alloc_point(curve->g.ndigits); + if (!pk) + return -ENOMEM; + + /* Public Key(pk) = priv * G. */ + rc = kmb_ecc_point_mult(tctx->ecc_dev, pk, &curve->g, tctx->private_key, + curve); + if (rc) + goto exit; + + /* SP800-56A rev 3 5.6.2.1.3 key check */ + if (kmb_ocs_ecc_is_pubkey_valid_full(tctx->ecc_dev, curve, pk)) { + rc = -EAGAIN; + goto exit; + } + + /* Copy public key from point to buffer. */ + ecc_swap_digits(pk->x, pubk_buf, pk->ndigits); + ecc_swap_digits(pk->y, &pubk_buf[pk->ndigits], pk->ndigits); + + /* Copy public key to req->dst. */ + copied = sg_copy_from_buffer(req->dst, + sg_nents_for_len(req->dst, pubk_len), + pubk_buf, pubk_len); + + if (copied != pubk_len) + rc = -EINVAL; + +exit: + ecc_free_point(pk); + + return rc; +} + +static int kmb_ocs_ecc_do_one_request(struct crypto_engine *engine, + void *areq) +{ + struct kpp_request *req = container_of(areq, struct kpp_request, base); + struct ocs_ecc_ctx *tctx = kmb_ocs_ecc_tctx(req); + struct ocs_ecc_dev *ecc_dev = tctx->ecc_dev; + int rc; + + if (req->src) + rc = kmb_ecc_do_shared_secret(tctx, req); + else + rc = kmb_ecc_do_public_key(tctx, req); + + crypto_finalize_kpp_request(ecc_dev->engine, req, rc); + + return 0; +} + +static int kmb_ocs_ecdh_generate_public_key(struct kpp_request *req) +{ + struct ocs_ecc_ctx *tctx = kmb_ocs_ecc_tctx(req); + const struct ecc_curve *curve = tctx->curve; + + /* Ensure kmb_ocs_ecdh_set_secret() has been successfully called. */ + if (!tctx->curve) + return -EINVAL; + + /* Ensure dst is present. */ + if (!req->dst) + return -EINVAL; + + /* Check the request dst is big enough to hold the public key. */ + if (req->dst_len < (2 * digits_to_bytes(curve->g.ndigits))) + return -EINVAL; + + /* 'src' is not supposed to be present when generate pubk is called. */ + if (req->src) + return -EINVAL; + + return crypto_transfer_kpp_request_to_engine(tctx->ecc_dev->engine, + req); +} + +static int kmb_ocs_ecdh_compute_shared_secret(struct kpp_request *req) +{ + struct ocs_ecc_ctx *tctx = kmb_ocs_ecc_tctx(req); + const struct ecc_curve *curve = tctx->curve; + + /* Ensure kmb_ocs_ecdh_set_secret() has been successfully called. */ + if (!tctx->curve) + return -EINVAL; + + /* Ensure dst is present. */ + if (!req->dst) + return -EINVAL; + + /* Ensure src is present. */ + if (!req->src) + return -EINVAL; + + /* + * req->src is expected to the (other-side) public key, so its length + * must be 2 * coordinate size (in bytes). + */ + if (req->src_len != 2 * digits_to_bytes(curve->g.ndigits)) + return -EINVAL; + + return crypto_transfer_kpp_request_to_engine(tctx->ecc_dev->engine, + req); +} + +static int kmb_ecc_tctx_init(struct ocs_ecc_ctx *tctx, unsigned int curve_id) +{ + memset(tctx, 0, sizeof(*tctx)); + + tctx->ecc_dev = kmb_ocs_ecc_find_dev(tctx); + + if (IS_ERR(tctx->ecc_dev)) { + pr_err("Failed to find the device : %ld\n", + PTR_ERR(tctx->ecc_dev)); + return PTR_ERR(tctx->ecc_dev); + } + + tctx->curve = ecc_get_curve(curve_id); + if (!tctx->curve) + return -EOPNOTSUPP; + + return 0; +} + +static int kmb_ocs_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) +{ + struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); + + return kmb_ecc_tctx_init(tctx, ECC_CURVE_NIST_P256); +} + +static int kmb_ocs_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm) +{ + struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); + + return kmb_ecc_tctx_init(tctx, ECC_CURVE_NIST_P384); +} + +static void kmb_ocs_ecdh_exit_tfm(struct crypto_kpp *tfm) +{ + struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); + + memzero_explicit(tctx->private_key, sizeof(*tctx->private_key)); +} + +static unsigned int kmb_ocs_ecdh_max_size(struct crypto_kpp *tfm) +{ + struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); + + /* Public key is made of two coordinates, so double the digits. */ + return digits_to_bytes(tctx->curve->g.ndigits) * 2; +} + +static struct kpp_engine_alg ocs_ecdh_p256 = { + .base.set_secret = kmb_ocs_ecdh_set_secret, + .base.generate_public_key = kmb_ocs_ecdh_generate_public_key, + .base.compute_shared_secret = kmb_ocs_ecdh_compute_shared_secret, + .base.init = kmb_ocs_ecdh_nist_p256_init_tfm, + .base.exit = kmb_ocs_ecdh_exit_tfm, + .base.max_size = kmb_ocs_ecdh_max_size, + .base.base = { + .cra_name = "ecdh-nist-p256", + .cra_driver_name = "ecdh-nist-p256-keembay-ocs", + .cra_priority = KMB_OCS_ECC_PRIORITY, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct ocs_ecc_ctx), + }, + .op.do_one_request = kmb_ocs_ecc_do_one_request, +}; + +static struct kpp_engine_alg ocs_ecdh_p384 = { + .base.set_secret = kmb_ocs_ecdh_set_secret, + .base.generate_public_key = kmb_ocs_ecdh_generate_public_key, + .base.compute_shared_secret = kmb_ocs_ecdh_compute_shared_secret, + .base.init = kmb_ocs_ecdh_nist_p384_init_tfm, + .base.exit = kmb_ocs_ecdh_exit_tfm, + .base.max_size = kmb_ocs_ecdh_max_size, + .base.base = { + .cra_name = "ecdh-nist-p384", + .cra_driver_name = "ecdh-nist-p384-keembay-ocs", + .cra_priority = KMB_OCS_ECC_PRIORITY, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct ocs_ecc_ctx), + }, + .op.do_one_request = kmb_ocs_ecc_do_one_request, +}; + +static irqreturn_t ocs_ecc_irq_handler(int irq, void *dev_id) +{ + struct ocs_ecc_dev *ecc_dev = dev_id; + u32 status; + + /* + * Read the status register and write it back to clear the + * DONE_INT_STATUS bit. + */ + status = ioread32(ecc_dev->base_reg + HW_OFFS_OCS_ECC_ISR); + iowrite32(status, ecc_dev->base_reg + HW_OFFS_OCS_ECC_ISR); + + if (!(status & HW_OCS_ECC_ISR_INT_STATUS_DONE)) + return IRQ_NONE; + + complete(&ecc_dev->irq_done); + + return IRQ_HANDLED; +} + +static int kmb_ocs_ecc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct ocs_ecc_dev *ecc_dev; + int rc; + + ecc_dev = devm_kzalloc(dev, sizeof(*ecc_dev), GFP_KERNEL); + if (!ecc_dev) + return -ENOMEM; + + ecc_dev->dev = dev; + + platform_set_drvdata(pdev, ecc_dev); + + INIT_LIST_HEAD(&ecc_dev->list); + init_completion(&ecc_dev->irq_done); + + /* Get base register address. */ + ecc_dev->base_reg = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(ecc_dev->base_reg)) { + dev_err(dev, "Failed to get base address\n"); + rc = PTR_ERR(ecc_dev->base_reg); + goto list_del; + } + + /* Get and request IRQ */ + ecc_dev->irq = platform_get_irq(pdev, 0); + if (ecc_dev->irq < 0) { + rc = ecc_dev->irq; + goto list_del; + } + + rc = devm_request_threaded_irq(dev, ecc_dev->irq, ocs_ecc_irq_handler, + NULL, 0, "keembay-ocs-ecc", ecc_dev); + if (rc < 0) + goto list_del; + + /* Add device to the list of OCS ECC devices. */ + spin_lock(&ocs_ecc.lock); + list_add_tail(&ecc_dev->list, &ocs_ecc.dev_list); + spin_unlock(&ocs_ecc.lock); + + /* Initialize crypto engine. */ + ecc_dev->engine = crypto_engine_alloc_init(dev, 1); + if (!ecc_dev->engine) { + dev_err(dev, "Could not allocate crypto engine\n"); + rc = -ENOMEM; + goto list_del; + } + + rc = crypto_engine_start(ecc_dev->engine); + if (rc) { + dev_err(dev, "Could not start crypto engine\n"); + goto cleanup; + } + + /* Register the KPP algo. */ + rc = crypto_engine_register_kpp(&ocs_ecdh_p256); + if (rc) { + dev_err(dev, + "Could not register OCS algorithms with Crypto API\n"); + goto cleanup; + } + + rc = crypto_engine_register_kpp(&ocs_ecdh_p384); + if (rc) { + dev_err(dev, + "Could not register OCS algorithms with Crypto API\n"); + goto ocs_ecdh_p384_error; + } + + return 0; + +ocs_ecdh_p384_error: + crypto_engine_unregister_kpp(&ocs_ecdh_p256); + +cleanup: + crypto_engine_exit(ecc_dev->engine); + +list_del: + spin_lock(&ocs_ecc.lock); + list_del(&ecc_dev->list); + spin_unlock(&ocs_ecc.lock); + + return rc; +} + +static void kmb_ocs_ecc_remove(struct platform_device *pdev) +{ + struct ocs_ecc_dev *ecc_dev; + + ecc_dev = platform_get_drvdata(pdev); + + crypto_engine_unregister_kpp(&ocs_ecdh_p384); + crypto_engine_unregister_kpp(&ocs_ecdh_p256); + + spin_lock(&ocs_ecc.lock); + list_del(&ecc_dev->list); + spin_unlock(&ocs_ecc.lock); + + crypto_engine_exit(ecc_dev->engine); +} + +/* Device tree driver match. */ +static const struct of_device_id kmb_ocs_ecc_of_match[] = { + { + .compatible = "intel,keembay-ocs-ecc", + }, + {} +}; +MODULE_DEVICE_TABLE(of, kmb_ocs_ecc_of_match); + +/* The OCS driver is a platform device. */ +static struct platform_driver kmb_ocs_ecc_driver = { + .probe = kmb_ocs_ecc_probe, + .remove = kmb_ocs_ecc_remove, + .driver = { + .name = DRV_NAME, + .of_match_table = kmb_ocs_ecc_of_match, + }, +}; +module_platform_driver(kmb_ocs_ecc_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Intel Keem Bay OCS ECC Driver"); +MODULE_ALIAS_CRYPTO("ecdh-nist-p256"); +MODULE_ALIAS_CRYPTO("ecdh-nist-p384"); +MODULE_ALIAS_CRYPTO("ecdh-nist-p256-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ecdh-nist-p384-keembay-ocs"); diff --git a/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c b/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c new file mode 100644 index 000000000..9a67bb4ec --- /dev/null +++ b/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c @@ -0,0 +1,1261 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel Keem Bay OCS HCU Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ocs-hcu.h" + +#define DRV_NAME "keembay-ocs-hcu" + +/* Flag marking a final request. */ +#define REQ_FINAL BIT(0) +/* Flag marking a HMAC request. */ +#define REQ_FLAGS_HMAC BIT(1) +/* Flag set when HW HMAC is being used. */ +#define REQ_FLAGS_HMAC_HW BIT(2) +/* Flag set when SW HMAC is being used. */ +#define REQ_FLAGS_HMAC_SW BIT(3) + +/** + * struct ocs_hcu_ctx: OCS HCU Transform context. + * @hcu_dev: The OCS HCU device used by the transformation. + * @key: The key (used only for HMAC transformations). + * @key_len: The length of the key. + * @is_sm3_tfm: Whether or not this is an SM3 transformation. + * @is_hmac_tfm: Whether or not this is a HMAC transformation. + */ +struct ocs_hcu_ctx { + struct ocs_hcu_dev *hcu_dev; + u8 key[SHA512_BLOCK_SIZE]; + size_t key_len; + bool is_sm3_tfm; + bool is_hmac_tfm; +}; + +/** + * struct ocs_hcu_rctx - Context for the request. + * @hcu_dev: OCS HCU device to be used to service the request. + * @flags: Flags tracking request status. + * @algo: Algorithm to use for the request. + * @blk_sz: Block size of the transformation / request. + * @dig_sz: Digest size of the transformation / request. + * @dma_list: OCS DMA linked list. + * @hash_ctx: OCS HCU hashing context. + * @buffer: Buffer to store: partial block of data and SW HMAC + * artifacts (ipad, opad, etc.). + * @buf_cnt: Number of bytes currently stored in the buffer. + * @buf_dma_addr: The DMA address of @buffer (when mapped). + * @buf_dma_count: The number of bytes in @buffer currently DMA-mapped. + * @sg: Head of the scatterlist entries containing data. + * @sg_data_total: Total data in the SG list at any time. + * @sg_data_offset: Offset into the data of the current individual SG node. + * @sg_dma_nents: Number of sg entries mapped in dma_list. + * @nents: Number of entries in the scatterlist. + */ +struct ocs_hcu_rctx { + struct ocs_hcu_dev *hcu_dev; + u32 flags; + enum ocs_hcu_algo algo; + size_t blk_sz; + size_t dig_sz; + struct ocs_hcu_dma_list *dma_list; + struct ocs_hcu_hash_ctx hash_ctx; + /* + * Buffer is double the block size because we need space for SW HMAC + * artifacts, i.e: + * - ipad (1 block) + a possible partial block of data. + * - opad (1 block) + digest of H(k ^ ipad || m) + */ + u8 buffer[2 * SHA512_BLOCK_SIZE]; + size_t buf_cnt; + dma_addr_t buf_dma_addr; + size_t buf_dma_count; + struct scatterlist *sg; + unsigned int sg_data_total; + unsigned int sg_data_offset; + unsigned int sg_dma_nents; + unsigned int nents; +}; + +/** + * struct ocs_hcu_drv - Driver data + * @dev_list: The list of HCU devices. + * @lock: The lock protecting dev_list. + */ +struct ocs_hcu_drv { + struct list_head dev_list; + spinlock_t lock; /* Protects dev_list. */ +}; + +static struct ocs_hcu_drv ocs_hcu = { + .dev_list = LIST_HEAD_INIT(ocs_hcu.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(ocs_hcu.lock), +}; + +/* + * Return the total amount of data in the request; that is: the data in the + * request buffer + the data in the sg list. + */ +static inline unsigned int kmb_get_total_data(struct ocs_hcu_rctx *rctx) +{ + return rctx->sg_data_total + rctx->buf_cnt; +} + +/* Move remaining content of scatter-gather list to context buffer. */ +static int flush_sg_to_ocs_buffer(struct ocs_hcu_rctx *rctx) +{ + size_t count; + + if (rctx->sg_data_total > (sizeof(rctx->buffer) - rctx->buf_cnt)) { + WARN(1, "%s: sg data does not fit in buffer\n", __func__); + return -EINVAL; + } + + while (rctx->sg_data_total) { + if (!rctx->sg) { + WARN(1, "%s: unexpected NULL sg\n", __func__); + return -EINVAL; + } + /* + * If current sg has been fully processed, skip to the next + * one. + */ + if (rctx->sg_data_offset == rctx->sg->length) { + rctx->sg = sg_next(rctx->sg); + rctx->sg_data_offset = 0; + continue; + } + /* + * Determine the maximum data available to copy from the node. + * Minimum of the length left in the sg node, or the total data + * in the request. + */ + count = min(rctx->sg->length - rctx->sg_data_offset, + rctx->sg_data_total); + /* Copy from scatter-list entry to context buffer. */ + scatterwalk_map_and_copy(&rctx->buffer[rctx->buf_cnt], + rctx->sg, rctx->sg_data_offset, + count, 0); + + rctx->sg_data_offset += count; + rctx->sg_data_total -= count; + rctx->buf_cnt += count; + } + + return 0; +} + +static struct ocs_hcu_dev *kmb_ocs_hcu_find_dev(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ocs_hcu_ctx *tctx = crypto_ahash_ctx(tfm); + + /* If the HCU device for the request was previously set, return it. */ + if (tctx->hcu_dev) + return tctx->hcu_dev; + + /* + * Otherwise, get the first HCU device available (there should be one + * and only one device). + */ + spin_lock_bh(&ocs_hcu.lock); + tctx->hcu_dev = list_first_entry_or_null(&ocs_hcu.dev_list, + struct ocs_hcu_dev, + list); + spin_unlock_bh(&ocs_hcu.lock); + + return tctx->hcu_dev; +} + +/* Free OCS DMA linked list and DMA-able context buffer. */ +static void kmb_ocs_hcu_dma_cleanup(struct ahash_request *req, + struct ocs_hcu_rctx *rctx) +{ + struct ocs_hcu_dev *hcu_dev = rctx->hcu_dev; + struct device *dev = hcu_dev->dev; + + /* Unmap rctx->buffer (if mapped). */ + if (rctx->buf_dma_count) { + dma_unmap_single(dev, rctx->buf_dma_addr, rctx->buf_dma_count, + DMA_TO_DEVICE); + rctx->buf_dma_count = 0; + } + + /* Unmap req->src (if mapped). */ + if (rctx->sg_dma_nents) { + dma_unmap_sg(dev, req->src, rctx->nents, DMA_TO_DEVICE); + rctx->sg_dma_nents = 0; + } + + /* Free dma_list (if allocated). */ + if (rctx->dma_list) { + ocs_hcu_dma_list_free(hcu_dev, rctx->dma_list); + rctx->dma_list = NULL; + } +} + +/* + * Prepare for DMA operation: + * - DMA-map request context buffer (if needed) + * - DMA-map SG list (only the entries to be processed, see note below) + * - Allocate OCS HCU DMA linked list (number of elements = SG entries to + * process + context buffer (if not empty)). + * - Add DMA-mapped request context buffer to OCS HCU DMA list. + * - Add SG entries to DMA list. + * + * Note: if this is a final request, we process all the data in the SG list, + * otherwise we can only process up to the maximum amount of block-aligned data + * (the remainder will be put into the context buffer and processed in the next + * request). + */ +static int kmb_ocs_dma_prepare(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); + struct device *dev = rctx->hcu_dev->dev; + unsigned int remainder = 0; + unsigned int total; + int nents; + size_t count; + int rc; + int i; + + /* This function should be called only when there is data to process. */ + total = kmb_get_total_data(rctx); + if (!total) + return -EINVAL; + + /* + * If this is not a final DMA (terminated DMA), the data passed to the + * HCU must be aligned to the block size; compute the remainder data to + * be processed in the next request. + */ + if (!(rctx->flags & REQ_FINAL)) + remainder = total % rctx->blk_sz; + + /* Determine the number of scatter gather list entries to process. */ + nents = sg_nents_for_len(req->src, rctx->sg_data_total - remainder); + + if (nents < 0) + return nents; + + /* If there are entries to process, map them. */ + if (nents) { + rctx->sg_dma_nents = dma_map_sg(dev, req->src, nents, + DMA_TO_DEVICE); + if (!rctx->sg_dma_nents) { + dev_err(dev, "Failed to MAP SG\n"); + rc = -ENOMEM; + goto cleanup; + } + + /* Save the value of nents to pass to dma_unmap_sg. */ + rctx->nents = nents; + + /* + * The value returned by dma_map_sg() can be < nents; so update + * nents accordingly. + */ + nents = rctx->sg_dma_nents; + } + + /* + * If context buffer is not empty, map it and add extra DMA entry for + * it. + */ + if (rctx->buf_cnt) { + rctx->buf_dma_addr = dma_map_single(dev, rctx->buffer, + rctx->buf_cnt, + DMA_TO_DEVICE); + if (dma_mapping_error(dev, rctx->buf_dma_addr)) { + dev_err(dev, "Failed to map request context buffer\n"); + rc = -ENOMEM; + goto cleanup; + } + rctx->buf_dma_count = rctx->buf_cnt; + /* Increase number of dma entries. */ + nents++; + } + + /* Allocate OCS HCU DMA list. */ + rctx->dma_list = ocs_hcu_dma_list_alloc(rctx->hcu_dev, nents); + if (!rctx->dma_list) { + rc = -ENOMEM; + goto cleanup; + } + + /* Add request context buffer (if previously DMA-mapped) */ + if (rctx->buf_dma_count) { + rc = ocs_hcu_dma_list_add_tail(rctx->hcu_dev, rctx->dma_list, + rctx->buf_dma_addr, + rctx->buf_dma_count); + if (rc) + goto cleanup; + } + + /* Add the SG nodes to be processed to the DMA linked list. */ + for_each_sg(req->src, rctx->sg, rctx->sg_dma_nents, i) { + /* + * The number of bytes to add to the list entry is the minimum + * between: + * - The DMA length of the SG entry. + * - The data left to be processed. + */ + count = min(rctx->sg_data_total - remainder, + sg_dma_len(rctx->sg) - rctx->sg_data_offset); + /* + * Do not create a zero length DMA descriptor. Check in case of + * zero length SG node. + */ + if (count == 0) + continue; + /* Add sg to HCU DMA list. */ + rc = ocs_hcu_dma_list_add_tail(rctx->hcu_dev, + rctx->dma_list, + rctx->sg->dma_address, + count); + if (rc) + goto cleanup; + + /* Update amount of data remaining in SG list. */ + rctx->sg_data_total -= count; + + /* + * If remaining data is equal to remainder (note: 'less than' + * case should never happen in practice), we are done: update + * offset and exit the loop. + */ + if (rctx->sg_data_total <= remainder) { + WARN_ON(rctx->sg_data_total < remainder); + rctx->sg_data_offset += count; + break; + } + + /* + * If we get here is because we need to process the next sg in + * the list; set offset within the sg to 0. + */ + rctx->sg_data_offset = 0; + } + + return 0; +cleanup: + dev_err(dev, "Failed to prepare DMA.\n"); + kmb_ocs_hcu_dma_cleanup(req, rctx); + + return rc; +} + +static void kmb_ocs_hcu_secure_cleanup(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); + + /* Clear buffer of any data. */ + memzero_explicit(rctx->buffer, sizeof(rctx->buffer)); +} + +static int kmb_ocs_hcu_handle_queue(struct ahash_request *req) +{ + struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req); + + if (!hcu_dev) + return -ENOENT; + + return crypto_transfer_hash_request_to_engine(hcu_dev->engine, req); +} + +static int prepare_ipad(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm); + int i; + + WARN(rctx->buf_cnt, "%s: Context buffer is not empty\n", __func__); + WARN(!(rctx->flags & REQ_FLAGS_HMAC_SW), + "%s: HMAC_SW flag is not set\n", __func__); + /* + * Key length must be equal to block size. If key is shorter, + * we pad it with zero (note: key cannot be longer, since + * longer keys are hashed by kmb_ocs_hcu_setkey()). + */ + if (ctx->key_len > rctx->blk_sz) { + WARN(1, "%s: Invalid key length in tfm context\n", __func__); + return -EINVAL; + } + memzero_explicit(&ctx->key[ctx->key_len], + rctx->blk_sz - ctx->key_len); + ctx->key_len = rctx->blk_sz; + /* + * Prepare IPAD for HMAC. Only done for first block. + * HMAC(k,m) = H(k ^ opad || H(k ^ ipad || m)) + * k ^ ipad will be first hashed block. + * k ^ opad will be calculated in the final request. + * Only needed if not using HW HMAC. + */ + for (i = 0; i < rctx->blk_sz; i++) + rctx->buffer[i] = ctx->key[i] ^ HMAC_IPAD_VALUE; + rctx->buf_cnt = rctx->blk_sz; + + return 0; +} + +static int kmb_ocs_hcu_do_one_request(struct crypto_engine *engine, void *areq) +{ + struct ahash_request *req = container_of(areq, struct ahash_request, + base); + struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); + struct ocs_hcu_ctx *tctx = crypto_ahash_ctx(tfm); + int rc; + int i; + + if (!hcu_dev) { + rc = -ENOENT; + goto error; + } + + /* + * If hardware HMAC flag is set, perform HMAC in hardware. + * + * NOTE: this flag implies REQ_FINAL && kmb_get_total_data(rctx) + */ + if (rctx->flags & REQ_FLAGS_HMAC_HW) { + /* Map input data into the HCU DMA linked list. */ + rc = kmb_ocs_dma_prepare(req); + if (rc) + goto error; + + rc = ocs_hcu_hmac(hcu_dev, rctx->algo, tctx->key, tctx->key_len, + rctx->dma_list, req->result, rctx->dig_sz); + + /* Unmap data and free DMA list regardless of return code. */ + kmb_ocs_hcu_dma_cleanup(req, rctx); + + /* Process previous return code. */ + if (rc) + goto error; + + goto done; + } + + /* Handle update request case. */ + if (!(rctx->flags & REQ_FINAL)) { + /* Update should always have input data. */ + if (!kmb_get_total_data(rctx)) + return -EINVAL; + + /* Map input data into the HCU DMA linked list. */ + rc = kmb_ocs_dma_prepare(req); + if (rc) + goto error; + + /* Do hashing step. */ + rc = ocs_hcu_hash_update(hcu_dev, &rctx->hash_ctx, + rctx->dma_list); + + /* Unmap data and free DMA list regardless of return code. */ + kmb_ocs_hcu_dma_cleanup(req, rctx); + + /* Process previous return code. */ + if (rc) + goto error; + + /* + * Reset request buffer count (data in the buffer was just + * processed). + */ + rctx->buf_cnt = 0; + /* + * Move remaining sg data into the request buffer, so that it + * will be processed during the next request. + * + * NOTE: we have remaining data if kmb_get_total_data() was not + * a multiple of block size. + */ + rc = flush_sg_to_ocs_buffer(rctx); + if (rc) + goto error; + + goto done; + } + + /* If we get here, this is a final request. */ + + /* If there is data to process, use finup. */ + if (kmb_get_total_data(rctx)) { + /* Map input data into the HCU DMA linked list. */ + rc = kmb_ocs_dma_prepare(req); + if (rc) + goto error; + + /* Do hashing step. */ + rc = ocs_hcu_hash_finup(hcu_dev, &rctx->hash_ctx, + rctx->dma_list, + req->result, rctx->dig_sz); + /* Free DMA list regardless of return code. */ + kmb_ocs_hcu_dma_cleanup(req, rctx); + + /* Process previous return code. */ + if (rc) + goto error; + + } else { /* Otherwise (if we have no data), use final. */ + rc = ocs_hcu_hash_final(hcu_dev, &rctx->hash_ctx, req->result, + rctx->dig_sz); + if (rc) + goto error; + } + + /* + * If we are finalizing a SW HMAC request, we just computed the result + * of: H(k ^ ipad || m). + * + * We now need to complete the HMAC calculation with the OPAD step, + * that is, we need to compute H(k ^ opad || digest), where digest is + * the digest we just obtained, i.e., H(k ^ ipad || m). + */ + if (rctx->flags & REQ_FLAGS_HMAC_SW) { + /* + * Compute k ^ opad and store it in the request buffer (which + * is not used anymore at this point). + * Note: key has been padded / hashed already (so keylen == + * blksz) . + */ + WARN_ON(tctx->key_len != rctx->blk_sz); + for (i = 0; i < rctx->blk_sz; i++) + rctx->buffer[i] = tctx->key[i] ^ HMAC_OPAD_VALUE; + /* Now append the digest to the rest of the buffer. */ + for (i = 0; (i < rctx->dig_sz); i++) + rctx->buffer[rctx->blk_sz + i] = req->result[i]; + + /* Now hash the buffer to obtain the final HMAC. */ + rc = ocs_hcu_digest(hcu_dev, rctx->algo, rctx->buffer, + rctx->blk_sz + rctx->dig_sz, req->result, + rctx->dig_sz); + if (rc) + goto error; + } + + /* Perform secure clean-up. */ + kmb_ocs_hcu_secure_cleanup(req); +done: + crypto_finalize_hash_request(hcu_dev->engine, req, 0); + + return 0; + +error: + kmb_ocs_hcu_secure_cleanup(req); + return rc; +} + +static int kmb_ocs_hcu_init(struct ahash_request *req) +{ + struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req); + struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm); + + if (!hcu_dev) + return -ENOENT; + + /* Initialize entire request context to zero. */ + memset(rctx, 0, sizeof(*rctx)); + + rctx->hcu_dev = hcu_dev; + rctx->dig_sz = crypto_ahash_digestsize(tfm); + + switch (rctx->dig_sz) { +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 + case SHA224_DIGEST_SIZE: + rctx->blk_sz = SHA224_BLOCK_SIZE; + rctx->algo = OCS_HCU_ALGO_SHA224; + break; +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */ + case SHA256_DIGEST_SIZE: + rctx->blk_sz = SHA256_BLOCK_SIZE; + /* + * SHA256 and SM3 have the same digest size: use info from tfm + * context to find out which one we should use. + */ + rctx->algo = ctx->is_sm3_tfm ? OCS_HCU_ALGO_SM3 : + OCS_HCU_ALGO_SHA256; + break; + case SHA384_DIGEST_SIZE: + rctx->blk_sz = SHA384_BLOCK_SIZE; + rctx->algo = OCS_HCU_ALGO_SHA384; + break; + case SHA512_DIGEST_SIZE: + rctx->blk_sz = SHA512_BLOCK_SIZE; + rctx->algo = OCS_HCU_ALGO_SHA512; + break; + default: + return -EINVAL; + } + + /* Initialize intermediate data. */ + ocs_hcu_hash_init(&rctx->hash_ctx, rctx->algo); + + /* If this a HMAC request, set HMAC flag. */ + if (ctx->is_hmac_tfm) + rctx->flags |= REQ_FLAGS_HMAC; + + return 0; +} + +static int kmb_ocs_hcu_update(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); + int rc; + + if (!req->nbytes) + return 0; + + rctx->sg_data_total = req->nbytes; + rctx->sg_data_offset = 0; + rctx->sg = req->src; + + /* + * If we are doing HMAC, then we must use SW-assisted HMAC, since HW + * HMAC does not support context switching (there it can only be used + * with finup() or digest()). + */ + if (rctx->flags & REQ_FLAGS_HMAC && + !(rctx->flags & REQ_FLAGS_HMAC_SW)) { + rctx->flags |= REQ_FLAGS_HMAC_SW; + rc = prepare_ipad(req); + if (rc) + return rc; + } + + /* + * If remaining sg_data fits into ctx buffer, just copy it there; we'll + * process it at the next update() or final(). + */ + if (rctx->sg_data_total <= (sizeof(rctx->buffer) - rctx->buf_cnt)) + return flush_sg_to_ocs_buffer(rctx); + + return kmb_ocs_hcu_handle_queue(req); +} + +/* Common logic for kmb_ocs_hcu_final() and kmb_ocs_hcu_finup(). */ +static int kmb_ocs_hcu_fin_common(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm); + int rc; + + rctx->flags |= REQ_FINAL; + + /* + * If this is a HMAC request and, so far, we didn't have to switch to + * SW HMAC, check if we can use HW HMAC. + */ + if (rctx->flags & REQ_FLAGS_HMAC && + !(rctx->flags & REQ_FLAGS_HMAC_SW)) { + /* + * If we are here, it means we never processed any data so far, + * so we can use HW HMAC, but only if there is some data to + * process (since OCS HW MAC does not support zero-length + * messages) and the key length is supported by the hardware + * (OCS HCU HW only supports length <= 64); if HW HMAC cannot + * be used, fall back to SW-assisted HMAC. + */ + if (kmb_get_total_data(rctx) && + ctx->key_len <= OCS_HCU_HW_KEY_LEN) { + rctx->flags |= REQ_FLAGS_HMAC_HW; + } else { + rctx->flags |= REQ_FLAGS_HMAC_SW; + rc = prepare_ipad(req); + if (rc) + return rc; + } + } + + return kmb_ocs_hcu_handle_queue(req); +} + +static int kmb_ocs_hcu_final(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); + + rctx->sg_data_total = 0; + rctx->sg_data_offset = 0; + rctx->sg = NULL; + + return kmb_ocs_hcu_fin_common(req); +} + +static int kmb_ocs_hcu_finup(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); + + rctx->sg_data_total = req->nbytes; + rctx->sg_data_offset = 0; + rctx->sg = req->src; + + return kmb_ocs_hcu_fin_common(req); +} + +static int kmb_ocs_hcu_digest(struct ahash_request *req) +{ + int rc = 0; + struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req); + + if (!hcu_dev) + return -ENOENT; + + rc = kmb_ocs_hcu_init(req); + if (rc) + return rc; + + rc = kmb_ocs_hcu_finup(req); + + return rc; +} + +static int kmb_ocs_hcu_export(struct ahash_request *req, void *out) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); + + /* Intermediate data is always stored and applied per request. */ + memcpy(out, rctx, sizeof(*rctx)); + + return 0; +} + +static int kmb_ocs_hcu_import(struct ahash_request *req, const void *in) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); + + /* Intermediate data is always stored and applied per request. */ + memcpy(rctx, in, sizeof(*rctx)); + + return 0; +} + +static int kmb_ocs_hcu_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + unsigned int digestsize = crypto_ahash_digestsize(tfm); + struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm); + size_t blk_sz = crypto_ahash_blocksize(tfm); + struct crypto_ahash *ahash_tfm; + struct ahash_request *req; + struct crypto_wait wait; + struct scatterlist sg; + const char *alg_name; + int rc; + + /* + * Key length must be equal to block size: + * - If key is shorter, we are done for now (the key will be padded + * later on); this is to maximize the use of HW HMAC (which works + * only for keys <= 64 bytes). + * - If key is longer, we hash it. + */ + if (keylen <= blk_sz) { + memcpy(ctx->key, key, keylen); + ctx->key_len = keylen; + return 0; + } + + switch (digestsize) { +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 + case SHA224_DIGEST_SIZE: + alg_name = "sha224-keembay-ocs"; + break; +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */ + case SHA256_DIGEST_SIZE: + alg_name = ctx->is_sm3_tfm ? "sm3-keembay-ocs" : + "sha256-keembay-ocs"; + break; + case SHA384_DIGEST_SIZE: + alg_name = "sha384-keembay-ocs"; + break; + case SHA512_DIGEST_SIZE: + alg_name = "sha512-keembay-ocs"; + break; + default: + return -EINVAL; + } + + ahash_tfm = crypto_alloc_ahash(alg_name, 0, 0); + if (IS_ERR(ahash_tfm)) + return PTR_ERR(ahash_tfm); + + req = ahash_request_alloc(ahash_tfm, GFP_KERNEL); + if (!req) { + rc = -ENOMEM; + goto err_free_ahash; + } + + crypto_init_wait(&wait); + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + crypto_ahash_clear_flags(ahash_tfm, ~0); + + sg_init_one(&sg, key, keylen); + ahash_request_set_crypt(req, &sg, ctx->key, keylen); + + rc = crypto_wait_req(crypto_ahash_digest(req), &wait); + if (rc == 0) + ctx->key_len = digestsize; + + ahash_request_free(req); +err_free_ahash: + crypto_free_ahash(ahash_tfm); + + return rc; +} + +/* Set request size and initialize tfm context. */ +static void __cra_init(struct crypto_tfm *tfm, struct ocs_hcu_ctx *ctx) +{ + crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm), + sizeof(struct ocs_hcu_rctx)); +} + +static int kmb_ocs_hcu_sha_cra_init(struct crypto_tfm *tfm) +{ + struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); + + __cra_init(tfm, ctx); + + return 0; +} + +static int kmb_ocs_hcu_sm3_cra_init(struct crypto_tfm *tfm) +{ + struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); + + __cra_init(tfm, ctx); + + ctx->is_sm3_tfm = true; + + return 0; +} + +static int kmb_ocs_hcu_hmac_sm3_cra_init(struct crypto_tfm *tfm) +{ + struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); + + __cra_init(tfm, ctx); + + ctx->is_sm3_tfm = true; + ctx->is_hmac_tfm = true; + + return 0; +} + +static int kmb_ocs_hcu_hmac_cra_init(struct crypto_tfm *tfm) +{ + struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); + + __cra_init(tfm, ctx); + + ctx->is_hmac_tfm = true; + + return 0; +} + +/* Function called when 'tfm' is de-initialized. */ +static void kmb_ocs_hcu_hmac_cra_exit(struct crypto_tfm *tfm) +{ + struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); + + /* Clear the key. */ + memzero_explicit(ctx->key, sizeof(ctx->key)); +} + +static struct ahash_engine_alg ocs_hcu_algs[] = { +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 +{ + .base.init = kmb_ocs_hcu_init, + .base.update = kmb_ocs_hcu_update, + .base.final = kmb_ocs_hcu_final, + .base.finup = kmb_ocs_hcu_finup, + .base.digest = kmb_ocs_hcu_digest, + .base.export = kmb_ocs_hcu_export, + .base.import = kmb_ocs_hcu_import, + .base.halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "sha224", + .cra_driver_name = "sha224-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_sha_cra_init, + } + }, + .op.do_one_request = kmb_ocs_hcu_do_one_request, +}, +{ + .base.init = kmb_ocs_hcu_init, + .base.update = kmb_ocs_hcu_update, + .base.final = kmb_ocs_hcu_final, + .base.finup = kmb_ocs_hcu_finup, + .base.digest = kmb_ocs_hcu_digest, + .base.export = kmb_ocs_hcu_export, + .base.import = kmb_ocs_hcu_import, + .base.setkey = kmb_ocs_hcu_setkey, + .base.halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "hmac(sha224)", + .cra_driver_name = "hmac-sha224-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_hmac_cra_init, + .cra_exit = kmb_ocs_hcu_hmac_cra_exit, + } + }, + .op.do_one_request = kmb_ocs_hcu_do_one_request, +}, +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */ +{ + .base.init = kmb_ocs_hcu_init, + .base.update = kmb_ocs_hcu_update, + .base.final = kmb_ocs_hcu_final, + .base.finup = kmb_ocs_hcu_finup, + .base.digest = kmb_ocs_hcu_digest, + .base.export = kmb_ocs_hcu_export, + .base.import = kmb_ocs_hcu_import, + .base.halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_sha_cra_init, + } + }, + .op.do_one_request = kmb_ocs_hcu_do_one_request, +}, +{ + .base.init = kmb_ocs_hcu_init, + .base.update = kmb_ocs_hcu_update, + .base.final = kmb_ocs_hcu_final, + .base.finup = kmb_ocs_hcu_finup, + .base.digest = kmb_ocs_hcu_digest, + .base.export = kmb_ocs_hcu_export, + .base.import = kmb_ocs_hcu_import, + .base.setkey = kmb_ocs_hcu_setkey, + .base.halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "hmac-sha256-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_hmac_cra_init, + .cra_exit = kmb_ocs_hcu_hmac_cra_exit, + } + }, + .op.do_one_request = kmb_ocs_hcu_do_one_request, +}, +{ + .base.init = kmb_ocs_hcu_init, + .base.update = kmb_ocs_hcu_update, + .base.final = kmb_ocs_hcu_final, + .base.finup = kmb_ocs_hcu_finup, + .base.digest = kmb_ocs_hcu_digest, + .base.export = kmb_ocs_hcu_export, + .base.import = kmb_ocs_hcu_import, + .base.halg = { + .digestsize = SM3_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "sm3", + .cra_driver_name = "sm3-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SM3_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_sm3_cra_init, + } + }, + .op.do_one_request = kmb_ocs_hcu_do_one_request, +}, +{ + .base.init = kmb_ocs_hcu_init, + .base.update = kmb_ocs_hcu_update, + .base.final = kmb_ocs_hcu_final, + .base.finup = kmb_ocs_hcu_finup, + .base.digest = kmb_ocs_hcu_digest, + .base.export = kmb_ocs_hcu_export, + .base.import = kmb_ocs_hcu_import, + .base.setkey = kmb_ocs_hcu_setkey, + .base.halg = { + .digestsize = SM3_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "hmac(sm3)", + .cra_driver_name = "hmac-sm3-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SM3_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_hmac_sm3_cra_init, + .cra_exit = kmb_ocs_hcu_hmac_cra_exit, + } + }, + .op.do_one_request = kmb_ocs_hcu_do_one_request, +}, +{ + .base.init = kmb_ocs_hcu_init, + .base.update = kmb_ocs_hcu_update, + .base.final = kmb_ocs_hcu_final, + .base.finup = kmb_ocs_hcu_finup, + .base.digest = kmb_ocs_hcu_digest, + .base.export = kmb_ocs_hcu_export, + .base.import = kmb_ocs_hcu_import, + .base.halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "sha384", + .cra_driver_name = "sha384-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_sha_cra_init, + } + }, + .op.do_one_request = kmb_ocs_hcu_do_one_request, +}, +{ + .base.init = kmb_ocs_hcu_init, + .base.update = kmb_ocs_hcu_update, + .base.final = kmb_ocs_hcu_final, + .base.finup = kmb_ocs_hcu_finup, + .base.digest = kmb_ocs_hcu_digest, + .base.export = kmb_ocs_hcu_export, + .base.import = kmb_ocs_hcu_import, + .base.setkey = kmb_ocs_hcu_setkey, + .base.halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "hmac(sha384)", + .cra_driver_name = "hmac-sha384-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_hmac_cra_init, + .cra_exit = kmb_ocs_hcu_hmac_cra_exit, + } + }, + .op.do_one_request = kmb_ocs_hcu_do_one_request, +}, +{ + .base.init = kmb_ocs_hcu_init, + .base.update = kmb_ocs_hcu_update, + .base.final = kmb_ocs_hcu_final, + .base.finup = kmb_ocs_hcu_finup, + .base.digest = kmb_ocs_hcu_digest, + .base.export = kmb_ocs_hcu_export, + .base.import = kmb_ocs_hcu_import, + .base.halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "sha512", + .cra_driver_name = "sha512-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_sha_cra_init, + } + }, + .op.do_one_request = kmb_ocs_hcu_do_one_request, +}, +{ + .base.init = kmb_ocs_hcu_init, + .base.update = kmb_ocs_hcu_update, + .base.final = kmb_ocs_hcu_final, + .base.finup = kmb_ocs_hcu_finup, + .base.digest = kmb_ocs_hcu_digest, + .base.export = kmb_ocs_hcu_export, + .base.import = kmb_ocs_hcu_import, + .base.setkey = kmb_ocs_hcu_setkey, + .base.halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "hmac(sha512)", + .cra_driver_name = "hmac-sha512-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_hmac_cra_init, + .cra_exit = kmb_ocs_hcu_hmac_cra_exit, + } + }, + .op.do_one_request = kmb_ocs_hcu_do_one_request, +}, +}; + +/* Device tree driver match. */ +static const struct of_device_id kmb_ocs_hcu_of_match[] = { + { + .compatible = "intel,keembay-ocs-hcu", + }, + {} +}; +MODULE_DEVICE_TABLE(of, kmb_ocs_hcu_of_match); + +static void kmb_ocs_hcu_remove(struct platform_device *pdev) +{ + struct ocs_hcu_dev *hcu_dev = platform_get_drvdata(pdev); + + crypto_engine_unregister_ahashes(ocs_hcu_algs, ARRAY_SIZE(ocs_hcu_algs)); + + crypto_engine_exit(hcu_dev->engine); + + spin_lock_bh(&ocs_hcu.lock); + list_del(&hcu_dev->list); + spin_unlock_bh(&ocs_hcu.lock); +} + +static int kmb_ocs_hcu_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct ocs_hcu_dev *hcu_dev; + int rc; + + hcu_dev = devm_kzalloc(dev, sizeof(*hcu_dev), GFP_KERNEL); + if (!hcu_dev) + return -ENOMEM; + + hcu_dev->dev = dev; + + platform_set_drvdata(pdev, hcu_dev); + rc = dma_set_mask_and_coherent(&pdev->dev, OCS_HCU_DMA_BIT_MASK); + if (rc) + return rc; + + hcu_dev->io_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(hcu_dev->io_base)) + return PTR_ERR(hcu_dev->io_base); + + init_completion(&hcu_dev->irq_done); + + /* Get and request IRQ. */ + hcu_dev->irq = platform_get_irq(pdev, 0); + if (hcu_dev->irq < 0) + return hcu_dev->irq; + + rc = devm_request_threaded_irq(&pdev->dev, hcu_dev->irq, + ocs_hcu_irq_handler, NULL, 0, + "keembay-ocs-hcu", hcu_dev); + if (rc < 0) + return rc; + + INIT_LIST_HEAD(&hcu_dev->list); + + spin_lock_bh(&ocs_hcu.lock); + list_add_tail(&hcu_dev->list, &ocs_hcu.dev_list); + spin_unlock_bh(&ocs_hcu.lock); + + /* Initialize crypto engine */ + hcu_dev->engine = crypto_engine_alloc_init(dev, 1); + if (!hcu_dev->engine) { + rc = -ENOMEM; + goto list_del; + } + + rc = crypto_engine_start(hcu_dev->engine); + if (rc) { + dev_err(dev, "Could not start engine.\n"); + goto cleanup; + } + + /* Security infrastructure guarantees OCS clock is enabled. */ + + rc = crypto_engine_register_ahashes(ocs_hcu_algs, ARRAY_SIZE(ocs_hcu_algs)); + if (rc) { + dev_err(dev, "Could not register algorithms.\n"); + goto cleanup; + } + + return 0; + +cleanup: + crypto_engine_exit(hcu_dev->engine); +list_del: + spin_lock_bh(&ocs_hcu.lock); + list_del(&hcu_dev->list); + spin_unlock_bh(&ocs_hcu.lock); + + return rc; +} + +/* The OCS driver is a platform device. */ +static struct platform_driver kmb_ocs_hcu_driver = { + .probe = kmb_ocs_hcu_probe, + .remove = kmb_ocs_hcu_remove, + .driver = { + .name = DRV_NAME, + .of_match_table = kmb_ocs_hcu_of_match, + }, +}; + +module_platform_driver(kmb_ocs_hcu_driver); + +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/intel/keembay/ocs-aes.c b/drivers/crypto/intel/keembay/ocs-aes.c new file mode 100644 index 000000000..13ba75736 --- /dev/null +++ b/drivers/crypto/intel/keembay/ocs-aes.c @@ -0,0 +1,1490 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel Keem Bay OCS AES Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include "ocs-aes.h" + +#define AES_COMMAND_OFFSET 0x0000 +#define AES_KEY_0_OFFSET 0x0004 +#define AES_KEY_1_OFFSET 0x0008 +#define AES_KEY_2_OFFSET 0x000C +#define AES_KEY_3_OFFSET 0x0010 +#define AES_KEY_4_OFFSET 0x0014 +#define AES_KEY_5_OFFSET 0x0018 +#define AES_KEY_6_OFFSET 0x001C +#define AES_KEY_7_OFFSET 0x0020 +#define AES_IV_0_OFFSET 0x0024 +#define AES_IV_1_OFFSET 0x0028 +#define AES_IV_2_OFFSET 0x002C +#define AES_IV_3_OFFSET 0x0030 +#define AES_ACTIVE_OFFSET 0x0034 +#define AES_STATUS_OFFSET 0x0038 +#define AES_KEY_SIZE_OFFSET 0x0044 +#define AES_IER_OFFSET 0x0048 +#define AES_ISR_OFFSET 0x005C +#define AES_MULTIPURPOSE1_0_OFFSET 0x0200 +#define AES_MULTIPURPOSE1_1_OFFSET 0x0204 +#define AES_MULTIPURPOSE1_2_OFFSET 0x0208 +#define AES_MULTIPURPOSE1_3_OFFSET 0x020C +#define AES_MULTIPURPOSE2_0_OFFSET 0x0220 +#define AES_MULTIPURPOSE2_1_OFFSET 0x0224 +#define AES_MULTIPURPOSE2_2_OFFSET 0x0228 +#define AES_MULTIPURPOSE2_3_OFFSET 0x022C +#define AES_BYTE_ORDER_CFG_OFFSET 0x02C0 +#define AES_TLEN_OFFSET 0x0300 +#define AES_T_MAC_0_OFFSET 0x0304 +#define AES_T_MAC_1_OFFSET 0x0308 +#define AES_T_MAC_2_OFFSET 0x030C +#define AES_T_MAC_3_OFFSET 0x0310 +#define AES_PLEN_OFFSET 0x0314 +#define AES_A_DMA_SRC_ADDR_OFFSET 0x0400 +#define AES_A_DMA_DST_ADDR_OFFSET 0x0404 +#define AES_A_DMA_SRC_SIZE_OFFSET 0x0408 +#define AES_A_DMA_DST_SIZE_OFFSET 0x040C +#define AES_A_DMA_DMA_MODE_OFFSET 0x0410 +#define AES_A_DMA_NEXT_SRC_DESCR_OFFSET 0x0418 +#define AES_A_DMA_NEXT_DST_DESCR_OFFSET 0x041C +#define AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET 0x0420 +#define AES_A_DMA_LOG_OFFSET 0x0424 +#define AES_A_DMA_STATUS_OFFSET 0x0428 +#define AES_A_DMA_PERF_CNTR_OFFSET 0x042C +#define AES_A_DMA_MSI_ISR_OFFSET 0x0480 +#define AES_A_DMA_MSI_IER_OFFSET 0x0484 +#define AES_A_DMA_MSI_MASK_OFFSET 0x0488 +#define AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET 0x0600 +#define AES_A_DMA_OUTBUFFER_READ_FIFO_OFFSET 0x0700 + +/* + * AES_A_DMA_DMA_MODE register. + * Default: 0x00000000. + * bit[31] ACTIVE + * This bit activates the DMA. When the DMA finishes, it resets + * this bit to zero. + * bit[30:26] Unused by this driver. + * bit[25] SRC_LINK_LIST_EN + * Source link list enable bit. When the linked list is terminated + * this bit is reset by the DMA. + * bit[24] DST_LINK_LIST_EN + * Destination link list enable bit. When the linked list is + * terminated this bit is reset by the DMA. + * bit[23:0] Unused by this driver. + */ +#define AES_A_DMA_DMA_MODE_ACTIVE BIT(31) +#define AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN BIT(25) +#define AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN BIT(24) + +/* + * AES_ACTIVE register + * default 0x00000000 + * bit[31:10] Reserved + * bit[9] LAST_ADATA + * bit[8] LAST_GCX + * bit[7:2] Reserved + * bit[1] TERMINATION + * bit[0] TRIGGER + */ +#define AES_ACTIVE_LAST_ADATA BIT(9) +#define AES_ACTIVE_LAST_CCM_GCM BIT(8) +#define AES_ACTIVE_TERMINATION BIT(1) +#define AES_ACTIVE_TRIGGER BIT(0) + +#define AES_DISABLE_INT 0x00000000 +#define AES_DMA_CPD_ERR_INT BIT(8) +#define AES_DMA_OUTBUF_RD_ERR_INT BIT(7) +#define AES_DMA_OUTBUF_WR_ERR_INT BIT(6) +#define AES_DMA_INBUF_RD_ERR_INT BIT(5) +#define AES_DMA_INBUF_WR_ERR_INT BIT(4) +#define AES_DMA_BAD_COMP_INT BIT(3) +#define AES_DMA_SAI_INT BIT(2) +#define AES_DMA_SRC_DONE_INT BIT(0) +#define AES_COMPLETE_INT BIT(1) + +#define AES_DMA_MSI_MASK_CLEAR BIT(0) + +#define AES_128_BIT_KEY 0x00000000 +#define AES_256_BIT_KEY BIT(0) + +#define AES_DEACTIVATE_PERF_CNTR 0x00000000 +#define AES_ACTIVATE_PERF_CNTR BIT(0) + +#define AES_MAX_TAG_SIZE_U32 4 + +#define OCS_LL_DMA_FLAG_TERMINATE BIT(31) + +/* + * There is an inconsistency in the documentation. This is documented as a + * 11-bit value, but it is actually 10-bits. + */ +#define AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK 0x3FF + +/* + * During CCM decrypt, the OCS block needs to finish processing the ciphertext + * before the tag is written. For 128-bit mode this required delay is 28 OCS + * clock cycles. For 256-bit mode it is 36 OCS clock cycles. + */ +#define CCM_DECRYPT_DELAY_TAG_CLK_COUNT 36UL + +/* + * During CCM decrypt there must be a delay of at least 42 OCS clock cycles + * between setting the TRIGGER bit in AES_ACTIVE and setting the LAST_CCM_GCM + * bit in the same register (as stated in the OCS databook) + */ +#define CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT 42UL + +/* See RFC3610 section 2.2 */ +#define L_PRIME_MIN (1) +#define L_PRIME_MAX (7) +/* + * CCM IV format from RFC 3610 section 2.3 + * + * Octet Number Contents + * ------------ --------- + * 0 Flags + * 1 ... 15-L Nonce N + * 16-L ... 15 Counter i + * + * Flags = L' = L - 1 + */ +#define L_PRIME_IDX 0 +#define COUNTER_START(lprime) (16 - ((lprime) + 1)) +#define COUNTER_LEN(lprime) ((lprime) + 1) + +enum aes_counter_mode { + AES_CTR_M_NO_INC = 0, + AES_CTR_M_32_INC = 1, + AES_CTR_M_64_INC = 2, + AES_CTR_M_128_INC = 3, +}; + +/** + * struct ocs_dma_linked_list - OCS DMA linked list entry. + * @src_addr: Source address of the data. + * @src_len: Length of data to be fetched. + * @next: Next dma_list to fetch. + * @ll_flags: Flags (Freeze @ terminate) for the DMA engine. + */ +struct ocs_dma_linked_list { + u32 src_addr; + u32 src_len; + u32 next; + u32 ll_flags; +} __packed; + +/* + * Set endianness of inputs and outputs + * AES_BYTE_ORDER_CFG + * default 0x00000000 + * bit [10] - KEY_HI_LO_SWAP + * bit [9] - KEY_HI_SWAP_DWORDS_IN_OCTWORD + * bit [8] - KEY_HI_SWAP_BYTES_IN_DWORD + * bit [7] - KEY_LO_SWAP_DWORDS_IN_OCTWORD + * bit [6] - KEY_LO_SWAP_BYTES_IN_DWORD + * bit [5] - IV_SWAP_DWORDS_IN_OCTWORD + * bit [4] - IV_SWAP_BYTES_IN_DWORD + * bit [3] - DOUT_SWAP_DWORDS_IN_OCTWORD + * bit [2] - DOUT_SWAP_BYTES_IN_DWORD + * bit [1] - DOUT_SWAP_DWORDS_IN_OCTWORD + * bit [0] - DOUT_SWAP_BYTES_IN_DWORD + */ +static inline void aes_a_set_endianness(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(0x7FF, aes_dev->base_reg + AES_BYTE_ORDER_CFG_OFFSET); +} + +/* Trigger AES process start. */ +static inline void aes_a_op_trigger(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_ACTIVE_TRIGGER, aes_dev->base_reg + AES_ACTIVE_OFFSET); +} + +/* Indicate last bulk of data. */ +static inline void aes_a_op_termination(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_ACTIVE_TERMINATION, + aes_dev->base_reg + AES_ACTIVE_OFFSET); +} + +/* + * Set LAST_CCM_GCM in AES_ACTIVE register and clear all other bits. + * + * Called when DMA is programmed to fetch the last batch of data. + * - For AES-CCM it is called for the last batch of Payload data and Ciphertext + * data. + * - For AES-GCM, it is called for the last batch of Plaintext data and + * Ciphertext data. + */ +static inline void aes_a_set_last_gcx(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_ACTIVE_LAST_CCM_GCM, + aes_dev->base_reg + AES_ACTIVE_OFFSET); +} + +/* Wait for LAST_CCM_GCM bit to be unset. */ +static inline void aes_a_wait_last_gcx(const struct ocs_aes_dev *aes_dev) +{ + u32 aes_active_reg; + + do { + aes_active_reg = ioread32(aes_dev->base_reg + + AES_ACTIVE_OFFSET); + } while (aes_active_reg & AES_ACTIVE_LAST_CCM_GCM); +} + +/* Wait for 10 bits of input occupancy. */ +static void aes_a_dma_wait_input_buffer_occupancy(const struct ocs_aes_dev *aes_dev) +{ + u32 reg; + + do { + reg = ioread32(aes_dev->base_reg + AES_A_DMA_STATUS_OFFSET); + } while (reg & AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK); +} + + /* + * Set LAST_CCM_GCM and LAST_ADATA bits in AES_ACTIVE register (and clear all + * other bits). + * + * Called when DMA is programmed to fetch the last batch of Associated Data + * (CCM case) or Additional Authenticated Data (GCM case). + */ +static inline void aes_a_set_last_gcx_and_adata(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_ACTIVE_LAST_ADATA | AES_ACTIVE_LAST_CCM_GCM, + aes_dev->base_reg + AES_ACTIVE_OFFSET); +} + +/* Set DMA src and dst transfer size to 0 */ +static inline void aes_a_dma_set_xfer_size_zero(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET); + iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET); +} + +/* Activate DMA for zero-byte transfer case. */ +static inline void aes_a_dma_active(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_A_DMA_DMA_MODE_ACTIVE, + aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); +} + +/* Activate DMA and enable src linked list */ +static inline void aes_a_dma_active_src_ll_en(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_A_DMA_DMA_MODE_ACTIVE | + AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN, + aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); +} + +/* Activate DMA and enable dst linked list */ +static inline void aes_a_dma_active_dst_ll_en(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_A_DMA_DMA_MODE_ACTIVE | + AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN, + aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); +} + +/* Activate DMA and enable src and dst linked lists */ +static inline void aes_a_dma_active_src_dst_ll_en(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_A_DMA_DMA_MODE_ACTIVE | + AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN | + AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN, + aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); +} + +/* Reset PERF_CNTR to 0 and activate it */ +static inline void aes_a_dma_reset_and_activate_perf_cntr(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(0x00000000, aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET); + iowrite32(AES_ACTIVATE_PERF_CNTR, + aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET); +} + +/* Wait until PERF_CNTR is > delay, then deactivate it */ +static inline void aes_a_dma_wait_and_deactivate_perf_cntr(const struct ocs_aes_dev *aes_dev, + int delay) +{ + while (ioread32(aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET) < delay) + ; + iowrite32(AES_DEACTIVATE_PERF_CNTR, + aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET); +} + +/* Disable AES and DMA IRQ. */ +static void aes_irq_disable(struct ocs_aes_dev *aes_dev) +{ + u32 isr_val = 0; + + /* Disable interrupts */ + iowrite32(AES_DISABLE_INT, + aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); + iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET); + + /* Clear any pending interrupt */ + isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); + if (isr_val) + iowrite32(isr_val, + aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); + + isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET); + if (isr_val) + iowrite32(isr_val, + aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET); + + isr_val = ioread32(aes_dev->base_reg + AES_ISR_OFFSET); + if (isr_val) + iowrite32(isr_val, aes_dev->base_reg + AES_ISR_OFFSET); +} + +/* Enable AES or DMA IRQ. IRQ is disabled once fired. */ +static void aes_irq_enable(struct ocs_aes_dev *aes_dev, u8 irq) +{ + if (irq == AES_COMPLETE_INT) { + /* Ensure DMA error interrupts are enabled */ + iowrite32(AES_DMA_CPD_ERR_INT | + AES_DMA_OUTBUF_RD_ERR_INT | + AES_DMA_OUTBUF_WR_ERR_INT | + AES_DMA_INBUF_RD_ERR_INT | + AES_DMA_INBUF_WR_ERR_INT | + AES_DMA_BAD_COMP_INT | + AES_DMA_SAI_INT, + aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); + /* + * AES_IER + * default 0x00000000 + * bits [31:3] - reserved + * bit [2] - EN_SKS_ERR + * bit [1] - EN_AES_COMPLETE + * bit [0] - reserved + */ + iowrite32(AES_COMPLETE_INT, aes_dev->base_reg + AES_IER_OFFSET); + return; + } + if (irq == AES_DMA_SRC_DONE_INT) { + /* Ensure AES interrupts are disabled */ + iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET); + /* + * DMA_MSI_IER + * default 0x00000000 + * bits [31:9] - reserved + * bit [8] - CPD_ERR_INT_EN + * bit [7] - OUTBUF_RD_ERR_INT_EN + * bit [6] - OUTBUF_WR_ERR_INT_EN + * bit [5] - INBUF_RD_ERR_INT_EN + * bit [4] - INBUF_WR_ERR_INT_EN + * bit [3] - BAD_COMP_INT_EN + * bit [2] - SAI_INT_EN + * bit [1] - DST_DONE_INT_EN + * bit [0] - SRC_DONE_INT_EN + */ + iowrite32(AES_DMA_CPD_ERR_INT | + AES_DMA_OUTBUF_RD_ERR_INT | + AES_DMA_OUTBUF_WR_ERR_INT | + AES_DMA_INBUF_RD_ERR_INT | + AES_DMA_INBUF_WR_ERR_INT | + AES_DMA_BAD_COMP_INT | + AES_DMA_SAI_INT | + AES_DMA_SRC_DONE_INT, + aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); + } +} + +/* Enable and wait for IRQ (either from OCS AES engine or DMA) */ +static int ocs_aes_irq_enable_and_wait(struct ocs_aes_dev *aes_dev, u8 irq) +{ + int rc; + + reinit_completion(&aes_dev->irq_completion); + aes_irq_enable(aes_dev, irq); + rc = wait_for_completion_interruptible(&aes_dev->irq_completion); + if (rc) + return rc; + + return aes_dev->dma_err_mask ? -EIO : 0; +} + +/* Configure DMA to OCS, linked list mode */ +static inline void dma_to_ocs_aes_ll(struct ocs_aes_dev *aes_dev, + dma_addr_t dma_list) +{ + iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET); + iowrite32(dma_list, + aes_dev->base_reg + AES_A_DMA_NEXT_SRC_DESCR_OFFSET); +} + +/* Configure DMA from OCS, linked list mode */ +static inline void dma_from_ocs_aes_ll(struct ocs_aes_dev *aes_dev, + dma_addr_t dma_list) +{ + iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET); + iowrite32(dma_list, + aes_dev->base_reg + AES_A_DMA_NEXT_DST_DESCR_OFFSET); +} + +irqreturn_t ocs_aes_irq_handler(int irq, void *dev_id) +{ + struct ocs_aes_dev *aes_dev = dev_id; + u32 aes_dma_isr; + + /* Read DMA ISR status. */ + aes_dma_isr = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); + + /* Disable and clear interrupts. */ + aes_irq_disable(aes_dev); + + /* Save DMA error status. */ + aes_dev->dma_err_mask = aes_dma_isr & + (AES_DMA_CPD_ERR_INT | + AES_DMA_OUTBUF_RD_ERR_INT | + AES_DMA_OUTBUF_WR_ERR_INT | + AES_DMA_INBUF_RD_ERR_INT | + AES_DMA_INBUF_WR_ERR_INT | + AES_DMA_BAD_COMP_INT | + AES_DMA_SAI_INT); + + /* Signal IRQ completion. */ + complete(&aes_dev->irq_completion); + + return IRQ_HANDLED; +} + +/** + * ocs_aes_set_key() - Write key into OCS AES hardware. + * @aes_dev: The OCS AES device to write the key to. + * @key_size: The size of the key (in bytes). + * @key: The key to write. + * @cipher: The cipher the key is for. + * + * For AES @key_size must be either 16 or 32. For SM4 @key_size must be 16. + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_aes_set_key(struct ocs_aes_dev *aes_dev, u32 key_size, const u8 *key, + enum ocs_cipher cipher) +{ + const u32 *key_u32; + u32 val; + int i; + + /* OCS AES supports 128-bit and 256-bit keys only. */ + if (cipher == OCS_AES && !(key_size == 32 || key_size == 16)) { + dev_err(aes_dev->dev, + "%d-bit keys not supported by AES cipher\n", + key_size * 8); + return -EINVAL; + } + /* OCS SM4 supports 128-bit keys only. */ + if (cipher == OCS_SM4 && key_size != 16) { + dev_err(aes_dev->dev, + "%d-bit keys not supported for SM4 cipher\n", + key_size * 8); + return -EINVAL; + } + + if (!key) + return -EINVAL; + + key_u32 = (const u32 *)key; + + /* Write key to AES_KEY[0-7] registers */ + for (i = 0; i < (key_size / sizeof(u32)); i++) { + iowrite32(key_u32[i], + aes_dev->base_reg + AES_KEY_0_OFFSET + + (i * sizeof(u32))); + } + /* + * Write key size + * bits [31:1] - reserved + * bit [0] - AES_KEY_SIZE + * 0 - 128 bit key + * 1 - 256 bit key + */ + val = (key_size == 16) ? AES_128_BIT_KEY : AES_256_BIT_KEY; + iowrite32(val, aes_dev->base_reg + AES_KEY_SIZE_OFFSET); + + return 0; +} + +/* Write AES_COMMAND */ +static inline void set_ocs_aes_command(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_mode mode, + enum ocs_instruction instruction) +{ + u32 val; + + /* AES_COMMAND + * default 0x000000CC + * bit [14] - CIPHER_SELECT + * 0 - AES + * 1 - SM4 + * bits [11:8] - OCS_AES_MODE + * 0000 - ECB + * 0001 - CBC + * 0010 - CTR + * 0110 - CCM + * 0111 - GCM + * 1001 - CTS + * bits [7:6] - AES_INSTRUCTION + * 00 - ENCRYPT + * 01 - DECRYPT + * 10 - EXPAND + * 11 - BYPASS + * bits [3:2] - CTR_M_BITS + * 00 - No increment + * 01 - Least significant 32 bits are incremented + * 10 - Least significant 64 bits are incremented + * 11 - Full 128 bits are incremented + */ + val = (cipher << 14) | (mode << 8) | (instruction << 6) | + (AES_CTR_M_128_INC << 2); + iowrite32(val, aes_dev->base_reg + AES_COMMAND_OFFSET); +} + +static void ocs_aes_init(struct ocs_aes_dev *aes_dev, + enum ocs_mode mode, + enum ocs_cipher cipher, + enum ocs_instruction instruction) +{ + /* Ensure interrupts are disabled and pending interrupts cleared. */ + aes_irq_disable(aes_dev); + + /* Set endianness recommended by data-sheet. */ + aes_a_set_endianness(aes_dev); + + /* Set AES_COMMAND register. */ + set_ocs_aes_command(aes_dev, cipher, mode, instruction); +} + +/* + * Write the byte length of the last AES/SM4 block of Payload data (without + * zero padding and without the length of the MAC) in register AES_PLEN. + */ +static inline void ocs_aes_write_last_data_blk_len(struct ocs_aes_dev *aes_dev, + u32 size) +{ + u32 val; + + if (size == 0) { + val = 0; + goto exit; + } + + val = size % AES_BLOCK_SIZE; + if (val == 0) + val = AES_BLOCK_SIZE; + +exit: + iowrite32(val, aes_dev->base_reg + AES_PLEN_OFFSET); +} + +/* + * Validate inputs according to mode. + * If OK return 0; else return -EINVAL. + */ +static int ocs_aes_validate_inputs(dma_addr_t src_dma_list, u32 src_size, + const u8 *iv, u32 iv_size, + dma_addr_t aad_dma_list, u32 aad_size, + const u8 *tag, u32 tag_size, + enum ocs_cipher cipher, enum ocs_mode mode, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list) +{ + /* Ensure cipher, mode and instruction are valid. */ + if (!(cipher == OCS_AES || cipher == OCS_SM4)) + return -EINVAL; + + if (mode != OCS_MODE_ECB && mode != OCS_MODE_CBC && + mode != OCS_MODE_CTR && mode != OCS_MODE_CCM && + mode != OCS_MODE_GCM && mode != OCS_MODE_CTS) + return -EINVAL; + + if (instruction != OCS_ENCRYPT && instruction != OCS_DECRYPT && + instruction != OCS_EXPAND && instruction != OCS_BYPASS) + return -EINVAL; + + /* + * When instruction is OCS_BYPASS, OCS simply copies data from source + * to destination using DMA. + * + * AES mode is irrelevant, but both source and destination DMA + * linked-list must be defined. + */ + if (instruction == OCS_BYPASS) { + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + return 0; + } + + /* + * For performance reasons switch based on mode to limit unnecessary + * conditionals for each mode + */ + switch (mode) { + case OCS_MODE_ECB: + /* Ensure input length is multiple of block size */ + if (src_size % AES_BLOCK_SIZE != 0) + return -EINVAL; + + /* Ensure source and destination linked lists are created */ + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + return 0; + + case OCS_MODE_CBC: + /* Ensure input length is multiple of block size */ + if (src_size % AES_BLOCK_SIZE != 0) + return -EINVAL; + + /* Ensure source and destination linked lists are created */ + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + + case OCS_MODE_CTR: + /* Ensure input length of 1 byte or greater */ + if (src_size == 0) + return -EINVAL; + + /* Ensure source and destination linked lists are created */ + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + + case OCS_MODE_CTS: + /* Ensure input length >= block size */ + if (src_size < AES_BLOCK_SIZE) + return -EINVAL; + + /* Ensure source and destination linked lists are created */ + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + + case OCS_MODE_GCM: + /* Ensure IV is present and GCM_AES_IV_SIZE in length */ + if (!iv || iv_size != GCM_AES_IV_SIZE) + return -EINVAL; + + /* + * If input data present ensure source and destination linked + * lists are created + */ + if (src_size && (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR)) + return -EINVAL; + + /* If aad present ensure aad linked list is created */ + if (aad_size && aad_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Ensure tag destination is set */ + if (!tag) + return -EINVAL; + + /* Just ensure that tag_size doesn't cause overflows. */ + if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32))) + return -EINVAL; + + return 0; + + case OCS_MODE_CCM: + /* Ensure IV is present and block size in length */ + if (!iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + /* 2 <= L <= 8, so 1 <= L' <= 7 */ + if (iv[L_PRIME_IDX] < L_PRIME_MIN || + iv[L_PRIME_IDX] > L_PRIME_MAX) + return -EINVAL; + + /* If aad present ensure aad linked list is created */ + if (aad_size && aad_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Just ensure that tag_size doesn't cause overflows. */ + if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32))) + return -EINVAL; + + if (instruction == OCS_DECRYPT) { + /* + * If input data present ensure source and destination + * linked lists are created + */ + if (src_size && (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR)) + return -EINVAL; + + /* Ensure input tag is present */ + if (!tag) + return -EINVAL; + + return 0; + } + + /* Instruction == OCS_ENCRYPT */ + + /* + * Destination linked list always required (for tag even if no + * input data) + */ + if (dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* If input data present ensure src linked list is created */ + if (src_size && src_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + return 0; + + default: + return -EINVAL; + } +} + +/** + * ocs_aes_op() - Perform AES/SM4 operation. + * @aes_dev: The OCS AES device to use. + * @mode: The mode to use (ECB, CBC, CTR, or CTS). + * @cipher: The cipher to use (AES or SM4). + * @instruction: The instruction to perform (encrypt or decrypt). + * @dst_dma_list: The OCS DMA list mapping output memory. + * @src_dma_list: The OCS DMA list mapping input payload data. + * @src_size: The amount of data mapped by @src_dma_list. + * @iv: The IV vector. + * @iv_size: The size (in bytes) of @iv. + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_aes_op(struct ocs_aes_dev *aes_dev, + enum ocs_mode mode, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + u8 *iv, + u32 iv_size) +{ + u32 *iv32; + int rc; + + rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, iv_size, 0, 0, + NULL, 0, cipher, mode, instruction, + dst_dma_list); + if (rc) + return rc; + /* + * ocs_aes_validate_inputs() is a generic check, now ensure mode is not + * GCM or CCM. + */ + if (mode == OCS_MODE_GCM || mode == OCS_MODE_CCM) + return -EINVAL; + + /* Cast IV to u32 array. */ + iv32 = (u32 *)iv; + + ocs_aes_init(aes_dev, mode, cipher, instruction); + + if (mode == OCS_MODE_CTS) { + /* Write the byte length of the last data block to engine. */ + ocs_aes_write_last_data_blk_len(aes_dev, src_size); + } + + /* ECB is the only mode that doesn't use IV. */ + if (mode != OCS_MODE_ECB) { + iowrite32(iv32[0], aes_dev->base_reg + AES_IV_0_OFFSET); + iowrite32(iv32[1], aes_dev->base_reg + AES_IV_1_OFFSET); + iowrite32(iv32[2], aes_dev->base_reg + AES_IV_2_OFFSET); + iowrite32(iv32[3], aes_dev->base_reg + AES_IV_3_OFFSET); + } + + /* Set AES_ACTIVE.TRIGGER to start the operation. */ + aes_a_op_trigger(aes_dev); + + /* Configure and activate input / output DMA. */ + dma_to_ocs_aes_ll(aes_dev, src_dma_list); + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_src_dst_ll_en(aes_dev); + + if (mode == OCS_MODE_CTS) { + /* + * For CTS mode, instruct engine to activate ciphertext + * stealing if last block of data is incomplete. + */ + aes_a_set_last_gcx(aes_dev); + } else { + /* For all other modes, just write the 'termination' bit. */ + aes_a_op_termination(aes_dev); + } + + /* Wait for engine to complete processing. */ + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); + if (rc) + return rc; + + if (mode == OCS_MODE_CTR) { + /* Read back IV for streaming mode */ + iv32[0] = ioread32(aes_dev->base_reg + AES_IV_0_OFFSET); + iv32[1] = ioread32(aes_dev->base_reg + AES_IV_1_OFFSET); + iv32[2] = ioread32(aes_dev->base_reg + AES_IV_2_OFFSET); + iv32[3] = ioread32(aes_dev->base_reg + AES_IV_3_OFFSET); + } + + return 0; +} + +/* Compute and write J0 to engine registers. */ +static void ocs_aes_gcm_write_j0(const struct ocs_aes_dev *aes_dev, + const u8 *iv) +{ + const u32 *j0 = (u32 *)iv; + + /* + * IV must be 12 bytes; Other sizes not supported as Linux crypto API + * does only expects/allows 12 byte IV for GCM + */ + iowrite32(0x00000001, aes_dev->base_reg + AES_IV_0_OFFSET); + iowrite32(__swab32(j0[2]), aes_dev->base_reg + AES_IV_1_OFFSET); + iowrite32(__swab32(j0[1]), aes_dev->base_reg + AES_IV_2_OFFSET); + iowrite32(__swab32(j0[0]), aes_dev->base_reg + AES_IV_3_OFFSET); +} + +/* Read GCM tag from engine registers. */ +static inline void ocs_aes_gcm_read_tag(struct ocs_aes_dev *aes_dev, + u8 *tag, u32 tag_size) +{ + u32 tag_u32[AES_MAX_TAG_SIZE_U32]; + + /* + * The Authentication Tag T is stored in Little Endian order in the + * registers with the most significant bytes stored from AES_T_MAC[3] + * downward. + */ + tag_u32[0] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_3_OFFSET)); + tag_u32[1] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_2_OFFSET)); + tag_u32[2] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_1_OFFSET)); + tag_u32[3] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_0_OFFSET)); + + memcpy(tag, tag_u32, tag_size); +} + +/** + * ocs_aes_gcm_op() - Perform GCM operation. + * @aes_dev: The OCS AES device to use. + * @cipher: The Cipher to use (AES or SM4). + * @instruction: The instruction to perform (encrypt or decrypt). + * @dst_dma_list: The OCS DMA list mapping output memory. + * @src_dma_list: The OCS DMA list mapping input payload data. + * @src_size: The amount of data mapped by @src_dma_list. + * @iv: The input IV vector. + * @aad_dma_list: The OCS DMA list mapping input AAD data. + * @aad_size: The amount of data mapped by @aad_dma_list. + * @out_tag: Where to store computed tag. + * @tag_size: The size (in bytes) of @out_tag. + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + const u8 *iv, + dma_addr_t aad_dma_list, + u32 aad_size, + u8 *out_tag, + u32 tag_size) +{ + u64 bit_len; + u32 val; + int rc; + + rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, + GCM_AES_IV_SIZE, aad_dma_list, + aad_size, out_tag, tag_size, cipher, + OCS_MODE_GCM, instruction, + dst_dma_list); + if (rc) + return rc; + + ocs_aes_init(aes_dev, OCS_MODE_GCM, cipher, instruction); + + /* Compute and write J0 to OCS HW. */ + ocs_aes_gcm_write_j0(aes_dev, iv); + + /* Write out_tag byte length */ + iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET); + + /* Write the byte length of the last plaintext / ciphertext block. */ + ocs_aes_write_last_data_blk_len(aes_dev, src_size); + + /* Write ciphertext bit length */ + bit_len = (u64)src_size * 8; + val = bit_len & 0xFFFFFFFF; + iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_0_OFFSET); + val = bit_len >> 32; + iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_1_OFFSET); + + /* Write aad bit length */ + bit_len = (u64)aad_size * 8; + val = bit_len & 0xFFFFFFFF; + iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_2_OFFSET); + val = bit_len >> 32; + iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_3_OFFSET); + + /* Set AES_ACTIVE.TRIGGER to start the operation. */ + aes_a_op_trigger(aes_dev); + + /* Process AAD. */ + if (aad_size) { + /* If aad present, configure DMA to feed it to the engine. */ + dma_to_ocs_aes_ll(aes_dev, aad_dma_list); + aes_a_dma_active_src_ll_en(aes_dev); + + /* Instructs engine to pad last block of aad, if needed. */ + aes_a_set_last_gcx_and_adata(aes_dev); + + /* Wait for DMA transfer to complete. */ + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT); + if (rc) + return rc; + } else { + aes_a_set_last_gcx_and_adata(aes_dev); + } + + /* Wait until adata (if present) has been processed. */ + aes_a_wait_last_gcx(aes_dev); + aes_a_dma_wait_input_buffer_occupancy(aes_dev); + + /* Now process payload. */ + if (src_size) { + /* Configure and activate DMA for both input and output data. */ + dma_to_ocs_aes_ll(aes_dev, src_dma_list); + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_src_dst_ll_en(aes_dev); + } else { + aes_a_dma_set_xfer_size_zero(aes_dev); + aes_a_dma_active(aes_dev); + } + + /* Instruct AES/SMA4 engine payload processing is over. */ + aes_a_set_last_gcx(aes_dev); + + /* Wait for OCS AES engine to complete processing. */ + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); + if (rc) + return rc; + + ocs_aes_gcm_read_tag(aes_dev, out_tag, tag_size); + + return 0; +} + +/* Write encrypted tag to AES/SM4 engine. */ +static void ocs_aes_ccm_write_encrypted_tag(struct ocs_aes_dev *aes_dev, + const u8 *in_tag, u32 tag_size) +{ + int i; + + /* Ensure DMA input buffer is empty */ + aes_a_dma_wait_input_buffer_occupancy(aes_dev); + + /* + * During CCM decrypt, the OCS block needs to finish processing the + * ciphertext before the tag is written. So delay needed after DMA has + * completed writing the ciphertext + */ + aes_a_dma_reset_and_activate_perf_cntr(aes_dev); + aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev, + CCM_DECRYPT_DELAY_TAG_CLK_COUNT); + + /* Write encrypted tag to AES/SM4 engine. */ + for (i = 0; i < tag_size; i++) { + iowrite8(in_tag[i], aes_dev->base_reg + + AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET); + } +} + +/* + * Write B0 CCM block to OCS AES HW. + * + * Note: B0 format is documented in NIST Special Publication 800-38C + * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf + * (see Section A.2.1) + */ +static int ocs_aes_ccm_write_b0(const struct ocs_aes_dev *aes_dev, + const u8 *iv, u32 adata_size, u32 tag_size, + u32 cryptlen) +{ + u8 b0[16]; /* CCM B0 block is 16 bytes long. */ + int i, q; + + /* Initialize B0 to 0. */ + memset(b0, 0, sizeof(b0)); + + /* + * B0[0] is the 'Flags Octet' and has the following structure: + * bit 7: Reserved + * bit 6: Adata flag + * bit 5-3: t value encoded as (t-2)/2 + * bit 2-0: q value encoded as q - 1 + */ + /* If there is AAD data, set the Adata flag. */ + if (adata_size) + b0[0] |= BIT(6); + /* + * t denotes the octet length of T. + * t can only be an element of { 4, 6, 8, 10, 12, 14, 16} and is + * encoded as (t - 2) / 2 + */ + b0[0] |= (((tag_size - 2) / 2) & 0x7) << 3; + /* + * q is the octet length of Q. + * q can only be an element of {2, 3, 4, 5, 6, 7, 8} and is encoded as + * q - 1 == iv[0] & 0x7; + */ + b0[0] |= iv[0] & 0x7; + /* + * Copy the Nonce N from IV to B0; N is located in iv[1]..iv[15 - q] + * and must be copied to b0[1]..b0[15-q]. + * q == (iv[0] & 0x7) + 1 + */ + q = (iv[0] & 0x7) + 1; + for (i = 1; i <= 15 - q; i++) + b0[i] = iv[i]; + /* + * The rest of B0 must contain Q, i.e., the message length. + * Q is encoded in q octets, in big-endian order, so to write it, we + * start from the end of B0 and we move backward. + */ + i = sizeof(b0) - 1; + while (q) { + b0[i] = cryptlen & 0xff; + cryptlen >>= 8; + i--; + q--; + } + /* + * If cryptlen is not zero at this point, it means that its original + * value was too big. + */ + if (cryptlen) + return -EOVERFLOW; + /* Now write B0 to OCS AES input buffer. */ + for (i = 0; i < sizeof(b0); i++) + iowrite8(b0[i], aes_dev->base_reg + + AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET); + return 0; +} + +/* + * Write adata length to OCS AES HW. + * + * Note: adata len encoding is documented in NIST Special Publication 800-38C + * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf + * (see Section A.2.2) + */ +static void ocs_aes_ccm_write_adata_len(const struct ocs_aes_dev *aes_dev, + u64 adata_len) +{ + u8 enc_a[10]; /* Maximum encoded size: 10 octets. */ + int i, len; + + /* + * adata_len ('a') is encoded as follows: + * If 0 < a < 2^16 - 2^8 ==> 'a' encoded as [a]16, i.e., two octets + * (big endian). + * If 2^16 - 2^8 ≤ a < 2^32 ==> 'a' encoded as 0xff || 0xfe || [a]32, + * i.e., six octets (big endian). + * If 2^32 ≤ a < 2^64 ==> 'a' encoded as 0xff || 0xff || [a]64, + * i.e., ten octets (big endian). + */ + if (adata_len < 65280) { + len = 2; + *(__be16 *)enc_a = cpu_to_be16(adata_len); + } else if (adata_len <= 0xFFFFFFFF) { + len = 6; + *(__be16 *)enc_a = cpu_to_be16(0xfffe); + *(__be32 *)&enc_a[2] = cpu_to_be32(adata_len); + } else { /* adata_len >= 2^32 */ + len = 10; + *(__be16 *)enc_a = cpu_to_be16(0xffff); + *(__be64 *)&enc_a[2] = cpu_to_be64(adata_len); + } + for (i = 0; i < len; i++) + iowrite8(enc_a[i], + aes_dev->base_reg + + AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET); +} + +static int ocs_aes_ccm_do_adata(struct ocs_aes_dev *aes_dev, + dma_addr_t adata_dma_list, u32 adata_size) +{ + int rc; + + if (!adata_size) { + /* Since no aad the LAST_GCX bit can be set now */ + aes_a_set_last_gcx_and_adata(aes_dev); + goto exit; + } + + /* Adata case. */ + + /* + * Form the encoding of the Associated data length and write it + * to the AES/SM4 input buffer. + */ + ocs_aes_ccm_write_adata_len(aes_dev, adata_size); + + /* Configure the AES/SM4 DMA to fetch the Associated Data */ + dma_to_ocs_aes_ll(aes_dev, adata_dma_list); + + /* Activate DMA to fetch Associated data. */ + aes_a_dma_active_src_ll_en(aes_dev); + + /* Set LAST_GCX and LAST_ADATA in AES ACTIVE register. */ + aes_a_set_last_gcx_and_adata(aes_dev); + + /* Wait for DMA transfer to complete. */ + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT); + if (rc) + return rc; + +exit: + /* Wait until adata (if present) has been processed. */ + aes_a_wait_last_gcx(aes_dev); + aes_a_dma_wait_input_buffer_occupancy(aes_dev); + + return 0; +} + +static int ocs_aes_ccm_encrypt_do_payload(struct ocs_aes_dev *aes_dev, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size) +{ + if (src_size) { + /* + * Configure and activate DMA for both input and output + * data. + */ + dma_to_ocs_aes_ll(aes_dev, src_dma_list); + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_src_dst_ll_en(aes_dev); + } else { + /* Configure and activate DMA for output data only. */ + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_dst_ll_en(aes_dev); + } + + /* + * Set the LAST GCX bit in AES_ACTIVE Register to instruct + * AES/SM4 engine to pad the last block of data. + */ + aes_a_set_last_gcx(aes_dev); + + /* We are done, wait for IRQ and return. */ + return ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); +} + +static int ocs_aes_ccm_decrypt_do_payload(struct ocs_aes_dev *aes_dev, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size) +{ + if (!src_size) { + /* Let engine process 0-length input. */ + aes_a_dma_set_xfer_size_zero(aes_dev); + aes_a_dma_active(aes_dev); + aes_a_set_last_gcx(aes_dev); + + return 0; + } + + /* + * Configure and activate DMA for both input and output + * data. + */ + dma_to_ocs_aes_ll(aes_dev, src_dma_list); + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_src_dst_ll_en(aes_dev); + /* + * Set the LAST GCX bit in AES_ACTIVE Register; this allows the + * AES/SM4 engine to differentiate between encrypted data and + * encrypted MAC. + */ + aes_a_set_last_gcx(aes_dev); + /* + * Enable DMA DONE interrupt; once DMA transfer is over, + * interrupt handler will process the MAC/tag. + */ + return ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT); +} + +/* + * Compare Tag to Yr. + * + * Only used at the end of CCM decrypt. If tag == yr, message authentication + * has succeeded. + */ +static inline int ccm_compare_tag_to_yr(struct ocs_aes_dev *aes_dev, + u8 tag_size_bytes) +{ + u32 tag[AES_MAX_TAG_SIZE_U32]; + u32 yr[AES_MAX_TAG_SIZE_U32]; + u8 i; + + /* Read Tag and Yr from AES registers. */ + for (i = 0; i < AES_MAX_TAG_SIZE_U32; i++) { + tag[i] = ioread32(aes_dev->base_reg + + AES_T_MAC_0_OFFSET + (i * sizeof(u32))); + yr[i] = ioread32(aes_dev->base_reg + + AES_MULTIPURPOSE2_0_OFFSET + + (i * sizeof(u32))); + } + + return crypto_memneq(tag, yr, tag_size_bytes) ? -EBADMSG : 0; +} + +/** + * ocs_aes_ccm_op() - Perform CCM operation. + * @aes_dev: The OCS AES device to use. + * @cipher: The Cipher to use (AES or SM4). + * @instruction: The instruction to perform (encrypt or decrypt). + * @dst_dma_list: The OCS DMA list mapping output memory. + * @src_dma_list: The OCS DMA list mapping input payload data. + * @src_size: The amount of data mapped by @src_dma_list. + * @iv: The input IV vector. + * @adata_dma_list: The OCS DMA list mapping input A-data. + * @adata_size: The amount of data mapped by @adata_dma_list. + * @in_tag: Input tag. + * @tag_size: The size (in bytes) of @in_tag. + * + * Note: for encrypt the tag is appended to the ciphertext (in the memory + * mapped by @dst_dma_list). + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_aes_ccm_op(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + u8 *iv, + dma_addr_t adata_dma_list, + u32 adata_size, + u8 *in_tag, + u32 tag_size) +{ + u32 *iv_32; + u8 lprime; + int rc; + + rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, + AES_BLOCK_SIZE, adata_dma_list, adata_size, + in_tag, tag_size, cipher, OCS_MODE_CCM, + instruction, dst_dma_list); + if (rc) + return rc; + + ocs_aes_init(aes_dev, OCS_MODE_CCM, cipher, instruction); + + /* + * Note: rfc 3610 and NIST 800-38C require counter of zero to encrypt + * auth tag so ensure this is the case + */ + lprime = iv[L_PRIME_IDX]; + memset(&iv[COUNTER_START(lprime)], 0, COUNTER_LEN(lprime)); + + /* + * Nonce is already converted to ctr0 before being passed into this + * function as iv. + */ + iv_32 = (u32 *)iv; + iowrite32(__swab32(iv_32[0]), + aes_dev->base_reg + AES_MULTIPURPOSE1_3_OFFSET); + iowrite32(__swab32(iv_32[1]), + aes_dev->base_reg + AES_MULTIPURPOSE1_2_OFFSET); + iowrite32(__swab32(iv_32[2]), + aes_dev->base_reg + AES_MULTIPURPOSE1_1_OFFSET); + iowrite32(__swab32(iv_32[3]), + aes_dev->base_reg + AES_MULTIPURPOSE1_0_OFFSET); + + /* Write MAC/tag length in register AES_TLEN */ + iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET); + /* + * Write the byte length of the last AES/SM4 block of Payload data + * (without zero padding and without the length of the MAC) in register + * AES_PLEN. + */ + ocs_aes_write_last_data_blk_len(aes_dev, src_size); + + /* Set AES_ACTIVE.TRIGGER to start the operation. */ + aes_a_op_trigger(aes_dev); + + aes_a_dma_reset_and_activate_perf_cntr(aes_dev); + + /* Form block B0 and write it to the AES/SM4 input buffer. */ + rc = ocs_aes_ccm_write_b0(aes_dev, iv, adata_size, tag_size, src_size); + if (rc) + return rc; + /* + * Ensure there has been at least CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT + * clock cycles since TRIGGER bit was set + */ + aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev, + CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT); + + /* Process Adata. */ + ocs_aes_ccm_do_adata(aes_dev, adata_dma_list, adata_size); + + /* For Encrypt case we just process the payload and return. */ + if (instruction == OCS_ENCRYPT) { + return ocs_aes_ccm_encrypt_do_payload(aes_dev, dst_dma_list, + src_dma_list, src_size); + } + /* For Decypt we need to process the payload and then the tag. */ + rc = ocs_aes_ccm_decrypt_do_payload(aes_dev, dst_dma_list, + src_dma_list, src_size); + if (rc) + return rc; + + /* Process MAC/tag directly: feed tag to engine and wait for IRQ. */ + ocs_aes_ccm_write_encrypted_tag(aes_dev, in_tag, tag_size); + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); + if (rc) + return rc; + + return ccm_compare_tag_to_yr(aes_dev, tag_size); +} + +/** + * ocs_create_linked_list_from_sg() - Create OCS DMA linked list from SG list. + * @aes_dev: The OCS AES device the list will be created for. + * @sg: The SG list OCS DMA linked list will be created from. When + * passed to this function, @sg must have been already mapped + * with dma_map_sg(). + * @sg_dma_count: The number of DMA-mapped entries in @sg. This must be the + * value returned by dma_map_sg() when @sg was mapped. + * @dll_desc: The OCS DMA dma_list to use to store information about the + * created linked list. + * @data_size: The size of the data (from the SG list) to be mapped into the + * OCS DMA linked list. + * @data_offset: The offset (within the SG list) of the data to be mapped. + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_create_linked_list_from_sg(const struct ocs_aes_dev *aes_dev, + struct scatterlist *sg, + int sg_dma_count, + struct ocs_dll_desc *dll_desc, + size_t data_size, size_t data_offset) +{ + struct ocs_dma_linked_list *ll = NULL; + struct scatterlist *sg_tmp; + unsigned int tmp; + int dma_nents; + int i; + + if (!dll_desc || !sg || !aes_dev) + return -EINVAL; + + /* Default values for when no ddl_desc is created. */ + dll_desc->vaddr = NULL; + dll_desc->dma_addr = DMA_MAPPING_ERROR; + dll_desc->size = 0; + + if (data_size == 0) + return 0; + + /* Loop over sg_list until we reach entry at specified offset. */ + while (data_offset >= sg_dma_len(sg)) { + data_offset -= sg_dma_len(sg); + sg_dma_count--; + sg = sg_next(sg); + /* If we reach the end of the list, offset was invalid. */ + if (!sg || sg_dma_count == 0) + return -EINVAL; + } + + /* Compute number of DMA-mapped SG entries to add into OCS DMA list. */ + dma_nents = 0; + tmp = 0; + sg_tmp = sg; + while (tmp < data_offset + data_size) { + /* If we reach the end of the list, data_size was invalid. */ + if (!sg_tmp) + return -EINVAL; + tmp += sg_dma_len(sg_tmp); + dma_nents++; + sg_tmp = sg_next(sg_tmp); + } + if (dma_nents > sg_dma_count) + return -EINVAL; + + /* Allocate the DMA list, one entry for each SG entry. */ + dll_desc->size = sizeof(struct ocs_dma_linked_list) * dma_nents; + dll_desc->vaddr = dma_alloc_coherent(aes_dev->dev, dll_desc->size, + &dll_desc->dma_addr, GFP_KERNEL); + if (!dll_desc->vaddr) + return -ENOMEM; + + /* Populate DMA linked list entries. */ + ll = dll_desc->vaddr; + for (i = 0; i < dma_nents; i++, sg = sg_next(sg)) { + ll[i].src_addr = sg_dma_address(sg) + data_offset; + ll[i].src_len = min(sg_dma_len(sg) - data_offset, data_size); + data_offset = 0; + data_size -= ll[i].src_len; + /* Current element points to the DMA address of the next one. */ + ll[i].next = dll_desc->dma_addr + (sizeof(*ll) * (i + 1)); + ll[i].ll_flags = 0; + } + /* Terminate last element. */ + ll[i - 1].next = 0; + ll[i - 1].ll_flags = OCS_LL_DMA_FLAG_TERMINATE; + + return 0; +} diff --git a/drivers/crypto/intel/keembay/ocs-aes.h b/drivers/crypto/intel/keembay/ocs-aes.h new file mode 100644 index 000000000..c035fc48b --- /dev/null +++ b/drivers/crypto/intel/keembay/ocs-aes.h @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Intel Keem Bay OCS AES Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#ifndef _CRYPTO_OCS_AES_H +#define _CRYPTO_OCS_AES_H + +#include + +enum ocs_cipher { + OCS_AES = 0, + OCS_SM4 = 1, +}; + +enum ocs_mode { + OCS_MODE_ECB = 0, + OCS_MODE_CBC = 1, + OCS_MODE_CTR = 2, + OCS_MODE_CCM = 6, + OCS_MODE_GCM = 7, + OCS_MODE_CTS = 9, +}; + +enum ocs_instruction { + OCS_ENCRYPT = 0, + OCS_DECRYPT = 1, + OCS_EXPAND = 2, + OCS_BYPASS = 3, +}; + +/** + * struct ocs_aes_dev - AES device context. + * @list: List head for insertion into device list hold + * by driver. + * @dev: OCS AES device. + * @irq: IRQ number. + * @base_reg: IO base address of OCS AES. + * @irq_copy_completion: Completion to indicate IRQ has been triggered. + * @dma_err_mask: Error reported by OCS DMA interrupts. + * @engine: Crypto engine for the device. + */ +struct ocs_aes_dev { + struct list_head list; + struct device *dev; + int irq; + void __iomem *base_reg; + struct completion irq_completion; + u32 dma_err_mask; + struct crypto_engine *engine; +}; + +/** + * struct ocs_dll_desc - Descriptor of an OCS DMA Linked List. + * @vaddr: Virtual address of the linked list head. + * @dma_addr: DMA address of the linked list head. + * @size: Size (in bytes) of the linked list. + */ +struct ocs_dll_desc { + void *vaddr; + dma_addr_t dma_addr; + size_t size; +}; + +int ocs_aes_set_key(struct ocs_aes_dev *aes_dev, const u32 key_size, + const u8 *key, const enum ocs_cipher cipher); + +int ocs_aes_op(struct ocs_aes_dev *aes_dev, + enum ocs_mode mode, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + u8 *iv, + u32 iv_size); + +/** + * ocs_aes_bypass_op() - Use OCS DMA to copy data. + * @aes_dev: The OCS AES device to use. + * @dst_dma_list: The OCS DMA list mapping the memory where input data + * will be copied to. + * @src_dma_list: The OCS DMA list mapping input data. + * @src_size: The amount of data to copy. + */ +static inline int ocs_aes_bypass_op(struct ocs_aes_dev *aes_dev, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, u32 src_size) +{ + return ocs_aes_op(aes_dev, OCS_MODE_ECB, OCS_AES, OCS_BYPASS, + dst_dma_list, src_dma_list, src_size, NULL, 0); +} + +int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + const u8 *iv, + dma_addr_t aad_dma_list, + u32 aad_size, + u8 *out_tag, + u32 tag_size); + +int ocs_aes_ccm_op(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + u8 *iv, + dma_addr_t adata_dma_list, + u32 adata_size, + u8 *in_tag, + u32 tag_size); + +int ocs_create_linked_list_from_sg(const struct ocs_aes_dev *aes_dev, + struct scatterlist *sg, + int sg_dma_count, + struct ocs_dll_desc *dll_desc, + size_t data_size, + size_t data_offset); + +irqreturn_t ocs_aes_irq_handler(int irq, void *dev_id); + +#endif diff --git a/drivers/crypto/intel/keembay/ocs-hcu.c b/drivers/crypto/intel/keembay/ocs-hcu.c new file mode 100644 index 000000000..87711068d --- /dev/null +++ b/drivers/crypto/intel/keembay/ocs-hcu.c @@ -0,0 +1,841 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel Keem Bay OCS HCU Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#include +#include +#include +#include +#include + +#include + +#include "ocs-hcu.h" + +/* Registers. */ +#define OCS_HCU_MODE 0x00 +#define OCS_HCU_CHAIN 0x04 +#define OCS_HCU_OPERATION 0x08 +#define OCS_HCU_KEY_0 0x0C +#define OCS_HCU_ISR 0x50 +#define OCS_HCU_IER 0x54 +#define OCS_HCU_STATUS 0x58 +#define OCS_HCU_MSG_LEN_LO 0x60 +#define OCS_HCU_MSG_LEN_HI 0x64 +#define OCS_HCU_KEY_BYTE_ORDER_CFG 0x80 +#define OCS_HCU_DMA_SRC_ADDR 0x400 +#define OCS_HCU_DMA_SRC_SIZE 0x408 +#define OCS_HCU_DMA_DST_SIZE 0x40C +#define OCS_HCU_DMA_DMA_MODE 0x410 +#define OCS_HCU_DMA_NEXT_SRC_DESCR 0x418 +#define OCS_HCU_DMA_MSI_ISR 0x480 +#define OCS_HCU_DMA_MSI_IER 0x484 +#define OCS_HCU_DMA_MSI_MASK 0x488 + +/* Register bit definitions. */ +#define HCU_MODE_ALGO_SHIFT 16 +#define HCU_MODE_HMAC_SHIFT 22 + +#define HCU_STATUS_BUSY BIT(0) + +#define HCU_BYTE_ORDER_SWAP BIT(0) + +#define HCU_IRQ_HASH_DONE BIT(2) +#define HCU_IRQ_HASH_ERR_MASK (BIT(3) | BIT(1) | BIT(0)) + +#define HCU_DMA_IRQ_SRC_DONE BIT(0) +#define HCU_DMA_IRQ_SAI_ERR BIT(2) +#define HCU_DMA_IRQ_BAD_COMP_ERR BIT(3) +#define HCU_DMA_IRQ_INBUF_RD_ERR BIT(4) +#define HCU_DMA_IRQ_INBUF_WD_ERR BIT(5) +#define HCU_DMA_IRQ_OUTBUF_WR_ERR BIT(6) +#define HCU_DMA_IRQ_OUTBUF_RD_ERR BIT(7) +#define HCU_DMA_IRQ_CRD_ERR BIT(8) +#define HCU_DMA_IRQ_ERR_MASK (HCU_DMA_IRQ_SAI_ERR | \ + HCU_DMA_IRQ_BAD_COMP_ERR | \ + HCU_DMA_IRQ_INBUF_RD_ERR | \ + HCU_DMA_IRQ_INBUF_WD_ERR | \ + HCU_DMA_IRQ_OUTBUF_WR_ERR | \ + HCU_DMA_IRQ_OUTBUF_RD_ERR | \ + HCU_DMA_IRQ_CRD_ERR) + +#define HCU_DMA_SNOOP_MASK (0x7 << 28) +#define HCU_DMA_SRC_LL_EN BIT(25) +#define HCU_DMA_EN BIT(31) + +#define OCS_HCU_ENDIANNESS_VALUE 0x2A + +#define HCU_DMA_MSI_UNMASK BIT(0) +#define HCU_DMA_MSI_DISABLE 0 +#define HCU_IRQ_DISABLE 0 + +#define OCS_HCU_START BIT(0) +#define OCS_HCU_TERMINATE BIT(1) + +#define OCS_LL_DMA_FLAG_TERMINATE BIT(31) + +#define OCS_HCU_HW_KEY_LEN_U32 (OCS_HCU_HW_KEY_LEN / sizeof(u32)) + +#define HCU_DATA_WRITE_ENDIANNESS_OFFSET 26 + +#define OCS_HCU_NUM_CHAINS_SHA256_224_SM3 (SHA256_DIGEST_SIZE / sizeof(u32)) +#define OCS_HCU_NUM_CHAINS_SHA384_512 (SHA512_DIGEST_SIZE / sizeof(u32)) + +/* + * While polling on a busy HCU, wait maximum 200us between one check and the + * other. + */ +#define OCS_HCU_WAIT_BUSY_RETRY_DELAY_US 200 +/* Wait on a busy HCU for maximum 1 second. */ +#define OCS_HCU_WAIT_BUSY_TIMEOUT_US 1000000 + +/** + * struct ocs_hcu_dma_entry - An entry in an OCS DMA linked list. + * @src_addr: Source address of the data. + * @src_len: Length of data to be fetched. + * @nxt_desc: Next descriptor to fetch. + * @ll_flags: Flags (Freeze @ terminate) for the DMA engine. + */ +struct ocs_hcu_dma_entry { + u32 src_addr; + u32 src_len; + u32 nxt_desc; + u32 ll_flags; +}; + +/** + * struct ocs_hcu_dma_list - OCS-specific DMA linked list. + * @head: The head of the list (points to the array backing the list). + * @tail: The current tail of the list; NULL if the list is empty. + * @dma_addr: The DMA address of @head (i.e., the DMA address of the backing + * array). + * @max_nents: Maximum number of entries in the list (i.e., number of elements + * in the backing array). + * + * The OCS DMA list is an array-backed list of OCS DMA descriptors. The array + * backing the list is allocated with dma_alloc_coherent() and pointed by + * @head. + */ +struct ocs_hcu_dma_list { + struct ocs_hcu_dma_entry *head; + struct ocs_hcu_dma_entry *tail; + dma_addr_t dma_addr; + size_t max_nents; +}; + +static inline u32 ocs_hcu_num_chains(enum ocs_hcu_algo algo) +{ + switch (algo) { + case OCS_HCU_ALGO_SHA224: + case OCS_HCU_ALGO_SHA256: + case OCS_HCU_ALGO_SM3: + return OCS_HCU_NUM_CHAINS_SHA256_224_SM3; + case OCS_HCU_ALGO_SHA384: + case OCS_HCU_ALGO_SHA512: + return OCS_HCU_NUM_CHAINS_SHA384_512; + default: + return 0; + }; +} + +static inline u32 ocs_hcu_digest_size(enum ocs_hcu_algo algo) +{ + switch (algo) { + case OCS_HCU_ALGO_SHA224: + return SHA224_DIGEST_SIZE; + case OCS_HCU_ALGO_SHA256: + case OCS_HCU_ALGO_SM3: + /* SM3 shares the same block size. */ + return SHA256_DIGEST_SIZE; + case OCS_HCU_ALGO_SHA384: + return SHA384_DIGEST_SIZE; + case OCS_HCU_ALGO_SHA512: + return SHA512_DIGEST_SIZE; + default: + return 0; + } +} + +/** + * ocs_hcu_wait_busy() - Wait for HCU OCS hardware to became usable. + * @hcu_dev: OCS HCU device to wait for. + * + * Return: 0 if device free, -ETIMEOUT if device busy and internal timeout has + * expired. + */ +static int ocs_hcu_wait_busy(struct ocs_hcu_dev *hcu_dev) +{ + long val; + + return readl_poll_timeout(hcu_dev->io_base + OCS_HCU_STATUS, val, + !(val & HCU_STATUS_BUSY), + OCS_HCU_WAIT_BUSY_RETRY_DELAY_US, + OCS_HCU_WAIT_BUSY_TIMEOUT_US); +} + +static void ocs_hcu_done_irq_en(struct ocs_hcu_dev *hcu_dev) +{ + /* Clear any pending interrupts. */ + writel(0xFFFFFFFF, hcu_dev->io_base + OCS_HCU_ISR); + hcu_dev->irq_err = false; + /* Enable error and HCU done interrupts. */ + writel(HCU_IRQ_HASH_DONE | HCU_IRQ_HASH_ERR_MASK, + hcu_dev->io_base + OCS_HCU_IER); +} + +static void ocs_hcu_dma_irq_en(struct ocs_hcu_dev *hcu_dev) +{ + /* Clear any pending interrupts. */ + writel(0xFFFFFFFF, hcu_dev->io_base + OCS_HCU_DMA_MSI_ISR); + hcu_dev->irq_err = false; + /* Only operating on DMA source completion and error interrupts. */ + writel(HCU_DMA_IRQ_ERR_MASK | HCU_DMA_IRQ_SRC_DONE, + hcu_dev->io_base + OCS_HCU_DMA_MSI_IER); + /* Unmask */ + writel(HCU_DMA_MSI_UNMASK, hcu_dev->io_base + OCS_HCU_DMA_MSI_MASK); +} + +static void ocs_hcu_irq_dis(struct ocs_hcu_dev *hcu_dev) +{ + writel(HCU_IRQ_DISABLE, hcu_dev->io_base + OCS_HCU_IER); + writel(HCU_DMA_MSI_DISABLE, hcu_dev->io_base + OCS_HCU_DMA_MSI_IER); +} + +static int ocs_hcu_wait_and_disable_irq(struct ocs_hcu_dev *hcu_dev) +{ + int rc; + + rc = wait_for_completion_interruptible(&hcu_dev->irq_done); + if (rc) + goto exit; + + if (hcu_dev->irq_err) { + /* Unset flag and return error. */ + hcu_dev->irq_err = false; + rc = -EIO; + goto exit; + } + +exit: + ocs_hcu_irq_dis(hcu_dev); + + return rc; +} + +/** + * ocs_hcu_get_intermediate_data() - Get intermediate data. + * @hcu_dev: The target HCU device. + * @data: Where to store the intermediate. + * @algo: The algorithm being used. + * + * This function is used to save the current hashing process state in order to + * continue it in the future. + * + * Note: once all data has been processed, the intermediate data actually + * contains the hashing result. So this function is also used to retrieve the + * final result of a hashing process. + * + * Return: 0 on success, negative error code otherwise. + */ +static int ocs_hcu_get_intermediate_data(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_idata *data, + enum ocs_hcu_algo algo) +{ + const int n = ocs_hcu_num_chains(algo); + u32 *chain; + int rc; + int i; + + /* Data not requested. */ + if (!data) + return -EINVAL; + + chain = (u32 *)data->digest; + + /* Ensure that the OCS is no longer busy before reading the chains. */ + rc = ocs_hcu_wait_busy(hcu_dev); + if (rc) + return rc; + + /* + * This loops is safe because data->digest is an array of + * SHA512_DIGEST_SIZE bytes and the maximum value returned by + * ocs_hcu_num_chains() is OCS_HCU_NUM_CHAINS_SHA384_512 which is equal + * to SHA512_DIGEST_SIZE / sizeof(u32). + */ + for (i = 0; i < n; i++) + chain[i] = readl(hcu_dev->io_base + OCS_HCU_CHAIN); + + data->msg_len_lo = readl(hcu_dev->io_base + OCS_HCU_MSG_LEN_LO); + data->msg_len_hi = readl(hcu_dev->io_base + OCS_HCU_MSG_LEN_HI); + + return 0; +} + +/** + * ocs_hcu_set_intermediate_data() - Set intermediate data. + * @hcu_dev: The target HCU device. + * @data: The intermediate data to be set. + * @algo: The algorithm being used. + * + * This function is used to continue a previous hashing process. + */ +static void ocs_hcu_set_intermediate_data(struct ocs_hcu_dev *hcu_dev, + const struct ocs_hcu_idata *data, + enum ocs_hcu_algo algo) +{ + const int n = ocs_hcu_num_chains(algo); + u32 *chain = (u32 *)data->digest; + int i; + + /* + * This loops is safe because data->digest is an array of + * SHA512_DIGEST_SIZE bytes and the maximum value returned by + * ocs_hcu_num_chains() is OCS_HCU_NUM_CHAINS_SHA384_512 which is equal + * to SHA512_DIGEST_SIZE / sizeof(u32). + */ + for (i = 0; i < n; i++) + writel(chain[i], hcu_dev->io_base + OCS_HCU_CHAIN); + + writel(data->msg_len_lo, hcu_dev->io_base + OCS_HCU_MSG_LEN_LO); + writel(data->msg_len_hi, hcu_dev->io_base + OCS_HCU_MSG_LEN_HI); +} + +static int ocs_hcu_get_digest(struct ocs_hcu_dev *hcu_dev, + enum ocs_hcu_algo algo, u8 *dgst, size_t dgst_len) +{ + u32 *chain; + int rc; + int i; + + if (!dgst) + return -EINVAL; + + /* Length of the output buffer must match the algo digest size. */ + if (dgst_len != ocs_hcu_digest_size(algo)) + return -EINVAL; + + /* Ensure that the OCS is no longer busy before reading the chains. */ + rc = ocs_hcu_wait_busy(hcu_dev); + if (rc) + return rc; + + chain = (u32 *)dgst; + for (i = 0; i < dgst_len / sizeof(u32); i++) + chain[i] = readl(hcu_dev->io_base + OCS_HCU_CHAIN); + + return 0; +} + +/** + * ocs_hcu_hw_cfg() - Configure the HCU hardware. + * @hcu_dev: The HCU device to configure. + * @algo: The algorithm to be used by the HCU device. + * @use_hmac: Whether or not HW HMAC should be used. + * + * Return: 0 on success, negative error code otherwise. + */ +static int ocs_hcu_hw_cfg(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, + bool use_hmac) +{ + u32 cfg; + int rc; + + if (algo != OCS_HCU_ALGO_SHA256 && algo != OCS_HCU_ALGO_SHA224 && + algo != OCS_HCU_ALGO_SHA384 && algo != OCS_HCU_ALGO_SHA512 && + algo != OCS_HCU_ALGO_SM3) + return -EINVAL; + + rc = ocs_hcu_wait_busy(hcu_dev); + if (rc) + return rc; + + /* Ensure interrupts are disabled. */ + ocs_hcu_irq_dis(hcu_dev); + + /* Configure endianness, hashing algorithm and HW HMAC (if needed) */ + cfg = OCS_HCU_ENDIANNESS_VALUE << HCU_DATA_WRITE_ENDIANNESS_OFFSET; + cfg |= algo << HCU_MODE_ALGO_SHIFT; + if (use_hmac) + cfg |= BIT(HCU_MODE_HMAC_SHIFT); + + writel(cfg, hcu_dev->io_base + OCS_HCU_MODE); + + return 0; +} + +/** + * ocs_hcu_clear_key() - Clear key stored in OCS HMAC KEY registers. + * @hcu_dev: The OCS HCU device whose key registers should be cleared. + */ +static void ocs_hcu_clear_key(struct ocs_hcu_dev *hcu_dev) +{ + int reg_off; + + /* Clear OCS_HCU_KEY_[0..15] */ + for (reg_off = 0; reg_off < OCS_HCU_HW_KEY_LEN; reg_off += sizeof(u32)) + writel(0, hcu_dev->io_base + OCS_HCU_KEY_0 + reg_off); +} + +/** + * ocs_hcu_write_key() - Write key to OCS HMAC KEY registers. + * @hcu_dev: The OCS HCU device the key should be written to. + * @key: The key to be written. + * @len: The size of the key to write. It must be OCS_HCU_HW_KEY_LEN. + * + * Return: 0 on success, negative error code otherwise. + */ +static int ocs_hcu_write_key(struct ocs_hcu_dev *hcu_dev, const u8 *key, size_t len) +{ + u32 key_u32[OCS_HCU_HW_KEY_LEN_U32]; + int i; + + if (len > OCS_HCU_HW_KEY_LEN) + return -EINVAL; + + /* Copy key into temporary u32 array. */ + memcpy(key_u32, key, len); + + /* + * Hardware requires all the bytes of the HW Key vector to be + * written. So pad with zero until we reach OCS_HCU_HW_KEY_LEN. + */ + memzero_explicit((u8 *)key_u32 + len, OCS_HCU_HW_KEY_LEN - len); + + /* + * OCS hardware expects the MSB of the key to be written at the highest + * address of the HCU Key vector; in other word, the key must be + * written in reverse order. + * + * Therefore, we first enable byte swapping for the HCU key vector; + * so that bytes of 32-bit word written to OCS_HCU_KEY_[0..15] will be + * swapped: + * 3 <---> 0, 2 <---> 1. + */ + writel(HCU_BYTE_ORDER_SWAP, + hcu_dev->io_base + OCS_HCU_KEY_BYTE_ORDER_CFG); + /* + * And then we write the 32-bit words composing the key starting from + * the end of the key. + */ + for (i = 0; i < OCS_HCU_HW_KEY_LEN_U32; i++) + writel(key_u32[OCS_HCU_HW_KEY_LEN_U32 - 1 - i], + hcu_dev->io_base + OCS_HCU_KEY_0 + (sizeof(u32) * i)); + + memzero_explicit(key_u32, OCS_HCU_HW_KEY_LEN); + + return 0; +} + +/** + * ocs_hcu_ll_dma_start() - Start OCS HCU hashing via DMA + * @hcu_dev: The OCS HCU device to use. + * @dma_list: The OCS DMA list mapping the data to hash. + * @finalize: Whether or not this is the last hashing operation and therefore + * the final hash should be compute even if data is not + * block-aligned. + * + * Return: 0 on success, negative error code otherwise. + */ +static int ocs_hcu_ll_dma_start(struct ocs_hcu_dev *hcu_dev, + const struct ocs_hcu_dma_list *dma_list, + bool finalize) +{ + u32 cfg = HCU_DMA_SNOOP_MASK | HCU_DMA_SRC_LL_EN | HCU_DMA_EN; + int rc; + + if (!dma_list) + return -EINVAL; + + /* + * For final requests we use HCU_DONE IRQ to be notified when all input + * data has been processed by the HCU; however, we cannot do so for + * non-final requests, because we don't get a HCU_DONE IRQ when we + * don't terminate the operation. + * + * Therefore, for non-final requests, we use the DMA IRQ, which + * triggers when DMA has finishing feeding all the input data to the + * HCU, but the HCU may still be processing it. This is fine, since we + * will wait for the HCU processing to be completed when we try to read + * intermediate results, in ocs_hcu_get_intermediate_data(). + */ + if (finalize) + ocs_hcu_done_irq_en(hcu_dev); + else + ocs_hcu_dma_irq_en(hcu_dev); + + reinit_completion(&hcu_dev->irq_done); + writel(dma_list->dma_addr, hcu_dev->io_base + OCS_HCU_DMA_NEXT_SRC_DESCR); + writel(0, hcu_dev->io_base + OCS_HCU_DMA_SRC_SIZE); + writel(0, hcu_dev->io_base + OCS_HCU_DMA_DST_SIZE); + + writel(OCS_HCU_START, hcu_dev->io_base + OCS_HCU_OPERATION); + + writel(cfg, hcu_dev->io_base + OCS_HCU_DMA_DMA_MODE); + + if (finalize) + writel(OCS_HCU_TERMINATE, hcu_dev->io_base + OCS_HCU_OPERATION); + + rc = ocs_hcu_wait_and_disable_irq(hcu_dev); + if (rc) + return rc; + + return 0; +} + +struct ocs_hcu_dma_list *ocs_hcu_dma_list_alloc(struct ocs_hcu_dev *hcu_dev, + int max_nents) +{ + struct ocs_hcu_dma_list *dma_list; + + dma_list = kmalloc_obj(*dma_list); + if (!dma_list) + return NULL; + + /* Total size of the DMA list to allocate. */ + dma_list->head = dma_alloc_coherent(hcu_dev->dev, + sizeof(*dma_list->head) * max_nents, + &dma_list->dma_addr, GFP_KERNEL); + if (!dma_list->head) { + kfree(dma_list); + return NULL; + } + dma_list->max_nents = max_nents; + dma_list->tail = NULL; + + return dma_list; +} + +void ocs_hcu_dma_list_free(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_dma_list *dma_list) +{ + if (!dma_list) + return; + + dma_free_coherent(hcu_dev->dev, + sizeof(*dma_list->head) * dma_list->max_nents, + dma_list->head, dma_list->dma_addr); + + kfree(dma_list); +} + +/* Add a new DMA entry at the end of the OCS DMA list. */ +int ocs_hcu_dma_list_add_tail(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_dma_list *dma_list, + dma_addr_t addr, u32 len) +{ + struct device *dev = hcu_dev->dev; + struct ocs_hcu_dma_entry *old_tail; + struct ocs_hcu_dma_entry *new_tail; + + if (!len) + return 0; + + if (!dma_list) + return -EINVAL; + + if (addr & ~OCS_HCU_DMA_BIT_MASK) { + dev_err(dev, + "Unexpected error: Invalid DMA address for OCS HCU\n"); + return -EINVAL; + } + + old_tail = dma_list->tail; + new_tail = old_tail ? old_tail + 1 : dma_list->head; + + /* Check if list is full. */ + if (new_tail - dma_list->head >= dma_list->max_nents) + return -ENOMEM; + + /* + * If there was an old tail (i.e., this is not the first element we are + * adding), un-terminate the old tail and make it point to the new one. + */ + if (old_tail) { + old_tail->ll_flags &= ~OCS_LL_DMA_FLAG_TERMINATE; + /* + * The old tail 'nxt_desc' must point to the DMA address of the + * new tail. + */ + old_tail->nxt_desc = dma_list->dma_addr + + sizeof(*dma_list->tail) * (new_tail - + dma_list->head); + } + + new_tail->src_addr = (u32)addr; + new_tail->src_len = (u32)len; + new_tail->ll_flags = OCS_LL_DMA_FLAG_TERMINATE; + new_tail->nxt_desc = 0; + + /* Update list tail with new tail. */ + dma_list->tail = new_tail; + + return 0; +} + +/** + * ocs_hcu_hash_init() - Initialize hash operation context. + * @ctx: The context to initialize. + * @algo: The hashing algorithm to use. + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_hcu_hash_init(struct ocs_hcu_hash_ctx *ctx, enum ocs_hcu_algo algo) +{ + if (!ctx) + return -EINVAL; + + ctx->algo = algo; + ctx->idata.msg_len_lo = 0; + ctx->idata.msg_len_hi = 0; + /* No need to set idata.digest to 0. */ + + return 0; +} + +/** + * ocs_hcu_hash_update() - Perform a hashing iteration. + * @hcu_dev: The OCS HCU device to use. + * @ctx: The OCS HCU hashing context. + * @dma_list: The OCS DMA list mapping the input data to process. + * + * Return: 0 on success; negative error code otherwise. + */ +int ocs_hcu_hash_update(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_hash_ctx *ctx, + const struct ocs_hcu_dma_list *dma_list) +{ + int rc; + + if (!hcu_dev || !ctx) + return -EINVAL; + + /* Configure the hardware for the current request. */ + rc = ocs_hcu_hw_cfg(hcu_dev, ctx->algo, false); + if (rc) + return rc; + + /* If we already processed some data, idata needs to be set. */ + if (ctx->idata.msg_len_lo || ctx->idata.msg_len_hi) + ocs_hcu_set_intermediate_data(hcu_dev, &ctx->idata, ctx->algo); + + /* Start linked-list DMA hashing. */ + rc = ocs_hcu_ll_dma_start(hcu_dev, dma_list, false); + if (rc) + return rc; + + /* Update idata and return. */ + return ocs_hcu_get_intermediate_data(hcu_dev, &ctx->idata, ctx->algo); +} + +/** + * ocs_hcu_hash_finup() - Update and finalize hash computation. + * @hcu_dev: The OCS HCU device to use. + * @ctx: The OCS HCU hashing context. + * @dma_list: The OCS DMA list mapping the input data to process. + * @dgst: The buffer where to save the computed digest. + * @dgst_len: The length of @dgst. + * + * Return: 0 on success; negative error code otherwise. + */ +int ocs_hcu_hash_finup(struct ocs_hcu_dev *hcu_dev, + const struct ocs_hcu_hash_ctx *ctx, + const struct ocs_hcu_dma_list *dma_list, + u8 *dgst, size_t dgst_len) +{ + int rc; + + if (!hcu_dev || !ctx) + return -EINVAL; + + /* Configure the hardware for the current request. */ + rc = ocs_hcu_hw_cfg(hcu_dev, ctx->algo, false); + if (rc) + return rc; + + /* If we already processed some data, idata needs to be set. */ + if (ctx->idata.msg_len_lo || ctx->idata.msg_len_hi) + ocs_hcu_set_intermediate_data(hcu_dev, &ctx->idata, ctx->algo); + + /* Start linked-list DMA hashing. */ + rc = ocs_hcu_ll_dma_start(hcu_dev, dma_list, true); + if (rc) + return rc; + + /* Get digest and return. */ + return ocs_hcu_get_digest(hcu_dev, ctx->algo, dgst, dgst_len); +} + +/** + * ocs_hcu_hash_final() - Finalize hash computation. + * @hcu_dev: The OCS HCU device to use. + * @ctx: The OCS HCU hashing context. + * @dgst: The buffer where to save the computed digest. + * @dgst_len: The length of @dgst. + * + * Return: 0 on success; negative error code otherwise. + */ +int ocs_hcu_hash_final(struct ocs_hcu_dev *hcu_dev, + const struct ocs_hcu_hash_ctx *ctx, u8 *dgst, + size_t dgst_len) +{ + int rc; + + if (!hcu_dev || !ctx) + return -EINVAL; + + /* Configure the hardware for the current request. */ + rc = ocs_hcu_hw_cfg(hcu_dev, ctx->algo, false); + if (rc) + return rc; + + /* If we already processed some data, idata needs to be set. */ + if (ctx->idata.msg_len_lo || ctx->idata.msg_len_hi) + ocs_hcu_set_intermediate_data(hcu_dev, &ctx->idata, ctx->algo); + + /* + * Enable HCU interrupts, so that HCU_DONE will be triggered once the + * final hash is computed. + */ + ocs_hcu_done_irq_en(hcu_dev); + reinit_completion(&hcu_dev->irq_done); + writel(OCS_HCU_TERMINATE, hcu_dev->io_base + OCS_HCU_OPERATION); + + rc = ocs_hcu_wait_and_disable_irq(hcu_dev); + if (rc) + return rc; + + /* Get digest and return. */ + return ocs_hcu_get_digest(hcu_dev, ctx->algo, dgst, dgst_len); +} + +/** + * ocs_hcu_digest() - Compute hash digest. + * @hcu_dev: The OCS HCU device to use. + * @algo: The hash algorithm to use. + * @data: The input data to process. + * @data_len: The length of @data. + * @dgst: The buffer where to save the computed digest. + * @dgst_len: The length of @dgst. + * + * Return: 0 on success; negative error code otherwise. + */ +int ocs_hcu_digest(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, + void *data, size_t data_len, u8 *dgst, size_t dgst_len) +{ + struct device *dev = hcu_dev->dev; + dma_addr_t dma_handle; + u32 reg; + int rc; + + /* Configure the hardware for the current request. */ + rc = ocs_hcu_hw_cfg(hcu_dev, algo, false); + if (rc) + return rc; + + dma_handle = dma_map_single(dev, data, data_len, DMA_TO_DEVICE); + if (dma_mapping_error(dev, dma_handle)) + return -EIO; + + reg = HCU_DMA_SNOOP_MASK | HCU_DMA_EN; + + ocs_hcu_done_irq_en(hcu_dev); + + reinit_completion(&hcu_dev->irq_done); + + writel(dma_handle, hcu_dev->io_base + OCS_HCU_DMA_SRC_ADDR); + writel(data_len, hcu_dev->io_base + OCS_HCU_DMA_SRC_SIZE); + writel(OCS_HCU_START, hcu_dev->io_base + OCS_HCU_OPERATION); + writel(reg, hcu_dev->io_base + OCS_HCU_DMA_DMA_MODE); + + writel(OCS_HCU_TERMINATE, hcu_dev->io_base + OCS_HCU_OPERATION); + + rc = ocs_hcu_wait_and_disable_irq(hcu_dev); + if (rc) + return rc; + + dma_unmap_single(dev, dma_handle, data_len, DMA_TO_DEVICE); + + return ocs_hcu_get_digest(hcu_dev, algo, dgst, dgst_len); +} + +/** + * ocs_hcu_hmac() - Compute HMAC. + * @hcu_dev: The OCS HCU device to use. + * @algo: The hash algorithm to use with HMAC. + * @key: The key to use. + * @dma_list: The OCS DMA list mapping the input data to process. + * @key_len: The length of @key. + * @dgst: The buffer where to save the computed HMAC. + * @dgst_len: The length of @dgst. + * + * Return: 0 on success; negative error code otherwise. + */ +int ocs_hcu_hmac(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, + const u8 *key, size_t key_len, + const struct ocs_hcu_dma_list *dma_list, + u8 *dgst, size_t dgst_len) +{ + int rc; + + /* Ensure 'key' is not NULL. */ + if (!key || key_len == 0) + return -EINVAL; + + /* Configure the hardware for the current request. */ + rc = ocs_hcu_hw_cfg(hcu_dev, algo, true); + if (rc) + return rc; + + rc = ocs_hcu_write_key(hcu_dev, key, key_len); + if (rc) + return rc; + + rc = ocs_hcu_ll_dma_start(hcu_dev, dma_list, true); + + /* Clear HW key before processing return code. */ + ocs_hcu_clear_key(hcu_dev); + + if (rc) + return rc; + + return ocs_hcu_get_digest(hcu_dev, algo, dgst, dgst_len); +} + +irqreturn_t ocs_hcu_irq_handler(int irq, void *dev_id) +{ + struct ocs_hcu_dev *hcu_dev = dev_id; + u32 hcu_irq; + u32 dma_irq; + + /* Read and clear the HCU interrupt. */ + hcu_irq = readl(hcu_dev->io_base + OCS_HCU_ISR); + writel(hcu_irq, hcu_dev->io_base + OCS_HCU_ISR); + + /* Read and clear the HCU DMA interrupt. */ + dma_irq = readl(hcu_dev->io_base + OCS_HCU_DMA_MSI_ISR); + writel(dma_irq, hcu_dev->io_base + OCS_HCU_DMA_MSI_ISR); + + /* Check for errors. */ + if (hcu_irq & HCU_IRQ_HASH_ERR_MASK || dma_irq & HCU_DMA_IRQ_ERR_MASK) { + hcu_dev->irq_err = true; + goto complete; + } + + /* Check for DONE IRQs. */ + if (hcu_irq & HCU_IRQ_HASH_DONE || dma_irq & HCU_DMA_IRQ_SRC_DONE) + goto complete; + + return IRQ_NONE; + +complete: + complete(&hcu_dev->irq_done); + + return IRQ_HANDLED; +} + +MODULE_DESCRIPTION("Intel Keem Bay OCS HCU Crypto Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/intel/keembay/ocs-hcu.h b/drivers/crypto/intel/keembay/ocs-hcu.h new file mode 100644 index 000000000..fbbbb92a0 --- /dev/null +++ b/drivers/crypto/intel/keembay/ocs-hcu.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Intel Keem Bay OCS HCU Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#include + +#ifndef _CRYPTO_OCS_HCU_H +#define _CRYPTO_OCS_HCU_H + +#define OCS_HCU_DMA_BIT_MASK DMA_BIT_MASK(32) + +#define OCS_HCU_HW_KEY_LEN 64 + +struct ocs_hcu_dma_list; + +enum ocs_hcu_algo { + OCS_HCU_ALGO_SHA256 = 2, + OCS_HCU_ALGO_SHA224 = 3, + OCS_HCU_ALGO_SHA384 = 4, + OCS_HCU_ALGO_SHA512 = 5, + OCS_HCU_ALGO_SM3 = 6, +}; + +/** + * struct ocs_hcu_dev - OCS HCU device context. + * @list: List of device contexts. + * @dev: OCS HCU device. + * @io_base: Base address of OCS HCU registers. + * @engine: Crypto engine for the device. + * @irq: IRQ number. + * @irq_done: Completion for IRQ. + * @irq_err: Flag indicating an IRQ error has happened. + */ +struct ocs_hcu_dev { + struct list_head list; + struct device *dev; + void __iomem *io_base; + struct crypto_engine *engine; + int irq; + struct completion irq_done; + bool irq_err; +}; + +/** + * struct ocs_hcu_idata - Intermediate data generated by the HCU. + * @msg_len_lo: Length of data the HCU has operated on in bits, low 32b. + * @msg_len_hi: Length of data the HCU has operated on in bits, high 32b. + * @digest: The digest read from the HCU. If the HCU is terminated, it will + * contain the actual hash digest. Otherwise it is the intermediate + * state. + */ +struct ocs_hcu_idata { + u32 msg_len_lo; + u32 msg_len_hi; + u8 digest[SHA512_DIGEST_SIZE]; +}; + +/** + * struct ocs_hcu_hash_ctx - Context for OCS HCU hashing operation. + * @algo: The hashing algorithm being used. + * @idata: The current intermediate data. + */ +struct ocs_hcu_hash_ctx { + enum ocs_hcu_algo algo; + struct ocs_hcu_idata idata; +}; + +irqreturn_t ocs_hcu_irq_handler(int irq, void *dev_id); + +struct ocs_hcu_dma_list *ocs_hcu_dma_list_alloc(struct ocs_hcu_dev *hcu_dev, + int max_nents); + +void ocs_hcu_dma_list_free(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_dma_list *dma_list); + +int ocs_hcu_dma_list_add_tail(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_dma_list *dma_list, + dma_addr_t addr, u32 len); + +int ocs_hcu_hash_init(struct ocs_hcu_hash_ctx *ctx, enum ocs_hcu_algo algo); + +int ocs_hcu_hash_update(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_hash_ctx *ctx, + const struct ocs_hcu_dma_list *dma_list); + +int ocs_hcu_hash_finup(struct ocs_hcu_dev *hcu_dev, + const struct ocs_hcu_hash_ctx *ctx, + const struct ocs_hcu_dma_list *dma_list, + u8 *dgst, size_t dgst_len); + +int ocs_hcu_hash_final(struct ocs_hcu_dev *hcu_dev, + const struct ocs_hcu_hash_ctx *ctx, u8 *dgst, + size_t dgst_len); + +int ocs_hcu_digest(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, + void *data, size_t data_len, u8 *dgst, size_t dgst_len); + +int ocs_hcu_hmac(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, + const u8 *key, size_t key_len, + const struct ocs_hcu_dma_list *dma_list, + u8 *dgst, size_t dgst_len); + +#endif /* _CRYPTO_OCS_HCU_H */ diff --git a/drivers/crypto/intel/qat/Kconfig b/drivers/crypto/intel/qat/Kconfig new file mode 100644 index 000000000..9d6e6f52d --- /dev/null +++ b/drivers/crypto/intel/qat/Kconfig @@ -0,0 +1,135 @@ +# SPDX-License-Identifier: GPL-2.0-only +config CRYPTO_DEV_QAT + tristate + select CRYPTO_ACOMP + select CRYPTO_AEAD + select CRYPTO_AUTHENC + select CRYPTO_SKCIPHER + select CRYPTO_AKCIPHER + select CRYPTO_DH + select CRYPTO_RSA + select CRYPTO_LIB_AES + select CRYPTO_LIB_SHA1 + select CRYPTO_LIB_SHA256 + select CRYPTO_LIB_SHA512 + select CRYPTO_ZSTD + select FW_LOADER + select CRC8 + +config CRYPTO_DEV_QAT_DH895xCC + tristate "Support for Intel(R) DH895xCC" + depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) + select CRYPTO_DEV_QAT + help + Support for Intel(R) DH895xcc with Intel(R) QuickAssist Technology + for accelerating crypto and compression workloads. + + To compile this as a module, choose M here: the module + will be called qat_dh895xcc. + +config CRYPTO_DEV_QAT_C3XXX + tristate "Support for Intel(R) C3XXX" + depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) + select CRYPTO_DEV_QAT + help + Support for Intel(R) C3xxx with Intel(R) QuickAssist Technology + for accelerating crypto and compression workloads. + + To compile this as a module, choose M here: the module + will be called qat_c3xxx. + +config CRYPTO_DEV_QAT_C62X + tristate "Support for Intel(R) C62X" + depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) + select CRYPTO_DEV_QAT + help + Support for Intel(R) C62x with Intel(R) QuickAssist Technology + for accelerating crypto and compression workloads. + + To compile this as a module, choose M here: the module + will be called qat_c62x. + +config CRYPTO_DEV_QAT_4XXX + tristate "Support for Intel(R) QAT_4XXX" + depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) + select CRYPTO_DEV_QAT + help + Support for Intel(R) QuickAssist Technology QAT_4xxx + for accelerating crypto and compression workloads. + + To compile this as a module, choose M here: the module + will be called qat_4xxx. + +config CRYPTO_DEV_QAT_420XX + tristate "Support for Intel(R) QAT_420XX" + depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) + select CRYPTO_DEV_QAT + help + Support for Intel(R) QuickAssist Technology QAT_420xx + for accelerating crypto and compression workloads. + + To compile this as a module, choose M here: the module + will be called qat_420xx. + +config CRYPTO_DEV_QAT_6XXX + tristate "Support for Intel(R) QuickAssist Technology QAT_6XXX" + depends on (X86 || COMPILE_TEST) + depends on PCI + select CRYPTO_DEV_QAT + help + Support for Intel(R) QuickAssist Technology QAT_6xxx + for accelerating crypto and compression workloads. + + To compile this as a module, choose M here: the module + will be called qat_6xxx. + +config CRYPTO_DEV_QAT_DH895xCCVF + tristate "Support for Intel(R) DH895xCC Virtual Function" + depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) + select PCI_IOV + select CRYPTO_DEV_QAT + + help + Support for Intel(R) DH895xcc with Intel(R) QuickAssist Technology + Virtual Function for accelerating crypto and compression workloads. + + To compile this as a module, choose M here: the module + will be called qat_dh895xccvf. + +config CRYPTO_DEV_QAT_C3XXXVF + tristate "Support for Intel(R) C3XXX Virtual Function" + depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) + select PCI_IOV + select CRYPTO_DEV_QAT + help + Support for Intel(R) C3xxx with Intel(R) QuickAssist Technology + Virtual Function for accelerating crypto and compression workloads. + + To compile this as a module, choose M here: the module + will be called qat_c3xxxvf. + +config CRYPTO_DEV_QAT_C62XVF + tristate "Support for Intel(R) C62X Virtual Function" + depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) + select PCI_IOV + select CRYPTO_DEV_QAT + help + Support for Intel(R) C62x with Intel(R) QuickAssist Technology + Virtual Function for accelerating crypto and compression workloads. + + To compile this as a module, choose M here: the module + will be called qat_c62xvf. + +config CRYPTO_DEV_QAT_ERROR_INJECTION + bool "Support for Intel(R) QAT Devices Heartbeat Error Injection" + depends on CRYPTO_DEV_QAT + depends on DEBUG_FS + help + Enables a mechanism that allows to inject a heartbeat error on + Intel(R) QuickAssist devices for testing purposes. + + This is intended for developer use only. + If unsure, say N. + + This functionality is available via debugfs entry of the Intel(R) + QuickAssist device diff --git a/drivers/crypto/intel/qat/Makefile b/drivers/crypto/intel/qat/Makefile new file mode 100644 index 000000000..abef14207 --- /dev/null +++ b/drivers/crypto/intel/qat/Makefile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0 +subdir-ccflags-y := -I$(src)/qat_common +obj-$(CONFIG_CRYPTO_DEV_QAT) += qat_common/ +obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc/ +obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXX) += qat_c3xxx/ +obj-$(CONFIG_CRYPTO_DEV_QAT_C62X) += qat_c62x/ +obj-$(CONFIG_CRYPTO_DEV_QAT_4XXX) += qat_4xxx/ +obj-$(CONFIG_CRYPTO_DEV_QAT_420XX) += qat_420xx/ +obj-$(CONFIG_CRYPTO_DEV_QAT_6XXX) += qat_6xxx/ +obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCCVF) += qat_dh895xccvf/ +obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXXVF) += qat_c3xxxvf/ +obj-$(CONFIG_CRYPTO_DEV_QAT_C62XVF) += qat_c62xvf/ diff --git a/drivers/crypto/intel/qat/qat_420xx/Makefile b/drivers/crypto/intel/qat/qat_420xx/Makefile new file mode 100644 index 000000000..f6df54d29 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_420xx/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_QAT_420XX) += qat_420xx.o +qat_420xx-y := adf_drv.o adf_420xx_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c new file mode 100644 index 000000000..19f9f7386 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c @@ -0,0 +1,506 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "adf_420xx_hw_data.h" +#include "icp_qat_hw.h" + +#define ADF_AE_GROUP_0 GENMASK(3, 0) +#define ADF_AE_GROUP_1 GENMASK(7, 4) +#define ADF_AE_GROUP_2 GENMASK(11, 8) +#define ADF_AE_GROUP_3 GENMASK(15, 12) +#define ADF_AE_GROUP_4 BIT(16) + +#define ENA_THD_MASK_ASYM GENMASK(1, 0) +#define ENA_THD_MASK_SYM GENMASK(3, 0) +#define ENA_THD_MASK_DC GENMASK(1, 0) + +static const char * const adf_420xx_fw_objs[] = { + [ADF_FW_SYM_OBJ] = ADF_420XX_SYM_OBJ, + [ADF_FW_ASYM_OBJ] = ADF_420XX_ASYM_OBJ, + [ADF_FW_DC_OBJ] = ADF_420XX_DC_OBJ, + [ADF_FW_ADMIN_OBJ] = ADF_420XX_ADMIN_OBJ, +}; + +static const struct adf_fw_config adf_fw_cy_config[] = { + {ADF_AE_GROUP_3, ADF_FW_SYM_OBJ}, + {ADF_AE_GROUP_2, ADF_FW_ASYM_OBJ}, + {ADF_AE_GROUP_1, ADF_FW_SYM_OBJ}, + {ADF_AE_GROUP_0, ADF_FW_ASYM_OBJ}, + {ADF_AE_GROUP_4, ADF_FW_ADMIN_OBJ}, +}; + +static const struct adf_fw_config adf_fw_dc_config[] = { + {ADF_AE_GROUP_1, ADF_FW_DC_OBJ}, + {ADF_AE_GROUP_0, ADF_FW_DC_OBJ}, + {ADF_AE_GROUP_4, ADF_FW_ADMIN_OBJ}, +}; + +static const struct adf_fw_config adf_fw_sym_config[] = { + {ADF_AE_GROUP_3, ADF_FW_SYM_OBJ}, + {ADF_AE_GROUP_2, ADF_FW_SYM_OBJ}, + {ADF_AE_GROUP_1, ADF_FW_SYM_OBJ}, + {ADF_AE_GROUP_0, ADF_FW_SYM_OBJ}, + {ADF_AE_GROUP_4, ADF_FW_ADMIN_OBJ}, +}; + +static const struct adf_fw_config adf_fw_asym_config[] = { + {ADF_AE_GROUP_3, ADF_FW_ASYM_OBJ}, + {ADF_AE_GROUP_2, ADF_FW_ASYM_OBJ}, + {ADF_AE_GROUP_1, ADF_FW_ASYM_OBJ}, + {ADF_AE_GROUP_0, ADF_FW_ASYM_OBJ}, + {ADF_AE_GROUP_4, ADF_FW_ADMIN_OBJ}, +}; + +static const struct adf_fw_config adf_fw_asym_dc_config[] = { + {ADF_AE_GROUP_3, ADF_FW_ASYM_OBJ}, + {ADF_AE_GROUP_2, ADF_FW_ASYM_OBJ}, + {ADF_AE_GROUP_1, ADF_FW_ASYM_OBJ}, + {ADF_AE_GROUP_0, ADF_FW_DC_OBJ}, + {ADF_AE_GROUP_4, ADF_FW_ADMIN_OBJ}, +}; + +static const struct adf_fw_config adf_fw_sym_dc_config[] = { + {ADF_AE_GROUP_2, ADF_FW_SYM_OBJ}, + {ADF_AE_GROUP_1, ADF_FW_SYM_OBJ}, + {ADF_AE_GROUP_0, ADF_FW_DC_OBJ}, + {ADF_AE_GROUP_4, ADF_FW_ADMIN_OBJ}, +}; + +static const struct adf_fw_config adf_fw_dcc_config[] = { + {ADF_AE_GROUP_1, ADF_FW_DC_OBJ}, + {ADF_AE_GROUP_0, ADF_FW_SYM_OBJ}, + {ADF_AE_GROUP_4, ADF_FW_ADMIN_OBJ}, +}; + + +static struct adf_hw_device_class adf_420xx_class = { + .name = ADF_420XX_DEVICE_NAME, + .type = DEV_420XX, +}; + +static u32 get_ae_mask(struct adf_hw_device_data *self) +{ + unsigned long fuses = self->fuses[ADF_FUSECTL4]; + u32 mask = ADF_420XX_ACCELENGINES_MASK; + + if (test_bit(0, &fuses)) + mask &= ~ADF_AE_GROUP_0; + + if (test_bit(4, &fuses)) + mask &= ~ADF_AE_GROUP_1; + + if (test_bit(8, &fuses)) + mask &= ~ADF_AE_GROUP_2; + + if (test_bit(12, &fuses)) + mask &= ~ADF_AE_GROUP_3; + + if (test_bit(16, &fuses)) + mask &= ~ADF_AE_GROUP_4; + + return mask; +} + +static u32 uof_get_num_objs(struct adf_accel_dev *accel_dev) +{ + switch (adf_get_service_enabled(accel_dev)) { + case SVC_SYM_ASYM: + return ARRAY_SIZE(adf_fw_cy_config); + case SVC_DC: + return ARRAY_SIZE(adf_fw_dc_config); + case SVC_DCC: + return ARRAY_SIZE(adf_fw_dcc_config); + case SVC_SYM: + return ARRAY_SIZE(adf_fw_sym_config); + case SVC_ASYM: + return ARRAY_SIZE(adf_fw_asym_config); + case SVC_ASYM_DC: + return ARRAY_SIZE(adf_fw_asym_dc_config); + case SVC_SYM_DC: + return ARRAY_SIZE(adf_fw_sym_dc_config); + default: + return 0; + } +} + +static const struct adf_fw_config *get_fw_config(struct adf_accel_dev *accel_dev) +{ + switch (adf_get_service_enabled(accel_dev)) { + case SVC_SYM_ASYM: + return adf_fw_cy_config; + case SVC_DC: + return adf_fw_dc_config; + case SVC_DCC: + return adf_fw_dcc_config; + case SVC_SYM: + return adf_fw_sym_config; + case SVC_ASYM: + return adf_fw_asym_config; + case SVC_ASYM_DC: + return adf_fw_asym_dc_config; + case SVC_SYM_DC: + return adf_fw_sym_dc_config; + default: + return NULL; + } +} + +static void update_ae_mask(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + const struct adf_fw_config *fw_config; + u32 config_ae_mask = 0; + u32 ae_mask, num_objs; + int i; + + ae_mask = get_ae_mask(hw_data); + + /* Modify the AE mask based on the firmware configuration loaded */ + fw_config = get_fw_config(accel_dev); + num_objs = uof_get_num_objs(accel_dev); + + config_ae_mask |= ADF_420XX_ADMIN_AE_MASK; + for (i = 0; i < num_objs; i++) + config_ae_mask |= fw_config[i].ae_mask; + + hw_data->ae_mask = ae_mask & config_ae_mask; +} + +static u32 get_accel_cap(struct adf_accel_dev *accel_dev) +{ + u32 capabilities_sym, capabilities_asym, capabilities_dc; + struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev; + u32 capabilities_dcc; + u32 fusectl1; + + /* As a side effect, update ae_mask based on configuration */ + update_ae_mask(accel_dev); + + /* Read accelerator capabilities mask */ + pci_read_config_dword(pdev, ADF_GEN4_FUSECTL1_OFFSET, &fusectl1); + + capabilities_sym = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | + ICP_ACCEL_CAPABILITIES_CIPHER | + ICP_ACCEL_CAPABILITIES_AUTHENTICATION | + ICP_ACCEL_CAPABILITIES_SHA3 | + ICP_ACCEL_CAPABILITIES_SHA3_EXT | + ICP_ACCEL_CAPABILITIES_HKDF | + ICP_ACCEL_CAPABILITIES_CHACHA_POLY | + ICP_ACCEL_CAPABILITIES_AESGCM_SPC | + ICP_ACCEL_CAPABILITIES_SM3 | + ICP_ACCEL_CAPABILITIES_SM4 | + ICP_ACCEL_CAPABILITIES_AES_V2 | + ICP_ACCEL_CAPABILITIES_ZUC | + ICP_ACCEL_CAPABILITIES_WIRELESS_CRYPTO_EXT | + ICP_ACCEL_CAPABILITIES_EXT_ALGCHAIN; + + /* A set bit in fusectl1 means the feature is OFF in this SKU */ + if (fusectl1 & ICP_ACCEL_GEN4_MASK_CIPHER_SLICE) { + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_HKDF; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + } + + if (fusectl1 & ICP_ACCEL_GEN4_MASK_UCS_SLICE) { + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CHACHA_POLY; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AESGCM_SPC; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AES_V2; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + } + + if (fusectl1 & ICP_ACCEL_GEN4_MASK_AUTH_SLICE) { + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SHA3; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SHA3_EXT; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + } + + if (fusectl1 & ICP_ACCEL_GEN4_MASK_SMX_SLICE) { + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SM3; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SM4; + } + + if (fusectl1 & ICP_ACCEL_GEN4_MASK_WCP_WAT_SLICE) { + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_ZUC; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_WIRELESS_CRYPTO_EXT; + } + + if (fusectl1 & ICP_ACCEL_GEN4_MASK_EIA3_SLICE) + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_ZUC; + + capabilities_asym = ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | + ICP_ACCEL_CAPABILITIES_SM2 | + ICP_ACCEL_CAPABILITIES_ECEDMONT; + + if (fusectl1 & ICP_ACCEL_GEN4_MASK_PKE_SLICE) { + capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_SM2; + capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_ECEDMONT; + } + + capabilities_dc = ICP_ACCEL_CAPABILITIES_COMPRESSION | + ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION | + ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION | + ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64; + + if (fusectl1 & ICP_ACCEL_GEN4_MASK_COMPRESS_SLICE) { + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION; + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION; + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION; + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64; + } + + switch (adf_get_service_enabled(accel_dev)) { + case SVC_SYM_ASYM: + return capabilities_sym | capabilities_asym; + case SVC_DC: + return capabilities_dc; + case SVC_DCC: + /* + * Sym capabilities are available for chaining operations, + * but sym crypto instances cannot be supported + */ + capabilities_dcc = capabilities_dc | capabilities_sym; + capabilities_dcc &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + return capabilities_dcc; + case SVC_SYM: + return capabilities_sym; + case SVC_ASYM: + return capabilities_asym; + case SVC_ASYM_DC: + return capabilities_asym | capabilities_dc; + case SVC_SYM_DC: + return capabilities_sym | capabilities_dc; + default: + return 0; + } +} + +static const u32 *adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev) +{ + if (adf_gen4_init_thd2arb_map(accel_dev)) + dev_warn(&GET_DEV(accel_dev), + "Failed to generate thread to arbiter mapping"); + + return GET_HW_DATA(accel_dev)->thd_to_arb_map; +} + +static void adf_init_rl_data(struct adf_rl_hw_data *rl_data) +{ + rl_data->pciout_tb_offset = ADF_GEN4_RL_TOKEN_PCIEOUT_BUCKET_OFFSET; + rl_data->pciin_tb_offset = ADF_GEN4_RL_TOKEN_PCIEIN_BUCKET_OFFSET; + rl_data->r2l_offset = ADF_GEN4_RL_R2L_OFFSET; + rl_data->l2c_offset = ADF_GEN4_RL_L2C_OFFSET; + rl_data->c2s_offset = ADF_GEN4_RL_C2S_OFFSET; + + rl_data->pcie_scale_div = ADF_420XX_RL_PCIE_SCALE_FACTOR_DIV; + rl_data->pcie_scale_mul = ADF_420XX_RL_PCIE_SCALE_FACTOR_MUL; + rl_data->dcpr_correction = ADF_420XX_RL_DCPR_CORRECTION; + rl_data->max_tp[SVC_ASYM] = ADF_420XX_RL_MAX_TP_ASYM; + rl_data->max_tp[SVC_SYM] = ADF_420XX_RL_MAX_TP_SYM; + rl_data->max_tp[SVC_DC] = ADF_420XX_RL_MAX_TP_DC; + rl_data->scan_interval = ADF_420XX_RL_SCANS_PER_SEC; + rl_data->scale_ref = ADF_420XX_RL_SLICE_REF; + + adf_gen4_init_num_svc_aes(rl_data); +} + +static int get_rp_group(struct adf_accel_dev *accel_dev, u32 ae_mask) +{ + switch (ae_mask) { + case ADF_AE_GROUP_0: + return RP_GROUP_0; + case ADF_AE_GROUP_1: + case ADF_AE_GROUP_3: + return RP_GROUP_1; + case ADF_AE_GROUP_2: + if (get_fw_config(accel_dev) == adf_fw_cy_config) + return RP_GROUP_0; + else + return RP_GROUP_1; + default: + dev_dbg(&GET_DEV(accel_dev), "ae_mask not recognized"); + return -EINVAL; + } +} + +static u32 get_ena_thd_mask(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + const struct adf_fw_config *fw_config; + + if (obj_num >= uof_get_num_objs(accel_dev)) + return ADF_GEN4_ENA_THD_MASK_ERROR; + + fw_config = get_fw_config(accel_dev); + if (!fw_config) + return ADF_GEN4_ENA_THD_MASK_ERROR; + + switch (fw_config[obj_num].obj) { + case ADF_FW_ASYM_OBJ: + return ENA_THD_MASK_ASYM; + case ADF_FW_SYM_OBJ: + return ENA_THD_MASK_SYM; + case ADF_FW_DC_OBJ: + return ENA_THD_MASK_DC; + default: + return ADF_GEN4_ENA_THD_MASK_ERROR; + } +} + +static const char *uof_get_name(struct adf_accel_dev *accel_dev, u32 obj_num, + const char * const fw_objs[], int num_objs) +{ + const struct adf_fw_config *fw_config; + int id; + + fw_config = get_fw_config(accel_dev); + if (fw_config) + id = fw_config[obj_num].obj; + else + id = -EINVAL; + + if (id < 0 || id >= num_objs) + return NULL; + + return fw_objs[id]; +} + +static const char *uof_get_name_420xx(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + int num_fw_objs = ARRAY_SIZE(adf_420xx_fw_objs); + + return uof_get_name(accel_dev, obj_num, adf_420xx_fw_objs, num_fw_objs); +} + +static int uof_get_obj_type(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + const struct adf_fw_config *fw_config; + + if (obj_num >= uof_get_num_objs(accel_dev)) + return -EINVAL; + + fw_config = get_fw_config(accel_dev); + if (!fw_config) + return -EINVAL; + + return fw_config[obj_num].obj; +} + +static u32 uof_get_ae_mask(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + const struct adf_fw_config *fw_config; + + fw_config = get_fw_config(accel_dev); + if (!fw_config) + return 0; + + return fw_config[obj_num].ae_mask; +} + +static void adf_gen4_set_err_mask(struct adf_dev_err_mask *dev_err_mask) +{ + dev_err_mask->cppagentcmdpar_mask = ADF_420XX_HICPPAGENTCMDPARERRLOG_MASK; + dev_err_mask->parerr_ath_cph_mask = ADF_420XX_PARITYERRORMASK_ATH_CPH_MASK; + dev_err_mask->parerr_cpr_xlt_mask = ADF_420XX_PARITYERRORMASK_CPR_XLT_MASK; + dev_err_mask->parerr_dcpr_ucs_mask = ADF_420XX_PARITYERRORMASK_DCPR_UCS_MASK; + dev_err_mask->parerr_pke_mask = ADF_420XX_PARITYERRORMASK_PKE_MASK; + dev_err_mask->parerr_wat_wcp_mask = ADF_420XX_PARITYERRORMASK_WAT_WCP_MASK; + dev_err_mask->ssmfeatren_mask = ADF_420XX_SSMFEATREN_MASK; +} + +void adf_init_hw_data_420xx(struct adf_hw_device_data *hw_data, u32 dev_id) +{ + hw_data->dev_class = &adf_420xx_class; + hw_data->instance_id = adf_420xx_class.instances++; + hw_data->num_banks = ADF_GEN4_ETR_MAX_BANKS; + hw_data->num_banks_per_vf = ADF_GEN4_NUM_BANKS_PER_VF; + hw_data->num_rings_per_bank = ADF_GEN4_NUM_RINGS_PER_BANK; + hw_data->num_accel = ADF_GEN4_MAX_ACCELERATORS; + hw_data->num_engines = ADF_420XX_MAX_ACCELENGINES; + hw_data->num_logical_accel = 1; + hw_data->tx_rx_gap = ADF_GEN4_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_GEN4_TX_RINGS_MASK; + hw_data->ring_to_svc_map = ADF_GEN4_DEFAULT_RING_TO_SRV_MAP; + hw_data->alloc_irq = adf_isr_resource_alloc; + hw_data->free_irq = adf_isr_resource_free; + hw_data->enable_error_correction = adf_gen4_enable_error_correction; + hw_data->get_accel_mask = adf_gen4_get_accel_mask; + hw_data->get_ae_mask = get_ae_mask; + hw_data->get_num_accels = adf_gen4_get_num_accels; + hw_data->get_num_aes = adf_gen4_get_num_aes; + hw_data->get_sram_bar_id = adf_gen4_get_sram_bar_id; + hw_data->get_etr_bar_id = adf_gen4_get_etr_bar_id; + hw_data->get_misc_bar_id = adf_gen4_get_misc_bar_id; + hw_data->get_arb_info = adf_gen4_get_arb_info; + hw_data->get_admin_info = adf_gen4_get_admin_info; + hw_data->get_accel_cap = get_accel_cap; + hw_data->get_sku = adf_gen4_get_sku; + hw_data->init_admin_comms = adf_init_admin_comms; + hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->send_admin_init = adf_send_admin_init; + hw_data->init_arb = adf_init_arb; + hw_data->exit_arb = adf_exit_arb; + hw_data->get_arb_mapping = adf_get_arbiter_mapping; + hw_data->enable_ints = adf_gen4_enable_ints; + hw_data->init_device = adf_gen4_init_device; + hw_data->reset_device = adf_reset_flr; + hw_data->admin_ae_mask = ADF_420XX_ADMIN_AE_MASK; + hw_data->num_rps = ADF_GEN4_MAX_RPS; + hw_data->fw_name = ADF_420XX_FW; + hw_data->fw_mmp_name = ADF_420XX_MMP; + hw_data->uof_get_name = uof_get_name_420xx; + hw_data->uof_get_num_objs = uof_get_num_objs; + hw_data->uof_get_obj_type = uof_get_obj_type; + hw_data->uof_get_ae_mask = uof_get_ae_mask; + hw_data->get_rp_group = get_rp_group; + hw_data->get_ena_thd_mask = get_ena_thd_mask; + hw_data->set_msix_rttable = adf_gen4_set_msix_default_rttable; + hw_data->set_ssm_wdtimer = adf_gen4_set_ssm_wdtimer; + hw_data->get_ring_to_svc_map = adf_gen4_get_ring_to_svc_map; + hw_data->disable_iov = adf_disable_sriov; + hw_data->ring_pair_reset = adf_gen4_ring_pair_reset; + hw_data->bank_state_save = adf_bank_state_save; + hw_data->bank_state_restore = adf_bank_state_restore; + hw_data->enable_pm = adf_gen4_enable_pm; + hw_data->handle_pm_interrupt = adf_gen4_handle_pm_interrupt; + hw_data->dev_config = adf_gen4_dev_config; + hw_data->start_timer = adf_timer_start; + hw_data->stop_timer = adf_timer_stop; + hw_data->get_hb_clock = adf_gen4_get_heartbeat_clock; + hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE; + hw_data->clock_frequency = ADF_420XX_AE_FREQ; + hw_data->services_supported = adf_gen4_services_supported; + hw_data->get_svc_slice_cnt = adf_gen4_get_svc_slice_cnt; + hw_data->accel_capabilities_ext_mask = ADF_ACCEL_CAPABILITIES_EXT_ZSTD_LZ4S; + + adf_gen4_set_err_mask(&hw_data->dev_err_mask); + adf_gen4_init_hw_csr_ops(&hw_data->csr_ops); + adf_gen4_init_pf_pfvf_ops(&hw_data->pfvf_ops); + adf_gen4_init_dc_ops(&hw_data->dc_ops); + adf_gen4_init_ras_ops(&hw_data->ras_ops); + adf_gen4_init_tl_data(&hw_data->tl_data); + adf_gen4_init_vf_mig_ops(&hw_data->vfmig_ops); + adf_init_rl_data(&hw_data->rl_data); +} + +void adf_clean_hw_data_420xx(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class->instances--; +} diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.h b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.h new file mode 100644 index 000000000..99abbfc14 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ +#ifndef ADF_420XX_HW_DATA_H_ +#define ADF_420XX_HW_DATA_H_ + +#include + +#define ADF_420XX_MAX_ACCELENGINES 17 + +#define ADF_420XX_ACCELENGINES_MASK 0x1FFFF +#define ADF_420XX_ADMIN_AE_MASK 0x10000 + +#define ADF_420XX_HICPPAGENTCMDPARERRLOG_MASK (0xFF) +#define ADF_420XX_PARITYERRORMASK_ATH_CPH_MASK (0xFF00FF) +#define ADF_420XX_PARITYERRORMASK_CPR_XLT_MASK (0x10001) +#define ADF_420XX_PARITYERRORMASK_DCPR_UCS_MASK (0xF0007) +#define ADF_420XX_PARITYERRORMASK_PKE_MASK (0xFFF) +#define ADF_420XX_PARITYERRORMASK_WAT_WCP_MASK (0x3FF03FF) + +/* + * SSMFEATREN bit mask + * BIT(4) - enables parity detection on CPP + * BIT(12) - enables the logging of push/pull data errors + * in pperr register + * BIT(16) - BIT(27) - enable parity detection on SPPs + */ +#define ADF_420XX_SSMFEATREN_MASK \ + (BIT(4) | BIT(12) | BIT(16) | BIT(17) | BIT(18) | BIT(19) | BIT(20) | \ + BIT(21) | BIT(22) | BIT(23) | BIT(24) | BIT(25) | BIT(26) | BIT(27)) + +/* Firmware Binaries */ +#define ADF_420XX_FW "qat_420xx.bin" +#define ADF_420XX_MMP "qat_420xx_mmp.bin" +#define ADF_420XX_SYM_OBJ "qat_420xx_sym.bin" +#define ADF_420XX_DC_OBJ "qat_420xx_dc.bin" +#define ADF_420XX_ASYM_OBJ "qat_420xx_asym.bin" +#define ADF_420XX_ADMIN_OBJ "qat_420xx_admin.bin" + +/* RL constants */ +#define ADF_420XX_RL_PCIE_SCALE_FACTOR_DIV 100 +#define ADF_420XX_RL_PCIE_SCALE_FACTOR_MUL 102 +#define ADF_420XX_RL_DCPR_CORRECTION 1 +#define ADF_420XX_RL_SCANS_PER_SEC 954 +#define ADF_420XX_RL_MAX_TP_ASYM 173750UL +#define ADF_420XX_RL_MAX_TP_SYM 95000UL +#define ADF_420XX_RL_MAX_TP_DC 40000UL +#define ADF_420XX_RL_SLICE_REF 1000UL + +/* Clocks frequency */ +#define ADF_420XX_AE_FREQ (1000 * HZ_PER_MHZ) + +void adf_init_hw_data_420xx(struct adf_hw_device_data *hw_data, u32 dev_id); +void adf_clean_hw_data_420xx(struct adf_hw_device_data *hw_data); + +#endif diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_drv.c b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c new file mode 100644 index 000000000..0f0827e2b --- /dev/null +++ b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c @@ -0,0 +1,212 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "adf_420xx_hw_data.h" + +static const struct pci_device_id adf_pci_tbl[] = { + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_420XX) }, + { } +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) +{ + if (accel_dev->hw_device) { + adf_clean_hw_data_420xx(accel_dev->hw_device); + accel_dev->hw_device = NULL; + } + adf_dbgfs_exit(accel_dev); + adf_cfg_dev_remove(accel_dev); + adf_devmgr_rm_dev(accel_dev, NULL); +} + +static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct adf_accel_dev *accel_dev; + struct adf_accel_pci *accel_pci_dev; + struct adf_hw_device_data *hw_data; + unsigned int i, bar_nr; + unsigned long bar_mask; + struct adf_bar *bar; + int ret; + + if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) { + /* + * If the accelerator is connected to a node with no memory + * there is no point in using the accelerator since the remote + * memory transaction will be very slow. + */ + dev_err(&pdev->dev, "Invalid NUMA configuration.\n"); + return -EINVAL; + } + + accel_dev = devm_kzalloc(&pdev->dev, sizeof(*accel_dev), GFP_KERNEL); + if (!accel_dev) + return -ENOMEM; + + INIT_LIST_HEAD(&accel_dev->crypto_list); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = pdev; + + /* + * Add accel device to accel table + * This should be called before adf_cleanup_accel is called + */ + if (adf_devmgr_add_dev(accel_dev, NULL)) { + dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); + return -EFAULT; + } + + accel_dev->owner = THIS_MODULE; + /* Allocate and initialise device hardware meta-data structure */ + hw_data = devm_kzalloc(&pdev->dev, sizeof(*hw_data), GFP_KERNEL); + if (!hw_data) { + ret = -ENOMEM; + goto out_err; + } + + accel_dev->hw_device = hw_data; + adf_init_hw_data_420xx(accel_dev->hw_device, ent->device); + + pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); + pci_read_config_dword(pdev, ADF_GEN4_FUSECTL4_OFFSET, &hw_data->fuses[ADF_FUSECTL4]); + + /* Get Accelerators and Accelerators Engines masks */ + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); + accel_pci_dev->sku = hw_data->get_sku(hw_data); + /* If the device has no acceleration engines then ignore it */ + if (!hw_data->accel_mask || !hw_data->ae_mask || + (~hw_data->ae_mask & 0x01)) { + dev_err(&pdev->dev, "No acceleration units found.\n"); + ret = -EFAULT; + goto out_err; + } + + /* Create device configuration table */ + ret = adf_cfg_dev_add(accel_dev); + if (ret) + goto out_err; + + /* Enable PCI device */ + ret = pcim_enable_device(pdev); + if (ret) { + pci_err(pdev, "Can't enable PCI device.\n"); + goto out_err; + } + + /* Set DMA identifier */ + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (ret) { + dev_err(&pdev->dev, "No usable DMA configuration.\n"); + goto out_err; + } + + ret = adf_gen4_cfg_dev_init(accel_dev); + if (ret) { + dev_err(&pdev->dev, "Failed to initialize configuration.\n"); + goto out_err; + } + + /* Get accelerator capabilities mask */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); + if (!hw_data->accel_capabilities_mask) { + dev_err(&pdev->dev, "Failed to get capabilities mask.\n"); + ret = -EINVAL; + goto out_err; + } + + /* Find and map all the device's BARS */ + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM) & ADF_GEN4_BAR_MASK; + + ret = pcim_request_all_regions(pdev, pci_name(pdev)); + if (ret) { + pci_err(pdev, "Failed to request PCI regions.\n"); + goto out_err; + } + + i = 0; + for_each_set_bit(bar_nr, &bar_mask, PCI_STD_NUM_BARS) { + bar = &accel_pci_dev->pci_bars[i++]; + bar->virt_addr = pcim_iomap(pdev, bar_nr, 0); + if (!bar->virt_addr) { + pci_err(pdev, "Failed to ioremap PCI region.\n"); + ret = -ENOMEM; + goto out_err; + } + } + + if (pci_save_state(pdev)) { + pci_err(pdev, "Failed to save pci state.\n"); + ret = -ENOMEM; + goto out_err; + } + + accel_dev->ras_errors.enabled = true; + adf_dbgfs_init(accel_dev); + + ret = adf_dev_up(accel_dev, true); + if (ret) + goto out_err_dev_stop; + + ret = adf_sysfs_init(accel_dev); + if (ret) + goto out_err_dev_stop; + + return ret; + +out_err_dev_stop: + adf_dev_down(accel_dev); +out_err: + adf_cleanup_accel(accel_dev); + return ret; +} + +static void adf_remove(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + if (!accel_dev) { + pr_err("QAT: Driver removal failed\n"); + return; + } + adf_dev_down(accel_dev); + adf_cleanup_accel(accel_dev); +} + +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_420XX_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, + .shutdown = adf_shutdown, + .sriov_configure = adf_sriov_configure, + .err_handler = &adf_err_handler, +}; + +module_pci_driver(adf_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Intel"); +MODULE_FIRMWARE(ADF_420XX_FW); +MODULE_FIRMWARE(ADF_420XX_MMP); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); +MODULE_SOFTDEP("pre: crypto-intel_qat"); +MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/intel/qat/qat_4xxx/Makefile b/drivers/crypto/intel/qat/qat_4xxx/Makefile new file mode 100644 index 000000000..188b61144 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_4xxx/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +obj-$(CONFIG_CRYPTO_DEV_QAT_4XXX) += qat_4xxx.o +qat_4xxx-y := adf_drv.o adf_4xxx_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c new file mode 100644 index 000000000..49b425be3 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -0,0 +1,491 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2020 - 2021 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "adf_gen4_ras.h" +#include +#include +#include +#include "adf_4xxx_hw_data.h" +#include "icp_qat_hw.h" + +#define ADF_AE_GROUP_0 GENMASK(3, 0) +#define ADF_AE_GROUP_1 GENMASK(7, 4) +#define ADF_AE_GROUP_2 BIT(8) + +#define ENA_THD_MASK_ASYM GENMASK(1, 0) +#define ENA_THD_MASK_ASYM_401XX GENMASK(5, 0) +#define ENA_THD_MASK_SYM GENMASK(6, 0) +#define ENA_THD_MASK_DC GENMASK(1, 0) + +static const char * const adf_4xxx_fw_objs[] = { + [ADF_FW_SYM_OBJ] = ADF_4XXX_SYM_OBJ, + [ADF_FW_ASYM_OBJ] = ADF_4XXX_ASYM_OBJ, + [ADF_FW_DC_OBJ] = ADF_4XXX_DC_OBJ, + [ADF_FW_ADMIN_OBJ] = ADF_4XXX_ADMIN_OBJ, +}; + +static const char * const adf_402xx_fw_objs[] = { + [ADF_FW_SYM_OBJ] = ADF_402XX_SYM_OBJ, + [ADF_FW_ASYM_OBJ] = ADF_402XX_ASYM_OBJ, + [ADF_FW_DC_OBJ] = ADF_402XX_DC_OBJ, + [ADF_FW_ADMIN_OBJ] = ADF_402XX_ADMIN_OBJ, +}; + +static const struct adf_fw_config adf_fw_cy_config[] = { + {ADF_AE_GROUP_1, ADF_FW_SYM_OBJ}, + {ADF_AE_GROUP_0, ADF_FW_ASYM_OBJ}, + {ADF_AE_GROUP_2, ADF_FW_ADMIN_OBJ}, +}; + +static const struct adf_fw_config adf_fw_dc_config[] = { + {ADF_AE_GROUP_1, ADF_FW_DC_OBJ}, + {ADF_AE_GROUP_0, ADF_FW_DC_OBJ}, + {ADF_AE_GROUP_2, ADF_FW_ADMIN_OBJ}, +}; + +static const struct adf_fw_config adf_fw_sym_config[] = { + {ADF_AE_GROUP_1, ADF_FW_SYM_OBJ}, + {ADF_AE_GROUP_0, ADF_FW_SYM_OBJ}, + {ADF_AE_GROUP_2, ADF_FW_ADMIN_OBJ}, +}; + +static const struct adf_fw_config adf_fw_asym_config[] = { + {ADF_AE_GROUP_1, ADF_FW_ASYM_OBJ}, + {ADF_AE_GROUP_0, ADF_FW_ASYM_OBJ}, + {ADF_AE_GROUP_2, ADF_FW_ADMIN_OBJ}, +}; + +static const struct adf_fw_config adf_fw_asym_dc_config[] = { + {ADF_AE_GROUP_1, ADF_FW_ASYM_OBJ}, + {ADF_AE_GROUP_0, ADF_FW_DC_OBJ}, + {ADF_AE_GROUP_2, ADF_FW_ADMIN_OBJ}, +}; + +static const struct adf_fw_config adf_fw_sym_dc_config[] = { + {ADF_AE_GROUP_1, ADF_FW_SYM_OBJ}, + {ADF_AE_GROUP_0, ADF_FW_DC_OBJ}, + {ADF_AE_GROUP_2, ADF_FW_ADMIN_OBJ}, +}; + +static const struct adf_fw_config adf_fw_dcc_config[] = { + {ADF_AE_GROUP_1, ADF_FW_DC_OBJ}, + {ADF_AE_GROUP_0, ADF_FW_SYM_OBJ}, + {ADF_AE_GROUP_2, ADF_FW_ADMIN_OBJ}, +}; + +static_assert(ARRAY_SIZE(adf_fw_cy_config) == ARRAY_SIZE(adf_fw_dc_config)); +static_assert(ARRAY_SIZE(adf_fw_cy_config) == ARRAY_SIZE(adf_fw_sym_config)); +static_assert(ARRAY_SIZE(adf_fw_cy_config) == ARRAY_SIZE(adf_fw_asym_config)); +static_assert(ARRAY_SIZE(adf_fw_cy_config) == ARRAY_SIZE(adf_fw_asym_dc_config)); +static_assert(ARRAY_SIZE(adf_fw_cy_config) == ARRAY_SIZE(adf_fw_sym_dc_config)); +static_assert(ARRAY_SIZE(adf_fw_cy_config) == ARRAY_SIZE(adf_fw_dcc_config)); + +static struct adf_hw_device_class adf_4xxx_class = { + .name = ADF_4XXX_DEVICE_NAME, + .type = DEV_4XXX, +}; + +static u32 get_ae_mask(struct adf_hw_device_data *self) +{ + unsigned long fuses = self->fuses[ADF_FUSECTL4]; + u32 mask = ADF_4XXX_ACCELENGINES_MASK; + + if (test_bit(0, &fuses)) + mask &= ~ADF_AE_GROUP_0; + + if (test_bit(4, &fuses)) + mask &= ~ADF_AE_GROUP_1; + + if (test_bit(8, &fuses)) + mask &= ~ADF_AE_GROUP_2; + + return mask; +} + +static u32 get_accel_cap(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev; + u32 capabilities_sym, capabilities_asym, capabilities_dc; + u32 capabilities_dcc; + u32 fusectl1; + + /* Read accelerator capabilities mask */ + pci_read_config_dword(pdev, ADF_GEN4_FUSECTL1_OFFSET, &fusectl1); + + capabilities_sym = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | + ICP_ACCEL_CAPABILITIES_CIPHER | + ICP_ACCEL_CAPABILITIES_AUTHENTICATION | + ICP_ACCEL_CAPABILITIES_SHA3 | + ICP_ACCEL_CAPABILITIES_SHA3_EXT | + ICP_ACCEL_CAPABILITIES_HKDF | + ICP_ACCEL_CAPABILITIES_CHACHA_POLY | + ICP_ACCEL_CAPABILITIES_AESGCM_SPC | + ICP_ACCEL_CAPABILITIES_SM3 | + ICP_ACCEL_CAPABILITIES_SM4 | + ICP_ACCEL_CAPABILITIES_AES_V2; + + /* A set bit in fusectl1 means the feature is OFF in this SKU */ + if (fusectl1 & ICP_ACCEL_GEN4_MASK_CIPHER_SLICE) { + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_HKDF; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + } + + if (fusectl1 & ICP_ACCEL_GEN4_MASK_UCS_SLICE) { + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CHACHA_POLY; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AESGCM_SPC; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AES_V2; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + } + + if (fusectl1 & ICP_ACCEL_GEN4_MASK_AUTH_SLICE) { + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SHA3; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SHA3_EXT; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + } + + if (fusectl1 & ICP_ACCEL_GEN4_MASK_SMX_SLICE) { + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SM3; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SM4; + } + + capabilities_asym = ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | + ICP_ACCEL_CAPABILITIES_CIPHER | + ICP_ACCEL_CAPABILITIES_SM2 | + ICP_ACCEL_CAPABILITIES_ECEDMONT; + + if (fusectl1 & ICP_ACCEL_GEN4_MASK_PKE_SLICE) { + capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_SM2; + capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_ECEDMONT; + } + + capabilities_dc = ICP_ACCEL_CAPABILITIES_COMPRESSION | + ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION | + ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION | + ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64; + + if (fusectl1 & ICP_ACCEL_GEN4_MASK_COMPRESS_SLICE) { + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION; + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION; + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION; + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64; + } + + switch (adf_get_service_enabled(accel_dev)) { + case SVC_SYM_ASYM: + return capabilities_sym | capabilities_asym; + case SVC_DC: + return capabilities_dc; + case SVC_DCC: + /* + * Sym capabilities are available for chaining operations, + * but sym crypto instances cannot be supported + */ + capabilities_dcc = capabilities_dc | capabilities_sym; + capabilities_dcc &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + return capabilities_dcc; + case SVC_SYM: + return capabilities_sym; + case SVC_ASYM: + return capabilities_asym; + case SVC_ASYM_DC: + return capabilities_asym | capabilities_dc; + case SVC_SYM_DC: + return capabilities_sym | capabilities_dc; + default: + return 0; + } +} + +static const u32 *adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev) +{ + if (adf_gen4_init_thd2arb_map(accel_dev)) + dev_warn(&GET_DEV(accel_dev), + "Failed to generate thread to arbiter mapping"); + + return GET_HW_DATA(accel_dev)->thd_to_arb_map; +} + +static void adf_init_rl_data(struct adf_rl_hw_data *rl_data) +{ + rl_data->pciout_tb_offset = ADF_GEN4_RL_TOKEN_PCIEOUT_BUCKET_OFFSET; + rl_data->pciin_tb_offset = ADF_GEN4_RL_TOKEN_PCIEIN_BUCKET_OFFSET; + rl_data->r2l_offset = ADF_GEN4_RL_R2L_OFFSET; + rl_data->l2c_offset = ADF_GEN4_RL_L2C_OFFSET; + rl_data->c2s_offset = ADF_GEN4_RL_C2S_OFFSET; + + rl_data->pcie_scale_div = ADF_4XXX_RL_PCIE_SCALE_FACTOR_DIV; + rl_data->pcie_scale_mul = ADF_4XXX_RL_PCIE_SCALE_FACTOR_MUL; + rl_data->dcpr_correction = ADF_4XXX_RL_DCPR_CORRECTION; + rl_data->max_tp[SVC_ASYM] = ADF_4XXX_RL_MAX_TP_ASYM; + rl_data->max_tp[SVC_SYM] = ADF_4XXX_RL_MAX_TP_SYM; + rl_data->max_tp[SVC_DC] = ADF_4XXX_RL_MAX_TP_DC; + rl_data->scan_interval = ADF_4XXX_RL_SCANS_PER_SEC; + rl_data->scale_ref = ADF_4XXX_RL_SLICE_REF; + + adf_gen4_init_num_svc_aes(rl_data); +} + +static u32 uof_get_num_objs(struct adf_accel_dev *accel_dev) +{ + return ARRAY_SIZE(adf_fw_cy_config); +} + +static const struct adf_fw_config *get_fw_config(struct adf_accel_dev *accel_dev) +{ + switch (adf_get_service_enabled(accel_dev)) { + case SVC_SYM_ASYM: + return adf_fw_cy_config; + case SVC_DC: + return adf_fw_dc_config; + case SVC_DCC: + return adf_fw_dcc_config; + case SVC_SYM: + return adf_fw_sym_config; + case SVC_ASYM: + return adf_fw_asym_config; + case SVC_ASYM_DC: + return adf_fw_asym_dc_config; + case SVC_SYM_DC: + return adf_fw_sym_dc_config; + default: + return NULL; + } +} + +static int get_rp_group(struct adf_accel_dev *accel_dev, u32 ae_mask) +{ + switch (ae_mask) { + case ADF_AE_GROUP_0: + return RP_GROUP_0; + case ADF_AE_GROUP_1: + return RP_GROUP_1; + default: + dev_dbg(&GET_DEV(accel_dev), "ae_mask not recognized"); + return -EINVAL; + } +} + +static u32 get_ena_thd_mask(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + const struct adf_fw_config *fw_config; + + if (obj_num >= uof_get_num_objs(accel_dev)) + return ADF_GEN4_ENA_THD_MASK_ERROR; + + fw_config = get_fw_config(accel_dev); + if (!fw_config) + return ADF_GEN4_ENA_THD_MASK_ERROR; + + switch (fw_config[obj_num].obj) { + case ADF_FW_ASYM_OBJ: + return ENA_THD_MASK_ASYM; + case ADF_FW_SYM_OBJ: + return ENA_THD_MASK_SYM; + case ADF_FW_DC_OBJ: + return ENA_THD_MASK_DC; + default: + return ADF_GEN4_ENA_THD_MASK_ERROR; + } +} + +static u32 get_ena_thd_mask_401xx(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + const struct adf_fw_config *fw_config; + + if (obj_num >= uof_get_num_objs(accel_dev)) + return ADF_GEN4_ENA_THD_MASK_ERROR; + + fw_config = get_fw_config(accel_dev); + if (!fw_config) + return ADF_GEN4_ENA_THD_MASK_ERROR; + + switch (fw_config[obj_num].obj) { + case ADF_FW_ASYM_OBJ: + return ENA_THD_MASK_ASYM_401XX; + case ADF_FW_SYM_OBJ: + return ENA_THD_MASK_SYM; + case ADF_FW_DC_OBJ: + return ENA_THD_MASK_DC; + default: + return ADF_GEN4_ENA_THD_MASK_ERROR; + } +} + +static const char *uof_get_name(struct adf_accel_dev *accel_dev, u32 obj_num, + const char * const fw_objs[], int num_objs) +{ + const struct adf_fw_config *fw_config; + int id; + + fw_config = get_fw_config(accel_dev); + if (fw_config) + id = fw_config[obj_num].obj; + else + id = -EINVAL; + + if (id < 0 || id >= num_objs) + return NULL; + + return fw_objs[id]; +} + +static const char *uof_get_name_4xxx(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + int num_fw_objs = ARRAY_SIZE(adf_4xxx_fw_objs); + + return uof_get_name(accel_dev, obj_num, adf_4xxx_fw_objs, num_fw_objs); +} + +static const char *uof_get_name_402xx(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + int num_fw_objs = ARRAY_SIZE(adf_402xx_fw_objs); + + return uof_get_name(accel_dev, obj_num, adf_402xx_fw_objs, num_fw_objs); +} + +static int uof_get_obj_type(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + const struct adf_fw_config *fw_config; + + if (obj_num >= uof_get_num_objs(accel_dev)) + return -EINVAL; + + fw_config = get_fw_config(accel_dev); + if (!fw_config) + return -EINVAL; + + return fw_config[obj_num].obj; +} + +static u32 uof_get_ae_mask(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + const struct adf_fw_config *fw_config; + + fw_config = get_fw_config(accel_dev); + if (!fw_config) + return 0; + + return fw_config[obj_num].ae_mask; +} + +static void adf_gen4_set_err_mask(struct adf_dev_err_mask *dev_err_mask) +{ + dev_err_mask->cppagentcmdpar_mask = ADF_4XXX_HICPPAGENTCMDPARERRLOG_MASK; + dev_err_mask->parerr_ath_cph_mask = ADF_4XXX_PARITYERRORMASK_ATH_CPH_MASK; + dev_err_mask->parerr_cpr_xlt_mask = ADF_4XXX_PARITYERRORMASK_CPR_XLT_MASK; + dev_err_mask->parerr_dcpr_ucs_mask = ADF_4XXX_PARITYERRORMASK_DCPR_UCS_MASK; + dev_err_mask->parerr_pke_mask = ADF_4XXX_PARITYERRORMASK_PKE_MASK; + dev_err_mask->ssmfeatren_mask = ADF_4XXX_SSMFEATREN_MASK; +} + +void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 dev_id) +{ + hw_data->dev_class = &adf_4xxx_class; + hw_data->instance_id = adf_4xxx_class.instances++; + hw_data->num_banks = ADF_GEN4_ETR_MAX_BANKS; + hw_data->num_banks_per_vf = ADF_GEN4_NUM_BANKS_PER_VF; + hw_data->num_rings_per_bank = ADF_GEN4_NUM_RINGS_PER_BANK; + hw_data->num_accel = ADF_GEN4_MAX_ACCELERATORS; + hw_data->num_engines = ADF_4XXX_MAX_ACCELENGINES; + hw_data->num_logical_accel = 1; + hw_data->tx_rx_gap = ADF_GEN4_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_GEN4_TX_RINGS_MASK; + hw_data->ring_to_svc_map = ADF_GEN4_DEFAULT_RING_TO_SRV_MAP; + hw_data->alloc_irq = adf_isr_resource_alloc; + hw_data->free_irq = adf_isr_resource_free; + hw_data->enable_error_correction = adf_gen4_enable_error_correction; + hw_data->get_accel_mask = adf_gen4_get_accel_mask; + hw_data->get_ae_mask = get_ae_mask; + hw_data->get_num_accels = adf_gen4_get_num_accels; + hw_data->get_num_aes = adf_gen4_get_num_aes; + hw_data->get_sram_bar_id = adf_gen4_get_sram_bar_id; + hw_data->get_etr_bar_id = adf_gen4_get_etr_bar_id; + hw_data->get_misc_bar_id = adf_gen4_get_misc_bar_id; + hw_data->get_arb_info = adf_gen4_get_arb_info; + hw_data->get_admin_info = adf_gen4_get_admin_info; + hw_data->get_accel_cap = get_accel_cap; + hw_data->get_sku = adf_gen4_get_sku; + hw_data->init_admin_comms = adf_init_admin_comms; + hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->send_admin_init = adf_send_admin_init; + hw_data->init_arb = adf_init_arb; + hw_data->exit_arb = adf_exit_arb; + hw_data->get_arb_mapping = adf_get_arbiter_mapping; + hw_data->enable_ints = adf_gen4_enable_ints; + hw_data->init_device = adf_gen4_init_device; + hw_data->reset_device = adf_reset_flr; + hw_data->admin_ae_mask = ADF_4XXX_ADMIN_AE_MASK; + hw_data->num_rps = ADF_GEN4_MAX_RPS; + switch (dev_id) { + case PCI_DEVICE_ID_INTEL_QAT_402XX: + hw_data->fw_name = ADF_402XX_FW; + hw_data->fw_mmp_name = ADF_402XX_MMP; + hw_data->uof_get_name = uof_get_name_402xx; + hw_data->get_ena_thd_mask = get_ena_thd_mask; + break; + case PCI_DEVICE_ID_INTEL_QAT_401XX: + hw_data->fw_name = ADF_4XXX_FW; + hw_data->fw_mmp_name = ADF_4XXX_MMP; + hw_data->uof_get_name = uof_get_name_4xxx; + hw_data->get_ena_thd_mask = get_ena_thd_mask_401xx; + break; + default: + hw_data->fw_name = ADF_4XXX_FW; + hw_data->fw_mmp_name = ADF_4XXX_MMP; + hw_data->uof_get_name = uof_get_name_4xxx; + hw_data->get_ena_thd_mask = get_ena_thd_mask; + break; + } + hw_data->uof_get_num_objs = uof_get_num_objs; + hw_data->uof_get_obj_type = uof_get_obj_type; + hw_data->uof_get_ae_mask = uof_get_ae_mask; + hw_data->get_rp_group = get_rp_group; + hw_data->set_msix_rttable = adf_gen4_set_msix_default_rttable; + hw_data->set_ssm_wdtimer = adf_gen4_set_ssm_wdtimer; + hw_data->get_ring_to_svc_map = adf_gen4_get_ring_to_svc_map; + hw_data->disable_iov = adf_disable_sriov; + hw_data->ring_pair_reset = adf_gen4_ring_pair_reset; + hw_data->bank_state_save = adf_bank_state_save; + hw_data->bank_state_restore = adf_bank_state_restore; + hw_data->enable_pm = adf_gen4_enable_pm; + hw_data->handle_pm_interrupt = adf_gen4_handle_pm_interrupt; + hw_data->dev_config = adf_gen4_dev_config; + hw_data->start_timer = adf_timer_start; + hw_data->stop_timer = adf_timer_stop; + hw_data->get_hb_clock = adf_gen4_get_heartbeat_clock; + hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE; + hw_data->clock_frequency = ADF_4XXX_AE_FREQ; + hw_data->services_supported = adf_gen4_services_supported; + hw_data->get_svc_slice_cnt = adf_gen4_get_svc_slice_cnt; + hw_data->accel_capabilities_ext_mask = ADF_ACCEL_CAPABILITIES_EXT_ZSTD_LZ4S; + + adf_gen4_set_err_mask(&hw_data->dev_err_mask); + adf_gen4_init_hw_csr_ops(&hw_data->csr_ops); + adf_gen4_init_pf_pfvf_ops(&hw_data->pfvf_ops); + adf_gen4_init_dc_ops(&hw_data->dc_ops); + adf_gen4_init_ras_ops(&hw_data->ras_ops); + adf_gen4_init_tl_data(&hw_data->tl_data); + adf_gen4_init_vf_mig_ops(&hw_data->vfmig_ops); + adf_init_rl_data(&hw_data->rl_data); +} + +void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class->instances--; +} diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.h b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.h new file mode 100644 index 000000000..76388363e --- /dev/null +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#ifndef ADF_4XXX_HW_DATA_H_ +#define ADF_4XXX_HW_DATA_H_ + +#include +#include + +#define ADF_4XXX_MAX_ACCELENGINES 9 + +#define ADF_4XXX_ACCELENGINES_MASK (0x1FF) +#define ADF_4XXX_ADMIN_AE_MASK (0x100) + +#define ADF_4XXX_HICPPAGENTCMDPARERRLOG_MASK 0x1F +#define ADF_4XXX_PARITYERRORMASK_ATH_CPH_MASK 0xF000F +#define ADF_4XXX_PARITYERRORMASK_CPR_XLT_MASK 0x10001 +#define ADF_4XXX_PARITYERRORMASK_DCPR_UCS_MASK 0x30007 +#define ADF_4XXX_PARITYERRORMASK_PKE_MASK 0x3F + +/* + * SSMFEATREN bit mask + * BIT(4) - enables parity detection on CPP + * BIT(12) - enables the logging of push/pull data errors + * in pperr register + * BIT(16) - BIT(23) - enable parity detection on SPPs + */ +#define ADF_4XXX_SSMFEATREN_MASK \ + (BIT(4) | BIT(12) | BIT(16) | BIT(17) | BIT(18) | \ + BIT(19) | BIT(20) | BIT(21) | BIT(22) | BIT(23)) + +/* Firmware Binaries */ +#define ADF_4XXX_FW "qat_4xxx.bin" +#define ADF_4XXX_MMP "qat_4xxx_mmp.bin" +#define ADF_4XXX_SYM_OBJ "qat_4xxx_sym.bin" +#define ADF_4XXX_DC_OBJ "qat_4xxx_dc.bin" +#define ADF_4XXX_ASYM_OBJ "qat_4xxx_asym.bin" +#define ADF_4XXX_ADMIN_OBJ "qat_4xxx_admin.bin" +/* Firmware for 402XXX */ +#define ADF_402XX_FW "qat_402xx.bin" +#define ADF_402XX_MMP "qat_402xx_mmp.bin" +#define ADF_402XX_SYM_OBJ "qat_402xx_sym.bin" +#define ADF_402XX_DC_OBJ "qat_402xx_dc.bin" +#define ADF_402XX_ASYM_OBJ "qat_402xx_asym.bin" +#define ADF_402XX_ADMIN_OBJ "qat_402xx_admin.bin" + +/* RL constants */ +#define ADF_4XXX_RL_PCIE_SCALE_FACTOR_DIV 100 +#define ADF_4XXX_RL_PCIE_SCALE_FACTOR_MUL 102 +#define ADF_4XXX_RL_DCPR_CORRECTION 1 +#define ADF_4XXX_RL_SCANS_PER_SEC 954 +#define ADF_4XXX_RL_MAX_TP_ASYM 173750UL +#define ADF_4XXX_RL_MAX_TP_SYM 95000UL +#define ADF_4XXX_RL_MAX_TP_DC 45000UL +#define ADF_4XXX_RL_SLICE_REF 1000UL + +/* Clocks frequency */ +#define ADF_4XXX_AE_FREQ (1000 * HZ_PER_MHZ) + +void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 dev_id); +void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data); + +#endif diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c new file mode 100644 index 000000000..aa95f762c --- /dev/null +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2020 Intel Corporation */ +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "adf_4xxx_hw_data.h" + +static const struct pci_device_id adf_pci_tbl[] = { + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_4XXX) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_401XX) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_402XX) }, + { } +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) +{ + if (accel_dev->hw_device) { + adf_clean_hw_data_4xxx(accel_dev->hw_device); + accel_dev->hw_device = NULL; + } + adf_dbgfs_exit(accel_dev); + adf_cfg_dev_remove(accel_dev); + adf_devmgr_rm_dev(accel_dev, NULL); +} + +static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct adf_accel_dev *accel_dev; + struct adf_accel_pci *accel_pci_dev; + struct adf_hw_device_data *hw_data; + unsigned int i, bar_nr; + unsigned long bar_mask; + struct adf_bar *bar; + int ret; + + if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) { + /* + * If the accelerator is connected to a node with no memory + * there is no point in using the accelerator since the remote + * memory transaction will be very slow. + */ + dev_err(&pdev->dev, "Invalid NUMA configuration.\n"); + return -EINVAL; + } + + accel_dev = devm_kzalloc(&pdev->dev, sizeof(*accel_dev), GFP_KERNEL); + if (!accel_dev) + return -ENOMEM; + + INIT_LIST_HEAD(&accel_dev->crypto_list); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = pdev; + + /* + * Add accel device to accel table + * This should be called before adf_cleanup_accel is called + */ + if (adf_devmgr_add_dev(accel_dev, NULL)) { + dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); + return -EFAULT; + } + + accel_dev->owner = THIS_MODULE; + /* Allocate and initialise device hardware meta-data structure */ + hw_data = devm_kzalloc(&pdev->dev, sizeof(*hw_data), GFP_KERNEL); + if (!hw_data) { + ret = -ENOMEM; + goto out_err; + } + + accel_dev->hw_device = hw_data; + adf_init_hw_data_4xxx(accel_dev->hw_device, ent->device); + + pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); + pci_read_config_dword(pdev, ADF_GEN4_FUSECTL4_OFFSET, &hw_data->fuses[ADF_FUSECTL4]); + + /* Get Accelerators and Accelerators Engines masks */ + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); + accel_pci_dev->sku = hw_data->get_sku(hw_data); + /* If the device has no acceleration engines then ignore it */ + if (!hw_data->accel_mask || !hw_data->ae_mask || + (~hw_data->ae_mask & 0x01)) { + dev_err(&pdev->dev, "No acceleration units found.\n"); + ret = -EFAULT; + goto out_err; + } + + /* Create device configuration table */ + ret = adf_cfg_dev_add(accel_dev); + if (ret) + goto out_err; + + /* Enable PCI device */ + ret = pcim_enable_device(pdev); + if (ret) { + pci_err(pdev, "Can't enable PCI device.\n"); + goto out_err; + } + + /* Set DMA identifier */ + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (ret) { + dev_err(&pdev->dev, "No usable DMA configuration.\n"); + goto out_err; + } + + ret = adf_gen4_cfg_dev_init(accel_dev); + if (ret) { + dev_err(&pdev->dev, "Failed to initialize configuration.\n"); + goto out_err; + } + + /* Get accelerator capabilities mask */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); + if (!hw_data->accel_capabilities_mask) { + dev_err(&pdev->dev, "Failed to get capabilities mask.\n"); + ret = -EINVAL; + goto out_err; + } + + /* Find and map all the device's BARS */ + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM) & ADF_GEN4_BAR_MASK; + + ret = pcim_request_all_regions(pdev, pci_name(pdev)); + if (ret) { + pci_err(pdev, "Failed to request PCI regions.\n"); + goto out_err; + } + + i = 0; + for_each_set_bit(bar_nr, &bar_mask, PCI_STD_NUM_BARS) { + bar = &accel_pci_dev->pci_bars[i++]; + bar->virt_addr = pcim_iomap(pdev, bar_nr, 0); + if (!bar->virt_addr) { + pci_err(pdev, "Failed to ioremap PCI region.\n"); + ret = -ENOMEM; + goto out_err; + } + } + + if (pci_save_state(pdev)) { + pci_err(pdev, "Failed to save pci state.\n"); + ret = -ENOMEM; + goto out_err; + } + + accel_dev->ras_errors.enabled = true; + adf_dbgfs_init(accel_dev); + + ret = adf_dev_up(accel_dev, true); + if (ret) + goto out_err_dev_stop; + + ret = adf_sysfs_init(accel_dev); + if (ret) + goto out_err_dev_stop; + + return ret; + +out_err_dev_stop: + adf_dev_down(accel_dev); +out_err: + adf_cleanup_accel(accel_dev); + return ret; +} + +static void adf_remove(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + if (!accel_dev) { + pr_err("QAT: Driver removal failed\n"); + return; + } + adf_dev_down(accel_dev); + adf_cleanup_accel(accel_dev); +} + +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_4XXX_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, + .shutdown = adf_shutdown, + .sriov_configure = adf_sriov_configure, + .err_handler = &adf_err_handler, +}; + +module_pci_driver(adf_driver); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Intel"); +MODULE_FIRMWARE(ADF_4XXX_FW); +MODULE_FIRMWARE(ADF_402XX_FW); +MODULE_FIRMWARE(ADF_4XXX_MMP); +MODULE_FIRMWARE(ADF_402XX_MMP); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); +MODULE_SOFTDEP("pre: crypto-intel_qat"); +MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/intel/qat/qat_6xxx/Makefile b/drivers/crypto/intel/qat/qat_6xxx/Makefile new file mode 100644 index 000000000..4b4de67cb --- /dev/null +++ b/drivers/crypto/intel/qat/qat_6xxx/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_QAT_6XXX) += qat_6xxx.o +qat_6xxx-y := adf_drv.o adf_6xxx_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c b/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c new file mode 100644 index 000000000..388087e64 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c @@ -0,0 +1,1085 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2025 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "adf_6xxx_hw_data.h" +#include "icp_qat_fw_comp.h" +#include "icp_qat_hw_51_comp.h" + +#define RP_GROUP_0_MASK (BIT(0) | BIT(2)) +#define RP_GROUP_1_MASK (BIT(1) | BIT(3)) +#define RP_GROUP_ALL_MASK (RP_GROUP_0_MASK | RP_GROUP_1_MASK) + +#define ADF_AE_GROUP_0 GENMASK(3, 0) +#define ADF_AE_GROUP_1 GENMASK(7, 4) +#define ADF_AE_GROUP_2 BIT(8) + +#define ASB_MULTIPLIER 9 + +struct adf_ring_config { + u32 ring_mask; + enum adf_cfg_service_type ring_type; + const unsigned long *thrd_mask; +}; + +static u32 rmask_two_services[] = { + RP_GROUP_0_MASK, + RP_GROUP_1_MASK, +}; + +enum adf_gen6_rps { + RP0 = 0, + RP1 = 1, + RP2 = 2, + RP3 = 3, + RP_MAX = RP3 +}; + +/* + * thrd_mask_[sym|asym|cpr|dcc]: these static arrays define the thread + * configuration for handling requests of specific services across the + * accelerator engines. Each element in an array corresponds to an + * accelerator engine, with the value being a bitmask that specifies which + * threads within that engine are capable of processing the particular service. + * + * For example, a value of 0x0C means that threads 2 and 3 are enabled for the + * service in the respective accelerator engine. + */ +static const unsigned long thrd_mask_sym[ADF_6XXX_MAX_ACCELENGINES] = { + 0x0C, 0x0C, 0x0C, 0x0C, 0x1C, 0x1C, 0x1C, 0x1C, 0x00 +}; + +static const unsigned long thrd_mask_asym[ADF_6XXX_MAX_ACCELENGINES] = { + 0x70, 0x70, 0x70, 0x70, 0x60, 0x60, 0x60, 0x60, 0x00 +}; + +static const unsigned long thrd_mask_cpr[ADF_6XXX_MAX_ACCELENGINES] = { + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00 +}; + +static const unsigned long thrd_mask_dcc[ADF_6XXX_MAX_ACCELENGINES] = { + 0x00, 0x00, 0x00, 0x00, 0x07, 0x07, 0x03, 0x03, 0x00 +}; + +static const unsigned long thrd_mask_dcpr[ADF_6XXX_MAX_ACCELENGINES] = { + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x00 +}; + +static const unsigned long thrd_mask_wcy[ADF_6XXX_MAX_ACCELENGINES] = { + 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x00 +}; + +static const char *const adf_6xxx_fw_objs[] = { + [ADF_FW_CY_OBJ] = ADF_6XXX_CY_OBJ, + [ADF_FW_DC_OBJ] = ADF_6XXX_DC_OBJ, + [ADF_FW_ADMIN_OBJ] = ADF_6XXX_ADMIN_OBJ, + [ADF_FW_WCY_OBJ] = ADF_6XXX_WCY_OBJ, +}; + +static const struct adf_fw_config adf_default_fw_config[] = { + { ADF_AE_GROUP_1, ADF_FW_DC_OBJ }, + { ADF_AE_GROUP_0, ADF_FW_CY_OBJ }, + { ADF_AE_GROUP_2, ADF_FW_ADMIN_OBJ }, +}; + +static const struct adf_fw_config adf_wcy_fw_config[] = { + { ADF_AE_GROUP_1, ADF_FW_WCY_OBJ }, + { ADF_AE_GROUP_0, ADF_FW_WCY_OBJ }, + { ADF_AE_GROUP_2, ADF_FW_ADMIN_OBJ }, +}; + +static struct adf_hw_device_class adf_6xxx_class = { + .name = ADF_6XXX_DEVICE_NAME, + .type = DEV_6XXX, +}; + +static bool services_supported(unsigned long mask) +{ + int num_svc; + + if (mask >= BIT(SVC_COUNT)) + return false; + + num_svc = hweight_long(mask); + switch (num_svc) { + case ADF_ONE_SERVICE: + return true; + case ADF_TWO_SERVICES: + case ADF_THREE_SERVICES: + return !test_bit(SVC_DCC, &mask); + default: + return false; + } +} + +static bool wcy_services_supported(unsigned long mask) +{ + /* The wireless SKU supports only the symmetric crypto service */ + return mask == BIT(SVC_SYM); +} + +static int get_service(unsigned long *mask) +{ + if (test_and_clear_bit(SVC_ASYM, mask)) + return SVC_ASYM; + + if (test_and_clear_bit(SVC_SYM, mask)) + return SVC_SYM; + + if (test_and_clear_bit(SVC_DC, mask)) + return SVC_DC; + + if (test_and_clear_bit(SVC_DCC, mask)) + return SVC_DCC; + + if (test_and_clear_bit(SVC_DECOMP, mask)) + return SVC_DECOMP; + + return -EINVAL; +} + +static enum adf_cfg_service_type get_ring_type(unsigned int service) +{ + switch (service) { + case SVC_SYM: + return SYM; + case SVC_ASYM: + return ASYM; + case SVC_DC: + case SVC_DCC: + return COMP; + case SVC_DECOMP: + return DECOMP; + default: + return UNUSED; + } +} + +static const unsigned long *get_thrd_mask(struct adf_accel_dev *accel_dev, + unsigned int service) +{ + if (adf_6xxx_is_wcy(GET_HW_DATA(accel_dev))) + return (service == SVC_SYM) ? thrd_mask_wcy : NULL; + + switch (service) { + case SVC_SYM: + return thrd_mask_sym; + case SVC_ASYM: + return thrd_mask_asym; + case SVC_DC: + return thrd_mask_cpr; + case SVC_DCC: + return thrd_mask_dcc; + case SVC_DECOMP: + return thrd_mask_dcpr; + default: + return NULL; + } +} + +static int get_rp_config(struct adf_accel_dev *accel_dev, struct adf_ring_config *rp_config, + unsigned int *num_services) +{ + unsigned int i, nservices; + unsigned long mask; + int ret, service; + + ret = adf_get_service_mask(accel_dev, &mask); + if (ret) + return ret; + + nservices = hweight_long(mask); + if (nservices > MAX_NUM_CONCURR_SVC) + return -EINVAL; + + for (i = 0; i < nservices; i++) { + service = get_service(&mask); + if (service < 0) + return service; + + rp_config[i].ring_type = get_ring_type(service); + rp_config[i].thrd_mask = get_thrd_mask(accel_dev, service); + + /* + * If there is only one service enabled, use all ring pairs for + * that service. + * If there are two services enabled, use ring pairs 0 and 2 for + * one service and ring pairs 1 and 3 for the other service. + */ + switch (nservices) { + case ADF_ONE_SERVICE: + rp_config[i].ring_mask = RP_GROUP_ALL_MASK; + break; + case ADF_TWO_SERVICES: + rp_config[i].ring_mask = rmask_two_services[i]; + break; + case ADF_THREE_SERVICES: + rp_config[i].ring_mask = BIT(i); + + /* If ASYM is enabled, use additional ring pair */ + if (service == SVC_ASYM) + rp_config[i].ring_mask |= BIT(RP3); + + break; + default: + return -EINVAL; + } + } + + *num_services = nservices; + + return 0; +} + +static u32 adf_gen6_get_arb_mask(struct adf_accel_dev *accel_dev, unsigned int ae) +{ + struct adf_ring_config rp_config[MAX_NUM_CONCURR_SVC]; + unsigned int num_services, i, thrd; + u32 ring_mask, thd2arb_mask = 0; + const unsigned long *p_mask; + + if (get_rp_config(accel_dev, rp_config, &num_services)) + return 0; + + /* + * The thd2arb_mask maps ring pairs to threads within an accelerator engine. + * It ensures that jobs submitted to ring pairs are scheduled on threads capable + * of handling the specified service type. + * + * Each group of 4 bits in the mask corresponds to a thread, with each bit + * indicating whether a job from a ring pair can be scheduled on that thread. + * The use of 4 bits is due to the organization of ring pairs into groups of + * four, where each group shares the same configuration. + */ + for (i = 0; i < num_services; i++) { + p_mask = &rp_config[i].thrd_mask[ae]; + ring_mask = rp_config[i].ring_mask; + + for_each_set_bit(thrd, p_mask, ADF_NUM_THREADS_PER_AE) + thd2arb_mask |= ring_mask << (thrd * 4); + } + + return thd2arb_mask; +} + +static u16 get_ring_to_svc_map(struct adf_accel_dev *accel_dev) +{ + enum adf_cfg_service_type rps[ADF_GEN6_NUM_BANKS_PER_VF] = { }; + struct adf_ring_config rp_config[MAX_NUM_CONCURR_SVC]; + unsigned int num_services, rp_num, i; + unsigned long cfg_mask; + u16 ring_to_svc_map; + + if (get_rp_config(accel_dev, rp_config, &num_services)) + return 0; + + /* + * Loop through the configured services and populate the `rps` array that + * contains what service that particular ring pair can handle (i.e. symmetric + * crypto, asymmetric crypto, data compression or compression chaining). + */ + for (i = 0; i < num_services; i++) { + cfg_mask = rp_config[i].ring_mask; + for_each_set_bit(rp_num, &cfg_mask, ADF_GEN6_NUM_BANKS_PER_VF) + rps[rp_num] = rp_config[i].ring_type; + } + + /* + * The ring_mask is structured into segments of 3 bits, with each + * segment representing the service configuration for a specific ring pair. + * Since ring pairs are organized into groups of 4, the ring_mask contains 4 + * such 3-bit segments, each corresponding to one ring pair. + * + * The device has 64 ring pairs, which are organized in groups of 4, namely + * 16 groups. Each group has the same configuration, represented here by + * `ring_to_svc_map`. + */ + ring_to_svc_map = rps[RP0] << ADF_CFG_SERV_RING_PAIR_0_SHIFT | + rps[RP1] << ADF_CFG_SERV_RING_PAIR_1_SHIFT | + rps[RP2] << ADF_CFG_SERV_RING_PAIR_2_SHIFT | + rps[RP3] << ADF_CFG_SERV_RING_PAIR_3_SHIFT; + + return ring_to_svc_map; +} + +static u32 get_accel_mask(struct adf_hw_device_data *self) +{ + return ADF_GEN6_ACCELERATORS_MASK; +} + +static u32 get_num_accels(struct adf_hw_device_data *self) +{ + return ADF_GEN6_MAX_ACCELERATORS; +} + +static u32 get_num_aes(struct adf_hw_device_data *self) +{ + return self ? hweight32(self->ae_mask) : 0; +} + +static u32 get_misc_bar_id(struct adf_hw_device_data *self) +{ + return ADF_GEN6_PMISC_BAR; +} + +static u32 get_etr_bar_id(struct adf_hw_device_data *self) +{ + return ADF_GEN6_ETR_BAR; +} + +static u32 get_sram_bar_id(struct adf_hw_device_data *self) +{ + return ADF_GEN6_SRAM_BAR; +} + +static enum dev_sku_info get_sku(struct adf_hw_device_data *self) +{ + return DEV_SKU_1; +} + +static void get_arb_info(struct arb_info *arb_info) +{ + arb_info->arb_cfg = ADF_GEN6_ARB_CONFIG; + arb_info->arb_offset = ADF_GEN6_ARB_OFFSET; + arb_info->wt2sam_offset = ADF_GEN6_ARB_WRK_2_SER_MAP_OFFSET; +} + +static void get_admin_info(struct admin_info *admin_csrs_info) +{ + admin_csrs_info->mailbox_offset = ADF_GEN6_MAILBOX_BASE_OFFSET; + admin_csrs_info->admin_msg_ur = ADF_GEN6_ADMINMSGUR_OFFSET; + admin_csrs_info->admin_msg_lr = ADF_GEN6_ADMINMSGLR_OFFSET; +} + +static u32 get_heartbeat_clock(struct adf_hw_device_data *self) +{ + return ADF_GEN6_COUNTER_FREQ; +} + +static void enable_error_correction(struct adf_accel_dev *accel_dev) +{ + void __iomem *csr = adf_get_pmisc_base(accel_dev); + + /* + * Enable all error notification bits in errsou3 except VFLR + * notification on host. + */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK3, ADF_GEN6_VFLNOTIFY); +} + +static void enable_ints(struct adf_accel_dev *accel_dev) +{ + void __iomem *addr = adf_get_pmisc_base(accel_dev); + + /* Enable bundle interrupts */ + ADF_CSR_WR(addr, ADF_GEN6_SMIAPF_RP_X0_MASK_OFFSET, 0); + ADF_CSR_WR(addr, ADF_GEN6_SMIAPF_RP_X1_MASK_OFFSET, 0); + + /* Enable misc interrupts */ + ADF_CSR_WR(addr, ADF_GEN6_SMIAPF_MASK_OFFSET, 0); +} + +static void set_ssm_wdtimer(struct adf_accel_dev *accel_dev) +{ + void __iomem *addr = adf_get_pmisc_base(accel_dev); + u64 val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE; + u64 val = ADF_SSM_WDT_DEFAULT_VALUE; + + /* Enable watchdog timer for sym and dc */ + ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTATHL_OFFSET, ADF_SSMWDTATHH_OFFSET, val); + ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTCNVL_OFFSET, ADF_SSMWDTCNVH_OFFSET, val); + ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTUCSL_OFFSET, ADF_SSMWDTUCSH_OFFSET, val); + ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTDCPRL_OFFSET, ADF_SSMWDTDCPRH_OFFSET, val); + ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTWCPL_OFFSET, ADF_SSMWDTWCPH_OFFSET, val); + ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTWATL_OFFSET, ADF_SSMWDTWATH_OFFSET, val); + + /* Enable watchdog timer for pke */ + ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTPKEL_OFFSET, ADF_SSMWDTPKEH_OFFSET, val_pke); +} + +/* + * The vector routing table is used to select the MSI-X entry to use for each + * interrupt source. + * The first ADF_GEN6_ETR_MAX_BANKS entries correspond to ring interrupts. + * The final entry corresponds to VF2PF or error interrupts. + * This vector table could be used to configure one MSI-X entry to be shared + * between multiple interrupt sources. + * + * The default routing is set to have a one to one correspondence between the + * interrupt source and the MSI-X entry used. + */ +static void set_msix_default_rttable(struct adf_accel_dev *accel_dev) +{ + void __iomem *csr = adf_get_pmisc_base(accel_dev); + unsigned int i; + + for (i = 0; i <= ADF_GEN6_ETR_MAX_BANKS; i++) + ADF_CSR_WR(csr, ADF_GEN6_MSIX_RTTABLE_OFFSET(i), i); +} + +static int reset_ring_pair(void __iomem *csr, u32 bank_number) +{ + u32 status; + int ret; + + /* + * Write rpresetctl register BIT(0) as 1. + * Since rpresetctl registers have no RW fields, no need to preserve + * values for other bits. Just write directly. + */ + ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETCTL(bank_number), + ADF_WQM_CSR_RPRESETCTL_RESET); + + /* Read rpresetsts register and wait for rp reset to complete */ + ret = read_poll_timeout(ADF_CSR_RD, status, + status & ADF_WQM_CSR_RPRESETSTS_STATUS, + ADF_RPRESET_POLL_DELAY_US, + ADF_RPRESET_POLL_TIMEOUT_US, true, + csr, ADF_WQM_CSR_RPRESETSTS(bank_number)); + if (ret) + return ret; + + /* When ring pair reset is done, clear rpresetsts */ + ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETSTS(bank_number), ADF_WQM_CSR_RPRESETSTS_STATUS); + + return 0; +} + +static bool adf_anti_rb_enabled(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + + return !!(hw_data->fuses[0] & ADF_GEN6_ANTI_RB_FUSE_BIT); +} + +static void adf_gen6_init_anti_rb(struct adf_anti_rb_hw_data *anti_rb_data) +{ + anti_rb_data->anti_rb_enabled = adf_anti_rb_enabled; + anti_rb_data->svncheck_offset = ADF_GEN6_SVNCHECK_CSR_MSG; + anti_rb_data->svncheck_retry = 0; + anti_rb_data->sysfs_added = false; +} + +static int ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *csr = adf_get_etr_base(accel_dev); + int ret; + + if (bank_number >= hw_data->num_banks) + return -EINVAL; + + dev_dbg(&GET_DEV(accel_dev), "ring pair reset for bank:%d\n", bank_number); + + ret = reset_ring_pair(csr, bank_number); + if (ret) + dev_err(&GET_DEV(accel_dev), "ring pair reset failed (timeout)\n"); + else + dev_dbg(&GET_DEV(accel_dev), "ring pair reset successful\n"); + + return ret; +} + +static int build_comp_block(void *ctx, enum adf_dc_algo algo) +{ + struct icp_qat_fw_comp_req *req_tmpl = ctx; + struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_hw_comp_51_config_csr_lower hw_comp_lower_csr = { }; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + u32 lower_val; + + switch (algo) { + case QAT_DEFLATE: + header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DYNAMIC; + break; + case QAT_ZSTD: + header->service_cmd_id = ICP_QAT_FW_COMP_CMD_ZSTD_COMPRESS; + break; + default: + return -EINVAL; + } + + hw_comp_lower_csr.lllbd = ICP_QAT_HW_COMP_51_LLLBD_CTRL_LLLBD_DISABLED; + hw_comp_lower_csr.sd = ICP_QAT_HW_COMP_51_SEARCH_DEPTH_LEVEL_1; + lower_val = ICP_QAT_FW_COMP_51_BUILD_CONFIG_LOWER(hw_comp_lower_csr); + cd_pars->u.sl.comp_slice_cfg_word[0] = lower_val; + cd_pars->u.sl.comp_slice_cfg_word[1] = 0; + + /* + * Store Auto Select Best (ASB) multiplier in the request template. + * This will be used in the data path to set the actual threshold + * value based on the input data size. + */ + req_tmpl->u3.asb_threshold.asb_value = ASB_MULTIPLIER; + + return 0; +} + +static int build_decomp_block(void *ctx, enum adf_dc_algo algo) +{ + struct icp_qat_fw_comp_req *req_tmpl = ctx; + struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + + switch (algo) { + case QAT_DEFLATE: + header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS; + break; + case QAT_ZSTD: + header->service_cmd_id = ICP_QAT_FW_COMP_CMD_ZSTD_DECOMPRESS; + break; + default: + return -EINVAL; + } + + cd_pars->u.sl.comp_slice_cfg_word[0] = 0; + cd_pars->u.sl.comp_slice_cfg_word[1] = 0; + req_tmpl->u3.asb_threshold.asb_value = 0; + + return 0; +} + +static void adf_gen6_init_dc_ops(struct adf_dc_ops *dc_ops) +{ + dc_ops->build_comp_block = build_comp_block; + dc_ops->build_decomp_block = build_decomp_block; +} + +static int adf_gen6_init_thd2arb_map(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + u32 *thd2arb_map = hw_data->thd_to_arb_map; + unsigned int i; + + for (i = 0; i < hw_data->num_engines; i++) { + thd2arb_map[i] = adf_gen6_get_arb_mask(accel_dev, i); + dev_dbg(&GET_DEV(accel_dev), "ME:%d arb_mask:%#x\n", i, thd2arb_map[i]); + } + + return 0; +} + +static void init_num_svc_aes(struct adf_rl_hw_data *device_data) +{ + enum adf_fw_objs obj_type, obj_iter; + unsigned int svc, i, num_grp; + u32 ae_mask; + + for (svc = 0; svc < SVC_BASE_COUNT; svc++) { + switch (svc) { + case SVC_SYM: + case SVC_ASYM: + obj_type = ADF_FW_CY_OBJ; + break; + case SVC_DC: + case SVC_DECOMP: + obj_type = ADF_FW_DC_OBJ; + break; + } + + num_grp = ARRAY_SIZE(adf_default_fw_config); + for (i = 0; i < num_grp; i++) { + obj_iter = adf_default_fw_config[i].obj; + if (obj_iter == obj_type) { + ae_mask = adf_default_fw_config[i].ae_mask; + device_data->svc_ae_mask[svc] = hweight32(ae_mask); + break; + } + } + } +} + +static u32 adf_gen6_get_svc_slice_cnt(struct adf_accel_dev *accel_dev, + enum adf_base_services svc) +{ + struct adf_rl_hw_data *device_data = &accel_dev->hw_device->rl_data; + + switch (svc) { + case SVC_SYM: + return device_data->slices.cph_cnt; + case SVC_ASYM: + return device_data->slices.pke_cnt; + case SVC_DC: + return device_data->slices.cpr_cnt + device_data->slices.dcpr_cnt; + case SVC_DECOMP: + return device_data->slices.dcpr_cnt; + default: + return 0; + } +} + +static void set_vc_csr_for_bank(void __iomem *csr, u32 bank_number) +{ + u32 value; + + /* + * After each PF FLR, for each of the 64 ring pairs in the PF, the + * driver must program the ringmodectl CSRs. + */ + value = ADF_CSR_RD(csr, ADF_GEN6_CSR_RINGMODECTL(bank_number)); + FIELD_MODIFY(ADF_GEN6_RINGMODECTL_TC_MASK, &value, ADF_GEN6_RINGMODECTL_TC_DEFAULT); + FIELD_MODIFY(ADF_GEN6_RINGMODECTL_TC_EN_MASK, &value, ADF_GEN6_RINGMODECTL_TC_EN_OP1); + ADF_CSR_WR(csr, ADF_GEN6_CSR_RINGMODECTL(bank_number), value); +} + +static int set_vc_config(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + u32 value; + int err; + + /* + * After each PF FLR, the driver must program the Port Virtual Channel (VC) + * Control Registers. + * Read PVC0CTL then write the masked values. + */ + pci_read_config_dword(pdev, ADF_GEN6_PVC0CTL_OFFSET, &value); + FIELD_MODIFY(ADF_GEN6_PVC0CTL_TCVCMAP_MASK, &value, ADF_GEN6_PVC0CTL_TCVCMAP_DEFAULT); + err = pci_write_config_dword(pdev, ADF_GEN6_PVC0CTL_OFFSET, value); + if (err) { + dev_err(&GET_DEV(accel_dev), "pci write to PVC0CTL failed\n"); + return pcibios_err_to_errno(err); + } + + /* Read PVC1CTL then write masked values */ + pci_read_config_dword(pdev, ADF_GEN6_PVC1CTL_OFFSET, &value); + FIELD_MODIFY(ADF_GEN6_PVC1CTL_TCVCMAP_MASK, &value, ADF_GEN6_PVC1CTL_TCVCMAP_DEFAULT); + FIELD_MODIFY(ADF_GEN6_PVC1CTL_VCEN_MASK, &value, ADF_GEN6_PVC1CTL_VCEN_ON); + err = pci_write_config_dword(pdev, ADF_GEN6_PVC1CTL_OFFSET, value); + if (err) + dev_err(&GET_DEV(accel_dev), "pci write to PVC1CTL failed\n"); + + return pcibios_err_to_errno(err); +} + +static int adf_gen6_set_vc(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + void __iomem *csr = adf_get_etr_base(accel_dev); + u32 i; + + for (i = 0; i < hw_data->num_banks; i++) { + dev_dbg(&GET_DEV(accel_dev), "set virtual channels for bank:%d\n", i); + set_vc_csr_for_bank(csr, i); + } + + return set_vc_config(accel_dev); +} + +static const struct adf_fw_config *get_fw_config(struct adf_accel_dev *accel_dev) +{ + return adf_6xxx_is_wcy(GET_HW_DATA(accel_dev)) ? adf_wcy_fw_config : + adf_default_fw_config; +} + +static u32 get_ae_mask(struct adf_hw_device_data *self) +{ + unsigned long fuses = self->fuses[ADF_FUSECTL4]; + u32 mask = ADF_6XXX_ACCELENGINES_MASK; + + /* + * If bit 0 is set in the fuses, the first 4 engines are disabled. + * If bit 4 is set, the second group of 4 engines are disabled. + * If bit 8 is set, the admin engine (bit 8) is disabled. + */ + if (test_bit(0, &fuses)) + mask &= ~ADF_AE_GROUP_0; + + if (test_bit(4, &fuses)) + mask &= ~ADF_AE_GROUP_1; + + if (test_bit(8, &fuses)) + mask &= ~ADF_AE_GROUP_2; + + return mask; +} + +static u32 get_accel_cap_wcy(struct adf_accel_dev *accel_dev) +{ + u32 capabilities_sym; + u32 fuse; + + fuse = GET_HW_DATA(accel_dev)->fuses[ADF_FUSECTL1]; + + capabilities_sym = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | + ICP_ACCEL_CAPABILITIES_CIPHER | + ICP_ACCEL_CAPABILITIES_AUTHENTICATION | + ICP_ACCEL_CAPABILITIES_WIRELESS_CRYPTO_EXT | + ICP_ACCEL_CAPABILITIES_5G | + ICP_ACCEL_CAPABILITIES_ZUC | + ICP_ACCEL_CAPABILITIES_ZUC_256 | + ICP_ACCEL_CAPABILITIES_EXT_ALGCHAIN; + + if (fuse & ICP_ACCEL_GEN6_MASK_EIA3_SLICE) { + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_ZUC; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_ZUC_256; + } + if (fuse & ICP_ACCEL_GEN6_MASK_ZUC_256_SLICE) + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_ZUC_256; + + if (fuse & ICP_ACCEL_GEN6_MASK_5G_SLICE) + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_5G; + + if (adf_get_service_enabled(accel_dev) == SVC_SYM) + return capabilities_sym; + + return 0; +} + +static u32 get_accel_cap(struct adf_accel_dev *accel_dev) +{ + u32 capabilities_sym, capabilities_asym; + u32 capabilities_dc; + unsigned long mask; + u32 caps = 0; + u32 fusectl1; + u32 fusectl0; + + if (adf_6xxx_is_wcy(GET_HW_DATA(accel_dev))) + return get_accel_cap_wcy(accel_dev); + + fusectl0 = GET_HW_DATA(accel_dev)->fuses[ADF_FUSECTL0]; + fusectl1 = GET_HW_DATA(accel_dev)->fuses[ADF_FUSECTL1]; + + /* Read accelerator capabilities mask */ + capabilities_sym = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | + ICP_ACCEL_CAPABILITIES_CIPHER | + ICP_ACCEL_CAPABILITIES_AUTHENTICATION | + ICP_ACCEL_CAPABILITIES_SHA3 | + ICP_ACCEL_CAPABILITIES_SHA3_EXT | + ICP_ACCEL_CAPABILITIES_CHACHA_POLY | + ICP_ACCEL_CAPABILITIES_AESGCM_SPC | + ICP_ACCEL_CAPABILITIES_AES_V2; + + /* A set bit in fusectl1 means the corresponding feature is OFF in this SKU */ + if (fusectl1 & ICP_ACCEL_GEN6_MASK_UCS_SLICE) { + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CHACHA_POLY; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AESGCM_SPC; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AES_V2; + } + if (fusectl1 & ICP_ACCEL_GEN6_MASK_AUTH_SLICE) { + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SHA3; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SHA3_EXT; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + } + + capabilities_asym = ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | + ICP_ACCEL_CAPABILITIES_SM2 | + ICP_ACCEL_CAPABILITIES_ECEDMONT | + ICP_ACCEL_CAPABILITIES_KPT; + + if (fusectl1 & ICP_ACCEL_GEN6_MASK_PKE_SLICE) { + capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_SM2; + capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_ECEDMONT; + } + + if (fusectl0 & ADF_GEN6_KPT_FUSE_BIT) + capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_KPT; + + capabilities_dc = ICP_ACCEL_CAPABILITIES_COMPRESSION | + ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION | + ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION | + ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64; + + if (fusectl1 & ICP_ACCEL_GEN6_MASK_CPR_SLICE) { + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION; + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION; + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION; + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64; + } + + if (adf_get_service_mask(accel_dev, &mask)) + return 0; + + if (test_bit(SVC_ASYM, &mask)) + caps |= capabilities_asym; + if (test_bit(SVC_SYM, &mask)) + caps |= capabilities_sym; + if (test_bit(SVC_DC, &mask) || test_bit(SVC_DECOMP, &mask)) + caps |= capabilities_dc; + if (test_bit(SVC_DCC, &mask)) { + /* + * Sym capabilities are available for chaining operations, + * but sym crypto instances cannot be supported + */ + caps = capabilities_dc | capabilities_sym; + caps &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + } + + return caps; +} + +static u32 uof_get_num_objs(struct adf_accel_dev *accel_dev) +{ + return adf_6xxx_is_wcy(GET_HW_DATA(accel_dev)) ? + ARRAY_SIZE(adf_wcy_fw_config) : + ARRAY_SIZE(adf_default_fw_config); +} + +static const char *uof_get_name(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + int num_fw_objs = ARRAY_SIZE(adf_6xxx_fw_objs); + const struct adf_fw_config *fw_config; + int id; + + fw_config = get_fw_config(accel_dev); + id = fw_config[obj_num].obj; + if (id >= num_fw_objs) + return NULL; + + return adf_6xxx_fw_objs[id]; +} + +static const char *uof_get_name_6xxx(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + return uof_get_name(accel_dev, obj_num); +} + +static int uof_get_obj_type(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + const struct adf_fw_config *fw_config; + + if (obj_num >= uof_get_num_objs(accel_dev)) + return -EINVAL; + + fw_config = get_fw_config(accel_dev); + + return fw_config[obj_num].obj; +} + +static u32 uof_get_ae_mask(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + const struct adf_fw_config *fw_config; + + fw_config = get_fw_config(accel_dev); + return fw_config[obj_num].ae_mask; +} + +static const u32 *adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev) +{ + if (adf_gen6_init_thd2arb_map(accel_dev)) + dev_warn(&GET_DEV(accel_dev), + "Failed to generate thread to arbiter mapping"); + + return GET_HW_DATA(accel_dev)->thd_to_arb_map; +} + +static int adf_init_device(struct adf_accel_dev *accel_dev) +{ + void __iomem *addr = adf_get_pmisc_base(accel_dev); + u32 status; + u32 csr; + int ret; + + /* Temporarily mask PM interrupt */ + csr = ADF_CSR_RD(addr, ADF_GEN6_ERRMSK2); + csr |= ADF_GEN6_PM_SOU; + ADF_CSR_WR(addr, ADF_GEN6_ERRMSK2, csr); + + /* Set DRV_ACTIVE bit to power up the device */ + ADF_CSR_WR(addr, ADF_GEN6_PM_INTERRUPT, ADF_GEN6_PM_DRV_ACTIVE); + + /* Poll status register to make sure the device is powered up */ + ret = read_poll_timeout(ADF_CSR_RD, status, + status & ADF_GEN6_PM_INIT_STATE, + ADF_GEN6_PM_POLL_DELAY_US, + ADF_GEN6_PM_POLL_TIMEOUT_US, true, addr, + ADF_GEN6_PM_STATUS); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Failed to power up the device\n"); + return ret; + } + + dev_dbg(&GET_DEV(accel_dev), "Setting virtual channels for device qat_dev%d\n", + accel_dev->accel_id); + + ret = adf_gen6_set_vc(accel_dev); + if (ret) + dev_err(&GET_DEV(accel_dev), "Failed to set virtual channels\n"); + + return ret; +} + +static int enable_pm(struct adf_accel_dev *accel_dev) +{ + int ret; + + ret = adf_init_admin_pm(accel_dev, ADF_GEN6_PM_DEFAULT_IDLE_FILTER); + if (ret) + return ret; + + /* Initialize PM internal data */ + adf_gen6_init_dev_pm_data(accel_dev); + + return 0; +} + +static int dev_config(struct adf_accel_dev *accel_dev) +{ + int ret; + + ret = adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC); + if (ret) + return ret; + + ret = adf_cfg_section_add(accel_dev, "Accelerator0"); + if (ret) + return ret; + + switch (adf_get_service_enabled(accel_dev)) { + case SVC_DC: + case SVC_DCC: + ret = adf_gen6_comp_dev_config(accel_dev); + break; + default: + ret = adf_gen6_no_dev_config(accel_dev); + break; + } + if (ret) + return ret; + + __set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); + + return ret; +} + +static void adf_gen6_init_kpt(struct adf_kpt_hw_data *kpt_data) +{ + kpt_data->max_swk_cnt_per_fn_pasid = ADF_6XXX_KPT_MAX_SWK_COUNT_PER_FNPASID; + kpt_data->max_swk_ttl = ADF_6XXX_KPT_MAX_SWK_TTL; + + kpt_data->user_input.enable = false; + kpt_data->user_input.swk_shared = ADF_6XXX_KPT_DEFAULT_SWK_SHARED_MODE; + kpt_data->user_input.swk_max_ttl = ADF_6XXX_KPT_DEFAULT_SWK_TTL; + kpt_data->user_input.swk_cnt_per_fn = ADF_6XXX_KPT_DEFAULT_SWK_CNT_PER_FN; + kpt_data->user_input.swk_cnt_per_pasid = ADF_6XXX_KPT_DEFAULT_SWK_CNT_PER_PASID; +} + +static void adf_gen6_init_rl_data(struct adf_rl_hw_data *rl_data) +{ + rl_data->pciout_tb_offset = ADF_GEN6_RL_TOKEN_PCIEOUT_BUCKET_OFFSET; + rl_data->pciin_tb_offset = ADF_GEN6_RL_TOKEN_PCIEIN_BUCKET_OFFSET; + rl_data->r2l_offset = ADF_GEN6_RL_R2L_OFFSET; + rl_data->l2c_offset = ADF_GEN6_RL_L2C_OFFSET; + rl_data->c2s_offset = ADF_GEN6_RL_C2S_OFFSET; + rl_data->pcie_scale_div = ADF_6XXX_RL_PCIE_SCALE_FACTOR_DIV; + rl_data->pcie_scale_mul = ADF_6XXX_RL_PCIE_SCALE_FACTOR_MUL; + rl_data->max_tp[SVC_ASYM] = ADF_6XXX_RL_MAX_TP_ASYM; + rl_data->max_tp[SVC_SYM] = ADF_6XXX_RL_MAX_TP_SYM; + rl_data->max_tp[SVC_DC] = ADF_6XXX_RL_MAX_TP_DC; + rl_data->max_tp[SVC_DECOMP] = ADF_6XXX_RL_MAX_TP_DECOMP; + rl_data->scan_interval = ADF_6XXX_RL_SCANS_PER_SEC; + rl_data->scale_ref = ADF_6XXX_RL_SLICE_REF; + + init_num_svc_aes(rl_data); +} + +static void adf_gen6_init_services_supported(struct adf_hw_device_data *hw_data) +{ + if (adf_6xxx_is_wcy(hw_data)) + hw_data->services_supported = wcy_services_supported; + else + hw_data->services_supported = services_supported; +} + +void adf_init_hw_data_6xxx(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class = &adf_6xxx_class; + hw_data->instance_id = adf_6xxx_class.instances++; + hw_data->num_banks = ADF_GEN6_ETR_MAX_BANKS; + hw_data->num_banks_per_vf = ADF_GEN6_NUM_BANKS_PER_VF; + hw_data->num_rings_per_bank = ADF_GEN6_NUM_RINGS_PER_BANK; + hw_data->num_accel = ADF_GEN6_MAX_ACCELERATORS; + hw_data->num_engines = ADF_6XXX_MAX_ACCELENGINES; + hw_data->num_logical_accel = 1; + hw_data->tx_rx_gap = ADF_GEN6_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_GEN6_TX_RINGS_MASK; + hw_data->ring_to_svc_map = 0; + hw_data->alloc_irq = adf_isr_resource_alloc; + hw_data->free_irq = adf_isr_resource_free; + hw_data->enable_error_correction = enable_error_correction; + hw_data->get_accel_mask = get_accel_mask; + hw_data->get_ae_mask = get_ae_mask; + hw_data->get_num_accels = get_num_accels; + hw_data->get_num_aes = get_num_aes; + hw_data->get_sram_bar_id = get_sram_bar_id; + hw_data->get_etr_bar_id = get_etr_bar_id; + hw_data->get_misc_bar_id = get_misc_bar_id; + hw_data->get_arb_info = get_arb_info; + hw_data->get_admin_info = get_admin_info; + hw_data->get_accel_cap = get_accel_cap; + hw_data->get_sku = get_sku; + hw_data->init_admin_comms = adf_init_admin_comms; + hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->send_admin_init = adf_send_admin_init; + hw_data->init_arb = adf_init_arb; + hw_data->exit_arb = adf_exit_arb; + hw_data->get_arb_mapping = adf_get_arbiter_mapping; + hw_data->enable_ints = enable_ints; + hw_data->reset_device = adf_reset_flr; + hw_data->admin_ae_mask = ADF_6XXX_ADMIN_AE_MASK; + hw_data->fw_name = ADF_6XXX_FW; + hw_data->fw_mmp_name = ADF_6XXX_MMP; + hw_data->uof_get_name = uof_get_name_6xxx; + hw_data->uof_get_num_objs = uof_get_num_objs; + hw_data->uof_get_obj_type = uof_get_obj_type; + hw_data->uof_get_ae_mask = uof_get_ae_mask; + hw_data->set_msix_rttable = set_msix_default_rttable; + hw_data->set_ssm_wdtimer = set_ssm_wdtimer; + hw_data->get_ring_to_svc_map = get_ring_to_svc_map; + hw_data->disable_iov = adf_disable_sriov; + hw_data->ring_pair_reset = ring_pair_reset; + hw_data->dev_config = dev_config; + hw_data->bank_state_save = adf_bank_state_save; + hw_data->bank_state_restore = adf_bank_state_restore; + hw_data->get_hb_clock = get_heartbeat_clock; + hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE; + hw_data->start_timer = adf_timer_start; + hw_data->stop_timer = adf_timer_stop; + hw_data->init_device = adf_init_device; + hw_data->enable_pm = enable_pm; + hw_data->num_rps = ADF_GEN6_ETR_MAX_BANKS; + hw_data->clock_frequency = ADF_6XXX_AE_FREQ; + hw_data->get_svc_slice_cnt = adf_gen6_get_svc_slice_cnt; + hw_data->accel_capabilities_ext_mask = ADF_ACCEL_CAPABILITIES_EXT_ZSTD; + + adf_gen6_init_services_supported(hw_data); + adf_gen6_init_hw_csr_ops(&hw_data->csr_ops); + adf_gen6_init_pf_pfvf_ops(&hw_data->pfvf_ops); + adf_gen6_init_dc_ops(&hw_data->dc_ops); + adf_gen6_init_vf_mig_ops(&hw_data->vfmig_ops); + adf_gen6_init_ras_ops(&hw_data->ras_ops); + adf_gen6_init_tl_data(&hw_data->tl_data); + adf_gen6_init_rl_data(&hw_data->rl_data); + adf_gen6_init_anti_rb(&hw_data->anti_rb_data); + adf_gen6_init_kpt(&hw_data->kpt_data); +} + +void adf_clean_hw_data_6xxx(struct adf_hw_device_data *hw_data) +{ + if (hw_data->dev_class->instances) + hw_data->dev_class->instances--; +} diff --git a/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.h b/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.h new file mode 100644 index 000000000..2719ddbc1 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.h @@ -0,0 +1,197 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ADF_6XXX_HW_DATA_H_ +#define ADF_6XXX_HW_DATA_H_ + +#include +#include +#include + +#include "adf_accel_devices.h" +#include "adf_cfg_common.h" +#include "adf_dc.h" + +/* PCIe configuration space */ +#define ADF_GEN6_BAR_MASK (BIT(0) | BIT(2) | BIT(4)) +#define ADF_GEN6_SRAM_BAR 0 +#define ADF_GEN6_PMISC_BAR 1 +#define ADF_GEN6_ETR_BAR 2 +#define ADF_6XXX_MAX_ACCELENGINES 9 + +/* Clocks frequency */ +#define ADF_GEN6_COUNTER_FREQ (100 * HZ_PER_MHZ) + +/* Physical function fuses */ +#define ADF_GEN6_FUSECTL0_OFFSET 0x2C8 +#define ADF_GEN6_FUSECTL1_OFFSET 0x2CC +#define ADF_GEN6_FUSECTL4_OFFSET 0x2D8 + +/* Accelerators */ +#define ADF_GEN6_ACCELERATORS_MASK 0x1 +#define ADF_GEN6_MAX_ACCELERATORS 1 + +/* MSI-X interrupt */ +#define ADF_GEN6_SMIAPF_RP_X0_MASK_OFFSET 0x41A040 +#define ADF_GEN6_SMIAPF_RP_X1_MASK_OFFSET 0x41A044 +#define ADF_GEN6_SMIAPF_MASK_OFFSET 0x41A084 +#define ADF_GEN6_MSIX_RTTABLE_OFFSET(i) (0x409000 + ((i) * 4)) + +/* Bank and ring configuration */ +#define ADF_GEN6_NUM_RINGS_PER_BANK 2 +#define ADF_GEN6_NUM_BANKS_PER_VF 4 +#define ADF_GEN6_ETR_MAX_BANKS 64 +#define ADF_GEN6_RX_RINGS_OFFSET 1 +#define ADF_GEN6_TX_RINGS_MASK 0x1 + +/* Arbiter configuration */ +#define ADF_GEN6_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0)) +#define ADF_GEN6_ARB_OFFSET 0x000 +#define ADF_GEN6_ARB_WRK_2_SER_MAP_OFFSET 0x400 + +/* Admin interface configuration */ +#define ADF_GEN6_ADMINMSGUR_OFFSET 0x500574 +#define ADF_GEN6_ADMINMSGLR_OFFSET 0x500578 +#define ADF_GEN6_MAILBOX_BASE_OFFSET 0x600970 + +/* Anti-rollback */ +#define ADF_GEN6_SVNCHECK_CSR_MSG 0x640004 + +/* Fuse bits */ +#define ADF_GEN6_ANTI_RB_FUSE_BIT BIT(24) +#define ADF_GEN6_KPT_FUSE_BIT BIT(16) + +/* + * Watchdog timers + * Timeout is in cycles. Clock speed may vary across products but this + * value should be a few milli-seconds. + */ +#define ADF_SSM_WDT_DEFAULT_VALUE 0x7000000ULL +#define ADF_SSM_WDT_PKE_DEFAULT_VALUE 0x8000000ULL +#define ADF_SSMWDTATHL_OFFSET 0x5208 +#define ADF_SSMWDTATHH_OFFSET 0x520C +#define ADF_SSMWDTCNVL_OFFSET 0x5408 +#define ADF_SSMWDTCNVH_OFFSET 0x540C +#define ADF_SSMWDTWCPL_OFFSET 0x5608 +#define ADF_SSMWDTWCPH_OFFSET 0x560C +#define ADF_SSMWDTUCSL_OFFSET 0x5808 +#define ADF_SSMWDTUCSH_OFFSET 0x580C +#define ADF_SSMWDTDCPRL_OFFSET 0x5A08 +#define ADF_SSMWDTDCPRH_OFFSET 0x5A0C +#define ADF_SSMWDTWATL_OFFSET 0x5C08 +#define ADF_SSMWDTWATH_OFFSET 0x5C0C +#define ADF_SSMWDTPKEL_OFFSET 0x5E08 +#define ADF_SSMWDTPKEH_OFFSET 0x5E0C + +/* Ring reset */ +#define ADF_RPRESET_POLL_TIMEOUT_US (5 * USEC_PER_SEC) +#define ADF_RPRESET_POLL_DELAY_US 20 +#define ADF_WQM_CSR_RPRESETCTL_RESET BIT(0) +#define ADF_WQM_CSR_RPRESETCTL(bank) (0x6000 + (bank) * 8) +#define ADF_WQM_CSR_RPRESETSTS_STATUS BIT(0) +#define ADF_WQM_CSR_RPRESETSTS(bank) (ADF_WQM_CSR_RPRESETCTL(bank) + 4) + +/* Controls and sets up the corresponding ring mode of operation */ +#define ADF_GEN6_CSR_RINGMODECTL(bank) (0x9000 + (bank) * 4) + +/* Specifies the traffic class to use for the transactions to/from the ring */ +#define ADF_GEN6_RINGMODECTL_TC_MASK GENMASK(18, 16) +#define ADF_GEN6_RINGMODECTL_TC_DEFAULT 0x7 + +/* Specifies usage of tc for the transactions to/from this ring */ +#define ADF_GEN6_RINGMODECTL_TC_EN_MASK GENMASK(20, 19) + +/* + * Use the value programmed in the tc field for request descriptor + * and metadata read transactions + */ +#define ADF_GEN6_RINGMODECTL_TC_EN_OP1 0x1 + +/* VC0 Resource Control Register */ +#define ADF_GEN6_PVC0CTL_OFFSET 0x204 +#define ADF_GEN6_PVC0CTL_TCVCMAP_OFFSET 1 +#define ADF_GEN6_PVC0CTL_TCVCMAP_MASK GENMASK(7, 1) +#define ADF_GEN6_PVC0CTL_TCVCMAP_DEFAULT 0x3F + +/* VC1 Resource Control Register */ +#define ADF_GEN6_PVC1CTL_OFFSET 0x210 +#define ADF_GEN6_PVC1CTL_TCVCMAP_OFFSET 1 +#define ADF_GEN6_PVC1CTL_TCVCMAP_MASK GENMASK(7, 1) +#define ADF_GEN6_PVC1CTL_TCVCMAP_DEFAULT 0x40 +#define ADF_GEN6_PVC1CTL_VCEN_OFFSET 31 +#define ADF_GEN6_PVC1CTL_VCEN_MASK BIT(31) +/* RW bit: 0x1 - enables a Virtual Channel, 0x0 - disables */ +#define ADF_GEN6_PVC1CTL_VCEN_ON 0x1 + +/* Error source mask registers */ +#define ADF_GEN6_ERRMSK0 0x41A210 +#define ADF_GEN6_ERRMSK1 0x41A214 +#define ADF_GEN6_ERRMSK2 0x41A218 +#define ADF_GEN6_ERRMSK3 0x41A21C + +#define ADF_GEN6_VFLNOTIFY BIT(7) + +/* Number of heartbeat counter pairs */ +#define ADF_NUM_HB_CNT_PER_AE ADF_NUM_THREADS_PER_AE + +/* Rate Limiting */ +#define ADF_GEN6_RL_R2L_OFFSET 0x508000 +#define ADF_GEN6_RL_L2C_OFFSET 0x509000 +#define ADF_GEN6_RL_C2S_OFFSET 0x508818 +#define ADF_GEN6_RL_TOKEN_PCIEIN_BUCKET_OFFSET 0x508800 +#define ADF_GEN6_RL_TOKEN_PCIEOUT_BUCKET_OFFSET 0x508804 + +/* Physical function fuses */ +#define ADF_6XXX_ACCELENGINES_MASK GENMASK(8, 0) +#define ADF_6XXX_ADMIN_AE_MASK GENMASK(8, 8) + +/* Firmware binaries */ +#define ADF_6XXX_FW "qat_6xxx.bin" +#define ADF_6XXX_MMP "qat_6xxx_mmp.bin" +#define ADF_6XXX_CY_OBJ "qat_6xxx_cy.bin" +#define ADF_6XXX_DC_OBJ "qat_6xxx_dc.bin" +#define ADF_6XXX_ADMIN_OBJ "qat_6xxx_admin.bin" +#define ADF_6XXX_WCY_OBJ "qat_6xxx_wcy.bin" + +/* RL constants */ +#define ADF_6XXX_RL_PCIE_SCALE_FACTOR_DIV 100 +#define ADF_6XXX_RL_PCIE_SCALE_FACTOR_MUL 102 +#define ADF_6XXX_RL_SCANS_PER_SEC 954 +#define ADF_6XXX_RL_MAX_TP_ASYM 173750UL +#define ADF_6XXX_RL_MAX_TP_SYM 95000UL +#define ADF_6XXX_RL_MAX_TP_DC 40000UL +#define ADF_6XXX_RL_MAX_TP_DECOMP 40000UL +#define ADF_6XXX_RL_SLICE_REF 1000UL + +/* Clock frequency */ +#define ADF_6XXX_AE_FREQ (1000 * HZ_PER_MHZ) + +/* KPT */ +#define ADF_6XXX_KPT_MAX_SWK_COUNT_PER_FNPASID 128 +#define ADF_6XXX_KPT_MAX_SWK_TTL 31536000 +#define ADF_6XXX_KPT_DEFAULT_SWK_SHARED_MODE 1 +#define ADF_6XXX_KPT_DEFAULT_SWK_TTL 0 +#define ADF_6XXX_KPT_DEFAULT_SWK_CNT_PER_FN 0 +#define ADF_6XXX_KPT_DEFAULT_SWK_CNT_PER_PASID 0 + +enum icp_qat_gen6_slice_mask { + ICP_ACCEL_GEN6_MASK_UCS_SLICE = BIT(0), + ICP_ACCEL_GEN6_MASK_AUTH_SLICE = BIT(1), + ICP_ACCEL_GEN6_MASK_PKE_SLICE = BIT(2), + ICP_ACCEL_GEN6_MASK_CPR_SLICE = BIT(3), + ICP_ACCEL_GEN6_MASK_DCPRZ_SLICE = BIT(4), + ICP_ACCEL_GEN6_MASK_EIA3_SLICE = BIT(5), + ICP_ACCEL_GEN6_MASK_WCP_WAT_SLICE = BIT(6), + ICP_ACCEL_GEN6_MASK_ZUC_256_SLICE = BIT(7), + ICP_ACCEL_GEN6_MASK_5G_SLICE = BIT(8), +}; + +/* Return true if the device is a wireless crypto (WCY) SKU */ +static inline bool adf_6xxx_is_wcy(struct adf_hw_device_data *hw_data) +{ + return !(hw_data->fuses[ADF_FUSECTL1] & ICP_ACCEL_GEN6_MASK_WCP_WAT_SLICE); +} + +void adf_init_hw_data_6xxx(struct adf_hw_device_data *hw_data); +void adf_clean_hw_data_6xxx(struct adf_hw_device_data *hw_data); + +#endif /* ADF_6XXX_HW_DATA_H_ */ diff --git a/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c new file mode 100644 index 000000000..319b4251a --- /dev/null +++ b/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2025 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "adf_gen6_shared.h" +#include "adf_6xxx_hw_data.h" +#include "adf_heartbeat.h" + +static int bar_map[] = { + 0, /* SRAM */ + 2, /* PMISC */ + 4, /* ETR */ +}; + +static void adf_device_down(void *accel_dev) +{ + adf_dev_down(accel_dev); +} + +static void adf_dbgfs_cleanup(void *accel_dev) +{ + adf_dbgfs_exit(accel_dev); +} + +static void adf_cfg_device_remove(void *accel_dev) +{ + adf_cfg_dev_remove(accel_dev); +} + +static void adf_cleanup_hw_data(void *accel_dev) +{ + struct adf_accel_dev *accel_device = accel_dev; + + if (accel_device->hw_device) { + adf_clean_hw_data_6xxx(accel_device->hw_device); + accel_device->hw_device = NULL; + } +} + +static void adf_devmgr_remove(void *accel_dev) +{ + adf_devmgr_rm_dev(accel_dev, NULL); +} + +static int adf_gen6_cfg_dev_init(struct adf_accel_dev *accel_dev) +{ + const char *config; + int ret; + + /* + * Wireless SKU - symmetric crypto service only + * Non-wireless SKU - crypto service for even devices and compression for odd devices + */ + if (adf_6xxx_is_wcy(GET_HW_DATA(accel_dev))) + config = ADF_CFG_SYM; + else + config = accel_dev->accel_id % 2 ? ADF_CFG_DC : ADF_CFG_CY; + + ret = adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC); + if (ret) + return ret; + + ret = adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC, + ADF_SERVICES_ENABLED, config, + ADF_STR); + if (ret) + return ret; + + adf_heartbeat_save_cfg_param(accel_dev, ADF_CFG_HB_TIMER_MIN_MS); + + return 0; +} + +static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct adf_accel_pci *accel_pci_dev; + struct adf_hw_device_data *hw_data; + struct device *dev = &pdev->dev; + struct adf_accel_dev *accel_dev; + struct adf_bar *bar; + unsigned int i; + int ret; + + if (num_possible_nodes() > 1 && dev_to_node(dev) < 0) { + /* + * If the accelerator is connected to a node with no memory + * there is no point in using the accelerator since the remote + * memory transaction will be very slow. + */ + return dev_err_probe(dev, -EINVAL, "Invalid NUMA configuration.\n"); + } + + accel_dev = devm_kzalloc(dev, sizeof(*accel_dev), GFP_KERNEL); + if (!accel_dev) + return -ENOMEM; + + INIT_LIST_HEAD(&accel_dev->crypto_list); + INIT_LIST_HEAD(&accel_dev->list); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = pdev; + accel_dev->owner = THIS_MODULE; + + hw_data = devm_kzalloc(dev, sizeof(*hw_data), GFP_KERNEL); + if (!hw_data) + return -ENOMEM; + + pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); + pci_read_config_dword(pdev, ADF_GEN6_FUSECTL4_OFFSET, &hw_data->fuses[ADF_FUSECTL4]); + pci_read_config_dword(pdev, ADF_GEN6_FUSECTL0_OFFSET, &hw_data->fuses[ADF_FUSECTL0]); + pci_read_config_dword(pdev, ADF_GEN6_FUSECTL1_OFFSET, &hw_data->fuses[ADF_FUSECTL1]); + + /* Enable PCI device */ + ret = pcim_enable_device(pdev); + if (ret) + return dev_err_probe(dev, ret, "Cannot enable PCI device.\n"); + + ret = adf_devmgr_add_dev(accel_dev, NULL); + if (ret) + return dev_err_probe(dev, ret, "Failed to add new accelerator device.\n"); + + ret = devm_add_action_or_reset(dev, adf_devmgr_remove, accel_dev); + if (ret) + return ret; + + accel_dev->hw_device = hw_data; + adf_init_hw_data_6xxx(accel_dev->hw_device); + + ret = devm_add_action_or_reset(dev, adf_cleanup_hw_data, accel_dev); + if (ret) + return ret; + + /* Get Accelerators and Accelerator Engine masks */ + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); + accel_pci_dev->sku = hw_data->get_sku(hw_data); + + /* If the device has no acceleration engines then ignore it */ + if (!hw_data->accel_mask || !hw_data->ae_mask || + (~hw_data->ae_mask & ADF_GEN6_ACCELERATORS_MASK)) { + ret = -EFAULT; + return dev_err_probe(dev, ret, "No acceleration units were found.\n"); + } + + /* Create device configuration table */ + ret = adf_cfg_dev_add(accel_dev); + if (ret) + return ret; + + ret = devm_add_action_or_reset(dev, adf_cfg_device_remove, accel_dev); + if (ret) + return ret; + + /* Set DMA identifier */ + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); + if (ret) + return dev_err_probe(dev, ret, "No usable DMA configuration.\n"); + + ret = adf_gen6_cfg_dev_init(accel_dev); + if (ret) + return dev_err_probe(dev, ret, "Failed to initialize configuration.\n"); + + /* Get accelerator capability mask */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); + if (!hw_data->accel_capabilities_mask) { + ret = -EINVAL; + return dev_err_probe(dev, ret, "Failed to get capabilities mask.\n"); + } + + for (i = 0; i < ARRAY_SIZE(bar_map); i++) { + bar = &accel_pci_dev->pci_bars[i]; + + /* Map 64-bit PCIe BAR */ + bar->virt_addr = pcim_iomap_region(pdev, bar_map[i], pci_name(pdev)); + if (IS_ERR(bar->virt_addr)) { + ret = PTR_ERR(bar->virt_addr); + return dev_err_probe(dev, ret, "Failed to ioremap PCI region.\n"); + } + } + + /* + * The PCI config space is saved at this point and will be restored + * after a Function Level Reset (FLR) as the FLR does not completely + * restore it. + */ + ret = pci_save_state(pdev); + if (ret) + return dev_err_probe(dev, ret, "Failed to save pci state.\n"); + + accel_dev->ras_errors.enabled = true; + + adf_dbgfs_init(accel_dev); + + ret = devm_add_action_or_reset(dev, adf_dbgfs_cleanup, accel_dev); + if (ret) + return ret; + + ret = adf_dev_up(accel_dev, true); + if (ret) { + adf_dev_down(accel_dev); + return ret; + } + + ret = devm_add_action_or_reset(dev, adf_device_down, accel_dev); + if (ret) + return ret; + + ret = adf_sysfs_init(accel_dev); + if (ret) + return ret; + + if (!(hw_data->fuses[ADF_FUSECTL0] & ADF_GEN6_KPT_FUSE_BIT)) + ret = adf_sysfs_init_kpt(accel_dev); + + return ret; +} + +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + +static const struct pci_device_id adf_pci_tbl[] = { + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_6XXX) }, + { } +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_6XXX_DEVICE_NAME, + .probe = adf_probe, + .shutdown = adf_shutdown, + .sriov_configure = adf_sriov_configure, + .err_handler = &adf_err_handler, +}; +module_pci_driver(adf_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Intel"); +MODULE_FIRMWARE(ADF_6XXX_FW); +MODULE_FIRMWARE(ADF_6XXX_MMP); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology for GEN6 Devices"); +MODULE_SOFTDEP("pre: crypto-intel_qat"); +MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/intel/qat/qat_c3xxx/Makefile b/drivers/crypto/intel/qat/qat_c3xxx/Makefile new file mode 100644 index 000000000..43604c025 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_c3xxx/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXX) += qat_c3xxx.o +qat_c3xxx-y := adf_drv.o adf_c3xxx_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c new file mode 100644 index 000000000..07f2c42a6 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2021 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "adf_c3xxx_hw_data.h" +#include "adf_heartbeat.h" +#include "icp_qat_hw.h" + +/* Worker thread to service arbiter mappings */ +static const u32 thrd_to_arb_map[ADF_C3XXX_MAX_ACCELENGINES] = { + 0x12222AAA, 0x11222AAA, 0x12222AAA, + 0x11222AAA, 0x12222AAA, 0x11222AAA +}; + +static struct adf_hw_device_class c3xxx_class = { + .name = ADF_C3XXX_DEVICE_NAME, + .type = DEV_C3XXX, +}; + +static u32 get_accel_mask(struct adf_hw_device_data *self) +{ + u32 fuses = self->fuses[ADF_FUSECTL0]; + u32 straps = self->straps; + u32 accel; + + accel = ~(fuses | straps) >> ADF_C3XXX_ACCELERATORS_REG_OFFSET; + accel &= ADF_C3XXX_ACCELERATORS_MASK; + + return accel; +} + +static u32 get_ae_mask(struct adf_hw_device_data *self) +{ + u32 fuses = self->fuses[ADF_FUSECTL0]; + u32 straps = self->straps; + unsigned long disabled; + u32 ae_disable; + int accel; + + /* If an accel is disabled, then disable the corresponding two AEs */ + disabled = ~get_accel_mask(self) & ADF_C3XXX_ACCELERATORS_MASK; + ae_disable = BIT(1) | BIT(0); + for_each_set_bit(accel, &disabled, ADF_C3XXX_MAX_ACCELERATORS) + straps |= ae_disable << (accel << 1); + + return ~(fuses | straps) & ADF_C3XXX_ACCELENGINES_MASK; +} + +static u32 get_ts_clock(struct adf_hw_device_data *self) +{ + /* + * Timestamp update interval is 16 AE clock ticks for c3xxx. + */ + return self->clock_frequency / 16; +} + +static int measure_clock(struct adf_accel_dev *accel_dev) +{ + u32 frequency; + int ret; + + ret = adf_dev_measure_clock(accel_dev, &frequency, ADF_C3XXX_MIN_AE_FREQ, + ADF_C3XXX_MAX_AE_FREQ); + if (ret) + return ret; + + accel_dev->hw_device->clock_frequency = frequency; + return 0; +} + +static u32 get_misc_bar_id(struct adf_hw_device_data *self) +{ + return ADF_C3XXX_PMISC_BAR; +} + +static u32 get_etr_bar_id(struct adf_hw_device_data *self) +{ + return ADF_C3XXX_ETR_BAR; +} + +static u32 get_sram_bar_id(struct adf_hw_device_data *self) +{ + return ADF_C3XXX_SRAM_BAR; +} + +static enum dev_sku_info get_sku(struct adf_hw_device_data *self) +{ + int aes = self->get_num_aes(self); + + if (aes == 6) + return DEV_SKU_4; + + return DEV_SKU_UNKNOWN; +} + +static const u32 *adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev) +{ + return thrd_to_arb_map; +} + +static void configure_iov_threads(struct adf_accel_dev *accel_dev, bool enable) +{ + adf_gen2_cfg_iov_thds(accel_dev, enable, + ADF_C3XXX_AE2FUNC_MAP_GRP_A_NUM_REGS, + ADF_C3XXX_AE2FUNC_MAP_GRP_B_NUM_REGS); +} + +void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class = &c3xxx_class; + hw_data->instance_id = c3xxx_class.instances++; + hw_data->num_banks = ADF_C3XXX_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; + hw_data->num_accel = ADF_C3XXX_MAX_ACCELERATORS; + hw_data->num_logical_accel = 1; + hw_data->num_engines = ADF_C3XXX_MAX_ACCELENGINES; + hw_data->tx_rx_gap = ADF_GEN2_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_GEN2_TX_RINGS_MASK; + hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP; + hw_data->alloc_irq = adf_isr_resource_alloc; + hw_data->free_irq = adf_isr_resource_free; + hw_data->enable_error_correction = adf_gen2_enable_error_correction; + hw_data->get_accel_mask = get_accel_mask; + hw_data->get_ae_mask = get_ae_mask; + hw_data->get_accel_cap = adf_gen2_get_accel_cap; + hw_data->get_num_accels = adf_gen2_get_num_accels; + hw_data->get_num_aes = adf_gen2_get_num_aes; + hw_data->get_sram_bar_id = get_sram_bar_id; + hw_data->get_etr_bar_id = get_etr_bar_id; + hw_data->get_misc_bar_id = get_misc_bar_id; + hw_data->get_admin_info = adf_gen2_get_admin_info; + hw_data->get_arb_info = adf_gen2_get_arb_info; + hw_data->get_sku = get_sku; + hw_data->fw_name = ADF_C3XXX_FW; + hw_data->fw_mmp_name = ADF_C3XXX_MMP; + hw_data->init_admin_comms = adf_init_admin_comms; + hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->configure_iov_threads = configure_iov_threads; + hw_data->send_admin_init = adf_send_admin_init; + hw_data->init_arb = adf_init_arb; + hw_data->exit_arb = adf_exit_arb; + hw_data->get_arb_mapping = adf_get_arbiter_mapping; + hw_data->enable_ints = adf_gen2_enable_ints; + hw_data->reset_device = adf_reset_flr; + hw_data->set_ssm_wdtimer = adf_gen2_set_ssm_wdtimer; + hw_data->disable_iov = adf_disable_sriov; + hw_data->dev_config = adf_gen2_dev_config; + hw_data->measure_clock = measure_clock; + hw_data->get_hb_clock = get_ts_clock; + hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE; + hw_data->check_hb_ctrs = adf_heartbeat_check_ctrs; + + adf_gen2_init_pf_pfvf_ops(&hw_data->pfvf_ops); + adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); + adf_gen2_init_dc_ops(&hw_data->dc_ops); +} + +void adf_clean_hw_data_c3xxx(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class->instances--; +} diff --git a/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.h b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.h new file mode 100644 index 000000000..690c6a1aa --- /dev/null +++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#ifndef ADF_C3XXX_HW_DATA_H_ +#define ADF_C3XXX_HW_DATA_H_ + +#include + +/* PCIe configuration space */ +#define ADF_C3XXX_PMISC_BAR 0 +#define ADF_C3XXX_ETR_BAR 1 +#define ADF_C3XXX_SRAM_BAR 0 +#define ADF_C3XXX_MAX_ACCELERATORS 3 +#define ADF_C3XXX_MAX_ACCELENGINES 6 +#define ADF_C3XXX_ACCELERATORS_REG_OFFSET 16 +#define ADF_C3XXX_ACCELERATORS_MASK 0x7 +#define ADF_C3XXX_ACCELENGINES_MASK 0x3F +#define ADF_C3XXX_ETR_MAX_BANKS 16 +#define ADF_C3XXX_SOFTSTRAP_CSR_OFFSET 0x2EC + +/* AE to function mapping */ +#define ADF_C3XXX_AE2FUNC_MAP_GRP_A_NUM_REGS 48 +#define ADF_C3XXX_AE2FUNC_MAP_GRP_B_NUM_REGS 6 + +/* Clocks frequency */ +#define ADF_C3XXX_AE_FREQ (685 * HZ_PER_MHZ) +#define ADF_C3XXX_MIN_AE_FREQ (533 * HZ_PER_MHZ) +#define ADF_C3XXX_MAX_AE_FREQ (685 * HZ_PER_MHZ) + +/* Firmware Binary */ +#define ADF_C3XXX_FW "qat_c3xxx.bin" +#define ADF_C3XXX_MMP "qat_c3xxx_mmp.bin" + +void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data); +void adf_clean_hw_data_c3xxx(struct adf_hw_device_data *hw_data); +#endif diff --git a/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c new file mode 100644 index 000000000..e816cc006 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "adf_c3xxx_hw_data.h" + +static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) +{ + pci_release_regions(accel_dev->accel_pci_dev.pci_dev); + pci_disable_device(accel_dev->accel_pci_dev.pci_dev); +} + +static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; + int i; + + for (i = 0; i < ADF_PCI_MAX_BARS; i++) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; + + if (bar->virt_addr) + pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr); + } + + if (accel_dev->hw_device) { + switch (accel_pci_dev->pci_dev->device) { + case PCI_DEVICE_ID_INTEL_QAT_C3XXX: + adf_clean_hw_data_c3xxx(accel_dev->hw_device); + break; + default: + break; + } + kfree(accel_dev->hw_device); + accel_dev->hw_device = NULL; + } + adf_dbgfs_exit(accel_dev); + adf_cfg_dev_remove(accel_dev); + adf_devmgr_rm_dev(accel_dev, NULL); +} + +static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct adf_accel_dev *accel_dev; + struct adf_accel_pci *accel_pci_dev; + struct adf_hw_device_data *hw_data; + unsigned int i, bar_nr; + unsigned long bar_mask; + int ret; + + switch (ent->device) { + case PCI_DEVICE_ID_INTEL_QAT_C3XXX: + break; + default: + dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); + return -ENODEV; + } + + if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) { + /* If the accelerator is connected to a node with no memory + * there is no point in using the accelerator since the remote + * memory transaction will be very slow. */ + dev_err(&pdev->dev, "Invalid NUMA configuration.\n"); + return -EINVAL; + } + + accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!accel_dev) + return -ENOMEM; + + INIT_LIST_HEAD(&accel_dev->crypto_list); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = pdev; + + /* Add accel device to accel table. + * This should be called before adf_cleanup_accel is called */ + if (adf_devmgr_add_dev(accel_dev, NULL)) { + dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); + kfree(accel_dev); + return -EFAULT; + } + + accel_dev->owner = THIS_MODULE; + /* Allocate and configure device configuration structure */ + hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!hw_data) { + ret = -ENOMEM; + goto out_err; + } + + accel_dev->hw_device = hw_data; + adf_init_hw_data_c3xxx(accel_dev->hw_device); + pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); + pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, + &hw_data->fuses[ADF_FUSECTL0]); + pci_read_config_dword(pdev, ADF_C3XXX_SOFTSTRAP_CSR_OFFSET, + &hw_data->straps); + + /* Get Accelerators and Accelerators Engines masks */ + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); + accel_pci_dev->sku = hw_data->get_sku(hw_data); + /* If the device has no acceleration engines then ignore it. */ + if (!hw_data->accel_mask || !hw_data->ae_mask || + ((~hw_data->ae_mask) & 0x01)) { + dev_err(&pdev->dev, "No acceleration units found"); + ret = -EFAULT; + goto out_err; + } + + /* Create device configuration table */ + ret = adf_cfg_dev_add(accel_dev); + if (ret) + goto out_err; + + /* enable PCI device */ + if (pci_enable_device(pdev)) { + ret = -EFAULT; + goto out_err; + } + + /* set dma identifier */ + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); + if (ret) { + dev_err(&pdev->dev, "No usable DMA configuration\n"); + goto out_err_disable; + } + + if (pci_request_regions(pdev, ADF_C3XXX_DEVICE_NAME)) { + ret = -EFAULT; + goto out_err_disable; + } + + /* Get accelerator capabilities mask */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); + + /* Find and map all the device's BARS */ + i = 0; + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); + for_each_set_bit(bar_nr, &bar_mask, ADF_PCI_MAX_BARS * 2) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; + + bar->base_addr = pci_resource_start(pdev, bar_nr); + if (!bar->base_addr) + break; + bar->size = pci_resource_len(pdev, bar_nr); + bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); + if (!bar->virt_addr) { + pci_err(pdev, "Failed to map BAR %d\n", bar_nr); + ret = -EFAULT; + goto out_err_free_reg; + } + } + + if (pci_save_state(pdev)) { + pci_err(pdev, "Failed to save pci state\n"); + ret = -ENOMEM; + goto out_err_free_reg; + } + + adf_dbgfs_init(accel_dev); + + ret = adf_dev_up(accel_dev, true); + if (ret) + goto out_err_dev_stop; + + return ret; + +out_err_dev_stop: + adf_dev_down(accel_dev); +out_err_free_reg: + pci_release_regions(accel_pci_dev->pci_dev); +out_err_disable: + pci_disable_device(accel_pci_dev->pci_dev); +out_err: + adf_cleanup_accel(accel_dev); + kfree(accel_dev); + return ret; +} + +static void adf_remove(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + if (!accel_dev) { + pr_err("QAT: Driver removal failed\n"); + return; + } + adf_dev_down(accel_dev); + adf_cleanup_accel(accel_dev); + adf_cleanup_pci_dev(accel_dev); + kfree(accel_dev); +} + +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + +static const struct pci_device_id adf_pci_tbl[] = { + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C3XXX) }, + { } +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_C3XXX_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, + .shutdown = adf_shutdown, + .sriov_configure = adf_sriov_configure, + .err_handler = &adf_err_handler, +}; + +static int __init adfdrv_init(void) +{ + request_module("intel_qat"); + + if (pci_register_driver(&adf_driver)) { + pr_err("QAT: Driver initialization failed\n"); + return -EFAULT; + } + return 0; +} + +static void __exit adfdrv_release(void) +{ + pci_unregister_driver(&adf_driver); +} + +module_init(adfdrv_init); +module_exit(adfdrv_release); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Intel"); +MODULE_FIRMWARE(ADF_C3XXX_FW); +MODULE_FIRMWARE(ADF_C3XXX_MMP); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); +MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile b/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile new file mode 100644 index 000000000..03f6745b4 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXXVF) += qat_c3xxxvf.o +qat_c3xxxvf-y := adf_drv.o adf_c3xxxvf_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c new file mode 100644 index 000000000..db3c33fa1 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2015 - 2021 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include +#include "adf_c3xxxvf_hw_data.h" + +static struct adf_hw_device_class c3xxxiov_class = { + .name = ADF_C3XXXVF_DEVICE_NAME, + .type = DEV_C3XXXVF, +}; + +static u32 get_accel_mask(struct adf_hw_device_data *self) +{ + return ADF_C3XXXIOV_ACCELERATORS_MASK; +} + +static u32 get_ae_mask(struct adf_hw_device_data *self) +{ + return ADF_C3XXXIOV_ACCELENGINES_MASK; +} + +static u32 get_num_accels(struct adf_hw_device_data *self) +{ + return ADF_C3XXXIOV_MAX_ACCELERATORS; +} + +static u32 get_num_aes(struct adf_hw_device_data *self) +{ + return ADF_C3XXXIOV_MAX_ACCELENGINES; +} + +static u32 get_misc_bar_id(struct adf_hw_device_data *self) +{ + return ADF_C3XXXIOV_PMISC_BAR; +} + +static u32 get_etr_bar_id(struct adf_hw_device_data *self) +{ + return ADF_C3XXXIOV_ETR_BAR; +} + +static enum dev_sku_info get_sku(struct adf_hw_device_data *self) +{ + return DEV_SKU_VF; +} + +static int adf_vf_int_noop(struct adf_accel_dev *accel_dev) +{ + return 0; +} + +static void adf_vf_void_noop(struct adf_accel_dev *accel_dev) +{ +} + +void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class = &c3xxxiov_class; + hw_data->num_banks = ADF_C3XXXIOV_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; + hw_data->num_accel = ADF_C3XXXIOV_MAX_ACCELERATORS; + hw_data->num_logical_accel = 1; + hw_data->num_engines = ADF_C3XXXIOV_MAX_ACCELENGINES; + hw_data->tx_rx_gap = ADF_C3XXXIOV_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_C3XXXIOV_TX_RINGS_MASK; + hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP; + hw_data->alloc_irq = adf_vf_isr_resource_alloc; + hw_data->free_irq = adf_vf_isr_resource_free; + hw_data->enable_error_correction = adf_vf_void_noop; + hw_data->init_admin_comms = adf_vf_int_noop; + hw_data->exit_admin_comms = adf_vf_void_noop; + hw_data->send_admin_init = adf_vf2pf_notify_init; + hw_data->init_arb = adf_vf_int_noop; + hw_data->exit_arb = adf_vf_void_noop; + hw_data->disable_iov = adf_vf2pf_notify_shutdown; + hw_data->get_accel_mask = get_accel_mask; + hw_data->get_ae_mask = get_ae_mask; + hw_data->get_num_accels = get_num_accels; + hw_data->get_num_aes = get_num_aes; + hw_data->get_etr_bar_id = get_etr_bar_id; + hw_data->get_misc_bar_id = get_misc_bar_id; + hw_data->get_sku = get_sku; + hw_data->enable_ints = adf_vf_void_noop; + hw_data->dev_class->instances++; + hw_data->dev_config = adf_gen2_dev_config; + adf_devmgr_update_class_index(hw_data); + adf_gen2_init_vf_pfvf_ops(&hw_data->pfvf_ops); + adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); + adf_gen2_init_dc_ops(&hw_data->dc_ops); +} + +void adf_clean_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class->instances--; + adf_devmgr_update_class_index(hw_data); +} diff --git a/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h new file mode 100644 index 000000000..6b4bf181d --- /dev/null +++ b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2015 - 2020 Intel Corporation */ +#ifndef ADF_C3XXXVF_HW_DATA_H_ +#define ADF_C3XXXVF_HW_DATA_H_ + +#define ADF_C3XXXIOV_PMISC_BAR 1 +#define ADF_C3XXXIOV_ACCELERATORS_MASK 0x1 +#define ADF_C3XXXIOV_ACCELENGINES_MASK 0x1 +#define ADF_C3XXXIOV_MAX_ACCELERATORS 1 +#define ADF_C3XXXIOV_MAX_ACCELENGINES 1 +#define ADF_C3XXXIOV_RX_RINGS_OFFSET 8 +#define ADF_C3XXXIOV_TX_RINGS_MASK 0xFF +#define ADF_C3XXXIOV_ETR_BAR 0 +#define ADF_C3XXXIOV_ETR_MAX_BANKS 1 + +void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data); +void adf_clean_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data); +#endif diff --git a/drivers/crypto/intel/qat/qat_c3xxxvf/adf_drv.c b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_drv.c new file mode 100644 index 000000000..1c77f0a18 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_drv.c @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "adf_c3xxxvf_hw_data.h" + +static const struct pci_device_id adf_pci_tbl[] = { + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C3XXX_VF), }, + { } +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); +static void adf_remove(struct pci_dev *dev); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_C3XXXVF_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, +}; + +static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) +{ + pci_release_regions(accel_dev->accel_pci_dev.pci_dev); + pci_disable_device(accel_dev->accel_pci_dev.pci_dev); +} + +static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; + struct adf_accel_dev *pf; + int i; + + for (i = 0; i < ADF_PCI_MAX_BARS; i++) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; + + if (bar->virt_addr) + pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr); + } + + if (accel_dev->hw_device) { + switch (accel_pci_dev->pci_dev->device) { + case PCI_DEVICE_ID_INTEL_QAT_C3XXX_VF: + adf_clean_hw_data_c3xxxiov(accel_dev->hw_device); + break; + default: + break; + } + kfree(accel_dev->hw_device); + accel_dev->hw_device = NULL; + } + adf_dbgfs_exit(accel_dev); + adf_cfg_dev_remove(accel_dev); + pf = adf_devmgr_pci_to_accel_dev(accel_pci_dev->pci_dev->physfn); + adf_devmgr_rm_dev(accel_dev, pf); +} + +static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct adf_accel_dev *accel_dev; + struct adf_accel_dev *pf; + struct adf_accel_pci *accel_pci_dev; + struct adf_hw_device_data *hw_data; + unsigned int i, bar_nr; + unsigned long bar_mask; + int ret; + + switch (ent->device) { + case PCI_DEVICE_ID_INTEL_QAT_C3XXX_VF: + break; + default: + dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); + return -ENODEV; + } + + accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!accel_dev) + return -ENOMEM; + + accel_dev->is_vf = true; + pf = adf_devmgr_pci_to_accel_dev(pdev->physfn); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = pdev; + + /* Add accel device to accel table */ + if (adf_devmgr_add_dev(accel_dev, pf)) { + dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); + kfree(accel_dev); + return -EFAULT; + } + INIT_LIST_HEAD(&accel_dev->crypto_list); + + accel_dev->owner = THIS_MODULE; + /* Allocate and configure device configuration structure */ + hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!hw_data) { + ret = -ENOMEM; + goto out_err; + } + accel_dev->hw_device = hw_data; + adf_init_hw_data_c3xxxiov(accel_dev->hw_device); + + /* Get Accelerators and Accelerators Engines masks */ + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); + accel_pci_dev->sku = hw_data->get_sku(hw_data); + + /* Create device configuration table */ + ret = adf_cfg_dev_add(accel_dev); + if (ret) + goto out_err; + + /* enable PCI device */ + if (pci_enable_device(pdev)) { + ret = -EFAULT; + goto out_err; + } + + /* set dma identifier */ + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); + if (ret) { + dev_err(&pdev->dev, "No usable DMA configuration\n"); + goto out_err_disable; + } + + if (pci_request_regions(pdev, ADF_C3XXXVF_DEVICE_NAME)) { + ret = -EFAULT; + goto out_err_disable; + } + + /* Find and map all the device's BARS */ + i = 0; + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); + for_each_set_bit(bar_nr, &bar_mask, ADF_PCI_MAX_BARS * 2) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; + + bar->base_addr = pci_resource_start(pdev, bar_nr); + if (!bar->base_addr) + break; + bar->size = pci_resource_len(pdev, bar_nr); + bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); + if (!bar->virt_addr) { + pci_err(pdev, "Failed to map BAR %d\n", bar_nr); + ret = -EFAULT; + goto out_err_free_reg; + } + } + /* Completion for VF2PF request/response message exchange */ + init_completion(&accel_dev->vf.msg_received); + + adf_dbgfs_init(accel_dev); + + ret = adf_dev_up(accel_dev, false); + if (ret) + goto out_err_dev_stop; + + return ret; + +out_err_dev_stop: + adf_dev_down(accel_dev); +out_err_free_reg: + pci_release_regions(accel_pci_dev->pci_dev); +out_err_disable: + pci_disable_device(accel_pci_dev->pci_dev); +out_err: + adf_cleanup_accel(accel_dev); + kfree(accel_dev); + return ret; +} + +static void adf_remove(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + if (!accel_dev) { + pr_err("QAT: Driver removal failed\n"); + return; + } + adf_flush_vf_wq(accel_dev); + adf_dev_down(accel_dev); + adf_cleanup_accel(accel_dev); + adf_cleanup_pci_dev(accel_dev); + kfree(accel_dev); +} + +static int __init adfdrv_init(void) +{ + request_module("intel_qat"); + + if (pci_register_driver(&adf_driver)) { + pr_err("QAT: Driver initialization failed\n"); + return -EFAULT; + } + return 0; +} + +static void __exit adfdrv_release(void) +{ + pci_unregister_driver(&adf_driver); + adf_clean_vf_map(true); +} + +module_init(adfdrv_init); +module_exit(adfdrv_release); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Intel"); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); +MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/intel/qat/qat_c62x/Makefile b/drivers/crypto/intel/qat/qat_c62x/Makefile new file mode 100644 index 000000000..f3d722bef --- /dev/null +++ b/drivers/crypto/intel/qat/qat_c62x/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_QAT_C62X) += qat_c62x.o +qat_c62x-y := adf_drv.o adf_c62x_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c new file mode 100644 index 000000000..0b410b414 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2021 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "adf_c62x_hw_data.h" +#include "adf_heartbeat.h" +#include "icp_qat_hw.h" + +/* Worker thread to service arbiter mappings */ +static const u32 thrd_to_arb_map[ADF_C62X_MAX_ACCELENGINES] = { + 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA, + 0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA +}; + +static struct adf_hw_device_class c62x_class = { + .name = ADF_C62X_DEVICE_NAME, + .type = DEV_C62X, +}; + +static u32 get_accel_mask(struct adf_hw_device_data *self) +{ + u32 fuses = self->fuses[ADF_FUSECTL0]; + u32 straps = self->straps; + u32 accel; + + accel = ~(fuses | straps) >> ADF_C62X_ACCELERATORS_REG_OFFSET; + accel &= ADF_C62X_ACCELERATORS_MASK; + + return accel; +} + +static u32 get_ae_mask(struct adf_hw_device_data *self) +{ + u32 fuses = self->fuses[ADF_FUSECTL0]; + u32 straps = self->straps; + unsigned long disabled; + u32 ae_disable; + int accel; + + /* If an accel is disabled, then disable the corresponding two AEs */ + disabled = ~get_accel_mask(self) & ADF_C62X_ACCELERATORS_MASK; + ae_disable = BIT(1) | BIT(0); + for_each_set_bit(accel, &disabled, ADF_C62X_MAX_ACCELERATORS) + straps |= ae_disable << (accel << 1); + + return ~(fuses | straps) & ADF_C62X_ACCELENGINES_MASK; +} + +static u32 get_ts_clock(struct adf_hw_device_data *self) +{ + /* + * Timestamp update interval is 16 AE clock ticks for c62x. + */ + return self->clock_frequency / 16; +} + +static int measure_clock(struct adf_accel_dev *accel_dev) +{ + u32 frequency; + int ret; + + ret = adf_dev_measure_clock(accel_dev, &frequency, ADF_C62X_MIN_AE_FREQ, + ADF_C62X_MAX_AE_FREQ); + if (ret) + return ret; + + accel_dev->hw_device->clock_frequency = frequency; + return 0; +} + +static u32 get_misc_bar_id(struct adf_hw_device_data *self) +{ + return ADF_C62X_PMISC_BAR; +} + +static u32 get_etr_bar_id(struct adf_hw_device_data *self) +{ + return ADF_C62X_ETR_BAR; +} + +static u32 get_sram_bar_id(struct adf_hw_device_data *self) +{ + return ADF_C62X_SRAM_BAR; +} + +static enum dev_sku_info get_sku(struct adf_hw_device_data *self) +{ + int aes = self->get_num_aes(self); + + if (aes == 8) + return DEV_SKU_2; + else if (aes == 10) + return DEV_SKU_4; + + return DEV_SKU_UNKNOWN; +} + +static const u32 *adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev) +{ + return thrd_to_arb_map; +} + +static void configure_iov_threads(struct adf_accel_dev *accel_dev, bool enable) +{ + adf_gen2_cfg_iov_thds(accel_dev, enable, + ADF_C62X_AE2FUNC_MAP_GRP_A_NUM_REGS, + ADF_C62X_AE2FUNC_MAP_GRP_B_NUM_REGS); +} + +void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class = &c62x_class; + hw_data->instance_id = c62x_class.instances++; + hw_data->num_banks = ADF_C62X_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; + hw_data->num_accel = ADF_C62X_MAX_ACCELERATORS; + hw_data->num_logical_accel = 1; + hw_data->num_engines = ADF_C62X_MAX_ACCELENGINES; + hw_data->tx_rx_gap = ADF_GEN2_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_GEN2_TX_RINGS_MASK; + hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP; + hw_data->alloc_irq = adf_isr_resource_alloc; + hw_data->free_irq = adf_isr_resource_free; + hw_data->enable_error_correction = adf_gen2_enable_error_correction; + hw_data->get_accel_mask = get_accel_mask; + hw_data->get_ae_mask = get_ae_mask; + hw_data->get_accel_cap = adf_gen2_get_accel_cap; + hw_data->get_num_accels = adf_gen2_get_num_accels; + hw_data->get_num_aes = adf_gen2_get_num_aes; + hw_data->get_sram_bar_id = get_sram_bar_id; + hw_data->get_etr_bar_id = get_etr_bar_id; + hw_data->get_misc_bar_id = get_misc_bar_id; + hw_data->get_admin_info = adf_gen2_get_admin_info; + hw_data->get_arb_info = adf_gen2_get_arb_info; + hw_data->get_sku = get_sku; + hw_data->fw_name = ADF_C62X_FW; + hw_data->fw_mmp_name = ADF_C62X_MMP; + hw_data->init_admin_comms = adf_init_admin_comms; + hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->configure_iov_threads = configure_iov_threads; + hw_data->send_admin_init = adf_send_admin_init; + hw_data->init_arb = adf_init_arb; + hw_data->exit_arb = adf_exit_arb; + hw_data->get_arb_mapping = adf_get_arbiter_mapping; + hw_data->enable_ints = adf_gen2_enable_ints; + hw_data->reset_device = adf_reset_flr; + hw_data->set_ssm_wdtimer = adf_gen2_set_ssm_wdtimer; + hw_data->disable_iov = adf_disable_sriov; + hw_data->dev_config = adf_gen2_dev_config; + hw_data->measure_clock = measure_clock; + hw_data->get_hb_clock = get_ts_clock; + hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE; + hw_data->check_hb_ctrs = adf_heartbeat_check_ctrs; + + adf_gen2_init_pf_pfvf_ops(&hw_data->pfvf_ops); + adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); + adf_gen2_init_dc_ops(&hw_data->dc_ops); +} + +void adf_clean_hw_data_c62x(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class->instances--; +} diff --git a/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.h b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.h new file mode 100644 index 000000000..13e6ebf6f --- /dev/null +++ b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#ifndef ADF_C62X_HW_DATA_H_ +#define ADF_C62X_HW_DATA_H_ + +#include + +/* PCIe configuration space */ +#define ADF_C62X_SRAM_BAR 0 +#define ADF_C62X_PMISC_BAR 1 +#define ADF_C62X_ETR_BAR 2 +#define ADF_C62X_MAX_ACCELERATORS 5 +#define ADF_C62X_MAX_ACCELENGINES 10 +#define ADF_C62X_ACCELERATORS_REG_OFFSET 16 +#define ADF_C62X_ACCELERATORS_MASK 0x1F +#define ADF_C62X_ACCELENGINES_MASK 0x3FF +#define ADF_C62X_ETR_MAX_BANKS 16 +#define ADF_C62X_SOFTSTRAP_CSR_OFFSET 0x2EC + +/* AE to function mapping */ +#define ADF_C62X_AE2FUNC_MAP_GRP_A_NUM_REGS 80 +#define ADF_C62X_AE2FUNC_MAP_GRP_B_NUM_REGS 10 + +/* Clocks frequency */ +#define ADF_C62X_AE_FREQ (685 * HZ_PER_MHZ) +#define ADF_C62X_MIN_AE_FREQ (533 * HZ_PER_MHZ) +#define ADF_C62X_MAX_AE_FREQ (800 * HZ_PER_MHZ) + +/* Firmware Binary */ +#define ADF_C62X_FW "qat_c62x.bin" +#define ADF_C62X_MMP "qat_c62x_mmp.bin" + +void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data); +void adf_clean_hw_data_c62x(struct adf_hw_device_data *hw_data); +#endif diff --git a/drivers/crypto/intel/qat/qat_c62x/adf_drv.c b/drivers/crypto/intel/qat/qat_c62x/adf_drv.c new file mode 100644 index 000000000..f48f3b437 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_c62x/adf_drv.c @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "adf_c62x_hw_data.h" + +static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) +{ + pci_release_regions(accel_dev->accel_pci_dev.pci_dev); + pci_disable_device(accel_dev->accel_pci_dev.pci_dev); +} + +static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; + int i; + + for (i = 0; i < ADF_PCI_MAX_BARS; i++) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; + + if (bar->virt_addr) + pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr); + } + + if (accel_dev->hw_device) { + switch (accel_pci_dev->pci_dev->device) { + case PCI_DEVICE_ID_INTEL_QAT_C62X: + adf_clean_hw_data_c62x(accel_dev->hw_device); + break; + default: + break; + } + kfree(accel_dev->hw_device); + accel_dev->hw_device = NULL; + } + adf_dbgfs_exit(accel_dev); + adf_cfg_dev_remove(accel_dev); + adf_devmgr_rm_dev(accel_dev, NULL); +} + +static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct adf_accel_dev *accel_dev; + struct adf_accel_pci *accel_pci_dev; + struct adf_hw_device_data *hw_data; + unsigned int i, bar_nr; + unsigned long bar_mask; + int ret; + + switch (ent->device) { + case PCI_DEVICE_ID_INTEL_QAT_C62X: + break; + default: + dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); + return -ENODEV; + } + + if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) { + /* If the accelerator is connected to a node with no memory + * there is no point in using the accelerator since the remote + * memory transaction will be very slow. */ + dev_err(&pdev->dev, "Invalid NUMA configuration.\n"); + return -EINVAL; + } + + accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!accel_dev) + return -ENOMEM; + + INIT_LIST_HEAD(&accel_dev->crypto_list); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = pdev; + + /* Add accel device to accel table. + * This should be called before adf_cleanup_accel is called */ + if (adf_devmgr_add_dev(accel_dev, NULL)) { + dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); + kfree(accel_dev); + return -EFAULT; + } + + accel_dev->owner = THIS_MODULE; + /* Allocate and configure device configuration structure */ + hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!hw_data) { + ret = -ENOMEM; + goto out_err; + } + + accel_dev->hw_device = hw_data; + adf_init_hw_data_c62x(accel_dev->hw_device); + pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); + pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, + &hw_data->fuses[ADF_FUSECTL0]); + pci_read_config_dword(pdev, ADF_C62X_SOFTSTRAP_CSR_OFFSET, + &hw_data->straps); + + /* Get Accelerators and Accelerators Engines masks */ + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); + accel_pci_dev->sku = hw_data->get_sku(hw_data); + /* If the device has no acceleration engines then ignore it. */ + if (!hw_data->accel_mask || !hw_data->ae_mask || + ((~hw_data->ae_mask) & 0x01)) { + dev_err(&pdev->dev, "No acceleration units found"); + ret = -EFAULT; + goto out_err; + } + + /* Create device configuration table */ + ret = adf_cfg_dev_add(accel_dev); + if (ret) + goto out_err; + + /* enable PCI device */ + if (pci_enable_device(pdev)) { + ret = -EFAULT; + goto out_err; + } + + /* set dma identifier */ + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); + if (ret) { + dev_err(&pdev->dev, "No usable DMA configuration\n"); + goto out_err_disable; + } + + if (pci_request_regions(pdev, ADF_C62X_DEVICE_NAME)) { + ret = -EFAULT; + goto out_err_disable; + } + + /* Get accelerator capabilities mask */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); + + /* Find and map all the device's BARS */ + i = (hw_data->fuses[ADF_FUSECTL0] & ADF_DEVICE_FUSECTL_MASK) ? 1 : 0; + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); + for_each_set_bit(bar_nr, &bar_mask, ADF_PCI_MAX_BARS * 2) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; + + bar->base_addr = pci_resource_start(pdev, bar_nr); + if (!bar->base_addr) + break; + bar->size = pci_resource_len(pdev, bar_nr); + bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); + if (!bar->virt_addr) { + pci_err(pdev, "Failed to map BAR %d\n", bar_nr); + ret = -EFAULT; + goto out_err_free_reg; + } + } + + if (pci_save_state(pdev)) { + pci_err(pdev, "Failed to save pci state\n"); + ret = -ENOMEM; + goto out_err_free_reg; + } + + adf_dbgfs_init(accel_dev); + + ret = adf_dev_up(accel_dev, true); + if (ret) + goto out_err_dev_stop; + + return ret; + +out_err_dev_stop: + adf_dev_down(accel_dev); +out_err_free_reg: + pci_release_regions(accel_pci_dev->pci_dev); +out_err_disable: + pci_disable_device(accel_pci_dev->pci_dev); +out_err: + adf_cleanup_accel(accel_dev); + kfree(accel_dev); + return ret; +} + +static void adf_remove(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + if (!accel_dev) { + pr_err("QAT: Driver removal failed\n"); + return; + } + adf_dev_down(accel_dev); + adf_cleanup_accel(accel_dev); + adf_cleanup_pci_dev(accel_dev); + kfree(accel_dev); +} + +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + +static const struct pci_device_id adf_pci_tbl[] = { + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C62X) }, + { } +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_C62X_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, + .shutdown = adf_shutdown, + .sriov_configure = adf_sriov_configure, + .err_handler = &adf_err_handler, +}; + +static int __init adfdrv_init(void) +{ + request_module("intel_qat"); + + if (pci_register_driver(&adf_driver)) { + pr_err("QAT: Driver initialization failed\n"); + return -EFAULT; + } + return 0; +} + +static void __exit adfdrv_release(void) +{ + pci_unregister_driver(&adf_driver); +} + +module_init(adfdrv_init); +module_exit(adfdrv_release); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Intel"); +MODULE_FIRMWARE(ADF_C62X_FW); +MODULE_FIRMWARE(ADF_C62X_MMP); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); +MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/intel/qat/qat_c62xvf/Makefile b/drivers/crypto/intel/qat/qat_c62xvf/Makefile new file mode 100644 index 000000000..ed7f3f722 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_c62xvf/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_QAT_C62XVF) += qat_c62xvf.o +qat_c62xvf-y := adf_drv.o adf_c62xvf_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c b/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c new file mode 100644 index 000000000..7f00035d3 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2015 - 2021 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include +#include "adf_c62xvf_hw_data.h" + +static struct adf_hw_device_class c62xiov_class = { + .name = ADF_C62XVF_DEVICE_NAME, + .type = DEV_C62XVF, +}; + +static u32 get_accel_mask(struct adf_hw_device_data *self) +{ + return ADF_C62XIOV_ACCELERATORS_MASK; +} + +static u32 get_ae_mask(struct adf_hw_device_data *self) +{ + return ADF_C62XIOV_ACCELENGINES_MASK; +} + +static u32 get_num_accels(struct adf_hw_device_data *self) +{ + return ADF_C62XIOV_MAX_ACCELERATORS; +} + +static u32 get_num_aes(struct adf_hw_device_data *self) +{ + return ADF_C62XIOV_MAX_ACCELENGINES; +} + +static u32 get_misc_bar_id(struct adf_hw_device_data *self) +{ + return ADF_C62XIOV_PMISC_BAR; +} + +static u32 get_etr_bar_id(struct adf_hw_device_data *self) +{ + return ADF_C62XIOV_ETR_BAR; +} + +static enum dev_sku_info get_sku(struct adf_hw_device_data *self) +{ + return DEV_SKU_VF; +} + +static int adf_vf_int_noop(struct adf_accel_dev *accel_dev) +{ + return 0; +} + +static void adf_vf_void_noop(struct adf_accel_dev *accel_dev) +{ +} + +void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class = &c62xiov_class; + hw_data->num_banks = ADF_C62XIOV_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; + hw_data->num_accel = ADF_C62XIOV_MAX_ACCELERATORS; + hw_data->num_logical_accel = 1; + hw_data->num_engines = ADF_C62XIOV_MAX_ACCELENGINES; + hw_data->tx_rx_gap = ADF_C62XIOV_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_C62XIOV_TX_RINGS_MASK; + hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP; + hw_data->alloc_irq = adf_vf_isr_resource_alloc; + hw_data->free_irq = adf_vf_isr_resource_free; + hw_data->enable_error_correction = adf_vf_void_noop; + hw_data->init_admin_comms = adf_vf_int_noop; + hw_data->exit_admin_comms = adf_vf_void_noop; + hw_data->send_admin_init = adf_vf2pf_notify_init; + hw_data->init_arb = adf_vf_int_noop; + hw_data->exit_arb = adf_vf_void_noop; + hw_data->disable_iov = adf_vf2pf_notify_shutdown; + hw_data->get_accel_mask = get_accel_mask; + hw_data->get_ae_mask = get_ae_mask; + hw_data->get_num_accels = get_num_accels; + hw_data->get_num_aes = get_num_aes; + hw_data->get_etr_bar_id = get_etr_bar_id; + hw_data->get_misc_bar_id = get_misc_bar_id; + hw_data->get_sku = get_sku; + hw_data->enable_ints = adf_vf_void_noop; + hw_data->dev_class->instances++; + hw_data->dev_config = adf_gen2_dev_config; + adf_devmgr_update_class_index(hw_data); + adf_gen2_init_vf_pfvf_ops(&hw_data->pfvf_ops); + adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); + adf_gen2_init_dc_ops(&hw_data->dc_ops); +} + +void adf_clean_hw_data_c62xiov(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class->instances--; + adf_devmgr_update_class_index(hw_data); +} diff --git a/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.h b/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.h new file mode 100644 index 000000000..a1a62c003 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2015 - 2020 Intel Corporation */ +#ifndef ADF_C62XVF_HW_DATA_H_ +#define ADF_C62XVF_HW_DATA_H_ + +#define ADF_C62XIOV_PMISC_BAR 1 +#define ADF_C62XIOV_ACCELERATORS_MASK 0x1 +#define ADF_C62XIOV_ACCELENGINES_MASK 0x1 +#define ADF_C62XIOV_MAX_ACCELERATORS 1 +#define ADF_C62XIOV_MAX_ACCELENGINES 1 +#define ADF_C62XIOV_RX_RINGS_OFFSET 8 +#define ADF_C62XIOV_TX_RINGS_MASK 0xFF +#define ADF_C62XIOV_ETR_BAR 0 +#define ADF_C62XIOV_ETR_MAX_BANKS 1 + +void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data); +void adf_clean_hw_data_c62xiov(struct adf_hw_device_data *hw_data); +#endif diff --git a/drivers/crypto/intel/qat/qat_c62xvf/adf_drv.c b/drivers/crypto/intel/qat/qat_c62xvf/adf_drv.c new file mode 100644 index 000000000..b96f19e31 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_c62xvf/adf_drv.c @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "adf_c62xvf_hw_data.h" + +static const struct pci_device_id adf_pci_tbl[] = { + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C62X_VF), }, + { } +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); +static void adf_remove(struct pci_dev *dev); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_C62XVF_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, +}; + +static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) +{ + pci_release_regions(accel_dev->accel_pci_dev.pci_dev); + pci_disable_device(accel_dev->accel_pci_dev.pci_dev); +} + +static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; + struct adf_accel_dev *pf; + int i; + + for (i = 0; i < ADF_PCI_MAX_BARS; i++) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; + + if (bar->virt_addr) + pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr); + } + + if (accel_dev->hw_device) { + switch (accel_pci_dev->pci_dev->device) { + case PCI_DEVICE_ID_INTEL_QAT_C62X_VF: + adf_clean_hw_data_c62xiov(accel_dev->hw_device); + break; + default: + break; + } + kfree(accel_dev->hw_device); + accel_dev->hw_device = NULL; + } + adf_dbgfs_exit(accel_dev); + adf_cfg_dev_remove(accel_dev); + pf = adf_devmgr_pci_to_accel_dev(accel_pci_dev->pci_dev->physfn); + adf_devmgr_rm_dev(accel_dev, pf); +} + +static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct adf_accel_dev *accel_dev; + struct adf_accel_dev *pf; + struct adf_accel_pci *accel_pci_dev; + struct adf_hw_device_data *hw_data; + unsigned int i, bar_nr; + unsigned long bar_mask; + int ret; + + switch (ent->device) { + case PCI_DEVICE_ID_INTEL_QAT_C62X_VF: + break; + default: + dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); + return -ENODEV; + } + + accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!accel_dev) + return -ENOMEM; + + accel_dev->is_vf = true; + pf = adf_devmgr_pci_to_accel_dev(pdev->physfn); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = pdev; + + /* Add accel device to accel table */ + if (adf_devmgr_add_dev(accel_dev, pf)) { + dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); + kfree(accel_dev); + return -EFAULT; + } + INIT_LIST_HEAD(&accel_dev->crypto_list); + + accel_dev->owner = THIS_MODULE; + /* Allocate and configure device configuration structure */ + hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!hw_data) { + ret = -ENOMEM; + goto out_err; + } + accel_dev->hw_device = hw_data; + adf_init_hw_data_c62xiov(accel_dev->hw_device); + + /* Get Accelerators and Accelerators Engines masks */ + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); + accel_pci_dev->sku = hw_data->get_sku(hw_data); + + /* Create device configuration table */ + ret = adf_cfg_dev_add(accel_dev); + if (ret) + goto out_err; + + /* enable PCI device */ + if (pci_enable_device(pdev)) { + ret = -EFAULT; + goto out_err; + } + + /* set dma identifier */ + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); + if (ret) { + dev_err(&pdev->dev, "No usable DMA configuration\n"); + goto out_err_disable; + } + + if (pci_request_regions(pdev, ADF_C62XVF_DEVICE_NAME)) { + ret = -EFAULT; + goto out_err_disable; + } + + /* Find and map all the device's BARS */ + i = 0; + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); + for_each_set_bit(bar_nr, &bar_mask, ADF_PCI_MAX_BARS * 2) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; + + bar->base_addr = pci_resource_start(pdev, bar_nr); + if (!bar->base_addr) + break; + bar->size = pci_resource_len(pdev, bar_nr); + bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); + if (!bar->virt_addr) { + pci_err(pdev, "Failed to map BAR %d\n", bar_nr); + ret = -EFAULT; + goto out_err_free_reg; + } + } + /* Completion for VF2PF request/response message exchange */ + init_completion(&accel_dev->vf.msg_received); + + adf_dbgfs_init(accel_dev); + + ret = adf_dev_up(accel_dev, false); + if (ret) + goto out_err_dev_stop; + + return ret; + +out_err_dev_stop: + adf_dev_down(accel_dev); +out_err_free_reg: + pci_release_regions(accel_pci_dev->pci_dev); +out_err_disable: + pci_disable_device(accel_pci_dev->pci_dev); +out_err: + adf_cleanup_accel(accel_dev); + kfree(accel_dev); + return ret; +} + +static void adf_remove(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + if (!accel_dev) { + pr_err("QAT: Driver removal failed\n"); + return; + } + adf_flush_vf_wq(accel_dev); + adf_dev_down(accel_dev); + adf_cleanup_accel(accel_dev); + adf_cleanup_pci_dev(accel_dev); + kfree(accel_dev); +} + +static int __init adfdrv_init(void) +{ + request_module("intel_qat"); + + if (pci_register_driver(&adf_driver)) { + pr_err("QAT: Driver initialization failed\n"); + return -EFAULT; + } + return 0; +} + +static void __exit adfdrv_release(void) +{ + pci_unregister_driver(&adf_driver); + adf_clean_vf_map(true); +} + +module_init(adfdrv_init); +module_exit(adfdrv_release); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Intel"); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); +MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/intel/qat/qat_common/Makefile b/drivers/crypto/intel/qat/qat_common/Makefile new file mode 100644 index 000000000..306ec71bb --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/Makefile @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_CRYPTO_DEV_QAT) += intel_qat.o +ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE='"CRYPTO_QAT"' +intel_qat-y := adf_accel_engine.o \ + adf_admin.o \ + adf_aer.o \ + adf_anti_rb.o \ + adf_bank_state.o \ + adf_cfg.o \ + adf_cfg_services.o \ + adf_clock.o \ + adf_dc.o \ + adf_dev_mgr.o \ + adf_gen2_config.o \ + adf_gen2_hw_csr_data.o \ + adf_gen2_hw_data.o \ + adf_gen4_config.o \ + adf_gen4_hw_csr_data.o \ + adf_gen4_hw_data.o \ + adf_gen4_pm.o \ + adf_gen4_ras.o \ + adf_gen4_vf_mig.o \ + adf_gen6_ras.o \ + adf_gen6_shared.o \ + adf_hw_arbiter.o \ + adf_init.o \ + adf_isr.o \ + adf_kpt.o \ + adf_module.o \ + adf_mstate_mgr.o \ + adf_rl_admin.o \ + adf_rl.o \ + adf_sysfs.o \ + adf_sysfs_anti_rb.o \ + adf_sysfs_kpt.o \ + adf_sysfs_ras_counters.o \ + adf_sysfs_rl.o \ + adf_timer.o \ + adf_transport.o \ + qat_algs.o \ + qat_algs_send.o \ + qat_asym_algs.o \ + qat_bl.o \ + qat_comp_algs.o \ + qat_compression.o \ + qat_comp_zstd_utils.o \ + qat_crypto.o \ + qat_hal.o \ + qat_mig_dev.o \ + qat_uclo.o + +intel_qat-$(CONFIG_DEBUG_FS) += adf_cnv_dbgfs.o \ + adf_dbgfs.o \ + adf_fw_counters.o \ + adf_gen4_pm_debugfs.o \ + adf_gen4_tl.o \ + adf_gen6_pm_dbgfs.o \ + adf_gen6_tl.o \ + adf_heartbeat_dbgfs.o \ + adf_heartbeat.o \ + adf_pm_dbgfs.o \ + adf_pm_dbgfs_utils.o \ + adf_telemetry.o \ + adf_tl_debugfs.o \ + adf_transport_debug.o + +intel_qat-$(CONFIG_PCI_IOV) += adf_gen2_pfvf.o adf_gen4_pfvf.o \ + adf_pfvf_pf_msg.o adf_pfvf_pf_proto.o \ + adf_pfvf_utils.o adf_pfvf_vf_msg.o \ + adf_pfvf_vf_proto.o adf_sriov.o adf_vf_isr.o + +intel_qat-$(CONFIG_CRYPTO_DEV_QAT_ERROR_INJECTION) += adf_heartbeat_inject.o diff --git a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h new file mode 100644 index 000000000..a84380125 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h @@ -0,0 +1,501 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#ifndef ADF_ACCEL_DEVICES_H_ +#define ADF_ACCEL_DEVICES_H_ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "adf_anti_rb.h" +#include "adf_cfg_common.h" +#include "adf_dc.h" +#include "adf_kpt.h" +#include "adf_rl.h" +#include "adf_telemetry.h" +#include "adf_pfvf_msg.h" +#include "icp_qat_hw.h" + +#define ADF_DH895XCC_DEVICE_NAME "dh895xcc" +#define ADF_DH895XCCVF_DEVICE_NAME "dh895xccvf" +#define ADF_C62X_DEVICE_NAME "c6xx" +#define ADF_C62XVF_DEVICE_NAME "c6xxvf" +#define ADF_C3XXX_DEVICE_NAME "c3xxx" +#define ADF_C3XXXVF_DEVICE_NAME "c3xxxvf" +#define ADF_4XXX_DEVICE_NAME "4xxx" +#define ADF_420XX_DEVICE_NAME "420xx" +#define ADF_6XXX_DEVICE_NAME "6xxx" +#define PCI_DEVICE_ID_INTEL_QAT_4XXXIOV 0x4941 +#define PCI_DEVICE_ID_INTEL_QAT_401XXIOV 0x4943 +#define PCI_DEVICE_ID_INTEL_QAT_402XXIOV 0x4945 +#define PCI_DEVICE_ID_INTEL_QAT_420XXIOV 0x4947 +#define PCI_DEVICE_ID_INTEL_QAT_6XXX_IOV 0x4949 + +#define ADF_DEVICE_FUSECTL_OFFSET 0x40 +#define ADF_DEVICE_LEGFUSE_OFFSET 0x4C +#define ADF_DEVICE_FUSECTL_MASK 0x80000000 +#define ADF_PCI_MAX_BARS 3 +#define ADF_DEVICE_NAME_LENGTH 32 +#define ADF_ETR_MAX_RINGS_PER_BANK 16 +#define ADF_MAX_MSIX_VECTOR_NAME 48 +#define ADF_DEVICE_NAME_PREFIX "qat_" + +enum adf_accel_capabilities { + ADF_ACCEL_CAPABILITIES_NULL = 0, + ADF_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC = 1, + ADF_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC = 2, + ADF_ACCEL_CAPABILITIES_CIPHER = 4, + ADF_ACCEL_CAPABILITIES_AUTHENTICATION = 8, + ADF_ACCEL_CAPABILITIES_COMPRESSION = 32, + ADF_ACCEL_CAPABILITIES_LZS_COMPRESSION = 64, + ADF_ACCEL_CAPABILITIES_RANDOM_NUMBER = 128 +}; + +enum adf_accel_capabilities_ext { + ADF_ACCEL_CAPABILITIES_EXT_ZSTD_LZ4S = BIT(0), + ADF_ACCEL_CAPABILITIES_EXT_ZSTD = BIT(1), +}; + +enum adf_fuses { + ADF_FUSECTL0, + ADF_FUSECTL1, + ADF_FUSECTL2, + ADF_FUSECTL3, + ADF_FUSECTL4, + ADF_FUSECTL5, + ADF_MAX_FUSES +}; + +struct adf_bar { + resource_size_t base_addr; + void __iomem *virt_addr; + resource_size_t size; +}; + +struct adf_irq { + bool enabled; + char name[ADF_MAX_MSIX_VECTOR_NAME]; +}; + +struct adf_accel_msix { + struct adf_irq *irqs; + u32 num_entries; +}; + +struct adf_accel_pci { + struct pci_dev *pci_dev; + struct adf_accel_msix msix_entries; + struct adf_bar pci_bars[ADF_PCI_MAX_BARS]; + u8 revid; + u8 sku; +}; + +enum dev_state { + DEV_DOWN = 0, + DEV_UP +}; + +enum dev_sku_info { + DEV_SKU_1 = 0, + DEV_SKU_2, + DEV_SKU_3, + DEV_SKU_4, + DEV_SKU_VF, + DEV_SKU_UNKNOWN, +}; + +enum ras_errors { + ADF_RAS_CORR, + ADF_RAS_UNCORR, + ADF_RAS_FATAL, + ADF_RAS_ERRORS, +}; + +struct adf_error_counters { + atomic_t counter[ADF_RAS_ERRORS]; + bool sysfs_added; + bool enabled; +}; + +static inline const char *get_sku_info(enum dev_sku_info info) +{ + switch (info) { + case DEV_SKU_1: + return "SKU1"; + case DEV_SKU_2: + return "SKU2"; + case DEV_SKU_3: + return "SKU3"; + case DEV_SKU_4: + return "SKU4"; + case DEV_SKU_VF: + return "SKUVF"; + case DEV_SKU_UNKNOWN: + default: + break; + } + return "Unknown SKU"; +} + +struct adf_hw_device_class { + const char *name; + const enum adf_device_type type; + u32 instances; +}; + +struct arb_info { + u32 arb_cfg; + u32 arb_offset; + u32 wt2sam_offset; +}; + +struct admin_info { + u32 admin_msg_ur; + u32 admin_msg_lr; + u32 mailbox_offset; +}; + +struct adf_bank_state; + +struct adf_hw_csr_ops { + u64 (*build_csr_ring_base_addr)(dma_addr_t addr, u32 size); + u32 (*read_csr_ring_head)(void __iomem *csr_base_addr, u32 bank, + u32 ring); + void (*write_csr_ring_head)(void __iomem *csr_base_addr, u32 bank, + u32 ring, u32 value); + u32 (*read_csr_ring_tail)(void __iomem *csr_base_addr, u32 bank, + u32 ring); + void (*write_csr_ring_tail)(void __iomem *csr_base_addr, u32 bank, + u32 ring, u32 value); + u32 (*read_csr_stat)(void __iomem *csr_base_addr, u32 bank); + u32 (*read_csr_uo_stat)(void __iomem *csr_base_addr, u32 bank); + u32 (*read_csr_e_stat)(void __iomem *csr_base_addr, u32 bank); + u32 (*read_csr_ne_stat)(void __iomem *csr_base_addr, u32 bank); + u32 (*read_csr_nf_stat)(void __iomem *csr_base_addr, u32 bank); + u32 (*read_csr_f_stat)(void __iomem *csr_base_addr, u32 bank); + u32 (*read_csr_c_stat)(void __iomem *csr_base_addr, u32 bank); + u32 (*read_csr_exp_stat)(void __iomem *csr_base_addr, u32 bank); + u32 (*read_csr_exp_int_en)(void __iomem *csr_base_addr, u32 bank); + void (*write_csr_exp_int_en)(void __iomem *csr_base_addr, u32 bank, + u32 value); + u32 (*read_csr_ring_config)(void __iomem *csr_base_addr, u32 bank, + u32 ring); + void (*write_csr_ring_config)(void __iomem *csr_base_addr, u32 bank, + u32 ring, u32 value); + dma_addr_t (*read_csr_ring_base)(void __iomem *csr_base_addr, u32 bank, + u32 ring); + void (*write_csr_ring_base)(void __iomem *csr_base_addr, u32 bank, + u32 ring, dma_addr_t addr); + u32 (*read_csr_int_en)(void __iomem *csr_base_addr, u32 bank); + void (*write_csr_int_en)(void __iomem *csr_base_addr, u32 bank, + u32 value); + u32 (*read_csr_int_flag)(void __iomem *csr_base_addr, u32 bank); + void (*write_csr_int_flag)(void __iomem *csr_base_addr, u32 bank, + u32 value); + u32 (*read_csr_int_srcsel)(void __iomem *csr_base_addr, u32 bank); + void (*write_csr_int_srcsel)(void __iomem *csr_base_addr, u32 bank); + void (*write_csr_int_srcsel_w_val)(void __iomem *csr_base_addr, + u32 bank, u32 value); + u32 (*read_csr_int_col_en)(void __iomem *csr_base_addr, u32 bank); + void (*write_csr_int_col_en)(void __iomem *csr_base_addr, u32 bank, + u32 value); + u32 (*read_csr_int_col_ctl)(void __iomem *csr_base_addr, u32 bank); + void (*write_csr_int_col_ctl)(void __iomem *csr_base_addr, u32 bank, + u32 value); + u32 (*read_csr_int_flag_and_col)(void __iomem *csr_base_addr, + u32 bank); + void (*write_csr_int_flag_and_col)(void __iomem *csr_base_addr, + u32 bank, u32 value); + u32 (*read_csr_ring_srv_arb_en)(void __iomem *csr_base_addr, u32 bank); + void (*write_csr_ring_srv_arb_en)(void __iomem *csr_base_addr, u32 bank, + u32 value); + u32 (*get_int_col_ctl_enable_mask)(void); +}; + +struct adf_cfg_device_data; +struct adf_accel_dev; +struct adf_etr_data; +struct adf_etr_ring_data; + +struct adf_ras_ops { + void (*enable_ras_errors)(struct adf_accel_dev *accel_dev); + void (*disable_ras_errors)(struct adf_accel_dev *accel_dev); + bool (*handle_interrupt)(struct adf_accel_dev *accel_dev, + bool *reset_required); +}; + +struct adf_pfvf_ops { + int (*enable_comms)(struct adf_accel_dev *accel_dev); + u32 (*get_pf2vf_offset)(u32 i); + u32 (*get_vf2pf_offset)(u32 i); + void (*enable_vf2pf_interrupts)(void __iomem *pmisc_addr, u32 vf_mask); + void (*disable_all_vf2pf_interrupts)(void __iomem *pmisc_addr); + u32 (*disable_pending_vf2pf_interrupts)(void __iomem *pmisc_addr); + int (*send_msg)(struct adf_accel_dev *accel_dev, struct pfvf_message msg, + u32 pfvf_offset, struct mutex *csr_lock); + struct pfvf_message (*recv_msg)(struct adf_accel_dev *accel_dev, + u32 pfvf_offset, u8 compat_ver); +}; + +struct adf_dc_ops { + int (*build_comp_block)(void *ctx, enum adf_dc_algo algo); + int (*build_decomp_block)(void *ctx, enum adf_dc_algo algo); +}; + +struct qat_migdev_ops { + int (*init)(struct qat_mig_dev *mdev); + void (*cleanup)(struct qat_mig_dev *mdev); + void (*reset)(struct qat_mig_dev *mdev); + int (*open)(struct qat_mig_dev *mdev); + void (*close)(struct qat_mig_dev *mdev); + int (*suspend)(struct qat_mig_dev *mdev); + int (*resume)(struct qat_mig_dev *mdev); + int (*save_state)(struct qat_mig_dev *mdev); + int (*save_setup)(struct qat_mig_dev *mdev); + int (*load_state)(struct qat_mig_dev *mdev); + int (*load_setup)(struct qat_mig_dev *mdev, int size); +}; + +struct adf_dev_err_mask { + u32 cppagentcmdpar_mask; + u32 parerr_ath_cph_mask; + u32 parerr_cpr_xlt_mask; + u32 parerr_dcpr_ucs_mask; + u32 parerr_pke_mask; + u32 parerr_wat_wcp_mask; + u32 ssmfeatren_mask; +}; + +struct adf_hw_device_data { + struct adf_hw_device_class *dev_class; + u32 (*get_accel_mask)(struct adf_hw_device_data *self); + u32 (*get_ae_mask)(struct adf_hw_device_data *self); + u32 (*get_accel_cap)(struct adf_accel_dev *accel_dev); + u32 (*get_sram_bar_id)(struct adf_hw_device_data *self); + u32 (*get_misc_bar_id)(struct adf_hw_device_data *self); + u32 (*get_etr_bar_id)(struct adf_hw_device_data *self); + u32 (*get_num_aes)(struct adf_hw_device_data *self); + u32 (*get_num_accels)(struct adf_hw_device_data *self); + void (*get_arb_info)(struct arb_info *arb_csrs_info); + void (*get_admin_info)(struct admin_info *admin_csrs_info); + enum dev_sku_info (*get_sku)(struct adf_hw_device_data *self); + u16 (*get_ring_to_svc_map)(struct adf_accel_dev *accel_dev); + int (*alloc_irq)(struct adf_accel_dev *accel_dev); + void (*free_irq)(struct adf_accel_dev *accel_dev); + void (*enable_error_correction)(struct adf_accel_dev *accel_dev); + int (*init_admin_comms)(struct adf_accel_dev *accel_dev); + void (*exit_admin_comms)(struct adf_accel_dev *accel_dev); + int (*send_admin_init)(struct adf_accel_dev *accel_dev); + int (*start_timer)(struct adf_accel_dev *accel_dev); + void (*stop_timer)(struct adf_accel_dev *accel_dev); + void (*check_hb_ctrs)(struct adf_accel_dev *accel_dev); + uint32_t (*get_hb_clock)(struct adf_hw_device_data *self); + int (*measure_clock)(struct adf_accel_dev *accel_dev); + int (*init_arb)(struct adf_accel_dev *accel_dev); + void (*exit_arb)(struct adf_accel_dev *accel_dev); + const u32 *(*get_arb_mapping)(struct adf_accel_dev *accel_dev); + int (*init_device)(struct adf_accel_dev *accel_dev); + int (*enable_pm)(struct adf_accel_dev *accel_dev); + bool (*handle_pm_interrupt)(struct adf_accel_dev *accel_dev); + void (*disable_iov)(struct adf_accel_dev *accel_dev); + void (*configure_iov_threads)(struct adf_accel_dev *accel_dev, + bool enable); + void (*enable_ints)(struct adf_accel_dev *accel_dev); + void (*set_ssm_wdtimer)(struct adf_accel_dev *accel_dev); + int (*ring_pair_reset)(struct adf_accel_dev *accel_dev, u32 bank_nr); + int (*bank_state_save)(struct adf_accel_dev *accel_dev, u32 bank_number, + struct adf_bank_state *state); + int (*bank_state_restore)(struct adf_accel_dev *accel_dev, + u32 bank_number, struct adf_bank_state *state); + void (*reset_device)(struct adf_accel_dev *accel_dev); + void (*set_msix_rttable)(struct adf_accel_dev *accel_dev); + const char *(*uof_get_name)(struct adf_accel_dev *accel_dev, u32 obj_num); + u32 (*uof_get_num_objs)(struct adf_accel_dev *accel_dev); + int (*uof_get_obj_type)(struct adf_accel_dev *accel_dev, u32 obj_num); + u32 (*uof_get_ae_mask)(struct adf_accel_dev *accel_dev, u32 obj_num); + int (*get_rp_group)(struct adf_accel_dev *accel_dev, u32 ae_mask); + u32 (*get_ena_thd_mask)(struct adf_accel_dev *accel_dev, u32 obj_num); + int (*dev_config)(struct adf_accel_dev *accel_dev); + bool (*services_supported)(unsigned long mask); + u32 (*get_svc_slice_cnt)(struct adf_accel_dev *accel_dev, + enum adf_base_services svc); + struct adf_pfvf_ops pfvf_ops; + struct adf_hw_csr_ops csr_ops; + struct adf_dc_ops dc_ops; + struct adf_ras_ops ras_ops; + struct adf_dev_err_mask dev_err_mask; + struct adf_rl_hw_data rl_data; + struct adf_tl_hw_data tl_data; + struct adf_anti_rb_hw_data anti_rb_data; + struct adf_kpt_hw_data kpt_data; + struct qat_migdev_ops vfmig_ops; + const char *fw_name; + const char *fw_mmp_name; + u32 fuses[ADF_MAX_FUSES]; + u32 straps; + u32 accel_capabilities_mask; + u32 accel_capabilities_ext_mask; + u32 extended_dc_capabilities; + u16 fw_capabilities; + u32 clock_frequency; + u32 instance_id; + u16 accel_mask; + u32 ae_mask; + u32 admin_ae_mask; + u16 tx_rings_mask; + u16 ring_to_svc_map; + u32 thd_to_arb_map[ICP_QAT_HW_AE_DELIMITER]; + u8 tx_rx_gap; + u8 num_banks; + u16 num_banks_per_vf; + u8 num_rings_per_bank; + u8 num_accel; + u8 num_logical_accel; + u8 num_engines; + u32 num_hb_ctrs; + u8 num_rps; +}; + +/* CSR write macro */ +#define ADF_CSR_WR(csr_base, csr_offset, val) \ + __raw_writel(val, csr_base + csr_offset) +/* + * CSR write macro to handle cases where the high and low + * offsets are sparsely located. + */ +#define ADF_CSR_WR64_LO_HI(csr_base, csr_low_offset, csr_high_offset, val) \ +do { \ + ADF_CSR_WR(csr_base, csr_low_offset, lower_32_bits(val)); \ + ADF_CSR_WR(csr_base, csr_high_offset, upper_32_bits(val)); \ +} while (0) + +/* CSR read macro */ +#define ADF_CSR_RD(csr_base, csr_offset) __raw_readl(csr_base + csr_offset) + +#define ADF_CFG_NUM_SERVICES 4 +#define ADF_SRV_TYPE_BIT_LEN 3 +#define ADF_SRV_TYPE_MASK 0x7 +#define ADF_AE_ADMIN_THREAD 7 +#define ADF_NUM_THREADS_PER_AE 8 +#define ADF_NUM_PKE_STRAND 2 +#define ADF_AE_STRAND0_THREAD 8 +#define ADF_AE_STRAND1_THREAD 9 + +#define GET_DEV(accel_dev) ((accel_dev)->accel_pci_dev.pci_dev->dev) +#define GET_BARS(accel_dev) ((accel_dev)->accel_pci_dev.pci_bars) +#define GET_HW_DATA(accel_dev) (accel_dev->hw_device) +#define GET_MAX_BANKS(accel_dev) (GET_HW_DATA(accel_dev)->num_banks) +#define GET_NUM_RINGS_PER_BANK(accel_dev) \ + GET_HW_DATA(accel_dev)->num_rings_per_bank +#define GET_SRV_TYPE(accel_dev, idx) \ + (((GET_HW_DATA(accel_dev)->ring_to_svc_map) >> (ADF_SRV_TYPE_BIT_LEN * (idx))) \ + & ADF_SRV_TYPE_MASK) +#define GET_ERR_MASK(accel_dev) (&GET_HW_DATA(accel_dev)->dev_err_mask) +#define GET_MAX_ACCELENGINES(accel_dev) (GET_HW_DATA(accel_dev)->num_engines) +#define GET_CSR_OPS(accel_dev) (&(accel_dev)->hw_device->csr_ops) +#define GET_PFVF_OPS(accel_dev) (&(accel_dev)->hw_device->pfvf_ops) +#define GET_DC_OPS(accel_dev) (&(accel_dev)->hw_device->dc_ops) +#define GET_VFMIG_OPS(accel_dev) (&(accel_dev)->hw_device->vfmig_ops) +#define GET_TL_DATA(accel_dev) GET_HW_DATA(accel_dev)->tl_data +#define accel_to_pci_dev(accel_ptr) accel_ptr->accel_pci_dev.pci_dev + +struct adf_admin_comms; +struct icp_qat_fw_loader_handle; +struct adf_fw_loader_data { + struct icp_qat_fw_loader_handle *fw_loader; + const struct firmware *uof_fw; + const struct firmware *mmp_fw; +}; + +struct adf_accel_vf_info { + struct adf_accel_dev *accel_dev; + struct mutex pf2vf_lock; /* protect CSR access for PF2VF messages */ + struct mutex pfvf_mig_lock; /* protects PFVF state for migration */ + struct ratelimit_state vf2pf_ratelimit; + u32 vf_nr; + bool init; + bool restarting; + u8 vf_compat_ver; + /* + * Private area used for device migration. + * Memory allocation and free is managed by migration driver. + */ + void *mig_priv; +}; + +struct adf_dc_data { + u8 *ovf_buff; + size_t ovf_buff_sz; + dma_addr_t ovf_buff_p; +}; + +struct adf_pm { + struct dentry *debugfs_pm_status; + bool present; + int idle_irq_counters; + int throttle_irq_counters; + int fw_irq_counters; + int host_ack_counter; + int host_nack_counter; + ssize_t (*print_pm_status)(struct adf_accel_dev *accel_dev, + char __user *buf, size_t count, loff_t *pos); +}; + +struct adf_sysfs { + int ring_num; + struct rw_semaphore lock; /* protects access to the fields in this struct */ +}; + +struct adf_accel_dev { + struct adf_etr_data *transport; + struct adf_hw_device_data *hw_device; + struct adf_cfg_device_data *cfg; + struct adf_fw_loader_data *fw_loader; + struct adf_admin_comms *admin; + struct adf_telemetry *telemetry; + struct adf_dc_data *dc_data; + struct adf_pm power_management; + struct list_head crypto_list; + struct list_head compression_list; + unsigned long status; + atomic_t ref_count; + struct dentry *debugfs_dir; + struct dentry *fw_cntr_dbgfile; + struct dentry *cnv_dbgfile; + struct list_head list; + struct module *owner; + struct adf_accel_pci accel_pci_dev; + struct adf_timer *timer; + struct adf_heartbeat *heartbeat; + struct adf_rl *rate_limiting; + struct adf_sysfs sysfs; + union { + struct { + /* protects VF2PF interrupts access */ + spinlock_t vf2pf_ints_lock; + /* prevents VF2PF handling from racing with VF state teardown */ + bool vf2pf_disabled; + /* vf_info is non-zero when SR-IOV is init'ed */ + struct adf_accel_vf_info *vf_info; + } pf; + struct { + bool irq_enabled; + char irq_name[ADF_MAX_MSIX_VECTOR_NAME]; + struct tasklet_struct pf2vf_bh_tasklet; + struct mutex vf2pf_lock; /* protect CSR access */ + struct completion msg_received; + struct pfvf_message response; /* temp field holding pf2vf response */ + u8 pf_compat_ver; + } vf; + }; + struct adf_error_counters ras_errors; + struct mutex state_lock; /* protect state of the device */ + bool is_vf; + bool autoreset_on_error; + u32 accel_id; +}; +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_accel_engine.c b/drivers/crypto/intel/qat/qat_common/adf_accel_engine.c new file mode 100644 index 000000000..09d4f547e --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_accel_engine.c @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#include +#include +#include +#include "adf_cfg.h" +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "icp_qat_uclo.h" + +static int adf_ae_fw_load_images(struct adf_accel_dev *accel_dev, void *fw_addr, + u32 fw_size) +{ + struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + struct icp_qat_fw_loader_handle *loader; + const char *obj_name; + u32 num_objs; + u32 ae_mask; + int i; + + loader = loader_data->fw_loader; + num_objs = hw_device->uof_get_num_objs(accel_dev); + + for (i = 0; i < num_objs; i++) { + obj_name = hw_device->uof_get_name(accel_dev, i); + ae_mask = hw_device->uof_get_ae_mask(accel_dev, i); + if (!obj_name || !ae_mask) { + dev_err(&GET_DEV(accel_dev), "Invalid UOF image\n"); + goto out_err; + } + + if (qat_uclo_set_cfg_ae_mask(loader, ae_mask)) { + dev_err(&GET_DEV(accel_dev), + "Invalid mask for UOF image\n"); + goto out_err; + } + if (qat_uclo_map_obj(loader, fw_addr, fw_size, obj_name)) { + dev_err(&GET_DEV(accel_dev), + "Failed to map UOF firmware\n"); + goto out_err; + } + if (qat_uclo_wr_all_uimage(loader)) { + dev_err(&GET_DEV(accel_dev), + "Failed to load UOF firmware\n"); + goto out_err; + } + qat_uclo_del_obj(loader); + } + + return 0; + +out_err: + adf_ae_fw_release(accel_dev); + return -EFAULT; +} + +int adf_ae_fw_load(struct adf_accel_dev *accel_dev) +{ + struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + void *fw_addr, *mmp_addr; + u32 fw_size, mmp_size; + + if (!hw_device->fw_name) + return 0; + + if (request_firmware(&loader_data->mmp_fw, hw_device->fw_mmp_name, + &accel_dev->accel_pci_dev.pci_dev->dev)) { + dev_err(&GET_DEV(accel_dev), "Failed to load MMP firmware %s\n", + hw_device->fw_mmp_name); + return -EFAULT; + } + if (request_firmware(&loader_data->uof_fw, hw_device->fw_name, + &accel_dev->accel_pci_dev.pci_dev->dev)) { + dev_err(&GET_DEV(accel_dev), "Failed to load UOF firmware %s\n", + hw_device->fw_name); + goto out_err; + } + + fw_size = loader_data->uof_fw->size; + fw_addr = (void *)loader_data->uof_fw->data; + mmp_size = loader_data->mmp_fw->size; + mmp_addr = (void *)loader_data->mmp_fw->data; + + if (qat_uclo_wr_mimage(loader_data->fw_loader, mmp_addr, mmp_size)) { + dev_err(&GET_DEV(accel_dev), "Failed to load MMP\n"); + goto out_err; + } + + if (hw_device->uof_get_num_objs) + return adf_ae_fw_load_images(accel_dev, fw_addr, fw_size); + + if (qat_uclo_map_obj(loader_data->fw_loader, fw_addr, fw_size, NULL)) { + dev_err(&GET_DEV(accel_dev), "Failed to map FW\n"); + goto out_err; + } + if (qat_uclo_wr_all_uimage(loader_data->fw_loader)) { + dev_err(&GET_DEV(accel_dev), "Failed to load UOF\n"); + goto out_err; + } + return 0; + +out_err: + adf_ae_fw_release(accel_dev); + return -EFAULT; +} + +void adf_ae_fw_release(struct adf_accel_dev *accel_dev) +{ + struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + + if (!hw_device->fw_name) + return; + + qat_uclo_del_obj(loader_data->fw_loader); + qat_hal_deinit(loader_data->fw_loader); + release_firmware(loader_data->uof_fw); + release_firmware(loader_data->mmp_fw); + loader_data->uof_fw = NULL; + loader_data->mmp_fw = NULL; + loader_data->fw_loader = NULL; +} + +int adf_ae_start(struct adf_accel_dev *accel_dev) +{ + struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 ae_ctr; + + if (!hw_data->fw_name) + return 0; + + ae_ctr = qat_hal_start(loader_data->fw_loader); + dev_info(&GET_DEV(accel_dev), + "qat_dev%d started %d acceleration engines\n", + accel_dev->accel_id, ae_ctr); + return 0; +} + +int adf_ae_stop(struct adf_accel_dev *accel_dev) +{ + struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 ae_ctr, ae, max_aes = GET_MAX_ACCELENGINES(accel_dev); + + if (!hw_data->fw_name) + return 0; + + for (ae = 0, ae_ctr = 0; ae < max_aes; ae++) { + if (hw_data->ae_mask & (1 << ae)) { + qat_hal_stop(loader_data->fw_loader, ae, 0xFF); + ae_ctr++; + } + } + dev_info(&GET_DEV(accel_dev), + "qat_dev%d stopped %d acceleration engines\n", + accel_dev->accel_id, ae_ctr); + return 0; +} + +static int adf_ae_reset(struct adf_accel_dev *accel_dev, int ae) +{ + struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; + unsigned long reset_delay; + + qat_hal_reset(loader_data->fw_loader); + + reset_delay = loader_data->fw_loader->chip_info->reset_delay_us; + if (reset_delay) + fsleep(reset_delay); + + if (qat_hal_clr_reset(loader_data->fw_loader)) + return -EFAULT; + + return 0; +} + +int adf_ae_init(struct adf_accel_dev *accel_dev) +{ + struct adf_fw_loader_data *loader_data; + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + + if (!hw_device->fw_name) + return 0; + + loader_data = kzalloc_obj(*loader_data); + if (!loader_data) + return -ENOMEM; + + accel_dev->fw_loader = loader_data; + if (qat_hal_init(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Failed to init the AEs\n"); + kfree(loader_data); + return -EFAULT; + } + if (adf_ae_reset(accel_dev, 0)) { + dev_err(&GET_DEV(accel_dev), "Failed to reset the AEs\n"); + qat_hal_deinit(loader_data->fw_loader); + kfree(loader_data); + return -EFAULT; + } + return 0; +} + +int adf_ae_shutdown(struct adf_accel_dev *accel_dev) +{ + struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + + if (!hw_device->fw_name) + return 0; + + qat_hal_deinit(loader_data->fw_loader); + kfree(accel_dev->fw_loader); + accel_dev->fw_loader = NULL; + return 0; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_admin.c b/drivers/crypto/intel/qat/qat_common/adf_admin.c new file mode 100644 index 000000000..1eea74c2a --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_admin.c @@ -0,0 +1,719 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include +#include "adf_accel_devices.h" +#include "adf_admin.h" +#include "adf_anti_rb.h" +#include "adf_common_drv.h" +#include "adf_cfg.h" +#include "adf_heartbeat.h" +#include "adf_kpt.h" +#include "icp_qat_fw_init_admin.h" + +#define ADF_ADMIN_MAILBOX_STRIDE 0x1000 +#define ADF_ADMINMSG_LEN 32 +#define ADF_CONST_TABLE_SIZE 1024 +#define ADF_ADMIN_POLL_DELAY_US 20 +#define ADF_ADMIN_POLL_TIMEOUT_US (5 * USEC_PER_SEC) +#define ADF_ONE_AE 1 +#define ADF_ADMIN_RETRY_MAX 60 + +static const u8 const_tab[1024] __aligned(1024) = { +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x03, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x01, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x02, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x13, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, +0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x33, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76, +0x54, 0x32, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x45, 0x23, 0x01, 0xef, 0xcd, 0xab, +0x89, 0x98, 0xba, 0xdc, 0xfe, 0x10, 0x32, 0x54, 0x76, 0xc3, 0xd2, 0xe1, 0xf0, +0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc1, 0x05, 0x9e, +0xd8, 0x36, 0x7c, 0xd5, 0x07, 0x30, 0x70, 0xdd, 0x17, 0xf7, 0x0e, 0x59, 0x39, +0xff, 0xc0, 0x0b, 0x31, 0x68, 0x58, 0x15, 0x11, 0x64, 0xf9, 0x8f, 0xa7, 0xbe, +0xfa, 0x4f, 0xa4, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x09, 0xe6, 0x67, 0xbb, 0x67, 0xae, +0x85, 0x3c, 0x6e, 0xf3, 0x72, 0xa5, 0x4f, 0xf5, 0x3a, 0x51, 0x0e, 0x52, 0x7f, +0x9b, 0x05, 0x68, 0x8c, 0x1f, 0x83, 0xd9, 0xab, 0x5b, 0xe0, 0xcd, 0x19, 0x05, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8, 0x62, 0x9a, 0x29, +0x2a, 0x36, 0x7c, 0xd5, 0x07, 0x91, 0x59, 0x01, 0x5a, 0x30, 0x70, 0xdd, 0x17, +0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39, 0x67, 0x33, 0x26, 0x67, 0xff, +0xc0, 0x0b, 0x31, 0x8e, 0xb4, 0x4a, 0x87, 0x68, 0x58, 0x15, 0x11, 0xdb, 0x0c, +0x2e, 0x0d, 0x64, 0xf9, 0x8f, 0xa7, 0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f, +0xa4, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08, 0xbb, +0x67, 0xae, 0x85, 0x84, 0xca, 0xa7, 0x3b, 0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94, +0xf8, 0x2b, 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, 0x51, 0x0e, 0x52, +0x7f, 0xad, 0xe6, 0x82, 0xd1, 0x9b, 0x05, 0x68, 0x8c, 0x2b, 0x3e, 0x6c, 0x1f, +0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, 0x6b, 0x5b, 0xe0, 0xcd, 0x19, 0x13, +0x7e, 0x21, 0x79, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x18, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x01, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x15, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x02, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x14, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x02, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x24, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, +0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x45, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x01, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x01, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x2B, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + +struct adf_admin_comms { + dma_addr_t phy_addr; + dma_addr_t const_tbl_addr; + void *virt_addr; + void *virt_tbl_addr; + void __iomem *mailbox_addr; + struct mutex lock; /* protects adf_admin_comms struct */ +}; + +static int adf_put_admin_msg_sync(struct adf_accel_dev *accel_dev, u32 ae, + void *in, void *out) +{ + int ret; + u32 status; + struct adf_admin_comms *admin = accel_dev->admin; + int offset = ae * ADF_ADMINMSG_LEN * 2; + void __iomem *mailbox = admin->mailbox_addr; + int mb_offset = ae * ADF_ADMIN_MAILBOX_STRIDE; + struct icp_qat_fw_init_admin_req *request = in; + + mutex_lock(&admin->lock); + + if (ADF_CSR_RD(mailbox, mb_offset) == 1) { + mutex_unlock(&admin->lock); + return -EAGAIN; + } + + memcpy(admin->virt_addr + offset, in, ADF_ADMINMSG_LEN); + ADF_CSR_WR(mailbox, mb_offset, 1); + + ret = read_poll_timeout(ADF_CSR_RD, status, status == 0, + ADF_ADMIN_POLL_DELAY_US, + ADF_ADMIN_POLL_TIMEOUT_US, true, + mailbox, mb_offset); + if (ret < 0) { + /* Response timeout */ + dev_err(&GET_DEV(accel_dev), + "Failed to send admin msg %d to accelerator %d\n", + request->cmd_id, ae); + } else { + /* Response received from admin message, we can now + * make response data available in "out" parameter. + */ + memcpy(out, admin->virt_addr + offset + + ADF_ADMINMSG_LEN, ADF_ADMINMSG_LEN); + } + + mutex_unlock(&admin->lock); + return ret; +} + +static int adf_send_admin(struct adf_accel_dev *accel_dev, + struct icp_qat_fw_init_admin_req *req, + struct icp_qat_fw_init_admin_resp *resp, + const unsigned long ae_mask) +{ + u32 ae; + + for_each_set_bit(ae, &ae_mask, ICP_QAT_HW_AE_DELIMITER) + if (adf_put_admin_msg_sync(accel_dev, ae, req, resp) || + resp->status) + return -EFAULT; + + return 0; +} + +static int adf_init_ae(struct adf_accel_dev *accel_dev) +{ + struct icp_qat_fw_init_admin_req req; + struct icp_qat_fw_init_admin_resp resp; + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + u32 ae_mask = hw_device->ae_mask; + + memset(&req, 0, sizeof(req)); + memset(&resp, 0, sizeof(resp)); + req.cmd_id = ICP_QAT_FW_INIT_AE; + + return adf_send_admin(accel_dev, &req, &resp, ae_mask); +} + +static int adf_set_fw_constants(struct adf_accel_dev *accel_dev) +{ + struct icp_qat_fw_init_admin_req req; + struct icp_qat_fw_init_admin_resp resp; + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + u32 ae_mask = hw_device->admin_ae_mask ?: hw_device->ae_mask; + + memset(&req, 0, sizeof(req)); + memset(&resp, 0, sizeof(resp)); + req.cmd_id = ICP_QAT_FW_CONSTANTS_CFG; + + req.init_cfg_sz = ADF_CONST_TABLE_SIZE; + req.init_cfg_ptr = accel_dev->admin->const_tbl_addr; + + return adf_send_admin(accel_dev, &req, &resp, ae_mask); +} + +int adf_get_fw_timestamp(struct adf_accel_dev *accel_dev, u64 *timestamp) +{ + struct icp_qat_fw_init_admin_req req = { }; + struct icp_qat_fw_init_admin_resp resp; + unsigned int ae_mask = ADF_ONE_AE; + int ret; + + req.cmd_id = ICP_QAT_FW_TIMER_GET; + ret = adf_send_admin(accel_dev, &req, &resp, ae_mask); + if (ret) + return ret; + + *timestamp = resp.timestamp; + return 0; +} + +static int adf_set_chaining(struct adf_accel_dev *accel_dev) +{ + u32 ae_mask = GET_HW_DATA(accel_dev)->ae_mask; + struct icp_qat_fw_init_admin_resp resp = { }; + struct icp_qat_fw_init_admin_req req = { }; + + req.cmd_id = ICP_QAT_FW_DC_CHAIN_INIT; + + return adf_send_admin(accel_dev, &req, &resp, ae_mask); +} + +static int adf_get_dc_capabilities(struct adf_accel_dev *accel_dev, + u32 *capabilities) +{ + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + struct icp_qat_fw_init_admin_resp resp; + struct icp_qat_fw_init_admin_req req; + unsigned long ae_mask; + unsigned long ae; + int ret; + + /* Target only service accelerator engines */ + ae_mask = hw_device->ae_mask & ~hw_device->admin_ae_mask; + + memset(&req, 0, sizeof(req)); + memset(&resp, 0, sizeof(resp)); + req.cmd_id = ICP_QAT_FW_COMP_CAPABILITY_GET; + + *capabilities = 0; + for_each_set_bit(ae, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) { + ret = adf_send_admin(accel_dev, &req, &resp, 1ULL << ae); + if (ret) + return ret; + + *capabilities |= resp.extended_features; + } + + return 0; +} + +int adf_get_ae_fw_counters(struct adf_accel_dev *accel_dev, u16 ae, u64 *reqs, u64 *resps) +{ + struct icp_qat_fw_init_admin_resp resp = { }; + struct icp_qat_fw_init_admin_req req = { }; + int ret; + + req.cmd_id = ICP_QAT_FW_COUNTERS_GET; + + ret = adf_put_admin_msg_sync(accel_dev, ae, &req, &resp); + if (ret || resp.status) + return -EFAULT; + + *reqs = resp.req_rec_count; + *resps = resp.resp_sent_count; + + return 0; +} + +int adf_send_admin_tim_sync(struct adf_accel_dev *accel_dev, u32 cnt) +{ + u32 ae_mask = accel_dev->hw_device->ae_mask; + struct icp_qat_fw_init_admin_req req = { }; + struct icp_qat_fw_init_admin_resp resp = { }; + + req.cmd_id = ICP_QAT_FW_SYNC; + req.int_timer_ticks = cnt; + + return adf_send_admin(accel_dev, &req, &resp, ae_mask); +} + +int adf_send_admin_hb_timer(struct adf_accel_dev *accel_dev, uint32_t ticks) +{ + u32 ae_mask = accel_dev->hw_device->ae_mask; + struct icp_qat_fw_init_admin_req req = { }; + struct icp_qat_fw_init_admin_resp resp; + + req.cmd_id = ICP_QAT_FW_HEARTBEAT_TIMER_SET; + req.init_cfg_ptr = accel_dev->heartbeat->dma.phy_addr; + req.heartbeat_ticks = ticks; + + return adf_send_admin(accel_dev, &req, &resp, ae_mask); +} + +static bool is_dcc_enabled(struct adf_accel_dev *accel_dev) +{ + char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0}; + int ret; + + ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, + ADF_SERVICES_ENABLED, services); + if (ret) + return false; + + return !strcmp(services, "dcc"); +} + +static int adf_get_fw_capabilities(struct adf_accel_dev *accel_dev, u16 *caps) +{ + u32 ae_mask = accel_dev->hw_device->admin_ae_mask; + struct icp_qat_fw_init_admin_resp resp = { }; + struct icp_qat_fw_init_admin_req req = { }; + int ret; + + if (!ae_mask) + return 0; + + req.cmd_id = ICP_QAT_FW_CAPABILITIES_GET; + ret = adf_send_admin(accel_dev, &req, &resp, ae_mask); + if (ret) + return ret; + + *caps = resp.fw_capabilities; + + return 0; +} + +int adf_send_admin_rl_init(struct adf_accel_dev *accel_dev, + struct icp_qat_fw_init_admin_slice_cnt *slices) +{ + u32 ae_mask = accel_dev->hw_device->admin_ae_mask; + struct icp_qat_fw_init_admin_resp resp = { }; + struct icp_qat_fw_init_admin_req req = { }; + int ret; + + req.cmd_id = ICP_QAT_FW_RL_INIT; + + ret = adf_send_admin(accel_dev, &req, &resp, ae_mask); + if (ret) + return ret; + + memcpy(slices, &resp.slices, sizeof(*slices)); + + return 0; +} + +int adf_send_admin_rl_add_update(struct adf_accel_dev *accel_dev, + struct icp_qat_fw_init_admin_req *req) +{ + u32 ae_mask = accel_dev->hw_device->admin_ae_mask; + struct icp_qat_fw_init_admin_resp resp = { }; + + /* + * req struct filled in rl implementation. Used commands + * ICP_QAT_FW_RL_ADD for a new SLA + * ICP_QAT_FW_RL_UPDATE for update SLA + */ + return adf_send_admin(accel_dev, req, &resp, ae_mask); +} + +int adf_send_admin_rl_delete(struct adf_accel_dev *accel_dev, u16 node_id, + u8 node_type) +{ + u32 ae_mask = accel_dev->hw_device->admin_ae_mask; + struct icp_qat_fw_init_admin_resp resp = { }; + struct icp_qat_fw_init_admin_req req = { }; + + req.cmd_id = ICP_QAT_FW_RL_REMOVE; + req.node_id = node_id; + req.node_type = node_type; + + return adf_send_admin(accel_dev, &req, &resp, ae_mask); +} + +/** + * adf_send_admin_init() - Function sends init message to FW + * @accel_dev: Pointer to acceleration device. + * + * Function sends admin init message to the FW + * + * Return: 0 on success, error code otherwise. + */ +int adf_send_admin_init(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + u32 dc_capabilities = 0; + int ret; + + ret = adf_set_fw_constants(accel_dev); + if (ret) + return ret; + + if (is_dcc_enabled(accel_dev)) { + ret = adf_set_chaining(accel_dev); + if (ret) + return ret; + } + + ret = adf_get_dc_capabilities(accel_dev, &dc_capabilities); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Cannot get dc capabilities\n"); + return ret; + } + accel_dev->hw_device->extended_dc_capabilities = dc_capabilities; + + adf_get_fw_capabilities(accel_dev, &hw_data->fw_capabilities); + + return adf_init_ae(accel_dev); +} +EXPORT_SYMBOL_GPL(adf_send_admin_init); + +/** + * adf_init_admin_pm() - Function sends PM init message to FW + * @accel_dev: Pointer to acceleration device. + * @idle_delay: QAT HW idle time before power gating is initiated. + * 000 - 64us + * 001 - 128us + * 010 - 256us + * 011 - 512us + * 100 - 1ms + * 101 - 2ms + * 110 - 4ms + * 111 - 8ms + * + * Function sends to the FW the admin init message for the PM state + * configuration. + * + * Return: 0 on success, error code otherwise. + */ +int adf_init_admin_pm(struct adf_accel_dev *accel_dev, u32 idle_delay) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct icp_qat_fw_init_admin_resp resp = {0}; + struct icp_qat_fw_init_admin_req req = {0}; + u32 ae_mask = hw_data->admin_ae_mask; + + if (!accel_dev->admin) { + dev_err(&GET_DEV(accel_dev), "adf_admin is not available\n"); + return -EFAULT; + } + + req.cmd_id = ICP_QAT_FW_PM_STATE_CONFIG; + req.idle_filter = idle_delay; + + return adf_send_admin(accel_dev, &req, &resp, ae_mask); +} +EXPORT_SYMBOL_GPL(adf_init_admin_pm); + +int adf_get_pm_info(struct adf_accel_dev *accel_dev, dma_addr_t p_state_addr, + size_t buff_size) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct icp_qat_fw_init_admin_req req = { }; + struct icp_qat_fw_init_admin_resp resp; + u32 ae_mask = hw_data->admin_ae_mask; + int ret; + + /* Query pm info via init/admin cmd */ + if (!accel_dev->admin) { + dev_err(&GET_DEV(accel_dev), "adf_admin is not available\n"); + return -EFAULT; + } + + req.cmd_id = ICP_QAT_FW_PM_INFO; + req.init_cfg_sz = buff_size; + req.init_cfg_ptr = p_state_addr; + + ret = adf_send_admin(accel_dev, &req, &resp, ae_mask); + if (ret) + dev_err(&GET_DEV(accel_dev), + "Failed to query power-management info\n"); + + return ret; +} + +int adf_get_cnv_stats(struct adf_accel_dev *accel_dev, u16 ae, u16 *err_cnt, + u16 *latest_err) +{ + struct icp_qat_fw_init_admin_req req = { }; + struct icp_qat_fw_init_admin_resp resp; + int ret; + + req.cmd_id = ICP_QAT_FW_CNV_STATS_GET; + + ret = adf_put_admin_msg_sync(accel_dev, ae, &req, &resp); + if (ret) + return ret; + if (resp.status) + return -EPROTONOSUPPORT; + + *err_cnt = resp.error_count; + *latest_err = resp.latest_error; + + return ret; +} + +int adf_send_admin_tl_start(struct adf_accel_dev *accel_dev, + dma_addr_t tl_dma_addr, size_t layout_sz, u8 *rp_indexes, + struct icp_qat_fw_init_admin_slice_cnt *slice_count) +{ + u32 ae_mask = GET_HW_DATA(accel_dev)->admin_ae_mask; + struct icp_qat_fw_init_admin_resp resp = { }; + struct icp_qat_fw_init_admin_req req = { }; + int ret; + + req.cmd_id = ICP_QAT_FW_TL_START; + req.init_cfg_ptr = tl_dma_addr; + req.init_cfg_sz = layout_sz; + + if (rp_indexes) + memcpy(&req.rp_indexes, rp_indexes, sizeof(req.rp_indexes)); + + ret = adf_send_admin(accel_dev, &req, &resp, ae_mask); + if (ret) + return ret; + + memcpy(slice_count, &resp.slices, sizeof(*slice_count)); + + return 0; +} + +int adf_send_admin_tl_stop(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + struct icp_qat_fw_init_admin_resp resp = { }; + struct icp_qat_fw_init_admin_req req = { }; + u32 ae_mask = hw_data->admin_ae_mask; + + req.cmd_id = ICP_QAT_FW_TL_STOP; + + return adf_send_admin(accel_dev, &req, &resp, ae_mask); +} + +static int adf_send_admin_retry(struct adf_accel_dev *accel_dev, u8 cmd_id, + struct icp_qat_fw_init_admin_resp *resp, + unsigned int sleep_ms) +{ + u32 admin_ae_mask = GET_HW_DATA(accel_dev)->admin_ae_mask; + struct icp_qat_fw_init_admin_req req = { }; + unsigned int retries = ADF_ADMIN_RETRY_MAX; + int ret; + + req.cmd_id = cmd_id; + + do { + ret = adf_send_admin(accel_dev, &req, resp, admin_ae_mask); + if (!ret) + return 0; + + if (resp->status != ICP_QAT_FW_INIT_RESP_STATUS_RETRY) + return ret; + + msleep(sleep_ms); + } while (--retries); + + return -ETIMEDOUT; +} + +static int adf_send_admin_svn(struct adf_accel_dev *accel_dev, u8 cmd_id, + struct icp_qat_fw_init_admin_resp *resp) +{ + return adf_send_admin_retry(accel_dev, cmd_id, resp, ADF_SVN_RETRY_MS); +} + +int adf_send_admin_arb_query(struct adf_accel_dev *accel_dev, int cmd, u8 *svn) +{ + struct icp_qat_fw_init_admin_resp resp = { }; + int ret; + + ret = adf_send_admin_svn(accel_dev, ICP_QAT_FW_SVN_READ, &resp); + if (ret) + return ret; + + switch (cmd) { + case ARB_ENFORCED_MIN_SVN: + *svn = resp.enforced_min_svn; + break; + case ARB_PERMANENT_MIN_SVN: + *svn = resp.permanent_min_svn; + break; + case ARB_ACTIVE_SVN: + *svn = resp.active_svn; + break; + default: + *svn = 0; + dev_err(&GET_DEV(accel_dev), + "Unknown secure version number request\n"); + ret = -EINVAL; + } + + return ret; +} + +int adf_send_admin_arb_commit(struct adf_accel_dev *accel_dev) +{ + struct icp_qat_fw_init_admin_resp resp = { }; + + return adf_send_admin_svn(accel_dev, ICP_QAT_FW_SVN_COMMIT, &resp); +} + +static_assert(sizeof(struct icp_qat_fw_init_admin_kpt_cfg) < PAGE_SIZE); + +static void adf_cfg_kpt_config(struct adf_accel_dev *accel_dev, + struct icp_qat_fw_init_admin_kpt_cfg *kpt_config) +{ + struct adf_kpt_interface_data *user_data = GET_KPT_USER_DATA(accel_dev); + + kpt_config->swk_cnt_per_fn = user_data->swk_cnt_per_fn; + kpt_config->swk_cnt_per_pasid = user_data->swk_cnt_per_pasid; + kpt_config->swk_ttl_in_secs = user_data->swk_max_ttl; + kpt_config->swk_shared_disable = !user_data->swk_shared; +} + +int adf_send_admin_kpt_init(struct adf_accel_dev *accel_dev, void *init_cfg, + size_t init_cfg_sz, dma_addr_t init_ptr) +{ + struct icp_qat_fw_init_admin_kpt_cfg *kpt_config = init_cfg; + u32 ae_mask = GET_HW_DATA(accel_dev)->admin_ae_mask; + struct icp_qat_fw_init_admin_resp resp = { }; + struct icp_qat_fw_init_admin_req req = { }; + int ret; + + if (!kpt_config || init_cfg_sz < sizeof(*kpt_config)) + return -EINVAL; + + adf_cfg_kpt_config(accel_dev, kpt_config); + + req.cmd_id = ICP_QAT_FW_KPT_ENABLE; + req.init_cfg_ptr = init_ptr; + req.init_cfg_sz = sizeof(*kpt_config); + + ret = adf_send_admin(accel_dev, &req, &resp, ae_mask); + if (ret) + dev_err(&GET_DEV(accel_dev), "Failed to send KPT init admin message\n"); + + return ret; +} + +int adf_init_admin_comms(struct adf_accel_dev *accel_dev) +{ + struct adf_admin_comms *admin; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + struct admin_info admin_csrs_info; + u32 mailbox_offset, adminmsg_u, adminmsg_l; + void __iomem *mailbox; + u64 reg_val; + + admin = kzalloc_node(sizeof(*accel_dev->admin), GFP_KERNEL, + dev_to_node(&GET_DEV(accel_dev))); + if (!admin) + return -ENOMEM; + admin->virt_addr = dma_alloc_coherent(&GET_DEV(accel_dev), PAGE_SIZE, + &admin->phy_addr, GFP_KERNEL); + if (!admin->virt_addr) { + dev_err(&GET_DEV(accel_dev), "Failed to allocate dma buff\n"); + kfree(admin); + return -ENOMEM; + } + + admin->virt_tbl_addr = dma_alloc_coherent(&GET_DEV(accel_dev), + PAGE_SIZE, + &admin->const_tbl_addr, + GFP_KERNEL); + if (!admin->virt_tbl_addr) { + dev_err(&GET_DEV(accel_dev), "Failed to allocate const_tbl\n"); + dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, + admin->virt_addr, admin->phy_addr); + kfree(admin); + return -ENOMEM; + } + + memcpy(admin->virt_tbl_addr, const_tab, sizeof(const_tab)); + hw_data->get_admin_info(&admin_csrs_info); + + mailbox_offset = admin_csrs_info.mailbox_offset; + mailbox = pmisc_addr + mailbox_offset; + adminmsg_u = admin_csrs_info.admin_msg_ur; + adminmsg_l = admin_csrs_info.admin_msg_lr; + + reg_val = (u64)admin->phy_addr; + ADF_CSR_WR(pmisc_addr, adminmsg_u, upper_32_bits(reg_val)); + ADF_CSR_WR(pmisc_addr, adminmsg_l, lower_32_bits(reg_val)); + + mutex_init(&admin->lock); + admin->mailbox_addr = mailbox; + accel_dev->admin = admin; + return 0; +} +EXPORT_SYMBOL_GPL(adf_init_admin_comms); + +void adf_exit_admin_comms(struct adf_accel_dev *accel_dev) +{ + struct adf_admin_comms *admin = accel_dev->admin; + + if (!admin) + return; + + if (admin->virt_addr) + dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, + admin->virt_addr, admin->phy_addr); + if (admin->virt_tbl_addr) + dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, + admin->virt_tbl_addr, admin->const_tbl_addr); + + mutex_destroy(&admin->lock); + kfree(admin); + accel_dev->admin = NULL; +} +EXPORT_SYMBOL_GPL(adf_exit_admin_comms); diff --git a/drivers/crypto/intel/qat/qat_common/adf_admin.h b/drivers/crypto/intel/qat/qat_common/adf_admin.h new file mode 100644 index 000000000..44eb6dc92 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_admin.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ +#ifndef ADF_ADMIN +#define ADF_ADMIN + +#include "icp_qat_fw_init_admin.h" + +struct adf_accel_dev; + +int adf_init_admin_comms(struct adf_accel_dev *accel_dev); +void adf_exit_admin_comms(struct adf_accel_dev *accel_dev); +int adf_send_admin_init(struct adf_accel_dev *accel_dev); +int adf_get_ae_fw_counters(struct adf_accel_dev *accel_dev, u16 ae, u64 *reqs, u64 *resps); +int adf_init_admin_pm(struct adf_accel_dev *accel_dev, u32 idle_delay); +int adf_send_admin_tim_sync(struct adf_accel_dev *accel_dev, u32 cnt); +int adf_send_admin_hb_timer(struct adf_accel_dev *accel_dev, uint32_t ticks); +int adf_send_admin_rl_init(struct adf_accel_dev *accel_dev, + struct icp_qat_fw_init_admin_slice_cnt *slices); +int adf_send_admin_rl_add_update(struct adf_accel_dev *accel_dev, + struct icp_qat_fw_init_admin_req *req); +int adf_send_admin_rl_delete(struct adf_accel_dev *accel_dev, u16 node_id, + u8 node_type); +int adf_get_fw_timestamp(struct adf_accel_dev *accel_dev, u64 *timestamp); +int adf_get_pm_info(struct adf_accel_dev *accel_dev, dma_addr_t p_state_addr, size_t buff_size); +int adf_get_cnv_stats(struct adf_accel_dev *accel_dev, u16 ae, u16 *err_cnt, u16 *latest_err); +int adf_send_admin_tl_start(struct adf_accel_dev *accel_dev, + dma_addr_t tl_dma_addr, size_t layout_sz, u8 *rp_indexes, + struct icp_qat_fw_init_admin_slice_cnt *slice_count); +int adf_send_admin_tl_stop(struct adf_accel_dev *accel_dev); +int adf_send_admin_arb_query(struct adf_accel_dev *accel_dev, int cmd, u8 *svn); +int adf_send_admin_arb_commit(struct adf_accel_dev *accel_dev); +int adf_send_admin_kpt_init(struct adf_accel_dev *accel_dev, void *init_cfg, + size_t init_cfg_sz, dma_addr_t init_ptr); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_aer.c b/drivers/crypto/intel/qat/qat_common/adf_aer.c new file mode 100644 index 000000000..afded3030 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_aer.c @@ -0,0 +1,344 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#include +#include +#include +#include +#include +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_pfvf_pf_msg.h" + +struct adf_fatal_error_data { + struct adf_accel_dev *accel_dev; + struct work_struct work; +}; + +static struct workqueue_struct *device_reset_wq; +static struct workqueue_struct *device_sriov_wq; + +static pci_ers_result_t reset_prepare(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + if (!accel_dev) { + pci_err(pdev, "Can't find acceleration device\n"); + return PCI_ERS_RESULT_DISCONNECT; + } + + if (!adf_dev_started(accel_dev)) + return PCI_ERS_RESULT_CAN_RECOVER; + + set_bit(ADF_STATUS_RESTARTING, &accel_dev->status); + if (accel_dev->hw_device->exit_arb) { + dev_dbg(&pdev->dev, "Disabling arbitration\n"); + accel_dev->hw_device->exit_arb(accel_dev); + } + adf_dev_restarting_notify(accel_dev); + adf_dev_down(accel_dev); + + return PCI_ERS_RESULT_NEED_RESET; +} + +static pci_ers_result_t reset_done(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + int res; + + if (!accel_dev) { + pci_err(pdev, "Can't find acceleration device\n"); + return PCI_ERS_RESULT_DISCONNECT; + } + + if (!adf_devmgr_in_reset(accel_dev)) + goto reset_complete; + + pci_restore_state(pdev); + res = adf_dev_up(accel_dev, false); + if (res && res != -EALREADY) + return PCI_ERS_RESULT_DISCONNECT; + + adf_reenable_sriov(accel_dev); + adf_pf2vf_notify_restarted(accel_dev); + adf_dev_restarted_notify(accel_dev); + clear_bit(ADF_STATUS_RESTARTING, &accel_dev->status); + +reset_complete: + pci_info(pdev, "Device reset completed successfully\n"); + + return PCI_ERS_RESULT_RECOVERED; +} + +static pci_ers_result_t adf_error_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + pci_info(pdev, "Acceleration driver hardware error detected.\n"); + if (!accel_dev) { + pci_err(pdev, "Can't find acceleration device\n"); + return PCI_ERS_RESULT_DISCONNECT; + } + + if (state == pci_channel_io_perm_failure) { + pci_err(pdev, "Can't recover from device error\n"); + return PCI_ERS_RESULT_DISCONNECT; + } + + adf_error_notifier(accel_dev); + adf_pf2vf_notify_fatal_error(accel_dev); + + return reset_prepare(pdev); +} + +/* reset dev data */ +struct adf_reset_dev_data { + int mode; + struct adf_accel_dev *accel_dev; + struct completion compl; + struct work_struct reset_work; +}; + +/* sriov dev data */ +struct adf_sriov_dev_data { + struct adf_accel_dev *accel_dev; + struct completion compl; + struct work_struct sriov_work; +}; + +void adf_reset_sbr(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + struct pci_dev *parent = pdev->bus->self; + u16 bridge_ctl = 0; + + if (!parent) + parent = pdev; + + if (!pci_wait_for_pending_transaction(pdev)) + pci_info(pdev, "Transaction still in progress. Proceeding\n"); + + pci_info(pdev, "Secondary bus reset\n"); + + pci_read_config_word(parent, PCI_BRIDGE_CONTROL, &bridge_ctl); + bridge_ctl |= PCI_BRIDGE_CTL_BUS_RESET; + pci_write_config_word(parent, PCI_BRIDGE_CONTROL, bridge_ctl); + msleep(100); + bridge_ctl &= ~PCI_BRIDGE_CTL_BUS_RESET; + pci_write_config_word(parent, PCI_BRIDGE_CONTROL, bridge_ctl); + msleep(100); +} +EXPORT_SYMBOL_GPL(adf_reset_sbr); + +void adf_reset_flr(struct adf_accel_dev *accel_dev) +{ + pcie_flr(accel_to_pci_dev(accel_dev)); +} +EXPORT_SYMBOL_GPL(adf_reset_flr); + +void adf_dev_restore(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + + if (hw_device->reset_device) { + dev_info(&GET_DEV(accel_dev), "Resetting device qat_dev%d\n", + accel_dev->accel_id); + hw_device->reset_device(accel_dev); + pci_restore_state(pdev); + } +} + +void adf_set_bme(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + + pci_set_master(pdev); +} + +static void adf_device_sriov_worker(struct work_struct *work) +{ + struct adf_sriov_dev_data *sriov_data = + container_of(work, struct adf_sriov_dev_data, sriov_work); + + adf_reenable_sriov(sriov_data->accel_dev); + complete(&sriov_data->compl); +} + +static void adf_device_reset_worker(struct work_struct *work) +{ + struct adf_reset_dev_data *reset_data = + container_of(work, struct adf_reset_dev_data, reset_work); + struct adf_accel_dev *accel_dev = reset_data->accel_dev; + unsigned long wait_jiffies = msecs_to_jiffies(10000); + struct adf_sriov_dev_data sriov_data; + + adf_dev_restarting_notify(accel_dev); + if (adf_dev_restart(accel_dev)) { + /* The device hanged and we can't restart it so stop here */ + dev_err(&GET_DEV(accel_dev), "Restart device failed\n"); + if (reset_data->mode == ADF_DEV_RESET_ASYNC) + kfree(reset_data); + WARN(1, "QAT: device restart failed. Device is unusable\n"); + return; + } + + sriov_data.accel_dev = accel_dev; + init_completion(&sriov_data.compl); + INIT_WORK(&sriov_data.sriov_work, adf_device_sriov_worker); + queue_work(device_sriov_wq, &sriov_data.sriov_work); + if (wait_for_completion_timeout(&sriov_data.compl, wait_jiffies)) + adf_pf2vf_notify_restarted(accel_dev); + else + cancel_work_sync(&sriov_data.sriov_work); + + adf_dev_restarted_notify(accel_dev); + clear_bit(ADF_STATUS_RESTARTING, &accel_dev->status); + + /* The dev is back alive. Notify the caller if in sync mode */ + if (reset_data->mode == ADF_DEV_RESET_ASYNC) + kfree(reset_data); + else + complete(&reset_data->compl); +} + +static int adf_dev_aer_schedule_reset(struct adf_accel_dev *accel_dev, + enum adf_dev_reset_mode mode) +{ + struct adf_reset_dev_data *reset_data; + + if (!adf_dev_started(accel_dev) || + test_and_set_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) + return 0; + + reset_data = kzalloc_obj(*reset_data); + if (!reset_data) { + clear_bit(ADF_STATUS_RESTARTING, &accel_dev->status); + return -ENOMEM; + } + reset_data->accel_dev = accel_dev; + init_completion(&reset_data->compl); + reset_data->mode = mode; + INIT_WORK(&reset_data->reset_work, adf_device_reset_worker); + queue_work(device_reset_wq, &reset_data->reset_work); + + /* If in sync mode wait for the result */ + if (mode == ADF_DEV_RESET_SYNC) { + int ret = 0; + /* Maximum device reset time is 10 seconds */ + unsigned long wait_jiffies = msecs_to_jiffies(10000); + unsigned long timeout = wait_for_completion_timeout( + &reset_data->compl, wait_jiffies); + if (!timeout) { + dev_err(&GET_DEV(accel_dev), + "Reset device timeout expired\n"); + cancel_work_sync(&reset_data->reset_work); + ret = -EFAULT; + } + kfree(reset_data); + return ret; + } + return 0; +} + +static pci_ers_result_t adf_slot_reset(struct pci_dev *pdev) +{ + return reset_done(pdev); +} + +static void adf_resume(struct pci_dev *pdev) +{ + pci_info(pdev, "Acceleration driver reset completed\n"); + pci_info(pdev, "Device is up and running\n"); +} + +static void adf_reset_prepare(struct pci_dev *pdev) +{ + reset_prepare(pdev); +} + +static void adf_reset_done(struct pci_dev *pdev) +{ + reset_done(pdev); +} + +const struct pci_error_handlers adf_err_handler = { + .error_detected = adf_error_detected, + .slot_reset = adf_slot_reset, + .resume = adf_resume, + .reset_prepare = adf_reset_prepare, + .reset_done = adf_reset_done, +}; +EXPORT_SYMBOL_GPL(adf_err_handler); + +static int adf_dev_autoreset(struct adf_accel_dev *accel_dev) +{ + if (accel_dev->autoreset_on_error) + return adf_dev_aer_schedule_reset(accel_dev, ADF_DEV_RESET_ASYNC); + + return 0; +} + +static void adf_notify_fatal_error_worker(struct work_struct *work) +{ + struct adf_fatal_error_data *wq_data = + container_of(work, struct adf_fatal_error_data, work); + struct adf_accel_dev *accel_dev = wq_data->accel_dev; + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + + adf_error_notifier(accel_dev); + + if (!accel_dev->is_vf) { + /* Disable arbitration to stop processing of new requests */ + if (accel_dev->autoreset_on_error && hw_device->exit_arb) + hw_device->exit_arb(accel_dev); + if (accel_dev->pf.vf_info) + adf_pf2vf_notify_fatal_error(accel_dev); + adf_dev_autoreset(accel_dev); + } + + kfree(wq_data); +} + +int adf_notify_fatal_error(struct adf_accel_dev *accel_dev) +{ + struct adf_fatal_error_data *wq_data; + + wq_data = kzalloc_obj(*wq_data, GFP_ATOMIC); + if (!wq_data) + return -ENOMEM; + + wq_data->accel_dev = accel_dev; + INIT_WORK(&wq_data->work, adf_notify_fatal_error_worker); + adf_misc_wq_queue_work(&wq_data->work); + + return 0; +} + +int adf_init_aer(void) +{ + device_reset_wq = alloc_workqueue("qat_device_reset_wq", + WQ_MEM_RECLAIM | WQ_PERCPU, 0); + if (!device_reset_wq) + return -EFAULT; + + device_sriov_wq = alloc_workqueue("qat_device_sriov_wq", WQ_PERCPU, 0); + if (!device_sriov_wq) { + destroy_workqueue(device_reset_wq); + device_reset_wq = NULL; + return -EFAULT; + } + + return 0; +} + +void adf_exit_aer(void) +{ + if (device_reset_wq) + destroy_workqueue(device_reset_wq); + device_reset_wq = NULL; + + if (device_sriov_wq) + destroy_workqueue(device_sriov_wq); + device_sriov_wq = NULL; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_anti_rb.c b/drivers/crypto/intel/qat/qat_common/adf_anti_rb.c new file mode 100644 index 000000000..2c19a82d8 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_anti_rb.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2026 Intel Corporation */ +#include +#include +#include +#include + +#include "adf_accel_devices.h" +#include "adf_admin.h" +#include "adf_anti_rb.h" +#include "adf_common_drv.h" +#include "icp_qat_fw_init_admin.h" + +#define ADF_SVN_RETRY_MAX 60 + +int adf_anti_rb_commit(struct adf_accel_dev *accel_dev) +{ + return adf_send_admin_arb_commit(accel_dev); +} + +int adf_anti_rb_query(struct adf_accel_dev *accel_dev, enum anti_rb cmd, u8 *svn) +{ + return adf_send_admin_arb_query(accel_dev, cmd, svn); +} + +int adf_anti_rb_check(struct pci_dev *pdev) +{ + struct adf_anti_rb_hw_data *anti_rb; + u32 svncheck_sts, cfc_svncheck_sts; + struct adf_accel_dev *accel_dev; + void __iomem *pmisc_addr; + + accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + if (!accel_dev) + return -EINVAL; + + anti_rb = GET_ANTI_RB_DATA(accel_dev); + if (!anti_rb->anti_rb_enabled || !anti_rb->anti_rb_enabled(accel_dev)) + return 0; + + pmisc_addr = adf_get_pmisc_base(accel_dev); + + cfc_svncheck_sts = ADF_CSR_RD(pmisc_addr, anti_rb->svncheck_offset); + + svncheck_sts = FIELD_GET(ADF_SVN_STS_MASK, cfc_svncheck_sts); + switch (svncheck_sts) { + case ADF_SVN_NO_STS: + return 0; + case ADF_SVN_PASS_STS: + anti_rb->svncheck_retry = 0; + return 0; + case ADF_SVN_FAIL_STS: + dev_err(&GET_DEV(accel_dev), "Security Version Number failure\n"); + return -EIO; + case ADF_SVN_RETRY_STS: + if (anti_rb->svncheck_retry++ >= ADF_SVN_RETRY_MAX) { + anti_rb->svncheck_retry = 0; + return -ETIMEDOUT; + } + msleep(ADF_SVN_RETRY_MS); + return -EAGAIN; + default: + dev_err(&GET_DEV(accel_dev), "Invalid SVN check status\n"); + return -EINVAL; + } +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_anti_rb.h b/drivers/crypto/intel/qat/qat_common/adf_anti_rb.h new file mode 100644 index 000000000..531af41a3 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_anti_rb.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2026 Intel Corporation */ +#ifndef ADF_ANTI_RB_H_ +#define ADF_ANTI_RB_H_ + +#include + +#define GET_ANTI_RB_DATA(accel_dev) (&(accel_dev)->hw_device->anti_rb_data) + +#define ADF_SVN_NO_STS 0x00 +#define ADF_SVN_PASS_STS 0x01 +#define ADF_SVN_RETRY_STS 0x02 +#define ADF_SVN_FAIL_STS 0x03 +#define ADF_SVN_RETRY_MS 250 +#define ADF_SVN_STS_MASK GENMASK(7, 0) + +enum anti_rb { + ARB_ENFORCED_MIN_SVN, + ARB_PERMANENT_MIN_SVN, + ARB_ACTIVE_SVN, +}; + +struct adf_accel_dev; +struct pci_dev; + +struct adf_anti_rb_hw_data { + bool (*anti_rb_enabled)(struct adf_accel_dev *accel_dev); + u32 svncheck_offset; + u32 svncheck_retry; + bool sysfs_added; +}; + +int adf_anti_rb_commit(struct adf_accel_dev *accel_dev); +int adf_anti_rb_query(struct adf_accel_dev *accel_dev, enum anti_rb cmd, u8 *svn); +int adf_anti_rb_check(struct pci_dev *pdev); + +#endif /* ADF_ANTI_RB_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_bank_state.c b/drivers/crypto/intel/qat/qat_common/adf_bank_state.c new file mode 100644 index 000000000..225d55d56 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_bank_state.c @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2025 Intel Corporation */ + +#define pr_fmt(fmt) "QAT: " fmt + +#include +#include +#include +#include "adf_accel_devices.h" +#include "adf_bank_state.h" +#include "adf_common_drv.h" + +/* Ring interrupt masks */ +#define ADF_RP_INT_SRC_SEL_F_RISE_MASK GENMASK(1, 0) +#define ADF_RP_INT_SRC_SEL_F_FALL_MASK GENMASK(2, 0) +#define ADF_RP_INT_SRC_SEL_RANGE_WIDTH 4 + +static inline int check_stat(u32 (*op)(void __iomem *, u32), u32 expect_val, + const char *name, void __iomem *base, u32 bank) +{ + u32 actual_val = op(base, bank); + + if (expect_val == actual_val) + return 0; + + pr_err("Fail to restore %s register. Expected %#x, actual %#x\n", + name, expect_val, actual_val); + + return -EINVAL; +} + +static void bank_state_save(struct adf_hw_csr_ops *ops, void __iomem *base, + u32 bank, struct adf_bank_state *state, u32 num_rings) +{ + u32 i; + + state->ringstat0 = ops->read_csr_stat(base, bank); + state->ringuostat = ops->read_csr_uo_stat(base, bank); + state->ringestat = ops->read_csr_e_stat(base, bank); + state->ringnestat = ops->read_csr_ne_stat(base, bank); + state->ringnfstat = ops->read_csr_nf_stat(base, bank); + state->ringfstat = ops->read_csr_f_stat(base, bank); + state->ringcstat0 = ops->read_csr_c_stat(base, bank); + state->iaintflagen = ops->read_csr_int_en(base, bank); + state->iaintflagreg = ops->read_csr_int_flag(base, bank); + state->iaintflagsrcsel0 = ops->read_csr_int_srcsel(base, bank); + state->iaintcolen = ops->read_csr_int_col_en(base, bank); + state->iaintcolctl = ops->read_csr_int_col_ctl(base, bank); + state->iaintflagandcolen = ops->read_csr_int_flag_and_col(base, bank); + state->ringexpstat = ops->read_csr_exp_stat(base, bank); + state->ringexpintenable = ops->read_csr_exp_int_en(base, bank); + state->ringsrvarben = ops->read_csr_ring_srv_arb_en(base, bank); + + for (i = 0; i < num_rings; i++) { + state->rings[i].head = ops->read_csr_ring_head(base, bank, i); + state->rings[i].tail = ops->read_csr_ring_tail(base, bank, i); + state->rings[i].config = ops->read_csr_ring_config(base, bank, i); + state->rings[i].base = ops->read_csr_ring_base(base, bank, i); + } +} + +static int bank_state_restore(struct adf_hw_csr_ops *ops, void __iomem *base, + u32 bank, struct adf_bank_state *state, u32 num_rings, + int tx_rx_gap) +{ + u32 val, tmp_val, i; + int ret; + + for (i = 0; i < num_rings; i++) + ops->write_csr_ring_base(base, bank, i, state->rings[i].base); + + for (i = 0; i < num_rings; i++) + ops->write_csr_ring_config(base, bank, i, state->rings[i].config); + + for (i = 0; i < num_rings / 2; i++) { + int tx = i * (tx_rx_gap + 1); + int rx = tx + tx_rx_gap; + + ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head); + ops->write_csr_ring_tail(base, bank, tx, state->rings[tx].tail); + + /* + * The TX ring head needs to be updated again to make sure that + * the HW will not consider the ring as full when it is empty + * and the correct state flags are set to match the recovered state. + */ + if (state->ringestat & BIT(tx)) { + val = ops->read_csr_int_srcsel(base, bank); + val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK; + ops->write_csr_int_srcsel_w_val(base, bank, val); + ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head); + } + + ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail); + val = ops->read_csr_int_srcsel(base, bank); + val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH; + ops->write_csr_int_srcsel_w_val(base, bank, val); + + ops->write_csr_ring_head(base, bank, rx, state->rings[rx].head); + val = ops->read_csr_int_srcsel(base, bank); + val |= ADF_RP_INT_SRC_SEL_F_FALL_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH; + ops->write_csr_int_srcsel_w_val(base, bank, val); + + /* + * The RX ring tail needs to be updated again to make sure that + * the HW will not consider the ring as empty when it is full + * and the correct state flags are set to match the recovered state. + */ + if (state->ringfstat & BIT(rx)) + ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail); + } + + ops->write_csr_int_flag_and_col(base, bank, state->iaintflagandcolen); + ops->write_csr_int_en(base, bank, state->iaintflagen); + ops->write_csr_int_col_en(base, bank, state->iaintcolen); + ops->write_csr_int_srcsel_w_val(base, bank, state->iaintflagsrcsel0); + ops->write_csr_exp_int_en(base, bank, state->ringexpintenable); + ops->write_csr_int_col_ctl(base, bank, state->iaintcolctl); + + /* + * Verify whether any exceptions were raised during the bank save process. + * If exceptions occurred, the status and exception registers cannot + * be directly restored. Consequently, further restoration is not + * feasible, and the current state of the ring should be maintained. + */ + val = state->ringexpstat; + if (val) { + pr_info("Bank %u state not fully restored due to exception in saved state (%#x)\n", + bank, val); + return 0; + } + + /* Ensure that the restoration process completed without exceptions */ + tmp_val = ops->read_csr_exp_stat(base, bank); + if (tmp_val) { + pr_err("Bank %u restored with exception: %#x\n", bank, tmp_val); + return -EFAULT; + } + + ops->write_csr_ring_srv_arb_en(base, bank, state->ringsrvarben); + + /* Check that all ring statuses match the saved state. */ + ret = check_stat(ops->read_csr_stat, state->ringstat0, "ringstat", + base, bank); + if (ret) + return ret; + + ret = check_stat(ops->read_csr_e_stat, state->ringestat, "ringestat", + base, bank); + if (ret) + return ret; + + ret = check_stat(ops->read_csr_ne_stat, state->ringnestat, "ringnestat", + base, bank); + if (ret) + return ret; + + ret = check_stat(ops->read_csr_nf_stat, state->ringnfstat, "ringnfstat", + base, bank); + if (ret) + return ret; + + ret = check_stat(ops->read_csr_f_stat, state->ringfstat, "ringfstat", + base, bank); + if (ret) + return ret; + + ret = check_stat(ops->read_csr_c_stat, state->ringcstat0, "ringcstat", + base, bank); + if (ret) + return ret; + + return 0; +} + +/** + * adf_bank_state_save() - save state of bank-related registers + * @accel_dev: Pointer to the device structure + * @bank_number: Bank number + * @state: Pointer to bank state structure + * + * This function saves the state of a bank by reading the bank CSRs and + * writing them in the @state structure. + * + * Returns 0 on success, error code otherwise + */ +int adf_bank_state_save(struct adf_accel_dev *accel_dev, u32 bank_number, + struct adf_bank_state *state) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); + void __iomem *csr_base = adf_get_etr_base(accel_dev); + + if (bank_number >= hw_data->num_banks || !state) + return -EINVAL; + + dev_dbg(&GET_DEV(accel_dev), "Saving state of bank %d\n", bank_number); + + bank_state_save(csr_ops, csr_base, bank_number, state, + hw_data->num_rings_per_bank); + + return 0; +} +EXPORT_SYMBOL_GPL(adf_bank_state_save); + +/** + * adf_bank_state_restore() - restore state of bank-related registers + * @accel_dev: Pointer to the device structure + * @bank_number: Bank number + * @state: Pointer to bank state structure + * + * This function attempts to restore the state of a bank by writing the + * bank CSRs to the values in the state structure. + * + * Returns 0 on success, error code otherwise + */ +int adf_bank_state_restore(struct adf_accel_dev *accel_dev, u32 bank_number, + struct adf_bank_state *state) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); + void __iomem *csr_base = adf_get_etr_base(accel_dev); + int ret; + + if (bank_number >= hw_data->num_banks || !state) + return -EINVAL; + + dev_dbg(&GET_DEV(accel_dev), "Restoring state of bank %d\n", bank_number); + + ret = bank_state_restore(csr_ops, csr_base, bank_number, state, + hw_data->num_rings_per_bank, hw_data->tx_rx_gap); + if (ret) + dev_err(&GET_DEV(accel_dev), + "Unable to restore state of bank %d\n", bank_number); + + return ret; +} +EXPORT_SYMBOL_GPL(adf_bank_state_restore); diff --git a/drivers/crypto/intel/qat/qat_common/adf_bank_state.h b/drivers/crypto/intel/qat/qat_common/adf_bank_state.h new file mode 100644 index 000000000..48b573d69 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_bank_state.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ADF_BANK_STATE_H_ +#define ADF_BANK_STATE_H_ + +#include + +struct adf_accel_dev; + +struct ring_config { + u64 base; + u32 config; + u32 head; + u32 tail; + u32 reserved0; +}; + +struct adf_bank_state { + u32 ringstat0; + u32 ringstat1; + u32 ringuostat; + u32 ringestat; + u32 ringnestat; + u32 ringnfstat; + u32 ringfstat; + u32 ringcstat0; + u32 ringcstat1; + u32 ringcstat2; + u32 ringcstat3; + u32 iaintflagen; + u32 iaintflagreg; + u32 iaintflagsrcsel0; + u32 iaintflagsrcsel1; + u32 iaintcolen; + u32 iaintcolctl; + u32 iaintflagandcolen; + u32 ringexpstat; + u32 ringexpintenable; + u32 ringsrvarben; + u32 reserved0; + struct ring_config rings[ADF_ETR_MAX_RINGS_PER_BANK]; +}; + +int adf_bank_state_restore(struct adf_accel_dev *accel_dev, u32 bank_number, + struct adf_bank_state *state); +int adf_bank_state_save(struct adf_accel_dev *accel_dev, u32 bank_number, + struct adf_bank_state *state); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg.c b/drivers/crypto/intel/qat/qat_common/adf_cfg.c new file mode 100644 index 000000000..b88febf53 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg.c @@ -0,0 +1,372 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#include +#include +#include +#include +#include +#include "adf_accel_devices.h" +#include "adf_cfg.h" +#include "adf_common_drv.h" + +static DEFINE_MUTEX(qat_cfg_read_lock); + +static void *qat_dev_cfg_start(struct seq_file *sfile, loff_t *pos) +{ + struct adf_cfg_device_data *dev_cfg = sfile->private; + + mutex_lock(&qat_cfg_read_lock); + return seq_list_start(&dev_cfg->sec_list, *pos); +} + +static int qat_dev_cfg_show(struct seq_file *sfile, void *v) +{ + struct list_head *list; + struct adf_cfg_section *sec = + list_entry(v, struct adf_cfg_section, list); + + seq_printf(sfile, "[%s]\n", sec->name); + list_for_each(list, &sec->param_head) { + struct adf_cfg_key_val *ptr = + list_entry(list, struct adf_cfg_key_val, list); + seq_printf(sfile, "%s = %s\n", ptr->key, ptr->val); + } + return 0; +} + +static void *qat_dev_cfg_next(struct seq_file *sfile, void *v, loff_t *pos) +{ + struct adf_cfg_device_data *dev_cfg = sfile->private; + + return seq_list_next(v, &dev_cfg->sec_list, pos); +} + +static void qat_dev_cfg_stop(struct seq_file *sfile, void *v) +{ + mutex_unlock(&qat_cfg_read_lock); +} + +static const struct seq_operations qat_dev_cfg_sops = { + .start = qat_dev_cfg_start, + .next = qat_dev_cfg_next, + .stop = qat_dev_cfg_stop, + .show = qat_dev_cfg_show +}; + +DEFINE_SEQ_ATTRIBUTE(qat_dev_cfg); + +/** + * adf_cfg_dev_add() - Create an acceleration device configuration table. + * @accel_dev: Pointer to acceleration device. + * + * Function creates a configuration table for the given acceleration device. + * The table stores device specific config values. + * To be used by QAT device specific drivers. + * + * Return: 0 on success, error code otherwise. + */ +int adf_cfg_dev_add(struct adf_accel_dev *accel_dev) +{ + struct adf_cfg_device_data *dev_cfg_data; + + dev_cfg_data = kzalloc_obj(*dev_cfg_data); + if (!dev_cfg_data) + return -ENOMEM; + INIT_LIST_HEAD(&dev_cfg_data->sec_list); + init_rwsem(&dev_cfg_data->lock); + accel_dev->cfg = dev_cfg_data; + return 0; +} +EXPORT_SYMBOL_GPL(adf_cfg_dev_add); + +void adf_cfg_dev_dbgfs_add(struct adf_accel_dev *accel_dev) +{ + struct adf_cfg_device_data *dev_cfg_data = accel_dev->cfg; + + dev_cfg_data->debug = debugfs_create_file("dev_cfg", 0400, + accel_dev->debugfs_dir, + dev_cfg_data, + &qat_dev_cfg_fops); +} + +void adf_cfg_dev_dbgfs_rm(struct adf_accel_dev *accel_dev) +{ + struct adf_cfg_device_data *dev_cfg_data = accel_dev->cfg; + + if (!dev_cfg_data) + return; + + debugfs_remove(dev_cfg_data->debug); + dev_cfg_data->debug = NULL; +} + +static void adf_cfg_section_del_all(struct list_head *head); +static void adf_cfg_section_del_all_except(struct list_head *head, + const char *section_name); + +void adf_cfg_del_all_except(struct adf_accel_dev *accel_dev, + const char *section_name) +{ + struct adf_cfg_device_data *dev_cfg_data = accel_dev->cfg; + + down_write(&dev_cfg_data->lock); + adf_cfg_section_del_all_except(&dev_cfg_data->sec_list, section_name); + up_write(&dev_cfg_data->lock); + clear_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); +} + +/** + * adf_cfg_dev_remove() - Clears acceleration device configuration table. + * @accel_dev: Pointer to acceleration device. + * + * Function removes configuration table from the given acceleration device + * and frees all allocated memory. + * To be used by QAT device specific drivers. + * + * Return: void + */ +void adf_cfg_dev_remove(struct adf_accel_dev *accel_dev) +{ + struct adf_cfg_device_data *dev_cfg_data = accel_dev->cfg; + + if (!dev_cfg_data) + return; + + down_write(&dev_cfg_data->lock); + adf_cfg_section_del_all(&dev_cfg_data->sec_list); + up_write(&dev_cfg_data->lock); + kfree(dev_cfg_data); + accel_dev->cfg = NULL; +} +EXPORT_SYMBOL_GPL(adf_cfg_dev_remove); + +static void adf_cfg_keyval_add(struct adf_cfg_key_val *new, + struct adf_cfg_section *sec) +{ + list_add_tail(&new->list, &sec->param_head); +} + +static void adf_cfg_keyval_remove(const char *key, struct adf_cfg_section *sec) +{ + struct list_head *head = &sec->param_head; + struct list_head *list_ptr, *tmp; + + list_for_each_prev_safe(list_ptr, tmp, head) { + struct adf_cfg_key_val *ptr = + list_entry(list_ptr, struct adf_cfg_key_val, list); + + if (strncmp(ptr->key, key, sizeof(ptr->key))) + continue; + + list_del(list_ptr); + kfree(ptr); + break; + } +} + +static void adf_cfg_keyval_del_all(struct list_head *head) +{ + struct list_head *list_ptr, *tmp; + + list_for_each_prev_safe(list_ptr, tmp, head) { + struct adf_cfg_key_val *ptr = + list_entry(list_ptr, struct adf_cfg_key_val, list); + list_del(list_ptr); + kfree(ptr); + } +} + +static void adf_cfg_section_del_all(struct list_head *head) +{ + struct adf_cfg_section *ptr; + struct list_head *list, *tmp; + + list_for_each_prev_safe(list, tmp, head) { + ptr = list_entry(list, struct adf_cfg_section, list); + adf_cfg_keyval_del_all(&ptr->param_head); + list_del(list); + kfree(ptr); + } +} + +static void adf_cfg_section_del_all_except(struct list_head *head, + const char *section_name) +{ + struct list_head *list, *tmp; + struct adf_cfg_section *ptr; + + list_for_each_prev_safe(list, tmp, head) { + ptr = list_entry(list, struct adf_cfg_section, list); + if (!strcmp(ptr->name, section_name)) + continue; + adf_cfg_keyval_del_all(&ptr->param_head); + list_del(list); + kfree(ptr); + } +} + +static struct adf_cfg_key_val *adf_cfg_key_value_find(struct adf_cfg_section *s, + const char *key) +{ + struct list_head *list; + + list_for_each(list, &s->param_head) { + struct adf_cfg_key_val *ptr = + list_entry(list, struct adf_cfg_key_val, list); + if (!strcmp(ptr->key, key)) + return ptr; + } + return NULL; +} + +static struct adf_cfg_section *adf_cfg_sec_find(struct adf_accel_dev *accel_dev, + const char *sec_name) +{ + struct adf_cfg_device_data *cfg = accel_dev->cfg; + struct list_head *list; + + list_for_each(list, &cfg->sec_list) { + struct adf_cfg_section *ptr = + list_entry(list, struct adf_cfg_section, list); + if (!strcmp(ptr->name, sec_name)) + return ptr; + } + return NULL; +} + +static int adf_cfg_key_val_get(struct adf_accel_dev *accel_dev, + const char *sec_name, + const char *key_name, + char *val) +{ + struct adf_cfg_section *sec = adf_cfg_sec_find(accel_dev, sec_name); + struct adf_cfg_key_val *keyval = NULL; + + if (sec) + keyval = adf_cfg_key_value_find(sec, key_name); + if (keyval) { + memcpy(val, keyval->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES); + return 0; + } + return -ENODATA; +} + +/** + * adf_cfg_add_key_value_param() - Add key-value config entry to config table. + * @accel_dev: Pointer to acceleration device. + * @section_name: Name of the section where the param will be added + * @key: The key string + * @val: Value pain for the given @key + * @type: Type - string, int or address + * + * Function adds configuration key - value entry in the appropriate section + * in the given acceleration device. If the key exists already, the value + * is updated. + * To be used by QAT device specific drivers. + * + * Return: 0 on success, error code otherwise. + */ +int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev, + const char *section_name, + const char *key, const void *val, + enum adf_cfg_val_type type) +{ + struct adf_cfg_device_data *cfg = accel_dev->cfg; + struct adf_cfg_key_val *key_val; + struct adf_cfg_section *section = adf_cfg_sec_find(accel_dev, + section_name); + char temp_val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + + if (!section) + return -EFAULT; + + key_val = kzalloc_obj(*key_val); + if (!key_val) + return -ENOMEM; + + INIT_LIST_HEAD(&key_val->list); + strscpy(key_val->key, key); + + if (type == ADF_DEC) { + snprintf(key_val->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES, + "%ld", (*((long *)val))); + } else if (type == ADF_STR) { + strscpy(key_val->val, (char *)val); + } else { + dev_err(&GET_DEV(accel_dev), "Unknown type given.\n"); + kfree(key_val); + return -EINVAL; + } + key_val->type = type; + + /* Add the key-value pair as below policy: + * 1. if the key doesn't exist, add it; + * 2. if the key already exists with a different value then update it + * to the new value (the key is deleted and the newly created + * key_val containing the new value is added to the database); + * 3. if the key exists with the same value, then return without doing + * anything (the newly created key_val is freed). + */ + down_write(&cfg->lock); + if (!adf_cfg_key_val_get(accel_dev, section_name, key, temp_val)) { + if (strncmp(temp_val, key_val->val, sizeof(temp_val))) { + adf_cfg_keyval_remove(key, section); + } else { + kfree(key_val); + goto out; + } + } + + adf_cfg_keyval_add(key_val, section); + +out: + up_write(&cfg->lock); + return 0; +} +EXPORT_SYMBOL_GPL(adf_cfg_add_key_value_param); + +/** + * adf_cfg_section_add() - Add config section entry to config table. + * @accel_dev: Pointer to acceleration device. + * @name: Name of the section + * + * Function adds configuration section where key - value entries + * will be stored. + * To be used by QAT device specific drivers. + * + * Return: 0 on success, error code otherwise. + */ +int adf_cfg_section_add(struct adf_accel_dev *accel_dev, const char *name) +{ + struct adf_cfg_device_data *cfg = accel_dev->cfg; + struct adf_cfg_section *sec = adf_cfg_sec_find(accel_dev, name); + + if (sec) + return 0; + + sec = kzalloc_obj(*sec); + if (!sec) + return -ENOMEM; + + strscpy(sec->name, name); + INIT_LIST_HEAD(&sec->param_head); + down_write(&cfg->lock); + list_add_tail(&sec->list, &cfg->sec_list); + up_write(&cfg->lock); + return 0; +} +EXPORT_SYMBOL_GPL(adf_cfg_section_add); + +int adf_cfg_get_param_value(struct adf_accel_dev *accel_dev, + const char *section, const char *name, + char *value) +{ + struct adf_cfg_device_data *cfg = accel_dev->cfg; + int ret; + + down_read(&cfg->lock); + ret = adf_cfg_key_val_get(accel_dev, section, name, value); + up_read(&cfg->lock); + return ret; +} +EXPORT_SYMBOL_GPL(adf_cfg_get_param_value); diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg.h b/drivers/crypto/intel/qat/qat_common/adf_cfg.h new file mode 100644 index 000000000..108032b39 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#ifndef ADF_CFG_H_ +#define ADF_CFG_H_ + +#include +#include +#include +#include "adf_accel_devices.h" +#include "adf_cfg_common.h" +#include "adf_cfg_strings.h" + +struct adf_cfg_key_val { + char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + enum adf_cfg_val_type type; + struct list_head list; +}; + +struct adf_cfg_section { + char name[ADF_CFG_MAX_SECTION_LEN_IN_BYTES]; + struct list_head list; + struct list_head param_head; +}; + +struct adf_cfg_device_data { + struct list_head sec_list; + struct dentry *debug; + struct rw_semaphore lock; +}; + +int adf_cfg_dev_add(struct adf_accel_dev *accel_dev); +void adf_cfg_dev_remove(struct adf_accel_dev *accel_dev); +void adf_cfg_dev_dbgfs_add(struct adf_accel_dev *accel_dev); +void adf_cfg_dev_dbgfs_rm(struct adf_accel_dev *accel_dev); +int adf_cfg_section_add(struct adf_accel_dev *accel_dev, const char *name); +void adf_cfg_del_all_except(struct adf_accel_dev *accel_dev, + const char *section_name); +int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev, + const char *section_name, + const char *key, const void *val, + enum adf_cfg_val_type type); +int adf_cfg_get_param_value(struct adf_accel_dev *accel_dev, + const char *section, const char *name, char *value); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h b/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h new file mode 100644 index 000000000..5922d018f --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#ifndef ADF_CFG_COMMON_H_ +#define ADF_CFG_COMMON_H_ + +#include + +#define ADF_CFG_MAX_STR_LEN 64 +#define ADF_CFG_MAX_KEY_LEN_IN_BYTES ADF_CFG_MAX_STR_LEN +#define ADF_CFG_MAX_VAL_LEN_IN_BYTES ADF_CFG_MAX_STR_LEN +#define ADF_CFG_MAX_SECTION_LEN_IN_BYTES ADF_CFG_MAX_STR_LEN +#define ADF_MAX_DEVICES (32 * 32) +#define ADF_DEVS_ARRAY_SIZE BITS_TO_LONGS(ADF_MAX_DEVICES) + +#define ADF_CFG_SERV_RING_PAIR_0_SHIFT 0 +#define ADF_CFG_SERV_RING_PAIR_1_SHIFT 3 +#define ADF_CFG_SERV_RING_PAIR_2_SHIFT 6 +#define ADF_CFG_SERV_RING_PAIR_3_SHIFT 9 +enum adf_cfg_service_type { + UNUSED = 0, + CRYPTO, + COMP, + SYM, + ASYM, + DECOMP, + USED +}; + +enum adf_cfg_val_type { + ADF_DEC, + ADF_STR +}; + +enum adf_device_type { + DEV_UNKNOWN = 0, + DEV_DH895XCC, + DEV_DH895XCCVF, + DEV_C62X, + DEV_C62XVF, + DEV_C3XXX, + DEV_C3XXXVF, + DEV_4XXX, + DEV_420XX, + DEV_6XXX, +}; +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c new file mode 100644 index 000000000..f7237c7ef --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ + +#include +#include +#include +#include +#include +#include "adf_cfg.h" +#include "adf_cfg_common.h" +#include "adf_cfg_services.h" +#include "adf_cfg_strings.h" + +static const char *const adf_cfg_services[] = { + [SVC_ASYM] = ADF_CFG_ASYM, + [SVC_SYM] = ADF_CFG_SYM, + [SVC_DC] = ADF_CFG_DC, + [SVC_DCC] = ADF_CFG_DCC, + [SVC_DECOMP] = ADF_CFG_DECOMP, +}; + +/* + * Ensure that the size of the array matches the number of services, + * SVC_COUNT, that is used to size the bitmap. + */ +static_assert(ARRAY_SIZE(adf_cfg_services) == SVC_COUNT); + +/* + * Ensure that the maximum number of concurrent services that can be + * enabled on a device is less than or equal to the number of total + * supported services. + */ +static_assert(ARRAY_SIZE(adf_cfg_services) >= MAX_NUM_CONCURR_SVC); + +/* + * Ensure that the number of services fit a single unsigned long, as each + * service is represented by a bit in the mask. + */ +static_assert(BITS_PER_LONG >= SVC_COUNT); + +/* + * Ensure that size of the concatenation of all service strings is smaller + * than the size of the buffer that will contain them. + */ +static_assert(sizeof(ADF_CFG_SYM ADF_SERVICES_DELIMITER + ADF_CFG_ASYM ADF_SERVICES_DELIMITER + ADF_CFG_DC ADF_SERVICES_DELIMITER + ADF_CFG_DECOMP ADF_SERVICES_DELIMITER + ADF_CFG_DCC) < ADF_CFG_MAX_VAL_LEN_IN_BYTES); + +static int adf_service_string_to_mask(struct adf_accel_dev *accel_dev, const char *buf, + unsigned long *out_mask) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + unsigned long mask = 0; + char *substr, *token; + int id, num_svc = 0; + + if (strscpy_pad(services, buf) < 0) + return -EINVAL; + + substr = services; + + while ((token = strsep(&substr, ADF_SERVICES_DELIMITER))) { + id = sysfs_match_string(adf_cfg_services, token); + if (id < 0) + return id; + + if (test_and_set_bit(id, &mask)) + return -EINVAL; + + if (num_svc++ == MAX_NUM_CONCURR_SVC) + return -EINVAL; + } + + if (hw_data->services_supported && !hw_data->services_supported(mask)) + return -EINVAL; + + *out_mask = mask; + + return 0; +} + +static int adf_service_mask_to_string(unsigned long mask, char *buf, size_t len) +{ + int offset = 0; + int bit; + + if (len < ADF_CFG_MAX_VAL_LEN_IN_BYTES) + return -ENOSPC; + + for_each_set_bit(bit, &mask, SVC_COUNT) { + if (offset) + offset += scnprintf(buf + offset, len - offset, + ADF_SERVICES_DELIMITER "%s", adf_cfg_services[bit]); + else + offset += scnprintf(buf, len, "%s", adf_cfg_services[bit]); + } + + return 0; +} + +int adf_parse_service_string(struct adf_accel_dev *accel_dev, const char *in, + char *out, size_t out_len) +{ + unsigned long mask; + int ret; + + ret = adf_service_string_to_mask(accel_dev, in, &mask); + if (ret) + return ret; + + if (!mask) + return -EINVAL; + + return adf_service_mask_to_string(mask, out, out_len); +} + +int adf_get_service_mask(struct adf_accel_dev *accel_dev, unsigned long *mask) +{ + char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { }; + int ret; + + ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, + ADF_SERVICES_ENABLED, services); + if (ret) { + dev_err(&GET_DEV(accel_dev), "%s param not found\n", + ADF_SERVICES_ENABLED); + return ret; + } + + ret = adf_service_string_to_mask(accel_dev, services, mask); + if (ret) + dev_err(&GET_DEV(accel_dev), "Invalid value of %s param: %s\n", + ADF_SERVICES_ENABLED, services); + + return ret; +} +EXPORT_SYMBOL_GPL(adf_get_service_mask); + +int adf_get_service_enabled(struct adf_accel_dev *accel_dev) +{ + unsigned long mask; + int ret; + + ret = adf_get_service_mask(accel_dev, &mask); + if (ret) + return ret; + + if (test_bit(SVC_SYM, &mask) && test_bit(SVC_ASYM, &mask)) + return SVC_SYM_ASYM; + + if (test_bit(SVC_SYM, &mask) && test_bit(SVC_DC, &mask)) + return SVC_SYM_DC; + + if (test_bit(SVC_ASYM, &mask) && test_bit(SVC_DC, &mask)) + return SVC_ASYM_DC; + + if (test_bit(SVC_SYM, &mask)) + return SVC_SYM; + + if (test_bit(SVC_ASYM, &mask)) + return SVC_ASYM; + + if (test_bit(SVC_DC, &mask)) + return SVC_DC; + + if (test_bit(SVC_DECOMP, &mask)) + return SVC_DECOMP; + + if (test_bit(SVC_DCC, &mask)) + return SVC_DCC; + + return -EINVAL; +} +EXPORT_SYMBOL_GPL(adf_get_service_enabled); + +enum adf_cfg_service_type adf_srv_to_cfg_svc_type(enum adf_base_services svc) +{ + switch (svc) { + case SVC_ASYM: + return ASYM; + case SVC_SYM: + return SYM; + case SVC_DC: + return COMP; + case SVC_DECOMP: + return DECOMP; + default: + return UNUSED; + } +} + +bool adf_is_service_enabled(struct adf_accel_dev *accel_dev, enum adf_base_services svc) +{ + enum adf_cfg_service_type arb_srv = adf_srv_to_cfg_svc_type(svc); + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + u8 rps_per_bundle = hw_data->num_banks_per_vf; + int i; + + for (i = 0; i < rps_per_bundle; i++) { + if (GET_SRV_TYPE(accel_dev, i) == arb_srv) + return true; + } + + return false; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.h b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.h new file mode 100644 index 000000000..89be2f2c7 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ +#ifndef _ADF_CFG_SERVICES_H_ +#define _ADF_CFG_SERVICES_H_ + +#include "adf_cfg_strings.h" + +struct adf_accel_dev; + +enum adf_base_services { + SVC_ASYM = 0, + SVC_SYM, + SVC_DC, + SVC_DECOMP, + SVC_BASE_COUNT +}; + +enum adf_extended_services { + SVC_DCC = SVC_BASE_COUNT, + SVC_COUNT +}; + +enum adf_composed_services { + SVC_SYM_ASYM = SVC_COUNT, + SVC_SYM_DC, + SVC_ASYM_DC, +}; + +enum { + ADF_ONE_SERVICE = 1, + ADF_TWO_SERVICES, + ADF_THREE_SERVICES, +}; + +#define MAX_NUM_CONCURR_SVC ADF_THREE_SERVICES + +int adf_parse_service_string(struct adf_accel_dev *accel_dev, const char *in, + char *out, size_t out_len); +int adf_get_service_enabled(struct adf_accel_dev *accel_dev); +int adf_get_service_mask(struct adf_accel_dev *accel_dev, unsigned long *mask); +enum adf_cfg_service_type adf_srv_to_cfg_svc_type(enum adf_base_services svc); +bool adf_is_service_enabled(struct adf_accel_dev *accel_dev, enum adf_base_services svc); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_strings.h b/drivers/crypto/intel/qat/qat_common/adf_cfg_strings.h new file mode 100644 index 000000000..30107a02e --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_strings.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#ifndef ADF_CFG_STRINGS_H_ +#define ADF_CFG_STRINGS_H_ + +#define ADF_GENERAL_SEC "GENERAL" +#define ADF_KERNEL_SEC "KERNEL" +#define ADF_ACCEL_SEC "Accelerator" +#define ADF_NUM_CY "NumberCyInstances" +#define ADF_NUM_DC "NumberDcInstances" +#define ADF_RING_SYM_SIZE "NumConcurrentSymRequests" +#define ADF_RING_ASYM_SIZE "NumConcurrentAsymRequests" +#define ADF_RING_DC_SIZE "NumConcurrentRequests" +#define ADF_RING_ASYM_TX "RingAsymTx" +#define ADF_RING_SYM_TX "RingSymTx" +#define ADF_RING_ASYM_RX "RingAsymRx" +#define ADF_RING_SYM_RX "RingSymRx" +#define ADF_RING_DC_TX "RingTx" +#define ADF_RING_DC_RX "RingRx" +#define ADF_ETRMGR_BANK "Bank" +#define ADF_RING_SYM_BANK_NUM "BankSymNumber" +#define ADF_RING_ASYM_BANK_NUM "BankAsymNumber" +#define ADF_RING_DC_BANK_NUM "BankDcNumber" +#define ADF_CY "Cy" +#define ADF_DC "Dc" +#define ADF_CFG_DC "dc" +#define ADF_CFG_DECOMP "decomp" +#define ADF_CFG_CY "sym;asym" +#define ADF_CFG_SYM "sym" +#define ADF_CFG_ASYM "asym" +#define ADF_CFG_DCC "dcc" +#define ADF_SERVICES_ENABLED "ServicesEnabled" +#define ADF_SERVICES_DELIMITER ";" +#define ADF_PM_IDLE_SUPPORT "PmIdleSupport" +#define ADF_ETRMGR_COALESCING_ENABLED "InterruptCoalescingEnabled" +#define ADF_ETRMGR_COALESCING_ENABLED_FORMAT \ + ADF_ETRMGR_BANK "%d" ADF_ETRMGR_COALESCING_ENABLED +#define ADF_ETRMGR_COALESCE_TIMER "InterruptCoalescingTimerNs" +#define ADF_ETRMGR_COALESCE_TIMER_FORMAT \ + ADF_ETRMGR_BANK "%d" ADF_ETRMGR_COALESCE_TIMER +#define ADF_ETRMGR_COALESCING_MSG_ENABLED "InterruptCoalescingNumResponses" +#define ADF_ETRMGR_COALESCING_MSG_ENABLED_FORMAT \ + ADF_ETRMGR_BANK "%d" ADF_ETRMGR_COALESCING_MSG_ENABLED +#define ADF_ETRMGR_CORE_AFFINITY "CoreAffinity" +#define ADF_ETRMGR_CORE_AFFINITY_FORMAT \ + ADF_ETRMGR_BANK "%d" ADF_ETRMGR_CORE_AFFINITY +#define ADF_ACCEL_STR "Accelerator%d" +#define ADF_HEARTBEAT_TIMER "HeartbeatTimer" +#define ADF_SRIOV_ENABLED "SriovEnabled" + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_clock.c b/drivers/crypto/intel/qat/qat_common/adf_clock.c new file mode 100644 index 000000000..cf89f57de --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_clock.c @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "adf_admin.h" +#include "adf_accel_devices.h" +#include "adf_clock.h" +#include "adf_common_drv.h" + +#define MEASURE_CLOCK_RETRIES 10 +#define MEASURE_CLOCK_DELAY_US 10000 +#define ME_CLK_DIVIDER 16 +#define MEASURE_CLOCK_DELTA_THRESHOLD_US 100 + +static inline u64 timespec_to_us(const struct timespec64 *ts) +{ + return (u64)DIV_ROUND_CLOSEST_ULL(timespec64_to_ns(ts), NSEC_PER_USEC); +} + +static inline u64 timespec_to_ms(const struct timespec64 *ts) +{ + return (u64)DIV_ROUND_CLOSEST_ULL(timespec64_to_ns(ts), NSEC_PER_MSEC); +} + +u64 adf_clock_get_current_time(void) +{ + struct timespec64 ts; + + ktime_get_real_ts64(&ts); + return timespec_to_ms(&ts); +} + +static int measure_clock(struct adf_accel_dev *accel_dev, u32 *frequency) +{ + struct timespec64 ts1, ts2, ts3, ts4; + u64 timestamp1, timestamp2, temp; + u32 delta_us, tries; + int ret; + + tries = MEASURE_CLOCK_RETRIES; + do { + ktime_get_real_ts64(&ts1); + ret = adf_get_fw_timestamp(accel_dev, ×tamp1); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to get fw timestamp\n"); + return ret; + } + ktime_get_real_ts64(&ts2); + delta_us = timespec_to_us(&ts2) - timespec_to_us(&ts1); + } while (delta_us > MEASURE_CLOCK_DELTA_THRESHOLD_US && --tries); + + if (!tries) { + dev_err(&GET_DEV(accel_dev), "Excessive clock measure delay\n"); + return -ETIMEDOUT; + } + + fsleep(MEASURE_CLOCK_DELAY_US); + + tries = MEASURE_CLOCK_RETRIES; + do { + ktime_get_real_ts64(&ts3); + if (adf_get_fw_timestamp(accel_dev, ×tamp2)) { + dev_err(&GET_DEV(accel_dev), + "Failed to get fw timestamp\n"); + return -EIO; + } + ktime_get_real_ts64(&ts4); + delta_us = timespec_to_us(&ts4) - timespec_to_us(&ts3); + } while (delta_us > MEASURE_CLOCK_DELTA_THRESHOLD_US && --tries); + + if (!tries) { + dev_err(&GET_DEV(accel_dev), "Excessive clock measure delay\n"); + return -ETIMEDOUT; + } + + delta_us = timespec_to_us(&ts3) - timespec_to_us(&ts1); + if (!delta_us) + return -EINVAL; + + temp = (timestamp2 - timestamp1) * ME_CLK_DIVIDER * 10; + temp = DIV_ROUND_CLOSEST_ULL(temp, delta_us); + /* + * Enclose the division to allow the preprocessor to precalculate it, + * and avoid promoting r-value to 64-bit before division. + */ + *frequency = temp * (HZ_PER_MHZ / 10); + + return 0; +} + +/** + * adf_dev_measure_clock() - measures device clock frequency + * @accel_dev: Pointer to acceleration device. + * @frequency: Pointer to variable where result will be stored + * @min: Minimal allowed frequency value + * @max: Maximal allowed frequency value + * + * If the measurement result will go beyond the min/max thresholds the value + * will take the value of the crossed threshold. + * + * This algorithm compares the device firmware timestamp with the kernel + * timestamp. So we can't expect too high accuracy from this measurement. + * + * Return: + * * 0 - measurement succeed + * * -ETIMEDOUT - measurement failed + */ +int adf_dev_measure_clock(struct adf_accel_dev *accel_dev, + u32 *frequency, u32 min, u32 max) +{ + int ret; + u32 freq; + + ret = measure_clock(accel_dev, &freq); + if (ret) + return ret; + + *frequency = clamp(freq, min, max); + + if (*frequency != freq) + dev_warn(&GET_DEV(accel_dev), + "Measured clock %d Hz is out of range, assuming %d\n", + freq, *frequency); + return 0; +} +EXPORT_SYMBOL_GPL(adf_dev_measure_clock); diff --git a/drivers/crypto/intel/qat/qat_common/adf_clock.h b/drivers/crypto/intel/qat/qat_common/adf_clock.h new file mode 100644 index 000000000..e309bc0dc --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_clock.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ +#ifndef ADF_CLOCK_H +#define ADF_CLOCK_H + +#include + +struct adf_accel_dev; + +int adf_dev_measure_clock(struct adf_accel_dev *accel_dev, u32 *frequency, + u32 min, u32 max); +u64 adf_clock_get_current_time(void); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_cnv_dbgfs.c b/drivers/crypto/intel/qat/qat_common/adf_cnv_dbgfs.c new file mode 100644 index 000000000..627953a72 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_cnv_dbgfs.c @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ + +#include +#include +#include + +#include "adf_accel_devices.h" +#include "adf_admin.h" +#include "adf_common_drv.h" +#include "adf_cnv_dbgfs.h" +#include "qat_compression.h" + +#define CNV_DEBUGFS_FILENAME "cnv_errors" +#define CNV_MIN_PADDING 16 + +#define CNV_ERR_INFO_MASK GENMASK(11, 0) +#define CNV_ERR_TYPE_MASK GENMASK(15, 12) +#define CNV_SLICE_ERR_SIGN_BIT_INDEX 7 +#define CNV_DELTA_ERR_SIGN_BIT_INDEX 11 + +enum cnv_error_type { + CNV_ERR_TYPE_NONE, + CNV_ERR_TYPE_CHECKSUM, + CNV_ERR_TYPE_DECOMP_PRODUCED_LENGTH, + CNV_ERR_TYPE_DECOMPRESSION, + CNV_ERR_TYPE_TRANSLATION, + CNV_ERR_TYPE_DECOMP_CONSUMED_LENGTH, + CNV_ERR_TYPE_UNKNOWN, + CNV_ERR_TYPES_COUNT +}; + +#define CNV_ERROR_TYPE_GET(latest_err) \ + min_t(u16, u16_get_bits(latest_err, CNV_ERR_TYPE_MASK), CNV_ERR_TYPE_UNKNOWN) + +#define CNV_GET_DELTA_ERR_INFO(latest_error) \ + sign_extend32(latest_error, CNV_DELTA_ERR_SIGN_BIT_INDEX) + +#define CNV_GET_SLICE_ERR_INFO(latest_error) \ + sign_extend32(latest_error, CNV_SLICE_ERR_SIGN_BIT_INDEX) + +#define CNV_GET_DEFAULT_ERR_INFO(latest_error) \ + u16_get_bits(latest_error, CNV_ERR_INFO_MASK) + +enum cnv_fields { + CNV_ERR_COUNT, + CNV_LATEST_ERR, + CNV_FIELDS_COUNT +}; + +static const char * const cnv_field_names[CNV_FIELDS_COUNT] = { + [CNV_ERR_COUNT] = "Total Errors", + [CNV_LATEST_ERR] = "Last Error", +}; + +static const char * const cnv_error_names[CNV_ERR_TYPES_COUNT] = { + [CNV_ERR_TYPE_NONE] = "No Error", + [CNV_ERR_TYPE_CHECKSUM] = "Checksum Error", + [CNV_ERR_TYPE_DECOMP_PRODUCED_LENGTH] = "Length Error-P", + [CNV_ERR_TYPE_DECOMPRESSION] = "Decomp Error", + [CNV_ERR_TYPE_TRANSLATION] = "Xlat Error", + [CNV_ERR_TYPE_DECOMP_CONSUMED_LENGTH] = "Length Error-C", + [CNV_ERR_TYPE_UNKNOWN] = "Unknown Error", +}; + +struct ae_cnv_errors { + u16 ae; + u16 err_cnt; + u16 latest_err; + bool is_comp_ae; +}; + +struct cnv_err_stats { + u16 ae_count; + struct ae_cnv_errors ae_cnv_errors[]; +}; + +static s16 get_err_info(u8 error_type, u16 latest) +{ + switch (error_type) { + case CNV_ERR_TYPE_DECOMP_PRODUCED_LENGTH: + case CNV_ERR_TYPE_DECOMP_CONSUMED_LENGTH: + return CNV_GET_DELTA_ERR_INFO(latest); + case CNV_ERR_TYPE_DECOMPRESSION: + case CNV_ERR_TYPE_TRANSLATION: + return CNV_GET_SLICE_ERR_INFO(latest); + default: + return CNV_GET_DEFAULT_ERR_INFO(latest); + } +} + +static void *qat_cnv_errors_seq_start(struct seq_file *sfile, loff_t *pos) +{ + struct cnv_err_stats *err_stats = sfile->private; + + if (*pos == 0) + return SEQ_START_TOKEN; + + if (*pos > err_stats->ae_count) + return NULL; + + return &err_stats->ae_cnv_errors[*pos - 1]; +} + +static void *qat_cnv_errors_seq_next(struct seq_file *sfile, void *v, + loff_t *pos) +{ + struct cnv_err_stats *err_stats = sfile->private; + + (*pos)++; + + if (*pos > err_stats->ae_count) + return NULL; + + return &err_stats->ae_cnv_errors[*pos - 1]; +} + +static void qat_cnv_errors_seq_stop(struct seq_file *sfile, void *v) +{ +} + +static int qat_cnv_errors_seq_show(struct seq_file *sfile, void *v) +{ + struct ae_cnv_errors *ae_errors; + unsigned int i; + s16 err_info; + u8 err_type; + + if (v == SEQ_START_TOKEN) { + seq_puts(sfile, "AE "); + for (i = 0; i < CNV_FIELDS_COUNT; ++i) + seq_printf(sfile, " %*s", CNV_MIN_PADDING, + cnv_field_names[i]); + } else { + ae_errors = v; + + if (!ae_errors->is_comp_ae) + return 0; + + err_type = CNV_ERROR_TYPE_GET(ae_errors->latest_err); + err_info = get_err_info(err_type, ae_errors->latest_err); + + seq_printf(sfile, "%d:", ae_errors->ae); + seq_printf(sfile, " %*d", CNV_MIN_PADDING, ae_errors->err_cnt); + seq_printf(sfile, "%*s [%d]", CNV_MIN_PADDING, + cnv_error_names[err_type], err_info); + } + seq_putc(sfile, '\n'); + + return 0; +} + +static const struct seq_operations qat_cnv_errors_sops = { + .start = qat_cnv_errors_seq_start, + .next = qat_cnv_errors_seq_next, + .stop = qat_cnv_errors_seq_stop, + .show = qat_cnv_errors_seq_show, +}; + +/** + * cnv_err_stats_alloc() - Get CNV stats for the provided device. + * @accel_dev: Pointer to a QAT acceleration device + * + * Allocates and populates table of CNV errors statistics for each non-admin AE + * available through the supplied acceleration device. The caller becomes the + * owner of such memory and is responsible for the deallocation through a call + * to kfree(). + * + * Returns: a pointer to a dynamically allocated struct cnv_err_stats on success + * or a negative value on error. + */ +static struct cnv_err_stats *cnv_err_stats_alloc(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + struct cnv_err_stats *err_stats; + unsigned long ae_count; + unsigned long ae_mask; + size_t err_stats_size; + unsigned long ae; + unsigned int i; + u16 latest_err; + u16 err_cnt; + int ret; + + if (!adf_dev_started(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "QAT Device not started\n"); + return ERR_PTR(-EBUSY); + } + + /* Ignore the admin AEs */ + ae_mask = hw_data->ae_mask & ~hw_data->admin_ae_mask; + ae_count = hweight_long(ae_mask); + if (unlikely(!ae_count)) + return ERR_PTR(-EINVAL); + + err_stats_size = struct_size(err_stats, ae_cnv_errors, ae_count); + err_stats = kmalloc(err_stats_size, GFP_KERNEL); + if (!err_stats) + return ERR_PTR(-ENOMEM); + + err_stats->ae_count = ae_count; + + i = 0; + for_each_set_bit(ae, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) { + ret = adf_get_cnv_stats(accel_dev, ae, &err_cnt, &latest_err); + if (ret) { + dev_dbg(&GET_DEV(accel_dev), + "Failed to get CNV stats for ae %ld, [%d].\n", + ae, ret); + err_stats->ae_cnv_errors[i++].is_comp_ae = false; + continue; + } + err_stats->ae_cnv_errors[i].is_comp_ae = true; + err_stats->ae_cnv_errors[i].latest_err = latest_err; + err_stats->ae_cnv_errors[i].err_cnt = err_cnt; + err_stats->ae_cnv_errors[i].ae = ae; + i++; + } + + return err_stats; +} + +static int qat_cnv_errors_file_open(struct inode *inode, struct file *file) +{ + struct adf_accel_dev *accel_dev = inode->i_private; + struct seq_file *cnv_errors_seq_file; + struct cnv_err_stats *cnv_err_stats; + int ret; + + cnv_err_stats = cnv_err_stats_alloc(accel_dev); + if (IS_ERR(cnv_err_stats)) + return PTR_ERR(cnv_err_stats); + + ret = seq_open(file, &qat_cnv_errors_sops); + if (unlikely(ret)) { + kfree(cnv_err_stats); + return ret; + } + + cnv_errors_seq_file = file->private_data; + cnv_errors_seq_file->private = cnv_err_stats; + return ret; +} + +static int qat_cnv_errors_file_release(struct inode *inode, struct file *file) +{ + struct seq_file *cnv_errors_seq_file = file->private_data; + + kfree(cnv_errors_seq_file->private); + cnv_errors_seq_file->private = NULL; + + return seq_release(inode, file); +} + +static const struct file_operations qat_cnv_fops = { + .owner = THIS_MODULE, + .open = qat_cnv_errors_file_open, + .read = seq_read, + .llseek = seq_lseek, + .release = qat_cnv_errors_file_release, +}; + +static ssize_t no_comp_file_read(struct file *f, char __user *buf, size_t count, + loff_t *pos) +{ + char *file_msg = "No engine configured for comp\n"; + + return simple_read_from_buffer(buf, count, pos, file_msg, + strlen(file_msg)); +} + +static const struct file_operations qat_cnv_no_comp_fops = { + .owner = THIS_MODULE, + .read = no_comp_file_read, +}; + +void adf_cnv_dbgfs_add(struct adf_accel_dev *accel_dev) +{ + const struct file_operations *fops; + void *data; + + if (adf_hw_dev_has_compression(accel_dev)) { + fops = &qat_cnv_fops; + data = accel_dev; + } else { + fops = &qat_cnv_no_comp_fops; + data = NULL; + } + + accel_dev->cnv_dbgfile = debugfs_create_file(CNV_DEBUGFS_FILENAME, 0400, + accel_dev->debugfs_dir, + data, fops); +} + +void adf_cnv_dbgfs_rm(struct adf_accel_dev *accel_dev) +{ + debugfs_remove(accel_dev->cnv_dbgfile); + accel_dev->cnv_dbgfile = NULL; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_cnv_dbgfs.h b/drivers/crypto/intel/qat/qat_common/adf_cnv_dbgfs.h new file mode 100644 index 000000000..b02b0961c --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_cnv_dbgfs.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ +#ifndef ADF_CNV_DBG_H +#define ADF_CNV_DBG_H + +struct adf_accel_dev; + +void adf_cnv_dbgfs_add(struct adf_accel_dev *accel_dev); +void adf_cnv_dbgfs_rm(struct adf_accel_dev *accel_dev); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_common_drv.h b/drivers/crypto/intel/qat/qat_common/adf_common_drv.h new file mode 100644 index 000000000..762a0b5e7 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_common_drv.h @@ -0,0 +1,262 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2014 - 2021 Intel Corporation */ +#ifndef ADF_DRV_H +#define ADF_DRV_H + +#include +#include +#include "adf_accel_devices.h" +#include "icp_qat_fw_loader_handle.h" +#include "icp_qat_hal.h" + +#define ADF_STATUS_RESTARTING 0 +#define ADF_STATUS_STARTING 1 +#define ADF_STATUS_CONFIGURED 2 +#define ADF_STATUS_STARTED 3 +#define ADF_STATUS_AE_INITIALISED 4 +#define ADF_STATUS_AE_UCODE_LOADED 5 +#define ADF_STATUS_AE_STARTED 6 +#define ADF_STATUS_PF_RUNNING 7 +#define ADF_STATUS_IRQ_ALLOCATED 8 +#define ADF_STATUS_CRYPTO_ALGS_REGISTERED 9 +#define ADF_STATUS_COMP_ALGS_REGISTERED 10 + +enum adf_dev_reset_mode { + ADF_DEV_RESET_ASYNC = 0, + ADF_DEV_RESET_SYNC +}; + +enum adf_event { + ADF_EVENT_INIT = 0, + ADF_EVENT_START, + ADF_EVENT_STOP, + ADF_EVENT_SHUTDOWN, + ADF_EVENT_RESTARTING, + ADF_EVENT_RESTARTED, + ADF_EVENT_FATAL_ERROR, +}; + +struct service_hndl { + int (*event_hld)(struct adf_accel_dev *accel_dev, + enum adf_event event); + unsigned long init_status[ADF_DEVS_ARRAY_SIZE]; + unsigned long start_status[ADF_DEVS_ARRAY_SIZE]; + char *name; + struct list_head list; +}; + +int adf_service_register(struct service_hndl *service); +int adf_service_unregister(struct service_hndl *service); + +int adf_dev_up(struct adf_accel_dev *accel_dev, bool init_config); +int adf_dev_down(struct adf_accel_dev *accel_dev); +int adf_dev_restart(struct adf_accel_dev *accel_dev); + +void adf_devmgr_update_class_index(struct adf_hw_device_data *hw_data); +void adf_clean_vf_map(bool); +int adf_notify_fatal_error(struct adf_accel_dev *accel_dev); +void adf_error_notifier(struct adf_accel_dev *accel_dev); +int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev, + struct adf_accel_dev *pf); +void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev, + struct adf_accel_dev *pf); +struct list_head *adf_devmgr_get_head(void); +struct adf_accel_dev *adf_devmgr_pci_to_accel_dev(struct pci_dev *pci_dev); +int adf_devmgr_in_reset(struct adf_accel_dev *accel_dev); +int adf_dev_started(struct adf_accel_dev *accel_dev); +int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev); +int adf_dev_restarted_notify(struct adf_accel_dev *accel_dev); +int adf_ae_init(struct adf_accel_dev *accel_dev); +int adf_ae_shutdown(struct adf_accel_dev *accel_dev); +int adf_ae_fw_load(struct adf_accel_dev *accel_dev); +void adf_ae_fw_release(struct adf_accel_dev *accel_dev); +int adf_ae_start(struct adf_accel_dev *accel_dev); +int adf_ae_stop(struct adf_accel_dev *accel_dev); + +extern const struct pci_error_handlers adf_err_handler; +void adf_reset_sbr(struct adf_accel_dev *accel_dev); +void adf_reset_flr(struct adf_accel_dev *accel_dev); +void adf_dev_restore(struct adf_accel_dev *accel_dev); +void adf_set_bme(struct adf_accel_dev *accel_dev); +int adf_init_aer(void); +void adf_exit_aer(void); +int adf_init_arb(struct adf_accel_dev *accel_dev); +void adf_exit_arb(struct adf_accel_dev *accel_dev); +void adf_update_ring_arb(struct adf_etr_ring_data *ring); + +int adf_dev_get(struct adf_accel_dev *accel_dev); +void adf_dev_put(struct adf_accel_dev *accel_dev); +int adf_dev_in_use(struct adf_accel_dev *accel_dev); +int adf_init_etr_data(struct adf_accel_dev *accel_dev); +void adf_cleanup_etr_data(struct adf_accel_dev *accel_dev); +int qat_crypto_register(void); +int qat_crypto_unregister(void); +int qat_crypto_vf_dev_config(struct adf_accel_dev *accel_dev); +struct qat_crypto_instance *qat_crypto_get_instance_node(int node); +void qat_crypto_put_instance(struct qat_crypto_instance *inst); +void qat_alg_callback(void *resp); +void qat_alg_asym_callback(void *resp); +int qat_algs_register(void); +void qat_algs_unregister(void); +int qat_asym_algs_register(void); +void qat_asym_algs_unregister(void); + +struct qat_compression_instance *qat_compression_get_instance_node(int node, int alg); +void qat_compression_put_instance(struct qat_compression_instance *inst); +int qat_compression_register(void); +int qat_compression_unregister(void); +int qat_comp_algs_register(u32 caps); +void qat_comp_algs_unregister(u32 caps); +void qat_comp_alg_callback(void *resp); + +int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev); +void adf_isr_resource_free(struct adf_accel_dev *accel_dev); +void adf_isr_sync_ae_cluster(struct adf_accel_dev *accel_dev); +int adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev); +void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev); + +int adf_pfvf_comms_disabled(struct adf_accel_dev *accel_dev); + +int adf_sysfs_init(struct adf_accel_dev *accel_dev); + +int qat_hal_init(struct adf_accel_dev *accel_dev); +void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle); +int qat_hal_start(struct icp_qat_fw_loader_handle *handle); +void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae, + unsigned int ctx_mask); +void qat_hal_reset(struct icp_qat_fw_loader_handle *handle); +int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle); +void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int ctx_mask); +int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle, + unsigned int ae); +int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, enum icp_qat_uof_regtype lm_type, + unsigned char mode); +int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char mode); +int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char mode); +void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int ctx_mask, unsigned int upc); +void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int uaddr, + unsigned int words_num, u64 *uword); +void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle, unsigned char ae, + unsigned int uword_addr, unsigned int words_num, + unsigned int *data); +int qat_hal_get_ins_num(void); +int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, + struct icp_qat_uof_batch_init *lm_init_header); +int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned long ctx_mask, + enum icp_qat_uof_regtype reg_type, + unsigned short reg_num, unsigned int regdata); +int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned long ctx_mask, + enum icp_qat_uof_regtype reg_type, + unsigned short reg_num, unsigned int regdata); +int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned long ctx_mask, + enum icp_qat_uof_regtype reg_type, + unsigned short reg_num, unsigned int regdata); +int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned long ctx_mask, + unsigned short reg_num, unsigned int regdata); +void qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char mode); +int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle); +void qat_uclo_del_obj(struct icp_qat_fw_loader_handle *handle); +int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, void *addr_ptr, + int mem_size); +int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, + void *addr_ptr, u32 mem_size, const char *obj_name); +int qat_uclo_set_cfg_ae_mask(struct icp_qat_fw_loader_handle *handle, + unsigned int cfg_ae_mask); +int adf_init_misc_wq(void); +void adf_exit_misc_wq(void); +bool adf_misc_wq_queue_work(struct work_struct *work); +bool adf_misc_wq_queue_delayed_work(struct delayed_work *work, + unsigned long delay); +void adf_misc_wq_flush(void); +#if defined(CONFIG_PCI_IOV) +int adf_sriov_configure(struct pci_dev *pdev, int numvfs); +void adf_disable_sriov(struct adf_accel_dev *accel_dev); +void adf_reenable_sriov(struct adf_accel_dev *accel_dev); +void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask); +void adf_enable_all_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 num_vfs); +void adf_disable_all_vf2pf_interrupts(struct adf_accel_dev *accel_dev); +bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev); +bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr); +int adf_pf2vf_handle_pf_restarting(struct adf_accel_dev *accel_dev); +void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev); +void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev); +void adf_schedule_vf2pf_handler(struct adf_accel_vf_info *vf_info); +int adf_init_pf_wq(void); +void adf_exit_pf_wq(void); +int adf_init_vf_wq(void); +void adf_exit_vf_wq(void); +void adf_flush_vf_wq(struct adf_accel_dev *accel_dev); +#else +#define adf_sriov_configure NULL + +static inline void adf_disable_sriov(struct adf_accel_dev *accel_dev) +{ +} + +static inline void adf_reenable_sriov(struct adf_accel_dev *accel_dev) +{ +} + +static inline int adf_init_pf_wq(void) +{ + return 0; +} + +static inline void adf_exit_pf_wq(void) +{ +} + +static inline int adf_init_vf_wq(void) +{ + return 0; +} + +static inline void adf_exit_vf_wq(void) +{ +} + +#endif + +static inline void __iomem *adf_get_pmisc_base(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_bar *pmisc; + + pmisc = &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; + + return pmisc->virt_addr; +} + +static inline void __iomem *adf_get_etr_base(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_bar *etr; + + etr = &GET_BARS(accel_dev)[hw_data->get_etr_bar_id(hw_data)]; + + return etr->virt_addr; +} + +static inline void __iomem *adf_get_aram_base(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_bar *param; + + param = &GET_BARS(accel_dev)[hw_data->get_sram_bar_id(hw_data)]; + + return param->virt_addr; +} + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_dbgfs.c b/drivers/crypto/intel/qat/qat_common/adf_dbgfs.c new file mode 100644 index 000000000..4c11ad1eb --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_dbgfs.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ + +#include +#include "adf_accel_devices.h" +#include "adf_cfg.h" +#include "adf_common_drv.h" +#include "adf_cnv_dbgfs.h" +#include "adf_dbgfs.h" +#include "adf_fw_counters.h" +#include "adf_heartbeat_dbgfs.h" +#include "adf_pm_dbgfs.h" +#include "adf_tl_debugfs.h" + +/** + * adf_dbgfs_init() - add persistent debugfs entries + * @accel_dev: Pointer to acceleration device. + * + * This function creates debugfs entries that are persistent through a device + * state change (from up to down or vice versa). + */ +void adf_dbgfs_init(struct adf_accel_dev *accel_dev) +{ + char name[ADF_DEVICE_NAME_LENGTH]; + + /* Create dev top level debugfs entry */ + snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, + accel_dev->hw_device->dev_class->name, + pci_name(accel_dev->accel_pci_dev.pci_dev)); + + accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); + + adf_cfg_dev_dbgfs_add(accel_dev); +} +EXPORT_SYMBOL_GPL(adf_dbgfs_init); + +/** + * adf_dbgfs_exit() - remove persistent debugfs entries + * @accel_dev: Pointer to acceleration device. + */ +void adf_dbgfs_exit(struct adf_accel_dev *accel_dev) +{ + adf_cfg_dev_dbgfs_rm(accel_dev); + debugfs_remove(accel_dev->debugfs_dir); +} +EXPORT_SYMBOL_GPL(adf_dbgfs_exit); + +/** + * adf_dbgfs_add() - add non-persistent debugfs entries + * @accel_dev: Pointer to acceleration device. + * + * This function creates debugfs entries that are not persistent through + * a device state change (from up to down or vice versa). + */ +void adf_dbgfs_add(struct adf_accel_dev *accel_dev) +{ + if (!accel_dev->is_vf) { + adf_fw_counters_dbgfs_add(accel_dev); + adf_heartbeat_dbgfs_add(accel_dev); + adf_pm_dbgfs_add(accel_dev); + adf_cnv_dbgfs_add(accel_dev); + adf_tl_dbgfs_add(accel_dev); + } +} + +/** + * adf_dbgfs_rm() - remove non-persistent debugfs entries + * @accel_dev: Pointer to acceleration device. + */ +void adf_dbgfs_rm(struct adf_accel_dev *accel_dev) +{ + if (!accel_dev->is_vf) { + adf_tl_dbgfs_rm(accel_dev); + adf_cnv_dbgfs_rm(accel_dev); + adf_pm_dbgfs_rm(accel_dev); + adf_heartbeat_dbgfs_rm(accel_dev); + adf_fw_counters_dbgfs_rm(accel_dev); + } +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_dbgfs.h b/drivers/crypto/intel/qat/qat_common/adf_dbgfs.h new file mode 100644 index 000000000..e0cb2c2a2 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_dbgfs.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ + +#ifndef ADF_DBGFS_H +#define ADF_DBGFS_H + +#ifdef CONFIG_DEBUG_FS +void adf_dbgfs_init(struct adf_accel_dev *accel_dev); +void adf_dbgfs_add(struct adf_accel_dev *accel_dev); +void adf_dbgfs_rm(struct adf_accel_dev *accel_dev); +void adf_dbgfs_exit(struct adf_accel_dev *accel_dev); +#else +static inline void adf_dbgfs_init(struct adf_accel_dev *accel_dev) +{ +} + +static inline void adf_dbgfs_add(struct adf_accel_dev *accel_dev) +{ +} + +static inline void adf_dbgfs_rm(struct adf_accel_dev *accel_dev) +{ +} + +static inline void adf_dbgfs_exit(struct adf_accel_dev *accel_dev) +{ +} +#endif +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_dc.c b/drivers/crypto/intel/qat/qat_common/adf_dc.c new file mode 100644 index 000000000..3e8fb4e3e --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_dc.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2022 Intel Corporation */ +#include "adf_accel_devices.h" +#include "adf_dc.h" +#include "icp_qat_fw_comp.h" + +int qat_comp_build_ctx(struct adf_accel_dev *accel_dev, void *ctx, enum adf_dc_algo algo) +{ + struct icp_qat_fw_comp_req *req_tmpl = ctx; + struct icp_qat_fw_comp_cd_hdr *comp_cd_ctrl = &req_tmpl->comp_cd_ctrl; + struct icp_qat_fw_comp_req_params *req_pars = &req_tmpl->comp_pars; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + int ret; + + memset(req_tmpl, 0, sizeof(*req_tmpl)); + header->hdr_flags = + ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET); + header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_COMP; + header->comn_req_flags = + ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_16BYTE_DATA, + QAT_COMN_PTR_TYPE_SGL); + header->serv_specif_flags = + ICP_QAT_FW_COMP_FLAGS_BUILD(ICP_QAT_FW_COMP_STATELESS_SESSION, + ICP_QAT_FW_COMP_NOT_AUTO_SELECT_BEST, + ICP_QAT_FW_COMP_NOT_ENH_AUTO_SELECT_BEST, + ICP_QAT_FW_COMP_NOT_DISABLE_TYPE0_ENH_AUTO_SELECT_BEST, + ICP_QAT_FW_COMP_ENABLE_SECURE_RAM_USED_AS_INTMD_BUF); + + /* Build HW config block for compression */ + ret = GET_DC_OPS(accel_dev)->build_comp_block(ctx, algo); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Failed to build compression block\n"); + return ret; + } + + req_pars->crc.legacy.initial_adler = COMP_CPR_INITIAL_ADLER; + req_pars->crc.legacy.initial_crc32 = COMP_CPR_INITIAL_CRC; + req_pars->req_par_flags = + ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD(ICP_QAT_FW_COMP_SOP, + ICP_QAT_FW_COMP_EOP, + ICP_QAT_FW_COMP_BFINAL, + ICP_QAT_FW_COMP_CNV, + ICP_QAT_FW_COMP_CNV_RECOVERY, + ICP_QAT_FW_COMP_NO_CNV_DFX, + ICP_QAT_FW_COMP_CRC_MODE_LEGACY, + ICP_QAT_FW_COMP_NO_XXHASH_ACC, + ICP_QAT_FW_COMP_CNV_ERROR_NONE, + ICP_QAT_FW_COMP_NO_APPEND_CRC, + ICP_QAT_FW_COMP_NO_DROP_DATA, + ICP_QAT_FW_COMP_NO_PARTIAL_DECOMPRESS); + ICP_QAT_FW_COMN_NEXT_ID_SET(comp_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); + ICP_QAT_FW_COMN_CURR_ID_SET(comp_cd_ctrl, ICP_QAT_FW_SLICE_COMP); + + /* Fill second half of the template for decompression */ + memcpy(req_tmpl + 1, req_tmpl, sizeof(*req_tmpl)); + req_tmpl++; + + /* Build HW config block for decompression */ + ret = GET_DC_OPS(accel_dev)->build_decomp_block(req_tmpl, algo); + if (ret) + dev_err(&GET_DEV(accel_dev), "Failed to build decompression block\n"); + + return ret; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_dc.h b/drivers/crypto/intel/qat/qat_common/adf_dc.h new file mode 100644 index 000000000..6cb5e0905 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_dc.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ADF_DC_H +#define ADF_DC_H + +struct adf_accel_dev; + +enum adf_dc_algo { + QAT_DEFLATE, + QAT_LZ4, + QAT_LZ4S, + QAT_ZSTD, +}; + +int qat_comp_build_ctx(struct adf_accel_dev *accel_dev, void *ctx, enum adf_dc_algo algo); + +#endif /* ADF_DC_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_dev_mgr.c b/drivers/crypto/intel/qat/qat_common/adf_dev_mgr.c new file mode 100644 index 000000000..161863841 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_dev_mgr.c @@ -0,0 +1,372 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#include +#include +#include "adf_cfg.h" +#include "adf_common_drv.h" + +static LIST_HEAD(accel_table); +static LIST_HEAD(vfs_table); +static DEFINE_MUTEX(table_lock); +static u32 num_devices; +static u8 id_map[ADF_MAX_DEVICES]; + +struct vf_id_map { + u32 bdf; + u32 id; + u32 fake_id; + bool attached; + struct list_head list; +}; + +static int adf_get_vf_id(struct adf_accel_dev *vf) +{ + return ((7 * (PCI_SLOT(accel_to_pci_dev(vf)->devfn) - 1)) + + PCI_FUNC(accel_to_pci_dev(vf)->devfn) + + (PCI_SLOT(accel_to_pci_dev(vf)->devfn) - 1)); +} + +static int adf_get_vf_num(struct adf_accel_dev *vf) +{ + return (accel_to_pci_dev(vf)->bus->number << 8) | adf_get_vf_id(vf); +} + +static struct vf_id_map *adf_find_vf(u32 bdf) +{ + struct list_head *itr; + + list_for_each(itr, &vfs_table) { + struct vf_id_map *ptr = + list_entry(itr, struct vf_id_map, list); + + if (ptr->bdf == bdf) + return ptr; + } + return NULL; +} + +/** + * adf_clean_vf_map() - Cleans VF id mappings + * @vf: flag indicating whether mappings is cleaned + * for vfs only or for vfs and pfs + * + * Function cleans internal ids for virtual functions. + */ +void adf_clean_vf_map(bool vf) +{ + struct vf_id_map *map; + struct list_head *ptr, *tmp; + + mutex_lock(&table_lock); + list_for_each_safe(ptr, tmp, &vfs_table) { + map = list_entry(ptr, struct vf_id_map, list); + if (map->bdf != -1) { + id_map[map->id] = 0; + num_devices--; + } + + if (vf && map->bdf == -1) + continue; + + list_del(ptr); + kfree(map); + } + mutex_unlock(&table_lock); +} +EXPORT_SYMBOL_GPL(adf_clean_vf_map); + +/** + * adf_devmgr_update_class_index() - Update internal index + * @hw_data: Pointer to internal device data. + * + * Function updates internal dev index for VFs + */ +void adf_devmgr_update_class_index(struct adf_hw_device_data *hw_data) +{ + struct adf_hw_device_class *class = hw_data->dev_class; + struct list_head *itr; + int i = 0; + + list_for_each(itr, &accel_table) { + struct adf_accel_dev *ptr = + list_entry(itr, struct adf_accel_dev, list); + + if (ptr->hw_device->dev_class == class) + ptr->hw_device->instance_id = i++; + + if (i == class->instances) + break; + } +} +EXPORT_SYMBOL_GPL(adf_devmgr_update_class_index); + +static unsigned int adf_find_free_id(void) +{ + unsigned int i; + + for (i = 0; i < ADF_MAX_DEVICES; i++) { + if (!id_map[i]) { + id_map[i] = 1; + return i; + } + } + return ADF_MAX_DEVICES + 1; +} + +/** + * adf_devmgr_add_dev() - Add accel_dev to the acceleration framework + * @accel_dev: Pointer to acceleration device. + * @pf: Corresponding PF if the accel_dev is a VF + * + * Function adds acceleration device to the acceleration framework. + * To be used by QAT device specific drivers. + * + * Return: 0 on success, error code otherwise. + */ +int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev, + struct adf_accel_dev *pf) +{ + struct list_head *itr; + int ret = 0; + + if (num_devices == ADF_MAX_DEVICES) { + dev_err(&GET_DEV(accel_dev), "Only support up to %d devices\n", + ADF_MAX_DEVICES); + return -EFAULT; + } + + mutex_lock(&table_lock); + atomic_set(&accel_dev->ref_count, 0); + + /* PF on host or VF on guest - optimized to remove redundant is_vf */ + if (!accel_dev->is_vf || !pf) { + struct vf_id_map *map; + + list_for_each(itr, &accel_table) { + struct adf_accel_dev *ptr = + list_entry(itr, struct adf_accel_dev, list); + + if (ptr == accel_dev) { + ret = -EEXIST; + goto unlock; + } + } + + list_add_tail(&accel_dev->list, &accel_table); + accel_dev->accel_id = adf_find_free_id(); + if (accel_dev->accel_id > ADF_MAX_DEVICES) { + ret = -EFAULT; + goto unlock; + } + num_devices++; + map = kzalloc_obj(*map); + if (!map) { + ret = -ENOMEM; + goto unlock; + } + map->bdf = ~0; + map->id = accel_dev->accel_id; + map->fake_id = map->id; + map->attached = true; + list_add_tail(&map->list, &vfs_table); + } else if (accel_dev->is_vf && pf) { + /* VF on host */ + struct vf_id_map *map; + + map = adf_find_vf(adf_get_vf_num(accel_dev)); + if (map) { + struct vf_id_map *next; + + accel_dev->accel_id = map->id; + list_add_tail(&accel_dev->list, &accel_table); + map->fake_id++; + map->attached = true; + next = list_next_entry(map, list); + while (next && &next->list != &vfs_table) { + next->fake_id++; + next = list_next_entry(next, list); + } + + ret = 0; + goto unlock; + } + + map = kzalloc_obj(*map); + if (!map) { + ret = -ENOMEM; + goto unlock; + } + accel_dev->accel_id = adf_find_free_id(); + if (accel_dev->accel_id > ADF_MAX_DEVICES) { + kfree(map); + ret = -EFAULT; + goto unlock; + } + num_devices++; + list_add_tail(&accel_dev->list, &accel_table); + map->bdf = adf_get_vf_num(accel_dev); + map->id = accel_dev->accel_id; + map->fake_id = map->id; + map->attached = true; + list_add_tail(&map->list, &vfs_table); + } + mutex_init(&accel_dev->state_lock); +unlock: + mutex_unlock(&table_lock); + return ret; +} +EXPORT_SYMBOL_GPL(adf_devmgr_add_dev); + +struct list_head *adf_devmgr_get_head(void) +{ + return &accel_table; +} + +/** + * adf_devmgr_rm_dev() - Remove accel_dev from the acceleration framework. + * @accel_dev: Pointer to acceleration device. + * @pf: Corresponding PF if the accel_dev is a VF + * + * Function removes acceleration device from the acceleration framework. + * To be used by QAT device specific drivers. + * + * Return: void + */ +void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev, + struct adf_accel_dev *pf) +{ + mutex_lock(&table_lock); + /* PF on host or VF on guest - optimized to remove redundant is_vf */ + if (!accel_dev->is_vf || !pf) { + id_map[accel_dev->accel_id] = 0; + num_devices--; + } else if (accel_dev->is_vf && pf) { + struct vf_id_map *map, *next; + + map = adf_find_vf(adf_get_vf_num(accel_dev)); + if (!map) { + dev_err(&GET_DEV(accel_dev), "Failed to find VF map\n"); + goto unlock; + } + map->fake_id--; + map->attached = false; + next = list_next_entry(map, list); + while (next && &next->list != &vfs_table) { + next->fake_id--; + next = list_next_entry(next, list); + } + } +unlock: + mutex_destroy(&accel_dev->state_lock); + list_del(&accel_dev->list); + mutex_unlock(&table_lock); +} +EXPORT_SYMBOL_GPL(adf_devmgr_rm_dev); + +/** + * adf_devmgr_pci_to_accel_dev() - Get accel_dev associated with the pci_dev. + * @pci_dev: Pointer to PCI device. + * + * Function returns acceleration device associated with the given PCI device. + * To be used by QAT device specific drivers. + * + * Return: pointer to accel_dev or NULL if not found. + */ +struct adf_accel_dev *adf_devmgr_pci_to_accel_dev(struct pci_dev *pci_dev) +{ + struct list_head *itr; + + mutex_lock(&table_lock); + list_for_each(itr, &accel_table) { + struct adf_accel_dev *ptr = + list_entry(itr, struct adf_accel_dev, list); + + if (ptr->accel_pci_dev.pci_dev == pci_dev) { + mutex_unlock(&table_lock); + return ptr; + } + } + mutex_unlock(&table_lock); + return NULL; +} +EXPORT_SYMBOL_GPL(adf_devmgr_pci_to_accel_dev); + +/** + * adf_dev_in_use() - Check whether accel_dev is currently in use + * @accel_dev: Pointer to acceleration device. + * + * To be used by QAT device specific drivers. + * + * Return: 1 when device is in use, 0 otherwise. + */ +int adf_dev_in_use(struct adf_accel_dev *accel_dev) +{ + return atomic_read(&accel_dev->ref_count) != 0; +} +EXPORT_SYMBOL_GPL(adf_dev_in_use); + +/** + * adf_dev_get() - Increment accel_dev reference count + * @accel_dev: Pointer to acceleration device. + * + * Increment the accel_dev refcount and if this is the first time + * incrementing it during this period the accel_dev is in use, + * increment the module refcount too. + * To be used by QAT device specific drivers. + * + * Return: 0 when successful, EFAULT when fail to bump module refcount + */ +int adf_dev_get(struct adf_accel_dev *accel_dev) +{ + if (atomic_add_return(1, &accel_dev->ref_count) == 1) + if (!try_module_get(accel_dev->owner)) + return -EFAULT; + return 0; +} +EXPORT_SYMBOL_GPL(adf_dev_get); + +/** + * adf_dev_put() - Decrement accel_dev reference count + * @accel_dev: Pointer to acceleration device. + * + * Decrement the accel_dev refcount and if this is the last time + * decrementing it during this period the accel_dev is in use, + * decrement the module refcount too. + * To be used by QAT device specific drivers. + * + * Return: void + */ +void adf_dev_put(struct adf_accel_dev *accel_dev) +{ + if (atomic_sub_return(1, &accel_dev->ref_count) == 0) + module_put(accel_dev->owner); +} +EXPORT_SYMBOL_GPL(adf_dev_put); + +/** + * adf_devmgr_in_reset() - Check whether device is in reset + * @accel_dev: Pointer to acceleration device. + * + * To be used by QAT device specific drivers. + * + * Return: 1 when the device is being reset, 0 otherwise. + */ +int adf_devmgr_in_reset(struct adf_accel_dev *accel_dev) +{ + return test_bit(ADF_STATUS_RESTARTING, &accel_dev->status); +} +EXPORT_SYMBOL_GPL(adf_devmgr_in_reset); + +/** + * adf_dev_started() - Check whether device has started + * @accel_dev: Pointer to acceleration device. + * + * To be used by QAT device specific drivers. + * + * Return: 1 when the device has started, 0 otherwise + */ +int adf_dev_started(struct adf_accel_dev *accel_dev) +{ + return test_bit(ADF_STATUS_STARTED, &accel_dev->status); +} +EXPORT_SYMBOL_GPL(adf_dev_started); diff --git a/drivers/crypto/intel/qat/qat_common/adf_fw_config.h b/drivers/crypto/intel/qat/qat_common/adf_fw_config.h new file mode 100644 index 000000000..d5c578e3f --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_fw_config.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ +#ifndef ADF_FW_CONFIG_H_ +#define ADF_FW_CONFIG_H_ + +enum adf_fw_objs { + ADF_FW_SYM_OBJ, + ADF_FW_ASYM_OBJ, + ADF_FW_DC_OBJ, + ADF_FW_ADMIN_OBJ, + ADF_FW_CY_OBJ, + ADF_FW_WCY_OBJ, +}; + +struct adf_fw_config { + u32 ae_mask; + enum adf_fw_objs obj; +}; + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_fw_counters.c b/drivers/crypto/intel/qat/qat_common/adf_fw_counters.c new file mode 100644 index 000000000..608cd0213 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_fw_counters.c @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include + +#include "adf_accel_devices.h" +#include "adf_admin.h" +#include "adf_common_drv.h" +#include "adf_fw_counters.h" + +#define ADF_FW_COUNTERS_MAX_PADDING 16 + +enum adf_fw_counters_types { + ADF_FW_REQUESTS, + ADF_FW_RESPONSES, + ADF_FW_COUNTERS_COUNT +}; + +static const char * const adf_fw_counter_names[] = { + [ADF_FW_REQUESTS] = "Requests", + [ADF_FW_RESPONSES] = "Responses", +}; + +static_assert(ARRAY_SIZE(adf_fw_counter_names) == ADF_FW_COUNTERS_COUNT); + +struct adf_ae_counters { + u16 ae; + u64 values[ADF_FW_COUNTERS_COUNT]; +}; + +struct adf_fw_counters { + u16 ae_count; + struct adf_ae_counters ae_counters[] __counted_by(ae_count); +}; + +static void adf_fw_counters_parse_ae_values(struct adf_ae_counters *ae_counters, u32 ae, + u64 req_count, u64 resp_count) +{ + ae_counters->ae = ae; + ae_counters->values[ADF_FW_REQUESTS] = req_count; + ae_counters->values[ADF_FW_RESPONSES] = resp_count; +} + +static int adf_fw_counters_load_from_device(struct adf_accel_dev *accel_dev, + struct adf_fw_counters *fw_counters) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + unsigned long ae_mask; + unsigned int i; + unsigned long ae; + + /* Ignore the admin AEs */ + ae_mask = hw_data->ae_mask & ~hw_data->admin_ae_mask; + + if (hweight_long(ae_mask) > fw_counters->ae_count) + return -EINVAL; + + i = 0; + for_each_set_bit(ae, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) { + u64 req_count, resp_count; + int ret; + + ret = adf_get_ae_fw_counters(accel_dev, ae, &req_count, &resp_count); + if (ret) + return ret; + + adf_fw_counters_parse_ae_values(&fw_counters->ae_counters[i++], ae, + req_count, resp_count); + } + + return 0; +} + +static struct adf_fw_counters *adf_fw_counters_allocate(unsigned long ae_count) +{ + struct adf_fw_counters *fw_counters; + + if (unlikely(!ae_count)) + return ERR_PTR(-EINVAL); + + fw_counters = kmalloc_flex(*fw_counters, ae_counters, ae_count); + if (!fw_counters) + return ERR_PTR(-ENOMEM); + + fw_counters->ae_count = ae_count; + + return fw_counters; +} + +/** + * adf_fw_counters_get() - Return FW counters for the provided device. + * @accel_dev: Pointer to a QAT acceleration device + * + * Allocates and returns a table of counters containing execution statistics + * for each non-admin AE available through the supplied acceleration device. + * The caller becomes the owner of such memory and is responsible for + * the deallocation through a call to kfree(). + * + * Returns: a pointer to a dynamically allocated struct adf_fw_counters + * on success, or a negative value on error. + */ +static struct adf_fw_counters *adf_fw_counters_get(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + struct adf_fw_counters *fw_counters; + unsigned long ae_count; + int ret; + + if (!adf_dev_started(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "QAT Device not started\n"); + return ERR_PTR(-EFAULT); + } + + /* Ignore the admin AEs */ + ae_count = hweight_long(hw_data->ae_mask & ~hw_data->admin_ae_mask); + + fw_counters = adf_fw_counters_allocate(ae_count); + if (IS_ERR(fw_counters)) + return fw_counters; + + ret = adf_fw_counters_load_from_device(accel_dev, fw_counters); + if (ret) { + kfree(fw_counters); + dev_err(&GET_DEV(accel_dev), + "Failed to create QAT fw_counters file table [%d].\n", ret); + return ERR_PTR(ret); + } + + return fw_counters; +} + +static void *qat_fw_counters_seq_start(struct seq_file *sfile, loff_t *pos) +{ + struct adf_fw_counters *fw_counters = sfile->private; + + if (*pos == 0) + return SEQ_START_TOKEN; + + if (*pos > fw_counters->ae_count) + return NULL; + + return &fw_counters->ae_counters[*pos - 1]; +} + +static void *qat_fw_counters_seq_next(struct seq_file *sfile, void *v, loff_t *pos) +{ + struct adf_fw_counters *fw_counters = sfile->private; + + (*pos)++; + + if (*pos > fw_counters->ae_count) + return NULL; + + return &fw_counters->ae_counters[*pos - 1]; +} + +static void qat_fw_counters_seq_stop(struct seq_file *sfile, void *v) {} + +static int qat_fw_counters_seq_show(struct seq_file *sfile, void *v) +{ + int i; + + if (v == SEQ_START_TOKEN) { + seq_puts(sfile, "AE "); + for (i = 0; i < ADF_FW_COUNTERS_COUNT; ++i) + seq_printf(sfile, " %*s", ADF_FW_COUNTERS_MAX_PADDING, + adf_fw_counter_names[i]); + } else { + struct adf_ae_counters *ae_counters = (struct adf_ae_counters *)v; + + seq_printf(sfile, "%2d:", ae_counters->ae); + for (i = 0; i < ADF_FW_COUNTERS_COUNT; ++i) + seq_printf(sfile, " %*llu", ADF_FW_COUNTERS_MAX_PADDING, + ae_counters->values[i]); + } + seq_putc(sfile, '\n'); + + return 0; +} + +static const struct seq_operations qat_fw_counters_sops = { + .start = qat_fw_counters_seq_start, + .next = qat_fw_counters_seq_next, + .stop = qat_fw_counters_seq_stop, + .show = qat_fw_counters_seq_show, +}; + +static int qat_fw_counters_file_open(struct inode *inode, struct file *file) +{ + struct adf_accel_dev *accel_dev = inode->i_private; + struct seq_file *fw_counters_seq_file; + struct adf_fw_counters *fw_counters; + int ret; + + fw_counters = adf_fw_counters_get(accel_dev); + if (IS_ERR(fw_counters)) + return PTR_ERR(fw_counters); + + ret = seq_open(file, &qat_fw_counters_sops); + if (unlikely(ret)) { + kfree(fw_counters); + return ret; + } + + fw_counters_seq_file = file->private_data; + fw_counters_seq_file->private = fw_counters; + return ret; +} + +static int qat_fw_counters_file_release(struct inode *inode, struct file *file) +{ + struct seq_file *seq = file->private_data; + + kfree(seq->private); + seq->private = NULL; + + return seq_release(inode, file); } + +static const struct file_operations qat_fw_counters_fops = { + .owner = THIS_MODULE, + .open = qat_fw_counters_file_open, + .read = seq_read, + .llseek = seq_lseek, + .release = qat_fw_counters_file_release, +}; + +/** + * adf_fw_counters_dbgfs_add() - Create a debugfs file containing FW + * execution counters. + * @accel_dev: Pointer to a QAT acceleration device + * + * Function creates a file to display a table with statistics for the given + * QAT acceleration device. The table stores device specific execution values + * for each AE, such as the number of requests sent to the FW and responses + * received from the FW. + * + * Return: void + */ +void adf_fw_counters_dbgfs_add(struct adf_accel_dev *accel_dev) +{ + accel_dev->fw_cntr_dbgfile = debugfs_create_file("fw_counters", 0400, + accel_dev->debugfs_dir, + accel_dev, + &qat_fw_counters_fops); +} + +/** + * adf_fw_counters_dbgfs_rm() - Remove the debugfs file containing FW counters. + * @accel_dev: Pointer to a QAT acceleration device. + * + * Function removes the file providing the table of statistics for the given + * QAT acceleration device. + * + * Return: void + */ +void adf_fw_counters_dbgfs_rm(struct adf_accel_dev *accel_dev) +{ + debugfs_remove(accel_dev->fw_cntr_dbgfile); + accel_dev->fw_cntr_dbgfile = NULL; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_fw_counters.h b/drivers/crypto/intel/qat/qat_common/adf_fw_counters.h new file mode 100644 index 000000000..91b3b6a95 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_fw_counters.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ +#ifndef ADF_FW_COUNTERS_H +#define ADF_FW_COUNTERS_H + +struct adf_accel_dev; + +void adf_fw_counters_dbgfs_add(struct adf_accel_dev *accel_dev); +void adf_fw_counters_dbgfs_rm(struct adf_accel_dev *accel_dev); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_config.c b/drivers/crypto/intel/qat/qat_common/adf_gen2_config.c new file mode 100644 index 000000000..57025bffe --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_config.c @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2022 Intel Corporation */ +#include "adf_accel_devices.h" +#include "adf_cfg.h" +#include "adf_cfg_strings.h" +#include "adf_gen2_config.h" +#include "adf_common_drv.h" +#include "qat_crypto.h" +#include "qat_compression.h" +#include "adf_heartbeat.h" +#include "adf_transport_access_macros.h" + +static int adf_gen2_crypto_dev_config(struct adf_accel_dev *accel_dev) +{ + char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + int banks = GET_MAX_BANKS(accel_dev); + int cpus = num_online_cpus(); + unsigned long val; + int instances; + int ret; + int i; + + if (adf_hw_dev_has_crypto(accel_dev)) + instances = min(cpus, banks); + else + instances = 0; + + for (i = 0; i < instances; i++) { + val = i; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_BANK_NUM, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY, + i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); + val = 128; + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 512; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 0; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 2; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 8; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 10; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = ADF_COALESCING_DEF_TIME; + snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); + ret = adf_cfg_add_key_value_param(accel_dev, "Accelerator0", + key, &val, ADF_DEC); + if (ret) + goto err; + } + + val = i; + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY, + &val, ADF_DEC); + if (ret) + goto err; + + return ret; + +err: + dev_err(&GET_DEV(accel_dev), "Failed to add configuration for crypto\n"); + return ret; +} + +static int adf_gen2_comp_dev_config(struct adf_accel_dev *accel_dev) +{ + char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + int banks = GET_MAX_BANKS(accel_dev); + int cpus = num_online_cpus(); + unsigned long val; + int instances; + int ret; + int i; + + if (adf_hw_dev_has_compression(accel_dev)) + instances = min(cpus, banks); + else + instances = 0; + + for (i = 0; i < instances; i++) { + val = i; + snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_BANK_NUM, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 512; + snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_SIZE, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 6; + snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_TX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 14; + snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_RX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + } + + val = i; + return adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + ADF_NUM_DC, &val, ADF_DEC); + +err: + dev_err(&GET_DEV(accel_dev), "Failed to add configuration for compression\n"); + return ret; +} + +/** + * adf_gen2_dev_config() - create dev config required to create instances + * + * @accel_dev: Pointer to acceleration device. + * + * Function creates device configuration required to create instances + * + * Return: 0 on success, error code otherwise. + */ +int adf_gen2_dev_config(struct adf_accel_dev *accel_dev) +{ + int ret; + + ret = adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC); + if (ret) + goto err; + + ret = adf_cfg_section_add(accel_dev, "Accelerator0"); + if (ret) + goto err; + + ret = adf_gen2_crypto_dev_config(accel_dev); + if (ret) + goto err; + + ret = adf_gen2_comp_dev_config(accel_dev); + if (ret) + goto err; + + ret = adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC); + if (ret) + goto err; + + adf_heartbeat_save_cfg_param(accel_dev, ADF_CFG_HB_TIMER_DEFAULT_MS); + + set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); + + return ret; + +err: + dev_err(&GET_DEV(accel_dev), "Failed to configure QAT driver\n"); + return ret; +} +EXPORT_SYMBOL_GPL(adf_gen2_dev_config); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_config.h b/drivers/crypto/intel/qat/qat_common/adf_gen2_config.h new file mode 100644 index 000000000..4bf9da2de --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_config.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2022 Intel Corporation */ +#ifndef ADF_GEN2_CONFIG_H_ +#define ADF_GEN2_CONFIG_H_ + +#include "adf_accel_devices.h" + +int adf_gen2_dev_config(struct adf_accel_dev *accel_dev); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_csr_data.c b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_csr_data.c new file mode 100644 index 000000000..650c9edd8 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_csr_data.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2024 Intel Corporation */ +#include +#include "adf_gen2_hw_csr_data.h" + +static u64 build_csr_ring_base_addr(dma_addr_t addr, u32 size) +{ + return BUILD_RING_BASE_ADDR(addr, size); +} + +static u32 read_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring) +{ + return READ_CSR_RING_HEAD(csr_base_addr, bank, ring); +} + +static void write_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value); +} + +static u32 read_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring) +{ + return READ_CSR_RING_TAIL(csr_base_addr, bank, ring); +} + +static void write_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value); +} + +static u32 read_csr_e_stat(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_E_STAT(csr_base_addr, bank); +} + +static void write_csr_ring_config(void __iomem *csr_base_addr, u32 bank, + u32 ring, u32 value) +{ + WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value); +} + +static void write_csr_ring_base(void __iomem *csr_base_addr, u32 bank, u32 ring, + dma_addr_t addr) +{ + WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, addr); +} + +static void write_csr_int_flag(void __iomem *csr_base_addr, u32 bank, u32 value) +{ + WRITE_CSR_INT_FLAG(csr_base_addr, bank, value); +} + +static void write_csr_int_srcsel(void __iomem *csr_base_addr, u32 bank) +{ + WRITE_CSR_INT_SRCSEL(csr_base_addr, bank); +} + +static void write_csr_int_col_en(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value); +} + +static void write_csr_int_col_ctl(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value); +} + +static void write_csr_int_flag_and_col(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value); +} + +static void write_csr_ring_srv_arb_en(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value); +} + +void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) +{ + csr_ops->build_csr_ring_base_addr = build_csr_ring_base_addr; + csr_ops->read_csr_ring_head = read_csr_ring_head; + csr_ops->write_csr_ring_head = write_csr_ring_head; + csr_ops->read_csr_ring_tail = read_csr_ring_tail; + csr_ops->write_csr_ring_tail = write_csr_ring_tail; + csr_ops->read_csr_e_stat = read_csr_e_stat; + csr_ops->write_csr_ring_config = write_csr_ring_config; + csr_ops->write_csr_ring_base = write_csr_ring_base; + csr_ops->write_csr_int_flag = write_csr_int_flag; + csr_ops->write_csr_int_srcsel = write_csr_int_srcsel; + csr_ops->write_csr_int_col_en = write_csr_int_col_en; + csr_ops->write_csr_int_col_ctl = write_csr_int_col_ctl; + csr_ops->write_csr_int_flag_and_col = write_csr_int_flag_and_col; + csr_ops->write_csr_ring_srv_arb_en = write_csr_ring_srv_arb_en; +} +EXPORT_SYMBOL_GPL(adf_gen2_init_hw_csr_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_csr_data.h b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_csr_data.h new file mode 100644 index 000000000..55058b0f9 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_csr_data.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2024 Intel Corporation */ +#ifndef ADF_GEN2_HW_CSR_DATA_H_ +#define ADF_GEN2_HW_CSR_DATA_H_ + +#include +#include "adf_accel_devices.h" + +#define ADF_BANK_INT_SRC_SEL_MASK_0 0x4444444CUL +#define ADF_BANK_INT_SRC_SEL_MASK_X 0x44444444UL +#define ADF_RING_CSR_RING_CONFIG 0x000 +#define ADF_RING_CSR_RING_LBASE 0x040 +#define ADF_RING_CSR_RING_UBASE 0x080 +#define ADF_RING_CSR_RING_HEAD 0x0C0 +#define ADF_RING_CSR_RING_TAIL 0x100 +#define ADF_RING_CSR_E_STAT 0x14C +#define ADF_RING_CSR_INT_FLAG 0x170 +#define ADF_RING_CSR_INT_SRCSEL 0x174 +#define ADF_RING_CSR_INT_SRCSEL_2 0x178 +#define ADF_RING_CSR_INT_COL_EN 0x17C +#define ADF_RING_CSR_INT_COL_CTL 0x180 +#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184 +#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000 +#define ADF_RING_BUNDLE_SIZE 0x1000 +#define ADF_ARB_REG_SLOT 0x1000 +#define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C + +#define BUILD_RING_BASE_ADDR(addr, size) \ + (((addr) >> 6) & (GENMASK_ULL(63, 0) << (size))) +#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \ + ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_HEAD + ((ring) << 2)) +#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring) \ + ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_TAIL + ((ring) << 2)) +#define READ_CSR_E_STAT(csr_base_addr, bank) \ + ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_E_STAT) +#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_CONFIG + ((ring) << 2), value) +#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value) \ +do { \ + u32 l_base = 0, u_base = 0; \ + l_base = (u32)((value) & 0xFFFFFFFF); \ + u_base = (u32)(((value) & 0xFFFFFFFF00000000ULL) >> 32); \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_LBASE + ((ring) << 2), l_base); \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_UBASE + ((ring) << 2), u_base); \ +} while (0) + +#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_HEAD + ((ring) << 2), value) +#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_TAIL + ((ring) << 2), value) +#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_FLAG, value) +#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \ +do { \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_SRCSEL, ADF_BANK_INT_SRC_SEL_MASK_0); \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_SRCSEL_2, ADF_BANK_INT_SRC_SEL_MASK_X); \ +} while (0) +#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_COL_EN, value) +#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_COL_CTL, \ + ADF_RING_CSR_INT_COL_CTL_ENABLE | (value)) +#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_FLAG_AND_COL, value) + +#define WRITE_CSR_RING_SRV_ARB_EN(csr_addr, index, value) \ + ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \ + (ADF_ARB_REG_SLOT * (index)), value) + +void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c new file mode 100644 index 000000000..6a505e9a5 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2020 Intel Corporation */ +#include "adf_common_drv.h" +#include "adf_dc.h" +#include "adf_gen2_hw_data.h" +#include "icp_qat_fw_comp.h" +#include "icp_qat_hw.h" +#include + +u32 adf_gen2_get_num_accels(struct adf_hw_device_data *self) +{ + if (!self || !self->accel_mask) + return 0; + + return hweight16(self->accel_mask); +} +EXPORT_SYMBOL_GPL(adf_gen2_get_num_accels); + +u32 adf_gen2_get_num_aes(struct adf_hw_device_data *self) +{ + if (!self || !self->ae_mask) + return 0; + + return hweight32(self->ae_mask); +} +EXPORT_SYMBOL_GPL(adf_gen2_get_num_aes); + +void adf_gen2_enable_error_correction(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + unsigned long accel_mask = hw_data->accel_mask; + unsigned long ae_mask = hw_data->ae_mask; + unsigned int val, i; + + /* Enable Accel Engine error detection & correction */ + for_each_set_bit(i, &ae_mask, hw_data->num_engines) { + val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_AE_CTX_ENABLES(i)); + val |= ADF_GEN2_ENABLE_AE_ECC_ERR; + ADF_CSR_WR(pmisc_addr, ADF_GEN2_AE_CTX_ENABLES(i), val); + val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_AE_MISC_CONTROL(i)); + val |= ADF_GEN2_ENABLE_AE_ECC_PARITY_CORR; + ADF_CSR_WR(pmisc_addr, ADF_GEN2_AE_MISC_CONTROL(i), val); + } + + /* Enable shared memory error detection & correction */ + for_each_set_bit(i, &accel_mask, hw_data->num_accel) { + val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_UERRSSMSH(i)); + val |= ADF_GEN2_ERRSSMSH_EN; + ADF_CSR_WR(pmisc_addr, ADF_GEN2_UERRSSMSH(i), val); + val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_CERRSSMSH(i)); + val |= ADF_GEN2_ERRSSMSH_EN; + ADF_CSR_WR(pmisc_addr, ADF_GEN2_CERRSSMSH(i), val); + } +} +EXPORT_SYMBOL_GPL(adf_gen2_enable_error_correction); + +void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, + int num_a_regs, int num_b_regs) +{ + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + u32 reg; + int i; + + /* Set/Unset Valid bit in AE Thread to PCIe Function Mapping Group A */ + for (i = 0; i < num_a_regs; i++) { + reg = READ_CSR_AE2FUNCTION_MAP_A(pmisc_addr, i); + if (enable) + reg |= AE2FUNCTION_MAP_VALID; + else + reg &= ~AE2FUNCTION_MAP_VALID; + WRITE_CSR_AE2FUNCTION_MAP_A(pmisc_addr, i, reg); + } + + /* Set/Unset Valid bit in AE Thread to PCIe Function Mapping Group B */ + for (i = 0; i < num_b_regs; i++) { + reg = READ_CSR_AE2FUNCTION_MAP_B(pmisc_addr, i); + if (enable) + reg |= AE2FUNCTION_MAP_VALID; + else + reg &= ~AE2FUNCTION_MAP_VALID; + WRITE_CSR_AE2FUNCTION_MAP_B(pmisc_addr, i, reg); + } +} +EXPORT_SYMBOL_GPL(adf_gen2_cfg_iov_thds); + +void adf_gen2_get_admin_info(struct admin_info *admin_csrs_info) +{ + admin_csrs_info->mailbox_offset = ADF_MAILBOX_BASE_OFFSET; + admin_csrs_info->admin_msg_ur = ADF_ADMINMSGUR_OFFSET; + admin_csrs_info->admin_msg_lr = ADF_ADMINMSGLR_OFFSET; +} +EXPORT_SYMBOL_GPL(adf_gen2_get_admin_info); + +void adf_gen2_get_arb_info(struct arb_info *arb_info) +{ + arb_info->arb_cfg = ADF_ARB_CONFIG; + arb_info->arb_offset = ADF_ARB_OFFSET; + arb_info->wt2sam_offset = ADF_ARB_WRK_2_SER_MAP_OFFSET; +} +EXPORT_SYMBOL_GPL(adf_gen2_get_arb_info); + +void adf_gen2_enable_ints(struct adf_accel_dev *accel_dev) +{ + void __iomem *addr = adf_get_pmisc_base(accel_dev); + u32 val; + + val = accel_dev->pf.vf_info ? 0 : BIT_ULL(GET_MAX_BANKS(accel_dev)) - 1; + + /* Enable bundle and misc interrupts */ + ADF_CSR_WR(addr, ADF_GEN2_SMIAPF0_MASK_OFFSET, val); + ADF_CSR_WR(addr, ADF_GEN2_SMIAPF1_MASK_OFFSET, ADF_GEN2_SMIA1_MASK); +} +EXPORT_SYMBOL_GPL(adf_gen2_enable_ints); + +u32 adf_gen2_get_accel_cap(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev; + u32 fuses = hw_data->fuses[ADF_FUSECTL0]; + u32 straps = hw_data->straps; + u32 legfuses; + u32 capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | + ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | + ICP_ACCEL_CAPABILITIES_AUTHENTICATION | + ICP_ACCEL_CAPABILITIES_CIPHER | + ICP_ACCEL_CAPABILITIES_COMPRESSION; + + /* Read accelerator capabilities mask */ + pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, &legfuses); + + /* A set bit in legfuses means the feature is OFF in this SKU */ + if (legfuses & ICP_ACCEL_MASK_CIPHER_SLICE) { + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + } + if (legfuses & ICP_ACCEL_MASK_PKE_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + if (legfuses & ICP_ACCEL_MASK_AUTH_SLICE) { + capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + } + if (legfuses & ICP_ACCEL_MASK_COMPRESS_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION; + + if ((straps | fuses) & ADF_POWERGATE_PKE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + + if ((straps | fuses) & ADF_POWERGATE_DC) + capabilities &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION; + + return capabilities; +} +EXPORT_SYMBOL_GPL(adf_gen2_get_accel_cap); + +void adf_gen2_set_ssm_wdtimer(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + u32 timer_val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE; + u32 timer_val = ADF_SSM_WDT_DEFAULT_VALUE; + unsigned long accel_mask = hw_data->accel_mask; + u32 i = 0; + + /* Configures WDT timers */ + for_each_set_bit(i, &accel_mask, hw_data->num_accel) { + /* Enable WDT for sym and dc */ + ADF_CSR_WR(pmisc_addr, ADF_SSMWDT(i), timer_val); + /* Enable WDT for pke */ + ADF_CSR_WR(pmisc_addr, ADF_SSMWDTPKE(i), timer_val_pke); + } +} +EXPORT_SYMBOL_GPL(adf_gen2_set_ssm_wdtimer); + +static int adf_gen2_build_comp_block(void *ctx, enum adf_dc_algo algo) +{ + struct icp_qat_fw_comp_req *req_tmpl = ctx; + struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + + switch (algo) { + case QAT_DEFLATE: + header->service_cmd_id = ICP_QAT_FW_COMP_CMD_STATIC; + break; + default: + return -EINVAL; + } + + cd_pars->u.sl.comp_slice_cfg_word[0] = + ICP_QAT_HW_COMPRESSION_CONFIG_BUILD(ICP_QAT_HW_COMPRESSION_DIR_COMPRESS, + ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DISABLED, + ICP_QAT_HW_COMPRESSION_ALGO_DEFLATE, + ICP_QAT_HW_COMPRESSION_DEPTH_1, + ICP_QAT_HW_COMPRESSION_FILE_TYPE_0); + + return 0; +} + +static int adf_gen2_build_decomp_block(void *ctx, enum adf_dc_algo algo) +{ + struct icp_qat_fw_comp_req *req_tmpl = ctx; + struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + + switch (algo) { + case QAT_DEFLATE: + header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS; + break; + default: + return -EINVAL; + } + + cd_pars->u.sl.comp_slice_cfg_word[0] = + ICP_QAT_HW_COMPRESSION_CONFIG_BUILD(ICP_QAT_HW_COMPRESSION_DIR_DECOMPRESS, + ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DISABLED, + ICP_QAT_HW_COMPRESSION_ALGO_DEFLATE, + ICP_QAT_HW_COMPRESSION_DEPTH_1, + ICP_QAT_HW_COMPRESSION_FILE_TYPE_0); + + return 0; +} + +void adf_gen2_init_dc_ops(struct adf_dc_ops *dc_ops) +{ + dc_ops->build_comp_block = adf_gen2_build_comp_block; + dc_ops->build_decomp_block = adf_gen2_build_decomp_block; +} +EXPORT_SYMBOL_GPL(adf_gen2_init_dc_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h new file mode 100644 index 000000000..59bad368a --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2020 Intel Corporation */ +#ifndef ADF_GEN2_HW_DATA_H_ +#define ADF_GEN2_HW_DATA_H_ + +#include "adf_accel_devices.h" +#include "adf_cfg_common.h" + +#define ADF_GEN2_RX_RINGS_OFFSET 8 +#define ADF_GEN2_TX_RINGS_MASK 0xFF + +/* AE to function map */ +#define AE2FUNCTION_MAP_A_OFFSET (0x3A400 + 0x190) +#define AE2FUNCTION_MAP_B_OFFSET (0x3A400 + 0x310) +#define AE2FUNCTION_MAP_REG_SIZE 4 +#define AE2FUNCTION_MAP_VALID BIT(7) + +#define READ_CSR_AE2FUNCTION_MAP_A(pmisc_bar_addr, index) \ + ADF_CSR_RD(pmisc_bar_addr, AE2FUNCTION_MAP_A_OFFSET + \ + AE2FUNCTION_MAP_REG_SIZE * (index)) +#define WRITE_CSR_AE2FUNCTION_MAP_A(pmisc_bar_addr, index, value) \ + ADF_CSR_WR(pmisc_bar_addr, AE2FUNCTION_MAP_A_OFFSET + \ + AE2FUNCTION_MAP_REG_SIZE * (index), value) +#define READ_CSR_AE2FUNCTION_MAP_B(pmisc_bar_addr, index) \ + ADF_CSR_RD(pmisc_bar_addr, AE2FUNCTION_MAP_B_OFFSET + \ + AE2FUNCTION_MAP_REG_SIZE * (index)) +#define WRITE_CSR_AE2FUNCTION_MAP_B(pmisc_bar_addr, index, value) \ + ADF_CSR_WR(pmisc_bar_addr, AE2FUNCTION_MAP_B_OFFSET + \ + AE2FUNCTION_MAP_REG_SIZE * (index), value) + +/* Admin Interface Offsets */ +#define ADF_ADMINMSGUR_OFFSET (0x3A000 + 0x574) +#define ADF_ADMINMSGLR_OFFSET (0x3A000 + 0x578) +#define ADF_MAILBOX_BASE_OFFSET 0x20970 + +/* Arbiter configuration */ +#define ADF_ARB_OFFSET 0x30000 +#define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180 +#define ADF_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0)) + +/* Power gating */ +#define ADF_POWERGATE_DC BIT(23) +#define ADF_POWERGATE_PKE BIT(24) + +/* Default ring mapping */ +#define ADF_GEN2_DEFAULT_RING_TO_SRV_MAP \ + (CRYPTO << ADF_CFG_SERV_RING_PAIR_0_SHIFT | \ + CRYPTO << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \ + UNUSED << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \ + COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT) + +/* WDT timers + * + * Timeout is in cycles. Clock speed may vary across products but this + * value should be a few milli-seconds. + */ +#define ADF_SSM_WDT_DEFAULT_VALUE 0x200000 +#define ADF_SSM_WDT_PKE_DEFAULT_VALUE 0x2000000 +#define ADF_SSMWDT_OFFSET 0x54 +#define ADF_SSMWDTPKE_OFFSET 0x58 +#define ADF_SSMWDT(i) (ADF_SSMWDT_OFFSET + ((i) * 0x4000)) +#define ADF_SSMWDTPKE(i) (ADF_SSMWDTPKE_OFFSET + ((i) * 0x4000)) + +/* Error detection and correction */ +#define ADF_GEN2_AE_CTX_ENABLES(i) ((i) * 0x1000 + 0x20818) +#define ADF_GEN2_AE_MISC_CONTROL(i) ((i) * 0x1000 + 0x20960) +#define ADF_GEN2_ENABLE_AE_ECC_ERR BIT(28) +#define ADF_GEN2_ENABLE_AE_ECC_PARITY_CORR (BIT(24) | BIT(12)) +#define ADF_GEN2_UERRSSMSH(i) ((i) * 0x4000 + 0x18) +#define ADF_GEN2_CERRSSMSH(i) ((i) * 0x4000 + 0x10) +#define ADF_GEN2_ERRSSMSH_EN BIT(3) + +/* Number of heartbeat counter pairs */ +#define ADF_NUM_HB_CNT_PER_AE ADF_NUM_THREADS_PER_AE + +/* Interrupts */ +#define ADF_GEN2_SMIAPF0_MASK_OFFSET (0x3A000 + 0x28) +#define ADF_GEN2_SMIAPF1_MASK_OFFSET (0x3A000 + 0x30) +#define ADF_GEN2_SMIA1_MASK 0x1 + +u32 adf_gen2_get_num_accels(struct adf_hw_device_data *self); +u32 adf_gen2_get_num_aes(struct adf_hw_device_data *self); +void adf_gen2_enable_error_correction(struct adf_accel_dev *accel_dev); +void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, + int num_a_regs, int num_b_regs); +void adf_gen2_get_admin_info(struct admin_info *admin_csrs_info); +void adf_gen2_get_arb_info(struct arb_info *arb_info); +void adf_gen2_enable_ints(struct adf_accel_dev *accel_dev); +u32 adf_gen2_get_accel_cap(struct adf_accel_dev *accel_dev); +void adf_gen2_set_ssm_wdtimer(struct adf_accel_dev *accel_dev); +void adf_gen2_init_dc_ops(struct adf_dc_ops *dc_ops); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_pfvf.c b/drivers/crypto/intel/qat/qat_common/adf_gen2_pfvf.c new file mode 100644 index 000000000..43af81fca --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_pfvf.c @@ -0,0 +1,401 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2021 Intel Corporation */ +#include +#include +#include +#include +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_gen2_pfvf.h" +#include "adf_pfvf_msg.h" +#include "adf_pfvf_pf_proto.h" +#include "adf_pfvf_vf_proto.h" +#include "adf_pfvf_utils.h" + + /* VF2PF interrupts */ +#define ADF_GEN2_VF_MSK 0xFFFF +#define ADF_GEN2_ERR_REG_VF2PF(vf_src) (((vf_src) & 0x01FFFE00) >> 9) +#define ADF_GEN2_ERR_MSK_VF2PF(vf_mask) (((vf_mask) & ADF_GEN2_VF_MSK) << 9) + +#define ADF_GEN2_PF_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) +#define ADF_GEN2_VF_PF2VF_OFFSET 0x200 + +#define ADF_GEN2_CSR_IN_USE 0x6AC2 +#define ADF_GEN2_CSR_IN_USE_MASK 0xFFFE + +enum gen2_csr_pos { + ADF_GEN2_CSR_PF2VF_OFFSET = 0, + ADF_GEN2_CSR_VF2PF_OFFSET = 16, +}; + +#define ADF_PFVF_GEN2_MSGTYPE_SHIFT 2 +#define ADF_PFVF_GEN2_MSGTYPE_MASK 0x0F +#define ADF_PFVF_GEN2_MSGDATA_SHIFT 6 +#define ADF_PFVF_GEN2_MSGDATA_MASK 0x3FF + +static const struct pfvf_csr_format csr_gen2_fmt = { + { ADF_PFVF_GEN2_MSGTYPE_SHIFT, ADF_PFVF_GEN2_MSGTYPE_MASK }, + { ADF_PFVF_GEN2_MSGDATA_SHIFT, ADF_PFVF_GEN2_MSGDATA_MASK }, +}; + +#define ADF_PFVF_MSG_RETRY_DELAY 5 +#define ADF_PFVF_MSG_MAX_RETRIES 3 + +static u32 adf_gen2_pf_get_pfvf_offset(u32 i) +{ + return ADF_GEN2_PF_PF2VF_OFFSET(i); +} + +static u32 adf_gen2_vf_get_pfvf_offset(u32 i) +{ + return ADF_GEN2_VF_PF2VF_OFFSET; +} + +static void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) +{ + /* Enable VF2PF Messaging Ints - VFs 0 through 15 per vf_mask[15:0] */ + if (vf_mask & ADF_GEN2_VF_MSK) { + u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3) + & ~ADF_GEN2_ERR_MSK_VF2PF(vf_mask); + ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val); + } +} + +static void adf_gen2_disable_all_vf2pf_interrupts(void __iomem *pmisc_addr) +{ + /* Disable VF2PF interrupts for VFs 0 through 15 per vf_mask[15:0] */ + u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3) + | ADF_GEN2_ERR_MSK_VF2PF(ADF_GEN2_VF_MSK); + ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val); +} + +static u32 adf_gen2_disable_pending_vf2pf_interrupts(void __iomem *pmisc_addr) +{ + u32 sources, disabled, pending; + u32 errsou3, errmsk3; + + /* Get the interrupt sources triggered by VFs */ + errsou3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRSOU3); + sources = ADF_GEN2_ERR_REG_VF2PF(errsou3); + + if (!sources) + return 0; + + /* Get the already disabled interrupts */ + errmsk3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3); + disabled = ADF_GEN2_ERR_REG_VF2PF(errmsk3); + + pending = sources & ~disabled; + if (!pending) + return 0; + + /* Due to HW limitations, when disabling the interrupts, we can't + * just disable the requested sources, as this would lead to missed + * interrupts if ERRSOU3 changes just before writing to ERRMSK3. + * To work around it, disable all and re-enable only the sources that + * are not in vf_mask and were not already disabled. Re-enabling will + * trigger a new interrupt for the sources that have changed in the + * meantime, if any. + */ + errmsk3 |= ADF_GEN2_ERR_MSK_VF2PF(ADF_GEN2_VF_MSK); + ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, errmsk3); + + /* Update only section of errmsk3 related to VF2PF */ + errmsk3 &= ~ADF_GEN2_ERR_MSK_VF2PF(ADF_GEN2_VF_MSK); + errmsk3 |= ADF_GEN2_ERR_MSK_VF2PF(sources | disabled); + ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, errmsk3); + + /* Return the sources of the (new) interrupt(s) */ + return pending; +} + +static u32 gen2_csr_get_int_bit(enum gen2_csr_pos offset) +{ + return ADF_PFVF_INT << offset; +} + +static u32 gen2_csr_msg_to_position(u32 csr_msg, enum gen2_csr_pos offset) +{ + return (csr_msg & 0xFFFF) << offset; +} + +static u32 gen2_csr_msg_from_position(u32 csr_val, enum gen2_csr_pos offset) +{ + return (csr_val >> offset) & 0xFFFF; +} + +static bool gen2_csr_is_in_use(u32 msg, enum gen2_csr_pos offset) +{ + return ((msg >> offset) & ADF_GEN2_CSR_IN_USE_MASK) == ADF_GEN2_CSR_IN_USE; +} + +static void gen2_csr_clear_in_use(u32 *msg, enum gen2_csr_pos offset) +{ + *msg &= ~(ADF_GEN2_CSR_IN_USE_MASK << offset); +} + +static void gen2_csr_set_in_use(u32 *msg, enum gen2_csr_pos offset) +{ + *msg |= (ADF_GEN2_CSR_IN_USE << offset); +} + +static bool is_legacy_user_pfvf_message(u32 msg) +{ + return !(msg & ADF_PFVF_MSGORIGIN_SYSTEM); +} + +static bool is_pf2vf_notification(u8 msg_type) +{ + switch (msg_type) { + case ADF_PF2VF_MSGTYPE_RESTARTING: + return true; + default: + return false; + } +} + +static bool is_vf2pf_notification(u8 msg_type) +{ + switch (msg_type) { + case ADF_VF2PF_MSGTYPE_INIT: + case ADF_VF2PF_MSGTYPE_SHUTDOWN: + return true; + default: + return false; + } +} + +struct pfvf_gen2_params { + u32 pfvf_offset; + struct mutex *csr_lock; /* lock preventing concurrent access of CSR */ + enum gen2_csr_pos local_offset; + enum gen2_csr_pos remote_offset; + bool (*is_notification_message)(u8 msg_type); + u8 compat_ver; +}; + +static int adf_gen2_pfvf_send(struct adf_accel_dev *accel_dev, + struct pfvf_message msg, + struct pfvf_gen2_params *params) +{ + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + enum gen2_csr_pos remote_offset = params->remote_offset; + enum gen2_csr_pos local_offset = params->local_offset; + unsigned int retries = ADF_PFVF_MSG_MAX_RETRIES; + struct mutex *lock = params->csr_lock; + u32 pfvf_offset = params->pfvf_offset; + u32 int_bit; + u32 csr_val; + u32 csr_msg; + int ret; + + /* Gen2 messages, both PF->VF and VF->PF, are all 16 bits long. This + * allows us to build and read messages as if they where all 0 based. + * However, send and receive are in a single shared 32 bits register, + * so we need to shift and/or mask the message half before decoding + * it and after encoding it. Which one to shift depends on the + * direction. + */ + + int_bit = gen2_csr_get_int_bit(local_offset); + + csr_msg = adf_pfvf_csr_msg_of(accel_dev, msg, &csr_gen2_fmt); + if (unlikely(!csr_msg)) + return -EINVAL; + + /* Prepare for CSR format, shifting the wire message in place and + * setting the in use pattern + */ + csr_msg = gen2_csr_msg_to_position(csr_msg, local_offset); + gen2_csr_set_in_use(&csr_msg, remote_offset); + + mutex_lock(lock); + +start: + /* Check if the PFVF CSR is in use by remote function */ + csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset); + if (gen2_csr_is_in_use(csr_val, local_offset)) { + dev_dbg(&GET_DEV(accel_dev), + "PFVF CSR in use by remote function\n"); + goto retry; + } + + /* Attempt to get ownership of the PFVF CSR */ + ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_msg | int_bit); + + /* Wait for confirmation from remote func it received the message */ + ret = read_poll_timeout(ADF_CSR_RD, csr_val, !(csr_val & int_bit), + ADF_PFVF_MSG_ACK_DELAY_US, + ADF_PFVF_MSG_ACK_MAX_DELAY_US, + true, pmisc_addr, pfvf_offset); + if (unlikely(ret < 0)) { + dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n"); + csr_val &= ~int_bit; + } + + /* For fire-and-forget notifications, the receiver does not clear + * the in-use pattern. This is used to detect collisions. + */ + if (params->is_notification_message(msg.type) && csr_val != csr_msg) { + /* Collision must have overwritten the message */ + dev_err(&GET_DEV(accel_dev), + "Collision on notification - PFVF CSR overwritten by remote function\n"); + goto retry; + } + + /* If the far side did not clear the in-use pattern it is either + * 1) Notification - message left intact to detect collision + * 2) Older protocol (compatibility version < 3) on the far side + * where the sender is responsible for clearing the in-use + * pattern after the received has acknowledged receipt. + * In either case, clear the in-use pattern now. + */ + if (gen2_csr_is_in_use(csr_val, remote_offset)) { + gen2_csr_clear_in_use(&csr_val, remote_offset); + ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val); + } + +out: + mutex_unlock(lock); + return ret; + +retry: + if (--retries) { + msleep(ADF_PFVF_MSG_RETRY_DELAY); + goto start; + } else { + ret = -EBUSY; + goto out; + } +} + +static struct pfvf_message adf_gen2_pfvf_recv(struct adf_accel_dev *accel_dev, + struct pfvf_gen2_params *params) +{ + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + enum gen2_csr_pos remote_offset = params->remote_offset; + enum gen2_csr_pos local_offset = params->local_offset; + u32 pfvf_offset = params->pfvf_offset; + struct pfvf_message msg = { 0 }; + u32 int_bit; + u32 csr_val; + u16 csr_msg; + + int_bit = gen2_csr_get_int_bit(local_offset); + + /* Read message */ + csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset); + if (!(csr_val & int_bit)) { + dev_info(&GET_DEV(accel_dev), + "Spurious PFVF interrupt, msg 0x%.8x. Ignored\n", csr_val); + return msg; + } + + /* Extract the message from the CSR */ + csr_msg = gen2_csr_msg_from_position(csr_val, local_offset); + + /* Ignore legacy non-system (non-kernel) messages */ + if (unlikely(is_legacy_user_pfvf_message(csr_msg))) { + dev_dbg(&GET_DEV(accel_dev), + "Ignored non-system message (0x%.8x);\n", csr_val); + /* Because this must be a legacy message, the far side + * must clear the in-use pattern, so don't do it. + */ + return msg; + } + + /* Return the pfvf_message format */ + msg = adf_pfvf_message_of(accel_dev, csr_msg, &csr_gen2_fmt); + + /* The in-use pattern is not cleared for notifications (so that + * it can be used for collision detection) or older implementations + */ + if (params->compat_ver >= ADF_PFVF_COMPAT_FAST_ACK && + !params->is_notification_message(msg.type)) + gen2_csr_clear_in_use(&csr_val, remote_offset); + + /* To ACK, clear the INT bit */ + csr_val &= ~int_bit; + ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val); + + return msg; +} + +static int adf_gen2_pf2vf_send(struct adf_accel_dev *accel_dev, struct pfvf_message msg, + u32 pfvf_offset, struct mutex *csr_lock) +{ + struct pfvf_gen2_params params = { + .csr_lock = csr_lock, + .pfvf_offset = pfvf_offset, + .local_offset = ADF_GEN2_CSR_PF2VF_OFFSET, + .remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET, + .is_notification_message = is_pf2vf_notification, + }; + + return adf_gen2_pfvf_send(accel_dev, msg, ¶ms); +} + +static int adf_gen2_vf2pf_send(struct adf_accel_dev *accel_dev, struct pfvf_message msg, + u32 pfvf_offset, struct mutex *csr_lock) +{ + struct pfvf_gen2_params params = { + .csr_lock = csr_lock, + .pfvf_offset = pfvf_offset, + .local_offset = ADF_GEN2_CSR_VF2PF_OFFSET, + .remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET, + .is_notification_message = is_vf2pf_notification, + }; + + return adf_gen2_pfvf_send(accel_dev, msg, ¶ms); +} + +static struct pfvf_message adf_gen2_pf2vf_recv(struct adf_accel_dev *accel_dev, + u32 pfvf_offset, u8 compat_ver) +{ + struct pfvf_gen2_params params = { + .pfvf_offset = pfvf_offset, + .local_offset = ADF_GEN2_CSR_PF2VF_OFFSET, + .remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET, + .is_notification_message = is_pf2vf_notification, + .compat_ver = compat_ver, + }; + + return adf_gen2_pfvf_recv(accel_dev, ¶ms); +} + +static struct pfvf_message adf_gen2_vf2pf_recv(struct adf_accel_dev *accel_dev, + u32 pfvf_offset, u8 compat_ver) +{ + struct pfvf_gen2_params params = { + .pfvf_offset = pfvf_offset, + .local_offset = ADF_GEN2_CSR_VF2PF_OFFSET, + .remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET, + .is_notification_message = is_vf2pf_notification, + .compat_ver = compat_ver, + }; + + return adf_gen2_pfvf_recv(accel_dev, ¶ms); +} + +void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) +{ + pfvf_ops->enable_comms = adf_enable_pf2vf_comms; + pfvf_ops->get_pf2vf_offset = adf_gen2_pf_get_pfvf_offset; + pfvf_ops->get_vf2pf_offset = adf_gen2_pf_get_pfvf_offset; + pfvf_ops->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts; + pfvf_ops->disable_all_vf2pf_interrupts = adf_gen2_disable_all_vf2pf_interrupts; + pfvf_ops->disable_pending_vf2pf_interrupts = adf_gen2_disable_pending_vf2pf_interrupts; + pfvf_ops->send_msg = adf_gen2_pf2vf_send; + pfvf_ops->recv_msg = adf_gen2_vf2pf_recv; +} +EXPORT_SYMBOL_GPL(adf_gen2_init_pf_pfvf_ops); + +void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) +{ + pfvf_ops->enable_comms = adf_enable_vf2pf_comms; + pfvf_ops->get_pf2vf_offset = adf_gen2_vf_get_pfvf_offset; + pfvf_ops->get_vf2pf_offset = adf_gen2_vf_get_pfvf_offset; + pfvf_ops->send_msg = adf_gen2_vf2pf_send; + pfvf_ops->recv_msg = adf_gen2_pf2vf_recv; +} +EXPORT_SYMBOL_GPL(adf_gen2_init_vf_pfvf_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_pfvf.h b/drivers/crypto/intel/qat/qat_common/adf_gen2_pfvf.h new file mode 100644 index 000000000..34a63cf40 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_pfvf.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2021 Intel Corporation */ +#ifndef ADF_GEN2_PFVF_H +#define ADF_GEN2_PFVF_H + +#include +#include "adf_accel_devices.h" +#include "adf_common_drv.h" + +#define ADF_GEN2_ERRSOU3 (0x3A000 + 0x0C) +#define ADF_GEN2_ERRSOU5 (0x3A000 + 0xD8) +#define ADF_GEN2_ERRMSK3 (0x3A000 + 0x1C) +#define ADF_GEN2_ERRMSK5 (0x3A000 + 0xDC) + +#if defined(CONFIG_PCI_IOV) +void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops); +void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops); +#else +static inline void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) +{ + pfvf_ops->enable_comms = adf_pfvf_comms_disabled; +} + +static inline void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) +{ + pfvf_ops->enable_comms = adf_pfvf_comms_disabled; +} +#endif + +#endif /* ADF_GEN2_PFVF_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_config.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_config.c new file mode 100644 index 000000000..afcdfdd0a --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_config.c @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ +#include "adf_accel_devices.h" +#include "adf_cfg.h" +#include "adf_cfg_services.h" +#include "adf_cfg_strings.h" +#include "adf_common_drv.h" +#include "adf_gen4_config.h" +#include "adf_heartbeat.h" +#include "adf_transport_access_macros.h" +#include "qat_compression.h" +#include "qat_crypto.h" + +int adf_crypto_dev_config(struct adf_accel_dev *accel_dev) +{ + char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + int banks = GET_MAX_BANKS(accel_dev); + int cpus = num_online_cpus(); + unsigned long bank, val; + int instances; + int ret; + int i; + + if (adf_hw_dev_has_crypto(accel_dev)) + instances = min(cpus, banks / 2); + else + instances = 0; + + for (i = 0; i < instances; i++) { + val = i; + bank = i * 2; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &bank, ADF_DEC); + if (ret) + goto err; + + bank += 1; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_BANK_NUM, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &bank, ADF_DEC); + if (ret) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY, + i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); + val = 128; + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 512; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 0; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 0; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 1; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 1; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = ADF_COALESCING_DEF_TIME; + snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); + ret = adf_cfg_add_key_value_param(accel_dev, "Accelerator0", + key, &val, ADF_DEC); + if (ret) + goto err; + } + + val = i; + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY, + &val, ADF_DEC); + if (ret) + goto err; + + val = 0; + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_DC, + &val, ADF_DEC); + if (ret) + goto err; + + return 0; +err: + dev_err(&GET_DEV(accel_dev), "Failed to add configuration for crypto\n"); + return ret; +} + +int adf_comp_dev_config(struct adf_accel_dev *accel_dev) +{ + char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + int banks = GET_MAX_BANKS(accel_dev); + int cpus = num_online_cpus(); + unsigned long val; + int instances; + int ret; + int i; + + if (adf_hw_dev_has_compression(accel_dev)) + instances = min(cpus, banks); + else + instances = 0; + + for (i = 0; i < instances; i++) { + val = i; + snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_BANK_NUM, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 512; + snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_SIZE, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 0; + snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_TX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 1; + snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_RX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = ADF_COALESCING_DEF_TIME; + snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); + ret = adf_cfg_add_key_value_param(accel_dev, "Accelerator0", + key, &val, ADF_DEC); + if (ret) + goto err; + } + + val = i; + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_DC, + &val, ADF_DEC); + if (ret) + goto err; + + val = 0; + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY, + &val, ADF_DEC); + if (ret) + goto err; + + return 0; +err: + dev_err(&GET_DEV(accel_dev), "Failed to add configuration for compression\n"); + return ret; +} + +int adf_no_dev_config(struct adf_accel_dev *accel_dev) +{ + unsigned long val; + int ret; + + val = 0; + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_DC, + &val, ADF_DEC); + if (ret) + return ret; + + return adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY, + &val, ADF_DEC); +} + +/** + * adf_gen4_dev_config() - create dev config required to create instances + * + * @accel_dev: Pointer to acceleration device. + * + * Function creates device configuration required to create instances + * + * Return: 0 on success, error code otherwise. + */ +int adf_gen4_dev_config(struct adf_accel_dev *accel_dev) +{ + int ret; + + ret = adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC); + if (ret) + goto err; + + ret = adf_cfg_section_add(accel_dev, "Accelerator0"); + if (ret) + goto err; + + switch (adf_get_service_enabled(accel_dev)) { + case SVC_SYM_ASYM: + ret = adf_crypto_dev_config(accel_dev); + break; + case SVC_DC: + case SVC_DCC: + ret = adf_comp_dev_config(accel_dev); + break; + default: + ret = adf_no_dev_config(accel_dev); + break; + } + + if (ret) + goto err; + + set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); + + return ret; + +err: + dev_err(&GET_DEV(accel_dev), "Failed to configure QAT driver\n"); + return ret; +} +EXPORT_SYMBOL_GPL(adf_gen4_dev_config); + +int adf_gen4_cfg_dev_init(struct adf_accel_dev *accel_dev) +{ + const char *config; + int ret; + + config = accel_dev->accel_id % 2 ? ADF_CFG_DC : ADF_CFG_CY; + + ret = adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC); + if (ret) + return ret; + + /* Default configuration is crypto only for even devices + * and compression for odd devices + */ + ret = adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC, + ADF_SERVICES_ENABLED, config, + ADF_STR); + if (ret) + return ret; + + adf_heartbeat_save_cfg_param(accel_dev, ADF_CFG_HB_TIMER_MIN_MS); + + return 0; +} +EXPORT_SYMBOL_GPL(adf_gen4_cfg_dev_init); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_config.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_config.h new file mode 100644 index 000000000..38a674c27 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_config.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ +#ifndef ADF_GEN4_CONFIG_H_ +#define ADF_GEN4_CONFIG_H_ + +#include "adf_accel_devices.h" + +int adf_gen4_dev_config(struct adf_accel_dev *accel_dev); +int adf_gen4_cfg_dev_init(struct adf_accel_dev *accel_dev); +int adf_crypto_dev_config(struct adf_accel_dev *accel_dev); +int adf_comp_dev_config(struct adf_accel_dev *accel_dev); +int adf_no_dev_config(struct adf_accel_dev *accel_dev); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_csr_data.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_csr_data.c new file mode 100644 index 000000000..6609c248a --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_csr_data.c @@ -0,0 +1,231 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2024 Intel Corporation */ +#include +#include "adf_gen4_hw_csr_data.h" + +static u64 build_csr_ring_base_addr(dma_addr_t addr, u32 size) +{ + return BUILD_RING_BASE_ADDR(addr, size); +} + +static u32 read_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring) +{ + return READ_CSR_RING_HEAD(csr_base_addr, bank, ring); +} + +static void write_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value); +} + +static u32 read_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring) +{ + return READ_CSR_RING_TAIL(csr_base_addr, bank, ring); +} + +static void write_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value); +} + +static u32 read_csr_stat(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_STAT(csr_base_addr, bank); +} + +static u32 read_csr_uo_stat(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_UO_STAT(csr_base_addr, bank); +} + +static u32 read_csr_e_stat(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_E_STAT(csr_base_addr, bank); +} + +static u32 read_csr_ne_stat(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_NE_STAT(csr_base_addr, bank); +} + +static u32 read_csr_nf_stat(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_NF_STAT(csr_base_addr, bank); +} + +static u32 read_csr_f_stat(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_F_STAT(csr_base_addr, bank); +} + +static u32 read_csr_c_stat(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_C_STAT(csr_base_addr, bank); +} + +static u32 read_csr_exp_stat(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_EXP_STAT(csr_base_addr, bank); +} + +static u32 read_csr_exp_int_en(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_EXP_INT_EN(csr_base_addr, bank); +} + +static void write_csr_exp_int_en(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_EXP_INT_EN(csr_base_addr, bank, value); +} + +static u32 read_csr_ring_config(void __iomem *csr_base_addr, u32 bank, + u32 ring) +{ + return READ_CSR_RING_CONFIG(csr_base_addr, bank, ring); +} + +static void write_csr_ring_config(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value); +} + +static dma_addr_t read_csr_ring_base(void __iomem *csr_base_addr, u32 bank, + u32 ring) +{ + return READ_CSR_RING_BASE(csr_base_addr, bank, ring); +} + +static void write_csr_ring_base(void __iomem *csr_base_addr, u32 bank, u32 ring, + dma_addr_t addr) +{ + WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, addr); +} + +static u32 read_csr_int_en(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_INT_EN(csr_base_addr, bank); +} + +static void write_csr_int_en(void __iomem *csr_base_addr, u32 bank, u32 value) +{ + WRITE_CSR_INT_EN(csr_base_addr, bank, value); +} + +static u32 read_csr_int_flag(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_INT_FLAG(csr_base_addr, bank); +} + +static void write_csr_int_flag(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_FLAG(csr_base_addr, bank, value); +} + +static u32 read_csr_int_srcsel(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_INT_SRCSEL(csr_base_addr, bank); +} + +static void write_csr_int_srcsel(void __iomem *csr_base_addr, u32 bank) +{ + WRITE_CSR_INT_SRCSEL(csr_base_addr, bank); +} + +static void write_csr_int_srcsel_w_val(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_SRCSEL_W_VAL(csr_base_addr, bank, value); +} + +static u32 read_csr_int_col_en(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_INT_COL_EN(csr_base_addr, bank); +} + +static void write_csr_int_col_en(void __iomem *csr_base_addr, u32 bank, u32 value) +{ + WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value); +} + +static u32 read_csr_int_col_ctl(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_INT_COL_CTL(csr_base_addr, bank); +} + +static void write_csr_int_col_ctl(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value); +} + +static u32 read_csr_int_flag_and_col(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_INT_FLAG_AND_COL(csr_base_addr, bank); +} + +static void write_csr_int_flag_and_col(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value); +} + +static u32 read_csr_ring_srv_arb_en(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_RING_SRV_ARB_EN(csr_base_addr, bank); +} + +static void write_csr_ring_srv_arb_en(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value); +} + +static u32 get_int_col_ctl_enable_mask(void) +{ + return ADF_RING_CSR_INT_COL_CTL_ENABLE; +} + +void adf_gen4_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) +{ + csr_ops->build_csr_ring_base_addr = build_csr_ring_base_addr; + csr_ops->read_csr_ring_head = read_csr_ring_head; + csr_ops->write_csr_ring_head = write_csr_ring_head; + csr_ops->read_csr_ring_tail = read_csr_ring_tail; + csr_ops->write_csr_ring_tail = write_csr_ring_tail; + csr_ops->read_csr_stat = read_csr_stat; + csr_ops->read_csr_uo_stat = read_csr_uo_stat; + csr_ops->read_csr_e_stat = read_csr_e_stat; + csr_ops->read_csr_ne_stat = read_csr_ne_stat; + csr_ops->read_csr_nf_stat = read_csr_nf_stat; + csr_ops->read_csr_f_stat = read_csr_f_stat; + csr_ops->read_csr_c_stat = read_csr_c_stat; + csr_ops->read_csr_exp_stat = read_csr_exp_stat; + csr_ops->read_csr_exp_int_en = read_csr_exp_int_en; + csr_ops->write_csr_exp_int_en = write_csr_exp_int_en; + csr_ops->read_csr_ring_config = read_csr_ring_config; + csr_ops->write_csr_ring_config = write_csr_ring_config; + csr_ops->read_csr_ring_base = read_csr_ring_base; + csr_ops->write_csr_ring_base = write_csr_ring_base; + csr_ops->read_csr_int_en = read_csr_int_en; + csr_ops->write_csr_int_en = write_csr_int_en; + csr_ops->read_csr_int_flag = read_csr_int_flag; + csr_ops->write_csr_int_flag = write_csr_int_flag; + csr_ops->read_csr_int_srcsel = read_csr_int_srcsel; + csr_ops->write_csr_int_srcsel = write_csr_int_srcsel; + csr_ops->write_csr_int_srcsel_w_val = write_csr_int_srcsel_w_val; + csr_ops->read_csr_int_col_en = read_csr_int_col_en; + csr_ops->write_csr_int_col_en = write_csr_int_col_en; + csr_ops->read_csr_int_col_ctl = read_csr_int_col_ctl; + csr_ops->write_csr_int_col_ctl = write_csr_int_col_ctl; + csr_ops->read_csr_int_flag_and_col = read_csr_int_flag_and_col; + csr_ops->write_csr_int_flag_and_col = write_csr_int_flag_and_col; + csr_ops->read_csr_ring_srv_arb_en = read_csr_ring_srv_arb_en; + csr_ops->write_csr_ring_srv_arb_en = write_csr_ring_srv_arb_en; + csr_ops->get_int_col_ctl_enable_mask = get_int_col_ctl_enable_mask; +} +EXPORT_SYMBOL_GPL(adf_gen4_init_hw_csr_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_csr_data.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_csr_data.h new file mode 100644 index 000000000..6f33e7c87 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_csr_data.h @@ -0,0 +1,188 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2024 Intel Corporation */ +#ifndef ADF_GEN4_HW_CSR_DATA_H_ +#define ADF_GEN4_HW_CSR_DATA_H_ + +#include +#include "adf_accel_devices.h" + +#define ADF_BANK_INT_SRC_SEL_MASK 0x44UL +#define ADF_RING_CSR_RING_CONFIG 0x1000 +#define ADF_RING_CSR_RING_LBASE 0x1040 +#define ADF_RING_CSR_RING_UBASE 0x1080 +#define ADF_RING_CSR_RING_HEAD 0x0C0 +#define ADF_RING_CSR_RING_TAIL 0x100 +#define ADF_RING_CSR_STAT 0x140 +#define ADF_RING_CSR_UO_STAT 0x148 +#define ADF_RING_CSR_E_STAT 0x14C +#define ADF_RING_CSR_NE_STAT 0x150 +#define ADF_RING_CSR_NF_STAT 0x154 +#define ADF_RING_CSR_F_STAT 0x158 +#define ADF_RING_CSR_C_STAT 0x15C +#define ADF_RING_CSR_INT_FLAG_EN 0x16C +#define ADF_RING_CSR_INT_FLAG 0x170 +#define ADF_RING_CSR_INT_SRCSEL 0x174 +#define ADF_RING_CSR_INT_COL_EN 0x17C +#define ADF_RING_CSR_INT_COL_CTL 0x180 +#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184 +#define ADF_RING_CSR_EXP_STAT 0x188 +#define ADF_RING_CSR_EXP_INT_EN 0x18C +#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000 +#define ADF_RING_CSR_ADDR_OFFSET 0x100000 +#define ADF_RING_BUNDLE_SIZE 0x2000 +#define ADF_RING_CSR_RING_SRV_ARB_EN 0x19C + +#define BUILD_RING_BASE_ADDR(addr, size) \ + ((((addr) >> 6) & (GENMASK_ULL(63, 0) << (size))) << 6) +#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_HEAD + ((ring) << 2)) +#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_TAIL + ((ring) << 2)) +#define READ_CSR_STAT(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_STAT) +#define READ_CSR_UO_STAT(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_UO_STAT) +#define READ_CSR_E_STAT(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_E_STAT) +#define READ_CSR_NE_STAT(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_NE_STAT) +#define READ_CSR_NF_STAT(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_NF_STAT) +#define READ_CSR_F_STAT(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_F_STAT) +#define READ_CSR_C_STAT(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_C_STAT) +#define READ_CSR_EXP_STAT(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_EXP_STAT) +#define READ_CSR_EXP_INT_EN(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_EXP_INT_EN) +#define WRITE_CSR_EXP_INT_EN(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_EXP_INT_EN, value) +#define READ_CSR_RING_CONFIG(csr_base_addr, bank, ring) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_CONFIG + ((ring) << 2)) +#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_CONFIG + ((ring) << 2), value) +#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value) \ +do { \ + void __iomem *_csr_base_addr = csr_base_addr; \ + u32 _bank = bank; \ + u32 _ring = ring; \ + dma_addr_t _value = value; \ + u32 l_base = 0, u_base = 0; \ + l_base = lower_32_bits(_value); \ + u_base = upper_32_bits(_value); \ + ADF_CSR_WR((_csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (_bank) + \ + ADF_RING_CSR_RING_LBASE + ((_ring) << 2), l_base); \ + ADF_CSR_WR((_csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (_bank) + \ + ADF_RING_CSR_RING_UBASE + ((_ring) << 2), u_base); \ +} while (0) + +static inline u64 read_base(void __iomem *csr_base_addr, u32 bank, u32 ring) +{ + u32 l_base, u_base; + + /* + * Use special IO wrapper for ring base as LBASE and UBASE are + * not physically contigious + */ + l_base = ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + + ADF_RING_CSR_RING_LBASE + (ring << 2)); + u_base = ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + + ADF_RING_CSR_RING_UBASE + (ring << 2)); + + return (u64)u_base << 32 | (u64)l_base; +} + +#define READ_CSR_RING_BASE(csr_base_addr, bank, ring) \ + read_base((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, (bank), (ring)) + +#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_HEAD + ((ring) << 2), value) +#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_TAIL + ((ring) << 2), value) +#define READ_CSR_INT_EN(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_INT_FLAG_EN) +#define WRITE_CSR_INT_EN(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_FLAG_EN, (value)) +#define READ_CSR_INT_FLAG(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_INT_FLAG) +#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_FLAG, (value)) +#define READ_CSR_INT_SRCSEL(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_INT_SRCSEL) +#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_SRCSEL, ADF_BANK_INT_SRC_SEL_MASK) +#define WRITE_CSR_INT_SRCSEL_W_VAL(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_SRCSEL, (value)) +#define READ_CSR_INT_COL_EN(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_INT_COL_EN) +#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_COL_EN, (value)) +#define READ_CSR_INT_COL_CTL(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_INT_COL_CTL) +#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_COL_CTL, \ + ADF_RING_CSR_INT_COL_CTL_ENABLE | (value)) +#define READ_CSR_INT_FLAG_AND_COL(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_FLAG_AND_COL) +#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_FLAG_AND_COL, (value)) + +#define READ_CSR_RING_SRV_ARB_EN(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_SRV_ARB_EN) +#define WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_SRV_ARB_EN, (value)) + +void adf_gen4_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c new file mode 100644 index 000000000..f4a58f040 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c @@ -0,0 +1,610 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2020 Intel Corporation */ + +#define pr_fmt(fmt) "QAT: " fmt + +#include +#include +#include +#include "adf_accel_devices.h" +#include "adf_cfg_services.h" +#include "adf_common_drv.h" +#include "adf_fw_config.h" +#include "adf_gen4_hw_data.h" +#include "adf_gen4_pm.h" +#include "icp_qat_fw_comp.h" +#include "icp_qat_hw_20_comp.h" + +u32 adf_gen4_get_accel_mask(struct adf_hw_device_data *self) +{ + return ADF_GEN4_ACCELERATORS_MASK; +} +EXPORT_SYMBOL_GPL(adf_gen4_get_accel_mask); + +u32 adf_gen4_get_num_accels(struct adf_hw_device_data *self) +{ + return ADF_GEN4_MAX_ACCELERATORS; +} +EXPORT_SYMBOL_GPL(adf_gen4_get_num_accels); + +u32 adf_gen4_get_num_aes(struct adf_hw_device_data *self) +{ + if (!self || !self->ae_mask) + return 0; + + return hweight32(self->ae_mask); +} +EXPORT_SYMBOL_GPL(adf_gen4_get_num_aes); + +u32 adf_gen4_get_misc_bar_id(struct adf_hw_device_data *self) +{ + return ADF_GEN4_PMISC_BAR; +} +EXPORT_SYMBOL_GPL(adf_gen4_get_misc_bar_id); + +u32 adf_gen4_get_etr_bar_id(struct adf_hw_device_data *self) +{ + return ADF_GEN4_ETR_BAR; +} +EXPORT_SYMBOL_GPL(adf_gen4_get_etr_bar_id); + +u32 adf_gen4_get_sram_bar_id(struct adf_hw_device_data *self) +{ + return ADF_GEN4_SRAM_BAR; +} +EXPORT_SYMBOL_GPL(adf_gen4_get_sram_bar_id); + +enum dev_sku_info adf_gen4_get_sku(struct adf_hw_device_data *self) +{ + return DEV_SKU_1; +} +EXPORT_SYMBOL_GPL(adf_gen4_get_sku); + +void adf_gen4_get_arb_info(struct arb_info *arb_info) +{ + arb_info->arb_cfg = ADF_GEN4_ARB_CONFIG; + arb_info->arb_offset = ADF_GEN4_ARB_OFFSET; + arb_info->wt2sam_offset = ADF_GEN4_ARB_WRK_2_SER_MAP_OFFSET; +} +EXPORT_SYMBOL_GPL(adf_gen4_get_arb_info); + +void adf_gen4_get_admin_info(struct admin_info *admin_csrs_info) +{ + admin_csrs_info->mailbox_offset = ADF_GEN4_MAILBOX_BASE_OFFSET; + admin_csrs_info->admin_msg_ur = ADF_GEN4_ADMINMSGUR_OFFSET; + admin_csrs_info->admin_msg_lr = ADF_GEN4_ADMINMSGLR_OFFSET; +} +EXPORT_SYMBOL_GPL(adf_gen4_get_admin_info); + +u32 adf_gen4_get_heartbeat_clock(struct adf_hw_device_data *self) +{ + /* + * GEN4 uses KPT counter for HB + */ + return ADF_GEN4_KPT_COUNTER_FREQ; +} +EXPORT_SYMBOL_GPL(adf_gen4_get_heartbeat_clock); + +void adf_gen4_enable_error_correction(struct adf_accel_dev *accel_dev) +{ + struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_GEN4_PMISC_BAR]; + void __iomem *csr = misc_bar->virt_addr; + + /* Enable all in errsou3 except VFLR notification on host */ + ADF_CSR_WR(csr, ADF_GEN4_ERRMSK3, ADF_GEN4_VFLNOTIFY); +} +EXPORT_SYMBOL_GPL(adf_gen4_enable_error_correction); + +void adf_gen4_enable_ints(struct adf_accel_dev *accel_dev) +{ + void __iomem *addr; + + addr = (&GET_BARS(accel_dev)[ADF_GEN4_PMISC_BAR])->virt_addr; + + /* Enable bundle interrupts */ + ADF_CSR_WR(addr, ADF_GEN4_SMIAPF_RP_X0_MASK_OFFSET, 0); + ADF_CSR_WR(addr, ADF_GEN4_SMIAPF_RP_X1_MASK_OFFSET, 0); + + /* Enable misc interrupts */ + ADF_CSR_WR(addr, ADF_GEN4_SMIAPF_MASK_OFFSET, 0); +} +EXPORT_SYMBOL_GPL(adf_gen4_enable_ints); + +int adf_gen4_init_device(struct adf_accel_dev *accel_dev) +{ + void __iomem *addr; + u32 status; + u32 csr; + int ret; + + addr = (&GET_BARS(accel_dev)[ADF_GEN4_PMISC_BAR])->virt_addr; + + /* Temporarily mask PM interrupt */ + csr = ADF_CSR_RD(addr, ADF_GEN4_ERRMSK2); + csr |= ADF_GEN4_PM_SOU; + ADF_CSR_WR(addr, ADF_GEN4_ERRMSK2, csr); + + /* Set DRV_ACTIVE bit to power up the device */ + ADF_CSR_WR(addr, ADF_GEN4_PM_INTERRUPT, ADF_GEN4_PM_DRV_ACTIVE); + + /* Poll status register to make sure the device is powered up */ + ret = read_poll_timeout(ADF_CSR_RD, status, + status & ADF_GEN4_PM_INIT_STATE, + ADF_GEN4_PM_POLL_DELAY_US, + ADF_GEN4_PM_POLL_TIMEOUT_US, true, addr, + ADF_GEN4_PM_STATUS); + if (ret) + dev_err(&GET_DEV(accel_dev), "Failed to power up the device\n"); + + return ret; +} +EXPORT_SYMBOL_GPL(adf_gen4_init_device); + +void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev) +{ + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + u64 timer_val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE; + u64 timer_val = ADF_SSM_WDT_DEFAULT_VALUE; + + /* Enable watchdog timer for sym and dc */ + ADF_CSR_WR64_LO_HI(pmisc_addr, ADF_SSMWDTL_OFFSET, ADF_SSMWDTH_OFFSET, timer_val); + + /* Enable watchdog timer for pke */ + ADF_CSR_WR64_LO_HI(pmisc_addr, ADF_SSMWDTPKEL_OFFSET, ADF_SSMWDTPKEH_OFFSET, + timer_val_pke); +} +EXPORT_SYMBOL_GPL(adf_gen4_set_ssm_wdtimer); + +/* + * The vector routing table is used to select the MSI-X entry to use for each + * interrupt source. + * The first ADF_GEN4_ETR_MAX_BANKS entries correspond to ring interrupts. + * The final entry corresponds to VF2PF or error interrupts. + * This vector table could be used to configure one MSI-X entry to be shared + * between multiple interrupt sources. + * + * The default routing is set to have a one to one correspondence between the + * interrupt source and the MSI-X entry used. + */ +void adf_gen4_set_msix_default_rttable(struct adf_accel_dev *accel_dev) +{ + void __iomem *csr; + int i; + + csr = (&GET_BARS(accel_dev)[ADF_GEN4_PMISC_BAR])->virt_addr; + for (i = 0; i <= ADF_GEN4_ETR_MAX_BANKS; i++) + ADF_CSR_WR(csr, ADF_GEN4_MSIX_RTTABLE_OFFSET(i), i); +} +EXPORT_SYMBOL_GPL(adf_gen4_set_msix_default_rttable); + +int adf_pfvf_comms_disabled(struct adf_accel_dev *accel_dev) +{ + return 0; +} +EXPORT_SYMBOL_GPL(adf_pfvf_comms_disabled); + +static int reset_ring_pair(void __iomem *csr, u32 bank_number) +{ + u32 status; + int ret; + + /* Write rpresetctl register BIT(0) as 1 + * Since rpresetctl registers have no RW fields, no need to preserve + * values for other bits. Just write directly. + */ + ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETCTL(bank_number), + ADF_WQM_CSR_RPRESETCTL_RESET); + + /* Read rpresetsts register and wait for rp reset to complete */ + ret = read_poll_timeout(ADF_CSR_RD, status, + status & ADF_WQM_CSR_RPRESETSTS_STATUS, + ADF_RPRESET_POLL_DELAY_US, + ADF_RPRESET_POLL_TIMEOUT_US, true, + csr, ADF_WQM_CSR_RPRESETSTS(bank_number)); + if (!ret) { + /* When rp reset is done, clear rpresetsts */ + ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETSTS(bank_number), + ADF_WQM_CSR_RPRESETSTS_STATUS); + } + + return ret; +} + +int adf_gen4_ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *csr = adf_get_etr_base(accel_dev); + int ret; + + if (bank_number >= hw_data->num_banks) + return -EINVAL; + + dev_dbg(&GET_DEV(accel_dev), + "ring pair reset for bank:%d\n", bank_number); + + ret = reset_ring_pair(csr, bank_number); + if (ret) + dev_err(&GET_DEV(accel_dev), + "ring pair reset failed (timeout)\n"); + else + dev_dbg(&GET_DEV(accel_dev), "ring pair reset successful\n"); + + return ret; +} +EXPORT_SYMBOL_GPL(adf_gen4_ring_pair_reset); + +static const u32 thrd_to_arb_map_dcc[] = { + 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x0000FFFF, 0x0000FFFF, 0x0000FFFF, 0x0000FFFF, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x0 +}; + +static const u16 rp_group_to_arb_mask[] = { + [RP_GROUP_0] = 0x5, + [RP_GROUP_1] = 0xA, +}; + +static bool is_single_service(int service_id) +{ + switch (service_id) { + case SVC_DC: + case SVC_SYM: + case SVC_ASYM: + return true; + default: + return false; + } +} + +bool adf_gen4_services_supported(unsigned long mask) +{ + unsigned long num_svc = hweight_long(mask); + + if (mask >= BIT(SVC_COUNT)) + return false; + + if (test_bit(SVC_DECOMP, &mask)) + return false; + + switch (num_svc) { + case ADF_ONE_SERVICE: + return true; + case ADF_TWO_SERVICES: + return !test_bit(SVC_DCC, &mask); + default: + return false; + } +} +EXPORT_SYMBOL_GPL(adf_gen4_services_supported); + +int adf_gen4_init_thd2arb_map(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + u32 *thd2arb_map = hw_data->thd_to_arb_map; + unsigned int ae_cnt, worker_obj_cnt, i, j; + unsigned long ae_mask, thds_mask; + int srv_id, rp_group; + u32 thd2arb_map_base; + u16 arb_mask; + + if (!hw_data->get_rp_group || !hw_data->get_ena_thd_mask || + !hw_data->get_num_aes || !hw_data->uof_get_num_objs || + !hw_data->uof_get_ae_mask) + return -EFAULT; + + srv_id = adf_get_service_enabled(accel_dev); + if (srv_id < 0) + return srv_id; + + ae_cnt = hw_data->get_num_aes(hw_data); + worker_obj_cnt = hw_data->uof_get_num_objs(accel_dev) - + ADF_GEN4_ADMIN_ACCELENGINES; + + if (srv_id == SVC_DCC) { + if (ae_cnt > ICP_QAT_HW_AE_DELIMITER) + return -EINVAL; + + memcpy(thd2arb_map, thrd_to_arb_map_dcc, + array_size(sizeof(*thd2arb_map), ae_cnt)); + return 0; + } + + for (i = 0; i < worker_obj_cnt; i++) { + ae_mask = hw_data->uof_get_ae_mask(accel_dev, i); + rp_group = hw_data->get_rp_group(accel_dev, ae_mask); + thds_mask = hw_data->get_ena_thd_mask(accel_dev, i); + thd2arb_map_base = 0; + + if (rp_group >= RP_GROUP_COUNT || rp_group < RP_GROUP_0) + return -EINVAL; + + if (thds_mask == ADF_GEN4_ENA_THD_MASK_ERROR) + return -EINVAL; + + if (is_single_service(srv_id)) + arb_mask = rp_group_to_arb_mask[RP_GROUP_0] | + rp_group_to_arb_mask[RP_GROUP_1]; + else + arb_mask = rp_group_to_arb_mask[rp_group]; + + for_each_set_bit(j, &thds_mask, ADF_NUM_THREADS_PER_AE) + thd2arb_map_base |= arb_mask << (j * 4); + + for_each_set_bit(j, &ae_mask, ae_cnt) + thd2arb_map[j] = thd2arb_map_base; + } + return 0; +} +EXPORT_SYMBOL_GPL(adf_gen4_init_thd2arb_map); + +u16 adf_gen4_get_ring_to_svc_map(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + enum adf_cfg_service_type rps[RP_GROUP_COUNT] = { }; + unsigned int ae_mask, start_id, worker_obj_cnt, i; + u16 ring_to_svc_map; + int rp_group; + + if (!hw_data->get_rp_group || !hw_data->uof_get_ae_mask || + !hw_data->uof_get_obj_type || !hw_data->uof_get_num_objs) + return 0; + + /* If dcc, all rings handle compression requests */ + if (adf_get_service_enabled(accel_dev) == SVC_DCC) { + for (i = 0; i < RP_GROUP_COUNT; i++) + rps[i] = COMP; + goto set_mask; + } + + worker_obj_cnt = hw_data->uof_get_num_objs(accel_dev) - + ADF_GEN4_ADMIN_ACCELENGINES; + start_id = worker_obj_cnt - RP_GROUP_COUNT; + + for (i = start_id; i < worker_obj_cnt; i++) { + ae_mask = hw_data->uof_get_ae_mask(accel_dev, i); + rp_group = hw_data->get_rp_group(accel_dev, ae_mask); + if (rp_group >= RP_GROUP_COUNT || rp_group < RP_GROUP_0) + return 0; + + switch (hw_data->uof_get_obj_type(accel_dev, i)) { + case ADF_FW_SYM_OBJ: + rps[rp_group] = SYM; + break; + case ADF_FW_ASYM_OBJ: + rps[rp_group] = ASYM; + break; + case ADF_FW_DC_OBJ: + rps[rp_group] = COMP; + break; + default: + rps[rp_group] = 0; + break; + } + } + +set_mask: + ring_to_svc_map = rps[RP_GROUP_0] << ADF_CFG_SERV_RING_PAIR_0_SHIFT | + rps[RP_GROUP_1] << ADF_CFG_SERV_RING_PAIR_1_SHIFT | + rps[RP_GROUP_0] << ADF_CFG_SERV_RING_PAIR_2_SHIFT | + rps[RP_GROUP_1] << ADF_CFG_SERV_RING_PAIR_3_SHIFT; + + return ring_to_svc_map; +} +EXPORT_SYMBOL_GPL(adf_gen4_get_ring_to_svc_map); + +/* + * adf_gen4_bank_quiesce_coal_timer() - quiesce bank coalesced interrupt timer + * @accel_dev: Pointer to the device structure + * @bank_idx: Offset to the bank within this device + * @timeout_ms: Timeout in milliseconds for the operation + * + * This function tries to quiesce the coalesced interrupt timer of a bank if + * it has been enabled and triggered. + * + * Returns 0 on success, error code otherwise + * + */ +int adf_gen4_bank_quiesce_coal_timer(struct adf_accel_dev *accel_dev, + u32 bank_idx, int timeout_ms) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); + void __iomem *csr_misc = adf_get_pmisc_base(accel_dev); + void __iomem *csr_etr = adf_get_etr_base(accel_dev); + u32 int_col_ctl, int_col_mask, int_col_en; + u32 e_stat, intsrc; + u64 wait_us; + int ret; + + if (timeout_ms < 0) + return -EINVAL; + + int_col_ctl = csr_ops->read_csr_int_col_ctl(csr_etr, bank_idx); + int_col_mask = csr_ops->get_int_col_ctl_enable_mask(); + if (!(int_col_ctl & int_col_mask)) + return 0; + + int_col_en = csr_ops->read_csr_int_col_en(csr_etr, bank_idx); + int_col_en &= BIT(ADF_WQM_CSR_RP_IDX_RX); + + e_stat = csr_ops->read_csr_e_stat(csr_etr, bank_idx); + if (!(~e_stat & int_col_en)) + return 0; + + wait_us = 2 * ((int_col_ctl & ~int_col_mask) << 8) * USEC_PER_SEC; + do_div(wait_us, hw_data->clock_frequency); + wait_us = min(wait_us, (u64)timeout_ms * USEC_PER_MSEC); + dev_dbg(&GET_DEV(accel_dev), + "wait for bank %d - coalesced timer expires in %llu us (max=%u ms estat=0x%x intcolen=0x%x)\n", + bank_idx, wait_us, timeout_ms, e_stat, int_col_en); + + ret = read_poll_timeout(ADF_CSR_RD, intsrc, intsrc, + ADF_COALESCED_POLL_DELAY_US, wait_us, true, + csr_misc, ADF_WQM_CSR_RPINTSOU(bank_idx)); + if (ret) + dev_warn(&GET_DEV(accel_dev), + "coalesced timer for bank %d expired (%llu us)\n", + bank_idx, wait_us); + + return ret; +} +EXPORT_SYMBOL_GPL(adf_gen4_bank_quiesce_coal_timer); + +static int drain_bank(void __iomem *csr, u32 bank_number, int timeout_us) +{ + u32 status; + + ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETCTL(bank_number), + ADF_WQM_CSR_RPRESETCTL_DRAIN); + + return read_poll_timeout(ADF_CSR_RD, status, + status & ADF_WQM_CSR_RPRESETSTS_STATUS, + ADF_RPRESET_POLL_DELAY_US, timeout_us, true, + csr, ADF_WQM_CSR_RPRESETSTS(bank_number)); +} + +void adf_gen4_bank_drain_finish(struct adf_accel_dev *accel_dev, + u32 bank_number) +{ + void __iomem *csr = adf_get_etr_base(accel_dev); + + ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETSTS(bank_number), + ADF_WQM_CSR_RPRESETSTS_STATUS); +} + +int adf_gen4_bank_drain_start(struct adf_accel_dev *accel_dev, + u32 bank_number, int timeout_us) +{ + void __iomem *csr = adf_get_etr_base(accel_dev); + int ret; + + dev_dbg(&GET_DEV(accel_dev), "Drain bank %d\n", bank_number); + + ret = drain_bank(csr, bank_number, timeout_us); + if (ret) + dev_err(&GET_DEV(accel_dev), "Bank drain failed (timeout)\n"); + else + dev_dbg(&GET_DEV(accel_dev), "Bank drain successful\n"); + + return ret; +} + +static int adf_gen4_build_comp_block(void *ctx, enum adf_dc_algo algo) +{ + struct icp_qat_fw_comp_req *req_tmpl = ctx; + struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_hw_comp_20_config_csr_upper hw_comp_upper_csr = { }; + struct icp_qat_hw_comp_20_config_csr_lower hw_comp_lower_csr = { }; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + u32 upper_val; + u32 lower_val; + + switch (algo) { + case QAT_DEFLATE: + header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DYNAMIC; + hw_comp_lower_csr.algo = ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_ILZ77; + hw_comp_lower_csr.lllbd = ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_ENABLED; + hw_comp_lower_csr.skip_ctrl = ICP_QAT_HW_COMP_20_BYTE_SKIP_3BYTE_LITERAL; + break; + case QAT_LZ4S: + header->service_cmd_id = ICP_QAT_FW_COMP_20_CMD_LZ4S_COMPRESS; + hw_comp_lower_csr.algo = ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_LZ4S; + hw_comp_lower_csr.lllbd = ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_DISABLED; + hw_comp_lower_csr.abd = ICP_QAT_HW_COMP_20_ABD_ABD_DISABLED; + break; + default: + return -EINVAL; + } + + hw_comp_lower_csr.sd = ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_1; + hw_comp_lower_csr.hash_update = ICP_QAT_HW_COMP_20_SKIP_HASH_UPDATE_DONT_ALLOW; + hw_comp_lower_csr.edmm = ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_ENABLED; + hw_comp_upper_csr.nice = ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_DEFAULT_VAL; + hw_comp_upper_csr.lazy = ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_DEFAULT_VAL; + + upper_val = ICP_QAT_FW_COMP_20_BUILD_CONFIG_UPPER(hw_comp_upper_csr); + lower_val = ICP_QAT_FW_COMP_20_BUILD_CONFIG_LOWER(hw_comp_lower_csr); + + cd_pars->u.sl.comp_slice_cfg_word[0] = lower_val; + cd_pars->u.sl.comp_slice_cfg_word[1] = upper_val; + + return 0; +} + +static int adf_gen4_build_decomp_block(void *ctx, enum adf_dc_algo algo) +{ + struct icp_qat_fw_comp_req *req_tmpl = ctx; + struct icp_qat_hw_decomp_20_config_csr_lower hw_decomp_lower_csr = { }; + struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + u32 lower_val; + + switch (algo) { + case QAT_DEFLATE: + header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS; + hw_decomp_lower_csr.algo = ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_DEFLATE; + break; + case QAT_LZ4S: + header->service_cmd_id = ICP_QAT_FW_COMP_20_CMD_LZ4S_DECOMPRESS; + hw_decomp_lower_csr.algo = ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_LZ4S; + break; + default: + return -EINVAL; + } + + lower_val = ICP_QAT_FW_DECOMP_20_BUILD_CONFIG_LOWER(hw_decomp_lower_csr); + + cd_pars->u.sl.comp_slice_cfg_word[0] = lower_val; + cd_pars->u.sl.comp_slice_cfg_word[1] = 0; + + return 0; +} + +void adf_gen4_init_dc_ops(struct adf_dc_ops *dc_ops) +{ + dc_ops->build_comp_block = adf_gen4_build_comp_block; + dc_ops->build_decomp_block = adf_gen4_build_decomp_block; +} +EXPORT_SYMBOL_GPL(adf_gen4_init_dc_ops); + +void adf_gen4_init_num_svc_aes(struct adf_rl_hw_data *device_data) +{ + struct adf_hw_device_data *hw_data; + unsigned int i; + u32 ae_cnt; + + hw_data = container_of(device_data, struct adf_hw_device_data, rl_data); + ae_cnt = hweight32(hw_data->get_ae_mask(hw_data)); + if (!ae_cnt) + return; + + for (i = 0; i < SVC_BASE_COUNT; i++) + device_data->svc_ae_mask[i] = ae_cnt - 1; + + /* + * The decompression service is not supported on QAT GEN4 devices. + * Therefore, set svc_ae_mask to 0. + */ + device_data->svc_ae_mask[SVC_DECOMP] = 0; +} +EXPORT_SYMBOL_GPL(adf_gen4_init_num_svc_aes); + +u32 adf_gen4_get_svc_slice_cnt(struct adf_accel_dev *accel_dev, + enum adf_base_services svc) +{ + struct adf_rl_hw_data *device_data = &accel_dev->hw_device->rl_data; + + switch (svc) { + case SVC_SYM: + return device_data->slices.cph_cnt; + case SVC_ASYM: + return device_data->slices.pke_cnt; + case SVC_DC: + return device_data->slices.dcpr_cnt; + default: + return 0; + } +} +EXPORT_SYMBOL_GPL(adf_gen4_get_svc_slice_cnt); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h new file mode 100644 index 000000000..cd26b6724 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h @@ -0,0 +1,182 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2020 Intel Corporation */ +#ifndef ADF_GEN4_HW_DATA_H_ +#define ADF_GEN4_HW_DATA_H_ + +#include + +#include "adf_accel_devices.h" +#include "adf_cfg_common.h" +#include "adf_dc.h" + +/* PCIe configuration space */ +#define ADF_GEN4_BAR_MASK (BIT(0) | BIT(2) | BIT(4)) +#define ADF_GEN4_SRAM_BAR 0 +#define ADF_GEN4_PMISC_BAR 1 +#define ADF_GEN4_ETR_BAR 2 + +/* Clocks frequency */ +#define ADF_GEN4_KPT_COUNTER_FREQ (100 * HZ_PER_MHZ) + +/* Physical function fuses */ +#define ADF_GEN4_FUSECTL0_OFFSET 0x2C8 +#define ADF_GEN4_FUSECTL1_OFFSET 0x2CC +#define ADF_GEN4_FUSECTL2_OFFSET 0x2D0 +#define ADF_GEN4_FUSECTL3_OFFSET 0x2D4 +#define ADF_GEN4_FUSECTL4_OFFSET 0x2D8 +#define ADF_GEN4_FUSECTL5_OFFSET 0x2DC + +/* Accelerators */ +#define ADF_GEN4_ACCELERATORS_MASK 0x1 +#define ADF_GEN4_MAX_ACCELERATORS 1 +#define ADF_GEN4_ADMIN_ACCELENGINES 1 + +/* MSIX interrupt */ +#define ADF_GEN4_SMIAPF_RP_X0_MASK_OFFSET 0x41A040 +#define ADF_GEN4_SMIAPF_RP_X1_MASK_OFFSET 0x41A044 +#define ADF_GEN4_SMIAPF_MASK_OFFSET 0x41A084 +#define ADF_GEN4_MSIX_RTTABLE_OFFSET(i) (0x409000 + ((i) * 0x04)) + +/* Bank and ring configuration */ +#define ADF_GEN4_MAX_RPS 64 +#define ADF_GEN4_NUM_RINGS_PER_BANK 2 +#define ADF_GEN4_NUM_BANKS_PER_VF 4 +#define ADF_GEN4_ETR_MAX_BANKS 64 +#define ADF_GEN4_RX_RINGS_OFFSET 1 +#define ADF_GEN4_TX_RINGS_MASK 0x1 + +/* Arbiter configuration */ +#define ADF_GEN4_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0)) +#define ADF_GEN4_ARB_OFFSET 0x0 +#define ADF_GEN4_ARB_WRK_2_SER_MAP_OFFSET 0x400 + +/* Admin Interface Reg Offset */ +#define ADF_GEN4_ADMINMSGUR_OFFSET 0x500574 +#define ADF_GEN4_ADMINMSGLR_OFFSET 0x500578 +#define ADF_GEN4_MAILBOX_BASE_OFFSET 0x600970 + +/* Default ring mapping */ +#define ADF_GEN4_DEFAULT_RING_TO_SRV_MAP \ + (ASYM << ADF_CFG_SERV_RING_PAIR_0_SHIFT | \ + SYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \ + ASYM << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \ + SYM << ADF_CFG_SERV_RING_PAIR_3_SHIFT) + +/* WDT timers + * + * Timeout is in cycles. Clock speed may vary across products but this + * value should be a few milli-seconds. + */ +#define ADF_SSM_WDT_DEFAULT_VALUE 0x7000000ULL +#define ADF_SSM_WDT_PKE_DEFAULT_VALUE 0x8000000 +#define ADF_SSMWDTL_OFFSET 0x54 +#define ADF_SSMWDTH_OFFSET 0x5C +#define ADF_SSMWDTPKEL_OFFSET 0x58 +#define ADF_SSMWDTPKEH_OFFSET 0x60 + +/* Ring reset */ +#define ADF_RPRESET_POLL_TIMEOUT_US (5 * USEC_PER_SEC) +#define ADF_RPRESET_POLL_DELAY_US 20 +#define ADF_WQM_CSR_RPRESETCTL_RESET BIT(0) +#define ADF_WQM_CSR_RPRESETCTL_DRAIN BIT(2) +#define ADF_WQM_CSR_RPRESETCTL(bank) (0x6000 + ((bank) << 3)) +#define ADF_WQM_CSR_RPRESETSTS_STATUS BIT(0) +#define ADF_WQM_CSR_RPRESETSTS(bank) (ADF_WQM_CSR_RPRESETCTL(bank) + 4) + +/* Ring interrupt */ +#define ADF_COALESCED_POLL_TIMEOUT_US (1 * USEC_PER_SEC) +#define ADF_COALESCED_POLL_DELAY_US 1000 +#define ADF_WQM_CSR_RPINTSOU(bank) (0x200000 + ((bank) << 12)) +#define ADF_WQM_CSR_RP_IDX_RX 1 + +/* Error source registers */ +#define ADF_GEN4_ERRSOU0 (0x41A200) +#define ADF_GEN4_ERRSOU1 (0x41A204) +#define ADF_GEN4_ERRSOU2 (0x41A208) +#define ADF_GEN4_ERRSOU3 (0x41A20C) + +/* Error source mask registers */ +#define ADF_GEN4_ERRMSK0 (0x41A210) +#define ADF_GEN4_ERRMSK1 (0x41A214) +#define ADF_GEN4_ERRMSK2 (0x41A218) +#define ADF_GEN4_ERRMSK3 (0x41A21C) + +#define ADF_GEN4_VFLNOTIFY BIT(7) + +/* Number of heartbeat counter pairs */ +#define ADF_NUM_HB_CNT_PER_AE ADF_NUM_THREADS_PER_AE + +/* Rate Limiting */ +#define ADF_GEN4_RL_R2L_OFFSET 0x508000 +#define ADF_GEN4_RL_L2C_OFFSET 0x509000 +#define ADF_GEN4_RL_C2S_OFFSET 0x508818 +#define ADF_GEN4_RL_TOKEN_PCIEIN_BUCKET_OFFSET 0x508800 +#define ADF_GEN4_RL_TOKEN_PCIEOUT_BUCKET_OFFSET 0x508804 + +/* Arbiter threads mask with error value */ +#define ADF_GEN4_ENA_THD_MASK_ERROR GENMASK(ADF_NUM_THREADS_PER_AE, 0) + +/* PF2VM communication channel */ +#define ADF_GEN4_PF2VM_OFFSET(i) (0x40B010 + (i) * 0x20) +#define ADF_GEN4_VM2PF_OFFSET(i) (0x40B014 + (i) * 0x20) +#define ADF_GEN4_VINTMSKPF2VM_OFFSET(i) (0x40B00C + (i) * 0x20) +#define ADF_GEN4_VINTSOUPF2VM_OFFSET(i) (0x40B008 + (i) * 0x20) +#define ADF_GEN4_VINTMSK_OFFSET(i) (0x40B004 + (i) * 0x20) +#define ADF_GEN4_VINTSOU_OFFSET(i) (0x40B000 + (i) * 0x20) + +struct adf_gen4_vfmig { + struct adf_mstate_mgr *mstate_mgr; + bool bank_stopped[ADF_GEN4_NUM_BANKS_PER_VF]; +}; + +void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev); + +enum icp_qat_gen4_slice_mask { + ICP_ACCEL_GEN4_MASK_CIPHER_SLICE = BIT(0), + ICP_ACCEL_GEN4_MASK_AUTH_SLICE = BIT(1), + ICP_ACCEL_GEN4_MASK_PKE_SLICE = BIT(2), + ICP_ACCEL_GEN4_MASK_COMPRESS_SLICE = BIT(3), + ICP_ACCEL_GEN4_MASK_UCS_SLICE = BIT(4), + ICP_ACCEL_GEN4_MASK_EIA3_SLICE = BIT(5), + ICP_ACCEL_GEN4_MASK_SMX_SLICE = BIT(7), + ICP_ACCEL_GEN4_MASK_WCP_WAT_SLICE = BIT(8), + ICP_ACCEL_GEN4_MASK_ZUC_256_SLICE = BIT(9), +}; + +enum adf_gen4_rp_groups { + RP_GROUP_0, + RP_GROUP_1, + RP_GROUP_COUNT +}; + +void adf_gen4_enable_error_correction(struct adf_accel_dev *accel_dev); +void adf_gen4_enable_ints(struct adf_accel_dev *accel_dev); +u32 adf_gen4_get_accel_mask(struct adf_hw_device_data *self); +void adf_gen4_get_admin_info(struct admin_info *admin_csrs_info); +void adf_gen4_get_arb_info(struct arb_info *arb_info); +u32 adf_gen4_get_etr_bar_id(struct adf_hw_device_data *self); +u32 adf_gen4_get_heartbeat_clock(struct adf_hw_device_data *self); +u32 adf_gen4_get_misc_bar_id(struct adf_hw_device_data *self); +u32 adf_gen4_get_num_accels(struct adf_hw_device_data *self); +u32 adf_gen4_get_num_aes(struct adf_hw_device_data *self); +enum dev_sku_info adf_gen4_get_sku(struct adf_hw_device_data *self); +u32 adf_gen4_get_sram_bar_id(struct adf_hw_device_data *self); +int adf_gen4_init_device(struct adf_accel_dev *accel_dev); +int adf_gen4_ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number); +void adf_gen4_set_msix_default_rttable(struct adf_accel_dev *accel_dev); +void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev); +int adf_gen4_init_thd2arb_map(struct adf_accel_dev *accel_dev); +u16 adf_gen4_get_ring_to_svc_map(struct adf_accel_dev *accel_dev); +int adf_gen4_bank_quiesce_coal_timer(struct adf_accel_dev *accel_dev, + u32 bank_idx, int timeout_ms); +int adf_gen4_bank_drain_start(struct adf_accel_dev *accel_dev, + u32 bank_number, int timeout_us); +void adf_gen4_bank_drain_finish(struct adf_accel_dev *accel_dev, + u32 bank_number); +bool adf_gen4_services_supported(unsigned long service_mask); +void adf_gen4_init_dc_ops(struct adf_dc_ops *dc_ops); +void adf_gen4_init_num_svc_aes(struct adf_rl_hw_data *device_data); +u32 adf_gen4_get_svc_slice_cnt(struct adf_accel_dev *accel_dev, + enum adf_base_services svc); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.c new file mode 100644 index 000000000..21474d402 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.c @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2021 Intel Corporation */ +#include +#include +#include +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_gen4_pfvf.h" +#include "adf_gen4_hw_data.h" +#include "adf_pfvf_pf_proto.h" +#include "adf_pfvf_utils.h" + +/* VF2PF interrupt source registers */ +#define ADF_4XXX_VM2PF_SOU 0x41A180 +#define ADF_4XXX_VM2PF_MSK 0x41A1C0 +#define ADF_GEN4_VF_MSK 0xFFFF + +#define ADF_PFVF_GEN4_MSGTYPE_SHIFT 2 +#define ADF_PFVF_GEN4_MSGTYPE_MASK 0x3F +#define ADF_PFVF_GEN4_MSGDATA_SHIFT 8 +#define ADF_PFVF_GEN4_MSGDATA_MASK 0xFFFFFF + +static const struct pfvf_csr_format csr_gen4_fmt = { + { ADF_PFVF_GEN4_MSGTYPE_SHIFT, ADF_PFVF_GEN4_MSGTYPE_MASK }, + { ADF_PFVF_GEN4_MSGDATA_SHIFT, ADF_PFVF_GEN4_MSGDATA_MASK }, +}; + +static u32 adf_gen4_pf_get_pf2vf_offset(u32 i) +{ + return ADF_GEN4_PF2VM_OFFSET(i); +} + +static u32 adf_gen4_pf_get_vf2pf_offset(u32 i) +{ + return ADF_GEN4_VM2PF_OFFSET(i); +} + +static void adf_gen4_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) +{ + u32 val; + + val = ADF_CSR_RD(pmisc_addr, ADF_4XXX_VM2PF_MSK) & ~vf_mask; + ADF_CSR_WR(pmisc_addr, ADF_4XXX_VM2PF_MSK, val); +} + +static void adf_gen4_disable_all_vf2pf_interrupts(void __iomem *pmisc_addr) +{ + ADF_CSR_WR(pmisc_addr, ADF_4XXX_VM2PF_MSK, ADF_GEN4_VF_MSK); +} + +static u32 adf_gen4_disable_pending_vf2pf_interrupts(void __iomem *pmisc_addr) +{ + u32 sources, disabled, pending; + + /* Get the interrupt sources triggered by VFs */ + sources = ADF_CSR_RD(pmisc_addr, ADF_4XXX_VM2PF_SOU); + if (!sources) + return 0; + + /* Get the already disabled interrupts */ + disabled = ADF_CSR_RD(pmisc_addr, ADF_4XXX_VM2PF_MSK); + + pending = sources & ~disabled; + if (!pending) + return 0; + + /* Due to HW limitations, when disabling the interrupts, we can't + * just disable the requested sources, as this would lead to missed + * interrupts if VM2PF_SOU changes just before writing to VM2PF_MSK. + * To work around it, disable all and re-enable only the sources that + * are not in vf_mask and were not already disabled. Re-enabling will + * trigger a new interrupt for the sources that have changed in the + * meantime, if any. + */ + ADF_CSR_WR(pmisc_addr, ADF_4XXX_VM2PF_MSK, ADF_GEN4_VF_MSK); + ADF_CSR_WR(pmisc_addr, ADF_4XXX_VM2PF_MSK, disabled | sources); + + /* Return the sources of the (new) interrupt(s) */ + return pending; +} + +static int adf_gen4_pfvf_send(struct adf_accel_dev *accel_dev, + struct pfvf_message msg, u32 pfvf_offset, + struct mutex *csr_lock) +{ + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + u32 csr_val; + int ret; + + csr_val = adf_pfvf_csr_msg_of(accel_dev, msg, &csr_gen4_fmt); + if (unlikely(!csr_val)) + return -EINVAL; + + mutex_lock(csr_lock); + + ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val | ADF_PFVF_INT); + + /* Wait for confirmation from remote that it received the message */ + ret = read_poll_timeout(ADF_CSR_RD, csr_val, !(csr_val & ADF_PFVF_INT), + ADF_PFVF_MSG_ACK_DELAY_US, + ADF_PFVF_MSG_ACK_MAX_DELAY_US, + true, pmisc_addr, pfvf_offset); + if (ret < 0) + dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n"); + + mutex_unlock(csr_lock); + return ret; +} + +static struct pfvf_message adf_gen4_pfvf_recv(struct adf_accel_dev *accel_dev, + u32 pfvf_offset, u8 compat_ver) +{ + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + struct pfvf_message msg = { 0 }; + u32 csr_val; + + /* Read message from the CSR */ + csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset); + if (!(csr_val & ADF_PFVF_INT)) { + dev_info(&GET_DEV(accel_dev), + "Spurious PFVF interrupt, msg 0x%.8x. Ignored\n", csr_val); + return msg; + } + + /* We can now acknowledge the message reception by clearing the + * interrupt bit + */ + ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val & ~ADF_PFVF_INT); + + /* Return the pfvf_message format */ + return adf_pfvf_message_of(accel_dev, csr_val, &csr_gen4_fmt); +} + +void adf_gen4_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) +{ + pfvf_ops->enable_comms = adf_enable_pf2vf_comms; + pfvf_ops->get_pf2vf_offset = adf_gen4_pf_get_pf2vf_offset; + pfvf_ops->get_vf2pf_offset = adf_gen4_pf_get_vf2pf_offset; + pfvf_ops->enable_vf2pf_interrupts = adf_gen4_enable_vf2pf_interrupts; + pfvf_ops->disable_all_vf2pf_interrupts = adf_gen4_disable_all_vf2pf_interrupts; + pfvf_ops->disable_pending_vf2pf_interrupts = adf_gen4_disable_pending_vf2pf_interrupts; + pfvf_ops->send_msg = adf_gen4_pfvf_send; + pfvf_ops->recv_msg = adf_gen4_pfvf_recv; +} +EXPORT_SYMBOL_GPL(adf_gen4_init_pf_pfvf_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.h new file mode 100644 index 000000000..2c8708117 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2021 Intel Corporation */ +#ifndef ADF_GEN4_PFVF_H +#define ADF_GEN4_PFVF_H + +#include "adf_accel_devices.h" +#include "adf_common_drv.h" + +#ifdef CONFIG_PCI_IOV +void adf_gen4_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops); +#else +static inline void adf_gen4_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) +{ + pfvf_ops->enable_comms = adf_pfvf_comms_disabled; +} +#endif + +#endif /* ADF_GEN4_PFVF_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_pm.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_pm.c new file mode 100644 index 000000000..efeb3d06b --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_pm.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2022 Intel Corporation */ +#include +#include +#include + +#include "adf_accel_devices.h" +#include "adf_admin.h" +#include "adf_common_drv.h" +#include "adf_gen4_pm.h" +#include "adf_cfg_strings.h" +#include "icp_qat_fw_init_admin.h" +#include "adf_gen4_hw_data.h" +#include "adf_cfg.h" + +struct adf_gen4_pm_data { + struct work_struct pm_irq_work; + struct adf_accel_dev *accel_dev; + u32 pm_int_sts; +}; + +static int send_host_msg(struct adf_accel_dev *accel_dev) +{ + char pm_idle_support_cfg[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {}; + void __iomem *pmisc = adf_get_pmisc_base(accel_dev); + struct adf_pm *pm = &accel_dev->power_management; + bool pm_idle_support; + u32 msg; + int ret; + + msg = ADF_CSR_RD(pmisc, ADF_GEN4_PM_HOST_MSG); + if (msg & ADF_GEN4_PM_MSG_PENDING) + return -EBUSY; + + adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, + ADF_PM_IDLE_SUPPORT, pm_idle_support_cfg); + ret = kstrtobool(pm_idle_support_cfg, &pm_idle_support); + if (ret) + pm_idle_support = true; + + if (pm_idle_support) + pm->host_ack_counter++; + else + pm->host_nack_counter++; + + /* Send HOST_MSG */ + msg = FIELD_PREP(ADF_GEN4_PM_MSG_PAYLOAD_BIT_MASK, + pm_idle_support ? PM_SET_MIN : PM_NO_CHANGE); + msg |= ADF_GEN4_PM_MSG_PENDING; + ADF_CSR_WR(pmisc, ADF_GEN4_PM_HOST_MSG, msg); + + /* Poll status register to make sure the HOST_MSG has been processed */ + return read_poll_timeout(ADF_CSR_RD, msg, + !(msg & ADF_GEN4_PM_MSG_PENDING), + ADF_GEN4_PM_MSG_POLL_DELAY_US, + ADF_GEN4_PM_POLL_TIMEOUT_US, true, pmisc, + ADF_GEN4_PM_HOST_MSG); +} + +static void pm_bh_handler(struct work_struct *work) +{ + struct adf_gen4_pm_data *pm_data = + container_of(work, struct adf_gen4_pm_data, pm_irq_work); + struct adf_accel_dev *accel_dev = pm_data->accel_dev; + void __iomem *pmisc = adf_get_pmisc_base(accel_dev); + struct adf_pm *pm = &accel_dev->power_management; + u32 pm_int_sts = pm_data->pm_int_sts; + u32 val; + + /* PM Idle interrupt */ + if (pm_int_sts & ADF_GEN4_PM_IDLE_STS) { + pm->idle_irq_counters++; + /* Issue host message to FW */ + if (send_host_msg(accel_dev)) + dev_warn_ratelimited(&GET_DEV(accel_dev), + "Failed to send host msg to FW\n"); + } + + /* PM throttle interrupt */ + if (pm_int_sts & ADF_GEN4_PM_THR_STS) + pm->throttle_irq_counters++; + + /* PM fw interrupt */ + if (pm_int_sts & ADF_GEN4_PM_FW_INT_STS) + pm->fw_irq_counters++; + + /* Clear interrupt status */ + ADF_CSR_WR(pmisc, ADF_GEN4_PM_INTERRUPT, pm_int_sts); + + /* Reenable PM interrupt */ + val = ADF_CSR_RD(pmisc, ADF_GEN4_ERRMSK2); + val &= ~ADF_GEN4_PM_SOU; + ADF_CSR_WR(pmisc, ADF_GEN4_ERRMSK2, val); + + kfree(pm_data); +} + +bool adf_gen4_handle_pm_interrupt(struct adf_accel_dev *accel_dev) +{ + void __iomem *pmisc = adf_get_pmisc_base(accel_dev); + struct adf_gen4_pm_data *pm_data = NULL; + u32 errsou2; + u32 errmsk2; + u32 val; + + /* Only handle the interrupt triggered by PM */ + errmsk2 = ADF_CSR_RD(pmisc, ADF_GEN4_ERRMSK2); + if (errmsk2 & ADF_GEN4_PM_SOU) + return false; + + errsou2 = ADF_CSR_RD(pmisc, ADF_GEN4_ERRSOU2); + if (!(errsou2 & ADF_GEN4_PM_SOU)) + return false; + + /* Disable interrupt */ + val = ADF_CSR_RD(pmisc, ADF_GEN4_ERRMSK2); + val |= ADF_GEN4_PM_SOU; + ADF_CSR_WR(pmisc, ADF_GEN4_ERRMSK2, val); + + val = ADF_CSR_RD(pmisc, ADF_GEN4_PM_INTERRUPT); + + pm_data = kzalloc_obj(*pm_data, GFP_ATOMIC); + if (!pm_data) + return false; + + pm_data->pm_int_sts = val; + pm_data->accel_dev = accel_dev; + + INIT_WORK(&pm_data->pm_irq_work, pm_bh_handler); + adf_misc_wq_queue_work(&pm_data->pm_irq_work); + + return true; +} +EXPORT_SYMBOL_GPL(adf_gen4_handle_pm_interrupt); + +int adf_gen4_enable_pm(struct adf_accel_dev *accel_dev) +{ + void __iomem *pmisc = adf_get_pmisc_base(accel_dev); + int ret; + u32 val; + + ret = adf_init_admin_pm(accel_dev, ADF_GEN4_PM_DEFAULT_IDLE_FILTER); + if (ret) + return ret; + + /* Initialize PM internal data */ + adf_gen4_init_dev_pm_data(accel_dev); + + /* Enable default PM interrupts: IDLE, THROTTLE */ + val = ADF_CSR_RD(pmisc, ADF_GEN4_PM_INTERRUPT); + val |= ADF_GEN4_PM_INT_EN_DEFAULT; + + /* Clear interrupt status */ + val |= ADF_GEN4_PM_INT_STS_MASK; + ADF_CSR_WR(pmisc, ADF_GEN4_PM_INTERRUPT, val); + + /* Unmask PM Interrupt */ + val = ADF_CSR_RD(pmisc, ADF_GEN4_ERRMSK2); + val &= ~ADF_GEN4_PM_SOU; + ADF_CSR_WR(pmisc, ADF_GEN4_ERRMSK2, val); + + return 0; +} +EXPORT_SYMBOL_GPL(adf_gen4_enable_pm); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_pm.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_pm.h new file mode 100644 index 000000000..a49352b79 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_pm.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2022 Intel Corporation */ +#ifndef ADF_GEN4_PM_H +#define ADF_GEN4_PM_H + +#include + +struct adf_accel_dev; + +enum qat_pm_host_msg { + PM_NO_CHANGE = 0, + PM_SET_MIN, +}; + +/* Power management registers */ +#define ADF_GEN4_PM_HOST_MSG (0x50A01C) + +/* Power management */ +#define ADF_GEN4_PM_POLL_DELAY_US 20 +#define ADF_GEN4_PM_POLL_TIMEOUT_US USEC_PER_SEC +#define ADF_GEN4_PM_MSG_POLL_DELAY_US (10 * USEC_PER_MSEC) +#define ADF_GEN4_PM_STATUS (0x50A00C) +#define ADF_GEN4_PM_INTERRUPT (0x50A028) + +/* Power management source in ERRSOU2 and ERRMSK2 */ +#define ADF_GEN4_PM_SOU BIT(18) + +#define ADF_GEN4_PM_IDLE_INT_EN BIT(18) +#define ADF_GEN4_PM_THROTTLE_INT_EN BIT(19) +#define ADF_GEN4_PM_DRV_ACTIVE BIT(20) +#define ADF_GEN4_PM_INIT_STATE BIT(21) +#define ADF_GEN4_PM_INT_EN_DEFAULT (ADF_GEN4_PM_IDLE_INT_EN | \ + ADF_GEN4_PM_THROTTLE_INT_EN) + +#define ADF_GEN4_PM_THR_STS BIT(0) +#define ADF_GEN4_PM_IDLE_STS BIT(1) +#define ADF_GEN4_PM_FW_INT_STS BIT(2) +#define ADF_GEN4_PM_INT_STS_MASK (ADF_GEN4_PM_THR_STS | \ + ADF_GEN4_PM_IDLE_STS | \ + ADF_GEN4_PM_FW_INT_STS) + +#define ADF_GEN4_PM_MSG_PENDING BIT(0) +#define ADF_GEN4_PM_MSG_PAYLOAD_BIT_MASK GENMASK(28, 1) + +#define ADF_GEN4_PM_DEFAULT_IDLE_FILTER (0x6) +#define ADF_GEN4_PM_MAX_IDLE_FILTER (0x7) +#define ADF_GEN4_PM_DEFAULT_IDLE_SUPPORT (0x1) + +/* PM CSRs fields masks */ +#define ADF_GEN4_PM_DOMAIN_POWER_GATED_MASK GENMASK(15, 0) +#define ADF_GEN4_PM_SSM_PM_ENABLE_MASK GENMASK(15, 0) +#define ADF_GEN4_PM_IDLE_FILTER_MASK GENMASK(5, 3) +#define ADF_GEN4_PM_IDLE_ENABLE_MASK BIT(2) +#define ADF_GEN4_PM_ENABLE_PM_MASK BIT(21) +#define ADF_GEN4_PM_ENABLE_PM_IDLE_MASK BIT(22) +#define ADF_GEN4_PM_ENABLE_DEEP_PM_IDLE_MASK BIT(23) +#define ADF_GEN4_PM_CURRENT_WP_MASK GENMASK(19, 11) +#define ADF_GEN4_PM_CPM_PM_STATE_MASK GENMASK(22, 20) +#define ADF_GEN4_PM_PENDING_WP_MASK GENMASK(31, 23) +#define ADF_GEN4_PM_THR_VALUE_MASK GENMASK(6, 4) +#define ADF_GEN4_PM_MIN_PWR_ACK_MASK BIT(7) +#define ADF_GEN4_PM_MIN_PWR_ACK_PENDING_MASK BIT(17) +#define ADF_GEN4_PM_CPR_ACTIVE_COUNT_MASK BIT(0) +#define ADF_GEN4_PM_CPR_MANAGED_COUNT_MASK BIT(0) +#define ADF_GEN4_PM_XLT_ACTIVE_COUNT_MASK BIT(1) +#define ADF_GEN4_PM_XLT_MANAGED_COUNT_MASK BIT(1) +#define ADF_GEN4_PM_DCPR_ACTIVE_COUNT_MASK GENMASK(3, 2) +#define ADF_GEN4_PM_DCPR_MANAGED_COUNT_MASK GENMASK(3, 2) +#define ADF_GEN4_PM_PKE_ACTIVE_COUNT_MASK GENMASK(8, 4) +#define ADF_GEN4_PM_PKE_MANAGED_COUNT_MASK GENMASK(8, 4) +#define ADF_GEN4_PM_WAT_ACTIVE_COUNT_MASK GENMASK(13, 9) +#define ADF_GEN4_PM_WAT_MANAGED_COUNT_MASK GENMASK(13, 9) +#define ADF_GEN4_PM_WCP_ACTIVE_COUNT_MASK GENMASK(18, 14) +#define ADF_GEN4_PM_WCP_MANAGED_COUNT_MASK GENMASK(18, 14) +#define ADF_GEN4_PM_UCS_ACTIVE_COUNT_MASK GENMASK(20, 19) +#define ADF_GEN4_PM_UCS_MANAGED_COUNT_MASK GENMASK(20, 19) +#define ADF_GEN4_PM_CPH_ACTIVE_COUNT_MASK GENMASK(24, 21) +#define ADF_GEN4_PM_CPH_MANAGED_COUNT_MASK GENMASK(24, 21) +#define ADF_GEN4_PM_ATH_ACTIVE_COUNT_MASK GENMASK(28, 25) +#define ADF_GEN4_PM_ATH_MANAGED_COUNT_MASK GENMASK(28, 25) + +int adf_gen4_enable_pm(struct adf_accel_dev *accel_dev); +bool adf_gen4_handle_pm_interrupt(struct adf_accel_dev *accel_dev); + +#ifdef CONFIG_DEBUG_FS +void adf_gen4_init_dev_pm_data(struct adf_accel_dev *accel_dev); +#else +static inline void adf_gen4_init_dev_pm_data(struct adf_accel_dev *accel_dev) +{ +} +#endif /* CONFIG_DEBUG_FS */ + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_pm_debugfs.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_pm_debugfs.c new file mode 100644 index 000000000..b7e38842a --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_pm_debugfs.c @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ +#include +#include + +#include "adf_accel_devices.h" +#include "adf_admin.h" +#include "adf_common_drv.h" +#include "adf_gen4_pm.h" +#include "adf_pm_dbgfs_utils.h" +#include "icp_qat_fw_init_admin.h" + +#define PM_INFO_REGSET_ENTRY(_reg_, _field_) \ + PM_INFO_REGSET_ENTRY_MASK(_reg_, _field_, ADF_GEN4_PM_##_field_##_MASK) + +static const struct pm_status_row pm_fuse_rows[] = { + PM_INFO_REGSET_ENTRY(fusectl0, ENABLE_PM), + PM_INFO_REGSET_ENTRY(fusectl0, ENABLE_PM_IDLE), + PM_INFO_REGSET_ENTRY(fusectl0, ENABLE_DEEP_PM_IDLE), +}; + +static const struct pm_status_row pm_info_rows[] = { + PM_INFO_REGSET_ENTRY(pm.status, CPM_PM_STATE), + PM_INFO_REGSET_ENTRY(pm.status, PENDING_WP), + PM_INFO_REGSET_ENTRY(pm.status, CURRENT_WP), + PM_INFO_REGSET_ENTRY(pm.fw_init, IDLE_ENABLE), + PM_INFO_REGSET_ENTRY(pm.fw_init, IDLE_FILTER), + PM_INFO_REGSET_ENTRY(pm.main, MIN_PWR_ACK), + PM_INFO_REGSET_ENTRY(pm.thread, MIN_PWR_ACK_PENDING), + PM_INFO_REGSET_ENTRY(pm.main, THR_VALUE), +}; + +static const struct pm_status_row pm_ssm_rows[] = { + PM_INFO_REGSET_ENTRY(ssm.pm_enable, SSM_PM_ENABLE), + PM_INFO_REGSET_ENTRY32(ssm.active_constraint, ACTIVE_CONSTRAINT), + PM_INFO_REGSET_ENTRY(ssm.pm_domain_status, DOMAIN_POWER_GATED), + PM_INFO_REGSET_ENTRY(ssm.pm_active_status, ATH_ACTIVE_COUNT), + PM_INFO_REGSET_ENTRY(ssm.pm_active_status, CPH_ACTIVE_COUNT), + PM_INFO_REGSET_ENTRY(ssm.pm_active_status, PKE_ACTIVE_COUNT), + PM_INFO_REGSET_ENTRY(ssm.pm_active_status, CPR_ACTIVE_COUNT), + PM_INFO_REGSET_ENTRY(ssm.pm_active_status, DCPR_ACTIVE_COUNT), + PM_INFO_REGSET_ENTRY(ssm.pm_active_status, UCS_ACTIVE_COUNT), + PM_INFO_REGSET_ENTRY(ssm.pm_active_status, XLT_ACTIVE_COUNT), + PM_INFO_REGSET_ENTRY(ssm.pm_active_status, WAT_ACTIVE_COUNT), + PM_INFO_REGSET_ENTRY(ssm.pm_active_status, WCP_ACTIVE_COUNT), + PM_INFO_REGSET_ENTRY(ssm.pm_managed_status, ATH_MANAGED_COUNT), + PM_INFO_REGSET_ENTRY(ssm.pm_managed_status, CPH_MANAGED_COUNT), + PM_INFO_REGSET_ENTRY(ssm.pm_managed_status, PKE_MANAGED_COUNT), + PM_INFO_REGSET_ENTRY(ssm.pm_managed_status, CPR_MANAGED_COUNT), + PM_INFO_REGSET_ENTRY(ssm.pm_managed_status, DCPR_MANAGED_COUNT), + PM_INFO_REGSET_ENTRY(ssm.pm_managed_status, UCS_MANAGED_COUNT), + PM_INFO_REGSET_ENTRY(ssm.pm_managed_status, XLT_MANAGED_COUNT), + PM_INFO_REGSET_ENTRY(ssm.pm_managed_status, WAT_MANAGED_COUNT), + PM_INFO_REGSET_ENTRY(ssm.pm_managed_status, WCP_MANAGED_COUNT), +}; + +static const struct pm_status_row pm_log_rows[] = { + PM_INFO_REGSET_ENTRY32(event_counters.host_msg, HOST_MSG_EVENT_COUNT), + PM_INFO_REGSET_ENTRY32(event_counters.sys_pm, SYS_PM_EVENT_COUNT), + PM_INFO_REGSET_ENTRY32(event_counters.local_ssm, SSM_EVENT_COUNT), + PM_INFO_REGSET_ENTRY32(event_counters.timer, TIMER_EVENT_COUNT), + PM_INFO_REGSET_ENTRY32(event_counters.unknown, UNKNOWN_EVENT_COUNT), +}; + +static const struct pm_status_row pm_event_rows[ICP_QAT_NUMBER_OF_PM_EVENTS] = { + PM_INFO_REGSET_ENTRY32(event_log[0], EVENT0), + PM_INFO_REGSET_ENTRY32(event_log[1], EVENT1), + PM_INFO_REGSET_ENTRY32(event_log[2], EVENT2), + PM_INFO_REGSET_ENTRY32(event_log[3], EVENT3), + PM_INFO_REGSET_ENTRY32(event_log[4], EVENT4), + PM_INFO_REGSET_ENTRY32(event_log[5], EVENT5), + PM_INFO_REGSET_ENTRY32(event_log[6], EVENT6), + PM_INFO_REGSET_ENTRY32(event_log[7], EVENT7), +}; + +static const struct pm_status_row pm_csrs_rows[] = { + PM_INFO_REGSET_ENTRY32(pm.fw_init, CPM_PM_FW_INIT), + PM_INFO_REGSET_ENTRY32(pm.status, CPM_PM_STATUS), + PM_INFO_REGSET_ENTRY32(pm.main, CPM_PM_MASTER_FW), + PM_INFO_REGSET_ENTRY32(pm.pwrreq, CPM_PM_PWRREQ), +}; + +static_assert(sizeof(struct icp_qat_fw_init_admin_pm_info) < PAGE_SIZE); + +static ssize_t adf_gen4_print_pm_status(struct adf_accel_dev *accel_dev, + char __user *buf, size_t count, + loff_t *pos) +{ + void __iomem *pmisc = adf_get_pmisc_base(accel_dev); + struct adf_pm *pm = &accel_dev->power_management; + struct icp_qat_fw_init_admin_pm_info *pm_info; + dma_addr_t p_state_addr; + u32 *pm_info_regs; + char *pm_kv; + int len = 0; + u32 val; + int ret; + + pm_info = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!pm_info) + return -ENOMEM; + + pm_kv = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!pm_kv) { + ret = -ENOMEM; + goto out_free; + } + + p_state_addr = dma_map_single(&GET_DEV(accel_dev), pm_info, PAGE_SIZE, + DMA_FROM_DEVICE); + ret = dma_mapping_error(&GET_DEV(accel_dev), p_state_addr); + if (ret) + goto out_free; + + /* Query PM info from QAT FW */ + ret = adf_get_pm_info(accel_dev, p_state_addr, PAGE_SIZE); + dma_unmap_single(&GET_DEV(accel_dev), p_state_addr, PAGE_SIZE, + DMA_FROM_DEVICE); + if (ret) + goto out_free; + + pm_info_regs = (u32 *)pm_info; + + /* Fusectl related */ + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, + "----------- PM Fuse info ---------\n"); + len += adf_pm_scnprint_table_lower_keys(&pm_kv[len], pm_fuse_rows, + pm_info_regs, PAGE_SIZE - len, + ARRAY_SIZE(pm_fuse_rows)); + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "max_pwrreq: %#x\n", + pm_info->max_pwrreq); + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "min_pwrreq: %#x\n", + pm_info->min_pwrreq); + + /* PM related */ + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, + "------------ PM Info ------------\n"); + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "power_level: %s\n", + pm_info->pwr_state == PM_SET_MIN ? "min" : "max"); + len += adf_pm_scnprint_table_lower_keys(&pm_kv[len], pm_info_rows, + pm_info_regs, PAGE_SIZE - len, + ARRAY_SIZE(pm_info_rows)); + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "pm_mode: STATIC\n"); + + /* SSM related */ + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, + "----------- SSM_PM Info ----------\n"); + len += adf_pm_scnprint_table_lower_keys(&pm_kv[len], pm_ssm_rows, + pm_info_regs, PAGE_SIZE - len, + ARRAY_SIZE(pm_ssm_rows)); + + /* Log related */ + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, + "------------- PM Log -------------\n"); + len += adf_pm_scnprint_table_lower_keys(&pm_kv[len], pm_log_rows, + pm_info_regs, PAGE_SIZE - len, + ARRAY_SIZE(pm_log_rows)); + + len += adf_pm_scnprint_table_lower_keys(&pm_kv[len], pm_event_rows, + pm_info_regs, PAGE_SIZE - len, + ARRAY_SIZE(pm_event_rows)); + + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "idle_irq_count: %#x\n", + pm->idle_irq_counters); + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "fw_irq_count: %#x\n", + pm->fw_irq_counters); + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, + "throttle_irq_count: %#x\n", pm->throttle_irq_counters); + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "host_ack_count: %#x\n", + pm->host_ack_counter); + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "host_nack_count: %#x\n", + pm->host_nack_counter); + + /* CSRs content */ + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, + "----------- HW PM CSRs -----------\n"); + len += adf_pm_scnprint_table_upper_keys(&pm_kv[len], pm_csrs_rows, + pm_info_regs, PAGE_SIZE - len, + ARRAY_SIZE(pm_csrs_rows)); + + val = ADF_CSR_RD(pmisc, ADF_GEN4_PM_HOST_MSG); + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, + "CPM_PM_HOST_MSG: %#x\n", val); + val = ADF_CSR_RD(pmisc, ADF_GEN4_PM_INTERRUPT); + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, + "CPM_PM_INTERRUPT: %#x\n", val); + ret = simple_read_from_buffer(buf, count, pos, pm_kv, len); + +out_free: + kfree(pm_info); + kfree(pm_kv); + return ret; +} + +void adf_gen4_init_dev_pm_data(struct adf_accel_dev *accel_dev) +{ + accel_dev->power_management.print_pm_status = adf_gen4_print_pm_status; + accel_dev->power_management.present = true; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_ras.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_ras.c new file mode 100644 index 000000000..0f7f00a19 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_ras.c @@ -0,0 +1,1517 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ +#include "adf_common_drv.h" +#include "adf_gen4_hw_data.h" +#include "adf_gen4_ras.h" +#include "adf_sysfs_ras_counters.h" + +#define BITS_PER_REG(_n_) (sizeof(_n_) * BITS_PER_BYTE) + +static void enable_errsou_reporting(void __iomem *csr) +{ + /* Enable correctable error reporting in ERRSOU0 */ + ADF_CSR_WR(csr, ADF_GEN4_ERRMSK0, 0); + + /* Enable uncorrectable error reporting in ERRSOU1 */ + ADF_CSR_WR(csr, ADF_GEN4_ERRMSK1, 0); + + /* + * Enable uncorrectable error reporting in ERRSOU2 + * but disable PM interrupt and CFC attention interrupt by default + */ + ADF_CSR_WR(csr, ADF_GEN4_ERRMSK2, + ADF_GEN4_ERRSOU2_PM_INT_BIT | + ADF_GEN4_ERRSOU2_CPP_CFC_ATT_INT_BITMASK); + + /* + * Enable uncorrectable error reporting in ERRSOU3 + * but disable RLT error interrupt and VFLR notify interrupt by default + */ + ADF_CSR_WR(csr, ADF_GEN4_ERRMSK3, + ADF_GEN4_ERRSOU3_RLTERROR_BIT | + ADF_GEN4_ERRSOU3_VFLRNOTIFY_BIT); +} + +static void disable_errsou_reporting(void __iomem *csr) +{ + u32 val = 0; + + /* Disable correctable error reporting in ERRSOU0 */ + ADF_CSR_WR(csr, ADF_GEN4_ERRMSK0, ADF_GEN4_ERRSOU0_BIT); + + /* Disable uncorrectable error reporting in ERRSOU1 */ + ADF_CSR_WR(csr, ADF_GEN4_ERRMSK1, ADF_GEN4_ERRSOU1_BITMASK); + + /* Disable uncorrectable error reporting in ERRSOU2 */ + val = ADF_CSR_RD(csr, ADF_GEN4_ERRMSK2); + val |= ADF_GEN4_ERRSOU2_DIS_BITMASK; + ADF_CSR_WR(csr, ADF_GEN4_ERRMSK2, val); + + /* Disable uncorrectable error reporting in ERRSOU3 */ + ADF_CSR_WR(csr, ADF_GEN4_ERRMSK3, ADF_GEN4_ERRSOU3_BITMASK); +} + +static void enable_ae_error_reporting(struct adf_accel_dev *accel_dev, + void __iomem *csr) +{ + u32 ae_mask = GET_HW_DATA(accel_dev)->ae_mask; + + /* Enable Acceleration Engine correctable error reporting */ + ADF_CSR_WR(csr, ADF_GEN4_HIAECORERRLOGENABLE_CPP0, ae_mask); + + /* Enable Acceleration Engine uncorrectable error reporting */ + ADF_CSR_WR(csr, ADF_GEN4_HIAEUNCERRLOGENABLE_CPP0, ae_mask); +} + +static void disable_ae_error_reporting(void __iomem *csr) +{ + /* Disable Acceleration Engine correctable error reporting */ + ADF_CSR_WR(csr, ADF_GEN4_HIAECORERRLOGENABLE_CPP0, 0); + + /* Disable Acceleration Engine uncorrectable error reporting */ + ADF_CSR_WR(csr, ADF_GEN4_HIAEUNCERRLOGENABLE_CPP0, 0); +} + +static void enable_cpp_error_reporting(struct adf_accel_dev *accel_dev, + void __iomem *csr) +{ + struct adf_dev_err_mask *err_mask = GET_ERR_MASK(accel_dev); + + /* Enable HI CPP Agents Command Parity Error Reporting */ + ADF_CSR_WR(csr, ADF_GEN4_HICPPAGENTCMDPARERRLOGENABLE, + err_mask->cppagentcmdpar_mask); + + ADF_CSR_WR(csr, ADF_GEN4_CPP_CFC_ERR_CTRL, + ADF_GEN4_CPP_CFC_ERR_CTRL_BITMASK); +} + +static void disable_cpp_error_reporting(void __iomem *csr) +{ + /* Disable HI CPP Agents Command Parity Error Reporting */ + ADF_CSR_WR(csr, ADF_GEN4_HICPPAGENTCMDPARERRLOGENABLE, 0); + + ADF_CSR_WR(csr, ADF_GEN4_CPP_CFC_ERR_CTRL, + ADF_GEN4_CPP_CFC_ERR_CTRL_DIS_BITMASK); +} + +static void enable_ti_ri_error_reporting(void __iomem *csr) +{ + u32 reg; + + /* Enable RI Memory error reporting */ + ADF_CSR_WR(csr, ADF_GEN4_RI_MEM_PAR_ERR_EN0, + ADF_GEN4_RIMEM_PARERR_STS_FATAL_BITMASK | + ADF_GEN4_RIMEM_PARERR_STS_UNCERR_BITMASK); + + /* Enable IOSF Primary Command Parity error Reporting */ + ADF_CSR_WR(csr, ADF_GEN4_RIMISCCTL, ADF_GEN4_RIMISCSTS_BIT); + + /* Enable TI Internal Memory Parity Error reporting */ + ADF_CSR_WR(csr, ADF_GEN4_TI_CI_PAR_ERR_MASK, 0); + ADF_CSR_WR(csr, ADF_GEN4_TI_PULL0FUB_PAR_ERR_MASK, 0); + ADF_CSR_WR(csr, ADF_GEN4_TI_PUSHFUB_PAR_ERR_MASK, 0); + ADF_CSR_WR(csr, ADF_GEN4_TI_CD_PAR_ERR_MASK, 0); + ADF_CSR_WR(csr, ADF_GEN4_TI_TRNSB_PAR_ERR_MASK, 0); + + /* Enable error handling in RI, TI CPP interface control registers */ + ADF_CSR_WR(csr, ADF_GEN4_RICPPINTCTL, ADF_GEN4_RICPPINTCTL_BITMASK); + + ADF_CSR_WR(csr, ADF_GEN4_TICPPINTCTL, ADF_GEN4_TICPPINTCTL_BITMASK); + + /* + * Enable error detection and reporting in TIMISCSTS + * with bits 1, 2 and 30 value preserved + */ + reg = ADF_CSR_RD(csr, ADF_GEN4_TIMISCCTL); + reg &= ADF_GEN4_TIMSCCTL_RELAY_BITMASK; + reg |= ADF_GEN4_TIMISCCTL_BIT; + ADF_CSR_WR(csr, ADF_GEN4_TIMISCCTL, reg); +} + +static void disable_ti_ri_error_reporting(void __iomem *csr) +{ + u32 reg; + + /* Disable RI Memory error reporting */ + ADF_CSR_WR(csr, ADF_GEN4_RI_MEM_PAR_ERR_EN0, 0); + + /* Disable IOSF Primary Command Parity error Reporting */ + ADF_CSR_WR(csr, ADF_GEN4_RIMISCCTL, 0); + + /* Disable TI Internal Memory Parity Error reporting */ + ADF_CSR_WR(csr, ADF_GEN4_TI_CI_PAR_ERR_MASK, + ADF_GEN4_TI_CI_PAR_STS_BITMASK); + ADF_CSR_WR(csr, ADF_GEN4_TI_PULL0FUB_PAR_ERR_MASK, + ADF_GEN4_TI_PULL0FUB_PAR_STS_BITMASK); + ADF_CSR_WR(csr, ADF_GEN4_TI_PUSHFUB_PAR_ERR_MASK, + ADF_GEN4_TI_PUSHFUB_PAR_STS_BITMASK); + ADF_CSR_WR(csr, ADF_GEN4_TI_CD_PAR_ERR_MASK, + ADF_GEN4_TI_CD_PAR_STS_BITMASK); + ADF_CSR_WR(csr, ADF_GEN4_TI_TRNSB_PAR_ERR_MASK, + ADF_GEN4_TI_TRNSB_PAR_STS_BITMASK); + + /* Disable error handling in RI, TI CPP interface control registers */ + ADF_CSR_WR(csr, ADF_GEN4_RICPPINTCTL, 0); + + ADF_CSR_WR(csr, ADF_GEN4_TICPPINTCTL, 0); + + /* + * Disable error detection and reporting in TIMISCSTS + * with bits 1, 2 and 30 value preserved + */ + reg = ADF_CSR_RD(csr, ADF_GEN4_TIMISCCTL); + reg &= ADF_GEN4_TIMSCCTL_RELAY_BITMASK; + ADF_CSR_WR(csr, ADF_GEN4_TIMISCCTL, reg); +} + +static void enable_rf_error_reporting(struct adf_accel_dev *accel_dev, + void __iomem *csr) +{ + struct adf_dev_err_mask *err_mask = GET_ERR_MASK(accel_dev); + + /* Enable RF parity error in Shared RAM */ + ADF_CSR_WR(csr, ADF_GEN4_SSMSOFTERRORPARITYMASK_SRC, 0); + ADF_CSR_WR(csr, ADF_GEN4_SSMSOFTERRORPARITYMASK_ATH_CPH, 0); + ADF_CSR_WR(csr, ADF_GEN4_SSMSOFTERRORPARITYMASK_CPR_XLT, 0); + ADF_CSR_WR(csr, ADF_GEN4_SSMSOFTERRORPARITYMASK_DCPR_UCS, 0); + ADF_CSR_WR(csr, ADF_GEN4_SSMSOFTERRORPARITYMASK_PKE, 0); + + if (err_mask->parerr_wat_wcp_mask) + ADF_CSR_WR(csr, ADF_GEN4_SSMSOFTERRORPARITYMASK_WAT_WCP, 0); +} + +static void disable_rf_error_reporting(struct adf_accel_dev *accel_dev, + void __iomem *csr) +{ + struct adf_dev_err_mask *err_mask = GET_ERR_MASK(accel_dev); + + /* Disable RF Parity Error reporting in Shared RAM */ + ADF_CSR_WR(csr, ADF_GEN4_SSMSOFTERRORPARITYMASK_SRC, + ADF_GEN4_SSMSOFTERRORPARITY_SRC_BIT); + + ADF_CSR_WR(csr, ADF_GEN4_SSMSOFTERRORPARITYMASK_ATH_CPH, + err_mask->parerr_ath_cph_mask); + + ADF_CSR_WR(csr, ADF_GEN4_SSMSOFTERRORPARITYMASK_CPR_XLT, + err_mask->parerr_cpr_xlt_mask); + + ADF_CSR_WR(csr, ADF_GEN4_SSMSOFTERRORPARITYMASK_DCPR_UCS, + err_mask->parerr_dcpr_ucs_mask); + + ADF_CSR_WR(csr, ADF_GEN4_SSMSOFTERRORPARITYMASK_PKE, + err_mask->parerr_pke_mask); + + if (err_mask->parerr_wat_wcp_mask) + ADF_CSR_WR(csr, ADF_GEN4_SSMSOFTERRORPARITYMASK_WAT_WCP, + err_mask->parerr_wat_wcp_mask); +} + +static void enable_ssm_error_reporting(struct adf_accel_dev *accel_dev, + void __iomem *csr) +{ + struct adf_dev_err_mask *err_mask = GET_ERR_MASK(accel_dev); + u32 val = 0; + + /* Enable SSM interrupts */ + ADF_CSR_WR(csr, ADF_GEN4_INTMASKSSM, 0); + + /* Enable shared memory error detection & correction */ + val = ADF_CSR_RD(csr, ADF_GEN4_SSMFEATREN); + val |= err_mask->ssmfeatren_mask; + ADF_CSR_WR(csr, ADF_GEN4_SSMFEATREN, val); + + /* Enable SER detection in SER_err_ssmsh register */ + ADF_CSR_WR(csr, ADF_GEN4_SER_EN_SSMSH, + ADF_GEN4_SER_EN_SSMSH_BITMASK); + + /* Enable SSM soft parity error */ + ADF_CSR_WR(csr, ADF_GEN4_SPPPARERRMSK_ATH_CPH, 0); + ADF_CSR_WR(csr, ADF_GEN4_SPPPARERRMSK_CPR_XLT, 0); + ADF_CSR_WR(csr, ADF_GEN4_SPPPARERRMSK_DCPR_UCS, 0); + ADF_CSR_WR(csr, ADF_GEN4_SPPPARERRMSK_PKE, 0); + + if (err_mask->parerr_wat_wcp_mask) + ADF_CSR_WR(csr, ADF_GEN4_SPPPARERRMSK_WAT_WCP, 0); + + /* Enable slice hang interrupt reporting */ + ADF_CSR_WR(csr, ADF_GEN4_SHINTMASKSSM_ATH_CPH, 0); + ADF_CSR_WR(csr, ADF_GEN4_SHINTMASKSSM_CPR_XLT, 0); + ADF_CSR_WR(csr, ADF_GEN4_SHINTMASKSSM_DCPR_UCS, 0); + ADF_CSR_WR(csr, ADF_GEN4_SHINTMASKSSM_PKE, 0); + + if (err_mask->parerr_wat_wcp_mask) + ADF_CSR_WR(csr, ADF_GEN4_SHINTMASKSSM_WAT_WCP, 0); +} + +static void disable_ssm_error_reporting(struct adf_accel_dev *accel_dev, + void __iomem *csr) +{ + struct adf_dev_err_mask *err_mask = GET_ERR_MASK(accel_dev); + u32 val = 0; + + /* Disable SSM interrupts */ + ADF_CSR_WR(csr, ADF_GEN4_INTMASKSSM, + ADF_GEN4_INTMASKSSM_BITMASK); + + /* Disable shared memory error detection & correction */ + val = ADF_CSR_RD(csr, ADF_GEN4_SSMFEATREN); + val &= ADF_GEN4_SSMFEATREN_DIS_BITMASK; + ADF_CSR_WR(csr, ADF_GEN4_SSMFEATREN, val); + + /* Disable SER detection in SER_err_ssmsh register */ + ADF_CSR_WR(csr, ADF_GEN4_SER_EN_SSMSH, 0); + + /* Disable SSM soft parity error */ + ADF_CSR_WR(csr, ADF_GEN4_SPPPARERRMSK_ATH_CPH, + err_mask->parerr_ath_cph_mask); + + ADF_CSR_WR(csr, ADF_GEN4_SPPPARERRMSK_CPR_XLT, + err_mask->parerr_cpr_xlt_mask); + + ADF_CSR_WR(csr, ADF_GEN4_SPPPARERRMSK_DCPR_UCS, + err_mask->parerr_dcpr_ucs_mask); + + ADF_CSR_WR(csr, ADF_GEN4_SPPPARERRMSK_PKE, + err_mask->parerr_pke_mask); + + if (err_mask->parerr_wat_wcp_mask) + ADF_CSR_WR(csr, ADF_GEN4_SPPPARERRMSK_WAT_WCP, + err_mask->parerr_wat_wcp_mask); + + /* Disable slice hang interrupt reporting */ + ADF_CSR_WR(csr, ADF_GEN4_SHINTMASKSSM_ATH_CPH, + err_mask->parerr_ath_cph_mask); + + ADF_CSR_WR(csr, ADF_GEN4_SHINTMASKSSM_CPR_XLT, + err_mask->parerr_cpr_xlt_mask); + + ADF_CSR_WR(csr, ADF_GEN4_SHINTMASKSSM_DCPR_UCS, + err_mask->parerr_dcpr_ucs_mask); + + ADF_CSR_WR(csr, ADF_GEN4_SHINTMASKSSM_PKE, + err_mask->parerr_pke_mask); + + if (err_mask->parerr_wat_wcp_mask) + ADF_CSR_WR(csr, ADF_GEN4_SHINTMASKSSM_WAT_WCP, + err_mask->parerr_wat_wcp_mask); +} + +static void enable_aram_error_reporting(void __iomem *csr) +{ + ADF_CSR_WR(csr, ADF_GEN4_REG_ARAMCERRUERR_EN, + ADF_GEN4_REG_ARAMCERRUERR_EN_BITMASK); + + ADF_CSR_WR(csr, ADF_GEN4_REG_ARAMCERR, + ADF_GEN4_REG_ARAMCERR_EN_BITMASK); + + ADF_CSR_WR(csr, ADF_GEN4_REG_ARAMUERR, + ADF_GEN4_REG_ARAMUERR_EN_BITMASK); + + ADF_CSR_WR(csr, ADF_GEN4_REG_CPPMEMTGTERR, + ADF_GEN4_REG_CPPMEMTGTERR_EN_BITMASK); +} + +static void disable_aram_error_reporting(void __iomem *csr) +{ + ADF_CSR_WR(csr, ADF_GEN4_REG_ARAMCERRUERR_EN, 0); + ADF_CSR_WR(csr, ADF_GEN4_REG_ARAMCERR, 0); + ADF_CSR_WR(csr, ADF_GEN4_REG_ARAMUERR, 0); + ADF_CSR_WR(csr, ADF_GEN4_REG_CPPMEMTGTERR, 0); +} + +static void adf_gen4_enable_ras(struct adf_accel_dev *accel_dev) +{ + void __iomem *aram_csr = adf_get_aram_base(accel_dev); + void __iomem *csr = adf_get_pmisc_base(accel_dev); + + enable_errsou_reporting(csr); + enable_ae_error_reporting(accel_dev, csr); + enable_cpp_error_reporting(accel_dev, csr); + enable_ti_ri_error_reporting(csr); + enable_rf_error_reporting(accel_dev, csr); + enable_ssm_error_reporting(accel_dev, csr); + enable_aram_error_reporting(aram_csr); +} + +static void adf_gen4_disable_ras(struct adf_accel_dev *accel_dev) +{ + void __iomem *aram_csr = adf_get_aram_base(accel_dev); + void __iomem *csr = adf_get_pmisc_base(accel_dev); + + disable_errsou_reporting(csr); + disable_ae_error_reporting(csr); + disable_cpp_error_reporting(csr); + disable_ti_ri_error_reporting(csr); + disable_rf_error_reporting(accel_dev, csr); + disable_ssm_error_reporting(accel_dev, csr); + disable_aram_error_reporting(aram_csr); +} + +static void adf_gen4_process_errsou0(struct adf_accel_dev *accel_dev, + void __iomem *csr) +{ + u32 aecorrerr = ADF_CSR_RD(csr, ADF_GEN4_HIAECORERRLOG_CPP0); + + aecorrerr &= GET_HW_DATA(accel_dev)->ae_mask; + + dev_warn(&GET_DEV(accel_dev), + "Correctable error detected in AE: 0x%x\n", + aecorrerr); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_CORR); + + /* Clear interrupt from ERRSOU0 */ + ADF_CSR_WR(csr, ADF_GEN4_HIAECORERRLOG_CPP0, aecorrerr); +} + +static bool adf_handle_cpp_aeunc(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 errsou) +{ + u32 aeuncorerr; + + if (!(errsou & ADF_GEN4_ERRSOU1_HIAEUNCERRLOG_CPP0_BIT)) + return false; + + aeuncorerr = ADF_CSR_RD(csr, ADF_GEN4_HIAEUNCERRLOG_CPP0); + aeuncorerr &= GET_HW_DATA(accel_dev)->ae_mask; + + dev_err(&GET_DEV(accel_dev), + "Uncorrectable error detected in AE: 0x%x\n", + aeuncorerr); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + + ADF_CSR_WR(csr, ADF_GEN4_HIAEUNCERRLOG_CPP0, aeuncorerr); + + return false; +} + +static bool adf_handle_cppcmdparerr(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 errsou) +{ + struct adf_dev_err_mask *err_mask = GET_ERR_MASK(accel_dev); + u32 cmdparerr; + + if (!(errsou & ADF_GEN4_ERRSOU1_HICPPAGENTCMDPARERRLOG_BIT)) + return false; + + cmdparerr = ADF_CSR_RD(csr, ADF_GEN4_HICPPAGENTCMDPARERRLOG); + cmdparerr &= err_mask->cppagentcmdpar_mask; + + dev_err(&GET_DEV(accel_dev), + "HI CPP agent command parity error: 0x%x\n", + cmdparerr); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + + ADF_CSR_WR(csr, ADF_GEN4_HICPPAGENTCMDPARERRLOG, cmdparerr); + + return true; +} + +static bool adf_handle_ri_mem_par_err(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 errsou) +{ + bool reset_required = false; + u32 rimem_parerr_sts; + + if (!(errsou & ADF_GEN4_ERRSOU1_RIMEM_PARERR_STS_BIT)) + return false; + + rimem_parerr_sts = ADF_CSR_RD(csr, ADF_GEN4_RIMEM_PARERR_STS); + rimem_parerr_sts &= ADF_GEN4_RIMEM_PARERR_STS_UNCERR_BITMASK | + ADF_GEN4_RIMEM_PARERR_STS_FATAL_BITMASK; + + if (rimem_parerr_sts & ADF_GEN4_RIMEM_PARERR_STS_UNCERR_BITMASK) { + dev_err(&GET_DEV(accel_dev), + "RI Memory Parity uncorrectable error: 0x%x\n", + rimem_parerr_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + } + + if (rimem_parerr_sts & ADF_GEN4_RIMEM_PARERR_STS_FATAL_BITMASK) { + dev_err(&GET_DEV(accel_dev), + "RI Memory Parity fatal error: 0x%x\n", + rimem_parerr_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + reset_required = true; + } + + ADF_CSR_WR(csr, ADF_GEN4_RIMEM_PARERR_STS, rimem_parerr_sts); + + return reset_required; +} + +static bool adf_handle_ti_ci_par_sts(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 errsou) +{ + u32 ti_ci_par_sts; + + if (!(errsou & ADF_GEN4_ERRSOU1_TIMEM_PARERR_STS_BIT)) + return false; + + ti_ci_par_sts = ADF_CSR_RD(csr, ADF_GEN4_TI_CI_PAR_STS); + ti_ci_par_sts &= ADF_GEN4_TI_CI_PAR_STS_BITMASK; + + if (ti_ci_par_sts) { + dev_err(&GET_DEV(accel_dev), + "TI Memory Parity Error: 0x%x\n", ti_ci_par_sts); + ADF_CSR_WR(csr, ADF_GEN4_TI_CI_PAR_STS, ti_ci_par_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + } + + return false; +} + +static bool adf_handle_ti_pullfub_par_sts(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 errsou) +{ + u32 ti_pullfub_par_sts; + + if (!(errsou & ADF_GEN4_ERRSOU1_TIMEM_PARERR_STS_BIT)) + return false; + + ti_pullfub_par_sts = ADF_CSR_RD(csr, ADF_GEN4_TI_PULL0FUB_PAR_STS); + ti_pullfub_par_sts &= ADF_GEN4_TI_PULL0FUB_PAR_STS_BITMASK; + + if (ti_pullfub_par_sts) { + dev_err(&GET_DEV(accel_dev), + "TI Pull Parity Error: 0x%x\n", ti_pullfub_par_sts); + + ADF_CSR_WR(csr, ADF_GEN4_TI_PULL0FUB_PAR_STS, + ti_pullfub_par_sts); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + } + + return false; +} + +static bool adf_handle_ti_pushfub_par_sts(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 errsou) +{ + u32 ti_pushfub_par_sts; + + if (!(errsou & ADF_GEN4_ERRSOU1_TIMEM_PARERR_STS_BIT)) + return false; + + ti_pushfub_par_sts = ADF_CSR_RD(csr, ADF_GEN4_TI_PUSHFUB_PAR_STS); + ti_pushfub_par_sts &= ADF_GEN4_TI_PUSHFUB_PAR_STS_BITMASK; + + if (ti_pushfub_par_sts) { + dev_err(&GET_DEV(accel_dev), + "TI Push Parity Error: 0x%x\n", ti_pushfub_par_sts); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + + ADF_CSR_WR(csr, ADF_GEN4_TI_PUSHFUB_PAR_STS, + ti_pushfub_par_sts); + } + + return false; +} + +static bool adf_handle_ti_cd_par_sts(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 errsou) +{ + u32 ti_cd_par_sts; + + if (!(errsou & ADF_GEN4_ERRSOU1_TIMEM_PARERR_STS_BIT)) + return false; + + ti_cd_par_sts = ADF_CSR_RD(csr, ADF_GEN4_TI_CD_PAR_STS); + ti_cd_par_sts &= ADF_GEN4_TI_CD_PAR_STS_BITMASK; + + if (ti_cd_par_sts) { + dev_err(&GET_DEV(accel_dev), + "TI CD Parity Error: 0x%x\n", ti_cd_par_sts); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + + ADF_CSR_WR(csr, ADF_GEN4_TI_CD_PAR_STS, ti_cd_par_sts); + } + + return false; +} + +static bool adf_handle_ti_trnsb_par_sts(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 errsou) +{ + u32 ti_trnsb_par_sts; + + if (!(errsou & ADF_GEN4_ERRSOU1_TIMEM_PARERR_STS_BIT)) + return false; + + ti_trnsb_par_sts = ADF_CSR_RD(csr, ADF_GEN4_TI_TRNSB_PAR_STS); + ti_trnsb_par_sts &= ADF_GEN4_TI_TRNSB_PAR_STS_BITMASK; + + if (ti_trnsb_par_sts) { + dev_err(&GET_DEV(accel_dev), + "TI TRNSB Parity Error: 0x%x\n", ti_trnsb_par_sts); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + + ADF_CSR_WR(csr, ADF_GEN4_TI_TRNSB_PAR_STS, ti_trnsb_par_sts); + } + + return false; +} + +static bool adf_handle_iosfp_cmd_parerr(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 errsou) +{ + u32 rimiscsts; + + if (!(errsou & ADF_GEN4_ERRSOU1_TIMEM_PARERR_STS_BIT)) + return false; + + rimiscsts = ADF_CSR_RD(csr, ADF_GEN4_RIMISCSTS); + rimiscsts &= ADF_GEN4_RIMISCSTS_BIT; + + dev_err(&GET_DEV(accel_dev), + "Command Parity error detected on IOSFP: 0x%x\n", + rimiscsts); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + + ADF_CSR_WR(csr, ADF_GEN4_RIMISCSTS, rimiscsts); + + return true; +} + +static void adf_gen4_process_errsou1(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 errsou, + bool *reset_required) +{ + *reset_required |= adf_handle_cpp_aeunc(accel_dev, csr, errsou); + *reset_required |= adf_handle_cppcmdparerr(accel_dev, csr, errsou); + *reset_required |= adf_handle_ri_mem_par_err(accel_dev, csr, errsou); + *reset_required |= adf_handle_ti_ci_par_sts(accel_dev, csr, errsou); + *reset_required |= adf_handle_ti_pullfub_par_sts(accel_dev, csr, errsou); + *reset_required |= adf_handle_ti_pushfub_par_sts(accel_dev, csr, errsou); + *reset_required |= adf_handle_ti_cd_par_sts(accel_dev, csr, errsou); + *reset_required |= adf_handle_ti_trnsb_par_sts(accel_dev, csr, errsou); + *reset_required |= adf_handle_iosfp_cmd_parerr(accel_dev, csr, errsou); +} + +static bool adf_handle_uerrssmsh(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN4_IAINTSTATSSM_UERRSSMSH_BIT)) + return false; + + reg = ADF_CSR_RD(csr, ADF_GEN4_UERRSSMSH); + reg &= ADF_GEN4_UERRSSMSH_BITMASK; + + dev_err(&GET_DEV(accel_dev), + "Uncorrectable error on ssm shared memory: 0x%x\n", + reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + + ADF_CSR_WR(csr, ADF_GEN4_UERRSSMSH, reg); + + return false; +} + +static bool adf_handle_cerrssmsh(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN4_IAINTSTATSSM_CERRSSMSH_BIT)) + return false; + + reg = ADF_CSR_RD(csr, ADF_GEN4_CERRSSMSH); + reg &= ADF_GEN4_CERRSSMSH_ERROR_BIT; + + dev_warn(&GET_DEV(accel_dev), + "Correctable error on ssm shared memory: 0x%x\n", + reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_CORR); + + ADF_CSR_WR(csr, ADF_GEN4_CERRSSMSH, reg); + + return false; +} + +static bool adf_handle_pperr_err(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN4_IAINTSTATSSM_PPERR_BIT)) + return false; + + reg = ADF_CSR_RD(csr, ADF_GEN4_PPERR); + reg &= ADF_GEN4_PPERR_BITMASK; + + dev_err(&GET_DEV(accel_dev), + "Uncorrectable error CPP transaction on memory target: 0x%x\n", + reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + + ADF_CSR_WR(csr, ADF_GEN4_PPERR, reg); + + return false; +} + +static void adf_poll_slicehang_csr(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 slice_hang_offset, + char *slice_name) +{ + u32 slice_hang_reg = ADF_CSR_RD(csr, slice_hang_offset); + + if (!slice_hang_reg) + return; + + dev_err(&GET_DEV(accel_dev), + "Slice %s hang error encountered\n", slice_name); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); +} + +static bool adf_handle_slice_hang_error(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 iastatssm) +{ + struct adf_dev_err_mask *err_mask = GET_ERR_MASK(accel_dev); + + if (!(iastatssm & ADF_GEN4_IAINTSTATSSM_SLICEHANG_ERR_BIT)) + return false; + + adf_poll_slicehang_csr(accel_dev, csr, + ADF_GEN4_SLICEHANGSTATUS_ATH_CPH, "ath_cph"); + adf_poll_slicehang_csr(accel_dev, csr, + ADF_GEN4_SLICEHANGSTATUS_CPR_XLT, "cpr_xlt"); + adf_poll_slicehang_csr(accel_dev, csr, + ADF_GEN4_SLICEHANGSTATUS_DCPR_UCS, "dcpr_ucs"); + adf_poll_slicehang_csr(accel_dev, csr, + ADF_GEN4_SLICEHANGSTATUS_PKE, "pke"); + + if (err_mask->parerr_wat_wcp_mask) + adf_poll_slicehang_csr(accel_dev, csr, + ADF_GEN4_SLICEHANGSTATUS_WAT_WCP, + "wat_wcp"); + + return false; +} + +static bool adf_handle_spp_pullcmd_err(struct adf_accel_dev *accel_dev, + void __iomem *csr) +{ + struct adf_dev_err_mask *err_mask = GET_ERR_MASK(accel_dev); + bool reset_required = false; + u32 reg; + + reg = ADF_CSR_RD(csr, ADF_GEN4_SPPPULLCMDPARERR_ATH_CPH); + reg &= err_mask->parerr_ath_cph_mask; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "SPP pull command fatal error ATH_CPH: 0x%x\n", reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + + ADF_CSR_WR(csr, ADF_GEN4_SPPPULLCMDPARERR_ATH_CPH, reg); + + reset_required = true; + } + + reg = ADF_CSR_RD(csr, ADF_GEN4_SPPPULLCMDPARERR_CPR_XLT); + reg &= err_mask->parerr_cpr_xlt_mask; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "SPP pull command fatal error CPR_XLT: 0x%x\n", reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + + ADF_CSR_WR(csr, ADF_GEN4_SPPPULLCMDPARERR_CPR_XLT, reg); + + reset_required = true; + } + + reg = ADF_CSR_RD(csr, ADF_GEN4_SPPPULLCMDPARERR_DCPR_UCS); + reg &= err_mask->parerr_dcpr_ucs_mask; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "SPP pull command fatal error DCPR_UCS: 0x%x\n", reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + + ADF_CSR_WR(csr, ADF_GEN4_SPPPULLCMDPARERR_DCPR_UCS, reg); + + reset_required = true; + } + + reg = ADF_CSR_RD(csr, ADF_GEN4_SPPPULLCMDPARERR_PKE); + reg &= err_mask->parerr_pke_mask; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "SPP pull command fatal error PKE: 0x%x\n", reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + + ADF_CSR_WR(csr, ADF_GEN4_SPPPULLCMDPARERR_PKE, reg); + + reset_required = true; + } + + if (err_mask->parerr_wat_wcp_mask) { + reg = ADF_CSR_RD(csr, ADF_GEN4_SPPPULLCMDPARERR_WAT_WCP); + reg &= err_mask->parerr_wat_wcp_mask; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "SPP pull command fatal error WAT_WCP: 0x%x\n", reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + + ADF_CSR_WR(csr, ADF_GEN4_SPPPULLCMDPARERR_WAT_WCP, reg); + + reset_required = true; + } + } + + return reset_required; +} + +static bool adf_handle_spp_pulldata_err(struct adf_accel_dev *accel_dev, + void __iomem *csr) +{ + struct adf_dev_err_mask *err_mask = GET_ERR_MASK(accel_dev); + u32 reg; + + reg = ADF_CSR_RD(csr, ADF_GEN4_SPPPULLDATAPARERR_ATH_CPH); + reg &= err_mask->parerr_ath_cph_mask; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "SPP pull data err ATH_CPH: 0x%x\n", reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + + ADF_CSR_WR(csr, ADF_GEN4_SPPPULLDATAPARERR_ATH_CPH, reg); + } + + reg = ADF_CSR_RD(csr, ADF_GEN4_SPPPULLDATAPARERR_CPR_XLT); + reg &= err_mask->parerr_cpr_xlt_mask; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "SPP pull data err CPR_XLT: 0x%x\n", reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + + ADF_CSR_WR(csr, ADF_GEN4_SPPPULLDATAPARERR_CPR_XLT, reg); + } + + reg = ADF_CSR_RD(csr, ADF_GEN4_SPPPULLDATAPARERR_DCPR_UCS); + reg &= err_mask->parerr_dcpr_ucs_mask; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "SPP pull data err DCPR_UCS: 0x%x\n", reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + + ADF_CSR_WR(csr, ADF_GEN4_SPPPULLDATAPARERR_DCPR_UCS, reg); + } + + reg = ADF_CSR_RD(csr, ADF_GEN4_SPPPULLDATAPARERR_PKE); + reg &= err_mask->parerr_pke_mask; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "SPP pull data err PKE: 0x%x\n", reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + + ADF_CSR_WR(csr, ADF_GEN4_SPPPULLDATAPARERR_PKE, reg); + } + + if (err_mask->parerr_wat_wcp_mask) { + reg = ADF_CSR_RD(csr, ADF_GEN4_SPPPULLDATAPARERR_WAT_WCP); + reg &= err_mask->parerr_wat_wcp_mask; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "SPP pull data err WAT_WCP: 0x%x\n", reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + + ADF_CSR_WR(csr, ADF_GEN4_SPPPULLDATAPARERR_WAT_WCP, reg); + } + } + + return false; +} + +static bool adf_handle_spp_pushcmd_err(struct adf_accel_dev *accel_dev, + void __iomem *csr) +{ + struct adf_dev_err_mask *err_mask = GET_ERR_MASK(accel_dev); + bool reset_required = false; + u32 reg; + + reg = ADF_CSR_RD(csr, ADF_GEN4_SPPPUSHCMDPARERR_ATH_CPH); + reg &= err_mask->parerr_ath_cph_mask; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "SPP push command fatal error ATH_CPH: 0x%x\n", reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + + ADF_CSR_WR(csr, ADF_GEN4_SPPPUSHCMDPARERR_ATH_CPH, reg); + + reset_required = true; + } + + reg = ADF_CSR_RD(csr, ADF_GEN4_SPPPUSHCMDPARERR_CPR_XLT); + reg &= err_mask->parerr_cpr_xlt_mask; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "SPP push command fatal error CPR_XLT: 0x%x\n", reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + + ADF_CSR_WR(csr, ADF_GEN4_SPPPUSHCMDPARERR_CPR_XLT, reg); + + reset_required = true; + } + + reg = ADF_CSR_RD(csr, ADF_GEN4_SPPPUSHCMDPARERR_DCPR_UCS); + reg &= err_mask->parerr_dcpr_ucs_mask; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "SPP push command fatal error DCPR_UCS: 0x%x\n", reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + + ADF_CSR_WR(csr, ADF_GEN4_SPPPUSHCMDPARERR_DCPR_UCS, reg); + + reset_required = true; + } + + reg = ADF_CSR_RD(csr, ADF_GEN4_SPPPUSHCMDPARERR_PKE); + reg &= err_mask->parerr_pke_mask; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "SPP push command fatal error PKE: 0x%x\n", + reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + + ADF_CSR_WR(csr, ADF_GEN4_SPPPUSHCMDPARERR_PKE, reg); + + reset_required = true; + } + + if (err_mask->parerr_wat_wcp_mask) { + reg = ADF_CSR_RD(csr, ADF_GEN4_SPPPUSHCMDPARERR_WAT_WCP); + reg &= err_mask->parerr_wat_wcp_mask; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "SPP push command fatal error WAT_WCP: 0x%x\n", reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + + ADF_CSR_WR(csr, ADF_GEN4_SPPPUSHCMDPARERR_WAT_WCP, reg); + + reset_required = true; + } + } + + return reset_required; +} + +static bool adf_handle_spp_pushdata_err(struct adf_accel_dev *accel_dev, + void __iomem *csr) +{ + struct adf_dev_err_mask *err_mask = GET_ERR_MASK(accel_dev); + u32 reg; + + reg = ADF_CSR_RD(csr, ADF_GEN4_SPPPUSHDATAPARERR_ATH_CPH); + reg &= err_mask->parerr_ath_cph_mask; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "SPP push data err ATH_CPH: 0x%x\n", reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + + ADF_CSR_WR(csr, ADF_GEN4_SPPPUSHDATAPARERR_ATH_CPH, reg); + } + + reg = ADF_CSR_RD(csr, ADF_GEN4_SPPPUSHDATAPARERR_CPR_XLT); + reg &= err_mask->parerr_cpr_xlt_mask; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "SPP push data err CPR_XLT: 0x%x\n", reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + + ADF_CSR_WR(csr, ADF_GEN4_SPPPUSHDATAPARERR_CPR_XLT, reg); + } + + reg = ADF_CSR_RD(csr, ADF_GEN4_SPPPUSHDATAPARERR_DCPR_UCS); + reg &= err_mask->parerr_dcpr_ucs_mask; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "SPP push data err DCPR_UCS: 0x%x\n", reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + + ADF_CSR_WR(csr, ADF_GEN4_SPPPUSHDATAPARERR_DCPR_UCS, reg); + } + + reg = ADF_CSR_RD(csr, ADF_GEN4_SPPPUSHDATAPARERR_PKE); + reg &= err_mask->parerr_pke_mask; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "SPP push data err PKE: 0x%x\n", reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + + ADF_CSR_WR(csr, ADF_GEN4_SPPPUSHDATAPARERR_PKE, reg); + } + + if (err_mask->parerr_wat_wcp_mask) { + reg = ADF_CSR_RD(csr, ADF_GEN4_SPPPUSHDATAPARERR_WAT_WCP); + reg &= err_mask->parerr_wat_wcp_mask; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "SPP push data err WAT_WCP: 0x%x\n", reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + + ADF_CSR_WR(csr, ADF_GEN4_SPPPUSHDATAPARERR_WAT_WCP, + reg); + } + } + + return false; +} + +static bool adf_handle_spppar_err(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 iastatssm) +{ + bool reset_required; + + if (!(iastatssm & ADF_GEN4_IAINTSTATSSM_SPPPARERR_BIT)) + return false; + + reset_required = adf_handle_spp_pullcmd_err(accel_dev, csr); + reset_required |= adf_handle_spp_pulldata_err(accel_dev, csr); + reset_required |= adf_handle_spp_pushcmd_err(accel_dev, csr); + reset_required |= adf_handle_spp_pushdata_err(accel_dev, csr); + + return reset_required; +} + +static bool adf_handle_ssmcpppar_err(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 iastatssm) +{ + u32 reg, bits_num = BITS_PER_REG(reg); + bool reset_required = false; + unsigned long errs_bits; + u32 bit_iterator; + + if (!(iastatssm & ADF_GEN4_IAINTSTATSSM_SSMCPPERR_BIT)) + return false; + + reg = ADF_CSR_RD(csr, ADF_GEN4_SSMCPPERR); + reg &= ADF_GEN4_SSMCPPERR_FATAL_BITMASK | ADF_GEN4_SSMCPPERR_UNCERR_BITMASK; + if (reg & ADF_GEN4_SSMCPPERR_FATAL_BITMASK) { + dev_err(&GET_DEV(accel_dev), + "Fatal SSM CPP parity error: 0x%x\n", reg); + + errs_bits = reg & ADF_GEN4_SSMCPPERR_FATAL_BITMASK; + for_each_set_bit(bit_iterator, &errs_bits, bits_num) { + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + } + reset_required = true; + } + + if (reg & ADF_GEN4_SSMCPPERR_UNCERR_BITMASK) { + dev_err(&GET_DEV(accel_dev), + "non-Fatal SSM CPP parity error: 0x%x\n", reg); + errs_bits = reg & ADF_GEN4_SSMCPPERR_UNCERR_BITMASK; + + for_each_set_bit(bit_iterator, &errs_bits, bits_num) { + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + } + } + + ADF_CSR_WR(csr, ADF_GEN4_SSMCPPERR, reg); + + return reset_required; +} + +static void adf_handle_rf_parr_err(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 iastatssm) +{ + if (!(iastatssm & ADF_GEN4_IAINTSTATSSM_SSMSOFTERRORPARITY_BIT)) + return; + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + dev_err(&GET_DEV(accel_dev), "Slice ssm soft parity error reported"); + + return; +} + +static bool adf_handle_ser_err_ssmsh(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 iastatssm) +{ + u32 reg, bits_num = BITS_PER_REG(reg); + bool reset_required = false; + unsigned long errs_bits; + u32 bit_iterator; + + if (!(iastatssm & (ADF_GEN4_IAINTSTATSSM_SER_ERR_SSMSH_CERR_BIT | + ADF_GEN4_IAINTSTATSSM_SER_ERR_SSMSH_UNCERR_BIT))) + return false; + + reg = ADF_CSR_RD(csr, ADF_GEN4_SER_ERR_SSMSH); + reg &= ADF_GEN4_SER_ERR_SSMSH_FATAL_BITMASK | + ADF_GEN4_SER_ERR_SSMSH_UNCERR_BITMASK | + ADF_GEN4_SER_ERR_SSMSH_CERR_BITMASK; + if (reg & ADF_GEN4_SER_ERR_SSMSH_FATAL_BITMASK) { + dev_err(&GET_DEV(accel_dev), + "Fatal SER_SSMSH_ERR: 0x%x\n", reg); + + errs_bits = reg & ADF_GEN4_SER_ERR_SSMSH_FATAL_BITMASK; + for_each_set_bit(bit_iterator, &errs_bits, bits_num) { + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + } + + reset_required = true; + } + + if (reg & ADF_GEN4_SER_ERR_SSMSH_UNCERR_BITMASK) { + dev_err(&GET_DEV(accel_dev), + "non-fatal SER_SSMSH_ERR: 0x%x\n", reg); + + errs_bits = reg & ADF_GEN4_SER_ERR_SSMSH_UNCERR_BITMASK; + for_each_set_bit(bit_iterator, &errs_bits, bits_num) { + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + } + } + + if (reg & ADF_GEN4_SER_ERR_SSMSH_CERR_BITMASK) { + dev_warn(&GET_DEV(accel_dev), + "Correctable SER_SSMSH_ERR: 0x%x\n", reg); + + errs_bits = reg & ADF_GEN4_SER_ERR_SSMSH_CERR_BITMASK; + for_each_set_bit(bit_iterator, &errs_bits, bits_num) { + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_CORR); + } + } + + ADF_CSR_WR(csr, ADF_GEN4_SER_ERR_SSMSH, reg); + + return reset_required; +} + +static bool adf_handle_iaintstatssm(struct adf_accel_dev *accel_dev, + void __iomem *csr) +{ + u32 iastatssm = ADF_CSR_RD(csr, ADF_GEN4_IAINTSTATSSM); + bool reset_required; + + iastatssm &= ADF_GEN4_IAINTSTATSSM_BITMASK; + if (!iastatssm) + return false; + + reset_required = adf_handle_uerrssmsh(accel_dev, csr, iastatssm); + reset_required |= adf_handle_cerrssmsh(accel_dev, csr, iastatssm); + reset_required |= adf_handle_pperr_err(accel_dev, csr, iastatssm); + reset_required |= adf_handle_slice_hang_error(accel_dev, csr, iastatssm); + reset_required |= adf_handle_spppar_err(accel_dev, csr, iastatssm); + reset_required |= adf_handle_ssmcpppar_err(accel_dev, csr, iastatssm); + reset_required |= adf_handle_ser_err_ssmsh(accel_dev, csr, iastatssm); + adf_handle_rf_parr_err(accel_dev, csr, iastatssm); + + ADF_CSR_WR(csr, ADF_GEN4_IAINTSTATSSM, iastatssm); + + return reset_required; +} + +static bool adf_handle_exprpssmcmpr(struct adf_accel_dev *accel_dev, + void __iomem *csr) +{ + u32 reg = ADF_CSR_RD(csr, ADF_GEN4_EXPRPSSMCPR); + + reg &= ADF_GEN4_EXPRPSSMCPR_UNCERR_BITMASK; + if (!reg) + return false; + + dev_err(&GET_DEV(accel_dev), + "Uncorrectable error exception in SSM CMP: 0x%x", reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + + ADF_CSR_WR(csr, ADF_GEN4_EXPRPSSMCPR, reg); + + return false; +} + +static bool adf_handle_exprpssmxlt(struct adf_accel_dev *accel_dev, + void __iomem *csr) +{ + u32 reg = ADF_CSR_RD(csr, ADF_GEN4_EXPRPSSMXLT); + + reg &= ADF_GEN4_EXPRPSSMXLT_UNCERR_BITMASK | + ADF_GEN4_EXPRPSSMXLT_CERR_BIT; + if (!reg) + return false; + + if (reg & ADF_GEN4_EXPRPSSMXLT_UNCERR_BITMASK) { + dev_err(&GET_DEV(accel_dev), + "Uncorrectable error exception in SSM XLT: 0x%x", reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + } + + if (reg & ADF_GEN4_EXPRPSSMXLT_CERR_BIT) { + dev_warn(&GET_DEV(accel_dev), + "Correctable error exception in SSM XLT: 0x%x", reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_CORR); + } + + ADF_CSR_WR(csr, ADF_GEN4_EXPRPSSMXLT, reg); + + return false; +} + +static bool adf_handle_exprpssmdcpr(struct adf_accel_dev *accel_dev, + void __iomem *csr) +{ + u32 reg; + int i; + + for (i = 0; i < ADF_GEN4_DCPR_SLICES_NUM; i++) { + reg = ADF_CSR_RD(csr, ADF_GEN4_EXPRPSSMDCPR(i)); + reg &= ADF_GEN4_EXPRPSSMDCPR_UNCERR_BITMASK | + ADF_GEN4_EXPRPSSMDCPR_CERR_BITMASK; + if (!reg) + continue; + + if (reg & ADF_GEN4_EXPRPSSMDCPR_UNCERR_BITMASK) { + dev_err(&GET_DEV(accel_dev), + "Uncorrectable error exception in SSM DCMP: 0x%x", reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + } + + if (reg & ADF_GEN4_EXPRPSSMDCPR_CERR_BITMASK) { + dev_warn(&GET_DEV(accel_dev), + "Correctable error exception in SSM DCMP: 0x%x", reg); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_CORR); + } + + ADF_CSR_WR(csr, ADF_GEN4_EXPRPSSMDCPR(i), reg); + } + + return false; +} + +static bool adf_handle_ssm(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + bool reset_required; + + if (!(errsou & ADF_GEN4_ERRSOU2_SSM_ERR_BIT)) + return false; + + reset_required = adf_handle_iaintstatssm(accel_dev, csr); + reset_required |= adf_handle_exprpssmcmpr(accel_dev, csr); + reset_required |= adf_handle_exprpssmxlt(accel_dev, csr); + reset_required |= adf_handle_exprpssmdcpr(accel_dev, csr); + + return reset_required; +} + +static bool adf_handle_cpp_cfc_err(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 errsou) +{ + bool reset_required = false; + u32 reg; + + if (!(errsou & ADF_GEN4_ERRSOU2_CPP_CFC_ERR_STATUS_BIT)) + return false; + + reg = ADF_CSR_RD(csr, ADF_GEN4_CPP_CFC_ERR_STATUS); + if (reg & ADF_GEN4_CPP_CFC_ERR_STATUS_DATAPAR_BIT) { + dev_err(&GET_DEV(accel_dev), + "CPP_CFC_ERR: data parity: 0x%x", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + } + + if (reg & ADF_GEN4_CPP_CFC_ERR_STATUS_CMDPAR_BIT) { + dev_err(&GET_DEV(accel_dev), + "CPP_CFC_ERR: command parity: 0x%x", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + + reset_required = true; + } + + if (reg & ADF_GEN4_CPP_CFC_ERR_STATUS_MERR_BIT) { + dev_err(&GET_DEV(accel_dev), + "CPP_CFC_ERR: multiple errors: 0x%x", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + + reset_required = true; + } + + ADF_CSR_WR(csr, ADF_GEN4_CPP_CFC_ERR_STATUS_CLR, + ADF_GEN4_CPP_CFC_ERR_STATUS_CLR_BITMASK); + + return reset_required; +} + +static void adf_gen4_process_errsou2(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 errsou, + bool *reset_required) +{ + *reset_required |= adf_handle_ssm(accel_dev, csr, errsou); + *reset_required |= adf_handle_cpp_cfc_err(accel_dev, csr, errsou); +} + +static bool adf_handle_timiscsts(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 errsou) +{ + u32 timiscsts; + + if (!(errsou & ADF_GEN4_ERRSOU3_TIMISCSTS_BIT)) + return false; + + timiscsts = ADF_CSR_RD(csr, ADF_GEN4_TIMISCSTS); + + dev_err(&GET_DEV(accel_dev), + "Fatal error in Transmit Interface: 0x%x\n", timiscsts); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + + return true; +} + +static bool adf_handle_ricppintsts(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 errsou) +{ + u32 ricppintsts; + + if (!(errsou & ADF_GEN4_ERRSOU3_RICPPINTSTS_BITMASK)) + return false; + + ricppintsts = ADF_CSR_RD(csr, ADF_GEN4_RICPPINTSTS); + ricppintsts &= ADF_GEN4_RICPPINTSTS_BITMASK; + + dev_err(&GET_DEV(accel_dev), + "RI CPP Uncorrectable Error: 0x%x\n", ricppintsts); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + + ADF_CSR_WR(csr, ADF_GEN4_RICPPINTSTS, ricppintsts); + + return false; +} + +static bool adf_handle_ticppintsts(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 errsou) +{ + u32 ticppintsts; + + if (!(errsou & ADF_GEN4_ERRSOU3_TICPPINTSTS_BITMASK)) + return false; + + ticppintsts = ADF_CSR_RD(csr, ADF_GEN4_TICPPINTSTS); + ticppintsts &= ADF_GEN4_TICPPINTSTS_BITMASK; + + dev_err(&GET_DEV(accel_dev), + "TI CPP Uncorrectable Error: 0x%x\n", ticppintsts); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + + ADF_CSR_WR(csr, ADF_GEN4_TICPPINTSTS, ticppintsts); + + return false; +} + +static bool adf_handle_aramcerr(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 errsou) +{ + u32 aram_cerr; + + if (!(errsou & ADF_GEN4_ERRSOU3_REG_ARAMCERR_BIT)) + return false; + + aram_cerr = ADF_CSR_RD(csr, ADF_GEN4_REG_ARAMCERR); + aram_cerr &= ADF_GEN4_REG_ARAMCERR_BIT; + + dev_warn(&GET_DEV(accel_dev), + "ARAM correctable error : 0x%x\n", aram_cerr); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_CORR); + + aram_cerr |= ADF_GEN4_REG_ARAMCERR_EN_BITMASK; + + ADF_CSR_WR(csr, ADF_GEN4_REG_ARAMCERR, aram_cerr); + + return false; +} + +static bool adf_handle_aramuerr(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 errsou) +{ + bool reset_required = false; + u32 aramuerr; + + if (!(errsou & ADF_GEN4_ERRSOU3_REG_ARAMUERR_BIT)) + return false; + + aramuerr = ADF_CSR_RD(csr, ADF_GEN4_REG_ARAMUERR); + aramuerr &= ADF_GEN4_REG_ARAMUERR_ERROR_BIT | + ADF_GEN4_REG_ARAMUERR_MULTI_ERRORS_BIT; + + if (!aramuerr) + return false; + + if (aramuerr & ADF_GEN4_REG_ARAMUERR_MULTI_ERRORS_BIT) { + dev_err(&GET_DEV(accel_dev), + "ARAM multiple uncorrectable errors: 0x%x\n", aramuerr); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + + reset_required = true; + } else { + dev_err(&GET_DEV(accel_dev), + "ARAM uncorrectable error: 0x%x\n", aramuerr); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + } + + aramuerr |= ADF_GEN4_REG_ARAMUERR_EN_BITMASK; + + ADF_CSR_WR(csr, ADF_GEN4_REG_ARAMUERR, aramuerr); + + return reset_required; +} + +static bool adf_handle_reg_cppmemtgterr(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 errsou) +{ + bool reset_required = false; + u32 cppmemtgterr; + + if (!(errsou & ADF_GEN4_ERRSOU3_REG_ARAMUERR_BIT)) + return false; + + cppmemtgterr = ADF_CSR_RD(csr, ADF_GEN4_REG_CPPMEMTGTERR); + cppmemtgterr &= ADF_GEN4_REG_CPPMEMTGTERR_BITMASK | + ADF_GEN4_REG_CPPMEMTGTERR_MULTI_ERRORS_BIT; + if (!cppmemtgterr) + return false; + + if (cppmemtgterr & ADF_GEN4_REG_CPPMEMTGTERR_MULTI_ERRORS_BIT) { + dev_err(&GET_DEV(accel_dev), + "Misc memory target multiple uncorrectable errors: 0x%x\n", + cppmemtgterr); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + + reset_required = true; + } else { + dev_err(&GET_DEV(accel_dev), + "Misc memory target uncorrectable error: 0x%x\n", cppmemtgterr); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + } + + cppmemtgterr |= ADF_GEN4_REG_CPPMEMTGTERR_EN_BITMASK; + + ADF_CSR_WR(csr, ADF_GEN4_REG_CPPMEMTGTERR, cppmemtgterr); + + return reset_required; +} + +static bool adf_handle_atufaultstatus(struct adf_accel_dev *accel_dev, + void __iomem *csr, u32 errsou) +{ + u32 i; + u32 max_rp_num = GET_HW_DATA(accel_dev)->num_banks; + + if (!(errsou & ADF_GEN4_ERRSOU3_ATUFAULTSTATUS_BIT)) + return false; + + for (i = 0; i < max_rp_num; i++) { + u32 atufaultstatus = ADF_CSR_RD(csr, ADF_GEN4_ATUFAULTSTATUS(i)); + + atufaultstatus &= ADF_GEN4_ATUFAULTSTATUS_BIT; + + if (atufaultstatus) { + dev_err(&GET_DEV(accel_dev), + "Ring Pair (%u) ATU detected fault: 0x%x\n", i, + atufaultstatus); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + + ADF_CSR_WR(csr, ADF_GEN4_ATUFAULTSTATUS(i), atufaultstatus); + } + } + + return false; +} + +static void adf_gen4_process_errsou3(struct adf_accel_dev *accel_dev, + void __iomem *csr, void __iomem *aram_csr, + u32 errsou, bool *reset_required) +{ + *reset_required |= adf_handle_timiscsts(accel_dev, csr, errsou); + *reset_required |= adf_handle_ricppintsts(accel_dev, csr, errsou); + *reset_required |= adf_handle_ticppintsts(accel_dev, csr, errsou); + *reset_required |= adf_handle_aramcerr(accel_dev, aram_csr, errsou); + *reset_required |= adf_handle_aramuerr(accel_dev, aram_csr, errsou); + *reset_required |= adf_handle_reg_cppmemtgterr(accel_dev, aram_csr, errsou); + *reset_required |= adf_handle_atufaultstatus(accel_dev, csr, errsou); +} + +static bool adf_gen4_handle_interrupt(struct adf_accel_dev *accel_dev, + bool *reset_required) +{ + void __iomem *aram_csr = adf_get_aram_base(accel_dev); + void __iomem *csr = adf_get_pmisc_base(accel_dev); + u32 errsou = ADF_CSR_RD(csr, ADF_GEN4_ERRSOU0); + bool handled = false; + + *reset_required = false; + + if (errsou & ADF_GEN4_ERRSOU0_BIT) { + adf_gen4_process_errsou0(accel_dev, csr); + handled = true; + } + + errsou = ADF_CSR_RD(csr, ADF_GEN4_ERRSOU1); + if (errsou & ADF_GEN4_ERRSOU1_BITMASK) { + adf_gen4_process_errsou1(accel_dev, csr, errsou, reset_required); + handled = true; + } + + errsou = ADF_CSR_RD(csr, ADF_GEN4_ERRSOU2); + if (errsou & ADF_GEN4_ERRSOU2_BITMASK) { + adf_gen4_process_errsou2(accel_dev, csr, errsou, reset_required); + handled = true; + } + + errsou = ADF_CSR_RD(csr, ADF_GEN4_ERRSOU3); + if (errsou & ADF_GEN4_ERRSOU3_BITMASK) { + adf_gen4_process_errsou3(accel_dev, csr, aram_csr, errsou, reset_required); + handled = true; + } + + return handled; +} + +void adf_gen4_init_ras_ops(struct adf_ras_ops *ras_ops) +{ + ras_ops->enable_ras_errors = adf_gen4_enable_ras; + ras_ops->disable_ras_errors = adf_gen4_disable_ras; + ras_ops->handle_interrupt = adf_gen4_handle_interrupt; +} +EXPORT_SYMBOL_GPL(adf_gen4_init_ras_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_ras.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_ras.h new file mode 100644 index 000000000..53352083c --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_ras.h @@ -0,0 +1,825 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ +#ifndef ADF_GEN4_RAS_H_ +#define ADF_GEN4_RAS_H_ + +#include + +struct adf_ras_ops; + +/* ERRSOU0 Correctable error mask*/ +#define ADF_GEN4_ERRSOU0_BIT BIT(0) + +/* HI AE Correctable error log */ +#define ADF_GEN4_HIAECORERRLOG_CPP0 0x41A308 + +/* HI AE Correctable error log enable */ +#define ADF_GEN4_HIAECORERRLOGENABLE_CPP0 0x41A318 +#define ADF_GEN4_ERRSOU1_HIAEUNCERRLOG_CPP0_BIT BIT(0) +#define ADF_GEN4_ERRSOU1_HICPPAGENTCMDPARERRLOG_BIT BIT(1) +#define ADF_GEN4_ERRSOU1_RIMEM_PARERR_STS_BIT BIT(2) +#define ADF_GEN4_ERRSOU1_TIMEM_PARERR_STS_BIT BIT(3) +#define ADF_GEN4_ERRSOU1_RIMISCSTS_BIT BIT(4) + +#define ADF_GEN4_ERRSOU1_BITMASK ( \ + (ADF_GEN4_ERRSOU1_HIAEUNCERRLOG_CPP0_BIT) | \ + (ADF_GEN4_ERRSOU1_HICPPAGENTCMDPARERRLOG_BIT) | \ + (ADF_GEN4_ERRSOU1_RIMEM_PARERR_STS_BIT) | \ + (ADF_GEN4_ERRSOU1_TIMEM_PARERR_STS_BIT) | \ + (ADF_GEN4_ERRSOU1_RIMISCSTS_BIT)) + +/* HI AE Uncorrectable error log */ +#define ADF_GEN4_HIAEUNCERRLOG_CPP0 0x41A300 + +/* HI AE Uncorrectable error log enable */ +#define ADF_GEN4_HIAEUNCERRLOGENABLE_CPP0 0x41A320 + +/* HI CPP Agent Command parity error log */ +#define ADF_GEN4_HICPPAGENTCMDPARERRLOG 0x41A310 + +/* HI CPP Agent Command parity error logging enable */ +#define ADF_GEN4_HICPPAGENTCMDPARERRLOGENABLE 0x41A314 + +/* RI Memory parity error status register */ +#define ADF_GEN4_RIMEM_PARERR_STS 0x41B128 + +/* RI Memory parity error reporting enable */ +#define ADF_GEN4_RI_MEM_PAR_ERR_EN0 0x41B12C + +/* + * RI Memory parity error mask + * BIT(0) - BIT(3) - ri_iosf_pdata_rxq[0:3] parity error + * BIT(4) - ri_tlq_phdr parity error + * BIT(5) - ri_tlq_pdata parity error + * BIT(6) - ri_tlq_nphdr parity error + * BIT(7) - ri_tlq_npdata parity error + * BIT(8) - BIT(9) - ri_tlq_cplhdr[0:1] parity error + * BIT(10) - BIT(17) - ri_tlq_cpldata[0:7] parity error + * BIT(18) - set this bit to 1 to enable logging status to ri_mem_par_err_sts0 + * BIT(19) - ri_cds_cmd_fifo parity error + * BIT(20) - ri_obc_ricpl_fifo parity error + * BIT(21) - ri_obc_tiricpl_fifo parity error + * BIT(22) - ri_obc_cppcpl_fifo parity error + * BIT(23) - ri_obc_pendcpl_fifo parity error + * BIT(24) - ri_cpp_cmd_fifo parity error + * BIT(25) - ri_cds_ticmd_fifo parity error + * BIT(26) - riti_cmd_fifo parity error + * BIT(27) - ri_int_msixtbl parity error + * BIT(28) - ri_int_imstbl parity error + * BIT(30) - ri_kpt_fuses parity error + */ +#define ADF_GEN4_RIMEM_PARERR_STS_UNCERR_BITMASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(5) | \ + BIT(7) | BIT(10) | BIT(11) | BIT(12) | BIT(13) | \ + BIT(14) | BIT(15) | BIT(16) | BIT(17) | BIT(18) | BIT(19) | \ + BIT(20) | BIT(21) | BIT(22) | BIT(23) | BIT(24) | BIT(25) | \ + BIT(26) | BIT(27) | BIT(28) | BIT(30)) + +#define ADF_GEN4_RIMEM_PARERR_STS_FATAL_BITMASK \ + (BIT(4) | BIT(6) | BIT(8) | BIT(9)) + +/* TI CI parity status */ +#define ADF_GEN4_TI_CI_PAR_STS 0x50060C + +/* TI CI parity reporting mask */ +#define ADF_GEN4_TI_CI_PAR_ERR_MASK 0x500608 + +/* + * TI CI parity status mask + * BIT(0) - CdCmdQ_sts patiry error status + * BIT(1) - CdDataQ_sts parity error status + * BIT(3) - CPP_SkidQ_sts parity error status + * BIT(7) - CPP_SkidQ_sc_sts parity error status + */ +#define ADF_GEN4_TI_CI_PAR_STS_BITMASK \ + (BIT(0) | BIT(1) | BIT(3) | BIT(7)) + +/* TI PULLFUB parity status */ +#define ADF_GEN4_TI_PULL0FUB_PAR_STS 0x500618 + +/* TI PULLFUB parity error reporting mask */ +#define ADF_GEN4_TI_PULL0FUB_PAR_ERR_MASK 0x500614 + +/* + * TI PULLFUB parity status mask + * BIT(0) - TrnPullReqQ_sts parity status + * BIT(1) - TrnSharedDataQ_sts parity status + * BIT(2) - TrnPullReqDataQ_sts parity status + * BIT(4) - CPP_CiPullReqQ_sts parity status + * BIT(5) - CPP_TrnPullReqQ_sts parity status + * BIT(6) - CPP_PullidQ_sts parity status + * BIT(7) - CPP_WaitDataQ_sts parity status + * BIT(8) - CPP_CdDataQ_sts parity status + * BIT(9) - CPP_TrnDataQP0_sts parity status + * BIT(10) - BIT(11) - CPP_TrnDataQRF[00:01]_sts parity status + * BIT(12) - CPP_TrnDataQP1_sts parity status + * BIT(13) - BIT(14) - CPP_TrnDataQRF[10:11]_sts parity status + */ +#define ADF_GEN4_TI_PULL0FUB_PAR_STS_BITMASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(4) | BIT(5) | BIT(6) | BIT(7) | \ + BIT(8) | BIT(9) | BIT(10) | BIT(11) | BIT(12) | BIT(13) | BIT(14)) + +/* TI PUSHUB parity status */ +#define ADF_GEN4_TI_PUSHFUB_PAR_STS 0x500630 + +/* TI PUSHFUB parity error reporting mask */ +#define ADF_GEN4_TI_PUSHFUB_PAR_ERR_MASK 0x50062C + +/* + * TI PUSHUB parity status mask + * BIT(0) - SbPushReqQ_sts parity status + * BIT(1) - BIT(2) - SbPushDataQ[0:1]_sts parity status + * BIT(4) - CPP_CdPushReqQ_sts parity status + * BIT(5) - BIT(6) - CPP_CdPushDataQ[0:1]_sts parity status + * BIT(7) - CPP_SbPushReqQ_sts parity status + * BIT(8) - CPP_SbPushDataQP_sts parity status + * BIT(9) - BIT(10) - CPP_SbPushDataQRF[0:1]_sts parity status + */ +#define ADF_GEN4_TI_PUSHFUB_PAR_STS_BITMASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(4) | BIT(5) | \ + BIT(6) | BIT(7) | BIT(8) | BIT(9) | BIT(10)) + +/* TI CD parity status */ +#define ADF_GEN4_TI_CD_PAR_STS 0x50063C + +/* TI CD parity error mask */ +#define ADF_GEN4_TI_CD_PAR_ERR_MASK 0x500638 + +/* + * TI CD parity status mask + * BIT(0) - BIT(15) - CtxMdRam[0:15]_sts parity status + * BIT(16) - Leaf2ClusterRam_sts parity status + * BIT(17) - BIT(18) - Ring2LeafRam[0:1]_sts parity status + * BIT(19) - VirtualQ_sts parity status + * BIT(20) - DtRdQ_sts parity status + * BIT(21) - DtWrQ_sts parity status + * BIT(22) - RiCmdQ_sts parity status + * BIT(23) - BypassQ_sts parity status + * BIT(24) - DtRdQ_sc_sts parity status + * BIT(25) - DtWrQ_sc_sts parity status + */ +#define ADF_GEN4_TI_CD_PAR_STS_BITMASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4) | BIT(5) | BIT(6) | \ + BIT(7) | BIT(8) | BIT(9) | BIT(10) | BIT(11) | BIT(12) | BIT(13) | \ + BIT(14) | BIT(15) | BIT(16) | BIT(17) | BIT(18) | BIT(19) | BIT(20) | \ + BIT(21) | BIT(22) | BIT(23) | BIT(24) | BIT(25)) + +/* TI TRNSB parity status */ +#define ADF_GEN4_TI_TRNSB_PAR_STS 0x500648 + +/* TI TRNSB Parity error reporting mask */ +#define ADF_GEN4_TI_TRNSB_PAR_ERR_MASK 0x500644 + +/* + * TI TRNSB parity status mask + * BIT(0) - TrnPHdrQP_sts parity status + * BIT(1) - TrnPHdrQRF_sts parity status + * BIT(2) - TrnPDataQP_sts parity status + * BIT(3) - BIT(6) - TrnPDataQRF[0:3]_sts parity status + * BIT(7) - TrnNpHdrQP_sts parity status + * BIT(8) - BIT(9) - TrnNpHdrQRF[0:1]_sts parity status + * BIT(10) - TrnCplHdrQ_sts parity status + * BIT(11) - TrnPutObsReqQ_sts parity status + * BIT(12) - TrnPushReqQ_sts parity status + * BIT(13) - SbSplitIdRam_sts parity status + * BIT(14) - SbReqCountQ_sts parity status + * BIT(15) - SbCplTrkRam_sts parity status + * BIT(16) - SbGetObsReqQ_sts parity status + * BIT(17) - SbEpochIdQ_sts parity status + * BIT(18) - SbAtCplHdrQ_sts parity status + * BIT(19) - SbAtCplDataQ_sts parity status + * BIT(20) - SbReqCountRam_sts parity status + * BIT(21) - SbAtCplHdrQ_sc_sts parity status + */ +#define ADF_GEN4_TI_TRNSB_PAR_STS_BITMASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4) | BIT(5) | BIT(6) | \ + BIT(7) | BIT(8) | BIT(9) | BIT(10) | BIT(11) | BIT(12) | \ + BIT(13) | BIT(14) | BIT(15) | BIT(16) | BIT(17) | BIT(18) | \ + BIT(19) | BIT(20) | BIT(21)) + +/* Status register to log misc error on RI */ +#define ADF_GEN4_RIMISCSTS 0x41B1B8 + +/* Status control register to log misc RI error */ +#define ADF_GEN4_RIMISCCTL 0x41B1BC + +/* + * ERRSOU2 bit mask + * BIT(0) - SSM Interrupt Mask + * BIT(1) - CFC on CPP. ORed of CFC Push error and Pull error + * BIT(2) - BIT(4) - CPP attention interrupts, deprecated on gen4 devices + * BIT(18) - PM interrupt + */ +#define ADF_GEN4_ERRSOU2_SSM_ERR_BIT BIT(0) +#define ADF_GEN4_ERRSOU2_CPP_CFC_ERR_STATUS_BIT BIT(1) +#define ADF_GEN4_ERRSOU2_CPP_CFC_ATT_INT_BITMASK \ + (BIT(2) | BIT(3) | BIT(4)) + +#define ADF_GEN4_ERRSOU2_PM_INT_BIT BIT(18) + +#define ADF_GEN4_ERRSOU2_BITMASK \ + (ADF_GEN4_ERRSOU2_SSM_ERR_BIT | \ + ADF_GEN4_ERRSOU2_CPP_CFC_ERR_STATUS_BIT) + +#define ADF_GEN4_ERRSOU2_DIS_BITMASK \ + (ADF_GEN4_ERRSOU2_SSM_ERR_BIT | \ + ADF_GEN4_ERRSOU2_CPP_CFC_ERR_STATUS_BIT | \ + ADF_GEN4_ERRSOU2_CPP_CFC_ATT_INT_BITMASK) + +#define ADF_GEN4_IAINTSTATSSM 0x28 + +/* IAINTSTATSSM error bit mask definitions */ +#define ADF_GEN4_IAINTSTATSSM_UERRSSMSH_BIT BIT(0) +#define ADF_GEN4_IAINTSTATSSM_CERRSSMSH_BIT BIT(1) +#define ADF_GEN4_IAINTSTATSSM_PPERR_BIT BIT(2) +#define ADF_GEN4_IAINTSTATSSM_SLICEHANG_ERR_BIT BIT(3) +#define ADF_GEN4_IAINTSTATSSM_SPPPARERR_BIT BIT(4) +#define ADF_GEN4_IAINTSTATSSM_SSMCPPERR_BIT BIT(5) +#define ADF_GEN4_IAINTSTATSSM_SSMSOFTERRORPARITY_BIT BIT(6) +#define ADF_GEN4_IAINTSTATSSM_SER_ERR_SSMSH_CERR_BIT BIT(7) +#define ADF_GEN4_IAINTSTATSSM_SER_ERR_SSMSH_UNCERR_BIT BIT(8) + +#define ADF_GEN4_IAINTSTATSSM_BITMASK \ + (ADF_GEN4_IAINTSTATSSM_UERRSSMSH_BIT | \ + ADF_GEN4_IAINTSTATSSM_CERRSSMSH_BIT | \ + ADF_GEN4_IAINTSTATSSM_PPERR_BIT | \ + ADF_GEN4_IAINTSTATSSM_SLICEHANG_ERR_BIT | \ + ADF_GEN4_IAINTSTATSSM_SPPPARERR_BIT | \ + ADF_GEN4_IAINTSTATSSM_SSMCPPERR_BIT | \ + ADF_GEN4_IAINTSTATSSM_SSMSOFTERRORPARITY_BIT | \ + ADF_GEN4_IAINTSTATSSM_SER_ERR_SSMSH_CERR_BIT | \ + ADF_GEN4_IAINTSTATSSM_SER_ERR_SSMSH_UNCERR_BIT) + +#define ADF_GEN4_UERRSSMSH 0x18 + +/* + * UERRSSMSH error bit masks definitions + * + * BIT(0) - Indicates one uncorrectable error + * BIT(15) - Indicates multiple uncorrectable errors + * in device shared memory + */ +#define ADF_GEN4_UERRSSMSH_BITMASK (BIT(0) | BIT(15)) + +#define ADF_GEN4_UERRSSMSHAD 0x1C + +#define ADF_GEN4_CERRSSMSH 0x10 + +/* + * CERRSSMSH error bit + * BIT(0) - Indicates one correctable error + */ +#define ADF_GEN4_CERRSSMSH_ERROR_BIT BIT(0) + +#define ADF_GEN4_CERRSSMSHAD 0x14 + +/* SSM error handling features enable register */ +#define ADF_GEN4_SSMFEATREN 0x198 + +/* + * Disable SSM error detection and reporting features + * enabled by device driver on RAS initialization + * + * following bits should be cleared : + * BIT(4) - Disable parity for CPP parity + * BIT(12) - Disable logging push/pull data error in pperr register. + * BIT(16) - BIT(23) - Disable parity for SPPs + * BIT(24) - BIT(27) - Disable parity for SPPs, if it's supported on the device. + */ +#define ADF_GEN4_SSMFEATREN_DIS_BITMASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(5) | BIT(6) | BIT(7) | \ + BIT(8) | BIT(9) | BIT(10) | BIT(11) | BIT(13) | BIT(14) | BIT(15)) + +#define ADF_GEN4_INTMASKSSM 0x0 + +/* + * Error reporting mask in INTMASKSSM + * BIT(0) - Shared memory uncorrectable interrupt mask + * BIT(1) - Shared memory correctable interrupt mask + * BIT(2) - PPERR interrupt mask + * BIT(3) - CPP parity error Interrupt mask + * BIT(4) - SSM interrupt generated by SER correctable error mask + * BIT(5) - SSM interrupt generated by SER uncorrectable error + * - not stop and scream - mask + */ +#define ADF_GEN4_INTMASKSSM_BITMASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4) | BIT(5)) + +/* CPP push or pull error */ +#define ADF_GEN4_PPERR 0x8 + +#define ADF_GEN4_PPERR_BITMASK (BIT(0) | BIT(1)) + +#define ADF_GEN4_PPERRID 0xC + +/* Slice hang handling related registers */ +#define ADF_GEN4_SLICEHANGSTATUS_ATH_CPH 0x84 +#define ADF_GEN4_SLICEHANGSTATUS_CPR_XLT 0x88 +#define ADF_GEN4_SLICEHANGSTATUS_DCPR_UCS 0x90 +#define ADF_GEN4_SLICEHANGSTATUS_WAT_WCP 0x8C +#define ADF_GEN4_SLICEHANGSTATUS_PKE 0x94 + +#define ADF_GEN4_SHINTMASKSSM_ATH_CPH 0xF0 +#define ADF_GEN4_SHINTMASKSSM_CPR_XLT 0xF4 +#define ADF_GEN4_SHINTMASKSSM_DCPR_UCS 0xFC +#define ADF_GEN4_SHINTMASKSSM_WAT_WCP 0xF8 +#define ADF_GEN4_SHINTMASKSSM_PKE 0x100 + +/* SPP pull cmd parity err_*slice* CSR */ +#define ADF_GEN4_SPPPULLCMDPARERR_ATH_CPH 0x1A4 +#define ADF_GEN4_SPPPULLCMDPARERR_CPR_XLT 0x1A8 +#define ADF_GEN4_SPPPULLCMDPARERR_DCPR_UCS 0x1B0 +#define ADF_GEN4_SPPPULLCMDPARERR_PKE 0x1B4 +#define ADF_GEN4_SPPPULLCMDPARERR_WAT_WCP 0x1AC + +/* SPP pull data parity err_*slice* CSR */ +#define ADF_GEN4_SPPPULLDATAPARERR_ATH_CPH 0x1BC +#define ADF_GEN4_SPPPULLDATAPARERR_CPR_XLT 0x1C0 +#define ADF_GEN4_SPPPULLDATAPARERR_DCPR_UCS 0x1C8 +#define ADF_GEN4_SPPPULLDATAPARERR_PKE 0x1CC +#define ADF_GEN4_SPPPULLDATAPARERR_WAT_WCP 0x1C4 + +/* SPP push cmd parity err_*slice* CSR */ +#define ADF_GEN4_SPPPUSHCMDPARERR_ATH_CPH 0x1D4 +#define ADF_GEN4_SPPPUSHCMDPARERR_CPR_XLT 0x1D8 +#define ADF_GEN4_SPPPUSHCMDPARERR_DCPR_UCS 0x1E0 +#define ADF_GEN4_SPPPUSHCMDPARERR_PKE 0x1E4 +#define ADF_GEN4_SPPPUSHCMDPARERR_WAT_WCP 0x1DC + +/* SPP push data parity err_*slice* CSR */ +#define ADF_GEN4_SPPPUSHDATAPARERR_ATH_CPH 0x1EC +#define ADF_GEN4_SPPPUSHDATAPARERR_CPR_XLT 0x1F0 +#define ADF_GEN4_SPPPUSHDATAPARERR_DCPR_UCS 0x1F8 +#define ADF_GEN4_SPPPUSHDATAPARERR_PKE 0x1FC +#define ADF_GEN4_SPPPUSHDATAPARERR_WAT_WCP 0x1F4 + +/* Accelerator SPP parity error mask registers */ +#define ADF_GEN4_SPPPARERRMSK_ATH_CPH 0x204 +#define ADF_GEN4_SPPPARERRMSK_CPR_XLT 0x208 +#define ADF_GEN4_SPPPARERRMSK_DCPR_UCS 0x210 +#define ADF_GEN4_SPPPARERRMSK_PKE 0x214 +#define ADF_GEN4_SPPPARERRMSK_WAT_WCP 0x20C + +#define ADF_GEN4_SSMCPPERR 0x224 + +/* + * Uncorrectable error mask in SSMCPPERR + * BIT(0) - indicates CPP command parity error + * BIT(1) - indicates CPP Main Push PPID parity error + * BIT(2) - indicates CPP Main ePPID parity error + * BIT(3) - indicates CPP Main push data parity error + * BIT(4) - indicates CPP Main Pull PPID parity error + * BIT(5) - indicates CPP target pull data parity error + */ +#define ADF_GEN4_SSMCPPERR_FATAL_BITMASK \ + (BIT(0) | BIT(1) | BIT(4)) + +#define ADF_GEN4_SSMCPPERR_UNCERR_BITMASK \ + (BIT(2) | BIT(3) | BIT(5)) + +#define ADF_GEN4_SSMSOFTERRORPARITY_SRC 0x9C +#define ADF_GEN4_SSMSOFTERRORPARITYMASK_SRC 0xB8 + +#define ADF_GEN4_SSMSOFTERRORPARITY_ATH_CPH 0xA0 +#define ADF_GEN4_SSMSOFTERRORPARITYMASK_ATH_CPH 0xBC + +#define ADF_GEN4_SSMSOFTERRORPARITY_CPR_XLT 0xA4 +#define ADF_GEN4_SSMSOFTERRORPARITYMASK_CPR_XLT 0xC0 + +#define ADF_GEN4_SSMSOFTERRORPARITY_DCPR_UCS 0xAC +#define ADF_GEN4_SSMSOFTERRORPARITYMASK_DCPR_UCS 0xC8 + +#define ADF_GEN4_SSMSOFTERRORPARITY_PKE 0xB0 +#define ADF_GEN4_SSMSOFTERRORPARITYMASK_PKE 0xCC + +#define ADF_GEN4_SSMSOFTERRORPARITY_WAT_WCP 0xA8 +#define ADF_GEN4_SSMSOFTERRORPARITYMASK_WAT_WCP 0xC4 + +/* RF parity error detected in SharedRAM */ +#define ADF_GEN4_SSMSOFTERRORPARITY_SRC_BIT BIT(0) + +#define ADF_GEN4_SER_ERR_SSMSH 0x44C + +/* + * Fatal error mask in SER_ERR_SSMSH + * BIT(0) - Indicates an uncorrectable error has occurred in the + * accelerator controller command RFs + * BIT(2) - Parity error occurred in the bank SPP fifos + * BIT(3) - Indicates Parity error occurred in following fifos in + * the design + * BIT(4) - Parity error occurred in flops in the design + * BIT(5) - Uncorrectable error has occurred in the + * target push and pull data register flop + * BIT(7) - Indicates Parity error occurred in the Resource Manager + * pending lock request fifos + * BIT(8) - Indicates Parity error occurred in the Resource Manager + * MECTX command queues logic + * BIT(9) - Indicates Parity error occurred in the Resource Manager + * MECTX sigdone fifo flops + * BIT(10) - Indicates an uncorrectable error has occurred in the + * Resource Manager MECTX command RFs + * BIT(14) - Parity error occurred in Buffer Manager sigdone FIFO + */ + #define ADF_GEN4_SER_ERR_SSMSH_FATAL_BITMASK \ + (BIT(0) | BIT(2) | BIT(3) | BIT(4) | BIT(5) | BIT(7) | \ + BIT(8) | BIT(9) | BIT(10) | BIT(14)) + +/* + * Uncorrectable error mask in SER_ERR_SSMSH + * BIT(12) Parity error occurred in Buffer Manager pool 0 + * BIT(13) Parity error occurred in Buffer Manager pool 1 + */ +#define ADF_GEN4_SER_ERR_SSMSH_UNCERR_BITMASK \ + (BIT(12) | BIT(13)) + +/* + * Correctable error mask in SER_ERR_SSMSH + * BIT(1) - Indicates a correctable Error has occurred + * in the slice controller command RFs + * BIT(6) - Indicates a correctable Error has occurred in + * the target push and pull data RFs + * BIT(11) - Indicates an correctable Error has occurred in + * the Resource Manager MECTX command RFs + */ +#define ADF_GEN4_SER_ERR_SSMSH_CERR_BITMASK \ + (BIT(1) | BIT(6) | BIT(11)) + +/* SSM shared memory SER error reporting mask */ +#define ADF_GEN4_SER_EN_SSMSH 0x450 + +/* + * SSM SER error reporting mask in SER_en_err_ssmsh + * BIT(0) - Enables uncorrectable Error detection in : + * 1) slice controller command RFs. + * 2) target push/pull data registers + * BIT(1) - Enables correctable Error detection in : + * 1) slice controller command RFs + * 2) target push/pull data registers + * BIT(2) - Enables Parity error detection in + * 1) bank SPP fifos + * 2) gen4_pull_id_queue + * 3) gen4_push_id_queue + * 4) AE_pull_sigdn_fifo + * 5) DT_push_sigdn_fifo + * 6) slx_push_sigdn_fifo + * 7) secure_push_cmd_fifo + * 8) secure_pull_cmd_fifo + * 9) Head register in FIFO wrapper + * 10) current_cmd in individual push queue + * 11) current_cmd in individual pull queue + * 12) push_command_rxp arbitrated in ssm_push_cmd_queues + * 13) pull_command_rxp arbitrated in ssm_pull_cmd_queues + * BIT(3) - Enables uncorrectable Error detection in + * the resource manager mectx cmd RFs. + * BIT(4) - Enables correctable error detection in the Resource Manager + * mectx command RFs + * BIT(5) - Enables Parity error detection in + * 1) resource manager lock request fifo + * 2) mectx cmdqueues logic + * 3) mectx sigdone fifo + * BIT(6) - Enables Parity error detection in Buffer Manager pools + * and sigdone fifo + */ +#define ADF_GEN4_SER_EN_SSMSH_BITMASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4) | BIT(5) | BIT(6)) + +#define ADF_GEN4_CPP_CFC_ERR_STATUS 0x640C04 + +/* + * BIT(1) - Indicates multiple CPP CFC errors + * BIT(7) - Indicates CPP CFC command parity error type + * BIT(8) - Indicated CPP CFC data parity error type + */ +#define ADF_GEN4_CPP_CFC_ERR_STATUS_MERR_BIT BIT(1) +#define ADF_GEN4_CPP_CFC_ERR_STATUS_CMDPAR_BIT BIT(7) +#define ADF_GEN4_CPP_CFC_ERR_STATUS_DATAPAR_BIT BIT(8) + +/* + * BIT(0) - Enables CFC to detect and log push/pull data error + * BIT(1) - Enables CFC to generate interrupt to PCIEP for CPP error + * BIT(4) - When 1 Parity detection is disabled + * BIT(5) - When 1 Parity detection is disabled on CPP command bus + * BIT(6) - When 1 Parity detection is disabled on CPP push/pull bus + * BIT(9) - When 1 RF parity error detection is disabled + */ +#define ADF_GEN4_CPP_CFC_ERR_CTRL_BITMASK (BIT(0) | BIT(1)) + +#define ADF_GEN4_CPP_CFC_ERR_CTRL_DIS_BITMASK \ + (BIT(4) | BIT(5) | BIT(6) | BIT(9) | BIT(10)) + +#define ADF_GEN4_CPP_CFC_ERR_CTRL 0x640C00 + +/* + * BIT(0) - Clears bit(0) of ADF_GEN4_CPP_CFC_ERR_STATUS + * when an error is reported on CPP + * BIT(1) - Clears bit(1) of ADF_GEN4_CPP_CFC_ERR_STATUS + * when multiple errors are reported on CPP + * BIT(2) - Clears bit(2) of ADF_GEN4_CPP_CFC_ERR_STATUS + * when attention interrupt is reported + */ +#define ADF_GEN4_CPP_CFC_ERR_STATUS_CLR_BITMASK (BIT(0) | BIT(1) | BIT(2)) +#define ADF_GEN4_CPP_CFC_ERR_STATUS_CLR 0x640C08 + +#define ADF_GEN4_CPP_CFC_ERR_PPID_LO 0x640C0C +#define ADF_GEN4_CPP_CFC_ERR_PPID_HI 0x640C10 + +/* Exception reporting in QAT SSM CMP */ +#define ADF_GEN4_EXPRPSSMCPR 0x2000 + +/* + * Uncorrectable error mask in EXPRPSSMCPR + * BIT(2) - Hard fatal error + * BIT(16) - Parity error detected in CPR Push FIFO + * BIT(17) - Parity error detected in CPR Pull FIFO + * BIT(18) - Parity error detected in CPR Hash Table + * BIT(19) - Parity error detected in CPR History Buffer Copy 0 + * BIT(20) - Parity error detected in CPR History Buffer Copy 1 + * BIT(21) - Parity error detected in CPR History Buffer Copy 2 + * BIT(22) - Parity error detected in CPR History Buffer Copy 3 + * BIT(23) - Parity error detected in CPR History Buffer Copy 4 + * BIT(24) - Parity error detected in CPR History Buffer Copy 5 + * BIT(25) - Parity error detected in CPR History Buffer Copy 6 + * BIT(26) - Parity error detected in CPR History Buffer Copy 7 + */ +#define ADF_GEN4_EXPRPSSMCPR_UNCERR_BITMASK \ + (BIT(2) | BIT(16) | BIT(17) | BIT(18) | BIT(19) | BIT(20) | \ + BIT(21) | BIT(22) | BIT(23) | BIT(24) | BIT(25) | BIT(26)) + +/* Exception reporting in QAT SSM XLT */ +#define ADF_GEN4_EXPRPSSMXLT 0xA000 + +/* + * Uncorrectable error mask in EXPRPSSMXLT + * BIT(2) - If set, an Uncorrectable Error event occurred + * BIT(16) - Parity error detected in XLT Push FIFO + * BIT(17) - Parity error detected in XLT Pull FIFO + * BIT(18) - Parity error detected in XLT HCTB0 + * BIT(19) - Parity error detected in XLT HCTB1 + * BIT(20) - Parity error detected in XLT HCTB2 + * BIT(21) - Parity error detected in XLT HCTB3 + * BIT(22) - Parity error detected in XLT CBCL + * BIT(23) - Parity error detected in XLT LITPTR + */ +#define ADF_GEN4_EXPRPSSMXLT_UNCERR_BITMASK \ + (BIT(2) | BIT(16) | BIT(17) | BIT(18) | BIT(19) | BIT(20) | BIT(21) | \ + BIT(22) | BIT(23)) + +/* + * Correctable error mask in EXPRPSSMXLT + * BIT(3) - Correctable error event occurred. + */ +#define ADF_GEN4_EXPRPSSMXLT_CERR_BIT BIT(3) + +/* Exception reporting in QAT SSM DCMP */ +#define ADF_GEN4_EXPRPSSMDCPR(_n_) (0x12000 + (_n_) * 0x80) + +/* + * Uncorrectable error mask in EXPRPSSMDCPR + * BIT(2) - Even hard fatal error + * BIT(4) - Odd hard fatal error + * BIT(6) - decode soft error + * BIT(16) - Parity error detected in CPR Push FIFO + * BIT(17) - Parity error detected in CPR Pull FIFO + * BIT(18) - Parity error detected in the Input Buffer + * BIT(19) - symbuf0parerr + * Parity error detected in CPR Push FIFO + * BIT(20) - symbuf1parerr + * Parity error detected in CPR Push FIFO + */ +#define ADF_GEN4_EXPRPSSMDCPR_UNCERR_BITMASK \ + (BIT(2) | BIT(4) | BIT(6) | BIT(16) | BIT(17) | \ + BIT(18) | BIT(19) | BIT(20)) + +/* + * Correctable error mask in EXPRPSSMDCPR + * BIT(3) - Even ecc correctable error + * BIT(5) - Odd ecc correctable error + */ +#define ADF_GEN4_EXPRPSSMDCPR_CERR_BITMASK (BIT(3) | BIT(5)) + +#define ADF_GEN4_DCPR_SLICES_NUM 3 + +/* + * ERRSOU3 bit masks + * BIT(0) - indicates error Response Order Overflow and/or BME error + * BIT(1) - indicates RI push/pull error + * BIT(2) - indicates TI push/pull error + * BIT(3) - indicates ARAM correctable error + * BIT(4) - indicates ARAM uncorrectable error + * BIT(5) - indicates TI pull parity error + * BIT(6) - indicates RI push parity error + * BIT(7) - indicates VFLR interrupt + * BIT(8) - indicates ring pair interrupts for ATU detected fault + * BIT(9) - indicates error when accessing RLT block + */ +#define ADF_GEN4_ERRSOU3_TIMISCSTS_BIT BIT(0) +#define ADF_GEN4_ERRSOU3_RICPPINTSTS_BITMASK (BIT(1) | BIT(6)) +#define ADF_GEN4_ERRSOU3_TICPPINTSTS_BITMASK (BIT(2) | BIT(5)) +#define ADF_GEN4_ERRSOU3_REG_ARAMCERR_BIT BIT(3) +#define ADF_GEN4_ERRSOU3_REG_ARAMUERR_BIT BIT(4) +#define ADF_GEN4_ERRSOU3_VFLRNOTIFY_BIT BIT(7) +#define ADF_GEN4_ERRSOU3_ATUFAULTSTATUS_BIT BIT(8) +#define ADF_GEN4_ERRSOU3_RLTERROR_BIT BIT(9) + +#define ADF_GEN4_ERRSOU3_BITMASK ( \ + (ADF_GEN4_ERRSOU3_TIMISCSTS_BIT) | \ + (ADF_GEN4_ERRSOU3_RICPPINTSTS_BITMASK) | \ + (ADF_GEN4_ERRSOU3_TICPPINTSTS_BITMASK) | \ + (ADF_GEN4_ERRSOU3_REG_ARAMCERR_BIT) | \ + (ADF_GEN4_ERRSOU3_REG_ARAMUERR_BIT) | \ + (ADF_GEN4_ERRSOU3_VFLRNOTIFY_BIT) | \ + (ADF_GEN4_ERRSOU3_ATUFAULTSTATUS_BIT) | \ + (ADF_GEN4_ERRSOU3_RLTERROR_BIT)) + +/* TI Misc status register */ +#define ADF_GEN4_TIMISCSTS 0x50054C + +/* TI Misc error reporting mask */ +#define ADF_GEN4_TIMISCCTL 0x500548 + +/* + * TI Misc error reporting control mask + * BIT(0) - Enables error detection and logging in TIMISCSTS register + * BIT(1) - It has effect only when SRIOV enabled, this bit is 0 by default + * BIT(2) - Enables the D-F-x counter within the dispatch arbiter + * to start based on the command triggered from + * BIT(30) - Disables VFLR functionality + * By setting this bit will revert to CPM1.x functionality + * bits 1, 2 and 30 value should be preserved and not meant to be changed + * within RAS. + */ +#define ADF_GEN4_TIMISCCTL_BIT BIT(0) +#define ADF_GEN4_TIMSCCTL_RELAY_BITMASK (BIT(1) | BIT(2) | BIT(30)) + +/* RI CPP interface status register */ +#define ADF_GEN4_RICPPINTSTS 0x41A330 + +/* + * Uncorrectable error mask in RICPPINTSTS register + * BIT(0) - RI asserted the CPP error signal during a push + * BIT(1) - RI detected the CPP error signal asserted during a pull + * BIT(2) - RI detected a push data parity error + * BIT(3) - RI detected a push valid parity error + */ +#define ADF_GEN4_RICPPINTSTS_BITMASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3)) + +/* RI CPP interface status register control */ +#define ADF_GEN4_RICPPINTCTL 0x41A32C + +/* + * Control bit mask for RICPPINTCTL register + * BIT(0) - value of 1 enables error detection and reporting + * on the RI CPP Push interface + * BIT(1) - value of 1 enables error detection and reporting + * on the RI CPP Pull interface + * BIT(2) - value of 1 enables error detection and reporting + * on the RI Parity + * BIT(3) - value of 1 enable checking parity on CPP + * BIT(4) - value of 1 enables the stop feature of the stop and stream + * for all RI CPP Command RFs + */ +#define ADF_GEN4_RICPPINTCTL_BITMASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4)) + +/* Push ID of the command which triggered the transaction error on RI */ +#define ADF_GEN4_RIERRPUSHID 0x41A334 + +/* Pull ID of the command which triggered the transaction error on RI */ +#define ADF_GEN4_RIERRPULLID 0x41A338 + +/* TI CPP interface status register */ +#define ADF_GEN4_TICPPINTSTS 0x50053C + +/* + * Uncorrectable error mask in TICPPINTSTS register + * BIT(0) - value of 1 indicates that the TI asserted + * the CPP error signal during a push + * BIT(1) - value of 1 indicates that the TI detected + * the CPP error signal asserted during a pull + * BIT(2) - value of 1 indicates that the TI detected + * a pull data parity error + */ +#define ADF_GEN4_TICPPINTSTS_BITMASK \ + (BIT(0) | BIT(1) | BIT(2)) + +/* TI CPP interface status register control */ +#define ADF_GEN4_TICPPINTCTL 0x500538 + +/* + * Control bit mask for TICPPINTCTL register + * BIT(0) - value of 1 enables error detection and reporting on + * the TI CPP Push interface + * BIT(1) - value of 1 enables error detection and reporting on + * the TI CPP Push interface + * BIT(2) - value of 1 enables parity error detection and logging on + * the TI CPP Pull interface + * BIT(3) - value of 1 enables CPP CMD and Pull Data parity checking + * BIT(4) - value of 1 enables TI stop part of stop and scream mode on + * CPP/RF Parity error + */ +#define ADF_GEN4_TICPPINTCTL_BITMASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4)) + +/* Push ID of the command which triggered the transaction error on TI */ +#define ADF_GEN4_TIERRPUSHID 0x500540 + +/* Pull ID of the command which triggered the transaction error on TI */ +#define ADF_GEN4_TIERRPULLID 0x500544 + +/* Correctable error in ARAM agent register */ +#define ADF_GEN4_REG_ARAMCERR 0x1700 + +#define ADF_GEN4_REG_ARAMCERR_BIT BIT(0) + +/* + * Correctable error enablement in ARAM bit mask + * BIT(3) - enable ARAM RAM to fix and log correctable error + * BIT(26) - enables ARAM agent to generate interrupt for correctable error + */ +#define ADF_GEN4_REG_ARAMCERR_EN_BITMASK (BIT(3) | BIT(26)) + +/* Correctable error address in ARAM agent register */ +#define ADF_GEN4_REG_ARAMCERRAD 0x1708 + +/* Uncorrectable error in ARAM agent register */ +#define ADF_GEN4_REG_ARAMUERR 0x1704 + +/* + * ARAM error bit mask + * BIT(0) - indicates error logged in ARAMCERR or ARAMUCERR + * BIT(18) - indicates uncorrectable multiple errors in ARAM agent + */ +#define ADF_GEN4_REG_ARAMUERR_ERROR_BIT BIT(0) +#define ADF_GEN4_REG_ARAMUERR_MULTI_ERRORS_BIT BIT(18) + +/* + * Uncorrectable error enablement in ARAM bit mask + * BIT(3) - enable ARAM RAM to fix and log uncorrectable error + * BIT(19) - enables ARAM agent to generate interrupt for uncorrectable error + */ +#define ADF_GEN4_REG_ARAMUERR_EN_BITMASK (BIT(3) | BIT(19)) + +/* Unorrectable error address in ARAM agent register */ +#define ADF_GEN4_REG_ARAMUERRAD 0x170C + +/* Uncorrectable error transaction push/pull ID registers*/ +#define ADF_GEN4_REG_ERRPPID_LO 0x1714 +#define ADF_GEN4_REG_ERRPPID_HI 0x1718 + +/* ARAM ECC block error enablement */ +#define ADF_GEN4_REG_ARAMCERRUERR_EN 0x1808 + +/* + * ARAM ECC block error control bit masks + * BIT(0) - enable ARAM CD ECC block error detecting + * BIT(1) - enable ARAM pull request ECC error detecting + * BIT(2) - enable ARAM command dispatch ECC error detecting + * BIT(3) - enable ARAM read datapath push ECC error detecting + * BIT(4) - enable ARAM read datapath pull ECC error detecting + * BIT(5) - enable ARAM RMW ECC error detecting + * BIT(6) - enable ARAM write datapath RMW ECC error detecting + * BIT(7) - enable ARAM write datapath ECC error detecting + */ +#define ADF_GEN4_REG_ARAMCERRUERR_EN_BITMASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4) | \ + BIT(5) | BIT(6) | BIT(7)) + +/* ARAM misc memory target error registers*/ +#define ADF_GEN4_REG_CPPMEMTGTERR 0x1710 + +/* + * ARAM misc memory target error bit masks + * BIT(0) - indicates an error in ARAM target memory + * BIT(1) - indicates multiple errors in ARAM target memory + * BIT(4) - indicates pull error in ARAM target memory + * BIT(5) - indicates parity pull error in ARAM target memory + * BIT(6) - indicates push error in ARAM target memory + */ +#define ADF_GEN4_REG_CPPMEMTGTERR_BITMASK \ + (BIT(0) | BIT(4) | BIT(5) | BIT(6)) + +#define ADF_GEN4_REG_CPPMEMTGTERR_MULTI_ERRORS_BIT BIT(1) + +/* + * ARAM misc memory target error enablement mask + * BIT(2) - enables CPP memory to detect and log push/pull data error + * BIT(7) - enables push/pull error to generate interrupts to RI + * BIT(8) - enables ARAM to check parity on pull data and CPP command buses + * BIT(9) - enables ARAM to autopush to AE when push/parity error is detected + * on lookaside DT + */ +#define ADF_GEN4_REG_CPPMEMTGTERR_EN_BITMASK \ + (BIT(2) | BIT(7) | BIT(8) | BIT(9)) + +/* ATU fault status register */ +#define ADF_GEN4_ATUFAULTSTATUS(i) (0x506000 + ((i) * 0x4)) + +#define ADF_GEN4_ATUFAULTSTATUS_BIT BIT(0) + +/* Command Parity error detected on IOSFP Command to QAT */ +#define ADF_GEN4_RIMISCSTS_BIT BIT(0) + +void adf_gen4_init_ras_ops(struct adf_ras_ops *ras_ops); + +#endif /* ADF_GEN4_RAS_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_tl.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_tl.c new file mode 100644 index 000000000..c7ad8cf07 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_tl.c @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2023 Intel Corporation. */ +#include +#include + +#include "adf_gen4_tl.h" +#include "adf_telemetry.h" +#include "adf_tl_debugfs.h" + +#define ADF_GEN4_TL_DEV_REG_OFF(reg) ADF_TL_DEV_REG_OFF(reg, gen4) + +#define ADF_GEN4_TL_RP_REG_OFF(reg) ADF_TL_RP_REG_OFF(reg, gen4) + +#define ADF_GEN4_TL_SL_UTIL_COUNTER(_name) \ + ADF_TL_COUNTER("util_" #_name, \ + ADF_TL_SIMPLE_COUNT, \ + ADF_TL_SLICE_REG_OFF(_name, reg_tm_slice_util, gen4)) + +#define ADF_GEN4_TL_SL_EXEC_COUNTER(_name) \ + ADF_TL_COUNTER("exec_" #_name, \ + ADF_TL_SIMPLE_COUNT, \ + ADF_TL_SLICE_REG_OFF(_name, reg_tm_slice_exec_cnt, gen4)) + +/* Device level counters. */ +static const struct adf_tl_dbg_counter dev_counters[] = { + /* PCIe partial transactions. */ + ADF_TL_COUNTER(PCI_TRANS_CNT_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_pci_trans_cnt)), + /* Max read latency[ns]. */ + ADF_TL_COUNTER(MAX_RD_LAT_NAME, ADF_TL_COUNTER_NS, + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_rd_lat_max)), + /* Read latency average[ns]. */ + ADF_TL_COUNTER_LATENCY(RD_LAT_ACC_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_rd_lat_acc), + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_rd_cmpl_cnt)), + /* Max get to put latency[ns]. */ + ADF_TL_COUNTER(MAX_LAT_NAME, ADF_TL_COUNTER_NS, + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_gp_lat_max)), + /* Get to put latency average[ns]. */ + ADF_TL_COUNTER_LATENCY(LAT_ACC_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_gp_lat_acc), + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_ae_put_cnt)), + /* PCIe write bandwidth[Mbps]. */ + ADF_TL_COUNTER(BW_IN_NAME, ADF_TL_COUNTER_MBPS, + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_bw_in)), + /* PCIe read bandwidth[Mbps]. */ + ADF_TL_COUNTER(BW_OUT_NAME, ADF_TL_COUNTER_MBPS, + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_bw_out)), + /* Page request latency average[ns]. */ + ADF_TL_COUNTER_LATENCY(PAGE_REQ_LAT_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_at_page_req_lat_acc), + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_at_page_req_cnt)), + /* Page translation latency average[ns]. */ + ADF_TL_COUNTER_LATENCY(AT_TRANS_LAT_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_at_trans_lat_acc), + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_at_trans_lat_cnt)), + /* Maximum uTLB used. */ + ADF_TL_COUNTER(AT_MAX_UTLB_USED_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_at_max_tlb_used)), +}; + +/* Slice utilization counters. */ +static const struct adf_tl_dbg_counter sl_util_counters[ADF_TL_SL_CNT_COUNT] = { + /* Compression slice utilization. */ + ADF_GEN4_TL_SL_UTIL_COUNTER(cpr), + /* Translator slice utilization. */ + ADF_GEN4_TL_SL_UTIL_COUNTER(xlt), + /* Decompression slice utilization. */ + ADF_GEN4_TL_SL_UTIL_COUNTER(dcpr), + /* PKE utilization. */ + ADF_GEN4_TL_SL_UTIL_COUNTER(pke), + /* Wireless Authentication slice utilization. */ + ADF_GEN4_TL_SL_UTIL_COUNTER(wat), + /* Wireless Cipher slice utilization. */ + ADF_GEN4_TL_SL_UTIL_COUNTER(wcp), + /* UCS slice utilization. */ + ADF_GEN4_TL_SL_UTIL_COUNTER(ucs), + /* Cipher slice utilization. */ + ADF_GEN4_TL_SL_UTIL_COUNTER(cph), + /* Authentication slice utilization. */ + ADF_GEN4_TL_SL_UTIL_COUNTER(ath), +}; + +/* Slice execution counters. */ +static const struct adf_tl_dbg_counter sl_exec_counters[ADF_TL_SL_CNT_COUNT] = { + /* Compression slice execution count. */ + ADF_GEN4_TL_SL_EXEC_COUNTER(cpr), + /* Translator slice execution count. */ + ADF_GEN4_TL_SL_EXEC_COUNTER(xlt), + /* Decompression slice execution count. */ + ADF_GEN4_TL_SL_EXEC_COUNTER(dcpr), + /* PKE execution count. */ + ADF_GEN4_TL_SL_EXEC_COUNTER(pke), + /* Wireless Authentication slice execution count. */ + ADF_GEN4_TL_SL_EXEC_COUNTER(wat), + /* Wireless Cipher slice execution count. */ + ADF_GEN4_TL_SL_EXEC_COUNTER(wcp), + /* UCS slice execution count. */ + ADF_GEN4_TL_SL_EXEC_COUNTER(ucs), + /* Cipher slice execution count. */ + ADF_GEN4_TL_SL_EXEC_COUNTER(cph), + /* Authentication slice execution count. */ + ADF_GEN4_TL_SL_EXEC_COUNTER(ath), +}; + +/* Ring pair counters. */ +static const struct adf_tl_dbg_counter rp_counters[] = { + /* PCIe partial transactions. */ + ADF_TL_COUNTER(PCI_TRANS_CNT_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN4_TL_RP_REG_OFF(reg_tl_pci_trans_cnt)), + /* Get to put latency average[ns]. */ + ADF_TL_COUNTER_LATENCY(LAT_ACC_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN4_TL_RP_REG_OFF(reg_tl_gp_lat_acc), + ADF_GEN4_TL_RP_REG_OFF(reg_tl_ae_put_cnt)), + /* PCIe write bandwidth[Mbps]. */ + ADF_TL_COUNTER(BW_IN_NAME, ADF_TL_COUNTER_MBPS, + ADF_GEN4_TL_RP_REG_OFF(reg_tl_bw_in)), + /* PCIe read bandwidth[Mbps]. */ + ADF_TL_COUNTER(BW_OUT_NAME, ADF_TL_COUNTER_MBPS, + ADF_GEN4_TL_RP_REG_OFF(reg_tl_bw_out)), + /* Message descriptor DevTLB hit rate. */ + ADF_TL_COUNTER(AT_GLOB_DTLB_HIT_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN4_TL_RP_REG_OFF(reg_tl_at_glob_devtlb_hit)), + /* Message descriptor DevTLB miss rate. */ + ADF_TL_COUNTER(AT_GLOB_DTLB_MISS_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN4_TL_RP_REG_OFF(reg_tl_at_glob_devtlb_miss)), + /* Payload DevTLB hit rate. */ + ADF_TL_COUNTER(AT_PAYLD_DTLB_HIT_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN4_TL_RP_REG_OFF(reg_tl_at_payld_devtlb_hit)), + /* Payload DevTLB miss rate. */ + ADF_TL_COUNTER(AT_PAYLD_DTLB_MISS_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN4_TL_RP_REG_OFF(reg_tl_at_payld_devtlb_miss)), +}; + +void adf_gen4_init_tl_data(struct adf_tl_hw_data *tl_data) +{ + tl_data->layout_sz = ADF_GEN4_TL_LAYOUT_SZ; + tl_data->slice_reg_sz = ADF_GEN4_TL_SLICE_REG_SZ; + tl_data->rp_reg_sz = ADF_GEN4_TL_RP_REG_SZ; + tl_data->num_hbuff = ADF_GEN4_TL_NUM_HIST_BUFFS; + tl_data->max_rp = ADF_GEN4_TL_MAX_RP_NUM; + tl_data->msg_cnt_off = ADF_GEN4_TL_MSG_CNT_OFF; + tl_data->cpp_ns_per_cycle = ADF_GEN4_CPP_NS_PER_CYCLE; + tl_data->bw_units_to_bytes = ADF_GEN4_TL_BW_HW_UNITS_TO_BYTES; + + tl_data->dev_counters = dev_counters; + tl_data->num_dev_counters = ARRAY_SIZE(dev_counters); + tl_data->sl_util_counters = sl_util_counters; + tl_data->sl_exec_counters = sl_exec_counters; + tl_data->rp_counters = rp_counters; + tl_data->num_rp_counters = ARRAY_SIZE(rp_counters); + tl_data->max_sl_cnt = ADF_GEN4_TL_MAX_SLICES_PER_TYPE; +} +EXPORT_SYMBOL_GPL(adf_gen4_init_tl_data); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_tl.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_tl.h new file mode 100644 index 000000000..32df4163b --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_tl.h @@ -0,0 +1,158 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2023 Intel Corporation. */ +#ifndef ADF_GEN4_TL_H +#define ADF_GEN4_TL_H + +#include +#include + +struct adf_tl_hw_data; + +/* Computation constants. */ +#define ADF_GEN4_CPP_NS_PER_CYCLE 2 +#define ADF_GEN4_TL_BW_HW_UNITS_TO_BYTES 64 + +/* Maximum aggregation time. Value in milliseconds. */ +#define ADF_GEN4_TL_MAX_AGGR_TIME_MS 4000 +/* Num of buffers to store historic values. */ +#define ADF_GEN4_TL_NUM_HIST_BUFFS \ + (ADF_GEN4_TL_MAX_AGGR_TIME_MS / ADF_TL_DATA_WR_INTERVAL_MS) + +/* Max number of HW resources of one type. */ +#define ADF_GEN4_TL_MAX_SLICES_PER_TYPE 24 + +/* Max number of simultaneously monitored ring pairs. */ +#define ADF_GEN4_TL_MAX_RP_NUM 4 + +/** + * struct adf_gen4_tl_slice_data_regs - HW slice data as populated by FW. + * @reg_tm_slice_exec_cnt: Slice execution count. + * @reg_tm_slice_util: Slice utilization. + */ +struct adf_gen4_tl_slice_data_regs { + __u32 reg_tm_slice_exec_cnt; + __u32 reg_tm_slice_util; +}; + +#define ADF_GEN4_TL_SLICE_REG_SZ sizeof(struct adf_gen4_tl_slice_data_regs) + +/** + * struct adf_gen4_tl_device_data_regs - This structure stores device telemetry + * counter values as are being populated periodically by device. + * @reg_tl_rd_lat_acc: read latency accumulator + * @reg_tl_gp_lat_acc: get-put latency accumulator + * @reg_tl_at_page_req_lat_acc: AT/DevTLB page request latency accumulator + * @reg_tl_at_trans_lat_acc: DevTLB transaction latency accumulator + * @reg_tl_re_acc: accumulated ring empty time + * @reg_tl_pci_trans_cnt: PCIe partial transactions + * @reg_tl_rd_lat_max: maximum logged read latency + * @reg_tl_rd_cmpl_cnt: read requests completed count + * @reg_tl_gp_lat_max: maximum logged get to put latency + * @reg_tl_ae_put_cnt: Accelerator Engine put counts across all rings + * @reg_tl_bw_in: PCIe write bandwidth + * @reg_tl_bw_out: PCIe read bandwidth + * @reg_tl_at_page_req_cnt: DevTLB page requests count + * @reg_tl_at_trans_lat_cnt: DevTLB transaction latency samples count + * @reg_tl_at_max_tlb_used: maximum uTLB used + * @reg_tl_re_cnt: ring empty time samples count + * @reserved: reserved + * @ath_slices: array of Authentication slices utilization registers + * @cph_slices: array of Cipher slices utilization registers + * @cpr_slices: array of Compression slices utilization registers + * @xlt_slices: array of Translator slices utilization registers + * @dcpr_slices: array of Decompression slices utilization registers + * @pke_slices: array of PKE slices utilization registers + * @ucs_slices: array of UCS slices utilization registers + * @wat_slices: array of Wireless Authentication slices utilization registers + * @wcp_slices: array of Wireless Cipher slices utilization registers + */ +struct adf_gen4_tl_device_data_regs { + __u64 reg_tl_rd_lat_acc; + __u64 reg_tl_gp_lat_acc; + __u64 reg_tl_at_page_req_lat_acc; + __u64 reg_tl_at_trans_lat_acc; + __u64 reg_tl_re_acc; + __u32 reg_tl_pci_trans_cnt; + __u32 reg_tl_rd_lat_max; + __u32 reg_tl_rd_cmpl_cnt; + __u32 reg_tl_gp_lat_max; + __u32 reg_tl_ae_put_cnt; + __u32 reg_tl_bw_in; + __u32 reg_tl_bw_out; + __u32 reg_tl_at_page_req_cnt; + __u32 reg_tl_at_trans_lat_cnt; + __u32 reg_tl_at_max_tlb_used; + __u32 reg_tl_re_cnt; + __u32 reserved; + struct adf_gen4_tl_slice_data_regs ath_slices[ADF_GEN4_TL_MAX_SLICES_PER_TYPE]; + struct adf_gen4_tl_slice_data_regs cph_slices[ADF_GEN4_TL_MAX_SLICES_PER_TYPE]; + struct adf_gen4_tl_slice_data_regs cpr_slices[ADF_GEN4_TL_MAX_SLICES_PER_TYPE]; + struct adf_gen4_tl_slice_data_regs xlt_slices[ADF_GEN4_TL_MAX_SLICES_PER_TYPE]; + struct adf_gen4_tl_slice_data_regs dcpr_slices[ADF_GEN4_TL_MAX_SLICES_PER_TYPE]; + struct adf_gen4_tl_slice_data_regs pke_slices[ADF_GEN4_TL_MAX_SLICES_PER_TYPE]; + struct adf_gen4_tl_slice_data_regs ucs_slices[ADF_GEN4_TL_MAX_SLICES_PER_TYPE]; + struct adf_gen4_tl_slice_data_regs wat_slices[ADF_GEN4_TL_MAX_SLICES_PER_TYPE]; + struct adf_gen4_tl_slice_data_regs wcp_slices[ADF_GEN4_TL_MAX_SLICES_PER_TYPE]; +}; + +/** + * struct adf_gen4_tl_ring_pair_data_regs - This structure stores Ring Pair + * telemetry counter values as are being populated periodically by device. + * @reg_tl_gp_lat_acc: get-put latency accumulator + * @reserved: reserved + * @reg_tl_pci_trans_cnt: PCIe partial transactions + * @reg_tl_ae_put_cnt: Accelerator Engine put counts across all rings + * @reg_tl_bw_in: PCIe write bandwidth + * @reg_tl_bw_out: PCIe read bandwidth + * @reg_tl_at_glob_devtlb_hit: Message descriptor DevTLB hit rate + * @reg_tl_at_glob_devtlb_miss: Message descriptor DevTLB miss rate + * @reg_tl_at_payld_devtlb_hit: Payload DevTLB hit rate + * @reg_tl_at_payld_devtlb_miss: Payload DevTLB miss rate + * @reg_tl_re_cnt: ring empty time samples count + * @reserved1: reserved + */ +struct adf_gen4_tl_ring_pair_data_regs { + __u64 reg_tl_gp_lat_acc; + __u64 reserved; + __u32 reg_tl_pci_trans_cnt; + __u32 reg_tl_ae_put_cnt; + __u32 reg_tl_bw_in; + __u32 reg_tl_bw_out; + __u32 reg_tl_at_glob_devtlb_hit; + __u32 reg_tl_at_glob_devtlb_miss; + __u32 reg_tl_at_payld_devtlb_hit; + __u32 reg_tl_at_payld_devtlb_miss; + __u32 reg_tl_re_cnt; + __u32 reserved1; +}; + +#define ADF_GEN4_TL_RP_REG_SZ sizeof(struct adf_gen4_tl_ring_pair_data_regs) + +/** + * struct adf_gen4_tl_layout - This structure represents entire telemetry + * counters data: Device + 4 Ring Pairs as are being populated periodically + * by device. + * @tl_device_data_regs: structure of device telemetry registers + * @tl_ring_pairs_data_regs: array of ring pairs telemetry registers + * @reg_tl_msg_cnt: telemetry messages counter + * @reserved: reserved + */ +struct adf_gen4_tl_layout { + struct adf_gen4_tl_device_data_regs tl_device_data_regs; + struct adf_gen4_tl_ring_pair_data_regs + tl_ring_pairs_data_regs[ADF_GEN4_TL_MAX_RP_NUM]; + __u32 reg_tl_msg_cnt; + __u32 reserved; +}; + +#define ADF_GEN4_TL_LAYOUT_SZ sizeof(struct adf_gen4_tl_layout) +#define ADF_GEN4_TL_MSG_CNT_OFF offsetof(struct adf_gen4_tl_layout, reg_tl_msg_cnt) + +#ifdef CONFIG_DEBUG_FS +void adf_gen4_init_tl_data(struct adf_tl_hw_data *tl_data); +#else +static inline void adf_gen4_init_tl_data(struct adf_tl_hw_data *tl_data) +{ +} +#endif /* CONFIG_DEBUG_FS */ +#endif /* ADF_GEN4_TL_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_vf_mig.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_vf_mig.c new file mode 100644 index 000000000..bf2bae78a --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_vf_mig.c @@ -0,0 +1,1011 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2024 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include + +#include "adf_accel_devices.h" +#include "adf_bank_state.h" +#include "adf_common_drv.h" +#include "adf_gen4_hw_data.h" +#include "adf_gen4_pfvf.h" +#include "adf_pfvf_utils.h" +#include "adf_mstate_mgr.h" +#include "adf_gen4_vf_mig.h" + +#define ADF_GEN4_VF_MSTATE_SIZE 4096 +#define ADF_GEN4_PFVF_RSP_TIMEOUT_US 5000 + +static int adf_gen4_vfmig_save_setup(struct qat_mig_dev *mdev); +static int adf_gen4_vfmig_load_setup(struct qat_mig_dev *mdev, int len); + +static int adf_gen4_vfmig_init_device(struct qat_mig_dev *mdev) +{ + u8 *state; + + state = kmalloc(ADF_GEN4_VF_MSTATE_SIZE, GFP_KERNEL); + if (!state) + return -ENOMEM; + + mdev->state = state; + mdev->state_size = ADF_GEN4_VF_MSTATE_SIZE; + mdev->setup_size = 0; + mdev->remote_setup_size = 0; + + return 0; +} + +static void adf_gen4_vfmig_cleanup_device(struct qat_mig_dev *mdev) +{ + kfree(mdev->state); + mdev->state = NULL; +} + +static void adf_gen4_vfmig_reset_device(struct qat_mig_dev *mdev) +{ + mdev->setup_size = 0; + mdev->remote_setup_size = 0; +} + +static int adf_gen4_vfmig_open_device(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + struct adf_accel_vf_info *vf_info; + struct adf_gen4_vfmig *vfmig; + + vf_info = &accel_dev->pf.vf_info[mdev->vf_id]; + + vfmig = kzalloc_obj(*vfmig); + if (!vfmig) + return -ENOMEM; + + vfmig->mstate_mgr = adf_mstate_mgr_new(mdev->state, mdev->state_size); + if (!vfmig->mstate_mgr) { + kfree(vfmig); + return -ENOMEM; + } + vf_info->mig_priv = vfmig; + mdev->setup_size = 0; + mdev->remote_setup_size = 0; + + return 0; +} + +static void adf_gen4_vfmig_close_device(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + struct adf_accel_vf_info *vf_info; + struct adf_gen4_vfmig *vfmig; + + vf_info = &accel_dev->pf.vf_info[mdev->vf_id]; + if (vf_info->mig_priv) { + vfmig = vf_info->mig_priv; + adf_mstate_mgr_destroy(vfmig->mstate_mgr); + kfree(vfmig); + vf_info->mig_priv = NULL; + } +} + +static int adf_gen4_vfmig_suspend_device(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_accel_vf_info *vf_info; + struct adf_gen4_vfmig *vf_mig; + u32 vf_nr = mdev->vf_id; + int ret, i; + + vf_info = &accel_dev->pf.vf_info[vf_nr]; + vf_mig = vf_info->mig_priv; + + /* Stop all inflight jobs */ + for (i = 0; i < hw_data->num_banks_per_vf; i++) { + u32 pf_bank_nr = i + vf_nr * hw_data->num_banks_per_vf; + + ret = adf_gen4_bank_drain_start(accel_dev, pf_bank_nr, + ADF_RPRESET_POLL_TIMEOUT_US); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to drain bank %d for vf_nr %d\n", i, + vf_nr); + return ret; + } + vf_mig->bank_stopped[i] = true; + + adf_gen4_bank_quiesce_coal_timer(accel_dev, pf_bank_nr, + ADF_COALESCED_POLL_TIMEOUT_US); + } + + return 0; +} + +static int adf_gen4_vfmig_resume_device(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_accel_vf_info *vf_info; + struct adf_gen4_vfmig *vf_mig; + u32 vf_nr = mdev->vf_id; + int i; + + vf_info = &accel_dev->pf.vf_info[vf_nr]; + vf_mig = vf_info->mig_priv; + + for (i = 0; i < hw_data->num_banks_per_vf; i++) { + u32 pf_bank_nr = i + vf_nr * hw_data->num_banks_per_vf; + + if (vf_mig->bank_stopped[i]) { + adf_gen4_bank_drain_finish(accel_dev, pf_bank_nr); + vf_mig->bank_stopped[i] = false; + } + } + + return 0; +} + +struct adf_vf_bank_info { + struct adf_accel_dev *accel_dev; + u32 vf_nr; + u32 bank_nr; +}; + +struct mig_user_sla { + enum adf_base_services srv; + u64 rp_mask; + u32 cir; + u32 pir; +}; + +static int adf_mstate_sla_check(struct adf_mstate_mgr *sub_mgr, u8 *src_buf, + u32 src_size, void *opaque) +{ + struct adf_mstate_vreginfo _sinfo = { src_buf, src_size }; + struct adf_mstate_vreginfo *sinfo = &_sinfo, *dinfo = opaque; + u32 src_sla_cnt = sinfo->size / sizeof(struct mig_user_sla); + u32 dst_sla_cnt = dinfo->size / sizeof(struct mig_user_sla); + struct mig_user_sla *src_slas = sinfo->addr; + struct mig_user_sla *dst_slas = dinfo->addr; + int i, j; + + for (i = 0; i < src_sla_cnt; i++) { + for (j = 0; j < dst_sla_cnt; j++) { + if (src_slas[i].srv != dst_slas[j].srv || + src_slas[i].rp_mask != dst_slas[j].rp_mask) + continue; + + if (src_slas[i].cir > dst_slas[j].cir || + src_slas[i].pir > dst_slas[j].pir) { + pr_err("QAT: DST VF rate limiting mismatch.\n"); + return -EINVAL; + } + break; + } + + if (j == dst_sla_cnt) { + pr_err("QAT: SRC VF rate limiting mismatch - SRC srv %d and rp_mask 0x%llx.\n", + src_slas[i].srv, src_slas[i].rp_mask); + return -EINVAL; + } + } + + return 0; +} + +static inline int adf_mstate_check_cap_size(u32 src_sz, u32 dst_sz, u32 max_sz) +{ + if (src_sz > max_sz || dst_sz > max_sz) + return -EINVAL; + else + return 0; +} + +static int adf_mstate_compatver_check(struct adf_mstate_mgr *sub_mgr, + u8 *src_buf, u32 src_sz, void *opaque) +{ + struct adf_mstate_vreginfo *info = opaque; + u8 compat = 0; + u8 *pcompat; + + if (src_sz != info->size) { + pr_debug("QAT: State mismatch (compat version size), current %u, expected %u\n", + src_sz, info->size); + return -EINVAL; + } + + memcpy(info->addr, src_buf, info->size); + pcompat = info->addr; + if (*pcompat == 0) { + pr_warn("QAT: Unable to determine the version of VF\n"); + return 0; + } + + compat = adf_vf_compat_checker(*pcompat); + if (compat == ADF_PF2VF_VF_INCOMPATIBLE) { + pr_debug("QAT: SRC VF driver (ver=%u) is incompatible with DST PF driver (ver=%u)\n", + *pcompat, ADF_PFVF_COMPAT_THIS_VERSION); + return -EINVAL; + } + + if (compat == ADF_PF2VF_VF_COMPAT_UNKNOWN) + pr_debug("QAT: SRC VF driver (ver=%u) is newer than DST PF driver (ver=%u)\n", + *pcompat, ADF_PFVF_COMPAT_THIS_VERSION); + + return 0; +} + +/* + * adf_mstate_capmask_compare() - compare QAT device capability mask + * @sinfo: Pointer to source capability info + * @dinfo: Pointer to target capability info + * + * This function compares the capability mask between source VF and target VF + * + * Returns: 0 if target capability mask is identical to source capability mask, + * 1 if target mask can represent all the capabilities represented by source mask, + * -1 if target mask can't represent all the capabilities represented by source + * mask. + */ +static int adf_mstate_capmask_compare(struct adf_mstate_vreginfo *sinfo, + struct adf_mstate_vreginfo *dinfo) +{ + u64 src = 0, dst = 0; + + if (adf_mstate_check_cap_size(sinfo->size, dinfo->size, sizeof(u64))) { + pr_debug("QAT: Unexpected capability size %u %u %zu\n", + sinfo->size, dinfo->size, sizeof(u64)); + return -1; + } + + memcpy(&src, sinfo->addr, sinfo->size); + memcpy(&dst, dinfo->addr, dinfo->size); + + pr_debug("QAT: Check cap compatibility of cap %llu %llu\n", src, dst); + + if (src == dst) + return 0; + + if ((src | dst) == dst) + return 1; + + return -1; +} + +static int adf_mstate_capmask_superset(struct adf_mstate_mgr *sub_mgr, u8 *buf, + u32 size, void *opa) +{ + struct adf_mstate_vreginfo sinfo = { buf, size }; + + if (adf_mstate_capmask_compare(&sinfo, opa) >= 0) + return 0; + + return -EINVAL; +} + +static int adf_mstate_capmask_equal(struct adf_mstate_mgr *sub_mgr, u8 *buf, + u32 size, void *opa) +{ + struct adf_mstate_vreginfo sinfo = { buf, size }; + + if (adf_mstate_capmask_compare(&sinfo, opa) == 0) + return 0; + + return -EINVAL; +} + +static int adf_mstate_set_vreg(struct adf_mstate_mgr *sub_mgr, u8 *buf, + u32 size, void *opa) +{ + struct adf_mstate_vreginfo *info = opa; + + if (size != info->size) { + pr_debug("QAT: Unexpected cap size %u %u\n", size, info->size); + return -EINVAL; + } + memcpy(info->addr, buf, info->size); + + return 0; +} + +static u32 adf_gen4_vfmig_get_slas(struct adf_accel_dev *accel_dev, u32 vf_nr, + struct mig_user_sla *pmig_slas) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_rl *rl_data = accel_dev->rate_limiting; + struct rl_sla **sla_type_arr = NULL; + u64 rp_mask, rp_index; + u32 max_num_sla; + u32 sla_cnt = 0; + int i, j; + + if (!accel_dev->rate_limiting) + return 0; + + rp_index = vf_nr * hw_data->num_banks_per_vf; + max_num_sla = adf_rl_get_sla_arr_of_type(rl_data, RL_LEAF, &sla_type_arr); + + for (i = 0; i < max_num_sla; i++) { + if (!sla_type_arr[i]) + continue; + + rp_mask = 0; + for (j = 0; j < sla_type_arr[i]->ring_pairs_cnt; j++) + rp_mask |= BIT(sla_type_arr[i]->ring_pairs_ids[j]); + + if (rp_mask & GENMASK_ULL(rp_index + 3, rp_index)) { + pmig_slas->rp_mask = rp_mask; + pmig_slas->cir = sla_type_arr[i]->cir; + pmig_slas->pir = sla_type_arr[i]->pir; + pmig_slas->srv = sla_type_arr[i]->srv; + pmig_slas++; + sla_cnt++; + } + } + + return sla_cnt; +} + +static int adf_gen4_vfmig_load_etr_regs(struct adf_mstate_mgr *sub_mgr, + u8 *state, u32 size, void *opa) +{ + struct adf_vf_bank_info *vf_bank_info = opa; + struct adf_accel_dev *accel_dev = vf_bank_info->accel_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 pf_bank_nr; + int ret; + + pf_bank_nr = vf_bank_info->bank_nr + vf_bank_info->vf_nr * hw_data->num_banks_per_vf; + ret = hw_data->bank_state_restore(accel_dev, pf_bank_nr, + (struct adf_bank_state *)state); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to load regs for vf%d bank%d\n", + vf_bank_info->vf_nr, vf_bank_info->bank_nr); + return ret; + } + + return 0; +} + +static int adf_gen4_vfmig_load_etr_bank(struct adf_accel_dev *accel_dev, + u32 vf_nr, u32 bank_nr, + struct adf_mstate_mgr *mstate_mgr) +{ + struct adf_vf_bank_info vf_bank_info = {accel_dev, vf_nr, bank_nr}; + struct adf_mstate_sect_h *subsec, *l2_subsec; + struct adf_mstate_mgr sub_sects_mgr; + char bank_ids[ADF_MSTATE_ID_LEN]; + + snprintf(bank_ids, sizeof(bank_ids), ADF_MSTATE_BANK_IDX_IDS "%x", bank_nr); + subsec = adf_mstate_sect_lookup(mstate_mgr, bank_ids, NULL, NULL); + if (!subsec) { + dev_err(&GET_DEV(accel_dev), + "Failed to lookup sec %s for vf%d bank%d\n", + ADF_MSTATE_BANK_IDX_IDS, vf_nr, bank_nr); + return -EINVAL; + } + + adf_mstate_mgr_init_from_psect(&sub_sects_mgr, subsec); + l2_subsec = adf_mstate_sect_lookup(&sub_sects_mgr, ADF_MSTATE_ETR_REGS_IDS, + adf_gen4_vfmig_load_etr_regs, + &vf_bank_info); + if (!l2_subsec) { + dev_err(&GET_DEV(accel_dev), + "Failed to add sec %s for vf%d bank%d\n", + ADF_MSTATE_ETR_REGS_IDS, vf_nr, bank_nr); + return -EINVAL; + } + + return 0; +} + +static int adf_gen4_vfmig_load_etr(struct adf_accel_dev *accel_dev, u32 vf_nr) +{ + struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; + struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; + struct adf_mstate_mgr sub_sects_mgr; + struct adf_mstate_sect_h *subsec; + int ret, i; + + subsec = adf_mstate_sect_lookup(mstate_mgr, ADF_MSTATE_ETRB_IDS, NULL, + NULL); + if (!subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to load sec %s\n", + ADF_MSTATE_ETRB_IDS); + return -EINVAL; + } + + adf_mstate_mgr_init_from_psect(&sub_sects_mgr, subsec); + for (i = 0; i < hw_data->num_banks_per_vf; i++) { + ret = adf_gen4_vfmig_load_etr_bank(accel_dev, vf_nr, i, + &sub_sects_mgr); + if (ret) + return ret; + } + + return 0; +} + +static int adf_gen4_vfmig_load_misc(struct adf_accel_dev *accel_dev, u32 vf_nr) +{ + struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; + struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; + void __iomem *csr = adf_get_pmisc_base(accel_dev); + struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; + struct adf_mstate_sect_h *subsec, *l2_subsec; + struct adf_mstate_mgr sub_sects_mgr; + struct { + char *id; + u64 ofs; + } misc_states[] = { + {ADF_MSTATE_VINTMSK_IDS, ADF_GEN4_VINTMSK_OFFSET(vf_nr)}, + {ADF_MSTATE_VINTMSK_PF2VM_IDS, ADF_GEN4_VINTMSKPF2VM_OFFSET(vf_nr)}, + {ADF_MSTATE_PF2VM_IDS, ADF_GEN4_PF2VM_OFFSET(vf_nr)}, + {ADF_MSTATE_VM2PF_IDS, ADF_GEN4_VM2PF_OFFSET(vf_nr)}, + }; + int i; + + subsec = adf_mstate_sect_lookup(mstate_mgr, ADF_MSTATE_MISCB_IDS, NULL, + NULL); + if (!subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to load sec %s\n", + ADF_MSTATE_MISCB_IDS); + return -EINVAL; + } + + adf_mstate_mgr_init_from_psect(&sub_sects_mgr, subsec); + for (i = 0; i < ARRAY_SIZE(misc_states); i++) { + struct adf_mstate_vreginfo info; + u32 regv; + + info.addr = ®v; + info.size = sizeof(regv); + l2_subsec = adf_mstate_sect_lookup(&sub_sects_mgr, + misc_states[i].id, + adf_mstate_set_vreg, + &info); + if (!l2_subsec) { + dev_err(&GET_DEV(accel_dev), + "Failed to load sec %s\n", misc_states[i].id); + return -EINVAL; + } + ADF_CSR_WR(csr, misc_states[i].ofs, regv); + } + + return 0; +} + +static int adf_gen4_vfmig_load_generic(struct adf_accel_dev *accel_dev, u32 vf_nr) +{ + struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; + struct mig_user_sla dst_slas[RL_RP_CNT_PER_LEAF_MAX] = { }; + struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; + struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; + struct adf_mstate_sect_h *subsec, *l2_subsec; + struct adf_mstate_mgr sub_sects_mgr; + u32 dst_sla_cnt; + struct { + char *id; + int (*action)(struct adf_mstate_mgr *sub_mgr, u8 *buf, u32 size, void *opa); + struct adf_mstate_vreginfo info; + } gen_states[] = { + {ADF_MSTATE_IOV_INIT_IDS, adf_mstate_set_vreg, + {&vf_info->init, sizeof(vf_info->init)}}, + {ADF_MSTATE_COMPAT_VER_IDS, adf_mstate_compatver_check, + {&vf_info->vf_compat_ver, sizeof(vf_info->vf_compat_ver)}}, + {ADF_MSTATE_SLA_IDS, adf_mstate_sla_check, {dst_slas, 0}}, + }; + int i; + + subsec = adf_mstate_sect_lookup(mstate_mgr, ADF_MSTATE_GEN_IDS, NULL, NULL); + if (!subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to load sec %s\n", + ADF_MSTATE_GEN_IDS); + return -EINVAL; + } + + adf_mstate_mgr_init_from_psect(&sub_sects_mgr, subsec); + for (i = 0; i < ARRAY_SIZE(gen_states); i++) { + if (gen_states[i].info.addr == dst_slas) { + dst_sla_cnt = adf_gen4_vfmig_get_slas(accel_dev, vf_nr, dst_slas); + gen_states[i].info.size = dst_sla_cnt * sizeof(struct mig_user_sla); + } + + l2_subsec = adf_mstate_sect_lookup(&sub_sects_mgr, + gen_states[i].id, + gen_states[i].action, + &gen_states[i].info); + if (!l2_subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to load sec %s\n", + gen_states[i].id); + return -EINVAL; + } + } + + return 0; +} + +static int adf_gen4_vfmig_load_config(struct adf_accel_dev *accel_dev, u32 vf_nr) +{ + struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; + struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; + struct adf_mstate_sect_h *subsec, *l2_subsec; + struct adf_mstate_mgr sub_sects_mgr; + struct { + char *id; + int (*action)(struct adf_mstate_mgr *sub_mgr, u8 *buf, u32 size, void *opa); + struct adf_mstate_vreginfo info; + } setups[] = { + {ADF_MSTATE_GEN_CAP_IDS, adf_mstate_capmask_superset, + {&hw_data->accel_capabilities_mask, sizeof(hw_data->accel_capabilities_mask)}}, + {ADF_MSTATE_GEN_SVCMAP_IDS, adf_mstate_capmask_equal, + {&hw_data->ring_to_svc_map, sizeof(hw_data->ring_to_svc_map)}}, + {ADF_MSTATE_GEN_EXTDC_IDS, adf_mstate_capmask_superset, + {&hw_data->extended_dc_capabilities, sizeof(hw_data->extended_dc_capabilities)}}, + }; + int i; + + subsec = adf_mstate_sect_lookup(mstate_mgr, ADF_MSTATE_CONFIG_IDS, NULL, NULL); + if (!subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to load sec %s\n", + ADF_MSTATE_CONFIG_IDS); + return -EINVAL; + } + + adf_mstate_mgr_init_from_psect(&sub_sects_mgr, subsec); + for (i = 0; i < ARRAY_SIZE(setups); i++) { + l2_subsec = adf_mstate_sect_lookup(&sub_sects_mgr, setups[i].id, + setups[i].action, &setups[i].info); + if (!l2_subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to load sec %s\n", + setups[i].id); + return -EINVAL; + } + } + + return 0; +} + +static int adf_gen4_vfmig_save_etr_regs(struct adf_mstate_mgr *subs, u8 *state, + u32 size, void *opa) +{ + struct adf_vf_bank_info *vf_bank_info = opa; + struct adf_accel_dev *accel_dev = vf_bank_info->accel_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 pf_bank_nr; + int ret; + + pf_bank_nr = vf_bank_info->bank_nr; + pf_bank_nr += vf_bank_info->vf_nr * hw_data->num_banks_per_vf; + + ret = hw_data->bank_state_save(accel_dev, pf_bank_nr, + (struct adf_bank_state *)state); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to save regs for vf%d bank%d\n", + vf_bank_info->vf_nr, vf_bank_info->bank_nr); + return ret; + } + + return sizeof(struct adf_bank_state); +} + +static int adf_gen4_vfmig_save_etr_bank(struct adf_accel_dev *accel_dev, + u32 vf_nr, u32 bank_nr, + struct adf_mstate_mgr *mstate_mgr) +{ + struct adf_mstate_sect_h *subsec, *l2_subsec; + struct adf_vf_bank_info vf_bank_info; + struct adf_mstate_mgr sub_sects_mgr; + char bank_ids[ADF_MSTATE_ID_LEN]; + + snprintf(bank_ids, sizeof(bank_ids), ADF_MSTATE_BANK_IDX_IDS "%x", bank_nr); + + subsec = adf_mstate_sect_add(mstate_mgr, bank_ids, NULL, NULL); + if (!subsec) { + dev_err(&GET_DEV(accel_dev), + "Failed to add sec %s for vf%d bank%d\n", + ADF_MSTATE_BANK_IDX_IDS, vf_nr, bank_nr); + return -EINVAL; + } + + adf_mstate_mgr_init_from_parent(&sub_sects_mgr, mstate_mgr); + vf_bank_info.accel_dev = accel_dev; + vf_bank_info.vf_nr = vf_nr; + vf_bank_info.bank_nr = bank_nr; + l2_subsec = adf_mstate_sect_add(&sub_sects_mgr, ADF_MSTATE_ETR_REGS_IDS, + adf_gen4_vfmig_save_etr_regs, + &vf_bank_info); + if (!l2_subsec) { + dev_err(&GET_DEV(accel_dev), + "Failed to add sec %s for vf%d bank%d\n", + ADF_MSTATE_ETR_REGS_IDS, vf_nr, bank_nr); + return -EINVAL; + } + adf_mstate_sect_update(mstate_mgr, &sub_sects_mgr, subsec); + + return 0; +} + +static int adf_gen4_vfmig_save_etr(struct adf_accel_dev *accel_dev, u32 vf_nr) +{ + struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; + struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; + struct adf_mstate_mgr sub_sects_mgr; + struct adf_mstate_sect_h *subsec; + int ret, i; + + subsec = adf_mstate_sect_add(mstate_mgr, ADF_MSTATE_ETRB_IDS, NULL, NULL); + if (!subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", + ADF_MSTATE_ETRB_IDS); + return -EINVAL; + } + + adf_mstate_mgr_init_from_parent(&sub_sects_mgr, mstate_mgr); + for (i = 0; i < hw_data->num_banks_per_vf; i++) { + ret = adf_gen4_vfmig_save_etr_bank(accel_dev, vf_nr, i, + &sub_sects_mgr); + if (ret) + return ret; + } + adf_mstate_sect_update(mstate_mgr, &sub_sects_mgr, subsec); + + return 0; +} + +static int adf_gen4_vfmig_save_misc(struct adf_accel_dev *accel_dev, u32 vf_nr) +{ + struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; + struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; + struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; + void __iomem *csr = adf_get_pmisc_base(accel_dev); + struct adf_mstate_sect_h *subsec, *l2_subsec; + struct adf_mstate_mgr sub_sects_mgr; + struct { + char *id; + u64 offset; + } misc_states[] = { + {ADF_MSTATE_VINTSRC_IDS, ADF_GEN4_VINTSOU_OFFSET(vf_nr)}, + {ADF_MSTATE_VINTMSK_IDS, ADF_GEN4_VINTMSK_OFFSET(vf_nr)}, + {ADF_MSTATE_VINTSRC_PF2VM_IDS, ADF_GEN4_VINTSOUPF2VM_OFFSET(vf_nr)}, + {ADF_MSTATE_VINTMSK_PF2VM_IDS, ADF_GEN4_VINTMSKPF2VM_OFFSET(vf_nr)}, + {ADF_MSTATE_PF2VM_IDS, ADF_GEN4_PF2VM_OFFSET(vf_nr)}, + {ADF_MSTATE_VM2PF_IDS, ADF_GEN4_VM2PF_OFFSET(vf_nr)}, + }; + ktime_t time_exp; + int i; + + subsec = adf_mstate_sect_add(mstate_mgr, ADF_MSTATE_MISCB_IDS, NULL, NULL); + if (!subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", + ADF_MSTATE_MISCB_IDS); + return -EINVAL; + } + + time_exp = ktime_add_us(ktime_get(), ADF_GEN4_PFVF_RSP_TIMEOUT_US); + while (!mutex_trylock(&vf_info->pfvf_mig_lock)) { + if (ktime_after(ktime_get(), time_exp)) { + dev_err(&GET_DEV(accel_dev), "Failed to get pfvf mig lock\n"); + return -ETIMEDOUT; + } + usleep_range(500, 1000); + } + + adf_mstate_mgr_init_from_parent(&sub_sects_mgr, mstate_mgr); + for (i = 0; i < ARRAY_SIZE(misc_states); i++) { + struct adf_mstate_vreginfo info; + u32 regv; + + info.addr = ®v; + info.size = sizeof(regv); + regv = ADF_CSR_RD(csr, misc_states[i].offset); + + l2_subsec = adf_mstate_sect_add_vreg(&sub_sects_mgr, + misc_states[i].id, + &info); + if (!l2_subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", + misc_states[i].id); + mutex_unlock(&vf_info->pfvf_mig_lock); + return -EINVAL; + } + } + + mutex_unlock(&vf_info->pfvf_mig_lock); + adf_mstate_sect_update(mstate_mgr, &sub_sects_mgr, subsec); + + return 0; +} + +static int adf_gen4_vfmig_save_generic(struct adf_accel_dev *accel_dev, u32 vf_nr) +{ + struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; + struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; + struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; + struct adf_mstate_mgr sub_sects_mgr; + struct adf_mstate_sect_h *subsec, *l2_subsec; + struct mig_user_sla src_slas[RL_RP_CNT_PER_LEAF_MAX] = { }; + u32 src_sla_cnt; + struct { + char *id; + struct adf_mstate_vreginfo info; + } gen_states[] = { + {ADF_MSTATE_IOV_INIT_IDS, + {&vf_info->init, sizeof(vf_info->init)}}, + {ADF_MSTATE_COMPAT_VER_IDS, + {&vf_info->vf_compat_ver, sizeof(vf_info->vf_compat_ver)}}, + {ADF_MSTATE_SLA_IDS, {src_slas, 0}}, + }; + int i; + + subsec = adf_mstate_sect_add(mstate_mgr, ADF_MSTATE_GEN_IDS, NULL, NULL); + if (!subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", + ADF_MSTATE_GEN_IDS); + return -EINVAL; + } + + adf_mstate_mgr_init_from_parent(&sub_sects_mgr, mstate_mgr); + for (i = 0; i < ARRAY_SIZE(gen_states); i++) { + if (gen_states[i].info.addr == src_slas) { + src_sla_cnt = adf_gen4_vfmig_get_slas(accel_dev, vf_nr, src_slas); + gen_states[i].info.size = src_sla_cnt * sizeof(struct mig_user_sla); + } + + l2_subsec = adf_mstate_sect_add_vreg(&sub_sects_mgr, + gen_states[i].id, + &gen_states[i].info); + if (!l2_subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", + gen_states[i].id); + return -EINVAL; + } + } + adf_mstate_sect_update(mstate_mgr, &sub_sects_mgr, subsec); + + return 0; +} + +static int adf_gen4_vfmig_save_config(struct adf_accel_dev *accel_dev, u32 vf_nr) +{ + struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; + struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; + struct adf_mstate_mgr sub_sects_mgr; + struct adf_mstate_sect_h *subsec, *l2_subsec; + struct { + char *id; + struct adf_mstate_vreginfo info; + } setups[] = { + {ADF_MSTATE_GEN_CAP_IDS, + {&hw_data->accel_capabilities_mask, sizeof(hw_data->accel_capabilities_mask)}}, + {ADF_MSTATE_GEN_SVCMAP_IDS, + {&hw_data->ring_to_svc_map, sizeof(hw_data->ring_to_svc_map)}}, + {ADF_MSTATE_GEN_EXTDC_IDS, + {&hw_data->extended_dc_capabilities, sizeof(hw_data->extended_dc_capabilities)}}, + }; + int i; + + subsec = adf_mstate_sect_add(mstate_mgr, ADF_MSTATE_CONFIG_IDS, NULL, NULL); + if (!subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", + ADF_MSTATE_CONFIG_IDS); + return -EINVAL; + } + + adf_mstate_mgr_init_from_parent(&sub_sects_mgr, mstate_mgr); + for (i = 0; i < ARRAY_SIZE(setups); i++) { + l2_subsec = adf_mstate_sect_add_vreg(&sub_sects_mgr, setups[i].id, + &setups[i].info); + if (!l2_subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", + setups[i].id); + return -EINVAL; + } + } + adf_mstate_sect_update(mstate_mgr, &sub_sects_mgr, subsec); + + return 0; +} + +static int adf_gen4_vfmig_save_state(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + struct adf_accel_vf_info *vf_info; + struct adf_gen4_vfmig *vfmig; + u32 vf_nr = mdev->vf_id; + int ret; + + vf_info = &accel_dev->pf.vf_info[vf_nr]; + vfmig = vf_info->mig_priv; + + ret = adf_gen4_vfmig_save_setup(mdev); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to save setup for vf_nr %d\n", vf_nr); + return ret; + } + + adf_mstate_mgr_init(vfmig->mstate_mgr, mdev->state + mdev->setup_size, + mdev->state_size - mdev->setup_size); + if (!adf_mstate_preamble_add(vfmig->mstate_mgr)) + return -EINVAL; + + ret = adf_gen4_vfmig_save_generic(accel_dev, vf_nr); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to save generic state for vf_nr %d\n", vf_nr); + return ret; + } + + ret = adf_gen4_vfmig_save_misc(accel_dev, vf_nr); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to save misc bar state for vf_nr %d\n", vf_nr); + return ret; + } + + ret = adf_gen4_vfmig_save_etr(accel_dev, vf_nr); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to save etr bar state for vf_nr %d\n", vf_nr); + return ret; + } + + adf_mstate_preamble_update(vfmig->mstate_mgr); + + return 0; +} + +static int adf_gen4_vfmig_load_state(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + struct adf_accel_vf_info *vf_info; + struct adf_gen4_vfmig *vfmig; + u32 vf_nr = mdev->vf_id; + int ret; + + vf_info = &accel_dev->pf.vf_info[vf_nr]; + vfmig = vf_info->mig_priv; + + ret = adf_gen4_vfmig_load_setup(mdev, mdev->state_size); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Failed to load setup for vf_nr %d\n", + vf_nr); + return ret; + } + + ret = adf_mstate_mgr_init_from_remote(vfmig->mstate_mgr, + mdev->state + mdev->remote_setup_size, + mdev->state_size - mdev->remote_setup_size, + NULL, NULL); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Invalid state for vf_nr %d\n", + vf_nr); + return ret; + } + + ret = adf_gen4_vfmig_load_generic(accel_dev, vf_nr); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to load general state for vf_nr %d\n", vf_nr); + return ret; + } + + ret = adf_gen4_vfmig_load_misc(accel_dev, vf_nr); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to load misc bar state for vf_nr %d\n", vf_nr); + return ret; + } + + ret = adf_gen4_vfmig_load_etr(accel_dev, vf_nr); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to load etr bar state for vf_nr %d\n", vf_nr); + return ret; + } + + return 0; +} + +static int adf_gen4_vfmig_save_setup(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + struct adf_accel_vf_info *vf_info; + struct adf_gen4_vfmig *vfmig; + u32 vf_nr = mdev->vf_id; + int ret; + + vf_info = &accel_dev->pf.vf_info[vf_nr]; + vfmig = vf_info->mig_priv; + + if (mdev->setup_size) + return 0; + + adf_mstate_mgr_init(vfmig->mstate_mgr, mdev->state, mdev->state_size); + if (!adf_mstate_preamble_add(vfmig->mstate_mgr)) + return -EINVAL; + + ret = adf_gen4_vfmig_save_config(accel_dev, mdev->vf_id); + if (ret) + return ret; + + adf_mstate_preamble_update(vfmig->mstate_mgr); + mdev->setup_size = adf_mstate_state_size(vfmig->mstate_mgr); + + return 0; +} + +static int adf_gen4_vfmig_load_setup(struct qat_mig_dev *mdev, int len) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + struct adf_accel_vf_info *vf_info; + struct adf_gen4_vfmig *vfmig; + u32 vf_nr = mdev->vf_id; + u32 setup_size; + int ret; + + vf_info = &accel_dev->pf.vf_info[vf_nr]; + vfmig = vf_info->mig_priv; + + if (mdev->remote_setup_size) + return 0; + + if (len < sizeof(struct adf_mstate_preh)) + return -EAGAIN; + + adf_mstate_mgr_init(vfmig->mstate_mgr, mdev->state, mdev->state_size); + setup_size = adf_mstate_state_size_from_remote(vfmig->mstate_mgr); + if (setup_size > mdev->state_size) + return -EINVAL; + + if (len < setup_size) + return -EAGAIN; + + ret = adf_mstate_mgr_init_from_remote(vfmig->mstate_mgr, mdev->state, + setup_size, NULL, NULL); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Invalid setup for vf_nr %d\n", + vf_nr); + return ret; + } + + mdev->remote_setup_size = setup_size; + + ret = adf_gen4_vfmig_load_config(accel_dev, vf_nr); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to load config for vf_nr %d\n", vf_nr); + return ret; + } + + return 0; +} + +void adf_gen4_init_vf_mig_ops(struct qat_migdev_ops *vfmig_ops) +{ + vfmig_ops->init = adf_gen4_vfmig_init_device; + vfmig_ops->cleanup = adf_gen4_vfmig_cleanup_device; + vfmig_ops->reset = adf_gen4_vfmig_reset_device; + vfmig_ops->open = adf_gen4_vfmig_open_device; + vfmig_ops->close = adf_gen4_vfmig_close_device; + vfmig_ops->suspend = adf_gen4_vfmig_suspend_device; + vfmig_ops->resume = adf_gen4_vfmig_resume_device; + vfmig_ops->save_state = adf_gen4_vfmig_save_state; + vfmig_ops->load_state = adf_gen4_vfmig_load_state; + vfmig_ops->load_setup = adf_gen4_vfmig_load_setup; + vfmig_ops->save_setup = adf_gen4_vfmig_save_setup; +} +EXPORT_SYMBOL_GPL(adf_gen4_init_vf_mig_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_vf_mig.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_vf_mig.h new file mode 100644 index 000000000..72216d078 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_vf_mig.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2024 Intel Corporation */ +#ifndef ADF_GEN4_VF_MIG_H_ +#define ADF_GEN4_VF_MIG_H_ + +#include "adf_accel_devices.h" + +void adf_gen4_init_vf_mig_ops(struct qat_migdev_ops *vfmig_ops); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_pm.h b/drivers/crypto/intel/qat/qat_common/adf_gen6_pm.h new file mode 100644 index 000000000..4c0d576e8 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_pm.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ADF_GEN6_PM_H +#define ADF_GEN6_PM_H + +#include +#include + +struct adf_accel_dev; + +/* Power management */ +#define ADF_GEN6_PM_POLL_DELAY_US 20 +#define ADF_GEN6_PM_POLL_TIMEOUT_US USEC_PER_SEC +#define ADF_GEN6_PM_STATUS 0x50A00C +#define ADF_GEN6_PM_INTERRUPT 0x50A028 + +/* Power management source in ERRSOU2 and ERRMSK2 */ +#define ADF_GEN6_PM_SOU BIT(18) + +/* cpm_pm_interrupt bitfields */ +#define ADF_GEN6_PM_DRV_ACTIVE BIT(20) + +#define ADF_GEN6_PM_DEFAULT_IDLE_FILTER 0x6 + +/* cpm_pm_status bitfields */ +#define ADF_GEN6_PM_INIT_STATE BIT(21) +#define ADF_GEN6_PM_CPM_PM_STATE_MASK GENMASK(22, 20) + +/* fusectl0 bitfields */ +#define ADF_GEN6_PM_ENABLE_PM_MASK BIT(21) +#define ADF_GEN6_PM_ENABLE_PM_IDLE_MASK BIT(22) +#define ADF_GEN6_PM_ENABLE_DEEP_PM_IDLE_MASK BIT(23) + +/* cpm_pm_fw_init bitfields */ +#define ADF_GEN6_PM_IDLE_FILTER_MASK GENMASK(5, 3) +#define ADF_GEN6_PM_IDLE_ENABLE_MASK BIT(2) + +/* ssm_pm_enable bitfield */ +#define ADF_GEN6_PM_SSM_PM_ENABLE_MASK BIT(0) + +/* ssm_pm_domain_status bitfield */ +#define ADF_GEN6_PM_DOMAIN_POWERED_UP_MASK BIT(0) + +#ifdef CONFIG_DEBUG_FS +void adf_gen6_init_dev_pm_data(struct adf_accel_dev *accel_dev); +#else +static inline void adf_gen6_init_dev_pm_data(struct adf_accel_dev *accel_dev) +{ +} +#endif /* CONFIG_DEBUG_FS */ + +#endif /* ADF_GEN6_PM_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_pm_dbgfs.c b/drivers/crypto/intel/qat/qat_common/adf_gen6_pm_dbgfs.c new file mode 100644 index 000000000..603aefba0 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_pm_dbgfs.c @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2025 Intel Corporation */ +#include +#include +#include + +#include "adf_admin.h" +#include "adf_common_drv.h" +#include "adf_gen6_pm.h" +#include "adf_pm_dbgfs_utils.h" +#include "icp_qat_fw_init_admin.h" + +#define PM_INFO_REGSET_ENTRY(_reg_, _field_) \ + PM_INFO_REGSET_ENTRY_MASK(_reg_, _field_, ADF_GEN6_PM_##_field_##_MASK) + +static struct pm_status_row pm_fuse_rows[] = { + PM_INFO_REGSET_ENTRY(fusectl0, ENABLE_PM), + PM_INFO_REGSET_ENTRY(fusectl0, ENABLE_PM_IDLE), + PM_INFO_REGSET_ENTRY(fusectl0, ENABLE_DEEP_PM_IDLE), +}; + +static struct pm_status_row pm_info_rows[] = { + PM_INFO_REGSET_ENTRY(pm.status, CPM_PM_STATE), + PM_INFO_REGSET_ENTRY(pm.fw_init, IDLE_ENABLE), + PM_INFO_REGSET_ENTRY(pm.fw_init, IDLE_FILTER), +}; + +static struct pm_status_row pm_ssm_rows[] = { + PM_INFO_REGSET_ENTRY(ssm.pm_enable, SSM_PM_ENABLE), + PM_INFO_REGSET_ENTRY(ssm.pm_domain_status, DOMAIN_POWERED_UP), +}; + +static struct pm_status_row pm_csrs_rows[] = { + PM_INFO_REGSET_ENTRY32(pm.fw_init, CPM_PM_FW_INIT), + PM_INFO_REGSET_ENTRY32(pm.status, CPM_PM_STATUS), +}; + +static_assert(sizeof(struct icp_qat_fw_init_admin_pm_info) < PAGE_SIZE); + +static ssize_t adf_gen6_print_pm_status(struct adf_accel_dev *accel_dev, + char __user *buf, size_t count, + loff_t *pos) +{ + void __iomem *pmisc = adf_get_pmisc_base(accel_dev); + struct icp_qat_fw_init_admin_pm_info *pm_info; + dma_addr_t p_state_addr; + u32 *pm_info_regs; + size_t len = 0; + char *pm_kv; + u32 val; + int ret; + + pm_info = kzalloc(PAGE_SIZE, GFP_KERNEL); + if (!pm_info) + return -ENOMEM; + + pm_kv = kzalloc(PAGE_SIZE, GFP_KERNEL); + if (!pm_kv) { + kfree(pm_info); + return -ENOMEM; + } + + p_state_addr = dma_map_single(&GET_DEV(accel_dev), pm_info, PAGE_SIZE, + DMA_FROM_DEVICE); + ret = dma_mapping_error(&GET_DEV(accel_dev), p_state_addr); + if (ret) + goto out_free; + + /* Query power management information from QAT FW */ + ret = adf_get_pm_info(accel_dev, p_state_addr, PAGE_SIZE); + dma_unmap_single(&GET_DEV(accel_dev), p_state_addr, PAGE_SIZE, + DMA_FROM_DEVICE); + if (ret) + goto out_free; + + pm_info_regs = (u32 *)pm_info; + + /* Fuse control register */ + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, + "----------- PM Fuse info ---------\n"); + len += adf_pm_scnprint_table_lower_keys(&pm_kv[len], pm_fuse_rows, + pm_info_regs, PAGE_SIZE - len, + ARRAY_SIZE(pm_fuse_rows)); + + /* Power management */ + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, + "----------- PM Info --------------\n"); + + len += adf_pm_scnprint_table_lower_keys(&pm_kv[len], pm_info_rows, + pm_info_regs, PAGE_SIZE - len, + ARRAY_SIZE(pm_info_rows)); + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "pm_mode: ACTIVE\n"); + + /* Shared Slice Module */ + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, + "----------- SSM_PM Info ----------\n"); + len += adf_pm_scnprint_table_lower_keys(&pm_kv[len], pm_ssm_rows, + pm_info_regs, PAGE_SIZE - len, + ARRAY_SIZE(pm_ssm_rows)); + + /* Control status register content */ + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, + "----------- HW PM CSRs -----------\n"); + len += adf_pm_scnprint_table_upper_keys(&pm_kv[len], pm_csrs_rows, + pm_info_regs, PAGE_SIZE - len, + ARRAY_SIZE(pm_csrs_rows)); + + val = ADF_CSR_RD(pmisc, ADF_GEN6_PM_INTERRUPT); + len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "CPM_PM_INTERRUPT: %#x\n", val); + ret = simple_read_from_buffer(buf, count, pos, pm_kv, len); + +out_free: + kfree(pm_info); + kfree(pm_kv); + + return ret; +} + +void adf_gen6_init_dev_pm_data(struct adf_accel_dev *accel_dev) +{ + accel_dev->power_management.print_pm_status = adf_gen6_print_pm_status; + accel_dev->power_management.present = true; +} +EXPORT_SYMBOL_GPL(adf_gen6_init_dev_pm_data); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.c b/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.c new file mode 100644 index 000000000..967253082 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.c @@ -0,0 +1,818 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2025 Intel Corporation */ +#include +#include + +#include "adf_common_drv.h" +#include "adf_gen6_ras.h" +#include "adf_sysfs_ras_counters.h" + +static void enable_errsou_reporting(void __iomem *csr) +{ + /* Enable correctable error reporting in ERRSOU0 */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK0, 0); + + /* Enable uncorrectable error reporting in ERRSOU1 */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK1, 0); + + /* + * Enable uncorrectable error reporting in ERRSOU2 + * but disable PM interrupt by default + */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK2, ADF_GEN6_ERRSOU2_PM_INT_BIT); + + /* Enable uncorrectable error reporting in ERRSOU3 */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK3, 0); +} + +static void enable_ae_error_reporting(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ae_mask = GET_HW_DATA(accel_dev)->ae_mask; + + /* Enable acceleration engine correctable error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_HIAECORERRLOGENABLE_CPP0, ae_mask); + + /* Enable acceleration engine uncorrectable error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_HIAEUNCERRLOGENABLE_CPP0, ae_mask); +} + +static void enable_cpp_error_reporting(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + /* Enable HI CPP agents command parity error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_HICPPAGENTCMDPARERRLOGENABLE, + ADF_6XXX_HICPPAGENTCMDPARERRLOG_MASK); + + ADF_CSR_WR(csr, ADF_GEN6_CPP_CFC_ERR_CTRL, ADF_GEN6_CPP_CFC_ERR_CTRL_MASK); +} + +static void enable_ti_ri_error_reporting(void __iomem *csr) +{ + u32 reg, mask; + + /* Enable RI memory error reporting */ + mask = ADF_GEN6_RIMEM_PARERR_FATAL_MASK | ADF_GEN6_RIMEM_PARERR_CERR_MASK; + ADF_CSR_WR(csr, ADF_GEN6_RI_MEM_PAR_ERR_EN0, mask); + + /* Enable IOSF primary command parity error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_RIMISCCTL, ADF_GEN6_RIMISCSTS_BIT); + + /* Enable TI internal memory parity error reporting */ + reg = ADF_CSR_RD(csr, ADF_GEN6_TI_CI_PAR_ERR_MASK); + reg &= ~ADF_GEN6_TI_CI_PAR_STS_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TI_CI_PAR_ERR_MASK, reg); + + reg = ADF_CSR_RD(csr, ADF_GEN6_TI_PULL0FUB_PAR_ERR_MASK); + reg &= ~ADF_GEN6_TI_PULL0FUB_PAR_STS_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TI_PULL0FUB_PAR_ERR_MASK, reg); + + reg = ADF_CSR_RD(csr, ADF_GEN6_TI_PUSHFUB_PAR_ERR_MASK); + reg &= ~ADF_GEN6_TI_PUSHFUB_PAR_STS_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TI_PUSHFUB_PAR_ERR_MASK, reg); + + reg = ADF_CSR_RD(csr, ADF_GEN6_TI_CD_PAR_ERR_MASK); + reg &= ~ADF_GEN6_TI_CD_PAR_STS_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TI_CD_PAR_ERR_MASK, reg); + + reg = ADF_CSR_RD(csr, ADF_GEN6_TI_TRNSB_PAR_ERR_MASK); + reg &= ~ADF_GEN6_TI_TRNSB_PAR_STS_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TI_TRNSB_PAR_ERR_MASK, reg); + + /* Enable error handling in RI, TI CPP interface control registers */ + ADF_CSR_WR(csr, ADF_GEN6_RICPPINTCTL, ADF_GEN6_RICPPINTCTL_MASK); + ADF_CSR_WR(csr, ADF_GEN6_TICPPINTCTL, ADF_GEN6_TICPPINTCTL_MASK); + + /* + * Enable error detection and reporting in TIMISCSTS + * with bits 1, 2 and 30 value preserved + */ + reg = ADF_CSR_RD(csr, ADF_GEN6_TIMISCCTL); + reg &= ADF_GEN6_TIMSCCTL_RELAY_MASK; + reg |= ADF_GEN6_TIMISCCTL_BIT; + ADF_CSR_WR(csr, ADF_GEN6_TIMISCCTL, reg); +} + +static void enable_ssm_error_reporting(struct adf_accel_dev *accel_dev, + void __iomem *csr) +{ + /* Enable SSM interrupts */ + ADF_CSR_WR(csr, ADF_GEN6_INTMASKSSM, 0); +} + +static void adf_gen6_enable_ras(struct adf_accel_dev *accel_dev) +{ + void __iomem *csr = adf_get_pmisc_base(accel_dev); + + enable_errsou_reporting(csr); + enable_ae_error_reporting(accel_dev, csr); + enable_cpp_error_reporting(accel_dev, csr); + enable_ti_ri_error_reporting(csr); + enable_ssm_error_reporting(accel_dev, csr); +} + +static void disable_errsou_reporting(void __iomem *csr) +{ + u32 val; + + /* Disable correctable error reporting in ERRSOU0 */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK0, ADF_GEN6_ERRSOU0_MASK); + + /* Disable uncorrectable error reporting in ERRSOU1 */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK1, ADF_GEN6_ERRMSK1_MASK); + + /* Disable uncorrectable error reporting in ERRSOU2 */ + val = ADF_CSR_RD(csr, ADF_GEN6_ERRMSK2); + val |= ADF_GEN6_ERRSOU2_DIS_MASK; + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK2, val); + + /* Disable uncorrectable error reporting in ERRSOU3 */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK3, ADF_GEN6_ERRSOU3_DIS_MASK); +} + +static void disable_ae_error_reporting(void __iomem *csr) +{ + /* Disable acceleration engine correctable error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_HIAECORERRLOGENABLE_CPP0, 0); + + /* Disable acceleration engine uncorrectable error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_HIAEUNCERRLOGENABLE_CPP0, 0); +} + +static void disable_cpp_error_reporting(void __iomem *csr) +{ + /* Disable HI CPP agents command parity error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_HICPPAGENTCMDPARERRLOGENABLE, 0); + + ADF_CSR_WR(csr, ADF_GEN6_CPP_CFC_ERR_CTRL, ADF_GEN6_CPP_CFC_ERR_CTRL_DIS_MASK); +} + +static void disable_ti_ri_error_reporting(void __iomem *csr) +{ + u32 reg; + + /* Disable RI memory error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_RI_MEM_PAR_ERR_EN0, 0); + + /* Disable IOSF primary command parity error reporting */ + reg = ADF_CSR_RD(csr, ADF_GEN6_RIMISCCTL); + reg &= ~ADF_GEN6_RIMISCSTS_BIT; + ADF_CSR_WR(csr, ADF_GEN6_RIMISCCTL, reg); + + /* Disable TI internal memory parity error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_TI_CI_PAR_ERR_MASK, ADF_GEN6_TI_CI_PAR_STS_MASK); + ADF_CSR_WR(csr, ADF_GEN6_TI_PULL0FUB_PAR_ERR_MASK, ADF_GEN6_TI_PULL0FUB_PAR_STS_MASK); + ADF_CSR_WR(csr, ADF_GEN6_TI_PUSHFUB_PAR_ERR_MASK, ADF_GEN6_TI_PUSHFUB_PAR_STS_MASK); + ADF_CSR_WR(csr, ADF_GEN6_TI_CD_PAR_ERR_MASK, ADF_GEN6_TI_CD_PAR_STS_MASK); + ADF_CSR_WR(csr, ADF_GEN6_TI_TRNSB_PAR_ERR_MASK, ADF_GEN6_TI_TRNSB_PAR_STS_MASK); + + /* Disable error handling in RI, TI CPP interface control registers */ + reg = ADF_CSR_RD(csr, ADF_GEN6_RICPPINTCTL); + reg &= ~ADF_GEN6_RICPPINTCTL_MASK; + ADF_CSR_WR(csr, ADF_GEN6_RICPPINTCTL, reg); + + reg = ADF_CSR_RD(csr, ADF_GEN6_TICPPINTCTL); + reg &= ~ADF_GEN6_TICPPINTCTL_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TICPPINTCTL, reg); + + /* + * Disable error detection and reporting in TIMISCSTS + * with bits 1, 2 and 30 value preserved + */ + reg = ADF_CSR_RD(csr, ADF_GEN6_TIMISCCTL); + reg &= ADF_GEN6_TIMSCCTL_RELAY_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TIMISCCTL, reg); +} + +static void disable_ssm_error_reporting(void __iomem *csr) +{ + /* Disable SSM interrupts */ + ADF_CSR_WR(csr, ADF_GEN6_INTMASKSSM, ADF_GEN6_INTMASKSSM_MASK); +} + +static void adf_gen6_disable_ras(struct adf_accel_dev *accel_dev) +{ + void __iomem *csr = adf_get_pmisc_base(accel_dev); + + disable_errsou_reporting(csr); + disable_ae_error_reporting(csr); + disable_cpp_error_reporting(csr); + disable_ti_ri_error_reporting(csr); + disable_ssm_error_reporting(csr); +} + +static void adf_gen6_process_errsou0(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ae, errsou; + + ae = ADF_CSR_RD(csr, ADF_GEN6_HIAECORERRLOG_CPP0); + ae &= GET_HW_DATA(accel_dev)->ae_mask; + + dev_warn(&GET_DEV(accel_dev), "Correctable error detected: %#x\n", ae); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_CORR); + + /* Clear interrupt from ERRSOU0 */ + ADF_CSR_WR(csr, ADF_GEN6_HIAECORERRLOG_CPP0, ae); + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU0); + if (errsou & ADF_GEN6_ERRSOU0_MASK) + dev_warn(&GET_DEV(accel_dev), "errsou0 still set: %#x\n", errsou); +} + +static void adf_handle_cpp_ae_unc(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 ae; + + if (!(errsou & ADF_GEN6_ERRSOU1_CPP0_MEUNC_BIT)) + return; + + ae = ADF_CSR_RD(csr, ADF_GEN6_HIAEUNCERRLOG_CPP0); + ae &= GET_HW_DATA(accel_dev)->ae_mask; + if (ae) { + dev_err(&GET_DEV(accel_dev), "Uncorrectable error detected: %#x\n", ae); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_HIAEUNCERRLOG_CPP0, ae); + } +} + +static void adf_handle_cpp_cmd_par_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 cmd_par_err; + + if (!(errsou & ADF_GEN6_ERRSOU1_CPP_CMDPARERR_BIT)) + return; + + cmd_par_err = ADF_CSR_RD(csr, ADF_GEN6_HICPPAGENTCMDPARERRLOG); + cmd_par_err &= ADF_6XXX_HICPPAGENTCMDPARERRLOG_MASK; + if (cmd_par_err) { + dev_err(&GET_DEV(accel_dev), "HI CPP agent command parity error: %#x\n", + cmd_par_err); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_HICPPAGENTCMDPARERRLOG, cmd_par_err); + } +} + +static void adf_handle_ri_mem_par_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 rimem_parerr_sts; + + if (!(errsou & ADF_GEN6_ERRSOU1_RIMEM_PARERR_STS_BIT)) + return; + + rimem_parerr_sts = ADF_CSR_RD(csr, ADF_GEN6_RIMEM_PARERR_STS); + rimem_parerr_sts &= ADF_GEN6_RIMEM_PARERR_CERR_MASK | + ADF_GEN6_RIMEM_PARERR_FATAL_MASK; + if (rimem_parerr_sts & ADF_GEN6_RIMEM_PARERR_CERR_MASK) { + dev_err(&GET_DEV(accel_dev), "RI memory parity correctable error: %#x\n", + rimem_parerr_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_CORR); + } + + if (rimem_parerr_sts & ADF_GEN6_RIMEM_PARERR_FATAL_MASK) { + dev_err(&GET_DEV(accel_dev), "RI memory parity fatal error: %#x\n", + rimem_parerr_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + } + + ADF_CSR_WR(csr, ADF_GEN6_RIMEM_PARERR_STS, rimem_parerr_sts); +} + +static void adf_handle_ti_ci_par_sts(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ti_ci_par_sts; + + ti_ci_par_sts = ADF_CSR_RD(csr, ADF_GEN6_TI_CI_PAR_STS); + ti_ci_par_sts &= ADF_GEN6_TI_CI_PAR_STS_MASK; + if (ti_ci_par_sts) { + dev_err(&GET_DEV(accel_dev), "TI memory parity error: %#x\n", ti_ci_par_sts); + ADF_CSR_WR(csr, ADF_GEN6_TI_CI_PAR_STS, ti_ci_par_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + } +} + +static void adf_handle_ti_pullfub_par_sts(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ti_pullfub_par_sts; + + ti_pullfub_par_sts = ADF_CSR_RD(csr, ADF_GEN6_TI_PULL0FUB_PAR_STS); + ti_pullfub_par_sts &= ADF_GEN6_TI_PULL0FUB_PAR_STS_MASK; + if (ti_pullfub_par_sts) { + dev_err(&GET_DEV(accel_dev), "TI pull parity error: %#x\n", ti_pullfub_par_sts); + ADF_CSR_WR(csr, ADF_GEN6_TI_PULL0FUB_PAR_STS, ti_pullfub_par_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + } +} + +static void adf_handle_ti_pushfub_par_sts(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ti_pushfub_par_sts; + + ti_pushfub_par_sts = ADF_CSR_RD(csr, ADF_GEN6_TI_PUSHFUB_PAR_STS); + ti_pushfub_par_sts &= ADF_GEN6_TI_PUSHFUB_PAR_STS_MASK; + if (ti_pushfub_par_sts) { + dev_err(&GET_DEV(accel_dev), "TI push parity error: %#x\n", ti_pushfub_par_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_TI_PUSHFUB_PAR_STS, ti_pushfub_par_sts); + } +} + +static void adf_handle_ti_cd_par_sts(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ti_cd_par_sts; + + ti_cd_par_sts = ADF_CSR_RD(csr, ADF_GEN6_TI_CD_PAR_STS); + ti_cd_par_sts &= ADF_GEN6_TI_CD_PAR_STS_MASK; + if (ti_cd_par_sts) { + dev_err(&GET_DEV(accel_dev), "TI CD parity error: %#x\n", ti_cd_par_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_TI_CD_PAR_STS, ti_cd_par_sts); + } +} + +static void adf_handle_ti_trnsb_par_sts(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ti_trnsb_par_sts; + + ti_trnsb_par_sts = ADF_CSR_RD(csr, ADF_GEN6_TI_TRNSB_PAR_STS); + ti_trnsb_par_sts &= ADF_GEN6_TI_TRNSB_PAR_STS_MASK; + if (ti_trnsb_par_sts) { + dev_err(&GET_DEV(accel_dev), "TI TRNSB parity error: %#x\n", ti_trnsb_par_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_TI_TRNSB_PAR_STS, ti_trnsb_par_sts); + } +} + +static void adf_handle_iosfp_cmd_parerr(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 rimiscsts; + + rimiscsts = ADF_CSR_RD(csr, ADF_GEN6_RIMISCSTS); + rimiscsts &= ADF_GEN6_RIMISCSTS_BIT; + if (rimiscsts) { + dev_err(&GET_DEV(accel_dev), "Command parity error detected on IOSFP: %#x\n", + rimiscsts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_RIMISCSTS, rimiscsts); + } +} + +static void adf_handle_ti_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU1_TIMEM_PARERR_STS_BIT)) + return; + + adf_handle_ti_ci_par_sts(accel_dev, csr); + adf_handle_ti_pullfub_par_sts(accel_dev, csr); + adf_handle_ti_pushfub_par_sts(accel_dev, csr); + adf_handle_ti_cd_par_sts(accel_dev, csr); + adf_handle_ti_trnsb_par_sts(accel_dev, csr); + adf_handle_iosfp_cmd_parerr(accel_dev, csr); +} + +static void adf_handle_sfi_cmd_parerr(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU1_SFICMD_PARERR_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), + "Command parity error detected on streaming fabric interface\n"); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); +} + +static void adf_gen6_process_errsou1(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + adf_handle_cpp_ae_unc(accel_dev, csr, errsou); + adf_handle_cpp_cmd_par_err(accel_dev, csr, errsou); + adf_handle_ri_mem_par_err(accel_dev, csr, errsou); + adf_handle_ti_err(accel_dev, csr, errsou); + adf_handle_sfi_cmd_parerr(accel_dev, csr, errsou); + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU1); + if (errsou & ADF_GEN6_ERRSOU1_MASK) + dev_warn(&GET_DEV(accel_dev), "errsou1 still set: %#x\n", errsou); +} + +static void adf_handle_cerrssmsh(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 reg; + + reg = ADF_CSR_RD(csr, ADF_GEN6_CERRSSMSH); + reg &= ADF_GEN6_CERRSSMSH_ERROR_BIT; + if (reg) { + dev_warn(&GET_DEV(accel_dev), + "Correctable error on ssm shared memory: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_CORR); + ADF_CSR_WR(csr, ADF_GEN6_CERRSSMSH, reg); + } +} + +static void adf_handle_uerrssmsh(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN6_IAINTSTATSSM_SH_ERR_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_UERRSSMSH); + reg &= ADF_GEN6_UERRSSMSH_MASK; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "Fatal error on ssm shared memory: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_UERRSSMSH, reg); + } +} + +static void adf_handle_pperr_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN6_IAINTSTATSSM_PPERR_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_PPERR); + reg &= ADF_GEN6_PPERR_MASK; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "Fatal push or pull data error: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_PPERR, reg); + } +} + +static void adf_handle_scmpar_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN6_IAINTSTATSSM_SCMPAR_ERR_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_SSM_FERR_STATUS); + reg &= ADF_GEN6_SCM_PAR_ERR_MASK; + if (reg) { + dev_err(&GET_DEV(accel_dev), "Fatal error on SCM: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_SSM_FERR_STATUS, reg); + } +} + +static void adf_handle_cpppar_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN6_IAINTSTATSSM_CPPPAR_ERR_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_SSM_FERR_STATUS); + reg &= ADF_GEN6_CPP_PAR_ERR_MASK; + if (reg) { + dev_err(&GET_DEV(accel_dev), "Fatal error on CPP: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_SSM_FERR_STATUS, reg); + } +} + +static void adf_handle_rfpar_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN6_IAINTSTATSSM_RFPAR_ERR_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_SSM_FERR_STATUS); + reg &= ADF_GEN6_RF_PAR_ERR_MASK; + if (reg) { + dev_err(&GET_DEV(accel_dev), "Fatal error on RF Parity: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_SSM_FERR_STATUS, reg); + } +} + +static void adf_handle_unexp_cpl_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN6_IAINTSTATSSM_UNEXP_CPL_ERR_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_SSM_FERR_STATUS); + reg &= ADF_GEN6_UNEXP_CPL_ERR_MASK; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "Fatal error for AXI unexpected tag/length: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_SSM_FERR_STATUS, reg); + } +} + +static void adf_handle_iaintstatssm(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 iastatssm = ADF_CSR_RD(csr, ADF_GEN6_IAINTSTATSSM); + + iastatssm &= ADF_GEN6_IAINTSTATSSM_MASK; + if (!iastatssm) + return; + + adf_handle_uerrssmsh(accel_dev, csr, iastatssm); + adf_handle_pperr_err(accel_dev, csr, iastatssm); + adf_handle_scmpar_err(accel_dev, csr, iastatssm); + adf_handle_cpppar_err(accel_dev, csr, iastatssm); + adf_handle_rfpar_err(accel_dev, csr, iastatssm); + adf_handle_unexp_cpl_err(accel_dev, csr, iastatssm); + + ADF_CSR_WR(csr, ADF_GEN6_IAINTSTATSSM, iastatssm); +} + +static void adf_handle_ssm(struct adf_accel_dev *accel_dev, void __iomem *csr, u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU2_SSM_ERR_BIT)) + return; + + adf_handle_cerrssmsh(accel_dev, csr); + adf_handle_iaintstatssm(accel_dev, csr); +} + +static void adf_handle_cpp_cfc_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 reg; + + if (!(errsou & ADF_GEN6_ERRSOU2_CPP_CFC_ERR_STATUS_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_CPP_CFC_ERR_STATUS); + if (reg & ADF_GEN6_CPP_CFC_ERR_STATUS_DATAPAR_BIT) { + dev_err(&GET_DEV(accel_dev), "CPP_CFC_ERR: data parity: %#x", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + } + + if (reg & ADF_GEN6_CPP_CFC_ERR_STATUS_CMDPAR_BIT) { + dev_err(&GET_DEV(accel_dev), "CPP_CFC_ERR: command parity: %#x", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + } + + if (reg & ADF_GEN6_CPP_CFC_FATAL_ERR_BIT) { + dev_err(&GET_DEV(accel_dev), "CPP_CFC_ERR: errors: %#x", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + } + + ADF_CSR_WR(csr, ADF_GEN6_CPP_CFC_ERR_STATUS_CLR, + ADF_GEN6_CPP_CFC_ERR_STATUS_CLR_MASK); +} + +static void adf_gen6_process_errsou2(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + adf_handle_ssm(accel_dev, csr, errsou); + adf_handle_cpp_cfc_err(accel_dev, csr, errsou); + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU2); + if (errsou & ADF_GEN6_ERRSOU2_MASK) + dev_warn(&GET_DEV(accel_dev), "errsou2 still set: %#x\n", errsou); +} + +static void adf_handle_timiscsts(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 timiscsts; + + if (!(errsou & ADF_GEN6_ERRSOU3_TIMISCSTS_BIT)) + return; + + timiscsts = ADF_CSR_RD(csr, ADF_GEN6_TIMISCSTS); + if (timiscsts) { + dev_err(&GET_DEV(accel_dev), "Fatal error in transmit interface: %#x\n", + timiscsts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + } +} + +static void adf_handle_ricppintsts(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 ricppintsts; + + if (!(errsou & ADF_GEN6_ERRSOU3_RICPPINTSTS_MASK)) + return; + + ricppintsts = ADF_CSR_RD(csr, ADF_GEN6_RICPPINTSTS); + ricppintsts &= ADF_GEN6_RICPPINTSTS_MASK; + if (ricppintsts) { + dev_err(&GET_DEV(accel_dev), "RI push pull error: %#x\n", ricppintsts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_RICPPINTSTS, ricppintsts); + } +} + +static void adf_handle_ticppintsts(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 ticppintsts; + + if (!(errsou & ADF_GEN6_ERRSOU3_TICPPINTSTS_MASK)) + return; + + ticppintsts = ADF_CSR_RD(csr, ADF_GEN6_TICPPINTSTS); + ticppintsts &= ADF_GEN6_TICPPINTSTS_MASK; + if (ticppintsts) { + dev_err(&GET_DEV(accel_dev), "TI push pull error: %#x\n", ticppintsts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_TICPPINTSTS, ticppintsts); + } +} + +static void adf_handle_atufaultstatus(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 max_rp_num = GET_HW_DATA(accel_dev)->num_banks; + u32 atufaultstatus; + u32 i; + + if (!(errsou & ADF_GEN6_ERRSOU3_ATUFAULTSTATUS_BIT)) + return; + + for (i = 0; i < max_rp_num; i++) { + atufaultstatus = ADF_CSR_RD(csr, ADF_GEN6_ATUFAULTSTATUS(i)); + + atufaultstatus &= ADF_GEN6_ATUFAULTSTATUS_BIT; + if (atufaultstatus) { + dev_err(&GET_DEV(accel_dev), "Ring pair (%u) ATU detected fault: %#x\n", i, + atufaultstatus); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_ATUFAULTSTATUS(i), atufaultstatus); + } + } +} + +static void adf_handle_rlterror(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 rlterror; + + if (!(errsou & ADF_GEN6_ERRSOU3_RLTERROR_BIT)) + return; + + rlterror = ADF_CSR_RD(csr, ADF_GEN6_RLT_ERRLOG); + rlterror &= ADF_GEN6_RLT_ERRLOG_MASK; + if (rlterror) { + dev_err(&GET_DEV(accel_dev), "Error in rate limiting block: %#x\n", rlterror); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_RLT_ERRLOG, rlterror); + } +} + +static void adf_handle_vflr(struct adf_accel_dev *accel_dev, void __iomem *csr, u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU3_VFLRNOTIFY_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), "Uncorrectable error in VF\n"); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); +} + +static void adf_handle_tc_vc_map_error(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU3_TC_VC_MAP_ERROR_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), "Violation of PCIe TC VC mapping\n"); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); +} + +static void adf_handle_pcie_devhalt(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU3_PCIE_DEVHALT_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), + "DEVHALT due to an error in an incoming transaction\n"); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); +} + +static void adf_handle_pg_req_devhalt(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU3_PG_REQ_DEVHALT_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), + "Error due to response failure in response to a page request\n"); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); +} + +static void adf_handle_xlt_cpl_devhalt(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU3_XLT_CPL_DEVHALT_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), "Error status for a address translation request\n"); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); +} + +static void adf_handle_ti_int_err_devhalt(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU3_TI_INT_ERR_DEVHALT_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), "DEVHALT due to a TI internal memory error\n"); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); +} + +static void adf_gen6_process_errsou3(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + adf_handle_timiscsts(accel_dev, csr, errsou); + adf_handle_ricppintsts(accel_dev, csr, errsou); + adf_handle_ticppintsts(accel_dev, csr, errsou); + adf_handle_atufaultstatus(accel_dev, csr, errsou); + adf_handle_rlterror(accel_dev, csr, errsou); + adf_handle_vflr(accel_dev, csr, errsou); + adf_handle_tc_vc_map_error(accel_dev, csr, errsou); + adf_handle_pcie_devhalt(accel_dev, csr, errsou); + adf_handle_pg_req_devhalt(accel_dev, csr, errsou); + adf_handle_xlt_cpl_devhalt(accel_dev, csr, errsou); + adf_handle_ti_int_err_devhalt(accel_dev, csr, errsou); + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU3); + if (errsou & ADF_GEN6_ERRSOU3_MASK) + dev_warn(&GET_DEV(accel_dev), "errsou3 still set: %#x\n", errsou); +} + +static void adf_gen6_is_reset_required(struct adf_accel_dev *accel_dev, void __iomem *csr, + bool *reset_required) +{ + u8 reset, dev_state; + u32 gensts; + + gensts = ADF_CSR_RD(csr, ADF_GEN6_GENSTS); + dev_state = FIELD_GET(ADF_GEN6_GENSTS_DEVICE_STATE_MASK, gensts); + reset = FIELD_GET(ADF_GEN6_GENSTS_RESET_TYPE_MASK, gensts); + if (dev_state == ADF_GEN6_GENSTS_DEVHALT && reset == ADF_GEN6_GENSTS_PFLR) { + *reset_required = true; + return; + } + + if (reset == ADF_GEN6_GENSTS_COLD_RESET) + dev_err(&GET_DEV(accel_dev), "Fatal error, cold reset required\n"); + + *reset_required = false; +} + +static bool adf_gen6_handle_interrupt(struct adf_accel_dev *accel_dev, bool *reset_required) +{ + void __iomem *csr = adf_get_pmisc_base(accel_dev); + bool handled = false; + u32 errsou; + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU0); + if (errsou & ADF_GEN6_ERRSOU0_MASK) { + adf_gen6_process_errsou0(accel_dev, csr); + handled = true; + } + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU1); + if (errsou & ADF_GEN6_ERRSOU1_MASK) { + adf_gen6_process_errsou1(accel_dev, csr, errsou); + handled = true; + } + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU2); + if (errsou & ADF_GEN6_ERRSOU2_MASK) { + adf_gen6_process_errsou2(accel_dev, csr, errsou); + handled = true; + } + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU3); + if (errsou & ADF_GEN6_ERRSOU3_MASK) { + adf_gen6_process_errsou3(accel_dev, csr, errsou); + handled = true; + } + + adf_gen6_is_reset_required(accel_dev, csr, reset_required); + + return handled; +} + +void adf_gen6_init_ras_ops(struct adf_ras_ops *ras_ops) +{ + ras_ops->enable_ras_errors = adf_gen6_enable_ras; + ras_ops->disable_ras_errors = adf_gen6_disable_ras; + ras_ops->handle_interrupt = adf_gen6_handle_interrupt; +} +EXPORT_SYMBOL_GPL(adf_gen6_init_ras_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.h b/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.h new file mode 100644 index 000000000..66ced271d --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.h @@ -0,0 +1,504 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ADF_GEN6_RAS_H_ +#define ADF_GEN6_RAS_H_ + +#include + +struct adf_ras_ops; + +/* Error source registers */ +#define ADF_GEN6_ERRSOU0 0x41A200 +#define ADF_GEN6_ERRSOU1 0x41A204 +#define ADF_GEN6_ERRSOU2 0x41A208 +#define ADF_GEN6_ERRSOU3 0x41A20C + +/* Error source mask registers */ +#define ADF_GEN6_ERRMSK0 0x41A210 +#define ADF_GEN6_ERRMSK1 0x41A214 +#define ADF_GEN6_ERRMSK2 0x41A218 +#define ADF_GEN6_ERRMSK3 0x41A21C + +/* ERRSOU0 Correctable error mask */ +#define ADF_GEN6_ERRSOU0_MASK BIT(0) + +#define ADF_GEN6_ERRSOU1_CPP0_MEUNC_BIT BIT(0) +#define ADF_GEN6_ERRSOU1_CPP_CMDPARERR_BIT BIT(1) +#define ADF_GEN6_ERRSOU1_RIMEM_PARERR_STS_BIT BIT(2) +#define ADF_GEN6_ERRSOU1_TIMEM_PARERR_STS_BIT BIT(3) +#define ADF_GEN6_ERRSOU1_SFICMD_PARERR_BIT BIT(4) + +#define ADF_GEN6_ERRSOU1_MASK ( \ + (ADF_GEN6_ERRSOU1_CPP0_MEUNC_BIT) | \ + (ADF_GEN6_ERRSOU1_CPP_CMDPARERR_BIT) | \ + (ADF_GEN6_ERRSOU1_RIMEM_PARERR_STS_BIT) | \ + (ADF_GEN6_ERRSOU1_TIMEM_PARERR_STS_BIT) | \ + (ADF_GEN6_ERRSOU1_SFICMD_PARERR_BIT)) + +#define ADF_GEN6_ERRMSK1_CPP0_MEUNC_BIT BIT(0) +#define ADF_GEN6_ERRMSK1_CPP_CMDPARERR_BIT BIT(1) +#define ADF_GEN6_ERRMSK1_RIMEM_PARERR_STS_BIT BIT(2) +#define ADF_GEN6_ERRMSK1_TIMEM_PARERR_STS_BIT BIT(3) +#define ADF_GEN6_ERRMSK1_IOSFCMD_PARERR_BIT BIT(4) + +#define ADF_GEN6_ERRMSK1_MASK ( \ + (ADF_GEN6_ERRMSK1_CPP0_MEUNC_BIT) | \ + (ADF_GEN6_ERRMSK1_CPP_CMDPARERR_BIT) | \ + (ADF_GEN6_ERRMSK1_RIMEM_PARERR_STS_BIT) | \ + (ADF_GEN6_ERRMSK1_TIMEM_PARERR_STS_BIT) | \ + (ADF_GEN6_ERRMSK1_IOSFCMD_PARERR_BIT)) + +/* HI AE Uncorrectable error log */ +#define ADF_GEN6_HIAEUNCERRLOG_CPP0 0x41A300 + +/* HI AE Uncorrectable error log enable */ +#define ADF_GEN6_HIAEUNCERRLOGENABLE_CPP0 0x41A320 + +/* HI AE Correctable error log */ +#define ADF_GEN6_HIAECORERRLOG_CPP0 0x41A308 + +/* HI AE Correctable error log enable */ +#define ADF_GEN6_HIAECORERRLOGENABLE_CPP0 0x41A318 + +/* HI CPP Agent Command parity error log */ +#define ADF_GEN6_HICPPAGENTCMDPARERRLOG 0x41A310 + +/* HI CPP Agent command parity error logging enable */ +#define ADF_GEN6_HICPPAGENTCMDPARERRLOGENABLE 0x41A314 + +#define ADF_6XXX_HICPPAGENTCMDPARERRLOG_MASK 0x1B + +/* RI Memory parity error status register */ +#define ADF_GEN6_RIMEM_PARERR_STS 0x41B128 + +/* RI Memory parity error reporting enable */ +#define ADF_GEN6_RI_MEM_PAR_ERR_EN0 0x41B12C + +/* + * RI Memory parity error mask + * BIT(4) - ri_tlq_phdr parity error + * BIT(5) - ri_tlq_pdata parity error + * BIT(6) - ri_tlq_nphdr parity error + * BIT(7) - ri_tlq_npdata parity error + * BIT(8) - ri_tlq_cplhdr parity error + * BIT(10) - BIT(13) - ri_tlq_cpldata[0:3] parity error + * BIT(19) - ri_cds_cmd_fifo parity error + * BIT(20) - ri_obc_ricpl_fifo parity error + * BIT(21) - ri_obc_tiricpl_fifo parity error + * BIT(22) - ri_obc_cppcpl_fifo parity error + * BIT(23) - ri_obc_pendcpl_fifo parity error + * BIT(24) - ri_cpp_cmd_fifo parity error + * BIT(25) - ri_cds_ticmd_fifo parity error + * BIT(26) - riti_cmd_fifo parity error + * BIT(27) - ri_int_msixtbl parity error + * BIT(28) - ri_int_imstbl parity error + * BIT(30) - ri_kpt_fuses parity error + */ +#define ADF_GEN6_RIMEM_PARERR_FATAL_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(4) | BIT(5) | BIT(6) | \ + BIT(7) | BIT(8) | BIT(18) | BIT(19) | BIT(20) | BIT(21) | \ + BIT(22) | BIT(23) | BIT(24) | BIT(25) | BIT(26) | BIT(27) | \ + BIT(28) | BIT(30)) + +#define ADF_GEN6_RIMEM_PARERR_CERR_MASK \ + (BIT(10) | BIT(11) | BIT(12) | BIT(13)) + +/* TI CI parity status */ +#define ADF_GEN6_TI_CI_PAR_STS 0x50060C + +/* TI CI parity reporting mask */ +#define ADF_GEN6_TI_CI_PAR_ERR_MASK 0x500608 + +/* + * TI CI parity status mask + * BIT(0) - CdCmdQ_sts patiry error status + * BIT(1) - CdDataQ_sts parity error status + * BIT(3) - CPP_SkidQ_sts parity error status + */ +#define ADF_GEN6_TI_CI_PAR_STS_MASK \ + (BIT(0) | BIT(1) | BIT(3)) + +/* TI PULLFUB parity status */ +#define ADF_GEN6_TI_PULL0FUB_PAR_STS 0x500618 + +/* TI PULLFUB parity error reporting mask */ +#define ADF_GEN6_TI_PULL0FUB_PAR_ERR_MASK 0x500614 + +/* + * TI PULLFUB parity status mask + * BIT(0) - TrnPullReqQ_sts parity status + * BIT(1) - TrnSharedDataQ_sts parity status + * BIT(2) - TrnPullReqDataQ_sts parity status + * BIT(4) - CPP_CiPullReqQ_sts parity status + * BIT(5) - CPP_TrnPullReqQ_sts parity status + * BIT(6) - CPP_PullidQ_sts parity status + * BIT(7) - CPP_WaitDataQ_sts parity status + * BIT(8) - CPP_CdDataQ_sts parity status + * BIT(9) - CPP_TrnDataQP0_sts parity status + * BIT(10) - BIT(11) - CPP_TrnDataQRF[00:01]_sts parity status + * BIT(12) - CPP_TrnDataQP1_sts parity status + * BIT(13) - BIT(14) - CPP_TrnDataQRF[10:11]_sts parity status + */ +#define ADF_GEN6_TI_PULL0FUB_PAR_STS_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(4) | BIT(5) | BIT(6) | BIT(7) | \ + BIT(8) | BIT(9) | BIT(10) | BIT(11) | BIT(12) | BIT(13) | BIT(14)) + +/* TI PUSHUB parity status */ +#define ADF_GEN6_TI_PUSHFUB_PAR_STS 0x500630 + +/* TI PUSHFUB parity error reporting mask */ +#define ADF_GEN6_TI_PUSHFUB_PAR_ERR_MASK 0x50062C + +/* + * TI PUSHUB parity status mask + * BIT(0) - SbPushReqQ_sts parity status + * BIT(1) - BIT(2) - SbPushDataQ[0:1]_sts parity status + * BIT(4) - CPP_CdPushReqQ_sts parity status + * BIT(5) - BIT(6) - CPP_CdPushDataQ[0:1]_sts parity status + * BIT(7) - CPP_SbPushReqQ_sts parity status + * BIT(8) - CPP_SbPushDataQP_sts parity status + * BIT(9) - BIT(10) - CPP_SbPushDataQRF[0:1]_sts parity status + */ +#define ADF_GEN6_TI_PUSHFUB_PAR_STS_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(4) | BIT(5) | \ + BIT(6) | BIT(7) | BIT(8) | BIT(9) | BIT(10)) + +/* TI CD parity status */ +#define ADF_GEN6_TI_CD_PAR_STS 0x50063C + +/* TI CD parity error mask */ +#define ADF_GEN6_TI_CD_PAR_ERR_MASK 0x500638 + +/* + * TI CD parity status mask + * BIT(0) - BIT(15) - CtxMdRam[0:15]_sts parity status + * BIT(16) - Leaf2ClusterRam_sts parity status + * BIT(17) - BIT(18) - Ring2LeafRam[0:1]_sts parity status + * BIT(19) - VirtualQ_sts parity status + * BIT(20) - DtRdQ_sts parity status + * BIT(21) - DtWrQ_sts parity status + * BIT(22) - RiCmdQ_sts parity status + * BIT(23) - BypassQ_sts parity status + * BIT(24) - DtRdQ_sc_sts parity status + * BIT(25) - DtWrQ_sc_sts parity status + */ +#define ADF_GEN6_TI_CD_PAR_STS_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4) | BIT(5) | BIT(6) | \ + BIT(7) | BIT(8) | BIT(9) | BIT(10) | BIT(11) | BIT(12) | BIT(13) | \ + BIT(14) | BIT(15) | BIT(16) | BIT(17) | BIT(18) | BIT(19) | BIT(20) | \ + BIT(21) | BIT(22) | BIT(23) | BIT(24) | BIT(25)) + +/* TI TRNSB parity status */ +#define ADF_GEN6_TI_TRNSB_PAR_STS 0x500648 + +/* TI TRNSB parity error reporting mask */ +#define ADF_GEN6_TI_TRNSB_PAR_ERR_MASK 0x500644 + +/* + * TI TRNSB parity status mask + * BIT(0) - TrnPHdrQP_sts parity status + * BIT(1) - TrnPHdrQRF_sts parity status + * BIT(2) - TrnPDataQP_sts parity status + * BIT(3) - BIT(6) - TrnPDataQRF[0:3]_sts parity status + * BIT(7) - TrnNpHdrQP_sts parity status + * BIT(8) - BIT(9) - TrnNpHdrQRF[0:1]_sts parity status + * BIT(10) - TrnCplHdrQ_sts parity status + * BIT(11) - TrnPutObsReqQ_sts parity status + * BIT(12) - TrnPushReqQ_sts parity status + * BIT(13) - SbSplitIdRam_sts parity status + * BIT(14) - SbReqCountQ_sts parity status + * BIT(15) - SbCplTrkRam_sts parity status + * BIT(16) - SbGetObsReqQ_sts parity status + * BIT(17) - SbEpochIdQ_sts parity status + * BIT(18) - SbAtCplHdrQ_sts parity status + * BIT(19) - SbAtCplDataQ_sts parity status + * BIT(20) - SbReqCountRam_sts parity status + * BIT(21) - SbAtCplHdrQ_sc_sts parity status + */ +#define ADF_GEN6_TI_TRNSB_PAR_STS_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4) | BIT(5) | BIT(6) | \ + BIT(7) | BIT(8) | BIT(9) | BIT(10) | BIT(11) | BIT(12) | \ + BIT(13) | BIT(14) | BIT(15) | BIT(16) | BIT(17) | BIT(18) | \ + BIT(19) | BIT(20) | BIT(21)) + +/* Status register to log misc error on RI */ +#define ADF_GEN6_RIMISCSTS 0x41B1B8 + +/* Status control register to log misc RI error */ +#define ADF_GEN6_RIMISCCTL 0x41B1BC + +/* + * ERRSOU2 bit mask + * BIT(0) - SSM Interrupt Mask + * BIT(1) - CFC on CPP. ORed of CFC Push error and Pull error + * BIT(2) - BIT(4) - CPP attention interrupts + * BIT(18) - PM interrupt + */ +#define ADF_GEN6_ERRSOU2_SSM_ERR_BIT BIT(0) +#define ADF_GEN6_ERRSOU2_CPP_CFC_ERR_STATUS_BIT BIT(1) +#define ADF_GEN6_ERRSOU2_CPP_CFC_ATT_INT_MASK \ + (BIT(2) | BIT(3) | BIT(4)) + +#define ADF_GEN6_ERRSOU2_PM_INT_BIT BIT(18) + +#define ADF_GEN6_ERRSOU2_MASK \ + (ADF_GEN6_ERRSOU2_SSM_ERR_BIT | \ + ADF_GEN6_ERRSOU2_CPP_CFC_ERR_STATUS_BIT) + +#define ADF_GEN6_ERRSOU2_DIS_MASK \ + (ADF_GEN6_ERRSOU2_SSM_ERR_BIT | \ + ADF_GEN6_ERRSOU2_CPP_CFC_ERR_STATUS_BIT | \ + ADF_GEN6_ERRSOU2_CPP_CFC_ATT_INT_MASK) + +#define ADF_GEN6_IAINTSTATSSM 0x28 + +/* IAINTSTATSSM error bit mask definitions */ +#define ADF_GEN6_IAINTSTATSSM_SH_ERR_BIT BIT(0) +#define ADF_GEN6_IAINTSTATSSM_PPERR_BIT BIT(2) +#define ADF_GEN6_IAINTSTATSSM_SCMPAR_ERR_BIT BIT(4) +#define ADF_GEN6_IAINTSTATSSM_CPPPAR_ERR_BIT BIT(5) +#define ADF_GEN6_IAINTSTATSSM_RFPAR_ERR_BIT BIT(6) +#define ADF_GEN6_IAINTSTATSSM_UNEXP_CPL_ERR_BIT BIT(7) + +#define ADF_GEN6_IAINTSTATSSM_MASK \ + (ADF_GEN6_IAINTSTATSSM_SH_ERR_BIT | \ + ADF_GEN6_IAINTSTATSSM_PPERR_BIT | \ + ADF_GEN6_IAINTSTATSSM_SCMPAR_ERR_BIT | \ + ADF_GEN6_IAINTSTATSSM_CPPPAR_ERR_BIT | \ + ADF_GEN6_IAINTSTATSSM_RFPAR_ERR_BIT | \ + ADF_GEN6_IAINTSTATSSM_UNEXP_CPL_ERR_BIT) + +#define ADF_GEN6_UERRSSMSH 0x18 + +/* + * UERRSSMSH error bit mask definitions + * + * BIT(0) - Indicates one uncorrectable error + * BIT(15) - Indicates multiple uncorrectable errors + * in device shared memory + */ +#define ADF_GEN6_UERRSSMSH_MASK (BIT(0) | BIT(15)) + +/* + * CERRSSMSH error bit + * BIT(0) - Indicates one correctable error + */ +#define ADF_GEN6_CERRSSMSH_ERROR_BIT (BIT(0) | BIT(15) | BIT(24)) +#define ADF_GEN6_CERRSSMSH 0x10 + +#define ADF_GEN6_INTMASKSSM 0x0 + +/* + * Error reporting mask in INTMASKSSM + * BIT(0) - Shared memory uncorrectable interrupt mask + * BIT(2) - PPERR interrupt mask + * BIT(4) - SCM parity error interrupt mask + * BIT(5) - CPP parity error interrupt mask + * BIT(6) - SHRAM RF parity error interrupt mask + * BIT(7) - AXI unexpected completion error mask + */ +#define ADF_GEN6_INTMASKSSM_MASK \ + (BIT(0) | BIT(2) | BIT(4) | BIT(5) | BIT(6) | BIT(7)) + +/* CPP push or pull error */ +#define ADF_GEN6_PPERR 0x8 + +#define ADF_GEN6_PPERR_MASK (BIT(0) | BIT(1)) + +/* + * SSM_FERR_STATUS error bit mask definitions + */ +#define ADF_GEN6_SCM_PAR_ERR_MASK BIT(5) +#define ADF_GEN6_CPP_PAR_ERR_MASK (BIT(0) | BIT(1) | BIT(2)) +#define ADF_GEN6_UNEXP_CPL_ERR_MASK (BIT(3) | BIT(4) | BIT(10) | BIT(11)) +#define ADF_GEN6_RF_PAR_ERR_MASK BIT(16) + +#define ADF_GEN6_SSM_FERR_STATUS 0x9C + +#define ADF_GEN6_CPP_CFC_ERR_STATUS 0x640C04 + +/* + * BIT(0) - Indicates one or more CPP CFC errors + * BIT(1) - Indicates multiple CPP CFC errors + * BIT(7) - Indicates CPP CFC command parity error type + * BIT(8) - Indicates CPP CFC data parity error type + */ +#define ADF_GEN6_CPP_CFC_ERR_STATUS_ERR_BIT BIT(0) +#define ADF_GEN6_CPP_CFC_ERR_STATUS_MERR_BIT BIT(1) +#define ADF_GEN6_CPP_CFC_ERR_STATUS_CMDPAR_BIT BIT(7) +#define ADF_GEN6_CPP_CFC_ERR_STATUS_DATAPAR_BIT BIT(8) +#define ADF_GEN6_CPP_CFC_FATAL_ERR_BIT \ + (ADF_GEN6_CPP_CFC_ERR_STATUS_ERR_BIT | \ + ADF_GEN6_CPP_CFC_ERR_STATUS_MERR_BIT) + +/* + * BIT(0) - Enables CFC to detect and log a push/pull data error + * BIT(1) - Enables CFC to generate interrupt to PCIEP for a CPP error + * BIT(4) - When 1 parity detection is disabled + * BIT(5) - When 1 parity detection is disabled on CPP command bus + * BIT(6) - When 1 parity detection is disabled on CPP push/pull bus + * BIT(9) - When 1 RF parity error detection is disabled + */ +#define ADF_GEN6_CPP_CFC_ERR_CTRL_MASK (BIT(0) | BIT(1)) + +#define ADF_GEN6_CPP_CFC_ERR_CTRL_DIS_MASK \ + (BIT(4) | BIT(5) | BIT(6) | BIT(9) | BIT(10)) + +#define ADF_GEN6_CPP_CFC_ERR_CTRL 0x640C00 + +/* + * BIT(0) - Clears bit(0) of ADF_GEN6_CPP_CFC_ERR_STATUS + * when an error is reported on CPP + * BIT(1) - Clears bit(1) of ADF_GEN6_CPP_CFC_ERR_STATUS + * when multiple errors are reported on CPP + * BIT(2) - Clears bit(2) of ADF_GEN6_CPP_CFC_ERR_STATUS + * when attention interrupt is reported + */ +#define ADF_GEN6_CPP_CFC_ERR_STATUS_CLR_MASK (BIT(0) | BIT(1) | BIT(2)) +#define ADF_GEN6_CPP_CFC_ERR_STATUS_CLR 0x640C08 + +/* + * ERRSOU3 bit masks + * BIT(0) - indicates error response order overflow and/or BME error + * BIT(1) - indicates RI push/pull error + * BIT(2) - indicates TI push/pull error + * BIT(5) - indicates TI pull parity error + * BIT(6) - indicates RI push parity error + * BIT(7) - indicates VFLR interrupt + * BIT(8) - indicates ring pair interrupts for ATU detected fault + * BIT(9) - indicates rate limiting error + */ +#define ADF_GEN6_ERRSOU3_TIMISCSTS_BIT BIT(0) +#define ADF_GEN6_ERRSOU3_RICPPINTSTS_MASK (BIT(1) | BIT(6)) +#define ADF_GEN6_ERRSOU3_TICPPINTSTS_MASK (BIT(2) | BIT(5)) +#define ADF_GEN6_ERRSOU3_VFLRNOTIFY_BIT BIT(7) +#define ADF_GEN6_ERRSOU3_ATUFAULTSTATUS_BIT BIT(8) +#define ADF_GEN6_ERRSOU3_RLTERROR_BIT BIT(9) +#define ADF_GEN6_ERRSOU3_TC_VC_MAP_ERROR_BIT BIT(16) +#define ADF_GEN6_ERRSOU3_PCIE_DEVHALT_BIT BIT(17) +#define ADF_GEN6_ERRSOU3_PG_REQ_DEVHALT_BIT BIT(18) +#define ADF_GEN6_ERRSOU3_XLT_CPL_DEVHALT_BIT BIT(19) +#define ADF_GEN6_ERRSOU3_TI_INT_ERR_DEVHALT_BIT BIT(20) + +#define ADF_GEN6_ERRSOU3_MASK ( \ + (ADF_GEN6_ERRSOU3_TIMISCSTS_BIT) | \ + (ADF_GEN6_ERRSOU3_RICPPINTSTS_MASK) | \ + (ADF_GEN6_ERRSOU3_TICPPINTSTS_MASK) | \ + (ADF_GEN6_ERRSOU3_VFLRNOTIFY_BIT) | \ + (ADF_GEN6_ERRSOU3_ATUFAULTSTATUS_BIT) | \ + (ADF_GEN6_ERRSOU3_RLTERROR_BIT) | \ + (ADF_GEN6_ERRSOU3_TC_VC_MAP_ERROR_BIT) | \ + (ADF_GEN6_ERRSOU3_PCIE_DEVHALT_BIT) | \ + (ADF_GEN6_ERRSOU3_PG_REQ_DEVHALT_BIT) | \ + (ADF_GEN6_ERRSOU3_XLT_CPL_DEVHALT_BIT) | \ + (ADF_GEN6_ERRSOU3_TI_INT_ERR_DEVHALT_BIT)) + +#define ADF_GEN6_ERRSOU3_DIS_MASK ( \ + (ADF_GEN6_ERRSOU3_TIMISCSTS_BIT) | \ + (ADF_GEN6_ERRSOU3_RICPPINTSTS_MASK) | \ + (ADF_GEN6_ERRSOU3_TICPPINTSTS_MASK) | \ + (ADF_GEN6_ERRSOU3_VFLRNOTIFY_BIT) | \ + (ADF_GEN6_ERRSOU3_ATUFAULTSTATUS_BIT) | \ + (ADF_GEN6_ERRSOU3_RLTERROR_BIT) | \ + (ADF_GEN6_ERRSOU3_TC_VC_MAP_ERROR_BIT)) + +/* Rate limiting error log register */ +#define ADF_GEN6_RLT_ERRLOG 0x508814 + +#define ADF_GEN6_RLT_ERRLOG_MASK (BIT(0) | BIT(1) | BIT(2) | BIT(3)) + +/* TI misc status register */ +#define ADF_GEN6_TIMISCSTS 0x50054C + +/* TI misc error reporting mask */ +#define ADF_GEN6_TIMISCCTL 0x500548 + +/* + * TI Misc error reporting control mask + * BIT(0) - Enables error detection and logging in TIMISCSTS register + * BIT(1) - It has effect only when SRIOV enabled, this bit is 0 by default + * BIT(2) - Enables the D-F-x counter within the dispatch arbiter + * to start based on the command triggered from + * BIT(30) - Disables VFLR functionality + * bits 1, 2 and 30 value should be preserved and not meant to be changed + * within RAS. + */ +#define ADF_GEN6_TIMISCCTL_BIT BIT(0) +#define ADF_GEN6_TIMSCCTL_RELAY_MASK (BIT(1) | BIT(2) | BIT(30)) + +/* RI CPP interface status register */ +#define ADF_GEN6_RICPPINTSTS 0x41A330 + +/* + * Uncorrectable error mask in RICPPINTSTS register + * BIT(0) - RI asserted the CPP error signal during a push + * BIT(1) - RI detected the CPP error signal asserted during a pull + * BIT(2) - RI detected a push data parity error + * BIT(3) - RI detected a push valid parity error + */ +#define ADF_GEN6_RICPPINTSTS_MASK (BIT(0) | BIT(1) | BIT(2) | BIT(3)) + +/* RI CPP interface register control */ +#define ADF_GEN6_RICPPINTCTL 0x41A32C + +/* + * Control bit mask for RICPPINTCTL register + * BIT(0) - value of 1 enables error detection and reporting + * on the RI CPP Push interface + * BIT(1) - value of 1 enables error detection and reporting + * on the RI CPP Pull interface + * BIT(2) - value of 1 enables error detection and reporting + * on the RI Parity + * BIT(3) - value of 1 enable checking parity on CPP + */ +#define ADF_GEN6_RICPPINTCTL_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4)) + +/* TI CPP interface status register */ +#define ADF_GEN6_TICPPINTSTS 0x50053C + +/* + * Uncorrectable error mask in TICPPINTSTS register + * BIT(0) - value of 1 indicates that the TI asserted + * the CPP error signal during a push + * BIT(1) - value of 1 indicates that the TI detected + * the CPP error signal asserted during a pull + * BIT(2) - value of 1 indicates that the TI detected + * a pull data parity error + */ +#define ADF_GEN6_TICPPINTSTS_MASK (BIT(0) | BIT(1) | BIT(2)) + +/* TI CPP interface status register control */ +#define ADF_GEN6_TICPPINTCTL 0x500538 + +/* + * Control bit mask for TICPPINTCTL register + * BIT(0) - value of 1 enables error detection and reporting on + * the TI CPP Push interface + * BIT(1) - value of 1 enables error detection and reporting on + * the TI CPP Push interface + * BIT(2) - value of 1 enables parity error detection and logging on + * the TI CPP Pull interface + * BIT(3) - value of 1 enables CPP CMD and Pull Data parity checking + */ +#define ADF_GEN6_TICPPINTCTL_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4)) + +/* ATU fault status register */ +#define ADF_GEN6_ATUFAULTSTATUS(i) (0x506000 + ((i) * 0x4)) + +#define ADF_GEN6_ATUFAULTSTATUS_BIT BIT(0) + +/* Command parity error detected on IOSFP command to QAT */ +#define ADF_GEN6_RIMISCSTS_BIT BIT(0) + +#define ADF_GEN6_GENSTS 0x41A220 +#define ADF_GEN6_GENSTS_DEVICE_STATE_MASK GENMASK(1, 0) +#define ADF_GEN6_GENSTS_RESET_TYPE_MASK GENMASK(3, 2) +#define ADF_GEN6_GENSTS_PFLR 0x1 +#define ADF_GEN6_GENSTS_COLD_RESET 0x3 +#define ADF_GEN6_GENSTS_DEVHALT 0x1 + +void adf_gen6_init_ras_ops(struct adf_ras_ops *ras_ops); + +#endif /* ADF_GEN6_RAS_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_shared.c b/drivers/crypto/intel/qat/qat_common/adf_gen6_shared.c new file mode 100644 index 000000000..ffe4525a1 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_shared.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2025 Intel Corporation */ +#include + +#include "adf_gen4_config.h" +#include "adf_gen4_hw_csr_data.h" +#include "adf_gen4_pfvf.h" +#include "adf_gen4_vf_mig.h" +#include "adf_gen6_shared.h" + +struct adf_accel_dev; +struct adf_pfvf_ops; +struct adf_hw_csr_ops; + +/* + * QAT GEN4 and GEN6 devices often differ in terms of supported features, + * options and internal logic. However, some of the mechanisms and register + * layout are shared between those two GENs. This file serves as an abstraction + * layer that allows to use existing GEN4 implementation that is also + * applicable to GEN6 without additional overhead and complexity. + */ +void adf_gen6_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) +{ + adf_gen4_init_pf_pfvf_ops(pfvf_ops); +} +EXPORT_SYMBOL_GPL(adf_gen6_init_pf_pfvf_ops); + +void adf_gen6_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) +{ + return adf_gen4_init_hw_csr_ops(csr_ops); +} +EXPORT_SYMBOL_GPL(adf_gen6_init_hw_csr_ops); + +int adf_gen6_comp_dev_config(struct adf_accel_dev *accel_dev) +{ + return adf_comp_dev_config(accel_dev); +} +EXPORT_SYMBOL_GPL(adf_gen6_comp_dev_config); + +int adf_gen6_no_dev_config(struct adf_accel_dev *accel_dev) +{ + return adf_no_dev_config(accel_dev); +} +EXPORT_SYMBOL_GPL(adf_gen6_no_dev_config); + +void adf_gen6_init_vf_mig_ops(struct qat_migdev_ops *vfmig_ops) +{ + adf_gen4_init_vf_mig_ops(vfmig_ops); +} +EXPORT_SYMBOL_GPL(adf_gen6_init_vf_mig_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_shared.h b/drivers/crypto/intel/qat/qat_common/adf_gen6_shared.h new file mode 100644 index 000000000..072115a53 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_shared.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ADF_GEN6_SHARED_H_ +#define ADF_GEN6_SHARED_H_ + +struct adf_hw_csr_ops; +struct qat_migdev_ops; +struct adf_accel_dev; +struct adf_pfvf_ops; + +void adf_gen6_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops); +void adf_gen6_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); +int adf_gen6_comp_dev_config(struct adf_accel_dev *accel_dev); +int adf_gen6_no_dev_config(struct adf_accel_dev *accel_dev); +void adf_gen6_init_vf_mig_ops(struct qat_migdev_ops *vfmig_ops); +#endif/* ADF_GEN6_SHARED_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_tl.c b/drivers/crypto/intel/qat/qat_common/adf_gen6_tl.c new file mode 100644 index 000000000..faa60b04c --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_tl.c @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2025 Intel Corporation. */ +#include + +#include "adf_gen6_tl.h" +#include "adf_telemetry.h" +#include "adf_tl_debugfs.h" +#include "icp_qat_fw_init_admin.h" + +#define ADF_GEN6_TL_DEV_REG_OFF(reg) ADF_TL_DEV_REG_OFF(reg, gen6) + +#define ADF_GEN6_TL_RP_REG_OFF(reg) ADF_TL_RP_REG_OFF(reg, gen6) + +#define ADF_GEN6_TL_SL_UTIL_COUNTER(_name) \ + ADF_TL_COUNTER("util_" #_name, ADF_TL_SIMPLE_COUNT, \ + ADF_TL_SLICE_REG_OFF(_name, reg_tm_slice_util, gen6)) + +#define ADF_GEN6_TL_SL_EXEC_COUNTER(_name) \ + ADF_TL_COUNTER("exec_" #_name, ADF_TL_SIMPLE_COUNT, \ + ADF_TL_SLICE_REG_OFF(_name, reg_tm_slice_exec_cnt, gen6)) + +#define SLICE_IDX(sl) offsetof(struct icp_qat_fw_init_admin_slice_cnt, sl##_cnt) + +#define ADF_GEN6_TL_CMDQ_WAIT_COUNTER(_name) \ + ADF_TL_COUNTER("cmdq_wait_" #_name, ADF_TL_SIMPLE_COUNT, \ + ADF_TL_CMDQ_REG_OFF(_name, reg_tm_cmdq_wait_cnt, gen6)) +#define ADF_GEN6_TL_CMDQ_EXEC_COUNTER(_name) \ + ADF_TL_COUNTER("cmdq_exec_" #_name, ADF_TL_SIMPLE_COUNT, \ + ADF_TL_CMDQ_REG_OFF(_name, reg_tm_cmdq_exec_cnt, gen6)) +#define ADF_GEN6_TL_CMDQ_DRAIN_COUNTER(_name) \ + ADF_TL_COUNTER("cmdq_drain_" #_name, ADF_TL_SIMPLE_COUNT, \ + ADF_TL_CMDQ_REG_OFF(_name, reg_tm_cmdq_drain_cnt, \ + gen6)) + +#define CPR_QUEUE_COUNT 5 +#define DCPR_QUEUE_COUNT 3 +#define PKE_QUEUE_COUNT 1 +#define WAT_QUEUE_COUNT 7 +#define WCP_QUEUE_COUNT 7 +#define USC_QUEUE_COUNT 3 +#define ATH_QUEUE_COUNT 2 + +/* Device level counters. */ +static const struct adf_tl_dbg_counter dev_counters[] = { + /* PCIe partial transactions. */ + ADF_TL_COUNTER(PCI_TRANS_CNT_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_prt_trans_cnt)), + /* Max read latency[ns]. */ + ADF_TL_COUNTER(MAX_RD_LAT_NAME, ADF_TL_COUNTER_NS, + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_rd_lat_max)), + /* Read latency average[ns]. */ + ADF_TL_COUNTER_LATENCY(RD_LAT_ACC_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_rd_lat_acc), + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_rd_cmpl_cnt)), + /* Max "get to put" latency[ns]. */ + ADF_TL_COUNTER(MAX_LAT_NAME, ADF_TL_COUNTER_NS, + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_gp_lat_max)), + /* "Get to put" latency average[ns]. */ + ADF_TL_COUNTER_LATENCY(LAT_ACC_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_gp_lat_acc), + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_ae_put_cnt)), + /* PCIe write bandwidth[Mbps]. */ + ADF_TL_COUNTER(BW_IN_NAME, ADF_TL_COUNTER_MBPS, + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_bw_in)), + /* PCIe read bandwidth[Mbps]. */ + ADF_TL_COUNTER(BW_OUT_NAME, ADF_TL_COUNTER_MBPS, + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_bw_out)), + /* Page request latency average[ns]. */ + ADF_TL_COUNTER_LATENCY(PAGE_REQ_LAT_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_at_page_req_lat_acc), + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_at_page_req_cnt)), + /* Page translation latency average[ns]. */ + ADF_TL_COUNTER_LATENCY(AT_TRANS_LAT_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_at_trans_lat_acc), + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_at_trans_lat_cnt)), + /* Maximum uTLB used. */ + ADF_TL_COUNTER(AT_MAX_UTLB_USED_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_at_max_utlb_used)), + /* Ring Empty average[ns] across all rings */ + ADF_TL_COUNTER_LATENCY(RE_ACC_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_re_acc), + ADF_GEN6_TL_DEV_REG_OFF(reg_tl_re_cnt)), +}; + +/* Accelerator utilization counters */ +static const struct adf_tl_dbg_counter sl_util_counters[ADF_TL_SL_CNT_COUNT] = { + /* Compression accelerator utilization. */ + [SLICE_IDX(cpr)] = ADF_GEN6_TL_SL_UTIL_COUNTER(cnv), + /* Decompression accelerator utilization. */ + [SLICE_IDX(dcpr)] = ADF_GEN6_TL_SL_UTIL_COUNTER(dcprz), + /* PKE accelerator utilization. */ + [SLICE_IDX(pke)] = ADF_GEN6_TL_SL_UTIL_COUNTER(pke), + /* Wireless Authentication accelerator utilization. */ + [SLICE_IDX(wat)] = ADF_GEN6_TL_SL_UTIL_COUNTER(wat), + /* Wireless Cipher accelerator utilization. */ + [SLICE_IDX(wcp)] = ADF_GEN6_TL_SL_UTIL_COUNTER(wcp), + /* UCS accelerator utilization. */ + [SLICE_IDX(ucs)] = ADF_GEN6_TL_SL_UTIL_COUNTER(ucs), + /* Authentication accelerator utilization. */ + [SLICE_IDX(ath)] = ADF_GEN6_TL_SL_UTIL_COUNTER(ath), +}; + +/* Accelerator execution counters */ +static const struct adf_tl_dbg_counter sl_exec_counters[ADF_TL_SL_CNT_COUNT] = { + /* Compression accelerator execution count. */ + [SLICE_IDX(cpr)] = ADF_GEN6_TL_SL_EXEC_COUNTER(cnv), + /* Decompression accelerator execution count. */ + [SLICE_IDX(dcpr)] = ADF_GEN6_TL_SL_EXEC_COUNTER(dcprz), + /* PKE execution count. */ + [SLICE_IDX(pke)] = ADF_GEN6_TL_SL_EXEC_COUNTER(pke), + /* Wireless Authentication accelerator execution count. */ + [SLICE_IDX(wat)] = ADF_GEN6_TL_SL_EXEC_COUNTER(wat), + /* Wireless Cipher accelerator execution count. */ + [SLICE_IDX(wcp)] = ADF_GEN6_TL_SL_EXEC_COUNTER(wcp), + /* UCS accelerator execution count. */ + [SLICE_IDX(ucs)] = ADF_GEN6_TL_SL_EXEC_COUNTER(ucs), + /* Authentication accelerator execution count. */ + [SLICE_IDX(ath)] = ADF_GEN6_TL_SL_EXEC_COUNTER(ath), +}; + +static const struct adf_tl_dbg_counter cnv_cmdq_counters[] = { + ADF_GEN6_TL_CMDQ_WAIT_COUNTER(cnv), + ADF_GEN6_TL_CMDQ_EXEC_COUNTER(cnv), + ADF_GEN6_TL_CMDQ_DRAIN_COUNTER(cnv) +}; + +#define NUM_CMDQ_COUNTERS ARRAY_SIZE(cnv_cmdq_counters) + +static const struct adf_tl_dbg_counter dcprz_cmdq_counters[] = { + ADF_GEN6_TL_CMDQ_WAIT_COUNTER(dcprz), + ADF_GEN6_TL_CMDQ_EXEC_COUNTER(dcprz), + ADF_GEN6_TL_CMDQ_DRAIN_COUNTER(dcprz) +}; + +static_assert(ARRAY_SIZE(dcprz_cmdq_counters) == NUM_CMDQ_COUNTERS); + +static const struct adf_tl_dbg_counter pke_cmdq_counters[] = { + ADF_GEN6_TL_CMDQ_WAIT_COUNTER(pke), + ADF_GEN6_TL_CMDQ_EXEC_COUNTER(pke), + ADF_GEN6_TL_CMDQ_DRAIN_COUNTER(pke) +}; + +static_assert(ARRAY_SIZE(pke_cmdq_counters) == NUM_CMDQ_COUNTERS); + +static const struct adf_tl_dbg_counter wat_cmdq_counters[] = { + ADF_GEN6_TL_CMDQ_WAIT_COUNTER(wat), + ADF_GEN6_TL_CMDQ_EXEC_COUNTER(wat), + ADF_GEN6_TL_CMDQ_DRAIN_COUNTER(wat) +}; + +static_assert(ARRAY_SIZE(wat_cmdq_counters) == NUM_CMDQ_COUNTERS); + +static const struct adf_tl_dbg_counter wcp_cmdq_counters[] = { + ADF_GEN6_TL_CMDQ_WAIT_COUNTER(wcp), + ADF_GEN6_TL_CMDQ_EXEC_COUNTER(wcp), + ADF_GEN6_TL_CMDQ_DRAIN_COUNTER(wcp) +}; + +static_assert(ARRAY_SIZE(wcp_cmdq_counters) == NUM_CMDQ_COUNTERS); + +static const struct adf_tl_dbg_counter ucs_cmdq_counters[] = { + ADF_GEN6_TL_CMDQ_WAIT_COUNTER(ucs), + ADF_GEN6_TL_CMDQ_EXEC_COUNTER(ucs), + ADF_GEN6_TL_CMDQ_DRAIN_COUNTER(ucs) +}; + +static_assert(ARRAY_SIZE(ucs_cmdq_counters) == NUM_CMDQ_COUNTERS); + +static const struct adf_tl_dbg_counter ath_cmdq_counters[] = { + ADF_GEN6_TL_CMDQ_WAIT_COUNTER(ath), + ADF_GEN6_TL_CMDQ_EXEC_COUNTER(ath), + ADF_GEN6_TL_CMDQ_DRAIN_COUNTER(ath) +}; + +static_assert(ARRAY_SIZE(ath_cmdq_counters) == NUM_CMDQ_COUNTERS); + +/* CMDQ drain counters. */ +static const struct adf_tl_dbg_counter *cmdq_counters[ADF_TL_SL_CNT_COUNT] = { + /* Compression accelerator execution count. */ + [SLICE_IDX(cpr)] = cnv_cmdq_counters, + /* Decompression accelerator execution count. */ + [SLICE_IDX(dcpr)] = dcprz_cmdq_counters, + /* PKE execution count. */ + [SLICE_IDX(pke)] = pke_cmdq_counters, + /* Wireless Authentication accelerator execution count. */ + [SLICE_IDX(wat)] = wat_cmdq_counters, + /* Wireless Cipher accelerator execution count. */ + [SLICE_IDX(wcp)] = wcp_cmdq_counters, + /* UCS accelerator execution count. */ + [SLICE_IDX(ucs)] = ucs_cmdq_counters, + /* Authentication accelerator execution count. */ + [SLICE_IDX(ath)] = ath_cmdq_counters, +}; + +/* Ring pair counters. */ +static const struct adf_tl_dbg_counter rp_counters[] = { + /* PCIe partial transactions. */ + ADF_TL_COUNTER(PCI_TRANS_CNT_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN6_TL_RP_REG_OFF(reg_tl_prt_trans_cnt)), + /* "Get to put" latency average[ns]. */ + ADF_TL_COUNTER_LATENCY(LAT_ACC_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN6_TL_RP_REG_OFF(reg_tl_gp_lat_acc), + ADF_GEN6_TL_RP_REG_OFF(reg_tl_ae_put_cnt)), + /* PCIe write bandwidth[Mbps]. */ + ADF_TL_COUNTER(BW_IN_NAME, ADF_TL_COUNTER_MBPS, + ADF_GEN6_TL_RP_REG_OFF(reg_tl_bw_in)), + /* PCIe read bandwidth[Mbps]. */ + ADF_TL_COUNTER(BW_OUT_NAME, ADF_TL_COUNTER_MBPS, + ADF_GEN6_TL_RP_REG_OFF(reg_tl_bw_out)), + /* Message descriptor DevTLB hit rate. */ + ADF_TL_COUNTER(AT_GLOB_DTLB_HIT_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN6_TL_RP_REG_OFF(reg_tl_at_glob_devtlb_hit)), + /* Message descriptor DevTLB miss rate. */ + ADF_TL_COUNTER(AT_GLOB_DTLB_MISS_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN6_TL_RP_REG_OFF(reg_tl_at_glob_devtlb_miss)), + /* Payload DevTLB hit rate. */ + ADF_TL_COUNTER(AT_PAYLD_DTLB_HIT_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN6_TL_RP_REG_OFF(reg_tl_at_payld_devtlb_hit)), + /* Payload DevTLB miss rate. */ + ADF_TL_COUNTER(AT_PAYLD_DTLB_MISS_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN6_TL_RP_REG_OFF(reg_tl_at_payld_devtlb_miss)), + /* Ring Empty average[ns]. */ + ADF_TL_COUNTER_LATENCY(RE_ACC_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN6_TL_RP_REG_OFF(reg_tl_re_acc), + ADF_GEN6_TL_RP_REG_OFF(reg_tl_re_cnt)), +}; + +void adf_gen6_init_tl_data(struct adf_tl_hw_data *tl_data) +{ + tl_data->layout_sz = ADF_GEN6_TL_LAYOUT_SZ; + tl_data->slice_reg_sz = ADF_GEN6_TL_SLICE_REG_SZ; + tl_data->cmdq_reg_sz = ADF_GEN6_TL_CMDQ_REG_SZ; + tl_data->rp_reg_sz = ADF_GEN6_TL_RP_REG_SZ; + tl_data->num_hbuff = ADF_GEN6_TL_NUM_HIST_BUFFS; + tl_data->max_rp = ADF_GEN6_TL_MAX_RP_NUM; + tl_data->msg_cnt_off = ADF_GEN6_TL_MSG_CNT_OFF; + tl_data->cpp_ns_per_cycle = ADF_GEN6_CPP_NS_PER_CYCLE; + tl_data->bw_units_to_bytes = ADF_GEN6_TL_BW_HW_UNITS_TO_BYTES; + + tl_data->dev_counters = dev_counters; + tl_data->num_dev_counters = ARRAY_SIZE(dev_counters); + tl_data->sl_util_counters = sl_util_counters; + tl_data->sl_exec_counters = sl_exec_counters; + tl_data->cmdq_counters = cmdq_counters; + tl_data->num_cmdq_counters = NUM_CMDQ_COUNTERS; + tl_data->rp_counters = rp_counters; + tl_data->num_rp_counters = ARRAY_SIZE(rp_counters); + tl_data->max_sl_cnt = ADF_GEN6_TL_MAX_SLICES_PER_TYPE; + + tl_data->multiplier.cpr_cnt = CPR_QUEUE_COUNT; + tl_data->multiplier.dcpr_cnt = DCPR_QUEUE_COUNT; + tl_data->multiplier.pke_cnt = PKE_QUEUE_COUNT; + tl_data->multiplier.wat_cnt = WAT_QUEUE_COUNT; + tl_data->multiplier.wcp_cnt = WCP_QUEUE_COUNT; + tl_data->multiplier.ucs_cnt = USC_QUEUE_COUNT; + tl_data->multiplier.ath_cnt = ATH_QUEUE_COUNT; +} +EXPORT_SYMBOL_GPL(adf_gen6_init_tl_data); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_tl.h b/drivers/crypto/intel/qat/qat_common/adf_gen6_tl.h new file mode 100644 index 000000000..49db660b8 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_tl.h @@ -0,0 +1,198 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2025 Intel Corporation. */ +#ifndef ADF_GEN6_TL_H +#define ADF_GEN6_TL_H + +#include + +struct adf_tl_hw_data; + +/* Computation constants. */ +#define ADF_GEN6_CPP_NS_PER_CYCLE 2 +#define ADF_GEN6_TL_BW_HW_UNITS_TO_BYTES 64 + +/* Maximum aggregation time. Value is in milliseconds. */ +#define ADF_GEN6_TL_MAX_AGGR_TIME_MS 4000 +/* Number of buffers to store historic values. */ +#define ADF_GEN6_TL_NUM_HIST_BUFFS \ + (ADF_GEN6_TL_MAX_AGGR_TIME_MS / ADF_TL_DATA_WR_INTERVAL_MS) + +/* Max number of HW resources of one type */ +#define ADF_GEN6_TL_MAX_SLICES_PER_TYPE 32 +#define MAX_ATH_SL_COUNT 7 +#define MAX_CNV_SL_COUNT 2 +#define MAX_DCPRZ_SL_COUNT 2 +#define MAX_PKE_SL_COUNT 32 +#define MAX_UCS_SL_COUNT 4 +#define MAX_WAT_SL_COUNT 5 +#define MAX_WCP_SL_COUNT 5 + +#define MAX_ATH_CMDQ_COUNT 14 +#define MAX_CNV_CMDQ_COUNT 6 +#define MAX_DCPRZ_CMDQ_COUNT 6 +#define MAX_PKE_CMDQ_COUNT 32 +#define MAX_UCS_CMDQ_COUNT 12 +#define MAX_WAT_CMDQ_COUNT 35 +#define MAX_WCP_CMDQ_COUNT 35 + +/* Max number of simultaneously monitored ring pairs. */ +#define ADF_GEN6_TL_MAX_RP_NUM 4 + +/** + * struct adf_gen6_tl_slice_data_regs - HW slice data as populated by FW. + * @reg_tm_slice_exec_cnt: Slice execution count. + * @reg_tm_slice_util: Slice utilization. + */ +struct adf_gen6_tl_slice_data_regs { + __u32 reg_tm_slice_exec_cnt; + __u32 reg_tm_slice_util; +}; + +#define ADF_GEN6_TL_SLICE_REG_SZ sizeof(struct adf_gen6_tl_slice_data_regs) + +/** + * struct adf_gen6_tl_cmdq_data_regs - HW CMDQ data as populated by FW. + * @reg_tm_cmdq_wait_cnt: CMDQ wait count. + * @reg_tm_cmdq_exec_cnt: CMDQ execution count. + * @reg_tm_cmdq_drain_cnt: CMDQ drain count. + */ +struct adf_gen6_tl_cmdq_data_regs { + __u32 reg_tm_cmdq_wait_cnt; + __u32 reg_tm_cmdq_exec_cnt; + __u32 reg_tm_cmdq_drain_cnt; + __u32 reserved; +}; + +#define ADF_GEN6_TL_CMDQ_REG_SZ sizeof(struct adf_gen6_tl_cmdq_data_regs) + +/** + * struct adf_gen6_tl_device_data_regs - This structure stores device telemetry + * counter values as are being populated periodically by device. + * @reg_tl_rd_lat_acc: read latency accumulator + * @reg_tl_gp_lat_acc: "get to put" latency accumulator + * @reg_tl_at_page_req_lat_acc: AT/DevTLB page request latency accumulator + * @reg_tl_at_trans_lat_acc: DevTLB transaction latency accumulator + * @reg_tl_re_acc: accumulated ring empty time + * @reg_tl_prt_trans_cnt: PCIe partial transactions + * @reg_tl_rd_lat_max: maximum logged read latency + * @reg_tl_rd_cmpl_cnt: read requests completed count + * @reg_tl_gp_lat_max: maximum logged get to put latency + * @reg_tl_ae_put_cnt: Accelerator Engine put counts across all rings + * @reg_tl_bw_in: PCIe write bandwidth + * @reg_tl_bw_out: PCIe read bandwidth + * @reg_tl_at_page_req_cnt: DevTLB page requests count + * @reg_tl_at_trans_lat_cnt: DevTLB transaction latency samples count + * @reg_tl_at_max_utlb_used: maximum uTLB used + * @reg_tl_re_cnt: ring empty time samples count + * @reserved: reserved + * @ath_slices: array of Authentication slices utilization registers + * @cnv_slices: array of Compression slices utilization registers + * @dcprz_slices: array of Decompression slices utilization registers + * @pke_slices: array of PKE slices utilization registers + * @ucs_slices: array of UCS slices utilization registers + * @wat_slices: array of Wireless Authentication slices utilization registers + * @wcp_slices: array of Wireless Cipher slices utilization registers + * @ath_cmdq: array of Authentication cmdq telemetry registers + * @cnv_cmdq: array of Compression cmdq telemetry registers + * @dcprz_cmdq: array of Decomopression cmdq telemetry registers + * @pke_cmdq: array of PKE cmdq telemetry registers + * @ucs_cmdq: array of UCS cmdq telemetry registers + * @wat_cmdq: array of Wireless Authentication cmdq telemetry registers + * @wcp_cmdq: array of Wireless Cipher cmdq telemetry registers + */ +struct adf_gen6_tl_device_data_regs { + __u64 reg_tl_rd_lat_acc; + __u64 reg_tl_gp_lat_acc; + __u64 reg_tl_at_page_req_lat_acc; + __u64 reg_tl_at_trans_lat_acc; + __u64 reg_tl_re_acc; + __u32 reg_tl_prt_trans_cnt; + __u32 reg_tl_rd_lat_max; + __u32 reg_tl_rd_cmpl_cnt; + __u32 reg_tl_gp_lat_max; + __u32 reg_tl_ae_put_cnt; + __u32 reg_tl_bw_in; + __u32 reg_tl_bw_out; + __u32 reg_tl_at_page_req_cnt; + __u32 reg_tl_at_trans_lat_cnt; + __u32 reg_tl_at_max_utlb_used; + __u32 reg_tl_re_cnt; + __u32 reserved; + struct adf_gen6_tl_slice_data_regs ath_slices[MAX_ATH_SL_COUNT]; + struct adf_gen6_tl_slice_data_regs cnv_slices[MAX_CNV_SL_COUNT]; + struct adf_gen6_tl_slice_data_regs dcprz_slices[MAX_DCPRZ_SL_COUNT]; + struct adf_gen6_tl_slice_data_regs pke_slices[MAX_PKE_SL_COUNT]; + struct adf_gen6_tl_slice_data_regs ucs_slices[MAX_UCS_SL_COUNT]; + struct adf_gen6_tl_slice_data_regs wat_slices[MAX_WAT_SL_COUNT]; + struct adf_gen6_tl_slice_data_regs wcp_slices[MAX_WCP_SL_COUNT]; + struct adf_gen6_tl_cmdq_data_regs ath_cmdq[MAX_ATH_CMDQ_COUNT]; + struct adf_gen6_tl_cmdq_data_regs cnv_cmdq[MAX_CNV_CMDQ_COUNT]; + struct adf_gen6_tl_cmdq_data_regs dcprz_cmdq[MAX_DCPRZ_CMDQ_COUNT]; + struct adf_gen6_tl_cmdq_data_regs pke_cmdq[MAX_PKE_CMDQ_COUNT]; + struct adf_gen6_tl_cmdq_data_regs ucs_cmdq[MAX_UCS_CMDQ_COUNT]; + struct adf_gen6_tl_cmdq_data_regs wat_cmdq[MAX_WAT_CMDQ_COUNT]; + struct adf_gen6_tl_cmdq_data_regs wcp_cmdq[MAX_WCP_CMDQ_COUNT]; +}; + +/** + * struct adf_gen6_tl_ring_pair_data_regs - This structure stores ring pair + * telemetry counter values as they are being populated periodically by device. + * @reg_tl_gp_lat_acc: get-put latency accumulator + * @reg_tl_re_acc: accumulated ring empty time + * @reg_tl_pci_trans_cnt: PCIe partial transactions + * @reg_tl_ae_put_cnt: Accelerator Engine put counts across all rings + * @reg_tl_bw_in: PCIe write bandwidth + * @reg_tl_bw_out: PCIe read bandwidth + * @reg_tl_at_glob_devtlb_hit: Message descriptor DevTLB hit rate + * @reg_tl_at_glob_devtlb_miss: Message descriptor DevTLB miss rate + * @reg_tl_at_payld_devtlb_hit: Payload DevTLB hit rate + * @reg_tl_at_payld_devtlb_miss: Payload DevTLB miss rate + * @reg_tl_re_cnt: ring empty time samples count + * @reserved1: reserved + */ +struct adf_gen6_tl_ring_pair_data_regs { + __u64 reg_tl_gp_lat_acc; + __u64 reg_tl_re_acc; + __u32 reg_tl_prt_trans_cnt; + __u32 reg_tl_ae_put_cnt; + __u32 reg_tl_bw_in; + __u32 reg_tl_bw_out; + __u32 reg_tl_at_glob_devtlb_hit; + __u32 reg_tl_at_glob_devtlb_miss; + __u32 reg_tl_at_payld_devtlb_hit; + __u32 reg_tl_at_payld_devtlb_miss; + __u32 reg_tl_re_cnt; + __u32 reserved1; +}; + +#define ADF_GEN6_TL_RP_REG_SZ sizeof(struct adf_gen6_tl_ring_pair_data_regs) + +/** + * struct adf_gen6_tl_layout - This structure represents the entire telemetry + * counters data: Device + 4 Ring Pairs as they are being populated periodically + * by device. + * @tl_device_data_regs: structure of device telemetry registers + * @tl_ring_pairs_data_regs: array of ring pairs telemetry registers + * @reg_tl_msg_cnt: telemetry message counter + * @reserved: reserved + */ +struct adf_gen6_tl_layout { + struct adf_gen6_tl_device_data_regs tl_device_data_regs; + struct adf_gen6_tl_ring_pair_data_regs + tl_ring_pairs_data_regs[ADF_GEN6_TL_MAX_RP_NUM]; + __u32 reg_tl_msg_cnt; + __u32 reserved; +}; + +#define ADF_GEN6_TL_LAYOUT_SZ sizeof(struct adf_gen6_tl_layout) +#define ADF_GEN6_TL_MSG_CNT_OFF \ + offsetof(struct adf_gen6_tl_layout, reg_tl_msg_cnt) + +#ifdef CONFIG_DEBUG_FS +void adf_gen6_init_tl_data(struct adf_tl_hw_data *tl_data); +#else +static inline void adf_gen6_init_tl_data(struct adf_tl_hw_data *tl_data) +{ +} +#endif /* CONFIG_DEBUG_FS */ +#endif /* ADF_GEN6_TL_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_heartbeat.c b/drivers/crypto/intel/qat/qat_common/adf_heartbeat.c new file mode 100644 index 000000000..0cdb4241f --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_heartbeat.c @@ -0,0 +1,345 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "adf_accel_devices.h" +#include "adf_admin.h" +#include "adf_cfg.h" +#include "adf_cfg_strings.h" +#include "adf_clock.h" +#include "adf_common_drv.h" +#include "adf_heartbeat.h" +#include "adf_transport_internal.h" +#include "icp_qat_fw_init_admin.h" + +#define ADF_HB_EMPTY_SIG 0xA5A5A5A5 + +static int adf_hb_check_polling_freq(struct adf_accel_dev *accel_dev) +{ + u64 curr_time = adf_clock_get_current_time(); + u64 polling_time = curr_time - accel_dev->heartbeat->last_hb_check_time; + + if (polling_time < accel_dev->heartbeat->hb_timer) { + dev_warn(&GET_DEV(accel_dev), + "HB polling too frequent. Configured HB timer %d ms\n", + accel_dev->heartbeat->hb_timer); + return -EINVAL; + } + + accel_dev->heartbeat->last_hb_check_time = curr_time; + return 0; +} + +/** + * validate_hb_ctrs_cnt() - checks if the number of heartbeat counters should + * be updated by one to support the currently loaded firmware. + * @accel_dev: Pointer to acceleration device. + * + * Return: + * * true - hb_ctrs must increased by ADF_NUM_PKE_STRAND + * * false - no changes needed + */ +static bool validate_hb_ctrs_cnt(struct adf_accel_dev *accel_dev) +{ + const size_t hb_ctrs = accel_dev->hw_device->num_hb_ctrs; + const size_t max_aes = accel_dev->hw_device->num_engines; + const size_t hb_struct_size = sizeof(struct hb_cnt_pair); + const size_t exp_diff_size = array3_size(ADF_NUM_PKE_STRAND, max_aes, + hb_struct_size); + const size_t dev_ctrs = size_mul(max_aes, hb_ctrs); + const size_t stats_size = size_mul(dev_ctrs, hb_struct_size); + const u32 exp_diff_cnt = exp_diff_size / sizeof(u32); + const u32 stats_el_cnt = stats_size / sizeof(u32); + struct hb_cnt_pair *hb_stats = accel_dev->heartbeat->dma.virt_addr; + const u32 *mem_to_chk = (u32 *)(hb_stats + dev_ctrs); + u32 el_diff_cnt = 0; + int i; + + /* count how many bytes are different from pattern */ + for (i = 0; i < stats_el_cnt; i++) { + if (mem_to_chk[i] == ADF_HB_EMPTY_SIG) + break; + + el_diff_cnt++; + } + + return el_diff_cnt && el_diff_cnt == exp_diff_cnt; +} + +void adf_heartbeat_check_ctrs(struct adf_accel_dev *accel_dev) +{ + struct hb_cnt_pair *hb_stats = accel_dev->heartbeat->dma.virt_addr; + const size_t hb_ctrs = accel_dev->hw_device->num_hb_ctrs; + const size_t max_aes = accel_dev->hw_device->num_engines; + const size_t dev_ctrs = size_mul(max_aes, hb_ctrs); + const size_t stats_size = size_mul(dev_ctrs, sizeof(struct hb_cnt_pair)); + const size_t mem_items_to_fill = size_mul(stats_size, 2) / sizeof(u32); + + /* fill hb stats memory with pattern */ + memset32((uint32_t *)hb_stats, ADF_HB_EMPTY_SIG, mem_items_to_fill); + accel_dev->heartbeat->ctrs_cnt_checked = false; +} +EXPORT_SYMBOL_GPL(adf_heartbeat_check_ctrs); + +static int get_timer_ticks(struct adf_accel_dev *accel_dev, unsigned int *value) +{ + char timer_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { }; + u32 timer_ms = ADF_CFG_HB_TIMER_DEFAULT_MS; + int cfg_read_status; + u32 ticks; + int ret; + + cfg_read_status = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, + ADF_HEARTBEAT_TIMER, timer_str); + if (cfg_read_status == 0) { + if (kstrtouint(timer_str, 10, &timer_ms)) + dev_dbg(&GET_DEV(accel_dev), + "kstrtouint failed to parse the %s, param value", + ADF_HEARTBEAT_TIMER); + } + + if (timer_ms < ADF_CFG_HB_TIMER_MIN_MS) { + dev_err(&GET_DEV(accel_dev), "Timer cannot be less than %u\n", + ADF_CFG_HB_TIMER_MIN_MS); + return -EINVAL; + } + + /* + * On 4xxx devices adf_timer is responsible for HB updates and + * its period is fixed to 200ms + */ + if (accel_dev->timer) + timer_ms = ADF_CFG_HB_TIMER_MIN_MS; + + ret = adf_heartbeat_ms_to_ticks(accel_dev, timer_ms, &ticks); + if (ret) + return ret; + + adf_heartbeat_save_cfg_param(accel_dev, timer_ms); + + accel_dev->heartbeat->hb_timer = timer_ms; + *value = ticks; + + return 0; +} + +static int check_ae(struct hb_cnt_pair *curr, struct hb_cnt_pair *prev, + u16 *count, const size_t hb_ctrs) +{ + size_t thr; + + /* loop through all threads in AE */ + for (thr = 0; thr < hb_ctrs; thr++) { + u16 req = curr[thr].req_heartbeat_cnt; + u16 resp = curr[thr].resp_heartbeat_cnt; + u16 last = prev[thr].resp_heartbeat_cnt; + + if ((thr == ADF_AE_ADMIN_THREAD || req != resp) && resp == last) { + u16 retry = ++count[thr]; + + if (retry >= ADF_CFG_HB_COUNT_THRESHOLD) + return -EIO; + + } else { + count[thr] = 0; + } + } + return 0; +} + +static int adf_hb_get_status(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + struct hb_cnt_pair *live_stats, *last_stats, *curr_stats; + const size_t hb_ctrs = hw_device->num_hb_ctrs; + const unsigned long ae_mask = hw_device->ae_mask; + const size_t max_aes = hw_device->num_engines; + const size_t dev_ctrs = size_mul(max_aes, hb_ctrs); + const size_t stats_size = size_mul(dev_ctrs, sizeof(*curr_stats)); + struct hb_cnt_pair *ae_curr_p, *ae_prev_p; + u16 *count_fails, *ae_count_p; + size_t ae_offset; + size_t ae = 0; + int ret = 0; + + if (!accel_dev->heartbeat->ctrs_cnt_checked) { + if (validate_hb_ctrs_cnt(accel_dev)) + hw_device->num_hb_ctrs += ADF_NUM_PKE_STRAND; + + accel_dev->heartbeat->ctrs_cnt_checked = true; + } + + live_stats = accel_dev->heartbeat->dma.virt_addr; + last_stats = live_stats + dev_ctrs; + count_fails = (u16 *)(last_stats + dev_ctrs); + + curr_stats = kmemdup(live_stats, stats_size, GFP_KERNEL); + if (!curr_stats) + return -ENOMEM; + + /* loop through active AEs */ + for_each_set_bit(ae, &ae_mask, max_aes) { + ae_offset = size_mul(ae, hb_ctrs); + ae_curr_p = curr_stats + ae_offset; + ae_prev_p = last_stats + ae_offset; + ae_count_p = count_fails + ae_offset; + + ret = check_ae(ae_curr_p, ae_prev_p, ae_count_p, hb_ctrs); + if (ret) + break; + } + + /* Copy current stats for the next iteration */ + memcpy(last_stats, curr_stats, stats_size); + kfree(curr_stats); + + return ret; +} + +static void adf_heartbeat_reset(struct adf_accel_dev *accel_dev) +{ + u64 curr_time = adf_clock_get_current_time(); + u64 time_since_reset = curr_time - accel_dev->heartbeat->last_hb_reset_time; + + if (time_since_reset < ADF_CFG_HB_RESET_MS) + return; + + accel_dev->heartbeat->last_hb_reset_time = curr_time; + if (adf_notify_fatal_error(accel_dev)) + dev_err(&GET_DEV(accel_dev), "Failed to notify fatal error\n"); +} + +void adf_heartbeat_status(struct adf_accel_dev *accel_dev, + enum adf_device_heartbeat_status *hb_status) +{ + struct adf_heartbeat *hb; + + if (!adf_dev_started(accel_dev) || + test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) { + *hb_status = HB_DEV_UNRESPONSIVE; + return; + } + + if (adf_hb_check_polling_freq(accel_dev) == -EINVAL) { + *hb_status = HB_DEV_UNSUPPORTED; + return; + } + + hb = accel_dev->heartbeat; + hb->hb_sent_counter++; + + if (adf_hb_get_status(accel_dev)) { + dev_err(&GET_DEV(accel_dev), + "Heartbeat ERROR: QAT is not responding.\n"); + *hb_status = HB_DEV_UNRESPONSIVE; + hb->hb_failed_counter++; + adf_heartbeat_reset(accel_dev); + return; + } + + *hb_status = HB_DEV_ALIVE; +} + +int adf_heartbeat_ms_to_ticks(struct adf_accel_dev *accel_dev, unsigned int time_ms, + u32 *value) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 clk_per_sec; + + /* HB clock may be different than AE clock */ + if (!hw_data->get_hb_clock) + return -EINVAL; + + clk_per_sec = hw_data->get_hb_clock(hw_data); + *value = time_ms * (clk_per_sec / MSEC_PER_SEC); + + return 0; +} + +int adf_heartbeat_save_cfg_param(struct adf_accel_dev *accel_dev, + unsigned int timer_ms) +{ + char timer_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + + snprintf(timer_str, sizeof(timer_str), "%u", timer_ms); + return adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC, + ADF_HEARTBEAT_TIMER, timer_str, + ADF_STR); +} +EXPORT_SYMBOL_GPL(adf_heartbeat_save_cfg_param); + +int adf_heartbeat_init(struct adf_accel_dev *accel_dev) +{ + struct adf_heartbeat *hb; + + hb = kzalloc_obj(*hb); + if (!hb) + goto err_ret; + + hb->dma.virt_addr = dma_alloc_coherent(&GET_DEV(accel_dev), PAGE_SIZE, + &hb->dma.phy_addr, GFP_KERNEL); + if (!hb->dma.virt_addr) + goto err_free; + + /* + * Default set this flag as true to avoid unnecessary checks, + * it will be reset on platforms that need such a check + */ + hb->ctrs_cnt_checked = true; + accel_dev->heartbeat = hb; + + return 0; + +err_free: + kfree(hb); +err_ret: + return -ENOMEM; +} + +int adf_heartbeat_start(struct adf_accel_dev *accel_dev) +{ + unsigned int timer_ticks; + int ret; + + if (!accel_dev->heartbeat) { + dev_warn(&GET_DEV(accel_dev), "Heartbeat instance not found!"); + return -EFAULT; + } + + if (accel_dev->hw_device->check_hb_ctrs) + accel_dev->hw_device->check_hb_ctrs(accel_dev); + + ret = get_timer_ticks(accel_dev, &timer_ticks); + if (ret) + return ret; + + ret = adf_send_admin_hb_timer(accel_dev, timer_ticks); + if (ret) + dev_warn(&GET_DEV(accel_dev), "Heartbeat not supported!"); + + return ret; +} + +void adf_heartbeat_shutdown(struct adf_accel_dev *accel_dev) +{ + struct adf_heartbeat *hb = accel_dev->heartbeat; + + if (!hb) + return; + + if (hb->dma.virt_addr) + dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, + hb->dma.virt_addr, hb->dma.phy_addr); + + kfree(hb); + accel_dev->heartbeat = NULL; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_heartbeat.h b/drivers/crypto/intel/qat/qat_common/adf_heartbeat.h new file mode 100644 index 000000000..16fdfb48b --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_heartbeat.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ + +#ifndef ADF_HEARTBEAT_H_ +#define ADF_HEARTBEAT_H_ + +#include + +struct adf_accel_dev; +struct dentry; + +#define ADF_CFG_HB_TIMER_MIN_MS 200 +#define ADF_CFG_HB_TIMER_DEFAULT_MS 500 +#define ADF_CFG_HB_COUNT_THRESHOLD 3 + +#define ADF_CFG_HB_RESET_MS 5000 + +enum adf_device_heartbeat_status { + HB_DEV_UNRESPONSIVE = 0, + HB_DEV_ALIVE, + HB_DEV_UNSUPPORTED, +}; + +/* Heartbeat counter pair */ +struct hb_cnt_pair { + __u16 resp_heartbeat_cnt; + __u16 req_heartbeat_cnt; +}; + +struct adf_heartbeat { + unsigned int hb_sent_counter; + unsigned int hb_failed_counter; + unsigned int hb_timer; + u64 last_hb_check_time; + u64 last_hb_reset_time; + bool ctrs_cnt_checked; + struct hb_dma_addr { + dma_addr_t phy_addr; + void *virt_addr; + } dma; + struct { + struct dentry *base_dir; + struct dentry *status; + struct dentry *cfg; + struct dentry *sent; + struct dentry *failed; +#ifdef CONFIG_CRYPTO_DEV_QAT_ERROR_INJECTION + struct dentry *inject_error; +#endif + } dbgfs; +}; + +#ifdef CONFIG_DEBUG_FS +int adf_heartbeat_init(struct adf_accel_dev *accel_dev); +int adf_heartbeat_start(struct adf_accel_dev *accel_dev); +void adf_heartbeat_shutdown(struct adf_accel_dev *accel_dev); + +int adf_heartbeat_ms_to_ticks(struct adf_accel_dev *accel_dev, unsigned int time_ms, + uint32_t *value); +int adf_heartbeat_save_cfg_param(struct adf_accel_dev *accel_dev, + unsigned int timer_ms); +void adf_heartbeat_status(struct adf_accel_dev *accel_dev, + enum adf_device_heartbeat_status *hb_status); +void adf_heartbeat_check_ctrs(struct adf_accel_dev *accel_dev); + +#ifdef CONFIG_CRYPTO_DEV_QAT_ERROR_INJECTION +int adf_heartbeat_inject_error(struct adf_accel_dev *accel_dev); +#else +static inline int adf_heartbeat_inject_error(struct adf_accel_dev *accel_dev) +{ + return -EPERM; +} +#endif + +#else +static inline int adf_heartbeat_init(struct adf_accel_dev *accel_dev) +{ + return 0; +} + +static inline int adf_heartbeat_start(struct adf_accel_dev *accel_dev) +{ + return 0; +} + +static inline void adf_heartbeat_shutdown(struct adf_accel_dev *accel_dev) +{ +} + +static inline int adf_heartbeat_save_cfg_param(struct adf_accel_dev *accel_dev, + unsigned int timer_ms) +{ + return 0; +} + +static inline void adf_heartbeat_check_ctrs(struct adf_accel_dev *accel_dev) +{ +} +#endif +#endif /* ADF_HEARTBEAT_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_heartbeat_dbgfs.c b/drivers/crypto/intel/qat/qat_common/adf_heartbeat_dbgfs.c new file mode 100644 index 000000000..cccdff24b --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_heartbeat_dbgfs.c @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ + +#include +#include +#include +#include +#include +#include +#include +#include "adf_admin.h" +#include "adf_cfg.h" +#include "adf_common_drv.h" +#include "adf_heartbeat.h" +#include "adf_heartbeat_dbgfs.h" + +#define HB_OK 0 +#define HB_ERROR -1 +#define HB_STATUS_MAX_STRLEN 4 +#define HB_STATS_MAX_STRLEN 16 + +static ssize_t adf_hb_stats_read(struct file *file, char __user *user_buffer, + size_t count, loff_t *ppos) +{ + char buf[HB_STATS_MAX_STRLEN]; + unsigned int *value; + int len; + + if (*ppos > 0) + return 0; + + value = file->private_data; + len = scnprintf(buf, sizeof(buf), "%u\n", *value); + + return simple_read_from_buffer(user_buffer, count, ppos, buf, len + 1); +} + +static const struct file_operations adf_hb_stats_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = adf_hb_stats_read, +}; + +static ssize_t adf_hb_status_read(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + enum adf_device_heartbeat_status hb_status; + char ret_str[HB_STATUS_MAX_STRLEN]; + struct adf_accel_dev *accel_dev; + int ret_code; + size_t len; + + if (*ppos > 0) + return 0; + + accel_dev = file->private_data; + ret_code = HB_OK; + + adf_heartbeat_status(accel_dev, &hb_status); + + if (hb_status != HB_DEV_ALIVE) + ret_code = HB_ERROR; + + len = scnprintf(ret_str, sizeof(ret_str), "%d\n", ret_code); + + return simple_read_from_buffer(user_buf, count, ppos, ret_str, len + 1); +} + +static const struct file_operations adf_hb_status_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = adf_hb_status_read, +}; + +static ssize_t adf_hb_cfg_read(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + char timer_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + struct adf_accel_dev *accel_dev; + unsigned int timer_ms; + int len; + + if (*ppos > 0) + return 0; + + accel_dev = file->private_data; + timer_ms = accel_dev->heartbeat->hb_timer; + len = scnprintf(timer_str, sizeof(timer_str), "%u\n", timer_ms); + + return simple_read_from_buffer(user_buf, count, ppos, timer_str, + len + 1); +} + +static ssize_t adf_hb_cfg_write(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + char input_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { }; + struct adf_accel_dev *accel_dev; + int ret, written_chars; + unsigned int timer_ms; + u32 ticks; + + accel_dev = file->private_data; + timer_ms = ADF_CFG_HB_TIMER_DEFAULT_MS; + + /* last byte left as string termination */ + if (count > sizeof(input_str) - 1) + return -EINVAL; + + written_chars = simple_write_to_buffer(input_str, sizeof(input_str) - 1, + ppos, user_buf, count); + if (written_chars > 0) { + ret = kstrtouint(input_str, 10, &timer_ms); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "heartbeat_cfg: Invalid value\n"); + return ret; + } + + if (timer_ms < ADF_CFG_HB_TIMER_MIN_MS) { + dev_err(&GET_DEV(accel_dev), + "heartbeat_cfg: Invalid value\n"); + return -EINVAL; + } + + /* + * On 4xxx devices adf_timer is responsible for HB updates and + * its period is fixed to 200ms + */ + if (accel_dev->timer) + timer_ms = ADF_CFG_HB_TIMER_MIN_MS; + + ret = adf_heartbeat_save_cfg_param(accel_dev, timer_ms); + if (ret) + return ret; + + ret = adf_heartbeat_ms_to_ticks(accel_dev, timer_ms, &ticks); + if (ret) + return ret; + + ret = adf_send_admin_hb_timer(accel_dev, ticks); + if (ret) + return ret; + + accel_dev->heartbeat->hb_timer = timer_ms; + } + + return written_chars; +} + +static const struct file_operations adf_hb_cfg_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = adf_hb_cfg_read, + .write = adf_hb_cfg_write, +}; + +static ssize_t adf_hb_error_inject_write(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct adf_accel_dev *accel_dev = file->private_data; + char buf[3]; + int ret; + + /* last byte left as string termination */ + if (*ppos != 0 || count != 2) + return -EINVAL; + + if (copy_from_user(buf, user_buf, count)) + return -EFAULT; + buf[count] = '\0'; + + if (buf[0] != '1') + return -EINVAL; + + ret = adf_heartbeat_inject_error(accel_dev); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Heartbeat error injection failed with status %d\n", + ret); + return ret; + } + + dev_info(&GET_DEV(accel_dev), "Heartbeat error injection enabled\n"); + + return count; +} + +static const struct file_operations adf_hb_error_inject_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = adf_hb_error_inject_write, +}; + +void adf_heartbeat_dbgfs_add(struct adf_accel_dev *accel_dev) +{ + struct adf_heartbeat *hb = accel_dev->heartbeat; + + if (!hb) + return; + + hb->dbgfs.base_dir = debugfs_create_dir("heartbeat", accel_dev->debugfs_dir); + hb->dbgfs.status = debugfs_create_file("status", 0400, hb->dbgfs.base_dir, + accel_dev, &adf_hb_status_fops); + hb->dbgfs.sent = debugfs_create_file("queries_sent", 0400, hb->dbgfs.base_dir, + &hb->hb_sent_counter, &adf_hb_stats_fops); + hb->dbgfs.failed = debugfs_create_file("queries_failed", 0400, hb->dbgfs.base_dir, + &hb->hb_failed_counter, &adf_hb_stats_fops); + hb->dbgfs.cfg = debugfs_create_file("config", 0600, hb->dbgfs.base_dir, + accel_dev, &adf_hb_cfg_fops); + + if (IS_ENABLED(CONFIG_CRYPTO_DEV_QAT_ERROR_INJECTION)) { + struct dentry *inject_error __maybe_unused; + + inject_error = debugfs_create_file("inject_error", 0200, + hb->dbgfs.base_dir, accel_dev, + &adf_hb_error_inject_fops); +#ifdef CONFIG_CRYPTO_DEV_QAT_ERROR_INJECTION + hb->dbgfs.inject_error = inject_error; +#endif + } +} +EXPORT_SYMBOL_GPL(adf_heartbeat_dbgfs_add); + +void adf_heartbeat_dbgfs_rm(struct adf_accel_dev *accel_dev) +{ + struct adf_heartbeat *hb = accel_dev->heartbeat; + + if (!hb) + return; + + debugfs_remove(hb->dbgfs.status); + hb->dbgfs.status = NULL; + debugfs_remove(hb->dbgfs.sent); + hb->dbgfs.sent = NULL; + debugfs_remove(hb->dbgfs.failed); + hb->dbgfs.failed = NULL; + debugfs_remove(hb->dbgfs.cfg); + hb->dbgfs.cfg = NULL; +#ifdef CONFIG_CRYPTO_DEV_QAT_ERROR_INJECTION + debugfs_remove(hb->dbgfs.inject_error); + hb->dbgfs.inject_error = NULL; +#endif + debugfs_remove(hb->dbgfs.base_dir); + hb->dbgfs.base_dir = NULL; +} +EXPORT_SYMBOL_GPL(adf_heartbeat_dbgfs_rm); diff --git a/drivers/crypto/intel/qat/qat_common/adf_heartbeat_dbgfs.h b/drivers/crypto/intel/qat/qat_common/adf_heartbeat_dbgfs.h new file mode 100644 index 000000000..84dd29ea6 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_heartbeat_dbgfs.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ + +#ifndef ADF_HEARTBEAT_DBGFS_H_ +#define ADF_HEARTBEAT_DBGFS_H_ + +struct adf_accel_dev; + +void adf_heartbeat_dbgfs_add(struct adf_accel_dev *accel_dev); +void adf_heartbeat_dbgfs_rm(struct adf_accel_dev *accel_dev); + +#endif /* ADF_HEARTBEAT_DBGFS_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_heartbeat_inject.c b/drivers/crypto/intel/qat/qat_common/adf_heartbeat_inject.c new file mode 100644 index 000000000..023c5f1e7 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_heartbeat_inject.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ +#include + +#include "adf_admin.h" +#include "adf_common_drv.h" +#include "adf_heartbeat.h" + +#define MAX_HB_TICKS 0xFFFFFFFF + +static int adf_hb_set_timer_to_max(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + + accel_dev->heartbeat->hb_timer = 0; + + if (hw_data->stop_timer) + hw_data->stop_timer(accel_dev); + + return adf_send_admin_hb_timer(accel_dev, MAX_HB_TICKS); +} + +static void adf_set_hb_counters_fail(struct adf_accel_dev *accel_dev, u32 ae, + u32 thr) +{ + struct hb_cnt_pair *stats = accel_dev->heartbeat->dma.virt_addr; + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + const size_t max_aes = hw_device->get_num_aes(hw_device); + const size_t hb_ctrs = hw_device->num_hb_ctrs; + size_t thr_id = ae * hb_ctrs + thr; + u16 num_rsp = stats[thr_id].resp_heartbeat_cnt; + + /* + * Inject live.req != live.rsp and live.rsp == last.rsp + * to trigger the heartbeat error detection + */ + stats[thr_id].req_heartbeat_cnt++; + stats += (max_aes * hb_ctrs); + stats[thr_id].resp_heartbeat_cnt = num_rsp; +} + +int adf_heartbeat_inject_error(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + const size_t max_aes = hw_device->get_num_aes(hw_device); + const size_t hb_ctrs = hw_device->num_hb_ctrs; + u32 rand, rand_ae, rand_thr; + unsigned long ae_mask; + int ret; + + ae_mask = hw_device->ae_mask; + + do { + /* Ensure we have a valid ae */ + get_random_bytes(&rand, sizeof(rand)); + rand_ae = rand % max_aes; + } while (!test_bit(rand_ae, &ae_mask)); + + get_random_bytes(&rand, sizeof(rand)); + rand_thr = rand % hb_ctrs; + + /* Increase the heartbeat timer to prevent FW updating HB counters */ + ret = adf_hb_set_timer_to_max(accel_dev); + if (ret) + return ret; + + /* Disable arbiter to stop processing any packet */ + hw_device->exit_arb(accel_dev); + + /* Change HB counters memory to simulate a hang */ + adf_set_hb_counters_fail(accel_dev, rand_ae, rand_thr); + + return 0; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_hw_arbiter.c b/drivers/crypto/intel/qat/qat_common/adf_hw_arbiter.c new file mode 100644 index 000000000..dd9a31c20 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_hw_arbiter.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_transport_internal.h" + +#define ADF_ARB_NUM 4 +#define ADF_ARB_REG_SIZE 0x4 + +#define WRITE_CSR_ARB_SARCONFIG(csr_addr, arb_offset, index, value) \ + ADF_CSR_WR(csr_addr, (arb_offset) + \ + (ADF_ARB_REG_SIZE * (index)), value) + +#define WRITE_CSR_ARB_WT2SAM(csr_addr, arb_offset, wt_offset, index, value) \ + ADF_CSR_WR(csr_addr, ((arb_offset) + (wt_offset)) + \ + (ADF_ARB_REG_SIZE * (index)), value) + +int adf_init_arb(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *csr = accel_dev->transport->banks[0].csr_addr; + unsigned long ae_mask = hw_data->ae_mask; + u32 arb_off, wt_off, arb_cfg; + const u32 *thd_2_arb_cfg; + struct arb_info info; + int arb, i; + + hw_data->get_arb_info(&info); + arb_cfg = info.arb_cfg; + arb_off = info.arb_offset; + wt_off = info.wt2sam_offset; + + /* Service arb configured for 32 bytes responses and + * ring flow control check enabled. */ + for (arb = 0; arb < ADF_ARB_NUM; arb++) + WRITE_CSR_ARB_SARCONFIG(csr, arb_off, arb, arb_cfg); + + /* Map worker threads to service arbiters */ + thd_2_arb_cfg = hw_data->get_arb_mapping(accel_dev); + + for_each_set_bit(i, &ae_mask, hw_data->num_engines) + WRITE_CSR_ARB_WT2SAM(csr, arb_off, wt_off, i, thd_2_arb_cfg[i]); + + return 0; +} +EXPORT_SYMBOL_GPL(adf_init_arb); + +void adf_update_ring_arb(struct adf_etr_ring_data *ring) +{ + struct adf_accel_dev *accel_dev = ring->bank->accel_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); + u32 tx_ring_mask = hw_data->tx_rings_mask; + u32 shift = hw_data->tx_rx_gap; + u32 arben, arben_tx, arben_rx; + u32 rx_ring_mask; + + /* + * Enable arbitration on a ring only if the TX half of the ring mask + * matches the RX part. This results in writes to CSR on both TX and + * RX update - only one is necessary, but both are done for + * simplicity. + */ + rx_ring_mask = tx_ring_mask << shift; + arben_tx = (ring->bank->ring_mask & tx_ring_mask) >> 0; + arben_rx = (ring->bank->ring_mask & rx_ring_mask) >> shift; + arben = arben_tx & arben_rx; + + csr_ops->write_csr_ring_srv_arb_en(ring->bank->csr_addr, + ring->bank->bank_number, arben); +} + +void adf_exit_arb(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); + u32 arb_off, wt_off; + struct arb_info info; + void __iomem *csr; + unsigned int i; + + hw_data->get_arb_info(&info); + arb_off = info.arb_offset; + wt_off = info.wt2sam_offset; + + if (!accel_dev->transport) + return; + + csr = accel_dev->transport->banks[0].csr_addr; + + hw_data->get_arb_info(&info); + + /* Unmap worker threads to service arbiters */ + for (i = 0; i < hw_data->num_engines; i++) + WRITE_CSR_ARB_WT2SAM(csr, arb_off, wt_off, i, 0); + + /* Disable arbitration on all rings */ + for (i = 0; i < GET_MAX_BANKS(accel_dev); i++) + csr_ops->write_csr_ring_srv_arb_en(csr, i, 0); +} +EXPORT_SYMBOL_GPL(adf_exit_arb); diff --git a/drivers/crypto/intel/qat/qat_common/adf_init.c b/drivers/crypto/intel/qat/qat_common/adf_init.c new file mode 100644 index 000000000..3e39c5381 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_init.c @@ -0,0 +1,547 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#include +#include +#include +#include +#include "adf_accel_devices.h" +#include "adf_cfg.h" +#include "adf_common_drv.h" +#include "adf_dbgfs.h" +#include "adf_heartbeat.h" +#include "adf_rl.h" +#include "adf_kpt.h" +#include "adf_sysfs_anti_rb.h" +#include "adf_sysfs_ras_counters.h" +#include "adf_telemetry.h" + +static LIST_HEAD(service_table); +static DEFINE_MUTEX(service_lock); + +static void adf_service_add(struct service_hndl *service) +{ + mutex_lock(&service_lock); + list_add(&service->list, &service_table); + mutex_unlock(&service_lock); +} + +int adf_service_register(struct service_hndl *service) +{ + memset(service->init_status, 0, sizeof(service->init_status)); + memset(service->start_status, 0, sizeof(service->start_status)); + adf_service_add(service); + return 0; +} + +static void adf_service_remove(struct service_hndl *service) +{ + mutex_lock(&service_lock); + list_del(&service->list); + mutex_unlock(&service_lock); +} + +int adf_service_unregister(struct service_hndl *service) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(service->init_status); i++) { + if (service->init_status[i] || service->start_status[i]) { + pr_err("QAT: Could not remove active service\n"); + return -EFAULT; + } + } + adf_service_remove(service); + return 0; +} + +/** + * adf_dev_init() - Init data structures and services for the given accel device + * @accel_dev: Pointer to acceleration device. + * + * Initialize the ring data structures and the admin comms and arbitration + * services. + * + * Return: 0 on success, error code otherwise. + */ +static int adf_dev_init(struct adf_accel_dev *accel_dev) +{ + struct service_hndl *service; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + int ret; + + if (!hw_data) { + dev_err(&GET_DEV(accel_dev), + "Failed to init device - hw_data not set\n"); + return -EFAULT; + } + + adf_set_bme(accel_dev); + + if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status) && + !accel_dev->is_vf) { + dev_err(&GET_DEV(accel_dev), "Device not configured\n"); + return -EFAULT; + } + + if (adf_init_etr_data(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Failed initialize etr\n"); + return -EFAULT; + } + + if (hw_data->init_device && hw_data->init_device(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Failed to initialize device\n"); + return -EFAULT; + } + + if (hw_data->init_admin_comms && hw_data->init_admin_comms(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Failed initialize admin comms\n"); + return -EFAULT; + } + + if (hw_data->init_arb && hw_data->init_arb(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Failed initialize hw arbiter\n"); + return -EFAULT; + } + + if (hw_data->get_ring_to_svc_map) + hw_data->ring_to_svc_map = hw_data->get_ring_to_svc_map(accel_dev); + + if (adf_ae_init(accel_dev)) { + dev_err(&GET_DEV(accel_dev), + "Failed to initialise Acceleration Engine\n"); + return -EFAULT; + } + set_bit(ADF_STATUS_AE_INITIALISED, &accel_dev->status); + + if (adf_ae_fw_load(accel_dev)) { + dev_err(&GET_DEV(accel_dev), + "Failed to load acceleration FW\n"); + return -EFAULT; + } + set_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status); + + if (hw_data->alloc_irq(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Failed to allocate interrupts\n"); + return -EFAULT; + } + set_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status); + + if (hw_data->ras_ops.enable_ras_errors) + hw_data->ras_ops.enable_ras_errors(accel_dev); + + hw_data->enable_ints(accel_dev); + hw_data->enable_error_correction(accel_dev); + + ret = hw_data->pfvf_ops.enable_comms(accel_dev); + if (ret) + return ret; + + if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status) && + accel_dev->is_vf) { + if (qat_crypto_vf_dev_config(accel_dev)) + return -EFAULT; + } + + adf_heartbeat_init(accel_dev); + ret = adf_rl_init(accel_dev); + if (ret && ret != -EOPNOTSUPP) + return ret; + + ret = adf_tl_init(accel_dev); + if (ret && ret != -EOPNOTSUPP) + return ret; + + /* + * Subservice initialisation is divided into two stages: init and start. + * This is to facilitate any ordering dependencies between services + * prior to starting any of the accelerators. + */ + mutex_lock(&service_lock); + list_for_each_entry(service, &service_table, list) { + if (service->event_hld(accel_dev, ADF_EVENT_INIT)) { + dev_err(&GET_DEV(accel_dev), + "Failed to initialise service %s\n", + service->name); + mutex_unlock(&service_lock); + return -EFAULT; + } + set_bit(accel_dev->accel_id, service->init_status); + } + mutex_unlock(&service_lock); + + return 0; +} + +/** + * adf_dev_start() - Start acceleration service for the given accel device + * @accel_dev: Pointer to acceleration device. + * + * Function notifies all the registered services that the acceleration device + * is ready to be used. + * To be used by QAT device specific drivers. + * + * Return: 0 on success, error code otherwise. + */ +static int adf_dev_start(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct service_hndl *service; + u32 caps; + int ret; + + set_bit(ADF_STATUS_STARTING, &accel_dev->status); + + if (adf_ae_start(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "AE Start Failed\n"); + return -EFAULT; + } + set_bit(ADF_STATUS_AE_STARTED, &accel_dev->status); + + if (hw_data->send_admin_init(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Failed to send init message\n"); + return -EFAULT; + } + + if (hw_data->measure_clock) { + ret = hw_data->measure_clock(accel_dev); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Failed measure device clock\n"); + return ret; + } + } + + /* Set ssm watch dog timer */ + if (hw_data->set_ssm_wdtimer) + hw_data->set_ssm_wdtimer(accel_dev); + + /* Enable Power Management */ + if (hw_data->enable_pm && hw_data->enable_pm(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Failed to configure Power Management\n"); + return -EFAULT; + } + + /* Enable Key Protection Technology (KPT) */ + ret = adf_enable_kpt(accel_dev); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Failed to enable KPT\n"); + return ret; + } + + if (hw_data->start_timer) { + ret = hw_data->start_timer(accel_dev); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Failed to start internal sync timer\n"); + return ret; + } + } + + adf_heartbeat_start(accel_dev); + ret = adf_rl_start(accel_dev); + if (ret && ret != -EOPNOTSUPP) + return ret; + + ret = adf_tl_start(accel_dev); + if (ret && ret != -EOPNOTSUPP) + return ret; + + mutex_lock(&service_lock); + list_for_each_entry(service, &service_table, list) { + if (service->event_hld(accel_dev, ADF_EVENT_START)) { + dev_err(&GET_DEV(accel_dev), + "Failed to start service %s\n", + service->name); + mutex_unlock(&service_lock); + return -EFAULT; + } + set_bit(accel_dev->accel_id, service->start_status); + } + mutex_unlock(&service_lock); + + clear_bit(ADF_STATUS_STARTING, &accel_dev->status); + set_bit(ADF_STATUS_STARTED, &accel_dev->status); + + if (!list_empty(&accel_dev->crypto_list) && + (qat_algs_register() || qat_asym_algs_register())) { + dev_err(&GET_DEV(accel_dev), + "Failed to register crypto algs\n"); + set_bit(ADF_STATUS_STARTING, &accel_dev->status); + clear_bit(ADF_STATUS_STARTED, &accel_dev->status); + return -EFAULT; + } + set_bit(ADF_STATUS_CRYPTO_ALGS_REGISTERED, &accel_dev->status); + + caps = hw_data->accel_capabilities_ext_mask; + if (!list_empty(&accel_dev->compression_list) && qat_comp_algs_register(caps)) { + dev_err(&GET_DEV(accel_dev), + "Failed to register compression algs\n"); + set_bit(ADF_STATUS_STARTING, &accel_dev->status); + clear_bit(ADF_STATUS_STARTED, &accel_dev->status); + return -EFAULT; + } + set_bit(ADF_STATUS_COMP_ALGS_REGISTERED, &accel_dev->status); + + adf_dbgfs_add(accel_dev); + adf_sysfs_start_ras(accel_dev); + adf_sysfs_start_arb(accel_dev); + + return 0; +} + +/** + * adf_dev_stop() - Stop acceleration service for the given accel device + * @accel_dev: Pointer to acceleration device. + * + * Function notifies all the registered services that the acceleration device + * is shuting down. + * To be used by QAT device specific drivers. + * + * Return: void + */ +static void adf_dev_stop(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct service_hndl *service; + bool wait = false; + int ret; + + if (!adf_dev_started(accel_dev) && + !test_bit(ADF_STATUS_STARTING, &accel_dev->status)) + return; + + adf_tl_stop(accel_dev); + adf_rl_stop(accel_dev); + adf_dbgfs_rm(accel_dev); + adf_sysfs_stop_ras(accel_dev); + adf_sysfs_stop_arb(accel_dev); + + clear_bit(ADF_STATUS_STARTING, &accel_dev->status); + clear_bit(ADF_STATUS_STARTED, &accel_dev->status); + + if (!list_empty(&accel_dev->crypto_list) && + test_bit(ADF_STATUS_CRYPTO_ALGS_REGISTERED, &accel_dev->status)) { + qat_algs_unregister(); + qat_asym_algs_unregister(); + } + clear_bit(ADF_STATUS_CRYPTO_ALGS_REGISTERED, &accel_dev->status); + + if (!list_empty(&accel_dev->compression_list) && + test_bit(ADF_STATUS_COMP_ALGS_REGISTERED, &accel_dev->status)) + qat_comp_algs_unregister(hw_data->accel_capabilities_ext_mask); + clear_bit(ADF_STATUS_COMP_ALGS_REGISTERED, &accel_dev->status); + + mutex_lock(&service_lock); + list_for_each_entry(service, &service_table, list) { + if (!test_bit(accel_dev->accel_id, service->start_status)) + continue; + ret = service->event_hld(accel_dev, ADF_EVENT_STOP); + if (!ret) { + clear_bit(accel_dev->accel_id, service->start_status); + } else if (ret == -EAGAIN) { + wait = true; + clear_bit(accel_dev->accel_id, service->start_status); + } + } + mutex_unlock(&service_lock); + + if (hw_data->stop_timer) + hw_data->stop_timer(accel_dev); + + hw_data->disable_iov(accel_dev); + + if (wait) + msleep(100); + + if (test_bit(ADF_STATUS_AE_STARTED, &accel_dev->status)) { + if (adf_ae_stop(accel_dev)) + dev_err(&GET_DEV(accel_dev), "failed to stop AE\n"); + else + clear_bit(ADF_STATUS_AE_STARTED, &accel_dev->status); + } +} + +/** + * adf_dev_shutdown() - shutdown acceleration services and data strucutures + * @accel_dev: Pointer to acceleration device + * + * Cleanup the ring data structures and the admin comms and arbitration + * services. + */ +static void adf_dev_shutdown(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct service_hndl *service; + + if (!hw_data) { + dev_err(&GET_DEV(accel_dev), + "QAT: Failed to shutdown device - hw_data not set\n"); + return; + } + + if (test_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status)) { + adf_ae_fw_release(accel_dev); + clear_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status); + } + + if (test_bit(ADF_STATUS_AE_INITIALISED, &accel_dev->status)) { + if (adf_ae_shutdown(accel_dev)) + dev_err(&GET_DEV(accel_dev), + "Failed to shutdown Accel Engine\n"); + else + clear_bit(ADF_STATUS_AE_INITIALISED, + &accel_dev->status); + } + + mutex_lock(&service_lock); + list_for_each_entry(service, &service_table, list) { + if (!test_bit(accel_dev->accel_id, service->init_status)) + continue; + if (service->event_hld(accel_dev, ADF_EVENT_SHUTDOWN)) + dev_err(&GET_DEV(accel_dev), + "Failed to shutdown service %s\n", + service->name); + else + clear_bit(accel_dev->accel_id, service->init_status); + } + mutex_unlock(&service_lock); + + adf_rl_exit(accel_dev); + + if (hw_data->ras_ops.disable_ras_errors) + hw_data->ras_ops.disable_ras_errors(accel_dev); + + adf_heartbeat_shutdown(accel_dev); + + adf_tl_shutdown(accel_dev); + + if (test_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status)) { + hw_data->free_irq(accel_dev); + clear_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status); + } + + /* If not restarting, delete all cfg sections except for GENERAL */ + if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) + adf_cfg_del_all_except(accel_dev, ADF_GENERAL_SEC); + + if (hw_data->exit_arb) + hw_data->exit_arb(accel_dev); + + if (hw_data->exit_admin_comms) + hw_data->exit_admin_comms(accel_dev); + + adf_cleanup_etr_data(accel_dev); + adf_misc_wq_flush(); + adf_dev_restore(accel_dev); +} + +int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev) +{ + struct service_hndl *service; + + mutex_lock(&service_lock); + list_for_each_entry(service, &service_table, list) { + if (service->event_hld(accel_dev, ADF_EVENT_RESTARTING)) + dev_err(&GET_DEV(accel_dev), + "Failed to restart service %s.\n", + service->name); + } + mutex_unlock(&service_lock); + return 0; +} + +int adf_dev_restarted_notify(struct adf_accel_dev *accel_dev) +{ + struct service_hndl *service; + + mutex_lock(&service_lock); + list_for_each_entry(service, &service_table, list) { + if (service->event_hld(accel_dev, ADF_EVENT_RESTARTED)) + dev_err(&GET_DEV(accel_dev), + "Failed to restart service %s.\n", + service->name); + } + mutex_unlock(&service_lock); + return 0; +} + +void adf_error_notifier(struct adf_accel_dev *accel_dev) +{ + struct service_hndl *service; + + mutex_lock(&service_lock); + list_for_each_entry(service, &service_table, list) { + if (service->event_hld(accel_dev, ADF_EVENT_FATAL_ERROR)) + dev_err(&GET_DEV(accel_dev), + "Failed to send error event to %s.\n", + service->name); + } + mutex_unlock(&service_lock); +} + +int adf_dev_down(struct adf_accel_dev *accel_dev) +{ + int ret = 0; + + if (!accel_dev) + return -EINVAL; + + mutex_lock(&accel_dev->state_lock); + + adf_dev_stop(accel_dev); + adf_dev_shutdown(accel_dev); + + mutex_unlock(&accel_dev->state_lock); + return ret; +} +EXPORT_SYMBOL_GPL(adf_dev_down); + +int adf_dev_up(struct adf_accel_dev *accel_dev, bool config) +{ + int ret = 0; + + if (!accel_dev) + return -EINVAL; + + mutex_lock(&accel_dev->state_lock); + + if (adf_dev_started(accel_dev)) { + dev_info(&GET_DEV(accel_dev), "Device qat_dev%d already up\n", + accel_dev->accel_id); + ret = -EALREADY; + goto out; + } + + if (config && GET_HW_DATA(accel_dev)->dev_config) { + ret = GET_HW_DATA(accel_dev)->dev_config(accel_dev); + if (unlikely(ret)) + goto out; + } + + ret = adf_dev_init(accel_dev); + if (unlikely(ret)) + goto out; + + ret = adf_dev_start(accel_dev); + +out: + mutex_unlock(&accel_dev->state_lock); + return ret; +} +EXPORT_SYMBOL_GPL(adf_dev_up); + +int adf_dev_restart(struct adf_accel_dev *accel_dev) +{ + int ret = 0; + + if (!accel_dev) + return -EFAULT; + + adf_dev_down(accel_dev); + + ret = adf_dev_up(accel_dev, false); + /* if device is already up return success*/ + if (ret == -EALREADY) + return 0; + + return ret; +} +EXPORT_SYMBOL_GPL(adf_dev_restart); diff --git a/drivers/crypto/intel/qat/qat_common/adf_isr.c b/drivers/crypto/intel/qat/qat_common/adf_isr.c new file mode 100644 index 000000000..159e91a50 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_isr.c @@ -0,0 +1,454 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_cfg.h" +#include "adf_cfg_strings.h" +#include "adf_cfg_common.h" +#include "adf_transport_access_macros.h" +#include "adf_transport_internal.h" + +#define ADF_MAX_NUM_VFS 32 +static struct workqueue_struct *adf_misc_wq; + +static int adf_enable_msix(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 msix_num_entries = hw_data->num_banks + 1; + int ret; + + if (hw_data->set_msix_rttable) + hw_data->set_msix_rttable(accel_dev); + + ret = pci_alloc_irq_vectors(pci_dev_info->pci_dev, msix_num_entries, + msix_num_entries, PCI_IRQ_MSIX); + if (unlikely(ret < 0)) { + dev_err(&GET_DEV(accel_dev), + "Failed to allocate %d MSI-X vectors\n", + msix_num_entries); + return ret; + } + return 0; +} + +static void adf_disable_msix(struct adf_accel_pci *pci_dev_info) +{ + pci_free_irq_vectors(pci_dev_info->pci_dev); +} + +static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr) +{ + struct adf_etr_bank_data *bank = bank_ptr; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); + + csr_ops->write_csr_int_flag_and_col(bank->csr_addr, bank->bank_number, + 0); + tasklet_hi_schedule(&bank->resp_handler); + return IRQ_HANDLED; +} + +#ifdef CONFIG_PCI_IOV +void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask) +{ + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + unsigned long flags; + + spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags); + if (!READ_ONCE(accel_dev->pf.vf2pf_disabled)) + GET_PFVF_OPS(accel_dev)->enable_vf2pf_interrupts(pmisc_addr, vf_mask); + spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags); +} + +void adf_enable_all_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 num_vfs) +{ + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + unsigned long flags; + u32 vf_mask; + + vf_mask = BIT_ULL(num_vfs) - 1; + if (!vf_mask) + return; + + spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags); + WRITE_ONCE(accel_dev->pf.vf2pf_disabled, false); + GET_PFVF_OPS(accel_dev)->enable_vf2pf_interrupts(pmisc_addr, vf_mask); + spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags); +} + +void adf_disable_all_vf2pf_interrupts(struct adf_accel_dev *accel_dev) +{ + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + unsigned long flags; + + spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags); + WRITE_ONCE(accel_dev->pf.vf2pf_disabled, true); + GET_PFVF_OPS(accel_dev)->disable_all_vf2pf_interrupts(pmisc_addr); + spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags); +} + +static u32 adf_disable_pending_vf2pf_interrupts(struct adf_accel_dev *accel_dev) +{ + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + u32 pending; + + spin_lock(&accel_dev->pf.vf2pf_ints_lock); + pending = GET_PFVF_OPS(accel_dev)->disable_pending_vf2pf_interrupts(pmisc_addr); + spin_unlock(&accel_dev->pf.vf2pf_ints_lock); + + return pending; +} + +static bool adf_handle_vf2pf_int(struct adf_accel_dev *accel_dev) +{ + bool irq_handled = false; + unsigned long vf_mask; + + /* Get the interrupt sources triggered by VFs, except for those already disabled */ + vf_mask = adf_disable_pending_vf2pf_interrupts(accel_dev); + if (vf_mask) { + struct adf_accel_vf_info *vf_info; + int i; + + /* + * Handle VF2PF interrupt unless the VF is malicious and + * is attempting to flood the host OS with VF2PF interrupts. + */ + for_each_set_bit(i, &vf_mask, ADF_MAX_NUM_VFS) { + vf_info = accel_dev->pf.vf_info + i; + + if (!__ratelimit(&vf_info->vf2pf_ratelimit)) { + dev_info(&GET_DEV(accel_dev), + "Too many ints from VF%d\n", + vf_info->vf_nr); + continue; + } + + adf_schedule_vf2pf_handler(vf_info); + irq_handled = true; + } + } + return irq_handled; +} +#endif /* CONFIG_PCI_IOV */ + +static bool adf_handle_pm_int(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + + if (hw_data->handle_pm_interrupt && + hw_data->handle_pm_interrupt(accel_dev)) + return true; + + return false; +} + +static bool adf_handle_ras_int(struct adf_accel_dev *accel_dev) +{ + struct adf_ras_ops *ras_ops = &accel_dev->hw_device->ras_ops; + bool reset_required; + + if (ras_ops->handle_interrupt && + ras_ops->handle_interrupt(accel_dev, &reset_required)) { + if (reset_required) { + dev_err(&GET_DEV(accel_dev), "Fatal error, reset required\n"); + if (adf_notify_fatal_error(accel_dev)) + dev_err(&GET_DEV(accel_dev), + "Failed to notify fatal error\n"); + } + + return true; + } + + return false; +} + +static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr) +{ + struct adf_accel_dev *accel_dev = dev_ptr; + +#ifdef CONFIG_PCI_IOV + /* If SR-IOV is enabled (vf_info is non-NULL), check for VF->PF ints */ + if (accel_dev->pf.vf_info && adf_handle_vf2pf_int(accel_dev)) + return IRQ_HANDLED; +#endif /* CONFIG_PCI_IOV */ + + if (adf_handle_pm_int(accel_dev)) + return IRQ_HANDLED; + + if (adf_handle_ras_int(accel_dev)) + return IRQ_HANDLED; + + dev_dbg(&GET_DEV(accel_dev), "qat_dev%d spurious AE interrupt\n", + accel_dev->accel_id); + + return IRQ_NONE; +} + +void adf_isr_sync_ae_cluster(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + u32 num_entries = pci_dev_info->msix_entries.num_entries; + struct adf_irq *irqs = pci_dev_info->msix_entries.irqs; + u32 irq_idx; + int irq; + + if (!test_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status) || !irqs) + return; + + irq_idx = num_entries > 1 ? hw_data->num_banks : 0; + if (irq_idx >= num_entries || !irqs[irq_idx].enabled) + return; + + irq = pci_irq_vector(pci_dev_info->pci_dev, hw_data->num_banks); + if (irq > 0) + synchronize_irq(irq); +} + +static void adf_free_irqs(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_irq *irqs = pci_dev_info->msix_entries.irqs; + struct adf_etr_data *etr_data = accel_dev->transport; + int clust_irq = hw_data->num_banks; + int irq, i = 0; + + if (pci_dev_info->msix_entries.num_entries > 1) { + for (i = 0; i < hw_data->num_banks; i++) { + if (irqs[i].enabled) { + irq = pci_irq_vector(pci_dev_info->pci_dev, i); + irq_set_affinity_hint(irq, NULL); + free_irq(irq, &etr_data->banks[i]); + } + } + } + + if (irqs[i].enabled) { + irq = pci_irq_vector(pci_dev_info->pci_dev, clust_irq); + free_irq(irq, accel_dev); + } +} + +static int adf_request_irqs(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_irq *irqs = pci_dev_info->msix_entries.irqs; + struct adf_etr_data *etr_data = accel_dev->transport; + int clust_irq = hw_data->num_banks; + int ret, irq, i = 0; + char *name; + + /* Request msix irq for all banks unless SR-IOV enabled */ + if (!accel_dev->pf.vf_info) { + for (i = 0; i < hw_data->num_banks; i++) { + struct adf_etr_bank_data *bank = &etr_data->banks[i]; + unsigned int cpu, cpus = num_online_cpus(); + + name = irqs[i].name; + snprintf(name, ADF_MAX_MSIX_VECTOR_NAME, + "qat%d-bundle%d", accel_dev->accel_id, i); + irq = pci_irq_vector(pci_dev_info->pci_dev, i); + if (unlikely(irq < 0)) { + dev_err(&GET_DEV(accel_dev), + "Failed to get IRQ number of device vector %d - %s\n", + i, name); + ret = irq; + goto err; + } + ret = request_irq(irq, adf_msix_isr_bundle, 0, + &name[0], bank); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to allocate IRQ %d for %s\n", + irq, name); + goto err; + } + + cpu = ((accel_dev->accel_id * hw_data->num_banks) + + i) % cpus; + irq_set_affinity_hint(irq, get_cpu_mask(cpu)); + irqs[i].enabled = true; + } + } + + /* Request msix irq for AE */ + name = irqs[i].name; + snprintf(name, ADF_MAX_MSIX_VECTOR_NAME, + "qat%d-ae-cluster", accel_dev->accel_id); + irq = pci_irq_vector(pci_dev_info->pci_dev, clust_irq); + if (unlikely(irq < 0)) { + dev_err(&GET_DEV(accel_dev), + "Failed to get IRQ number of device vector %d - %s\n", + i, name); + ret = irq; + goto err; + } + ret = request_irq(irq, adf_msix_isr_ae, 0, &name[0], accel_dev); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to allocate IRQ %d for %s\n", irq, name); + goto err; + } + irqs[i].enabled = true; + return ret; +err: + adf_free_irqs(accel_dev); + return ret; +} + +static int adf_isr_alloc_msix_vectors_data(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 msix_num_entries = 1; + struct adf_irq *irqs; + + /* If SR-IOV is disabled (vf_info is NULL), add entries for each bank */ + if (!accel_dev->pf.vf_info) + msix_num_entries += hw_data->num_banks; + + irqs = kcalloc_node(msix_num_entries, sizeof(*irqs), + GFP_KERNEL, dev_to_node(&GET_DEV(accel_dev))); + if (!irqs) + return -ENOMEM; + + accel_dev->accel_pci_dev.msix_entries.num_entries = msix_num_entries; + accel_dev->accel_pci_dev.msix_entries.irqs = irqs; + return 0; +} + +static void adf_isr_free_msix_vectors_data(struct adf_accel_dev *accel_dev) +{ + kfree(accel_dev->accel_pci_dev.msix_entries.irqs); + accel_dev->accel_pci_dev.msix_entries.irqs = NULL; +} + +static int adf_setup_bh(struct adf_accel_dev *accel_dev) +{ + struct adf_etr_data *priv_data = accel_dev->transport; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + int i; + + for (i = 0; i < hw_data->num_banks; i++) + tasklet_init(&priv_data->banks[i].resp_handler, + adf_response_handler, + (unsigned long)&priv_data->banks[i]); + return 0; +} + +static void adf_cleanup_bh(struct adf_accel_dev *accel_dev) +{ + struct adf_etr_data *priv_data = accel_dev->transport; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + int i; + + for (i = 0; i < hw_data->num_banks; i++) { + tasklet_disable(&priv_data->banks[i].resp_handler); + tasklet_kill(&priv_data->banks[i].resp_handler); + } +} + +/** + * adf_isr_resource_free() - Free IRQ for acceleration device + * @accel_dev: Pointer to acceleration device. + * + * Function frees interrupts for acceleration device. + */ +void adf_isr_resource_free(struct adf_accel_dev *accel_dev) +{ + adf_free_irqs(accel_dev); + adf_cleanup_bh(accel_dev); + adf_disable_msix(&accel_dev->accel_pci_dev); + adf_isr_free_msix_vectors_data(accel_dev); +} +EXPORT_SYMBOL_GPL(adf_isr_resource_free); + +/** + * adf_isr_resource_alloc() - Allocate IRQ for acceleration device + * @accel_dev: Pointer to acceleration device. + * + * Function allocates interrupts for acceleration device. + * + * Return: 0 on success, error code otherwise. + */ +int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev) +{ + int ret; + + ret = adf_isr_alloc_msix_vectors_data(accel_dev); + if (ret) + goto err_out; + + ret = adf_enable_msix(accel_dev); + if (ret) + goto err_free_msix_table; + + ret = adf_setup_bh(accel_dev); + if (ret) + goto err_disable_msix; + + ret = adf_request_irqs(accel_dev); + if (ret) + goto err_cleanup_bh; + + return 0; + +err_cleanup_bh: + adf_cleanup_bh(accel_dev); + +err_disable_msix: + adf_disable_msix(&accel_dev->accel_pci_dev); + +err_free_msix_table: + adf_isr_free_msix_vectors_data(accel_dev); + +err_out: + return ret; +} +EXPORT_SYMBOL_GPL(adf_isr_resource_alloc); + +/** + * adf_init_misc_wq() - Init misc workqueue + * + * Return: 0 on success, error code otherwise. + */ +int __init adf_init_misc_wq(void) +{ + adf_misc_wq = alloc_workqueue("qat_misc_wq", + WQ_MEM_RECLAIM | WQ_PERCPU, 0); + + return !adf_misc_wq ? -ENOMEM : 0; +} + +void adf_exit_misc_wq(void) +{ + if (adf_misc_wq) + destroy_workqueue(adf_misc_wq); + + adf_misc_wq = NULL; +} + +bool adf_misc_wq_queue_work(struct work_struct *work) +{ + return queue_work(adf_misc_wq, work); +} + +bool adf_misc_wq_queue_delayed_work(struct delayed_work *work, + unsigned long delay) +{ + return queue_delayed_work(adf_misc_wq, work, delay); +} + +void adf_misc_wq_flush(void) +{ + flush_workqueue(adf_misc_wq); +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_kpt.c b/drivers/crypto/intel/qat/qat_common/adf_kpt.c new file mode 100644 index 000000000..a0430141e --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_kpt.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2026 Intel Corporation */ +#include + +#include "adf_admin.h" +#include "adf_cfg_services.h" +#include "adf_common_drv.h" +#include "adf_kpt.h" + +static bool adf_kpt_supported(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + + return hw_data->accel_capabilities_mask & ICP_ACCEL_CAPABILITIES_KPT; +} + +int adf_enable_kpt(struct adf_accel_dev *accel_dev) +{ + struct adf_kpt_interface_data *user_data = GET_KPT_USER_DATA(accel_dev); + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + dma_addr_t paddr; + void *vaddr; + int ret; + int svc; + + /* Return 0 if KPT is not supported by the hardware */ + if (!adf_kpt_supported(accel_dev)) + return 0; + + if (!user_data->enable) { + /* Disable KPT capability if user has not enabled it */ + hw_data->accel_capabilities_mask &= ~ICP_ACCEL_CAPABILITIES_KPT; + return 0; + } + + svc = adf_get_service_enabled(accel_dev); + if (svc < 0) + return svc; + + if (svc != SVC_ASYM) { + dev_err(&GET_DEV(accel_dev), + "KPT can only be enabled when service is configured as 'asym'\n"); + return -EINVAL; + } + + vaddr = dma_alloc_coherent(&GET_DEV(accel_dev), PAGE_SIZE, &paddr, + GFP_KERNEL); + if (!vaddr) + return -ENOMEM; + + ret = adf_send_admin_kpt_init(accel_dev, vaddr, PAGE_SIZE, paddr); + + dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, vaddr, paddr); + + return ret; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_kpt.h b/drivers/crypto/intel/qat/qat_common/adf_kpt.h new file mode 100644 index 000000000..17d07d24a --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_kpt.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2026 Intel Corporation */ +#ifndef ADF_KPT_H_ +#define ADF_KPT_H_ + +#include + +#define GET_KPT_CFG_DATA(accel_dev) (&(accel_dev)->hw_device->kpt_data) +#define GET_KPT_USER_DATA(accel_dev) (&(accel_dev)->hw_device->kpt_data.user_input) + +struct adf_accel_dev; + +struct adf_kpt_interface_data { + bool enable; + bool swk_shared; + unsigned int swk_cnt_per_fn; + unsigned int swk_cnt_per_pasid; + unsigned int swk_max_ttl; +}; + +struct adf_kpt_hw_data { + unsigned int max_swk_cnt_per_fn_pasid; + unsigned int max_swk_ttl; + struct adf_kpt_interface_data user_input; +}; + +int adf_enable_kpt(struct adf_accel_dev *accel_dev); + +#endif /* ADF_KPT_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_module.c b/drivers/crypto/intel/qat/qat_common/adf_module.c new file mode 100644 index 000000000..30069feec --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_module.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2020 Intel Corporation */ + +#include +#include +#include + +#include "adf_common_drv.h" + +static int __init adf_register_module(void) +{ + if (adf_init_misc_wq()) + goto err_misc_wq; + + if (adf_init_aer()) + goto err_aer; + + if (adf_init_pf_wq()) + goto err_pf_wq; + + if (adf_init_vf_wq()) + goto err_vf_wq; + + if (qat_crypto_register()) + goto err_crypto_register; + + if (qat_compression_register()) + goto err_compression_register; + + return 0; + +err_compression_register: + qat_crypto_unregister(); +err_crypto_register: + adf_exit_vf_wq(); +err_vf_wq: + adf_exit_pf_wq(); +err_pf_wq: + adf_exit_aer(); +err_aer: + adf_exit_misc_wq(); +err_misc_wq: + return -EFAULT; +} + +static void __exit adf_unregister_module(void) +{ + adf_exit_misc_wq(); + adf_exit_aer(); + adf_exit_vf_wq(); + adf_exit_pf_wq(); + qat_crypto_unregister(); + qat_compression_unregister(); + adf_clean_vf_map(false); +} + +module_init(adf_register_module); +module_exit(adf_unregister_module); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Intel"); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); +MODULE_ALIAS_CRYPTO("intel_qat"); +MODULE_IMPORT_NS("CRYPTO_INTERNAL"); diff --git a/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.c b/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.c new file mode 100644 index 000000000..32aeb795c --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.c @@ -0,0 +1,319 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2024 Intel Corporation */ + +#include +#include +#include +#include "adf_mstate_mgr.h" + +#define ADF_MSTATE_MAGIC 0xADF5CAEA +#define ADF_MSTATE_VERSION 0x1 + +struct adf_mstate_sect_h { + u8 id[ADF_MSTATE_ID_LEN]; + u32 size; + u32 sub_sects; + u8 state[]; +}; + +u32 adf_mstate_state_size(struct adf_mstate_mgr *mgr) +{ + return mgr->state - mgr->buf; +} + +static inline u32 adf_mstate_avail_room(struct adf_mstate_mgr *mgr) +{ + return mgr->buf + mgr->size - mgr->state; +} + +void adf_mstate_mgr_init(struct adf_mstate_mgr *mgr, u8 *buf, u32 size) +{ + mgr->buf = buf; + mgr->state = buf; + mgr->size = size; + mgr->n_sects = 0; +}; + +struct adf_mstate_mgr *adf_mstate_mgr_new(u8 *buf, u32 size) +{ + struct adf_mstate_mgr *mgr; + + mgr = kzalloc_obj(*mgr); + if (!mgr) + return NULL; + + adf_mstate_mgr_init(mgr, buf, size); + + return mgr; +} + +void adf_mstate_mgr_destroy(struct adf_mstate_mgr *mgr) +{ + kfree(mgr); +} + +void adf_mstate_mgr_init_from_parent(struct adf_mstate_mgr *mgr, + struct adf_mstate_mgr *p_mgr) +{ + adf_mstate_mgr_init(mgr, p_mgr->state, + p_mgr->size - adf_mstate_state_size(p_mgr)); +} + +void adf_mstate_mgr_init_from_psect(struct adf_mstate_mgr *mgr, + struct adf_mstate_sect_h *p_sect) +{ + adf_mstate_mgr_init(mgr, p_sect->state, p_sect->size); + mgr->n_sects = p_sect->sub_sects; +} + +static void adf_mstate_preamble_init(struct adf_mstate_preh *preamble) +{ + preamble->magic = ADF_MSTATE_MAGIC; + preamble->version = ADF_MSTATE_VERSION; + preamble->preh_len = sizeof(*preamble); + preamble->size = 0; + preamble->n_sects = 0; +} + +/* default preambles checker */ +static int adf_mstate_preamble_def_checker(struct adf_mstate_preh *preamble, + void *opaque) +{ + struct adf_mstate_mgr *mgr = opaque; + + if (preamble->magic != ADF_MSTATE_MAGIC || + preamble->version > ADF_MSTATE_VERSION || + preamble->preh_len > mgr->size) { + pr_debug("QAT: LM - Invalid state (magic=%#x, version=%#x, hlen=%u), state_size=%u\n", + preamble->magic, preamble->version, preamble->preh_len, + mgr->size); + return -EINVAL; + } + + return 0; +} + +struct adf_mstate_preh *adf_mstate_preamble_add(struct adf_mstate_mgr *mgr) +{ + struct adf_mstate_preh *pre = (struct adf_mstate_preh *)mgr->buf; + + if (adf_mstate_avail_room(mgr) < sizeof(*pre)) { + pr_err("QAT: LM - Not enough space for preamble\n"); + return NULL; + } + + adf_mstate_preamble_init(pre); + mgr->state += pre->preh_len; + + return pre; +} + +int adf_mstate_preamble_update(struct adf_mstate_mgr *mgr) +{ + struct adf_mstate_preh *preamble = (struct adf_mstate_preh *)mgr->buf; + + preamble->size = adf_mstate_state_size(mgr) - preamble->preh_len; + preamble->n_sects = mgr->n_sects; + + return 0; +} + +static void adf_mstate_dump_sect(struct adf_mstate_sect_h *sect, + const char *prefix) +{ + pr_debug("QAT: LM - %s QAT state section %s\n", prefix, sect->id); + print_hex_dump_debug("h-", DUMP_PREFIX_OFFSET, 16, 2, sect, + sizeof(*sect), true); + print_hex_dump_debug("s-", DUMP_PREFIX_OFFSET, 16, 2, sect->state, + sect->size, true); +} + +static inline void __adf_mstate_sect_update(struct adf_mstate_mgr *mgr, + struct adf_mstate_sect_h *sect, + u32 size, + u32 n_subsects) +{ + sect->size += size; + sect->sub_sects += n_subsects; + mgr->n_sects++; + mgr->state += sect->size; + + adf_mstate_dump_sect(sect, "Add"); +} + +void adf_mstate_sect_update(struct adf_mstate_mgr *p_mgr, + struct adf_mstate_mgr *curr_mgr, + struct adf_mstate_sect_h *sect) +{ + __adf_mstate_sect_update(p_mgr, sect, adf_mstate_state_size(curr_mgr), + curr_mgr->n_sects); +} + +static struct adf_mstate_sect_h *adf_mstate_sect_add_header(struct adf_mstate_mgr *mgr, + const char *id) +{ + struct adf_mstate_sect_h *sect = (struct adf_mstate_sect_h *)(mgr->state); + + if (adf_mstate_avail_room(mgr) < sizeof(*sect)) { + pr_debug("QAT: LM - Not enough space for header of QAT state sect %s\n", id); + return NULL; + } + + strscpy(sect->id, id); + sect->size = 0; + sect->sub_sects = 0; + mgr->state += sizeof(*sect); + + return sect; +} + +struct adf_mstate_sect_h *adf_mstate_sect_add_vreg(struct adf_mstate_mgr *mgr, + const char *id, + struct adf_mstate_vreginfo *info) +{ + struct adf_mstate_sect_h *sect; + + sect = adf_mstate_sect_add_header(mgr, id); + if (!sect) + return NULL; + + if (adf_mstate_avail_room(mgr) < info->size) { + pr_debug("QAT: LM - Not enough space for QAT state sect %s, requires %u\n", + id, info->size); + return NULL; + } + + memcpy(sect->state, info->addr, info->size); + __adf_mstate_sect_update(mgr, sect, info->size, 0); + + return sect; +} + +struct adf_mstate_sect_h *adf_mstate_sect_add(struct adf_mstate_mgr *mgr, + const char *id, + adf_mstate_populate populate, + void *opaque) +{ + struct adf_mstate_mgr sub_sects_mgr; + struct adf_mstate_sect_h *sect; + int avail_room, size; + + sect = adf_mstate_sect_add_header(mgr, id); + if (!sect) + return NULL; + + if (!populate) + return sect; + + avail_room = adf_mstate_avail_room(mgr); + adf_mstate_mgr_init_from_parent(&sub_sects_mgr, mgr); + + size = (*populate)(&sub_sects_mgr, sect->state, avail_room, opaque); + if (size < 0) + return NULL; + + size += adf_mstate_state_size(&sub_sects_mgr); + if (avail_room < size) { + pr_debug("QAT: LM - Not enough space for QAT state sect %s, requires %u\n", + id, size); + return NULL; + } + __adf_mstate_sect_update(mgr, sect, size, sub_sects_mgr.n_sects); + + return sect; +} + +static int adf_mstate_sect_validate(struct adf_mstate_mgr *mgr) +{ + struct adf_mstate_sect_h *start = (struct adf_mstate_sect_h *)mgr->state; + struct adf_mstate_sect_h *sect = start; + u64 end; + int i; + + end = (uintptr_t)mgr->buf + mgr->size; + for (i = 0; i < mgr->n_sects; i++) { + uintptr_t s_start = (uintptr_t)sect->state; + uintptr_t s_end = s_start + sect->size; + + if (s_end < s_start || s_end > end) { + pr_debug("QAT: LM - Corrupted state section (index=%u, size=%u) in state_mgr (size=%u, secs=%u)\n", + i, sect->size, mgr->size, mgr->n_sects); + return -EINVAL; + } + sect = (struct adf_mstate_sect_h *)s_end; + } + + pr_debug("QAT: LM - Scanned section (last child=%s, size=%lu) in state_mgr (size=%u, secs=%u)\n", + start->id, sizeof(struct adf_mstate_sect_h) * (ulong)(sect - start), + mgr->size, mgr->n_sects); + + return 0; +} + +u32 adf_mstate_state_size_from_remote(struct adf_mstate_mgr *mgr) +{ + struct adf_mstate_preh *preh = (struct adf_mstate_preh *)mgr->buf; + + return preh->preh_len + preh->size; +} + +int adf_mstate_mgr_init_from_remote(struct adf_mstate_mgr *mgr, u8 *buf, u32 size, + adf_mstate_preamble_checker pre_checker, + void *opaque) +{ + struct adf_mstate_preh *pre; + int ret; + + adf_mstate_mgr_init(mgr, buf, size); + pre = (struct adf_mstate_preh *)(mgr->buf); + + pr_debug("QAT: LM - Dump state preambles\n"); + print_hex_dump_debug("", DUMP_PREFIX_OFFSET, 16, 2, pre, pre->preh_len, 0); + + if (pre_checker) + ret = (*pre_checker)(pre, opaque); + else + ret = adf_mstate_preamble_def_checker(pre, mgr); + if (ret) + return ret; + + mgr->state = mgr->buf + pre->preh_len; + mgr->n_sects = pre->n_sects; + + return adf_mstate_sect_validate(mgr); +} + +struct adf_mstate_sect_h *adf_mstate_sect_lookup(struct adf_mstate_mgr *mgr, + const char *id, + adf_mstate_action action, + void *opaque) +{ + struct adf_mstate_sect_h *sect = (struct adf_mstate_sect_h *)mgr->state; + struct adf_mstate_mgr sub_sects_mgr; + int i, ret; + + for (i = 0; i < mgr->n_sects; i++) { + if (!strncmp(sect->id, id, sizeof(sect->id))) + goto found; + + sect = (struct adf_mstate_sect_h *)(sect->state + sect->size); + } + + return NULL; + +found: + adf_mstate_dump_sect(sect, "Found"); + + adf_mstate_mgr_init_from_psect(&sub_sects_mgr, sect); + if (sect->sub_sects && adf_mstate_sect_validate(&sub_sects_mgr)) + return NULL; + + if (!action) + return sect; + + ret = (*action)(&sub_sects_mgr, sect->state, sect->size, opaque); + if (ret) + return NULL; + + return sect; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.h b/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.h new file mode 100644 index 000000000..81d263a59 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2024 Intel Corporation */ + +#ifndef ADF_MSTATE_MGR_H +#define ADF_MSTATE_MGR_H + +#define ADF_MSTATE_ID_LEN 8 + +#define ADF_MSTATE_ETRB_IDS "ETRBAR" +#define ADF_MSTATE_MISCB_IDS "MISCBAR" +#define ADF_MSTATE_EXTB_IDS "EXTBAR" +#define ADF_MSTATE_GEN_IDS "GENER" +#define ADF_MSTATE_CONFIG_IDS "CONFIG" +#define ADF_MSTATE_SECTION_NUM 5 + +#define ADF_MSTATE_BANK_IDX_IDS "bnk" + +#define ADF_MSTATE_ETR_REGS_IDS "mregs" +#define ADF_MSTATE_VINTSRC_IDS "visrc" +#define ADF_MSTATE_VINTMSK_IDS "vimsk" +#define ADF_MSTATE_SLA_IDS "sla" +#define ADF_MSTATE_IOV_INIT_IDS "iovinit" +#define ADF_MSTATE_COMPAT_VER_IDS "compver" +#define ADF_MSTATE_GEN_CAP_IDS "gencap" +#define ADF_MSTATE_GEN_SVCMAP_IDS "svcmap" +#define ADF_MSTATE_GEN_EXTDC_IDS "extdc" +#define ADF_MSTATE_VINTSRC_PF2VM_IDS "vispv" +#define ADF_MSTATE_VINTMSK_PF2VM_IDS "vimpv" +#define ADF_MSTATE_VM2PF_IDS "vm2pf" +#define ADF_MSTATE_PF2VM_IDS "pf2vm" + +struct adf_mstate_mgr { + u8 *buf; + u8 *state; + u32 size; + u32 n_sects; +}; + +struct adf_mstate_preh { + u32 magic; + u32 version; + u16 preh_len; + u16 n_sects; + u32 size; +}; + +struct adf_mstate_vreginfo { + void *addr; + u32 size; +}; + +struct adf_mstate_sect_h; + +typedef int (*adf_mstate_preamble_checker)(struct adf_mstate_preh *preamble, void *opa); +typedef int (*adf_mstate_populate)(struct adf_mstate_mgr *sub_mgr, u8 *buf, + u32 size, void *opa); +typedef int (*adf_mstate_action)(struct adf_mstate_mgr *sub_mgr, u8 *buf, u32 size, + void *opa); + +struct adf_mstate_mgr *adf_mstate_mgr_new(u8 *buf, u32 size); +void adf_mstate_mgr_destroy(struct adf_mstate_mgr *mgr); +void adf_mstate_mgr_init(struct adf_mstate_mgr *mgr, u8 *buf, u32 size); +void adf_mstate_mgr_init_from_parent(struct adf_mstate_mgr *mgr, + struct adf_mstate_mgr *p_mgr); +void adf_mstate_mgr_init_from_psect(struct adf_mstate_mgr *mgr, + struct adf_mstate_sect_h *p_sect); +int adf_mstate_mgr_init_from_remote(struct adf_mstate_mgr *mgr, + u8 *buf, u32 size, + adf_mstate_preamble_checker checker, + void *opaque); +struct adf_mstate_preh *adf_mstate_preamble_add(struct adf_mstate_mgr *mgr); +int adf_mstate_preamble_update(struct adf_mstate_mgr *mgr); +u32 adf_mstate_state_size(struct adf_mstate_mgr *mgr); +u32 adf_mstate_state_size_from_remote(struct adf_mstate_mgr *mgr); +void adf_mstate_sect_update(struct adf_mstate_mgr *p_mgr, + struct adf_mstate_mgr *curr_mgr, + struct adf_mstate_sect_h *sect); +struct adf_mstate_sect_h *adf_mstate_sect_add_vreg(struct adf_mstate_mgr *mgr, + const char *id, + struct adf_mstate_vreginfo *info); +struct adf_mstate_sect_h *adf_mstate_sect_add(struct adf_mstate_mgr *mgr, + const char *id, + adf_mstate_populate populate, + void *opaque); +struct adf_mstate_sect_h *adf_mstate_sect_lookup(struct adf_mstate_mgr *mgr, + const char *id, + adf_mstate_action action, + void *opaque); +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_pfvf_msg.h b/drivers/crypto/intel/qat/qat_common/adf_pfvf_msg.h new file mode 100644 index 000000000..d1b3ef9ca --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_msg.h @@ -0,0 +1,264 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2015 - 2021 Intel Corporation */ +#ifndef ADF_PFVF_MSG_H +#define ADF_PFVF_MSG_H + +#include + +/* + * PF<->VF Gen2 Messaging format + * + * The PF has an array of 32-bit PF2VF registers, one for each VF. The + * PF can access all these registers while each VF can access only the one + * register associated with that particular VF. + * + * The register functionally is split into two parts: + * The bottom half is for PF->VF messages. In particular when the first + * bit of this register (bit 0) gets set an interrupt will be triggered + * in the respective VF. + * The top half is for VF->PF messages. In particular when the first bit + * of this half of register (bit 16) gets set an interrupt will be triggered + * in the PF. + * + * The remaining bits within this register are available to encode messages. + * and implement a collision control mechanism to prevent concurrent use of + * the PF2VF register by both the PF and VF. + * + * 31 30 29 28 27 26 25 24 23 22 21 20 19 18 17 16 + * _______________________________________________ + * | | | | | | | | | | | | | | | | | + * +-----------------------------------------------+ + * \___________________________/ \_________/ ^ ^ + * ^ ^ | | + * | | | VF2PF Int + * | | Message Origin + * | Message Type + * Message-specific Data/Reserved + * + * 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0 + * _______________________________________________ + * | | | | | | | | | | | | | | | | | + * +-----------------------------------------------+ + * \___________________________/ \_________/ ^ ^ + * ^ ^ | | + * | | | PF2VF Int + * | | Message Origin + * | Message Type + * Message-specific Data/Reserved + * + * Message Origin (Should always be 1) + * A legacy out-of-tree QAT driver allowed for a set of messages not supported + * by this driver; these had a Msg Origin of 0 and are ignored by this driver. + * + * When a PF or VF attempts to send a message in the lower or upper 16 bits, + * respectively, the other 16 bits are written to first with a defined + * IN_USE_BY pattern as part of a collision control scheme (see function + * adf_gen2_pfvf_send() in adf_pf2vf_msg.c). + * + * + * PF<->VF Gen4 Messaging format + * + * Similarly to the gen2 messaging format, 32-bit long registers are used for + * communication between PF and VFs. However, each VF and PF share a pair of + * 32-bits register to avoid collisions: one for PV to VF messages and one + * for VF to PF messages. + * + * Both the Interrupt bit and the Message Origin bit retain the same position + * and meaning, although non-system messages are now deprecated and not + * expected. + * + * 31 30 9 8 7 6 5 4 3 2 1 0 + * _______________________________________________ + * | | | . . . | | | | | | | | | | | + * +-----------------------------------------------+ + * \_____________________/ \_______________/ ^ ^ + * ^ ^ | | + * | | | PF/VF Int + * | | Message Origin + * | Message Type + * Message-specific Data/Reserved + * + * For both formats, the message reception is acknowledged by lowering the + * interrupt bit on the register where the message was sent. + */ + +/* PFVF message common bits */ +#define ADF_PFVF_INT BIT(0) +#define ADF_PFVF_MSGORIGIN_SYSTEM BIT(1) + +/* Different generations have different CSR layouts, use this struct + * to abstract these differences away + */ +struct pfvf_message { + u8 type; + u32 data; +}; + +/* PF->VF messages */ +enum pf2vf_msgtype { + ADF_PF2VF_MSGTYPE_RESTARTING = 0x01, + ADF_PF2VF_MSGTYPE_VERSION_RESP = 0x02, + ADF_PF2VF_MSGTYPE_BLKMSG_RESP = 0x03, + ADF_PF2VF_MSGTYPE_FATAL_ERROR = 0x04, + ADF_PF2VF_MSGTYPE_RESTARTED = 0x05, +/* Values from 0x10 are Gen4 specific, message type is only 4 bits in Gen2 devices. */ + ADF_PF2VF_MSGTYPE_RP_RESET_RESP = 0x10, +}; + +/* VF->PF messages */ +enum vf2pf_msgtype { + ADF_VF2PF_MSGTYPE_INIT = 0x03, + ADF_VF2PF_MSGTYPE_SHUTDOWN = 0x04, + ADF_VF2PF_MSGTYPE_VERSION_REQ = 0x05, + ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ = 0x06, + ADF_VF2PF_MSGTYPE_LARGE_BLOCK_REQ = 0x07, + ADF_VF2PF_MSGTYPE_MEDIUM_BLOCK_REQ = 0x08, + ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ = 0x09, + ADF_VF2PF_MSGTYPE_RESTARTING_COMPLETE = 0x0a, +/* Values from 0x10 are Gen4 specific, message type is only 4 bits in Gen2 devices. */ + ADF_VF2PF_MSGTYPE_RP_RESET = 0x10, +}; + +/* VF/PF compatibility version. */ +enum pfvf_compatibility_version { + /* Support for extended capabilities */ + ADF_PFVF_COMPAT_CAPABILITIES = 0x02, + /* In-use pattern cleared by receiver */ + ADF_PFVF_COMPAT_FAST_ACK = 0x03, + /* Ring to service mapping support for non-standard mappings */ + ADF_PFVF_COMPAT_RING_TO_SVC_MAP = 0x04, + /* Fallback compat */ + ADF_PFVF_COMPAT_FALLBACK = 0x05, + /* Reference to the latest version */ + ADF_PFVF_COMPAT_THIS_VERSION = 0x05, +}; + +/* PF->VF Version Response */ +#define ADF_PF2VF_VERSION_RESP_VERS_MASK GENMASK(7, 0) +#define ADF_PF2VF_VERSION_RESP_RESULT_MASK GENMASK(9, 8) + +enum pf2vf_compat_response { + ADF_PF2VF_VF_COMPATIBLE = 0x01, + ADF_PF2VF_VF_INCOMPATIBLE = 0x02, + ADF_PF2VF_VF_COMPAT_UNKNOWN = 0x03, +}; + +enum ring_reset_result { + RPRESET_SUCCESS = 0x00, + RPRESET_NOT_SUPPORTED = 0x01, + RPRESET_INVAL_BANK = 0x02, + RPRESET_TIMEOUT = 0x03, +}; + +#define ADF_VF2PF_RNG_RESET_RP_MASK GENMASK(1, 0) +#define ADF_VF2PF_RNG_RESET_RSVD_MASK GENMASK(25, 2) + +/* PF->VF Block Responses */ +#define ADF_PF2VF_BLKMSG_RESP_TYPE_MASK GENMASK(1, 0) +#define ADF_PF2VF_BLKMSG_RESP_DATA_MASK GENMASK(9, 2) + +enum pf2vf_blkmsg_resp_type { + ADF_PF2VF_BLKMSG_RESP_TYPE_DATA = 0x00, + ADF_PF2VF_BLKMSG_RESP_TYPE_CRC = 0x01, + ADF_PF2VF_BLKMSG_RESP_TYPE_ERROR = 0x02, +}; + +/* PF->VF Block Error Code */ +enum pf2vf_blkmsg_error { + ADF_PF2VF_INVALID_BLOCK_TYPE = 0x00, + ADF_PF2VF_INVALID_BYTE_NUM_REQ = 0x01, + ADF_PF2VF_PAYLOAD_TRUNCATED = 0x02, + ADF_PF2VF_UNSPECIFIED_ERROR = 0x03, +}; + +/* VF->PF Block Requests */ +#define ADF_VF2PF_LARGE_BLOCK_TYPE_MASK GENMASK(1, 0) +#define ADF_VF2PF_LARGE_BLOCK_BYTE_MASK GENMASK(8, 2) +#define ADF_VF2PF_MEDIUM_BLOCK_TYPE_MASK GENMASK(2, 0) +#define ADF_VF2PF_MEDIUM_BLOCK_BYTE_MASK GENMASK(8, 3) +#define ADF_VF2PF_SMALL_BLOCK_TYPE_MASK GENMASK(3, 0) +#define ADF_VF2PF_SMALL_BLOCK_BYTE_MASK GENMASK(8, 4) +#define ADF_VF2PF_BLOCK_CRC_REQ_MASK BIT(9) + +/* PF->VF Block Request Types + * 0..15 - 32 byte message + * 16..23 - 64 byte message + * 24..27 - 128 byte message + */ +enum vf2pf_blkmsg_req_type { + ADF_VF2PF_BLKMSG_REQ_CAP_SUMMARY = 0x02, + ADF_VF2PF_BLKMSG_REQ_RING_SVC_MAP = 0x03, +}; + +#define ADF_VF2PF_SMALL_BLOCK_TYPE_MAX \ + (FIELD_MAX(ADF_VF2PF_SMALL_BLOCK_TYPE_MASK)) + +#define ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX \ + (FIELD_MAX(ADF_VF2PF_MEDIUM_BLOCK_TYPE_MASK) + \ + ADF_VF2PF_SMALL_BLOCK_TYPE_MAX + 1) + +#define ADF_VF2PF_LARGE_BLOCK_TYPE_MAX \ + (FIELD_MAX(ADF_VF2PF_LARGE_BLOCK_TYPE_MASK) + \ + ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX) + +#define ADF_VF2PF_SMALL_BLOCK_BYTE_MAX \ + FIELD_MAX(ADF_VF2PF_SMALL_BLOCK_BYTE_MASK) + +#define ADF_VF2PF_MEDIUM_BLOCK_BYTE_MAX \ + FIELD_MAX(ADF_VF2PF_MEDIUM_BLOCK_BYTE_MASK) + +#define ADF_VF2PF_LARGE_BLOCK_BYTE_MAX \ + FIELD_MAX(ADF_VF2PF_LARGE_BLOCK_BYTE_MASK) + +struct pfvf_blkmsg_header { + u8 version; + u8 payload_size; +} __packed; + +#define ADF_PFVF_BLKMSG_HEADER_SIZE (sizeof(struct pfvf_blkmsg_header)) +#define ADF_PFVF_BLKMSG_PAYLOAD_SIZE(blkmsg) (sizeof(blkmsg) - \ + ADF_PFVF_BLKMSG_HEADER_SIZE) +#define ADF_PFVF_BLKMSG_MSG_SIZE(blkmsg) (ADF_PFVF_BLKMSG_HEADER_SIZE + \ + (blkmsg)->hdr.payload_size) +#define ADF_PFVF_BLKMSG_MSG_MAX_SIZE 128 + +/* PF->VF Block message header bytes */ +#define ADF_PFVF_BLKMSG_VER_BYTE 0 +#define ADF_PFVF_BLKMSG_LEN_BYTE 1 + +/* PF/VF Capabilities message values */ +enum blkmsg_capabilities_versions { + ADF_PFVF_CAPABILITIES_V1_VERSION = 0x01, + ADF_PFVF_CAPABILITIES_V2_VERSION = 0x02, + ADF_PFVF_CAPABILITIES_V3_VERSION = 0x03, +}; + +struct capabilities_v1 { + struct pfvf_blkmsg_header hdr; + u32 ext_dc_caps; +} __packed; + +struct capabilities_v2 { + struct pfvf_blkmsg_header hdr; + u32 ext_dc_caps; + u32 capabilities; +} __packed; + +struct capabilities_v3 { + struct pfvf_blkmsg_header hdr; + u32 ext_dc_caps; + u32 capabilities; + u32 frequency; +} __packed; + +/* PF/VF Ring to service mapping values */ +enum blkmsg_ring_to_svc_versions { + ADF_PFVF_RING_TO_SVC_VERSION = 0x01, +}; + +struct ring_to_svc_map_v1 { + struct pfvf_blkmsg_header hdr; + u16 map; +} __packed; + +#endif /* ADF_PFVF_MSG_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_msg.c b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_msg.c new file mode 100644 index 000000000..0cee3b23d --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_msg.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2015 - 2021 Intel Corporation */ +#include +#include +#include "adf_accel_devices.h" +#include "adf_pfvf_msg.h" +#include "adf_pfvf_pf_msg.h" +#include "adf_pfvf_pf_proto.h" + +#define ADF_PF_WAIT_RESTARTING_COMPLETE_DELAY 100 +#define ADF_VF_SHUTDOWN_RETRY 100 + +void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_vf_info *vf; + struct pfvf_message msg = { .type = ADF_PF2VF_MSGTYPE_RESTARTING }; + int i, num_vfs = pci_num_vf(accel_to_pci_dev(accel_dev)); + + dev_dbg(&GET_DEV(accel_dev), "pf2vf notify restarting\n"); + for (i = 0, vf = accel_dev->pf.vf_info; i < num_vfs; i++, vf++) { + if (vf->init && vf->vf_compat_ver >= ADF_PFVF_COMPAT_FALLBACK) + vf->restarting = true; + else + vf->restarting = false; + + if (!vf->init) + continue; + + if (adf_send_pf2vf_msg(accel_dev, i, msg)) + dev_err(&GET_DEV(accel_dev), + "Failed to send restarting msg to VF%d\n", i); + } +} + +void adf_pf2vf_wait_for_restarting_complete(struct adf_accel_dev *accel_dev) +{ + int num_vfs = pci_num_vf(accel_to_pci_dev(accel_dev)); + int i, retries = ADF_VF_SHUTDOWN_RETRY; + struct adf_accel_vf_info *vf; + bool vf_running; + + dev_dbg(&GET_DEV(accel_dev), "pf2vf wait for restarting complete\n"); + do { + vf_running = false; + for (i = 0, vf = accel_dev->pf.vf_info; i < num_vfs; i++, vf++) + if (vf->restarting) + vf_running = true; + if (!vf_running) + break; + msleep(ADF_PF_WAIT_RESTARTING_COMPLETE_DELAY); + } while (--retries); + + if (vf_running) + dev_warn(&GET_DEV(accel_dev), "Some VFs are still running\n"); +} + +void adf_pf2vf_notify_restarted(struct adf_accel_dev *accel_dev) +{ + struct pfvf_message msg = { .type = ADF_PF2VF_MSGTYPE_RESTARTED }; + int i, num_vfs = pci_num_vf(accel_to_pci_dev(accel_dev)); + struct adf_accel_vf_info *vf; + + dev_dbg(&GET_DEV(accel_dev), "pf2vf notify restarted\n"); + for (i = 0, vf = accel_dev->pf.vf_info; i < num_vfs; i++, vf++) { + if (vf->init && vf->vf_compat_ver >= ADF_PFVF_COMPAT_FALLBACK && + adf_send_pf2vf_msg(accel_dev, i, msg)) + dev_err(&GET_DEV(accel_dev), + "Failed to send restarted msg to VF%d\n", i); + } +} + +void adf_pf2vf_notify_fatal_error(struct adf_accel_dev *accel_dev) +{ + struct pfvf_message msg = { .type = ADF_PF2VF_MSGTYPE_FATAL_ERROR }; + int i, num_vfs = pci_num_vf(accel_to_pci_dev(accel_dev)); + struct adf_accel_vf_info *vf; + + dev_dbg(&GET_DEV(accel_dev), "pf2vf notify fatal error\n"); + for (i = 0, vf = accel_dev->pf.vf_info; i < num_vfs; i++, vf++) { + if (vf->init && vf->vf_compat_ver >= ADF_PFVF_COMPAT_FALLBACK && + adf_send_pf2vf_msg(accel_dev, i, msg)) + dev_err(&GET_DEV(accel_dev), + "Failed to send fatal error msg to VF%d\n", i); + } +} + +int adf_pf_capabilities_msg_provider(struct adf_accel_dev *accel_dev, + u8 *buffer, u8 compat) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct capabilities_v2 caps_msg; + + caps_msg.ext_dc_caps = hw_data->extended_dc_capabilities; + caps_msg.capabilities = hw_data->accel_capabilities_mask; + + caps_msg.hdr.version = ADF_PFVF_CAPABILITIES_V2_VERSION; + caps_msg.hdr.payload_size = + ADF_PFVF_BLKMSG_PAYLOAD_SIZE(struct capabilities_v2); + + memcpy(buffer, &caps_msg, sizeof(caps_msg)); + + return 0; +} + +int adf_pf_ring_to_svc_msg_provider(struct adf_accel_dev *accel_dev, + u8 *buffer, u8 compat) +{ + struct ring_to_svc_map_v1 rts_map_msg; + + rts_map_msg.map = accel_dev->hw_device->ring_to_svc_map; + rts_map_msg.hdr.version = ADF_PFVF_RING_TO_SVC_VERSION; + rts_map_msg.hdr.payload_size = ADF_PFVF_BLKMSG_PAYLOAD_SIZE(rts_map_msg); + + memcpy(buffer, &rts_map_msg, sizeof(rts_map_msg)); + + return 0; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_msg.h b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_msg.h new file mode 100644 index 000000000..f203d88c9 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_msg.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2021 Intel Corporation */ +#ifndef ADF_PFVF_PF_MSG_H +#define ADF_PFVF_PF_MSG_H + +#include "adf_accel_devices.h" + +#if defined(CONFIG_PCI_IOV) +void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev); +void adf_pf2vf_wait_for_restarting_complete(struct adf_accel_dev *accel_dev); +void adf_pf2vf_notify_restarted(struct adf_accel_dev *accel_dev); +void adf_pf2vf_notify_fatal_error(struct adf_accel_dev *accel_dev); +#else +static inline void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev) +{ +} + +static inline void adf_pf2vf_wait_for_restarting_complete(struct adf_accel_dev *accel_dev) +{ +} + +static inline void adf_pf2vf_notify_restarted(struct adf_accel_dev *accel_dev) +{ +} + +static inline void adf_pf2vf_notify_fatal_error(struct adf_accel_dev *accel_dev) +{ +} +#endif + +typedef int (*adf_pf2vf_blkmsg_provider)(struct adf_accel_dev *accel_dev, + u8 *buffer, u8 compat); + +int adf_pf_capabilities_msg_provider(struct adf_accel_dev *accel_dev, + u8 *buffer, u8 comapt); +int adf_pf_ring_to_svc_msg_provider(struct adf_accel_dev *accel_dev, + u8 *buffer, u8 comapt); + +#endif /* ADF_PFVF_PF_MSG_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c new file mode 100644 index 000000000..af8dbc751 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c @@ -0,0 +1,360 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2015 - 2021 Intel Corporation */ +#include +#include +#include +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_pfvf_msg.h" +#include "adf_pfvf_pf_msg.h" +#include "adf_pfvf_pf_proto.h" +#include "adf_pfvf_utils.h" + +typedef u8 (*pf2vf_blkmsg_data_getter_fn)(u8 const *blkmsg, u8 byte); + +static const adf_pf2vf_blkmsg_provider pf2vf_blkmsg_providers[] = { + NULL, /* no message type defined for value 0 */ + NULL, /* no message type defined for value 1 */ + adf_pf_capabilities_msg_provider, /* ADF_VF2PF_BLKMSG_REQ_CAP_SUMMARY */ + adf_pf_ring_to_svc_msg_provider, /* ADF_VF2PF_BLKMSG_REQ_RING_SVC_MAP */ +}; + +/** + * adf_send_pf2vf_msg() - send PF to VF message + * @accel_dev: Pointer to acceleration device + * @vf_nr: VF number to which the message will be sent + * @msg: Message to send + * + * This function allows the PF to send a message to a specific VF. + * + * Return: 0 on success, error code otherwise. + */ +int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, struct pfvf_message msg) +{ + struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev); + u32 pfvf_offset = pfvf_ops->get_pf2vf_offset(vf_nr); + + return pfvf_ops->send_msg(accel_dev, msg, pfvf_offset, + &accel_dev->pf.vf_info[vf_nr].pf2vf_lock); +} + +/** + * adf_recv_vf2pf_msg() - receive a VF to PF message + * @accel_dev: Pointer to acceleration device + * @vf_nr: Number of the VF from where the message will be received + * + * This function allows the PF to receive a message from a specific VF. + * + * Return: a valid message on success, zero otherwise. + */ +static struct pfvf_message adf_recv_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr) +{ + struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; + struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev); + u32 pfvf_offset = pfvf_ops->get_vf2pf_offset(vf_nr); + + return pfvf_ops->recv_msg(accel_dev, pfvf_offset, vf_info->vf_compat_ver); +} + +static adf_pf2vf_blkmsg_provider get_blkmsg_response_provider(u8 type) +{ + if (type >= ARRAY_SIZE(pf2vf_blkmsg_providers)) + return NULL; + + return pf2vf_blkmsg_providers[type]; +} + +/* Byte pf2vf_blkmsg_data_getter_fn callback */ +static u8 adf_pf2vf_blkmsg_get_byte(u8 const *blkmsg, u8 index) +{ + return blkmsg[index]; +} + +/* CRC pf2vf_blkmsg_data_getter_fn callback */ +static u8 adf_pf2vf_blkmsg_get_crc(u8 const *blkmsg, u8 count) +{ + /* count is 0-based, turn it into a length */ + return adf_pfvf_calc_blkmsg_crc(blkmsg, count + 1); +} + +static int adf_pf2vf_blkmsg_get_data(struct adf_accel_vf_info *vf_info, + u8 type, u8 byte, u8 max_size, u8 *data, + pf2vf_blkmsg_data_getter_fn data_getter) +{ + u8 blkmsg[ADF_PFVF_BLKMSG_MSG_MAX_SIZE] = { 0 }; + struct adf_accel_dev *accel_dev = vf_info->accel_dev; + adf_pf2vf_blkmsg_provider provider; + u8 msg_size; + + provider = get_blkmsg_response_provider(type); + + if (unlikely(!provider)) { + pr_err("QAT: No registered provider for message %d\n", type); + *data = ADF_PF2VF_INVALID_BLOCK_TYPE; + return -EINVAL; + } + + if (unlikely((*provider)(accel_dev, blkmsg, vf_info->vf_compat_ver))) { + pr_err("QAT: unknown error from provider for message %d\n", type); + *data = ADF_PF2VF_UNSPECIFIED_ERROR; + return -EINVAL; + } + + msg_size = ADF_PFVF_BLKMSG_HEADER_SIZE + blkmsg[ADF_PFVF_BLKMSG_LEN_BYTE]; + + if (unlikely(msg_size >= max_size)) { + pr_err("QAT: Invalid size %d provided for message type %d\n", + msg_size, type); + *data = ADF_PF2VF_PAYLOAD_TRUNCATED; + return -EINVAL; + } + + if (unlikely(byte >= msg_size)) { + pr_err("QAT: Out-of-bound byte number %d (msg size %d)\n", + byte, msg_size); + *data = ADF_PF2VF_INVALID_BYTE_NUM_REQ; + return -EINVAL; + } + + *data = data_getter(blkmsg, byte); + return 0; +} + +static struct pfvf_message handle_blkmsg_req(struct adf_accel_vf_info *vf_info, + struct pfvf_message req) +{ + u8 resp_type = ADF_PF2VF_BLKMSG_RESP_TYPE_ERROR; + struct pfvf_message resp = { 0 }; + u8 resp_data = 0; + u8 blk_type; + u8 blk_byte; + u8 byte_max; + + switch (req.type) { + case ADF_VF2PF_MSGTYPE_LARGE_BLOCK_REQ: + blk_type = FIELD_GET(ADF_VF2PF_LARGE_BLOCK_TYPE_MASK, req.data) + + ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX + 1; + blk_byte = FIELD_GET(ADF_VF2PF_LARGE_BLOCK_BYTE_MASK, req.data); + byte_max = ADF_VF2PF_LARGE_BLOCK_BYTE_MAX; + break; + case ADF_VF2PF_MSGTYPE_MEDIUM_BLOCK_REQ: + blk_type = FIELD_GET(ADF_VF2PF_MEDIUM_BLOCK_TYPE_MASK, req.data) + + ADF_VF2PF_SMALL_BLOCK_TYPE_MAX + 1; + blk_byte = FIELD_GET(ADF_VF2PF_MEDIUM_BLOCK_BYTE_MASK, req.data); + byte_max = ADF_VF2PF_MEDIUM_BLOCK_BYTE_MAX; + break; + case ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ: + blk_type = FIELD_GET(ADF_VF2PF_SMALL_BLOCK_TYPE_MASK, req.data); + blk_byte = FIELD_GET(ADF_VF2PF_SMALL_BLOCK_BYTE_MASK, req.data); + byte_max = ADF_VF2PF_SMALL_BLOCK_BYTE_MAX; + break; + default: + dev_err(&GET_DEV(vf_info->accel_dev), + "Invalid BlockMsg type 0x%.4x received from VF%u\n", + req.type, vf_info->vf_nr); + resp.type = ADF_PF2VF_MSGTYPE_BLKMSG_RESP; + resp.data = FIELD_PREP(ADF_PF2VF_BLKMSG_RESP_TYPE_MASK, + ADF_PF2VF_BLKMSG_RESP_TYPE_ERROR) | + FIELD_PREP(ADF_PF2VF_BLKMSG_RESP_DATA_MASK, + ADF_PF2VF_UNSPECIFIED_ERROR); + return resp; + } + + /* Is this a request for CRC or data? */ + if (FIELD_GET(ADF_VF2PF_BLOCK_CRC_REQ_MASK, req.data)) { + dev_dbg(&GET_DEV(vf_info->accel_dev), + "BlockMsg of type %d for CRC over %d bytes received from VF%d\n", + blk_type, blk_byte + 1, vf_info->vf_nr); + + if (!adf_pf2vf_blkmsg_get_data(vf_info, blk_type, blk_byte, + byte_max, &resp_data, + adf_pf2vf_blkmsg_get_crc)) + resp_type = ADF_PF2VF_BLKMSG_RESP_TYPE_CRC; + } else { + dev_dbg(&GET_DEV(vf_info->accel_dev), + "BlockMsg of type %d for data byte %d received from VF%d\n", + blk_type, blk_byte, vf_info->vf_nr); + + if (!adf_pf2vf_blkmsg_get_data(vf_info, blk_type, blk_byte, + byte_max, &resp_data, + adf_pf2vf_blkmsg_get_byte)) + resp_type = ADF_PF2VF_BLKMSG_RESP_TYPE_DATA; + } + + resp.type = ADF_PF2VF_MSGTYPE_BLKMSG_RESP; + resp.data = FIELD_PREP(ADF_PF2VF_BLKMSG_RESP_TYPE_MASK, resp_type) | + FIELD_PREP(ADF_PF2VF_BLKMSG_RESP_DATA_MASK, resp_data); + + return resp; +} + +static struct pfvf_message handle_rp_reset_req(struct adf_accel_dev *accel_dev, u8 vf_nr, + struct pfvf_message req) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct pfvf_message resp = { + .type = ADF_PF2VF_MSGTYPE_RP_RESET_RESP, + .data = RPRESET_SUCCESS + }; + u32 bank_number; + u32 rsvd_field; + + bank_number = FIELD_GET(ADF_VF2PF_RNG_RESET_RP_MASK, req.data); + rsvd_field = FIELD_GET(ADF_VF2PF_RNG_RESET_RSVD_MASK, req.data); + + dev_dbg(&GET_DEV(accel_dev), + "Ring Pair Reset Message received from VF%d for bank 0x%x\n", + vf_nr, bank_number); + + if (!hw_data->ring_pair_reset || rsvd_field) { + dev_dbg(&GET_DEV(accel_dev), + "Ring Pair Reset for VF%d is not supported\n", vf_nr); + resp.data = RPRESET_NOT_SUPPORTED; + goto out; + } + + if (bank_number >= hw_data->num_banks_per_vf) { + dev_err(&GET_DEV(accel_dev), + "Invalid bank number (0x%x) from VF%d for Ring Reset\n", + bank_number, vf_nr); + resp.data = RPRESET_INVAL_BANK; + goto out; + } + + /* Convert the VF provided value to PF bank number */ + bank_number = vf_nr * hw_data->num_banks_per_vf + bank_number; + if (hw_data->ring_pair_reset(accel_dev, bank_number)) { + dev_dbg(&GET_DEV(accel_dev), + "Ring pair reset for VF%d failure\n", vf_nr); + resp.data = RPRESET_TIMEOUT; + goto out; + } + + dev_dbg(&GET_DEV(accel_dev), + "Ring pair reset for VF%d successfully\n", vf_nr); + +out: + return resp; +} + +static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, + struct pfvf_message msg, struct pfvf_message *resp) +{ + struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; + + switch (msg.type) { + case ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ: + { + u8 vf_compat_ver = msg.data; + u8 compat; + + dev_dbg(&GET_DEV(accel_dev), + "VersionRequest received from VF%d (vers %d) to PF (vers %d)\n", + vf_nr, vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION); + + compat = adf_vf_compat_checker(vf_compat_ver); + vf_info->vf_compat_ver = vf_compat_ver; + + resp->type = ADF_PF2VF_MSGTYPE_VERSION_RESP; + resp->data = FIELD_PREP(ADF_PF2VF_VERSION_RESP_VERS_MASK, + ADF_PFVF_COMPAT_THIS_VERSION) | + FIELD_PREP(ADF_PF2VF_VERSION_RESP_RESULT_MASK, compat); + } + break; + case ADF_VF2PF_MSGTYPE_VERSION_REQ: + { + u8 compat; + + dev_dbg(&GET_DEV(accel_dev), + "Legacy VersionRequest received from VF%d to PF (vers 1.1)\n", + vf_nr); + + /* legacy driver, VF compat_ver is 0 */ + vf_info->vf_compat_ver = 0; + + /* PF always newer than legacy VF */ + compat = ADF_PF2VF_VF_COMPATIBLE; + + /* Set legacy major and minor version to the latest, 1.1 */ + resp->type = ADF_PF2VF_MSGTYPE_VERSION_RESP; + resp->data = FIELD_PREP(ADF_PF2VF_VERSION_RESP_VERS_MASK, 0x11) | + FIELD_PREP(ADF_PF2VF_VERSION_RESP_RESULT_MASK, compat); + } + break; + case ADF_VF2PF_MSGTYPE_INIT: + { + dev_dbg(&GET_DEV(accel_dev), + "Init message received from VF%d\n", vf_nr); + vf_info->init = true; + } + break; + case ADF_VF2PF_MSGTYPE_SHUTDOWN: + { + dev_dbg(&GET_DEV(accel_dev), + "Shutdown message received from VF%d\n", vf_nr); + vf_info->init = false; + } + break; + case ADF_VF2PF_MSGTYPE_RESTARTING_COMPLETE: + { + dev_dbg(&GET_DEV(accel_dev), + "Restarting Complete received from VF%d\n", vf_nr); + vf_info->restarting = false; + vf_info->init = false; + } + break; + case ADF_VF2PF_MSGTYPE_LARGE_BLOCK_REQ: + case ADF_VF2PF_MSGTYPE_MEDIUM_BLOCK_REQ: + case ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ: + *resp = handle_blkmsg_req(vf_info, msg); + break; + case ADF_VF2PF_MSGTYPE_RP_RESET: + *resp = handle_rp_reset_req(accel_dev, vf_nr, msg); + break; + default: + dev_dbg(&GET_DEV(accel_dev), + "Unknown message from VF%d (type 0x%.4x, data: 0x%.4x)\n", + vf_nr, msg.type, msg.data); + return -ENOMSG; + } + + return 0; +} + +bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr) +{ + struct pfvf_message req; + struct pfvf_message resp = {0}; + + req = adf_recv_vf2pf_msg(accel_dev, vf_nr); + if (!req.type) /* Legacy or no message */ + return true; + + if (adf_handle_vf2pf_msg(accel_dev, vf_nr, req, &resp)) + return false; + + if (resp.type && adf_send_pf2vf_msg(accel_dev, vf_nr, resp)) + dev_err(&GET_DEV(accel_dev), + "Failed to send response to VF%d\n", vf_nr); + + return true; +} + +/** + * adf_enable_pf2vf_comms() - Function enables communication from pf to vf + * + * @accel_dev: Pointer to acceleration device virtual function. + * + * This function carries out the necessary steps to setup and start the PFVF + * communication channel, if any. + * + * Return: 0 on success, error code otherwise. + */ +int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev) +{ + adf_pfvf_crc_init(); + spin_lock_init(&accel_dev->pf.vf2pf_ints_lock); + + return 0; +} +EXPORT_SYMBOL_GPL(adf_enable_pf2vf_comms); diff --git a/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.h b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.h new file mode 100644 index 000000000..165d266d0 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2021 Intel Corporation */ +#ifndef ADF_PFVF_PF_PROTO_H +#define ADF_PFVF_PF_PROTO_H + +#include +#include "adf_accel_devices.h" + +int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, struct pfvf_message msg); + +int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev); + +#endif /* ADF_PFVF_PF_PROTO_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_pfvf_utils.c b/drivers/crypto/intel/qat/qat_common/adf_pfvf_utils.c new file mode 100644 index 000000000..c5f6d77d4 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_utils.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2021 Intel Corporation */ +#include +#include +#include +#include "adf_accel_devices.h" +#include "adf_pfvf_msg.h" +#include "adf_pfvf_utils.h" + +/* CRC Calculation */ +DECLARE_CRC8_TABLE(pfvf_crc8_table); +#define ADF_PFVF_CRC8_POLYNOMIAL 0x97 + +void adf_pfvf_crc_init(void) +{ + crc8_populate_msb(pfvf_crc8_table, ADF_PFVF_CRC8_POLYNOMIAL); +} + +u8 adf_pfvf_calc_blkmsg_crc(u8 const *buf, u8 buf_len) +{ + return crc8(pfvf_crc8_table, buf, buf_len, CRC8_INIT_VALUE); +} + +static bool set_value_on_csr_msg(struct adf_accel_dev *accel_dev, u32 *csr_msg, + u32 value, const struct pfvf_field_format *fmt) +{ + if (unlikely((value & fmt->mask) != value)) { + dev_err(&GET_DEV(accel_dev), + "PFVF message value 0x%X out of range, %u max allowed\n", + value, fmt->mask); + return false; + } + + *csr_msg |= value << fmt->offset; + + return true; +} + +u32 adf_pfvf_csr_msg_of(struct adf_accel_dev *accel_dev, + struct pfvf_message msg, + const struct pfvf_csr_format *fmt) +{ + u32 csr_msg = 0; + + if (!set_value_on_csr_msg(accel_dev, &csr_msg, msg.type, &fmt->type) || + !set_value_on_csr_msg(accel_dev, &csr_msg, msg.data, &fmt->data)) + return 0; + + return csr_msg | ADF_PFVF_MSGORIGIN_SYSTEM; +} + +struct pfvf_message adf_pfvf_message_of(struct adf_accel_dev *accel_dev, u32 csr_msg, + const struct pfvf_csr_format *fmt) +{ + struct pfvf_message msg = { 0 }; + + msg.type = (csr_msg >> fmt->type.offset) & fmt->type.mask; + msg.data = (csr_msg >> fmt->data.offset) & fmt->data.mask; + + if (unlikely(!msg.type)) + dev_err(&GET_DEV(accel_dev), + "Invalid PFVF msg with no type received\n"); + + return msg; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_pfvf_utils.h b/drivers/crypto/intel/qat/qat_common/adf_pfvf_utils.h new file mode 100644 index 000000000..1a044297d --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_utils.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2021 Intel Corporation */ +#ifndef ADF_PFVF_UTILS_H +#define ADF_PFVF_UTILS_H + +#include +#include "adf_pfvf_msg.h" + +/* How long to wait for far side to acknowledge receipt */ +#define ADF_PFVF_MSG_ACK_DELAY_US 4 +#define ADF_PFVF_MSG_ACK_MAX_DELAY_US (1 * USEC_PER_SEC) + +u8 adf_pfvf_calc_blkmsg_crc(u8 const *buf, u8 buf_len); +void adf_pfvf_crc_init(void); + +struct pfvf_field_format { + u8 offset; + u32 mask; +}; + +struct pfvf_csr_format { + struct pfvf_field_format type; + struct pfvf_field_format data; +}; + +u32 adf_pfvf_csr_msg_of(struct adf_accel_dev *accel_dev, struct pfvf_message msg, + const struct pfvf_csr_format *fmt); +struct pfvf_message adf_pfvf_message_of(struct adf_accel_dev *accel_dev, u32 raw_msg, + const struct pfvf_csr_format *fmt); + +static inline u8 adf_vf_compat_checker(u8 vf_compat_ver) +{ + if (vf_compat_ver == 0) + return ADF_PF2VF_VF_INCOMPATIBLE; + + if (vf_compat_ver <= ADF_PFVF_COMPAT_THIS_VERSION) + return ADF_PF2VF_VF_COMPATIBLE; + + return ADF_PF2VF_VF_COMPAT_UNKNOWN; +} + +#endif /* ADF_PFVF_UTILS_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_pfvf_vf_msg.c b/drivers/crypto/intel/qat/qat_common/adf_pfvf_vf_msg.c new file mode 100644 index 000000000..10c91e56d --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_vf_msg.c @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2015 - 2021 Intel Corporation */ +#include +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_pfvf_msg.h" +#include "adf_pfvf_vf_msg.h" +#include "adf_pfvf_vf_proto.h" + +/** + * adf_vf2pf_notify_init() - send init msg to PF + * @accel_dev: Pointer to acceleration VF device. + * + * Function sends an init message from the VF to a PF + * + * Return: 0 on success, error code otherwise. + */ +int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev) +{ + struct pfvf_message msg = { .type = ADF_VF2PF_MSGTYPE_INIT }; + + if (adf_send_vf2pf_msg(accel_dev, msg)) { + dev_err(&GET_DEV(accel_dev), + "Failed to send Init event to PF\n"); + return -EFAULT; + } + set_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status); + return 0; +} +EXPORT_SYMBOL_GPL(adf_vf2pf_notify_init); + +/** + * adf_vf2pf_notify_shutdown() - send shutdown msg to PF + * @accel_dev: Pointer to acceleration VF device. + * + * Function sends a shutdown message from the VF to a PF + * + * Return: void + */ +void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev) +{ + struct pfvf_message msg = { .type = ADF_VF2PF_MSGTYPE_SHUTDOWN }; + + if (test_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status)) + if (adf_send_vf2pf_msg(accel_dev, msg)) + dev_err(&GET_DEV(accel_dev), + "Failed to send Shutdown event to PF\n"); +} +EXPORT_SYMBOL_GPL(adf_vf2pf_notify_shutdown); + +void adf_vf2pf_notify_restart_complete(struct adf_accel_dev *accel_dev) +{ + struct pfvf_message msg = { .type = ADF_VF2PF_MSGTYPE_RESTARTING_COMPLETE }; + + /* Check compatibility version */ + if (accel_dev->vf.pf_compat_ver < ADF_PFVF_COMPAT_FALLBACK) + return; + + if (adf_send_vf2pf_msg(accel_dev, msg)) + dev_err(&GET_DEV(accel_dev), + "Failed to send Restarting complete event to PF\n"); +} +EXPORT_SYMBOL_GPL(adf_vf2pf_notify_restart_complete); + +int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev) +{ + u8 pf_version; + int compat; + int ret; + struct pfvf_message resp; + struct pfvf_message msg = { + .type = ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ, + .data = ADF_PFVF_COMPAT_THIS_VERSION, + }; + + BUILD_BUG_ON(ADF_PFVF_COMPAT_THIS_VERSION > 255); + + ret = adf_send_vf2pf_req(accel_dev, msg, &resp); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to send Compatibility Version Request.\n"); + return ret; + } + + pf_version = FIELD_GET(ADF_PF2VF_VERSION_RESP_VERS_MASK, resp.data); + compat = FIELD_GET(ADF_PF2VF_VERSION_RESP_RESULT_MASK, resp.data); + + /* Response from PF received, check compatibility */ + switch (compat) { + case ADF_PF2VF_VF_COMPATIBLE: + break; + case ADF_PF2VF_VF_COMPAT_UNKNOWN: + /* VF is newer than PF - compatible for now */ + break; + case ADF_PF2VF_VF_INCOMPATIBLE: + dev_err(&GET_DEV(accel_dev), + "PF (vers %d) and VF (vers %d) are not compatible\n", + pf_version, ADF_PFVF_COMPAT_THIS_VERSION); + return -EINVAL; + default: + dev_err(&GET_DEV(accel_dev), + "Invalid response from PF; assume not compatible\n"); + return -EINVAL; + } + + accel_dev->vf.pf_compat_ver = pf_version; + return 0; +} + +int adf_vf2pf_get_capabilities(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct capabilities_v3 cap_msg = { 0 }; + unsigned int len = sizeof(cap_msg); + + if (accel_dev->vf.pf_compat_ver < ADF_PFVF_COMPAT_CAPABILITIES) + /* The PF is too old to support the extended capabilities */ + return 0; + + if (adf_send_vf2pf_blkmsg_req(accel_dev, ADF_VF2PF_BLKMSG_REQ_CAP_SUMMARY, + (u8 *)&cap_msg, &len)) { + dev_err(&GET_DEV(accel_dev), + "QAT: Failed to get block message response\n"); + return -EFAULT; + } + + switch (cap_msg.hdr.version) { + default: + /* Newer version received, handle only the know parts */ + fallthrough; + case ADF_PFVF_CAPABILITIES_V3_VERSION: + if (likely(len >= sizeof(struct capabilities_v3))) + hw_data->clock_frequency = cap_msg.frequency; + else + dev_info(&GET_DEV(accel_dev), "Could not get frequency"); + fallthrough; + case ADF_PFVF_CAPABILITIES_V2_VERSION: + if (likely(len >= sizeof(struct capabilities_v2))) + hw_data->accel_capabilities_mask = cap_msg.capabilities; + else + dev_info(&GET_DEV(accel_dev), "Could not get capabilities"); + fallthrough; + case ADF_PFVF_CAPABILITIES_V1_VERSION: + if (likely(len >= sizeof(struct capabilities_v1))) { + hw_data->extended_dc_capabilities = cap_msg.ext_dc_caps; + } else { + dev_err(&GET_DEV(accel_dev), + "Capabilities message truncated to %d bytes\n", len); + return -EFAULT; + } + } + + return 0; +} + +int adf_vf2pf_get_ring_to_svc(struct adf_accel_dev *accel_dev) +{ + struct ring_to_svc_map_v1 rts_map_msg = { 0 }; + unsigned int len = sizeof(rts_map_msg); + + if (accel_dev->vf.pf_compat_ver < ADF_PFVF_COMPAT_RING_TO_SVC_MAP) + /* Use already set default mappings */ + return 0; + + if (adf_send_vf2pf_blkmsg_req(accel_dev, ADF_VF2PF_BLKMSG_REQ_RING_SVC_MAP, + (u8 *)&rts_map_msg, &len)) { + dev_err(&GET_DEV(accel_dev), + "QAT: Failed to get block message response\n"); + return -EFAULT; + } + + if (unlikely(len < sizeof(struct ring_to_svc_map_v1))) { + dev_err(&GET_DEV(accel_dev), + "RING_TO_SVC message truncated to %d bytes\n", len); + return -EFAULT; + } + + /* Only v1 at present */ + accel_dev->hw_device->ring_to_svc_map = rts_map_msg.map; + return 0; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_pfvf_vf_msg.h b/drivers/crypto/intel/qat/qat_common/adf_pfvf_vf_msg.h new file mode 100644 index 000000000..d79340ab3 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_vf_msg.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2021 Intel Corporation */ +#ifndef ADF_PFVF_VF_MSG_H +#define ADF_PFVF_VF_MSG_H + +#if defined(CONFIG_PCI_IOV) +int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev); +void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev); +void adf_vf2pf_notify_restart_complete(struct adf_accel_dev *accel_dev); +int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev); +int adf_vf2pf_get_capabilities(struct adf_accel_dev *accel_dev); +int adf_vf2pf_get_ring_to_svc(struct adf_accel_dev *accel_dev); +#else +static inline int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev) +{ + return 0; +} + +static inline void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev) +{ +} +#endif + +#endif /* ADF_PFVF_VF_MSG_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_pfvf_vf_proto.c b/drivers/crypto/intel/qat/qat_common/adf_pfvf_vf_proto.c new file mode 100644 index 000000000..dc284a089 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_vf_proto.c @@ -0,0 +1,374 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2015 - 2021 Intel Corporation */ +#include +#include +#include +#include +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_pfvf_msg.h" +#include "adf_pfvf_utils.h" +#include "adf_pfvf_vf_msg.h" +#include "adf_pfvf_vf_proto.h" + +#define ADF_PFVF_MSG_COLLISION_DETECT_DELAY 10 +#define ADF_PFVF_MSG_ACK_DELAY 2 +#define ADF_PFVF_MSG_ACK_MAX_RETRY 100 + +/* How often to retry if there is no response */ +#define ADF_PFVF_MSG_RESP_RETRIES 5 +#define ADF_PFVF_MSG_RESP_TIMEOUT (ADF_PFVF_MSG_ACK_DELAY * \ + ADF_PFVF_MSG_ACK_MAX_RETRY + \ + ADF_PFVF_MSG_COLLISION_DETECT_DELAY) + +/** + * adf_send_vf2pf_msg() - send VF to PF message + * @accel_dev: Pointer to acceleration device + * @msg: Message to send + * + * This function allows the VF to send a message to the PF. + * + * Return: 0 on success, error code otherwise. + */ +int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, struct pfvf_message msg) +{ + struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev); + u32 pfvf_offset = pfvf_ops->get_vf2pf_offset(0); + + return pfvf_ops->send_msg(accel_dev, msg, pfvf_offset, + &accel_dev->vf.vf2pf_lock); +} + +/** + * adf_recv_pf2vf_msg() - receive a PF to VF message + * @accel_dev: Pointer to acceleration device + * + * This function allows the VF to receive a message from the PF. + * + * Return: a valid message on success, zero otherwise. + */ +static struct pfvf_message adf_recv_pf2vf_msg(struct adf_accel_dev *accel_dev) +{ + struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev); + u32 pfvf_offset = pfvf_ops->get_pf2vf_offset(0); + + return pfvf_ops->recv_msg(accel_dev, pfvf_offset, accel_dev->vf.pf_compat_ver); +} + +/** + * adf_send_vf2pf_req() - send VF2PF request message + * @accel_dev: Pointer to acceleration device. + * @msg: Request message to send + * @resp: Returned PF response + * + * This function sends a message that requires a response from the VF to the PF + * and waits for a reply. + * + * Return: 0 on success, error code otherwise. + */ +int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, struct pfvf_message msg, + struct pfvf_message *resp) +{ + unsigned long timeout = msecs_to_jiffies(ADF_PFVF_MSG_RESP_TIMEOUT); + unsigned int retries = ADF_PFVF_MSG_RESP_RETRIES; + int ret; + + reinit_completion(&accel_dev->vf.msg_received); + + /* Send request from VF to PF */ + do { + ret = adf_send_vf2pf_msg(accel_dev, msg); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to send request msg to PF\n"); + return ret; + } + + /* Wait for response, if it times out retry */ + ret = wait_for_completion_timeout(&accel_dev->vf.msg_received, + timeout); + if (ret) { + if (likely(resp)) + *resp = accel_dev->vf.response; + + /* Once copied, set to an invalid value */ + accel_dev->vf.response.type = 0; + + return 0; + } + + dev_err(&GET_DEV(accel_dev), "PFVF response message timeout\n"); + } while (--retries); + + return -EIO; +} + +static int adf_vf2pf_blkmsg_data_req(struct adf_accel_dev *accel_dev, bool crc, + u8 *type, u8 *data) +{ + struct pfvf_message req = { 0 }; + struct pfvf_message resp = { 0 }; + u8 blk_type; + u8 blk_byte; + u8 msg_type; + u8 max_data; + int err; + + /* Convert the block type to {small, medium, large} size category */ + if (*type <= ADF_VF2PF_SMALL_BLOCK_TYPE_MAX) { + msg_type = ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ; + blk_type = FIELD_PREP(ADF_VF2PF_SMALL_BLOCK_TYPE_MASK, *type); + blk_byte = FIELD_PREP(ADF_VF2PF_SMALL_BLOCK_BYTE_MASK, *data); + max_data = ADF_VF2PF_SMALL_BLOCK_BYTE_MAX; + } else if (*type <= ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX) { + msg_type = ADF_VF2PF_MSGTYPE_MEDIUM_BLOCK_REQ; + blk_type = FIELD_PREP(ADF_VF2PF_MEDIUM_BLOCK_TYPE_MASK, + *type - ADF_VF2PF_SMALL_BLOCK_TYPE_MAX); + blk_byte = FIELD_PREP(ADF_VF2PF_MEDIUM_BLOCK_BYTE_MASK, *data); + max_data = ADF_VF2PF_MEDIUM_BLOCK_BYTE_MAX; + } else if (*type <= ADF_VF2PF_LARGE_BLOCK_TYPE_MAX) { + msg_type = ADF_VF2PF_MSGTYPE_LARGE_BLOCK_REQ; + blk_type = FIELD_PREP(ADF_VF2PF_LARGE_BLOCK_TYPE_MASK, + *type - ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX); + blk_byte = FIELD_PREP(ADF_VF2PF_LARGE_BLOCK_BYTE_MASK, *data); + max_data = ADF_VF2PF_LARGE_BLOCK_BYTE_MAX; + } else { + dev_err(&GET_DEV(accel_dev), "Invalid message type %u\n", *type); + return -EINVAL; + } + + /* Sanity check */ + if (*data > max_data) { + dev_err(&GET_DEV(accel_dev), + "Invalid byte %s %u for message type %u\n", + crc ? "count" : "index", *data, *type); + return -EINVAL; + } + + /* Build the block message */ + req.type = msg_type; + req.data = blk_type | blk_byte | FIELD_PREP(ADF_VF2PF_BLOCK_CRC_REQ_MASK, crc); + + err = adf_send_vf2pf_req(accel_dev, req, &resp); + if (err) + return err; + + *type = FIELD_GET(ADF_PF2VF_BLKMSG_RESP_TYPE_MASK, resp.data); + *data = FIELD_GET(ADF_PF2VF_BLKMSG_RESP_DATA_MASK, resp.data); + + return 0; +} + +static int adf_vf2pf_blkmsg_get_byte(struct adf_accel_dev *accel_dev, u8 type, + u8 index, u8 *data) +{ + int ret; + + ret = adf_vf2pf_blkmsg_data_req(accel_dev, false, &type, &index); + if (ret < 0) + return ret; + + if (unlikely(type != ADF_PF2VF_BLKMSG_RESP_TYPE_DATA)) { + dev_err(&GET_DEV(accel_dev), + "Unexpected BLKMSG response type %u, byte 0x%x\n", + type, index); + return -EFAULT; + } + + *data = index; + return 0; +} + +static int adf_vf2pf_blkmsg_get_crc(struct adf_accel_dev *accel_dev, u8 type, + u8 bytes, u8 *crc) +{ + int ret; + + /* The count of bytes refers to a length, however shift it to a 0-based + * count to avoid overflows. Thus, a request for 0 bytes is technically + * valid. + */ + --bytes; + + ret = adf_vf2pf_blkmsg_data_req(accel_dev, true, &type, &bytes); + if (ret < 0) + return ret; + + if (unlikely(type != ADF_PF2VF_BLKMSG_RESP_TYPE_CRC)) { + dev_err(&GET_DEV(accel_dev), + "Unexpected CRC BLKMSG response type %u, crc 0x%x\n", + type, bytes); + return -EFAULT; + } + + *crc = bytes; + return 0; +} + +/** + * adf_send_vf2pf_blkmsg_req() - retrieve block message + * @accel_dev: Pointer to acceleration VF device. + * @type: The block message type, see adf_pfvf_msg.h for allowed values + * @buffer: input buffer where to place the received data + * @buffer_len: buffer length as input, the amount of written bytes on output + * + * Request a message of type 'type' over the block message transport. + * This function will send the required amount block message requests and + * return the overall content back to the caller through the provided buffer. + * The buffer should be large enough to contain the requested message type, + * otherwise the response will be truncated. + * + * Return: 0 on success, error code otherwise. + */ +int adf_send_vf2pf_blkmsg_req(struct adf_accel_dev *accel_dev, u8 type, + u8 *buffer, unsigned int *buffer_len) +{ + unsigned int index; + unsigned int msg_len; + int ret; + u8 remote_crc; + u8 local_crc; + + if (unlikely(type > ADF_VF2PF_LARGE_BLOCK_TYPE_MAX)) { + dev_err(&GET_DEV(accel_dev), "Invalid block message type %d\n", + type); + return -EINVAL; + } + + if (unlikely(*buffer_len < ADF_PFVF_BLKMSG_HEADER_SIZE)) { + dev_err(&GET_DEV(accel_dev), + "Buffer size too small for a block message\n"); + return -EINVAL; + } + + ret = adf_vf2pf_blkmsg_get_byte(accel_dev, type, + ADF_PFVF_BLKMSG_VER_BYTE, + &buffer[ADF_PFVF_BLKMSG_VER_BYTE]); + if (unlikely(ret)) + return ret; + + if (unlikely(!buffer[ADF_PFVF_BLKMSG_VER_BYTE])) { + dev_err(&GET_DEV(accel_dev), + "Invalid version 0 received for block request %u", type); + return -EFAULT; + } + + ret = adf_vf2pf_blkmsg_get_byte(accel_dev, type, + ADF_PFVF_BLKMSG_LEN_BYTE, + &buffer[ADF_PFVF_BLKMSG_LEN_BYTE]); + if (unlikely(ret)) + return ret; + + if (unlikely(!buffer[ADF_PFVF_BLKMSG_LEN_BYTE])) { + dev_err(&GET_DEV(accel_dev), + "Invalid size 0 received for block request %u", type); + return -EFAULT; + } + + /* We need to pick the minimum since there is no way to request a + * specific version. As a consequence any scenario is possible: + * - PF has a newer (longer) version which doesn't fit in the buffer + * - VF expects a newer (longer) version, so we must not ask for + * bytes in excess + * - PF and VF share the same version, no problem + */ + msg_len = ADF_PFVF_BLKMSG_HEADER_SIZE + buffer[ADF_PFVF_BLKMSG_LEN_BYTE]; + msg_len = min(*buffer_len, msg_len); + + /* Get the payload */ + for (index = ADF_PFVF_BLKMSG_HEADER_SIZE; index < msg_len; index++) { + ret = adf_vf2pf_blkmsg_get_byte(accel_dev, type, index, + &buffer[index]); + if (unlikely(ret)) + return ret; + } + + ret = adf_vf2pf_blkmsg_get_crc(accel_dev, type, msg_len, &remote_crc); + if (unlikely(ret)) + return ret; + + local_crc = adf_pfvf_calc_blkmsg_crc(buffer, msg_len); + if (unlikely(local_crc != remote_crc)) { + dev_err(&GET_DEV(accel_dev), + "CRC error on msg type %d. Local %02X, remote %02X\n", + type, local_crc, remote_crc); + return -EIO; + } + + *buffer_len = msg_len; + return 0; +} + +static bool adf_handle_pf2vf_msg(struct adf_accel_dev *accel_dev, + struct pfvf_message msg) +{ + switch (msg.type) { + case ADF_PF2VF_MSGTYPE_RESTARTING: + dev_dbg(&GET_DEV(accel_dev), "Restarting message received from PF\n"); + + adf_pf2vf_handle_pf_restarting(accel_dev); + return false; + case ADF_PF2VF_MSGTYPE_RESTARTED: + dev_dbg(&GET_DEV(accel_dev), "Restarted message received from PF\n"); + return true; + case ADF_PF2VF_MSGTYPE_FATAL_ERROR: + dev_err(&GET_DEV(accel_dev), "Fatal error received from PF\n"); + return true; + case ADF_PF2VF_MSGTYPE_VERSION_RESP: + case ADF_PF2VF_MSGTYPE_BLKMSG_RESP: + case ADF_PF2VF_MSGTYPE_RP_RESET_RESP: + dev_dbg(&GET_DEV(accel_dev), + "Response Message received from PF (type 0x%.4x, data 0x%.4x)\n", + msg.type, msg.data); + accel_dev->vf.response = msg; + complete(&accel_dev->vf.msg_received); + return true; + default: + dev_err(&GET_DEV(accel_dev), + "Unknown message from PF (type 0x%.4x, data: 0x%.4x)\n", + msg.type, msg.data); + } + + return false; +} + +bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev) +{ + struct pfvf_message msg; + + msg = adf_recv_pf2vf_msg(accel_dev); + if (msg.type) /* Invalid or no message */ + return adf_handle_pf2vf_msg(accel_dev, msg); + + /* No replies for PF->VF messages at present */ + + return true; +} + +/** + * adf_enable_vf2pf_comms() - Function enables communication from vf to pf + * + * @accel_dev: Pointer to acceleration device virtual function. + * + * Return: 0 on success, error code otherwise. + */ +int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) +{ + int ret; + + adf_pfvf_crc_init(); + adf_enable_pf2vf_interrupts(accel_dev); + + ret = adf_vf2pf_request_version(accel_dev); + if (ret) + return ret; + + ret = adf_vf2pf_get_capabilities(accel_dev); + if (ret) + return ret; + + ret = adf_vf2pf_get_ring_to_svc(accel_dev); + + return ret; +} +EXPORT_SYMBOL_GPL(adf_enable_vf2pf_comms); diff --git a/drivers/crypto/intel/qat/qat_common/adf_pfvf_vf_proto.h b/drivers/crypto/intel/qat/qat_common/adf_pfvf_vf_proto.h new file mode 100644 index 000000000..f6ee9b38c --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_vf_proto.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2021 Intel Corporation */ +#ifndef ADF_PFVF_VF_PROTO_H +#define ADF_PFVF_VF_PROTO_H + +#include +#include "adf_accel_devices.h" + +int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, struct pfvf_message msg); +int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, struct pfvf_message msg, + struct pfvf_message *resp); +int adf_send_vf2pf_blkmsg_req(struct adf_accel_dev *accel_dev, u8 type, + u8 *buffer, unsigned int *buffer_len); + +int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev); + +#endif /* ADF_PFVF_VF_PROTO_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs.c b/drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs.c new file mode 100644 index 000000000..f0a13c190 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs.c @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ +#include +#include +#include + +#include "adf_accel_devices.h" +#include "adf_pm_dbgfs.h" + +static ssize_t pm_status_read(struct file *f, char __user *buf, size_t count, + loff_t *pos) +{ + struct adf_accel_dev *accel_dev = file_inode(f)->i_private; + struct adf_pm pm = accel_dev->power_management; + + if (pm.print_pm_status) + return pm.print_pm_status(accel_dev, buf, count, pos); + + return count; +} + +static const struct file_operations pm_status_fops = { + .owner = THIS_MODULE, + .read = pm_status_read, +}; + +void adf_pm_dbgfs_add(struct adf_accel_dev *accel_dev) +{ + struct adf_pm *pm = &accel_dev->power_management; + + if (!pm->present || !pm->print_pm_status) + return; + + pm->debugfs_pm_status = debugfs_create_file("pm_status", 0400, + accel_dev->debugfs_dir, + accel_dev, &pm_status_fops); +} + +void adf_pm_dbgfs_rm(struct adf_accel_dev *accel_dev) +{ + struct adf_pm *pm = &accel_dev->power_management; + + if (!pm->present) + return; + + debugfs_remove(pm->debugfs_pm_status); + pm->debugfs_pm_status = NULL; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs.h b/drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs.h new file mode 100644 index 000000000..83632e5aa --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ + +#ifndef ADF_PM_DBGFS_H_ +#define ADF_PM_DBGFS_H_ + +struct adf_accel_dev; + +void adf_pm_dbgfs_rm(struct adf_accel_dev *accel_dev); +void adf_pm_dbgfs_add(struct adf_accel_dev *accel_dev); + +#endif /* ADF_PM_DBGFS_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs_utils.c b/drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs_utils.c new file mode 100644 index 000000000..4ccc94ed9 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs_utils.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2025 Intel Corporation */ +#include +#include +#include +#include + +#include "adf_pm_dbgfs_utils.h" + +#define PM_INFO_MAX_KEY_LEN 21 + +static int pm_scnprint_table(char *buff, const struct pm_status_row *table, + u32 *pm_info_regs, size_t buff_size, int table_len, + bool lowercase) +{ + char key[PM_INFO_MAX_KEY_LEN]; + int wr = 0; + int i; + + for (i = 0; i < table_len; i++) { + if (lowercase) + string_lower(key, table[i].key); + else + string_upper(key, table[i].key); + + wr += scnprintf(&buff[wr], buff_size - wr, "%s: %#x\n", key, + field_get(table[i].field_mask, + pm_info_regs[table[i].reg_offset])); + } + + return wr; +} + +int adf_pm_scnprint_table_upper_keys(char *buff, const struct pm_status_row *table, + u32 *pm_info_regs, size_t buff_size, int table_len) +{ + return pm_scnprint_table(buff, table, pm_info_regs, buff_size, + table_len, false); +} + +int adf_pm_scnprint_table_lower_keys(char *buff, const struct pm_status_row *table, + u32 *pm_info_regs, size_t buff_size, int table_len) +{ + return pm_scnprint_table(buff, table, pm_info_regs, buff_size, + table_len, true); +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs_utils.h b/drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs_utils.h new file mode 100644 index 000000000..854f058b3 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs_utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ADF_PM_DBGFS_UTILS_H_ +#define ADF_PM_DBGFS_UTILS_H_ + +#include +#include +#include +#include "icp_qat_fw_init_admin.h" + +#define PM_INFO_MEMBER_OFF(member) \ + (offsetof(struct icp_qat_fw_init_admin_pm_info, member) / sizeof(u32)) + +#define PM_INFO_REGSET_ENTRY_MASK(_reg_, _field_, _mask_) \ +{ \ + .reg_offset = PM_INFO_MEMBER_OFF(_reg_), \ + .key = __stringify(_field_), \ + .field_mask = _mask_, \ +} + +#define PM_INFO_REGSET_ENTRY32(_reg_, _field_) \ + PM_INFO_REGSET_ENTRY_MASK(_reg_, _field_, GENMASK(31, 0)) + +struct pm_status_row { + int reg_offset; + u32 field_mask; + const char *key; +}; + +int adf_pm_scnprint_table_upper_keys(char *buff, const struct pm_status_row *table, + u32 *pm_info_regs, size_t buff_size, int table_len); + +int adf_pm_scnprint_table_lower_keys(char *buff, const struct pm_status_row *table, + u32 *pm_info_regs, size_t buff_size, int table_len); + +#endif /* ADF_PM_DBGFS_UTILS_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_rl.c b/drivers/crypto/intel/qat/qat_common/adf_rl.c new file mode 100644 index 000000000..997ef448a --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_rl.c @@ -0,0 +1,1155 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ + +#define dev_fmt(fmt) "RateLimiting: " fmt + +#include +#include + +#include +#include +#include +#include +#include + +#include "adf_accel_devices.h" +#include "adf_cfg_services.h" +#include "adf_common_drv.h" +#include "adf_rl_admin.h" +#include "adf_rl.h" +#include "adf_sysfs_rl.h" + +#define RL_TOKEN_GRANULARITY_PCIEIN_BUCKET 0U +#define RL_TOKEN_GRANULARITY_PCIEOUT_BUCKET 0U +#define RL_TOKEN_PCIE_SIZE 64 +#define RL_TOKEN_ASYM_SIZE 1024 +#define RL_CSR_SIZE 4U +#define RL_CAPABILITY_MASK GENMASK(6, 4) +#define RL_CAPABILITY_VALUE 0x70 +#define RL_VALIDATE_NON_ZERO(input) ((input) == 0) +#define ROOT_MASK GENMASK(1, 0) +#define CLUSTER_MASK GENMASK(3, 0) +#define LEAF_MASK GENMASK(5, 0) + +static int validate_user_input(struct adf_accel_dev *accel_dev, + struct adf_rl_sla_input_data *sla_in, + bool is_update) +{ + const unsigned long rp_mask = sla_in->rp_mask; + size_t rp_mask_size; + int i, cnt; + + if (sla_in->pir < sla_in->cir) { + dev_notice(&GET_DEV(accel_dev), + "PIR must be >= CIR, setting PIR to CIR\n"); + sla_in->pir = sla_in->cir; + } + + if (!is_update) { + cnt = 0; + rp_mask_size = sizeof(sla_in->rp_mask) * BITS_PER_BYTE; + for_each_set_bit(i, &rp_mask, rp_mask_size) { + if (++cnt > RL_RP_CNT_PER_LEAF_MAX) { + dev_notice(&GET_DEV(accel_dev), + "Too many ring pairs selected for this SLA\n"); + return -EINVAL; + } + } + + if (sla_in->srv >= SVC_BASE_COUNT) { + dev_notice(&GET_DEV(accel_dev), + "Wrong service type\n"); + return -EINVAL; + } + + if (sla_in->type > RL_LEAF) { + dev_notice(&GET_DEV(accel_dev), + "Wrong node type\n"); + return -EINVAL; + } + + if (sla_in->parent_id < RL_PARENT_DEFAULT_ID || + sla_in->parent_id >= RL_NODES_CNT_MAX) { + dev_notice(&GET_DEV(accel_dev), + "Wrong parent ID\n"); + return -EINVAL; + } + } + + return 0; +} + +static int validate_sla_id(struct adf_accel_dev *accel_dev, int sla_id) +{ + struct rl_sla *sla; + + if (sla_id <= RL_SLA_EMPTY_ID || sla_id >= RL_NODES_CNT_MAX) { + dev_notice(&GET_DEV(accel_dev), "Provided ID is out of bounds\n"); + return -EINVAL; + } + + sla = accel_dev->rate_limiting->sla[sla_id]; + + if (!sla) { + dev_notice(&GET_DEV(accel_dev), "SLA with provided ID does not exist\n"); + return -EINVAL; + } + + if (sla->type != RL_LEAF) { + dev_notice(&GET_DEV(accel_dev), "This ID is reserved for internal use\n"); + return -EINVAL; + } + + return 0; +} + +/** + * find_parent() - Find the parent for a new SLA + * @rl_data: pointer to ratelimiting data + * @sla_in: pointer to user input data for a new SLA + * + * Function returns a pointer to the parent SLA. If the parent ID is provided + * as input in the user data, then such ID is validated and the parent SLA + * is returned. + * Otherwise, it returns the default parent SLA (root or cluster) for + * the new object. + * + * Return: + * * Pointer to the parent SLA object + * * NULL - when parent cannot be found + */ +static struct rl_sla *find_parent(struct adf_rl *rl_data, + struct adf_rl_sla_input_data *sla_in) +{ + int input_parent_id = sla_in->parent_id; + struct rl_sla *root = NULL; + struct rl_sla *parent_sla; + int i; + + if (sla_in->type == RL_ROOT) + return NULL; + + if (input_parent_id > RL_PARENT_DEFAULT_ID) { + parent_sla = rl_data->sla[input_parent_id]; + /* + * SLA can be a parent if it has the same service as the child + * and its type is higher in the hierarchy, + * for example the parent type of a LEAF must be a CLUSTER. + */ + if (parent_sla && parent_sla->srv == sla_in->srv && + parent_sla->type == sla_in->type - 1) + return parent_sla; + + return NULL; + } + + /* If input_parent_id is not valid, get root for this service type. */ + for (i = 0; i < RL_ROOT_MAX; i++) { + if (rl_data->root[i] && rl_data->root[i]->srv == sla_in->srv) { + root = rl_data->root[i]; + break; + } + } + + if (!root) + return NULL; + + /* + * If the type of this SLA is cluster, then return the root. + * Otherwise, find the default (i.e. first) cluster for this service. + */ + if (sla_in->type == RL_CLUSTER) + return root; + + for (i = 0; i < RL_CLUSTER_MAX; i++) { + if (rl_data->cluster[i] && rl_data->cluster[i]->parent == root) + return rl_data->cluster[i]; + } + + return NULL; +} + +/** + * adf_rl_get_sla_arr_of_type() - Returns a pointer to SLA type specific array + * @rl_data: pointer to ratelimiting data + * @type: SLA type + * @sla_arr: pointer to variable where requested pointer will be stored + * + * Return: Max number of elements allowed for the returned array + */ +u32 adf_rl_get_sla_arr_of_type(struct adf_rl *rl_data, enum rl_node_type type, + struct rl_sla ***sla_arr) +{ + switch (type) { + case RL_LEAF: + *sla_arr = rl_data->leaf; + return RL_LEAF_MAX; + case RL_CLUSTER: + *sla_arr = rl_data->cluster; + return RL_CLUSTER_MAX; + case RL_ROOT: + *sla_arr = rl_data->root; + return RL_ROOT_MAX; + default: + *sla_arr = NULL; + return 0; + } +} + +/** + * prepare_rp_ids() - Creates an array of ring pair IDs from bitmask + * @accel_dev: pointer to acceleration device structure + * @sla: SLA object data where result will be written + * @rp_mask: bitmask of ring pair IDs + * + * Function tries to convert provided bitmap to an array of IDs. It checks if + * RPs aren't in use, are assigned to SLA service or if a number of provided + * IDs is not too big. If successful, writes the result into the field + * sla->ring_pairs_cnt. + * + * Return: + * * 0 - ok + * * -EINVAL - ring pairs array cannot be created from provided mask + */ +static int prepare_rp_ids(struct adf_accel_dev *accel_dev, struct rl_sla *sla, + const unsigned long rp_mask) +{ + enum adf_cfg_service_type arb_srv = adf_srv_to_cfg_svc_type(sla->srv); + u16 rps_per_bundle = GET_HW_DATA(accel_dev)->num_banks_per_vf; + bool *rp_in_use = accel_dev->rate_limiting->rp_in_use; + size_t rp_cnt_max = ARRAY_SIZE(sla->ring_pairs_ids); + u16 rp_id_max = GET_HW_DATA(accel_dev)->num_banks; + u16 cnt = 0; + u16 rp_id; + + for_each_set_bit(rp_id, &rp_mask, rp_id_max) { + if (cnt >= rp_cnt_max) { + dev_notice(&GET_DEV(accel_dev), + "Assigned more ring pairs than supported"); + return -EINVAL; + } + + if (rp_in_use[rp_id]) { + dev_notice(&GET_DEV(accel_dev), + "RP %u already assigned to other SLA", rp_id); + return -EINVAL; + } + + if (GET_SRV_TYPE(accel_dev, rp_id % rps_per_bundle) != arb_srv) { + dev_notice(&GET_DEV(accel_dev), + "RP %u does not support SLA service", rp_id); + return -EINVAL; + } + + sla->ring_pairs_ids[cnt++] = rp_id; + } + + sla->ring_pairs_cnt = cnt; + + return 0; +} + +static void mark_rps_usage(struct rl_sla *sla, bool *rp_in_use, bool used) +{ + u16 rp_id; + int i; + + for (i = 0; i < sla->ring_pairs_cnt; i++) { + rp_id = sla->ring_pairs_ids[i]; + rp_in_use[rp_id] = used; + } +} + +static void assign_rps_to_leaf(struct adf_accel_dev *accel_dev, + struct rl_sla *sla, bool clear) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + u32 base_offset = hw_data->rl_data.r2l_offset; + u32 node_id = clear ? 0U : (sla->node_id & LEAF_MASK); + u32 offset; + int i; + + for (i = 0; i < sla->ring_pairs_cnt; i++) { + offset = base_offset + (RL_CSR_SIZE * sla->ring_pairs_ids[i]); + ADF_CSR_WR(pmisc_addr, offset, node_id); + } +} + +static void assign_leaf_to_cluster(struct adf_accel_dev *accel_dev, + struct rl_sla *sla, bool clear) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + u32 base_offset = hw_data->rl_data.l2c_offset; + u32 node_id = sla->node_id & LEAF_MASK; + u32 parent_id = clear ? 0U : (sla->parent->node_id & CLUSTER_MASK); + u32 offset; + + offset = base_offset + (RL_CSR_SIZE * node_id); + ADF_CSR_WR(pmisc_addr, offset, parent_id); +} + +static void assign_cluster_to_root(struct adf_accel_dev *accel_dev, + struct rl_sla *sla, bool clear) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + u32 base_offset = hw_data->rl_data.c2s_offset; + u32 node_id = sla->node_id & CLUSTER_MASK; + u32 parent_id = clear ? 0U : (sla->parent->node_id & ROOT_MASK); + u32 offset; + + offset = base_offset + (RL_CSR_SIZE * node_id); + ADF_CSR_WR(pmisc_addr, offset, parent_id); +} + +static void assign_node_to_parent(struct adf_accel_dev *accel_dev, + struct rl_sla *sla, bool clear_assignment) +{ + switch (sla->type) { + case RL_LEAF: + assign_rps_to_leaf(accel_dev, sla, clear_assignment); + assign_leaf_to_cluster(accel_dev, sla, clear_assignment); + break; + case RL_CLUSTER: + assign_cluster_to_root(accel_dev, sla, clear_assignment); + break; + default: + break; + } +} + +/** + * can_parent_afford_sla() - Verifies if parent allows to create an SLA + * @sla_in: pointer to user input data for a new SLA + * @sla_parent: pointer to parent SLA object + * @sla_cir: current child CIR value (only for update) + * @is_update: request is a update + * + * Algorithm verifies if parent has enough remaining budget to take assignment + * of a child with provided parameters. In update case current CIR value must be + * returned to budget first. + * PIR value cannot exceed the PIR assigned to parent. + * + * Return: + * * true - SLA can be created + * * false - SLA cannot be created + */ +static bool can_parent_afford_sla(struct adf_rl_sla_input_data *sla_in, + struct rl_sla *sla_parent, u32 sla_cir, + bool is_update) +{ + u32 rem_cir = sla_parent->rem_cir; + + if (is_update) + rem_cir += sla_cir; + + if (sla_in->cir > rem_cir || sla_in->pir > sla_parent->pir) + return false; + + return true; +} + +/** + * can_node_afford_update() - Verifies if SLA can be updated with input data + * @sla_in: pointer to user input data for a new SLA + * @sla: pointer to SLA object selected for update + * + * Algorithm verifies if a new CIR value is big enough to satisfy currently + * assigned child SLAs and if PIR can be updated + * + * Return: + * * true - SLA can be updated + * * false - SLA cannot be updated + */ +static bool can_node_afford_update(struct adf_rl_sla_input_data *sla_in, + struct rl_sla *sla) +{ + u32 cir_in_use = sla->cir - sla->rem_cir; + + /* new CIR cannot be smaller then currently consumed value */ + if (cir_in_use > sla_in->cir) + return false; + + /* PIR of root/cluster cannot be reduced in node with assigned children */ + if (sla_in->pir < sla->pir && sla->type != RL_LEAF && cir_in_use > 0) + return false; + + return true; +} + +static bool is_enough_budget(struct adf_rl *rl_data, struct rl_sla *sla, + struct adf_rl_sla_input_data *sla_in, + bool is_update) +{ + u32 max_val = rl_data->device_data->scale_ref; + struct rl_sla *parent = sla->parent; + bool ret = true; + + if (sla_in->cir > max_val || sla_in->pir > max_val) + ret = false; + + switch (sla->type) { + case RL_LEAF: + ret &= can_parent_afford_sla(sla_in, parent, sla->cir, + is_update); + break; + case RL_CLUSTER: + ret &= can_parent_afford_sla(sla_in, parent, sla->cir, + is_update); + + if (is_update) + ret &= can_node_afford_update(sla_in, sla); + + break; + case RL_ROOT: + if (is_update) + ret &= can_node_afford_update(sla_in, sla); + + break; + default: + ret = false; + break; + } + + return ret; +} + +static void update_budget(struct rl_sla *sla, u32 old_cir, bool is_update) +{ + switch (sla->type) { + case RL_LEAF: + if (is_update) + sla->parent->rem_cir += old_cir; + + sla->parent->rem_cir -= sla->cir; + sla->rem_cir = 0; + break; + case RL_CLUSTER: + if (is_update) { + sla->parent->rem_cir += old_cir; + sla->rem_cir = sla->cir - (old_cir - sla->rem_cir); + } else { + sla->rem_cir = sla->cir; + } + + sla->parent->rem_cir -= sla->cir; + break; + case RL_ROOT: + if (is_update) + sla->rem_cir = sla->cir - (old_cir - sla->rem_cir); + else + sla->rem_cir = sla->cir; + break; + default: + break; + } +} + +/** + * get_next_free_sla_id() - finds next free ID in the SLA array + * @rl_data: Pointer to ratelimiting data structure + * + * Return: + * * 0 : RL_NODES_CNT_MAX - correct ID + * * -ENOSPC - all SLA slots are in use + */ +static int get_next_free_sla_id(struct adf_rl *rl_data) +{ + int i = 0; + + while (i < RL_NODES_CNT_MAX && rl_data->sla[i++]) + ; + + if (i == RL_NODES_CNT_MAX) + return -ENOSPC; + + return i - 1; +} + +/** + * get_next_free_node_id() - finds next free ID in the array of that node type + * @rl_data: Pointer to ratelimiting data structure + * @sla: Pointer to SLA object for which the ID is searched + * + * Return: + * * 0 : RL_[NODE_TYPE]_MAX - correct ID + * * -ENOSPC - all slots of that type are in use + */ +static int get_next_free_node_id(struct adf_rl *rl_data, struct rl_sla *sla) +{ + struct adf_hw_device_data *hw_device = GET_HW_DATA(rl_data->accel_dev); + int max_id, i, step, rp_per_leaf; + struct rl_sla **sla_list; + + rp_per_leaf = hw_device->num_banks / hw_device->num_banks_per_vf; + + /* + * Static nodes mapping: + * root0 - cluster[0,4,8,12] - leaf[0-15] + * root1 - cluster[1,5,9,13] - leaf[16-31] + * root2 - cluster[2,6,10,14] - leaf[32-47] + */ + switch (sla->type) { + case RL_LEAF: + i = sla->srv * rp_per_leaf; + step = 1; + max_id = i + rp_per_leaf; + sla_list = rl_data->leaf; + break; + case RL_CLUSTER: + i = sla->srv; + step = 4; + max_id = RL_CLUSTER_MAX; + sla_list = rl_data->cluster; + break; + case RL_ROOT: + return sla->srv; + default: + return -EINVAL; + } + + while (i < max_id && sla_list[i]) + i += step; + + if (i >= max_id) + return -ENOSPC; + + return i; +} + +u32 adf_rl_calculate_slice_tokens(struct adf_accel_dev *accel_dev, u32 sla_val, + enum adf_base_services svc_type) +{ + struct adf_rl_hw_data *device_data = &accel_dev->hw_device->rl_data; + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + u64 avail_slice_cycles, allocated_tokens; + + if (!sla_val) + return 0; + + /* Handle generation specific slice count adjustment */ + avail_slice_cycles = hw_data->clock_frequency; + avail_slice_cycles *= hw_data->get_svc_slice_cnt(accel_dev, svc_type); + + do_div(avail_slice_cycles, device_data->scan_interval); + allocated_tokens = avail_slice_cycles * sla_val; + do_div(allocated_tokens, device_data->scale_ref); + + return allocated_tokens; +} + +static u32 adf_rl_get_num_svc_aes(struct adf_accel_dev *accel_dev, + enum adf_base_services svc) +{ + struct adf_rl_hw_data *device_data = &accel_dev->hw_device->rl_data; + + if (svc >= SVC_BASE_COUNT) + return 0; + + return device_data->svc_ae_mask[svc]; +} + +u32 adf_rl_calculate_ae_cycles(struct adf_accel_dev *accel_dev, u32 sla_val, + enum adf_base_services svc_type) +{ + struct adf_rl_hw_data *device_data = &accel_dev->hw_device->rl_data; + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + u64 allocated_ae_cycles, avail_ae_cycles; + + if (!sla_val) + return 0; + + avail_ae_cycles = hw_data->clock_frequency; + avail_ae_cycles *= adf_rl_get_num_svc_aes(accel_dev, svc_type); + do_div(avail_ae_cycles, device_data->scan_interval); + + sla_val *= device_data->max_tp[svc_type]; + sla_val /= device_data->scale_ref; + + allocated_ae_cycles = (sla_val * avail_ae_cycles); + do_div(allocated_ae_cycles, device_data->max_tp[svc_type]); + + return allocated_ae_cycles; +} + +u32 adf_rl_calculate_pci_bw(struct adf_accel_dev *accel_dev, u32 sla_val, + enum adf_base_services svc_type, bool is_bw_out) +{ + struct adf_rl_hw_data *device_data = &accel_dev->hw_device->rl_data; + u64 sla_to_bytes, allocated_bw, sla_scaled; + + if (!sla_val) + return 0; + + sla_to_bytes = sla_val; + sla_to_bytes *= device_data->max_tp[svc_type]; + do_div(sla_to_bytes, device_data->scale_ref); + + sla_to_bytes *= (svc_type == SVC_ASYM) ? RL_TOKEN_ASYM_SIZE : BYTES_PER_MBIT; + if (svc_type == SVC_DC && is_bw_out) + sla_to_bytes *= device_data->slices.dcpr_cnt - + device_data->dcpr_correction; + + sla_scaled = sla_to_bytes * device_data->pcie_scale_mul; + do_div(sla_scaled, device_data->pcie_scale_div); + allocated_bw = sla_scaled; + do_div(allocated_bw, RL_TOKEN_PCIE_SIZE); + do_div(allocated_bw, device_data->scan_interval); + + return allocated_bw; +} + +/** + * add_new_sla_entry() - creates a new SLA object and fills it with user data + * @accel_dev: pointer to acceleration device structure + * @sla_in: pointer to user input data for a new SLA + * @sla_out: Pointer to variable that will contain the address of a new + * SLA object if the operation succeeds + * + * Return: + * * 0 - ok + * * -ENOMEM - memory allocation failed + * * -EINVAL - invalid user input + * * -ENOSPC - all available SLAs are in use + */ +static int add_new_sla_entry(struct adf_accel_dev *accel_dev, + struct adf_rl_sla_input_data *sla_in, + struct rl_sla **sla_out) +{ + struct adf_rl *rl_data = accel_dev->rate_limiting; + struct rl_sla *sla; + int ret = 0; + + sla = kzalloc_obj(*sla); + if (!sla) { + ret = -ENOMEM; + goto ret_err; + } + *sla_out = sla; + + if (!adf_is_service_enabled(accel_dev, sla_in->srv)) { + dev_notice(&GET_DEV(accel_dev), + "Provided service is not enabled\n"); + ret = -EINVAL; + goto ret_err; + } + + sla->srv = sla_in->srv; + sla->type = sla_in->type; + ret = get_next_free_node_id(rl_data, sla); + if (ret < 0) { + dev_notice(&GET_DEV(accel_dev), + "Exceeded number of available nodes for that service\n"); + goto ret_err; + } + sla->node_id = ret; + + ret = get_next_free_sla_id(rl_data); + if (ret < 0) { + dev_notice(&GET_DEV(accel_dev), + "Allocated maximum SLAs number\n"); + goto ret_err; + } + sla->sla_id = ret; + + sla->parent = find_parent(rl_data, sla_in); + if (!sla->parent && sla->type != RL_ROOT) { + if (sla_in->parent_id != RL_PARENT_DEFAULT_ID) + dev_notice(&GET_DEV(accel_dev), + "Provided parent ID does not exist or cannot be parent for this SLA."); + else + dev_notice(&GET_DEV(accel_dev), + "Unable to find parent node for this service. Is service enabled?"); + ret = -EINVAL; + goto ret_err; + } + + if (sla->type == RL_LEAF) { + ret = prepare_rp_ids(accel_dev, sla, sla_in->rp_mask); + if (!sla->ring_pairs_cnt || ret) { + dev_notice(&GET_DEV(accel_dev), + "Unable to find ring pairs to assign to the leaf"); + if (!ret) + ret = -EINVAL; + + goto ret_err; + } + } + + return 0; + +ret_err: + kfree(sla); + *sla_out = NULL; + + return ret; +} + +static int initialize_default_nodes(struct adf_accel_dev *accel_dev) +{ + struct adf_rl *rl_data = accel_dev->rate_limiting; + struct adf_rl_hw_data *device_data = rl_data->device_data; + struct adf_rl_sla_input_data sla_in = { }; + int ret = 0; + int i; + + /* Init root for each enabled service */ + sla_in.type = RL_ROOT; + sla_in.parent_id = RL_PARENT_DEFAULT_ID; + + for (i = 0; i < SVC_BASE_COUNT; i++) { + if (!adf_is_service_enabled(accel_dev, i)) + continue; + + sla_in.cir = device_data->scale_ref; + sla_in.pir = sla_in.cir; + sla_in.srv = i; + + ret = adf_rl_add_sla(accel_dev, &sla_in); + if (ret) + return ret; + } + + /* Init default cluster for each root */ + sla_in.type = RL_CLUSTER; + for (i = 0; i < SVC_BASE_COUNT; i++) { + if (!rl_data->root[i]) + continue; + sla_in.cir = rl_data->root[i]->cir; + sla_in.pir = sla_in.cir; + sla_in.srv = rl_data->root[i]->srv; + + ret = adf_rl_add_sla(accel_dev, &sla_in); + if (ret) + return ret; + } + + return 0; +} + +static void clear_sla(struct adf_rl *rl_data, struct rl_sla *sla) +{ + bool *rp_in_use = rl_data->rp_in_use; + struct rl_sla **sla_type_arr = NULL; + int i, sla_id, node_id; + u32 old_cir; + + sla_id = sla->sla_id; + node_id = sla->node_id; + old_cir = sla->cir; + sla->cir = 0; + sla->pir = 0; + + for (i = 0; i < sla->ring_pairs_cnt; i++) + rp_in_use[sla->ring_pairs_ids[i]] = false; + + update_budget(sla, old_cir, true); + adf_rl_get_sla_arr_of_type(rl_data, sla->type, &sla_type_arr); + assign_node_to_parent(rl_data->accel_dev, sla, true); + adf_rl_send_admin_delete_msg(rl_data->accel_dev, node_id, sla->type); + mark_rps_usage(sla, rl_data->rp_in_use, false); + + kfree(sla); + rl_data->sla[sla_id] = NULL; + sla_type_arr[node_id] = NULL; +} + +static void free_all_sla(struct adf_accel_dev *accel_dev) +{ + struct adf_rl *rl_data = accel_dev->rate_limiting; + int sla_id; + + mutex_lock(&rl_data->rl_lock); + + for (sla_id = 0; sla_id < RL_NODES_CNT_MAX; sla_id++) { + if (!rl_data->sla[sla_id]) + continue; + + kfree(rl_data->sla[sla_id]); + rl_data->sla[sla_id] = NULL; + } + + mutex_unlock(&rl_data->rl_lock); +} + +/** + * add_update_sla() - handles the creation and the update of an SLA + * @accel_dev: pointer to acceleration device structure + * @sla_in: pointer to user input data for a new/updated SLA + * @is_update: flag to indicate if this is an update or an add operation + * + * Return: + * * 0 - ok + * * -ENOMEM - memory allocation failed + * * -EINVAL - user input data cannot be used to create SLA + * * -ENOSPC - all available SLAs are in use + */ +static int add_update_sla(struct adf_accel_dev *accel_dev, + struct adf_rl_sla_input_data *sla_in, bool is_update) +{ + struct adf_rl *rl_data = accel_dev->rate_limiting; + struct rl_sla **sla_type_arr = NULL; + struct rl_sla *sla = NULL; + u32 old_cir = 0; + int ret; + + if (!sla_in) { + dev_warn(&GET_DEV(accel_dev), + "SLA input data pointer is missing\n"); + return -EFAULT; + } + + mutex_lock(&rl_data->rl_lock); + + /* Input validation */ + ret = validate_user_input(accel_dev, sla_in, is_update); + if (ret) + goto ret_err; + + if (is_update) { + ret = validate_sla_id(accel_dev, sla_in->sla_id); + if (ret) + goto ret_err; + + sla = rl_data->sla[sla_in->sla_id]; + old_cir = sla->cir; + } else { + ret = add_new_sla_entry(accel_dev, sla_in, &sla); + if (ret) + goto ret_err; + } + + if (!is_enough_budget(rl_data, sla, sla_in, is_update)) { + dev_notice(&GET_DEV(accel_dev), + "Input value exceeds the remaining budget%s\n", + is_update ? " or more budget is already in use" : ""); + ret = -EINVAL; + goto ret_err; + } + sla->cir = sla_in->cir; + sla->pir = sla_in->pir; + + /* Apply SLA */ + assign_node_to_parent(accel_dev, sla, false); + ret = adf_rl_send_admin_add_update_msg(accel_dev, sla, is_update); + if (ret) { + dev_notice(&GET_DEV(accel_dev), + "Failed to apply an SLA\n"); + goto ret_err; + } + update_budget(sla, old_cir, is_update); + + if (!is_update) { + mark_rps_usage(sla, rl_data->rp_in_use, true); + adf_rl_get_sla_arr_of_type(rl_data, sla->type, &sla_type_arr); + sla_type_arr[sla->node_id] = sla; + rl_data->sla[sla->sla_id] = sla; + } + + sla_in->sla_id = sla->sla_id; + goto ret_ok; + +ret_err: + if (!is_update) { + sla_in->sla_id = -1; + kfree(sla); + } +ret_ok: + mutex_unlock(&rl_data->rl_lock); + return ret; +} + +/** + * adf_rl_add_sla() - handles the creation of an SLA + * @accel_dev: pointer to acceleration device structure + * @sla_in: pointer to user input data required to add an SLA + * + * Return: + * * 0 - ok + * * -ENOMEM - memory allocation failed + * * -EINVAL - invalid user input + * * -ENOSPC - all available SLAs are in use + */ +int adf_rl_add_sla(struct adf_accel_dev *accel_dev, + struct adf_rl_sla_input_data *sla_in) +{ + return add_update_sla(accel_dev, sla_in, false); +} + +/** + * adf_rl_update_sla() - handles the update of an SLA + * @accel_dev: pointer to acceleration device structure + * @sla_in: pointer to user input data required to update an SLA + * + * Return: + * * 0 - ok + * * -EINVAL - user input data cannot be used to update SLA + */ +int adf_rl_update_sla(struct adf_accel_dev *accel_dev, + struct adf_rl_sla_input_data *sla_in) +{ + return add_update_sla(accel_dev, sla_in, true); +} + +/** + * adf_rl_get_sla() - returns an existing SLA data + * @accel_dev: pointer to acceleration device structure + * @sla_in: pointer to user data where SLA info will be stored + * + * The sla_id for which data are requested should be set in sla_id structure + * + * Return: + * * 0 - ok + * * -EINVAL - provided sla_id does not exist + */ +int adf_rl_get_sla(struct adf_accel_dev *accel_dev, + struct adf_rl_sla_input_data *sla_in) +{ + struct rl_sla *sla; + int ret, i; + + ret = validate_sla_id(accel_dev, sla_in->sla_id); + if (ret) + return ret; + + sla = accel_dev->rate_limiting->sla[sla_in->sla_id]; + sla_in->type = sla->type; + sla_in->srv = sla->srv; + sla_in->cir = sla->cir; + sla_in->pir = sla->pir; + sla_in->rp_mask = 0U; + if (sla->parent) + sla_in->parent_id = sla->parent->sla_id; + else + sla_in->parent_id = RL_PARENT_DEFAULT_ID; + + for (i = 0; i < sla->ring_pairs_cnt; i++) + sla_in->rp_mask |= BIT(sla->ring_pairs_ids[i]); + + return 0; +} + +/** + * adf_rl_get_capability_remaining() - returns the remaining SLA value (CIR) for + * selected service or provided sla_id + * @accel_dev: pointer to acceleration device structure + * @srv: service ID for which capability is requested + * @sla_id: ID of the cluster or root to which we want assign a new SLA + * + * Check if the provided SLA id is valid. If it is and the service matches + * the requested service and the type is cluster or root, return the remaining + * capability. + * If the provided ID does not match the service or type, return the remaining + * capacity of the default cluster for that service. + * + * Return: + * * Positive value - correct remaining value + * * -EINVAL - algorithm cannot find a remaining value for provided data + */ +int adf_rl_get_capability_remaining(struct adf_accel_dev *accel_dev, + enum adf_base_services srv, int sla_id) +{ + struct adf_rl *rl_data = accel_dev->rate_limiting; + struct rl_sla *sla = NULL; + int i; + + if (srv >= SVC_BASE_COUNT) + return -EINVAL; + + if (sla_id > RL_SLA_EMPTY_ID && !validate_sla_id(accel_dev, sla_id)) { + sla = rl_data->sla[sla_id]; + + if (sla->srv == srv && sla->type <= RL_CLUSTER) + goto ret_ok; + } + + for (i = 0; i < RL_CLUSTER_MAX; i++) { + if (!rl_data->cluster[i]) + continue; + + if (rl_data->cluster[i]->srv == srv) { + sla = rl_data->cluster[i]; + goto ret_ok; + } + } + + return -EINVAL; +ret_ok: + return sla->rem_cir; +} + +/** + * adf_rl_remove_sla() - removes provided sla_id + * @accel_dev: pointer to acceleration device structure + * @sla_id: ID of the cluster or root to which we want assign an new SLA + * + * Return: + * * 0 - ok + * * -EINVAL - wrong sla_id or it still have assigned children + */ +int adf_rl_remove_sla(struct adf_accel_dev *accel_dev, u32 sla_id) +{ + struct adf_rl *rl_data = accel_dev->rate_limiting; + struct rl_sla *sla; + int ret = 0; + + mutex_lock(&rl_data->rl_lock); + ret = validate_sla_id(accel_dev, sla_id); + if (ret) + goto err_ret; + + sla = rl_data->sla[sla_id]; + + if (sla->type < RL_LEAF && sla->rem_cir != sla->cir) { + dev_notice(&GET_DEV(accel_dev), + "To remove parent SLA all its children must be removed first"); + ret = -EINVAL; + goto err_ret; + } + + clear_sla(rl_data, sla); + +err_ret: + mutex_unlock(&rl_data->rl_lock); + return ret; +} + +/** + * adf_rl_remove_sla_all() - removes all SLAs from device + * @accel_dev: pointer to acceleration device structure + * @incl_default: set to true if default SLAs also should be removed + */ +void adf_rl_remove_sla_all(struct adf_accel_dev *accel_dev, bool incl_default) +{ + struct adf_rl *rl_data = accel_dev->rate_limiting; + int end_type = incl_default ? RL_ROOT : RL_LEAF; + struct rl_sla **sla_type_arr = NULL; + u32 max_id; + int i, j; + + mutex_lock(&rl_data->rl_lock); + + /* Unregister and remove all SLAs */ + for (j = RL_LEAF; j >= end_type; j--) { + max_id = adf_rl_get_sla_arr_of_type(rl_data, j, &sla_type_arr); + + for (i = 0; i < max_id; i++) { + if (!sla_type_arr[i]) + continue; + + clear_sla(rl_data, sla_type_arr[i]); + } + } + + mutex_unlock(&rl_data->rl_lock); +} + +int adf_rl_init(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + struct adf_rl_hw_data *rl_hw_data = &hw_data->rl_data; + struct adf_rl *rl; + int ret = 0; + + /* Validate device parameters */ + if (RL_VALIDATE_NON_ZERO(rl_hw_data->max_tp[SVC_ASYM]) || + RL_VALIDATE_NON_ZERO(rl_hw_data->max_tp[SVC_SYM]) || + RL_VALIDATE_NON_ZERO(rl_hw_data->max_tp[SVC_DC]) || + RL_VALIDATE_NON_ZERO(rl_hw_data->scan_interval) || + RL_VALIDATE_NON_ZERO(rl_hw_data->pcie_scale_div) || + RL_VALIDATE_NON_ZERO(rl_hw_data->pcie_scale_mul) || + RL_VALIDATE_NON_ZERO(rl_hw_data->scale_ref)) { + ret = -EOPNOTSUPP; + goto err_ret; + } + + rl = kzalloc_obj(*rl); + if (!rl) { + ret = -ENOMEM; + goto err_ret; + } + + mutex_init(&rl->rl_lock); + rl->device_data = &accel_dev->hw_device->rl_data; + rl->accel_dev = accel_dev; + init_rwsem(&rl->user_input.lock); + accel_dev->rate_limiting = rl; + +err_ret: + return ret; +} + +int adf_rl_start(struct adf_accel_dev *accel_dev) +{ + struct adf_rl_hw_data *rl_hw_data = &GET_HW_DATA(accel_dev)->rl_data; + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + u16 fw_caps = GET_HW_DATA(accel_dev)->fw_capabilities; + int ret; + + if (!accel_dev->rate_limiting) { + ret = -EOPNOTSUPP; + goto ret_err; + } + + if ((fw_caps & RL_CAPABILITY_MASK) != RL_CAPABILITY_VALUE) { + dev_info(&GET_DEV(accel_dev), "feature not supported by FW\n"); + ret = -EOPNOTSUPP; + goto ret_free; + } + + ADF_CSR_WR(pmisc_addr, rl_hw_data->pciin_tb_offset, + RL_TOKEN_GRANULARITY_PCIEIN_BUCKET); + ADF_CSR_WR(pmisc_addr, rl_hw_data->pciout_tb_offset, + RL_TOKEN_GRANULARITY_PCIEOUT_BUCKET); + + ret = adf_rl_send_admin_init_msg(accel_dev, &rl_hw_data->slices); + if (ret) { + dev_err(&GET_DEV(accel_dev), "initialization failed\n"); + goto ret_free; + } + + ret = initialize_default_nodes(accel_dev); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "failed to initialize default SLAs\n"); + goto ret_sla_rm; + } + + ret = adf_sysfs_rl_add(accel_dev); + if (ret) { + dev_err(&GET_DEV(accel_dev), "failed to add sysfs interface\n"); + goto ret_sysfs_rm; + } + + return 0; + +ret_sysfs_rm: + adf_sysfs_rl_rm(accel_dev); +ret_sla_rm: + adf_rl_remove_sla_all(accel_dev, true); +ret_free: + kfree(accel_dev->rate_limiting); + accel_dev->rate_limiting = NULL; +ret_err: + return ret; +} + +void adf_rl_stop(struct adf_accel_dev *accel_dev) +{ + if (!accel_dev->rate_limiting) + return; + + adf_sysfs_rl_rm(accel_dev); + free_all_sla(accel_dev); +} + +void adf_rl_exit(struct adf_accel_dev *accel_dev) +{ + if (!accel_dev->rate_limiting) + return; + + kfree(accel_dev->rate_limiting); + accel_dev->rate_limiting = NULL; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_rl.h b/drivers/crypto/intel/qat/qat_common/adf_rl.h new file mode 100644 index 000000000..c1f3f9a51 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_rl.h @@ -0,0 +1,176 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ + +#ifndef ADF_RL_H_ +#define ADF_RL_H_ + +#include +#include + +#include "adf_cfg_services.h" + +struct adf_accel_dev; + +#define RL_ROOT_MAX 4 +#define RL_CLUSTER_MAX 16 +#define RL_LEAF_MAX 64 +#define RL_NODES_CNT_MAX (RL_ROOT_MAX + RL_CLUSTER_MAX + RL_LEAF_MAX) +#define RL_RP_CNT_PER_LEAF_MAX 4U +#define RL_RP_CNT_MAX 64 +#define RL_SLA_EMPTY_ID -1 +#define RL_PARENT_DEFAULT_ID -1 + +enum rl_node_type { + RL_ROOT, + RL_CLUSTER, + RL_LEAF, +}; + +/** + * struct adf_rl_sla_input_data - ratelimiting user input data structure + * @rp_mask: 64 bit bitmask of ring pair IDs which will be assigned to SLA. + * Eg. 0x5 -> RP0 and RP2 assigned; 0xA005 -> RP0,2,13,15 assigned. + * @sla_id: ID of current SLA for operations update, rm, get. For the add + * operation, this field will be updated with the ID of the newly + * added SLA + * @parent_id: ID of the SLA to which the current one should be assigned. + * Set to -1 to refer to the default parent. + * @cir: Committed information rate. Rate guaranteed to be achieved. Input value + * is expressed in permille scale, i.e. 1000 refers to the maximum + * device throughput for a selected service. + * @pir: Peak information rate. Maximum rate available that the SLA can achieve. + * Input value is expressed in permille scale, i.e. 1000 refers to + * the maximum device throughput for a selected service. + * @type: SLA type: root, cluster, node + * @srv: Service associated to the SLA: asym, sym dc. + * + * This structure is used to perform operations on an SLA. + * Depending on the operation, some of the parameters are ignored. + * The following list reports which parameters should be set for each operation. + * - add: all except sla_id + * - update: cir, pir, sla_id + * - rm: sla_id + * - rm_all: - + * - get: sla_id + * - get_capability_rem: srv, sla_id + */ +struct adf_rl_sla_input_data { + u64 rp_mask; + int sla_id; + int parent_id; + unsigned int cir; + unsigned int pir; + enum rl_node_type type; + enum adf_base_services srv; +}; + +struct rl_slice_cnt { + u8 dcpr_cnt; + u8 pke_cnt; + u8 cph_cnt; + u8 cpr_cnt; +}; + +struct adf_rl_interface_data { + struct adf_rl_sla_input_data input; + enum adf_base_services cap_rem_srv; + struct rw_semaphore lock; + bool sysfs_added; +}; + +struct adf_rl_hw_data { + u32 scale_ref; + u32 scan_interval; + u32 r2l_offset; + u32 l2c_offset; + u32 c2s_offset; + u32 pciin_tb_offset; + u32 pciout_tb_offset; + u32 pcie_scale_mul; + u32 pcie_scale_div; + u32 dcpr_correction; + u32 max_tp[RL_ROOT_MAX]; + u32 svc_ae_mask[SVC_BASE_COUNT]; + struct rl_slice_cnt slices; +}; + +/** + * struct adf_rl - ratelimiting data structure + * @accel_dev: pointer to acceleration device data + * @device_data: pointer to rate limiting data specific to a device type (or revision) + * @sla: array of pointers to SLA objects + * @root: array of pointers to root type SLAs, element number reflects node_id + * @cluster: array of pointers to cluster type SLAs, element number reflects node_id + * @leaf: array of pointers to leaf type SLAs, element number reflects node_id + * @rp_in_use: array of ring pair IDs already used in one of SLAs + * @rl_lock: mutex object which is protecting data in this structure + * @input: structure which is used for holding the data received from user + */ +struct adf_rl { + struct adf_accel_dev *accel_dev; + struct adf_rl_hw_data *device_data; + /* mapping sla_id to SLA objects */ + struct rl_sla *sla[RL_NODES_CNT_MAX]; + struct rl_sla *root[RL_ROOT_MAX]; + struct rl_sla *cluster[RL_CLUSTER_MAX]; + struct rl_sla *leaf[RL_LEAF_MAX]; + bool rp_in_use[RL_RP_CNT_MAX]; + /* Mutex protecting writing to SLAs lists */ + struct mutex rl_lock; + struct adf_rl_interface_data user_input; +}; + +/** + * struct rl_sla - SLA object data structure + * @parent: pointer to the parent SLA (root/cluster) + * @type: SLA type + * @srv: service associated with this SLA + * @sla_id: ID of the SLA, used as element number in SLA array and as identifier + * shared with the user + * @node_id: ID of node, each of SLA type have a separate ID list + * @cir: committed information rate + * @pir: peak information rate (PIR >= CIR) + * @rem_cir: if this SLA is a parent then this field represents a remaining + * value to be used by child SLAs. + * @ring_pairs_ids: array with numeric ring pairs IDs assigned to this SLA + * @ring_pairs_cnt: number of assigned ring pairs listed in the array above + */ +struct rl_sla { + struct rl_sla *parent; + enum rl_node_type type; + enum adf_base_services srv; + u32 sla_id; + u32 node_id; + u32 cir; + u32 pir; + u32 rem_cir; + u16 ring_pairs_ids[RL_RP_CNT_PER_LEAF_MAX]; + u16 ring_pairs_cnt; +}; + +u32 adf_rl_get_sla_arr_of_type(struct adf_rl *rl_data, enum rl_node_type type, + struct rl_sla ***sla_arr); +int adf_rl_add_sla(struct adf_accel_dev *accel_dev, + struct adf_rl_sla_input_data *sla_in); +int adf_rl_update_sla(struct adf_accel_dev *accel_dev, + struct adf_rl_sla_input_data *sla_in); +int adf_rl_get_sla(struct adf_accel_dev *accel_dev, + struct adf_rl_sla_input_data *sla_in); +int adf_rl_get_capability_remaining(struct adf_accel_dev *accel_dev, + enum adf_base_services srv, int sla_id); +int adf_rl_remove_sla(struct adf_accel_dev *accel_dev, u32 sla_id); +void adf_rl_remove_sla_all(struct adf_accel_dev *accel_dev, bool incl_default); + +int adf_rl_init(struct adf_accel_dev *accel_dev); +int adf_rl_start(struct adf_accel_dev *accel_dev); +void adf_rl_stop(struct adf_accel_dev *accel_dev); +void adf_rl_exit(struct adf_accel_dev *accel_dev); + +u32 adf_rl_calculate_pci_bw(struct adf_accel_dev *accel_dev, u32 sla_val, + enum adf_base_services svc_type, bool is_bw_out); +u32 adf_rl_calculate_ae_cycles(struct adf_accel_dev *accel_dev, u32 sla_val, + enum adf_base_services svc_type); +u32 adf_rl_calculate_slice_tokens(struct adf_accel_dev *accel_dev, u32 sla_val, + enum adf_base_services svc_type); + +#endif /* ADF_RL_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_rl_admin.c b/drivers/crypto/intel/qat/qat_common/adf_rl_admin.c new file mode 100644 index 000000000..4a3e0591f --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_rl_admin.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ + +#include +#include + +#include "adf_admin.h" +#include "adf_accel_devices.h" +#include "adf_rl_admin.h" + +static void +prep_admin_req_msg(struct rl_sla *sla, dma_addr_t dma_addr, + struct icp_qat_fw_init_admin_sla_config_params *fw_params, + struct icp_qat_fw_init_admin_req *req, bool is_update) +{ + req->cmd_id = is_update ? ICP_QAT_FW_RL_UPDATE : ICP_QAT_FW_RL_ADD; + req->init_cfg_ptr = dma_addr; + req->init_cfg_sz = sizeof(*fw_params); + req->node_id = sla->node_id; + req->node_type = sla->type; + req->rp_count = sla->ring_pairs_cnt; + req->svc_type = sla->srv; +} + +static void +prep_admin_req_params(struct adf_accel_dev *accel_dev, struct rl_sla *sla, + struct icp_qat_fw_init_admin_sla_config_params *fw_params) +{ + fw_params->pcie_in_cir = + adf_rl_calculate_pci_bw(accel_dev, sla->cir, sla->srv, false); + fw_params->pcie_in_pir = + adf_rl_calculate_pci_bw(accel_dev, sla->pir, sla->srv, false); + fw_params->pcie_out_cir = + adf_rl_calculate_pci_bw(accel_dev, sla->cir, sla->srv, true); + fw_params->pcie_out_pir = + adf_rl_calculate_pci_bw(accel_dev, sla->pir, sla->srv, true); + + fw_params->slice_util_cir = + adf_rl_calculate_slice_tokens(accel_dev, sla->cir, sla->srv); + fw_params->slice_util_pir = + adf_rl_calculate_slice_tokens(accel_dev, sla->pir, sla->srv); + + fw_params->ae_util_cir = + adf_rl_calculate_ae_cycles(accel_dev, sla->cir, sla->srv); + fw_params->ae_util_pir = + adf_rl_calculate_ae_cycles(accel_dev, sla->pir, sla->srv); + + memcpy(fw_params->rp_ids, sla->ring_pairs_ids, + sizeof(sla->ring_pairs_ids)); +} + +int adf_rl_send_admin_init_msg(struct adf_accel_dev *accel_dev, + struct rl_slice_cnt *slices_int) +{ + struct icp_qat_fw_init_admin_slice_cnt slices_resp = { }; + int ret; + + ret = adf_send_admin_rl_init(accel_dev, &slices_resp); + if (ret) + return ret; + + slices_int->dcpr_cnt = slices_resp.dcpr_cnt; + slices_int->pke_cnt = slices_resp.pke_cnt; + /* For symmetric crypto, slice tokens are relative to the UCS slice */ + slices_int->cph_cnt = slices_resp.ucs_cnt; + slices_int->cpr_cnt = slices_resp.cpr_cnt; + + return 0; +} + +int adf_rl_send_admin_add_update_msg(struct adf_accel_dev *accel_dev, + struct rl_sla *sla, bool is_update) +{ + struct icp_qat_fw_init_admin_sla_config_params *fw_params; + struct icp_qat_fw_init_admin_req req = { }; + dma_addr_t dma_addr; + int ret; + + fw_params = dma_alloc_coherent(&GET_DEV(accel_dev), sizeof(*fw_params), + &dma_addr, GFP_KERNEL); + if (!fw_params) + return -ENOMEM; + + prep_admin_req_params(accel_dev, sla, fw_params); + prep_admin_req_msg(sla, dma_addr, fw_params, &req, is_update); + ret = adf_send_admin_rl_add_update(accel_dev, &req); + + dma_free_coherent(&GET_DEV(accel_dev), sizeof(*fw_params), fw_params, + dma_addr); + + return ret; +} + +int adf_rl_send_admin_delete_msg(struct adf_accel_dev *accel_dev, u16 node_id, + u8 node_type) +{ + return adf_send_admin_rl_delete(accel_dev, node_id, node_type); +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_rl_admin.h b/drivers/crypto/intel/qat/qat_common/adf_rl_admin.h new file mode 100644 index 000000000..dd5419b7e --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_rl_admin.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ + +#ifndef ADF_RL_ADMIN_H_ +#define ADF_RL_ADMIN_H_ + +#include + +#include "adf_rl.h" + +int adf_rl_send_admin_init_msg(struct adf_accel_dev *accel_dev, + struct rl_slice_cnt *slices_int); +int adf_rl_send_admin_add_update_msg(struct adf_accel_dev *accel_dev, + struct rl_sla *sla, bool is_update); +int adf_rl_send_admin_delete_msg(struct adf_accel_dev *accel_dev, u16 node_id, + u8 node_type); + +#endif /* ADF_RL_ADMIN_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_sriov.c b/drivers/crypto/intel/qat/qat_common/adf_sriov.c new file mode 100644 index 000000000..f45ca2eec --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_sriov.c @@ -0,0 +1,339 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2015 - 2021 Intel Corporation */ +#include +#include +#include +#include "adf_common_drv.h" +#include "adf_cfg.h" +#include "adf_pfvf_pf_msg.h" + +#define ADF_VF2PF_RATELIMIT_INTERVAL 8 +#define ADF_VF2PF_RATELIMIT_BURST 130 + +static struct workqueue_struct *pf2vf_resp_wq; + +struct adf_pf2vf_resp { + struct work_struct pf2vf_resp_work; + struct adf_accel_vf_info *vf_info; +}; + +static void adf_iov_send_resp(struct work_struct *work) +{ + struct adf_pf2vf_resp *pf2vf_resp = + container_of(work, struct adf_pf2vf_resp, pf2vf_resp_work); + struct adf_accel_vf_info *vf_info = pf2vf_resp->vf_info; + struct adf_accel_dev *accel_dev = vf_info->accel_dev; + u32 vf_nr = vf_info->vf_nr; + bool ret; + + if (READ_ONCE(accel_dev->pf.vf2pf_disabled)) + goto out; + + mutex_lock(&vf_info->pfvf_mig_lock); + ret = adf_recv_and_handle_vf2pf_msg(accel_dev, vf_nr); + if (ret) + /* re-enable interrupt on PF from this VF */ + adf_enable_vf2pf_interrupts(accel_dev, 1 << vf_nr); + mutex_unlock(&vf_info->pfvf_mig_lock); + +out: + kfree(pf2vf_resp); +} + +void adf_schedule_vf2pf_handler(struct adf_accel_vf_info *vf_info) +{ + struct adf_accel_dev *accel_dev = vf_info->accel_dev; + struct adf_pf2vf_resp *pf2vf_resp; + + if (READ_ONCE(accel_dev->pf.vf2pf_disabled)) + return; + + pf2vf_resp = kzalloc_obj(*pf2vf_resp, GFP_ATOMIC); + if (!pf2vf_resp) + return; + + pf2vf_resp->vf_info = vf_info; + INIT_WORK(&pf2vf_resp->pf2vf_resp_work, adf_iov_send_resp); + queue_work(pf2vf_resp_wq, &pf2vf_resp->pf2vf_resp_work); +} + +static void adf_flush_pf2vf_resp_wq(void) +{ + if (pf2vf_resp_wq) + flush_workqueue(pf2vf_resp_wq); +} + +static int adf_enable_sriov(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + int totalvfs = pci_sriov_get_totalvfs(pdev); + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_accel_vf_info *vf_info; + int i; + + for (i = 0, vf_info = accel_dev->pf.vf_info; i < totalvfs; + i++, vf_info++) { + /* This ptr will be populated when VFs will be created */ + vf_info->accel_dev = accel_dev; + vf_info->vf_nr = i; + + mutex_init(&vf_info->pf2vf_lock); + mutex_init(&vf_info->pfvf_mig_lock); + ratelimit_state_init(&vf_info->vf2pf_ratelimit, + ADF_VF2PF_RATELIMIT_INTERVAL, + ADF_VF2PF_RATELIMIT_BURST); + } + + /* Set Valid bits in AE Thread to PCIe Function Mapping */ + if (hw_data->configure_iov_threads) + hw_data->configure_iov_threads(accel_dev, true); + + /* Enable VF to PF interrupts for all VFs */ + adf_enable_all_vf2pf_interrupts(accel_dev, totalvfs); + + /* Do not enable SR-IOV if already enabled */ + if (pci_num_vf(pdev)) + return 0; + + /* + * Due to the hardware design, when SR-IOV and the ring arbiter + * are enabled all the VFs supported in hardware must be enabled in + * order for all the hardware resources (i.e. bundles) to be usable. + * When SR-IOV is enabled, each of the VFs will own one bundle. + */ + return pci_enable_sriov(pdev, totalvfs); +} + +static int adf_add_sriov_configuration(struct adf_accel_dev *accel_dev) +{ + unsigned long val = 0; + int ret; + + ret = adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC); + if (ret) + return ret; + + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY, + &val, ADF_DEC); + if (ret) + return ret; + + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_DC, + &val, ADF_DEC); + if (ret) + return ret; + + set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); + + return ret; +} + +static int adf_do_disable_sriov(struct adf_accel_dev *accel_dev) +{ + int ret; + + if (adf_dev_in_use(accel_dev)) { + dev_err(&GET_DEV(accel_dev), + "Cannot disable SR-IOV, device in use\n"); + return -EBUSY; + } + + if (adf_dev_started(accel_dev)) { + if (adf_devmgr_in_reset(accel_dev)) { + dev_err(&GET_DEV(accel_dev), + "Cannot disable SR-IOV, device in reset\n"); + return -EBUSY; + } + + ret = adf_dev_down(accel_dev); + if (ret) + goto err_del_cfg; + } + + adf_disable_sriov(accel_dev); + + ret = adf_dev_up(accel_dev, true); + if (ret) + goto err_del_cfg; + + return 0; + +err_del_cfg: + adf_cfg_del_all_except(accel_dev, ADF_GENERAL_SEC); + return ret; +} + +static int adf_do_enable_sriov(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + int totalvfs = pci_sriov_get_totalvfs(pdev); + unsigned long val; + int ret; + + if (!device_iommu_mapped(&GET_DEV(accel_dev))) { + dev_warn(&GET_DEV(accel_dev), + "IOMMU should be enabled for SR-IOV to work correctly\n"); + } + + if (adf_dev_started(accel_dev)) { + if (adf_devmgr_in_reset(accel_dev) || adf_dev_in_use(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Device busy\n"); + return -EBUSY; + } + + ret = adf_dev_down(accel_dev); + if (ret) + return ret; + } + + ret = adf_add_sriov_configuration(accel_dev); + if (ret) + goto err_del_cfg; + + /* Allocate memory for VF info structs */ + accel_dev->pf.vf_info = kzalloc_objs(struct adf_accel_vf_info, totalvfs); + ret = -ENOMEM; + if (!accel_dev->pf.vf_info) + goto err_del_cfg; + + ret = adf_dev_up(accel_dev, false); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Failed to start qat_dev%d\n", + accel_dev->accel_id); + goto err_free_vf_info; + } + + ret = adf_enable_sriov(accel_dev); + if (ret) + goto err_free_vf_info; + + val = 1; + ret = adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC, ADF_SRIOV_ENABLED, + &val, ADF_DEC); + if (ret) + goto err_free_vf_info; + + return totalvfs; + +err_free_vf_info: + adf_dev_down(accel_dev); + kfree(accel_dev->pf.vf_info); + accel_dev->pf.vf_info = NULL; + return ret; +err_del_cfg: + adf_cfg_del_all_except(accel_dev, ADF_GENERAL_SEC); + return ret; +} + +void adf_reenable_sriov(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + char cfg[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0}; + + if (adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, + ADF_SRIOV_ENABLED, cfg)) + return; + + if (!accel_dev->pf.vf_info) + return; + + if (adf_add_sriov_configuration(accel_dev)) + return; + + pci_dbg(pdev, "Re-enabling SRIOV\n"); + adf_enable_sriov(accel_dev); +} + +/** + * adf_disable_sriov() - Disable SRIOV for the device + * @accel_dev: Pointer to accel device. + * + * Function disables SRIOV for the accel device. + * + * Return: 0 on success, error code otherwise. + */ +void adf_disable_sriov(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + int totalvfs = pci_sriov_get_totalvfs(accel_to_pci_dev(accel_dev)); + struct adf_accel_vf_info *vf; + int i; + + if (!accel_dev->pf.vf_info) + return; + + adf_pf2vf_notify_restarting(accel_dev); + adf_pf2vf_wait_for_restarting_complete(accel_dev); + /* + * When the device is restarting, preserve VF PCI devices across + * the reset by skipping pci_disable_sriov(). VFs are notified to + * quiesce regardless so the PF can safely shut down. + */ + if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) + pci_disable_sriov(accel_to_pci_dev(accel_dev)); + + /* Block VF2PF work and disable VF to PF interrupts */ + adf_disable_all_vf2pf_interrupts(accel_dev); + adf_isr_sync_ae_cluster(accel_dev); + adf_flush_pf2vf_resp_wq(); + + /* Clear Valid bits in AE Thread to PCIe Function Mapping */ + if (hw_data->configure_iov_threads) + hw_data->configure_iov_threads(accel_dev, false); + + for (i = 0, vf = accel_dev->pf.vf_info; i < totalvfs; i++, vf++) { + mutex_destroy(&vf->pf2vf_lock); + mutex_destroy(&vf->pfvf_mig_lock); + } + + if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) { + kfree(accel_dev->pf.vf_info); + accel_dev->pf.vf_info = NULL; + } +} +EXPORT_SYMBOL_GPL(adf_disable_sriov); + +/** + * adf_sriov_configure() - Enable SRIOV for the device + * @pdev: Pointer to PCI device. + * @numvfs: Number of virtual functions (VFs) to enable. + * + * Note that the @numvfs parameter is ignored and all VFs supported by the + * device are enabled due to the design of the hardware. + * + * Function enables SRIOV for the PCI device. + * + * Return: number of VFs enabled on success, error code otherwise. + */ +int adf_sriov_configure(struct pci_dev *pdev, int numvfs) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + if (!accel_dev) { + dev_err(&pdev->dev, "Failed to find accel_dev\n"); + return -EFAULT; + } + + if (numvfs) + return adf_do_enable_sriov(accel_dev); + else + return adf_do_disable_sriov(accel_dev); +} +EXPORT_SYMBOL_GPL(adf_sriov_configure); + +int __init adf_init_pf_wq(void) +{ + /* Workqueue for PF2VF responses */ + pf2vf_resp_wq = alloc_workqueue("qat_pf2vf_resp_wq", + WQ_MEM_RECLAIM | WQ_PERCPU, 0); + + return !pf2vf_resp_wq ? -ENOMEM : 0; +} + +void adf_exit_pf_wq(void) +{ + if (pf2vf_resp_wq) { + destroy_workqueue(pf2vf_resp_wq); + pf2vf_resp_wq = NULL; + } +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_sysfs.c b/drivers/crypto/intel/qat/qat_common/adf_sysfs.c new file mode 100644 index 000000000..8daa69a76 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_sysfs.c @@ -0,0 +1,353 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2022 Intel Corporation */ +#include +#include +#include +#include +#include "adf_accel_devices.h" +#include "adf_cfg.h" +#include "adf_cfg_services.h" +#include "adf_common_drv.h" + +#define UNSET_RING_NUM -1 + +static const char * const state_operations[] = { + [DEV_DOWN] = "down", + [DEV_UP] = "up", +}; + +static ssize_t state_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct adf_accel_dev *accel_dev; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + return sysfs_emit(buf, "%s\n", str_up_down(adf_dev_started(accel_dev))); +} + +static ssize_t state_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct adf_accel_dev *accel_dev; + u32 accel_id; + int ret; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + accel_id = accel_dev->accel_id; + + if (adf_devmgr_in_reset(accel_dev) || adf_dev_in_use(accel_dev)) { + dev_info(dev, "Device qat_dev%d is busy\n", accel_id); + return -EBUSY; + } + + ret = sysfs_match_string(state_operations, buf); + if (ret < 0) + return ret; + + switch (ret) { + case DEV_DOWN: + dev_info(dev, "Stopping device qat_dev%d\n", accel_id); + + if (!adf_dev_started(accel_dev)) { + dev_info(&GET_DEV(accel_dev), "Device qat_dev%d already down\n", + accel_id); + + break; + } + + ret = adf_dev_down(accel_dev); + if (ret) + return ret; + + break; + case DEV_UP: + dev_info(dev, "Starting device qat_dev%d\n", accel_id); + + ret = adf_dev_up(accel_dev, true); + if (ret == -EALREADY) { + break; + } else if (ret) { + dev_err(dev, "Failed to start device qat_dev%d\n", + accel_id); + adf_dev_down(accel_dev); + return ret; + } + break; + default: + return -EINVAL; + } + + return count; +} + +static ssize_t cfg_services_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0}; + struct adf_accel_dev *accel_dev; + int ret; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, + ADF_SERVICES_ENABLED, services); + if (ret) + return ret; + + return sysfs_emit(buf, "%s\n", services); +} + +static int adf_sysfs_update_dev_config(struct adf_accel_dev *accel_dev, + const char *services) +{ + return adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC, + ADF_SERVICES_ENABLED, services, + ADF_STR); +} + +static ssize_t cfg_services_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { }; + struct adf_hw_device_data *hw_data; + struct adf_accel_dev *accel_dev; + int ret; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + ret = adf_parse_service_string(accel_dev, buf, services, + ADF_CFG_MAX_VAL_LEN_IN_BYTES); + if (ret) + return ret; + + if (adf_dev_started(accel_dev)) { + dev_info(dev, "Device qat_dev%d must be down to reconfigure the service.\n", + accel_dev->accel_id); + return -EINVAL; + } + + ret = adf_sysfs_update_dev_config(accel_dev, services); + if (ret < 0) + return ret; + + hw_data = GET_HW_DATA(accel_dev); + + /* Update capabilities mask after change in configuration. + * A call to this function is required as capabilities are, at the + * moment, tied to configuration + */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); + if (!hw_data->accel_capabilities_mask) + return -EINVAL; + + return count; +} + +static ssize_t pm_idle_enabled_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + char pm_idle_enabled[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {}; + struct adf_accel_dev *accel_dev; + int ret; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, + ADF_PM_IDLE_SUPPORT, pm_idle_enabled); + if (ret) + return sysfs_emit(buf, "1\n"); + + return sysfs_emit(buf, "%s\n", pm_idle_enabled); +} + +static ssize_t pm_idle_enabled_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + unsigned long pm_idle_enabled_cfg_val; + struct adf_accel_dev *accel_dev; + bool pm_idle_enabled; + int ret; + + ret = kstrtobool(buf, &pm_idle_enabled); + if (ret) + return ret; + + pm_idle_enabled_cfg_val = pm_idle_enabled; + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + if (adf_dev_started(accel_dev)) { + dev_info(dev, "Device qat_dev%d must be down to set pm_idle_enabled.\n", + accel_dev->accel_id); + return -EINVAL; + } + + ret = adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC, + ADF_PM_IDLE_SUPPORT, &pm_idle_enabled_cfg_val, + ADF_DEC); + if (ret) + return ret; + + return count; +} +static DEVICE_ATTR_RW(pm_idle_enabled); + +static ssize_t auto_reset_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct adf_accel_dev *accel_dev; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + return sysfs_emit(buf, "%s\n", str_on_off(accel_dev->autoreset_on_error)); +} + +static ssize_t auto_reset_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct adf_accel_dev *accel_dev; + bool enabled = false; + int ret; + + ret = kstrtobool(buf, &enabled); + if (ret) + return ret; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + accel_dev->autoreset_on_error = enabled; + + return count; +} +static DEVICE_ATTR_RW(auto_reset); + +static DEVICE_ATTR_RW(state); +static DEVICE_ATTR_RW(cfg_services); + +static ssize_t rp2srv_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct adf_hw_device_data *hw_data; + struct adf_accel_dev *accel_dev; + enum adf_cfg_service_type svc; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + hw_data = GET_HW_DATA(accel_dev); + + if (accel_dev->sysfs.ring_num == UNSET_RING_NUM) + return -EINVAL; + + down_read(&accel_dev->sysfs.lock); + svc = GET_SRV_TYPE(accel_dev, accel_dev->sysfs.ring_num % + hw_data->num_banks_per_vf); + up_read(&accel_dev->sysfs.lock); + + switch (svc) { + case COMP: + return sysfs_emit(buf, "%s\n", ADF_CFG_DC); + case SYM: + return sysfs_emit(buf, "%s\n", ADF_CFG_SYM); + case ASYM: + return sysfs_emit(buf, "%s\n", ADF_CFG_ASYM); + case DECOMP: + return sysfs_emit(buf, "%s\n", ADF_CFG_DECOMP); + default: + break; + } + return -EINVAL; +} + +static ssize_t rp2srv_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct adf_accel_dev *accel_dev; + int num_rings, ret; + unsigned int ring; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + ret = kstrtouint(buf, 10, &ring); + if (ret) + return ret; + + num_rings = GET_MAX_BANKS(accel_dev); + if (ring >= num_rings) { + dev_err(&GET_DEV(accel_dev), + "Device does not support more than %u ring pairs\n", + num_rings); + return -EINVAL; + } + + down_write(&accel_dev->sysfs.lock); + accel_dev->sysfs.ring_num = ring; + up_write(&accel_dev->sysfs.lock); + + return count; +} +static DEVICE_ATTR_RW(rp2srv); + +static ssize_t num_rps_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct adf_accel_dev *accel_dev; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + return sysfs_emit(buf, "%u\n", GET_MAX_BANKS(accel_dev)); +} +static DEVICE_ATTR_RO(num_rps); + +static struct attribute *qat_attrs[] = { + &dev_attr_state.attr, + &dev_attr_cfg_services.attr, + &dev_attr_pm_idle_enabled.attr, + &dev_attr_rp2srv.attr, + &dev_attr_num_rps.attr, + &dev_attr_auto_reset.attr, + NULL, +}; + +static struct attribute_group qat_group = { + .attrs = qat_attrs, + .name = "qat", +}; + +int adf_sysfs_init(struct adf_accel_dev *accel_dev) +{ + int ret; + + ret = devm_device_add_group(&GET_DEV(accel_dev), &qat_group); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to create qat attribute group: %d\n", ret); + } + + accel_dev->sysfs.ring_num = UNSET_RING_NUM; + + return ret; +} +EXPORT_SYMBOL_GPL(adf_sysfs_init); diff --git a/drivers/crypto/intel/qat/qat_common/adf_sysfs_anti_rb.c b/drivers/crypto/intel/qat/qat_common/adf_sysfs_anti_rb.c new file mode 100644 index 000000000..789341ad1 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_sysfs_anti_rb.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2026 Intel Corporation */ +#include +#include + +#include "adf_anti_rb.h" +#include "adf_common_drv.h" +#include "adf_sysfs_anti_rb.h" + +static ssize_t enforced_min_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct adf_accel_dev *accel_dev; + int err; + u8 svn; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + err = adf_anti_rb_query(accel_dev, ARB_ENFORCED_MIN_SVN, &svn); + if (err) + return err; + + return sysfs_emit(buf, "%u\n", svn); +} +static DEVICE_ATTR_RO(enforced_min); + +static ssize_t active_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct adf_accel_dev *accel_dev; + int err; + u8 svn; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + err = adf_anti_rb_query(accel_dev, ARB_ACTIVE_SVN, &svn); + if (err) + return err; + + return sysfs_emit(buf, "%u\n", svn); +} +static DEVICE_ATTR_RO(active); + +static ssize_t permanent_min_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct adf_accel_dev *accel_dev; + int err; + u8 svn; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + err = adf_anti_rb_query(accel_dev, ARB_PERMANENT_MIN_SVN, &svn); + if (err) + return err; + + return sysfs_emit(buf, "%u\n", svn); +} +static DEVICE_ATTR_RO(permanent_min); + +static ssize_t commit_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct adf_accel_dev *accel_dev; + bool val; + int err; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + err = kstrtobool(buf, &val); + if (err) + return err; + + if (!val) + return -EINVAL; + + err = adf_anti_rb_commit(accel_dev); + if (err) + return err; + + return count; +} +static DEVICE_ATTR_WO(commit); + +static struct attribute *qat_svn_attrs[] = { + &dev_attr_commit.attr, + &dev_attr_active.attr, + &dev_attr_enforced_min.attr, + &dev_attr_permanent_min.attr, + NULL +}; + +static const struct attribute_group qat_svn_group = { + .attrs = qat_svn_attrs, + .name = "qat_svn", +}; + +void adf_sysfs_start_arb(struct adf_accel_dev *accel_dev) +{ + struct adf_anti_rb_hw_data *anti_rb = GET_ANTI_RB_DATA(accel_dev); + + if (!anti_rb->anti_rb_enabled || !anti_rb->anti_rb_enabled(accel_dev)) + return; + + if (device_add_group(&GET_DEV(accel_dev), &qat_svn_group)) { + dev_warn(&GET_DEV(accel_dev), + "Failed to create qat_svn attribute group\n"); + return; + } + + anti_rb->sysfs_added = true; +} + +void adf_sysfs_stop_arb(struct adf_accel_dev *accel_dev) +{ + struct adf_anti_rb_hw_data *anti_rb = GET_ANTI_RB_DATA(accel_dev); + + if (!anti_rb->sysfs_added) + return; + + device_remove_group(&GET_DEV(accel_dev), &qat_svn_group); + + anti_rb->sysfs_added = false; + anti_rb->svncheck_retry = 0; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_sysfs_anti_rb.h b/drivers/crypto/intel/qat/qat_common/adf_sysfs_anti_rb.h new file mode 100644 index 000000000..f0c2b6e46 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_sysfs_anti_rb.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2026 Intel Corporation */ +#ifndef ADF_SYSFS_ANTI_RB_H_ +#define ADF_SYSFS_ANTI_RB_H_ + +struct adf_accel_dev; + +void adf_sysfs_start_arb(struct adf_accel_dev *accel_dev); +void adf_sysfs_stop_arb(struct adf_accel_dev *accel_dev); + +#endif /* ADF_SYSFS_ANTI_RB_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_sysfs_kpt.c b/drivers/crypto/intel/qat/qat_common/adf_sysfs_kpt.c new file mode 100644 index 000000000..1f733ae80 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_sysfs_kpt.c @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2026 Intel Corporation */ +#include +#include + +#include "adf_cfg.h" +#include "adf_cfg_services.h" +#include "adf_common_drv.h" +#include "adf_kpt.h" +#include "adf_sysfs_kpt.h" + +static ssize_t enable_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct adf_kpt_interface_data *user_data; + struct adf_accel_dev *accel_dev; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + user_data = GET_KPT_USER_DATA(accel_dev); + + return sysfs_emit(buf, "%d\n", user_data->enable); +} + +static ssize_t enable_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct adf_kpt_interface_data *user_data; + struct adf_hw_device_data *hw_data; + struct adf_accel_dev *accel_dev; + bool enable; + int ret; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + if (adf_dev_started(accel_dev)) { + dev_info(dev, "Device qat_dev%d must be down before enabling KPT\n", + accel_dev->accel_id); + return -EINVAL; + } + + if (adf_get_service_enabled(accel_dev) != SVC_ASYM) { + dev_info(dev, "KPT can only be enabled when the asymmetric service is enabled\n"); + return -EINVAL; + } + + hw_data = GET_HW_DATA(accel_dev); + + /* + * Restore the KPT capability bit in the device's capabilities mask + * before processing user input, as the bit may have been cleared if + * KPT was previously disabled by the user. + */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); + if (!hw_data->accel_capabilities_mask) + return -EINVAL; + + ret = kstrtobool(buf, &enable); + if (ret) + return ret; + + user_data = GET_KPT_USER_DATA(accel_dev); + user_data->enable = enable; + + return count; +} +static DEVICE_ATTR_RW(enable); + +static ssize_t swk_shared_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct adf_kpt_interface_data *user_data; + struct adf_accel_dev *accel_dev; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + user_data = GET_KPT_USER_DATA(accel_dev); + + return sysfs_emit(buf, "%d\n", user_data->swk_shared); +} + +static ssize_t swk_shared_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct adf_kpt_interface_data *user_data; + struct adf_accel_dev *accel_dev; + bool swk_shared; + int ret; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + if (adf_dev_started(accel_dev)) { + dev_info(dev, "Device qat_dev%d must be down before setting swk_shared\n", + accel_dev->accel_id); + return -EINVAL; + } + + ret = kstrtobool(buf, &swk_shared); + if (ret) + return ret; + + user_data = GET_KPT_USER_DATA(accel_dev); + user_data->swk_shared = swk_shared; + + return count; +} +static DEVICE_ATTR_RW(swk_shared); + +static ssize_t swk_max_ttl_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct adf_kpt_interface_data *user_data; + struct adf_accel_dev *accel_dev; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + user_data = GET_KPT_USER_DATA(accel_dev); + + return sysfs_emit(buf, "%u\n", user_data->swk_max_ttl); +} + +static ssize_t swk_max_ttl_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct adf_kpt_hw_data *kpt_data; + struct adf_accel_dev *accel_dev; + unsigned int swk_max_ttl; + int ret; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + if (adf_dev_started(accel_dev)) { + dev_info(dev, "Device qat_dev%d must be down before setting swk_max_ttl\n", + accel_dev->accel_id); + return -EINVAL; + } + + ret = kstrtouint(buf, 10, &swk_max_ttl); + if (ret) + return ret; + + kpt_data = GET_KPT_CFG_DATA(accel_dev); + + if (swk_max_ttl > kpt_data->max_swk_ttl) { + dev_info(dev, "Configuration value is out of range (%u - %u)\n", + 0, kpt_data->max_swk_ttl); + return -EINVAL; + } + + kpt_data->user_input.swk_max_ttl = swk_max_ttl; + + return count; +} +static DEVICE_ATTR_RW(swk_max_ttl); + +static ssize_t swk_cnt_per_fn_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct adf_kpt_interface_data *user_data; + struct adf_accel_dev *accel_dev; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + user_data = GET_KPT_USER_DATA(accel_dev); + + return sysfs_emit(buf, "%u\n", user_data->swk_cnt_per_fn); +} + +static ssize_t swk_cnt_per_fn_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct adf_kpt_hw_data *kpt_data; + struct adf_accel_dev *accel_dev; + unsigned int swk_cnt_per_fn; + int ret; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + if (adf_dev_started(accel_dev)) { + dev_info(dev, "Device qat_dev%d must be down before setting swk_cnt_per_fn\n", + accel_dev->accel_id); + return -EINVAL; + } + + ret = kstrtouint(buf, 10, &swk_cnt_per_fn); + if (ret) + return ret; + + kpt_data = GET_KPT_CFG_DATA(accel_dev); + + if (swk_cnt_per_fn > kpt_data->max_swk_cnt_per_fn_pasid) { + dev_info(dev, "swk_cnt_per_fn: value out of range (0 - %u)\n", + kpt_data->max_swk_cnt_per_fn_pasid); + return -EINVAL; + } + + kpt_data->user_input.swk_cnt_per_fn = swk_cnt_per_fn; + + return count; +} +static DEVICE_ATTR_RW(swk_cnt_per_fn); + +static ssize_t swk_cnt_per_pasid_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct adf_kpt_interface_data *user_data; + struct adf_accel_dev *accel_dev; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + user_data = GET_KPT_USER_DATA(accel_dev); + + return sysfs_emit(buf, "%u\n", user_data->swk_cnt_per_pasid); +} + +static ssize_t swk_cnt_per_pasid_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct adf_kpt_hw_data *kpt_data; + struct adf_accel_dev *accel_dev; + unsigned int swk_cnt_per_pasid; + int ret; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + if (adf_dev_started(accel_dev)) { + dev_info(dev, "Device qat_dev%d must be down before setting swk_cnt_per_pasid\n", + accel_dev->accel_id); + return -EINVAL; + } + + ret = kstrtouint(buf, 10, &swk_cnt_per_pasid); + if (ret) + return ret; + + kpt_data = GET_KPT_CFG_DATA(accel_dev); + + if (swk_cnt_per_pasid > kpt_data->max_swk_cnt_per_fn_pasid) { + dev_info(dev, "swk_cnt_per_pasid: value out of range (0 - %u)\n", + kpt_data->max_swk_cnt_per_fn_pasid); + return -EINVAL; + } + + kpt_data->user_input.swk_cnt_per_pasid = swk_cnt_per_pasid; + + return count; +} +static DEVICE_ATTR_RW(swk_cnt_per_pasid); + +static struct attribute *qat_kpt_attrs[] = { + &dev_attr_enable.attr, + &dev_attr_swk_shared.attr, + &dev_attr_swk_max_ttl.attr, + &dev_attr_swk_cnt_per_fn.attr, + &dev_attr_swk_cnt_per_pasid.attr, + NULL, +}; + +static const struct attribute_group qat_kpt_group = { + .attrs = qat_kpt_attrs, + .name = "qat_kpt", +}; + +int adf_sysfs_init_kpt(struct adf_accel_dev *accel_dev) +{ + int ret; + + ret = devm_device_add_group(&GET_DEV(accel_dev), &qat_kpt_group); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Failed to create qat_kpt attribute group\n"); + return ret; + } + + return 0; +} +EXPORT_SYMBOL_GPL(adf_sysfs_init_kpt); diff --git a/drivers/crypto/intel/qat/qat_common/adf_sysfs_kpt.h b/drivers/crypto/intel/qat/qat_common/adf_sysfs_kpt.h new file mode 100644 index 000000000..d9d065f75 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_sysfs_kpt.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2026 Intel Corporation */ +#ifndef ADF_SYSFS_KPT_H_ +#define ADF_SYSFS_KPT_H_ + +struct adf_accel_dev; + +int adf_sysfs_init_kpt(struct adf_accel_dev *accel_dev); + +#endif /* ADF_SYSFS_KPT_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_sysfs_ras_counters.c b/drivers/crypto/intel/qat/qat_common/adf_sysfs_ras_counters.c new file mode 100644 index 000000000..ef1420199 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_sysfs_ras_counters.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ + +#include +#include +#include + +#include "adf_common_drv.h" +#include "adf_sysfs_ras_counters.h" + +static ssize_t errors_correctable_show(struct device *dev, + struct device_attribute *dev_attr, + char *buf) +{ + struct adf_accel_dev *accel_dev; + int counter; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + counter = ADF_RAS_ERR_CTR_READ(accel_dev->ras_errors, ADF_RAS_CORR); + return sysfs_emit(buf, "%d\n", counter); +} + +static ssize_t errors_nonfatal_show(struct device *dev, + struct device_attribute *dev_attr, + char *buf) +{ + struct adf_accel_dev *accel_dev; + int counter; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + counter = ADF_RAS_ERR_CTR_READ(accel_dev->ras_errors, ADF_RAS_UNCORR); + return sysfs_emit(buf, "%d\n", counter); +} + +static ssize_t errors_fatal_show(struct device *dev, + struct device_attribute *dev_attr, + char *buf) +{ + struct adf_accel_dev *accel_dev; + int counter; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + counter = ADF_RAS_ERR_CTR_READ(accel_dev->ras_errors, ADF_RAS_FATAL); + return sysfs_emit(buf, "%d\n", counter); +} + +static ssize_t reset_error_counters_store(struct device *dev, + struct device_attribute *dev_attr, + const char *buf, size_t count) +{ + struct adf_accel_dev *accel_dev; + + if (buf[0] != '1' || count != 2) + return -EINVAL; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + ADF_RAS_ERR_CTR_CLEAR(accel_dev->ras_errors); + + return count; +} + +static DEVICE_ATTR_RO(errors_correctable); +static DEVICE_ATTR_RO(errors_nonfatal); +static DEVICE_ATTR_RO(errors_fatal); +static DEVICE_ATTR_WO(reset_error_counters); + +static struct attribute *qat_ras_attrs[] = { + &dev_attr_errors_correctable.attr, + &dev_attr_errors_nonfatal.attr, + &dev_attr_errors_fatal.attr, + &dev_attr_reset_error_counters.attr, + NULL, +}; + +static struct attribute_group qat_ras_group = { + .attrs = qat_ras_attrs, + .name = "qat_ras", +}; + +void adf_sysfs_start_ras(struct adf_accel_dev *accel_dev) +{ + if (!accel_dev->ras_errors.enabled) + return; + + ADF_RAS_ERR_CTR_CLEAR(accel_dev->ras_errors); + + if (device_add_group(&GET_DEV(accel_dev), &qat_ras_group)) + dev_err(&GET_DEV(accel_dev), + "Failed to create qat_ras attribute group.\n"); + + accel_dev->ras_errors.sysfs_added = true; +} + +void adf_sysfs_stop_ras(struct adf_accel_dev *accel_dev) +{ + if (!accel_dev->ras_errors.enabled) + return; + + if (accel_dev->ras_errors.sysfs_added) { + device_remove_group(&GET_DEV(accel_dev), &qat_ras_group); + accel_dev->ras_errors.sysfs_added = false; + } + + ADF_RAS_ERR_CTR_CLEAR(accel_dev->ras_errors); +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_sysfs_ras_counters.h b/drivers/crypto/intel/qat/qat_common/adf_sysfs_ras_counters.h new file mode 100644 index 000000000..99e9d9cf5 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_sysfs_ras_counters.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ + +#ifndef ADF_RAS_H +#define ADF_RAS_H + +#include +#include + +struct adf_accel_dev; + +void adf_sysfs_start_ras(struct adf_accel_dev *accel_dev); +void adf_sysfs_stop_ras(struct adf_accel_dev *accel_dev); + +#define ADF_RAS_ERR_CTR_READ(ras_errors, ERR) \ + atomic_read(&(ras_errors).counter[ERR]) + +#define ADF_RAS_ERR_CTR_CLEAR(ras_errors) \ + do { \ + for (int err = 0; err < ADF_RAS_ERRORS; ++err) \ + atomic_set(&(ras_errors).counter[err], 0); \ + } while (0) + +#define ADF_RAS_ERR_CTR_INC(ras_errors, ERR) \ + atomic_inc(&(ras_errors).counter[ERR]) + +#endif /* ADF_RAS_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.c b/drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.c new file mode 100644 index 000000000..89bfd8761 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.c @@ -0,0 +1,464 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ + +#define dev_fmt(fmt) "RateLimiting: " fmt + +#include +#include +#include +#include + +#include "adf_common_drv.h" +#include "adf_rl.h" +#include "adf_sysfs_rl.h" + +#define GET_RL_STRUCT(accel_dev) ((accel_dev)->rate_limiting->user_input) + +enum rl_ops { + ADD, + UPDATE, + RM, + RM_ALL, + GET, +}; + +enum rl_params { + RP_MASK, + ID, + CIR, + PIR, + SRV, + CAP_REM_SRV, +}; + +static const char *const rl_services[] = { + [SVC_ASYM] = "asym", + [SVC_SYM] = "sym", + [SVC_DC] = "dc", + [SVC_DECOMP] = "decomp", +}; + +static const char *const rl_operations[] = { + [ADD] = "add", + [UPDATE] = "update", + [RM] = "rm", + [RM_ALL] = "rm_all", + [GET] = "get", +}; + +static int set_param_u(struct device *dev, enum rl_params param, u64 set) +{ + struct adf_rl_interface_data *data; + struct adf_accel_dev *accel_dev; + int ret = 0; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + data = &GET_RL_STRUCT(accel_dev); + + down_write(&data->lock); + switch (param) { + case RP_MASK: + data->input.rp_mask = set; + break; + case CIR: + data->input.cir = set; + break; + case PIR: + data->input.pir = set; + break; + case SRV: + data->input.srv = set; + break; + case CAP_REM_SRV: + data->cap_rem_srv = set; + break; + default: + ret = -EINVAL; + break; + } + up_write(&data->lock); + + return ret; +} + +static int set_param_s(struct device *dev, enum rl_params param, int set) +{ + struct adf_rl_interface_data *data; + struct adf_accel_dev *accel_dev; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev || param != ID) + return -EINVAL; + + data = &GET_RL_STRUCT(accel_dev); + + down_write(&data->lock); + data->input.sla_id = set; + up_write(&data->lock); + + return 0; +} + +static int get_param_u(struct device *dev, enum rl_params param, u64 *get) +{ + struct adf_rl_interface_data *data; + struct adf_accel_dev *accel_dev; + int ret = 0; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + data = &GET_RL_STRUCT(accel_dev); + + down_read(&data->lock); + switch (param) { + case RP_MASK: + *get = data->input.rp_mask; + break; + case CIR: + *get = data->input.cir; + break; + case PIR: + *get = data->input.pir; + break; + case SRV: + *get = data->input.srv; + break; + default: + ret = -EINVAL; + } + up_read(&data->lock); + + return ret; +} + +static int get_param_s(struct device *dev, enum rl_params param) +{ + struct adf_rl_interface_data *data; + struct adf_accel_dev *accel_dev; + int ret = 0; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + data = &GET_RL_STRUCT(accel_dev); + + down_read(&data->lock); + if (param == ID) + ret = data->input.sla_id; + up_read(&data->lock); + + return ret; +} + +static ssize_t rp_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + int ret; + u64 get; + + ret = get_param_u(dev, RP_MASK, &get); + if (ret) + return ret; + + return sysfs_emit(buf, "%#llx\n", get); +} + +static ssize_t rp_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + int err; + u64 val; + + err = kstrtou64(buf, 16, &val); + if (err) + return err; + + err = set_param_u(dev, RP_MASK, val); + if (err) + return err; + + return count; +} +static DEVICE_ATTR_RW(rp); + +static ssize_t id_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%d\n", get_param_s(dev, ID)); +} + +static ssize_t id_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + int err; + int val; + + err = kstrtoint(buf, 10, &val); + if (err) + return err; + + err = set_param_s(dev, ID, val); + if (err) + return err; + + return count; +} +static DEVICE_ATTR_RW(id); + +static ssize_t cir_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + int ret; + u64 get; + + ret = get_param_u(dev, CIR, &get); + if (ret) + return ret; + + return sysfs_emit(buf, "%llu\n", get); +} + +static ssize_t cir_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + unsigned int val; + int err; + + err = kstrtouint(buf, 10, &val); + if (err) + return err; + + err = set_param_u(dev, CIR, val); + if (err) + return err; + + return count; +} +static DEVICE_ATTR_RW(cir); + +static ssize_t pir_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + int ret; + u64 get; + + ret = get_param_u(dev, PIR, &get); + if (ret) + return ret; + + return sysfs_emit(buf, "%llu\n", get); +} + +static ssize_t pir_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + unsigned int val; + int err; + + err = kstrtouint(buf, 10, &val); + if (err) + return err; + + err = set_param_u(dev, PIR, val); + if (err) + return err; + + return count; +} +static DEVICE_ATTR_RW(pir); + +static ssize_t srv_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + int ret; + u64 get; + + ret = get_param_u(dev, SRV, &get); + if (ret) + return ret; + + if (get == SVC_BASE_COUNT) + return -EINVAL; + + return sysfs_emit(buf, "%s\n", rl_services[get]); +} + +static ssize_t srv_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct adf_accel_dev *accel_dev; + unsigned int val; + int ret; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + ret = sysfs_match_string(rl_services, buf); + if (ret < 0) + return ret; + + val = ret; + if (!adf_is_service_enabled(accel_dev, val)) + return -EINVAL; + + ret = set_param_u(dev, SRV, val); + if (ret) + return ret; + + return count; +} +static DEVICE_ATTR_RW(srv); + +static ssize_t cap_rem_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct adf_rl_interface_data *data; + struct adf_accel_dev *accel_dev; + int rem_cap; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + data = &GET_RL_STRUCT(accel_dev); + + down_read(&data->lock); + rem_cap = adf_rl_get_capability_remaining(accel_dev, data->cap_rem_srv, + RL_SLA_EMPTY_ID); + up_read(&data->lock); + if (rem_cap < 0) + return rem_cap; + + return sysfs_emit(buf, "%u\n", rem_cap); +} + +static ssize_t cap_rem_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + int ret; + + ret = sysfs_match_string(rl_services, buf); + if (ret < 0) + return ret; + + ret = set_param_u(dev, CAP_REM_SRV, ret); + if (ret) + return ret; + + return count; +} +static DEVICE_ATTR_RW(cap_rem); + +static ssize_t sla_op_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct adf_rl_interface_data *data; + struct adf_accel_dev *accel_dev; + int ret; + + accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + + data = &GET_RL_STRUCT(accel_dev); + + ret = sysfs_match_string(rl_operations, buf); + if (ret < 0) + return ret; + + down_write(&data->lock); + switch (ret) { + case ADD: + data->input.parent_id = RL_PARENT_DEFAULT_ID; + data->input.type = RL_LEAF; + data->input.sla_id = 0; + ret = adf_rl_add_sla(accel_dev, &data->input); + if (ret) + goto err_free_lock; + break; + case UPDATE: + ret = adf_rl_update_sla(accel_dev, &data->input); + if (ret) + goto err_free_lock; + break; + case RM: + ret = adf_rl_remove_sla(accel_dev, data->input.sla_id); + if (ret) + goto err_free_lock; + break; + case RM_ALL: + adf_rl_remove_sla_all(accel_dev, false); + break; + case GET: + ret = adf_rl_get_sla(accel_dev, &data->input); + if (ret) + goto err_free_lock; + break; + default: + ret = -EINVAL; + goto err_free_lock; + } + up_write(&data->lock); + + return count; + +err_free_lock: + up_write(&data->lock); + + return ret; +} +static DEVICE_ATTR_WO(sla_op); + +static struct attribute *qat_rl_attrs[] = { + &dev_attr_rp.attr, + &dev_attr_id.attr, + &dev_attr_cir.attr, + &dev_attr_pir.attr, + &dev_attr_srv.attr, + &dev_attr_cap_rem.attr, + &dev_attr_sla_op.attr, + NULL, +}; + +static struct attribute_group qat_rl_group = { + .attrs = qat_rl_attrs, + .name = "qat_rl", +}; + +int adf_sysfs_rl_add(struct adf_accel_dev *accel_dev) +{ + struct adf_rl_interface_data *data; + int ret; + + data = &GET_RL_STRUCT(accel_dev); + + ret = device_add_group(&GET_DEV(accel_dev), &qat_rl_group); + if (ret) + dev_err(&GET_DEV(accel_dev), + "Failed to create qat_rl attribute group\n"); + + data->cap_rem_srv = SVC_BASE_COUNT; + data->input.srv = SVC_BASE_COUNT; + data->sysfs_added = true; + + return ret; +} + +void adf_sysfs_rl_rm(struct adf_accel_dev *accel_dev) +{ + struct adf_rl_interface_data *data; + + data = &GET_RL_STRUCT(accel_dev); + if (!data->sysfs_added) + return; + + device_remove_group(&GET_DEV(accel_dev), &qat_rl_group); + data->sysfs_added = false; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.h b/drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.h new file mode 100644 index 000000000..22d36aa8a --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ +#ifndef ADF_SYSFS_RL_H_ +#define ADF_SYSFS_RL_H_ + +struct adf_accel_dev; + +int adf_sysfs_rl_add(struct adf_accel_dev *accel_dev); +void adf_sysfs_rl_rm(struct adf_accel_dev *accel_dev); + +#endif /* ADF_SYSFS_RL_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_telemetry.c b/drivers/crypto/intel/qat/qat_common/adf_telemetry.c new file mode 100644 index 000000000..8c575113c --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_telemetry.c @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2023 Intel Corporation. */ +#define dev_fmt(fmt) "Telemetry: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "adf_admin.h" +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_telemetry.h" + +#define TL_IS_ZERO(input) ((input) == 0) + +static bool is_tl_supported(struct adf_accel_dev *accel_dev) +{ + u16 fw_caps = GET_HW_DATA(accel_dev)->fw_capabilities; + + return fw_caps & TL_CAPABILITY_BIT; +} + +static int validate_tl_data(struct adf_tl_hw_data *tl_data) +{ + if (!tl_data->dev_counters || + TL_IS_ZERO(tl_data->num_dev_counters) || + !tl_data->sl_util_counters || + !tl_data->sl_exec_counters || + !tl_data->rp_counters || + TL_IS_ZERO(tl_data->num_rp_counters)) + return -EOPNOTSUPP; + + return 0; +} + +static int validate_tl_slice_counters(struct icp_qat_fw_init_admin_slice_cnt *slice_count, + u8 max_slices_per_type) +{ + u8 *sl_counter = (u8 *)slice_count; + int i; + + for (i = 0; i < ADF_TL_SL_CNT_COUNT; i++) { + if (sl_counter[i] > max_slices_per_type) + return -EINVAL; + } + + return 0; +} + +static int adf_tl_alloc_mem(struct adf_accel_dev *accel_dev) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(accel_dev); + struct device *dev = &GET_DEV(accel_dev); + size_t regs_sz = tl_data->layout_sz; + struct adf_telemetry *telemetry; + int node = dev_to_node(dev); + void *tl_data_regs; + unsigned int i; + + telemetry = kzalloc_node(sizeof(*telemetry), GFP_KERNEL, node); + if (!telemetry) + return -ENOMEM; + + telemetry->rp_num_indexes = kmalloc_array(tl_data->max_rp, + sizeof(*telemetry->rp_num_indexes), + GFP_KERNEL); + if (!telemetry->rp_num_indexes) + goto err_free_tl; + + telemetry->regs_hist_buff = kmalloc_objs(*telemetry->regs_hist_buff, + tl_data->num_hbuff); + if (!telemetry->regs_hist_buff) + goto err_free_rp_indexes; + + telemetry->regs_data = dma_alloc_coherent(dev, regs_sz, + &telemetry->regs_data_p, + GFP_KERNEL); + if (!telemetry->regs_data) + goto err_free_regs_hist_buff; + + for (i = 0; i < tl_data->num_hbuff; i++) { + tl_data_regs = kzalloc_node(regs_sz, GFP_KERNEL, node); + if (!tl_data_regs) + goto err_free_dma; + + telemetry->regs_hist_buff[i] = tl_data_regs; + } + + accel_dev->telemetry = telemetry; + + return 0; + +err_free_dma: + dma_free_coherent(dev, regs_sz, telemetry->regs_data, + telemetry->regs_data_p); + + while (i--) + kfree(telemetry->regs_hist_buff[i]); + +err_free_regs_hist_buff: + kfree(telemetry->regs_hist_buff); +err_free_rp_indexes: + kfree(telemetry->rp_num_indexes); +err_free_tl: + kfree(telemetry); + + return -ENOMEM; +} + +static void adf_tl_free_mem(struct adf_accel_dev *accel_dev) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(accel_dev); + struct adf_telemetry *telemetry = accel_dev->telemetry; + struct device *dev = &GET_DEV(accel_dev); + size_t regs_sz = tl_data->layout_sz; + unsigned int i; + + for (i = 0; i < tl_data->num_hbuff; i++) + kfree(telemetry->regs_hist_buff[i]); + + dma_free_coherent(dev, regs_sz, telemetry->regs_data, + telemetry->regs_data_p); + + kfree(telemetry->regs_hist_buff); + kfree(telemetry->rp_num_indexes); + kfree(telemetry); + accel_dev->telemetry = NULL; +} + +static unsigned long get_next_timeout(void) +{ + return msecs_to_jiffies(ADF_TL_TIMER_INT_MS); +} + +static void snapshot_regs(struct adf_telemetry *telemetry, size_t size) +{ + void *dst = telemetry->regs_hist_buff[telemetry->hb_num]; + void *src = telemetry->regs_data; + + memcpy(dst, src, size); +} + +static void tl_work_handler(struct work_struct *work) +{ + struct delayed_work *delayed_work; + struct adf_telemetry *telemetry; + struct adf_tl_hw_data *tl_data; + u32 msg_cnt, old_msg_cnt; + size_t layout_sz; + u32 *regs_data; + size_t id; + + delayed_work = to_delayed_work(work); + telemetry = container_of(delayed_work, struct adf_telemetry, work_ctx); + tl_data = &GET_TL_DATA(telemetry->accel_dev); + regs_data = telemetry->regs_data; + + id = tl_data->msg_cnt_off / sizeof(*regs_data); + layout_sz = tl_data->layout_sz; + + if (!atomic_read(&telemetry->state)) { + cancel_delayed_work_sync(&telemetry->work_ctx); + return; + } + + msg_cnt = regs_data[id]; + old_msg_cnt = msg_cnt; + if (msg_cnt == telemetry->msg_cnt) + goto out; + + mutex_lock(&telemetry->regs_hist_lock); + + snapshot_regs(telemetry, layout_sz); + + /* Check if data changed while updating it */ + msg_cnt = regs_data[id]; + if (old_msg_cnt != msg_cnt) + snapshot_regs(telemetry, layout_sz); + + telemetry->msg_cnt = msg_cnt; + telemetry->hb_num++; + telemetry->hb_num %= telemetry->hbuffs; + + mutex_unlock(&telemetry->regs_hist_lock); + +out: + adf_misc_wq_queue_delayed_work(&telemetry->work_ctx, get_next_timeout()); +} + +int adf_tl_halt(struct adf_accel_dev *accel_dev) +{ + struct adf_telemetry *telemetry = accel_dev->telemetry; + struct device *dev = &GET_DEV(accel_dev); + int ret; + + cancel_delayed_work_sync(&telemetry->work_ctx); + atomic_set(&telemetry->state, 0); + + ret = adf_send_admin_tl_stop(accel_dev); + if (ret) + dev_err(dev, "failed to stop telemetry\n"); + + return ret; +} + +static void adf_set_cmdq_cnt(struct adf_accel_dev *accel_dev, + struct adf_tl_hw_data *tl_data) +{ + struct icp_qat_fw_init_admin_slice_cnt *slice_cnt, *cmdq_cnt; + + slice_cnt = &accel_dev->telemetry->slice_cnt; + cmdq_cnt = &accel_dev->telemetry->cmdq_cnt; + + cmdq_cnt->cpr_cnt = slice_cnt->cpr_cnt * tl_data->multiplier.cpr_cnt; + cmdq_cnt->dcpr_cnt = slice_cnt->dcpr_cnt * tl_data->multiplier.dcpr_cnt; + cmdq_cnt->pke_cnt = slice_cnt->pke_cnt * tl_data->multiplier.pke_cnt; + cmdq_cnt->wat_cnt = slice_cnt->wat_cnt * tl_data->multiplier.wat_cnt; + cmdq_cnt->wcp_cnt = slice_cnt->wcp_cnt * tl_data->multiplier.wcp_cnt; + cmdq_cnt->ucs_cnt = slice_cnt->ucs_cnt * tl_data->multiplier.ucs_cnt; + cmdq_cnt->ath_cnt = slice_cnt->ath_cnt * tl_data->multiplier.ath_cnt; +} + +int adf_tl_run(struct adf_accel_dev *accel_dev, int state) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(accel_dev); + struct adf_telemetry *telemetry = accel_dev->telemetry; + struct device *dev = &GET_DEV(accel_dev); + size_t layout_sz = tl_data->layout_sz; + int ret; + + ret = adf_send_admin_tl_start(accel_dev, telemetry->regs_data_p, + layout_sz, telemetry->rp_num_indexes, + &telemetry->slice_cnt); + if (ret) { + dev_err(dev, "failed to start telemetry\n"); + return ret; + } + + ret = validate_tl_slice_counters(&telemetry->slice_cnt, tl_data->max_sl_cnt); + if (ret) { + dev_err(dev, "invalid value returned by FW\n"); + adf_send_admin_tl_stop(accel_dev); + return ret; + } + + adf_set_cmdq_cnt(accel_dev, tl_data); + + telemetry->hbuffs = state; + atomic_set(&telemetry->state, state); + + adf_misc_wq_queue_delayed_work(&telemetry->work_ctx, get_next_timeout()); + + return 0; +} + +int adf_tl_init(struct adf_accel_dev *accel_dev) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(accel_dev); + u8 max_rp = GET_TL_DATA(accel_dev).max_rp; + struct device *dev = &GET_DEV(accel_dev); + struct adf_telemetry *telemetry; + unsigned int i; + int ret; + + ret = validate_tl_data(tl_data); + if (ret) + return ret; + + ret = adf_tl_alloc_mem(accel_dev); + if (ret) { + dev_err(dev, "failed to initialize: %d\n", ret); + return ret; + } + + telemetry = accel_dev->telemetry; + telemetry->accel_dev = accel_dev; + + mutex_init(&telemetry->wr_lock); + mutex_init(&telemetry->regs_hist_lock); + INIT_DELAYED_WORK(&telemetry->work_ctx, tl_work_handler); + + for (i = 0; i < max_rp; i++) + telemetry->rp_num_indexes[i] = ADF_TL_RP_REGS_DISABLED; + + return 0; +} + +int adf_tl_start(struct adf_accel_dev *accel_dev) +{ + struct device *dev = &GET_DEV(accel_dev); + + if (!accel_dev->telemetry) + return -EOPNOTSUPP; + + if (!is_tl_supported(accel_dev)) { + dev_info(dev, "feature not supported by FW\n"); + adf_tl_free_mem(accel_dev); + return -EOPNOTSUPP; + } + + return 0; +} + +void adf_tl_stop(struct adf_accel_dev *accel_dev) +{ + if (!accel_dev->telemetry) + return; + + if (atomic_read(&accel_dev->telemetry->state)) + adf_tl_halt(accel_dev); +} + +void adf_tl_shutdown(struct adf_accel_dev *accel_dev) +{ + if (!accel_dev->telemetry) + return; + + adf_tl_free_mem(accel_dev); +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_telemetry.h b/drivers/crypto/intel/qat/qat_common/adf_telemetry.h new file mode 100644 index 000000000..02d75c3c2 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_telemetry.h @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2023 Intel Corporation. */ +#ifndef ADF_TELEMETRY_H +#define ADF_TELEMETRY_H + +#include +#include +#include +#include + +#include "icp_qat_fw_init_admin.h" + +struct adf_accel_dev; +struct adf_tl_dbg_counter; +struct dentry; + +#define ADF_TL_SL_CNT_COUNT \ + (sizeof(struct icp_qat_fw_init_admin_slice_cnt) / sizeof(__u8)) + +#define TL_CAPABILITY_BIT BIT(1) +/* Interval within device writes data to DMA region. Value in milliseconds. */ +#define ADF_TL_DATA_WR_INTERVAL_MS 1000 +/* Interval within timer interrupt should be handled. Value in milliseconds. */ +#define ADF_TL_TIMER_INT_MS (ADF_TL_DATA_WR_INTERVAL_MS / 2) + +#define ADF_TL_RP_REGS_DISABLED (0xff) + +struct adf_tl_hw_data { + size_t layout_sz; + size_t slice_reg_sz; + size_t cmdq_reg_sz; + size_t rp_reg_sz; + size_t msg_cnt_off; + const struct adf_tl_dbg_counter *dev_counters; + const struct adf_tl_dbg_counter *sl_util_counters; + const struct adf_tl_dbg_counter *sl_exec_counters; + const struct adf_tl_dbg_counter **cmdq_counters; + const struct adf_tl_dbg_counter *rp_counters; + u8 num_hbuff; + u8 cpp_ns_per_cycle; + u8 bw_units_to_bytes; + u8 num_dev_counters; + u8 num_rp_counters; + u8 num_cmdq_counters; + u8 max_rp; + u8 max_sl_cnt; + struct icp_qat_fw_init_admin_slice_cnt multiplier; +}; + +struct adf_telemetry { + struct adf_accel_dev *accel_dev; + atomic_t state; + u32 hbuffs; + int hb_num; + u32 msg_cnt; + dma_addr_t regs_data_p; /* bus address for DMA mapping */ + void *regs_data; /* virtual address for DMA mapping */ + /** + * @regs_hist_buff: array of pointers to copies of the last @hbuffs + * values of @regs_data + */ + void **regs_hist_buff; + struct dentry *dbg_dir; + u8 *rp_num_indexes; + /** + * @regs_hist_lock: protects from race conditions between write and read + * to the copies referenced by @regs_hist_buff + */ + struct mutex regs_hist_lock; + /** + * @wr_lock: protects from concurrent writes to debugfs telemetry files + */ + struct mutex wr_lock; + struct delayed_work work_ctx; + struct icp_qat_fw_init_admin_slice_cnt slice_cnt; + struct icp_qat_fw_init_admin_slice_cnt cmdq_cnt; +}; + +#ifdef CONFIG_DEBUG_FS +int adf_tl_init(struct adf_accel_dev *accel_dev); +int adf_tl_start(struct adf_accel_dev *accel_dev); +void adf_tl_stop(struct adf_accel_dev *accel_dev); +void adf_tl_shutdown(struct adf_accel_dev *accel_dev); +int adf_tl_run(struct adf_accel_dev *accel_dev, int state); +int adf_tl_halt(struct adf_accel_dev *accel_dev); +#else +static inline int adf_tl_init(struct adf_accel_dev *accel_dev) +{ + return 0; +} + +static inline int adf_tl_start(struct adf_accel_dev *accel_dev) +{ + return 0; +} + +static inline void adf_tl_stop(struct adf_accel_dev *accel_dev) +{ +} + +static inline void adf_tl_shutdown(struct adf_accel_dev *accel_dev) +{ +} +#endif /* CONFIG_DEBUG_FS */ +#endif /* ADF_TELEMETRY_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_timer.c b/drivers/crypto/intel/qat/qat_common/adf_timer.c new file mode 100644 index 000000000..cee965f93 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_timer.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ + +#include +#include +#include +#include +#include +#include +#include + +#include "adf_admin.h" +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_timer.h" + +#define ADF_DEFAULT_TIMER_PERIOD_MS 200 + +/* This periodic update is used to trigger HB, RL & TL fw events */ +static void work_handler(struct work_struct *work) +{ + struct adf_accel_dev *accel_dev; + struct adf_timer *timer_ctx; + u32 time_periods; + + timer_ctx = container_of(to_delayed_work(work), struct adf_timer, work_ctx); + accel_dev = timer_ctx->accel_dev; + + adf_misc_wq_queue_delayed_work(&timer_ctx->work_ctx, + msecs_to_jiffies(ADF_DEFAULT_TIMER_PERIOD_MS)); + + time_periods = div_u64(ktime_ms_delta(ktime_get_real(), timer_ctx->initial_ktime), + ADF_DEFAULT_TIMER_PERIOD_MS); + + if (adf_send_admin_tim_sync(accel_dev, time_periods)) + dev_err(&GET_DEV(accel_dev), "Failed to synchronize qat timer\n"); +} + +int adf_timer_start(struct adf_accel_dev *accel_dev) +{ + struct adf_timer *timer_ctx; + + timer_ctx = kzalloc_obj(*timer_ctx); + if (!timer_ctx) + return -ENOMEM; + + timer_ctx->accel_dev = accel_dev; + accel_dev->timer = timer_ctx; + timer_ctx->initial_ktime = ktime_get_real(); + + INIT_DELAYED_WORK(&timer_ctx->work_ctx, work_handler); + adf_misc_wq_queue_delayed_work(&timer_ctx->work_ctx, + msecs_to_jiffies(ADF_DEFAULT_TIMER_PERIOD_MS)); + + return 0; +} +EXPORT_SYMBOL_GPL(adf_timer_start); + +void adf_timer_stop(struct adf_accel_dev *accel_dev) +{ + struct adf_timer *timer_ctx = accel_dev->timer; + + if (!timer_ctx) + return; + + cancel_delayed_work_sync(&timer_ctx->work_ctx); + + kfree(timer_ctx); + accel_dev->timer = NULL; +} +EXPORT_SYMBOL_GPL(adf_timer_stop); diff --git a/drivers/crypto/intel/qat/qat_common/adf_timer.h b/drivers/crypto/intel/qat/qat_common/adf_timer.h new file mode 100644 index 000000000..68e5136d6 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_timer.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ + +#ifndef ADF_TIMER_H_ +#define ADF_TIMER_H_ + +#include +#include + +struct adf_accel_dev; + +struct adf_timer { + struct adf_accel_dev *accel_dev; + struct delayed_work work_ctx; + ktime_t initial_ktime; +}; + +int adf_timer_start(struct adf_accel_dev *accel_dev); +void adf_timer_stop(struct adf_accel_dev *accel_dev); + +#endif /* ADF_TIMER_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.c b/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.c new file mode 100644 index 000000000..b81f70576 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.c @@ -0,0 +1,737 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2023 Intel Corporation. */ +#define dev_fmt(fmt) "Telemetry debugfs: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "adf_accel_devices.h" +#include "adf_cfg_strings.h" +#include "adf_telemetry.h" +#include "adf_tl_debugfs.h" + +#define TL_VALUE_MIN_PADDING 20 +#define TL_KEY_MIN_PADDING 23 +#define TL_RP_SRV_UNKNOWN "Unknown" + +static int tl_collect_values_u32(struct adf_telemetry *telemetry, + size_t counter_offset, u64 *arr) +{ + unsigned int samples, hb_idx, i; + u32 *regs_hist_buff; + u32 counter_val; + + samples = min(telemetry->msg_cnt, telemetry->hbuffs); + hb_idx = telemetry->hb_num + telemetry->hbuffs - samples; + + mutex_lock(&telemetry->regs_hist_lock); + + for (i = 0; i < samples; i++) { + regs_hist_buff = telemetry->regs_hist_buff[hb_idx % telemetry->hbuffs]; + counter_val = regs_hist_buff[counter_offset / sizeof(counter_val)]; + arr[i] = counter_val; + hb_idx++; + } + + mutex_unlock(&telemetry->regs_hist_lock); + + return samples; +} + +static int tl_collect_values_u64(struct adf_telemetry *telemetry, + size_t counter_offset, u64 *arr) +{ + unsigned int samples, hb_idx, i; + u64 *regs_hist_buff; + u64 counter_val; + + samples = min(telemetry->msg_cnt, telemetry->hbuffs); + hb_idx = telemetry->hb_num + telemetry->hbuffs - samples; + + mutex_lock(&telemetry->regs_hist_lock); + + for (i = 0; i < samples; i++) { + regs_hist_buff = telemetry->regs_hist_buff[hb_idx % telemetry->hbuffs]; + counter_val = regs_hist_buff[counter_offset / sizeof(counter_val)]; + arr[i] = counter_val; + hb_idx++; + } + + mutex_unlock(&telemetry->regs_hist_lock); + + return samples; +} + +/** + * avg_array() - Return average of values within an array. + * @array: Array of values. + * @len: Number of elements. + * + * This algorithm computes average of an array without running into overflow. + * + * Return: average of values. + */ +#define avg_array(array, len) ( \ +{ \ + typeof(&(array)[0]) _array = (array); \ + __unqual_scalar_typeof(_array[0]) _x = 0; \ + __unqual_scalar_typeof(_array[0]) _y = 0; \ + __unqual_scalar_typeof(_array[0]) _a, _b; \ + typeof(len) _len = (len); \ + size_t _i; \ + \ + for (_i = 0; _i < _len; _i++) { \ + _a = _array[_i]; \ + _b = do_div(_a, _len); \ + _x += _a; \ + if (_y >= _len - _b) { \ + _x++; \ + _y -= _len - _b; \ + } else { \ + _y += _b; \ + } \ + } \ + do_div(_y, _len); \ + (_x + _y); \ +}) + +/* Calculation function for simple counter. */ +static int tl_calc_count(struct adf_telemetry *telemetry, + const struct adf_tl_dbg_counter *ctr, + struct adf_tl_dbg_aggr_values *vals) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(telemetry->accel_dev); + u64 *hist_vals; + int sample_cnt; + int ret = 0; + + hist_vals = kmalloc_array(tl_data->num_hbuff, sizeof(*hist_vals), + GFP_KERNEL); + if (!hist_vals) + return -ENOMEM; + + memset(vals, 0, sizeof(*vals)); + sample_cnt = tl_collect_values_u32(telemetry, ctr->offset1, hist_vals); + if (!sample_cnt) + goto out_free_hist_vals; + + vals->curr = hist_vals[sample_cnt - 1]; + vals->min = min_array(hist_vals, sample_cnt); + vals->max = max_array(hist_vals, sample_cnt); + vals->avg = avg_array(hist_vals, sample_cnt); + +out_free_hist_vals: + kfree(hist_vals); + return ret; +} + +/* Convert CPP bus cycles to ns. */ +static int tl_cycles_to_ns(struct adf_telemetry *telemetry, + const struct adf_tl_dbg_counter *ctr, + struct adf_tl_dbg_aggr_values *vals) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(telemetry->accel_dev); + u8 cpp_ns_per_cycle = tl_data->cpp_ns_per_cycle; + int ret; + + ret = tl_calc_count(telemetry, ctr, vals); + if (ret) + return ret; + + vals->curr *= cpp_ns_per_cycle; + vals->min *= cpp_ns_per_cycle; + vals->max *= cpp_ns_per_cycle; + vals->avg *= cpp_ns_per_cycle; + + return 0; +} + +/* + * Compute latency cumulative average with division of accumulated value + * by sample count. Returned value is in ns. + */ +static int tl_lat_acc_avg(struct adf_telemetry *telemetry, + const struct adf_tl_dbg_counter *ctr, + struct adf_tl_dbg_aggr_values *vals) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(telemetry->accel_dev); + u8 cpp_ns_per_cycle = tl_data->cpp_ns_per_cycle; + u8 num_hbuff = tl_data->num_hbuff; + int sample_cnt, i; + u64 *hist_vals; + u64 *hist_cnt; + int ret = 0; + + hist_vals = kmalloc_array(num_hbuff, sizeof(*hist_vals), GFP_KERNEL); + if (!hist_vals) + return -ENOMEM; + + hist_cnt = kmalloc_array(num_hbuff, sizeof(*hist_cnt), GFP_KERNEL); + if (!hist_cnt) { + ret = -ENOMEM; + goto out_free_hist_vals; + } + + memset(vals, 0, sizeof(*vals)); + sample_cnt = tl_collect_values_u64(telemetry, ctr->offset1, hist_vals); + if (!sample_cnt) + goto out_free_hist_cnt; + + tl_collect_values_u32(telemetry, ctr->offset2, hist_cnt); + + for (i = 0; i < sample_cnt; i++) { + /* Avoid division by 0 if count is 0. */ + if (hist_cnt[i]) + hist_vals[i] = div_u64(hist_vals[i] * cpp_ns_per_cycle, + hist_cnt[i]); + else + hist_vals[i] = 0; + } + + vals->curr = hist_vals[sample_cnt - 1]; + vals->min = min_array(hist_vals, sample_cnt); + vals->max = max_array(hist_vals, sample_cnt); + vals->avg = avg_array(hist_vals, sample_cnt); + +out_free_hist_cnt: + kfree(hist_cnt); +out_free_hist_vals: + kfree(hist_vals); + return ret; +} + +/* Convert HW raw bandwidth units to Mbps. */ +static int tl_bw_hw_units_to_mbps(struct adf_telemetry *telemetry, + const struct adf_tl_dbg_counter *ctr, + struct adf_tl_dbg_aggr_values *vals) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(telemetry->accel_dev); + u16 bw_hw_2_bits = tl_data->bw_units_to_bytes * BITS_PER_BYTE; + u64 *hist_vals; + int sample_cnt; + int ret = 0; + + hist_vals = kmalloc_array(tl_data->num_hbuff, sizeof(*hist_vals), + GFP_KERNEL); + if (!hist_vals) + return -ENOMEM; + + memset(vals, 0, sizeof(*vals)); + sample_cnt = tl_collect_values_u32(telemetry, ctr->offset1, hist_vals); + if (!sample_cnt) + goto out_free_hist_vals; + + vals->curr = div_u64(hist_vals[sample_cnt - 1] * bw_hw_2_bits, MEGA); + vals->min = div_u64(min_array(hist_vals, sample_cnt) * bw_hw_2_bits, MEGA); + vals->max = div_u64(max_array(hist_vals, sample_cnt) * bw_hw_2_bits, MEGA); + vals->avg = div_u64(avg_array(hist_vals, sample_cnt) * bw_hw_2_bits, MEGA); + +out_free_hist_vals: + kfree(hist_vals); + return ret; +} + +static void tl_seq_printf_counter(struct adf_telemetry *telemetry, + struct seq_file *s, const char *name, + struct adf_tl_dbg_aggr_values *vals) +{ + seq_printf(s, "%-*s", TL_KEY_MIN_PADDING, name); + seq_printf(s, "%*llu", TL_VALUE_MIN_PADDING, vals->curr); + if (atomic_read(&telemetry->state) > 1) { + seq_printf(s, "%*llu", TL_VALUE_MIN_PADDING, vals->min); + seq_printf(s, "%*llu", TL_VALUE_MIN_PADDING, vals->max); + seq_printf(s, "%*llu", TL_VALUE_MIN_PADDING, vals->avg); + } + seq_puts(s, "\n"); +} + +static int tl_calc_and_print_counter(struct adf_telemetry *telemetry, + struct seq_file *s, + const struct adf_tl_dbg_counter *ctr, + const char *name) +{ + const char *counter_name = name ? name : ctr->name; + enum adf_tl_counter_type type = ctr->type; + struct adf_tl_dbg_aggr_values vals; + int ret; + + switch (type) { + case ADF_TL_SIMPLE_COUNT: + ret = tl_calc_count(telemetry, ctr, &vals); + break; + case ADF_TL_COUNTER_NS: + ret = tl_cycles_to_ns(telemetry, ctr, &vals); + break; + case ADF_TL_COUNTER_NS_AVG: + ret = tl_lat_acc_avg(telemetry, ctr, &vals); + break; + case ADF_TL_COUNTER_MBPS: + ret = tl_bw_hw_units_to_mbps(telemetry, ctr, &vals); + break; + default: + return -EINVAL; + } + + if (ret) + return ret; + + tl_seq_printf_counter(telemetry, s, counter_name, &vals); + + return 0; +} + +static int tl_print_sl_counter(struct adf_telemetry *telemetry, + const struct adf_tl_dbg_counter *ctr, + struct seq_file *s, u8 cnt_id) +{ + size_t sl_regs_sz = GET_TL_DATA(telemetry->accel_dev).slice_reg_sz; + struct adf_tl_dbg_counter slice_ctr; + size_t offset_inc = cnt_id * sl_regs_sz; + char cnt_name[MAX_COUNT_NAME_SIZE]; + + snprintf(cnt_name, MAX_COUNT_NAME_SIZE, "%s%d", ctr->name, cnt_id); + slice_ctr = *ctr; + slice_ctr.offset1 += offset_inc; + + return tl_calc_and_print_counter(telemetry, s, &slice_ctr, cnt_name); +} + +static int tl_calc_and_print_sl_counters(struct adf_accel_dev *accel_dev, + struct seq_file *s, u8 cnt_type, u8 cnt_id) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(accel_dev); + struct adf_telemetry *telemetry = accel_dev->telemetry; + const struct adf_tl_dbg_counter *sl_tl_util_counters; + const struct adf_tl_dbg_counter *sl_tl_exec_counters; + const struct adf_tl_dbg_counter *ctr; + int ret; + + sl_tl_util_counters = tl_data->sl_util_counters; + sl_tl_exec_counters = tl_data->sl_exec_counters; + + ctr = &sl_tl_util_counters[cnt_type]; + + ret = tl_print_sl_counter(telemetry, ctr, s, cnt_id); + if (ret) { + dev_notice(&GET_DEV(accel_dev), + "invalid slice utilization counter type\n"); + return ret; + } + + ctr = &sl_tl_exec_counters[cnt_type]; + + ret = tl_print_sl_counter(telemetry, ctr, s, cnt_id); + if (ret) { + dev_notice(&GET_DEV(accel_dev), + "invalid slice execution counter type\n"); + return ret; + } + + return 0; +} + +static int tl_print_cmdq_counter(struct adf_telemetry *telemetry, + const struct adf_tl_dbg_counter *ctr, + struct seq_file *s, u8 cnt_id, u8 counter) +{ + size_t cmdq_regs_sz = GET_TL_DATA(telemetry->accel_dev).cmdq_reg_sz; + size_t offset_inc = cnt_id * cmdq_regs_sz; + struct adf_tl_dbg_counter slice_ctr; + char cnt_name[MAX_COUNT_NAME_SIZE]; + + slice_ctr = *(ctr + counter); + slice_ctr.offset1 += offset_inc; + snprintf(cnt_name, MAX_COUNT_NAME_SIZE, "%s%d", slice_ctr.name, cnt_id); + + return tl_calc_and_print_counter(telemetry, s, &slice_ctr, cnt_name); +} + +static int tl_calc_and_print_cmdq_counters(struct adf_accel_dev *accel_dev, + struct seq_file *s, u8 cnt_type, + u8 cnt_id) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(accel_dev); + struct adf_telemetry *telemetry = accel_dev->telemetry; + const struct adf_tl_dbg_counter **cmdq_tl_counters; + const struct adf_tl_dbg_counter *ctr; + u8 counter; + int ret; + + cmdq_tl_counters = tl_data->cmdq_counters; + ctr = cmdq_tl_counters[cnt_type]; + + for (counter = 0; counter < tl_data->num_cmdq_counters; counter++) { + ret = tl_print_cmdq_counter(telemetry, ctr, s, cnt_id, counter); + if (ret) { + dev_notice(&GET_DEV(accel_dev), + "invalid slice utilization counter type\n"); + return ret; + } + } + + return 0; +} + +static void tl_print_msg_cnt(struct seq_file *s, u32 msg_cnt) +{ + seq_printf(s, "%-*s", TL_KEY_MIN_PADDING, SNAPSHOT_CNT_MSG); + seq_printf(s, "%*u\n", TL_VALUE_MIN_PADDING, msg_cnt); +} + +static int tl_print_dev_data(struct adf_accel_dev *accel_dev, + struct seq_file *s) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(accel_dev); + struct adf_telemetry *telemetry = accel_dev->telemetry; + const struct adf_tl_dbg_counter *dev_tl_counters; + u8 num_dev_counters = tl_data->num_dev_counters; + u8 *cmdq_cnt = (u8 *)&telemetry->cmdq_cnt; + u8 *sl_cnt = (u8 *)&telemetry->slice_cnt; + const struct adf_tl_dbg_counter *ctr; + unsigned int i; + int ret; + u8 j; + + if (!atomic_read(&telemetry->state)) { + dev_info(&GET_DEV(accel_dev), "not enabled\n"); + return -EPERM; + } + + dev_tl_counters = tl_data->dev_counters; + + tl_print_msg_cnt(s, telemetry->msg_cnt); + + /* Print device level telemetry. */ + for (i = 0; i < num_dev_counters; i++) { + ctr = &dev_tl_counters[i]; + ret = tl_calc_and_print_counter(telemetry, s, ctr, NULL); + if (ret) { + dev_notice(&GET_DEV(accel_dev), + "invalid counter type\n"); + return ret; + } + } + + /* Print per slice telemetry. */ + for (i = 0; i < ADF_TL_SL_CNT_COUNT; i++) { + for (j = 0; j < sl_cnt[i]; j++) { + ret = tl_calc_and_print_sl_counters(accel_dev, s, i, j); + if (ret) + return ret; + } + } + + /* Print per command queue telemetry. */ + for (i = 0; i < ADF_TL_SL_CNT_COUNT; i++) { + for (j = 0; j < cmdq_cnt[i]; j++) { + ret = tl_calc_and_print_cmdq_counters(accel_dev, s, i, j); + if (ret) + return ret; + } + } + + return 0; +} + +static int tl_dev_data_show(struct seq_file *s, void *unused) +{ + struct adf_accel_dev *accel_dev = s->private; + + if (!accel_dev) + return -EINVAL; + + return tl_print_dev_data(accel_dev, s); +} +DEFINE_SHOW_ATTRIBUTE(tl_dev_data); + +static int tl_control_show(struct seq_file *s, void *unused) +{ + struct adf_accel_dev *accel_dev = s->private; + + if (!accel_dev) + return -EINVAL; + + seq_printf(s, "%d\n", atomic_read(&accel_dev->telemetry->state)); + + return 0; +} + +static ssize_t tl_control_write(struct file *file, const char __user *userbuf, + size_t count, loff_t *ppos) +{ + struct seq_file *seq_f = file->private_data; + struct adf_accel_dev *accel_dev; + struct adf_telemetry *telemetry; + struct adf_tl_hw_data *tl_data; + struct device *dev; + u32 input; + int ret; + + accel_dev = seq_f->private; + if (!accel_dev) + return -EINVAL; + + tl_data = &GET_TL_DATA(accel_dev); + telemetry = accel_dev->telemetry; + dev = &GET_DEV(accel_dev); + + mutex_lock(&telemetry->wr_lock); + + ret = kstrtou32_from_user(userbuf, count, 10, &input); + if (ret) + goto unlock_and_exit; + + if (input > tl_data->num_hbuff) { + dev_info(dev, "invalid control input\n"); + ret = -EINVAL; + goto unlock_and_exit; + } + + /* If input is 0, just stop telemetry. */ + if (!input) { + ret = adf_tl_halt(accel_dev); + if (!ret) + ret = count; + + goto unlock_and_exit; + } + + /* If TL is already enabled, stop it. */ + if (atomic_read(&telemetry->state)) { + dev_info(dev, "already enabled, restarting.\n"); + ret = adf_tl_halt(accel_dev); + if (ret) + goto unlock_and_exit; + } + + ret = adf_tl_run(accel_dev, input); + if (ret) + goto unlock_and_exit; + + ret = count; + +unlock_and_exit: + mutex_unlock(&telemetry->wr_lock); + return ret; +} +DEFINE_SHOW_STORE_ATTRIBUTE(tl_control); + +static int adf_tl_dbg_change_rp_index(struct adf_accel_dev *accel_dev, + unsigned int new_rp_num, + unsigned int rp_regs_index) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + struct adf_telemetry *telemetry = accel_dev->telemetry; + struct device *dev = &GET_DEV(accel_dev); + unsigned int i; + u8 curr_state; + int ret; + + if (new_rp_num >= hw_data->num_rps) { + dev_info(dev, "invalid Ring Pair number selected\n"); + return -EINVAL; + } + + for (i = 0; i < hw_data->tl_data.max_rp; i++) { + if (telemetry->rp_num_indexes[i] == new_rp_num) { + dev_info(dev, "RP nr: %d is already selected in slot rp_%c_data\n", + new_rp_num, ADF_TL_DBG_RP_ALPHA_INDEX(i)); + return 0; + } + } + + dev_dbg(dev, "selecting RP nr %u into slot rp_%c_data\n", + new_rp_num, ADF_TL_DBG_RP_ALPHA_INDEX(rp_regs_index)); + + curr_state = atomic_read(&telemetry->state); + + if (curr_state) { + ret = adf_tl_halt(accel_dev); + if (ret) + return ret; + + telemetry->rp_num_indexes[rp_regs_index] = new_rp_num; + + ret = adf_tl_run(accel_dev, curr_state); + if (ret) + return ret; + } else { + telemetry->rp_num_indexes[rp_regs_index] = new_rp_num; + } + + return 0; +} + +static void tl_print_rp_srv(struct adf_accel_dev *accel_dev, struct seq_file *s, + u8 rp_idx) +{ + u32 banks_per_vf = GET_HW_DATA(accel_dev)->num_banks_per_vf; + enum adf_cfg_service_type svc; + + seq_printf(s, "%-*s", TL_KEY_MIN_PADDING, RP_SERVICE_TYPE); + + svc = GET_SRV_TYPE(accel_dev, rp_idx % banks_per_vf); + switch (svc) { + case COMP: + seq_printf(s, "%*s\n", TL_VALUE_MIN_PADDING, ADF_CFG_DC); + break; + case SYM: + seq_printf(s, "%*s\n", TL_VALUE_MIN_PADDING, ADF_CFG_SYM); + break; + case ASYM: + seq_printf(s, "%*s\n", TL_VALUE_MIN_PADDING, ADF_CFG_ASYM); + break; + case DECOMP: + seq_printf(s, "%*s\n", TL_VALUE_MIN_PADDING, ADF_CFG_DECOMP); + break; + default: + seq_printf(s, "%*s\n", TL_VALUE_MIN_PADDING, TL_RP_SRV_UNKNOWN); + break; + } +} + +static int tl_print_rp_data(struct adf_accel_dev *accel_dev, struct seq_file *s, + u8 rp_regs_index) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(accel_dev); + struct adf_telemetry *telemetry = accel_dev->telemetry; + const struct adf_tl_dbg_counter *rp_tl_counters; + u8 num_rp_counters = tl_data->num_rp_counters; + size_t rp_regs_sz = tl_data->rp_reg_sz; + struct adf_tl_dbg_counter ctr; + unsigned int i; + u8 rp_idx; + int ret; + + if (!atomic_read(&telemetry->state)) { + dev_info(&GET_DEV(accel_dev), "not enabled\n"); + return -EPERM; + } + + rp_tl_counters = tl_data->rp_counters; + rp_idx = telemetry->rp_num_indexes[rp_regs_index]; + + if (rp_idx == ADF_TL_RP_REGS_DISABLED) { + dev_info(&GET_DEV(accel_dev), "no RP number selected in rp_%c_data\n", + ADF_TL_DBG_RP_ALPHA_INDEX(rp_regs_index)); + return -EPERM; + } + + tl_print_msg_cnt(s, telemetry->msg_cnt); + seq_printf(s, "%-*s", TL_KEY_MIN_PADDING, RP_NUM_INDEX); + seq_printf(s, "%*d\n", TL_VALUE_MIN_PADDING, rp_idx); + tl_print_rp_srv(accel_dev, s, rp_idx); + + for (i = 0; i < num_rp_counters; i++) { + ctr = rp_tl_counters[i]; + ctr.offset1 += rp_regs_sz * rp_regs_index; + ctr.offset2 += rp_regs_sz * rp_regs_index; + ret = tl_calc_and_print_counter(telemetry, s, &ctr, NULL); + if (ret) { + dev_dbg(&GET_DEV(accel_dev), + "invalid RP counter type\n"); + return ret; + } + } + + return 0; +} + +static int tl_rp_data_show(struct seq_file *s, void *unused) +{ + struct adf_accel_dev *accel_dev = s->private; + u8 rp_regs_index; + + if (!accel_dev) + return -EINVAL; + + rp_regs_index = debugfs_get_aux_num(s->file); + + return tl_print_rp_data(accel_dev, s, rp_regs_index); +} + +static ssize_t tl_rp_data_write(struct file *file, const char __user *userbuf, + size_t count, loff_t *ppos) +{ + struct seq_file *seq_f = file->private_data; + struct adf_accel_dev *accel_dev; + struct adf_telemetry *telemetry; + unsigned int new_rp_num; + u8 rp_regs_index; + int ret; + + accel_dev = seq_f->private; + if (!accel_dev) + return -EINVAL; + + telemetry = accel_dev->telemetry; + + mutex_lock(&telemetry->wr_lock); + + rp_regs_index = debugfs_get_aux_num(file); + + ret = kstrtou32_from_user(userbuf, count, 10, &new_rp_num); + if (ret) + goto unlock_and_exit; + + ret = adf_tl_dbg_change_rp_index(accel_dev, new_rp_num, rp_regs_index); + if (ret) + goto unlock_and_exit; + + ret = count; + +unlock_and_exit: + mutex_unlock(&telemetry->wr_lock); + return ret; +} +DEFINE_SHOW_STORE_ATTRIBUTE(tl_rp_data); + +void adf_tl_dbgfs_add(struct adf_accel_dev *accel_dev) +{ + struct adf_telemetry *telemetry = accel_dev->telemetry; + struct dentry *parent = accel_dev->debugfs_dir; + u8 max_rp = GET_TL_DATA(accel_dev).max_rp; + char name[ADF_TL_RP_REGS_FNAME_SIZE]; + struct dentry *dir; + unsigned int i; + + if (!telemetry) + return; + + dir = debugfs_create_dir("telemetry", parent); + accel_dev->telemetry->dbg_dir = dir; + debugfs_create_file("device_data", 0444, dir, accel_dev, &tl_dev_data_fops); + debugfs_create_file("control", 0644, dir, accel_dev, &tl_control_fops); + + for (i = 0; i < max_rp; i++) { + snprintf(name, sizeof(name), ADF_TL_RP_REGS_FNAME, + ADF_TL_DBG_RP_ALPHA_INDEX(i)); + debugfs_create_file_aux_num(name, 0644, dir, accel_dev, i, + &tl_rp_data_fops); + } +} + +void adf_tl_dbgfs_rm(struct adf_accel_dev *accel_dev) +{ + struct adf_telemetry *telemetry = accel_dev->telemetry; + struct dentry *dbg_dir; + + if (!telemetry) + return; + + dbg_dir = telemetry->dbg_dir; + + debugfs_remove_recursive(dbg_dir); + + if (atomic_read(&telemetry->state)) + adf_tl_halt(accel_dev); +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.h b/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.h new file mode 100644 index 000000000..97c5eeaa1 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.h @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2023 Intel Corporation. */ +#ifndef ADF_TL_DEBUGFS_H +#define ADF_TL_DEBUGFS_H + +#include + +struct adf_accel_dev; + +#define MAX_COUNT_NAME_SIZE 32 +#define SNAPSHOT_CNT_MSG "sample_cnt" +#define RP_NUM_INDEX "rp_num" +#define PCI_TRANS_CNT_NAME "pci_trans_cnt" +#define MAX_RD_LAT_NAME "max_rd_lat" +#define RD_LAT_ACC_NAME "rd_lat_acc_avg" +#define MAX_LAT_NAME "max_gp_lat" +#define LAT_ACC_NAME "gp_lat_acc_avg" +#define BW_IN_NAME "bw_in" +#define BW_OUT_NAME "bw_out" +#define RE_ACC_NAME "re_acc_avg" +#define PAGE_REQ_LAT_NAME "at_page_req_lat_avg" +#define AT_TRANS_LAT_NAME "at_trans_lat_avg" +#define AT_MAX_UTLB_USED_NAME "at_max_tlb_used" +#define AT_GLOB_DTLB_HIT_NAME "at_glob_devtlb_hit" +#define AT_GLOB_DTLB_MISS_NAME "at_glob_devtlb_miss" +#define AT_PAYLD_DTLB_HIT_NAME "tl_at_payld_devtlb_hit" +#define AT_PAYLD_DTLB_MISS_NAME "tl_at_payld_devtlb_miss" +#define RP_SERVICE_TYPE "service_type" + +#define ADF_TL_DBG_RP_ALPHA_INDEX(index) ((index) + 'A') +#define ADF_TL_DBG_RP_INDEX_ALPHA(alpha) ((alpha) - 'A') + +#define ADF_TL_RP_REGS_FNAME "rp_%c_data" +#define ADF_TL_RP_REGS_FNAME_SIZE 16 + +#define ADF_TL_DATA_REG_OFF(reg, qat_gen) \ + offsetof(struct adf_##qat_gen##_tl_layout, reg) + +#define ADF_TL_DEV_REG_OFF(reg, qat_gen) \ + (ADF_TL_DATA_REG_OFF(tl_device_data_regs, qat_gen) + \ + offsetof(struct adf_##qat_gen##_tl_device_data_regs, reg)) + +#define ADF_TL_SLICE_REG_OFF(slice, reg, qat_gen) \ + (ADF_TL_DEV_REG_OFF(slice##_slices[0], qat_gen) + \ + offsetof(struct adf_##qat_gen##_tl_slice_data_regs, reg)) + +#define ADF_TL_CMDQ_REG_OFF(slice, reg, qat_gen) \ + (ADF_TL_DEV_REG_OFF(slice##_cmdq[0], qat_gen) + \ + offsetof(struct adf_##qat_gen##_tl_cmdq_data_regs, reg)) + +#define ADF_TL_RP_REG_OFF(reg, qat_gen) \ + (ADF_TL_DATA_REG_OFF(tl_ring_pairs_data_regs[0], qat_gen) + \ + offsetof(struct adf_##qat_gen##_tl_ring_pair_data_regs, reg)) + +/** + * enum adf_tl_counter_type - telemetry counter types + * @ADF_TL_COUNTER_UNSUPPORTED: unsupported counter + * @ADF_TL_SIMPLE_COUNT: simple counter + * @ADF_TL_COUNTER_NS: latency counter, value in ns + * @ADF_TL_COUNTER_NS_AVG: accumulated average latency counter, value in ns + * @ADF_TL_COUNTER_MBPS: bandwidth, value in MBps + */ +enum adf_tl_counter_type { + ADF_TL_COUNTER_UNSUPPORTED, + ADF_TL_SIMPLE_COUNT, + ADF_TL_COUNTER_NS, + ADF_TL_COUNTER_NS_AVG, + ADF_TL_COUNTER_MBPS, +}; + +/** + * struct adf_tl_dbg_counter - telemetry counter definition + * @name: name of the counter as printed in the report + * @adf_tl_counter_type: type of the counter + * @offset1: offset of 1st register + * @offset2: offset of 2nd optional register + */ +struct adf_tl_dbg_counter { + const char *name; + enum adf_tl_counter_type type; + size_t offset1; + size_t offset2; +}; + +#define ADF_TL_COUNTER(_name, _type, _offset) \ +{ .name = _name, \ + .type = _type, \ + .offset1 = _offset \ +} + +#define ADF_TL_COUNTER_LATENCY(_name, _type, _offset1, _offset2) \ +{ .name = _name, \ + .type = _type, \ + .offset1 = _offset1, \ + .offset2 = _offset2 \ +} + +/* Telemetry counter aggregated values. */ +struct adf_tl_dbg_aggr_values { + u64 curr; + u64 min; + u64 max; + u64 avg; +}; + +/** + * adf_tl_dbgfs_add() - Add telemetry's debug fs entries. + * @accel_dev: Pointer to acceleration device. + * + * Creates telemetry's debug fs folder and attributes in QAT debug fs root. + */ +void adf_tl_dbgfs_add(struct adf_accel_dev *accel_dev); + +/** + * adf_tl_dbgfs_rm() - Remove telemetry's debug fs entries. + * @accel_dev: Pointer to acceleration device. + * + * Removes telemetry's debug fs folder and attributes from QAT debug fs root. + */ +void adf_tl_dbgfs_rm(struct adf_accel_dev *accel_dev); + +#endif /* ADF_TL_DEBUGFS_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_transport.c b/drivers/crypto/intel/qat/qat_common/adf_transport.c new file mode 100644 index 000000000..1efdf4649 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_transport.c @@ -0,0 +1,575 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#include +#include +#include "adf_accel_devices.h" +#include "adf_transport_internal.h" +#include "adf_transport_access_macros.h" +#include "adf_cfg.h" +#include "adf_common_drv.h" + +#define ADF_MAX_RING_THRESHOLD 80 +#define ADF_PERCENT(tot, percent) (((tot) * (percent)) / 100) + +static inline u32 adf_modulo(u32 data, u32 shift) +{ + u32 div = data >> shift; + u32 mult = div << shift; + + return data - mult; +} + +static inline int adf_check_ring_alignment(u64 addr, u64 size) +{ + if (((size - 1) & addr) != 0) + return -EFAULT; + return 0; +} + +static int adf_verify_ring_size(u32 msg_size, u32 msg_num) +{ + int i = ADF_MIN_RING_SIZE; + + for (; i <= ADF_MAX_RING_SIZE; i++) + if ((msg_size * msg_num) == ADF_SIZE_TO_RING_SIZE_IN_BYTES(i)) + return i; + + return ADF_DEFAULT_RING_SIZE; +} + +static int adf_reserve_ring(struct adf_etr_bank_data *bank, u32 ring) +{ + spin_lock(&bank->lock); + if (bank->ring_mask & (1 << ring)) { + spin_unlock(&bank->lock); + return -EFAULT; + } + bank->ring_mask |= (1 << ring); + spin_unlock(&bank->lock); + return 0; +} + +static void adf_unreserve_ring(struct adf_etr_bank_data *bank, u32 ring) +{ + spin_lock(&bank->lock); + bank->ring_mask &= ~(1 << ring); + spin_unlock(&bank->lock); +} + +static void adf_enable_ring_irq(struct adf_etr_bank_data *bank, u32 ring) +{ + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); + + spin_lock_bh(&bank->lock); + bank->irq_mask |= (1 << ring); + spin_unlock_bh(&bank->lock); + csr_ops->write_csr_int_col_en(bank->csr_addr, bank->bank_number, + bank->irq_mask); + csr_ops->write_csr_int_col_ctl(bank->csr_addr, bank->bank_number, + bank->irq_coalesc_timer); +} + +static void adf_disable_ring_irq(struct adf_etr_bank_data *bank, u32 ring) +{ + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); + + spin_lock_bh(&bank->lock); + bank->irq_mask &= ~(1 << ring); + spin_unlock_bh(&bank->lock); + csr_ops->write_csr_int_col_en(bank->csr_addr, bank->bank_number, + bank->irq_mask); +} + +bool adf_ring_nearly_full(struct adf_etr_ring_data *ring) +{ + return atomic_read(ring->inflights) > ring->threshold; +} + +int adf_send_message(struct adf_etr_ring_data *ring, u32 *msg) +{ + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); + + if (atomic_add_return(1, ring->inflights) > + ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size)) { + atomic_dec(ring->inflights); + return -EAGAIN; + } + spin_lock_bh(&ring->lock); + memcpy((void *)((uintptr_t)ring->base_addr + ring->tail), msg, + ADF_MSG_SIZE_TO_BYTES(ring->msg_size)); + + ring->tail = adf_modulo(ring->tail + + ADF_MSG_SIZE_TO_BYTES(ring->msg_size), + ADF_RING_SIZE_MODULO(ring->ring_size)); + csr_ops->write_csr_ring_tail(ring->bank->csr_addr, + ring->bank->bank_number, ring->ring_number, + ring->tail); + spin_unlock_bh(&ring->lock); + + return 0; +} + +static int adf_handle_response(struct adf_etr_ring_data *ring) +{ + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); + u32 msg_counter = 0; + u32 *msg = (u32 *)((uintptr_t)ring->base_addr + ring->head); + + while (*msg != ADF_RING_EMPTY_SIG) { + ring->callback((u32 *)msg); + atomic_dec(ring->inflights); + *msg = ADF_RING_EMPTY_SIG; + ring->head = adf_modulo(ring->head + + ADF_MSG_SIZE_TO_BYTES(ring->msg_size), + ADF_RING_SIZE_MODULO(ring->ring_size)); + msg_counter++; + msg = (u32 *)((uintptr_t)ring->base_addr + ring->head); + } + if (msg_counter > 0) { + csr_ops->write_csr_ring_head(ring->bank->csr_addr, + ring->bank->bank_number, + ring->ring_number, ring->head); + } + return 0; +} + +static void adf_configure_tx_ring(struct adf_etr_ring_data *ring) +{ + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); + u32 ring_config = BUILD_RING_CONFIG(ring->ring_size); + + csr_ops->write_csr_ring_config(ring->bank->csr_addr, + ring->bank->bank_number, + ring->ring_number, ring_config); + +} + +static void adf_configure_rx_ring(struct adf_etr_ring_data *ring) +{ + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); + u32 ring_config = + BUILD_RESP_RING_CONFIG(ring->ring_size, + ADF_RING_NEAR_WATERMARK_512, + ADF_RING_NEAR_WATERMARK_0); + + csr_ops->write_csr_ring_config(ring->bank->csr_addr, + ring->bank->bank_number, + ring->ring_number, ring_config); +} + +static int adf_init_ring(struct adf_etr_ring_data *ring) +{ + struct adf_etr_bank_data *bank = ring->bank; + struct adf_accel_dev *accel_dev = bank->accel_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); + u64 ring_base; + u32 ring_size_bytes = + ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size); + + ring_size_bytes = ADF_RING_SIZE_BYTES_MIN(ring_size_bytes); + ring->base_addr = dma_alloc_coherent(&GET_DEV(accel_dev), + ring_size_bytes, &ring->dma_addr, + GFP_KERNEL); + if (!ring->base_addr) + return -ENOMEM; + + memset(ring->base_addr, 0x7F, ring_size_bytes); + /* The base_addr has to be aligned to the size of the buffer */ + if (adf_check_ring_alignment(ring->dma_addr, ring_size_bytes)) { + dev_err(&GET_DEV(accel_dev), "Ring address not aligned\n"); + dma_free_coherent(&GET_DEV(accel_dev), ring_size_bytes, + ring->base_addr, ring->dma_addr); + ring->base_addr = NULL; + return -EFAULT; + } + + if (hw_data->tx_rings_mask & (1 << ring->ring_number)) + adf_configure_tx_ring(ring); + + else + adf_configure_rx_ring(ring); + + ring_base = csr_ops->build_csr_ring_base_addr(ring->dma_addr, + ring->ring_size); + + csr_ops->write_csr_ring_base(ring->bank->csr_addr, + ring->bank->bank_number, ring->ring_number, + ring_base); + spin_lock_init(&ring->lock); + return 0; +} + +static void adf_cleanup_ring(struct adf_etr_ring_data *ring) +{ + u32 ring_size_bytes = + ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size); + ring_size_bytes = ADF_RING_SIZE_BYTES_MIN(ring_size_bytes); + + if (ring->base_addr) { + memset(ring->base_addr, 0x7F, ring_size_bytes); + dma_free_coherent(&GET_DEV(ring->bank->accel_dev), + ring_size_bytes, ring->base_addr, + ring->dma_addr); + } +} + +int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section, + u32 bank_num, u32 num_msgs, + u32 msg_size, const char *ring_name, + adf_callback_fn callback, int poll_mode, + struct adf_etr_ring_data **ring_ptr) +{ + struct adf_etr_data *transport_data = accel_dev->transport; + u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(accel_dev); + struct adf_etr_bank_data *bank; + struct adf_etr_ring_data *ring; + char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + int max_inflights; + u32 ring_num; + int ret; + + if (bank_num >= GET_MAX_BANKS(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Invalid bank number\n"); + return -EFAULT; + } + if (msg_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) { + dev_err(&GET_DEV(accel_dev), "Invalid msg size\n"); + return -EFAULT; + } + if (ADF_MAX_INFLIGHTS(adf_verify_ring_size(msg_size, num_msgs), + ADF_BYTES_TO_MSG_SIZE(msg_size)) < 2) { + dev_err(&GET_DEV(accel_dev), + "Invalid ring size for given msg size\n"); + return -EFAULT; + } + if (adf_cfg_get_param_value(accel_dev, section, ring_name, val)) { + dev_err(&GET_DEV(accel_dev), "Section %s, no such entry : %s\n", + section, ring_name); + return -EFAULT; + } + if (kstrtouint(val, 10, &ring_num)) { + dev_err(&GET_DEV(accel_dev), "Can't get ring number\n"); + return -EFAULT; + } + if (ring_num >= num_rings_per_bank) { + dev_err(&GET_DEV(accel_dev), "Invalid ring number\n"); + return -EFAULT; + } + + ring_num = array_index_nospec(ring_num, num_rings_per_bank); + bank = &transport_data->banks[bank_num]; + if (adf_reserve_ring(bank, ring_num)) { + dev_err(&GET_DEV(accel_dev), "Ring %d, %s already exists.\n", + ring_num, ring_name); + return -EFAULT; + } + ring = &bank->rings[ring_num]; + ring->ring_number = ring_num; + ring->bank = bank; + ring->callback = callback; + ring->msg_size = ADF_BYTES_TO_MSG_SIZE(msg_size); + ring->ring_size = adf_verify_ring_size(msg_size, num_msgs); + ring->head = 0; + ring->tail = 0; + max_inflights = ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size); + ring->threshold = ADF_PERCENT(max_inflights, ADF_MAX_RING_THRESHOLD); + atomic_set(ring->inflights, 0); + ret = adf_init_ring(ring); + if (ret) + goto err; + + /* Enable HW arbitration for the given ring */ + adf_update_ring_arb(ring); + + if (adf_ring_debugfs_add(ring, ring_name)) { + dev_err(&GET_DEV(accel_dev), + "Couldn't add ring debugfs entry\n"); + ret = -EFAULT; + goto err; + } + + /* Enable interrupts if needed */ + if (callback && (!poll_mode)) + adf_enable_ring_irq(bank, ring->ring_number); + *ring_ptr = ring; + return 0; +err: + adf_cleanup_ring(ring); + adf_unreserve_ring(bank, ring_num); + adf_update_ring_arb(ring); + return ret; +} + +void adf_remove_ring(struct adf_etr_ring_data *ring) +{ + struct adf_etr_bank_data *bank = ring->bank; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); + + /* Disable interrupts for the given ring */ + adf_disable_ring_irq(bank, ring->ring_number); + + /* Clear PCI config space */ + + csr_ops->write_csr_ring_config(bank->csr_addr, bank->bank_number, + ring->ring_number, 0); + csr_ops->write_csr_ring_base(bank->csr_addr, bank->bank_number, + ring->ring_number, 0); + adf_ring_debugfs_rm(ring); + adf_unreserve_ring(bank, ring->ring_number); + /* Disable HW arbitration for the given ring */ + adf_update_ring_arb(ring); + adf_cleanup_ring(ring); +} + +static void adf_ring_response_handler(struct adf_etr_bank_data *bank) +{ + struct adf_accel_dev *accel_dev = bank->accel_dev; + u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(accel_dev); + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); + unsigned long empty_rings; + int i; + + empty_rings = csr_ops->read_csr_e_stat(bank->csr_addr, + bank->bank_number); + empty_rings = ~empty_rings & bank->irq_mask; + + for_each_set_bit(i, &empty_rings, num_rings_per_bank) + adf_handle_response(&bank->rings[i]); +} + +void adf_response_handler(uintptr_t bank_addr) +{ + struct adf_etr_bank_data *bank = (void *)bank_addr; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); + + /* Handle all the responses and reenable IRQs */ + adf_ring_response_handler(bank); + + csr_ops->write_csr_int_flag_and_col(bank->csr_addr, bank->bank_number, + bank->irq_mask); +} + +static inline int adf_get_cfg_int(struct adf_accel_dev *accel_dev, + const char *section, const char *format, + u32 key, u32 *value) +{ + char key_buf[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + char val_buf[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + + snprintf(key_buf, ADF_CFG_MAX_KEY_LEN_IN_BYTES, format, key); + + if (adf_cfg_get_param_value(accel_dev, section, key_buf, val_buf)) + return -EFAULT; + + if (kstrtouint(val_buf, 10, value)) + return -EFAULT; + return 0; +} + +static void adf_get_coalesc_timer(struct adf_etr_bank_data *bank, + const char *section, + u32 bank_num_in_accel) +{ + if (adf_get_cfg_int(bank->accel_dev, section, + ADF_ETRMGR_COALESCE_TIMER_FORMAT, + bank_num_in_accel, &bank->irq_coalesc_timer)) + bank->irq_coalesc_timer = ADF_COALESCING_DEF_TIME; + + if (ADF_COALESCING_MAX_TIME < bank->irq_coalesc_timer || + ADF_COALESCING_MIN_TIME > bank->irq_coalesc_timer) + bank->irq_coalesc_timer = ADF_COALESCING_DEF_TIME; +} + +static int adf_init_bank(struct adf_accel_dev *accel_dev, + struct adf_etr_bank_data *bank, + u32 bank_num, void __iomem *csr_addr) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u8 num_rings_per_bank = hw_data->num_rings_per_bank; + struct adf_hw_csr_ops *csr_ops = &hw_data->csr_ops; + u32 irq_mask = BIT(num_rings_per_bank) - 1; + struct adf_etr_ring_data *ring; + struct adf_etr_ring_data *tx_ring; + u32 i, coalesc_enabled = 0; + unsigned long ring_mask; + int size; + + memset(bank, 0, sizeof(*bank)); + bank->bank_number = bank_num; + bank->csr_addr = csr_addr; + bank->accel_dev = accel_dev; + spin_lock_init(&bank->lock); + + /* Allocate the rings in the bank */ + size = num_rings_per_bank * sizeof(struct adf_etr_ring_data); + bank->rings = kzalloc_node(size, GFP_KERNEL, + dev_to_node(&GET_DEV(accel_dev))); + if (!bank->rings) + return -ENOMEM; + + /* Enable IRQ coalescing always. This will allow to use + * the optimised flag and coalesc register. + * If it is disabled in the config file just use min time value */ + if ((adf_get_cfg_int(accel_dev, "Accelerator0", + ADF_ETRMGR_COALESCING_ENABLED_FORMAT, bank_num, + &coalesc_enabled) == 0) && coalesc_enabled) + adf_get_coalesc_timer(bank, "Accelerator0", bank_num); + else + bank->irq_coalesc_timer = ADF_COALESCING_MIN_TIME; + + for (i = 0; i < num_rings_per_bank; i++) { + csr_ops->write_csr_ring_config(csr_addr, bank_num, i, 0); + csr_ops->write_csr_ring_base(csr_addr, bank_num, i, 0); + + ring = &bank->rings[i]; + if (hw_data->tx_rings_mask & (1 << i)) { + ring->inflights = + kzalloc_node(sizeof(atomic_t), + GFP_KERNEL, + dev_to_node(&GET_DEV(accel_dev))); + if (!ring->inflights) + goto err; + } else { + if (i < hw_data->tx_rx_gap) { + dev_err(&GET_DEV(accel_dev), + "Invalid tx rings mask config\n"); + goto err; + } + tx_ring = &bank->rings[i - hw_data->tx_rx_gap]; + ring->inflights = tx_ring->inflights; + } + } + if (adf_bank_debugfs_add(bank)) { + dev_err(&GET_DEV(accel_dev), + "Failed to add bank debugfs entry\n"); + goto err; + } + + csr_ops->write_csr_int_flag(csr_addr, bank_num, irq_mask); + csr_ops->write_csr_int_srcsel(csr_addr, bank_num); + + return 0; +err: + ring_mask = hw_data->tx_rings_mask; + for_each_set_bit(i, &ring_mask, num_rings_per_bank) { + ring = &bank->rings[i]; + kfree(ring->inflights); + ring->inflights = NULL; + } + kfree(bank->rings); + return -ENOMEM; +} + +/** + * adf_init_etr_data() - Initialize transport rings for acceleration device + * @accel_dev: Pointer to acceleration device. + * + * Function is the initializes the communications channels (rings) to the + * acceleration device accel_dev. + * To be used by QAT device specific drivers. + * + * Return: 0 on success, error code otherwise. + */ +int adf_init_etr_data(struct adf_accel_dev *accel_dev) +{ + struct adf_etr_data *etr_data; + void __iomem *csr_addr; + u32 size; + u32 num_banks = 0; + int i, ret; + + etr_data = kzalloc_node(sizeof(*etr_data), GFP_KERNEL, + dev_to_node(&GET_DEV(accel_dev))); + if (!etr_data) + return -ENOMEM; + + num_banks = GET_MAX_BANKS(accel_dev); + size = num_banks * sizeof(struct adf_etr_bank_data); + etr_data->banks = kzalloc_node(size, GFP_KERNEL, + dev_to_node(&GET_DEV(accel_dev))); + if (!etr_data->banks) { + ret = -ENOMEM; + goto err_bank; + } + + accel_dev->transport = etr_data; + csr_addr = adf_get_etr_base(accel_dev); + + /* accel_dev->debugfs_dir should always be non-NULL here */ + etr_data->debug = debugfs_create_dir("transport", + accel_dev->debugfs_dir); + + for (i = 0; i < num_banks; i++) { + ret = adf_init_bank(accel_dev, &etr_data->banks[i], i, + csr_addr); + if (ret) + goto err_bank_all; + } + + return 0; + +err_bank_all: + debugfs_remove(etr_data->debug); + kfree(etr_data->banks); +err_bank: + kfree(etr_data); + accel_dev->transport = NULL; + return ret; +} +EXPORT_SYMBOL_GPL(adf_init_etr_data); + +static void cleanup_bank(struct adf_etr_bank_data *bank) +{ + struct adf_accel_dev *accel_dev = bank->accel_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u8 num_rings_per_bank = hw_data->num_rings_per_bank; + u32 i; + + for (i = 0; i < num_rings_per_bank; i++) { + struct adf_etr_ring_data *ring = &bank->rings[i]; + + if (bank->ring_mask & (1 << i)) + adf_cleanup_ring(ring); + + if (hw_data->tx_rings_mask & (1 << i)) + kfree(ring->inflights); + } + kfree(bank->rings); + adf_bank_debugfs_rm(bank); + memset(bank, 0, sizeof(*bank)); +} + +static void adf_cleanup_etr_handles(struct adf_accel_dev *accel_dev) +{ + struct adf_etr_data *etr_data = accel_dev->transport; + u32 i, num_banks = GET_MAX_BANKS(accel_dev); + + for (i = 0; i < num_banks; i++) + cleanup_bank(&etr_data->banks[i]); +} + +/** + * adf_cleanup_etr_data() - Clear transport rings for acceleration device + * @accel_dev: Pointer to acceleration device. + * + * Function is the clears the communications channels (rings) of the + * acceleration device accel_dev. + * To be used by QAT device specific drivers. + * + * Return: void + */ +void adf_cleanup_etr_data(struct adf_accel_dev *accel_dev) +{ + struct adf_etr_data *etr_data = accel_dev->transport; + + if (etr_data) { + adf_cleanup_etr_handles(accel_dev); + debugfs_remove(etr_data->debug); + kfree(etr_data->banks->rings); + kfree(etr_data->banks); + kfree(etr_data); + accel_dev->transport = NULL; + } +} +EXPORT_SYMBOL_GPL(adf_cleanup_etr_data); diff --git a/drivers/crypto/intel/qat/qat_common/adf_transport.h b/drivers/crypto/intel/qat/qat_common/adf_transport.h new file mode 100644 index 000000000..e6ef6f9b7 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_transport.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#ifndef ADF_TRANSPORT_H +#define ADF_TRANSPORT_H + +#include "adf_accel_devices.h" + +struct adf_etr_ring_data; + +typedef void (*adf_callback_fn)(void *resp_msg); + +int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section, + u32 bank_num, u32 num_mgs, u32 msg_size, + const char *ring_name, adf_callback_fn callback, + int poll_mode, struct adf_etr_ring_data **ring_ptr); + +bool adf_ring_nearly_full(struct adf_etr_ring_data *ring); +int adf_send_message(struct adf_etr_ring_data *ring, u32 *msg); +void adf_remove_ring(struct adf_etr_ring_data *ring); +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_transport_access_macros.h b/drivers/crypto/intel/qat/qat_common/adf_transport_access_macros.h new file mode 100644 index 000000000..d3667dbd9 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_transport_access_macros.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#ifndef ADF_TRANSPORT_ACCESS_MACROS_H +#define ADF_TRANSPORT_ACCESS_MACROS_H + +#include "adf_accel_devices.h" +#define ADF_RING_CONFIG_NEAR_FULL_WM 0x0A +#define ADF_RING_CONFIG_NEAR_EMPTY_WM 0x05 +#define ADF_COALESCING_MIN_TIME 0x1FF +#define ADF_COALESCING_MAX_TIME 0xFFFFF +#define ADF_COALESCING_DEF_TIME 0x27FF +#define ADF_RING_NEAR_WATERMARK_512 0x08 +#define ADF_RING_NEAR_WATERMARK_0 0x00 +#define ADF_RING_EMPTY_SIG 0x7F7F7F7F + +/* Valid internal ring size values */ +#define ADF_RING_SIZE_128 0x01 +#define ADF_RING_SIZE_256 0x02 +#define ADF_RING_SIZE_512 0x03 +#define ADF_RING_SIZE_4K 0x06 +#define ADF_RING_SIZE_16K 0x08 +#define ADF_RING_SIZE_4M 0x10 +#define ADF_MIN_RING_SIZE ADF_RING_SIZE_128 +#define ADF_MAX_RING_SIZE ADF_RING_SIZE_4M +#define ADF_DEFAULT_RING_SIZE ADF_RING_SIZE_16K + +/* Valid internal msg size values */ +#define ADF_MSG_SIZE_32 0x01 +#define ADF_MSG_SIZE_64 0x02 +#define ADF_MSG_SIZE_128 0x04 +#define ADF_MIN_MSG_SIZE ADF_MSG_SIZE_32 +#define ADF_MAX_MSG_SIZE ADF_MSG_SIZE_128 + +/* Size to bytes conversion macros for ring and msg size values */ +#define ADF_MSG_SIZE_TO_BYTES(SIZE) (SIZE << 5) +#define ADF_BYTES_TO_MSG_SIZE(SIZE) (SIZE >> 5) +#define ADF_SIZE_TO_RING_SIZE_IN_BYTES(SIZE) ((1 << (SIZE - 1)) << 7) +#define ADF_RING_SIZE_IN_BYTES_TO_SIZE(SIZE) ((1 << (SIZE - 1)) >> 7) + +/* Minimum ring buffer size for memory allocation */ +#define ADF_RING_SIZE_BYTES_MIN(SIZE) \ + ((SIZE < ADF_SIZE_TO_RING_SIZE_IN_BYTES(ADF_RING_SIZE_4K)) ? \ + ADF_SIZE_TO_RING_SIZE_IN_BYTES(ADF_RING_SIZE_4K) : SIZE) +#define ADF_RING_SIZE_MODULO(SIZE) (SIZE + 0x6) +#define ADF_SIZE_TO_POW(SIZE) ((((SIZE & 0x4) >> 1) | ((SIZE & 0x4) >> 2) | \ + SIZE) & ~0x4) +/* Max outstanding requests */ +#define ADF_MAX_INFLIGHTS(RING_SIZE, MSG_SIZE) \ + ((((1 << (RING_SIZE - 1)) << 3) >> ADF_SIZE_TO_POW(MSG_SIZE)) - 1) +#define BUILD_RING_CONFIG(size) \ + ((ADF_RING_NEAR_WATERMARK_0 << ADF_RING_CONFIG_NEAR_FULL_WM) \ + | (ADF_RING_NEAR_WATERMARK_0 << ADF_RING_CONFIG_NEAR_EMPTY_WM) \ + | size) +#define BUILD_RESP_RING_CONFIG(size, watermark_nf, watermark_ne) \ + ((watermark_nf << ADF_RING_CONFIG_NEAR_FULL_WM) \ + | (watermark_ne << ADF_RING_CONFIG_NEAR_EMPTY_WM) \ + | size) +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c b/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c new file mode 100644 index 000000000..fc5d88a2b --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#include +#include +#include +#include +#include "adf_accel_devices.h" +#include "adf_transport_internal.h" +#include "adf_transport_access_macros.h" + +static DEFINE_MUTEX(ring_read_lock); +static DEFINE_MUTEX(bank_read_lock); + +#define ADF_RING_NUM_MSGS(ring) \ + (ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size) / \ + ADF_MSG_SIZE_TO_BYTES(ring->msg_size)) + +static void *adf_ring_start(struct seq_file *sfile, loff_t *pos) +{ + struct adf_etr_ring_data *ring = sfile->private; + unsigned int num_msg = ADF_RING_NUM_MSGS(ring); + loff_t val = *pos; + + mutex_lock(&ring_read_lock); + if (val == 0) + return SEQ_START_TOKEN; + + if (val >= num_msg) + return NULL; + + return ring->base_addr + + (ADF_MSG_SIZE_TO_BYTES(ring->msg_size) * (*pos)++); +} + +static void *adf_ring_next(struct seq_file *sfile, void *v, loff_t *pos) +{ + struct adf_etr_ring_data *ring = sfile->private; + unsigned int num_msg = ADF_RING_NUM_MSGS(ring); + loff_t val = *pos; + + (*pos)++; + + if (val >= num_msg) + return NULL; + + return ring->base_addr + (ADF_MSG_SIZE_TO_BYTES(ring->msg_size) * val); +} + +static int adf_ring_show(struct seq_file *sfile, void *v) +{ + struct adf_etr_ring_data *ring = sfile->private; + struct adf_etr_bank_data *bank = ring->bank; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); + void __iomem *csr = ring->bank->csr_addr; + + if (v == SEQ_START_TOKEN) { + int head, tail, empty; + + head = csr_ops->read_csr_ring_head(csr, bank->bank_number, + ring->ring_number); + tail = csr_ops->read_csr_ring_tail(csr, bank->bank_number, + ring->ring_number); + empty = csr_ops->read_csr_e_stat(csr, bank->bank_number); + + seq_puts(sfile, "------- Ring configuration -------\n"); + seq_printf(sfile, "ring name: %s\n", + ring->ring_debug->ring_name); + seq_printf(sfile, "ring num %d, bank num %d\n", + ring->ring_number, ring->bank->bank_number); + seq_printf(sfile, "head %x, tail %x, empty: %d\n", + head, tail, (empty & 1 << ring->ring_number) + >> ring->ring_number); + seq_printf(sfile, "ring size %lld, msg size %d\n", + (long long)ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size), + ADF_MSG_SIZE_TO_BYTES(ring->msg_size)); + seq_puts(sfile, "----------- Ring data ------------\n"); + return 0; + } + seq_hex_dump(sfile, "", DUMP_PREFIX_ADDRESS, 32, 4, + v, ADF_MSG_SIZE_TO_BYTES(ring->msg_size), false); + return 0; +} + +static void adf_ring_stop(struct seq_file *sfile, void *v) +{ + mutex_unlock(&ring_read_lock); +} + +static const struct seq_operations adf_ring_debug_sops = { + .start = adf_ring_start, + .next = adf_ring_next, + .stop = adf_ring_stop, + .show = adf_ring_show +}; + +DEFINE_SEQ_ATTRIBUTE(adf_ring_debug); + +int adf_ring_debugfs_add(struct adf_etr_ring_data *ring, const char *name) +{ + struct adf_etr_ring_debug_entry *ring_debug; + char entry_name[16]; + + ring_debug = kzalloc_obj(*ring_debug); + if (!ring_debug) + return -ENOMEM; + + strscpy(ring_debug->ring_name, name); + snprintf(entry_name, sizeof(entry_name), "ring_%02d", + ring->ring_number); + + ring_debug->debug = debugfs_create_file(entry_name, S_IRUSR, + ring->bank->bank_debug_dir, + ring, &adf_ring_debug_fops); + ring->ring_debug = ring_debug; + return 0; +} + +void adf_ring_debugfs_rm(struct adf_etr_ring_data *ring) +{ + if (ring->ring_debug) { + debugfs_remove(ring->ring_debug->debug); + kfree(ring->ring_debug); + ring->ring_debug = NULL; + } +} + +static void *adf_bank_start(struct seq_file *sfile, loff_t *pos) +{ + struct adf_etr_bank_data *bank = sfile->private; + u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(bank->accel_dev); + + mutex_lock(&bank_read_lock); + if (*pos == 0) + return SEQ_START_TOKEN; + + if (*pos >= num_rings_per_bank) + return NULL; + + return pos; +} + +static void *adf_bank_next(struct seq_file *sfile, void *v, loff_t *pos) +{ + struct adf_etr_bank_data *bank = sfile->private; + u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(bank->accel_dev); + + if (++(*pos) >= num_rings_per_bank) + return NULL; + + return pos; +} + +static int adf_bank_show(struct seq_file *sfile, void *v) +{ + struct adf_etr_bank_data *bank = sfile->private; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); + + if (v == SEQ_START_TOKEN) { + seq_printf(sfile, "------- Bank %d configuration -------\n", + bank->bank_number); + } else { + int ring_id = *((int *)v) - 1; + struct adf_etr_ring_data *ring = &bank->rings[ring_id]; + void __iomem *csr = bank->csr_addr; + int head, tail, empty; + + if (!(bank->ring_mask & 1 << ring_id)) + return 0; + + head = csr_ops->read_csr_ring_head(csr, bank->bank_number, + ring->ring_number); + tail = csr_ops->read_csr_ring_tail(csr, bank->bank_number, + ring->ring_number); + empty = csr_ops->read_csr_e_stat(csr, bank->bank_number); + + seq_printf(sfile, + "ring num %02d, head %04x, tail %04x, empty: %d\n", + ring->ring_number, head, tail, + (empty & 1 << ring->ring_number) >> + ring->ring_number); + } + return 0; +} + +static void adf_bank_stop(struct seq_file *sfile, void *v) +{ + mutex_unlock(&bank_read_lock); +} + +static const struct seq_operations adf_bank_debug_sops = { + .start = adf_bank_start, + .next = adf_bank_next, + .stop = adf_bank_stop, + .show = adf_bank_show +}; + +DEFINE_SEQ_ATTRIBUTE(adf_bank_debug); + +int adf_bank_debugfs_add(struct adf_etr_bank_data *bank) +{ + struct adf_accel_dev *accel_dev = bank->accel_dev; + struct dentry *parent = accel_dev->transport->debug; + char name[16]; + + snprintf(name, sizeof(name), "bank_%02d", bank->bank_number); + bank->bank_debug_dir = debugfs_create_dir(name, parent); + bank->bank_debug_cfg = debugfs_create_file("config", S_IRUSR, + bank->bank_debug_dir, bank, + &adf_bank_debug_fops); + return 0; +} + +void adf_bank_debugfs_rm(struct adf_etr_bank_data *bank) +{ + debugfs_remove(bank->bank_debug_cfg); + debugfs_remove(bank->bank_debug_dir); +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_transport_internal.h b/drivers/crypto/intel/qat/qat_common/adf_transport_internal.h new file mode 100644 index 000000000..8b2c92ba7 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_transport_internal.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#ifndef ADF_TRANSPORT_INTRN_H +#define ADF_TRANSPORT_INTRN_H + +#include +#include +#include "adf_transport.h" + +struct adf_etr_ring_debug_entry { + char ring_name[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + struct dentry *debug; +}; + +struct adf_etr_ring_data { + void *base_addr; + atomic_t *inflights; + adf_callback_fn callback; + struct adf_etr_bank_data *bank; + dma_addr_t dma_addr; + struct adf_etr_ring_debug_entry *ring_debug; + spinlock_t lock; /* protects ring data struct */ + u16 head; + u16 tail; + u32 threshold; + u8 ring_number; + u8 ring_size; + u8 msg_size; +}; + +struct adf_etr_bank_data { + struct adf_etr_ring_data *rings; + struct tasklet_struct resp_handler; + void __iomem *csr_addr; + u32 irq_coalesc_timer; + u32 bank_number; + u16 ring_mask; + u16 irq_mask; + spinlock_t lock; /* protects bank data struct */ + struct adf_accel_dev *accel_dev; + struct dentry *bank_debug_dir; + struct dentry *bank_debug_cfg; +}; + +struct adf_etr_data { + struct adf_etr_bank_data *banks; + struct dentry *debug; +}; + +void adf_response_handler(uintptr_t bank_addr); +#ifdef CONFIG_DEBUG_FS +#include +int adf_bank_debugfs_add(struct adf_etr_bank_data *bank); +void adf_bank_debugfs_rm(struct adf_etr_bank_data *bank); +int adf_ring_debugfs_add(struct adf_etr_ring_data *ring, const char *name); +void adf_ring_debugfs_rm(struct adf_etr_ring_data *ring); +#else +static inline int adf_bank_debugfs_add(struct adf_etr_bank_data *bank) +{ + return 0; +} + +#define adf_bank_debugfs_rm(bank) do {} while (0) + +static inline int adf_ring_debugfs_add(struct adf_etr_ring_data *ring, + const char *name) +{ + return 0; +} + +#define adf_ring_debugfs_rm(ring) do {} while (0) +#endif +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_vf_isr.c b/drivers/crypto/intel/qat/qat_common/adf_vf_isr.c new file mode 100644 index 000000000..9ffaa56de --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_vf_isr.c @@ -0,0 +1,314 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_cfg.h" +#include "adf_cfg_strings.h" +#include "adf_cfg_common.h" +#include "adf_pfvf_vf_msg.h" +#include "adf_transport_access_macros.h" +#include "adf_transport_internal.h" + +#define ADF_VINTSOU_OFFSET 0x204 +#define ADF_VINTMSK_OFFSET 0x208 +#define ADF_VINTSOU_BUN BIT(0) +#define ADF_VINTSOU_PF2VF BIT(1) + +static struct workqueue_struct *adf_vf_stop_wq; + +struct adf_vf_stop_data { + struct adf_accel_dev *accel_dev; + struct work_struct work; +}; + +void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev) +{ + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + + ADF_CSR_WR(pmisc_addr, ADF_VINTMSK_OFFSET, 0x0); +} + +void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev) +{ + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + + ADF_CSR_WR(pmisc_addr, ADF_VINTMSK_OFFSET, 0x2); +} +EXPORT_SYMBOL_GPL(adf_disable_pf2vf_interrupts); + +static int adf_enable_msi(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; + int stat = pci_alloc_irq_vectors(pci_dev_info->pci_dev, 1, 1, + PCI_IRQ_MSI); + if (unlikely(stat < 0)) { + dev_err(&GET_DEV(accel_dev), + "Failed to enable MSI interrupt: %d\n", stat); + return stat; + } + + return 0; +} + +static void adf_disable_msi(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + + pci_free_irq_vectors(pdev); +} + +static void adf_dev_stop_async(struct work_struct *work) +{ + struct adf_vf_stop_data *stop_data = + container_of(work, struct adf_vf_stop_data, work); + struct adf_accel_dev *accel_dev = stop_data->accel_dev; + + adf_dev_restarting_notify(accel_dev); + adf_dev_down(accel_dev); + + /* Re-enable PF2VF interrupts */ + adf_enable_pf2vf_interrupts(accel_dev); + adf_vf2pf_notify_restart_complete(accel_dev); + kfree(stop_data); +} + +int adf_pf2vf_handle_pf_restarting(struct adf_accel_dev *accel_dev) +{ + struct adf_vf_stop_data *stop_data; + + clear_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status); + stop_data = kzalloc_obj(*stop_data, GFP_ATOMIC); + if (!stop_data) { + dev_err(&GET_DEV(accel_dev), + "Couldn't schedule stop for vf_%d\n", + accel_dev->accel_id); + return -ENOMEM; + } + stop_data->accel_dev = accel_dev; + INIT_WORK(&stop_data->work, adf_dev_stop_async); + queue_work(adf_vf_stop_wq, &stop_data->work); + + return 0; +} + +static void adf_pf2vf_bh_handler(void *data) +{ + struct adf_accel_dev *accel_dev = data; + bool ret; + + ret = adf_recv_and_handle_pf2vf_msg(accel_dev); + if (ret) + /* Re-enable PF2VF interrupts */ + adf_enable_pf2vf_interrupts(accel_dev); + + return; + +} + +static int adf_setup_pf2vf_bh(struct adf_accel_dev *accel_dev) +{ + tasklet_init(&accel_dev->vf.pf2vf_bh_tasklet, + (void *)adf_pf2vf_bh_handler, (unsigned long)accel_dev); + + mutex_init(&accel_dev->vf.vf2pf_lock); + return 0; +} + +static void adf_cleanup_pf2vf_bh(struct adf_accel_dev *accel_dev) +{ + tasklet_disable(&accel_dev->vf.pf2vf_bh_tasklet); + tasklet_kill(&accel_dev->vf.pf2vf_bh_tasklet); + mutex_destroy(&accel_dev->vf.vf2pf_lock); +} + +static irqreturn_t adf_isr(int irq, void *privdata) +{ + struct adf_accel_dev *accel_dev = privdata; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_hw_csr_ops *csr_ops = &hw_data->csr_ops; + struct adf_bar *pmisc = + &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; + void __iomem *pmisc_bar_addr = pmisc->virt_addr; + bool handled = false; + u32 v_int, v_mask; + + /* Read VF INT source CSR to determine the source of VF interrupt */ + v_int = ADF_CSR_RD(pmisc_bar_addr, ADF_VINTSOU_OFFSET); + + /* Read VF INT mask CSR to determine which sources are masked */ + v_mask = ADF_CSR_RD(pmisc_bar_addr, ADF_VINTMSK_OFFSET); + + /* + * Recompute v_int ignoring sources that are masked. This is to + * avoid rescheduling the tasklet for interrupts already handled + */ + v_int &= ~v_mask; + + /* Check for PF2VF interrupt */ + if (v_int & ADF_VINTSOU_PF2VF) { + /* Disable PF to VF interrupt */ + adf_disable_pf2vf_interrupts(accel_dev); + + /* Schedule tasklet to handle interrupt BH */ + tasklet_hi_schedule(&accel_dev->vf.pf2vf_bh_tasklet); + handled = true; + } + + /* Check bundle interrupt */ + if (v_int & ADF_VINTSOU_BUN) { + struct adf_etr_data *etr_data = accel_dev->transport; + struct adf_etr_bank_data *bank = &etr_data->banks[0]; + + /* Disable Flag and Coalesce Ring Interrupts */ + csr_ops->write_csr_int_flag_and_col(bank->csr_addr, + bank->bank_number, 0); + tasklet_hi_schedule(&bank->resp_handler); + handled = true; + } + + return handled ? IRQ_HANDLED : IRQ_NONE; +} + +static int adf_request_msi_irq(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + unsigned int cpu; + int ret; + + snprintf(accel_dev->vf.irq_name, ADF_MAX_MSIX_VECTOR_NAME, + "qat_%02x:%02d.%02d", pdev->bus->number, PCI_SLOT(pdev->devfn), + PCI_FUNC(pdev->devfn)); + ret = request_irq(pdev->irq, adf_isr, 0, accel_dev->vf.irq_name, + (void *)accel_dev); + if (ret) { + dev_err(&GET_DEV(accel_dev), "failed to enable irq for %s\n", + accel_dev->vf.irq_name); + return ret; + } + cpu = accel_dev->accel_id % num_online_cpus(); + irq_set_affinity_hint(pdev->irq, get_cpu_mask(cpu)); + accel_dev->vf.irq_enabled = true; + + return ret; +} + +static int adf_setup_bh(struct adf_accel_dev *accel_dev) +{ + struct adf_etr_data *priv_data = accel_dev->transport; + + tasklet_init(&priv_data->banks[0].resp_handler, adf_response_handler, + (unsigned long)priv_data->banks); + return 0; +} + +static void adf_cleanup_bh(struct adf_accel_dev *accel_dev) +{ + struct adf_etr_data *priv_data = accel_dev->transport; + + tasklet_disable(&priv_data->banks[0].resp_handler); + tasklet_kill(&priv_data->banks[0].resp_handler); +} + +/** + * adf_vf_isr_resource_free() - Free IRQ for acceleration device + * @accel_dev: Pointer to acceleration device. + * + * Function frees interrupts for acceleration device virtual function. + */ +void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + + if (accel_dev->vf.irq_enabled) { + irq_set_affinity_hint(pdev->irq, NULL); + free_irq(pdev->irq, accel_dev); + } + adf_cleanup_bh(accel_dev); + adf_cleanup_pf2vf_bh(accel_dev); + adf_disable_msi(accel_dev); +} +EXPORT_SYMBOL_GPL(adf_vf_isr_resource_free); + +/** + * adf_vf_isr_resource_alloc() - Allocate IRQ for acceleration device + * @accel_dev: Pointer to acceleration device. + * + * Function allocates interrupts for acceleration device virtual function. + * + * Return: 0 on success, error code otherwise. + */ +int adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev) +{ + if (adf_enable_msi(accel_dev)) + goto err_out; + + if (adf_setup_pf2vf_bh(accel_dev)) + goto err_disable_msi; + + if (adf_setup_bh(accel_dev)) + goto err_cleanup_pf2vf_bh; + + if (adf_request_msi_irq(accel_dev)) + goto err_cleanup_bh; + + return 0; + +err_cleanup_bh: + adf_cleanup_bh(accel_dev); + +err_cleanup_pf2vf_bh: + adf_cleanup_pf2vf_bh(accel_dev); + +err_disable_msi: + adf_disable_msi(accel_dev); + +err_out: + return -EFAULT; +} +EXPORT_SYMBOL_GPL(adf_vf_isr_resource_alloc); + +/** + * adf_flush_vf_wq() - Flush workqueue for VF + * @accel_dev: Pointer to acceleration device. + * + * Function disables the PF/VF interrupts on the VF so that no new messages + * are received and flushes the workqueue 'adf_vf_stop_wq'. + * + * Return: void. + */ +void adf_flush_vf_wq(struct adf_accel_dev *accel_dev) +{ + adf_disable_pf2vf_interrupts(accel_dev); + + flush_workqueue(adf_vf_stop_wq); +} +EXPORT_SYMBOL_GPL(adf_flush_vf_wq); + +/** + * adf_init_vf_wq() - Init workqueue for VF + * + * Return: 0 on success, error code otherwise. + */ +int __init adf_init_vf_wq(void) +{ + adf_vf_stop_wq = alloc_workqueue("adf_vf_stop_wq", + WQ_MEM_RECLAIM | WQ_PERCPU, 0); + + return !adf_vf_stop_wq ? -EFAULT : 0; +} + +void adf_exit_vf_wq(void) +{ + if (adf_vf_stop_wq) + destroy_workqueue(adf_vf_stop_wq); + + adf_vf_stop_wq = NULL; +} diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_fw.h b/drivers/crypto/intel/qat/qat_common/icp_qat_fw.h new file mode 100644 index 000000000..2fea30a78 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_fw.h @@ -0,0 +1,305 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#ifndef _ICP_QAT_FW_H_ +#define _ICP_QAT_FW_H_ +#include +#include "icp_qat_hw.h" + +#define QAT_FIELD_SET(flags, val, bitpos, mask) \ +{ (flags) = (((flags) & (~((mask) << (bitpos)))) | \ + (((val) & (mask)) << (bitpos))) ; } + +#define QAT_FIELD_GET(flags, bitpos, mask) \ + (((flags) >> (bitpos)) & (mask)) + +#define ICP_QAT_FW_REQ_DEFAULT_SZ 128 +#define ICP_QAT_FW_RESP_DEFAULT_SZ 32 +#define ICP_QAT_FW_COMN_ONE_BYTE_SHIFT 8 +#define ICP_QAT_FW_COMN_SINGLE_BYTE_MASK 0xFF +#define ICP_QAT_FW_NUM_LONGWORDS_1 1 +#define ICP_QAT_FW_NUM_LONGWORDS_2 2 +#define ICP_QAT_FW_NUM_LONGWORDS_3 3 +#define ICP_QAT_FW_NUM_LONGWORDS_4 4 +#define ICP_QAT_FW_NUM_LONGWORDS_5 5 +#define ICP_QAT_FW_NUM_LONGWORDS_6 6 +#define ICP_QAT_FW_NUM_LONGWORDS_7 7 +#define ICP_QAT_FW_NUM_LONGWORDS_10 10 +#define ICP_QAT_FW_NUM_LONGWORDS_13 13 +#define ICP_QAT_FW_NULL_REQ_SERV_ID 1 + +enum icp_qat_fw_comn_resp_serv_id { + ICP_QAT_FW_COMN_RESP_SERV_NULL, + ICP_QAT_FW_COMN_RESP_SERV_CPM_FW, + ICP_QAT_FW_COMN_RESP_SERV_DELIMITER +}; + +enum icp_qat_fw_comn_request_id { + ICP_QAT_FW_COMN_REQ_NULL = 0, + ICP_QAT_FW_COMN_REQ_CPM_FW_PKE = 3, + ICP_QAT_FW_COMN_REQ_CPM_FW_LA = 4, + ICP_QAT_FW_COMN_REQ_CPM_FW_DMA = 7, + ICP_QAT_FW_COMN_REQ_CPM_FW_COMP = 9, + ICP_QAT_FW_COMN_REQ_DELIMITER +}; + +struct icp_qat_fw_comn_req_hdr_cd_pars { + union { + struct { + __u64 content_desc_addr; + __u16 content_desc_resrvd1; + __u8 content_desc_params_sz; + __u8 content_desc_hdr_resrvd2; + __u32 content_desc_resrvd3; + } s; + struct { + __u32 serv_specif_fields[4]; + } s1; + } u; +}; + +struct icp_qat_fw_comn_req_mid { + __u64 opaque_data; + __u64 src_data_addr; + __u64 dest_data_addr; + __u32 src_length; + __u32 dst_length; +}; + +struct icp_qat_fw_comn_req_cd_ctrl { + __u32 content_desc_ctrl_lw[ICP_QAT_FW_NUM_LONGWORDS_5]; +}; + +struct icp_qat_fw_comn_req_hdr { + __u8 resrvd1; + __u8 service_cmd_id; + __u8 service_type; + __u8 hdr_flags; + __u16 serv_specif_flags; + __u16 comn_req_flags; +}; + +struct icp_qat_fw_comn_req_rqpars { + __u32 serv_specif_rqpars_lw[ICP_QAT_FW_NUM_LONGWORDS_13]; +}; + +struct icp_qat_fw_comn_req { + struct icp_qat_fw_comn_req_hdr comn_hdr; + struct icp_qat_fw_comn_req_hdr_cd_pars cd_pars; + struct icp_qat_fw_comn_req_mid comn_mid; + struct icp_qat_fw_comn_req_rqpars serv_specif_rqpars; + struct icp_qat_fw_comn_req_cd_ctrl cd_ctrl; +}; + +struct icp_qat_fw_comn_error { + __u8 xlat_err_code; + __u8 cmp_err_code; +}; + +struct icp_qat_fw_comn_resp_hdr { + __u8 resrvd1; + __u8 service_id; + __u8 response_type; + __u8 hdr_flags; + struct icp_qat_fw_comn_error comn_error; + __u8 comn_status; + __u8 cmd_id; +}; + +struct icp_qat_fw_comn_resp { + struct icp_qat_fw_comn_resp_hdr comn_hdr; + __u64 opaque_data; + __u32 resrvd[ICP_QAT_FW_NUM_LONGWORDS_4]; +}; + +#define ICP_QAT_FW_COMN_REQ_FLAG_SET 1 +#define ICP_QAT_FW_COMN_REQ_FLAG_CLR 0 +#define ICP_QAT_FW_COMN_VALID_FLAG_BITPOS 7 +#define ICP_QAT_FW_COMN_VALID_FLAG_MASK 0x1 +#define ICP_QAT_FW_COMN_HDR_RESRVD_FLD_MASK 0x7F +#define ICP_QAT_FW_COMN_CNV_FLAG_BITPOS 6 +#define ICP_QAT_FW_COMN_CNV_FLAG_MASK 0x1 +#define ICP_QAT_FW_COMN_CNVNR_FLAG_BITPOS 5 +#define ICP_QAT_FW_COMN_CNVNR_FLAG_MASK 0x1 + +#define ICP_QAT_FW_COMN_OV_SRV_TYPE_GET(icp_qat_fw_comn_req_hdr_t) \ + icp_qat_fw_comn_req_hdr_t.service_type + +#define ICP_QAT_FW_COMN_OV_SRV_TYPE_SET(icp_qat_fw_comn_req_hdr_t, val) \ + icp_qat_fw_comn_req_hdr_t.service_type = val + +#define ICP_QAT_FW_COMN_OV_SRV_CMD_ID_GET(icp_qat_fw_comn_req_hdr_t) \ + icp_qat_fw_comn_req_hdr_t.service_cmd_id + +#define ICP_QAT_FW_COMN_OV_SRV_CMD_ID_SET(icp_qat_fw_comn_req_hdr_t, val) \ + icp_qat_fw_comn_req_hdr_t.service_cmd_id = val + +#define ICP_QAT_FW_COMN_HDR_VALID_FLAG_GET(hdr_t) \ + ICP_QAT_FW_COMN_VALID_FLAG_GET(hdr_t.hdr_flags) + +#define ICP_QAT_FW_COMN_HDR_CNVNR_FLAG_GET(hdr_flags) \ + QAT_FIELD_GET(hdr_flags, \ + ICP_QAT_FW_COMN_CNVNR_FLAG_BITPOS, \ + ICP_QAT_FW_COMN_CNVNR_FLAG_MASK) + +#define ICP_QAT_FW_COMN_HDR_CNVNR_FLAG_SET(hdr_t, val) \ + QAT_FIELD_SET((hdr_t.hdr_flags), (val), \ + ICP_QAT_FW_COMN_CNVNR_FLAG_BITPOS, \ + ICP_QAT_FW_COMN_CNVNR_FLAG_MASK) + +#define ICP_QAT_FW_COMN_HDR_CNV_FLAG_GET(hdr_flags) \ + QAT_FIELD_GET(hdr_flags, \ + ICP_QAT_FW_COMN_CNV_FLAG_BITPOS, \ + ICP_QAT_FW_COMN_CNV_FLAG_MASK) + +#define ICP_QAT_FW_COMN_ST_BLK_FLAG_BITPOS 4 +#define ICP_QAT_FW_COMN_ST_BLK_FLAG_MASK 0x1 +#define ICP_QAT_FW_COMN_HDR_ST_BLK_FLAG_GET(hdr_flags) \ + QAT_FIELD_GET(hdr_flags, \ + ICP_QAT_FW_COMN_ST_BLK_FLAG_BITPOS, \ + ICP_QAT_FW_COMN_ST_BLK_FLAG_MASK) + +#define ICP_QAT_FW_COMN_HDR_CNV_FLAG_SET(hdr_t, val) \ + QAT_FIELD_SET((hdr_t.hdr_flags), (val), \ + ICP_QAT_FW_COMN_CNV_FLAG_BITPOS, \ + ICP_QAT_FW_COMN_CNV_FLAG_MASK) + +#define ICP_QAT_FW_COMN_HDR_VALID_FLAG_SET(hdr_t, val) \ + ICP_QAT_FW_COMN_VALID_FLAG_SET(hdr_t, val) + +#define ICP_QAT_FW_COMN_VALID_FLAG_GET(hdr_flags) \ + QAT_FIELD_GET(hdr_flags, \ + ICP_QAT_FW_COMN_VALID_FLAG_BITPOS, \ + ICP_QAT_FW_COMN_VALID_FLAG_MASK) + +#define ICP_QAT_FW_COMN_HDR_RESRVD_FLD_GET(hdr_flags) \ + (hdr_flags & ICP_QAT_FW_COMN_HDR_RESRVD_FLD_MASK) + +#define ICP_QAT_FW_COMN_VALID_FLAG_SET(hdr_t, val) \ + QAT_FIELD_SET((hdr_t.hdr_flags), (val), \ + ICP_QAT_FW_COMN_VALID_FLAG_BITPOS, \ + ICP_QAT_FW_COMN_VALID_FLAG_MASK) + +#define ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(valid) \ + (((valid) & ICP_QAT_FW_COMN_VALID_FLAG_MASK) << \ + ICP_QAT_FW_COMN_VALID_FLAG_BITPOS) + +#define QAT_COMN_PTR_TYPE_BITPOS 0 +#define QAT_COMN_PTR_TYPE_MASK 0x1 +#define QAT_COMN_CD_FLD_TYPE_BITPOS 1 +#define QAT_COMN_CD_FLD_TYPE_MASK 0x1 +#define QAT_COMN_PTR_TYPE_FLAT 0x0 +#define QAT_COMN_PTR_TYPE_SGL 0x1 +#define QAT_COMN_CD_FLD_TYPE_64BIT_ADR 0x0 +#define QAT_COMN_CD_FLD_TYPE_16BYTE_DATA 0x1 + +#define ICP_QAT_FW_COMN_FLAGS_BUILD(cdt, ptr) \ + ((((cdt) & QAT_COMN_CD_FLD_TYPE_MASK) << QAT_COMN_CD_FLD_TYPE_BITPOS) \ + | (((ptr) & QAT_COMN_PTR_TYPE_MASK) << QAT_COMN_PTR_TYPE_BITPOS)) + +#define ICP_QAT_FW_COMN_PTR_TYPE_GET(flags) \ + QAT_FIELD_GET(flags, QAT_COMN_PTR_TYPE_BITPOS, QAT_COMN_PTR_TYPE_MASK) + +#define ICP_QAT_FW_COMN_CD_FLD_TYPE_GET(flags) \ + QAT_FIELD_GET(flags, QAT_COMN_CD_FLD_TYPE_BITPOS, \ + QAT_COMN_CD_FLD_TYPE_MASK) + +#define ICP_QAT_FW_COMN_PTR_TYPE_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_COMN_PTR_TYPE_BITPOS, \ + QAT_COMN_PTR_TYPE_MASK) + +#define ICP_QAT_FW_COMN_CD_FLD_TYPE_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_COMN_CD_FLD_TYPE_BITPOS, \ + QAT_COMN_CD_FLD_TYPE_MASK) + +#define ICP_QAT_FW_COMN_NEXT_ID_BITPOS 4 +#define ICP_QAT_FW_COMN_NEXT_ID_MASK 0xF0 +#define ICP_QAT_FW_COMN_CURR_ID_BITPOS 0 +#define ICP_QAT_FW_COMN_CURR_ID_MASK 0x0F + +#define ICP_QAT_FW_COMN_NEXT_ID_GET(cd_ctrl_hdr_t) \ + ((((cd_ctrl_hdr_t)->next_curr_id) & ICP_QAT_FW_COMN_NEXT_ID_MASK) \ + >> (ICP_QAT_FW_COMN_NEXT_ID_BITPOS)) + +#define ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl_hdr_t, val) \ + { ((cd_ctrl_hdr_t)->next_curr_id) = ((((cd_ctrl_hdr_t)->next_curr_id) \ + & ICP_QAT_FW_COMN_CURR_ID_MASK) | \ + ((val << ICP_QAT_FW_COMN_NEXT_ID_BITPOS) \ + & ICP_QAT_FW_COMN_NEXT_ID_MASK)); } + +#define ICP_QAT_FW_COMN_CURR_ID_GET(cd_ctrl_hdr_t) \ + (((cd_ctrl_hdr_t)->next_curr_id) & ICP_QAT_FW_COMN_CURR_ID_MASK) + +#define ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl_hdr_t, val) \ + { ((cd_ctrl_hdr_t)->next_curr_id) = ((((cd_ctrl_hdr_t)->next_curr_id) \ + & ICP_QAT_FW_COMN_NEXT_ID_MASK) | \ + ((val) & ICP_QAT_FW_COMN_CURR_ID_MASK)); } + +#define QAT_COMN_RESP_CRYPTO_STATUS_BITPOS 7 +#define QAT_COMN_RESP_CRYPTO_STATUS_MASK 0x1 +#define QAT_COMN_RESP_PKE_STATUS_BITPOS 6 +#define QAT_COMN_RESP_PKE_STATUS_MASK 0x1 +#define QAT_COMN_RESP_CMP_STATUS_BITPOS 5 +#define QAT_COMN_RESP_CMP_STATUS_MASK 0x1 +#define QAT_COMN_RESP_XLAT_STATUS_BITPOS 4 +#define QAT_COMN_RESP_XLAT_STATUS_MASK 0x1 +#define QAT_COMN_RESP_CMP_END_OF_LAST_BLK_BITPOS 3 +#define QAT_COMN_RESP_CMP_END_OF_LAST_BLK_MASK 0x1 + +#define ICP_QAT_FW_COMN_RESP_STATUS_BUILD(crypto, comp, xlat, eolb) \ + ((((crypto) & QAT_COMN_RESP_CRYPTO_STATUS_MASK) << \ + QAT_COMN_RESP_CRYPTO_STATUS_BITPOS) | \ + (((comp) & QAT_COMN_RESP_CMP_STATUS_MASK) << \ + QAT_COMN_RESP_CMP_STATUS_BITPOS) | \ + (((xlat) & QAT_COMN_RESP_XLAT_STATUS_MASK) << \ + QAT_COMN_RESP_XLAT_STATUS_BITPOS) | \ + (((eolb) & QAT_COMN_RESP_CMP_END_OF_LAST_BLK_MASK) << \ + QAT_COMN_RESP_CMP_END_OF_LAST_BLK_BITPOS)) + +#define ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(status) \ + QAT_FIELD_GET(status, QAT_COMN_RESP_CRYPTO_STATUS_BITPOS, \ + QAT_COMN_RESP_CRYPTO_STATUS_MASK) + +#define ICP_QAT_FW_COMN_RESP_CMP_STAT_GET(status) \ + QAT_FIELD_GET(status, QAT_COMN_RESP_CMP_STATUS_BITPOS, \ + QAT_COMN_RESP_CMP_STATUS_MASK) + +#define ICP_QAT_FW_COMN_RESP_XLAT_STAT_GET(status) \ + QAT_FIELD_GET(status, QAT_COMN_RESP_XLAT_STATUS_BITPOS, \ + QAT_COMN_RESP_XLAT_STATUS_MASK) + +#define ICP_QAT_FW_COMN_RESP_CMP_END_OF_LAST_BLK_FLAG_GET(status) \ + QAT_FIELD_GET(status, QAT_COMN_RESP_CMP_END_OF_LAST_BLK_BITPOS, \ + QAT_COMN_RESP_CMP_END_OF_LAST_BLK_MASK) + +#define ICP_QAT_FW_COMN_STATUS_FLAG_OK 0 +#define ICP_QAT_FW_COMN_STATUS_FLAG_ERROR 1 +#define ICP_QAT_FW_COMN_STATUS_CMP_END_OF_LAST_BLK_FLAG_CLR 0 +#define ICP_QAT_FW_COMN_STATUS_CMP_END_OF_LAST_BLK_FLAG_SET 1 +#define ERR_CODE_NO_ERROR 0 +#define ERR_CODE_INVALID_BLOCK_TYPE -1 +#define ERR_CODE_NO_MATCH_ONES_COMP -2 +#define ERR_CODE_TOO_MANY_LEN_OR_DIS -3 +#define ERR_CODE_INCOMPLETE_LEN -4 +#define ERR_CODE_RPT_LEN_NO_FIRST_LEN -5 +#define ERR_CODE_RPT_GT_SPEC_LEN -6 +#define ERR_CODE_INV_LIT_LEN_CODE_LEN -7 +#define ERR_CODE_INV_DIS_CODE_LEN -8 +#define ERR_CODE_INV_LIT_LEN_DIS_IN_BLK -9 +#define ERR_CODE_DIS_TOO_FAR_BACK -10 +#define ERR_CODE_OVERFLOW_ERROR -11 +#define ERR_CODE_SOFT_ERROR -12 +#define ERR_CODE_FATAL_ERROR -13 +#define ERR_CODE_SSM_ERROR -14 +#define ERR_CODE_ENDPOINT_ERROR -15 + +enum icp_qat_fw_slice { + ICP_QAT_FW_SLICE_NULL = 0, + ICP_QAT_FW_SLICE_CIPHER = 1, + ICP_QAT_FW_SLICE_AUTH = 2, + ICP_QAT_FW_SLICE_DRAM_RD = 3, + ICP_QAT_FW_SLICE_DRAM_WR = 4, + ICP_QAT_FW_SLICE_COMP = 5, + ICP_QAT_FW_SLICE_XLAT = 6, + ICP_QAT_FW_SLICE_DELIMITER +}; +#endif diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_comp.h b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_comp.h new file mode 100644 index 000000000..2526053ee --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_comp.h @@ -0,0 +1,423 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2022 Intel Corporation */ +#ifndef _ICP_QAT_FW_COMP_H_ +#define _ICP_QAT_FW_COMP_H_ +#include "icp_qat_fw.h" + +enum icp_qat_fw_comp_cmd_id { + ICP_QAT_FW_COMP_CMD_STATIC = 0, + ICP_QAT_FW_COMP_CMD_DYNAMIC = 1, + ICP_QAT_FW_COMP_CMD_DECOMPRESS = 2, + ICP_QAT_FW_COMP_CMD_ZSTD_COMPRESS = 10, + ICP_QAT_FW_COMP_CMD_ZSTD_DECOMPRESS = 11, + ICP_QAT_FW_COMP_CMD_DELIMITER +}; + +enum icp_qat_fw_comp_20_cmd_id { + ICP_QAT_FW_COMP_20_CMD_LZ4_COMPRESS = 3, + ICP_QAT_FW_COMP_20_CMD_LZ4_DECOMPRESS = 4, + ICP_QAT_FW_COMP_20_CMD_LZ4S_COMPRESS = 5, + ICP_QAT_FW_COMP_20_CMD_LZ4S_DECOMPRESS = 6, + ICP_QAT_FW_COMP_20_CMD_RESERVED_7 = 7, + ICP_QAT_FW_COMP_20_CMD_RESERVED_8 = 8, + ICP_QAT_FW_COMP_20_CMD_RESERVED_9 = 9, + ICP_QAT_FW_COMP_23_CMD_ZSTD_COMPRESS = 10, + ICP_QAT_FW_COMP_23_CMD_ZSTD_DECOMPRESS = 11, + ICP_QAT_FW_COMP_20_CMD_DELIMITER +}; + +#define ICP_QAT_FW_COMP_STATELESS_SESSION 0 +#define ICP_QAT_FW_COMP_STATEFUL_SESSION 1 +#define ICP_QAT_FW_COMP_NOT_AUTO_SELECT_BEST 0 +#define ICP_QAT_FW_COMP_AUTO_SELECT_BEST 1 +#define ICP_QAT_FW_COMP_NOT_ENH_AUTO_SELECT_BEST 0 +#define ICP_QAT_FW_COMP_ENH_AUTO_SELECT_BEST 1 +#define ICP_QAT_FW_COMP_NOT_DISABLE_TYPE0_ENH_AUTO_SELECT_BEST 0 +#define ICP_QAT_FW_COMP_DISABLE_TYPE0_ENH_AUTO_SELECT_BEST 1 +#define ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_USED_AS_INTMD_BUF 1 +#define ICP_QAT_FW_COMP_ENABLE_SECURE_RAM_USED_AS_INTMD_BUF 0 +#define ICP_QAT_FW_COMP_SESSION_TYPE_BITPOS 2 +#define ICP_QAT_FW_COMP_SESSION_TYPE_MASK 0x1 +#define ICP_QAT_FW_COMP_AUTO_SELECT_BEST_BITPOS 3 +#define ICP_QAT_FW_COMP_AUTO_SELECT_BEST_MASK 0x1 +#define ICP_QAT_FW_COMP_ENHANCED_AUTO_SELECT_BEST_BITPOS 4 +#define ICP_QAT_FW_COMP_ENHANCED_AUTO_SELECT_BEST_MASK 0x1 +#define ICP_QAT_FW_COMP_RET_DISABLE_TYPE0_HEADER_DATA_BITPOS 5 +#define ICP_QAT_FW_COMP_RET_DISABLE_TYPE0_HEADER_DATA_MASK 0x1 +#define ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_BITPOS 7 +#define ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_MASK 0x1 +#define ICP_QAT_FW_COMP_AUTO_SELECT_BEST_MAX_VALUE 0xFFFFFFFF + +#define ICP_QAT_FW_COMP_FLAGS_BUILD(sesstype, autoselect, enhanced_asb, \ + ret_uncomp, secure_ram) \ + ((((sesstype) & ICP_QAT_FW_COMP_SESSION_TYPE_MASK) << \ + ICP_QAT_FW_COMP_SESSION_TYPE_BITPOS) | \ + (((autoselect) & ICP_QAT_FW_COMP_AUTO_SELECT_BEST_MASK) << \ + ICP_QAT_FW_COMP_AUTO_SELECT_BEST_BITPOS) | \ + (((enhanced_asb) & ICP_QAT_FW_COMP_ENHANCED_AUTO_SELECT_BEST_MASK) << \ + ICP_QAT_FW_COMP_ENHANCED_AUTO_SELECT_BEST_BITPOS) | \ + (((ret_uncomp) & ICP_QAT_FW_COMP_RET_DISABLE_TYPE0_HEADER_DATA_MASK) << \ + ICP_QAT_FW_COMP_RET_DISABLE_TYPE0_HEADER_DATA_BITPOS) | \ + (((secure_ram) & ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_MASK) << \ + ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_BITPOS)) + +#define ICP_QAT_FW_COMP_SESSION_TYPE_GET(flags) \ + QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_SESSION_TYPE_BITPOS, \ + ICP_QAT_FW_COMP_SESSION_TYPE_MASK) + +#define ICP_QAT_FW_COMP_SESSION_TYPE_SET(flags, val) \ + QAT_FIELD_SET(flags, val, ICP_QAT_FW_COMP_SESSION_TYPE_BITPOS, \ + ICP_QAT_FW_COMP_SESSION_TYPE_MASK) + +#define ICP_QAT_FW_COMP_AUTO_SELECT_BEST_GET(flags) \ + QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_AUTO_SELECT_BEST_BITPOS, \ + ICP_QAT_FW_COMP_AUTO_SELECT_BEST_MASK) + +#define ICP_QAT_FW_COMP_EN_ASB_GET(flags) \ + QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_ENHANCED_AUTO_SELECT_BEST_BITPOS, \ + ICP_QAT_FW_COMP_ENHANCED_AUTO_SELECT_BEST_MASK) + +#define ICP_QAT_FW_COMP_RET_UNCOMP_GET(flags) \ + QAT_FIELD_GET(flags, \ + ICP_QAT_FW_COMP_RET_DISABLE_TYPE0_HEADER_DATA_BITPOS, \ + ICP_QAT_FW_COMP_RET_DISABLE_TYPE0_HEADER_DATA_MASK) + +#define ICP_QAT_FW_COMP_SECURE_RAM_USE_GET(flags) \ + QAT_FIELD_GET(flags, \ + ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_BITPOS, \ + ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_MASK) + +struct icp_qat_fw_comp_req_hdr_cd_pars { + union { + struct { + __u64 content_desc_addr; + __u16 content_desc_resrvd1; + __u8 content_desc_params_sz; + __u8 content_desc_hdr_resrvd2; + __u32 content_desc_resrvd3; + } s; + struct { + __u32 comp_slice_cfg_word[ICP_QAT_FW_NUM_LONGWORDS_2]; + __u32 content_desc_resrvd4; + } sl; + } u; +}; + +struct icp_qat_fw_comp_req_params { + __u32 comp_len; + __u32 out_buffer_sz; + union { + struct { + __u32 initial_crc32; + __u32 initial_adler; + } legacy; + __u64 crc_data_addr; + } crc; + __u32 req_par_flags; + __u32 rsrvd; +}; + +#define ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD(sop, eop, bfinal, cnv, cnvnr, \ + cnvdfx, crc, xxhash_acc, \ + cnv_error_type, append_crc, \ + drop_data, partial_decomp) \ + ((((sop) & ICP_QAT_FW_COMP_SOP_MASK) << \ + ICP_QAT_FW_COMP_SOP_BITPOS) | \ + (((eop) & ICP_QAT_FW_COMP_EOP_MASK) << \ + ICP_QAT_FW_COMP_EOP_BITPOS) | \ + (((bfinal) & ICP_QAT_FW_COMP_BFINAL_MASK) \ + << ICP_QAT_FW_COMP_BFINAL_BITPOS) | \ + (((cnv) & ICP_QAT_FW_COMP_CNV_MASK) << \ + ICP_QAT_FW_COMP_CNV_BITPOS) | \ + (((cnvnr) & ICP_QAT_FW_COMP_CNVNR_MASK) \ + << ICP_QAT_FW_COMP_CNVNR_BITPOS) | \ + (((cnvdfx) & ICP_QAT_FW_COMP_CNV_DFX_MASK) \ + << ICP_QAT_FW_COMP_CNV_DFX_BITPOS) | \ + (((crc) & ICP_QAT_FW_COMP_CRC_MODE_MASK) \ + << ICP_QAT_FW_COMP_CRC_MODE_BITPOS) | \ + (((xxhash_acc) & ICP_QAT_FW_COMP_XXHASH_ACC_MODE_MASK) \ + << ICP_QAT_FW_COMP_XXHASH_ACC_MODE_BITPOS) | \ + (((cnv_error_type) & ICP_QAT_FW_COMP_CNV_ERROR_MASK) \ + << ICP_QAT_FW_COMP_CNV_ERROR_BITPOS) | \ + (((append_crc) & ICP_QAT_FW_COMP_APPEND_CRC_MASK) \ + << ICP_QAT_FW_COMP_APPEND_CRC_BITPOS) | \ + (((drop_data) & ICP_QAT_FW_COMP_DROP_DATA_MASK) \ + << ICP_QAT_FW_COMP_DROP_DATA_BITPOS) | \ + (((partial_decomp) & ICP_QAT_FW_COMP_PARTIAL_DECOMP_MASK) \ + << ICP_QAT_FW_COMP_PARTIAL_DECOMP_BITPOS)) + +#define ICP_QAT_FW_COMP_NOT_SOP 0 +#define ICP_QAT_FW_COMP_SOP 1 +#define ICP_QAT_FW_COMP_NOT_EOP 0 +#define ICP_QAT_FW_COMP_EOP 1 +#define ICP_QAT_FW_COMP_NOT_BFINAL 0 +#define ICP_QAT_FW_COMP_BFINAL 1 +#define ICP_QAT_FW_COMP_NO_CNV 0 +#define ICP_QAT_FW_COMP_CNV 1 +#define ICP_QAT_FW_COMP_NO_CNV_RECOVERY 0 +#define ICP_QAT_FW_COMP_CNV_RECOVERY 1 +#define ICP_QAT_FW_COMP_NO_CNV_DFX 0 +#define ICP_QAT_FW_COMP_CNV_DFX 1 +#define ICP_QAT_FW_COMP_CRC_MODE_LEGACY 0 +#define ICP_QAT_FW_COMP_CRC_MODE_E2E 1 +#define ICP_QAT_FW_COMP_NO_XXHASH_ACC 0 +#define ICP_QAT_FW_COMP_XXHASH_ACC 1 +#define ICP_QAT_FW_COMP_APPEND_CRC 1 +#define ICP_QAT_FW_COMP_NO_APPEND_CRC 0 +#define ICP_QAT_FW_COMP_DROP_DATA 1 +#define ICP_QAT_FW_COMP_NO_DROP_DATA 0 +#define ICP_QAT_FW_COMP_PARTIAL_DECOMPRESS 1 +#define ICP_QAT_FW_COMP_NO_PARTIAL_DECOMPRESS 0 +#define ICP_QAT_FW_COMP_SOP_BITPOS 0 +#define ICP_QAT_FW_COMP_SOP_MASK 0x1 +#define ICP_QAT_FW_COMP_EOP_BITPOS 1 +#define ICP_QAT_FW_COMP_EOP_MASK 0x1 +#define ICP_QAT_FW_COMP_BFINAL_BITPOS 6 +#define ICP_QAT_FW_COMP_BFINAL_MASK 0x1 +#define ICP_QAT_FW_COMP_CNV_BITPOS 16 +#define ICP_QAT_FW_COMP_CNV_MASK 0x1 +#define ICP_QAT_FW_COMP_CNVNR_BITPOS 17 +#define ICP_QAT_FW_COMP_CNVNR_MASK 0x1 +#define ICP_QAT_FW_COMP_CNV_DFX_BITPOS 18 +#define ICP_QAT_FW_COMP_CNV_DFX_MASK 0x1 +#define ICP_QAT_FW_COMP_CRC_MODE_BITPOS 19 +#define ICP_QAT_FW_COMP_CRC_MODE_MASK 0x1 +#define ICP_QAT_FW_COMP_XXHASH_ACC_MODE_BITPOS 20 +#define ICP_QAT_FW_COMP_XXHASH_ACC_MODE_MASK 0x1 +#define ICP_QAT_FW_COMP_CNV_ERROR_BITPOS 21 +#define ICP_QAT_FW_COMP_CNV_ERROR_MASK 0b111 +#define ICP_QAT_FW_COMP_CNV_ERROR_NONE 0b000 +#define ICP_QAT_FW_COMP_CNV_ERROR_CHECKSUM 0b001 +#define ICP_QAT_FW_COMP_CNV_ERROR_DCPR_OBC_DIFF 0b010 +#define ICP_QAT_FW_COMP_CNV_ERROR_DCPR 0b011 +#define ICP_QAT_FW_COMP_CNV_ERROR_XLT 0b100 +#define ICP_QAT_FW_COMP_CNV_ERROR_DCPR_IBC_DIFF 0b101 +#define ICP_QAT_FW_COMP_APPEND_CRC_BITPOS 24 +#define ICP_QAT_FW_COMP_APPEND_CRC_MASK 0x1 +#define ICP_QAT_FW_COMP_DROP_DATA_BITPOS 25 +#define ICP_QAT_FW_COMP_DROP_DATA_MASK 0x1 +#define ICP_QAT_FW_COMP_PARTIAL_DECOMP_BITPOS 27 +#define ICP_QAT_FW_COMP_PARTIAL_DECOMP_MASK 0x1 + +#define ICP_QAT_FW_COMP_SOP_GET(flags) \ + QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_SOP_BITPOS, \ + ICP_QAT_FW_COMP_SOP_MASK) + +#define ICP_QAT_FW_COMP_SOP_SET(flags, val) \ + QAT_FIELD_SET(flags, val, ICP_QAT_FW_COMP_SOP_BITPOS, \ + ICP_QAT_FW_COMP_SOP_MASK) + +#define ICP_QAT_FW_COMP_EOP_GET(flags) \ + QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_EOP_BITPOS, \ + ICP_QAT_FW_COMP_EOP_MASK) + +#define ICP_QAT_FW_COMP_EOP_SET(flags, val) \ + QAT_FIELD_SET(flags, val, ICP_QAT_FW_COMP_EOP_BITPOS, \ + ICP_QAT_FW_COMP_EOP_MASK) + +#define ICP_QAT_FW_COMP_BFINAL_GET(flags) \ + QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_BFINAL_BITPOS, \ + ICP_QAT_FW_COMP_BFINAL_MASK) + +#define ICP_QAT_FW_COMP_BFINAL_SET(flags, val) \ + QAT_FIELD_SET(flags, val, ICP_QAT_FW_COMP_BFINAL_BITPOS, \ + ICP_QAT_FW_COMP_BFINAL_MASK) + +#define ICP_QAT_FW_COMP_CNV_GET(flags) \ + QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_CNV_BITPOS, \ + ICP_QAT_FW_COMP_CNV_MASK) + +#define ICP_QAT_FW_COMP_CNVNR_GET(flags) \ + QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_CNVNR_BITPOS, \ + ICP_QAT_FW_COMP_CNVNR_MASK) + +#define ICP_QAT_FW_COMP_CNV_DFX_GET(flags) \ + QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_CNV_DFX_BITPOS, \ + ICP_QAT_FW_COMP_CNV_DFX_MASK) + +#define ICP_QAT_FW_COMP_CNV_DFX_SET(flags, val) \ + QAT_FIELD_SET(flags, val, ICP_QAT_FW_COMP_CNV_DFX_BITPOS, \ + ICP_QAT_FW_COMP_CNV_DFX_MASK) + +#define ICP_QAT_FW_COMP_CRC_MODE_GET(flags) \ + QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_CRC_MODE_BITPOS, \ + ICP_QAT_FW_COMP_CRC_MODE_MASK) + +#define ICP_QAT_FW_COMP_XXHASH_ACC_MODE_GET(flags) \ + QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_XXHASH_ACC_MODE_BITPOS, \ + ICP_QAT_FW_COMP_XXHASH_ACC_MODE_MASK) + +#define ICP_QAT_FW_COMP_XXHASH_ACC_MODE_SET(flags, val) \ + QAT_FIELD_SET(flags, val, ICP_QAT_FW_COMP_XXHASH_ACC_MODE_BITPOS, \ + ICP_QAT_FW_COMP_XXHASH_ACC_MODE_MASK) + +#define ICP_QAT_FW_COMP_CNV_ERROR_TYPE_GET(flags) \ + QAT_FIELD_GET(flags, ICP_QAT_FW_COMP_CNV_ERROR_BITPOS, \ + ICP_QAT_FW_COMP_CNV_ERROR_MASK) + +#define ICP_QAT_FW_COMP_CNV_ERROR_TYPE_SET(flags, val) \ + QAT_FIELD_SET(flags, val, ICP_QAT_FW_COMP_CNV_ERROR_BITPOS, \ + ICP_QAT_FW_COMP_CNV_ERROR_MASK) + +struct icp_qat_fw_xlt_req_params { + __u64 inter_buff_ptr; +}; + +struct icp_qat_fw_comp_cd_hdr { + __u16 ram_bank_flags; + __u8 comp_cfg_offset; + __u8 next_curr_id; + __u32 resrvd; + __u64 comp_state_addr; + __u64 ram_banks_addr; +}; + +#define COMP_CPR_INITIAL_CRC 0 +#define COMP_CPR_INITIAL_ADLER 1 + +struct icp_qat_fw_xlt_cd_hdr { + __u16 resrvd1; + __u8 resrvd2; + __u8 next_curr_id; + __u32 resrvd3; +}; + +struct icp_qat_fw_comp_req { + struct icp_qat_fw_comn_req_hdr comn_hdr; + struct icp_qat_fw_comp_req_hdr_cd_pars cd_pars; + struct icp_qat_fw_comn_req_mid comn_mid; + struct icp_qat_fw_comp_req_params comp_pars; + union { + struct icp_qat_fw_xlt_req_params xlt_pars; + __u32 resrvd1[ICP_QAT_FW_NUM_LONGWORDS_2]; + struct { + __u32 partial_decompress_length; + __u32 partial_decompress_offset; + } partial_decompress; + } u1; + union { + __u32 resrvd2[ICP_QAT_FW_NUM_LONGWORDS_2]; + struct { + __u32 asb_value; + __u32 reserved; + } asb_threshold; + } u3; + struct icp_qat_fw_comp_cd_hdr comp_cd_ctrl; + union { + struct icp_qat_fw_xlt_cd_hdr xlt_cd_ctrl; + __u32 resrvd3[ICP_QAT_FW_NUM_LONGWORDS_2]; + } u2; +}; + +struct icp_qat_fw_resp_comp_pars { + __u32 input_byte_counter; + __u32 output_byte_counter; + union { + struct { + __u32 curr_crc32; + __u32 curr_adler_32; + } legacy; + __u32 resrvd[ICP_QAT_FW_NUM_LONGWORDS_2]; + } crc; +}; + +struct icp_qat_fw_comp_state { + __u32 rd8_counter; + __u32 status_flags; + __u32 in_counter; + __u32 out_counter; + __u64 intermediate_state; + __u32 lobc; + __u32 replaybc; + __u64 pcrc64_poly; + __u32 crc32; + __u32 adler_xxhash32; + __u64 pcrc64_xorout; + __u32 out_buf_size; + __u32 in_buf_size; + __u64 in_pcrc64; + __u64 out_pcrc64; + __u32 lobs; + __u32 libc; + __u64 reserved; + __u32 xxhash_state[4]; + __u32 cleartext[4]; +}; + +struct icp_qat_fw_comp_resp { + struct icp_qat_fw_comn_resp_hdr comn_resp; + __u64 opaque_data; + struct icp_qat_fw_resp_comp_pars comp_resp_pars; +}; + +#define QAT_FW_COMP_BANK_FLAG_MASK 0x1 +#define QAT_FW_COMP_BANK_I_BITPOS 8 +#define QAT_FW_COMP_BANK_H_BITPOS 7 +#define QAT_FW_COMP_BANK_G_BITPOS 6 +#define QAT_FW_COMP_BANK_F_BITPOS 5 +#define QAT_FW_COMP_BANK_E_BITPOS 4 +#define QAT_FW_COMP_BANK_D_BITPOS 3 +#define QAT_FW_COMP_BANK_C_BITPOS 2 +#define QAT_FW_COMP_BANK_B_BITPOS 1 +#define QAT_FW_COMP_BANK_A_BITPOS 0 + +enum icp_qat_fw_comp_bank_enabled { + ICP_QAT_FW_COMP_BANK_DISABLED = 0, + ICP_QAT_FW_COMP_BANK_ENABLED = 1, + ICP_QAT_FW_COMP_BANK_DELIMITER = 2 +}; + +#define ICP_QAT_FW_COMP_RAM_FLAGS_BUILD(bank_i_enable, bank_h_enable, \ + bank_g_enable, bank_f_enable, \ + bank_e_enable, bank_d_enable, \ + bank_c_enable, bank_b_enable, \ + bank_a_enable) \ + ((((bank_i_enable) & QAT_FW_COMP_BANK_FLAG_MASK) << \ + QAT_FW_COMP_BANK_I_BITPOS) | \ + (((bank_h_enable) & QAT_FW_COMP_BANK_FLAG_MASK) << \ + QAT_FW_COMP_BANK_H_BITPOS) | \ + (((bank_g_enable) & QAT_FW_COMP_BANK_FLAG_MASK) << \ + QAT_FW_COMP_BANK_G_BITPOS) | \ + (((bank_f_enable) & QAT_FW_COMP_BANK_FLAG_MASK) << \ + QAT_FW_COMP_BANK_F_BITPOS) | \ + (((bank_e_enable) & QAT_FW_COMP_BANK_FLAG_MASK) << \ + QAT_FW_COMP_BANK_E_BITPOS) | \ + (((bank_d_enable) & QAT_FW_COMP_BANK_FLAG_MASK) << \ + QAT_FW_COMP_BANK_D_BITPOS) | \ + (((bank_c_enable) & QAT_FW_COMP_BANK_FLAG_MASK) << \ + QAT_FW_COMP_BANK_C_BITPOS) | \ + (((bank_b_enable) & QAT_FW_COMP_BANK_FLAG_MASK) << \ + QAT_FW_COMP_BANK_B_BITPOS) | \ + (((bank_a_enable) & QAT_FW_COMP_BANK_FLAG_MASK) << \ + QAT_FW_COMP_BANK_A_BITPOS)) + +struct icp_qat_fw_comp_crc_data_struct { + __u32 crc32; + union { + __u32 adler; + __u32 xxhash; + } adler_xxhash_u; + __u32 cpr_in_crc_lo; + __u32 cpr_in_crc_hi; + __u32 cpr_out_crc_lo; + __u32 cpr_out_crc_hi; + __u32 xlt_in_crc_lo; + __u32 xlt_in_crc_hi; + __u32 xlt_out_crc_lo; + __u32 xlt_out_crc_hi; + __u32 prog_crc_poly_lo; + __u32 prog_crc_poly_hi; + __u32 xor_out_lo; + __u32 xor_out_hi; + __u32 append_crc_lo; + __u32 append_crc_hi; +}; + +struct xxhash_acc_state_buff { + __u32 in_counter; + __u32 out_counter; + __u32 xxhash_state[4]; + __u32 clear_txt[4]; +}; + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_init_admin.h b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_init_admin.h new file mode 100644 index 000000000..95f7f514c --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_init_admin.h @@ -0,0 +1,223 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#ifndef _ICP_QAT_FW_INIT_ADMIN_H_ +#define _ICP_QAT_FW_INIT_ADMIN_H_ + +#include "icp_qat_fw.h" + +#define RL_MAX_RP_IDS 16 + +enum icp_qat_fw_init_admin_cmd_id { + ICP_QAT_FW_INIT_AE = 0, + ICP_QAT_FW_TRNG_ENABLE = 1, + ICP_QAT_FW_TRNG_DISABLE = 2, + ICP_QAT_FW_CONSTANTS_CFG = 3, + ICP_QAT_FW_STATUS_GET = 4, + ICP_QAT_FW_COUNTERS_GET = 5, + ICP_QAT_FW_LOOPBACK = 6, + ICP_QAT_FW_HEARTBEAT_SYNC = 7, + ICP_QAT_FW_HEARTBEAT_GET = 8, + ICP_QAT_FW_COMP_CAPABILITY_GET = 9, + ICP_QAT_FW_CRYPTO_CAPABILITY_GET = 10, + ICP_QAT_FW_DC_CHAIN_INIT = 11, + ICP_QAT_FW_HEARTBEAT_TIMER_SET = 13, + ICP_QAT_FW_RL_INIT = 15, + ICP_QAT_FW_TIMER_GET = 19, + ICP_QAT_FW_CNV_STATS_GET = 20, + ICP_QAT_FW_PM_STATE_CONFIG = 128, + ICP_QAT_FW_PM_INFO = 129, + ICP_QAT_FW_RL_ADD = 134, + ICP_QAT_FW_RL_UPDATE = 135, + ICP_QAT_FW_RL_REMOVE = 136, + ICP_QAT_FW_TL_START = 137, + ICP_QAT_FW_TL_STOP = 138, + ICP_QAT_FW_KPT_ENABLE = 144, + ICP_QAT_FW_SVN_READ = 146, + ICP_QAT_FW_SVN_COMMIT = 147, +}; + +enum icp_qat_fw_init_admin_resp_status { + ICP_QAT_FW_INIT_RESP_STATUS_SUCCESS = 0, + ICP_QAT_FW_INIT_RESP_STATUS_FAIL = 1, + ICP_QAT_FW_INIT_RESP_STATUS_RETRY = 2, + ICP_QAT_FW_INIT_RESP_STATUS_UNSUPPORTED = 4, +}; + +struct icp_qat_fw_init_admin_tl_rp_indexes { + __u8 rp_num_index_0; + __u8 rp_num_index_1; + __u8 rp_num_index_2; + __u8 rp_num_index_3; +}; + +struct icp_qat_fw_init_admin_slice_cnt { + __u8 cpr_cnt; + __u8 xlt_cnt; + __u8 dcpr_cnt; + __u8 pke_cnt; + __u8 wat_cnt; + __u8 wcp_cnt; + __u8 ucs_cnt; + __u8 cph_cnt; + __u8 ath_cnt; +}; + +struct icp_qat_fw_init_admin_sla_config_params { + __u32 pcie_in_cir; + __u32 pcie_in_pir; + __u32 pcie_out_cir; + __u32 pcie_out_pir; + __u32 slice_util_cir; + __u32 slice_util_pir; + __u32 ae_util_cir; + __u32 ae_util_pir; + __u16 rp_ids[RL_MAX_RP_IDS]; +}; + +struct icp_qat_fw_init_admin_req { + __u16 init_cfg_sz; + __u8 resrvd1; + __u8 cmd_id; + __u32 resrvd2; + __u64 opaque_data; + __u64 init_cfg_ptr; + + union { + struct { + __u16 ibuf_size_in_kb; + __u16 resrvd3; + }; + struct { + __u32 int_timer_ticks; + }; + struct { + __u32 heartbeat_ticks; + }; + struct { + __u16 node_id; + __u8 node_type; + __u8 svc_type; + __u8 resrvd5[3]; + __u8 rp_count; + }; + __u32 idle_filter; + struct icp_qat_fw_init_admin_tl_rp_indexes rp_indexes; + }; + + __u32 resrvd4; +} __packed; + +struct icp_qat_fw_init_admin_resp { + __u8 flags; + __u8 resrvd1; + __u8 status; + __u8 cmd_id; + union { + __u32 resrvd2; + struct { + __u16 version_minor_num; + __u16 version_major_num; + }; + __u32 extended_features; + struct { + __u16 error_count; + __u16 latest_error; + }; + }; + __u64 opaque_data; + union { + __u32 resrvd3[ICP_QAT_FW_NUM_LONGWORDS_4]; + struct { + __u32 version_patch_num; + __u8 context_id; + __u8 ae_id; + __u16 resrvd4; + __u64 resrvd5; + }; + struct { + __u64 req_rec_count; + __u64 resp_sent_count; + }; + struct { + __u16 compression_algos; + __u16 checksum_algos; + __u32 deflate_capabilities; + __u32 resrvd6; + __u32 lzs_capabilities; + }; + struct { + __u32 cipher_algos; + __u32 hash_algos; + __u16 keygen_algos; + __u16 other; + __u16 public_key_algos; + __u16 prime_algos; + }; + struct { + __u64 timestamp; + __u64 resrvd7; + }; + struct { + __u32 successful_count; + __u32 unsuccessful_count; + __u64 resrvd8; + }; + struct icp_qat_fw_init_admin_slice_cnt slices; + __u16 fw_capabilities; + struct { + __u8 enforced_min_svn; + __u8 permanent_min_svn; + __u8 active_svn; + __u8 resrvd9; + __u16 svn_status; + __u16 resrvd10; + __u64 resrvd11; + }; + }; +} __packed; + +#define ICP_QAT_FW_SYNC ICP_QAT_FW_HEARTBEAT_SYNC +#define ICP_QAT_FW_CAPABILITIES_GET ICP_QAT_FW_CRYPTO_CAPABILITY_GET + +#define ICP_QAT_NUMBER_OF_PM_EVENTS 8 + +struct icp_qat_fw_init_admin_pm_info { + __u16 max_pwrreq; + __u16 min_pwrreq; + __u16 resvrd1; + __u8 pwr_state; + __u8 resvrd2; + __u32 fusectl0; + struct_group(event_counters, + __u32 sys_pm; + __u32 host_msg; + __u32 unknown; + __u32 local_ssm; + __u32 timer; + ); + __u32 event_log[ICP_QAT_NUMBER_OF_PM_EVENTS]; + struct_group(pm, + __u32 fw_init; + __u32 pwrreq; + __u32 status; + __u32 main; + __u32 thread; + ); + struct_group(ssm, + __u32 pm_enable; + __u32 pm_active_status; + __u32 pm_managed_status; + __u32 pm_domain_status; + __u32 active_constraint; + ); + __u32 resvrd3[6]; +}; + +struct icp_qat_fw_init_admin_kpt_cfg { + __u32 swk_cnt_per_fn; + __u32 swk_cnt_per_pasid; + __u32 swk_ttl_in_secs; + __u32 swk_shared_disable; +}; + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_la.h b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_la.h new file mode 100644 index 000000000..28fa17f14 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_la.h @@ -0,0 +1,367 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#ifndef _ICP_QAT_FW_LA_H_ +#define _ICP_QAT_FW_LA_H_ +#include "icp_qat_fw.h" + +enum icp_qat_fw_la_cmd_id { + ICP_QAT_FW_LA_CMD_CIPHER = 0, + ICP_QAT_FW_LA_CMD_AUTH = 1, + ICP_QAT_FW_LA_CMD_CIPHER_HASH = 2, + ICP_QAT_FW_LA_CMD_HASH_CIPHER = 3, + ICP_QAT_FW_LA_CMD_TRNG_GET_RANDOM = 4, + ICP_QAT_FW_LA_CMD_TRNG_TEST = 5, + ICP_QAT_FW_LA_CMD_SSL3_KEY_DERIVE = 6, + ICP_QAT_FW_LA_CMD_TLS_V1_1_KEY_DERIVE = 7, + ICP_QAT_FW_LA_CMD_TLS_V1_2_KEY_DERIVE = 8, + ICP_QAT_FW_LA_CMD_MGF1 = 9, + ICP_QAT_FW_LA_CMD_AUTH_PRE_COMP = 10, + ICP_QAT_FW_LA_CMD_CIPHER_PRE_COMP = 11, + ICP_QAT_FW_LA_CMD_DELIMITER = 12 +}; + +#define ICP_QAT_FW_LA_ICV_VER_STATUS_PASS ICP_QAT_FW_COMN_STATUS_FLAG_OK +#define ICP_QAT_FW_LA_ICV_VER_STATUS_FAIL ICP_QAT_FW_COMN_STATUS_FLAG_ERROR +#define ICP_QAT_FW_LA_TRNG_STATUS_PASS ICP_QAT_FW_COMN_STATUS_FLAG_OK +#define ICP_QAT_FW_LA_TRNG_STATUS_FAIL ICP_QAT_FW_COMN_STATUS_FLAG_ERROR + +struct icp_qat_fw_la_bulk_req { + struct icp_qat_fw_comn_req_hdr comn_hdr; + struct icp_qat_fw_comn_req_hdr_cd_pars cd_pars; + struct icp_qat_fw_comn_req_mid comn_mid; + struct icp_qat_fw_comn_req_rqpars serv_specif_rqpars; + struct icp_qat_fw_comn_req_cd_ctrl cd_ctrl; +}; + +#define ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE 1 +#define QAT_LA_SLICE_TYPE_BITPOS 14 +#define QAT_LA_SLICE_TYPE_MASK 0x3 +#define ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS 1 +#define ICP_QAT_FW_LA_GCM_IV_LEN_NOT_12_OCTETS 0 +#define QAT_FW_LA_ZUC_3G_PROTO_FLAG_BITPOS 12 +#define ICP_QAT_FW_LA_ZUC_3G_PROTO 1 +#define QAT_FW_LA_ZUC_3G_PROTO_FLAG_MASK 0x1 +#define QAT_LA_GCM_IV_LEN_FLAG_BITPOS 11 +#define QAT_LA_GCM_IV_LEN_FLAG_MASK 0x1 +#define ICP_QAT_FW_LA_DIGEST_IN_BUFFER 1 +#define ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER 0 +#define QAT_LA_DIGEST_IN_BUFFER_BITPOS 10 +#define QAT_LA_DIGEST_IN_BUFFER_MASK 0x1 +#define ICP_QAT_FW_LA_SNOW_3G_PROTO 4 +#define ICP_QAT_FW_LA_GCM_PROTO 2 +#define ICP_QAT_FW_LA_CCM_PROTO 1 +#define ICP_QAT_FW_LA_NO_PROTO 0 +#define QAT_LA_PROTO_BITPOS 7 +#define QAT_LA_PROTO_MASK 0x7 +#define ICP_QAT_FW_LA_CMP_AUTH_RES 1 +#define ICP_QAT_FW_LA_NO_CMP_AUTH_RES 0 +#define QAT_LA_CMP_AUTH_RES_BITPOS 6 +#define QAT_LA_CMP_AUTH_RES_MASK 0x1 +#define ICP_QAT_FW_LA_RET_AUTH_RES 1 +#define ICP_QAT_FW_LA_NO_RET_AUTH_RES 0 +#define QAT_LA_RET_AUTH_RES_BITPOS 5 +#define QAT_LA_RET_AUTH_RES_MASK 0x1 +#define ICP_QAT_FW_LA_UPDATE_STATE 1 +#define ICP_QAT_FW_LA_NO_UPDATE_STATE 0 +#define QAT_LA_UPDATE_STATE_BITPOS 4 +#define QAT_LA_UPDATE_STATE_MASK 0x1 +#define ICP_QAT_FW_CIPH_AUTH_CFG_OFFSET_IN_CD_SETUP 0 +#define ICP_QAT_FW_CIPH_AUTH_CFG_OFFSET_IN_SHRAM_CP 1 +#define QAT_LA_CIPH_AUTH_CFG_OFFSET_BITPOS 3 +#define QAT_LA_CIPH_AUTH_CFG_OFFSET_MASK 0x1 +#define ICP_QAT_FW_CIPH_IV_64BIT_PTR 0 +#define ICP_QAT_FW_CIPH_IV_16BYTE_DATA 1 +#define QAT_LA_CIPH_IV_FLD_BITPOS 2 +#define QAT_LA_CIPH_IV_FLD_MASK 0x1 +#define ICP_QAT_FW_LA_PARTIAL_NONE 0 +#define ICP_QAT_FW_LA_PARTIAL_START 1 +#define ICP_QAT_FW_LA_PARTIAL_MID 3 +#define ICP_QAT_FW_LA_PARTIAL_END 2 +#define QAT_LA_PARTIAL_BITPOS 0 +#define QAT_LA_PARTIAL_MASK 0x3 +#define ICP_QAT_FW_LA_FLAGS_BUILD(zuc_proto, gcm_iv_len, auth_rslt, proto, \ + cmp_auth, ret_auth, update_state, \ + ciph_iv, ciphcfg, partial) \ + (((zuc_proto & QAT_FW_LA_ZUC_3G_PROTO_FLAG_MASK) << \ + QAT_FW_LA_ZUC_3G_PROTO_FLAG_BITPOS) | \ + ((gcm_iv_len & QAT_LA_GCM_IV_LEN_FLAG_MASK) << \ + QAT_LA_GCM_IV_LEN_FLAG_BITPOS) | \ + ((auth_rslt & QAT_LA_DIGEST_IN_BUFFER_MASK) << \ + QAT_LA_DIGEST_IN_BUFFER_BITPOS) | \ + ((proto & QAT_LA_PROTO_MASK) << \ + QAT_LA_PROTO_BITPOS) | \ + ((cmp_auth & QAT_LA_CMP_AUTH_RES_MASK) << \ + QAT_LA_CMP_AUTH_RES_BITPOS) | \ + ((ret_auth & QAT_LA_RET_AUTH_RES_MASK) << \ + QAT_LA_RET_AUTH_RES_BITPOS) | \ + ((update_state & QAT_LA_UPDATE_STATE_MASK) << \ + QAT_LA_UPDATE_STATE_BITPOS) | \ + ((ciph_iv & QAT_LA_CIPH_IV_FLD_MASK) << \ + QAT_LA_CIPH_IV_FLD_BITPOS) | \ + ((ciphcfg & QAT_LA_CIPH_AUTH_CFG_OFFSET_MASK) << \ + QAT_LA_CIPH_AUTH_CFG_OFFSET_BITPOS) | \ + ((partial & QAT_LA_PARTIAL_MASK) << \ + QAT_LA_PARTIAL_BITPOS)) + +#define ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_GET(flags) \ + QAT_FIELD_GET(flags, QAT_LA_CIPH_IV_FLD_BITPOS, \ + QAT_LA_CIPH_IV_FLD_MASK) + +#define ICP_QAT_FW_LA_CIPH_AUTH_CFG_OFFSET_FLAG_GET(flags) \ + QAT_FIELD_GET(flags, QAT_LA_CIPH_AUTH_CFG_OFFSET_BITPOS, \ + QAT_LA_CIPH_AUTH_CFG_OFFSET_MASK) + +#define ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_GET(flags) \ + QAT_FIELD_GET(flags, QAT_FW_LA_ZUC_3G_PROTO_FLAG_BITPOS, \ + QAT_FW_LA_ZUC_3G_PROTO_FLAG_MASK) + +#define ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_GET(flags) \ + QAT_FIELD_GET(flags, QAT_LA_GCM_IV_LEN_FLAG_BITPOS, \ + QAT_LA_GCM_IV_LEN_FLAG_MASK) + +#define ICP_QAT_FW_LA_PROTO_GET(flags) \ + QAT_FIELD_GET(flags, QAT_LA_PROTO_BITPOS, QAT_LA_PROTO_MASK) + +#define ICP_QAT_FW_LA_CMP_AUTH_GET(flags) \ + QAT_FIELD_GET(flags, QAT_LA_CMP_AUTH_RES_BITPOS, \ + QAT_LA_CMP_AUTH_RES_MASK) + +#define ICP_QAT_FW_LA_RET_AUTH_GET(flags) \ + QAT_FIELD_GET(flags, QAT_LA_RET_AUTH_RES_BITPOS, \ + QAT_LA_RET_AUTH_RES_MASK) + +#define ICP_QAT_FW_LA_DIGEST_IN_BUFFER_GET(flags) \ + QAT_FIELD_GET(flags, QAT_LA_DIGEST_IN_BUFFER_BITPOS, \ + QAT_LA_DIGEST_IN_BUFFER_MASK) + +#define ICP_QAT_FW_LA_UPDATE_STATE_GET(flags) \ + QAT_FIELD_GET(flags, QAT_LA_UPDATE_STATE_BITPOS, \ + QAT_LA_UPDATE_STATE_MASK) + +#define ICP_QAT_FW_LA_PARTIAL_GET(flags) \ + QAT_FIELD_GET(flags, QAT_LA_PARTIAL_BITPOS, \ + QAT_LA_PARTIAL_MASK) + +#define ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_LA_CIPH_IV_FLD_BITPOS, \ + QAT_LA_CIPH_IV_FLD_MASK) + +#define ICP_QAT_FW_LA_CIPH_AUTH_CFG_OFFSET_FLAG_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_LA_CIPH_AUTH_CFG_OFFSET_BITPOS, \ + QAT_LA_CIPH_AUTH_CFG_OFFSET_MASK) + +#define ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_FW_LA_ZUC_3G_PROTO_FLAG_BITPOS, \ + QAT_FW_LA_ZUC_3G_PROTO_FLAG_MASK) + +#define ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_LA_GCM_IV_LEN_FLAG_BITPOS, \ + QAT_LA_GCM_IV_LEN_FLAG_MASK) + +#define ICP_QAT_FW_LA_PROTO_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_LA_PROTO_BITPOS, \ + QAT_LA_PROTO_MASK) + +#define ICP_QAT_FW_LA_CMP_AUTH_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_LA_CMP_AUTH_RES_BITPOS, \ + QAT_LA_CMP_AUTH_RES_MASK) + +#define ICP_QAT_FW_LA_RET_AUTH_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_LA_RET_AUTH_RES_BITPOS, \ + QAT_LA_RET_AUTH_RES_MASK) + +#define ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_LA_DIGEST_IN_BUFFER_BITPOS, \ + QAT_LA_DIGEST_IN_BUFFER_MASK) + +#define ICP_QAT_FW_LA_UPDATE_STATE_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_LA_UPDATE_STATE_BITPOS, \ + QAT_LA_UPDATE_STATE_MASK) + +#define ICP_QAT_FW_LA_PARTIAL_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_LA_PARTIAL_BITPOS, \ + QAT_LA_PARTIAL_MASK) + +#define ICP_QAT_FW_LA_SLICE_TYPE_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_LA_SLICE_TYPE_BITPOS, \ + QAT_LA_SLICE_TYPE_MASK) + +struct icp_qat_fw_cipher_req_hdr_cd_pars { + union { + struct { + __u64 content_desc_addr; + __u16 content_desc_resrvd1; + __u8 content_desc_params_sz; + __u8 content_desc_hdr_resrvd2; + __u32 content_desc_resrvd3; + } s; + struct { + __u32 cipher_key_array[ICP_QAT_FW_NUM_LONGWORDS_4]; + } s1; + } u; +}; + +struct icp_qat_fw_cipher_auth_req_hdr_cd_pars { + union { + struct { + __u64 content_desc_addr; + __u16 content_desc_resrvd1; + __u8 content_desc_params_sz; + __u8 content_desc_hdr_resrvd2; + __u32 content_desc_resrvd3; + } s; + struct { + __u32 cipher_key_array[ICP_QAT_FW_NUM_LONGWORDS_4]; + } sl; + } u; +}; + +struct icp_qat_fw_cipher_cd_ctrl_hdr { + __u8 cipher_state_sz; + __u8 cipher_key_sz; + __u8 cipher_cfg_offset; + __u8 next_curr_id; + __u8 cipher_padding_sz; + __u8 resrvd1; + __u16 resrvd2; + __u32 resrvd3[ICP_QAT_FW_NUM_LONGWORDS_3]; +}; + +struct icp_qat_fw_auth_cd_ctrl_hdr { + __u32 resrvd1; + __u8 resrvd2; + __u8 hash_flags; + __u8 hash_cfg_offset; + __u8 next_curr_id; + __u8 resrvd3; + __u8 outer_prefix_sz; + __u8 final_sz; + __u8 inner_res_sz; + __u8 resrvd4; + __u8 inner_state1_sz; + __u8 inner_state2_offset; + __u8 inner_state2_sz; + __u8 outer_config_offset; + __u8 outer_state1_sz; + __u8 outer_res_sz; + __u8 outer_prefix_offset; +}; + +struct icp_qat_fw_cipher_auth_cd_ctrl_hdr { + __u8 cipher_state_sz; + __u8 cipher_key_sz; + __u8 cipher_cfg_offset; + __u8 next_curr_id_cipher; + __u8 cipher_padding_sz; + __u8 hash_flags; + __u8 hash_cfg_offset; + __u8 next_curr_id_auth; + __u8 resrvd1; + __u8 outer_prefix_sz; + __u8 final_sz; + __u8 inner_res_sz; + __u8 resrvd2; + __u8 inner_state1_sz; + __u8 inner_state2_offset; + __u8 inner_state2_sz; + __u8 outer_config_offset; + __u8 outer_state1_sz; + __u8 outer_res_sz; + __u8 outer_prefix_offset; +}; + +#define ICP_QAT_FW_AUTH_HDR_FLAG_DO_NESTED 1 +#define ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED 0 +#define ICP_QAT_FW_CCM_GCM_AAD_SZ_MAX 240 +#define ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET \ + (sizeof(struct icp_qat_fw_la_cipher_req_params_t)) +#define ICP_QAT_FW_CIPHER_REQUEST_PARAMETERS_OFFSET (0) + +struct icp_qat_fw_la_cipher_req_params { + __u32 cipher_offset; + __u32 cipher_length; + union { + __u32 cipher_IV_array[ICP_QAT_FW_NUM_LONGWORDS_4]; + struct { + __u64 cipher_IV_ptr; + __u64 resrvd1; + } s; + } u; +}; + +struct icp_qat_fw_la_auth_req_params { + __u32 auth_off; + __u32 auth_len; + union { + __u64 auth_partial_st_prefix; + __u64 aad_adr; + } u1; + __u64 auth_res_addr; + union { + __u8 inner_prefix_sz; + __u8 aad_sz; + } u2; + __u8 resrvd1; + __u8 hash_state_sz; + __u8 auth_res_sz; +} __packed; + +struct icp_qat_fw_la_auth_req_params_resrvd_flds { + __u32 resrvd[ICP_QAT_FW_NUM_LONGWORDS_6]; + union { + __u8 inner_prefix_sz; + __u8 aad_sz; + } u2; + __u8 resrvd1; + __u16 resrvd2; +}; + +struct icp_qat_fw_la_resp { + struct icp_qat_fw_comn_resp_hdr comn_resp; + __u64 opaque_data; + __u32 resrvd[ICP_QAT_FW_NUM_LONGWORDS_4]; +}; + +#define ICP_QAT_FW_CIPHER_NEXT_ID_GET(cd_ctrl_hdr_t) \ + ((((cd_ctrl_hdr_t)->next_curr_id_cipher) & \ + ICP_QAT_FW_COMN_NEXT_ID_MASK) >> (ICP_QAT_FW_COMN_NEXT_ID_BITPOS)) + +#define ICP_QAT_FW_CIPHER_NEXT_ID_SET(cd_ctrl_hdr_t, val) \ +{ (cd_ctrl_hdr_t)->next_curr_id_cipher = \ + ((((cd_ctrl_hdr_t)->next_curr_id_cipher) \ + & ICP_QAT_FW_COMN_CURR_ID_MASK) | \ + ((val << ICP_QAT_FW_COMN_NEXT_ID_BITPOS) \ + & ICP_QAT_FW_COMN_NEXT_ID_MASK)) } + +#define ICP_QAT_FW_CIPHER_CURR_ID_GET(cd_ctrl_hdr_t) \ + (((cd_ctrl_hdr_t)->next_curr_id_cipher) \ + & ICP_QAT_FW_COMN_CURR_ID_MASK) + +#define ICP_QAT_FW_CIPHER_CURR_ID_SET(cd_ctrl_hdr_t, val) \ +{ (cd_ctrl_hdr_t)->next_curr_id_cipher = \ + ((((cd_ctrl_hdr_t)->next_curr_id_cipher) \ + & ICP_QAT_FW_COMN_NEXT_ID_MASK) | \ + ((val) & ICP_QAT_FW_COMN_CURR_ID_MASK)) } + +#define ICP_QAT_FW_AUTH_NEXT_ID_GET(cd_ctrl_hdr_t) \ + ((((cd_ctrl_hdr_t)->next_curr_id_auth) & ICP_QAT_FW_COMN_NEXT_ID_MASK) \ + >> (ICP_QAT_FW_COMN_NEXT_ID_BITPOS)) + +#define ICP_QAT_FW_AUTH_NEXT_ID_SET(cd_ctrl_hdr_t, val) \ +{ (cd_ctrl_hdr_t)->next_curr_id_auth = \ + ((((cd_ctrl_hdr_t)->next_curr_id_auth) \ + & ICP_QAT_FW_COMN_CURR_ID_MASK) | \ + ((val << ICP_QAT_FW_COMN_NEXT_ID_BITPOS) \ + & ICP_QAT_FW_COMN_NEXT_ID_MASK)) } + +#define ICP_QAT_FW_AUTH_CURR_ID_GET(cd_ctrl_hdr_t) \ + (((cd_ctrl_hdr_t)->next_curr_id_auth) \ + & ICP_QAT_FW_COMN_CURR_ID_MASK) + +#define ICP_QAT_FW_AUTH_CURR_ID_SET(cd_ctrl_hdr_t, val) \ +{ (cd_ctrl_hdr_t)->next_curr_id_auth = \ + ((((cd_ctrl_hdr_t)->next_curr_id_auth) \ + & ICP_QAT_FW_COMN_NEXT_ID_MASK) | \ + ((val) & ICP_QAT_FW_COMN_CURR_ID_MASK)) } + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h new file mode 100644 index 000000000..e74cafa95 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#ifndef __ICP_QAT_FW_LOADER_HANDLE_H__ +#define __ICP_QAT_FW_LOADER_HANDLE_H__ +#include "icp_qat_uclo.h" + +struct icp_qat_fw_loader_ae_data { + unsigned int state; + unsigned int ustore_size; + unsigned int free_addr; + unsigned int free_size; + unsigned int live_ctx_mask; +}; + +struct icp_qat_fw_loader_hal_handle { + struct icp_qat_fw_loader_ae_data aes[ICP_QAT_UCLO_MAX_AE]; + unsigned int ae_mask; + unsigned int admin_ae_mask; + unsigned int slice_mask; + unsigned int revision_id; + unsigned int ae_max_num; + unsigned int upc_mask; + unsigned int max_ustore; +}; + +struct icp_qat_fw_loader_chip_info { + int mmp_sram_size; + bool nn; + bool lm2lm3; + u16 reset_delay_us; + u32 lm_size; + u32 icp_rst_csr; + u32 icp_rst_mask; + u32 glb_clk_enable_csr; + u32 misc_ctl_csr; + u32 wakeup_event_val; + bool fw_auth; + bool css_3k; + bool dual_sign; + bool tgroup_share_ustore; + u32 fcu_ctl_csr; + u32 fcu_sts_csr; + u32 fcu_dram_addr_hi; + u32 fcu_dram_addr_lo; + u32 fcu_loaded_ae_csr; + u8 fcu_loaded_ae_pos; +}; + +struct icp_qat_fw_loader_handle { + struct icp_qat_fw_loader_hal_handle *hal_handle; + struct icp_qat_fw_loader_chip_info *chip_info; + struct pci_dev *pci_dev; + void *obj_handle; + void *sobj_handle; + void *mobj_handle; + unsigned int cfg_ae_mask; + void __iomem *hal_sram_addr_v; + void __iomem *hal_cap_g_ctl_csr_addr_v; + void __iomem *hal_cap_ae_xfer_csr_addr_v; + void __iomem *hal_cap_ae_local_csr_addr_v; + void __iomem *hal_ep_csr_addr_v; +}; + +struct icp_firml_dram_desc { + void __iomem *dram_base_addr; + void *dram_base_addr_v; + dma_addr_t dram_bus_addr; + u64 dram_size; +}; +#endif diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_pke.h b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_pke.h new file mode 100644 index 000000000..9dddae000 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_pke.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#ifndef _ICP_QAT_FW_PKE_ +#define _ICP_QAT_FW_PKE_ + +#include "icp_qat_fw.h" + +struct icp_qat_fw_req_hdr_pke_cd_pars { + __u64 content_desc_addr; + __u32 content_desc_resrvd; + __u32 func_id; +}; + +struct icp_qat_fw_req_pke_mid { + __u64 opaque; + __u64 src_data_addr; + __u64 dest_data_addr; +}; + +struct icp_qat_fw_req_pke_hdr { + __u8 resrvd1; + __u8 resrvd2; + __u8 service_type; + __u8 hdr_flags; + __u16 comn_req_flags; + __u16 resrvd4; + struct icp_qat_fw_req_hdr_pke_cd_pars cd_pars; +}; + +struct icp_qat_fw_pke_request { + struct icp_qat_fw_req_pke_hdr pke_hdr; + struct icp_qat_fw_req_pke_mid pke_mid; + __u8 output_param_count; + __u8 input_param_count; + __u16 resrvd1; + __u32 resrvd2; + __u64 next_req_adr; +}; + +struct icp_qat_fw_resp_pke_hdr { + __u8 resrvd1; + __u8 resrvd2; + __u8 response_type; + __u8 hdr_flags; + __u16 comn_resp_flags; + __u16 resrvd4; +}; + +struct icp_qat_fw_pke_resp { + struct icp_qat_fw_resp_pke_hdr pke_resp_hdr; + __u64 opaque; + __u64 src_data_addr; + __u64 dest_data_addr; +}; + +#define ICP_QAT_FW_PKE_HDR_VALID_FLAG_BITPOS 7 +#define ICP_QAT_FW_PKE_HDR_VALID_FLAG_MASK 0x1 +#define ICP_QAT_FW_PKE_RESP_PKE_STAT_GET(status_word) \ + QAT_FIELD_GET(((status_word >> ICP_QAT_FW_COMN_ONE_BYTE_SHIFT) & \ + ICP_QAT_FW_COMN_SINGLE_BYTE_MASK), \ + QAT_COMN_RESP_PKE_STATUS_BITPOS, \ + QAT_COMN_RESP_PKE_STATUS_MASK) + +#define ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(hdr_t, val) \ + QAT_FIELD_SET((hdr_t.hdr_flags), (val), \ + ICP_QAT_FW_PKE_HDR_VALID_FLAG_BITPOS, \ + ICP_QAT_FW_PKE_HDR_VALID_FLAG_MASK) +#endif diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_hal.h b/drivers/crypto/intel/qat/qat_common/icp_qat_hal.h new file mode 100644 index 000000000..20b2ee1fc --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_hal.h @@ -0,0 +1,143 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#ifndef __ICP_QAT_HAL_H +#define __ICP_QAT_HAL_H +#include "icp_qat_fw_loader_handle.h" + +enum hal_global_csr { + MISC_CONTROL = 0xA04, + ICP_RESET = 0xA0c, + ICP_GLOBAL_CLK_ENABLE = 0xA50 +}; + +enum { + MISC_CONTROL_C4XXX = 0xAA0, + ICP_RESET_CPP0 = 0x938, + ICP_RESET_CPP1 = 0x93c, + ICP_GLOBAL_CLK_ENABLE_CPP0 = 0x964, + ICP_GLOBAL_CLK_ENABLE_CPP1 = 0x968 +}; + +enum hal_ae_csr { + USTORE_ADDRESS = 0x000, + USTORE_DATA_LOWER = 0x004, + USTORE_DATA_UPPER = 0x008, + ALU_OUT = 0x010, + CTX_ARB_CNTL = 0x014, + CTX_ENABLES = 0x018, + CC_ENABLE = 0x01c, + CSR_CTX_POINTER = 0x020, + CTX_STS_INDIRECT = 0x040, + ACTIVE_CTX_STATUS = 0x044, + CTX_SIG_EVENTS_INDIRECT = 0x048, + CTX_SIG_EVENTS_ACTIVE = 0x04c, + CTX_WAKEUP_EVENTS_INDIRECT = 0x050, + LM_ADDR_0_INDIRECT = 0x060, + LM_ADDR_1_INDIRECT = 0x068, + LM_ADDR_2_INDIRECT = 0x0cc, + LM_ADDR_3_INDIRECT = 0x0d4, + INDIRECT_LM_ADDR_0_BYTE_INDEX = 0x0e0, + INDIRECT_LM_ADDR_1_BYTE_INDEX = 0x0e8, + INDIRECT_LM_ADDR_2_BYTE_INDEX = 0x10c, + INDIRECT_LM_ADDR_3_BYTE_INDEX = 0x114, + INDIRECT_T_INDEX = 0x0f8, + INDIRECT_T_INDEX_BYTE_INDEX = 0x0fc, + FUTURE_COUNT_SIGNAL_INDIRECT = 0x078, + TIMESTAMP_LOW = 0x0c0, + TIMESTAMP_HIGH = 0x0c4, + PROFILE_COUNT = 0x144, + SIGNATURE_ENABLE = 0x150, + AE_MISC_CONTROL = 0x160, + LOCAL_CSR_STATUS = 0x180, +}; + +enum fcu_csr { + FCU_CONTROL = 0x8c0, + FCU_STATUS = 0x8c4, + FCU_STATUS1 = 0x8c8, + FCU_DRAM_ADDR_LO = 0x8cc, + FCU_DRAM_ADDR_HI = 0x8d0, + FCU_RAMBASE_ADDR_HI = 0x8d4, + FCU_RAMBASE_ADDR_LO = 0x8d8 +}; + +enum fcu_csr_4xxx { + FCU_CONTROL_4XXX = 0x1000, + FCU_STATUS_4XXX = 0x1004, + FCU_ME_BROADCAST_MASK_TYPE = 0x1008, + FCU_AE_LOADED_4XXX = 0x1010, + FCU_DRAM_ADDR_LO_4XXX = 0x1014, + FCU_DRAM_ADDR_HI_4XXX = 0x1018, +}; + +enum fcu_cmd { + FCU_CTRL_CMD_NOOP = 0, + FCU_CTRL_CMD_AUTH = 1, + FCU_CTRL_CMD_LOAD = 2, + FCU_CTRL_CMD_START = 3 +}; + +enum fcu_sts { + FCU_STS_NO_STS = 0, + FCU_STS_VERI_DONE = 1, + FCU_STS_LOAD_DONE = 2, + FCU_STS_VERI_FAIL = 3, + FCU_STS_LOAD_FAIL = 4, + FCU_STS_BUSY = 5 +}; + +#define ALL_AE_MASK 0xFFFFFFFF +#define UA_ECS (0x1 << 31) +#define ACS_ABO_BITPOS 31 +#define ACS_ACNO 0x7 +#define CE_ENABLE_BITPOS 0x8 +#define CE_LMADDR_0_GLOBAL_BITPOS 16 +#define CE_LMADDR_1_GLOBAL_BITPOS 17 +#define CE_LMADDR_2_GLOBAL_BITPOS 22 +#define CE_LMADDR_3_GLOBAL_BITPOS 23 +#define CE_T_INDEX_GLOBAL_BITPOS 21 +#define CE_NN_MODE_BITPOS 20 +#define CE_REG_PAR_ERR_BITPOS 25 +#define CE_BREAKPOINT_BITPOS 27 +#define CE_CNTL_STORE_PARITY_ERROR_BITPOS 29 +#define CE_INUSE_CONTEXTS_BITPOS 31 +#define CE_NN_MODE (0x1 << CE_NN_MODE_BITPOS) +#define CE_INUSE_CONTEXTS (0x1 << CE_INUSE_CONTEXTS_BITPOS) +#define XCWE_VOLUNTARY (0x1) +#define LCS_STATUS (0x1) +#define MMC_SHARE_CS_BITPOS 2 +#define WAKEUP_EVENT 0x10000 +#define FCU_CTRL_BROADCAST_POS 0x4 +#define FCU_CTRL_AE_POS 0x8 +#define FCU_AUTH_STS_MASK 0x7 +#define FCU_STS_DONE_POS 0x9 +#define FCU_STS_AUTHFWLD_POS 0X8 +#define FCU_LOADED_AE_POS 0x16 +#define FW_AUTH_WAIT_PERIOD 10 +#define FW_AUTH_MAX_RETRY 300 +#define ICP_QAT_AE_OFFSET 0x20000 +#define ICP_QAT_CAP_OFFSET (ICP_QAT_AE_OFFSET + 0x10000) +#define LOCAL_TO_XFER_REG_OFFSET 0x800 +#define ICP_QAT_EP_OFFSET 0x3a000 +#define ICP_QAT_EP_OFFSET_4XXX 0x200000 /* HI MMIO CSRs */ +#define ICP_QAT_AE_OFFSET_4XXX 0x600000 +#define ICP_QAT_CAP_OFFSET_4XXX 0x640000 +#define SET_CAP_CSR(handle, csr, val) \ + ADF_CSR_WR((handle)->hal_cap_g_ctl_csr_addr_v, csr, val) +#define GET_CAP_CSR(handle, csr) \ + ADF_CSR_RD((handle)->hal_cap_g_ctl_csr_addr_v, csr) +#define AE_CSR(handle, ae) \ + ((char __iomem *)(handle)->hal_cap_ae_local_csr_addr_v + ((ae) << 12)) +#define AE_CSR_ADDR(handle, ae, csr) (AE_CSR(handle, ae) + (0x3ff & (csr))) +#define SET_AE_CSR(handle, ae, csr, val) \ + ADF_CSR_WR(AE_CSR_ADDR(handle, ae, csr), 0, val) +#define GET_AE_CSR(handle, ae, csr) ADF_CSR_RD(AE_CSR_ADDR(handle, ae, csr), 0) +#define AE_XFER(handle, ae) \ + ((char __iomem *)(handle)->hal_cap_ae_xfer_csr_addr_v + ((ae) << 12)) +#define AE_XFER_ADDR(handle, ae, reg) (AE_XFER(handle, ae) + \ + (((reg) & 0xff) << 2)) +#define SET_AE_XFER(handle, ae, reg, val) \ + ADF_CSR_WR(AE_XFER_ADDR(handle, ae, reg), 0, val) +#define SRAM_WRITE(handle, addr, val) \ + ADF_CSR_WR((handle)->hal_sram_addr_v, addr, val) +#endif diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_hw.h b/drivers/crypto/intel/qat/qat_common/icp_qat_hw.h new file mode 100644 index 000000000..e38115d3c --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_hw.h @@ -0,0 +1,391 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#ifndef _ICP_QAT_HW_H_ +#define _ICP_QAT_HW_H_ + +#include + +enum icp_qat_hw_ae_id { + ICP_QAT_HW_AE_0 = 0, + ICP_QAT_HW_AE_1 = 1, + ICP_QAT_HW_AE_2 = 2, + ICP_QAT_HW_AE_3 = 3, + ICP_QAT_HW_AE_4 = 4, + ICP_QAT_HW_AE_5 = 5, + ICP_QAT_HW_AE_6 = 6, + ICP_QAT_HW_AE_7 = 7, + ICP_QAT_HW_AE_8 = 8, + ICP_QAT_HW_AE_9 = 9, + ICP_QAT_HW_AE_10 = 10, + ICP_QAT_HW_AE_11 = 11, + ICP_QAT_HW_AE_12 = 12, + ICP_QAT_HW_AE_13 = 13, + ICP_QAT_HW_AE_14 = 14, + ICP_QAT_HW_AE_15 = 15, + ICP_QAT_HW_AE_16 = 16, + ICP_QAT_HW_AE_DELIMITER = 17 +}; + +enum icp_qat_hw_qat_id { + ICP_QAT_HW_QAT_0 = 0, + ICP_QAT_HW_QAT_1 = 1, + ICP_QAT_HW_QAT_2 = 2, + ICP_QAT_HW_QAT_3 = 3, + ICP_QAT_HW_QAT_4 = 4, + ICP_QAT_HW_QAT_5 = 5, + ICP_QAT_HW_QAT_DELIMITER = 6 +}; + +enum icp_qat_hw_auth_algo { + ICP_QAT_HW_AUTH_ALGO_NULL = 0, + ICP_QAT_HW_AUTH_ALGO_SHA1 = 1, + ICP_QAT_HW_AUTH_ALGO_MD5 = 2, + ICP_QAT_HW_AUTH_ALGO_SHA224 = 3, + ICP_QAT_HW_AUTH_ALGO_SHA256 = 4, + ICP_QAT_HW_AUTH_ALGO_SHA384 = 5, + ICP_QAT_HW_AUTH_ALGO_SHA512 = 6, + ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC = 7, + ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC = 8, + ICP_QAT_HW_AUTH_ALGO_AES_F9 = 9, + ICP_QAT_HW_AUTH_ALGO_GALOIS_128 = 10, + ICP_QAT_HW_AUTH_ALGO_GALOIS_64 = 11, + ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 = 12, + ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 = 13, + ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3 = 14, + ICP_QAT_HW_AUTH_RESERVED_1 = 15, + ICP_QAT_HW_AUTH_RESERVED_2 = 16, + ICP_QAT_HW_AUTH_ALGO_SHA3_256 = 17, + ICP_QAT_HW_AUTH_RESERVED_3 = 18, + ICP_QAT_HW_AUTH_ALGO_SHA3_512 = 19, + ICP_QAT_HW_AUTH_ALGO_DELIMITER = 20 +}; + +enum icp_qat_hw_auth_mode { + ICP_QAT_HW_AUTH_MODE0 = 0, + ICP_QAT_HW_AUTH_MODE1 = 1, + ICP_QAT_HW_AUTH_MODE2 = 2, + ICP_QAT_HW_AUTH_MODE_DELIMITER = 3 +}; + +struct icp_qat_hw_auth_config { + __u32 config; + __u32 reserved; +}; + +struct icp_qat_hw_ucs_cipher_config { + __u32 val; + __u32 reserved[3]; +}; + +enum icp_qat_slice_mask { + ICP_ACCEL_MASK_CIPHER_SLICE = BIT(0), + ICP_ACCEL_MASK_AUTH_SLICE = BIT(1), + ICP_ACCEL_MASK_PKE_SLICE = BIT(2), + ICP_ACCEL_MASK_COMPRESS_SLICE = BIT(3), + ICP_ACCEL_MASK_LZS_SLICE = BIT(4), + ICP_ACCEL_MASK_EIA3_SLICE = BIT(5), + ICP_ACCEL_MASK_SHA3_SLICE = BIT(6), +}; + +enum icp_qat_capabilities_mask { + ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC = BIT(0), + ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC = BIT(1), + ICP_ACCEL_CAPABILITIES_CIPHER = BIT(2), + ICP_ACCEL_CAPABILITIES_AUTHENTICATION = BIT(3), + ICP_ACCEL_CAPABILITIES_RESERVED_1 = BIT(4), + ICP_ACCEL_CAPABILITIES_COMPRESSION = BIT(5), + /* Bit 6 is currently reserved */ + ICP_ACCEL_CAPABILITIES_5G = BIT(7), + ICP_ACCEL_CAPABILITIES_ZUC = BIT(8), + ICP_ACCEL_CAPABILITIES_SHA3 = BIT(9), + /* Bits 10-11 are currently reserved */ + ICP_ACCEL_CAPABILITIES_HKDF = BIT(12), + ICP_ACCEL_CAPABILITIES_ECEDMONT = BIT(13), + ICP_ACCEL_CAPABILITIES_EXT_ALGCHAIN = BIT(14), + ICP_ACCEL_CAPABILITIES_SHA3_EXT = BIT(15), + ICP_ACCEL_CAPABILITIES_AESGCM_SPC = BIT(16), + ICP_ACCEL_CAPABILITIES_CHACHA_POLY = BIT(17), + ICP_ACCEL_CAPABILITIES_SM2 = BIT(18), + ICP_ACCEL_CAPABILITIES_SM3 = BIT(19), + ICP_ACCEL_CAPABILITIES_SM4 = BIT(20), + /* Bit 21 is currently reserved */ + ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY = BIT(22), + ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64 = BIT(23), + ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION = BIT(24), + ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION = BIT(25), + ICP_ACCEL_CAPABILITIES_AES_V2 = BIT(26), + ICP_ACCEL_CAPABILITIES_KPT = BIT(27), + /* Bit 28 is currently reserved */ + ICP_ACCEL_CAPABILITIES_ZUC_256 = BIT(29), + ICP_ACCEL_CAPABILITIES_WIRELESS_CRYPTO_EXT = BIT(30), +}; + +#define QAT_AUTH_MODE_BITPOS 4 +#define QAT_AUTH_MODE_MASK 0xF +#define QAT_AUTH_ALGO_BITPOS 0 +#define QAT_AUTH_ALGO_MASK 0xF +#define QAT_AUTH_CMP_BITPOS 8 +#define QAT_AUTH_CMP_MASK 0x7F +#define QAT_AUTH_SHA3_PADDING_BITPOS 16 +#define QAT_AUTH_SHA3_PADDING_MASK 0x1 +#define QAT_AUTH_ALGO_SHA3_BITPOS 22 +#define QAT_AUTH_ALGO_SHA3_MASK 0x3 +#define ICP_QAT_HW_AUTH_CONFIG_BUILD(mode, algo, cmp_len) \ + (((mode & QAT_AUTH_MODE_MASK) << QAT_AUTH_MODE_BITPOS) | \ + ((algo & QAT_AUTH_ALGO_MASK) << QAT_AUTH_ALGO_BITPOS) | \ + (((algo >> 4) & QAT_AUTH_ALGO_SHA3_MASK) << \ + QAT_AUTH_ALGO_SHA3_BITPOS) | \ + (((((algo == ICP_QAT_HW_AUTH_ALGO_SHA3_256) || \ + (algo == ICP_QAT_HW_AUTH_ALGO_SHA3_512)) ? 1 : 0) \ + & QAT_AUTH_SHA3_PADDING_MASK) << QAT_AUTH_SHA3_PADDING_BITPOS) | \ + ((cmp_len & QAT_AUTH_CMP_MASK) << QAT_AUTH_CMP_BITPOS)) + +struct icp_qat_hw_auth_counter { + __be32 counter; + __u32 reserved; +}; + +#define QAT_AUTH_COUNT_MASK 0xFFFFFFFF +#define QAT_AUTH_COUNT_BITPOS 0 +#define ICP_QAT_HW_AUTH_COUNT_BUILD(val) \ + (((val) & QAT_AUTH_COUNT_MASK) << QAT_AUTH_COUNT_BITPOS) + +struct icp_qat_hw_auth_setup { + struct icp_qat_hw_auth_config auth_config; + struct icp_qat_hw_auth_counter auth_counter; +}; + +#define QAT_HW_DEFAULT_ALIGNMENT 8 +#define QAT_HW_ROUND_UP(val, n) (((val) + ((n) - 1)) & (~(n - 1))) +#define ICP_QAT_HW_NULL_STATE1_SZ 32 +#define ICP_QAT_HW_MD5_STATE1_SZ 16 +#define ICP_QAT_HW_SHA1_STATE1_SZ 20 +#define ICP_QAT_HW_SHA224_STATE1_SZ 32 +#define ICP_QAT_HW_SHA256_STATE1_SZ 32 +#define ICP_QAT_HW_SHA3_256_STATE1_SZ 32 +#define ICP_QAT_HW_SHA384_STATE1_SZ 64 +#define ICP_QAT_HW_SHA512_STATE1_SZ 64 +#define ICP_QAT_HW_SHA3_512_STATE1_SZ 64 +#define ICP_QAT_HW_SHA3_224_STATE1_SZ 28 +#define ICP_QAT_HW_SHA3_384_STATE1_SZ 48 +#define ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ 16 +#define ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ 16 +#define ICP_QAT_HW_AES_F9_STATE1_SZ 32 +#define ICP_QAT_HW_KASUMI_F9_STATE1_SZ 16 +#define ICP_QAT_HW_GALOIS_128_STATE1_SZ 16 +#define ICP_QAT_HW_SNOW_3G_UIA2_STATE1_SZ 8 +#define ICP_QAT_HW_ZUC_3G_EIA3_STATE1_SZ 8 +#define ICP_QAT_HW_NULL_STATE2_SZ 32 +#define ICP_QAT_HW_MD5_STATE2_SZ 16 +#define ICP_QAT_HW_SHA1_STATE2_SZ 20 +#define ICP_QAT_HW_SHA224_STATE2_SZ 32 +#define ICP_QAT_HW_SHA256_STATE2_SZ 32 +#define ICP_QAT_HW_SHA3_256_STATE2_SZ 0 +#define ICP_QAT_HW_SHA384_STATE2_SZ 64 +#define ICP_QAT_HW_SHA512_STATE2_SZ 64 +#define ICP_QAT_HW_SHA3_512_STATE2_SZ 0 +#define ICP_QAT_HW_SHA3_224_STATE2_SZ 0 +#define ICP_QAT_HW_SHA3_384_STATE2_SZ 0 +#define ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ 16 +#define ICP_QAT_HW_AES_CBC_MAC_KEY_SZ 16 +#define ICP_QAT_HW_AES_CCM_CBC_E_CTR0_SZ 16 +#define ICP_QAT_HW_F9_IK_SZ 16 +#define ICP_QAT_HW_F9_FK_SZ 16 +#define ICP_QAT_HW_KASUMI_F9_STATE2_SZ (ICP_QAT_HW_F9_IK_SZ + \ + ICP_QAT_HW_F9_FK_SZ) +#define ICP_QAT_HW_AES_F9_STATE2_SZ ICP_QAT_HW_KASUMI_F9_STATE2_SZ +#define ICP_QAT_HW_SNOW_3G_UIA2_STATE2_SZ 24 +#define ICP_QAT_HW_ZUC_3G_EIA3_STATE2_SZ 32 +#define ICP_QAT_HW_GALOIS_H_SZ 16 +#define ICP_QAT_HW_GALOIS_LEN_A_SZ 8 +#define ICP_QAT_HW_GALOIS_E_CTR0_SZ 16 + +struct icp_qat_hw_auth_sha512 { + struct icp_qat_hw_auth_setup inner_setup; + __u8 state1[ICP_QAT_HW_SHA512_STATE1_SZ]; + struct icp_qat_hw_auth_setup outer_setup; + __u8 state2[ICP_QAT_HW_SHA512_STATE2_SZ]; +}; + +struct icp_qat_hw_auth_algo_blk { + struct icp_qat_hw_auth_sha512 sha; +}; + +#define ICP_QAT_HW_GALOIS_LEN_A_BITPOS 0 +#define ICP_QAT_HW_GALOIS_LEN_A_MASK 0xFFFFFFFF + +enum icp_qat_hw_cipher_algo { + ICP_QAT_HW_CIPHER_ALGO_NULL = 0, + ICP_QAT_HW_CIPHER_ALGO_DES = 1, + ICP_QAT_HW_CIPHER_ALGO_3DES = 2, + ICP_QAT_HW_CIPHER_ALGO_AES128 = 3, + ICP_QAT_HW_CIPHER_ALGO_AES192 = 4, + ICP_QAT_HW_CIPHER_ALGO_AES256 = 5, + ICP_QAT_HW_CIPHER_ALGO_ARC4 = 6, + ICP_QAT_HW_CIPHER_ALGO_KASUMI = 7, + ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 = 8, + ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3 = 9, + ICP_QAT_HW_CIPHER_DELIMITER = 10 +}; + +enum icp_qat_hw_cipher_mode { + ICP_QAT_HW_CIPHER_ECB_MODE = 0, + ICP_QAT_HW_CIPHER_CBC_MODE = 1, + ICP_QAT_HW_CIPHER_CTR_MODE = 2, + ICP_QAT_HW_CIPHER_F8_MODE = 3, + ICP_QAT_HW_CIPHER_XTS_MODE = 6, + ICP_QAT_HW_CIPHER_MODE_DELIMITER = 7 +}; + +struct icp_qat_hw_cipher_config { + __u32 val; + __u32 reserved; +}; + +enum icp_qat_hw_cipher_dir { + ICP_QAT_HW_CIPHER_ENCRYPT = 0, + ICP_QAT_HW_CIPHER_DECRYPT = 1, +}; + +enum icp_qat_hw_cipher_convert { + ICP_QAT_HW_CIPHER_NO_CONVERT = 0, + ICP_QAT_HW_CIPHER_KEY_CONVERT = 1, +}; + +#define QAT_CIPHER_MODE_BITPOS 4 +#define QAT_CIPHER_MODE_MASK 0xF +#define QAT_CIPHER_ALGO_BITPOS 0 +#define QAT_CIPHER_ALGO_MASK 0xF +#define QAT_CIPHER_CONVERT_BITPOS 9 +#define QAT_CIPHER_CONVERT_MASK 0x1 +#define QAT_CIPHER_DIR_BITPOS 8 +#define QAT_CIPHER_DIR_MASK 0x1 +#define QAT_CIPHER_MODE_F8_KEY_SZ_MULT 2 +#define QAT_CIPHER_MODE_XTS_KEY_SZ_MULT 2 +#define ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, algo, convert, dir) \ + (((mode & QAT_CIPHER_MODE_MASK) << QAT_CIPHER_MODE_BITPOS) | \ + ((algo & QAT_CIPHER_ALGO_MASK) << QAT_CIPHER_ALGO_BITPOS) | \ + ((convert & QAT_CIPHER_CONVERT_MASK) << QAT_CIPHER_CONVERT_BITPOS) | \ + ((dir & QAT_CIPHER_DIR_MASK) << QAT_CIPHER_DIR_BITPOS)) +#define ICP_QAT_HW_DES_BLK_SZ 8 +#define ICP_QAT_HW_3DES_BLK_SZ 8 +#define ICP_QAT_HW_NULL_BLK_SZ 8 +#define ICP_QAT_HW_AES_BLK_SZ 16 +#define ICP_QAT_HW_KASUMI_BLK_SZ 8 +#define ICP_QAT_HW_SNOW_3G_BLK_SZ 8 +#define ICP_QAT_HW_ZUC_3G_BLK_SZ 8 +#define ICP_QAT_HW_NULL_KEY_SZ 256 +#define ICP_QAT_HW_DES_KEY_SZ 8 +#define ICP_QAT_HW_3DES_KEY_SZ 24 +#define ICP_QAT_HW_AES_128_KEY_SZ 16 +#define ICP_QAT_HW_AES_192_KEY_SZ 24 +#define ICP_QAT_HW_AES_256_KEY_SZ 32 +#define ICP_QAT_HW_AES_128_F8_KEY_SZ (ICP_QAT_HW_AES_128_KEY_SZ * \ + QAT_CIPHER_MODE_F8_KEY_SZ_MULT) +#define ICP_QAT_HW_AES_192_F8_KEY_SZ (ICP_QAT_HW_AES_192_KEY_SZ * \ + QAT_CIPHER_MODE_F8_KEY_SZ_MULT) +#define ICP_QAT_HW_AES_256_F8_KEY_SZ (ICP_QAT_HW_AES_256_KEY_SZ * \ + QAT_CIPHER_MODE_F8_KEY_SZ_MULT) +#define ICP_QAT_HW_AES_128_XTS_KEY_SZ (ICP_QAT_HW_AES_128_KEY_SZ * \ + QAT_CIPHER_MODE_XTS_KEY_SZ_MULT) +#define ICP_QAT_HW_AES_256_XTS_KEY_SZ (ICP_QAT_HW_AES_256_KEY_SZ * \ + QAT_CIPHER_MODE_XTS_KEY_SZ_MULT) +#define ICP_QAT_HW_KASUMI_KEY_SZ 16 +#define ICP_QAT_HW_KASUMI_F8_KEY_SZ (ICP_QAT_HW_KASUMI_KEY_SZ * \ + QAT_CIPHER_MODE_F8_KEY_SZ_MULT) +#define ICP_QAT_HW_AES_128_XTS_KEY_SZ (ICP_QAT_HW_AES_128_KEY_SZ * \ + QAT_CIPHER_MODE_XTS_KEY_SZ_MULT) +#define ICP_QAT_HW_AES_256_XTS_KEY_SZ (ICP_QAT_HW_AES_256_KEY_SZ * \ + QAT_CIPHER_MODE_XTS_KEY_SZ_MULT) +#define ICP_QAT_HW_ARC4_KEY_SZ 256 +#define ICP_QAT_HW_SNOW_3G_UEA2_KEY_SZ 16 +#define ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ 16 +#define ICP_QAT_HW_ZUC_3G_EEA3_KEY_SZ 16 +#define ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ 16 +#define ICP_QAT_HW_MODE_F8_NUM_REG_TO_CLEAR 2 +#define INIT_SHRAM_CONSTANTS_TABLE_SZ 1024 + +struct icp_qat_hw_cipher_aes256_f8 { + struct icp_qat_hw_cipher_config cipher_config; + __u8 key[ICP_QAT_HW_AES_256_F8_KEY_SZ]; +}; + +struct icp_qat_hw_ucs_cipher_aes256_f8 { + struct icp_qat_hw_ucs_cipher_config cipher_config; + __u8 key[ICP_QAT_HW_AES_256_F8_KEY_SZ]; +}; + +struct icp_qat_hw_cipher_algo_blk { + union { + struct icp_qat_hw_cipher_aes256_f8 aes; + struct icp_qat_hw_ucs_cipher_aes256_f8 ucs_aes; + }; +} __aligned(64); + +enum icp_qat_hw_compression_direction { + ICP_QAT_HW_COMPRESSION_DIR_COMPRESS = 0, + ICP_QAT_HW_COMPRESSION_DIR_DECOMPRESS = 1, + ICP_QAT_HW_COMPRESSION_DIR_DELIMITER = 2 +}; + +enum icp_qat_hw_compression_delayed_match { + ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DISABLED = 0, + ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_ENABLED = 1, + ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DELIMITER = 2 +}; + +enum icp_qat_hw_compression_algo { + ICP_QAT_HW_COMPRESSION_ALGO_DEFLATE = 0, + ICP_QAT_HW_COMPRESSION_ALGO_LZS = 1, + ICP_QAT_HW_COMPRESSION_ALGO_ZSTD = 2, + ICP_QAT_HW_COMPRESSION_ALGO_DELIMITER +}; + +enum icp_qat_hw_compression_depth { + ICP_QAT_HW_COMPRESSION_DEPTH_1 = 0, + ICP_QAT_HW_COMPRESSION_DEPTH_4 = 1, + ICP_QAT_HW_COMPRESSION_DEPTH_8 = 2, + ICP_QAT_HW_COMPRESSION_DEPTH_16 = 3, + ICP_QAT_HW_COMPRESSION_DEPTH_128 = 4, + ICP_QAT_HW_COMPRESSION_DEPTH_DELIMITER = 5 +}; + +enum icp_qat_hw_compression_file_type { + ICP_QAT_HW_COMPRESSION_FILE_TYPE_0 = 0, + ICP_QAT_HW_COMPRESSION_FILE_TYPE_1 = 1, + ICP_QAT_HW_COMPRESSION_FILE_TYPE_2 = 2, + ICP_QAT_HW_COMPRESSION_FILE_TYPE_3 = 3, + ICP_QAT_HW_COMPRESSION_FILE_TYPE_4 = 4, + ICP_QAT_HW_COMPRESSION_FILE_TYPE_DELIMITER = 5 +}; + +struct icp_qat_hw_compression_config { + __u32 lower_val; + __u32 upper_val; +}; + +#define QAT_COMPRESSION_DIR_BITPOS 4 +#define QAT_COMPRESSION_DIR_MASK 0x7 +#define QAT_COMPRESSION_DELAYED_MATCH_BITPOS 16 +#define QAT_COMPRESSION_DELAYED_MATCH_MASK 0x1 +#define QAT_COMPRESSION_ALGO_BITPOS 31 +#define QAT_COMPRESSION_ALGO_MASK 0x1 +#define QAT_COMPRESSION_DEPTH_BITPOS 28 +#define QAT_COMPRESSION_DEPTH_MASK 0x7 +#define QAT_COMPRESSION_FILE_TYPE_BITPOS 24 +#define QAT_COMPRESSION_FILE_TYPE_MASK 0xF + +#define ICP_QAT_HW_COMPRESSION_CONFIG_BUILD(dir, delayed, \ + algo, depth, filetype) \ + ((((dir) & QAT_COMPRESSION_DIR_MASK) << \ + QAT_COMPRESSION_DIR_BITPOS) | \ + (((delayed) & QAT_COMPRESSION_DELAYED_MATCH_MASK) << \ + QAT_COMPRESSION_DELAYED_MATCH_BITPOS) | \ + (((algo) & QAT_COMPRESSION_ALGO_MASK) << \ + QAT_COMPRESSION_ALGO_BITPOS) | \ + (((depth) & QAT_COMPRESSION_DEPTH_MASK) << \ + QAT_COMPRESSION_DEPTH_BITPOS) | \ + (((filetype) & QAT_COMPRESSION_FILE_TYPE_MASK) << \ + QAT_COMPRESSION_FILE_TYPE_BITPOS)) + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_hw_20_comp.h b/drivers/crypto/intel/qat/qat_common/icp_qat_hw_20_comp.h new file mode 100644 index 000000000..d28732225 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_hw_20_comp.h @@ -0,0 +1,166 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2022 Intel Corporation */ +#ifndef _ICP_QAT_HW_20_COMP_H_ +#define _ICP_QAT_HW_20_COMP_H_ + +#include + +#include "icp_qat_hw_20_comp_defs.h" +#include "icp_qat_fw.h" + +struct icp_qat_hw_comp_20_config_csr_lower { + enum icp_qat_hw_comp_20_extended_delay_match_mode edmm; + enum icp_qat_hw_comp_20_hw_comp_format algo; + enum icp_qat_hw_comp_20_search_depth sd; + enum icp_qat_hw_comp_20_hbs_control hbs; + enum icp_qat_hw_comp_20_abd abd; + enum icp_qat_hw_comp_20_lllbd_ctrl lllbd; + enum icp_qat_hw_comp_20_min_match_control mmctrl; + enum icp_qat_hw_comp_20_skip_hash_collision hash_col; + enum icp_qat_hw_comp_20_skip_hash_update hash_update; + enum icp_qat_hw_comp_20_byte_skip skip_ctrl; +}; + +static inline __u32 +ICP_QAT_FW_COMP_20_BUILD_CONFIG_LOWER(struct icp_qat_hw_comp_20_config_csr_lower csr) +{ + u32 val32 = 0; + + QAT_FIELD_SET(val32, csr.algo, + ICP_QAT_HW_COMP_20_CONFIG_CSR_HW_COMP_FORMAT_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_HW_COMP_FORMAT_MASK); + QAT_FIELD_SET(val32, csr.sd, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SEARCH_DEPTH_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SEARCH_DEPTH_MASK); + QAT_FIELD_SET(val32, csr.edmm, + ICP_QAT_HW_COMP_20_CONFIG_CSR_EXTENDED_DELAY_MATCH_MODE_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_EXTENDED_DELAY_MATCH_MODE_MASK); + QAT_FIELD_SET(val32, csr.hbs, + ICP_QAT_HW_COMP_20_CONFIG_CSR_HBS_CONTROL_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_HBS_CONTROL_MASK); + QAT_FIELD_SET(val32, csr.lllbd, + ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_MASK); + QAT_FIELD_SET(val32, csr.mmctrl, + ICP_QAT_HW_COMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_MASK); + QAT_FIELD_SET(val32, csr.hash_col, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_COLLISION_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_COLLISION_MASK); + QAT_FIELD_SET(val32, csr.hash_update, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_UPDATE_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_UPDATE_MASK); + QAT_FIELD_SET(val32, csr.skip_ctrl, + ICP_QAT_HW_COMP_20_CONFIG_CSR_BYTE_SKIP_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_BYTE_SKIP_MASK); + QAT_FIELD_SET(val32, csr.abd, ICP_QAT_HW_COMP_20_CONFIG_CSR_ABD_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_ABD_MASK); + + return swab32(val32); +} + +struct icp_qat_hw_comp_20_config_csr_upper { + enum icp_qat_hw_comp_20_scb_control scb_ctrl; + enum icp_qat_hw_comp_20_rmb_control rmb_ctrl; + enum icp_qat_hw_comp_20_som_control som_ctrl; + enum icp_qat_hw_comp_20_skip_hash_rd_control skip_hash_ctrl; + enum icp_qat_hw_comp_20_scb_unload_control scb_unload_ctrl; + enum icp_qat_hw_comp_20_disable_token_fusion_control disable_token_fusion_ctrl; + enum icp_qat_hw_comp_20_lbms lbms; + enum icp_qat_hw_comp_20_scb_mode_reset_mask scb_mode_reset; + __u16 lazy; + __u16 nice; +}; + +static inline __u32 +ICP_QAT_FW_COMP_20_BUILD_CONFIG_UPPER(struct icp_qat_hw_comp_20_config_csr_upper csr) +{ + u32 val32 = 0; + + QAT_FIELD_SET(val32, csr.scb_ctrl, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_CONTROL_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_CONTROL_MASK); + QAT_FIELD_SET(val32, csr.rmb_ctrl, + ICP_QAT_HW_COMP_20_CONFIG_CSR_RMB_CONTROL_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_RMB_CONTROL_MASK); + QAT_FIELD_SET(val32, csr.som_ctrl, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SOM_CONTROL_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SOM_CONTROL_MASK); + QAT_FIELD_SET(val32, csr.skip_hash_ctrl, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_RD_CONTROL_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_RD_CONTROL_MASK); + QAT_FIELD_SET(val32, csr.scb_unload_ctrl, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_UNLOAD_CONTROL_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_UNLOAD_CONTROL_MASK); + QAT_FIELD_SET(val32, csr.disable_token_fusion_ctrl, + ICP_QAT_HW_COMP_20_CONFIG_CSR_DISABLE_TOKEN_FUSION_CONTROL_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_DISABLE_TOKEN_FUSION_CONTROL_MASK); + QAT_FIELD_SET(val32, csr.lbms, + ICP_QAT_HW_COMP_20_CONFIG_CSR_LBMS_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_LBMS_MASK); + QAT_FIELD_SET(val32, csr.scb_mode_reset, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_MASK); + QAT_FIELD_SET(val32, csr.lazy, + ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_MASK); + QAT_FIELD_SET(val32, csr.nice, + ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_MASK); + + return swab32(val32); +} + +struct icp_qat_hw_decomp_20_config_csr_lower { + enum icp_qat_hw_decomp_20_hbs_control hbs; + enum icp_qat_hw_decomp_20_lbms lbms; + enum icp_qat_hw_decomp_20_hw_comp_format algo; + enum icp_qat_hw_decomp_20_min_match_control mmctrl; + enum icp_qat_hw_decomp_20_lz4_block_checksum_present lbc; +}; + +static inline __u32 +ICP_QAT_FW_DECOMP_20_BUILD_CONFIG_LOWER(struct icp_qat_hw_decomp_20_config_csr_lower csr) +{ + u32 val32 = 0; + + QAT_FIELD_SET(val32, csr.hbs, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HBS_CONTROL_BITPOS, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HBS_CONTROL_MASK); + QAT_FIELD_SET(val32, csr.lbms, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LBMS_BITPOS, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LBMS_MASK); + QAT_FIELD_SET(val32, csr.algo, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HW_DECOMP_FORMAT_BITPOS, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HW_DECOMP_FORMAT_MASK); + QAT_FIELD_SET(val32, csr.mmctrl, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_BITPOS, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_MASK); + QAT_FIELD_SET(val32, csr.lbc, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_PRESENT_BITPOS, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_PRESENT_MASK); + + return swab32(val32); +} + +struct icp_qat_hw_decomp_20_config_csr_upper { + enum icp_qat_hw_decomp_20_speculative_decoder_control sdc; + enum icp_qat_hw_decomp_20_mini_cam_control mcc; +}; + +static inline __u32 +ICP_QAT_FW_DECOMP_20_BUILD_CONFIG_UPPER(struct icp_qat_hw_decomp_20_config_csr_upper csr) +{ + u32 val32 = 0; + + QAT_FIELD_SET(val32, csr.sdc, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_SPECULATIVE_DECODER_CONTROL_BITPOS, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_SPECULATIVE_DECODER_CONTROL_MASK); + QAT_FIELD_SET(val32, csr.mcc, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_BITPOS, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_MASK); + + return swab32(val32); +} + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_hw_20_comp_defs.h b/drivers/crypto/intel/qat/qat_common/icp_qat_hw_20_comp_defs.h new file mode 100644 index 000000000..208d45542 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_hw_20_comp_defs.h @@ -0,0 +1,300 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2022 Intel Corporation */ +#ifndef _ICP_QAT_HW_20_COMP_DEFS_H +#define _ICP_QAT_HW_20_COMP_DEFS_H + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_CONTROL_BITPOS 31 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_CONTROL_MASK 0x1 + +enum icp_qat_hw_comp_20_scb_control { + ICP_QAT_HW_COMP_20_SCB_CONTROL_ENABLE = 0x0, + ICP_QAT_HW_COMP_20_SCB_CONTROL_DISABLE = 0x1, +}; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_SCB_CONTROL_DISABLE + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_RMB_CONTROL_BITPOS 30 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_RMB_CONTROL_MASK 0x1 + +enum icp_qat_hw_comp_20_rmb_control { + ICP_QAT_HW_COMP_20_RMB_CONTROL_RESET_ALL = 0x0, + ICP_QAT_HW_COMP_20_RMB_CONTROL_RESET_FC_ONLY = 0x1, +}; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_RMB_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_RMB_CONTROL_RESET_ALL + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SOM_CONTROL_BITPOS 28 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SOM_CONTROL_MASK 0x3 + +enum icp_qat_hw_comp_20_som_control { + ICP_QAT_HW_COMP_20_SOM_CONTROL_NORMAL_MODE = 0x0, + ICP_QAT_HW_COMP_20_SOM_CONTROL_REPLAY_MODE = 0x1, + ICP_QAT_HW_COMP_20_SOM_CONTROL_INPUT_CRC = 0x2, + ICP_QAT_HW_COMP_20_SOM_CONTROL_RESERVED_MODE = 0x3, +}; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SOM_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_SOM_CONTROL_NORMAL_MODE + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_RD_CONTROL_BITPOS 27 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_RD_CONTROL_MASK 0x1 + +enum icp_qat_hw_comp_20_skip_hash_rd_control { + ICP_QAT_HW_COMP_20_SKIP_HASH_RD_CONTROL_NO_SKIP = 0x0, + ICP_QAT_HW_COMP_20_SKIP_HASH_RD_CONTROL_SKIP_HASH_READS = 0x1, +}; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_RD_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_SKIP_HASH_RD_CONTROL_NO_SKIP + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_UNLOAD_CONTROL_BITPOS 26 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_UNLOAD_CONTROL_MASK 0x1 + +enum icp_qat_hw_comp_20_scb_unload_control { + ICP_QAT_HW_COMP_20_SCB_UNLOAD_CONTROL_UNLOAD = 0x0, + ICP_QAT_HW_COMP_20_SCB_UNLOAD_CONTROL_NO_UNLOAD = 0x1, +}; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_UNLOAD_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_SCB_UNLOAD_CONTROL_UNLOAD + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_DISABLE_TOKEN_FUSION_CONTROL_BITPOS 21 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_DISABLE_TOKEN_FUSION_CONTROL_MASK 0x1 + +enum icp_qat_hw_comp_20_disable_token_fusion_control { + ICP_QAT_HW_COMP_20_DISABLE_TOKEN_FUSION_CONTROL_ENABLE = 0x0, + ICP_QAT_HW_COMP_20_DISABLE_TOKEN_FUSION_CONTROL_DISABLE = 0x1, +}; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_DISABLE_TOKEN_FUSION_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_DISABLE_TOKEN_FUSION_CONTROL_ENABLE + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LBMS_BITPOS 19 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LBMS_MASK 0x3 + +enum icp_qat_hw_comp_20_lbms { + ICP_QAT_HW_COMP_20_LBMS_LBMS_64KB = 0x0, + ICP_QAT_HW_COMP_20_LBMS_LBMS_256KB = 0x1, + ICP_QAT_HW_COMP_20_LBMS_LBMS_1MB = 0x2, + ICP_QAT_HW_COMP_20_LBMS_LBMS_4MB = 0x3, +}; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LBMS_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_LBMS_LBMS_64KB + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_BITPOS 18 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_MASK 0x1 + +enum icp_qat_hw_comp_20_scb_mode_reset_mask { + ICP_QAT_HW_COMP_20_SCB_MODE_RESET_MASK_RESET_COUNTERS = 0x0, + ICP_QAT_HW_COMP_20_SCB_MODE_RESET_MASK_RESET_COUNTERS_AND_HISTORY = 0x1, +}; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_SCB_MODE_RESET_MASK_RESET_COUNTERS + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_BITPOS 9 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_MASK 0x1ff +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_DEFAULT_VAL 258 + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_BITPOS 0 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_MASK 0x1ff +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_DEFAULT_VAL 259 + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_HBS_CONTROL_BITPOS 14 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_HBS_CONTROL_MASK 0x7 + +enum icp_qat_hw_comp_20_hbs_control { + ICP_QAT_HW_COMP_20_HBS_CONTROL_HBS_IS_32KB = 0x0, + ICP_QAT_HW_COMP_23_HBS_CONTROL_HBS_IS_64KB = 0x1, +}; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_HBS_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_HBS_CONTROL_HBS_IS_32KB + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_ABD_BITPOS 13 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_ABD_MASK 0x1 + +enum icp_qat_hw_comp_20_abd { + ICP_QAT_HW_COMP_20_ABD_ABD_ENABLED = 0x0, + ICP_QAT_HW_COMP_20_ABD_ABD_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_ABD_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_ABD_ABD_ENABLED + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_BITPOS 12 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_MASK 0x1 + +enum icp_qat_hw_comp_20_lllbd_ctrl { + ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_ENABLED = 0x0, + ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_ENABLED + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SEARCH_DEPTH_BITPOS 8 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SEARCH_DEPTH_MASK 0xf + +enum icp_qat_hw_comp_20_search_depth { + ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_1 = 0x1, + ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_6 = 0x3, + ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_9 = 0x4, +}; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SEARCH_DEPTH_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_1 + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_HW_COMP_FORMAT_BITPOS 5 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_HW_COMP_FORMAT_MASK 0x7 + +enum icp_qat_hw_comp_20_hw_comp_format { + ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_ILZ77 = 0x0, + ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_DEFLATE = 0x1, + ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_LZ4 = 0x2, + ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_LZ4S = 0x3, + ICP_QAT_HW_COMP_23_HW_COMP_FORMAT_ZSTD = 0x4, +}; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_HW_COMP_FORMAT_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_DEFLATE + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_BITPOS 4 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_MASK 0x1 + +enum icp_qat_hw_comp_20_min_match_control { + ICP_QAT_HW_COMP_20_MIN_MATCH_CONTROL_MATCH_3B = 0x0, + ICP_QAT_HW_COMP_20_MIN_MATCH_CONTROL_MATCH_4B = 0x1, +}; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_MIN_MATCH_CONTROL_MATCH_3B + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_COLLISION_BITPOS 3 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_COLLISION_MASK 0x1 + +enum icp_qat_hw_comp_20_skip_hash_collision { + ICP_QAT_HW_COMP_20_SKIP_HASH_COLLISION_ALLOW = 0x0, + ICP_QAT_HW_COMP_20_SKIP_HASH_COLLISION_DONT_ALLOW = 0x1, +}; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_COLLISION_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_SKIP_HASH_COLLISION_ALLOW + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_UPDATE_BITPOS 2 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_UPDATE_MASK 0x1 + +enum icp_qat_hw_comp_20_skip_hash_update { + ICP_QAT_HW_COMP_20_SKIP_HASH_UPDATE_ALLOW = 0x0, + ICP_QAT_HW_COMP_20_SKIP_HASH_UPDATE_DONT_ALLOW = 0x1, +}; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_UPDATE_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_SKIP_HASH_UPDATE_ALLOW + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_BYTE_SKIP_BITPOS 1 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_BYTE_SKIP_MASK 0x1 + +enum icp_qat_hw_comp_20_byte_skip { + ICP_QAT_HW_COMP_20_BYTE_SKIP_3BYTE_TOKEN = 0x0, + ICP_QAT_HW_COMP_20_BYTE_SKIP_3BYTE_LITERAL = 0x1, +}; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_BYTE_SKIP_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_BYTE_SKIP_3BYTE_TOKEN + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_EXTENDED_DELAY_MATCH_MODE_BITPOS 0 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_EXTENDED_DELAY_MATCH_MODE_MASK 0x1 + +enum icp_qat_hw_comp_20_extended_delay_match_mode { + ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_DISABLED = 0x0, + ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_ENABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_EXTENDED_DELAY_MATCH_MODE_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_DISABLED + +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_SPECULATIVE_DECODER_CONTROL_BITPOS 31 +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_SPECULATIVE_DECODER_CONTROL_MASK 0x1 + +enum icp_qat_hw_decomp_20_speculative_decoder_control { + ICP_QAT_HW_DECOMP_20_SPECULATIVE_DECODER_CONTROL_ENABLE = 0x0, + ICP_QAT_HW_DECOMP_20_SPECULATIVE_DECODER_CONTROL_DISABLE = 0x1, +}; + +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_SPECULATIVE_DECODER_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_20_SPECULATIVE_DECODER_CONTROL_ENABLE + +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_BITPOS 30 +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_MASK 0x1 + +enum icp_qat_hw_decomp_20_mini_cam_control { + ICP_QAT_HW_DECOMP_20_MINI_CAM_CONTROL_ENABLE = 0x0, + ICP_QAT_HW_DECOMP_20_MINI_CAM_CONTROL_DISABLE = 0x1, +}; + +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_20_MINI_CAM_CONTROL_ENABLE + +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HBS_CONTROL_BITPOS 14 +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HBS_CONTROL_MASK 0x7 + +enum icp_qat_hw_decomp_20_hbs_control { + ICP_QAT_HW_DECOMP_20_HBS_CONTROL_HBS_IS_32KB = 0x0, +}; + +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HBS_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_20_HBS_CONTROL_HBS_IS_32KB + +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LBMS_BITPOS 8 +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LBMS_MASK 0x3 + +enum icp_qat_hw_decomp_20_lbms { + ICP_QAT_HW_DECOMP_20_LBMS_LBMS_64KB = 0x0, + ICP_QAT_HW_DECOMP_20_LBMS_LBMS_256KB = 0x1, + ICP_QAT_HW_DECOMP_20_LBMS_LBMS_1MB = 0x2, + ICP_QAT_HW_DECOMP_20_LBMS_LBMS_4MB = 0x3, +}; + +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LBMS_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_20_LBMS_LBMS_64KB + +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HW_DECOMP_FORMAT_BITPOS 5 +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HW_DECOMP_FORMAT_MASK 0x7 + +enum icp_qat_hw_decomp_20_hw_comp_format { + ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_DEFLATE = 0x1, + ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_LZ4 = 0x2, + ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_LZ4S = 0x3, + ICP_QAT_HW_DECOMP_23_HW_DECOMP_FORMAT_ZSTD = 0x4, +}; + +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HW_DECOMP_FORMAT_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_DEFLATE + +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_BITPOS 4 +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_MASK 0x1 + +enum icp_qat_hw_decomp_20_min_match_control { + ICP_QAT_HW_DECOMP_20_MIN_MATCH_CONTROL_MATCH_3B = 0x0, + ICP_QAT_HW_DECOMP_20_MIN_MATCH_CONTROL_MATCH_4B = 0x1, +}; + +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_20_MIN_MATCH_CONTROL_MATCH_3B + +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_PRESENT_BITPOS 3 +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_PRESENT_MASK 0x1 + +enum icp_qat_hw_decomp_20_lz4_block_checksum_present { + ICP_QAT_HW_DECOMP_20_LZ4_BLOCK_CHKSUM_ABSENT = 0x0, + ICP_QAT_HW_DECOMP_20_LZ4_BLOCK_CHKSUM_PRESENT = 0x1, +}; + +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_PRESENT_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_20_LZ4_BLOCK_CHKSUM_ABSENT + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_hw_51_comp.h b/drivers/crypto/intel/qat/qat_common/icp_qat_hw_51_comp.h new file mode 100644 index 000000000..dce639152 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_hw_51_comp.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ICP_QAT_HW_51_COMP_H_ +#define ICP_QAT_HW_51_COMP_H_ + +#include + +#include "icp_qat_fw.h" +#include "icp_qat_hw_51_comp_defs.h" + +struct icp_qat_hw_comp_51_config_csr_lower { + enum icp_qat_hw_comp_51_abd abd; + enum icp_qat_hw_comp_51_lllbd_ctrl lllbd; + enum icp_qat_hw_comp_51_search_depth sd; + enum icp_qat_hw_comp_51_min_match_control mmctrl; + enum icp_qat_hw_comp_51_lz4_block_checksum lbc; +}; + +static inline u32 +ICP_QAT_FW_COMP_51_BUILD_CONFIG_LOWER(struct icp_qat_hw_comp_51_config_csr_lower csr) +{ + u32 val32 = 0; + + QAT_FIELD_SET(val32, csr.abd, + ICP_QAT_HW_COMP_51_CONFIG_CSR_ABD_BITPOS, + ICP_QAT_HW_COMP_51_CONFIG_CSR_ABD_MASK); + QAT_FIELD_SET(val32, csr.lllbd, + ICP_QAT_HW_COMP_51_CONFIG_CSR_LLLBD_CTRL_BITPOS, + ICP_QAT_HW_COMP_51_CONFIG_CSR_LLLBD_CTRL_MASK); + QAT_FIELD_SET(val32, csr.sd, + ICP_QAT_HW_COMP_51_CONFIG_CSR_SEARCH_DEPTH_BITPOS, + ICP_QAT_HW_COMP_51_CONFIG_CSR_SEARCH_DEPTH_MASK); + QAT_FIELD_SET(val32, csr.mmctrl, + ICP_QAT_HW_COMP_51_CONFIG_CSR_MIN_MATCH_CONTROL_BITPOS, + ICP_QAT_HW_COMP_51_CONFIG_CSR_MIN_MATCH_CONTROL_MASK); + QAT_FIELD_SET(val32, csr.lbc, + ICP_QAT_HW_COMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_BITPOS, + ICP_QAT_HW_COMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_MASK); + + return val32; +} + +struct icp_qat_hw_comp_51_config_csr_upper { + enum icp_qat_hw_comp_51_dmm_algorithm edmm; + enum icp_qat_hw_comp_51_bms bms; + enum icp_qat_hw_comp_51_scb_mode_reset_mask scb_mode_reset; +}; + +static inline u32 +ICP_QAT_FW_COMP_51_BUILD_CONFIG_UPPER(struct icp_qat_hw_comp_51_config_csr_upper csr) +{ + u32 val32 = 0; + + QAT_FIELD_SET(val32, csr.edmm, + ICP_QAT_HW_COMP_51_CONFIG_CSR_DMM_ALGORITHM_BITPOS, + ICP_QAT_HW_COMP_51_CONFIG_CSR_DMM_ALGORITHM_MASK); + QAT_FIELD_SET(val32, csr.bms, + ICP_QAT_HW_COMP_51_CONFIG_CSR_BMS_BITPOS, + ICP_QAT_HW_COMP_51_CONFIG_CSR_BMS_MASK); + QAT_FIELD_SET(val32, csr.scb_mode_reset, + ICP_QAT_HW_COMP_51_CONFIG_CSR_SCB_MODE_RESET_MASK_BITPOS, + ICP_QAT_HW_COMP_51_CONFIG_CSR_SCB_MODE_RESET_MASK_MASK); + + return val32; +} + +struct icp_qat_hw_decomp_51_config_csr_lower { + enum icp_qat_hw_decomp_51_lz4_block_checksum lbc; +}; + +static inline u32 +ICP_QAT_FW_DECOMP_51_BUILD_CONFIG_LOWER(struct icp_qat_hw_decomp_51_config_csr_lower csr) +{ + u32 val32 = 0; + + QAT_FIELD_SET(val32, csr.lbc, + ICP_QAT_HW_DECOMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_BITPOS, + ICP_QAT_HW_DECOMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_MASK); + + return val32; +} + +struct icp_qat_hw_decomp_51_config_csr_upper { + enum icp_qat_hw_decomp_51_bms bms; +}; + +static inline u32 +ICP_QAT_FW_DECOMP_51_BUILD_CONFIG_UPPER(struct icp_qat_hw_decomp_51_config_csr_upper csr) +{ + u32 val32 = 0; + + QAT_FIELD_SET(val32, csr.bms, + ICP_QAT_HW_DECOMP_51_CONFIG_CSR_BMS_BITPOS, + ICP_QAT_HW_DECOMP_51_CONFIG_CSR_BMS_MASK); + + return val32; +} + +#endif /* ICP_QAT_HW_51_COMP_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_hw_51_comp_defs.h b/drivers/crypto/intel/qat/qat_common/icp_qat_hw_51_comp_defs.h new file mode 100644 index 000000000..e745688c5 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_hw_51_comp_defs.h @@ -0,0 +1,318 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ICP_QAT_HW_51_COMP_DEFS_H_ +#define ICP_QAT_HW_51_COMP_DEFS_H_ + +#include + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SOM_CONTROL_BITPOS 28 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SOM_CONTROL_MASK GENMASK(1, 0) +enum icp_qat_hw_comp_51_som_control { + ICP_QAT_HW_COMP_51_SOM_CONTROL_NORMAL_MODE = 0x0, + ICP_QAT_HW_COMP_51_SOM_CONTROL_DICTIONARY_MODE = 0x1, + ICP_QAT_HW_COMP_51_SOM_CONTROL_INPUT_CRC = 0x2, + ICP_QAT_HW_COMP_51_SOM_CONTROL_RESERVED_MODE = 0x3, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SOM_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_SOM_CONTROL_NORMAL_MODE +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_RD_CONTROL_BITPOS 27 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_RD_CONTROL_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_skip_hash_rd_control { + ICP_QAT_HW_COMP_51_SKIP_HASH_RD_CONTROL_NO_SKIP = 0x0, + ICP_QAT_HW_COMP_51_SKIP_HASH_RD_CONTROL_SKIP_HASH_READS = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_RD_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_SKIP_HASH_RD_CONTROL_NO_SKIP +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BYPASS_COMPRESSION_BITPOS 25 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BYPASS_COMPRESSION_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_bypass_compression { + ICP_QAT_HW_COMP_51_BYPASS_COMPRESSION_DISABLED = 0x0, + ICP_QAT_HW_COMP_51_BYPASS_COMPRESSION_ENABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BYPASS_COMPRESSION_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_BYPASS_COMPRESSION_DISABLED +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_DMM_ALGORITHM_BITPOS 22 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_DMM_ALGORITHM_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_dmm_algorithm { + ICP_QAT_HW_COMP_51_DMM_ALGORITHM_EDMM_ENABLED = 0x0, + ICP_QAT_HW_COMP_51_DMM_ALGORITHM_ZSTD_DMM_LITE = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_DMM_ALGORITHM_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_DMM_ALGORITHM_EDMM_ENABLED +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_TOKEN_FUSION_INTERNAL_ONLY_BITPOS 21 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_TOKEN_FUSION_INTERNAL_ONLY_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_token_fusion_internal_only { + ICP_QAT_HW_COMP_51_TOKEN_FUSION_INTERNAL_ONLY_ENABLED = 0x0, + ICP_QAT_HW_COMP_51_TOKEN_FUSION_INTERNAL_ONLY_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_TOKEN_FUSION_INTERNAL_ONLY_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_TOKEN_FUSION_INTERNAL_ONLY_ENABLED +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BMS_BITPOS 19 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BMS_MASK GENMASK(1, 0) +enum icp_qat_hw_comp_51_bms { + ICP_QAT_HW_COMP_51_BMS_BMS_64KB = 0x0, + ICP_QAT_HW_COMP_51_BMS_BMS_256KB = 0x1, + ICP_QAT_HW_COMP_51_BMS_BMS_1MB = 0x2, + ICP_QAT_HW_COMP_51_BMS_BMS_4MB = 0x3, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BMS_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_BMS_BMS_64KB +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SCB_MODE_RESET_MASK_BITPOS 18 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SCB_MODE_RESET_MASK_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_scb_mode_reset_mask { + ICP_QAT_HW_COMP_51_SCB_MODE_RESET_MASK_DO_NOT_RESET_HB_HT = 0x0, + ICP_QAT_HW_COMP_51_SCB_MODE_RESET_MASK_RESET_HB_HT = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SCB_MODE_RESET_MASK_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_SCB_MODE_RESET_MASK_DO_NOT_RESET_HB_HT +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ZSTD_FRAME_GEN_DEC_EN_BITPOS 2 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ZSTD_FRAME_GEN_DEC_EN_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_zstd_frame_gen_dec_en { + ICP_QAT_HW_COMP_51_ZSTD_FRAME_GEN_DEC_EN_ZSTD_FRAME_HDR_DISABLE = 0x0, + ICP_QAT_HW_COMP_51_ZSTD_FRAME_GEN_DEC_EN_ZSTD_FRAME_HDR_ENABLE = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ZSTD_FRAME_GEN_DEC_EN_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_ZSTD_FRAME_GEN_DEC_EN_ZSTD_FRAME_HDR_ENABLE +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_CNV_DISABLE_BITPOS 1 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_CNV_DISABLE_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_cnv_disable { + ICP_QAT_HW_COMP_51_CNV_DISABLE_CNV_ENABLED = 0x0, + ICP_QAT_HW_COMP_51_CNV_DISABLE_CNV_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_CNV_DISABLE_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_CNV_DISABLE_CNV_ENABLED +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ASB_DISABLE_BITPOS 0 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ASB_DISABLE_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_asb_disable { + ICP_QAT_HW_COMP_51_ASB_DISABLE_ASB_ENABLED = 0x0, + ICP_QAT_HW_COMP_51_ASB_DISABLE_ASB_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ASB_DISABLE_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_ASB_DISABLE_ASB_ENABLED +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SPEC_DECODER_INTERNAL_ONLY_BITPOS 21 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SPEC_DECODER_INTERNAL_ONLY_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_spec_decoder_internal_only { + ICP_QAT_HW_COMP_51_SPEC_DECODER_INTERNAL_ONLY_NORMAL = 0x0, + ICP_QAT_HW_COMP_51_SPEC_DECODER_INTERNAL_ONLY_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SPEC_DECODER_INTERNAL_ONLY_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_SPEC_DECODER_INTERNAL_ONLY_NORMAL +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_MINI_XCAM_INTERNAL_ONLY_BITPOS 20 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_MINI_XCAM_INTERNAL_ONLY_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_mini_xcam_internal_only { + ICP_QAT_HW_COMP_51_MINI_XCAM_INTERNAL_ONLY_NORMAL = 0x0, + ICP_QAT_HW_COMP_51_MINI_XCAM_INTERNAL_ONLY_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_MINI_XCAM_INTERNAL_ONLY_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_MINI_XCAM_INTERNAL_ONLY_NORMAL +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_REP_OFF_ENC_INTERNAL_ONLY_BITPOS 19 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_REP_OFF_ENC_INTERNAL_ONLY_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_rep_off_enc_internal_only { + ICP_QAT_HW_COMP_51_REP_OFF_ENC_INTERNAL_ONLY_ENABLED = 0x0, + ICP_QAT_HW_COMP_51_REP_OFF_ENC_INTERNAL_ONLY_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_REP_OFF_ENC_INTERNAL_ONLY_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_REP_OFF_ENC_INTERNAL_ONLY_ENABLED +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_PROG_BLOCK_DROP_INTERNAL_ONLY_BITPOS 18 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_PROG_BLOCK_DROP_INTERNAL_ONLY_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_prog_block_drop_internal_only { + ICP_QAT_HW_COMP_51_PROG_BLOCK_DROP_INTERNAL_ONLY_DISABLE = 0x0, + ICP_QAT_HW_COMP_51_PROG_BLOCK_DROP_INTERNAL_ONLY_ENABLE = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_PROG_BLOCK_DROP_INTERNAL_ONLY_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_PROG_BLOCK_DROP_INTERNAL_ONLY_DISABLE +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_OVERRIDE_INTERNAL_ONLY_BITPOS 17 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_OVERRIDE_INTERNAL_ONLY_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_skip_hash_override_internal_only { + ICP_QAT_HW_COMP_51_SKIP_HASH_OVERRIDE_INTERNAL_ONLY_DETERMINE_HASH_PARAMS = 0x0, + ICP_QAT_HW_COMP_51_SKIP_HASH_OVERRIDE_INTERNAL_ONLY_OVERRIDE_HASH_PARAMS = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_OVERRIDE_INTERNAL_ONLY_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_SKIP_HASH_OVERRIDE_INTERNAL_ONLY_DETERMINE_HASH_PARAMS +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_HBS_BITPOS 14 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_HBS_MASK GENMASK(2, 0) +enum icp_qat_hw_comp_51_hbs { + ICP_QAT_HW_COMP_51_HBS_32KB = 0x0, + ICP_QAT_HW_COMP_51_HBS_64KB = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_HBS_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_HBS_32KB +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ABD_BITPOS 13 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ABD_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_abd { + ICP_QAT_HW_COMP_51_ABD_ABD_ENABLED = 0x0, + ICP_QAT_HW_COMP_51_ABD_ABD_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_ABD_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_ABD_ABD_ENABLED +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_LLLBD_CTRL_BITPOS 12 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_LLLBD_CTRL_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_lllbd_ctrl { + ICP_QAT_HW_COMP_51_LLLBD_CTRL_LLLBD_ENABLED = 0x0, + ICP_QAT_HW_COMP_51_LLLBD_CTRL_LLLBD_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_LLLBD_CTRL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_LLLBD_CTRL_LLLBD_ENABLED +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SEARCH_DEPTH_BITPOS 8 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SEARCH_DEPTH_MASK GENMASK(3, 0) +enum icp_qat_hw_comp_51_search_depth { + ICP_QAT_HW_COMP_51_SEARCH_DEPTH_LEVEL_1 = 0x1, + ICP_QAT_HW_COMP_51_SEARCH_DEPTH_LEVEL_6 = 0x3, + ICP_QAT_HW_COMP_51_SEARCH_DEPTH_LEVEL_9 = 0x4, + ICP_QAT_HW_COMP_51_SEARCH_DEPTH_LEVEL_10 = 0x4, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SEARCH_DEPTH_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_SEARCH_DEPTH_LEVEL_1 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_FORMAT_BITPOS 5 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_FORMAT_MASK GENMASK(2, 0) +enum icp_qat_hw_comp_51_format { + ICP_QAT_HW_COMP_51_FORMAT_ILZ77 = 0x1, + ICP_QAT_HW_COMP_51_FORMAT_LZ4 = 0x2, + ICP_QAT_HW_COMP_51_FORMAT_LZ4s = 0x3, + ICP_QAT_HW_COMP_51_FORMAT_ZSTD = 0x4, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_FORMAT_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_FORMAT_ILZ77 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_MIN_MATCH_CONTROL_BITPOS 4 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_MIN_MATCH_CONTROL_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_min_match_control { + ICP_QAT_HW_COMP_51_MIN_MATCH_CONTROL_MATCH_3B = 0x0, + ICP_QAT_HW_COMP_51_MIN_MATCH_CONTROL_MATCH_4B = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_MIN_MATCH_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_MIN_MATCH_CONTROL_MATCH_3B +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_COLLISION_BITPOS 3 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_COLLISION_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_skip_hash_collision { + ICP_QAT_HW_COMP_51_SKIP_HASH_COLLISION_ALLOW = 0x0, + ICP_QAT_HW_COMP_51_SKIP_HASH_COLLISION_DONT_ALLOW = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_COLLISION_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_SKIP_HASH_COLLISION_ALLOW +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_UPDATE_BITPOS 2 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_UPDATE_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_skip_hash_update { + ICP_QAT_HW_COMP_51_SKIP_HASH_UPDATE_ALLOW = 0x0, + ICP_QAT_HW_COMP_51_SKIP_HASH_UPDATE_DONT_ALLOW = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_SKIP_HASH_UPDATE_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_SKIP_HASH_UPDATE_ALLOW +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BYTE_SKIP_BITPOS 1 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BYTE_SKIP_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_byte_skip { + ICP_QAT_HW_COMP_51_BYTE_SKIP_3BYTE_TOKEN = 0x0, + ICP_QAT_HW_COMP_51_BYTE_SKIP_3BYTE_LITERAL = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_BYTE_SKIP_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_BYTE_SKIP_3BYTE_TOKEN +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_BITPOS 0 +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_MASK GENMASK(0, 0) +enum icp_qat_hw_comp_51_lz4_block_checksum { + ICP_QAT_HW_COMP_51_LZ4_BLOCK_CHECKSUM_ABSENT = 0x0, + ICP_QAT_HW_COMP_51_LZ4_BLOCK_CHECKSUM_PRESENT = 0x1, +}; + +#define ICP_QAT_HW_COMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_DEFAULT_VAL \ + ICP_QAT_HW_COMP_51_LZ4_BLOCK_CHECKSUM_ABSENT +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_DISCARD_DATA_BITPOS 26 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_DISCARD_DATA_MASK GENMASK(0, 0) +enum icp_qat_hw_decomp_51_discard_data { + ICP_QAT_HW_DECOMP_51_DISCARD_DATA_DISABLED = 0x0, + ICP_QAT_HW_DECOMP_51_DISCARD_DATA_ENABLED = 0x1, +}; + +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_DISCARD_DATA_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_51_DISCARD_DATA_DISABLED +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_BMS_BITPOS 19 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_BMS_MASK GENMASK(1, 0) +enum icp_qat_hw_decomp_51_bms { + ICP_QAT_HW_DECOMP_51_BMS_BMS_64KB = 0x0, + ICP_QAT_HW_DECOMP_51_BMS_BMS_256KB = 0x1, + ICP_QAT_HW_DECOMP_51_BMS_BMS_1MB = 0x2, + ICP_QAT_HW_DECOMP_51_BMS_BMS_4MB = 0x3, +}; + +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_BMS_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_51_BMS_BMS_64KB +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_ZSTD_FRAME_GEN_DEC_EN_BITPOS 2 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_ZSTD_FRAME_GEN_DEC_EN_MASK GENMASK(0, 0) +enum icp_qat_hw_decomp_51_zstd_frame_gen_dec_en { + ICP_QAT_HW_DECOMP_51_ZSTD_FRAME_GEN_DEC_EN_ZSTD_FRAME_HDR_DISABLE = 0x0, + ICP_QAT_HW_DECOMP_51_ZSTD_FRAME_GEN_DEC_EN_ZSTD_FRAME_HDR_ENABLE = 0x1, +}; + +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_ZSTD_FRAME_GEN_DEC_EN_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_51_ZSTD_FRAME_GEN_DEC_EN_ZSTD_FRAME_HDR_ENABLE +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_SPEC_DECODER_INTERNAL_ONLY_BITPOS 21 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_SPEC_DECODER_INTERNAL_ONLY_MASK GENMASK(0, 0) +enum icp_qat_hw_decomp_51_spec_decoder_internal_only { + ICP_QAT_HW_DECOMP_51_SPEC_DECODER_INTERNAL_ONLY_NORMAL = 0x0, + ICP_QAT_HW_DECOMP_51_SPEC_DECODER_INTERNAL_ONLY_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_SPEC_DECODER_INTERNAL_ONLY_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_51_SPEC_DECODER_INTERNAL_ONLY_NORMAL +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_MINI_XCAM_INTERNAL_ONLY_BITPOS 20 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_MINI_XCAM_INTERNAL_ONLY_MASK GENMASK(0, 0) +enum icp_qat_hw_decomp_51_mini_xcam_internal_only { + ICP_QAT_HW_DECOMP_51_MINI_XCAM_INTERNAL_ONLY_NORMAL = 0x0, + ICP_QAT_HW_DECOMP_51_MINI_XCAM_INTERNAL_ONLY_DISABLED = 0x1, +}; + +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_MINI_XCAM_INTERNAL_ONLY_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_51_MINI_XCAM_INTERNAL_ONLY_NORMAL +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_HBS_BITPOS 14 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_HBS_MASK GENMASK(2, 0) +enum icp_qat_hw_decomp_51_hbs { + ICP_QAT_HW_DECOMP_51_HBS_32KB = 0x0, + ICP_QAT_HW_DECOMP_51_HBS_64KB = 0x1, +}; + +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_HBS_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_51_HBS_32KB +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_FORMAT_BITPOS 5 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_FORMAT_MASK GENMASK(2, 0) +enum icp_qat_hw_decomp_51_format { + ICP_QAT_HW_DECOMP_51_FORMAT_ILZ77 = 0x1, + ICP_QAT_HW_DECOMP_51_FORMAT_LZ4 = 0x2, + ICP_QAT_HW_DECOMP_51_FORMAT_RESERVED = 0x3, + ICP_QAT_HW_DECOMP_51_FORMAT_ZSTD = 0x4, +}; + +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_FORMAT_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_51_FORMAT_ILZ77 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_BITPOS 0 +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_MASK GENMASK(0, 0) +enum icp_qat_hw_decomp_51_lz4_block_checksum { + ICP_QAT_HW_DECOMP_51_LZ4_BLOCK_CHECKSUM_ABSENT = 0x0, + ICP_QAT_HW_DECOMP_51_LZ4_BLOCK_CHECKSUM_PRESENT = 0x1, +}; + +#define ICP_QAT_HW_DECOMP_51_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_51_LZ4_BLOCK_CHECKSUM_ABSENT + +#endif /* ICP_QAT_HW_51_COMP_DEFS_H_ */ diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_uclo.h b/drivers/crypto/intel/qat/qat_common/icp_qat_uclo.h new file mode 100644 index 000000000..6313c35ef --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_uclo.h @@ -0,0 +1,598 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#ifndef __ICP_QAT_UCLO_H__ +#define __ICP_QAT_UCLO_H__ + +#define ICP_QAT_AC_895XCC_DEV_TYPE 0x00400000 +#define ICP_QAT_AC_C62X_DEV_TYPE 0x01000000 +#define ICP_QAT_AC_C3XXX_DEV_TYPE 0x02000000 +#define ICP_QAT_AC_4XXX_A_DEV_TYPE 0x08000000 +#define ICP_QAT_AC_6XXX_DEV_TYPE 0x80000000 +#define ICP_QAT_UCLO_MAX_AE 17 +#define ICP_QAT_UCLO_MAX_CTX 8 +#define ICP_QAT_UCLO_MAX_UIMAGE (ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX) +#define ICP_QAT_UCLO_MAX_USTORE 0x4000 +#define ICP_QAT_UCLO_MAX_XFER_REG 128 +#define ICP_QAT_UCLO_MAX_GPR_REG 128 +#define ICP_QAT_UCLO_MAX_LMEM_REG 1024 +#define ICP_QAT_UCLO_MAX_LMEM_REG_2X 1280 +#define ICP_QAT_UCLO_AE_ALL_CTX 0xff +#define ICP_QAT_UOF_OBJID_LEN 8 +#define ICP_QAT_UOF_FID 0xc6c2 +#define ICP_QAT_UOF_MAJVER 0x4 +#define ICP_QAT_UOF_MINVER 0x11 +#define ICP_QAT_UOF_OBJS "UOF_OBJS" +#define ICP_QAT_UOF_STRT "UOF_STRT" +#define ICP_QAT_UOF_IMAG "UOF_IMAG" +#define ICP_QAT_UOF_IMEM "UOF_IMEM" +#define ICP_QAT_UOF_LOCAL_SCOPE 1 +#define ICP_QAT_UOF_INIT_EXPR 0 +#define ICP_QAT_UOF_INIT_REG 1 +#define ICP_QAT_UOF_INIT_REG_CTX 2 +#define ICP_QAT_UOF_INIT_EXPR_ENDIAN_SWAP 3 +#define ICP_QAT_SUOF_OBJ_ID_LEN 8 +#define ICP_QAT_SUOF_FID 0x53554f46 +#define ICP_QAT_SUOF_MAJVER 0x0 +#define ICP_QAT_SUOF_MINVER 0x1 +#define ICP_QAT_SUOF_OBJ_NAME_LEN 128 +#define ICP_QAT_MOF_OBJ_ID_LEN 8 +#define ICP_QAT_MOF_OBJ_CHUNKID_LEN 8 +#define ICP_QAT_MOF_FID 0x00666f6d +#define ICP_QAT_MOF_MAJVER 0x0 +#define ICP_QAT_MOF_MINVER 0x1 +#define ICP_QAT_MOF_SYM_OBJS "SYM_OBJS" +#define ICP_QAT_SUOF_OBJS "SUF_OBJS" +#define ICP_QAT_SUOF_IMAG "SUF_IMAG" +#define ICP_QAT_SIMG_AE_INIT_SEQ_LEN (50 * sizeof(unsigned long long)) + +#define DSS_FWSK_MODULUS_LEN 384 /* RSA3K */ +#define DSS_FWSK_EXPONENT_LEN 4 +#define DSS_FWSK_PADDING_LEN 380 +#define DSS_SIGNATURE_LEN 384 /* RSA3K */ + +#define CSS_FWSK_MODULUS_LEN 256 /* RSA2K */ +#define CSS_FWSK_EXPONENT_LEN 4 +#define CSS_FWSK_PADDING_LEN 252 +#define CSS_SIGNATURE_LEN 256 /* RSA2K */ + +#define ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) ((handle)->chip_info->css_3k ? \ + DSS_FWSK_MODULUS_LEN : \ + CSS_FWSK_MODULUS_LEN) + +#define ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle) ((handle)->chip_info->css_3k ? \ + DSS_FWSK_EXPONENT_LEN : \ + CSS_FWSK_EXPONENT_LEN) + +#define ICP_QAT_CSS_FWSK_PAD_LEN(handle) ((handle)->chip_info->css_3k ? \ + DSS_FWSK_PADDING_LEN : \ + CSS_FWSK_PADDING_LEN) + +#define ICP_QAT_CSS_FWSK_PUB_LEN(handle) (ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + \ + ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle) + \ + ICP_QAT_CSS_FWSK_PAD_LEN(handle)) + +#define ICP_QAT_CSS_SIGNATURE_LEN(handle) ((handle)->chip_info->css_3k ? \ + DSS_SIGNATURE_LEN : \ + CSS_SIGNATURE_LEN) + +#define ICP_QAT_AE_IMG_OFFSET(handle) (sizeof(struct icp_qat_css_hdr) + \ + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + \ + ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle) + \ + ICP_QAT_CSS_SIGNATURE_LEN(handle)) +#define ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN 0x40000 +#define ICP_QAT_CSS_RSA3K_MAX_IMAGE_LEN 0x30000 + +/* All lengths below are in bytes */ +#define ICP_QAT_DUALSIGN_OPAQUE_HDR_LEN 12 +#define ICP_QAT_DUALSIGN_OPAQUE_HDR_ALIGN_LEN 16 +#define ICP_QAT_DUALSIGN_OPAQUE_DATA_LEN 3540 +#define ICP_QAT_DUALSIGN_XMSS_PUBKEY_LEN 64 +#define ICP_QAT_DUALSIGN_XMSS_SIG_LEN 2692 +#define ICP_QAT_DUALSIGN_XMSS_SIG_ALIGN_LEN 2696 +#define ICP_QAT_DUALSIGN_MISC_INFO_LEN 16 +#define ICP_QAT_DUALSIGN_FW_TYPE_LEN 7 +#define ICP_QAT_DUALSIGN_MODULE_TYPE 0x14 +#define ICP_QAT_DUALSIGN_HDR_LEN 0x375 +#define ICP_QAT_DUALSIGN_HDR_VER 0x40001 +#define ICP_QAT_DUALSIGN_HDR_LEN_OFFSET 4 +#define ICP_QAT_DUALSIGN_HDR_VER_OFFSET 8 + +#define ICP_QAT_CTX_MODE(ae_mode) ((ae_mode) & 0xf) +#define ICP_QAT_NN_MODE(ae_mode) (((ae_mode) >> 0x4) & 0xf) +#define ICP_QAT_SHARED_USTORE_MODE(ae_mode) (((ae_mode) >> 0xb) & 0x1) +#define RELOADABLE_CTX_SHARED_MODE(ae_mode) (((ae_mode) >> 0xc) & 0x1) + +#define ICP_QAT_LOC_MEM0_MODE(ae_mode) (((ae_mode) >> 0x8) & 0x1) +#define ICP_QAT_LOC_MEM1_MODE(ae_mode) (((ae_mode) >> 0x9) & 0x1) +#define ICP_QAT_LOC_MEM2_MODE(ae_mode) (((ae_mode) >> 0x6) & 0x1) +#define ICP_QAT_LOC_MEM3_MODE(ae_mode) (((ae_mode) >> 0x7) & 0x1) +#define ICP_QAT_LOC_TINDEX_MODE(ae_mode) (((ae_mode) >> 0xe) & 0x1) + +enum icp_qat_uof_mem_region { + ICP_QAT_UOF_SRAM_REGION = 0x0, + ICP_QAT_UOF_LMEM_REGION = 0x3, + ICP_QAT_UOF_UMEM_REGION = 0x5 +}; + +enum icp_qat_uof_regtype { + ICP_NO_DEST = 0, + ICP_GPA_REL = 1, + ICP_GPA_ABS = 2, + ICP_GPB_REL = 3, + ICP_GPB_ABS = 4, + ICP_SR_REL = 5, + ICP_SR_RD_REL = 6, + ICP_SR_WR_REL = 7, + ICP_SR_ABS = 8, + ICP_SR_RD_ABS = 9, + ICP_SR_WR_ABS = 10, + ICP_DR_REL = 19, + ICP_DR_RD_REL = 20, + ICP_DR_WR_REL = 21, + ICP_DR_ABS = 22, + ICP_DR_RD_ABS = 23, + ICP_DR_WR_ABS = 24, + ICP_LMEM = 26, + ICP_LMEM0 = 27, + ICP_LMEM1 = 28, + ICP_NEIGH_REL = 31, + ICP_LMEM2 = 61, + ICP_LMEM3 = 62, +}; + +enum icp_qat_css_fwtype { + CSS_AE_FIRMWARE = 0, + CSS_MMP_FIRMWARE = 1 +}; + +struct icp_qat_uclo_page { + struct icp_qat_uclo_encap_page *encap_page; + struct icp_qat_uclo_region *region; + unsigned int flags; +}; + +struct icp_qat_uclo_region { + struct icp_qat_uclo_page *loaded; + struct icp_qat_uclo_page *page; +}; + +struct icp_qat_uclo_aeslice { + struct icp_qat_uclo_region *region; + struct icp_qat_uclo_page *page; + struct icp_qat_uclo_page *cur_page[ICP_QAT_UCLO_MAX_CTX]; + struct icp_qat_uclo_encapme *encap_image; + unsigned int ctx_mask_assigned; + unsigned int new_uaddr[ICP_QAT_UCLO_MAX_CTX]; +}; + +struct icp_qat_uclo_aedata { + unsigned int slice_num; + unsigned int eff_ustore_size; + struct icp_qat_uclo_aeslice ae_slices[ICP_QAT_UCLO_MAX_CTX]; +}; + +struct icp_qat_uof_encap_obj { + char *beg_uof; + struct icp_qat_uof_objhdr *obj_hdr; + struct icp_qat_uof_chunkhdr *chunk_hdr; + struct icp_qat_uof_varmem_seg *var_mem_seg; +}; + +struct icp_qat_uclo_encap_uwblock { + unsigned int start_addr; + unsigned int words_num; + u64 micro_words; +}; + +struct icp_qat_uclo_encap_page { + unsigned int def_page; + unsigned int page_region; + unsigned int beg_addr_v; + unsigned int beg_addr_p; + unsigned int micro_words_num; + unsigned int uwblock_num; + struct icp_qat_uclo_encap_uwblock *uwblock; +}; + +struct icp_qat_uclo_encapme { + struct icp_qat_uof_image *img_ptr; + struct icp_qat_uclo_encap_page *page; + unsigned int ae_reg_num; + struct icp_qat_uof_ae_reg *ae_reg; + unsigned int init_regsym_num; + struct icp_qat_uof_init_regsym *init_regsym; + unsigned int sbreak_num; + struct icp_qat_uof_sbreak *sbreak; + unsigned int uwords_num; +}; + +struct icp_qat_uclo_init_mem_table { + unsigned int entry_num; + struct icp_qat_uof_initmem *init_mem; +}; + +struct icp_qat_uclo_objhdr { + char *file_buff; + unsigned int checksum; + unsigned int size; +}; + +struct icp_qat_uof_strtable { + unsigned int table_len; + unsigned int reserved; + u64 strings; +}; + +struct icp_qat_uclo_objhandle { + unsigned int prod_type; + unsigned int prod_rev; + struct icp_qat_uclo_objhdr *obj_hdr; + struct icp_qat_uof_encap_obj encap_uof_obj; + struct icp_qat_uof_strtable str_table; + struct icp_qat_uclo_encapme ae_uimage[ICP_QAT_UCLO_MAX_UIMAGE]; + struct icp_qat_uclo_aedata ae_data[ICP_QAT_UCLO_MAX_AE]; + struct icp_qat_uclo_init_mem_table init_mem_tab; + struct icp_qat_uof_batch_init *lm_init_tab[ICP_QAT_UCLO_MAX_AE]; + struct icp_qat_uof_batch_init *umem_init_tab[ICP_QAT_UCLO_MAX_AE]; + int uimage_num; + int uword_in_bytes; + int global_inited; + unsigned int ae_num; + unsigned int ustore_phy_size; + void *obj_buf; + u64 *uword_buf; +}; + +struct icp_qat_uof_uword_block { + unsigned int start_addr; + unsigned int words_num; + unsigned int uword_offset; + unsigned int reserved; +}; + +struct icp_qat_uof_filehdr { + unsigned short file_id; + unsigned short reserved1; + char min_ver; + char maj_ver; + unsigned short reserved2; + unsigned short max_chunks; + unsigned short num_chunks; +}; + +struct icp_qat_uof_filechunkhdr { + char chunk_id[ICP_QAT_UOF_OBJID_LEN]; + unsigned int checksum; + unsigned int offset; + unsigned int size; +}; + +struct icp_qat_uof_objhdr { + unsigned int ac_dev_type; + unsigned short min_cpu_ver; + unsigned short max_cpu_ver; + short max_chunks; + short num_chunks; + unsigned int reserved1; + unsigned int reserved2; +}; + +struct icp_qat_uof_chunkhdr { + char chunk_id[ICP_QAT_UOF_OBJID_LEN]; + unsigned int offset; + unsigned int size; +}; + +struct icp_qat_uof_memvar_attr { + unsigned int offset_in_byte; + unsigned int value; +}; + +struct icp_qat_uof_initmem { + unsigned int sym_name; + char region; + char scope; + unsigned short reserved1; + unsigned int addr; + unsigned int num_in_bytes; + unsigned int val_attr_num; +}; + +struct icp_qat_uof_init_regsym { + unsigned int sym_name; + char init_type; + char value_type; + char reg_type; + unsigned char ctx; + unsigned int reg_addr; + unsigned int value; +}; + +struct icp_qat_uof_varmem_seg { + unsigned int sram_base; + unsigned int sram_size; + unsigned int sram_alignment; + unsigned int sdram_base; + unsigned int sdram_size; + unsigned int sdram_alignment; + unsigned int sdram1_base; + unsigned int sdram1_size; + unsigned int sdram1_alignment; + unsigned int scratch_base; + unsigned int scratch_size; + unsigned int scratch_alignment; +}; + +struct icp_qat_uof_gtid { + char tool_id[ICP_QAT_UOF_OBJID_LEN]; + int tool_ver; + unsigned int reserved1; + unsigned int reserved2; +}; + +struct icp_qat_uof_sbreak { + unsigned int page_num; + unsigned int virt_uaddr; + unsigned char sbreak_type; + unsigned char reg_type; + unsigned short reserved1; + unsigned int addr_offset; + unsigned int reg_addr; +}; + +struct icp_qat_uof_code_page { + unsigned int page_region; + unsigned int page_num; + unsigned char def_page; + unsigned char reserved2; + unsigned short reserved1; + unsigned int beg_addr_v; + unsigned int beg_addr_p; + unsigned int neigh_reg_tab_offset; + unsigned int uc_var_tab_offset; + unsigned int imp_var_tab_offset; + unsigned int imp_expr_tab_offset; + unsigned int code_area_offset; +}; + +struct icp_qat_uof_image { + unsigned int img_name; + unsigned int ae_assigned; + unsigned int ctx_assigned; + unsigned int ac_dev_type; + unsigned int entry_address; + unsigned int fill_pattern[2]; + unsigned int reloadable_size; + unsigned char sensitivity; + unsigned char reserved; + unsigned short ae_mode; + unsigned short max_ver; + unsigned short min_ver; + unsigned short image_attrib; + unsigned short reserved2; + unsigned short page_region_num; + unsigned short numpages; + unsigned int reg_tab_offset; + unsigned int init_reg_sym_tab; + unsigned int sbreak_tab; + unsigned int app_metadata; +}; + +struct icp_qat_uof_objtable { + unsigned int entry_num; +}; + +struct icp_qat_uof_ae_reg { + unsigned int name; + unsigned int vis_name; + unsigned short type; + unsigned short addr; + unsigned short access_mode; + unsigned char visible; + unsigned char reserved1; + unsigned short ref_count; + unsigned short reserved2; + unsigned int xo_id; +}; + +struct icp_qat_uof_code_area { + unsigned int micro_words_num; + unsigned int uword_block_tab; +}; + +struct icp_qat_uof_batch_init { + unsigned int ae; + unsigned int addr; + unsigned int *value; + unsigned int size; + struct icp_qat_uof_batch_init *next; +}; + +struct icp_qat_suof_img_hdr { + char *simg_buf; + unsigned long simg_len; + char *css_header; + char *css_simg; + unsigned long simg_size; + unsigned int ae_num; + unsigned int ae_mask; + unsigned int fw_type; + unsigned long simg_name; + unsigned long appmeta_data; +}; + +struct icp_qat_suof_img_tbl { + unsigned int num_simgs; + struct icp_qat_suof_img_hdr *simg_hdr; +}; + +struct icp_qat_suof_handle { + unsigned int file_id; + unsigned int check_sum; + char min_ver; + char maj_ver; + char fw_type; + char *suof_buf; + unsigned int suof_size; + char *sym_str; + unsigned int sym_size; + struct icp_qat_suof_img_tbl img_table; +}; + +struct icp_qat_fw_auth_desc { + unsigned int img_len; + unsigned int ae_mask; + unsigned int css_hdr_high; + unsigned int css_hdr_low; + unsigned int img_high; + unsigned int img_low; + unsigned int signature_high; + unsigned int signature_low; + unsigned int fwsk_pub_high; + unsigned int fwsk_pub_low; + unsigned int img_ae_mode_data_high; + unsigned int img_ae_mode_data_low; + unsigned int img_ae_init_data_high; + unsigned int img_ae_init_data_low; + unsigned int img_ae_insts_high; + unsigned int img_ae_insts_low; + unsigned int cpp_mask; + unsigned int reserved; + unsigned int xmss_pubkey_high; + unsigned int xmss_pubkey_low; + unsigned int xmss_sig_high; + unsigned int xmss_sig_low; + unsigned int reserved2[2]; +}; + +struct icp_qat_auth_chunk { + struct icp_qat_fw_auth_desc fw_auth_desc; + u64 chunk_size; + u64 chunk_bus_addr; +}; + +struct icp_qat_css_hdr { + unsigned int module_type; + unsigned int header_len; + unsigned int header_ver; + unsigned int module_id; + unsigned int module_vendor; + unsigned int date; + unsigned int size; + unsigned int key_size; + unsigned int module_size; + unsigned int exponent_size; + unsigned int fw_type; + unsigned int reserved[21]; +}; + +struct icp_qat_simg_ae_mode { + unsigned int file_id; + unsigned short maj_ver; + unsigned short min_ver; + unsigned int dev_type; + unsigned short devmax_ver; + unsigned short devmin_ver; + unsigned int ae_mask; + unsigned int ctx_enables; + char fw_type; + char ctx_mode; + char nn_mode; + char lm0_mode; + char lm1_mode; + char scs_mode; + char lm2_mode; + char lm3_mode; + char tindex_mode; + unsigned char reserved[7]; + char simg_name[256]; + char appmeta_data[256]; +}; + +struct icp_qat_suof_filehdr { + unsigned int file_id; + unsigned int check_sum; + char min_ver; + char maj_ver; + char fw_type; + char reserved; + unsigned short max_chunks; + unsigned short num_chunks; +}; + +struct icp_qat_suof_chunk_hdr { + char chunk_id[ICP_QAT_SUOF_OBJ_ID_LEN]; + u64 offset; + u64 size; +}; + +struct icp_qat_suof_strtable { + unsigned int tab_length; + unsigned int strings; +}; + +struct icp_qat_suof_objhdr { + unsigned int img_length; + unsigned int reserved; +}; + +struct icp_qat_mof_file_hdr { + unsigned int file_id; + unsigned int checksum; + char min_ver; + char maj_ver; + unsigned short reserved; + unsigned short max_chunks; + unsigned short num_chunks; +}; + +struct icp_qat_mof_chunkhdr { + char chunk_id[ICP_QAT_MOF_OBJ_ID_LEN]; + u64 offset; + u64 size; +}; + +struct icp_qat_mof_str_table { + unsigned int tab_len; + unsigned int strings; +}; + +struct icp_qat_mof_obj_hdr { + unsigned short max_chunks; + unsigned short num_chunks; + unsigned int reserved; +}; + +struct icp_qat_mof_obj_chunkhdr { + char chunk_id[ICP_QAT_MOF_OBJ_CHUNKID_LEN]; + u64 offset; + u64 size; + unsigned int name; + unsigned int reserved; +}; + +struct icp_qat_mof_objhdr { + char *obj_name; + char *obj_buf; + unsigned int obj_size; +}; + +struct icp_qat_mof_table { + unsigned int num_objs; + struct icp_qat_mof_objhdr *obj_hdr; +}; + +struct icp_qat_mof_handle { + unsigned int file_id; + unsigned int checksum; + char min_ver; + char maj_ver; + char *mof_buf; + u32 mof_size; + char *sym_str; + unsigned int sym_size; + char *uobjs_hdr; + char *sobjs_hdr; + struct icp_qat_mof_table obj_table; +}; +#endif diff --git a/drivers/crypto/intel/qat/qat_common/qat_algs.c b/drivers/crypto/intel/qat/qat_common/qat_algs.c new file mode 100644 index 000000000..91663805d --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/qat_algs.c @@ -0,0 +1,1349 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "adf_accel_devices.h" +#include "qat_algs_send.h" +#include "adf_common_drv.h" +#include "qat_crypto.h" +#include "icp_qat_hw.h" +#include "icp_qat_fw.h" +#include "icp_qat_fw_la.h" +#include "qat_bl.h" + +#define QAT_AES_HW_CONFIG_ENC(alg, mode) \ + ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \ + ICP_QAT_HW_CIPHER_NO_CONVERT, \ + ICP_QAT_HW_CIPHER_ENCRYPT) + +#define QAT_AES_HW_CONFIG_DEC(alg, mode) \ + ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \ + ICP_QAT_HW_CIPHER_KEY_CONVERT, \ + ICP_QAT_HW_CIPHER_DECRYPT) + +#define QAT_AES_HW_CONFIG_DEC_NO_CONV(alg, mode) \ + ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \ + ICP_QAT_HW_CIPHER_NO_CONVERT, \ + ICP_QAT_HW_CIPHER_DECRYPT) + +#define HW_CAP_AES_V2(accel_dev) \ + (GET_HW_DATA(accel_dev)->accel_capabilities_mask & \ + ICP_ACCEL_CAPABILITIES_AES_V2) + +static DEFINE_MUTEX(algs_lock); +static unsigned int active_devs; + +/* Common content descriptor */ +struct qat_alg_cd { + union { + struct qat_enc { /* Encrypt content desc */ + struct icp_qat_hw_cipher_algo_blk cipher; + struct icp_qat_hw_auth_algo_blk hash; + } qat_enc_cd; + struct qat_dec { /* Decrypt content desc */ + struct icp_qat_hw_auth_algo_blk hash; + struct icp_qat_hw_cipher_algo_blk cipher; + } qat_dec_cd; + }; +} __aligned(64); + +struct qat_alg_aead_ctx { + struct qat_alg_cd *enc_cd; + struct qat_alg_cd *dec_cd; + dma_addr_t enc_cd_paddr; + dma_addr_t dec_cd_paddr; + struct icp_qat_fw_la_bulk_req enc_fw_req; + struct icp_qat_fw_la_bulk_req dec_fw_req; + enum icp_qat_hw_auth_algo qat_hash_alg; + unsigned int hash_digestsize; + unsigned int hash_blocksize; + struct qat_crypto_instance *inst; +}; + +struct qat_alg_skcipher_ctx { + struct icp_qat_hw_cipher_algo_blk *enc_cd; + struct icp_qat_hw_cipher_algo_blk *dec_cd; + dma_addr_t enc_cd_paddr; + dma_addr_t dec_cd_paddr; + struct icp_qat_fw_la_bulk_req enc_fw_req; + struct icp_qat_fw_la_bulk_req dec_fw_req; + struct qat_crypto_instance *inst; + struct crypto_skcipher *ftfm; + struct crypto_cipher *tweak; + bool fallback; + int mode; +}; + +static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash, + struct qat_alg_aead_ctx *ctx, + const u8 *auth_key, + unsigned int auth_keylen) +{ + switch (ctx->qat_hash_alg) { + case ICP_QAT_HW_AUTH_ALGO_SHA1: { + struct hmac_sha1_key key; + __be32 *istate = (__be32 *)hash->sha.state1; + __be32 *ostate = (__be32 *)(hash->sha.state1 + + round_up(sizeof(key.istate.h), 8)); + + hmac_sha1_preparekey(&key, auth_key, auth_keylen); + for (int i = 0; i < ARRAY_SIZE(key.istate.h); i++) { + istate[i] = cpu_to_be32(key.istate.h[i]); + ostate[i] = cpu_to_be32(key.ostate.h[i]); + } + memzero_explicit(&key, sizeof(key)); + return 0; + } + case ICP_QAT_HW_AUTH_ALGO_SHA256: { + struct hmac_sha256_key key; + __be32 *istate = (__be32 *)hash->sha.state1; + __be32 *ostate = (__be32 *)(hash->sha.state1 + + sizeof(key.key.istate.h)); + + hmac_sha256_preparekey(&key, auth_key, auth_keylen); + for (int i = 0; i < ARRAY_SIZE(key.key.istate.h); i++) { + istate[i] = cpu_to_be32(key.key.istate.h[i]); + ostate[i] = cpu_to_be32(key.key.ostate.h[i]); + } + memzero_explicit(&key, sizeof(key)); + return 0; + } + case ICP_QAT_HW_AUTH_ALGO_SHA512: { + struct hmac_sha512_key key; + __be64 *istate = (__be64 *)hash->sha.state1; + __be64 *ostate = (__be64 *)(hash->sha.state1 + + sizeof(key.key.istate.h)); + + hmac_sha512_preparekey(&key, auth_key, auth_keylen); + for (int i = 0; i < ARRAY_SIZE(key.key.istate.h); i++) { + istate[i] = cpu_to_be64(key.key.istate.h[i]); + ostate[i] = cpu_to_be64(key.key.ostate.h[i]); + } + memzero_explicit(&key, sizeof(key)); + return 0; + } + default: + return -EFAULT; + } +} + +static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header) +{ + header->hdr_flags = + ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET); + header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_LA; + header->comn_req_flags = + ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR, + QAT_COMN_PTR_TYPE_SGL); + ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_PARTIAL_NONE); + ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags, + ICP_QAT_FW_CIPH_IV_16BYTE_DATA); + ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_PROTO); + ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_UPDATE_STATE); +} + +static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm, + int alg, + struct crypto_authenc_keys *keys, + int mode) +{ + struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm); + unsigned int digestsize = crypto_aead_authsize(aead_tfm); + struct qat_enc *enc_ctx = &ctx->enc_cd->qat_enc_cd; + struct icp_qat_hw_cipher_algo_blk *cipher = &enc_ctx->cipher; + struct icp_qat_hw_auth_algo_blk *hash = + (struct icp_qat_hw_auth_algo_blk *)((char *)enc_ctx + + sizeof(struct icp_qat_hw_auth_setup) + keys->enckeylen); + struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req; + struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + void *ptr = &req_tmpl->cd_ctrl; + struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr; + struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr; + + /* CD setup */ + cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode); + memcpy(cipher->aes.key, keys->enckey, keys->enckeylen); + hash->sha.inner_setup.auth_config.config = + ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1, + ctx->qat_hash_alg, digestsize); + hash->sha.inner_setup.auth_counter.counter = + cpu_to_be32(ctx->hash_blocksize); + + if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen)) + return -EFAULT; + + /* Request setup */ + qat_alg_init_common_hdr(header); + header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH; + ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_DIGEST_IN_BUFFER); + ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_RET_AUTH_RES); + ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_CMP_AUTH_RES); + cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr; + cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3; + + /* Cipher CD config setup */ + cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3; + cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3; + cipher_cd_ctrl->cipher_cfg_offset = 0; + ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); + ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_AUTH); + /* Auth CD config setup */ + hash_cd_ctrl->hash_cfg_offset = ((char *)hash - (char *)cipher) >> 3; + hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED; + hash_cd_ctrl->inner_res_sz = digestsize; + hash_cd_ctrl->final_sz = digestsize; + + switch (ctx->qat_hash_alg) { + case ICP_QAT_HW_AUTH_ALGO_SHA1: + hash_cd_ctrl->inner_state1_sz = + round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8); + hash_cd_ctrl->inner_state2_sz = + round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8); + break; + case ICP_QAT_HW_AUTH_ALGO_SHA256: + hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ; + hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ; + break; + case ICP_QAT_HW_AUTH_ALGO_SHA512: + hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ; + hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ; + break; + default: + break; + } + hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset + + ((sizeof(struct icp_qat_hw_auth_setup) + + round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3); + ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH); + ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); + return 0; +} + +static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm, + int alg, + struct crypto_authenc_keys *keys, + int mode) +{ + struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm); + unsigned int digestsize = crypto_aead_authsize(aead_tfm); + struct qat_dec *dec_ctx = &ctx->dec_cd->qat_dec_cd; + struct icp_qat_hw_auth_algo_blk *hash = &dec_ctx->hash; + struct icp_qat_hw_cipher_algo_blk *cipher = + (struct icp_qat_hw_cipher_algo_blk *)((char *)dec_ctx + + sizeof(struct icp_qat_hw_auth_setup) + + roundup(ctx->hash_digestsize, 8) * 2); + struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req; + struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + void *ptr = &req_tmpl->cd_ctrl; + struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr; + struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr; + struct icp_qat_fw_la_auth_req_params *auth_param = + (struct icp_qat_fw_la_auth_req_params *) + ((char *)&req_tmpl->serv_specif_rqpars + + sizeof(struct icp_qat_fw_la_cipher_req_params)); + + /* CD setup */ + cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_DEC(alg, mode); + memcpy(cipher->aes.key, keys->enckey, keys->enckeylen); + hash->sha.inner_setup.auth_config.config = + ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1, + ctx->qat_hash_alg, + digestsize); + hash->sha.inner_setup.auth_counter.counter = + cpu_to_be32(ctx->hash_blocksize); + + if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen)) + return -EFAULT; + + /* Request setup */ + qat_alg_init_common_hdr(header); + header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER; + ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_DIGEST_IN_BUFFER); + ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_RET_AUTH_RES); + ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_CMP_AUTH_RES); + cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr; + cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3; + + /* Cipher CD config setup */ + cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3; + cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3; + cipher_cd_ctrl->cipher_cfg_offset = + (sizeof(struct icp_qat_hw_auth_setup) + + roundup(ctx->hash_digestsize, 8) * 2) >> 3; + ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); + ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); + + /* Auth CD config setup */ + hash_cd_ctrl->hash_cfg_offset = 0; + hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED; + hash_cd_ctrl->inner_res_sz = digestsize; + hash_cd_ctrl->final_sz = digestsize; + + switch (ctx->qat_hash_alg) { + case ICP_QAT_HW_AUTH_ALGO_SHA1: + hash_cd_ctrl->inner_state1_sz = + round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8); + hash_cd_ctrl->inner_state2_sz = + round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8); + break; + case ICP_QAT_HW_AUTH_ALGO_SHA256: + hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ; + hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ; + break; + case ICP_QAT_HW_AUTH_ALGO_SHA512: + hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ; + hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ; + break; + default: + break; + } + + hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset + + ((sizeof(struct icp_qat_hw_auth_setup) + + round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3); + auth_param->auth_res_sz = digestsize; + ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH); + ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); + return 0; +} + +static void qat_alg_skcipher_init_com(struct qat_alg_skcipher_ctx *ctx, + struct icp_qat_fw_la_bulk_req *req, + struct icp_qat_hw_cipher_algo_blk *cd, + const u8 *key, unsigned int keylen) +{ + struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; + struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr; + struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl; + bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev); + int mode = ctx->mode; + + qat_alg_init_common_hdr(header); + header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER; + cd_pars->u.s.content_desc_params_sz = + sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3; + + if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_XTS_MODE) { + ICP_QAT_FW_LA_SLICE_TYPE_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE); + + /* Store both XTS keys in CD, only the first key is sent + * to the HW, the second key is used for tweak calculation + */ + memcpy(cd->ucs_aes.key, key, keylen); + keylen = keylen / 2; + } else if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_CTR_MODE) { + ICP_QAT_FW_LA_SLICE_TYPE_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE); + memcpy(cd->ucs_aes.key, key, keylen); + keylen = round_up(keylen, 16); + } else { + memcpy(cd->aes.key, key, keylen); + } + + /* Cipher CD config setup */ + cd_ctrl->cipher_key_sz = keylen >> 3; + cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3; + cd_ctrl->cipher_cfg_offset = 0; + ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); + ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); +} + +static void qat_alg_skcipher_init_enc(struct qat_alg_skcipher_ctx *ctx, + int alg, const u8 *key, + unsigned int keylen, int mode) +{ + struct icp_qat_hw_cipher_algo_blk *enc_cd = ctx->enc_cd; + struct icp_qat_fw_la_bulk_req *req = &ctx->enc_fw_req; + struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; + + qat_alg_skcipher_init_com(ctx, req, enc_cd, key, keylen); + cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr; + enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode); +} + +static void qat_alg_xts_reverse_key(const u8 *key_forward, unsigned int keylen, + u8 *key_reverse) +{ + struct crypto_aes_ctx aes_expanded; + int nrounds; + u8 *key; + + aes_expandkey(&aes_expanded, key_forward, keylen); + if (keylen == AES_KEYSIZE_128) { + nrounds = 10; + key = (u8 *)aes_expanded.key_enc + (AES_BLOCK_SIZE * nrounds); + memcpy(key_reverse, key, AES_BLOCK_SIZE); + } else { + /* AES_KEYSIZE_256 */ + nrounds = 14; + key = (u8 *)aes_expanded.key_enc + (AES_BLOCK_SIZE * nrounds); + memcpy(key_reverse, key, AES_BLOCK_SIZE); + memcpy(key_reverse + AES_BLOCK_SIZE, key - AES_BLOCK_SIZE, + AES_BLOCK_SIZE); + } + memzero_explicit(&aes_expanded, sizeof(aes_expanded)); +} + +static void qat_alg_skcipher_init_dec(struct qat_alg_skcipher_ctx *ctx, + int alg, const u8 *key, + unsigned int keylen, int mode) +{ + struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd; + struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req; + struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; + bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev); + + qat_alg_skcipher_init_com(ctx, req, dec_cd, key, keylen); + cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr; + + if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_XTS_MODE) { + /* Key reversing not supported, set no convert */ + dec_cd->aes.cipher_config.val = + QAT_AES_HW_CONFIG_DEC_NO_CONV(alg, mode); + + /* In-place key reversal */ + qat_alg_xts_reverse_key(dec_cd->ucs_aes.key, keylen / 2, + dec_cd->ucs_aes.key); + } else if (mode != ICP_QAT_HW_CIPHER_CTR_MODE) { + dec_cd->aes.cipher_config.val = + QAT_AES_HW_CONFIG_DEC(alg, mode); + } else { + dec_cd->aes.cipher_config.val = + QAT_AES_HW_CONFIG_ENC(alg, mode); + } +} + +static int qat_alg_validate_key(int key_len, int *alg, int mode) +{ + if (mode != ICP_QAT_HW_CIPHER_XTS_MODE) { + switch (key_len) { + case AES_KEYSIZE_128: + *alg = ICP_QAT_HW_CIPHER_ALGO_AES128; + break; + case AES_KEYSIZE_192: + *alg = ICP_QAT_HW_CIPHER_ALGO_AES192; + break; + case AES_KEYSIZE_256: + *alg = ICP_QAT_HW_CIPHER_ALGO_AES256; + break; + default: + return -EINVAL; + } + } else { + switch (key_len) { + case AES_KEYSIZE_128 << 1: + *alg = ICP_QAT_HW_CIPHER_ALGO_AES128; + break; + case AES_KEYSIZE_256 << 1: + *alg = ICP_QAT_HW_CIPHER_ALGO_AES256; + break; + default: + return -EINVAL; + } + } + return 0; +} + +static int qat_alg_aead_init_sessions(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen, int mode) +{ + struct crypto_authenc_keys keys; + int alg; + + if (crypto_authenc_extractkeys(&keys, key, keylen)) + goto bad_key; + + if (qat_alg_validate_key(keys.enckeylen, &alg, mode)) + goto bad_key; + + if (qat_alg_aead_init_enc_session(tfm, alg, &keys, mode)) + goto error; + + if (qat_alg_aead_init_dec_session(tfm, alg, &keys, mode)) + goto error; + + memzero_explicit(&keys, sizeof(keys)); + return 0; +bad_key: + memzero_explicit(&keys, sizeof(keys)); + return -EINVAL; +error: + memzero_explicit(&keys, sizeof(keys)); + return -EFAULT; +} + +static int qat_alg_skcipher_init_sessions(struct qat_alg_skcipher_ctx *ctx, + const u8 *key, + unsigned int keylen, + int mode) +{ + int alg; + + if (qat_alg_validate_key(keylen, &alg, mode)) + return -EINVAL; + + qat_alg_skcipher_init_enc(ctx, alg, key, keylen, mode); + qat_alg_skcipher_init_dec(ctx, alg, key, keylen, mode); + return 0; +} + +static int qat_alg_aead_rekey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); + + memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd)); + memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd)); + memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req)); + memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req)); + + return qat_alg_aead_init_sessions(tfm, key, keylen, + ICP_QAT_HW_CIPHER_CBC_MODE); +} + +static int qat_alg_aead_newkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct qat_crypto_instance *inst = NULL; + int node = numa_node_id(); + struct device *dev; + int ret; + + inst = qat_crypto_get_instance_node(node); + if (!inst) + return -EINVAL; + dev = &GET_DEV(inst->accel_dev); + ctx->inst = inst; + ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd), + &ctx->enc_cd_paddr, + GFP_ATOMIC); + if (!ctx->enc_cd) { + ret = -ENOMEM; + goto out_free_inst; + } + ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd), + &ctx->dec_cd_paddr, + GFP_ATOMIC); + if (!ctx->dec_cd) { + ret = -ENOMEM; + goto out_free_enc; + } + + ret = qat_alg_aead_init_sessions(tfm, key, keylen, + ICP_QAT_HW_CIPHER_CBC_MODE); + if (ret) + goto out_free_all; + + return 0; + +out_free_all: + memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd)); + dma_free_coherent(dev, sizeof(struct qat_alg_cd), + ctx->dec_cd, ctx->dec_cd_paddr); + ctx->dec_cd = NULL; +out_free_enc: + memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd)); + dma_free_coherent(dev, sizeof(struct qat_alg_cd), + ctx->enc_cd, ctx->enc_cd_paddr); + ctx->enc_cd = NULL; +out_free_inst: + ctx->inst = NULL; + qat_crypto_put_instance(inst); + return ret; +} + +static int qat_alg_aead_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); + + if (ctx->enc_cd) + return qat_alg_aead_rekey(tfm, key, keylen); + else + return qat_alg_aead_newkey(tfm, key, keylen); +} + +static void qat_aead_alg_callback(struct icp_qat_fw_la_resp *qat_resp, + struct qat_crypto_request *qat_req) +{ + struct qat_alg_aead_ctx *ctx = qat_req->aead_ctx; + struct qat_crypto_instance *inst = ctx->inst; + struct aead_request *areq = qat_req->aead_req; + u8 stat_filed = qat_resp->comn_resp.comn_status; + int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed); + + qat_bl_free_bufl(inst->accel_dev, &qat_req->buf); + if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK)) + res = -EBADMSG; + aead_request_complete(areq, res); +} + +static void qat_alg_update_iv_ctr_mode(struct qat_crypto_request *qat_req) +{ + struct skcipher_request *sreq = qat_req->skcipher_req; + u64 iv_lo_prev; + u64 iv_lo; + u64 iv_hi; + + memcpy(qat_req->iv, sreq->iv, AES_BLOCK_SIZE); + + iv_lo = be64_to_cpu(qat_req->iv_lo); + iv_hi = be64_to_cpu(qat_req->iv_hi); + + iv_lo_prev = iv_lo; + iv_lo += DIV_ROUND_UP(sreq->cryptlen, AES_BLOCK_SIZE); + if (iv_lo < iv_lo_prev) + iv_hi++; + + qat_req->iv_lo = cpu_to_be64(iv_lo); + qat_req->iv_hi = cpu_to_be64(iv_hi); +} + +static void qat_alg_update_iv_cbc_mode(struct qat_crypto_request *qat_req) +{ + struct skcipher_request *sreq = qat_req->skcipher_req; + int offset = sreq->cryptlen - AES_BLOCK_SIZE; + struct scatterlist *sgl; + + if (qat_req->encryption) + sgl = sreq->dst; + else + sgl = sreq->src; + + scatterwalk_map_and_copy(qat_req->iv, sgl, offset, AES_BLOCK_SIZE, 0); +} + +static void qat_alg_update_iv(struct qat_crypto_request *qat_req) +{ + struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx; + struct device *dev = &GET_DEV(ctx->inst->accel_dev); + + switch (ctx->mode) { + case ICP_QAT_HW_CIPHER_CTR_MODE: + qat_alg_update_iv_ctr_mode(qat_req); + break; + case ICP_QAT_HW_CIPHER_CBC_MODE: + qat_alg_update_iv_cbc_mode(qat_req); + break; + case ICP_QAT_HW_CIPHER_XTS_MODE: + break; + default: + dev_warn(dev, "Unsupported IV update for cipher mode %d\n", + ctx->mode); + } +} + +static void qat_skcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp, + struct qat_crypto_request *qat_req) +{ + struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx; + struct qat_crypto_instance *inst = ctx->inst; + struct skcipher_request *sreq = qat_req->skcipher_req; + u8 stat_filed = qat_resp->comn_resp.comn_status; + int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed); + + qat_bl_free_bufl(inst->accel_dev, &qat_req->buf); + if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK)) + res = -EINVAL; + + if (qat_req->encryption) + qat_alg_update_iv(qat_req); + + memcpy(sreq->iv, qat_req->iv, AES_BLOCK_SIZE); + + skcipher_request_complete(sreq, res); +} + +void qat_alg_callback(void *resp) +{ + struct icp_qat_fw_la_resp *qat_resp = resp; + struct qat_crypto_request *qat_req = + (void *)(__force long)qat_resp->opaque_data; + struct qat_instance_backlog *backlog = qat_req->alg_req.backlog; + + qat_req->cb(qat_resp, qat_req); + + qat_alg_send_backlog(backlog); +} + +static int qat_alg_send_sym_message(struct qat_crypto_request *qat_req, + struct qat_crypto_instance *inst, + struct crypto_async_request *base) +{ + struct qat_alg_req *alg_req = &qat_req->alg_req; + + alg_req->fw_req = (u32 *)&qat_req->req; + alg_req->tx_ring = inst->sym_tx; + alg_req->base = base; + alg_req->backlog = &inst->backlog; + + return qat_alg_send_message(alg_req); +} + +static int qat_alg_aead_dec(struct aead_request *areq) +{ + struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq); + struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm); + struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_crypto_request *qat_req = aead_request_ctx(areq); + struct icp_qat_fw_la_cipher_req_params *cipher_param; + struct icp_qat_fw_la_auth_req_params *auth_param; + struct icp_qat_fw_la_bulk_req *msg; + int digst_size = crypto_aead_authsize(aead_tfm); + gfp_t f = qat_algs_alloc_flags(&areq->base); + int ret; + u32 cipher_len; + + cipher_len = areq->cryptlen - digst_size; + if (cipher_len % AES_BLOCK_SIZE != 0) + return -EINVAL; + + ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, areq->src, areq->dst, + &qat_req->buf, NULL, f); + if (unlikely(ret)) + return ret; + + msg = &qat_req->req; + *msg = ctx->dec_fw_req; + qat_req->aead_ctx = ctx; + qat_req->aead_req = areq; + qat_req->cb = qat_aead_alg_callback; + qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req; + qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; + qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; + cipher_param = (void *)&qat_req->req.serv_specif_rqpars; + cipher_param->cipher_length = cipher_len; + cipher_param->cipher_offset = areq->assoclen; + memcpy(cipher_param->u.cipher_IV_array, areq->iv, AES_BLOCK_SIZE); + auth_param = (void *)((u8 *)cipher_param + sizeof(*cipher_param)); + auth_param->auth_off = 0; + auth_param->auth_len = areq->assoclen + cipher_param->cipher_length; + + ret = qat_alg_send_sym_message(qat_req, ctx->inst, &areq->base); + if (ret == -ENOSPC) + qat_bl_free_bufl(ctx->inst->accel_dev, &qat_req->buf); + + return ret; +} + +static int qat_alg_aead_enc(struct aead_request *areq) +{ + struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq); + struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm); + struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_crypto_request *qat_req = aead_request_ctx(areq); + struct icp_qat_fw_la_cipher_req_params *cipher_param; + struct icp_qat_fw_la_auth_req_params *auth_param; + gfp_t f = qat_algs_alloc_flags(&areq->base); + struct icp_qat_fw_la_bulk_req *msg; + u8 *iv = areq->iv; + int ret; + + if (areq->cryptlen % AES_BLOCK_SIZE != 0) + return -EINVAL; + + ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, areq->src, areq->dst, + &qat_req->buf, NULL, f); + if (unlikely(ret)) + return ret; + + msg = &qat_req->req; + *msg = ctx->enc_fw_req; + qat_req->aead_ctx = ctx; + qat_req->aead_req = areq; + qat_req->cb = qat_aead_alg_callback; + qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req; + qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; + qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; + cipher_param = (void *)&qat_req->req.serv_specif_rqpars; + auth_param = (void *)((u8 *)cipher_param + sizeof(*cipher_param)); + + memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE); + cipher_param->cipher_length = areq->cryptlen; + cipher_param->cipher_offset = areq->assoclen; + + auth_param->auth_off = 0; + auth_param->auth_len = areq->assoclen + areq->cryptlen; + + ret = qat_alg_send_sym_message(qat_req, ctx->inst, &areq->base); + if (ret == -ENOSPC) + qat_bl_free_bufl(ctx->inst->accel_dev, &qat_req->buf); + + return ret; +} + +static int qat_alg_skcipher_rekey(struct qat_alg_skcipher_ctx *ctx, + const u8 *key, unsigned int keylen, + int mode) +{ + memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd)); + memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd)); + memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req)); + memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req)); + + return qat_alg_skcipher_init_sessions(ctx, key, keylen, mode); +} + +static int qat_alg_skcipher_newkey(struct qat_alg_skcipher_ctx *ctx, + const u8 *key, unsigned int keylen, + int mode) +{ + struct qat_crypto_instance *inst = NULL; + struct device *dev; + int node = numa_node_id(); + int ret; + + inst = qat_crypto_get_instance_node(node); + if (!inst) + return -EINVAL; + dev = &GET_DEV(inst->accel_dev); + ctx->inst = inst; + ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd), + &ctx->enc_cd_paddr, + GFP_ATOMIC); + if (!ctx->enc_cd) { + ret = -ENOMEM; + goto out_free_instance; + } + ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd), + &ctx->dec_cd_paddr, + GFP_ATOMIC); + if (!ctx->dec_cd) { + ret = -ENOMEM; + goto out_free_enc; + } + + ret = qat_alg_skcipher_init_sessions(ctx, key, keylen, mode); + if (ret) + goto out_free_all; + + return 0; + +out_free_all: + memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd)); + dma_free_coherent(dev, sizeof(*ctx->dec_cd), + ctx->dec_cd, ctx->dec_cd_paddr); + ctx->dec_cd = NULL; +out_free_enc: + memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd)); + dma_free_coherent(dev, sizeof(*ctx->enc_cd), + ctx->enc_cd, ctx->enc_cd_paddr); + ctx->enc_cd = NULL; +out_free_instance: + ctx->inst = NULL; + qat_crypto_put_instance(inst); + return ret; +} + +static int qat_alg_skcipher_setkey(struct crypto_skcipher *tfm, + const u8 *key, unsigned int keylen, + int mode) +{ + struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); + + ctx->mode = mode; + + if (ctx->enc_cd) + return qat_alg_skcipher_rekey(ctx, key, keylen, mode); + else + return qat_alg_skcipher_newkey(ctx, key, keylen, mode); +} + +static int qat_alg_skcipher_cbc_setkey(struct crypto_skcipher *tfm, + const u8 *key, unsigned int keylen) +{ + return qat_alg_skcipher_setkey(tfm, key, keylen, + ICP_QAT_HW_CIPHER_CBC_MODE); +} + +static int qat_alg_skcipher_ctr_setkey(struct crypto_skcipher *tfm, + const u8 *key, unsigned int keylen) +{ + return qat_alg_skcipher_setkey(tfm, key, keylen, + ICP_QAT_HW_CIPHER_CTR_MODE); +} + +static int qat_alg_skcipher_xts_setkey(struct crypto_skcipher *tfm, + const u8 *key, unsigned int keylen) +{ + struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); + int ret; + + ret = xts_verify_key(tfm, key, keylen); + if (ret) + return ret; + + if (keylen >> 1 == AES_KEYSIZE_192) { + ret = crypto_skcipher_setkey(ctx->ftfm, key, keylen); + if (ret) + return ret; + + ctx->fallback = true; + + return 0; + } + + ctx->fallback = false; + + ret = qat_alg_skcipher_setkey(tfm, key, keylen, + ICP_QAT_HW_CIPHER_XTS_MODE); + if (ret) + return ret; + + if (HW_CAP_AES_V2(ctx->inst->accel_dev)) + ret = crypto_cipher_setkey(ctx->tweak, key + (keylen / 2), + keylen / 2); + + return ret; +} + +static void qat_alg_set_req_iv(struct qat_crypto_request *qat_req) +{ + struct icp_qat_fw_la_cipher_req_params *cipher_param; + struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx; + bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev); + u8 *iv = qat_req->skcipher_req->iv; + + cipher_param = (void *)&qat_req->req.serv_specif_rqpars; + + if (aes_v2_capable && ctx->mode == ICP_QAT_HW_CIPHER_XTS_MODE) + crypto_cipher_encrypt_one(ctx->tweak, + (u8 *)cipher_param->u.cipher_IV_array, + iv); + else + memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE); +} + +static int qat_alg_skcipher_encrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); + struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm); + struct qat_alg_skcipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_crypto_request *qat_req = skcipher_request_ctx(req); + struct icp_qat_fw_la_cipher_req_params *cipher_param; + gfp_t f = qat_algs_alloc_flags(&req->base); + struct icp_qat_fw_la_bulk_req *msg; + int ret; + + if (req->cryptlen == 0) + return 0; + + ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, req->src, req->dst, + &qat_req->buf, NULL, f); + if (unlikely(ret)) + return ret; + + msg = &qat_req->req; + *msg = ctx->enc_fw_req; + qat_req->skcipher_ctx = ctx; + qat_req->skcipher_req = req; + qat_req->cb = qat_skcipher_alg_callback; + qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req; + qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; + qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; + qat_req->encryption = true; + cipher_param = (void *)&qat_req->req.serv_specif_rqpars; + cipher_param->cipher_length = req->cryptlen; + cipher_param->cipher_offset = 0; + + qat_alg_set_req_iv(qat_req); + + ret = qat_alg_send_sym_message(qat_req, ctx->inst, &req->base); + if (ret == -ENOSPC) + qat_bl_free_bufl(ctx->inst->accel_dev, &qat_req->buf); + + return ret; +} + +static int qat_alg_skcipher_blk_encrypt(struct skcipher_request *req) +{ + if (req->cryptlen % AES_BLOCK_SIZE != 0) + return -EINVAL; + + return qat_alg_skcipher_encrypt(req); +} + +static int qat_alg_skcipher_xts_encrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); + struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(stfm); + struct skcipher_request *nreq = skcipher_request_ctx(req); + + if (req->cryptlen < XTS_BLOCK_SIZE) + return -EINVAL; + + if (ctx->fallback) { + memcpy(nreq, req, sizeof(*req)); + skcipher_request_set_tfm(nreq, ctx->ftfm); + return crypto_skcipher_encrypt(nreq); + } + + return qat_alg_skcipher_encrypt(req); +} + +static int qat_alg_skcipher_decrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); + struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm); + struct qat_alg_skcipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_crypto_request *qat_req = skcipher_request_ctx(req); + struct icp_qat_fw_la_cipher_req_params *cipher_param; + gfp_t f = qat_algs_alloc_flags(&req->base); + struct icp_qat_fw_la_bulk_req *msg; + int ret; + + if (req->cryptlen == 0) + return 0; + + ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, req->src, req->dst, + &qat_req->buf, NULL, f); + if (unlikely(ret)) + return ret; + + msg = &qat_req->req; + *msg = ctx->dec_fw_req; + qat_req->skcipher_ctx = ctx; + qat_req->skcipher_req = req; + qat_req->cb = qat_skcipher_alg_callback; + qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req; + qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; + qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; + qat_req->encryption = false; + cipher_param = (void *)&qat_req->req.serv_specif_rqpars; + cipher_param->cipher_length = req->cryptlen; + cipher_param->cipher_offset = 0; + + qat_alg_set_req_iv(qat_req); + qat_alg_update_iv(qat_req); + + ret = qat_alg_send_sym_message(qat_req, ctx->inst, &req->base); + if (ret == -ENOSPC) + qat_bl_free_bufl(ctx->inst->accel_dev, &qat_req->buf); + + return ret; +} + +static int qat_alg_skcipher_blk_decrypt(struct skcipher_request *req) +{ + if (req->cryptlen % AES_BLOCK_SIZE != 0) + return -EINVAL; + + return qat_alg_skcipher_decrypt(req); +} + +static int qat_alg_skcipher_xts_decrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); + struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(stfm); + struct skcipher_request *nreq = skcipher_request_ctx(req); + + if (req->cryptlen < XTS_BLOCK_SIZE) + return -EINVAL; + + if (ctx->fallback) { + memcpy(nreq, req, sizeof(*req)); + skcipher_request_set_tfm(nreq, ctx->ftfm); + return crypto_skcipher_decrypt(nreq); + } + + return qat_alg_skcipher_decrypt(req); +} + +static int qat_alg_aead_init(struct crypto_aead *tfm, + enum icp_qat_hw_auth_algo hash_alg, + unsigned int hash_digestsize, + unsigned int hash_blocksize) +{ + struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); + + ctx->qat_hash_alg = hash_alg; + ctx->hash_digestsize = hash_digestsize; + ctx->hash_blocksize = hash_blocksize; + crypto_aead_set_reqsize(tfm, sizeof(struct qat_crypto_request)); + return 0; +} + +static int qat_alg_aead_sha1_init(struct crypto_aead *tfm) +{ + return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, + SHA1_DIGEST_SIZE, SHA1_BLOCK_SIZE); +} + +static int qat_alg_aead_sha256_init(struct crypto_aead *tfm) +{ + return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, + SHA256_DIGEST_SIZE, SHA256_BLOCK_SIZE); +} + +static int qat_alg_aead_sha512_init(struct crypto_aead *tfm) +{ + return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, + SHA512_DIGEST_SIZE, SHA512_BLOCK_SIZE); +} + +static void qat_alg_aead_exit(struct crypto_aead *tfm) +{ + struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev; + + if (!inst) + return; + + dev = &GET_DEV(inst->accel_dev); + if (ctx->enc_cd) { + memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd)); + dma_free_coherent(dev, sizeof(struct qat_alg_cd), + ctx->enc_cd, ctx->enc_cd_paddr); + } + if (ctx->dec_cd) { + memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd)); + dma_free_coherent(dev, sizeof(struct qat_alg_cd), + ctx->dec_cd, ctx->dec_cd_paddr); + } + qat_crypto_put_instance(inst); +} + +static int qat_alg_skcipher_init_tfm(struct crypto_skcipher *tfm) +{ + crypto_skcipher_set_reqsize(tfm, sizeof(struct qat_crypto_request)); + return 0; +} + +static int qat_alg_skcipher_init_xts_tfm(struct crypto_skcipher *tfm) +{ + struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); + int reqsize; + + ctx->ftfm = crypto_alloc_skcipher("xts(aes)", 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->ftfm)) + return PTR_ERR(ctx->ftfm); + + ctx->tweak = crypto_alloc_cipher("aes", 0, 0); + if (IS_ERR(ctx->tweak)) { + crypto_free_skcipher(ctx->ftfm); + return PTR_ERR(ctx->tweak); + } + + reqsize = max(sizeof(struct qat_crypto_request), + sizeof(struct skcipher_request) + + crypto_skcipher_reqsize(ctx->ftfm)); + crypto_skcipher_set_reqsize(tfm, reqsize); + + return 0; +} + +static void qat_alg_skcipher_exit_tfm(struct crypto_skcipher *tfm) +{ + struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev; + + if (!inst) + return; + + dev = &GET_DEV(inst->accel_dev); + if (ctx->enc_cd) { + memset(ctx->enc_cd, 0, + sizeof(struct icp_qat_hw_cipher_algo_blk)); + dma_free_coherent(dev, + sizeof(struct icp_qat_hw_cipher_algo_blk), + ctx->enc_cd, ctx->enc_cd_paddr); + } + if (ctx->dec_cd) { + memset(ctx->dec_cd, 0, + sizeof(struct icp_qat_hw_cipher_algo_blk)); + dma_free_coherent(dev, + sizeof(struct icp_qat_hw_cipher_algo_blk), + ctx->dec_cd, ctx->dec_cd_paddr); + } + qat_crypto_put_instance(inst); +} + +static void qat_alg_skcipher_exit_xts_tfm(struct crypto_skcipher *tfm) +{ + struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (ctx->ftfm) + crypto_free_skcipher(ctx->ftfm); + + if (ctx->tweak) + crypto_free_cipher(ctx->tweak); + + qat_alg_skcipher_exit_tfm(tfm); +} + +static struct aead_alg qat_aeads[] = { { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "qat_aes_cbc_hmac_sha1", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), + .cra_module = THIS_MODULE, + }, + .init = qat_alg_aead_sha1_init, + .exit = qat_alg_aead_exit, + .setkey = qat_alg_aead_setkey, + .decrypt = qat_alg_aead_dec, + .encrypt = qat_alg_aead_enc, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "qat_aes_cbc_hmac_sha256", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), + .cra_module = THIS_MODULE, + }, + .init = qat_alg_aead_sha256_init, + .exit = qat_alg_aead_exit, + .setkey = qat_alg_aead_setkey, + .decrypt = qat_alg_aead_dec, + .encrypt = qat_alg_aead_enc, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(aes))", + .cra_driver_name = "qat_aes_cbc_hmac_sha512", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), + .cra_module = THIS_MODULE, + }, + .init = qat_alg_aead_sha512_init, + .exit = qat_alg_aead_exit, + .setkey = qat_alg_aead_setkey, + .decrypt = qat_alg_aead_dec, + .encrypt = qat_alg_aead_enc, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, +} }; + +static struct skcipher_alg qat_skciphers[] = { { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "qat_aes_cbc", + .base.cra_priority = 100, + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx), + .base.cra_alignmask = 0, + .base.cra_module = THIS_MODULE, + + .init = qat_alg_skcipher_init_tfm, + .exit = qat_alg_skcipher_exit_tfm, + .setkey = qat_alg_skcipher_cbc_setkey, + .decrypt = qat_alg_skcipher_blk_decrypt, + .encrypt = qat_alg_skcipher_blk_encrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, +}, { + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "qat_aes_ctr", + .base.cra_priority = 100, + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx), + .base.cra_alignmask = 0, + .base.cra_module = THIS_MODULE, + + .init = qat_alg_skcipher_init_tfm, + .exit = qat_alg_skcipher_exit_tfm, + .setkey = qat_alg_skcipher_ctr_setkey, + .decrypt = qat_alg_skcipher_decrypt, + .encrypt = qat_alg_skcipher_encrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, +}, { + .base.cra_name = "xts(aes)", + .base.cra_driver_name = "qat_aes_xts", + .base.cra_priority = 100, + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ALLOCATES_MEMORY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx), + .base.cra_alignmask = 0, + .base.cra_module = THIS_MODULE, + + .init = qat_alg_skcipher_init_xts_tfm, + .exit = qat_alg_skcipher_exit_xts_tfm, + .setkey = qat_alg_skcipher_xts_setkey, + .decrypt = qat_alg_skcipher_xts_decrypt, + .encrypt = qat_alg_skcipher_xts_encrypt, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, +} }; + +int qat_algs_register(void) +{ + int ret = 0; + + mutex_lock(&algs_lock); + if (++active_devs != 1) + goto unlock; + + ret = crypto_register_skciphers(qat_skciphers, + ARRAY_SIZE(qat_skciphers)); + if (ret) + goto unlock; + + ret = crypto_register_aeads(qat_aeads, ARRAY_SIZE(qat_aeads)); + if (ret) + goto unreg_algs; + +unlock: + mutex_unlock(&algs_lock); + return ret; + +unreg_algs: + crypto_unregister_skciphers(qat_skciphers, ARRAY_SIZE(qat_skciphers)); + goto unlock; +} + +void qat_algs_unregister(void) +{ + mutex_lock(&algs_lock); + if (--active_devs != 0) + goto unlock; + + crypto_unregister_aeads(qat_aeads, ARRAY_SIZE(qat_aeads)); + crypto_unregister_skciphers(qat_skciphers, ARRAY_SIZE(qat_skciphers)); + +unlock: + mutex_unlock(&algs_lock); +} diff --git a/drivers/crypto/intel/qat/qat_common/qat_algs_send.c b/drivers/crypto/intel/qat/qat_common/qat_algs_send.c new file mode 100644 index 000000000..b97b67882 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/qat_algs_send.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2022 Intel Corporation */ +#include +#include "adf_transport.h" +#include "qat_algs_send.h" +#include "qat_crypto.h" + +#define ADF_MAX_RETRIES 20 + +static int qat_alg_send_message_retry(struct qat_alg_req *req) +{ + int ret = 0, ctr = 0; + + do { + ret = adf_send_message(req->tx_ring, req->fw_req); + } while (ret == -EAGAIN && ctr++ < ADF_MAX_RETRIES); + + if (ret == -EAGAIN) + return -ENOSPC; + + return -EINPROGRESS; +} + +void qat_alg_send_backlog(struct qat_instance_backlog *backlog) +{ + struct qat_alg_req *req, *tmp; + + spin_lock_bh(&backlog->lock); + list_for_each_entry_safe(req, tmp, &backlog->list, list) { + if (adf_send_message(req->tx_ring, req->fw_req)) { + /* The HW ring is full. Do nothing. + * qat_alg_send_backlog() will be invoked again by + * another callback. + */ + break; + } + list_del(&req->list); + crypto_request_complete(req->base, -EINPROGRESS); + } + spin_unlock_bh(&backlog->lock); +} + +static bool qat_alg_try_enqueue(struct qat_alg_req *req) +{ + struct qat_instance_backlog *backlog = req->backlog; + struct adf_etr_ring_data *tx_ring = req->tx_ring; + u32 *fw_req = req->fw_req; + + /* Check if any request is already backlogged */ + if (!list_empty(&backlog->list)) + return false; + + /* Check if ring is nearly full */ + if (adf_ring_nearly_full(tx_ring)) + return false; + + /* Try to enqueue to HW ring */ + if (adf_send_message(tx_ring, fw_req)) + return false; + + return true; +} + + +static int qat_alg_send_message_maybacklog(struct qat_alg_req *req) +{ + struct qat_instance_backlog *backlog = req->backlog; + int ret = -EINPROGRESS; + + if (qat_alg_try_enqueue(req)) + return ret; + + spin_lock_bh(&backlog->lock); + if (!qat_alg_try_enqueue(req)) { + list_add_tail(&req->list, &backlog->list); + ret = -EBUSY; + } + spin_unlock_bh(&backlog->lock); + + return ret; +} + +int qat_alg_send_message(struct qat_alg_req *req) +{ + u32 flags = req->base->flags; + + if (flags & CRYPTO_TFM_REQ_MAY_BACKLOG) + return qat_alg_send_message_maybacklog(req); + else + return qat_alg_send_message_retry(req); +} diff --git a/drivers/crypto/intel/qat/qat_common/qat_algs_send.h b/drivers/crypto/intel/qat/qat_common/qat_algs_send.h new file mode 100644 index 000000000..0baca16e1 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/qat_algs_send.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2022 Intel Corporation */ +#ifndef QAT_ALGS_SEND_H +#define QAT_ALGS_SEND_H + +#include +#include "adf_transport_internal.h" + +struct qat_instance_backlog { + struct list_head list; + spinlock_t lock; /* protects backlog list */ +}; + +struct qat_alg_req { + u32 *fw_req; + struct adf_etr_ring_data *tx_ring; + struct crypto_async_request *base; + struct list_head list; + struct qat_instance_backlog *backlog; +}; + +int qat_alg_send_message(struct qat_alg_req *req); +void qat_alg_send_backlog(struct qat_instance_backlog *backlog); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c b/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c new file mode 100644 index 000000000..75c15c8e4 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c @@ -0,0 +1,1361 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "icp_qat_fw_pke.h" +#include "adf_accel_devices.h" +#include "qat_algs_send.h" +#include "adf_transport.h" +#include "adf_common_drv.h" +#include "qat_crypto.h" + +static DEFINE_MUTEX(algs_lock); +static unsigned int active_devs; + +struct qat_rsa_input_params { + union { + struct { + dma_addr_t m; + dma_addr_t e; + dma_addr_t n; + } enc; + struct { + dma_addr_t c; + dma_addr_t d; + dma_addr_t n; + } dec; + struct { + dma_addr_t c; + dma_addr_t p; + dma_addr_t q; + dma_addr_t dp; + dma_addr_t dq; + dma_addr_t qinv; + } dec_crt; + u64 in_tab[8]; + }; +} __packed __aligned(64); + +struct qat_rsa_output_params { + union { + struct { + dma_addr_t c; + } enc; + struct { + dma_addr_t m; + } dec; + u64 out_tab[8]; + }; +} __packed __aligned(64); + +struct qat_rsa_ctx { + char *n; + char *e; + char *d; + char *p; + char *q; + char *dp; + char *dq; + char *qinv; + dma_addr_t dma_n; + dma_addr_t dma_e; + dma_addr_t dma_d; + dma_addr_t dma_p; + dma_addr_t dma_q; + dma_addr_t dma_dp; + dma_addr_t dma_dq; + dma_addr_t dma_qinv; + unsigned int key_sz; + bool crt_mode; + struct qat_crypto_instance *inst; +} __packed __aligned(64); + +struct qat_dh_input_params { + union { + struct { + dma_addr_t b; + dma_addr_t xa; + dma_addr_t p; + } in; + struct { + dma_addr_t xa; + dma_addr_t p; + } in_g2; + u64 in_tab[8]; + }; +} __packed __aligned(64); + +struct qat_dh_output_params { + union { + dma_addr_t r; + u64 out_tab[8]; + }; +} __packed __aligned(64); + +struct qat_dh_ctx { + char *g; + char *xa; + char *p; + dma_addr_t dma_g; + dma_addr_t dma_xa; + dma_addr_t dma_p; + unsigned int p_size; + bool g2; + struct qat_crypto_instance *inst; + struct crypto_kpp *ftfm; + bool fallback; +} __packed __aligned(64); + +struct qat_asym_request { + union { + struct qat_rsa_input_params rsa; + struct qat_dh_input_params dh; + } in; + union { + struct qat_rsa_output_params rsa; + struct qat_dh_output_params dh; + } out; + dma_addr_t phy_in; + dma_addr_t phy_out; + char *src_align; + char *dst_align; + struct icp_qat_fw_pke_request req; + union { + struct qat_rsa_ctx *rsa; + struct qat_dh_ctx *dh; + } ctx; + union { + struct akcipher_request *rsa; + struct kpp_request *dh; + } areq; + int err; + void (*cb)(struct icp_qat_fw_pke_resp *resp); + struct qat_alg_req alg_req; +} __aligned(64); + +static int qat_alg_send_asym_message(struct qat_asym_request *qat_req, + struct qat_crypto_instance *inst, + struct crypto_async_request *base) +{ + struct qat_alg_req *alg_req = &qat_req->alg_req; + + alg_req->fw_req = (u32 *)&qat_req->req; + alg_req->tx_ring = inst->pke_tx; + alg_req->base = base; + alg_req->backlog = &inst->backlog; + + return qat_alg_send_message(alg_req); +} + +static void qat_dh_cb(struct icp_qat_fw_pke_resp *resp) +{ + struct qat_asym_request *req = (void *)(__force long)resp->opaque; + struct kpp_request *areq = req->areq.dh; + struct device *dev = &GET_DEV(req->ctx.dh->inst->accel_dev); + int err = ICP_QAT_FW_PKE_RESP_PKE_STAT_GET( + resp->pke_resp_hdr.comn_resp_flags); + + err = (err == ICP_QAT_FW_COMN_STATUS_FLAG_OK) ? 0 : -EINVAL; + + if (areq->src) { + dma_unmap_single(dev, req->in.dh.in.b, req->ctx.dh->p_size, + DMA_TO_DEVICE); + kfree_sensitive(req->src_align); + } + + areq->dst_len = req->ctx.dh->p_size; + dma_unmap_single(dev, req->out.dh.r, req->ctx.dh->p_size, + DMA_FROM_DEVICE); + if (req->dst_align) { + scatterwalk_map_and_copy(req->dst_align, areq->dst, 0, + areq->dst_len, 1); + kfree_sensitive(req->dst_align); + } + + dma_unmap_single(dev, req->phy_in, sizeof(struct qat_dh_input_params), + DMA_TO_DEVICE); + dma_unmap_single(dev, req->phy_out, + sizeof(struct qat_dh_output_params), + DMA_TO_DEVICE); + + kpp_request_complete(areq, err); +} + +#define PKE_DH_1536 0x390c1a49 +#define PKE_DH_G2_1536 0x2e0b1a3e +#define PKE_DH_2048 0x4d0c1a60 +#define PKE_DH_G2_2048 0x3e0b1a55 +#define PKE_DH_3072 0x510c1a77 +#define PKE_DH_G2_3072 0x3a0b1a6c +#define PKE_DH_4096 0x690c1a8e +#define PKE_DH_G2_4096 0x4a0b1a83 + +static unsigned long qat_dh_fn_id(unsigned int len, bool g2) +{ + unsigned int bitslen = len << 3; + + switch (bitslen) { + case 1536: + return g2 ? PKE_DH_G2_1536 : PKE_DH_1536; + case 2048: + return g2 ? PKE_DH_G2_2048 : PKE_DH_2048; + case 3072: + return g2 ? PKE_DH_G2_3072 : PKE_DH_3072; + case 4096: + return g2 ? PKE_DH_G2_4096 : PKE_DH_4096; + default: + return 0; + } +} + +static int qat_dh_compute_value(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev = &GET_DEV(inst->accel_dev); + struct qat_asym_request *qat_req = + PTR_ALIGN(kpp_request_ctx(req), 64); + struct icp_qat_fw_pke_request *msg = &qat_req->req; + gfp_t flags = qat_algs_alloc_flags(&req->base); + int n_input_params = 0; + u8 *vaddr; + int ret; + + if (unlikely(!ctx->xa)) + return -EINVAL; + + if (req->dst_len < ctx->p_size) { + req->dst_len = ctx->p_size; + return -EOVERFLOW; + } + + if (req->src_len > ctx->p_size) + return -EINVAL; + + memset(msg, '\0', sizeof(*msg)); + ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(msg->pke_hdr, + ICP_QAT_FW_COMN_REQ_FLAG_SET); + + msg->pke_hdr.cd_pars.func_id = qat_dh_fn_id(ctx->p_size, + !req->src && ctx->g2); + if (unlikely(!msg->pke_hdr.cd_pars.func_id)) + return -EINVAL; + + qat_req->cb = qat_dh_cb; + qat_req->ctx.dh = ctx; + qat_req->areq.dh = req; + msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; + msg->pke_hdr.comn_req_flags = + ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR, + QAT_COMN_PTR_TYPE_FLAT); + + /* + * If no source is provided use g as base + */ + if (req->src) { + qat_req->in.dh.in.xa = ctx->dma_xa; + qat_req->in.dh.in.p = ctx->dma_p; + n_input_params = 3; + } else { + if (ctx->g2) { + qat_req->in.dh.in_g2.xa = ctx->dma_xa; + qat_req->in.dh.in_g2.p = ctx->dma_p; + n_input_params = 2; + } else { + qat_req->in.dh.in.b = ctx->dma_g; + qat_req->in.dh.in.xa = ctx->dma_xa; + qat_req->in.dh.in.p = ctx->dma_p; + n_input_params = 3; + } + } + + ret = -ENOMEM; + if (req->src) { + /* + * src can be of any size in valid range, but HW expects it to + * be the same as modulo p so in case it is different we need + * to allocate a new buf and copy src data. + * In other case we just need to map the user provided buffer. + * Also need to make sure that it is in contiguous buffer. + */ + if (sg_is_last(req->src) && req->src_len == ctx->p_size) { + qat_req->src_align = NULL; + vaddr = sg_virt(req->src); + } else { + int shift = ctx->p_size - req->src_len; + + qat_req->src_align = kzalloc(ctx->p_size, flags); + if (unlikely(!qat_req->src_align)) + return ret; + + scatterwalk_map_and_copy(qat_req->src_align + shift, + req->src, 0, req->src_len, 0); + + vaddr = qat_req->src_align; + } + + qat_req->in.dh.in.b = dma_map_single(dev, vaddr, ctx->p_size, + DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, qat_req->in.dh.in.b))) + goto unmap_src; + } + /* + * dst can be of any size in valid range, but HW expects it to be the + * same as modulo m so in case it is different we need to allocate a + * new buf and copy src data. + * In other case we just need to map the user provided buffer. + * Also need to make sure that it is in contiguous buffer. + */ + if (sg_is_last(req->dst) && req->dst_len == ctx->p_size) { + qat_req->dst_align = NULL; + vaddr = sg_virt(req->dst); + } else { + qat_req->dst_align = kzalloc(ctx->p_size, flags); + if (unlikely(!qat_req->dst_align)) + goto unmap_src; + + vaddr = qat_req->dst_align; + } + qat_req->out.dh.r = dma_map_single(dev, vaddr, ctx->p_size, + DMA_FROM_DEVICE); + if (unlikely(dma_mapping_error(dev, qat_req->out.dh.r))) + goto unmap_dst; + + qat_req->in.dh.in_tab[n_input_params] = 0; + qat_req->out.dh.out_tab[1] = 0; + /* Mapping in.in.b or in.in_g2.xa is the same */ + qat_req->phy_in = dma_map_single(dev, &qat_req->in.dh, + sizeof(struct qat_dh_input_params), + DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) + goto unmap_dst; + + qat_req->phy_out = dma_map_single(dev, &qat_req->out.dh, + sizeof(struct qat_dh_output_params), + DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) + goto unmap_in_params; + + msg->pke_mid.src_data_addr = qat_req->phy_in; + msg->pke_mid.dest_data_addr = qat_req->phy_out; + msg->pke_mid.opaque = (u64)(__force long)qat_req; + msg->input_param_count = n_input_params; + msg->output_param_count = 1; + + ret = qat_alg_send_asym_message(qat_req, inst, &req->base); + if (ret == -ENOSPC) + goto unmap_all; + + return ret; + +unmap_all: + if (!dma_mapping_error(dev, qat_req->phy_out)) + dma_unmap_single(dev, qat_req->phy_out, + sizeof(struct qat_dh_output_params), + DMA_TO_DEVICE); +unmap_in_params: + if (!dma_mapping_error(dev, qat_req->phy_in)) + dma_unmap_single(dev, qat_req->phy_in, + sizeof(struct qat_dh_input_params), + DMA_TO_DEVICE); +unmap_dst: + if (!dma_mapping_error(dev, qat_req->out.dh.r)) + dma_unmap_single(dev, qat_req->out.dh.r, ctx->p_size, + DMA_FROM_DEVICE); + kfree_sensitive(qat_req->dst_align); +unmap_src: + if (req->src) { + if (!dma_mapping_error(dev, qat_req->in.dh.in.b)) + dma_unmap_single(dev, qat_req->in.dh.in.b, + ctx->p_size, + DMA_TO_DEVICE); + kfree_sensitive(qat_req->src_align); + } + return ret; +} + +static int qat_dh_generate_public_key(struct kpp_request *req) +{ + struct kpp_request *nreq = kpp_request_ctx(req); + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); + + if (ctx->fallback) { + memcpy(nreq, req, sizeof(*req)); + kpp_request_set_tfm(nreq, ctx->ftfm); + return crypto_kpp_generate_public_key(nreq); + } + + return qat_dh_compute_value(req); +} + +static int qat_dh_compute_shared_secret(struct kpp_request *req) +{ + struct kpp_request *nreq = kpp_request_ctx(req); + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); + + if (ctx->fallback) { + memcpy(nreq, req, sizeof(*req)); + kpp_request_set_tfm(nreq, ctx->ftfm); + return crypto_kpp_compute_shared_secret(nreq); + } + + return qat_dh_compute_value(req); +} + +static int qat_dh_check_params_length(unsigned int p_len) +{ + switch (p_len) { + case 1536: + case 2048: + case 3072: + case 4096: + return 0; + } + return -EINVAL; +} + +static int qat_dh_set_params(struct qat_dh_ctx *ctx, struct dh *params) +{ + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev = &GET_DEV(inst->accel_dev); + + ctx->p_size = params->p_size; + ctx->p = dma_alloc_coherent(dev, ctx->p_size, &ctx->dma_p, GFP_KERNEL); + if (!ctx->p) + return -ENOMEM; + memcpy(ctx->p, params->p, ctx->p_size); + + /* If g equals 2 don't copy it */ + if (params->g_size == 1 && *(char *)params->g == 0x02) { + ctx->g2 = true; + return 0; + } + + ctx->g = dma_alloc_coherent(dev, ctx->p_size, &ctx->dma_g, GFP_KERNEL); + if (!ctx->g) + return -ENOMEM; + memcpy(ctx->g + (ctx->p_size - params->g_size), params->g, + params->g_size); + + return 0; +} + +static void qat_dh_clear_ctx(struct device *dev, struct qat_dh_ctx *ctx) +{ + if (ctx->g) { + memset(ctx->g, 0, ctx->p_size); + dma_free_coherent(dev, ctx->p_size, ctx->g, ctx->dma_g); + ctx->g = NULL; + } + if (ctx->xa) { + memset(ctx->xa, 0, ctx->p_size); + dma_free_coherent(dev, ctx->p_size, ctx->xa, ctx->dma_xa); + ctx->xa = NULL; + } + if (ctx->p) { + memset(ctx->p, 0, ctx->p_size); + dma_free_coherent(dev, ctx->p_size, ctx->p, ctx->dma_p); + ctx->p = NULL; + } + ctx->p_size = 0; + ctx->g2 = false; +} + +static int qat_dh_set_secret(struct crypto_kpp *tfm, const void *buf, + unsigned int len) +{ + struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); + struct device *dev = &GET_DEV(ctx->inst->accel_dev); + struct dh params; + int ret; + + if (crypto_dh_decode_key(buf, len, ¶ms) < 0) + return -EINVAL; + + if (qat_dh_check_params_length(params.p_size << 3)) { + ctx->fallback = true; + return crypto_kpp_set_secret(ctx->ftfm, buf, len); + } + + ctx->fallback = false; + + /* Free old secret if any */ + qat_dh_clear_ctx(dev, ctx); + + ret = qat_dh_set_params(ctx, ¶ms); + if (ret < 0) + goto err_clear_ctx; + + ctx->xa = dma_alloc_coherent(dev, ctx->p_size, &ctx->dma_xa, + GFP_KERNEL); + if (!ctx->xa) { + ret = -ENOMEM; + goto err_clear_ctx; + } + memcpy(ctx->xa + (ctx->p_size - params.key_size), params.key, + params.key_size); + + return 0; + +err_clear_ctx: + qat_dh_clear_ctx(dev, ctx); + return ret; +} + +static unsigned int qat_dh_max_size(struct crypto_kpp *tfm) +{ + struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); + + if (ctx->fallback) + return crypto_kpp_maxsize(ctx->ftfm); + + return ctx->p_size; +} + +static int qat_dh_init_tfm(struct crypto_kpp *tfm) +{ + struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); + struct qat_crypto_instance *inst = + qat_crypto_get_instance_node(numa_node_id()); + const char *alg = kpp_alg_name(tfm); + unsigned int reqsize; + + if (!inst) + return -EINVAL; + + ctx->ftfm = crypto_alloc_kpp(alg, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->ftfm)) + return PTR_ERR(ctx->ftfm); + + crypto_kpp_set_flags(ctx->ftfm, crypto_kpp_get_flags(tfm)); + + reqsize = max(sizeof(struct qat_asym_request) + 64, + sizeof(struct kpp_request) + crypto_kpp_reqsize(ctx->ftfm)); + + kpp_set_reqsize(tfm, reqsize); + + ctx->p_size = 0; + ctx->g2 = false; + ctx->inst = inst; + return 0; +} + +static void qat_dh_exit_tfm(struct crypto_kpp *tfm) +{ + struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); + struct device *dev = &GET_DEV(ctx->inst->accel_dev); + + if (ctx->ftfm) + crypto_free_kpp(ctx->ftfm); + + qat_dh_clear_ctx(dev, ctx); + qat_crypto_put_instance(ctx->inst); +} + +static void qat_rsa_cb(struct icp_qat_fw_pke_resp *resp) +{ + struct qat_asym_request *req = (void *)(__force long)resp->opaque; + struct akcipher_request *areq = req->areq.rsa; + struct device *dev = &GET_DEV(req->ctx.rsa->inst->accel_dev); + int err = ICP_QAT_FW_PKE_RESP_PKE_STAT_GET( + resp->pke_resp_hdr.comn_resp_flags); + + err = (err == ICP_QAT_FW_COMN_STATUS_FLAG_OK) ? 0 : -EINVAL; + + dma_unmap_single(dev, req->in.rsa.enc.m, req->ctx.rsa->key_sz, + DMA_TO_DEVICE); + + kfree_sensitive(req->src_align); + + areq->dst_len = req->ctx.rsa->key_sz; + dma_unmap_single(dev, req->out.rsa.enc.c, req->ctx.rsa->key_sz, + DMA_FROM_DEVICE); + if (req->dst_align) { + scatterwalk_map_and_copy(req->dst_align, areq->dst, 0, + areq->dst_len, 1); + + kfree_sensitive(req->dst_align); + } + + dma_unmap_single(dev, req->phy_in, sizeof(struct qat_rsa_input_params), + DMA_TO_DEVICE); + dma_unmap_single(dev, req->phy_out, + sizeof(struct qat_rsa_output_params), + DMA_TO_DEVICE); + + akcipher_request_complete(areq, err); +} + +void qat_alg_asym_callback(void *_resp) +{ + struct icp_qat_fw_pke_resp *resp = _resp; + struct qat_asym_request *areq = (void *)(__force long)resp->opaque; + struct qat_instance_backlog *backlog = areq->alg_req.backlog; + + areq->cb(resp); + + qat_alg_send_backlog(backlog); +} + +#define PKE_RSA_EP_512 0x1c161b21 +#define PKE_RSA_EP_1024 0x35111bf7 +#define PKE_RSA_EP_1536 0x4d111cdc +#define PKE_RSA_EP_2048 0x6e111dba +#define PKE_RSA_EP_3072 0x7d111ea3 +#define PKE_RSA_EP_4096 0xa5101f7e + +static unsigned long qat_rsa_enc_fn_id(unsigned int len) +{ + unsigned int bitslen = len << 3; + + switch (bitslen) { + case 512: + return PKE_RSA_EP_512; + case 1024: + return PKE_RSA_EP_1024; + case 1536: + return PKE_RSA_EP_1536; + case 2048: + return PKE_RSA_EP_2048; + case 3072: + return PKE_RSA_EP_3072; + case 4096: + return PKE_RSA_EP_4096; + default: + return 0; + } +} + +#define PKE_RSA_DP1_512 0x1c161b3c +#define PKE_RSA_DP1_1024 0x35111c12 +#define PKE_RSA_DP1_1536 0x4d111cf7 +#define PKE_RSA_DP1_2048 0x6e111dda +#define PKE_RSA_DP1_3072 0x7d111ebe +#define PKE_RSA_DP1_4096 0xa5101f98 + +static unsigned long qat_rsa_dec_fn_id(unsigned int len) +{ + unsigned int bitslen = len << 3; + + switch (bitslen) { + case 512: + return PKE_RSA_DP1_512; + case 1024: + return PKE_RSA_DP1_1024; + case 1536: + return PKE_RSA_DP1_1536; + case 2048: + return PKE_RSA_DP1_2048; + case 3072: + return PKE_RSA_DP1_3072; + case 4096: + return PKE_RSA_DP1_4096; + default: + return 0; + } +} + +#define PKE_RSA_DP2_512 0x1c131b57 +#define PKE_RSA_DP2_1024 0x26131c2d +#define PKE_RSA_DP2_1536 0x45111d12 +#define PKE_RSA_DP2_2048 0x59121dfa +#define PKE_RSA_DP2_3072 0x81121ed9 +#define PKE_RSA_DP2_4096 0xb1111fb2 + +static unsigned long qat_rsa_dec_fn_id_crt(unsigned int len) +{ + unsigned int bitslen = len << 3; + + switch (bitslen) { + case 512: + return PKE_RSA_DP2_512; + case 1024: + return PKE_RSA_DP2_1024; + case 1536: + return PKE_RSA_DP2_1536; + case 2048: + return PKE_RSA_DP2_2048; + case 3072: + return PKE_RSA_DP2_3072; + case 4096: + return PKE_RSA_DP2_4096; + default: + return 0; + } +} + +static int qat_rsa_enc(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev = &GET_DEV(inst->accel_dev); + struct qat_asym_request *qat_req = + PTR_ALIGN(akcipher_request_ctx(req), 64); + struct icp_qat_fw_pke_request *msg = &qat_req->req; + gfp_t flags = qat_algs_alloc_flags(&req->base); + u8 *vaddr; + int ret; + + if (unlikely(!ctx->n || !ctx->e)) + return -EINVAL; + + if (req->dst_len < ctx->key_sz) { + req->dst_len = ctx->key_sz; + return -EOVERFLOW; + } + + if (req->src_len > ctx->key_sz) + return -EINVAL; + + memset(msg, '\0', sizeof(*msg)); + ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(msg->pke_hdr, + ICP_QAT_FW_COMN_REQ_FLAG_SET); + msg->pke_hdr.cd_pars.func_id = qat_rsa_enc_fn_id(ctx->key_sz); + if (unlikely(!msg->pke_hdr.cd_pars.func_id)) + return -EINVAL; + + qat_req->cb = qat_rsa_cb; + qat_req->ctx.rsa = ctx; + qat_req->areq.rsa = req; + msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; + msg->pke_hdr.comn_req_flags = + ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR, + QAT_COMN_PTR_TYPE_FLAT); + + qat_req->in.rsa.enc.e = ctx->dma_e; + qat_req->in.rsa.enc.n = ctx->dma_n; + ret = -ENOMEM; + + /* + * src can be of any size in valid range, but HW expects it to be the + * same as modulo n so in case it is different we need to allocate a + * new buf and copy src data. + * In other case we just need to map the user provided buffer. + * Also need to make sure that it is in contiguous buffer. + */ + if (sg_is_last(req->src) && req->src_len == ctx->key_sz) { + qat_req->src_align = NULL; + vaddr = sg_virt(req->src); + } else { + int shift = ctx->key_sz - req->src_len; + + qat_req->src_align = kzalloc(ctx->key_sz, flags); + if (unlikely(!qat_req->src_align)) + return ret; + + scatterwalk_map_and_copy(qat_req->src_align + shift, req->src, + 0, req->src_len, 0); + vaddr = qat_req->src_align; + } + + qat_req->in.rsa.enc.m = dma_map_single(dev, vaddr, ctx->key_sz, + DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, qat_req->in.rsa.enc.m))) + goto unmap_src; + + if (sg_is_last(req->dst) && req->dst_len == ctx->key_sz) { + qat_req->dst_align = NULL; + vaddr = sg_virt(req->dst); + } else { + qat_req->dst_align = kzalloc(ctx->key_sz, flags); + if (unlikely(!qat_req->dst_align)) + goto unmap_src; + vaddr = qat_req->dst_align; + } + + qat_req->out.rsa.enc.c = dma_map_single(dev, vaddr, ctx->key_sz, + DMA_FROM_DEVICE); + if (unlikely(dma_mapping_error(dev, qat_req->out.rsa.enc.c))) + goto unmap_dst; + + qat_req->in.rsa.in_tab[3] = 0; + qat_req->out.rsa.out_tab[1] = 0; + qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa, + sizeof(struct qat_rsa_input_params), + DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) + goto unmap_dst; + + qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa, + sizeof(struct qat_rsa_output_params), + DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) + goto unmap_in_params; + + msg->pke_mid.src_data_addr = qat_req->phy_in; + msg->pke_mid.dest_data_addr = qat_req->phy_out; + msg->pke_mid.opaque = (u64)(__force long)qat_req; + msg->input_param_count = 3; + msg->output_param_count = 1; + + ret = qat_alg_send_asym_message(qat_req, inst, &req->base); + if (ret == -ENOSPC) + goto unmap_all; + + return ret; + +unmap_all: + if (!dma_mapping_error(dev, qat_req->phy_out)) + dma_unmap_single(dev, qat_req->phy_out, + sizeof(struct qat_rsa_output_params), + DMA_TO_DEVICE); +unmap_in_params: + if (!dma_mapping_error(dev, qat_req->phy_in)) + dma_unmap_single(dev, qat_req->phy_in, + sizeof(struct qat_rsa_input_params), + DMA_TO_DEVICE); +unmap_dst: + if (!dma_mapping_error(dev, qat_req->out.rsa.enc.c)) + dma_unmap_single(dev, qat_req->out.rsa.enc.c, + ctx->key_sz, DMA_FROM_DEVICE); + kfree_sensitive(qat_req->dst_align); +unmap_src: + if (!dma_mapping_error(dev, qat_req->in.rsa.enc.m)) + dma_unmap_single(dev, qat_req->in.rsa.enc.m, ctx->key_sz, + DMA_TO_DEVICE); + kfree_sensitive(qat_req->src_align); + return ret; +} + +static int qat_rsa_dec(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev = &GET_DEV(inst->accel_dev); + struct qat_asym_request *qat_req = + PTR_ALIGN(akcipher_request_ctx(req), 64); + struct icp_qat_fw_pke_request *msg = &qat_req->req; + gfp_t flags = qat_algs_alloc_flags(&req->base); + u8 *vaddr; + int ret; + + if (unlikely(!ctx->n || !ctx->d)) + return -EINVAL; + + if (req->dst_len < ctx->key_sz) { + req->dst_len = ctx->key_sz; + return -EOVERFLOW; + } + + if (req->src_len > ctx->key_sz) + return -EINVAL; + + memset(msg, '\0', sizeof(*msg)); + ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(msg->pke_hdr, + ICP_QAT_FW_COMN_REQ_FLAG_SET); + msg->pke_hdr.cd_pars.func_id = ctx->crt_mode ? + qat_rsa_dec_fn_id_crt(ctx->key_sz) : + qat_rsa_dec_fn_id(ctx->key_sz); + if (unlikely(!msg->pke_hdr.cd_pars.func_id)) + return -EINVAL; + + qat_req->cb = qat_rsa_cb; + qat_req->ctx.rsa = ctx; + qat_req->areq.rsa = req; + msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; + msg->pke_hdr.comn_req_flags = + ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR, + QAT_COMN_PTR_TYPE_FLAT); + + if (ctx->crt_mode) { + qat_req->in.rsa.dec_crt.p = ctx->dma_p; + qat_req->in.rsa.dec_crt.q = ctx->dma_q; + qat_req->in.rsa.dec_crt.dp = ctx->dma_dp; + qat_req->in.rsa.dec_crt.dq = ctx->dma_dq; + qat_req->in.rsa.dec_crt.qinv = ctx->dma_qinv; + } else { + qat_req->in.rsa.dec.d = ctx->dma_d; + qat_req->in.rsa.dec.n = ctx->dma_n; + } + ret = -ENOMEM; + + /* + * src can be of any size in valid range, but HW expects it to be the + * same as modulo n so in case it is different we need to allocate a + * new buf and copy src data. + * In other case we just need to map the user provided buffer. + * Also need to make sure that it is in contiguous buffer. + */ + if (sg_is_last(req->src) && req->src_len == ctx->key_sz) { + qat_req->src_align = NULL; + vaddr = sg_virt(req->src); + } else { + int shift = ctx->key_sz - req->src_len; + + qat_req->src_align = kzalloc(ctx->key_sz, flags); + if (unlikely(!qat_req->src_align)) + return ret; + + scatterwalk_map_and_copy(qat_req->src_align + shift, req->src, + 0, req->src_len, 0); + vaddr = qat_req->src_align; + } + + qat_req->in.rsa.dec.c = dma_map_single(dev, vaddr, ctx->key_sz, + DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, qat_req->in.rsa.dec.c))) + goto unmap_src; + + if (sg_is_last(req->dst) && req->dst_len == ctx->key_sz) { + qat_req->dst_align = NULL; + vaddr = sg_virt(req->dst); + } else { + qat_req->dst_align = kzalloc(ctx->key_sz, flags); + if (unlikely(!qat_req->dst_align)) + goto unmap_src; + vaddr = qat_req->dst_align; + } + qat_req->out.rsa.dec.m = dma_map_single(dev, vaddr, ctx->key_sz, + DMA_FROM_DEVICE); + if (unlikely(dma_mapping_error(dev, qat_req->out.rsa.dec.m))) + goto unmap_dst; + + if (ctx->crt_mode) + qat_req->in.rsa.in_tab[6] = 0; + else + qat_req->in.rsa.in_tab[3] = 0; + qat_req->out.rsa.out_tab[1] = 0; + qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa, + sizeof(struct qat_rsa_input_params), + DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) + goto unmap_dst; + + qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa, + sizeof(struct qat_rsa_output_params), + DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) + goto unmap_in_params; + + msg->pke_mid.src_data_addr = qat_req->phy_in; + msg->pke_mid.dest_data_addr = qat_req->phy_out; + msg->pke_mid.opaque = (u64)(__force long)qat_req; + if (ctx->crt_mode) + msg->input_param_count = 6; + else + msg->input_param_count = 3; + + msg->output_param_count = 1; + + ret = qat_alg_send_asym_message(qat_req, inst, &req->base); + if (ret == -ENOSPC) + goto unmap_all; + + return ret; + +unmap_all: + if (!dma_mapping_error(dev, qat_req->phy_out)) + dma_unmap_single(dev, qat_req->phy_out, + sizeof(struct qat_rsa_output_params), + DMA_TO_DEVICE); +unmap_in_params: + if (!dma_mapping_error(dev, qat_req->phy_in)) + dma_unmap_single(dev, qat_req->phy_in, + sizeof(struct qat_rsa_input_params), + DMA_TO_DEVICE); +unmap_dst: + if (!dma_mapping_error(dev, qat_req->out.rsa.dec.m)) + dma_unmap_single(dev, qat_req->out.rsa.dec.m, + ctx->key_sz, DMA_FROM_DEVICE); + kfree_sensitive(qat_req->dst_align); +unmap_src: + if (!dma_mapping_error(dev, qat_req->in.rsa.dec.c)) + dma_unmap_single(dev, qat_req->in.rsa.dec.c, ctx->key_sz, + DMA_TO_DEVICE); + kfree_sensitive(qat_req->src_align); + return ret; +} + +static int qat_rsa_set_n(struct qat_rsa_ctx *ctx, const char *value, + size_t vlen) +{ + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev = &GET_DEV(inst->accel_dev); + const char *ptr = value; + int ret; + + while (!*ptr && vlen) { + ptr++; + vlen--; + } + + ctx->key_sz = vlen; + ret = -EINVAL; + /* invalid key size provided */ + if (!qat_rsa_enc_fn_id(ctx->key_sz)) + goto err; + + ret = -ENOMEM; + ctx->n = dma_alloc_coherent(dev, ctx->key_sz, &ctx->dma_n, GFP_KERNEL); + if (!ctx->n) + goto err; + + memcpy(ctx->n, ptr, ctx->key_sz); + return 0; +err: + ctx->key_sz = 0; + ctx->n = NULL; + return ret; +} + +static int qat_rsa_set_e(struct qat_rsa_ctx *ctx, const char *value, + size_t vlen) +{ + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev = &GET_DEV(inst->accel_dev); + const char *ptr = value; + + while (!*ptr && vlen) { + ptr++; + vlen--; + } + + if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) { + ctx->e = NULL; + return -EINVAL; + } + + ctx->e = dma_alloc_coherent(dev, ctx->key_sz, &ctx->dma_e, GFP_KERNEL); + if (!ctx->e) + return -ENOMEM; + + memcpy(ctx->e + (ctx->key_sz - vlen), ptr, vlen); + return 0; +} + +static int qat_rsa_set_d(struct qat_rsa_ctx *ctx, const char *value, + size_t vlen) +{ + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev = &GET_DEV(inst->accel_dev); + const char *ptr = value; + int ret; + + while (!*ptr && vlen) { + ptr++; + vlen--; + } + + ret = -EINVAL; + if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) + goto err; + + ret = -ENOMEM; + ctx->d = dma_alloc_coherent(dev, ctx->key_sz, &ctx->dma_d, GFP_KERNEL); + if (!ctx->d) + goto err; + + memcpy(ctx->d + (ctx->key_sz - vlen), ptr, vlen); + return 0; +err: + ctx->d = NULL; + return ret; +} + +static void qat_rsa_drop_leading_zeros(const char **ptr, unsigned int *len) +{ + while (!**ptr && *len) { + (*ptr)++; + (*len)--; + } +} + +static void qat_rsa_setkey_crt(struct qat_rsa_ctx *ctx, struct rsa_key *rsa_key) +{ + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev = &GET_DEV(inst->accel_dev); + const char *ptr; + unsigned int len; + unsigned int half_key_sz = ctx->key_sz / 2; + + /* p */ + ptr = rsa_key->p; + len = rsa_key->p_sz; + qat_rsa_drop_leading_zeros(&ptr, &len); + if (!len || len > half_key_sz) + goto err; + ctx->p = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_p, GFP_KERNEL); + if (!ctx->p) + goto err; + memcpy(ctx->p + (half_key_sz - len), ptr, len); + + /* q */ + ptr = rsa_key->q; + len = rsa_key->q_sz; + qat_rsa_drop_leading_zeros(&ptr, &len); + if (!len || len > half_key_sz) + goto free_p; + ctx->q = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_q, GFP_KERNEL); + if (!ctx->q) + goto free_p; + memcpy(ctx->q + (half_key_sz - len), ptr, len); + + /* dp */ + ptr = rsa_key->dp; + len = rsa_key->dp_sz; + qat_rsa_drop_leading_zeros(&ptr, &len); + if (!len || len > half_key_sz) + goto free_q; + ctx->dp = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_dp, + GFP_KERNEL); + if (!ctx->dp) + goto free_q; + memcpy(ctx->dp + (half_key_sz - len), ptr, len); + + /* dq */ + ptr = rsa_key->dq; + len = rsa_key->dq_sz; + qat_rsa_drop_leading_zeros(&ptr, &len); + if (!len || len > half_key_sz) + goto free_dp; + ctx->dq = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_dq, + GFP_KERNEL); + if (!ctx->dq) + goto free_dp; + memcpy(ctx->dq + (half_key_sz - len), ptr, len); + + /* qinv */ + ptr = rsa_key->qinv; + len = rsa_key->qinv_sz; + qat_rsa_drop_leading_zeros(&ptr, &len); + if (!len || len > half_key_sz) + goto free_dq; + ctx->qinv = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_qinv, + GFP_KERNEL); + if (!ctx->qinv) + goto free_dq; + memcpy(ctx->qinv + (half_key_sz - len), ptr, len); + + ctx->crt_mode = true; + return; + +free_dq: + memset(ctx->dq, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->dq, ctx->dma_dq); + ctx->dq = NULL; +free_dp: + memset(ctx->dp, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->dp, ctx->dma_dp); + ctx->dp = NULL; +free_q: + memset(ctx->q, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->q, ctx->dma_q); + ctx->q = NULL; +free_p: + memset(ctx->p, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->p, ctx->dma_p); + ctx->p = NULL; +err: + ctx->crt_mode = false; +} + +static void qat_rsa_clear_ctx(struct device *dev, struct qat_rsa_ctx *ctx) +{ + unsigned int half_key_sz = ctx->key_sz / 2; + + /* Free the old key if any */ + if (ctx->n) + dma_free_coherent(dev, ctx->key_sz, ctx->n, ctx->dma_n); + if (ctx->e) + dma_free_coherent(dev, ctx->key_sz, ctx->e, ctx->dma_e); + if (ctx->d) { + memset(ctx->d, '\0', ctx->key_sz); + dma_free_coherent(dev, ctx->key_sz, ctx->d, ctx->dma_d); + } + if (ctx->p) { + memset(ctx->p, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->p, ctx->dma_p); + } + if (ctx->q) { + memset(ctx->q, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->q, ctx->dma_q); + } + if (ctx->dp) { + memset(ctx->dp, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->dp, ctx->dma_dp); + } + if (ctx->dq) { + memset(ctx->dq, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->dq, ctx->dma_dq); + } + if (ctx->qinv) { + memset(ctx->qinv, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->qinv, ctx->dma_qinv); + } + + ctx->n = NULL; + ctx->e = NULL; + ctx->d = NULL; + ctx->p = NULL; + ctx->q = NULL; + ctx->dp = NULL; + ctx->dq = NULL; + ctx->qinv = NULL; + ctx->crt_mode = false; + ctx->key_sz = 0; +} + +static int qat_rsa_setkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen, bool private) +{ + struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct device *dev = &GET_DEV(ctx->inst->accel_dev); + struct rsa_key rsa_key; + int ret; + + qat_rsa_clear_ctx(dev, ctx); + + if (private) + ret = rsa_parse_priv_key(&rsa_key, key, keylen); + else + ret = rsa_parse_pub_key(&rsa_key, key, keylen); + if (ret < 0) + goto free; + + ret = qat_rsa_set_n(ctx, rsa_key.n, rsa_key.n_sz); + if (ret < 0) + goto free; + ret = qat_rsa_set_e(ctx, rsa_key.e, rsa_key.e_sz); + if (ret < 0) + goto free; + if (private) { + ret = qat_rsa_set_d(ctx, rsa_key.d, rsa_key.d_sz); + if (ret < 0) + goto free; + qat_rsa_setkey_crt(ctx, &rsa_key); + } + + if (!ctx->n || !ctx->e) { + /* invalid key provided */ + ret = -EINVAL; + goto free; + } + if (private && !ctx->d) { + /* invalid private key provided */ + ret = -EINVAL; + goto free; + } + + return 0; +free: + qat_rsa_clear_ctx(dev, ctx); + return ret; +} + +static int qat_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + return qat_rsa_setkey(tfm, key, keylen, false); +} + +static int qat_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + return qat_rsa_setkey(tfm, key, keylen, true); +} + +static unsigned int qat_rsa_max_size(struct crypto_akcipher *tfm) +{ + struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + + return ctx->key_sz; +} + +static int qat_rsa_init_tfm(struct crypto_akcipher *tfm) +{ + struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct qat_crypto_instance *inst = + qat_crypto_get_instance_node(numa_node_id()); + + if (!inst) + return -EINVAL; + + akcipher_set_reqsize(tfm, sizeof(struct qat_asym_request) + 64); + + ctx->key_sz = 0; + ctx->inst = inst; + return 0; +} + +static void qat_rsa_exit_tfm(struct crypto_akcipher *tfm) +{ + struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct device *dev = &GET_DEV(ctx->inst->accel_dev); + + qat_rsa_clear_ctx(dev, ctx); + qat_crypto_put_instance(ctx->inst); +} + +static struct akcipher_alg rsa = { + .encrypt = qat_rsa_enc, + .decrypt = qat_rsa_dec, + .set_pub_key = qat_rsa_setpubkey, + .set_priv_key = qat_rsa_setprivkey, + .max_size = qat_rsa_max_size, + .init = qat_rsa_init_tfm, + .exit = qat_rsa_exit_tfm, + .base = { + .cra_name = "rsa", + .cra_driver_name = "qat-rsa", + .cra_priority = 1000, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct qat_rsa_ctx), + }, +}; + +static struct kpp_alg dh = { + .set_secret = qat_dh_set_secret, + .generate_public_key = qat_dh_generate_public_key, + .compute_shared_secret = qat_dh_compute_shared_secret, + .max_size = qat_dh_max_size, + .init = qat_dh_init_tfm, + .exit = qat_dh_exit_tfm, + .base = { + .cra_name = "dh", + .cra_driver_name = "qat-dh", + .cra_priority = 1000, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct qat_dh_ctx), + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, + }, +}; + +int qat_asym_algs_register(void) +{ + int ret = 0; + + mutex_lock(&algs_lock); + if (++active_devs == 1) { + rsa.base.cra_flags = 0; + ret = crypto_register_akcipher(&rsa); + if (ret) + goto unlock; + ret = crypto_register_kpp(&dh); + } +unlock: + mutex_unlock(&algs_lock); + return ret; +} + +void qat_asym_algs_unregister(void) +{ + mutex_lock(&algs_lock); + if (--active_devs == 0) { + crypto_unregister_akcipher(&rsa); + crypto_unregister_kpp(&dh); + } + mutex_unlock(&algs_lock); +} diff --git a/drivers/crypto/intel/qat/qat_common/qat_bl.c b/drivers/crypto/intel/qat/qat_common/qat_bl.c new file mode 100644 index 000000000..9b2338f58 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/qat_bl.c @@ -0,0 +1,253 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2014 - 2022 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include "adf_accel_devices.h" +#include "qat_bl.h" +#include "qat_crypto.h" + +void qat_bl_free_bufl(struct adf_accel_dev *accel_dev, + struct qat_request_buffs *buf) +{ + struct device *dev = &GET_DEV(accel_dev); + struct qat_alg_buf_list *bl = buf->bl; + struct qat_alg_buf_list *blout = buf->blout; + dma_addr_t blp = buf->blp; + dma_addr_t blpout = buf->bloutp; + size_t sz = buf->sz; + size_t sz_out = buf->sz_out; + int bl_dma_dir; + int i; + + bl_dma_dir = blp != blpout ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; + + for (i = 0; i < bl->num_bufs; i++) + dma_unmap_single(dev, bl->buffers[i].addr, + bl->buffers[i].len, bl_dma_dir); + + dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE); + + if (!buf->sgl_src_valid) + kfree(bl); + + if (blp != blpout) { + for (i = 0; i < blout->num_mapped_bufs; i++) { + dma_unmap_single(dev, blout->buffers[i].addr, + blout->buffers[i].len, + DMA_BIDIRECTIONAL); + } + dma_unmap_single(dev, blpout, sz_out, DMA_TO_DEVICE); + + if (!buf->sgl_dst_valid) + kfree(blout); + } +} + +static int __qat_bl_sgl_to_bufl(struct adf_accel_dev *accel_dev, + struct scatterlist *sgl, + struct scatterlist *sglout, + struct qat_request_buffs *buf, + dma_addr_t extra_dst_buff, + size_t sz_extra_dst_buff, + unsigned int sskip, + unsigned int dskip, + gfp_t flags) +{ + struct device *dev = &GET_DEV(accel_dev); + int i, sg_nctr = 0; + int n = sg_nents(sgl); + struct qat_alg_buf_list *bufl; + struct qat_alg_buf_list *buflout = NULL; + dma_addr_t blp = DMA_MAPPING_ERROR; + dma_addr_t bloutp = DMA_MAPPING_ERROR; + struct scatterlist *sg; + size_t sz_out, sz = struct_size(bufl, buffers, n); + int node = dev_to_node(&GET_DEV(accel_dev)); + unsigned int left; + int bufl_dma_dir; + + if (unlikely(!n)) + return -EINVAL; + + buf->sgl_src_valid = false; + buf->sgl_dst_valid = false; + + if (n > QAT_MAX_BUFF_DESC) { + bufl = kzalloc_node(sz, flags, node); + if (unlikely(!bufl)) + return -ENOMEM; + } else { + bufl = container_of(&buf->sgl_src.sgl_hdr, + struct qat_alg_buf_list, hdr); + memset(bufl, 0, sizeof(struct qat_alg_buf_list)); + buf->sgl_src_valid = true; + } + + bufl_dma_dir = sgl != sglout ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; + + for (i = 0; i < n; i++) + bufl->buffers[i].addr = DMA_MAPPING_ERROR; + + left = sskip; + + for_each_sg(sgl, sg, n, i) { + int y = sg_nctr; + + if (!sg->length) + continue; + + if (left >= sg->length) { + left -= sg->length; + continue; + } + bufl->buffers[y].addr = dma_map_single(dev, sg_virt(sg) + left, + sg->length - left, + bufl_dma_dir); + bufl->buffers[y].len = sg->length; + if (unlikely(dma_mapping_error(dev, bufl->buffers[y].addr))) + goto err_in; + sg_nctr++; + if (left) { + bufl->buffers[y].len -= left; + left = 0; + } + } + bufl->num_bufs = sg_nctr; + blp = dma_map_single(dev, bufl, sz, DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, blp))) + goto err_in; + buf->bl = bufl; + buf->blp = blp; + buf->sz = sz; + /* Handle out of place operation */ + if (sgl != sglout) { + struct qat_alg_buf *buffers; + int extra_buff = extra_dst_buff ? 1 : 0; + int n_sglout = sg_nents(sglout); + + n = n_sglout + extra_buff; + sz_out = struct_size(buflout, buffers, n); + left = dskip; + + sg_nctr = 0; + + if (n > QAT_MAX_BUFF_DESC) { + buflout = kzalloc_node(sz_out, flags, node); + if (unlikely(!buflout)) + goto err_in; + } else { + buflout = container_of(&buf->sgl_dst.sgl_hdr, + struct qat_alg_buf_list, hdr); + memset(buflout, 0, sizeof(struct qat_alg_buf_list)); + buf->sgl_dst_valid = true; + } + + buffers = buflout->buffers; + for (i = 0; i < n; i++) + buffers[i].addr = DMA_MAPPING_ERROR; + + for_each_sg(sglout, sg, n_sglout, i) { + int y = sg_nctr; + + if (!sg->length) + continue; + + if (left >= sg->length) { + left -= sg->length; + continue; + } + buffers[y].addr = dma_map_single(dev, sg_virt(sg) + left, + sg->length - left, + DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(dev, buffers[y].addr))) + goto err_out; + buffers[y].len = sg->length; + sg_nctr++; + if (left) { + buffers[y].len -= left; + left = 0; + } + } + if (extra_buff) { + buffers[sg_nctr].addr = extra_dst_buff; + buffers[sg_nctr].len = sz_extra_dst_buff; + } + + buflout->num_bufs = sg_nctr; + buflout->num_bufs += extra_buff; + buflout->num_mapped_bufs = sg_nctr; + bloutp = dma_map_single(dev, buflout, sz_out, DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, bloutp))) + goto err_out; + buf->blout = buflout; + buf->bloutp = bloutp; + buf->sz_out = sz_out; + } else { + /* Otherwise set the src and dst to the same address */ + buf->bloutp = buf->blp; + buf->sz_out = 0; + } + return 0; + +err_out: + if (!dma_mapping_error(dev, bloutp)) + dma_unmap_single(dev, bloutp, sz_out, DMA_TO_DEVICE); + + n = sg_nents(sglout); + for (i = 0; i < n; i++) { + if (buflout->buffers[i].addr == extra_dst_buff) + break; + if (!dma_mapping_error(dev, buflout->buffers[i].addr)) + dma_unmap_single(dev, buflout->buffers[i].addr, + buflout->buffers[i].len, + DMA_BIDIRECTIONAL); + } + + if (!buf->sgl_dst_valid) + kfree(buflout); + +err_in: + if (!dma_mapping_error(dev, blp)) + dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE); + + n = sg_nents(sgl); + for (i = 0; i < n; i++) + if (!dma_mapping_error(dev, bufl->buffers[i].addr)) + dma_unmap_single(dev, bufl->buffers[i].addr, + bufl->buffers[i].len, + bufl_dma_dir); + + if (!buf->sgl_src_valid) + kfree(bufl); + + dev_err(dev, "Failed to map buf for dma\n"); + return -ENOMEM; +} + +int qat_bl_sgl_to_bufl(struct adf_accel_dev *accel_dev, + struct scatterlist *sgl, + struct scatterlist *sglout, + struct qat_request_buffs *buf, + struct qat_sgl_to_bufl_params *params, + gfp_t flags) +{ + dma_addr_t extra_dst_buff = 0; + size_t sz_extra_dst_buff = 0; + unsigned int sskip = 0; + unsigned int dskip = 0; + + if (params) { + extra_dst_buff = params->extra_dst_buff; + sz_extra_dst_buff = params->sz_extra_dst_buff; + sskip = params->sskip; + dskip = params->dskip; + } + + return __qat_bl_sgl_to_bufl(accel_dev, sgl, sglout, buf, + extra_dst_buff, sz_extra_dst_buff, + sskip, dskip, flags); +} diff --git a/drivers/crypto/intel/qat/qat_common/qat_bl.h b/drivers/crypto/intel/qat/qat_common/qat_bl.h new file mode 100644 index 000000000..2827d5055 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/qat_bl.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2014 - 2022 Intel Corporation */ +#ifndef QAT_BL_H +#define QAT_BL_H +#include +#include +#include + +#define QAT_MAX_BUFF_DESC 4 + +struct qat_alg_buf { + u32 len; + u32 resrvd; + u64 addr; +} __packed; + +struct qat_alg_buf_list { + /* New members must be added within the __struct_group() macro below. */ + __struct_group(qat_alg_buf_list_hdr, hdr, __packed, + u64 resrvd; + u32 num_bufs; + u32 num_mapped_bufs; + ); + struct qat_alg_buf buffers[]; +} __packed; +static_assert(offsetof(struct qat_alg_buf_list, buffers) == sizeof(struct qat_alg_buf_list_hdr), + "struct member likely outside of __struct_group()"); + +struct qat_alg_fixed_buf_list { + struct qat_alg_buf_list_hdr sgl_hdr; + struct qat_alg_buf descriptors[QAT_MAX_BUFF_DESC]; +} __packed __aligned(64); + +struct qat_request_buffs { + struct qat_alg_buf_list *bl; + dma_addr_t blp; + struct qat_alg_buf_list *blout; + dma_addr_t bloutp; + size_t sz; + size_t sz_out; + bool sgl_src_valid; + bool sgl_dst_valid; + struct qat_alg_fixed_buf_list sgl_src; + struct qat_alg_fixed_buf_list sgl_dst; +}; + +struct qat_sgl_to_bufl_params { + dma_addr_t extra_dst_buff; + size_t sz_extra_dst_buff; + unsigned int sskip; + unsigned int dskip; +}; + +void qat_bl_free_bufl(struct adf_accel_dev *accel_dev, + struct qat_request_buffs *buf); +int qat_bl_sgl_to_bufl(struct adf_accel_dev *accel_dev, + struct scatterlist *sgl, + struct scatterlist *sglout, + struct qat_request_buffs *buf, + struct qat_sgl_to_bufl_params *params, + gfp_t flags); + +static inline gfp_t qat_algs_alloc_flags(struct crypto_async_request *req) +{ + return req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; +} + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c b/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c new file mode 100644 index 000000000..50e444da9 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c @@ -0,0 +1,758 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2022 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_dc.h" +#include "qat_bl.h" +#include "qat_comp_req.h" +#include "qat_compression.h" +#include "qat_algs_send.h" +#include "qat_comp_zstd_utils.h" + +#define QAT_ZSTD_SCRATCH_SIZE 524288 +#define QAT_ZSTD_MAX_BLOCK_SIZE 65535 +#define QAT_ZSTD_MAX_CONTENT_SIZE 4096 +#define QAT_LZ4S_MIN_INPUT_SIZE 8192 +#define QAT_LZ4S_MAX_OUTPUT_SIZE QAT_ZSTD_SCRATCH_SIZE +#define QAT_MAX_SEQUENCES (128 * 1024) + +static DEFINE_MUTEX(algs_lock); +static unsigned int active_devs_deflate; +static unsigned int active_devs_lz4s; +static unsigned int active_devs_zstd; + +struct qat_zstd_scratch { + size_t cctx_buffer_size; + void *lz4s; + void *literals; + void *out_seqs; + void *workspace; + ZSTD_CCtx *ctx; +}; + +static void *qat_zstd_alloc_scratch(void) +{ + struct qat_zstd_scratch *scratch; + ZSTD_parameters params; + size_t cctx_size; + ZSTD_CCtx *ctx; + size_t zret; + int ret; + + ret = -ENOMEM; + scratch = kzalloc_obj(*scratch); + if (!scratch) + return ERR_PTR(ret); + + scratch->lz4s = kvmalloc(QAT_ZSTD_SCRATCH_SIZE, GFP_KERNEL); + if (!scratch->lz4s) + goto error; + + scratch->literals = kvmalloc(QAT_ZSTD_SCRATCH_SIZE, GFP_KERNEL); + if (!scratch->literals) + goto error; + + scratch->out_seqs = kvzalloc_objs(ZSTD_Sequence, QAT_MAX_SEQUENCES); + if (!scratch->out_seqs) + goto error; + + params = zstd_get_params(zstd_max_clevel(), QAT_ZSTD_SCRATCH_SIZE); + cctx_size = zstd_cctx_workspace_bound(¶ms.cParams); + + scratch->workspace = kvmalloc(cctx_size, GFP_KERNEL | __GFP_ZERO); + if (!scratch->workspace) + goto error; + + ret = -EINVAL; + ctx = zstd_init_cctx(scratch->workspace, cctx_size); + if (!ctx) + goto error; + + scratch->ctx = ctx; + scratch->cctx_buffer_size = cctx_size; + + zret = zstd_cctx_set_param(ctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters); + if (zstd_is_error(zret)) + goto error; + + return scratch; + +error: + kvfree(scratch->lz4s); + kvfree(scratch->literals); + kvfree(scratch->out_seqs); + kvfree(scratch->workspace); + kfree(scratch); + return ERR_PTR(ret); +} + +static void qat_zstd_free_scratch(void *ctx) +{ + struct qat_zstd_scratch *scratch = ctx; + + if (!scratch) + return; + + kvfree(scratch->lz4s); + kvfree(scratch->literals); + kvfree(scratch->out_seqs); + kvfree(scratch->workspace); + kfree(scratch); +} + +static struct crypto_acomp_streams qat_zstd_streams = { + .alloc_ctx = qat_zstd_alloc_scratch, + .free_ctx = qat_zstd_free_scratch, +}; + +enum direction { + DECOMPRESSION = 0, + COMPRESSION = 1, +}; + +struct qat_compression_req; + +struct qat_callback_params { + unsigned int produced; + unsigned int dlen; + bool plain; +}; + +struct qat_compression_ctx { + u8 comp_ctx[QAT_COMP_CTX_SIZE]; + struct qat_compression_instance *inst; + int (*qat_comp_callback)(struct qat_compression_req *qat_req, void *resp, + struct qat_callback_params *params); + struct crypto_acomp *ftfm; +}; + +struct qat_compression_req { + u8 req[QAT_COMP_REQ_SIZE]; + struct qat_compression_ctx *qat_compression_ctx; + struct acomp_req *acompress_req; + struct qat_request_buffs buf; + enum direction dir; + int actual_dlen; + struct qat_alg_req alg_req; +}; + +static int qat_alg_send_dc_message(struct qat_compression_req *qat_req, + struct qat_compression_instance *inst, + struct crypto_async_request *base) +{ + struct qat_alg_req *alg_req = &qat_req->alg_req; + + alg_req->fw_req = (u32 *)&qat_req->req; + alg_req->tx_ring = inst->dc_tx; + alg_req->base = base; + alg_req->backlog = &inst->backlog; + + return qat_alg_send_message(alg_req); +} + +static void qat_comp_generic_callback(struct qat_compression_req *qat_req, + void *resp) +{ + struct acomp_req *areq = qat_req->acompress_req; + struct qat_compression_ctx *ctx = qat_req->qat_compression_ctx; + struct adf_accel_dev *accel_dev = ctx->inst->accel_dev; + struct crypto_acomp *tfm = crypto_acomp_reqtfm(areq); + struct qat_compression_instance *inst = ctx->inst; + struct qat_callback_params params = { }; + int consumed, produced; + s8 cmp_err, xlt_err; + int res = -EBADMSG; + int status; + u8 cnv; + + status = qat_comp_get_cmp_status(resp); + status |= qat_comp_get_xlt_status(resp); + cmp_err = qat_comp_get_cmp_err(resp); + xlt_err = qat_comp_get_xlt_err(resp); + + consumed = qat_comp_get_consumed_ctr(resp); + produced = qat_comp_get_produced_ctr(resp); + + /* Cache parameters for algorithm specific callback */ + params.produced = produced; + params.dlen = areq->dlen; + + dev_dbg(&GET_DEV(accel_dev), + "[%s][%s][%s] slen = %8d dlen = %8d consumed = %8d produced = %8d cmp_err = %3d xlt_err = %3d", + crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm)), + qat_req->dir == COMPRESSION ? "comp " : "decomp", + status ? "ERR" : "OK ", + areq->slen, areq->dlen, consumed, produced, cmp_err, xlt_err); + + if (unlikely(status != ICP_QAT_FW_COMN_STATUS_FLAG_OK)) { + if (cmp_err == ERR_CODE_OVERFLOW_ERROR || xlt_err == ERR_CODE_OVERFLOW_ERROR) + res = -E2BIG; + + areq->dlen = 0; + goto end; + } + + if (qat_req->dir == COMPRESSION) { + cnv = qat_comp_get_cmp_cnv_flag(resp); + if (unlikely(!cnv)) { + dev_err(&GET_DEV(accel_dev), + "Verified compression not supported\n"); + areq->dlen = 0; + goto end; + } + + if (unlikely(produced > qat_req->actual_dlen)) { + memset(inst->dc_data->ovf_buff, 0, + inst->dc_data->ovf_buff_sz); + dev_dbg(&GET_DEV(accel_dev), + "Actual buffer overflow: produced=%d, dlen=%d\n", + produced, qat_req->actual_dlen); + + res = -E2BIG; + areq->dlen = 0; + goto end; + } + + params.plain = !!qat_comp_get_cmp_uncomp_flag(resp); + } + + res = 0; + areq->dlen = produced; + + if (ctx->qat_comp_callback) + res = ctx->qat_comp_callback(qat_req, resp, ¶ms); + +end: + qat_bl_free_bufl(accel_dev, &qat_req->buf); + acomp_request_complete(areq, res); + qat_alg_send_backlog(qat_req->alg_req.backlog); +} + +void qat_comp_alg_callback(void *resp) +{ + struct qat_compression_req *qat_req = + (void *)(__force long)qat_comp_get_opaque(resp); + + qat_comp_generic_callback(qat_req, resp); +} + +static int qat_comp_alg_init_tfm(struct crypto_acomp *acomp_tfm, int alg) +{ + struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm); + struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm); + struct qat_compression_instance *inst; + int node, ret; + + if (tfm->node == NUMA_NO_NODE) + node = numa_node_id(); + else + node = tfm->node; + + memset(ctx, 0, sizeof(*ctx)); + inst = qat_compression_get_instance_node(node, alg); + if (!inst) + return -EINVAL; + ctx->inst = inst; + + ret = qat_comp_build_ctx(inst->accel_dev, ctx->comp_ctx, alg); + if (ret) { + qat_compression_put_instance(inst); + memset(ctx, 0, sizeof(*ctx)); + } + + return ret; +} + +static int qat_comp_alg_deflate_init_tfm(struct crypto_acomp *acomp_tfm) +{ + return qat_comp_alg_init_tfm(acomp_tfm, QAT_DEFLATE); +} + +static void qat_comp_alg_exit_tfm(struct crypto_acomp *acomp_tfm) +{ + struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm); + + qat_compression_put_instance(ctx->inst); + memset(ctx, 0, sizeof(*ctx)); +} + +static int qat_comp_alg_compress_decompress(struct acomp_req *areq, enum direction dir, + unsigned int shdr, unsigned int sftr, + unsigned int dhdr, unsigned int dftr) +{ + struct qat_compression_req *qat_req = acomp_request_ctx(areq); + struct crypto_acomp *acomp_tfm = crypto_acomp_reqtfm(areq); + struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm); + struct qat_compression_instance *inst = ctx->inst; + gfp_t f = qat_algs_alloc_flags(&areq->base); + struct qat_sgl_to_bufl_params params = {0}; + int slen = areq->slen - shdr - sftr; + int dlen = areq->dlen - dhdr - dftr; + dma_addr_t sfbuf, dfbuf; + u8 *req = qat_req->req; + size_t ovf_buff_sz; + int ret; + + params.sskip = shdr; + params.dskip = dhdr; + + if (!areq->src || !slen) + return -EINVAL; + + if (!areq->dst || !dlen) + return -EINVAL; + + if (dir == COMPRESSION) { + params.extra_dst_buff = inst->dc_data->ovf_buff_p; + ovf_buff_sz = inst->dc_data->ovf_buff_sz; + params.sz_extra_dst_buff = ovf_buff_sz; + } + + ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, areq->src, areq->dst, + &qat_req->buf, ¶ms, f); + if (unlikely(ret)) + return ret; + + sfbuf = qat_req->buf.blp; + dfbuf = qat_req->buf.bloutp; + qat_req->qat_compression_ctx = ctx; + qat_req->acompress_req = areq; + qat_req->dir = dir; + + if (dir == COMPRESSION) { + qat_req->actual_dlen = dlen; + dlen += ovf_buff_sz; + qat_comp_create_compression_req(ctx->comp_ctx, req, + (u64)(__force long)sfbuf, slen, + (u64)(__force long)dfbuf, dlen, + (u64)(__force long)qat_req); + } else { + qat_comp_create_decompression_req(ctx->comp_ctx, req, + (u64)(__force long)sfbuf, slen, + (u64)(__force long)dfbuf, dlen, + (u64)(__force long)qat_req); + } + + ret = qat_alg_send_dc_message(qat_req, inst, &areq->base); + if (ret == -ENOSPC) + qat_bl_free_bufl(inst->accel_dev, &qat_req->buf); + + return ret; +} + +static int qat_comp_alg_compress(struct acomp_req *req) +{ + return qat_comp_alg_compress_decompress(req, COMPRESSION, 0, 0, 0, 0); +} + +static int qat_comp_alg_decompress(struct acomp_req *req) +{ + return qat_comp_alg_compress_decompress(req, DECOMPRESSION, 0, 0, 0, 0); +} + +static int qat_comp_alg_zstd_decompress(struct acomp_req *req) +{ + struct crypto_acomp *acomp_tfm = crypto_acomp_reqtfm(req); + struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm); + struct acomp_req *nreq = acomp_request_ctx(req); + zstd_frame_header header; + void *buffer; + size_t zret; + int ret; + + buffer = kmap_local_page(sg_page(req->src)) + req->src->offset; + zret = zstd_get_frame_header(&header, buffer, req->src->length); + kunmap_local(buffer); + if (zret) { + dev_err(&GET_DEV(ctx->inst->accel_dev), + "ZSTD-compressed data has an incomplete frame header\n"); + return -EINVAL; + } + + if (header.windowSize > QAT_ZSTD_MAX_BLOCK_SIZE || + header.frameContentSize >= QAT_ZSTD_MAX_CONTENT_SIZE) { + dev_dbg(&GET_DEV(ctx->inst->accel_dev), "Window size=0x%llx\n", + header.windowSize); + + memcpy(nreq, req, sizeof(*req)); + acomp_request_set_tfm(nreq, ctx->ftfm); + + ret = crypto_acomp_decompress(nreq); + req->dlen = nreq->dlen; + + return ret; + } + + return qat_comp_alg_compress_decompress(req, DECOMPRESSION, 0, 0, 0, 0); +} + +static int qat_comp_lz4s_zstd_callback(struct qat_compression_req *qat_req, void *resp, + struct qat_callback_params *params) +{ + struct qat_compression_ctx *qat_ctx = qat_req->qat_compression_ctx; + struct acomp_req *areq = qat_req->acompress_req; + struct qat_zstd_scratch *scratch; + struct crypto_acomp_stream *s; + unsigned int lit_len = 0; + ZSTD_Sequence *out_seqs; + void *lz4s, *zstd; + size_t comp_size; + ZSTD_CCtx *ctx; + void *literals; + int seq_count; + int ret = 0; + + if (params->produced + QAT_ZSTD_LIT_COPY_LEN > QAT_ZSTD_SCRATCH_SIZE) { + dev_dbg(&GET_DEV(qat_ctx->inst->accel_dev), + "LZ4s-ZSTD: produced size (%u) + COPY_SIZE > QAT_ZSTD_SCRATCH_SIZE (%u)\n", + params->produced, QAT_ZSTD_SCRATCH_SIZE); + areq->dlen = 0; + return -E2BIG; + } + + s = crypto_acomp_lock_stream_bh(&qat_zstd_streams); + scratch = s->ctx; + + lz4s = scratch->lz4s; + zstd = lz4s; /* Output buffer is same as lz4s */ + out_seqs = scratch->out_seqs; + ctx = scratch->ctx; + literals = scratch->literals; + + if (likely(!params->plain)) { + if (likely(sg_nents(areq->dst) == 1)) { + zstd = sg_virt(areq->dst); + lz4s = zstd; + } else { + memcpy_from_sglist(lz4s, areq->dst, 0, params->produced); + } + + seq_count = qat_alg_dec_lz4s(out_seqs, QAT_MAX_SEQUENCES, lz4s, + params->produced, literals, &lit_len); + if (seq_count < 0) { + ret = seq_count; + comp_size = 0; + goto out; + } + } else { + out_seqs[0].litLength = areq->slen; + out_seqs[0].offset = 0; + out_seqs[0].matchLength = 0; + + seq_count = 1; + } + + comp_size = zstd_compress_sequences_and_literals(ctx, zstd, params->dlen, + out_seqs, seq_count, + literals, lit_len, + QAT_ZSTD_SCRATCH_SIZE, + areq->slen); + if (zstd_is_error(comp_size)) { + if (comp_size == ZSTD_error_cannotProduce_uncompressedBlock) + ret = -E2BIG; + else + ret = -EOPNOTSUPP; + + comp_size = 0; + goto out; + } + + if (comp_size > params->dlen) { + dev_dbg(&GET_DEV(qat_ctx->inst->accel_dev), + "LZ4s-ZSTD: compressed_size (%u) > output buffer size (%u)\n", + (unsigned int)comp_size, params->dlen); + ret = -EOVERFLOW; + goto out; + } + + if (unlikely(sg_nents(areq->dst) != 1)) + memcpy_to_sglist(areq->dst, 0, zstd, comp_size); + +out: + areq->dlen = comp_size; + crypto_acomp_unlock_stream_bh(s); + + return ret; +} + +static int qat_comp_alg_lz4s_zstd_init_tfm(struct crypto_acomp *acomp_tfm) +{ + struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm); + struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm); + int reqsize; + int ret; + + /* qat_comp_alg_init_tfm() wipes out the ctx */ + ret = qat_comp_alg_init_tfm(acomp_tfm, QAT_LZ4S); + if (ret) + return ret; + + ctx->ftfm = crypto_alloc_acomp_node("zstd", 0, CRYPTO_ALG_NEED_FALLBACK, + tfm->node); + if (IS_ERR(ctx->ftfm)) { + qat_comp_alg_exit_tfm(acomp_tfm); + return PTR_ERR(ctx->ftfm); + } + + reqsize = max(sizeof(struct qat_compression_req), + sizeof(struct acomp_req) + crypto_acomp_reqsize(ctx->ftfm)); + + acomp_tfm->reqsize = reqsize; + + ctx->qat_comp_callback = qat_comp_lz4s_zstd_callback; + + return 0; +} + +static int qat_comp_alg_zstd_init_tfm(struct crypto_acomp *acomp_tfm) +{ + struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm); + struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm); + int reqsize; + int ret; + + /* qat_comp_alg_init_tfm() wipes out the ctx */ + ret = qat_comp_alg_init_tfm(acomp_tfm, QAT_ZSTD); + if (ret) + return ret; + + ctx->ftfm = crypto_alloc_acomp_node("zstd", 0, CRYPTO_ALG_NEED_FALLBACK, + tfm->node); + if (IS_ERR(ctx->ftfm)) { + qat_comp_alg_exit_tfm(acomp_tfm); + return PTR_ERR(ctx->ftfm); + } + + reqsize = max(sizeof(struct qat_compression_req), + sizeof(struct acomp_req) + crypto_acomp_reqsize(ctx->ftfm)); + + acomp_tfm->reqsize = reqsize; + + return 0; +} + +static void qat_comp_alg_zstd_exit_tfm(struct crypto_acomp *acomp_tfm) +{ + struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm); + + if (ctx->ftfm) + crypto_free_acomp(ctx->ftfm); + + qat_comp_alg_exit_tfm(acomp_tfm); +} + +static int qat_comp_alg_lz4s_zstd_compress(struct acomp_req *req) +{ + struct crypto_acomp *acomp_tfm = crypto_acomp_reqtfm(req); + struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm); + struct acomp_req *nreq = acomp_request_ctx(req); + int ret; + + if (req->slen >= QAT_LZ4S_MIN_INPUT_SIZE && req->dlen >= QAT_LZ4S_MIN_INPUT_SIZE && + req->slen <= QAT_LZ4S_MAX_OUTPUT_SIZE && req->dlen <= QAT_LZ4S_MAX_OUTPUT_SIZE) + return qat_comp_alg_compress(req); + + memcpy(nreq, req, sizeof(*req)); + acomp_request_set_tfm(nreq, ctx->ftfm); + + ret = crypto_acomp_compress(nreq); + req->dlen = nreq->dlen; + + return ret; +} + +static int qat_comp_alg_sw_decompress(struct acomp_req *req) +{ + struct crypto_acomp *acomp_tfm = crypto_acomp_reqtfm(req); + struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm); + struct acomp_req *nreq = acomp_request_ctx(req); + int ret; + + memcpy(nreq, req, sizeof(*req)); + acomp_request_set_tfm(nreq, ctx->ftfm); + + ret = crypto_acomp_decompress(nreq); + req->dlen = nreq->dlen; + + return ret; +} + +static struct acomp_alg qat_acomp_deflate[] = { { + .base = { + .cra_name = "deflate", + .cra_driver_name = "qat_deflate", + .cra_priority = 4001, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_ctxsize = sizeof(struct qat_compression_ctx), + .cra_reqsize = sizeof(struct qat_compression_req), + .cra_module = THIS_MODULE, + }, + .init = qat_comp_alg_deflate_init_tfm, + .exit = qat_comp_alg_exit_tfm, + .compress = qat_comp_alg_compress, + .decompress = qat_comp_alg_decompress, +}}; + +static struct acomp_alg qat_acomp_zstd_lz4s = { + .base = { + .cra_name = "zstd", + .cra_driver_name = "qat_zstd", + .cra_priority = 4001, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_reqsize = sizeof(struct qat_compression_req), + .cra_ctxsize = sizeof(struct qat_compression_ctx), + .cra_module = THIS_MODULE, + }, + .init = qat_comp_alg_lz4s_zstd_init_tfm, + .exit = qat_comp_alg_zstd_exit_tfm, + .compress = qat_comp_alg_lz4s_zstd_compress, + .decompress = qat_comp_alg_sw_decompress, +}; + +static struct acomp_alg qat_acomp_zstd_native = { + .base = { + .cra_name = "zstd", + .cra_driver_name = "qat_zstd", + .cra_priority = 4001, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_reqsize = sizeof(struct qat_compression_req), + .cra_ctxsize = sizeof(struct qat_compression_ctx), + .cra_module = THIS_MODULE, + }, + .init = qat_comp_alg_zstd_init_tfm, + .exit = qat_comp_alg_zstd_exit_tfm, + .compress = qat_comp_alg_compress, + .decompress = qat_comp_alg_zstd_decompress, +}; + +static int qat_comp_algs_register_deflate(void) +{ + int ret = 0; + + mutex_lock(&algs_lock); + if (++active_devs_deflate == 1) { + ret = crypto_register_acomps(qat_acomp_deflate, + ARRAY_SIZE(qat_acomp_deflate)); + if (ret) + active_devs_deflate--; + } + mutex_unlock(&algs_lock); + + return ret; +} + +static void qat_comp_algs_unregister_deflate(void) +{ + mutex_lock(&algs_lock); + if (--active_devs_deflate == 0) + crypto_unregister_acomps(qat_acomp_deflate, ARRAY_SIZE(qat_acomp_deflate)); + mutex_unlock(&algs_lock); +} + +static int qat_comp_algs_register_lz4s(void) +{ + int ret = 0; + + mutex_lock(&algs_lock); + if (++active_devs_lz4s == 1) { + ret = crypto_acomp_alloc_streams(&qat_zstd_streams); + if (ret) { + active_devs_lz4s--; + goto unlock; + } + + ret = crypto_register_acomp(&qat_acomp_zstd_lz4s); + if (ret) { + crypto_acomp_free_streams(&qat_zstd_streams); + active_devs_lz4s--; + } + } +unlock: + mutex_unlock(&algs_lock); + + return ret; +} + +static void qat_comp_algs_unregister_lz4s(void) +{ + mutex_lock(&algs_lock); + if (--active_devs_lz4s == 0) { + crypto_unregister_acomp(&qat_acomp_zstd_lz4s); + crypto_acomp_free_streams(&qat_zstd_streams); + } + mutex_unlock(&algs_lock); +} + +static int qat_comp_algs_register_zstd(void) +{ + int ret = 0; + + mutex_lock(&algs_lock); + if (++active_devs_zstd == 1) { + ret = crypto_register_acomp(&qat_acomp_zstd_native); + if (ret) + active_devs_zstd--; + } + mutex_unlock(&algs_lock); + + return ret; +} + +static void qat_comp_algs_unregister_zstd(void) +{ + mutex_lock(&algs_lock); + if (--active_devs_zstd == 0) + crypto_unregister_acomp(&qat_acomp_zstd_native); + mutex_unlock(&algs_lock); +} + +int qat_comp_algs_register(u32 caps) +{ + int ret; + + ret = qat_comp_algs_register_deflate(); + if (ret) + return ret; + + if (caps & ADF_ACCEL_CAPABILITIES_EXT_ZSTD_LZ4S) { + ret = qat_comp_algs_register_lz4s(); + if (ret) + goto err_unregister_deflate; + } + + if (caps & ADF_ACCEL_CAPABILITIES_EXT_ZSTD) { + ret = qat_comp_algs_register_zstd(); + if (ret) + goto err_unregister_lz4s; + } + + return ret; + +err_unregister_lz4s: + if (caps & ADF_ACCEL_CAPABILITIES_EXT_ZSTD_LZ4S) + qat_comp_algs_unregister_lz4s(); +err_unregister_deflate: + qat_comp_algs_unregister_deflate(); + + return ret; +} + +void qat_comp_algs_unregister(u32 caps) +{ + qat_comp_algs_unregister_deflate(); + + if (caps & ADF_ACCEL_CAPABILITIES_EXT_ZSTD_LZ4S) + qat_comp_algs_unregister_lz4s(); + + if (caps & ADF_ACCEL_CAPABILITIES_EXT_ZSTD) + qat_comp_algs_unregister_zstd(); +} diff --git a/drivers/crypto/intel/qat/qat_common/qat_comp_req.h b/drivers/crypto/intel/qat/qat_common/qat_comp_req.h new file mode 100644 index 000000000..f165d28aa --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/qat_comp_req.h @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2022 Intel Corporation */ +#ifndef _QAT_COMP_REQ_H_ +#define _QAT_COMP_REQ_H_ + +#include "icp_qat_fw_comp.h" + +#define QAT_COMP_REQ_SIZE (sizeof(struct icp_qat_fw_comp_req)) +#define QAT_COMP_CTX_SIZE (QAT_COMP_REQ_SIZE * 2) + +static inline void qat_comp_create_req(void *ctx, void *req, u64 src, u32 slen, + u64 dst, u32 dlen, u64 opaque) +{ + struct icp_qat_fw_comp_req *fw_tmpl = ctx; + struct icp_qat_fw_comp_req *fw_req = req; + struct icp_qat_fw_comp_req_params *req_pars = &fw_req->comp_pars; + + memcpy(fw_req, fw_tmpl, sizeof(*fw_req)); + fw_req->comn_mid.src_data_addr = src; + fw_req->comn_mid.src_length = slen; + fw_req->comn_mid.dest_data_addr = dst; + fw_req->comn_mid.dst_length = dlen; + fw_req->comn_mid.opaque_data = opaque; + req_pars->comp_len = slen; + req_pars->out_buffer_sz = dlen; + fw_req->u3.asb_threshold.asb_value *= slen >> 4; +} + +static inline void qat_comp_create_compression_req(void *ctx, void *req, + u64 src, u32 slen, + u64 dst, u32 dlen, + u64 opaque) +{ + qat_comp_create_req(ctx, req, src, slen, dst, dlen, opaque); +} + +static inline void qat_comp_create_decompression_req(void *ctx, void *req, + u64 src, u32 slen, + u64 dst, u32 dlen, + u64 opaque) +{ + struct icp_qat_fw_comp_req *fw_tmpl = ctx; + + fw_tmpl++; + qat_comp_create_req(fw_tmpl, req, src, slen, dst, dlen, opaque); +} + +static inline u32 qat_comp_get_consumed_ctr(void *resp) +{ + struct icp_qat_fw_comp_resp *qat_resp = resp; + + return qat_resp->comp_resp_pars.input_byte_counter; +} + +static inline u32 qat_comp_get_produced_ctr(void *resp) +{ + struct icp_qat_fw_comp_resp *qat_resp = resp; + + return qat_resp->comp_resp_pars.output_byte_counter; +} + +static inline u32 qat_comp_get_produced_adler32(void *resp) +{ + struct icp_qat_fw_comp_resp *qat_resp = resp; + + return qat_resp->comp_resp_pars.crc.legacy.curr_adler_32; +} + +static inline u64 qat_comp_get_opaque(void *resp) +{ + struct icp_qat_fw_comp_resp *qat_resp = resp; + + return qat_resp->opaque_data; +} + +static inline s8 qat_comp_get_cmp_err(void *resp) +{ + struct icp_qat_fw_comp_resp *qat_resp = resp; + + return qat_resp->comn_resp.comn_error.cmp_err_code; +} + +static inline s8 qat_comp_get_xlt_err(void *resp) +{ + struct icp_qat_fw_comp_resp *qat_resp = resp; + + return qat_resp->comn_resp.comn_error.xlat_err_code; +} + +static inline s8 qat_comp_get_cmp_status(void *resp) +{ + struct icp_qat_fw_comp_resp *qat_resp = resp; + u8 stat_filed = qat_resp->comn_resp.comn_status; + + return ICP_QAT_FW_COMN_RESP_CMP_STAT_GET(stat_filed); +} + +static inline s8 qat_comp_get_xlt_status(void *resp) +{ + struct icp_qat_fw_comp_resp *qat_resp = resp; + u8 stat_filed = qat_resp->comn_resp.comn_status; + + return ICP_QAT_FW_COMN_RESP_XLAT_STAT_GET(stat_filed); +} + +static inline u8 qat_comp_get_cmp_cnv_flag(void *resp) +{ + struct icp_qat_fw_comp_resp *qat_resp = resp; + u8 flags = qat_resp->comn_resp.hdr_flags; + + return ICP_QAT_FW_COMN_HDR_CNV_FLAG_GET(flags); +} + +static inline u8 qat_comp_get_cmp_uncomp_flag(void *resp) +{ + struct icp_qat_fw_comp_resp *qat_resp = resp; + u8 flags = qat_resp->comn_resp.hdr_flags; + + return ICP_QAT_FW_COMN_HDR_ST_BLK_FLAG_GET(flags); +} + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/qat_comp_zstd_utils.c b/drivers/crypto/intel/qat/qat_common/qat_comp_zstd_utils.c new file mode 100644 index 000000000..62ec2d5c3 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/qat_comp_zstd_utils.c @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2026 Intel Corporation */ +#include +#include +#include +#include +#include + +#include "qat_comp_zstd_utils.h" + +#define ML_BITS 4 +#define ML_MASK ((1U << ML_BITS) - 1) +#define RUN_BITS (8 - ML_BITS) +#define RUN_MASK ((1U << RUN_BITS) - 1) +#define LZ4S_MINMATCH 2 + +/* + * ZSTD blocks can decompress to at most min(windowSize, 128KB) bytes. + * Insert explicit block delimiters to keep blocks within this limit. + */ +#define QAT_ZSTD_BLOCK_MAX ZSTD_BLOCKSIZE_MAX + +static int emit_delimiter(ZSTD_Sequence *out_seqs, size_t *seqs_idx, + size_t out_seqs_capacity, unsigned int lz4s_buff_size) +{ + if (*seqs_idx >= out_seqs_capacity - 1) { + pr_debug("QAT ZSTD: sequence overflow (seqs_idx:%zu, capacity:%zu, lz4s_size:%u)\n", + *seqs_idx, out_seqs_capacity, lz4s_buff_size); + return -EOVERFLOW; + } + + out_seqs[*seqs_idx].offset = 0; + out_seqs[*seqs_idx].litLength = 0; + out_seqs[*seqs_idx].matchLength = 0; + (*seqs_idx)++; + + return 0; +} + +int qat_alg_dec_lz4s(ZSTD_Sequence *out_seqs, size_t out_seqs_capacity, + unsigned char *lz4s_buff, unsigned int lz4s_buff_size, + unsigned char *literals, unsigned int *lit_len) +{ + unsigned char *end_ip = lz4s_buff + lz4s_buff_size; + unsigned char *start, *dest, *dest_end; + unsigned int hist_literal_len = 0; + unsigned char *ip = lz4s_buff; + size_t block_decomp_size = 0; + size_t seqs_idx = 0; + int ret; + + *lit_len = 0; + + if (!lz4s_buff_size) + return 0; + + while (ip < end_ip) { + size_t literal_len = 0, match_len = 0; + const unsigned int token = *ip++; + size_t length = 0; + size_t offset = 0; + + /* Get literal length */ + length = token >> ML_BITS; + if (length == RUN_MASK) { + unsigned int s; + + do { + s = *ip++; + length += s; + } while (s == 255); + } + + literal_len = length; + + start = ip; + dest = literals; + dest_end = literals + length; + + do { + memcpy(dest, start, QAT_ZSTD_LIT_COPY_LEN); + dest += QAT_ZSTD_LIT_COPY_LEN; + start += QAT_ZSTD_LIT_COPY_LEN; + } while (dest < dest_end); + + literals += length; + *lit_len += length; + + ip += length; + if (ip == end_ip) { + literal_len += hist_literal_len; + /* + * If adding trailing literals would overflow the + * current block, close it first. + */ + if (block_decomp_size + literal_len > QAT_ZSTD_BLOCK_MAX) { + ret = emit_delimiter(out_seqs, &seqs_idx, + out_seqs_capacity, + lz4s_buff_size); + if (ret) + return ret; + } + out_seqs[seqs_idx].litLength = literal_len; + out_seqs[seqs_idx].offset = offset; + out_seqs[seqs_idx].matchLength = match_len; + break; + } + + offset = get_unaligned_le16(ip); + ip += 2; + + length = token & ML_MASK; + if (length == ML_MASK) { + unsigned int s; + + do { + s = *ip++; + length += s; + } while (s == 255); + } + if (length != 0) { + length += LZ4S_MINMATCH; + match_len = (unsigned short)length; + literal_len += hist_literal_len; + + /* + * If this sequence would push the current block past + * the ZSTD maximum, close the block first. + */ + if (block_decomp_size + literal_len + match_len > QAT_ZSTD_BLOCK_MAX) { + ret = emit_delimiter(out_seqs, &seqs_idx, + out_seqs_capacity, + lz4s_buff_size); + if (ret) + return ret; + + block_decomp_size = 0; + } + + out_seqs[seqs_idx].offset = offset; + out_seqs[seqs_idx].litLength = literal_len; + out_seqs[seqs_idx].matchLength = match_len; + hist_literal_len = 0; + seqs_idx++; + if (seqs_idx >= out_seqs_capacity - 1) { + pr_debug("QAT ZSTD: sequence overflow (seqs_idx:%zu, capacity:%zu, lz4s_size:%u)\n", + seqs_idx, out_seqs_capacity, lz4s_buff_size); + return -EOVERFLOW; + } + + block_decomp_size += literal_len + match_len; + } else { + if (literal_len > 0) { + /* + * When match length is 0, the literal length needs + * to be temporarily stored and processed together + * with the next data block. + */ + hist_literal_len += literal_len; + } + } + } + + return seqs_idx + 1; +} diff --git a/drivers/crypto/intel/qat/qat_common/qat_comp_zstd_utils.h b/drivers/crypto/intel/qat/qat_common/qat_comp_zstd_utils.h new file mode 100644 index 000000000..55c7a1b9b --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/qat_comp_zstd_utils.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2026 Intel Corporation */ +#ifndef QAT_COMP_ZSTD_UTILS_H_ +#define QAT_COMP_ZSTD_UTILS_H_ +#include + +#define QAT_ZSTD_LIT_COPY_LEN 8 + +int qat_alg_dec_lz4s(ZSTD_Sequence *out_seqs, size_t out_seqs_capacity, + unsigned char *lz4s_buff, unsigned int lz4s_buff_size, + unsigned char *literals, unsigned int *lit_len); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/qat_compression.c b/drivers/crypto/intel/qat/qat_common/qat_compression.c new file mode 100644 index 000000000..8129ad0c3 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/qat_compression.c @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2022 Intel Corporation */ +#include +#include +#include +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_transport.h" +#include "adf_transport_access_macros.h" +#include "adf_cfg.h" +#include "adf_cfg_strings.h" +#include "qat_compression.h" +#include "icp_qat_fw.h" + +#define SEC ADF_KERNEL_SEC + +static struct service_hndl qat_compression; + +void qat_compression_put_instance(struct qat_compression_instance *inst) +{ + atomic_dec(&inst->refctr); + adf_dev_put(inst->accel_dev); +} + +static int qat_compression_free_instances(struct adf_accel_dev *accel_dev) +{ + struct qat_compression_instance *inst; + struct list_head *list_ptr, *tmp; + int i; + + list_for_each_safe(list_ptr, tmp, &accel_dev->compression_list) { + inst = list_entry(list_ptr, + struct qat_compression_instance, list); + + for (i = 0; i < atomic_read(&inst->refctr); i++) + qat_compression_put_instance(inst); + + if (inst->dc_tx) + adf_remove_ring(inst->dc_tx); + + if (inst->dc_rx) + adf_remove_ring(inst->dc_rx); + + list_del(list_ptr); + kfree(inst); + } + return 0; +} + +struct qat_compression_instance *qat_compression_get_instance_node(int node, int alg) +{ + struct qat_compression_instance *inst = NULL; + struct adf_hw_device_data *hw_data = NULL; + struct adf_accel_dev *accel_dev = NULL; + unsigned long best = ~0; + struct list_head *itr; + u32 caps, mask; + + list_for_each(itr, adf_devmgr_get_head()) { + struct adf_accel_dev *tmp_dev; + unsigned long ctr; + int tmp_dev_node; + + tmp_dev = list_entry(itr, struct adf_accel_dev, list); + tmp_dev_node = dev_to_node(&GET_DEV(tmp_dev)); + + if (alg == QAT_ZSTD || alg == QAT_LZ4S) { + hw_data = tmp_dev->hw_device; + caps = hw_data->accel_capabilities_ext_mask; + mask = ADF_ACCEL_CAPABILITIES_EXT_ZSTD | + ADF_ACCEL_CAPABILITIES_EXT_ZSTD_LZ4S; + if (!(caps & mask)) + continue; + } + + if ((node == tmp_dev_node || tmp_dev_node < 0) && + adf_dev_started(tmp_dev) && !list_empty(&tmp_dev->compression_list)) { + ctr = atomic_read(&tmp_dev->ref_count); + if (best > ctr) { + accel_dev = tmp_dev; + best = ctr; + } + } + } + + if (!accel_dev) { + pr_debug_ratelimited("QAT: Could not find a device on node %d\n", node); + /* Get any started device */ + list_for_each(itr, adf_devmgr_get_head()) { + struct adf_accel_dev *tmp_dev; + + tmp_dev = list_entry(itr, struct adf_accel_dev, list); + + if (alg == QAT_ZSTD || alg == QAT_LZ4S) { + hw_data = tmp_dev->hw_device; + caps = hw_data->accel_capabilities_ext_mask; + mask = ADF_ACCEL_CAPABILITIES_EXT_ZSTD | + ADF_ACCEL_CAPABILITIES_EXT_ZSTD_LZ4S; + if (!(caps & mask)) + continue; + } + + if (adf_dev_started(tmp_dev) && + !list_empty(&tmp_dev->compression_list)) { + accel_dev = tmp_dev; + break; + } + } + } + + if (!accel_dev) + return NULL; + + best = ~0; + list_for_each(itr, &accel_dev->compression_list) { + struct qat_compression_instance *tmp_inst; + unsigned long ctr; + + tmp_inst = list_entry(itr, struct qat_compression_instance, list); + ctr = atomic_read(&tmp_inst->refctr); + if (best > ctr) { + inst = tmp_inst; + best = ctr; + } + } + if (inst) { + if (adf_dev_get(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Could not increment dev refctr\n"); + return NULL; + } + atomic_inc(&inst->refctr); + } + return inst; +} + +static int qat_compression_create_instances(struct adf_accel_dev *accel_dev) +{ + struct qat_compression_instance *inst; + char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + unsigned long num_inst, num_msg_dc; + unsigned long bank; + int msg_size; + int ret; + int i; + + INIT_LIST_HEAD(&accel_dev->compression_list); + strscpy(key, ADF_NUM_DC); + ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); + if (ret) + return ret; + + ret = kstrtoul(val, 10, &num_inst); + if (ret) + return ret; + + for (i = 0; i < num_inst; i++) { + inst = kzalloc_node(sizeof(*inst), GFP_KERNEL, + dev_to_node(&GET_DEV(accel_dev))); + if (!inst) { + ret = -ENOMEM; + goto err; + } + + list_add_tail(&inst->list, &accel_dev->compression_list); + inst->id = i; + atomic_set(&inst->refctr, 0); + inst->accel_dev = accel_dev; + + snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_BANK_NUM, i); + ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); + if (ret) + return ret; + + ret = kstrtoul(val, 10, &bank); + if (ret) + return ret; + + snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_SIZE, i); + ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); + if (ret) + return ret; + + ret = kstrtoul(val, 10, &num_msg_dc); + if (ret) + return ret; + + msg_size = ICP_QAT_FW_REQ_DEFAULT_SZ; + snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_TX, i); + ret = adf_create_ring(accel_dev, SEC, bank, num_msg_dc, + msg_size, key, NULL, 0, &inst->dc_tx); + if (ret) + return ret; + + msg_size = ICP_QAT_FW_RESP_DEFAULT_SZ; + snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_RX, i); + ret = adf_create_ring(accel_dev, SEC, bank, num_msg_dc, + msg_size, key, qat_comp_alg_callback, 0, + &inst->dc_rx); + if (ret) + return ret; + + inst->dc_data = accel_dev->dc_data; + INIT_LIST_HEAD(&inst->backlog.list); + spin_lock_init(&inst->backlog.lock); + } + return 0; +err: + qat_compression_free_instances(accel_dev); + return ret; +} + +static int qat_compression_alloc_dc_data(struct adf_accel_dev *accel_dev) +{ + struct device *dev = &GET_DEV(accel_dev); + dma_addr_t obuff_p = DMA_MAPPING_ERROR; + size_t ovf_buff_sz = QAT_COMP_MAX_SKID; + struct adf_dc_data *dc_data = NULL; + u8 *obuff = NULL; + + dc_data = kzalloc_node(sizeof(*dc_data), GFP_KERNEL, dev_to_node(dev)); + if (!dc_data) + goto err; + + obuff = kzalloc_node(ovf_buff_sz, GFP_KERNEL, dev_to_node(dev)); + if (!obuff) + goto err; + + obuff_p = dma_map_single(dev, obuff, ovf_buff_sz, DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(dev, obuff_p))) + goto err; + + dc_data->ovf_buff = obuff; + dc_data->ovf_buff_p = obuff_p; + dc_data->ovf_buff_sz = ovf_buff_sz; + + accel_dev->dc_data = dc_data; + + return 0; + +err: + accel_dev->dc_data = NULL; + kfree(obuff); + devm_kfree(dev, dc_data); + return -ENOMEM; +} + +static void qat_free_dc_data(struct adf_accel_dev *accel_dev) +{ + struct adf_dc_data *dc_data = accel_dev->dc_data; + struct device *dev = &GET_DEV(accel_dev); + + if (!dc_data) + return; + + dma_unmap_single(dev, dc_data->ovf_buff_p, dc_data->ovf_buff_sz, + DMA_BIDIRECTIONAL); + kfree_sensitive(dc_data->ovf_buff); + kfree(dc_data); + accel_dev->dc_data = NULL; +} + +static int qat_compression_init(struct adf_accel_dev *accel_dev) +{ + int ret; + + ret = qat_compression_alloc_dc_data(accel_dev); + if (ret) + return ret; + + ret = qat_compression_create_instances(accel_dev); + if (ret) + qat_free_dc_data(accel_dev); + + return ret; +} + +static int qat_compression_shutdown(struct adf_accel_dev *accel_dev) +{ + qat_free_dc_data(accel_dev); + return qat_compression_free_instances(accel_dev); +} + +static int qat_compression_event_handler(struct adf_accel_dev *accel_dev, + enum adf_event event) +{ + int ret; + + switch (event) { + case ADF_EVENT_INIT: + ret = qat_compression_init(accel_dev); + break; + case ADF_EVENT_SHUTDOWN: + ret = qat_compression_shutdown(accel_dev); + break; + case ADF_EVENT_RESTARTING: + case ADF_EVENT_RESTARTED: + case ADF_EVENT_START: + case ADF_EVENT_STOP: + default: + ret = 0; + } + return ret; +} + +int qat_compression_register(void) +{ + memset(&qat_compression, 0, sizeof(qat_compression)); + qat_compression.event_hld = qat_compression_event_handler; + qat_compression.name = "qat_compression"; + return adf_service_register(&qat_compression); +} + +int qat_compression_unregister(void) +{ + return adf_service_unregister(&qat_compression); +} diff --git a/drivers/crypto/intel/qat/qat_common/qat_compression.h b/drivers/crypto/intel/qat/qat_common/qat_compression.h new file mode 100644 index 000000000..5ced3ed0e --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/qat_compression.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2022 Intel Corporation */ +#ifndef _QAT_COMPRESSION_H_ +#define _QAT_COMPRESSION_H_ + +#include +#include +#include "adf_accel_devices.h" +#include "qat_algs_send.h" + +#define QAT_COMP_MAX_SKID 4096 + +struct qat_compression_instance { + struct adf_etr_ring_data *dc_tx; + struct adf_etr_ring_data *dc_rx; + struct adf_accel_dev *accel_dev; + struct list_head list; + unsigned long state; + int id; + atomic_t refctr; + struct qat_instance_backlog backlog; + struct adf_dc_data *dc_data; +}; + +static inline bool adf_hw_dev_has_compression(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + u32 mask = ~hw_device->accel_capabilities_mask; + + if (mask & ADF_ACCEL_CAPABILITIES_COMPRESSION) + return false; + + return true; +} + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/qat_crypto.c b/drivers/crypto/intel/qat/qat_common/qat_crypto.c new file mode 100644 index 000000000..101c6ea41 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/qat_crypto.c @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#include +#include +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_transport.h" +#include "adf_cfg.h" +#include "adf_cfg_strings.h" +#include "adf_gen2_hw_data.h" +#include "qat_crypto.h" +#include "icp_qat_fw.h" + +#define SEC ADF_KERNEL_SEC + +static struct service_hndl qat_crypto; + +void qat_crypto_put_instance(struct qat_crypto_instance *inst) +{ + atomic_dec(&inst->refctr); + adf_dev_put(inst->accel_dev); +} + +static int qat_crypto_free_instances(struct adf_accel_dev *accel_dev) +{ + struct qat_crypto_instance *inst, *tmp; + int i; + + list_for_each_entry_safe(inst, tmp, &accel_dev->crypto_list, list) { + for (i = 0; i < atomic_read(&inst->refctr); i++) + qat_crypto_put_instance(inst); + + if (inst->sym_tx) + adf_remove_ring(inst->sym_tx); + + if (inst->sym_rx) + adf_remove_ring(inst->sym_rx); + + if (inst->pke_tx) + adf_remove_ring(inst->pke_tx); + + if (inst->pke_rx) + adf_remove_ring(inst->pke_rx); + + list_del(&inst->list); + kfree(inst); + } + return 0; +} + +struct qat_crypto_instance *qat_crypto_get_instance_node(int node) +{ + struct adf_accel_dev *accel_dev = NULL, *tmp_dev; + struct qat_crypto_instance *inst = NULL, *tmp_inst; + unsigned long best = ~0; + + list_for_each_entry(tmp_dev, adf_devmgr_get_head(), list) { + unsigned long ctr; + + if ((node == dev_to_node(&GET_DEV(tmp_dev)) || + dev_to_node(&GET_DEV(tmp_dev)) < 0) && + adf_dev_started(tmp_dev) && + !list_empty(&tmp_dev->crypto_list)) { + ctr = atomic_read(&tmp_dev->ref_count); + if (best > ctr) { + accel_dev = tmp_dev; + best = ctr; + } + } + } + + if (!accel_dev) { + pr_debug_ratelimited("QAT: Could not find a device on node %d\n", node); + /* Get any started device */ + list_for_each_entry(tmp_dev, adf_devmgr_get_head(), list) { + if (adf_dev_started(tmp_dev) && + !list_empty(&tmp_dev->crypto_list)) { + accel_dev = tmp_dev; + break; + } + } + } + + if (!accel_dev) + return NULL; + + best = ~0; + list_for_each_entry(tmp_inst, &accel_dev->crypto_list, list) { + unsigned long ctr; + + ctr = atomic_read(&tmp_inst->refctr); + if (best > ctr) { + inst = tmp_inst; + best = ctr; + } + } + if (inst) { + if (adf_dev_get(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Could not increment dev refctr\n"); + return NULL; + } + atomic_inc(&inst->refctr); + } + return inst; +} + +/** + * qat_crypto_vf_dev_config() - create dev config required to create + * crypto inst. + * + * @accel_dev: Pointer to acceleration device. + * + * Function creates device configuration required to create + * asym, sym or, crypto instances + * + * Return: 0 on success, error code otherwise. + */ +int qat_crypto_vf_dev_config(struct adf_accel_dev *accel_dev) +{ + u16 ring_to_svc_map = GET_HW_DATA(accel_dev)->ring_to_svc_map; + + if (ring_to_svc_map != ADF_GEN2_DEFAULT_RING_TO_SRV_MAP) { + dev_err(&GET_DEV(accel_dev), + "Unsupported ring/service mapping present on PF"); + return -EFAULT; + } + + return GET_HW_DATA(accel_dev)->dev_config(accel_dev); +} + +static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev) +{ + unsigned long num_inst, num_msg_sym, num_msg_asym; + char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + unsigned long sym_bank, asym_bank; + struct qat_crypto_instance *inst; + int msg_size; + int ret; + int i; + + INIT_LIST_HEAD(&accel_dev->crypto_list); + ret = adf_cfg_get_param_value(accel_dev, SEC, ADF_NUM_CY, val); + if (ret) + return ret; + + ret = kstrtoul(val, 0, &num_inst); + if (ret) + return ret; + + for (i = 0; i < num_inst; i++) { + inst = kzalloc_node(sizeof(*inst), GFP_KERNEL, + dev_to_node(&GET_DEV(accel_dev))); + if (!inst) { + ret = -ENOMEM; + goto err; + } + + list_add_tail(&inst->list, &accel_dev->crypto_list); + inst->id = i; + atomic_set(&inst->refctr, 0); + inst->accel_dev = accel_dev; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_BANK_NUM, i); + ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); + if (ret) + goto err; + + ret = kstrtoul(val, 10, &sym_bank); + if (ret) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i); + ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); + if (ret) + goto err; + + ret = kstrtoul(val, 10, &asym_bank); + if (ret) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); + ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); + if (ret) + goto err; + + ret = kstrtoul(val, 10, &num_msg_sym); + if (ret) + goto err; + + num_msg_sym = num_msg_sym >> 1; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); + ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); + if (ret) + goto err; + + ret = kstrtoul(val, 10, &num_msg_asym); + if (ret) + goto err; + num_msg_asym = num_msg_asym >> 1; + + msg_size = ICP_QAT_FW_REQ_DEFAULT_SZ; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); + ret = adf_create_ring(accel_dev, SEC, sym_bank, num_msg_sym, + msg_size, key, NULL, 0, &inst->sym_tx); + if (ret) + goto err; + + msg_size = msg_size >> 1; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); + ret = adf_create_ring(accel_dev, SEC, asym_bank, num_msg_asym, + msg_size, key, NULL, 0, &inst->pke_tx); + if (ret) + goto err; + + msg_size = ICP_QAT_FW_RESP_DEFAULT_SZ; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); + ret = adf_create_ring(accel_dev, SEC, sym_bank, num_msg_sym, + msg_size, key, qat_alg_callback, 0, + &inst->sym_rx); + if (ret) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); + ret = adf_create_ring(accel_dev, SEC, asym_bank, num_msg_asym, + msg_size, key, qat_alg_asym_callback, 0, + &inst->pke_rx); + if (ret) + goto err; + + INIT_LIST_HEAD(&inst->backlog.list); + spin_lock_init(&inst->backlog.lock); + } + return 0; +err: + qat_crypto_free_instances(accel_dev); + return ret; +} + +static int qat_crypto_init(struct adf_accel_dev *accel_dev) +{ + if (qat_crypto_create_instances(accel_dev)) + return -EFAULT; + + return 0; +} + +static int qat_crypto_shutdown(struct adf_accel_dev *accel_dev) +{ + return qat_crypto_free_instances(accel_dev); +} + +static int qat_crypto_event_handler(struct adf_accel_dev *accel_dev, + enum adf_event event) +{ + int ret; + + switch (event) { + case ADF_EVENT_INIT: + ret = qat_crypto_init(accel_dev); + break; + case ADF_EVENT_SHUTDOWN: + ret = qat_crypto_shutdown(accel_dev); + break; + case ADF_EVENT_RESTARTING: + case ADF_EVENT_RESTARTED: + case ADF_EVENT_START: + case ADF_EVENT_STOP: + default: + ret = 0; + } + return ret; +} + +int qat_crypto_register(void) +{ + memset(&qat_crypto, 0, sizeof(qat_crypto)); + qat_crypto.event_hld = qat_crypto_event_handler; + qat_crypto.name = "qat_crypto"; + return adf_service_register(&qat_crypto); +} + +int qat_crypto_unregister(void) +{ + return adf_service_unregister(&qat_crypto); +} diff --git a/drivers/crypto/intel/qat/qat_common/qat_crypto.h b/drivers/crypto/intel/qat/qat_common/qat_crypto.h new file mode 100644 index 000000000..6a0e961bb --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/qat_crypto.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#ifndef _QAT_CRYPTO_INSTANCE_H_ +#define _QAT_CRYPTO_INSTANCE_H_ + +#include +#include +#include +#include "adf_accel_devices.h" +#include "icp_qat_fw_la.h" +#include "qat_algs_send.h" +#include "qat_bl.h" + +struct qat_crypto_instance { + struct adf_etr_ring_data *sym_tx; + struct adf_etr_ring_data *sym_rx; + struct adf_etr_ring_data *pke_tx; + struct adf_etr_ring_data *pke_rx; + struct adf_accel_dev *accel_dev; + struct list_head list; + unsigned long state; + int id; + atomic_t refctr; + struct qat_instance_backlog backlog; +}; + +struct qat_crypto_request; + +struct qat_crypto_request { + struct icp_qat_fw_la_bulk_req req; + union { + struct qat_alg_aead_ctx *aead_ctx; + struct qat_alg_skcipher_ctx *skcipher_ctx; + }; + union { + struct aead_request *aead_req; + struct skcipher_request *skcipher_req; + }; + struct qat_request_buffs buf; + void (*cb)(struct icp_qat_fw_la_resp *resp, + struct qat_crypto_request *req); + union { + struct { + __be64 iv_hi; + __be64 iv_lo; + }; + u8 iv[AES_BLOCK_SIZE]; + }; + bool encryption; + struct qat_alg_req alg_req; +}; + +static inline bool adf_hw_dev_has_crypto(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + u32 mask = ~hw_device->accel_capabilities_mask; + + if (mask & ADF_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC) + return false; + if (mask & ADF_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC) + return false; + if (mask & ADF_ACCEL_CAPABILITIES_AUTHENTICATION) + return false; + + return true; +} + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/qat_hal.c b/drivers/crypto/intel/qat/qat_common/qat_hal.c new file mode 100644 index 000000000..1c3d1311f --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/qat_hal.c @@ -0,0 +1,1604 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#include +#include +#include + +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "icp_qat_hal.h" +#include "icp_qat_uclo.h" + +#define BAD_REGADDR 0xffff +#define MAX_RETRY_TIMES 10000 +#define INIT_CTX_ARB_VALUE 0x0 +#define INIT_CTX_ENABLE_VALUE 0x0 +#define INIT_PC_VALUE 0x0 +#define INIT_WAKEUP_EVENTS_VALUE 0x1 +#define INIT_SIG_EVENTS_VALUE 0x1 +#define INIT_CCENABLE_VALUE 0x2000 +#define RST_CSR_QAT_LSB 20 +#define RST_CSR_AE_LSB 0 +#define MC_TIMESTAMP_ENABLE (0x1 << 7) +#define MIN_RESET_DELAY_US 3 + +#define IGNORE_W1C_MASK ((~(1 << CE_BREAKPOINT_BITPOS)) & \ + (~(1 << CE_CNTL_STORE_PARITY_ERROR_BITPOS)) & \ + (~(1 << CE_REG_PAR_ERR_BITPOS))) +#define INSERT_IMMED_GPRA_CONST(inst, const_val) \ + (inst = ((inst & 0xFFFF00C03FFull) | \ + ((((const_val) << 12) & 0x0FF00000ull) | \ + (((const_val) << 10) & 0x0003FC00ull)))) +#define INSERT_IMMED_GPRB_CONST(inst, const_val) \ + (inst = ((inst & 0xFFFF00FFF00ull) | \ + ((((const_val) << 12) & 0x0FF00000ull) | \ + (((const_val) << 0) & 0x000000FFull)))) + +#define AE(handle, ae) ((handle)->hal_handle->aes[ae]) + +static const u64 inst_4b[] = { + 0x0F0400C0000ull, 0x0F4400C0000ull, 0x0F040000300ull, 0x0F440000300ull, + 0x0FC066C0000ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, + 0x0A021000000ull +}; + +static const u64 inst[] = { + 0x0F0000C0000ull, 0x0F000000380ull, 0x0D805000011ull, 0x0FC082C0300ull, + 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, + 0x0A0643C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0F0000C0001ull, + 0x0FC066C0001ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, + 0x0F000400300ull, 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D804400101ull, + 0x0A0580C0000ull, 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull, + 0x0A0584C0000ull, 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull, + 0x0A0588C0000ull, 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull, + 0x0A058CC0000ull, 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull, + 0x0A05C0C0000ull, 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull, + 0x0A05C4C0000ull, 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull, + 0x0A05C8C0000ull, 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull, + 0x0A05CCC0000ull, 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull, + 0x0A0400C0000ull, 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull, + 0x0A0402C0000ull, 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull, + 0x0A0404C0000ull, 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull, + 0x0A0406C0000ull, 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull, + 0x0A0408C0000ull, 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull, + 0x0A040AC0000ull, 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull, + 0x0A040CC0000ull, 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull, + 0x0A040EC0000ull, 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull, + 0x0D81581C010ull, 0x0E000010000ull, 0x0E000010000ull, +}; + +void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int ctx_mask) +{ + AE(handle, ae).live_ctx_mask = ctx_mask; +} + +#define CSR_RETRY_TIMES 500 +static int qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int csr) +{ + unsigned int iterations = CSR_RETRY_TIMES; + int value; + + do { + value = GET_AE_CSR(handle, ae, csr); + if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS)) + return value; + } while (iterations--); + + pr_err("QAT: Read CSR timeout\n"); + return 0; +} + +static int qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int csr, + unsigned int value) +{ + unsigned int iterations = CSR_RETRY_TIMES; + + do { + SET_AE_CSR(handle, ae, csr, value); + if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS)) + return 0; + } while (iterations--); + + pr_err("QAT: Write CSR Timeout\n"); + return -EFAULT; +} + +static void qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx, + unsigned int *events) +{ + unsigned int cur_ctx; + + cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); + qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); + *events = qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT); + qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); +} + +static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int cycles, + int chk_inactive) +{ + unsigned int base_cnt = 0, cur_cnt = 0; + unsigned int csr = (1 << ACS_ABO_BITPOS); + int times = MAX_RETRY_TIMES; + int elapsed_cycles = 0; + + base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT); + base_cnt &= 0xffff; + while ((int)cycles > elapsed_cycles && times--) { + if (chk_inactive) + csr = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); + + cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT); + cur_cnt &= 0xffff; + elapsed_cycles = cur_cnt - base_cnt; + + if (elapsed_cycles < 0) + elapsed_cycles += 0x10000; + + /* ensure at least 8 time cycles elapsed in wait_cycles */ + if (elapsed_cycles >= 8 && !(csr & (1 << ACS_ABO_BITPOS))) + return 0; + } + if (times < 0) { + pr_err("QAT: wait_num_cycles time out\n"); + return -EFAULT; + } + return 0; +} + +#define CLR_BIT(wrd, bit) ((wrd) & ~(1 << (bit))) +#define SET_BIT(wrd, bit) ((wrd) | 1 << (bit)) + +int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char mode) +{ + unsigned int csr, new_csr; + + if (mode != 4 && mode != 8) { + pr_err("QAT: bad ctx mode=%d\n", mode); + return -EINVAL; + } + + /* Sets the acceleration engine context mode to either four or eight */ + csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); + csr = IGNORE_W1C_MASK & csr; + new_csr = (mode == 4) ? + SET_BIT(csr, CE_INUSE_CONTEXTS_BITPOS) : + CLR_BIT(csr, CE_INUSE_CONTEXTS_BITPOS); + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); + return 0; +} + +int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char mode) +{ + unsigned int csr, new_csr; + + csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); + csr &= IGNORE_W1C_MASK; + + new_csr = (mode) ? + SET_BIT(csr, CE_NN_MODE_BITPOS) : + CLR_BIT(csr, CE_NN_MODE_BITPOS); + + if (new_csr != csr) + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); + + return 0; +} + +int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, enum icp_qat_uof_regtype lm_type, + unsigned char mode) +{ + unsigned int csr, new_csr; + + csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); + csr &= IGNORE_W1C_MASK; + switch (lm_type) { + case ICP_LMEM0: + new_csr = (mode) ? + SET_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS) : + CLR_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS); + break; + case ICP_LMEM1: + new_csr = (mode) ? + SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) : + CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS); + break; + case ICP_LMEM2: + new_csr = (mode) ? + SET_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS) : + CLR_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS); + break; + case ICP_LMEM3: + new_csr = (mode) ? + SET_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS) : + CLR_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS); + break; + default: + pr_err("QAT: lmType = 0x%x\n", lm_type); + return -EINVAL; + } + + if (new_csr != csr) + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); + return 0; +} + +void qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char mode) +{ + unsigned int csr, new_csr; + + csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); + csr &= IGNORE_W1C_MASK; + new_csr = (mode) ? + SET_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS) : + CLR_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS); + if (new_csr != csr) + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); +} + +static unsigned short qat_hal_get_reg_addr(unsigned int type, + unsigned short reg_num) +{ + unsigned short reg_addr; + + switch (type) { + case ICP_GPA_ABS: + case ICP_GPB_ABS: + reg_addr = 0x80 | (reg_num & 0x7f); + break; + case ICP_GPA_REL: + case ICP_GPB_REL: + reg_addr = reg_num & 0x1f; + break; + case ICP_SR_RD_REL: + case ICP_SR_WR_REL: + case ICP_SR_REL: + reg_addr = 0x180 | (reg_num & 0x1f); + break; + case ICP_SR_ABS: + reg_addr = 0x140 | ((reg_num & 0x3) << 1); + break; + case ICP_DR_RD_REL: + case ICP_DR_WR_REL: + case ICP_DR_REL: + reg_addr = 0x1c0 | (reg_num & 0x1f); + break; + case ICP_DR_ABS: + reg_addr = 0x100 | ((reg_num & 0x3) << 1); + break; + case ICP_NEIGH_REL: + reg_addr = 0x280 | (reg_num & 0x1f); + break; + case ICP_LMEM0: + reg_addr = 0x200; + break; + case ICP_LMEM1: + reg_addr = 0x220; + break; + case ICP_LMEM2: + reg_addr = 0x2c0; + break; + case ICP_LMEM3: + reg_addr = 0x2e0; + break; + case ICP_NO_DEST: + reg_addr = 0x300 | (reg_num & 0xff); + break; + default: + reg_addr = BAD_REGADDR; + break; + } + return reg_addr; +} + +void qat_hal_reset(struct icp_qat_fw_loader_handle *handle) +{ + unsigned int reset_mask = handle->chip_info->icp_rst_mask; + unsigned int reset_csr = handle->chip_info->icp_rst_csr; + unsigned int csr_val; + + csr_val = GET_CAP_CSR(handle, reset_csr); + csr_val |= reset_mask; + SET_CAP_CSR(handle, reset_csr, csr_val); +} + +static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int ctx_mask, + unsigned int ae_csr, unsigned int csr_val) +{ + unsigned int ctx, cur_ctx; + + cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); + + for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { + if (!(ctx_mask & (1 << ctx))) + continue; + qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); + qat_hal_wr_ae_csr(handle, ae, ae_csr, csr_val); + } + + qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); +} + +static unsigned int qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx, + unsigned int ae_csr) +{ + unsigned int cur_ctx, csr_val; + + cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); + qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); + csr_val = qat_hal_rd_ae_csr(handle, ae, ae_csr); + qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); + + return csr_val; +} + +static void qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int ctx_mask, + unsigned int events) +{ + unsigned int ctx, cur_ctx; + + cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); + for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { + if (!(ctx_mask & (1 << ctx))) + continue; + qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); + qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_INDIRECT, events); + } + qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); +} + +static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int ctx_mask, + unsigned int events) +{ + unsigned int ctx, cur_ctx; + + cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER); + for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { + if (!(ctx_mask & (1 << ctx))) + continue; + qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); + qat_hal_wr_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT, + events); + } + qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); +} + +static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle) +{ + unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned int base_cnt, cur_cnt; + unsigned char ae; + int times = MAX_RETRY_TIMES; + + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT); + base_cnt &= 0xffff; + + do { + cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT); + cur_cnt &= 0xffff; + } while (times-- && (cur_cnt == base_cnt)); + + if (times < 0) { + pr_err("QAT: AE%d is inactive!!\n", ae); + return -EFAULT; + } + } + + return 0; +} + +int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle, + unsigned int ae) +{ + unsigned int enable = 0, active = 0; + + enable = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); + active = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); + if ((enable & (0xff << CE_ENABLE_BITPOS)) || + (active & (1 << ACS_ABO_BITPOS))) + return 1; + else + return 0; +} + +static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) +{ + unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned int misc_ctl_csr, misc_ctl; + unsigned char ae; + + misc_ctl_csr = handle->chip_info->misc_ctl_csr; + /* stop the timestamp timers */ + misc_ctl = GET_CAP_CSR(handle, misc_ctl_csr); + if (misc_ctl & MC_TIMESTAMP_ENABLE) + SET_CAP_CSR(handle, misc_ctl_csr, misc_ctl & + (~MC_TIMESTAMP_ENABLE)); + + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0); + qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0); + } + /* start timestamp timers */ + SET_CAP_CSR(handle, misc_ctl_csr, misc_ctl | MC_TIMESTAMP_ENABLE); +} + +#define ESRAM_AUTO_TINIT BIT(2) +#define ESRAM_AUTO_TINIT_DONE BIT(3) +#define ESRAM_AUTO_INIT_USED_CYCLES (1640) +#define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C +static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle) +{ + void __iomem *csr_addr = + (void __iomem *)((uintptr_t)handle->hal_ep_csr_addr_v + + ESRAM_AUTO_INIT_CSR_OFFSET); + unsigned int csr_val; + int times = 30; + + if (handle->pci_dev->device != PCI_DEVICE_ID_INTEL_QAT_DH895XCC) + return 0; + + csr_val = ADF_CSR_RD(csr_addr, 0); + if ((csr_val & ESRAM_AUTO_TINIT) && (csr_val & ESRAM_AUTO_TINIT_DONE)) + return 0; + + csr_val = ADF_CSR_RD(csr_addr, 0); + csr_val |= ESRAM_AUTO_TINIT; + ADF_CSR_WR(csr_addr, 0, csr_val); + + do { + qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0); + csr_val = ADF_CSR_RD(csr_addr, 0); + } while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--); + if (times < 0) { + pr_err("QAT: Fail to init eSram!\n"); + return -EFAULT; + } + return 0; +} + +#define SHRAM_INIT_CYCLES 2060 +int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle) +{ + unsigned int clk_csr = handle->chip_info->glb_clk_enable_csr; + unsigned int reset_mask = handle->chip_info->icp_rst_mask; + unsigned int reset_csr = handle->chip_info->icp_rst_csr; + unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned char ae = 0; + unsigned int times = 100; + unsigned int csr_val; + + /* write to the reset csr */ + csr_val = GET_CAP_CSR(handle, reset_csr); + csr_val &= ~reset_mask; + do { + SET_CAP_CSR(handle, reset_csr, csr_val); + if (!(times--)) + goto out_err; + csr_val = GET_CAP_CSR(handle, reset_csr); + csr_val &= reset_mask; + } while (csr_val); + /* enable clock */ + csr_val = GET_CAP_CSR(handle, clk_csr); + csr_val |= reset_mask; + SET_CAP_CSR(handle, clk_csr, csr_val); + if (qat_hal_check_ae_alive(handle)) + goto out_err; + + /* Set undefined power-up/reset states to reasonable default values */ + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, + INIT_CTX_ENABLE_VALUE); + qat_hal_wr_indr_csr(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX, + CTX_STS_INDIRECT, + handle->hal_handle->upc_mask & + INIT_PC_VALUE); + qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE); + qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE); + qat_hal_put_wakeup_event(handle, ae, + ICP_QAT_UCLO_AE_ALL_CTX, + INIT_WAKEUP_EVENTS_VALUE); + qat_hal_put_sig_event(handle, ae, + ICP_QAT_UCLO_AE_ALL_CTX, + INIT_SIG_EVENTS_VALUE); + } + if (qat_hal_init_esram(handle)) + goto out_err; + if (qat_hal_wait_cycles(handle, 0, SHRAM_INIT_CYCLES, 0)) + goto out_err; + qat_hal_reset_timestamp(handle); + + return 0; +out_err: + pr_err("QAT: failed to get device out of reset\n"); + return -EFAULT; +} + +static void qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int ctx_mask) +{ + unsigned int ctx; + + ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); + ctx &= IGNORE_W1C_MASK & + (~((ctx_mask & ICP_QAT_UCLO_AE_ALL_CTX) << CE_ENABLE_BITPOS)); + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx); +} + +static u64 qat_hal_parity_64bit(u64 word) +{ + word ^= word >> 1; + word ^= word >> 2; + word ^= word >> 4; + word ^= word >> 8; + word ^= word >> 16; + word ^= word >> 32; + return word & 1; +} + +static u64 qat_hal_set_uword_ecc(u64 uword) +{ + u64 bit0_mask = 0xff800007fffULL, bit1_mask = 0x1f801ff801fULL, + bit2_mask = 0xe387e0781e1ULL, bit3_mask = 0x7cb8e388e22ULL, + bit4_mask = 0xaf5b2c93244ULL, bit5_mask = 0xf56d5525488ULL, + bit6_mask = 0xdaf69a46910ULL; + + /* clear the ecc bits */ + uword &= ~(0x7fULL << 0x2C); + uword |= qat_hal_parity_64bit(bit0_mask & uword) << 0x2C; + uword |= qat_hal_parity_64bit(bit1_mask & uword) << 0x2D; + uword |= qat_hal_parity_64bit(bit2_mask & uword) << 0x2E; + uword |= qat_hal_parity_64bit(bit3_mask & uword) << 0x2F; + uword |= qat_hal_parity_64bit(bit4_mask & uword) << 0x30; + uword |= qat_hal_parity_64bit(bit5_mask & uword) << 0x31; + uword |= qat_hal_parity_64bit(bit6_mask & uword) << 0x32; + return uword; +} + +void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int uaddr, + unsigned int words_num, u64 *uword) +{ + unsigned int ustore_addr; + unsigned int i; + + ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS); + uaddr |= UA_ECS; + qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); + for (i = 0; i < words_num; i++) { + unsigned int uwrd_lo, uwrd_hi; + u64 tmp; + + tmp = qat_hal_set_uword_ecc(uword[i]); + uwrd_lo = (unsigned int)(tmp & 0xffffffff); + uwrd_hi = (unsigned int)(tmp >> 0x20); + qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo); + qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi); + } + qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); +} + +static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int ctx_mask) +{ + unsigned int ctx; + + ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); + ctx &= IGNORE_W1C_MASK; + ctx_mask &= (ctx & CE_INUSE_CONTEXTS) ? 0x55 : 0xFF; + ctx |= (ctx_mask << CE_ENABLE_BITPOS); + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx); +} + +static void qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle) +{ + unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned char ae; + unsigned short reg; + + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) { + qat_hal_init_rd_xfer(handle, ae, 0, ICP_SR_RD_ABS, + reg, 0); + qat_hal_init_rd_xfer(handle, ae, 0, ICP_DR_RD_ABS, + reg, 0); + } + } +} + +static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) +{ + unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned char ae; + unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX; + int times = MAX_RETRY_TIMES; + unsigned int csr_val = 0; + unsigned int savctx = 0; + int ret = 0; + + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL); + csr_val &= ~(1 << MMC_SHARE_CS_BITPOS); + qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val); + csr_val = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); + csr_val &= IGNORE_W1C_MASK; + if (handle->chip_info->nn) + csr_val |= CE_NN_MODE; + + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val); + qat_hal_wr_uwords(handle, ae, 0, ARRAY_SIZE(inst), + (u64 *)inst); + qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT, + handle->hal_handle->upc_mask & + INIT_PC_VALUE); + savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); + qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0); + qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY); + qat_hal_wr_indr_csr(handle, ae, ctx_mask, + CTX_SIG_EVENTS_INDIRECT, 0); + qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0); + qat_hal_enable_ctx(handle, ae, ctx_mask); + } + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + /* wait for AE to finish */ + do { + ret = qat_hal_wait_cycles(handle, ae, 20, 1); + } while (ret && times--); + + if (times < 0) { + pr_err("QAT: clear GPR of AE %d failed", ae); + return -EINVAL; + } + qat_hal_disable_ctx(handle, ae, ctx_mask); + qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, + savctx & ACS_ACNO); + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, + INIT_CTX_ENABLE_VALUE); + qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT, + handle->hal_handle->upc_mask & + INIT_PC_VALUE); + qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE); + qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE); + qat_hal_put_wakeup_event(handle, ae, ctx_mask, + INIT_WAKEUP_EVENTS_VALUE); + qat_hal_put_sig_event(handle, ae, ctx_mask, + INIT_SIG_EVENTS_VALUE); + } + return 0; +} + +static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, + struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); + unsigned int max_en_ae_id = 0; + struct adf_bar *sram_bar; + unsigned int csr_val = 0; + unsigned long ae_mask; + unsigned char ae = 0; + int ret = 0; + + handle->pci_dev = pci_info->pci_dev; + switch (handle->pci_dev->device) { + case PCI_DEVICE_ID_INTEL_QAT_4XXX: + case PCI_DEVICE_ID_INTEL_QAT_401XX: + case PCI_DEVICE_ID_INTEL_QAT_402XX: + case PCI_DEVICE_ID_INTEL_QAT_420XX: + case PCI_DEVICE_ID_INTEL_QAT_6XXX: + handle->chip_info->mmp_sram_size = 0; + handle->chip_info->nn = false; + handle->chip_info->lm2lm3 = true; + handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG_2X; + handle->chip_info->icp_rst_csr = ICP_RESET_CPP0; + if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_420XX) + handle->chip_info->icp_rst_mask = 0x100155; + else + handle->chip_info->icp_rst_mask = 0x100015; + handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE_CPP0; + handle->chip_info->misc_ctl_csr = MISC_CONTROL_C4XXX; + handle->chip_info->wakeup_event_val = 0x80000000; + handle->chip_info->fw_auth = true; + handle->chip_info->css_3k = true; + if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_6XXX) { + handle->chip_info->dual_sign = true; + handle->chip_info->reset_delay_us = MIN_RESET_DELAY_US; + } + handle->chip_info->tgroup_share_ustore = true; + handle->chip_info->fcu_ctl_csr = FCU_CONTROL_4XXX; + handle->chip_info->fcu_sts_csr = FCU_STATUS_4XXX; + handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI_4XXX; + handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO_4XXX; + handle->chip_info->fcu_loaded_ae_csr = FCU_AE_LOADED_4XXX; + handle->chip_info->fcu_loaded_ae_pos = 0; + + handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET_4XXX; + handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET_4XXX; + handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET_4XXX; + handle->hal_cap_ae_local_csr_addr_v = + (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + + LOCAL_TO_XFER_REG_OFFSET); + break; + case PCI_DEVICE_ID_INTEL_QAT_C62X: + case PCI_DEVICE_ID_INTEL_QAT_C3XXX: + handle->chip_info->mmp_sram_size = 0; + handle->chip_info->nn = true; + handle->chip_info->lm2lm3 = false; + handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; + handle->chip_info->icp_rst_csr = ICP_RESET; + handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) | + (hw_data->accel_mask << RST_CSR_QAT_LSB); + handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE; + handle->chip_info->misc_ctl_csr = MISC_CONTROL; + handle->chip_info->wakeup_event_val = WAKEUP_EVENT; + handle->chip_info->fw_auth = true; + handle->chip_info->css_3k = false; + handle->chip_info->tgroup_share_ustore = false; + handle->chip_info->fcu_ctl_csr = FCU_CONTROL; + handle->chip_info->fcu_sts_csr = FCU_STATUS; + handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI; + handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO; + handle->chip_info->fcu_loaded_ae_csr = FCU_STATUS; + handle->chip_info->fcu_loaded_ae_pos = FCU_LOADED_AE_POS; + handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET; + handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET; + handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET; + handle->hal_cap_ae_local_csr_addr_v = + (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + + LOCAL_TO_XFER_REG_OFFSET); + break; + case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: + handle->chip_info->mmp_sram_size = 0x40000; + handle->chip_info->nn = true; + handle->chip_info->lm2lm3 = false; + handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; + handle->chip_info->icp_rst_csr = ICP_RESET; + handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) | + (hw_data->accel_mask << RST_CSR_QAT_LSB); + handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE; + handle->chip_info->misc_ctl_csr = MISC_CONTROL; + handle->chip_info->wakeup_event_val = WAKEUP_EVENT; + handle->chip_info->fw_auth = false; + handle->chip_info->css_3k = false; + handle->chip_info->tgroup_share_ustore = false; + handle->chip_info->fcu_ctl_csr = 0; + handle->chip_info->fcu_sts_csr = 0; + handle->chip_info->fcu_dram_addr_hi = 0; + handle->chip_info->fcu_dram_addr_lo = 0; + handle->chip_info->fcu_loaded_ae_csr = 0; + handle->chip_info->fcu_loaded_ae_pos = 0; + handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET; + handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET; + handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET; + handle->hal_cap_ae_local_csr_addr_v = + (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + + LOCAL_TO_XFER_REG_OFFSET); + break; + default: + ret = -EINVAL; + goto out_err; + } + + if (handle->chip_info->mmp_sram_size > 0) { + sram_bar = + &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)]; + handle->hal_sram_addr_v = sram_bar->virt_addr; + } + handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid; + handle->hal_handle->ae_mask = hw_data->ae_mask; + handle->hal_handle->admin_ae_mask = hw_data->admin_ae_mask; + handle->hal_handle->slice_mask = hw_data->accel_mask; + handle->cfg_ae_mask = ALL_AE_MASK; + /* create AE objects */ + handle->hal_handle->upc_mask = 0x1ffff; + handle->hal_handle->max_ustore = 0x4000; + + ae_mask = handle->hal_handle->ae_mask; + for_each_set_bit(ae, &ae_mask, ICP_QAT_UCLO_MAX_AE) { + handle->hal_handle->aes[ae].free_addr = 0; + handle->hal_handle->aes[ae].free_size = + handle->hal_handle->max_ustore; + handle->hal_handle->aes[ae].ustore_size = + handle->hal_handle->max_ustore; + handle->hal_handle->aes[ae].live_ctx_mask = + ICP_QAT_UCLO_AE_ALL_CTX; + max_en_ae_id = ae; + } + handle->hal_handle->ae_max_num = max_en_ae_id + 1; + + /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */ + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + csr_val = qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE); + csr_val |= 0x1; + qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val); + } +out_err: + return ret; +} + +int qat_hal_init(struct adf_accel_dev *accel_dev) +{ + struct icp_qat_fw_loader_handle *handle; + int ret = 0; + + handle = kzalloc_obj(*handle); + if (!handle) + return -ENOMEM; + + handle->hal_handle = kzalloc_obj(*handle->hal_handle); + if (!handle->hal_handle) { + ret = -ENOMEM; + goto out_hal_handle; + } + + handle->chip_info = kzalloc_obj(*handle->chip_info); + if (!handle->chip_info) { + ret = -ENOMEM; + goto out_chip_info; + } + + ret = qat_hal_chip_init(handle, accel_dev); + if (ret) { + dev_err(&GET_DEV(accel_dev), "qat_hal_chip_init error\n"); + goto out_err; + } + + /* take all AEs out of reset */ + ret = qat_hal_clr_reset(handle); + if (ret) { + dev_err(&GET_DEV(accel_dev), "qat_hal_clr_reset error\n"); + goto out_err; + } + + qat_hal_clear_xfer(handle); + if (!handle->chip_info->fw_auth) { + ret = qat_hal_clear_gpr(handle); + if (ret) + goto out_err; + } + + accel_dev->fw_loader->fw_loader = handle; + return 0; + +out_err: + kfree(handle->chip_info); +out_chip_info: + kfree(handle->hal_handle); +out_hal_handle: + kfree(handle); + return ret; +} + +void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle) +{ + if (!handle) + return; + kfree(handle->chip_info); + kfree(handle->hal_handle); + kfree(handle); +} + +int qat_hal_start(struct icp_qat_fw_loader_handle *handle) +{ + unsigned long ae_mask = handle->hal_handle->ae_mask; + u32 wakeup_val = handle->chip_info->wakeup_event_val; + u32 fcu_ctl_csr, fcu_sts_csr; + unsigned int fcu_sts; + unsigned char ae; + u32 ae_ctr = 0; + int retry = 0; + + if (handle->chip_info->fw_auth) { + fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; + fcu_sts_csr = handle->chip_info->fcu_sts_csr; + ae_ctr = hweight32(ae_mask); + SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_START); + do { + msleep(FW_AUTH_WAIT_PERIOD); + fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); + if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1)) + return ae_ctr; + } while (retry++ < FW_AUTH_MAX_RETRY); + pr_err("QAT: start error (FCU_STS = 0x%x)\n", fcu_sts); + return 0; + } else { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + qat_hal_put_wakeup_event(handle, ae, 0, wakeup_val); + qat_hal_enable_ctx(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX); + ae_ctr++; + } + return ae_ctr; + } +} + +void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae, + unsigned int ctx_mask) +{ + if (!handle->chip_info->fw_auth) + qat_hal_disable_ctx(handle, ae, ctx_mask); +} + +void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int ctx_mask, unsigned int upc) +{ + qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT, + handle->hal_handle->upc_mask & upc); +} + +static void qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int uaddr, + unsigned int words_num, u64 *uword) +{ + unsigned int i, uwrd_lo, uwrd_hi; + unsigned int ustore_addr, misc_control; + + misc_control = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL); + qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, + misc_control & 0xfffffffb); + ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS); + uaddr |= UA_ECS; + for (i = 0; i < words_num; i++) { + qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); + uaddr++; + uwrd_lo = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER); + uwrd_hi = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER); + uword[i] = uwrd_hi; + uword[i] = (uword[i] << 0x20) | uwrd_lo; + } + qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, misc_control); + qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); +} + +void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int uaddr, + unsigned int words_num, unsigned int *data) +{ + unsigned int i, ustore_addr; + + ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS); + uaddr |= UA_ECS; + qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); + for (i = 0; i < words_num; i++) { + unsigned int uwrd_lo, uwrd_hi, tmp; + + uwrd_lo = ((data[i] & 0xfff0000) << 4) | (0x3 << 18) | + ((data[i] & 0xff00) << 2) | + (0x3 << 8) | (data[i] & 0xff); + uwrd_hi = (0xf << 4) | ((data[i] & 0xf0000000) >> 28); + uwrd_hi |= (hweight32(data[i] & 0xffff) & 0x1) << 8; + tmp = ((data[i] >> 0x10) & 0xffff); + uwrd_hi |= (hweight32(tmp) & 0x1) << 9; + qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo); + qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi); + } + qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); +} + +#define MAX_EXEC_INST 100 +static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx, + u64 *micro_inst, unsigned int inst_num, + int code_off, unsigned int max_cycle, + unsigned int *endpc) +{ + unsigned int ind_lm_addr_byte0 = 0, ind_lm_addr_byte1 = 0; + unsigned int ind_lm_addr_byte2 = 0, ind_lm_addr_byte3 = 0; + unsigned int ind_t_index = 0, ind_t_index_byte = 0; + unsigned int ind_lm_addr0 = 0, ind_lm_addr1 = 0; + unsigned int ind_lm_addr2 = 0, ind_lm_addr3 = 0; + u64 savuwords[MAX_EXEC_INST]; + unsigned int ind_cnt_sig; + unsigned int ind_sig, act_sig; + unsigned int csr_val = 0, newcsr_val; + unsigned int savctx; + unsigned int savcc, wakeup_events, savpc; + unsigned int ctxarb_ctl, ctx_enables; + + if ((inst_num > handle->hal_handle->max_ustore) || !micro_inst) { + pr_err("QAT: invalid instruction num %d\n", inst_num); + return -EINVAL; + } + /* save current context */ + ind_lm_addr0 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT); + ind_lm_addr1 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT); + ind_lm_addr_byte0 = qat_hal_rd_indr_csr(handle, ae, ctx, + INDIRECT_LM_ADDR_0_BYTE_INDEX); + ind_lm_addr_byte1 = qat_hal_rd_indr_csr(handle, ae, ctx, + INDIRECT_LM_ADDR_1_BYTE_INDEX); + if (handle->chip_info->lm2lm3) { + ind_lm_addr2 = qat_hal_rd_indr_csr(handle, ae, ctx, + LM_ADDR_2_INDIRECT); + ind_lm_addr3 = qat_hal_rd_indr_csr(handle, ae, ctx, + LM_ADDR_3_INDIRECT); + ind_lm_addr_byte2 = qat_hal_rd_indr_csr(handle, ae, ctx, + INDIRECT_LM_ADDR_2_BYTE_INDEX); + ind_lm_addr_byte3 = qat_hal_rd_indr_csr(handle, ae, ctx, + INDIRECT_LM_ADDR_3_BYTE_INDEX); + ind_t_index = qat_hal_rd_indr_csr(handle, ae, ctx, + INDIRECT_T_INDEX); + ind_t_index_byte = qat_hal_rd_indr_csr(handle, ae, ctx, + INDIRECT_T_INDEX_BYTE_INDEX); + } + if (inst_num <= MAX_EXEC_INST) + qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords); + qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events); + savpc = qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT); + savpc = (savpc & handle->hal_handle->upc_mask) >> 0; + ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); + ctx_enables &= IGNORE_W1C_MASK; + savcc = qat_hal_rd_ae_csr(handle, ae, CC_ENABLE); + savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); + ctxarb_ctl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL); + ind_cnt_sig = qat_hal_rd_indr_csr(handle, ae, ctx, + FUTURE_COUNT_SIGNAL_INDIRECT); + ind_sig = qat_hal_rd_indr_csr(handle, ae, ctx, + CTX_SIG_EVENTS_INDIRECT); + act_sig = qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE); + /* execute micro codes */ + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); + qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst); + qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, 0); + qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, ctx & ACS_ACNO); + if (code_off) + qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc & 0xffffdfff); + qat_hal_put_wakeup_event(handle, ae, (1 << ctx), XCWE_VOLUNTARY); + qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, 0); + qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0); + qat_hal_enable_ctx(handle, ae, (1 << ctx)); + /* wait for micro codes to finish */ + if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0) + return -EFAULT; + if (endpc) { + unsigned int ctx_status; + + ctx_status = qat_hal_rd_indr_csr(handle, ae, ctx, + CTX_STS_INDIRECT); + *endpc = ctx_status & handle->hal_handle->upc_mask; + } + /* retore to saved context */ + qat_hal_disable_ctx(handle, ae, (1 << ctx)); + if (inst_num <= MAX_EXEC_INST) + qat_hal_wr_uwords(handle, ae, 0, inst_num, savuwords); + qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events); + qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, + handle->hal_handle->upc_mask & savpc); + csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL); + newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS); + qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val); + qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc); + qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, savctx & ACS_ACNO); + qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_ctl); + qat_hal_wr_indr_csr(handle, ae, (1 << ctx), + LM_ADDR_0_INDIRECT, ind_lm_addr0); + qat_hal_wr_indr_csr(handle, ae, (1 << ctx), + LM_ADDR_1_INDIRECT, ind_lm_addr1); + qat_hal_wr_indr_csr(handle, ae, (1 << ctx), + INDIRECT_LM_ADDR_0_BYTE_INDEX, ind_lm_addr_byte0); + qat_hal_wr_indr_csr(handle, ae, (1 << ctx), + INDIRECT_LM_ADDR_1_BYTE_INDEX, ind_lm_addr_byte1); + if (handle->chip_info->lm2lm3) { + qat_hal_wr_indr_csr(handle, ae, BIT(ctx), LM_ADDR_2_INDIRECT, + ind_lm_addr2); + qat_hal_wr_indr_csr(handle, ae, BIT(ctx), LM_ADDR_3_INDIRECT, + ind_lm_addr3); + qat_hal_wr_indr_csr(handle, ae, BIT(ctx), + INDIRECT_LM_ADDR_2_BYTE_INDEX, + ind_lm_addr_byte2); + qat_hal_wr_indr_csr(handle, ae, BIT(ctx), + INDIRECT_LM_ADDR_3_BYTE_INDEX, + ind_lm_addr_byte3); + qat_hal_wr_indr_csr(handle, ae, BIT(ctx), + INDIRECT_T_INDEX, ind_t_index); + qat_hal_wr_indr_csr(handle, ae, BIT(ctx), + INDIRECT_T_INDEX_BYTE_INDEX, + ind_t_index_byte); + } + qat_hal_wr_indr_csr(handle, ae, (1 << ctx), + FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig); + qat_hal_wr_indr_csr(handle, ae, (1 << ctx), + CTX_SIG_EVENTS_INDIRECT, ind_sig); + qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, act_sig); + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); + + return 0; +} + +static int qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx, + enum icp_qat_uof_regtype reg_type, + unsigned short reg_num, unsigned int *data) +{ + unsigned int savctx, uaddr, uwrd_lo, uwrd_hi; + unsigned int ctxarb_cntl, ustore_addr, ctx_enables; + unsigned short reg_addr; + int status = 0; + u64 insts, savuword; + + reg_addr = qat_hal_get_reg_addr(reg_type, reg_num); + if (reg_addr == BAD_REGADDR) { + pr_err("QAT: bad regaddr=0x%x\n", reg_addr); + return -EINVAL; + } + switch (reg_type) { + case ICP_GPA_REL: + insts = 0xA070000000ull | (reg_addr & 0x3ff); + break; + default: + insts = (u64)0xA030000000ull | ((reg_addr & 0x3ff) << 10); + break; + } + savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS); + ctxarb_cntl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL); + ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); + ctx_enables &= IGNORE_W1C_MASK; + if (ctx != (savctx & ACS_ACNO)) + qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, + ctx & ACS_ACNO); + qat_hal_get_uwords(handle, ae, 0, 1, &savuword); + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); + ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS); + uaddr = UA_ECS; + qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); + insts = qat_hal_set_uword_ecc(insts); + uwrd_lo = (unsigned int)(insts & 0xffffffff); + uwrd_hi = (unsigned int)(insts >> 0x20); + qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo); + qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi); + qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); + /* delay for at least 8 cycles */ + qat_hal_wait_cycles(handle, ae, 0x8, 0); + /* + * read ALU output + * the instruction should have been executed + * prior to clearing the ECS in putUwords + */ + *data = qat_hal_rd_ae_csr(handle, ae, ALU_OUT); + qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); + qat_hal_wr_uwords(handle, ae, 0, 1, &savuword); + if (ctx != (savctx & ACS_ACNO)) + qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, + savctx & ACS_ACNO); + qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_cntl); + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); + + return status; +} + +static int qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx, + enum icp_qat_uof_regtype reg_type, + unsigned short reg_num, unsigned int data) +{ + unsigned short src_hiaddr, src_lowaddr, dest_addr, data16hi, data16lo; + u64 insts[] = { + 0x0F440000000ull, + 0x0F040000000ull, + 0x0F0000C0300ull, + 0x0E000010000ull + }; + const int num_inst = ARRAY_SIZE(insts), code_off = 1; + const int imm_w1 = 0, imm_w0 = 1; + + dest_addr = qat_hal_get_reg_addr(reg_type, reg_num); + if (dest_addr == BAD_REGADDR) { + pr_err("QAT: bad destAddr=0x%x\n", dest_addr); + return -EINVAL; + } + + data16lo = 0xffff & data; + data16hi = 0xffff & (data >> 0x10); + src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short) + (0xff & data16hi)); + src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short) + (0xff & data16lo)); + switch (reg_type) { + case ICP_GPA_REL: + insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) | + ((src_hiaddr & 0x3ff) << 10) | (dest_addr & 0x3ff); + insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) | + ((src_lowaddr & 0x3ff) << 10) | (dest_addr & 0x3ff); + break; + default: + insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) | + ((dest_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff); + + insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) | + ((dest_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff); + break; + } + + return qat_hal_exec_micro_inst(handle, ae, ctx, insts, num_inst, + code_off, num_inst * 0x5, NULL); +} + +int qat_hal_get_ins_num(void) +{ + return ARRAY_SIZE(inst_4b); +} + +static int qat_hal_concat_micro_code(u64 *micro_inst, + unsigned int inst_num, unsigned int size, + unsigned int addr, unsigned int *value) +{ + int i; + unsigned int cur_value; + const u64 *inst_arr; + int fixup_offset; + int usize = 0; + int orig_num; + + orig_num = inst_num; + cur_value = value[0]; + inst_arr = inst_4b; + usize = ARRAY_SIZE(inst_4b); + fixup_offset = inst_num; + for (i = 0; i < usize; i++) + micro_inst[inst_num++] = inst_arr[i]; + INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], (addr)); + fixup_offset++; + INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], 0); + fixup_offset++; + INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0)); + fixup_offset++; + INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0x10)); + + return inst_num - orig_num; +} + +static int qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx, + int *pfirst_exec, u64 *micro_inst, + unsigned int inst_num) +{ + int stat = 0; + unsigned int gpra0 = 0, gpra1 = 0, gpra2 = 0; + unsigned int gprb0 = 0, gprb1 = 0; + + if (*pfirst_exec) { + qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, &gpra0); + qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, &gpra1); + qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, &gpra2); + qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, &gprb0); + qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, &gprb1); + *pfirst_exec = 0; + } + stat = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, inst_num, 1, + inst_num * 0x5, NULL); + if (stat != 0) + return -EFAULT; + qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, gpra0); + qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, gpra1); + qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, gpra2); + qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, gprb0); + qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, gprb1); + + return 0; +} + +int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, + struct icp_qat_uof_batch_init *lm_init_header) +{ + struct icp_qat_uof_batch_init *plm_init; + u64 *micro_inst_arry; + int micro_inst_num; + int alloc_inst_size; + int first_exec = 1; + int stat = 0; + + plm_init = lm_init_header->next; + alloc_inst_size = lm_init_header->size; + if ((unsigned int)alloc_inst_size > handle->hal_handle->max_ustore) + alloc_inst_size = handle->hal_handle->max_ustore; + micro_inst_arry = kmalloc_array(alloc_inst_size, sizeof(u64), + GFP_KERNEL); + if (!micro_inst_arry) + return -ENOMEM; + micro_inst_num = 0; + while (plm_init) { + unsigned int addr, *value, size; + + ae = plm_init->ae; + addr = plm_init->addr; + value = plm_init->value; + size = plm_init->size; + micro_inst_num += qat_hal_concat_micro_code(micro_inst_arry, + micro_inst_num, + size, addr, value); + plm_init = plm_init->next; + } + /* exec micro codes */ + if (micro_inst_arry && micro_inst_num > 0) { + micro_inst_arry[micro_inst_num++] = 0x0E000010000ull; + stat = qat_hal_exec_micro_init_lm(handle, ae, 0, &first_exec, + micro_inst_arry, + micro_inst_num); + } + kfree(micro_inst_arry); + return stat; +} + +static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx, + enum icp_qat_uof_regtype reg_type, + unsigned short reg_num, unsigned int val) +{ + int status = 0; + unsigned int reg_addr; + unsigned int ctx_enables; + unsigned short mask; + unsigned short dr_offset = 0x10; + + ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); + if (CE_INUSE_CONTEXTS & ctx_enables) { + if (ctx & 0x1) { + pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx); + return -EINVAL; + } + mask = 0x1f; + dr_offset = 0x20; + } else { + mask = 0x0f; + } + if (reg_num & ~mask) + return -EINVAL; + reg_addr = reg_num + (ctx << 0x5); + switch (reg_type) { + case ICP_SR_RD_REL: + case ICP_SR_REL: + SET_AE_XFER(handle, ae, reg_addr, val); + break; + case ICP_DR_RD_REL: + case ICP_DR_REL: + SET_AE_XFER(handle, ae, (reg_addr + dr_offset), val); + break; + default: + status = -EINVAL; + break; + } + return status; +} + +static int qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx, + enum icp_qat_uof_regtype reg_type, + unsigned short reg_num, unsigned int data) +{ + unsigned int gprval, ctx_enables; + unsigned short src_hiaddr, src_lowaddr, gpr_addr, xfr_addr, data16hi, + data16low; + unsigned short reg_mask; + int status = 0; + u64 micro_inst[] = { + 0x0F440000000ull, + 0x0F040000000ull, + 0x0A000000000ull, + 0x0F0000C0300ull, + 0x0E000010000ull + }; + const int num_inst = ARRAY_SIZE(micro_inst), code_off = 1; + const unsigned short gprnum = 0, dly = num_inst * 0x5; + + ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); + if (CE_INUSE_CONTEXTS & ctx_enables) { + if (ctx & 0x1) { + pr_err("QAT: 4-ctx mode,ctx=0x%x\n", ctx); + return -EINVAL; + } + reg_mask = (unsigned short)~0x1f; + } else { + reg_mask = (unsigned short)~0xf; + } + if (reg_num & reg_mask) + return -EINVAL; + xfr_addr = qat_hal_get_reg_addr(reg_type, reg_num); + if (xfr_addr == BAD_REGADDR) { + pr_err("QAT: bad xfrAddr=0x%x\n", xfr_addr); + return -EINVAL; + } + status = qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval); + if (status) { + pr_err("QAT: failed to read register"); + return status; + } + gpr_addr = qat_hal_get_reg_addr(ICP_GPB_REL, gprnum); + data16low = 0xffff & data; + data16hi = 0xffff & (data >> 0x10); + src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, + (unsigned short)(0xff & data16hi)); + src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, + (unsigned short)(0xff & data16low)); + micro_inst[0] = micro_inst[0x0] | ((data16hi >> 8) << 20) | + ((gpr_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff); + micro_inst[1] = micro_inst[0x1] | ((data16low >> 8) << 20) | + ((gpr_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff); + micro_inst[0x2] = micro_inst[0x2] | + ((xfr_addr & 0x3ff) << 20) | ((gpr_addr & 0x3ff) << 10); + status = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, num_inst, + code_off, dly, NULL); + qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, gprval); + return status; +} + +static int qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx, + unsigned short nn, unsigned int val) +{ + unsigned int ctx_enables; + int stat = 0; + + ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); + ctx_enables &= IGNORE_W1C_MASK; + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE); + + stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, ICP_NEIGH_REL, nn, val); + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); + return stat; +} + +static int qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle + *handle, unsigned char ae, + unsigned short absreg_num, + unsigned short *relreg, + unsigned char *ctx) +{ + unsigned int ctx_enables; + + ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); + if (ctx_enables & CE_INUSE_CONTEXTS) { + /* 4-ctx mode */ + *relreg = absreg_num & 0x1F; + *ctx = (absreg_num >> 0x4) & 0x6; + } else { + /* 8-ctx mode */ + *relreg = absreg_num & 0x0F; + *ctx = (absreg_num >> 0x4) & 0x7; + } + return 0; +} + +int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned long ctx_mask, + enum icp_qat_uof_regtype reg_type, + unsigned short reg_num, unsigned int regdata) +{ + int stat = 0; + unsigned short reg; + unsigned char ctx = 0; + enum icp_qat_uof_regtype type; + + if (reg_num >= ICP_QAT_UCLO_MAX_GPR_REG) + return -EINVAL; + + do { + if (ctx_mask == 0) { + qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®, + &ctx); + type = reg_type - 1; + } else { + reg = reg_num; + type = reg_type; + if (!test_bit(ctx, &ctx_mask)) + continue; + } + stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata); + if (stat) { + pr_err("QAT: write gpr fail\n"); + return -EINVAL; + } + } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX)); + + return 0; +} + +int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned long ctx_mask, + enum icp_qat_uof_regtype reg_type, + unsigned short reg_num, unsigned int regdata) +{ + int stat = 0; + unsigned short reg; + unsigned char ctx = 0; + enum icp_qat_uof_regtype type; + + if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG) + return -EINVAL; + + do { + if (ctx_mask == 0) { + qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®, + &ctx); + type = reg_type - 3; + } else { + reg = reg_num; + type = reg_type; + if (!test_bit(ctx, &ctx_mask)) + continue; + } + stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, type, reg, + regdata); + if (stat) { + pr_err("QAT: write wr xfer fail\n"); + return -EINVAL; + } + } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX)); + + return 0; +} + +int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned long ctx_mask, + enum icp_qat_uof_regtype reg_type, + unsigned short reg_num, unsigned int regdata) +{ + int stat = 0; + unsigned short reg; + unsigned char ctx = 0; + enum icp_qat_uof_regtype type; + + if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG) + return -EINVAL; + + do { + if (ctx_mask == 0) { + qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®, + &ctx); + type = reg_type - 3; + } else { + reg = reg_num; + type = reg_type; + if (!test_bit(ctx, &ctx_mask)) + continue; + } + stat = qat_hal_put_rel_rd_xfer(handle, ae, ctx, type, reg, + regdata); + if (stat) { + pr_err("QAT: write rd xfer fail\n"); + return -EINVAL; + } + } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX)); + + return 0; +} + +int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned long ctx_mask, + unsigned short reg_num, unsigned int regdata) +{ + int stat = 0; + unsigned char ctx; + if (!handle->chip_info->nn) { + dev_err(&handle->pci_dev->dev, "QAT: No next neigh in 0x%x\n", + handle->pci_dev->device); + return -EINVAL; + } + + if (ctx_mask == 0) + return -EINVAL; + + for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { + if (!test_bit(ctx, &ctx_mask)) + continue; + stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata); + if (stat) { + pr_err("QAT: write neigh error\n"); + return -EINVAL; + } + } + + return 0; +} diff --git a/drivers/crypto/intel/qat/qat_common/qat_mig_dev.c b/drivers/crypto/intel/qat/qat_common/qat_mig_dev.c new file mode 100644 index 000000000..3869ada01 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/qat_mig_dev.c @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2024 Intel Corporation */ +#include +#include +#include +#include +#include +#include "adf_accel_devices.h" +#include "adf_common_drv.h" + +struct qat_mig_dev *qat_vfmig_create(struct pci_dev *pdev, int vf_id) +{ + struct adf_accel_dev *accel_dev; + struct qat_migdev_ops *ops; + struct qat_mig_dev *mdev; + + accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + if (!accel_dev) + return ERR_PTR(-ENODEV); + + ops = GET_VFMIG_OPS(accel_dev); + if (!ops || !ops->init || !ops->cleanup || !ops->reset || !ops->open || + !ops->close || !ops->suspend || !ops->resume || !ops->save_state || + !ops->load_state || !ops->save_setup || !ops->load_setup) + return ERR_PTR(-EINVAL); + + mdev = kmalloc_obj(*mdev); + if (!mdev) + return ERR_PTR(-ENOMEM); + + mdev->vf_id = vf_id; + mdev->parent_accel_dev = accel_dev; + + return mdev; +} +EXPORT_SYMBOL_GPL(qat_vfmig_create); + +int qat_vfmig_init(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + + return GET_VFMIG_OPS(accel_dev)->init(mdev); +} +EXPORT_SYMBOL_GPL(qat_vfmig_init); + +void qat_vfmig_cleanup(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + + return GET_VFMIG_OPS(accel_dev)->cleanup(mdev); +} +EXPORT_SYMBOL_GPL(qat_vfmig_cleanup); + +void qat_vfmig_reset(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + + return GET_VFMIG_OPS(accel_dev)->reset(mdev); +} +EXPORT_SYMBOL_GPL(qat_vfmig_reset); + +int qat_vfmig_open(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + + return GET_VFMIG_OPS(accel_dev)->open(mdev); +} +EXPORT_SYMBOL_GPL(qat_vfmig_open); + +void qat_vfmig_close(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + + GET_VFMIG_OPS(accel_dev)->close(mdev); +} +EXPORT_SYMBOL_GPL(qat_vfmig_close); + +int qat_vfmig_suspend(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + + return GET_VFMIG_OPS(accel_dev)->suspend(mdev); +} +EXPORT_SYMBOL_GPL(qat_vfmig_suspend); + +int qat_vfmig_resume(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + + return GET_VFMIG_OPS(accel_dev)->resume(mdev); +} +EXPORT_SYMBOL_GPL(qat_vfmig_resume); + +int qat_vfmig_save_state(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + + return GET_VFMIG_OPS(accel_dev)->save_state(mdev); +} +EXPORT_SYMBOL_GPL(qat_vfmig_save_state); + +int qat_vfmig_save_setup(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + + return GET_VFMIG_OPS(accel_dev)->save_setup(mdev); +} +EXPORT_SYMBOL_GPL(qat_vfmig_save_setup); + +int qat_vfmig_load_state(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + + return GET_VFMIG_OPS(accel_dev)->load_state(mdev); +} +EXPORT_SYMBOL_GPL(qat_vfmig_load_state); + +int qat_vfmig_load_setup(struct qat_mig_dev *mdev, int size) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + + return GET_VFMIG_OPS(accel_dev)->load_setup(mdev, size); +} +EXPORT_SYMBOL_GPL(qat_vfmig_load_setup); + +void qat_vfmig_destroy(struct qat_mig_dev *mdev) +{ + kfree(mdev); +} +EXPORT_SYMBOL_GPL(qat_vfmig_destroy); diff --git a/drivers/crypto/intel/qat/qat_common/qat_uclo.c b/drivers/crypto/intel/qat/qat_common/qat_uclo.c new file mode 100644 index 000000000..a00ca2a09 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/qat_uclo.c @@ -0,0 +1,2291 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2020 Intel Corporation */ + +#define pr_fmt(fmt) "QAT: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include "adf_accel_devices.h" +#include "adf_anti_rb.h" +#include "adf_common_drv.h" +#include "icp_qat_uclo.h" +#include "icp_qat_hal.h" +#include "icp_qat_fw_loader_handle.h" + +#define UWORD_CPYBUF_SIZE 1024U +#define INVLD_UWORD 0xffffffffffull +#define PID_MINOR_REV 0xf +#define PID_MAJOR_REV (0xf << 4) + +static int qat_uclo_init_ae_data(struct icp_qat_uclo_objhandle *obj_handle, + unsigned int ae, unsigned int image_num) +{ + struct icp_qat_uclo_aedata *ae_data; + struct icp_qat_uclo_encapme *encap_image; + struct icp_qat_uclo_page *page = NULL; + struct icp_qat_uclo_aeslice *ae_slice = NULL; + + ae_data = &obj_handle->ae_data[ae]; + encap_image = &obj_handle->ae_uimage[image_num]; + ae_slice = &ae_data->ae_slices[ae_data->slice_num]; + ae_slice->encap_image = encap_image; + + if (encap_image->img_ptr) { + ae_slice->ctx_mask_assigned = + encap_image->img_ptr->ctx_assigned; + ae_data->eff_ustore_size = obj_handle->ustore_phy_size; + } else { + ae_slice->ctx_mask_assigned = 0; + } + ae_slice->region = kzalloc_obj(*ae_slice->region); + if (!ae_slice->region) + return -ENOMEM; + ae_slice->page = kzalloc_obj(*ae_slice->page); + if (!ae_slice->page) + goto out_err; + page = ae_slice->page; + page->encap_page = encap_image->page; + ae_slice->page->region = ae_slice->region; + ae_data->slice_num++; + return 0; +out_err: + kfree(ae_slice->region); + ae_slice->region = NULL; + return -ENOMEM; +} + +static int qat_uclo_free_ae_data(struct icp_qat_uclo_aedata *ae_data) +{ + unsigned int i; + + if (!ae_data) { + pr_err("bad argument, ae_data is NULL\n"); + return -EINVAL; + } + + for (i = 0; i < ae_data->slice_num; i++) { + kfree(ae_data->ae_slices[i].region); + ae_data->ae_slices[i].region = NULL; + kfree(ae_data->ae_slices[i].page); + ae_data->ae_slices[i].page = NULL; + } + return 0; +} + +static char *qat_uclo_get_string(struct icp_qat_uof_strtable *str_table, + unsigned int str_offset) +{ + if (!str_table->table_len || str_offset > str_table->table_len) + return NULL; + return (char *)(((uintptr_t)(str_table->strings)) + str_offset); +} + +static int qat_uclo_check_uof_format(struct icp_qat_uof_filehdr *hdr) +{ + int maj = hdr->maj_ver & 0xff; + int min = hdr->min_ver & 0xff; + + if (hdr->file_id != ICP_QAT_UOF_FID) { + pr_err("Invalid header 0x%x\n", hdr->file_id); + return -EINVAL; + } + if (min != ICP_QAT_UOF_MINVER || maj != ICP_QAT_UOF_MAJVER) { + pr_err("bad UOF version, major 0x%x, minor 0x%x\n", maj, min); + return -EINVAL; + } + return 0; +} + +static int qat_uclo_check_suof_format(struct icp_qat_suof_filehdr *suof_hdr) +{ + int maj = suof_hdr->maj_ver & 0xff; + int min = suof_hdr->min_ver & 0xff; + + if (suof_hdr->file_id != ICP_QAT_SUOF_FID) { + pr_err("invalid header 0x%x\n", suof_hdr->file_id); + return -EINVAL; + } + if (suof_hdr->fw_type != 0) { + pr_err("unsupported firmware type\n"); + return -EINVAL; + } + if (suof_hdr->num_chunks <= 0x1) { + pr_err("SUOF chunk amount is incorrect\n"); + return -EINVAL; + } + if (maj != ICP_QAT_SUOF_MAJVER || min != ICP_QAT_SUOF_MINVER) { + pr_err("bad SUOF version, major 0x%x, minor 0x%x\n", maj, min); + return -EINVAL; + } + return 0; +} + +static void qat_uclo_wr_sram_by_words(struct icp_qat_fw_loader_handle *handle, + unsigned int addr, unsigned int *val, + unsigned int num_in_bytes) +{ + unsigned int outval; + unsigned char *ptr = (unsigned char *)val; + + while (num_in_bytes) { + memcpy(&outval, ptr, 4); + SRAM_WRITE(handle, addr, outval); + num_in_bytes -= 4; + ptr += 4; + addr += 4; + } +} + +static void qat_uclo_wr_umem_by_words(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int addr, + unsigned int *val, + unsigned int num_in_bytes) +{ + unsigned int outval; + unsigned char *ptr = (unsigned char *)val; + + addr >>= 0x2; /* convert to uword address */ + + while (num_in_bytes) { + memcpy(&outval, ptr, 4); + qat_hal_wr_umem(handle, ae, addr++, 1, &outval); + num_in_bytes -= 4; + ptr += 4; + } +} + +static void qat_uclo_batch_wr_umem(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, + struct icp_qat_uof_batch_init + *umem_init_header) +{ + struct icp_qat_uof_batch_init *umem_init; + + if (!umem_init_header) + return; + umem_init = umem_init_header->next; + while (umem_init) { + unsigned int addr, *value, size; + + ae = umem_init->ae; + addr = umem_init->addr; + value = umem_init->value; + size = umem_init->size; + qat_uclo_wr_umem_by_words(handle, ae, addr, value, size); + umem_init = umem_init->next; + } +} + +static void +qat_uclo_cleanup_batch_init_list(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_uof_batch_init **base) +{ + struct icp_qat_uof_batch_init *umem_init; + + umem_init = *base; + while (umem_init) { + struct icp_qat_uof_batch_init *pre; + + pre = umem_init; + umem_init = umem_init->next; + kfree(pre); + } + *base = NULL; +} + +static int qat_uclo_parse_num(char *str, unsigned int *num) +{ + unsigned long long ae; + char *end; + + ae = simple_strtoull(str, &end, 10); + if (ae > UINT_MAX || str == end || (end - str) > 19) + return -EINVAL; + *num = (unsigned int)ae; + return 0; +} + +static int qat_uclo_fetch_initmem_ae(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_uof_initmem *init_mem, + unsigned int size_range, unsigned int *ae) +{ + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + char *str; + + if ((init_mem->addr + init_mem->num_in_bytes) > (size_range << 0x2)) { + pr_err("initmem is out of range"); + return -EINVAL; + } + if (init_mem->scope != ICP_QAT_UOF_LOCAL_SCOPE) { + pr_err("Memory scope for init_mem error\n"); + return -EINVAL; + } + str = qat_uclo_get_string(&obj_handle->str_table, init_mem->sym_name); + if (!str) { + pr_err("AE name assigned in UOF init table is NULL\n"); + return -EINVAL; + } + if (qat_uclo_parse_num(str, ae)) { + pr_err("Parse num for AE number failed\n"); + return -EINVAL; + } + if (*ae >= ICP_QAT_UCLO_MAX_AE) { + pr_err("ae %d out of range\n", *ae); + return -EINVAL; + } + return 0; +} + +static int qat_uclo_create_batch_init_list(struct icp_qat_fw_loader_handle + *handle, struct icp_qat_uof_initmem + *init_mem, unsigned int ae, + struct icp_qat_uof_batch_init + **init_tab_base) +{ + struct icp_qat_uof_batch_init *init_header, *tail; + struct icp_qat_uof_batch_init *mem_init, *tail_old; + struct icp_qat_uof_memvar_attr *mem_val_attr; + unsigned int i, flag = 0; + + mem_val_attr = + (struct icp_qat_uof_memvar_attr *)((uintptr_t)init_mem + + sizeof(struct icp_qat_uof_initmem)); + + init_header = *init_tab_base; + if (!init_header) { + init_header = kzalloc_obj(*init_header); + if (!init_header) + return -ENOMEM; + init_header->size = 1; + *init_tab_base = init_header; + flag = 1; + } + tail_old = init_header; + while (tail_old->next) + tail_old = tail_old->next; + tail = tail_old; + for (i = 0; i < init_mem->val_attr_num; i++) { + mem_init = kzalloc_obj(*mem_init); + if (!mem_init) + goto out_err; + mem_init->ae = ae; + mem_init->addr = init_mem->addr + mem_val_attr->offset_in_byte; + mem_init->value = &mem_val_attr->value; + mem_init->size = 4; + mem_init->next = NULL; + tail->next = mem_init; + tail = mem_init; + init_header->size += qat_hal_get_ins_num(); + mem_val_attr++; + } + return 0; +out_err: + /* Do not free the list head unless we allocated it. */ + tail_old = tail_old->next; + if (flag) { + kfree(*init_tab_base); + *init_tab_base = NULL; + } + + while (tail_old) { + mem_init = tail_old->next; + kfree(tail_old); + tail_old = mem_init; + } + return -ENOMEM; +} + +static int qat_uclo_init_lmem_seg(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_uof_initmem *init_mem) +{ + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned int ae; + + if (qat_uclo_fetch_initmem_ae(handle, init_mem, + handle->chip_info->lm_size, &ae)) + return -EINVAL; + if (qat_uclo_create_batch_init_list(handle, init_mem, ae, + &obj_handle->lm_init_tab[ae])) + return -EINVAL; + return 0; +} + +static int qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_uof_initmem *init_mem) +{ + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned int ae, ustore_size, uaddr, i; + struct icp_qat_uclo_aedata *aed; + + ustore_size = obj_handle->ustore_phy_size; + if (qat_uclo_fetch_initmem_ae(handle, init_mem, ustore_size, &ae)) + return -EINVAL; + if (qat_uclo_create_batch_init_list(handle, init_mem, ae, + &obj_handle->umem_init_tab[ae])) + return -EINVAL; + /* set the highest ustore address referenced */ + uaddr = (init_mem->addr + init_mem->num_in_bytes) >> 0x2; + aed = &obj_handle->ae_data[ae]; + for (i = 0; i < aed->slice_num; i++) { + if (aed->ae_slices[i].encap_image->uwords_num < uaddr) + aed->ae_slices[i].encap_image->uwords_num = uaddr; + } + return 0; +} + +static int qat_uclo_init_ae_memory(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_uof_initmem *init_mem) +{ + switch (init_mem->region) { + case ICP_QAT_UOF_LMEM_REGION: + if (qat_uclo_init_lmem_seg(handle, init_mem)) + return -EINVAL; + break; + case ICP_QAT_UOF_UMEM_REGION: + if (qat_uclo_init_umem_seg(handle, init_mem)) + return -EINVAL; + break; + default: + pr_err("initmem region error. region type=0x%x\n", init_mem->region); + return -EINVAL; + } + return 0; +} + +static int qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_uclo_encapme *image) +{ + unsigned int i; + struct icp_qat_uclo_encap_page *page; + struct icp_qat_uof_image *uof_image; + unsigned char ae; + unsigned int ustore_size; + unsigned int patt_pos; + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned long cfg_ae_mask = handle->cfg_ae_mask; + u64 *fill_data; + + uof_image = image->img_ptr; + fill_data = kcalloc(ICP_QAT_UCLO_MAX_USTORE, sizeof(u64), + GFP_KERNEL); + if (!fill_data) + return -ENOMEM; + for (i = 0; i < ICP_QAT_UCLO_MAX_USTORE; i++) + memcpy(&fill_data[i], &uof_image->fill_pattern, + sizeof(u64)); + page = image->page; + + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + unsigned long ae_assigned = uof_image->ae_assigned; + + if (!test_bit(ae, &ae_assigned)) + continue; + + if (!test_bit(ae, &cfg_ae_mask)) + continue; + + ustore_size = obj_handle->ae_data[ae].eff_ustore_size; + patt_pos = page->beg_addr_p + page->micro_words_num; + + qat_hal_wr_uwords(handle, (unsigned char)ae, 0, + page->beg_addr_p, &fill_data[0]); + qat_hal_wr_uwords(handle, (unsigned char)ae, patt_pos, + ustore_size - patt_pos + 1, + &fill_data[page->beg_addr_p]); + } + kfree(fill_data); + return 0; +} + +static int qat_uclo_init_memory(struct icp_qat_fw_loader_handle *handle) +{ + int i, ae; + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + struct icp_qat_uof_initmem *initmem = obj_handle->init_mem_tab.init_mem; + unsigned long ae_mask = handle->hal_handle->ae_mask; + + for (i = 0; i < obj_handle->init_mem_tab.entry_num; i++) { + if (initmem->num_in_bytes) { + if (qat_uclo_init_ae_memory(handle, initmem)) + return -EINVAL; + } + initmem = (struct icp_qat_uof_initmem *)((uintptr_t)( + (uintptr_t)initmem + + sizeof(struct icp_qat_uof_initmem)) + + (sizeof(struct icp_qat_uof_memvar_attr) * + initmem->val_attr_num)); + } + + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + if (qat_hal_batch_wr_lm(handle, ae, + obj_handle->lm_init_tab[ae])) { + pr_err("fail to batch init lmem for AE %d\n", ae); + return -EINVAL; + } + qat_uclo_cleanup_batch_init_list(handle, + &obj_handle->lm_init_tab[ae]); + qat_uclo_batch_wr_umem(handle, ae, + obj_handle->umem_init_tab[ae]); + qat_uclo_cleanup_batch_init_list(handle, + &obj_handle-> + umem_init_tab[ae]); + } + return 0; +} + +static void *qat_uclo_find_chunk(struct icp_qat_uof_objhdr *obj_hdr, + char *chunk_id, void *cur) +{ + int i; + struct icp_qat_uof_chunkhdr *chunk_hdr = + (struct icp_qat_uof_chunkhdr *) + ((uintptr_t)obj_hdr + sizeof(struct icp_qat_uof_objhdr)); + + for (i = 0; i < obj_hdr->num_chunks; i++) { + if ((cur < (void *)&chunk_hdr[i]) && + !strncmp(chunk_hdr[i].chunk_id, chunk_id, + ICP_QAT_UOF_OBJID_LEN)) { + return &chunk_hdr[i]; + } + } + return NULL; +} + +static unsigned int qat_uclo_calc_checksum(unsigned int reg, int ch) +{ + int i; + unsigned int topbit = 1 << 0xF; + unsigned int inbyte = (unsigned int)((reg >> 0x18) ^ ch); + + reg ^= inbyte << 0x8; + for (i = 0; i < 0x8; i++) { + if (reg & topbit) + reg = (reg << 1) ^ 0x1021; + else + reg <<= 1; + } + return reg & 0xFFFF; +} + +static unsigned int qat_uclo_calc_str_checksum(char *ptr, int num) +{ + unsigned int chksum = 0; + + if (ptr) + while (num--) + chksum = qat_uclo_calc_checksum(chksum, *ptr++); + return chksum; +} + +static struct icp_qat_uclo_objhdr * +qat_uclo_map_chunk(char *buf, struct icp_qat_uof_filehdr *file_hdr, + char *chunk_id) +{ + struct icp_qat_uof_filechunkhdr *file_chunk; + struct icp_qat_uclo_objhdr *obj_hdr; + char *chunk; + int i; + + file_chunk = (struct icp_qat_uof_filechunkhdr *) + (buf + sizeof(struct icp_qat_uof_filehdr)); + for (i = 0; i < file_hdr->num_chunks; i++) { + if (!strncmp(file_chunk->chunk_id, chunk_id, + ICP_QAT_UOF_OBJID_LEN)) { + chunk = buf + file_chunk->offset; + if (file_chunk->checksum != qat_uclo_calc_str_checksum( + chunk, file_chunk->size)) + break; + obj_hdr = kzalloc_obj(*obj_hdr); + if (!obj_hdr) + break; + obj_hdr->file_buff = chunk; + obj_hdr->checksum = file_chunk->checksum; + obj_hdr->size = file_chunk->size; + return obj_hdr; + } + file_chunk++; + } + return NULL; +} + +static int +qat_uclo_check_image_compat(struct icp_qat_uof_encap_obj *encap_uof_obj, + struct icp_qat_uof_image *image) +{ + struct icp_qat_uof_objtable *uc_var_tab, *imp_var_tab, *imp_expr_tab; + struct icp_qat_uof_objtable *neigh_reg_tab; + struct icp_qat_uof_code_page *code_page; + + code_page = (struct icp_qat_uof_code_page *) + ((char *)image + sizeof(struct icp_qat_uof_image)); + uc_var_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof + + code_page->uc_var_tab_offset); + imp_var_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof + + code_page->imp_var_tab_offset); + imp_expr_tab = (struct icp_qat_uof_objtable *) + (encap_uof_obj->beg_uof + + code_page->imp_expr_tab_offset); + if (uc_var_tab->entry_num || imp_var_tab->entry_num || + imp_expr_tab->entry_num) { + pr_err("UOF can't contain imported variable to be parsed\n"); + return -EINVAL; + } + neigh_reg_tab = (struct icp_qat_uof_objtable *) + (encap_uof_obj->beg_uof + + code_page->neigh_reg_tab_offset); + if (neigh_reg_tab->entry_num) { + pr_err("UOF can't contain neighbor register table\n"); + return -EINVAL; + } + if (image->numpages > 1) { + pr_err("UOF can't contain multiple pages\n"); + return -EINVAL; + } + if (ICP_QAT_SHARED_USTORE_MODE(image->ae_mode)) { + pr_err("UOF can't use shared control store feature\n"); + return -EFAULT; + } + if (RELOADABLE_CTX_SHARED_MODE(image->ae_mode)) { + pr_err("UOF can't use reloadable feature\n"); + return -EFAULT; + } + return 0; +} + +static void qat_uclo_map_image_page(struct icp_qat_uof_encap_obj + *encap_uof_obj, + struct icp_qat_uof_image *img, + struct icp_qat_uclo_encap_page *page) +{ + struct icp_qat_uof_code_page *code_page; + struct icp_qat_uof_code_area *code_area; + struct icp_qat_uof_objtable *uword_block_tab; + struct icp_qat_uof_uword_block *uwblock; + int i; + + code_page = (struct icp_qat_uof_code_page *) + ((char *)img + sizeof(struct icp_qat_uof_image)); + page->def_page = code_page->def_page; + page->page_region = code_page->page_region; + page->beg_addr_v = code_page->beg_addr_v; + page->beg_addr_p = code_page->beg_addr_p; + code_area = (struct icp_qat_uof_code_area *)(encap_uof_obj->beg_uof + + code_page->code_area_offset); + page->micro_words_num = code_area->micro_words_num; + uword_block_tab = (struct icp_qat_uof_objtable *) + (encap_uof_obj->beg_uof + + code_area->uword_block_tab); + page->uwblock_num = uword_block_tab->entry_num; + uwblock = (struct icp_qat_uof_uword_block *)((char *)uword_block_tab + + sizeof(struct icp_qat_uof_objtable)); + page->uwblock = (struct icp_qat_uclo_encap_uwblock *)uwblock; + for (i = 0; i < uword_block_tab->entry_num; i++) + page->uwblock[i].micro_words = + (uintptr_t)encap_uof_obj->beg_uof + uwblock[i].uword_offset; +} + +static int qat_uclo_map_uimage(struct icp_qat_uclo_objhandle *obj_handle, + struct icp_qat_uclo_encapme *ae_uimage, + int max_image) +{ + int i, j; + struct icp_qat_uof_chunkhdr *chunk_hdr = NULL; + struct icp_qat_uof_image *image; + struct icp_qat_uof_objtable *ae_regtab; + struct icp_qat_uof_objtable *init_reg_sym_tab; + struct icp_qat_uof_objtable *sbreak_tab; + struct icp_qat_uof_encap_obj *encap_uof_obj = + &obj_handle->encap_uof_obj; + + for (j = 0; j < max_image; j++) { + chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr, + ICP_QAT_UOF_IMAG, chunk_hdr); + if (!chunk_hdr) + break; + image = (struct icp_qat_uof_image *)(encap_uof_obj->beg_uof + + chunk_hdr->offset); + ae_regtab = (struct icp_qat_uof_objtable *) + (image->reg_tab_offset + + obj_handle->obj_hdr->file_buff); + ae_uimage[j].ae_reg_num = ae_regtab->entry_num; + ae_uimage[j].ae_reg = (struct icp_qat_uof_ae_reg *) + (((char *)ae_regtab) + + sizeof(struct icp_qat_uof_objtable)); + init_reg_sym_tab = (struct icp_qat_uof_objtable *) + (image->init_reg_sym_tab + + obj_handle->obj_hdr->file_buff); + ae_uimage[j].init_regsym_num = init_reg_sym_tab->entry_num; + ae_uimage[j].init_regsym = (struct icp_qat_uof_init_regsym *) + (((char *)init_reg_sym_tab) + + sizeof(struct icp_qat_uof_objtable)); + sbreak_tab = (struct icp_qat_uof_objtable *) + (image->sbreak_tab + obj_handle->obj_hdr->file_buff); + ae_uimage[j].sbreak_num = sbreak_tab->entry_num; + ae_uimage[j].sbreak = (struct icp_qat_uof_sbreak *) + (((char *)sbreak_tab) + + sizeof(struct icp_qat_uof_objtable)); + ae_uimage[j].img_ptr = image; + if (qat_uclo_check_image_compat(encap_uof_obj, image)) + goto out_err; + ae_uimage[j].page = + kzalloc_obj(struct icp_qat_uclo_encap_page); + if (!ae_uimage[j].page) + goto out_err; + qat_uclo_map_image_page(encap_uof_obj, image, + ae_uimage[j].page); + } + return j; +out_err: + for (i = 0; i < j; i++) + kfree(ae_uimage[i].page); + return 0; +} + +static int qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae) +{ + int i, ae; + int mflag = 0; + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned long cfg_ae_mask = handle->cfg_ae_mask; + + for_each_set_bit(ae, &ae_mask, max_ae) { + if (!test_bit(ae, &cfg_ae_mask)) + continue; + + for (i = 0; i < obj_handle->uimage_num; i++) { + unsigned long ae_assigned = obj_handle->ae_uimage[i].img_ptr->ae_assigned; + + if (!test_bit(ae, &ae_assigned)) + continue; + mflag = 1; + if (qat_uclo_init_ae_data(obj_handle, ae, i)) + return -EINVAL; + } + } + if (!mflag) { + pr_err("uimage uses AE not set\n"); + return -EINVAL; + } + return 0; +} + +static struct icp_qat_uof_strtable * +qat_uclo_map_str_table(struct icp_qat_uclo_objhdr *obj_hdr, + char *tab_name, struct icp_qat_uof_strtable *str_table) +{ + struct icp_qat_uof_chunkhdr *chunk_hdr; + + chunk_hdr = qat_uclo_find_chunk((struct icp_qat_uof_objhdr *) + obj_hdr->file_buff, tab_name, NULL); + if (chunk_hdr) { + int hdr_size; + + memcpy(&str_table->table_len, obj_hdr->file_buff + + chunk_hdr->offset, sizeof(str_table->table_len)); + hdr_size = (char *)&str_table->strings - (char *)str_table; + str_table->strings = (uintptr_t)obj_hdr->file_buff + + chunk_hdr->offset + hdr_size; + return str_table; + } + return NULL; +} + +static void +qat_uclo_map_initmem_table(struct icp_qat_uof_encap_obj *encap_uof_obj, + struct icp_qat_uclo_init_mem_table *init_mem_tab) +{ + struct icp_qat_uof_chunkhdr *chunk_hdr; + + chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr, + ICP_QAT_UOF_IMEM, NULL); + if (chunk_hdr) { + memmove(&init_mem_tab->entry_num, encap_uof_obj->beg_uof + + chunk_hdr->offset, sizeof(unsigned int)); + init_mem_tab->init_mem = (struct icp_qat_uof_initmem *) + (encap_uof_obj->beg_uof + chunk_hdr->offset + + sizeof(unsigned int)); + } +} + +static unsigned int +qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle) +{ + switch (handle->pci_dev->device) { + case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: + return ICP_QAT_AC_895XCC_DEV_TYPE; + case PCI_DEVICE_ID_INTEL_QAT_C62X: + return ICP_QAT_AC_C62X_DEV_TYPE; + case PCI_DEVICE_ID_INTEL_QAT_C3XXX: + return ICP_QAT_AC_C3XXX_DEV_TYPE; + case PCI_DEVICE_ID_INTEL_QAT_4XXX: + case PCI_DEVICE_ID_INTEL_QAT_401XX: + case PCI_DEVICE_ID_INTEL_QAT_402XX: + case PCI_DEVICE_ID_INTEL_QAT_420XX: + return ICP_QAT_AC_4XXX_A_DEV_TYPE; + case PCI_DEVICE_ID_INTEL_QAT_6XXX: + return ICP_QAT_AC_6XXX_DEV_TYPE; + default: + pr_err("unsupported device 0x%x\n", handle->pci_dev->device); + return 0; + } +} + +static int qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle *obj_handle) +{ + unsigned int maj_ver, prod_type = obj_handle->prod_type; + + if (!(prod_type & obj_handle->encap_uof_obj.obj_hdr->ac_dev_type)) { + pr_err("UOF type 0x%x doesn't match with platform 0x%x\n", + obj_handle->encap_uof_obj.obj_hdr->ac_dev_type, + prod_type); + return -EINVAL; + } + maj_ver = obj_handle->prod_rev & 0xff; + if (obj_handle->encap_uof_obj.obj_hdr->max_cpu_ver < maj_ver || + obj_handle->encap_uof_obj.obj_hdr->min_cpu_ver > maj_ver) { + pr_err("UOF majVer 0x%x out of range\n", maj_ver); + return -EINVAL; + } + return 0; +} + +static int qat_uclo_init_reg(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx_mask, + enum icp_qat_uof_regtype reg_type, + unsigned short reg_addr, unsigned int value) +{ + switch (reg_type) { + case ICP_GPA_ABS: + case ICP_GPB_ABS: + ctx_mask = 0; + fallthrough; + case ICP_GPA_REL: + case ICP_GPB_REL: + return qat_hal_init_gpr(handle, ae, ctx_mask, reg_type, + reg_addr, value); + case ICP_SR_ABS: + case ICP_DR_ABS: + case ICP_SR_RD_ABS: + case ICP_DR_RD_ABS: + ctx_mask = 0; + fallthrough; + case ICP_SR_REL: + case ICP_DR_REL: + case ICP_SR_RD_REL: + case ICP_DR_RD_REL: + return qat_hal_init_rd_xfer(handle, ae, ctx_mask, reg_type, + reg_addr, value); + case ICP_SR_WR_ABS: + case ICP_DR_WR_ABS: + ctx_mask = 0; + fallthrough; + case ICP_SR_WR_REL: + case ICP_DR_WR_REL: + return qat_hal_init_wr_xfer(handle, ae, ctx_mask, reg_type, + reg_addr, value); + case ICP_NEIGH_REL: + return qat_hal_init_nn(handle, ae, ctx_mask, reg_addr, value); + default: + pr_err("UOF uses not supported reg type 0x%x\n", reg_type); + return -EFAULT; + } + return 0; +} + +static int qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle, + unsigned int ae, + struct icp_qat_uclo_encapme *encap_ae) +{ + unsigned int i; + unsigned char ctx_mask; + struct icp_qat_uof_init_regsym *init_regsym; + + if (ICP_QAT_CTX_MODE(encap_ae->img_ptr->ae_mode) == + ICP_QAT_UCLO_MAX_CTX) + ctx_mask = 0xff; + else + ctx_mask = 0x55; + + for (i = 0; i < encap_ae->init_regsym_num; i++) { + unsigned int exp_res; + + init_regsym = &encap_ae->init_regsym[i]; + exp_res = init_regsym->value; + switch (init_regsym->init_type) { + case ICP_QAT_UOF_INIT_REG: + qat_uclo_init_reg(handle, ae, ctx_mask, + (enum icp_qat_uof_regtype) + init_regsym->reg_type, + (unsigned short)init_regsym->reg_addr, + exp_res); + break; + case ICP_QAT_UOF_INIT_REG_CTX: + /* check if ctx is appropriate for the ctxMode */ + if (!((1 << init_regsym->ctx) & ctx_mask)) { + pr_err("invalid ctx num = 0x%x\n", init_regsym->ctx); + return -EINVAL; + } + qat_uclo_init_reg(handle, ae, + (unsigned char) + (1 << init_regsym->ctx), + (enum icp_qat_uof_regtype) + init_regsym->reg_type, + (unsigned short)init_regsym->reg_addr, + exp_res); + break; + case ICP_QAT_UOF_INIT_EXPR: + pr_err("INIT_EXPR feature not supported\n"); + return -EINVAL; + case ICP_QAT_UOF_INIT_EXPR_ENDIAN_SWAP: + pr_err("INIT_EXPR_ENDIAN_SWAP feature not supported\n"); + return -EINVAL; + default: + break; + } + } + return 0; +} + +static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle) +{ + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned long ae_mask = handle->hal_handle->ae_mask; + struct icp_qat_uclo_aedata *aed; + unsigned int s, ae; + + if (obj_handle->global_inited) + return 0; + if (obj_handle->init_mem_tab.entry_num) { + if (qat_uclo_init_memory(handle)) { + pr_err("initialize memory failed\n"); + return -EINVAL; + } + } + + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + aed = &obj_handle->ae_data[ae]; + for (s = 0; s < aed->slice_num; s++) { + if (!aed->ae_slices[s].encap_image) + continue; + if (qat_uclo_init_reg_sym(handle, ae, aed->ae_slices[s].encap_image)) + return -EINVAL; + } + } + obj_handle->global_inited = 1; + return 0; +} + +static int qat_hal_set_modes(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_uclo_objhandle *obj_handle, + unsigned char ae, + struct icp_qat_uof_image *uof_image) +{ + unsigned char mode; + int ret; + + mode = ICP_QAT_CTX_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_ctx_mode(handle, ae, mode); + if (ret) { + pr_err("qat_hal_set_ae_ctx_mode error\n"); + return ret; + } + if (handle->chip_info->nn) { + mode = ICP_QAT_NN_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_nn_mode(handle, ae, mode); + if (ret) { + pr_err("qat_hal_set_ae_nn_mode error\n"); + return ret; + } + } + mode = ICP_QAT_LOC_MEM0_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, mode); + if (ret) { + pr_err("qat_hal_set_ae_lm_mode LMEM0 error\n"); + return ret; + } + mode = ICP_QAT_LOC_MEM1_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM1, mode); + if (ret) { + pr_err("qat_hal_set_ae_lm_mode LMEM1 error\n"); + return ret; + } + if (handle->chip_info->lm2lm3) { + mode = ICP_QAT_LOC_MEM2_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM2, mode); + if (ret) { + pr_err("qat_hal_set_ae_lm_mode LMEM2 error\n"); + return ret; + } + mode = ICP_QAT_LOC_MEM3_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM3, mode); + if (ret) { + pr_err("qat_hal_set_ae_lm_mode LMEM3 error\n"); + return ret; + } + mode = ICP_QAT_LOC_TINDEX_MODE(uof_image->ae_mode); + qat_hal_set_ae_tindex_mode(handle, ae, mode); + } + return 0; +} + +static int qat_uclo_set_ae_mode(struct icp_qat_fw_loader_handle *handle) +{ + struct icp_qat_uof_image *uof_image; + struct icp_qat_uclo_aedata *ae_data; + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned long cfg_ae_mask = handle->cfg_ae_mask; + unsigned char ae, s; + int error; + + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + if (!test_bit(ae, &cfg_ae_mask)) + continue; + + ae_data = &obj_handle->ae_data[ae]; + for (s = 0; s < min_t(unsigned int, ae_data->slice_num, + ICP_QAT_UCLO_MAX_CTX); s++) { + if (!obj_handle->ae_data[ae].ae_slices[s].encap_image) + continue; + uof_image = ae_data->ae_slices[s].encap_image->img_ptr; + error = qat_hal_set_modes(handle, obj_handle, ae, + uof_image); + if (error) + return error; + } + } + return 0; +} + +static void qat_uclo_init_uword_num(struct icp_qat_fw_loader_handle *handle) +{ + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + struct icp_qat_uclo_encapme *image; + int a; + + for (a = 0; a < obj_handle->uimage_num; a++) { + image = &obj_handle->ae_uimage[a]; + image->uwords_num = image->page->beg_addr_p + + image->page->micro_words_num; + } +} + +static int qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle) +{ + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned int ae; + + obj_handle->encap_uof_obj.beg_uof = obj_handle->obj_hdr->file_buff; + obj_handle->encap_uof_obj.obj_hdr = (struct icp_qat_uof_objhdr *) + obj_handle->obj_hdr->file_buff; + obj_handle->uword_in_bytes = 6; + obj_handle->prod_type = qat_uclo_get_dev_type(handle); + obj_handle->prod_rev = PID_MAJOR_REV | + (PID_MINOR_REV & handle->hal_handle->revision_id); + if (qat_uclo_check_uof_compat(obj_handle)) { + pr_err("UOF incompatible\n"); + return -EINVAL; + } + obj_handle->uword_buf = kcalloc(UWORD_CPYBUF_SIZE, sizeof(u64), + GFP_KERNEL); + if (!obj_handle->uword_buf) + return -ENOMEM; + obj_handle->ustore_phy_size = ICP_QAT_UCLO_MAX_USTORE; + if (!obj_handle->obj_hdr->file_buff || + !qat_uclo_map_str_table(obj_handle->obj_hdr, ICP_QAT_UOF_STRT, + &obj_handle->str_table)) { + pr_err("UOF doesn't have effective images\n"); + goto out_err; + } + obj_handle->uimage_num = + qat_uclo_map_uimage(obj_handle, obj_handle->ae_uimage, + ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX); + if (!obj_handle->uimage_num) + goto out_err; + if (qat_uclo_map_ae(handle, handle->hal_handle->ae_max_num)) { + pr_err("Bad object\n"); + goto out_check_uof_aemask_err; + } + qat_uclo_init_uword_num(handle); + qat_uclo_map_initmem_table(&obj_handle->encap_uof_obj, + &obj_handle->init_mem_tab); + if (qat_uclo_set_ae_mode(handle)) + goto out_check_uof_aemask_err; + return 0; +out_check_uof_aemask_err: + for (ae = 0; ae < obj_handle->uimage_num; ae++) + kfree(obj_handle->ae_uimage[ae].page); +out_err: + kfree(obj_handle->uword_buf); + return -EFAULT; +} + +static unsigned int qat_uclo_simg_hdr2sign_len(struct icp_qat_fw_loader_handle *handle) +{ + if (handle->chip_info->dual_sign) + return ICP_QAT_DUALSIGN_OPAQUE_DATA_LEN; + + return ICP_QAT_AE_IMG_OFFSET(handle); +} + +static unsigned int qat_uclo_simg_hdr2cont_len(struct icp_qat_fw_loader_handle *handle) +{ + if (handle->chip_info->dual_sign) + return ICP_QAT_DUALSIGN_OPAQUE_DATA_LEN + ICP_QAT_DUALSIGN_MISC_INFO_LEN; + + return ICP_QAT_AE_IMG_OFFSET(handle); +} + +static unsigned int qat_uclo_simg_fw_type(struct icp_qat_fw_loader_handle *handle, void *img_ptr) +{ + struct icp_qat_css_hdr *hdr = img_ptr; + char *fw_hdr = img_ptr; + unsigned int offset; + + if (handle->chip_info->dual_sign) { + offset = qat_uclo_simg_hdr2sign_len(handle) + ICP_QAT_DUALSIGN_FW_TYPE_LEN; + return *(fw_hdr + offset); + } + + return hdr->fw_type; +} + +static int qat_uclo_map_suof_file_hdr(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_suof_filehdr *suof_ptr, + int suof_size) +{ + unsigned int check_sum = 0; + unsigned int min_ver_offset = 0; + struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; + + suof_handle->file_id = ICP_QAT_SUOF_FID; + suof_handle->suof_buf = (char *)suof_ptr; + suof_handle->suof_size = suof_size; + min_ver_offset = suof_size - offsetof(struct icp_qat_suof_filehdr, + min_ver); + check_sum = qat_uclo_calc_str_checksum((char *)&suof_ptr->min_ver, + min_ver_offset); + if (check_sum != suof_ptr->check_sum) { + pr_err("incorrect SUOF checksum\n"); + return -EINVAL; + } + suof_handle->check_sum = suof_ptr->check_sum; + suof_handle->min_ver = suof_ptr->min_ver; + suof_handle->maj_ver = suof_ptr->maj_ver; + suof_handle->fw_type = suof_ptr->fw_type; + return 0; +} + +static void qat_uclo_map_simg(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_suof_img_hdr *suof_img_hdr, + struct icp_qat_suof_chunk_hdr *suof_chunk_hdr) +{ + struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; + unsigned int offset = qat_uclo_simg_hdr2cont_len(handle); + struct icp_qat_suof_objhdr *suof_objhdr; + struct icp_qat_simg_ae_mode *ae_mode; + + suof_img_hdr->simg_buf = (suof_handle->suof_buf + + suof_chunk_hdr->offset + + sizeof(*suof_objhdr)); + suof_img_hdr->simg_len = ((struct icp_qat_suof_objhdr *)(uintptr_t) + (suof_handle->suof_buf + + suof_chunk_hdr->offset))->img_length; + + suof_img_hdr->css_header = suof_img_hdr->simg_buf; + suof_img_hdr->css_simg = suof_img_hdr->css_header + offset; + + ae_mode = (struct icp_qat_simg_ae_mode *)(suof_img_hdr->css_simg); + suof_img_hdr->ae_mask = ae_mode->ae_mask; + suof_img_hdr->simg_name = (unsigned long)&ae_mode->simg_name; + suof_img_hdr->appmeta_data = (unsigned long)&ae_mode->appmeta_data; + suof_img_hdr->fw_type = ae_mode->fw_type; +} + +static void +qat_uclo_map_suof_symobjs(struct icp_qat_suof_handle *suof_handle, + struct icp_qat_suof_chunk_hdr *suof_chunk_hdr) +{ + char **sym_str = (char **)&suof_handle->sym_str; + unsigned int *sym_size = &suof_handle->sym_size; + struct icp_qat_suof_strtable *str_table_obj; + + *sym_size = *(unsigned int *)(uintptr_t) + (suof_chunk_hdr->offset + suof_handle->suof_buf); + *sym_str = (char *)(uintptr_t) + (suof_handle->suof_buf + suof_chunk_hdr->offset + + sizeof(str_table_obj->tab_length)); +} + +static int qat_uclo_check_simg_compat(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_suof_img_hdr *img_hdr) +{ + struct icp_qat_simg_ae_mode *img_ae_mode = NULL; + unsigned int prod_rev, maj_ver, prod_type; + + prod_type = qat_uclo_get_dev_type(handle); + img_ae_mode = (struct icp_qat_simg_ae_mode *)img_hdr->css_simg; + prod_rev = PID_MAJOR_REV | + (PID_MINOR_REV & handle->hal_handle->revision_id); + if (img_ae_mode->dev_type != prod_type) { + pr_err("incompatible product type %x\n", img_ae_mode->dev_type); + return -EINVAL; + } + maj_ver = prod_rev & 0xff; + if (maj_ver > img_ae_mode->devmax_ver || + maj_ver < img_ae_mode->devmin_ver) { + pr_err("incompatible device majver 0x%x\n", maj_ver); + return -EINVAL; + } + return 0; +} + +static void qat_uclo_del_suof(struct icp_qat_fw_loader_handle *handle) +{ + struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle; + + kfree(sobj_handle->img_table.simg_hdr); + sobj_handle->img_table.simg_hdr = NULL; + kfree(handle->sobj_handle); + handle->sobj_handle = NULL; +} + +static void qat_uclo_tail_img(struct icp_qat_suof_img_hdr *suof_img_hdr, + unsigned int img_id, unsigned int num_simgs) +{ + struct icp_qat_suof_img_hdr img_header; + + if (img_id != num_simgs - 1) { + memcpy(&img_header, &suof_img_hdr[num_simgs - 1], + sizeof(*suof_img_hdr)); + memcpy(&suof_img_hdr[num_simgs - 1], &suof_img_hdr[img_id], + sizeof(*suof_img_hdr)); + memcpy(&suof_img_hdr[img_id], &img_header, + sizeof(*suof_img_hdr)); + } +} + +static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_suof_filehdr *suof_ptr, + int suof_size) +{ + struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; + struct icp_qat_suof_chunk_hdr *suof_chunk_hdr = NULL; + struct icp_qat_suof_img_hdr *suof_img_hdr = NULL; + int ret = 0, ae0_img = ICP_QAT_UCLO_MAX_AE; + unsigned int i = 0; + struct icp_qat_suof_img_hdr img_header; + + if (!suof_ptr || suof_size == 0) { + pr_err("input parameter SUOF pointer/size is NULL\n"); + return -EINVAL; + } + if (qat_uclo_check_suof_format(suof_ptr)) + return -EINVAL; + ret = qat_uclo_map_suof_file_hdr(handle, suof_ptr, suof_size); + if (ret) + return ret; + suof_chunk_hdr = (struct icp_qat_suof_chunk_hdr *) + ((uintptr_t)suof_ptr + sizeof(*suof_ptr)); + + qat_uclo_map_suof_symobjs(suof_handle, suof_chunk_hdr); + suof_handle->img_table.num_simgs = suof_ptr->num_chunks - 1; + + if (suof_handle->img_table.num_simgs != 0) { + suof_img_hdr = kzalloc_objs(img_header, + suof_handle->img_table.num_simgs); + if (!suof_img_hdr) + return -ENOMEM; + suof_handle->img_table.simg_hdr = suof_img_hdr; + + for (i = 0; i < suof_handle->img_table.num_simgs; i++) { + qat_uclo_map_simg(handle, &suof_img_hdr[i], + &suof_chunk_hdr[1 + i]); + ret = qat_uclo_check_simg_compat(handle, + &suof_img_hdr[i]); + if (ret) + return ret; + suof_img_hdr[i].ae_mask &= handle->cfg_ae_mask; + if ((suof_img_hdr[i].ae_mask & 0x1) != 0) + ae0_img = i; + } + + if (!handle->chip_info->tgroup_share_ustore) { + qat_uclo_tail_img(suof_img_hdr, ae0_img, + suof_handle->img_table.num_simgs); + } + } + return 0; +} + +#define ADD_ADDR(high, low) ((((u64)high) << 32) + low) + +static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_fw_auth_desc *desc) +{ + unsigned int retries = FW_AUTH_MAX_RETRY; + u32 fcu_ctl_csr, fcu_sts_csr; + u32 fcu_dram_hi_csr, fcu_dram_lo_csr; + u64 bus_addr; + u32 fcu_sts; + + bus_addr = ADD_ADDR(desc->css_hdr_high, desc->css_hdr_low) + - sizeof(struct icp_qat_auth_chunk); + + fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; + fcu_sts_csr = handle->chip_info->fcu_sts_csr; + fcu_dram_hi_csr = handle->chip_info->fcu_dram_addr_hi; + fcu_dram_lo_csr = handle->chip_info->fcu_dram_addr_lo; + + SET_CAP_CSR(handle, fcu_dram_hi_csr, bus_addr >> BITS_PER_TYPE(u32)); + SET_CAP_CSR(handle, fcu_dram_lo_csr, bus_addr); + SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_AUTH); + + do { + int arb_ret; + + msleep(FW_AUTH_WAIT_PERIOD); + fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); + + arb_ret = adf_anti_rb_check(handle->pci_dev); + if (arb_ret == -EAGAIN) { + if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_FAIL) { + SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_AUTH); + continue; + } + } else if (arb_ret) { + goto auth_fail; + } + + if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_FAIL) + goto auth_fail; + + if (((fcu_sts >> FCU_STS_AUTHFWLD_POS) & 0x1)) + if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_DONE) + return 0; + } while (--retries); + +auth_fail: + pr_err("authentication error (FCU_STATUS = 0x%x)\n", fcu_sts & FCU_AUTH_STS_MASK); + + return -EINVAL; +} + +static bool qat_uclo_is_broadcast(struct icp_qat_fw_loader_handle *handle, + int imgid) +{ + struct icp_qat_suof_handle *sobj_handle; + + if (!handle->chip_info->tgroup_share_ustore) + return false; + + sobj_handle = (struct icp_qat_suof_handle *)handle->sobj_handle; + if (handle->hal_handle->admin_ae_mask & + sobj_handle->img_table.simg_hdr[imgid].ae_mask) + return false; + + return true; +} + +static int qat_uclo_broadcast_load_fw(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_fw_auth_desc *desc) +{ + unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned long desc_ae_mask = desc->ae_mask; + u32 fcu_sts, ae_broadcast_mask = 0; + u32 fcu_loaded_csr, ae_loaded; + u32 fcu_sts_csr, fcu_ctl_csr; + unsigned int ae, retry = 0; + + if (handle->chip_info->tgroup_share_ustore) { + fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; + fcu_sts_csr = handle->chip_info->fcu_sts_csr; + fcu_loaded_csr = handle->chip_info->fcu_loaded_ae_csr; + } else { + pr_err("Chip 0x%x doesn't support broadcast load\n", handle->pci_dev->device); + return -EINVAL; + } + + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + if (qat_hal_check_ae_active(handle, (unsigned char)ae)) { + pr_err("Broadcast load failed. AE is not enabled or active.\n"); + return -EINVAL; + } + + if (test_bit(ae, &desc_ae_mask)) + ae_broadcast_mask |= 1 << ae; + } + + if (ae_broadcast_mask) { + SET_CAP_CSR(handle, FCU_ME_BROADCAST_MASK_TYPE, + ae_broadcast_mask); + + SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_LOAD); + + do { + msleep(FW_AUTH_WAIT_PERIOD); + fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); + fcu_sts &= FCU_AUTH_STS_MASK; + + if (fcu_sts == FCU_STS_LOAD_FAIL) { + pr_err("Broadcast load failed: 0x%x)\n", fcu_sts); + return -EINVAL; + } else if (fcu_sts == FCU_STS_LOAD_DONE) { + ae_loaded = GET_CAP_CSR(handle, fcu_loaded_csr); + ae_loaded >>= handle->chip_info->fcu_loaded_ae_pos; + + if ((ae_loaded & ae_broadcast_mask) == ae_broadcast_mask) + break; + } + } while (retry++ < FW_AUTH_MAX_RETRY); + + if (retry > FW_AUTH_MAX_RETRY) { + pr_err("broadcast load failed timeout %d\n", retry); + return -EINVAL; + } + } + return 0; +} + +static int qat_uclo_simg_alloc(struct icp_qat_fw_loader_handle *handle, + struct icp_firml_dram_desc *dram_desc, + unsigned int size) +{ + void *vptr; + dma_addr_t ptr; + + vptr = dma_alloc_coherent(&handle->pci_dev->dev, + size, &ptr, GFP_KERNEL); + if (!vptr) + return -ENOMEM; + dram_desc->dram_base_addr_v = vptr; + dram_desc->dram_bus_addr = ptr; + dram_desc->dram_size = size; + return 0; +} + +static void qat_uclo_simg_free(struct icp_qat_fw_loader_handle *handle, + struct icp_firml_dram_desc *dram_desc) +{ + if (handle && dram_desc && dram_desc->dram_base_addr_v) { + dma_free_coherent(&handle->pci_dev->dev, + (size_t)(dram_desc->dram_size), + dram_desc->dram_base_addr_v, + dram_desc->dram_bus_addr); + } + + if (dram_desc) + memset(dram_desc, 0, sizeof(*dram_desc)); +} + +static void qat_uclo_ummap_auth_fw(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_fw_auth_desc **desc) +{ + struct icp_firml_dram_desc dram_desc; + + if (*desc) { + dram_desc.dram_base_addr_v = *desc; + dram_desc.dram_bus_addr = ((struct icp_qat_auth_chunk *) + (*desc))->chunk_bus_addr; + dram_desc.dram_size = ((struct icp_qat_auth_chunk *) + (*desc))->chunk_size; + qat_uclo_simg_free(handle, &dram_desc); + } +} + +static int qat_uclo_check_image(struct icp_qat_fw_loader_handle *handle, + void *image, unsigned int size, + unsigned int fw_type) +{ + char *fw_type_name = fw_type ? "MMP" : "AE"; + unsigned int css_dword_size = sizeof(u32); + unsigned int header_len, simg_type; + struct icp_qat_css_hdr *css_hdr; + + if (handle->chip_info->fw_auth) { + header_len = qat_uclo_simg_hdr2sign_len(handle); + simg_type = qat_uclo_simg_fw_type(handle, image); + css_hdr = image; + + if (handle->chip_info->dual_sign) { + if (css_hdr->module_type != ICP_QAT_DUALSIGN_MODULE_TYPE) + goto err; + if (css_hdr->header_len != ICP_QAT_DUALSIGN_HDR_LEN) + goto err; + if (css_hdr->header_ver != ICP_QAT_DUALSIGN_HDR_VER) + goto err; + } else { + if (css_hdr->header_len * css_dword_size != header_len) + goto err; + if (css_hdr->size * css_dword_size != size) + goto err; + if (size <= header_len) + goto err; + } + + if (fw_type != simg_type) + goto err; + + size -= header_len; + } + + if (fw_type == CSS_AE_FIRMWARE) { + if (size < sizeof(struct icp_qat_simg_ae_mode *) + + ICP_QAT_SIMG_AE_INIT_SEQ_LEN) + goto err; + if (size > ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN) + goto err; + } else if (fw_type == CSS_MMP_FIRMWARE) { + if (size > ICP_QAT_CSS_RSA3K_MAX_IMAGE_LEN) + goto err; + } else { + pr_err("Unsupported firmware type\n"); + return -EINVAL; + } + return 0; + +err: + pr_err("Invalid %s firmware image\n", fw_type_name); + return -EINVAL; +} + +static int qat_uclo_build_auth_desc_RSA(struct icp_qat_fw_loader_handle *handle, + char *image, unsigned int size, + struct icp_firml_dram_desc *dram_desc, + unsigned int fw_type, struct icp_qat_fw_auth_desc **desc) +{ + struct icp_qat_css_hdr *css_hdr = (struct icp_qat_css_hdr *)image; + struct icp_qat_simg_ae_mode *simg_ae_mode; + struct icp_qat_fw_auth_desc *auth_desc; + char *virt_addr, *virt_base; + u64 bus_addr; + + virt_base = dram_desc->dram_base_addr_v; + virt_base += sizeof(struct icp_qat_auth_chunk); + bus_addr = dram_desc->dram_bus_addr + sizeof(struct icp_qat_auth_chunk); + auth_desc = dram_desc->dram_base_addr_v; + auth_desc->css_hdr_high = upper_32_bits(bus_addr); + auth_desc->css_hdr_low = lower_32_bits(bus_addr); + virt_addr = virt_base; + + memcpy(virt_addr, image, sizeof(*css_hdr)); + /* pub key */ + bus_addr = ADD_ADDR(auth_desc->css_hdr_high, auth_desc->css_hdr_low) + + sizeof(*css_hdr); + virt_addr = virt_addr + sizeof(*css_hdr); + + auth_desc->fwsk_pub_high = upper_32_bits(bus_addr); + auth_desc->fwsk_pub_low = lower_32_bits(bus_addr); + + memcpy(virt_addr, image + sizeof(*css_hdr), ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)); + /* padding */ + memset((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)), + 0, ICP_QAT_CSS_FWSK_PAD_LEN(handle)); + + /* exponent */ + memcpy(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + + ICP_QAT_CSS_FWSK_PAD_LEN(handle), image + sizeof(*css_hdr) + + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle), sizeof(unsigned int)); + + /* signature */ + bus_addr = ADD_ADDR(auth_desc->fwsk_pub_high, + auth_desc->fwsk_pub_low) + + ICP_QAT_CSS_FWSK_PUB_LEN(handle); + virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN(handle); + auth_desc->signature_high = upper_32_bits(bus_addr); + auth_desc->signature_low = lower_32_bits(bus_addr); + + memcpy(virt_addr, image + sizeof(*css_hdr) + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + + ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle), ICP_QAT_CSS_SIGNATURE_LEN(handle)); + + bus_addr = ADD_ADDR(auth_desc->signature_high, + auth_desc->signature_low) + + ICP_QAT_CSS_SIGNATURE_LEN(handle); + virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle); + + auth_desc->img_high = upper_32_bits(bus_addr); + auth_desc->img_low = lower_32_bits(bus_addr); + auth_desc->img_len = size - qat_uclo_simg_hdr2sign_len(handle); + if (bus_addr + auth_desc->img_len > + dram_desc->dram_bus_addr + ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN) { + pr_err("insufficient memory size for authentication data\n"); + qat_uclo_simg_free(handle, dram_desc); + return -ENOMEM; + } + + memcpy(virt_addr, image + qat_uclo_simg_hdr2sign_len(handle), auth_desc->img_len); + virt_addr = virt_base; + /* AE firmware */ + if (fw_type == CSS_AE_FIRMWARE) { + auth_desc->img_ae_mode_data_high = auth_desc->img_high; + auth_desc->img_ae_mode_data_low = auth_desc->img_low; + bus_addr = ADD_ADDR(auth_desc->img_ae_mode_data_high, + auth_desc->img_ae_mode_data_low) + + sizeof(struct icp_qat_simg_ae_mode); + + auth_desc->img_ae_init_data_high = upper_32_bits(bus_addr); + auth_desc->img_ae_init_data_low = lower_32_bits(bus_addr); + bus_addr += ICP_QAT_SIMG_AE_INIT_SEQ_LEN; + auth_desc->img_ae_insts_high = upper_32_bits(bus_addr); + auth_desc->img_ae_insts_low = lower_32_bits(bus_addr); + virt_addr += sizeof(struct icp_qat_css_hdr); + virt_addr += ICP_QAT_CSS_FWSK_PUB_LEN(handle); + virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle); + simg_ae_mode = (struct icp_qat_simg_ae_mode *)(uintptr_t)virt_addr; + auth_desc->ae_mask = simg_ae_mode->ae_mask & handle->cfg_ae_mask; + } else { + auth_desc->img_ae_insts_high = auth_desc->img_high; + auth_desc->img_ae_insts_low = auth_desc->img_low; + } + *desc = auth_desc; + return 0; +} + +static int qat_uclo_build_auth_desc_dualsign(struct icp_qat_fw_loader_handle *handle, + char *image, unsigned int size, + struct icp_firml_dram_desc *dram_desc, + unsigned int fw_type, + struct icp_qat_fw_auth_desc **desc) +{ + struct icp_qat_simg_ae_mode *simg_ae_mode; + struct icp_qat_fw_auth_desc *auth_desc; + unsigned int chunk_offset, img_offset; + u64 bus_addr, addr; + char *virt_addr; + + virt_addr = dram_desc->dram_base_addr_v; + virt_addr += sizeof(struct icp_qat_auth_chunk); + bus_addr = dram_desc->dram_bus_addr + sizeof(struct icp_qat_auth_chunk); + + auth_desc = dram_desc->dram_base_addr_v; + auth_desc->img_len = size - qat_uclo_simg_hdr2sign_len(handle); + auth_desc->css_hdr_high = upper_32_bits(bus_addr); + auth_desc->css_hdr_low = lower_32_bits(bus_addr); + memcpy(virt_addr, image, ICP_QAT_DUALSIGN_OPAQUE_HDR_LEN); + + img_offset = ICP_QAT_DUALSIGN_OPAQUE_HDR_LEN; + chunk_offset = ICP_QAT_DUALSIGN_OPAQUE_HDR_ALIGN_LEN; + + /* RSA pub key */ + addr = bus_addr + chunk_offset; + auth_desc->fwsk_pub_high = upper_32_bits(addr); + auth_desc->fwsk_pub_low = lower_32_bits(addr); + memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)); + + img_offset += ICP_QAT_CSS_FWSK_MODULUS_LEN(handle); + chunk_offset += ICP_QAT_CSS_FWSK_MODULUS_LEN(handle); + /* RSA padding */ + memset(virt_addr + chunk_offset, 0, ICP_QAT_CSS_FWSK_PAD_LEN(handle)); + + chunk_offset += ICP_QAT_CSS_FWSK_PAD_LEN(handle); + /* RSA exponent */ + memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle)); + + img_offset += ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle); + chunk_offset += ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle); + /* RSA signature */ + addr = bus_addr + chunk_offset; + auth_desc->signature_high = upper_32_bits(addr); + auth_desc->signature_low = lower_32_bits(addr); + memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_CSS_SIGNATURE_LEN(handle)); + + img_offset += ICP_QAT_CSS_SIGNATURE_LEN(handle); + chunk_offset += ICP_QAT_CSS_SIGNATURE_LEN(handle); + /* XMSS pubkey */ + addr = bus_addr + chunk_offset; + auth_desc->xmss_pubkey_high = upper_32_bits(addr); + auth_desc->xmss_pubkey_low = lower_32_bits(addr); + memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_DUALSIGN_XMSS_PUBKEY_LEN); + + img_offset += ICP_QAT_DUALSIGN_XMSS_PUBKEY_LEN; + chunk_offset += ICP_QAT_DUALSIGN_XMSS_PUBKEY_LEN; + /* XMSS signature */ + addr = bus_addr + chunk_offset; + auth_desc->xmss_sig_high = upper_32_bits(addr); + auth_desc->xmss_sig_low = lower_32_bits(addr); + memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_DUALSIGN_XMSS_SIG_LEN); + + img_offset += ICP_QAT_DUALSIGN_XMSS_SIG_LEN; + chunk_offset += ICP_QAT_DUALSIGN_XMSS_SIG_ALIGN_LEN; + + if (dram_desc->dram_size < (chunk_offset + auth_desc->img_len)) { + pr_err("auth chunk memory size is not enough to store data\n"); + return -ENOMEM; + } + + /* Signed data */ + addr = bus_addr + chunk_offset; + auth_desc->img_high = upper_32_bits(addr); + auth_desc->img_low = lower_32_bits(addr); + memcpy(virt_addr + chunk_offset, image + img_offset, auth_desc->img_len); + + chunk_offset += ICP_QAT_DUALSIGN_MISC_INFO_LEN; + /* AE firmware */ + if (fw_type == CSS_AE_FIRMWARE) { + /* AE mode data */ + addr = bus_addr + chunk_offset; + auth_desc->img_ae_mode_data_high = upper_32_bits(addr); + auth_desc->img_ae_mode_data_low = lower_32_bits(addr); + simg_ae_mode = + (struct icp_qat_simg_ae_mode *)(virt_addr + chunk_offset); + auth_desc->ae_mask = simg_ae_mode->ae_mask & handle->cfg_ae_mask; + + chunk_offset += sizeof(struct icp_qat_simg_ae_mode); + /* AE init seq */ + addr = bus_addr + chunk_offset; + auth_desc->img_ae_init_data_high = upper_32_bits(addr); + auth_desc->img_ae_init_data_low = lower_32_bits(addr); + + chunk_offset += ICP_QAT_SIMG_AE_INIT_SEQ_LEN; + /* AE instructions */ + addr = bus_addr + chunk_offset; + auth_desc->img_ae_insts_high = upper_32_bits(addr); + auth_desc->img_ae_insts_low = lower_32_bits(addr); + } else { + addr = bus_addr + chunk_offset; + auth_desc->img_ae_insts_high = upper_32_bits(addr); + auth_desc->img_ae_insts_low = lower_32_bits(addr); + } + *desc = auth_desc; + return 0; +} + +static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, + char *image, unsigned int size, + struct icp_qat_fw_auth_desc **desc) +{ + struct icp_qat_auth_chunk *auth_chunk; + struct icp_firml_dram_desc img_desc; + unsigned int simg_fw_type; + int ret; + + ret = qat_uclo_simg_alloc(handle, &img_desc, ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN); + if (ret) + return ret; + + simg_fw_type = qat_uclo_simg_fw_type(handle, image); + auth_chunk = img_desc.dram_base_addr_v; + auth_chunk->chunk_size = img_desc.dram_size; + auth_chunk->chunk_bus_addr = img_desc.dram_bus_addr; + + if (handle->chip_info->dual_sign) + return qat_uclo_build_auth_desc_dualsign(handle, image, size, &img_desc, + simg_fw_type, desc); + + return qat_uclo_build_auth_desc_RSA(handle, image, size, &img_desc, + simg_fw_type, desc); +} + +static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_fw_auth_desc *desc) +{ + unsigned long ae_mask = handle->hal_handle->ae_mask; + u32 fcu_sts_csr, fcu_ctl_csr; + u32 loaded_aes, loaded_csr; + unsigned int i; + u32 fcu_sts; + + fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; + fcu_sts_csr = handle->chip_info->fcu_sts_csr; + loaded_csr = handle->chip_info->fcu_loaded_ae_csr; + + for_each_set_bit(i, &ae_mask, handle->hal_handle->ae_max_num) { + int retry = 0; + + if (!((desc->ae_mask >> i) & 0x1)) + continue; + if (qat_hal_check_ae_active(handle, i)) { + pr_err("AE %d is active\n", i); + return -EINVAL; + } + SET_CAP_CSR(handle, fcu_ctl_csr, + (FCU_CTRL_CMD_LOAD | + (1 << FCU_CTRL_BROADCAST_POS) | + (i << FCU_CTRL_AE_POS))); + + do { + msleep(FW_AUTH_WAIT_PERIOD); + fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); + if ((fcu_sts & FCU_AUTH_STS_MASK) == + FCU_STS_LOAD_DONE) { + loaded_aes = GET_CAP_CSR(handle, loaded_csr); + loaded_aes >>= handle->chip_info->fcu_loaded_ae_pos; + if (loaded_aes & (1 << i)) + break; + } + } while (retry++ < FW_AUTH_MAX_RETRY); + if (retry > FW_AUTH_MAX_RETRY) { + pr_err("firmware load failed timeout %x\n", retry); + return -EINVAL; + } + } + return 0; +} + +static int qat_uclo_map_suof_obj(struct icp_qat_fw_loader_handle *handle, + void *addr_ptr, int mem_size) +{ + struct icp_qat_suof_handle *suof_handle; + + suof_handle = kzalloc_obj(*suof_handle); + if (!suof_handle) + return -ENOMEM; + handle->sobj_handle = suof_handle; + if (qat_uclo_map_suof(handle, addr_ptr, mem_size)) { + qat_uclo_del_suof(handle); + pr_err("map SUOF failed\n"); + return -EINVAL; + } + return 0; +} + +int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, + void *addr_ptr, int mem_size) +{ + struct icp_qat_fw_auth_desc *desc = NULL; + int status = 0; + int ret; + + ret = qat_uclo_check_image(handle, addr_ptr, mem_size, CSS_MMP_FIRMWARE); + if (ret) + return ret; + + if (handle->chip_info->fw_auth) { + status = qat_uclo_map_auth_fw(handle, addr_ptr, mem_size, &desc); + if (!status) + status = qat_uclo_auth_fw(handle, desc); + qat_uclo_ummap_auth_fw(handle, &desc); + } else { + if (handle->chip_info->mmp_sram_size < mem_size) { + pr_err("MMP size is too large: 0x%x\n", mem_size); + return -EFBIG; + } + qat_uclo_wr_sram_by_words(handle, 0, addr_ptr, mem_size); + } + return status; +} + +static int qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle, + void *addr_ptr, int mem_size) +{ + struct icp_qat_uof_filehdr *filehdr; + struct icp_qat_uclo_objhandle *objhdl; + + objhdl = kzalloc_obj(*objhdl); + if (!objhdl) + return -ENOMEM; + objhdl->obj_buf = kmemdup(addr_ptr, mem_size, GFP_KERNEL); + if (!objhdl->obj_buf) + goto out_objbuf_err; + filehdr = (struct icp_qat_uof_filehdr *)objhdl->obj_buf; + if (qat_uclo_check_uof_format(filehdr)) + goto out_objhdr_err; + objhdl->obj_hdr = qat_uclo_map_chunk((char *)objhdl->obj_buf, filehdr, + ICP_QAT_UOF_OBJS); + if (!objhdl->obj_hdr) { + pr_err("object file chunk is null\n"); + goto out_objhdr_err; + } + handle->obj_handle = objhdl; + if (qat_uclo_parse_uof_obj(handle)) + goto out_overlay_obj_err; + return 0; + +out_overlay_obj_err: + handle->obj_handle = NULL; + kfree(objhdl->obj_hdr); +out_objhdr_err: + kfree(objhdl->obj_buf); +out_objbuf_err: + kfree(objhdl); + return -ENOMEM; +} + +static int qat_uclo_map_mof_file_hdr(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_mof_file_hdr *mof_ptr, + u32 mof_size) +{ + struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle; + unsigned int min_ver_offset; + unsigned int checksum; + + mobj_handle->file_id = ICP_QAT_MOF_FID; + mobj_handle->mof_buf = (char *)mof_ptr; + mobj_handle->mof_size = mof_size; + + min_ver_offset = mof_size - offsetof(struct icp_qat_mof_file_hdr, + min_ver); + checksum = qat_uclo_calc_str_checksum(&mof_ptr->min_ver, + min_ver_offset); + if (checksum != mof_ptr->checksum) { + pr_err("incorrect MOF checksum\n"); + return -EINVAL; + } + + mobj_handle->checksum = mof_ptr->checksum; + mobj_handle->min_ver = mof_ptr->min_ver; + mobj_handle->maj_ver = mof_ptr->maj_ver; + return 0; +} + +static void qat_uclo_del_mof(struct icp_qat_fw_loader_handle *handle) +{ + struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle; + + kfree(mobj_handle->obj_table.obj_hdr); + mobj_handle->obj_table.obj_hdr = NULL; + kfree(handle->mobj_handle); + handle->mobj_handle = NULL; +} + +static int qat_uclo_seek_obj_inside_mof(struct icp_qat_mof_handle *mobj_handle, + const char *obj_name, char **obj_ptr, + unsigned int *obj_size) +{ + struct icp_qat_mof_objhdr *obj_hdr = mobj_handle->obj_table.obj_hdr; + unsigned int i; + + for (i = 0; i < mobj_handle->obj_table.num_objs; i++) { + if (!strncmp(obj_hdr[i].obj_name, obj_name, + ICP_QAT_SUOF_OBJ_NAME_LEN)) { + *obj_ptr = obj_hdr[i].obj_buf; + *obj_size = obj_hdr[i].obj_size; + return 0; + } + } + + pr_err("object %s is not found inside MOF\n", obj_name); + return -EINVAL; +} + +static int qat_uclo_map_obj_from_mof(struct icp_qat_mof_handle *mobj_handle, + struct icp_qat_mof_objhdr *mobj_hdr, + struct icp_qat_mof_obj_chunkhdr *obj_chunkhdr) +{ + u8 *obj; + + if (!strncmp(obj_chunkhdr->chunk_id, ICP_QAT_UOF_IMAG, + ICP_QAT_MOF_OBJ_CHUNKID_LEN)) { + obj = mobj_handle->uobjs_hdr + obj_chunkhdr->offset; + } else if (!strncmp(obj_chunkhdr->chunk_id, ICP_QAT_SUOF_IMAG, + ICP_QAT_MOF_OBJ_CHUNKID_LEN)) { + obj = mobj_handle->sobjs_hdr + obj_chunkhdr->offset; + } else { + pr_err("unsupported chunk id\n"); + return -EINVAL; + } + mobj_hdr->obj_buf = obj; + mobj_hdr->obj_size = (unsigned int)obj_chunkhdr->size; + mobj_hdr->obj_name = obj_chunkhdr->name + mobj_handle->sym_str; + return 0; +} + +static int qat_uclo_map_objs_from_mof(struct icp_qat_mof_handle *mobj_handle) +{ + struct icp_qat_mof_obj_chunkhdr *uobj_chunkhdr; + struct icp_qat_mof_obj_chunkhdr *sobj_chunkhdr; + struct icp_qat_mof_obj_hdr *uobj_hdr; + struct icp_qat_mof_obj_hdr *sobj_hdr; + struct icp_qat_mof_objhdr *mobj_hdr; + unsigned int uobj_chunk_num = 0; + unsigned int sobj_chunk_num = 0; + unsigned int *valid_chunk; + int ret, i; + + uobj_hdr = (struct icp_qat_mof_obj_hdr *)mobj_handle->uobjs_hdr; + sobj_hdr = (struct icp_qat_mof_obj_hdr *)mobj_handle->sobjs_hdr; + if (uobj_hdr) + uobj_chunk_num = uobj_hdr->num_chunks; + if (sobj_hdr) + sobj_chunk_num = sobj_hdr->num_chunks; + + mobj_hdr = kzalloc_objs(*mobj_hdr, + size_add(uobj_chunk_num, sobj_chunk_num)); + if (!mobj_hdr) + return -ENOMEM; + + mobj_handle->obj_table.obj_hdr = mobj_hdr; + valid_chunk = &mobj_handle->obj_table.num_objs; + uobj_chunkhdr = (struct icp_qat_mof_obj_chunkhdr *) + ((uintptr_t)uobj_hdr + sizeof(*uobj_hdr)); + sobj_chunkhdr = (struct icp_qat_mof_obj_chunkhdr *) + ((uintptr_t)sobj_hdr + sizeof(*sobj_hdr)); + + /* map uof objects */ + for (i = 0; i < uobj_chunk_num; i++) { + ret = qat_uclo_map_obj_from_mof(mobj_handle, + &mobj_hdr[*valid_chunk], + &uobj_chunkhdr[i]); + if (ret) + return ret; + (*valid_chunk)++; + } + + /* map suof objects */ + for (i = 0; i < sobj_chunk_num; i++) { + ret = qat_uclo_map_obj_from_mof(mobj_handle, + &mobj_hdr[*valid_chunk], + &sobj_chunkhdr[i]); + if (ret) + return ret; + (*valid_chunk)++; + } + + if ((uobj_chunk_num + sobj_chunk_num) != *valid_chunk) { + pr_err("inconsistent UOF/SUOF chunk amount\n"); + return -EINVAL; + } + return 0; +} + +static void qat_uclo_map_mof_symobjs(struct icp_qat_mof_handle *mobj_handle, + struct icp_qat_mof_chunkhdr *mof_chunkhdr) +{ + char **sym_str = (char **)&mobj_handle->sym_str; + unsigned int *sym_size = &mobj_handle->sym_size; + struct icp_qat_mof_str_table *str_table_obj; + + *sym_size = *(unsigned int *)(uintptr_t) + (mof_chunkhdr->offset + mobj_handle->mof_buf); + *sym_str = (char *)(uintptr_t) + (mobj_handle->mof_buf + mof_chunkhdr->offset + + sizeof(str_table_obj->tab_len)); +} + +static void qat_uclo_map_mof_chunk(struct icp_qat_mof_handle *mobj_handle, + struct icp_qat_mof_chunkhdr *mof_chunkhdr) +{ + char *chunk_id = mof_chunkhdr->chunk_id; + + if (!strncmp(chunk_id, ICP_QAT_MOF_SYM_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) + qat_uclo_map_mof_symobjs(mobj_handle, mof_chunkhdr); + else if (!strncmp(chunk_id, ICP_QAT_UOF_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) + mobj_handle->uobjs_hdr = mobj_handle->mof_buf + + mof_chunkhdr->offset; + else if (!strncmp(chunk_id, ICP_QAT_SUOF_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) + mobj_handle->sobjs_hdr = mobj_handle->mof_buf + + mof_chunkhdr->offset; +} + +static int qat_uclo_check_mof_format(struct icp_qat_mof_file_hdr *mof_hdr) +{ + int maj = mof_hdr->maj_ver & 0xff; + int min = mof_hdr->min_ver & 0xff; + + if (mof_hdr->file_id != ICP_QAT_MOF_FID) { + pr_err("invalid header 0x%x\n", mof_hdr->file_id); + return -EINVAL; + } + + if (mof_hdr->num_chunks <= 0x1) { + pr_err("MOF chunk amount is incorrect\n"); + return -EINVAL; + } + if (maj != ICP_QAT_MOF_MAJVER || min != ICP_QAT_MOF_MINVER) { + pr_err("bad MOF version, major 0x%x, minor 0x%x\n", maj, min); + return -EINVAL; + } + return 0; +} + +static int qat_uclo_map_mof_obj(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_mof_file_hdr *mof_ptr, + u32 mof_size, const char *obj_name, + char **obj_ptr, unsigned int *obj_size) +{ + struct icp_qat_mof_chunkhdr *mof_chunkhdr; + unsigned int file_id = mof_ptr->file_id; + struct icp_qat_mof_handle *mobj_handle; + unsigned short chunks_num; + unsigned int i; + int ret; + + if (file_id == ICP_QAT_UOF_FID || file_id == ICP_QAT_SUOF_FID) { + if (obj_ptr) + *obj_ptr = (char *)mof_ptr; + if (obj_size) + *obj_size = mof_size; + return 0; + } + if (qat_uclo_check_mof_format(mof_ptr)) + return -EINVAL; + + mobj_handle = kzalloc_obj(*mobj_handle); + if (!mobj_handle) + return -ENOMEM; + + handle->mobj_handle = mobj_handle; + ret = qat_uclo_map_mof_file_hdr(handle, mof_ptr, mof_size); + if (ret) + return ret; + + mof_chunkhdr = (void *)mof_ptr + sizeof(*mof_ptr); + chunks_num = mof_ptr->num_chunks; + + /* Parse MOF file chunks */ + for (i = 0; i < chunks_num; i++) + qat_uclo_map_mof_chunk(mobj_handle, &mof_chunkhdr[i]); + + /* All sym_objs uobjs and sobjs should be available */ + if (!mobj_handle->sym_str || + (!mobj_handle->uobjs_hdr && !mobj_handle->sobjs_hdr)) + return -EINVAL; + + ret = qat_uclo_map_objs_from_mof(mobj_handle); + if (ret) + return ret; + + /* Seek specified uof object in MOF */ + return qat_uclo_seek_obj_inside_mof(mobj_handle, obj_name, + obj_ptr, obj_size); +} + +int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, + void *addr_ptr, u32 mem_size, const char *obj_name) +{ + char *obj_addr; + u32 obj_size; + int ret; + + BUILD_BUG_ON(ICP_QAT_UCLO_MAX_AE >= + (sizeof(handle->hal_handle->ae_mask) * 8)); + + if (!handle || !addr_ptr || mem_size < 24) + return -EINVAL; + + if (obj_name) { + ret = qat_uclo_map_mof_obj(handle, addr_ptr, mem_size, obj_name, + &obj_addr, &obj_size); + if (ret) + return ret; + } else { + obj_addr = addr_ptr; + obj_size = mem_size; + } + + return (handle->chip_info->fw_auth) ? + qat_uclo_map_suof_obj(handle, obj_addr, obj_size) : + qat_uclo_map_uof_obj(handle, obj_addr, obj_size); +} + +void qat_uclo_del_obj(struct icp_qat_fw_loader_handle *handle) +{ + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned int a; + + if (handle->mobj_handle) + qat_uclo_del_mof(handle); + if (handle->sobj_handle) + qat_uclo_del_suof(handle); + if (!obj_handle) + return; + + kfree(obj_handle->uword_buf); + for (a = 0; a < obj_handle->uimage_num; a++) + kfree(obj_handle->ae_uimage[a].page); + + for (a = 0; a < handle->hal_handle->ae_max_num; a++) + qat_uclo_free_ae_data(&obj_handle->ae_data[a]); + + kfree(obj_handle->obj_hdr); + kfree(obj_handle->obj_buf); + kfree(obj_handle); + handle->obj_handle = NULL; +} + +static void qat_uclo_fill_uwords(struct icp_qat_uclo_objhandle *obj_handle, + struct icp_qat_uclo_encap_page *encap_page, + u64 *uword, unsigned int addr_p, + unsigned int raddr, u64 fill) +{ + unsigned int i, addr; + u64 uwrd = 0; + + if (!encap_page) { + *uword = fill; + return; + } + addr = (encap_page->page_region) ? raddr : addr_p; + for (i = 0; i < encap_page->uwblock_num; i++) { + if (addr >= encap_page->uwblock[i].start_addr && + addr <= encap_page->uwblock[i].start_addr + + encap_page->uwblock[i].words_num - 1) { + addr -= encap_page->uwblock[i].start_addr; + addr *= obj_handle->uword_in_bytes; + memcpy(&uwrd, (void *)(((uintptr_t) + encap_page->uwblock[i].micro_words) + addr), + obj_handle->uword_in_bytes); + uwrd = uwrd & GENMASK_ULL(43, 0); + } + } + *uword = uwrd; + if (*uword == INVLD_UWORD) + *uword = fill; +} + +static void qat_uclo_wr_uimage_raw_page(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_uclo_encap_page + *encap_page, unsigned int ae) +{ + unsigned int uw_physical_addr, uw_relative_addr, i, words_num, cpylen; + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + u64 fill_pat; + + /* load the page starting at appropriate ustore address */ + /* get fill-pattern from an image -- they are all the same */ + memcpy(&fill_pat, obj_handle->ae_uimage[0].img_ptr->fill_pattern, + sizeof(u64)); + uw_physical_addr = encap_page->beg_addr_p; + uw_relative_addr = 0; + words_num = encap_page->micro_words_num; + while (words_num) { + cpylen = min(words_num, UWORD_CPYBUF_SIZE); + + /* load the buffer */ + for (i = 0; i < cpylen; i++) + qat_uclo_fill_uwords(obj_handle, encap_page, + &obj_handle->uword_buf[i], + uw_physical_addr + i, + uw_relative_addr + i, fill_pat); + + /* copy the buffer to ustore */ + qat_hal_wr_uwords(handle, (unsigned char)ae, + uw_physical_addr, cpylen, + obj_handle->uword_buf); + + uw_physical_addr += cpylen; + uw_relative_addr += cpylen; + words_num -= cpylen; + } +} + +static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_uof_image *image) +{ + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned long cfg_ae_mask = handle->cfg_ae_mask; + unsigned long ae_assigned = image->ae_assigned; + struct icp_qat_uclo_aedata *aed; + unsigned int ctx_mask, s; + struct icp_qat_uclo_page *page; + unsigned char ae; + int ctx; + + if (ICP_QAT_CTX_MODE(image->ae_mode) == ICP_QAT_UCLO_MAX_CTX) + ctx_mask = 0xff; + else + ctx_mask = 0x55; + /* load the default page and set assigned CTX PC + * to the entrypoint address */ + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + if (!test_bit(ae, &cfg_ae_mask)) + continue; + + if (!test_bit(ae, &ae_assigned)) + continue; + + aed = &obj_handle->ae_data[ae]; + /* find the slice to which this image is assigned */ + for (s = 0; s < aed->slice_num; s++) { + if (image->ctx_assigned & + aed->ae_slices[s].ctx_mask_assigned) + break; + } + if (s >= aed->slice_num) + continue; + page = aed->ae_slices[s].page; + if (!page->encap_page->def_page) + continue; + qat_uclo_wr_uimage_raw_page(handle, page->encap_page, ae); + + page = aed->ae_slices[s].page; + for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) + aed->ae_slices[s].cur_page[ctx] = + (ctx_mask & (1 << ctx)) ? page : NULL; + qat_hal_set_live_ctx(handle, (unsigned char)ae, + image->ctx_assigned); + qat_hal_set_pc(handle, (unsigned char)ae, image->ctx_assigned, + image->entry_address); + } +} + +static int qat_uclo_wr_suof_img(struct icp_qat_fw_loader_handle *handle) +{ + unsigned int i; + struct icp_qat_fw_auth_desc *desc = NULL; + struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle; + struct icp_qat_suof_img_hdr *simg_hdr = sobj_handle->img_table.simg_hdr; + int ret; + + for (i = 0; i < sobj_handle->img_table.num_simgs; i++) { + ret = qat_uclo_check_image(handle, simg_hdr[i].simg_buf, + simg_hdr[i].simg_len, + CSS_AE_FIRMWARE); + if (ret) + return ret; + + if (qat_uclo_map_auth_fw(handle, + (char *)simg_hdr[i].simg_buf, + (unsigned int) + simg_hdr[i].simg_len, + &desc)) + goto wr_err; + if (qat_uclo_auth_fw(handle, desc)) + goto wr_err; + if (qat_uclo_is_broadcast(handle, i)) { + if (qat_uclo_broadcast_load_fw(handle, desc)) + goto wr_err; + } else { + if (qat_uclo_load_fw(handle, desc)) + goto wr_err; + } + qat_uclo_ummap_auth_fw(handle, &desc); + } + return 0; +wr_err: + qat_uclo_ummap_auth_fw(handle, &desc); + return -EINVAL; +} + +static int qat_uclo_wr_uof_img(struct icp_qat_fw_loader_handle *handle) +{ + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned int i; + + if (qat_uclo_init_globals(handle)) + return -EINVAL; + for (i = 0; i < obj_handle->uimage_num; i++) { + if (!obj_handle->ae_uimage[i].img_ptr) + return -EINVAL; + if (qat_uclo_init_ustore(handle, &obj_handle->ae_uimage[i])) + return -EINVAL; + qat_uclo_wr_uimage_page(handle, + obj_handle->ae_uimage[i].img_ptr); + } + return 0; +} + +int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle) +{ + return (handle->chip_info->fw_auth) ? qat_uclo_wr_suof_img(handle) : + qat_uclo_wr_uof_img(handle); +} + +int qat_uclo_set_cfg_ae_mask(struct icp_qat_fw_loader_handle *handle, + unsigned int cfg_ae_mask) +{ + if (!cfg_ae_mask) + return -EINVAL; + + handle->cfg_ae_mask = cfg_ae_mask; + return 0; +} diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/Makefile b/drivers/crypto/intel/qat/qat_dh895xcc/Makefile new file mode 100644 index 000000000..1427fe76f --- /dev/null +++ b/drivers/crypto/intel/qat/qat_dh895xcc/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc.o +qat_dh895xcc-y := adf_drv.o adf_dh895xcc_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c new file mode 100644 index 000000000..5b4bd0ba1 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -0,0 +1,269 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2021 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include +#include "adf_dh895xcc_hw_data.h" +#include "adf_heartbeat.h" +#include "icp_qat_hw.h" + +#define ADF_DH895XCC_VF_MSK 0xFFFFFFFF + +/* Worker thread to service arbiter mappings */ +static const u32 thrd_to_arb_map[ADF_DH895XCC_MAX_ACCELENGINES] = { + 0x12222AAA, 0x11666666, 0x12222AAA, 0x11666666, + 0x12222AAA, 0x11222222, 0x12222AAA, 0x11222222, + 0x12222AAA, 0x11222222, 0x12222AAA, 0x11222222 +}; + +static struct adf_hw_device_class dh895xcc_class = { + .name = ADF_DH895XCC_DEVICE_NAME, + .type = DEV_DH895XCC, +}; + +static u32 get_accel_mask(struct adf_hw_device_data *self) +{ + u32 fuses = self->fuses[ADF_FUSECTL0]; + + return ~fuses >> ADF_DH895XCC_ACCELERATORS_REG_OFFSET & + ADF_DH895XCC_ACCELERATORS_MASK; +} + +static u32 get_ae_mask(struct adf_hw_device_data *self) +{ + u32 fuses = self->fuses[ADF_FUSECTL0]; + + return ~fuses & ADF_DH895XCC_ACCELENGINES_MASK; +} + +static u32 get_misc_bar_id(struct adf_hw_device_data *self) +{ + return ADF_DH895XCC_PMISC_BAR; +} + +static u32 get_ts_clock(struct adf_hw_device_data *self) +{ + /* + * Timestamp update interval is 16 AE clock ticks for dh895xcc. + */ + return self->clock_frequency / 16; +} + +static u32 get_etr_bar_id(struct adf_hw_device_data *self) +{ + return ADF_DH895XCC_ETR_BAR; +} + +static u32 get_sram_bar_id(struct adf_hw_device_data *self) +{ + return ADF_DH895XCC_SRAM_BAR; +} + +static u32 get_accel_cap(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev; + u32 capabilities; + u32 legfuses; + + capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | + ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | + ICP_ACCEL_CAPABILITIES_AUTHENTICATION | + ICP_ACCEL_CAPABILITIES_CIPHER | + ICP_ACCEL_CAPABILITIES_COMPRESSION; + + /* Read accelerator capabilities mask */ + pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, &legfuses); + + /* A set bit in legfuses means the feature is OFF in this SKU */ + if (legfuses & ICP_ACCEL_MASK_CIPHER_SLICE) { + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + } + if (legfuses & ICP_ACCEL_MASK_PKE_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + if (legfuses & ICP_ACCEL_MASK_AUTH_SLICE) { + capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + } + if (legfuses & ICP_ACCEL_MASK_COMPRESS_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION; + + return capabilities; +} + +static enum dev_sku_info get_sku(struct adf_hw_device_data *self) +{ + int sku = (self->fuses[ADF_FUSECTL0] & ADF_DH895XCC_FUSECTL_SKU_MASK) + >> ADF_DH895XCC_FUSECTL_SKU_SHIFT; + + switch (sku) { + case ADF_DH895XCC_FUSECTL_SKU_1: + return DEV_SKU_1; + case ADF_DH895XCC_FUSECTL_SKU_2: + return DEV_SKU_2; + case ADF_DH895XCC_FUSECTL_SKU_3: + return DEV_SKU_3; + case ADF_DH895XCC_FUSECTL_SKU_4: + return DEV_SKU_4; + default: + return DEV_SKU_UNKNOWN; + } + return DEV_SKU_UNKNOWN; +} + +static const u32 *adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev) +{ + return thrd_to_arb_map; +} + +static void enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) +{ + /* Enable VF2PF Messaging Ints - VFs 0 through 15 per vf_mask[15:0] */ + if (vf_mask & 0xFFFF) { + u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3) + & ~ADF_DH895XCC_ERR_MSK_VF2PF_L(vf_mask); + ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val); + } + + /* Enable VF2PF Messaging Ints - VFs 16 through 31 per vf_mask[31:16] */ + if (vf_mask >> 16) { + u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK5) + & ~ADF_DH895XCC_ERR_MSK_VF2PF_U(vf_mask); + ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK5, val); + } +} + +static void disable_all_vf2pf_interrupts(void __iomem *pmisc_addr) +{ + u32 val; + + /* Disable VF2PF interrupts for VFs 0 through 15 per vf_mask[15:0] */ + val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3) + | ADF_DH895XCC_ERR_MSK_VF2PF_L(ADF_DH895XCC_VF_MSK); + ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val); + + /* Disable VF2PF interrupts for VFs 16 through 31 per vf_mask[31:16] */ + val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK5) + | ADF_DH895XCC_ERR_MSK_VF2PF_U(ADF_DH895XCC_VF_MSK); + ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK5, val); +} + +static u32 disable_pending_vf2pf_interrupts(void __iomem *pmisc_addr) +{ + u32 sources, pending, disabled; + u32 errsou3, errmsk3; + u32 errsou5, errmsk5; + + /* Get the interrupt sources triggered by VFs */ + errsou3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRSOU3); + errsou5 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRSOU5); + sources = ADF_DH895XCC_ERR_REG_VF2PF_L(errsou3) + | ADF_DH895XCC_ERR_REG_VF2PF_U(errsou5); + + if (!sources) + return 0; + + /* Get the already disabled interrupts */ + errmsk3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3); + errmsk5 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK5); + disabled = ADF_DH895XCC_ERR_REG_VF2PF_L(errmsk3) + | ADF_DH895XCC_ERR_REG_VF2PF_U(errmsk5); + + pending = sources & ~disabled; + if (!pending) + return 0; + + /* Due to HW limitations, when disabling the interrupts, we can't + * just disable the requested sources, as this would lead to missed + * interrupts if sources changes just before writing to ERRMSK3 and + * ERRMSK5. + * To work around it, disable all and re-enable only the sources that + * are not in vf_mask and were not already disabled. Re-enabling will + * trigger a new interrupt for the sources that have changed in the + * meantime, if any. + */ + errmsk3 |= ADF_DH895XCC_ERR_MSK_VF2PF_L(ADF_DH895XCC_VF_MSK); + errmsk5 |= ADF_DH895XCC_ERR_MSK_VF2PF_U(ADF_DH895XCC_VF_MSK); + ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, errmsk3); + ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK5, errmsk5); + + /* Update only section of errmsk3 and errmsk5 related to VF2PF */ + errmsk3 &= ~ADF_DH895XCC_ERR_MSK_VF2PF_L(ADF_DH895XCC_VF_MSK); + errmsk5 &= ~ADF_DH895XCC_ERR_MSK_VF2PF_U(ADF_DH895XCC_VF_MSK); + + errmsk3 |= ADF_DH895XCC_ERR_MSK_VF2PF_L(sources | disabled); + errmsk5 |= ADF_DH895XCC_ERR_MSK_VF2PF_U(sources | disabled); + ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, errmsk3); + ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK5, errmsk5); + + /* Return the sources of the (new) interrupt(s) */ + return pending; +} + +static void configure_iov_threads(struct adf_accel_dev *accel_dev, bool enable) +{ + adf_gen2_cfg_iov_thds(accel_dev, enable, + ADF_DH895XCC_AE2FUNC_MAP_GRP_A_NUM_REGS, + ADF_DH895XCC_AE2FUNC_MAP_GRP_B_NUM_REGS); +} + +void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class = &dh895xcc_class; + hw_data->instance_id = dh895xcc_class.instances++; + hw_data->num_banks = ADF_DH895XCC_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; + hw_data->num_accel = ADF_DH895XCC_MAX_ACCELERATORS; + hw_data->num_logical_accel = 1; + hw_data->num_engines = ADF_DH895XCC_MAX_ACCELENGINES; + hw_data->tx_rx_gap = ADF_GEN2_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_GEN2_TX_RINGS_MASK; + hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP; + hw_data->alloc_irq = adf_isr_resource_alloc; + hw_data->free_irq = adf_isr_resource_free; + hw_data->enable_error_correction = adf_gen2_enable_error_correction; + hw_data->get_accel_mask = get_accel_mask; + hw_data->get_ae_mask = get_ae_mask; + hw_data->get_accel_cap = get_accel_cap; + hw_data->get_num_accels = adf_gen2_get_num_accels; + hw_data->get_num_aes = adf_gen2_get_num_aes; + hw_data->get_etr_bar_id = get_etr_bar_id; + hw_data->get_misc_bar_id = get_misc_bar_id; + hw_data->get_admin_info = adf_gen2_get_admin_info; + hw_data->get_arb_info = adf_gen2_get_arb_info; + hw_data->get_sram_bar_id = get_sram_bar_id; + hw_data->get_sku = get_sku; + hw_data->fw_name = ADF_DH895XCC_FW; + hw_data->fw_mmp_name = ADF_DH895XCC_MMP; + hw_data->init_admin_comms = adf_init_admin_comms; + hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->configure_iov_threads = configure_iov_threads; + hw_data->send_admin_init = adf_send_admin_init; + hw_data->init_arb = adf_init_arb; + hw_data->exit_arb = adf_exit_arb; + hw_data->get_arb_mapping = adf_get_arbiter_mapping; + hw_data->enable_ints = adf_gen2_enable_ints; + hw_data->reset_device = adf_reset_sbr; + hw_data->disable_iov = adf_disable_sriov; + hw_data->dev_config = adf_gen2_dev_config; + hw_data->clock_frequency = ADF_DH895X_AE_FREQ; + hw_data->get_hb_clock = get_ts_clock; + hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE; + hw_data->check_hb_ctrs = adf_heartbeat_check_ctrs; + + adf_gen2_init_pf_pfvf_ops(&hw_data->pfvf_ops); + hw_data->pfvf_ops.enable_vf2pf_interrupts = enable_vf2pf_interrupts; + hw_data->pfvf_ops.disable_all_vf2pf_interrupts = disable_all_vf2pf_interrupts; + hw_data->pfvf_ops.disable_pending_vf2pf_interrupts = disable_pending_vf2pf_interrupts; + adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); + adf_gen2_init_dc_ops(&hw_data->dc_ops); +} + +void adf_clean_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class->instances--; +} diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h new file mode 100644 index 000000000..cd3a21985 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#ifndef ADF_DH895x_HW_DATA_H_ +#define ADF_DH895x_HW_DATA_H_ + +#include + +/* PCIe configuration space */ +#define ADF_DH895XCC_SRAM_BAR 0 +#define ADF_DH895XCC_PMISC_BAR 1 +#define ADF_DH895XCC_ETR_BAR 2 +#define ADF_DH895XCC_FUSECTL_SKU_MASK 0x300000 +#define ADF_DH895XCC_FUSECTL_SKU_SHIFT 20 +#define ADF_DH895XCC_FUSECTL_SKU_1 0x0 +#define ADF_DH895XCC_FUSECTL_SKU_2 0x1 +#define ADF_DH895XCC_FUSECTL_SKU_3 0x2 +#define ADF_DH895XCC_FUSECTL_SKU_4 0x3 +#define ADF_DH895XCC_MAX_ACCELERATORS 6 +#define ADF_DH895XCC_MAX_ACCELENGINES 12 +#define ADF_DH895XCC_ACCELERATORS_REG_OFFSET 13 +#define ADF_DH895XCC_ACCELERATORS_MASK 0x3F +#define ADF_DH895XCC_ACCELENGINES_MASK 0xFFF +#define ADF_DH895XCC_ETR_MAX_BANKS 32 + +/* Masks for VF2PF interrupts */ +#define ADF_DH895XCC_ERR_REG_VF2PF_L(vf_src) (((vf_src) & 0x01FFFE00) >> 9) +#define ADF_DH895XCC_ERR_MSK_VF2PF_L(vf_mask) (((vf_mask) & 0xFFFF) << 9) +#define ADF_DH895XCC_ERR_REG_VF2PF_U(vf_src) (((vf_src) & 0x0000FFFF) << 16) +#define ADF_DH895XCC_ERR_MSK_VF2PF_U(vf_mask) ((vf_mask) >> 16) + +/* AE to function mapping */ +#define ADF_DH895XCC_AE2FUNC_MAP_GRP_A_NUM_REGS 96 +#define ADF_DH895XCC_AE2FUNC_MAP_GRP_B_NUM_REGS 12 + +/* Clocks frequency */ +#define ADF_DH895X_AE_FREQ (933 * HZ_PER_MHZ) + +/* FW names */ +#define ADF_DH895XCC_FW "qat_895xcc.bin" +#define ADF_DH895XCC_MMP "qat_895xcc_mmp.bin" + +void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data); +void adf_clean_hw_data_dh895xcc(struct adf_hw_device_data *hw_data); +#endif diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c new file mode 100644 index 000000000..571f302ed --- /dev/null +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "adf_dh895xcc_hw_data.h" + +static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) +{ + pci_release_regions(accel_dev->accel_pci_dev.pci_dev); + pci_disable_device(accel_dev->accel_pci_dev.pci_dev); +} + +static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; + int i; + + for (i = 0; i < ADF_PCI_MAX_BARS; i++) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; + + if (bar->virt_addr) + pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr); + } + + if (accel_dev->hw_device) { + switch (accel_pci_dev->pci_dev->device) { + case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: + adf_clean_hw_data_dh895xcc(accel_dev->hw_device); + break; + default: + break; + } + kfree(accel_dev->hw_device); + accel_dev->hw_device = NULL; + } + adf_dbgfs_exit(accel_dev); + adf_cfg_dev_remove(accel_dev); + adf_devmgr_rm_dev(accel_dev, NULL); +} + +static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct adf_accel_dev *accel_dev; + struct adf_accel_pci *accel_pci_dev; + struct adf_hw_device_data *hw_data; + unsigned int i, bar_nr; + unsigned long bar_mask; + int ret; + + switch (ent->device) { + case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: + break; + default: + dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); + return -ENODEV; + } + + if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) { + /* If the accelerator is connected to a node with no memory + * there is no point in using the accelerator since the remote + * memory transaction will be very slow. */ + dev_err(&pdev->dev, "Invalid NUMA configuration.\n"); + return -EINVAL; + } + + accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!accel_dev) + return -ENOMEM; + + INIT_LIST_HEAD(&accel_dev->crypto_list); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = pdev; + + /* Add accel device to accel table. + * This should be called before adf_cleanup_accel is called */ + if (adf_devmgr_add_dev(accel_dev, NULL)) { + dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); + kfree(accel_dev); + return -EFAULT; + } + + accel_dev->owner = THIS_MODULE; + /* Allocate and configure device configuration structure */ + hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!hw_data) { + ret = -ENOMEM; + goto out_err; + } + + accel_dev->hw_device = hw_data; + adf_init_hw_data_dh895xcc(accel_dev->hw_device); + pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); + pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, + &hw_data->fuses[ADF_FUSECTL0]); + + /* Get Accelerators and Accelerators Engines masks */ + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); + accel_pci_dev->sku = hw_data->get_sku(hw_data); + /* If the device has no acceleration engines then ignore it. */ + if (!hw_data->accel_mask || !hw_data->ae_mask || + ((~hw_data->ae_mask) & 0x01)) { + dev_err(&pdev->dev, "No acceleration units found"); + ret = -EFAULT; + goto out_err; + } + + /* Create device configuration table */ + ret = adf_cfg_dev_add(accel_dev); + if (ret) + goto out_err; + + pcie_set_readrq(pdev, 1024); + + /* enable PCI device */ + if (pci_enable_device(pdev)) { + ret = -EFAULT; + goto out_err; + } + + /* set dma identifier */ + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); + if (ret) { + dev_err(&pdev->dev, "No usable DMA configuration\n"); + goto out_err_disable; + } + + if (pci_request_regions(pdev, ADF_DH895XCC_DEVICE_NAME)) { + ret = -EFAULT; + goto out_err_disable; + } + + /* Get accelerator capabilities mask */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); + + /* Find and map all the device's BARS */ + i = 0; + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); + for_each_set_bit(bar_nr, &bar_mask, ADF_PCI_MAX_BARS * 2) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; + + bar->base_addr = pci_resource_start(pdev, bar_nr); + if (!bar->base_addr) + break; + bar->size = pci_resource_len(pdev, bar_nr); + bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); + if (!bar->virt_addr) { + pci_err(pdev, "Failed to map BAR %d\n", bar_nr); + ret = -EFAULT; + goto out_err_free_reg; + } + } + + if (pci_save_state(pdev)) { + pci_err(pdev, "Failed to save pci state\n"); + ret = -ENOMEM; + goto out_err_free_reg; + } + + adf_dbgfs_init(accel_dev); + + ret = adf_dev_up(accel_dev, true); + if (ret) + goto out_err_dev_stop; + + return ret; + +out_err_dev_stop: + adf_dev_down(accel_dev); +out_err_free_reg: + pci_release_regions(accel_pci_dev->pci_dev); +out_err_disable: + pci_disable_device(accel_pci_dev->pci_dev); +out_err: + adf_cleanup_accel(accel_dev); + kfree(accel_dev); + return ret; +} + +static void adf_remove(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + if (!accel_dev) { + pr_err("QAT: Driver removal failed\n"); + return; + } + adf_dev_down(accel_dev); + adf_cleanup_accel(accel_dev); + adf_cleanup_pci_dev(accel_dev); + kfree(accel_dev); +} + +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + +static const struct pci_device_id adf_pci_tbl[] = { + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_DH895XCC) }, + { } +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_DH895XCC_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, + .shutdown = adf_shutdown, + .sriov_configure = adf_sriov_configure, + .err_handler = &adf_err_handler, +}; + +static int __init adfdrv_init(void) +{ + request_module("intel_qat"); + + if (pci_register_driver(&adf_driver)) { + pr_err("QAT: Driver initialization failed\n"); + return -EFAULT; + } + return 0; +} + +static void __exit adfdrv_release(void) +{ + pci_unregister_driver(&adf_driver); +} + +module_init(adfdrv_init); +module_exit(adfdrv_release); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Intel"); +MODULE_FIRMWARE(ADF_DH895XCC_FW); +MODULE_FIRMWARE(ADF_DH895XCC_MMP); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); +MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile b/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile new file mode 100644 index 000000000..c2fdb6e0f --- /dev/null +++ b/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCCVF) += qat_dh895xccvf.o +qat_dh895xccvf-y := adf_drv.o adf_dh895xccvf_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c new file mode 100644 index 000000000..828456c43 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2015 - 2021 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include +#include "adf_dh895xccvf_hw_data.h" + +static struct adf_hw_device_class dh895xcciov_class = { + .name = ADF_DH895XCCVF_DEVICE_NAME, + .type = DEV_DH895XCCVF, +}; + +static u32 get_accel_mask(struct adf_hw_device_data *self) +{ + return ADF_DH895XCCIOV_ACCELERATORS_MASK; +} + +static u32 get_ae_mask(struct adf_hw_device_data *self) +{ + return ADF_DH895XCCIOV_ACCELENGINES_MASK; +} + +static u32 get_num_accels(struct adf_hw_device_data *self) +{ + return ADF_DH895XCCIOV_MAX_ACCELERATORS; +} + +static u32 get_num_aes(struct adf_hw_device_data *self) +{ + return ADF_DH895XCCIOV_MAX_ACCELENGINES; +} + +static u32 get_misc_bar_id(struct adf_hw_device_data *self) +{ + return ADF_DH895XCCIOV_PMISC_BAR; +} + +static u32 get_etr_bar_id(struct adf_hw_device_data *self) +{ + return ADF_DH895XCCIOV_ETR_BAR; +} + +static enum dev_sku_info get_sku(struct adf_hw_device_data *self) +{ + return DEV_SKU_VF; +} + +static int adf_vf_int_noop(struct adf_accel_dev *accel_dev) +{ + return 0; +} + +static void adf_vf_void_noop(struct adf_accel_dev *accel_dev) +{ +} + +void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class = &dh895xcciov_class; + hw_data->num_banks = ADF_DH895XCCIOV_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; + hw_data->num_accel = ADF_DH895XCCIOV_MAX_ACCELERATORS; + hw_data->num_logical_accel = 1; + hw_data->num_engines = ADF_DH895XCCIOV_MAX_ACCELENGINES; + hw_data->tx_rx_gap = ADF_DH895XCCIOV_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_DH895XCCIOV_TX_RINGS_MASK; + hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP; + hw_data->alloc_irq = adf_vf_isr_resource_alloc; + hw_data->free_irq = adf_vf_isr_resource_free; + hw_data->enable_error_correction = adf_vf_void_noop; + hw_data->init_admin_comms = adf_vf_int_noop; + hw_data->exit_admin_comms = adf_vf_void_noop; + hw_data->send_admin_init = adf_vf2pf_notify_init; + hw_data->init_arb = adf_vf_int_noop; + hw_data->exit_arb = adf_vf_void_noop; + hw_data->disable_iov = adf_vf2pf_notify_shutdown; + hw_data->get_accel_mask = get_accel_mask; + hw_data->get_ae_mask = get_ae_mask; + hw_data->get_num_accels = get_num_accels; + hw_data->get_num_aes = get_num_aes; + hw_data->get_etr_bar_id = get_etr_bar_id; + hw_data->get_misc_bar_id = get_misc_bar_id; + hw_data->get_sku = get_sku; + hw_data->enable_ints = adf_vf_void_noop; + hw_data->dev_class->instances++; + hw_data->dev_config = adf_gen2_dev_config; + adf_devmgr_update_class_index(hw_data); + adf_gen2_init_vf_pfvf_ops(&hw_data->pfvf_ops); + adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); + adf_gen2_init_dc_ops(&hw_data->dc_ops); +} + +void adf_clean_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class->instances--; + adf_devmgr_update_class_index(hw_data); +} diff --git a/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h new file mode 100644 index 000000000..6973fa967 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2015 - 2020 Intel Corporation */ +#ifndef ADF_DH895XVF_HW_DATA_H_ +#define ADF_DH895XVF_HW_DATA_H_ + +#define ADF_DH895XCCIOV_PMISC_BAR 1 +#define ADF_DH895XCCIOV_ACCELERATORS_MASK 0x1 +#define ADF_DH895XCCIOV_ACCELENGINES_MASK 0x1 +#define ADF_DH895XCCIOV_MAX_ACCELERATORS 1 +#define ADF_DH895XCCIOV_MAX_ACCELENGINES 1 +#define ADF_DH895XCCIOV_RX_RINGS_OFFSET 8 +#define ADF_DH895XCCIOV_TX_RINGS_MASK 0xFF +#define ADF_DH895XCCIOV_ETR_BAR 0 +#define ADF_DH895XCCIOV_ETR_MAX_BANKS 1 + +void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data); +void adf_clean_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data); +#endif diff --git a/drivers/crypto/intel/qat/qat_dh895xccvf/adf_drv.c b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_drv.c new file mode 100644 index 000000000..481551a08 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_drv.c @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "adf_dh895xccvf_hw_data.h" + +static const struct pci_device_id adf_pci_tbl[] = { + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_DH895XCC_VF), }, + { } +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); +static void adf_remove(struct pci_dev *dev); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_DH895XCCVF_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, +}; + +static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) +{ + pci_release_regions(accel_dev->accel_pci_dev.pci_dev); + pci_disable_device(accel_dev->accel_pci_dev.pci_dev); +} + +static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; + struct adf_accel_dev *pf; + int i; + + for (i = 0; i < ADF_PCI_MAX_BARS; i++) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; + + if (bar->virt_addr) + pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr); + } + + if (accel_dev->hw_device) { + switch (accel_pci_dev->pci_dev->device) { + case PCI_DEVICE_ID_INTEL_QAT_DH895XCC_VF: + adf_clean_hw_data_dh895xcciov(accel_dev->hw_device); + break; + default: + break; + } + kfree(accel_dev->hw_device); + accel_dev->hw_device = NULL; + } + adf_dbgfs_exit(accel_dev); + adf_cfg_dev_remove(accel_dev); + pf = adf_devmgr_pci_to_accel_dev(accel_pci_dev->pci_dev->physfn); + adf_devmgr_rm_dev(accel_dev, pf); +} + +static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct adf_accel_dev *accel_dev; + struct adf_accel_dev *pf; + struct adf_accel_pci *accel_pci_dev; + struct adf_hw_device_data *hw_data; + unsigned int i, bar_nr; + unsigned long bar_mask; + int ret; + + switch (ent->device) { + case PCI_DEVICE_ID_INTEL_QAT_DH895XCC_VF: + break; + default: + dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); + return -ENODEV; + } + + accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!accel_dev) + return -ENOMEM; + + accel_dev->is_vf = true; + pf = adf_devmgr_pci_to_accel_dev(pdev->physfn); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = pdev; + + /* Add accel device to accel table */ + if (adf_devmgr_add_dev(accel_dev, pf)) { + dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); + kfree(accel_dev); + return -EFAULT; + } + INIT_LIST_HEAD(&accel_dev->crypto_list); + + accel_dev->owner = THIS_MODULE; + /* Allocate and configure device configuration structure */ + hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!hw_data) { + ret = -ENOMEM; + goto out_err; + } + accel_dev->hw_device = hw_data; + adf_init_hw_data_dh895xcciov(accel_dev->hw_device); + + /* Get Accelerators and Accelerators Engines masks */ + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); + accel_pci_dev->sku = hw_data->get_sku(hw_data); + + /* Create device configuration table */ + ret = adf_cfg_dev_add(accel_dev); + if (ret) + goto out_err; + + /* enable PCI device */ + if (pci_enable_device(pdev)) { + ret = -EFAULT; + goto out_err; + } + + /* set dma identifier */ + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); + if (ret) { + dev_err(&pdev->dev, "No usable DMA configuration\n"); + goto out_err_disable; + } + + if (pci_request_regions(pdev, ADF_DH895XCCVF_DEVICE_NAME)) { + ret = -EFAULT; + goto out_err_disable; + } + + /* Find and map all the device's BARS */ + i = 0; + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); + for_each_set_bit(bar_nr, &bar_mask, ADF_PCI_MAX_BARS * 2) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; + + bar->base_addr = pci_resource_start(pdev, bar_nr); + if (!bar->base_addr) + break; + bar->size = pci_resource_len(pdev, bar_nr); + bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); + if (!bar->virt_addr) { + pci_err(pdev, "Failed to map BAR %d\n", bar_nr); + ret = -EFAULT; + goto out_err_free_reg; + } + } + /* Completion for VF2PF request/response message exchange */ + init_completion(&accel_dev->vf.msg_received); + + adf_dbgfs_init(accel_dev); + + ret = adf_dev_up(accel_dev, false); + if (ret) + goto out_err_dev_stop; + + return ret; + +out_err_dev_stop: + adf_dev_down(accel_dev); +out_err_free_reg: + pci_release_regions(accel_pci_dev->pci_dev); +out_err_disable: + pci_disable_device(accel_pci_dev->pci_dev); +out_err: + adf_cleanup_accel(accel_dev); + kfree(accel_dev); + return ret; +} + +static void adf_remove(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + if (!accel_dev) { + pr_err("QAT: Driver removal failed\n"); + return; + } + adf_flush_vf_wq(accel_dev); + adf_dev_down(accel_dev); + adf_cleanup_accel(accel_dev); + adf_cleanup_pci_dev(accel_dev); + kfree(accel_dev); +} + +static int __init adfdrv_init(void) +{ + request_module("intel_qat"); + + if (pci_register_driver(&adf_driver)) { + pr_err("QAT: Driver initialization failed\n"); + return -EFAULT; + } + return 0; +} + +static void __exit adfdrv_release(void) +{ + pci_unregister_driver(&adf_driver); + adf_clean_vf_map(true); +} + +module_init(adfdrv_init); +module_exit(adfdrv_release); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Intel"); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); +MODULE_IMPORT_NS("CRYPTO_QAT"); diff --git a/drivers/crypto/marvell/Kconfig b/drivers/crypto/marvell/Kconfig new file mode 100644 index 000000000..aa269abb0 --- /dev/null +++ b/drivers/crypto/marvell/Kconfig @@ -0,0 +1,55 @@ +# +# Marvell crypto drivers configuration +# + +config CRYPTO_DEV_MARVELL + tristate + +config CRYPTO_DEV_MARVELL_CESA + tristate "Marvell's Cryptographic Engine driver" + depends on PLAT_ORION || ARCH_MVEBU || COMPILE_TEST + select CRYPTO_LIB_AES + select CRYPTO_LIB_DES + select CRYPTO_SKCIPHER + select CRYPTO_HASH + select SRAM + select CRYPTO_DEV_MARVELL + help + This driver allows you to utilize the Cryptographic Engines and + Security Accelerator (CESA) which can be found on MVEBU and ORION + platforms. + This driver supports CPU offload through DMA transfers. + +config CRYPTO_DEV_OCTEONTX_CPT + tristate "Support for Marvell OcteonTX CPT driver" + depends on ARCH_THUNDER || COMPILE_TEST + depends on PCI_MSI && 64BIT + select CRYPTO_LIB_AES + select CRYPTO_SKCIPHER + select CRYPTO_HASH + select CRYPTO_AEAD + select CRYPTO_AUTHENC + select CRYPTO_DEV_MARVELL + help + This driver allows you to utilize the Marvell Cryptographic + Accelerator Unit(CPT) found in OcteonTX series of processors. + + To compile this driver as module, choose M here: + the modules will be called octeontx-cpt and octeontx-cptvf + +config CRYPTO_DEV_OCTEONTX2_CPT + tristate "Marvell OcteonTX2 CPT driver" + depends on ARCH_THUNDER2 || COMPILE_TEST + depends on PCI_MSI && 64BIT + depends on NET_VENDOR_MARVELL + select OCTEONTX2_MBOX + select CRYPTO_DEV_MARVELL + select CRYPTO_LIB_AES + select CRYPTO_SKCIPHER + select CRYPTO_HASH + select CRYPTO_AEAD + select CRYPTO_AUTHENC + select NET_DEVLINK + help + This driver allows you to utilize the Marvell Cryptographic + Accelerator Unit(CPT) found in OcteonTX2 series of processors. diff --git a/drivers/crypto/marvell/Makefile b/drivers/crypto/marvell/Makefile new file mode 100644 index 000000000..39db6d9c0 --- /dev/null +++ b/drivers/crypto/marvell/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_CRYPTO_DEV_MARVELL_CESA) += cesa/ +obj-$(CONFIG_CRYPTO_DEV_OCTEONTX_CPT) += octeontx/ +obj-$(CONFIG_CRYPTO_DEV_OCTEONTX2_CPT) += octeontx2/ diff --git a/drivers/crypto/marvell/cesa/Makefile b/drivers/crypto/marvell/cesa/Makefile new file mode 100644 index 000000000..b27cab65e --- /dev/null +++ b/drivers/crypto/marvell/cesa/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_MARVELL_CESA) += marvell-cesa.o +marvell-cesa-objs := cesa.o cipher.o hash.o tdma.o diff --git a/drivers/crypto/marvell/cesa/cesa.c b/drivers/crypto/marvell/cesa/cesa.c new file mode 100644 index 000000000..564b09773 --- /dev/null +++ b/drivers/crypto/marvell/cesa/cesa.c @@ -0,0 +1,571 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Support for Marvell's Cryptographic Engine and Security Accelerator (CESA) + * that can be found on the following platform: Orion, Kirkwood, Armada. This + * driver supports the TDMA engine on platforms on which it is available. + * + * Author: Boris Brezillon + * Author: Arnaud Ebalard + * + * This work is based on an initial version written by + * Sebastian Andrzej Siewior < sebastian at breakpoint dot cc > + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cesa.h" + +/* Limit of the crypto queue before reaching the backlog */ +#define CESA_CRYPTO_DEFAULT_MAX_QLEN 128 + +struct mv_cesa_dev *cesa_dev; + +struct crypto_async_request * +mv_cesa_dequeue_req_locked(struct mv_cesa_engine *engine, + struct crypto_async_request **backlog) +{ + *backlog = crypto_get_backlog(&engine->queue); + + return crypto_dequeue_request(&engine->queue); +} + +static void mv_cesa_rearm_engine(struct mv_cesa_engine *engine) +{ + struct crypto_async_request *req = NULL, *backlog = NULL; + struct mv_cesa_ctx *ctx; + + + spin_lock_bh(&engine->lock); + if (!engine->req) { + req = mv_cesa_dequeue_req_locked(engine, &backlog); + engine->req = req; + } + spin_unlock_bh(&engine->lock); + + if (!req) + return; + + if (backlog) + crypto_request_complete(backlog, -EINPROGRESS); + + ctx = crypto_tfm_ctx(req->tfm); + ctx->ops->step(req); +} + +static int mv_cesa_std_process(struct mv_cesa_engine *engine, u32 status) +{ + struct crypto_async_request *req; + struct mv_cesa_ctx *ctx; + int res; + + req = engine->req; + ctx = crypto_tfm_ctx(req->tfm); + res = ctx->ops->process(req, status); + + if (res == 0) { + ctx->ops->complete(req); + mv_cesa_engine_enqueue_complete_request(engine, req); + } else if (res == -EINPROGRESS) { + ctx->ops->step(req); + } + + return res; +} + +static int mv_cesa_int_process(struct mv_cesa_engine *engine, u32 status) +{ + if (engine->chain_hw.first && engine->chain_hw.last) + return mv_cesa_tdma_process(engine, status); + + return mv_cesa_std_process(engine, status); +} + +static inline void +mv_cesa_complete_req(struct mv_cesa_ctx *ctx, struct crypto_async_request *req, + int res) +{ + ctx->ops->cleanup(req); + local_bh_disable(); + crypto_request_complete(req, res); + local_bh_enable(); +} + +static irqreturn_t mv_cesa_int(int irq, void *priv) +{ + struct mv_cesa_engine *engine = priv; + struct crypto_async_request *req; + struct mv_cesa_ctx *ctx; + u32 status, mask; + irqreturn_t ret = IRQ_NONE; + + while (true) { + int res; + + mask = mv_cesa_get_int_mask(engine); + status = readl(engine->regs + CESA_SA_INT_STATUS); + + if (!(status & mask)) + break; + + /* + * TODO: avoid clearing the FPGA_INT_STATUS if this not + * relevant on some platforms. + */ + writel(~status, engine->regs + CESA_SA_FPGA_INT_STATUS); + writel(~status, engine->regs + CESA_SA_INT_STATUS); + + /* Process fetched requests */ + res = mv_cesa_int_process(engine, status & mask); + ret = IRQ_HANDLED; + + spin_lock_bh(&engine->lock); + req = engine->req; + if (res != -EINPROGRESS) + engine->req = NULL; + spin_unlock_bh(&engine->lock); + + ctx = crypto_tfm_ctx(req->tfm); + + if (res && res != -EINPROGRESS) + mv_cesa_complete_req(ctx, req, res); + + /* Launch the next pending request */ + mv_cesa_rearm_engine(engine); + + /* Iterate over the complete queue */ + while (true) { + req = mv_cesa_engine_dequeue_complete_request(engine); + if (!req) + break; + + ctx = crypto_tfm_ctx(req->tfm); + mv_cesa_complete_req(ctx, req, 0); + } + } + + return ret; +} + +int mv_cesa_queue_req(struct crypto_async_request *req, + struct mv_cesa_req *creq) +{ + int ret; + struct mv_cesa_engine *engine = creq->engine; + + spin_lock_bh(&engine->lock); + ret = crypto_enqueue_request(&engine->queue, req); + if ((mv_cesa_req_get_type(creq) == CESA_DMA_REQ) && + (ret == -EINPROGRESS || ret == -EBUSY)) + mv_cesa_tdma_chain(engine, creq); + spin_unlock_bh(&engine->lock); + + if (ret != -EINPROGRESS) + return ret; + + mv_cesa_rearm_engine(engine); + + return -EINPROGRESS; +} + +static int mv_cesa_add_algs(struct mv_cesa_dev *cesa) +{ + int ret; + int i, j; + + for (i = 0; i < cesa->caps->ncipher_algs; i++) { + ret = crypto_register_skcipher(cesa->caps->cipher_algs[i]); + if (ret) + goto err_unregister_crypto; + } + + for (i = 0; i < cesa->caps->nahash_algs; i++) { + ret = crypto_register_ahash(cesa->caps->ahash_algs[i]); + if (ret) + goto err_unregister_ahash; + } + + return 0; + +err_unregister_ahash: + for (j = 0; j < i; j++) + crypto_unregister_ahash(cesa->caps->ahash_algs[j]); + i = cesa->caps->ncipher_algs; + +err_unregister_crypto: + for (j = 0; j < i; j++) + crypto_unregister_skcipher(cesa->caps->cipher_algs[j]); + + return ret; +} + +static void mv_cesa_remove_algs(struct mv_cesa_dev *cesa) +{ + int i; + + for (i = 0; i < cesa->caps->nahash_algs; i++) + crypto_unregister_ahash(cesa->caps->ahash_algs[i]); + + for (i = 0; i < cesa->caps->ncipher_algs; i++) + crypto_unregister_skcipher(cesa->caps->cipher_algs[i]); +} + +static struct skcipher_alg *orion_cipher_algs[] = { + &mv_cesa_ecb_des_alg, + &mv_cesa_cbc_des_alg, + &mv_cesa_ecb_des3_ede_alg, + &mv_cesa_cbc_des3_ede_alg, + &mv_cesa_ecb_aes_alg, + &mv_cesa_cbc_aes_alg, +}; + +static struct ahash_alg *orion_ahash_algs[] = { + &mv_md5_alg, + &mv_sha1_alg, + &mv_ahmac_md5_alg, + &mv_ahmac_sha1_alg, +}; + +static struct skcipher_alg *armada_370_cipher_algs[] = { + &mv_cesa_ecb_des_alg, + &mv_cesa_cbc_des_alg, + &mv_cesa_ecb_des3_ede_alg, + &mv_cesa_cbc_des3_ede_alg, + &mv_cesa_ecb_aes_alg, + &mv_cesa_cbc_aes_alg, +}; + +static struct ahash_alg *armada_370_ahash_algs[] = { + &mv_md5_alg, + &mv_sha1_alg, + &mv_sha256_alg, + &mv_ahmac_md5_alg, + &mv_ahmac_sha1_alg, + &mv_ahmac_sha256_alg, +}; + +static const struct mv_cesa_caps orion_caps = { + .nengines = 1, + .cipher_algs = orion_cipher_algs, + .ncipher_algs = ARRAY_SIZE(orion_cipher_algs), + .ahash_algs = orion_ahash_algs, + .nahash_algs = ARRAY_SIZE(orion_ahash_algs), + .has_tdma = false, +}; + +static const struct mv_cesa_caps kirkwood_caps = { + .nengines = 1, + .cipher_algs = orion_cipher_algs, + .ncipher_algs = ARRAY_SIZE(orion_cipher_algs), + .ahash_algs = orion_ahash_algs, + .nahash_algs = ARRAY_SIZE(orion_ahash_algs), + .has_tdma = true, +}; + +static const struct mv_cesa_caps armada_370_caps = { + .nengines = 1, + .cipher_algs = armada_370_cipher_algs, + .ncipher_algs = ARRAY_SIZE(armada_370_cipher_algs), + .ahash_algs = armada_370_ahash_algs, + .nahash_algs = ARRAY_SIZE(armada_370_ahash_algs), + .has_tdma = true, +}; + +static const struct mv_cesa_caps armada_xp_caps = { + .nengines = 2, + .cipher_algs = armada_370_cipher_algs, + .ncipher_algs = ARRAY_SIZE(armada_370_cipher_algs), + .ahash_algs = armada_370_ahash_algs, + .nahash_algs = ARRAY_SIZE(armada_370_ahash_algs), + .has_tdma = true, +}; + +static const struct of_device_id mv_cesa_of_match_table[] = { + { .compatible = "marvell,orion-crypto", .data = &orion_caps }, + { .compatible = "marvell,kirkwood-crypto", .data = &kirkwood_caps }, + { .compatible = "marvell,dove-crypto", .data = &kirkwood_caps }, + { .compatible = "marvell,armada-370-crypto", .data = &armada_370_caps }, + { .compatible = "marvell,armada-xp-crypto", .data = &armada_xp_caps }, + { .compatible = "marvell,armada-375-crypto", .data = &armada_xp_caps }, + { .compatible = "marvell,armada-38x-crypto", .data = &armada_xp_caps }, + {} +}; +MODULE_DEVICE_TABLE(of, mv_cesa_of_match_table); + +static void +mv_cesa_conf_mbus_windows(struct mv_cesa_engine *engine, + const struct mbus_dram_target_info *dram) +{ + void __iomem *iobase = engine->regs; + int i; + + for (i = 0; i < 4; i++) { + writel(0, iobase + CESA_TDMA_WINDOW_CTRL(i)); + writel(0, iobase + CESA_TDMA_WINDOW_BASE(i)); + } + + for (i = 0; i < dram->num_cs; i++) { + const struct mbus_dram_window *cs = dram->cs + i; + + writel(((cs->size - 1) & 0xffff0000) | + (cs->mbus_attr << 8) | + (dram->mbus_dram_target_id << 4) | 1, + iobase + CESA_TDMA_WINDOW_CTRL(i)); + writel(cs->base, iobase + CESA_TDMA_WINDOW_BASE(i)); + } +} + +static int mv_cesa_dev_dma_init(struct mv_cesa_dev *cesa) +{ + struct device *dev = cesa->dev; + struct mv_cesa_dev_dma *dma; + + if (!cesa->caps->has_tdma) + return 0; + + dma = devm_kzalloc(dev, sizeof(*dma), GFP_KERNEL); + if (!dma) + return -ENOMEM; + + dma->tdma_desc_pool = dmam_pool_create("tdma_desc", dev, + sizeof(struct mv_cesa_tdma_desc), + 16, 0); + if (!dma->tdma_desc_pool) + return -ENOMEM; + + dma->op_pool = dmam_pool_create("cesa_op", dev, + sizeof(struct mv_cesa_op_ctx), 16, 0); + if (!dma->op_pool) + return -ENOMEM; + + dma->cache_pool = dmam_pool_create("cesa_cache", dev, + CESA_MAX_HASH_BLOCK_SIZE, 1, 0); + if (!dma->cache_pool) + return -ENOMEM; + + dma->padding_pool = dmam_pool_create("cesa_padding", dev, 72, 1, 0); + if (!dma->padding_pool) + return -ENOMEM; + + cesa->dma = dma; + + return 0; +} + +static void mv_cesa_release_sram(void *data); + +static int mv_cesa_get_sram(struct platform_device *pdev, int idx) +{ + struct mv_cesa_dev *cesa = platform_get_drvdata(pdev); + struct mv_cesa_engine *engine = &cesa->engines[idx]; + struct resource *res; + + engine->pool = of_gen_pool_get(cesa->dev->of_node, + "marvell,crypto-srams", idx); + if (engine->pool) { + engine->sram_pool = gen_pool_dma_alloc(engine->pool, + cesa->sram_size, + &engine->sram_dma); + if (!engine->sram_pool) { + engine->pool = NULL; + return -ENOMEM; + } + + return devm_add_action_or_reset(cesa->dev, mv_cesa_release_sram, + engine); + } + + engine->sram = devm_platform_get_and_ioremap_resource(pdev, idx, &res); + if (IS_ERR(engine->sram)) + return PTR_ERR(engine->sram); + + engine->sram_dma = dma_map_resource(cesa->dev, res->start, + cesa->sram_size, + DMA_BIDIRECTIONAL, 0); + if (dma_mapping_error(cesa->dev, engine->sram_dma)) + return -ENOMEM; + + return devm_add_action_or_reset(cesa->dev, mv_cesa_release_sram, engine); +} + +static void mv_cesa_release_sram(void *data) +{ + struct mv_cesa_engine *engine = data; + struct mv_cesa_dev *cesa = engine->cesa; + + if (engine->pool) + gen_pool_free(engine->pool, (unsigned long)engine->sram_pool, + cesa->sram_size); + else + dma_unmap_resource(cesa->dev, engine->sram_dma, + cesa->sram_size, DMA_BIDIRECTIONAL, 0); +} + +static int mv_cesa_probe(struct platform_device *pdev) +{ + const struct mv_cesa_caps *caps = &orion_caps; + const struct mbus_dram_target_info *dram; + struct device *dev = &pdev->dev; + struct mv_cesa_dev *cesa; + struct mv_cesa_engine *engine; + int irq, ret, i, cpu; + u32 sram_size; + + if (cesa_dev) { + dev_err(&pdev->dev, "Only one CESA device authorized\n"); + return -EEXIST; + } + + if (dev->of_node) { + caps = of_device_get_match_data(dev); + if (!caps) + return -ENOTSUPP; + } + + cesa = devm_kzalloc(dev, struct_size(cesa, engines, caps->nengines), + GFP_KERNEL); + if (!cesa) + return -ENOMEM; + + cesa->caps = caps; + cesa->dev = dev; + + sram_size = CESA_SA_DEFAULT_SRAM_SIZE; + of_property_read_u32(cesa->dev->of_node, "marvell,crypto-sram-size", + &sram_size); + + cesa->sram_size = max(sram_size, CESA_SA_MIN_SRAM_SIZE); + + spin_lock_init(&cesa->lock); + + cesa->regs = devm_platform_ioremap_resource_byname(pdev, "regs"); + if (IS_ERR(cesa->regs)) + return PTR_ERR(cesa->regs); + + ret = mv_cesa_dev_dma_init(cesa); + if (ret) + return ret; + + dram = mv_mbus_dram_info_nooverlap(); + + platform_set_drvdata(pdev, cesa); + + for (i = 0; i < caps->nengines; i++) { + engine = &cesa->engines[i]; + char res_name[16]; + + engine->id = i; + engine->cesa = cesa; + spin_lock_init(&engine->lock); + + ret = mv_cesa_get_sram(pdev, i); + if (ret) + return ret; + + irq = platform_get_irq(pdev, i); + if (irq < 0) + return irq; + + engine->irq = irq; + + /* + * Not all platforms can gate the CESA clocks: do not complain + * if the clock does not exist. + */ + snprintf(res_name, sizeof(res_name), "cesa%u", i); + engine->clk = devm_clk_get_optional_enabled(dev, res_name); + if (IS_ERR(engine->clk)) { + engine->clk = devm_clk_get_optional_enabled(dev, NULL); + if (IS_ERR(engine->clk)) + return PTR_ERR(engine->clk); + } + + snprintf(res_name, sizeof(res_name), "cesaz%u", i); + engine->zclk = devm_clk_get_optional_enabled(dev, res_name); + if (IS_ERR(engine->zclk)) + return PTR_ERR(engine->zclk); + + engine->regs = cesa->regs + CESA_ENGINE_OFF(i); + + if (dram && cesa->caps->has_tdma) + mv_cesa_conf_mbus_windows(engine, dram); + + writel(0, engine->regs + CESA_SA_INT_STATUS); + writel(CESA_SA_CFG_STOP_DIG_ERR, + engine->regs + CESA_SA_CFG); + writel(engine->sram_dma & CESA_SA_SRAM_MSK, + engine->regs + CESA_SA_DESC_P0); + + ret = devm_request_threaded_irq(dev, irq, NULL, mv_cesa_int, + IRQF_ONESHOT, + dev_name(&pdev->dev), + engine); + if (ret) + return ret; + + /* Set affinity */ + cpu = cpumask_local_spread(engine->id, NUMA_NO_NODE); + irq_set_affinity_hint(irq, get_cpu_mask(cpu)); + + crypto_init_queue(&engine->queue, CESA_CRYPTO_DEFAULT_MAX_QLEN); + atomic_set(&engine->load, 0); + INIT_LIST_HEAD(&engine->complete_queue); + } + + cesa_dev = cesa; + + ret = mv_cesa_add_algs(cesa); + if (ret) { + cesa_dev = NULL; + return ret; + } + + dev_info(dev, "CESA device successfully registered\n"); + + return 0; +} + +static void mv_cesa_remove(struct platform_device *pdev) +{ + struct mv_cesa_dev *cesa = platform_get_drvdata(pdev); + + mv_cesa_remove_algs(cesa); + + cesa_dev = NULL; +} + +static const struct platform_device_id mv_cesa_plat_id_table[] = { + { .name = "mv_crypto" }, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(platform, mv_cesa_plat_id_table); + +static struct platform_driver marvell_cesa = { + .probe = mv_cesa_probe, + .remove = mv_cesa_remove, + .id_table = mv_cesa_plat_id_table, + .driver = { + .name = "marvell-cesa", + .of_match_table = mv_cesa_of_match_table, + }, +}; +module_platform_driver(marvell_cesa); + +MODULE_AUTHOR("Boris Brezillon "); +MODULE_AUTHOR("Arnaud Ebalard "); +MODULE_DESCRIPTION("Support for Marvell's cryptographic engine"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/crypto/marvell/cesa/cesa.h b/drivers/crypto/marvell/cesa/cesa.h new file mode 100644 index 000000000..44351b252 --- /dev/null +++ b/drivers/crypto/marvell/cesa/cesa.h @@ -0,0 +1,920 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __MARVELL_CESA_H__ +#define __MARVELL_CESA_H__ + +#include +#include + +#include +#include + +#define CESA_ENGINE_OFF(i) (((i) * 0x2000)) + +#define CESA_TDMA_BYTE_CNT 0x800 +#define CESA_TDMA_SRC_ADDR 0x810 +#define CESA_TDMA_DST_ADDR 0x820 +#define CESA_TDMA_NEXT_ADDR 0x830 + +#define CESA_TDMA_CONTROL 0x840 +#define CESA_TDMA_DST_BURST GENMASK(2, 0) +#define CESA_TDMA_DST_BURST_32B 3 +#define CESA_TDMA_DST_BURST_128B 4 +#define CESA_TDMA_OUT_RD_EN BIT(4) +#define CESA_TDMA_SRC_BURST GENMASK(8, 6) +#define CESA_TDMA_SRC_BURST_32B (3 << 6) +#define CESA_TDMA_SRC_BURST_128B (4 << 6) +#define CESA_TDMA_CHAIN BIT(9) +#define CESA_TDMA_BYTE_SWAP BIT(11) +#define CESA_TDMA_NO_BYTE_SWAP BIT(11) +#define CESA_TDMA_EN BIT(12) +#define CESA_TDMA_FETCH_ND BIT(13) +#define CESA_TDMA_ACT BIT(14) + +#define CESA_TDMA_CUR 0x870 +#define CESA_TDMA_ERROR_CAUSE 0x8c8 +#define CESA_TDMA_ERROR_MSK 0x8cc + +#define CESA_TDMA_WINDOW_BASE(x) (((x) * 0x8) + 0xa00) +#define CESA_TDMA_WINDOW_CTRL(x) (((x) * 0x8) + 0xa04) + +#define CESA_IVDIG(x) (0xdd00 + ((x) * 4) + \ + (((x) < 5) ? 0 : 0x14)) + +#define CESA_SA_CMD 0xde00 +#define CESA_SA_CMD_EN_CESA_SA_ACCL0 BIT(0) +#define CESA_SA_CMD_EN_CESA_SA_ACCL1 BIT(1) +#define CESA_SA_CMD_DISABLE_SEC BIT(2) + +#define CESA_SA_DESC_P0 0xde04 + +#define CESA_SA_DESC_P1 0xde14 + +#define CESA_SA_CFG 0xde08 +#define CESA_SA_CFG_STOP_DIG_ERR GENMASK(1, 0) +#define CESA_SA_CFG_DIG_ERR_CONT 0 +#define CESA_SA_CFG_DIG_ERR_SKIP 1 +#define CESA_SA_CFG_DIG_ERR_STOP 3 +#define CESA_SA_CFG_CH0_W_IDMA BIT(7) +#define CESA_SA_CFG_CH1_W_IDMA BIT(8) +#define CESA_SA_CFG_ACT_CH0_IDMA BIT(9) +#define CESA_SA_CFG_ACT_CH1_IDMA BIT(10) +#define CESA_SA_CFG_MULTI_PKT BIT(11) +#define CESA_SA_CFG_PARA_DIS BIT(13) + +#define CESA_SA_ACCEL_STATUS 0xde0c +#define CESA_SA_ST_ACT_0 BIT(0) +#define CESA_SA_ST_ACT_1 BIT(1) + +/* + * CESA_SA_FPGA_INT_STATUS looks like an FPGA leftover and is documented only + * in Errata 4.12. It looks like that it was part of an IRQ-controller in FPGA + * and someone forgot to remove it while switching to the core and moving to + * CESA_SA_INT_STATUS. + */ +#define CESA_SA_FPGA_INT_STATUS 0xdd68 +#define CESA_SA_INT_STATUS 0xde20 +#define CESA_SA_INT_AUTH_DONE BIT(0) +#define CESA_SA_INT_DES_E_DONE BIT(1) +#define CESA_SA_INT_AES_E_DONE BIT(2) +#define CESA_SA_INT_AES_D_DONE BIT(3) +#define CESA_SA_INT_ENC_DONE BIT(4) +#define CESA_SA_INT_ACCEL0_DONE BIT(5) +#define CESA_SA_INT_ACCEL1_DONE BIT(6) +#define CESA_SA_INT_ACC0_IDMA_DONE BIT(7) +#define CESA_SA_INT_ACC1_IDMA_DONE BIT(8) +#define CESA_SA_INT_IDMA_DONE BIT(9) +#define CESA_SA_INT_IDMA_OWN_ERR BIT(10) + +#define CESA_SA_INT_MSK 0xde24 + +#define CESA_SA_DESC_CFG_OP_MAC_ONLY 0 +#define CESA_SA_DESC_CFG_OP_CRYPT_ONLY 1 +#define CESA_SA_DESC_CFG_OP_MAC_CRYPT 2 +#define CESA_SA_DESC_CFG_OP_CRYPT_MAC 3 +#define CESA_SA_DESC_CFG_OP_MSK GENMASK(1, 0) +#define CESA_SA_DESC_CFG_MACM_SHA256 (1 << 4) +#define CESA_SA_DESC_CFG_MACM_HMAC_SHA256 (3 << 4) +#define CESA_SA_DESC_CFG_MACM_MD5 (4 << 4) +#define CESA_SA_DESC_CFG_MACM_SHA1 (5 << 4) +#define CESA_SA_DESC_CFG_MACM_HMAC_MD5 (6 << 4) +#define CESA_SA_DESC_CFG_MACM_HMAC_SHA1 (7 << 4) +#define CESA_SA_DESC_CFG_MACM_MSK GENMASK(6, 4) +#define CESA_SA_DESC_CFG_CRYPTM_DES (1 << 8) +#define CESA_SA_DESC_CFG_CRYPTM_3DES (2 << 8) +#define CESA_SA_DESC_CFG_CRYPTM_AES (3 << 8) +#define CESA_SA_DESC_CFG_CRYPTM_MSK GENMASK(9, 8) +#define CESA_SA_DESC_CFG_DIR_ENC (0 << 12) +#define CESA_SA_DESC_CFG_DIR_DEC (1 << 12) +#define CESA_SA_DESC_CFG_CRYPTCM_ECB (0 << 16) +#define CESA_SA_DESC_CFG_CRYPTCM_CBC (1 << 16) +#define CESA_SA_DESC_CFG_CRYPTCM_MSK BIT(16) +#define CESA_SA_DESC_CFG_3DES_EEE (0 << 20) +#define CESA_SA_DESC_CFG_3DES_EDE (1 << 20) +#define CESA_SA_DESC_CFG_AES_LEN_128 (0 << 24) +#define CESA_SA_DESC_CFG_AES_LEN_192 (1 << 24) +#define CESA_SA_DESC_CFG_AES_LEN_256 (2 << 24) +#define CESA_SA_DESC_CFG_AES_LEN_MSK GENMASK(25, 24) +#define CESA_SA_DESC_CFG_NOT_FRAG (0 << 30) +#define CESA_SA_DESC_CFG_FIRST_FRAG (1 << 30) +#define CESA_SA_DESC_CFG_LAST_FRAG (2 << 30) +#define CESA_SA_DESC_CFG_MID_FRAG (3 << 30) +#define CESA_SA_DESC_CFG_FRAG_MSK GENMASK(31, 30) + +/* + * /-----------\ 0 + * | ACCEL CFG | 4 * 8 + * |-----------| 0x20 + * | CRYPT KEY | 8 * 4 + * |-----------| 0x40 + * | IV IN | 4 * 4 + * |-----------| 0x40 (inplace) + * | IV BUF | 4 * 4 + * |-----------| 0x80 + * | DATA IN | 16 * x (max ->max_req_size) + * |-----------| 0x80 (inplace operation) + * | DATA OUT | 16 * x (max ->max_req_size) + * \-----------/ SRAM size + */ + +/* + * Hashing memory map: + * /-----------\ 0 + * | ACCEL CFG | 4 * 8 + * |-----------| 0x20 + * | Inner IV | 8 * 4 + * |-----------| 0x40 + * | Outer IV | 8 * 4 + * |-----------| 0x60 + * | Output BUF| 8 * 4 + * |-----------| 0x80 + * | DATA IN | 64 * x (max ->max_req_size) + * \-----------/ SRAM size + */ + +#define CESA_SA_CFG_SRAM_OFFSET 0x00 +#define CESA_SA_DATA_SRAM_OFFSET 0x80 + +#define CESA_SA_CRYPT_KEY_SRAM_OFFSET 0x20 +#define CESA_SA_CRYPT_IV_SRAM_OFFSET 0x40 + +#define CESA_SA_MAC_IIV_SRAM_OFFSET 0x20 +#define CESA_SA_MAC_OIV_SRAM_OFFSET 0x40 +#define CESA_SA_MAC_DIG_SRAM_OFFSET 0x60 + +#define CESA_SA_DESC_CRYPT_DATA(offset) \ + cpu_to_le32((CESA_SA_DATA_SRAM_OFFSET + (offset)) | \ + ((CESA_SA_DATA_SRAM_OFFSET + (offset)) << 16)) + +#define CESA_SA_DESC_CRYPT_IV(offset) \ + cpu_to_le32((CESA_SA_CRYPT_IV_SRAM_OFFSET + (offset)) | \ + ((CESA_SA_CRYPT_IV_SRAM_OFFSET + (offset)) << 16)) + +#define CESA_SA_DESC_CRYPT_KEY(offset) \ + cpu_to_le32(CESA_SA_CRYPT_KEY_SRAM_OFFSET + (offset)) + +#define CESA_SA_DESC_MAC_DATA(offset) \ + cpu_to_le32(CESA_SA_DATA_SRAM_OFFSET + (offset)) +#define CESA_SA_DESC_MAC_DATA_MSK cpu_to_le32(GENMASK(15, 0)) + +#define CESA_SA_DESC_MAC_TOTAL_LEN(total_len) cpu_to_le32((total_len) << 16) +#define CESA_SA_DESC_MAC_TOTAL_LEN_MSK cpu_to_le32(GENMASK(31, 16)) + +#define CESA_SA_DESC_MAC_SRC_TOTAL_LEN_MAX 0xffff + +#define CESA_SA_DESC_MAC_DIGEST(offset) \ + cpu_to_le32(CESA_SA_MAC_DIG_SRAM_OFFSET + (offset)) +#define CESA_SA_DESC_MAC_DIGEST_MSK cpu_to_le32(GENMASK(15, 0)) + +#define CESA_SA_DESC_MAC_FRAG_LEN(frag_len) cpu_to_le32((frag_len) << 16) +#define CESA_SA_DESC_MAC_FRAG_LEN_MSK cpu_to_le32(GENMASK(31, 16)) + +#define CESA_SA_DESC_MAC_IV(offset) \ + cpu_to_le32((CESA_SA_MAC_IIV_SRAM_OFFSET + (offset)) | \ + ((CESA_SA_MAC_OIV_SRAM_OFFSET + (offset)) << 16)) + +#define CESA_SA_SRAM_SIZE 2048 +#define CESA_SA_SRAM_PAYLOAD_SIZE (cesa_dev->sram_size - \ + CESA_SA_DATA_SRAM_OFFSET) + +#define CESA_SA_DEFAULT_SRAM_SIZE 2048 +#define CESA_SA_MIN_SRAM_SIZE 1024 + +#define CESA_SA_SRAM_MSK (2048 - 1) + +#define CESA_MAX_HASH_BLOCK_SIZE 64 +#define CESA_HASH_BLOCK_SIZE_MSK (CESA_MAX_HASH_BLOCK_SIZE - 1) + +/** + * struct mv_cesa_sec_accel_desc - security accelerator descriptor + * @config: engine config + * @enc_p: input and output data pointers for a cipher operation + * @enc_len: cipher operation length + * @enc_key_p: cipher key pointer + * @enc_iv: cipher IV pointers + * @mac_src_p: input pointer and total hash length + * @mac_digest: digest pointer and hash operation length + * @mac_iv: hmac IV pointers + * + * Structure passed to the CESA engine to describe the crypto operation + * to be executed. + */ +struct mv_cesa_sec_accel_desc { + __le32 config; + __le32 enc_p; + __le32 enc_len; + __le32 enc_key_p; + __le32 enc_iv; + __le32 mac_src_p; + __le32 mac_digest; + __le32 mac_iv; +}; + +/** + * struct mv_cesa_skcipher_op_ctx - cipher operation context + * @key: cipher key + * @iv: cipher IV + * + * Context associated to a cipher operation. + */ +struct mv_cesa_skcipher_op_ctx { + __le32 key[8]; + u32 iv[4]; +}; + +/** + * struct mv_cesa_hash_op_ctx - hash or hmac operation context + * @key: cipher key + * @iv: cipher IV + * + * Context associated to an hash or hmac operation. + */ +struct mv_cesa_hash_op_ctx { + u32 iv[16]; + __le32 hash[8]; +}; + +/** + * struct mv_cesa_op_ctx - crypto operation context + * @desc: CESA descriptor + * @ctx: context associated to the crypto operation + * + * Context associated to a crypto operation. + */ +struct mv_cesa_op_ctx { + struct mv_cesa_sec_accel_desc desc; + union { + struct mv_cesa_skcipher_op_ctx skcipher; + struct mv_cesa_hash_op_ctx hash; + } ctx; +}; + +/* TDMA descriptor flags */ +#define CESA_TDMA_DST_IN_SRAM BIT(31) +#define CESA_TDMA_SRC_IN_SRAM BIT(30) +#define CESA_TDMA_END_OF_REQ BIT(29) +#define CESA_TDMA_BREAK_CHAIN BIT(28) +#define CESA_TDMA_SET_STATE BIT(27) +#define CESA_TDMA_TYPE_MSK GENMASK(26, 0) +#define CESA_TDMA_DUMMY 0 +#define CESA_TDMA_DATA 1 +#define CESA_TDMA_OP 2 +#define CESA_TDMA_RESULT 3 + +/** + * struct mv_cesa_tdma_desc - TDMA descriptor + * @byte_cnt: number of bytes to transfer + * @src: DMA address of the source + * @dst: DMA address of the destination + * @next_dma: DMA address of the next TDMA descriptor + * @cur_dma: DMA address of this TDMA descriptor + * @next: pointer to the next TDMA descriptor + * @op: CESA operation attached to this TDMA descriptor + * @data: raw data attached to this TDMA descriptor + * @flags: flags describing the TDMA transfer. See the + * "TDMA descriptor flags" section above + * + * TDMA descriptor used to create a transfer chain describing a crypto + * operation. + */ +struct mv_cesa_tdma_desc { + __le32 byte_cnt; + union { + __le32 src; + u32 src_dma; + }; + union { + __le32 dst; + u32 dst_dma; + }; + __le32 next_dma; + + /* Software state */ + dma_addr_t cur_dma; + struct mv_cesa_tdma_desc *next; + union { + struct mv_cesa_op_ctx *op; + void *data; + }; + u32 flags; +}; + +/** + * struct mv_cesa_sg_dma_iter - scatter-gather iterator + * @dir: transfer direction + * @sg: scatter list + * @offset: current position in the scatter list + * @op_offset: current position in the crypto operation + * + * Iterator used to iterate over a scatterlist while creating a TDMA chain for + * a crypto operation. + */ +struct mv_cesa_sg_dma_iter { + enum dma_data_direction dir; + struct scatterlist *sg; + unsigned int offset; + unsigned int op_offset; +}; + +/** + * struct mv_cesa_dma_iter - crypto operation iterator + * @len: the crypto operation length + * @offset: current position in the crypto operation + * @op_len: sub-operation length (the crypto engine can only act on 2kb + * chunks) + * + * Iterator used to create a TDMA chain for a given crypto operation. + */ +struct mv_cesa_dma_iter { + unsigned int len; + unsigned int offset; + unsigned int op_len; +}; + +/** + * struct mv_cesa_tdma_chain - TDMA chain + * @first: first entry in the TDMA chain + * @last: last entry in the TDMA chain + * + * Stores a TDMA chain for a specific crypto operation. + */ +struct mv_cesa_tdma_chain { + struct mv_cesa_tdma_desc *first; + struct mv_cesa_tdma_desc *last; +}; + +struct mv_cesa_engine; + +/** + * struct mv_cesa_caps - CESA device capabilities + * @engines: number of engines + * @has_tdma: whether this device has a TDMA block + * @cipher_algs: supported cipher algorithms + * @ncipher_algs: number of supported cipher algorithms + * @ahash_algs: supported hash algorithms + * @nahash_algs: number of supported hash algorithms + * + * Structure used to describe CESA device capabilities. + */ +struct mv_cesa_caps { + int nengines; + bool has_tdma; + struct skcipher_alg **cipher_algs; + int ncipher_algs; + struct ahash_alg **ahash_algs; + int nahash_algs; +}; + +/** + * struct mv_cesa_dev_dma - DMA pools + * @tdma_desc_pool: TDMA desc pool + * @op_pool: crypto operation pool + * @cache_pool: data cache pool (used by hash implementation when the + * hash request is smaller than the hash block size) + * @padding_pool: padding pool (used by hash implementation when hardware + * padding cannot be used) + * + * Structure containing the different DMA pools used by this driver. + */ +struct mv_cesa_dev_dma { + struct dma_pool *tdma_desc_pool; + struct dma_pool *op_pool; + struct dma_pool *cache_pool; + struct dma_pool *padding_pool; +}; + +/** + * struct mv_cesa_engine - CESA engine + * @id: engine id + * @regs: engine registers + * @sram: SRAM memory region + * @sram_pool: SRAM memory region from pool + * @sram_dma: DMA address of the SRAM memory region + * @lock: engine lock + * @req: current crypto request + * @clk: engine clk + * @zclk: engine zclk + * @max_req_len: maximum chunk length (useful to create the TDMA chain) + * @int_mask: interrupt mask cache + * @cesa: back-pointer to the parent CESA device + * @pool: memory pool pointing to the memory region reserved in + * SRAM + * @queue: fifo of the pending crypto requests + * @load: engine load counter, useful for load balancing + * @chain_hw: list of the current tdma descriptors being processed + * by the hardware. + * @chain_sw: list of the current tdma descriptors that will be + * submitted to the hardware. + * @complete_queue: fifo of the processed requests by the engine + * + * Structure storing CESA engine information. + */ +struct mv_cesa_engine { + int id; + void __iomem *regs; + union { + void __iomem *sram; + void *sram_pool; + }; + dma_addr_t sram_dma; + spinlock_t lock; + struct crypto_async_request *req; + struct clk *clk; + struct clk *zclk; + size_t max_req_len; + u32 int_mask; + struct mv_cesa_dev *cesa; + struct gen_pool *pool; + struct crypto_queue queue; + atomic_t load; + struct mv_cesa_tdma_chain chain_hw; + struct mv_cesa_tdma_chain chain_sw; + struct list_head complete_queue; + int irq; +}; + +/** + * struct mv_cesa_dev - CESA device + * @caps: device capabilities + * @regs: device registers + * @sram_size: usable SRAM size + * @lock: device lock + * @dma: dma pools + * @engines: array of engines + * + * Structure storing CESA device information. + */ +struct mv_cesa_dev { + const struct mv_cesa_caps *caps; + void __iomem *regs; + struct device *dev; + unsigned int sram_size; + spinlock_t lock; + struct mv_cesa_dev_dma *dma; + struct mv_cesa_engine engines[]; +}; + +/** + * struct mv_cesa_req_ops - CESA request operations + * @process: process a request chunk result (should return 0 if the + * operation, -EINPROGRESS if it needs more steps or an error + * code) + * @step: launch the crypto operation on the next chunk + * @cleanup: cleanup the crypto request (release associated data) + * @complete: complete the request, i.e copy result or context from sram when + * needed. + */ +struct mv_cesa_req_ops { + int (*process)(struct crypto_async_request *req, u32 status); + void (*step)(struct crypto_async_request *req); + void (*cleanup)(struct crypto_async_request *req); + void (*complete)(struct crypto_async_request *req); +}; + +/** + * struct mv_cesa_ctx - CESA operation context + * @ops: crypto operations + * + * Base context structure inherited by operation specific ones. + */ +struct mv_cesa_ctx { + const struct mv_cesa_req_ops *ops; +}; + +/** + * struct mv_cesa_hash_ctx - CESA hash operation context + * @base: base context structure + * + * Hash context structure. + */ +struct mv_cesa_hash_ctx { + struct mv_cesa_ctx base; +}; + +/** + * struct mv_cesa_hash_ctx - CESA hmac operation context + * @base: base context structure + * @iv: initialization vectors + * + * HMAC context structure. + */ +struct mv_cesa_hmac_ctx { + struct mv_cesa_ctx base; + __be32 iv[16]; +}; + +/** + * enum mv_cesa_req_type - request type definitions + * @CESA_STD_REQ: standard request + * @CESA_DMA_REQ: DMA request + */ +enum mv_cesa_req_type { + CESA_STD_REQ, + CESA_DMA_REQ, +}; + +/** + * struct mv_cesa_req - CESA request + * @engine: engine associated with this request + * @chain: list of tdma descriptors associated with this request + */ +struct mv_cesa_req { + struct mv_cesa_engine *engine; + struct mv_cesa_tdma_chain chain; +}; + +/** + * struct mv_cesa_sg_std_iter - CESA scatter-gather iterator for standard + * requests + * @iter: sg mapping iterator + * @offset: current offset in the SG entry mapped in memory + */ +struct mv_cesa_sg_std_iter { + struct sg_mapping_iter iter; + unsigned int offset; +}; + +/** + * struct mv_cesa_skcipher_std_req - cipher standard request + * @op: operation context + * @offset: current operation offset + * @size: size of the crypto operation + */ +struct mv_cesa_skcipher_std_req { + struct mv_cesa_op_ctx op; + unsigned int offset; + unsigned int size; + bool skip_ctx; +}; + +/** + * struct mv_cesa_skcipher_req - cipher request + * @req: type specific request information + * @src_nents: number of entries in the src sg list + * @dst_nents: number of entries in the dest sg list + */ +struct mv_cesa_skcipher_req { + struct mv_cesa_req base; + struct mv_cesa_skcipher_std_req std; + int src_nents; + int dst_nents; +}; + +/** + * struct mv_cesa_ahash_std_req - standard hash request + * @offset: current operation offset + */ +struct mv_cesa_ahash_std_req { + unsigned int offset; +}; + +/** + * struct mv_cesa_ahash_dma_req - DMA hash request + * @padding: padding buffer + * @padding_dma: DMA address of the padding buffer + * @cache_dma: DMA address of the cache buffer + */ +struct mv_cesa_ahash_dma_req { + u8 *padding; + dma_addr_t padding_dma; + u8 *cache; + dma_addr_t cache_dma; +}; + +/** + * struct mv_cesa_ahash_req - hash request + * @req: type specific request information + * @cache: cache buffer + * @cache_ptr: write pointer in the cache buffer + * @len: hash total length + * @src_nents: number of entries in the scatterlist + * @last_req: define whether the current operation is the last one + * or not + * @state: hash state + */ +struct mv_cesa_ahash_req { + struct mv_cesa_req base; + union { + struct mv_cesa_ahash_dma_req dma; + struct mv_cesa_ahash_std_req std; + } req; + struct mv_cesa_op_ctx op_tmpl; + u8 cache[CESA_MAX_HASH_BLOCK_SIZE]; + unsigned int cache_ptr; + u64 len; + int src_nents; + bool last_req; + bool algo_le; + u32 state[8]; +}; + +/* CESA functions */ + +extern struct mv_cesa_dev *cesa_dev; + + +static inline void +mv_cesa_engine_enqueue_complete_request(struct mv_cesa_engine *engine, + struct crypto_async_request *req) +{ + list_add_tail(&req->list, &engine->complete_queue); +} + +static inline struct crypto_async_request * +mv_cesa_engine_dequeue_complete_request(struct mv_cesa_engine *engine) +{ + struct crypto_async_request *req; + + req = list_first_entry_or_null(&engine->complete_queue, + struct crypto_async_request, + list); + if (req) + list_del(&req->list); + + return req; +} + + +static inline enum mv_cesa_req_type +mv_cesa_req_get_type(struct mv_cesa_req *req) +{ + return req->chain.first ? CESA_DMA_REQ : CESA_STD_REQ; +} + +static inline void mv_cesa_update_op_cfg(struct mv_cesa_op_ctx *op, + u32 cfg, u32 mask) +{ + op->desc.config &= cpu_to_le32(~mask); + op->desc.config |= cpu_to_le32(cfg); +} + +static inline u32 mv_cesa_get_op_cfg(const struct mv_cesa_op_ctx *op) +{ + return le32_to_cpu(op->desc.config); +} + +static inline void mv_cesa_set_op_cfg(struct mv_cesa_op_ctx *op, u32 cfg) +{ + op->desc.config = cpu_to_le32(cfg); +} + +static inline void mv_cesa_adjust_op(struct mv_cesa_engine *engine, + struct mv_cesa_op_ctx *op) +{ + u32 offset = engine->sram_dma & CESA_SA_SRAM_MSK; + + op->desc.enc_p = CESA_SA_DESC_CRYPT_DATA(offset); + op->desc.enc_key_p = CESA_SA_DESC_CRYPT_KEY(offset); + op->desc.enc_iv = CESA_SA_DESC_CRYPT_IV(offset); + op->desc.mac_src_p &= ~CESA_SA_DESC_MAC_DATA_MSK; + op->desc.mac_src_p |= CESA_SA_DESC_MAC_DATA(offset); + op->desc.mac_digest &= ~CESA_SA_DESC_MAC_DIGEST_MSK; + op->desc.mac_digest |= CESA_SA_DESC_MAC_DIGEST(offset); + op->desc.mac_iv = CESA_SA_DESC_MAC_IV(offset); +} + +static inline void mv_cesa_set_crypt_op_len(struct mv_cesa_op_ctx *op, int len) +{ + op->desc.enc_len = cpu_to_le32(len); +} + +static inline void mv_cesa_set_mac_op_total_len(struct mv_cesa_op_ctx *op, + int len) +{ + op->desc.mac_src_p &= ~CESA_SA_DESC_MAC_TOTAL_LEN_MSK; + op->desc.mac_src_p |= CESA_SA_DESC_MAC_TOTAL_LEN(len); +} + +static inline void mv_cesa_set_mac_op_frag_len(struct mv_cesa_op_ctx *op, + int len) +{ + op->desc.mac_digest &= ~CESA_SA_DESC_MAC_FRAG_LEN_MSK; + op->desc.mac_digest |= CESA_SA_DESC_MAC_FRAG_LEN(len); +} + +static inline void mv_cesa_set_int_mask(struct mv_cesa_engine *engine, + u32 int_mask) +{ + if (int_mask == engine->int_mask) + return; + + writel_relaxed(int_mask, engine->regs + CESA_SA_INT_MSK); + engine->int_mask = int_mask; +} + +static inline u32 mv_cesa_get_int_mask(struct mv_cesa_engine *engine) +{ + return engine->int_mask; +} + +static inline bool mv_cesa_mac_op_is_first_frag(const struct mv_cesa_op_ctx *op) +{ + return (mv_cesa_get_op_cfg(op) & CESA_SA_DESC_CFG_FRAG_MSK) == + CESA_SA_DESC_CFG_FIRST_FRAG; +} + +int mv_cesa_queue_req(struct crypto_async_request *req, + struct mv_cesa_req *creq); + +struct crypto_async_request * +mv_cesa_dequeue_req_locked(struct mv_cesa_engine *engine, + struct crypto_async_request **backlog); + +static inline struct mv_cesa_engine *mv_cesa_select_engine(int weight) +{ + int i; + u32 min_load = U32_MAX; + struct mv_cesa_engine *selected = NULL; + + for (i = 0; i < cesa_dev->caps->nengines; i++) { + struct mv_cesa_engine *engine = cesa_dev->engines + i; + u32 load = atomic_read(&engine->load); + + if (load < min_load) { + min_load = load; + selected = engine; + } + } + + atomic_add(weight, &selected->load); + + return selected; +} + +/* + * Helper function that indicates whether a crypto request needs to be + * cleaned up or not after being enqueued using mv_cesa_queue_req(). + */ +static inline int mv_cesa_req_needs_cleanup(struct crypto_async_request *req, + int ret) +{ + /* + * The queue still had some space, the request was queued + * normally, so there's no need to clean it up. + */ + if (ret == -EINPROGRESS) + return false; + + /* + * The queue had not space left, but since the request is + * flagged with CRYPTO_TFM_REQ_MAY_BACKLOG, it was added to + * the backlog and will be processed later. There's no need to + * clean it up. + */ + if (ret == -EBUSY) + return false; + + /* Request wasn't queued, we need to clean it up */ + return true; +} + +/* TDMA functions */ + +static inline void mv_cesa_req_dma_iter_init(struct mv_cesa_dma_iter *iter, + unsigned int len) +{ + iter->len = len; + iter->op_len = min(len, CESA_SA_SRAM_PAYLOAD_SIZE); + iter->offset = 0; +} + +static inline void mv_cesa_sg_dma_iter_init(struct mv_cesa_sg_dma_iter *iter, + struct scatterlist *sg, + enum dma_data_direction dir) +{ + iter->op_offset = 0; + iter->offset = 0; + iter->sg = sg; + iter->dir = dir; +} + +static inline unsigned int +mv_cesa_req_dma_iter_transfer_len(struct mv_cesa_dma_iter *iter, + struct mv_cesa_sg_dma_iter *sgiter) +{ + return min(iter->op_len - sgiter->op_offset, + sg_dma_len(sgiter->sg) - sgiter->offset); +} + +bool mv_cesa_req_dma_iter_next_transfer(struct mv_cesa_dma_iter *chain, + struct mv_cesa_sg_dma_iter *sgiter, + unsigned int len); + +static inline bool mv_cesa_req_dma_iter_next_op(struct mv_cesa_dma_iter *iter) +{ + iter->offset += iter->op_len; + iter->op_len = min(iter->len - iter->offset, + CESA_SA_SRAM_PAYLOAD_SIZE); + + return iter->op_len; +} + +void mv_cesa_dma_step(struct mv_cesa_req *dreq); + +static inline int mv_cesa_dma_process(struct mv_cesa_req *dreq, + u32 status) +{ + if (!(status & CESA_SA_INT_ACC0_IDMA_DONE)) + return -EINPROGRESS; + + if (status & CESA_SA_INT_IDMA_OWN_ERR) + return -EINVAL; + + return 0; +} + +void mv_cesa_dma_prepare(struct mv_cesa_req *dreq, + struct mv_cesa_engine *engine); +void mv_cesa_dma_cleanup(struct mv_cesa_req *dreq); +void mv_cesa_tdma_chain(struct mv_cesa_engine *engine, + struct mv_cesa_req *dreq); +int mv_cesa_tdma_process(struct mv_cesa_engine *engine, u32 status); + + +static inline void +mv_cesa_tdma_desc_iter_init(struct mv_cesa_tdma_chain *chain) +{ + memset(chain, 0, sizeof(*chain)); +} + +int mv_cesa_dma_add_result_op(struct mv_cesa_tdma_chain *chain, dma_addr_t src, + u32 size, u32 flags, gfp_t gfp_flags); + +struct mv_cesa_op_ctx *mv_cesa_dma_add_op(struct mv_cesa_tdma_chain *chain, + const struct mv_cesa_op_ctx *op_templ, + bool skip_ctx, + gfp_t flags); + +int mv_cesa_dma_add_data_transfer(struct mv_cesa_tdma_chain *chain, + dma_addr_t dst, dma_addr_t src, u32 size, + u32 flags, gfp_t gfp_flags); + +int mv_cesa_dma_add_dummy_launch(struct mv_cesa_tdma_chain *chain, gfp_t flags); +int mv_cesa_dma_add_dummy_end(struct mv_cesa_tdma_chain *chain, gfp_t flags); + +int mv_cesa_dma_add_op_transfers(struct mv_cesa_tdma_chain *chain, + struct mv_cesa_dma_iter *dma_iter, + struct mv_cesa_sg_dma_iter *sgiter, + gfp_t gfp_flags); + +size_t mv_cesa_sg_copy(struct mv_cesa_engine *engine, + struct scatterlist *sgl, unsigned int nents, + unsigned int sram_off, size_t buflen, off_t skip, + bool to_sram); + +static inline size_t mv_cesa_sg_copy_to_sram(struct mv_cesa_engine *engine, + struct scatterlist *sgl, + unsigned int nents, + unsigned int sram_off, + size_t buflen, off_t skip) +{ + return mv_cesa_sg_copy(engine, sgl, nents, sram_off, buflen, skip, + true); +} + +static inline size_t mv_cesa_sg_copy_from_sram(struct mv_cesa_engine *engine, + struct scatterlist *sgl, + unsigned int nents, + unsigned int sram_off, + size_t buflen, off_t skip) +{ + return mv_cesa_sg_copy(engine, sgl, nents, sram_off, buflen, skip, + false); +} + +/* Algorithm definitions */ + +extern struct ahash_alg mv_md5_alg; +extern struct ahash_alg mv_sha1_alg; +extern struct ahash_alg mv_sha256_alg; +extern struct ahash_alg mv_ahmac_md5_alg; +extern struct ahash_alg mv_ahmac_sha1_alg; +extern struct ahash_alg mv_ahmac_sha256_alg; + +extern struct skcipher_alg mv_cesa_ecb_des_alg; +extern struct skcipher_alg mv_cesa_cbc_des_alg; +extern struct skcipher_alg mv_cesa_ecb_des3_ede_alg; +extern struct skcipher_alg mv_cesa_cbc_des3_ede_alg; +extern struct skcipher_alg mv_cesa_ecb_aes_alg; +extern struct skcipher_alg mv_cesa_cbc_aes_alg; + +#endif /* __MARVELL_CESA_H__ */ diff --git a/drivers/crypto/marvell/cesa/cipher.c b/drivers/crypto/marvell/cesa/cipher.c new file mode 100644 index 000000000..3fe0fd922 --- /dev/null +++ b/drivers/crypto/marvell/cesa/cipher.c @@ -0,0 +1,822 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Cipher algorithms supported by the CESA: DES, 3DES and AES. + * + * Author: Boris Brezillon + * Author: Arnaud Ebalard + * + * This work is based on an initial version written by + * Sebastian Andrzej Siewior < sebastian at breakpoint dot cc > + */ + +#include +#include +#include +#include + +#include "cesa.h" + +struct mv_cesa_des_ctx { + struct mv_cesa_ctx base; + u8 key[DES_KEY_SIZE]; +}; + +struct mv_cesa_des3_ctx { + struct mv_cesa_ctx base; + u8 key[DES3_EDE_KEY_SIZE]; +}; + +struct mv_cesa_aes_ctx { + struct mv_cesa_ctx base; + struct crypto_aes_ctx aes; +}; + +struct mv_cesa_skcipher_dma_iter { + struct mv_cesa_dma_iter base; + struct mv_cesa_sg_dma_iter src; + struct mv_cesa_sg_dma_iter dst; +}; + +static inline void +mv_cesa_skcipher_req_iter_init(struct mv_cesa_skcipher_dma_iter *iter, + struct skcipher_request *req) +{ + mv_cesa_req_dma_iter_init(&iter->base, req->cryptlen); + mv_cesa_sg_dma_iter_init(&iter->src, req->src, DMA_TO_DEVICE); + mv_cesa_sg_dma_iter_init(&iter->dst, req->dst, DMA_FROM_DEVICE); +} + +static inline bool +mv_cesa_skcipher_req_iter_next_op(struct mv_cesa_skcipher_dma_iter *iter) +{ + iter->src.op_offset = 0; + iter->dst.op_offset = 0; + + return mv_cesa_req_dma_iter_next_op(&iter->base); +} + +static inline void +mv_cesa_skcipher_dma_cleanup(struct skcipher_request *req) +{ + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); + + if (req->dst != req->src) { + dma_unmap_sg(cesa_dev->dev, req->dst, creq->dst_nents, + DMA_FROM_DEVICE); + dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, + DMA_TO_DEVICE); + } else { + dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, + DMA_BIDIRECTIONAL); + } + mv_cesa_dma_cleanup(&creq->base); +} + +static inline void mv_cesa_skcipher_cleanup(struct skcipher_request *req) +{ + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); + struct mv_cesa_engine *engine = creq->base.engine; + + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) + mv_cesa_skcipher_dma_cleanup(req); + + atomic_sub(req->cryptlen, &engine->load); +} + +static void mv_cesa_skcipher_std_step(struct skcipher_request *req) +{ + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); + struct mv_cesa_skcipher_std_req *sreq = &creq->std; + struct mv_cesa_engine *engine = creq->base.engine; + size_t len = min_t(size_t, req->cryptlen - sreq->offset, + CESA_SA_SRAM_PAYLOAD_SIZE); + + mv_cesa_adjust_op(engine, &sreq->op); + if (engine->pool) + memcpy(engine->sram_pool, &sreq->op, sizeof(sreq->op)); + else + memcpy_toio(engine->sram, &sreq->op, sizeof(sreq->op)); + + len = mv_cesa_sg_copy_to_sram(engine, req->src, creq->src_nents, + CESA_SA_DATA_SRAM_OFFSET, len, + sreq->offset); + + sreq->size = len; + mv_cesa_set_crypt_op_len(&sreq->op, len); + + /* FIXME: only update enc_len field */ + if (!sreq->skip_ctx) { + if (engine->pool) + memcpy(engine->sram_pool, &sreq->op, sizeof(sreq->op)); + else + memcpy_toio(engine->sram, &sreq->op, sizeof(sreq->op)); + sreq->skip_ctx = true; + } else if (engine->pool) + memcpy(engine->sram_pool, &sreq->op, sizeof(sreq->op.desc)); + else + memcpy_toio(engine->sram, &sreq->op, sizeof(sreq->op.desc)); + + mv_cesa_set_int_mask(engine, CESA_SA_INT_ACCEL0_DONE); + writel_relaxed(CESA_SA_CFG_PARA_DIS, engine->regs + CESA_SA_CFG); + WARN_ON(readl(engine->regs + CESA_SA_CMD) & + CESA_SA_CMD_EN_CESA_SA_ACCL0); + writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD); +} + +static int mv_cesa_skcipher_std_process(struct skcipher_request *req, + u32 status) +{ + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); + struct mv_cesa_skcipher_std_req *sreq = &creq->std; + struct mv_cesa_engine *engine = creq->base.engine; + size_t len; + + len = mv_cesa_sg_copy_from_sram(engine, req->dst, creq->dst_nents, + CESA_SA_DATA_SRAM_OFFSET, sreq->size, + sreq->offset); + + sreq->offset += len; + if (sreq->offset < req->cryptlen) + return -EINPROGRESS; + + return 0; +} + +static int mv_cesa_skcipher_process(struct crypto_async_request *req, + u32 status) +{ + struct skcipher_request *skreq = skcipher_request_cast(req); + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(skreq); + struct mv_cesa_req *basereq = &creq->base; + + if (mv_cesa_req_get_type(basereq) == CESA_STD_REQ) + return mv_cesa_skcipher_std_process(skreq, status); + + return mv_cesa_dma_process(basereq, status); +} + +static void mv_cesa_skcipher_step(struct crypto_async_request *req) +{ + struct skcipher_request *skreq = skcipher_request_cast(req); + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(skreq); + + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) + mv_cesa_dma_step(&creq->base); + else + mv_cesa_skcipher_std_step(skreq); +} + +static inline void +mv_cesa_skcipher_dma_prepare(struct skcipher_request *req) +{ + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); + struct mv_cesa_req *basereq = &creq->base; + + mv_cesa_dma_prepare(basereq, basereq->engine); +} + +static inline void +mv_cesa_skcipher_std_prepare(struct skcipher_request *req) +{ + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); + struct mv_cesa_skcipher_std_req *sreq = &creq->std; + + sreq->size = 0; + sreq->offset = 0; +} + +static inline void mv_cesa_skcipher_prepare(struct crypto_async_request *req, + struct mv_cesa_engine *engine) +{ + struct skcipher_request *skreq = skcipher_request_cast(req); + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(skreq); + + creq->base.engine = engine; + + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) + mv_cesa_skcipher_dma_prepare(skreq); + else + mv_cesa_skcipher_std_prepare(skreq); +} + +static inline void +mv_cesa_skcipher_req_cleanup(struct crypto_async_request *req) +{ + struct skcipher_request *skreq = skcipher_request_cast(req); + + mv_cesa_skcipher_cleanup(skreq); +} + +static void +mv_cesa_skcipher_complete(struct crypto_async_request *req) +{ + struct skcipher_request *skreq = skcipher_request_cast(req); + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(skreq); + struct mv_cesa_engine *engine = creq->base.engine; + unsigned int ivsize; + + ivsize = crypto_skcipher_ivsize(crypto_skcipher_reqtfm(skreq)); + + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) { + struct mv_cesa_req *basereq; + + basereq = &creq->base; + memcpy(skreq->iv, basereq->chain.last->op->ctx.skcipher.iv, + ivsize); + } else if (engine->pool) + memcpy(skreq->iv, + engine->sram_pool + CESA_SA_CRYPT_IV_SRAM_OFFSET, + ivsize); + else + memcpy_fromio(skreq->iv, + engine->sram + CESA_SA_CRYPT_IV_SRAM_OFFSET, + ivsize); +} + +static const struct mv_cesa_req_ops mv_cesa_skcipher_req_ops = { + .step = mv_cesa_skcipher_step, + .process = mv_cesa_skcipher_process, + .cleanup = mv_cesa_skcipher_req_cleanup, + .complete = mv_cesa_skcipher_complete, +}; + +static void mv_cesa_skcipher_cra_exit(struct crypto_tfm *tfm) +{ + void *ctx = crypto_tfm_ctx(tfm); + + memzero_explicit(ctx, tfm->__crt_alg->cra_ctxsize); +} + +static int mv_cesa_skcipher_cra_init(struct crypto_tfm *tfm) +{ + struct mv_cesa_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->ops = &mv_cesa_skcipher_req_ops; + + crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm), + sizeof(struct mv_cesa_skcipher_req)); + + return 0; +} + +static int mv_cesa_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, + unsigned int len) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); + struct mv_cesa_aes_ctx *ctx = crypto_tfm_ctx(tfm); + int remaining; + int offset; + int ret; + int i; + + ret = aes_expandkey(&ctx->aes, key, len); + if (ret) + return ret; + + remaining = (ctx->aes.key_length - 16) / 4; + offset = ctx->aes.key_length + 24 - remaining; + for (i = 0; i < remaining; i++) + ctx->aes.key_dec[4 + i] = ctx->aes.key_enc[offset + i]; + + return 0; +} + +static int mv_cesa_des_setkey(struct crypto_skcipher *cipher, const u8 *key, + unsigned int len) +{ + struct mv_cesa_des_ctx *ctx = crypto_skcipher_ctx(cipher); + int err; + + err = verify_skcipher_des_key(cipher, key); + if (err) + return err; + + memcpy(ctx->key, key, DES_KEY_SIZE); + + return 0; +} + +static int mv_cesa_des3_ede_setkey(struct crypto_skcipher *cipher, + const u8 *key, unsigned int len) +{ + struct mv_cesa_des3_ctx *ctx = crypto_skcipher_ctx(cipher); + int err; + + err = verify_skcipher_des3_key(cipher, key); + if (err) + return err; + + memcpy(ctx->key, key, DES3_EDE_KEY_SIZE); + + return 0; +} + +static int mv_cesa_skcipher_dma_req_init(struct skcipher_request *req, + const struct mv_cesa_op_ctx *op_templ) +{ + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + struct mv_cesa_req *basereq = &creq->base; + struct mv_cesa_skcipher_dma_iter iter; + bool skip_ctx = false; + int ret; + + basereq->chain.first = NULL; + basereq->chain.last = NULL; + + if (req->src != req->dst) { + ret = dma_map_sg(cesa_dev->dev, req->src, creq->src_nents, + DMA_TO_DEVICE); + if (!ret) + return -ENOMEM; + + ret = dma_map_sg(cesa_dev->dev, req->dst, creq->dst_nents, + DMA_FROM_DEVICE); + if (!ret) { + ret = -ENOMEM; + goto err_unmap_src; + } + } else { + ret = dma_map_sg(cesa_dev->dev, req->src, creq->src_nents, + DMA_BIDIRECTIONAL); + if (!ret) + return -ENOMEM; + } + + mv_cesa_tdma_desc_iter_init(&basereq->chain); + mv_cesa_skcipher_req_iter_init(&iter, req); + + do { + struct mv_cesa_op_ctx *op; + + op = mv_cesa_dma_add_op(&basereq->chain, op_templ, skip_ctx, + flags); + if (IS_ERR(op)) { + ret = PTR_ERR(op); + goto err_free_tdma; + } + skip_ctx = true; + + mv_cesa_set_crypt_op_len(op, iter.base.op_len); + + /* Add input transfers */ + ret = mv_cesa_dma_add_op_transfers(&basereq->chain, &iter.base, + &iter.src, flags); + if (ret) + goto err_free_tdma; + + /* Add dummy desc to launch the crypto operation */ + ret = mv_cesa_dma_add_dummy_launch(&basereq->chain, flags); + if (ret) + goto err_free_tdma; + + /* Add output transfers */ + ret = mv_cesa_dma_add_op_transfers(&basereq->chain, &iter.base, + &iter.dst, flags); + if (ret) + goto err_free_tdma; + + } while (mv_cesa_skcipher_req_iter_next_op(&iter)); + + /* Add output data for IV */ + ret = mv_cesa_dma_add_result_op(&basereq->chain, + CESA_SA_CFG_SRAM_OFFSET, + CESA_SA_DATA_SRAM_OFFSET, + CESA_TDMA_SRC_IN_SRAM, flags); + + if (ret) + goto err_free_tdma; + + basereq->chain.last->flags |= CESA_TDMA_END_OF_REQ; + + return 0; + +err_free_tdma: + mv_cesa_dma_cleanup(basereq); + if (req->dst != req->src) + dma_unmap_sg(cesa_dev->dev, req->dst, creq->dst_nents, + DMA_FROM_DEVICE); + +err_unmap_src: + dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, + req->dst != req->src ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL); + + return ret; +} + +static inline int +mv_cesa_skcipher_std_req_init(struct skcipher_request *req, + const struct mv_cesa_op_ctx *op_templ) +{ + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); + struct mv_cesa_skcipher_std_req *sreq = &creq->std; + struct mv_cesa_req *basereq = &creq->base; + + sreq->op = *op_templ; + sreq->skip_ctx = false; + basereq->chain.first = NULL; + basereq->chain.last = NULL; + + return 0; +} + +static int mv_cesa_skcipher_req_init(struct skcipher_request *req, + struct mv_cesa_op_ctx *tmpl) +{ + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + unsigned int blksize = crypto_skcipher_blocksize(tfm); + int ret; + + if (!IS_ALIGNED(req->cryptlen, blksize)) + return -EINVAL; + + creq->src_nents = sg_nents_for_len(req->src, req->cryptlen); + if (creq->src_nents < 0) { + dev_err(cesa_dev->dev, "Invalid number of src SG"); + return creq->src_nents; + } + creq->dst_nents = sg_nents_for_len(req->dst, req->cryptlen); + if (creq->dst_nents < 0) { + dev_err(cesa_dev->dev, "Invalid number of dst SG"); + return creq->dst_nents; + } + + mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_OP_CRYPT_ONLY, + CESA_SA_DESC_CFG_OP_MSK); + + if (cesa_dev->caps->has_tdma) + ret = mv_cesa_skcipher_dma_req_init(req, tmpl); + else + ret = mv_cesa_skcipher_std_req_init(req, tmpl); + + return ret; +} + +static int mv_cesa_skcipher_queue_req(struct skcipher_request *req, + struct mv_cesa_op_ctx *tmpl) +{ + int ret; + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); + struct mv_cesa_engine *engine; + + if (!req->cryptlen) + return 0; + + ret = mv_cesa_skcipher_req_init(req, tmpl); + if (ret) + return ret; + + engine = mv_cesa_select_engine(req->cryptlen); + mv_cesa_skcipher_prepare(&req->base, engine); + + ret = mv_cesa_queue_req(&req->base, &creq->base); + + if (mv_cesa_req_needs_cleanup(&req->base, ret)) + mv_cesa_skcipher_cleanup(req); + + return ret; +} + +static int mv_cesa_des_op(struct skcipher_request *req, + struct mv_cesa_op_ctx *tmpl) +{ + struct mv_cesa_des_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + + mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_CRYPTM_DES, + CESA_SA_DESC_CFG_CRYPTM_MSK); + + memcpy(tmpl->ctx.skcipher.key, ctx->key, DES_KEY_SIZE); + + return mv_cesa_skcipher_queue_req(req, tmpl); +} + +static int mv_cesa_ecb_des_encrypt(struct skcipher_request *req) +{ + struct mv_cesa_op_ctx tmpl = { }; + + mv_cesa_set_op_cfg(&tmpl, + CESA_SA_DESC_CFG_CRYPTCM_ECB | + CESA_SA_DESC_CFG_DIR_ENC); + + return mv_cesa_des_op(req, &tmpl); +} + +static int mv_cesa_ecb_des_decrypt(struct skcipher_request *req) +{ + struct mv_cesa_op_ctx tmpl = { }; + + mv_cesa_set_op_cfg(&tmpl, + CESA_SA_DESC_CFG_CRYPTCM_ECB | + CESA_SA_DESC_CFG_DIR_DEC); + + return mv_cesa_des_op(req, &tmpl); +} + +struct skcipher_alg mv_cesa_ecb_des_alg = { + .setkey = mv_cesa_des_setkey, + .encrypt = mv_cesa_ecb_des_encrypt, + .decrypt = mv_cesa_ecb_des_decrypt, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .base = { + .cra_name = "ecb(des)", + .cra_driver_name = "mv-ecb-des", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_des_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = mv_cesa_skcipher_cra_init, + .cra_exit = mv_cesa_skcipher_cra_exit, + }, +}; + +static int mv_cesa_cbc_des_op(struct skcipher_request *req, + struct mv_cesa_op_ctx *tmpl) +{ + mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_CRYPTCM_CBC, + CESA_SA_DESC_CFG_CRYPTCM_MSK); + + memcpy(tmpl->ctx.skcipher.iv, req->iv, DES_BLOCK_SIZE); + + return mv_cesa_des_op(req, tmpl); +} + +static int mv_cesa_cbc_des_encrypt(struct skcipher_request *req) +{ + struct mv_cesa_op_ctx tmpl = { }; + + mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_DIR_ENC); + + return mv_cesa_cbc_des_op(req, &tmpl); +} + +static int mv_cesa_cbc_des_decrypt(struct skcipher_request *req) +{ + struct mv_cesa_op_ctx tmpl = { }; + + mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_DIR_DEC); + + return mv_cesa_cbc_des_op(req, &tmpl); +} + +struct skcipher_alg mv_cesa_cbc_des_alg = { + .setkey = mv_cesa_des_setkey, + .encrypt = mv_cesa_cbc_des_encrypt, + .decrypt = mv_cesa_cbc_des_decrypt, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .base = { + .cra_name = "cbc(des)", + .cra_driver_name = "mv-cbc-des", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_des_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = mv_cesa_skcipher_cra_init, + .cra_exit = mv_cesa_skcipher_cra_exit, + }, +}; + +static int mv_cesa_des3_op(struct skcipher_request *req, + struct mv_cesa_op_ctx *tmpl) +{ + struct mv_cesa_des3_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + + mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_CRYPTM_3DES, + CESA_SA_DESC_CFG_CRYPTM_MSK); + + memcpy(tmpl->ctx.skcipher.key, ctx->key, DES3_EDE_KEY_SIZE); + + return mv_cesa_skcipher_queue_req(req, tmpl); +} + +static int mv_cesa_ecb_des3_ede_encrypt(struct skcipher_request *req) +{ + struct mv_cesa_op_ctx tmpl = { }; + + mv_cesa_set_op_cfg(&tmpl, + CESA_SA_DESC_CFG_CRYPTCM_ECB | + CESA_SA_DESC_CFG_3DES_EDE | + CESA_SA_DESC_CFG_DIR_ENC); + + return mv_cesa_des3_op(req, &tmpl); +} + +static int mv_cesa_ecb_des3_ede_decrypt(struct skcipher_request *req) +{ + struct mv_cesa_op_ctx tmpl = { }; + + mv_cesa_set_op_cfg(&tmpl, + CESA_SA_DESC_CFG_CRYPTCM_ECB | + CESA_SA_DESC_CFG_3DES_EDE | + CESA_SA_DESC_CFG_DIR_DEC); + + return mv_cesa_des3_op(req, &tmpl); +} + +struct skcipher_alg mv_cesa_ecb_des3_ede_alg = { + .setkey = mv_cesa_des3_ede_setkey, + .encrypt = mv_cesa_ecb_des3_ede_encrypt, + .decrypt = mv_cesa_ecb_des3_ede_decrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .base = { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "mv-ecb-des3-ede", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_des3_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = mv_cesa_skcipher_cra_init, + .cra_exit = mv_cesa_skcipher_cra_exit, + }, +}; + +static int mv_cesa_cbc_des3_op(struct skcipher_request *req, + struct mv_cesa_op_ctx *tmpl) +{ + memcpy(tmpl->ctx.skcipher.iv, req->iv, DES3_EDE_BLOCK_SIZE); + + return mv_cesa_des3_op(req, tmpl); +} + +static int mv_cesa_cbc_des3_ede_encrypt(struct skcipher_request *req) +{ + struct mv_cesa_op_ctx tmpl = { }; + + mv_cesa_set_op_cfg(&tmpl, + CESA_SA_DESC_CFG_CRYPTCM_CBC | + CESA_SA_DESC_CFG_3DES_EDE | + CESA_SA_DESC_CFG_DIR_ENC); + + return mv_cesa_cbc_des3_op(req, &tmpl); +} + +static int mv_cesa_cbc_des3_ede_decrypt(struct skcipher_request *req) +{ + struct mv_cesa_op_ctx tmpl = { }; + + mv_cesa_set_op_cfg(&tmpl, + CESA_SA_DESC_CFG_CRYPTCM_CBC | + CESA_SA_DESC_CFG_3DES_EDE | + CESA_SA_DESC_CFG_DIR_DEC); + + return mv_cesa_cbc_des3_op(req, &tmpl); +} + +struct skcipher_alg mv_cesa_cbc_des3_ede_alg = { + .setkey = mv_cesa_des3_ede_setkey, + .encrypt = mv_cesa_cbc_des3_ede_encrypt, + .decrypt = mv_cesa_cbc_des3_ede_decrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .base = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "mv-cbc-des3-ede", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_des3_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = mv_cesa_skcipher_cra_init, + .cra_exit = mv_cesa_skcipher_cra_exit, + }, +}; + +static int mv_cesa_aes_op(struct skcipher_request *req, + struct mv_cesa_op_ctx *tmpl) +{ + struct mv_cesa_aes_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + int i; + u32 *key; + u32 cfg; + + cfg = CESA_SA_DESC_CFG_CRYPTM_AES; + + if (mv_cesa_get_op_cfg(tmpl) & CESA_SA_DESC_CFG_DIR_DEC) + key = ctx->aes.key_dec; + else + key = ctx->aes.key_enc; + + for (i = 0; i < ctx->aes.key_length / sizeof(u32); i++) + tmpl->ctx.skcipher.key[i] = cpu_to_le32(key[i]); + + if (ctx->aes.key_length == 24) + cfg |= CESA_SA_DESC_CFG_AES_LEN_192; + else if (ctx->aes.key_length == 32) + cfg |= CESA_SA_DESC_CFG_AES_LEN_256; + + mv_cesa_update_op_cfg(tmpl, cfg, + CESA_SA_DESC_CFG_CRYPTM_MSK | + CESA_SA_DESC_CFG_AES_LEN_MSK); + + return mv_cesa_skcipher_queue_req(req, tmpl); +} + +static int mv_cesa_ecb_aes_encrypt(struct skcipher_request *req) +{ + struct mv_cesa_op_ctx tmpl = { }; + + mv_cesa_set_op_cfg(&tmpl, + CESA_SA_DESC_CFG_CRYPTCM_ECB | + CESA_SA_DESC_CFG_DIR_ENC); + + return mv_cesa_aes_op(req, &tmpl); +} + +static int mv_cesa_ecb_aes_decrypt(struct skcipher_request *req) +{ + struct mv_cesa_op_ctx tmpl = { }; + + mv_cesa_set_op_cfg(&tmpl, + CESA_SA_DESC_CFG_CRYPTCM_ECB | + CESA_SA_DESC_CFG_DIR_DEC); + + return mv_cesa_aes_op(req, &tmpl); +} + +struct skcipher_alg mv_cesa_ecb_aes_alg = { + .setkey = mv_cesa_aes_setkey, + .encrypt = mv_cesa_ecb_aes_encrypt, + .decrypt = mv_cesa_ecb_aes_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "mv-ecb-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_aes_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = mv_cesa_skcipher_cra_init, + .cra_exit = mv_cesa_skcipher_cra_exit, + }, +}; + +static int mv_cesa_cbc_aes_op(struct skcipher_request *req, + struct mv_cesa_op_ctx *tmpl) +{ + mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_CRYPTCM_CBC, + CESA_SA_DESC_CFG_CRYPTCM_MSK); + memcpy(tmpl->ctx.skcipher.iv, req->iv, AES_BLOCK_SIZE); + + return mv_cesa_aes_op(req, tmpl); +} + +static int mv_cesa_cbc_aes_encrypt(struct skcipher_request *req) +{ + struct mv_cesa_op_ctx tmpl = { }; + + mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_DIR_ENC); + + return mv_cesa_cbc_aes_op(req, &tmpl); +} + +static int mv_cesa_cbc_aes_decrypt(struct skcipher_request *req) +{ + struct mv_cesa_op_ctx tmpl = { }; + + mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_DIR_DEC); + + return mv_cesa_cbc_aes_op(req, &tmpl); +} + +struct skcipher_alg mv_cesa_cbc_aes_alg = { + .setkey = mv_cesa_aes_setkey, + .encrypt = mv_cesa_cbc_aes_encrypt, + .decrypt = mv_cesa_cbc_aes_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "mv-cbc-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_aes_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = mv_cesa_skcipher_cra_init, + .cra_exit = mv_cesa_skcipher_cra_exit, + }, +}; diff --git a/drivers/crypto/marvell/cesa/hash.c b/drivers/crypto/marvell/cesa/hash.c new file mode 100644 index 000000000..2f203042d --- /dev/null +++ b/drivers/crypto/marvell/cesa/hash.c @@ -0,0 +1,1454 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Hash algorithms supported by the CESA: MD5, SHA1 and SHA256. + * + * Author: Boris Brezillon + * Author: Arnaud Ebalard + * + * This work is based on an initial version written by + * Sebastian Andrzej Siewior < sebastian at breakpoint dot cc > + */ + +#include +#include +#include +#include +#include +#include + +#include "cesa.h" + +struct mv_cesa_ahash_dma_iter { + struct mv_cesa_dma_iter base; + struct mv_cesa_sg_dma_iter src; +}; + +static inline void +mv_cesa_ahash_req_iter_init(struct mv_cesa_ahash_dma_iter *iter, + struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + unsigned int len = req->nbytes + creq->cache_ptr; + + if (!creq->last_req) + len &= ~CESA_HASH_BLOCK_SIZE_MSK; + + mv_cesa_req_dma_iter_init(&iter->base, len); + mv_cesa_sg_dma_iter_init(&iter->src, req->src, DMA_TO_DEVICE); + iter->src.op_offset = creq->cache_ptr; +} + +static inline bool +mv_cesa_ahash_req_iter_next_op(struct mv_cesa_ahash_dma_iter *iter) +{ + iter->src.op_offset = 0; + + return mv_cesa_req_dma_iter_next_op(&iter->base); +} + +static inline int +mv_cesa_ahash_dma_alloc_cache(struct mv_cesa_ahash_dma_req *req, gfp_t flags) +{ + req->cache = dma_pool_alloc(cesa_dev->dma->cache_pool, flags, + &req->cache_dma); + if (!req->cache) + return -ENOMEM; + + return 0; +} + +static inline void +mv_cesa_ahash_dma_free_cache(struct mv_cesa_ahash_dma_req *req) +{ + if (!req->cache) + return; + + dma_pool_free(cesa_dev->dma->cache_pool, req->cache, + req->cache_dma); +} + +static int mv_cesa_ahash_dma_alloc_padding(struct mv_cesa_ahash_dma_req *req, + gfp_t flags) +{ + if (req->padding) + return 0; + + req->padding = dma_pool_alloc(cesa_dev->dma->padding_pool, flags, + &req->padding_dma); + if (!req->padding) + return -ENOMEM; + + return 0; +} + +static void mv_cesa_ahash_dma_free_padding(struct mv_cesa_ahash_dma_req *req) +{ + if (!req->padding) + return; + + dma_pool_free(cesa_dev->dma->padding_pool, req->padding, + req->padding_dma); + req->padding = NULL; +} + +static inline void mv_cesa_ahash_dma_last_cleanup(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + + mv_cesa_ahash_dma_free_padding(&creq->req.dma); +} + +static inline void mv_cesa_ahash_dma_cleanup(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + + dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, DMA_TO_DEVICE); + mv_cesa_ahash_dma_free_cache(&creq->req.dma); + mv_cesa_dma_cleanup(&creq->base); +} + +static inline void mv_cesa_ahash_cleanup(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + struct mv_cesa_engine *engine = creq->base.engine; + + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) + mv_cesa_ahash_dma_cleanup(req); + + atomic_sub(req->nbytes, &engine->load); +} + +static void mv_cesa_ahash_last_cleanup(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) + mv_cesa_ahash_dma_last_cleanup(req); +} + +static int mv_cesa_ahash_pad_len(struct mv_cesa_ahash_req *creq) +{ + unsigned int index, padlen; + + index = creq->len & CESA_HASH_BLOCK_SIZE_MSK; + padlen = (index < 56) ? (56 - index) : (64 + 56 - index); + + return padlen; +} + +static int mv_cesa_ahash_pad_req(struct mv_cesa_ahash_req *creq, u8 *buf) +{ + unsigned int padlen; + + buf[0] = 0x80; + /* Pad out to 56 mod 64 */ + padlen = mv_cesa_ahash_pad_len(creq); + memset(buf + 1, 0, padlen - 1); + + if (creq->algo_le) { + __le64 bits = cpu_to_le64(creq->len << 3); + + memcpy(buf + padlen, &bits, sizeof(bits)); + } else { + __be64 bits = cpu_to_be64(creq->len << 3); + + memcpy(buf + padlen, &bits, sizeof(bits)); + } + + return padlen + 8; +} + +static void mv_cesa_ahash_std_step(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + struct mv_cesa_ahash_std_req *sreq = &creq->req.std; + struct mv_cesa_engine *engine = creq->base.engine; + struct mv_cesa_op_ctx *op; + unsigned int new_cache_ptr = 0; + u32 frag_mode; + size_t len; + unsigned int digsize; + int i; + + mv_cesa_adjust_op(engine, &creq->op_tmpl); + if (engine->pool) + memcpy(engine->sram_pool, &creq->op_tmpl, + sizeof(creq->op_tmpl)); + else + memcpy_toio(engine->sram, &creq->op_tmpl, + sizeof(creq->op_tmpl)); + + if (!sreq->offset) { + digsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req)); + for (i = 0; i < digsize / 4; i++) + writel_relaxed(creq->state[i], + engine->regs + CESA_IVDIG(i)); + } + + if (creq->cache_ptr) { + if (engine->pool) + memcpy(engine->sram_pool + CESA_SA_DATA_SRAM_OFFSET, + creq->cache, creq->cache_ptr); + else + memcpy_toio(engine->sram + CESA_SA_DATA_SRAM_OFFSET, + creq->cache, creq->cache_ptr); + } + + len = min_t(size_t, req->nbytes + creq->cache_ptr - sreq->offset, + CESA_SA_SRAM_PAYLOAD_SIZE); + + if (!creq->last_req) { + new_cache_ptr = len & CESA_HASH_BLOCK_SIZE_MSK; + len &= ~CESA_HASH_BLOCK_SIZE_MSK; + } + + if (len - creq->cache_ptr) + sreq->offset += mv_cesa_sg_copy_to_sram( + engine, req->src, creq->src_nents, + CESA_SA_DATA_SRAM_OFFSET + creq->cache_ptr, + len - creq->cache_ptr, sreq->offset); + + op = &creq->op_tmpl; + + frag_mode = mv_cesa_get_op_cfg(op) & CESA_SA_DESC_CFG_FRAG_MSK; + + if (creq->last_req && sreq->offset == req->nbytes && + creq->len <= CESA_SA_DESC_MAC_SRC_TOTAL_LEN_MAX) { + if (frag_mode == CESA_SA_DESC_CFG_FIRST_FRAG) + frag_mode = CESA_SA_DESC_CFG_NOT_FRAG; + else if (frag_mode == CESA_SA_DESC_CFG_MID_FRAG) + frag_mode = CESA_SA_DESC_CFG_LAST_FRAG; + } + + if (frag_mode == CESA_SA_DESC_CFG_NOT_FRAG || + frag_mode == CESA_SA_DESC_CFG_LAST_FRAG) { + if (len && + creq->len <= CESA_SA_DESC_MAC_SRC_TOTAL_LEN_MAX) { + mv_cesa_set_mac_op_total_len(op, creq->len); + } else { + int trailerlen = mv_cesa_ahash_pad_len(creq) + 8; + + if (len + trailerlen > CESA_SA_SRAM_PAYLOAD_SIZE) { + len &= CESA_HASH_BLOCK_SIZE_MSK; + new_cache_ptr = 64 - trailerlen; + if (engine->pool) + memcpy(creq->cache, + engine->sram_pool + + CESA_SA_DATA_SRAM_OFFSET + len, + new_cache_ptr); + else + memcpy_fromio(creq->cache, + engine->sram + + CESA_SA_DATA_SRAM_OFFSET + + len, + new_cache_ptr); + } else { + i = mv_cesa_ahash_pad_req(creq, creq->cache); + len += i; + if (engine->pool) + memcpy(engine->sram_pool + len + + CESA_SA_DATA_SRAM_OFFSET, + creq->cache, i); + else + memcpy_toio(engine->sram + len + + CESA_SA_DATA_SRAM_OFFSET, + creq->cache, i); + } + + if (frag_mode == CESA_SA_DESC_CFG_LAST_FRAG) + frag_mode = CESA_SA_DESC_CFG_MID_FRAG; + else + frag_mode = CESA_SA_DESC_CFG_FIRST_FRAG; + } + } + + mv_cesa_set_mac_op_frag_len(op, len); + mv_cesa_update_op_cfg(op, frag_mode, CESA_SA_DESC_CFG_FRAG_MSK); + + /* FIXME: only update enc_len field */ + if (engine->pool) + memcpy(engine->sram_pool, op, sizeof(*op)); + else + memcpy_toio(engine->sram, op, sizeof(*op)); + + if (frag_mode == CESA_SA_DESC_CFG_FIRST_FRAG) + mv_cesa_update_op_cfg(op, CESA_SA_DESC_CFG_MID_FRAG, + CESA_SA_DESC_CFG_FRAG_MSK); + + creq->cache_ptr = new_cache_ptr; + + mv_cesa_set_int_mask(engine, CESA_SA_INT_ACCEL0_DONE); + writel_relaxed(CESA_SA_CFG_PARA_DIS, engine->regs + CESA_SA_CFG); + WARN_ON(readl(engine->regs + CESA_SA_CMD) & + CESA_SA_CMD_EN_CESA_SA_ACCL0); + writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD); +} + +static int mv_cesa_ahash_std_process(struct ahash_request *req, u32 status) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + struct mv_cesa_ahash_std_req *sreq = &creq->req.std; + + if (sreq->offset < (req->nbytes - creq->cache_ptr)) + return -EINPROGRESS; + + return 0; +} + +static inline void mv_cesa_ahash_dma_prepare(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + struct mv_cesa_req *basereq = &creq->base; + + mv_cesa_dma_prepare(basereq, basereq->engine); +} + +static void mv_cesa_ahash_std_prepare(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + struct mv_cesa_ahash_std_req *sreq = &creq->req.std; + + sreq->offset = 0; +} + +static void mv_cesa_ahash_dma_step(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + struct mv_cesa_req *base = &creq->base; + + /* We must explicitly set the digest state. */ + if (base->chain.first->flags & CESA_TDMA_SET_STATE) { + struct mv_cesa_engine *engine = base->engine; + int i; + + /* Set the hash state in the IVDIG regs. */ + for (i = 0; i < ARRAY_SIZE(creq->state); i++) + writel_relaxed(creq->state[i], engine->regs + + CESA_IVDIG(i)); + } + + mv_cesa_dma_step(base); +} + +static void mv_cesa_ahash_step(struct crypto_async_request *req) +{ + struct ahash_request *ahashreq = ahash_request_cast(req); + struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq); + + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) + mv_cesa_ahash_dma_step(ahashreq); + else + mv_cesa_ahash_std_step(ahashreq); +} + +static int mv_cesa_ahash_process(struct crypto_async_request *req, u32 status) +{ + struct ahash_request *ahashreq = ahash_request_cast(req); + struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq); + + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) + return mv_cesa_dma_process(&creq->base, status); + + return mv_cesa_ahash_std_process(ahashreq, status); +} + +static void mv_cesa_ahash_complete(struct crypto_async_request *req) +{ + struct ahash_request *ahashreq = ahash_request_cast(req); + struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq); + struct mv_cesa_engine *engine = creq->base.engine; + unsigned int digsize; + int i; + + digsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(ahashreq)); + + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ && + (creq->base.chain.last->flags & CESA_TDMA_TYPE_MSK) == + CESA_TDMA_RESULT) { + const void *data; + + /* + * Result is already in the correct endianness when the SA is + * used + */ + data = creq->base.chain.last->op->ctx.hash.hash; + memcpy(ahashreq->result, data, digsize); + } else { + for (i = 0; i < digsize / 4; i++) + creq->state[i] = readl_relaxed(engine->regs + + CESA_IVDIG(i)); + if (creq->last_req) { + /* + * Hardware's MD5 digest is in little endian format, but + * SHA in big endian format + */ + if (creq->algo_le) { + __le32 *result = (void *)ahashreq->result; + + for (i = 0; i < digsize / 4; i++) + result[i] = cpu_to_le32(creq->state[i]); + } else { + __be32 *result = (void *)ahashreq->result; + + for (i = 0; i < digsize / 4; i++) + result[i] = cpu_to_be32(creq->state[i]); + } + } + } +} + +static void mv_cesa_ahash_prepare(struct crypto_async_request *req, + struct mv_cesa_engine *engine) +{ + struct ahash_request *ahashreq = ahash_request_cast(req); + struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq); + + creq->base.engine = engine; + + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) + mv_cesa_ahash_dma_prepare(ahashreq); + else + mv_cesa_ahash_std_prepare(ahashreq); +} + +static void mv_cesa_ahash_req_cleanup(struct crypto_async_request *req) +{ + struct ahash_request *ahashreq = ahash_request_cast(req); + struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq); + + if (creq->last_req) + mv_cesa_ahash_last_cleanup(ahashreq); + + mv_cesa_ahash_cleanup(ahashreq); + + if (creq->cache_ptr) + sg_pcopy_to_buffer(ahashreq->src, creq->src_nents, + creq->cache, + creq->cache_ptr, + ahashreq->nbytes - creq->cache_ptr); +} + +static const struct mv_cesa_req_ops mv_cesa_ahash_req_ops = { + .step = mv_cesa_ahash_step, + .process = mv_cesa_ahash_process, + .cleanup = mv_cesa_ahash_req_cleanup, + .complete = mv_cesa_ahash_complete, +}; + +static void mv_cesa_ahash_init(struct ahash_request *req, + struct mv_cesa_op_ctx *tmpl, bool algo_le) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + + memset(creq, 0, sizeof(*creq)); + mv_cesa_update_op_cfg(tmpl, + CESA_SA_DESC_CFG_OP_MAC_ONLY | + CESA_SA_DESC_CFG_FIRST_FRAG, + CESA_SA_DESC_CFG_OP_MSK | + CESA_SA_DESC_CFG_FRAG_MSK); + mv_cesa_set_mac_op_total_len(tmpl, 0); + mv_cesa_set_mac_op_frag_len(tmpl, 0); + creq->op_tmpl = *tmpl; + creq->len = 0; + creq->algo_le = algo_le; +} + +static inline int mv_cesa_ahash_cra_init(struct crypto_tfm *tfm) +{ + struct mv_cesa_hash_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->base.ops = &mv_cesa_ahash_req_ops; + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct mv_cesa_ahash_req)); + return 0; +} + +static bool mv_cesa_ahash_cache_req(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + bool cached = false; + + if (creq->cache_ptr + req->nbytes < CESA_MAX_HASH_BLOCK_SIZE && + !creq->last_req) { + cached = true; + + if (!req->nbytes) + return cached; + + sg_pcopy_to_buffer(req->src, creq->src_nents, + creq->cache + creq->cache_ptr, + req->nbytes, 0); + + creq->cache_ptr += req->nbytes; + } + + return cached; +} + +static struct mv_cesa_op_ctx * +mv_cesa_dma_add_frag(struct mv_cesa_tdma_chain *chain, + struct mv_cesa_op_ctx *tmpl, unsigned int frag_len, + gfp_t flags) +{ + struct mv_cesa_op_ctx *op; + int ret; + + op = mv_cesa_dma_add_op(chain, tmpl, false, flags); + if (IS_ERR(op)) + return op; + + /* Set the operation block fragment length. */ + mv_cesa_set_mac_op_frag_len(op, frag_len); + + /* Append dummy desc to launch operation */ + ret = mv_cesa_dma_add_dummy_launch(chain, flags); + if (ret) + return ERR_PTR(ret); + + if (mv_cesa_mac_op_is_first_frag(tmpl)) + mv_cesa_update_op_cfg(tmpl, + CESA_SA_DESC_CFG_MID_FRAG, + CESA_SA_DESC_CFG_FRAG_MSK); + + return op; +} + +static int +mv_cesa_ahash_dma_add_cache(struct mv_cesa_tdma_chain *chain, + struct mv_cesa_ahash_req *creq, + gfp_t flags) +{ + struct mv_cesa_ahash_dma_req *ahashdreq = &creq->req.dma; + int ret; + + if (!creq->cache_ptr) + return 0; + + ret = mv_cesa_ahash_dma_alloc_cache(ahashdreq, flags); + if (ret) + return ret; + + memcpy(ahashdreq->cache, creq->cache, creq->cache_ptr); + + return mv_cesa_dma_add_data_transfer(chain, + CESA_SA_DATA_SRAM_OFFSET, + ahashdreq->cache_dma, + creq->cache_ptr, + CESA_TDMA_DST_IN_SRAM, + flags); +} + +static struct mv_cesa_op_ctx * +mv_cesa_ahash_dma_last_req(struct mv_cesa_tdma_chain *chain, + struct mv_cesa_ahash_dma_iter *dma_iter, + struct mv_cesa_ahash_req *creq, + unsigned int frag_len, gfp_t flags) +{ + struct mv_cesa_ahash_dma_req *ahashdreq = &creq->req.dma; + unsigned int len, trailerlen, padoff = 0; + struct mv_cesa_op_ctx *op; + int ret; + + /* + * If the transfer is smaller than our maximum length, and we have + * some data outstanding, we can ask the engine to finish the hash. + */ + if (creq->len <= CESA_SA_DESC_MAC_SRC_TOTAL_LEN_MAX && frag_len) { + op = mv_cesa_dma_add_frag(chain, &creq->op_tmpl, frag_len, + flags); + if (IS_ERR(op)) + return op; + + mv_cesa_set_mac_op_total_len(op, creq->len); + mv_cesa_update_op_cfg(op, mv_cesa_mac_op_is_first_frag(op) ? + CESA_SA_DESC_CFG_NOT_FRAG : + CESA_SA_DESC_CFG_LAST_FRAG, + CESA_SA_DESC_CFG_FRAG_MSK); + + ret = mv_cesa_dma_add_result_op(chain, + CESA_SA_CFG_SRAM_OFFSET, + CESA_SA_DATA_SRAM_OFFSET, + CESA_TDMA_SRC_IN_SRAM, flags); + if (ret) + return ERR_PTR(-ENOMEM); + return op; + } + + /* + * The request is longer than the engine can handle, or we have + * no data outstanding. Manually generate the padding, adding it + * as a "mid" fragment. + */ + ret = mv_cesa_ahash_dma_alloc_padding(ahashdreq, flags); + if (ret) + return ERR_PTR(ret); + + trailerlen = mv_cesa_ahash_pad_req(creq, ahashdreq->padding); + + len = min(CESA_SA_SRAM_PAYLOAD_SIZE - frag_len, trailerlen); + if (len) { + ret = mv_cesa_dma_add_data_transfer(chain, + CESA_SA_DATA_SRAM_OFFSET + + frag_len, + ahashdreq->padding_dma, + len, CESA_TDMA_DST_IN_SRAM, + flags); + if (ret) + return ERR_PTR(ret); + + op = mv_cesa_dma_add_frag(chain, &creq->op_tmpl, frag_len + len, + flags); + if (IS_ERR(op)) + return op; + + if (len == trailerlen) + return op; + + padoff += len; + } + + ret = mv_cesa_dma_add_data_transfer(chain, + CESA_SA_DATA_SRAM_OFFSET, + ahashdreq->padding_dma + + padoff, + trailerlen - padoff, + CESA_TDMA_DST_IN_SRAM, + flags); + if (ret) + return ERR_PTR(ret); + + return mv_cesa_dma_add_frag(chain, &creq->op_tmpl, trailerlen - padoff, + flags); +} + +static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + struct mv_cesa_req *basereq = &creq->base; + struct mv_cesa_ahash_dma_iter iter; + struct mv_cesa_op_ctx *op = NULL; + unsigned int frag_len; + bool set_state = false; + int ret; + u32 type; + + basereq->chain.first = NULL; + basereq->chain.last = NULL; + + if (!mv_cesa_mac_op_is_first_frag(&creq->op_tmpl)) + set_state = true; + + if (creq->src_nents) { + ret = dma_map_sg(cesa_dev->dev, req->src, creq->src_nents, + DMA_TO_DEVICE); + if (!ret) { + ret = -ENOMEM; + goto err; + } + } + + mv_cesa_tdma_desc_iter_init(&basereq->chain); + mv_cesa_ahash_req_iter_init(&iter, req); + + /* + * Add the cache (left-over data from a previous block) first. + * This will never overflow the SRAM size. + */ + ret = mv_cesa_ahash_dma_add_cache(&basereq->chain, creq, flags); + if (ret) + goto err_free_tdma; + + if (iter.base.len > iter.src.op_offset) { + /* + * Add all the new data, inserting an operation block and + * launch command between each full SRAM block-worth of + * data. We intentionally do not add the final op block. + */ + while (true) { + ret = mv_cesa_dma_add_op_transfers(&basereq->chain, + &iter.base, + &iter.src, flags); + if (ret) + goto err_free_tdma; + + frag_len = iter.base.op_len; + + if (!mv_cesa_ahash_req_iter_next_op(&iter)) + break; + + op = mv_cesa_dma_add_frag(&basereq->chain, + &creq->op_tmpl, + frag_len, flags); + if (IS_ERR(op)) { + ret = PTR_ERR(op); + goto err_free_tdma; + } + } + } else { + /* Account for the data that was in the cache. */ + frag_len = iter.base.op_len; + } + + /* + * At this point, frag_len indicates whether we have any data + * outstanding which needs an operation. Queue up the final + * operation, which depends whether this is the final request. + */ + if (creq->last_req) + op = mv_cesa_ahash_dma_last_req(&basereq->chain, &iter, creq, + frag_len, flags); + else if (frag_len) + op = mv_cesa_dma_add_frag(&basereq->chain, &creq->op_tmpl, + frag_len, flags); + + if (IS_ERR(op)) { + ret = PTR_ERR(op); + goto err_free_tdma; + } + + /* + * If results are copied via DMA, this means that this + * request can be directly processed by the engine, + * without partial updates. So we can chain it at the + * DMA level with other requests. + */ + type = basereq->chain.last->flags & CESA_TDMA_TYPE_MSK; + + if (op && type != CESA_TDMA_RESULT) { + /* Add dummy desc to wait for crypto operation end */ + ret = mv_cesa_dma_add_dummy_end(&basereq->chain, flags); + if (ret) + goto err_free_tdma; + } + + if (!creq->last_req) + creq->cache_ptr = req->nbytes + creq->cache_ptr - + iter.base.len; + else + creq->cache_ptr = 0; + + basereq->chain.last->flags |= CESA_TDMA_END_OF_REQ; + + if (type != CESA_TDMA_RESULT) + basereq->chain.last->flags |= CESA_TDMA_BREAK_CHAIN; + + if (set_state) { + /* + * Put the CESA_TDMA_SET_STATE flag on the first tdma desc to + * let the step logic know that the IVDIG registers should be + * explicitly set before launching a TDMA chain. + */ + basereq->chain.first->flags |= CESA_TDMA_SET_STATE; + } + + return 0; + +err_free_tdma: + mv_cesa_dma_cleanup(basereq); + dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, DMA_TO_DEVICE); + +err: + mv_cesa_ahash_last_cleanup(req); + + return ret; +} + +static int mv_cesa_ahash_req_init(struct ahash_request *req, bool *cached) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + + creq->src_nents = sg_nents_for_len(req->src, req->nbytes); + if (creq->src_nents < 0) { + dev_err(cesa_dev->dev, "Invalid number of src SG"); + return creq->src_nents; + } + + *cached = mv_cesa_ahash_cache_req(req); + + if (*cached) + return 0; + + if (cesa_dev->caps->has_tdma) + return mv_cesa_ahash_dma_req_init(req); + else + return 0; +} + +static int mv_cesa_ahash_queue_req(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + struct mv_cesa_engine *engine; + bool cached = false; + int ret; + + ret = mv_cesa_ahash_req_init(req, &cached); + if (ret) + return ret; + + if (cached) + return 0; + + engine = mv_cesa_select_engine(req->nbytes); + mv_cesa_ahash_prepare(&req->base, engine); + + ret = mv_cesa_queue_req(&req->base, &creq->base); + + if (mv_cesa_req_needs_cleanup(&req->base, ret)) + mv_cesa_ahash_cleanup(req); + + return ret; +} + +static int mv_cesa_ahash_update(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + + creq->len += req->nbytes; + + return mv_cesa_ahash_queue_req(req); +} + +static int mv_cesa_ahash_final(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + struct mv_cesa_op_ctx *tmpl = &creq->op_tmpl; + + mv_cesa_set_mac_op_total_len(tmpl, creq->len); + creq->last_req = true; + req->nbytes = 0; + + return mv_cesa_ahash_queue_req(req); +} + +static int mv_cesa_ahash_finup(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + struct mv_cesa_op_ctx *tmpl = &creq->op_tmpl; + + creq->len += req->nbytes; + mv_cesa_set_mac_op_total_len(tmpl, creq->len); + creq->last_req = true; + + return mv_cesa_ahash_queue_req(req); +} + +static int mv_cesa_ahash_export(struct ahash_request *req, void *hash, + u64 *len, void *cache) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + unsigned int digsize = crypto_ahash_digestsize(ahash); + unsigned int blocksize; + + blocksize = crypto_ahash_blocksize(ahash); + + *len = creq->len; + memcpy(hash, creq->state, digsize); + memcpy_and_pad(cache, blocksize, creq->cache, creq->cache_ptr, 0); + + return 0; +} + +static int mv_cesa_ahash_import(struct ahash_request *req, const void *hash, + u64 len, const void *cache) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + unsigned int digsize = crypto_ahash_digestsize(ahash); + unsigned int blocksize; + unsigned int cache_ptr; + int ret; + + ret = crypto_ahash_init(req); + if (ret) + return ret; + + blocksize = crypto_ahash_blocksize(ahash); + if (len >= blocksize) + mv_cesa_update_op_cfg(&creq->op_tmpl, + CESA_SA_DESC_CFG_MID_FRAG, + CESA_SA_DESC_CFG_FRAG_MSK); + + creq->len = len; + memcpy(creq->state, hash, digsize); + creq->cache_ptr = 0; + + cache_ptr = do_div(len, blocksize); + if (!cache_ptr) + return 0; + + memcpy(creq->cache, cache, cache_ptr); + creq->cache_ptr = cache_ptr; + + return 0; +} + +static int mv_cesa_md5_init(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + struct mv_cesa_op_ctx tmpl = { }; + + mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_MD5); + + mv_cesa_ahash_init(req, &tmpl, true); + + creq->state[0] = MD5_H0; + creq->state[1] = MD5_H1; + creq->state[2] = MD5_H2; + creq->state[3] = MD5_H3; + + return 0; +} + +static int mv_cesa_md5_export(struct ahash_request *req, void *out) +{ + struct md5_state *out_state = out; + + return mv_cesa_ahash_export(req, out_state->hash, + &out_state->byte_count, out_state->block); +} + +static int mv_cesa_md5_import(struct ahash_request *req, const void *in) +{ + const struct md5_state *in_state = in; + + return mv_cesa_ahash_import(req, in_state->hash, in_state->byte_count, + in_state->block); +} + +static int mv_cesa_md5_digest(struct ahash_request *req) +{ + int ret; + + ret = mv_cesa_md5_init(req); + if (ret) + return ret; + + return mv_cesa_ahash_finup(req); +} + +struct ahash_alg mv_md5_alg = { + .init = mv_cesa_md5_init, + .update = mv_cesa_ahash_update, + .final = mv_cesa_ahash_final, + .finup = mv_cesa_ahash_finup, + .digest = mv_cesa_md5_digest, + .export = mv_cesa_md5_export, + .import = mv_cesa_md5_import, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct md5_state), + .base = { + .cra_name = "md5", + .cra_driver_name = "mv-md5", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_hash_ctx), + .cra_init = mv_cesa_ahash_cra_init, + .cra_module = THIS_MODULE, + } + } +}; + +static int mv_cesa_sha1_init(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + struct mv_cesa_op_ctx tmpl = { }; + + mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_SHA1); + + mv_cesa_ahash_init(req, &tmpl, false); + + creq->state[0] = SHA1_H0; + creq->state[1] = SHA1_H1; + creq->state[2] = SHA1_H2; + creq->state[3] = SHA1_H3; + creq->state[4] = SHA1_H4; + + return 0; +} + +static int mv_cesa_sha1_export(struct ahash_request *req, void *out) +{ + struct sha1_state *out_state = out; + + return mv_cesa_ahash_export(req, out_state->state, &out_state->count, + out_state->buffer); +} + +static int mv_cesa_sha1_import(struct ahash_request *req, const void *in) +{ + const struct sha1_state *in_state = in; + + return mv_cesa_ahash_import(req, in_state->state, in_state->count, + in_state->buffer); +} + +static int mv_cesa_sha1_digest(struct ahash_request *req) +{ + int ret; + + ret = mv_cesa_sha1_init(req); + if (ret) + return ret; + + return mv_cesa_ahash_finup(req); +} + +struct ahash_alg mv_sha1_alg = { + .init = mv_cesa_sha1_init, + .update = mv_cesa_ahash_update, + .final = mv_cesa_ahash_final, + .finup = mv_cesa_ahash_finup, + .digest = mv_cesa_sha1_digest, + .export = mv_cesa_sha1_export, + .import = mv_cesa_sha1_import, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct sha1_state), + .base = { + .cra_name = "sha1", + .cra_driver_name = "mv-sha1", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_hash_ctx), + .cra_init = mv_cesa_ahash_cra_init, + .cra_module = THIS_MODULE, + } + } +}; + +static int mv_cesa_sha256_init(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + struct mv_cesa_op_ctx tmpl = { }; + + mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_SHA256); + + mv_cesa_ahash_init(req, &tmpl, false); + + creq->state[0] = SHA256_H0; + creq->state[1] = SHA256_H1; + creq->state[2] = SHA256_H2; + creq->state[3] = SHA256_H3; + creq->state[4] = SHA256_H4; + creq->state[5] = SHA256_H5; + creq->state[6] = SHA256_H6; + creq->state[7] = SHA256_H7; + + return 0; +} + +static int mv_cesa_sha256_digest(struct ahash_request *req) +{ + int ret; + + ret = mv_cesa_sha256_init(req); + if (ret) + return ret; + + return mv_cesa_ahash_finup(req); +} + +static int mv_cesa_sha256_export(struct ahash_request *req, void *out) +{ + struct sha256_state *out_state = out; + + return mv_cesa_ahash_export(req, out_state->state, &out_state->count, + out_state->buf); +} + +static int mv_cesa_sha256_import(struct ahash_request *req, const void *in) +{ + const struct sha256_state *in_state = in; + + return mv_cesa_ahash_import(req, in_state->state, in_state->count, + in_state->buf); +} + +struct ahash_alg mv_sha256_alg = { + .init = mv_cesa_sha256_init, + .update = mv_cesa_ahash_update, + .final = mv_cesa_ahash_final, + .finup = mv_cesa_ahash_finup, + .digest = mv_cesa_sha256_digest, + .export = mv_cesa_sha256_export, + .import = mv_cesa_sha256_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha256", + .cra_driver_name = "mv-sha256", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_hash_ctx), + .cra_init = mv_cesa_ahash_cra_init, + .cra_module = THIS_MODULE, + } + } +}; + +static int mv_cesa_ahmac_iv_state_init(struct ahash_request *req, u8 *pad, + void *state, unsigned int blocksize) +{ + DECLARE_CRYPTO_WAIT(result); + struct scatterlist sg; + int ret; + + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &result); + sg_init_one(&sg, pad, blocksize); + ahash_request_set_crypt(req, &sg, pad, blocksize); + + ret = crypto_ahash_init(req); + if (ret) + return ret; + + ret = crypto_ahash_update(req); + ret = crypto_wait_req(ret, &result); + + if (ret) + return ret; + + ret = crypto_ahash_export(req, state); + if (ret) + return ret; + + return 0; +} + +static int mv_cesa_ahmac_pad_init(struct ahash_request *req, + const u8 *key, unsigned int keylen, + u8 *ipad, u8 *opad, + unsigned int blocksize) +{ + DECLARE_CRYPTO_WAIT(result); + struct scatterlist sg; + int ret; + int i; + + if (keylen <= blocksize) { + memcpy(ipad, key, keylen); + } else { + u8 *keydup = kmemdup(key, keylen, GFP_KERNEL); + + if (!keydup) + return -ENOMEM; + + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &result); + sg_init_one(&sg, keydup, keylen); + ahash_request_set_crypt(req, &sg, ipad, keylen); + + ret = crypto_ahash_digest(req); + ret = crypto_wait_req(ret, &result); + + /* Set the memory region to 0 to avoid any leak. */ + kfree_sensitive(keydup); + + if (ret) + return ret; + + keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(req)); + } + + memset(ipad + keylen, 0, blocksize - keylen); + memcpy(opad, ipad, blocksize); + + for (i = 0; i < blocksize; i++) { + ipad[i] ^= HMAC_IPAD_VALUE; + opad[i] ^= HMAC_OPAD_VALUE; + } + + return 0; +} + +static int mv_cesa_ahmac_setkey(const char *hash_alg_name, + const u8 *key, unsigned int keylen, + void *istate, void *ostate) +{ + struct ahash_request *req; + struct crypto_ahash *tfm; + unsigned int blocksize; + u8 *ipad = NULL; + u8 *opad; + int ret; + + tfm = crypto_alloc_ahash(hash_alg_name, 0, 0); + if (IS_ERR(tfm)) + return PTR_ERR(tfm); + + req = ahash_request_alloc(tfm, GFP_KERNEL); + if (!req) { + ret = -ENOMEM; + goto free_ahash; + } + + crypto_ahash_clear_flags(tfm, ~0); + + blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + + ipad = kcalloc(2, blocksize, GFP_KERNEL); + if (!ipad) { + ret = -ENOMEM; + goto free_req; + } + + opad = ipad + blocksize; + + ret = mv_cesa_ahmac_pad_init(req, key, keylen, ipad, opad, blocksize); + if (ret) + goto free_ipad; + + ret = mv_cesa_ahmac_iv_state_init(req, ipad, istate, blocksize); + if (ret) + goto free_ipad; + + ret = mv_cesa_ahmac_iv_state_init(req, opad, ostate, blocksize); + +free_ipad: + kfree(ipad); +free_req: + ahash_request_free(req); +free_ahash: + crypto_free_ahash(tfm); + + return ret; +} + +static int mv_cesa_ahmac_cra_init(struct crypto_tfm *tfm) +{ + struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->base.ops = &mv_cesa_ahash_req_ops; + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct mv_cesa_ahash_req)); + return 0; +} + +static int mv_cesa_ahmac_md5_init(struct ahash_request *req) +{ + struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct mv_cesa_op_ctx tmpl = { }; + + mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_HMAC_MD5); + memcpy(tmpl.ctx.hash.iv, ctx->iv, sizeof(ctx->iv)); + + mv_cesa_ahash_init(req, &tmpl, true); + + return 0; +} + +static int mv_cesa_ahmac_md5_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); + struct md5_state istate, ostate; + int ret, i; + + ret = mv_cesa_ahmac_setkey("mv-md5", key, keylen, &istate, &ostate); + if (ret) + return ret; + + for (i = 0; i < ARRAY_SIZE(istate.hash); i++) + ctx->iv[i] = cpu_to_be32(istate.hash[i]); + + for (i = 0; i < ARRAY_SIZE(ostate.hash); i++) + ctx->iv[i + 8] = cpu_to_be32(ostate.hash[i]); + + return 0; +} + +static int mv_cesa_ahmac_md5_digest(struct ahash_request *req) +{ + int ret; + + ret = mv_cesa_ahmac_md5_init(req); + if (ret) + return ret; + + return mv_cesa_ahash_finup(req); +} + +struct ahash_alg mv_ahmac_md5_alg = { + .init = mv_cesa_ahmac_md5_init, + .update = mv_cesa_ahash_update, + .final = mv_cesa_ahash_final, + .finup = mv_cesa_ahash_finup, + .digest = mv_cesa_ahmac_md5_digest, + .setkey = mv_cesa_ahmac_md5_setkey, + .export = mv_cesa_md5_export, + .import = mv_cesa_md5_import, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct md5_state), + .base = { + .cra_name = "hmac(md5)", + .cra_driver_name = "mv-hmac-md5", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_hmac_ctx), + .cra_init = mv_cesa_ahmac_cra_init, + .cra_module = THIS_MODULE, + } + } +}; + +static int mv_cesa_ahmac_sha1_init(struct ahash_request *req) +{ + struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct mv_cesa_op_ctx tmpl = { }; + + mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_HMAC_SHA1); + memcpy(tmpl.ctx.hash.iv, ctx->iv, sizeof(ctx->iv)); + + mv_cesa_ahash_init(req, &tmpl, false); + + return 0; +} + +static int mv_cesa_ahmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); + struct sha1_state istate, ostate; + int ret, i; + + ret = mv_cesa_ahmac_setkey("mv-sha1", key, keylen, &istate, &ostate); + if (ret) + return ret; + + for (i = 0; i < ARRAY_SIZE(istate.state); i++) + ctx->iv[i] = cpu_to_be32(istate.state[i]); + + for (i = 0; i < ARRAY_SIZE(ostate.state); i++) + ctx->iv[i + 8] = cpu_to_be32(ostate.state[i]); + + return 0; +} + +static int mv_cesa_ahmac_sha1_digest(struct ahash_request *req) +{ + int ret; + + ret = mv_cesa_ahmac_sha1_init(req); + if (ret) + return ret; + + return mv_cesa_ahash_finup(req); +} + +struct ahash_alg mv_ahmac_sha1_alg = { + .init = mv_cesa_ahmac_sha1_init, + .update = mv_cesa_ahash_update, + .final = mv_cesa_ahash_final, + .finup = mv_cesa_ahash_finup, + .digest = mv_cesa_ahmac_sha1_digest, + .setkey = mv_cesa_ahmac_sha1_setkey, + .export = mv_cesa_sha1_export, + .import = mv_cesa_sha1_import, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct sha1_state), + .base = { + .cra_name = "hmac(sha1)", + .cra_driver_name = "mv-hmac-sha1", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_hmac_ctx), + .cra_init = mv_cesa_ahmac_cra_init, + .cra_module = THIS_MODULE, + } + } +}; + +static int mv_cesa_ahmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); + struct sha256_state istate, ostate; + int ret, i; + + ret = mv_cesa_ahmac_setkey("mv-sha256", key, keylen, &istate, &ostate); + if (ret) + return ret; + + for (i = 0; i < ARRAY_SIZE(istate.state); i++) + ctx->iv[i] = cpu_to_be32(istate.state[i]); + + for (i = 0; i < ARRAY_SIZE(ostate.state); i++) + ctx->iv[i + 8] = cpu_to_be32(ostate.state[i]); + + return 0; +} + +static int mv_cesa_ahmac_sha256_init(struct ahash_request *req) +{ + struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct mv_cesa_op_ctx tmpl = { }; + + mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_HMAC_SHA256); + memcpy(tmpl.ctx.hash.iv, ctx->iv, sizeof(ctx->iv)); + + mv_cesa_ahash_init(req, &tmpl, false); + + return 0; +} + +static int mv_cesa_ahmac_sha256_digest(struct ahash_request *req) +{ + int ret; + + ret = mv_cesa_ahmac_sha256_init(req); + if (ret) + return ret; + + return mv_cesa_ahash_finup(req); +} + +struct ahash_alg mv_ahmac_sha256_alg = { + .init = mv_cesa_ahmac_sha256_init, + .update = mv_cesa_ahash_update, + .final = mv_cesa_ahash_final, + .finup = mv_cesa_ahash_finup, + .digest = mv_cesa_ahmac_sha256_digest, + .setkey = mv_cesa_ahmac_sha256_setkey, + .export = mv_cesa_sha256_export, + .import = mv_cesa_sha256_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "mv-hmac-sha256", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_hmac_ctx), + .cra_init = mv_cesa_ahmac_cra_init, + .cra_module = THIS_MODULE, + } + } +}; diff --git a/drivers/crypto/marvell/cesa/tdma.c b/drivers/crypto/marvell/cesa/tdma.c new file mode 100644 index 000000000..243305354 --- /dev/null +++ b/drivers/crypto/marvell/cesa/tdma.c @@ -0,0 +1,413 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Provide TDMA helper functions used by cipher and hash algorithm + * implementations. + * + * Author: Boris Brezillon + * Author: Arnaud Ebalard + * + * This work is based on an initial version written by + * Sebastian Andrzej Siewior < sebastian at breakpoint dot cc > + */ + +#include "cesa.h" + +bool mv_cesa_req_dma_iter_next_transfer(struct mv_cesa_dma_iter *iter, + struct mv_cesa_sg_dma_iter *sgiter, + unsigned int len) +{ + if (!sgiter->sg) + return false; + + sgiter->op_offset += len; + sgiter->offset += len; + if (sgiter->offset == sg_dma_len(sgiter->sg)) { + if (sg_is_last(sgiter->sg)) + return false; + sgiter->offset = 0; + sgiter->sg = sg_next(sgiter->sg); + } + + if (sgiter->op_offset == iter->op_len) + return false; + + return true; +} + +void mv_cesa_dma_step(struct mv_cesa_req *dreq) +{ + struct mv_cesa_engine *engine = dreq->engine; + + spin_lock_bh(&engine->lock); + if (engine->chain_sw.first == dreq->chain.first) { + engine->chain_sw.first = NULL; + engine->chain_sw.last = NULL; + } + engine->chain_hw.first = dreq->chain.first; + engine->chain_hw.last = dreq->chain.last; + spin_unlock_bh(&engine->lock); + + writel_relaxed(0, engine->regs + CESA_SA_CFG); + + mv_cesa_set_int_mask(engine, CESA_SA_INT_ACC0_IDMA_DONE); + writel_relaxed(CESA_TDMA_DST_BURST_128B | CESA_TDMA_SRC_BURST_128B | + CESA_TDMA_NO_BYTE_SWAP | CESA_TDMA_EN, + engine->regs + CESA_TDMA_CONTROL); + + writel_relaxed(CESA_SA_CFG_ACT_CH0_IDMA | CESA_SA_CFG_MULTI_PKT | + CESA_SA_CFG_CH0_W_IDMA | CESA_SA_CFG_PARA_DIS, + engine->regs + CESA_SA_CFG); + writel_relaxed(dreq->chain.first->cur_dma, + engine->regs + CESA_TDMA_NEXT_ADDR); + WARN_ON(readl(engine->regs + CESA_SA_CMD) & + CESA_SA_CMD_EN_CESA_SA_ACCL0); + writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD); +} + +void mv_cesa_dma_cleanup(struct mv_cesa_req *dreq) +{ + struct mv_cesa_tdma_desc *tdma; + + for (tdma = dreq->chain.first; tdma;) { + struct mv_cesa_tdma_desc *old_tdma = tdma; + u32 type = tdma->flags & CESA_TDMA_TYPE_MSK; + + if (type == CESA_TDMA_OP) + dma_pool_free(cesa_dev->dma->op_pool, tdma->op, + le32_to_cpu(tdma->src)); + + tdma = tdma->next; + dma_pool_free(cesa_dev->dma->tdma_desc_pool, old_tdma, + old_tdma->cur_dma); + } + + dreq->chain.first = NULL; + dreq->chain.last = NULL; +} + +void mv_cesa_dma_prepare(struct mv_cesa_req *dreq, + struct mv_cesa_engine *engine) +{ + struct mv_cesa_tdma_desc *tdma; + + for (tdma = dreq->chain.first; tdma; tdma = tdma->next) { + if (tdma->flags & CESA_TDMA_DST_IN_SRAM) + tdma->dst = cpu_to_le32(tdma->dst_dma + engine->sram_dma); + + if (tdma->flags & CESA_TDMA_SRC_IN_SRAM) + tdma->src = cpu_to_le32(tdma->src_dma + engine->sram_dma); + + if ((tdma->flags & CESA_TDMA_TYPE_MSK) == CESA_TDMA_OP) + mv_cesa_adjust_op(engine, tdma->op); + } +} + +void mv_cesa_tdma_chain(struct mv_cesa_engine *engine, + struct mv_cesa_req *dreq) +{ + struct mv_cesa_tdma_desc *last = engine->chain_sw.last; + + /* + * Break the DMA chain if the request being queued needs the IV + * regs to be set before lauching the request. + */ + if (!last || dreq->chain.first->flags & CESA_TDMA_SET_STATE) + engine->chain_sw.first = dreq->chain.first; + else { + last->next = dreq->chain.first; + last->next_dma = cpu_to_le32(dreq->chain.first->cur_dma); + } + last = dreq->chain.last; + engine->chain_sw.last = last; + /* + * Break the DMA chain if the CESA_TDMA_BREAK_CHAIN is set on + * the last element of the current chain. + */ + if (last->flags & CESA_TDMA_BREAK_CHAIN) { + engine->chain_sw.first = NULL; + engine->chain_sw.last = NULL; + } +} + +int mv_cesa_tdma_process(struct mv_cesa_engine *engine, u32 status) +{ + struct crypto_async_request *req = NULL; + struct mv_cesa_tdma_desc *tdma = NULL, *next = NULL; + dma_addr_t tdma_cur; + int res = 0; + + tdma_cur = readl(engine->regs + CESA_TDMA_CUR); + + for (tdma = engine->chain_hw.first; tdma; tdma = next) { + spin_lock_bh(&engine->lock); + next = tdma->next; + spin_unlock_bh(&engine->lock); + + if (tdma->flags & CESA_TDMA_END_OF_REQ) { + struct crypto_async_request *backlog = NULL; + struct mv_cesa_ctx *ctx; + u32 current_status; + + spin_lock_bh(&engine->lock); + /* + * if req is NULL, this means we're processing the + * request in engine->req. + */ + if (!req) + req = engine->req; + else + req = mv_cesa_dequeue_req_locked(engine, + &backlog); + + /* Re-chaining to the next request */ + engine->chain_hw.first = tdma->next; + tdma->next = NULL; + + /* If this is the last request, clear the chain */ + if (engine->chain_hw.first == NULL) + engine->chain_hw.last = NULL; + spin_unlock_bh(&engine->lock); + + ctx = crypto_tfm_ctx(req->tfm); + current_status = (tdma->cur_dma == tdma_cur) ? + status : CESA_SA_INT_ACC0_IDMA_DONE; + res = ctx->ops->process(req, current_status); + ctx->ops->complete(req); + + if (res == 0) + mv_cesa_engine_enqueue_complete_request(engine, + req); + + if (backlog) + crypto_request_complete(backlog, -EINPROGRESS); + } + + if (res || tdma->cur_dma == tdma_cur) + break; + } + + /* + * Save the last request in error to engine->req, so that the core + * knows which request was faulty + */ + if (res) { + spin_lock_bh(&engine->lock); + engine->req = req; + spin_unlock_bh(&engine->lock); + } + + return res; +} + +static struct mv_cesa_tdma_desc * +mv_cesa_dma_add_desc(struct mv_cesa_tdma_chain *chain, gfp_t flags) +{ + struct mv_cesa_tdma_desc *new_tdma = NULL; + dma_addr_t dma_handle; + + new_tdma = dma_pool_zalloc(cesa_dev->dma->tdma_desc_pool, flags, + &dma_handle); + if (!new_tdma) + return ERR_PTR(-ENOMEM); + + new_tdma->cur_dma = dma_handle; + if (chain->last) { + chain->last->next_dma = cpu_to_le32(dma_handle); + chain->last->next = new_tdma; + } else { + chain->first = new_tdma; + } + + chain->last = new_tdma; + + return new_tdma; +} + +int mv_cesa_dma_add_result_op(struct mv_cesa_tdma_chain *chain, dma_addr_t src, + u32 size, u32 flags, gfp_t gfp_flags) +{ + struct mv_cesa_tdma_desc *tdma, *op_desc; + + tdma = mv_cesa_dma_add_desc(chain, gfp_flags); + if (IS_ERR(tdma)) + return PTR_ERR(tdma); + + /* We re-use an existing op_desc object to retrieve the context + * and result instead of allocating a new one. + * There is at least one object of this type in a CESA crypto + * req, just pick the first one in the chain. + */ + for (op_desc = chain->first; op_desc; op_desc = op_desc->next) { + u32 type = op_desc->flags & CESA_TDMA_TYPE_MSK; + + if (type == CESA_TDMA_OP) + break; + } + + if (!op_desc) + return -EIO; + + tdma->byte_cnt = cpu_to_le32(size | BIT(31)); + tdma->src_dma = src; + tdma->dst_dma = op_desc->src_dma; + tdma->op = op_desc->op; + + flags &= (CESA_TDMA_DST_IN_SRAM | CESA_TDMA_SRC_IN_SRAM); + tdma->flags = flags | CESA_TDMA_RESULT; + return 0; +} + +struct mv_cesa_op_ctx *mv_cesa_dma_add_op(struct mv_cesa_tdma_chain *chain, + const struct mv_cesa_op_ctx *op_templ, + bool skip_ctx, + gfp_t flags) +{ + struct mv_cesa_tdma_desc *tdma; + struct mv_cesa_op_ctx *op; + dma_addr_t dma_handle; + unsigned int size; + + tdma = mv_cesa_dma_add_desc(chain, flags); + if (IS_ERR(tdma)) + return ERR_CAST(tdma); + + op = dma_pool_alloc(cesa_dev->dma->op_pool, flags, &dma_handle); + if (!op) + return ERR_PTR(-ENOMEM); + + *op = *op_templ; + + size = skip_ctx ? sizeof(op->desc) : sizeof(*op); + + tdma = chain->last; + tdma->op = op; + tdma->byte_cnt = cpu_to_le32(size | BIT(31)); + tdma->src = cpu_to_le32(dma_handle); + tdma->dst_dma = CESA_SA_CFG_SRAM_OFFSET; + tdma->flags = CESA_TDMA_DST_IN_SRAM | CESA_TDMA_OP; + + return op; +} + +int mv_cesa_dma_add_data_transfer(struct mv_cesa_tdma_chain *chain, + dma_addr_t dst, dma_addr_t src, u32 size, + u32 flags, gfp_t gfp_flags) +{ + struct mv_cesa_tdma_desc *tdma; + + tdma = mv_cesa_dma_add_desc(chain, gfp_flags); + if (IS_ERR(tdma)) + return PTR_ERR(tdma); + + tdma->byte_cnt = cpu_to_le32(size | BIT(31)); + tdma->src_dma = src; + tdma->dst_dma = dst; + + flags &= (CESA_TDMA_DST_IN_SRAM | CESA_TDMA_SRC_IN_SRAM); + tdma->flags = flags | CESA_TDMA_DATA; + + return 0; +} + +int mv_cesa_dma_add_dummy_launch(struct mv_cesa_tdma_chain *chain, gfp_t flags) +{ + struct mv_cesa_tdma_desc *tdma; + + tdma = mv_cesa_dma_add_desc(chain, flags); + return PTR_ERR_OR_ZERO(tdma); +} + +int mv_cesa_dma_add_dummy_end(struct mv_cesa_tdma_chain *chain, gfp_t flags) +{ + struct mv_cesa_tdma_desc *tdma; + + tdma = mv_cesa_dma_add_desc(chain, flags); + if (IS_ERR(tdma)) + return PTR_ERR(tdma); + + tdma->byte_cnt = cpu_to_le32(BIT(31)); + + return 0; +} + +int mv_cesa_dma_add_op_transfers(struct mv_cesa_tdma_chain *chain, + struct mv_cesa_dma_iter *dma_iter, + struct mv_cesa_sg_dma_iter *sgiter, + gfp_t gfp_flags) +{ + u32 flags = sgiter->dir == DMA_TO_DEVICE ? + CESA_TDMA_DST_IN_SRAM : CESA_TDMA_SRC_IN_SRAM; + unsigned int len; + + do { + dma_addr_t dst, src; + int ret; + + len = mv_cesa_req_dma_iter_transfer_len(dma_iter, sgiter); + if (sgiter->dir == DMA_TO_DEVICE) { + dst = CESA_SA_DATA_SRAM_OFFSET + sgiter->op_offset; + src = sg_dma_address(sgiter->sg) + sgiter->offset; + } else { + dst = sg_dma_address(sgiter->sg) + sgiter->offset; + src = CESA_SA_DATA_SRAM_OFFSET + sgiter->op_offset; + } + + ret = mv_cesa_dma_add_data_transfer(chain, dst, src, len, + flags, gfp_flags); + if (ret) + return ret; + + } while (mv_cesa_req_dma_iter_next_transfer(dma_iter, sgiter, len)); + + return 0; +} + +size_t mv_cesa_sg_copy(struct mv_cesa_engine *engine, + struct scatterlist *sgl, unsigned int nents, + unsigned int sram_off, size_t buflen, off_t skip, + bool to_sram) +{ + unsigned int sg_flags = SG_MITER_ATOMIC; + struct sg_mapping_iter miter; + unsigned int offset = 0; + + if (to_sram) + sg_flags |= SG_MITER_FROM_SG; + else + sg_flags |= SG_MITER_TO_SG; + + sg_miter_start(&miter, sgl, nents, sg_flags); + + if (!sg_miter_skip(&miter, skip)) + return 0; + + while ((offset < buflen) && sg_miter_next(&miter)) { + unsigned int len; + + len = min(miter.length, buflen - offset); + + if (to_sram) { + if (engine->pool) + memcpy(engine->sram_pool + sram_off + offset, + miter.addr, len); + else + memcpy_toio(engine->sram + sram_off + offset, + miter.addr, len); + } else { + if (engine->pool) + memcpy(miter.addr, + engine->sram_pool + sram_off + offset, + len); + else + memcpy_fromio(miter.addr, + engine->sram + sram_off + offset, + len); + } + + offset += len; + } + + sg_miter_stop(&miter); + + return offset; +} diff --git a/drivers/crypto/marvell/octeontx/Makefile b/drivers/crypto/marvell/octeontx/Makefile new file mode 100644 index 000000000..5e956fe1a --- /dev/null +++ b/drivers/crypto/marvell/octeontx/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_CRYPTO_DEV_OCTEONTX_CPT) += octeontx-cpt.o octeontx-cptvf.o + +octeontx-cpt-objs := otx_cptpf_main.o otx_cptpf_mbox.o otx_cptpf_ucode.o +octeontx-cptvf-objs := otx_cptvf_main.o otx_cptvf_mbox.o otx_cptvf_reqmgr.o \ + otx_cptvf_algs.o diff --git a/drivers/crypto/marvell/octeontx/otx_cpt_common.h b/drivers/crypto/marvell/octeontx/otx_cpt_common.h new file mode 100644 index 000000000..ca704a7a2 --- /dev/null +++ b/drivers/crypto/marvell/octeontx/otx_cpt_common.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 + * Marvell OcteonTX CPT driver + * + * Copyright (C) 2019 Marvell International Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef __OTX_CPT_COMMON_H +#define __OTX_CPT_COMMON_H + +#include +#include +#include + +#define OTX_CPT_MAX_MBOX_DATA_STR_SIZE 64 + +enum otx_cptpf_type { + OTX_CPT_AE = 2, + OTX_CPT_SE = 3, + BAD_OTX_CPTPF_TYPE, +}; + +enum otx_cptvf_type { + OTX_CPT_AE_TYPES = 1, + OTX_CPT_SE_TYPES = 2, + BAD_OTX_CPTVF_TYPE, +}; + +/* VF-PF message opcodes */ +enum otx_cpt_mbox_opcode { + OTX_CPT_MSG_VF_UP = 1, + OTX_CPT_MSG_VF_DOWN, + OTX_CPT_MSG_READY, + OTX_CPT_MSG_QLEN, + OTX_CPT_MSG_QBIND_GRP, + OTX_CPT_MSG_VQ_PRIORITY, + OTX_CPT_MSG_PF_TYPE, + OTX_CPT_MSG_ACK, + OTX_CPT_MSG_NACK +}; + +/* OcteonTX CPT mailbox structure */ +struct otx_cpt_mbox { + u64 msg; /* Message type MBOX[0] */ + u64 data;/* Data MBOX[1] */ +}; + +#endif /* __OTX_CPT_COMMON_H */ diff --git a/drivers/crypto/marvell/octeontx/otx_cpt_hw_types.h b/drivers/crypto/marvell/octeontx/otx_cpt_hw_types.h new file mode 100644 index 000000000..f8aedafdf --- /dev/null +++ b/drivers/crypto/marvell/octeontx/otx_cpt_hw_types.h @@ -0,0 +1,824 @@ +/* SPDX-License-Identifier: GPL-2.0 + * Marvell OcteonTX CPT driver + * + * Copyright (C) 2019 Marvell International Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef __OTX_CPT_HW_TYPES_H +#define __OTX_CPT_HW_TYPES_H + +#include + +/* Device IDs */ +#define OTX_CPT_PCI_PF_DEVICE_ID 0xa040 +#define OTX_CPT_PCI_VF_DEVICE_ID 0xa041 + +#define OTX_CPT_PCI_PF_SUBSYS_ID 0xa340 +#define OTX_CPT_PCI_VF_SUBSYS_ID 0xa341 + +/* Configuration and status registers are in BAR0 on OcteonTX platform */ +#define OTX_CPT_PF_PCI_CFG_BAR 0 +#define OTX_CPT_VF_PCI_CFG_BAR 0 + +#define OTX_CPT_BAR_E_CPTX_VFX_BAR0_OFFSET(a, b) \ + (0x000020000000ll + 0x1000000000ll * (a) + 0x100000ll * (b)) +#define OTX_CPT_BAR_E_CPTX_VFX_BAR0_SIZE 0x400000 + +/* Mailbox interrupts offset */ +#define OTX_CPT_PF_MBOX_INT 3 +#define OTX_CPT_PF_INT_VEC_E_MBOXX(x, a) ((x) + (a)) +/* Number of MSIX supported in PF */ +#define OTX_CPT_PF_MSIX_VECTORS 4 +/* Maximum supported microcode groups */ +#define OTX_CPT_MAX_ENGINE_GROUPS 8 + +/* CPT instruction size in bytes */ +#define OTX_CPT_INST_SIZE 64 +/* CPT queue next chunk pointer size in bytes */ +#define OTX_CPT_NEXT_CHUNK_PTR_SIZE 8 + +/* OcteonTX CPT VF MSIX vectors and their offsets */ +#define OTX_CPT_VF_MSIX_VECTORS 2 +#define OTX_CPT_VF_INTR_MBOX_MASK BIT(0) +#define OTX_CPT_VF_INTR_DOVF_MASK BIT(1) +#define OTX_CPT_VF_INTR_IRDE_MASK BIT(2) +#define OTX_CPT_VF_INTR_NWRP_MASK BIT(3) +#define OTX_CPT_VF_INTR_SERR_MASK BIT(4) + +/* OcteonTX CPT PF registers */ +#define OTX_CPT_PF_CONSTANTS (0x0ll) +#define OTX_CPT_PF_RESET (0x100ll) +#define OTX_CPT_PF_DIAG (0x120ll) +#define OTX_CPT_PF_BIST_STATUS (0x160ll) +#define OTX_CPT_PF_ECC0_CTL (0x200ll) +#define OTX_CPT_PF_ECC0_FLIP (0x210ll) +#define OTX_CPT_PF_ECC0_INT (0x220ll) +#define OTX_CPT_PF_ECC0_INT_W1S (0x230ll) +#define OTX_CPT_PF_ECC0_ENA_W1S (0x240ll) +#define OTX_CPT_PF_ECC0_ENA_W1C (0x250ll) +#define OTX_CPT_PF_MBOX_INTX(b) (0x400ll | (u64)(b) << 3) +#define OTX_CPT_PF_MBOX_INT_W1SX(b) (0x420ll | (u64)(b) << 3) +#define OTX_CPT_PF_MBOX_ENA_W1CX(b) (0x440ll | (u64)(b) << 3) +#define OTX_CPT_PF_MBOX_ENA_W1SX(b) (0x460ll | (u64)(b) << 3) +#define OTX_CPT_PF_EXEC_INT (0x500ll) +#define OTX_CPT_PF_EXEC_INT_W1S (0x520ll) +#define OTX_CPT_PF_EXEC_ENA_W1C (0x540ll) +#define OTX_CPT_PF_EXEC_ENA_W1S (0x560ll) +#define OTX_CPT_PF_GX_EN(b) (0x600ll | (u64)(b) << 3) +#define OTX_CPT_PF_EXEC_INFO (0x700ll) +#define OTX_CPT_PF_EXEC_BUSY (0x800ll) +#define OTX_CPT_PF_EXEC_INFO0 (0x900ll) +#define OTX_CPT_PF_EXEC_INFO1 (0x910ll) +#define OTX_CPT_PF_INST_REQ_PC (0x10000ll) +#define OTX_CPT_PF_INST_LATENCY_PC (0x10020ll) +#define OTX_CPT_PF_RD_REQ_PC (0x10040ll) +#define OTX_CPT_PF_RD_LATENCY_PC (0x10060ll) +#define OTX_CPT_PF_RD_UC_PC (0x10080ll) +#define OTX_CPT_PF_ACTIVE_CYCLES_PC (0x10100ll) +#define OTX_CPT_PF_EXE_CTL (0x4000000ll) +#define OTX_CPT_PF_EXE_STATUS (0x4000008ll) +#define OTX_CPT_PF_EXE_CLK (0x4000010ll) +#define OTX_CPT_PF_EXE_DBG_CTL (0x4000018ll) +#define OTX_CPT_PF_EXE_DBG_DATA (0x4000020ll) +#define OTX_CPT_PF_EXE_BIST_STATUS (0x4000028ll) +#define OTX_CPT_PF_EXE_REQ_TIMER (0x4000030ll) +#define OTX_CPT_PF_EXE_MEM_CTL (0x4000038ll) +#define OTX_CPT_PF_EXE_PERF_CTL (0x4001000ll) +#define OTX_CPT_PF_EXE_DBG_CNTX(b) (0x4001100ll | (u64)(b) << 3) +#define OTX_CPT_PF_EXE_PERF_EVENT_CNT (0x4001180ll) +#define OTX_CPT_PF_EXE_EPCI_INBX_CNT(b) (0x4001200ll | (u64)(b) << 3) +#define OTX_CPT_PF_EXE_EPCI_OUTBX_CNT(b) (0x4001240ll | (u64)(b) << 3) +#define OTX_CPT_PF_ENGX_UCODE_BASE(b) (0x4002000ll | (u64)(b) << 3) +#define OTX_CPT_PF_QX_CTL(b) (0x8000000ll | (u64)(b) << 20) +#define OTX_CPT_PF_QX_GMCTL(b) (0x8000020ll | (u64)(b) << 20) +#define OTX_CPT_PF_QX_CTL2(b) (0x8000100ll | (u64)(b) << 20) +#define OTX_CPT_PF_VFX_MBOXX(b, c) (0x8001000ll | (u64)(b) << 20 | \ + (u64)(c) << 8) + +/* OcteonTX CPT VF registers */ +#define OTX_CPT_VQX_CTL(b) (0x100ll | (u64)(b) << 20) +#define OTX_CPT_VQX_SADDR(b) (0x200ll | (u64)(b) << 20) +#define OTX_CPT_VQX_DONE_WAIT(b) (0x400ll | (u64)(b) << 20) +#define OTX_CPT_VQX_INPROG(b) (0x410ll | (u64)(b) << 20) +#define OTX_CPT_VQX_DONE(b) (0x420ll | (u64)(b) << 20) +#define OTX_CPT_VQX_DONE_ACK(b) (0x440ll | (u64)(b) << 20) +#define OTX_CPT_VQX_DONE_INT_W1S(b) (0x460ll | (u64)(b) << 20) +#define OTX_CPT_VQX_DONE_INT_W1C(b) (0x468ll | (u64)(b) << 20) +#define OTX_CPT_VQX_DONE_ENA_W1S(b) (0x470ll | (u64)(b) << 20) +#define OTX_CPT_VQX_DONE_ENA_W1C(b) (0x478ll | (u64)(b) << 20) +#define OTX_CPT_VQX_MISC_INT(b) (0x500ll | (u64)(b) << 20) +#define OTX_CPT_VQX_MISC_INT_W1S(b) (0x508ll | (u64)(b) << 20) +#define OTX_CPT_VQX_MISC_ENA_W1S(b) (0x510ll | (u64)(b) << 20) +#define OTX_CPT_VQX_MISC_ENA_W1C(b) (0x518ll | (u64)(b) << 20) +#define OTX_CPT_VQX_DOORBELL(b) (0x600ll | (u64)(b) << 20) +#define OTX_CPT_VFX_PF_MBOXX(b, c) (0x1000ll | ((b) << 20) | ((c) << 3)) + +/* + * Enumeration otx_cpt_ucode_error_code_e + * + * Enumerates ucode errors + */ +enum otx_cpt_ucode_error_code_e { + CPT_NO_UCODE_ERROR = 0x00, + ERR_OPCODE_UNSUPPORTED = 0x01, + + /* Scatter gather */ + ERR_SCATTER_GATHER_WRITE_LENGTH = 0x02, + ERR_SCATTER_GATHER_LIST = 0x03, + ERR_SCATTER_GATHER_NOT_SUPPORTED = 0x04, + +}; + +/* + * Enumeration otx_cpt_comp_e + * + * CPT OcteonTX Completion Enumeration + * Enumerates the values of CPT_RES_S[COMPCODE]. + */ +enum otx_cpt_comp_e { + CPT_COMP_E_NOTDONE = 0x00, + CPT_COMP_E_GOOD = 0x01, + CPT_COMP_E_FAULT = 0x02, + CPT_COMP_E_SWERR = 0x03, + CPT_COMP_E_HWERR = 0x04, + CPT_COMP_E_LAST_ENTRY = 0x05 +}; + +/* + * Enumeration otx_cpt_vf_int_vec_e + * + * CPT OcteonTX VF MSI-X Vector Enumeration + * Enumerates the MSI-X interrupt vectors. + */ +enum otx_cpt_vf_int_vec_e { + CPT_VF_INT_VEC_E_MISC = 0x00, + CPT_VF_INT_VEC_E_DONE = 0x01 +}; + +/* + * Structure cpt_inst_s + * + * CPT Instruction Structure + * This structure specifies the instruction layout. Instructions are + * stored in memory as little-endian unless CPT()_PF_Q()_CTL[INST_BE] is set. + * cpt_inst_s_s + * Word 0 + * doneint:1 Done interrupt. + * 0 = No interrupts related to this instruction. + * 1 = When the instruction completes, CPT()_VQ()_DONE[DONE] will be + * incremented,and based on the rules described there an interrupt may + * occur. + * Word 1 + * res_addr [127: 64] Result IOVA. + * If nonzero, specifies where to write CPT_RES_S. + * If zero, no result structure will be written. + * Address must be 16-byte aligned. + * Bits <63:49> are ignored by hardware; software should use a + * sign-extended bit <48> for forward compatibility. + * Word 2 + * grp:10 [171:162] If [WQ_PTR] is nonzero, the SSO guest-group to use when + * CPT submits work SSO. + * For the SSO to not discard the add-work request, FPA_PF_MAP() must map + * [GRP] and CPT()_PF_Q()_GMCTL[GMID] as valid. + * tt:2 [161:160] If [WQ_PTR] is nonzero, the SSO tag type to use when CPT + * submits work to SSO + * tag:32 [159:128] If [WQ_PTR] is nonzero, the SSO tag to use when CPT + * submits work to SSO. + * Word 3 + * wq_ptr [255:192] If [WQ_PTR] is nonzero, it is a pointer to a + * work-queue entry that CPT submits work to SSO after all context, + * output data, and result write operations are visible to other + * CNXXXX units and the cores. Bits <2:0> must be zero. + * Bits <63:49> are ignored by hardware; software should + * use a sign-extended bit <48> for forward compatibility. + * Internal: + * Bits <63:49>, <2:0> are ignored by hardware, treated as always 0x0. + * Word 4 + * ei0; [319:256] Engine instruction word 0. Passed to the AE/SE. + * Word 5 + * ei1; [383:320] Engine instruction word 1. Passed to the AE/SE. + * Word 6 + * ei2; [447:384] Engine instruction word 1. Passed to the AE/SE. + * Word 7 + * ei3; [511:448] Engine instruction word 1. Passed to the AE/SE. + * + */ +union otx_cpt_inst_s { + u64 u[8]; + + struct { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_17_63:47; + u64 doneint:1; + u64 reserved_0_15:16; +#else /* Word 0 - Little Endian */ + u64 reserved_0_15:16; + u64 doneint:1; + u64 reserved_17_63:47; +#endif /* Word 0 - End */ + u64 res_addr; +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 2 - Big Endian */ + u64 reserved_172_191:20; + u64 grp:10; + u64 tt:2; + u64 tag:32; +#else /* Word 2 - Little Endian */ + u64 tag:32; + u64 tt:2; + u64 grp:10; + u64 reserved_172_191:20; +#endif /* Word 2 - End */ + u64 wq_ptr; + u64 ei0; + u64 ei1; + u64 ei2; + u64 ei3; + } s; +}; + +/* + * Structure cpt_res_s + * + * CPT Result Structure + * The CPT coprocessor writes the result structure after it completes a + * CPT_INST_S instruction. The result structure is exactly 16 bytes, and + * each instruction completion produces exactly one result structure. + * + * This structure is stored in memory as little-endian unless + * CPT()_PF_Q()_CTL[INST_BE] is set. + * cpt_res_s_s + * Word 0 + * doneint:1 [16:16] Done interrupt. This bit is copied from the + * corresponding instruction's CPT_INST_S[DONEINT]. + * compcode:8 [7:0] Indicates completion/error status of the CPT coprocessor + * for the associated instruction, as enumerated by CPT_COMP_E. + * Core software may write the memory location containing [COMPCODE] to + * 0x0 before ringing the doorbell, and then poll for completion by + * checking for a nonzero value. + * Once the core observes a nonzero [COMPCODE] value in this case,the CPT + * coprocessor will have also completed L2/DRAM write operations. + * Word 1 + * reserved + * + */ +union otx_cpt_res_s { + u64 u[2]; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_17_63:47; + u64 doneint:1; + u64 reserved_8_15:8; + u64 compcode:8; +#else /* Word 0 - Little Endian */ + u64 compcode:8; + u64 reserved_8_15:8; + u64 doneint:1; + u64 reserved_17_63:47; +#endif /* Word 0 - End */ + u64 reserved_64_127; + } s; +}; + +/* + * Register (NCB) otx_cpt#_pf_bist_status + * + * CPT PF Control Bist Status Register + * This register has the BIST status of memories. Each bit is the BIST result + * of an individual memory (per bit, 0 = pass and 1 = fail). + * otx_cptx_pf_bist_status_s + * Word0 + * bstatus [29:0](RO/H) BIST status. One bit per memory, enumerated by + * CPT_RAMS_E. + */ +union otx_cptx_pf_bist_status { + u64 u; + struct otx_cptx_pf_bist_status_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_30_63:34; + u64 bstatus:30; +#else /* Word 0 - Little Endian */ + u64 bstatus:30; + u64 reserved_30_63:34; +#endif /* Word 0 - End */ + } s; +}; + +/* + * Register (NCB) otx_cpt#_pf_constants + * + * CPT PF Constants Register + * This register contains implementation-related parameters of CPT in CNXXXX. + * otx_cptx_pf_constants_s + * Word 0 + * reserved_40_63:24 [63:40] Reserved. + * epcis:8 [39:32](RO) Number of EPCI busses. + * grps:8 [31:24](RO) Number of engine groups implemented. + * ae:8 [23:16](RO/H) Number of AEs. In CNXXXX, for CPT0 returns 0x0, + * for CPT1 returns 0x18, or less if there are fuse-disables. + * se:8 [15:8](RO/H) Number of SEs. In CNXXXX, for CPT0 returns 0x30, + * or less if there are fuse-disables, for CPT1 returns 0x0. + * vq:8 [7:0](RO) Number of VQs. + */ +union otx_cptx_pf_constants { + u64 u; + struct otx_cptx_pf_constants_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_40_63:24; + u64 epcis:8; + u64 grps:8; + u64 ae:8; + u64 se:8; + u64 vq:8; +#else /* Word 0 - Little Endian */ + u64 vq:8; + u64 se:8; + u64 ae:8; + u64 grps:8; + u64 epcis:8; + u64 reserved_40_63:24; +#endif /* Word 0 - End */ + } s; +}; + +/* + * Register (NCB) otx_cpt#_pf_exe_bist_status + * + * CPT PF Engine Bist Status Register + * This register has the BIST status of each engine. Each bit is the + * BIST result of an individual engine (per bit, 0 = pass and 1 = fail). + * otx_cptx_pf_exe_bist_status_s + * Word0 + * reserved_48_63:16 [63:48] reserved + * bstatus:48 [47:0](RO/H) BIST status. One bit per engine. + * + */ +union otx_cptx_pf_exe_bist_status { + u64 u; + struct otx_cptx_pf_exe_bist_status_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_48_63:16; + u64 bstatus:48; +#else /* Word 0 - Little Endian */ + u64 bstatus:48; + u64 reserved_48_63:16; +#endif /* Word 0 - End */ + } s; +}; + +/* + * Register (NCB) otx_cpt#_pf_q#_ctl + * + * CPT Queue Control Register + * This register configures queues. This register should be changed only + * when quiescent (see CPT()_VQ()_INPROG[INFLIGHT]). + * otx_cptx_pf_qx_ctl_s + * Word0 + * reserved_60_63:4 [63:60] reserved. + * aura:12; [59:48](R/W) Guest-aura for returning this queue's + * instruction-chunk buffers to FPA. Only used when [INST_FREE] is set. + * For the FPA to not discard the request, FPA_PF_MAP() must map + * [AURA] and CPT()_PF_Q()_GMCTL[GMID] as valid. + * reserved_45_47:3 [47:45] reserved. + * size:13 [44:32](R/W) Command-buffer size, in number of 64-bit words per + * command buffer segment. Must be 8*n + 1, where n is the number of + * instructions per buffer segment. + * reserved_11_31:21 [31:11] Reserved. + * cont_err:1 [10:10](R/W) Continue on error. + * 0 = When CPT()_VQ()_MISC_INT[NWRP], CPT()_VQ()_MISC_INT[IRDE] or + * CPT()_VQ()_MISC_INT[DOVF] are set by hardware or software via + * CPT()_VQ()_MISC_INT_W1S, then CPT()_VQ()_CTL[ENA] is cleared. Due to + * pipelining, additional instructions may have been processed between the + * instruction causing the error and the next instruction in the disabled + * queue (the instruction at CPT()_VQ()_SADDR). + * 1 = Ignore errors and continue processing instructions. + * For diagnostic use only. + * inst_free:1 [9:9](R/W) Instruction FPA free. When set, when CPT reaches the + * end of an instruction chunk, that chunk will be freed to the FPA. + * inst_be:1 [8:8](R/W) Instruction big-endian control. When set, instructions, + * instruction next chunk pointers, and result structures are stored in + * big-endian format in memory. + * iqb_ldwb:1 [7:7](R/W) Instruction load don't write back. + * 0 = The hardware issues NCB transient load (LDT) towards the cache, + * which if the line hits and is dirty will cause the line to be + * written back before being replaced. + * 1 = The hardware issues NCB LDWB read-and-invalidate command towards + * the cache when fetching the last word of instructions; as a result the + * line will not be written back when replaced. This improves + * performance, but software must not read the instructions after they are + * posted to the hardware. Reads that do not consume the last word of a + * cache line always use LDI. + * reserved_4_6:3 [6:4] Reserved. + * grp:3; [3:1](R/W) Engine group. + * pri:1; [0:0](R/W) Queue priority. + * 1 = This queue has higher priority. Round-robin between higher + * priority queues. + * 0 = This queue has lower priority. Round-robin between lower + * priority queues. + */ +union otx_cptx_pf_qx_ctl { + u64 u; + struct otx_cptx_pf_qx_ctl_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_60_63:4; + u64 aura:12; + u64 reserved_45_47:3; + u64 size:13; + u64 reserved_11_31:21; + u64 cont_err:1; + u64 inst_free:1; + u64 inst_be:1; + u64 iqb_ldwb:1; + u64 reserved_4_6:3; + u64 grp:3; + u64 pri:1; +#else /* Word 0 - Little Endian */ + u64 pri:1; + u64 grp:3; + u64 reserved_4_6:3; + u64 iqb_ldwb:1; + u64 inst_be:1; + u64 inst_free:1; + u64 cont_err:1; + u64 reserved_11_31:21; + u64 size:13; + u64 reserved_45_47:3; + u64 aura:12; + u64 reserved_60_63:4; +#endif /* Word 0 - End */ + } s; +}; + +/* + * Register (NCB) otx_cpt#_vq#_saddr + * + * CPT Queue Starting Buffer Address Registers + * These registers set the instruction buffer starting address. + * otx_cptx_vqx_saddr_s + * Word0 + * reserved_49_63:15 [63:49] Reserved. + * ptr:43 [48:6](R/W/H) Instruction buffer IOVA <48:6> (64-byte aligned). + * When written, it is the initial buffer starting address; when read, + * it is the next read pointer to be requested from L2C. The PTR field + * is overwritten with the next pointer each time that the command buffer + * segment is exhausted. New commands will then be read from the newly + * specified command buffer pointer. + * reserved_0_5:6 [5:0] Reserved. + * + */ +union otx_cptx_vqx_saddr { + u64 u; + struct otx_cptx_vqx_saddr_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_49_63:15; + u64 ptr:43; + u64 reserved_0_5:6; +#else /* Word 0 - Little Endian */ + u64 reserved_0_5:6; + u64 ptr:43; + u64 reserved_49_63:15; +#endif /* Word 0 - End */ + } s; +}; + +/* + * Register (NCB) otx_cpt#_vq#_misc_ena_w1s + * + * CPT Queue Misc Interrupt Enable Set Register + * This register sets interrupt enable bits. + * otx_cptx_vqx_misc_ena_w1s_s + * Word0 + * reserved_5_63:59 [63:5] Reserved. + * swerr:1 [4:4](R/W1S/H) Reads or sets enable for + * CPT(0..1)_VQ(0..63)_MISC_INT[SWERR]. + * nwrp:1 [3:3](R/W1S/H) Reads or sets enable for + * CPT(0..1)_VQ(0..63)_MISC_INT[NWRP]. + * irde:1 [2:2](R/W1S/H) Reads or sets enable for + * CPT(0..1)_VQ(0..63)_MISC_INT[IRDE]. + * dovf:1 [1:1](R/W1S/H) Reads or sets enable for + * CPT(0..1)_VQ(0..63)_MISC_INT[DOVF]. + * mbox:1 [0:0](R/W1S/H) Reads or sets enable for + * CPT(0..1)_VQ(0..63)_MISC_INT[MBOX]. + * + */ +union otx_cptx_vqx_misc_ena_w1s { + u64 u; + struct otx_cptx_vqx_misc_ena_w1s_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_5_63:59; + u64 swerr:1; + u64 nwrp:1; + u64 irde:1; + u64 dovf:1; + u64 mbox:1; +#else /* Word 0 - Little Endian */ + u64 mbox:1; + u64 dovf:1; + u64 irde:1; + u64 nwrp:1; + u64 swerr:1; + u64 reserved_5_63:59; +#endif /* Word 0 - End */ + } s; +}; + +/* + * Register (NCB) otx_cpt#_vq#_doorbell + * + * CPT Queue Doorbell Registers + * Doorbells for the CPT instruction queues. + * otx_cptx_vqx_doorbell_s + * Word0 + * reserved_20_63:44 [63:20] Reserved. + * dbell_cnt:20 [19:0](R/W/H) Number of instruction queue 64-bit words to add + * to the CPT instruction doorbell count. Readback value is the + * current number of pending doorbell requests. If counter overflows + * CPT()_VQ()_MISC_INT[DBELL_DOVF] is set. To reset the count back to + * zero, write one to clear CPT()_VQ()_MISC_INT_ENA_W1C[DBELL_DOVF], + * then write a value of 2^20 minus the read [DBELL_CNT], then write one + * to CPT()_VQ()_MISC_INT_W1C[DBELL_DOVF] and + * CPT()_VQ()_MISC_INT_ENA_W1S[DBELL_DOVF]. Must be a multiple of 8. + * All CPT instructions are 8 words and require a doorbell count of + * multiple of 8. + */ +union otx_cptx_vqx_doorbell { + u64 u; + struct otx_cptx_vqx_doorbell_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_20_63:44; + u64 dbell_cnt:20; +#else /* Word 0 - Little Endian */ + u64 dbell_cnt:20; + u64 reserved_20_63:44; +#endif /* Word 0 - End */ + } s; +}; + +/* + * Register (NCB) otx_cpt#_vq#_inprog + * + * CPT Queue In Progress Count Registers + * These registers contain the per-queue instruction in flight registers. + * otx_cptx_vqx_inprog_s + * Word0 + * reserved_8_63:56 [63:8] Reserved. + * inflight:8 [7:0](RO/H) Inflight count. Counts the number of instructions + * for the VF for which CPT is fetching, executing or responding to + * instructions. However this does not include any interrupts that are + * awaiting software handling (CPT()_VQ()_DONE[DONE] != 0x0). + * A queue may not be reconfigured until: + * 1. CPT()_VQ()_CTL[ENA] is cleared by software. + * 2. [INFLIGHT] is polled until equals to zero. + */ +union otx_cptx_vqx_inprog { + u64 u; + struct otx_cptx_vqx_inprog_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_8_63:56; + u64 inflight:8; +#else /* Word 0 - Little Endian */ + u64 inflight:8; + u64 reserved_8_63:56; +#endif /* Word 0 - End */ + } s; +}; + +/* + * Register (NCB) otx_cpt#_vq#_misc_int + * + * CPT Queue Misc Interrupt Register + * These registers contain the per-queue miscellaneous interrupts. + * otx_cptx_vqx_misc_int_s + * Word 0 + * reserved_5_63:59 [63:5] Reserved. + * swerr:1 [4:4](R/W1C/H) Software error from engines. + * nwrp:1 [3:3](R/W1C/H) NCB result write response error. + * irde:1 [2:2](R/W1C/H) Instruction NCB read response error. + * dovf:1 [1:1](R/W1C/H) Doorbell overflow. + * mbox:1 [0:0](R/W1C/H) PF to VF mailbox interrupt. Set when + * CPT()_VF()_PF_MBOX(0) is written. + * + */ +union otx_cptx_vqx_misc_int { + u64 u; + struct otx_cptx_vqx_misc_int_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_5_63:59; + u64 swerr:1; + u64 nwrp:1; + u64 irde:1; + u64 dovf:1; + u64 mbox:1; +#else /* Word 0 - Little Endian */ + u64 mbox:1; + u64 dovf:1; + u64 irde:1; + u64 nwrp:1; + u64 swerr:1; + u64 reserved_5_63:59; +#endif /* Word 0 - End */ + } s; +}; + +/* + * Register (NCB) otx_cpt#_vq#_done_ack + * + * CPT Queue Done Count Ack Registers + * This register is written by software to acknowledge interrupts. + * otx_cptx_vqx_done_ack_s + * Word0 + * reserved_20_63:44 [63:20] Reserved. + * done_ack:20 [19:0](R/W/H) Number of decrements to CPT()_VQ()_DONE[DONE]. + * Reads CPT()_VQ()_DONE[DONE]. Written by software to acknowledge + * interrupts. If CPT()_VQ()_DONE[DONE] is still nonzero the interrupt + * will be re-sent if the conditions described in CPT()_VQ()_DONE[DONE] + * are satisfied. + * + */ +union otx_cptx_vqx_done_ack { + u64 u; + struct otx_cptx_vqx_done_ack_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_20_63:44; + u64 done_ack:20; +#else /* Word 0 - Little Endian */ + u64 done_ack:20; + u64 reserved_20_63:44; +#endif /* Word 0 - End */ + } s; +}; + +/* + * Register (NCB) otx_cpt#_vq#_done + * + * CPT Queue Done Count Registers + * These registers contain the per-queue instruction done count. + * cptx_vqx_done_s + * Word0 + * reserved_20_63:44 [63:20] Reserved. + * done:20 [19:0](R/W/H) Done count. When CPT_INST_S[DONEINT] set and that + * instruction completes, CPT()_VQ()_DONE[DONE] is incremented when the + * instruction finishes. Write to this field are for diagnostic use only; + * instead software writes CPT()_VQ()_DONE_ACK with the number of + * decrements for this field. + * Interrupts are sent as follows: + * * When CPT()_VQ()_DONE[DONE] = 0, then no results are pending, the + * interrupt coalescing timer is held to zero, and an interrupt is not + * sent. + * * When CPT()_VQ()_DONE[DONE] != 0, then the interrupt coalescing timer + * counts. If the counter is >= CPT()_VQ()_DONE_WAIT[TIME_WAIT]*1024, or + * CPT()_VQ()_DONE[DONE] >= CPT()_VQ()_DONE_WAIT[NUM_WAIT], i.e. enough + * time has passed or enough results have arrived, then the interrupt is + * sent. + * * When CPT()_VQ()_DONE_ACK is written (or CPT()_VQ()_DONE is written + * but this is not typical), the interrupt coalescing timer restarts. + * Note after decrementing this interrupt equation is recomputed, + * for example if CPT()_VQ()_DONE[DONE] >= CPT()_VQ()_DONE_WAIT[NUM_WAIT] + * and because the timer is zero, the interrupt will be resent immediately. + * (This covers the race case between software acknowledging an interrupt + * and a result returning.) + * * When CPT()_VQ()_DONE_ENA_W1S[DONE] = 0, interrupts are not sent, + * but the counting described above still occurs. + * Since CPT instructions complete out-of-order, if software is using + * completion interrupts the suggested scheme is to request a DONEINT on + * each request, and when an interrupt arrives perform a "greedy" scan for + * completions; even if a later command is acknowledged first this will + * not result in missing a completion. + * Software is responsible for making sure [DONE] does not overflow; + * for example by insuring there are not more than 2^20-1 instructions in + * flight that may request interrupts. + * + */ +union otx_cptx_vqx_done { + u64 u; + struct otx_cptx_vqx_done_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_20_63:44; + u64 done:20; +#else /* Word 0 - Little Endian */ + u64 done:20; + u64 reserved_20_63:44; +#endif /* Word 0 - End */ + } s; +}; + +/* + * Register (NCB) otx_cpt#_vq#_done_wait + * + * CPT Queue Done Interrupt Coalescing Wait Registers + * Specifies the per queue interrupt coalescing settings. + * cptx_vqx_done_wait_s + * Word0 + * reserved_48_63:16 [63:48] Reserved. + * time_wait:16; [47:32](R/W) Time hold-off. When CPT()_VQ()_DONE[DONE] = 0 + * or CPT()_VQ()_DONE_ACK is written a timer is cleared. When the timer + * reaches [TIME_WAIT]*1024 then interrupt coalescing ends. + * see CPT()_VQ()_DONE[DONE]. If 0x0, time coalescing is disabled. + * reserved_20_31:12 [31:20] Reserved. + * num_wait:20 [19:0](R/W) Number of messages hold-off. + * When CPT()_VQ()_DONE[DONE] >= [NUM_WAIT] then interrupt coalescing ends + * see CPT()_VQ()_DONE[DONE]. If 0x0, same behavior as 0x1. + * + */ +union otx_cptx_vqx_done_wait { + u64 u; + struct otx_cptx_vqx_done_wait_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_48_63:16; + u64 time_wait:16; + u64 reserved_20_31:12; + u64 num_wait:20; +#else /* Word 0 - Little Endian */ + u64 num_wait:20; + u64 reserved_20_31:12; + u64 time_wait:16; + u64 reserved_48_63:16; +#endif /* Word 0 - End */ + } s; +}; + +/* + * Register (NCB) otx_cpt#_vq#_done_ena_w1s + * + * CPT Queue Done Interrupt Enable Set Registers + * Write 1 to these registers will enable the DONEINT interrupt for the queue. + * cptx_vqx_done_ena_w1s_s + * Word0 + * reserved_1_63:63 [63:1] Reserved. + * done:1 [0:0](R/W1S/H) Write 1 will enable DONEINT for this queue. + * Write 0 has no effect. Read will return the enable bit. + */ +union otx_cptx_vqx_done_ena_w1s { + u64 u; + struct otx_cptx_vqx_done_ena_w1s_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_1_63:63; + u64 done:1; +#else /* Word 0 - Little Endian */ + u64 done:1; + u64 reserved_1_63:63; +#endif /* Word 0 - End */ + } s; +}; + +/* + * Register (NCB) otx_cpt#_vq#_ctl + * + * CPT VF Queue Control Registers + * This register configures queues. This register should be changed (other than + * clearing [ENA]) only when quiescent (see CPT()_VQ()_INPROG[INFLIGHT]). + * cptx_vqx_ctl_s + * Word0 + * reserved_1_63:63 [63:1] Reserved. + * ena:1 [0:0](R/W/H) Enables the logical instruction queue. + * See also CPT()_PF_Q()_CTL[CONT_ERR] and CPT()_VQ()_INPROG[INFLIGHT]. + * 1 = Queue is enabled. + * 0 = Queue is disabled. + */ +union otx_cptx_vqx_ctl { + u64 u; + struct otx_cptx_vqx_ctl_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + u64 reserved_1_63:63; + u64 ena:1; +#else /* Word 0 - Little Endian */ + u64 ena:1; + u64 reserved_1_63:63; +#endif /* Word 0 - End */ + } s; +}; + +/* + * Error Address/Error Codes + * + * In the event of a severe error, microcode writes an 8-byte Error Code + * value (ECODE) to host memory at the Rptr address specified by the host + * system (in the 64-byte request). + * + * Word0 + * [63:56](R) 8-bit completion code + * [55:48](R) Number of the core that reported the severe error + * [47:0] Lower 6 bytes of M-Inst word2. Used to assist in uniquely + * identifying which specific instruction caused the error. This assumes + * that each instruction has a unique result location (RPTR), at least + * for a given period of time. + */ +union otx_cpt_error_code { + u64 u; + struct otx_cpt_error_code_s { +#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */ + uint64_t ccode:8; + uint64_t coreid:8; + uint64_t rptr6:48; +#else /* Word 0 - Little Endian */ + uint64_t rptr6:48; + uint64_t coreid:8; + uint64_t ccode:8; +#endif /* Word 0 - End */ + } s; +}; + +#endif /*__OTX_CPT_HW_TYPES_H */ diff --git a/drivers/crypto/marvell/octeontx/otx_cptpf.h b/drivers/crypto/marvell/octeontx/otx_cptpf.h new file mode 100644 index 000000000..73cd0a9bc --- /dev/null +++ b/drivers/crypto/marvell/octeontx/otx_cptpf.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 + * Marvell OcteonTX CPT driver + * + * Copyright (C) 2019 Marvell International Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef __OTX_CPTPF_H +#define __OTX_CPTPF_H + +#include +#include +#include "otx_cptpf_ucode.h" + +/* + * OcteonTX CPT device structure + */ +struct otx_cpt_device { + void __iomem *reg_base; /* Register start address */ + struct pci_dev *pdev; /* Pci device handle */ + struct otx_cpt_eng_grps eng_grps;/* Engine groups information */ + struct list_head list; + u8 pf_type; /* PF type SE or AE */ + u8 max_vfs; /* Maximum number of VFs supported by the CPT */ + u8 vfs_enabled; /* Number of enabled VFs */ +}; + +void otx_cpt_mbox_intr_handler(struct otx_cpt_device *cpt, int mbx); +void otx_cpt_disable_all_cores(struct otx_cpt_device *cpt); + +#endif /* __OTX_CPTPF_H */ diff --git a/drivers/crypto/marvell/octeontx/otx_cptpf_main.c b/drivers/crypto/marvell/octeontx/otx_cptpf_main.c new file mode 100644 index 000000000..e4c828606 --- /dev/null +++ b/drivers/crypto/marvell/octeontx/otx_cptpf_main.c @@ -0,0 +1,301 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell OcteonTX CPT driver + * + * Copyright (C) 2019 Marvell International Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include "otx_cpt_common.h" +#include "otx_cptpf.h" + +#define DRV_NAME "octeontx-cpt" +#define DRV_VERSION "1.0" + +static void otx_cpt_disable_mbox_interrupts(struct otx_cpt_device *cpt) +{ + /* Disable mbox(0) interrupts for all VFs */ + writeq(~0ull, cpt->reg_base + OTX_CPT_PF_MBOX_ENA_W1CX(0)); +} + +static void otx_cpt_enable_mbox_interrupts(struct otx_cpt_device *cpt) +{ + /* Enable mbox(0) interrupts for all VFs */ + writeq(~0ull, cpt->reg_base + OTX_CPT_PF_MBOX_ENA_W1SX(0)); +} + +static irqreturn_t otx_cpt_mbx0_intr_handler(int __always_unused irq, + void *cpt) +{ + otx_cpt_mbox_intr_handler(cpt, 0); + + return IRQ_HANDLED; +} + +static void otx_cpt_reset(struct otx_cpt_device *cpt) +{ + writeq(1, cpt->reg_base + OTX_CPT_PF_RESET); +} + +static void otx_cpt_find_max_enabled_cores(struct otx_cpt_device *cpt) +{ + union otx_cptx_pf_constants pf_cnsts = {0}; + + pf_cnsts.u = readq(cpt->reg_base + OTX_CPT_PF_CONSTANTS); + cpt->eng_grps.avail.max_se_cnt = pf_cnsts.s.se; + cpt->eng_grps.avail.max_ae_cnt = pf_cnsts.s.ae; +} + +static u32 otx_cpt_check_bist_status(struct otx_cpt_device *cpt) +{ + union otx_cptx_pf_bist_status bist_sts = {0}; + + bist_sts.u = readq(cpt->reg_base + OTX_CPT_PF_BIST_STATUS); + return bist_sts.u; +} + +static u64 otx_cpt_check_exe_bist_status(struct otx_cpt_device *cpt) +{ + union otx_cptx_pf_exe_bist_status bist_sts = {0}; + + bist_sts.u = readq(cpt->reg_base + OTX_CPT_PF_EXE_BIST_STATUS); + return bist_sts.u; +} + +static int otx_cpt_device_init(struct otx_cpt_device *cpt) +{ + struct device *dev = &cpt->pdev->dev; + u16 sdevid; + u64 bist; + + /* Reset the PF when probed first */ + otx_cpt_reset(cpt); + mdelay(100); + + pci_read_config_word(cpt->pdev, PCI_SUBSYSTEM_ID, &sdevid); + + /* Check BIST status */ + bist = (u64)otx_cpt_check_bist_status(cpt); + if (bist) { + dev_err(dev, "RAM BIST failed with code 0x%llx\n", bist); + return -ENODEV; + } + + bist = otx_cpt_check_exe_bist_status(cpt); + if (bist) { + dev_err(dev, "Engine BIST failed with code 0x%llx\n", bist); + return -ENODEV; + } + + /* Get max enabled cores */ + otx_cpt_find_max_enabled_cores(cpt); + + if ((sdevid == OTX_CPT_PCI_PF_SUBSYS_ID) && + (cpt->eng_grps.avail.max_se_cnt == 0)) { + cpt->pf_type = OTX_CPT_AE; + } else if ((sdevid == OTX_CPT_PCI_PF_SUBSYS_ID) && + (cpt->eng_grps.avail.max_ae_cnt == 0)) { + cpt->pf_type = OTX_CPT_SE; + } + + /* Get max VQs/VFs supported by the device */ + cpt->max_vfs = pci_sriov_get_totalvfs(cpt->pdev); + + /* Disable all cores */ + otx_cpt_disable_all_cores(cpt); + + return 0; +} + +static int otx_cpt_register_interrupts(struct otx_cpt_device *cpt) +{ + struct device *dev = &cpt->pdev->dev; + u32 mbox_int_idx = OTX_CPT_PF_MBOX_INT; + u32 num_vec = OTX_CPT_PF_MSIX_VECTORS; + int ret; + + /* Enable MSI-X */ + ret = pci_alloc_irq_vectors(cpt->pdev, num_vec, num_vec, PCI_IRQ_MSIX); + if (ret < 0) { + dev_err(&cpt->pdev->dev, + "Request for #%d msix vectors failed\n", + num_vec); + return ret; + } + + /* Register mailbox interrupt handlers */ + ret = request_irq(pci_irq_vector(cpt->pdev, + OTX_CPT_PF_INT_VEC_E_MBOXX(mbox_int_idx, 0)), + otx_cpt_mbx0_intr_handler, 0, "CPT Mbox0", cpt); + if (ret) { + dev_err(dev, "Request irq failed\n"); + pci_free_irq_vectors(cpt->pdev); + return ret; + } + /* Enable mailbox interrupt */ + otx_cpt_enable_mbox_interrupts(cpt); + return 0; +} + +static void otx_cpt_unregister_interrupts(struct otx_cpt_device *cpt) +{ + u32 mbox_int_idx = OTX_CPT_PF_MBOX_INT; + + otx_cpt_disable_mbox_interrupts(cpt); + free_irq(pci_irq_vector(cpt->pdev, + OTX_CPT_PF_INT_VEC_E_MBOXX(mbox_int_idx, 0)), + cpt); + pci_free_irq_vectors(cpt->pdev); +} + + +static int otx_cpt_sriov_configure(struct pci_dev *pdev, int numvfs) +{ + struct otx_cpt_device *cpt = pci_get_drvdata(pdev); + int ret = 0; + + if (numvfs > cpt->max_vfs) + numvfs = cpt->max_vfs; + + if (numvfs > 0) { + ret = otx_cpt_try_create_default_eng_grps(cpt->pdev, + &cpt->eng_grps, + cpt->pf_type); + if (ret) + return ret; + + cpt->vfs_enabled = numvfs; + ret = pci_enable_sriov(pdev, numvfs); + if (ret) { + cpt->vfs_enabled = 0; + return ret; + } + otx_cpt_set_eng_grps_is_rdonly(&cpt->eng_grps, true); + try_module_get(THIS_MODULE); + ret = numvfs; + } else { + pci_disable_sriov(pdev); + otx_cpt_set_eng_grps_is_rdonly(&cpt->eng_grps, false); + module_put(THIS_MODULE); + cpt->vfs_enabled = 0; + } + dev_notice(&cpt->pdev->dev, "VFs enabled: %d\n", ret); + + return ret; +} + +static int otx_cpt_probe(struct pci_dev *pdev, + const struct pci_device_id __always_unused *ent) +{ + struct device *dev = &pdev->dev; + struct otx_cpt_device *cpt; + int err; + + cpt = devm_kzalloc(dev, sizeof(*cpt), GFP_KERNEL); + if (!cpt) + return -ENOMEM; + + pci_set_drvdata(pdev, cpt); + cpt->pdev = pdev; + + err = pci_enable_device(pdev); + if (err) { + dev_err(dev, "Failed to enable PCI device\n"); + goto err_clear_drvdata; + } + + err = pci_request_regions(pdev, DRV_NAME); + if (err) { + dev_err(dev, "PCI request regions failed 0x%x\n", err); + goto err_disable_device; + } + + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); + if (err) { + dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); + goto err_release_regions; + } + + /* MAP PF's configuration registers */ + cpt->reg_base = pci_iomap(pdev, OTX_CPT_PF_PCI_CFG_BAR, 0); + if (!cpt->reg_base) { + dev_err(dev, "Cannot map config register space, aborting\n"); + err = -ENOMEM; + goto err_release_regions; + } + + /* CPT device HW initialization */ + err = otx_cpt_device_init(cpt); + if (err) + goto err_unmap_region; + + /* Register interrupts */ + err = otx_cpt_register_interrupts(cpt); + if (err) + goto err_unmap_region; + + /* Initialize engine groups */ + err = otx_cpt_init_eng_grps(pdev, &cpt->eng_grps, cpt->pf_type); + if (err) + goto err_unregister_interrupts; + + return 0; + +err_unregister_interrupts: + otx_cpt_unregister_interrupts(cpt); +err_unmap_region: + pci_iounmap(pdev, cpt->reg_base); +err_release_regions: + pci_release_regions(pdev); +err_disable_device: + pci_disable_device(pdev); +err_clear_drvdata: + pci_set_drvdata(pdev, NULL); + + return err; +} + +static void otx_cpt_remove(struct pci_dev *pdev) +{ + struct otx_cpt_device *cpt = pci_get_drvdata(pdev); + + if (!cpt) + return; + + /* Disable VFs */ + pci_disable_sriov(pdev); + /* Cleanup engine groups */ + otx_cpt_cleanup_eng_grps(pdev, &cpt->eng_grps); + /* Disable CPT PF interrupts */ + otx_cpt_unregister_interrupts(cpt); + /* Disengage SE and AE cores from all groups */ + otx_cpt_disable_all_cores(cpt); + pci_iounmap(pdev, cpt->reg_base); + pci_release_regions(pdev); + pci_disable_device(pdev); + pci_set_drvdata(pdev, NULL); +} + +/* Supported devices */ +static const struct pci_device_id otx_cpt_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OTX_CPT_PCI_PF_DEVICE_ID) }, + { 0, } /* end of table */ +}; +MODULE_DEVICE_TABLE(pci, otx_cpt_id_table); + +static struct pci_driver otx_cpt_pci_driver = { + .name = DRV_NAME, + .id_table = otx_cpt_id_table, + .probe = otx_cpt_probe, + .remove = otx_cpt_remove, + .sriov_configure = otx_cpt_sriov_configure +}; + +module_pci_driver(otx_cpt_pci_driver); + +MODULE_AUTHOR("Marvell International Ltd."); +MODULE_DESCRIPTION("Marvell OcteonTX CPT Physical Function Driver"); +MODULE_LICENSE("GPL v2"); +MODULE_VERSION(DRV_VERSION); diff --git a/drivers/crypto/marvell/octeontx/otx_cptpf_mbox.c b/drivers/crypto/marvell/octeontx/otx_cptpf_mbox.c new file mode 100644 index 000000000..a9e3de658 --- /dev/null +++ b/drivers/crypto/marvell/octeontx/otx_cptpf_mbox.c @@ -0,0 +1,253 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell OcteonTX CPT driver + * + * Copyright (C) 2019 Marvell International Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include "otx_cpt_common.h" +#include "otx_cptpf.h" + +static char *get_mbox_opcode_str(int msg_opcode) +{ + char *str = "Unknown"; + + switch (msg_opcode) { + case OTX_CPT_MSG_VF_UP: + str = "UP"; + break; + + case OTX_CPT_MSG_VF_DOWN: + str = "DOWN"; + break; + + case OTX_CPT_MSG_READY: + str = "READY"; + break; + + case OTX_CPT_MSG_QLEN: + str = "QLEN"; + break; + + case OTX_CPT_MSG_QBIND_GRP: + str = "QBIND_GRP"; + break; + + case OTX_CPT_MSG_VQ_PRIORITY: + str = "VQ_PRIORITY"; + break; + + case OTX_CPT_MSG_PF_TYPE: + str = "PF_TYPE"; + break; + + case OTX_CPT_MSG_ACK: + str = "ACK"; + break; + + case OTX_CPT_MSG_NACK: + str = "NACK"; + break; + } + + return str; +} + +static void dump_mbox_msg(struct otx_cpt_mbox *mbox_msg, int vf_id) +{ + char raw_data_str[OTX_CPT_MAX_MBOX_DATA_STR_SIZE]; + + hex_dump_to_buffer(mbox_msg, sizeof(struct otx_cpt_mbox), 16, 8, + raw_data_str, OTX_CPT_MAX_MBOX_DATA_STR_SIZE, false); + if (vf_id >= 0) + pr_debug("MBOX opcode %s received from VF%d raw_data %s\n", + get_mbox_opcode_str(mbox_msg->msg), vf_id, + raw_data_str); + else + pr_debug("MBOX opcode %s received from PF raw_data %s\n", + get_mbox_opcode_str(mbox_msg->msg), raw_data_str); +} + +static void otx_cpt_send_msg_to_vf(struct otx_cpt_device *cpt, int vf, + struct otx_cpt_mbox *mbx) +{ + /* Writing mbox(0) causes interrupt */ + writeq(mbx->data, cpt->reg_base + OTX_CPT_PF_VFX_MBOXX(vf, 1)); + writeq(mbx->msg, cpt->reg_base + OTX_CPT_PF_VFX_MBOXX(vf, 0)); +} + +/* + * ACKs VF's mailbox message + * @vf: VF to which ACK to be sent + */ +static void otx_cpt_mbox_send_ack(struct otx_cpt_device *cpt, int vf, + struct otx_cpt_mbox *mbx) +{ + mbx->data = 0ull; + mbx->msg = OTX_CPT_MSG_ACK; + otx_cpt_send_msg_to_vf(cpt, vf, mbx); +} + +/* NACKs VF's mailbox message that PF is not able to complete the action */ +static void otx_cptpf_mbox_send_nack(struct otx_cpt_device *cpt, int vf, + struct otx_cpt_mbox *mbx) +{ + mbx->data = 0ull; + mbx->msg = OTX_CPT_MSG_NACK; + otx_cpt_send_msg_to_vf(cpt, vf, mbx); +} + +static void otx_cpt_clear_mbox_intr(struct otx_cpt_device *cpt, u32 vf) +{ + /* W1C for the VF */ + writeq(1ull << vf, cpt->reg_base + OTX_CPT_PF_MBOX_INTX(0)); +} + +/* + * Configure QLEN/Chunk sizes for VF + */ +static void otx_cpt_cfg_qlen_for_vf(struct otx_cpt_device *cpt, int vf, + u32 size) +{ + union otx_cptx_pf_qx_ctl pf_qx_ctl; + + pf_qx_ctl.u = readq(cpt->reg_base + OTX_CPT_PF_QX_CTL(vf)); + pf_qx_ctl.s.size = size; + pf_qx_ctl.s.cont_err = true; + writeq(pf_qx_ctl.u, cpt->reg_base + OTX_CPT_PF_QX_CTL(vf)); +} + +/* + * Configure VQ priority + */ +static void otx_cpt_cfg_vq_priority(struct otx_cpt_device *cpt, int vf, u32 pri) +{ + union otx_cptx_pf_qx_ctl pf_qx_ctl; + + pf_qx_ctl.u = readq(cpt->reg_base + OTX_CPT_PF_QX_CTL(vf)); + pf_qx_ctl.s.pri = pri; + writeq(pf_qx_ctl.u, cpt->reg_base + OTX_CPT_PF_QX_CTL(vf)); +} + +static int otx_cpt_bind_vq_to_grp(struct otx_cpt_device *cpt, u8 q, u8 grp) +{ + struct device *dev = &cpt->pdev->dev; + struct otx_cpt_eng_grp_info *eng_grp; + union otx_cptx_pf_qx_ctl pf_qx_ctl; + struct otx_cpt_ucode *ucode; + + if (q >= cpt->max_vfs) { + dev_err(dev, "Requested queue %d is > than maximum avail %d\n", + q, cpt->max_vfs); + return -EINVAL; + } + + if (grp >= OTX_CPT_MAX_ENGINE_GROUPS) { + dev_err(dev, "Requested group %d is > than maximum avail %d\n", + grp, OTX_CPT_MAX_ENGINE_GROUPS); + return -EINVAL; + } + + eng_grp = &cpt->eng_grps.grp[grp]; + if (!eng_grp->is_enabled) { + dev_err(dev, "Requested engine group %d is disabled\n", grp); + return -EINVAL; + } + + pf_qx_ctl.u = readq(cpt->reg_base + OTX_CPT_PF_QX_CTL(q)); + pf_qx_ctl.s.grp = grp; + writeq(pf_qx_ctl.u, cpt->reg_base + OTX_CPT_PF_QX_CTL(q)); + + if (eng_grp->mirror.is_ena) + ucode = &eng_grp->g->grp[eng_grp->mirror.idx].ucode[0]; + else + ucode = &eng_grp->ucode[0]; + + if (otx_cpt_uc_supports_eng_type(ucode, OTX_CPT_SE_TYPES)) + return OTX_CPT_SE_TYPES; + else if (otx_cpt_uc_supports_eng_type(ucode, OTX_CPT_AE_TYPES)) + return OTX_CPT_AE_TYPES; + else + return BAD_OTX_CPTVF_TYPE; +} + +/* Interrupt handler to handle mailbox messages from VFs */ +static void otx_cpt_handle_mbox_intr(struct otx_cpt_device *cpt, int vf) +{ + int vftype = 0; + struct otx_cpt_mbox mbx = {}; + struct device *dev = &cpt->pdev->dev; + /* + * MBOX[0] contains msg + * MBOX[1] contains data + */ + mbx.msg = readq(cpt->reg_base + OTX_CPT_PF_VFX_MBOXX(vf, 0)); + mbx.data = readq(cpt->reg_base + OTX_CPT_PF_VFX_MBOXX(vf, 1)); + + dump_mbox_msg(&mbx, vf); + + switch (mbx.msg) { + case OTX_CPT_MSG_VF_UP: + mbx.msg = OTX_CPT_MSG_VF_UP; + mbx.data = cpt->vfs_enabled; + otx_cpt_send_msg_to_vf(cpt, vf, &mbx); + break; + case OTX_CPT_MSG_READY: + mbx.msg = OTX_CPT_MSG_READY; + mbx.data = vf; + otx_cpt_send_msg_to_vf(cpt, vf, &mbx); + break; + case OTX_CPT_MSG_VF_DOWN: + /* First msg in VF teardown sequence */ + otx_cpt_mbox_send_ack(cpt, vf, &mbx); + break; + case OTX_CPT_MSG_QLEN: + otx_cpt_cfg_qlen_for_vf(cpt, vf, mbx.data); + otx_cpt_mbox_send_ack(cpt, vf, &mbx); + break; + case OTX_CPT_MSG_QBIND_GRP: + vftype = otx_cpt_bind_vq_to_grp(cpt, vf, (u8)mbx.data); + if ((vftype != OTX_CPT_AE_TYPES) && + (vftype != OTX_CPT_SE_TYPES)) { + dev_err(dev, "VF%d binding to eng group %llu failed\n", + vf, mbx.data); + otx_cptpf_mbox_send_nack(cpt, vf, &mbx); + } else { + mbx.msg = OTX_CPT_MSG_QBIND_GRP; + mbx.data = vftype; + otx_cpt_send_msg_to_vf(cpt, vf, &mbx); + } + break; + case OTX_CPT_MSG_PF_TYPE: + mbx.msg = OTX_CPT_MSG_PF_TYPE; + mbx.data = cpt->pf_type; + otx_cpt_send_msg_to_vf(cpt, vf, &mbx); + break; + case OTX_CPT_MSG_VQ_PRIORITY: + otx_cpt_cfg_vq_priority(cpt, vf, mbx.data); + otx_cpt_mbox_send_ack(cpt, vf, &mbx); + break; + default: + dev_err(&cpt->pdev->dev, "Invalid msg from VF%d, msg 0x%llx\n", + vf, mbx.msg); + break; + } +} + +void otx_cpt_mbox_intr_handler (struct otx_cpt_device *cpt, int mbx) +{ + u64 intr; + u8 vf; + + intr = readq(cpt->reg_base + OTX_CPT_PF_MBOX_INTX(0)); + pr_debug("PF interrupt mbox%d mask 0x%llx\n", mbx, intr); + for (vf = 0; vf < cpt->max_vfs; vf++) { + if (intr & (1ULL << vf)) { + otx_cpt_handle_mbox_intr(cpt, vf); + otx_cpt_clear_mbox_intr(cpt, vf); + } + } +} diff --git a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c new file mode 100644 index 000000000..c66cd3229 --- /dev/null +++ b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c @@ -0,0 +1,1675 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell OcteonTX CPT driver + * + * Copyright (C) 2019 Marvell International Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include "otx_cpt_common.h" +#include "otx_cptpf_ucode.h" +#include "otx_cptpf.h" + +#define CSR_DELAY 30 +/* Tar archive defines */ +#define TAR_MAGIC "ustar" +#define TAR_MAGIC_LEN 6 +#define TAR_BLOCK_LEN 512 +#define REGTYPE '0' +#define AREGTYPE '\0' + +/* tar header as defined in POSIX 1003.1-1990. */ +struct tar_hdr_t { + char name[100]; + char mode[8]; + char uid[8]; + char gid[8]; + char size[12]; + char mtime[12]; + char chksum[8]; + char typeflag; + char linkname[100]; + char magic[6]; + char version[2]; + char uname[32]; + char gname[32]; + char devmajor[8]; + char devminor[8]; + char prefix[155]; +}; + +struct tar_blk_t { + union { + struct tar_hdr_t hdr; + char block[TAR_BLOCK_LEN]; + }; +}; + +struct tar_arch_info_t { + struct list_head ucodes; + const struct firmware *fw; +}; + +static struct otx_cpt_bitmap get_cores_bmap(struct device *dev, + struct otx_cpt_eng_grp_info *eng_grp) +{ + struct otx_cpt_bitmap bmap = { {0} }; + bool found = false; + int i; + + if (eng_grp->g->engs_num > OTX_CPT_MAX_ENGINES) { + dev_err(dev, "unsupported number of engines %d on octeontx\n", + eng_grp->g->engs_num); + return bmap; + } + + for (i = 0; i < OTX_CPT_MAX_ETYPES_PER_GRP; i++) { + if (eng_grp->engs[i].type) { + bitmap_or(bmap.bits, bmap.bits, + eng_grp->engs[i].bmap, + eng_grp->g->engs_num); + bmap.size = eng_grp->g->engs_num; + found = true; + } + } + + if (!found) + dev_err(dev, "No engines reserved for engine group %d\n", + eng_grp->idx); + return bmap; +} + +static int is_eng_type(int val, int eng_type) +{ + return val & (1 << eng_type); +} + +static int dev_supports_eng_type(struct otx_cpt_eng_grps *eng_grps, + int eng_type) +{ + return is_eng_type(eng_grps->eng_types_supported, eng_type); +} + +static void set_ucode_filename(struct otx_cpt_ucode *ucode, + const char *filename) +{ + strscpy(ucode->filename, filename); +} + +static char *get_eng_type_str(int eng_type) +{ + switch (eng_type) { + case OTX_CPT_SE_TYPES: + return "SE"; + case OTX_CPT_AE_TYPES: + return "AE"; + default: + return "unknown"; + } +} + +static char *get_ucode_type_str(int ucode_type) +{ + switch (ucode_type) { + case (1 << OTX_CPT_SE_TYPES): + return "SE"; + case (1 << OTX_CPT_AE_TYPES): + return "AE"; + default: + return "unknown"; + } +} + +static int get_ucode_type(struct otx_cpt_ucode_hdr *ucode_hdr, int *ucode_type) +{ + char tmp_ver_str[OTX_CPT_UCODE_VER_STR_SZ]; + u32 i, val = 0; + u8 nn; + + strscpy(tmp_ver_str, ucode_hdr->ver_str); + for (i = 0; i < strlen(tmp_ver_str); i++) + tmp_ver_str[i] = tolower(tmp_ver_str[i]); + + nn = ucode_hdr->ver_num.nn; + if (strnstr(tmp_ver_str, "se-", OTX_CPT_UCODE_VER_STR_SZ) && + (nn == OTX_CPT_SE_UC_TYPE1 || nn == OTX_CPT_SE_UC_TYPE2 || + nn == OTX_CPT_SE_UC_TYPE3)) + val |= 1 << OTX_CPT_SE_TYPES; + if (strnstr(tmp_ver_str, "ae", OTX_CPT_UCODE_VER_STR_SZ) && + nn == OTX_CPT_AE_UC_TYPE) + val |= 1 << OTX_CPT_AE_TYPES; + + *ucode_type = val; + + if (!val) + return -EINVAL; + if (is_eng_type(val, OTX_CPT_AE_TYPES) && + is_eng_type(val, OTX_CPT_SE_TYPES)) + return -EINVAL; + return 0; +} + +static int is_mem_zero(const char *ptr, int size) +{ + int i; + + for (i = 0; i < size; i++) { + if (ptr[i]) + return 0; + } + return 1; +} + +static int cpt_set_ucode_base(struct otx_cpt_eng_grp_info *eng_grp, void *obj) +{ + struct otx_cpt_device *cpt = (struct otx_cpt_device *) obj; + dma_addr_t dma_addr; + struct otx_cpt_bitmap bmap; + int i; + + bmap = get_cores_bmap(&cpt->pdev->dev, eng_grp); + if (!bmap.size) + return -EINVAL; + + if (eng_grp->mirror.is_ena) + dma_addr = + eng_grp->g->grp[eng_grp->mirror.idx].ucode[0].align_dma; + else + dma_addr = eng_grp->ucode[0].align_dma; + + /* + * Set UCODE_BASE only for the cores which are not used, + * other cores should have already valid UCODE_BASE set + */ + for_each_set_bit(i, bmap.bits, bmap.size) + if (!eng_grp->g->eng_ref_cnt[i]) + writeq((u64) dma_addr, cpt->reg_base + + OTX_CPT_PF_ENGX_UCODE_BASE(i)); + return 0; +} + +static int cpt_detach_and_disable_cores(struct otx_cpt_eng_grp_info *eng_grp, + void *obj) +{ + struct otx_cpt_device *cpt = (struct otx_cpt_device *) obj; + struct otx_cpt_bitmap bmap = { {0} }; + int timeout = 10; + int i, busy; + u64 reg; + + bmap = get_cores_bmap(&cpt->pdev->dev, eng_grp); + if (!bmap.size) + return -EINVAL; + + /* Detach the cores from group */ + reg = readq(cpt->reg_base + OTX_CPT_PF_GX_EN(eng_grp->idx)); + for_each_set_bit(i, bmap.bits, bmap.size) { + if (reg & (1ull << i)) { + eng_grp->g->eng_ref_cnt[i]--; + reg &= ~(1ull << i); + } + } + writeq(reg, cpt->reg_base + OTX_CPT_PF_GX_EN(eng_grp->idx)); + + /* Wait for cores to become idle */ + do { + busy = 0; + usleep_range(10000, 20000); + if (timeout-- < 0) + return -EBUSY; + + reg = readq(cpt->reg_base + OTX_CPT_PF_EXEC_BUSY); + for_each_set_bit(i, bmap.bits, bmap.size) + if (reg & (1ull << i)) { + busy = 1; + break; + } + } while (busy); + + /* Disable the cores only if they are not used anymore */ + reg = readq(cpt->reg_base + OTX_CPT_PF_EXE_CTL); + for_each_set_bit(i, bmap.bits, bmap.size) + if (!eng_grp->g->eng_ref_cnt[i]) + reg &= ~(1ull << i); + writeq(reg, cpt->reg_base + OTX_CPT_PF_EXE_CTL); + + return 0; +} + +static int cpt_attach_and_enable_cores(struct otx_cpt_eng_grp_info *eng_grp, + void *obj) +{ + struct otx_cpt_device *cpt = (struct otx_cpt_device *) obj; + struct otx_cpt_bitmap bmap; + u64 reg; + int i; + + bmap = get_cores_bmap(&cpt->pdev->dev, eng_grp); + if (!bmap.size) + return -EINVAL; + + /* Attach the cores to the group */ + reg = readq(cpt->reg_base + OTX_CPT_PF_GX_EN(eng_grp->idx)); + for_each_set_bit(i, bmap.bits, bmap.size) { + if (!(reg & (1ull << i))) { + eng_grp->g->eng_ref_cnt[i]++; + reg |= 1ull << i; + } + } + writeq(reg, cpt->reg_base + OTX_CPT_PF_GX_EN(eng_grp->idx)); + + /* Enable the cores */ + reg = readq(cpt->reg_base + OTX_CPT_PF_EXE_CTL); + for_each_set_bit(i, bmap.bits, bmap.size) + reg |= 1ull << i; + writeq(reg, cpt->reg_base + OTX_CPT_PF_EXE_CTL); + + return 0; +} + +static int process_tar_file(struct device *dev, + struct tar_arch_info_t *tar_arch, char *filename, + const u8 *data, u32 size) +{ + struct tar_ucode_info_t *tar_info; + struct otx_cpt_ucode_hdr *ucode_hdr; + int ucode_type, ucode_size; + unsigned int code_length; + + /* + * If size is less than microcode header size then don't report + * an error because it might not be microcode file, just process + * next file from archive + */ + if (size < sizeof(struct otx_cpt_ucode_hdr)) + return 0; + + ucode_hdr = (struct otx_cpt_ucode_hdr *) data; + /* + * If microcode version can't be found don't report an error + * because it might not be microcode file, just process next file + */ + if (get_ucode_type(ucode_hdr, &ucode_type)) + return 0; + + code_length = ntohl(ucode_hdr->code_length); + if (code_length >= INT_MAX / 2) { + dev_err(dev, "Invalid code_length %u\n", code_length); + return -EINVAL; + } + + ucode_size = code_length * 2; + if (!ucode_size || (size < round_up(ucode_size, 16) + + sizeof(struct otx_cpt_ucode_hdr) + OTX_CPT_UCODE_SIGN_LEN)) { + dev_err(dev, "Ucode %s invalid size\n", filename); + return -EINVAL; + } + + tar_info = kzalloc_obj(struct tar_ucode_info_t); + if (!tar_info) + return -ENOMEM; + + tar_info->ucode_ptr = data; + set_ucode_filename(&tar_info->ucode, filename); + memcpy(tar_info->ucode.ver_str, ucode_hdr->ver_str, + OTX_CPT_UCODE_VER_STR_SZ); + tar_info->ucode.ver_num = ucode_hdr->ver_num; + tar_info->ucode.type = ucode_type; + tar_info->ucode.size = ucode_size; + list_add_tail(&tar_info->list, &tar_arch->ucodes); + + return 0; +} + +static void release_tar_archive(struct tar_arch_info_t *tar_arch) +{ + struct tar_ucode_info_t *curr, *temp; + + if (!tar_arch) + return; + + list_for_each_entry_safe(curr, temp, &tar_arch->ucodes, list) { + list_del(&curr->list); + kfree(curr); + } + + release_firmware(tar_arch->fw); + kfree(tar_arch); +} + +static struct tar_ucode_info_t *get_uc_from_tar_archive( + struct tar_arch_info_t *tar_arch, + int ucode_type) +{ + struct tar_ucode_info_t *curr, *uc_found = NULL; + + list_for_each_entry(curr, &tar_arch->ucodes, list) { + if (!is_eng_type(curr->ucode.type, ucode_type)) + continue; + + if (!uc_found) { + uc_found = curr; + continue; + } + + switch (ucode_type) { + case OTX_CPT_AE_TYPES: + break; + + case OTX_CPT_SE_TYPES: + if (uc_found->ucode.ver_num.nn == OTX_CPT_SE_UC_TYPE2 || + (uc_found->ucode.ver_num.nn == OTX_CPT_SE_UC_TYPE3 + && curr->ucode.ver_num.nn == OTX_CPT_SE_UC_TYPE1)) + uc_found = curr; + break; + } + } + + return uc_found; +} + +static void print_tar_dbg_info(struct tar_arch_info_t *tar_arch, + char *tar_filename) +{ + struct tar_ucode_info_t *curr; + + pr_debug("Tar archive filename %s\n", tar_filename); + pr_debug("Tar archive pointer %p, size %ld\n", tar_arch->fw->data, + tar_arch->fw->size); + list_for_each_entry(curr, &tar_arch->ucodes, list) { + pr_debug("Ucode filename %s\n", curr->ucode.filename); + pr_debug("Ucode version string %s\n", curr->ucode.ver_str); + pr_debug("Ucode version %d.%d.%d.%d\n", + curr->ucode.ver_num.nn, curr->ucode.ver_num.xx, + curr->ucode.ver_num.yy, curr->ucode.ver_num.zz); + pr_debug("Ucode type (%d) %s\n", curr->ucode.type, + get_ucode_type_str(curr->ucode.type)); + pr_debug("Ucode size %d\n", curr->ucode.size); + pr_debug("Ucode ptr %p\n", curr->ucode_ptr); + } +} + +static struct tar_arch_info_t *load_tar_archive(struct device *dev, + char *tar_filename) +{ + struct tar_arch_info_t *tar_arch = NULL; + struct tar_blk_t *tar_blk; + unsigned int cur_size; + size_t tar_offs = 0; + size_t tar_size; + int ret; + + tar_arch = kzalloc_obj(struct tar_arch_info_t); + if (!tar_arch) + return NULL; + + INIT_LIST_HEAD(&tar_arch->ucodes); + + /* Load tar archive */ + ret = request_firmware(&tar_arch->fw, tar_filename, dev); + if (ret) + goto release_tar_arch; + + if (tar_arch->fw->size < TAR_BLOCK_LEN) { + dev_err(dev, "Invalid tar archive %s\n", tar_filename); + goto release_tar_arch; + } + + tar_size = tar_arch->fw->size; + tar_blk = (struct tar_blk_t *) tar_arch->fw->data; + if (strncmp(tar_blk->hdr.magic, TAR_MAGIC, TAR_MAGIC_LEN - 1)) { + dev_err(dev, "Unsupported format of tar archive %s\n", + tar_filename); + goto release_tar_arch; + } + + while (1) { + /* Read current file size */ + ret = kstrtouint(tar_blk->hdr.size, 8, &cur_size); + if (ret) + goto release_tar_arch; + + if (tar_offs + cur_size > tar_size || + tar_offs + 2*TAR_BLOCK_LEN > tar_size) { + dev_err(dev, "Invalid tar archive %s\n", tar_filename); + goto release_tar_arch; + } + + tar_offs += TAR_BLOCK_LEN; + if (tar_blk->hdr.typeflag == REGTYPE || + tar_blk->hdr.typeflag == AREGTYPE) { + ret = process_tar_file(dev, tar_arch, + tar_blk->hdr.name, + &tar_arch->fw->data[tar_offs], + cur_size); + if (ret) + goto release_tar_arch; + } + + tar_offs += (cur_size/TAR_BLOCK_LEN) * TAR_BLOCK_LEN; + if (cur_size % TAR_BLOCK_LEN) + tar_offs += TAR_BLOCK_LEN; + + /* Check for the end of the archive */ + if (tar_offs + 2*TAR_BLOCK_LEN > tar_size) { + dev_err(dev, "Invalid tar archive %s\n", tar_filename); + goto release_tar_arch; + } + + if (is_mem_zero(&tar_arch->fw->data[tar_offs], + 2*TAR_BLOCK_LEN)) + break; + + /* Read next block from tar archive */ + tar_blk = (struct tar_blk_t *) &tar_arch->fw->data[tar_offs]; + } + + print_tar_dbg_info(tar_arch, tar_filename); + return tar_arch; +release_tar_arch: + release_tar_archive(tar_arch); + return NULL; +} + +static struct otx_cpt_engs_rsvd *find_engines_by_type( + struct otx_cpt_eng_grp_info *eng_grp, + int eng_type) +{ + int i; + + for (i = 0; i < OTX_CPT_MAX_ETYPES_PER_GRP; i++) { + if (!eng_grp->engs[i].type) + continue; + + if (eng_grp->engs[i].type == eng_type) + return &eng_grp->engs[i]; + } + return NULL; +} + +int otx_cpt_uc_supports_eng_type(struct otx_cpt_ucode *ucode, int eng_type) +{ + return is_eng_type(ucode->type, eng_type); +} +EXPORT_SYMBOL_GPL(otx_cpt_uc_supports_eng_type); + +static void print_ucode_info(struct otx_cpt_eng_grp_info *eng_grp, + char *buf, int size) +{ + if (eng_grp->mirror.is_ena) + scnprintf(buf, size, "%s (shared with engine_group%d)", + eng_grp->g->grp[eng_grp->mirror.idx].ucode[0].ver_str, + eng_grp->mirror.idx); + else + strscpy(buf, eng_grp->ucode[0].ver_str, size); +} + +static void print_engs_info(struct otx_cpt_eng_grp_info *eng_grp, + char *buf, int size, int idx) +{ + struct otx_cpt_engs_rsvd *mirrored_engs = NULL; + struct otx_cpt_engs_rsvd *engs; + int len, i; + + buf[0] = '\0'; + for (i = 0; i < OTX_CPT_MAX_ETYPES_PER_GRP; i++) { + engs = &eng_grp->engs[i]; + if (!engs->type) + continue; + if (idx != -1 && idx != i) + continue; + + if (eng_grp->mirror.is_ena) + mirrored_engs = find_engines_by_type( + &eng_grp->g->grp[eng_grp->mirror.idx], + engs->type); + if (i > 0 && idx == -1) { + len = strlen(buf); + scnprintf(buf+len, size-len, ", "); + } + + len = strlen(buf); + scnprintf(buf+len, size-len, "%d %s ", mirrored_engs ? + engs->count + mirrored_engs->count : engs->count, + get_eng_type_str(engs->type)); + if (mirrored_engs) { + len = strlen(buf); + scnprintf(buf+len, size-len, + "(%d shared with engine_group%d) ", + engs->count <= 0 ? engs->count + + mirrored_engs->count : mirrored_engs->count, + eng_grp->mirror.idx); + } + } +} + +static void print_ucode_dbg_info(struct otx_cpt_ucode *ucode) +{ + pr_debug("Ucode info\n"); + pr_debug("Ucode version string %s\n", ucode->ver_str); + pr_debug("Ucode version %d.%d.%d.%d\n", ucode->ver_num.nn, + ucode->ver_num.xx, ucode->ver_num.yy, ucode->ver_num.zz); + pr_debug("Ucode type %s\n", get_ucode_type_str(ucode->type)); + pr_debug("Ucode size %d\n", ucode->size); + pr_debug("Ucode virt address %16.16llx\n", (u64)ucode->align_va); + pr_debug("Ucode phys address %16.16llx\n", ucode->align_dma); +} + +static void cpt_print_engines_mask(struct otx_cpt_eng_grp_info *eng_grp, + struct device *dev, char *buf, int size) +{ + struct otx_cpt_bitmap bmap; + u32 mask[2]; + + bmap = get_cores_bmap(dev, eng_grp); + if (!bmap.size) { + scnprintf(buf, size, "unknown"); + return; + } + bitmap_to_arr32(mask, bmap.bits, bmap.size); + scnprintf(buf, size, "%8.8x %8.8x", mask[1], mask[0]); +} + + +static void print_dbg_info(struct device *dev, + struct otx_cpt_eng_grps *eng_grps) +{ + char engs_info[2*OTX_CPT_UCODE_NAME_LENGTH]; + struct otx_cpt_eng_grp_info *mirrored_grp; + char engs_mask[OTX_CPT_UCODE_NAME_LENGTH]; + struct otx_cpt_eng_grp_info *grp; + struct otx_cpt_engs_rsvd *engs; + u32 mask[4]; + int i, j; + + pr_debug("Engine groups global info\n"); + pr_debug("max SE %d, max AE %d\n", + eng_grps->avail.max_se_cnt, eng_grps->avail.max_ae_cnt); + pr_debug("free SE %d\n", eng_grps->avail.se_cnt); + pr_debug("free AE %d\n", eng_grps->avail.ae_cnt); + + for (i = 0; i < OTX_CPT_MAX_ENGINE_GROUPS; i++) { + grp = &eng_grps->grp[i]; + pr_debug("engine_group%d, state %s\n", i, + str_enabled_disabled(grp->is_enabled)); + if (grp->is_enabled) { + mirrored_grp = &eng_grps->grp[grp->mirror.idx]; + pr_debug("Ucode0 filename %s, version %s\n", + grp->mirror.is_ena ? + mirrored_grp->ucode[0].filename : + grp->ucode[0].filename, + grp->mirror.is_ena ? + mirrored_grp->ucode[0].ver_str : + grp->ucode[0].ver_str); + } + + for (j = 0; j < OTX_CPT_MAX_ETYPES_PER_GRP; j++) { + engs = &grp->engs[j]; + if (engs->type) { + print_engs_info(grp, engs_info, + 2*OTX_CPT_UCODE_NAME_LENGTH, j); + pr_debug("Slot%d: %s\n", j, engs_info); + bitmap_to_arr32(mask, engs->bmap, + eng_grps->engs_num); + pr_debug("Mask: %8.8x %8.8x %8.8x %8.8x\n", + mask[3], mask[2], mask[1], mask[0]); + } else + pr_debug("Slot%d not used\n", j); + } + if (grp->is_enabled) { + cpt_print_engines_mask(grp, dev, engs_mask, + OTX_CPT_UCODE_NAME_LENGTH); + pr_debug("Cmask: %s\n", engs_mask); + } + } +} + +static int update_engines_avail_count(struct device *dev, + struct otx_cpt_engs_available *avail, + struct otx_cpt_engs_rsvd *engs, int val) +{ + switch (engs->type) { + case OTX_CPT_SE_TYPES: + avail->se_cnt += val; + break; + + case OTX_CPT_AE_TYPES: + avail->ae_cnt += val; + break; + + default: + dev_err(dev, "Invalid engine type %d\n", engs->type); + return -EINVAL; + } + + return 0; +} + +static int update_engines_offset(struct device *dev, + struct otx_cpt_engs_available *avail, + struct otx_cpt_engs_rsvd *engs) +{ + switch (engs->type) { + case OTX_CPT_SE_TYPES: + engs->offset = 0; + break; + + case OTX_CPT_AE_TYPES: + engs->offset = avail->max_se_cnt; + break; + + default: + dev_err(dev, "Invalid engine type %d\n", engs->type); + return -EINVAL; + } + + return 0; +} + +static int release_engines(struct device *dev, struct otx_cpt_eng_grp_info *grp) +{ + int i, ret = 0; + + for (i = 0; i < OTX_CPT_MAX_ETYPES_PER_GRP; i++) { + if (!grp->engs[i].type) + continue; + + if (grp->engs[i].count > 0) { + ret = update_engines_avail_count(dev, &grp->g->avail, + &grp->engs[i], + grp->engs[i].count); + if (ret) + return ret; + } + + grp->engs[i].type = 0; + grp->engs[i].count = 0; + grp->engs[i].offset = 0; + grp->engs[i].ucode = NULL; + bitmap_zero(grp->engs[i].bmap, grp->g->engs_num); + } + + return 0; +} + +static int do_reserve_engines(struct device *dev, + struct otx_cpt_eng_grp_info *grp, + struct otx_cpt_engines *req_engs) +{ + struct otx_cpt_engs_rsvd *engs = NULL; + int i, ret; + + for (i = 0; i < OTX_CPT_MAX_ETYPES_PER_GRP; i++) { + if (!grp->engs[i].type) { + engs = &grp->engs[i]; + break; + } + } + + if (!engs) + return -ENOMEM; + + engs->type = req_engs->type; + engs->count = req_engs->count; + + ret = update_engines_offset(dev, &grp->g->avail, engs); + if (ret) + return ret; + + if (engs->count > 0) { + ret = update_engines_avail_count(dev, &grp->g->avail, engs, + -engs->count); + if (ret) + return ret; + } + + return 0; +} + +static int check_engines_availability(struct device *dev, + struct otx_cpt_eng_grp_info *grp, + struct otx_cpt_engines *req_eng) +{ + int avail_cnt = 0; + + switch (req_eng->type) { + case OTX_CPT_SE_TYPES: + avail_cnt = grp->g->avail.se_cnt; + break; + + case OTX_CPT_AE_TYPES: + avail_cnt = grp->g->avail.ae_cnt; + break; + + default: + dev_err(dev, "Invalid engine type %d\n", req_eng->type); + return -EINVAL; + } + + if (avail_cnt < req_eng->count) { + dev_err(dev, + "Error available %s engines %d < than requested %d\n", + get_eng_type_str(req_eng->type), + avail_cnt, req_eng->count); + return -EBUSY; + } + + return 0; +} + +static int reserve_engines(struct device *dev, struct otx_cpt_eng_grp_info *grp, + struct otx_cpt_engines *req_engs, int req_cnt) +{ + int i, ret; + + /* Validate if a number of requested engines is available */ + for (i = 0; i < req_cnt; i++) { + ret = check_engines_availability(dev, grp, &req_engs[i]); + if (ret) + return ret; + } + + /* Reserve requested engines for this engine group */ + for (i = 0; i < req_cnt; i++) { + ret = do_reserve_engines(dev, grp, &req_engs[i]); + if (ret) + return ret; + } + return 0; +} + +static ssize_t eng_grp_info_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + char ucode_info[2*OTX_CPT_UCODE_NAME_LENGTH]; + char engs_info[2*OTX_CPT_UCODE_NAME_LENGTH]; + char engs_mask[OTX_CPT_UCODE_NAME_LENGTH]; + struct otx_cpt_eng_grp_info *eng_grp; + int ret; + + eng_grp = container_of(attr, struct otx_cpt_eng_grp_info, info_attr); + mutex_lock(&eng_grp->g->lock); + + print_engs_info(eng_grp, engs_info, 2*OTX_CPT_UCODE_NAME_LENGTH, -1); + print_ucode_info(eng_grp, ucode_info, 2*OTX_CPT_UCODE_NAME_LENGTH); + cpt_print_engines_mask(eng_grp, dev, engs_mask, + OTX_CPT_UCODE_NAME_LENGTH); + ret = scnprintf(buf, PAGE_SIZE, + "Microcode : %s\nEngines: %s\nEngines mask: %s\n", + ucode_info, engs_info, engs_mask); + + mutex_unlock(&eng_grp->g->lock); + return ret; +} + +static int create_sysfs_eng_grps_info(struct device *dev, + struct otx_cpt_eng_grp_info *eng_grp) +{ + eng_grp->info_attr.show = eng_grp_info_show; + eng_grp->info_attr.store = NULL; + eng_grp->info_attr.attr.name = eng_grp->sysfs_info_name; + eng_grp->info_attr.attr.mode = 0440; + sysfs_attr_init(&eng_grp->info_attr.attr); + return device_create_file(dev, &eng_grp->info_attr); +} + +static void ucode_unload(struct device *dev, struct otx_cpt_ucode *ucode) +{ + if (ucode->va) { + dma_free_coherent(dev, ucode->size + OTX_CPT_UCODE_ALIGNMENT, + ucode->va, ucode->dma); + ucode->va = NULL; + ucode->align_va = NULL; + ucode->dma = 0; + ucode->align_dma = 0; + ucode->size = 0; + } + + memset(&ucode->ver_str, 0, OTX_CPT_UCODE_VER_STR_SZ); + memset(&ucode->ver_num, 0, sizeof(struct otx_cpt_ucode_ver_num)); + set_ucode_filename(ucode, ""); + ucode->type = 0; +} + +static int copy_ucode_to_dma_mem(struct device *dev, + struct otx_cpt_ucode *ucode, + const u8 *ucode_data) +{ + u32 i; + + /* Allocate DMAable space */ + ucode->va = dma_alloc_coherent(dev, ucode->size + + OTX_CPT_UCODE_ALIGNMENT, + &ucode->dma, GFP_KERNEL); + if (!ucode->va) { + dev_err(dev, "Unable to allocate space for microcode\n"); + return -ENOMEM; + } + ucode->align_va = PTR_ALIGN(ucode->va, OTX_CPT_UCODE_ALIGNMENT); + ucode->align_dma = PTR_ALIGN(ucode->dma, OTX_CPT_UCODE_ALIGNMENT); + + memcpy((void *) ucode->align_va, (void *) ucode_data + + sizeof(struct otx_cpt_ucode_hdr), ucode->size); + + /* Byte swap 64-bit */ + for (i = 0; i < (ucode->size / 8); i++) + ((__be64 *)ucode->align_va)[i] = + cpu_to_be64(((u64 *)ucode->align_va)[i]); + /* Ucode needs 16-bit swap */ + for (i = 0; i < (ucode->size / 2); i++) + ((__be16 *)ucode->align_va)[i] = + cpu_to_be16(((u16 *)ucode->align_va)[i]); + return 0; +} + +static int ucode_load(struct device *dev, struct otx_cpt_ucode *ucode, + const char *ucode_filename) +{ + struct otx_cpt_ucode_hdr *ucode_hdr; + const struct firmware *fw; + unsigned int code_length; + int ret; + + set_ucode_filename(ucode, ucode_filename); + ret = request_firmware(&fw, ucode->filename, dev); + if (ret) + return ret; + + ucode_hdr = (struct otx_cpt_ucode_hdr *) fw->data; + memcpy(ucode->ver_str, ucode_hdr->ver_str, OTX_CPT_UCODE_VER_STR_SZ); + ucode->ver_num = ucode_hdr->ver_num; + code_length = ntohl(ucode_hdr->code_length); + if (code_length >= INT_MAX / 2) { + dev_err(dev, "Ucode invalid code_length %u\n", code_length); + ret = -EINVAL; + goto release_fw; + } + ucode->size = code_length * 2; + if (!ucode->size || (fw->size < round_up(ucode->size, 16) + + sizeof(struct otx_cpt_ucode_hdr) + OTX_CPT_UCODE_SIGN_LEN)) { + dev_err(dev, "Ucode %s invalid size\n", ucode_filename); + ret = -EINVAL; + goto release_fw; + } + + ret = get_ucode_type(ucode_hdr, &ucode->type); + if (ret) { + dev_err(dev, "Microcode %s unknown type 0x%x\n", + ucode->filename, ucode->type); + goto release_fw; + } + + ret = copy_ucode_to_dma_mem(dev, ucode, fw->data); + if (ret) + goto release_fw; + + print_ucode_dbg_info(ucode); +release_fw: + release_firmware(fw); + return ret; +} + +static int enable_eng_grp(struct otx_cpt_eng_grp_info *eng_grp, + void *obj) +{ + int ret; + + ret = cpt_set_ucode_base(eng_grp, obj); + if (ret) + return ret; + + ret = cpt_attach_and_enable_cores(eng_grp, obj); + return ret; +} + +static int disable_eng_grp(struct device *dev, + struct otx_cpt_eng_grp_info *eng_grp, + void *obj) +{ + int i, ret; + + ret = cpt_detach_and_disable_cores(eng_grp, obj); + if (ret) + return ret; + + /* Unload ucode used by this engine group */ + ucode_unload(dev, &eng_grp->ucode[0]); + + for (i = 0; i < OTX_CPT_MAX_ETYPES_PER_GRP; i++) { + if (!eng_grp->engs[i].type) + continue; + + eng_grp->engs[i].ucode = &eng_grp->ucode[0]; + } + + ret = cpt_set_ucode_base(eng_grp, obj); + + return ret; +} + +static void setup_eng_grp_mirroring(struct otx_cpt_eng_grp_info *dst_grp, + struct otx_cpt_eng_grp_info *src_grp) +{ + /* Setup fields for engine group which is mirrored */ + src_grp->mirror.is_ena = false; + src_grp->mirror.idx = 0; + src_grp->mirror.ref_count++; + + /* Setup fields for mirroring engine group */ + dst_grp->mirror.is_ena = true; + dst_grp->mirror.idx = src_grp->idx; + dst_grp->mirror.ref_count = 0; +} + +static void remove_eng_grp_mirroring(struct otx_cpt_eng_grp_info *dst_grp) +{ + struct otx_cpt_eng_grp_info *src_grp; + + if (!dst_grp->mirror.is_ena) + return; + + src_grp = &dst_grp->g->grp[dst_grp->mirror.idx]; + + src_grp->mirror.ref_count--; + dst_grp->mirror.is_ena = false; + dst_grp->mirror.idx = 0; + dst_grp->mirror.ref_count = 0; +} + +static void update_requested_engs(struct otx_cpt_eng_grp_info *mirrored_eng_grp, + struct otx_cpt_engines *engs, int engs_cnt) +{ + struct otx_cpt_engs_rsvd *mirrored_engs; + int i; + + for (i = 0; i < engs_cnt; i++) { + mirrored_engs = find_engines_by_type(mirrored_eng_grp, + engs[i].type); + if (!mirrored_engs) + continue; + + /* + * If mirrored group has this type of engines attached then + * there are 3 scenarios possible: + * 1) mirrored_engs.count == engs[i].count then all engines + * from mirrored engine group will be shared with this engine + * group + * 2) mirrored_engs.count > engs[i].count then only a subset of + * engines from mirrored engine group will be shared with this + * engine group + * 3) mirrored_engs.count < engs[i].count then all engines + * from mirrored engine group will be shared with this group + * and additional engines will be reserved for exclusively use + * by this engine group + */ + engs[i].count -= mirrored_engs->count; + } +} + +static struct otx_cpt_eng_grp_info *find_mirrored_eng_grp( + struct otx_cpt_eng_grp_info *grp) +{ + struct otx_cpt_eng_grps *eng_grps = grp->g; + int i; + + for (i = 0; i < OTX_CPT_MAX_ENGINE_GROUPS; i++) { + if (!eng_grps->grp[i].is_enabled) + continue; + if (eng_grps->grp[i].ucode[0].type) + continue; + if (grp->idx == i) + continue; + if (!strncasecmp(eng_grps->grp[i].ucode[0].ver_str, + grp->ucode[0].ver_str, + OTX_CPT_UCODE_VER_STR_SZ)) + return &eng_grps->grp[i]; + } + + return NULL; +} + +static struct otx_cpt_eng_grp_info *find_unused_eng_grp( + struct otx_cpt_eng_grps *eng_grps) +{ + int i; + + for (i = 0; i < OTX_CPT_MAX_ENGINE_GROUPS; i++) { + if (!eng_grps->grp[i].is_enabled) + return &eng_grps->grp[i]; + } + return NULL; +} + +static int eng_grp_update_masks(struct device *dev, + struct otx_cpt_eng_grp_info *eng_grp) +{ + struct otx_cpt_engs_rsvd *engs, *mirrored_engs; + struct otx_cpt_bitmap tmp_bmap = { {0} }; + int i, j, cnt, max_cnt; + int bit; + + for (i = 0; i < OTX_CPT_MAX_ETYPES_PER_GRP; i++) { + engs = &eng_grp->engs[i]; + if (!engs->type) + continue; + if (engs->count <= 0) + continue; + + switch (engs->type) { + case OTX_CPT_SE_TYPES: + max_cnt = eng_grp->g->avail.max_se_cnt; + break; + + case OTX_CPT_AE_TYPES: + max_cnt = eng_grp->g->avail.max_ae_cnt; + break; + + default: + dev_err(dev, "Invalid engine type %d\n", engs->type); + return -EINVAL; + } + + cnt = engs->count; + WARN_ON(engs->offset + max_cnt > OTX_CPT_MAX_ENGINES); + bitmap_zero(tmp_bmap.bits, eng_grp->g->engs_num); + for (j = engs->offset; j < engs->offset + max_cnt; j++) { + if (!eng_grp->g->eng_ref_cnt[j]) { + bitmap_set(tmp_bmap.bits, j, 1); + cnt--; + if (!cnt) + break; + } + } + + if (cnt) + return -ENOSPC; + + bitmap_copy(engs->bmap, tmp_bmap.bits, eng_grp->g->engs_num); + } + + if (!eng_grp->mirror.is_ena) + return 0; + + for (i = 0; i < OTX_CPT_MAX_ETYPES_PER_GRP; i++) { + engs = &eng_grp->engs[i]; + if (!engs->type) + continue; + + mirrored_engs = find_engines_by_type( + &eng_grp->g->grp[eng_grp->mirror.idx], + engs->type); + WARN_ON(!mirrored_engs && engs->count <= 0); + if (!mirrored_engs) + continue; + + bitmap_copy(tmp_bmap.bits, mirrored_engs->bmap, + eng_grp->g->engs_num); + if (engs->count < 0) { + bit = find_first_bit(mirrored_engs->bmap, + eng_grp->g->engs_num); + bitmap_clear(tmp_bmap.bits, bit, -engs->count); + } + bitmap_or(engs->bmap, engs->bmap, tmp_bmap.bits, + eng_grp->g->engs_num); + } + return 0; +} + +static int delete_engine_group(struct device *dev, + struct otx_cpt_eng_grp_info *eng_grp) +{ + int i, ret; + + if (!eng_grp->is_enabled) + return -EINVAL; + + if (eng_grp->mirror.ref_count) { + dev_err(dev, "Can't delete engine_group%d as it is used by engine_group(s):", + eng_grp->idx); + for (i = 0; i < OTX_CPT_MAX_ENGINE_GROUPS; i++) { + if (eng_grp->g->grp[i].mirror.is_ena && + eng_grp->g->grp[i].mirror.idx == eng_grp->idx) + pr_cont(" %d", i); + } + pr_cont("\n"); + return -EINVAL; + } + + /* Removing engine group mirroring if enabled */ + remove_eng_grp_mirroring(eng_grp); + + /* Disable engine group */ + ret = disable_eng_grp(dev, eng_grp, eng_grp->g->obj); + if (ret) + return ret; + + /* Release all engines held by this engine group */ + ret = release_engines(dev, eng_grp); + if (ret) + return ret; + + device_remove_file(dev, &eng_grp->info_attr); + eng_grp->is_enabled = false; + + return 0; +} + +static int validate_1_ucode_scenario(struct device *dev, + struct otx_cpt_eng_grp_info *eng_grp, + struct otx_cpt_engines *engs, int engs_cnt) +{ + int i; + + /* Verify that ucode loaded supports requested engine types */ + for (i = 0; i < engs_cnt; i++) { + if (!otx_cpt_uc_supports_eng_type(&eng_grp->ucode[0], + engs[i].type)) { + dev_err(dev, + "Microcode %s does not support %s engines\n", + eng_grp->ucode[0].filename, + get_eng_type_str(engs[i].type)); + return -EINVAL; + } + } + return 0; +} + +static void update_ucode_ptrs(struct otx_cpt_eng_grp_info *eng_grp) +{ + struct otx_cpt_ucode *ucode; + + if (eng_grp->mirror.is_ena) + ucode = &eng_grp->g->grp[eng_grp->mirror.idx].ucode[0]; + else + ucode = &eng_grp->ucode[0]; + WARN_ON(!eng_grp->engs[0].type); + eng_grp->engs[0].ucode = ucode; +} + +static int create_engine_group(struct device *dev, + struct otx_cpt_eng_grps *eng_grps, + struct otx_cpt_engines *engs, int engs_cnt, + void *ucode_data[], int ucodes_cnt, + bool use_uc_from_tar_arch) +{ + struct otx_cpt_eng_grp_info *mirrored_eng_grp; + struct tar_ucode_info_t *tar_info; + struct otx_cpt_eng_grp_info *eng_grp; + int i, ret = 0; + + if (ucodes_cnt > OTX_CPT_MAX_ETYPES_PER_GRP) + return -EINVAL; + + /* Validate if requested engine types are supported by this device */ + for (i = 0; i < engs_cnt; i++) + if (!dev_supports_eng_type(eng_grps, engs[i].type)) { + dev_err(dev, "Device does not support %s engines\n", + get_eng_type_str(engs[i].type)); + return -EPERM; + } + + /* Find engine group which is not used */ + eng_grp = find_unused_eng_grp(eng_grps); + if (!eng_grp) { + dev_err(dev, "Error all engine groups are being used\n"); + return -ENOSPC; + } + + /* Load ucode */ + for (i = 0; i < ucodes_cnt; i++) { + if (use_uc_from_tar_arch) { + tar_info = (struct tar_ucode_info_t *) ucode_data[i]; + eng_grp->ucode[i] = tar_info->ucode; + ret = copy_ucode_to_dma_mem(dev, &eng_grp->ucode[i], + tar_info->ucode_ptr); + } else + ret = ucode_load(dev, &eng_grp->ucode[i], + (char *) ucode_data[i]); + if (ret) + goto err_ucode_unload; + } + + /* Validate scenario where 1 ucode is used */ + ret = validate_1_ucode_scenario(dev, eng_grp, engs, engs_cnt); + if (ret) + goto err_ucode_unload; + + /* Check if this group mirrors another existing engine group */ + mirrored_eng_grp = find_mirrored_eng_grp(eng_grp); + if (mirrored_eng_grp) { + /* Setup mirroring */ + setup_eng_grp_mirroring(eng_grp, mirrored_eng_grp); + + /* + * Update count of requested engines because some + * of them might be shared with mirrored group + */ + update_requested_engs(mirrored_eng_grp, engs, engs_cnt); + } + + /* Reserve engines */ + ret = reserve_engines(dev, eng_grp, engs, engs_cnt); + if (ret) + goto err_ucode_unload; + + /* Update ucode pointers used by engines */ + update_ucode_ptrs(eng_grp); + + /* Update engine masks used by this group */ + ret = eng_grp_update_masks(dev, eng_grp); + if (ret) + goto err_release_engs; + + /* Create sysfs entry for engine group info */ + ret = create_sysfs_eng_grps_info(dev, eng_grp); + if (ret) + goto err_release_engs; + + /* Enable engine group */ + ret = enable_eng_grp(eng_grp, eng_grps->obj); + if (ret) + goto err_release_engs; + + /* + * If this engine group mirrors another engine group + * then we need to unload ucode as we will use ucode + * from mirrored engine group + */ + if (eng_grp->mirror.is_ena) + ucode_unload(dev, &eng_grp->ucode[0]); + + eng_grp->is_enabled = true; + if (eng_grp->mirror.is_ena) + dev_info(dev, + "Engine_group%d: reuse microcode %s from group %d\n", + eng_grp->idx, mirrored_eng_grp->ucode[0].ver_str, + mirrored_eng_grp->idx); + else + dev_info(dev, "Engine_group%d: microcode loaded %s\n", + eng_grp->idx, eng_grp->ucode[0].ver_str); + + return 0; + +err_release_engs: + release_engines(dev, eng_grp); +err_ucode_unload: + ucode_unload(dev, &eng_grp->ucode[0]); + return ret; +} + +static ssize_t ucode_load_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct otx_cpt_engines engs[OTX_CPT_MAX_ETYPES_PER_GRP] = { {0} }; + char *ucode_filename[OTX_CPT_MAX_ETYPES_PER_GRP]; + char tmp_buf[OTX_CPT_UCODE_NAME_LENGTH]; + char *start, *val, *err_msg, *tmp; + struct otx_cpt_eng_grps *eng_grps; + int grp_idx = 0, ret = -EINVAL; + bool has_se, has_ie, has_ae; + int del_grp_idx = -1; + int ucode_idx = 0; + + if (strscpy_pad(tmp_buf, buf) < 0) + return -EINVAL; + + eng_grps = container_of(attr, struct otx_cpt_eng_grps, ucode_load_attr); + err_msg = "Invalid engine group format"; + start = tmp_buf; + + has_se = has_ie = has_ae = false; + + for (;;) { + val = strsep(&start, ";"); + if (!val) + break; + val = strim(val); + if (!*val) + continue; + + if (!strncasecmp(val, "engine_group", 12)) { + if (del_grp_idx != -1) + goto err_print; + tmp = strim(strsep(&val, ":")); + if (!val) + goto err_print; + if (strlen(tmp) != 13) + goto err_print; + if (kstrtoint((tmp + 12), 10, &del_grp_idx)) + goto err_print; + val = strim(val); + if (strncasecmp(val, "null", 4)) + goto err_print; + if (strlen(val) != 4) + goto err_print; + } else if (!strncasecmp(val, "se", 2) && strchr(val, ':')) { + if (has_se || ucode_idx) + goto err_print; + tmp = strim(strsep(&val, ":")); + if (!val) + goto err_print; + if (strlen(tmp) != 2) + goto err_print; + if (kstrtoint(strim(val), 10, &engs[grp_idx].count)) + goto err_print; + engs[grp_idx++].type = OTX_CPT_SE_TYPES; + has_se = true; + } else if (!strncasecmp(val, "ae", 2) && strchr(val, ':')) { + if (has_ae || ucode_idx) + goto err_print; + tmp = strim(strsep(&val, ":")); + if (!val) + goto err_print; + if (strlen(tmp) != 2) + goto err_print; + if (kstrtoint(strim(val), 10, &engs[grp_idx].count)) + goto err_print; + engs[grp_idx++].type = OTX_CPT_AE_TYPES; + has_ae = true; + } else { + if (ucode_idx > 1) + goto err_print; + if (!strlen(val)) + goto err_print; + if (strnstr(val, " ", strlen(val))) + goto err_print; + ucode_filename[ucode_idx++] = val; + } + } + + /* Validate input parameters */ + if (del_grp_idx == -1) { + if (!(grp_idx && ucode_idx)) + goto err_print; + + if (ucode_idx > 1 && grp_idx < 2) + goto err_print; + + if (grp_idx > OTX_CPT_MAX_ETYPES_PER_GRP) { + err_msg = "Error max 2 engine types can be attached"; + goto err_print; + } + + } else { + if (del_grp_idx < 0 || + del_grp_idx >= OTX_CPT_MAX_ENGINE_GROUPS) { + dev_err(dev, "Invalid engine group index %d\n", + del_grp_idx); + ret = -EINVAL; + return ret; + } + + if (!eng_grps->grp[del_grp_idx].is_enabled) { + dev_err(dev, "Error engine_group%d is not configured\n", + del_grp_idx); + ret = -EINVAL; + return ret; + } + + if (grp_idx || ucode_idx) + goto err_print; + } + + mutex_lock(&eng_grps->lock); + + if (eng_grps->is_rdonly) { + dev_err(dev, "Disable VFs before modifying engine groups\n"); + ret = -EACCES; + goto err_unlock; + } + + if (del_grp_idx == -1) + /* create engine group */ + ret = create_engine_group(dev, eng_grps, engs, grp_idx, + (void **) ucode_filename, + ucode_idx, false); + else + /* delete engine group */ + ret = delete_engine_group(dev, &eng_grps->grp[del_grp_idx]); + if (ret) + goto err_unlock; + + print_dbg_info(dev, eng_grps); +err_unlock: + mutex_unlock(&eng_grps->lock); + return ret ? ret : count; +err_print: + dev_err(dev, "%s\n", err_msg); + + return ret; +} + +int otx_cpt_try_create_default_eng_grps(struct pci_dev *pdev, + struct otx_cpt_eng_grps *eng_grps, + int pf_type) +{ + struct tar_ucode_info_t *tar_info[OTX_CPT_MAX_ETYPES_PER_GRP] = {}; + struct otx_cpt_engines engs[OTX_CPT_MAX_ETYPES_PER_GRP] = {}; + struct tar_arch_info_t *tar_arch = NULL; + char *tar_filename; + int i, ret = 0; + + mutex_lock(&eng_grps->lock); + + /* + * We don't create engine group for kernel crypto if attempt to create + * it was already made (when user enabled VFs for the first time) + */ + if (eng_grps->is_first_try) + goto unlock_mutex; + eng_grps->is_first_try = true; + + /* We create group for kcrypto only if no groups are configured */ + for (i = 0; i < OTX_CPT_MAX_ENGINE_GROUPS; i++) + if (eng_grps->grp[i].is_enabled) + goto unlock_mutex; + + switch (pf_type) { + case OTX_CPT_AE: + case OTX_CPT_SE: + tar_filename = OTX_CPT_UCODE_TAR_FILE_NAME; + break; + + default: + dev_err(&pdev->dev, "Unknown PF type %d\n", pf_type); + ret = -EINVAL; + goto unlock_mutex; + } + + tar_arch = load_tar_archive(&pdev->dev, tar_filename); + if (!tar_arch) + goto unlock_mutex; + + /* + * If device supports SE engines and there is SE microcode in tar + * archive try to create engine group with SE engines for kernel + * crypto functionality (symmetric crypto) + */ + tar_info[0] = get_uc_from_tar_archive(tar_arch, OTX_CPT_SE_TYPES); + if (tar_info[0] && + dev_supports_eng_type(eng_grps, OTX_CPT_SE_TYPES)) { + + engs[0].type = OTX_CPT_SE_TYPES; + engs[0].count = eng_grps->avail.max_se_cnt; + + ret = create_engine_group(&pdev->dev, eng_grps, engs, 1, + (void **) tar_info, 1, true); + if (ret) + goto release_tar_arch; + } + /* + * If device supports AE engines and there is AE microcode in tar + * archive try to create engine group with AE engines for asymmetric + * crypto functionality. + */ + tar_info[0] = get_uc_from_tar_archive(tar_arch, OTX_CPT_AE_TYPES); + if (tar_info[0] && + dev_supports_eng_type(eng_grps, OTX_CPT_AE_TYPES)) { + + engs[0].type = OTX_CPT_AE_TYPES; + engs[0].count = eng_grps->avail.max_ae_cnt; + + ret = create_engine_group(&pdev->dev, eng_grps, engs, 1, + (void **) tar_info, 1, true); + if (ret) + goto release_tar_arch; + } + + print_dbg_info(&pdev->dev, eng_grps); +release_tar_arch: + release_tar_archive(tar_arch); +unlock_mutex: + mutex_unlock(&eng_grps->lock); + return ret; +} + +void otx_cpt_set_eng_grps_is_rdonly(struct otx_cpt_eng_grps *eng_grps, + bool is_rdonly) +{ + mutex_lock(&eng_grps->lock); + + eng_grps->is_rdonly = is_rdonly; + + mutex_unlock(&eng_grps->lock); +} + +void otx_cpt_disable_all_cores(struct otx_cpt_device *cpt) +{ + int grp, timeout = 100; + u64 reg; + + /* Disengage the cores from groups */ + for (grp = 0; grp < OTX_CPT_MAX_ENGINE_GROUPS; grp++) { + writeq(0, cpt->reg_base + OTX_CPT_PF_GX_EN(grp)); + udelay(CSR_DELAY); + } + + reg = readq(cpt->reg_base + OTX_CPT_PF_EXEC_BUSY); + while (reg) { + udelay(CSR_DELAY); + reg = readq(cpt->reg_base + OTX_CPT_PF_EXEC_BUSY); + if (timeout--) { + dev_warn(&cpt->pdev->dev, "Cores still busy\n"); + break; + } + } + + /* Disable the cores */ + writeq(0, cpt->reg_base + OTX_CPT_PF_EXE_CTL); +} + +void otx_cpt_cleanup_eng_grps(struct pci_dev *pdev, + struct otx_cpt_eng_grps *eng_grps) +{ + struct otx_cpt_eng_grp_info *grp; + int i, j; + + mutex_lock(&eng_grps->lock); + if (eng_grps->is_ucode_load_created) { + device_remove_file(&pdev->dev, + &eng_grps->ucode_load_attr); + eng_grps->is_ucode_load_created = false; + } + + /* First delete all mirroring engine groups */ + for (i = 0; i < OTX_CPT_MAX_ENGINE_GROUPS; i++) + if (eng_grps->grp[i].mirror.is_ena) + delete_engine_group(&pdev->dev, &eng_grps->grp[i]); + + /* Delete remaining engine groups */ + for (i = 0; i < OTX_CPT_MAX_ENGINE_GROUPS; i++) + delete_engine_group(&pdev->dev, &eng_grps->grp[i]); + + /* Release memory */ + for (i = 0; i < OTX_CPT_MAX_ENGINE_GROUPS; i++) { + grp = &eng_grps->grp[i]; + for (j = 0; j < OTX_CPT_MAX_ETYPES_PER_GRP; j++) { + kfree(grp->engs[j].bmap); + grp->engs[j].bmap = NULL; + } + } + + mutex_unlock(&eng_grps->lock); +} + +int otx_cpt_init_eng_grps(struct pci_dev *pdev, + struct otx_cpt_eng_grps *eng_grps, int pf_type) +{ + struct otx_cpt_eng_grp_info *grp; + int i, j, ret = 0; + + mutex_init(&eng_grps->lock); + eng_grps->obj = pci_get_drvdata(pdev); + eng_grps->avail.se_cnt = eng_grps->avail.max_se_cnt; + eng_grps->avail.ae_cnt = eng_grps->avail.max_ae_cnt; + + eng_grps->engs_num = eng_grps->avail.max_se_cnt + + eng_grps->avail.max_ae_cnt; + if (eng_grps->engs_num > OTX_CPT_MAX_ENGINES) { + dev_err(&pdev->dev, + "Number of engines %d > than max supported %d\n", + eng_grps->engs_num, OTX_CPT_MAX_ENGINES); + ret = -EINVAL; + goto err; + } + + for (i = 0; i < OTX_CPT_MAX_ENGINE_GROUPS; i++) { + grp = &eng_grps->grp[i]; + grp->g = eng_grps; + grp->idx = i; + + snprintf(grp->sysfs_info_name, OTX_CPT_UCODE_NAME_LENGTH, + "engine_group%d", i); + for (j = 0; j < OTX_CPT_MAX_ETYPES_PER_GRP; j++) { + grp->engs[j].bmap = + kcalloc(BITS_TO_LONGS(eng_grps->engs_num), + sizeof(long), GFP_KERNEL); + if (!grp->engs[j].bmap) { + ret = -ENOMEM; + goto err; + } + } + } + + switch (pf_type) { + case OTX_CPT_SE: + /* OcteonTX 83XX SE CPT PF has only SE engines attached */ + eng_grps->eng_types_supported = 1 << OTX_CPT_SE_TYPES; + break; + + case OTX_CPT_AE: + /* OcteonTX 83XX AE CPT PF has only AE engines attached */ + eng_grps->eng_types_supported = 1 << OTX_CPT_AE_TYPES; + break; + + default: + dev_err(&pdev->dev, "Unknown PF type %d\n", pf_type); + ret = -EINVAL; + goto err; + } + + eng_grps->ucode_load_attr.show = NULL; + eng_grps->ucode_load_attr.store = ucode_load_store; + eng_grps->ucode_load_attr.attr.name = "ucode_load"; + eng_grps->ucode_load_attr.attr.mode = 0220; + sysfs_attr_init(&eng_grps->ucode_load_attr.attr); + ret = device_create_file(&pdev->dev, + &eng_grps->ucode_load_attr); + if (ret) + goto err; + eng_grps->is_ucode_load_created = true; + + print_dbg_info(&pdev->dev, eng_grps); + return ret; +err: + otx_cpt_cleanup_eng_grps(pdev, eng_grps); + return ret; +} diff --git a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.h b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.h new file mode 100644 index 000000000..df79ee416 --- /dev/null +++ b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.h @@ -0,0 +1,178 @@ +/* SPDX-License-Identifier: GPL-2.0 + * Marvell OcteonTX CPT driver + * + * Copyright (C) 2019 Marvell International Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef __OTX_CPTPF_UCODE_H +#define __OTX_CPTPF_UCODE_H + +#include +#include +#include +#include "otx_cpt_hw_types.h" + +/* CPT ucode name maximum length */ +#define OTX_CPT_UCODE_NAME_LENGTH 64 +/* + * On OcteonTX 83xx platform, only one type of engines is allowed to be + * attached to an engine group. + */ +#define OTX_CPT_MAX_ETYPES_PER_GRP 1 + +/* Default tar archive file names */ +#define OTX_CPT_UCODE_TAR_FILE_NAME "cpt8x-mc.tar" + +/* CPT ucode alignment */ +#define OTX_CPT_UCODE_ALIGNMENT 128 + +/* CPT ucode signature size */ +#define OTX_CPT_UCODE_SIGN_LEN 256 + +/* Microcode version string length */ +#define OTX_CPT_UCODE_VER_STR_SZ 44 + +/* Maximum number of supported engines/cores on OcteonTX 83XX platform */ +#define OTX_CPT_MAX_ENGINES 64 + +#define OTX_CPT_ENGS_BITMASK_LEN (OTX_CPT_MAX_ENGINES/(BITS_PER_BYTE * \ + sizeof(unsigned long))) + +/* Microcode types */ +enum otx_cpt_ucode_type { + OTX_CPT_AE_UC_TYPE = 1, /* AE-MAIN */ + OTX_CPT_SE_UC_TYPE1 = 20, /* SE-MAIN - combination of 21 and 22 */ + OTX_CPT_SE_UC_TYPE2 = 21, /* Fast Path IPSec + AirCrypto */ + OTX_CPT_SE_UC_TYPE3 = 22, /* + * Hash + HMAC + FlexiCrypto + RNG + Full + * Feature IPSec + AirCrypto + Kasumi + */ +}; + +struct otx_cpt_bitmap { + unsigned long bits[OTX_CPT_ENGS_BITMASK_LEN]; + int size; +}; + +struct otx_cpt_engines { + int type; + int count; +}; + +/* Microcode version number */ +struct otx_cpt_ucode_ver_num { + u8 nn; + u8 xx; + u8 yy; + u8 zz; +}; + +struct otx_cpt_ucode_hdr { + struct otx_cpt_ucode_ver_num ver_num; + u8 ver_str[OTX_CPT_UCODE_VER_STR_SZ]; + __be32 code_length; + u32 padding[3]; +}; + +struct otx_cpt_ucode { + u8 ver_str[OTX_CPT_UCODE_VER_STR_SZ];/* + * ucode version in readable format + */ + struct otx_cpt_ucode_ver_num ver_num;/* ucode version number */ + char filename[OTX_CPT_UCODE_NAME_LENGTH]; /* ucode filename */ + dma_addr_t dma; /* phys address of ucode image */ + dma_addr_t align_dma; /* aligned phys address of ucode image */ + void *va; /* virt address of ucode image */ + void *align_va; /* aligned virt address of ucode image */ + u32 size; /* ucode image size */ + int type; /* ucode image type SE or AE */ +}; + +struct tar_ucode_info_t { + struct list_head list; + struct otx_cpt_ucode ucode;/* microcode information */ + const u8 *ucode_ptr; /* pointer to microcode in tar archive */ +}; + +/* Maximum and current number of engines available for all engine groups */ +struct otx_cpt_engs_available { + int max_se_cnt; + int max_ae_cnt; + int se_cnt; + int ae_cnt; +}; + +/* Engines reserved to an engine group */ +struct otx_cpt_engs_rsvd { + int type; /* engine type */ + int count; /* number of engines attached */ + int offset; /* constant offset of engine type in the bitmap */ + unsigned long *bmap; /* attached engines bitmap */ + struct otx_cpt_ucode *ucode; /* ucode used by these engines */ +}; + +struct otx_cpt_mirror_info { + int is_ena; /* + * is mirroring enabled, it is set only for engine + * group which mirrors another engine group + */ + int idx; /* + * index of engine group which is mirrored by this + * group, set only for engine group which mirrors + * another group + */ + int ref_count; /* + * number of times this engine group is mirrored by + * other groups, this is set only for engine group + * which is mirrored by other group(s) + */ +}; + +struct otx_cpt_eng_grp_info { + struct otx_cpt_eng_grps *g; /* pointer to engine_groups structure */ + struct device_attribute info_attr; /* group info entry attr */ + /* engines attached */ + struct otx_cpt_engs_rsvd engs[OTX_CPT_MAX_ETYPES_PER_GRP]; + /* Microcode information */ + struct otx_cpt_ucode ucode[OTX_CPT_MAX_ETYPES_PER_GRP]; + /* sysfs info entry name */ + char sysfs_info_name[OTX_CPT_UCODE_NAME_LENGTH]; + /* engine group mirroring information */ + struct otx_cpt_mirror_info mirror; + int idx; /* engine group index */ + bool is_enabled; /* + * is engine group enabled, engine group is enabled + * when it has engines attached and ucode loaded + */ +}; + +struct otx_cpt_eng_grps { + struct otx_cpt_eng_grp_info grp[OTX_CPT_MAX_ENGINE_GROUPS]; + struct device_attribute ucode_load_attr;/* ucode load attr */ + struct otx_cpt_engs_available avail; + struct mutex lock; + void *obj; + int engs_num; /* total number of engines supported */ + int eng_types_supported; /* engine types supported SE, AE */ + u8 eng_ref_cnt[OTX_CPT_MAX_ENGINES];/* engines reference count */ + bool is_ucode_load_created; /* is ucode_load sysfs entry created */ + bool is_first_try; /* is this first try to create kcrypto engine grp */ + bool is_rdonly; /* do engine groups configuration can be modified */ +}; + +int otx_cpt_init_eng_grps(struct pci_dev *pdev, + struct otx_cpt_eng_grps *eng_grps, int pf_type); +void otx_cpt_cleanup_eng_grps(struct pci_dev *pdev, + struct otx_cpt_eng_grps *eng_grps); +int otx_cpt_try_create_default_eng_grps(struct pci_dev *pdev, + struct otx_cpt_eng_grps *eng_grps, + int pf_type); +void otx_cpt_set_eng_grps_is_rdonly(struct otx_cpt_eng_grps *eng_grps, + bool is_rdonly); +int otx_cpt_uc_supports_eng_type(struct otx_cpt_ucode *ucode, int eng_type); + +#endif /* __OTX_CPTPF_UCODE_H */ diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf.h b/drivers/crypto/marvell/octeontx/otx_cptvf.h new file mode 100644 index 000000000..dd02f2165 --- /dev/null +++ b/drivers/crypto/marvell/octeontx/otx_cptvf.h @@ -0,0 +1,104 @@ +/* SPDX-License-Identifier: GPL-2.0 + * Marvell OcteonTX CPT driver + * + * Copyright (C) 2019 Marvell International Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef __OTX_CPTVF_H +#define __OTX_CPTVF_H + +#include +#include +#include +#include "otx_cpt_common.h" +#include "otx_cptvf_reqmgr.h" + +/* Flags to indicate the features supported */ +#define OTX_CPT_FLAG_DEVICE_READY BIT(1) +#define otx_cpt_device_ready(cpt) ((cpt)->flags & OTX_CPT_FLAG_DEVICE_READY) +/* Default command queue length */ +#define OTX_CPT_CMD_QLEN (4*2046) +#define OTX_CPT_CMD_QCHUNK_SIZE 1023 +#define OTX_CPT_NUM_QS_PER_VF 1 + +struct otx_cpt_cmd_chunk { + u8 *head; + dma_addr_t dma_addr; + u32 size; /* Chunk size, max OTX_CPT_INST_CHUNK_MAX_SIZE */ + struct list_head nextchunk; +}; + +struct otx_cpt_cmd_queue { + u32 idx; /* Command queue host write idx */ + u32 num_chunks; /* Number of command chunks */ + struct otx_cpt_cmd_chunk *qhead;/* + * Command queue head, instructions + * are inserted here + */ + struct otx_cpt_cmd_chunk *base; + struct list_head chead; +}; + +struct otx_cpt_cmd_qinfo { + u32 qchunksize; /* Command queue chunk size */ + struct otx_cpt_cmd_queue queue[OTX_CPT_NUM_QS_PER_VF]; +}; + +struct otx_cpt_pending_qinfo { + u32 num_queues; /* Number of queues supported */ + struct otx_cpt_pending_queue queue[OTX_CPT_NUM_QS_PER_VF]; +}; + +#define for_each_pending_queue(qinfo, q, i) \ + for (i = 0, q = &qinfo->queue[i]; i < qinfo->num_queues; i++, \ + q = &qinfo->queue[i]) + +struct otx_cptvf_wqe { + struct tasklet_struct twork; + struct otx_cptvf *cptvf; +}; + +struct otx_cptvf_wqe_info { + struct otx_cptvf_wqe vq_wqe[OTX_CPT_NUM_QS_PER_VF]; +}; + +struct otx_cptvf { + u16 flags; /* Flags to hold device status bits */ + u8 vfid; /* Device Index 0...OTX_CPT_MAX_VF_NUM */ + u8 num_vfs; /* Number of enabled VFs */ + u8 vftype; /* VF type of SE_TYPE(2) or AE_TYPE(1) */ + u8 vfgrp; /* VF group (0 - 8) */ + u8 node; /* Operating node: Bits (46:44) in BAR0 address */ + u8 priority; /* + * VF priority ring: 1-High proirity round + * robin ring;0-Low priority round robin ring; + */ + struct pci_dev *pdev; /* Pci device handle */ + void __iomem *reg_base; /* Register start address */ + void *wqe_info; /* BH worker info */ + /* MSI-X */ + cpumask_var_t affinity_mask[OTX_CPT_VF_MSIX_VECTORS]; + /* Command and Pending queues */ + u32 qsize; + u32 num_queues; + struct otx_cpt_cmd_qinfo cqinfo; /* Command queue information */ + struct otx_cpt_pending_qinfo pqinfo; /* Pending queue information */ + /* VF-PF mailbox communication */ + bool pf_acked; + bool pf_nacked; +}; + +int otx_cptvf_send_vf_up(struct otx_cptvf *cptvf); +int otx_cptvf_send_vf_down(struct otx_cptvf *cptvf); +int otx_cptvf_send_vf_to_grp_msg(struct otx_cptvf *cptvf, int group); +int otx_cptvf_send_vf_priority_msg(struct otx_cptvf *cptvf); +int otx_cptvf_send_vq_size_msg(struct otx_cptvf *cptvf); +int otx_cptvf_check_pf_ready(struct otx_cptvf *cptvf); +void otx_cptvf_handle_mbox_intr(struct otx_cptvf *cptvf); +void otx_cptvf_write_vq_doorbell(struct otx_cptvf *cptvf, u32 val); + +#endif /* __OTX_CPTVF_H */ diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c b/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c new file mode 100644 index 000000000..096be42e9 --- /dev/null +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c @@ -0,0 +1,1637 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell OcteonTX CPT driver + * + * Copyright (C) 2019 Marvell International Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "otx_cptvf.h" +#include "otx_cptvf_algs.h" +#include "otx_cptvf_reqmgr.h" + +#define CPT_MAX_VF_NUM 64 +/* Size of salt in AES GCM mode */ +#define AES_GCM_SALT_SIZE 4 +/* Size of IV in AES GCM mode */ +#define AES_GCM_IV_SIZE 8 +/* Size of ICV (Integrity Check Value) in AES GCM mode */ +#define AES_GCM_ICV_SIZE 16 +/* Offset of IV in AES GCM mode */ +#define AES_GCM_IV_OFFSET 8 +#define CONTROL_WORD_LEN 8 +#define KEY2_OFFSET 48 +#define DMA_MODE_FLAG(dma_mode) \ + (((dma_mode) == OTX_CPT_DMA_GATHER_SCATTER) ? (1 << 7) : 0) + +/* Truncated SHA digest size */ +#define SHA1_TRUNC_DIGEST_SIZE 12 +#define SHA256_TRUNC_DIGEST_SIZE 16 +#define SHA384_TRUNC_DIGEST_SIZE 24 +#define SHA512_TRUNC_DIGEST_SIZE 32 + +static DEFINE_MUTEX(mutex); +static int is_crypto_registered; + +struct cpt_device_desc { + enum otx_cptpf_type pf_type; + struct pci_dev *dev; + int num_queues; +}; + +struct cpt_device_table { + atomic_t count; + struct cpt_device_desc desc[CPT_MAX_VF_NUM]; +}; + +static struct cpt_device_table se_devices = { + .count = ATOMIC_INIT(0) +}; + +static struct cpt_device_table ae_devices = { + .count = ATOMIC_INIT(0) +}; + +static struct otx_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg); + +static inline int get_se_device(struct pci_dev **pdev, int *cpu_num) +{ + int count, ret = 0; + + count = atomic_read(&se_devices.count); + if (count < 1) + return -ENODEV; + + *cpu_num = get_cpu(); + + if (se_devices.desc[0].pf_type == OTX_CPT_SE) { + /* + * On OcteonTX platform there is one CPT instruction queue bound + * to each VF. We get maximum performance if one CPT queue + * is available for each cpu otherwise CPT queues need to be + * shared between cpus. + */ + if (*cpu_num >= count) + *cpu_num %= count; + *pdev = se_devices.desc[*cpu_num].dev; + } else { + pr_err("Unknown PF type %d\n", se_devices.desc[0].pf_type); + ret = -EINVAL; + } + put_cpu(); + + return ret; +} + +static inline int validate_hmac_cipher_null(struct otx_cpt_req_info *cpt_req) +{ + struct otx_cpt_req_ctx *rctx; + struct aead_request *req; + struct crypto_aead *tfm; + + req = container_of(cpt_req->areq, struct aead_request, base); + tfm = crypto_aead_reqtfm(req); + rctx = aead_request_ctx_dma(req); + if (memcmp(rctx->fctx.hmac.s.hmac_calc, + rctx->fctx.hmac.s.hmac_recv, + crypto_aead_authsize(tfm)) != 0) + return -EBADMSG; + + return 0; +} + +static void otx_cpt_aead_callback(int status, void *arg1, void *arg2) +{ + struct otx_cpt_info_buffer *cpt_info = arg2; + struct crypto_async_request *areq = arg1; + struct otx_cpt_req_info *cpt_req; + struct pci_dev *pdev; + + if (!cpt_info) + goto complete; + + cpt_req = cpt_info->req; + if (!status) { + /* + * When selected cipher is NULL we need to manually + * verify whether calculated hmac value matches + * received hmac value + */ + if (cpt_req->req_type == OTX_CPT_AEAD_ENC_DEC_NULL_REQ && + !cpt_req->is_enc) + status = validate_hmac_cipher_null(cpt_req); + } + pdev = cpt_info->pdev; + do_request_cleanup(pdev, cpt_info); + +complete: + if (areq) + crypto_request_complete(areq, status); +} + +static void output_iv_copyback(struct crypto_async_request *areq) +{ + struct otx_cpt_req_info *req_info; + struct skcipher_request *sreq; + struct crypto_skcipher *stfm; + struct otx_cpt_req_ctx *rctx; + struct otx_cpt_enc_ctx *ctx; + u32 start, ivsize; + + sreq = container_of(areq, struct skcipher_request, base); + stfm = crypto_skcipher_reqtfm(sreq); + ctx = crypto_skcipher_ctx(stfm); + if (ctx->cipher_type == OTX_CPT_AES_CBC || + ctx->cipher_type == OTX_CPT_DES3_CBC) { + rctx = skcipher_request_ctx_dma(sreq); + req_info = &rctx->cpt_req; + ivsize = crypto_skcipher_ivsize(stfm); + start = sreq->cryptlen - ivsize; + + if (req_info->is_enc) { + scatterwalk_map_and_copy(sreq->iv, sreq->dst, start, + ivsize, 0); + } else { + if (sreq->src != sreq->dst) { + scatterwalk_map_and_copy(sreq->iv, sreq->src, + start, ivsize, 0); + } else { + memcpy(sreq->iv, req_info->iv_out, ivsize); + kfree(req_info->iv_out); + } + } + } +} + +static void otx_cpt_skcipher_callback(int status, void *arg1, void *arg2) +{ + struct otx_cpt_info_buffer *cpt_info = arg2; + struct crypto_async_request *areq = arg1; + struct pci_dev *pdev; + + if (areq) { + if (!status) + output_iv_copyback(areq); + if (cpt_info) { + pdev = cpt_info->pdev; + do_request_cleanup(pdev, cpt_info); + } + crypto_request_complete(areq, status); + } +} + +static inline void update_input_data(struct otx_cpt_req_info *req_info, + struct scatterlist *inp_sg, + u32 nbytes, u32 *argcnt) +{ + req_info->req.dlen += nbytes; + + while (nbytes) { + u32 len = min(nbytes, inp_sg->length); + u8 *ptr = sg_virt(inp_sg); + + req_info->in[*argcnt].vptr = (void *)ptr; + req_info->in[*argcnt].size = len; + nbytes -= len; + ++(*argcnt); + inp_sg = sg_next(inp_sg); + } +} + +static inline void update_output_data(struct otx_cpt_req_info *req_info, + struct scatterlist *outp_sg, + u32 offset, u32 nbytes, u32 *argcnt) +{ + req_info->rlen += nbytes; + + while (nbytes) { + u32 len = min(nbytes, outp_sg->length - offset); + u8 *ptr = sg_virt(outp_sg); + + req_info->out[*argcnt].vptr = (void *) (ptr + offset); + req_info->out[*argcnt].size = len; + nbytes -= len; + ++(*argcnt); + offset = 0; + outp_sg = sg_next(outp_sg); + } +} + +static inline u32 create_ctx_hdr(struct skcipher_request *req, u32 enc, + u32 *argcnt) +{ + struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); + struct otx_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req); + struct otx_cpt_req_info *req_info = &rctx->cpt_req; + struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm); + struct otx_cpt_enc_ctx *ctx = crypto_tfm_ctx(tfm); + struct otx_cpt_fc_ctx *fctx = &rctx->fctx; + int ivsize = crypto_skcipher_ivsize(stfm); + u32 start = req->cryptlen - ivsize; + gfp_t flags; + + flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + req_info->ctrl.s.dma_mode = OTX_CPT_DMA_GATHER_SCATTER; + req_info->ctrl.s.se_req = OTX_CPT_SE_CORE_REQ; + + req_info->req.opcode.s.major = OTX_CPT_MAJOR_OP_FC | + DMA_MODE_FLAG(OTX_CPT_DMA_GATHER_SCATTER); + if (enc) { + req_info->req.opcode.s.minor = 2; + } else { + req_info->req.opcode.s.minor = 3; + if ((ctx->cipher_type == OTX_CPT_AES_CBC || + ctx->cipher_type == OTX_CPT_DES3_CBC) && + req->src == req->dst) { + req_info->iv_out = kmalloc(ivsize, flags); + if (!req_info->iv_out) + return -ENOMEM; + + scatterwalk_map_and_copy(req_info->iv_out, req->src, + start, ivsize, 0); + } + } + /* Encryption data length */ + req_info->req.param1 = req->cryptlen; + /* Authentication data length */ + req_info->req.param2 = 0; + + fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type; + fctx->enc.enc_ctrl.e.aes_key = ctx->key_type; + fctx->enc.enc_ctrl.e.iv_source = OTX_CPT_FROM_CPTR; + + if (ctx->cipher_type == OTX_CPT_AES_XTS) + memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2); + else + memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len); + + memcpy(fctx->enc.encr_iv, req->iv, crypto_skcipher_ivsize(stfm)); + + fctx->enc.enc_ctrl.flags = cpu_to_be64(fctx->enc.enc_ctrl.cflags); + + /* + * Storing Packet Data Information in offset + * Control Word First 8 bytes + */ + req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word; + req_info->in[*argcnt].size = CONTROL_WORD_LEN; + req_info->req.dlen += CONTROL_WORD_LEN; + ++(*argcnt); + + req_info->in[*argcnt].vptr = (u8 *)fctx; + req_info->in[*argcnt].size = sizeof(struct otx_cpt_fc_ctx); + req_info->req.dlen += sizeof(struct otx_cpt_fc_ctx); + + ++(*argcnt); + + return 0; +} + +static inline u32 create_input_list(struct skcipher_request *req, u32 enc, + u32 enc_iv_len) +{ + struct otx_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req); + struct otx_cpt_req_info *req_info = &rctx->cpt_req; + u32 argcnt = 0; + int ret; + + ret = create_ctx_hdr(req, enc, &argcnt); + if (ret) + return ret; + + update_input_data(req_info, req->src, req->cryptlen, &argcnt); + req_info->incnt = argcnt; + + return 0; +} + +static inline void create_output_list(struct skcipher_request *req, + u32 enc_iv_len) +{ + struct otx_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req); + struct otx_cpt_req_info *req_info = &rctx->cpt_req; + u32 argcnt = 0; + + /* + * OUTPUT Buffer Processing + * AES encryption/decryption output would be + * received in the following format + * + * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----| + * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ] + */ + update_output_data(req_info, req->dst, 0, req->cryptlen, &argcnt); + req_info->outcnt = argcnt; +} + +static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc) +{ + struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); + struct otx_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req); + struct otx_cpt_req_info *req_info = &rctx->cpt_req; + u32 enc_iv_len = crypto_skcipher_ivsize(stfm); + struct pci_dev *pdev; + int status, cpu_num; + + /* Validate that request doesn't exceed maximum CPT supported size */ + if (req->cryptlen > OTX_CPT_MAX_REQ_SIZE) + return -E2BIG; + + /* Clear control words */ + rctx->ctrl_word.flags = 0; + rctx->fctx.enc.enc_ctrl.flags = 0; + + status = create_input_list(req, enc, enc_iv_len); + if (status) + return status; + create_output_list(req, enc_iv_len); + + status = get_se_device(&pdev, &cpu_num); + if (status) + return status; + + req_info->callback = (void *)otx_cpt_skcipher_callback; + req_info->areq = &req->base; + req_info->req_type = OTX_CPT_ENC_DEC_REQ; + req_info->is_enc = enc; + req_info->is_trunc_hmac = false; + req_info->ctrl.s.grp = 0; + + /* + * We perform an asynchronous send and once + * the request is completed the driver would + * intimate through registered call back functions + */ + status = otx_cpt_do_request(pdev, req_info, cpu_num); + + return status; +} + +static int otx_cpt_skcipher_encrypt(struct skcipher_request *req) +{ + return cpt_enc_dec(req, true); +} + +static int otx_cpt_skcipher_decrypt(struct skcipher_request *req) +{ + return cpt_enc_dec(req, false); +} + +static int otx_cpt_skcipher_xts_setkey(struct crypto_skcipher *tfm, + const u8 *key, u32 keylen) +{ + struct otx_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); + const u8 *key2 = key + (keylen / 2); + const u8 *key1 = key; + int ret; + + ret = xts_verify_key(tfm, key, keylen); + if (ret) + return ret; + ctx->key_len = keylen; + memcpy(ctx->enc_key, key1, keylen / 2); + memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2); + ctx->cipher_type = OTX_CPT_AES_XTS; + switch (ctx->key_len) { + case 2 * AES_KEYSIZE_128: + ctx->key_type = OTX_CPT_AES_128_BIT; + break; + case 2 * AES_KEYSIZE_256: + ctx->key_type = OTX_CPT_AES_256_BIT; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int cpt_des_setkey(struct crypto_skcipher *tfm, const u8 *key, + u32 keylen, u8 cipher_type) +{ + struct otx_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (keylen != DES3_EDE_KEY_SIZE) + return -EINVAL; + + ctx->key_len = keylen; + ctx->cipher_type = cipher_type; + + memcpy(ctx->enc_key, key, keylen); + + return 0; +} + +static int cpt_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + u32 keylen, u8 cipher_type) +{ + struct otx_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); + + switch (keylen) { + case AES_KEYSIZE_128: + ctx->key_type = OTX_CPT_AES_128_BIT; + break; + case AES_KEYSIZE_192: + ctx->key_type = OTX_CPT_AES_192_BIT; + break; + case AES_KEYSIZE_256: + ctx->key_type = OTX_CPT_AES_256_BIT; + break; + default: + return -EINVAL; + } + ctx->key_len = keylen; + ctx->cipher_type = cipher_type; + + memcpy(ctx->enc_key, key, keylen); + + return 0; +} + +static int otx_cpt_skcipher_cbc_aes_setkey(struct crypto_skcipher *tfm, + const u8 *key, u32 keylen) +{ + return cpt_aes_setkey(tfm, key, keylen, OTX_CPT_AES_CBC); +} + +static int otx_cpt_skcipher_ecb_aes_setkey(struct crypto_skcipher *tfm, + const u8 *key, u32 keylen) +{ + return cpt_aes_setkey(tfm, key, keylen, OTX_CPT_AES_ECB); +} + +static int otx_cpt_skcipher_cbc_des3_setkey(struct crypto_skcipher *tfm, + const u8 *key, u32 keylen) +{ + return cpt_des_setkey(tfm, key, keylen, OTX_CPT_DES3_CBC); +} + +static int otx_cpt_skcipher_ecb_des3_setkey(struct crypto_skcipher *tfm, + const u8 *key, u32 keylen) +{ + return cpt_des_setkey(tfm, key, keylen, OTX_CPT_DES3_ECB); +} + +static int otx_cpt_enc_dec_init(struct crypto_skcipher *tfm) +{ + struct otx_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); + + memset(ctx, 0, sizeof(*ctx)); + /* + * Additional memory for skcipher_request is + * allocated since the cryptd daemon uses + * this memory for request_ctx information + */ + crypto_skcipher_set_reqsize_dma( + tfm, sizeof(struct otx_cpt_req_ctx) + + sizeof(struct skcipher_request)); + + return 0; +} + +static int cpt_aead_init(struct crypto_aead *tfm, u8 cipher_type, u8 mac_type) +{ + struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); + + ctx->cipher_type = cipher_type; + ctx->mac_type = mac_type; + + switch (ctx->mac_type) { + case OTX_CPT_SHA1: + ctx->hashalg = crypto_alloc_shash("sha1", 0, 0); + break; + + case OTX_CPT_SHA256: + ctx->hashalg = crypto_alloc_shash("sha256", 0, 0); + break; + + case OTX_CPT_SHA384: + ctx->hashalg = crypto_alloc_shash("sha384", 0, 0); + break; + + case OTX_CPT_SHA512: + ctx->hashalg = crypto_alloc_shash("sha512", 0, 0); + break; + } + + if (IS_ERR(ctx->hashalg)) + return PTR_ERR(ctx->hashalg); + + crypto_aead_set_reqsize_dma(tfm, sizeof(struct otx_cpt_req_ctx)); + + if (!ctx->hashalg) + return 0; + + /* + * When selected cipher is NULL we use HMAC opcode instead of + * FLEXICRYPTO opcode therefore we don't need to use HASH algorithms + * for calculating ipad and opad + */ + if (ctx->cipher_type != OTX_CPT_CIPHER_NULL) { + int ss = crypto_shash_statesize(ctx->hashalg); + + ctx->ipad = kzalloc(ss, GFP_KERNEL); + if (!ctx->ipad) { + crypto_free_shash(ctx->hashalg); + return -ENOMEM; + } + + ctx->opad = kzalloc(ss, GFP_KERNEL); + if (!ctx->opad) { + kfree(ctx->ipad); + crypto_free_shash(ctx->hashalg); + return -ENOMEM; + } + } + + ctx->sdesc = alloc_sdesc(ctx->hashalg); + if (!ctx->sdesc) { + kfree(ctx->opad); + kfree(ctx->ipad); + crypto_free_shash(ctx->hashalg); + return -ENOMEM; + } + + return 0; +} + +static int otx_cpt_aead_cbc_aes_sha1_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX_CPT_AES_CBC, OTX_CPT_SHA1); +} + +static int otx_cpt_aead_cbc_aes_sha256_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX_CPT_AES_CBC, OTX_CPT_SHA256); +} + +static int otx_cpt_aead_cbc_aes_sha384_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX_CPT_AES_CBC, OTX_CPT_SHA384); +} + +static int otx_cpt_aead_cbc_aes_sha512_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX_CPT_AES_CBC, OTX_CPT_SHA512); +} + +static int otx_cpt_aead_ecb_null_sha1_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX_CPT_CIPHER_NULL, OTX_CPT_SHA1); +} + +static int otx_cpt_aead_ecb_null_sha256_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX_CPT_CIPHER_NULL, OTX_CPT_SHA256); +} + +static int otx_cpt_aead_ecb_null_sha384_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX_CPT_CIPHER_NULL, OTX_CPT_SHA384); +} + +static int otx_cpt_aead_ecb_null_sha512_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX_CPT_CIPHER_NULL, OTX_CPT_SHA512); +} + +static int otx_cpt_aead_gcm_aes_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX_CPT_AES_GCM, OTX_CPT_MAC_NULL); +} + +static void otx_cpt_aead_exit(struct crypto_aead *tfm) +{ + struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); + + kfree(ctx->ipad); + kfree(ctx->opad); + crypto_free_shash(ctx->hashalg); + kfree(ctx->sdesc); +} + +/* + * This is the Integrity Check Value validation (aka the authentication tag + * length) + */ +static int otx_cpt_aead_set_authsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); + + switch (ctx->mac_type) { + case OTX_CPT_SHA1: + if (authsize != SHA1_DIGEST_SIZE && + authsize != SHA1_TRUNC_DIGEST_SIZE) + return -EINVAL; + + if (authsize == SHA1_TRUNC_DIGEST_SIZE) + ctx->is_trunc_hmac = true; + break; + + case OTX_CPT_SHA256: + if (authsize != SHA256_DIGEST_SIZE && + authsize != SHA256_TRUNC_DIGEST_SIZE) + return -EINVAL; + + if (authsize == SHA256_TRUNC_DIGEST_SIZE) + ctx->is_trunc_hmac = true; + break; + + case OTX_CPT_SHA384: + if (authsize != SHA384_DIGEST_SIZE && + authsize != SHA384_TRUNC_DIGEST_SIZE) + return -EINVAL; + + if (authsize == SHA384_TRUNC_DIGEST_SIZE) + ctx->is_trunc_hmac = true; + break; + + case OTX_CPT_SHA512: + if (authsize != SHA512_DIGEST_SIZE && + authsize != SHA512_TRUNC_DIGEST_SIZE) + return -EINVAL; + + if (authsize == SHA512_TRUNC_DIGEST_SIZE) + ctx->is_trunc_hmac = true; + break; + + case OTX_CPT_MAC_NULL: + if (ctx->cipher_type == OTX_CPT_AES_GCM) { + if (authsize != AES_GCM_ICV_SIZE) + return -EINVAL; + } else + return -EINVAL; + break; + + default: + return -EINVAL; + } + + tfm->authsize = authsize; + return 0; +} + +static struct otx_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg) +{ + struct otx_cpt_sdesc *sdesc; + int size; + + size = sizeof(struct shash_desc) + crypto_shash_descsize(alg); + sdesc = kmalloc(size, GFP_KERNEL); + if (!sdesc) + return NULL; + + sdesc->shash.tfm = alg; + + return sdesc; +} + +static inline void swap_data32(void *buf, u32 len) +{ + cpu_to_be32_array(buf, buf, len / 4); +} + +static inline void swap_data64(void *buf, u32 len) +{ + __be64 *dst = buf; + u64 *src = buf; + int i = 0; + + for (i = 0 ; i < len / 8; i++, src++, dst++) + *dst = cpu_to_be64p(src); +} + +static int swap_pad(u8 mac_type, u8 *pad) +{ + struct sha512_state *sha512; + struct sha256_state *sha256; + struct sha1_state *sha1; + + switch (mac_type) { + case OTX_CPT_SHA1: + sha1 = (struct sha1_state *)pad; + swap_data32(sha1->state, SHA1_DIGEST_SIZE); + break; + + case OTX_CPT_SHA256: + sha256 = (struct sha256_state *)pad; + swap_data32(sha256->state, SHA256_DIGEST_SIZE); + break; + + case OTX_CPT_SHA384: + case OTX_CPT_SHA512: + sha512 = (struct sha512_state *)pad; + swap_data64(sha512->state, SHA512_DIGEST_SIZE); + break; + + default: + return -EINVAL; + } + + return 0; +} + +static int aead_hmac_init(struct crypto_aead *cipher, + struct crypto_authenc_keys *keys) +{ + struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher); + int ds = crypto_shash_digestsize(ctx->hashalg); + int bs = crypto_shash_blocksize(ctx->hashalg); + int authkeylen = keys->authkeylen; + u8 *ipad = NULL, *opad = NULL; + int icount = 0; + int ret; + + if (authkeylen > bs) { + ret = crypto_shash_digest(&ctx->sdesc->shash, keys->authkey, + authkeylen, ctx->key); + if (ret) + return ret; + authkeylen = ds; + } else + memcpy(ctx->key, keys->authkey, authkeylen); + + ctx->enc_key_len = keys->enckeylen; + ctx->auth_key_len = authkeylen; + + if (ctx->cipher_type == OTX_CPT_CIPHER_NULL) + return keys->enckeylen ? -EINVAL : 0; + + switch (keys->enckeylen) { + case AES_KEYSIZE_128: + ctx->key_type = OTX_CPT_AES_128_BIT; + break; + case AES_KEYSIZE_192: + ctx->key_type = OTX_CPT_AES_192_BIT; + break; + case AES_KEYSIZE_256: + ctx->key_type = OTX_CPT_AES_256_BIT; + break; + default: + /* Invalid key length */ + return -EINVAL; + } + + memcpy(ctx->key + authkeylen, keys->enckey, keys->enckeylen); + + ipad = ctx->ipad; + opad = ctx->opad; + + memcpy(ipad, ctx->key, authkeylen); + memset(ipad + authkeylen, 0, bs - authkeylen); + memcpy(opad, ipad, bs); + + for (icount = 0; icount < bs; icount++) { + ipad[icount] ^= 0x36; + opad[icount] ^= 0x5c; + } + + /* + * Partial Hash calculated from the software + * algorithm is retrieved for IPAD & OPAD + */ + + /* IPAD Calculation */ + crypto_shash_init(&ctx->sdesc->shash); + crypto_shash_update(&ctx->sdesc->shash, ipad, bs); + crypto_shash_export(&ctx->sdesc->shash, ipad); + ret = swap_pad(ctx->mac_type, ipad); + if (ret) + goto calc_fail; + + /* OPAD Calculation */ + crypto_shash_init(&ctx->sdesc->shash); + crypto_shash_update(&ctx->sdesc->shash, opad, bs); + crypto_shash_export(&ctx->sdesc->shash, opad); + ret = swap_pad(ctx->mac_type, opad); + +calc_fail: + return ret; +} + +static int otx_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead *cipher, + const unsigned char *key, + unsigned int keylen) +{ + struct crypto_authenc_keys authenc_keys; + int status; + + status = crypto_authenc_extractkeys(&authenc_keys, key, keylen); + if (status) + goto badkey; + + status = aead_hmac_init(cipher, &authenc_keys); + +badkey: + return status; +} + +static int otx_cpt_aead_ecb_null_sha_setkey(struct crypto_aead *cipher, + const unsigned char *key, + unsigned int keylen) +{ + return otx_cpt_aead_cbc_aes_sha_setkey(cipher, key, keylen); +} + +static int otx_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher, + const unsigned char *key, + unsigned int keylen) +{ + struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher); + + /* + * For aes gcm we expect to get encryption key (16, 24, 32 bytes) + * and salt (4 bytes) + */ + switch (keylen) { + case AES_KEYSIZE_128 + AES_GCM_SALT_SIZE: + ctx->key_type = OTX_CPT_AES_128_BIT; + ctx->enc_key_len = AES_KEYSIZE_128; + break; + case AES_KEYSIZE_192 + AES_GCM_SALT_SIZE: + ctx->key_type = OTX_CPT_AES_192_BIT; + ctx->enc_key_len = AES_KEYSIZE_192; + break; + case AES_KEYSIZE_256 + AES_GCM_SALT_SIZE: + ctx->key_type = OTX_CPT_AES_256_BIT; + ctx->enc_key_len = AES_KEYSIZE_256; + break; + default: + /* Invalid key and salt length */ + return -EINVAL; + } + + /* Store encryption key and salt */ + memcpy(ctx->key, key, keylen); + + return 0; +} + +static inline u32 create_aead_ctx_hdr(struct aead_request *req, u32 enc, + u32 *argcnt) +{ + struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); + struct otx_cpt_req_info *req_info = &rctx->cpt_req; + struct otx_cpt_fc_ctx *fctx = &rctx->fctx; + int mac_len = crypto_aead_authsize(tfm); + int ds; + + rctx->ctrl_word.e.enc_data_offset = req->assoclen; + + switch (ctx->cipher_type) { + case OTX_CPT_AES_CBC: + fctx->enc.enc_ctrl.e.iv_source = OTX_CPT_FROM_CPTR; + /* Copy encryption key to context */ + memcpy(fctx->enc.encr_key, ctx->key + ctx->auth_key_len, + ctx->enc_key_len); + /* Copy IV to context */ + memcpy(fctx->enc.encr_iv, req->iv, crypto_aead_ivsize(tfm)); + + ds = crypto_shash_digestsize(ctx->hashalg); + if (ctx->mac_type == OTX_CPT_SHA384) + ds = SHA512_DIGEST_SIZE; + if (ctx->ipad) + memcpy(fctx->hmac.e.ipad, ctx->ipad, ds); + if (ctx->opad) + memcpy(fctx->hmac.e.opad, ctx->opad, ds); + break; + + case OTX_CPT_AES_GCM: + fctx->enc.enc_ctrl.e.iv_source = OTX_CPT_FROM_DPTR; + /* Copy encryption key to context */ + memcpy(fctx->enc.encr_key, ctx->key, ctx->enc_key_len); + /* Copy salt to context */ + memcpy(fctx->enc.encr_iv, ctx->key + ctx->enc_key_len, + AES_GCM_SALT_SIZE); + + rctx->ctrl_word.e.iv_offset = req->assoclen - AES_GCM_IV_OFFSET; + break; + + default: + /* Unknown cipher type */ + return -EINVAL; + } + rctx->ctrl_word.flags = cpu_to_be64(rctx->ctrl_word.cflags); + + req_info->ctrl.s.dma_mode = OTX_CPT_DMA_GATHER_SCATTER; + req_info->ctrl.s.se_req = OTX_CPT_SE_CORE_REQ; + req_info->req.opcode.s.major = OTX_CPT_MAJOR_OP_FC | + DMA_MODE_FLAG(OTX_CPT_DMA_GATHER_SCATTER); + if (enc) { + req_info->req.opcode.s.minor = 2; + req_info->req.param1 = req->cryptlen; + req_info->req.param2 = req->cryptlen + req->assoclen; + } else { + req_info->req.opcode.s.minor = 3; + req_info->req.param1 = req->cryptlen - mac_len; + req_info->req.param2 = req->cryptlen + req->assoclen - mac_len; + } + + fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type; + fctx->enc.enc_ctrl.e.aes_key = ctx->key_type; + fctx->enc.enc_ctrl.e.mac_type = ctx->mac_type; + fctx->enc.enc_ctrl.e.mac_len = mac_len; + fctx->enc.enc_ctrl.flags = cpu_to_be64(fctx->enc.enc_ctrl.cflags); + + /* + * Storing Packet Data Information in offset + * Control Word First 8 bytes + */ + req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word; + req_info->in[*argcnt].size = CONTROL_WORD_LEN; + req_info->req.dlen += CONTROL_WORD_LEN; + ++(*argcnt); + + req_info->in[*argcnt].vptr = (u8 *)fctx; + req_info->in[*argcnt].size = sizeof(struct otx_cpt_fc_ctx); + req_info->req.dlen += sizeof(struct otx_cpt_fc_ctx); + ++(*argcnt); + + return 0; +} + +static inline u32 create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt, + u32 enc) +{ + struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); + struct otx_cpt_req_info *req_info = &rctx->cpt_req; + + req_info->ctrl.s.dma_mode = OTX_CPT_DMA_GATHER_SCATTER; + req_info->ctrl.s.se_req = OTX_CPT_SE_CORE_REQ; + req_info->req.opcode.s.major = OTX_CPT_MAJOR_OP_HMAC | + DMA_MODE_FLAG(OTX_CPT_DMA_GATHER_SCATTER); + req_info->is_trunc_hmac = ctx->is_trunc_hmac; + + req_info->req.opcode.s.minor = 0; + req_info->req.param1 = ctx->auth_key_len; + req_info->req.param2 = ctx->mac_type << 8; + + /* Add authentication key */ + req_info->in[*argcnt].vptr = ctx->key; + req_info->in[*argcnt].size = round_up(ctx->auth_key_len, 8); + req_info->req.dlen += round_up(ctx->auth_key_len, 8); + ++(*argcnt); + + return 0; +} + +static inline u32 create_aead_input_list(struct aead_request *req, u32 enc) +{ + struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req); + struct otx_cpt_req_info *req_info = &rctx->cpt_req; + u32 inputlen = req->cryptlen + req->assoclen; + u32 status, argcnt = 0; + + status = create_aead_ctx_hdr(req, enc, &argcnt); + if (status) + return status; + update_input_data(req_info, req->src, inputlen, &argcnt); + req_info->incnt = argcnt; + + return 0; +} + +static inline u32 create_aead_output_list(struct aead_request *req, u32 enc, + u32 mac_len) +{ + struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req); + struct otx_cpt_req_info *req_info = &rctx->cpt_req; + u32 argcnt = 0, outputlen = 0; + + if (enc) + outputlen = req->cryptlen + req->assoclen + mac_len; + else + outputlen = req->cryptlen + req->assoclen - mac_len; + + update_output_data(req_info, req->dst, 0, outputlen, &argcnt); + req_info->outcnt = argcnt; + + return 0; +} + +static inline u32 create_aead_null_input_list(struct aead_request *req, + u32 enc, u32 mac_len) +{ + struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req); + struct otx_cpt_req_info *req_info = &rctx->cpt_req; + u32 inputlen, argcnt = 0; + + if (enc) + inputlen = req->cryptlen + req->assoclen; + else + inputlen = req->cryptlen + req->assoclen - mac_len; + + create_hmac_ctx_hdr(req, &argcnt, enc); + update_input_data(req_info, req->src, inputlen, &argcnt); + req_info->incnt = argcnt; + + return 0; +} + +static inline u32 create_aead_null_output_list(struct aead_request *req, + u32 enc, u32 mac_len) +{ + struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req); + struct otx_cpt_req_info *req_info = &rctx->cpt_req; + struct scatterlist *dst; + u8 *ptr = NULL; + int argcnt = 0, status, offset; + u32 inputlen; + + if (enc) + inputlen = req->cryptlen + req->assoclen; + else + inputlen = req->cryptlen + req->assoclen - mac_len; + + /* + * If source and destination are different + * then copy payload to destination + */ + if (req->src != req->dst) { + + ptr = kmalloc(inputlen, (req_info->areq->flags & + CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC); + if (!ptr) { + status = -ENOMEM; + goto error; + } + + status = sg_copy_to_buffer(req->src, sg_nents(req->src), ptr, + inputlen); + if (status != inputlen) { + status = -EINVAL; + goto error_free; + } + status = sg_copy_from_buffer(req->dst, sg_nents(req->dst), ptr, + inputlen); + if (status != inputlen) { + status = -EINVAL; + goto error_free; + } + kfree(ptr); + } + + if (enc) { + /* + * In an encryption scenario hmac needs + * to be appended after payload + */ + dst = req->dst; + offset = inputlen; + while (offset >= dst->length) { + offset -= dst->length; + dst = sg_next(dst); + if (!dst) { + status = -ENOENT; + goto error; + } + } + + update_output_data(req_info, dst, offset, mac_len, &argcnt); + } else { + /* + * In a decryption scenario calculated hmac for received + * payload needs to be compare with hmac received + */ + status = sg_copy_buffer(req->src, sg_nents(req->src), + rctx->fctx.hmac.s.hmac_recv, mac_len, + inputlen, true); + if (status != mac_len) { + status = -EINVAL; + goto error; + } + + req_info->out[argcnt].vptr = rctx->fctx.hmac.s.hmac_calc; + req_info->out[argcnt].size = mac_len; + argcnt++; + } + + req_info->outcnt = argcnt; + return 0; + +error_free: + kfree(ptr); +error: + return status; +} + +static u32 cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc) +{ + struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req); + struct otx_cpt_req_info *req_info = &rctx->cpt_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct pci_dev *pdev; + u32 status, cpu_num; + + /* Clear control words */ + rctx->ctrl_word.flags = 0; + rctx->fctx.enc.enc_ctrl.flags = 0; + + req_info->callback = otx_cpt_aead_callback; + req_info->areq = &req->base; + req_info->req_type = reg_type; + req_info->is_enc = enc; + req_info->is_trunc_hmac = false; + + switch (reg_type) { + case OTX_CPT_AEAD_ENC_DEC_REQ: + status = create_aead_input_list(req, enc); + if (status) + return status; + status = create_aead_output_list(req, enc, + crypto_aead_authsize(tfm)); + if (status) + return status; + break; + + case OTX_CPT_AEAD_ENC_DEC_NULL_REQ: + status = create_aead_null_input_list(req, enc, + crypto_aead_authsize(tfm)); + if (status) + return status; + status = create_aead_null_output_list(req, enc, + crypto_aead_authsize(tfm)); + if (status) + return status; + break; + + default: + return -EINVAL; + } + + /* Validate that request doesn't exceed maximum CPT supported size */ + if (req_info->req.param1 > OTX_CPT_MAX_REQ_SIZE || + req_info->req.param2 > OTX_CPT_MAX_REQ_SIZE) + return -E2BIG; + + status = get_se_device(&pdev, &cpu_num); + if (status) + return status; + + req_info->ctrl.s.grp = 0; + + status = otx_cpt_do_request(pdev, req_info, cpu_num); + /* + * We perform an asynchronous send and once + * the request is completed the driver would + * intimate through registered call back functions + */ + return status; +} + +static int otx_cpt_aead_encrypt(struct aead_request *req) +{ + return cpt_aead_enc_dec(req, OTX_CPT_AEAD_ENC_DEC_REQ, true); +} + +static int otx_cpt_aead_decrypt(struct aead_request *req) +{ + return cpt_aead_enc_dec(req, OTX_CPT_AEAD_ENC_DEC_REQ, false); +} + +static int otx_cpt_aead_null_encrypt(struct aead_request *req) +{ + return cpt_aead_enc_dec(req, OTX_CPT_AEAD_ENC_DEC_NULL_REQ, true); +} + +static int otx_cpt_aead_null_decrypt(struct aead_request *req) +{ + return cpt_aead_enc_dec(req, OTX_CPT_AEAD_ENC_DEC_NULL_REQ, false); +} + +static struct skcipher_alg otx_cpt_skciphers[] = { { + .base.cra_name = "xts(aes)", + .base.cra_driver_name = "cpt_xts_aes", + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct otx_cpt_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_module = THIS_MODULE, + + .init = otx_cpt_enc_dec_init, + .ivsize = AES_BLOCK_SIZE, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .setkey = otx_cpt_skcipher_xts_setkey, + .encrypt = otx_cpt_skcipher_encrypt, + .decrypt = otx_cpt_skcipher_decrypt, +}, { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cpt_cbc_aes", + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct otx_cpt_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_module = THIS_MODULE, + + .init = otx_cpt_enc_dec_init, + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = otx_cpt_skcipher_cbc_aes_setkey, + .encrypt = otx_cpt_skcipher_encrypt, + .decrypt = otx_cpt_skcipher_decrypt, +}, { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "cpt_ecb_aes", + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct otx_cpt_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_module = THIS_MODULE, + + .init = otx_cpt_enc_dec_init, + .ivsize = 0, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = otx_cpt_skcipher_ecb_aes_setkey, + .encrypt = otx_cpt_skcipher_encrypt, + .decrypt = otx_cpt_skcipher_decrypt, +}, { + .base.cra_name = "cbc(des3_ede)", + .base.cra_driver_name = "cpt_cbc_des3_ede", + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct otx_cpt_des3_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_module = THIS_MODULE, + + .init = otx_cpt_enc_dec_init, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = otx_cpt_skcipher_cbc_des3_setkey, + .encrypt = otx_cpt_skcipher_encrypt, + .decrypt = otx_cpt_skcipher_decrypt, +}, { + .base.cra_name = "ecb(des3_ede)", + .base.cra_driver_name = "cpt_ecb_des3_ede", + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct otx_cpt_des3_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_module = THIS_MODULE, + + .init = otx_cpt_enc_dec_init, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = 0, + .setkey = otx_cpt_skcipher_ecb_des3_setkey, + .encrypt = otx_cpt_skcipher_encrypt, + .decrypt = otx_cpt_skcipher_decrypt, +} }; + +static struct aead_alg otx_cpt_aeads[] = { { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "cpt_hmac_sha1_cbc_aes", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx_cpt_aead_cbc_aes_sha1_init, + .exit = otx_cpt_aead_exit, + .setkey = otx_cpt_aead_cbc_aes_sha_setkey, + .setauthsize = otx_cpt_aead_set_authsize, + .encrypt = otx_cpt_aead_encrypt, + .decrypt = otx_cpt_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "cpt_hmac_sha256_cbc_aes", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx_cpt_aead_cbc_aes_sha256_init, + .exit = otx_cpt_aead_exit, + .setkey = otx_cpt_aead_cbc_aes_sha_setkey, + .setauthsize = otx_cpt_aead_set_authsize, + .encrypt = otx_cpt_aead_encrypt, + .decrypt = otx_cpt_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(aes))", + .cra_driver_name = "cpt_hmac_sha384_cbc_aes", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx_cpt_aead_cbc_aes_sha384_init, + .exit = otx_cpt_aead_exit, + .setkey = otx_cpt_aead_cbc_aes_sha_setkey, + .setauthsize = otx_cpt_aead_set_authsize, + .encrypt = otx_cpt_aead_encrypt, + .decrypt = otx_cpt_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(aes))", + .cra_driver_name = "cpt_hmac_sha512_cbc_aes", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx_cpt_aead_cbc_aes_sha512_init, + .exit = otx_cpt_aead_exit, + .setkey = otx_cpt_aead_cbc_aes_sha_setkey, + .setauthsize = otx_cpt_aead_set_authsize, + .encrypt = otx_cpt_aead_encrypt, + .decrypt = otx_cpt_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha1),ecb(cipher_null))", + .cra_driver_name = "cpt_hmac_sha1_ecb_null", + .cra_blocksize = 1, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx_cpt_aead_ecb_null_sha1_init, + .exit = otx_cpt_aead_exit, + .setkey = otx_cpt_aead_ecb_null_sha_setkey, + .setauthsize = otx_cpt_aead_set_authsize, + .encrypt = otx_cpt_aead_null_encrypt, + .decrypt = otx_cpt_aead_null_decrypt, + .ivsize = 0, + .maxauthsize = SHA1_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha256),ecb(cipher_null))", + .cra_driver_name = "cpt_hmac_sha256_ecb_null", + .cra_blocksize = 1, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx_cpt_aead_ecb_null_sha256_init, + .exit = otx_cpt_aead_exit, + .setkey = otx_cpt_aead_ecb_null_sha_setkey, + .setauthsize = otx_cpt_aead_set_authsize, + .encrypt = otx_cpt_aead_null_encrypt, + .decrypt = otx_cpt_aead_null_decrypt, + .ivsize = 0, + .maxauthsize = SHA256_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha384),ecb(cipher_null))", + .cra_driver_name = "cpt_hmac_sha384_ecb_null", + .cra_blocksize = 1, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx_cpt_aead_ecb_null_sha384_init, + .exit = otx_cpt_aead_exit, + .setkey = otx_cpt_aead_ecb_null_sha_setkey, + .setauthsize = otx_cpt_aead_set_authsize, + .encrypt = otx_cpt_aead_null_encrypt, + .decrypt = otx_cpt_aead_null_decrypt, + .ivsize = 0, + .maxauthsize = SHA384_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha512),ecb(cipher_null))", + .cra_driver_name = "cpt_hmac_sha512_ecb_null", + .cra_blocksize = 1, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx_cpt_aead_ecb_null_sha512_init, + .exit = otx_cpt_aead_exit, + .setkey = otx_cpt_aead_ecb_null_sha_setkey, + .setauthsize = otx_cpt_aead_set_authsize, + .encrypt = otx_cpt_aead_null_encrypt, + .decrypt = otx_cpt_aead_null_decrypt, + .ivsize = 0, + .maxauthsize = SHA512_DIGEST_SIZE, +}, { + .base = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "cpt_rfc4106_gcm_aes", + .cra_blocksize = 1, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, + .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx_cpt_aead_gcm_aes_init, + .exit = otx_cpt_aead_exit, + .setkey = otx_cpt_aead_gcm_aes_setkey, + .setauthsize = otx_cpt_aead_set_authsize, + .encrypt = otx_cpt_aead_encrypt, + .decrypt = otx_cpt_aead_decrypt, + .ivsize = AES_GCM_IV_SIZE, + .maxauthsize = AES_GCM_ICV_SIZE, +} }; + +static inline int is_any_alg_used(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(otx_cpt_skciphers); i++) + if (refcount_read(&otx_cpt_skciphers[i].base.cra_refcnt) != 1) + return true; + for (i = 0; i < ARRAY_SIZE(otx_cpt_aeads); i++) + if (refcount_read(&otx_cpt_aeads[i].base.cra_refcnt) != 1) + return true; + return false; +} + +static inline int cpt_register_algs(void) +{ + int i, err = 0; + + if (!IS_ENABLED(CONFIG_DM_CRYPT)) { + for (i = 0; i < ARRAY_SIZE(otx_cpt_skciphers); i++) + otx_cpt_skciphers[i].base.cra_flags &= ~CRYPTO_ALG_DEAD; + + err = crypto_register_skciphers(otx_cpt_skciphers, + ARRAY_SIZE(otx_cpt_skciphers)); + if (err) + return err; + } + + for (i = 0; i < ARRAY_SIZE(otx_cpt_aeads); i++) + otx_cpt_aeads[i].base.cra_flags &= ~CRYPTO_ALG_DEAD; + + err = crypto_register_aeads(otx_cpt_aeads, ARRAY_SIZE(otx_cpt_aeads)); + if (err) { + crypto_unregister_skciphers(otx_cpt_skciphers, + ARRAY_SIZE(otx_cpt_skciphers)); + return err; + } + + return 0; +} + +static inline void cpt_unregister_algs(void) +{ + crypto_unregister_skciphers(otx_cpt_skciphers, + ARRAY_SIZE(otx_cpt_skciphers)); + crypto_unregister_aeads(otx_cpt_aeads, ARRAY_SIZE(otx_cpt_aeads)); +} + +static int compare_func(const void *lptr, const void *rptr) +{ + struct cpt_device_desc *ldesc = (struct cpt_device_desc *) lptr; + struct cpt_device_desc *rdesc = (struct cpt_device_desc *) rptr; + + if (ldesc->dev->devfn < rdesc->dev->devfn) + return -1; + if (ldesc->dev->devfn > rdesc->dev->devfn) + return 1; + return 0; +} + +int otx_cpt_crypto_init(struct pci_dev *pdev, struct module *mod, + enum otx_cptpf_type pf_type, + enum otx_cptvf_type engine_type, + int num_queues, int num_devices) +{ + int ret = 0; + int count; + + mutex_lock(&mutex); + switch (engine_type) { + case OTX_CPT_SE_TYPES: + count = atomic_read(&se_devices.count); + if (count >= CPT_MAX_VF_NUM) { + dev_err(&pdev->dev, "No space to add a new device\n"); + ret = -ENOSPC; + goto err; + } + se_devices.desc[count].pf_type = pf_type; + se_devices.desc[count].num_queues = num_queues; + se_devices.desc[count++].dev = pdev; + atomic_inc(&se_devices.count); + + if (atomic_read(&se_devices.count) == num_devices && + is_crypto_registered == false) { + if (cpt_register_algs()) { + dev_err(&pdev->dev, + "Error in registering crypto algorithms\n"); + ret = -EINVAL; + goto err; + } + try_module_get(mod); + is_crypto_registered = true; + } + sort(se_devices.desc, count, sizeof(struct cpt_device_desc), + compare_func, NULL); + break; + + case OTX_CPT_AE_TYPES: + count = atomic_read(&ae_devices.count); + if (count >= CPT_MAX_VF_NUM) { + dev_err(&pdev->dev, "No space to a add new device\n"); + ret = -ENOSPC; + goto err; + } + ae_devices.desc[count].pf_type = pf_type; + ae_devices.desc[count].num_queues = num_queues; + ae_devices.desc[count++].dev = pdev; + atomic_inc(&ae_devices.count); + sort(ae_devices.desc, count, sizeof(struct cpt_device_desc), + compare_func, NULL); + break; + + default: + dev_err(&pdev->dev, "Unknown VF type %d\n", engine_type); + ret = BAD_OTX_CPTVF_TYPE; + } +err: + mutex_unlock(&mutex); + return ret; +} + +void otx_cpt_crypto_exit(struct pci_dev *pdev, struct module *mod, + enum otx_cptvf_type engine_type) +{ + struct cpt_device_table *dev_tbl; + bool dev_found = false; + int i, j, count; + + mutex_lock(&mutex); + + dev_tbl = (engine_type == OTX_CPT_AE_TYPES) ? &ae_devices : &se_devices; + count = atomic_read(&dev_tbl->count); + for (i = 0; i < count; i++) + if (pdev == dev_tbl->desc[i].dev) { + for (j = i; j < count-1; j++) + dev_tbl->desc[j] = dev_tbl->desc[j+1]; + dev_found = true; + break; + } + + if (!dev_found) { + dev_err(&pdev->dev, "%s device not found\n", __func__); + goto exit; + } + + if (engine_type != OTX_CPT_AE_TYPES) { + if (atomic_dec_and_test(&se_devices.count) && + !is_any_alg_used()) { + cpt_unregister_algs(); + module_put(mod); + is_crypto_registered = false; + } + } else + atomic_dec(&ae_devices.count); +exit: + mutex_unlock(&mutex); +} diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_algs.h b/drivers/crypto/marvell/octeontx/otx_cptvf_algs.h new file mode 100644 index 000000000..a50b5e2f8 --- /dev/null +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_algs.h @@ -0,0 +1,189 @@ +/* SPDX-License-Identifier: GPL-2.0 + * Marvell OcteonTX CPT driver + * + * Copyright (C) 2019 Marvell International Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef __OTX_CPT_ALGS_H +#define __OTX_CPT_ALGS_H + +#include +#include "otx_cpt_common.h" + +#define OTX_CPT_MAX_ENC_KEY_SIZE 32 +#define OTX_CPT_MAX_HASH_KEY_SIZE 64 +#define OTX_CPT_MAX_KEY_SIZE (OTX_CPT_MAX_ENC_KEY_SIZE + \ + OTX_CPT_MAX_HASH_KEY_SIZE) +enum otx_cpt_request_type { + OTX_CPT_ENC_DEC_REQ = 0x1, + OTX_CPT_AEAD_ENC_DEC_REQ = 0x2, + OTX_CPT_AEAD_ENC_DEC_NULL_REQ = 0x3, + OTX_CPT_PASSTHROUGH_REQ = 0x4 +}; + +enum otx_cpt_major_opcodes { + OTX_CPT_MAJOR_OP_MISC = 0x01, + OTX_CPT_MAJOR_OP_FC = 0x33, + OTX_CPT_MAJOR_OP_HMAC = 0x35, +}; + +enum otx_cpt_req_type { + OTX_CPT_AE_CORE_REQ, + OTX_CPT_SE_CORE_REQ +}; + +enum otx_cpt_cipher_type { + OTX_CPT_CIPHER_NULL = 0x0, + OTX_CPT_DES3_CBC = 0x1, + OTX_CPT_DES3_ECB = 0x2, + OTX_CPT_AES_CBC = 0x3, + OTX_CPT_AES_ECB = 0x4, + OTX_CPT_AES_CFB = 0x5, + OTX_CPT_AES_CTR = 0x6, + OTX_CPT_AES_GCM = 0x7, + OTX_CPT_AES_XTS = 0x8 +}; + +enum otx_cpt_mac_type { + OTX_CPT_MAC_NULL = 0x0, + OTX_CPT_MD5 = 0x1, + OTX_CPT_SHA1 = 0x2, + OTX_CPT_SHA224 = 0x3, + OTX_CPT_SHA256 = 0x4, + OTX_CPT_SHA384 = 0x5, + OTX_CPT_SHA512 = 0x6, + OTX_CPT_GMAC = 0x7 +}; + +enum otx_cpt_aes_key_len { + OTX_CPT_AES_128_BIT = 0x1, + OTX_CPT_AES_192_BIT = 0x2, + OTX_CPT_AES_256_BIT = 0x3 +}; + +union otx_cpt_encr_ctrl { + __be64 flags; + u64 cflags; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 enc_cipher:4; + u64 reserved1:1; + u64 aes_key:2; + u64 iv_source:1; + u64 mac_type:4; + u64 reserved2:3; + u64 auth_input_type:1; + u64 mac_len:8; + u64 reserved3:8; + u64 encr_offset:16; + u64 iv_offset:8; + u64 auth_offset:8; +#else + u64 auth_offset:8; + u64 iv_offset:8; + u64 encr_offset:16; + u64 reserved3:8; + u64 mac_len:8; + u64 auth_input_type:1; + u64 reserved2:3; + u64 mac_type:4; + u64 iv_source:1; + u64 aes_key:2; + u64 reserved1:1; + u64 enc_cipher:4; +#endif + } e; +}; + +struct otx_cpt_cipher { + const char *name; + u8 value; +}; + +struct otx_cpt_enc_context { + union otx_cpt_encr_ctrl enc_ctrl; + u8 encr_key[32]; + u8 encr_iv[16]; +}; + +union otx_cpt_fchmac_ctx { + struct { + u8 ipad[64]; + u8 opad[64]; + } e; + struct { + u8 hmac_calc[64]; /* HMAC calculated */ + u8 hmac_recv[64]; /* HMAC received */ + } s; +}; + +struct otx_cpt_fc_ctx { + struct otx_cpt_enc_context enc; + union otx_cpt_fchmac_ctx hmac; +}; + +struct otx_cpt_enc_ctx { + u32 key_len; + u8 enc_key[OTX_CPT_MAX_KEY_SIZE]; + u8 cipher_type; + u8 key_type; +}; + +struct otx_cpt_des3_ctx { + u32 key_len; + u8 des3_key[OTX_CPT_MAX_KEY_SIZE]; +}; + +union otx_cpt_offset_ctrl_word { + __be64 flags; + u64 cflags; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved:32; + u64 enc_data_offset:16; + u64 iv_offset:8; + u64 auth_offset:8; +#else + u64 auth_offset:8; + u64 iv_offset:8; + u64 enc_data_offset:16; + u64 reserved:32; +#endif + } e; +}; + +struct otx_cpt_req_ctx { + struct otx_cpt_req_info cpt_req; + union otx_cpt_offset_ctrl_word ctrl_word; + struct otx_cpt_fc_ctx fctx; +}; + +struct otx_cpt_sdesc { + struct shash_desc shash; +}; + +struct otx_cpt_aead_ctx { + u8 key[OTX_CPT_MAX_KEY_SIZE]; + struct crypto_shash *hashalg; + struct otx_cpt_sdesc *sdesc; + u8 *ipad; + u8 *opad; + u32 enc_key_len; + u32 auth_key_len; + u8 cipher_type; + u8 mac_type; + u8 key_type; + u8 is_trunc_hmac; +}; +int otx_cpt_crypto_init(struct pci_dev *pdev, struct module *mod, + enum otx_cptpf_type pf_type, + enum otx_cptvf_type engine_type, + int num_queues, int num_devices); +void otx_cpt_crypto_exit(struct pci_dev *pdev, struct module *mod, + enum otx_cptvf_type engine_type); + +#endif /* __OTX_CPT_ALGS_H */ diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c new file mode 100644 index 000000000..159c1d290 --- /dev/null +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c @@ -0,0 +1,977 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell OcteonTX CPT driver + * + * Copyright (C) 2019 Marvell International Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include "otx_cptvf.h" +#include "otx_cptvf_algs.h" +#include "otx_cptvf_reqmgr.h" + +#define DRV_NAME "octeontx-cptvf" +#define DRV_VERSION "1.0" + +static void vq_work_handler(unsigned long data) +{ + struct otx_cptvf_wqe_info *cwqe_info = + (struct otx_cptvf_wqe_info *) data; + + otx_cpt_post_process(&cwqe_info->vq_wqe[0]); +} + +static int init_worker_threads(struct otx_cptvf *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + struct otx_cptvf_wqe_info *cwqe_info; + int i; + + cwqe_info = kzalloc_obj(*cwqe_info); + if (!cwqe_info) + return -ENOMEM; + + if (cptvf->num_queues) { + dev_dbg(&pdev->dev, "Creating VQ worker threads (%d)\n", + cptvf->num_queues); + } + + for (i = 0; i < cptvf->num_queues; i++) { + tasklet_init(&cwqe_info->vq_wqe[i].twork, vq_work_handler, + (u64)cwqe_info); + cwqe_info->vq_wqe[i].cptvf = cptvf; + } + cptvf->wqe_info = cwqe_info; + + return 0; +} + +static void cleanup_worker_threads(struct otx_cptvf *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + struct otx_cptvf_wqe_info *cwqe_info; + int i; + + cwqe_info = (struct otx_cptvf_wqe_info *)cptvf->wqe_info; + if (!cwqe_info) + return; + + if (cptvf->num_queues) { + dev_dbg(&pdev->dev, "Cleaning VQ worker threads (%u)\n", + cptvf->num_queues); + } + + for (i = 0; i < cptvf->num_queues; i++) + tasklet_kill(&cwqe_info->vq_wqe[i].twork); + + kfree_sensitive(cwqe_info); + cptvf->wqe_info = NULL; +} + +static void free_pending_queues(struct otx_cpt_pending_qinfo *pqinfo) +{ + struct otx_cpt_pending_queue *queue; + int i; + + for_each_pending_queue(pqinfo, queue, i) { + if (!queue->head) + continue; + + /* free single queue */ + kfree_sensitive((queue->head)); + queue->front = 0; + queue->rear = 0; + queue->qlen = 0; + } + pqinfo->num_queues = 0; +} + +static int alloc_pending_queues(struct otx_cpt_pending_qinfo *pqinfo, u32 qlen, + u32 num_queues) +{ + struct otx_cpt_pending_queue *queue = NULL; + int ret; + u32 i; + + pqinfo->num_queues = num_queues; + + for_each_pending_queue(pqinfo, queue, i) { + queue->head = kzalloc_objs(*queue->head, qlen); + if (!queue->head) { + ret = -ENOMEM; + goto pending_qfail; + } + + queue->pending_count = 0; + queue->front = 0; + queue->rear = 0; + queue->qlen = qlen; + + /* init queue spin lock */ + spin_lock_init(&queue->lock); + } + return 0; + +pending_qfail: + free_pending_queues(pqinfo); + + return ret; +} + +static int init_pending_queues(struct otx_cptvf *cptvf, u32 qlen, + u32 num_queues) +{ + struct pci_dev *pdev = cptvf->pdev; + int ret; + + if (!num_queues) + return 0; + + ret = alloc_pending_queues(&cptvf->pqinfo, qlen, num_queues); + if (ret) { + dev_err(&pdev->dev, "Failed to setup pending queues (%u)\n", + num_queues); + return ret; + } + return 0; +} + +static void cleanup_pending_queues(struct otx_cptvf *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + + if (!cptvf->num_queues) + return; + + dev_dbg(&pdev->dev, "Cleaning VQ pending queue (%u)\n", + cptvf->num_queues); + free_pending_queues(&cptvf->pqinfo); +} + +static void free_command_queues(struct otx_cptvf *cptvf, + struct otx_cpt_cmd_qinfo *cqinfo) +{ + struct otx_cpt_cmd_queue *queue = NULL; + struct otx_cpt_cmd_chunk *chunk = NULL; + struct pci_dev *pdev = cptvf->pdev; + int i; + + /* clean up for each queue */ + for (i = 0; i < cptvf->num_queues; i++) { + queue = &cqinfo->queue[i]; + + while (!list_empty(&cqinfo->queue[i].chead)) { + chunk = list_first_entry(&cqinfo->queue[i].chead, + struct otx_cpt_cmd_chunk, nextchunk); + + dma_free_coherent(&pdev->dev, + chunk->size + OTX_CPT_NEXT_CHUNK_PTR_SIZE, + chunk->head, + chunk->dma_addr); + chunk->head = NULL; + chunk->dma_addr = 0; + list_del(&chunk->nextchunk); + kfree_sensitive(chunk); + } + queue->num_chunks = 0; + queue->idx = 0; + + } +} + +static int alloc_command_queues(struct otx_cptvf *cptvf, + struct otx_cpt_cmd_qinfo *cqinfo, + u32 qlen) +{ + struct otx_cpt_cmd_chunk *curr, *first, *last; + struct otx_cpt_cmd_queue *queue = NULL; + struct pci_dev *pdev = cptvf->pdev; + size_t q_size, c_size, rem_q_size; + u32 qcsize_bytes; + int i; + + + /* Qsize in dwords, needed for SADDR config, 1-next chunk pointer */ + cptvf->qsize = min(qlen, cqinfo->qchunksize) * + OTX_CPT_NEXT_CHUNK_PTR_SIZE + 1; + /* Qsize in bytes to create space for alignment */ + q_size = qlen * OTX_CPT_INST_SIZE; + + qcsize_bytes = cqinfo->qchunksize * OTX_CPT_INST_SIZE; + + /* per queue initialization */ + for (i = 0; i < cptvf->num_queues; i++) { + rem_q_size = q_size; + first = NULL; + last = NULL; + + queue = &cqinfo->queue[i]; + INIT_LIST_HEAD(&queue->chead); + do { + curr = kzalloc_obj(*curr); + if (!curr) + goto cmd_qfail; + + c_size = (rem_q_size > qcsize_bytes) ? qcsize_bytes : + rem_q_size; + curr->head = dma_alloc_coherent(&pdev->dev, + c_size + OTX_CPT_NEXT_CHUNK_PTR_SIZE, + &curr->dma_addr, GFP_KERNEL); + if (!curr->head) { + dev_err(&pdev->dev, + "Command Q (%d) chunk (%d) allocation failed\n", + i, queue->num_chunks); + goto free_curr; + } + curr->size = c_size; + + if (queue->num_chunks == 0) { + first = curr; + queue->base = first; + } + list_add_tail(&curr->nextchunk, + &cqinfo->queue[i].chead); + + queue->num_chunks++; + rem_q_size -= c_size; + if (last) + *((u64 *)(&last->head[last->size])) = + (u64)curr->dma_addr; + + last = curr; + } while (rem_q_size); + + /* + * Make the queue circular, tie back last chunk entry to head + */ + curr = first; + *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr; + queue->qhead = curr; + } + return 0; +free_curr: + kfree(curr); +cmd_qfail: + free_command_queues(cptvf, cqinfo); + return -ENOMEM; +} + +static int init_command_queues(struct otx_cptvf *cptvf, u32 qlen) +{ + struct pci_dev *pdev = cptvf->pdev; + int ret; + + /* setup command queues */ + ret = alloc_command_queues(cptvf, &cptvf->cqinfo, qlen); + if (ret) { + dev_err(&pdev->dev, "Failed to allocate command queues (%u)\n", + cptvf->num_queues); + return ret; + } + return ret; +} + +static void cleanup_command_queues(struct otx_cptvf *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + + if (!cptvf->num_queues) + return; + + dev_dbg(&pdev->dev, "Cleaning VQ command queue (%u)\n", + cptvf->num_queues); + free_command_queues(cptvf, &cptvf->cqinfo); +} + +static void cptvf_sw_cleanup(struct otx_cptvf *cptvf) +{ + cleanup_worker_threads(cptvf); + cleanup_pending_queues(cptvf); + cleanup_command_queues(cptvf); +} + +static int cptvf_sw_init(struct otx_cptvf *cptvf, u32 qlen, u32 num_queues) +{ + struct pci_dev *pdev = cptvf->pdev; + u32 max_dev_queues = 0; + int ret; + + max_dev_queues = OTX_CPT_NUM_QS_PER_VF; + /* possible cpus */ + num_queues = min_t(u32, num_queues, max_dev_queues); + cptvf->num_queues = num_queues; + + ret = init_command_queues(cptvf, qlen); + if (ret) { + dev_err(&pdev->dev, "Failed to setup command queues (%u)\n", + num_queues); + return ret; + } + + ret = init_pending_queues(cptvf, qlen, num_queues); + if (ret) { + dev_err(&pdev->dev, "Failed to setup pending queues (%u)\n", + num_queues); + goto setup_pqfail; + } + + /* Create worker threads for BH processing */ + ret = init_worker_threads(cptvf); + if (ret) { + dev_err(&pdev->dev, "Failed to setup worker threads\n"); + goto init_work_fail; + } + return 0; + +init_work_fail: + cleanup_worker_threads(cptvf); + cleanup_pending_queues(cptvf); + +setup_pqfail: + cleanup_command_queues(cptvf); + + return ret; +} + +static void cptvf_free_irq_affinity(struct otx_cptvf *cptvf, int vec) +{ + irq_set_affinity_hint(pci_irq_vector(cptvf->pdev, vec), NULL); + free_cpumask_var(cptvf->affinity_mask[vec]); +} + +static void cptvf_write_vq_ctl(struct otx_cptvf *cptvf, bool val) +{ + union otx_cptx_vqx_ctl vqx_ctl; + + vqx_ctl.u = readq(cptvf->reg_base + OTX_CPT_VQX_CTL(0)); + vqx_ctl.s.ena = val; + writeq(vqx_ctl.u, cptvf->reg_base + OTX_CPT_VQX_CTL(0)); +} + +void otx_cptvf_write_vq_doorbell(struct otx_cptvf *cptvf, u32 val) +{ + union otx_cptx_vqx_doorbell vqx_dbell; + + vqx_dbell.u = readq(cptvf->reg_base + OTX_CPT_VQX_DOORBELL(0)); + vqx_dbell.s.dbell_cnt = val * 8; /* Num of Instructions * 8 words */ + writeq(vqx_dbell.u, cptvf->reg_base + OTX_CPT_VQX_DOORBELL(0)); +} + +static void cptvf_write_vq_inprog(struct otx_cptvf *cptvf, u8 val) +{ + union otx_cptx_vqx_inprog vqx_inprg; + + vqx_inprg.u = readq(cptvf->reg_base + OTX_CPT_VQX_INPROG(0)); + vqx_inprg.s.inflight = val; + writeq(vqx_inprg.u, cptvf->reg_base + OTX_CPT_VQX_INPROG(0)); +} + +static void cptvf_write_vq_done_numwait(struct otx_cptvf *cptvf, u32 val) +{ + union otx_cptx_vqx_done_wait vqx_dwait; + + vqx_dwait.u = readq(cptvf->reg_base + OTX_CPT_VQX_DONE_WAIT(0)); + vqx_dwait.s.num_wait = val; + writeq(vqx_dwait.u, cptvf->reg_base + OTX_CPT_VQX_DONE_WAIT(0)); +} + +static u32 cptvf_read_vq_done_numwait(struct otx_cptvf *cptvf) +{ + union otx_cptx_vqx_done_wait vqx_dwait; + + vqx_dwait.u = readq(cptvf->reg_base + OTX_CPT_VQX_DONE_WAIT(0)); + return vqx_dwait.s.num_wait; +} + +static void cptvf_write_vq_done_timewait(struct otx_cptvf *cptvf, u16 time) +{ + union otx_cptx_vqx_done_wait vqx_dwait; + + vqx_dwait.u = readq(cptvf->reg_base + OTX_CPT_VQX_DONE_WAIT(0)); + vqx_dwait.s.time_wait = time; + writeq(vqx_dwait.u, cptvf->reg_base + OTX_CPT_VQX_DONE_WAIT(0)); +} + + +static u16 cptvf_read_vq_done_timewait(struct otx_cptvf *cptvf) +{ + union otx_cptx_vqx_done_wait vqx_dwait; + + vqx_dwait.u = readq(cptvf->reg_base + OTX_CPT_VQX_DONE_WAIT(0)); + return vqx_dwait.s.time_wait; +} + +static void cptvf_enable_swerr_interrupts(struct otx_cptvf *cptvf) +{ + union otx_cptx_vqx_misc_ena_w1s vqx_misc_ena; + + vqx_misc_ena.u = readq(cptvf->reg_base + OTX_CPT_VQX_MISC_ENA_W1S(0)); + /* Enable SWERR interrupts for the requested VF */ + vqx_misc_ena.s.swerr = 1; + writeq(vqx_misc_ena.u, cptvf->reg_base + OTX_CPT_VQX_MISC_ENA_W1S(0)); +} + +static void cptvf_enable_mbox_interrupts(struct otx_cptvf *cptvf) +{ + union otx_cptx_vqx_misc_ena_w1s vqx_misc_ena; + + vqx_misc_ena.u = readq(cptvf->reg_base + OTX_CPT_VQX_MISC_ENA_W1S(0)); + /* Enable MBOX interrupt for the requested VF */ + vqx_misc_ena.s.mbox = 1; + writeq(vqx_misc_ena.u, cptvf->reg_base + OTX_CPT_VQX_MISC_ENA_W1S(0)); +} + +static void cptvf_enable_done_interrupts(struct otx_cptvf *cptvf) +{ + union otx_cptx_vqx_done_ena_w1s vqx_done_ena; + + vqx_done_ena.u = readq(cptvf->reg_base + OTX_CPT_VQX_DONE_ENA_W1S(0)); + /* Enable DONE interrupt for the requested VF */ + vqx_done_ena.s.done = 1; + writeq(vqx_done_ena.u, cptvf->reg_base + OTX_CPT_VQX_DONE_ENA_W1S(0)); +} + +static void cptvf_clear_dovf_intr(struct otx_cptvf *cptvf) +{ + union otx_cptx_vqx_misc_int vqx_misc_int; + + vqx_misc_int.u = readq(cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0)); + /* W1C for the VF */ + vqx_misc_int.s.dovf = 1; + writeq(vqx_misc_int.u, cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0)); +} + +static void cptvf_clear_irde_intr(struct otx_cptvf *cptvf) +{ + union otx_cptx_vqx_misc_int vqx_misc_int; + + vqx_misc_int.u = readq(cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0)); + /* W1C for the VF */ + vqx_misc_int.s.irde = 1; + writeq(vqx_misc_int.u, cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0)); +} + +static void cptvf_clear_nwrp_intr(struct otx_cptvf *cptvf) +{ + union otx_cptx_vqx_misc_int vqx_misc_int; + + vqx_misc_int.u = readq(cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0)); + /* W1C for the VF */ + vqx_misc_int.s.nwrp = 1; + writeq(vqx_misc_int.u, cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0)); +} + +static void cptvf_clear_mbox_intr(struct otx_cptvf *cptvf) +{ + union otx_cptx_vqx_misc_int vqx_misc_int; + + vqx_misc_int.u = readq(cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0)); + /* W1C for the VF */ + vqx_misc_int.s.mbox = 1; + writeq(vqx_misc_int.u, cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0)); +} + +static void cptvf_clear_swerr_intr(struct otx_cptvf *cptvf) +{ + union otx_cptx_vqx_misc_int vqx_misc_int; + + vqx_misc_int.u = readq(cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0)); + /* W1C for the VF */ + vqx_misc_int.s.swerr = 1; + writeq(vqx_misc_int.u, cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0)); +} + +static u64 cptvf_read_vf_misc_intr_status(struct otx_cptvf *cptvf) +{ + return readq(cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0)); +} + +static irqreturn_t cptvf_misc_intr_handler(int __always_unused irq, + void *arg) +{ + struct otx_cptvf *cptvf = arg; + struct pci_dev *pdev = cptvf->pdev; + u64 intr; + + intr = cptvf_read_vf_misc_intr_status(cptvf); + /* Check for MISC interrupt types */ + if (likely(intr & OTX_CPT_VF_INTR_MBOX_MASK)) { + dev_dbg(&pdev->dev, "Mailbox interrupt 0x%llx on CPT VF %d\n", + intr, cptvf->vfid); + otx_cptvf_handle_mbox_intr(cptvf); + cptvf_clear_mbox_intr(cptvf); + } else if (unlikely(intr & OTX_CPT_VF_INTR_DOVF_MASK)) { + cptvf_clear_dovf_intr(cptvf); + /* Clear doorbell count */ + otx_cptvf_write_vq_doorbell(cptvf, 0); + dev_err(&pdev->dev, + "Doorbell overflow error interrupt 0x%llx on CPT VF %d\n", + intr, cptvf->vfid); + } else if (unlikely(intr & OTX_CPT_VF_INTR_IRDE_MASK)) { + cptvf_clear_irde_intr(cptvf); + dev_err(&pdev->dev, + "Instruction NCB read error interrupt 0x%llx on CPT VF %d\n", + intr, cptvf->vfid); + } else if (unlikely(intr & OTX_CPT_VF_INTR_NWRP_MASK)) { + cptvf_clear_nwrp_intr(cptvf); + dev_err(&pdev->dev, + "NCB response write error interrupt 0x%llx on CPT VF %d\n", + intr, cptvf->vfid); + } else if (unlikely(intr & OTX_CPT_VF_INTR_SERR_MASK)) { + cptvf_clear_swerr_intr(cptvf); + dev_err(&pdev->dev, + "Software error interrupt 0x%llx on CPT VF %d\n", + intr, cptvf->vfid); + } else { + dev_err(&pdev->dev, "Unhandled interrupt in OTX_CPT VF %d\n", + cptvf->vfid); + } + + return IRQ_HANDLED; +} + +static inline struct otx_cptvf_wqe *get_cptvf_vq_wqe(struct otx_cptvf *cptvf, + int qno) +{ + struct otx_cptvf_wqe_info *nwqe_info; + + if (unlikely(qno >= cptvf->num_queues)) + return NULL; + nwqe_info = (struct otx_cptvf_wqe_info *)cptvf->wqe_info; + + return &nwqe_info->vq_wqe[qno]; +} + +static inline u32 cptvf_read_vq_done_count(struct otx_cptvf *cptvf) +{ + union otx_cptx_vqx_done vqx_done; + + vqx_done.u = readq(cptvf->reg_base + OTX_CPT_VQX_DONE(0)); + return vqx_done.s.done; +} + +static inline void cptvf_write_vq_done_ack(struct otx_cptvf *cptvf, + u32 ackcnt) +{ + union otx_cptx_vqx_done_ack vqx_dack_cnt; + + vqx_dack_cnt.u = readq(cptvf->reg_base + OTX_CPT_VQX_DONE_ACK(0)); + vqx_dack_cnt.s.done_ack = ackcnt; + writeq(vqx_dack_cnt.u, cptvf->reg_base + OTX_CPT_VQX_DONE_ACK(0)); +} + +static irqreturn_t cptvf_done_intr_handler(int __always_unused irq, + void *cptvf_dev) +{ + struct otx_cptvf *cptvf = (struct otx_cptvf *)cptvf_dev; + struct pci_dev *pdev = cptvf->pdev; + /* Read the number of completions */ + u32 intr = cptvf_read_vq_done_count(cptvf); + + if (intr) { + struct otx_cptvf_wqe *wqe; + + /* + * Acknowledge the number of scheduled completions for + * processing + */ + cptvf_write_vq_done_ack(cptvf, intr); + wqe = get_cptvf_vq_wqe(cptvf, 0); + if (unlikely(!wqe)) { + dev_err(&pdev->dev, "No work to schedule for VF (%d)\n", + cptvf->vfid); + return IRQ_NONE; + } + tasklet_hi_schedule(&wqe->twork); + } + + return IRQ_HANDLED; +} + +static void cptvf_set_irq_affinity(struct otx_cptvf *cptvf, int vec) +{ + struct pci_dev *pdev = cptvf->pdev; + int cpu; + + if (!zalloc_cpumask_var(&cptvf->affinity_mask[vec], + GFP_KERNEL)) { + dev_err(&pdev->dev, + "Allocation failed for affinity_mask for VF %d\n", + cptvf->vfid); + return; + } + + cpu = cptvf->vfid % num_online_cpus(); + cpumask_set_cpu(cpumask_local_spread(cpu, cptvf->node), + cptvf->affinity_mask[vec]); + irq_set_affinity_hint(pci_irq_vector(pdev, vec), + cptvf->affinity_mask[vec]); +} + +static void cptvf_write_vq_saddr(struct otx_cptvf *cptvf, u64 val) +{ + union otx_cptx_vqx_saddr vqx_saddr; + + vqx_saddr.u = val; + writeq(vqx_saddr.u, cptvf->reg_base + OTX_CPT_VQX_SADDR(0)); +} + +static void cptvf_device_init(struct otx_cptvf *cptvf) +{ + u64 base_addr = 0; + + /* Disable the VQ */ + cptvf_write_vq_ctl(cptvf, 0); + /* Reset the doorbell */ + otx_cptvf_write_vq_doorbell(cptvf, 0); + /* Clear inflight */ + cptvf_write_vq_inprog(cptvf, 0); + /* Write VQ SADDR */ + base_addr = (u64)(cptvf->cqinfo.queue[0].qhead->dma_addr); + cptvf_write_vq_saddr(cptvf, base_addr); + /* Configure timerhold / coalescence */ + cptvf_write_vq_done_timewait(cptvf, OTX_CPT_TIMER_HOLD); + cptvf_write_vq_done_numwait(cptvf, OTX_CPT_COUNT_HOLD); + /* Enable the VQ */ + cptvf_write_vq_ctl(cptvf, 1); + /* Flag the VF ready */ + cptvf->flags |= OTX_CPT_FLAG_DEVICE_READY; +} + +static ssize_t vf_type_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct otx_cptvf *cptvf = dev_get_drvdata(dev); + char *msg; + + switch (cptvf->vftype) { + case OTX_CPT_AE_TYPES: + msg = "AE"; + break; + + case OTX_CPT_SE_TYPES: + msg = "SE"; + break; + + default: + msg = "Invalid"; + } + + return sysfs_emit(buf, "%s\n", msg); +} + +static ssize_t vf_engine_group_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct otx_cptvf *cptvf = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%d\n", cptvf->vfgrp); +} + +static ssize_t vf_engine_group_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct otx_cptvf *cptvf = dev_get_drvdata(dev); + int val, ret; + + ret = kstrtoint(buf, 10, &val); + if (ret) + return ret; + + if (val < 0) + return -EINVAL; + + if (val >= OTX_CPT_MAX_ENGINE_GROUPS) { + dev_err(dev, "Engine group >= than max available groups %d\n", + OTX_CPT_MAX_ENGINE_GROUPS); + return -EINVAL; + } + + ret = otx_cptvf_send_vf_to_grp_msg(cptvf, val); + if (ret) + return ret; + + return count; +} + +static ssize_t vf_coalesc_time_wait_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct otx_cptvf *cptvf = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%d\n", + cptvf_read_vq_done_timewait(cptvf)); +} + +static ssize_t vf_coalesc_num_wait_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct otx_cptvf *cptvf = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%d\n", + cptvf_read_vq_done_numwait(cptvf)); +} + +static ssize_t vf_coalesc_time_wait_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct otx_cptvf *cptvf = dev_get_drvdata(dev); + long val; + int ret; + + ret = kstrtol(buf, 10, &val); + if (ret != 0) + return ret; + + if (val < OTX_CPT_COALESC_MIN_TIME_WAIT || + val > OTX_CPT_COALESC_MAX_TIME_WAIT) + return -EINVAL; + + cptvf_write_vq_done_timewait(cptvf, val); + return count; +} + +static ssize_t vf_coalesc_num_wait_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct otx_cptvf *cptvf = dev_get_drvdata(dev); + long val; + int ret; + + ret = kstrtol(buf, 10, &val); + if (ret != 0) + return ret; + + if (val < OTX_CPT_COALESC_MIN_NUM_WAIT || + val > OTX_CPT_COALESC_MAX_NUM_WAIT) + return -EINVAL; + + cptvf_write_vq_done_numwait(cptvf, val); + return count; +} + +static DEVICE_ATTR_RO(vf_type); +static DEVICE_ATTR_RW(vf_engine_group); +static DEVICE_ATTR_RW(vf_coalesc_time_wait); +static DEVICE_ATTR_RW(vf_coalesc_num_wait); + +static struct attribute *otx_cptvf_attrs[] = { + &dev_attr_vf_type.attr, + &dev_attr_vf_engine_group.attr, + &dev_attr_vf_coalesc_time_wait.attr, + &dev_attr_vf_coalesc_num_wait.attr, + NULL +}; + +static const struct attribute_group otx_cptvf_sysfs_group = { + .attrs = otx_cptvf_attrs, +}; + +static int otx_cptvf_probe(struct pci_dev *pdev, + const struct pci_device_id *ent) +{ + struct device *dev = &pdev->dev; + struct otx_cptvf *cptvf; + int err; + + cptvf = devm_kzalloc(dev, sizeof(*cptvf), GFP_KERNEL); + if (!cptvf) + return -ENOMEM; + + pci_set_drvdata(pdev, cptvf); + cptvf->pdev = pdev; + + err = pci_enable_device(pdev); + if (err) { + dev_err(dev, "Failed to enable PCI device\n"); + goto clear_drvdata; + } + err = pci_request_regions(pdev, DRV_NAME); + if (err) { + dev_err(dev, "PCI request regions failed 0x%x\n", err); + goto disable_device; + } + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); + if (err) { + dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); + goto release_regions; + } + + /* MAP PF's configuration registers */ + cptvf->reg_base = pci_iomap(pdev, OTX_CPT_VF_PCI_CFG_BAR, 0); + if (!cptvf->reg_base) { + dev_err(dev, "Cannot map config register space, aborting\n"); + err = -ENOMEM; + goto release_regions; + } + + cptvf->node = dev_to_node(&pdev->dev); + err = pci_alloc_irq_vectors(pdev, OTX_CPT_VF_MSIX_VECTORS, + OTX_CPT_VF_MSIX_VECTORS, PCI_IRQ_MSIX); + if (err < 0) { + dev_err(dev, "Request for #%d msix vectors failed\n", + OTX_CPT_VF_MSIX_VECTORS); + goto unmap_region; + } + + err = request_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), + cptvf_misc_intr_handler, 0, "CPT VF misc intr", + cptvf); + if (err) { + dev_err(dev, "Failed to request misc irq\n"); + goto free_vectors; + } + + /* Enable mailbox interrupt */ + cptvf_enable_mbox_interrupts(cptvf); + cptvf_enable_swerr_interrupts(cptvf); + + /* Check cpt pf status, gets chip ID / device Id from PF if ready */ + err = otx_cptvf_check_pf_ready(cptvf); + if (err) + goto free_misc_irq; + + /* CPT VF software resources initialization */ + cptvf->cqinfo.qchunksize = OTX_CPT_CMD_QCHUNK_SIZE; + err = cptvf_sw_init(cptvf, OTX_CPT_CMD_QLEN, OTX_CPT_NUM_QS_PER_VF); + if (err) { + dev_err(dev, "cptvf_sw_init() failed\n"); + goto free_misc_irq; + } + /* Convey VQ LEN to PF */ + err = otx_cptvf_send_vq_size_msg(cptvf); + if (err) + goto sw_cleanup; + + /* CPT VF device initialization */ + cptvf_device_init(cptvf); + /* Send msg to PF to assign currnet Q to required group */ + err = otx_cptvf_send_vf_to_grp_msg(cptvf, cptvf->vfgrp); + if (err) + goto sw_cleanup; + + cptvf->priority = 1; + err = otx_cptvf_send_vf_priority_msg(cptvf); + if (err) + goto sw_cleanup; + + err = request_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE), + cptvf_done_intr_handler, 0, "CPT VF done intr", + cptvf); + if (err) { + dev_err(dev, "Failed to request done irq\n"); + goto free_done_irq; + } + + /* Enable done interrupt */ + cptvf_enable_done_interrupts(cptvf); + + /* Set irq affinity masks */ + cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC); + cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE); + + err = otx_cptvf_send_vf_up(cptvf); + if (err) + goto free_irq_affinity; + + /* Initialize algorithms and set ops */ + err = otx_cpt_crypto_init(pdev, THIS_MODULE, + cptvf->vftype == OTX_CPT_SE_TYPES ? OTX_CPT_SE : OTX_CPT_AE, + cptvf->vftype, 1, cptvf->num_vfs); + if (err) { + dev_err(dev, "Failed to register crypto algs\n"); + goto free_irq_affinity; + } + + err = sysfs_create_group(&dev->kobj, &otx_cptvf_sysfs_group); + if (err) { + dev_err(dev, "Creating sysfs entries failed\n"); + goto crypto_exit; + } + + return 0; + +crypto_exit: + otx_cpt_crypto_exit(pdev, THIS_MODULE, cptvf->vftype); +free_irq_affinity: + cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE); + cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC); +free_done_irq: + free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE), cptvf); +sw_cleanup: + cptvf_sw_cleanup(cptvf); +free_misc_irq: + free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf); +free_vectors: + pci_free_irq_vectors(cptvf->pdev); +unmap_region: + pci_iounmap(pdev, cptvf->reg_base); +release_regions: + pci_release_regions(pdev); +disable_device: + pci_disable_device(pdev); +clear_drvdata: + pci_set_drvdata(pdev, NULL); + + return err; +} + +static void otx_cptvf_remove(struct pci_dev *pdev) +{ + struct otx_cptvf *cptvf = pci_get_drvdata(pdev); + + if (!cptvf) { + dev_err(&pdev->dev, "Invalid CPT-VF device\n"); + return; + } + + /* Convey DOWN to PF */ + if (otx_cptvf_send_vf_down(cptvf)) { + dev_err(&pdev->dev, "PF not responding to DOWN msg\n"); + } else { + sysfs_remove_group(&pdev->dev.kobj, &otx_cptvf_sysfs_group); + otx_cpt_crypto_exit(pdev, THIS_MODULE, cptvf->vftype); + cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE); + cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC); + free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE), cptvf); + free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf); + cptvf_sw_cleanup(cptvf); + pci_free_irq_vectors(cptvf->pdev); + pci_iounmap(pdev, cptvf->reg_base); + pci_release_regions(pdev); + pci_disable_device(pdev); + pci_set_drvdata(pdev, NULL); + } +} + +/* Supported devices */ +static const struct pci_device_id otx_cptvf_id_table[] = { + { PCI_VDEVICE(CAVIUM, OTX_CPT_PCI_VF_DEVICE_ID) }, + { } /* end of table */ +}; +MODULE_DEVICE_TABLE(pci, otx_cptvf_id_table); + +static struct pci_driver otx_cptvf_pci_driver = { + .name = DRV_NAME, + .id_table = otx_cptvf_id_table, + .probe = otx_cptvf_probe, + .remove = otx_cptvf_remove, +}; + +module_pci_driver(otx_cptvf_pci_driver); + +MODULE_AUTHOR("Marvell International Ltd."); +MODULE_DESCRIPTION("Marvell OcteonTX CPT Virtual Function Driver"); +MODULE_LICENSE("GPL v2"); +MODULE_VERSION(DRV_VERSION); diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_mbox.c b/drivers/crypto/marvell/octeontx/otx_cptvf_mbox.c new file mode 100644 index 000000000..90fdafb7c --- /dev/null +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_mbox.c @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell OcteonTX CPT driver + * + * Copyright (C) 2019 Marvell International Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include "otx_cptvf.h" + +#define CPT_MBOX_MSG_TIMEOUT 2000 + +static char *get_mbox_opcode_str(int msg_opcode) +{ + char *str = "Unknown"; + + switch (msg_opcode) { + case OTX_CPT_MSG_VF_UP: + str = "UP"; + break; + + case OTX_CPT_MSG_VF_DOWN: + str = "DOWN"; + break; + + case OTX_CPT_MSG_READY: + str = "READY"; + break; + + case OTX_CPT_MSG_QLEN: + str = "QLEN"; + break; + + case OTX_CPT_MSG_QBIND_GRP: + str = "QBIND_GRP"; + break; + + case OTX_CPT_MSG_VQ_PRIORITY: + str = "VQ_PRIORITY"; + break; + + case OTX_CPT_MSG_PF_TYPE: + str = "PF_TYPE"; + break; + + case OTX_CPT_MSG_ACK: + str = "ACK"; + break; + + case OTX_CPT_MSG_NACK: + str = "NACK"; + break; + } + return str; +} + +static void dump_mbox_msg(struct otx_cpt_mbox *mbox_msg, int vf_id) +{ + char raw_data_str[OTX_CPT_MAX_MBOX_DATA_STR_SIZE]; + + hex_dump_to_buffer(mbox_msg, sizeof(struct otx_cpt_mbox), 16, 8, + raw_data_str, OTX_CPT_MAX_MBOX_DATA_STR_SIZE, false); + if (vf_id >= 0) + pr_debug("MBOX msg %s received from VF%d raw_data %s", + get_mbox_opcode_str(mbox_msg->msg), vf_id, + raw_data_str); + else + pr_debug("MBOX msg %s received from PF raw_data %s", + get_mbox_opcode_str(mbox_msg->msg), raw_data_str); +} + +static void cptvf_send_msg_to_pf(struct otx_cptvf *cptvf, + struct otx_cpt_mbox *mbx) +{ + /* Writing mbox(1) causes interrupt */ + writeq(mbx->msg, cptvf->reg_base + OTX_CPT_VFX_PF_MBOXX(0, 0)); + writeq(mbx->data, cptvf->reg_base + OTX_CPT_VFX_PF_MBOXX(0, 1)); +} + +/* Interrupt handler to handle mailbox messages from VFs */ +void otx_cptvf_handle_mbox_intr(struct otx_cptvf *cptvf) +{ + struct otx_cpt_mbox mbx = {}; + + /* + * MBOX[0] contains msg + * MBOX[1] contains data + */ + mbx.msg = readq(cptvf->reg_base + OTX_CPT_VFX_PF_MBOXX(0, 0)); + mbx.data = readq(cptvf->reg_base + OTX_CPT_VFX_PF_MBOXX(0, 1)); + + dump_mbox_msg(&mbx, -1); + + switch (mbx.msg) { + case OTX_CPT_MSG_VF_UP: + cptvf->pf_acked = true; + cptvf->num_vfs = mbx.data; + break; + case OTX_CPT_MSG_READY: + cptvf->pf_acked = true; + cptvf->vfid = mbx.data; + dev_dbg(&cptvf->pdev->dev, "Received VFID %d\n", cptvf->vfid); + break; + case OTX_CPT_MSG_QBIND_GRP: + cptvf->pf_acked = true; + cptvf->vftype = mbx.data; + dev_dbg(&cptvf->pdev->dev, "VF %d type %s group %d\n", + cptvf->vfid, + ((mbx.data == OTX_CPT_SE_TYPES) ? "SE" : "AE"), + cptvf->vfgrp); + break; + case OTX_CPT_MSG_ACK: + cptvf->pf_acked = true; + break; + case OTX_CPT_MSG_NACK: + cptvf->pf_nacked = true; + break; + default: + dev_err(&cptvf->pdev->dev, "Invalid msg from PF, msg 0x%llx\n", + mbx.msg); + break; + } +} + +static int cptvf_send_msg_to_pf_timeout(struct otx_cptvf *cptvf, + struct otx_cpt_mbox *mbx) +{ + int timeout = CPT_MBOX_MSG_TIMEOUT; + int sleep = 10; + + cptvf->pf_acked = false; + cptvf->pf_nacked = false; + cptvf_send_msg_to_pf(cptvf, mbx); + /* Wait for previous message to be acked, timeout 2sec */ + while (!cptvf->pf_acked) { + if (cptvf->pf_nacked) + return -EINVAL; + msleep(sleep); + if (cptvf->pf_acked) + break; + timeout -= sleep; + if (!timeout) { + dev_err(&cptvf->pdev->dev, + "PF didn't ack to mbox msg %llx from VF%u\n", + mbx->msg, cptvf->vfid); + return -EBUSY; + } + } + return 0; +} + +/* + * Checks if VF is able to comminicate with PF + * and also gets the CPT number this VF is associated to. + */ +int otx_cptvf_check_pf_ready(struct otx_cptvf *cptvf) +{ + struct otx_cpt_mbox mbx = {}; + + mbx.msg = OTX_CPT_MSG_READY; + + return cptvf_send_msg_to_pf_timeout(cptvf, &mbx); +} + +/* + * Communicate VQs size to PF to program CPT(0)_PF_Q(0-15)_CTL of the VF. + * Must be ACKed. + */ +int otx_cptvf_send_vq_size_msg(struct otx_cptvf *cptvf) +{ + struct otx_cpt_mbox mbx = {}; + + mbx.msg = OTX_CPT_MSG_QLEN; + mbx.data = cptvf->qsize; + + return cptvf_send_msg_to_pf_timeout(cptvf, &mbx); +} + +/* + * Communicate VF group required to PF and get the VQ binded to that group + */ +int otx_cptvf_send_vf_to_grp_msg(struct otx_cptvf *cptvf, int group) +{ + struct otx_cpt_mbox mbx = {}; + int ret; + + mbx.msg = OTX_CPT_MSG_QBIND_GRP; + /* Convey group of the VF */ + mbx.data = group; + ret = cptvf_send_msg_to_pf_timeout(cptvf, &mbx); + if (ret) + return ret; + cptvf->vfgrp = group; + + return 0; +} + +/* + * Communicate VF group required to PF and get the VQ binded to that group + */ +int otx_cptvf_send_vf_priority_msg(struct otx_cptvf *cptvf) +{ + struct otx_cpt_mbox mbx = {}; + + mbx.msg = OTX_CPT_MSG_VQ_PRIORITY; + /* Convey group of the VF */ + mbx.data = cptvf->priority; + + return cptvf_send_msg_to_pf_timeout(cptvf, &mbx); +} + +/* + * Communicate to PF that VF is UP and running + */ +int otx_cptvf_send_vf_up(struct otx_cptvf *cptvf) +{ + struct otx_cpt_mbox mbx = {}; + + mbx.msg = OTX_CPT_MSG_VF_UP; + + return cptvf_send_msg_to_pf_timeout(cptvf, &mbx); +} + +/* + * Communicate to PF that VF is DOWN and running + */ +int otx_cptvf_send_vf_down(struct otx_cptvf *cptvf) +{ + struct otx_cpt_mbox mbx = {}; + + mbx.msg = OTX_CPT_MSG_VF_DOWN; + + return cptvf_send_msg_to_pf_timeout(cptvf, &mbx); +} diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_reqmgr.c b/drivers/crypto/marvell/octeontx/otx_cptvf_reqmgr.c new file mode 100644 index 000000000..89030e271 --- /dev/null +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_reqmgr.c @@ -0,0 +1,609 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell OcteonTX CPT driver + * + * Copyright (C) 2019 Marvell International Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include "otx_cptvf.h" +#include "otx_cptvf_algs.h" + +/* Completion code size and initial value */ +#define COMPLETION_CODE_SIZE 8 +#define COMPLETION_CODE_INIT 0 + +/* SG list header size in bytes */ +#define SG_LIST_HDR_SIZE 8 + +/* Default timeout when waiting for free pending entry in us */ +#define CPT_PENTRY_TIMEOUT 1000 +#define CPT_PENTRY_STEP 50 + +/* Default threshold for stopping and resuming sender requests */ +#define CPT_IQ_STOP_MARGIN 128 +#define CPT_IQ_RESUME_MARGIN 512 + +#define CPT_DMA_ALIGN 128 + +void otx_cpt_dump_sg_list(struct pci_dev *pdev, struct otx_cpt_req_info *req) +{ + int i; + + pr_debug("Gather list size %d\n", req->incnt); + for (i = 0; i < req->incnt; i++) { + pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%p\n", i, + req->in[i].size, req->in[i].vptr, + (void *) req->in[i].dma_addr); + pr_debug("Buffer hexdump (%d bytes)\n", + req->in[i].size); + print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, + req->in[i].vptr, req->in[i].size, false); + } + + pr_debug("Scatter list size %d\n", req->outcnt); + for (i = 0; i < req->outcnt; i++) { + pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%p\n", i, + req->out[i].size, req->out[i].vptr, + (void *) req->out[i].dma_addr); + pr_debug("Buffer hexdump (%d bytes)\n", req->out[i].size); + print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, + req->out[i].vptr, req->out[i].size, false); + } +} + +static inline struct otx_cpt_pending_entry *get_free_pending_entry( + struct otx_cpt_pending_queue *q, + int qlen) +{ + struct otx_cpt_pending_entry *ent = NULL; + + ent = &q->head[q->rear]; + if (unlikely(ent->busy)) + return NULL; + + q->rear++; + if (unlikely(q->rear == qlen)) + q->rear = 0; + + return ent; +} + +static inline u32 modulo_inc(u32 index, u32 length, u32 inc) +{ + if (WARN_ON(inc > length)) + inc = length; + + index += inc; + if (unlikely(index >= length)) + index -= length; + + return index; +} + +static inline void free_pentry(struct otx_cpt_pending_entry *pentry) +{ + pentry->completion_addr = NULL; + pentry->info = NULL; + pentry->callback = NULL; + pentry->areq = NULL; + pentry->resume_sender = false; + pentry->busy = false; +} + +static inline int setup_sgio_components(struct pci_dev *pdev, + struct otx_cpt_buf_ptr *list, + int buf_count, u8 *buffer) +{ + struct otx_cpt_sglist_component *sg_ptr = NULL; + int ret = 0, i, j; + int components; + + if (unlikely(!list)) { + dev_err(&pdev->dev, "Input list pointer is NULL\n"); + return -EFAULT; + } + + for (i = 0; i < buf_count; i++) { + if (likely(list[i].vptr)) { + list[i].dma_addr = dma_map_single(&pdev->dev, + list[i].vptr, + list[i].size, + DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(&pdev->dev, + list[i].dma_addr))) { + dev_err(&pdev->dev, "Dma mapping failed\n"); + ret = -EIO; + goto sg_cleanup; + } + } + } + + components = buf_count / 4; + sg_ptr = (struct otx_cpt_sglist_component *)buffer; + for (i = 0; i < components; i++) { + sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size); + sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size); + sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size); + sg_ptr->u.s.len3 = cpu_to_be16(list[i * 4 + 3].size); + sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr); + sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr); + sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr); + sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr); + sg_ptr++; + } + components = buf_count % 4; + + switch (components) { + case 3: + sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size); + sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr); + fallthrough; + case 2: + sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size); + sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr); + fallthrough; + case 1: + sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size); + sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr); + break; + default: + break; + } + return ret; + +sg_cleanup: + for (j = 0; j < i; j++) { + if (list[j].dma_addr) { + dma_unmap_single(&pdev->dev, list[j].dma_addr, + list[j].size, DMA_BIDIRECTIONAL); + } + + list[j].dma_addr = 0; + } + return ret; +} + +static inline int setup_sgio_list(struct pci_dev *pdev, + struct otx_cpt_info_buffer **pinfo, + struct otx_cpt_req_info *req, gfp_t gfp) +{ + u32 dlen, align_dlen, info_len, rlen; + struct otx_cpt_info_buffer *info; + u16 g_sz_bytes, s_sz_bytes; + int align = CPT_DMA_ALIGN; + u32 total_mem_len; + + if (unlikely(req->incnt > OTX_CPT_MAX_SG_IN_CNT || + req->outcnt > OTX_CPT_MAX_SG_OUT_CNT)) { + dev_err(&pdev->dev, "Error too many sg components\n"); + return -EINVAL; + } + + g_sz_bytes = ((req->incnt + 3) / 4) * + sizeof(struct otx_cpt_sglist_component); + s_sz_bytes = ((req->outcnt + 3) / 4) * + sizeof(struct otx_cpt_sglist_component); + + dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE; + align_dlen = ALIGN(dlen, align); + info_len = ALIGN(sizeof(*info), align); + rlen = ALIGN(sizeof(union otx_cpt_res_s), align); + total_mem_len = align_dlen + info_len + rlen + COMPLETION_CODE_SIZE; + + info = kzalloc(total_mem_len, gfp); + if (unlikely(!info)) { + dev_err(&pdev->dev, "Memory allocation failed\n"); + return -ENOMEM; + } + *pinfo = info; + info->dlen = dlen; + info->in_buffer = (u8 *)info + info_len; + + ((__be16 *)info->in_buffer)[0] = cpu_to_be16(req->outcnt); + ((__be16 *)info->in_buffer)[1] = cpu_to_be16(req->incnt); + ((u16 *)info->in_buffer)[2] = 0; + ((u16 *)info->in_buffer)[3] = 0; + + /* Setup gather (input) components */ + if (setup_sgio_components(pdev, req->in, req->incnt, + &info->in_buffer[8])) { + dev_err(&pdev->dev, "Failed to setup gather list\n"); + return -EFAULT; + } + + if (setup_sgio_components(pdev, req->out, req->outcnt, + &info->in_buffer[8 + g_sz_bytes])) { + dev_err(&pdev->dev, "Failed to setup scatter list\n"); + return -EFAULT; + } + + info->dma_len = total_mem_len - info_len; + info->dptr_baddr = dma_map_single(&pdev->dev, (void *)info->in_buffer, + info->dma_len, DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(&pdev->dev, info->dptr_baddr))) { + dev_err(&pdev->dev, "DMA Mapping failed for cpt req\n"); + return -EIO; + } + /* + * Get buffer for union otx_cpt_res_s response + * structure and its physical address + */ + info->completion_addr = (u64 *)(info->in_buffer + align_dlen); + info->comp_baddr = info->dptr_baddr + align_dlen; + + /* Create and initialize RPTR */ + info->out_buffer = (u8 *)info->completion_addr + rlen; + info->rptr_baddr = info->comp_baddr + rlen; + + *((u64 *) info->out_buffer) = ~((u64) COMPLETION_CODE_INIT); + + return 0; +} + + +static void cpt_fill_inst(union otx_cpt_inst_s *inst, + struct otx_cpt_info_buffer *info, + struct otx_cpt_iq_cmd *cmd) +{ + inst->u[0] = 0x0; + inst->s.doneint = true; + inst->s.res_addr = (u64)info->comp_baddr; + inst->u[2] = 0x0; + inst->s.wq_ptr = 0; + inst->s.ei0 = cmd->cmd.u64; + inst->s.ei1 = cmd->dptr; + inst->s.ei2 = cmd->rptr; + inst->s.ei3 = cmd->cptr.u64; +} + +/* + * On OcteonTX platform the parameter db_count is used as a count for ringing + * door bell. The valid values for db_count are: + * 0 - 1 CPT instruction will be enqueued however CPT will not be informed + * 1 - 1 CPT instruction will be enqueued and CPT will be informed + */ +static void cpt_send_cmd(union otx_cpt_inst_s *cptinst, struct otx_cptvf *cptvf) +{ + struct otx_cpt_cmd_qinfo *qinfo = &cptvf->cqinfo; + struct otx_cpt_cmd_queue *queue; + struct otx_cpt_cmd_chunk *curr; + u8 *ent; + + queue = &qinfo->queue[0]; + /* + * cpt_send_cmd is currently called only from critical section + * therefore no locking is required for accessing instruction queue + */ + ent = &queue->qhead->head[queue->idx * OTX_CPT_INST_SIZE]; + memcpy(ent, (void *) cptinst, OTX_CPT_INST_SIZE); + + if (++queue->idx >= queue->qhead->size / 64) { + curr = queue->qhead; + + if (list_is_last(&curr->nextchunk, &queue->chead)) + queue->qhead = queue->base; + else + queue->qhead = list_next_entry(queue->qhead, nextchunk); + queue->idx = 0; + } + /* make sure all memory stores are done before ringing doorbell */ + smp_wmb(); + otx_cptvf_write_vq_doorbell(cptvf, 1); +} + +static int process_request(struct pci_dev *pdev, struct otx_cpt_req_info *req, + struct otx_cpt_pending_queue *pqueue, + struct otx_cptvf *cptvf) +{ + struct otx_cptvf_request *cpt_req = &req->req; + struct otx_cpt_pending_entry *pentry = NULL; + union otx_cpt_ctrl_info *ctrl = &req->ctrl; + struct otx_cpt_info_buffer *info = NULL; + union otx_cpt_res_s *result = NULL; + struct otx_cpt_iq_cmd iq_cmd; + union otx_cpt_inst_s cptinst; + int retry, ret = 0; + u8 resume_sender; + gfp_t gfp; + + gfp = (req->areq->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : + GFP_ATOMIC; + ret = setup_sgio_list(pdev, &info, req, gfp); + if (unlikely(ret)) { + dev_err(&pdev->dev, "Setting up SG list failed\n"); + goto request_cleanup; + } + cpt_req->dlen = info->dlen; + + result = (union otx_cpt_res_s *) info->completion_addr; + result->s.compcode = COMPLETION_CODE_INIT; + + spin_lock_bh(&pqueue->lock); + pentry = get_free_pending_entry(pqueue, pqueue->qlen); + retry = CPT_PENTRY_TIMEOUT / CPT_PENTRY_STEP; + while (unlikely(!pentry) && retry--) { + spin_unlock_bh(&pqueue->lock); + udelay(CPT_PENTRY_STEP); + spin_lock_bh(&pqueue->lock); + pentry = get_free_pending_entry(pqueue, pqueue->qlen); + } + + if (unlikely(!pentry)) { + ret = -ENOSPC; + spin_unlock_bh(&pqueue->lock); + goto request_cleanup; + } + + /* + * Check if we are close to filling in entire pending queue, + * if so then tell the sender to stop/sleep by returning -EBUSY + * We do it only for context which can sleep (GFP_KERNEL) + */ + if (gfp == GFP_KERNEL && + pqueue->pending_count > (pqueue->qlen - CPT_IQ_STOP_MARGIN)) { + pentry->resume_sender = true; + } else + pentry->resume_sender = false; + resume_sender = pentry->resume_sender; + pqueue->pending_count++; + + pentry->completion_addr = info->completion_addr; + pentry->info = info; + pentry->callback = req->callback; + pentry->areq = req->areq; + pentry->busy = true; + info->pentry = pentry; + info->time_in = jiffies; + info->req = req; + + /* Fill in the command */ + iq_cmd.cmd.u64 = 0; + iq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags); + iq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1); + iq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2); + iq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen); + + iq_cmd.dptr = info->dptr_baddr; + iq_cmd.rptr = info->rptr_baddr; + iq_cmd.cptr.u64 = 0; + iq_cmd.cptr.s.grp = ctrl->s.grp; + + /* Fill in the CPT_INST_S type command for HW interpretation */ + cpt_fill_inst(&cptinst, info, &iq_cmd); + + /* Print debug info if enabled */ + otx_cpt_dump_sg_list(pdev, req); + pr_debug("Cpt_inst_s hexdump (%d bytes)\n", OTX_CPT_INST_SIZE); + print_hex_dump_debug("", 0, 16, 1, &cptinst, OTX_CPT_INST_SIZE, false); + pr_debug("Dptr hexdump (%d bytes)\n", cpt_req->dlen); + print_hex_dump_debug("", 0, 16, 1, info->in_buffer, + cpt_req->dlen, false); + + /* Send CPT command */ + cpt_send_cmd(&cptinst, cptvf); + + /* + * We allocate and prepare pending queue entry in critical section + * together with submitting CPT instruction to CPT instruction queue + * to make sure that order of CPT requests is the same in both + * pending and instruction queues + */ + spin_unlock_bh(&pqueue->lock); + + ret = resume_sender ? -EBUSY : -EINPROGRESS; + return ret; + +request_cleanup: + do_request_cleanup(pdev, info); + return ret; +} + +int otx_cpt_do_request(struct pci_dev *pdev, struct otx_cpt_req_info *req, + int cpu_num) +{ + struct otx_cptvf *cptvf = pci_get_drvdata(pdev); + + if (!otx_cpt_device_ready(cptvf)) { + dev_err(&pdev->dev, "CPT Device is not ready\n"); + return -ENODEV; + } + + if ((cptvf->vftype == OTX_CPT_SE_TYPES) && (!req->ctrl.s.se_req)) { + dev_err(&pdev->dev, "CPTVF-%d of SE TYPE got AE request\n", + cptvf->vfid); + return -EINVAL; + } else if ((cptvf->vftype == OTX_CPT_AE_TYPES) && + (req->ctrl.s.se_req)) { + dev_err(&pdev->dev, "CPTVF-%d of AE TYPE got SE request\n", + cptvf->vfid); + return -EINVAL; + } + + return process_request(pdev, req, &cptvf->pqinfo.queue[0], cptvf); +} + +static int cpt_process_ccode(struct pci_dev *pdev, + union otx_cpt_res_s *cpt_status, + struct otx_cpt_info_buffer *cpt_info, + struct otx_cpt_req_info *req, u32 *res_code) +{ + u8 ccode = cpt_status->s.compcode; + union otx_cpt_error_code ecode; + + ecode.u = be64_to_cpup((__be64 *)cpt_info->out_buffer); + switch (ccode) { + case CPT_COMP_E_FAULT: + dev_err(&pdev->dev, + "Request failed with DMA fault\n"); + otx_cpt_dump_sg_list(pdev, req); + break; + + case CPT_COMP_E_SWERR: + dev_err(&pdev->dev, + "Request failed with software error code %d\n", + ecode.s.ccode); + otx_cpt_dump_sg_list(pdev, req); + break; + + case CPT_COMP_E_HWERR: + dev_err(&pdev->dev, + "Request failed with hardware error\n"); + otx_cpt_dump_sg_list(pdev, req); + break; + + case COMPLETION_CODE_INIT: + /* check for timeout */ + if (time_after_eq(jiffies, cpt_info->time_in + + OTX_CPT_COMMAND_TIMEOUT * HZ)) + dev_warn(&pdev->dev, "Request timed out 0x%p\n", req); + else if (cpt_info->extra_time < OTX_CPT_TIME_IN_RESET_COUNT) { + cpt_info->time_in = jiffies; + cpt_info->extra_time++; + } + return 1; + + case CPT_COMP_E_GOOD: + /* Check microcode completion code */ + if (ecode.s.ccode) { + /* + * If requested hmac is truncated and ucode returns + * s/g write length error then we report success + * because ucode writes as many bytes of calculated + * hmac as available in gather buffer and reports + * s/g write length error if number of bytes in gather + * buffer is less than full hmac size. + */ + if (req->is_trunc_hmac && + ecode.s.ccode == ERR_SCATTER_GATHER_WRITE_LENGTH) { + *res_code = 0; + break; + } + + dev_err(&pdev->dev, + "Request failed with software error code 0x%x\n", + ecode.s.ccode); + otx_cpt_dump_sg_list(pdev, req); + break; + } + + /* Request has been processed with success */ + *res_code = 0; + break; + + default: + dev_err(&pdev->dev, "Request returned invalid status\n"); + break; + } + + return 0; +} + +static inline void process_pending_queue(struct pci_dev *pdev, + struct otx_cpt_pending_queue *pqueue) +{ + void (*callback)(int status, void *arg1, void *arg2); + struct otx_cpt_pending_entry *resume_pentry = NULL; + struct otx_cpt_pending_entry *pentry = NULL; + struct otx_cpt_info_buffer *cpt_info = NULL; + union otx_cpt_res_s *cpt_status = NULL; + struct otx_cpt_req_info *req = NULL; + struct crypto_async_request *areq; + u32 res_code, resume_index; + + while (1) { + spin_lock_bh(&pqueue->lock); + pentry = &pqueue->head[pqueue->front]; + + if (WARN_ON(!pentry)) { + spin_unlock_bh(&pqueue->lock); + break; + } + + res_code = -EINVAL; + if (unlikely(!pentry->busy)) { + spin_unlock_bh(&pqueue->lock); + break; + } + + if (unlikely(!pentry->callback)) { + dev_err(&pdev->dev, "Callback NULL\n"); + goto process_pentry; + } + + cpt_info = pentry->info; + if (unlikely(!cpt_info)) { + dev_err(&pdev->dev, "Pending entry post arg NULL\n"); + goto process_pentry; + } + + req = cpt_info->req; + if (unlikely(!req)) { + dev_err(&pdev->dev, "Request NULL\n"); + goto process_pentry; + } + + cpt_status = (union otx_cpt_res_s *) pentry->completion_addr; + if (unlikely(!cpt_status)) { + dev_err(&pdev->dev, "Completion address NULL\n"); + goto process_pentry; + } + + if (cpt_process_ccode(pdev, cpt_status, cpt_info, req, + &res_code)) { + spin_unlock_bh(&pqueue->lock); + return; + } + cpt_info->pdev = pdev; + +process_pentry: + /* + * Check if we should inform sending side to resume + * We do it CPT_IQ_RESUME_MARGIN elements in advance before + * pending queue becomes empty + */ + resume_index = modulo_inc(pqueue->front, pqueue->qlen, + CPT_IQ_RESUME_MARGIN); + resume_pentry = &pqueue->head[resume_index]; + if (resume_pentry && + resume_pentry->resume_sender) { + resume_pentry->resume_sender = false; + callback = resume_pentry->callback; + areq = resume_pentry->areq; + + if (callback) { + spin_unlock_bh(&pqueue->lock); + + /* + * EINPROGRESS is an indication for sending + * side that it can resume sending requests + */ + callback(-EINPROGRESS, areq, cpt_info); + spin_lock_bh(&pqueue->lock); + } + } + + callback = pentry->callback; + areq = pentry->areq; + free_pentry(pentry); + + pqueue->pending_count--; + pqueue->front = modulo_inc(pqueue->front, pqueue->qlen, 1); + spin_unlock_bh(&pqueue->lock); + + /* + * Call callback after current pending entry has been + * processed, we don't do it if the callback pointer is + * invalid. + */ + if (callback) + callback(res_code, areq, cpt_info); + } +} + +void otx_cpt_post_process(struct otx_cptvf_wqe *wqe) +{ + process_pending_queue(wqe->cptvf->pdev, &wqe->cptvf->pqinfo.queue[0]); +} diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_reqmgr.h b/drivers/crypto/marvell/octeontx/otx_cptvf_reqmgr.h new file mode 100644 index 000000000..a02d059fb --- /dev/null +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_reqmgr.h @@ -0,0 +1,227 @@ +/* SPDX-License-Identifier: GPL-2.0 + * Marvell OcteonTX CPT driver + * + * Copyright (C) 2019 Marvell International Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef __OTX_CPTVF_REQUEST_MANAGER_H +#define __OTX_CPTVF_REQUEST_MANAGER_H + +#include +#include +#include +#include "otx_cpt_hw_types.h" + +/* + * Maximum total number of SG buffers is 100, we divide it equally + * between input and output + */ +#define OTX_CPT_MAX_SG_IN_CNT 50 +#define OTX_CPT_MAX_SG_OUT_CNT 50 + +/* DMA mode direct or SG */ +#define OTX_CPT_DMA_DIRECT_DIRECT 0 +#define OTX_CPT_DMA_GATHER_SCATTER 1 + +/* Context source CPTR or DPTR */ +#define OTX_CPT_FROM_CPTR 0 +#define OTX_CPT_FROM_DPTR 1 + +/* CPT instruction queue alignment */ +#define OTX_CPT_INST_Q_ALIGNMENT 128 +#define OTX_CPT_MAX_REQ_SIZE 65535 + +/* Default command timeout in seconds */ +#define OTX_CPT_COMMAND_TIMEOUT 4 +#define OTX_CPT_TIMER_HOLD 0x03F +#define OTX_CPT_COUNT_HOLD 32 +#define OTX_CPT_TIME_IN_RESET_COUNT 5 + +/* Minimum and maximum values for interrupt coalescing */ +#define OTX_CPT_COALESC_MIN_TIME_WAIT 0x0 +#define OTX_CPT_COALESC_MAX_TIME_WAIT ((1<<16)-1) +#define OTX_CPT_COALESC_MIN_NUM_WAIT 0x0 +#define OTX_CPT_COALESC_MAX_NUM_WAIT ((1<<20)-1) + +union otx_cpt_opcode_info { + u16 flags; + struct { + u8 major; + u8 minor; + } s; +}; + +struct otx_cptvf_request { + u32 param1; + u32 param2; + u16 dlen; + union otx_cpt_opcode_info opcode; +}; + +struct otx_cpt_buf_ptr { + u8 *vptr; + dma_addr_t dma_addr; + u16 size; +}; + +union otx_cpt_ctrl_info { + u32 flags; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u32 reserved0:26; + u32 grp:3; /* Group bits */ + u32 dma_mode:2; /* DMA mode */ + u32 se_req:1; /* To SE core */ +#else + u32 se_req:1; /* To SE core */ + u32 dma_mode:2; /* DMA mode */ + u32 grp:3; /* Group bits */ + u32 reserved0:26; +#endif + } s; +}; + +/* + * CPT_INST_S software command definitions + * Words EI (0-3) + */ +union otx_cpt_iq_cmd_word0 { + u64 u64; + struct { + __be16 opcode; + __be16 param1; + __be16 param2; + __be16 dlen; + } s; +}; + +union otx_cpt_iq_cmd_word3 { + u64 u64; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 grp:3; + u64 cptr:61; +#else + u64 cptr:61; + u64 grp:3; +#endif + } s; +}; + +struct otx_cpt_iq_cmd { + union otx_cpt_iq_cmd_word0 cmd; + u64 dptr; + u64 rptr; + union otx_cpt_iq_cmd_word3 cptr; +}; + +struct otx_cpt_sglist_component { + union { + u64 len; + struct { + __be16 len0; + __be16 len1; + __be16 len2; + __be16 len3; + } s; + } u; + __be64 ptr0; + __be64 ptr1; + __be64 ptr2; + __be64 ptr3; +}; + +struct otx_cpt_pending_entry { + u64 *completion_addr; /* Completion address */ + struct otx_cpt_info_buffer *info; + /* Kernel async request callback */ + void (*callback)(int status, void *arg1, void *arg2); + struct crypto_async_request *areq; /* Async request callback arg */ + u8 resume_sender; /* Notify sender to resume sending requests */ + u8 busy; /* Entry status (free/busy) */ +}; + +struct otx_cpt_pending_queue { + struct otx_cpt_pending_entry *head; /* Head of the queue */ + u32 front; /* Process work from here */ + u32 rear; /* Append new work here */ + u32 pending_count; /* Pending requests count */ + u32 qlen; /* Queue length */ + spinlock_t lock; /* Queue lock */ +}; + +struct otx_cpt_req_info { + /* Kernel async request callback */ + void (*callback)(int status, void *arg1, void *arg2); + struct crypto_async_request *areq; /* Async request callback arg */ + struct otx_cptvf_request req;/* Request information (core specific) */ + union otx_cpt_ctrl_info ctrl;/* User control information */ + struct otx_cpt_buf_ptr in[OTX_CPT_MAX_SG_IN_CNT]; + struct otx_cpt_buf_ptr out[OTX_CPT_MAX_SG_OUT_CNT]; + u8 *iv_out; /* IV to send back */ + u16 rlen; /* Output length */ + u8 incnt; /* Number of input buffers */ + u8 outcnt; /* Number of output buffers */ + u8 req_type; /* Type of request */ + u8 is_enc; /* Is a request an encryption request */ + u8 is_trunc_hmac;/* Is truncated hmac used */ +}; + +struct otx_cpt_info_buffer { + struct otx_cpt_pending_entry *pentry; + struct otx_cpt_req_info *req; + struct pci_dev *pdev; + u64 *completion_addr; + u8 *out_buffer; + u8 *in_buffer; + dma_addr_t dptr_baddr; + dma_addr_t rptr_baddr; + dma_addr_t comp_baddr; + unsigned long time_in; + u32 dlen; + u32 dma_len; + u8 extra_time; +}; + +static inline void do_request_cleanup(struct pci_dev *pdev, + struct otx_cpt_info_buffer *info) +{ + struct otx_cpt_req_info *req; + int i; + + if (info->dptr_baddr) + dma_unmap_single(&pdev->dev, info->dptr_baddr, + info->dma_len, DMA_BIDIRECTIONAL); + + if (info->req) { + req = info->req; + for (i = 0; i < req->outcnt; i++) { + if (req->out[i].dma_addr) + dma_unmap_single(&pdev->dev, + req->out[i].dma_addr, + req->out[i].size, + DMA_BIDIRECTIONAL); + } + + for (i = 0; i < req->incnt; i++) { + if (req->in[i].dma_addr) + dma_unmap_single(&pdev->dev, + req->in[i].dma_addr, + req->in[i].size, + DMA_BIDIRECTIONAL); + } + } + kfree_sensitive(info); +} + +struct otx_cptvf_wqe; +void otx_cpt_dump_sg_list(struct pci_dev *pdev, struct otx_cpt_req_info *req); +void otx_cpt_post_process(struct otx_cptvf_wqe *wqe); +int otx_cpt_do_request(struct pci_dev *pdev, struct otx_cpt_req_info *req, + int cpu_num); + +#endif /* __OTX_CPTVF_REQUEST_MANAGER_H */ diff --git a/drivers/crypto/marvell/octeontx2/Makefile b/drivers/crypto/marvell/octeontx2/Makefile new file mode 100644 index 000000000..f0f2942c1 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_OCTEONTX2_CPT) += rvu_cptcommon.o rvu_cptpf.o rvu_cptvf.o + +rvu_cptcommon-objs := cn10k_cpt.o otx2_cptlf.o otx2_cpt_mbox_common.o +rvu_cptpf-objs := otx2_cptpf_main.o otx2_cptpf_mbox.o \ + otx2_cptpf_ucode.o otx2_cpt_devlink.o +rvu_cptvf-objs := otx2_cptvf_main.o otx2_cptvf_mbox.o \ + otx2_cptvf_reqmgr.o otx2_cptvf_algs.o + +ccflags-y += -I$(srctree)/drivers/net/ethernet/marvell/octeontx2/af diff --git a/drivers/crypto/marvell/octeontx2/cn10k_cpt.c b/drivers/crypto/marvell/octeontx2/cn10k_cpt.c new file mode 100644 index 000000000..d4aab9e20 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/cn10k_cpt.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2021 Marvell. */ + +#include +#include "otx2_cptpf.h" +#include "otx2_cptvf.h" +#include "otx2_cptlf.h" +#include "cn10k_cpt.h" +#include "otx2_cpt_common.h" + +static void cn10k_cpt_send_cmd(union otx2_cpt_inst_s *cptinst, u32 insts_num, + struct otx2_cptlf_info *lf); + +static struct cpt_hw_ops otx2_hw_ops = { + .send_cmd = otx2_cpt_send_cmd, + .cpt_get_compcode = otx2_cpt_get_compcode, + .cpt_get_uc_compcode = otx2_cpt_get_uc_compcode, + .cpt_sg_info_create = otx2_sg_info_create, +}; + +static struct cpt_hw_ops cn10k_hw_ops = { + .send_cmd = cn10k_cpt_send_cmd, + .cpt_get_compcode = cn10k_cpt_get_compcode, + .cpt_get_uc_compcode = cn10k_cpt_get_uc_compcode, + .cpt_sg_info_create = otx2_sg_info_create, +}; + +static void cn10k_cpt_send_cmd(union otx2_cpt_inst_s *cptinst, u32 insts_num, + struct otx2_cptlf_info *lf) +{ + void *lmtline = lf->lfs->lmt_info.base + (lf->slot * LMTLINE_SIZE); + u64 val = (lf->slot & 0x7FF); + u64 tar_addr = 0; + + /* tar_addr<6:4> = Size of first LMTST - 1 in units of 128b. */ + tar_addr |= (__force u64)lf->ioreg | + (((OTX2_CPT_INST_SIZE/16) - 1) & 0x7) << 4; + /* + * Make sure memory areas pointed in CPT_INST_S + * are flushed before the instruction is sent to CPT + */ + dma_wmb(); + + /* Copy CPT command to LMTLINE */ + memcpy(lmtline, cptinst, insts_num * OTX2_CPT_INST_SIZE); + cn10k_lmt_flush(val, tar_addr); +} + +void cn10k_cpt_lmtst_free(struct pci_dev *pdev, struct otx2_cptlfs_info *lfs) +{ + struct otx2_lmt_info *lmt_info = &lfs->lmt_info; + + if (!lmt_info->base) + return; + + dma_free_attrs(&pdev->dev, lmt_info->size, + lmt_info->base - lmt_info->align, + lmt_info->iova - lmt_info->align, + DMA_ATTR_FORCE_CONTIGUOUS); +} +EXPORT_SYMBOL_NS_GPL(cn10k_cpt_lmtst_free, "CRYPTO_DEV_OCTEONTX2_CPT"); + +static int cn10k_cpt_lmtst_alloc(struct pci_dev *pdev, + struct otx2_cptlfs_info *lfs, u32 size) +{ + struct otx2_lmt_info *lmt_info = &lfs->lmt_info; + dma_addr_t align_iova; + dma_addr_t iova; + + lmt_info->base = dma_alloc_attrs(&pdev->dev, size, &iova, GFP_KERNEL, + DMA_ATTR_FORCE_CONTIGUOUS); + if (!lmt_info->base) + return -ENOMEM; + + align_iova = ALIGN((u64)iova, LMTLINE_ALIGN); + lmt_info->iova = align_iova; + lmt_info->align = align_iova - iova; + lmt_info->size = size; + lmt_info->base += lmt_info->align; + return 0; +} + +int cn10k_cptpf_lmtst_init(struct otx2_cptpf_dev *cptpf) +{ + struct pci_dev *pdev = cptpf->pdev; + u32 size; + int ret; + + if (!test_bit(CN10K_LMTST, &cptpf->cap_flag)) { + cptpf->lfs.ops = &otx2_hw_ops; + return 0; + } + + cptpf->lfs.ops = &cn10k_hw_ops; + size = OTX2_CPT_MAX_VFS_NUM * LMTLINE_SIZE + LMTLINE_ALIGN; + ret = cn10k_cpt_lmtst_alloc(pdev, &cptpf->lfs, size); + if (ret) { + dev_err(&pdev->dev, "PF-%d LMTLINE memory allocation failed\n", + cptpf->pf_id); + return ret; + } + + ret = otx2_cpt_lmtst_tbl_setup_msg(&cptpf->lfs); + if (ret) { + dev_err(&pdev->dev, "PF-%d: LMTST Table setup failed\n", + cptpf->pf_id); + cn10k_cpt_lmtst_free(pdev, &cptpf->lfs); + } + + return 0; +} +EXPORT_SYMBOL_NS_GPL(cn10k_cptpf_lmtst_init, "CRYPTO_DEV_OCTEONTX2_CPT"); + +int cn10k_cptvf_lmtst_init(struct otx2_cptvf_dev *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + u32 size; + int ret; + + if (!test_bit(CN10K_LMTST, &cptvf->cap_flag)) + return 0; + + size = cptvf->lfs.lfs_num * LMTLINE_SIZE + LMTLINE_ALIGN; + ret = cn10k_cpt_lmtst_alloc(pdev, &cptvf->lfs, size); + if (ret) { + dev_err(&pdev->dev, "VF-%d LMTLINE memory allocation failed\n", + cptvf->vf_id); + return ret; + } + + ret = otx2_cpt_lmtst_tbl_setup_msg(&cptvf->lfs); + if (ret) { + dev_err(&pdev->dev, "VF-%d: LMTST Table setup failed\n", + cptvf->vf_id); + cn10k_cpt_lmtst_free(pdev, &cptvf->lfs); + } + + return 0; +} +EXPORT_SYMBOL_NS_GPL(cn10k_cptvf_lmtst_init, "CRYPTO_DEV_OCTEONTX2_CPT"); + +void cn10k_cpt_hw_ctx_clear(struct pci_dev *pdev, + struct cn10k_cpt_errata_ctx *er_ctx) +{ + u64 cptr_dma; + + if (!is_dev_cn10ka_ax(pdev)) + return; + + cptr_dma = er_ctx->cptr_dma & ~(BIT_ULL(60)); + cn10k_cpt_ctx_flush(pdev, cptr_dma, true); + dma_unmap_single(&pdev->dev, cptr_dma, CN10K_CPT_HW_CTX_SIZE, + DMA_BIDIRECTIONAL); + kfree(er_ctx->hw_ctx); +} +EXPORT_SYMBOL_NS_GPL(cn10k_cpt_hw_ctx_clear, "CRYPTO_DEV_OCTEONTX2_CPT"); + +void cn10k_cpt_hw_ctx_set(union cn10k_cpt_hw_ctx *hctx, u16 ctx_sz) +{ + hctx->w0.aop_valid = 1; + hctx->w0.ctx_hdr_sz = 0; + hctx->w0.ctx_sz = ctx_sz; + hctx->w0.ctx_push_sz = 1; +} +EXPORT_SYMBOL_NS_GPL(cn10k_cpt_hw_ctx_set, "CRYPTO_DEV_OCTEONTX2_CPT"); + +int cn10k_cpt_hw_ctx_init(struct pci_dev *pdev, + struct cn10k_cpt_errata_ctx *er_ctx) +{ + union cn10k_cpt_hw_ctx *hctx; + u64 cptr_dma; + + er_ctx->cptr_dma = 0; + er_ctx->hw_ctx = NULL; + + if (!is_dev_cn10ka_ax(pdev)) + return 0; + + hctx = kmalloc(CN10K_CPT_HW_CTX_SIZE, GFP_KERNEL); + if (unlikely(!hctx)) + return -ENOMEM; + cptr_dma = dma_map_single(&pdev->dev, hctx, CN10K_CPT_HW_CTX_SIZE, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(&pdev->dev, cptr_dma)) { + kfree(hctx); + return -ENOMEM; + } + + cn10k_cpt_hw_ctx_set(hctx, 1); + er_ctx->hw_ctx = hctx; + er_ctx->cptr_dma = cptr_dma | BIT_ULL(60); + + return 0; +} +EXPORT_SYMBOL_NS_GPL(cn10k_cpt_hw_ctx_init, "CRYPTO_DEV_OCTEONTX2_CPT"); + +void cn10k_cpt_ctx_flush(struct pci_dev *pdev, u64 cptr, bool inval) +{ + struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev); + struct otx2_cptlfs_info *lfs = &cptvf->lfs; + u64 reg; + + reg = (uintptr_t)cptr >> 7; + if (inval) + reg = reg | BIT_ULL(46); + + otx2_cpt_write64(lfs->reg_base, lfs->blkaddr, lfs->lf[0].slot, + OTX2_CPT_LF_CTX_FLUSH, reg); + /* Make sure that the FLUSH operation is complete */ + wmb(); + otx2_cpt_read64(lfs->reg_base, lfs->blkaddr, lfs->lf[0].slot, + OTX2_CPT_LF_CTX_ERR); +} +EXPORT_SYMBOL_NS_GPL(cn10k_cpt_ctx_flush, "CRYPTO_DEV_OCTEONTX2_CPT"); + +void cptvf_hw_ops_get(struct otx2_cptvf_dev *cptvf) +{ + if (test_bit(CN10K_LMTST, &cptvf->cap_flag)) + cptvf->lfs.ops = &cn10k_hw_ops; + else + cptvf->lfs.ops = &otx2_hw_ops; +} +EXPORT_SYMBOL_NS_GPL(cptvf_hw_ops_get, "CRYPTO_DEV_OCTEONTX2_CPT"); diff --git a/drivers/crypto/marvell/octeontx2/cn10k_cpt.h b/drivers/crypto/marvell/octeontx2/cn10k_cpt.h new file mode 100644 index 000000000..ea5990048 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/cn10k_cpt.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2021 Marvell. + */ +#ifndef __CN10K_CPT_H +#define __CN10K_CPT_H + +#include "otx2_cpt_common.h" +#include "otx2_cptpf.h" +#include "otx2_cptvf.h" + +#define CN10K_CPT_HW_CTX_SIZE 256 + +union cn10k_cpt_hw_ctx { + u64 u; + struct { + u64 reserved_0_47:48; + u64 ctx_push_sz:7; + u64 reserved_55:1; + u64 ctx_hdr_sz:2; + u64 aop_valid:1; + u64 reserved_59:1; + u64 ctx_sz:4; + } w0; +}; + +struct cn10k_cpt_errata_ctx { + union cn10k_cpt_hw_ctx *hw_ctx; + u64 cptr_dma; +}; + +static inline u8 cn10k_cpt_get_compcode(union otx2_cpt_res_s *result) +{ + return ((struct cn10k_cpt_res_s *)result)->compcode; +} + +static inline u8 cn10k_cpt_get_uc_compcode(union otx2_cpt_res_s *result) +{ + return ((struct cn10k_cpt_res_s *)result)->uc_compcode; +} + +static inline u8 otx2_cpt_get_compcode(union otx2_cpt_res_s *result) +{ + return ((struct cn9k_cpt_res_s *)result)->compcode; +} + +static inline u8 otx2_cpt_get_uc_compcode(union otx2_cpt_res_s *result) +{ + return ((struct cn9k_cpt_res_s *)result)->uc_compcode; +} + +int cn10k_cptpf_lmtst_init(struct otx2_cptpf_dev *cptpf); +int cn10k_cptvf_lmtst_init(struct otx2_cptvf_dev *cptvf); +void cn10k_cpt_lmtst_free(struct pci_dev *pdev, struct otx2_cptlfs_info *lfs); +void cn10k_cpt_ctx_flush(struct pci_dev *pdev, u64 cptr, bool inval); +int cn10k_cpt_hw_ctx_init(struct pci_dev *pdev, + struct cn10k_cpt_errata_ctx *er_ctx); +void cn10k_cpt_hw_ctx_clear(struct pci_dev *pdev, + struct cn10k_cpt_errata_ctx *er_ctx); +void cn10k_cpt_hw_ctx_set(union cn10k_cpt_hw_ctx *hctx, u16 ctx_sz); +void cptvf_hw_ops_get(struct otx2_cptvf_dev *cptvf); + +#endif /* __CN10K_CPTLF_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h new file mode 100644 index 000000000..062def303 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h @@ -0,0 +1,213 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2020 Marvell. + */ + +#ifndef __OTX2_CPT_COMMON_H +#define __OTX2_CPT_COMMON_H + +#include +#include +#include +#include +#include +#include +#include "otx2_cpt_hw_types.h" +#include "rvu.h" +#include "mbox.h" + +#define OTX2_CPT_MAX_VFS_NUM 128 +#define OTX2_CPT_RVU_FUNC_ADDR_S(blk, slot, offs) \ + (((blk) << 20) | ((slot) << 12) | (offs)) + +#define OTX2_CPT_RVU_PFFUNC(pdev, pf, func) rvu_make_pcifunc(pdev, pf, func) + +#define OTX2_CPT_INVALID_CRYPTO_ENG_GRP 0xFF +#define OTX2_CPT_NAME_LENGTH 64 +#define OTX2_CPT_DMA_MINALIGN 128 + +/* HW capability flags */ +#define CN10K_MBOX 0 +#define CN10K_LMTST 1 + +#define BAD_OTX2_CPT_ENG_TYPE OTX2_CPT_MAX_ENG_TYPES + +enum otx2_cpt_eng_type { + OTX2_CPT_AE_TYPES = 1, + OTX2_CPT_SE_TYPES = 2, + OTX2_CPT_IE_TYPES = 3, + OTX2_CPT_MAX_ENG_TYPES, +}; + +/* Take mbox id from end of CPT mbox range in AF (range 0xA00 - 0xBFF) */ +#define MBOX_MSG_RX_INLINE_IPSEC_LF_CFG 0xBFE +#define MBOX_MSG_GET_ENG_GRP_NUM 0xBFF +#define MBOX_MSG_GET_CAPS 0xBFD +#define MBOX_MSG_GET_KVF_LIMITS 0xBFC + +/* + * Message request to config cpt lf for inline inbound ipsec. + * This message is only used between CPT PF <-> CPT VF + */ +struct otx2_cpt_rx_inline_lf_cfg { + struct mbox_msghdr hdr; + u16 sso_pf_func; + u16 param1; + u16 param2; + u16 opcode; + u32 credit; + u32 credit_th; + u16 bpid; + u32 reserved; + u8 ctx_ilen_valid : 1; + u8 ctx_ilen : 7; +}; + +/* + * Message request and response to get engine group number + * which has attached a given type of engines (SE, AE, IE) + * This messages are only used between CPT PF <=> CPT VF + */ +struct otx2_cpt_egrp_num_msg { + struct mbox_msghdr hdr; + u8 eng_type; +}; + +struct otx2_cpt_egrp_num_rsp { + struct mbox_msghdr hdr; + u8 eng_type; + u8 eng_grp_num; +}; + +/* + * Message request and response to get kernel crypto limits + * This messages are only used between CPT PF <-> CPT VF + */ +struct otx2_cpt_kvf_limits_msg { + struct mbox_msghdr hdr; +}; + +struct otx2_cpt_kvf_limits_rsp { + struct mbox_msghdr hdr; + u8 kvf_limits; +}; + +/* CPT HW capabilities */ +union otx2_cpt_eng_caps { + u64 u; + struct { + u64 reserved_0_4:5; + u64 mul:1; + u64 sha1_sha2:1; + u64 chacha20:1; + u64 zuc_snow3g:1; + u64 sha3:1; + u64 aes:1; + u64 kasumi:1; + u64 des:1; + u64 crc:1; + u64 mmul:1; + u64 reserved_15_33:19; + u64 pdcp_chain:1; + u64 reserved_35_63:29; + }; +}; + +/* + * Message request and response to get HW capabilities for each + * engine type (SE, IE, AE). + * This messages are only used between CPT PF <=> CPT VF + */ +struct otx2_cpt_caps_msg { + struct mbox_msghdr hdr; +}; + +struct otx2_cpt_caps_rsp { + struct mbox_msghdr hdr; + u16 cpt_pf_drv_version; + u8 cpt_revision; + union otx2_cpt_eng_caps eng_caps[OTX2_CPT_MAX_ENG_TYPES]; +}; + +static inline void otx2_cpt_write64(void __iomem *reg_base, u64 blk, u64 slot, + u64 offs, u64 val) +{ + writeq_relaxed(val, reg_base + + OTX2_CPT_RVU_FUNC_ADDR_S(blk, slot, offs)); +} + +static inline u64 otx2_cpt_read64(void __iomem *reg_base, u64 blk, u64 slot, + u64 offs) +{ + return readq_relaxed(reg_base + + OTX2_CPT_RVU_FUNC_ADDR_S(blk, slot, offs)); +} + +static inline bool is_dev_otx2(struct pci_dev *pdev) +{ + return pdev->device == OTX2_CPT_PCI_PF_DEVICE_ID || + pdev->device == OTX2_CPT_PCI_VF_DEVICE_ID; +} + +static inline bool is_dev_cn10ka(struct pci_dev *pdev) +{ + return pdev->subsystem_device == CPT_PCI_SUBSYS_DEVID_CN10K_A; +} + +static inline bool is_dev_cn10ka_ax(struct pci_dev *pdev) +{ + return pdev->subsystem_device == CPT_PCI_SUBSYS_DEVID_CN10K_A && + ((pdev->revision & 0xFF) == 4 || + (pdev->revision & 0xFF) == 0x50 || + (pdev->revision & 0xFF) == 0x51); +} + +static inline bool is_dev_cn10kb(struct pci_dev *pdev) +{ + return pdev->subsystem_device == CPT_PCI_SUBSYS_DEVID_CN10K_B; +} + +static inline bool is_dev_cn10ka_b0(struct pci_dev *pdev) +{ + return pdev->subsystem_device == CPT_PCI_SUBSYS_DEVID_CN10K_A && + (pdev->revision & 0xFF) == 0x54; +} + +static inline void otx2_cpt_set_hw_caps(struct pci_dev *pdev, + unsigned long *cap_flag) +{ + if (!is_dev_otx2(pdev)) { + __set_bit(CN10K_MBOX, cap_flag); + __set_bit(CN10K_LMTST, cap_flag); + } +} + +static inline bool cpt_is_errata_38550_exists(struct pci_dev *pdev) +{ + return is_dev_otx2(pdev) || is_dev_cn10ka_ax(pdev); +} + +static inline bool cpt_feature_sgv2(struct pci_dev *pdev) +{ + return !is_dev_otx2(pdev) && !is_dev_cn10ka_ax(pdev); +} + +int otx2_cpt_send_ready_msg(struct otx2_mbox *mbox, struct pci_dev *pdev); +int otx2_cpt_send_mbox_msg(struct otx2_mbox *mbox, struct pci_dev *pdev); + +int otx2_cpt_send_af_reg_requests(struct otx2_mbox *mbox, + struct pci_dev *pdev); +int otx2_cpt_add_write_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, + u64 reg, u64 val, int blkaddr); +int otx2_cpt_read_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, + u64 reg, u64 *val, int blkaddr); +int otx2_cpt_write_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, + u64 reg, u64 val, int blkaddr); +struct otx2_cptlfs_info; +int otx2_cpt_attach_rscrs_msg(struct otx2_cptlfs_info *lfs); +int otx2_cpt_detach_rsrcs_msg(struct otx2_cptlfs_info *lfs); +int otx2_cpt_msix_offset_msg(struct otx2_cptlfs_info *lfs); +int otx2_cpt_sync_mbox_msg(struct otx2_mbox *mbox); +int otx2_cpt_lf_reset_msg(struct otx2_cptlfs_info *lfs, int slot); +int otx2_cpt_lmtst_tbl_setup_msg(struct otx2_cptlfs_info *lfs); + +#endif /* __OTX2_CPT_COMMON_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c b/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c new file mode 100644 index 000000000..07a74f702 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2021 Marvell. */ + +#include "otx2_cpt_devlink.h" + +static int otx2_cpt_dl_egrp_create(struct devlink *dl, u32 id, + struct devlink_param_gset_ctx *ctx, + struct netlink_ext_ack *extack) +{ + struct otx2_cpt_devlink *cpt_dl = devlink_priv(dl); + struct otx2_cptpf_dev *cptpf = cpt_dl->cptpf; + + return otx2_cpt_dl_custom_egrp_create(cptpf, ctx); +} + +static int otx2_cpt_dl_egrp_delete(struct devlink *dl, u32 id, + struct devlink_param_gset_ctx *ctx, + struct netlink_ext_ack *extack) +{ + struct otx2_cpt_devlink *cpt_dl = devlink_priv(dl); + struct otx2_cptpf_dev *cptpf = cpt_dl->cptpf; + + return otx2_cpt_dl_custom_egrp_delete(cptpf, ctx); +} + +static int otx2_cpt_dl_uc_info(struct devlink *dl, u32 id, + struct devlink_param_gset_ctx *ctx, + struct netlink_ext_ack *extack) +{ + ctx->val.vstr[0] = '\0'; + + return 0; +} + +static int otx2_cpt_dl_t106_mode_get(struct devlink *dl, u32 id, + struct devlink_param_gset_ctx *ctx, + struct netlink_ext_ack *extack) +{ + struct otx2_cpt_devlink *cpt_dl = devlink_priv(dl); + struct otx2_cptpf_dev *cptpf = cpt_dl->cptpf; + struct pci_dev *pdev = cptpf->pdev; + u64 reg_val = 0; + + otx2_cpt_read_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_CTL, ®_val, + BLKADDR_CPT0); + ctx->val.vu8 = (reg_val >> 18) & 0x1; + + return 0; +} + +static int otx2_cpt_dl_t106_mode_set(struct devlink *dl, u32 id, + struct devlink_param_gset_ctx *ctx, + struct netlink_ext_ack *extack) +{ + struct otx2_cpt_devlink *cpt_dl = devlink_priv(dl); + struct otx2_cptpf_dev *cptpf = cpt_dl->cptpf; + struct pci_dev *pdev = cptpf->pdev; + u64 reg_val = 0; + + if (cptpf->enabled_vfs != 0 || cptpf->eng_grps.is_grps_created) + return -EPERM; + + if (cpt_feature_sgv2(pdev)) { + otx2_cpt_read_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_CTL, + ®_val, BLKADDR_CPT0); + reg_val &= ~(0x1ULL << 18); + reg_val |= ((u64)ctx->val.vu8 & 0x1) << 18; + return otx2_cpt_write_af_reg(&cptpf->afpf_mbox, pdev, + CPT_AF_CTL, reg_val, BLKADDR_CPT0); + } + + return 0; +} + +enum otx2_cpt_dl_param_id { + OTX2_CPT_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX, + OTX2_CPT_DEVLINK_PARAM_ID_EGRP_CREATE, + OTX2_CPT_DEVLINK_PARAM_ID_EGRP_DELETE, + OTX2_CPT_DEVLINK_PARAM_ID_T106_MODE, +}; + +static const struct devlink_param otx2_cpt_dl_params[] = { + DEVLINK_PARAM_DRIVER(OTX2_CPT_DEVLINK_PARAM_ID_EGRP_CREATE, + "egrp_create", DEVLINK_PARAM_TYPE_STRING, + BIT(DEVLINK_PARAM_CMODE_RUNTIME), + otx2_cpt_dl_uc_info, otx2_cpt_dl_egrp_create, + NULL), + DEVLINK_PARAM_DRIVER(OTX2_CPT_DEVLINK_PARAM_ID_EGRP_DELETE, + "egrp_delete", DEVLINK_PARAM_TYPE_STRING, + BIT(DEVLINK_PARAM_CMODE_RUNTIME), + otx2_cpt_dl_uc_info, otx2_cpt_dl_egrp_delete, + NULL), + DEVLINK_PARAM_DRIVER(OTX2_CPT_DEVLINK_PARAM_ID_T106_MODE, + "t106_mode", DEVLINK_PARAM_TYPE_U8, + BIT(DEVLINK_PARAM_CMODE_RUNTIME), + otx2_cpt_dl_t106_mode_get, otx2_cpt_dl_t106_mode_set, + NULL), +}; + +static int otx2_cpt_dl_info_firmware_version_put(struct devlink_info_req *req, + struct otx2_cpt_eng_grp_info grp[], + const char *ver_name, int eng_type) +{ + struct otx2_cpt_engs_rsvd *eng; + int i; + + for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) { + eng = find_engines_by_type(&grp[i], eng_type); + if (eng) + return devlink_info_version_running_put(req, ver_name, + eng->ucode->ver_str); + } + + return 0; +} + +static int otx2_cpt_devlink_info_get(struct devlink *dl, + struct devlink_info_req *req, + struct netlink_ext_ack *extack) +{ + struct otx2_cpt_devlink *cpt_dl = devlink_priv(dl); + struct otx2_cptpf_dev *cptpf = cpt_dl->cptpf; + int err; + + err = otx2_cpt_dl_info_firmware_version_put(req, cptpf->eng_grps.grp, + "fw.ae", OTX2_CPT_AE_TYPES); + if (err) + return err; + + err = otx2_cpt_dl_info_firmware_version_put(req, cptpf->eng_grps.grp, + "fw.se", OTX2_CPT_SE_TYPES); + if (err) + return err; + + return otx2_cpt_dl_info_firmware_version_put(req, cptpf->eng_grps.grp, + "fw.ie", OTX2_CPT_IE_TYPES); +} + +static const struct devlink_ops otx2_cpt_devlink_ops = { + .info_get = otx2_cpt_devlink_info_get, +}; + +int otx2_cpt_register_dl(struct otx2_cptpf_dev *cptpf) +{ + struct device *dev = &cptpf->pdev->dev; + struct otx2_cpt_devlink *cpt_dl; + struct devlink *dl; + int ret; + + dl = devlink_alloc(&otx2_cpt_devlink_ops, + sizeof(struct otx2_cpt_devlink), dev); + if (!dl) { + dev_warn(dev, "devlink_alloc failed\n"); + return -ENOMEM; + } + + cpt_dl = devlink_priv(dl); + cpt_dl->dl = dl; + cpt_dl->cptpf = cptpf; + cptpf->dl = dl; + ret = devlink_params_register(dl, otx2_cpt_dl_params, + ARRAY_SIZE(otx2_cpt_dl_params)); + if (ret) { + dev_err(dev, "devlink params register failed with error %d", + ret); + devlink_free(dl); + return ret; + } + devlink_register(dl); + + return 0; +} + +void otx2_cpt_unregister_dl(struct otx2_cptpf_dev *cptpf) +{ + struct devlink *dl = cptpf->dl; + + if (!dl) + return; + + devlink_unregister(dl); + devlink_params_unregister(dl, otx2_cpt_dl_params, + ARRAY_SIZE(otx2_cpt_dl_params)); + devlink_free(dl); +} diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.h new file mode 100644 index 000000000..8b7d88c5d --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2021 Marvell. + */ + +#ifndef __OTX2_CPT_DEVLINK_H +#define __OTX2_CPT_DEVLINK_H + +#include "otx2_cpt_common.h" +#include "otx2_cptpf.h" + +struct otx2_cpt_devlink { + struct devlink *dl; + struct otx2_cptpf_dev *cptpf; +}; + +/* Devlink APIs */ +int otx2_cpt_register_dl(struct otx2_cptpf_dev *cptpf); +void otx2_cpt_unregister_dl(struct otx2_cptpf_dev *cptpf); + +#endif /* __OTX2_CPT_DEVLINK_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h new file mode 100644 index 000000000..7e746a4de --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h @@ -0,0 +1,483 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2020 Marvell. + */ + +#ifndef __OTX2_CPT_HW_TYPES_H +#define __OTX2_CPT_HW_TYPES_H + +#include + +/* Device IDs */ +#define OTX2_CPT_PCI_PF_DEVICE_ID 0xA0FD +#define OTX2_CPT_PCI_VF_DEVICE_ID 0xA0FE +#define CN10K_CPT_PCI_PF_DEVICE_ID 0xA0F2 +#define CN10K_CPT_PCI_VF_DEVICE_ID 0xA0F3 + +#define CPT_PCI_SUBSYS_DEVID_CN10K_A 0xB900 +#define CPT_PCI_SUBSYS_DEVID_CN10K_B 0xBD00 + +/* Mailbox interrupts offset */ +#define OTX2_CPT_PF_MBOX_INT 6 +#define OTX2_CPT_PF_INT_VEC_E_MBOXX(x, a) ((x) + (a)) + +/* Maximum supported microcode groups */ +#define OTX2_CPT_MAX_ENGINE_GROUPS 8 + +/* CPT instruction size in bytes */ +#define OTX2_CPT_INST_SIZE 64 +/* + * CPT VF MSIX vectors and their offsets + */ +#define OTX2_CPT_VF_MSIX_VECTORS 1 +#define OTX2_CPT_VF_INTR_MBOX_MASK BIT(0) +#define CN10K_CPT_VF_MBOX_REGION (0xC0000) + +/* CPT LF MSIX vectors */ +#define OTX2_CPT_LF_MSIX_VECTORS 2 + +/* OcteonTX2 CPT PF registers */ +#define OTX2_CPT_PF_CONSTANTS (0x0) +#define OTX2_CPT_PF_RESET (0x100) +#define OTX2_CPT_PF_DIAG (0x120) +#define OTX2_CPT_PF_BIST_STATUS (0x160) +#define OTX2_CPT_PF_ECC0_CTL (0x200) +#define OTX2_CPT_PF_ECC0_FLIP (0x210) +#define OTX2_CPT_PF_ECC0_INT (0x220) +#define OTX2_CPT_PF_ECC0_INT_W1S (0x230) +#define OTX2_CPT_PF_ECC0_ENA_W1S (0x240) +#define OTX2_CPT_PF_ECC0_ENA_W1C (0x250) +#define OTX2_CPT_PF_MBOX_INTX(b) (0x400 | (b) << 3) +#define OTX2_CPT_PF_MBOX_INT_W1SX(b) (0x420 | (b) << 3) +#define OTX2_CPT_PF_MBOX_ENA_W1CX(b) (0x440 | (b) << 3) +#define OTX2_CPT_PF_MBOX_ENA_W1SX(b) (0x460 | (b) << 3) +#define OTX2_CPT_PF_EXEC_INT (0x500) +#define OTX2_CPT_PF_EXEC_INT_W1S (0x520) +#define OTX2_CPT_PF_EXEC_ENA_W1C (0x540) +#define OTX2_CPT_PF_EXEC_ENA_W1S (0x560) +#define OTX2_CPT_PF_GX_EN(b) (0x600 | (b) << 3) +#define OTX2_CPT_PF_EXEC_INFO (0x700) +#define OTX2_CPT_PF_EXEC_BUSY (0x800) +#define OTX2_CPT_PF_EXEC_INFO0 (0x900) +#define OTX2_CPT_PF_EXEC_INFO1 (0x910) +#define OTX2_CPT_PF_INST_REQ_PC (0x10000) +#define OTX2_CPT_PF_INST_LATENCY_PC (0x10020) +#define OTX2_CPT_PF_RD_REQ_PC (0x10040) +#define OTX2_CPT_PF_RD_LATENCY_PC (0x10060) +#define OTX2_CPT_PF_RD_UC_PC (0x10080) +#define OTX2_CPT_PF_ACTIVE_CYCLES_PC (0x10100) +#define OTX2_CPT_PF_EXE_CTL (0x4000000) +#define OTX2_CPT_PF_EXE_STATUS (0x4000008) +#define OTX2_CPT_PF_EXE_CLK (0x4000010) +#define OTX2_CPT_PF_EXE_DBG_CTL (0x4000018) +#define OTX2_CPT_PF_EXE_DBG_DATA (0x4000020) +#define OTX2_CPT_PF_EXE_BIST_STATUS (0x4000028) +#define OTX2_CPT_PF_EXE_REQ_TIMER (0x4000030) +#define OTX2_CPT_PF_EXE_MEM_CTL (0x4000038) +#define OTX2_CPT_PF_EXE_PERF_CTL (0x4001000) +#define OTX2_CPT_PF_EXE_DBG_CNTX(b) (0x4001100 | (b) << 3) +#define OTX2_CPT_PF_EXE_PERF_EVENT_CNT (0x4001180) +#define OTX2_CPT_PF_EXE_EPCI_INBX_CNT(b) (0x4001200 | (b) << 3) +#define OTX2_CPT_PF_EXE_EPCI_OUTBX_CNT(b) (0x4001240 | (b) << 3) +#define OTX2_CPT_PF_ENGX_UCODE_BASE(b) (0x4002000 | (b) << 3) +#define OTX2_CPT_PF_QX_CTL(b) (0x8000000 | (b) << 20) +#define OTX2_CPT_PF_QX_GMCTL(b) (0x8000020 | (b) << 20) +#define OTX2_CPT_PF_QX_CTL2(b) (0x8000100 | (b) << 20) +#define OTX2_CPT_PF_VFX_MBOXX(b, c) (0x8001000 | (b) << 20 | \ + (c) << 8) + +/* OcteonTX2 CPT LF registers */ +#define OTX2_CPT_LF_CTL (0x10) +#define OTX2_CPT_LF_DONE_WAIT (0x30) +#define OTX2_CPT_LF_INPROG (0x40) +#define OTX2_CPT_LF_DONE (0x50) +#define OTX2_CPT_LF_DONE_ACK (0x60) +#define OTX2_CPT_LF_DONE_INT_ENA_W1S (0x90) +#define OTX2_CPT_LF_DONE_INT_ENA_W1C (0xa0) +#define OTX2_CPT_LF_MISC_INT (0xb0) +#define OTX2_CPT_LF_MISC_INT_W1S (0xc0) +#define OTX2_CPT_LF_MISC_INT_ENA_W1S (0xd0) +#define OTX2_CPT_LF_MISC_INT_ENA_W1C (0xe0) +#define OTX2_CPT_LF_Q_BASE (0xf0) +#define OTX2_CPT_LF_Q_SIZE (0x100) +#define OTX2_CPT_LF_Q_INST_PTR (0x110) +#define OTX2_CPT_LF_Q_GRP_PTR (0x120) +#define OTX2_CPT_LF_NQX(a) (0x400 | (a) << 3) +#define OTX2_CPT_LF_CTX_CTL (0x500) +#define OTX2_CPT_LF_CTX_FLUSH (0x510) +#define OTX2_CPT_LF_CTX_ERR (0x520) +#define OTX2_CPT_RVU_FUNC_BLKADDR_SHIFT 20 +/* LMT LF registers */ +#define OTX2_CPT_LMT_LFBASE BIT_ULL(OTX2_CPT_RVU_FUNC_BLKADDR_SHIFT) +#define OTX2_CPT_LMT_LF_LMTLINEX(a) (OTX2_CPT_LMT_LFBASE | 0x000 | \ + (a) << 12) +/* RVU VF registers */ +#define OTX2_RVU_VF_INT (0x20) +#define OTX2_RVU_VF_INT_W1S (0x28) +#define OTX2_RVU_VF_INT_ENA_W1S (0x30) +#define OTX2_RVU_VF_INT_ENA_W1C (0x38) + +/* + * Enumeration otx2_cpt_ucode_error_code_e + * + * Enumerates ucode errors + */ +enum otx2_cpt_ucode_comp_code_e { + OTX2_CPT_UCC_SUCCESS = 0x00, + OTX2_CPT_UCC_INVALID_OPCODE = 0x01, + + /* Scatter gather */ + OTX2_CPT_UCC_SG_WRITE_LENGTH = 0x02, + OTX2_CPT_UCC_SG_LIST = 0x03, + OTX2_CPT_UCC_SG_NOT_SUPPORTED = 0x04, + +}; + +/* + * Enumeration otx2_cpt_comp_e + * + * OcteonTX2 CPT Completion Enumeration + * Enumerates the values of CPT_RES_S[COMPCODE]. + */ +enum otx2_cpt_comp_e { + OTX2_CPT_COMP_E_NOTDONE = 0x00, + OTX2_CPT_COMP_E_GOOD = 0x01, + OTX2_CPT_COMP_E_FAULT = 0x02, + OTX2_CPT_COMP_E_HWERR = 0x04, + OTX2_CPT_COMP_E_INSTERR = 0x05, + OTX2_CPT_COMP_E_WARN = 0x06 +}; + +/* + * Enumeration otx2_cpt_vf_int_vec_e + * + * OcteonTX2 CPT VF MSI-X Vector Enumeration + * Enumerates the MSI-X interrupt vectors. + */ +enum otx2_cpt_vf_int_vec_e { + OTX2_CPT_VF_INT_VEC_E_MBOX = 0x00 +}; + +/* + * Enumeration otx2_cpt_lf_int_vec_e + * + * OcteonTX2 CPT LF MSI-X Vector Enumeration + * Enumerates the MSI-X interrupt vectors. + */ +enum otx2_cpt_lf_int_vec_e { + OTX2_CPT_LF_INT_VEC_E_MISC = 0x00, + OTX2_CPT_LF_INT_VEC_E_DONE = 0x01 +}; + +/* + * Structure otx2_cpt_inst_s + * + * CPT Instruction Structure + * This structure specifies the instruction layout. Instructions are + * stored in memory as little-endian unless CPT()_PF_Q()_CTL[INST_BE] is set. + * cpt_inst_s_s + * Word 0 + * doneint:1 Done interrupt. + * 0 = No interrupts related to this instruction. + * 1 = When the instruction completes, CPT()_VQ()_DONE[DONE] will be + * incremented,and based on the rules described there an interrupt may + * occur. + * Word 1 + * res_addr [127: 64] Result IOVA. + * If nonzero, specifies where to write CPT_RES_S. + * If zero, no result structure will be written. + * Address must be 16-byte aligned. + * Bits <63:49> are ignored by hardware; software should use a + * sign-extended bit <48> for forward compatibility. + * Word 2 + * grp:10 [171:162] If [WQ_PTR] is nonzero, the SSO guest-group to use when + * CPT submits work SSO. + * For the SSO to not discard the add-work request, FPA_PF_MAP() must map + * [GRP] and CPT()_PF_Q()_GMCTL[GMID] as valid. + * tt:2 [161:160] If [WQ_PTR] is nonzero, the SSO tag type to use when CPT + * submits work to SSO + * tag:32 [159:128] If [WQ_PTR] is nonzero, the SSO tag to use when CPT + * submits work to SSO. + * Word 3 + * wq_ptr [255:192] If [WQ_PTR] is nonzero, it is a pointer to a + * work-queue entry that CPT submits work to SSO after all context, + * output data, and result write operations are visible to other + * CNXXXX units and the cores. Bits <2:0> must be zero. + * Bits <63:49> are ignored by hardware; software should + * use a sign-extended bit <48> for forward compatibility. + * Internal: + * Bits <63:49>, <2:0> are ignored by hardware, treated as always 0x0. + * Word 4 + * ei0; [319:256] Engine instruction word 0. Passed to the AE/SE. + * Word 5 + * ei1; [383:320] Engine instruction word 1. Passed to the AE/SE. + * Word 6 + * ei2; [447:384] Engine instruction word 1. Passed to the AE/SE. + * Word 7 + * ei3; [511:448] Engine instruction word 1. Passed to the AE/SE. + * + */ +union otx2_cpt_inst_s { + u64 u[8]; + + struct { + /* Word 0 */ + u64 nixtxl:3; + u64 doneint:1; + u64 nixtx_addr:60; + /* Word 1 */ + u64 res_addr; + /* Word 2 */ + u64 tag:32; + u64 tt:2; + u64 grp:10; + u64 reserved_172_175:4; + u64 rvu_pf_func:16; + /* Word 3 */ + u64 qord:1; + u64 reserved_194_193:2; + u64 wq_ptr:61; + /* Word 4 */ + u64 ei0; + /* Word 5 */ + u64 ei1; + /* Word 6 */ + u64 ei2; + /* Word 7 */ + u64 ei3; + } s; +}; + +/* + * Structure otx2_cpt_res_s + * + * CPT Result Structure + * The CPT coprocessor writes the result structure after it completes a + * CPT_INST_S instruction. The result structure is exactly 16 bytes, and + * each instruction completion produces exactly one result structure. + * + * This structure is stored in memory as little-endian unless + * CPT()_PF_Q()_CTL[INST_BE] is set. + * cpt_res_s_s + * Word 0 + * doneint:1 [16:16] Done interrupt. This bit is copied from the + * corresponding instruction's CPT_INST_S[DONEINT]. + * compcode:8 [7:0] Indicates completion/error status of the CPT coprocessor + * for the associated instruction, as enumerated by CPT_COMP_E. + * Core software may write the memory location containing [COMPCODE] to + * 0x0 before ringing the doorbell, and then poll for completion by + * checking for a nonzero value. + * Once the core observes a nonzero [COMPCODE] value in this case,the CPT + * coprocessor will have also completed L2/DRAM write operations. + * Word 1 + * reserved + * + */ +union otx2_cpt_res_s { + u64 u[2]; + + struct cn9k_cpt_res_s { + u64 compcode:8; + u64 uc_compcode:8; + u64 doneint:1; + u64 reserved_17_63:47; + u64 reserved_64_127; + } s; + + struct cn10k_cpt_res_s { + u64 compcode:7; + u64 doneint:1; + u64 uc_compcode:8; + u64 rlen:16; + u64 spi:32; + u64 esn; + } cn10k; +}; + +/* + * Register (RVU_PF_BAR0) cpt#_af_constants1 + * + * CPT AF Constants Register + * This register contains implementation-related parameters of CPT. + */ +union otx2_cptx_af_constants1 { + u64 u; + struct otx2_cptx_af_constants1_s { + u64 se:16; + u64 ie:16; + u64 ae:16; + u64 reserved_48_63:16; + } s; +}; + +/* + * RVU_PFVF_BAR2 - cpt_lf_misc_int + * + * This register contain the per-queue miscellaneous interrupts. + * + */ +union otx2_cptx_lf_misc_int { + u64 u; + struct otx2_cptx_lf_misc_int_s { + u64 reserved_0:1; + u64 nqerr:1; + u64 irde:1; + u64 nwrp:1; + u64 reserved_4:1; + u64 hwerr:1; + u64 fault:1; + u64 reserved_7_63:57; + } s; +}; + +/* + * RVU_PFVF_BAR2 - cpt_lf_misc_int_ena_w1s + * + * This register sets interrupt enable bits. + * + */ +union otx2_cptx_lf_misc_int_ena_w1s { + u64 u; + struct otx2_cptx_lf_misc_int_ena_w1s_s { + u64 reserved_0:1; + u64 nqerr:1; + u64 irde:1; + u64 nwrp:1; + u64 reserved_4:1; + u64 hwerr:1; + u64 fault:1; + u64 reserved_7_63:57; + } s; +}; + +/* + * RVU_PFVF_BAR2 - cpt_lf_ctl + * + * This register configures the queue. + * + * When the queue is not execution-quiescent (see CPT_LF_INPROG[EENA,INFLIGHT]), + * software must only write this register with [ENA]=0. + */ +union otx2_cptx_lf_ctl { + u64 u; + struct otx2_cptx_lf_ctl_s { + u64 ena:1; + u64 fc_ena:1; + u64 fc_up_crossing:1; + u64 reserved_3:1; + u64 fc_hyst_bits:4; + u64 reserved_8_63:56; + } s; +}; + +/* + * RVU_PFVF_BAR2 - cpt_lf_done_wait + * + * This register specifies the per-queue interrupt coalescing settings. + */ +union otx2_cptx_lf_done_wait { + u64 u; + struct otx2_cptx_lf_done_wait_s { + u64 num_wait:20; + u64 reserved_20_31:12; + u64 time_wait:16; + u64 reserved_48_63:16; + } s; +}; + +/* + * RVU_PFVF_BAR2 - cpt_lf_done + * + * This register contain the per-queue instruction done count. + */ +union otx2_cptx_lf_done { + u64 u; + struct otx2_cptx_lf_done_s { + u64 done:20; + u64 reserved_20_63:44; + } s; +}; + +/* + * RVU_PFVF_BAR2 - cpt_lf_inprog + * + * These registers contain the per-queue instruction in flight registers. + * + */ +union otx2_cptx_lf_inprog { + u64 u; + struct otx2_cptx_lf_inprog_s { + u64 inflight:9; + u64 reserved_9_15:7; + u64 eena:1; + u64 grp_drp:1; + u64 reserved_18_30:13; + u64 grb_partial:1; + u64 grb_cnt:8; + u64 gwb_cnt:8; + u64 reserved_48_63:16; + } s; +}; + +/* + * RVU_PFVF_BAR2 - cpt_lf_q_base + * + * CPT initializes these CSR fields to these values on any CPT_LF_Q_BASE write: + * _ CPT_LF_Q_INST_PTR[XQ_XOR]=0. + * _ CPT_LF_Q_INST_PTR[NQ_PTR]=2. + * _ CPT_LF_Q_INST_PTR[DQ_PTR]=2. + * _ CPT_LF_Q_GRP_PTR[XQ_XOR]=0. + * _ CPT_LF_Q_GRP_PTR[NQ_PTR]=1. + * _ CPT_LF_Q_GRP_PTR[DQ_PTR]=1. + */ +union otx2_cptx_lf_q_base { + u64 u; + struct otx2_cptx_lf_q_base_s { + u64 fault:1; + u64 reserved_1_6:6; + u64 addr:46; + u64 reserved_53_63:11; + } s; +}; + +/* + * RVU_PFVF_BAR2 - cpt_lf_q_size + * + * CPT initializes these CSR fields to these values on any CPT_LF_Q_SIZE write: + * _ CPT_LF_Q_INST_PTR[XQ_XOR]=0. + * _ CPT_LF_Q_INST_PTR[NQ_PTR]=2. + * _ CPT_LF_Q_INST_PTR[DQ_PTR]=2. + * _ CPT_LF_Q_GRP_PTR[XQ_XOR]=0. + * _ CPT_LF_Q_GRP_PTR[NQ_PTR]=1. + * _ CPT_LF_Q_GRP_PTR[DQ_PTR]=1. + */ +union otx2_cptx_lf_q_size { + u64 u; + struct otx2_cptx_lf_q_size_s { + u64 size_div40:15; + u64 reserved_15_63:49; + } s; +}; + +/* + * RVU_PF_BAR0 - cpt_af_lf_ctl + * + * This register configures queues. This register should be written only + * when the queue is execution-quiescent (see CPT_LF_INPROG[INFLIGHT]). + */ +union otx2_cptx_af_lf_ctrl { + u64 u; + struct otx2_cptx_af_lf_ctrl_s { + u64 pri:1; + u64 reserved_1_8:8; + u64 pf_func_inst:1; + u64 cont_err:1; + u64 reserved_11_15:5; + u64 nixtx_en:1; + u64 ctx_ilen:3; + u64 reserved_17_47:28; + u64 grp:8; + u64 reserved_56_63:8; + } s; +}; + +#endif /* __OTX2_CPT_HW_TYPES_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_mbox_common.c b/drivers/crypto/marvell/octeontx2/otx2_cpt_mbox_common.c new file mode 100644 index 000000000..1542a110b --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_mbox_common.c @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include "otx2_cpt_common.h" +#include "otx2_cptlf.h" + +int otx2_cpt_send_mbox_msg(struct otx2_mbox *mbox, struct pci_dev *pdev) +{ + int ret; + + otx2_mbox_msg_send(mbox, 0); + ret = otx2_mbox_wait_for_rsp(mbox, 0); + if (ret == -EIO) { + dev_err(&pdev->dev, "RVU MBOX timeout.\n"); + return ret; + } else if (ret) { + dev_err(&pdev->dev, "RVU MBOX error: %d.\n", ret); + return -EFAULT; + } + return ret; +} +EXPORT_SYMBOL_NS_GPL(otx2_cpt_send_mbox_msg, "CRYPTO_DEV_OCTEONTX2_CPT"); + +int otx2_cpt_send_ready_msg(struct otx2_mbox *mbox, struct pci_dev *pdev) +{ + struct mbox_msghdr *req; + + req = otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct ready_msg_rsp)); + if (req == NULL) { + dev_err(&pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + req->id = MBOX_MSG_READY; + req->sig = OTX2_MBOX_REQ_SIG; + req->pcifunc = 0; + + return otx2_cpt_send_mbox_msg(mbox, pdev); +} +EXPORT_SYMBOL_NS_GPL(otx2_cpt_send_ready_msg, "CRYPTO_DEV_OCTEONTX2_CPT"); + +int otx2_cpt_send_af_reg_requests(struct otx2_mbox *mbox, struct pci_dev *pdev) +{ + return otx2_cpt_send_mbox_msg(mbox, pdev); +} +EXPORT_SYMBOL_NS_GPL(otx2_cpt_send_af_reg_requests, "CRYPTO_DEV_OCTEONTX2_CPT"); + +static int otx2_cpt_add_read_af_reg(struct otx2_mbox *mbox, + struct pci_dev *pdev, u64 reg, + u64 *val, int blkaddr) +{ + struct cpt_rd_wr_reg_msg *reg_msg; + + reg_msg = (struct cpt_rd_wr_reg_msg *) + otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*reg_msg), + sizeof(*reg_msg)); + if (reg_msg == NULL) { + dev_err(&pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + + reg_msg->hdr.id = MBOX_MSG_CPT_RD_WR_REGISTER; + reg_msg->hdr.sig = OTX2_MBOX_REQ_SIG; + reg_msg->hdr.pcifunc = 0; + + reg_msg->is_write = 0; + reg_msg->reg_offset = reg; + reg_msg->ret_val = val; + reg_msg->blkaddr = blkaddr; + + return 0; +} + +int otx2_cpt_add_write_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, + u64 reg, u64 val, int blkaddr) +{ + struct cpt_rd_wr_reg_msg *reg_msg; + + reg_msg = (struct cpt_rd_wr_reg_msg *) + otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*reg_msg), + sizeof(*reg_msg)); + if (reg_msg == NULL) { + dev_err(&pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + + reg_msg->hdr.id = MBOX_MSG_CPT_RD_WR_REGISTER; + reg_msg->hdr.sig = OTX2_MBOX_REQ_SIG; + reg_msg->hdr.pcifunc = 0; + + reg_msg->is_write = 1; + reg_msg->reg_offset = reg; + reg_msg->val = val; + reg_msg->blkaddr = blkaddr; + + return 0; +} +EXPORT_SYMBOL_NS_GPL(otx2_cpt_add_write_af_reg, "CRYPTO_DEV_OCTEONTX2_CPT"); + +int otx2_cpt_read_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, + u64 reg, u64 *val, int blkaddr) +{ + int ret; + + ret = otx2_cpt_add_read_af_reg(mbox, pdev, reg, val, blkaddr); + if (ret) + return ret; + + return otx2_cpt_send_mbox_msg(mbox, pdev); +} +EXPORT_SYMBOL_NS_GPL(otx2_cpt_read_af_reg, "CRYPTO_DEV_OCTEONTX2_CPT"); + +int otx2_cpt_write_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, + u64 reg, u64 val, int blkaddr) +{ + int ret; + + ret = otx2_cpt_add_write_af_reg(mbox, pdev, reg, val, blkaddr); + if (ret) + return ret; + + return otx2_cpt_send_mbox_msg(mbox, pdev); +} +EXPORT_SYMBOL_NS_GPL(otx2_cpt_write_af_reg, "CRYPTO_DEV_OCTEONTX2_CPT"); + +int otx2_cpt_attach_rscrs_msg(struct otx2_cptlfs_info *lfs) +{ + struct otx2_mbox *mbox = lfs->mbox; + struct rsrc_attach *req; + int ret; + + req = (struct rsrc_attach *) + otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct msg_rsp)); + if (req == NULL) { + dev_err(&lfs->pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + + req->hdr.id = MBOX_MSG_ATTACH_RESOURCES; + req->hdr.sig = OTX2_MBOX_REQ_SIG; + req->hdr.pcifunc = 0; + req->cptlfs = lfs->lfs_num; + req->cpt_blkaddr = lfs->blkaddr; + req->modify = 1; + ret = otx2_cpt_send_mbox_msg(mbox, lfs->pdev); + if (ret) + return ret; + + if (!lfs->are_lfs_attached) + ret = -EINVAL; + + return ret; +} + +int otx2_cpt_detach_rsrcs_msg(struct otx2_cptlfs_info *lfs) +{ + struct otx2_mbox *mbox = lfs->mbox; + struct rsrc_detach *req; + int ret; + + req = (struct rsrc_detach *) + otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct msg_rsp)); + if (req == NULL) { + dev_err(&lfs->pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + + req->hdr.id = MBOX_MSG_DETACH_RESOURCES; + req->hdr.sig = OTX2_MBOX_REQ_SIG; + req->hdr.pcifunc = 0; + req->cptlfs = 1; + ret = otx2_cpt_send_mbox_msg(mbox, lfs->pdev); + if (ret) + return ret; + + if (lfs->are_lfs_attached) + ret = -EINVAL; + + return ret; +} +EXPORT_SYMBOL_NS_GPL(otx2_cpt_detach_rsrcs_msg, "CRYPTO_DEV_OCTEONTX2_CPT"); + +int otx2_cpt_msix_offset_msg(struct otx2_cptlfs_info *lfs) +{ + struct otx2_mbox *mbox = lfs->mbox; + struct pci_dev *pdev = lfs->pdev; + struct mbox_msghdr *req; + int ret, i; + + req = otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct msix_offset_rsp)); + if (req == NULL) { + dev_err(&pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + + req->id = MBOX_MSG_MSIX_OFFSET; + req->sig = OTX2_MBOX_REQ_SIG; + req->pcifunc = 0; + ret = otx2_cpt_send_mbox_msg(mbox, pdev); + if (ret) + return ret; + + for (i = 0; i < lfs->lfs_num; i++) { + if (lfs->lf[i].msix_offset == MSIX_VECTOR_INVALID) { + dev_err(&pdev->dev, + "Invalid msix offset %d for LF %d\n", + lfs->lf[i].msix_offset, i); + return -EINVAL; + } + } + return ret; +} +EXPORT_SYMBOL_NS_GPL(otx2_cpt_msix_offset_msg, "CRYPTO_DEV_OCTEONTX2_CPT"); + +int otx2_cpt_sync_mbox_msg(struct otx2_mbox *mbox) +{ + int err; + + if (!otx2_mbox_nonempty(mbox, 0)) + return 0; + otx2_mbox_msg_send(mbox, 0); + err = otx2_mbox_wait_for_rsp(mbox, 0); + if (err) + return err; + + return otx2_mbox_check_rsp_msgs(mbox, 0); +} +EXPORT_SYMBOL_NS_GPL(otx2_cpt_sync_mbox_msg, "CRYPTO_DEV_OCTEONTX2_CPT"); + +int otx2_cpt_lf_reset_msg(struct otx2_cptlfs_info *lfs, int slot) +{ + struct otx2_mbox *mbox = lfs->mbox; + struct pci_dev *pdev = lfs->pdev; + struct cpt_lf_rst_req *req; + + req = (struct cpt_lf_rst_req *)otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct msg_rsp)); + if (!req) { + dev_err(&pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + + req->hdr.id = MBOX_MSG_CPT_LF_RESET; + req->hdr.sig = OTX2_MBOX_REQ_SIG; + req->hdr.pcifunc = 0; + req->slot = slot; + return otx2_cpt_send_mbox_msg(mbox, pdev); +} +EXPORT_SYMBOL_NS_GPL(otx2_cpt_lf_reset_msg, "CRYPTO_DEV_OCTEONTX2_CPT"); + +int otx2_cpt_lmtst_tbl_setup_msg(struct otx2_cptlfs_info *lfs) +{ + struct otx2_mbox *mbox = lfs->mbox; + struct pci_dev *pdev = lfs->pdev; + struct lmtst_tbl_setup_req *req; + + req = (struct lmtst_tbl_setup_req *) + otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct msg_rsp)); + if (!req) { + dev_err(&pdev->dev, "RVU MBOX failed to alloc message.\n"); + return -EFAULT; + } + + req->hdr.id = MBOX_MSG_LMTST_TBL_SETUP; + req->hdr.sig = OTX2_MBOX_REQ_SIG; + req->hdr.pcifunc = 0; + + req->use_local_lmt_region = true; + req->lmt_iova = lfs->lmt_info.iova; + + return otx2_cpt_send_mbox_msg(mbox, pdev); +} +EXPORT_SYMBOL_NS_GPL(otx2_cpt_lmtst_tbl_setup_msg, "CRYPTO_DEV_OCTEONTX2_CPT"); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h new file mode 100644 index 000000000..e64ca3033 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h @@ -0,0 +1,561 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2020 Marvell. + */ + +#ifndef __OTX2_CPT_REQMGR_H +#define __OTX2_CPT_REQMGR_H + +#include "otx2_cpt_common.h" + +/* Completion code size and initial value */ +#define OTX2_CPT_COMPLETION_CODE_SIZE 8 +#define OTX2_CPT_COMPLETION_CODE_INIT OTX2_CPT_COMP_E_NOTDONE +/* + * Maximum total number of SG buffers is 100, we divide it equally + * between input and output + */ +#define OTX2_CPT_MAX_SG_IN_CNT 50 +#define OTX2_CPT_MAX_SG_OUT_CNT 50 + +/* DMA mode direct or SG */ +#define OTX2_CPT_DMA_MODE_DIRECT 0 +#define OTX2_CPT_DMA_MODE_SG 1 + +/* Context source CPTR or DPTR */ +#define OTX2_CPT_FROM_CPTR 0 +#define OTX2_CPT_FROM_DPTR 1 + +#define OTX2_CPT_MAX_REQ_SIZE 65535 + +#define SG_COMPS_MAX 4 +#define SGV2_COMPS_MAX 3 + +#define SG_COMP_3 3 +#define SG_COMP_2 2 +#define SG_COMP_1 1 + +#define OTX2_CPT_DPTR_RPTR_ALIGN 8 +#define OTX2_CPT_RES_ADDR_ALIGN 32 + +union otx2_cpt_opcode { + u16 flags; + struct { + u8 major; + u8 minor; + } s; +}; + +struct otx2_cptvf_request { + u32 param1; + u32 param2; + u16 dlen; + union otx2_cpt_opcode opcode; + dma_addr_t cptr_dma; + void *cptr; +}; + +/* + * CPT_INST_S software command definitions + * Words EI (0-3) + */ +union otx2_cpt_iq_cmd_word0 { + u64 u; + struct { + __be16 opcode; + __be16 param1; + __be16 param2; + __be16 dlen; + } s; +}; + +union otx2_cpt_iq_cmd_word3 { + u64 u; + struct { + u64 cptr:61; + u64 grp:3; + } s; +}; + +struct otx2_cpt_iq_command { + union otx2_cpt_iq_cmd_word0 cmd; + u64 dptr; + u64 rptr; + union otx2_cpt_iq_cmd_word3 cptr; +}; + +struct otx2_cpt_pending_entry { + void *completion_addr; /* Completion address */ + void *info; + /* Kernel async request callback */ + void (*callback)(int status, void *arg1, void *arg2); + struct crypto_async_request *areq; /* Async request callback arg */ + u8 resume_sender; /* Notify sender to resume sending requests */ + u8 busy; /* Entry status (free/busy) */ +}; + +struct otx2_cpt_pending_queue { + struct otx2_cpt_pending_entry *head; /* Head of the queue */ + u32 front; /* Process work from here */ + u32 rear; /* Append new work here */ + u32 pending_count; /* Pending requests count */ + u32 qlen; /* Queue length */ + spinlock_t lock; /* Queue lock */ +}; + +struct otx2_cpt_buf_ptr { + u8 *vptr; + dma_addr_t dma_addr; + u16 size; +}; + +union otx2_cpt_ctrl_info { + u32 flags; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u32 reserved_6_31:26; + u32 grp:3; /* Group bits */ + u32 dma_mode:2; /* DMA mode */ + u32 se_req:1; /* To SE core */ +#else + u32 se_req:1; /* To SE core */ + u32 dma_mode:2; /* DMA mode */ + u32 grp:3; /* Group bits */ + u32 reserved_6_31:26; +#endif + } s; +}; + +struct otx2_cpt_req_info { + /* Kernel async request callback */ + void (*callback)(int status, void *arg1, void *arg2); + struct crypto_async_request *areq; /* Async request callback arg */ + struct otx2_cptvf_request req;/* Request information (core specific) */ + union otx2_cpt_ctrl_info ctrl;/* User control information */ + struct otx2_cpt_buf_ptr in[OTX2_CPT_MAX_SG_IN_CNT]; + struct otx2_cpt_buf_ptr out[OTX2_CPT_MAX_SG_OUT_CNT]; + u8 *iv_out; /* IV to send back */ + u16 rlen; /* Output length */ + u8 in_cnt; /* Number of input buffers */ + u8 out_cnt; /* Number of output buffers */ + u8 req_type; /* Type of request */ + u8 is_enc; /* Is a request an encryption request */ + u8 is_trunc_hmac;/* Is truncated hmac used */ +}; + +struct otx2_cpt_inst_info { + struct otx2_cpt_pending_entry *pentry; + struct otx2_cpt_req_info *req; + struct pci_dev *pdev; + void *completion_addr; + u8 *out_buffer; + u8 *in_buffer; + dma_addr_t dptr_baddr; + dma_addr_t rptr_baddr; + dma_addr_t comp_baddr; + unsigned long time_in; + u32 dlen; + u32 dma_len; + u64 gthr_sz; + u64 sctr_sz; + u8 extra_time; +}; + +struct otx2_cpt_sglist_component { + __be16 len0; + __be16 len1; + __be16 len2; + __be16 len3; + __be64 ptr0; + __be64 ptr1; + __be64 ptr2; + __be64 ptr3; +}; + +struct cn10kb_cpt_sglist_component { + u16 len0; + u16 len1; + u16 len2; + u16 valid_segs; + u64 ptr0; + u64 ptr1; + u64 ptr2; +}; + +static inline void otx2_cpt_info_destroy(struct pci_dev *pdev, + struct otx2_cpt_inst_info *info) +{ + struct otx2_cpt_req_info *req; + int i; + + if (info->dptr_baddr) + dma_unmap_single(&pdev->dev, info->dptr_baddr, + info->dma_len, DMA_BIDIRECTIONAL); + + if (info->req) { + req = info->req; + for (i = 0; i < req->out_cnt; i++) { + if (req->out[i].dma_addr) + dma_unmap_single(&pdev->dev, + req->out[i].dma_addr, + req->out[i].size, + DMA_BIDIRECTIONAL); + } + + for (i = 0; i < req->in_cnt; i++) { + if (req->in[i].dma_addr) + dma_unmap_single(&pdev->dev, + req->in[i].dma_addr, + req->in[i].size, + DMA_BIDIRECTIONAL); + } + } + kfree(info); +} + +static inline int setup_sgio_components(struct pci_dev *pdev, + struct otx2_cpt_buf_ptr *list, + int buf_count, u8 *buffer) +{ + struct otx2_cpt_sglist_component *sg_ptr; + int components; + int i, j; + + if (unlikely(!list)) { + dev_err(&pdev->dev, "Input list pointer is NULL\n"); + return -EINVAL; + } + + for (i = 0; i < buf_count; i++) { + if (unlikely(!list[i].vptr)) + continue; + list[i].dma_addr = dma_map_single(&pdev->dev, list[i].vptr, + list[i].size, + DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(&pdev->dev, list[i].dma_addr))) { + dev_err(&pdev->dev, "Dma mapping failed\n"); + goto sg_cleanup; + } + } + components = buf_count / SG_COMPS_MAX; + sg_ptr = (struct otx2_cpt_sglist_component *)buffer; + for (i = 0; i < components; i++) { + sg_ptr->len0 = cpu_to_be16(list[i * SG_COMPS_MAX + 0].size); + sg_ptr->len1 = cpu_to_be16(list[i * SG_COMPS_MAX + 1].size); + sg_ptr->len2 = cpu_to_be16(list[i * SG_COMPS_MAX + 2].size); + sg_ptr->len3 = cpu_to_be16(list[i * SG_COMPS_MAX + 3].size); + sg_ptr->ptr0 = cpu_to_be64(list[i * SG_COMPS_MAX + 0].dma_addr); + sg_ptr->ptr1 = cpu_to_be64(list[i * SG_COMPS_MAX + 1].dma_addr); + sg_ptr->ptr2 = cpu_to_be64(list[i * SG_COMPS_MAX + 2].dma_addr); + sg_ptr->ptr3 = cpu_to_be64(list[i * SG_COMPS_MAX + 3].dma_addr); + sg_ptr++; + } + components = buf_count % SG_COMPS_MAX; + + switch (components) { + case SG_COMP_3: + sg_ptr->len2 = cpu_to_be16(list[i * SG_COMPS_MAX + 2].size); + sg_ptr->ptr2 = cpu_to_be64(list[i * SG_COMPS_MAX + 2].dma_addr); + fallthrough; + case SG_COMP_2: + sg_ptr->len1 = cpu_to_be16(list[i * SG_COMPS_MAX + 1].size); + sg_ptr->ptr1 = cpu_to_be64(list[i * SG_COMPS_MAX + 1].dma_addr); + fallthrough; + case SG_COMP_1: + sg_ptr->len0 = cpu_to_be16(list[i * SG_COMPS_MAX + 0].size); + sg_ptr->ptr0 = cpu_to_be64(list[i * SG_COMPS_MAX + 0].dma_addr); + break; + default: + break; + } + return 0; + +sg_cleanup: + for (j = 0; j < i; j++) { + if (list[j].dma_addr) { + dma_unmap_single(&pdev->dev, list[j].dma_addr, + list[j].size, DMA_BIDIRECTIONAL); + } + + list[j].dma_addr = 0; + } + return -EIO; +} + +static inline int sgv2io_components_setup(struct pci_dev *pdev, + struct otx2_cpt_buf_ptr *list, + int buf_count, u8 *buffer) +{ + struct cn10kb_cpt_sglist_component *sg_ptr; + int components; + int i, j; + + if (unlikely(!list)) { + dev_err(&pdev->dev, "Input list pointer is NULL\n"); + return -EFAULT; + } + + for (i = 0; i < buf_count; i++) { + if (unlikely(!list[i].vptr)) + continue; + list[i].dma_addr = dma_map_single(&pdev->dev, list[i].vptr, + list[i].size, + DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(&pdev->dev, list[i].dma_addr))) { + dev_err(&pdev->dev, "Dma mapping failed\n"); + goto sg_cleanup; + } + } + components = buf_count / SGV2_COMPS_MAX; + sg_ptr = (struct cn10kb_cpt_sglist_component *)buffer; + for (i = 0; i < components; i++) { + sg_ptr->len0 = list[i * SGV2_COMPS_MAX + 0].size; + sg_ptr->len1 = list[i * SGV2_COMPS_MAX + 1].size; + sg_ptr->len2 = list[i * SGV2_COMPS_MAX + 2].size; + sg_ptr->ptr0 = list[i * SGV2_COMPS_MAX + 0].dma_addr; + sg_ptr->ptr1 = list[i * SGV2_COMPS_MAX + 1].dma_addr; + sg_ptr->ptr2 = list[i * SGV2_COMPS_MAX + 2].dma_addr; + sg_ptr->valid_segs = SGV2_COMPS_MAX; + sg_ptr++; + } + components = buf_count % SGV2_COMPS_MAX; + + sg_ptr->valid_segs = components; + switch (components) { + case SG_COMP_2: + sg_ptr->len1 = list[i * SGV2_COMPS_MAX + 1].size; + sg_ptr->ptr1 = list[i * SGV2_COMPS_MAX + 1].dma_addr; + fallthrough; + case SG_COMP_1: + sg_ptr->len0 = list[i * SGV2_COMPS_MAX + 0].size; + sg_ptr->ptr0 = list[i * SGV2_COMPS_MAX + 0].dma_addr; + break; + default: + break; + } + return 0; + +sg_cleanup: + for (j = 0; j < i; j++) { + if (list[j].dma_addr) { + dma_unmap_single(&pdev->dev, list[j].dma_addr, + list[j].size, DMA_BIDIRECTIONAL); + } + + list[j].dma_addr = 0; + } + return -EIO; +} + +static inline struct otx2_cpt_inst_info * +cn10k_sgv2_info_create(struct pci_dev *pdev, struct otx2_cpt_req_info *req, + gfp_t gfp) +{ + u32 dlen = 0, g_len, s_len, sg_len, info_len; + struct otx2_cpt_inst_info *info; + u32 total_mem_len; + int i; + + /* Allocate memory to meet below alignment requirement: + * ------------------------------------ + * | struct otx2_cpt_inst_info | + * | (No alignment required) | + * | --------------------------------| + * | | padding for ARCH_DMA_MINALIGN | + * | | alignment | + * |------------------------------------| + * | SG List Gather/Input memory | + * | Length = multiple of 32Bytes | + * | Alignment = 8Byte | + * |---------------------------------- | + * | SG List Scatter/Output memory | + * | Length = multiple of 32Bytes | + * | Alignment = 8Byte | + * | -------------------------------| + * | | padding for 32B alignment | + * |------------------------------------| + * | Result response memory | + * | Alignment = 32Byte | + * ------------------------------------ + */ + + info_len = sizeof(*info); + + g_len = ((req->in_cnt + 2) / 3) * + sizeof(struct cn10kb_cpt_sglist_component); + s_len = ((req->out_cnt + 2) / 3) * + sizeof(struct cn10kb_cpt_sglist_component); + sg_len = g_len + s_len; + + /* Allocate extra memory for SG and response address alignment */ + total_mem_len = ALIGN(info_len, OTX2_CPT_DPTR_RPTR_ALIGN); + total_mem_len += (ARCH_DMA_MINALIGN - 1) & + ~(OTX2_CPT_DPTR_RPTR_ALIGN - 1); + total_mem_len += ALIGN(sg_len, OTX2_CPT_RES_ADDR_ALIGN); + total_mem_len += sizeof(union otx2_cpt_res_s); + + info = kzalloc(total_mem_len, gfp); + if (unlikely(!info)) + return NULL; + + for (i = 0; i < req->in_cnt; i++) + dlen += req->in[i].size; + + info->dlen = dlen; + info->in_buffer = PTR_ALIGN((u8 *)info + info_len, ARCH_DMA_MINALIGN); + info->out_buffer = info->in_buffer + g_len; + info->gthr_sz = req->in_cnt; + info->sctr_sz = req->out_cnt; + + /* Setup gather (input) components */ + if (sgv2io_components_setup(pdev, req->in, req->in_cnt, + info->in_buffer)) { + dev_err(&pdev->dev, "Failed to setup gather list\n"); + goto destroy_info; + } + + if (sgv2io_components_setup(pdev, req->out, req->out_cnt, + info->out_buffer)) { + dev_err(&pdev->dev, "Failed to setup scatter list\n"); + goto destroy_info; + } + + info->dma_len = total_mem_len - info_len; + info->dptr_baddr = dma_map_single(&pdev->dev, info->in_buffer, + info->dma_len, DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(&pdev->dev, info->dptr_baddr))) { + dev_err(&pdev->dev, "DMA Mapping failed for cpt req\n"); + goto destroy_info; + } + info->rptr_baddr = info->dptr_baddr + g_len; + /* + * Get buffer for union otx2_cpt_res_s response + * structure and its physical address + */ + info->completion_addr = PTR_ALIGN((info->in_buffer + sg_len), + OTX2_CPT_RES_ADDR_ALIGN); + info->comp_baddr = ALIGN((info->dptr_baddr + sg_len), + OTX2_CPT_RES_ADDR_ALIGN); + + return info; + +destroy_info: + otx2_cpt_info_destroy(pdev, info); + return NULL; +} + +/* SG list header size in bytes */ +#define SG_LIST_HDR_SIZE 8 +static inline struct otx2_cpt_inst_info * +otx2_sg_info_create(struct pci_dev *pdev, struct otx2_cpt_req_info *req, + gfp_t gfp) +{ + struct otx2_cpt_inst_info *info; + u32 dlen, info_len; + u16 g_len, s_len; + u32 total_mem_len; + + if (unlikely(req->in_cnt > OTX2_CPT_MAX_SG_IN_CNT || + req->out_cnt > OTX2_CPT_MAX_SG_OUT_CNT)) { + dev_err(&pdev->dev, "Error too many sg components\n"); + return NULL; + } + + /* Allocate memory to meet below alignment requirement: + * ------------------------------------ + * | struct otx2_cpt_inst_info | + * | (No alignment required) | + * | --------------------------------| + * | | padding for ARCH_DMA_MINALIGN | + * | | alignment | + * |------------------------------------| + * | SG List Header of 8 Byte | + * |------------------------------------| + * | SG List Gather/Input memory | + * | Length = multiple of 32Bytes | + * | Alignment = 8Byte | + * |---------------------------------- | + * | SG List Scatter/Output memory | + * | Length = multiple of 32Bytes | + * | Alignment = 8Byte | + * | -------------------------------| + * | | padding for 32B alignment | + * |------------------------------------| + * | Result response memory | + * | Alignment = 32Byte | + * ------------------------------------ + */ + + info_len = sizeof(*info); + + g_len = ((req->in_cnt + 3) / 4) * + sizeof(struct otx2_cpt_sglist_component); + s_len = ((req->out_cnt + 3) / 4) * + sizeof(struct otx2_cpt_sglist_component); + + dlen = g_len + s_len + SG_LIST_HDR_SIZE; + + /* Allocate extra memory for SG and response address alignment */ + total_mem_len = ALIGN(info_len, OTX2_CPT_DPTR_RPTR_ALIGN); + total_mem_len += (ARCH_DMA_MINALIGN - 1) & + ~(OTX2_CPT_DPTR_RPTR_ALIGN - 1); + total_mem_len += ALIGN(dlen, OTX2_CPT_RES_ADDR_ALIGN); + total_mem_len += sizeof(union otx2_cpt_res_s); + + info = kzalloc(total_mem_len, gfp); + if (unlikely(!info)) + return NULL; + + info->dlen = dlen; + info->in_buffer = PTR_ALIGN((u8 *)info + info_len, ARCH_DMA_MINALIGN); + info->out_buffer = info->in_buffer + SG_LIST_HDR_SIZE + g_len; + + ((u16 *)info->in_buffer)[0] = req->out_cnt; + ((u16 *)info->in_buffer)[1] = req->in_cnt; + ((u16 *)info->in_buffer)[2] = 0; + ((u16 *)info->in_buffer)[3] = 0; + cpu_to_be64s((u64 *)info->in_buffer); + + /* Setup gather (input) components */ + if (setup_sgio_components(pdev, req->in, req->in_cnt, + &info->in_buffer[8])) { + dev_err(&pdev->dev, "Failed to setup gather list\n"); + goto destroy_info; + } + + if (setup_sgio_components(pdev, req->out, req->out_cnt, + info->out_buffer)) { + dev_err(&pdev->dev, "Failed to setup scatter list\n"); + goto destroy_info; + } + + info->dma_len = total_mem_len - info_len; + info->dptr_baddr = dma_map_single(&pdev->dev, info->in_buffer, + info->dma_len, DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(&pdev->dev, info->dptr_baddr))) { + dev_err(&pdev->dev, "DMA Mapping failed for cpt req\n"); + goto destroy_info; + } + /* + * Get buffer for union otx2_cpt_res_s response + * structure and its physical address + */ + info->completion_addr = PTR_ALIGN((info->in_buffer + dlen), + OTX2_CPT_RES_ADDR_ALIGN); + info->comp_baddr = ALIGN((info->dptr_baddr + dlen), + OTX2_CPT_RES_ADDR_ALIGN); + + return info; + +destroy_info: + otx2_cpt_info_destroy(pdev, info); + return NULL; +} + +struct otx2_cptlf_wqe; +int otx2_cpt_do_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, + int cpu_num); +void otx2_cpt_post_process(struct otx2_cptlf_wqe *wqe); +int otx2_cpt_get_eng_grp_num(struct pci_dev *pdev, + enum otx2_cpt_eng_type); + +#endif /* __OTX2_CPT_REQMGR_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptlf.c b/drivers/crypto/marvell/octeontx2/otx2_cptlf.c new file mode 100644 index 000000000..dc7c7a265 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptlf.c @@ -0,0 +1,498 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include "otx2_cpt_common.h" +#include "otx2_cptlf.h" +#include "rvu_reg.h" + +#define CPT_TIMER_HOLD 0x03F +#define CPT_COUNT_HOLD 32 + +static void cptlf_do_set_done_time_wait(struct otx2_cptlf_info *lf, + int time_wait) +{ + union otx2_cptx_lf_done_wait done_wait; + + done_wait.u = otx2_cpt_read64(lf->lfs->reg_base, lf->lfs->blkaddr, + lf->slot, OTX2_CPT_LF_DONE_WAIT); + done_wait.s.time_wait = time_wait; + otx2_cpt_write64(lf->lfs->reg_base, lf->lfs->blkaddr, lf->slot, + OTX2_CPT_LF_DONE_WAIT, done_wait.u); +} + +static void cptlf_do_set_done_num_wait(struct otx2_cptlf_info *lf, int num_wait) +{ + union otx2_cptx_lf_done_wait done_wait; + + done_wait.u = otx2_cpt_read64(lf->lfs->reg_base, lf->lfs->blkaddr, + lf->slot, OTX2_CPT_LF_DONE_WAIT); + done_wait.s.num_wait = num_wait; + otx2_cpt_write64(lf->lfs->reg_base, lf->lfs->blkaddr, lf->slot, + OTX2_CPT_LF_DONE_WAIT, done_wait.u); +} + +static void cptlf_set_done_time_wait(struct otx2_cptlfs_info *lfs, + int time_wait) +{ + int slot; + + for (slot = 0; slot < lfs->lfs_num; slot++) + cptlf_do_set_done_time_wait(&lfs->lf[slot], time_wait); +} + +static void cptlf_set_done_num_wait(struct otx2_cptlfs_info *lfs, int num_wait) +{ + int slot; + + for (slot = 0; slot < lfs->lfs_num; slot++) + cptlf_do_set_done_num_wait(&lfs->lf[slot], num_wait); +} + +static int cptlf_set_pri(struct otx2_cptlf_info *lf, int pri) +{ + struct otx2_cptlfs_info *lfs = lf->lfs; + union otx2_cptx_af_lf_ctrl lf_ctrl; + int ret; + + ret = otx2_cpt_read_af_reg(lfs->mbox, lfs->pdev, + CPT_AF_LFX_CTL(lf->slot), + &lf_ctrl.u, lfs->blkaddr); + if (ret) + return ret; + + lf_ctrl.s.pri = pri ? 1 : 0; + + ret = otx2_cpt_write_af_reg(lfs->mbox, lfs->pdev, + CPT_AF_LFX_CTL(lf->slot), + lf_ctrl.u, lfs->blkaddr); + return ret; +} + +static int cptlf_set_eng_grps_mask(struct otx2_cptlf_info *lf, + int eng_grps_mask) +{ + struct otx2_cptlfs_info *lfs = lf->lfs; + union otx2_cptx_af_lf_ctrl lf_ctrl; + int ret; + + ret = otx2_cpt_read_af_reg(lfs->mbox, lfs->pdev, + CPT_AF_LFX_CTL(lf->slot), + &lf_ctrl.u, lfs->blkaddr); + if (ret) + return ret; + + lf_ctrl.s.grp = eng_grps_mask; + + ret = otx2_cpt_write_af_reg(lfs->mbox, lfs->pdev, + CPT_AF_LFX_CTL(lf->slot), + lf_ctrl.u, lfs->blkaddr); + return ret; +} + +static int cptlf_set_grp_and_pri(struct otx2_cptlfs_info *lfs, + int eng_grp_mask, int pri) +{ + int slot, ret = 0; + + for (slot = 0; slot < lfs->lfs_num; slot++) { + ret = cptlf_set_pri(&lfs->lf[slot], pri); + if (ret) + return ret; + + ret = cptlf_set_eng_grps_mask(&lfs->lf[slot], eng_grp_mask); + if (ret) + return ret; + } + return ret; +} + +static int cptlf_set_ctx_ilen(struct otx2_cptlfs_info *lfs, int ctx_ilen) +{ + union otx2_cptx_af_lf_ctrl lf_ctrl; + struct otx2_cptlf_info *lf; + int slot, ret = 0; + + for (slot = 0; slot < lfs->lfs_num; slot++) { + lf = &lfs->lf[slot]; + + ret = otx2_cpt_read_af_reg(lfs->mbox, lfs->pdev, + CPT_AF_LFX_CTL(lf->slot), + &lf_ctrl.u, lfs->blkaddr); + if (ret) + return ret; + + lf_ctrl.s.ctx_ilen = ctx_ilen; + + ret = otx2_cpt_write_af_reg(lfs->mbox, lfs->pdev, + CPT_AF_LFX_CTL(lf->slot), + lf_ctrl.u, lfs->blkaddr); + if (ret) + return ret; + } + return ret; +} + +static void cptlf_hw_init(struct otx2_cptlfs_info *lfs) +{ + /* Disable instruction queues */ + otx2_cptlf_disable_iqueues(lfs); + + /* Set instruction queues base addresses */ + otx2_cptlf_set_iqueues_base_addr(lfs); + + /* Set instruction queues sizes */ + otx2_cptlf_set_iqueues_size(lfs); + + /* Set done interrupts time wait */ + cptlf_set_done_time_wait(lfs, CPT_TIMER_HOLD); + + /* Set done interrupts num wait */ + cptlf_set_done_num_wait(lfs, CPT_COUNT_HOLD); + + /* Enable instruction queues */ + otx2_cptlf_enable_iqueues(lfs); +} + +static void cptlf_hw_cleanup(struct otx2_cptlfs_info *lfs) +{ + /* Disable instruction queues */ + otx2_cptlf_disable_iqueues(lfs); +} + +static void cptlf_set_misc_intrs(struct otx2_cptlfs_info *lfs, u8 enable) +{ + union otx2_cptx_lf_misc_int_ena_w1s irq_misc = { .u = 0x0 }; + u64 reg = enable ? OTX2_CPT_LF_MISC_INT_ENA_W1S : + OTX2_CPT_LF_MISC_INT_ENA_W1C; + int slot; + + irq_misc.s.fault = 0x1; + irq_misc.s.hwerr = 0x1; + irq_misc.s.irde = 0x1; + irq_misc.s.nqerr = 0x1; + irq_misc.s.nwrp = 0x1; + + for (slot = 0; slot < lfs->lfs_num; slot++) + otx2_cpt_write64(lfs->reg_base, lfs->blkaddr, slot, reg, + irq_misc.u); +} + +static void cptlf_set_done_intrs(struct otx2_cptlfs_info *lfs, u8 enable) +{ + u64 reg = enable ? OTX2_CPT_LF_DONE_INT_ENA_W1S : + OTX2_CPT_LF_DONE_INT_ENA_W1C; + int slot; + + for (slot = 0; slot < lfs->lfs_num; slot++) + otx2_cpt_write64(lfs->reg_base, lfs->blkaddr, slot, reg, 0x1); +} + +static inline int cptlf_read_done_cnt(struct otx2_cptlf_info *lf) +{ + union otx2_cptx_lf_done irq_cnt; + + irq_cnt.u = otx2_cpt_read64(lf->lfs->reg_base, lf->lfs->blkaddr, lf->slot, + OTX2_CPT_LF_DONE); + return irq_cnt.s.done; +} + +static irqreturn_t cptlf_misc_intr_handler(int __always_unused irq, void *arg) +{ + union otx2_cptx_lf_misc_int irq_misc, irq_misc_ack; + struct otx2_cptlf_info *lf = arg; + struct device *dev; + + dev = &lf->lfs->pdev->dev; + irq_misc.u = otx2_cpt_read64(lf->lfs->reg_base, lf->lfs->blkaddr, + lf->slot, OTX2_CPT_LF_MISC_INT); + irq_misc_ack.u = 0x0; + + if (irq_misc.s.fault) { + dev_err(dev, "Memory error detected while executing CPT_INST_S, LF %d.\n", + lf->slot); + irq_misc_ack.s.fault = 0x1; + + } else if (irq_misc.s.hwerr) { + dev_err(dev, "HW error from an engine executing CPT_INST_S, LF %d.", + lf->slot); + irq_misc_ack.s.hwerr = 0x1; + + } else if (irq_misc.s.nwrp) { + dev_err(dev, "SMMU fault while writing CPT_RES_S to CPT_INST_S[RES_ADDR], LF %d.\n", + lf->slot); + irq_misc_ack.s.nwrp = 0x1; + + } else if (irq_misc.s.irde) { + dev_err(dev, "Memory error when accessing instruction memory queue CPT_LF_Q_BASE[ADDR].\n"); + irq_misc_ack.s.irde = 0x1; + + } else if (irq_misc.s.nqerr) { + dev_err(dev, "Error enqueuing an instruction received at CPT_LF_NQ.\n"); + irq_misc_ack.s.nqerr = 0x1; + + } else { + dev_err(dev, "Unhandled interrupt in CPT LF %d\n", lf->slot); + return IRQ_NONE; + } + + /* Acknowledge interrupts */ + otx2_cpt_write64(lf->lfs->reg_base, lf->lfs->blkaddr, lf->slot, + OTX2_CPT_LF_MISC_INT, irq_misc_ack.u); + + return IRQ_HANDLED; +} + +static irqreturn_t cptlf_done_intr_handler(int irq, void *arg) +{ + union otx2_cptx_lf_done_wait done_wait; + struct otx2_cptlf_info *lf = arg; + int irq_cnt; + + /* Read the number of completed requests */ + irq_cnt = cptlf_read_done_cnt(lf); + if (irq_cnt) { + done_wait.u = otx2_cpt_read64(lf->lfs->reg_base, lf->lfs->blkaddr, + lf->slot, OTX2_CPT_LF_DONE_WAIT); + /* Acknowledge the number of completed requests */ + otx2_cpt_write64(lf->lfs->reg_base, lf->lfs->blkaddr, lf->slot, + OTX2_CPT_LF_DONE_ACK, irq_cnt); + + otx2_cpt_write64(lf->lfs->reg_base, lf->lfs->blkaddr, lf->slot, + OTX2_CPT_LF_DONE_WAIT, done_wait.u); + if (unlikely(!lf->wqe)) { + dev_err(&lf->lfs->pdev->dev, "No work for LF %d\n", + lf->slot); + return IRQ_NONE; + } + + /* Schedule processing of completed requests */ + tasklet_hi_schedule(&lf->wqe->work); + } + return IRQ_HANDLED; +} + +void otx2_cptlf_unregister_misc_interrupts(struct otx2_cptlfs_info *lfs) +{ + int i, irq_offs, vector; + + irq_offs = OTX2_CPT_LF_INT_VEC_E_MISC; + for (i = 0; i < lfs->lfs_num; i++) { + if (!lfs->lf[i].is_irq_reg[irq_offs]) + continue; + + vector = pci_irq_vector(lfs->pdev, + lfs->lf[i].msix_offset + irq_offs); + free_irq(vector, &lfs->lf[i]); + lfs->lf[i].is_irq_reg[irq_offs] = false; + } + + cptlf_set_misc_intrs(lfs, false); +} +EXPORT_SYMBOL_NS_GPL(otx2_cptlf_unregister_misc_interrupts, "CRYPTO_DEV_OCTEONTX2_CPT"); + +void otx2_cptlf_unregister_done_interrupts(struct otx2_cptlfs_info *lfs) +{ + int i, irq_offs, vector; + + irq_offs = OTX2_CPT_LF_INT_VEC_E_DONE; + for (i = 0; i < lfs->lfs_num; i++) { + if (!lfs->lf[i].is_irq_reg[irq_offs]) + continue; + + vector = pci_irq_vector(lfs->pdev, + lfs->lf[i].msix_offset + irq_offs); + free_irq(vector, &lfs->lf[i]); + lfs->lf[i].is_irq_reg[irq_offs] = false; + } + + cptlf_set_done_intrs(lfs, false); +} +EXPORT_SYMBOL_NS_GPL(otx2_cptlf_unregister_done_interrupts, "CRYPTO_DEV_OCTEONTX2_CPT"); + +static int cptlf_do_register_interrrupts(struct otx2_cptlfs_info *lfs, + int lf_num, int irq_offset, + irq_handler_t handler) +{ + int ret, vector; + + vector = pci_irq_vector(lfs->pdev, lfs->lf[lf_num].msix_offset + + irq_offset); + ret = request_irq(vector, handler, 0, + lfs->lf[lf_num].irq_name[irq_offset], + &lfs->lf[lf_num]); + if (ret) + return ret; + + lfs->lf[lf_num].is_irq_reg[irq_offset] = true; + + return ret; +} + +int otx2_cptlf_register_misc_interrupts(struct otx2_cptlfs_info *lfs) +{ + bool is_cpt1 = (lfs->blkaddr == BLKADDR_CPT1); + int irq_offs, ret, i; + + irq_offs = OTX2_CPT_LF_INT_VEC_E_MISC; + for (i = 0; i < lfs->lfs_num; i++) { + snprintf(lfs->lf[i].irq_name[irq_offs], 32, "CPT%dLF Misc%d", + is_cpt1, i); + ret = cptlf_do_register_interrrupts(lfs, i, irq_offs, + cptlf_misc_intr_handler); + if (ret) + goto free_irq; + } + cptlf_set_misc_intrs(lfs, true); + return 0; + +free_irq: + otx2_cptlf_unregister_misc_interrupts(lfs); + return ret; +} +EXPORT_SYMBOL_NS_GPL(otx2_cptlf_register_misc_interrupts, "CRYPTO_DEV_OCTEONTX2_CPT"); + +int otx2_cptlf_register_done_interrupts(struct otx2_cptlfs_info *lfs) +{ + bool is_cpt1 = (lfs->blkaddr == BLKADDR_CPT1); + int irq_offs, ret, i; + + irq_offs = OTX2_CPT_LF_INT_VEC_E_DONE; + for (i = 0; i < lfs->lfs_num; i++) { + snprintf(lfs->lf[i].irq_name[irq_offs], 32, + "OTX2_CPT%dLF Done%d", is_cpt1, i); + ret = cptlf_do_register_interrrupts(lfs, i, irq_offs, + cptlf_done_intr_handler); + if (ret) + goto free_irq; + } + cptlf_set_done_intrs(lfs, true); + return 0; + +free_irq: + otx2_cptlf_unregister_done_interrupts(lfs); + return ret; +} +EXPORT_SYMBOL_NS_GPL(otx2_cptlf_register_done_interrupts, "CRYPTO_DEV_OCTEONTX2_CPT"); + +void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs) +{ + int slot, offs; + + for (slot = 0; slot < lfs->lfs_num; slot++) { + for (offs = 0; offs < OTX2_CPT_LF_MSIX_VECTORS; offs++) + irq_set_affinity_hint(pci_irq_vector(lfs->pdev, + lfs->lf[slot].msix_offset + + offs), NULL); + free_cpumask_var(lfs->lf[slot].affinity_mask); + } +} +EXPORT_SYMBOL_NS_GPL(otx2_cptlf_free_irqs_affinity, "CRYPTO_DEV_OCTEONTX2_CPT"); + +int otx2_cptlf_set_irqs_affinity(struct otx2_cptlfs_info *lfs) +{ + struct otx2_cptlf_info *lf = lfs->lf; + int slot, offs, ret; + + for (slot = 0; slot < lfs->lfs_num; slot++) { + if (!zalloc_cpumask_var(&lf[slot].affinity_mask, GFP_KERNEL)) { + dev_err(&lfs->pdev->dev, + "cpumask allocation failed for LF %d", slot); + ret = -ENOMEM; + goto free_affinity_mask; + } + + cpumask_set_cpu(cpumask_local_spread(slot, + dev_to_node(&lfs->pdev->dev)), + lf[slot].affinity_mask); + + for (offs = 0; offs < OTX2_CPT_LF_MSIX_VECTORS; offs++) { + ret = irq_set_affinity_hint(pci_irq_vector(lfs->pdev, + lf[slot].msix_offset + offs), + lf[slot].affinity_mask); + if (ret) + goto free_affinity_mask; + } + } + return 0; + +free_affinity_mask: + otx2_cptlf_free_irqs_affinity(lfs); + return ret; +} +EXPORT_SYMBOL_NS_GPL(otx2_cptlf_set_irqs_affinity, "CRYPTO_DEV_OCTEONTX2_CPT"); + +int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_mask, int pri, + int lfs_num) +{ + int slot, ret; + + if (!lfs->pdev || !lfs->reg_base) + return -EINVAL; + + lfs->lfs_num = lfs_num; + for (slot = 0; slot < lfs->lfs_num; slot++) { + lfs->lf[slot].lfs = lfs; + lfs->lf[slot].slot = slot; + if (!lfs->lmt_info.base) + lfs->lf[slot].lmtline = lfs->reg_base + + OTX2_CPT_RVU_FUNC_ADDR_S(BLKADDR_LMT, slot, + OTX2_CPT_LMT_LF_LMTLINEX(0)); + + lfs->lf[slot].ioreg = lfs->reg_base + + OTX2_CPT_RVU_FUNC_ADDR_S(lfs->blkaddr, slot, + OTX2_CPT_LF_NQX(0)); + } + /* Send request to attach LFs */ + ret = otx2_cpt_attach_rscrs_msg(lfs); + if (ret) + goto clear_lfs_num; + + ret = otx2_cpt_alloc_instruction_queues(lfs); + if (ret) { + dev_err(&lfs->pdev->dev, + "Allocating instruction queues failed\n"); + goto detach_rsrcs; + } + cptlf_hw_init(lfs); + /* + * Allow each LF to execute requests destined to any of 8 engine + * groups and set queue priority of each LF to high + */ + ret = cptlf_set_grp_and_pri(lfs, eng_grp_mask, pri); + if (ret) + goto free_iq; + + if (lfs->ctx_ilen_ovrd) { + ret = cptlf_set_ctx_ilen(lfs, lfs->ctx_ilen); + if (ret) + goto free_iq; + } + + return 0; + +free_iq: + cptlf_hw_cleanup(lfs); + otx2_cpt_free_instruction_queues(lfs); +detach_rsrcs: + otx2_cpt_detach_rsrcs_msg(lfs); +clear_lfs_num: + lfs->lfs_num = 0; + return ret; +} +EXPORT_SYMBOL_NS_GPL(otx2_cptlf_init, "CRYPTO_DEV_OCTEONTX2_CPT"); + +void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs) +{ + /* Cleanup LFs hardware side */ + cptlf_hw_cleanup(lfs); + /* Free instruction queues */ + otx2_cpt_free_instruction_queues(lfs); + /* Send request to detach LFs */ + otx2_cpt_detach_rsrcs_msg(lfs); + lfs->lfs_num = 0; +} +EXPORT_SYMBOL_NS_GPL(otx2_cptlf_shutdown, "CRYPTO_DEV_OCTEONTX2_CPT"); + +MODULE_AUTHOR("Marvell"); +MODULE_DESCRIPTION("Marvell RVU CPT Common module"); +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptlf.h b/drivers/crypto/marvell/octeontx2/otx2_cptlf.h new file mode 100644 index 000000000..1b9f75214 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptlf.h @@ -0,0 +1,448 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2020 Marvell. + */ +#ifndef __OTX2_CPTLF_H +#define __OTX2_CPTLF_H + +#include +#include +#include +#include +#include "otx2_cpt_common.h" +#include "otx2_cpt_reqmgr.h" + +/* + * CPT instruction and pending queues user requested length in CPT_INST_S msgs + */ +#define OTX2_CPT_USER_REQUESTED_QLEN_MSGS 8200 + +/* + * CPT instruction queue size passed to HW is in units of 40*CPT_INST_S + * messages. + */ +#define OTX2_CPT_SIZE_DIV40 (OTX2_CPT_USER_REQUESTED_QLEN_MSGS/40) + +/* + * CPT instruction and pending queues length in CPT_INST_S messages + */ +#define OTX2_CPT_INST_QLEN_MSGS ((OTX2_CPT_SIZE_DIV40 - 1) * 40) + +/* + * LDWB is getting incorrectly used when IQB_LDWB = 1 and CPT instruction + * queue has less than 320 free entries. So, increase HW instruction queue + * size by 320 and give 320 entries less for SW/NIX RX as a workaround. + */ +#define OTX2_CPT_INST_QLEN_EXTRA_BYTES (320 * OTX2_CPT_INST_SIZE) +#define OTX2_CPT_EXTRA_SIZE_DIV40 (320/40) + +/* CPT instruction queue length in bytes */ +#define OTX2_CPT_INST_QLEN_BYTES \ + ((OTX2_CPT_SIZE_DIV40 * 40 * OTX2_CPT_INST_SIZE) + \ + OTX2_CPT_INST_QLEN_EXTRA_BYTES) + +/* CPT instruction group queue length in bytes */ +#define OTX2_CPT_INST_GRP_QLEN_BYTES \ + ((OTX2_CPT_SIZE_DIV40 + OTX2_CPT_EXTRA_SIZE_DIV40) * 16) + +/* CPT FC length in bytes */ +#define OTX2_CPT_Q_FC_LEN 128 + +/* CPT instruction queue alignment */ +#define OTX2_CPT_INST_Q_ALIGNMENT 128 + +/* Mask which selects all engine groups */ +#define OTX2_CPT_ALL_ENG_GRPS_MASK 0xFF + +/* Maximum LFs supported in OcteonTX2 for CPT */ +#define OTX2_CPT_MAX_LFS_NUM 64 + +/* Queue priority */ +#define OTX2_CPT_QUEUE_HI_PRIO 0x1 +#define OTX2_CPT_QUEUE_LOW_PRIO 0x0 + +enum otx2_cptlf_state { + OTX2_CPTLF_IN_RESET, + OTX2_CPTLF_STARTED, +}; + +struct otx2_cpt_inst_queue { + u8 *vaddr; + u8 *real_vaddr; + dma_addr_t dma_addr; + dma_addr_t real_dma_addr; + u32 size; +}; + +struct otx2_cptlfs_info; +struct otx2_cptlf_wqe { + struct tasklet_struct work; + struct otx2_cptlfs_info *lfs; + u8 lf_num; +}; + +struct otx2_cptlf_info { + struct otx2_cptlfs_info *lfs; /* Ptr to cptlfs_info struct */ + void __iomem *lmtline; /* Address of LMTLINE */ + void __iomem *ioreg; /* LMTLINE send register */ + int msix_offset; /* MSI-X interrupts offset */ + cpumask_var_t affinity_mask; /* IRQs affinity mask */ + u8 irq_name[OTX2_CPT_LF_MSIX_VECTORS][32];/* Interrupts name */ + u8 is_irq_reg[OTX2_CPT_LF_MSIX_VECTORS]; /* Is interrupt registered */ + u8 slot; /* Slot number of this LF */ + + struct otx2_cpt_inst_queue iqueue;/* Instruction queue */ + struct otx2_cpt_pending_queue pqueue; /* Pending queue */ + struct otx2_cptlf_wqe *wqe; /* Tasklet work info */ +}; + +struct cpt_hw_ops { + void (*send_cmd)(union otx2_cpt_inst_s *cptinst, u32 insts_num, + struct otx2_cptlf_info *lf); + u8 (*cpt_get_compcode)(union otx2_cpt_res_s *result); + u8 (*cpt_get_uc_compcode)(union otx2_cpt_res_s *result); + struct otx2_cpt_inst_info * + (*cpt_sg_info_create)(struct pci_dev *pdev, struct otx2_cpt_req_info *req, + gfp_t gfp); +}; + +#define LMTLINE_SIZE 128 +#define LMTLINE_ALIGN 128 +struct otx2_lmt_info { + void *base; + dma_addr_t iova; + u32 size; + u8 align; +}; + +struct otx2_cptlfs_info { + /* Registers start address of VF/PF LFs are attached to */ + void __iomem *reg_base; + struct otx2_lmt_info lmt_info; + struct pci_dev *pdev; /* Device LFs are attached to */ + struct otx2_cptlf_info lf[OTX2_CPT_MAX_LFS_NUM]; + struct otx2_mbox *mbox; + struct cpt_hw_ops *ops; + u8 are_lfs_attached; /* Whether CPT LFs are attached */ + u8 lfs_num; /* Number of CPT LFs */ + u8 kcrypto_se_eng_grp_num; /* Crypto symmetric engine group number */ + u8 kcrypto_ae_eng_grp_num; /* Crypto asymmetric engine group number */ + u8 kvf_limits; /* Kernel crypto limits */ + atomic_t state; /* LF's state. started/reset */ + int blkaddr; /* CPT blkaddr: BLKADDR_CPT0/BLKADDR_CPT1 */ + int global_slot; /* Global slot across the blocks */ + u8 ctx_ilen; + u8 ctx_ilen_ovrd; +}; + +static inline void otx2_cpt_free_instruction_queues( + struct otx2_cptlfs_info *lfs) +{ + struct otx2_cpt_inst_queue *iq; + int i; + + for (i = 0; i < lfs->lfs_num; i++) { + iq = &lfs->lf[i].iqueue; + if (iq->real_vaddr) + dma_free_coherent(&lfs->pdev->dev, + iq->size, + iq->real_vaddr, + iq->real_dma_addr); + iq->real_vaddr = NULL; + iq->vaddr = NULL; + } +} + +static inline int otx2_cpt_alloc_instruction_queues( + struct otx2_cptlfs_info *lfs) +{ + struct otx2_cpt_inst_queue *iq; + int ret = 0, i; + + if (!lfs->lfs_num) + return -EINVAL; + + for (i = 0; i < lfs->lfs_num; i++) { + iq = &lfs->lf[i].iqueue; + iq->size = OTX2_CPT_INST_QLEN_BYTES + + OTX2_CPT_Q_FC_LEN + + OTX2_CPT_INST_GRP_QLEN_BYTES + + OTX2_CPT_INST_Q_ALIGNMENT; + iq->real_vaddr = dma_alloc_coherent(&lfs->pdev->dev, iq->size, + &iq->real_dma_addr, GFP_KERNEL); + if (!iq->real_vaddr) { + ret = -ENOMEM; + goto error; + } + iq->vaddr = iq->real_vaddr + OTX2_CPT_INST_GRP_QLEN_BYTES; + iq->dma_addr = iq->real_dma_addr + OTX2_CPT_INST_GRP_QLEN_BYTES; + + /* Align pointers */ + iq->vaddr = PTR_ALIGN(iq->vaddr, OTX2_CPT_INST_Q_ALIGNMENT); + iq->dma_addr = PTR_ALIGN(iq->dma_addr, + OTX2_CPT_INST_Q_ALIGNMENT); + } + return 0; + +error: + otx2_cpt_free_instruction_queues(lfs); + return ret; +} + +static inline void otx2_cptlf_set_iqueues_base_addr( + struct otx2_cptlfs_info *lfs) +{ + union otx2_cptx_lf_q_base lf_q_base; + int slot; + + for (slot = 0; slot < lfs->lfs_num; slot++) { + lf_q_base.u = lfs->lf[slot].iqueue.dma_addr; + otx2_cpt_write64(lfs->reg_base, lfs->blkaddr, slot, + OTX2_CPT_LF_Q_BASE, lf_q_base.u); + } +} + +static inline void otx2_cptlf_do_set_iqueue_size(struct otx2_cptlf_info *lf) +{ + union otx2_cptx_lf_q_size lf_q_size = { .u = 0x0 }; + + lf_q_size.s.size_div40 = OTX2_CPT_SIZE_DIV40 + + OTX2_CPT_EXTRA_SIZE_DIV40; + otx2_cpt_write64(lf->lfs->reg_base, lf->lfs->blkaddr, lf->slot, + OTX2_CPT_LF_Q_SIZE, lf_q_size.u); +} + +static inline void otx2_cptlf_set_iqueues_size(struct otx2_cptlfs_info *lfs) +{ + int slot; + + for (slot = 0; slot < lfs->lfs_num; slot++) + otx2_cptlf_do_set_iqueue_size(&lfs->lf[slot]); +} + +#define INFLIGHT GENMASK_ULL(8, 0) +#define GRB_CNT GENMASK_ULL(39, 32) +#define GWB_CNT GENMASK_ULL(47, 40) +#define XQ_XOR GENMASK_ULL(63, 63) +#define DQPTR GENMASK_ULL(19, 0) +#define NQPTR GENMASK_ULL(51, 32) + +static inline void otx2_cptlf_do_disable_iqueue(struct otx2_cptlf_info *lf) +{ + void __iomem *reg_base = lf->lfs->reg_base; + struct pci_dev *pdev = lf->lfs->pdev; + u8 blkaddr = lf->lfs->blkaddr; + int timeout = 1000000; + u64 inprog, inst_ptr; + u64 slot = lf->slot; + u64 qsize, pending; + int i = 0; + + /* Disable instructions enqueuing */ + otx2_cpt_write64(reg_base, blkaddr, slot, OTX2_CPT_LF_CTL, 0x0); + + inprog = otx2_cpt_read64(reg_base, blkaddr, slot, OTX2_CPT_LF_INPROG); + inprog |= BIT_ULL(16); + otx2_cpt_write64(reg_base, blkaddr, slot, OTX2_CPT_LF_INPROG, inprog); + + qsize = otx2_cpt_read64(reg_base, blkaddr, slot, OTX2_CPT_LF_Q_SIZE) & 0x7FFF; + do { + inst_ptr = otx2_cpt_read64(reg_base, blkaddr, slot, OTX2_CPT_LF_Q_INST_PTR); + pending = (FIELD_GET(XQ_XOR, inst_ptr) * qsize * 40) + + FIELD_GET(NQPTR, inst_ptr) - FIELD_GET(DQPTR, inst_ptr); + udelay(1); + timeout--; + } while ((pending != 0) && (timeout != 0)); + + if (timeout == 0) + dev_warn(&pdev->dev, "TIMEOUT: CPT poll on pending instructions\n"); + + timeout = 1000000; + /* Wait for CPT queue to become execution-quiescent */ + do { + inprog = otx2_cpt_read64(reg_base, blkaddr, slot, OTX2_CPT_LF_INPROG); + + if ((FIELD_GET(INFLIGHT, inprog) == 0) && + (FIELD_GET(GRB_CNT, inprog) == 0)) { + i++; + } else { + i = 0; + timeout--; + } + } while ((timeout != 0) && (i < 10)); + + if (timeout == 0) + dev_warn(&pdev->dev, "TIMEOUT: CPT poll on inflight count\n"); + /* Wait for 2 us to flush all queue writes to memory */ + udelay(2); +} + +static inline void otx2_cptlf_disable_iqueues(struct otx2_cptlfs_info *lfs) +{ + int slot; + + for (slot = 0; slot < lfs->lfs_num; slot++) { + otx2_cptlf_do_disable_iqueue(&lfs->lf[slot]); + otx2_cpt_lf_reset_msg(lfs, lfs->global_slot + slot); + } +} + +static inline void otx2_cptlf_set_iqueue_enq(struct otx2_cptlf_info *lf, + bool enable) +{ + u8 blkaddr = lf->lfs->blkaddr; + union otx2_cptx_lf_ctl lf_ctl; + + lf_ctl.u = otx2_cpt_read64(lf->lfs->reg_base, blkaddr, lf->slot, + OTX2_CPT_LF_CTL); + + /* Set iqueue's enqueuing */ + lf_ctl.s.ena = enable ? 0x1 : 0x0; + otx2_cpt_write64(lf->lfs->reg_base, blkaddr, lf->slot, + OTX2_CPT_LF_CTL, lf_ctl.u); +} + +static inline void otx2_cptlf_enable_iqueue_enq(struct otx2_cptlf_info *lf) +{ + otx2_cptlf_set_iqueue_enq(lf, true); +} + +static inline void otx2_cptlf_set_iqueue_exec(struct otx2_cptlf_info *lf, + bool enable) +{ + union otx2_cptx_lf_inprog lf_inprog; + u8 blkaddr = lf->lfs->blkaddr; + + lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, blkaddr, lf->slot, + OTX2_CPT_LF_INPROG); + + /* Set iqueue's execution */ + lf_inprog.s.eena = enable ? 0x1 : 0x0; + otx2_cpt_write64(lf->lfs->reg_base, blkaddr, lf->slot, + OTX2_CPT_LF_INPROG, lf_inprog.u); +} + +static inline void otx2_cptlf_set_ctx_flr_flush(struct otx2_cptlf_info *lf) +{ + u8 blkaddr = lf->lfs->blkaddr; + u64 val; + + val = otx2_cpt_read64(lf->lfs->reg_base, blkaddr, lf->slot, + OTX2_CPT_LF_CTX_CTL); + val |= BIT_ULL(0); + + otx2_cpt_write64(lf->lfs->reg_base, blkaddr, lf->slot, + OTX2_CPT_LF_CTX_CTL, val); +} + +static inline void otx2_cptlf_enable_iqueue_exec(struct otx2_cptlf_info *lf) +{ + otx2_cptlf_set_iqueue_exec(lf, true); +} + +static inline void otx2_cptlf_disable_iqueue_exec(struct otx2_cptlf_info *lf) +{ + otx2_cptlf_set_iqueue_exec(lf, false); +} + +static inline void otx2_cptlf_enable_iqueues(struct otx2_cptlfs_info *lfs) +{ + int slot; + + for (slot = 0; slot < lfs->lfs_num; slot++) { + /* Enable flush on FLR for Errata */ + if (is_dev_cn10kb(lfs->pdev)) + otx2_cptlf_set_ctx_flr_flush(&lfs->lf[slot]); + + otx2_cptlf_enable_iqueue_exec(&lfs->lf[slot]); + otx2_cptlf_enable_iqueue_enq(&lfs->lf[slot]); + } +} + +static inline void otx2_cpt_fill_inst(union otx2_cpt_inst_s *cptinst, + struct otx2_cpt_iq_command *iq_cmd, + u64 comp_baddr) +{ + cptinst->u[0] = 0x0; + cptinst->s.doneint = true; + cptinst->s.res_addr = comp_baddr; + cptinst->u[2] = 0x0; + cptinst->u[3] = 0x0; + cptinst->s.ei0 = iq_cmd->cmd.u; + cptinst->s.ei1 = iq_cmd->dptr; + cptinst->s.ei2 = iq_cmd->rptr; + cptinst->s.ei3 = iq_cmd->cptr.u; +} + +/* + * On OcteonTX2 platform the parameter insts_num is used as a count of + * instructions to be enqueued. The valid values for insts_num are: + * 1 - 1 CPT instruction will be enqueued during LMTST operation + * 2 - 2 CPT instructions will be enqueued during LMTST operation + */ +static inline void otx2_cpt_send_cmd(union otx2_cpt_inst_s *cptinst, + u32 insts_num, struct otx2_cptlf_info *lf) +{ + void __iomem *lmtline = lf->lmtline; + long ret; + + /* + * Make sure memory areas pointed in CPT_INST_S + * are flushed before the instruction is sent to CPT + */ + dma_wmb(); + + do { + /* Copy CPT command to LMTLINE */ + memcpy_toio(lmtline, cptinst, insts_num * OTX2_CPT_INST_SIZE); + + /* + * LDEOR initiates atomic transfer to I/O device + * The following will cause the LMTST to fail (the LDEOR + * returns zero): + * - No stores have been performed to the LMTLINE since it was + * last invalidated. + * - The bytes which have been stored to LMTLINE since it was + * last invalidated form a pattern that is non-contiguous, does + * not start at byte 0, or does not end on a 8-byte boundary. + * (i.e.comprises a formation of other than 1–16 8-byte + * words.) + * + * These rules are designed such that an operating system + * context switch or hypervisor guest switch need have no + * knowledge of the LMTST operations; the switch code does not + * need to store to LMTCANCEL. Also note as LMTLINE data cannot + * be read, there is no information leakage between processes. + */ + ret = otx2_lmt_flush(lf->ioreg); + + } while (!ret); +} + +static inline bool otx2_cptlf_started(struct otx2_cptlfs_info *lfs) +{ + return atomic_read(&lfs->state) == OTX2_CPTLF_STARTED; +} + +static inline void otx2_cptlf_set_dev_info(struct otx2_cptlfs_info *lfs, + struct pci_dev *pdev, + void __iomem *reg_base, + struct otx2_mbox *mbox, + int blkaddr) +{ + lfs->pdev = pdev; + lfs->reg_base = reg_base; + lfs->mbox = mbox; + lfs->blkaddr = blkaddr; +} + +int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_msk, int pri, + int lfs_num); +void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs); +int otx2_cptlf_register_misc_interrupts(struct otx2_cptlfs_info *lfs); +int otx2_cptlf_register_done_interrupts(struct otx2_cptlfs_info *lfs); +void otx2_cptlf_unregister_misc_interrupts(struct otx2_cptlfs_info *lfs); +void otx2_cptlf_unregister_done_interrupts(struct otx2_cptlfs_info *lfs); +void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs); +int otx2_cptlf_set_irqs_affinity(struct otx2_cptlfs_info *lfs); + +#endif /* __OTX2_CPTLF_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf.h b/drivers/crypto/marvell/octeontx2/otx2_cptpf.h new file mode 100644 index 000000000..e5859a1e1 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2020 Marvell. + */ + +#ifndef __OTX2_CPTPF_H +#define __OTX2_CPTPF_H + +#include "otx2_cpt_common.h" +#include "otx2_cptpf_ucode.h" +#include "otx2_cptlf.h" + +struct otx2_cptpf_dev; +struct otx2_cptvf_info { + struct otx2_cptpf_dev *cptpf; /* PF pointer this VF belongs to */ + struct work_struct vfpf_mbox_work; + struct pci_dev *vf_dev; + int vf_id; + int intr_idx; +}; + +struct cptpf_flr_work { + struct work_struct work; + struct otx2_cptpf_dev *pf; +}; + +struct otx2_cptpf_dev { + void __iomem *reg_base; /* CPT PF registers start address */ + void __iomem *afpf_mbox_base; /* PF-AF mbox start address */ + void __iomem *vfpf_mbox_base; /* VF-PF mbox start address */ + struct pci_dev *pdev; /* PCI device handle */ + struct otx2_cptvf_info vf[OTX2_CPT_MAX_VFS_NUM]; + struct otx2_cpt_eng_grps eng_grps;/* Engine groups information */ + struct otx2_cptlfs_info lfs; /* CPT LFs attached to this PF */ + struct otx2_cptlfs_info cpt1_lfs; /* CPT1 LFs attached to this PF */ + /* HW capabilities for each engine type */ + union otx2_cpt_eng_caps eng_caps[OTX2_CPT_MAX_ENG_TYPES]; + bool is_eng_caps_discovered; + + /* AF <=> PF mbox */ + struct otx2_mbox afpf_mbox; + struct work_struct afpf_mbox_work; + struct workqueue_struct *afpf_mbox_wq; + + struct otx2_mbox afpf_mbox_up; + struct work_struct afpf_mbox_up_work; + + /* VF <=> PF mbox */ + struct otx2_mbox vfpf_mbox; + struct workqueue_struct *vfpf_mbox_wq; + + struct workqueue_struct *flr_wq; + struct cptpf_flr_work *flr_work; + struct mutex lock; /* serialize mailbox access */ + + unsigned long cap_flag; + u8 pf_id; /* RVU PF number */ + u8 max_vfs; /* Maximum number of VFs supported by CPT */ + u8 enabled_vfs; /* Number of enabled VFs */ + u8 sso_pf_func_ovrd; /* SSO PF_FUNC override bit */ + u8 kvf_limits; /* Kernel crypto limits */ + bool has_cpt1; + u8 rsrc_req_blkaddr; + + /* Devlink */ + struct devlink *dl; +}; + +irqreturn_t otx2_cptpf_afpf_mbox_intr(int irq, void *arg); +void otx2_cptpf_afpf_mbox_handler(struct work_struct *work); +void otx2_cptpf_afpf_mbox_up_handler(struct work_struct *work); +irqreturn_t otx2_cptpf_vfpf_mbox_intr(int irq, void *arg); +void otx2_cptpf_vfpf_mbox_handler(struct work_struct *work); + +int otx2_inline_cptlf_setup(struct otx2_cptpf_dev *cptpf, + struct otx2_cptlfs_info *lfs, u8 egrp, int num_lfs); +void otx2_inline_cptlf_cleanup(struct otx2_cptlfs_info *lfs); + +#endif /* __OTX2_CPTPF_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c new file mode 100644 index 000000000..79f6c2bb9 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c @@ -0,0 +1,883 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include +#include +#include "otx2_cpt_hw_types.h" +#include "otx2_cpt_common.h" +#include "otx2_cpt_devlink.h" +#include "otx2_cptpf_ucode.h" +#include "otx2_cptpf.h" +#include "cn10k_cpt.h" +#include "rvu_reg.h" + +#define OTX2_CPT_DRV_NAME "rvu_cptpf" +#define OTX2_CPT_DRV_STRING "Marvell RVU CPT Physical Function Driver" + +#define CPT_UC_RID_CN9K_B0 1 +#define CPT_UC_RID_CN10K_A 4 +#define CPT_UC_RID_CN10K_B 5 + +static void cptpf_enable_vfpf_mbox_intr(struct otx2_cptpf_dev *cptpf, + int num_vfs) +{ + int ena_bits; + + /* Clear any pending interrupts */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INTX(0), ~0x0ULL); + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INTX(1), ~0x0ULL); + + /* Enable VF interrupts for VFs from 0 to 63 */ + ena_bits = ((num_vfs - 1) % 64); + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INT_ENA_W1SX(0), + GENMASK_ULL(ena_bits, 0)); + + if (num_vfs > 64) { + /* Enable VF interrupts for VFs from 64 to 127 */ + ena_bits = num_vfs - 64 - 1; + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INT_ENA_W1SX(1), + GENMASK_ULL(ena_bits, 0)); + } +} + +static void cptpf_disable_vfpf_mbox_intr(struct otx2_cptpf_dev *cptpf, + int num_vfs) +{ + int vector; + + /* Disable VF-PF interrupts */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INT_ENA_W1CX(0), ~0ULL); + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INT_ENA_W1CX(1), ~0ULL); + /* Clear any pending interrupts */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INTX(0), ~0ULL); + + vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFPF_MBOX0); + free_irq(vector, cptpf); + + if (num_vfs > 64) { + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INTX(1), ~0ULL); + vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFPF_MBOX1); + free_irq(vector, cptpf); + } +} + +static void cptpf_enable_vf_flr_me_intrs(struct otx2_cptpf_dev *cptpf, + int num_vfs) +{ + /* Clear FLR interrupt if any */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFFLR_INTX(0), + INTR_MASK(num_vfs)); + + /* Enable VF FLR interrupts */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INT_ENA_W1SX(0), INTR_MASK(num_vfs)); + /* Clear ME interrupt if any */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFME_INTX(0), + INTR_MASK(num_vfs)); + /* Enable VF ME interrupts */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFME_INT_ENA_W1SX(0), INTR_MASK(num_vfs)); + + if (num_vfs <= 64) + return; + + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFFLR_INTX(1), + INTR_MASK(num_vfs - 64)); + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INT_ENA_W1SX(1), INTR_MASK(num_vfs - 64)); + + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFME_INTX(1), + INTR_MASK(num_vfs - 64)); + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFME_INT_ENA_W1SX(1), INTR_MASK(num_vfs - 64)); +} + +static void cptpf_disable_vf_flr_me_intrs(struct otx2_cptpf_dev *cptpf, + int num_vfs) +{ + int vector; + + /* Disable VF FLR interrupts */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INT_ENA_W1CX(0), INTR_MASK(num_vfs)); + vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFFLR0); + free_irq(vector, cptpf); + + /* Disable VF ME interrupts */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFME_INT_ENA_W1CX(0), INTR_MASK(num_vfs)); + vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFME0); + free_irq(vector, cptpf); + + if (num_vfs <= 64) + return; + + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INT_ENA_W1CX(1), INTR_MASK(num_vfs - 64)); + vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFFLR1); + free_irq(vector, cptpf); + + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFME_INT_ENA_W1CX(1), INTR_MASK(num_vfs - 64)); + vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFME1); + free_irq(vector, cptpf); +} + +static void cptpf_flr_wq_handler(struct work_struct *work) +{ + struct cptpf_flr_work *flr_work; + struct otx2_cptpf_dev *pf; + struct mbox_msghdr *req; + struct otx2_mbox *mbox; + int vf, reg = 0; + + flr_work = container_of(work, struct cptpf_flr_work, work); + pf = flr_work->pf; + mbox = &pf->afpf_mbox; + + vf = flr_work - pf->flr_work; + + mutex_lock(&pf->lock); + req = otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct msg_rsp)); + if (!req) { + mutex_unlock(&pf->lock); + return; + } + + req->sig = OTX2_MBOX_REQ_SIG; + req->id = MBOX_MSG_VF_FLR; + req->pcifunc &= RVU_PFVF_FUNC_MASK; + req->pcifunc |= (vf + 1) & RVU_PFVF_FUNC_MASK; + + otx2_cpt_send_mbox_msg(mbox, pf->pdev); + if (!otx2_cpt_sync_mbox_msg(&pf->afpf_mbox)) { + + if (vf >= 64) { + reg = 1; + vf = vf - 64; + } + /* Clear transaction pending register */ + otx2_cpt_write64(pf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFTRPENDX(reg), BIT_ULL(vf)); + otx2_cpt_write64(pf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INT_ENA_W1SX(reg), BIT_ULL(vf)); + } + mutex_unlock(&pf->lock); +} + +static irqreturn_t cptpf_vf_flr_intr(int __always_unused irq, void *arg) +{ + int reg, dev, vf, start_vf, num_reg = 1; + struct otx2_cptpf_dev *cptpf = arg; + u64 intr; + + if (cptpf->max_vfs > 64) + num_reg = 2; + + for (reg = 0; reg < num_reg; reg++) { + intr = otx2_cpt_read64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INTX(reg)); + if (!intr) + continue; + start_vf = 64 * reg; + for (vf = 0; vf < 64; vf++) { + if (!(intr & BIT_ULL(vf))) + continue; + dev = vf + start_vf; + queue_work(cptpf->flr_wq, &cptpf->flr_work[dev].work); + /* Clear interrupt */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INTX(reg), BIT_ULL(vf)); + /* Disable the interrupt */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INT_ENA_W1CX(reg), + BIT_ULL(vf)); + } + } + return IRQ_HANDLED; +} + +static irqreturn_t cptpf_vf_me_intr(int __always_unused irq, void *arg) +{ + struct otx2_cptpf_dev *cptpf = arg; + int reg, vf, num_reg = 1; + u64 intr; + + if (cptpf->max_vfs > 64) + num_reg = 2; + + for (reg = 0; reg < num_reg; reg++) { + intr = otx2_cpt_read64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFME_INTX(reg)); + if (!intr) + continue; + for (vf = 0; vf < 64; vf++) { + if (!(intr & BIT_ULL(vf))) + continue; + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFTRPENDX(reg), BIT_ULL(vf)); + /* Clear interrupt */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFME_INTX(reg), BIT_ULL(vf)); + } + } + return IRQ_HANDLED; +} + +static void cptpf_unregister_vfpf_intr(struct otx2_cptpf_dev *cptpf, + int num_vfs) +{ + cptpf_disable_vfpf_mbox_intr(cptpf, num_vfs); + cptpf_disable_vf_flr_me_intrs(cptpf, num_vfs); +} + +static int cptpf_register_vfpf_intr(struct otx2_cptpf_dev *cptpf, int num_vfs) +{ + struct pci_dev *pdev = cptpf->pdev; + struct device *dev = &pdev->dev; + int ret, vector; + + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFPF_MBOX0); + /* Register VF-PF mailbox interrupt handler */ + ret = request_irq(vector, otx2_cptpf_vfpf_mbox_intr, 0, "CPTVFPF Mbox0", + cptpf); + if (ret) { + dev_err(dev, + "IRQ registration failed for PFVF mbox0 irq\n"); + return ret; + } + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFFLR0); + /* Register VF FLR interrupt handler */ + ret = request_irq(vector, cptpf_vf_flr_intr, 0, "CPTPF FLR0", cptpf); + if (ret) { + dev_err(dev, + "IRQ registration failed for VFFLR0 irq\n"); + goto free_mbox0_irq; + } + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFME0); + /* Register VF ME interrupt handler */ + ret = request_irq(vector, cptpf_vf_me_intr, 0, "CPTPF ME0", cptpf); + if (ret) { + dev_err(dev, + "IRQ registration failed for PFVF mbox0 irq\n"); + goto free_flr0_irq; + } + + if (num_vfs > 64) { + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFPF_MBOX1); + ret = request_irq(vector, otx2_cptpf_vfpf_mbox_intr, 0, + "CPTVFPF Mbox1", cptpf); + if (ret) { + dev_err(dev, + "IRQ registration failed for PFVF mbox1 irq\n"); + goto free_me0_irq; + } + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFFLR1); + /* Register VF FLR interrupt handler */ + ret = request_irq(vector, cptpf_vf_flr_intr, 0, "CPTPF FLR1", + cptpf); + if (ret) { + dev_err(dev, + "IRQ registration failed for VFFLR1 irq\n"); + goto free_mbox1_irq; + } + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFME1); + /* Register VF FLR interrupt handler */ + ret = request_irq(vector, cptpf_vf_me_intr, 0, "CPTPF ME1", + cptpf); + if (ret) { + dev_err(dev, + "IRQ registration failed for VFFLR1 irq\n"); + goto free_flr1_irq; + } + } + cptpf_enable_vfpf_mbox_intr(cptpf, num_vfs); + cptpf_enable_vf_flr_me_intrs(cptpf, num_vfs); + + return 0; + +free_flr1_irq: + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFFLR1); + free_irq(vector, cptpf); +free_mbox1_irq: + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFPF_MBOX1); + free_irq(vector, cptpf); +free_me0_irq: + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFME0); + free_irq(vector, cptpf); +free_flr0_irq: + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFFLR0); + free_irq(vector, cptpf); +free_mbox0_irq: + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFPF_MBOX0); + free_irq(vector, cptpf); + return ret; +} + +static void cptpf_flr_wq_destroy(struct otx2_cptpf_dev *pf) +{ + if (!pf->flr_wq) + return; + destroy_workqueue(pf->flr_wq); + pf->flr_wq = NULL; + kfree(pf->flr_work); +} + +static int cptpf_flr_wq_init(struct otx2_cptpf_dev *cptpf, int num_vfs) +{ + int vf; + + cptpf->flr_wq = alloc_ordered_workqueue("cptpf_flr_wq", 0); + if (!cptpf->flr_wq) + return -ENOMEM; + + cptpf->flr_work = kzalloc_objs(struct cptpf_flr_work, num_vfs); + if (!cptpf->flr_work) + goto destroy_wq; + + for (vf = 0; vf < num_vfs; vf++) { + cptpf->flr_work[vf].pf = cptpf; + INIT_WORK(&cptpf->flr_work[vf].work, cptpf_flr_wq_handler); + } + return 0; + +destroy_wq: + destroy_workqueue(cptpf->flr_wq); + return -ENOMEM; +} + +static int cptpf_vfpf_mbox_init(struct otx2_cptpf_dev *cptpf, int num_vfs) +{ + struct device *dev = &cptpf->pdev->dev; + u64 vfpf_mbox_base; + int err, i; + + cptpf->vfpf_mbox_wq = + alloc_ordered_workqueue("cpt_vfpf_mailbox", + WQ_HIGHPRI | WQ_MEM_RECLAIM); + if (!cptpf->vfpf_mbox_wq) + return -ENOMEM; + + /* Map VF-PF mailbox memory */ + if (test_bit(CN10K_MBOX, &cptpf->cap_flag)) + vfpf_mbox_base = readq(cptpf->reg_base + RVU_PF_VF_MBOX_ADDR); + else + vfpf_mbox_base = readq(cptpf->reg_base + RVU_PF_VF_BAR4_ADDR); + + if (!vfpf_mbox_base) { + dev_err(dev, "VF-PF mailbox address not configured\n"); + err = -ENOMEM; + goto free_wqe; + } + cptpf->vfpf_mbox_base = devm_ioremap_wc(dev, vfpf_mbox_base, + MBOX_SIZE * cptpf->max_vfs); + if (!cptpf->vfpf_mbox_base) { + dev_err(dev, "Mapping of VF-PF mailbox address failed\n"); + err = -ENOMEM; + goto free_wqe; + } + err = otx2_mbox_init(&cptpf->vfpf_mbox, cptpf->vfpf_mbox_base, + cptpf->pdev, cptpf->reg_base, MBOX_DIR_PFVF, + num_vfs); + if (err) + goto free_wqe; + + for (i = 0; i < num_vfs; i++) { + cptpf->vf[i].vf_id = i; + cptpf->vf[i].cptpf = cptpf; + cptpf->vf[i].intr_idx = i % 64; + INIT_WORK(&cptpf->vf[i].vfpf_mbox_work, + otx2_cptpf_vfpf_mbox_handler); + } + return 0; + +free_wqe: + destroy_workqueue(cptpf->vfpf_mbox_wq); + return err; +} + +static void cptpf_vfpf_mbox_destroy(struct otx2_cptpf_dev *cptpf) +{ + destroy_workqueue(cptpf->vfpf_mbox_wq); + otx2_mbox_destroy(&cptpf->vfpf_mbox); +} + +static void cptpf_disable_afpf_mbox_intr(struct otx2_cptpf_dev *cptpf) +{ + /* Disable AF-PF interrupt */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_INT_ENA_W1C, + 0x1ULL); + /* Clear interrupt if any */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_INT, 0x1ULL); +} + +static int cptpf_register_afpf_mbox_intr(struct otx2_cptpf_dev *cptpf) +{ + struct pci_dev *pdev = cptpf->pdev; + struct device *dev = &pdev->dev; + int ret, irq; + + irq = pci_irq_vector(pdev, RVU_PF_INT_VEC_AFPF_MBOX); + /* Register AF-PF mailbox interrupt handler */ + ret = devm_request_irq(dev, irq, otx2_cptpf_afpf_mbox_intr, 0, + "CPTAFPF Mbox", cptpf); + if (ret) + return ret; + /* Clear interrupt if any, to avoid spurious interrupts */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_INT, 0x1ULL); + /* Enable AF-PF interrupt */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_INT_ENA_W1S, + 0x1ULL); + + ret = otx2_cpt_send_ready_msg(&cptpf->afpf_mbox, cptpf->pdev); + if (ret) { + dev_warn(dev, + "AF not responding to mailbox, deferring probe\n"); + cptpf_disable_afpf_mbox_intr(cptpf); + return -EPROBE_DEFER; + } + return 0; +} + +static int cptpf_afpf_mbox_init(struct otx2_cptpf_dev *cptpf) +{ + struct pci_dev *pdev = cptpf->pdev; + resource_size_t offset; + int err; + + cptpf->afpf_mbox_wq = + alloc_ordered_workqueue("cpt_afpf_mailbox", + WQ_HIGHPRI | WQ_MEM_RECLAIM); + if (!cptpf->afpf_mbox_wq) + return -ENOMEM; + + offset = pci_resource_start(pdev, PCI_MBOX_BAR_NUM); + /* Map AF-PF mailbox memory */ + cptpf->afpf_mbox_base = devm_ioremap_wc(&pdev->dev, offset, MBOX_SIZE); + if (!cptpf->afpf_mbox_base) { + dev_err(&pdev->dev, "Unable to map BAR4\n"); + err = -ENOMEM; + goto error; + } + + err = otx2_mbox_init(&cptpf->afpf_mbox, cptpf->afpf_mbox_base, + pdev, cptpf->reg_base, MBOX_DIR_PFAF, 1); + if (err) + goto error; + + err = otx2_mbox_init(&cptpf->afpf_mbox_up, cptpf->afpf_mbox_base, + pdev, cptpf->reg_base, MBOX_DIR_PFAF_UP, 1); + if (err) + goto mbox_cleanup; + + INIT_WORK(&cptpf->afpf_mbox_work, otx2_cptpf_afpf_mbox_handler); + INIT_WORK(&cptpf->afpf_mbox_up_work, otx2_cptpf_afpf_mbox_up_handler); + mutex_init(&cptpf->lock); + + return 0; + +mbox_cleanup: + otx2_mbox_destroy(&cptpf->afpf_mbox); +error: + destroy_workqueue(cptpf->afpf_mbox_wq); + return err; +} + +static void cptpf_afpf_mbox_destroy(struct otx2_cptpf_dev *cptpf) +{ + destroy_workqueue(cptpf->afpf_mbox_wq); + otx2_mbox_destroy(&cptpf->afpf_mbox); + otx2_mbox_destroy(&cptpf->afpf_mbox_up); +} + +static ssize_t sso_pf_func_ovrd_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct otx2_cptpf_dev *cptpf = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%d\n", cptpf->sso_pf_func_ovrd); +} + +static ssize_t sso_pf_func_ovrd_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct otx2_cptpf_dev *cptpf = dev_get_drvdata(dev); + u8 sso_pf_func_ovrd; + + if (!(cptpf->pdev->revision == CPT_UC_RID_CN9K_B0)) + return count; + + if (kstrtou8(buf, 0, &sso_pf_func_ovrd)) + return -EINVAL; + + cptpf->sso_pf_func_ovrd = sso_pf_func_ovrd; + + return count; +} + +static ssize_t kvf_limits_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct otx2_cptpf_dev *cptpf = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%d\n", cptpf->kvf_limits); +} + +static ssize_t kvf_limits_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct otx2_cptpf_dev *cptpf = dev_get_drvdata(dev); + int lfs_num; + int ret; + + ret = kstrtoint(buf, 0, &lfs_num); + if (ret) + return ret; + if (lfs_num < 1 || lfs_num > num_online_cpus()) { + dev_err(dev, "lfs count %d must be in range [1 - %d]\n", + lfs_num, num_online_cpus()); + return -EINVAL; + } + cptpf->kvf_limits = lfs_num; + + return count; +} + +static DEVICE_ATTR_RW(kvf_limits); +static DEVICE_ATTR_RW(sso_pf_func_ovrd); + +static struct attribute *cptpf_attrs[] = { + &dev_attr_kvf_limits.attr, + &dev_attr_sso_pf_func_ovrd.attr, + NULL +}; + +static const struct attribute_group cptpf_sysfs_group = { + .attrs = cptpf_attrs, +}; + +static int cpt_is_pf_usable(struct otx2_cptpf_dev *cptpf) +{ + u64 rev; + + rev = otx2_cpt_read64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_BLOCK_ADDRX_DISC(BLKADDR_RVUM)); + rev = (rev >> 12) & 0xFF; + /* + * Check if AF has setup revision for RVUM block, otherwise + * driver probe should be deferred until AF driver comes up + */ + if (!rev) { + dev_warn(&cptpf->pdev->dev, + "AF is not initialized, deferring probe\n"); + return -EPROBE_DEFER; + } + return 0; +} + +static void cptpf_get_rid(struct pci_dev *pdev, struct otx2_cptpf_dev *cptpf) +{ + struct otx2_cpt_eng_grps *eng_grps = &cptpf->eng_grps; + u64 reg_val = 0x0; + + if (is_dev_otx2(pdev)) { + eng_grps->rid = pdev->revision; + return; + } + otx2_cpt_read_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_CTL, ®_val, + BLKADDR_CPT0); + if ((cpt_feature_sgv2(pdev) && (reg_val & BIT_ULL(18))) || + is_dev_cn10ka_ax(pdev)) + eng_grps->rid = CPT_UC_RID_CN10K_A; + else if (cpt_feature_sgv2(pdev)) + eng_grps->rid = CPT_UC_RID_CN10K_B; +} + +static void cptpf_check_block_implemented(struct otx2_cptpf_dev *cptpf) +{ + u64 cfg; + + cfg = otx2_cpt_read64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_BLOCK_ADDRX_DISC(BLKADDR_CPT1)); + if (cfg & BIT_ULL(11)) + cptpf->has_cpt1 = true; +} + +static int cptpf_device_init(struct otx2_cptpf_dev *cptpf) +{ + union otx2_cptx_af_constants1 af_cnsts1 = {0}; + int ret = 0; + + /* check if 'implemented' bit is set for block BLKADDR_CPT1 */ + cptpf_check_block_implemented(cptpf); + + /* Get number of SE, IE and AE engines */ + ret = otx2_cpt_read_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_CONSTANTS1, &af_cnsts1.u, + BLKADDR_CPT0); + if (ret) + return ret; + + cptpf->eng_grps.avail.max_se_cnt = af_cnsts1.s.se; + cptpf->eng_grps.avail.max_ie_cnt = af_cnsts1.s.ie; + cptpf->eng_grps.avail.max_ae_cnt = af_cnsts1.s.ae; + + /* Disable all cores */ + ret = otx2_cpt_disable_all_cores(cptpf); + + otx2_cptlf_set_dev_info(&cptpf->lfs, cptpf->pdev, cptpf->reg_base, + &cptpf->afpf_mbox, BLKADDR_CPT0); + if (cptpf->has_cpt1) + otx2_cptlf_set_dev_info(&cptpf->cpt1_lfs, cptpf->pdev, + cptpf->reg_base, &cptpf->afpf_mbox, + BLKADDR_CPT1); + return ret; +} + +static int cptpf_sriov_disable(struct pci_dev *pdev) +{ + struct otx2_cptpf_dev *cptpf = pci_get_drvdata(pdev); + int num_vfs = pci_num_vf(pdev); + + if (!num_vfs) + return 0; + + pci_disable_sriov(pdev); + cptpf_unregister_vfpf_intr(cptpf, num_vfs); + cptpf_flr_wq_destroy(cptpf); + cptpf_vfpf_mbox_destroy(cptpf); + module_put(THIS_MODULE); + cptpf->enabled_vfs = 0; + + return 0; +} + +static int cptpf_sriov_enable(struct pci_dev *pdev, int num_vfs) +{ + struct otx2_cptpf_dev *cptpf = pci_get_drvdata(pdev); + int ret; + + /* Initialize VF<=>PF mailbox */ + ret = cptpf_vfpf_mbox_init(cptpf, num_vfs); + if (ret) + return ret; + + ret = cptpf_flr_wq_init(cptpf, num_vfs); + if (ret) + goto destroy_mbox; + /* Register VF<=>PF mailbox interrupt */ + ret = cptpf_register_vfpf_intr(cptpf, num_vfs); + if (ret) + goto destroy_flr; + + cptpf_get_rid(pdev, cptpf); + /* Get CPT HW capabilities using LOAD_FVC operation. */ + ret = otx2_cpt_discover_eng_capabilities(cptpf); + if (ret) + goto disable_intr; + + ret = otx2_cpt_create_eng_grps(cptpf, &cptpf->eng_grps); + if (ret) + goto disable_intr; + + cptpf->enabled_vfs = num_vfs; + ret = pci_enable_sriov(pdev, num_vfs); + if (ret) + goto disable_intr; + + dev_notice(&cptpf->pdev->dev, "VFs enabled: %d\n", num_vfs); + + try_module_get(THIS_MODULE); + return num_vfs; + +disable_intr: + cptpf_unregister_vfpf_intr(cptpf, num_vfs); + cptpf->enabled_vfs = 0; +destroy_flr: + cptpf_flr_wq_destroy(cptpf); +destroy_mbox: + cptpf_vfpf_mbox_destroy(cptpf); + return ret; +} + +static int otx2_cptpf_sriov_configure(struct pci_dev *pdev, int num_vfs) +{ + if (num_vfs > 0) { + return cptpf_sriov_enable(pdev, num_vfs); + } else { + return cptpf_sriov_disable(pdev); + } +} + +static int otx2_cptpf_probe(struct pci_dev *pdev, + const struct pci_device_id *ent) +{ + struct device *dev = &pdev->dev; + struct otx2_cptpf_dev *cptpf; + int err, num_vec; + + cptpf = devm_kzalloc(dev, sizeof(*cptpf), GFP_KERNEL); + if (!cptpf) + return -ENOMEM; + + err = pcim_enable_device(pdev); + if (err) { + dev_err(dev, "Failed to enable PCI device\n"); + goto clear_drvdata; + } + + err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); + if (err) { + dev_err(dev, "Unable to get usable DMA configuration\n"); + goto clear_drvdata; + } + err = pcim_request_all_regions(pdev, OTX2_CPT_DRV_NAME); + if (err) { + dev_err(dev, "Couldn't request PCI resources 0x%x\n", err); + goto clear_drvdata; + } + pci_set_master(pdev); + pci_set_drvdata(pdev, cptpf); + cptpf->pdev = pdev; + + /* Map PF's configuration registers */ + cptpf->reg_base = pcim_iomap(pdev, PCI_PF_REG_BAR_NUM, 0); + if (!cptpf->reg_base) { + err = -ENOMEM; + dev_err(dev, "Couldn't ioremap PCI resource 0x%x\n", err); + goto clear_drvdata; + } + + /* Check if AF driver is up, otherwise defer probe */ + err = cpt_is_pf_usable(cptpf); + if (err) + goto clear_drvdata; + + num_vec = pci_msix_vec_count(cptpf->pdev); + if (num_vec <= 0) { + err = -EINVAL; + goto clear_drvdata; + } + + err = pci_alloc_irq_vectors(pdev, num_vec, num_vec, PCI_IRQ_MSIX); + if (err < 0) { + dev_err(dev, "Request for %d msix vectors failed\n", + RVU_PF_INT_VEC_CNT); + goto clear_drvdata; + } + otx2_cpt_set_hw_caps(pdev, &cptpf->cap_flag); + /* Initialize AF-PF mailbox */ + err = cptpf_afpf_mbox_init(cptpf); + if (err) + goto clear_drvdata; + /* Register mailbox interrupt */ + err = cptpf_register_afpf_mbox_intr(cptpf); + if (err) + goto destroy_afpf_mbox; + + cptpf->max_vfs = pci_sriov_get_totalvfs(pdev); + cptpf->kvf_limits = 1; + + /* Initialize CPT PF device */ + err = cptpf_device_init(cptpf); + if (err) + goto unregister_intr; + + err = cn10k_cptpf_lmtst_init(cptpf); + if (err) + goto unregister_intr; + + /* Initialize engine groups */ + err = otx2_cpt_init_eng_grps(pdev, &cptpf->eng_grps); + if (err) + goto free_lmtst; + + err = sysfs_create_group(&dev->kobj, &cptpf_sysfs_group); + if (err) + goto cleanup_eng_grps; + + err = otx2_cpt_register_dl(cptpf); + if (err) + goto sysfs_grp_del; + + return 0; + +sysfs_grp_del: + sysfs_remove_group(&dev->kobj, &cptpf_sysfs_group); +cleanup_eng_grps: + otx2_cpt_cleanup_eng_grps(pdev, &cptpf->eng_grps); +free_lmtst: + cn10k_cpt_lmtst_free(pdev, &cptpf->lfs); +unregister_intr: + cptpf_disable_afpf_mbox_intr(cptpf); +destroy_afpf_mbox: + cptpf_afpf_mbox_destroy(cptpf); +clear_drvdata: + pci_set_drvdata(pdev, NULL); + return err; +} + +static void otx2_cptpf_remove(struct pci_dev *pdev) +{ + struct otx2_cptpf_dev *cptpf = pci_get_drvdata(pdev); + + if (!cptpf) + return; + + cptpf_sriov_disable(pdev); + otx2_cpt_unregister_dl(cptpf); + + /* Cleanup Inline CPT LF's if attached */ + if (cptpf->lfs.lfs_num) + otx2_inline_cptlf_cleanup(&cptpf->lfs); + + if (cptpf->cpt1_lfs.lfs_num) + otx2_inline_cptlf_cleanup(&cptpf->cpt1_lfs); + + /* Delete sysfs entry created for kernel VF limits */ + sysfs_remove_group(&pdev->dev.kobj, &cptpf_sysfs_group); + /* Cleanup engine groups */ + otx2_cpt_cleanup_eng_grps(pdev, &cptpf->eng_grps); + /* Disable AF-PF mailbox interrupt */ + cptpf_disable_afpf_mbox_intr(cptpf); + /* Destroy AF-PF mbox */ + cptpf_afpf_mbox_destroy(cptpf); + /* Free LMTST memory */ + cn10k_cpt_lmtst_free(pdev, &cptpf->lfs); + pci_set_drvdata(pdev, NULL); +} + +/* Supported devices */ +static const struct pci_device_id otx2_cpt_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OTX2_CPT_PCI_PF_DEVICE_ID) }, + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, CN10K_CPT_PCI_PF_DEVICE_ID) }, + { 0, } /* end of table */ +}; +MODULE_DEVICE_TABLE(pci, otx2_cpt_id_table); + +static struct pci_driver otx2_cpt_pci_driver = { + .name = OTX2_CPT_DRV_NAME, + .id_table = otx2_cpt_id_table, + .probe = otx2_cptpf_probe, + .remove = otx2_cptpf_remove, + .sriov_configure = otx2_cptpf_sriov_configure +}; + +module_pci_driver(otx2_cpt_pci_driver); + +MODULE_IMPORT_NS("CRYPTO_DEV_OCTEONTX2_CPT"); + +MODULE_AUTHOR("Marvell"); +MODULE_DESCRIPTION(OTX2_CPT_DRV_STRING); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c new file mode 100644 index 000000000..b4b2d3d1c --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c @@ -0,0 +1,661 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include "otx2_cpt_common.h" +#include "otx2_cptpf.h" +#include "rvu_reg.h" + +/* Fastpath ipsec opcode with inplace processing */ +#define CPT_INLINE_RX_OPCODE (0x26 | (1 << 6)) +#define CN10K_CPT_INLINE_RX_OPCODE (0x29 | (1 << 6)) + +#define cpt_inline_rx_opcode(pdev) \ +({ \ + u8 opcode; \ + if (is_dev_otx2(pdev)) \ + opcode = CPT_INLINE_RX_OPCODE; \ + else \ + opcode = CN10K_CPT_INLINE_RX_OPCODE; \ + (opcode); \ +}) + +/* + * CPT PF driver version, It will be incremented by 1 for every feature + * addition in CPT mailbox messages. + */ +#define OTX2_CPT_PF_DRV_VERSION 0x1 + +static int forward_to_af(struct otx2_cptpf_dev *cptpf, + struct otx2_cptvf_info *vf, + struct mbox_msghdr *req, int size) +{ + struct mbox_msghdr *msg; + int ret; + + mutex_lock(&cptpf->lock); + msg = otx2_mbox_alloc_msg(&cptpf->afpf_mbox, 0, size); + if (msg == NULL) { + mutex_unlock(&cptpf->lock); + return -ENOMEM; + } + + memcpy((uint8_t *)msg + sizeof(struct mbox_msghdr), + (uint8_t *)req + sizeof(struct mbox_msghdr), size); + msg->id = req->id; + msg->pcifunc = req->pcifunc; + msg->sig = req->sig; + msg->ver = req->ver; + + ret = otx2_cpt_sync_mbox_msg(&cptpf->afpf_mbox); + /* Error code -EIO indicate there is a communication failure + * to the AF. Rest of the error codes indicate that AF processed + * VF messages and set the error codes in response messages + * (if any) so simply forward responses to VF. + */ + if (ret == -EIO) { + dev_warn(&cptpf->pdev->dev, + "AF not responding to VF%d messages\n", vf->vf_id); + mutex_unlock(&cptpf->lock); + return ret; + } + mutex_unlock(&cptpf->lock); + return 0; +} + +static int handle_msg_get_caps(struct otx2_cptpf_dev *cptpf, + struct otx2_cptvf_info *vf, + struct mbox_msghdr *req) +{ + struct otx2_cpt_caps_rsp *rsp; + + rsp = (struct otx2_cpt_caps_rsp *) + otx2_mbox_alloc_msg(&cptpf->vfpf_mbox, vf->vf_id, + sizeof(*rsp)); + if (!rsp) + return -ENOMEM; + + rsp->hdr.id = MBOX_MSG_GET_CAPS; + rsp->hdr.sig = OTX2_MBOX_RSP_SIG; + rsp->hdr.pcifunc = req->pcifunc; + rsp->cpt_pf_drv_version = OTX2_CPT_PF_DRV_VERSION; + rsp->cpt_revision = cptpf->eng_grps.rid; + memcpy(&rsp->eng_caps, &cptpf->eng_caps, sizeof(rsp->eng_caps)); + + return 0; +} + +static int handle_msg_get_eng_grp_num(struct otx2_cptpf_dev *cptpf, + struct otx2_cptvf_info *vf, + struct mbox_msghdr *req) +{ + struct otx2_cpt_egrp_num_msg *grp_req; + struct otx2_cpt_egrp_num_rsp *rsp; + + grp_req = (struct otx2_cpt_egrp_num_msg *)req; + rsp = (struct otx2_cpt_egrp_num_rsp *) + otx2_mbox_alloc_msg(&cptpf->vfpf_mbox, vf->vf_id, sizeof(*rsp)); + if (!rsp) + return -ENOMEM; + + rsp->hdr.id = MBOX_MSG_GET_ENG_GRP_NUM; + rsp->hdr.sig = OTX2_MBOX_RSP_SIG; + rsp->hdr.pcifunc = req->pcifunc; + rsp->eng_type = grp_req->eng_type; + rsp->eng_grp_num = otx2_cpt_get_eng_grp(&cptpf->eng_grps, + grp_req->eng_type); + + return 0; +} + +static int handle_msg_kvf_limits(struct otx2_cptpf_dev *cptpf, + struct otx2_cptvf_info *vf, + struct mbox_msghdr *req) +{ + struct otx2_cpt_kvf_limits_rsp *rsp; + + rsp = (struct otx2_cpt_kvf_limits_rsp *) + otx2_mbox_alloc_msg(&cptpf->vfpf_mbox, vf->vf_id, sizeof(*rsp)); + if (!rsp) + return -ENOMEM; + + rsp->hdr.id = MBOX_MSG_GET_KVF_LIMITS; + rsp->hdr.sig = OTX2_MBOX_RSP_SIG; + rsp->hdr.pcifunc = req->pcifunc; + rsp->kvf_limits = cptpf->kvf_limits; + + return 0; +} + +static int send_inline_ipsec_inbound_msg(struct otx2_cptpf_dev *cptpf, + int sso_pf_func, u8 slot) +{ + struct cpt_inline_ipsec_cfg_msg *req; + struct pci_dev *pdev = cptpf->pdev; + + req = (struct cpt_inline_ipsec_cfg_msg *) + otx2_mbox_alloc_msg_rsp(&cptpf->afpf_mbox, 0, + sizeof(*req), sizeof(struct msg_rsp)); + if (req == NULL) { + dev_err(&pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + memset(req, 0, sizeof(*req)); + req->hdr.id = MBOX_MSG_CPT_INLINE_IPSEC_CFG; + req->hdr.sig = OTX2_MBOX_REQ_SIG; + req->hdr.pcifunc = OTX2_CPT_RVU_PFFUNC(cptpf->pdev, cptpf->pf_id, 0); + req->dir = CPT_INLINE_INBOUND; + req->slot = slot; + req->sso_pf_func_ovrd = cptpf->sso_pf_func_ovrd; + req->sso_pf_func = sso_pf_func; + req->enable = 1; + + return otx2_cpt_send_mbox_msg(&cptpf->afpf_mbox, pdev); +} + +static int rx_inline_ipsec_lf_cfg(struct otx2_cptpf_dev *cptpf, u8 egrp, + struct otx2_cpt_rx_inline_lf_cfg *req) +{ + struct nix_inline_ipsec_cfg *nix_req; + struct pci_dev *pdev = cptpf->pdev; + int ret; + + nix_req = (struct nix_inline_ipsec_cfg *) + otx2_mbox_alloc_msg_rsp(&cptpf->afpf_mbox, 0, + sizeof(*nix_req), + sizeof(struct msg_rsp)); + if (nix_req == NULL) { + dev_err(&pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + memset(nix_req, 0, sizeof(*nix_req)); + nix_req->hdr.id = MBOX_MSG_NIX_INLINE_IPSEC_CFG; + nix_req->hdr.sig = OTX2_MBOX_REQ_SIG; + nix_req->enable = 1; + nix_req->credit_th = req->credit_th; + nix_req->bpid = req->bpid; + if (!req->credit || req->credit > OTX2_CPT_INST_QLEN_MSGS) + nix_req->cpt_credit = OTX2_CPT_INST_QLEN_MSGS - 1; + else + nix_req->cpt_credit = req->credit - 1; + nix_req->gen_cfg.egrp = egrp; + if (req->opcode) + nix_req->gen_cfg.opcode = req->opcode; + else + nix_req->gen_cfg.opcode = cpt_inline_rx_opcode(pdev); + nix_req->gen_cfg.param1 = req->param1; + nix_req->gen_cfg.param2 = req->param2; + nix_req->inst_qsel.cpt_pf_func = + OTX2_CPT_RVU_PFFUNC(cptpf->pdev, cptpf->pf_id, 0); + nix_req->inst_qsel.cpt_slot = 0; + ret = otx2_cpt_send_mbox_msg(&cptpf->afpf_mbox, pdev); + if (ret) + return ret; + + if (cptpf->has_cpt1) { + ret = send_inline_ipsec_inbound_msg(cptpf, req->sso_pf_func, 1); + if (ret) + return ret; + } + + return send_inline_ipsec_inbound_msg(cptpf, req->sso_pf_func, 0); +} + +int +otx2_inline_cptlf_setup(struct otx2_cptpf_dev *cptpf, + struct otx2_cptlfs_info *lfs, u8 egrp, int num_lfs) +{ + int ret; + + ret = otx2_cptlf_init(lfs, 1 << egrp, OTX2_CPT_QUEUE_HI_PRIO, 1); + if (ret) { + dev_err(&cptpf->pdev->dev, + "LF configuration failed for RX inline ipsec.\n"); + return ret; + } + + /* Get msix offsets for attached LFs */ + ret = otx2_cpt_msix_offset_msg(lfs); + if (ret) + goto cleanup_lf; + + /* Register for CPT LF Misc interrupts */ + ret = otx2_cptlf_register_misc_interrupts(lfs); + if (ret) + goto free_irq; + + return 0; +free_irq: + otx2_cptlf_unregister_misc_interrupts(lfs); +cleanup_lf: + otx2_cptlf_shutdown(lfs); + return ret; +} + +void +otx2_inline_cptlf_cleanup(struct otx2_cptlfs_info *lfs) +{ + /* Unregister misc interrupt */ + otx2_cptlf_unregister_misc_interrupts(lfs); + + /* Cleanup LFs */ + otx2_cptlf_shutdown(lfs); +} + +static int handle_msg_rx_inline_ipsec_lf_cfg(struct otx2_cptpf_dev *cptpf, + struct mbox_msghdr *req) +{ + struct otx2_cpt_rx_inline_lf_cfg *cfg_req; + int num_lfs = 1, ret; + u8 egrp; + + cfg_req = (struct otx2_cpt_rx_inline_lf_cfg *)req; + if (cptpf->lfs.lfs_num) { + dev_err(&cptpf->pdev->dev, + "LF is already configured for RX inline ipsec.\n"); + return -EEXIST; + } + /* + * Allow LFs to execute requests destined to only grp IE_TYPES and + * set queue priority of each LF to high + */ + egrp = otx2_cpt_get_eng_grp(&cptpf->eng_grps, OTX2_CPT_IE_TYPES); + if (egrp == OTX2_CPT_INVALID_CRYPTO_ENG_GRP) { + dev_err(&cptpf->pdev->dev, + "Engine group for inline ipsec is not available\n"); + return -ENOENT; + } + + cptpf->lfs.global_slot = 0; + cptpf->lfs.ctx_ilen_ovrd = cfg_req->ctx_ilen_valid; + cptpf->lfs.ctx_ilen = cfg_req->ctx_ilen; + + ret = otx2_inline_cptlf_setup(cptpf, &cptpf->lfs, egrp, num_lfs); + if (ret) { + dev_err(&cptpf->pdev->dev, "Inline-Ipsec CPT0 LF setup failed.\n"); + return ret; + } + + if (cptpf->has_cpt1) { + cptpf->rsrc_req_blkaddr = BLKADDR_CPT1; + cptpf->cpt1_lfs.global_slot = num_lfs; + cptpf->cpt1_lfs.ctx_ilen_ovrd = cfg_req->ctx_ilen_valid; + cptpf->cpt1_lfs.ctx_ilen = cfg_req->ctx_ilen; + ret = otx2_inline_cptlf_setup(cptpf, &cptpf->cpt1_lfs, egrp, + num_lfs); + if (ret) { + dev_err(&cptpf->pdev->dev, "Inline CPT1 LF setup failed.\n"); + goto lf_cleanup; + } + cptpf->rsrc_req_blkaddr = 0; + } + + ret = rx_inline_ipsec_lf_cfg(cptpf, egrp, cfg_req); + if (ret) + goto lf1_cleanup; + + return 0; + +lf1_cleanup: + otx2_inline_cptlf_cleanup(&cptpf->cpt1_lfs); +lf_cleanup: + otx2_inline_cptlf_cleanup(&cptpf->lfs); + return ret; +} + +static int cptpf_handle_vf_req(struct otx2_cptpf_dev *cptpf, + struct otx2_cptvf_info *vf, + struct mbox_msghdr *req, int size) +{ + int err = 0; + + /* Check if msg is valid, if not reply with an invalid msg */ + if (req->sig != OTX2_MBOX_REQ_SIG) + goto inval_msg; + + switch (req->id) { + case MBOX_MSG_GET_ENG_GRP_NUM: + err = handle_msg_get_eng_grp_num(cptpf, vf, req); + break; + case MBOX_MSG_GET_CAPS: + err = handle_msg_get_caps(cptpf, vf, req); + break; + case MBOX_MSG_GET_KVF_LIMITS: + err = handle_msg_kvf_limits(cptpf, vf, req); + break; + case MBOX_MSG_RX_INLINE_IPSEC_LF_CFG: + err = handle_msg_rx_inline_ipsec_lf_cfg(cptpf, req); + break; + + default: + err = forward_to_af(cptpf, vf, req, size); + break; + } + return err; + +inval_msg: + otx2_reply_invalid_msg(&cptpf->vfpf_mbox, vf->vf_id, 0, req->id); + otx2_mbox_msg_send(&cptpf->vfpf_mbox, vf->vf_id); + return err; +} + +irqreturn_t otx2_cptpf_vfpf_mbox_intr(int __always_unused irq, void *arg) +{ + struct otx2_cptpf_dev *cptpf = arg; + struct otx2_cptvf_info *vf; + int i, vf_idx; + u64 intr; + + /* + * Check which VF has raised an interrupt and schedule + * corresponding work queue to process the messages + */ + for (i = 0; i < 2; i++) { + /* Read the interrupt bits */ + intr = otx2_cpt_read64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INTX(i)); + + for (vf_idx = i * 64; vf_idx < cptpf->enabled_vfs; vf_idx++) { + vf = &cptpf->vf[vf_idx]; + if (intr & (1ULL << vf->intr_idx)) { + queue_work(cptpf->vfpf_mbox_wq, + &vf->vfpf_mbox_work); + /* Clear the interrupt */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, + 0, RVU_PF_VFPF_MBOX_INTX(i), + BIT_ULL(vf->intr_idx)); + } + } + } + return IRQ_HANDLED; +} + +void otx2_cptpf_vfpf_mbox_handler(struct work_struct *work) +{ + struct otx2_cptpf_dev *cptpf; + struct otx2_cptvf_info *vf; + struct otx2_mbox_dev *mdev; + struct mbox_hdr *req_hdr; + struct mbox_msghdr *msg; + struct otx2_mbox *mbox; + int offset, i, err; + + vf = container_of(work, struct otx2_cptvf_info, vfpf_mbox_work); + cptpf = vf->cptpf; + mbox = &cptpf->vfpf_mbox; + /* sync with mbox memory region */ + smp_rmb(); + mdev = &mbox->dev[vf->vf_id]; + /* Process received mbox messages */ + req_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start); + offset = mbox->rx_start + ALIGN(sizeof(*req_hdr), MBOX_MSG_ALIGN); + + for (i = 0; i < req_hdr->num_msgs; i++) { + msg = (struct mbox_msghdr *)(mdev->mbase + offset); + + /* Set which VF sent this message based on mbox IRQ */ + msg->pcifunc = rvu_make_pcifunc(cptpf->pdev, cptpf->pf_id, + (vf->vf_id + 1)); + err = cptpf_handle_vf_req(cptpf, vf, msg, + msg->next_msgoff - offset); + /* + * Behave as the AF, drop the msg if there is + * no memory, timeout handling also goes here + */ + if (err == -ENOMEM || err == -EIO) + break; + offset = msg->next_msgoff; + /* Write barrier required for VF responses which are handled by + * PF driver and not forwarded to AF. + */ + smp_wmb(); + } + /* Send mbox responses to VF */ + if (mdev->num_msgs) + otx2_mbox_msg_send(mbox, vf->vf_id); +} + +irqreturn_t otx2_cptpf_afpf_mbox_intr(int __always_unused irq, void *arg) +{ + struct otx2_cptpf_dev *cptpf = arg; + struct otx2_mbox_dev *mdev; + struct otx2_mbox *mbox; + struct mbox_hdr *hdr; + u64 intr; + + /* Read the interrupt bits */ + intr = otx2_cpt_read64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_INT); + + if (intr & 0x1ULL) { + mbox = &cptpf->afpf_mbox; + mdev = &mbox->dev[0]; + hdr = mdev->mbase + mbox->rx_start; + if (hdr->num_msgs) + /* Schedule work queue function to process the MBOX request */ + queue_work(cptpf->afpf_mbox_wq, &cptpf->afpf_mbox_work); + + mbox = &cptpf->afpf_mbox_up; + mdev = &mbox->dev[0]; + hdr = mdev->mbase + mbox->rx_start; + if (hdr->num_msgs) + /* Schedule work queue function to process the MBOX request */ + queue_work(cptpf->afpf_mbox_wq, &cptpf->afpf_mbox_up_work); + /* Clear and ack the interrupt */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_INT, + 0x1ULL); + } + return IRQ_HANDLED; +} + +static void process_afpf_mbox_msg(struct otx2_cptpf_dev *cptpf, + struct mbox_msghdr *msg) +{ + struct otx2_cptlfs_info *lfs = &cptpf->lfs; + struct device *dev = &cptpf->pdev->dev; + struct cpt_rd_wr_reg_msg *rsp_rd_wr; + struct msix_offset_rsp *rsp_msix; + int i; + + if (msg->id >= MBOX_MSG_MAX) { + dev_err(dev, "MBOX msg with unknown ID %d\n", msg->id); + return; + } + if (msg->sig != OTX2_MBOX_RSP_SIG) { + dev_err(dev, "MBOX msg with wrong signature %x, ID %d\n", + msg->sig, msg->id); + return; + } + if (cptpf->rsrc_req_blkaddr == BLKADDR_CPT1) + lfs = &cptpf->cpt1_lfs; + + switch (msg->id) { + case MBOX_MSG_READY: + cptpf->pf_id = rvu_get_pf(cptpf->pdev, msg->pcifunc); + break; + case MBOX_MSG_MSIX_OFFSET: + rsp_msix = (struct msix_offset_rsp *) msg; + for (i = 0; i < rsp_msix->cptlfs; i++) + lfs->lf[i].msix_offset = rsp_msix->cptlf_msixoff[i]; + + for (i = 0; i < rsp_msix->cpt1_lfs; i++) + lfs->lf[i].msix_offset = rsp_msix->cpt1_lf_msixoff[i]; + break; + case MBOX_MSG_CPT_RD_WR_REGISTER: + rsp_rd_wr = (struct cpt_rd_wr_reg_msg *)msg; + if (msg->rc) { + dev_err(dev, "Reg %llx rd/wr(%d) failed %d\n", + rsp_rd_wr->reg_offset, rsp_rd_wr->is_write, + msg->rc); + return; + } + if (!rsp_rd_wr->is_write) + *rsp_rd_wr->ret_val = rsp_rd_wr->val; + break; + case MBOX_MSG_ATTACH_RESOURCES: + if (!msg->rc) + lfs->are_lfs_attached = 1; + break; + case MBOX_MSG_DETACH_RESOURCES: + if (!msg->rc) + lfs->are_lfs_attached = 0; + break; + case MBOX_MSG_CPT_INLINE_IPSEC_CFG: + case MBOX_MSG_NIX_INLINE_IPSEC_CFG: + case MBOX_MSG_CPT_LF_RESET: + case MBOX_MSG_LMTST_TBL_SETUP: + break; + + default: + dev_err(dev, + "Unsupported msg %d received.\n", msg->id); + break; + } +} + +static void forward_to_vf(struct otx2_cptpf_dev *cptpf, struct mbox_msghdr *msg, + int vf_id, int size) +{ + struct otx2_mbox *vfpf_mbox; + struct mbox_msghdr *fwd; + + if (msg->id >= MBOX_MSG_MAX) { + dev_err(&cptpf->pdev->dev, + "MBOX msg with unknown ID %d\n", msg->id); + return; + } + if (msg->sig != OTX2_MBOX_RSP_SIG) { + dev_err(&cptpf->pdev->dev, + "MBOX msg with wrong signature %x, ID %d\n", + msg->sig, msg->id); + return; + } + vfpf_mbox = &cptpf->vfpf_mbox; + vf_id--; + if (vf_id >= cptpf->enabled_vfs) { + dev_err(&cptpf->pdev->dev, + "MBOX msg to unknown VF: %d >= %d\n", + vf_id, cptpf->enabled_vfs); + return; + } + if (msg->id == MBOX_MSG_VF_FLR) + return; + + fwd = otx2_mbox_alloc_msg(vfpf_mbox, vf_id, size); + if (!fwd) { + dev_err(&cptpf->pdev->dev, + "Forwarding to VF%d failed.\n", vf_id); + return; + } + memcpy((uint8_t *)fwd + sizeof(struct mbox_msghdr), + (uint8_t *)msg + sizeof(struct mbox_msghdr), size); + fwd->id = msg->id; + fwd->pcifunc = msg->pcifunc; + fwd->sig = msg->sig; + fwd->ver = msg->ver; + fwd->rc = msg->rc; +} + +/* Handle mailbox messages received from AF */ +void otx2_cptpf_afpf_mbox_handler(struct work_struct *work) +{ + struct otx2_cptpf_dev *cptpf; + struct otx2_mbox *afpf_mbox; + struct otx2_mbox_dev *mdev; + struct mbox_hdr *rsp_hdr; + struct mbox_msghdr *msg; + int offset, vf_id, i; + + cptpf = container_of(work, struct otx2_cptpf_dev, afpf_mbox_work); + afpf_mbox = &cptpf->afpf_mbox; + mdev = &afpf_mbox->dev[0]; + /* Sync mbox data into memory */ + smp_wmb(); + + rsp_hdr = (struct mbox_hdr *)(mdev->mbase + afpf_mbox->rx_start); + offset = ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN); + + for (i = 0; i < rsp_hdr->num_msgs; i++) { + msg = (struct mbox_msghdr *)(mdev->mbase + afpf_mbox->rx_start + + offset); + vf_id = (msg->pcifunc >> RVU_PFVF_FUNC_SHIFT) & + RVU_PFVF_FUNC_MASK; + if (vf_id > 0) + forward_to_vf(cptpf, msg, vf_id, + msg->next_msgoff - offset); + else + process_afpf_mbox_msg(cptpf, msg); + + offset = msg->next_msgoff; + /* Sync VF response ready to be sent */ + smp_wmb(); + mdev->msgs_acked++; + } + otx2_mbox_reset(afpf_mbox, 0); +} + +static void handle_msg_cpt_inst_lmtst(struct otx2_cptpf_dev *cptpf, + struct mbox_msghdr *msg) +{ + struct cpt_inst_lmtst_req *req = (struct cpt_inst_lmtst_req *)msg; + struct otx2_cptlfs_info *lfs = &cptpf->lfs; + struct msg_rsp *rsp; + + if (cptpf->lfs.lfs_num) + lfs->ops->send_cmd((union otx2_cpt_inst_s *)req->inst, 1, + &lfs->lf[0]); + + rsp = (struct msg_rsp *)otx2_mbox_alloc_msg(&cptpf->afpf_mbox_up, 0, + sizeof(*rsp)); + if (!rsp) + return; + + rsp->hdr.id = msg->id; + rsp->hdr.sig = OTX2_MBOX_RSP_SIG; + rsp->hdr.pcifunc = 0; + rsp->hdr.rc = 0; +} + +static void process_afpf_mbox_up_msg(struct otx2_cptpf_dev *cptpf, + struct mbox_msghdr *msg) +{ + if (msg->id >= MBOX_MSG_MAX) { + dev_err(&cptpf->pdev->dev, + "MBOX msg with unknown ID %d\n", msg->id); + return; + } + + switch (msg->id) { + case MBOX_MSG_CPT_INST_LMTST: + handle_msg_cpt_inst_lmtst(cptpf, msg); + break; + default: + otx2_reply_invalid_msg(&cptpf->afpf_mbox_up, 0, 0, msg->id); + } +} + +void otx2_cptpf_afpf_mbox_up_handler(struct work_struct *work) +{ + struct otx2_cptpf_dev *cptpf; + struct otx2_mbox_dev *mdev; + struct mbox_hdr *rsp_hdr; + struct mbox_msghdr *msg; + struct otx2_mbox *mbox; + int offset, i; + + cptpf = container_of(work, struct otx2_cptpf_dev, afpf_mbox_up_work); + mbox = &cptpf->afpf_mbox_up; + mdev = &mbox->dev[0]; + /* Sync mbox data into memory */ + smp_wmb(); + + rsp_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start); + offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN); + + for (i = 0; i < rsp_hdr->num_msgs; i++) { + msg = (struct mbox_msghdr *)(mdev->mbase + offset); + + process_afpf_mbox_up_msg(cptpf, msg); + + offset = mbox->rx_start + msg->next_msgoff; + } + otx2_mbox_msg_send(mbox, 0); +} diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c new file mode 100644 index 000000000..465f00e74 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c @@ -0,0 +1,1797 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include +#include +#include +#include +#include "otx2_cptpf_ucode.h" +#include "otx2_cpt_common.h" +#include "otx2_cptpf.h" +#include "otx2_cptlf.h" +#include "otx2_cpt_reqmgr.h" +#include "rvu_reg.h" + +#define CSR_DELAY 30 + +#define LOADFVC_RLEN 8 +#define LOADFVC_MAJOR_OP 0x01 +#define LOADFVC_MINOR_OP 0x08 + +/* + * Interval to flush dirty data for next CTX entry. The interval is measured + * in increments of 10ns(interval time = CTX_FLUSH_TIMER_COUNT * 10ns). + */ +#define CTX_FLUSH_TIMER_CNT 0x2FAF0 + +struct fw_info_t { + struct list_head ucodes; +}; + +static struct otx2_cpt_bitmap get_cores_bmap(struct device *dev, + struct otx2_cpt_eng_grp_info *eng_grp) +{ + struct otx2_cpt_bitmap bmap = { {0} }; + bool found = false; + int i; + + if (eng_grp->g->engs_num < 0 || + eng_grp->g->engs_num > OTX2_CPT_MAX_ENGINES) { + dev_err(dev, "unsupported number of engines %d on octeontx2\n", + eng_grp->g->engs_num); + return bmap; + } + + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + if (eng_grp->engs[i].type) { + bitmap_or(bmap.bits, bmap.bits, + eng_grp->engs[i].bmap, + eng_grp->g->engs_num); + bmap.size = eng_grp->g->engs_num; + found = true; + } + } + + if (!found) + dev_err(dev, "No engines reserved for engine group %d\n", + eng_grp->idx); + return bmap; +} + +static int is_eng_type(int val, int eng_type) +{ + return val & (1 << eng_type); +} + +static int is_2nd_ucode_used(struct otx2_cpt_eng_grp_info *eng_grp) +{ + if (eng_grp->ucode[1].type) + return true; + else + return false; +} + +static void set_ucode_filename(struct otx2_cpt_ucode *ucode, + const char *filename) +{ + strscpy(ucode->filename, filename); +} + +static char *get_eng_type_str(int eng_type) +{ + char *str = "unknown"; + + switch (eng_type) { + case OTX2_CPT_SE_TYPES: + str = "SE"; + break; + + case OTX2_CPT_IE_TYPES: + str = "IE"; + break; + + case OTX2_CPT_AE_TYPES: + str = "AE"; + break; + } + return str; +} + +static char *get_ucode_type_str(int ucode_type) +{ + char *str = "unknown"; + + switch (ucode_type) { + case (1 << OTX2_CPT_SE_TYPES): + str = "SE"; + break; + + case (1 << OTX2_CPT_IE_TYPES): + str = "IE"; + break; + + case (1 << OTX2_CPT_AE_TYPES): + str = "AE"; + break; + + case (1 << OTX2_CPT_SE_TYPES | 1 << OTX2_CPT_IE_TYPES): + str = "SE+IPSEC"; + break; + } + return str; +} + +static int get_ucode_type(struct device *dev, + struct otx2_cpt_ucode_hdr *ucode_hdr, + int *ucode_type, u16 rid) +{ + char ver_str_prefix[OTX2_CPT_UCODE_VER_STR_SZ]; + char tmp_ver_str[OTX2_CPT_UCODE_VER_STR_SZ]; + int i, val = 0; + u8 nn; + + strscpy(tmp_ver_str, ucode_hdr->ver_str); + for (i = 0; i < strlen(tmp_ver_str); i++) + tmp_ver_str[i] = tolower(tmp_ver_str[i]); + + sprintf(ver_str_prefix, "ocpt-%02d", rid); + if (!strnstr(tmp_ver_str, ver_str_prefix, OTX2_CPT_UCODE_VER_STR_SZ)) + return -EINVAL; + + nn = ucode_hdr->ver_num.nn; + if (strnstr(tmp_ver_str, "se-", OTX2_CPT_UCODE_VER_STR_SZ) && + (nn == OTX2_CPT_SE_UC_TYPE1 || nn == OTX2_CPT_SE_UC_TYPE2 || + nn == OTX2_CPT_SE_UC_TYPE3)) + val |= 1 << OTX2_CPT_SE_TYPES; + if (strnstr(tmp_ver_str, "ie-", OTX2_CPT_UCODE_VER_STR_SZ) && + (nn == OTX2_CPT_IE_UC_TYPE1 || nn == OTX2_CPT_IE_UC_TYPE2 || + nn == OTX2_CPT_IE_UC_TYPE3)) + val |= 1 << OTX2_CPT_IE_TYPES; + if (strnstr(tmp_ver_str, "ae", OTX2_CPT_UCODE_VER_STR_SZ) && + nn == OTX2_CPT_AE_UC_TYPE) + val |= 1 << OTX2_CPT_AE_TYPES; + + *ucode_type = val; + + if (!val) + return -EINVAL; + + return 0; +} + +static int __write_ucode_base(struct otx2_cptpf_dev *cptpf, int eng, + dma_addr_t dma_addr, int blkaddr) +{ + return otx2_cpt_write_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_EXEX_UCODE_BASE(eng), + (u64)dma_addr, blkaddr); +} + +static int cptx_set_ucode_base(struct otx2_cpt_eng_grp_info *eng_grp, + struct otx2_cptpf_dev *cptpf, int blkaddr) +{ + struct otx2_cpt_engs_rsvd *engs; + dma_addr_t dma_addr; + int i, bit, ret; + + /* Set PF number for microcode fetches */ + ret = otx2_cpt_write_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_PF_FUNC, + rvu_make_pcifunc(cptpf->pdev, + cptpf->pf_id, 0), + blkaddr); + if (ret) + return ret; + + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + engs = &eng_grp->engs[i]; + if (!engs->type) + continue; + + dma_addr = engs->ucode->dma; + + /* + * Set UCODE_BASE only for the cores which are not used, + * other cores should have already valid UCODE_BASE set + */ + for_each_set_bit(bit, engs->bmap, eng_grp->g->engs_num) + if (!eng_grp->g->eng_ref_cnt[bit]) { + ret = __write_ucode_base(cptpf, bit, dma_addr, + blkaddr); + if (ret) + return ret; + } + } + return 0; +} + +static int cpt_set_ucode_base(struct otx2_cpt_eng_grp_info *eng_grp, void *obj) +{ + struct otx2_cptpf_dev *cptpf = obj; + int ret; + + if (cptpf->has_cpt1) { + ret = cptx_set_ucode_base(eng_grp, cptpf, BLKADDR_CPT1); + if (ret) + return ret; + } + return cptx_set_ucode_base(eng_grp, cptpf, BLKADDR_CPT0); +} + +static int cptx_detach_and_disable_cores(struct otx2_cpt_eng_grp_info *eng_grp, + struct otx2_cptpf_dev *cptpf, + struct otx2_cpt_bitmap bmap, + int blkaddr) +{ + int i, timeout = 10; + int busy, ret; + u64 reg = 0; + + /* Detach the cores from group */ + for_each_set_bit(i, bmap.bits, bmap.size) { + ret = otx2_cpt_read_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_EXEX_CTL2(i), ®, blkaddr); + if (ret) + return ret; + + if (reg & (1ull << eng_grp->idx)) { + eng_grp->g->eng_ref_cnt[i]--; + reg &= ~(1ull << eng_grp->idx); + + ret = otx2_cpt_write_af_reg(&cptpf->afpf_mbox, + cptpf->pdev, + CPT_AF_EXEX_CTL2(i), reg, + blkaddr); + if (ret) + return ret; + } + } + + /* Wait for cores to become idle */ + do { + busy = 0; + usleep_range(10000, 20000); + if (timeout-- < 0) + return -EBUSY; + + for_each_set_bit(i, bmap.bits, bmap.size) { + ret = otx2_cpt_read_af_reg(&cptpf->afpf_mbox, + cptpf->pdev, + CPT_AF_EXEX_STS(i), ®, + blkaddr); + if (ret) + return ret; + + if (reg & 0x1) { + busy = 1; + break; + } + } + } while (busy); + + /* Disable the cores only if they are not used anymore */ + for_each_set_bit(i, bmap.bits, bmap.size) { + if (!eng_grp->g->eng_ref_cnt[i]) { + ret = otx2_cpt_write_af_reg(&cptpf->afpf_mbox, + cptpf->pdev, + CPT_AF_EXEX_CTL(i), 0x0, + blkaddr); + if (ret) + return ret; + } + } + + return 0; +} + +static int cpt_detach_and_disable_cores(struct otx2_cpt_eng_grp_info *eng_grp, + void *obj) +{ + struct otx2_cptpf_dev *cptpf = obj; + struct otx2_cpt_bitmap bmap; + int ret; + + bmap = get_cores_bmap(&cptpf->pdev->dev, eng_grp); + if (!bmap.size) + return -EINVAL; + + if (cptpf->has_cpt1) { + ret = cptx_detach_and_disable_cores(eng_grp, cptpf, bmap, + BLKADDR_CPT1); + if (ret) + return ret; + } + return cptx_detach_and_disable_cores(eng_grp, cptpf, bmap, + BLKADDR_CPT0); +} + +static int cptx_attach_and_enable_cores(struct otx2_cpt_eng_grp_info *eng_grp, + struct otx2_cptpf_dev *cptpf, + struct otx2_cpt_bitmap bmap, + int blkaddr) +{ + u64 reg = 0; + int i, ret; + + /* Attach the cores to the group */ + for_each_set_bit(i, bmap.bits, bmap.size) { + ret = otx2_cpt_read_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_EXEX_CTL2(i), ®, blkaddr); + if (ret) + return ret; + + if (!(reg & (1ull << eng_grp->idx))) { + eng_grp->g->eng_ref_cnt[i]++; + reg |= 1ull << eng_grp->idx; + + ret = otx2_cpt_write_af_reg(&cptpf->afpf_mbox, + cptpf->pdev, + CPT_AF_EXEX_CTL2(i), reg, + blkaddr); + if (ret) + return ret; + } + } + + /* Enable the cores */ + for_each_set_bit(i, bmap.bits, bmap.size) { + ret = otx2_cpt_add_write_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_EXEX_CTL(i), 0x1, + blkaddr); + if (ret) + return ret; + } + return otx2_cpt_send_af_reg_requests(&cptpf->afpf_mbox, cptpf->pdev); +} + +static int cpt_attach_and_enable_cores(struct otx2_cpt_eng_grp_info *eng_grp, + void *obj) +{ + struct otx2_cptpf_dev *cptpf = obj; + struct otx2_cpt_bitmap bmap; + int ret; + + bmap = get_cores_bmap(&cptpf->pdev->dev, eng_grp); + if (!bmap.size) + return -EINVAL; + + if (cptpf->has_cpt1) { + ret = cptx_attach_and_enable_cores(eng_grp, cptpf, bmap, + BLKADDR_CPT1); + if (ret) + return ret; + } + return cptx_attach_and_enable_cores(eng_grp, cptpf, bmap, BLKADDR_CPT0); +} + +static int load_fw(struct device *dev, struct fw_info_t *fw_info, + char *filename, u16 rid) +{ + struct otx2_cpt_ucode_hdr *ucode_hdr; + struct otx2_cpt_uc_info_t *uc_info; + int ucode_type, ucode_size; + int ret; + + uc_info = kzalloc_obj(*uc_info); + if (!uc_info) + return -ENOMEM; + + ret = request_firmware(&uc_info->fw, filename, dev); + if (ret) + goto free_uc_info; + + ucode_hdr = (struct otx2_cpt_ucode_hdr *)uc_info->fw->data; + ret = get_ucode_type(dev, ucode_hdr, &ucode_type, rid); + if (ret) + goto release_fw; + + ucode_size = ntohl(ucode_hdr->code_length) * 2; + if (!ucode_size) { + dev_err(dev, "Ucode %s invalid size\n", filename); + ret = -EINVAL; + goto release_fw; + } + + set_ucode_filename(&uc_info->ucode, filename); + memcpy(uc_info->ucode.ver_str, ucode_hdr->ver_str, + OTX2_CPT_UCODE_VER_STR_SZ); + uc_info->ucode.ver_str[OTX2_CPT_UCODE_VER_STR_SZ] = 0; + uc_info->ucode.ver_num = ucode_hdr->ver_num; + uc_info->ucode.type = ucode_type; + uc_info->ucode.size = ucode_size; + list_add_tail(&uc_info->list, &fw_info->ucodes); + + return 0; + +release_fw: + release_firmware(uc_info->fw); +free_uc_info: + kfree(uc_info); + return ret; +} + +static void cpt_ucode_release_fw(struct fw_info_t *fw_info) +{ + struct otx2_cpt_uc_info_t *curr, *temp; + + if (!fw_info) + return; + + list_for_each_entry_safe(curr, temp, &fw_info->ucodes, list) { + list_del(&curr->list); + release_firmware(curr->fw); + kfree(curr); + } +} + +static struct otx2_cpt_uc_info_t *get_ucode(struct fw_info_t *fw_info, + int ucode_type) +{ + struct otx2_cpt_uc_info_t *curr; + + list_for_each_entry(curr, &fw_info->ucodes, list) { + if (!is_eng_type(curr->ucode.type, ucode_type)) + continue; + + return curr; + } + return NULL; +} + +static void print_uc_info(struct fw_info_t *fw_info) +{ + struct otx2_cpt_uc_info_t *curr; + + list_for_each_entry(curr, &fw_info->ucodes, list) { + pr_debug("Ucode filename %s\n", curr->ucode.filename); + pr_debug("Ucode version string %s\n", curr->ucode.ver_str); + pr_debug("Ucode version %d.%d.%d.%d\n", + curr->ucode.ver_num.nn, curr->ucode.ver_num.xx, + curr->ucode.ver_num.yy, curr->ucode.ver_num.zz); + pr_debug("Ucode type (%d) %s\n", curr->ucode.type, + get_ucode_type_str(curr->ucode.type)); + pr_debug("Ucode size %d\n", curr->ucode.size); + pr_debug("Ucode ptr %p\n", curr->fw->data); + } +} + +static int cpt_ucode_load_fw(struct pci_dev *pdev, struct fw_info_t *fw_info, + u16 rid) +{ + char filename[OTX2_CPT_NAME_LENGTH]; + char eng_type[8]; + int ret, e, i; + + INIT_LIST_HEAD(&fw_info->ucodes); + + for (e = 1; e < OTX2_CPT_MAX_ENG_TYPES; e++) { + strscpy(eng_type, get_eng_type_str(e)); + for (i = 0; i < strlen(eng_type); i++) + eng_type[i] = tolower(eng_type[i]); + + snprintf(filename, sizeof(filename), "mrvl/cpt%02d/%s.out", + rid, eng_type); + /* Request firmware for each engine type */ + ret = load_fw(&pdev->dev, fw_info, filename, rid); + if (ret) + goto release_fw; + } + print_uc_info(fw_info); + return 0; + +release_fw: + cpt_ucode_release_fw(fw_info); + return ret; +} + +struct otx2_cpt_engs_rsvd *find_engines_by_type( + struct otx2_cpt_eng_grp_info *eng_grp, + int eng_type) +{ + int i; + + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + if (!eng_grp->engs[i].type) + continue; + + if (eng_grp->engs[i].type == eng_type) + return &eng_grp->engs[i]; + } + return NULL; +} + +static int eng_grp_has_eng_type(struct otx2_cpt_eng_grp_info *eng_grp, + int eng_type) +{ + struct otx2_cpt_engs_rsvd *engs; + + engs = find_engines_by_type(eng_grp, eng_type); + + return (engs != NULL ? 1 : 0); +} + +static int update_engines_avail_count(struct device *dev, + struct otx2_cpt_engs_available *avail, + struct otx2_cpt_engs_rsvd *engs, int val) +{ + switch (engs->type) { + case OTX2_CPT_SE_TYPES: + avail->se_cnt += val; + break; + + case OTX2_CPT_IE_TYPES: + avail->ie_cnt += val; + break; + + case OTX2_CPT_AE_TYPES: + avail->ae_cnt += val; + break; + + default: + dev_err(dev, "Invalid engine type %d\n", engs->type); + return -EINVAL; + } + return 0; +} + +static int update_engines_offset(struct device *dev, + struct otx2_cpt_engs_available *avail, + struct otx2_cpt_engs_rsvd *engs) +{ + switch (engs->type) { + case OTX2_CPT_SE_TYPES: + engs->offset = 0; + break; + + case OTX2_CPT_IE_TYPES: + engs->offset = avail->max_se_cnt; + break; + + case OTX2_CPT_AE_TYPES: + engs->offset = avail->max_se_cnt + avail->max_ie_cnt; + break; + + default: + dev_err(dev, "Invalid engine type %d\n", engs->type); + return -EINVAL; + } + return 0; +} + +static int release_engines(struct device *dev, + struct otx2_cpt_eng_grp_info *grp) +{ + int i, ret = 0; + + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + if (!grp->engs[i].type) + continue; + + if (grp->engs[i].count > 0) { + ret = update_engines_avail_count(dev, &grp->g->avail, + &grp->engs[i], + grp->engs[i].count); + if (ret) + return ret; + } + + grp->engs[i].type = 0; + grp->engs[i].count = 0; + grp->engs[i].offset = 0; + grp->engs[i].ucode = NULL; + bitmap_zero(grp->engs[i].bmap, grp->g->engs_num); + } + return 0; +} + +static int do_reserve_engines(struct device *dev, + struct otx2_cpt_eng_grp_info *grp, + struct otx2_cpt_engines *req_engs) +{ + struct otx2_cpt_engs_rsvd *engs = NULL; + int i, ret; + + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + if (!grp->engs[i].type) { + engs = &grp->engs[i]; + break; + } + } + + if (!engs) + return -ENOMEM; + + engs->type = req_engs->type; + engs->count = req_engs->count; + + ret = update_engines_offset(dev, &grp->g->avail, engs); + if (ret) + return ret; + + if (engs->count > 0) { + ret = update_engines_avail_count(dev, &grp->g->avail, engs, + -engs->count); + if (ret) + return ret; + } + + return 0; +} + +static int check_engines_availability(struct device *dev, + struct otx2_cpt_eng_grp_info *grp, + struct otx2_cpt_engines *req_eng) +{ + int avail_cnt = 0; + + switch (req_eng->type) { + case OTX2_CPT_SE_TYPES: + avail_cnt = grp->g->avail.se_cnt; + break; + + case OTX2_CPT_IE_TYPES: + avail_cnt = grp->g->avail.ie_cnt; + break; + + case OTX2_CPT_AE_TYPES: + avail_cnt = grp->g->avail.ae_cnt; + break; + + default: + dev_err(dev, "Invalid engine type %d\n", req_eng->type); + return -EINVAL; + } + + if (avail_cnt < req_eng->count) { + dev_err(dev, + "Error available %s engines %d < than requested %d\n", + get_eng_type_str(req_eng->type), + avail_cnt, req_eng->count); + return -EBUSY; + } + return 0; +} + +static int reserve_engines(struct device *dev, + struct otx2_cpt_eng_grp_info *grp, + struct otx2_cpt_engines *req_engs, int ucodes_cnt) +{ + int i, ret = 0; + + /* Validate if a number of requested engines are available */ + for (i = 0; i < ucodes_cnt; i++) { + ret = check_engines_availability(dev, grp, &req_engs[i]); + if (ret) + return ret; + } + + /* Reserve requested engines for this engine group */ + for (i = 0; i < ucodes_cnt; i++) { + ret = do_reserve_engines(dev, grp, &req_engs[i]); + if (ret) + return ret; + } + return 0; +} + +static void ucode_unload(struct device *dev, struct otx2_cpt_ucode *ucode) +{ + if (ucode->va) { + dma_free_coherent(dev, OTX2_CPT_UCODE_SZ, ucode->va, + ucode->dma); + ucode->va = NULL; + ucode->dma = 0; + ucode->size = 0; + } + + memset(&ucode->ver_str, 0, OTX2_CPT_UCODE_VER_STR_SZ); + memset(&ucode->ver_num, 0, sizeof(struct otx2_cpt_ucode_ver_num)); + set_ucode_filename(ucode, ""); + ucode->type = 0; +} + +static int copy_ucode_to_dma_mem(struct device *dev, + struct otx2_cpt_ucode *ucode, + const u8 *ucode_data) +{ + u32 i; + + /* Allocate DMAable space */ + ucode->va = dma_alloc_coherent(dev, OTX2_CPT_UCODE_SZ, &ucode->dma, + GFP_KERNEL); + if (!ucode->va) + return -ENOMEM; + + memcpy(ucode->va, ucode_data + sizeof(struct otx2_cpt_ucode_hdr), + ucode->size); + + /* Byte swap 64-bit */ + for (i = 0; i < (ucode->size / 8); i++) + cpu_to_be64s(&((u64 *)ucode->va)[i]); + /* Ucode needs 16-bit swap */ + for (i = 0; i < (ucode->size / 2); i++) + cpu_to_be16s(&((u16 *)ucode->va)[i]); + return 0; +} + +static int enable_eng_grp(struct otx2_cpt_eng_grp_info *eng_grp, + void *obj) +{ + int ret; + + /* Point microcode to each core of the group */ + ret = cpt_set_ucode_base(eng_grp, obj); + if (ret) + return ret; + + /* Attach the cores to the group and enable them */ + ret = cpt_attach_and_enable_cores(eng_grp, obj); + + return ret; +} + +static int disable_eng_grp(struct device *dev, + struct otx2_cpt_eng_grp_info *eng_grp, + void *obj) +{ + int i, ret; + + /* Disable all engines used by this group */ + ret = cpt_detach_and_disable_cores(eng_grp, obj); + if (ret) + return ret; + + /* Unload ucode used by this engine group */ + ucode_unload(dev, &eng_grp->ucode[0]); + ucode_unload(dev, &eng_grp->ucode[1]); + + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + if (!eng_grp->engs[i].type) + continue; + + eng_grp->engs[i].ucode = &eng_grp->ucode[0]; + } + + /* Clear UCODE_BASE register for each engine used by this group */ + ret = cpt_set_ucode_base(eng_grp, obj); + + return ret; +} + +static void setup_eng_grp_mirroring(struct otx2_cpt_eng_grp_info *dst_grp, + struct otx2_cpt_eng_grp_info *src_grp) +{ + /* Setup fields for engine group which is mirrored */ + src_grp->mirror.is_ena = false; + src_grp->mirror.idx = 0; + src_grp->mirror.ref_count++; + + /* Setup fields for mirroring engine group */ + dst_grp->mirror.is_ena = true; + dst_grp->mirror.idx = src_grp->idx; + dst_grp->mirror.ref_count = 0; +} + +static void remove_eng_grp_mirroring(struct otx2_cpt_eng_grp_info *dst_grp) +{ + struct otx2_cpt_eng_grp_info *src_grp; + + if (!dst_grp->mirror.is_ena) + return; + + src_grp = &dst_grp->g->grp[dst_grp->mirror.idx]; + + src_grp->mirror.ref_count--; + dst_grp->mirror.is_ena = false; + dst_grp->mirror.idx = 0; + dst_grp->mirror.ref_count = 0; +} + +static void update_requested_engs(struct otx2_cpt_eng_grp_info *mirror_eng_grp, + struct otx2_cpt_engines *engs, int engs_cnt) +{ + struct otx2_cpt_engs_rsvd *mirrored_engs; + int i; + + for (i = 0; i < engs_cnt; i++) { + mirrored_engs = find_engines_by_type(mirror_eng_grp, + engs[i].type); + if (!mirrored_engs) + continue; + + /* + * If mirrored group has this type of engines attached then + * there are 3 scenarios possible: + * 1) mirrored_engs.count == engs[i].count then all engines + * from mirrored engine group will be shared with this engine + * group + * 2) mirrored_engs.count > engs[i].count then only a subset of + * engines from mirrored engine group will be shared with this + * engine group + * 3) mirrored_engs.count < engs[i].count then all engines + * from mirrored engine group will be shared with this group + * and additional engines will be reserved for exclusively use + * by this engine group + */ + engs[i].count -= mirrored_engs->count; + } +} + +static struct otx2_cpt_eng_grp_info *find_mirrored_eng_grp( + struct otx2_cpt_eng_grp_info *grp) +{ + struct otx2_cpt_eng_grps *eng_grps = grp->g; + int i; + + for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) { + if (!eng_grps->grp[i].is_enabled) + continue; + if (eng_grps->grp[i].ucode[0].type && + eng_grps->grp[i].ucode[1].type) + continue; + if (grp->idx == i) + continue; + if (!strncasecmp(eng_grps->grp[i].ucode[0].ver_str, + grp->ucode[0].ver_str, + OTX2_CPT_UCODE_VER_STR_SZ)) + return &eng_grps->grp[i]; + } + + return NULL; +} + +static struct otx2_cpt_eng_grp_info *find_unused_eng_grp( + struct otx2_cpt_eng_grps *eng_grps) +{ + int i; + + for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) { + if (!eng_grps->grp[i].is_enabled) + return &eng_grps->grp[i]; + } + return NULL; +} + +static int eng_grp_update_masks(struct device *dev, + struct otx2_cpt_eng_grp_info *eng_grp) +{ + struct otx2_cpt_engs_rsvd *engs, *mirrored_engs; + struct otx2_cpt_bitmap tmp_bmap = { {0} }; + int i, j, cnt, max_cnt; + int bit; + + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + engs = &eng_grp->engs[i]; + if (!engs->type) + continue; + if (engs->count <= 0) + continue; + + switch (engs->type) { + case OTX2_CPT_SE_TYPES: + max_cnt = eng_grp->g->avail.max_se_cnt; + break; + + case OTX2_CPT_IE_TYPES: + max_cnt = eng_grp->g->avail.max_ie_cnt; + break; + + case OTX2_CPT_AE_TYPES: + max_cnt = eng_grp->g->avail.max_ae_cnt; + break; + + default: + dev_err(dev, "Invalid engine type %d\n", engs->type); + return -EINVAL; + } + + cnt = engs->count; + WARN_ON(engs->offset + max_cnt > OTX2_CPT_MAX_ENGINES); + bitmap_zero(tmp_bmap.bits, eng_grp->g->engs_num); + for (j = engs->offset; j < engs->offset + max_cnt; j++) { + if (!eng_grp->g->eng_ref_cnt[j]) { + bitmap_set(tmp_bmap.bits, j, 1); + cnt--; + if (!cnt) + break; + } + } + + if (cnt) + return -ENOSPC; + + bitmap_copy(engs->bmap, tmp_bmap.bits, eng_grp->g->engs_num); + } + + if (!eng_grp->mirror.is_ena) + return 0; + + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + engs = &eng_grp->engs[i]; + if (!engs->type) + continue; + + mirrored_engs = find_engines_by_type( + &eng_grp->g->grp[eng_grp->mirror.idx], + engs->type); + WARN_ON(!mirrored_engs && engs->count <= 0); + if (!mirrored_engs) + continue; + + bitmap_copy(tmp_bmap.bits, mirrored_engs->bmap, + eng_grp->g->engs_num); + if (engs->count < 0) { + bit = find_first_bit(mirrored_engs->bmap, + eng_grp->g->engs_num); + bitmap_clear(tmp_bmap.bits, bit, -engs->count); + } + bitmap_or(engs->bmap, engs->bmap, tmp_bmap.bits, + eng_grp->g->engs_num); + } + return 0; +} + +static int delete_engine_group(struct device *dev, + struct otx2_cpt_eng_grp_info *eng_grp) +{ + int ret; + + if (!eng_grp->is_enabled) + return 0; + + if (eng_grp->mirror.ref_count) + return -EINVAL; + + /* Removing engine group mirroring if enabled */ + remove_eng_grp_mirroring(eng_grp); + + /* Disable engine group */ + ret = disable_eng_grp(dev, eng_grp, eng_grp->g->obj); + if (ret) + return ret; + + /* Release all engines held by this engine group */ + ret = release_engines(dev, eng_grp); + if (ret) + return ret; + + eng_grp->is_enabled = false; + + return 0; +} + +static void update_ucode_ptrs(struct otx2_cpt_eng_grp_info *eng_grp) +{ + struct otx2_cpt_ucode *ucode; + + if (eng_grp->mirror.is_ena) + ucode = &eng_grp->g->grp[eng_grp->mirror.idx].ucode[0]; + else + ucode = &eng_grp->ucode[0]; + WARN_ON(!eng_grp->engs[0].type); + eng_grp->engs[0].ucode = ucode; + + if (eng_grp->engs[1].type) { + if (is_2nd_ucode_used(eng_grp)) + eng_grp->engs[1].ucode = &eng_grp->ucode[1]; + else + eng_grp->engs[1].ucode = ucode; + } +} + +static int create_engine_group(struct device *dev, + struct otx2_cpt_eng_grps *eng_grps, + struct otx2_cpt_engines *engs, int ucodes_cnt, + void *ucode_data[], int is_print) +{ + struct otx2_cpt_eng_grp_info *mirrored_eng_grp; + struct otx2_cpt_eng_grp_info *eng_grp; + struct otx2_cpt_uc_info_t *uc_info; + int i, ret = 0; + + /* Find engine group which is not used */ + eng_grp = find_unused_eng_grp(eng_grps); + if (!eng_grp) { + dev_err(dev, "Error all engine groups are being used\n"); + return -ENOSPC; + } + /* Load ucode */ + for (i = 0; i < ucodes_cnt; i++) { + uc_info = (struct otx2_cpt_uc_info_t *) ucode_data[i]; + eng_grp->ucode[i] = uc_info->ucode; + ret = copy_ucode_to_dma_mem(dev, &eng_grp->ucode[i], + uc_info->fw->data); + if (ret) + goto unload_ucode; + } + + /* Check if this group mirrors another existing engine group */ + mirrored_eng_grp = find_mirrored_eng_grp(eng_grp); + if (mirrored_eng_grp) { + /* Setup mirroring */ + setup_eng_grp_mirroring(eng_grp, mirrored_eng_grp); + + /* + * Update count of requested engines because some + * of them might be shared with mirrored group + */ + update_requested_engs(mirrored_eng_grp, engs, ucodes_cnt); + } + ret = reserve_engines(dev, eng_grp, engs, ucodes_cnt); + if (ret) + goto unload_ucode; + + /* Update ucode pointers used by engines */ + update_ucode_ptrs(eng_grp); + + /* Update engine masks used by this group */ + ret = eng_grp_update_masks(dev, eng_grp); + if (ret) + goto release_engs; + + /* Enable engine group */ + ret = enable_eng_grp(eng_grp, eng_grps->obj); + if (ret) + goto release_engs; + + /* + * If this engine group mirrors another engine group + * then we need to unload ucode as we will use ucode + * from mirrored engine group + */ + if (eng_grp->mirror.is_ena) + ucode_unload(dev, &eng_grp->ucode[0]); + + eng_grp->is_enabled = true; + + if (!is_print) + return 0; + + if (mirrored_eng_grp) + dev_info(dev, + "Engine_group%d: reuse microcode %s from group %d\n", + eng_grp->idx, mirrored_eng_grp->ucode[0].ver_str, + mirrored_eng_grp->idx); + else + dev_info(dev, "Engine_group%d: microcode loaded %s\n", + eng_grp->idx, eng_grp->ucode[0].ver_str); + if (is_2nd_ucode_used(eng_grp)) + dev_info(dev, "Engine_group%d: microcode loaded %s\n", + eng_grp->idx, eng_grp->ucode[1].ver_str); + + return 0; + +release_engs: + release_engines(dev, eng_grp); +unload_ucode: + ucode_unload(dev, &eng_grp->ucode[0]); + ucode_unload(dev, &eng_grp->ucode[1]); + return ret; +} + +static void delete_engine_grps(struct pci_dev *pdev, + struct otx2_cpt_eng_grps *eng_grps) +{ + int i; + + /* First delete all mirroring engine groups */ + for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) + if (eng_grps->grp[i].mirror.is_ena) + delete_engine_group(&pdev->dev, &eng_grps->grp[i]); + + /* Delete remaining engine groups */ + for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) + delete_engine_group(&pdev->dev, &eng_grps->grp[i]); +} + +#define PCI_DEVID_CN10K_RNM 0xA098 +#define RNM_ENTROPY_STATUS 0x8 + +static void rnm_to_cpt_errata_fixup(struct device *dev) +{ + struct pci_dev *pdev; + void __iomem *base; + int timeout = 5000; + + pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN10K_RNM, NULL); + if (!pdev) + return; + + base = pci_ioremap_bar(pdev, 0); + if (!base) + goto put_pdev; + + while ((readq(base + RNM_ENTROPY_STATUS) & 0x7F) != 0x40) { + cpu_relax(); + udelay(1); + timeout--; + if (!timeout) { + dev_warn(dev, "RNM is not producing entropy\n"); + break; + } + } + + iounmap(base); + +put_pdev: + pci_dev_put(pdev); +} + +int otx2_cpt_get_eng_grp(struct otx2_cpt_eng_grps *eng_grps, int eng_type) +{ + + int eng_grp_num = OTX2_CPT_INVALID_CRYPTO_ENG_GRP; + struct otx2_cpt_eng_grp_info *grp; + int i; + + for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) { + grp = &eng_grps->grp[i]; + if (!grp->is_enabled) + continue; + + if (eng_type == OTX2_CPT_SE_TYPES) { + if (eng_grp_has_eng_type(grp, eng_type) && + !eng_grp_has_eng_type(grp, OTX2_CPT_IE_TYPES)) { + eng_grp_num = i; + break; + } + } else { + if (eng_grp_has_eng_type(grp, eng_type)) { + eng_grp_num = i; + break; + } + } + } + return eng_grp_num; +} + +int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf, + struct otx2_cpt_eng_grps *eng_grps) +{ + struct otx2_cpt_uc_info_t *uc_info[OTX2_CPT_MAX_ETYPES_PER_GRP] = { }; + struct otx2_cpt_engines engs[OTX2_CPT_MAX_ETYPES_PER_GRP] = { {0} }; + struct pci_dev *pdev = cptpf->pdev; + struct fw_info_t fw_info; + u64 reg_val; + int ret = 0; + + mutex_lock(&eng_grps->lock); + /* + * We don't create engine groups if it was already + * made (when user enabled VFs for the first time) + */ + if (eng_grps->is_grps_created) + goto unlock; + + ret = cpt_ucode_load_fw(pdev, &fw_info, eng_grps->rid); + if (ret) + goto unlock; + + /* + * Create engine group with SE engines for kernel + * crypto functionality (symmetric crypto) + */ + uc_info[0] = get_ucode(&fw_info, OTX2_CPT_SE_TYPES); + if (uc_info[0] == NULL) { + dev_err(&pdev->dev, "Unable to find firmware for SE\n"); + ret = -EINVAL; + goto release_fw; + } + engs[0].type = OTX2_CPT_SE_TYPES; + engs[0].count = eng_grps->avail.max_se_cnt; + + ret = create_engine_group(&pdev->dev, eng_grps, engs, 1, + (void **) uc_info, 1); + if (ret) + goto release_fw; + + /* + * Create engine group with SE+IE engines for IPSec. + * All SE engines will be shared with engine group 0. + */ + uc_info[0] = get_ucode(&fw_info, OTX2_CPT_SE_TYPES); + uc_info[1] = get_ucode(&fw_info, OTX2_CPT_IE_TYPES); + + if (uc_info[1] == NULL) { + dev_err(&pdev->dev, "Unable to find firmware for IE"); + ret = -EINVAL; + goto delete_eng_grp; + } + engs[0].type = OTX2_CPT_SE_TYPES; + engs[0].count = eng_grps->avail.max_se_cnt; + engs[1].type = OTX2_CPT_IE_TYPES; + engs[1].count = eng_grps->avail.max_ie_cnt; + + ret = create_engine_group(&pdev->dev, eng_grps, engs, 2, + (void **) uc_info, 1); + if (ret) + goto delete_eng_grp; + + /* + * Create engine group with AE engines for asymmetric + * crypto functionality. + */ + uc_info[0] = get_ucode(&fw_info, OTX2_CPT_AE_TYPES); + if (uc_info[0] == NULL) { + dev_err(&pdev->dev, "Unable to find firmware for AE"); + ret = -EINVAL; + goto delete_eng_grp; + } + engs[0].type = OTX2_CPT_AE_TYPES; + engs[0].count = eng_grps->avail.max_ae_cnt; + + ret = create_engine_group(&pdev->dev, eng_grps, engs, 1, + (void **) uc_info, 1); + if (ret) + goto delete_eng_grp; + + eng_grps->is_grps_created = true; + + cpt_ucode_release_fw(&fw_info); + + if (is_dev_otx2(pdev)) + goto unlock; + + /* + * Ensure RNM_ENTROPY_STATUS[NORMAL_CNT] = 0x40 before writing + * CPT_AF_CTL[RNM_REQ_EN] = 1 as a workaround for HW errata. + */ + rnm_to_cpt_errata_fixup(&pdev->dev); + + otx2_cpt_read_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_CTL, ®_val, + BLKADDR_CPT0); + /* + * Configure engine group mask to allow context prefetching + * for the groups and enable random number request, to enable + * CPT to request random numbers from RNM. + */ + reg_val |= OTX2_CPT_ALL_ENG_GRPS_MASK << 3 | BIT_ULL(16); + otx2_cpt_write_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_CTL, + reg_val, BLKADDR_CPT0); + /* + * Set interval to periodically flush dirty data for the next + * CTX cache entry. Set the interval count to maximum supported + * value. + */ + otx2_cpt_write_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_CTX_FLUSH_TIMER, + CTX_FLUSH_TIMER_CNT, BLKADDR_CPT0); + + /* + * Set CPT_AF_DIAG[FLT_DIS], as a workaround for HW errata, when + * CPT_AF_DIAG[FLT_DIS] = 0 and a CPT engine access to LLC/DRAM + * encounters a fault/poison, a rare case may result in + * unpredictable data being delivered to a CPT engine. + */ + if (cpt_is_errata_38550_exists(pdev)) { + otx2_cpt_read_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_DIAG, + ®_val, BLKADDR_CPT0); + otx2_cpt_write_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_DIAG, + reg_val | BIT_ULL(24), BLKADDR_CPT0); + } + + mutex_unlock(&eng_grps->lock); + return 0; + +delete_eng_grp: + delete_engine_grps(pdev, eng_grps); +release_fw: + cpt_ucode_release_fw(&fw_info); +unlock: + mutex_unlock(&eng_grps->lock); + return ret; +} + +static int cptx_disable_all_cores(struct otx2_cptpf_dev *cptpf, int total_cores, + int blkaddr) +{ + int timeout = 10, ret; + int i, busy; + u64 reg; + + /* Disengage the cores from groups */ + for (i = 0; i < total_cores; i++) { + ret = otx2_cpt_add_write_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_EXEX_CTL2(i), 0x0, + blkaddr); + if (ret) + return ret; + + cptpf->eng_grps.eng_ref_cnt[i] = 0; + } + ret = otx2_cpt_send_af_reg_requests(&cptpf->afpf_mbox, cptpf->pdev); + if (ret) + return ret; + + /* Wait for cores to become idle */ + do { + busy = 0; + usleep_range(10000, 20000); + if (timeout-- < 0) + return -EBUSY; + + for (i = 0; i < total_cores; i++) { + ret = otx2_cpt_read_af_reg(&cptpf->afpf_mbox, + cptpf->pdev, + CPT_AF_EXEX_STS(i), ®, + blkaddr); + if (ret) + return ret; + + if (reg & 0x1) { + busy = 1; + break; + } + } + } while (busy); + + /* Disable the cores */ + for (i = 0; i < total_cores; i++) { + ret = otx2_cpt_add_write_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_EXEX_CTL(i), 0x0, + blkaddr); + if (ret) + return ret; + } + return otx2_cpt_send_af_reg_requests(&cptpf->afpf_mbox, cptpf->pdev); +} + +int otx2_cpt_disable_all_cores(struct otx2_cptpf_dev *cptpf) +{ + int total_cores, ret; + + total_cores = cptpf->eng_grps.avail.max_se_cnt + + cptpf->eng_grps.avail.max_ie_cnt + + cptpf->eng_grps.avail.max_ae_cnt; + + if (cptpf->has_cpt1) { + ret = cptx_disable_all_cores(cptpf, total_cores, BLKADDR_CPT1); + if (ret) + return ret; + } + return cptx_disable_all_cores(cptpf, total_cores, BLKADDR_CPT0); +} + +void otx2_cpt_cleanup_eng_grps(struct pci_dev *pdev, + struct otx2_cpt_eng_grps *eng_grps) +{ + struct otx2_cpt_eng_grp_info *grp; + int i, j; + + mutex_lock(&eng_grps->lock); + delete_engine_grps(pdev, eng_grps); + /* Release memory */ + for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) { + grp = &eng_grps->grp[i]; + for (j = 0; j < OTX2_CPT_MAX_ETYPES_PER_GRP; j++) { + kfree(grp->engs[j].bmap); + grp->engs[j].bmap = NULL; + } + } + mutex_unlock(&eng_grps->lock); +} + +int otx2_cpt_init_eng_grps(struct pci_dev *pdev, + struct otx2_cpt_eng_grps *eng_grps) +{ + struct otx2_cpt_eng_grp_info *grp; + int i, j, ret; + + mutex_init(&eng_grps->lock); + eng_grps->obj = pci_get_drvdata(pdev); + eng_grps->avail.se_cnt = eng_grps->avail.max_se_cnt; + eng_grps->avail.ie_cnt = eng_grps->avail.max_ie_cnt; + eng_grps->avail.ae_cnt = eng_grps->avail.max_ae_cnt; + + eng_grps->engs_num = eng_grps->avail.max_se_cnt + + eng_grps->avail.max_ie_cnt + + eng_grps->avail.max_ae_cnt; + if (eng_grps->engs_num > OTX2_CPT_MAX_ENGINES) { + dev_err(&pdev->dev, + "Number of engines %d > than max supported %d\n", + eng_grps->engs_num, OTX2_CPT_MAX_ENGINES); + ret = -EINVAL; + goto cleanup_eng_grps; + } + + for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) { + grp = &eng_grps->grp[i]; + grp->g = eng_grps; + grp->idx = i; + + for (j = 0; j < OTX2_CPT_MAX_ETYPES_PER_GRP; j++) { + grp->engs[j].bmap = + kcalloc(BITS_TO_LONGS(eng_grps->engs_num), + sizeof(long), GFP_KERNEL); + if (!grp->engs[j].bmap) { + ret = -ENOMEM; + goto cleanup_eng_grps; + } + } + } + return 0; + +cleanup_eng_grps: + otx2_cpt_cleanup_eng_grps(pdev, eng_grps); + return ret; +} + +static int create_eng_caps_discovery_grps(struct pci_dev *pdev, + struct otx2_cpt_eng_grps *eng_grps) +{ + struct otx2_cpt_uc_info_t *uc_info[OTX2_CPT_MAX_ETYPES_PER_GRP] = { }; + struct otx2_cpt_engines engs[OTX2_CPT_MAX_ETYPES_PER_GRP] = { {0} }; + struct fw_info_t fw_info; + int ret; + + mutex_lock(&eng_grps->lock); + ret = cpt_ucode_load_fw(pdev, &fw_info, eng_grps->rid); + if (ret) { + mutex_unlock(&eng_grps->lock); + return ret; + } + + uc_info[0] = get_ucode(&fw_info, OTX2_CPT_AE_TYPES); + if (uc_info[0] == NULL) { + dev_err(&pdev->dev, "Unable to find firmware for AE\n"); + ret = -EINVAL; + goto release_fw; + } + engs[0].type = OTX2_CPT_AE_TYPES; + engs[0].count = 2; + + ret = create_engine_group(&pdev->dev, eng_grps, engs, 1, + (void **) uc_info, 0); + if (ret) + goto release_fw; + + uc_info[0] = get_ucode(&fw_info, OTX2_CPT_SE_TYPES); + if (uc_info[0] == NULL) { + dev_err(&pdev->dev, "Unable to find firmware for SE\n"); + ret = -EINVAL; + goto delete_eng_grp; + } + engs[0].type = OTX2_CPT_SE_TYPES; + engs[0].count = 2; + + ret = create_engine_group(&pdev->dev, eng_grps, engs, 1, + (void **) uc_info, 0); + if (ret) + goto delete_eng_grp; + + uc_info[0] = get_ucode(&fw_info, OTX2_CPT_IE_TYPES); + if (uc_info[0] == NULL) { + dev_err(&pdev->dev, "Unable to find firmware for IE\n"); + ret = -EINVAL; + goto delete_eng_grp; + } + engs[0].type = OTX2_CPT_IE_TYPES; + engs[0].count = 2; + + ret = create_engine_group(&pdev->dev, eng_grps, engs, 1, + (void **) uc_info, 0); + if (ret) + goto delete_eng_grp; + + cpt_ucode_release_fw(&fw_info); + mutex_unlock(&eng_grps->lock); + return 0; + +delete_eng_grp: + delete_engine_grps(pdev, eng_grps); +release_fw: + cpt_ucode_release_fw(&fw_info); + mutex_unlock(&eng_grps->lock); + return ret; +} + +/* + * Get CPT HW capabilities using LOAD_FVC operation. + */ +int otx2_cpt_discover_eng_capabilities(struct otx2_cptpf_dev *cptpf) +{ + struct otx2_cptlfs_info *lfs = &cptpf->lfs; + struct otx2_cpt_iq_command iq_cmd; + union otx2_cpt_opcode opcode; + union otx2_cpt_res_s *result; + union otx2_cpt_inst_s inst; + dma_addr_t result_baddr; + dma_addr_t rptr_baddr; + struct pci_dev *pdev; + int timeout = 10000; + void *base, *rptr; + int ret, etype; + u32 len; + + /* + * We don't get capabilities if it was already done + * (when user enabled VFs for the first time) + */ + if (cptpf->is_eng_caps_discovered) + return 0; + + pdev = cptpf->pdev; + /* + * Create engine groups for each type to submit LOAD_FVC op and + * get engine's capabilities. + */ + ret = create_eng_caps_discovery_grps(pdev, &cptpf->eng_grps); + if (ret) + goto delete_grps; + + ret = otx2_cptlf_init(lfs, OTX2_CPT_ALL_ENG_GRPS_MASK, + OTX2_CPT_QUEUE_HI_PRIO, 1); + if (ret) + goto delete_grps; + + /* Allocate extra memory for "rptr" and "result" pointer alignment */ + len = LOADFVC_RLEN + ARCH_DMA_MINALIGN + + sizeof(union otx2_cpt_res_s) + OTX2_CPT_RES_ADDR_ALIGN; + + base = kzalloc(len, GFP_KERNEL); + if (!base) { + ret = -ENOMEM; + goto lf_cleanup; + } + + rptr = PTR_ALIGN(base, ARCH_DMA_MINALIGN); + rptr_baddr = dma_map_single(&pdev->dev, rptr, len, DMA_BIDIRECTIONAL); + if (dma_mapping_error(&pdev->dev, rptr_baddr)) { + dev_err(&pdev->dev, "DMA mapping failed\n"); + ret = -EFAULT; + goto free_rptr; + } + + result = (union otx2_cpt_res_s *)PTR_ALIGN(rptr + LOADFVC_RLEN, + OTX2_CPT_RES_ADDR_ALIGN); + result_baddr = ALIGN(rptr_baddr + LOADFVC_RLEN, + OTX2_CPT_RES_ADDR_ALIGN); + + /* Fill in the command */ + opcode.s.major = LOADFVC_MAJOR_OP; + opcode.s.minor = LOADFVC_MINOR_OP; + + iq_cmd.cmd.u = 0; + iq_cmd.cmd.s.opcode = cpu_to_be16(opcode.flags); + + /* 64-bit swap for microcode data reads, not needed for addresses */ + cpu_to_be64s(&iq_cmd.cmd.u); + iq_cmd.dptr = 0; + iq_cmd.rptr = rptr_baddr; + iq_cmd.cptr.u = 0; + + for (etype = 1; etype < OTX2_CPT_MAX_ENG_TYPES; etype++) { + result->s.compcode = OTX2_CPT_COMPLETION_CODE_INIT; + iq_cmd.cptr.s.grp = otx2_cpt_get_eng_grp(&cptpf->eng_grps, + etype); + otx2_cpt_fill_inst(&inst, &iq_cmd, result_baddr); + lfs->ops->send_cmd(&inst, 1, &cptpf->lfs.lf[0]); + timeout = 10000; + + while (lfs->ops->cpt_get_compcode(result) == + OTX2_CPT_COMPLETION_CODE_INIT) { + cpu_relax(); + udelay(1); + timeout--; + if (!timeout) { + ret = -ENODEV; + cptpf->is_eng_caps_discovered = false; + dev_warn(&pdev->dev, "Timeout on CPT load_fvc completion poll\n"); + goto error_no_response; + } + } + + cptpf->eng_caps[etype].u = be64_to_cpup(rptr); + } + cptpf->is_eng_caps_discovered = true; + +error_no_response: + dma_unmap_single(&pdev->dev, rptr_baddr, len, DMA_BIDIRECTIONAL); +free_rptr: + kfree(base); +lf_cleanup: + otx2_cptlf_shutdown(lfs); +delete_grps: + delete_engine_grps(pdev, &cptpf->eng_grps); + + return ret; +} + +int otx2_cpt_dl_custom_egrp_create(struct otx2_cptpf_dev *cptpf, + struct devlink_param_gset_ctx *ctx) +{ + struct otx2_cpt_engines engs[OTX2_CPT_MAX_ETYPES_PER_GRP] = { { 0 } }; + struct otx2_cpt_uc_info_t *uc_info[OTX2_CPT_MAX_ETYPES_PER_GRP] = {}; + struct otx2_cpt_eng_grps *eng_grps = &cptpf->eng_grps; + char *ucode_filename[OTX2_CPT_MAX_ETYPES_PER_GRP]; + char tmp_buf[OTX2_CPT_NAME_LENGTH] = { 0 }; + struct device *dev = &cptpf->pdev->dev; + char *start, *val, *err_msg, *tmp; + int grp_idx = 0, ret = -EINVAL; + bool has_se, has_ie, has_ae; + struct fw_info_t fw_info; + int ucode_idx = 0; + + if (!eng_grps->is_grps_created) { + dev_err(dev, "Not allowed before creating the default groups\n"); + return -EINVAL; + } + err_msg = "Invalid engine group format"; + strscpy(tmp_buf, ctx->val.vstr); + start = tmp_buf; + + has_se = has_ie = has_ae = false; + + for (;;) { + val = strsep(&start, ";"); + if (!val) + break; + val = strim(val); + if (!*val) + continue; + + if (!strncasecmp(val, "se", 2) && strchr(val, ':')) { + if (has_se || ucode_idx) + goto err_print; + tmp = strsep(&val, ":"); + if (!tmp) + goto err_print; + tmp = strim(tmp); + if (!val) + goto err_print; + if (strlen(tmp) != 2) + goto err_print; + if (kstrtoint(strim(val), 10, &engs[grp_idx].count)) + goto err_print; + engs[grp_idx++].type = OTX2_CPT_SE_TYPES; + has_se = true; + } else if (!strncasecmp(val, "ae", 2) && strchr(val, ':')) { + if (has_ae || ucode_idx) + goto err_print; + tmp = strsep(&val, ":"); + if (!tmp) + goto err_print; + tmp = strim(tmp); + if (!val) + goto err_print; + if (strlen(tmp) != 2) + goto err_print; + if (kstrtoint(strim(val), 10, &engs[grp_idx].count)) + goto err_print; + engs[grp_idx++].type = OTX2_CPT_AE_TYPES; + has_ae = true; + } else if (!strncasecmp(val, "ie", 2) && strchr(val, ':')) { + if (has_ie || ucode_idx) + goto err_print; + tmp = strsep(&val, ":"); + if (!tmp) + goto err_print; + tmp = strim(tmp); + if (!val) + goto err_print; + if (strlen(tmp) != 2) + goto err_print; + if (kstrtoint(strim(val), 10, &engs[grp_idx].count)) + goto err_print; + engs[grp_idx++].type = OTX2_CPT_IE_TYPES; + has_ie = true; + } else { + if (ucode_idx > 1) + goto err_print; + if (!strlen(val)) + goto err_print; + if (strnstr(val, " ", strlen(val))) + goto err_print; + ucode_filename[ucode_idx++] = val; + } + } + + /* Validate input parameters */ + if (!(grp_idx && ucode_idx)) + goto err_print; + + if (ucode_idx > 1 && grp_idx < 2) + goto err_print; + + if (grp_idx > OTX2_CPT_MAX_ETYPES_PER_GRP) { + err_msg = "Error max 2 engine types can be attached"; + goto err_print; + } + + if (grp_idx > 1) { + if ((engs[0].type + engs[1].type) != + (OTX2_CPT_SE_TYPES + OTX2_CPT_IE_TYPES)) { + err_msg = "Only combination of SE+IE engines is allowed"; + goto err_print; + } + /* Keep SE engines at zero index */ + if (engs[1].type == OTX2_CPT_SE_TYPES) + swap(engs[0], engs[1]); + } + mutex_lock(&eng_grps->lock); + + if (cptpf->enabled_vfs) { + dev_err(dev, "Disable VFs before modifying engine groups\n"); + ret = -EACCES; + goto err_unlock; + } + INIT_LIST_HEAD(&fw_info.ucodes); + + ret = load_fw(dev, &fw_info, ucode_filename[0], eng_grps->rid); + if (ret) { + dev_err(dev, "Unable to load firmware %s\n", ucode_filename[0]); + goto err_unlock; + } + if (ucode_idx > 1) { + ret = load_fw(dev, &fw_info, ucode_filename[1], eng_grps->rid); + if (ret) { + dev_err(dev, "Unable to load firmware %s\n", + ucode_filename[1]); + goto release_fw; + } + } + uc_info[0] = get_ucode(&fw_info, engs[0].type); + if (uc_info[0] == NULL) { + dev_err(dev, "Unable to find firmware for %s\n", + get_eng_type_str(engs[0].type)); + ret = -EINVAL; + goto release_fw; + } + if (ucode_idx > 1) { + uc_info[1] = get_ucode(&fw_info, engs[1].type); + if (uc_info[1] == NULL) { + dev_err(dev, "Unable to find firmware for %s\n", + get_eng_type_str(engs[1].type)); + ret = -EINVAL; + goto release_fw; + } + } + ret = create_engine_group(dev, eng_grps, engs, grp_idx, + (void **)uc_info, 1); + +release_fw: + cpt_ucode_release_fw(&fw_info); +err_unlock: + mutex_unlock(&eng_grps->lock); + return ret; +err_print: + dev_err(dev, "%s\n", err_msg); + return ret; +} + +int otx2_cpt_dl_custom_egrp_delete(struct otx2_cptpf_dev *cptpf, + struct devlink_param_gset_ctx *ctx) +{ + struct otx2_cpt_eng_grps *eng_grps = &cptpf->eng_grps; + struct device *dev = &cptpf->pdev->dev; + char *tmp, *err_msg; + int egrp; + int ret; + + err_msg = "Invalid input string format(ex: egrp:0)"; + if (strncasecmp(ctx->val.vstr, "egrp", 4)) + goto err_print; + tmp = ctx->val.vstr; + strsep(&tmp, ":"); + if (!tmp) + goto err_print; + if (kstrtoint(tmp, 10, &egrp)) + goto err_print; + + if (egrp < 0 || egrp >= OTX2_CPT_MAX_ENGINE_GROUPS) { + dev_err(dev, "Invalid engine group %d", egrp); + return -EINVAL; + } + if (!eng_grps->grp[egrp].is_enabled) { + dev_err(dev, "Error engine_group%d is not configured", egrp); + return -EINVAL; + } + mutex_lock(&eng_grps->lock); + ret = delete_engine_group(dev, &eng_grps->grp[egrp]); + mutex_unlock(&eng_grps->lock); + + return ret; + +err_print: + dev_err(dev, "%s\n", err_msg); + return -EINVAL; +} diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h new file mode 100644 index 000000000..7e6a6a4ec --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h @@ -0,0 +1,172 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2020 Marvell. + */ + +#ifndef __OTX2_CPTPF_UCODE_H +#define __OTX2_CPTPF_UCODE_H + +#include +#include +#include +#include "otx2_cpt_hw_types.h" +#include "otx2_cpt_common.h" + +/* + * On OcteonTX2 platform IPSec ucode can use both IE and SE engines therefore + * IE and SE engines can be attached to the same engine group. + */ +#define OTX2_CPT_MAX_ETYPES_PER_GRP 2 + +/* CPT ucode signature size */ +#define OTX2_CPT_UCODE_SIGN_LEN 256 + +/* Microcode version string length */ +#define OTX2_CPT_UCODE_VER_STR_SZ 44 + +/* Maximum number of supported engines/cores on OcteonTX2/CN10K platform */ +#define OTX2_CPT_MAX_ENGINES 144 + +#define OTX2_CPT_ENGS_BITMASK_LEN BITS_TO_LONGS(OTX2_CPT_MAX_ENGINES) + +#define OTX2_CPT_UCODE_SZ (64 * 1024) + +/* Microcode types */ +enum otx2_cpt_ucode_type { + OTX2_CPT_AE_UC_TYPE = 1, /* AE-MAIN */ + OTX2_CPT_SE_UC_TYPE1 = 20,/* SE-MAIN - combination of 21 and 22 */ + OTX2_CPT_SE_UC_TYPE2 = 21,/* Fast Path IPSec + AirCrypto */ + OTX2_CPT_SE_UC_TYPE3 = 22,/* + * Hash + HMAC + FlexiCrypto + RNG + + * Full Feature IPSec + AirCrypto + Kasumi + */ + OTX2_CPT_IE_UC_TYPE1 = 30, /* IE-MAIN - combination of 31 and 32 */ + OTX2_CPT_IE_UC_TYPE2 = 31, /* Fast Path IPSec */ + OTX2_CPT_IE_UC_TYPE3 = 32, /* + * Hash + HMAC + FlexiCrypto + RNG + + * Full Future IPSec + */ +}; + +struct otx2_cpt_bitmap { + unsigned long bits[OTX2_CPT_ENGS_BITMASK_LEN]; + int size; +}; + +struct otx2_cpt_engines { + int type; + int count; +}; + +/* Microcode version number */ +struct otx2_cpt_ucode_ver_num { + u8 nn; + u8 xx; + u8 yy; + u8 zz; +}; + +struct otx2_cpt_ucode_hdr { + struct otx2_cpt_ucode_ver_num ver_num; + u8 ver_str[OTX2_CPT_UCODE_VER_STR_SZ]; + __be32 code_length; + u32 padding[3]; +}; + +struct otx2_cpt_ucode { + u8 ver_str[OTX2_CPT_UCODE_VER_STR_SZ + 1];/* + * ucode version in readable + * format + */ + struct otx2_cpt_ucode_ver_num ver_num;/* ucode version number */ + char filename[OTX2_CPT_NAME_LENGTH];/* ucode filename */ + dma_addr_t dma; /* phys address of ucode image */ + void *va; /* virt address of ucode image */ + u32 size; /* ucode image size */ + int type; /* ucode image type SE, IE, AE or SE+IE */ +}; + +struct otx2_cpt_uc_info_t { + struct list_head list; + struct otx2_cpt_ucode ucode;/* microcode information */ + const struct firmware *fw; +}; + +/* Maximum and current number of engines available for all engine groups */ +struct otx2_cpt_engs_available { + int max_se_cnt; + int max_ie_cnt; + int max_ae_cnt; + int se_cnt; + int ie_cnt; + int ae_cnt; +}; + +/* Engines reserved to an engine group */ +struct otx2_cpt_engs_rsvd { + int type; /* engine type */ + int count; /* number of engines attached */ + int offset; /* constant offset of engine type in the bitmap */ + unsigned long *bmap; /* attached engines bitmap */ + struct otx2_cpt_ucode *ucode; /* ucode used by these engines */ +}; + +struct otx2_cpt_mirror_info { + int is_ena; /* + * is mirroring enabled, it is set only for engine + * group which mirrors another engine group + */ + int idx; /* + * index of engine group which is mirrored by this + * group, set only for engine group which mirrors + * another group + */ + int ref_count; /* + * number of times this engine group is mirrored by + * other groups, this is set only for engine group + * which is mirrored by other group(s) + */ +}; + +struct otx2_cpt_eng_grp_info { + struct otx2_cpt_eng_grps *g; /* pointer to engine_groups structure */ + /* engines attached */ + struct otx2_cpt_engs_rsvd engs[OTX2_CPT_MAX_ETYPES_PER_GRP]; + /* ucodes information */ + struct otx2_cpt_ucode ucode[OTX2_CPT_MAX_ETYPES_PER_GRP]; + /* engine group mirroring information */ + struct otx2_cpt_mirror_info mirror; + int idx; /* engine group index */ + bool is_enabled; /* + * is engine group enabled, engine group is enabled + * when it has engines attached and ucode loaded + */ +}; + +struct otx2_cpt_eng_grps { + struct mutex lock; + struct otx2_cpt_eng_grp_info grp[OTX2_CPT_MAX_ENGINE_GROUPS]; + struct otx2_cpt_engs_available avail; + void *obj; /* device specific data */ + int engs_num; /* total number of engines supported */ + u8 eng_ref_cnt[OTX2_CPT_MAX_ENGINES];/* engines reference count */ + bool is_grps_created; /* Is the engine groups are already created */ + u16 rid; +}; +struct otx2_cptpf_dev; +int otx2_cpt_init_eng_grps(struct pci_dev *pdev, + struct otx2_cpt_eng_grps *eng_grps); +void otx2_cpt_cleanup_eng_grps(struct pci_dev *pdev, + struct otx2_cpt_eng_grps *eng_grps); +int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf, + struct otx2_cpt_eng_grps *eng_grps); +int otx2_cpt_disable_all_cores(struct otx2_cptpf_dev *cptpf); +int otx2_cpt_get_eng_grp(struct otx2_cpt_eng_grps *eng_grps, int eng_type); +int otx2_cpt_discover_eng_capabilities(struct otx2_cptpf_dev *cptpf); +int otx2_cpt_dl_custom_egrp_create(struct otx2_cptpf_dev *cptpf, + struct devlink_param_gset_ctx *ctx); +int otx2_cpt_dl_custom_egrp_delete(struct otx2_cptpf_dev *cptpf, + struct devlink_param_gset_ctx *ctx); +struct otx2_cpt_engs_rsvd *find_engines_by_type( + struct otx2_cpt_eng_grp_info *eng_grp, + int eng_type); +#endif /* __OTX2_CPTPF_UCODE_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf.h b/drivers/crypto/marvell/octeontx2/otx2_cptvf.h new file mode 100644 index 000000000..11ab9af1d --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2020 Marvell. + */ + +#ifndef __OTX2_CPTVF_H +#define __OTX2_CPTVF_H + +#include "mbox.h" +#include "otx2_cptlf.h" + +struct otx2_cptvf_dev { + void __iomem *reg_base; /* Register start address */ + void __iomem *pfvf_mbox_base; /* PF-VF mbox start address */ + struct pci_dev *pdev; /* PCI device handle */ + struct otx2_cptlfs_info lfs; /* CPT LFs attached to this VF */ + u8 vf_id; /* Virtual function index */ + + /* PF <=> VF mbox */ + struct otx2_mbox pfvf_mbox; + struct work_struct pfvf_mbox_work; + struct workqueue_struct *pfvf_mbox_wq; + int blkaddr; + void *bbuf_base; + unsigned long cap_flag; + u64 eng_caps[OTX2_CPT_MAX_ENG_TYPES]; +}; + +irqreturn_t otx2_cptvf_pfvf_mbox_intr(int irq, void *arg); +void otx2_cptvf_pfvf_mbox_handler(struct work_struct *work); +int otx2_cptvf_send_eng_grp_num_msg(struct otx2_cptvf_dev *cptvf, int eng_type); +int otx2_cptvf_send_kvf_limits_msg(struct otx2_cptvf_dev *cptvf); +int otx2_cpt_mbox_bbuf_init(struct otx2_cptvf_dev *cptvf, struct pci_dev *pdev); +int otx2_cptvf_send_caps_msg(struct otx2_cptvf_dev *cptvf); + +#endif /* __OTX2_CPTVF_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c new file mode 100644 index 000000000..8d9f394d6 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c @@ -0,0 +1,1700 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "otx2_cptvf.h" +#include "otx2_cptvf_algs.h" +#include "otx2_cpt_reqmgr.h" +#include "cn10k_cpt.h" + +/* Size of salt in AES GCM mode */ +#define AES_GCM_SALT_SIZE 4 +/* Size of IV in AES GCM mode */ +#define AES_GCM_IV_SIZE 8 +/* Size of ICV (Integrity Check Value) in AES GCM mode */ +#define AES_GCM_ICV_SIZE 16 +/* Offset of IV in AES GCM mode */ +#define AES_GCM_IV_OFFSET 8 +#define CONTROL_WORD_LEN 8 +#define KEY2_OFFSET 48 +#define DMA_MODE_FLAG(dma_mode) \ + (((dma_mode) == OTX2_CPT_DMA_MODE_SG) ? (1 << 7) : 0) + +/* Truncated SHA digest size */ +#define SHA1_TRUNC_DIGEST_SIZE 12 +#define SHA256_TRUNC_DIGEST_SIZE 16 +#define SHA384_TRUNC_DIGEST_SIZE 24 +#define SHA512_TRUNC_DIGEST_SIZE 32 + +static DEFINE_MUTEX(mutex); +static int is_crypto_registered; + +struct cpt_device_desc { + struct pci_dev *dev; + int num_queues; +}; + +struct cpt_device_table { + atomic_t count; + struct cpt_device_desc desc[OTX2_CPT_MAX_LFS_NUM]; +}; + +static struct cpt_device_table se_devices = { + .count = ATOMIC_INIT(0) +}; + +static struct otx2_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg); + +static inline int get_se_device(struct pci_dev **pdev, int *cpu_num) +{ + int count; + + count = atomic_read(&se_devices.count); + if (count < 1) + return -ENODEV; + + *cpu_num = get_cpu(); + /* + * On OcteonTX2 platform CPT instruction queue is bound to each + * local function LF, in turn LFs can be attached to PF + * or VF therefore we always use first device. We get maximum + * performance if one CPT queue is available for each cpu + * otherwise CPT queues need to be shared between cpus. + */ + if (*cpu_num >= se_devices.desc[0].num_queues) + *cpu_num %= se_devices.desc[0].num_queues; + *pdev = se_devices.desc[0].dev; + + put_cpu(); + + return 0; +} + +static inline int validate_hmac_cipher_null(struct otx2_cpt_req_info *cpt_req) +{ + struct otx2_cpt_req_ctx *rctx; + struct aead_request *req; + struct crypto_aead *tfm; + + req = container_of(cpt_req->areq, struct aead_request, base); + tfm = crypto_aead_reqtfm(req); + rctx = aead_request_ctx_dma(req); + if (memcmp(rctx->fctx.hmac.s.hmac_calc, + rctx->fctx.hmac.s.hmac_recv, + crypto_aead_authsize(tfm)) != 0) + return -EBADMSG; + + return 0; +} + +static void otx2_cpt_aead_callback(int status, void *arg1, void *arg2) +{ + struct otx2_cpt_inst_info *inst_info = arg2; + struct crypto_async_request *areq = arg1; + struct otx2_cpt_req_info *cpt_req; + struct pci_dev *pdev; + + if (inst_info) { + cpt_req = inst_info->req; + if (!status) { + /* + * When selected cipher is NULL we need to manually + * verify whether calculated hmac value matches + * received hmac value + */ + if (cpt_req->req_type == + OTX2_CPT_AEAD_ENC_DEC_NULL_REQ && + !cpt_req->is_enc) + status = validate_hmac_cipher_null(cpt_req); + } + pdev = inst_info->pdev; + otx2_cpt_info_destroy(pdev, inst_info); + } + if (areq) + crypto_request_complete(areq, status); +} + +static void output_iv_copyback(struct crypto_async_request *areq) +{ + struct otx2_cpt_req_info *req_info; + struct otx2_cpt_req_ctx *rctx; + struct skcipher_request *sreq; + struct crypto_skcipher *stfm; + struct otx2_cpt_enc_ctx *ctx; + u32 start, ivsize; + + sreq = container_of(areq, struct skcipher_request, base); + stfm = crypto_skcipher_reqtfm(sreq); + ctx = crypto_skcipher_ctx(stfm); + if (ctx->cipher_type == OTX2_CPT_AES_CBC || + ctx->cipher_type == OTX2_CPT_DES3_CBC) { + rctx = skcipher_request_ctx_dma(sreq); + req_info = &rctx->cpt_req; + ivsize = crypto_skcipher_ivsize(stfm); + start = sreq->cryptlen - ivsize; + + if (req_info->is_enc) { + scatterwalk_map_and_copy(sreq->iv, sreq->dst, start, + ivsize, 0); + } else { + if (sreq->src != sreq->dst) { + scatterwalk_map_and_copy(sreq->iv, sreq->src, + start, ivsize, 0); + } else { + memcpy(sreq->iv, req_info->iv_out, ivsize); + kfree(req_info->iv_out); + } + } + } +} + +static void otx2_cpt_skcipher_callback(int status, void *arg1, void *arg2) +{ + struct otx2_cpt_inst_info *inst_info = arg2; + struct crypto_async_request *areq = arg1; + struct pci_dev *pdev; + + if (areq) { + if (!status) + output_iv_copyback(areq); + if (inst_info) { + pdev = inst_info->pdev; + otx2_cpt_info_destroy(pdev, inst_info); + } + crypto_request_complete(areq, status); + } +} + +static inline void update_input_data(struct otx2_cpt_req_info *req_info, + struct scatterlist *inp_sg, + u32 nbytes, u32 *argcnt) +{ + req_info->req.dlen += nbytes; + + while (nbytes) { + u32 len = (nbytes < inp_sg->length) ? nbytes : inp_sg->length; + u8 *ptr = sg_virt(inp_sg); + + req_info->in[*argcnt].vptr = (void *)ptr; + req_info->in[*argcnt].size = len; + nbytes -= len; + ++(*argcnt); + inp_sg = sg_next(inp_sg); + } +} + +static inline void update_output_data(struct otx2_cpt_req_info *req_info, + struct scatterlist *outp_sg, + u32 offset, u32 nbytes, u32 *argcnt) +{ + u32 len, sg_len; + u8 *ptr; + + req_info->rlen += nbytes; + + while (nbytes) { + sg_len = outp_sg->length - offset; + len = (nbytes < sg_len) ? nbytes : sg_len; + ptr = sg_virt(outp_sg); + + req_info->out[*argcnt].vptr = (void *) (ptr + offset); + req_info->out[*argcnt].size = len; + nbytes -= len; + ++(*argcnt); + offset = 0; + outp_sg = sg_next(outp_sg); + } +} + +static inline int create_ctx_hdr(struct skcipher_request *req, u32 enc, + u32 *argcnt) +{ + struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); + struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req); + struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + struct otx2_cpt_fc_ctx *fctx = &rctx->fctx; + int ivsize = crypto_skcipher_ivsize(stfm); + u32 start = req->cryptlen - ivsize; + gfp_t flags; + + flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG; + req_info->ctrl.s.se_req = 1; + + req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC | + DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG); + if (enc) { + req_info->req.opcode.s.minor = 2; + } else { + req_info->req.opcode.s.minor = 3; + if ((ctx->cipher_type == OTX2_CPT_AES_CBC || + ctx->cipher_type == OTX2_CPT_DES3_CBC) && + req->src == req->dst) { + req_info->iv_out = kmalloc(ivsize, flags); + if (!req_info->iv_out) + return -ENOMEM; + + scatterwalk_map_and_copy(req_info->iv_out, req->src, + start, ivsize, 0); + } + } + /* Encryption data length */ + req_info->req.param1 = req->cryptlen; + /* Authentication data length */ + req_info->req.param2 = 0; + + fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type; + fctx->enc.enc_ctrl.e.aes_key = ctx->key_type; + fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR; + + if (ctx->cipher_type == OTX2_CPT_AES_XTS) + memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2); + else + memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len); + + memcpy(fctx->enc.encr_iv, req->iv, crypto_skcipher_ivsize(stfm)); + + cpu_to_be64s(&fctx->enc.enc_ctrl.u); + + /* + * Storing Packet Data Information in offset + * Control Word First 8 bytes + */ + req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word; + req_info->in[*argcnt].size = CONTROL_WORD_LEN; + req_info->req.dlen += CONTROL_WORD_LEN; + ++(*argcnt); + + req_info->in[*argcnt].vptr = (u8 *)fctx; + req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx); + req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx); + + ++(*argcnt); + + return 0; +} + +static inline int create_input_list(struct skcipher_request *req, u32 enc, + u32 enc_iv_len) +{ + struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + u32 argcnt = 0; + int ret; + + ret = create_ctx_hdr(req, enc, &argcnt); + if (ret) + return ret; + + update_input_data(req_info, req->src, req->cryptlen, &argcnt); + req_info->in_cnt = argcnt; + + return 0; +} + +static inline void create_output_list(struct skcipher_request *req, + u32 enc_iv_len) +{ + struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + u32 argcnt = 0; + + /* + * OUTPUT Buffer Processing + * AES encryption/decryption output would be + * received in the following format + * + * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----| + * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ] + */ + update_output_data(req_info, req->dst, 0, req->cryptlen, &argcnt); + req_info->out_cnt = argcnt; +} + +static int skcipher_do_fallback(struct skcipher_request *req, bool is_enc) +{ + struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); + struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req); + struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); + int ret; + + if (ctx->fbk_cipher) { + skcipher_request_set_tfm(&rctx->sk_fbk_req, ctx->fbk_cipher); + skcipher_request_set_callback(&rctx->sk_fbk_req, + req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(&rctx->sk_fbk_req, req->src, + req->dst, req->cryptlen, req->iv); + ret = is_enc ? crypto_skcipher_encrypt(&rctx->sk_fbk_req) : + crypto_skcipher_decrypt(&rctx->sk_fbk_req); + } else { + ret = -EINVAL; + } + return ret; +} + +static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc) +{ + struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); + struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req); + struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + u32 enc_iv_len = crypto_skcipher_ivsize(stfm); + struct pci_dev *pdev; + int status, cpu_num; + + if (req->cryptlen == 0) + return 0; + + if (!IS_ALIGNED(req->cryptlen, ctx->enc_align_len)) + return -EINVAL; + + if (req->cryptlen > OTX2_CPT_MAX_REQ_SIZE) + return skcipher_do_fallback(req, enc); + + /* Clear control words */ + rctx->ctrl_word.flags = 0; + rctx->fctx.enc.enc_ctrl.u = 0; + + status = create_input_list(req, enc, enc_iv_len); + if (status) + return status; + create_output_list(req, enc_iv_len); + + status = get_se_device(&pdev, &cpu_num); + if (status) + return status; + + req_info->callback = otx2_cpt_skcipher_callback; + req_info->areq = &req->base; + req_info->req_type = OTX2_CPT_ENC_DEC_REQ; + req_info->is_enc = enc; + req_info->is_trunc_hmac = false; + req_info->ctrl.s.grp = otx2_cpt_get_eng_grp_num(pdev, + OTX2_CPT_SE_TYPES); + + req_info->req.cptr = ctx->er_ctx.hw_ctx; + req_info->req.cptr_dma = ctx->er_ctx.cptr_dma; + + /* + * We perform an asynchronous send and once + * the request is completed the driver would + * intimate through registered call back functions + */ + status = otx2_cpt_do_request(pdev, req_info, cpu_num); + + return status; +} + +static int otx2_cpt_skcipher_encrypt(struct skcipher_request *req) +{ + return cpt_enc_dec(req, true); +} + +static int otx2_cpt_skcipher_decrypt(struct skcipher_request *req) +{ + return cpt_enc_dec(req, false); +} + +static int otx2_cpt_skcipher_xts_setkey(struct crypto_skcipher *tfm, + const u8 *key, u32 keylen) +{ + struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); + const u8 *key2 = key + (keylen / 2); + const u8 *key1 = key; + int ret; + + ret = xts_verify_key(tfm, key, keylen); + if (ret) + return ret; + ctx->key_len = keylen; + ctx->enc_align_len = 1; + memcpy(ctx->enc_key, key1, keylen / 2); + memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2); + ctx->cipher_type = OTX2_CPT_AES_XTS; + switch (ctx->key_len) { + case 2 * AES_KEYSIZE_128: + ctx->key_type = OTX2_CPT_AES_128_BIT; + break; + case 2 * AES_KEYSIZE_192: + ctx->key_type = OTX2_CPT_AES_192_BIT; + break; + case 2 * AES_KEYSIZE_256: + ctx->key_type = OTX2_CPT_AES_256_BIT; + break; + default: + return -EINVAL; + } + return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen); +} + +static int cpt_des_setkey(struct crypto_skcipher *tfm, const u8 *key, + u32 keylen, u8 cipher_type) +{ + struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (keylen != DES3_EDE_KEY_SIZE) + return -EINVAL; + + ctx->key_len = keylen; + ctx->cipher_type = cipher_type; + ctx->enc_align_len = 8; + + memcpy(ctx->enc_key, key, keylen); + + return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen); +} + +static int cpt_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + u32 keylen, u8 cipher_type) +{ + struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); + + switch (keylen) { + case AES_KEYSIZE_128: + ctx->key_type = OTX2_CPT_AES_128_BIT; + break; + case AES_KEYSIZE_192: + ctx->key_type = OTX2_CPT_AES_192_BIT; + break; + case AES_KEYSIZE_256: + ctx->key_type = OTX2_CPT_AES_256_BIT; + break; + default: + return -EINVAL; + } + if (cipher_type == OTX2_CPT_AES_CBC || cipher_type == OTX2_CPT_AES_ECB) + ctx->enc_align_len = 16; + else + ctx->enc_align_len = 1; + + ctx->key_len = keylen; + ctx->cipher_type = cipher_type; + + memcpy(ctx->enc_key, key, keylen); + + return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen); +} + +static int otx2_cpt_skcipher_cbc_aes_setkey(struct crypto_skcipher *tfm, + const u8 *key, u32 keylen) +{ + return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_CBC); +} + +static int otx2_cpt_skcipher_ecb_aes_setkey(struct crypto_skcipher *tfm, + const u8 *key, u32 keylen) +{ + return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_ECB); +} + +static int otx2_cpt_skcipher_cbc_des3_setkey(struct crypto_skcipher *tfm, + const u8 *key, u32 keylen) +{ + return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_CBC); +} + +static int otx2_cpt_skcipher_ecb_des3_setkey(struct crypto_skcipher *tfm, + const u8 *key, u32 keylen) +{ + return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_ECB); +} + +static int cpt_skcipher_fallback_init(struct otx2_cpt_enc_ctx *ctx, + struct crypto_alg *alg) +{ + if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) { + ctx->fbk_cipher = + crypto_alloc_skcipher(alg->cra_name, 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fbk_cipher)) { + pr_err("%s() failed to allocate fallback for %s\n", + __func__, alg->cra_name); + return PTR_ERR(ctx->fbk_cipher); + } + } + return 0; +} + +static int otx2_cpt_enc_dec_init(struct crypto_skcipher *stfm) +{ + struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); + struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm); + struct crypto_alg *alg = tfm->__crt_alg; + struct pci_dev *pdev; + int ret, cpu_num; + + memset(ctx, 0, sizeof(*ctx)); + /* + * Additional memory for skcipher_request is + * allocated since the cryptd daemon uses + * this memory for request_ctx information + */ + crypto_skcipher_set_reqsize_dma( + stfm, sizeof(struct otx2_cpt_req_ctx) + + sizeof(struct skcipher_request)); + + ret = get_se_device(&pdev, &cpu_num); + if (ret) + return ret; + + ctx->pdev = pdev; + ret = cn10k_cpt_hw_ctx_init(pdev, &ctx->er_ctx); + if (ret) + return ret; + + return cpt_skcipher_fallback_init(ctx, alg); +} + +static void otx2_cpt_skcipher_exit(struct crypto_skcipher *tfm) +{ + struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (ctx->fbk_cipher) { + crypto_free_skcipher(ctx->fbk_cipher); + ctx->fbk_cipher = NULL; + } + cn10k_cpt_hw_ctx_clear(ctx->pdev, &ctx->er_ctx); +} + +static int cpt_aead_fallback_init(struct otx2_cpt_aead_ctx *ctx, + struct crypto_alg *alg) +{ + if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) { + ctx->fbk_cipher = + crypto_alloc_aead(alg->cra_name, 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fbk_cipher)) { + pr_err("%s() failed to allocate fallback for %s\n", + __func__, alg->cra_name); + return PTR_ERR(ctx->fbk_cipher); + } + } + return 0; +} + +static int cpt_aead_init(struct crypto_aead *atfm, u8 cipher_type, u8 mac_type) +{ + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(atfm); + struct crypto_tfm *tfm = crypto_aead_tfm(atfm); + struct crypto_alg *alg = tfm->__crt_alg; + struct pci_dev *pdev; + int ret, cpu_num; + + ctx->cipher_type = cipher_type; + ctx->mac_type = mac_type; + + switch (ctx->mac_type) { + case OTX2_CPT_SHA1: + ctx->hashalg = crypto_alloc_shash("sha1", 0, 0); + break; + + case OTX2_CPT_SHA256: + ctx->hashalg = crypto_alloc_shash("sha256", 0, 0); + break; + + case OTX2_CPT_SHA384: + ctx->hashalg = crypto_alloc_shash("sha384", 0, 0); + break; + + case OTX2_CPT_SHA512: + ctx->hashalg = crypto_alloc_shash("sha512", 0, 0); + break; + } + + if (IS_ERR(ctx->hashalg)) + return PTR_ERR(ctx->hashalg); + + if (ctx->hashalg) { + ctx->sdesc = alloc_sdesc(ctx->hashalg); + if (!ctx->sdesc) { + crypto_free_shash(ctx->hashalg); + return -ENOMEM; + } + } + + /* + * When selected cipher is NULL we use HMAC opcode instead of + * FLEXICRYPTO opcode therefore we don't need to use HASH algorithms + * for calculating ipad and opad + */ + if (ctx->cipher_type != OTX2_CPT_CIPHER_NULL && ctx->hashalg) { + int ss = crypto_shash_statesize(ctx->hashalg); + + ctx->ipad = kzalloc(ss, GFP_KERNEL); + if (!ctx->ipad) { + kfree(ctx->sdesc); + crypto_free_shash(ctx->hashalg); + return -ENOMEM; + } + + ctx->opad = kzalloc(ss, GFP_KERNEL); + if (!ctx->opad) { + kfree(ctx->ipad); + kfree(ctx->sdesc); + crypto_free_shash(ctx->hashalg); + return -ENOMEM; + } + } + switch (ctx->cipher_type) { + case OTX2_CPT_AES_CBC: + case OTX2_CPT_AES_ECB: + ctx->enc_align_len = 16; + break; + case OTX2_CPT_DES3_CBC: + case OTX2_CPT_DES3_ECB: + ctx->enc_align_len = 8; + break; + case OTX2_CPT_AES_GCM: + case OTX2_CPT_CIPHER_NULL: + ctx->enc_align_len = 1; + break; + } + crypto_aead_set_reqsize_dma(atfm, sizeof(struct otx2_cpt_req_ctx)); + + ret = get_se_device(&pdev, &cpu_num); + if (ret) + return ret; + + ctx->pdev = pdev; + ret = cn10k_cpt_hw_ctx_init(pdev, &ctx->er_ctx); + if (ret) + return ret; + + return cpt_aead_fallback_init(ctx, alg); +} + +static int otx2_cpt_aead_cbc_aes_sha1_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA1); +} + +static int otx2_cpt_aead_cbc_aes_sha256_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA256); +} + +static int otx2_cpt_aead_cbc_aes_sha384_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA384); +} + +static int otx2_cpt_aead_cbc_aes_sha512_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA512); +} + +static int otx2_cpt_aead_ecb_null_sha1_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA1); +} + +static int otx2_cpt_aead_ecb_null_sha256_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA256); +} + +static int otx2_cpt_aead_ecb_null_sha384_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA384); +} + +static int otx2_cpt_aead_ecb_null_sha512_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA512); +} + +static int otx2_cpt_aead_gcm_aes_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_AES_GCM, OTX2_CPT_MAC_NULL); +} + +static void otx2_cpt_aead_exit(struct crypto_aead *tfm) +{ + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); + + kfree(ctx->ipad); + kfree(ctx->opad); + crypto_free_shash(ctx->hashalg); + kfree(ctx->sdesc); + + if (ctx->fbk_cipher) { + crypto_free_aead(ctx->fbk_cipher); + ctx->fbk_cipher = NULL; + } + cn10k_cpt_hw_ctx_clear(ctx->pdev, &ctx->er_ctx); +} + +static int otx2_cpt_aead_gcm_set_authsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); + + if (crypto_rfc4106_check_authsize(authsize)) + return -EINVAL; + + tfm->authsize = authsize; + /* Set authsize for fallback case */ + if (ctx->fbk_cipher) + ctx->fbk_cipher->authsize = authsize; + + return 0; +} + +static int otx2_cpt_aead_set_authsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + tfm->authsize = authsize; + + return 0; +} + +static int otx2_cpt_aead_null_set_authsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); + + ctx->is_trunc_hmac = true; + tfm->authsize = authsize; + + return 0; +} + +static struct otx2_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg) +{ + struct otx2_cpt_sdesc *sdesc; + int size; + + size = sizeof(struct shash_desc) + crypto_shash_descsize(alg); + sdesc = kmalloc(size, GFP_KERNEL); + if (!sdesc) + return NULL; + + sdesc->shash.tfm = alg; + + return sdesc; +} + +static inline void swap_data32(void *buf, u32 len) +{ + cpu_to_be32_array(buf, buf, len / 4); +} + +static inline void swap_data64(void *buf, u32 len) +{ + u64 *src = buf; + int i = 0; + + for (i = 0 ; i < len / 8; i++, src++) + cpu_to_be64s(src); +} + +static int swap_pad(u8 mac_type, u8 *pad) +{ + struct sha512_state *sha512; + struct sha256_state *sha256; + struct sha1_state *sha1; + + switch (mac_type) { + case OTX2_CPT_SHA1: + sha1 = (struct sha1_state *)pad; + swap_data32(sha1->state, SHA1_DIGEST_SIZE); + break; + + case OTX2_CPT_SHA256: + sha256 = (struct sha256_state *)pad; + swap_data32(sha256->state, SHA256_DIGEST_SIZE); + break; + + case OTX2_CPT_SHA384: + case OTX2_CPT_SHA512: + sha512 = (struct sha512_state *)pad; + swap_data64(sha512->state, SHA512_DIGEST_SIZE); + break; + + default: + return -EINVAL; + } + + return 0; +} + +static int aead_hmac_init(struct crypto_aead *cipher, + struct crypto_authenc_keys *keys) +{ + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher); + int ds = crypto_shash_digestsize(ctx->hashalg); + int bs = crypto_shash_blocksize(ctx->hashalg); + int authkeylen = keys->authkeylen; + u8 *ipad = NULL, *opad = NULL; + int icount = 0; + int ret; + + if (authkeylen > bs) { + ret = crypto_shash_digest(&ctx->sdesc->shash, keys->authkey, + authkeylen, ctx->key); + if (ret) + goto calc_fail; + + authkeylen = ds; + } else + memcpy(ctx->key, keys->authkey, authkeylen); + + ctx->enc_key_len = keys->enckeylen; + ctx->auth_key_len = authkeylen; + + if (ctx->cipher_type == OTX2_CPT_CIPHER_NULL) + return keys->enckeylen ? -EINVAL : 0; + + switch (keys->enckeylen) { + case AES_KEYSIZE_128: + ctx->key_type = OTX2_CPT_AES_128_BIT; + break; + case AES_KEYSIZE_192: + ctx->key_type = OTX2_CPT_AES_192_BIT; + break; + case AES_KEYSIZE_256: + ctx->key_type = OTX2_CPT_AES_256_BIT; + break; + default: + /* Invalid key length */ + return -EINVAL; + } + + memcpy(ctx->key + authkeylen, keys->enckey, keys->enckeylen); + + ipad = ctx->ipad; + opad = ctx->opad; + + memcpy(ipad, ctx->key, authkeylen); + memset(ipad + authkeylen, 0, bs - authkeylen); + memcpy(opad, ipad, bs); + + for (icount = 0; icount < bs; icount++) { + ipad[icount] ^= 0x36; + opad[icount] ^= 0x5c; + } + + /* + * Partial Hash calculated from the software + * algorithm is retrieved for IPAD & OPAD + */ + + /* IPAD Calculation */ + crypto_shash_init(&ctx->sdesc->shash); + crypto_shash_update(&ctx->sdesc->shash, ipad, bs); + crypto_shash_export(&ctx->sdesc->shash, ipad); + ret = swap_pad(ctx->mac_type, ipad); + if (ret) + goto calc_fail; + + /* OPAD Calculation */ + crypto_shash_init(&ctx->sdesc->shash); + crypto_shash_update(&ctx->sdesc->shash, opad, bs); + crypto_shash_export(&ctx->sdesc->shash, opad); + ret = swap_pad(ctx->mac_type, opad); + +calc_fail: + return ret; +} + +static int otx2_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead *cipher, + const unsigned char *key, + unsigned int keylen) +{ + struct crypto_authenc_keys authenc_keys; + + return crypto_authenc_extractkeys(&authenc_keys, key, keylen) ?: + aead_hmac_init(cipher, &authenc_keys); +} + +static int otx2_cpt_aead_ecb_null_sha_setkey(struct crypto_aead *cipher, + const unsigned char *key, + unsigned int keylen) +{ + return otx2_cpt_aead_cbc_aes_sha_setkey(cipher, key, keylen); +} + +static int otx2_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher, + const unsigned char *key, + unsigned int keylen) +{ + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher); + + /* + * For aes gcm we expect to get encryption key (16, 24, 32 bytes) + * and salt (4 bytes) + */ + switch (keylen) { + case AES_KEYSIZE_128 + AES_GCM_SALT_SIZE: + ctx->key_type = OTX2_CPT_AES_128_BIT; + ctx->enc_key_len = AES_KEYSIZE_128; + break; + case AES_KEYSIZE_192 + AES_GCM_SALT_SIZE: + ctx->key_type = OTX2_CPT_AES_192_BIT; + ctx->enc_key_len = AES_KEYSIZE_192; + break; + case AES_KEYSIZE_256 + AES_GCM_SALT_SIZE: + ctx->key_type = OTX2_CPT_AES_256_BIT; + ctx->enc_key_len = AES_KEYSIZE_256; + break; + default: + /* Invalid key and salt length */ + return -EINVAL; + } + + /* Store encryption key and salt */ + memcpy(ctx->key, key, keylen); + + return crypto_aead_setkey(ctx->fbk_cipher, key, keylen); +} + +static inline int create_aead_ctx_hdr(struct aead_request *req, u32 enc, + u32 *argcnt) +{ + struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + struct otx2_cpt_fc_ctx *fctx = &rctx->fctx; + int mac_len = crypto_aead_authsize(tfm); + int ds; + + rctx->ctrl_word.e.enc_data_offset = req->assoclen; + + switch (ctx->cipher_type) { + case OTX2_CPT_AES_CBC: + if (req->assoclen > 248 || !IS_ALIGNED(req->assoclen, 8)) + return -EINVAL; + + fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR; + /* Copy encryption key to context */ + memcpy(fctx->enc.encr_key, ctx->key + ctx->auth_key_len, + ctx->enc_key_len); + /* Copy IV to context */ + memcpy(fctx->enc.encr_iv, req->iv, crypto_aead_ivsize(tfm)); + + ds = crypto_shash_digestsize(ctx->hashalg); + if (ctx->mac_type == OTX2_CPT_SHA384) + ds = SHA512_DIGEST_SIZE; + if (ctx->ipad) + memcpy(fctx->hmac.e.ipad, ctx->ipad, ds); + if (ctx->opad) + memcpy(fctx->hmac.e.opad, ctx->opad, ds); + break; + + case OTX2_CPT_AES_GCM: + if (crypto_ipsec_check_assoclen(req->assoclen)) + return -EINVAL; + + fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_DPTR; + /* Copy encryption key to context */ + memcpy(fctx->enc.encr_key, ctx->key, ctx->enc_key_len); + /* Copy salt to context */ + memcpy(fctx->enc.encr_iv, ctx->key + ctx->enc_key_len, + AES_GCM_SALT_SIZE); + + rctx->ctrl_word.e.iv_offset = req->assoclen - AES_GCM_IV_OFFSET; + break; + + default: + /* Unknown cipher type */ + return -EINVAL; + } + cpu_to_be64s(&rctx->ctrl_word.flags); + + req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG; + req_info->ctrl.s.se_req = 1; + req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC | + DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG); + if (enc) { + req_info->req.opcode.s.minor = 2; + req_info->req.param1 = req->cryptlen; + req_info->req.param2 = req->cryptlen + req->assoclen; + } else { + req_info->req.opcode.s.minor = 3; + req_info->req.param1 = req->cryptlen - mac_len; + req_info->req.param2 = req->cryptlen + req->assoclen - mac_len; + } + + fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type; + fctx->enc.enc_ctrl.e.aes_key = ctx->key_type; + fctx->enc.enc_ctrl.e.mac_type = ctx->mac_type; + fctx->enc.enc_ctrl.e.mac_len = mac_len; + cpu_to_be64s(&fctx->enc.enc_ctrl.u); + + /* + * Storing Packet Data Information in offset + * Control Word First 8 bytes + */ + req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word; + req_info->in[*argcnt].size = CONTROL_WORD_LEN; + req_info->req.dlen += CONTROL_WORD_LEN; + ++(*argcnt); + + req_info->in[*argcnt].vptr = (u8 *)fctx; + req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx); + req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx); + ++(*argcnt); + + return 0; +} + +static inline void create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt, + u32 enc) +{ + struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + + req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG; + req_info->ctrl.s.se_req = 1; + req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_HMAC | + DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG); + req_info->is_trunc_hmac = ctx->is_trunc_hmac; + + req_info->req.opcode.s.minor = 0; + req_info->req.param1 = ctx->auth_key_len; + req_info->req.param2 = ctx->mac_type << 8; + + /* Add authentication key */ + req_info->in[*argcnt].vptr = ctx->key; + req_info->in[*argcnt].size = round_up(ctx->auth_key_len, 8); + req_info->req.dlen += round_up(ctx->auth_key_len, 8); + ++(*argcnt); +} + +static inline int create_aead_input_list(struct aead_request *req, u32 enc) +{ + struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + u32 inputlen = req->cryptlen + req->assoclen; + u32 status, argcnt = 0; + + status = create_aead_ctx_hdr(req, enc, &argcnt); + if (status) + return status; + update_input_data(req_info, req->src, inputlen, &argcnt); + req_info->in_cnt = argcnt; + + return 0; +} + +static inline void create_aead_output_list(struct aead_request *req, u32 enc, + u32 mac_len) +{ + struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + u32 argcnt = 0, outputlen = 0; + + if (enc) + outputlen = req->cryptlen + req->assoclen + mac_len; + else + outputlen = req->cryptlen + req->assoclen - mac_len; + + update_output_data(req_info, req->dst, 0, outputlen, &argcnt); + req_info->out_cnt = argcnt; +} + +static inline void create_aead_null_input_list(struct aead_request *req, + u32 enc, u32 mac_len) +{ + struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + u32 inputlen, argcnt = 0; + + if (enc) + inputlen = req->cryptlen + req->assoclen; + else + inputlen = req->cryptlen + req->assoclen - mac_len; + + create_hmac_ctx_hdr(req, &argcnt, enc); + update_input_data(req_info, req->src, inputlen, &argcnt); + req_info->in_cnt = argcnt; +} + +static inline int create_aead_null_output_list(struct aead_request *req, + u32 enc, u32 mac_len) +{ + struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + struct scatterlist *dst; + u8 *ptr = NULL; + int argcnt = 0, status, offset; + u32 inputlen; + + if (enc) + inputlen = req->cryptlen + req->assoclen; + else + inputlen = req->cryptlen + req->assoclen - mac_len; + + /* + * If source and destination are different + * then copy payload to destination + */ + if (req->src != req->dst) { + + ptr = kmalloc(inputlen, (req_info->areq->flags & + CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC); + if (!ptr) + return -ENOMEM; + + status = sg_copy_to_buffer(req->src, sg_nents(req->src), ptr, + inputlen); + if (status != inputlen) { + status = -EINVAL; + goto error_free; + } + status = sg_copy_from_buffer(req->dst, sg_nents(req->dst), ptr, + inputlen); + if (status != inputlen) { + status = -EINVAL; + goto error_free; + } + kfree(ptr); + } + + if (enc) { + /* + * In an encryption scenario hmac needs + * to be appended after payload + */ + dst = req->dst; + offset = inputlen; + while (offset >= dst->length) { + offset -= dst->length; + dst = sg_next(dst); + if (!dst) + return -ENOENT; + } + + update_output_data(req_info, dst, offset, mac_len, &argcnt); + } else { + /* + * In a decryption scenario calculated hmac for received + * payload needs to be compare with hmac received + */ + status = sg_copy_buffer(req->src, sg_nents(req->src), + rctx->fctx.hmac.s.hmac_recv, mac_len, + inputlen, true); + if (status != mac_len) + return -EINVAL; + + req_info->out[argcnt].vptr = rctx->fctx.hmac.s.hmac_calc; + req_info->out[argcnt].size = mac_len; + argcnt++; + } + + req_info->out_cnt = argcnt; + return 0; + +error_free: + kfree(ptr); + return status; +} + +static int aead_do_fallback(struct aead_request *req, bool is_enc) +{ + struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(aead); + int ret; + + if (ctx->fbk_cipher) { + /* Store the cipher tfm and then use the fallback tfm */ + aead_request_set_tfm(&rctx->fbk_req, ctx->fbk_cipher); + aead_request_set_callback(&rctx->fbk_req, req->base.flags, + req->base.complete, req->base.data); + aead_request_set_crypt(&rctx->fbk_req, req->src, + req->dst, req->cryptlen, req->iv); + aead_request_set_ad(&rctx->fbk_req, req->assoclen); + ret = is_enc ? crypto_aead_encrypt(&rctx->fbk_req) : + crypto_aead_decrypt(&rctx->fbk_req); + } else { + ret = -EINVAL; + } + + return ret; +} + +static int cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc) +{ + struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); + struct pci_dev *pdev; + int status, cpu_num; + + /* Clear control words */ + rctx->ctrl_word.flags = 0; + rctx->fctx.enc.enc_ctrl.u = 0; + + req_info->callback = otx2_cpt_aead_callback; + req_info->areq = &req->base; + req_info->req_type = reg_type; + req_info->is_enc = enc; + req_info->is_trunc_hmac = false; + + req_info->req.cptr = ctx->er_ctx.hw_ctx; + req_info->req.cptr_dma = ctx->er_ctx.cptr_dma; + + switch (reg_type) { + case OTX2_CPT_AEAD_ENC_DEC_REQ: + status = create_aead_input_list(req, enc); + if (status) + return status; + create_aead_output_list(req, enc, crypto_aead_authsize(tfm)); + break; + + case OTX2_CPT_AEAD_ENC_DEC_NULL_REQ: + create_aead_null_input_list(req, enc, + crypto_aead_authsize(tfm)); + status = create_aead_null_output_list(req, enc, + crypto_aead_authsize(tfm)); + if (status) + return status; + break; + + default: + return -EINVAL; + } + if (!IS_ALIGNED(req_info->req.param1, ctx->enc_align_len)) + return -EINVAL; + + if (!req_info->req.param2 || + (req_info->req.param1 > OTX2_CPT_MAX_REQ_SIZE) || + (req_info->req.param2 > OTX2_CPT_MAX_REQ_SIZE)) + return aead_do_fallback(req, enc); + + status = get_se_device(&pdev, &cpu_num); + if (status) + return status; + + req_info->ctrl.s.grp = otx2_cpt_get_eng_grp_num(pdev, + OTX2_CPT_SE_TYPES); + + /* + * We perform an asynchronous send and once + * the request is completed the driver would + * intimate through registered call back functions + */ + return otx2_cpt_do_request(pdev, req_info, cpu_num); +} + +static int otx2_cpt_aead_encrypt(struct aead_request *req) +{ + return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, true); +} + +static int otx2_cpt_aead_decrypt(struct aead_request *req) +{ + return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, false); +} + +static int otx2_cpt_aead_null_encrypt(struct aead_request *req) +{ + return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, true); +} + +static int otx2_cpt_aead_null_decrypt(struct aead_request *req) +{ + return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, false); +} + +static struct skcipher_alg otx2_cpt_skciphers[] = { { + .base.cra_name = "xts(aes)", + .base.cra_driver_name = "cpt_xts_aes", + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_module = THIS_MODULE, + + .init = otx2_cpt_enc_dec_init, + .exit = otx2_cpt_skcipher_exit, + .ivsize = AES_BLOCK_SIZE, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .setkey = otx2_cpt_skcipher_xts_setkey, + .encrypt = otx2_cpt_skcipher_encrypt, + .decrypt = otx2_cpt_skcipher_decrypt, +}, { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cpt_cbc_aes", + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_module = THIS_MODULE, + + .init = otx2_cpt_enc_dec_init, + .exit = otx2_cpt_skcipher_exit, + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = otx2_cpt_skcipher_cbc_aes_setkey, + .encrypt = otx2_cpt_skcipher_encrypt, + .decrypt = otx2_cpt_skcipher_decrypt, +}, { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "cpt_ecb_aes", + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_module = THIS_MODULE, + + .init = otx2_cpt_enc_dec_init, + .exit = otx2_cpt_skcipher_exit, + .ivsize = 0, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = otx2_cpt_skcipher_ecb_aes_setkey, + .encrypt = otx2_cpt_skcipher_encrypt, + .decrypt = otx2_cpt_skcipher_decrypt, +}, { + .base.cra_name = "cbc(des3_ede)", + .base.cra_driver_name = "cpt_cbc_des3_ede", + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_module = THIS_MODULE, + + .init = otx2_cpt_enc_dec_init, + .exit = otx2_cpt_skcipher_exit, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = otx2_cpt_skcipher_cbc_des3_setkey, + .encrypt = otx2_cpt_skcipher_encrypt, + .decrypt = otx2_cpt_skcipher_decrypt, +}, { + .base.cra_name = "ecb(des3_ede)", + .base.cra_driver_name = "cpt_ecb_des3_ede", + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_module = THIS_MODULE, + + .init = otx2_cpt_enc_dec_init, + .exit = otx2_cpt_skcipher_exit, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = 0, + .setkey = otx2_cpt_skcipher_ecb_des3_setkey, + .encrypt = otx2_cpt_skcipher_encrypt, + .decrypt = otx2_cpt_skcipher_decrypt, +} }; + +static struct aead_alg otx2_cpt_aeads[] = { { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "cpt_hmac_sha1_cbc_aes", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_cbc_aes_sha1_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_cbc_aes_sha_setkey, + .setauthsize = otx2_cpt_aead_set_authsize, + .encrypt = otx2_cpt_aead_encrypt, + .decrypt = otx2_cpt_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "cpt_hmac_sha256_cbc_aes", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_cbc_aes_sha256_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_cbc_aes_sha_setkey, + .setauthsize = otx2_cpt_aead_set_authsize, + .encrypt = otx2_cpt_aead_encrypt, + .decrypt = otx2_cpt_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(aes))", + .cra_driver_name = "cpt_hmac_sha384_cbc_aes", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_cbc_aes_sha384_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_cbc_aes_sha_setkey, + .setauthsize = otx2_cpt_aead_set_authsize, + .encrypt = otx2_cpt_aead_encrypt, + .decrypt = otx2_cpt_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(aes))", + .cra_driver_name = "cpt_hmac_sha512_cbc_aes", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_cbc_aes_sha512_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_cbc_aes_sha_setkey, + .setauthsize = otx2_cpt_aead_set_authsize, + .encrypt = otx2_cpt_aead_encrypt, + .decrypt = otx2_cpt_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha1),ecb(cipher_null))", + .cra_driver_name = "cpt_hmac_sha1_ecb_null", + .cra_blocksize = 1, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_ecb_null_sha1_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_ecb_null_sha_setkey, + .setauthsize = otx2_cpt_aead_null_set_authsize, + .encrypt = otx2_cpt_aead_null_encrypt, + .decrypt = otx2_cpt_aead_null_decrypt, + .ivsize = 0, + .maxauthsize = SHA1_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha256),ecb(cipher_null))", + .cra_driver_name = "cpt_hmac_sha256_ecb_null", + .cra_blocksize = 1, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_ecb_null_sha256_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_ecb_null_sha_setkey, + .setauthsize = otx2_cpt_aead_null_set_authsize, + .encrypt = otx2_cpt_aead_null_encrypt, + .decrypt = otx2_cpt_aead_null_decrypt, + .ivsize = 0, + .maxauthsize = SHA256_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha384),ecb(cipher_null))", + .cra_driver_name = "cpt_hmac_sha384_ecb_null", + .cra_blocksize = 1, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_ecb_null_sha384_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_ecb_null_sha_setkey, + .setauthsize = otx2_cpt_aead_null_set_authsize, + .encrypt = otx2_cpt_aead_null_encrypt, + .decrypt = otx2_cpt_aead_null_decrypt, + .ivsize = 0, + .maxauthsize = SHA384_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha512),ecb(cipher_null))", + .cra_driver_name = "cpt_hmac_sha512_ecb_null", + .cra_blocksize = 1, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_ecb_null_sha512_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_ecb_null_sha_setkey, + .setauthsize = otx2_cpt_aead_null_set_authsize, + .encrypt = otx2_cpt_aead_null_encrypt, + .decrypt = otx2_cpt_aead_null_decrypt, + .ivsize = 0, + .maxauthsize = SHA512_DIGEST_SIZE, +}, { + .base = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "cpt_rfc4106_gcm_aes", + .cra_blocksize = 1, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_gcm_aes_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_gcm_aes_setkey, + .setauthsize = otx2_cpt_aead_gcm_set_authsize, + .encrypt = otx2_cpt_aead_encrypt, + .decrypt = otx2_cpt_aead_decrypt, + .ivsize = AES_GCM_IV_SIZE, + .maxauthsize = AES_GCM_ICV_SIZE, +} }; + +static inline int cpt_register_algs(void) +{ + int i, err = 0; + + for (i = 0; i < ARRAY_SIZE(otx2_cpt_skciphers); i++) + otx2_cpt_skciphers[i].base.cra_flags &= ~CRYPTO_ALG_DEAD; + + err = crypto_register_skciphers(otx2_cpt_skciphers, + ARRAY_SIZE(otx2_cpt_skciphers)); + if (err) + return err; + + for (i = 0; i < ARRAY_SIZE(otx2_cpt_aeads); i++) + otx2_cpt_aeads[i].base.cra_flags &= ~CRYPTO_ALG_DEAD; + + err = crypto_register_aeads(otx2_cpt_aeads, + ARRAY_SIZE(otx2_cpt_aeads)); + if (err) { + crypto_unregister_skciphers(otx2_cpt_skciphers, + ARRAY_SIZE(otx2_cpt_skciphers)); + return err; + } + + return 0; +} + +static inline void cpt_unregister_algs(void) +{ + crypto_unregister_skciphers(otx2_cpt_skciphers, + ARRAY_SIZE(otx2_cpt_skciphers)); + crypto_unregister_aeads(otx2_cpt_aeads, ARRAY_SIZE(otx2_cpt_aeads)); +} + +static int compare_func(const void *lptr, const void *rptr) +{ + const struct cpt_device_desc *ldesc = (struct cpt_device_desc *) lptr; + const struct cpt_device_desc *rdesc = (struct cpt_device_desc *) rptr; + + if (ldesc->dev->devfn < rdesc->dev->devfn) + return -1; + if (ldesc->dev->devfn > rdesc->dev->devfn) + return 1; + return 0; +} + +int otx2_cpt_crypto_init(struct pci_dev *pdev, struct module *mod, + int num_queues, int num_devices) +{ + int ret = 0; + int count; + + mutex_lock(&mutex); + count = atomic_read(&se_devices.count); + if (count >= OTX2_CPT_MAX_LFS_NUM) { + dev_err(&pdev->dev, "No space to add a new device\n"); + ret = -ENOSPC; + goto unlock; + } + se_devices.desc[count].num_queues = num_queues; + se_devices.desc[count++].dev = pdev; + atomic_inc(&se_devices.count); + + if (atomic_read(&se_devices.count) == num_devices && + is_crypto_registered == false) { + if (cpt_register_algs()) { + dev_err(&pdev->dev, + "Error in registering crypto algorithms\n"); + ret = -EINVAL; + goto unlock; + } + try_module_get(mod); + is_crypto_registered = true; + } + sort(se_devices.desc, count, sizeof(struct cpt_device_desc), + compare_func, NULL); + +unlock: + mutex_unlock(&mutex); + return ret; +} + +void otx2_cpt_crypto_exit(struct pci_dev *pdev, struct module *mod) +{ + struct cpt_device_table *dev_tbl; + bool dev_found = false; + int i, j, count; + + mutex_lock(&mutex); + + dev_tbl = &se_devices; + count = atomic_read(&dev_tbl->count); + for (i = 0; i < count; i++) { + if (pdev == dev_tbl->desc[i].dev) { + for (j = i; j < count-1; j++) + dev_tbl->desc[j] = dev_tbl->desc[j+1]; + dev_found = true; + break; + } + } + + if (!dev_found) { + dev_err(&pdev->dev, "%s device not found\n", __func__); + goto unlock; + } + if (atomic_dec_and_test(&se_devices.count)) { + cpt_unregister_algs(); + module_put(mod); + is_crypto_registered = false; + } + +unlock: + mutex_unlock(&mutex); +} diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.h b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.h new file mode 100644 index 000000000..d29f84f01 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.h @@ -0,0 +1,183 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2020 Marvell. + */ + +#ifndef __OTX2_CPT_ALGS_H +#define __OTX2_CPT_ALGS_H + +#include +#include +#include +#include "otx2_cpt_common.h" +#include "cn10k_cpt.h" + +#define OTX2_CPT_MAX_ENC_KEY_SIZE 32 +#define OTX2_CPT_MAX_HASH_KEY_SIZE 64 +#define OTX2_CPT_MAX_KEY_SIZE (OTX2_CPT_MAX_ENC_KEY_SIZE + \ + OTX2_CPT_MAX_HASH_KEY_SIZE) +enum otx2_cpt_request_type { + OTX2_CPT_ENC_DEC_REQ = 0x1, + OTX2_CPT_AEAD_ENC_DEC_REQ = 0x2, + OTX2_CPT_AEAD_ENC_DEC_NULL_REQ = 0x3, + OTX2_CPT_PASSTHROUGH_REQ = 0x4 +}; + +enum otx2_cpt_major_opcodes { + OTX2_CPT_MAJOR_OP_MISC = 0x01, + OTX2_CPT_MAJOR_OP_FC = 0x33, + OTX2_CPT_MAJOR_OP_HMAC = 0x35, +}; + +enum otx2_cpt_cipher_type { + OTX2_CPT_CIPHER_NULL = 0x0, + OTX2_CPT_DES3_CBC = 0x1, + OTX2_CPT_DES3_ECB = 0x2, + OTX2_CPT_AES_CBC = 0x3, + OTX2_CPT_AES_ECB = 0x4, + OTX2_CPT_AES_CFB = 0x5, + OTX2_CPT_AES_CTR = 0x6, + OTX2_CPT_AES_GCM = 0x7, + OTX2_CPT_AES_XTS = 0x8 +}; + +enum otx2_cpt_mac_type { + OTX2_CPT_MAC_NULL = 0x0, + OTX2_CPT_MD5 = 0x1, + OTX2_CPT_SHA1 = 0x2, + OTX2_CPT_SHA224 = 0x3, + OTX2_CPT_SHA256 = 0x4, + OTX2_CPT_SHA384 = 0x5, + OTX2_CPT_SHA512 = 0x6, + OTX2_CPT_GMAC = 0x7 +}; + +enum otx2_cpt_aes_key_len { + OTX2_CPT_AES_128_BIT = 0x1, + OTX2_CPT_AES_192_BIT = 0x2, + OTX2_CPT_AES_256_BIT = 0x3 +}; + +union otx2_cpt_encr_ctrl { + u64 u; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 enc_cipher:4; + u64 reserved_59:1; + u64 aes_key:2; + u64 iv_source:1; + u64 mac_type:4; + u64 reserved_49_51:3; + u64 auth_input_type:1; + u64 mac_len:8; + u64 reserved_32_39:8; + u64 encr_offset:16; + u64 iv_offset:8; + u64 auth_offset:8; +#else + u64 auth_offset:8; + u64 iv_offset:8; + u64 encr_offset:16; + u64 reserved_32_39:8; + u64 mac_len:8; + u64 auth_input_type:1; + u64 reserved_49_51:3; + u64 mac_type:4; + u64 iv_source:1; + u64 aes_key:2; + u64 reserved_59:1; + u64 enc_cipher:4; +#endif + } e; +}; + +struct otx2_cpt_cipher { + const char *name; + u8 value; +}; + +struct otx2_cpt_fc_enc_ctx { + union otx2_cpt_encr_ctrl enc_ctrl; + u8 encr_key[32]; + u8 encr_iv[16]; +}; + +union otx2_cpt_fc_hmac_ctx { + struct { + u8 ipad[64]; + u8 opad[64]; + } e; + struct { + u8 hmac_calc[64]; /* HMAC calculated */ + u8 hmac_recv[64]; /* HMAC received */ + } s; +}; + +struct otx2_cpt_fc_ctx { + struct otx2_cpt_fc_enc_ctx enc; + union otx2_cpt_fc_hmac_ctx hmac; +}; + +struct otx2_cpt_enc_ctx { + u32 key_len; + u8 enc_key[OTX2_CPT_MAX_KEY_SIZE]; + u8 cipher_type; + u8 key_type; + u8 enc_align_len; + struct crypto_skcipher *fbk_cipher; + struct pci_dev *pdev; + struct cn10k_cpt_errata_ctx er_ctx; +}; + +union otx2_cpt_offset_ctrl { + u64 flags; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved:32; + u64 enc_data_offset:16; + u64 iv_offset:8; + u64 auth_offset:8; +#else + u64 auth_offset:8; + u64 iv_offset:8; + u64 enc_data_offset:16; + u64 reserved:32; +#endif + } e; +}; + +struct otx2_cpt_req_ctx { + struct otx2_cpt_req_info cpt_req; + union otx2_cpt_offset_ctrl ctrl_word; + struct otx2_cpt_fc_ctx fctx; + union { + struct skcipher_request sk_fbk_req; + struct aead_request fbk_req; + }; +}; + +struct otx2_cpt_sdesc { + struct shash_desc shash; +}; + +struct otx2_cpt_aead_ctx { + u8 key[OTX2_CPT_MAX_KEY_SIZE]; + struct crypto_shash *hashalg; + struct otx2_cpt_sdesc *sdesc; + struct crypto_aead *fbk_cipher; + struct cn10k_cpt_errata_ctx er_ctx; + struct pci_dev *pdev; + u8 *ipad; + u8 *opad; + u32 enc_key_len; + u32 auth_key_len; + u8 cipher_type; + u8 mac_type; + u8 key_type; + u8 is_trunc_hmac; + u8 enc_align_len; +}; +int otx2_cpt_crypto_init(struct pci_dev *pdev, struct module *mod, + int num_queues, int num_devices); +void otx2_cpt_crypto_exit(struct pci_dev *pdev, struct module *mod); + +#endif /* __OTX2_CPT_ALGS_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c new file mode 100644 index 000000000..09ab85e60 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c @@ -0,0 +1,482 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include "otx2_cpt_common.h" +#include "otx2_cptvf.h" +#include "otx2_cptlf.h" +#include "otx2_cptvf_algs.h" +#include "cn10k_cpt.h" +#include + +#define OTX2_CPTVF_DRV_NAME "rvu_cptvf" + +static void cptvf_enable_pfvf_mbox_intrs(struct otx2_cptvf_dev *cptvf) +{ + /* Clear interrupt if any */ + otx2_cpt_write64(cptvf->reg_base, BLKADDR_RVUM, 0, OTX2_RVU_VF_INT, + 0x1ULL); + + /* Enable PF-VF interrupt */ + otx2_cpt_write64(cptvf->reg_base, BLKADDR_RVUM, 0, + OTX2_RVU_VF_INT_ENA_W1S, 0x1ULL); +} + +static void cptvf_disable_pfvf_mbox_intrs(struct otx2_cptvf_dev *cptvf) +{ + /* Disable PF-VF interrupt */ + otx2_cpt_write64(cptvf->reg_base, BLKADDR_RVUM, 0, + OTX2_RVU_VF_INT_ENA_W1C, 0x1ULL); + + /* Clear interrupt if any */ + otx2_cpt_write64(cptvf->reg_base, BLKADDR_RVUM, 0, OTX2_RVU_VF_INT, + 0x1ULL); +} + +static int cptvf_register_interrupts(struct otx2_cptvf_dev *cptvf) +{ + int ret, irq; + int num_vec; + + num_vec = pci_msix_vec_count(cptvf->pdev); + if (num_vec <= 0) + return -EINVAL; + + /* Enable MSI-X */ + ret = pci_alloc_irq_vectors(cptvf->pdev, num_vec, num_vec, + PCI_IRQ_MSIX); + if (ret < 0) { + dev_err(&cptvf->pdev->dev, + "Request for %d msix vectors failed\n", num_vec); + return ret; + } + irq = pci_irq_vector(cptvf->pdev, OTX2_CPT_VF_INT_VEC_E_MBOX); + /* Register VF<=>PF mailbox interrupt handler */ + ret = devm_request_irq(&cptvf->pdev->dev, irq, + otx2_cptvf_pfvf_mbox_intr, 0, + "CPTPFVF Mbox", cptvf); + if (ret) + return ret; + /* Enable PF-VF mailbox interrupts */ + cptvf_enable_pfvf_mbox_intrs(cptvf); + + ret = otx2_cpt_send_ready_msg(&cptvf->pfvf_mbox, cptvf->pdev); + if (ret) { + dev_warn(&cptvf->pdev->dev, + "PF not responding to mailbox, deferring probe\n"); + cptvf_disable_pfvf_mbox_intrs(cptvf); + return -EPROBE_DEFER; + } + return 0; +} + +static int cptvf_pfvf_mbox_init(struct otx2_cptvf_dev *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + resource_size_t offset, size; + int ret; + + cptvf->pfvf_mbox_wq = + alloc_ordered_workqueue("cpt_pfvf_mailbox", + WQ_HIGHPRI | WQ_MEM_RECLAIM); + if (!cptvf->pfvf_mbox_wq) + return -ENOMEM; + + if (test_bit(CN10K_MBOX, &cptvf->cap_flag)) { + /* For cn10k platform, VF mailbox region is in its BAR2 + * register space + */ + cptvf->pfvf_mbox_base = cptvf->reg_base + + CN10K_CPT_VF_MBOX_REGION; + } else { + offset = pci_resource_start(pdev, PCI_MBOX_BAR_NUM); + size = pci_resource_len(pdev, PCI_MBOX_BAR_NUM); + /* Map PF-VF mailbox memory */ + cptvf->pfvf_mbox_base = devm_ioremap_wc(&pdev->dev, offset, + size); + if (!cptvf->pfvf_mbox_base) { + dev_err(&pdev->dev, "Unable to map BAR4\n"); + ret = -ENOMEM; + goto free_wqe; + } + } + + ret = otx2_mbox_init(&cptvf->pfvf_mbox, cptvf->pfvf_mbox_base, + pdev, cptvf->reg_base, MBOX_DIR_VFPF, 1); + if (ret) + goto free_wqe; + + ret = otx2_cpt_mbox_bbuf_init(cptvf, pdev); + if (ret) + goto destroy_mbox; + + INIT_WORK(&cptvf->pfvf_mbox_work, otx2_cptvf_pfvf_mbox_handler); + return 0; + +destroy_mbox: + otx2_mbox_destroy(&cptvf->pfvf_mbox); +free_wqe: + destroy_workqueue(cptvf->pfvf_mbox_wq); + return ret; +} + +static void cptvf_pfvf_mbox_destroy(struct otx2_cptvf_dev *cptvf) +{ + destroy_workqueue(cptvf->pfvf_mbox_wq); + otx2_mbox_destroy(&cptvf->pfvf_mbox); +} + +static void cptlf_work_handler(unsigned long data) +{ + otx2_cpt_post_process((struct otx2_cptlf_wqe *) data); +} + +static void cleanup_tasklet_work(struct otx2_cptlfs_info *lfs) +{ + int i; + + for (i = 0; i < lfs->lfs_num; i++) { + if (!lfs->lf[i].wqe) + continue; + + tasklet_kill(&lfs->lf[i].wqe->work); + kfree(lfs->lf[i].wqe); + lfs->lf[i].wqe = NULL; + } +} + +static int init_tasklet_work(struct otx2_cptlfs_info *lfs) +{ + struct otx2_cptlf_wqe *wqe; + int i, ret = 0; + + for (i = 0; i < lfs->lfs_num; i++) { + wqe = kzalloc_obj(struct otx2_cptlf_wqe); + if (!wqe) { + ret = -ENOMEM; + goto cleanup_tasklet; + } + + tasklet_init(&wqe->work, cptlf_work_handler, (u64) wqe); + wqe->lfs = lfs; + wqe->lf_num = i; + lfs->lf[i].wqe = wqe; + } + return 0; + +cleanup_tasklet: + cleanup_tasklet_work(lfs); + return ret; +} + +static void free_pending_queues(struct otx2_cptlfs_info *lfs) +{ + int i; + + for (i = 0; i < lfs->lfs_num; i++) { + kfree(lfs->lf[i].pqueue.head); + lfs->lf[i].pqueue.head = NULL; + } +} + +static int alloc_pending_queues(struct otx2_cptlfs_info *lfs) +{ + int size, ret, i; + + if (!lfs->lfs_num) + return -EINVAL; + + for (i = 0; i < lfs->lfs_num; i++) { + lfs->lf[i].pqueue.qlen = OTX2_CPT_INST_QLEN_MSGS; + size = lfs->lf[i].pqueue.qlen * + sizeof(struct otx2_cpt_pending_entry); + + lfs->lf[i].pqueue.head = kzalloc(size, GFP_KERNEL); + if (!lfs->lf[i].pqueue.head) { + ret = -ENOMEM; + goto error; + } + + /* Initialize spin lock */ + spin_lock_init(&lfs->lf[i].pqueue.lock); + } + return 0; + +error: + free_pending_queues(lfs); + return ret; +} + +static void lf_sw_cleanup(struct otx2_cptlfs_info *lfs) +{ + cleanup_tasklet_work(lfs); + free_pending_queues(lfs); +} + +static int lf_sw_init(struct otx2_cptlfs_info *lfs) +{ + int ret; + + ret = alloc_pending_queues(lfs); + if (ret) { + dev_err(&lfs->pdev->dev, + "Allocating pending queues failed\n"); + return ret; + } + ret = init_tasklet_work(lfs); + if (ret) { + dev_err(&lfs->pdev->dev, + "Tasklet work init failed\n"); + goto pending_queues_free; + } + return 0; + +pending_queues_free: + free_pending_queues(lfs); + return ret; +} + +static void cptvf_lf_shutdown(struct otx2_cptlfs_info *lfs) +{ + atomic_set(&lfs->state, OTX2_CPTLF_IN_RESET); + + /* Remove interrupts affinity */ + otx2_cptlf_free_irqs_affinity(lfs); + /* Disable instruction queue */ + otx2_cptlf_disable_iqueues(lfs); + /* Unregister crypto algorithms */ + otx2_cpt_crypto_exit(lfs->pdev, THIS_MODULE); + /* Unregister LFs interrupts */ + otx2_cptlf_unregister_misc_interrupts(lfs); + otx2_cptlf_unregister_done_interrupts(lfs); + /* Cleanup LFs software side */ + lf_sw_cleanup(lfs); + /* Free instruction queues */ + otx2_cpt_free_instruction_queues(lfs); + /* Send request to detach LFs */ + otx2_cpt_detach_rsrcs_msg(lfs); + lfs->lfs_num = 0; +} + +static int cptvf_lf_init(struct otx2_cptvf_dev *cptvf) +{ + struct otx2_cptlfs_info *lfs = &cptvf->lfs; + struct device *dev = &cptvf->pdev->dev; + int ret, lfs_num; + u8 eng_grp_msk; + + /* Get engine group number for symmetric crypto */ + cptvf->lfs.kcrypto_se_eng_grp_num = OTX2_CPT_INVALID_CRYPTO_ENG_GRP; + ret = otx2_cptvf_send_eng_grp_num_msg(cptvf, OTX2_CPT_SE_TYPES); + if (ret) + return ret; + + if (cptvf->lfs.kcrypto_se_eng_grp_num == + OTX2_CPT_INVALID_CRYPTO_ENG_GRP) { + dev_err(dev, + "Symmetric Engine group for crypto not available\n"); + return -ENOENT; + } + + /* Get engine group number for asymmetric crypto */ + cptvf->lfs.kcrypto_ae_eng_grp_num = OTX2_CPT_INVALID_CRYPTO_ENG_GRP; + ret = otx2_cptvf_send_eng_grp_num_msg(cptvf, OTX2_CPT_AE_TYPES); + if (ret) + return ret; + + if (cptvf->lfs.kcrypto_ae_eng_grp_num == + OTX2_CPT_INVALID_CRYPTO_ENG_GRP) { + dev_err(dev, + "Asymmetric Engine group for crypto not available\n"); + return -ENOENT; + } + + eng_grp_msk = BIT(cptvf->lfs.kcrypto_se_eng_grp_num) | + BIT(cptvf->lfs.kcrypto_ae_eng_grp_num); + + ret = otx2_cptvf_send_kvf_limits_msg(cptvf); + if (ret) + return ret; + + lfs_num = cptvf->lfs.kvf_limits; + + ret = otx2_cptlf_init(lfs, eng_grp_msk, OTX2_CPT_QUEUE_HI_PRIO, + lfs_num); + if (ret) + return ret; + + /* Get msix offsets for attached LFs */ + ret = otx2_cpt_msix_offset_msg(lfs); + if (ret) + goto cleanup_lf; + + /* Initialize LFs software side */ + ret = lf_sw_init(lfs); + if (ret) + goto cleanup_lf; + + /* Register LFs interrupts */ + ret = otx2_cptlf_register_misc_interrupts(lfs); + if (ret) + goto cleanup_lf_sw; + + ret = otx2_cptlf_register_done_interrupts(lfs); + if (ret) + goto cleanup_lf_sw; + + /* Set interrupts affinity */ + ret = otx2_cptlf_set_irqs_affinity(lfs); + if (ret) + goto unregister_intr; + + atomic_set(&lfs->state, OTX2_CPTLF_STARTED); + /* Register crypto algorithms */ + ret = otx2_cpt_crypto_init(lfs->pdev, THIS_MODULE, lfs_num, 1); + if (ret) { + dev_err(&lfs->pdev->dev, "algorithms registration failed\n"); + goto disable_irqs; + } + return 0; + +disable_irqs: + otx2_cptlf_free_irqs_affinity(lfs); +unregister_intr: + otx2_cptlf_unregister_misc_interrupts(lfs); + otx2_cptlf_unregister_done_interrupts(lfs); +cleanup_lf_sw: + lf_sw_cleanup(lfs); +cleanup_lf: + otx2_cptlf_shutdown(lfs); + + return ret; +} + +static int otx2_cptvf_probe(struct pci_dev *pdev, + const struct pci_device_id *ent) +{ + struct device *dev = &pdev->dev; + struct otx2_cptvf_dev *cptvf; + int ret; + + cptvf = devm_kzalloc(dev, sizeof(*cptvf), GFP_KERNEL); + if (!cptvf) + return -ENOMEM; + + ret = pcim_enable_device(pdev); + if (ret) { + dev_err(dev, "Failed to enable PCI device\n"); + goto clear_drvdata; + } + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); + if (ret) { + dev_err(dev, "Unable to get usable DMA configuration\n"); + goto clear_drvdata; + } + + ret = pcim_request_all_regions(pdev, OTX2_CPTVF_DRV_NAME); + if (ret) { + dev_err(dev, "Couldn't get PCI resources 0x%x\n", ret); + goto clear_drvdata; + } + pci_set_master(pdev); + pci_set_drvdata(pdev, cptvf); + cptvf->pdev = pdev; + + /* Map VF's configuration registers */ + cptvf->reg_base = pcim_iomap(pdev, PCI_PF_REG_BAR_NUM, 0); + if (!cptvf->reg_base) { + ret = -ENOMEM; + dev_err(dev, "Couldn't ioremap PCI resource 0x%x\n", ret); + goto clear_drvdata; + } + + otx2_cpt_set_hw_caps(pdev, &cptvf->cap_flag); + + /* Initialize PF<=>VF mailbox */ + ret = cptvf_pfvf_mbox_init(cptvf); + if (ret) + goto clear_drvdata; + + /* Register interrupts */ + ret = cptvf_register_interrupts(cptvf); + if (ret) + goto destroy_pfvf_mbox; + + cptvf->blkaddr = BLKADDR_CPT0; + + cptvf_hw_ops_get(cptvf); + + otx2_cptlf_set_dev_info(&cptvf->lfs, cptvf->pdev, cptvf->reg_base, + &cptvf->pfvf_mbox, cptvf->blkaddr); + + ret = otx2_cptvf_send_caps_msg(cptvf); + if (ret) { + dev_err(&pdev->dev, "Couldn't get CPT engine capabilities.\n"); + goto unregister_interrupts; + } + if (cptvf->eng_caps[OTX2_CPT_SE_TYPES] & BIT_ULL(35)) + cptvf->lfs.ops->cpt_sg_info_create = cn10k_sgv2_info_create; + + ret = cn10k_cptvf_lmtst_init(cptvf); + if (ret) + goto unregister_interrupts; + + /* Initialize CPT LFs */ + ret = cptvf_lf_init(cptvf); + if (ret) + goto free_lmtst; + + return 0; + +free_lmtst: + cn10k_cpt_lmtst_free(pdev, &cptvf->lfs); +unregister_interrupts: + cptvf_disable_pfvf_mbox_intrs(cptvf); +destroy_pfvf_mbox: + cptvf_pfvf_mbox_destroy(cptvf); +clear_drvdata: + pci_set_drvdata(pdev, NULL); + + return ret; +} + +static void otx2_cptvf_remove(struct pci_dev *pdev) +{ + struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev); + + if (!cptvf) { + dev_err(&pdev->dev, "Invalid CPT VF device.\n"); + return; + } + cptvf_lf_shutdown(&cptvf->lfs); + /* Disable PF-VF mailbox interrupt */ + cptvf_disable_pfvf_mbox_intrs(cptvf); + /* Destroy PF-VF mbox */ + cptvf_pfvf_mbox_destroy(cptvf); + /* Free LMTST memory */ + cn10k_cpt_lmtst_free(pdev, &cptvf->lfs); + pci_set_drvdata(pdev, NULL); +} + +/* Supported devices */ +static const struct pci_device_id otx2_cptvf_id_table[] = { + { PCI_VDEVICE(CAVIUM, OTX2_CPT_PCI_VF_DEVICE_ID) }, + { PCI_VDEVICE(CAVIUM, CN10K_CPT_PCI_VF_DEVICE_ID) }, + { } /* end of table */ +}; +MODULE_DEVICE_TABLE(pci, otx2_cptvf_id_table); + +static struct pci_driver otx2_cptvf_pci_driver = { + .name = OTX2_CPTVF_DRV_NAME, + .id_table = otx2_cptvf_id_table, + .probe = otx2_cptvf_probe, + .remove = otx2_cptvf_remove, +}; + +module_pci_driver(otx2_cptvf_pci_driver); + +MODULE_IMPORT_NS("CRYPTO_DEV_OCTEONTX2_CPT"); + +MODULE_AUTHOR("Marvell"); +MODULE_DESCRIPTION("Marvell RVU CPT Virtual Function Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c new file mode 100644 index 000000000..5277bcfa2 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c @@ -0,0 +1,241 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include "otx2_cpt_common.h" +#include "otx2_cptvf.h" +#include + +int otx2_cpt_mbox_bbuf_init(struct otx2_cptvf_dev *cptvf, struct pci_dev *pdev) +{ + struct otx2_mbox_dev *mdev; + struct otx2_mbox *otx2_mbox; + + cptvf->bbuf_base = devm_kmalloc(&pdev->dev, MBOX_SIZE, GFP_KERNEL); + if (!cptvf->bbuf_base) + return -ENOMEM; + /* + * Overwrite mbox mbase to point to bounce buffer, so that PF/VF + * prepare all mbox messages in bounce buffer instead of directly + * in hw mbox memory. + */ + otx2_mbox = &cptvf->pfvf_mbox; + mdev = &otx2_mbox->dev[0]; + mdev->mbase = cptvf->bbuf_base; + + return 0; +} + +static void otx2_cpt_sync_mbox_bbuf(struct otx2_mbox *mbox, int devid) +{ + u16 msgs_offset = ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN); + void *hw_mbase = mbox->hwbase + (devid * MBOX_SIZE); + struct otx2_mbox_dev *mdev = &mbox->dev[devid]; + struct mbox_hdr *hdr; + u64 msg_size; + + if (mdev->mbase == hw_mbase) + return; + + hdr = hw_mbase + mbox->rx_start; + msg_size = hdr->msg_size; + + if (msg_size > mbox->rx_size - msgs_offset) + msg_size = mbox->rx_size - msgs_offset; + + /* Copy mbox messages from mbox memory to bounce buffer */ + memcpy(mdev->mbase + mbox->rx_start, + hw_mbase + mbox->rx_start, msg_size + msgs_offset); +} + +irqreturn_t otx2_cptvf_pfvf_mbox_intr(int __always_unused irq, void *arg) +{ + struct otx2_cptvf_dev *cptvf = arg; + u64 intr; + + /* Read the interrupt bits */ + intr = otx2_cpt_read64(cptvf->reg_base, BLKADDR_RVUM, 0, + OTX2_RVU_VF_INT); + + if (intr & 0x1ULL) { + /* Schedule work queue function to process the MBOX request */ + queue_work(cptvf->pfvf_mbox_wq, &cptvf->pfvf_mbox_work); + /* Clear and ack the interrupt */ + otx2_cpt_write64(cptvf->reg_base, BLKADDR_RVUM, 0, + OTX2_RVU_VF_INT, 0x1ULL); + } + return IRQ_HANDLED; +} + +static void process_pfvf_mbox_mbox_msg(struct otx2_cptvf_dev *cptvf, + struct mbox_msghdr *msg) +{ + struct otx2_cptlfs_info *lfs = &cptvf->lfs; + struct otx2_cpt_kvf_limits_rsp *rsp_limits; + struct otx2_cpt_egrp_num_rsp *rsp_grp; + struct otx2_cpt_caps_rsp *eng_caps; + struct cpt_rd_wr_reg_msg *rsp_reg; + struct msix_offset_rsp *rsp_msix; + u8 grp_num; + int i; + + if (msg->id >= MBOX_MSG_MAX) { + dev_err(&cptvf->pdev->dev, + "MBOX msg with unknown ID %d\n", msg->id); + return; + } + if (msg->sig != OTX2_MBOX_RSP_SIG) { + dev_err(&cptvf->pdev->dev, + "MBOX msg with wrong signature %x, ID %d\n", + msg->sig, msg->id); + return; + } + switch (msg->id) { + case MBOX_MSG_READY: + cptvf->vf_id = ((msg->pcifunc >> RVU_PFVF_FUNC_SHIFT) + & RVU_PFVF_FUNC_MASK) - 1; + break; + case MBOX_MSG_ATTACH_RESOURCES: + /* Check if resources were successfully attached */ + if (!msg->rc) + lfs->are_lfs_attached = 1; + break; + case MBOX_MSG_DETACH_RESOURCES: + /* Check if resources were successfully detached */ + if (!msg->rc) + lfs->are_lfs_attached = 0; + break; + case MBOX_MSG_MSIX_OFFSET: + rsp_msix = (struct msix_offset_rsp *) msg; + for (i = 0; i < rsp_msix->cptlfs; i++) + lfs->lf[i].msix_offset = rsp_msix->cptlf_msixoff[i]; + break; + case MBOX_MSG_CPT_RD_WR_REGISTER: + rsp_reg = (struct cpt_rd_wr_reg_msg *) msg; + if (msg->rc) { + dev_err(&cptvf->pdev->dev, + "Reg %llx rd/wr(%d) failed %d\n", + rsp_reg->reg_offset, rsp_reg->is_write, + msg->rc); + return; + } + if (!rsp_reg->is_write) + *rsp_reg->ret_val = rsp_reg->val; + break; + case MBOX_MSG_GET_ENG_GRP_NUM: + rsp_grp = (struct otx2_cpt_egrp_num_rsp *) msg; + grp_num = rsp_grp->eng_grp_num; + if (rsp_grp->eng_type == OTX2_CPT_SE_TYPES) + cptvf->lfs.kcrypto_se_eng_grp_num = grp_num; + else if (rsp_grp->eng_type == OTX2_CPT_AE_TYPES) + cptvf->lfs.kcrypto_ae_eng_grp_num = grp_num; + break; + case MBOX_MSG_GET_KVF_LIMITS: + rsp_limits = (struct otx2_cpt_kvf_limits_rsp *) msg; + cptvf->lfs.kvf_limits = rsp_limits->kvf_limits; + break; + case MBOX_MSG_GET_CAPS: + eng_caps = (struct otx2_cpt_caps_rsp *)msg; + memcpy(cptvf->eng_caps, eng_caps->eng_caps, + sizeof(cptvf->eng_caps)); + break; + case MBOX_MSG_CPT_LF_RESET: + case MBOX_MSG_LMTST_TBL_SETUP: + break; + default: + dev_err(&cptvf->pdev->dev, "Unsupported msg %d received.\n", + msg->id); + break; + } +} + +void otx2_cptvf_pfvf_mbox_handler(struct work_struct *work) +{ + struct otx2_cptvf_dev *cptvf; + struct otx2_mbox *pfvf_mbox; + struct otx2_mbox_dev *mdev; + struct mbox_hdr *rsp_hdr; + struct mbox_msghdr *msg; + int offset, i; + + /* sync with mbox memory region */ + smp_rmb(); + + cptvf = container_of(work, struct otx2_cptvf_dev, pfvf_mbox_work); + pfvf_mbox = &cptvf->pfvf_mbox; + otx2_cpt_sync_mbox_bbuf(pfvf_mbox, 0); + mdev = &pfvf_mbox->dev[0]; + rsp_hdr = (struct mbox_hdr *)(mdev->mbase + pfvf_mbox->rx_start); + if (rsp_hdr->num_msgs == 0) + return; + offset = ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN); + + for (i = 0; i < rsp_hdr->num_msgs; i++) { + msg = (struct mbox_msghdr *)(mdev->mbase + pfvf_mbox->rx_start + + offset); + process_pfvf_mbox_mbox_msg(cptvf, msg); + offset = msg->next_msgoff; + mdev->msgs_acked++; + } + otx2_mbox_reset(pfvf_mbox, 0); +} + +int otx2_cptvf_send_eng_grp_num_msg(struct otx2_cptvf_dev *cptvf, int eng_type) +{ + struct otx2_mbox *mbox = &cptvf->pfvf_mbox; + struct pci_dev *pdev = cptvf->pdev; + struct otx2_cpt_egrp_num_msg *req; + + req = (struct otx2_cpt_egrp_num_msg *) + otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct otx2_cpt_egrp_num_rsp)); + if (req == NULL) { + dev_err(&pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + req->hdr.id = MBOX_MSG_GET_ENG_GRP_NUM; + req->hdr.sig = OTX2_MBOX_REQ_SIG; + req->hdr.pcifunc = OTX2_CPT_RVU_PFFUNC(cptvf->pdev, cptvf->vf_id, 0); + req->eng_type = eng_type; + + return otx2_cpt_send_mbox_msg(mbox, pdev); +} + +int otx2_cptvf_send_kvf_limits_msg(struct otx2_cptvf_dev *cptvf) +{ + struct otx2_mbox *mbox = &cptvf->pfvf_mbox; + struct pci_dev *pdev = cptvf->pdev; + struct mbox_msghdr *req; + + req = (struct mbox_msghdr *) + otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct otx2_cpt_kvf_limits_rsp)); + if (req == NULL) { + dev_err(&pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + req->id = MBOX_MSG_GET_KVF_LIMITS; + req->sig = OTX2_MBOX_REQ_SIG; + req->pcifunc = OTX2_CPT_RVU_PFFUNC(cptvf->pdev, cptvf->vf_id, 0); + + return otx2_cpt_send_mbox_msg(mbox, pdev); +} + +int otx2_cptvf_send_caps_msg(struct otx2_cptvf_dev *cptvf) +{ + struct otx2_mbox *mbox = &cptvf->pfvf_mbox; + struct pci_dev *pdev = cptvf->pdev; + struct mbox_msghdr *req; + + req = (struct mbox_msghdr *) + otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct otx2_cpt_caps_rsp)); + if (!req) { + dev_err(&pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + req->id = MBOX_MSG_GET_CAPS; + req->sig = OTX2_MBOX_REQ_SIG; + req->pcifunc = OTX2_CPT_RVU_PFFUNC(cptvf->pdev, cptvf->vf_id, 0); + + return otx2_cpt_send_mbox_msg(mbox, pdev); +} diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c new file mode 100644 index 000000000..e71494486 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c @@ -0,0 +1,409 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include "otx2_cptvf.h" +#include "otx2_cpt_common.h" + +/* Default timeout when waiting for free pending entry in us */ +#define CPT_PENTRY_TIMEOUT 1000 +#define CPT_PENTRY_STEP 50 + +/* Default threshold for stopping and resuming sender requests */ +#define CPT_IQ_STOP_MARGIN 128 +#define CPT_IQ_RESUME_MARGIN 512 + +/* Default command timeout in seconds */ +#define CPT_COMMAND_TIMEOUT 4 +#define CPT_TIME_IN_RESET_COUNT 5 + +static void otx2_cpt_dump_sg_list(struct pci_dev *pdev, + struct otx2_cpt_req_info *req) +{ + int i; + + pr_debug("Gather list size %d\n", req->in_cnt); + for (i = 0; i < req->in_cnt; i++) { + pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%llx\n", i, + req->in[i].size, req->in[i].vptr, + req->in[i].dma_addr); + pr_debug("Buffer hexdump (%d bytes)\n", + req->in[i].size); + print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, + req->in[i].vptr, req->in[i].size, false); + } + pr_debug("Scatter list size %d\n", req->out_cnt); + for (i = 0; i < req->out_cnt; i++) { + pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%llx\n", i, + req->out[i].size, req->out[i].vptr, + req->out[i].dma_addr); + pr_debug("Buffer hexdump (%d bytes)\n", req->out[i].size); + print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, + req->out[i].vptr, req->out[i].size, false); + } +} + +static inline struct otx2_cpt_pending_entry *get_free_pending_entry( + struct otx2_cpt_pending_queue *q, + int qlen) +{ + struct otx2_cpt_pending_entry *ent = NULL; + + ent = &q->head[q->rear]; + if (unlikely(ent->busy)) + return NULL; + + q->rear++; + if (unlikely(q->rear == qlen)) + q->rear = 0; + + return ent; +} + +static inline u32 modulo_inc(u32 index, u32 length, u32 inc) +{ + if (WARN_ON(inc > length)) + inc = length; + + index += inc; + if (unlikely(index >= length)) + index -= length; + + return index; +} + +static inline void free_pentry(struct otx2_cpt_pending_entry *pentry) +{ + pentry->completion_addr = NULL; + pentry->info = NULL; + pentry->callback = NULL; + pentry->areq = NULL; + pentry->resume_sender = false; + pentry->busy = false; +} + +static int process_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, + struct otx2_cpt_pending_queue *pqueue, + struct otx2_cptlf_info *lf) +{ + struct otx2_cptvf_request *cpt_req = &req->req; + struct otx2_cpt_pending_entry *pentry = NULL; + union otx2_cpt_ctrl_info *ctrl = &req->ctrl; + struct otx2_cpt_inst_info *info = NULL; + union otx2_cpt_res_s *result = NULL; + struct otx2_cpt_iq_command iq_cmd; + union otx2_cpt_inst_s cptinst; + int retry, ret = 0; + u8 resume_sender; + gfp_t gfp; + + gfp = (req->areq->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : + GFP_ATOMIC; + if (unlikely(!otx2_cptlf_started(lf->lfs))) + return -ENODEV; + + info = lf->lfs->ops->cpt_sg_info_create(pdev, req, gfp); + if (unlikely(!info)) { + dev_err(&pdev->dev, "Setting up cpt inst info failed"); + return -ENOMEM; + } + cpt_req->dlen = info->dlen; + + result = info->completion_addr; + result->s.compcode = OTX2_CPT_COMPLETION_CODE_INIT; + + spin_lock_bh(&pqueue->lock); + pentry = get_free_pending_entry(pqueue, pqueue->qlen); + retry = CPT_PENTRY_TIMEOUT / CPT_PENTRY_STEP; + while (unlikely(!pentry) && retry--) { + spin_unlock_bh(&pqueue->lock); + udelay(CPT_PENTRY_STEP); + spin_lock_bh(&pqueue->lock); + pentry = get_free_pending_entry(pqueue, pqueue->qlen); + } + + if (unlikely(!pentry)) { + ret = -ENOSPC; + goto destroy_info; + } + + /* + * Check if we are close to filling in entire pending queue, + * if so then tell the sender to stop/sleep by returning -EBUSY + * We do it only for context which can sleep (GFP_KERNEL) + */ + if (gfp == GFP_KERNEL && + pqueue->pending_count > (pqueue->qlen - CPT_IQ_STOP_MARGIN)) { + pentry->resume_sender = true; + } else + pentry->resume_sender = false; + resume_sender = pentry->resume_sender; + pqueue->pending_count++; + + pentry->completion_addr = info->completion_addr; + pentry->info = info; + pentry->callback = req->callback; + pentry->areq = req->areq; + pentry->busy = true; + info->pentry = pentry; + info->time_in = jiffies; + info->req = req; + + /* Fill in the command */ + iq_cmd.cmd.u = 0; + iq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags); + iq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1); + iq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2); + iq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen); + + /* 64-bit swap for microcode data reads, not needed for addresses*/ + cpu_to_be64s(&iq_cmd.cmd.u); + iq_cmd.dptr = info->dptr_baddr | info->gthr_sz << 60; + iq_cmd.rptr = info->rptr_baddr | info->sctr_sz << 60; + iq_cmd.cptr.s.cptr = cpt_req->cptr_dma; + iq_cmd.cptr.s.grp = ctrl->s.grp; + + /* Fill in the CPT_INST_S type command for HW interpretation */ + otx2_cpt_fill_inst(&cptinst, &iq_cmd, info->comp_baddr); + + /* Print debug info if enabled */ + otx2_cpt_dump_sg_list(pdev, req); + pr_debug("Cpt_inst_s hexdump (%d bytes)\n", OTX2_CPT_INST_SIZE); + print_hex_dump_debug("", 0, 16, 1, &cptinst, OTX2_CPT_INST_SIZE, false); + pr_debug("Dptr hexdump (%d bytes)\n", cpt_req->dlen); + print_hex_dump_debug("", 0, 16, 1, info->in_buffer, + cpt_req->dlen, false); + + /* Send CPT command */ + lf->lfs->ops->send_cmd(&cptinst, 1, lf); + + /* + * We allocate and prepare pending queue entry in critical section + * together with submitting CPT instruction to CPT instruction queue + * to make sure that order of CPT requests is the same in both + * pending and instruction queues + */ + spin_unlock_bh(&pqueue->lock); + + ret = resume_sender ? -EBUSY : -EINPROGRESS; + return ret; + +destroy_info: + spin_unlock_bh(&pqueue->lock); + otx2_cpt_info_destroy(pdev, info); + return ret; +} + +int otx2_cpt_do_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, + int cpu_num) +{ + struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev); + struct otx2_cptlfs_info *lfs = &cptvf->lfs; + + return process_request(lfs->pdev, req, &lfs->lf[cpu_num].pqueue, + &lfs->lf[cpu_num]); +} + +static int cpt_process_ccode(struct otx2_cptlfs_info *lfs, + union otx2_cpt_res_s *cpt_status, + struct otx2_cpt_inst_info *info, + u32 *res_code) +{ + u8 uc_ccode = lfs->ops->cpt_get_uc_compcode(cpt_status); + u8 ccode = lfs->ops->cpt_get_compcode(cpt_status); + struct pci_dev *pdev = lfs->pdev; + + switch (ccode) { + case OTX2_CPT_COMP_E_FAULT: + dev_err(&pdev->dev, + "Request failed with DMA fault\n"); + otx2_cpt_dump_sg_list(pdev, info->req); + break; + + case OTX2_CPT_COMP_E_HWERR: + dev_err(&pdev->dev, + "Request failed with hardware error\n"); + otx2_cpt_dump_sg_list(pdev, info->req); + break; + + case OTX2_CPT_COMP_E_INSTERR: + dev_err(&pdev->dev, + "Request failed with instruction error\n"); + otx2_cpt_dump_sg_list(pdev, info->req); + break; + + case OTX2_CPT_COMP_E_NOTDONE: + /* check for timeout */ + if (time_after_eq(jiffies, info->time_in + + CPT_COMMAND_TIMEOUT * HZ)) + dev_warn(&pdev->dev, + "Request timed out 0x%p", info->req); + else if (info->extra_time < CPT_TIME_IN_RESET_COUNT) { + info->time_in = jiffies; + info->extra_time++; + } + return 1; + + case OTX2_CPT_COMP_E_GOOD: + case OTX2_CPT_COMP_E_WARN: + /* + * Check microcode completion code, it is only valid + * when completion code is CPT_COMP_E::GOOD + */ + if (uc_ccode != OTX2_CPT_UCC_SUCCESS) { + /* + * If requested hmac is truncated and ucode returns + * s/g write length error then we report success + * because ucode writes as many bytes of calculated + * hmac as available in gather buffer and reports + * s/g write length error if number of bytes in gather + * buffer is less than full hmac size. + */ + if (info->req->is_trunc_hmac && + uc_ccode == OTX2_CPT_UCC_SG_WRITE_LENGTH) { + *res_code = 0; + break; + } + + pr_debug("Request failed with software error code 0x%x: algo = %s driver = %s\n", + cpt_status->s.uc_compcode, + info->req->areq->tfm->__crt_alg->cra_name, + info->req->areq->tfm->__crt_alg->cra_driver_name); + otx2_cpt_dump_sg_list(pdev, info->req); + break; + } + /* Request has been processed with success */ + *res_code = 0; + break; + + default: + dev_err(&pdev->dev, + "Request returned invalid status %d\n", ccode); + break; + } + return 0; +} + +static inline void process_pending_queue(struct otx2_cptlfs_info *lfs, + struct otx2_cpt_pending_queue *pqueue) +{ + struct otx2_cpt_pending_entry *resume_pentry = NULL; + void (*callback)(int status, void *arg, void *req); + struct otx2_cpt_pending_entry *pentry = NULL; + union otx2_cpt_res_s *cpt_status = NULL; + struct otx2_cpt_inst_info *info = NULL; + struct otx2_cpt_req_info *req = NULL; + struct crypto_async_request *areq; + struct pci_dev *pdev = lfs->pdev; + u32 res_code, resume_index; + + while (1) { + spin_lock_bh(&pqueue->lock); + pentry = &pqueue->head[pqueue->front]; + + if (WARN_ON(!pentry)) { + spin_unlock_bh(&pqueue->lock); + break; + } + + res_code = -EINVAL; + if (unlikely(!pentry->busy)) { + spin_unlock_bh(&pqueue->lock); + break; + } + + if (unlikely(!pentry->callback)) { + dev_err(&pdev->dev, "Callback NULL\n"); + goto process_pentry; + } + + info = pentry->info; + if (unlikely(!info)) { + dev_err(&pdev->dev, "Pending entry post arg NULL\n"); + goto process_pentry; + } + + req = info->req; + if (unlikely(!req)) { + dev_err(&pdev->dev, "Request NULL\n"); + goto process_pentry; + } + + cpt_status = pentry->completion_addr; + if (unlikely(!cpt_status)) { + dev_err(&pdev->dev, "Completion address NULL\n"); + goto process_pentry; + } + + if (cpt_process_ccode(lfs, cpt_status, info, &res_code)) { + spin_unlock_bh(&pqueue->lock); + return; + } + info->pdev = pdev; + +process_pentry: + /* + * Check if we should inform sending side to resume + * We do it CPT_IQ_RESUME_MARGIN elements in advance before + * pending queue becomes empty + */ + resume_index = modulo_inc(pqueue->front, pqueue->qlen, + CPT_IQ_RESUME_MARGIN); + resume_pentry = &pqueue->head[resume_index]; + if (resume_pentry && + resume_pentry->resume_sender) { + resume_pentry->resume_sender = false; + callback = resume_pentry->callback; + areq = resume_pentry->areq; + + if (callback) { + spin_unlock_bh(&pqueue->lock); + + /* + * EINPROGRESS is an indication for sending + * side that it can resume sending requests + */ + callback(-EINPROGRESS, areq, info); + spin_lock_bh(&pqueue->lock); + } + } + + callback = pentry->callback; + areq = pentry->areq; + free_pentry(pentry); + + pqueue->pending_count--; + pqueue->front = modulo_inc(pqueue->front, pqueue->qlen, 1); + spin_unlock_bh(&pqueue->lock); + + /* + * Call callback after current pending entry has been + * processed, we don't do it if the callback pointer is + * invalid. + */ + if (callback) + callback(res_code, areq, info); + } +} + +void otx2_cpt_post_process(struct otx2_cptlf_wqe *wqe) +{ + process_pending_queue(wqe->lfs, + &wqe->lfs->lf[wqe->lf_num].pqueue); +} + +int otx2_cpt_get_eng_grp_num(struct pci_dev *pdev, + enum otx2_cpt_eng_type eng_type) +{ + struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev); + + switch (eng_type) { + case OTX2_CPT_SE_TYPES: + return cptvf->lfs.kcrypto_se_eng_grp_num; + case OTX2_CPT_AE_TYPES: + return cptvf->lfs.kcrypto_ae_eng_grp_num; + default: + dev_err(&cptvf->pdev->dev, "Unsupported engine type"); + break; + } + return -ENXIO; +} diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c new file mode 100644 index 000000000..c75a06999 --- /dev/null +++ b/drivers/crypto/mxs-dcp.c @@ -0,0 +1,1258 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Freescale i.MX23/i.MX28 Data Co-Processor driver + * + * Copyright (C) 2013 Marek Vasut + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#define DCP_MAX_CHANS 4 +#define DCP_BUF_SZ PAGE_SIZE +#define DCP_SHA_PAY_SZ 64 + +#define DCP_ALIGNMENT 64 + +/* + * Null hashes to align with hw behavior on imx6sl and ull + * these are flipped for consistency with hw output + */ +static const uint8_t sha1_null_hash[] = + "\x09\x07\xd8\xaf\x90\x18\x60\x95\xef\xbf" + "\x55\x32\x0d\x4b\x6b\x5e\xee\xa3\x39\xda"; + +static const uint8_t sha256_null_hash[] = + "\x55\xb8\x52\x78\x1b\x99\x95\xa4" + "\x4c\x93\x9b\x64\xe4\x41\xae\x27" + "\x24\xb9\x6f\x99\xc8\xf4\xfb\x9a" + "\x14\x1c\xfc\x98\x42\xc4\xb0\xe3"; + +/* DCP DMA descriptor. */ +struct dcp_dma_desc { + uint32_t next_cmd_addr; + uint32_t control0; + uint32_t control1; + uint32_t source; + uint32_t destination; + uint32_t size; + uint32_t payload; + uint32_t status; +}; + +/* Coherent aligned block for bounce buffering. */ +struct dcp_coherent_block { + uint8_t aes_in_buf[DCP_BUF_SZ]; + uint8_t aes_out_buf[DCP_BUF_SZ]; + uint8_t sha_in_buf[DCP_BUF_SZ]; + uint8_t sha_out_buf[DCP_SHA_PAY_SZ]; + + uint8_t aes_key[2 * AES_KEYSIZE_128]; + + struct dcp_dma_desc desc[DCP_MAX_CHANS]; +}; + +struct dcp { + struct device *dev; + void __iomem *base; + + uint32_t caps; + + struct dcp_coherent_block *coh; + + struct completion completion[DCP_MAX_CHANS]; + spinlock_t lock[DCP_MAX_CHANS]; + struct task_struct *thread[DCP_MAX_CHANS]; + struct crypto_queue queue[DCP_MAX_CHANS]; + struct clk *dcp_clk; +}; + +enum dcp_chan { + DCP_CHAN_HASH_SHA = 0, + DCP_CHAN_CRYPTO = 2, +}; + +struct dcp_async_ctx { + /* Common context */ + enum dcp_chan chan; + uint32_t fill; + + /* SHA Hash-specific context */ + struct mutex mutex; + uint32_t alg; + unsigned int hot:1; + + /* Crypto-specific context */ + struct crypto_skcipher *fallback; + unsigned int key_len; + uint8_t key[AES_KEYSIZE_128]; + bool key_referenced; +}; + +struct dcp_aes_req_ctx { + unsigned int enc:1; + unsigned int ecb:1; + struct skcipher_request fallback_req; // keep at the end +}; + +struct dcp_sha_req_ctx { + unsigned int init:1; + unsigned int fini:1; +}; + +struct dcp_export_state { + struct dcp_sha_req_ctx req_ctx; + struct dcp_async_ctx async_ctx; +}; + +/* + * There can even be only one instance of the MXS DCP due to the + * design of Linux Crypto API. + */ +static struct dcp *global_sdcp; + +/* DCP register layout. */ +#define MXS_DCP_CTRL 0x00 +#define MXS_DCP_CTRL_GATHER_RESIDUAL_WRITES (1 << 23) +#define MXS_DCP_CTRL_ENABLE_CONTEXT_CACHING (1 << 22) + +#define MXS_DCP_STAT 0x10 +#define MXS_DCP_STAT_CLR 0x18 +#define MXS_DCP_STAT_IRQ_MASK 0xf + +#define MXS_DCP_CHANNELCTRL 0x20 +#define MXS_DCP_CHANNELCTRL_ENABLE_CHANNEL_MASK 0xff + +#define MXS_DCP_CAPABILITY1 0x40 +#define MXS_DCP_CAPABILITY1_SHA256 (4 << 16) +#define MXS_DCP_CAPABILITY1_SHA1 (1 << 16) +#define MXS_DCP_CAPABILITY1_AES128 (1 << 0) + +#define MXS_DCP_CONTEXT 0x50 + +#define MXS_DCP_CH_N_CMDPTR(n) (0x100 + ((n) * 0x40)) + +#define MXS_DCP_CH_N_SEMA(n) (0x110 + ((n) * 0x40)) + +#define MXS_DCP_CH_N_STAT(n) (0x120 + ((n) * 0x40)) +#define MXS_DCP_CH_N_STAT_CLR(n) (0x128 + ((n) * 0x40)) + +/* DMA descriptor bits. */ +#define MXS_DCP_CONTROL0_HASH_TERM (1 << 13) +#define MXS_DCP_CONTROL0_HASH_INIT (1 << 12) +#define MXS_DCP_CONTROL0_PAYLOAD_KEY (1 << 11) +#define MXS_DCP_CONTROL0_OTP_KEY (1 << 10) +#define MXS_DCP_CONTROL0_CIPHER_ENCRYPT (1 << 8) +#define MXS_DCP_CONTROL0_CIPHER_INIT (1 << 9) +#define MXS_DCP_CONTROL0_ENABLE_HASH (1 << 6) +#define MXS_DCP_CONTROL0_ENABLE_CIPHER (1 << 5) +#define MXS_DCP_CONTROL0_DECR_SEMAPHORE (1 << 1) +#define MXS_DCP_CONTROL0_INTERRUPT (1 << 0) + +#define MXS_DCP_CONTROL1_HASH_SELECT_SHA256 (2 << 16) +#define MXS_DCP_CONTROL1_HASH_SELECT_SHA1 (0 << 16) +#define MXS_DCP_CONTROL1_CIPHER_MODE_CBC (1 << 4) +#define MXS_DCP_CONTROL1_CIPHER_MODE_ECB (0 << 4) +#define MXS_DCP_CONTROL1_CIPHER_SELECT_AES128 (0 << 0) + +#define MXS_DCP_CONTROL1_KEY_SELECT_SHIFT 8 + +static int mxs_dcp_start_dma(struct dcp_async_ctx *actx) +{ + int dma_err; + struct dcp *sdcp = global_sdcp; + const int chan = actx->chan; + uint32_t stat; + unsigned long ret; + struct dcp_dma_desc *desc = &sdcp->coh->desc[actx->chan]; + dma_addr_t desc_phys = dma_map_single(sdcp->dev, desc, sizeof(*desc), + DMA_TO_DEVICE); + + dma_err = dma_mapping_error(sdcp->dev, desc_phys); + if (dma_err) + return dma_err; + + reinit_completion(&sdcp->completion[chan]); + + /* Clear status register. */ + writel(0xffffffff, sdcp->base + MXS_DCP_CH_N_STAT_CLR(chan)); + + /* Load the DMA descriptor. */ + writel(desc_phys, sdcp->base + MXS_DCP_CH_N_CMDPTR(chan)); + + /* Increment the semaphore to start the DMA transfer. */ + writel(1, sdcp->base + MXS_DCP_CH_N_SEMA(chan)); + + ret = wait_for_completion_timeout(&sdcp->completion[chan], + msecs_to_jiffies(1000)); + if (!ret) { + dev_err(sdcp->dev, "Channel %i timeout (DCP_STAT=0x%08x)\n", + chan, readl(sdcp->base + MXS_DCP_STAT)); + return -ETIMEDOUT; + } + + stat = readl(sdcp->base + MXS_DCP_CH_N_STAT(chan)); + if (stat & 0xff) { + dev_err(sdcp->dev, "Channel %i error (CH_STAT=0x%08x)\n", + chan, stat); + return -EINVAL; + } + + dma_unmap_single(sdcp->dev, desc_phys, sizeof(*desc), DMA_TO_DEVICE); + + return 0; +} + +/* + * Encryption (AES128) + */ +static int mxs_dcp_run_aes(struct dcp_async_ctx *actx, + struct skcipher_request *req, int init) +{ + dma_addr_t key_phys, src_phys, dst_phys; + struct dcp *sdcp = global_sdcp; + struct dcp_dma_desc *desc = &sdcp->coh->desc[actx->chan]; + struct dcp_aes_req_ctx *rctx = skcipher_request_ctx(req); + bool key_referenced = actx->key_referenced; + int ret; + + if (key_referenced) + key_phys = dma_map_single(sdcp->dev, sdcp->coh->aes_key + AES_KEYSIZE_128, + AES_KEYSIZE_128, DMA_TO_DEVICE); + else + key_phys = dma_map_single(sdcp->dev, sdcp->coh->aes_key, + 2 * AES_KEYSIZE_128, DMA_TO_DEVICE); + ret = dma_mapping_error(sdcp->dev, key_phys); + if (ret) + return ret; + + src_phys = dma_map_single(sdcp->dev, sdcp->coh->aes_in_buf, + DCP_BUF_SZ, DMA_TO_DEVICE); + ret = dma_mapping_error(sdcp->dev, src_phys); + if (ret) + goto err_src; + + dst_phys = dma_map_single(sdcp->dev, sdcp->coh->aes_out_buf, + DCP_BUF_SZ, DMA_FROM_DEVICE); + ret = dma_mapping_error(sdcp->dev, dst_phys); + if (ret) + goto err_dst; + + if (actx->fill % AES_BLOCK_SIZE) { + dev_err(sdcp->dev, "Invalid block size!\n"); + ret = -EINVAL; + goto aes_done_run; + } + + /* Fill in the DMA descriptor. */ + desc->control0 = MXS_DCP_CONTROL0_DECR_SEMAPHORE | + MXS_DCP_CONTROL0_INTERRUPT | + MXS_DCP_CONTROL0_ENABLE_CIPHER; + + if (!key_referenced) + /* Payload contains the key. */ + desc->control0 |= MXS_DCP_CONTROL0_PAYLOAD_KEY; + else if (actx->key[0] == DCP_PAES_KEY_OTP) + /* Set OTP key bit to select the key via KEY_SELECT. */ + desc->control0 |= MXS_DCP_CONTROL0_OTP_KEY; + + if (rctx->enc) + desc->control0 |= MXS_DCP_CONTROL0_CIPHER_ENCRYPT; + if (init) + desc->control0 |= MXS_DCP_CONTROL0_CIPHER_INIT; + + desc->control1 = MXS_DCP_CONTROL1_CIPHER_SELECT_AES128; + + if (rctx->ecb) + desc->control1 |= MXS_DCP_CONTROL1_CIPHER_MODE_ECB; + else + desc->control1 |= MXS_DCP_CONTROL1_CIPHER_MODE_CBC; + + if (key_referenced) + desc->control1 |= sdcp->coh->aes_key[0] << MXS_DCP_CONTROL1_KEY_SELECT_SHIFT; + + desc->next_cmd_addr = 0; + desc->source = src_phys; + desc->destination = dst_phys; + desc->size = actx->fill; + desc->payload = key_phys; + desc->status = 0; + + ret = mxs_dcp_start_dma(actx); + +aes_done_run: + dma_unmap_single(sdcp->dev, dst_phys, DCP_BUF_SZ, DMA_FROM_DEVICE); +err_dst: + dma_unmap_single(sdcp->dev, src_phys, DCP_BUF_SZ, DMA_TO_DEVICE); +err_src: + if (key_referenced) + dma_unmap_single(sdcp->dev, key_phys, AES_KEYSIZE_128, + DMA_TO_DEVICE); + else + dma_unmap_single(sdcp->dev, key_phys, 2 * AES_KEYSIZE_128, + DMA_TO_DEVICE); + return ret; +} + +static int mxs_dcp_aes_block_crypt(struct crypto_async_request *arq) +{ + struct dcp *sdcp = global_sdcp; + + struct skcipher_request *req = skcipher_request_cast(arq); + struct dcp_async_ctx *actx = crypto_tfm_ctx(arq->tfm); + struct dcp_aes_req_ctx *rctx = skcipher_request_ctx(req); + + struct scatterlist *dst = req->dst; + struct scatterlist *src = req->src; + int dst_nents = sg_nents(dst); + + const int out_off = DCP_BUF_SZ; + uint8_t *in_buf = sdcp->coh->aes_in_buf; + uint8_t *out_buf = sdcp->coh->aes_out_buf; + + uint32_t dst_off = 0; + uint8_t *src_buf = NULL; + uint32_t last_out_len = 0; + + uint8_t *key = sdcp->coh->aes_key; + + int ret = 0; + unsigned int i, len, clen, tlen = 0; + int init = 0; + bool limit_hit = false; + + actx->fill = 0; + + /* Copy the key from the temporary location. */ + memcpy(key, actx->key, actx->key_len); + + if (!rctx->ecb) { + /* Copy the CBC IV just past the key. */ + memcpy(key + AES_KEYSIZE_128, req->iv, AES_KEYSIZE_128); + /* CBC needs the INIT set. */ + init = 1; + } else { + memset(key + AES_KEYSIZE_128, 0, AES_KEYSIZE_128); + } + + for_each_sg(req->src, src, sg_nents(req->src), i) { + src_buf = sg_virt(src); + len = src->length; + tlen += len; + limit_hit = tlen > req->cryptlen; + + if (limit_hit) + len = req->cryptlen - (tlen - len); + + do { + if (actx->fill + len > out_off) + clen = out_off - actx->fill; + else + clen = len; + + memcpy(in_buf + actx->fill, src_buf, clen); + len -= clen; + src_buf += clen; + actx->fill += clen; + + /* + * If we filled the buffer or this is the last SG, + * submit the buffer. + */ + if (actx->fill == out_off || sg_is_last(src) || + limit_hit) { + ret = mxs_dcp_run_aes(actx, req, init); + if (ret) + return ret; + init = 0; + + sg_pcopy_from_buffer(dst, dst_nents, out_buf, + actx->fill, dst_off); + dst_off += actx->fill; + last_out_len = actx->fill; + actx->fill = 0; + } + } while (len); + + if (limit_hit) + break; + } + + /* Copy the IV for CBC for chaining */ + if (!rctx->ecb) { + if (rctx->enc) + memcpy(req->iv, out_buf+(last_out_len-AES_BLOCK_SIZE), + AES_BLOCK_SIZE); + else + memcpy(req->iv, in_buf+(last_out_len-AES_BLOCK_SIZE), + AES_BLOCK_SIZE); + } + + return ret; +} + +static int dcp_chan_thread_aes(void *data) +{ + struct dcp *sdcp = global_sdcp; + const int chan = DCP_CHAN_CRYPTO; + + struct crypto_async_request *backlog; + struct crypto_async_request *arq; + + int ret; + + while (!kthread_should_stop()) { + set_current_state(TASK_INTERRUPTIBLE); + + spin_lock(&sdcp->lock[chan]); + backlog = crypto_get_backlog(&sdcp->queue[chan]); + arq = crypto_dequeue_request(&sdcp->queue[chan]); + spin_unlock(&sdcp->lock[chan]); + + if (!backlog && !arq) { + schedule(); + continue; + } + + set_current_state(TASK_RUNNING); + + if (backlog) + crypto_request_complete(backlog, -EINPROGRESS); + + if (arq) { + ret = mxs_dcp_aes_block_crypt(arq); + crypto_request_complete(arq, ret); + } + } + + return 0; +} + +static int mxs_dcp_block_fallback(struct skcipher_request *req, int enc) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct dcp_aes_req_ctx *rctx = skcipher_request_ctx(req); + struct dcp_async_ctx *ctx = crypto_skcipher_ctx(tfm); + int ret; + + skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback); + skcipher_request_set_callback(&rctx->fallback_req, req->base.flags, + req->base.complete, req->base.data); + skcipher_request_set_crypt(&rctx->fallback_req, req->src, req->dst, + req->cryptlen, req->iv); + + if (enc) + ret = crypto_skcipher_encrypt(&rctx->fallback_req); + else + ret = crypto_skcipher_decrypt(&rctx->fallback_req); + + return ret; +} + +static int mxs_dcp_aes_enqueue(struct skcipher_request *req, int enc, int ecb) +{ + struct dcp *sdcp = global_sdcp; + struct crypto_async_request *arq = &req->base; + struct dcp_async_ctx *actx = crypto_tfm_ctx(arq->tfm); + struct dcp_aes_req_ctx *rctx = skcipher_request_ctx(req); + int ret; + + if (unlikely(actx->key_len != AES_KEYSIZE_128 && !actx->key_referenced)) + return mxs_dcp_block_fallback(req, enc); + + rctx->enc = enc; + rctx->ecb = ecb; + actx->chan = DCP_CHAN_CRYPTO; + + spin_lock(&sdcp->lock[actx->chan]); + ret = crypto_enqueue_request(&sdcp->queue[actx->chan], &req->base); + spin_unlock(&sdcp->lock[actx->chan]); + + wake_up_process(sdcp->thread[actx->chan]); + + return ret; +} + +static int mxs_dcp_aes_ecb_decrypt(struct skcipher_request *req) +{ + return mxs_dcp_aes_enqueue(req, 0, 1); +} + +static int mxs_dcp_aes_ecb_encrypt(struct skcipher_request *req) +{ + return mxs_dcp_aes_enqueue(req, 1, 1); +} + +static int mxs_dcp_aes_cbc_decrypt(struct skcipher_request *req) +{ + return mxs_dcp_aes_enqueue(req, 0, 0); +} + +static int mxs_dcp_aes_cbc_encrypt(struct skcipher_request *req) +{ + return mxs_dcp_aes_enqueue(req, 1, 0); +} + +static int mxs_dcp_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int len) +{ + struct dcp_async_ctx *actx = crypto_skcipher_ctx(tfm); + + /* + * AES 128 is supposed by the hardware, store key into temporary + * buffer and exit. We must use the temporary buffer here, since + * there can still be an operation in progress. + */ + actx->key_len = len; + actx->key_referenced = false; + if (len == AES_KEYSIZE_128) { + memcpy(actx->key, key, len); + return 0; + } + + /* + * If the requested AES key size is not supported by the hardware, + * but is supported by in-kernel software implementation, we use + * software fallback. + */ + crypto_skcipher_clear_flags(actx->fallback, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(actx->fallback, + tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); + return crypto_skcipher_setkey(actx->fallback, key, len); +} + +static int mxs_dcp_aes_setrefkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int len) +{ + struct dcp_async_ctx *actx = crypto_skcipher_ctx(tfm); + + if (len != DCP_PAES_KEYSIZE) + return -EINVAL; + + switch (key[0]) { + case DCP_PAES_KEY_SLOT0: + case DCP_PAES_KEY_SLOT1: + case DCP_PAES_KEY_SLOT2: + case DCP_PAES_KEY_SLOT3: + case DCP_PAES_KEY_UNIQUE: + case DCP_PAES_KEY_OTP: + memcpy(actx->key, key, len); + actx->key_len = len; + actx->key_referenced = true; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int mxs_dcp_aes_fallback_init_tfm(struct crypto_skcipher *tfm) +{ + const char *name = crypto_tfm_alg_name(crypto_skcipher_tfm(tfm)); + struct dcp_async_ctx *actx = crypto_skcipher_ctx(tfm); + struct crypto_skcipher *blk; + + blk = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(blk)) + return PTR_ERR(blk); + + actx->fallback = blk; + crypto_skcipher_set_reqsize(tfm, sizeof(struct dcp_aes_req_ctx) + + crypto_skcipher_reqsize(blk)); + return 0; +} + +static void mxs_dcp_aes_fallback_exit_tfm(struct crypto_skcipher *tfm) +{ + struct dcp_async_ctx *actx = crypto_skcipher_ctx(tfm); + + crypto_free_skcipher(actx->fallback); +} + +static int mxs_dcp_paes_init_tfm(struct crypto_skcipher *tfm) +{ + crypto_skcipher_set_reqsize(tfm, sizeof(struct dcp_aes_req_ctx)); + + return 0; +} + +/* + * Hashing (SHA1/SHA256) + */ +static int mxs_dcp_run_sha(struct ahash_request *req) +{ + struct dcp *sdcp = global_sdcp; + int ret; + + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm); + struct dcp_sha_req_ctx *rctx = ahash_request_ctx(req); + struct dcp_dma_desc *desc = &sdcp->coh->desc[actx->chan]; + + dma_addr_t digest_phys = 0; + dma_addr_t buf_phys = dma_map_single(sdcp->dev, sdcp->coh->sha_in_buf, + DCP_BUF_SZ, DMA_TO_DEVICE); + + ret = dma_mapping_error(sdcp->dev, buf_phys); + if (ret) + return ret; + + /* Fill in the DMA descriptor. */ + desc->control0 = MXS_DCP_CONTROL0_DECR_SEMAPHORE | + MXS_DCP_CONTROL0_INTERRUPT | + MXS_DCP_CONTROL0_ENABLE_HASH; + if (rctx->init) + desc->control0 |= MXS_DCP_CONTROL0_HASH_INIT; + + desc->control1 = actx->alg; + desc->next_cmd_addr = 0; + desc->source = buf_phys; + desc->destination = 0; + desc->size = actx->fill; + desc->payload = 0; + desc->status = 0; + + /* + * Align driver with hw behavior when generating null hashes + */ + if (rctx->init && rctx->fini && desc->size == 0) { + struct hash_alg_common *halg = crypto_hash_alg_common(tfm); + const uint8_t *sha_buf = + (actx->alg == MXS_DCP_CONTROL1_HASH_SELECT_SHA1) ? + sha1_null_hash : sha256_null_hash; + memcpy(sdcp->coh->sha_out_buf, sha_buf, halg->digestsize); + ret = 0; + goto done_run; + } + + /* Set HASH_TERM bit for last transfer block. */ + if (rctx->fini) { + digest_phys = dma_map_single(sdcp->dev, sdcp->coh->sha_out_buf, + DCP_SHA_PAY_SZ, DMA_FROM_DEVICE); + ret = dma_mapping_error(sdcp->dev, digest_phys); + if (ret) + goto done_run; + + desc->control0 |= MXS_DCP_CONTROL0_HASH_TERM; + desc->payload = digest_phys; + } + + ret = mxs_dcp_start_dma(actx); + + if (rctx->fini) + dma_unmap_single(sdcp->dev, digest_phys, DCP_SHA_PAY_SZ, + DMA_FROM_DEVICE); + +done_run: + dma_unmap_single(sdcp->dev, buf_phys, DCP_BUF_SZ, DMA_TO_DEVICE); + + return ret; +} + +static int dcp_sha_req_to_buf(struct crypto_async_request *arq) +{ + struct dcp *sdcp = global_sdcp; + + struct ahash_request *req = ahash_request_cast(arq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm); + struct dcp_sha_req_ctx *rctx = ahash_request_ctx(req); + struct hash_alg_common *halg = crypto_hash_alg_common(tfm); + + uint8_t *in_buf = sdcp->coh->sha_in_buf; + uint8_t *out_buf = sdcp->coh->sha_out_buf; + + struct scatterlist *src; + + unsigned int i, len, clen, oft = 0; + int ret; + + int fin = rctx->fini; + if (fin) + rctx->fini = 0; + + src = req->src; + len = req->nbytes; + + while (len) { + if (actx->fill + len > DCP_BUF_SZ) + clen = DCP_BUF_SZ - actx->fill; + else + clen = len; + + scatterwalk_map_and_copy(in_buf + actx->fill, src, oft, clen, + 0); + + len -= clen; + oft += clen; + actx->fill += clen; + + /* + * If we filled the buffer and still have some + * more data, submit the buffer. + */ + if (len && actx->fill == DCP_BUF_SZ) { + ret = mxs_dcp_run_sha(req); + if (ret) + return ret; + actx->fill = 0; + rctx->init = 0; + } + } + + if (fin) { + rctx->fini = 1; + + /* Submit whatever is left. */ + if (!req->result) + return -EINVAL; + + ret = mxs_dcp_run_sha(req); + if (ret) + return ret; + + actx->fill = 0; + + /* For some reason the result is flipped */ + for (i = 0; i < halg->digestsize; i++) + req->result[i] = out_buf[halg->digestsize - i - 1]; + } + + return 0; +} + +static int dcp_chan_thread_sha(void *data) +{ + struct dcp *sdcp = global_sdcp; + const int chan = DCP_CHAN_HASH_SHA; + + struct crypto_async_request *backlog; + struct crypto_async_request *arq; + int ret; + + while (!kthread_should_stop()) { + set_current_state(TASK_INTERRUPTIBLE); + + spin_lock(&sdcp->lock[chan]); + backlog = crypto_get_backlog(&sdcp->queue[chan]); + arq = crypto_dequeue_request(&sdcp->queue[chan]); + spin_unlock(&sdcp->lock[chan]); + + if (!backlog && !arq) { + schedule(); + continue; + } + + set_current_state(TASK_RUNNING); + + if (backlog) + crypto_request_complete(backlog, -EINPROGRESS); + + if (arq) { + ret = dcp_sha_req_to_buf(arq); + crypto_request_complete(arq, ret); + } + } + + return 0; +} + +static int dcp_sha_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm); + + struct hash_alg_common *halg = crypto_hash_alg_common(tfm); + + /* + * Start hashing session. The code below only inits the + * hashing session context, nothing more. + */ + memset(actx, 0, sizeof(*actx)); + + if (strcmp(halg->base.cra_name, "sha1") == 0) + actx->alg = MXS_DCP_CONTROL1_HASH_SELECT_SHA1; + else + actx->alg = MXS_DCP_CONTROL1_HASH_SELECT_SHA256; + + actx->fill = 0; + actx->hot = 0; + actx->chan = DCP_CHAN_HASH_SHA; + + mutex_init(&actx->mutex); + + return 0; +} + +static int dcp_sha_update_fx(struct ahash_request *req, int fini) +{ + struct dcp *sdcp = global_sdcp; + + struct dcp_sha_req_ctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm); + + int ret; + + /* + * Ignore requests that have no data in them and are not + * the trailing requests in the stream of requests. + */ + if (!req->nbytes && !fini) + return 0; + + mutex_lock(&actx->mutex); + + rctx->fini = fini; + + if (!actx->hot) { + actx->hot = 1; + rctx->init = 1; + } + + spin_lock(&sdcp->lock[actx->chan]); + ret = crypto_enqueue_request(&sdcp->queue[actx->chan], &req->base); + spin_unlock(&sdcp->lock[actx->chan]); + + wake_up_process(sdcp->thread[actx->chan]); + mutex_unlock(&actx->mutex); + + return ret; +} + +static int dcp_sha_update(struct ahash_request *req) +{ + return dcp_sha_update_fx(req, 0); +} + +static int dcp_sha_final(struct ahash_request *req) +{ + ahash_request_set_crypt(req, NULL, req->result, 0); + req->nbytes = 0; + return dcp_sha_update_fx(req, 1); +} + +static int dcp_sha_finup(struct ahash_request *req) +{ + return dcp_sha_update_fx(req, 1); +} + +static int dcp_sha_digest(struct ahash_request *req) +{ + int ret; + + ret = dcp_sha_init(req); + if (ret) + return ret; + + return dcp_sha_finup(req); +} + +static int dcp_sha_import(struct ahash_request *req, const void *in) +{ + struct dcp_sha_req_ctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm); + const struct dcp_export_state *export = in; + + memset(rctx, 0, sizeof(struct dcp_sha_req_ctx)); + memset(actx, 0, sizeof(struct dcp_async_ctx)); + memcpy(rctx, &export->req_ctx, sizeof(struct dcp_sha_req_ctx)); + memcpy(actx, &export->async_ctx, sizeof(struct dcp_async_ctx)); + + return 0; +} + +static int dcp_sha_export(struct ahash_request *req, void *out) +{ + struct dcp_sha_req_ctx *rctx_state = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct dcp_async_ctx *actx_state = crypto_ahash_ctx(tfm); + struct dcp_export_state *export = out; + + memcpy(&export->req_ctx, rctx_state, sizeof(struct dcp_sha_req_ctx)); + memcpy(&export->async_ctx, actx_state, sizeof(struct dcp_async_ctx)); + + return 0; +} + +static int dcp_sha_cra_init(struct crypto_tfm *tfm) +{ + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct dcp_sha_req_ctx)); + return 0; +} + +static void dcp_sha_cra_exit(struct crypto_tfm *tfm) +{ +} + +/* AES 128 ECB and AES 128 CBC */ +static struct skcipher_alg dcp_aes_algs[] = { + { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-dcp", + .base.cra_priority = 400, + .base.cra_alignmask = 15, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct dcp_async_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = mxs_dcp_aes_setkey, + .encrypt = mxs_dcp_aes_ecb_encrypt, + .decrypt = mxs_dcp_aes_ecb_decrypt, + .init = mxs_dcp_aes_fallback_init_tfm, + .exit = mxs_dcp_aes_fallback_exit_tfm, + }, { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-dcp", + .base.cra_priority = 400, + .base.cra_alignmask = 15, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct dcp_async_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = mxs_dcp_aes_setkey, + .encrypt = mxs_dcp_aes_cbc_encrypt, + .decrypt = mxs_dcp_aes_cbc_decrypt, + .ivsize = AES_BLOCK_SIZE, + .init = mxs_dcp_aes_fallback_init_tfm, + .exit = mxs_dcp_aes_fallback_exit_tfm, + }, { + .base.cra_name = "ecb(paes)", + .base.cra_driver_name = "ecb-paes-dcp", + .base.cra_priority = 401, + .base.cra_alignmask = 15, + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_INTERNAL, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct dcp_async_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = DCP_PAES_KEYSIZE, + .max_keysize = DCP_PAES_KEYSIZE, + .setkey = mxs_dcp_aes_setrefkey, + .encrypt = mxs_dcp_aes_ecb_encrypt, + .decrypt = mxs_dcp_aes_ecb_decrypt, + .init = mxs_dcp_paes_init_tfm, + }, { + .base.cra_name = "cbc(paes)", + .base.cra_driver_name = "cbc-paes-dcp", + .base.cra_priority = 401, + .base.cra_alignmask = 15, + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_INTERNAL, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct dcp_async_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = DCP_PAES_KEYSIZE, + .max_keysize = DCP_PAES_KEYSIZE, + .setkey = mxs_dcp_aes_setrefkey, + .encrypt = mxs_dcp_aes_cbc_encrypt, + .decrypt = mxs_dcp_aes_cbc_decrypt, + .ivsize = AES_BLOCK_SIZE, + .init = mxs_dcp_paes_init_tfm, + }, +}; + +/* SHA1 */ +static struct ahash_alg dcp_sha1_alg = { + .init = dcp_sha_init, + .update = dcp_sha_update, + .final = dcp_sha_final, + .finup = dcp_sha_finup, + .digest = dcp_sha_digest, + .import = dcp_sha_import, + .export = dcp_sha_export, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct dcp_export_state), + .base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-dcp", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct dcp_async_ctx), + .cra_module = THIS_MODULE, + .cra_init = dcp_sha_cra_init, + .cra_exit = dcp_sha_cra_exit, + }, + }, +}; + +/* SHA256 */ +static struct ahash_alg dcp_sha256_alg = { + .init = dcp_sha_init, + .update = dcp_sha_update, + .final = dcp_sha_final, + .finup = dcp_sha_finup, + .digest = dcp_sha_digest, + .import = dcp_sha_import, + .export = dcp_sha_export, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct dcp_export_state), + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-dcp", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct dcp_async_ctx), + .cra_module = THIS_MODULE, + .cra_init = dcp_sha_cra_init, + .cra_exit = dcp_sha_cra_exit, + }, + }, +}; + +static irqreturn_t mxs_dcp_irq(int irq, void *context) +{ + struct dcp *sdcp = context; + uint32_t stat; + int i; + + stat = readl(sdcp->base + MXS_DCP_STAT); + stat &= MXS_DCP_STAT_IRQ_MASK; + if (!stat) + return IRQ_NONE; + + /* Clear the interrupts. */ + writel(stat, sdcp->base + MXS_DCP_STAT_CLR); + + /* Complete the DMA requests that finished. */ + for (i = 0; i < DCP_MAX_CHANS; i++) + if (stat & (1 << i)) + complete(&sdcp->completion[i]); + + return IRQ_HANDLED; +} + +static int mxs_dcp_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dcp *sdcp = NULL; + int i, ret; + int dcp_vmi_irq, dcp_irq; + + if (global_sdcp) { + dev_err(dev, "Only one DCP instance allowed!\n"); + return -ENODEV; + } + + dcp_vmi_irq = platform_get_irq(pdev, 0); + if (dcp_vmi_irq < 0) + return dcp_vmi_irq; + + dcp_irq = platform_get_irq(pdev, 1); + if (dcp_irq < 0) + return dcp_irq; + + sdcp = devm_kzalloc(dev, sizeof(*sdcp), GFP_KERNEL); + if (!sdcp) + return -ENOMEM; + + sdcp->dev = dev; + sdcp->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(sdcp->base)) + return PTR_ERR(sdcp->base); + + + ret = devm_request_irq(dev, dcp_vmi_irq, mxs_dcp_irq, 0, + "dcp-vmi-irq", sdcp); + if (ret) + return ret; + + ret = devm_request_irq(dev, dcp_irq, mxs_dcp_irq, 0, + "dcp-irq", sdcp); + if (ret) + return ret; + + /* Allocate coherent helper block. */ + sdcp->coh = devm_kzalloc(dev, sizeof(*sdcp->coh) + DCP_ALIGNMENT, + GFP_KERNEL); + if (!sdcp->coh) + return -ENOMEM; + + /* Re-align the structure so it fits the DCP constraints. */ + sdcp->coh = PTR_ALIGN(sdcp->coh, DCP_ALIGNMENT); + + /* DCP clock is optional, only used on some SOCs */ + sdcp->dcp_clk = devm_clk_get_optional_enabled(dev, "dcp"); + if (IS_ERR(sdcp->dcp_clk)) + return PTR_ERR(sdcp->dcp_clk); + + /* Restart the DCP block. */ + ret = stmp_reset_block(sdcp->base); + if (ret) { + dev_err(dev, "Failed reset\n"); + return ret; + } + + /* Initialize control register. */ + writel(MXS_DCP_CTRL_GATHER_RESIDUAL_WRITES | + MXS_DCP_CTRL_ENABLE_CONTEXT_CACHING | 0xf, + sdcp->base + MXS_DCP_CTRL); + + /* Enable all DCP DMA channels. */ + writel(MXS_DCP_CHANNELCTRL_ENABLE_CHANNEL_MASK, + sdcp->base + MXS_DCP_CHANNELCTRL); + + /* + * We do not enable context switching. Give the context buffer a + * pointer to an illegal address so if context switching is + * inadvertantly enabled, the DCP will return an error instead of + * trashing good memory. The DCP DMA cannot access ROM, so any ROM + * address will do. + */ + writel(0xffff0000, sdcp->base + MXS_DCP_CONTEXT); + for (i = 0; i < DCP_MAX_CHANS; i++) + writel(0xffffffff, sdcp->base + MXS_DCP_CH_N_STAT_CLR(i)); + writel(0xffffffff, sdcp->base + MXS_DCP_STAT_CLR); + + global_sdcp = sdcp; + + platform_set_drvdata(pdev, sdcp); + + for (i = 0; i < DCP_MAX_CHANS; i++) { + spin_lock_init(&sdcp->lock[i]); + init_completion(&sdcp->completion[i]); + crypto_init_queue(&sdcp->queue[i], 50); + } + + /* Create the SHA and AES handler threads. */ + sdcp->thread[DCP_CHAN_HASH_SHA] = kthread_run(dcp_chan_thread_sha, + NULL, "mxs_dcp_chan/sha"); + if (IS_ERR(sdcp->thread[DCP_CHAN_HASH_SHA])) { + dev_err(dev, "Error starting SHA thread!\n"); + ret = PTR_ERR(sdcp->thread[DCP_CHAN_HASH_SHA]); + return ret; + } + + sdcp->thread[DCP_CHAN_CRYPTO] = kthread_run(dcp_chan_thread_aes, + NULL, "mxs_dcp_chan/aes"); + if (IS_ERR(sdcp->thread[DCP_CHAN_CRYPTO])) { + dev_err(dev, "Error starting SHA thread!\n"); + ret = PTR_ERR(sdcp->thread[DCP_CHAN_CRYPTO]); + goto err_destroy_sha_thread; + } + + /* Register the various crypto algorithms. */ + sdcp->caps = readl(sdcp->base + MXS_DCP_CAPABILITY1); + + if (sdcp->caps & MXS_DCP_CAPABILITY1_AES128) { + ret = crypto_register_skciphers(dcp_aes_algs, + ARRAY_SIZE(dcp_aes_algs)); + if (ret) { + /* Failed to register algorithm. */ + dev_err(dev, "Failed to register AES crypto!\n"); + goto err_destroy_aes_thread; + } + } + + if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA1) { + ret = crypto_register_ahash(&dcp_sha1_alg); + if (ret) { + dev_err(dev, "Failed to register %s hash!\n", + dcp_sha1_alg.halg.base.cra_name); + goto err_unregister_aes; + } + } + + if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA256) { + ret = crypto_register_ahash(&dcp_sha256_alg); + if (ret) { + dev_err(dev, "Failed to register %s hash!\n", + dcp_sha256_alg.halg.base.cra_name); + goto err_unregister_sha1; + } + } + + return 0; + +err_unregister_sha1: + if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA1) + crypto_unregister_ahash(&dcp_sha1_alg); + +err_unregister_aes: + if (sdcp->caps & MXS_DCP_CAPABILITY1_AES128) + crypto_unregister_skciphers(dcp_aes_algs, ARRAY_SIZE(dcp_aes_algs)); + +err_destroy_aes_thread: + kthread_stop(sdcp->thread[DCP_CHAN_CRYPTO]); + +err_destroy_sha_thread: + kthread_stop(sdcp->thread[DCP_CHAN_HASH_SHA]); + + return ret; +} + +static void mxs_dcp_remove(struct platform_device *pdev) +{ + struct dcp *sdcp = platform_get_drvdata(pdev); + + if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA256) + crypto_unregister_ahash(&dcp_sha256_alg); + + if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA1) + crypto_unregister_ahash(&dcp_sha1_alg); + + if (sdcp->caps & MXS_DCP_CAPABILITY1_AES128) + crypto_unregister_skciphers(dcp_aes_algs, ARRAY_SIZE(dcp_aes_algs)); + + kthread_stop(sdcp->thread[DCP_CHAN_HASH_SHA]); + kthread_stop(sdcp->thread[DCP_CHAN_CRYPTO]); + + platform_set_drvdata(pdev, NULL); + + global_sdcp = NULL; +} + +static const struct of_device_id mxs_dcp_dt_ids[] = { + { .compatible = "fsl,imx23-dcp", .data = NULL, }, + { .compatible = "fsl,imx28-dcp", .data = NULL, }, + { /* sentinel */ } +}; + +MODULE_DEVICE_TABLE(of, mxs_dcp_dt_ids); + +static struct platform_driver mxs_dcp_driver = { + .probe = mxs_dcp_probe, + .remove = mxs_dcp_remove, + .driver = { + .name = "mxs-dcp", + .of_match_table = mxs_dcp_dt_ids, + }, +}; + +module_platform_driver(mxs_dcp_driver); + +MODULE_AUTHOR("Marek Vasut "); +MODULE_DESCRIPTION("Freescale MXS DCP Driver"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:mxs-dcp"); diff --git a/drivers/crypto/nx/Kconfig b/drivers/crypto/nx/Kconfig new file mode 100644 index 000000000..2a35e0e78 --- /dev/null +++ b/drivers/crypto/nx/Kconfig @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config CRYPTO_DEV_NX_ENCRYPT + tristate "Encryption acceleration support on pSeries platform" + depends on PPC_PSERIES && IBMVIO && !CPU_LITTLE_ENDIAN + default y + select CRYPTO_AES + select CRYPTO_CCM + help + Support for PowerPC Nest (NX) encryption acceleration. This + module supports acceleration for AES and SHA2 algorithms on + the pSeries platform. If you choose 'M' here, this module + will be called nx_crypto. + +config CRYPTO_DEV_NX_COMPRESS + tristate "Compression acceleration support" + default y + select CRYPTO_ALGAPI + select 842_DECOMPRESS + help + Support for PowerPC Nest (NX) compression acceleration. This + module supports acceleration for compressing memory with the 842 + algorithm using the cryptographic API. One of the platform + drivers must be selected also. If you choose 'M' here, this + module will be called nx_compress. + +if CRYPTO_DEV_NX_COMPRESS + +config CRYPTO_DEV_NX_COMPRESS_PSERIES + tristate "Compression acceleration support on pSeries platform" + depends on PPC_PSERIES && IBMVIO + depends on PPC_VAS + default y + help + Support for PowerPC Nest (NX) compression acceleration. This + module supports acceleration for compressing memory with the 842 + algorithm. This supports NX hardware on the pSeries platform. + If you choose 'M' here, this module will be called nx_compress_pseries. + +config CRYPTO_DEV_NX_COMPRESS_POWERNV + tristate "Compression acceleration support on PowerNV platform" + depends on PPC_POWERNV + depends on PPC_VAS + default y + help + Support for PowerPC Nest (NX) compression acceleration. This + module supports acceleration for compressing memory with the 842 + algorithm. This supports NX hardware on the PowerNV platform. + If you choose 'M' here, this module will be called nx_compress_powernv. + +endif diff --git a/drivers/crypto/nx/Makefile b/drivers/crypto/nx/Makefile new file mode 100644 index 000000000..483cef62a --- /dev/null +++ b/drivers/crypto/nx/Makefile @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_CRYPTO_DEV_NX_ENCRYPT) += nx-crypto.o +nx-crypto-objs := nx.o \ + nx-aes-cbc.o \ + nx-aes-ecb.o \ + nx-aes-gcm.o \ + nx-aes-ccm.o \ + nx-aes-ctr.o \ + nx-aes-xcbc.o \ + nx-sha256.o \ + nx-sha512.o + +nx-crypto-$(CONFIG_DEBUG_FS) += nx_debugfs.o +obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_PSERIES) += nx-compress-pseries.o nx-compress.o +obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_POWERNV) += nx-compress-powernv.o nx-compress.o +nx-compress-objs := nx-842.o +nx-compress-pseries-objs := nx-common-pseries.o +nx-compress-powernv-objs := nx-common-powernv.o diff --git a/drivers/crypto/nx/nx-842.c b/drivers/crypto/nx/nx-842.c new file mode 100644 index 000000000..a61208cbc --- /dev/null +++ b/drivers/crypto/nx/nx-842.c @@ -0,0 +1,526 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Cryptographic API for the NX-842 hardware compression. + * + * Copyright (C) IBM Corporation, 2011-2015 + * + * Designer of the Power data compression engine: + * Bulent Abali + * + * Original Authors: Robert Jennings + * Seth Jennings + * + * Rewrite: Dan Streetman + * + * This is an interface to the NX-842 compression hardware in PowerPC + * processors. Most of the complexity of this drvier is due to the fact that + * the NX-842 compression hardware requires the input and output data buffers + * to be specifically aligned, to be a specific multiple in length, and within + * specific minimum and maximum lengths. Those restrictions, provided by the + * nx-842 driver via nx842_constraints, mean this driver must use bounce + * buffers and headers to correct misaligned in or out buffers, and to split + * input buffers that are too large. + * + * This driver will fall back to software decompression if the hardware + * decompression fails, so this driver's decompression should never fail as + * long as the provided compressed buffer is valid. Any compressed buffer + * created by this driver will have a header (except ones where the input + * perfectly matches the constraints); so users of this driver cannot simply + * pass a compressed buffer created by this driver over to the 842 software + * decompression library. Instead, users must use this driver to decompress; + * if the hardware fails or is unavailable, the compressed buffer will be + * parsed and the header removed, and the raw 842 buffer(s) passed to the 842 + * software decompression library. + * + * This does not fall back to software compression, however, since the caller + * of this function is specifically requesting hardware compression; if the + * hardware compression fails, the caller can fall back to software + * compression, and the raw 842 compressed buffer that the software compressor + * creates can be passed to this driver for hardware decompression; any + * buffer without our specific header magic is assumed to be a raw 842 buffer + * and passed directly to the hardware. Note that the software compression + * library will produce a compressed buffer that is incompatible with the + * hardware decompressor if the original input buffer length is not a multiple + * of 8; if such a compressed buffer is passed to this driver for + * decompression, the hardware will reject it and this driver will then pass + * it over to the software library for decompression. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include + +#include "nx-842.h" + +/* The first 5 bits of this magic are 0x1f, which is an invalid 842 5-bit + * template (see lib/842/842.h), so this magic number will never appear at + * the start of a raw 842 compressed buffer. That is important, as any buffer + * passed to us without this magic is assumed to be a raw 842 compressed + * buffer, and passed directly to the hardware to decompress. + */ +#define NX842_CRYPTO_MAGIC (0xf842) +#define NX842_CRYPTO_HEADER_SIZE(g) \ + (sizeof(struct nx842_crypto_header) + \ + sizeof(struct nx842_crypto_header_group) * (g)) +#define NX842_CRYPTO_HEADER_MAX_SIZE \ + NX842_CRYPTO_HEADER_SIZE(NX842_CRYPTO_GROUP_MAX) + +/* bounce buffer size */ +#define BOUNCE_BUFFER_ORDER (2) +#define BOUNCE_BUFFER_SIZE \ + ((unsigned int)(PAGE_SIZE << BOUNCE_BUFFER_ORDER)) + +/* try longer on comp because we can fallback to sw decomp if hw is busy */ +#define COMP_BUSY_TIMEOUT (250) /* ms */ +#define DECOMP_BUSY_TIMEOUT (50) /* ms */ + +struct nx842_crypto_param { + u8 *in; + unsigned int iremain; + u8 *out; + unsigned int oremain; + unsigned int ototal; +}; + +static int update_param(struct nx842_crypto_param *p, + unsigned int slen, unsigned int dlen) +{ + if (p->iremain < slen) + return -EOVERFLOW; + if (p->oremain < dlen) + return -ENOSPC; + + p->in += slen; + p->iremain -= slen; + p->out += dlen; + p->oremain -= dlen; + p->ototal += dlen; + + return 0; +} + +void *nx842_crypto_alloc_ctx(struct nx842_driver *driver) +{ + struct nx842_crypto_ctx *ctx; + + ctx = kzalloc_obj(*ctx); + if (!ctx) + return ERR_PTR(-ENOMEM); + + spin_lock_init(&ctx->lock); + ctx->driver = driver; + ctx->wmem = kmalloc(driver->workmem_size, GFP_KERNEL); + ctx->sbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER); + ctx->dbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER); + if (!ctx->wmem || !ctx->sbounce || !ctx->dbounce) { + nx842_crypto_free_ctx(ctx); + return ERR_PTR(-ENOMEM); + } + + return ctx; +} +EXPORT_SYMBOL_GPL(nx842_crypto_alloc_ctx); + +void nx842_crypto_free_ctx(void *p) +{ + struct nx842_crypto_ctx *ctx = p; + + kfree(ctx->wmem); + free_pages((unsigned long)ctx->sbounce, BOUNCE_BUFFER_ORDER); + free_pages((unsigned long)ctx->dbounce, BOUNCE_BUFFER_ORDER); + kfree(ctx); +} +EXPORT_SYMBOL_GPL(nx842_crypto_free_ctx); + +static void check_constraints(struct nx842_constraints *c) +{ + /* limit maximum, to always have enough bounce buffer to decompress */ + if (c->maximum > BOUNCE_BUFFER_SIZE) + c->maximum = BOUNCE_BUFFER_SIZE; +} + +static int nx842_crypto_add_header(struct nx842_crypto_header *hdr, u8 *buf) +{ + int s = NX842_CRYPTO_HEADER_SIZE(hdr->groups); + + /* compress should have added space for header */ + if (s > be16_to_cpu(hdr->group[0].padding)) { + pr_err("Internal error: no space for header\n"); + return -EINVAL; + } + + memcpy(buf, hdr, s); + + print_hex_dump_debug("header ", DUMP_PREFIX_OFFSET, 16, 1, buf, s, 0); + + return 0; +} + +static int compress(struct nx842_crypto_ctx *ctx, + struct nx842_crypto_param *p, + struct nx842_crypto_header_group *g, + struct nx842_constraints *c, + u16 *ignore, + unsigned int hdrsize) +{ + unsigned int slen = p->iremain, dlen = p->oremain, tmplen; + unsigned int adj_slen = slen; + u8 *src = p->in, *dst = p->out; + int ret, dskip = 0; + ktime_t timeout; + + if (p->iremain == 0) + return -EOVERFLOW; + + if (p->oremain == 0 || hdrsize + c->minimum > dlen) + return -ENOSPC; + + if (slen % c->multiple) + adj_slen = round_up(slen, c->multiple); + if (slen < c->minimum) + adj_slen = c->minimum; + if (slen > c->maximum) + adj_slen = slen = c->maximum; + if (adj_slen > slen || (u64)src % c->alignment) { + adj_slen = min(adj_slen, BOUNCE_BUFFER_SIZE); + slen = min(slen, BOUNCE_BUFFER_SIZE); + if (adj_slen > slen) + memset(ctx->sbounce + slen, 0, adj_slen - slen); + memcpy(ctx->sbounce, src, slen); + src = ctx->sbounce; + slen = adj_slen; + pr_debug("using comp sbounce buffer, len %x\n", slen); + } + + dst += hdrsize; + dlen -= hdrsize; + + if ((u64)dst % c->alignment) { + dskip = (int)(PTR_ALIGN(dst, c->alignment) - dst); + dst += dskip; + dlen -= dskip; + } + if (dlen % c->multiple) + dlen = round_down(dlen, c->multiple); + if (dlen < c->minimum) { +nospc: + dst = ctx->dbounce; + dlen = min(p->oremain, BOUNCE_BUFFER_SIZE); + dlen = round_down(dlen, c->multiple); + dskip = 0; + pr_debug("using comp dbounce buffer, len %x\n", dlen); + } + if (dlen > c->maximum) + dlen = c->maximum; + + tmplen = dlen; + timeout = ktime_add_ms(ktime_get(), COMP_BUSY_TIMEOUT); + do { + dlen = tmplen; /* reset dlen, if we're retrying */ + ret = ctx->driver->compress(src, slen, dst, &dlen, ctx->wmem); + /* possibly we should reduce the slen here, instead of + * retrying with the dbounce buffer? + */ + if (ret == -ENOSPC && dst != ctx->dbounce) + goto nospc; + } while (ret == -EBUSY && ktime_before(ktime_get(), timeout)); + if (ret) + return ret; + + dskip += hdrsize; + + if (dst == ctx->dbounce) + memcpy(p->out + dskip, dst, dlen); + + g->padding = cpu_to_be16(dskip); + g->compressed_length = cpu_to_be32(dlen); + g->uncompressed_length = cpu_to_be32(slen); + + if (p->iremain < slen) { + *ignore = slen - p->iremain; + slen = p->iremain; + } + + pr_debug("compress slen %x ignore %x dlen %x padding %x\n", + slen, *ignore, dlen, dskip); + + return update_param(p, slen, dskip + dlen); +} + +int nx842_crypto_compress(struct crypto_scomp *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen, void *pctx) +{ + struct nx842_crypto_ctx *ctx = pctx; + struct nx842_crypto_header *hdr = + container_of(&ctx->header, + struct nx842_crypto_header, hdr); + struct nx842_crypto_param p; + struct nx842_constraints c = *ctx->driver->constraints; + unsigned int groups, hdrsize, h; + int ret, n; + bool add_header; + u16 ignore = 0; + + check_constraints(&c); + + p.in = (u8 *)src; + p.iremain = slen; + p.out = dst; + p.oremain = *dlen; + p.ototal = 0; + + *dlen = 0; + + groups = min_t(unsigned int, NX842_CRYPTO_GROUP_MAX, + DIV_ROUND_UP(p.iremain, c.maximum)); + hdrsize = NX842_CRYPTO_HEADER_SIZE(groups); + + spin_lock_bh(&ctx->lock); + + /* skip adding header if the buffers meet all constraints */ + add_header = (p.iremain % c.multiple || + p.iremain < c.minimum || + p.iremain > c.maximum || + (u64)p.in % c.alignment || + p.oremain % c.multiple || + p.oremain < c.minimum || + p.oremain > c.maximum || + (u64)p.out % c.alignment); + + hdr->magic = cpu_to_be16(NX842_CRYPTO_MAGIC); + hdr->groups = 0; + hdr->ignore = 0; + + while (p.iremain > 0) { + n = hdr->groups++; + ret = -ENOSPC; + if (hdr->groups > NX842_CRYPTO_GROUP_MAX) + goto unlock; + + /* header goes before first group */ + h = !n && add_header ? hdrsize : 0; + + if (ignore) + pr_warn("internal error, ignore is set %x\n", ignore); + + ret = compress(ctx, &p, &hdr->group[n], &c, &ignore, h); + if (ret) + goto unlock; + } + + if (!add_header && hdr->groups > 1) { + pr_err("Internal error: No header but multiple groups\n"); + ret = -EINVAL; + goto unlock; + } + + /* ignore indicates the input stream needed to be padded */ + hdr->ignore = cpu_to_be16(ignore); + if (ignore) + pr_debug("marked %d bytes as ignore\n", ignore); + + if (add_header) + ret = nx842_crypto_add_header(hdr, dst); + if (ret) + goto unlock; + + *dlen = p.ototal; + + pr_debug("compress total slen %x dlen %x\n", slen, *dlen); + +unlock: + spin_unlock_bh(&ctx->lock); + return ret; +} +EXPORT_SYMBOL_GPL(nx842_crypto_compress); + +static int decompress(struct nx842_crypto_ctx *ctx, + struct nx842_crypto_param *p, + struct nx842_crypto_header_group *g, + struct nx842_constraints *c, + u16 ignore) +{ + unsigned int slen = be32_to_cpu(g->compressed_length); + unsigned int required_len = be32_to_cpu(g->uncompressed_length); + unsigned int dlen = p->oremain, tmplen; + unsigned int adj_slen = slen; + u8 *src = p->in, *dst = p->out; + u16 padding = be16_to_cpu(g->padding); + int ret, spadding = 0; + ktime_t timeout; + + if (!slen || !required_len) + return -EINVAL; + + if (p->iremain <= 0 || padding + slen > p->iremain) + return -EOVERFLOW; + + if (p->oremain <= 0 || required_len - ignore > p->oremain) + return -ENOSPC; + + src += padding; + + if (slen % c->multiple) + adj_slen = round_up(slen, c->multiple); + if (slen < c->minimum) + adj_slen = c->minimum; + if (slen > c->maximum) + goto usesw; + if (slen < adj_slen || (u64)src % c->alignment) { + /* we can append padding bytes because the 842 format defines + * an "end" template (see lib/842/842_decompress.c) and will + * ignore any bytes following it. + */ + if (slen < adj_slen) + memset(ctx->sbounce + slen, 0, adj_slen - slen); + memcpy(ctx->sbounce, src, slen); + src = ctx->sbounce; + spadding = adj_slen - slen; + slen = adj_slen; + pr_debug("using decomp sbounce buffer, len %x\n", slen); + } + + if (dlen % c->multiple) + dlen = round_down(dlen, c->multiple); + if (dlen < required_len || (u64)dst % c->alignment) { + dst = ctx->dbounce; + dlen = min(required_len, BOUNCE_BUFFER_SIZE); + pr_debug("using decomp dbounce buffer, len %x\n", dlen); + } + if (dlen < c->minimum) + goto usesw; + if (dlen > c->maximum) + dlen = c->maximum; + + tmplen = dlen; + timeout = ktime_add_ms(ktime_get(), DECOMP_BUSY_TIMEOUT); + do { + dlen = tmplen; /* reset dlen, if we're retrying */ + ret = ctx->driver->decompress(src, slen, dst, &dlen, ctx->wmem); + } while (ret == -EBUSY && ktime_before(ktime_get(), timeout)); + if (ret) { +usesw: + /* reset everything, sw doesn't have constraints */ + src = p->in + padding; + slen = be32_to_cpu(g->compressed_length); + spadding = 0; + dst = p->out; + dlen = p->oremain; + if (dlen < required_len) { /* have ignore bytes */ + dst = ctx->dbounce; + dlen = BOUNCE_BUFFER_SIZE; + } + pr_info_ratelimited("using software 842 decompression\n"); + ret = sw842_decompress(src, slen, dst, &dlen); + } + if (ret) + return ret; + + slen -= spadding; + + dlen -= ignore; + if (ignore) + pr_debug("ignoring last %x bytes\n", ignore); + + if (dst == ctx->dbounce) + memcpy(p->out, dst, dlen); + + pr_debug("decompress slen %x padding %x dlen %x ignore %x\n", + slen, padding, dlen, ignore); + + return update_param(p, slen + padding, dlen); +} + +int nx842_crypto_decompress(struct crypto_scomp *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen, void *pctx) +{ + struct nx842_crypto_ctx *ctx = pctx; + struct nx842_crypto_header *hdr; + struct nx842_crypto_param p; + struct nx842_constraints c = *ctx->driver->constraints; + int n, ret, hdr_len; + u16 ignore = 0; + + check_constraints(&c); + + p.in = (u8 *)src; + p.iremain = slen; + p.out = dst; + p.oremain = *dlen; + p.ototal = 0; + + *dlen = 0; + + hdr = (struct nx842_crypto_header *)src; + + spin_lock_bh(&ctx->lock); + + /* If it doesn't start with our header magic number, assume it's a raw + * 842 compressed buffer and pass it directly to the hardware driver + */ + if (be16_to_cpu(hdr->magic) != NX842_CRYPTO_MAGIC) { + struct nx842_crypto_header_group g = { + .padding = 0, + .compressed_length = cpu_to_be32(p.iremain), + .uncompressed_length = cpu_to_be32(p.oremain), + }; + + ret = decompress(ctx, &p, &g, &c, 0); + if (ret) + goto unlock; + + goto success; + } + + if (!hdr->groups) { + pr_err("header has no groups\n"); + ret = -EINVAL; + goto unlock; + } + if (hdr->groups > NX842_CRYPTO_GROUP_MAX) { + pr_err("header has too many groups %x, max %x\n", + hdr->groups, NX842_CRYPTO_GROUP_MAX); + ret = -EINVAL; + goto unlock; + } + + hdr_len = NX842_CRYPTO_HEADER_SIZE(hdr->groups); + if (hdr_len > slen) { + ret = -EOVERFLOW; + goto unlock; + } + + memcpy(&ctx->header, src, hdr_len); + hdr = container_of(&ctx->header, struct nx842_crypto_header, hdr); + + for (n = 0; n < hdr->groups; n++) { + /* ignore applies to last group */ + if (n + 1 == hdr->groups) + ignore = be16_to_cpu(hdr->ignore); + + ret = decompress(ctx, &p, &hdr->group[n], &c, ignore); + if (ret) + goto unlock; + } + +success: + *dlen = p.ototal; + + pr_debug("decompress total slen %x dlen %x\n", slen, *dlen); + + ret = 0; + +unlock: + spin_unlock_bh(&ctx->lock); + + return ret; +} +EXPORT_SYMBOL_GPL(nx842_crypto_decompress); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("IBM PowerPC Nest (NX) 842 Hardware Compression Driver"); +MODULE_AUTHOR("Dan Streetman "); diff --git a/drivers/crypto/nx/nx-842.h b/drivers/crypto/nx/nx-842.h new file mode 100644 index 000000000..c401cdf1a --- /dev/null +++ b/drivers/crypto/nx/nx-842.h @@ -0,0 +1,195 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __NX_842_H__ +#define __NX_842_H__ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* Restrictions on Data Descriptor List (DDL) and Entry (DDE) buffers + * + * From NX P8 workbook, sec 4.9.1 "842 details" + * Each DDE buffer is 128 byte aligned + * Each DDE buffer size is a multiple of 32 bytes (except the last) + * The last DDE buffer size is a multiple of 8 bytes + */ +#define DDE_BUFFER_ALIGN (128) +#define DDE_BUFFER_SIZE_MULT (32) +#define DDE_BUFFER_LAST_MULT (8) + +/* Arbitrary DDL length limit + * Allows max buffer size of MAX-1 to MAX pages + * (depending on alignment) + */ +#define DDL_LEN_MAX (17) + +/* CCW 842 CI/FC masks + * NX P8 workbook, section 4.3.1, figure 4-6 + * "CI/FC Boundary by NX CT type" + */ +#define CCW_CI_842 (0x00003ff8) +#define CCW_FC_842 (0x00000007) + +/* CCW Function Codes (FC) for 842 + * NX P8 workbook, section 4.9, table 4-28 + * "Function Code Definitions for 842 Memory Compression" + */ +#define CCW_FC_842_COMP_NOCRC (0) +#define CCW_FC_842_COMP_CRC (1) +#define CCW_FC_842_DECOMP_NOCRC (2) +#define CCW_FC_842_DECOMP_CRC (3) +#define CCW_FC_842_MOVE (4) + +/* CSB CC Error Types for 842 + * NX P8 workbook, section 4.10.3, table 4-30 + * "Reported Error Types Summary Table" + */ +/* These are all duplicates of existing codes defined in icswx.h. */ +#define CSB_CC_TRANSLATION_DUP1 (80) +#define CSB_CC_TRANSLATION_DUP2 (82) +#define CSB_CC_TRANSLATION_DUP3 (84) +#define CSB_CC_TRANSLATION_DUP4 (86) +#define CSB_CC_TRANSLATION_DUP5 (92) +#define CSB_CC_TRANSLATION_DUP6 (94) +#define CSB_CC_PROTECTION_DUP1 (81) +#define CSB_CC_PROTECTION_DUP2 (83) +#define CSB_CC_PROTECTION_DUP3 (85) +#define CSB_CC_PROTECTION_DUP4 (87) +#define CSB_CC_PROTECTION_DUP5 (93) +#define CSB_CC_PROTECTION_DUP6 (95) +#define CSB_CC_RD_EXTERNAL_DUP1 (89) +#define CSB_CC_RD_EXTERNAL_DUP2 (90) +#define CSB_CC_RD_EXTERNAL_DUP3 (91) +/* These are specific to NX */ +/* 842 codes */ +#define CSB_CC_TPBC_GT_SPBC (64) /* no error, but >1 comp ratio */ +#define CSB_CC_CRC_MISMATCH (65) /* decomp crc mismatch */ +#define CSB_CC_TEMPL_INVALID (66) /* decomp invalid template value */ +#define CSB_CC_TEMPL_OVERFLOW (67) /* decomp template shows data after end */ +/* sym crypt codes */ +#define CSB_CC_DECRYPT_OVERFLOW (64) +/* asym crypt codes */ +#define CSB_CC_MINV_OVERFLOW (128) +/* + * HW error - Job did not finish in the maximum time allowed. + * Job terminated. + */ +#define CSB_CC_HW_EXPIRED_TIMER (224) +/* These are reserved for hypervisor use */ +#define CSB_CC_HYP_RESERVE_START (240) +#define CSB_CC_HYP_RESERVE_END (253) +#define CSB_CC_HYP_RESERVE_P9_END (251) +/* No valid interrupt server (P9 or later). */ +#define CSB_CC_HYP_RESERVE_NO_INTR_SERVER (252) +#define CSB_CC_HYP_NO_HW (254) +#define CSB_CC_HYP_HANG_ABORTED (255) + +/* CCB Completion Modes (CM) for 842 + * NX P8 workbook, section 4.3, figure 4-5 + * "CRB Details - Normal Cop_Req (CL=00, C=1)" + */ +#define CCB_CM_EXTRA_WRITE (CCB_CM0_ALL_COMPLETIONS & CCB_CM12_STORE) +#define CCB_CM_INTERRUPT (CCB_CM0_ALL_COMPLETIONS & CCB_CM12_INTERRUPT) + +#define LEN_ON_SIZE(pa, size) ((size) - ((pa) & ((size) - 1))) +#define LEN_ON_PAGE(pa) LEN_ON_SIZE(pa, PAGE_SIZE) + +struct crypto_scomp; + +static inline unsigned long nx842_get_pa(void *addr) +{ + if (!is_vmalloc_addr(addr)) + return __pa(addr); + + return page_to_phys(vmalloc_to_page(addr)) + offset_in_page(addr); +} + +/** + * This provides the driver's constraints. Different nx842 implementations + * may have varying requirements. The constraints are: + * @alignment: All buffers should be aligned to this + * @multiple: All buffer lengths should be a multiple of this + * @minimum: Buffer lengths must not be less than this amount + * @maximum: Buffer lengths must not be more than this amount + * + * The constraints apply to all buffers and lengths, both input and output, + * for both compression and decompression, except for the minimum which + * only applies to compression input and decompression output; the + * compressed data can be less than the minimum constraint. It can be + * assumed that compressed data will always adhere to the multiple + * constraint. + * + * The driver may succeed even if these constraints are violated; + * however the driver can return failure or suffer reduced performance + * if any constraint is not met. + */ +struct nx842_constraints { + int alignment; + int multiple; + int minimum; + int maximum; +}; + +struct nx842_driver { + char *name; + struct module *owner; + size_t workmem_size; + + struct nx842_constraints *constraints; + + int (*compress)(const unsigned char *in, unsigned int in_len, + unsigned char *out, unsigned int *out_len, + void *wrkmem); + int (*decompress)(const unsigned char *in, unsigned int in_len, + unsigned char *out, unsigned int *out_len, + void *wrkmem); +}; + +struct nx842_crypto_header_group { + __be16 padding; /* unused bytes at start of group */ + __be32 compressed_length; /* compressed bytes in group */ + __be32 uncompressed_length; /* bytes after decompression */ +} __packed; + +struct nx842_crypto_header { + /* New members MUST be added within the struct_group() macro below. */ + __struct_group(nx842_crypto_header_hdr, hdr, __packed, + __be16 magic; /* NX842_CRYPTO_MAGIC */ + __be16 ignore; /* decompressed end bytes to ignore */ + u8 groups; /* total groups in this header */ + ); + struct nx842_crypto_header_group group[] __counted_by(groups); +} __packed; +static_assert(offsetof(struct nx842_crypto_header, group) == sizeof(struct nx842_crypto_header_hdr), + "struct member likely outside of __struct_group()"); + +#define NX842_CRYPTO_GROUP_MAX (0x20) + +struct nx842_crypto_ctx { + spinlock_t lock; + + u8 *wmem; + u8 *sbounce, *dbounce; + + struct nx842_crypto_header_hdr header; + struct nx842_crypto_header_group group[NX842_CRYPTO_GROUP_MAX]; + + struct nx842_driver *driver; +}; + +void *nx842_crypto_alloc_ctx(struct nx842_driver *driver); +void nx842_crypto_free_ctx(void *ctx); +int nx842_crypto_compress(struct crypto_scomp *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen, void *ctx); +int nx842_crypto_decompress(struct crypto_scomp *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen, void *ctx); + +#endif /* __NX_842_H__ */ diff --git a/drivers/crypto/nx/nx-aes-cbc.c b/drivers/crypto/nx/nx-aes-cbc.c new file mode 100644 index 000000000..35fa5bad1 --- /dev/null +++ b/drivers/crypto/nx/nx-aes-cbc.c @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AES CBC routines supporting the Power 7+ Nest Accelerators driver + * + * Copyright (C) 2011-2012 International Business Machines Inc. + * + * Author: Kent Yoder + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nx_csbcpb.h" +#include "nx.h" + + +static int cbc_aes_nx_set_key(struct crypto_skcipher *tfm, + const u8 *in_key, + unsigned int key_len) +{ + struct nx_crypto_ctx *nx_ctx = crypto_skcipher_ctx(tfm); + struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; + + nx_ctx_init(nx_ctx, HCOP_FC_AES); + + switch (key_len) { + case AES_KEYSIZE_128: + NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128); + nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128]; + break; + case AES_KEYSIZE_192: + NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_192); + nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_192]; + break; + case AES_KEYSIZE_256: + NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_256); + nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_256]; + break; + default: + return -EINVAL; + } + + csbcpb->cpb.hdr.mode = NX_MODE_AES_CBC; + memcpy(csbcpb->cpb.aes_cbc.key, in_key, key_len); + + return 0; +} + +static int cbc_aes_nx_crypt(struct skcipher_request *req, + int enc) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct nx_crypto_ctx *nx_ctx = crypto_skcipher_ctx(tfm); + struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; + unsigned long irq_flags; + unsigned int processed = 0, to_process; + int rc; + + spin_lock_irqsave(&nx_ctx->lock, irq_flags); + + if (enc) + NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT; + else + NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT; + + do { + to_process = req->cryptlen - processed; + + rc = nx_build_sg_lists(nx_ctx, req->iv, req->dst, req->src, + &to_process, processed, + csbcpb->cpb.aes_cbc.iv); + if (rc) + goto out; + + if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) { + rc = -EINVAL; + goto out; + } + + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP); + if (rc) + goto out; + + memcpy(req->iv, csbcpb->cpb.aes_cbc.cv, AES_BLOCK_SIZE); + atomic_inc(&(nx_ctx->stats->aes_ops)); + atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count), + &(nx_ctx->stats->aes_bytes)); + + processed += to_process; + } while (processed < req->cryptlen); +out: + spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); + return rc; +} + +static int cbc_aes_nx_encrypt(struct skcipher_request *req) +{ + return cbc_aes_nx_crypt(req, 1); +} + +static int cbc_aes_nx_decrypt(struct skcipher_request *req) +{ + return cbc_aes_nx_crypt(req, 0); +} + +struct skcipher_alg nx_cbc_aes_alg = { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-nx", + .base.cra_priority = 300, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct nx_crypto_ctx), + .base.cra_alignmask = 0xf, + .base.cra_module = THIS_MODULE, + .init = nx_crypto_ctx_aes_cbc_init, + .exit = nx_crypto_ctx_skcipher_exit, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = cbc_aes_nx_set_key, + .encrypt = cbc_aes_nx_encrypt, + .decrypt = cbc_aes_nx_decrypt, +}; diff --git a/drivers/crypto/nx/nx-aes-ccm.c b/drivers/crypto/nx/nx-aes-ccm.c new file mode 100644 index 000000000..56a0b3a67 --- /dev/null +++ b/drivers/crypto/nx/nx-aes-ccm.c @@ -0,0 +1,559 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AES CCM routines supporting the Power 7+ Nest Accelerators driver + * + * Copyright (C) 2012 International Business Machines Inc. + * + * Author: Kent Yoder + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nx_csbcpb.h" +#include "nx.h" + + +static int ccm_aes_nx_set_key(struct crypto_aead *tfm, + const u8 *in_key, + unsigned int key_len) +{ + struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base); + struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; + struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead; + + nx_ctx_init(nx_ctx, HCOP_FC_AES); + + switch (key_len) { + case AES_KEYSIZE_128: + NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128); + NX_CPB_SET_KEY_SIZE(csbcpb_aead, NX_KS_AES_128); + nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128]; + break; + default: + return -EINVAL; + } + + csbcpb->cpb.hdr.mode = NX_MODE_AES_CCM; + memcpy(csbcpb->cpb.aes_ccm.key, in_key, key_len); + + csbcpb_aead->cpb.hdr.mode = NX_MODE_AES_CCA; + memcpy(csbcpb_aead->cpb.aes_cca.key, in_key, key_len); + + return 0; + +} + +static int ccm4309_aes_nx_set_key(struct crypto_aead *tfm, + const u8 *in_key, + unsigned int key_len) +{ + struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base); + + if (key_len < 3) + return -EINVAL; + + key_len -= 3; + + memcpy(nx_ctx->priv.ccm.nonce, in_key + key_len, 3); + + return ccm_aes_nx_set_key(tfm, in_key, key_len); +} + +static int ccm_aes_nx_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + switch (authsize) { + case 4: + case 6: + case 8: + case 10: + case 12: + case 14: + case 16: + break; + default: + return -EINVAL; + } + + return 0; +} + +static int ccm4309_aes_nx_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + switch (authsize) { + case 8: + case 12: + case 16: + break; + default: + return -EINVAL; + } + + return 0; +} + +/* taken from crypto/ccm.c */ +static int set_msg_len(u8 *block, unsigned int msglen, int csize) +{ + __be32 data; + + memset(block, 0, csize); + block += csize; + + if (csize >= 4) + csize = 4; + else if (msglen > (unsigned int)(1 << (8 * csize))) + return -EOVERFLOW; + + data = cpu_to_be32(msglen); + memcpy(block - csize, (u8 *)&data + 4 - csize, csize); + + return 0; +} + +/* taken from crypto/ccm.c */ +static inline int crypto_ccm_check_iv(const u8 *iv) +{ + /* 2 <= L <= 8, so 1 <= L' <= 7. */ + if (1 > iv[0] || iv[0] > 7) + return -EINVAL; + + return 0; +} + +/* based on code from crypto/ccm.c */ +static int generate_b0(u8 *iv, unsigned int assoclen, unsigned int authsize, + unsigned int cryptlen, u8 *b0) +{ + unsigned int l, lp, m = authsize; + + memcpy(b0, iv, 16); + + lp = b0[0]; + l = lp + 1; + + /* set m, bits 3-5 */ + *b0 |= (8 * ((m - 2) / 2)); + + /* set adata, bit 6, if associated data is used */ + if (assoclen) + *b0 |= 64; + + return set_msg_len(b0 + 16 - l, cryptlen, l); +} + +static int generate_pat(u8 *iv, + struct aead_request *req, + struct nx_crypto_ctx *nx_ctx, + unsigned int authsize, + unsigned int nbytes, + unsigned int assoclen, + u8 *out) +{ + struct nx_sg *nx_insg = nx_ctx->in_sg; + struct nx_sg *nx_outsg = nx_ctx->out_sg; + unsigned int iauth_len = 0; + u8 tmp[16], *b1 = NULL, *b0 = NULL, *result = NULL; + int rc; + unsigned int max_sg_len; + + /* zero the ctr value */ + memset(iv + 15 - iv[0], 0, iv[0] + 1); + + /* page 78 of nx_wb.pdf has, + * Note: RFC3610 allows the AAD data to be up to 2^64 -1 bytes + * in length. If a full message is used, the AES CCA implementation + * restricts the maximum AAD length to 2^32 -1 bytes. + * If partial messages are used, the implementation supports + * 2^64 -1 bytes maximum AAD length. + * + * However, in the cryptoapi's aead_request structure, + * assoclen is an unsigned int, thus it cannot hold a length + * value greater than 2^32 - 1. + * Thus the AAD is further constrained by this and is never + * greater than 2^32. + */ + + if (!assoclen) { + b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0; + } else if (assoclen <= 14) { + /* if associated data is 14 bytes or less, we do 1 GCM + * operation on 2 AES blocks, B0 (stored in the csbcpb) and B1, + * which is fed in through the source buffers here */ + b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0; + b1 = nx_ctx->priv.ccm.iauth_tag; + iauth_len = assoclen; + } else if (assoclen <= 65280) { + /* if associated data is less than (2^16 - 2^8), we construct + * B1 differently and feed in the associated data to a CCA + * operation */ + b0 = nx_ctx->csbcpb_aead->cpb.aes_cca.b0; + b1 = nx_ctx->csbcpb_aead->cpb.aes_cca.b1; + iauth_len = 14; + } else { + b0 = nx_ctx->csbcpb_aead->cpb.aes_cca.b0; + b1 = nx_ctx->csbcpb_aead->cpb.aes_cca.b1; + iauth_len = 10; + } + + /* generate B0 */ + rc = generate_b0(iv, assoclen, authsize, nbytes, b0); + if (rc) + return rc; + + /* generate B1: + * add control info for associated data + * RFC 3610 and NIST Special Publication 800-38C + */ + if (b1) { + memset(b1, 0, 16); + if (assoclen <= 65280) { + *(u16 *)b1 = assoclen; + memcpy_from_sglist(b1 + 2, req->src, 0, iauth_len); + } else { + *(u16 *)b1 = (u16)(0xfffe); + *(u32 *)&b1[2] = assoclen; + memcpy_from_sglist(b1 + 6, req->src, 0, iauth_len); + } + } + + /* now copy any remaining AAD to scatterlist and call nx... */ + if (!assoclen) { + return rc; + } else if (assoclen <= 14) { + unsigned int len = 16; + + nx_insg = nx_build_sg_list(nx_insg, b1, &len, nx_ctx->ap->sglen); + + if (len != 16) + return -EINVAL; + + nx_outsg = nx_build_sg_list(nx_outsg, tmp, &len, + nx_ctx->ap->sglen); + + if (len != 16) + return -EINVAL; + + /* inlen should be negative, indicating to phyp that its a + * pointer to an sg list */ + nx_ctx->op.inlen = (nx_ctx->in_sg - nx_insg) * + sizeof(struct nx_sg); + nx_ctx->op.outlen = (nx_ctx->out_sg - nx_outsg) * + sizeof(struct nx_sg); + + NX_CPB_FDM(nx_ctx->csbcpb) |= NX_FDM_ENDE_ENCRYPT; + NX_CPB_FDM(nx_ctx->csbcpb) |= NX_FDM_INTERMEDIATE; + + result = nx_ctx->csbcpb->cpb.aes_ccm.out_pat_or_mac; + + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP); + if (rc) + return rc; + + atomic_inc(&(nx_ctx->stats->aes_ops)); + atomic64_add(assoclen, &nx_ctx->stats->aes_bytes); + + } else { + unsigned int processed = 0, to_process; + + processed += iauth_len; + + /* page_limit: number of sg entries that fit on one page */ + max_sg_len = min_t(u64, nx_ctx->ap->sglen, + nx_driver.of.max_sg_len/sizeof(struct nx_sg)); + max_sg_len = min_t(u64, max_sg_len, + nx_ctx->ap->databytelen/NX_PAGE_SIZE); + + do { + to_process = min_t(u32, assoclen - processed, + nx_ctx->ap->databytelen); + + nx_insg = nx_walk_and_build(nx_ctx->in_sg, + nx_ctx->ap->sglen, + req->src, processed, + &to_process); + + if ((to_process + processed) < assoclen) { + NX_CPB_FDM(nx_ctx->csbcpb_aead) |= + NX_FDM_INTERMEDIATE; + } else { + NX_CPB_FDM(nx_ctx->csbcpb_aead) &= + ~NX_FDM_INTERMEDIATE; + } + + + nx_ctx->op_aead.inlen = (nx_ctx->in_sg - nx_insg) * + sizeof(struct nx_sg); + + result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0; + + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op_aead, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP); + if (rc) + return rc; + + memcpy(nx_ctx->csbcpb_aead->cpb.aes_cca.b0, + nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0, + AES_BLOCK_SIZE); + + NX_CPB_FDM(nx_ctx->csbcpb_aead) |= NX_FDM_CONTINUATION; + + atomic_inc(&(nx_ctx->stats->aes_ops)); + atomic64_add(assoclen, &nx_ctx->stats->aes_bytes); + + processed += to_process; + } while (processed < assoclen); + + result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0; + } + + memcpy(out, result, AES_BLOCK_SIZE); + + return rc; +} + +static int ccm_nx_decrypt(struct aead_request *req, + u8 *iv, + unsigned int assoclen) +{ + struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); + struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; + unsigned int nbytes = req->cryptlen; + unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req)); + struct nx_ccm_priv *priv = &nx_ctx->priv.ccm; + unsigned long irq_flags; + unsigned int processed = 0, to_process; + int rc = -1; + + spin_lock_irqsave(&nx_ctx->lock, irq_flags); + + nbytes -= authsize; + + /* copy out the auth tag to compare with later */ + memcpy_from_sglist(priv->oauth_tag, req->src, nbytes + req->assoclen, + authsize); + + rc = generate_pat(iv, req, nx_ctx, authsize, nbytes, assoclen, + csbcpb->cpb.aes_ccm.in_pat_or_b0); + if (rc) + goto out; + + do { + + /* to_process: the AES_BLOCK_SIZE data chunk to process in this + * update. This value is bound by sg list limits. + */ + to_process = nbytes - processed; + + if ((to_process + processed) < nbytes) + NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; + else + NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; + + NX_CPB_FDM(nx_ctx->csbcpb) &= ~NX_FDM_ENDE_ENCRYPT; + + rc = nx_build_sg_lists(nx_ctx, iv, req->dst, req->src, + &to_process, processed + req->assoclen, + csbcpb->cpb.aes_ccm.iv_or_ctr); + if (rc) + goto out; + + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP); + if (rc) + goto out; + + /* for partial completion, copy following for next + * entry into loop... + */ + memcpy(iv, csbcpb->cpb.aes_ccm.out_ctr, AES_BLOCK_SIZE); + memcpy(csbcpb->cpb.aes_ccm.in_pat_or_b0, + csbcpb->cpb.aes_ccm.out_pat_or_mac, AES_BLOCK_SIZE); + memcpy(csbcpb->cpb.aes_ccm.in_s0, + csbcpb->cpb.aes_ccm.out_s0, AES_BLOCK_SIZE); + + NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; + + /* update stats */ + atomic_inc(&(nx_ctx->stats->aes_ops)); + atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count), + &(nx_ctx->stats->aes_bytes)); + + processed += to_process; + } while (processed < nbytes); + + rc = crypto_memneq(csbcpb->cpb.aes_ccm.out_pat_or_mac, priv->oauth_tag, + authsize) ? -EBADMSG : 0; +out: + spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); + return rc; +} + +static int ccm_nx_encrypt(struct aead_request *req, + u8 *iv, + unsigned int assoclen) +{ + struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); + struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; + unsigned int nbytes = req->cryptlen; + unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req)); + unsigned long irq_flags; + unsigned int processed = 0, to_process; + int rc = -1; + + spin_lock_irqsave(&nx_ctx->lock, irq_flags); + + rc = generate_pat(iv, req, nx_ctx, authsize, nbytes, assoclen, + csbcpb->cpb.aes_ccm.in_pat_or_b0); + if (rc) + goto out; + + do { + /* to process: the AES_BLOCK_SIZE data chunk to process in this + * update. This value is bound by sg list limits. + */ + to_process = nbytes - processed; + + if ((to_process + processed) < nbytes) + NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; + else + NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; + + NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT; + + rc = nx_build_sg_lists(nx_ctx, iv, req->dst, req->src, + &to_process, processed + req->assoclen, + csbcpb->cpb.aes_ccm.iv_or_ctr); + if (rc) + goto out; + + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP); + if (rc) + goto out; + + /* for partial completion, copy following for next + * entry into loop... + */ + memcpy(iv, csbcpb->cpb.aes_ccm.out_ctr, AES_BLOCK_SIZE); + memcpy(csbcpb->cpb.aes_ccm.in_pat_or_b0, + csbcpb->cpb.aes_ccm.out_pat_or_mac, AES_BLOCK_SIZE); + memcpy(csbcpb->cpb.aes_ccm.in_s0, + csbcpb->cpb.aes_ccm.out_s0, AES_BLOCK_SIZE); + + NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; + + /* update stats */ + atomic_inc(&(nx_ctx->stats->aes_ops)); + atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count), + &(nx_ctx->stats->aes_bytes)); + + processed += to_process; + + } while (processed < nbytes); + + /* copy out the auth tag */ + memcpy_to_sglist(req->dst, nbytes + req->assoclen, + csbcpb->cpb.aes_ccm.out_pat_or_mac, authsize); + +out: + spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); + return rc; +} + +static int ccm4309_aes_nx_encrypt(struct aead_request *req) +{ + struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); + struct nx_gcm_rctx *rctx = aead_request_ctx(req); + u8 *iv = rctx->iv; + + iv[0] = 3; + memcpy(iv + 1, nx_ctx->priv.ccm.nonce, 3); + memcpy(iv + 4, req->iv, 8); + + return ccm_nx_encrypt(req, iv, req->assoclen - 8); +} + +static int ccm_aes_nx_encrypt(struct aead_request *req) +{ + int rc; + + rc = crypto_ccm_check_iv(req->iv); + if (rc) + return rc; + + return ccm_nx_encrypt(req, req->iv, req->assoclen); +} + +static int ccm4309_aes_nx_decrypt(struct aead_request *req) +{ + struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); + struct nx_gcm_rctx *rctx = aead_request_ctx(req); + u8 *iv = rctx->iv; + + iv[0] = 3; + memcpy(iv + 1, nx_ctx->priv.ccm.nonce, 3); + memcpy(iv + 4, req->iv, 8); + + return ccm_nx_decrypt(req, iv, req->assoclen - 8); +} + +static int ccm_aes_nx_decrypt(struct aead_request *req) +{ + int rc; + + rc = crypto_ccm_check_iv(req->iv); + if (rc) + return rc; + + return ccm_nx_decrypt(req, req->iv, req->assoclen); +} + +struct aead_alg nx_ccm_aes_alg = { + .base = { + .cra_name = "ccm(aes)", + .cra_driver_name = "ccm-aes-nx", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct nx_crypto_ctx), + .cra_module = THIS_MODULE, + }, + .init = nx_crypto_ctx_aes_ccm_init, + .exit = nx_crypto_ctx_aead_exit, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .setkey = ccm_aes_nx_set_key, + .setauthsize = ccm_aes_nx_setauthsize, + .encrypt = ccm_aes_nx_encrypt, + .decrypt = ccm_aes_nx_decrypt, +}; + +struct aead_alg nx_ccm4309_aes_alg = { + .base = { + .cra_name = "rfc4309(ccm(aes))", + .cra_driver_name = "rfc4309-ccm-aes-nx", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct nx_crypto_ctx), + .cra_module = THIS_MODULE, + }, + .init = nx_crypto_ctx_aes_ccm_init, + .exit = nx_crypto_ctx_aead_exit, + .ivsize = 8, + .maxauthsize = AES_BLOCK_SIZE, + .setkey = ccm4309_aes_nx_set_key, + .setauthsize = ccm4309_aes_nx_setauthsize, + .encrypt = ccm4309_aes_nx_encrypt, + .decrypt = ccm4309_aes_nx_decrypt, +}; diff --git a/drivers/crypto/nx/nx-aes-ctr.c b/drivers/crypto/nx/nx-aes-ctr.c new file mode 100644 index 000000000..709b3ee74 --- /dev/null +++ b/drivers/crypto/nx/nx-aes-ctr.c @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AES CTR routines supporting the Power 7+ Nest Accelerators driver + * + * Copyright (C) 2011-2012 International Business Machines Inc. + * + * Author: Kent Yoder + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nx_csbcpb.h" +#include "nx.h" + + +static int ctr_aes_nx_set_key(struct crypto_skcipher *tfm, + const u8 *in_key, + unsigned int key_len) +{ + struct nx_crypto_ctx *nx_ctx = crypto_skcipher_ctx(tfm); + struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; + + nx_ctx_init(nx_ctx, HCOP_FC_AES); + + switch (key_len) { + case AES_KEYSIZE_128: + NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128); + nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128]; + break; + case AES_KEYSIZE_192: + NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_192); + nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_192]; + break; + case AES_KEYSIZE_256: + NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_256); + nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_256]; + break; + default: + return -EINVAL; + } + + csbcpb->cpb.hdr.mode = NX_MODE_AES_CTR; + memcpy(csbcpb->cpb.aes_ctr.key, in_key, key_len); + + return 0; +} + +static int ctr3686_aes_nx_set_key(struct crypto_skcipher *tfm, + const u8 *in_key, + unsigned int key_len) +{ + struct nx_crypto_ctx *nx_ctx = crypto_skcipher_ctx(tfm); + + if (key_len < CTR_RFC3686_NONCE_SIZE) + return -EINVAL; + + memcpy(nx_ctx->priv.ctr.nonce, + in_key + key_len - CTR_RFC3686_NONCE_SIZE, + CTR_RFC3686_NONCE_SIZE); + + key_len -= CTR_RFC3686_NONCE_SIZE; + + return ctr_aes_nx_set_key(tfm, in_key, key_len); +} + +static int ctr_aes_nx_crypt(struct skcipher_request *req, u8 *iv) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct nx_crypto_ctx *nx_ctx = crypto_skcipher_ctx(tfm); + struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; + unsigned long irq_flags; + unsigned int processed = 0, to_process; + int rc; + + spin_lock_irqsave(&nx_ctx->lock, irq_flags); + + do { + to_process = req->cryptlen - processed; + + rc = nx_build_sg_lists(nx_ctx, iv, req->dst, req->src, + &to_process, processed, + csbcpb->cpb.aes_ctr.iv); + if (rc) + goto out; + + if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) { + rc = -EINVAL; + goto out; + } + + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP); + if (rc) + goto out; + + memcpy(iv, csbcpb->cpb.aes_cbc.cv, AES_BLOCK_SIZE); + + atomic_inc(&(nx_ctx->stats->aes_ops)); + atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count), + &(nx_ctx->stats->aes_bytes)); + + processed += to_process; + } while (processed < req->cryptlen); +out: + spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); + return rc; +} + +static int ctr3686_aes_nx_crypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct nx_crypto_ctx *nx_ctx = crypto_skcipher_ctx(tfm); + u8 iv[16]; + + memcpy(iv, nx_ctx->priv.ctr.nonce, CTR_RFC3686_NONCE_SIZE); + memcpy(iv + CTR_RFC3686_NONCE_SIZE, req->iv, CTR_RFC3686_IV_SIZE); + iv[12] = iv[13] = iv[14] = 0; + iv[15] = 1; + + return ctr_aes_nx_crypt(req, iv); +} + +struct skcipher_alg nx_ctr3686_aes_alg = { + .base.cra_name = "rfc3686(ctr(aes))", + .base.cra_driver_name = "rfc3686-ctr-aes-nx", + .base.cra_priority = 300, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct nx_crypto_ctx), + .base.cra_module = THIS_MODULE, + .init = nx_crypto_ctx_aes_ctr_init, + .exit = nx_crypto_ctx_skcipher_exit, + .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + .setkey = ctr3686_aes_nx_set_key, + .encrypt = ctr3686_aes_nx_crypt, + .decrypt = ctr3686_aes_nx_crypt, + .chunksize = AES_BLOCK_SIZE, +}; diff --git a/drivers/crypto/nx/nx-aes-ecb.c b/drivers/crypto/nx/nx-aes-ecb.c new file mode 100644 index 000000000..4039cf3b2 --- /dev/null +++ b/drivers/crypto/nx/nx-aes-ecb.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AES ECB routines supporting the Power 7+ Nest Accelerators driver + * + * Copyright (C) 2011-2012 International Business Machines Inc. + * + * Author: Kent Yoder + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nx_csbcpb.h" +#include "nx.h" + + +static int ecb_aes_nx_set_key(struct crypto_skcipher *tfm, + const u8 *in_key, + unsigned int key_len) +{ + struct nx_crypto_ctx *nx_ctx = crypto_skcipher_ctx(tfm); + struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; + + nx_ctx_init(nx_ctx, HCOP_FC_AES); + + switch (key_len) { + case AES_KEYSIZE_128: + NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128); + nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128]; + break; + case AES_KEYSIZE_192: + NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_192); + nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_192]; + break; + case AES_KEYSIZE_256: + NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_256); + nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_256]; + break; + default: + return -EINVAL; + } + + csbcpb->cpb.hdr.mode = NX_MODE_AES_ECB; + memcpy(csbcpb->cpb.aes_ecb.key, in_key, key_len); + + return 0; +} + +static int ecb_aes_nx_crypt(struct skcipher_request *req, + int enc) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct nx_crypto_ctx *nx_ctx = crypto_skcipher_ctx(tfm); + struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; + unsigned long irq_flags; + unsigned int processed = 0, to_process; + int rc; + + spin_lock_irqsave(&nx_ctx->lock, irq_flags); + + if (enc) + NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT; + else + NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT; + + do { + to_process = req->cryptlen - processed; + + rc = nx_build_sg_lists(nx_ctx, NULL, req->dst, req->src, + &to_process, processed, NULL); + if (rc) + goto out; + + if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) { + rc = -EINVAL; + goto out; + } + + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP); + if (rc) + goto out; + + atomic_inc(&(nx_ctx->stats->aes_ops)); + atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count), + &(nx_ctx->stats->aes_bytes)); + + processed += to_process; + } while (processed < req->cryptlen); + +out: + spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); + return rc; +} + +static int ecb_aes_nx_encrypt(struct skcipher_request *req) +{ + return ecb_aes_nx_crypt(req, 1); +} + +static int ecb_aes_nx_decrypt(struct skcipher_request *req) +{ + return ecb_aes_nx_crypt(req, 0); +} + +struct skcipher_alg nx_ecb_aes_alg = { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-nx", + .base.cra_priority = 300, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_alignmask = 0xf, + .base.cra_ctxsize = sizeof(struct nx_crypto_ctx), + .base.cra_module = THIS_MODULE, + .init = nx_crypto_ctx_aes_ecb_init, + .exit = nx_crypto_ctx_skcipher_exit, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = ecb_aes_nx_set_key, + .encrypt = ecb_aes_nx_encrypt, + .decrypt = ecb_aes_nx_decrypt, +}; diff --git a/drivers/crypto/nx/nx-aes-gcm.c b/drivers/crypto/nx/nx-aes-gcm.c new file mode 100644 index 000000000..b7fe2de96 --- /dev/null +++ b/drivers/crypto/nx/nx-aes-gcm.c @@ -0,0 +1,500 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AES GCM routines supporting the Power 7+ Nest Accelerators driver + * + * Copyright (C) 2012 International Business Machines Inc. + * + * Author: Kent Yoder + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nx_csbcpb.h" +#include "nx.h" + + +static int gcm_aes_nx_set_key(struct crypto_aead *tfm, + const u8 *in_key, + unsigned int key_len) +{ + struct nx_crypto_ctx *nx_ctx = crypto_aead_ctx(tfm); + struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; + struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead; + + nx_ctx_init(nx_ctx, HCOP_FC_AES); + + switch (key_len) { + case AES_KEYSIZE_128: + NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128); + NX_CPB_SET_KEY_SIZE(csbcpb_aead, NX_KS_AES_128); + nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128]; + break; + case AES_KEYSIZE_192: + NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_192); + NX_CPB_SET_KEY_SIZE(csbcpb_aead, NX_KS_AES_192); + nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_192]; + break; + case AES_KEYSIZE_256: + NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_256); + NX_CPB_SET_KEY_SIZE(csbcpb_aead, NX_KS_AES_256); + nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_256]; + break; + default: + return -EINVAL; + } + + csbcpb->cpb.hdr.mode = NX_MODE_AES_GCM; + memcpy(csbcpb->cpb.aes_gcm.key, in_key, key_len); + + csbcpb_aead->cpb.hdr.mode = NX_MODE_AES_GCA; + memcpy(csbcpb_aead->cpb.aes_gca.key, in_key, key_len); + + return 0; +} + +static int gcm4106_aes_nx_set_key(struct crypto_aead *tfm, + const u8 *in_key, + unsigned int key_len) +{ + struct nx_crypto_ctx *nx_ctx = crypto_aead_ctx(tfm); + char *nonce = nx_ctx->priv.gcm.nonce; + int rc; + + if (key_len < 4) + return -EINVAL; + + key_len -= 4; + + rc = gcm_aes_nx_set_key(tfm, in_key, key_len); + if (rc) + goto out; + + memcpy(nonce, in_key + key_len, 4); +out: + return rc; +} + +static int gcm4106_aes_nx_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + switch (authsize) { + case 8: + case 12: + case 16: + break; + default: + return -EINVAL; + } + + return 0; +} + +static int nx_gca(struct nx_crypto_ctx *nx_ctx, + struct aead_request *req, + u8 *out, + unsigned int assoclen) +{ + int rc; + struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead; + struct nx_sg *nx_sg = nx_ctx->in_sg; + unsigned int nbytes = assoclen; + unsigned int processed = 0, to_process; + unsigned int max_sg_len; + + if (nbytes <= AES_BLOCK_SIZE) { + memcpy_from_sglist(out, req->src, 0, nbytes); + return 0; + } + + NX_CPB_FDM(csbcpb_aead) &= ~NX_FDM_CONTINUATION; + + /* page_limit: number of sg entries that fit on one page */ + max_sg_len = min_t(u64, nx_driver.of.max_sg_len/sizeof(struct nx_sg), + nx_ctx->ap->sglen); + max_sg_len = min_t(u64, max_sg_len, + nx_ctx->ap->databytelen/NX_PAGE_SIZE); + + do { + /* + * to_process: the data chunk to process in this update. + * This value is bound by sg list limits. + */ + to_process = min_t(u64, nbytes - processed, + nx_ctx->ap->databytelen); + to_process = min_t(u64, to_process, + NX_PAGE_SIZE * (max_sg_len - 1)); + + nx_sg = nx_walk_and_build(nx_ctx->in_sg, max_sg_len, + req->src, processed, &to_process); + + if ((to_process + processed) < nbytes) + NX_CPB_FDM(csbcpb_aead) |= NX_FDM_INTERMEDIATE; + else + NX_CPB_FDM(csbcpb_aead) &= ~NX_FDM_INTERMEDIATE; + + nx_ctx->op_aead.inlen = (nx_ctx->in_sg - nx_sg) + * sizeof(struct nx_sg); + + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op_aead, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP); + if (rc) + return rc; + + memcpy(csbcpb_aead->cpb.aes_gca.in_pat, + csbcpb_aead->cpb.aes_gca.out_pat, + AES_BLOCK_SIZE); + NX_CPB_FDM(csbcpb_aead) |= NX_FDM_CONTINUATION; + + atomic_inc(&(nx_ctx->stats->aes_ops)); + atomic64_add(assoclen, &(nx_ctx->stats->aes_bytes)); + + processed += to_process; + } while (processed < nbytes); + + memcpy(out, csbcpb_aead->cpb.aes_gca.out_pat, AES_BLOCK_SIZE); + + return rc; +} + +static int gmac(struct aead_request *req, const u8 *iv, unsigned int assoclen) +{ + int rc; + struct nx_crypto_ctx *nx_ctx = + crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; + struct nx_sg *nx_sg; + unsigned int nbytes = assoclen; + unsigned int processed = 0, to_process; + unsigned int max_sg_len; + + /* Set GMAC mode */ + csbcpb->cpb.hdr.mode = NX_MODE_AES_GMAC; + + NX_CPB_FDM(csbcpb) &= ~NX_FDM_CONTINUATION; + + /* page_limit: number of sg entries that fit on one page */ + max_sg_len = min_t(u64, nx_driver.of.max_sg_len/sizeof(struct nx_sg), + nx_ctx->ap->sglen); + max_sg_len = min_t(u64, max_sg_len, + nx_ctx->ap->databytelen/NX_PAGE_SIZE); + + /* Copy IV */ + memcpy(csbcpb->cpb.aes_gcm.iv_or_cnt, iv, AES_BLOCK_SIZE); + + do { + /* + * to_process: the data chunk to process in this update. + * This value is bound by sg list limits. + */ + to_process = min_t(u64, nbytes - processed, + nx_ctx->ap->databytelen); + to_process = min_t(u64, to_process, + NX_PAGE_SIZE * (max_sg_len - 1)); + + nx_sg = nx_walk_and_build(nx_ctx->in_sg, max_sg_len, + req->src, processed, &to_process); + + if ((to_process + processed) < nbytes) + NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; + else + NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; + + nx_ctx->op.inlen = (nx_ctx->in_sg - nx_sg) + * sizeof(struct nx_sg); + + csbcpb->cpb.aes_gcm.bit_length_data = 0; + csbcpb->cpb.aes_gcm.bit_length_aad = 8 * nbytes; + + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP); + if (rc) + goto out; + + memcpy(csbcpb->cpb.aes_gcm.in_pat_or_aad, + csbcpb->cpb.aes_gcm.out_pat_or_mac, AES_BLOCK_SIZE); + memcpy(csbcpb->cpb.aes_gcm.in_s0, + csbcpb->cpb.aes_gcm.out_s0, AES_BLOCK_SIZE); + + NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; + + atomic_inc(&(nx_ctx->stats->aes_ops)); + atomic64_add(assoclen, &(nx_ctx->stats->aes_bytes)); + + processed += to_process; + } while (processed < nbytes); + +out: + /* Restore GCM mode */ + csbcpb->cpb.hdr.mode = NX_MODE_AES_GCM; + return rc; +} + +static int gcm_empty(struct aead_request *req, const u8 *iv, int enc) +{ + int rc; + struct nx_crypto_ctx *nx_ctx = + crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; + char out[AES_BLOCK_SIZE]; + struct nx_sg *in_sg, *out_sg; + int len; + + /* For scenarios where the input message is zero length, AES CTR mode + * may be used. Set the source data to be a single block (16B) of all + * zeros, and set the input IV value to be the same as the GMAC IV + * value. - nx_wb 4.8.1.3 */ + + /* Change to ECB mode */ + csbcpb->cpb.hdr.mode = NX_MODE_AES_ECB; + memcpy(csbcpb->cpb.aes_ecb.key, csbcpb->cpb.aes_gcm.key, + sizeof(csbcpb->cpb.aes_ecb.key)); + if (enc) + NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT; + else + NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT; + + len = AES_BLOCK_SIZE; + + /* Encrypt the counter/IV */ + in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *) iv, + &len, nx_ctx->ap->sglen); + + if (len != AES_BLOCK_SIZE) + return -EINVAL; + + len = sizeof(out); + out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *) out, &len, + nx_ctx->ap->sglen); + + if (len != sizeof(out)) + return -EINVAL; + + nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg); + nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg); + + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP); + if (rc) + goto out; + atomic_inc(&(nx_ctx->stats->aes_ops)); + + /* Copy out the auth tag */ + memcpy(csbcpb->cpb.aes_gcm.out_pat_or_mac, out, + crypto_aead_authsize(crypto_aead_reqtfm(req))); +out: + /* Restore XCBC mode */ + csbcpb->cpb.hdr.mode = NX_MODE_AES_GCM; + + /* + * ECB key uses the same region that GCM AAD and counter, so it's safe + * to just fill it with zeroes. + */ + memset(csbcpb->cpb.aes_ecb.key, 0, sizeof(csbcpb->cpb.aes_ecb.key)); + + return rc; +} + +static int gcm_aes_nx_crypt(struct aead_request *req, int enc, + unsigned int assoclen) +{ + struct nx_crypto_ctx *nx_ctx = + crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct nx_gcm_rctx *rctx = aead_request_ctx(req); + struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; + unsigned int nbytes = req->cryptlen; + unsigned int processed = 0, to_process; + unsigned long irq_flags; + int rc = -EINVAL; + + spin_lock_irqsave(&nx_ctx->lock, irq_flags); + + /* initialize the counter */ + *(u32 *)&rctx->iv[NX_GCM_CTR_OFFSET] = 1; + + if (nbytes == 0) { + if (assoclen == 0) + rc = gcm_empty(req, rctx->iv, enc); + else + rc = gmac(req, rctx->iv, assoclen); + if (rc) + goto out; + else + goto mac; + } + + /* Process associated data */ + csbcpb->cpb.aes_gcm.bit_length_aad = assoclen * 8; + if (assoclen) { + rc = nx_gca(nx_ctx, req, csbcpb->cpb.aes_gcm.in_pat_or_aad, + assoclen); + if (rc) + goto out; + } + + /* Set flags for encryption */ + NX_CPB_FDM(csbcpb) &= ~NX_FDM_CONTINUATION; + if (enc) { + NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT; + } else { + NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT; + nbytes -= crypto_aead_authsize(crypto_aead_reqtfm(req)); + } + + do { + to_process = nbytes - processed; + + csbcpb->cpb.aes_gcm.bit_length_data = nbytes * 8; + rc = nx_build_sg_lists(nx_ctx, rctx->iv, req->dst, + req->src, &to_process, + processed + req->assoclen, + csbcpb->cpb.aes_gcm.iv_or_cnt); + + if (rc) + goto out; + + if ((to_process + processed) < nbytes) + NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; + else + NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; + + + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP); + if (rc) + goto out; + + memcpy(rctx->iv, csbcpb->cpb.aes_gcm.out_cnt, AES_BLOCK_SIZE); + memcpy(csbcpb->cpb.aes_gcm.in_pat_or_aad, + csbcpb->cpb.aes_gcm.out_pat_or_mac, AES_BLOCK_SIZE); + memcpy(csbcpb->cpb.aes_gcm.in_s0, + csbcpb->cpb.aes_gcm.out_s0, AES_BLOCK_SIZE); + + NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; + + atomic_inc(&(nx_ctx->stats->aes_ops)); + atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count), + &(nx_ctx->stats->aes_bytes)); + + processed += to_process; + } while (processed < nbytes); + +mac: + if (enc) { + /* copy out the auth tag */ + memcpy_to_sglist( + req->dst, req->assoclen + nbytes, + csbcpb->cpb.aes_gcm.out_pat_or_mac, + crypto_aead_authsize(crypto_aead_reqtfm(req))); + } else { + u8 *itag = nx_ctx->priv.gcm.iauth_tag; + u8 *otag = csbcpb->cpb.aes_gcm.out_pat_or_mac; + + memcpy_from_sglist( + itag, req->src, req->assoclen + nbytes, + crypto_aead_authsize(crypto_aead_reqtfm(req))); + rc = crypto_memneq(itag, otag, + crypto_aead_authsize(crypto_aead_reqtfm(req))) ? + -EBADMSG : 0; + } +out: + spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); + return rc; +} + +static int gcm_aes_nx_encrypt(struct aead_request *req) +{ + struct nx_gcm_rctx *rctx = aead_request_ctx(req); + char *iv = rctx->iv; + + memcpy(iv, req->iv, GCM_AES_IV_SIZE); + + return gcm_aes_nx_crypt(req, 1, req->assoclen); +} + +static int gcm_aes_nx_decrypt(struct aead_request *req) +{ + struct nx_gcm_rctx *rctx = aead_request_ctx(req); + char *iv = rctx->iv; + + memcpy(iv, req->iv, GCM_AES_IV_SIZE); + + return gcm_aes_nx_crypt(req, 0, req->assoclen); +} + +static int gcm4106_aes_nx_encrypt(struct aead_request *req) +{ + struct nx_crypto_ctx *nx_ctx = + crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct nx_gcm_rctx *rctx = aead_request_ctx(req); + char *iv = rctx->iv; + char *nonce = nx_ctx->priv.gcm.nonce; + + memcpy(iv, nonce, NX_GCM4106_NONCE_LEN); + memcpy(iv + NX_GCM4106_NONCE_LEN, req->iv, 8); + + if (req->assoclen < 8) + return -EINVAL; + + return gcm_aes_nx_crypt(req, 1, req->assoclen - 8); +} + +static int gcm4106_aes_nx_decrypt(struct aead_request *req) +{ + struct nx_crypto_ctx *nx_ctx = + crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct nx_gcm_rctx *rctx = aead_request_ctx(req); + char *iv = rctx->iv; + char *nonce = nx_ctx->priv.gcm.nonce; + + memcpy(iv, nonce, NX_GCM4106_NONCE_LEN); + memcpy(iv + NX_GCM4106_NONCE_LEN, req->iv, 8); + + if (req->assoclen < 8) + return -EINVAL; + + return gcm_aes_nx_crypt(req, 0, req->assoclen - 8); +} + +struct aead_alg nx_gcm_aes_alg = { + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "gcm-aes-nx", + .cra_priority = 300, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct nx_crypto_ctx), + .cra_module = THIS_MODULE, + }, + .init = nx_crypto_ctx_aes_gcm_init, + .exit = nx_crypto_ctx_aead_exit, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .setkey = gcm_aes_nx_set_key, + .encrypt = gcm_aes_nx_encrypt, + .decrypt = gcm_aes_nx_decrypt, +}; + +struct aead_alg nx_gcm4106_aes_alg = { + .base = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "rfc4106-gcm-aes-nx", + .cra_priority = 300, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct nx_crypto_ctx), + .cra_module = THIS_MODULE, + }, + .init = nx_crypto_ctx_aes_gcm_init, + .exit = nx_crypto_ctx_aead_exit, + .ivsize = GCM_RFC4106_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .setkey = gcm4106_aes_nx_set_key, + .setauthsize = gcm4106_aes_nx_setauthsize, + .encrypt = gcm4106_aes_nx_encrypt, + .decrypt = gcm4106_aes_nx_decrypt, +}; diff --git a/drivers/crypto/nx/nx-aes-xcbc.c b/drivers/crypto/nx/nx-aes-xcbc.c new file mode 100644 index 000000000..bf465d824 --- /dev/null +++ b/drivers/crypto/nx/nx-aes-xcbc.c @@ -0,0 +1,331 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AES XCBC routines supporting the Power 7+ Nest Accelerators driver + * + * Copyright (C) 2011-2012 International Business Machines Inc. + * + * Author: Kent Yoder + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nx_csbcpb.h" +#include "nx.h" + + +struct xcbc_state { + u8 state[AES_BLOCK_SIZE]; +}; + +static int nx_xcbc_set_key(struct crypto_shash *desc, + const u8 *in_key, + unsigned int key_len) +{ + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc); + struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; + + switch (key_len) { + case AES_KEYSIZE_128: + nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128]; + break; + default: + return -EINVAL; + } + + memcpy(csbcpb->cpb.aes_xcbc.key, in_key, key_len); + + return 0; +} + +/* + * Based on RFC 3566, for a zero-length message: + * + * n = 1 + * K1 = E(K, 0x01010101010101010101010101010101) + * K3 = E(K, 0x03030303030303030303030303030303) + * E[0] = 0x00000000000000000000000000000000 + * M[1] = 0x80000000000000000000000000000000 (0 length message with padding) + * E[1] = (K1, M[1] ^ E[0] ^ K3) + * Tag = M[1] + */ +static int nx_xcbc_empty(struct shash_desc *desc, u8 *out) +{ + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); + struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; + struct nx_sg *in_sg, *out_sg; + u8 keys[2][AES_BLOCK_SIZE]; + u8 key[32]; + int rc = 0; + int len; + + /* Change to ECB mode */ + csbcpb->cpb.hdr.mode = NX_MODE_AES_ECB; + memcpy(key, csbcpb->cpb.aes_xcbc.key, AES_BLOCK_SIZE); + memcpy(csbcpb->cpb.aes_ecb.key, key, AES_BLOCK_SIZE); + NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT; + + /* K1 and K3 base patterns */ + memset(keys[0], 0x01, sizeof(keys[0])); + memset(keys[1], 0x03, sizeof(keys[1])); + + len = sizeof(keys); + /* Generate K1 and K3 encrypting the patterns */ + in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *) keys, &len, + nx_ctx->ap->sglen); + + if (len != sizeof(keys)) + return -EINVAL; + + out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *) keys, &len, + nx_ctx->ap->sglen); + + if (len != sizeof(keys)) + return -EINVAL; + + nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg); + nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg); + + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 0); + if (rc) + goto out; + atomic_inc(&(nx_ctx->stats->aes_ops)); + + /* XOr K3 with the padding for a 0 length message */ + keys[1][0] ^= 0x80; + + len = sizeof(keys[1]); + + /* Encrypt the final result */ + memcpy(csbcpb->cpb.aes_ecb.key, keys[0], AES_BLOCK_SIZE); + in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *) keys[1], &len, + nx_ctx->ap->sglen); + + if (len != sizeof(keys[1])) + return -EINVAL; + + len = AES_BLOCK_SIZE; + out_sg = nx_build_sg_list(nx_ctx->out_sg, out, &len, + nx_ctx->ap->sglen); + + if (len != AES_BLOCK_SIZE) + return -EINVAL; + + nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg); + nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg); + + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 0); + if (rc) + goto out; + atomic_inc(&(nx_ctx->stats->aes_ops)); + +out: + /* Restore XCBC mode */ + csbcpb->cpb.hdr.mode = NX_MODE_AES_XCBC_MAC; + memcpy(csbcpb->cpb.aes_xcbc.key, key, AES_BLOCK_SIZE); + NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT; + + return rc; +} + +static int nx_crypto_ctx_aes_xcbc_init2(struct crypto_shash *tfm) +{ + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(tfm); + struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; + int err; + + err = nx_crypto_ctx_aes_xcbc_init(tfm); + if (err) + return err; + + nx_ctx_init(nx_ctx, HCOP_FC_AES); + + NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128); + csbcpb->cpb.hdr.mode = NX_MODE_AES_XCBC_MAC; + + return 0; +} + +static int nx_xcbc_init(struct shash_desc *desc) +{ + struct xcbc_state *sctx = shash_desc_ctx(desc); + + memset(sctx, 0, sizeof *sctx); + + return 0; +} + +static int nx_xcbc_update(struct shash_desc *desc, + const u8 *data, + unsigned int len) +{ + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); + struct xcbc_state *sctx = shash_desc_ctx(desc); + struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; + struct nx_sg *in_sg; + struct nx_sg *out_sg; + unsigned int max_sg_len; + unsigned long irq_flags; + u32 to_process, total; + int rc = 0; + int data_len; + + spin_lock_irqsave(&nx_ctx->lock, irq_flags); + + memcpy(csbcpb->cpb.aes_xcbc.out_cv_mac, sctx->state, AES_BLOCK_SIZE); + NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; + NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; + + total = len; + + in_sg = nx_ctx->in_sg; + max_sg_len = min_t(u64, nx_driver.of.max_sg_len/sizeof(struct nx_sg), + nx_ctx->ap->sglen); + max_sg_len = min_t(u64, max_sg_len, + nx_ctx->ap->databytelen/NX_PAGE_SIZE); + + data_len = AES_BLOCK_SIZE; + out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *)sctx->state, + &data_len, nx_ctx->ap->sglen); + + if (data_len != AES_BLOCK_SIZE) { + rc = -EINVAL; + goto out; + } + + nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg); + + do { + to_process = total & ~(AES_BLOCK_SIZE - 1); + + in_sg = nx_build_sg_list(in_sg, + (u8 *) data, + &to_process, + max_sg_len); + + nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * + sizeof(struct nx_sg); + + /* we've hit the nx chip previously and we're updating again, + * so copy over the partial digest */ + memcpy(csbcpb->cpb.aes_xcbc.cv, + csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE); + + if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) { + rc = -EINVAL; + goto out; + } + + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 0); + if (rc) + goto out; + + atomic_inc(&(nx_ctx->stats->aes_ops)); + + total -= to_process; + data += to_process; + in_sg = nx_ctx->in_sg; + } while (total >= AES_BLOCK_SIZE); + + rc = total; + memcpy(sctx->state, csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE); + +out: + spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); + return rc; +} + +static int nx_xcbc_finup(struct shash_desc *desc, const u8 *src, + unsigned int nbytes, u8 *out) +{ + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); + struct xcbc_state *sctx = shash_desc_ctx(desc); + struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; + struct nx_sg *in_sg, *out_sg; + unsigned long irq_flags; + int rc = 0; + int len; + + spin_lock_irqsave(&nx_ctx->lock, irq_flags); + + if (nbytes) { + /* non-zero final, so copy over the partial digest */ + memcpy(csbcpb->cpb.aes_xcbc.cv, sctx->state, AES_BLOCK_SIZE); + } else { + /* + * we've never seen an update, so this is a 0 byte op. The + * hardware cannot handle a 0 byte op, so just ECB to + * generate the hash. + */ + rc = nx_xcbc_empty(desc, out); + goto out; + } + + /* final is represented by continuing the operation and indicating that + * this is not an intermediate operation */ + NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; + + len = nbytes; + in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)src, &len, + nx_ctx->ap->sglen); + + if (len != nbytes) { + rc = -EINVAL; + goto out; + } + + len = AES_BLOCK_SIZE; + out_sg = nx_build_sg_list(nx_ctx->out_sg, out, &len, + nx_ctx->ap->sglen); + + if (len != AES_BLOCK_SIZE) { + rc = -EINVAL; + goto out; + } + + nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg); + nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg); + + if (!nx_ctx->op.outlen) { + rc = -EINVAL; + goto out; + } + + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 0); + if (rc) + goto out; + + atomic_inc(&(nx_ctx->stats->aes_ops)); + + memcpy(out, csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE); +out: + spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); + return rc; +} + +struct shash_alg nx_shash_aes_xcbc_alg = { + .digestsize = AES_BLOCK_SIZE, + .init = nx_xcbc_init, + .update = nx_xcbc_update, + .finup = nx_xcbc_finup, + .setkey = nx_xcbc_set_key, + .descsize = sizeof(struct xcbc_state), + .init_tfm = nx_crypto_ctx_aes_xcbc_init2, + .exit_tfm = nx_crypto_ctx_shash_exit, + .base = { + .cra_name = "xcbc(aes)", + .cra_driver_name = "xcbc-aes-nx", + .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINAL_NONZERO, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct nx_crypto_ctx), + } +}; diff --git a/drivers/crypto/nx/nx-common-powernv.c b/drivers/crypto/nx/nx-common-powernv.c new file mode 100644 index 000000000..d6185740f --- /dev/null +++ b/drivers/crypto/nx/nx-common-powernv.c @@ -0,0 +1,1133 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Driver for IBM PowerNV compression accelerator + * + * Copyright (C) 2015 Dan Streetman, IBM Corp + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include "nx-842.h" + +#include +#include + +#include +#include +#include +#include +#include +#include + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Dan Streetman "); +MODULE_DESCRIPTION("H/W Compression driver for IBM PowerNV processors"); +MODULE_ALIAS_CRYPTO("842"); +MODULE_ALIAS_CRYPTO("842-nx"); + +#define WORKMEM_ALIGN (CRB_ALIGN) +#define CSB_WAIT_MAX (5000) /* ms */ +#define VAS_RETRIES (10) + +struct nx842_workmem { + /* Below fields must be properly aligned */ + struct coprocessor_request_block crb; /* CRB_ALIGN align */ + struct data_descriptor_entry ddl_in[DDL_LEN_MAX]; /* DDE_ALIGN align */ + struct data_descriptor_entry ddl_out[DDL_LEN_MAX]; /* DDE_ALIGN align */ + /* Above fields must be properly aligned */ + + ktime_t start; + + char padding[WORKMEM_ALIGN]; /* unused, to allow alignment */ +} __packed __aligned(WORKMEM_ALIGN); + +struct nx_coproc { + unsigned int chip_id; + unsigned int ct; /* Can be 842 or GZIP high/normal*/ + unsigned int ci; /* Coprocessor instance, used with icswx */ + struct { + struct vas_window *rxwin; + int id; + } vas; + struct list_head list; +}; + +/* + * Send the request to NX engine on the chip for the corresponding CPU + * where the process is executing. Use with VAS function. + */ +static DEFINE_PER_CPU(struct vas_window *, cpu_txwin); + +/* no cpu hotplug on powernv, so this list never changes after init */ +static LIST_HEAD(nx_coprocs); +static unsigned int nx842_ct; /* used in icswx function */ + +/* + * Using same values as in skiboot or coprocessor type representing + * in NX workbook. + */ +#define NX_CT_GZIP (2) /* on P9 and later */ +#define NX_CT_842 (3) + +static int (*nx842_powernv_exec)(const unsigned char *in, + unsigned int inlen, unsigned char *out, + unsigned int *outlenp, void *workmem, int fc); + +/* + * setup_indirect_dde - Setup an indirect DDE + * + * The DDE is setup with the DDE count, byte count, and address of + * first direct DDE in the list. + */ +static void setup_indirect_dde(struct data_descriptor_entry *dde, + struct data_descriptor_entry *ddl, + unsigned int dde_count, unsigned int byte_count) +{ + dde->flags = 0; + dde->count = dde_count; + dde->index = 0; + dde->length = cpu_to_be32(byte_count); + dde->address = cpu_to_be64(nx842_get_pa(ddl)); +} + +/* + * setup_direct_dde - Setup single DDE from buffer + * + * The DDE is setup with the buffer and length. The buffer must be properly + * aligned. The used length is returned. + * Returns: + * N Successfully set up DDE with N bytes + */ +static unsigned int setup_direct_dde(struct data_descriptor_entry *dde, + unsigned long pa, unsigned int len) +{ + unsigned int l = min_t(unsigned int, len, LEN_ON_PAGE(pa)); + + dde->flags = 0; + dde->count = 0; + dde->index = 0; + dde->length = cpu_to_be32(l); + dde->address = cpu_to_be64(pa); + + return l; +} + +/* + * setup_ddl - Setup DDL from buffer + * + * Returns: + * 0 Successfully set up DDL + */ +static int setup_ddl(struct data_descriptor_entry *dde, + struct data_descriptor_entry *ddl, + unsigned char *buf, unsigned int len, + bool in) +{ + unsigned long pa = nx842_get_pa(buf); + int i, ret, total_len = len; + + if (!IS_ALIGNED(pa, DDE_BUFFER_ALIGN)) { + pr_debug("%s buffer pa 0x%lx not 0x%x-byte aligned\n", + in ? "input" : "output", pa, DDE_BUFFER_ALIGN); + return -EINVAL; + } + + /* only need to check last mult; since buffer must be + * DDE_BUFFER_ALIGN aligned, and that is a multiple of + * DDE_BUFFER_SIZE_MULT, and pre-last page DDE buffers + * are guaranteed a multiple of DDE_BUFFER_SIZE_MULT. + */ + if (len % DDE_BUFFER_LAST_MULT) { + pr_debug("%s buffer len 0x%x not a multiple of 0x%x\n", + in ? "input" : "output", len, DDE_BUFFER_LAST_MULT); + if (in) + return -EINVAL; + len = round_down(len, DDE_BUFFER_LAST_MULT); + } + + /* use a single direct DDE */ + if (len <= LEN_ON_PAGE(pa)) { + ret = setup_direct_dde(dde, pa, len); + WARN_ON(ret < len); + return 0; + } + + /* use the DDL */ + for (i = 0; i < DDL_LEN_MAX && len > 0; i++) { + ret = setup_direct_dde(&ddl[i], pa, len); + buf += ret; + len -= ret; + pa = nx842_get_pa(buf); + } + + if (len > 0) { + pr_debug("0x%x total %s bytes 0x%x too many for DDL.\n", + total_len, in ? "input" : "output", len); + if (in) + return -EMSGSIZE; + total_len -= len; + } + setup_indirect_dde(dde, ddl, i, total_len); + + return 0; +} + +#define CSB_ERR(csb, msg, ...) \ + pr_err("ERROR: " msg " : %02x %02x %02x %02x %08x\n", \ + ##__VA_ARGS__, (csb)->flags, \ + (csb)->cs, (csb)->cc, (csb)->ce, \ + be32_to_cpu((csb)->count)) + +#define CSB_ERR_ADDR(csb, msg, ...) \ + CSB_ERR(csb, msg " at %lx", ##__VA_ARGS__, \ + (unsigned long)be64_to_cpu((csb)->address)) + +static int wait_for_csb(struct nx842_workmem *wmem, + struct coprocessor_status_block *csb) +{ + ktime_t start = wmem->start, now = ktime_get(); + ktime_t timeout = ktime_add_ms(start, CSB_WAIT_MAX); + + while (!(READ_ONCE(csb->flags) & CSB_V)) { + cpu_relax(); + now = ktime_get(); + if (ktime_after(now, timeout)) + break; + } + + /* hw has updated csb and output buffer */ + barrier(); + + /* check CSB flags */ + if (!(csb->flags & CSB_V)) { + CSB_ERR(csb, "CSB still not valid after %ld us, giving up", + (long)ktime_us_delta(now, start)); + return -ETIMEDOUT; + } + if (csb->flags & CSB_F) { + CSB_ERR(csb, "Invalid CSB format"); + return -EPROTO; + } + if (csb->flags & CSB_CH) { + CSB_ERR(csb, "Invalid CSB chaining state"); + return -EPROTO; + } + + /* verify CSB completion sequence is 0 */ + if (csb->cs) { + CSB_ERR(csb, "Invalid CSB completion sequence"); + return -EPROTO; + } + + /* check CSB Completion Code */ + switch (csb->cc) { + /* no error */ + case CSB_CC_SUCCESS: + break; + case CSB_CC_TPBC_GT_SPBC: + /* not an error, but the compressed data is + * larger than the uncompressed data :( + */ + break; + + /* input data errors */ + case CSB_CC_OPERAND_OVERLAP: + /* input and output buffers overlap */ + CSB_ERR(csb, "Operand Overlap error"); + return -EINVAL; + case CSB_CC_INVALID_OPERAND: + CSB_ERR(csb, "Invalid operand"); + return -EINVAL; + case CSB_CC_NOSPC: + /* output buffer too small */ + return -ENOSPC; + case CSB_CC_ABORT: + CSB_ERR(csb, "Function aborted"); + return -EINTR; + case CSB_CC_CRC_MISMATCH: + CSB_ERR(csb, "CRC mismatch"); + return -EINVAL; + case CSB_CC_TEMPL_INVALID: + CSB_ERR(csb, "Compressed data template invalid"); + return -EINVAL; + case CSB_CC_TEMPL_OVERFLOW: + CSB_ERR(csb, "Compressed data template shows data past end"); + return -EINVAL; + case CSB_CC_EXCEED_BYTE_COUNT: /* P9 or later */ + /* + * DDE byte count exceeds the limit specified in Maximum + * byte count register. + */ + CSB_ERR(csb, "DDE byte count exceeds the limit"); + return -EINVAL; + + /* these should not happen */ + case CSB_CC_INVALID_ALIGN: + /* setup_ddl should have detected this */ + CSB_ERR_ADDR(csb, "Invalid alignment"); + return -EINVAL; + case CSB_CC_DATA_LENGTH: + /* setup_ddl should have detected this */ + CSB_ERR(csb, "Invalid data length"); + return -EINVAL; + case CSB_CC_WR_TRANSLATION: + case CSB_CC_TRANSLATION: + case CSB_CC_TRANSLATION_DUP1: + case CSB_CC_TRANSLATION_DUP2: + case CSB_CC_TRANSLATION_DUP3: + case CSB_CC_TRANSLATION_DUP4: + case CSB_CC_TRANSLATION_DUP5: + case CSB_CC_TRANSLATION_DUP6: + /* should not happen, we use physical addrs */ + CSB_ERR_ADDR(csb, "Translation error"); + return -EPROTO; + case CSB_CC_WR_PROTECTION: + case CSB_CC_PROTECTION: + case CSB_CC_PROTECTION_DUP1: + case CSB_CC_PROTECTION_DUP2: + case CSB_CC_PROTECTION_DUP3: + case CSB_CC_PROTECTION_DUP4: + case CSB_CC_PROTECTION_DUP5: + case CSB_CC_PROTECTION_DUP6: + /* should not happen, we use physical addrs */ + CSB_ERR_ADDR(csb, "Protection error"); + return -EPROTO; + case CSB_CC_PRIVILEGE: + /* shouldn't happen, we're in HYP mode */ + CSB_ERR(csb, "Insufficient Privilege error"); + return -EPROTO; + case CSB_CC_EXCESSIVE_DDE: + /* shouldn't happen, setup_ddl doesn't use many dde's */ + CSB_ERR(csb, "Too many DDEs in DDL"); + return -EINVAL; + case CSB_CC_TRANSPORT: + case CSB_CC_INVALID_CRB: /* P9 or later */ + /* shouldn't happen, we setup CRB correctly */ + CSB_ERR(csb, "Invalid CRB"); + return -EINVAL; + case CSB_CC_INVALID_DDE: /* P9 or later */ + /* + * shouldn't happen, setup_direct/indirect_dde creates + * DDE right + */ + CSB_ERR(csb, "Invalid DDE"); + return -EINVAL; + case CSB_CC_SEGMENTED_DDL: + /* shouldn't happen, setup_ddl creates DDL right */ + CSB_ERR(csb, "Segmented DDL error"); + return -EINVAL; + case CSB_CC_DDE_OVERFLOW: + /* shouldn't happen, setup_ddl creates DDL right */ + CSB_ERR(csb, "DDE overflow error"); + return -EINVAL; + case CSB_CC_SESSION: + /* should not happen with ICSWX */ + CSB_ERR(csb, "Session violation error"); + return -EPROTO; + case CSB_CC_CHAIN: + /* should not happen, we don't use chained CRBs */ + CSB_ERR(csb, "Chained CRB error"); + return -EPROTO; + case CSB_CC_SEQUENCE: + /* should not happen, we don't use chained CRBs */ + CSB_ERR(csb, "CRB sequence number error"); + return -EPROTO; + case CSB_CC_UNKNOWN_CODE: + CSB_ERR(csb, "Unknown subfunction code"); + return -EPROTO; + + /* hardware errors */ + case CSB_CC_RD_EXTERNAL: + case CSB_CC_RD_EXTERNAL_DUP1: + case CSB_CC_RD_EXTERNAL_DUP2: + case CSB_CC_RD_EXTERNAL_DUP3: + CSB_ERR_ADDR(csb, "Read error outside coprocessor"); + return -EPROTO; + case CSB_CC_WR_EXTERNAL: + CSB_ERR_ADDR(csb, "Write error outside coprocessor"); + return -EPROTO; + case CSB_CC_INTERNAL: + CSB_ERR(csb, "Internal error in coprocessor"); + return -EPROTO; + case CSB_CC_PROVISION: + CSB_ERR(csb, "Storage provision error"); + return -EPROTO; + case CSB_CC_HW: + CSB_ERR(csb, "Correctable hardware error"); + return -EPROTO; + case CSB_CC_HW_EXPIRED_TIMER: /* P9 or later */ + CSB_ERR(csb, "Job did not finish within allowed time"); + return -EPROTO; + + default: + CSB_ERR(csb, "Invalid CC %d", csb->cc); + return -EPROTO; + } + + /* check Completion Extension state */ + if (csb->ce & CSB_CE_TERMINATION) { + CSB_ERR(csb, "CSB request was terminated"); + return -EPROTO; + } + if (csb->ce & CSB_CE_INCOMPLETE) { + CSB_ERR(csb, "CSB request not complete"); + return -EPROTO; + } + if (!(csb->ce & CSB_CE_TPBC)) { + CSB_ERR(csb, "TPBC not provided, unknown target length"); + return -EPROTO; + } + + /* successful completion */ + pr_debug_ratelimited("Processed %u bytes in %lu us\n", + be32_to_cpu(csb->count), + (unsigned long)ktime_us_delta(now, start)); + + return 0; +} + +static int nx842_config_crb(const unsigned char *in, unsigned int inlen, + unsigned char *out, unsigned int outlen, + struct nx842_workmem *wmem) +{ + struct coprocessor_request_block *crb; + struct coprocessor_status_block *csb; + u64 csb_addr; + int ret; + + crb = &wmem->crb; + csb = &crb->csb; + + /* Clear any previous values */ + memset(crb, 0, sizeof(*crb)); + + /* set up DDLs */ + ret = setup_ddl(&crb->source, wmem->ddl_in, + (unsigned char *)in, inlen, true); + if (ret) + return ret; + + ret = setup_ddl(&crb->target, wmem->ddl_out, + out, outlen, false); + if (ret) + return ret; + + /* set up CRB's CSB addr */ + csb_addr = nx842_get_pa(csb) & CRB_CSB_ADDRESS; + csb_addr |= CRB_CSB_AT; /* Addrs are phys */ + crb->csb_addr = cpu_to_be64(csb_addr); + + return 0; +} + +/** + * nx842_exec_icswx - compress/decompress data using the 842 algorithm + * + * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems. + * This compresses or decompresses the provided input buffer into the provided + * output buffer. + * + * Upon return from this function @outlen contains the length of the + * output data. If there is an error then @outlen will be 0 and an + * error will be specified by the return code from this function. + * + * The @workmem buffer should only be used by one function call at a time. + * + * @in: input buffer pointer + * @inlen: input buffer size + * @out: output buffer pointer + * @outlenp: output buffer size pointer + * @workmem: working memory buffer pointer, size determined by + * nx842_powernv_driver.workmem_size + * @fc: function code, see CCW Function Codes in nx-842.h + * + * Returns: + * 0 Success, output of length @outlenp stored in the buffer at @out + * -ENODEV Hardware unavailable + * -ENOSPC Output buffer is to small + * -EMSGSIZE Input buffer too large + * -EINVAL buffer constraints do not fix nx842_constraints + * -EPROTO hardware error during operation + * -ETIMEDOUT hardware did not complete operation in reasonable time + * -EINTR operation was aborted + */ +static int nx842_exec_icswx(const unsigned char *in, unsigned int inlen, + unsigned char *out, unsigned int *outlenp, + void *workmem, int fc) +{ + struct coprocessor_request_block *crb; + struct coprocessor_status_block *csb; + struct nx842_workmem *wmem; + int ret; + u32 ccw; + unsigned int outlen = *outlenp; + + wmem = PTR_ALIGN(workmem, WORKMEM_ALIGN); + + *outlenp = 0; + + /* shoudn't happen, we don't load without a coproc */ + if (!nx842_ct) { + pr_err_ratelimited("coprocessor CT is 0"); + return -ENODEV; + } + + ret = nx842_config_crb(in, inlen, out, outlen, wmem); + if (ret) + return ret; + + crb = &wmem->crb; + csb = &crb->csb; + + /* set up CCW */ + ccw = 0; + ccw = SET_FIELD(CCW_CT, ccw, nx842_ct); + ccw = SET_FIELD(CCW_CI_842, ccw, 0); /* use 0 for hw auto-selection */ + ccw = SET_FIELD(CCW_FC_842, ccw, fc); + + wmem->start = ktime_get(); + + /* do ICSWX */ + ret = icswx(cpu_to_be32(ccw), crb); + + pr_debug_ratelimited("icswx CR %x ccw %x crb->ccw %x\n", ret, + (unsigned int)ccw, + (unsigned int)be32_to_cpu(crb->ccw)); + + /* + * NX842 coprocessor sets 3rd bit in CR register with XER[S0]. + * XER[S0] is the integer summary overflow bit which is nothing + * to do NX. Since this bit can be set with other return values, + * mask this bit. + */ + ret &= ~ICSWX_XERS0; + + switch (ret) { + case ICSWX_INITIATED: + ret = wait_for_csb(wmem, csb); + break; + case ICSWX_BUSY: + pr_debug_ratelimited("842 Coprocessor busy\n"); + ret = -EBUSY; + break; + case ICSWX_REJECTED: + pr_err_ratelimited("ICSWX rejected\n"); + ret = -EPROTO; + break; + } + + if (!ret) + *outlenp = be32_to_cpu(csb->count); + + return ret; +} + +/** + * nx842_exec_vas - compress/decompress data using the 842 algorithm + * + * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems. + * This compresses or decompresses the provided input buffer into the provided + * output buffer. + * + * Upon return from this function @outlen contains the length of the + * output data. If there is an error then @outlen will be 0 and an + * error will be specified by the return code from this function. + * + * The @workmem buffer should only be used by one function call at a time. + * + * @in: input buffer pointer + * @inlen: input buffer size + * @out: output buffer pointer + * @outlenp: output buffer size pointer + * @workmem: working memory buffer pointer, size determined by + * nx842_powernv_driver.workmem_size + * @fc: function code, see CCW Function Codes in nx-842.h + * + * Returns: + * 0 Success, output of length @outlenp stored in the buffer + * at @out + * -ENODEV Hardware unavailable + * -ENOSPC Output buffer is to small + * -EMSGSIZE Input buffer too large + * -EINVAL buffer constraints do not fix nx842_constraints + * -EPROTO hardware error during operation + * -ETIMEDOUT hardware did not complete operation in reasonable time + * -EINTR operation was aborted + */ +static int nx842_exec_vas(const unsigned char *in, unsigned int inlen, + unsigned char *out, unsigned int *outlenp, + void *workmem, int fc) +{ + struct coprocessor_request_block *crb; + struct coprocessor_status_block *csb; + struct nx842_workmem *wmem; + struct vas_window *txwin; + int ret, i = 0; + u32 ccw; + unsigned int outlen = *outlenp; + + wmem = PTR_ALIGN(workmem, WORKMEM_ALIGN); + + *outlenp = 0; + + crb = &wmem->crb; + csb = &crb->csb; + + ret = nx842_config_crb(in, inlen, out, outlen, wmem); + if (ret) + return ret; + + ccw = 0; + ccw = SET_FIELD(CCW_FC_842, ccw, fc); + crb->ccw = cpu_to_be32(ccw); + + do { + wmem->start = ktime_get(); + preempt_disable(); + txwin = this_cpu_read(cpu_txwin); + + /* + * VAS copy CRB into L2 cache. Refer . + * @crb and @offset. + */ + vas_copy_crb(crb, 0); + + /* + * VAS paste previously copied CRB to NX. + * @txwin, @offset and @last (must be true). + */ + ret = vas_paste_crb(txwin, 0, 1); + preempt_enable(); + /* + * Retry copy/paste function for VAS failures. + */ + } while (ret && (i++ < VAS_RETRIES)); + + if (ret) { + pr_err_ratelimited("VAS copy/paste failed\n"); + return ret; + } + + ret = wait_for_csb(wmem, csb); + if (!ret) + *outlenp = be32_to_cpu(csb->count); + + return ret; +} + +/** + * nx842_powernv_compress - Compress data using the 842 algorithm + * + * Compression provided by the NX842 coprocessor on IBM PowerNV systems. + * The input buffer is compressed and the result is stored in the + * provided output buffer. + * + * Upon return from this function @outlen contains the length of the + * compressed data. If there is an error then @outlen will be 0 and an + * error will be specified by the return code from this function. + * + * @in: input buffer pointer + * @inlen: input buffer size + * @out: output buffer pointer + * @outlenp: output buffer size pointer + * @wmem: working memory buffer pointer, size determined by + * nx842_powernv_driver.workmem_size + * + * Returns: see @nx842_powernv_exec() + */ +static int nx842_powernv_compress(const unsigned char *in, unsigned int inlen, + unsigned char *out, unsigned int *outlenp, + void *wmem) +{ + return nx842_powernv_exec(in, inlen, out, outlenp, + wmem, CCW_FC_842_COMP_CRC); +} + +/** + * nx842_powernv_decompress - Decompress data using the 842 algorithm + * + * Decompression provided by the NX842 coprocessor on IBM PowerNV systems. + * The input buffer is decompressed and the result is stored in the + * provided output buffer. + * + * Upon return from this function @outlen contains the length of the + * decompressed data. If there is an error then @outlen will be 0 and an + * error will be specified by the return code from this function. + * + * @in: input buffer pointer + * @inlen: input buffer size + * @out: output buffer pointer + * @outlenp: output buffer size pointer + * @wmem: working memory buffer pointer, size determined by + * nx842_powernv_driver.workmem_size + * + * Returns: see @nx842_powernv_exec() + */ +static int nx842_powernv_decompress(const unsigned char *in, unsigned int inlen, + unsigned char *out, unsigned int *outlenp, + void *wmem) +{ + return nx842_powernv_exec(in, inlen, out, outlenp, + wmem, CCW_FC_842_DECOMP_CRC); +} + +static inline void nx_add_coprocs_list(struct nx_coproc *coproc, + int chipid) +{ + coproc->chip_id = chipid; + INIT_LIST_HEAD(&coproc->list); + list_add(&coproc->list, &nx_coprocs); +} + +static struct vas_window *nx_alloc_txwin(struct nx_coproc *coproc) +{ + struct vas_window *txwin = NULL; + struct vas_tx_win_attr txattr; + + /* + * Kernel requests will be high priority. So open send + * windows only for high priority RxFIFO entries. + */ + vas_init_tx_win_attr(&txattr, coproc->ct); + txattr.lpid = 0; /* lpid is 0 for kernel requests */ + + /* + * Open a VAS send window which is used to send request to NX. + */ + txwin = vas_tx_win_open(coproc->vas.id, coproc->ct, &txattr); + if (IS_ERR(txwin)) + pr_err("ibm,nx-842: Can not open TX window: %ld\n", + PTR_ERR(txwin)); + + return txwin; +} + +/* + * Identify chip ID for each CPU, open send wndow for the corresponding NX + * engine and save txwin in percpu cpu_txwin. + * cpu_txwin is used in copy/paste operation for each compression / + * decompression request. + */ +static int nx_open_percpu_txwins(void) +{ + struct nx_coproc *coproc, *n; + unsigned int i, chip_id; + + for_each_possible_cpu(i) { + struct vas_window *txwin = NULL; + + chip_id = cpu_to_chip_id(i); + + list_for_each_entry_safe(coproc, n, &nx_coprocs, list) { + /* + * Kernel requests use only high priority FIFOs. So + * open send windows for these FIFOs. + * GZIP is not supported in kernel right now. + */ + + if (coproc->ct != VAS_COP_TYPE_842_HIPRI) + continue; + + if (coproc->chip_id == chip_id) { + txwin = nx_alloc_txwin(coproc); + if (IS_ERR(txwin)) + return PTR_ERR(txwin); + + per_cpu(cpu_txwin, i) = txwin; + break; + } + } + + if (!per_cpu(cpu_txwin, i)) { + /* shouldn't happen, Each chip will have NX engine */ + pr_err("NX engine is not available for CPU %d\n", i); + return -EINVAL; + } + } + + return 0; +} + +static int __init nx_set_ct(struct nx_coproc *coproc, const char *priority, + int high, int normal) +{ + if (!strcmp(priority, "High")) + coproc->ct = high; + else if (!strcmp(priority, "Normal")) + coproc->ct = normal; + else { + pr_err("Invalid RxFIFO priority value\n"); + return -EINVAL; + } + + return 0; +} + +static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id, + int vasid, int type, int *ct) +{ + struct vas_window *rxwin = NULL; + struct vas_rx_win_attr rxattr; + u32 lpid, pid, tid, fifo_size; + struct nx_coproc *coproc; + u64 rx_fifo; + const char *priority; + int ret; + + ret = of_property_read_u64(dn, "rx-fifo-address", &rx_fifo); + if (ret) { + pr_err("Missing rx-fifo-address property\n"); + return ret; + } + + ret = of_property_read_u32(dn, "rx-fifo-size", &fifo_size); + if (ret) { + pr_err("Missing rx-fifo-size property\n"); + return ret; + } + + ret = of_property_read_u32(dn, "lpid", &lpid); + if (ret) { + pr_err("Missing lpid property\n"); + return ret; + } + + ret = of_property_read_u32(dn, "pid", &pid); + if (ret) { + pr_err("Missing pid property\n"); + return ret; + } + + ret = of_property_read_u32(dn, "tid", &tid); + if (ret) { + pr_err("Missing tid property\n"); + return ret; + } + + ret = of_property_read_string(dn, "priority", &priority); + if (ret) { + pr_err("Missing priority property\n"); + return ret; + } + + coproc = kzalloc_obj(*coproc); + if (!coproc) + return -ENOMEM; + + if (type == NX_CT_842) + ret = nx_set_ct(coproc, priority, VAS_COP_TYPE_842_HIPRI, + VAS_COP_TYPE_842); + else if (type == NX_CT_GZIP) + ret = nx_set_ct(coproc, priority, VAS_COP_TYPE_GZIP_HIPRI, + VAS_COP_TYPE_GZIP); + + if (ret) + goto err_out; + + vas_init_rx_win_attr(&rxattr, coproc->ct); + rxattr.rx_fifo = rx_fifo; + rxattr.rx_fifo_size = fifo_size; + rxattr.lnotify_lpid = lpid; + rxattr.lnotify_pid = pid; + rxattr.lnotify_tid = tid; + /* + * Maximum RX window credits can not be more than #CRBs in + * RxFIFO. Otherwise, can get checkstop if RxFIFO overruns. + */ + rxattr.wcreds_max = fifo_size / CRB_SIZE; + + /* + * Open a VAS receice window which is used to configure RxFIFO + * for NX. + */ + rxwin = vas_rx_win_open(vasid, coproc->ct, &rxattr); + if (IS_ERR(rxwin)) { + ret = PTR_ERR(rxwin); + pr_err("setting RxFIFO with VAS failed: %d\n", + ret); + goto err_out; + } + + coproc->vas.rxwin = rxwin; + coproc->vas.id = vasid; + nx_add_coprocs_list(coproc, chip_id); + + /* + * (lpid, pid, tid) combination has to be unique for each + * coprocessor instance in the system. So to make it + * unique, skiboot uses coprocessor type such as 842 or + * GZIP for pid and provides this value to kernel in pid + * device-tree property. + */ + *ct = pid; + + return 0; + +err_out: + kfree(coproc); + return ret; +} + +static int __init nx_coproc_init(int chip_id, int ct_842, int ct_gzip) +{ + int ret = 0; + + if (opal_check_token(OPAL_NX_COPROC_INIT)) { + ret = opal_nx_coproc_init(chip_id, ct_842); + + if (!ret) + ret = opal_nx_coproc_init(chip_id, ct_gzip); + + if (ret) { + ret = opal_error_code(ret); + pr_err("Failed to initialize NX for chip(%d): %d\n", + chip_id, ret); + } + } else + pr_warn("Firmware doesn't support NX initialization\n"); + + return ret; +} + +static int __init find_nx_device_tree(struct device_node *dn, int chip_id, + int vasid, int type, char *devname, + int *ct) +{ + int ret = 0; + + if (of_device_is_compatible(dn, devname)) { + ret = vas_cfg_coproc_info(dn, chip_id, vasid, type, ct); + if (ret) + of_node_put(dn); + } + + return ret; +} + +static int __init nx_powernv_probe_vas(struct device_node *pn) +{ + int chip_id, vasid, ret = 0; + int ct_842 = 0, ct_gzip = 0; + + chip_id = of_get_ibm_chip_id(pn); + if (chip_id < 0) { + pr_err("ibm,chip-id missing\n"); + return -EINVAL; + } + + vasid = chip_to_vas_id(chip_id); + if (vasid < 0) { + pr_err("Unable to map chip_id %d to vasid\n", chip_id); + return -EINVAL; + } + + for_each_child_of_node_scoped(pn, dn) { + ret = find_nx_device_tree(dn, chip_id, vasid, NX_CT_842, + "ibm,p9-nx-842", &ct_842); + + if (!ret) + ret = find_nx_device_tree(dn, chip_id, vasid, + NX_CT_GZIP, "ibm,p9-nx-gzip", &ct_gzip); + + if (ret) + return ret; + } + + if (!ct_842 || !ct_gzip) { + pr_err("NX FIFO nodes are missing\n"); + return -EINVAL; + } + + /* + * Initialize NX instance for both high and normal priority FIFOs. + */ + ret = nx_coproc_init(chip_id, ct_842, ct_gzip); + + return ret; +} + +static int __init nx842_powernv_probe(struct device_node *dn) +{ + struct nx_coproc *coproc; + unsigned int ct, ci; + int chip_id; + + chip_id = of_get_ibm_chip_id(dn); + if (chip_id < 0) { + pr_err("ibm,chip-id missing\n"); + return -EINVAL; + } + + if (of_property_read_u32(dn, "ibm,842-coprocessor-type", &ct)) { + pr_err("ibm,842-coprocessor-type missing\n"); + return -EINVAL; + } + + if (of_property_read_u32(dn, "ibm,842-coprocessor-instance", &ci)) { + pr_err("ibm,842-coprocessor-instance missing\n"); + return -EINVAL; + } + + coproc = kzalloc_obj(*coproc); + if (!coproc) + return -ENOMEM; + + coproc->ct = ct; + coproc->ci = ci; + nx_add_coprocs_list(coproc, chip_id); + + pr_info("coprocessor found on chip %d, CT %d CI %d\n", chip_id, ct, ci); + + if (!nx842_ct) + nx842_ct = ct; + else if (nx842_ct != ct) + pr_err("NX842 chip %d, CT %d != first found CT %d\n", + chip_id, ct, nx842_ct); + + return 0; +} + +static void nx_delete_coprocs(void) +{ + struct nx_coproc *coproc, *n; + struct vas_window *txwin; + int i; + + /* + * close percpu txwins that are opened for the corresponding coproc. + */ + for_each_possible_cpu(i) { + txwin = per_cpu(cpu_txwin, i); + if (txwin) + vas_win_close(txwin); + + per_cpu(cpu_txwin, i) = NULL; + } + + list_for_each_entry_safe(coproc, n, &nx_coprocs, list) { + if (coproc->vas.rxwin) + vas_win_close(coproc->vas.rxwin); + + list_del(&coproc->list); + kfree(coproc); + } +} + +static struct nx842_constraints nx842_powernv_constraints = { + .alignment = DDE_BUFFER_ALIGN, + .multiple = DDE_BUFFER_LAST_MULT, + .minimum = DDE_BUFFER_LAST_MULT, + .maximum = (DDL_LEN_MAX - 1) * PAGE_SIZE, +}; + +static struct nx842_driver nx842_powernv_driver = { + .name = KBUILD_MODNAME, + .owner = THIS_MODULE, + .workmem_size = sizeof(struct nx842_workmem), + .constraints = &nx842_powernv_constraints, + .compress = nx842_powernv_compress, + .decompress = nx842_powernv_decompress, +}; + +static void *nx842_powernv_crypto_alloc_ctx(void) +{ + return nx842_crypto_alloc_ctx(&nx842_powernv_driver); +} + +static struct scomp_alg nx842_powernv_alg = { + .base.cra_name = "842", + .base.cra_driver_name = "842-nx", + .base.cra_priority = 300, + .base.cra_module = THIS_MODULE, + + .streams = { + .alloc_ctx = nx842_powernv_crypto_alloc_ctx, + .free_ctx = nx842_crypto_free_ctx, + }, + .compress = nx842_crypto_compress, + .decompress = nx842_crypto_decompress, +}; + +static __init int nx_compress_powernv_init(void) +{ + struct device_node *dn; + int ret; + + /* verify workmem size/align restrictions */ + BUILD_BUG_ON(WORKMEM_ALIGN % CRB_ALIGN); + BUILD_BUG_ON(CRB_ALIGN % DDE_ALIGN); + BUILD_BUG_ON(CRB_SIZE % DDE_ALIGN); + /* verify buffer size/align restrictions */ + BUILD_BUG_ON(PAGE_SIZE % DDE_BUFFER_ALIGN); + BUILD_BUG_ON(DDE_BUFFER_ALIGN % DDE_BUFFER_SIZE_MULT); + BUILD_BUG_ON(DDE_BUFFER_SIZE_MULT % DDE_BUFFER_LAST_MULT); + + for_each_compatible_node(dn, NULL, "ibm,power9-nx") { + ret = nx_powernv_probe_vas(dn); + if (ret) { + nx_delete_coprocs(); + of_node_put(dn); + return ret; + } + } + + if (list_empty(&nx_coprocs)) { + for_each_compatible_node(dn, NULL, "ibm,power-nx") + nx842_powernv_probe(dn); + + if (!nx842_ct) + return -ENODEV; + + nx842_powernv_exec = nx842_exec_icswx; + } else { + /* + * Register VAS user space API for NX GZIP so + * that user space can use GZIP engine. + * Using high FIFO priority for kernel requests and + * normal FIFO priority is assigned for userspace. + * 842 compression is supported only in kernel. + */ + ret = vas_register_api_powernv(THIS_MODULE, VAS_COP_TYPE_GZIP, + "nx-gzip"); + + /* + * GZIP is not supported in kernel right now. + * So open tx windows only for 842. + */ + if (!ret) + ret = nx_open_percpu_txwins(); + + if (ret) { + nx_delete_coprocs(); + return ret; + } + + nx842_powernv_exec = nx842_exec_vas; + } + + ret = crypto_register_scomp(&nx842_powernv_alg); + if (ret) { + nx_delete_coprocs(); + return ret; + } + + return 0; +} +module_init(nx_compress_powernv_init); + +static void __exit nx_compress_powernv_exit(void) +{ + /* + * GZIP engine is supported only in power9 or later and nx842_ct + * is used on power8 (icswx). + * VAS API for NX GZIP is registered during init for user space + * use. So delete this API use for GZIP engine. + */ + if (!nx842_ct) + vas_unregister_api_powernv(); + + crypto_unregister_scomp(&nx842_powernv_alg); + + nx_delete_coprocs(); +} +module_exit(nx_compress_powernv_exit); diff --git a/drivers/crypto/nx/nx-common-pseries.c b/drivers/crypto/nx/nx-common-pseries.c new file mode 100644 index 000000000..a6e65c3d2 --- /dev/null +++ b/drivers/crypto/nx/nx-common-pseries.c @@ -0,0 +1,1271 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Driver for IBM Power 842 compression accelerator + * + * Copyright (C) IBM Corporation, 2012 + * + * Authors: Robert Jennings + * Seth Jennings + */ + +#include +#include +#include +#include + +#include "nx-842.h" +#include "nx_csbcpb.h" /* struct nx_csbcpb */ + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Robert Jennings "); +MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors"); +MODULE_ALIAS_CRYPTO("842"); +MODULE_ALIAS_CRYPTO("842-nx"); + +/* + * Coprocessor type specific capabilities from the hypervisor. + */ +struct hv_nx_cop_caps { + __be64 descriptor; + __be64 req_max_processed_len; /* Max bytes in one GZIP request */ + __be64 min_compress_len; /* Min compression size in bytes */ + __be64 min_decompress_len; /* Min decompression size in bytes */ +} __packed __aligned(0x1000); + +/* + * Coprocessor type specific capabilities. + */ +struct nx_cop_caps { + u64 descriptor; + u64 req_max_processed_len; /* Max bytes in one GZIP request */ + u64 min_compress_len; /* Min compression in bytes */ + u64 min_decompress_len; /* Min decompression in bytes */ +}; + +static u64 caps_feat; +static struct nx_cop_caps nx_cop_caps; + +static struct nx842_constraints nx842_pseries_constraints = { + .alignment = DDE_BUFFER_ALIGN, + .multiple = DDE_BUFFER_LAST_MULT, + .minimum = DDE_BUFFER_LAST_MULT, + .maximum = PAGE_SIZE, /* dynamic, max_sync_size */ +}; + +static int check_constraints(unsigned long buf, unsigned int *len, bool in) +{ + if (!IS_ALIGNED(buf, nx842_pseries_constraints.alignment)) { + pr_debug("%s buffer 0x%lx not aligned to 0x%x\n", + in ? "input" : "output", buf, + nx842_pseries_constraints.alignment); + return -EINVAL; + } + if (*len % nx842_pseries_constraints.multiple) { + pr_debug("%s buffer len 0x%x not multiple of 0x%x\n", + in ? "input" : "output", *len, + nx842_pseries_constraints.multiple); + if (in) + return -EINVAL; + *len = round_down(*len, nx842_pseries_constraints.multiple); + } + if (*len < nx842_pseries_constraints.minimum) { + pr_debug("%s buffer len 0x%x under minimum 0x%x\n", + in ? "input" : "output", *len, + nx842_pseries_constraints.minimum); + return -EINVAL; + } + if (*len > nx842_pseries_constraints.maximum) { + pr_debug("%s buffer len 0x%x over maximum 0x%x\n", + in ? "input" : "output", *len, + nx842_pseries_constraints.maximum); + if (in) + return -EINVAL; + *len = nx842_pseries_constraints.maximum; + } + return 0; +} + +/* I assume we need to align the CSB? */ +#define WORKMEM_ALIGN (256) + +struct nx842_workmem { + /* scatterlist */ + char slin[4096]; + char slout[4096]; + /* coprocessor status/parameter block */ + struct nx_csbcpb csbcpb; + + char padding[WORKMEM_ALIGN]; +} __aligned(WORKMEM_ALIGN); + +/* Macros for fields within nx_csbcpb */ +/* Check the valid bit within the csbcpb valid field */ +#define NX842_CSBCBP_VALID_CHK(x) (x & BIT_MASK(7)) + +/* CE macros operate on the completion_extension field bits in the csbcpb. + * CE0 0=full completion, 1=partial completion + * CE1 0=CE0 indicates completion, 1=termination (output may be modified) + * CE2 0=processed_bytes is source bytes, 1=processed_bytes is target bytes */ +#define NX842_CSBCPB_CE0(x) (x & BIT_MASK(7)) +#define NX842_CSBCPB_CE1(x) (x & BIT_MASK(6)) +#define NX842_CSBCPB_CE2(x) (x & BIT_MASK(5)) + +/* The NX unit accepts data only on 4K page boundaries */ +#define NX842_HW_PAGE_SIZE (4096) +#define NX842_HW_PAGE_MASK (~(NX842_HW_PAGE_SIZE-1)) + +struct ibm_nx842_counters { + atomic64_t comp_complete; + atomic64_t comp_failed; + atomic64_t decomp_complete; + atomic64_t decomp_failed; + atomic64_t swdecomp; + atomic64_t comp_times[32]; + atomic64_t decomp_times[32]; +}; + +struct nx842_devdata { + struct vio_dev *vdev; + struct device *dev; + struct ibm_nx842_counters *counters; + unsigned int max_sg_len; + unsigned int max_sync_size; + unsigned int max_sync_sg; +}; + +static struct nx842_devdata __rcu *devdata; +static DEFINE_SPINLOCK(devdata_spinlock); + +#define NX842_COUNTER_INC(_x) \ +static inline void nx842_inc_##_x( \ + const struct nx842_devdata *dev) { \ + if (dev) \ + atomic64_inc(&dev->counters->_x); \ +} +NX842_COUNTER_INC(comp_complete); +NX842_COUNTER_INC(comp_failed); +NX842_COUNTER_INC(decomp_complete); +NX842_COUNTER_INC(decomp_failed); +NX842_COUNTER_INC(swdecomp); + +#define NX842_HIST_SLOTS 16 + +static void ibm_nx842_incr_hist(atomic64_t *times, unsigned int time) +{ + int bucket = fls(time); + + if (bucket) + bucket = min((NX842_HIST_SLOTS - 1), bucket - 1); + + atomic64_inc(×[bucket]); +} + +/* NX unit operation flags */ +#define NX842_OP_COMPRESS 0x0 +#define NX842_OP_CRC 0x1 +#define NX842_OP_DECOMPRESS 0x2 +#define NX842_OP_COMPRESS_CRC (NX842_OP_COMPRESS | NX842_OP_CRC) +#define NX842_OP_DECOMPRESS_CRC (NX842_OP_DECOMPRESS | NX842_OP_CRC) +#define NX842_OP_ASYNC (1<<23) +#define NX842_OP_NOTIFY (1<<22) +#define NX842_OP_NOTIFY_INT(x) ((x & 0xff)<<8) + +static unsigned long nx842_get_desired_dma(struct vio_dev *viodev) +{ + /* No use of DMA mappings within the driver. */ + return 0; +} + +struct nx842_slentry { + __be64 ptr; /* Real address (use __pa()) */ + __be64 len; +}; + +/* pHyp scatterlist entry */ +struct nx842_scatterlist { + int entry_nr; /* number of slentries */ + struct nx842_slentry *entries; /* ptr to array of slentries */ +}; + +/* Does not include sizeof(entry_nr) in the size */ +static inline unsigned long nx842_get_scatterlist_size( + struct nx842_scatterlist *sl) +{ + return sl->entry_nr * sizeof(struct nx842_slentry); +} + +static int nx842_build_scatterlist(unsigned long buf, int len, + struct nx842_scatterlist *sl) +{ + unsigned long entrylen; + struct nx842_slentry *entry; + + sl->entry_nr = 0; + + entry = sl->entries; + while (len) { + entry->ptr = cpu_to_be64(nx842_get_pa((void *)buf)); + entrylen = min_t(int, len, + LEN_ON_SIZE(buf, NX842_HW_PAGE_SIZE)); + entry->len = cpu_to_be64(entrylen); + + len -= entrylen; + buf += entrylen; + + sl->entry_nr++; + entry++; + } + + return 0; +} + +static int nx842_validate_result(struct device *dev, + struct cop_status_block *csb) +{ + /* The csb must be valid after returning from vio_h_cop_sync */ + if (!NX842_CSBCBP_VALID_CHK(csb->valid)) { + dev_err(dev, "%s: cspcbp not valid upon completion.\n", + __func__); + dev_dbg(dev, "valid:0x%02x cs:0x%02x cc:0x%02x ce:0x%02x\n", + csb->valid, + csb->crb_seq_number, + csb->completion_code, + csb->completion_extension); + dev_dbg(dev, "processed_bytes:%d address:0x%016lx\n", + be32_to_cpu(csb->processed_byte_count), + (unsigned long)be64_to_cpu(csb->address)); + return -EIO; + } + + /* Check return values from the hardware in the CSB */ + switch (csb->completion_code) { + case 0: /* Completed without error */ + break; + case 64: /* Compression ok, but output larger than input */ + dev_dbg(dev, "%s: output size larger than input size\n", + __func__); + break; + case 13: /* Output buffer too small */ + dev_dbg(dev, "%s: Out of space in output buffer\n", + __func__); + return -ENOSPC; + case 65: /* Calculated CRC doesn't match the passed value */ + dev_dbg(dev, "%s: CRC mismatch for decompression\n", + __func__); + return -EINVAL; + case 66: /* Input data contains an illegal template field */ + case 67: /* Template indicates data past the end of the input stream */ + dev_dbg(dev, "%s: Bad data for decompression (code:%d)\n", + __func__, csb->completion_code); + return -EINVAL; + default: + dev_dbg(dev, "%s: Unspecified error (code:%d)\n", + __func__, csb->completion_code); + return -EIO; + } + + /* Hardware sanity check */ + if (!NX842_CSBCPB_CE2(csb->completion_extension)) { + dev_err(dev, "%s: No error returned by hardware, but " + "data returned is unusable, contact support.\n" + "(Additional info: csbcbp->processed bytes " + "does not specify processed bytes for the " + "target buffer.)\n", __func__); + return -EIO; + } + + return 0; +} + +/** + * nx842_pseries_compress - Compress data using the 842 algorithm + * + * Compression provide by the NX842 coprocessor on IBM Power systems. + * The input buffer is compressed and the result is stored in the + * provided output buffer. + * + * Upon return from this function @outlen contains the length of the + * compressed data. If there is an error then @outlen will be 0 and an + * error will be specified by the return code from this function. + * + * @in: Pointer to input buffer + * @inlen: Length of input buffer + * @out: Pointer to output buffer + * @outlen: Length of output buffer + * @wmem: ptr to buffer for working memory, size determined by + * nx842_pseries_driver.workmem_size + * + * Returns: + * 0 Success, output of length @outlen stored in the buffer at @out + * -ENOMEM Unable to allocate internal buffers + * -ENOSPC Output buffer is to small + * -EIO Internal error + * -ENODEV Hardware unavailable + */ +static int nx842_pseries_compress(const unsigned char *in, unsigned int inlen, + unsigned char *out, unsigned int *outlen, + void *wmem) +{ + struct nx842_devdata *local_devdata; + struct device *dev = NULL; + struct nx842_workmem *workmem; + struct nx842_scatterlist slin, slout; + struct nx_csbcpb *csbcpb; + int ret = 0; + unsigned long inbuf, outbuf; + struct vio_pfo_op op = { + .done = NULL, + .handle = 0, + .timeout = 0, + }; + unsigned long start = get_tb(); + + inbuf = (unsigned long)in; + if (check_constraints(inbuf, &inlen, true)) + return -EINVAL; + + outbuf = (unsigned long)out; + if (check_constraints(outbuf, outlen, false)) + return -EINVAL; + + rcu_read_lock(); + local_devdata = rcu_dereference(devdata); + if (!local_devdata || !local_devdata->dev) { + rcu_read_unlock(); + return -ENODEV; + } + dev = local_devdata->dev; + + /* Init scatterlist */ + workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN); + slin.entries = (struct nx842_slentry *)workmem->slin; + slout.entries = (struct nx842_slentry *)workmem->slout; + + /* Init operation */ + op.flags = NX842_OP_COMPRESS_CRC; + csbcpb = &workmem->csbcpb; + memset(csbcpb, 0, sizeof(*csbcpb)); + op.csbcpb = nx842_get_pa(csbcpb); + + if ((inbuf & NX842_HW_PAGE_MASK) == + ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) { + /* Create direct DDE */ + op.in = nx842_get_pa((void *)inbuf); + op.inlen = inlen; + } else { + /* Create indirect DDE (scatterlist) */ + nx842_build_scatterlist(inbuf, inlen, &slin); + op.in = nx842_get_pa(slin.entries); + op.inlen = -nx842_get_scatterlist_size(&slin); + } + + if ((outbuf & NX842_HW_PAGE_MASK) == + ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) { + /* Create direct DDE */ + op.out = nx842_get_pa((void *)outbuf); + op.outlen = *outlen; + } else { + /* Create indirect DDE (scatterlist) */ + nx842_build_scatterlist(outbuf, *outlen, &slout); + op.out = nx842_get_pa(slout.entries); + op.outlen = -nx842_get_scatterlist_size(&slout); + } + + dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n", + __func__, (unsigned long)op.in, (long)op.inlen, + (unsigned long)op.out, (long)op.outlen); + + /* Send request to pHyp */ + ret = vio_h_cop_sync(local_devdata->vdev, &op); + + /* Check for pHyp error */ + if (ret) { + dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n", + __func__, ret, op.hcall_err); + ret = -EIO; + goto unlock; + } + + /* Check for hardware error */ + ret = nx842_validate_result(dev, &csbcpb->csb); + if (ret) + goto unlock; + + *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count); + dev_dbg(dev, "%s: processed_bytes=%d\n", __func__, *outlen); + +unlock: + if (ret) + nx842_inc_comp_failed(local_devdata); + else { + nx842_inc_comp_complete(local_devdata); + ibm_nx842_incr_hist(local_devdata->counters->comp_times, + (get_tb() - start) / tb_ticks_per_usec); + } + rcu_read_unlock(); + return ret; +} + +/** + * nx842_pseries_decompress - Decompress data using the 842 algorithm + * + * Decompression provide by the NX842 coprocessor on IBM Power systems. + * The input buffer is decompressed and the result is stored in the + * provided output buffer. The size allocated to the output buffer is + * provided by the caller of this function in @outlen. Upon return from + * this function @outlen contains the length of the decompressed data. + * If there is an error then @outlen will be 0 and an error will be + * specified by the return code from this function. + * + * @in: Pointer to input buffer + * @inlen: Length of input buffer + * @out: Pointer to output buffer + * @outlen: Length of output buffer + * @wmem: ptr to buffer for working memory, size determined by + * nx842_pseries_driver.workmem_size + * + * Returns: + * 0 Success, output of length @outlen stored in the buffer at @out + * -ENODEV Hardware decompression device is unavailable + * -ENOMEM Unable to allocate internal buffers + * -ENOSPC Output buffer is to small + * -EINVAL Bad input data encountered when attempting decompress + * -EIO Internal error + */ +static int nx842_pseries_decompress(const unsigned char *in, unsigned int inlen, + unsigned char *out, unsigned int *outlen, + void *wmem) +{ + struct nx842_devdata *local_devdata; + struct device *dev = NULL; + struct nx842_workmem *workmem; + struct nx842_scatterlist slin, slout; + struct nx_csbcpb *csbcpb; + int ret = 0; + unsigned long inbuf, outbuf; + struct vio_pfo_op op = { + .done = NULL, + .handle = 0, + .timeout = 0, + }; + unsigned long start = get_tb(); + + /* Ensure page alignment and size */ + inbuf = (unsigned long)in; + if (check_constraints(inbuf, &inlen, true)) + return -EINVAL; + + outbuf = (unsigned long)out; + if (check_constraints(outbuf, outlen, false)) + return -EINVAL; + + rcu_read_lock(); + local_devdata = rcu_dereference(devdata); + if (!local_devdata || !local_devdata->dev) { + rcu_read_unlock(); + return -ENODEV; + } + dev = local_devdata->dev; + + workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN); + + /* Init scatterlist */ + slin.entries = (struct nx842_slentry *)workmem->slin; + slout.entries = (struct nx842_slentry *)workmem->slout; + + /* Init operation */ + op.flags = NX842_OP_DECOMPRESS_CRC; + csbcpb = &workmem->csbcpb; + memset(csbcpb, 0, sizeof(*csbcpb)); + op.csbcpb = nx842_get_pa(csbcpb); + + if ((inbuf & NX842_HW_PAGE_MASK) == + ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) { + /* Create direct DDE */ + op.in = nx842_get_pa((void *)inbuf); + op.inlen = inlen; + } else { + /* Create indirect DDE (scatterlist) */ + nx842_build_scatterlist(inbuf, inlen, &slin); + op.in = nx842_get_pa(slin.entries); + op.inlen = -nx842_get_scatterlist_size(&slin); + } + + if ((outbuf & NX842_HW_PAGE_MASK) == + ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) { + /* Create direct DDE */ + op.out = nx842_get_pa((void *)outbuf); + op.outlen = *outlen; + } else { + /* Create indirect DDE (scatterlist) */ + nx842_build_scatterlist(outbuf, *outlen, &slout); + op.out = nx842_get_pa(slout.entries); + op.outlen = -nx842_get_scatterlist_size(&slout); + } + + dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n", + __func__, (unsigned long)op.in, (long)op.inlen, + (unsigned long)op.out, (long)op.outlen); + + /* Send request to pHyp */ + ret = vio_h_cop_sync(local_devdata->vdev, &op); + + /* Check for pHyp error */ + if (ret) { + dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n", + __func__, ret, op.hcall_err); + goto unlock; + } + + /* Check for hardware error */ + ret = nx842_validate_result(dev, &csbcpb->csb); + if (ret) + goto unlock; + + *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count); + +unlock: + if (ret) + /* decompress fail */ + nx842_inc_decomp_failed(local_devdata); + else { + nx842_inc_decomp_complete(local_devdata); + ibm_nx842_incr_hist(local_devdata->counters->decomp_times, + (get_tb() - start) / tb_ticks_per_usec); + } + + rcu_read_unlock(); + return ret; +} + +/** + * nx842_OF_set_defaults -- Set default (disabled) values for devdata + * + * @devdata: struct nx842_devdata to update + * + * Returns: + * 0 on success + * -ENOENT if @devdata ptr is NULL + */ +static int nx842_OF_set_defaults(struct nx842_devdata *devdata) +{ + if (devdata) { + devdata->max_sync_size = 0; + devdata->max_sync_sg = 0; + devdata->max_sg_len = 0; + return 0; + } else + return -ENOENT; +} + +/** + * nx842_OF_upd_status -- Check the device info from OF status prop + * + * The status property indicates if the accelerator is enabled. If the + * device is in the OF tree it indicates that the hardware is present. + * The status field indicates if the device is enabled when the status + * is 'okay'. Otherwise the device driver will be disabled. + * + * @devdata: struct nx842_devdata to use for dev_info + * @prop: struct property point containing the maxsyncop for the update + * + * Returns: + * 0 - Device is available + * -ENODEV - Device is not available + */ +static int nx842_OF_upd_status(struct nx842_devdata *devdata, + struct property *prop) +{ + const char *status = (const char *)prop->value; + + if (!strncmp(status, "okay", (size_t)prop->length)) + return 0; + if (!strncmp(status, "disabled", (size_t)prop->length)) + return -ENODEV; + dev_info(devdata->dev, "%s: unknown status '%s'\n", __func__, status); + + return -EINVAL; +} + +/** + * nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop + * + * Definition of the 'ibm,max-sg-len' OF property: + * This field indicates the maximum byte length of a scatter list + * for the platform facility. It is a single cell encoded as with encode-int. + * + * Example: + * # od -x ibm,max-sg-len + * 0000000 0000 0ff0 + * + * In this example, the maximum byte length of a scatter list is + * 0x0ff0 (4,080). + * + * @devdata: struct nx842_devdata to update + * @prop: struct property point containing the maxsyncop for the update + * + * Returns: + * 0 on success + * -EINVAL on failure + */ +static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata, + struct property *prop) { + int ret = 0; + const unsigned int maxsglen = of_read_number(prop->value, 1); + + if (prop->length != sizeof(maxsglen)) { + dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__); + dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__, + prop->length, sizeof(maxsglen)); + ret = -EINVAL; + } else { + devdata->max_sg_len = min_t(unsigned int, + maxsglen, NX842_HW_PAGE_SIZE); + } + + return ret; +} + +/** + * nx842_OF_upd_maxsyncop -- Update the device info from OF maxsyncop prop + * + * Definition of the 'ibm,max-sync-cop' OF property: + * Two series of cells. The first series of cells represents the maximums + * that can be synchronously compressed. The second series of cells + * represents the maximums that can be synchronously decompressed. + * 1. The first cell in each series contains the count of the number of + * data length, scatter list elements pairs that follow – each being + * of the form + * a. One cell data byte length + * b. One cell total number of scatter list elements + * + * Example: + * # od -x ibm,max-sync-cop + * 0000000 0000 0001 0000 1000 0000 01fe 0000 0001 + * 0000020 0000 1000 0000 01fe + * + * In this example, compression supports 0x1000 (4,096) data byte length + * and 0x1fe (510) total scatter list elements. Decompression supports + * 0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list + * elements. + * + * @devdata: struct nx842_devdata to update + * @prop: struct property point containing the maxsyncop for the update + * + * Returns: + * 0 on success + * -EINVAL on failure + */ +static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata, + struct property *prop) { + int ret = 0; + unsigned int comp_data_limit, decomp_data_limit; + unsigned int comp_sg_limit, decomp_sg_limit; + const struct maxsynccop_t { + __be32 comp_elements; + __be32 comp_data_limit; + __be32 comp_sg_limit; + __be32 decomp_elements; + __be32 decomp_data_limit; + __be32 decomp_sg_limit; + } *maxsynccop; + + if (prop->length != sizeof(*maxsynccop)) { + dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__); + dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length, + sizeof(*maxsynccop)); + ret = -EINVAL; + goto out; + } + + maxsynccop = (const struct maxsynccop_t *)prop->value; + comp_data_limit = be32_to_cpu(maxsynccop->comp_data_limit); + comp_sg_limit = be32_to_cpu(maxsynccop->comp_sg_limit); + decomp_data_limit = be32_to_cpu(maxsynccop->decomp_data_limit); + decomp_sg_limit = be32_to_cpu(maxsynccop->decomp_sg_limit); + + /* Use one limit rather than separate limits for compression and + * decompression. Set a maximum for this so as not to exceed the + * size that the header can support and round the value down to + * the hardware page size (4K) */ + devdata->max_sync_size = min(comp_data_limit, decomp_data_limit); + + devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size, + 65536); + + if (devdata->max_sync_size < 4096) { + dev_err(devdata->dev, "%s: hardware max data size (%u) is " + "less than the driver minimum, unable to use " + "the hardware device\n", + __func__, devdata->max_sync_size); + ret = -EINVAL; + goto out; + } + + nx842_pseries_constraints.maximum = devdata->max_sync_size; + + devdata->max_sync_sg = min(comp_sg_limit, decomp_sg_limit); + if (devdata->max_sync_sg < 1) { + dev_err(devdata->dev, "%s: hardware max sg size (%u) is " + "less than the driver minimum, unable to use " + "the hardware device\n", + __func__, devdata->max_sync_sg); + ret = -EINVAL; + goto out; + } + +out: + return ret; +} + +/** + * nx842_OF_upd -- Handle OF properties updates for the device. + * + * Set all properties from the OF tree. Optionally, a new property + * can be provided by the @new_prop pointer to overwrite an existing value. + * The device will remain disabled until all values are valid, this function + * will return an error for updates unless all values are valid. + * + * @new_prop: If not NULL, this property is being updated. If NULL, update + * all properties from the current values in the OF tree. + * + * Returns: + * 0 - Success + * -ENOMEM - Could not allocate memory for new devdata structure + * -EINVAL - property value not found, new_prop is not a recognized + * property for the device or property value is not valid. + * -ENODEV - Device is not available + */ +static int nx842_OF_upd(struct property *new_prop) +{ + struct nx842_devdata *old_devdata = NULL; + struct nx842_devdata *new_devdata = NULL; + struct device_node *of_node = NULL; + struct property *status = NULL; + struct property *maxsglen = NULL; + struct property *maxsyncop = NULL; + int ret = 0; + unsigned long flags; + + new_devdata = kzalloc_obj(*new_devdata, GFP_NOFS); + if (!new_devdata) + return -ENOMEM; + + spin_lock_irqsave(&devdata_spinlock, flags); + old_devdata = rcu_dereference_check(devdata, + lockdep_is_held(&devdata_spinlock)); + if (old_devdata) + of_node = old_devdata->dev->of_node; + + if (!old_devdata || !of_node) { + pr_err("%s: device is not available\n", __func__); + spin_unlock_irqrestore(&devdata_spinlock, flags); + kfree(new_devdata); + return -ENODEV; + } + + memcpy(new_devdata, old_devdata, sizeof(*old_devdata)); + new_devdata->counters = old_devdata->counters; + + /* Set ptrs for existing properties */ + status = of_find_property(of_node, "status", NULL); + maxsglen = of_find_property(of_node, "ibm,max-sg-len", NULL); + maxsyncop = of_find_property(of_node, "ibm,max-sync-cop", NULL); + if (!status || !maxsglen || !maxsyncop) { + dev_err(old_devdata->dev, "%s: Could not locate device properties\n", __func__); + ret = -EINVAL; + goto error_out; + } + + /* + * If this is a property update, there are only certain properties that + * we care about. Bail if it isn't in the below list + */ + if (new_prop && (strncmp(new_prop->name, "status", new_prop->length) || + strncmp(new_prop->name, "ibm,max-sg-len", new_prop->length) || + strncmp(new_prop->name, "ibm,max-sync-cop", new_prop->length))) + goto out; + + /* Perform property updates */ + ret = nx842_OF_upd_status(new_devdata, status); + if (ret) + goto error_out; + + ret = nx842_OF_upd_maxsglen(new_devdata, maxsglen); + if (ret) + goto error_out; + + ret = nx842_OF_upd_maxsyncop(new_devdata, maxsyncop); + if (ret) + goto error_out; + +out: + dev_info(old_devdata->dev, "%s: max_sync_size new:%u old:%u\n", + __func__, new_devdata->max_sync_size, + old_devdata->max_sync_size); + dev_info(old_devdata->dev, "%s: max_sync_sg new:%u old:%u\n", + __func__, new_devdata->max_sync_sg, + old_devdata->max_sync_sg); + dev_info(old_devdata->dev, "%s: max_sg_len new:%u old:%u\n", + __func__, new_devdata->max_sg_len, + old_devdata->max_sg_len); + + rcu_assign_pointer(devdata, new_devdata); + spin_unlock_irqrestore(&devdata_spinlock, flags); + synchronize_rcu(); + dev_set_drvdata(new_devdata->dev, new_devdata); + kfree(old_devdata); + return 0; + +error_out: + if (new_devdata) { + dev_info(old_devdata->dev, "%s: device disabled\n", __func__); + nx842_OF_set_defaults(new_devdata); + rcu_assign_pointer(devdata, new_devdata); + spin_unlock_irqrestore(&devdata_spinlock, flags); + synchronize_rcu(); + dev_set_drvdata(new_devdata->dev, new_devdata); + kfree(old_devdata); + } else { + dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__); + spin_unlock_irqrestore(&devdata_spinlock, flags); + } + + if (!ret) + ret = -EINVAL; + return ret; +} + +/** + * nx842_OF_notifier - Process updates to OF properties for the device + * + * @np: notifier block + * @action: notifier action + * @data: struct of_reconfig_data pointer + * + * Returns: + * NOTIFY_OK on success + * NOTIFY_BAD encoded with error number on failure, use + * notifier_to_errno() to decode this value + */ +static int nx842_OF_notifier(struct notifier_block *np, unsigned long action, + void *data) +{ + struct of_reconfig_data *upd = data; + struct nx842_devdata *local_devdata; + struct device_node *node = NULL; + + rcu_read_lock(); + local_devdata = rcu_dereference(devdata); + if (local_devdata) + node = local_devdata->dev->of_node; + + if (local_devdata && + action == OF_RECONFIG_UPDATE_PROPERTY && + !strcmp(upd->dn->name, node->name)) { + rcu_read_unlock(); + nx842_OF_upd(upd->prop); + } else + rcu_read_unlock(); + + return NOTIFY_OK; +} + +static struct notifier_block nx842_of_nb = { + .notifier_call = nx842_OF_notifier, +}; + +#define nx842_counter_read(_name) \ +static ssize_t nx842_##_name##_show(struct device *dev, \ + struct device_attribute *attr, \ + char *buf) { \ + struct nx842_devdata *local_devdata; \ + int p = 0; \ + rcu_read_lock(); \ + local_devdata = rcu_dereference(devdata); \ + if (local_devdata) \ + p = snprintf(buf, PAGE_SIZE, "%lld\n", \ + atomic64_read(&local_devdata->counters->_name)); \ + rcu_read_unlock(); \ + return p; \ +} + +#define NX842DEV_COUNTER_ATTR_RO(_name) \ + nx842_counter_read(_name); \ + static struct device_attribute dev_attr_##_name = __ATTR(_name, \ + 0444, \ + nx842_##_name##_show,\ + NULL); + +NX842DEV_COUNTER_ATTR_RO(comp_complete); +NX842DEV_COUNTER_ATTR_RO(comp_failed); +NX842DEV_COUNTER_ATTR_RO(decomp_complete); +NX842DEV_COUNTER_ATTR_RO(decomp_failed); +NX842DEV_COUNTER_ATTR_RO(swdecomp); + +static ssize_t nx842_timehist_show(struct device *, + struct device_attribute *, char *); + +static struct device_attribute dev_attr_comp_times = __ATTR(comp_times, 0444, + nx842_timehist_show, NULL); +static struct device_attribute dev_attr_decomp_times = __ATTR(decomp_times, + 0444, nx842_timehist_show, NULL); + +static ssize_t nx842_timehist_show(struct device *dev, + struct device_attribute *attr, char *buf) { + char *p = buf; + struct nx842_devdata *local_devdata; + atomic64_t *times; + int bytes_remain = PAGE_SIZE; + int bytes; + int i; + + rcu_read_lock(); + local_devdata = rcu_dereference(devdata); + if (!local_devdata) { + rcu_read_unlock(); + return 0; + } + + if (attr == &dev_attr_comp_times) + times = local_devdata->counters->comp_times; + else if (attr == &dev_attr_decomp_times) + times = local_devdata->counters->decomp_times; + else { + rcu_read_unlock(); + return 0; + } + + for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) { + bytes = snprintf(p, bytes_remain, "%u-%uus:\t%lld\n", + i ? (2<<(i-1)) : 0, (2<counters = kzalloc_obj(*new_devdata->counters, GFP_NOFS); + if (!new_devdata->counters) { + kfree(new_devdata); + return -ENOMEM; + } + + spin_lock_irqsave(&devdata_spinlock, flags); + old_devdata = rcu_dereference_check(devdata, + lockdep_is_held(&devdata_spinlock)); + + if (old_devdata && old_devdata->vdev != NULL) { + dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__); + ret = -1; + goto error_unlock; + } + + dev_set_drvdata(&viodev->dev, NULL); + + new_devdata->vdev = viodev; + new_devdata->dev = &viodev->dev; + nx842_OF_set_defaults(new_devdata); + + rcu_assign_pointer(devdata, new_devdata); + spin_unlock_irqrestore(&devdata_spinlock, flags); + synchronize_rcu(); + kfree(old_devdata); + + of_reconfig_notifier_register(&nx842_of_nb); + + ret = nx842_OF_upd(NULL); + if (ret) + goto error; + + ret = crypto_register_scomp(&nx842_pseries_alg); + if (ret) { + dev_err(&viodev->dev, "could not register comp alg: %d\n", ret); + goto error; + } + + rcu_read_lock(); + dev_set_drvdata(&viodev->dev, rcu_dereference(devdata)); + rcu_read_unlock(); + + if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) { + dev_err(&viodev->dev, "could not create sysfs device attributes\n"); + ret = -1; + goto error; + } + + if (caps_feat) { + if (sysfs_create_group(&viodev->dev.kobj, + &nxcop_caps_attr_group)) { + dev_err(&viodev->dev, + "Could not create sysfs NX capability entries\n"); + ret = -1; + goto error; + } + } + + return 0; + +error_unlock: + spin_unlock_irqrestore(&devdata_spinlock, flags); + if (new_devdata) + kfree(new_devdata->counters); + kfree(new_devdata); +error: + return ret; +} + +static void nx842_remove(struct vio_dev *viodev) +{ + struct nx842_devdata *old_devdata; + unsigned long flags; + + pr_info("Removing IBM Power 842 compression device\n"); + sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group); + + if (caps_feat) + sysfs_remove_group(&viodev->dev.kobj, &nxcop_caps_attr_group); + + crypto_unregister_scomp(&nx842_pseries_alg); + + of_reconfig_notifier_unregister(&nx842_of_nb); + + spin_lock_irqsave(&devdata_spinlock, flags); + old_devdata = rcu_dereference_check(devdata, + lockdep_is_held(&devdata_spinlock)); + RCU_INIT_POINTER(devdata, NULL); + spin_unlock_irqrestore(&devdata_spinlock, flags); + synchronize_rcu(); + dev_set_drvdata(&viodev->dev, NULL); + if (old_devdata) + kfree(old_devdata->counters); + kfree(old_devdata); +} + +/* + * Get NX capabilities from the hypervisor. + * Only NXGZIP capabilities are provided by the hypersvisor right + * now and these values are available to user space with sysfs. + */ +static void __init nxcop_get_capabilities(void) +{ + struct hv_vas_all_caps *hv_caps; + struct hv_nx_cop_caps *hv_nxc; + u64 feat; + int rc; + + hv_caps = kmalloc_obj(*hv_caps); + if (!hv_caps) + return; + /* + * Get NX overall capabilities with feature type=0 + */ + rc = h_query_vas_capabilities(H_QUERY_NX_CAPABILITIES, 0, + (u64)virt_to_phys(hv_caps)); + if (!rc) + feat = be64_to_cpu(hv_caps->feat_type); + kfree(hv_caps); + if (rc) + return; + if (!(feat & VAS_NX_GZIP_FEAT_BIT)) + return; + + /* + * NX-GZIP feature available + */ + hv_nxc = kmalloc_obj(*hv_nxc); + if (!hv_nxc) + return; + /* + * Get capabilities for NX-GZIP feature + */ + rc = h_query_vas_capabilities(H_QUERY_NX_CAPABILITIES, + VAS_NX_GZIP_FEAT, + (u64)virt_to_phys(hv_nxc)); + + if (!rc) { + nx_cop_caps.descriptor = be64_to_cpu(hv_nxc->descriptor); + nx_cop_caps.req_max_processed_len = + be64_to_cpu(hv_nxc->req_max_processed_len); + nx_cop_caps.min_compress_len = + be64_to_cpu(hv_nxc->min_compress_len); + nx_cop_caps.min_decompress_len = + be64_to_cpu(hv_nxc->min_decompress_len); + caps_feat = feat; + } + + kfree(hv_nxc); +} + +static const struct vio_device_id nx842_vio_driver_ids[] = { + {"ibm,compression-v1", "ibm,compression"}, + {"", ""}, +}; +MODULE_DEVICE_TABLE(vio, nx842_vio_driver_ids); + +static struct vio_driver nx842_vio_driver = { + .name = KBUILD_MODNAME, + .probe = nx842_probe, + .remove = nx842_remove, + .get_desired_dma = nx842_get_desired_dma, + .id_table = nx842_vio_driver_ids, +}; + +static int __init nx842_pseries_init(void) +{ + struct nx842_devdata *new_devdata; + struct device_node *np; + int ret; + + np = of_find_compatible_node(NULL, NULL, "ibm,compression"); + if (!np) + return -ENODEV; + of_node_put(np); + + RCU_INIT_POINTER(devdata, NULL); + new_devdata = kzalloc_obj(*new_devdata); + if (!new_devdata) + return -ENOMEM; + + RCU_INIT_POINTER(devdata, new_devdata); + /* + * Get NX capabilities from the hypervisor. + */ + nxcop_get_capabilities(); + + ret = vio_register_driver(&nx842_vio_driver); + if (ret) { + pr_err("Could not register VIO driver %d\n", ret); + + kfree(new_devdata); + return ret; + } + + ret = vas_register_api_pseries(THIS_MODULE, VAS_COP_TYPE_GZIP, + "nx-gzip"); + + if (ret) + pr_err("NX-GZIP is not supported. Returned=%d\n", ret); + + return 0; +} + +module_init(nx842_pseries_init); + +static void __exit nx842_pseries_exit(void) +{ + struct nx842_devdata *old_devdata; + unsigned long flags; + + vas_unregister_api_pseries(); + + crypto_unregister_scomp(&nx842_pseries_alg); + + spin_lock_irqsave(&devdata_spinlock, flags); + old_devdata = rcu_dereference_check(devdata, + lockdep_is_held(&devdata_spinlock)); + RCU_INIT_POINTER(devdata, NULL); + spin_unlock_irqrestore(&devdata_spinlock, flags); + synchronize_rcu(); + if (old_devdata && old_devdata->dev) + dev_set_drvdata(old_devdata->dev, NULL); + kfree(old_devdata); + vio_unregister_driver(&nx842_vio_driver); +} + +module_exit(nx842_pseries_exit); + diff --git a/drivers/crypto/nx/nx-sha256.c b/drivers/crypto/nx/nx-sha256.c new file mode 100644 index 000000000..5b29dd026 --- /dev/null +++ b/drivers/crypto/nx/nx-sha256.c @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * SHA-256 routines supporting the Power 7+ Nest Accelerators driver + * + * Copyright (C) 2011-2012 International Business Machines Inc. + * + * Author: Kent Yoder + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nx_csbcpb.h" +#include "nx.h" + +struct sha256_state_be { + __be32 state[SHA256_DIGEST_SIZE / 4]; + u64 count; +}; + +static int nx_crypto_ctx_sha256_init(struct crypto_shash *tfm) +{ + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(tfm); + int err; + + err = nx_crypto_ctx_sha_init(tfm); + if (err) + return err; + + nx_ctx_init(nx_ctx, HCOP_FC_SHA); + + nx_ctx->ap = &nx_ctx->props[NX_PROPS_SHA256]; + + NX_CPB_SET_DIGEST_SIZE(nx_ctx->csbcpb, NX_DS_SHA256); + + return 0; +} + +static int nx_sha256_init(struct shash_desc *desc) +{ + struct sha256_state_be *sctx = shash_desc_ctx(desc); + + sctx->state[0] = __cpu_to_be32(SHA256_H0); + sctx->state[1] = __cpu_to_be32(SHA256_H1); + sctx->state[2] = __cpu_to_be32(SHA256_H2); + sctx->state[3] = __cpu_to_be32(SHA256_H3); + sctx->state[4] = __cpu_to_be32(SHA256_H4); + sctx->state[5] = __cpu_to_be32(SHA256_H5); + sctx->state[6] = __cpu_to_be32(SHA256_H6); + sctx->state[7] = __cpu_to_be32(SHA256_H7); + sctx->count = 0; + + return 0; +} + +static int nx_sha256_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); + struct sha256_state_be *sctx = shash_desc_ctx(desc); + struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; + u64 to_process, leftover, total = len; + struct nx_sg *out_sg; + unsigned long irq_flags; + int rc = 0; + int data_len; + u32 max_sg_len; + + spin_lock_irqsave(&nx_ctx->lock, irq_flags); + + memcpy(csbcpb->cpb.sha256.message_digest, sctx->state, SHA256_DIGEST_SIZE); + NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; + NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; + + max_sg_len = min_t(u64, nx_ctx->ap->sglen, + nx_driver.of.max_sg_len/sizeof(struct nx_sg)); + max_sg_len = min_t(u64, max_sg_len, + nx_ctx->ap->databytelen/NX_PAGE_SIZE); + + data_len = SHA256_DIGEST_SIZE; + out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *)sctx->state, + &data_len, max_sg_len); + nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg); + + if (data_len != SHA256_DIGEST_SIZE) { + rc = -EINVAL; + goto out; + } + + do { + struct nx_sg *in_sg = nx_ctx->in_sg; + + to_process = total & ~(SHA256_BLOCK_SIZE - 1); + + data_len = to_process; + in_sg = nx_build_sg_list(in_sg, (u8 *) data, + &data_len, max_sg_len); + + nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg); + + to_process = data_len; + leftover = total - to_process; + + /* + * we've hit the nx chip previously and we're updating + * again, so copy over the partial digest. + */ + memcpy(csbcpb->cpb.sha256.input_partial_digest, + csbcpb->cpb.sha256.message_digest, + SHA256_DIGEST_SIZE); + + if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) { + rc = -EINVAL; + goto out; + } + + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 0); + if (rc) + goto out; + + atomic_inc(&(nx_ctx->stats->sha256_ops)); + + total -= to_process; + data += to_process; + sctx->count += to_process; + } while (leftover >= SHA256_BLOCK_SIZE); + + rc = leftover; + memcpy(sctx->state, csbcpb->cpb.sha256.message_digest, SHA256_DIGEST_SIZE); +out: + spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); + return rc; +} + +static int nx_sha256_finup(struct shash_desc *desc, const u8 *src, + unsigned int nbytes, u8 *out) +{ + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); + struct sha256_state_be *sctx = shash_desc_ctx(desc); + struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; + struct nx_sg *in_sg, *out_sg; + unsigned long irq_flags; + u32 max_sg_len; + int rc = 0; + int len; + + spin_lock_irqsave(&nx_ctx->lock, irq_flags); + + max_sg_len = min_t(u64, nx_ctx->ap->sglen, + nx_driver.of.max_sg_len/sizeof(struct nx_sg)); + max_sg_len = min_t(u64, max_sg_len, + nx_ctx->ap->databytelen/NX_PAGE_SIZE); + + /* final is represented by continuing the operation and indicating that + * this is not an intermediate operation + * copy over the partial digest */ + memcpy(csbcpb->cpb.sha256.input_partial_digest, sctx->state, SHA256_DIGEST_SIZE); + NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; + NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; + + sctx->count += nbytes; + csbcpb->cpb.sha256.message_bit_length = (u64) (sctx->count * 8); + + len = nbytes; + in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)src, &len, max_sg_len); + + if (len != nbytes) { + rc = -EINVAL; + goto out; + } + + len = SHA256_DIGEST_SIZE; + out_sg = nx_build_sg_list(nx_ctx->out_sg, out, &len, max_sg_len); + + if (len != SHA256_DIGEST_SIZE) { + rc = -EINVAL; + goto out; + } + + nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg); + nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg); + if (!nx_ctx->op.outlen) { + rc = -EINVAL; + goto out; + } + + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 0); + if (rc) + goto out; + + atomic_inc(&(nx_ctx->stats->sha256_ops)); + + atomic64_add(sctx->count, &(nx_ctx->stats->sha256_bytes)); + memcpy(out, csbcpb->cpb.sha256.message_digest, SHA256_DIGEST_SIZE); +out: + spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); + return rc; +} + +static int nx_sha256_export(struct shash_desc *desc, void *out) +{ + struct sha256_state_be *sctx = shash_desc_ctx(desc); + union { + u8 *u8; + u32 *u32; + u64 *u64; + } p = { .u8 = out }; + int i; + + for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(*p.u32); i++) + put_unaligned(be32_to_cpu(sctx->state[i]), p.u32++); + + put_unaligned(sctx->count, p.u64++); + return 0; +} + +static int nx_sha256_import(struct shash_desc *desc, const void *in) +{ + struct sha256_state_be *sctx = shash_desc_ctx(desc); + union { + const u8 *u8; + const u32 *u32; + const u64 *u64; + } p = { .u8 = in }; + int i; + + for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(*p.u32); i++) + sctx->state[i] = cpu_to_be32(get_unaligned(p.u32++)); + + sctx->count = get_unaligned(p.u64++); + return 0; +} + +struct shash_alg nx_shash_sha256_alg = { + .digestsize = SHA256_DIGEST_SIZE, + .init = nx_sha256_init, + .update = nx_sha256_update, + .finup = nx_sha256_finup, + .export = nx_sha256_export, + .import = nx_sha256_import, + .init_tfm = nx_crypto_ctx_sha256_init, + .exit_tfm = nx_crypto_ctx_shash_exit, + .descsize = sizeof(struct sha256_state_be), + .statesize = sizeof(struct sha256_state_be), + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-nx", + .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct nx_crypto_ctx), + } +}; diff --git a/drivers/crypto/nx/nx-sha512.c b/drivers/crypto/nx/nx-sha512.c new file mode 100644 index 000000000..f74776b7d --- /dev/null +++ b/drivers/crypto/nx/nx-sha512.c @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * SHA-512 routines supporting the Power 7+ Nest Accelerators driver + * + * Copyright (C) 2011-2012 International Business Machines Inc. + * + * Author: Kent Yoder + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nx_csbcpb.h" +#include "nx.h" + +struct sha512_state_be { + __be64 state[SHA512_DIGEST_SIZE / 8]; + u64 count[2]; +}; + +static int nx_crypto_ctx_sha512_init(struct crypto_shash *tfm) +{ + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(tfm); + int err; + + err = nx_crypto_ctx_sha_init(tfm); + if (err) + return err; + + nx_ctx_init(nx_ctx, HCOP_FC_SHA); + + nx_ctx->ap = &nx_ctx->props[NX_PROPS_SHA512]; + + NX_CPB_SET_DIGEST_SIZE(nx_ctx->csbcpb, NX_DS_SHA512); + + return 0; +} + +static int nx_sha512_init(struct shash_desc *desc) +{ + struct sha512_state_be *sctx = shash_desc_ctx(desc); + + sctx->state[0] = __cpu_to_be64(SHA512_H0); + sctx->state[1] = __cpu_to_be64(SHA512_H1); + sctx->state[2] = __cpu_to_be64(SHA512_H2); + sctx->state[3] = __cpu_to_be64(SHA512_H3); + sctx->state[4] = __cpu_to_be64(SHA512_H4); + sctx->state[5] = __cpu_to_be64(SHA512_H5); + sctx->state[6] = __cpu_to_be64(SHA512_H6); + sctx->state[7] = __cpu_to_be64(SHA512_H7); + sctx->count[0] = 0; + sctx->count[1] = 0; + + return 0; +} + +static int nx_sha512_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); + struct sha512_state_be *sctx = shash_desc_ctx(desc); + struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; + u64 to_process, leftover, total = len; + struct nx_sg *out_sg; + unsigned long irq_flags; + int rc = 0; + int data_len; + u32 max_sg_len; + + spin_lock_irqsave(&nx_ctx->lock, irq_flags); + + memcpy(csbcpb->cpb.sha512.message_digest, sctx->state, SHA512_DIGEST_SIZE); + NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; + NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; + + max_sg_len = min_t(u64, nx_ctx->ap->sglen, + nx_driver.of.max_sg_len/sizeof(struct nx_sg)); + max_sg_len = min_t(u64, max_sg_len, + nx_ctx->ap->databytelen/NX_PAGE_SIZE); + + data_len = SHA512_DIGEST_SIZE; + out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *)sctx->state, + &data_len, max_sg_len); + nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg); + + if (data_len != SHA512_DIGEST_SIZE) { + rc = -EINVAL; + goto out; + } + + do { + struct nx_sg *in_sg = nx_ctx->in_sg; + + to_process = total & ~(SHA512_BLOCK_SIZE - 1); + + data_len = to_process; + in_sg = nx_build_sg_list(in_sg, (u8 *) data, + &data_len, max_sg_len); + + nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg); + + to_process = data_len; + leftover = total - to_process; + + /* + * we've hit the nx chip previously and we're updating + * again, so copy over the partial digest. + */ + memcpy(csbcpb->cpb.sha512.input_partial_digest, + csbcpb->cpb.sha512.message_digest, + SHA512_DIGEST_SIZE); + + if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) { + rc = -EINVAL; + goto out; + } + + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 0); + if (rc) + goto out; + + atomic_inc(&(nx_ctx->stats->sha512_ops)); + + total -= to_process; + data += to_process; + sctx->count[0] += to_process; + if (sctx->count[0] < to_process) + sctx->count[1]++; + } while (leftover >= SHA512_BLOCK_SIZE); + + rc = leftover; + memcpy(sctx->state, csbcpb->cpb.sha512.message_digest, SHA512_DIGEST_SIZE); +out: + spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); + return rc; +} + +static int nx_sha512_finup(struct shash_desc *desc, const u8 *src, + unsigned int nbytes, u8 *out) +{ + struct sha512_state_be *sctx = shash_desc_ctx(desc); + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); + struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; + struct nx_sg *in_sg, *out_sg; + u32 max_sg_len; + unsigned long irq_flags; + u64 count0, count1; + int rc = 0; + int len; + + spin_lock_irqsave(&nx_ctx->lock, irq_flags); + + max_sg_len = min_t(u64, nx_ctx->ap->sglen, + nx_driver.of.max_sg_len/sizeof(struct nx_sg)); + max_sg_len = min_t(u64, max_sg_len, + nx_ctx->ap->databytelen/NX_PAGE_SIZE); + + /* final is represented by continuing the operation and indicating that + * this is not an intermediate operation + * copy over the partial digest */ + memcpy(csbcpb->cpb.sha512.input_partial_digest, sctx->state, SHA512_DIGEST_SIZE); + NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; + NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; + + count0 = sctx->count[0] + nbytes; + count1 = sctx->count[1]; + + csbcpb->cpb.sha512.message_bit_length_lo = count0 << 3; + csbcpb->cpb.sha512.message_bit_length_hi = (count1 << 3) | + (count0 >> 61); + + len = nbytes; + in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)src, &len, max_sg_len); + + if (len != nbytes) { + rc = -EINVAL; + goto out; + } + + len = SHA512_DIGEST_SIZE; + out_sg = nx_build_sg_list(nx_ctx->out_sg, out, &len, + max_sg_len); + + nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg); + nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg); + + if (!nx_ctx->op.outlen) { + rc = -EINVAL; + goto out; + } + + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 0); + if (rc) + goto out; + + atomic_inc(&(nx_ctx->stats->sha512_ops)); + atomic64_add(count0, &(nx_ctx->stats->sha512_bytes)); + + memcpy(out, csbcpb->cpb.sha512.message_digest, SHA512_DIGEST_SIZE); +out: + spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); + return rc; +} + +static int nx_sha512_export(struct shash_desc *desc, void *out) +{ + struct sha512_state_be *sctx = shash_desc_ctx(desc); + union { + u8 *u8; + u64 *u64; + } p = { .u8 = out }; + int i; + + for (i = 0; i < SHA512_DIGEST_SIZE / sizeof(*p.u64); i++) + put_unaligned(be64_to_cpu(sctx->state[i]), p.u64++); + + put_unaligned(sctx->count[0], p.u64++); + put_unaligned(sctx->count[1], p.u64++); + return 0; +} + +static int nx_sha512_import(struct shash_desc *desc, const void *in) +{ + struct sha512_state_be *sctx = shash_desc_ctx(desc); + union { + const u8 *u8; + const u64 *u64; + } p = { .u8 = in }; + int i; + + for (i = 0; i < SHA512_DIGEST_SIZE / sizeof(*p.u64); i++) + sctx->state[i] = cpu_to_be64(get_unaligned(p.u64++)); + + sctx->count[0] = get_unaligned(p.u64++); + sctx->count[1] = get_unaligned(p.u64++); + return 0; +} + +struct shash_alg nx_shash_sha512_alg = { + .digestsize = SHA512_DIGEST_SIZE, + .init = nx_sha512_init, + .update = nx_sha512_update, + .finup = nx_sha512_finup, + .export = nx_sha512_export, + .import = nx_sha512_import, + .init_tfm = nx_crypto_ctx_sha512_init, + .exit_tfm = nx_crypto_ctx_shash_exit, + .descsize = sizeof(struct sha512_state_be), + .statesize = sizeof(struct sha512_state_be), + .base = { + .cra_name = "sha512", + .cra_driver_name = "sha512-nx", + .cra_priority = 300, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct nx_crypto_ctx), + } +}; diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c new file mode 100644 index 000000000..1b7509e2c --- /dev/null +++ b/drivers/crypto/nx/nx.c @@ -0,0 +1,830 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Routines supporting the Power 7+ Nest Accelerators driver + * + * Copyright (C) 2011-2012 International Business Machines Inc. + * + * Author: Kent Yoder + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nx_csbcpb.h" +#include "nx.h" + + +/** + * nx_hcall_sync - make an H_COP_OP hcall for the passed in op structure + * + * @nx_ctx: the crypto context handle + * @op: PFO operation struct to pass in + * @may_sleep: flag indicating the request can sleep + * + * Make the hcall, retrying while the hardware is busy. If we cannot yield + * the thread, limit the number of retries to 10 here. + */ +int nx_hcall_sync(struct nx_crypto_ctx *nx_ctx, + struct vio_pfo_op *op, + u32 may_sleep) +{ + int rc, retries = 10; + struct vio_dev *viodev = nx_driver.viodev; + + atomic_inc(&(nx_ctx->stats->sync_ops)); + + do { + rc = vio_h_cop_sync(viodev, op); + } while (rc == -EBUSY && !may_sleep && retries--); + + if (rc) { + dev_dbg(&viodev->dev, "vio_h_cop_sync failed: rc: %d " + "hcall rc: %ld\n", rc, op->hcall_err); + atomic_inc(&(nx_ctx->stats->errors)); + atomic_set(&(nx_ctx->stats->last_error), op->hcall_err); + atomic_set(&(nx_ctx->stats->last_error_pid), current->pid); + } + + return rc; +} + +/** + * nx_build_sg_list - build an NX scatter list describing a single buffer + * + * @sg_head: pointer to the first scatter list element to build + * @start_addr: pointer to the linear buffer + * @len: length of the data at @start_addr + * @sgmax: the largest number of scatter list elements we're allowed to create + * + * This function will start writing nx_sg elements at @sg_head and keep + * writing them until all of the data from @start_addr is described or + * until sgmax elements have been written. Scatter list elements will be + * created such that none of the elements describes a buffer that crosses a 4K + * boundary. + */ +struct nx_sg *nx_build_sg_list(struct nx_sg *sg_head, + u8 *start_addr, + unsigned int *len, + u32 sgmax) +{ + unsigned int sg_len = 0; + struct nx_sg *sg; + u64 sg_addr = (u64)start_addr; + u64 end_addr; + + /* determine the start and end for this address range - slightly + * different if this is in VMALLOC_REGION */ + if (is_vmalloc_addr(start_addr)) + sg_addr = page_to_phys(vmalloc_to_page(start_addr)) + + offset_in_page(sg_addr); + else + sg_addr = __pa(sg_addr); + + end_addr = sg_addr + *len; + + /* each iteration will write one struct nx_sg element and add the + * length of data described by that element to sg_len. Once @len bytes + * have been described (or @sgmax elements have been written), the + * loop ends. min_t is used to ensure @end_addr falls on the same page + * as sg_addr, if not, we need to create another nx_sg element for the + * data on the next page. + * + * Also when using vmalloc'ed data, every time that a system page + * boundary is crossed the physical address needs to be re-calculated. + */ + for (sg = sg_head; sg_len < *len; sg++) { + u64 next_page; + + sg->addr = sg_addr; + sg_addr = min_t(u64, NX_PAGE_NUM(sg_addr + NX_PAGE_SIZE), + end_addr); + + next_page = (sg->addr & PAGE_MASK) + PAGE_SIZE; + sg->len = min_t(u64, sg_addr, next_page) - sg->addr; + sg_len += sg->len; + + if (sg_addr >= next_page && + is_vmalloc_addr(start_addr + sg_len)) { + sg_addr = page_to_phys(vmalloc_to_page( + start_addr + sg_len)); + end_addr = sg_addr + *len - sg_len; + } + + if ((sg - sg_head) == sgmax) { + sg++; + break; + } + } + *len = sg_len; + + /* return the moved sg_head pointer */ + return sg; +} + +/** + * nx_walk_and_build - walk a linux scatterlist and build an nx scatterlist + * + * @nx_dst: pointer to the first nx_sg element to write + * @sglen: max number of nx_sg entries we're allowed to write + * @sg_src: pointer to the source linux scatterlist to walk + * @start: number of bytes to fast-forward past at the beginning of @sg_src + * @src_len: number of bytes to walk in @sg_src + */ +struct nx_sg *nx_walk_and_build(struct nx_sg *nx_dst, + unsigned int sglen, + struct scatterlist *sg_src, + unsigned int start, + unsigned int *src_len) +{ + struct scatter_walk walk; + struct nx_sg *nx_sg = nx_dst; + unsigned int n, len = *src_len; + + /* we need to fast forward through @start bytes first */ + scatterwalk_start_at_pos(&walk, sg_src, start); + + while (len && (nx_sg - nx_dst) < sglen) { + n = scatterwalk_next(&walk, len); + + nx_sg = nx_build_sg_list(nx_sg, walk.addr, &n, sglen - (nx_sg - nx_dst)); + + scatterwalk_done_src(&walk, n); + len -= n; + } + /* update to_process */ + *src_len -= len; + + /* return the moved destination pointer */ + return nx_sg; +} + +/** + * trim_sg_list - ensures the bound in sg list. + * @sg: sg list head + * @end: sg lisg end + * @delta: is the amount we need to crop in order to bound the list. + * @nbytes: length of data in the scatterlists or data length - whichever + * is greater. + */ +static long int trim_sg_list(struct nx_sg *sg, + struct nx_sg *end, + unsigned int delta, + unsigned int *nbytes) +{ + long int oplen; + long int data_back; + unsigned int is_delta = delta; + + while (delta && end > sg) { + struct nx_sg *last = end - 1; + + if (last->len > delta) { + last->len -= delta; + delta = 0; + } else { + end--; + delta -= last->len; + } + } + + /* There are cases where we need to crop list in order to make it + * a block size multiple, but we also need to align data. In order to + * that we need to calculate how much we need to put back to be + * processed + */ + oplen = (sg - end) * sizeof(struct nx_sg); + if (is_delta) { + data_back = (abs(oplen) / AES_BLOCK_SIZE) * sg->len; + data_back = *nbytes - (data_back & ~(AES_BLOCK_SIZE - 1)); + *nbytes -= data_back; + } + + return oplen; +} + +/** + * nx_build_sg_lists - walk the input scatterlists and build arrays of NX + * scatterlists based on them. + * + * @nx_ctx: NX crypto context for the lists we're building + * @iv: iv data, if the algorithm requires it + * @dst: destination scatterlist + * @src: source scatterlist + * @nbytes: length of data described in the scatterlists + * @offset: number of bytes to fast-forward past at the beginning of + * scatterlists. + * @oiv: destination for the iv data, if the algorithm requires it + * + * This is common code shared by all the AES algorithms. It uses the crypto + * scatterlist walk routines to traverse input and output scatterlists, building + * corresponding NX scatterlists + */ +int nx_build_sg_lists(struct nx_crypto_ctx *nx_ctx, + const u8 *iv, + struct scatterlist *dst, + struct scatterlist *src, + unsigned int *nbytes, + unsigned int offset, + u8 *oiv) +{ + unsigned int delta = 0; + unsigned int total = *nbytes; + struct nx_sg *nx_insg = nx_ctx->in_sg; + struct nx_sg *nx_outsg = nx_ctx->out_sg; + unsigned int max_sg_len; + + max_sg_len = min_t(u64, nx_ctx->ap->sglen, + nx_driver.of.max_sg_len/sizeof(struct nx_sg)); + max_sg_len = min_t(u64, max_sg_len, + nx_ctx->ap->databytelen/NX_PAGE_SIZE); + + if (oiv) + memcpy(oiv, iv, AES_BLOCK_SIZE); + + *nbytes = min_t(u64, *nbytes, nx_ctx->ap->databytelen); + + nx_outsg = nx_walk_and_build(nx_outsg, max_sg_len, dst, + offset, nbytes); + nx_insg = nx_walk_and_build(nx_insg, max_sg_len, src, + offset, nbytes); + + if (*nbytes < total) + delta = *nbytes - (*nbytes & ~(AES_BLOCK_SIZE - 1)); + + /* these lengths should be negative, which will indicate to phyp that + * the input and output parameters are scatterlists, not linear + * buffers */ + nx_ctx->op.inlen = trim_sg_list(nx_ctx->in_sg, nx_insg, delta, nbytes); + nx_ctx->op.outlen = trim_sg_list(nx_ctx->out_sg, nx_outsg, delta, nbytes); + + return 0; +} + +/** + * nx_ctx_init - initialize an nx_ctx's vio_pfo_op struct + * + * @nx_ctx: the nx context to initialize + * @function: the function code for the op + */ +void nx_ctx_init(struct nx_crypto_ctx *nx_ctx, unsigned int function) +{ + spin_lock_init(&nx_ctx->lock); + memset(nx_ctx->kmem, 0, nx_ctx->kmem_len); + nx_ctx->csbcpb->csb.valid |= NX_CSB_VALID_BIT; + + nx_ctx->op.flags = function; + nx_ctx->op.csbcpb = __pa(nx_ctx->csbcpb); + nx_ctx->op.in = __pa(nx_ctx->in_sg); + nx_ctx->op.out = __pa(nx_ctx->out_sg); + + if (nx_ctx->csbcpb_aead) { + nx_ctx->csbcpb_aead->csb.valid |= NX_CSB_VALID_BIT; + + nx_ctx->op_aead.flags = function; + nx_ctx->op_aead.csbcpb = __pa(nx_ctx->csbcpb_aead); + nx_ctx->op_aead.in = __pa(nx_ctx->in_sg); + nx_ctx->op_aead.out = __pa(nx_ctx->out_sg); + } +} + +static void nx_of_update_status(struct device *dev, + struct property *p, + struct nx_of *props) +{ + if (!strncmp(p->value, "okay", p->length)) { + props->status = NX_WAITING; + props->flags |= NX_OF_FLAG_STATUS_SET; + } else { + dev_info(dev, "%s: status '%s' is not 'okay'\n", __func__, + (char *)p->value); + } +} + +static void nx_of_update_sglen(struct device *dev, + struct property *p, + struct nx_of *props) +{ + if (p->length != sizeof(props->max_sg_len)) { + dev_err(dev, "%s: unexpected format for " + "ibm,max-sg-len property\n", __func__); + dev_dbg(dev, "%s: ibm,max-sg-len is %d bytes " + "long, expected %zd bytes\n", __func__, + p->length, sizeof(props->max_sg_len)); + return; + } + + props->max_sg_len = *(u32 *)p->value; + props->flags |= NX_OF_FLAG_MAXSGLEN_SET; +} + +static void nx_of_update_msc(struct device *dev, + struct property *p, + struct nx_of *props) +{ + struct msc_triplet *trip; + struct max_sync_cop *msc; + unsigned int bytes_so_far, i, lenp; + + msc = (struct max_sync_cop *)p->value; + lenp = p->length; + + /* You can't tell if the data read in for this property is sane by its + * size alone. This is because there are sizes embedded in the data + * structure. The best we can do is check lengths as we parse and bail + * as soon as a length error is detected. */ + bytes_so_far = 0; + + while ((bytes_so_far + sizeof(struct max_sync_cop)) <= lenp) { + bytes_so_far += sizeof(struct max_sync_cop); + + trip = msc->trip; + + for (i = 0; + ((bytes_so_far + sizeof(struct msc_triplet)) <= lenp) && + i < msc->triplets; + i++) { + if (msc->fc >= NX_MAX_FC || msc->mode >= NX_MAX_MODE) { + dev_err(dev, "unknown function code/mode " + "combo: %d/%d (ignored)\n", msc->fc, + msc->mode); + goto next_loop; + } + + if (!trip->sglen || trip->databytelen < NX_PAGE_SIZE) { + dev_warn(dev, "bogus sglen/databytelen: " + "%u/%u (ignored)\n", trip->sglen, + trip->databytelen); + goto next_loop; + } + + switch (trip->keybitlen) { + case 128: + case 160: + props->ap[msc->fc][msc->mode][0].databytelen = + trip->databytelen; + props->ap[msc->fc][msc->mode][0].sglen = + trip->sglen; + break; + case 192: + props->ap[msc->fc][msc->mode][1].databytelen = + trip->databytelen; + props->ap[msc->fc][msc->mode][1].sglen = + trip->sglen; + break; + case 256: + if (msc->fc == NX_FC_AES) { + props->ap[msc->fc][msc->mode][2]. + databytelen = trip->databytelen; + props->ap[msc->fc][msc->mode][2].sglen = + trip->sglen; + } else if (msc->fc == NX_FC_AES_HMAC || + msc->fc == NX_FC_SHA) { + props->ap[msc->fc][msc->mode][1]. + databytelen = trip->databytelen; + props->ap[msc->fc][msc->mode][1].sglen = + trip->sglen; + } else { + dev_warn(dev, "unknown function " + "code/key bit len combo" + ": (%u/256)\n", msc->fc); + } + break; + case 512: + props->ap[msc->fc][msc->mode][2].databytelen = + trip->databytelen; + props->ap[msc->fc][msc->mode][2].sglen = + trip->sglen; + break; + default: + dev_warn(dev, "unknown function code/key bit " + "len combo: (%u/%u)\n", msc->fc, + trip->keybitlen); + break; + } +next_loop: + bytes_so_far += sizeof(struct msc_triplet); + trip++; + } + + msc = (struct max_sync_cop *)trip; + } + + props->flags |= NX_OF_FLAG_MAXSYNCCOP_SET; +} + +/** + * nx_of_init - read openFirmware values from the device tree + * + * @dev: device handle + * @props: pointer to struct to hold the properties values + * + * Called once at driver probe time, this function will read out the + * openFirmware properties we use at runtime. If all the OF properties are + * acceptable, when we exit this function props->flags will indicate that + * we're ready to register our crypto algorithms. + */ +static void nx_of_init(struct device *dev, struct nx_of *props) +{ + struct device_node *base_node = dev->of_node; + struct property *p; + + p = of_find_property(base_node, "status", NULL); + if (!p) + dev_info(dev, "%s: property 'status' not found\n", __func__); + else + nx_of_update_status(dev, p, props); + + p = of_find_property(base_node, "ibm,max-sg-len", NULL); + if (!p) + dev_info(dev, "%s: property 'ibm,max-sg-len' not found\n", + __func__); + else + nx_of_update_sglen(dev, p, props); + + p = of_find_property(base_node, "ibm,max-sync-cop", NULL); + if (!p) + dev_info(dev, "%s: property 'ibm,max-sync-cop' not found\n", + __func__); + else + nx_of_update_msc(dev, p, props); +} + +static bool nx_check_prop(struct device *dev, u32 fc, u32 mode, int slot) +{ + struct alg_props *props = &nx_driver.of.ap[fc][mode][slot]; + + if (!props->sglen || props->databytelen < NX_PAGE_SIZE) { + if (dev) + dev_warn(dev, "bogus sglen/databytelen for %u/%u/%u: " + "%u/%u (ignored)\n", fc, mode, slot, + props->sglen, props->databytelen); + return false; + } + + return true; +} + +static bool nx_check_props(struct device *dev, u32 fc, u32 mode) +{ + int i; + + for (i = 0; i < 3; i++) + if (!nx_check_prop(dev, fc, mode, i)) + return false; + + return true; +} + +static int nx_register_skcipher(struct skcipher_alg *alg, u32 fc, u32 mode) +{ + return nx_check_props(&nx_driver.viodev->dev, fc, mode) ? + crypto_register_skcipher(alg) : 0; +} + +static int nx_register_aead(struct aead_alg *alg, u32 fc, u32 mode) +{ + return nx_check_props(&nx_driver.viodev->dev, fc, mode) ? + crypto_register_aead(alg) : 0; +} + +static int nx_register_shash(struct shash_alg *alg, u32 fc, u32 mode, int slot) +{ + return (slot >= 0 ? nx_check_prop(&nx_driver.viodev->dev, + fc, mode, slot) : + nx_check_props(&nx_driver.viodev->dev, fc, mode)) ? + crypto_register_shash(alg) : 0; +} + +static void nx_unregister_skcipher(struct skcipher_alg *alg, u32 fc, u32 mode) +{ + if (nx_check_props(NULL, fc, mode)) + crypto_unregister_skcipher(alg); +} + +static void nx_unregister_aead(struct aead_alg *alg, u32 fc, u32 mode) +{ + if (nx_check_props(NULL, fc, mode)) + crypto_unregister_aead(alg); +} + +static void nx_unregister_shash(struct shash_alg *alg, u32 fc, u32 mode, + int slot) +{ + if (slot >= 0 ? nx_check_prop(NULL, fc, mode, slot) : + nx_check_props(NULL, fc, mode)) + crypto_unregister_shash(alg); +} + +/** + * nx_register_algs - register algorithms with the crypto API + * + * Called from nx_probe() + * + * If all OF properties are in an acceptable state, the driver flags will + * indicate that we're ready and we'll create our debugfs files and register + * out crypto algorithms. + */ +static int nx_register_algs(void) +{ + int rc = -1; + + if (nx_driver.of.flags != NX_OF_FLAG_MASK_READY) + goto out; + + memset(&nx_driver.stats, 0, sizeof(struct nx_stats)); + + NX_DEBUGFS_INIT(&nx_driver); + + nx_driver.of.status = NX_OKAY; + + rc = nx_register_skcipher(&nx_ecb_aes_alg, NX_FC_AES, NX_MODE_AES_ECB); + if (rc) + goto out; + + rc = nx_register_skcipher(&nx_cbc_aes_alg, NX_FC_AES, NX_MODE_AES_CBC); + if (rc) + goto out_unreg_ecb; + + rc = nx_register_skcipher(&nx_ctr3686_aes_alg, NX_FC_AES, + NX_MODE_AES_CTR); + if (rc) + goto out_unreg_cbc; + + rc = nx_register_aead(&nx_gcm_aes_alg, NX_FC_AES, NX_MODE_AES_GCM); + if (rc) + goto out_unreg_ctr3686; + + rc = nx_register_aead(&nx_gcm4106_aes_alg, NX_FC_AES, NX_MODE_AES_GCM); + if (rc) + goto out_unreg_gcm; + + rc = nx_register_aead(&nx_ccm_aes_alg, NX_FC_AES, NX_MODE_AES_CCM); + if (rc) + goto out_unreg_gcm4106; + + rc = nx_register_aead(&nx_ccm4309_aes_alg, NX_FC_AES, NX_MODE_AES_CCM); + if (rc) + goto out_unreg_ccm; + + rc = nx_register_shash(&nx_shash_sha256_alg, NX_FC_SHA, NX_MODE_SHA, + NX_PROPS_SHA256); + if (rc) + goto out_unreg_ccm4309; + + rc = nx_register_shash(&nx_shash_sha512_alg, NX_FC_SHA, NX_MODE_SHA, + NX_PROPS_SHA512); + if (rc) + goto out_unreg_s256; + + rc = nx_register_shash(&nx_shash_aes_xcbc_alg, + NX_FC_AES, NX_MODE_AES_XCBC_MAC, -1); + if (rc) + goto out_unreg_s512; + + goto out; + +out_unreg_s512: + nx_unregister_shash(&nx_shash_sha512_alg, NX_FC_SHA, NX_MODE_SHA, + NX_PROPS_SHA512); +out_unreg_s256: + nx_unregister_shash(&nx_shash_sha256_alg, NX_FC_SHA, NX_MODE_SHA, + NX_PROPS_SHA256); +out_unreg_ccm4309: + nx_unregister_aead(&nx_ccm4309_aes_alg, NX_FC_AES, NX_MODE_AES_CCM); +out_unreg_ccm: + nx_unregister_aead(&nx_ccm_aes_alg, NX_FC_AES, NX_MODE_AES_CCM); +out_unreg_gcm4106: + nx_unregister_aead(&nx_gcm4106_aes_alg, NX_FC_AES, NX_MODE_AES_GCM); +out_unreg_gcm: + nx_unregister_aead(&nx_gcm_aes_alg, NX_FC_AES, NX_MODE_AES_GCM); +out_unreg_ctr3686: + nx_unregister_skcipher(&nx_ctr3686_aes_alg, NX_FC_AES, NX_MODE_AES_CTR); +out_unreg_cbc: + nx_unregister_skcipher(&nx_cbc_aes_alg, NX_FC_AES, NX_MODE_AES_CBC); +out_unreg_ecb: + nx_unregister_skcipher(&nx_ecb_aes_alg, NX_FC_AES, NX_MODE_AES_ECB); +out: + return rc; +} + +/** + * nx_crypto_ctx_init - create and initialize a crypto api context + * + * @nx_ctx: the crypto api context + * @fc: function code for the context + * @mode: the function code specific mode for this context + */ +static int nx_crypto_ctx_init(struct nx_crypto_ctx *nx_ctx, u32 fc, u32 mode) +{ + if (nx_driver.of.status != NX_OKAY) { + pr_err("Attempt to initialize NX crypto context while device " + "is not available!\n"); + return -ENODEV; + } + + /* we need an extra page for csbcpb_aead for these modes */ + if (mode == NX_MODE_AES_GCM || mode == NX_MODE_AES_CCM) + nx_ctx->kmem_len = (5 * NX_PAGE_SIZE) + + sizeof(struct nx_csbcpb); + else + nx_ctx->kmem_len = (4 * NX_PAGE_SIZE) + + sizeof(struct nx_csbcpb); + + nx_ctx->kmem = kmalloc(nx_ctx->kmem_len, GFP_KERNEL); + if (!nx_ctx->kmem) + return -ENOMEM; + + /* the csbcpb and scatterlists must be 4K aligned pages */ + nx_ctx->csbcpb = (struct nx_csbcpb *)(round_up((u64)nx_ctx->kmem, + (u64)NX_PAGE_SIZE)); + nx_ctx->in_sg = (struct nx_sg *)((u8 *)nx_ctx->csbcpb + NX_PAGE_SIZE); + nx_ctx->out_sg = (struct nx_sg *)((u8 *)nx_ctx->in_sg + NX_PAGE_SIZE); + + if (mode == NX_MODE_AES_GCM || mode == NX_MODE_AES_CCM) + nx_ctx->csbcpb_aead = + (struct nx_csbcpb *)((u8 *)nx_ctx->out_sg + + NX_PAGE_SIZE); + + /* give each context a pointer to global stats and their OF + * properties */ + nx_ctx->stats = &nx_driver.stats; + memcpy(nx_ctx->props, nx_driver.of.ap[fc][mode], + sizeof(struct alg_props) * 3); + + return 0; +} + +/* entry points from the crypto tfm initializers */ +int nx_crypto_ctx_aes_ccm_init(struct crypto_aead *tfm) +{ + crypto_aead_set_reqsize(tfm, sizeof(struct nx_ccm_rctx)); + return nx_crypto_ctx_init(crypto_aead_ctx(tfm), NX_FC_AES, + NX_MODE_AES_CCM); +} + +int nx_crypto_ctx_aes_gcm_init(struct crypto_aead *tfm) +{ + crypto_aead_set_reqsize(tfm, sizeof(struct nx_gcm_rctx)); + return nx_crypto_ctx_init(crypto_aead_ctx(tfm), NX_FC_AES, + NX_MODE_AES_GCM); +} + +int nx_crypto_ctx_aes_ctr_init(struct crypto_skcipher *tfm) +{ + return nx_crypto_ctx_init(crypto_skcipher_ctx(tfm), NX_FC_AES, + NX_MODE_AES_CTR); +} + +int nx_crypto_ctx_aes_cbc_init(struct crypto_skcipher *tfm) +{ + return nx_crypto_ctx_init(crypto_skcipher_ctx(tfm), NX_FC_AES, + NX_MODE_AES_CBC); +} + +int nx_crypto_ctx_aes_ecb_init(struct crypto_skcipher *tfm) +{ + return nx_crypto_ctx_init(crypto_skcipher_ctx(tfm), NX_FC_AES, + NX_MODE_AES_ECB); +} + +int nx_crypto_ctx_sha_init(struct crypto_shash *tfm) +{ + return nx_crypto_ctx_init(crypto_shash_ctx(tfm), NX_FC_SHA, NX_MODE_SHA); +} + +int nx_crypto_ctx_aes_xcbc_init(struct crypto_shash *tfm) +{ + return nx_crypto_ctx_init(crypto_shash_ctx(tfm), NX_FC_AES, + NX_MODE_AES_XCBC_MAC); +} + +/** + * nx_crypto_ctx_exit - destroy a crypto api context + * + * @nx_ctx: the crypto api context + * + * As crypto API contexts are destroyed, this exit hook is called to free the + * memory associated with it. + */ +void nx_crypto_ctx_exit(struct nx_crypto_ctx *nx_ctx) +{ + kfree_sensitive(nx_ctx->kmem); + nx_ctx->csbcpb = NULL; + nx_ctx->csbcpb_aead = NULL; + nx_ctx->in_sg = NULL; + nx_ctx->out_sg = NULL; +} + +void nx_crypto_ctx_skcipher_exit(struct crypto_skcipher *tfm) +{ + nx_crypto_ctx_exit(crypto_skcipher_ctx(tfm)); +} + +void nx_crypto_ctx_aead_exit(struct crypto_aead *tfm) +{ + struct nx_crypto_ctx *nx_ctx = crypto_aead_ctx(tfm); + + kfree_sensitive(nx_ctx->kmem); +} + +void nx_crypto_ctx_shash_exit(struct crypto_shash *tfm) +{ + nx_crypto_ctx_exit(crypto_shash_ctx(tfm)); +} + +static int nx_probe(struct vio_dev *viodev, const struct vio_device_id *id) +{ + dev_dbg(&viodev->dev, "driver probed: %s resource id: 0x%x\n", + viodev->name, viodev->resource_id); + + if (nx_driver.viodev) { + dev_err(&viodev->dev, "%s: Attempt to register more than one " + "instance of the hardware\n", __func__); + return -EINVAL; + } + + nx_driver.viodev = viodev; + + nx_of_init(&viodev->dev, &nx_driver.of); + + return nx_register_algs(); +} + +static void nx_remove(struct vio_dev *viodev) +{ + dev_dbg(&viodev->dev, "entering nx_remove for UA 0x%x\n", + viodev->unit_address); + + if (nx_driver.of.status == NX_OKAY) { + NX_DEBUGFS_FINI(&nx_driver); + + nx_unregister_shash(&nx_shash_aes_xcbc_alg, + NX_FC_AES, NX_MODE_AES_XCBC_MAC, -1); + nx_unregister_shash(&nx_shash_sha512_alg, + NX_FC_SHA, NX_MODE_SHA, NX_PROPS_SHA256); + nx_unregister_shash(&nx_shash_sha256_alg, + NX_FC_SHA, NX_MODE_SHA, NX_PROPS_SHA512); + nx_unregister_aead(&nx_ccm4309_aes_alg, + NX_FC_AES, NX_MODE_AES_CCM); + nx_unregister_aead(&nx_ccm_aes_alg, NX_FC_AES, NX_MODE_AES_CCM); + nx_unregister_aead(&nx_gcm4106_aes_alg, + NX_FC_AES, NX_MODE_AES_GCM); + nx_unregister_aead(&nx_gcm_aes_alg, + NX_FC_AES, NX_MODE_AES_GCM); + nx_unregister_skcipher(&nx_ctr3686_aes_alg, + NX_FC_AES, NX_MODE_AES_CTR); + nx_unregister_skcipher(&nx_cbc_aes_alg, NX_FC_AES, + NX_MODE_AES_CBC); + nx_unregister_skcipher(&nx_ecb_aes_alg, NX_FC_AES, + NX_MODE_AES_ECB); + } +} + + +/* module wide initialization/cleanup */ +static int __init nx_init(void) +{ + return vio_register_driver(&nx_driver.viodriver); +} + +static void __exit nx_fini(void) +{ + vio_unregister_driver(&nx_driver.viodriver); +} + +static const struct vio_device_id nx_crypto_driver_ids[] = { + { "ibm,sym-encryption-v1", "ibm,sym-encryption" }, + { "", "" } +}; +MODULE_DEVICE_TABLE(vio, nx_crypto_driver_ids); + +/* driver state structure */ +struct nx_crypto_driver nx_driver = { + .viodriver = { + .id_table = nx_crypto_driver_ids, + .probe = nx_probe, + .remove = nx_remove, + .name = NX_NAME, + }, +}; + +module_init(nx_init); +module_exit(nx_fini); + +MODULE_AUTHOR("Kent Yoder "); +MODULE_DESCRIPTION(NX_STRING); +MODULE_LICENSE("GPL"); +MODULE_VERSION(NX_VERSION); diff --git a/drivers/crypto/nx/nx.h b/drivers/crypto/nx/nx.h new file mode 100644 index 000000000..6dfabfbf8 --- /dev/null +++ b/drivers/crypto/nx/nx.h @@ -0,0 +1,197 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __NX_H__ +#define __NX_H__ + +#include +#include +#include +#include +#include + +#define NX_NAME "nx-crypto" +#define NX_STRING "IBM Power7+ Nest Accelerator Crypto Driver" +#define NX_VERSION "1.0" + +/* a scatterlist in the format PHYP is expecting */ +struct nx_sg { + u64 addr; + u32 rsvd; + u32 len; +} __attribute((packed)); + +#define NX_PAGE_SIZE (4096) +#define NX_MAX_SG_ENTRIES (NX_PAGE_SIZE/(sizeof(struct nx_sg))) + +enum nx_status { + NX_DISABLED, + NX_WAITING, + NX_OKAY +}; + +/* msc_triplet and max_sync_cop are used only to assist in parsing the + * openFirmware property */ +struct msc_triplet { + u32 keybitlen; + u32 databytelen; + u32 sglen; +} __packed; + +struct max_sync_cop { + u32 fc; + u32 mode; + u32 triplets; + struct msc_triplet trip[]; +} __packed; + +struct alg_props { + u32 databytelen; + u32 sglen; +}; + +#define NX_OF_FLAG_MAXSGLEN_SET (1) +#define NX_OF_FLAG_STATUS_SET (2) +#define NX_OF_FLAG_MAXSYNCCOP_SET (4) +#define NX_OF_FLAG_MASK_READY (NX_OF_FLAG_MAXSGLEN_SET | \ + NX_OF_FLAG_STATUS_SET | \ + NX_OF_FLAG_MAXSYNCCOP_SET) +struct nx_of { + u32 flags; + u32 max_sg_len; + enum nx_status status; + struct alg_props ap[NX_MAX_FC][NX_MAX_MODE][3]; +}; + +struct nx_stats { + atomic_t aes_ops; + atomic64_t aes_bytes; + atomic_t sha256_ops; + atomic64_t sha256_bytes; + atomic_t sha512_ops; + atomic64_t sha512_bytes; + + atomic_t sync_ops; + + atomic_t errors; + atomic_t last_error; + atomic_t last_error_pid; +}; + +struct nx_crypto_driver { + struct nx_stats stats; + struct nx_of of; + struct vio_dev *viodev; + struct vio_driver viodriver; + struct dentry *dfs_root; +}; + +#define NX_GCM4106_NONCE_LEN (4) +#define NX_GCM_CTR_OFFSET (12) +struct nx_gcm_rctx { + u8 iv[16]; +}; + +struct nx_gcm_priv { + u8 iauth_tag[16]; + u8 nonce[NX_GCM4106_NONCE_LEN]; +}; + +#define NX_CCM_AES_KEY_LEN (16) +#define NX_CCM4309_AES_KEY_LEN (19) +#define NX_CCM4309_NONCE_LEN (3) +struct nx_ccm_rctx { + u8 iv[16]; +}; + +struct nx_ccm_priv { + u8 b0[16]; + u8 iauth_tag[16]; + u8 oauth_tag[16]; + u8 nonce[NX_CCM4309_NONCE_LEN]; +}; + +struct nx_xcbc_priv { + u8 key[16]; +}; + +struct nx_ctr_priv { + u8 nonce[CTR_RFC3686_NONCE_SIZE]; +}; + +struct nx_crypto_ctx { + spinlock_t lock; /* synchronize access to the context */ + void *kmem; /* unaligned, kmalloc'd buffer */ + size_t kmem_len; /* length of kmem */ + struct nx_csbcpb *csbcpb; /* aligned page given to phyp @ hcall time */ + struct vio_pfo_op op; /* operation struct with hcall parameters */ + struct nx_csbcpb *csbcpb_aead; /* secondary csbcpb used by AEAD algs */ + struct vio_pfo_op op_aead;/* operation struct for csbcpb_aead */ + + struct nx_sg *in_sg; /* aligned pointer into kmem to an sg list */ + struct nx_sg *out_sg; /* aligned pointer into kmem to an sg list */ + + struct alg_props *ap; /* pointer into props based on our key size */ + struct alg_props props[3];/* openFirmware properties for requests */ + struct nx_stats *stats; /* pointer into an nx_crypto_driver for stats + reporting */ + + union { + struct nx_gcm_priv gcm; + struct nx_ccm_priv ccm; + struct nx_xcbc_priv xcbc; + struct nx_ctr_priv ctr; + } priv; +}; + +struct scatterlist; + +/* prototypes */ +int nx_crypto_ctx_aes_ccm_init(struct crypto_aead *tfm); +int nx_crypto_ctx_aes_gcm_init(struct crypto_aead *tfm); +int nx_crypto_ctx_aes_xcbc_init(struct crypto_shash *tfm); +int nx_crypto_ctx_aes_ctr_init(struct crypto_skcipher *tfm); +int nx_crypto_ctx_aes_cbc_init(struct crypto_skcipher *tfm); +int nx_crypto_ctx_aes_ecb_init(struct crypto_skcipher *tfm); +int nx_crypto_ctx_sha_init(struct crypto_shash *tfm); +void nx_crypto_ctx_exit(struct nx_crypto_ctx *nx_ctx); +void nx_crypto_ctx_skcipher_exit(struct crypto_skcipher *tfm); +void nx_crypto_ctx_aead_exit(struct crypto_aead *tfm); +void nx_crypto_ctx_shash_exit(struct crypto_shash *tfm); +void nx_ctx_init(struct nx_crypto_ctx *nx_ctx, unsigned int function); +int nx_hcall_sync(struct nx_crypto_ctx *ctx, struct vio_pfo_op *op, + u32 may_sleep); +struct nx_sg *nx_build_sg_list(struct nx_sg *, u8 *, unsigned int *, u32); +int nx_build_sg_lists(struct nx_crypto_ctx *nx_ctx, const u8 *iv, + struct scatterlist *dst, struct scatterlist *src, + unsigned int *nbytes, unsigned int offset, u8 *oiv); +struct nx_sg *nx_walk_and_build(struct nx_sg *, unsigned int, + struct scatterlist *, unsigned int, + unsigned int *); + +#ifdef CONFIG_DEBUG_FS +#define NX_DEBUGFS_INIT(drv) nx_debugfs_init(drv) +#define NX_DEBUGFS_FINI(drv) nx_debugfs_fini(drv) + +void nx_debugfs_init(struct nx_crypto_driver *); +void nx_debugfs_fini(struct nx_crypto_driver *); +#else +#define NX_DEBUGFS_INIT(drv) do {} while (0) +#define NX_DEBUGFS_FINI(drv) do {} while (0) +#endif + +#define NX_PAGE_NUM(x) ((u64)(x) & 0xfffffffffffff000ULL) + +extern struct skcipher_alg nx_cbc_aes_alg; +extern struct skcipher_alg nx_ecb_aes_alg; +extern struct aead_alg nx_gcm_aes_alg; +extern struct aead_alg nx_gcm4106_aes_alg; +extern struct skcipher_alg nx_ctr3686_aes_alg; +extern struct aead_alg nx_ccm_aes_alg; +extern struct aead_alg nx_ccm4309_aes_alg; +extern struct shash_alg nx_shash_aes_xcbc_alg; +extern struct shash_alg nx_shash_sha512_alg; +extern struct shash_alg nx_shash_sha256_alg; + +extern struct nx_crypto_driver nx_driver; + +#endif diff --git a/drivers/crypto/nx/nx_csbcpb.h b/drivers/crypto/nx/nx_csbcpb.h new file mode 100644 index 000000000..e64f7e36f --- /dev/null +++ b/drivers/crypto/nx/nx_csbcpb.h @@ -0,0 +1,206 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __NX_CSBCPB_H__ +#define __NX_CSBCPB_H__ + +struct cop_symcpb_aes_ecb { + u8 key[32]; + u8 __rsvd[80]; +} __packed; + +struct cop_symcpb_aes_cbc { + u8 iv[16]; + u8 key[32]; + u8 cv[16]; + u32 spbc; + u8 __rsvd[44]; +} __packed; + +struct cop_symcpb_aes_gca { + u8 in_pat[16]; + u8 key[32]; + u8 out_pat[16]; + u32 spbc; + u8 __rsvd[44]; +} __packed; + +struct cop_symcpb_aes_gcm { + u8 in_pat_or_aad[16]; + u8 iv_or_cnt[16]; + u64 bit_length_aad; + u64 bit_length_data; + u8 in_s0[16]; + u8 key[32]; + u8 __rsvd1[16]; + u8 out_pat_or_mac[16]; + u8 out_s0[16]; + u8 out_cnt[16]; + u32 spbc; + u8 __rsvd2[12]; +} __packed; + +struct cop_symcpb_aes_ctr { + u8 iv[16]; + u8 key[32]; + u8 cv[16]; + u32 spbc; + u8 __rsvd2[44]; +} __packed; + +struct cop_symcpb_aes_cca { + u8 b0[16]; + u8 b1[16]; + u8 key[16]; + u8 out_pat_or_b0[16]; + u32 spbc; + u8 __rsvd[44]; +} __packed; + +struct cop_symcpb_aes_ccm { + u8 in_pat_or_b0[16]; + u8 iv_or_ctr[16]; + u8 in_s0[16]; + u8 key[16]; + u8 __rsvd1[48]; + u8 out_pat_or_mac[16]; + u8 out_s0[16]; + u8 out_ctr[16]; + u32 spbc; + u8 __rsvd2[12]; +} __packed; + +struct cop_symcpb_aes_xcbc { + u8 cv[16]; + u8 key[16]; + u8 __rsvd1[16]; + u8 out_cv_mac[16]; + u32 spbc; + u8 __rsvd2[44]; +} __packed; + +struct cop_symcpb_sha256 { + u64 message_bit_length; + u64 __rsvd1; + u8 input_partial_digest[32]; + u8 message_digest[32]; + u32 spbc; + u8 __rsvd2[44]; +} __packed; + +struct cop_symcpb_sha512 { + u64 message_bit_length_hi; + u64 message_bit_length_lo; + u8 input_partial_digest[64]; + u8 __rsvd1[32]; + u8 message_digest[64]; + u32 spbc; + u8 __rsvd2[76]; +} __packed; + +#define NX_FDM_INTERMEDIATE 0x01 +#define NX_FDM_CONTINUATION 0x02 +#define NX_FDM_ENDE_ENCRYPT 0x80 + +#define NX_CPB_FDM(c) ((c)->cpb.hdr.fdm) +#define NX_CPB_KS_DS(c) ((c)->cpb.hdr.ks_ds) + +#define NX_CPB_KEY_SIZE(c) (NX_CPB_KS_DS(c) >> 4) +#define NX_CPB_SET_KEY_SIZE(c, x) NX_CPB_KS_DS(c) |= ((x) << 4) +#define NX_CPB_SET_DIGEST_SIZE(c, x) NX_CPB_KS_DS(c) |= (x) + +struct cop_symcpb_header { + u8 mode; + u8 fdm; + u8 ks_ds; + u8 pad_byte; + u8 __rsvd[12]; +} __packed; + +struct cop_parameter_block { + struct cop_symcpb_header hdr; + union { + struct cop_symcpb_aes_ecb aes_ecb; + struct cop_symcpb_aes_cbc aes_cbc; + struct cop_symcpb_aes_gca aes_gca; + struct cop_symcpb_aes_gcm aes_gcm; + struct cop_symcpb_aes_cca aes_cca; + struct cop_symcpb_aes_ccm aes_ccm; + struct cop_symcpb_aes_ctr aes_ctr; + struct cop_symcpb_aes_xcbc aes_xcbc; + struct cop_symcpb_sha256 sha256; + struct cop_symcpb_sha512 sha512; + }; +} __packed; + +#define NX_CSB_VALID_BIT 0x80 + +/* co-processor status block */ +struct cop_status_block { + u8 valid; + u8 crb_seq_number; + u8 completion_code; + u8 completion_extension; + __be32 processed_byte_count; + __be64 address; +} __packed; + +/* Nest accelerator workbook section 4.4 */ +struct nx_csbcpb { + unsigned char __rsvd[112]; + struct cop_status_block csb; + struct cop_parameter_block cpb; +} __packed; + +/* nx_csbcpb related definitions */ +#define NX_MODE_AES_ECB 0 +#define NX_MODE_AES_CBC 1 +#define NX_MODE_AES_GMAC 2 +#define NX_MODE_AES_GCA 3 +#define NX_MODE_AES_GCM 4 +#define NX_MODE_AES_CCA 5 +#define NX_MODE_AES_CCM 6 +#define NX_MODE_AES_CTR 7 +#define NX_MODE_AES_XCBC_MAC 20 +#define NX_MODE_SHA 0 +#define NX_MODE_SHA_HMAC 1 +#define NX_MODE_AES_CBC_HMAC_ETA 8 +#define NX_MODE_AES_CBC_HMAC_ATE 9 +#define NX_MODE_AES_CBC_HMAC_EAA 10 +#define NX_MODE_AES_CTR_HMAC_ETA 12 +#define NX_MODE_AES_CTR_HMAC_ATE 13 +#define NX_MODE_AES_CTR_HMAC_EAA 14 + +#define NX_FDM_CI_FULL 0 +#define NX_FDM_CI_FIRST 1 +#define NX_FDM_CI_LAST 2 +#define NX_FDM_CI_MIDDLE 3 + +#define NX_FDM_PR_NONE 0 +#define NX_FDM_PR_PAD 1 + +#define NX_KS_AES_128 1 +#define NX_KS_AES_192 2 +#define NX_KS_AES_256 3 + +#define NX_DS_SHA256 2 +#define NX_DS_SHA512 3 + +#define NX_FC_AES 0 +#define NX_FC_SHA 2 +#define NX_FC_AES_HMAC 6 + +#define NX_MAX_FC (NX_FC_AES_HMAC + 1) +#define NX_MAX_MODE (NX_MODE_AES_XCBC_MAC + 1) + +#define HCOP_FC_AES NX_FC_AES +#define HCOP_FC_SHA NX_FC_SHA +#define HCOP_FC_AES_HMAC NX_FC_AES_HMAC + +/* indices into the array of algorithm properties */ +#define NX_PROPS_AES_128 0 +#define NX_PROPS_AES_192 1 +#define NX_PROPS_AES_256 2 +#define NX_PROPS_SHA256 1 +#define NX_PROPS_SHA512 2 + +#endif diff --git a/drivers/crypto/nx/nx_debugfs.c b/drivers/crypto/nx/nx_debugfs.c new file mode 100644 index 000000000..ee7cd88bb --- /dev/null +++ b/drivers/crypto/nx/nx_debugfs.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * debugfs routines supporting the Power 7+ Nest Accelerators driver + * + * Copyright (C) 2011-2012 International Business Machines Inc. + * + * Author: Kent Yoder + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nx_csbcpb.h" +#include "nx.h" + +#ifdef CONFIG_DEBUG_FS + +/* + * debugfs + * + * For documentation on these attributes, please see: + * + * Documentation/ABI/testing/debugfs-pfo-nx-crypto + */ + +void nx_debugfs_init(struct nx_crypto_driver *drv) +{ + struct dentry *root; + + root = debugfs_create_dir(NX_NAME, NULL); + drv->dfs_root = root; + + debugfs_create_u32("aes_ops", S_IRUSR | S_IRGRP | S_IROTH, + root, &drv->stats.aes_ops.counter); + debugfs_create_u32("sha256_ops", S_IRUSR | S_IRGRP | S_IROTH, + root, &drv->stats.sha256_ops.counter); + debugfs_create_u32("sha512_ops", S_IRUSR | S_IRGRP | S_IROTH, + root, &drv->stats.sha512_ops.counter); + debugfs_create_u64("aes_bytes", S_IRUSR | S_IRGRP | S_IROTH, + root, &drv->stats.aes_bytes.counter); + debugfs_create_u64("sha256_bytes", S_IRUSR | S_IRGRP | S_IROTH, + root, &drv->stats.sha256_bytes.counter); + debugfs_create_u64("sha512_bytes", S_IRUSR | S_IRGRP | S_IROTH, + root, &drv->stats.sha512_bytes.counter); + debugfs_create_u32("errors", S_IRUSR | S_IRGRP | S_IROTH, + root, &drv->stats.errors.counter); + debugfs_create_u32("last_error", S_IRUSR | S_IRGRP | S_IROTH, + root, &drv->stats.last_error.counter); + debugfs_create_u32("last_error_pid", S_IRUSR | S_IRGRP | S_IROTH, + root, &drv->stats.last_error_pid.counter); +} + +void +nx_debugfs_fini(struct nx_crypto_driver *drv) +{ + debugfs_remove_recursive(drv->dfs_root); +} + +#endif diff --git a/drivers/crypto/omap-aes-gcm.c b/drivers/crypto/omap-aes-gcm.c new file mode 100644 index 000000000..c652f9d00 --- /dev/null +++ b/drivers/crypto/omap-aes-gcm.c @@ -0,0 +1,387 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Cryptographic API. + * + * Support for OMAP AES GCM HW acceleration. + * + * Copyright (c) 2016 Texas Instruments Incorporated + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "omap-crypto.h" +#include "omap-aes.h" + +static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd, + struct aead_request *req); + +static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret) +{ + struct aead_request *req = dd->aead_req; + + dd->in_sg = NULL; + dd->out_sg = NULL; + + crypto_finalize_aead_request(dd->engine, req, ret); + + pm_runtime_put_autosuspend(dd->dev); +} + +static void omap_aes_gcm_done_task(struct omap_aes_dev *dd) +{ + u8 *tag; + int alen, clen, i, ret = 0, nsg; + struct omap_aes_reqctx *rctx; + + alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE); + clen = ALIGN(dd->total, AES_BLOCK_SIZE); + rctx = aead_request_ctx(dd->aead_req); + + nsg = !!(dd->assoc_len && dd->total); + + dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len, + DMA_FROM_DEVICE); + dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE); + dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE); + omap_aes_crypt_dma_stop(dd); + + omap_crypto_cleanup(dd->out_sg, dd->orig_out, + dd->aead_req->assoclen, dd->total, + FLAGS_OUT_DATA_ST_SHIFT, dd->flags); + + if (dd->flags & FLAGS_ENCRYPT) + scatterwalk_map_and_copy(rctx->auth_tag, + dd->aead_req->dst, + dd->total + dd->aead_req->assoclen, + dd->authsize, 1); + + omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen, + FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags); + + omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen, + FLAGS_IN_DATA_ST_SHIFT, dd->flags); + + if (!(dd->flags & FLAGS_ENCRYPT)) { + tag = (u8 *)rctx->auth_tag; + for (i = 0; i < dd->authsize; i++) { + if (tag[i]) { + ret = -EBADMSG; + } + } + } + + omap_aes_gcm_finish_req(dd, ret); +} + +static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd, + struct aead_request *req) +{ + int alen, clen, cryptlen, assoclen, ret; + struct crypto_aead *aead = crypto_aead_reqtfm(req); + unsigned int authlen = crypto_aead_authsize(aead); + struct scatterlist *tmp, sg_arr[2]; + int nsg; + u16 flags; + + assoclen = req->assoclen; + cryptlen = req->cryptlen; + + if (dd->flags & FLAGS_RFC4106_GCM) + assoclen -= 8; + + if (!(dd->flags & FLAGS_ENCRYPT)) + cryptlen -= authlen; + + alen = ALIGN(assoclen, AES_BLOCK_SIZE); + clen = ALIGN(cryptlen, AES_BLOCK_SIZE); + + nsg = !!(assoclen && cryptlen); + + omap_aes_clear_copy_flags(dd); + + sg_init_table(dd->in_sgl, nsg + 1); + if (assoclen) { + tmp = req->src; + ret = omap_crypto_align_sg(&tmp, assoclen, + AES_BLOCK_SIZE, dd->in_sgl, + OMAP_CRYPTO_COPY_DATA | + OMAP_CRYPTO_ZERO_BUF | + OMAP_CRYPTO_FORCE_SINGLE_ENTRY, + FLAGS_ASSOC_DATA_ST_SHIFT, + &dd->flags); + if (ret) + return ret; + } + + if (cryptlen) { + tmp = scatterwalk_ffwd(sg_arr, req->src, req->assoclen); + + if (nsg) + sg_unmark_end(dd->in_sgl); + + ret = omap_crypto_align_sg(&tmp, cryptlen, + AES_BLOCK_SIZE, &dd->in_sgl[nsg], + OMAP_CRYPTO_COPY_DATA | + OMAP_CRYPTO_ZERO_BUF | + OMAP_CRYPTO_FORCE_SINGLE_ENTRY, + FLAGS_IN_DATA_ST_SHIFT, + &dd->flags); + if (ret) + return ret; + } + + dd->in_sg = dd->in_sgl; + dd->total = cryptlen; + dd->assoc_len = assoclen; + dd->authsize = authlen; + + dd->out_sg = req->dst; + dd->orig_out = req->dst; + + dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, req->assoclen); + + flags = 0; + if (req->src == req->dst || dd->out_sg == sg_arr) + flags |= OMAP_CRYPTO_FORCE_COPY; + + if (cryptlen) { + ret = omap_crypto_align_sg(&dd->out_sg, cryptlen, + AES_BLOCK_SIZE, &dd->out_sgl, + flags, + FLAGS_OUT_DATA_ST_SHIFT, &dd->flags); + if (ret) + return ret; + } + + dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen); + dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen); + + return 0; +} + +static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv) +{ + struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + + aes_encrypt(&ctx->akey, (u8 *)tag, (const u8 *)iv); + return 0; +} + +void omap_aes_gcm_dma_out_callback(void *data) +{ + struct omap_aes_dev *dd = data; + struct omap_aes_reqctx *rctx; + int i, val; + u32 *auth_tag, tag[4]; + + if (!(dd->flags & FLAGS_ENCRYPT)) + scatterwalk_map_and_copy(tag, dd->aead_req->src, + dd->total + dd->aead_req->assoclen, + dd->authsize, 0); + + rctx = aead_request_ctx(dd->aead_req); + auth_tag = (u32 *)rctx->auth_tag; + for (i = 0; i < 4; i++) { + val = omap_aes_read(dd, AES_REG_TAG_N(dd, i)); + auth_tag[i] = val ^ auth_tag[i]; + if (!(dd->flags & FLAGS_ENCRYPT)) + auth_tag[i] = auth_tag[i] ^ tag[i]; + } + + omap_aes_gcm_done_task(dd); +} + +static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd, + struct aead_request *req) +{ + if (req) + return crypto_transfer_aead_request_to_engine(dd->engine, req); + + return 0; +} + +static int omap_aes_gcm_prepare_req(struct aead_request *req, + struct omap_aes_dev *dd) +{ + struct omap_aes_reqctx *rctx = aead_request_ctx(req); + struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + int err; + + dd->aead_req = req; + + rctx->mode &= FLAGS_MODE_MASK; + dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode; + + err = omap_aes_gcm_copy_buffers(dd, req); + if (err) + return err; + + dd->ctx = &ctx->octx; + + return omap_aes_write_ctrl(dd); +} + +static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode) +{ + struct omap_aes_reqctx *rctx = aead_request_ctx(req); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + unsigned int authlen = crypto_aead_authsize(aead); + struct omap_aes_dev *dd; + __be32 counter = cpu_to_be32(1); + int err, assoclen; + + memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag)); + memcpy(rctx->iv + GCM_AES_IV_SIZE, &counter, 4); + + err = do_encrypt_iv(req, (u32 *)rctx->auth_tag, (u32 *)rctx->iv); + if (err) + return err; + + if (mode & FLAGS_RFC4106_GCM) + assoclen = req->assoclen - 8; + else + assoclen = req->assoclen; + if (assoclen + req->cryptlen == 0) { + scatterwalk_map_and_copy(rctx->auth_tag, req->dst, 0, authlen, + 1); + return 0; + } + + dd = omap_aes_find_dev(rctx); + if (!dd) + return -ENODEV; + rctx->mode = mode; + + return omap_aes_gcm_handle_queue(dd, req); +} + +int omap_aes_gcm_encrypt(struct aead_request *req) +{ + struct omap_aes_reqctx *rctx = aead_request_ctx(req); + + memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); + return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM); +} + +int omap_aes_gcm_decrypt(struct aead_request *req) +{ + struct omap_aes_reqctx *rctx = aead_request_ctx(req); + + memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); + return omap_aes_gcm_crypt(req, FLAGS_GCM); +} + +int omap_aes_4106gcm_encrypt(struct aead_request *req) +{ + struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct omap_aes_reqctx *rctx = aead_request_ctx(req); + + memcpy(rctx->iv, ctx->octx.nonce, 4); + memcpy(rctx->iv + 4, req->iv, 8); + return crypto_ipsec_check_assoclen(req->assoclen) ?: + omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM | + FLAGS_RFC4106_GCM); +} + +int omap_aes_4106gcm_decrypt(struct aead_request *req) +{ + struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct omap_aes_reqctx *rctx = aead_request_ctx(req); + + memcpy(rctx->iv, ctx->octx.nonce, 4); + memcpy(rctx->iv + 4, req->iv, 8); + return crypto_ipsec_check_assoclen(req->assoclen) ?: + omap_aes_gcm_crypt(req, FLAGS_GCM | FLAGS_RFC4106_GCM); +} + +int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); + int ret; + + ret = aes_prepareenckey(&ctx->akey, key, keylen); + if (ret) + return ret; + + memcpy(ctx->octx.key, key, keylen); + ctx->octx.keylen = keylen; + + return 0; +} + +int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); + int ret; + + if (keylen < 4) + return -EINVAL; + keylen -= 4; + + ret = aes_prepareenckey(&ctx->akey, key, keylen); + if (ret) + return ret; + + memcpy(ctx->octx.key, key, keylen); + memcpy(ctx->octx.nonce, key + keylen, 4); + ctx->octx.keylen = keylen; + + return 0; +} + +int omap_aes_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) +{ + return crypto_gcm_check_authsize(authsize); +} + +int omap_aes_4106gcm_setauthsize(struct crypto_aead *parent, + unsigned int authsize) +{ + return crypto_rfc4106_check_authsize(authsize); +} + +int omap_aes_gcm_crypt_req(struct crypto_engine *engine, void *areq) +{ + struct aead_request *req = container_of(areq, struct aead_request, + base); + struct omap_aes_reqctx *rctx = aead_request_ctx(req); + struct omap_aes_dev *dd = rctx->dd; + int ret; + + if (!dd) + return -ENODEV; + + ret = omap_aes_gcm_prepare_req(req, dd); + if (ret) + return ret; + + if (dd->in_sg_len) + ret = omap_aes_crypt_dma_start(dd); + else + omap_aes_gcm_dma_out_callback(dd); + + return ret; +} + +int omap_aes_gcm_cra_init(struct crypto_aead *tfm) +{ + crypto_aead_set_reqsize(tfm, sizeof(struct omap_aes_reqctx)); + + return 0; +} diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c new file mode 100644 index 000000000..d40074e5a --- /dev/null +++ b/drivers/crypto/omap-aes.c @@ -0,0 +1,1245 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Cryptographic API. + * + * Support for OMAP AES HW acceleration. + * + * Copyright (c) 2010 Nokia Corporation + * Author: Dmitry Kasatkin + * Copyright (c) 2011 Texas Instruments Incorporated + */ + +#define pr_fmt(fmt) "%20s: " fmt, __func__ +#define prn(num) pr_debug(#num "=%d\n", num) +#define prx(num) pr_debug(#num "=%x\n", num) + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "omap-crypto.h" +#include "omap-aes.h" + +/* keep registered devices data here */ +static LIST_HEAD(dev_list); +static DEFINE_SPINLOCK(list_lock); + +static int aes_fallback_sz = 200; + +#ifdef DEBUG +#define omap_aes_read(dd, offset) \ +({ \ + int _read_ret; \ + _read_ret = __raw_readl(dd->io_base + offset); \ + pr_debug("omap_aes_read(" #offset "=%#x)= %#x\n", \ + offset, _read_ret); \ + _read_ret; \ +}) +#else +inline u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset) +{ + return __raw_readl(dd->io_base + offset); +} +#endif + +#ifdef DEBUG +#define omap_aes_write(dd, offset, value) \ + do { \ + pr_debug("omap_aes_write(" #offset "=%#x) value=%#x\n", \ + offset, value); \ + __raw_writel(value, dd->io_base + offset); \ + } while (0) +#else +inline void omap_aes_write(struct omap_aes_dev *dd, u32 offset, + u32 value) +{ + __raw_writel(value, dd->io_base + offset); +} +#endif + +static inline void omap_aes_write_mask(struct omap_aes_dev *dd, u32 offset, + u32 value, u32 mask) +{ + u32 val; + + val = omap_aes_read(dd, offset); + val &= ~mask; + val |= value; + omap_aes_write(dd, offset, val); +} + +static void omap_aes_write_n(struct omap_aes_dev *dd, u32 offset, + u32 *value, int count) +{ + for (; count--; value++, offset += 4) + omap_aes_write(dd, offset, *value); +} + +static int omap_aes_hw_init(struct omap_aes_dev *dd) +{ + int err; + + if (!(dd->flags & FLAGS_INIT)) { + dd->flags |= FLAGS_INIT; + dd->err = 0; + } + + err = pm_runtime_resume_and_get(dd->dev); + if (err < 0) { + dev_err(dd->dev, "failed to get sync: %d\n", err); + return err; + } + + return 0; +} + +void omap_aes_clear_copy_flags(struct omap_aes_dev *dd) +{ + dd->flags &= ~(OMAP_CRYPTO_COPY_MASK << FLAGS_IN_DATA_ST_SHIFT); + dd->flags &= ~(OMAP_CRYPTO_COPY_MASK << FLAGS_OUT_DATA_ST_SHIFT); + dd->flags &= ~(OMAP_CRYPTO_COPY_MASK << FLAGS_ASSOC_DATA_ST_SHIFT); +} + +int omap_aes_write_ctrl(struct omap_aes_dev *dd) +{ + struct omap_aes_reqctx *rctx; + unsigned int key32; + int i, err; + u32 val; + + err = omap_aes_hw_init(dd); + if (err) + return err; + + key32 = dd->ctx->keylen / sizeof(u32); + + /* RESET the key as previous HASH keys should not get affected*/ + if (dd->flags & FLAGS_GCM) + for (i = 0; i < 0x40; i = i + 4) + omap_aes_write(dd, i, 0x0); + + for (i = 0; i < key32; i++) { + omap_aes_write(dd, AES_REG_KEY(dd, i), + (__force u32)cpu_to_le32(dd->ctx->key[i])); + } + + if ((dd->flags & (FLAGS_CBC | FLAGS_CTR)) && dd->req->iv) + omap_aes_write_n(dd, AES_REG_IV(dd, 0), (void *)dd->req->iv, 4); + + if ((dd->flags & (FLAGS_GCM)) && dd->aead_req->iv) { + rctx = aead_request_ctx(dd->aead_req); + omap_aes_write_n(dd, AES_REG_IV(dd, 0), (u32 *)rctx->iv, 4); + } + + val = FLD_VAL(((dd->ctx->keylen >> 3) - 1), 4, 3); + if (dd->flags & FLAGS_CBC) + val |= AES_REG_CTRL_CBC; + + if (dd->flags & (FLAGS_CTR | FLAGS_GCM)) + val |= AES_REG_CTRL_CTR | AES_REG_CTRL_CTR_WIDTH_128; + + if (dd->flags & FLAGS_GCM) + val |= AES_REG_CTRL_GCM; + + if (dd->flags & FLAGS_ENCRYPT) + val |= AES_REG_CTRL_DIRECTION; + + omap_aes_write_mask(dd, AES_REG_CTRL(dd), val, AES_REG_CTRL_MASK); + + return 0; +} + +static void omap_aes_dma_trigger_omap2(struct omap_aes_dev *dd, int length) +{ + u32 mask, val; + + val = dd->pdata->dma_start; + + if (dd->dma_lch_out != NULL) + val |= dd->pdata->dma_enable_out; + if (dd->dma_lch_in != NULL) + val |= dd->pdata->dma_enable_in; + + mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in | + dd->pdata->dma_start; + + omap_aes_write_mask(dd, AES_REG_MASK(dd), val, mask); + +} + +static void omap_aes_dma_trigger_omap4(struct omap_aes_dev *dd, int length) +{ + omap_aes_write(dd, AES_REG_LENGTH_N(0), length); + omap_aes_write(dd, AES_REG_LENGTH_N(1), 0); + if (dd->flags & FLAGS_GCM) + omap_aes_write(dd, AES_REG_A_LEN, dd->assoc_len); + + omap_aes_dma_trigger_omap2(dd, length); +} + +static void omap_aes_dma_stop(struct omap_aes_dev *dd) +{ + u32 mask; + + mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in | + dd->pdata->dma_start; + + omap_aes_write_mask(dd, AES_REG_MASK(dd), 0, mask); +} + +struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_reqctx *rctx) +{ + struct omap_aes_dev *dd; + + spin_lock_bh(&list_lock); + dd = list_first_entry(&dev_list, struct omap_aes_dev, list); + list_move_tail(&dd->list, &dev_list); + rctx->dd = dd; + spin_unlock_bh(&list_lock); + + return dd; +} + +static void omap_aes_dma_out_callback(void *data) +{ + struct omap_aes_dev *dd = data; + + /* dma_lch_out - completed */ + queue_work(system_bh_wq, &dd->done_task); +} + +static int omap_aes_dma_init(struct omap_aes_dev *dd) +{ + int err; + + dd->dma_lch_out = NULL; + dd->dma_lch_in = NULL; + + dd->dma_lch_in = dma_request_chan(dd->dev, "rx"); + if (IS_ERR(dd->dma_lch_in)) { + dev_err(dd->dev, "Unable to request in DMA channel\n"); + return PTR_ERR(dd->dma_lch_in); + } + + dd->dma_lch_out = dma_request_chan(dd->dev, "tx"); + if (IS_ERR(dd->dma_lch_out)) { + dev_err(dd->dev, "Unable to request out DMA channel\n"); + err = PTR_ERR(dd->dma_lch_out); + goto err_dma_out; + } + + return 0; + +err_dma_out: + dma_release_channel(dd->dma_lch_in); + + return err; +} + +static void omap_aes_dma_cleanup(struct omap_aes_dev *dd) +{ + if (dd->pio_only) + return; + + dma_release_channel(dd->dma_lch_out); + dma_release_channel(dd->dma_lch_in); +} + +static int omap_aes_crypt_dma(struct omap_aes_dev *dd, + struct scatterlist *in_sg, + struct scatterlist *out_sg, + int in_sg_len, int out_sg_len) +{ + struct dma_async_tx_descriptor *tx_in, *tx_out = NULL, *cb_desc; + struct dma_slave_config cfg; + int ret; + + if (dd->pio_only) { + dd->in_sg_offset = 0; + if (out_sg_len) + dd->out_sg_offset = 0; + + /* Enable DATAIN interrupt and let it take + care of the rest */ + omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x2); + return 0; + } + + dma_sync_sg_for_device(dd->dev, dd->in_sg, in_sg_len, DMA_TO_DEVICE); + + memset(&cfg, 0, sizeof(cfg)); + + cfg.src_addr = dd->phys_base + AES_REG_DATA_N(dd, 0); + cfg.dst_addr = dd->phys_base + AES_REG_DATA_N(dd, 0); + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cfg.src_maxburst = DST_MAXBURST; + cfg.dst_maxburst = DST_MAXBURST; + + /* IN */ + ret = dmaengine_slave_config(dd->dma_lch_in, &cfg); + if (ret) { + dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n", + ret); + return ret; + } + + tx_in = dmaengine_prep_slave_sg(dd->dma_lch_in, in_sg, in_sg_len, + DMA_MEM_TO_DEV, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!tx_in) { + dev_err(dd->dev, "IN prep_slave_sg() failed\n"); + return -EINVAL; + } + + /* No callback necessary */ + tx_in->callback_param = dd; + tx_in->callback = NULL; + + /* OUT */ + if (out_sg_len) { + ret = dmaengine_slave_config(dd->dma_lch_out, &cfg); + if (ret) { + dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n", + ret); + return ret; + } + + tx_out = dmaengine_prep_slave_sg(dd->dma_lch_out, out_sg, + out_sg_len, + DMA_DEV_TO_MEM, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!tx_out) { + dev_err(dd->dev, "OUT prep_slave_sg() failed\n"); + return -EINVAL; + } + + cb_desc = tx_out; + } else { + cb_desc = tx_in; + } + + if (dd->flags & FLAGS_GCM) + cb_desc->callback = omap_aes_gcm_dma_out_callback; + else + cb_desc->callback = omap_aes_dma_out_callback; + cb_desc->callback_param = dd; + + + dmaengine_submit(tx_in); + if (tx_out) + dmaengine_submit(tx_out); + + dma_async_issue_pending(dd->dma_lch_in); + if (out_sg_len) + dma_async_issue_pending(dd->dma_lch_out); + + /* start DMA */ + dd->pdata->trigger(dd, dd->total); + + return 0; +} + +int omap_aes_crypt_dma_start(struct omap_aes_dev *dd) +{ + int err; + + pr_debug("total: %zu\n", dd->total); + + if (!dd->pio_only) { + err = dma_map_sg(dd->dev, dd->in_sg, dd->in_sg_len, + DMA_TO_DEVICE); + if (!err) { + dev_err(dd->dev, "dma_map_sg() error\n"); + return -EINVAL; + } + + if (dd->out_sg_len) { + err = dma_map_sg(dd->dev, dd->out_sg, dd->out_sg_len, + DMA_FROM_DEVICE); + if (!err) { + dev_err(dd->dev, "dma_map_sg() error\n"); + return -EINVAL; + } + } + } + + err = omap_aes_crypt_dma(dd, dd->in_sg, dd->out_sg, dd->in_sg_len, + dd->out_sg_len); + if (err && !dd->pio_only) { + dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE); + if (dd->out_sg_len) + dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, + DMA_FROM_DEVICE); + } + + return err; +} + +static void omap_aes_finish_req(struct omap_aes_dev *dd, int err) +{ + struct skcipher_request *req = dd->req; + + pr_debug("err: %d\n", err); + + crypto_finalize_skcipher_request(dd->engine, req, err); + + pm_runtime_put_autosuspend(dd->dev); +} + +int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd) +{ + pr_debug("total: %zu\n", dd->total); + + omap_aes_dma_stop(dd); + + + return 0; +} + +static int omap_aes_handle_queue(struct omap_aes_dev *dd, + struct skcipher_request *req) +{ + if (req) + return crypto_transfer_skcipher_request_to_engine(dd->engine, req); + + return 0; +} + +static int omap_aes_prepare_req(struct skcipher_request *req, + struct omap_aes_dev *dd) +{ + struct omap_aes_ctx *ctx = crypto_skcipher_ctx( + crypto_skcipher_reqtfm(req)); + struct omap_aes_reqctx *rctx = skcipher_request_ctx(req); + int ret; + u16 flags; + + /* assign new request to device */ + dd->req = req; + dd->total = req->cryptlen; + dd->total_save = req->cryptlen; + dd->in_sg = req->src; + dd->out_sg = req->dst; + dd->orig_out = req->dst; + + flags = OMAP_CRYPTO_COPY_DATA; + if (req->src == req->dst) + flags |= OMAP_CRYPTO_FORCE_COPY; + + ret = omap_crypto_align_sg(&dd->in_sg, dd->total, AES_BLOCK_SIZE, + dd->in_sgl, flags, + FLAGS_IN_DATA_ST_SHIFT, &dd->flags); + if (ret) + return ret; + + ret = omap_crypto_align_sg(&dd->out_sg, dd->total, AES_BLOCK_SIZE, + &dd->out_sgl, 0, + FLAGS_OUT_DATA_ST_SHIFT, &dd->flags); + if (ret) + return ret; + + dd->in_sg_len = sg_nents_for_len(dd->in_sg, dd->total); + if (dd->in_sg_len < 0) + return dd->in_sg_len; + + dd->out_sg_len = sg_nents_for_len(dd->out_sg, dd->total); + if (dd->out_sg_len < 0) + return dd->out_sg_len; + + rctx->mode &= FLAGS_MODE_MASK; + dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode; + + dd->ctx = ctx; + rctx->dd = dd; + + return omap_aes_write_ctrl(dd); +} + +static int omap_aes_crypt_req(struct crypto_engine *engine, + void *areq) +{ + struct skcipher_request *req = container_of(areq, struct skcipher_request, base); + struct omap_aes_reqctx *rctx = skcipher_request_ctx(req); + struct omap_aes_dev *dd = rctx->dd; + + if (!dd) + return -ENODEV; + + return omap_aes_prepare_req(req, dd) ?: + omap_aes_crypt_dma_start(dd); +} + +static void omap_aes_copy_ivout(struct omap_aes_dev *dd, u8 *ivbuf) +{ + int i; + + for (i = 0; i < 4; i++) + ((u32 *)ivbuf)[i] = omap_aes_read(dd, AES_REG_IV(dd, i)); +} + +static void omap_aes_done_task(struct work_struct *t) +{ + struct omap_aes_dev *dd = from_work(dd, t, done_task); + + pr_debug("enter done_task\n"); + + if (!dd->pio_only) { + dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len, + DMA_FROM_DEVICE); + dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE); + dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, + DMA_FROM_DEVICE); + omap_aes_crypt_dma_stop(dd); + } + + omap_crypto_cleanup(dd->in_sg, NULL, 0, dd->total_save, + FLAGS_IN_DATA_ST_SHIFT, dd->flags); + + omap_crypto_cleanup(dd->out_sg, dd->orig_out, 0, dd->total_save, + FLAGS_OUT_DATA_ST_SHIFT, dd->flags); + + /* Update IV output */ + if (dd->flags & (FLAGS_CBC | FLAGS_CTR)) + omap_aes_copy_ivout(dd, dd->req->iv); + + omap_aes_finish_req(dd, 0); + + pr_debug("exit\n"); +} + +static int omap_aes_crypt(struct skcipher_request *req, unsigned long mode) +{ + struct omap_aes_ctx *ctx = crypto_skcipher_ctx( + crypto_skcipher_reqtfm(req)); + struct omap_aes_reqctx *rctx = skcipher_request_ctx(req); + struct omap_aes_dev *dd; + int ret; + + if ((req->cryptlen % AES_BLOCK_SIZE) && !(mode & FLAGS_CTR)) + return -EINVAL; + + pr_debug("nbytes: %d, enc: %d, cbc: %d\n", req->cryptlen, + !!(mode & FLAGS_ENCRYPT), + !!(mode & FLAGS_CBC)); + + if (req->cryptlen < aes_fallback_sz) { + skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback); + skcipher_request_set_callback(&rctx->fallback_req, + req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(&rctx->fallback_req, req->src, + req->dst, req->cryptlen, req->iv); + + if (mode & FLAGS_ENCRYPT) + ret = crypto_skcipher_encrypt(&rctx->fallback_req); + else + ret = crypto_skcipher_decrypt(&rctx->fallback_req); + return ret; + } + dd = omap_aes_find_dev(rctx); + if (!dd) + return -ENODEV; + + rctx->mode = mode; + + return omap_aes_handle_queue(dd, req); +} + +/* ********************** ALG API ************************************ */ + +static int omap_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct omap_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + int ret; + + if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) + return -EINVAL; + + pr_debug("enter, keylen: %d\n", keylen); + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + crypto_skcipher_clear_flags(ctx->fallback, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(ctx->fallback, tfm->base.crt_flags & + CRYPTO_TFM_REQ_MASK); + + ret = crypto_skcipher_setkey(ctx->fallback, key, keylen); + if (!ret) + return 0; + + return 0; +} + +static int omap_aes_ecb_encrypt(struct skcipher_request *req) +{ + return omap_aes_crypt(req, FLAGS_ENCRYPT); +} + +static int omap_aes_ecb_decrypt(struct skcipher_request *req) +{ + return omap_aes_crypt(req, 0); +} + +static int omap_aes_cbc_encrypt(struct skcipher_request *req) +{ + return omap_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC); +} + +static int omap_aes_cbc_decrypt(struct skcipher_request *req) +{ + return omap_aes_crypt(req, FLAGS_CBC); +} + +static int omap_aes_ctr_encrypt(struct skcipher_request *req) +{ + return omap_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CTR); +} + +static int omap_aes_ctr_decrypt(struct skcipher_request *req) +{ + return omap_aes_crypt(req, FLAGS_CTR); +} + +static int omap_aes_init_tfm(struct crypto_skcipher *tfm) +{ + const char *name = crypto_tfm_alg_name(&tfm->base); + struct omap_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + struct crypto_skcipher *blk; + + blk = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(blk)) + return PTR_ERR(blk); + + ctx->fallback = blk; + + crypto_skcipher_set_reqsize(tfm, sizeof(struct omap_aes_reqctx) + + crypto_skcipher_reqsize(blk)); + + return 0; +} + +static void omap_aes_exit_tfm(struct crypto_skcipher *tfm) +{ + struct omap_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (ctx->fallback) + crypto_free_skcipher(ctx->fallback); + + ctx->fallback = NULL; +} + +/* ********************** ALGS ************************************ */ + +static struct skcipher_engine_alg algs_ecb_cbc[] = { +{ + .base = { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-omap", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct omap_aes_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = omap_aes_setkey, + .encrypt = omap_aes_ecb_encrypt, + .decrypt = omap_aes_ecb_decrypt, + .init = omap_aes_init_tfm, + .exit = omap_aes_exit_tfm, + }, + .op.do_one_request = omap_aes_crypt_req, +}, +{ + .base = { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-omap", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct omap_aes_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = omap_aes_setkey, + .encrypt = omap_aes_cbc_encrypt, + .decrypt = omap_aes_cbc_decrypt, + .init = omap_aes_init_tfm, + .exit = omap_aes_exit_tfm, + }, + .op.do_one_request = omap_aes_crypt_req, +} +}; + +static struct skcipher_engine_alg algs_ctr[] = { +{ + .base = { + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "ctr-aes-omap", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct omap_aes_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = omap_aes_setkey, + .encrypt = omap_aes_ctr_encrypt, + .decrypt = omap_aes_ctr_decrypt, + .init = omap_aes_init_tfm, + .exit = omap_aes_exit_tfm, + }, + .op.do_one_request = omap_aes_crypt_req, +} +}; + +static struct omap_aes_algs_info omap_aes_algs_info_ecb_cbc[] = { + { + .algs_list = algs_ecb_cbc, + .size = ARRAY_SIZE(algs_ecb_cbc), + }, +}; + +static struct aead_engine_alg algs_aead_gcm[] = { +{ + .base = { + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "gcm-aes-omap", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct omap_aes_gcm_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + .init = omap_aes_gcm_cra_init, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .setkey = omap_aes_gcm_setkey, + .setauthsize = omap_aes_gcm_setauthsize, + .encrypt = omap_aes_gcm_encrypt, + .decrypt = omap_aes_gcm_decrypt, + }, + .op.do_one_request = omap_aes_gcm_crypt_req, +}, +{ + .base = { + .base = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "rfc4106-gcm-aes-omap", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct omap_aes_gcm_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + .init = omap_aes_gcm_cra_init, + .maxauthsize = AES_BLOCK_SIZE, + .ivsize = GCM_RFC4106_IV_SIZE, + .setkey = omap_aes_4106gcm_setkey, + .setauthsize = omap_aes_4106gcm_setauthsize, + .encrypt = omap_aes_4106gcm_encrypt, + .decrypt = omap_aes_4106gcm_decrypt, + }, + .op.do_one_request = omap_aes_gcm_crypt_req, +}, +}; + +static struct omap_aes_aead_algs omap_aes_aead_info = { + .algs_list = algs_aead_gcm, + .size = ARRAY_SIZE(algs_aead_gcm), +}; + +static const struct omap_aes_pdata omap_aes_pdata_omap2 = { + .algs_info = omap_aes_algs_info_ecb_cbc, + .algs_info_size = ARRAY_SIZE(omap_aes_algs_info_ecb_cbc), + .trigger = omap_aes_dma_trigger_omap2, + .key_ofs = 0x1c, + .iv_ofs = 0x20, + .ctrl_ofs = 0x30, + .data_ofs = 0x34, + .rev_ofs = 0x44, + .mask_ofs = 0x48, + .dma_enable_in = BIT(2), + .dma_enable_out = BIT(3), + .dma_start = BIT(5), + .major_mask = 0xf0, + .major_shift = 4, + .minor_mask = 0x0f, + .minor_shift = 0, +}; + +#ifdef CONFIG_OF +static struct omap_aes_algs_info omap_aes_algs_info_ecb_cbc_ctr[] = { + { + .algs_list = algs_ecb_cbc, + .size = ARRAY_SIZE(algs_ecb_cbc), + }, + { + .algs_list = algs_ctr, + .size = ARRAY_SIZE(algs_ctr), + }, +}; + +static const struct omap_aes_pdata omap_aes_pdata_omap3 = { + .algs_info = omap_aes_algs_info_ecb_cbc_ctr, + .algs_info_size = ARRAY_SIZE(omap_aes_algs_info_ecb_cbc_ctr), + .trigger = omap_aes_dma_trigger_omap2, + .key_ofs = 0x1c, + .iv_ofs = 0x20, + .ctrl_ofs = 0x30, + .data_ofs = 0x34, + .rev_ofs = 0x44, + .mask_ofs = 0x48, + .dma_enable_in = BIT(2), + .dma_enable_out = BIT(3), + .dma_start = BIT(5), + .major_mask = 0xf0, + .major_shift = 4, + .minor_mask = 0x0f, + .minor_shift = 0, +}; + +static const struct omap_aes_pdata omap_aes_pdata_omap4 = { + .algs_info = omap_aes_algs_info_ecb_cbc_ctr, + .algs_info_size = ARRAY_SIZE(omap_aes_algs_info_ecb_cbc_ctr), + .aead_algs_info = &omap_aes_aead_info, + .trigger = omap_aes_dma_trigger_omap4, + .key_ofs = 0x3c, + .iv_ofs = 0x40, + .ctrl_ofs = 0x50, + .data_ofs = 0x60, + .rev_ofs = 0x80, + .mask_ofs = 0x84, + .irq_status_ofs = 0x8c, + .irq_enable_ofs = 0x90, + .dma_enable_in = BIT(5), + .dma_enable_out = BIT(6), + .major_mask = 0x0700, + .major_shift = 8, + .minor_mask = 0x003f, + .minor_shift = 0, +}; + +static irqreturn_t omap_aes_irq(int irq, void *dev_id) +{ + struct omap_aes_dev *dd = dev_id; + u32 status, i; + u32 *src, *dst; + + status = omap_aes_read(dd, AES_REG_IRQ_STATUS(dd)); + if (status & AES_REG_IRQ_DATA_IN) { + omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x0); + + BUG_ON(!dd->in_sg); + + BUG_ON(dd->in_sg_offset > dd->in_sg->length); + + src = sg_virt(dd->in_sg) + dd->in_sg_offset; + + for (i = 0; i < AES_BLOCK_WORDS; i++) { + omap_aes_write(dd, AES_REG_DATA_N(dd, i), *src); + dd->in_sg_offset += 4; + if (dd->in_sg_offset == dd->in_sg->length) { + dd->in_sg = sg_next(dd->in_sg); + if (dd->in_sg) { + dd->in_sg_offset = 0; + src = sg_virt(dd->in_sg); + } + } else { + src++; + } + } + + /* Clear IRQ status */ + status &= ~AES_REG_IRQ_DATA_IN; + omap_aes_write(dd, AES_REG_IRQ_STATUS(dd), status); + + /* Enable DATA_OUT interrupt */ + omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x4); + + } else if (status & AES_REG_IRQ_DATA_OUT) { + omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x0); + + BUG_ON(!dd->out_sg); + + BUG_ON(dd->out_sg_offset > dd->out_sg->length); + + dst = sg_virt(dd->out_sg) + dd->out_sg_offset; + + for (i = 0; i < AES_BLOCK_WORDS; i++) { + *dst = omap_aes_read(dd, AES_REG_DATA_N(dd, i)); + dd->out_sg_offset += 4; + if (dd->out_sg_offset == dd->out_sg->length) { + dd->out_sg = sg_next(dd->out_sg); + if (dd->out_sg) { + dd->out_sg_offset = 0; + dst = sg_virt(dd->out_sg); + } + } else { + dst++; + } + } + + dd->total -= min_t(size_t, AES_BLOCK_SIZE, dd->total); + + /* Clear IRQ status */ + status &= ~AES_REG_IRQ_DATA_OUT; + omap_aes_write(dd, AES_REG_IRQ_STATUS(dd), status); + + if (!dd->total) + /* All bytes read! */ + queue_work(system_bh_wq, &dd->done_task); + else + /* Enable DATA_IN interrupt for next block */ + omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x2); + } + + return IRQ_HANDLED; +} + +static const struct of_device_id omap_aes_of_match[] = { + { + .compatible = "ti,omap2-aes", + .data = &omap_aes_pdata_omap2, + }, + { + .compatible = "ti,omap3-aes", + .data = &omap_aes_pdata_omap3, + }, + { + .compatible = "ti,omap4-aes", + .data = &omap_aes_pdata_omap4, + }, + {}, +}; +MODULE_DEVICE_TABLE(of, omap_aes_of_match); +#endif + +static ssize_t fallback_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%d\n", aes_fallback_sz); +} + +static ssize_t fallback_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + ssize_t status; + long value; + + status = kstrtol(buf, 0, &value); + if (status) + return status; + + /* HW accelerator only works with buffers > 9 */ + if (value < 9) { + dev_err(dev, "minimum fallback size 9\n"); + return -EINVAL; + } + + aes_fallback_sz = value; + + return size; +} + +static ssize_t queue_len_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct omap_aes_dev *dd = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%d\n", dd->engine->queue.max_qlen); +} + +static ssize_t queue_len_store(struct device *dev, + struct device_attribute *attr, const char *buf, + size_t size) +{ + struct omap_aes_dev *dd; + ssize_t status; + long value; + unsigned long flags; + + status = kstrtol(buf, 0, &value); + if (status) + return status; + + if (value < 1) + return -EINVAL; + + /* + * Changing the queue size in fly is safe, if size becomes smaller + * than current size, it will just not accept new entries until + * it has shrank enough. + */ + spin_lock_bh(&list_lock); + list_for_each_entry(dd, &dev_list, list) { + spin_lock_irqsave(&dd->lock, flags); + dd->engine->queue.max_qlen = value; + dd->aead_queue.base.max_qlen = value; + spin_unlock_irqrestore(&dd->lock, flags); + } + spin_unlock_bh(&list_lock); + + return size; +} + +static DEVICE_ATTR_RW(queue_len); +static DEVICE_ATTR_RW(fallback); + +static struct attribute *omap_aes_attrs[] = { + &dev_attr_queue_len.attr, + &dev_attr_fallback.attr, + NULL, +}; +ATTRIBUTE_GROUPS(omap_aes); + +static void omap_aes_unregister_algs(const struct omap_aes_pdata *pdata) +{ + struct omap_aes_algs_info *alg_info; + int i; + + for (i = pdata->algs_info_size - 1; i >= 0; i--) { + alg_info = &pdata->algs_info[i]; + + crypto_engine_unregister_skciphers(alg_info->algs_list, + alg_info->registered); + alg_info->registered = 0; + } +} + +static int omap_aes_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct omap_aes_dev *dd; + struct skcipher_engine_alg *algp; + struct aead_engine_alg *aalg; + struct resource *res; + void __iomem *io_base; + int err = -ENOMEM, i, j, irq = -1; + u32 reg; + + io_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); + if (IS_ERR(io_base)) + return PTR_ERR(io_base); + + dd = devm_kzalloc(dev, sizeof(struct omap_aes_dev), GFP_KERNEL); + if (dd == NULL) { + dev_err(dev, "unable to alloc data struct.\n"); + goto err_data; + } + dd->dev = dev; + platform_set_drvdata(pdev, dd); + + aead_init_queue(&dd->aead_queue, OMAP_AES_QUEUE_LENGTH); + + dd->pdata = device_get_match_data(dev); + if (!dd->pdata) + dd->pdata = &omap_aes_pdata_omap2; + + dd->io_base = io_base; + dd->phys_base = res->start; + + pm_runtime_use_autosuspend(dev); + pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY); + + pm_runtime_enable(dev); + err = pm_runtime_resume_and_get(dev); + if (err < 0) { + dev_err(dev, "%s: failed to get_sync(%d)\n", + __func__, err); + goto err_pm_disable; + } + + omap_aes_dma_stop(dd); + + reg = omap_aes_read(dd, AES_REG_REV(dd)); + + pm_runtime_put_sync(dev); + + dev_info(dev, "OMAP AES hw accel rev: %u.%u\n", + (reg & dd->pdata->major_mask) >> dd->pdata->major_shift, + (reg & dd->pdata->minor_mask) >> dd->pdata->minor_shift); + + INIT_WORK(&dd->done_task, omap_aes_done_task); + + err = omap_aes_dma_init(dd); + if (err == -EPROBE_DEFER) { + goto err_irq; + } else if (err && AES_REG_IRQ_STATUS(dd) && AES_REG_IRQ_ENABLE(dd)) { + dd->pio_only = 1; + + irq = platform_get_irq(pdev, 0); + if (irq < 0) { + err = irq; + goto err_irq; + } + + err = devm_request_irq(dev, irq, omap_aes_irq, 0, + dev_name(dev), dd); + if (err) + goto err_irq; + } + + spin_lock_init(&dd->lock); + + INIT_LIST_HEAD(&dd->list); + spin_lock_bh(&list_lock); + list_add_tail(&dd->list, &dev_list); + spin_unlock_bh(&list_lock); + + /* Initialize crypto engine */ + dd->engine = crypto_engine_alloc_init(dev, 1); + if (!dd->engine) { + err = -ENOMEM; + goto err_engine; + } + + err = crypto_engine_start(dd->engine); + if (err) + goto err_engine; + + for (i = 0; i < dd->pdata->algs_info_size; i++) { + if (!dd->pdata->algs_info[i].registered) { + for (j = 0; j < dd->pdata->algs_info[i].size; j++) { + algp = &dd->pdata->algs_info[i].algs_list[j]; + + pr_debug("reg alg: %s\n", algp->base.base.cra_name); + + err = crypto_engine_register_skcipher(algp); + if (err) + goto err_algs; + + dd->pdata->algs_info[i].registered++; + } + } + } + + if (dd->pdata->aead_algs_info && + !dd->pdata->aead_algs_info->registered) { + for (i = 0; i < dd->pdata->aead_algs_info->size; i++) { + aalg = &dd->pdata->aead_algs_info->algs_list[i]; + + pr_debug("reg alg: %s\n", aalg->base.base.cra_name); + + err = crypto_engine_register_aead(aalg); + if (err) + goto err_aead_algs; + + dd->pdata->aead_algs_info->registered++; + } + } + + return 0; +err_aead_algs: + crypto_engine_unregister_aeads(dd->pdata->aead_algs_info->algs_list, + dd->pdata->aead_algs_info->registered); + dd->pdata->aead_algs_info->registered = 0; +err_algs: + omap_aes_unregister_algs(dd->pdata); + +err_engine: + if (dd->engine) + crypto_engine_exit(dd->engine); + + omap_aes_dma_cleanup(dd); +err_irq: + cancel_work_sync(&dd->done_task); +err_pm_disable: + pm_runtime_disable(dev); +err_data: + dev_err(dev, "initialization failed.\n"); + return err; +} + +static void omap_aes_remove(struct platform_device *pdev) +{ + struct omap_aes_dev *dd = platform_get_drvdata(pdev); + + spin_lock_bh(&list_lock); + list_del(&dd->list); + spin_unlock_bh(&list_lock); + + omap_aes_unregister_algs(dd->pdata); + + crypto_engine_unregister_aeads(dd->pdata->aead_algs_info->algs_list, + dd->pdata->aead_algs_info->registered); + dd->pdata->aead_algs_info->registered = 0; + + crypto_engine_exit(dd->engine); + + cancel_work_sync(&dd->done_task); + omap_aes_dma_cleanup(dd); + pm_runtime_disable(dd->dev); +} + +#ifdef CONFIG_PM_SLEEP +static int omap_aes_suspend(struct device *dev) +{ + pm_runtime_put_sync(dev); + return 0; +} + +static int omap_aes_resume(struct device *dev) +{ + pm_runtime_get_sync(dev); + return 0; +} +#endif + +static SIMPLE_DEV_PM_OPS(omap_aes_pm_ops, omap_aes_suspend, omap_aes_resume); + +static struct platform_driver omap_aes_driver = { + .probe = omap_aes_probe, + .remove = omap_aes_remove, + .driver = { + .name = "omap-aes", + .pm = &omap_aes_pm_ops, + .of_match_table = of_match_ptr(omap_aes_of_match), + .dev_groups = omap_aes_groups, + }, +}; + +module_platform_driver(omap_aes_driver); + +MODULE_DESCRIPTION("OMAP AES hw acceleration support."); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Dmitry Kasatkin"); + diff --git a/drivers/crypto/omap-aes.h b/drivers/crypto/omap-aes.h new file mode 100644 index 000000000..6aa70bde3 --- /dev/null +++ b/drivers/crypto/omap-aes.h @@ -0,0 +1,219 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Cryptographic API. + * + * Support for OMAP AES HW ACCELERATOR defines + * + * Copyright (c) 2015 Texas Instruments Incorporated + */ +#ifndef __OMAP_AES_H__ +#define __OMAP_AES_H__ + +#include + +#define DST_MAXBURST 4 +#define DMA_MIN (DST_MAXBURST * sizeof(u32)) + +/* + * OMAP TRM gives bitfields as start:end, where start is the higher bit + * number. For example 7:0 + */ +#define FLD_MASK(start, end) (((1 << ((start) - (end) + 1)) - 1) << (end)) +#define FLD_VAL(val, start, end) (((val) << (end)) & FLD_MASK(start, end)) + +#define AES_REG_KEY(dd, x) ((dd)->pdata->key_ofs - \ + (((x) ^ 0x01) * 0x04)) +#define AES_REG_IV(dd, x) ((dd)->pdata->iv_ofs + ((x) * 0x04)) + +#define AES_REG_CTRL(dd) ((dd)->pdata->ctrl_ofs) +#define AES_REG_CTRL_CONTEXT_READY BIT(31) +#define AES_REG_CTRL_CTR_WIDTH_MASK GENMASK(8, 7) +#define AES_REG_CTRL_CTR_WIDTH_32 0 +#define AES_REG_CTRL_CTR_WIDTH_64 BIT(7) +#define AES_REG_CTRL_CTR_WIDTH_96 BIT(8) +#define AES_REG_CTRL_CTR_WIDTH_128 GENMASK(8, 7) +#define AES_REG_CTRL_GCM GENMASK(17, 16) +#define AES_REG_CTRL_CTR BIT(6) +#define AES_REG_CTRL_CBC BIT(5) +#define AES_REG_CTRL_KEY_SIZE GENMASK(4, 3) +#define AES_REG_CTRL_DIRECTION BIT(2) +#define AES_REG_CTRL_INPUT_READY BIT(1) +#define AES_REG_CTRL_OUTPUT_READY BIT(0) +#define AES_REG_CTRL_MASK GENMASK(24, 2) + +#define AES_REG_C_LEN_0 0x54 +#define AES_REG_C_LEN_1 0x58 +#define AES_REG_A_LEN 0x5C + +#define AES_REG_DATA_N(dd, x) ((dd)->pdata->data_ofs + ((x) * 0x04)) +#define AES_REG_TAG_N(dd, x) (0x70 + ((x) * 0x04)) + +#define AES_REG_REV(dd) ((dd)->pdata->rev_ofs) + +#define AES_REG_MASK(dd) ((dd)->pdata->mask_ofs) +#define AES_REG_MASK_SIDLE BIT(6) +#define AES_REG_MASK_START BIT(5) +#define AES_REG_MASK_DMA_OUT_EN BIT(3) +#define AES_REG_MASK_DMA_IN_EN BIT(2) +#define AES_REG_MASK_SOFTRESET BIT(1) +#define AES_REG_AUTOIDLE BIT(0) + +#define AES_REG_LENGTH_N(x) (0x54 + ((x) * 0x04)) + +#define AES_REG_IRQ_STATUS(dd) ((dd)->pdata->irq_status_ofs) +#define AES_REG_IRQ_ENABLE(dd) ((dd)->pdata->irq_enable_ofs) +#define AES_REG_IRQ_DATA_IN BIT(1) +#define AES_REG_IRQ_DATA_OUT BIT(2) +#define DEFAULT_TIMEOUT (5 * HZ) + +#define DEFAULT_AUTOSUSPEND_DELAY 1000 + +#define FLAGS_MODE_MASK 0x001f +#define FLAGS_ENCRYPT BIT(0) +#define FLAGS_CBC BIT(1) +#define FLAGS_CTR BIT(2) +#define FLAGS_GCM BIT(3) +#define FLAGS_RFC4106_GCM BIT(4) + +#define FLAGS_INIT BIT(5) +#define FLAGS_FAST BIT(6) + +#define FLAGS_IN_DATA_ST_SHIFT 8 +#define FLAGS_OUT_DATA_ST_SHIFT 10 +#define FLAGS_ASSOC_DATA_ST_SHIFT 12 + +#define AES_BLOCK_WORDS (AES_BLOCK_SIZE >> 2) + +struct omap_aes_gcm_result { + struct completion completion; + int err; +}; + +struct omap_aes_ctx { + int keylen; + u32 key[AES_KEYSIZE_256 / sizeof(u32)]; + u8 nonce[4]; + struct crypto_skcipher *fallback; +}; + +struct omap_aes_gcm_ctx { + struct omap_aes_ctx octx; + struct aes_enckey akey; +}; + +struct omap_aes_reqctx { + struct omap_aes_dev *dd; + unsigned long mode; + u8 iv[AES_BLOCK_SIZE]; + u32 auth_tag[AES_BLOCK_SIZE / sizeof(u32)]; + struct skcipher_request fallback_req; // keep at the end +}; + +#define OMAP_AES_QUEUE_LENGTH 1 +#define OMAP_AES_CACHE_SIZE 0 + +struct omap_aes_algs_info { + struct skcipher_engine_alg *algs_list; + unsigned int size; + unsigned int registered; +}; + +struct omap_aes_aead_algs { + struct aead_engine_alg *algs_list; + unsigned int size; + unsigned int registered; +}; + +struct omap_aes_pdata { + struct omap_aes_algs_info *algs_info; + unsigned int algs_info_size; + struct omap_aes_aead_algs *aead_algs_info; + + void (*trigger)(struct omap_aes_dev *dd, int length); + + u32 key_ofs; + u32 iv_ofs; + u32 ctrl_ofs; + u32 data_ofs; + u32 rev_ofs; + u32 mask_ofs; + u32 irq_enable_ofs; + u32 irq_status_ofs; + + u32 dma_enable_in; + u32 dma_enable_out; + u32 dma_start; + + u32 major_mask; + u32 major_shift; + u32 minor_mask; + u32 minor_shift; +}; + +struct omap_aes_dev { + struct list_head list; + unsigned long phys_base; + void __iomem *io_base; + struct omap_aes_ctx *ctx; + struct device *dev; + unsigned long flags; + int err; + + struct work_struct done_task; + struct aead_queue aead_queue; + spinlock_t lock; + + struct skcipher_request *req; + struct aead_request *aead_req; + struct crypto_engine *engine; + + /* + * total is used by PIO mode for book keeping so introduce + * variable total_save as need it to calc page_order + */ + size_t total; + size_t total_save; + size_t assoc_len; + size_t authsize; + + struct scatterlist *in_sg; + struct scatterlist *out_sg; + + /* Buffers for copying for unaligned cases */ + struct scatterlist in_sgl[2]; + struct scatterlist out_sgl; + struct scatterlist *orig_out; + + unsigned int in_sg_offset; + unsigned int out_sg_offset; + struct dma_chan *dma_lch_in; + struct dma_chan *dma_lch_out; + int in_sg_len; + int out_sg_len; + int pio_only; + const struct omap_aes_pdata *pdata; +}; + +u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset); +void omap_aes_write(struct omap_aes_dev *dd, u32 offset, u32 value); +struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_reqctx *rctx); +int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen); +int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen); +int omap_aes_gcm_encrypt(struct aead_request *req); +int omap_aes_gcm_decrypt(struct aead_request *req); +int omap_aes_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize); +int omap_aes_4106gcm_encrypt(struct aead_request *req); +int omap_aes_4106gcm_decrypt(struct aead_request *req); +int omap_aes_4106gcm_setauthsize(struct crypto_aead *parent, + unsigned int authsize); +int omap_aes_gcm_cra_init(struct crypto_aead *tfm); +int omap_aes_write_ctrl(struct omap_aes_dev *dd); +int omap_aes_crypt_dma_start(struct omap_aes_dev *dd); +int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd); +void omap_aes_gcm_dma_out_callback(void *data); +void omap_aes_clear_copy_flags(struct omap_aes_dev *dd); +int omap_aes_gcm_crypt_req(struct crypto_engine *engine, void *areq); + +#endif diff --git a/drivers/crypto/omap-crypto.c b/drivers/crypto/omap-crypto.c new file mode 100644 index 000000000..7bd6869c2 --- /dev/null +++ b/drivers/crypto/omap-crypto.c @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * OMAP Crypto driver common support routines. + * + * Copyright (c) 2017 Texas Instruments Incorporated + * Tero Kristo + */ + +#include +#include +#include +#include + +#include "omap-crypto.h" + +static int omap_crypto_copy_sg_lists(int total, int bs, + struct scatterlist **sg, + struct scatterlist *new_sg, u16 flags) +{ + int n = sg_nents(*sg); + struct scatterlist *tmp; + + if (!(flags & OMAP_CRYPTO_FORCE_SINGLE_ENTRY)) { + new_sg = kmalloc_objs(*new_sg, n); + if (!new_sg) + return -ENOMEM; + + sg_init_table(new_sg, n); + } + + tmp = new_sg; + + while (*sg && total) { + int len = (*sg)->length; + + if (total < len) + len = total; + + if (len > 0) { + total -= len; + sg_set_page(tmp, sg_page(*sg), len, (*sg)->offset); + if (total <= 0) + sg_mark_end(tmp); + tmp = sg_next(tmp); + } + + *sg = sg_next(*sg); + } + + *sg = new_sg; + + return 0; +} + +static int omap_crypto_copy_sgs(int total, int bs, struct scatterlist **sg, + struct scatterlist *new_sg, u16 flags) +{ + void *buf; + int pages; + int new_len; + + new_len = ALIGN(total, bs); + pages = get_order(new_len); + + buf = (void *)__get_free_pages(GFP_ATOMIC, pages); + if (!buf) { + pr_err("%s: Couldn't allocate pages for unaligned cases.\n", + __func__); + return -ENOMEM; + } + + if (flags & OMAP_CRYPTO_COPY_DATA) { + scatterwalk_map_and_copy(buf, *sg, 0, total, 0); + if (flags & OMAP_CRYPTO_ZERO_BUF) + memset(buf + total, 0, new_len - total); + } + + if (!(flags & OMAP_CRYPTO_FORCE_SINGLE_ENTRY)) + sg_init_table(new_sg, 1); + + sg_set_buf(new_sg, buf, new_len); + + *sg = new_sg; + + return 0; +} + +static int omap_crypto_check_sg(struct scatterlist *sg, int total, int bs, + u16 flags) +{ + int len = 0; + int num_sg = 0; + + if (!IS_ALIGNED(total, bs)) + return OMAP_CRYPTO_NOT_ALIGNED; + + while (sg) { + num_sg++; + + if (!IS_ALIGNED(sg->offset, 4)) + return OMAP_CRYPTO_NOT_ALIGNED; + if (!IS_ALIGNED(sg->length, bs)) + return OMAP_CRYPTO_NOT_ALIGNED; +#ifdef CONFIG_ZONE_DMA + if (page_zonenum(sg_page(sg)) != ZONE_DMA) + return OMAP_CRYPTO_NOT_ALIGNED; +#endif + + len += sg->length; + sg = sg_next(sg); + + if (len >= total) + break; + } + + if ((flags & OMAP_CRYPTO_FORCE_SINGLE_ENTRY) && num_sg > 1) + return OMAP_CRYPTO_NOT_ALIGNED; + + if (len != total) + return OMAP_CRYPTO_BAD_DATA_LENGTH; + + return 0; +} + +int omap_crypto_align_sg(struct scatterlist **sg, int total, int bs, + struct scatterlist *new_sg, u16 flags, + u8 flags_shift, unsigned long *dd_flags) +{ + int ret; + + *dd_flags &= ~(OMAP_CRYPTO_COPY_MASK << flags_shift); + + if (flags & OMAP_CRYPTO_FORCE_COPY) + ret = OMAP_CRYPTO_NOT_ALIGNED; + else + ret = omap_crypto_check_sg(*sg, total, bs, flags); + + if (ret == OMAP_CRYPTO_NOT_ALIGNED) { + ret = omap_crypto_copy_sgs(total, bs, sg, new_sg, flags); + if (ret) + return ret; + *dd_flags |= OMAP_CRYPTO_DATA_COPIED << flags_shift; + } else if (ret == OMAP_CRYPTO_BAD_DATA_LENGTH) { + ret = omap_crypto_copy_sg_lists(total, bs, sg, new_sg, flags); + if (ret) + return ret; + if (!(flags & OMAP_CRYPTO_FORCE_SINGLE_ENTRY)) + *dd_flags |= OMAP_CRYPTO_SG_COPIED << flags_shift; + } else if (flags & OMAP_CRYPTO_FORCE_SINGLE_ENTRY) { + sg_set_buf(new_sg, sg_virt(*sg), (*sg)->length); + } + + return 0; +} +EXPORT_SYMBOL_GPL(omap_crypto_align_sg); + +static void omap_crypto_copy_data(struct scatterlist *src, + struct scatterlist *dst, + int offset, int len) +{ + int amt; + void *srcb, *dstb; + int srco = 0, dsto = offset; + + while (src && dst && len) { + if (srco >= src->length) { + srco -= src->length; + src = sg_next(src); + continue; + } + + if (dsto >= dst->length) { + dsto -= dst->length; + dst = sg_next(dst); + continue; + } + + amt = min(src->length - srco, dst->length - dsto); + amt = min(len, amt); + + srcb = kmap_atomic(sg_page(src)) + srco + src->offset; + dstb = kmap_atomic(sg_page(dst)) + dsto + dst->offset; + + memcpy(dstb, srcb, amt); + + flush_dcache_page(sg_page(dst)); + + kunmap_atomic(srcb); + kunmap_atomic(dstb); + + srco += amt; + dsto += amt; + len -= amt; + } +} + +void omap_crypto_cleanup(struct scatterlist *sg, struct scatterlist *orig, + int offset, int len, u8 flags_shift, + unsigned long flags) +{ + void *buf; + int pages; + + flags >>= flags_shift; + flags &= OMAP_CRYPTO_COPY_MASK; + + if (!flags) + return; + + buf = sg_virt(sg); + pages = get_order(len); + + if (orig && (flags & OMAP_CRYPTO_DATA_COPIED)) + omap_crypto_copy_data(sg, orig, offset, len); + + if (flags & OMAP_CRYPTO_DATA_COPIED) + free_pages((unsigned long)buf, pages); + else if (flags & OMAP_CRYPTO_SG_COPIED) + kfree(sg); +} +EXPORT_SYMBOL_GPL(omap_crypto_cleanup); + +MODULE_DESCRIPTION("OMAP crypto support library."); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Tero Kristo "); diff --git a/drivers/crypto/omap-crypto.h b/drivers/crypto/omap-crypto.h new file mode 100644 index 000000000..506ccde6f --- /dev/null +++ b/drivers/crypto/omap-crypto.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * OMAP Crypto driver common support routines. + * + * Copyright (c) 2017 Texas Instruments Incorporated + * Tero Kristo + */ + +#ifndef __CRYPTO_OMAP_CRYPTO_H +#define __CRYPTO_OMAP_CRYPTO_H + +enum { + OMAP_CRYPTO_NOT_ALIGNED = 1, + OMAP_CRYPTO_BAD_DATA_LENGTH, +}; + +#define OMAP_CRYPTO_DATA_COPIED BIT(0) +#define OMAP_CRYPTO_SG_COPIED BIT(1) + +#define OMAP_CRYPTO_COPY_MASK 0x3 + +#define OMAP_CRYPTO_COPY_DATA BIT(0) +#define OMAP_CRYPTO_FORCE_COPY BIT(1) +#define OMAP_CRYPTO_ZERO_BUF BIT(2) +#define OMAP_CRYPTO_FORCE_SINGLE_ENTRY BIT(3) + +int omap_crypto_align_sg(struct scatterlist **sg, int total, int bs, + struct scatterlist *new_sg, u16 flags, + u8 flags_shift, unsigned long *dd_flags); +void omap_crypto_cleanup(struct scatterlist *sg, struct scatterlist *orig, + int offset, int len, u8 flags_shift, + unsigned long flags); + +#endif diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c new file mode 100644 index 000000000..3b7104cb4 --- /dev/null +++ b/drivers/crypto/omap-des.c @@ -0,0 +1,1126 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Support for OMAP DES and Triple DES HW acceleration. + * + * Copyright (c) 2013 Texas Instruments Incorporated + * Author: Joel Fernandes + */ + +#define pr_fmt(fmt) "%s: " fmt, __func__ + +#ifdef DEBUG +#define prn(num) printk(#num "=%d\n", num) +#define prx(num) printk(#num "=%x\n", num) +#else +#define prn(num) do { } while (0) +#define prx(num) do { } while (0) +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "omap-crypto.h" + +#define DST_MAXBURST 2 + +#define DES_BLOCK_WORDS (DES_BLOCK_SIZE >> 2) + +#define DES_REG_KEY(dd, x) ((dd)->pdata->key_ofs - \ + ((x ^ 0x01) * 0x04)) + +#define DES_REG_IV(dd, x) ((dd)->pdata->iv_ofs + ((x) * 0x04)) + +#define DES_REG_CTRL(dd) ((dd)->pdata->ctrl_ofs) +#define DES_REG_CTRL_CBC BIT(4) +#define DES_REG_CTRL_TDES BIT(3) +#define DES_REG_CTRL_DIRECTION BIT(2) +#define DES_REG_CTRL_INPUT_READY BIT(1) +#define DES_REG_CTRL_OUTPUT_READY BIT(0) + +#define DES_REG_DATA_N(dd, x) ((dd)->pdata->data_ofs + ((x) * 0x04)) + +#define DES_REG_REV(dd) ((dd)->pdata->rev_ofs) + +#define DES_REG_MASK(dd) ((dd)->pdata->mask_ofs) + +#define DES_REG_LENGTH_N(x) (0x24 + ((x) * 0x04)) + +#define DES_REG_IRQ_STATUS(dd) ((dd)->pdata->irq_status_ofs) +#define DES_REG_IRQ_ENABLE(dd) ((dd)->pdata->irq_enable_ofs) +#define DES_REG_IRQ_DATA_IN BIT(1) +#define DES_REG_IRQ_DATA_OUT BIT(2) + +#define FLAGS_MODE_MASK 0x000f +#define FLAGS_ENCRYPT BIT(0) +#define FLAGS_CBC BIT(1) +#define FLAGS_INIT BIT(4) +#define FLAGS_BUSY BIT(6) + +#define DEFAULT_AUTOSUSPEND_DELAY 1000 + +#define FLAGS_IN_DATA_ST_SHIFT 8 +#define FLAGS_OUT_DATA_ST_SHIFT 10 + +struct omap_des_ctx { + struct omap_des_dev *dd; + + int keylen; + __le32 key[(3 * DES_KEY_SIZE) / sizeof(u32)]; + unsigned long flags; +}; + +struct omap_des_reqctx { + unsigned long mode; +}; + +#define OMAP_DES_QUEUE_LENGTH 1 +#define OMAP_DES_CACHE_SIZE 0 + +struct omap_des_algs_info { + struct skcipher_engine_alg *algs_list; + unsigned int size; + unsigned int registered; +}; + +struct omap_des_pdata { + struct omap_des_algs_info *algs_info; + unsigned int algs_info_size; + + void (*trigger)(struct omap_des_dev *dd, int length); + + u32 key_ofs; + u32 iv_ofs; + u32 ctrl_ofs; + u32 data_ofs; + u32 rev_ofs; + u32 mask_ofs; + u32 irq_enable_ofs; + u32 irq_status_ofs; + + u32 dma_enable_in; + u32 dma_enable_out; + u32 dma_start; + + u32 major_mask; + u32 major_shift; + u32 minor_mask; + u32 minor_shift; +}; + +struct omap_des_dev { + struct list_head list; + unsigned long phys_base; + void __iomem *io_base; + struct omap_des_ctx *ctx; + struct device *dev; + unsigned long flags; + int err; + + struct work_struct done_task; + + struct skcipher_request *req; + struct crypto_engine *engine; + /* + * total is used by PIO mode for book keeping so introduce + * variable total_save as need it to calc page_order + */ + size_t total; + size_t total_save; + + struct scatterlist *in_sg; + struct scatterlist *out_sg; + + /* Buffers for copying for unaligned cases */ + struct scatterlist in_sgl; + struct scatterlist out_sgl; + struct scatterlist *orig_out; + + unsigned int in_sg_offset; + unsigned int out_sg_offset; + struct dma_chan *dma_lch_in; + struct dma_chan *dma_lch_out; + int in_sg_len; + int out_sg_len; + int pio_only; + const struct omap_des_pdata *pdata; +}; + +/* keep registered devices data here */ +static LIST_HEAD(dev_list); +static DEFINE_SPINLOCK(list_lock); + +#ifdef DEBUG +#define omap_des_read(dd, offset) \ + ({ \ + int _read_ret; \ + _read_ret = __raw_readl(dd->io_base + offset); \ + pr_err("omap_des_read(" #offset "=%#x)= %#x\n", \ + offset, _read_ret); \ + _read_ret; \ + }) +#else +static inline u32 omap_des_read(struct omap_des_dev *dd, u32 offset) +{ + return __raw_readl(dd->io_base + offset); +} +#endif + +#ifdef DEBUG +#define omap_des_write(dd, offset, value) \ + do { \ + pr_err("omap_des_write(" #offset "=%#x) value=%#x\n", \ + offset, value); \ + __raw_writel(value, dd->io_base + offset); \ + } while (0) +#else +static inline void omap_des_write(struct omap_des_dev *dd, u32 offset, + u32 value) +{ + __raw_writel(value, dd->io_base + offset); +} +#endif + +static inline void omap_des_write_mask(struct omap_des_dev *dd, u32 offset, + u32 value, u32 mask) +{ + u32 val; + + val = omap_des_read(dd, offset); + val &= ~mask; + val |= value; + omap_des_write(dd, offset, val); +} + +static void omap_des_write_n(struct omap_des_dev *dd, u32 offset, + u32 *value, int count) +{ + for (; count--; value++, offset += 4) + omap_des_write(dd, offset, *value); +} + +static int omap_des_hw_init(struct omap_des_dev *dd) +{ + int err; + + /* + * clocks are enabled when request starts and disabled when finished. + * It may be long delays between requests. + * Device might go to off mode to save power. + */ + err = pm_runtime_resume_and_get(dd->dev); + if (err < 0) { + dev_err(dd->dev, "%s: failed to get_sync(%d)\n", __func__, err); + return err; + } + + if (!(dd->flags & FLAGS_INIT)) { + dd->flags |= FLAGS_INIT; + dd->err = 0; + } + + return 0; +} + +static int omap_des_write_ctrl(struct omap_des_dev *dd) +{ + unsigned int key32; + int i, err; + u32 val = 0, mask = 0; + + err = omap_des_hw_init(dd); + if (err) + return err; + + key32 = dd->ctx->keylen / sizeof(u32); + + /* it seems a key should always be set even if it has not changed */ + for (i = 0; i < key32; i++) { + omap_des_write(dd, DES_REG_KEY(dd, i), + __le32_to_cpu(dd->ctx->key[i])); + } + + if ((dd->flags & FLAGS_CBC) && dd->req->iv) + omap_des_write_n(dd, DES_REG_IV(dd, 0), (void *)dd->req->iv, 2); + + if (dd->flags & FLAGS_CBC) + val |= DES_REG_CTRL_CBC; + if (dd->flags & FLAGS_ENCRYPT) + val |= DES_REG_CTRL_DIRECTION; + if (key32 == 6) + val |= DES_REG_CTRL_TDES; + + mask |= DES_REG_CTRL_CBC | DES_REG_CTRL_DIRECTION | DES_REG_CTRL_TDES; + + omap_des_write_mask(dd, DES_REG_CTRL(dd), val, mask); + + return 0; +} + +static void omap_des_dma_trigger_omap4(struct omap_des_dev *dd, int length) +{ + u32 mask, val; + + omap_des_write(dd, DES_REG_LENGTH_N(0), length); + + val = dd->pdata->dma_start; + + if (dd->dma_lch_out != NULL) + val |= dd->pdata->dma_enable_out; + if (dd->dma_lch_in != NULL) + val |= dd->pdata->dma_enable_in; + + mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in | + dd->pdata->dma_start; + + omap_des_write_mask(dd, DES_REG_MASK(dd), val, mask); +} + +static void omap_des_dma_stop(struct omap_des_dev *dd) +{ + u32 mask; + + mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in | + dd->pdata->dma_start; + + omap_des_write_mask(dd, DES_REG_MASK(dd), 0, mask); +} + +static struct omap_des_dev *omap_des_find_dev(struct omap_des_ctx *ctx) +{ + struct omap_des_dev *dd = NULL, *tmp; + + spin_lock_bh(&list_lock); + if (!ctx->dd) { + list_for_each_entry(tmp, &dev_list, list) { + /* FIXME: take fist available des core */ + dd = tmp; + break; + } + ctx->dd = dd; + } else { + /* already found before */ + dd = ctx->dd; + } + spin_unlock_bh(&list_lock); + + return dd; +} + +static void omap_des_dma_out_callback(void *data) +{ + struct omap_des_dev *dd = data; + + /* dma_lch_out - completed */ + queue_work(system_bh_wq, &dd->done_task); +} + +static int omap_des_dma_init(struct omap_des_dev *dd) +{ + int err; + + dd->dma_lch_out = NULL; + dd->dma_lch_in = NULL; + + dd->dma_lch_in = dma_request_chan(dd->dev, "rx"); + if (IS_ERR(dd->dma_lch_in)) { + dev_err(dd->dev, "Unable to request in DMA channel\n"); + return PTR_ERR(dd->dma_lch_in); + } + + dd->dma_lch_out = dma_request_chan(dd->dev, "tx"); + if (IS_ERR(dd->dma_lch_out)) { + dev_err(dd->dev, "Unable to request out DMA channel\n"); + err = PTR_ERR(dd->dma_lch_out); + goto err_dma_out; + } + + return 0; + +err_dma_out: + dma_release_channel(dd->dma_lch_in); + + return err; +} + +static void omap_des_dma_cleanup(struct omap_des_dev *dd) +{ + if (dd->pio_only) + return; + + dma_release_channel(dd->dma_lch_out); + dma_release_channel(dd->dma_lch_in); +} + +static int omap_des_crypt_dma(struct crypto_tfm *tfm, + struct scatterlist *in_sg, struct scatterlist *out_sg, + int in_sg_len, int out_sg_len) +{ + struct omap_des_ctx *ctx = crypto_tfm_ctx(tfm); + struct omap_des_dev *dd = ctx->dd; + struct dma_async_tx_descriptor *tx_in, *tx_out; + struct dma_slave_config cfg; + int ret; + + if (dd->pio_only) { + dd->in_sg_offset = 0; + dd->out_sg_offset = 0; + + /* Enable DATAIN interrupt and let it take + care of the rest */ + omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x2); + return 0; + } + + dma_sync_sg_for_device(dd->dev, dd->in_sg, in_sg_len, DMA_TO_DEVICE); + + memset(&cfg, 0, sizeof(cfg)); + + cfg.src_addr = dd->phys_base + DES_REG_DATA_N(dd, 0); + cfg.dst_addr = dd->phys_base + DES_REG_DATA_N(dd, 0); + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cfg.src_maxburst = DST_MAXBURST; + cfg.dst_maxburst = DST_MAXBURST; + + /* IN */ + ret = dmaengine_slave_config(dd->dma_lch_in, &cfg); + if (ret) { + dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n", + ret); + return ret; + } + + tx_in = dmaengine_prep_slave_sg(dd->dma_lch_in, in_sg, in_sg_len, + DMA_MEM_TO_DEV, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!tx_in) { + dev_err(dd->dev, "IN prep_slave_sg() failed\n"); + return -EINVAL; + } + + /* No callback necessary */ + tx_in->callback_param = dd; + + /* OUT */ + ret = dmaengine_slave_config(dd->dma_lch_out, &cfg); + if (ret) { + dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n", + ret); + return ret; + } + + tx_out = dmaengine_prep_slave_sg(dd->dma_lch_out, out_sg, out_sg_len, + DMA_DEV_TO_MEM, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!tx_out) { + dev_err(dd->dev, "OUT prep_slave_sg() failed\n"); + return -EINVAL; + } + + tx_out->callback = omap_des_dma_out_callback; + tx_out->callback_param = dd; + + dmaengine_submit(tx_in); + dmaengine_submit(tx_out); + + dma_async_issue_pending(dd->dma_lch_in); + dma_async_issue_pending(dd->dma_lch_out); + + /* start DMA */ + dd->pdata->trigger(dd, dd->total); + + return 0; +} + +static int omap_des_crypt_dma_start(struct omap_des_dev *dd) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm( + crypto_skcipher_reqtfm(dd->req)); + int err; + + pr_debug("total: %zd\n", dd->total); + + if (!dd->pio_only) { + err = dma_map_sg(dd->dev, dd->in_sg, dd->in_sg_len, + DMA_TO_DEVICE); + if (!err) { + dev_err(dd->dev, "dma_map_sg() error\n"); + return -EINVAL; + } + + err = dma_map_sg(dd->dev, dd->out_sg, dd->out_sg_len, + DMA_FROM_DEVICE); + if (!err) { + dev_err(dd->dev, "dma_map_sg() error\n"); + return -EINVAL; + } + } + + err = omap_des_crypt_dma(tfm, dd->in_sg, dd->out_sg, dd->in_sg_len, + dd->out_sg_len); + if (err && !dd->pio_only) { + dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE); + dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, + DMA_FROM_DEVICE); + } + + return err; +} + +static void omap_des_finish_req(struct omap_des_dev *dd, int err) +{ + struct skcipher_request *req = dd->req; + + pr_debug("err: %d\n", err); + + crypto_finalize_skcipher_request(dd->engine, req, err); + + pm_runtime_put_autosuspend(dd->dev); +} + +static int omap_des_crypt_dma_stop(struct omap_des_dev *dd) +{ + pr_debug("total: %zd\n", dd->total); + + omap_des_dma_stop(dd); + + dmaengine_terminate_all(dd->dma_lch_in); + dmaengine_terminate_all(dd->dma_lch_out); + + return 0; +} + +static int omap_des_handle_queue(struct omap_des_dev *dd, + struct skcipher_request *req) +{ + if (req) + return crypto_transfer_skcipher_request_to_engine(dd->engine, req); + + return 0; +} + +static int omap_des_prepare_req(struct skcipher_request *req, + struct omap_des_dev *dd) +{ + struct omap_des_ctx *ctx = crypto_skcipher_ctx( + crypto_skcipher_reqtfm(req)); + struct omap_des_reqctx *rctx; + int ret; + u16 flags; + + /* assign new request to device */ + dd->req = req; + dd->total = req->cryptlen; + dd->total_save = req->cryptlen; + dd->in_sg = req->src; + dd->out_sg = req->dst; + dd->orig_out = req->dst; + + flags = OMAP_CRYPTO_COPY_DATA; + if (req->src == req->dst) + flags |= OMAP_CRYPTO_FORCE_COPY; + + ret = omap_crypto_align_sg(&dd->in_sg, dd->total, DES_BLOCK_SIZE, + &dd->in_sgl, flags, + FLAGS_IN_DATA_ST_SHIFT, &dd->flags); + if (ret) + return ret; + + ret = omap_crypto_align_sg(&dd->out_sg, dd->total, DES_BLOCK_SIZE, + &dd->out_sgl, 0, + FLAGS_OUT_DATA_ST_SHIFT, &dd->flags); + if (ret) + return ret; + + dd->in_sg_len = sg_nents_for_len(dd->in_sg, dd->total); + if (dd->in_sg_len < 0) + return dd->in_sg_len; + + dd->out_sg_len = sg_nents_for_len(dd->out_sg, dd->total); + if (dd->out_sg_len < 0) + return dd->out_sg_len; + + rctx = skcipher_request_ctx(req); + ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + rctx->mode &= FLAGS_MODE_MASK; + dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode; + + dd->ctx = ctx; + ctx->dd = dd; + + return omap_des_write_ctrl(dd); +} + +static int omap_des_crypt_req(struct crypto_engine *engine, + void *areq) +{ + struct skcipher_request *req = container_of(areq, struct skcipher_request, base); + struct omap_des_ctx *ctx = crypto_skcipher_ctx( + crypto_skcipher_reqtfm(req)); + struct omap_des_dev *dd = omap_des_find_dev(ctx); + + if (!dd) + return -ENODEV; + + return omap_des_prepare_req(req, dd) ?: + omap_des_crypt_dma_start(dd); +} + +static void omap_des_done_task(struct work_struct *t) +{ + struct omap_des_dev *dd = from_work(dd, t, done_task); + int i; + + pr_debug("enter done_task\n"); + + if (!dd->pio_only) { + dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len, + DMA_FROM_DEVICE); + dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE); + dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, + DMA_FROM_DEVICE); + omap_des_crypt_dma_stop(dd); + } + + omap_crypto_cleanup(&dd->in_sgl, NULL, 0, dd->total_save, + FLAGS_IN_DATA_ST_SHIFT, dd->flags); + + omap_crypto_cleanup(&dd->out_sgl, dd->orig_out, 0, dd->total_save, + FLAGS_OUT_DATA_ST_SHIFT, dd->flags); + + if ((dd->flags & FLAGS_CBC) && dd->req->iv) + for (i = 0; i < 2; i++) + ((u32 *)dd->req->iv)[i] = + omap_des_read(dd, DES_REG_IV(dd, i)); + + omap_des_finish_req(dd, 0); + + pr_debug("exit\n"); +} + +static int omap_des_crypt(struct skcipher_request *req, unsigned long mode) +{ + struct omap_des_ctx *ctx = crypto_skcipher_ctx( + crypto_skcipher_reqtfm(req)); + struct omap_des_reqctx *rctx = skcipher_request_ctx(req); + struct omap_des_dev *dd; + + pr_debug("nbytes: %d, enc: %d, cbc: %d\n", req->cryptlen, + !!(mode & FLAGS_ENCRYPT), + !!(mode & FLAGS_CBC)); + + if (!req->cryptlen) + return 0; + + if (!IS_ALIGNED(req->cryptlen, DES_BLOCK_SIZE)) + return -EINVAL; + + dd = omap_des_find_dev(ctx); + if (!dd) + return -ENODEV; + + rctx->mode = mode; + + return omap_des_handle_queue(dd, req); +} + +/* ********************** ALG API ************************************ */ + +static int omap_des_setkey(struct crypto_skcipher *cipher, const u8 *key, + unsigned int keylen) +{ + struct omap_des_ctx *ctx = crypto_skcipher_ctx(cipher); + int err; + + pr_debug("enter, keylen: %d\n", keylen); + + err = verify_skcipher_des_key(cipher, key); + if (err) + return err; + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + return 0; +} + +static int omap_des3_setkey(struct crypto_skcipher *cipher, const u8 *key, + unsigned int keylen) +{ + struct omap_des_ctx *ctx = crypto_skcipher_ctx(cipher); + int err; + + pr_debug("enter, keylen: %d\n", keylen); + + err = verify_skcipher_des3_key(cipher, key); + if (err) + return err; + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + return 0; +} + +static int omap_des_ecb_encrypt(struct skcipher_request *req) +{ + return omap_des_crypt(req, FLAGS_ENCRYPT); +} + +static int omap_des_ecb_decrypt(struct skcipher_request *req) +{ + return omap_des_crypt(req, 0); +} + +static int omap_des_cbc_encrypt(struct skcipher_request *req) +{ + return omap_des_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC); +} + +static int omap_des_cbc_decrypt(struct skcipher_request *req) +{ + return omap_des_crypt(req, FLAGS_CBC); +} + +static int omap_des_init_tfm(struct crypto_skcipher *tfm) +{ + pr_debug("enter\n"); + + crypto_skcipher_set_reqsize(tfm, sizeof(struct omap_des_reqctx)); + + return 0; +} + +/* ********************** ALGS ************************************ */ + +static struct skcipher_engine_alg algs_ecb_cbc[] = { +{ + .base = { + .base.cra_name = "ecb(des)", + .base.cra_driver_name = "ecb-des-omap", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct omap_des_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = omap_des_setkey, + .encrypt = omap_des_ecb_encrypt, + .decrypt = omap_des_ecb_decrypt, + .init = omap_des_init_tfm, + }, + .op.do_one_request = omap_des_crypt_req, +}, +{ + .base = { + .base.cra_name = "cbc(des)", + .base.cra_driver_name = "cbc-des-omap", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct omap_des_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = omap_des_setkey, + .encrypt = omap_des_cbc_encrypt, + .decrypt = omap_des_cbc_decrypt, + .init = omap_des_init_tfm, + }, + .op.do_one_request = omap_des_crypt_req, +}, +{ + .base = { + .base.cra_name = "ecb(des3_ede)", + .base.cra_driver_name = "ecb-des3-omap", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct omap_des_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = omap_des3_setkey, + .encrypt = omap_des_ecb_encrypt, + .decrypt = omap_des_ecb_decrypt, + .init = omap_des_init_tfm, + }, + .op.do_one_request = omap_des_crypt_req, +}, +{ + .base = { + .base.cra_name = "cbc(des3_ede)", + .base.cra_driver_name = "cbc-des3-omap", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct omap_des_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setkey = omap_des3_setkey, + .encrypt = omap_des_cbc_encrypt, + .decrypt = omap_des_cbc_decrypt, + .init = omap_des_init_tfm, + }, + .op.do_one_request = omap_des_crypt_req, +} +}; + +static struct omap_des_algs_info omap_des_algs_info_ecb_cbc[] = { + { + .algs_list = algs_ecb_cbc, + .size = ARRAY_SIZE(algs_ecb_cbc), + }, +}; + +static const struct omap_des_pdata omap_des_pdata_omap4 = { + .algs_info = omap_des_algs_info_ecb_cbc, + .algs_info_size = ARRAY_SIZE(omap_des_algs_info_ecb_cbc), + .trigger = omap_des_dma_trigger_omap4, + .key_ofs = 0x14, + .iv_ofs = 0x18, + .ctrl_ofs = 0x20, + .data_ofs = 0x28, + .rev_ofs = 0x30, + .mask_ofs = 0x34, + .irq_status_ofs = 0x3c, + .irq_enable_ofs = 0x40, + .dma_enable_in = BIT(5), + .dma_enable_out = BIT(6), + .major_mask = 0x0700, + .major_shift = 8, + .minor_mask = 0x003f, + .minor_shift = 0, +}; + +static irqreturn_t omap_des_irq(int irq, void *dev_id) +{ + struct omap_des_dev *dd = dev_id; + u32 status, i; + u32 *src, *dst; + + status = omap_des_read(dd, DES_REG_IRQ_STATUS(dd)); + if (status & DES_REG_IRQ_DATA_IN) { + omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x0); + + BUG_ON(!dd->in_sg); + + BUG_ON(dd->in_sg_offset > dd->in_sg->length); + + src = sg_virt(dd->in_sg) + dd->in_sg_offset; + + for (i = 0; i < DES_BLOCK_WORDS; i++) { + omap_des_write(dd, DES_REG_DATA_N(dd, i), *src); + dd->in_sg_offset += 4; + if (dd->in_sg_offset == dd->in_sg->length) { + dd->in_sg = sg_next(dd->in_sg); + if (dd->in_sg) { + dd->in_sg_offset = 0; + src = sg_virt(dd->in_sg); + } + } else { + src++; + } + } + + /* Clear IRQ status */ + status &= ~DES_REG_IRQ_DATA_IN; + omap_des_write(dd, DES_REG_IRQ_STATUS(dd), status); + + /* Enable DATA_OUT interrupt */ + omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x4); + + } else if (status & DES_REG_IRQ_DATA_OUT) { + omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x0); + + BUG_ON(!dd->out_sg); + + BUG_ON(dd->out_sg_offset > dd->out_sg->length); + + dst = sg_virt(dd->out_sg) + dd->out_sg_offset; + + for (i = 0; i < DES_BLOCK_WORDS; i++) { + *dst = omap_des_read(dd, DES_REG_DATA_N(dd, i)); + dd->out_sg_offset += 4; + if (dd->out_sg_offset == dd->out_sg->length) { + dd->out_sg = sg_next(dd->out_sg); + if (dd->out_sg) { + dd->out_sg_offset = 0; + dst = sg_virt(dd->out_sg); + } + } else { + dst++; + } + } + + BUG_ON(dd->total < DES_BLOCK_SIZE); + + dd->total -= DES_BLOCK_SIZE; + + /* Clear IRQ status */ + status &= ~DES_REG_IRQ_DATA_OUT; + omap_des_write(dd, DES_REG_IRQ_STATUS(dd), status); + + if (!dd->total) + /* All bytes read! */ + queue_work(system_bh_wq, &dd->done_task); + else + /* Enable DATA_IN interrupt for next block */ + omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x2); + } + + return IRQ_HANDLED; +} + +static const struct of_device_id omap_des_of_match[] = { + { + .compatible = "ti,omap4-des", + .data = &omap_des_pdata_omap4, + }, + {}, +}; +MODULE_DEVICE_TABLE(of, omap_des_of_match); + +#ifdef CONFIG_OF +static int omap_des_get_of(struct omap_des_dev *dd, + struct platform_device *pdev) +{ + + dd->pdata = of_device_get_match_data(&pdev->dev); + if (!dd->pdata) { + dev_err(&pdev->dev, "no compatible OF match\n"); + return -EINVAL; + } + + return 0; +} +#else +static int omap_des_get_of(struct omap_des_dev *dd, + struct platform_device *pdev) +{ + return -EINVAL; +} +#endif + +static int omap_des_get_pdev(struct omap_des_dev *dd, + struct platform_device *pdev) +{ + /* non-DT devices get pdata from pdev */ + dd->pdata = pdev->dev.platform_data; + + return 0; +} + +static void omap_des_unregister_algs(const struct omap_des_pdata *pdata) +{ + struct omap_des_algs_info *alg_info; + int i; + + for (i = pdata->algs_info_size - 1; i >= 0; i--) { + alg_info = &pdata->algs_info[i]; + + crypto_engine_unregister_skciphers(alg_info->algs_list, + alg_info->registered); + alg_info->registered = 0; + } +} + +static int omap_des_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct omap_des_dev *dd; + struct skcipher_engine_alg *algp; + struct resource *res; + int err = -ENOMEM, i, j, irq = -1; + u32 reg; + + dd = devm_kzalloc(dev, sizeof(struct omap_des_dev), GFP_KERNEL); + if (dd == NULL) { + dev_err(dev, "unable to alloc data struct.\n"); + goto err_data; + } + dd->dev = dev; + platform_set_drvdata(pdev, dd); + + err = (dev->of_node) ? omap_des_get_of(dd, pdev) : + omap_des_get_pdev(dd, pdev); + if (err) + goto err_res; + + dd->io_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); + if (IS_ERR(dd->io_base)) { + err = PTR_ERR(dd->io_base); + goto err_res; + } + dd->phys_base = res->start; + + pm_runtime_use_autosuspend(dev); + pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY); + + pm_runtime_enable(dev); + err = pm_runtime_resume_and_get(dev); + if (err < 0) { + dev_err(dd->dev, "%s: failed to get_sync(%d)\n", __func__, err); + goto err_get; + } + + omap_des_dma_stop(dd); + + reg = omap_des_read(dd, DES_REG_REV(dd)); + + pm_runtime_put_sync(dev); + + dev_info(dev, "OMAP DES hw accel rev: %u.%u\n", + (reg & dd->pdata->major_mask) >> dd->pdata->major_shift, + (reg & dd->pdata->minor_mask) >> dd->pdata->minor_shift); + + INIT_WORK(&dd->done_task, omap_des_done_task); + + err = omap_des_dma_init(dd); + if (err == -EPROBE_DEFER) { + goto err_irq; + } else if (err && DES_REG_IRQ_STATUS(dd) && DES_REG_IRQ_ENABLE(dd)) { + dd->pio_only = 1; + + irq = platform_get_irq(pdev, 0); + if (irq < 0) { + err = irq; + goto err_irq; + } + + err = devm_request_irq(dev, irq, omap_des_irq, 0, + dev_name(dev), dd); + if (err) + goto err_irq; + } + + + INIT_LIST_HEAD(&dd->list); + spin_lock_bh(&list_lock); + list_add_tail(&dd->list, &dev_list); + spin_unlock_bh(&list_lock); + + /* Initialize des crypto engine */ + dd->engine = crypto_engine_alloc_init(dev, 1); + if (!dd->engine) { + err = -ENOMEM; + goto err_engine; + } + + err = crypto_engine_start(dd->engine); + if (err) + goto err_engine; + + for (i = 0; i < dd->pdata->algs_info_size; i++) { + for (j = 0; j < dd->pdata->algs_info[i].size; j++) { + algp = &dd->pdata->algs_info[i].algs_list[j]; + + pr_debug("reg alg: %s\n", algp->base.base.cra_name); + + err = crypto_engine_register_skcipher(algp); + if (err) + goto err_algs; + + dd->pdata->algs_info[i].registered++; + } + } + + return 0; + +err_algs: + omap_des_unregister_algs(dd->pdata); +err_engine: + if (dd->engine) + crypto_engine_exit(dd->engine); + + omap_des_dma_cleanup(dd); +err_irq: + cancel_work_sync(&dd->done_task); +err_get: + pm_runtime_disable(dev); +err_res: + dd = NULL; +err_data: + dev_err(dev, "initialization failed.\n"); + return err; +} + +static void omap_des_remove(struct platform_device *pdev) +{ + struct omap_des_dev *dd = platform_get_drvdata(pdev); + + spin_lock_bh(&list_lock); + list_del(&dd->list); + spin_unlock_bh(&list_lock); + + omap_des_unregister_algs(dd->pdata); + + cancel_work_sync(&dd->done_task); + omap_des_dma_cleanup(dd); + pm_runtime_disable(dd->dev); +} + +#ifdef CONFIG_PM_SLEEP +static int omap_des_suspend(struct device *dev) +{ + pm_runtime_put_sync(dev); + return 0; +} + +static int omap_des_resume(struct device *dev) +{ + int err; + + err = pm_runtime_resume_and_get(dev); + if (err < 0) { + dev_err(dev, "%s: failed to get_sync(%d)\n", __func__, err); + return err; + } + return 0; +} +#endif + +static SIMPLE_DEV_PM_OPS(omap_des_pm_ops, omap_des_suspend, omap_des_resume); + +static struct platform_driver omap_des_driver = { + .probe = omap_des_probe, + .remove = omap_des_remove, + .driver = { + .name = "omap-des", + .pm = &omap_des_pm_ops, + .of_match_table = omap_des_of_match, + }, +}; + +module_platform_driver(omap_des_driver); + +MODULE_DESCRIPTION("OMAP DES hw acceleration support."); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Joel Fernandes "); diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c new file mode 100644 index 000000000..6d2dd0b88 --- /dev/null +++ b/drivers/crypto/omap-sham.c @@ -0,0 +1,2155 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Cryptographic API. + * + * Support for OMAP SHA1/MD5 HW acceleration. + * + * Copyright (c) 2010 Nokia Corporation + * Author: Dmitry Kasatkin + * Copyright (c) 2011 Texas Instruments Incorporated + * + * Some ideas are from old omap-sha1-md5.c driver. + */ + +#define pr_fmt(fmt) "%s: " fmt, __func__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MD5_DIGEST_SIZE 16 + +#define SHA_REG_IDIGEST(dd, x) ((dd)->pdata->idigest_ofs + ((x)*0x04)) +#define SHA_REG_DIN(dd, x) ((dd)->pdata->din_ofs + ((x) * 0x04)) +#define SHA_REG_DIGCNT(dd) ((dd)->pdata->digcnt_ofs) + +#define SHA_REG_ODIGEST(dd, x) ((dd)->pdata->odigest_ofs + (x * 0x04)) + +#define SHA_REG_CTRL 0x18 +#define SHA_REG_CTRL_LENGTH (0xFFFFFFFF << 5) +#define SHA_REG_CTRL_CLOSE_HASH (1 << 4) +#define SHA_REG_CTRL_ALGO_CONST (1 << 3) +#define SHA_REG_CTRL_ALGO (1 << 2) +#define SHA_REG_CTRL_INPUT_READY (1 << 1) +#define SHA_REG_CTRL_OUTPUT_READY (1 << 0) + +#define SHA_REG_REV(dd) ((dd)->pdata->rev_ofs) + +#define SHA_REG_MASK(dd) ((dd)->pdata->mask_ofs) +#define SHA_REG_MASK_DMA_EN (1 << 3) +#define SHA_REG_MASK_IT_EN (1 << 2) +#define SHA_REG_MASK_SOFTRESET (1 << 1) +#define SHA_REG_AUTOIDLE (1 << 0) + +#define SHA_REG_SYSSTATUS(dd) ((dd)->pdata->sysstatus_ofs) +#define SHA_REG_SYSSTATUS_RESETDONE (1 << 0) + +#define SHA_REG_MODE(dd) ((dd)->pdata->mode_ofs) +#define SHA_REG_MODE_HMAC_OUTER_HASH (1 << 7) +#define SHA_REG_MODE_HMAC_KEY_PROC (1 << 5) +#define SHA_REG_MODE_CLOSE_HASH (1 << 4) +#define SHA_REG_MODE_ALGO_CONSTANT (1 << 3) + +#define SHA_REG_MODE_ALGO_MASK (7 << 0) +#define SHA_REG_MODE_ALGO_MD5_128 (0 << 1) +#define SHA_REG_MODE_ALGO_SHA1_160 (1 << 1) +#define SHA_REG_MODE_ALGO_SHA2_224 (2 << 1) +#define SHA_REG_MODE_ALGO_SHA2_256 (3 << 1) +#define SHA_REG_MODE_ALGO_SHA2_384 (1 << 0) +#define SHA_REG_MODE_ALGO_SHA2_512 (3 << 0) + +#define SHA_REG_LENGTH(dd) ((dd)->pdata->length_ofs) + +#define SHA_REG_IRQSTATUS 0x118 +#define SHA_REG_IRQSTATUS_CTX_RDY (1 << 3) +#define SHA_REG_IRQSTATUS_PARTHASH_RDY (1 << 2) +#define SHA_REG_IRQSTATUS_INPUT_RDY (1 << 1) +#define SHA_REG_IRQSTATUS_OUTPUT_RDY (1 << 0) + +#define SHA_REG_IRQENA 0x11C +#define SHA_REG_IRQENA_CTX_RDY (1 << 3) +#define SHA_REG_IRQENA_PARTHASH_RDY (1 << 2) +#define SHA_REG_IRQENA_INPUT_RDY (1 << 1) +#define SHA_REG_IRQENA_OUTPUT_RDY (1 << 0) + +#define DEFAULT_TIMEOUT_INTERVAL HZ + +#define DEFAULT_AUTOSUSPEND_DELAY 1000 + +/* mostly device flags */ +#define FLAGS_FINAL 1 +#define FLAGS_DMA_ACTIVE 2 +#define FLAGS_OUTPUT_READY 3 +#define FLAGS_CPU 5 +#define FLAGS_DMA_READY 6 +#define FLAGS_AUTO_XOR 7 +#define FLAGS_BE32_SHA1 8 +#define FLAGS_SGS_COPIED 9 +#define FLAGS_SGS_ALLOCED 10 +#define FLAGS_HUGE 11 + +/* context flags */ +#define FLAGS_FINUP 16 + +#define FLAGS_MODE_SHIFT 18 +#define FLAGS_MODE_MASK (SHA_REG_MODE_ALGO_MASK << FLAGS_MODE_SHIFT) +#define FLAGS_MODE_MD5 (SHA_REG_MODE_ALGO_MD5_128 << FLAGS_MODE_SHIFT) +#define FLAGS_MODE_SHA1 (SHA_REG_MODE_ALGO_SHA1_160 << FLAGS_MODE_SHIFT) +#define FLAGS_MODE_SHA224 (SHA_REG_MODE_ALGO_SHA2_224 << FLAGS_MODE_SHIFT) +#define FLAGS_MODE_SHA256 (SHA_REG_MODE_ALGO_SHA2_256 << FLAGS_MODE_SHIFT) +#define FLAGS_MODE_SHA384 (SHA_REG_MODE_ALGO_SHA2_384 << FLAGS_MODE_SHIFT) +#define FLAGS_MODE_SHA512 (SHA_REG_MODE_ALGO_SHA2_512 << FLAGS_MODE_SHIFT) + +#define FLAGS_HMAC 21 +#define FLAGS_ERROR 22 + +#define OP_UPDATE 1 +#define OP_FINAL 2 + +#define OMAP_ALIGN_MASK (sizeof(u32)-1) +#define OMAP_ALIGNED __attribute__((aligned(sizeof(u32)))) + +#define BUFLEN SHA512_BLOCK_SIZE +#define OMAP_SHA_DMA_THRESHOLD 256 + +#define OMAP_SHA_MAX_DMA_LEN (1024 * 2048) + +struct omap_sham_dev; + +struct omap_sham_reqctx { + struct omap_sham_dev *dd; + unsigned long flags; + u8 op; + + u8 digest[SHA512_DIGEST_SIZE] OMAP_ALIGNED; + size_t digcnt; + size_t bufcnt; + + /* walk state */ + struct scatterlist *sg; + struct scatterlist sgl[2]; + int offset; /* offset in current sg */ + int sg_len; + unsigned int total; /* total request */ + + u8 buffer[BUFLEN] OMAP_ALIGNED; +}; + +struct omap_sham_hmac_ctx { + struct crypto_shash *shash; + u8 ipad[SHA512_BLOCK_SIZE] OMAP_ALIGNED; + u8 opad[SHA512_BLOCK_SIZE] OMAP_ALIGNED; +}; + +struct omap_sham_ctx { + unsigned long flags; + + /* fallback stuff */ + struct crypto_shash *fallback; + + struct omap_sham_hmac_ctx base[]; +}; + +#define OMAP_SHAM_QUEUE_LENGTH 10 + +struct omap_sham_algs_info { + struct ahash_engine_alg *algs_list; + unsigned int size; + unsigned int registered; +}; + +struct omap_sham_pdata { + struct omap_sham_algs_info *algs_info; + unsigned int algs_info_size; + unsigned long flags; + int digest_size; + + void (*copy_hash)(struct ahash_request *req, int out); + void (*write_ctrl)(struct omap_sham_dev *dd, size_t length, + int final, int dma); + void (*trigger)(struct omap_sham_dev *dd, size_t length); + int (*poll_irq)(struct omap_sham_dev *dd); + irqreturn_t (*intr_hdlr)(int irq, void *dev_id); + + u32 odigest_ofs; + u32 idigest_ofs; + u32 din_ofs; + u32 digcnt_ofs; + u32 rev_ofs; + u32 mask_ofs; + u32 sysstatus_ofs; + u32 mode_ofs; + u32 length_ofs; + + u32 major_mask; + u32 major_shift; + u32 minor_mask; + u32 minor_shift; +}; + +struct omap_sham_dev { + struct list_head list; + unsigned long phys_base; + struct device *dev; + void __iomem *io_base; + int irq; + int err; + struct dma_chan *dma_lch; + struct work_struct done_task; + u8 polling_mode; + u8 xmit_buf[BUFLEN] OMAP_ALIGNED; + + unsigned long flags; + int fallback_sz; + struct crypto_queue queue; + struct ahash_request *req; + struct crypto_engine *engine; + + const struct omap_sham_pdata *pdata; +}; + +struct omap_sham_drv { + struct list_head dev_list; + spinlock_t lock; + unsigned long flags; +}; + +static struct omap_sham_drv sham = { + .dev_list = LIST_HEAD_INIT(sham.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(sham.lock), +}; + +static int omap_sham_enqueue(struct ahash_request *req, unsigned int op); +static void omap_sham_finish_req(struct ahash_request *req, int err); + +static inline u32 omap_sham_read(struct omap_sham_dev *dd, u32 offset) +{ + return __raw_readl(dd->io_base + offset); +} + +static inline void omap_sham_write(struct omap_sham_dev *dd, + u32 offset, u32 value) +{ + __raw_writel(value, dd->io_base + offset); +} + +static inline void omap_sham_write_mask(struct omap_sham_dev *dd, u32 address, + u32 value, u32 mask) +{ + u32 val; + + val = omap_sham_read(dd, address); + val &= ~mask; + val |= value; + omap_sham_write(dd, address, val); +} + +static inline int omap_sham_wait(struct omap_sham_dev *dd, u32 offset, u32 bit) +{ + unsigned long timeout = jiffies + DEFAULT_TIMEOUT_INTERVAL; + + while (!(omap_sham_read(dd, offset) & bit)) { + if (time_is_before_jiffies(timeout)) + return -ETIMEDOUT; + } + + return 0; +} + +static void omap_sham_copy_hash_omap2(struct ahash_request *req, int out) +{ + struct omap_sham_reqctx *ctx = ahash_request_ctx(req); + struct omap_sham_dev *dd = ctx->dd; + u32 *hash = (u32 *)ctx->digest; + int i; + + for (i = 0; i < dd->pdata->digest_size / sizeof(u32); i++) { + if (out) + hash[i] = omap_sham_read(dd, SHA_REG_IDIGEST(dd, i)); + else + omap_sham_write(dd, SHA_REG_IDIGEST(dd, i), hash[i]); + } +} + +static void omap_sham_copy_hash_omap4(struct ahash_request *req, int out) +{ + struct omap_sham_reqctx *ctx = ahash_request_ctx(req); + struct omap_sham_dev *dd = ctx->dd; + int i; + + if (ctx->flags & BIT(FLAGS_HMAC)) { + struct crypto_ahash *tfm = crypto_ahash_reqtfm(dd->req); + struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct omap_sham_hmac_ctx *bctx = tctx->base; + u32 *opad = (u32 *)bctx->opad; + + for (i = 0; i < dd->pdata->digest_size / sizeof(u32); i++) { + if (out) + opad[i] = omap_sham_read(dd, + SHA_REG_ODIGEST(dd, i)); + else + omap_sham_write(dd, SHA_REG_ODIGEST(dd, i), + opad[i]); + } + } + + omap_sham_copy_hash_omap2(req, out); +} + +static void omap_sham_copy_ready_hash(struct ahash_request *req) +{ + struct omap_sham_reqctx *ctx = ahash_request_ctx(req); + u32 *in = (u32 *)ctx->digest; + u32 *hash = (u32 *)req->result; + int i, d, big_endian = 0; + + if (!hash) + return; + + switch (ctx->flags & FLAGS_MODE_MASK) { + case FLAGS_MODE_MD5: + d = MD5_DIGEST_SIZE / sizeof(u32); + break; + case FLAGS_MODE_SHA1: + /* OMAP2 SHA1 is big endian */ + if (test_bit(FLAGS_BE32_SHA1, &ctx->dd->flags)) + big_endian = 1; + d = SHA1_DIGEST_SIZE / sizeof(u32); + break; + case FLAGS_MODE_SHA224: + d = SHA224_DIGEST_SIZE / sizeof(u32); + break; + case FLAGS_MODE_SHA256: + d = SHA256_DIGEST_SIZE / sizeof(u32); + break; + case FLAGS_MODE_SHA384: + d = SHA384_DIGEST_SIZE / sizeof(u32); + break; + case FLAGS_MODE_SHA512: + d = SHA512_DIGEST_SIZE / sizeof(u32); + break; + default: + d = 0; + } + + if (big_endian) + for (i = 0; i < d; i++) + put_unaligned(be32_to_cpup((__be32 *)in + i), &hash[i]); + else + for (i = 0; i < d; i++) + put_unaligned(le32_to_cpup((__le32 *)in + i), &hash[i]); +} + +static void omap_sham_write_ctrl_omap2(struct omap_sham_dev *dd, size_t length, + int final, int dma) +{ + struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); + u32 val = length << 5, mask; + + if (likely(ctx->digcnt)) + omap_sham_write(dd, SHA_REG_DIGCNT(dd), ctx->digcnt); + + omap_sham_write_mask(dd, SHA_REG_MASK(dd), + SHA_REG_MASK_IT_EN | (dma ? SHA_REG_MASK_DMA_EN : 0), + SHA_REG_MASK_IT_EN | SHA_REG_MASK_DMA_EN); + /* + * Setting ALGO_CONST only for the first iteration + * and CLOSE_HASH only for the last one. + */ + if ((ctx->flags & FLAGS_MODE_MASK) == FLAGS_MODE_SHA1) + val |= SHA_REG_CTRL_ALGO; + if (!ctx->digcnt) + val |= SHA_REG_CTRL_ALGO_CONST; + if (final) + val |= SHA_REG_CTRL_CLOSE_HASH; + + mask = SHA_REG_CTRL_ALGO_CONST | SHA_REG_CTRL_CLOSE_HASH | + SHA_REG_CTRL_ALGO | SHA_REG_CTRL_LENGTH; + + omap_sham_write_mask(dd, SHA_REG_CTRL, val, mask); +} + +static void omap_sham_trigger_omap2(struct omap_sham_dev *dd, size_t length) +{ +} + +static int omap_sham_poll_irq_omap2(struct omap_sham_dev *dd) +{ + return omap_sham_wait(dd, SHA_REG_CTRL, SHA_REG_CTRL_INPUT_READY); +} + +static int get_block_size(struct omap_sham_reqctx *ctx) +{ + int d; + + switch (ctx->flags & FLAGS_MODE_MASK) { + case FLAGS_MODE_MD5: + case FLAGS_MODE_SHA1: + d = SHA1_BLOCK_SIZE; + break; + case FLAGS_MODE_SHA224: + case FLAGS_MODE_SHA256: + d = SHA256_BLOCK_SIZE; + break; + case FLAGS_MODE_SHA384: + case FLAGS_MODE_SHA512: + d = SHA512_BLOCK_SIZE; + break; + default: + d = 0; + } + + return d; +} + +static void omap_sham_write_n(struct omap_sham_dev *dd, u32 offset, + u32 *value, int count) +{ + for (; count--; value++, offset += 4) + omap_sham_write(dd, offset, *value); +} + +static void omap_sham_write_ctrl_omap4(struct omap_sham_dev *dd, size_t length, + int final, int dma) +{ + struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); + u32 val, mask; + + if (likely(ctx->digcnt)) + omap_sham_write(dd, SHA_REG_DIGCNT(dd), ctx->digcnt); + + /* + * Setting ALGO_CONST only for the first iteration and + * CLOSE_HASH only for the last one. Note that flags mode bits + * correspond to algorithm encoding in mode register. + */ + val = (ctx->flags & FLAGS_MODE_MASK) >> (FLAGS_MODE_SHIFT); + if (!ctx->digcnt) { + struct crypto_ahash *tfm = crypto_ahash_reqtfm(dd->req); + struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct omap_sham_hmac_ctx *bctx = tctx->base; + int bs, nr_dr; + + val |= SHA_REG_MODE_ALGO_CONSTANT; + + if (ctx->flags & BIT(FLAGS_HMAC)) { + bs = get_block_size(ctx); + nr_dr = bs / (2 * sizeof(u32)); + val |= SHA_REG_MODE_HMAC_KEY_PROC; + omap_sham_write_n(dd, SHA_REG_ODIGEST(dd, 0), + (u32 *)bctx->ipad, nr_dr); + omap_sham_write_n(dd, SHA_REG_IDIGEST(dd, 0), + (u32 *)bctx->ipad + nr_dr, nr_dr); + ctx->digcnt += bs; + } + } + + if (final) { + val |= SHA_REG_MODE_CLOSE_HASH; + + if (ctx->flags & BIT(FLAGS_HMAC)) + val |= SHA_REG_MODE_HMAC_OUTER_HASH; + } + + mask = SHA_REG_MODE_ALGO_CONSTANT | SHA_REG_MODE_CLOSE_HASH | + SHA_REG_MODE_ALGO_MASK | SHA_REG_MODE_HMAC_OUTER_HASH | + SHA_REG_MODE_HMAC_KEY_PROC; + + dev_dbg(dd->dev, "ctrl: %08x, flags: %08lx\n", val, ctx->flags); + omap_sham_write_mask(dd, SHA_REG_MODE(dd), val, mask); + omap_sham_write(dd, SHA_REG_IRQENA, SHA_REG_IRQENA_OUTPUT_RDY); + omap_sham_write_mask(dd, SHA_REG_MASK(dd), + SHA_REG_MASK_IT_EN | + (dma ? SHA_REG_MASK_DMA_EN : 0), + SHA_REG_MASK_IT_EN | SHA_REG_MASK_DMA_EN); +} + +static void omap_sham_trigger_omap4(struct omap_sham_dev *dd, size_t length) +{ + omap_sham_write(dd, SHA_REG_LENGTH(dd), length); +} + +static int omap_sham_poll_irq_omap4(struct omap_sham_dev *dd) +{ + return omap_sham_wait(dd, SHA_REG_IRQSTATUS, + SHA_REG_IRQSTATUS_INPUT_RDY); +} + +static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, size_t length, + int final) +{ + struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); + int count, len32, bs32, offset = 0; + const u32 *buffer; + int mlen; + struct sg_mapping_iter mi; + + dev_dbg(dd->dev, "xmit_cpu: digcnt: %zd, length: %zd, final: %d\n", + ctx->digcnt, length, final); + + dd->pdata->write_ctrl(dd, length, final, 0); + dd->pdata->trigger(dd, length); + + /* should be non-zero before next lines to disable clocks later */ + ctx->digcnt += length; + ctx->total -= length; + + if (final) + set_bit(FLAGS_FINAL, &dd->flags); /* catch last interrupt */ + + set_bit(FLAGS_CPU, &dd->flags); + + len32 = DIV_ROUND_UP(length, sizeof(u32)); + bs32 = get_block_size(ctx) / sizeof(u32); + + sg_miter_start(&mi, ctx->sg, ctx->sg_len, + SG_MITER_FROM_SG | SG_MITER_ATOMIC); + + mlen = 0; + + while (len32) { + if (dd->pdata->poll_irq(dd)) + return -ETIMEDOUT; + + for (count = 0; count < min(len32, bs32); count++, offset++) { + if (!mlen) { + sg_miter_next(&mi); + mlen = mi.length; + if (!mlen) { + pr_err("sg miter failure.\n"); + return -EINVAL; + } + offset = 0; + buffer = mi.addr; + } + omap_sham_write(dd, SHA_REG_DIN(dd, count), + buffer[offset]); + mlen -= 4; + } + len32 -= min(len32, bs32); + } + + sg_miter_stop(&mi); + + return -EINPROGRESS; +} + +static void omap_sham_dma_callback(void *param) +{ + struct omap_sham_dev *dd = param; + + set_bit(FLAGS_DMA_READY, &dd->flags); + queue_work(system_bh_wq, &dd->done_task); +} + +static int omap_sham_xmit_dma(struct omap_sham_dev *dd, size_t length, + int final) +{ + struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); + struct dma_async_tx_descriptor *tx; + struct dma_slave_config cfg; + int ret; + + dev_dbg(dd->dev, "xmit_dma: digcnt: %zd, length: %zd, final: %d\n", + ctx->digcnt, length, final); + + if (!dma_map_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE)) { + dev_err(dd->dev, "dma_map_sg error\n"); + return -EINVAL; + } + + memset(&cfg, 0, sizeof(cfg)); + + cfg.dst_addr = dd->phys_base + SHA_REG_DIN(dd, 0); + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cfg.dst_maxburst = get_block_size(ctx) / DMA_SLAVE_BUSWIDTH_4_BYTES; + + ret = dmaengine_slave_config(dd->dma_lch, &cfg); + if (ret) { + pr_err("omap-sham: can't configure dmaengine slave: %d\n", ret); + return ret; + } + + tx = dmaengine_prep_slave_sg(dd->dma_lch, ctx->sg, ctx->sg_len, + DMA_MEM_TO_DEV, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + + if (!tx) { + dev_err(dd->dev, "prep_slave_sg failed\n"); + return -EINVAL; + } + + tx->callback = omap_sham_dma_callback; + tx->callback_param = dd; + + dd->pdata->write_ctrl(dd, length, final, 1); + + ctx->digcnt += length; + ctx->total -= length; + + if (final) + set_bit(FLAGS_FINAL, &dd->flags); /* catch last interrupt */ + + set_bit(FLAGS_DMA_ACTIVE, &dd->flags); + + dmaengine_submit(tx); + dma_async_issue_pending(dd->dma_lch); + + dd->pdata->trigger(dd, length); + + return -EINPROGRESS; +} + +static int omap_sham_copy_sg_lists(struct omap_sham_reqctx *ctx, + struct scatterlist *sg, int bs, int new_len) +{ + int n = sg_nents(sg); + struct scatterlist *tmp; + int offset = ctx->offset; + + ctx->total = new_len; + + if (ctx->bufcnt) + n++; + + ctx->sg = kmalloc_objs(*sg, n); + if (!ctx->sg) + return -ENOMEM; + + sg_init_table(ctx->sg, n); + + tmp = ctx->sg; + + ctx->sg_len = 0; + + if (ctx->bufcnt) { + sg_set_buf(tmp, ctx->dd->xmit_buf, ctx->bufcnt); + tmp = sg_next(tmp); + ctx->sg_len++; + new_len -= ctx->bufcnt; + } + + while (sg && new_len) { + int len = sg->length - offset; + + if (len <= 0) { + offset -= sg->length; + sg = sg_next(sg); + continue; + } + + if (new_len < len) + len = new_len; + + if (len > 0) { + new_len -= len; + sg_set_page(tmp, sg_page(sg), len, sg->offset + offset); + offset = 0; + ctx->offset = 0; + ctx->sg_len++; + if (new_len <= 0) + break; + tmp = sg_next(tmp); + } + + sg = sg_next(sg); + } + + if (tmp) + sg_mark_end(tmp); + + set_bit(FLAGS_SGS_ALLOCED, &ctx->dd->flags); + + ctx->offset += new_len - ctx->bufcnt; + ctx->bufcnt = 0; + + return 0; +} + +static int omap_sham_copy_sgs(struct omap_sham_reqctx *ctx, + struct scatterlist *sg, int bs, + unsigned int new_len) +{ + int pages; + void *buf; + + pages = get_order(new_len); + + buf = (void *)__get_free_pages(GFP_ATOMIC, pages); + if (!buf) { + pr_err("Couldn't allocate pages for unaligned cases.\n"); + return -ENOMEM; + } + + if (ctx->bufcnt) + memcpy(buf, ctx->dd->xmit_buf, ctx->bufcnt); + + scatterwalk_map_and_copy(buf + ctx->bufcnt, sg, ctx->offset, + min(new_len, ctx->total) - ctx->bufcnt, 0); + sg_init_table(ctx->sgl, 1); + sg_set_buf(ctx->sgl, buf, new_len); + ctx->sg = ctx->sgl; + set_bit(FLAGS_SGS_COPIED, &ctx->dd->flags); + ctx->sg_len = 1; + ctx->offset += new_len - ctx->bufcnt; + ctx->bufcnt = 0; + ctx->total = new_len; + + return 0; +} + +static int omap_sham_align_sgs(struct scatterlist *sg, + int nbytes, int bs, bool final, + struct omap_sham_reqctx *rctx) +{ + int n = 0; + bool aligned = true; + bool list_ok = true; + struct scatterlist *sg_tmp = sg; + int new_len; + int offset = rctx->offset; + int bufcnt = rctx->bufcnt; + + if (!sg || !sg->length || !nbytes) { + if (bufcnt) { + bufcnt = DIV_ROUND_UP(bufcnt, bs) * bs; + sg_init_table(rctx->sgl, 1); + sg_set_buf(rctx->sgl, rctx->dd->xmit_buf, bufcnt); + rctx->sg = rctx->sgl; + rctx->sg_len = 1; + } + + return 0; + } + + new_len = nbytes; + + if (offset) + list_ok = false; + + if (final) + new_len = DIV_ROUND_UP(new_len, bs) * bs; + else + new_len = (new_len - 1) / bs * bs; + + if (!new_len) + return 0; + + if (nbytes != new_len) + list_ok = false; + + while (nbytes > 0 && sg_tmp) { + n++; + + if (bufcnt) { + if (!IS_ALIGNED(bufcnt, bs)) { + aligned = false; + break; + } + nbytes -= bufcnt; + bufcnt = 0; + if (!nbytes) + list_ok = false; + + continue; + } + +#ifdef CONFIG_ZONE_DMA + if (page_zonenum(sg_page(sg_tmp)) != ZONE_DMA) { + aligned = false; + break; + } +#endif + + if (offset < sg_tmp->length) { + if (!IS_ALIGNED(offset + sg_tmp->offset, 4)) { + aligned = false; + break; + } + + if (!IS_ALIGNED(sg_tmp->length - offset, bs)) { + aligned = false; + break; + } + } + + if (offset) { + offset -= sg_tmp->length; + if (offset < 0) { + nbytes += offset; + offset = 0; + } + } else { + nbytes -= sg_tmp->length; + } + + sg_tmp = sg_next(sg_tmp); + + if (nbytes < 0) { + list_ok = false; + break; + } + } + + if (new_len > OMAP_SHA_MAX_DMA_LEN) { + new_len = OMAP_SHA_MAX_DMA_LEN; + aligned = false; + } + + if (!aligned) + return omap_sham_copy_sgs(rctx, sg, bs, new_len); + else if (!list_ok) + return omap_sham_copy_sg_lists(rctx, sg, bs, new_len); + + rctx->total = new_len; + rctx->offset += new_len; + rctx->sg_len = n; + if (rctx->bufcnt) { + sg_init_table(rctx->sgl, 2); + sg_set_buf(rctx->sgl, rctx->dd->xmit_buf, rctx->bufcnt); + sg_chain(rctx->sgl, 2, sg); + rctx->sg = rctx->sgl; + } else { + rctx->sg = sg; + } + + return 0; +} + +static int omap_sham_prepare_request(struct crypto_engine *engine, void *areq) +{ + struct ahash_request *req = container_of(areq, struct ahash_request, + base); + struct omap_sham_reqctx *rctx = ahash_request_ctx(req); + int bs; + int ret; + unsigned int nbytes; + bool final = rctx->flags & BIT(FLAGS_FINUP); + bool update = rctx->op == OP_UPDATE; + int hash_later; + + bs = get_block_size(rctx); + + nbytes = rctx->bufcnt; + + if (update) + nbytes += req->nbytes - rctx->offset; + + dev_dbg(rctx->dd->dev, + "%s: nbytes=%d, bs=%d, total=%d, offset=%d, bufcnt=%zd\n", + __func__, nbytes, bs, rctx->total, rctx->offset, + rctx->bufcnt); + + if (!nbytes) + return 0; + + rctx->total = nbytes; + + if (update && req->nbytes && (!IS_ALIGNED(rctx->bufcnt, bs))) { + int len = bs - rctx->bufcnt % bs; + + if (len > req->nbytes) + len = req->nbytes; + scatterwalk_map_and_copy(rctx->buffer + rctx->bufcnt, req->src, + 0, len, 0); + rctx->bufcnt += len; + rctx->offset = len; + } + + if (rctx->bufcnt) + memcpy(rctx->dd->xmit_buf, rctx->buffer, rctx->bufcnt); + + ret = omap_sham_align_sgs(req->src, nbytes, bs, final, rctx); + if (ret) + return ret; + + hash_later = nbytes - rctx->total; + if (hash_later < 0) + hash_later = 0; + + if (hash_later && hash_later <= sizeof(rctx->buffer)) { + scatterwalk_map_and_copy(rctx->buffer, + req->src, + req->nbytes - hash_later, + hash_later, 0); + + rctx->bufcnt = hash_later; + } else { + rctx->bufcnt = 0; + } + + if (hash_later > sizeof(rctx->buffer)) + set_bit(FLAGS_HUGE, &rctx->dd->flags); + + rctx->total = min(nbytes, rctx->total); + + return 0; +} + +static int omap_sham_update_dma_stop(struct omap_sham_dev *dd) +{ + struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); + + dma_unmap_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE); + + clear_bit(FLAGS_DMA_ACTIVE, &dd->flags); + + return 0; +} + +static struct omap_sham_dev *omap_sham_find_dev(struct omap_sham_reqctx *ctx) +{ + struct omap_sham_dev *dd; + + if (ctx->dd) + return ctx->dd; + + spin_lock_bh(&sham.lock); + dd = list_first_entry(&sham.dev_list, struct omap_sham_dev, list); + list_move_tail(&dd->list, &sham.dev_list); + ctx->dd = dd; + spin_unlock_bh(&sham.lock); + + return dd; +} + +static int omap_sham_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct omap_sham_reqctx *ctx = ahash_request_ctx(req); + struct omap_sham_dev *dd; + int bs = 0; + + ctx->dd = NULL; + + dd = omap_sham_find_dev(ctx); + if (!dd) + return -ENODEV; + + ctx->flags = 0; + + dev_dbg(dd->dev, "init: digest size: %d\n", + crypto_ahash_digestsize(tfm)); + + switch (crypto_ahash_digestsize(tfm)) { + case MD5_DIGEST_SIZE: + ctx->flags |= FLAGS_MODE_MD5; + bs = SHA1_BLOCK_SIZE; + break; + case SHA1_DIGEST_SIZE: + ctx->flags |= FLAGS_MODE_SHA1; + bs = SHA1_BLOCK_SIZE; + break; + case SHA224_DIGEST_SIZE: + ctx->flags |= FLAGS_MODE_SHA224; + bs = SHA224_BLOCK_SIZE; + break; + case SHA256_DIGEST_SIZE: + ctx->flags |= FLAGS_MODE_SHA256; + bs = SHA256_BLOCK_SIZE; + break; + case SHA384_DIGEST_SIZE: + ctx->flags |= FLAGS_MODE_SHA384; + bs = SHA384_BLOCK_SIZE; + break; + case SHA512_DIGEST_SIZE: + ctx->flags |= FLAGS_MODE_SHA512; + bs = SHA512_BLOCK_SIZE; + break; + } + + ctx->bufcnt = 0; + ctx->digcnt = 0; + ctx->total = 0; + ctx->offset = 0; + + if (tctx->flags & BIT(FLAGS_HMAC)) { + if (!test_bit(FLAGS_AUTO_XOR, &dd->flags)) { + struct omap_sham_hmac_ctx *bctx = tctx->base; + + memcpy(ctx->buffer, bctx->ipad, bs); + ctx->bufcnt = bs; + } + + ctx->flags |= BIT(FLAGS_HMAC); + } + + return 0; + +} + +static int omap_sham_update_req(struct omap_sham_dev *dd) +{ + struct ahash_request *req = dd->req; + struct omap_sham_reqctx *ctx = ahash_request_ctx(req); + int err; + bool final = (ctx->flags & BIT(FLAGS_FINUP)) && + !(dd->flags & BIT(FLAGS_HUGE)); + + dev_dbg(dd->dev, "update_req: total: %u, digcnt: %zd, final: %d", + ctx->total, ctx->digcnt, final); + + if (ctx->total < get_block_size(ctx) || + ctx->total < dd->fallback_sz) + ctx->flags |= BIT(FLAGS_CPU); + + if (ctx->flags & BIT(FLAGS_CPU)) + err = omap_sham_xmit_cpu(dd, ctx->total, final); + else + err = omap_sham_xmit_dma(dd, ctx->total, final); + + /* wait for dma completion before can take more data */ + dev_dbg(dd->dev, "update: err: %d, digcnt: %zd\n", err, ctx->digcnt); + + return err; +} + +static int omap_sham_final_req(struct omap_sham_dev *dd) +{ + struct ahash_request *req = dd->req; + struct omap_sham_reqctx *ctx = ahash_request_ctx(req); + int err = 0, use_dma = 1; + + if (dd->flags & BIT(FLAGS_HUGE)) + return 0; + + if ((ctx->total <= get_block_size(ctx)) || dd->polling_mode) + /* + * faster to handle last block with cpu or + * use cpu when dma is not present. + */ + use_dma = 0; + + if (use_dma) + err = omap_sham_xmit_dma(dd, ctx->total, 1); + else + err = omap_sham_xmit_cpu(dd, ctx->total, 1); + + ctx->bufcnt = 0; + + dev_dbg(dd->dev, "final_req: err: %d\n", err); + + return err; +} + +static int omap_sham_hash_one_req(struct crypto_engine *engine, void *areq) +{ + struct ahash_request *req = container_of(areq, struct ahash_request, + base); + struct omap_sham_reqctx *ctx = ahash_request_ctx(req); + struct omap_sham_dev *dd = ctx->dd; + int err; + bool final = (ctx->flags & BIT(FLAGS_FINUP)) && + !(dd->flags & BIT(FLAGS_HUGE)); + + dev_dbg(dd->dev, "hash-one: op: %u, total: %u, digcnt: %zd, final: %d", + ctx->op, ctx->total, ctx->digcnt, final); + + err = omap_sham_prepare_request(engine, areq); + if (err) + return err; + + err = pm_runtime_resume_and_get(dd->dev); + if (err < 0) { + dev_err(dd->dev, "failed to get sync: %d\n", err); + return err; + } + + dd->err = 0; + dd->req = req; + + if (ctx->digcnt) + dd->pdata->copy_hash(req, 0); + + if (ctx->op == OP_UPDATE) + err = omap_sham_update_req(dd); + else if (ctx->op == OP_FINAL) + err = omap_sham_final_req(dd); + + if (err != -EINPROGRESS) + omap_sham_finish_req(req, err); + + return 0; +} + +static int omap_sham_finish_hmac(struct ahash_request *req) +{ + struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm); + struct omap_sham_hmac_ctx *bctx = tctx->base; + int bs = crypto_shash_blocksize(bctx->shash); + int ds = crypto_shash_digestsize(bctx->shash); + SHASH_DESC_ON_STACK(shash, bctx->shash); + + shash->tfm = bctx->shash; + + return crypto_shash_init(shash) ?: + crypto_shash_update(shash, bctx->opad, bs) ?: + crypto_shash_finup(shash, req->result, ds, req->result); +} + +static int omap_sham_finish(struct ahash_request *req) +{ + struct omap_sham_reqctx *ctx = ahash_request_ctx(req); + struct omap_sham_dev *dd = ctx->dd; + int err = 0; + + if (ctx->digcnt) { + omap_sham_copy_ready_hash(req); + if ((ctx->flags & BIT(FLAGS_HMAC)) && + !test_bit(FLAGS_AUTO_XOR, &dd->flags)) + err = omap_sham_finish_hmac(req); + } + + dev_dbg(dd->dev, "digcnt: %zd, bufcnt: %zd\n", ctx->digcnt, ctx->bufcnt); + + return err; +} + +static void omap_sham_finish_req(struct ahash_request *req, int err) +{ + struct omap_sham_reqctx *ctx = ahash_request_ctx(req); + struct omap_sham_dev *dd = ctx->dd; + + if (test_bit(FLAGS_SGS_COPIED, &dd->flags)) + free_pages((unsigned long)sg_virt(ctx->sg), + get_order(ctx->sg->length)); + + if (test_bit(FLAGS_SGS_ALLOCED, &dd->flags)) + kfree(ctx->sg); + + ctx->sg = NULL; + + dd->flags &= ~(BIT(FLAGS_SGS_ALLOCED) | BIT(FLAGS_SGS_COPIED) | + BIT(FLAGS_CPU) | BIT(FLAGS_DMA_READY) | + BIT(FLAGS_OUTPUT_READY)); + + if (!err) + dd->pdata->copy_hash(req, 1); + + if (dd->flags & BIT(FLAGS_HUGE)) { + /* Re-enqueue the request */ + omap_sham_enqueue(req, ctx->op); + return; + } + + if (!err) { + if (test_bit(FLAGS_FINAL, &dd->flags)) + err = omap_sham_finish(req); + } else { + ctx->flags |= BIT(FLAGS_ERROR); + } + + /* atomic operation is not needed here */ + dd->flags &= ~(BIT(FLAGS_FINAL) | BIT(FLAGS_CPU) | + BIT(FLAGS_DMA_READY) | BIT(FLAGS_OUTPUT_READY)); + + pm_runtime_put_autosuspend(dd->dev); + + ctx->offset = 0; + + crypto_finalize_hash_request(dd->engine, req, err); +} + +static int omap_sham_handle_queue(struct omap_sham_dev *dd, + struct ahash_request *req) +{ + return crypto_transfer_hash_request_to_engine(dd->engine, req); +} + +static int omap_sham_enqueue(struct ahash_request *req, unsigned int op) +{ + struct omap_sham_reqctx *ctx = ahash_request_ctx(req); + struct omap_sham_dev *dd = ctx->dd; + + ctx->op = op; + + return omap_sham_handle_queue(dd, req); +} + +static int omap_sham_update(struct ahash_request *req) +{ + struct omap_sham_reqctx *ctx = ahash_request_ctx(req); + struct omap_sham_dev *dd = omap_sham_find_dev(ctx); + + if (!req->nbytes) + return 0; + + if (ctx->bufcnt + req->nbytes <= sizeof(ctx->buffer)) { + scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, req->src, + 0, req->nbytes, 0); + ctx->bufcnt += req->nbytes; + return 0; + } + + if (dd->polling_mode) + ctx->flags |= BIT(FLAGS_CPU); + + return omap_sham_enqueue(req, OP_UPDATE); +} + +static int omap_sham_final_shash(struct ahash_request *req) +{ + struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm); + struct omap_sham_reqctx *ctx = ahash_request_ctx(req); + int offset = 0; + + /* + * If we are running HMAC on limited hardware support, skip + * the ipad in the beginning of the buffer if we are going for + * software fallback algorithm. + */ + if (test_bit(FLAGS_HMAC, &ctx->flags) && + !test_bit(FLAGS_AUTO_XOR, &ctx->dd->flags)) + offset = get_block_size(ctx); + + return crypto_shash_tfm_digest(tctx->fallback, ctx->buffer + offset, + ctx->bufcnt - offset, req->result); +} + +static int omap_sham_final(struct ahash_request *req) +{ + struct omap_sham_reqctx *ctx = ahash_request_ctx(req); + + ctx->flags |= BIT(FLAGS_FINUP); + + if (ctx->flags & BIT(FLAGS_ERROR)) + return 0; /* uncompleted hash is not needed */ + + /* + * OMAP HW accel works only with buffers >= 9. + * HMAC is always >= 9 because ipad == block size. + * If buffersize is less than fallback_sz, we use fallback + * SW encoding, as using DMA + HW in this case doesn't provide + * any benefit. + */ + if (!ctx->digcnt && ctx->bufcnt < ctx->dd->fallback_sz) + return omap_sham_final_shash(req); + else if (ctx->bufcnt) + return omap_sham_enqueue(req, OP_FINAL); + + /* copy ready hash (+ finalize hmac) */ + return omap_sham_finish(req); +} + +static int omap_sham_finup(struct ahash_request *req) +{ + struct omap_sham_reqctx *ctx = ahash_request_ctx(req); + int err1, err2; + + ctx->flags |= BIT(FLAGS_FINUP); + + err1 = omap_sham_update(req); + if (err1 == -EINPROGRESS || err1 == -EBUSY) + return err1; + /* + * final() has to be always called to cleanup resources + * even if udpate() failed, except EINPROGRESS + */ + err2 = omap_sham_final(req); + + return err1 ?: err2; +} + +static int omap_sham_digest(struct ahash_request *req) +{ + return omap_sham_init(req) ?: omap_sham_finup(req); +} + +static int omap_sham_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct omap_sham_hmac_ctx *bctx = tctx->base; + int bs = crypto_shash_blocksize(bctx->shash); + int ds = crypto_shash_digestsize(bctx->shash); + int err, i; + + err = crypto_shash_setkey(tctx->fallback, key, keylen); + if (err) + return err; + + if (keylen > bs) { + err = crypto_shash_tfm_digest(bctx->shash, key, keylen, + bctx->ipad); + if (err) + return err; + keylen = ds; + } else { + memcpy(bctx->ipad, key, keylen); + } + + memset(bctx->ipad + keylen, 0, bs - keylen); + + if (!test_bit(FLAGS_AUTO_XOR, &sham.flags)) { + memcpy(bctx->opad, bctx->ipad, bs); + + for (i = 0; i < bs; i++) { + bctx->ipad[i] ^= HMAC_IPAD_VALUE; + bctx->opad[i] ^= HMAC_OPAD_VALUE; + } + } + + return err; +} + +static int omap_sham_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) +{ + struct omap_sham_ctx *tctx = crypto_tfm_ctx(tfm); + const char *alg_name = crypto_tfm_alg_name(tfm); + + /* Allocate a fallback and abort if it failed. */ + tctx->fallback = crypto_alloc_shash(alg_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(tctx->fallback)) { + pr_err("omap-sham: fallback driver '%s' " + "could not be loaded.\n", alg_name); + return PTR_ERR(tctx->fallback); + } + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct omap_sham_reqctx)); + + if (alg_base) { + struct omap_sham_hmac_ctx *bctx = tctx->base; + tctx->flags |= BIT(FLAGS_HMAC); + bctx->shash = crypto_alloc_shash(alg_base, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(bctx->shash)) { + pr_err("omap-sham: base driver '%s' " + "could not be loaded.\n", alg_base); + crypto_free_shash(tctx->fallback); + return PTR_ERR(bctx->shash); + } + + } + + return 0; +} + +static int omap_sham_cra_init(struct crypto_tfm *tfm) +{ + return omap_sham_cra_init_alg(tfm, NULL); +} + +static int omap_sham_cra_sha1_init(struct crypto_tfm *tfm) +{ + return omap_sham_cra_init_alg(tfm, "sha1"); +} + +static int omap_sham_cra_sha224_init(struct crypto_tfm *tfm) +{ + return omap_sham_cra_init_alg(tfm, "sha224"); +} + +static int omap_sham_cra_sha256_init(struct crypto_tfm *tfm) +{ + return omap_sham_cra_init_alg(tfm, "sha256"); +} + +static int omap_sham_cra_md5_init(struct crypto_tfm *tfm) +{ + return omap_sham_cra_init_alg(tfm, "md5"); +} + +static int omap_sham_cra_sha384_init(struct crypto_tfm *tfm) +{ + return omap_sham_cra_init_alg(tfm, "sha384"); +} + +static int omap_sham_cra_sha512_init(struct crypto_tfm *tfm) +{ + return omap_sham_cra_init_alg(tfm, "sha512"); +} + +static void omap_sham_cra_exit(struct crypto_tfm *tfm) +{ + struct omap_sham_ctx *tctx = crypto_tfm_ctx(tfm); + + crypto_free_shash(tctx->fallback); + tctx->fallback = NULL; + + if (tctx->flags & BIT(FLAGS_HMAC)) { + struct omap_sham_hmac_ctx *bctx = tctx->base; + crypto_free_shash(bctx->shash); + } +} + +static int omap_sham_export(struct ahash_request *req, void *out) +{ + struct omap_sham_reqctx *rctx = ahash_request_ctx(req); + + memcpy(out, rctx, offsetof(struct omap_sham_reqctx, buffer) + + rctx->bufcnt); + + return 0; +} + +static int omap_sham_import(struct ahash_request *req, const void *in) +{ + struct omap_sham_reqctx *rctx = ahash_request_ctx(req); + const struct omap_sham_reqctx *ctx_in = in; + + memcpy(rctx, in, offsetof(struct omap_sham_reqctx, buffer) + + ctx_in->bufcnt); + + return 0; +} + +static struct ahash_engine_alg algs_sha1_md5[] = { +{ + .base.init = omap_sham_init, + .base.update = omap_sham_update, + .base.final = omap_sham_final, + .base.finup = omap_sham_finup, + .base.digest = omap_sham_digest, + .base.halg.digestsize = SHA1_DIGEST_SIZE, + .base.halg.base = { + .cra_name = "sha1", + .cra_driver_name = "omap-sha1", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct omap_sham_ctx), + .cra_module = THIS_MODULE, + .cra_init = omap_sham_cra_init, + .cra_exit = omap_sham_cra_exit, + }, + .op.do_one_request = omap_sham_hash_one_req, +}, +{ + .base.init = omap_sham_init, + .base.update = omap_sham_update, + .base.final = omap_sham_final, + .base.finup = omap_sham_finup, + .base.digest = omap_sham_digest, + .base.halg.digestsize = MD5_DIGEST_SIZE, + .base.halg.base = { + .cra_name = "md5", + .cra_driver_name = "omap-md5", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct omap_sham_ctx), + .cra_module = THIS_MODULE, + .cra_init = omap_sham_cra_init, + .cra_exit = omap_sham_cra_exit, + }, + .op.do_one_request = omap_sham_hash_one_req, +}, +{ + .base.init = omap_sham_init, + .base.update = omap_sham_update, + .base.final = omap_sham_final, + .base.finup = omap_sham_finup, + .base.digest = omap_sham_digest, + .base.setkey = omap_sham_setkey, + .base.halg.digestsize = SHA1_DIGEST_SIZE, + .base.halg.base = { + .cra_name = "hmac(sha1)", + .cra_driver_name = "omap-hmac-sha1", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct omap_sham_ctx) + + sizeof(struct omap_sham_hmac_ctx), + .cra_module = THIS_MODULE, + .cra_init = omap_sham_cra_sha1_init, + .cra_exit = omap_sham_cra_exit, + }, + .op.do_one_request = omap_sham_hash_one_req, +}, +{ + .base.init = omap_sham_init, + .base.update = omap_sham_update, + .base.final = omap_sham_final, + .base.finup = omap_sham_finup, + .base.digest = omap_sham_digest, + .base.setkey = omap_sham_setkey, + .base.halg.digestsize = MD5_DIGEST_SIZE, + .base.halg.base = { + .cra_name = "hmac(md5)", + .cra_driver_name = "omap-hmac-md5", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct omap_sham_ctx) + + sizeof(struct omap_sham_hmac_ctx), + .cra_module = THIS_MODULE, + .cra_init = omap_sham_cra_md5_init, + .cra_exit = omap_sham_cra_exit, + }, + .op.do_one_request = omap_sham_hash_one_req, +} +}; + +/* OMAP4 has some algs in addition to what OMAP2 has */ +static struct ahash_engine_alg algs_sha224_sha256[] = { +{ + .base.init = omap_sham_init, + .base.update = omap_sham_update, + .base.final = omap_sham_final, + .base.finup = omap_sham_finup, + .base.digest = omap_sham_digest, + .base.halg.digestsize = SHA224_DIGEST_SIZE, + .base.halg.base = { + .cra_name = "sha224", + .cra_driver_name = "omap-sha224", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct omap_sham_ctx), + .cra_module = THIS_MODULE, + .cra_init = omap_sham_cra_init, + .cra_exit = omap_sham_cra_exit, + }, + .op.do_one_request = omap_sham_hash_one_req, +}, +{ + .base.init = omap_sham_init, + .base.update = omap_sham_update, + .base.final = omap_sham_final, + .base.finup = omap_sham_finup, + .base.digest = omap_sham_digest, + .base.halg.digestsize = SHA256_DIGEST_SIZE, + .base.halg.base = { + .cra_name = "sha256", + .cra_driver_name = "omap-sha256", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct omap_sham_ctx), + .cra_module = THIS_MODULE, + .cra_init = omap_sham_cra_init, + .cra_exit = omap_sham_cra_exit, + }, + .op.do_one_request = omap_sham_hash_one_req, +}, +{ + .base.init = omap_sham_init, + .base.update = omap_sham_update, + .base.final = omap_sham_final, + .base.finup = omap_sham_finup, + .base.digest = omap_sham_digest, + .base.setkey = omap_sham_setkey, + .base.halg.digestsize = SHA224_DIGEST_SIZE, + .base.halg.base = { + .cra_name = "hmac(sha224)", + .cra_driver_name = "omap-hmac-sha224", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct omap_sham_ctx) + + sizeof(struct omap_sham_hmac_ctx), + .cra_module = THIS_MODULE, + .cra_init = omap_sham_cra_sha224_init, + .cra_exit = omap_sham_cra_exit, + }, + .op.do_one_request = omap_sham_hash_one_req, +}, +{ + .base.init = omap_sham_init, + .base.update = omap_sham_update, + .base.final = omap_sham_final, + .base.finup = omap_sham_finup, + .base.digest = omap_sham_digest, + .base.setkey = omap_sham_setkey, + .base.halg.digestsize = SHA256_DIGEST_SIZE, + .base.halg.base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "omap-hmac-sha256", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct omap_sham_ctx) + + sizeof(struct omap_sham_hmac_ctx), + .cra_module = THIS_MODULE, + .cra_init = omap_sham_cra_sha256_init, + .cra_exit = omap_sham_cra_exit, + }, + .op.do_one_request = omap_sham_hash_one_req, +}, +}; + +static struct ahash_engine_alg algs_sha384_sha512[] = { +{ + .base.init = omap_sham_init, + .base.update = omap_sham_update, + .base.final = omap_sham_final, + .base.finup = omap_sham_finup, + .base.digest = omap_sham_digest, + .base.halg.digestsize = SHA384_DIGEST_SIZE, + .base.halg.base = { + .cra_name = "sha384", + .cra_driver_name = "omap-sha384", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct omap_sham_ctx), + .cra_module = THIS_MODULE, + .cra_init = omap_sham_cra_init, + .cra_exit = omap_sham_cra_exit, + }, + .op.do_one_request = omap_sham_hash_one_req, +}, +{ + .base.init = omap_sham_init, + .base.update = omap_sham_update, + .base.final = omap_sham_final, + .base.finup = omap_sham_finup, + .base.digest = omap_sham_digest, + .base.halg.digestsize = SHA512_DIGEST_SIZE, + .base.halg.base = { + .cra_name = "sha512", + .cra_driver_name = "omap-sha512", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct omap_sham_ctx), + .cra_module = THIS_MODULE, + .cra_init = omap_sham_cra_init, + .cra_exit = omap_sham_cra_exit, + }, + .op.do_one_request = omap_sham_hash_one_req, +}, +{ + .base.init = omap_sham_init, + .base.update = omap_sham_update, + .base.final = omap_sham_final, + .base.finup = omap_sham_finup, + .base.digest = omap_sham_digest, + .base.setkey = omap_sham_setkey, + .base.halg.digestsize = SHA384_DIGEST_SIZE, + .base.halg.base = { + .cra_name = "hmac(sha384)", + .cra_driver_name = "omap-hmac-sha384", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct omap_sham_ctx) + + sizeof(struct omap_sham_hmac_ctx), + .cra_module = THIS_MODULE, + .cra_init = omap_sham_cra_sha384_init, + .cra_exit = omap_sham_cra_exit, + }, + .op.do_one_request = omap_sham_hash_one_req, +}, +{ + .base.init = omap_sham_init, + .base.update = omap_sham_update, + .base.final = omap_sham_final, + .base.finup = omap_sham_finup, + .base.digest = omap_sham_digest, + .base.setkey = omap_sham_setkey, + .base.halg.digestsize = SHA512_DIGEST_SIZE, + .base.halg.base = { + .cra_name = "hmac(sha512)", + .cra_driver_name = "omap-hmac-sha512", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct omap_sham_ctx) + + sizeof(struct omap_sham_hmac_ctx), + .cra_module = THIS_MODULE, + .cra_init = omap_sham_cra_sha512_init, + .cra_exit = omap_sham_cra_exit, + }, + .op.do_one_request = omap_sham_hash_one_req, +}, +}; + +static void omap_sham_done_task(struct work_struct *t) +{ + struct omap_sham_dev *dd = from_work(dd, t, done_task); + int err = 0; + + dev_dbg(dd->dev, "%s: flags=%lx\n", __func__, dd->flags); + + if (test_bit(FLAGS_CPU, &dd->flags)) { + if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags)) + goto finish; + } else if (test_bit(FLAGS_DMA_READY, &dd->flags)) { + if (test_bit(FLAGS_DMA_ACTIVE, &dd->flags)) { + omap_sham_update_dma_stop(dd); + if (dd->err) { + err = dd->err; + goto finish; + } + } + if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags)) { + /* hash or semi-hash ready */ + clear_bit(FLAGS_DMA_READY, &dd->flags); + goto finish; + } + } + + return; + +finish: + dev_dbg(dd->dev, "update done: err: %d\n", err); + /* finish curent request */ + omap_sham_finish_req(dd->req, err); +} + +static irqreturn_t omap_sham_irq_common(struct omap_sham_dev *dd) +{ + set_bit(FLAGS_OUTPUT_READY, &dd->flags); + queue_work(system_bh_wq, &dd->done_task); + + return IRQ_HANDLED; +} + +static irqreturn_t omap_sham_irq_omap2(int irq, void *dev_id) +{ + struct omap_sham_dev *dd = dev_id; + + if (unlikely(test_bit(FLAGS_FINAL, &dd->flags))) + /* final -> allow device to go to power-saving mode */ + omap_sham_write_mask(dd, SHA_REG_CTRL, 0, SHA_REG_CTRL_LENGTH); + + omap_sham_write_mask(dd, SHA_REG_CTRL, SHA_REG_CTRL_OUTPUT_READY, + SHA_REG_CTRL_OUTPUT_READY); + omap_sham_read(dd, SHA_REG_CTRL); + + return omap_sham_irq_common(dd); +} + +static irqreturn_t omap_sham_irq_omap4(int irq, void *dev_id) +{ + struct omap_sham_dev *dd = dev_id; + + omap_sham_write_mask(dd, SHA_REG_MASK(dd), 0, SHA_REG_MASK_IT_EN); + + return omap_sham_irq_common(dd); +} + +static struct omap_sham_algs_info omap_sham_algs_info_omap2[] = { + { + .algs_list = algs_sha1_md5, + .size = ARRAY_SIZE(algs_sha1_md5), + }, +}; + +static const struct omap_sham_pdata omap_sham_pdata_omap2 = { + .algs_info = omap_sham_algs_info_omap2, + .algs_info_size = ARRAY_SIZE(omap_sham_algs_info_omap2), + .flags = BIT(FLAGS_BE32_SHA1), + .digest_size = SHA1_DIGEST_SIZE, + .copy_hash = omap_sham_copy_hash_omap2, + .write_ctrl = omap_sham_write_ctrl_omap2, + .trigger = omap_sham_trigger_omap2, + .poll_irq = omap_sham_poll_irq_omap2, + .intr_hdlr = omap_sham_irq_omap2, + .idigest_ofs = 0x00, + .din_ofs = 0x1c, + .digcnt_ofs = 0x14, + .rev_ofs = 0x5c, + .mask_ofs = 0x60, + .sysstatus_ofs = 0x64, + .major_mask = 0xf0, + .major_shift = 4, + .minor_mask = 0x0f, + .minor_shift = 0, +}; + +#ifdef CONFIG_OF +static struct omap_sham_algs_info omap_sham_algs_info_omap4[] = { + { + .algs_list = algs_sha1_md5, + .size = ARRAY_SIZE(algs_sha1_md5), + }, + { + .algs_list = algs_sha224_sha256, + .size = ARRAY_SIZE(algs_sha224_sha256), + }, +}; + +static const struct omap_sham_pdata omap_sham_pdata_omap4 = { + .algs_info = omap_sham_algs_info_omap4, + .algs_info_size = ARRAY_SIZE(omap_sham_algs_info_omap4), + .flags = BIT(FLAGS_AUTO_XOR), + .digest_size = SHA256_DIGEST_SIZE, + .copy_hash = omap_sham_copy_hash_omap4, + .write_ctrl = omap_sham_write_ctrl_omap4, + .trigger = omap_sham_trigger_omap4, + .poll_irq = omap_sham_poll_irq_omap4, + .intr_hdlr = omap_sham_irq_omap4, + .idigest_ofs = 0x020, + .odigest_ofs = 0x0, + .din_ofs = 0x080, + .digcnt_ofs = 0x040, + .rev_ofs = 0x100, + .mask_ofs = 0x110, + .sysstatus_ofs = 0x114, + .mode_ofs = 0x44, + .length_ofs = 0x48, + .major_mask = 0x0700, + .major_shift = 8, + .minor_mask = 0x003f, + .minor_shift = 0, +}; + +static struct omap_sham_algs_info omap_sham_algs_info_omap5[] = { + { + .algs_list = algs_sha1_md5, + .size = ARRAY_SIZE(algs_sha1_md5), + }, + { + .algs_list = algs_sha224_sha256, + .size = ARRAY_SIZE(algs_sha224_sha256), + }, + { + .algs_list = algs_sha384_sha512, + .size = ARRAY_SIZE(algs_sha384_sha512), + }, +}; + +static const struct omap_sham_pdata omap_sham_pdata_omap5 = { + .algs_info = omap_sham_algs_info_omap5, + .algs_info_size = ARRAY_SIZE(omap_sham_algs_info_omap5), + .flags = BIT(FLAGS_AUTO_XOR), + .digest_size = SHA512_DIGEST_SIZE, + .copy_hash = omap_sham_copy_hash_omap4, + .write_ctrl = omap_sham_write_ctrl_omap4, + .trigger = omap_sham_trigger_omap4, + .poll_irq = omap_sham_poll_irq_omap4, + .intr_hdlr = omap_sham_irq_omap4, + .idigest_ofs = 0x240, + .odigest_ofs = 0x200, + .din_ofs = 0x080, + .digcnt_ofs = 0x280, + .rev_ofs = 0x100, + .mask_ofs = 0x110, + .sysstatus_ofs = 0x114, + .mode_ofs = 0x284, + .length_ofs = 0x288, + .major_mask = 0x0700, + .major_shift = 8, + .minor_mask = 0x003f, + .minor_shift = 0, +}; + +static const struct of_device_id omap_sham_of_match[] = { + { + .compatible = "ti,omap2-sham", + .data = &omap_sham_pdata_omap2, + }, + { + .compatible = "ti,omap3-sham", + .data = &omap_sham_pdata_omap2, + }, + { + .compatible = "ti,omap4-sham", + .data = &omap_sham_pdata_omap4, + }, + { + .compatible = "ti,omap5-sham", + .data = &omap_sham_pdata_omap5, + }, + {}, +}; +MODULE_DEVICE_TABLE(of, omap_sham_of_match); +#endif + +static ssize_t fallback_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct omap_sham_dev *dd = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%d\n", dd->fallback_sz); +} + +static ssize_t fallback_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + struct omap_sham_dev *dd = dev_get_drvdata(dev); + ssize_t status; + long value; + + status = kstrtol(buf, 0, &value); + if (status) + return status; + + /* HW accelerator only works with buffers > 9 */ + if (value < 9) { + dev_err(dev, "minimum fallback size 9\n"); + return -EINVAL; + } + + dd->fallback_sz = value; + + return size; +} + +static ssize_t queue_len_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct omap_sham_dev *dd = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%d\n", dd->queue.max_qlen); +} + +static ssize_t queue_len_store(struct device *dev, + struct device_attribute *attr, const char *buf, + size_t size) +{ + struct omap_sham_dev *dd = dev_get_drvdata(dev); + ssize_t status; + long value; + + status = kstrtol(buf, 0, &value); + if (status) + return status; + + if (value < 1) + return -EINVAL; + + /* + * Changing the queue size in fly is safe, if size becomes smaller + * than current size, it will just not accept new entries until + * it has shrank enough. + */ + dd->queue.max_qlen = value; + + return size; +} + +static DEVICE_ATTR_RW(queue_len); +static DEVICE_ATTR_RW(fallback); + +static struct attribute *omap_sham_attrs[] = { + &dev_attr_queue_len.attr, + &dev_attr_fallback.attr, + NULL, +}; +ATTRIBUTE_GROUPS(omap_sham); + +static void omap_sham_unregister_algs(const struct omap_sham_pdata *pdata) +{ + struct omap_sham_algs_info *alg_info; + int i; + + for (i = pdata->algs_info_size - 1; i >= 0; i--) { + alg_info = &pdata->algs_info[i]; + + crypto_engine_unregister_ahashes(alg_info->algs_list, + alg_info->registered); + alg_info->registered = 0; + } +} + +static int omap_sham_probe(struct platform_device *pdev) +{ + struct omap_sham_dev *dd; + struct device *dev = &pdev->dev; + void __iomem *io_base; + struct resource *res; + dma_cap_mask_t mask; + int err, i, j, irq; + u32 rev; + + io_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); + if (IS_ERR(io_base)) + return PTR_ERR(io_base); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + dd = devm_kzalloc(dev, sizeof(struct omap_sham_dev), GFP_KERNEL); + if (dd == NULL) { + dev_err(dev, "unable to alloc data struct.\n"); + err = -ENOMEM; + goto data_err; + } + dd->dev = dev; + platform_set_drvdata(pdev, dd); + + INIT_LIST_HEAD(&dd->list); + INIT_WORK(&dd->done_task, omap_sham_done_task); + crypto_init_queue(&dd->queue, OMAP_SHAM_QUEUE_LENGTH); + + dd->pdata = device_get_match_data(dev); + if (!dd->pdata) + dd->pdata = &omap_sham_pdata_omap2; + + dd->irq = irq; + dd->io_base = io_base; + dd->phys_base = res->start; + + err = devm_request_irq(dev, dd->irq, dd->pdata->intr_hdlr, + IRQF_TRIGGER_NONE, dev_name(dev), dd); + if (err) + goto data_err; + + dma_cap_zero(mask); + dma_cap_set(DMA_SLAVE, mask); + + dd->dma_lch = dma_request_chan(dev, "rx"); + if (IS_ERR(dd->dma_lch)) { + err = PTR_ERR(dd->dma_lch); + if (err == -EPROBE_DEFER) + goto data_err; + + dd->polling_mode = 1; + dev_dbg(dev, "using polling mode instead of dma\n"); + } + + dd->flags |= dd->pdata->flags; + sham.flags |= dd->pdata->flags; + + pm_runtime_use_autosuspend(dev); + pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY); + + dd->fallback_sz = OMAP_SHA_DMA_THRESHOLD; + + pm_runtime_enable(dev); + + err = pm_runtime_resume_and_get(dev); + if (err < 0) { + dev_err(dev, "failed to get sync: %d\n", err); + goto err_pm; + } + + rev = omap_sham_read(dd, SHA_REG_REV(dd)); + pm_runtime_put_sync(&pdev->dev); + + dev_info(dev, "hw accel on OMAP rev %u.%u\n", + (rev & dd->pdata->major_mask) >> dd->pdata->major_shift, + (rev & dd->pdata->minor_mask) >> dd->pdata->minor_shift); + + spin_lock_bh(&sham.lock); + list_add_tail(&dd->list, &sham.dev_list); + spin_unlock_bh(&sham.lock); + + dd->engine = crypto_engine_alloc_init(dev, 1); + if (!dd->engine) { + err = -ENOMEM; + goto err_engine; + } + + err = crypto_engine_start(dd->engine); + if (err) + goto err_engine_start; + + for (i = 0; i < dd->pdata->algs_info_size; i++) { + if (dd->pdata->algs_info[i].registered) + break; + + for (j = 0; j < dd->pdata->algs_info[i].size; j++) { + struct ahash_engine_alg *ealg; + struct ahash_alg *alg; + + ealg = &dd->pdata->algs_info[i].algs_list[j]; + alg = &ealg->base; + alg->export = omap_sham_export; + alg->import = omap_sham_import; + alg->halg.statesize = sizeof(struct omap_sham_reqctx); + err = crypto_engine_register_ahash(ealg); + if (err) + goto err_algs; + + dd->pdata->algs_info[i].registered++; + } + } + + return 0; + +err_algs: + omap_sham_unregister_algs(dd->pdata); +err_engine_start: + crypto_engine_exit(dd->engine); +err_engine: + spin_lock_bh(&sham.lock); + list_del(&dd->list); + spin_unlock_bh(&sham.lock); +err_pm: + pm_runtime_dont_use_autosuspend(dev); + pm_runtime_disable(dev); + if (!dd->polling_mode) + dma_release_channel(dd->dma_lch); +data_err: + dev_err(dev, "initialization failed.\n"); + + return err; +} + +static void omap_sham_remove(struct platform_device *pdev) +{ + struct omap_sham_dev *dd; + + dd = platform_get_drvdata(pdev); + + spin_lock_bh(&sham.lock); + list_del(&dd->list); + spin_unlock_bh(&sham.lock); + omap_sham_unregister_algs(dd->pdata); + cancel_work_sync(&dd->done_task); + pm_runtime_dont_use_autosuspend(&pdev->dev); + pm_runtime_disable(&pdev->dev); + + if (!dd->polling_mode) + dma_release_channel(dd->dma_lch); +} + +static struct platform_driver omap_sham_driver = { + .probe = omap_sham_probe, + .remove = omap_sham_remove, + .driver = { + .name = "omap-sham", + .of_match_table = of_match_ptr(omap_sham_of_match), + .dev_groups = omap_sham_groups, + }, +}; + +module_platform_driver(omap_sham_driver); + +MODULE_DESCRIPTION("OMAP SHA1/MD5 hw acceleration support."); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Dmitry Kasatkin"); +MODULE_ALIAS("platform:omap-sham"); diff --git a/drivers/crypto/padlock-aes.c b/drivers/crypto/padlock-aes.c new file mode 100644 index 000000000..1be549a07 --- /dev/null +++ b/drivers/crypto/padlock-aes.c @@ -0,0 +1,539 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Cryptographic API. + * + * Support for VIA PadLock hardware crypto engine. + * + * Copyright (c) 2004 Michal Ludvig + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Number of data blocks actually fetched for each xcrypt insn. + * Processors with prefetch errata will fetch extra blocks. + */ +static unsigned int ecb_fetch_blocks = 2; +#define MAX_ECB_FETCH_BLOCKS (8) +#define ecb_fetch_bytes (ecb_fetch_blocks * AES_BLOCK_SIZE) + +static unsigned int cbc_fetch_blocks = 1; +#define MAX_CBC_FETCH_BLOCKS (4) +#define cbc_fetch_bytes (cbc_fetch_blocks * AES_BLOCK_SIZE) + +/* Control word. */ +struct cword { + unsigned int __attribute__ ((__packed__)) + rounds:4, + algo:3, + keygen:1, + interm:1, + encdec:1, + ksize:2; +} __attribute__ ((__aligned__(PADLOCK_ALIGNMENT))); + +/* Whenever making any changes to the following + * structure *make sure* you keep E, d_data + * and cword aligned on 16 Bytes boundaries and + * the Hardware can access 16 * 16 bytes of E and d_data + * (only the first 15 * 16 bytes matter but the HW reads + * more). + */ +struct aes_ctx { + u32 E[AES_MAX_KEYLENGTH_U32] + __attribute__ ((__aligned__(PADLOCK_ALIGNMENT))); + u32 d_data[AES_MAX_KEYLENGTH_U32] + __attribute__ ((__aligned__(PADLOCK_ALIGNMENT))); + struct { + struct cword encrypt; + struct cword decrypt; + } cword; + u32 *D; +}; + +static DEFINE_PER_CPU(struct cword *, paes_last_cword); + +/* Tells whether the ACE is capable to generate + the extended key for a given key_len. */ +static inline int +aes_hw_extkey_available(uint8_t key_len) +{ + /* TODO: We should check the actual CPU model/stepping + as it's possible that the capability will be + added in the next CPU revisions. */ + if (key_len == 16) + return 1; + return 0; +} + +static inline struct aes_ctx *aes_ctx_common(void *ctx) +{ + unsigned long addr = (unsigned long)ctx; + unsigned long align = PADLOCK_ALIGNMENT; + + if (align <= crypto_tfm_ctx_alignment()) + align = 1; + return (struct aes_ctx *)ALIGN(addr, align); +} + +static inline struct aes_ctx *aes_ctx(struct crypto_tfm *tfm) +{ + return aes_ctx_common(crypto_tfm_ctx(tfm)); +} + +static inline struct aes_ctx *skcipher_aes_ctx(struct crypto_skcipher *tfm) +{ + return aes_ctx_common(crypto_skcipher_ctx(tfm)); +} + +static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct aes_ctx *ctx = aes_ctx(tfm); + const __le32 *key = (const __le32 *)in_key; + struct crypto_aes_ctx gen_aes; + int cpu; + + if (key_len % 8) + return -EINVAL; + + /* + * If the hardware is capable of generating the extended key + * itself we must supply the plain key for both encryption + * and decryption. + */ + ctx->D = ctx->E; + + ctx->E[0] = le32_to_cpu(key[0]); + ctx->E[1] = le32_to_cpu(key[1]); + ctx->E[2] = le32_to_cpu(key[2]); + ctx->E[3] = le32_to_cpu(key[3]); + + /* Prepare control words. */ + memset(&ctx->cword, 0, sizeof(ctx->cword)); + + ctx->cword.decrypt.encdec = 1; + ctx->cword.encrypt.rounds = 10 + (key_len - 16) / 4; + ctx->cword.decrypt.rounds = ctx->cword.encrypt.rounds; + ctx->cword.encrypt.ksize = (key_len - 16) / 8; + ctx->cword.decrypt.ksize = ctx->cword.encrypt.ksize; + + /* Don't generate extended keys if the hardware can do it. */ + if (aes_hw_extkey_available(key_len)) + goto ok; + + ctx->D = ctx->d_data; + ctx->cword.encrypt.keygen = 1; + ctx->cword.decrypt.keygen = 1; + + if (aes_expandkey(&gen_aes, in_key, key_len)) + return -EINVAL; + + memcpy(ctx->E, gen_aes.key_enc, AES_MAX_KEYLENGTH); + memcpy(ctx->D, gen_aes.key_dec, AES_MAX_KEYLENGTH); + +ok: + for_each_online_cpu(cpu) + if (&ctx->cword.encrypt == per_cpu(paes_last_cword, cpu) || + &ctx->cword.decrypt == per_cpu(paes_last_cword, cpu)) + per_cpu(paes_last_cword, cpu) = NULL; + + return 0; +} + +static int aes_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) +{ + return aes_set_key(crypto_skcipher_tfm(tfm), in_key, key_len); +} + +/* ====== Encryption/decryption routines ====== */ + +/* These are the real call to PadLock. */ +static inline void padlock_reset_key(struct cword *cword) +{ + int cpu = raw_smp_processor_id(); + + if (cword != per_cpu(paes_last_cword, cpu)) +#ifndef CONFIG_X86_64 + asm volatile ("pushfl; popfl"); +#else + asm volatile ("pushfq; popfq"); +#endif +} + +static inline void padlock_store_cword(struct cword *cword) +{ + per_cpu(paes_last_cword, raw_smp_processor_id()) = cword; +} + +/* + * While the padlock instructions don't use FP/SSE registers, they + * generate a spurious DNA fault when CR0.TS is '1'. Fortunately, + * the kernel doesn't use CR0.TS. + */ + +static inline void rep_xcrypt_ecb(const u8 *input, u8 *output, void *key, + struct cword *control_word, int count) +{ + asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */ + : "+S"(input), "+D"(output) + : "d"(control_word), "b"(key), "c"(count)); +} + +static inline u8 *rep_xcrypt_cbc(const u8 *input, u8 *output, void *key, + u8 *iv, struct cword *control_word, int count) +{ + asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */ + : "+S" (input), "+D" (output), "+a" (iv) + : "d" (control_word), "b" (key), "c" (count)); + return iv; +} + +static void ecb_crypt_copy(const u8 *in, u8 *out, u32 *key, + struct cword *cword, int count) +{ + /* + * Padlock prefetches extra data so we must provide mapped input buffers. + * Assume there are at least 16 bytes of stack already in use. + */ + u8 buf[AES_BLOCK_SIZE * (MAX_ECB_FETCH_BLOCKS - 1) + PADLOCK_ALIGNMENT - 1]; + u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT); + + memcpy(tmp, in, count * AES_BLOCK_SIZE); + rep_xcrypt_ecb(tmp, out, key, cword, count); +} + +static u8 *cbc_crypt_copy(const u8 *in, u8 *out, u32 *key, + u8 *iv, struct cword *cword, int count) +{ + /* + * Padlock prefetches extra data so we must provide mapped input buffers. + * Assume there are at least 16 bytes of stack already in use. + */ + u8 buf[AES_BLOCK_SIZE * (MAX_CBC_FETCH_BLOCKS - 1) + PADLOCK_ALIGNMENT - 1]; + u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT); + + memcpy(tmp, in, count * AES_BLOCK_SIZE); + return rep_xcrypt_cbc(tmp, out, key, iv, cword, count); +} + +static inline void ecb_crypt(const u8 *in, u8 *out, u32 *key, + struct cword *cword, int count) +{ + /* Padlock in ECB mode fetches at least ecb_fetch_bytes of data. + * We could avoid some copying here but it's probably not worth it. + */ + if (unlikely(offset_in_page(in) + ecb_fetch_bytes > PAGE_SIZE)) { + ecb_crypt_copy(in, out, key, cword, count); + return; + } + + rep_xcrypt_ecb(in, out, key, cword, count); +} + +static inline u8 *cbc_crypt(const u8 *in, u8 *out, u32 *key, + u8 *iv, struct cword *cword, int count) +{ + /* Padlock in CBC mode fetches at least cbc_fetch_bytes of data. */ + if (unlikely(offset_in_page(in) + cbc_fetch_bytes > PAGE_SIZE)) + return cbc_crypt_copy(in, out, key, iv, cword, count); + + return rep_xcrypt_cbc(in, out, key, iv, cword, count); +} + +static inline void padlock_xcrypt_ecb(const u8 *input, u8 *output, void *key, + void *control_word, u32 count) +{ + u32 initial = count & (ecb_fetch_blocks - 1); + + if (count < ecb_fetch_blocks) { + ecb_crypt(input, output, key, control_word, count); + return; + } + + count -= initial; + + if (initial) + asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */ + : "+S"(input), "+D"(output) + : "d"(control_word), "b"(key), "c"(initial)); + + asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */ + : "+S"(input), "+D"(output) + : "d"(control_word), "b"(key), "c"(count)); +} + +static inline u8 *padlock_xcrypt_cbc(const u8 *input, u8 *output, void *key, + u8 *iv, void *control_word, u32 count) +{ + u32 initial = count & (cbc_fetch_blocks - 1); + + if (count < cbc_fetch_blocks) + return cbc_crypt(input, output, key, iv, control_word, count); + + count -= initial; + + if (initial) + asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */ + : "+S" (input), "+D" (output), "+a" (iv) + : "d" (control_word), "b" (key), "c" (initial)); + + asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */ + : "+S" (input), "+D" (output), "+a" (iv) + : "d" (control_word), "b" (key), "c" (count)); + return iv; +} + +static void padlock_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) +{ + struct aes_ctx *ctx = aes_ctx(tfm); + + padlock_reset_key(&ctx->cword.encrypt); + ecb_crypt(in, out, ctx->E, &ctx->cword.encrypt, 1); + padlock_store_cword(&ctx->cword.encrypt); +} + +static void padlock_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) +{ + struct aes_ctx *ctx = aes_ctx(tfm); + + padlock_reset_key(&ctx->cword.encrypt); + ecb_crypt(in, out, ctx->D, &ctx->cword.decrypt, 1); + padlock_store_cword(&ctx->cword.encrypt); +} + +static struct crypto_alg aes_alg = { + .cra_name = "aes", + .cra_driver_name = "aes-padlock", + .cra_priority = PADLOCK_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_CIPHER, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aes_ctx), + .cra_alignmask = PADLOCK_ALIGNMENT - 1, + .cra_module = THIS_MODULE, + .cra_u = { + .cipher = { + .cia_min_keysize = AES_MIN_KEY_SIZE, + .cia_max_keysize = AES_MAX_KEY_SIZE, + .cia_setkey = aes_set_key, + .cia_encrypt = padlock_aes_encrypt, + .cia_decrypt = padlock_aes_decrypt, + } + } +}; + +static int ecb_aes_encrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct aes_ctx *ctx = skcipher_aes_ctx(tfm); + struct skcipher_walk walk; + unsigned int nbytes; + int err; + + padlock_reset_key(&ctx->cword.encrypt); + + err = skcipher_walk_virt(&walk, req, false); + + while ((nbytes = walk.nbytes) != 0) { + padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr, + ctx->E, &ctx->cword.encrypt, + nbytes / AES_BLOCK_SIZE); + nbytes &= AES_BLOCK_SIZE - 1; + err = skcipher_walk_done(&walk, nbytes); + } + + padlock_store_cword(&ctx->cword.encrypt); + + return err; +} + +static int ecb_aes_decrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct aes_ctx *ctx = skcipher_aes_ctx(tfm); + struct skcipher_walk walk; + unsigned int nbytes; + int err; + + padlock_reset_key(&ctx->cword.decrypt); + + err = skcipher_walk_virt(&walk, req, false); + + while ((nbytes = walk.nbytes) != 0) { + padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr, + ctx->D, &ctx->cword.decrypt, + nbytes / AES_BLOCK_SIZE); + nbytes &= AES_BLOCK_SIZE - 1; + err = skcipher_walk_done(&walk, nbytes); + } + + padlock_store_cword(&ctx->cword.encrypt); + + return err; +} + +static struct skcipher_alg ecb_aes_alg = { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-padlock", + .base.cra_priority = PADLOCK_COMPOSITE_PRIORITY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct aes_ctx), + .base.cra_alignmask = PADLOCK_ALIGNMENT - 1, + .base.cra_module = THIS_MODULE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = aes_set_key_skcipher, + .encrypt = ecb_aes_encrypt, + .decrypt = ecb_aes_decrypt, +}; + +static int cbc_aes_encrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct aes_ctx *ctx = skcipher_aes_ctx(tfm); + struct skcipher_walk walk; + unsigned int nbytes; + int err; + + padlock_reset_key(&ctx->cword.encrypt); + + err = skcipher_walk_virt(&walk, req, false); + + while ((nbytes = walk.nbytes) != 0) { + u8 *iv = padlock_xcrypt_cbc(walk.src.virt.addr, + walk.dst.virt.addr, ctx->E, + walk.iv, &ctx->cword.encrypt, + nbytes / AES_BLOCK_SIZE); + memcpy(walk.iv, iv, AES_BLOCK_SIZE); + nbytes &= AES_BLOCK_SIZE - 1; + err = skcipher_walk_done(&walk, nbytes); + } + + padlock_store_cword(&ctx->cword.decrypt); + + return err; +} + +static int cbc_aes_decrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct aes_ctx *ctx = skcipher_aes_ctx(tfm); + struct skcipher_walk walk; + unsigned int nbytes; + int err; + + padlock_reset_key(&ctx->cword.encrypt); + + err = skcipher_walk_virt(&walk, req, false); + + while ((nbytes = walk.nbytes) != 0) { + padlock_xcrypt_cbc(walk.src.virt.addr, walk.dst.virt.addr, + ctx->D, walk.iv, &ctx->cword.decrypt, + nbytes / AES_BLOCK_SIZE); + nbytes &= AES_BLOCK_SIZE - 1; + err = skcipher_walk_done(&walk, nbytes); + } + + padlock_store_cword(&ctx->cword.encrypt); + + return err; +} + +static struct skcipher_alg cbc_aes_alg = { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-padlock", + .base.cra_priority = PADLOCK_COMPOSITE_PRIORITY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct aes_ctx), + .base.cra_alignmask = PADLOCK_ALIGNMENT - 1, + .base.cra_module = THIS_MODULE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = aes_set_key_skcipher, + .encrypt = cbc_aes_encrypt, + .decrypt = cbc_aes_decrypt, +}; + +static const struct x86_cpu_id padlock_cpu_id[] = { + X86_MATCH_FEATURE(X86_FEATURE_XCRYPT, NULL), + {} +}; +MODULE_DEVICE_TABLE(x86cpu, padlock_cpu_id); + +static int __init padlock_init(void) +{ + int ret; + struct cpuinfo_x86 *c = &cpu_data(0); + + if (!x86_match_cpu(padlock_cpu_id)) + return -ENODEV; + + if (!boot_cpu_has(X86_FEATURE_XCRYPT_EN)) { + printk(KERN_NOTICE PFX "VIA PadLock detected, but not enabled. Hmm, strange...\n"); + return -ENODEV; + } + + if ((ret = crypto_register_alg(&aes_alg)) != 0) + goto aes_err; + + if ((ret = crypto_register_skcipher(&ecb_aes_alg)) != 0) + goto ecb_aes_err; + + if ((ret = crypto_register_skcipher(&cbc_aes_alg)) != 0) + goto cbc_aes_err; + + printk(KERN_NOTICE PFX "Using VIA PadLock ACE for AES algorithm.\n"); + + if (c->x86 == 6 && c->x86_model == 15 && c->x86_stepping == 2) { + ecb_fetch_blocks = MAX_ECB_FETCH_BLOCKS; + cbc_fetch_blocks = MAX_CBC_FETCH_BLOCKS; + printk(KERN_NOTICE PFX "VIA Nano stepping 2 detected: enabling workaround.\n"); + } + +out: + return ret; + +cbc_aes_err: + crypto_unregister_skcipher(&ecb_aes_alg); +ecb_aes_err: + crypto_unregister_alg(&aes_alg); +aes_err: + printk(KERN_ERR PFX "VIA PadLock AES initialization failed.\n"); + goto out; +} + +static void __exit padlock_fini(void) +{ + crypto_unregister_skcipher(&cbc_aes_alg); + crypto_unregister_skcipher(&ecb_aes_alg); + crypto_unregister_alg(&aes_alg); +} + +module_init(padlock_init); +module_exit(padlock_fini); + +MODULE_DESCRIPTION("VIA PadLock AES algorithm support"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Michal Ludvig"); + +MODULE_ALIAS_CRYPTO("aes"); diff --git a/drivers/crypto/padlock-sha.c b/drivers/crypto/padlock-sha.c new file mode 100644 index 000000000..9214bbfc8 --- /dev/null +++ b/drivers/crypto/padlock-sha.c @@ -0,0 +1,395 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Cryptographic API. + * + * Support for VIA PadLock hardware crypto engine. + * + * Copyright (c) 2006 Michal Ludvig + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define PADLOCK_SHA_DESCSIZE (128 + ((PADLOCK_ALIGNMENT - 1) & \ + ~(CRYPTO_MINALIGN - 1))) + +struct padlock_sha_ctx { + struct crypto_ahash *fallback; +}; + +static inline void *padlock_shash_desc_ctx(struct shash_desc *desc) +{ + return PTR_ALIGN(shash_desc_ctx(desc), PADLOCK_ALIGNMENT); +} + +static int padlock_sha1_init(struct shash_desc *desc) +{ + struct sha1_state *sctx = padlock_shash_desc_ctx(desc); + + *sctx = (struct sha1_state){ + .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, + }; + + return 0; +} + +static int padlock_sha256_init(struct shash_desc *desc) +{ + struct crypto_sha256_state *sctx = padlock_shash_desc_ctx(desc); + + sha256_block_init(sctx); + return 0; +} + +static int padlock_sha_update(struct shash_desc *desc, + const u8 *data, unsigned int length) +{ + u8 *state = padlock_shash_desc_ctx(desc); + struct crypto_shash *tfm = desc->tfm; + int err, remain; + + remain = length - round_down(length, crypto_shash_blocksize(tfm)); + { + struct padlock_sha_ctx *ctx = crypto_shash_ctx(tfm); + HASH_REQUEST_ON_STACK(req, ctx->fallback); + + ahash_request_set_callback(req, 0, NULL, NULL); + ahash_request_set_virt(req, data, NULL, length - remain); + err = crypto_ahash_import_core(req, state) ?: + crypto_ahash_update(req) ?: + crypto_ahash_export_core(req, state); + HASH_REQUEST_ZERO(req); + } + + return err ?: remain; +} + +static int padlock_sha_export(struct shash_desc *desc, void *out) +{ + memcpy(out, padlock_shash_desc_ctx(desc), + crypto_shash_coresize(desc->tfm)); + return 0; +} + +static int padlock_sha_import(struct shash_desc *desc, const void *in) +{ + unsigned int bs = crypto_shash_blocksize(desc->tfm); + unsigned int ss = crypto_shash_coresize(desc->tfm); + u64 *state = padlock_shash_desc_ctx(desc); + + memcpy(state, in, ss); + + /* Stop evil imports from generating a fault. */ + state[ss / 8 - 1] &= ~(bs - 1); + + return 0; +} + +static inline void padlock_output_block(uint32_t *src, + uint32_t *dst, size_t count) +{ + while (count--) + *dst++ = swab32(*src++); +} + +static int padlock_sha_finup(struct shash_desc *desc, const u8 *in, + unsigned int count, u8 *out) +{ + struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm); + HASH_REQUEST_ON_STACK(req, ctx->fallback); + + ahash_request_set_callback(req, 0, NULL, NULL); + ahash_request_set_virt(req, in, out, count); + return crypto_ahash_import_core(req, padlock_shash_desc_ctx(desc)) ?: + crypto_ahash_finup(req); +} + +static int padlock_sha1_finup(struct shash_desc *desc, const u8 *in, + unsigned int count, u8 *out) +{ + /* We can't store directly to *out as it may be unaligned. */ + /* BTW Don't reduce the buffer size below 128 Bytes! + * PadLock microcode needs it that big. */ + struct sha1_state *state = padlock_shash_desc_ctx(desc); + u64 start = state->count; + + if (start + count > ULONG_MAX) + return padlock_sha_finup(desc, in, count, out); + + asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" /* rep xsha1 */ + : \ + : "c"((unsigned long)start + count), \ + "a"((unsigned long)start), \ + "S"(in), "D"(state)); + + padlock_output_block(state->state, (uint32_t *)out, 5); + return 0; +} + +static int padlock_sha256_finup(struct shash_desc *desc, const u8 *in, + unsigned int count, u8 *out) +{ + /* We can't store directly to *out as it may be unaligned. */ + /* BTW Don't reduce the buffer size below 128 Bytes! + * PadLock microcode needs it that big. */ + struct sha256_state *state = padlock_shash_desc_ctx(desc); + u64 start = state->count; + + if (start + count > ULONG_MAX) + return padlock_sha_finup(desc, in, count, out); + + asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" /* rep xsha256 */ + : \ + : "c"((unsigned long)start + count), \ + "a"((unsigned long)start), \ + "S"(in), "D"(state)); + + padlock_output_block(state->state, (uint32_t *)out, 8); + return 0; +} + +static int padlock_init_tfm(struct crypto_shash *hash) +{ + const char *fallback_driver_name = crypto_shash_alg_name(hash); + struct padlock_sha_ctx *ctx = crypto_shash_ctx(hash); + struct crypto_ahash *fallback_tfm; + + /* Allocate a fallback and abort if it failed. */ + fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0, + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC); + if (IS_ERR(fallback_tfm)) { + printk(KERN_WARNING PFX "Fallback driver '%s' could not be loaded!\n", + fallback_driver_name); + return PTR_ERR(fallback_tfm); + } + + if (crypto_shash_statesize(hash) != + crypto_ahash_statesize(fallback_tfm)) { + crypto_free_ahash(fallback_tfm); + return -EINVAL; + } + + ctx->fallback = fallback_tfm; + + return 0; +} + +static void padlock_exit_tfm(struct crypto_shash *hash) +{ + struct padlock_sha_ctx *ctx = crypto_shash_ctx(hash); + + crypto_free_ahash(ctx->fallback); +} + +static struct shash_alg sha1_alg = { + .digestsize = SHA1_DIGEST_SIZE, + .init = padlock_sha1_init, + .update = padlock_sha_update, + .finup = padlock_sha1_finup, + .export = padlock_sha_export, + .import = padlock_sha_import, + .init_tfm = padlock_init_tfm, + .exit_tfm = padlock_exit_tfm, + .descsize = PADLOCK_SHA_DESCSIZE, + .statesize = SHA1_STATE_SIZE, + .base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-padlock", + .cra_priority = PADLOCK_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct padlock_sha_ctx), + .cra_module = THIS_MODULE, + } +}; + +static struct shash_alg sha256_alg = { + .digestsize = SHA256_DIGEST_SIZE, + .init = padlock_sha256_init, + .update = padlock_sha_update, + .finup = padlock_sha256_finup, + .init_tfm = padlock_init_tfm, + .export = padlock_sha_export, + .import = padlock_sha_import, + .exit_tfm = padlock_exit_tfm, + .descsize = PADLOCK_SHA_DESCSIZE, + .statesize = sizeof(struct crypto_sha256_state), + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-padlock", + .cra_priority = PADLOCK_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct padlock_sha_ctx), + .cra_module = THIS_MODULE, + } +}; + +/* Add two shash_alg instance for hardware-implemented * +* multiple-parts hash supported by VIA Nano Processor.*/ + +static int padlock_sha1_update_nano(struct shash_desc *desc, + const u8 *src, unsigned int len) +{ + /*The PHE require the out buffer must 128 bytes and 16-bytes aligned*/ + struct sha1_state *state = padlock_shash_desc_ctx(desc); + int blocks = len / SHA1_BLOCK_SIZE; + + len -= blocks * SHA1_BLOCK_SIZE; + state->count += blocks * SHA1_BLOCK_SIZE; + + /* Process the left bytes from the input data */ + asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" + : "+S"(src), "+D"(state) + : "a"((long)-1), + "c"((unsigned long)blocks)); + return len; +} + +static int padlock_sha256_update_nano(struct shash_desc *desc, const u8 *src, + unsigned int len) +{ + /*The PHE require the out buffer must 128 bytes and 16-bytes aligned*/ + struct crypto_sha256_state *state = padlock_shash_desc_ctx(desc); + int blocks = len / SHA256_BLOCK_SIZE; + + len -= blocks * SHA256_BLOCK_SIZE; + state->count += blocks * SHA256_BLOCK_SIZE; + + /* Process the left bytes from input data*/ + asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" + : "+S"(src), "+D"(state) + : "a"((long)-1), + "c"((unsigned long)blocks)); + return len; +} + +static struct shash_alg sha1_alg_nano = { + .digestsize = SHA1_DIGEST_SIZE, + .init = padlock_sha1_init, + .update = padlock_sha1_update_nano, + .finup = padlock_sha1_finup, + .export = padlock_sha_export, + .import = padlock_sha_import, + .descsize = PADLOCK_SHA_DESCSIZE, + .statesize = SHA1_STATE_SIZE, + .base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-padlock-nano", + .cra_priority = PADLOCK_CRA_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static struct shash_alg sha256_alg_nano = { + .digestsize = SHA256_DIGEST_SIZE, + .init = padlock_sha256_init, + .update = padlock_sha256_update_nano, + .finup = padlock_sha256_finup, + .export = padlock_sha_export, + .import = padlock_sha_import, + .descsize = PADLOCK_SHA_DESCSIZE, + .statesize = sizeof(struct crypto_sha256_state), + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-padlock-nano", + .cra_priority = PADLOCK_CRA_PRIORITY, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | + CRYPTO_AHASH_ALG_FINUP_MAX, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static const struct x86_cpu_id padlock_sha_ids[] = { + X86_MATCH_FEATURE(X86_FEATURE_PHE, NULL), + {} +}; +MODULE_DEVICE_TABLE(x86cpu, padlock_sha_ids); + +static int __init padlock_init(void) +{ + int rc = -ENODEV; + struct cpuinfo_x86 *c = &cpu_data(0); + struct shash_alg *sha1; + struct shash_alg *sha256; + + if (!x86_match_cpu(padlock_sha_ids) || !boot_cpu_has(X86_FEATURE_PHE_EN)) + return -ENODEV; + + /* + * Skip family 0x07 and newer used by Zhaoxin processors, + * as the driver's self-tests fail on these CPUs. + */ + if (c->x86 >= 0x07) + return -ENODEV; + + /* Register the newly added algorithm module if on * + * VIA Nano processor, or else just do as before */ + if (c->x86_model < 0x0f) { + sha1 = &sha1_alg; + sha256 = &sha256_alg; + } else { + sha1 = &sha1_alg_nano; + sha256 = &sha256_alg_nano; + } + + rc = crypto_register_shash(sha1); + if (rc) + goto out; + + rc = crypto_register_shash(sha256); + if (rc) + goto out_unreg1; + + printk(KERN_NOTICE PFX "Using VIA PadLock ACE for SHA1/SHA256 algorithms.\n"); + + return 0; + +out_unreg1: + crypto_unregister_shash(sha1); + +out: + printk(KERN_ERR PFX "VIA PadLock SHA1/SHA256 initialization failed.\n"); + return rc; +} + +static void __exit padlock_fini(void) +{ + struct cpuinfo_x86 *c = &cpu_data(0); + + if (c->x86_model >= 0x0f) { + crypto_unregister_shash(&sha1_alg_nano); + crypto_unregister_shash(&sha256_alg_nano); + } else { + crypto_unregister_shash(&sha1_alg); + crypto_unregister_shash(&sha256_alg); + } +} + +module_init(padlock_init); +module_exit(padlock_fini); + +MODULE_DESCRIPTION("VIA PadLock SHA1/SHA256 algorithms support."); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Michal Ludvig"); + +MODULE_ALIAS_CRYPTO("sha1-all"); +MODULE_ALIAS_CRYPTO("sha256-all"); +MODULE_ALIAS_CRYPTO("sha1-padlock"); +MODULE_ALIAS_CRYPTO("sha256-padlock"); diff --git a/drivers/crypto/qce/Makefile b/drivers/crypto/qce/Makefile new file mode 100644 index 000000000..2cf8984e1 --- /dev/null +++ b/drivers/crypto/qce/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_CRYPTO_DEV_QCE) += qcrypto.o +qcrypto-objs := core.o \ + common.o \ + dma.o + +qcrypto-$(CONFIG_CRYPTO_DEV_QCE_SHA) += sha.o +qcrypto-$(CONFIG_CRYPTO_DEV_QCE_SKCIPHER) += skcipher.o +qcrypto-$(CONFIG_CRYPTO_DEV_QCE_AEAD) += aead.o diff --git a/drivers/crypto/qce/aead.c b/drivers/crypto/qce/aead.c new file mode 100644 index 000000000..a9642c67f --- /dev/null +++ b/drivers/crypto/qce/aead.c @@ -0,0 +1,787 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/* + * Copyright (C) 2021, Linaro Limited. All rights reserved. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "aead.h" + +#define CCM_NONCE_ADATA_SHIFT 6 +#define CCM_NONCE_AUTHSIZE_SHIFT 3 +#define MAX_CCM_ADATA_HEADER_LEN 6 + +static LIST_HEAD(aead_algs); + +static void qce_aead_done(void *data) +{ + struct crypto_async_request *async_req = data; + struct aead_request *req = aead_request_cast(async_req); + struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); + struct qce_aead_ctx *ctx = crypto_tfm_ctx(async_req->tfm); + struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req)); + struct qce_device *qce = tmpl->qce; + struct qce_result_dump *result_buf = qce->dma.result_buf; + enum dma_data_direction dir_src, dir_dst; + bool diff_dst; + int error; + u32 status; + unsigned int totallen; + unsigned char tag[SHA256_DIGEST_SIZE] = {0}; + + diff_dst = (req->src != req->dst) ? true : false; + dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; + dir_dst = diff_dst ? DMA_FROM_DEVICE : DMA_BIDIRECTIONAL; + + error = qce_dma_terminate_all(&qce->dma); + if (error) + dev_dbg(qce->dev, "aead dma termination error (%d)\n", + error); + if (diff_dst) + dma_unmap_sg(qce->dev, rctx->src_sg, rctx->src_nents, dir_src); + + dma_unmap_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); + + if (IS_CCM(rctx->flags)) { + if (req->assoclen) { + sg_free_table(&rctx->src_tbl); + if (diff_dst) + sg_free_table(&rctx->dst_tbl); + } else { + if (!(IS_DECRYPT(rctx->flags) && !diff_dst)) + sg_free_table(&rctx->dst_tbl); + } + } else { + sg_free_table(&rctx->dst_tbl); + } + + error = qce_check_status(qce, &status); + if (error < 0 && (error != -EBADMSG)) + dev_err(qce->dev, "aead operation error (%x)\n", status); + + if (IS_ENCRYPT(rctx->flags)) { + totallen = req->cryptlen + req->assoclen; + if (IS_CCM(rctx->flags)) + scatterwalk_map_and_copy(rctx->ccmresult_buf, req->dst, + totallen, ctx->authsize, 1); + else + scatterwalk_map_and_copy(result_buf->auth_iv, req->dst, + totallen, ctx->authsize, 1); + + } else if (!IS_CCM(rctx->flags)) { + totallen = req->cryptlen + req->assoclen - ctx->authsize; + scatterwalk_map_and_copy(tag, req->src, totallen, ctx->authsize, 0); + if (memcmp(result_buf->auth_iv, tag, ctx->authsize)) { + pr_err("Bad message error\n"); + error = -EBADMSG; + } + } + + qce->async_req_done(qce, error); +} + +static struct scatterlist * +qce_aead_prepare_result_buf(struct sg_table *tbl, struct aead_request *req) +{ + struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); + struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req)); + struct qce_device *qce = tmpl->qce; + + sg_init_one(&rctx->result_sg, qce->dma.result_buf, QCE_RESULT_BUF_SZ); + return qce_sgtable_add(tbl, &rctx->result_sg, QCE_RESULT_BUF_SZ); +} + +static struct scatterlist * +qce_aead_prepare_ccm_result_buf(struct sg_table *tbl, struct aead_request *req) +{ + struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); + + sg_init_one(&rctx->result_sg, rctx->ccmresult_buf, QCE_BAM_BURST_SIZE); + return qce_sgtable_add(tbl, &rctx->result_sg, QCE_BAM_BURST_SIZE); +} + +static struct scatterlist * +qce_aead_prepare_dst_buf(struct aead_request *req) +{ + struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); + struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req)); + struct qce_device *qce = tmpl->qce; + struct scatterlist *sg, *msg_sg, __sg[2]; + gfp_t gfp; + unsigned int assoclen = req->assoclen; + unsigned int totallen; + int ret; + + totallen = rctx->cryptlen + assoclen; + rctx->dst_nents = sg_nents_for_len(req->dst, totallen); + if (rctx->dst_nents < 0) { + dev_err(qce->dev, "Invalid numbers of dst SG.\n"); + return ERR_PTR(-EINVAL); + } + if (IS_CCM(rctx->flags)) + rctx->dst_nents += 2; + else + rctx->dst_nents += 1; + + gfp = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + ret = sg_alloc_table(&rctx->dst_tbl, rctx->dst_nents, gfp); + if (ret) + return ERR_PTR(ret); + + if (IS_CCM(rctx->flags) && assoclen) { + /* Get the dst buffer */ + msg_sg = scatterwalk_ffwd(__sg, req->dst, assoclen); + + sg = qce_sgtable_add(&rctx->dst_tbl, &rctx->adata_sg, + rctx->assoclen); + if (IS_ERR(sg)) + goto dst_tbl_free; + /* dst buffer */ + sg = qce_sgtable_add(&rctx->dst_tbl, msg_sg, rctx->cryptlen); + if (IS_ERR(sg)) + goto dst_tbl_free; + totallen = rctx->cryptlen + rctx->assoclen; + } else { + if (totallen) { + sg = qce_sgtable_add(&rctx->dst_tbl, req->dst, totallen); + if (IS_ERR(sg)) + goto dst_tbl_free; + } + } + if (IS_CCM(rctx->flags)) + sg = qce_aead_prepare_ccm_result_buf(&rctx->dst_tbl, req); + else + sg = qce_aead_prepare_result_buf(&rctx->dst_tbl, req); + + if (IS_ERR(sg)) + goto dst_tbl_free; + + sg_mark_end(sg); + rctx->dst_sg = rctx->dst_tbl.sgl; + rctx->dst_nents = sg_nents_for_len(rctx->dst_sg, totallen) + 1; + + return sg; + +dst_tbl_free: + sg_free_table(&rctx->dst_tbl); + return sg; +} + +static int +qce_aead_ccm_prepare_buf_assoclen(struct aead_request *req) +{ + struct scatterlist *sg, *msg_sg, __sg[2]; + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); + struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); + unsigned int assoclen = rctx->assoclen; + unsigned int adata_header_len, cryptlen, totallen; + gfp_t gfp; + bool diff_dst; + int ret; + + if (IS_DECRYPT(rctx->flags)) + cryptlen = rctx->cryptlen + ctx->authsize; + else + cryptlen = rctx->cryptlen; + totallen = cryptlen + req->assoclen; + + /* Get the msg */ + msg_sg = scatterwalk_ffwd(__sg, req->src, req->assoclen); + + rctx->adata = kzalloc(ALIGN(assoclen + MAX_CCM_ADATA_HEADER_LEN, 16) * + sizeof(unsigned char), GFP_ATOMIC); + if (!rctx->adata) + return -ENOMEM; + + /* + * Format associated data (RFC3610 and NIST 800-38C) + * Even though specification allows for AAD to be up to 2^64 - 1 bytes, + * the assoclen field in aead_request is unsigned int and thus limits + * the AAD to be up to 2^32 - 1 bytes. So we handle only two scenarios + * while forming the header for AAD. + */ + if (assoclen < 0xff00) { + adata_header_len = 2; + *(__be16 *)rctx->adata = cpu_to_be16(assoclen); + } else { + adata_header_len = 6; + *(__be16 *)rctx->adata = cpu_to_be16(0xfffe); + *(__be32 *)(rctx->adata + 2) = cpu_to_be32(assoclen); + } + + /* Copy the associated data */ + if (sg_copy_to_buffer(req->src, sg_nents_for_len(req->src, assoclen), + rctx->adata + adata_header_len, + assoclen) != assoclen) + return -EINVAL; + + /* Pad associated data to block size */ + rctx->assoclen = ALIGN(assoclen + adata_header_len, 16); + + diff_dst = (req->src != req->dst) ? true : false; + + if (diff_dst) + rctx->src_nents = sg_nents_for_len(req->src, totallen) + 1; + else + rctx->src_nents = sg_nents_for_len(req->src, totallen) + 2; + + gfp = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; + ret = sg_alloc_table(&rctx->src_tbl, rctx->src_nents, gfp); + if (ret) + return ret; + + /* Associated Data */ + sg_init_one(&rctx->adata_sg, rctx->adata, rctx->assoclen); + sg = qce_sgtable_add(&rctx->src_tbl, &rctx->adata_sg, + rctx->assoclen); + if (IS_ERR(sg)) { + ret = PTR_ERR(sg); + goto err_free; + } + /* src msg */ + sg = qce_sgtable_add(&rctx->src_tbl, msg_sg, cryptlen); + if (IS_ERR(sg)) { + ret = PTR_ERR(sg); + goto err_free; + } + if (!diff_dst) { + /* + * For decrypt, when src and dst buffers are same, there is already space + * in the buffer for padded 0's which is output in lieu of + * the MAC that is input. So skip the below. + */ + if (!IS_DECRYPT(rctx->flags)) { + sg = qce_aead_prepare_ccm_result_buf(&rctx->src_tbl, req); + if (IS_ERR(sg)) { + ret = PTR_ERR(sg); + goto err_free; + } + } + } + sg_mark_end(sg); + rctx->src_sg = rctx->src_tbl.sgl; + totallen = cryptlen + rctx->assoclen; + rctx->src_nents = sg_nents_for_len(rctx->src_sg, totallen); + + if (diff_dst) { + sg = qce_aead_prepare_dst_buf(req); + if (IS_ERR(sg)) { + ret = PTR_ERR(sg); + goto err_free; + } + } else { + if (IS_ENCRYPT(rctx->flags)) + rctx->dst_nents = rctx->src_nents + 1; + else + rctx->dst_nents = rctx->src_nents; + rctx->dst_sg = rctx->src_sg; + } + + return 0; +err_free: + sg_free_table(&rctx->src_tbl); + return ret; +} + +static int qce_aead_prepare_buf(struct aead_request *req) +{ + struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); + struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req)); + struct qce_device *qce = tmpl->qce; + struct scatterlist *sg; + bool diff_dst = (req->src != req->dst) ? true : false; + unsigned int totallen; + + totallen = rctx->cryptlen + rctx->assoclen; + + sg = qce_aead_prepare_dst_buf(req); + if (IS_ERR(sg)) + return PTR_ERR(sg); + if (diff_dst) { + rctx->src_nents = sg_nents_for_len(req->src, totallen); + if (rctx->src_nents < 0) { + dev_err(qce->dev, "Invalid numbers of src SG.\n"); + return -EINVAL; + } + rctx->src_sg = req->src; + } else { + rctx->src_nents = rctx->dst_nents - 1; + rctx->src_sg = rctx->dst_sg; + } + return 0; +} + +static int qce_aead_ccm_prepare_buf(struct aead_request *req) +{ + struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct scatterlist *sg; + bool diff_dst = (req->src != req->dst) ? true : false; + unsigned int cryptlen; + + if (rctx->assoclen) + return qce_aead_ccm_prepare_buf_assoclen(req); + + if (IS_ENCRYPT(rctx->flags)) + return qce_aead_prepare_buf(req); + + cryptlen = rctx->cryptlen + ctx->authsize; + if (diff_dst) { + rctx->src_nents = sg_nents_for_len(req->src, cryptlen); + rctx->src_sg = req->src; + sg = qce_aead_prepare_dst_buf(req); + if (IS_ERR(sg)) + return PTR_ERR(sg); + } else { + rctx->src_nents = sg_nents_for_len(req->src, cryptlen); + rctx->src_sg = req->src; + rctx->dst_nents = rctx->src_nents; + rctx->dst_sg = rctx->src_sg; + } + + return 0; +} + +static int qce_aead_create_ccm_nonce(struct qce_aead_reqctx *rctx, struct qce_aead_ctx *ctx) +{ + unsigned int msglen_size, ivsize; + u8 msg_len[4]; + int i; + + if (!rctx || !rctx->iv) + return -EINVAL; + + msglen_size = rctx->iv[0] + 1; + + /* Verify that msg len size is valid */ + if (msglen_size < 2 || msglen_size > 8) + return -EINVAL; + + ivsize = rctx->ivsize; + + /* + * Clear the msglen bytes in IV. + * Else the h/w engine and nonce will use any stray value pending there. + */ + if (!IS_CCM_RFC4309(rctx->flags)) { + for (i = 0; i < msglen_size; i++) + rctx->iv[ivsize - i - 1] = 0; + } + + /* + * The crypto framework encodes cryptlen as unsigned int. Thus, even though + * spec allows for upto 8 bytes to encode msg_len only 4 bytes are needed. + */ + if (msglen_size > 4) + msglen_size = 4; + + memcpy(&msg_len[0], &rctx->cryptlen, 4); + + memcpy(&rctx->ccm_nonce[0], rctx->iv, rctx->ivsize); + if (rctx->assoclen) + rctx->ccm_nonce[0] |= 1 << CCM_NONCE_ADATA_SHIFT; + rctx->ccm_nonce[0] |= ((ctx->authsize - 2) / 2) << + CCM_NONCE_AUTHSIZE_SHIFT; + for (i = 0; i < msglen_size; i++) + rctx->ccm_nonce[QCE_MAX_NONCE - i - 1] = msg_len[i]; + + return 0; +} + +static int +qce_aead_async_req_handle(struct crypto_async_request *async_req) +{ + struct aead_request *req = aead_request_cast(async_req); + struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct qce_aead_ctx *ctx = crypto_tfm_ctx(async_req->tfm); + struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req)); + struct qce_device *qce = tmpl->qce; + enum dma_data_direction dir_src, dir_dst; + bool diff_dst; + int dst_nents, src_nents, ret; + + if (IS_CCM_RFC4309(rctx->flags)) { + memset(rctx->ccm_rfc4309_iv, 0, QCE_MAX_IV_SIZE); + rctx->ccm_rfc4309_iv[0] = 3; + memcpy(&rctx->ccm_rfc4309_iv[1], ctx->ccm4309_salt, QCE_CCM4309_SALT_SIZE); + memcpy(&rctx->ccm_rfc4309_iv[4], req->iv, 8); + rctx->iv = rctx->ccm_rfc4309_iv; + rctx->ivsize = AES_BLOCK_SIZE; + } else { + rctx->iv = req->iv; + rctx->ivsize = crypto_aead_ivsize(tfm); + } + if (IS_CCM_RFC4309(rctx->flags)) + rctx->assoclen = req->assoclen - 8; + else + rctx->assoclen = req->assoclen; + + diff_dst = (req->src != req->dst) ? true : false; + dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; + dir_dst = diff_dst ? DMA_FROM_DEVICE : DMA_BIDIRECTIONAL; + + if (IS_CCM(rctx->flags)) { + ret = qce_aead_create_ccm_nonce(rctx, ctx); + if (ret) + return ret; + } + if (IS_CCM(rctx->flags)) + ret = qce_aead_ccm_prepare_buf(req); + else + ret = qce_aead_prepare_buf(req); + + if (ret) + return ret; + dst_nents = dma_map_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); + if (!dst_nents) { + ret = -EIO; + goto error_free; + } + + if (diff_dst) { + src_nents = dma_map_sg(qce->dev, rctx->src_sg, rctx->src_nents, dir_src); + if (src_nents < 0) { + ret = src_nents; + goto error_unmap_dst; + } + } else { + if (IS_CCM(rctx->flags) && IS_DECRYPT(rctx->flags)) + src_nents = dst_nents; + else + src_nents = dst_nents - 1; + } + + ret = qce_dma_prep_sgs(&qce->dma, rctx->src_sg, src_nents, rctx->dst_sg, dst_nents, + qce_aead_done, async_req); + if (ret) + goto error_unmap_src; + + qce_dma_issue_pending(&qce->dma); + + ret = qce_start(async_req, tmpl->crypto_alg_type); + if (ret) + goto error_terminate; + + return 0; + +error_terminate: + qce_dma_terminate_all(&qce->dma); +error_unmap_src: + if (diff_dst) + dma_unmap_sg(qce->dev, req->src, rctx->src_nents, dir_src); +error_unmap_dst: + dma_unmap_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); +error_free: + if (IS_CCM(rctx->flags) && rctx->assoclen) { + sg_free_table(&rctx->src_tbl); + if (diff_dst) + sg_free_table(&rctx->dst_tbl); + } else { + sg_free_table(&rctx->dst_tbl); + } + return ret; +} + +static int qce_aead_crypt(struct aead_request *req, int encrypt) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); + struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct qce_alg_template *tmpl = to_aead_tmpl(tfm); + unsigned int blocksize = crypto_aead_blocksize(tfm); + + rctx->flags = tmpl->alg_flags; + rctx->flags |= encrypt ? QCE_ENCRYPT : QCE_DECRYPT; + + if (encrypt) + rctx->cryptlen = req->cryptlen; + else + rctx->cryptlen = req->cryptlen - ctx->authsize; + + /* CE does not handle 0 length messages */ + if (!rctx->cryptlen) { + if (!(IS_CCM(rctx->flags) && IS_DECRYPT(rctx->flags))) + ctx->need_fallback = true; + } + + /* If fallback is needed, schedule and exit */ + if (ctx->need_fallback) { + /* Reset need_fallback in case the same ctx is used for another transaction */ + ctx->need_fallback = false; + + aead_request_set_tfm(&rctx->fallback_req, ctx->fallback); + aead_request_set_callback(&rctx->fallback_req, req->base.flags, + req->base.complete, req->base.data); + aead_request_set_crypt(&rctx->fallback_req, req->src, + req->dst, req->cryptlen, req->iv); + aead_request_set_ad(&rctx->fallback_req, req->assoclen); + + return encrypt ? crypto_aead_encrypt(&rctx->fallback_req) : + crypto_aead_decrypt(&rctx->fallback_req); + } + + /* + * CBC algorithms require message lengths to be + * multiples of block size. + */ + if (IS_CBC(rctx->flags) && !IS_ALIGNED(rctx->cryptlen, blocksize)) + return -EINVAL; + + /* RFC4309 supported AAD size 16 bytes/20 bytes */ + if (IS_CCM_RFC4309(rctx->flags)) + if (crypto_ipsec_check_assoclen(req->assoclen)) + return -EINVAL; + + return tmpl->qce->async_req_enqueue(tmpl->qce, &req->base); +} + +static int qce_aead_encrypt(struct aead_request *req) +{ + return qce_aead_crypt(req, 1); +} + +static int qce_aead_decrypt(struct aead_request *req) +{ + return qce_aead_crypt(req, 0); +} + +static int qce_aead_ccm_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); + unsigned long flags = to_aead_tmpl(tfm)->alg_flags; + + if (IS_CCM_RFC4309(flags)) { + if (keylen < QCE_CCM4309_SALT_SIZE) + return -EINVAL; + keylen -= QCE_CCM4309_SALT_SIZE; + memcpy(ctx->ccm4309_salt, key + keylen, QCE_CCM4309_SALT_SIZE); + } + + if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_256 && keylen != AES_KEYSIZE_192) + return -EINVAL; + + ctx->enc_keylen = keylen; + ctx->auth_keylen = keylen; + + memcpy(ctx->enc_key, key, keylen); + memcpy(ctx->auth_key, key, keylen); + + if (keylen == AES_KEYSIZE_192) + ctx->need_fallback = true; + + return IS_CCM_RFC4309(flags) ? + crypto_aead_setkey(ctx->fallback, key, keylen + QCE_CCM4309_SALT_SIZE) : + crypto_aead_setkey(ctx->fallback, key, keylen); +} + +static int qce_aead_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int keylen) +{ + struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct crypto_authenc_keys authenc_keys; + unsigned long flags = to_aead_tmpl(tfm)->alg_flags; + int err; + + err = crypto_authenc_extractkeys(&authenc_keys, key, keylen); + if (err) + return err; + + if (authenc_keys.enckeylen > QCE_MAX_KEY_SIZE || + authenc_keys.authkeylen > QCE_MAX_KEY_SIZE) + return -EINVAL; + + if (IS_AES(flags)) { + /* No random key sizes */ + if (authenc_keys.enckeylen != AES_KEYSIZE_128 && + authenc_keys.enckeylen != AES_KEYSIZE_192 && + authenc_keys.enckeylen != AES_KEYSIZE_256) + return -EINVAL; + if (authenc_keys.enckeylen == AES_KEYSIZE_192) + ctx->need_fallback = true; + } + + ctx->enc_keylen = authenc_keys.enckeylen; + ctx->auth_keylen = authenc_keys.authkeylen; + + memcpy(ctx->enc_key, authenc_keys.enckey, authenc_keys.enckeylen); + + memcpy_and_pad(ctx->auth_key, sizeof(ctx->auth_key), + authenc_keys.authkey, authenc_keys.authkeylen, 0); + + return crypto_aead_setkey(ctx->fallback, key, keylen); +} + +static int qce_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) +{ + struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); + unsigned long flags = to_aead_tmpl(tfm)->alg_flags; + + if (IS_CCM(flags)) { + if (authsize < 4 || authsize > 16 || authsize % 2) + return -EINVAL; + if (IS_CCM_RFC4309(flags) && (authsize < 8 || authsize % 4)) + return -EINVAL; + } + ctx->authsize = authsize; + + return crypto_aead_setauthsize(ctx->fallback, authsize); +} + +static int qce_aead_init(struct crypto_aead *tfm) +{ + struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); + + ctx->need_fallback = false; + ctx->fallback = crypto_alloc_aead(crypto_tfm_alg_name(&tfm->base), + 0, CRYPTO_ALG_NEED_FALLBACK); + + if (IS_ERR(ctx->fallback)) + return PTR_ERR(ctx->fallback); + + crypto_aead_set_reqsize_dma(tfm, sizeof(struct qce_aead_reqctx) + + crypto_aead_reqsize(ctx->fallback)); + return 0; +} + +static void qce_aead_exit(struct crypto_aead *tfm) +{ + struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); + + crypto_free_aead(ctx->fallback); +} + +struct qce_aead_def { + unsigned long flags; + const char *name; + const char *drv_name; + unsigned int blocksize; + unsigned int chunksize; + unsigned int ivsize; + unsigned int maxauthsize; +}; + +static const struct qce_aead_def aead_def[] = { + { + .flags = QCE_ALG_AES | QCE_MODE_CBC | QCE_HASH_SHA256_HMAC, + .name = "authenc(hmac(sha256),cbc(aes))", + .drv_name = "authenc-hmac-sha256-cbc-aes-qce", + .blocksize = AES_BLOCK_SIZE, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + { + .flags = QCE_ALG_AES | QCE_MODE_CCM, + .name = "ccm(aes)", + .drv_name = "ccm-aes-qce", + .blocksize = 1, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + }, + { + .flags = QCE_ALG_AES | QCE_MODE_CCM | QCE_MODE_CCM_RFC4309, + .name = "rfc4309(ccm(aes))", + .drv_name = "rfc4309-ccm-aes-qce", + .blocksize = 1, + .ivsize = 8, + .maxauthsize = AES_BLOCK_SIZE, + }, +}; + +static int qce_aead_register_one(const struct qce_aead_def *def, struct qce_device *qce) +{ + struct qce_alg_template *tmpl; + struct aead_alg *alg; + int ret; + + tmpl = kzalloc_obj(*tmpl); + if (!tmpl) + return -ENOMEM; + + alg = &tmpl->alg.aead; + + strscpy(alg->base.cra_name, def->name); + strscpy(alg->base.cra_driver_name, def->drv_name); + + alg->base.cra_blocksize = def->blocksize; + alg->chunksize = def->chunksize; + alg->ivsize = def->ivsize; + alg->maxauthsize = def->maxauthsize; + if (IS_CCM(def->flags)) + alg->setkey = qce_aead_ccm_setkey; + else + alg->setkey = qce_aead_setkey; + alg->setauthsize = qce_aead_setauthsize; + alg->encrypt = qce_aead_encrypt; + alg->decrypt = qce_aead_decrypt; + alg->init = qce_aead_init; + alg->exit = qce_aead_exit; + + alg->base.cra_priority = 275; + alg->base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK; + alg->base.cra_ctxsize = sizeof(struct qce_aead_ctx); + alg->base.cra_alignmask = 0; + alg->base.cra_module = THIS_MODULE; + + INIT_LIST_HEAD(&tmpl->entry); + tmpl->crypto_alg_type = CRYPTO_ALG_TYPE_AEAD; + tmpl->alg_flags = def->flags; + tmpl->qce = qce; + + ret = crypto_register_aead(alg); + if (ret) { + dev_err(qce->dev, "%s registration failed\n", alg->base.cra_name); + kfree(tmpl); + return ret; + } + + list_add_tail(&tmpl->entry, &aead_algs); + dev_dbg(qce->dev, "%s is registered\n", alg->base.cra_name); + return 0; +} + +static void qce_aead_unregister(struct qce_device *qce) +{ + struct qce_alg_template *tmpl, *n; + + list_for_each_entry_safe(tmpl, n, &aead_algs, entry) { + crypto_unregister_aead(&tmpl->alg.aead); + list_del(&tmpl->entry); + kfree(tmpl); + } +} + +static int qce_aead_register(struct qce_device *qce) +{ + int ret, i; + + for (i = 0; i < ARRAY_SIZE(aead_def); i++) { + ret = qce_aead_register_one(&aead_def[i], qce); + if (ret) + goto err; + } + + return 0; +err: + qce_aead_unregister(qce); + return ret; +} + +const struct qce_algo_ops aead_ops = { + .type = CRYPTO_ALG_TYPE_AEAD, + .register_algs = qce_aead_register, + .unregister_algs = qce_aead_unregister, + .async_req_handle = qce_aead_async_req_handle, +}; diff --git a/drivers/crypto/qce/aead.h b/drivers/crypto/qce/aead.h new file mode 100644 index 000000000..efb8477cc --- /dev/null +++ b/drivers/crypto/qce/aead.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021, Linaro Limited. All rights reserved. + */ + +#ifndef _AEAD_H_ +#define _AEAD_H_ + +#include "common.h" +#include "core.h" + +#define QCE_MAX_KEY_SIZE 64 +#define QCE_CCM4309_SALT_SIZE 3 + +struct qce_aead_ctx { + u8 enc_key[QCE_MAX_KEY_SIZE]; + u8 auth_key[QCE_MAX_KEY_SIZE]; + u8 ccm4309_salt[QCE_CCM4309_SALT_SIZE]; + unsigned int enc_keylen; + unsigned int auth_keylen; + unsigned int authsize; + bool need_fallback; + struct crypto_aead *fallback; +}; + +struct qce_aead_reqctx { + unsigned long flags; + u8 *iv; + unsigned int ivsize; + int src_nents; + int dst_nents; + struct scatterlist result_sg; + struct scatterlist adata_sg; + struct sg_table dst_tbl; + struct sg_table src_tbl; + struct scatterlist *dst_sg; + struct scatterlist *src_sg; + unsigned int cryptlen; + unsigned int assoclen; + unsigned char *adata; + u8 ccm_nonce[QCE_MAX_NONCE]; + u8 ccmresult_buf[QCE_BAM_BURST_SIZE]; + u8 ccm_rfc4309_iv[QCE_MAX_IV_SIZE]; + struct aead_request fallback_req; +}; + +static inline struct qce_alg_template *to_aead_tmpl(struct crypto_aead *tfm) +{ + struct aead_alg *alg = crypto_aead_alg(tfm); + + return container_of(alg, struct qce_alg_template, alg.aead); +} + +extern const struct qce_algo_ops aead_ops; + +#endif /* _AEAD_H_ */ diff --git a/drivers/crypto/qce/cipher.h b/drivers/crypto/qce/cipher.h new file mode 100644 index 000000000..850f257d0 --- /dev/null +++ b/drivers/crypto/qce/cipher.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2010-2014, The Linux Foundation. All rights reserved. + */ + +#ifndef _CIPHER_H_ +#define _CIPHER_H_ + +#include "common.h" +#include "core.h" + +#define QCE_MAX_KEY_SIZE 64 + +struct qce_cipher_ctx { + u8 enc_key[QCE_MAX_KEY_SIZE]; + unsigned int enc_keylen; + struct crypto_skcipher *fallback; +}; + +/** + * struct qce_cipher_reqctx - holds private cipher objects per request + * @flags: operation flags + * @iv: pointer to the IV + * @ivsize: IV size + * @src_nents: source entries + * @dst_nents: destination entries + * @result_sg: scatterlist used for result buffer + * @dst_tbl: destination sg table + * @dst_sg: destination sg pointer table beginning + * @src_tbl: source sg table + * @src_sg: source sg pointer table beginning; + * @cryptlen: crypto length + */ +struct qce_cipher_reqctx { + unsigned long flags; + u8 *iv; + unsigned int ivsize; + int src_nents; + int dst_nents; + struct scatterlist result_sg; + struct sg_table dst_tbl; + struct scatterlist *dst_sg; + struct scatterlist *src_sg; + unsigned int cryptlen; + struct skcipher_request fallback_req; // keep at the end +}; + +static inline struct qce_alg_template *to_cipher_tmpl(struct crypto_skcipher *tfm) +{ + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + return container_of(alg, struct qce_alg_template, alg.skcipher); +} + +extern const struct qce_algo_ops skcipher_ops; + +#endif /* _CIPHER_H_ */ diff --git a/drivers/crypto/qce/common.c b/drivers/crypto/qce/common.c new file mode 100644 index 000000000..3081d765a --- /dev/null +++ b/drivers/crypto/qce/common.c @@ -0,0 +1,564 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2012-2014, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include + +#include "cipher.h" +#include "common.h" +#include "core.h" +#include "regs-v5.h" +#include "sha.h" +#include "aead.h" + +static inline u32 qce_read(struct qce_device *qce, u32 offset) +{ + return readl(qce->base + offset); +} + +static inline void qce_write(struct qce_device *qce, u32 offset, u32 val) +{ + writel(val, qce->base + offset); +} + +static inline void qce_write_array(struct qce_device *qce, u32 offset, + const u32 *val, unsigned int len) +{ + int i; + + for (i = 0; i < len; i++) + qce_write(qce, offset + i * sizeof(u32), val[i]); +} + +static inline void +qce_clear_array(struct qce_device *qce, u32 offset, unsigned int len) +{ + int i; + + for (i = 0; i < len; i++) + qce_write(qce, offset + i * sizeof(u32), 0); +} + +static u32 qce_config_reg(struct qce_device *qce, int little) +{ + u32 beats = (qce->burst_size >> 3) - 1; + u32 pipe_pair = qce->pipe_pair_id; + u32 config; + + config = (beats << REQ_SIZE_SHIFT) & REQ_SIZE_MASK; + config |= BIT(MASK_DOUT_INTR_SHIFT) | BIT(MASK_DIN_INTR_SHIFT) | + BIT(MASK_OP_DONE_INTR_SHIFT) | BIT(MASK_ERR_INTR_SHIFT); + config |= (pipe_pair << PIPE_SET_SELECT_SHIFT) & PIPE_SET_SELECT_MASK; + config &= ~HIGH_SPD_EN_N_SHIFT; + + if (little) + config |= BIT(LITTLE_ENDIAN_MODE_SHIFT); + + return config; +} + +void qce_cpu_to_be32p_array(__be32 *dst, const u8 *src, unsigned int len) +{ + __be32 *d = dst; + const u8 *s = src; + unsigned int n; + + n = len / sizeof(u32); + for (; n > 0; n--) { + *d = cpu_to_be32p((const __u32 *) s); + s += sizeof(__u32); + d++; + } +} + +static void qce_setup_config(struct qce_device *qce) +{ + u32 config; + + /* get big endianness */ + config = qce_config_reg(qce, 0); + + /* clear status */ + qce_write(qce, REG_STATUS, 0); + qce_write(qce, REG_CONFIG, config); +} + +static inline void qce_crypto_go(struct qce_device *qce, bool result_dump) +{ + if (result_dump) + qce_write(qce, REG_GOPROC, BIT(GO_SHIFT) | BIT(RESULTS_DUMP_SHIFT)); + else + qce_write(qce, REG_GOPROC, BIT(GO_SHIFT)); +} + +#if defined(CONFIG_CRYPTO_DEV_QCE_SHA) || defined(CONFIG_CRYPTO_DEV_QCE_AEAD) +static u32 qce_auth_cfg(unsigned long flags, u32 key_size, u32 auth_size) +{ + u32 cfg = 0; + + if (IS_CCM(flags) || IS_CMAC(flags)) + cfg |= AUTH_ALG_AES << AUTH_ALG_SHIFT; + else + cfg |= AUTH_ALG_SHA << AUTH_ALG_SHIFT; + + if (IS_CCM(flags) || IS_CMAC(flags)) { + if (key_size == AES_KEYSIZE_128) + cfg |= AUTH_KEY_SZ_AES128 << AUTH_KEY_SIZE_SHIFT; + else if (key_size == AES_KEYSIZE_256) + cfg |= AUTH_KEY_SZ_AES256 << AUTH_KEY_SIZE_SHIFT; + } + + if (IS_SHA256(flags) || IS_SHA256_HMAC(flags)) + cfg |= AUTH_SIZE_SHA256 << AUTH_SIZE_SHIFT; + else if (IS_CMAC(flags)) + cfg |= AUTH_SIZE_ENUM_16_BYTES << AUTH_SIZE_SHIFT; + else if (IS_CCM(flags)) + cfg |= (auth_size - 1) << AUTH_SIZE_SHIFT; + + if (IS_SHA256(flags)) + cfg |= AUTH_MODE_HASH << AUTH_MODE_SHIFT; + else if (IS_SHA256_HMAC(flags)) + cfg |= AUTH_MODE_HMAC << AUTH_MODE_SHIFT; + else if (IS_CCM(flags)) + cfg |= AUTH_MODE_CCM << AUTH_MODE_SHIFT; + else if (IS_CMAC(flags)) + cfg |= AUTH_MODE_CMAC << AUTH_MODE_SHIFT; + + if (IS_SHA(flags) || IS_SHA_HMAC(flags)) + cfg |= AUTH_POS_BEFORE << AUTH_POS_SHIFT; + + if (IS_CCM(flags)) + cfg |= QCE_MAX_NONCE_WORDS << AUTH_NONCE_NUM_WORDS_SHIFT; + + return cfg; +} +#endif + +#ifdef CONFIG_CRYPTO_DEV_QCE_SHA +static int qce_setup_regs_ahash(struct crypto_async_request *async_req) +{ + struct ahash_request *req = ahash_request_cast(async_req); + struct crypto_ahash *ahash = __crypto_ahash_cast(async_req->tfm); + struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req); + struct qce_alg_template *tmpl = to_ahash_tmpl(async_req->tfm); + struct qce_device *qce = tmpl->qce; + unsigned int digestsize = crypto_ahash_digestsize(ahash); + unsigned int blocksize = crypto_tfm_alg_blocksize(async_req->tfm); + __be32 auth[SHA256_DIGEST_SIZE / sizeof(__be32)] = {0}; + __be32 mackey[QCE_SHA_HMAC_KEY_SIZE / sizeof(__be32)] = {0}; + u32 auth_cfg = 0, config; + unsigned int iv_words; + + /* if not the last, the size has to be on the block boundary */ + if (!rctx->last_blk && req->nbytes % blocksize) + return -EINVAL; + + qce_setup_config(qce); + + if (IS_CMAC(rctx->flags)) { + qce_write(qce, REG_AUTH_SEG_CFG, 0); + qce_write(qce, REG_ENCR_SEG_CFG, 0); + qce_write(qce, REG_ENCR_SEG_SIZE, 0); + qce_clear_array(qce, REG_AUTH_IV0, 16); + qce_clear_array(qce, REG_AUTH_KEY0, 16); + qce_clear_array(qce, REG_AUTH_BYTECNT0, 4); + + auth_cfg = qce_auth_cfg(rctx->flags, rctx->authklen, digestsize); + } + + if (IS_SHA_HMAC(rctx->flags) || IS_CMAC(rctx->flags)) { + u32 authkey_words = rctx->authklen / sizeof(u32); + + qce_cpu_to_be32p_array(mackey, rctx->authkey, rctx->authklen); + qce_write_array(qce, REG_AUTH_KEY0, (u32 *)mackey, + authkey_words); + } + + if (IS_CMAC(rctx->flags)) + goto go_proc; + + if (rctx->first_blk) + memcpy(auth, rctx->digest, digestsize); + else + qce_cpu_to_be32p_array(auth, rctx->digest, digestsize); + + iv_words = 8; + qce_write_array(qce, REG_AUTH_IV0, (u32 *)auth, iv_words); + + if (rctx->first_blk) + qce_clear_array(qce, REG_AUTH_BYTECNT0, 4); + else + qce_write_array(qce, REG_AUTH_BYTECNT0, + (u32 *)rctx->byte_count, 2); + + auth_cfg = qce_auth_cfg(rctx->flags, 0, digestsize); + + if (rctx->last_blk) + auth_cfg |= BIT(AUTH_LAST_SHIFT); + else + auth_cfg &= ~BIT(AUTH_LAST_SHIFT); + + if (rctx->first_blk) + auth_cfg |= BIT(AUTH_FIRST_SHIFT); + else + auth_cfg &= ~BIT(AUTH_FIRST_SHIFT); + +go_proc: + qce_write(qce, REG_AUTH_SEG_CFG, auth_cfg); + qce_write(qce, REG_AUTH_SEG_SIZE, req->nbytes); + qce_write(qce, REG_AUTH_SEG_START, 0); + qce_write(qce, REG_ENCR_SEG_CFG, 0); + qce_write(qce, REG_SEG_SIZE, req->nbytes); + + /* get little endianness */ + config = qce_config_reg(qce, 1); + qce_write(qce, REG_CONFIG, config); + + qce_crypto_go(qce, true); + + return 0; +} +#endif + +#if defined(CONFIG_CRYPTO_DEV_QCE_SKCIPHER) || defined(CONFIG_CRYPTO_DEV_QCE_AEAD) +static u32 qce_encr_cfg(unsigned long flags, u32 aes_key_size) +{ + u32 cfg = 0; + + if (IS_AES(flags)) { + if (aes_key_size == AES_KEYSIZE_128) + cfg |= ENCR_KEY_SZ_AES128 << ENCR_KEY_SZ_SHIFT; + else if (aes_key_size == AES_KEYSIZE_256) + cfg |= ENCR_KEY_SZ_AES256 << ENCR_KEY_SZ_SHIFT; + } + + if (IS_AES(flags)) + cfg |= ENCR_ALG_AES << ENCR_ALG_SHIFT; + + switch (flags & QCE_MODE_MASK) { + case QCE_MODE_CBC: + cfg |= ENCR_MODE_CBC << ENCR_MODE_SHIFT; + break; + case QCE_MODE_CTR: + cfg |= ENCR_MODE_CTR << ENCR_MODE_SHIFT; + break; + case QCE_MODE_XTS: + cfg |= ENCR_MODE_XTS << ENCR_MODE_SHIFT; + break; + case QCE_MODE_CCM: + cfg |= ENCR_MODE_CCM << ENCR_MODE_SHIFT; + cfg |= LAST_CCM_XFR << LAST_CCM_SHIFT; + break; + default: + return ~0; + } + + return cfg; +} +#endif + +#ifdef CONFIG_CRYPTO_DEV_QCE_SKCIPHER +static void qce_xts_swapiv(__be32 *dst, const u8 *src, unsigned int ivsize) +{ + u8 swap[QCE_AES_IV_LENGTH] = {0}; + unsigned int i, offset; + + if (ivsize > QCE_AES_IV_LENGTH) + return; + + offset = QCE_AES_IV_LENGTH - ivsize; + + /* Reverse and right-align IV bytes. */ + for (i = 0; i < ivsize; i++) + swap[offset + i] = src[ivsize - 1 - i]; + + qce_cpu_to_be32p_array(dst, swap, QCE_AES_IV_LENGTH); +} + +static void qce_xtskey(struct qce_device *qce, const u8 *enckey, + unsigned int enckeylen, unsigned int cryptlen) +{ + u32 xtskey[QCE_MAX_CIPHER_KEY_SIZE / sizeof(u32)] = {0}; + unsigned int xtsklen = enckeylen / (2 * sizeof(u32)); + + qce_cpu_to_be32p_array((__be32 *)xtskey, enckey + enckeylen / 2, + enckeylen / 2); + qce_write_array(qce, REG_ENCR_XTS_KEY0, xtskey, xtsklen); + + /* Set data unit size to cryptlen. Anything else causes + * crypto engine to return back incorrect results. + */ + qce_write(qce, REG_ENCR_XTS_DU_SIZE, cryptlen); +} + +static int qce_setup_regs_skcipher(struct crypto_async_request *async_req) +{ + struct skcipher_request *req = skcipher_request_cast(async_req); + struct qce_cipher_reqctx *rctx = skcipher_request_ctx(req); + struct qce_cipher_ctx *ctx = crypto_tfm_ctx(async_req->tfm); + struct qce_alg_template *tmpl = to_cipher_tmpl(crypto_skcipher_reqtfm(req)); + struct qce_device *qce = tmpl->qce; + __be32 enckey[QCE_MAX_CIPHER_KEY_SIZE / sizeof(__be32)] = {0}; + __be32 enciv[QCE_MAX_IV_SIZE / sizeof(__be32)] = {0}; + unsigned int enckey_words, enciv_words; + unsigned int keylen; + u32 encr_cfg = 0, auth_cfg = 0, config; + unsigned int ivsize = rctx->ivsize; + unsigned long flags = rctx->flags; + + qce_setup_config(qce); + + if (IS_XTS(flags)) + keylen = ctx->enc_keylen / 2; + else + keylen = ctx->enc_keylen; + + qce_cpu_to_be32p_array(enckey, ctx->enc_key, keylen); + enckey_words = keylen / sizeof(u32); + + qce_write(qce, REG_AUTH_SEG_CFG, auth_cfg); + + encr_cfg = qce_encr_cfg(flags, keylen); + + if (IS_AES(flags)) { + if (IS_XTS(flags)) + qce_xtskey(qce, ctx->enc_key, ctx->enc_keylen, + rctx->cryptlen); + enciv_words = 4; + } else { + return -EINVAL; + } + + qce_write_array(qce, REG_ENCR_KEY0, (u32 *)enckey, enckey_words); + + if (IS_XTS(flags)) + qce_xts_swapiv(enciv, rctx->iv, ivsize); + else + qce_cpu_to_be32p_array(enciv, rctx->iv, ivsize); + + qce_write_array(qce, REG_CNTR0_IV0, (u32 *)enciv, enciv_words); + + if (IS_ENCRYPT(flags)) + encr_cfg |= BIT(ENCODE_SHIFT); + + qce_write(qce, REG_ENCR_SEG_CFG, encr_cfg); + qce_write(qce, REG_ENCR_SEG_SIZE, rctx->cryptlen); + qce_write(qce, REG_ENCR_SEG_START, 0); + + if (IS_CTR(flags)) { + qce_write(qce, REG_CNTR_MASK, ~0); + qce_write(qce, REG_CNTR_MASK0, ~0); + qce_write(qce, REG_CNTR_MASK1, ~0); + qce_write(qce, REG_CNTR_MASK2, ~0); + } + + qce_write(qce, REG_SEG_SIZE, rctx->cryptlen); + + /* get little endianness */ + config = qce_config_reg(qce, 1); + qce_write(qce, REG_CONFIG, config); + + qce_crypto_go(qce, true); + + return 0; +} +#endif + +#ifdef CONFIG_CRYPTO_DEV_QCE_AEAD +static const u32 std_iv_sha256[SHA256_DIGEST_SIZE / sizeof(u32)] = { + SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, + SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7 +}; + +static unsigned int qce_be32_to_cpu_array(u32 *dst, const u8 *src, unsigned int len) +{ + u32 *d = dst; + const u8 *s = src; + unsigned int n; + + n = len / sizeof(u32); + for (; n > 0; n--) { + *d = be32_to_cpup((const __be32 *)s); + s += sizeof(u32); + d++; + } + return DIV_ROUND_UP(len, sizeof(u32)); +} + +static int qce_setup_regs_aead(struct crypto_async_request *async_req) +{ + struct aead_request *req = aead_request_cast(async_req); + struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); + struct qce_aead_ctx *ctx = crypto_tfm_ctx(async_req->tfm); + struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req)); + struct qce_device *qce = tmpl->qce; + u32 enckey[QCE_MAX_CIPHER_KEY_SIZE / sizeof(u32)] = {0}; + u32 enciv[QCE_MAX_IV_SIZE / sizeof(u32)] = {0}; + u32 authkey[QCE_SHA_HMAC_KEY_SIZE / sizeof(u32)] = {0}; + u32 authiv[SHA256_DIGEST_SIZE / sizeof(u32)] = {0}; + u32 authnonce[QCE_MAX_NONCE / sizeof(u32)] = {0}; + unsigned int enc_keylen = ctx->enc_keylen; + unsigned int auth_keylen = ctx->auth_keylen; + unsigned int enc_ivsize = rctx->ivsize; + unsigned int auth_ivsize = 0; + unsigned int enckey_words, enciv_words; + unsigned int authkey_words, authiv_words, authnonce_words; + unsigned long flags = rctx->flags; + u32 encr_cfg, auth_cfg, config, totallen; + u32 iv_last_word; + + qce_setup_config(qce); + + /* Write encryption key */ + enckey_words = qce_be32_to_cpu_array(enckey, ctx->enc_key, enc_keylen); + qce_write_array(qce, REG_ENCR_KEY0, enckey, enckey_words); + + /* Write encryption iv */ + enciv_words = qce_be32_to_cpu_array(enciv, rctx->iv, enc_ivsize); + qce_write_array(qce, REG_CNTR0_IV0, enciv, enciv_words); + + if (IS_CCM(rctx->flags)) { + iv_last_word = enciv[enciv_words - 1]; + qce_write(qce, REG_CNTR3_IV3, iv_last_word + 1); + qce_write_array(qce, REG_ENCR_CCM_INT_CNTR0, (u32 *)enciv, enciv_words); + qce_write(qce, REG_CNTR_MASK, ~0); + qce_write(qce, REG_CNTR_MASK0, ~0); + qce_write(qce, REG_CNTR_MASK1, ~0); + qce_write(qce, REG_CNTR_MASK2, ~0); + } + + /* Clear authentication IV and KEY registers of previous values */ + qce_clear_array(qce, REG_AUTH_IV0, 16); + qce_clear_array(qce, REG_AUTH_KEY0, 16); + + /* Clear byte count */ + qce_clear_array(qce, REG_AUTH_BYTECNT0, 4); + + /* Write authentication key */ + authkey_words = qce_be32_to_cpu_array(authkey, ctx->auth_key, auth_keylen); + qce_write_array(qce, REG_AUTH_KEY0, (u32 *)authkey, authkey_words); + + /* Write initial authentication IV only for HMAC algorithms */ + if (IS_SHA_HMAC(rctx->flags)) { + /* Write default authentication iv */ + auth_ivsize = SHA256_DIGEST_SIZE; + memcpy(authiv, std_iv_sha256, auth_ivsize); + authiv_words = auth_ivsize / sizeof(u32); + qce_write_array(qce, REG_AUTH_IV0, (u32 *)authiv, authiv_words); + } else if (IS_CCM(rctx->flags)) { + /* Write nonce for CCM algorithms */ + authnonce_words = qce_be32_to_cpu_array(authnonce, rctx->ccm_nonce, QCE_MAX_NONCE); + qce_write_array(qce, REG_AUTH_INFO_NONCE0, authnonce, authnonce_words); + } + + /* Set up ENCR_SEG_CFG */ + encr_cfg = qce_encr_cfg(flags, enc_keylen); + if (IS_ENCRYPT(flags)) + encr_cfg |= BIT(ENCODE_SHIFT); + qce_write(qce, REG_ENCR_SEG_CFG, encr_cfg); + + /* Set up AUTH_SEG_CFG */ + auth_cfg = qce_auth_cfg(rctx->flags, auth_keylen, ctx->authsize); + auth_cfg |= BIT(AUTH_LAST_SHIFT); + auth_cfg |= BIT(AUTH_FIRST_SHIFT); + if (IS_ENCRYPT(flags)) { + if (IS_CCM(rctx->flags)) + auth_cfg |= AUTH_POS_BEFORE << AUTH_POS_SHIFT; + else + auth_cfg |= AUTH_POS_AFTER << AUTH_POS_SHIFT; + } else { + if (IS_CCM(rctx->flags)) + auth_cfg |= AUTH_POS_AFTER << AUTH_POS_SHIFT; + else + auth_cfg |= AUTH_POS_BEFORE << AUTH_POS_SHIFT; + } + qce_write(qce, REG_AUTH_SEG_CFG, auth_cfg); + + totallen = rctx->cryptlen + rctx->assoclen; + + /* Set the encryption size and start offset */ + if (IS_CCM(rctx->flags) && IS_DECRYPT(rctx->flags)) + qce_write(qce, REG_ENCR_SEG_SIZE, rctx->cryptlen + ctx->authsize); + else + qce_write(qce, REG_ENCR_SEG_SIZE, rctx->cryptlen); + qce_write(qce, REG_ENCR_SEG_START, rctx->assoclen & 0xffff); + + /* Set the authentication size and start offset */ + qce_write(qce, REG_AUTH_SEG_SIZE, totallen); + qce_write(qce, REG_AUTH_SEG_START, 0); + + /* Write total length */ + if (IS_CCM(rctx->flags) && IS_DECRYPT(rctx->flags)) + qce_write(qce, REG_SEG_SIZE, totallen + ctx->authsize); + else + qce_write(qce, REG_SEG_SIZE, totallen); + + /* get little endianness */ + config = qce_config_reg(qce, 1); + qce_write(qce, REG_CONFIG, config); + + /* Start the process */ + qce_crypto_go(qce, !IS_CCM(flags)); + + return 0; +} +#endif + +int qce_start(struct crypto_async_request *async_req, u32 type) +{ + switch (type) { +#ifdef CONFIG_CRYPTO_DEV_QCE_SKCIPHER + case CRYPTO_ALG_TYPE_SKCIPHER: + return qce_setup_regs_skcipher(async_req); +#endif +#ifdef CONFIG_CRYPTO_DEV_QCE_SHA + case CRYPTO_ALG_TYPE_AHASH: + return qce_setup_regs_ahash(async_req); +#endif +#ifdef CONFIG_CRYPTO_DEV_QCE_AEAD + case CRYPTO_ALG_TYPE_AEAD: + return qce_setup_regs_aead(async_req); +#endif + default: + return -EINVAL; + } +} + +#define STATUS_ERRORS \ + (BIT(SW_ERR_SHIFT) | BIT(AXI_ERR_SHIFT) | BIT(HSD_ERR_SHIFT)) + +int qce_check_status(struct qce_device *qce, u32 *status) +{ + int ret = 0; + + *status = qce_read(qce, REG_STATUS); + + /* + * Don't use result dump status. The operation may not be complete. + * Instead, use the status we just read from device. In case, we need to + * use result_status from result dump the result_status needs to be byte + * swapped, since we set the device to little endian. + */ + if (*status & STATUS_ERRORS || !(*status & BIT(OPERATION_DONE_SHIFT))) + ret = -ENXIO; + else if (*status & BIT(MAC_FAILED_SHIFT)) + ret = -EBADMSG; + + return ret; +} + +void qce_get_version(struct qce_device *qce, u32 *major, u32 *minor, u32 *step) +{ + u32 val; + + val = qce_read(qce, REG_VERSION); + *major = (val & CORE_MAJOR_REV_MASK) >> CORE_MAJOR_REV_SHIFT; + *minor = (val & CORE_MINOR_REV_MASK) >> CORE_MINOR_REV_SHIFT; + *step = (val & CORE_STEP_REV_MASK) >> CORE_STEP_REV_SHIFT; +} diff --git a/drivers/crypto/qce/common.h b/drivers/crypto/qce/common.h new file mode 100644 index 000000000..9cd2e6ed8 --- /dev/null +++ b/drivers/crypto/qce/common.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2010-2014, The Linux Foundation. All rights reserved. + */ + +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include +#include +#include +#include +#include +#include + +/* xts du size */ +#define QCE_SECTOR_SIZE 512 + +/* key size in bytes */ +#define QCE_SHA_HMAC_KEY_SIZE 64 +#define QCE_MAX_CIPHER_KEY_SIZE AES_KEYSIZE_256 + +/* IV length in bytes */ +#define QCE_AES_IV_LENGTH AES_BLOCK_SIZE +/* max of AES_BLOCK_SIZE */ +#define QCE_MAX_IV_SIZE AES_BLOCK_SIZE + +/* maximum nonce bytes */ +#define QCE_MAX_NONCE 16 +#define QCE_MAX_NONCE_WORDS (QCE_MAX_NONCE / sizeof(u32)) + +/* burst size alignment requirement */ +#define QCE_MAX_ALIGN_SIZE 64 + +/* cipher algorithms */ +#define QCE_ALG_AES BIT(2) + +/* hash and hmac algorithms */ +#define QCE_HASH_SHA256 BIT(4) +#define QCE_HASH_SHA256_HMAC BIT(6) +#define QCE_HASH_AES_CMAC BIT(7) + +/* cipher modes */ +#define QCE_MODE_CBC BIT(8) +#define QCE_MODE_ECB BIT(9) +#define QCE_MODE_CTR BIT(10) +#define QCE_MODE_XTS BIT(11) +#define QCE_MODE_CCM BIT(12) +#define QCE_MODE_MASK GENMASK(12, 8) + +#define QCE_MODE_CCM_RFC4309 BIT(13) + +/* cipher encryption/decryption operations */ +#define QCE_ENCRYPT BIT(30) +#define QCE_DECRYPT BIT(31) + +#define IS_AES(flags) (flags & QCE_ALG_AES) + +#define IS_SHA256(flags) (flags & QCE_HASH_SHA256) +#define IS_SHA256_HMAC(flags) (flags & QCE_HASH_SHA256_HMAC) +#define IS_CMAC(flags) (flags & QCE_HASH_AES_CMAC) +#define IS_SHA(flags) IS_SHA256(flags) +#define IS_SHA_HMAC(flags) IS_SHA256_HMAC(flags) + +#define IS_CBC(mode) (mode & QCE_MODE_CBC) +#define IS_CTR(mode) (mode & QCE_MODE_CTR) +#define IS_XTS(mode) (mode & QCE_MODE_XTS) +#define IS_CCM(mode) (mode & QCE_MODE_CCM) +#define IS_CCM_RFC4309(mode) ((mode) & QCE_MODE_CCM_RFC4309) + +#define IS_ENCRYPT(dir) (dir & QCE_ENCRYPT) +#define IS_DECRYPT(dir) (dir & QCE_DECRYPT) + +struct qce_alg_template { + struct list_head entry; + u32 crypto_alg_type; + unsigned long alg_flags; + const u32 *std_iv; + union { + struct skcipher_alg skcipher; + struct ahash_alg ahash; + struct aead_alg aead; + } alg; + struct qce_device *qce; + const u8 *hash_zero; + const u32 digest_size; +}; + +void qce_cpu_to_be32p_array(__be32 *dst, const u8 *src, unsigned int len); +int qce_check_status(struct qce_device *qce, u32 *status); +void qce_get_version(struct qce_device *qce, u32 *major, u32 *minor, u32 *step); +int qce_start(struct crypto_async_request *async_req, u32 type); + +#endif /* _COMMON_H_ */ diff --git a/drivers/crypto/qce/core.c b/drivers/crypto/qce/core.c new file mode 100644 index 000000000..bf3c1bc3b --- /dev/null +++ b/drivers/crypto/qce/core.c @@ -0,0 +1,353 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2010-2014, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core.h" +#include "cipher.h" +#include "sha.h" +#include "aead.h" + +#define QCE_QUEUE_LENGTH 1 + +#define QCE_DEFAULT_MEM_BANDWIDTH 393600 + +static const struct qce_algo_ops *qce_ops[] = { +#ifdef CONFIG_CRYPTO_DEV_QCE_SKCIPHER + &skcipher_ops, +#endif +#ifdef CONFIG_CRYPTO_DEV_QCE_SHA + &ahash_ops, +#endif +#ifdef CONFIG_CRYPTO_DEV_QCE_AEAD + &aead_ops, +#endif +}; + +static void qce_unregister_algs(void *data) +{ + const struct qce_algo_ops *ops; + struct qce_device *qce = data; + int i; + + for (i = 0; i < ARRAY_SIZE(qce_ops); i++) { + ops = qce_ops[i]; + ops->unregister_algs(qce); + } +} + +static int devm_qce_register_algs(struct qce_device *qce) +{ + const struct qce_algo_ops *ops; + int i, ret; + + for (i = 0; i < ARRAY_SIZE(qce_ops); i++) { + ops = qce_ops[i]; + ret = ops->register_algs(qce); + if (ret) { + while (i--) + qce_ops[i]->unregister_algs(qce); + return ret; + } + } + + return devm_add_action_or_reset(qce->dev, qce_unregister_algs, qce); +} + +static int qce_handle_request(struct crypto_async_request *async_req) +{ + int i; + const struct qce_algo_ops *ops; + u32 type = crypto_tfm_alg_type(async_req->tfm); + + for (i = 0; i < ARRAY_SIZE(qce_ops); i++) { + ops = qce_ops[i]; + if (type == ops->type) + return ops->async_req_handle(async_req); + } + + return -EINVAL; +} + +static int qce_handle_queue(struct qce_device *qce, + struct crypto_async_request *req) +{ + struct crypto_async_request *async_req, *backlog; + int ret, err; + + PM_RUNTIME_ACQUIRE_AUTOSUSPEND(qce->dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret) + return ret; + + scoped_guard(mutex, &qce->lock) { + if (req) + ret = crypto_enqueue_request(&qce->queue, req); + + /* busy, do not dequeue request */ + if (qce->req) + return ret; + + backlog = crypto_get_backlog(&qce->queue); + async_req = crypto_dequeue_request(&qce->queue); + if (async_req) + qce->req = async_req; + } + + if (!async_req) + return ret; + + if (backlog) { + scoped_guard(mutex, &qce->lock) + crypto_request_complete(backlog, -EINPROGRESS); + } + + err = qce_handle_request(async_req); + if (err) { + qce->result = err; + schedule_work(&qce->done_work); + } + + return ret; +} + +static void qce_req_done_work(struct work_struct *work) +{ + struct qce_device *qce = container_of(work, struct qce_device, + done_work); + struct crypto_async_request *req; + + scoped_guard(mutex, &qce->lock) { + req = qce->req; + qce->req = NULL; + } + + if (req) + crypto_request_complete(req, qce->result); + + qce_handle_queue(qce, NULL); +} + +static int qce_async_request_enqueue(struct qce_device *qce, + struct crypto_async_request *req) +{ + return qce_handle_queue(qce, req); +} + +static void qce_async_request_done(struct qce_device *qce, int ret) +{ + qce->result = ret; + schedule_work(&qce->done_work); +} + +static int qce_check_version(struct qce_device *qce) +{ + u32 major, minor, step; + + qce_get_version(qce, &major, &minor, &step); + + /* + * the driver does not support v5 with minor 0 because it has special + * alignment requirements. + */ + if (major == 5 && minor == 0) + return -ENODEV; + + qce->burst_size = QCE_BAM_BURST_SIZE; + + /* + * Rx and tx pipes are treated as a pair inside CE. + * Pipe pair number depends on the actual BAM dma pipe + * that is used for transfers. The BAM dma pipes are passed + * from the device tree and used to derive the pipe pair + * id in the CE driver as follows. + * BAM dma pipes(rx, tx) CE pipe pair id + * 0,1 0 + * 2,3 1 + * 4,5 2 + * 6,7 3 + * ... + */ + qce->pipe_pair_id = qce->dma.rxchan->chan_id >> 1; + + dev_dbg(qce->dev, "Crypto device found, version %d.%d.%d\n", + major, minor, step); + + return 0; +} + +static int qce_crypto_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct qce_device *qce; + int ret; + + qce = devm_kzalloc(dev, sizeof(*qce), GFP_KERNEL); + if (!qce) + return -ENOMEM; + + qce->dev = dev; + platform_set_drvdata(pdev, qce); + + qce->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(qce->base)) + return PTR_ERR(qce->base); + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + if (ret < 0) + return ret; + + qce->core = devm_clk_get_optional(qce->dev, "core"); + if (IS_ERR(qce->core)) + return PTR_ERR(qce->core); + + qce->iface = devm_clk_get_optional(qce->dev, "iface"); + if (IS_ERR(qce->iface)) + return PTR_ERR(qce->iface); + + qce->bus = devm_clk_get_optional(qce->dev, "bus"); + if (IS_ERR(qce->bus)) + return PTR_ERR(qce->bus); + + qce->mem_path = devm_of_icc_get(dev, "memory"); + if (IS_ERR(qce->mem_path)) + return PTR_ERR(qce->mem_path); + + /* + * Enable runtime PM after clocks and ICC path are acquired so that + * the resume callback can enable clocks and apply the ICC bandwidth + * vote before any hardware access takes place. + */ + ret = devm_pm_runtime_enable(dev); + if (ret) + return ret; + + PM_RUNTIME_ACQUIRE_AUTOSUSPEND(dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret) + return ret; + + ret = devm_qce_dma_request(qce->dev, &qce->dma); + if (ret) + return ret; + + ret = qce_check_version(qce); + if (ret) + return ret; + + ret = devm_mutex_init(qce->dev, &qce->lock); + if (ret) + return ret; + + INIT_WORK(&qce->done_work, qce_req_done_work); + crypto_init_queue(&qce->queue, QCE_QUEUE_LENGTH); + + qce->async_req_enqueue = qce_async_request_enqueue; + qce->async_req_done = qce_async_request_done; + + ret = devm_qce_register_algs(qce); + if (ret) + return ret; + + /* Configure autosuspend after successful init */ + pm_runtime_set_autosuspend_delay(dev, 100); + pm_runtime_use_autosuspend(dev); + pm_runtime_mark_last_busy(dev); + + return 0; +} + +static int qce_runtime_suspend(struct device *dev) +{ + struct qce_device *qce = dev_get_drvdata(dev); + int ret; + + clk_disable_unprepare(qce->core); + clk_disable_unprepare(qce->iface); + clk_disable_unprepare(qce->bus); + + ret = icc_set_bw(qce->mem_path, 0, 0); + if (ret) { + clk_prepare_enable(qce->bus); + clk_prepare_enable(qce->iface); + clk_prepare_enable(qce->core); + return ret; + } + + return 0; +} + +static int qce_runtime_resume(struct device *dev) +{ + struct qce_device *qce = dev_get_drvdata(dev); + int ret; + + ret = icc_set_bw(qce->mem_path, QCE_DEFAULT_MEM_BANDWIDTH, + QCE_DEFAULT_MEM_BANDWIDTH); + if (ret) + return ret; + + ret = clk_prepare_enable(qce->core); + if (ret) + goto err_core; + + ret = clk_prepare_enable(qce->iface); + if (ret) + goto err_iface; + + ret = clk_prepare_enable(qce->bus); + if (ret) + goto err_bus; + + return 0; + +err_bus: + clk_disable_unprepare(qce->iface); +err_iface: + clk_disable_unprepare(qce->core); +err_core: + icc_set_bw(qce->mem_path, 0, 0); + return ret; +} + +static const struct dev_pm_ops qce_crypto_pm_ops = { + RUNTIME_PM_OPS(qce_runtime_suspend, qce_runtime_resume, NULL) + SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) +}; + +static const struct of_device_id qce_crypto_of_match[] = { + { .compatible = "qcom,crypto-v5.1", }, + { .compatible = "qcom,crypto-v5.4", }, + { .compatible = "qcom,qce", }, + {} +}; +MODULE_DEVICE_TABLE(of, qce_crypto_of_match); + +static struct platform_driver qce_crypto_driver = { + .probe = qce_crypto_probe, + .driver = { + .name = KBUILD_MODNAME, + .of_match_table = qce_crypto_of_match, + .pm = pm_ptr(&qce_crypto_pm_ops), + }, +}; +module_platform_driver(qce_crypto_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Qualcomm crypto engine driver"); +MODULE_ALIAS("platform:" KBUILD_MODNAME); +MODULE_AUTHOR("The Linux Foundation"); diff --git a/drivers/crypto/qce/core.h b/drivers/crypto/qce/core.h new file mode 100644 index 000000000..eb6fa7a8b --- /dev/null +++ b/drivers/crypto/qce/core.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2010-2014, The Linux Foundation. All rights reserved. + */ + +#ifndef _CORE_H_ +#define _CORE_H_ + +#include +#include + +#include "dma.h" + +/** + * struct qce_device - crypto engine device structure + * @queue: crypto request queue + * @lock: the lock protects queue and req + * @done_work: workqueue context + * @req: current active request + * @result: result of current transform + * @base: virtual IO base + * @dev: pointer to device structure + * @core: core device clock + * @iface: interface clock + * @bus: bus clock + * @dma: pointer to dma data + * @burst_size: the crypto burst size + * @pipe_pair_id: which pipe pair id the device using + * @async_req_enqueue: invoked by every algorithm to enqueue a request + * @async_req_done: invoked by every algorithm to finish its request + */ +struct qce_device { + struct crypto_queue queue; + struct mutex lock; + struct work_struct done_work; + struct crypto_async_request *req; + int result; + void __iomem *base; + struct device *dev; + struct clk *core, *iface, *bus; + struct icc_path *mem_path; + struct qce_dma_data dma; + int burst_size; + unsigned int pipe_pair_id; + int (*async_req_enqueue)(struct qce_device *qce, + struct crypto_async_request *req); + void (*async_req_done)(struct qce_device *qce, int ret); +}; + +/** + * struct qce_algo_ops - algorithm operations per crypto type + * @type: should be CRYPTO_ALG_TYPE_XXX + * @register_algs: invoked by core to register the algorithms + * @unregister_algs: invoked by core to unregister the algorithms + * @async_req_handle: invoked by core to handle enqueued request + */ +struct qce_algo_ops { + u32 type; + int (*register_algs)(struct qce_device *qce); + void (*unregister_algs)(struct qce_device *qce); + int (*async_req_handle)(struct crypto_async_request *async_req); +}; + +#endif /* _CORE_H_ */ diff --git a/drivers/crypto/qce/dma.c b/drivers/crypto/qce/dma.c new file mode 100644 index 000000000..68cafd474 --- /dev/null +++ b/drivers/crypto/qce/dma.c @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2012-2014, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include + +#include "dma.h" + +static void qce_dma_release(void *data) +{ + struct qce_dma_data *dma = data; + + dma_release_channel(dma->txchan); + dma_release_channel(dma->rxchan); + kfree(dma->result_buf); +} + +int devm_qce_dma_request(struct device *dev, struct qce_dma_data *dma) +{ + int ret; + + dma->txchan = dma_request_chan(dev, "tx"); + if (IS_ERR(dma->txchan)) + return dev_err_probe(dev, PTR_ERR(dma->txchan), + "Failed to get TX DMA channel\n"); + + dma->rxchan = dma_request_chan(dev, "rx"); + if (IS_ERR(dma->rxchan)) { + ret = dev_err_probe(dev, PTR_ERR(dma->rxchan), + "Failed to get RX DMA channel\n"); + goto error_rx; + } + + dma->result_buf = kmalloc(QCE_RESULT_BUF_SZ + QCE_IGNORE_BUF_SZ, + GFP_KERNEL); + if (!dma->result_buf) { + ret = -ENOMEM; + goto error_nomem; + } + + dma->ignore_buf = dma->result_buf + QCE_RESULT_BUF_SZ; + + return devm_add_action_or_reset(dev, qce_dma_release, dma); + +error_nomem: + dma_release_channel(dma->rxchan); +error_rx: + dma_release_channel(dma->txchan); + return ret; +} + +struct scatterlist * +qce_sgtable_add(struct sg_table *sgt, struct scatterlist *new_sgl, + unsigned int max_len) +{ + struct scatterlist *sg = sgt->sgl, *sg_last = NULL; + unsigned int new_len; + + while (sg) { + if (!sg_page(sg)) + break; + sg = sg_next(sg); + } + + if (!sg) + return ERR_PTR(-EINVAL); + + while (new_sgl && sg && max_len) { + new_len = new_sgl->length > max_len ? max_len : new_sgl->length; + sg_set_page(sg, sg_page(new_sgl), new_len, new_sgl->offset); + sg_last = sg; + sg = sg_next(sg); + new_sgl = sg_next(new_sgl); + max_len -= new_len; + } + + return sg_last; +} + +static int qce_dma_prep_sg(struct dma_chan *chan, struct scatterlist *sg, + int nents, unsigned long flags, + enum dma_transfer_direction dir, + dma_async_tx_callback cb, void *cb_param) +{ + struct dma_async_tx_descriptor *desc; + dma_cookie_t cookie; + + if (!sg || !nents) + return -EINVAL; + + desc = dmaengine_prep_slave_sg(chan, sg, nents, dir, flags); + if (!desc) + return -EINVAL; + + desc->callback = cb; + desc->callback_param = cb_param; + cookie = dmaengine_submit(desc); + + return dma_submit_error(cookie); +} + +int qce_dma_prep_sgs(struct qce_dma_data *dma, struct scatterlist *rx_sg, + int rx_nents, struct scatterlist *tx_sg, int tx_nents, + dma_async_tx_callback cb, void *cb_param) +{ + struct dma_chan *rxchan = dma->rxchan; + struct dma_chan *txchan = dma->txchan; + unsigned long flags = DMA_PREP_INTERRUPT | DMA_CTRL_ACK; + int ret; + + ret = qce_dma_prep_sg(rxchan, rx_sg, rx_nents, flags, DMA_MEM_TO_DEV, + NULL, NULL); + if (ret) + return ret; + + return qce_dma_prep_sg(txchan, tx_sg, tx_nents, flags, DMA_DEV_TO_MEM, + cb, cb_param); +} + +void qce_dma_issue_pending(struct qce_dma_data *dma) +{ + dma_async_issue_pending(dma->rxchan); + dma_async_issue_pending(dma->txchan); +} + +int qce_dma_terminate_all(struct qce_dma_data *dma) +{ + int ret; + + ret = dmaengine_terminate_all(dma->rxchan); + return ret ?: dmaengine_terminate_all(dma->txchan); +} diff --git a/drivers/crypto/qce/dma.h b/drivers/crypto/qce/dma.h new file mode 100644 index 000000000..316291850 --- /dev/null +++ b/drivers/crypto/qce/dma.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2011-2014, The Linux Foundation. All rights reserved. + */ + +#ifndef _DMA_H_ +#define _DMA_H_ + +#include + +/* maximum data transfer block size between BAM and CE */ +#define QCE_BAM_BURST_SIZE 64 + +#define QCE_AUTHIV_REGS_CNT 16 +#define QCE_AUTH_BYTECOUNT_REGS_CNT 4 +#define QCE_CNTRIV_REGS_CNT 4 + +struct qce_result_dump { + u32 auth_iv[QCE_AUTHIV_REGS_CNT]; + u32 auth_byte_count[QCE_AUTH_BYTECOUNT_REGS_CNT]; + u32 encr_cntr_iv[QCE_CNTRIV_REGS_CNT]; + u32 status; + u32 status2; +}; + +#define QCE_IGNORE_BUF_SZ (2 * QCE_BAM_BURST_SIZE) +#define QCE_RESULT_BUF_SZ \ + ALIGN(sizeof(struct qce_result_dump), QCE_BAM_BURST_SIZE) + +struct qce_dma_data { + struct dma_chan *txchan; + struct dma_chan *rxchan; + struct qce_result_dump *result_buf; + void *ignore_buf; +}; + +int devm_qce_dma_request(struct device *dev, struct qce_dma_data *dma); +int qce_dma_prep_sgs(struct qce_dma_data *dma, struct scatterlist *sg_in, + int in_ents, struct scatterlist *sg_out, int out_ents, + dma_async_tx_callback cb, void *cb_param); +void qce_dma_issue_pending(struct qce_dma_data *dma); +int qce_dma_terminate_all(struct qce_dma_data *dma); +struct scatterlist * +qce_sgtable_add(struct sg_table *sgt, struct scatterlist *sg_add, + unsigned int max_len); + +#endif /* _DMA_H_ */ diff --git a/drivers/crypto/qce/regs-v5.h b/drivers/crypto/qce/regs-v5.h new file mode 100644 index 000000000..431a7db1a --- /dev/null +++ b/drivers/crypto/qce/regs-v5.h @@ -0,0 +1,322 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2012-2014, The Linux Foundation. All rights reserved. + */ + +#ifndef _REGS_V5_H_ +#define _REGS_V5_H_ + +#include + +#define REG_VERSION 0x000 +#define REG_STATUS 0x100 +#define REG_STATUS2 0x104 +#define REG_ENGINES_AVAIL 0x108 +#define REG_FIFO_SIZES 0x10c +#define REG_SEG_SIZE 0x110 +#define REG_GOPROC 0x120 +#define REG_ENCR_SEG_CFG 0x200 +#define REG_ENCR_SEG_SIZE 0x204 +#define REG_ENCR_SEG_START 0x208 +#define REG_CNTR0_IV0 0x20c +#define REG_CNTR1_IV1 0x210 +#define REG_CNTR2_IV2 0x214 +#define REG_CNTR3_IV3 0x218 +#define REG_CNTR_MASK 0x21C +#define REG_ENCR_CCM_INT_CNTR0 0x220 +#define REG_ENCR_CCM_INT_CNTR1 0x224 +#define REG_ENCR_CCM_INT_CNTR2 0x228 +#define REG_ENCR_CCM_INT_CNTR3 0x22c +#define REG_ENCR_XTS_DU_SIZE 0x230 +#define REG_CNTR_MASK2 0x234 +#define REG_CNTR_MASK1 0x238 +#define REG_CNTR_MASK0 0x23c +#define REG_AUTH_SEG_CFG 0x300 +#define REG_AUTH_SEG_SIZE 0x304 +#define REG_AUTH_SEG_START 0x308 +#define REG_AUTH_IV0 0x310 +#define REG_AUTH_IV1 0x314 +#define REG_AUTH_IV2 0x318 +#define REG_AUTH_IV3 0x31c +#define REG_AUTH_IV4 0x320 +#define REG_AUTH_IV5 0x324 +#define REG_AUTH_IV6 0x328 +#define REG_AUTH_IV7 0x32c +#define REG_AUTH_IV8 0x330 +#define REG_AUTH_IV9 0x334 +#define REG_AUTH_IV10 0x338 +#define REG_AUTH_IV11 0x33c +#define REG_AUTH_IV12 0x340 +#define REG_AUTH_IV13 0x344 +#define REG_AUTH_IV14 0x348 +#define REG_AUTH_IV15 0x34c +#define REG_AUTH_INFO_NONCE0 0x350 +#define REG_AUTH_INFO_NONCE1 0x354 +#define REG_AUTH_INFO_NONCE2 0x358 +#define REG_AUTH_INFO_NONCE3 0x35c +#define REG_AUTH_BYTECNT0 0x390 +#define REG_AUTH_BYTECNT1 0x394 +#define REG_AUTH_BYTECNT2 0x398 +#define REG_AUTH_BYTECNT3 0x39c +#define REG_AUTH_EXP_MAC0 0x3a0 +#define REG_AUTH_EXP_MAC1 0x3a4 +#define REG_AUTH_EXP_MAC2 0x3a8 +#define REG_AUTH_EXP_MAC3 0x3ac +#define REG_AUTH_EXP_MAC4 0x3b0 +#define REG_AUTH_EXP_MAC5 0x3b4 +#define REG_AUTH_EXP_MAC6 0x3b8 +#define REG_AUTH_EXP_MAC7 0x3bc +#define REG_CONFIG 0x400 +#define REG_GOPROC_QC_KEY 0x1000 +#define REG_GOPROC_OEM_KEY 0x2000 +#define REG_ENCR_KEY0 0x3000 +#define REG_ENCR_KEY1 0x3004 +#define REG_ENCR_KEY2 0x3008 +#define REG_ENCR_KEY3 0x300c +#define REG_ENCR_KEY4 0x3010 +#define REG_ENCR_KEY5 0x3014 +#define REG_ENCR_KEY6 0x3018 +#define REG_ENCR_KEY7 0x301c +#define REG_ENCR_XTS_KEY0 0x3020 +#define REG_ENCR_XTS_KEY1 0x3024 +#define REG_ENCR_XTS_KEY2 0x3028 +#define REG_ENCR_XTS_KEY3 0x302c +#define REG_ENCR_XTS_KEY4 0x3030 +#define REG_ENCR_XTS_KEY5 0x3034 +#define REG_ENCR_XTS_KEY6 0x3038 +#define REG_ENCR_XTS_KEY7 0x303c +#define REG_AUTH_KEY0 0x3040 +#define REG_AUTH_KEY1 0x3044 +#define REG_AUTH_KEY2 0x3048 +#define REG_AUTH_KEY3 0x304c +#define REG_AUTH_KEY4 0x3050 +#define REG_AUTH_KEY5 0x3054 +#define REG_AUTH_KEY6 0x3058 +#define REG_AUTH_KEY7 0x305c +#define REG_AUTH_KEY8 0x3060 +#define REG_AUTH_KEY9 0x3064 +#define REG_AUTH_KEY10 0x3068 +#define REG_AUTH_KEY11 0x306c +#define REG_AUTH_KEY12 0x3070 +#define REG_AUTH_KEY13 0x3074 +#define REG_AUTH_KEY14 0x3078 +#define REG_AUTH_KEY15 0x307c + +/* Register bits - REG_VERSION */ +#define CORE_STEP_REV_SHIFT 0 +#define CORE_STEP_REV_MASK GENMASK(15, 0) +#define CORE_MINOR_REV_SHIFT 16 +#define CORE_MINOR_REV_MASK GENMASK(23, 16) +#define CORE_MAJOR_REV_SHIFT 24 +#define CORE_MAJOR_REV_MASK GENMASK(31, 24) + +/* Register bits - REG_STATUS */ +#define MAC_FAILED_SHIFT 31 +#define DOUT_SIZE_AVAIL_SHIFT 26 +#define DOUT_SIZE_AVAIL_MASK GENMASK(30, 26) +#define DIN_SIZE_AVAIL_SHIFT 21 +#define DIN_SIZE_AVAIL_MASK GENMASK(25, 21) +#define HSD_ERR_SHIFT 20 +#define ACCESS_VIOL_SHIFT 19 +#define PIPE_ACTIVE_ERR_SHIFT 18 +#define CFG_CHNG_ERR_SHIFT 17 +#define DOUT_ERR_SHIFT 16 +#define DIN_ERR_SHIFT 15 +#define AXI_ERR_SHIFT 14 +#define CRYPTO_STATE_SHIFT 10 +#define CRYPTO_STATE_MASK GENMASK(13, 10) +#define ENCR_BUSY_SHIFT 9 +#define AUTH_BUSY_SHIFT 8 +#define DOUT_INTR_SHIFT 7 +#define DIN_INTR_SHIFT 6 +#define OP_DONE_INTR_SHIFT 5 +#define ERR_INTR_SHIFT 4 +#define DOUT_RDY_SHIFT 3 +#define DIN_RDY_SHIFT 2 +#define OPERATION_DONE_SHIFT 1 +#define SW_ERR_SHIFT 0 + +/* Register bits - REG_STATUS2 */ +#define AXI_EXTRA_SHIFT 1 +#define LOCKED_SHIFT 2 + +/* Register bits - REG_CONFIG */ +#define REQ_SIZE_SHIFT 17 +#define REQ_SIZE_MASK GENMASK(20, 17) +#define REQ_SIZE_ENUM_1_BEAT 0 +#define REQ_SIZE_ENUM_2_BEAT 1 +#define REQ_SIZE_ENUM_3_BEAT 2 +#define REQ_SIZE_ENUM_4_BEAT 3 +#define REQ_SIZE_ENUM_5_BEAT 4 +#define REQ_SIZE_ENUM_6_BEAT 5 +#define REQ_SIZE_ENUM_7_BEAT 6 +#define REQ_SIZE_ENUM_8_BEAT 7 +#define REQ_SIZE_ENUM_9_BEAT 8 +#define REQ_SIZE_ENUM_10_BEAT 9 +#define REQ_SIZE_ENUM_11_BEAT 10 +#define REQ_SIZE_ENUM_12_BEAT 11 +#define REQ_SIZE_ENUM_13_BEAT 12 +#define REQ_SIZE_ENUM_14_BEAT 13 +#define REQ_SIZE_ENUM_15_BEAT 14 +#define REQ_SIZE_ENUM_16_BEAT 15 + +#define MAX_QUEUED_REQ_SHIFT 14 +#define MAX_QUEUED_REQ_MASK GENMASK(24, 16) +#define ENUM_1_QUEUED_REQS 0 +#define ENUM_2_QUEUED_REQS 1 +#define ENUM_3_QUEUED_REQS 2 + +#define IRQ_ENABLES_SHIFT 10 +#define IRQ_ENABLES_MASK GENMASK(13, 10) + +#define LITTLE_ENDIAN_MODE_SHIFT 9 +#define PIPE_SET_SELECT_SHIFT 5 +#define PIPE_SET_SELECT_MASK GENMASK(8, 5) + +#define HIGH_SPD_EN_N_SHIFT 4 +#define MASK_DOUT_INTR_SHIFT 3 +#define MASK_DIN_INTR_SHIFT 2 +#define MASK_OP_DONE_INTR_SHIFT 1 +#define MASK_ERR_INTR_SHIFT 0 + +/* Register bits - REG_AUTH_SEG_CFG */ +#define COMP_EXP_MAC_SHIFT 24 +#define COMP_EXP_MAC_DISABLED 0 +#define COMP_EXP_MAC_ENABLED 1 + +#define F9_DIRECTION_SHIFT 23 +#define F9_DIRECTION_UPLINK 0 +#define F9_DIRECTION_DOWNLINK 1 + +#define AUTH_NONCE_NUM_WORDS_SHIFT 20 +#define AUTH_NONCE_NUM_WORDS_MASK GENMASK(22, 20) + +#define USE_PIPE_KEY_AUTH_SHIFT 19 +#define USE_HW_KEY_AUTH_SHIFT 18 +#define AUTH_FIRST_SHIFT 17 +#define AUTH_LAST_SHIFT 16 + +#define AUTH_POS_SHIFT 14 +#define AUTH_POS_MASK GENMASK(15, 14) +#define AUTH_POS_BEFORE 0 +#define AUTH_POS_AFTER 1 + +#define AUTH_SIZE_SHIFT 9 +#define AUTH_SIZE_MASK GENMASK(13, 9) +#define AUTH_SIZE_SHA256 1 +#define AUTH_SIZE_ENUM_1_BYTES 0 +#define AUTH_SIZE_ENUM_2_BYTES 1 +#define AUTH_SIZE_ENUM_3_BYTES 2 +#define AUTH_SIZE_ENUM_4_BYTES 3 +#define AUTH_SIZE_ENUM_5_BYTES 4 +#define AUTH_SIZE_ENUM_6_BYTES 5 +#define AUTH_SIZE_ENUM_7_BYTES 6 +#define AUTH_SIZE_ENUM_8_BYTES 7 +#define AUTH_SIZE_ENUM_9_BYTES 8 +#define AUTH_SIZE_ENUM_10_BYTES 9 +#define AUTH_SIZE_ENUM_11_BYTES 10 +#define AUTH_SIZE_ENUM_12_BYTES 11 +#define AUTH_SIZE_ENUM_13_BYTES 12 +#define AUTH_SIZE_ENUM_14_BYTES 13 +#define AUTH_SIZE_ENUM_15_BYTES 14 +#define AUTH_SIZE_ENUM_16_BYTES 15 + +#define AUTH_MODE_SHIFT 6 +#define AUTH_MODE_MASK GENMASK(8, 6) +#define AUTH_MODE_HASH 0 +#define AUTH_MODE_HMAC 1 +#define AUTH_MODE_CCM 0 +#define AUTH_MODE_CMAC 1 + +#define AUTH_KEY_SIZE_SHIFT 3 +#define AUTH_KEY_SIZE_MASK GENMASK(5, 3) +#define AUTH_KEY_SZ_AES128 0 +#define AUTH_KEY_SZ_AES256 2 + +#define AUTH_ALG_SHIFT 0 +#define AUTH_ALG_MASK GENMASK(2, 0) +#define AUTH_ALG_NONE 0 +#define AUTH_ALG_SHA 1 +#define AUTH_ALG_AES 2 +#define AUTH_ALG_KASUMI 3 +#define AUTH_ALG_SNOW3G 4 +#define AUTH_ALG_ZUC 5 + +/* Register bits - REG_ENCR_XTS_DU_SIZE */ +#define ENCR_XTS_DU_SIZE_SHIFT 0 +#define ENCR_XTS_DU_SIZE_MASK GENMASK(19, 0) + +/* Register bits - REG_ENCR_SEG_CFG */ +#define F8_KEYSTREAM_ENABLE_SHIFT 17 +#define F8_KEYSTREAM_DISABLED 0 +#define F8_KEYSTREAM_ENABLED 1 + +#define F8_DIRECTION_SHIFT 16 +#define F8_DIRECTION_UPLINK 0 +#define F8_DIRECTION_DOWNLINK 1 + +#define USE_PIPE_KEY_ENCR_SHIFT 15 +#define USE_PIPE_KEY_ENCR_ENABLED 1 +#define USE_KEY_REGISTERS 0 + +#define USE_HW_KEY_ENCR_SHIFT 14 +#define USE_KEY_REG 0 +#define USE_HW_KEY 1 + +#define LAST_CCM_SHIFT 13 +#define LAST_CCM_XFR 1 +#define INTERM_CCM_XFR 0 + +#define CNTR_ALG_SHIFT 11 +#define CNTR_ALG_MASK GENMASK(12, 11) +#define CNTR_ALG_NIST 0 + +#define ENCODE_SHIFT 10 + +#define ENCR_MODE_SHIFT 6 +#define ENCR_MODE_MASK GENMASK(9, 6) +#define ENCR_MODE_ECB 0 +#define ENCR_MODE_CBC 1 +#define ENCR_MODE_CTR 2 +#define ENCR_MODE_XTS 3 +#define ENCR_MODE_CCM 4 + +#define ENCR_KEY_SZ_SHIFT 3 +#define ENCR_KEY_SZ_MASK GENMASK(5, 3) +#define ENCR_KEY_SZ_AES128 0 +#define ENCR_KEY_SZ_AES256 2 + +#define ENCR_ALG_SHIFT 0 +#define ENCR_ALG_MASK GENMASK(2, 0) +#define ENCR_ALG_NONE 0 +#define ENCR_ALG_AES 2 +#define ENCR_ALG_KASUMI 4 +#define ENCR_ALG_SNOW_3G 5 +#define ENCR_ALG_ZUC 6 + +/* Register bits - REG_GOPROC */ +#define GO_SHIFT 0 +#define CLR_CNTXT_SHIFT 1 +#define RESULTS_DUMP_SHIFT 2 + +/* Register bits - REG_ENGINES_AVAIL */ +#define ENCR_AES_SEL_SHIFT 0 +#define DES_SEL_SHIFT 1 +#define ENCR_SNOW3G_SEL_SHIFT 2 +#define ENCR_KASUMI_SEL_SHIFT 3 +#define SHA_SEL_SHIFT 4 +#define SHA512_SEL_SHIFT 5 +#define AUTH_AES_SEL_SHIFT 6 +#define AUTH_SNOW3G_SEL_SHIFT 7 +#define AUTH_KASUMI_SEL_SHIFT 8 +#define BAM_PIPE_SETS_SHIFT 9 +#define BAM_PIPE_SETS_MASK GENMASK(12, 9) +#define AXI_WR_BEATS_SHIFT 13 +#define AXI_WR_BEATS_MASK GENMASK(18, 13) +#define AXI_RD_BEATS_SHIFT 19 +#define AXI_RD_BEATS_MASK GENMASK(24, 19) +#define ENCR_ZUC_SEL_SHIFT 26 +#define AUTH_ZUC_SEL_SHIFT 27 +#define ZUC_ENABLE_SHIFT 28 + +#endif /* _REGS_V5_H_ */ diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c new file mode 100644 index 000000000..eb8a5b0c4 --- /dev/null +++ b/drivers/crypto/qce/sha.c @@ -0,0 +1,498 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2010-2014, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include + +#include "common.h" +#include "core.h" +#include "sha.h" + +struct qce_sha_saved_state { + u8 pending_buf[QCE_SHA_MAX_BLOCKSIZE]; + u8 partial_digest[QCE_SHA_MAX_DIGESTSIZE]; + __be32 byte_count[2]; + unsigned int pending_buflen; + unsigned int flags; + u64 count; + bool first_blk; +}; + +static LIST_HEAD(ahash_algs); + +static const u32 std_iv_sha256[SHA256_DIGEST_SIZE / sizeof(u32)] = { + SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, + SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7 +}; + +static void qce_ahash_done(void *data) +{ + struct crypto_async_request *async_req = data; + struct ahash_request *req = ahash_request_cast(async_req); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req); + struct qce_alg_template *tmpl = to_ahash_tmpl(async_req->tfm); + struct qce_device *qce = tmpl->qce; + struct qce_result_dump *result = qce->dma.result_buf; + unsigned int digestsize = crypto_ahash_digestsize(ahash); + int error; + u32 status; + + error = qce_dma_terminate_all(&qce->dma); + if (error) + dev_dbg(qce->dev, "ahash dma termination error (%d)\n", error); + + dma_unmap_sg(qce->dev, req->src, rctx->src_nents, DMA_TO_DEVICE); + dma_unmap_sg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE); + + memcpy(rctx->digest, result->auth_iv, digestsize); + if (req->result && rctx->last_blk) + memcpy(req->result, result->auth_iv, digestsize); + + rctx->byte_count[0] = cpu_to_be32(result->auth_byte_count[0]); + rctx->byte_count[1] = cpu_to_be32(result->auth_byte_count[1]); + + error = qce_check_status(qce, &status); + if (error < 0) + dev_dbg(qce->dev, "ahash operation error (%x)\n", status); + + req->src = rctx->src_orig; + req->nbytes = rctx->nbytes_orig; + rctx->last_blk = false; + rctx->first_blk = false; + + qce->async_req_done(tmpl->qce, error); +} + +static int qce_ahash_async_req_handle(struct crypto_async_request *async_req) +{ + struct ahash_request *req = ahash_request_cast(async_req); + struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req); + struct qce_sha_ctx *ctx = crypto_tfm_ctx(async_req->tfm); + struct qce_alg_template *tmpl = to_ahash_tmpl(async_req->tfm); + struct qce_device *qce = tmpl->qce; + unsigned long flags = rctx->flags; + int ret; + + if (IS_SHA_HMAC(flags)) { + rctx->authkey = ctx->authkey; + rctx->authklen = QCE_SHA_HMAC_KEY_SIZE; + } else if (IS_CMAC(flags)) { + rctx->authkey = ctx->authkey; + rctx->authklen = AES_KEYSIZE_128; + } + + rctx->src_nents = sg_nents_for_len(req->src, req->nbytes); + if (rctx->src_nents < 0) { + dev_err(qce->dev, "Invalid numbers of src SG.\n"); + return rctx->src_nents; + } + + ret = dma_map_sg(qce->dev, req->src, rctx->src_nents, DMA_TO_DEVICE); + if (!ret) + return -EIO; + + sg_init_one(&rctx->result_sg, qce->dma.result_buf, QCE_RESULT_BUF_SZ); + + ret = dma_map_sg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE); + if (!ret) { + ret = -EIO; + goto error_unmap_src; + } + + ret = qce_dma_prep_sgs(&qce->dma, req->src, rctx->src_nents, + &rctx->result_sg, 1, qce_ahash_done, async_req); + if (ret) + goto error_unmap_dst; + + qce_dma_issue_pending(&qce->dma); + + ret = qce_start(async_req, tmpl->crypto_alg_type); + if (ret) + goto error_terminate; + + return 0; + +error_terminate: + qce_dma_terminate_all(&qce->dma); +error_unmap_dst: + dma_unmap_sg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE); +error_unmap_src: + dma_unmap_sg(qce->dev, req->src, rctx->src_nents, DMA_TO_DEVICE); + return ret; +} + +static int qce_ahash_init(struct ahash_request *req) +{ + struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req); + struct qce_alg_template *tmpl = to_ahash_tmpl(req->base.tfm); + const u32 *std_iv = tmpl->std_iv; + + memset(rctx, 0, sizeof(*rctx)); + rctx->first_blk = true; + rctx->last_blk = false; + rctx->flags = tmpl->alg_flags; + memcpy(rctx->digest, std_iv, sizeof(rctx->digest)); + + return 0; +} + +static int qce_ahash_export(struct ahash_request *req, void *out) +{ + struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req); + struct qce_sha_saved_state *export_state = out; + + memcpy(export_state->pending_buf, rctx->buf, rctx->buflen); + memcpy(export_state->partial_digest, rctx->digest, sizeof(rctx->digest)); + export_state->byte_count[0] = rctx->byte_count[0]; + export_state->byte_count[1] = rctx->byte_count[1]; + export_state->pending_buflen = rctx->buflen; + export_state->count = rctx->count; + export_state->first_blk = rctx->first_blk; + export_state->flags = rctx->flags; + + return 0; +} + +static int qce_ahash_import(struct ahash_request *req, const void *in) +{ + struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req); + const struct qce_sha_saved_state *import_state = in; + + memset(rctx, 0, sizeof(*rctx)); + rctx->count = import_state->count; + rctx->buflen = import_state->pending_buflen; + rctx->first_blk = import_state->first_blk; + rctx->flags = import_state->flags; + rctx->byte_count[0] = import_state->byte_count[0]; + rctx->byte_count[1] = import_state->byte_count[1]; + memcpy(rctx->buf, import_state->pending_buf, rctx->buflen); + memcpy(rctx->digest, import_state->partial_digest, sizeof(rctx->digest)); + + return 0; +} + +static int qce_ahash_update(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req); + struct qce_alg_template *tmpl = to_ahash_tmpl(req->base.tfm); + struct qce_device *qce = tmpl->qce; + unsigned int total; + unsigned int hash_later; + unsigned int blocksize; + + blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + rctx->count += req->nbytes; + + /* check for buffer from previous updates and append it */ + total = req->nbytes + rctx->buflen; + + if (total <= blocksize) { + scatterwalk_map_and_copy(rctx->buf + rctx->buflen, req->src, + 0, req->nbytes, 0); + rctx->buflen += req->nbytes; + return 0; + } + + /* save the original req structure fields */ + rctx->src_orig = req->src; + rctx->nbytes_orig = req->nbytes; + + /* + * if we have data from previous update copy them on buffer. The old + * data will be combined with current request bytes. + */ + if (rctx->buflen) + memcpy(rctx->tmpbuf, rctx->buf, rctx->buflen); + + /* calculate how many bytes will be hashed later */ + hash_later = total % blocksize; + + /* + * At this point, there is more than one block size of data. If + * the available data to transfer is exactly a multiple of block + * size, save the last block to be transferred in qce_ahash_final + * (with the last block bit set) if this is indeed the end of data + * stream. If not this saved block will be transferred as part of + * next update. If this block is not held back and if this is + * indeed the end of data stream, the digest obtained will be wrong + * since qce_ahash_final will see that rctx->buflen is 0 and return + * doing nothing which in turn means that a digest will not be + * copied to the destination result buffer. qce_ahash_final cannot + * be made to alter this behavior and allowed to proceed if + * rctx->buflen is 0 because the crypto engine BAM does not allow + * for zero length transfers. + */ + if (!hash_later) + hash_later = blocksize; + + scatterwalk_map_and_copy(rctx->buf, req->src, req->nbytes - hash_later, + hash_later, 0); + + if (rctx->buflen) { + sg_init_table(rctx->sg, 2); + sg_set_buf(rctx->sg, rctx->tmpbuf, rctx->buflen); + sg_chain(rctx->sg, 2, req->src); + req->src = rctx->sg; + } + + /* hash only complete blocks */ + req->nbytes = total - hash_later; + rctx->buflen = hash_later; + + return qce->async_req_enqueue(tmpl->qce, &req->base); +} + +static int qce_ahash_final(struct ahash_request *req) +{ + struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req); + struct qce_alg_template *tmpl = to_ahash_tmpl(req->base.tfm); + struct qce_device *qce = tmpl->qce; + + if (!rctx->buflen) { + if (tmpl->hash_zero) + memcpy(req->result, tmpl->hash_zero, + tmpl->alg.ahash.halg.digestsize); + return 0; + } + + rctx->last_blk = true; + + rctx->src_orig = req->src; + rctx->nbytes_orig = req->nbytes; + + memcpy(rctx->tmpbuf, rctx->buf, rctx->buflen); + sg_init_one(rctx->sg, rctx->tmpbuf, rctx->buflen); + + req->src = rctx->sg; + req->nbytes = rctx->buflen; + + return qce->async_req_enqueue(tmpl->qce, &req->base); +} + +static int qce_ahash_digest(struct ahash_request *req) +{ + struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req); + struct qce_alg_template *tmpl = to_ahash_tmpl(req->base.tfm); + struct qce_device *qce = tmpl->qce; + int ret; + + ret = qce_ahash_init(req); + if (ret) + return ret; + + rctx->src_orig = req->src; + rctx->nbytes_orig = req->nbytes; + rctx->first_blk = true; + rctx->last_blk = true; + + if (!rctx->nbytes_orig) { + if (tmpl->hash_zero) + memcpy(req->result, tmpl->hash_zero, + tmpl->alg.ahash.halg.digestsize); + return 0; + } + + return qce->async_req_enqueue(tmpl->qce, &req->base); +} + +static int qce_ahash_hmac_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + unsigned int digestsize = crypto_ahash_digestsize(tfm); + struct qce_sha_ctx *ctx = crypto_tfm_ctx(&tfm->base); + struct crypto_wait wait; + struct ahash_request *req; + struct scatterlist sg; + unsigned int blocksize; + struct crypto_ahash *ahash_tfm; + u8 *buf; + int ret; + const char *alg_name; + + blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + memset(ctx->authkey, 0, sizeof(ctx->authkey)); + + if (keylen <= blocksize) { + memcpy(ctx->authkey, key, keylen); + return 0; + } + + if (digestsize == SHA256_DIGEST_SIZE) + alg_name = "sha256-qce"; + else + return -EINVAL; + + ahash_tfm = crypto_alloc_ahash(alg_name, 0, 0); + if (IS_ERR(ahash_tfm)) + return PTR_ERR(ahash_tfm); + + req = ahash_request_alloc(ahash_tfm, GFP_KERNEL); + if (!req) { + ret = -ENOMEM; + goto err_free_ahash; + } + + crypto_init_wait(&wait); + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + crypto_ahash_clear_flags(ahash_tfm, ~0); + + buf = kzalloc(keylen + QCE_MAX_ALIGN_SIZE, GFP_KERNEL); + if (!buf) { + ret = -ENOMEM; + goto err_free_req; + } + + memcpy(buf, key, keylen); + sg_init_one(&sg, buf, keylen); + ahash_request_set_crypt(req, &sg, ctx->authkey, keylen); + + ret = crypto_wait_req(crypto_ahash_digest(req), &wait); + + kfree(buf); +err_free_req: + ahash_request_free(req); +err_free_ahash: + crypto_free_ahash(ahash_tfm); + return ret; +} + +static int qce_ahash_cra_init(struct crypto_tfm *tfm) +{ + struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); + struct qce_sha_ctx *ctx = crypto_tfm_ctx(tfm); + + crypto_ahash_set_reqsize_dma(ahash, sizeof(struct qce_sha_reqctx)); + memset(ctx, 0, sizeof(*ctx)); + return 0; +} + +struct qce_ahash_def { + unsigned long flags; + const char *name; + const char *drv_name; + unsigned int digestsize; + unsigned int blocksize; + unsigned int statesize; + const u32 *std_iv; +}; + +static const struct qce_ahash_def ahash_def[] = { + { + .flags = QCE_HASH_SHA256, + .name = "sha256", + .drv_name = "sha256-qce", + .digestsize = SHA256_DIGEST_SIZE, + .blocksize = SHA256_BLOCK_SIZE, + .statesize = sizeof(struct qce_sha_saved_state), + .std_iv = std_iv_sha256, + }, + { + .flags = QCE_HASH_SHA256_HMAC, + .name = "hmac(sha256)", + .drv_name = "hmac-sha256-qce", + .digestsize = SHA256_DIGEST_SIZE, + .blocksize = SHA256_BLOCK_SIZE, + .statesize = sizeof(struct qce_sha_saved_state), + .std_iv = std_iv_sha256, + }, +}; + +static int qce_ahash_register_one(const struct qce_ahash_def *def, + struct qce_device *qce) +{ + struct qce_alg_template *tmpl; + struct ahash_alg *alg; + struct crypto_alg *base; + int ret; + + tmpl = kzalloc_obj(*tmpl); + if (!tmpl) + return -ENOMEM; + + tmpl->std_iv = def->std_iv; + + alg = &tmpl->alg.ahash; + alg->init = qce_ahash_init; + alg->update = qce_ahash_update; + alg->final = qce_ahash_final; + alg->digest = qce_ahash_digest; + alg->export = qce_ahash_export; + alg->import = qce_ahash_import; + if (IS_SHA_HMAC(def->flags)) + alg->setkey = qce_ahash_hmac_setkey; + alg->halg.digestsize = def->digestsize; + alg->halg.statesize = def->statesize; + + if (IS_SHA256(def->flags)) + tmpl->hash_zero = sha256_zero_message_hash; + + base = &alg->halg.base; + base->cra_blocksize = def->blocksize; + base->cra_priority = 175; + base->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; + base->cra_ctxsize = sizeof(struct qce_sha_ctx); + base->cra_alignmask = 0; + base->cra_module = THIS_MODULE; + base->cra_init = qce_ahash_cra_init; + + strscpy(base->cra_name, def->name); + strscpy(base->cra_driver_name, def->drv_name); + + INIT_LIST_HEAD(&tmpl->entry); + tmpl->crypto_alg_type = CRYPTO_ALG_TYPE_AHASH; + tmpl->alg_flags = def->flags; + tmpl->qce = qce; + + ret = crypto_register_ahash(alg); + if (ret) { + dev_err(qce->dev, "%s registration failed\n", base->cra_name); + kfree(tmpl); + return ret; + } + + list_add_tail(&tmpl->entry, &ahash_algs); + dev_dbg(qce->dev, "%s is registered\n", base->cra_name); + return 0; +} + +static void qce_ahash_unregister(struct qce_device *qce) +{ + struct qce_alg_template *tmpl, *n; + + list_for_each_entry_safe(tmpl, n, &ahash_algs, entry) { + crypto_unregister_ahash(&tmpl->alg.ahash); + list_del(&tmpl->entry); + kfree(tmpl); + } +} + +static int qce_ahash_register(struct qce_device *qce) +{ + int ret, i; + + for (i = 0; i < ARRAY_SIZE(ahash_def); i++) { + ret = qce_ahash_register_one(&ahash_def[i], qce); + if (ret) + goto err; + } + + return 0; +err: + qce_ahash_unregister(qce); + return ret; +} + +const struct qce_algo_ops ahash_ops = { + .type = CRYPTO_ALG_TYPE_AHASH, + .register_algs = qce_ahash_register, + .unregister_algs = qce_ahash_unregister, + .async_req_handle = qce_ahash_async_req_handle, +}; diff --git a/drivers/crypto/qce/sha.h b/drivers/crypto/qce/sha.h new file mode 100644 index 000000000..cb822fc33 --- /dev/null +++ b/drivers/crypto/qce/sha.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2010-2014, The Linux Foundation. All rights reserved. + */ + +#ifndef _SHA_H_ +#define _SHA_H_ + +#include +#include + +#include "common.h" +#include "core.h" + +#define QCE_SHA_MAX_BLOCKSIZE SHA256_BLOCK_SIZE +#define QCE_SHA_MAX_DIGESTSIZE SHA256_DIGEST_SIZE + +struct qce_sha_ctx { + u8 authkey[QCE_SHA_MAX_BLOCKSIZE]; +}; + +/** + * struct qce_sha_reqctx - holds private ahash objects per request + * @buf: used during update, import and export + * @tmpbuf: buffer for internal use + * @digest: calculated digest buffer + * @buflen: length of the buffer + * @flags: operation flags + * @src_orig: original request sg list + * @nbytes_orig: original request number of bytes + * @src_nents: source number of entries + * @byte_count: byte count + * @count: save count in states during update, import and export + * @first_blk: is it the first block + * @last_blk: is it the last block + * @sg: used to chain sg lists + * @authkey: pointer to auth key in sha ctx + * @authklen: auth key length + * @result_sg: scatterlist used for result buffer + */ +struct qce_sha_reqctx { + u8 buf[QCE_SHA_MAX_BLOCKSIZE]; + u8 tmpbuf[QCE_SHA_MAX_BLOCKSIZE]; + u8 digest[QCE_SHA_MAX_DIGESTSIZE]; + unsigned int buflen; + unsigned long flags; + struct scatterlist *src_orig; + unsigned int nbytes_orig; + int src_nents; + __be32 byte_count[2]; + u64 count; + bool first_blk; + bool last_blk; + struct scatterlist sg[2]; + u8 *authkey; + unsigned int authklen; + struct scatterlist result_sg; +}; + +static inline struct qce_alg_template *to_ahash_tmpl(struct crypto_tfm *tfm) +{ + struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); + struct ahash_alg *alg = container_of(crypto_hash_alg_common(ahash), + struct ahash_alg, halg); + + return container_of(alg, struct qce_alg_template, alg.ahash); +} + +extern const struct qce_algo_ops ahash_ops; + +#endif /* _SHA_H_ */ diff --git a/drivers/crypto/qce/skcipher.c b/drivers/crypto/qce/skcipher.c new file mode 100644 index 000000000..4931b064f --- /dev/null +++ b/drivers/crypto/qce/skcipher.c @@ -0,0 +1,434 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2010-2014, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cipher.h" + +static unsigned int aes_sw_max_len = CONFIG_CRYPTO_DEV_QCE_SW_MAX_LEN; +module_param(aes_sw_max_len, uint, 0644); +MODULE_PARM_DESC(aes_sw_max_len, + "Only use hardware for AES requests larger than this " + "[0=always use hardware; anything <16 breaks AES-GCM; default=" + __stringify(CONFIG_CRYPTO_DEV_QCE_SW_MAX_LEN)"]"); + +static LIST_HEAD(skcipher_algs); + +static void qce_skcipher_done(void *data) +{ + struct crypto_async_request *async_req = data; + struct skcipher_request *req = skcipher_request_cast(async_req); + struct qce_cipher_reqctx *rctx = skcipher_request_ctx(req); + struct qce_alg_template *tmpl = to_cipher_tmpl(crypto_skcipher_reqtfm(req)); + struct qce_device *qce = tmpl->qce; + struct qce_result_dump *result_buf = qce->dma.result_buf; + enum dma_data_direction dir_src, dir_dst; + u32 status; + int error; + bool diff_dst; + + diff_dst = (req->src != req->dst) ? true : false; + dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; + dir_dst = diff_dst ? DMA_FROM_DEVICE : DMA_BIDIRECTIONAL; + + error = qce_dma_terminate_all(&qce->dma); + if (error) + dev_dbg(qce->dev, "skcipher dma termination error (%d)\n", + error); + + if (diff_dst) + dma_unmap_sg(qce->dev, rctx->src_sg, rctx->src_nents, dir_src); + dma_unmap_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); + + sg_free_table(&rctx->dst_tbl); + + error = qce_check_status(qce, &status); + if (error < 0) + dev_dbg(qce->dev, "skcipher operation error (%x)\n", status); + + memcpy(rctx->iv, result_buf->encr_cntr_iv, rctx->ivsize); + qce->async_req_done(tmpl->qce, error); +} + +static int +qce_skcipher_async_req_handle(struct crypto_async_request *async_req) +{ + struct skcipher_request *req = skcipher_request_cast(async_req); + struct qce_cipher_reqctx *rctx = skcipher_request_ctx(req); + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + struct qce_alg_template *tmpl = to_cipher_tmpl(crypto_skcipher_reqtfm(req)); + struct qce_device *qce = tmpl->qce; + enum dma_data_direction dir_src, dir_dst; + struct scatterlist *sg; + bool diff_dst; + gfp_t gfp; + int dst_nents, src_nents, ret; + + rctx->iv = req->iv; + rctx->ivsize = crypto_skcipher_ivsize(skcipher); + rctx->cryptlen = req->cryptlen; + + diff_dst = (req->src != req->dst) ? true : false; + dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; + dir_dst = diff_dst ? DMA_FROM_DEVICE : DMA_BIDIRECTIONAL; + + rctx->src_nents = sg_nents_for_len(req->src, req->cryptlen); + if (diff_dst) + rctx->dst_nents = sg_nents_for_len(req->dst, req->cryptlen); + else + rctx->dst_nents = rctx->src_nents; + if (rctx->src_nents < 0) { + dev_err(qce->dev, "Invalid numbers of src SG.\n"); + return rctx->src_nents; + } + if (rctx->dst_nents < 0) { + dev_err(qce->dev, "Invalid numbers of dst SG.\n"); + return -rctx->dst_nents; + } + + rctx->dst_nents += 1; + + gfp = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + + ret = sg_alloc_table(&rctx->dst_tbl, rctx->dst_nents, gfp); + if (ret) + return ret; + + sg_init_one(&rctx->result_sg, qce->dma.result_buf, QCE_RESULT_BUF_SZ); + + sg = qce_sgtable_add(&rctx->dst_tbl, req->dst, req->cryptlen); + if (IS_ERR(sg)) { + ret = PTR_ERR(sg); + goto error_free; + } + + sg = qce_sgtable_add(&rctx->dst_tbl, &rctx->result_sg, + QCE_RESULT_BUF_SZ); + if (IS_ERR(sg)) { + ret = PTR_ERR(sg); + goto error_free; + } + + sg_mark_end(sg); + rctx->dst_sg = rctx->dst_tbl.sgl; + + dst_nents = dma_map_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); + if (!dst_nents) { + ret = -EIO; + goto error_free; + } + + if (diff_dst) { + src_nents = dma_map_sg(qce->dev, req->src, rctx->src_nents, dir_src); + if (!src_nents) { + ret = -EIO; + goto error_unmap_dst; + } + rctx->src_sg = req->src; + } else { + rctx->src_sg = rctx->dst_sg; + src_nents = dst_nents - 1; + } + + ret = qce_dma_prep_sgs(&qce->dma, rctx->src_sg, src_nents, + rctx->dst_sg, dst_nents, + qce_skcipher_done, async_req); + if (ret) + goto error_unmap_src; + + qce_dma_issue_pending(&qce->dma); + + ret = qce_start(async_req, tmpl->crypto_alg_type); + if (ret) + goto error_terminate; + + return 0; + +error_terminate: + qce_dma_terminate_all(&qce->dma); +error_unmap_src: + if (diff_dst) + dma_unmap_sg(qce->dev, req->src, rctx->src_nents, dir_src); +error_unmap_dst: + dma_unmap_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); +error_free: + sg_free_table(&rctx->dst_tbl); + return ret; +} + +static int qce_skcipher_setkey(struct crypto_skcipher *ablk, const u8 *key, + unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(ablk); + struct qce_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + unsigned long flags = to_cipher_tmpl(ablk)->alg_flags; + unsigned int __keylen; + int ret; + + if (!key || !keylen) + return -EINVAL; + + /* + * AES XTS key1 = key2 not supported by crypto engine. + * Revisit to request a fallback cipher in this case. + */ + if (IS_XTS(flags)) { + __keylen = keylen >> 1; + if (!memcmp(key, key + __keylen, __keylen)) + return -ENOKEY; + } else { + __keylen = keylen; + } + + switch (__keylen) { + case AES_KEYSIZE_128: + case AES_KEYSIZE_256: + memcpy(ctx->enc_key, key, keylen); + break; + case AES_KEYSIZE_192: + break; + default: + return -EINVAL; + } + + ret = crypto_skcipher_setkey(ctx->fallback, key, keylen); + if (!ret) + ctx->enc_keylen = keylen; + return ret; +} + +static int qce_skcipher_crypt(struct skcipher_request *req, int encrypt) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct qce_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); + struct qce_cipher_reqctx *rctx = skcipher_request_ctx(req); + struct qce_alg_template *tmpl = to_cipher_tmpl(tfm); + unsigned int blocksize = crypto_skcipher_blocksize(tfm); + int keylen; + + rctx->flags = tmpl->alg_flags; + rctx->flags |= encrypt ? QCE_ENCRYPT : QCE_DECRYPT; + keylen = IS_XTS(rctx->flags) ? ctx->enc_keylen >> 1 : ctx->enc_keylen; + + /* CE does not handle 0 length messages */ + if (!req->cryptlen) + return 0; + + /* + * ECB and CBC algorithms require message lengths to be + * multiples of block size. + */ + if (IS_CBC(rctx->flags)) + if (!IS_ALIGNED(req->cryptlen, blocksize)) + return -EINVAL; + + /* + * Conditions for requesting a fallback cipher + * AES-192 (not supported by crypto engine (CE)) + * AES-XTS request with len <= 512 byte (not recommended to use CE) + * AES-XTS request with len > QCE_SECTOR_SIZE and + * is not a multiple of it.(Revisit this condition to check if it is + * needed in all versions of CE) + */ + if (IS_AES(rctx->flags) && + ((keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_256) || + (IS_XTS(rctx->flags) && ((req->cryptlen <= aes_sw_max_len) || + (req->cryptlen > QCE_SECTOR_SIZE && + req->cryptlen % QCE_SECTOR_SIZE))))) { + skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback); + skcipher_request_set_callback(&rctx->fallback_req, + req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(&rctx->fallback_req, req->src, + req->dst, req->cryptlen, req->iv); + return encrypt ? crypto_skcipher_encrypt(&rctx->fallback_req) : + crypto_skcipher_decrypt(&rctx->fallback_req); + } + + return tmpl->qce->async_req_enqueue(tmpl->qce, &req->base); +} + +static int qce_skcipher_encrypt(struct skcipher_request *req) +{ + return qce_skcipher_crypt(req, 1); +} + +static int qce_skcipher_decrypt(struct skcipher_request *req) +{ + return qce_skcipher_crypt(req, 0); +} + +static int qce_skcipher_init(struct crypto_skcipher *tfm) +{ + /* take the size without the fallback skcipher_request at the end */ + crypto_skcipher_set_reqsize(tfm, offsetof(struct qce_cipher_reqctx, + fallback_req)); + return 0; +} + +static int qce_skcipher_init_fallback(struct crypto_skcipher *tfm) +{ + struct qce_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); + + ctx->fallback = crypto_alloc_skcipher(crypto_tfm_alg_name(&tfm->base), + 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fallback)) + return PTR_ERR(ctx->fallback); + + crypto_skcipher_set_reqsize(tfm, sizeof(struct qce_cipher_reqctx) + + crypto_skcipher_reqsize(ctx->fallback)); + return 0; +} + +static void qce_skcipher_exit(struct crypto_skcipher *tfm) +{ + struct qce_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); + + crypto_free_skcipher(ctx->fallback); +} + +struct qce_skcipher_def { + unsigned long flags; + const char *name; + const char *drv_name; + unsigned int blocksize; + unsigned int chunksize; + unsigned int ivsize; + unsigned int min_keysize; + unsigned int max_keysize; +}; + +static const struct qce_skcipher_def skcipher_def[] = { + { + .flags = QCE_ALG_AES | QCE_MODE_CBC, + .name = "cbc(aes)", + .drv_name = "cbc-aes-qce", + .blocksize = AES_BLOCK_SIZE, + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + }, + { + .flags = QCE_ALG_AES | QCE_MODE_CTR, + .name = "ctr(aes)", + .drv_name = "ctr-aes-qce", + .blocksize = 1, + .chunksize = AES_BLOCK_SIZE, + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + }, + { + .flags = QCE_ALG_AES | QCE_MODE_XTS, + .name = "xts(aes)", + .drv_name = "xts-aes-qce", + .blocksize = AES_BLOCK_SIZE, + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE * 2, + .max_keysize = AES_MAX_KEY_SIZE * 2, + }, +}; + +static int qce_skcipher_register_one(const struct qce_skcipher_def *def, + struct qce_device *qce) +{ + struct qce_alg_template *tmpl; + struct skcipher_alg *alg; + int ret; + + tmpl = kzalloc_obj(*tmpl); + if (!tmpl) + return -ENOMEM; + + alg = &tmpl->alg.skcipher; + + strscpy(alg->base.cra_name, def->name); + strscpy(alg->base.cra_driver_name, def->drv_name); + + alg->base.cra_blocksize = def->blocksize; + alg->chunksize = def->chunksize; + alg->ivsize = def->ivsize; + alg->min_keysize = def->min_keysize; + alg->max_keysize = def->max_keysize; + alg->setkey = qce_skcipher_setkey; + alg->encrypt = qce_skcipher_encrypt; + alg->decrypt = qce_skcipher_decrypt; + + alg->base.cra_priority = 275; + alg->base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY; + alg->base.cra_ctxsize = sizeof(struct qce_cipher_ctx); + alg->base.cra_alignmask = 0; + alg->base.cra_module = THIS_MODULE; + + if (IS_AES(def->flags)) { + alg->base.cra_flags |= CRYPTO_ALG_NEED_FALLBACK; + alg->init = qce_skcipher_init_fallback; + alg->exit = qce_skcipher_exit; + } else { + alg->init = qce_skcipher_init; + } + + INIT_LIST_HEAD(&tmpl->entry); + tmpl->crypto_alg_type = CRYPTO_ALG_TYPE_SKCIPHER; + tmpl->alg_flags = def->flags; + tmpl->qce = qce; + + ret = crypto_register_skcipher(alg); + if (ret) { + dev_err(qce->dev, "%s registration failed\n", alg->base.cra_name); + kfree(tmpl); + return ret; + } + + list_add_tail(&tmpl->entry, &skcipher_algs); + dev_dbg(qce->dev, "%s is registered\n", alg->base.cra_name); + return 0; +} + +static void qce_skcipher_unregister(struct qce_device *qce) +{ + struct qce_alg_template *tmpl, *n; + + list_for_each_entry_safe(tmpl, n, &skcipher_algs, entry) { + crypto_unregister_skcipher(&tmpl->alg.skcipher); + list_del(&tmpl->entry); + kfree(tmpl); + } +} + +static int qce_skcipher_register(struct qce_device *qce) +{ + int ret, i; + + for (i = 0; i < ARRAY_SIZE(skcipher_def); i++) { + ret = qce_skcipher_register_one(&skcipher_def[i], qce); + if (ret) + goto err; + } + + return 0; +err: + qce_skcipher_unregister(qce); + return ret; +} + +const struct qce_algo_ops skcipher_ops = { + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .register_algs = qce_skcipher_register, + .unregister_algs = qce_skcipher_unregister, + .async_req_handle = qce_skcipher_async_req_handle, +}; diff --git a/drivers/crypto/rockchip/Makefile b/drivers/crypto/rockchip/Makefile new file mode 100644 index 000000000..785277aca --- /dev/null +++ b/drivers/crypto/rockchip/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP) += rk_crypto.o +rk_crypto-objs := rk3288_crypto.o \ + rk3288_crypto_skcipher.o \ + rk3288_crypto_ahash.o diff --git a/drivers/crypto/rockchip/rk3288_crypto.c b/drivers/crypto/rockchip/rk3288_crypto.c new file mode 100644 index 000000000..7cb1316b6 --- /dev/null +++ b/drivers/crypto/rockchip/rk3288_crypto.c @@ -0,0 +1,446 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Crypto acceleration support for Rockchip RK3288 + * + * Copyright (c) 2015, Fuzhou Rockchip Electronics Co., Ltd + * + * Author: Zain Wang + * + * Some ideas are from marvell-cesa.c and s5p-sss.c driver. + */ + +#include "rk3288_crypto.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static struct rockchip_ip rocklist = { + .dev_list = LIST_HEAD_INIT(rocklist.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(rocklist.lock), +}; + +struct rk_crypto_info *get_rk_crypto(void) +{ + struct rk_crypto_info *first; + + spin_lock(&rocklist.lock); + first = list_first_entry_or_null(&rocklist.dev_list, + struct rk_crypto_info, list); + list_rotate_left(&rocklist.dev_list); + spin_unlock(&rocklist.lock); + return first; +} + +static const struct rk_variant rk3288_variant = { + .num_clks = 4, + .rkclks = { + { "sclk", 150000000}, + } +}; + +static const struct rk_variant rk3328_variant = { + .num_clks = 3, +}; + +static const struct rk_variant rk3399_variant = { + .num_clks = 3, +}; + +static int rk_crypto_get_clks(struct rk_crypto_info *dev) +{ + int i, j, err; + unsigned long cr; + + dev->num_clks = devm_clk_bulk_get_all(dev->dev, &dev->clks); + if (dev->num_clks < dev->variant->num_clks) { + dev_err(dev->dev, "Missing clocks, got %d instead of %d\n", + dev->num_clks, dev->variant->num_clks); + return -EINVAL; + } + + for (i = 0; i < dev->num_clks; i++) { + cr = clk_get_rate(dev->clks[i].clk); + for (j = 0; j < ARRAY_SIZE(dev->variant->rkclks); j++) { + if (dev->variant->rkclks[j].max == 0) + continue; + if (strcmp(dev->variant->rkclks[j].name, dev->clks[i].id)) + continue; + if (cr > dev->variant->rkclks[j].max) { + err = clk_set_rate(dev->clks[i].clk, + dev->variant->rkclks[j].max); + if (err) + dev_err(dev->dev, "Fail downclocking %s from %lu to %lu\n", + dev->variant->rkclks[j].name, cr, + dev->variant->rkclks[j].max); + else + dev_info(dev->dev, "Downclocking %s from %lu to %lu\n", + dev->variant->rkclks[j].name, cr, + dev->variant->rkclks[j].max); + } + } + } + return 0; +} + +static int rk_crypto_enable_clk(struct rk_crypto_info *dev) +{ + int err; + + err = clk_bulk_prepare_enable(dev->num_clks, dev->clks); + if (err) + dev_err(dev->dev, "Could not enable clock clks\n"); + + return err; +} + +static void rk_crypto_disable_clk(struct rk_crypto_info *dev) +{ + clk_bulk_disable_unprepare(dev->num_clks, dev->clks); +} + +/* + * Power management strategy: The device is suspended until a request + * is handled. For avoiding suspend/resume yoyo, the autosuspend is set to 2s. + */ +static int rk_crypto_pm_suspend(struct device *dev) +{ + struct rk_crypto_info *rkdev = dev_get_drvdata(dev); + + rk_crypto_disable_clk(rkdev); + reset_control_assert(rkdev->rst); + + return 0; +} + +static int rk_crypto_pm_resume(struct device *dev) +{ + struct rk_crypto_info *rkdev = dev_get_drvdata(dev); + int ret; + + ret = rk_crypto_enable_clk(rkdev); + if (ret) + return ret; + + reset_control_deassert(rkdev->rst); + return 0; + +} + +static const struct dev_pm_ops rk_crypto_pm_ops = { + SET_RUNTIME_PM_OPS(rk_crypto_pm_suspend, rk_crypto_pm_resume, NULL) +}; + +static int rk_crypto_pm_init(struct rk_crypto_info *rkdev) +{ + int err; + + pm_runtime_use_autosuspend(rkdev->dev); + pm_runtime_set_autosuspend_delay(rkdev->dev, 2000); + + err = pm_runtime_set_suspended(rkdev->dev); + if (err) + return err; + pm_runtime_enable(rkdev->dev); + return err; +} + +static void rk_crypto_pm_exit(struct rk_crypto_info *rkdev) +{ + pm_runtime_disable(rkdev->dev); +} + +static irqreturn_t rk_crypto_irq_handle(int irq, void *dev_id) +{ + struct rk_crypto_info *dev = platform_get_drvdata(dev_id); + u32 interrupt_status; + + interrupt_status = CRYPTO_READ(dev, RK_CRYPTO_INTSTS); + CRYPTO_WRITE(dev, RK_CRYPTO_INTSTS, interrupt_status); + + dev->status = 1; + if (interrupt_status & 0x0a) { + dev_warn(dev->dev, "DMA Error\n"); + dev->status = 0; + } + complete(&dev->complete); + + return IRQ_HANDLED; +} + +static struct rk_crypto_tmp *rk_cipher_algs[] = { + &rk_ecb_aes_alg, + &rk_cbc_aes_alg, + &rk_ecb_des_alg, + &rk_cbc_des_alg, + &rk_ecb_des3_ede_alg, + &rk_cbc_des3_ede_alg, + &rk_ahash_sha1, + &rk_ahash_sha256, + &rk_ahash_md5, +}; + +static int rk_crypto_debugfs_show(struct seq_file *seq, void *v) +{ + struct rk_crypto_info *dd; + unsigned int i; + + spin_lock(&rocklist.lock); + list_for_each_entry(dd, &rocklist.dev_list, list) { + seq_printf(seq, "%s %s requests: %lu\n", + dev_driver_string(dd->dev), dev_name(dd->dev), + dd->nreq); + } + spin_unlock(&rocklist.lock); + + for (i = 0; i < ARRAY_SIZE(rk_cipher_algs); i++) { + if (!rk_cipher_algs[i]->dev) + continue; + switch (rk_cipher_algs[i]->type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n", + rk_cipher_algs[i]->alg.skcipher.base.base.cra_driver_name, + rk_cipher_algs[i]->alg.skcipher.base.base.cra_name, + rk_cipher_algs[i]->stat_req, rk_cipher_algs[i]->stat_fb); + seq_printf(seq, "\tfallback due to length: %lu\n", + rk_cipher_algs[i]->stat_fb_len); + seq_printf(seq, "\tfallback due to alignment: %lu\n", + rk_cipher_algs[i]->stat_fb_align); + seq_printf(seq, "\tfallback due to SGs: %lu\n", + rk_cipher_algs[i]->stat_fb_sgdiff); + break; + case CRYPTO_ALG_TYPE_AHASH: + seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n", + rk_cipher_algs[i]->alg.hash.base.halg.base.cra_driver_name, + rk_cipher_algs[i]->alg.hash.base.halg.base.cra_name, + rk_cipher_algs[i]->stat_req, rk_cipher_algs[i]->stat_fb); + break; + } + } + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(rk_crypto_debugfs); + +static void register_debugfs(struct rk_crypto_info *crypto_info) +{ + struct dentry *dbgfs_dir __maybe_unused; + struct dentry *dbgfs_stats __maybe_unused; + + /* Ignore error of debugfs */ + dbgfs_dir = debugfs_create_dir("rk3288_crypto", NULL); + dbgfs_stats = debugfs_create_file("stats", 0444, dbgfs_dir, &rocklist, + &rk_crypto_debugfs_fops); + +#ifdef CONFIG_CRYPTO_DEV_ROCKCHIP_DEBUG + rocklist.dbgfs_dir = dbgfs_dir; + rocklist.dbgfs_stats = dbgfs_stats; +#endif +} + +static int rk_crypto_register(struct rk_crypto_info *crypto_info) +{ + unsigned int i, k; + int err = 0; + + for (i = 0; i < ARRAY_SIZE(rk_cipher_algs); i++) { + rk_cipher_algs[i]->dev = crypto_info; + switch (rk_cipher_algs[i]->type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + dev_info(crypto_info->dev, "Register %s as %s\n", + rk_cipher_algs[i]->alg.skcipher.base.base.cra_name, + rk_cipher_algs[i]->alg.skcipher.base.base.cra_driver_name); + err = crypto_engine_register_skcipher(&rk_cipher_algs[i]->alg.skcipher); + break; + case CRYPTO_ALG_TYPE_AHASH: + dev_info(crypto_info->dev, "Register %s as %s\n", + rk_cipher_algs[i]->alg.hash.base.halg.base.cra_name, + rk_cipher_algs[i]->alg.hash.base.halg.base.cra_driver_name); + err = crypto_engine_register_ahash(&rk_cipher_algs[i]->alg.hash); + break; + default: + dev_err(crypto_info->dev, "unknown algorithm\n"); + } + if (err) + goto err_cipher_algs; + } + return 0; + +err_cipher_algs: + for (k = 0; k < i; k++) { + if (rk_cipher_algs[i]->type == CRYPTO_ALG_TYPE_SKCIPHER) + crypto_engine_unregister_skcipher(&rk_cipher_algs[k]->alg.skcipher); + else + crypto_engine_unregister_ahash(&rk_cipher_algs[i]->alg.hash); + } + return err; +} + +static void rk_crypto_unregister(void) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(rk_cipher_algs); i++) { + if (rk_cipher_algs[i]->type == CRYPTO_ALG_TYPE_SKCIPHER) + crypto_engine_unregister_skcipher(&rk_cipher_algs[i]->alg.skcipher); + else + crypto_engine_unregister_ahash(&rk_cipher_algs[i]->alg.hash); + } +} + +static const struct of_device_id crypto_of_id_table[] = { + { .compatible = "rockchip,rk3288-crypto", + .data = &rk3288_variant, + }, + { .compatible = "rockchip,rk3328-crypto", + .data = &rk3328_variant, + }, + { .compatible = "rockchip,rk3399-crypto", + .data = &rk3399_variant, + }, + {} +}; +MODULE_DEVICE_TABLE(of, crypto_of_id_table); + +static int rk_crypto_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct rk_crypto_info *crypto_info, *first; + int err = 0; + + crypto_info = devm_kzalloc(&pdev->dev, + sizeof(*crypto_info), GFP_KERNEL); + if (!crypto_info) { + err = -ENOMEM; + goto err_crypto; + } + + crypto_info->dev = &pdev->dev; + platform_set_drvdata(pdev, crypto_info); + + crypto_info->variant = of_device_get_match_data(&pdev->dev); + if (!crypto_info->variant) { + dev_err(&pdev->dev, "Missing variant\n"); + return -EINVAL; + } + + crypto_info->rst = devm_reset_control_array_get_exclusive(dev); + if (IS_ERR(crypto_info->rst)) { + err = PTR_ERR(crypto_info->rst); + goto err_crypto; + } + + reset_control_assert(crypto_info->rst); + usleep_range(10, 20); + reset_control_deassert(crypto_info->rst); + + crypto_info->reg = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(crypto_info->reg)) { + err = PTR_ERR(crypto_info->reg); + goto err_crypto; + } + + err = rk_crypto_get_clks(crypto_info); + if (err) + goto err_crypto; + + crypto_info->irq = platform_get_irq(pdev, 0); + if (crypto_info->irq < 0) { + err = crypto_info->irq; + goto err_crypto; + } + + err = devm_request_irq(&pdev->dev, crypto_info->irq, + rk_crypto_irq_handle, IRQF_SHARED, + "rk-crypto", pdev); + + if (err) + goto err_crypto; + + crypto_info->engine = crypto_engine_alloc_init(&pdev->dev, true); + if (!crypto_info->engine) { + err = -ENOMEM; + goto err_crypto; + } + + crypto_engine_start(crypto_info->engine); + init_completion(&crypto_info->complete); + + err = rk_crypto_pm_init(crypto_info); + if (err) + goto err_pm; + + spin_lock(&rocklist.lock); + first = list_first_entry_or_null(&rocklist.dev_list, + struct rk_crypto_info, list); + list_add_tail(&crypto_info->list, &rocklist.dev_list); + spin_unlock(&rocklist.lock); + + if (!first) { + err = rk_crypto_register(crypto_info); + if (err) { + dev_err(dev, "Fail to register crypto algorithms"); + goto err_register_alg; + } + + register_debugfs(crypto_info); + } + + return 0; + +err_register_alg: + rk_crypto_pm_exit(crypto_info); +err_pm: + crypto_engine_exit(crypto_info->engine); +err_crypto: + dev_err(dev, "Crypto Accelerator not successfully registered\n"); + return err; +} + +static void rk_crypto_remove(struct platform_device *pdev) +{ + struct rk_crypto_info *crypto_tmp = platform_get_drvdata(pdev); + struct rk_crypto_info *first; + + spin_lock_bh(&rocklist.lock); + list_del(&crypto_tmp->list); + first = list_first_entry_or_null(&rocklist.dev_list, + struct rk_crypto_info, list); + spin_unlock_bh(&rocklist.lock); + + if (!first) { +#ifdef CONFIG_CRYPTO_DEV_ROCKCHIP_DEBUG + debugfs_remove_recursive(rocklist.dbgfs_dir); +#endif + rk_crypto_unregister(); + } + rk_crypto_pm_exit(crypto_tmp); + crypto_engine_exit(crypto_tmp->engine); +} + +static struct platform_driver crypto_driver = { + .probe = rk_crypto_probe, + .remove = rk_crypto_remove, + .driver = { + .name = "rk3288-crypto", + .pm = &rk_crypto_pm_ops, + .of_match_table = crypto_of_id_table, + }, +}; + +module_platform_driver(crypto_driver); + +MODULE_AUTHOR("Zain Wang "); +MODULE_DESCRIPTION("Support for Rockchip's cryptographic engine"); +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/rockchip/rk3288_crypto.h b/drivers/crypto/rockchip/rk3288_crypto.h new file mode 100644 index 000000000..3aa03cbfb --- /dev/null +++ b/drivers/crypto/rockchip/rk3288_crypto.h @@ -0,0 +1,285 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __RK3288_CRYPTO_H__ +#define __RK3288_CRYPTO_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define _SBF(v, f) ((v) << (f)) + +/* Crypto control registers*/ +#define RK_CRYPTO_INTSTS 0x0000 +#define RK_CRYPTO_PKA_DONE_INT BIT(5) +#define RK_CRYPTO_HASH_DONE_INT BIT(4) +#define RK_CRYPTO_HRDMA_ERR_INT BIT(3) +#define RK_CRYPTO_HRDMA_DONE_INT BIT(2) +#define RK_CRYPTO_BCDMA_ERR_INT BIT(1) +#define RK_CRYPTO_BCDMA_DONE_INT BIT(0) + +#define RK_CRYPTO_INTENA 0x0004 +#define RK_CRYPTO_PKA_DONE_ENA BIT(5) +#define RK_CRYPTO_HASH_DONE_ENA BIT(4) +#define RK_CRYPTO_HRDMA_ERR_ENA BIT(3) +#define RK_CRYPTO_HRDMA_DONE_ENA BIT(2) +#define RK_CRYPTO_BCDMA_ERR_ENA BIT(1) +#define RK_CRYPTO_BCDMA_DONE_ENA BIT(0) + +#define RK_CRYPTO_CTRL 0x0008 +#define RK_CRYPTO_WRITE_MASK _SBF(0xFFFF, 16) +#define RK_CRYPTO_TRNG_FLUSH BIT(9) +#define RK_CRYPTO_TRNG_START BIT(8) +#define RK_CRYPTO_PKA_FLUSH BIT(7) +#define RK_CRYPTO_HASH_FLUSH BIT(6) +#define RK_CRYPTO_BLOCK_FLUSH BIT(5) +#define RK_CRYPTO_PKA_START BIT(4) +#define RK_CRYPTO_HASH_START BIT(3) +#define RK_CRYPTO_BLOCK_START BIT(2) +#define RK_CRYPTO_TDES_START BIT(1) +#define RK_CRYPTO_AES_START BIT(0) + +#define RK_CRYPTO_CONF 0x000c +/* HASH Receive DMA Address Mode: fix | increment */ +#define RK_CRYPTO_HR_ADDR_MODE BIT(8) +/* Block Transmit DMA Address Mode: fix | increment */ +#define RK_CRYPTO_BT_ADDR_MODE BIT(7) +/* Block Receive DMA Address Mode: fix | increment */ +#define RK_CRYPTO_BR_ADDR_MODE BIT(6) +#define RK_CRYPTO_BYTESWAP_HRFIFO BIT(5) +#define RK_CRYPTO_BYTESWAP_BTFIFO BIT(4) +#define RK_CRYPTO_BYTESWAP_BRFIFO BIT(3) +/* AES = 0 OR DES = 1 */ +#define RK_CRYPTO_DESSEL BIT(2) +#define RK_CYYPTO_HASHINSEL_INDEPENDENT_SOURCE _SBF(0x00, 0) +#define RK_CYYPTO_HASHINSEL_BLOCK_CIPHER_INPUT _SBF(0x01, 0) +#define RK_CYYPTO_HASHINSEL_BLOCK_CIPHER_OUTPUT _SBF(0x02, 0) + +/* Block Receiving DMA Start Address Register */ +#define RK_CRYPTO_BRDMAS 0x0010 +/* Block Transmitting DMA Start Address Register */ +#define RK_CRYPTO_BTDMAS 0x0014 +/* Block Receiving DMA Length Register */ +#define RK_CRYPTO_BRDMAL 0x0018 +/* Hash Receiving DMA Start Address Register */ +#define RK_CRYPTO_HRDMAS 0x001c +/* Hash Receiving DMA Length Register */ +#define RK_CRYPTO_HRDMAL 0x0020 + +/* AES registers */ +#define RK_CRYPTO_AES_CTRL 0x0080 +#define RK_CRYPTO_AES_BYTESWAP_CNT BIT(11) +#define RK_CRYPTO_AES_BYTESWAP_KEY BIT(10) +#define RK_CRYPTO_AES_BYTESWAP_IV BIT(9) +#define RK_CRYPTO_AES_BYTESWAP_DO BIT(8) +#define RK_CRYPTO_AES_BYTESWAP_DI BIT(7) +#define RK_CRYPTO_AES_KEY_CHANGE BIT(6) +#define RK_CRYPTO_AES_ECB_MODE _SBF(0x00, 4) +#define RK_CRYPTO_AES_CBC_MODE _SBF(0x01, 4) +#define RK_CRYPTO_AES_CTR_MODE _SBF(0x02, 4) +#define RK_CRYPTO_AES_128BIT_key _SBF(0x00, 2) +#define RK_CRYPTO_AES_192BIT_key _SBF(0x01, 2) +#define RK_CRYPTO_AES_256BIT_key _SBF(0x02, 2) +/* Slave = 0 / fifo = 1 */ +#define RK_CRYPTO_AES_FIFO_MODE BIT(1) +/* Encryption = 0 , Decryption = 1 */ +#define RK_CRYPTO_AES_DEC BIT(0) + +#define RK_CRYPTO_AES_STS 0x0084 +#define RK_CRYPTO_AES_DONE BIT(0) + +/* AES Input Data 0-3 Register */ +#define RK_CRYPTO_AES_DIN_0 0x0088 +#define RK_CRYPTO_AES_DIN_1 0x008c +#define RK_CRYPTO_AES_DIN_2 0x0090 +#define RK_CRYPTO_AES_DIN_3 0x0094 + +/* AES output Data 0-3 Register */ +#define RK_CRYPTO_AES_DOUT_0 0x0098 +#define RK_CRYPTO_AES_DOUT_1 0x009c +#define RK_CRYPTO_AES_DOUT_2 0x00a0 +#define RK_CRYPTO_AES_DOUT_3 0x00a4 + +/* AES IV Data 0-3 Register */ +#define RK_CRYPTO_AES_IV_0 0x00a8 +#define RK_CRYPTO_AES_IV_1 0x00ac +#define RK_CRYPTO_AES_IV_2 0x00b0 +#define RK_CRYPTO_AES_IV_3 0x00b4 + +/* AES Key Data 0-3 Register */ +#define RK_CRYPTO_AES_KEY_0 0x00b8 +#define RK_CRYPTO_AES_KEY_1 0x00bc +#define RK_CRYPTO_AES_KEY_2 0x00c0 +#define RK_CRYPTO_AES_KEY_3 0x00c4 +#define RK_CRYPTO_AES_KEY_4 0x00c8 +#define RK_CRYPTO_AES_KEY_5 0x00cc +#define RK_CRYPTO_AES_KEY_6 0x00d0 +#define RK_CRYPTO_AES_KEY_7 0x00d4 + +/* des/tdes */ +#define RK_CRYPTO_TDES_CTRL 0x0100 +#define RK_CRYPTO_TDES_BYTESWAP_KEY BIT(8) +#define RK_CRYPTO_TDES_BYTESWAP_IV BIT(7) +#define RK_CRYPTO_TDES_BYTESWAP_DO BIT(6) +#define RK_CRYPTO_TDES_BYTESWAP_DI BIT(5) +/* 0: ECB, 1: CBC */ +#define RK_CRYPTO_TDES_CHAINMODE_CBC BIT(4) +/* TDES Key Mode, 0 : EDE, 1 : EEE */ +#define RK_CRYPTO_TDES_EEE BIT(3) +/* 0: DES, 1:TDES */ +#define RK_CRYPTO_TDES_SELECT BIT(2) +/* 0: Slave, 1:Fifo */ +#define RK_CRYPTO_TDES_FIFO_MODE BIT(1) +/* Encryption = 0 , Decryption = 1 */ +#define RK_CRYPTO_TDES_DEC BIT(0) + +#define RK_CRYPTO_TDES_STS 0x0104 +#define RK_CRYPTO_TDES_DONE BIT(0) + +#define RK_CRYPTO_TDES_DIN_0 0x0108 +#define RK_CRYPTO_TDES_DIN_1 0x010c +#define RK_CRYPTO_TDES_DOUT_0 0x0110 +#define RK_CRYPTO_TDES_DOUT_1 0x0114 +#define RK_CRYPTO_TDES_IV_0 0x0118 +#define RK_CRYPTO_TDES_IV_1 0x011c +#define RK_CRYPTO_TDES_KEY1_0 0x0120 +#define RK_CRYPTO_TDES_KEY1_1 0x0124 +#define RK_CRYPTO_TDES_KEY2_0 0x0128 +#define RK_CRYPTO_TDES_KEY2_1 0x012c +#define RK_CRYPTO_TDES_KEY3_0 0x0130 +#define RK_CRYPTO_TDES_KEY3_1 0x0134 + +/* HASH */ +#define RK_CRYPTO_HASH_CTRL 0x0180 +#define RK_CRYPTO_HASH_SWAP_DO BIT(3) +#define RK_CRYPTO_HASH_SWAP_DI BIT(2) +#define RK_CRYPTO_HASH_SHA1 _SBF(0x00, 0) +#define RK_CRYPTO_HASH_MD5 _SBF(0x01, 0) +#define RK_CRYPTO_HASH_SHA256 _SBF(0x02, 0) +#define RK_CRYPTO_HASH_PRNG _SBF(0x03, 0) + +#define RK_CRYPTO_HASH_STS 0x0184 +#define RK_CRYPTO_HASH_DONE BIT(0) + +#define RK_CRYPTO_HASH_MSG_LEN 0x0188 +#define RK_CRYPTO_HASH_DOUT_0 0x018c +#define RK_CRYPTO_HASH_DOUT_1 0x0190 +#define RK_CRYPTO_HASH_DOUT_2 0x0194 +#define RK_CRYPTO_HASH_DOUT_3 0x0198 +#define RK_CRYPTO_HASH_DOUT_4 0x019c +#define RK_CRYPTO_HASH_DOUT_5 0x01a0 +#define RK_CRYPTO_HASH_DOUT_6 0x01a4 +#define RK_CRYPTO_HASH_DOUT_7 0x01a8 + +#define CRYPTO_READ(dev, offset) \ + readl_relaxed(((dev)->reg + (offset))) +#define CRYPTO_WRITE(dev, offset, val) \ + writel_relaxed((val), ((dev)->reg + (offset))) + +#define RK_MAX_CLKS 4 + +/* + * struct rockchip_ip - struct for managing a list of RK crypto instance + * @dev_list: Used for doing a list of rk_crypto_info + * @lock: Control access to dev_list + * @dbgfs_dir: Debugfs dentry for statistic directory + * @dbgfs_stats: Debugfs dentry for statistic counters + */ +struct rockchip_ip { + struct list_head dev_list; + spinlock_t lock; /* Control access to dev_list */ + struct dentry *dbgfs_dir; + struct dentry *dbgfs_stats; +}; + +struct rk_clks { + const char *name; + unsigned long max; +}; + +struct rk_variant { + int num_clks; + struct rk_clks rkclks[RK_MAX_CLKS]; +}; + +struct rk_crypto_info { + struct list_head list; + struct device *dev; + struct clk_bulk_data *clks; + int num_clks; + struct reset_control *rst; + void __iomem *reg; + int irq; + const struct rk_variant *variant; + unsigned long nreq; + struct crypto_engine *engine; + struct completion complete; + int status; +}; + +/* the private variable of hash */ +struct rk_ahash_ctx { + /* for fallback */ + struct crypto_ahash *fallback_tfm; +}; + +/* the private variable of hash for fallback */ +struct rk_ahash_rctx { + struct rk_crypto_info *dev; + struct ahash_request fallback_req; + u32 mode; + int nrsg; +}; + +/* the private variable of cipher */ +struct rk_cipher_ctx { + unsigned int keylen; + u8 key[AES_MAX_KEY_SIZE]; + u8 iv[AES_BLOCK_SIZE]; + struct crypto_skcipher *fallback_tfm; +}; + +struct rk_cipher_rctx { + struct rk_crypto_info *dev; + u8 backup_iv[AES_BLOCK_SIZE]; + u32 mode; + struct skcipher_request fallback_req; // keep at the end +}; + +struct rk_crypto_tmp { + u32 type; + struct rk_crypto_info *dev; + union { + struct skcipher_engine_alg skcipher; + struct ahash_engine_alg hash; + } alg; + unsigned long stat_req; + unsigned long stat_fb; + unsigned long stat_fb_len; + unsigned long stat_fb_sglen; + unsigned long stat_fb_align; + unsigned long stat_fb_sgdiff; +}; + +extern struct rk_crypto_tmp rk_ecb_aes_alg; +extern struct rk_crypto_tmp rk_cbc_aes_alg; +extern struct rk_crypto_tmp rk_ecb_des_alg; +extern struct rk_crypto_tmp rk_cbc_des_alg; +extern struct rk_crypto_tmp rk_ecb_des3_ede_alg; +extern struct rk_crypto_tmp rk_cbc_des3_ede_alg; + +extern struct rk_crypto_tmp rk_ahash_sha1; +extern struct rk_crypto_tmp rk_ahash_sha256; +extern struct rk_crypto_tmp rk_ahash_md5; + +struct rk_crypto_info *get_rk_crypto(void); +#endif diff --git a/drivers/crypto/rockchip/rk3288_crypto_ahash.c b/drivers/crypto/rockchip/rk3288_crypto_ahash.c new file mode 100644 index 000000000..d3482619a --- /dev/null +++ b/drivers/crypto/rockchip/rk3288_crypto_ahash.c @@ -0,0 +1,474 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Crypto acceleration support for Rockchip RK3288 + * + * Copyright (c) 2015, Fuzhou Rockchip Electronics Co., Ltd + * + * Author: Zain Wang + * + * Some ideas are from marvell/cesa.c and s5p-sss.c driver. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "rk3288_crypto.h" + +/* + * IC can not process zero message hash, + * so we put the fixed hash out when met zero message. + */ + +static bool rk_ahash_need_fallback(struct ahash_request *req) +{ + struct scatterlist *sg; + + sg = req->src; + while (sg) { + if (!IS_ALIGNED(sg->offset, sizeof(u32))) { + return true; + } + if (sg->length % 4) { + return true; + } + sg = sg_next(sg); + } + return false; +} + +static int rk_ahash_digest_fb(struct ahash_request *areq) +{ + struct rk_ahash_rctx *rctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct rk_ahash_ctx *tfmctx = crypto_ahash_ctx(tfm); + struct ahash_alg *alg = crypto_ahash_alg(tfm); + struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.hash.base); + + algt->stat_fb++; + + ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + areq->base.complete, areq->base.data); + ahash_request_set_crypt(&rctx->fallback_req, areq->src, areq->result, + areq->nbytes); + + return crypto_ahash_digest(&rctx->fallback_req); +} + +static int zero_message_process(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + int rk_digest_size = crypto_ahash_digestsize(tfm); + + switch (rk_digest_size) { + case SHA1_DIGEST_SIZE: + memcpy(req->result, sha1_zero_message_hash, rk_digest_size); + break; + case SHA256_DIGEST_SIZE: + memcpy(req->result, sha256_zero_message_hash, rk_digest_size); + break; + case MD5_DIGEST_SIZE: + memcpy(req->result, md5_zero_message_hash, rk_digest_size); + break; + default: + return -EINVAL; + } + + return 0; +} + +static void rk_ahash_reg_init(struct ahash_request *req, + struct rk_crypto_info *dev) +{ + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); + int reg_status; + + reg_status = CRYPTO_READ(dev, RK_CRYPTO_CTRL) | + RK_CRYPTO_HASH_FLUSH | _SBF(0xffff, 16); + CRYPTO_WRITE(dev, RK_CRYPTO_CTRL, reg_status); + + reg_status = CRYPTO_READ(dev, RK_CRYPTO_CTRL); + reg_status &= (~RK_CRYPTO_HASH_FLUSH); + reg_status |= _SBF(0xffff, 16); + CRYPTO_WRITE(dev, RK_CRYPTO_CTRL, reg_status); + + memset_io(dev->reg + RK_CRYPTO_HASH_DOUT_0, 0, 32); + + CRYPTO_WRITE(dev, RK_CRYPTO_INTENA, RK_CRYPTO_HRDMA_ERR_ENA | + RK_CRYPTO_HRDMA_DONE_ENA); + + CRYPTO_WRITE(dev, RK_CRYPTO_INTSTS, RK_CRYPTO_HRDMA_ERR_INT | + RK_CRYPTO_HRDMA_DONE_INT); + + CRYPTO_WRITE(dev, RK_CRYPTO_HASH_CTRL, rctx->mode | + RK_CRYPTO_HASH_SWAP_DO); + + CRYPTO_WRITE(dev, RK_CRYPTO_CONF, RK_CRYPTO_BYTESWAP_HRFIFO | + RK_CRYPTO_BYTESWAP_BRFIFO | + RK_CRYPTO_BYTESWAP_BTFIFO); + + CRYPTO_WRITE(dev, RK_CRYPTO_HASH_MSG_LEN, req->nbytes); +} + +static int rk_ahash_init(struct ahash_request *req) +{ + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + + return crypto_ahash_init(&rctx->fallback_req); +} + +static int rk_ahash_update(struct ahash_request *req) +{ + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, NULL, req->nbytes); + + return crypto_ahash_update(&rctx->fallback_req); +} + +static int rk_ahash_final(struct ahash_request *req) +{ + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, NULL, req->result, 0); + + return crypto_ahash_final(&rctx->fallback_req); +} + +static int rk_ahash_finup(struct ahash_request *req) +{ + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result, + req->nbytes); + + return crypto_ahash_finup(&rctx->fallback_req); +} + +static int rk_ahash_import(struct ahash_request *req, const void *in) +{ + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + + return crypto_ahash_import(&rctx->fallback_req, in); +} + +static int rk_ahash_export(struct ahash_request *req, void *out) +{ + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + + return crypto_ahash_export(&rctx->fallback_req, out); +} + +static int rk_ahash_digest(struct ahash_request *req) +{ + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); + struct rk_crypto_info *dev; + struct crypto_engine *engine; + + if (rk_ahash_need_fallback(req)) + return rk_ahash_digest_fb(req); + + if (!req->nbytes) + return zero_message_process(req); + + dev = get_rk_crypto(); + + rctx->dev = dev; + engine = dev->engine; + + return crypto_transfer_hash_request_to_engine(engine, req); +} + +static void crypto_ahash_dma_start(struct rk_crypto_info *dev, struct scatterlist *sg) +{ + CRYPTO_WRITE(dev, RK_CRYPTO_HRDMAS, sg_dma_address(sg)); + CRYPTO_WRITE(dev, RK_CRYPTO_HRDMAL, sg_dma_len(sg) / 4); + CRYPTO_WRITE(dev, RK_CRYPTO_CTRL, RK_CRYPTO_HASH_START | + (RK_CRYPTO_HASH_START << 16)); +} + +static int rk_hash_prepare(struct crypto_engine *engine, void *breq) +{ + struct ahash_request *areq = container_of(breq, struct ahash_request, base); + struct rk_ahash_rctx *rctx = ahash_request_ctx(areq); + struct rk_crypto_info *rkc = rctx->dev; + int ret; + + ret = dma_map_sg(rkc->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE); + if (ret <= 0) + return -EINVAL; + + rctx->nrsg = ret; + + return 0; +} + +static void rk_hash_unprepare(struct crypto_engine *engine, void *breq) +{ + struct ahash_request *areq = container_of(breq, struct ahash_request, base); + struct rk_ahash_rctx *rctx = ahash_request_ctx(areq); + struct rk_crypto_info *rkc = rctx->dev; + + dma_unmap_sg(rkc->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE); +} + +static int rk_hash_run(struct crypto_engine *engine, void *breq) +{ + struct ahash_request *areq = container_of(breq, struct ahash_request, base); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct rk_ahash_rctx *rctx = ahash_request_ctx(areq); + struct ahash_alg *alg = crypto_ahash_alg(tfm); + struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.hash.base); + struct scatterlist *sg = areq->src; + struct rk_crypto_info *rkc = rctx->dev; + int err; + int i; + u32 v; + + err = pm_runtime_resume_and_get(rkc->dev); + if (err) + return err; + + err = rk_hash_prepare(engine, breq); + if (err) + goto theend; + + rctx->mode = 0; + + algt->stat_req++; + rkc->nreq++; + + switch (crypto_ahash_digestsize(tfm)) { + case SHA1_DIGEST_SIZE: + rctx->mode = RK_CRYPTO_HASH_SHA1; + break; + case SHA256_DIGEST_SIZE: + rctx->mode = RK_CRYPTO_HASH_SHA256; + break; + case MD5_DIGEST_SIZE: + rctx->mode = RK_CRYPTO_HASH_MD5; + break; + default: + err = -EINVAL; + goto theend; + } + + rk_ahash_reg_init(areq, rkc); + + while (sg) { + reinit_completion(&rkc->complete); + rkc->status = 0; + crypto_ahash_dma_start(rkc, sg); + wait_for_completion_interruptible_timeout(&rkc->complete, + msecs_to_jiffies(2000)); + if (!rkc->status) { + dev_err(rkc->dev, "DMA timeout\n"); + err = -EFAULT; + goto theend; + } + sg = sg_next(sg); + } + + /* + * it will take some time to process date after last dma + * transmission. + * + * waiting time is relative with the last date len, + * so cannot set a fixed time here. + * 10us makes system not call here frequently wasting + * efficiency, and make it response quickly when dma + * complete. + */ + err = readl_poll_timeout(rkc->reg + RK_CRYPTO_HASH_STS, v, + v == 0, 10, 1000); + if (err) { + dev_err(rkc->dev, "HASH idle timeout\n"); + goto theend; + } + + for (i = 0; i < crypto_ahash_digestsize(tfm) / 4; i++) { + v = readl(rkc->reg + RK_CRYPTO_HASH_DOUT_0 + i * 4); + put_unaligned_le32(v, areq->result + i * 4); + } + +theend: + pm_runtime_put_autosuspend(rkc->dev); + + rk_hash_unprepare(engine, breq); + + local_bh_disable(); + crypto_finalize_hash_request(engine, breq, err); + local_bh_enable(); + + return 0; +} + +static int rk_hash_init_tfm(struct crypto_ahash *tfm) +{ + struct rk_ahash_ctx *tctx = crypto_ahash_ctx(tfm); + const char *alg_name = crypto_ahash_alg_name(tfm); + struct ahash_alg *alg = crypto_ahash_alg(tfm); + struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.hash.base); + + /* for fallback */ + tctx->fallback_tfm = crypto_alloc_ahash(alg_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(tctx->fallback_tfm)) { + dev_err(algt->dev->dev, "Could not load fallback driver.\n"); + return PTR_ERR(tctx->fallback_tfm); + } + + crypto_ahash_set_reqsize(tfm, + sizeof(struct rk_ahash_rctx) + + crypto_ahash_reqsize(tctx->fallback_tfm)); + + return 0; +} + +static void rk_hash_exit_tfm(struct crypto_ahash *tfm) +{ + struct rk_ahash_ctx *tctx = crypto_ahash_ctx(tfm); + + crypto_free_ahash(tctx->fallback_tfm); +} + +struct rk_crypto_tmp rk_ahash_sha1 = { + .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash.base = { + .init = rk_ahash_init, + .update = rk_ahash_update, + .final = rk_ahash_final, + .finup = rk_ahash_finup, + .export = rk_ahash_export, + .import = rk_ahash_import, + .digest = rk_ahash_digest, + .init_tfm = rk_hash_init_tfm, + .exit_tfm = rk_hash_exit_tfm, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct sha1_state), + .base = { + .cra_name = "sha1", + .cra_driver_name = "rk-sha1", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct rk_ahash_ctx), + .cra_module = THIS_MODULE, + } + } + }, + .alg.hash.op = { + .do_one_request = rk_hash_run, + }, +}; + +struct rk_crypto_tmp rk_ahash_sha256 = { + .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash.base = { + .init = rk_ahash_init, + .update = rk_ahash_update, + .final = rk_ahash_final, + .finup = rk_ahash_finup, + .export = rk_ahash_export, + .import = rk_ahash_import, + .digest = rk_ahash_digest, + .init_tfm = rk_hash_init_tfm, + .exit_tfm = rk_hash_exit_tfm, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha256", + .cra_driver_name = "rk-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct rk_ahash_ctx), + .cra_module = THIS_MODULE, + } + } + }, + .alg.hash.op = { + .do_one_request = rk_hash_run, + }, +}; + +struct rk_crypto_tmp rk_ahash_md5 = { + .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash.base = { + .init = rk_ahash_init, + .update = rk_ahash_update, + .final = rk_ahash_final, + .finup = rk_ahash_finup, + .export = rk_ahash_export, + .import = rk_ahash_import, + .digest = rk_ahash_digest, + .init_tfm = rk_hash_init_tfm, + .exit_tfm = rk_hash_exit_tfm, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct md5_state), + .base = { + .cra_name = "md5", + .cra_driver_name = "rk-md5", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct rk_ahash_ctx), + .cra_module = THIS_MODULE, + } + } + }, + .alg.hash.op = { + .do_one_request = rk_hash_run, + }, +}; diff --git a/drivers/crypto/rockchip/rk3288_crypto_skcipher.c b/drivers/crypto/rockchip/rk3288_crypto_skcipher.c new file mode 100644 index 000000000..e80f9148c --- /dev/null +++ b/drivers/crypto/rockchip/rk3288_crypto_skcipher.c @@ -0,0 +1,612 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Crypto acceleration support for Rockchip RK3288 + * + * Copyright (c) 2015, Fuzhou Rockchip Electronics Co., Ltd + * + * Author: Zain Wang + * + * Some ideas are from marvell-cesa.c and s5p-sss.c driver. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "rk3288_crypto.h" + +#define RK_CRYPTO_DEC BIT(0) + +static int rk_cipher_need_fallback(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.skcipher.base); + struct scatterlist *sgs, *sgd; + unsigned int stodo, dtodo, len; + unsigned int bs = crypto_skcipher_blocksize(tfm); + + if (!req->cryptlen) + return true; + + len = req->cryptlen; + sgs = req->src; + sgd = req->dst; + while (sgs && sgd) { + if (!IS_ALIGNED(sgs->offset, sizeof(u32))) { + algt->stat_fb_align++; + return true; + } + if (!IS_ALIGNED(sgd->offset, sizeof(u32))) { + algt->stat_fb_align++; + return true; + } + stodo = min(len, sgs->length); + if (stodo % bs) { + algt->stat_fb_len++; + return true; + } + dtodo = min(len, sgd->length); + if (dtodo % bs) { + algt->stat_fb_len++; + return true; + } + if (stodo != dtodo) { + algt->stat_fb_sgdiff++; + return true; + } + len -= stodo; + sgs = sg_next(sgs); + sgd = sg_next(sgd); + } + return false; +} + +static int rk_cipher_fallback(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct rk_cipher_ctx *op = crypto_skcipher_ctx(tfm); + struct rk_cipher_rctx *rctx = skcipher_request_ctx(areq); + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.skcipher.base); + int err; + + algt->stat_fb++; + + skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm); + skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags, + areq->base.complete, areq->base.data); + skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst, + areq->cryptlen, areq->iv); + if (rctx->mode & RK_CRYPTO_DEC) + err = crypto_skcipher_decrypt(&rctx->fallback_req); + else + err = crypto_skcipher_encrypt(&rctx->fallback_req); + return err; +} + +static int rk_cipher_handle_req(struct skcipher_request *req) +{ + struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); + struct rk_crypto_info *rkc; + struct crypto_engine *engine; + + if (rk_cipher_need_fallback(req)) + return rk_cipher_fallback(req); + + rkc = get_rk_crypto(); + + engine = rkc->engine; + rctx->dev = rkc; + + return crypto_transfer_skcipher_request_to_engine(engine, req); +} + +static int rk_aes_setkey(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); + struct rk_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) + return -EINVAL; + ctx->keylen = keylen; + memcpy(ctx->key, key, keylen); + + return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen); +} + +static int rk_des_setkey(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); + int err; + + err = verify_skcipher_des_key(cipher, key); + if (err) + return err; + + ctx->keylen = keylen; + memcpy(ctx->key, key, keylen); + + return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen); +} + +static int rk_tdes_setkey(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); + int err; + + err = verify_skcipher_des3_key(cipher, key); + if (err) + return err; + + ctx->keylen = keylen; + memcpy(ctx->key, key, keylen); + + return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen); +} + +static int rk_aes_ecb_encrypt(struct skcipher_request *req) +{ + struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); + + rctx->mode = RK_CRYPTO_AES_ECB_MODE; + return rk_cipher_handle_req(req); +} + +static int rk_aes_ecb_decrypt(struct skcipher_request *req) +{ + struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); + + rctx->mode = RK_CRYPTO_AES_ECB_MODE | RK_CRYPTO_DEC; + return rk_cipher_handle_req(req); +} + +static int rk_aes_cbc_encrypt(struct skcipher_request *req) +{ + struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); + + rctx->mode = RK_CRYPTO_AES_CBC_MODE; + return rk_cipher_handle_req(req); +} + +static int rk_aes_cbc_decrypt(struct skcipher_request *req) +{ + struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); + + rctx->mode = RK_CRYPTO_AES_CBC_MODE | RK_CRYPTO_DEC; + return rk_cipher_handle_req(req); +} + +static int rk_des_ecb_encrypt(struct skcipher_request *req) +{ + struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); + + rctx->mode = 0; + return rk_cipher_handle_req(req); +} + +static int rk_des_ecb_decrypt(struct skcipher_request *req) +{ + struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); + + rctx->mode = RK_CRYPTO_DEC; + return rk_cipher_handle_req(req); +} + +static int rk_des_cbc_encrypt(struct skcipher_request *req) +{ + struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); + + rctx->mode = RK_CRYPTO_TDES_CHAINMODE_CBC; + return rk_cipher_handle_req(req); +} + +static int rk_des_cbc_decrypt(struct skcipher_request *req) +{ + struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); + + rctx->mode = RK_CRYPTO_TDES_CHAINMODE_CBC | RK_CRYPTO_DEC; + return rk_cipher_handle_req(req); +} + +static int rk_des3_ede_ecb_encrypt(struct skcipher_request *req) +{ + struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); + + rctx->mode = RK_CRYPTO_TDES_SELECT; + return rk_cipher_handle_req(req); +} + +static int rk_des3_ede_ecb_decrypt(struct skcipher_request *req) +{ + struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); + + rctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_DEC; + return rk_cipher_handle_req(req); +} + +static int rk_des3_ede_cbc_encrypt(struct skcipher_request *req) +{ + struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); + + rctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC; + return rk_cipher_handle_req(req); +} + +static int rk_des3_ede_cbc_decrypt(struct skcipher_request *req) +{ + struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); + + rctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC | + RK_CRYPTO_DEC; + return rk_cipher_handle_req(req); +} + +static void rk_cipher_hw_init(struct rk_crypto_info *dev, struct skcipher_request *req) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); + struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); + struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); + u32 block, conf_reg = 0; + + block = crypto_tfm_alg_blocksize(tfm); + + if (block == DES_BLOCK_SIZE) { + rctx->mode |= RK_CRYPTO_TDES_FIFO_MODE | + RK_CRYPTO_TDES_BYTESWAP_KEY | + RK_CRYPTO_TDES_BYTESWAP_IV; + CRYPTO_WRITE(dev, RK_CRYPTO_TDES_CTRL, rctx->mode); + memcpy_toio(dev->reg + RK_CRYPTO_TDES_KEY1_0, ctx->key, ctx->keylen); + conf_reg = RK_CRYPTO_DESSEL; + } else { + rctx->mode |= RK_CRYPTO_AES_FIFO_MODE | + RK_CRYPTO_AES_KEY_CHANGE | + RK_CRYPTO_AES_BYTESWAP_KEY | + RK_CRYPTO_AES_BYTESWAP_IV; + if (ctx->keylen == AES_KEYSIZE_192) + rctx->mode |= RK_CRYPTO_AES_192BIT_key; + else if (ctx->keylen == AES_KEYSIZE_256) + rctx->mode |= RK_CRYPTO_AES_256BIT_key; + CRYPTO_WRITE(dev, RK_CRYPTO_AES_CTRL, rctx->mode); + memcpy_toio(dev->reg + RK_CRYPTO_AES_KEY_0, ctx->key, ctx->keylen); + } + conf_reg |= RK_CRYPTO_BYTESWAP_BTFIFO | + RK_CRYPTO_BYTESWAP_BRFIFO; + CRYPTO_WRITE(dev, RK_CRYPTO_CONF, conf_reg); + CRYPTO_WRITE(dev, RK_CRYPTO_INTENA, + RK_CRYPTO_BCDMA_ERR_ENA | RK_CRYPTO_BCDMA_DONE_ENA); +} + +static void crypto_dma_start(struct rk_crypto_info *dev, + struct scatterlist *sgs, + struct scatterlist *sgd, unsigned int todo) +{ + CRYPTO_WRITE(dev, RK_CRYPTO_BRDMAS, sg_dma_address(sgs)); + CRYPTO_WRITE(dev, RK_CRYPTO_BRDMAL, todo); + CRYPTO_WRITE(dev, RK_CRYPTO_BTDMAS, sg_dma_address(sgd)); + CRYPTO_WRITE(dev, RK_CRYPTO_CTRL, RK_CRYPTO_BLOCK_START | + _SBF(RK_CRYPTO_BLOCK_START, 16)); +} + +static int rk_cipher_run(struct crypto_engine *engine, void *async_req) +{ + struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct rk_cipher_rctx *rctx = skcipher_request_ctx(areq); + struct scatterlist *sgs, *sgd; + int err = 0; + int ivsize = crypto_skcipher_ivsize(tfm); + int offset; + u8 iv[AES_BLOCK_SIZE]; + u8 biv[AES_BLOCK_SIZE]; + u8 *ivtouse = areq->iv; + unsigned int len = areq->cryptlen; + unsigned int todo; + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.skcipher.base); + struct rk_crypto_info *rkc = rctx->dev; + + err = pm_runtime_resume_and_get(rkc->dev); + if (err) + return err; + + algt->stat_req++; + rkc->nreq++; + + if (areq->iv && ivsize > 0) { + if (rctx->mode & RK_CRYPTO_DEC) { + offset = areq->cryptlen - ivsize; + scatterwalk_map_and_copy(rctx->backup_iv, areq->src, + offset, ivsize, 0); + } + } + + sgs = areq->src; + sgd = areq->dst; + + while (sgs && sgd && len) { + if (!sgs->length) { + sgs = sg_next(sgs); + sgd = sg_next(sgd); + continue; + } + if (rctx->mode & RK_CRYPTO_DEC) { + /* we backup last block of source to be used as IV at next step */ + offset = sgs->length - ivsize; + scatterwalk_map_and_copy(biv, sgs, offset, ivsize, 0); + } + if (sgs == sgd) { + err = dma_map_sg(rkc->dev, sgs, 1, DMA_BIDIRECTIONAL); + if (err <= 0) { + err = -EINVAL; + goto theend_iv; + } + } else { + err = dma_map_sg(rkc->dev, sgs, 1, DMA_TO_DEVICE); + if (err <= 0) { + err = -EINVAL; + goto theend_iv; + } + err = dma_map_sg(rkc->dev, sgd, 1, DMA_FROM_DEVICE); + if (err <= 0) { + err = -EINVAL; + goto theend_sgs; + } + } + err = 0; + rk_cipher_hw_init(rkc, areq); + if (ivsize) { + if (ivsize == DES_BLOCK_SIZE) + memcpy_toio(rkc->reg + RK_CRYPTO_TDES_IV_0, ivtouse, ivsize); + else + memcpy_toio(rkc->reg + RK_CRYPTO_AES_IV_0, ivtouse, ivsize); + } + reinit_completion(&rkc->complete); + rkc->status = 0; + + todo = min(sg_dma_len(sgs), len); + len -= todo; + crypto_dma_start(rkc, sgs, sgd, todo / 4); + wait_for_completion_interruptible_timeout(&rkc->complete, + msecs_to_jiffies(2000)); + if (!rkc->status) { + dev_err(rkc->dev, "DMA timeout\n"); + err = -EFAULT; + goto theend; + } + if (sgs == sgd) { + dma_unmap_sg(rkc->dev, sgs, 1, DMA_BIDIRECTIONAL); + } else { + dma_unmap_sg(rkc->dev, sgs, 1, DMA_TO_DEVICE); + dma_unmap_sg(rkc->dev, sgd, 1, DMA_FROM_DEVICE); + } + if (rctx->mode & RK_CRYPTO_DEC) { + memcpy(iv, biv, ivsize); + ivtouse = iv; + } else { + offset = sgd->length - ivsize; + scatterwalk_map_and_copy(iv, sgd, offset, ivsize, 0); + ivtouse = iv; + } + sgs = sg_next(sgs); + sgd = sg_next(sgd); + } + + if (areq->iv && ivsize > 0) { + offset = areq->cryptlen - ivsize; + if (rctx->mode & RK_CRYPTO_DEC) { + memcpy(areq->iv, rctx->backup_iv, ivsize); + memzero_explicit(rctx->backup_iv, ivsize); + } else { + scatterwalk_map_and_copy(areq->iv, areq->dst, offset, + ivsize, 0); + } + } + +theend: + pm_runtime_put_autosuspend(rkc->dev); + + local_bh_disable(); + crypto_finalize_skcipher_request(engine, areq, err); + local_bh_enable(); + return 0; + +theend_sgs: + if (sgs == sgd) { + dma_unmap_sg(rkc->dev, sgs, 1, DMA_BIDIRECTIONAL); + } else { + dma_unmap_sg(rkc->dev, sgs, 1, DMA_TO_DEVICE); + dma_unmap_sg(rkc->dev, sgd, 1, DMA_FROM_DEVICE); + } +theend_iv: + return err; +} + +static int rk_cipher_tfm_init(struct crypto_skcipher *tfm) +{ + struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); + const char *name = crypto_tfm_alg_name(&tfm->base); + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.skcipher.base); + + ctx->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fallback_tfm)) { + dev_err(algt->dev->dev, "ERROR: Cannot allocate fallback for %s %ld\n", + name, PTR_ERR(ctx->fallback_tfm)); + return PTR_ERR(ctx->fallback_tfm); + } + + crypto_skcipher_set_reqsize(tfm, sizeof(struct rk_cipher_rctx) + + crypto_skcipher_reqsize(ctx->fallback_tfm)); + + return 0; +} + +static void rk_cipher_tfm_exit(struct crypto_skcipher *tfm) +{ + struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); + + memzero_explicit(ctx->key, ctx->keylen); + crypto_free_skcipher(ctx->fallback_tfm); +} + +struct rk_crypto_tmp rk_ecb_aes_alg = { + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher.base = { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-rk", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct rk_cipher_ctx), + .base.cra_alignmask = 0x0f, + .base.cra_module = THIS_MODULE, + + .init = rk_cipher_tfm_init, + .exit = rk_cipher_tfm_exit, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = rk_aes_setkey, + .encrypt = rk_aes_ecb_encrypt, + .decrypt = rk_aes_ecb_decrypt, + }, + .alg.skcipher.op = { + .do_one_request = rk_cipher_run, + }, +}; + +struct rk_crypto_tmp rk_cbc_aes_alg = { + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher.base = { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-rk", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct rk_cipher_ctx), + .base.cra_alignmask = 0x0f, + .base.cra_module = THIS_MODULE, + + .init = rk_cipher_tfm_init, + .exit = rk_cipher_tfm_exit, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = rk_aes_setkey, + .encrypt = rk_aes_cbc_encrypt, + .decrypt = rk_aes_cbc_decrypt, + }, + .alg.skcipher.op = { + .do_one_request = rk_cipher_run, + }, +}; + +struct rk_crypto_tmp rk_ecb_des_alg = { + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher.base = { + .base.cra_name = "ecb(des)", + .base.cra_driver_name = "ecb-des-rk", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct rk_cipher_ctx), + .base.cra_alignmask = 0x07, + .base.cra_module = THIS_MODULE, + + .init = rk_cipher_tfm_init, + .exit = rk_cipher_tfm_exit, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = rk_des_setkey, + .encrypt = rk_des_ecb_encrypt, + .decrypt = rk_des_ecb_decrypt, + }, + .alg.skcipher.op = { + .do_one_request = rk_cipher_run, + }, +}; + +struct rk_crypto_tmp rk_cbc_des_alg = { + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher.base = { + .base.cra_name = "cbc(des)", + .base.cra_driver_name = "cbc-des-rk", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct rk_cipher_ctx), + .base.cra_alignmask = 0x07, + .base.cra_module = THIS_MODULE, + + .init = rk_cipher_tfm_init, + .exit = rk_cipher_tfm_exit, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = rk_des_setkey, + .encrypt = rk_des_cbc_encrypt, + .decrypt = rk_des_cbc_decrypt, + }, + .alg.skcipher.op = { + .do_one_request = rk_cipher_run, + }, +}; + +struct rk_crypto_tmp rk_ecb_des3_ede_alg = { + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher.base = { + .base.cra_name = "ecb(des3_ede)", + .base.cra_driver_name = "ecb-des3-ede-rk", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct rk_cipher_ctx), + .base.cra_alignmask = 0x07, + .base.cra_module = THIS_MODULE, + + .init = rk_cipher_tfm_init, + .exit = rk_cipher_tfm_exit, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = rk_tdes_setkey, + .encrypt = rk_des3_ede_ecb_encrypt, + .decrypt = rk_des3_ede_ecb_decrypt, + }, + .alg.skcipher.op = { + .do_one_request = rk_cipher_run, + }, +}; + +struct rk_crypto_tmp rk_cbc_des3_ede_alg = { + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher.base = { + .base.cra_name = "cbc(des3_ede)", + .base.cra_driver_name = "cbc-des3-ede-rk", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct rk_cipher_ctx), + .base.cra_alignmask = 0x07, + .base.cra_module = THIS_MODULE, + + .init = rk_cipher_tfm_init, + .exit = rk_cipher_tfm_exit, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = rk_tdes_setkey, + .encrypt = rk_des3_ede_cbc_encrypt, + .decrypt = rk_des3_ede_cbc_decrypt, + }, + .alg.skcipher.op = { + .do_one_request = rk_cipher_run, + }, +}; diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c new file mode 100644 index 000000000..bdda7b39a --- /dev/null +++ b/drivers/crypto/s5p-sss.c @@ -0,0 +1,2325 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Cryptographic API. +// +// Support for Samsung S5PV210 and Exynos HW acceleration. +// +// Copyright (C) 2011 NetUP Inc. All rights reserved. +// Copyright (c) 2017 Samsung Electronics Co., Ltd. All rights reserved. +// +// Hash part based on omap-sham.c driver. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define _SBF(s, v) ((v) << (s)) + +/* Feed control registers */ +#define SSS_REG_FCINTSTAT 0x0000 +#define SSS_FCINTSTAT_HPARTINT BIT(7) +#define SSS_FCINTSTAT_HDONEINT BIT(5) +#define SSS_FCINTSTAT_BRDMAINT BIT(3) +#define SSS_FCINTSTAT_BTDMAINT BIT(2) +#define SSS_FCINTSTAT_HRDMAINT BIT(1) +#define SSS_FCINTSTAT_PKDMAINT BIT(0) + +#define SSS_REG_FCINTENSET 0x0004 +#define SSS_FCINTENSET_HPARTINTENSET BIT(7) +#define SSS_FCINTENSET_HDONEINTENSET BIT(5) +#define SSS_FCINTENSET_BRDMAINTENSET BIT(3) +#define SSS_FCINTENSET_BTDMAINTENSET BIT(2) +#define SSS_FCINTENSET_HRDMAINTENSET BIT(1) +#define SSS_FCINTENSET_PKDMAINTENSET BIT(0) + +#define SSS_REG_FCINTENCLR 0x0008 +#define SSS_FCINTENCLR_HPARTINTENCLR BIT(7) +#define SSS_FCINTENCLR_HDONEINTENCLR BIT(5) +#define SSS_FCINTENCLR_BRDMAINTENCLR BIT(3) +#define SSS_FCINTENCLR_BTDMAINTENCLR BIT(2) +#define SSS_FCINTENCLR_HRDMAINTENCLR BIT(1) +#define SSS_FCINTENCLR_PKDMAINTENCLR BIT(0) + +#define SSS_REG_FCINTPEND 0x000C +#define SSS_FCINTPEND_HPARTINTP BIT(7) +#define SSS_FCINTPEND_HDONEINTP BIT(5) +#define SSS_FCINTPEND_BRDMAINTP BIT(3) +#define SSS_FCINTPEND_BTDMAINTP BIT(2) +#define SSS_FCINTPEND_HRDMAINTP BIT(1) +#define SSS_FCINTPEND_PKDMAINTP BIT(0) + +#define SSS_REG_FCFIFOSTAT 0x0010 +#define SSS_FCFIFOSTAT_BRFIFOFUL BIT(7) +#define SSS_FCFIFOSTAT_BRFIFOEMP BIT(6) +#define SSS_FCFIFOSTAT_BTFIFOFUL BIT(5) +#define SSS_FCFIFOSTAT_BTFIFOEMP BIT(4) +#define SSS_FCFIFOSTAT_HRFIFOFUL BIT(3) +#define SSS_FCFIFOSTAT_HRFIFOEMP BIT(2) +#define SSS_FCFIFOSTAT_PKFIFOFUL BIT(1) +#define SSS_FCFIFOSTAT_PKFIFOEMP BIT(0) + +#define SSS_REG_FCFIFOCTRL 0x0014 +#define SSS_FCFIFOCTRL_DESSEL BIT(2) +#define SSS_HASHIN_INDEPENDENT _SBF(0, 0x00) +#define SSS_HASHIN_CIPHER_INPUT _SBF(0, 0x01) +#define SSS_HASHIN_CIPHER_OUTPUT _SBF(0, 0x02) +#define SSS_HASHIN_MASK _SBF(0, 0x03) + +#define SSS_REG_FCBRDMAS 0x0020 +#define SSS_REG_FCBRDMAL 0x0024 +#define SSS_REG_FCBRDMAC 0x0028 +#define SSS_FCBRDMAC_BYTESWAP BIT(1) +#define SSS_FCBRDMAC_FLUSH BIT(0) + +#define SSS_REG_FCBTDMAS 0x0030 +#define SSS_REG_FCBTDMAL 0x0034 +#define SSS_REG_FCBTDMAC 0x0038 +#define SSS_FCBTDMAC_BYTESWAP BIT(1) +#define SSS_FCBTDMAC_FLUSH BIT(0) + +#define SSS_REG_FCHRDMAS 0x0040 +#define SSS_REG_FCHRDMAL 0x0044 +#define SSS_REG_FCHRDMAC 0x0048 +#define SSS_FCHRDMAC_BYTESWAP BIT(1) +#define SSS_FCHRDMAC_FLUSH BIT(0) + +#define SSS_REG_FCPKDMAS 0x0050 +#define SSS_REG_FCPKDMAL 0x0054 +#define SSS_REG_FCPKDMAC 0x0058 +#define SSS_FCPKDMAC_BYTESWAP BIT(3) +#define SSS_FCPKDMAC_DESCEND BIT(2) +#define SSS_FCPKDMAC_TRANSMIT BIT(1) +#define SSS_FCPKDMAC_FLUSH BIT(0) + +#define SSS_REG_FCPKDMAO 0x005C + +/* AES registers */ +#define SSS_REG_AES_CONTROL 0x00 +#define SSS_AES_BYTESWAP_DI BIT(11) +#define SSS_AES_BYTESWAP_DO BIT(10) +#define SSS_AES_BYTESWAP_IV BIT(9) +#define SSS_AES_BYTESWAP_CNT BIT(8) +#define SSS_AES_BYTESWAP_KEY BIT(7) +#define SSS_AES_KEY_CHANGE_MODE BIT(6) +#define SSS_AES_KEY_SIZE_128 _SBF(4, 0x00) +#define SSS_AES_KEY_SIZE_192 _SBF(4, 0x01) +#define SSS_AES_KEY_SIZE_256 _SBF(4, 0x02) +#define SSS_AES_FIFO_MODE BIT(3) +#define SSS_AES_CHAIN_MODE_ECB _SBF(1, 0x00) +#define SSS_AES_CHAIN_MODE_CBC _SBF(1, 0x01) +#define SSS_AES_CHAIN_MODE_CTR _SBF(1, 0x02) +#define SSS_AES_MODE_DECRYPT BIT(0) + +#define SSS_REG_AES_STATUS 0x04 +#define SSS_AES_BUSY BIT(2) +#define SSS_AES_INPUT_READY BIT(1) +#define SSS_AES_OUTPUT_READY BIT(0) + +#define SSS_REG_AES_IN_DATA(s) (0x10 + (s << 2)) +#define SSS_REG_AES_OUT_DATA(s) (0x20 + (s << 2)) +#define SSS_REG_AES_IV_DATA(s) (0x30 + (s << 2)) +#define SSS_REG_AES_CNT_DATA(s) (0x40 + (s << 2)) +#define SSS_REG_AES_KEY_DATA(s) (0x80 + (s << 2)) + +#define SSS_REG(dev, reg) ((dev)->ioaddr + (SSS_REG_##reg)) +#define SSS_READ(dev, reg) __raw_readl(SSS_REG(dev, reg)) +#define SSS_WRITE(dev, reg, val) __raw_writel((val), SSS_REG(dev, reg)) + +#define SSS_AES_REG(dev, reg) ((dev)->aes_ioaddr + SSS_REG_##reg) +#define SSS_AES_WRITE(dev, reg, val) __raw_writel((val), \ + SSS_AES_REG(dev, reg)) + +/* HW engine modes */ +#define FLAGS_AES_DECRYPT BIT(0) +#define FLAGS_AES_MODE_MASK _SBF(1, 0x03) +#define FLAGS_AES_CBC _SBF(1, 0x01) +#define FLAGS_AES_CTR _SBF(1, 0x02) + +#define AES_KEY_LEN 16 +#define CRYPTO_QUEUE_LEN 1 + +/* HASH registers */ +#define SSS_REG_HASH_CTRL 0x00 + +#define SSS_HASH_USER_IV_EN BIT(5) +#define SSS_HASH_INIT_BIT BIT(4) +#define SSS_HASH_ENGINE_SHA1 _SBF(1, 0x00) +#define SSS_HASH_ENGINE_MD5 _SBF(1, 0x01) +#define SSS_HASH_ENGINE_SHA256 _SBF(1, 0x02) + +#define SSS_HASH_ENGINE_MASK _SBF(1, 0x03) + +#define SSS_REG_HASH_CTRL_PAUSE 0x04 + +#define SSS_HASH_PAUSE BIT(0) + +#define SSS_REG_HASH_CTRL_FIFO 0x08 + +#define SSS_HASH_FIFO_MODE_DMA BIT(0) +#define SSS_HASH_FIFO_MODE_CPU 0 + +#define SSS_REG_HASH_CTRL_SWAP 0x0C + +#define SSS_HASH_BYTESWAP_DI BIT(3) +#define SSS_HASH_BYTESWAP_DO BIT(2) +#define SSS_HASH_BYTESWAP_IV BIT(1) +#define SSS_HASH_BYTESWAP_KEY BIT(0) + +#define SSS_REG_HASH_STATUS 0x10 + +#define SSS_HASH_STATUS_MSG_DONE BIT(6) +#define SSS_HASH_STATUS_PARTIAL_DONE BIT(4) +#define SSS_HASH_STATUS_BUFFER_READY BIT(0) + +#define SSS_REG_HASH_MSG_SIZE_LOW 0x20 +#define SSS_REG_HASH_MSG_SIZE_HIGH 0x24 + +#define SSS_REG_HASH_PRE_MSG_SIZE_LOW 0x28 +#define SSS_REG_HASH_PRE_MSG_SIZE_HIGH 0x2C + +#define SSS_REG_HASH_IV(s) (0xB0 + ((s) << 2)) +#define SSS_REG_HASH_OUT(s) (0x100 + ((s) << 2)) + +#define HASH_BLOCK_SIZE 64 +#define HASH_REG_SIZEOF 4 +#define HASH_MD5_MAX_REG (MD5_DIGEST_SIZE / HASH_REG_SIZEOF) +#define HASH_SHA1_MAX_REG (SHA1_DIGEST_SIZE / HASH_REG_SIZEOF) +#define HASH_SHA256_MAX_REG (SHA256_DIGEST_SIZE / HASH_REG_SIZEOF) + +/* + * HASH bit numbers, used by device, setting in dev->hash_flags with + * functions set_bit(), clear_bit() or tested with test_bit() or BIT(), + * to keep HASH state BUSY or FREE, or to signal state from irq_handler + * to hash_tasklet. SGS keep track of allocated memory for scatterlist + */ +#define HASH_FLAGS_BUSY 0 +#define HASH_FLAGS_FINAL 1 +#define HASH_FLAGS_DMA_ACTIVE 2 +#define HASH_FLAGS_OUTPUT_READY 3 +#define HASH_FLAGS_DMA_READY 4 +#define HASH_FLAGS_SGS_COPIED 5 +#define HASH_FLAGS_SGS_ALLOCED 6 + +/* HASH HW constants */ +#define BUFLEN HASH_BLOCK_SIZE + +#define SSS_HASH_QUEUE_LENGTH 10 + +/** + * struct samsung_aes_variant - platform specific SSS driver data + * @aes_offset: AES register offset from SSS module's base. + * @hash_offset: HASH register offset from SSS module's base. + * @clk_names: names of clocks needed to run SSS IP + * + * Specifies platform specific configuration of SSS module. + * Note: A structure for driver specific platform data is used for future + * expansion of its usage. + */ +struct samsung_aes_variant { + unsigned int aes_offset; + unsigned int hash_offset; + const char *clk_names[2]; +}; + +struct s5p_aes_reqctx { + unsigned long mode; +}; + +struct s5p_aes_ctx { + struct s5p_aes_dev *dev; + + u8 aes_key[AES_MAX_KEY_SIZE]; + u8 nonce[CTR_RFC3686_NONCE_SIZE]; + int keylen; +}; + +/** + * struct s5p_aes_dev - Crypto device state container + * @dev: Associated device + * @clk: Clock for accessing hardware + * @pclk: APB bus clock necessary to access the hardware + * @ioaddr: Mapped IO memory region + * @aes_ioaddr: Per-varian offset for AES block IO memory + * @irq_fc: Feed control interrupt line + * @req: Crypto request currently handled by the device + * @ctx: Configuration for currently handled crypto request + * @sg_src: Scatter list with source data for currently handled block + * in device. This is DMA-mapped into device. + * @sg_dst: Scatter list with destination data for currently handled block + * in device. This is DMA-mapped into device. + * @sg_src_cpy: In case of unaligned access, copied scatter list + * with source data. + * @sg_dst_cpy: In case of unaligned access, copied scatter list + * with destination data. + * @tasklet: New request scheduling jib + * @queue: Crypto queue + * @busy: Indicates whether the device is currently handling some request + * thus it uses some of the fields from this state, like: + * req, ctx, sg_src/dst (and copies). This essentially + * protects against concurrent access to these fields. + * @lock: Lock for protecting both access to device hardware registers + * and fields related to current request (including the busy field). + * @res: Resources for hash. + * @io_hash_base: Per-variant offset for HASH block IO memory. + * @hash_lock: Lock for protecting hash_req, hash_queue and hash_flags + * variable. + * @hash_flags: Flags for current HASH op. + * @hash_queue: Async hash queue. + * @hash_tasklet: New HASH request scheduling job. + * @xmit_buf: Buffer for current HASH request transfer into SSS block. + * @hash_req: Current request sending to SSS HASH block. + * @hash_sg_iter: Scatterlist transferred through DMA into SSS HASH block. + * @hash_sg_cnt: Counter for hash_sg_iter. + * + * @use_hash: true if HASH algs enabled + */ +struct s5p_aes_dev { + struct device *dev; + struct clk *clk; + struct clk *pclk; + void __iomem *ioaddr; + void __iomem *aes_ioaddr; + int irq_fc; + + struct skcipher_request *req; + struct s5p_aes_ctx *ctx; + struct scatterlist *sg_src; + struct scatterlist *sg_dst; + + struct scatterlist *sg_src_cpy; + struct scatterlist *sg_dst_cpy; + + struct tasklet_struct tasklet; + struct crypto_queue queue; + bool busy; + spinlock_t lock; + + struct resource *res; + void __iomem *io_hash_base; + + spinlock_t hash_lock; /* protect hash_ vars */ + unsigned long hash_flags; + struct crypto_queue hash_queue; + struct tasklet_struct hash_tasklet; + + u8 xmit_buf[BUFLEN]; + struct ahash_request *hash_req; + struct scatterlist *hash_sg_iter; + unsigned int hash_sg_cnt; + + bool use_hash; +}; + +/** + * struct s5p_hash_reqctx - HASH request context + * @dd: Associated device + * @op_update: Current request operation (OP_UPDATE or OP_FINAL) + * @digcnt: Number of bytes processed by HW (without buffer[] ones) + * @digest: Digest message or IV for partial result + * @nregs: Number of HW registers for digest or IV read/write + * @engine: Bits for selecting type of HASH in SSS block + * @sg: sg for DMA transfer + * @sg_len: Length of sg for DMA transfer + * @sgl: sg for joining buffer and req->src scatterlist + * @skip: Skip offset in req->src for current op + * @total: Total number of bytes for current request + * @finup: Keep state for finup or final. + * @error: Keep track of error. + * @bufcnt: Number of bytes holded in buffer[] + * @buffer: For byte(s) from end of req->src in UPDATE op + */ +struct s5p_hash_reqctx { + struct s5p_aes_dev *dd; + bool op_update; + + u64 digcnt; + u8 digest[SHA256_DIGEST_SIZE]; + + unsigned int nregs; /* digest_size / sizeof(reg) */ + u32 engine; + + struct scatterlist *sg; + unsigned int sg_len; + struct scatterlist sgl[2]; + unsigned int skip; + unsigned int total; + bool finup; + bool error; + + u32 bufcnt; + u8 buffer[]; +}; + +/** + * struct s5p_hash_ctx - HASH transformation context + * @dd: Associated device + * @flags: Bits for algorithm HASH. + * @fallback: Software transformation for zero message or size < BUFLEN. + */ +struct s5p_hash_ctx { + struct s5p_aes_dev *dd; + unsigned long flags; + struct crypto_shash *fallback; +}; + +static const struct samsung_aes_variant s5p_aes_data = { + .aes_offset = 0x4000, + .hash_offset = 0x6000, + .clk_names = { "secss", }, +}; + +static const struct samsung_aes_variant exynos_aes_data = { + .aes_offset = 0x200, + .hash_offset = 0x400, + .clk_names = { "secss", }, +}; + +static const struct samsung_aes_variant exynos5433_slim_aes_data = { + .aes_offset = 0x400, + .hash_offset = 0x800, + .clk_names = { "aclk", "pclk", }, +}; + +static const struct of_device_id s5p_sss_dt_match[] = { + { + .compatible = "samsung,s5pv210-secss", + .data = &s5p_aes_data, + }, + { + .compatible = "samsung,exynos4210-secss", + .data = &exynos_aes_data, + }, + { + .compatible = "samsung,exynos5433-slim-sss", + .data = &exynos5433_slim_aes_data, + }, + { }, +}; +MODULE_DEVICE_TABLE(of, s5p_sss_dt_match); + +static inline const struct samsung_aes_variant *find_s5p_sss_version + (const struct platform_device *pdev) +{ + if (IS_ENABLED(CONFIG_OF) && (pdev->dev.of_node)) + return of_device_get_match_data(&pdev->dev); + + return (const struct samsung_aes_variant *) + platform_get_device_id(pdev)->driver_data; +} + +static struct s5p_aes_dev *s5p_dev; + +static void s5p_set_dma_indata(struct s5p_aes_dev *dev, + const struct scatterlist *sg) +{ + SSS_WRITE(dev, FCBRDMAS, sg_dma_address(sg)); + SSS_WRITE(dev, FCBRDMAL, sg_dma_len(sg)); +} + +static void s5p_set_dma_outdata(struct s5p_aes_dev *dev, + const struct scatterlist *sg) +{ + SSS_WRITE(dev, FCBTDMAS, sg_dma_address(sg)); + SSS_WRITE(dev, FCBTDMAL, sg_dma_len(sg)); +} + +static void s5p_free_sg_cpy(struct s5p_aes_dev *dev, struct scatterlist **sg) +{ + int len; + + if (!*sg) + return; + + len = ALIGN(dev->req->cryptlen, AES_BLOCK_SIZE); + free_pages((unsigned long)sg_virt(*sg), get_order(len)); + + kfree(*sg); + *sg = NULL; +} + +static void s5p_sg_done(struct s5p_aes_dev *dev) +{ + struct skcipher_request *req = dev->req; + struct s5p_aes_reqctx *reqctx = skcipher_request_ctx(req); + + if (dev->sg_dst_cpy) { + dev_dbg(dev->dev, + "Copying %d bytes of output data back to original place\n", + dev->req->cryptlen); + memcpy_to_sglist(dev->req->dst, 0, sg_virt(dev->sg_dst_cpy), + dev->req->cryptlen); + } + s5p_free_sg_cpy(dev, &dev->sg_src_cpy); + s5p_free_sg_cpy(dev, &dev->sg_dst_cpy); + if (reqctx->mode & FLAGS_AES_CBC) + memcpy_fromio(req->iv, dev->aes_ioaddr + SSS_REG_AES_IV_DATA(0), AES_BLOCK_SIZE); + + else if (reqctx->mode & FLAGS_AES_CTR) + memcpy_fromio(req->iv, dev->aes_ioaddr + SSS_REG_AES_CNT_DATA(0), AES_BLOCK_SIZE); +} + +/* Calls the completion. Cannot be called with dev->lock hold. */ +static void s5p_aes_complete(struct skcipher_request *req, int err) +{ + skcipher_request_complete(req, err); +} + +static void s5p_unset_outdata(struct s5p_aes_dev *dev) +{ + dma_unmap_sg(dev->dev, dev->sg_dst, 1, DMA_FROM_DEVICE); +} + +static void s5p_unset_indata(struct s5p_aes_dev *dev) +{ + dma_unmap_sg(dev->dev, dev->sg_src, 1, DMA_TO_DEVICE); +} + +static int s5p_make_sg_cpy(struct s5p_aes_dev *dev, struct scatterlist *src, + struct scatterlist **dst) +{ + void *pages; + int len; + + *dst = kmalloc_obj(**dst, GFP_ATOMIC); + if (!*dst) + return -ENOMEM; + + len = ALIGN(dev->req->cryptlen, AES_BLOCK_SIZE); + pages = (void *)__get_free_pages(GFP_ATOMIC, get_order(len)); + if (!pages) { + kfree(*dst); + *dst = NULL; + return -ENOMEM; + } + + memcpy_from_sglist(pages, src, 0, dev->req->cryptlen); + + sg_init_table(*dst, 1); + sg_set_buf(*dst, pages, len); + + return 0; +} + +static int s5p_set_outdata(struct s5p_aes_dev *dev, struct scatterlist *sg) +{ + if (!sg->length) + return -EINVAL; + + if (!dma_map_sg(dev->dev, sg, 1, DMA_FROM_DEVICE)) + return -ENOMEM; + + dev->sg_dst = sg; + + return 0; +} + +static int s5p_set_indata(struct s5p_aes_dev *dev, struct scatterlist *sg) +{ + if (!sg->length) + return -EINVAL; + + if (!dma_map_sg(dev->dev, sg, 1, DMA_TO_DEVICE)) + return -ENOMEM; + + dev->sg_src = sg; + + return 0; +} + +/* + * Returns -ERRNO on error (mapping of new data failed). + * On success returns: + * - 0 if there is no more data, + * - 1 if new transmitting (output) data is ready and its address+length + * have to be written to device (by calling s5p_set_dma_outdata()). + */ +static int s5p_aes_tx(struct s5p_aes_dev *dev) +{ + int ret = 0; + + s5p_unset_outdata(dev); + + if (!sg_is_last(dev->sg_dst)) { + ret = s5p_set_outdata(dev, sg_next(dev->sg_dst)); + if (!ret) + ret = 1; + } + + return ret; +} + +/* + * Returns -ERRNO on error (mapping of new data failed). + * On success returns: + * - 0 if there is no more data, + * - 1 if new receiving (input) data is ready and its address+length + * have to be written to device (by calling s5p_set_dma_indata()). + */ +static int s5p_aes_rx(struct s5p_aes_dev *dev/*, bool *set_dma*/) +{ + int ret = 0; + + s5p_unset_indata(dev); + + if (!sg_is_last(dev->sg_src)) { + ret = s5p_set_indata(dev, sg_next(dev->sg_src)); + if (!ret) + ret = 1; + } + + return ret; +} + +static inline u32 s5p_hash_read(struct s5p_aes_dev *dd, u32 offset) +{ + return __raw_readl(dd->io_hash_base + offset); +} + +static inline void s5p_hash_write(struct s5p_aes_dev *dd, + u32 offset, u32 value) +{ + __raw_writel(value, dd->io_hash_base + offset); +} + +/** + * s5p_set_dma_hashdata() - start DMA with sg + * @dev: device + * @sg: scatterlist ready to DMA transmit + */ +static void s5p_set_dma_hashdata(struct s5p_aes_dev *dev, + const struct scatterlist *sg) +{ + dev->hash_sg_cnt--; + SSS_WRITE(dev, FCHRDMAS, sg_dma_address(sg)); + SSS_WRITE(dev, FCHRDMAL, sg_dma_len(sg)); /* DMA starts */ +} + +/** + * s5p_hash_rx() - get next hash_sg_iter + * @dev: device + * + * Return: + * 2 if there is no more data and it is UPDATE op + * 1 if new receiving (input) data is ready and can be written to device + * 0 if there is no more data and it is FINAL op + */ +static int s5p_hash_rx(struct s5p_aes_dev *dev) +{ + if (dev->hash_sg_cnt > 0) { + dev->hash_sg_iter = sg_next(dev->hash_sg_iter); + return 1; + } + + set_bit(HASH_FLAGS_DMA_READY, &dev->hash_flags); + if (test_bit(HASH_FLAGS_FINAL, &dev->hash_flags)) + return 0; + + return 2; +} + +static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id) +{ + struct platform_device *pdev = dev_id; + struct s5p_aes_dev *dev = platform_get_drvdata(pdev); + struct skcipher_request *req; + int err_dma_tx = 0; + int err_dma_rx = 0; + int err_dma_hx = 0; + bool tx_end = false; + bool hx_end = false; + unsigned long flags; + u32 status, st_bits; + int err; + + spin_lock_irqsave(&dev->lock, flags); + + /* + * Handle rx or tx interrupt. If there is still data (scatterlist did not + * reach end), then map next scatterlist entry. + * In case of such mapping error, s5p_aes_complete() should be called. + * + * If there is no more data in tx scatter list, call s5p_aes_complete() + * and schedule new tasklet. + * + * Handle hx interrupt. If there is still data map next entry. + */ + status = SSS_READ(dev, FCINTSTAT); + if (status & SSS_FCINTSTAT_BRDMAINT) + err_dma_rx = s5p_aes_rx(dev); + + if (status & SSS_FCINTSTAT_BTDMAINT) { + if (sg_is_last(dev->sg_dst)) + tx_end = true; + err_dma_tx = s5p_aes_tx(dev); + } + + if (status & SSS_FCINTSTAT_HRDMAINT) + err_dma_hx = s5p_hash_rx(dev); + + st_bits = status & (SSS_FCINTSTAT_BRDMAINT | SSS_FCINTSTAT_BTDMAINT | + SSS_FCINTSTAT_HRDMAINT); + /* clear DMA bits */ + SSS_WRITE(dev, FCINTPEND, st_bits); + + /* clear HASH irq bits */ + if (status & (SSS_FCINTSTAT_HDONEINT | SSS_FCINTSTAT_HPARTINT)) { + /* cannot have both HPART and HDONE */ + if (status & SSS_FCINTSTAT_HPARTINT) + st_bits = SSS_HASH_STATUS_PARTIAL_DONE; + + if (status & SSS_FCINTSTAT_HDONEINT) + st_bits = SSS_HASH_STATUS_MSG_DONE; + + set_bit(HASH_FLAGS_OUTPUT_READY, &dev->hash_flags); + s5p_hash_write(dev, SSS_REG_HASH_STATUS, st_bits); + hx_end = true; + /* when DONE or PART, do not handle HASH DMA */ + err_dma_hx = 0; + } + + if (err_dma_rx < 0) { + err = err_dma_rx; + goto error; + } + if (err_dma_tx < 0) { + err = err_dma_tx; + goto error; + } + + if (tx_end) { + s5p_sg_done(dev); + if (err_dma_hx == 1) + s5p_set_dma_hashdata(dev, dev->hash_sg_iter); + + spin_unlock_irqrestore(&dev->lock, flags); + + s5p_aes_complete(dev->req, 0); + /* Device is still busy */ + tasklet_schedule(&dev->tasklet); + } else { + /* + * Writing length of DMA block (either receiving or + * transmitting) will start the operation immediately, so this + * should be done at the end (even after clearing pending + * interrupts to not miss the interrupt). + */ + if (err_dma_tx == 1) + s5p_set_dma_outdata(dev, dev->sg_dst); + if (err_dma_rx == 1) + s5p_set_dma_indata(dev, dev->sg_src); + if (err_dma_hx == 1) + s5p_set_dma_hashdata(dev, dev->hash_sg_iter); + + spin_unlock_irqrestore(&dev->lock, flags); + } + + goto hash_irq_end; + +error: + s5p_sg_done(dev); + dev->busy = false; + req = dev->req; + if (err_dma_hx == 1) + s5p_set_dma_hashdata(dev, dev->hash_sg_iter); + + spin_unlock_irqrestore(&dev->lock, flags); + s5p_aes_complete(req, err); + +hash_irq_end: + /* + * Note about else if: + * when hash_sg_iter reaches end and its UPDATE op, + * issue SSS_HASH_PAUSE and wait for HPART irq + */ + if (hx_end) + tasklet_schedule(&dev->hash_tasklet); + else if (err_dma_hx == 2) + s5p_hash_write(dev, SSS_REG_HASH_CTRL_PAUSE, + SSS_HASH_PAUSE); + + return IRQ_HANDLED; +} + +/** + * s5p_hash_read_msg() - read message or IV from HW + * @req: AHASH request + */ +static void s5p_hash_read_msg(struct ahash_request *req) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + struct s5p_aes_dev *dd = ctx->dd; + u32 *hash = (u32 *)ctx->digest; + unsigned int i; + + for (i = 0; i < ctx->nregs; i++) + hash[i] = s5p_hash_read(dd, SSS_REG_HASH_OUT(i)); +} + +/** + * s5p_hash_write_ctx_iv() - write IV for next partial/finup op. + * @dd: device + * @ctx: request context + */ +static void s5p_hash_write_ctx_iv(struct s5p_aes_dev *dd, + const struct s5p_hash_reqctx *ctx) +{ + const u32 *hash = (const u32 *)ctx->digest; + unsigned int i; + + for (i = 0; i < ctx->nregs; i++) + s5p_hash_write(dd, SSS_REG_HASH_IV(i), hash[i]); +} + +/** + * s5p_hash_write_iv() - write IV for next partial/finup op. + * @req: AHASH request + */ +static void s5p_hash_write_iv(struct ahash_request *req) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + + s5p_hash_write_ctx_iv(ctx->dd, ctx); +} + +/** + * s5p_hash_copy_result() - copy digest into req->result + * @req: AHASH request + */ +static void s5p_hash_copy_result(struct ahash_request *req) +{ + const struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + + if (!req->result) + return; + + memcpy(req->result, ctx->digest, ctx->nregs * HASH_REG_SIZEOF); +} + +/** + * s5p_hash_dma_flush() - flush HASH DMA + * @dev: secss device + */ +static void s5p_hash_dma_flush(struct s5p_aes_dev *dev) +{ + SSS_WRITE(dev, FCHRDMAC, SSS_FCHRDMAC_FLUSH); +} + +/** + * s5p_hash_dma_enable() - enable DMA mode for HASH + * @dev: secss device + * + * enable DMA mode for HASH + */ +static void s5p_hash_dma_enable(struct s5p_aes_dev *dev) +{ + s5p_hash_write(dev, SSS_REG_HASH_CTRL_FIFO, SSS_HASH_FIFO_MODE_DMA); +} + +/** + * s5p_hash_irq_disable() - disable irq HASH signals + * @dev: secss device + * @flags: bitfield with irq's to be disabled + */ +static void s5p_hash_irq_disable(struct s5p_aes_dev *dev, u32 flags) +{ + SSS_WRITE(dev, FCINTENCLR, flags); +} + +/** + * s5p_hash_irq_enable() - enable irq signals + * @dev: secss device + * @flags: bitfield with irq's to be enabled + */ +static void s5p_hash_irq_enable(struct s5p_aes_dev *dev, int flags) +{ + SSS_WRITE(dev, FCINTENSET, flags); +} + +/** + * s5p_hash_set_flow() - set flow inside SecSS AES/DES with/without HASH + * @dev: secss device + * @hashflow: HASH stream flow with/without crypto AES/DES + */ +static void s5p_hash_set_flow(struct s5p_aes_dev *dev, u32 hashflow) +{ + unsigned long flags; + u32 flow; + + spin_lock_irqsave(&dev->lock, flags); + + flow = SSS_READ(dev, FCFIFOCTRL); + flow &= ~SSS_HASHIN_MASK; + flow |= hashflow; + SSS_WRITE(dev, FCFIFOCTRL, flow); + + spin_unlock_irqrestore(&dev->lock, flags); +} + +/** + * s5p_ahash_dma_init() - enable DMA and set HASH flow inside SecSS + * @dev: secss device + * @hashflow: HASH stream flow with/without AES/DES + * + * flush HASH DMA and enable DMA, set HASH stream flow inside SecSS HW, + * enable HASH irq's HRDMA, HDONE, HPART + */ +static void s5p_ahash_dma_init(struct s5p_aes_dev *dev, u32 hashflow) +{ + s5p_hash_irq_disable(dev, SSS_FCINTENCLR_HRDMAINTENCLR | + SSS_FCINTENCLR_HDONEINTENCLR | + SSS_FCINTENCLR_HPARTINTENCLR); + s5p_hash_dma_flush(dev); + + s5p_hash_dma_enable(dev); + s5p_hash_set_flow(dev, hashflow & SSS_HASHIN_MASK); + s5p_hash_irq_enable(dev, SSS_FCINTENSET_HRDMAINTENSET | + SSS_FCINTENSET_HDONEINTENSET | + SSS_FCINTENSET_HPARTINTENSET); +} + +/** + * s5p_hash_write_ctrl() - prepare HASH block in SecSS for processing + * @dd: secss device + * @length: length for request + * @final: true if final op + * + * Prepare SSS HASH block for processing bytes in DMA mode. If it is called + * after previous updates, fill up IV words. For final, calculate and set + * lengths for HASH so SecSS can finalize hash. For partial, set SSS HASH + * length as 2^63 so it will be never reached and set to zero prelow and + * prehigh. + * + * This function does not start DMA transfer. + */ +static void s5p_hash_write_ctrl(struct s5p_aes_dev *dd, size_t length, + bool final) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(dd->hash_req); + u32 prelow, prehigh, low, high; + u32 configflags, swapflags; + u64 tmplen; + + configflags = ctx->engine | SSS_HASH_INIT_BIT; + + if (likely(ctx->digcnt)) { + s5p_hash_write_ctx_iv(dd, ctx); + configflags |= SSS_HASH_USER_IV_EN; + } + + if (final) { + /* number of bytes for last part */ + low = length; + high = 0; + /* total number of bits prev hashed */ + tmplen = ctx->digcnt * 8; + prelow = (u32)tmplen; + prehigh = (u32)(tmplen >> 32); + } else { + prelow = 0; + prehigh = 0; + low = 0; + high = BIT(31); + } + + swapflags = SSS_HASH_BYTESWAP_DI | SSS_HASH_BYTESWAP_DO | + SSS_HASH_BYTESWAP_IV | SSS_HASH_BYTESWAP_KEY; + + s5p_hash_write(dd, SSS_REG_HASH_MSG_SIZE_LOW, low); + s5p_hash_write(dd, SSS_REG_HASH_MSG_SIZE_HIGH, high); + s5p_hash_write(dd, SSS_REG_HASH_PRE_MSG_SIZE_LOW, prelow); + s5p_hash_write(dd, SSS_REG_HASH_PRE_MSG_SIZE_HIGH, prehigh); + + s5p_hash_write(dd, SSS_REG_HASH_CTRL_SWAP, swapflags); + s5p_hash_write(dd, SSS_REG_HASH_CTRL, configflags); +} + +/** + * s5p_hash_xmit_dma() - start DMA hash processing + * @dd: secss device + * @length: length for request + * @final: true if final op + * + * Update digcnt here, as it is needed for finup/final op. + */ +static int s5p_hash_xmit_dma(struct s5p_aes_dev *dd, size_t length, + bool final) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(dd->hash_req); + unsigned int cnt; + + cnt = dma_map_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE); + if (!cnt) { + dev_err(dd->dev, "dma_map_sg error\n"); + ctx->error = true; + return -EINVAL; + } + + set_bit(HASH_FLAGS_DMA_ACTIVE, &dd->hash_flags); + dd->hash_sg_iter = ctx->sg; + dd->hash_sg_cnt = cnt; + s5p_hash_write_ctrl(dd, length, final); + ctx->digcnt += length; + ctx->total -= length; + + /* catch last interrupt */ + if (final) + set_bit(HASH_FLAGS_FINAL, &dd->hash_flags); + + s5p_set_dma_hashdata(dd, dd->hash_sg_iter); /* DMA starts */ + + return -EINPROGRESS; +} + +/** + * s5p_hash_copy_sgs() - copy request's bytes into new buffer + * @ctx: request context + * @sg: source scatterlist request + * @new_len: number of bytes to process from sg + * + * Allocate new buffer, copy data for HASH into it. If there was xmit_buf + * filled, copy it first, then copy data from sg into it. Prepare one sgl[0] + * with allocated buffer. + * + * Set bit in dd->hash_flag so we can free it after irq ends processing. + */ +static int s5p_hash_copy_sgs(struct s5p_hash_reqctx *ctx, + struct scatterlist *sg, unsigned int new_len) +{ + unsigned int pages, len; + void *buf; + + len = new_len + ctx->bufcnt; + pages = get_order(len); + + buf = (void *)__get_free_pages(GFP_ATOMIC, pages); + if (!buf) { + dev_err(ctx->dd->dev, "alloc pages for unaligned case.\n"); + ctx->error = true; + return -ENOMEM; + } + + if (ctx->bufcnt) + memcpy(buf, ctx->dd->xmit_buf, ctx->bufcnt); + + memcpy_from_sglist(buf + ctx->bufcnt, sg, ctx->skip, new_len); + sg_init_table(ctx->sgl, 1); + sg_set_buf(ctx->sgl, buf, len); + ctx->sg = ctx->sgl; + ctx->sg_len = 1; + ctx->bufcnt = 0; + ctx->skip = 0; + set_bit(HASH_FLAGS_SGS_COPIED, &ctx->dd->hash_flags); + + return 0; +} + +/** + * s5p_hash_copy_sg_lists() - copy sg list and make fixes in copy + * @ctx: request context + * @sg: source scatterlist request + * @new_len: number of bytes to process from sg + * + * Allocate new scatterlist table, copy data for HASH into it. If there was + * xmit_buf filled, prepare it first, then copy page, length and offset from + * source sg into it, adjusting begin and/or end for skip offset and + * hash_later value. + * + * Resulting sg table will be assigned to ctx->sg. Set flag so we can free + * it after irq ends processing. + */ +static int s5p_hash_copy_sg_lists(struct s5p_hash_reqctx *ctx, + struct scatterlist *sg, unsigned int new_len) +{ + unsigned int skip = ctx->skip, n = sg_nents(sg); + struct scatterlist *tmp; + unsigned int len; + + if (ctx->bufcnt) + n++; + + ctx->sg = kmalloc_objs(*sg, n); + if (!ctx->sg) { + ctx->error = true; + return -ENOMEM; + } + + sg_init_table(ctx->sg, n); + + tmp = ctx->sg; + + ctx->sg_len = 0; + + if (ctx->bufcnt) { + sg_set_buf(tmp, ctx->dd->xmit_buf, ctx->bufcnt); + tmp = sg_next(tmp); + ctx->sg_len++; + } + + while (sg && skip >= sg->length) { + skip -= sg->length; + sg = sg_next(sg); + } + + while (sg && new_len) { + len = sg->length - skip; + if (new_len < len) + len = new_len; + + new_len -= len; + sg_set_page(tmp, sg_page(sg), len, sg->offset + skip); + skip = 0; + if (new_len <= 0) + sg_mark_end(tmp); + + tmp = sg_next(tmp); + ctx->sg_len++; + sg = sg_next(sg); + } + + set_bit(HASH_FLAGS_SGS_ALLOCED, &ctx->dd->hash_flags); + + return 0; +} + +/** + * s5p_hash_prepare_sgs() - prepare sg for processing + * @ctx: request context + * @sg: source scatterlist request + * @new_len: number of bytes to process from sg + * @final: final flag + * + * Check two conditions: (1) if buffers in sg have len aligned data, and (2) + * sg table have good aligned elements (list_ok). If one of this checks fails, + * then either (1) allocates new buffer for data with s5p_hash_copy_sgs, copy + * data into this buffer and prepare request in sgl, or (2) allocates new sg + * table and prepare sg elements. + * + * For digest or finup all conditions can be good, and we may not need any + * fixes. + */ +static int s5p_hash_prepare_sgs(struct s5p_hash_reqctx *ctx, + struct scatterlist *sg, + unsigned int new_len, bool final) +{ + unsigned int skip = ctx->skip, nbytes = new_len, n = 0; + bool aligned = true, list_ok = true; + struct scatterlist *sg_tmp = sg; + + if (!sg || !sg->length || !new_len) + return 0; + + if (skip || !final) + list_ok = false; + + while (nbytes > 0 && sg_tmp) { + n++; + if (skip >= sg_tmp->length) { + skip -= sg_tmp->length; + if (!sg_tmp->length) { + aligned = false; + break; + } + } else { + if (!IS_ALIGNED(sg_tmp->length - skip, BUFLEN)) { + aligned = false; + break; + } + + if (nbytes < sg_tmp->length - skip) { + list_ok = false; + break; + } + + nbytes -= sg_tmp->length - skip; + skip = 0; + } + + sg_tmp = sg_next(sg_tmp); + } + + if (!aligned) + return s5p_hash_copy_sgs(ctx, sg, new_len); + else if (!list_ok) + return s5p_hash_copy_sg_lists(ctx, sg, new_len); + + /* + * Have aligned data from previous operation and/or current + * Note: will enter here only if (digest or finup) and aligned + */ + if (ctx->bufcnt) { + ctx->sg_len = n; + sg_init_table(ctx->sgl, 2); + sg_set_buf(ctx->sgl, ctx->dd->xmit_buf, ctx->bufcnt); + sg_chain(ctx->sgl, 2, sg); + ctx->sg = ctx->sgl; + ctx->sg_len++; + } else { + ctx->sg = sg; + ctx->sg_len = n; + } + + return 0; +} + +/** + * s5p_hash_prepare_request() - prepare request for processing + * @req: AHASH request + * @update: true if UPDATE op + * + * Note 1: we can have update flag _and_ final flag at the same time. + * Note 2: we enter here when digcnt > BUFLEN (=HASH_BLOCK_SIZE) or + * either req->nbytes or ctx->bufcnt + req->nbytes is > BUFLEN or + * we have final op + */ +static int s5p_hash_prepare_request(struct ahash_request *req, bool update) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + bool final = ctx->finup; + int xmit_len, hash_later, nbytes; + int ret; + + if (update) + nbytes = req->nbytes; + else + nbytes = 0; + + ctx->total = nbytes + ctx->bufcnt; + if (!ctx->total) + return 0; + + if (nbytes && (!IS_ALIGNED(ctx->bufcnt, BUFLEN))) { + /* bytes left from previous request, so fill up to BUFLEN */ + int len = BUFLEN - ctx->bufcnt % BUFLEN; + + if (len > nbytes) + len = nbytes; + + memcpy_from_sglist(ctx->buffer + ctx->bufcnt, req->src, 0, len); + ctx->bufcnt += len; + nbytes -= len; + ctx->skip = len; + } else { + ctx->skip = 0; + } + + if (ctx->bufcnt) + memcpy(ctx->dd->xmit_buf, ctx->buffer, ctx->bufcnt); + + xmit_len = ctx->total; + if (final) { + hash_later = 0; + } else { + if (IS_ALIGNED(xmit_len, BUFLEN)) + xmit_len -= BUFLEN; + else + xmit_len -= xmit_len & (BUFLEN - 1); + + hash_later = ctx->total - xmit_len; + /* copy hash_later bytes from end of req->src */ + /* previous bytes are in xmit_buf, so no overwrite */ + memcpy_from_sglist(ctx->buffer, req->src, + req->nbytes - hash_later, hash_later); + } + + if (xmit_len > BUFLEN) { + ret = s5p_hash_prepare_sgs(ctx, req->src, nbytes - hash_later, + final); + if (ret) + return ret; + } else { + /* have buffered data only */ + if (unlikely(!ctx->bufcnt)) { + /* first update didn't fill up buffer */ + memcpy_from_sglist(ctx->dd->xmit_buf, req->src, + 0, xmit_len); + } + + sg_init_table(ctx->sgl, 1); + sg_set_buf(ctx->sgl, ctx->dd->xmit_buf, xmit_len); + + ctx->sg = ctx->sgl; + ctx->sg_len = 1; + } + + ctx->bufcnt = hash_later; + if (!final) + ctx->total = xmit_len; + + return 0; +} + +/** + * s5p_hash_update_dma_stop() - unmap DMA + * @dd: secss device + * + * Unmap scatterlist ctx->sg. + */ +static void s5p_hash_update_dma_stop(struct s5p_aes_dev *dd) +{ + const struct s5p_hash_reqctx *ctx = ahash_request_ctx(dd->hash_req); + + dma_unmap_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE); + clear_bit(HASH_FLAGS_DMA_ACTIVE, &dd->hash_flags); +} + +/** + * s5p_hash_finish() - copy calculated digest to crypto layer + * @req: AHASH request + */ +static void s5p_hash_finish(struct ahash_request *req) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + struct s5p_aes_dev *dd = ctx->dd; + + if (ctx->digcnt) + s5p_hash_copy_result(req); + + dev_dbg(dd->dev, "hash_finish digcnt: %lld\n", ctx->digcnt); +} + +/** + * s5p_hash_finish_req() - finish request + * @req: AHASH request + * @err: error + */ +static void s5p_hash_finish_req(struct ahash_request *req, int err) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + struct s5p_aes_dev *dd = ctx->dd; + unsigned long flags; + + if (test_bit(HASH_FLAGS_SGS_COPIED, &dd->hash_flags)) + free_pages((unsigned long)sg_virt(ctx->sg), + get_order(ctx->sg->length)); + + if (test_bit(HASH_FLAGS_SGS_ALLOCED, &dd->hash_flags)) + kfree(ctx->sg); + + ctx->sg = NULL; + dd->hash_flags &= ~(BIT(HASH_FLAGS_SGS_ALLOCED) | + BIT(HASH_FLAGS_SGS_COPIED)); + + if (!err && !ctx->error) { + s5p_hash_read_msg(req); + if (test_bit(HASH_FLAGS_FINAL, &dd->hash_flags)) + s5p_hash_finish(req); + } else { + ctx->error = true; + } + + spin_lock_irqsave(&dd->hash_lock, flags); + dd->hash_flags &= ~(BIT(HASH_FLAGS_BUSY) | BIT(HASH_FLAGS_FINAL) | + BIT(HASH_FLAGS_DMA_READY) | + BIT(HASH_FLAGS_OUTPUT_READY)); + spin_unlock_irqrestore(&dd->hash_lock, flags); + + if (req->base.complete) + ahash_request_complete(req, err); +} + +/** + * s5p_hash_handle_queue() - handle hash queue + * @dd: device s5p_aes_dev + * @req: AHASH request + * + * If req!=NULL enqueue it on dd->queue, if FLAGS_BUSY is not set on the + * device then processes the first request from the dd->queue + * + * Returns: see s5p_hash_final below. + */ +static int s5p_hash_handle_queue(struct s5p_aes_dev *dd, + struct ahash_request *req) +{ + struct crypto_async_request *async_req, *backlog; + struct s5p_hash_reqctx *ctx; + unsigned long flags; + int err = 0, ret = 0; + +retry: + spin_lock_irqsave(&dd->hash_lock, flags); + if (req) + ret = ahash_enqueue_request(&dd->hash_queue, req); + + if (test_bit(HASH_FLAGS_BUSY, &dd->hash_flags)) { + spin_unlock_irqrestore(&dd->hash_lock, flags); + return ret; + } + + backlog = crypto_get_backlog(&dd->hash_queue); + async_req = crypto_dequeue_request(&dd->hash_queue); + if (async_req) + set_bit(HASH_FLAGS_BUSY, &dd->hash_flags); + + spin_unlock_irqrestore(&dd->hash_lock, flags); + + if (!async_req) + return ret; + + if (backlog) + crypto_request_complete(backlog, -EINPROGRESS); + + req = ahash_request_cast(async_req); + dd->hash_req = req; + ctx = ahash_request_ctx(req); + + err = s5p_hash_prepare_request(req, ctx->op_update); + if (err || !ctx->total) + goto out; + + dev_dbg(dd->dev, "handling new req, op_update: %u, nbytes: %d\n", + ctx->op_update, req->nbytes); + + s5p_ahash_dma_init(dd, SSS_HASHIN_INDEPENDENT); + if (ctx->digcnt) + s5p_hash_write_iv(req); /* restore hash IV */ + + if (ctx->op_update) { /* HASH_OP_UPDATE */ + err = s5p_hash_xmit_dma(dd, ctx->total, ctx->finup); + if (err != -EINPROGRESS && ctx->finup && !ctx->error) + /* no final() after finup() */ + err = s5p_hash_xmit_dma(dd, ctx->total, true); + } else { /* HASH_OP_FINAL */ + err = s5p_hash_xmit_dma(dd, ctx->total, true); + } +out: + if (err != -EINPROGRESS) { + /* hash_tasklet_cb will not finish it, so do it here */ + s5p_hash_finish_req(req, err); + req = NULL; + + /* + * Execute next request immediately if there is anything + * in queue. + */ + goto retry; + } + + return ret; +} + +/** + * s5p_hash_tasklet_cb() - hash tasklet + * @data: ptr to s5p_aes_dev + */ +static void s5p_hash_tasklet_cb(unsigned long data) +{ + struct s5p_aes_dev *dd = (struct s5p_aes_dev *)data; + + if (!test_bit(HASH_FLAGS_BUSY, &dd->hash_flags)) { + s5p_hash_handle_queue(dd, NULL); + return; + } + + if (test_bit(HASH_FLAGS_DMA_READY, &dd->hash_flags)) { + if (test_and_clear_bit(HASH_FLAGS_DMA_ACTIVE, + &dd->hash_flags)) { + s5p_hash_update_dma_stop(dd); + } + + if (test_and_clear_bit(HASH_FLAGS_OUTPUT_READY, + &dd->hash_flags)) { + /* hash or semi-hash ready */ + clear_bit(HASH_FLAGS_DMA_READY, &dd->hash_flags); + goto finish; + } + } + + return; + +finish: + /* finish curent request */ + s5p_hash_finish_req(dd->hash_req, 0); + + /* If we are not busy, process next req */ + if (!test_bit(HASH_FLAGS_BUSY, &dd->hash_flags)) + s5p_hash_handle_queue(dd, NULL); +} + +/** + * s5p_hash_enqueue() - enqueue request + * @req: AHASH request + * @op: operation UPDATE (true) or FINAL (false) + * + * Returns: see s5p_hash_final below. + */ +static int s5p_hash_enqueue(struct ahash_request *req, bool op) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + struct s5p_hash_ctx *tctx = crypto_tfm_ctx(req->base.tfm); + + ctx->op_update = op; + + return s5p_hash_handle_queue(tctx->dd, req); +} + +/** + * s5p_hash_update() - process the hash input data + * @req: AHASH request + * + * If request will fit in buffer, copy it and return immediately + * else enqueue it with OP_UPDATE. + * + * Returns: see s5p_hash_final below. + */ +static int s5p_hash_update(struct ahash_request *req) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + + if (!req->nbytes) + return 0; + + if (ctx->bufcnt + req->nbytes <= BUFLEN) { + memcpy_from_sglist(ctx->buffer + ctx->bufcnt, req->src, + 0, req->nbytes); + ctx->bufcnt += req->nbytes; + return 0; + } + + return s5p_hash_enqueue(req, true); /* HASH_OP_UPDATE */ +} + +/** + * s5p_hash_final() - close up hash and calculate digest + * @req: AHASH request + * + * Note: in final req->src do not have any data, and req->nbytes can be + * non-zero. + * + * If there were no input data processed yet and the buffered hash data is + * less than BUFLEN (64) then calculate the final hash immediately by using + * SW algorithm fallback. + * + * Otherwise enqueues the current AHASH request with OP_FINAL operation op + * and finalize hash message in HW. Note that if digcnt!=0 then there were + * previous update op, so there are always some buffered bytes in ctx->buffer, + * which means that ctx->bufcnt!=0 + * + * Returns: + * 0 if the request has been processed immediately, + * -EINPROGRESS if the operation has been queued for later execution or is set + * to processing by HW, + * -EBUSY if queue is full and request should be resubmitted later, + * other negative values denotes an error. + */ +static int s5p_hash_final(struct ahash_request *req) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + + ctx->finup = true; + if (ctx->error) + return -EINVAL; /* uncompleted hash is not needed */ + + if (!ctx->digcnt && ctx->bufcnt < BUFLEN) { + struct s5p_hash_ctx *tctx = crypto_tfm_ctx(req->base.tfm); + + return crypto_shash_tfm_digest(tctx->fallback, ctx->buffer, + ctx->bufcnt, req->result); + } + + return s5p_hash_enqueue(req, false); /* HASH_OP_FINAL */ +} + +/** + * s5p_hash_finup() - process last req->src and calculate digest + * @req: AHASH request containing the last update data + * + * Return values: see s5p_hash_final above. + */ +static int s5p_hash_finup(struct ahash_request *req) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + int err1, err2; + + ctx->finup = true; + + err1 = s5p_hash_update(req); + if (err1 == -EINPROGRESS || err1 == -EBUSY) + return err1; + + /* + * final() has to be always called to cleanup resources even if + * update() failed, except EINPROGRESS or calculate digest for small + * size + */ + err2 = s5p_hash_final(req); + + return err1 ?: err2; +} + +/** + * s5p_hash_init() - initialize AHASH request contex + * @req: AHASH request + * + * Init async hash request context. + */ +static int s5p_hash_init(struct ahash_request *req) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct s5p_hash_ctx *tctx = crypto_ahash_ctx(tfm); + + ctx->dd = tctx->dd; + ctx->error = false; + ctx->finup = false; + ctx->bufcnt = 0; + ctx->digcnt = 0; + ctx->total = 0; + ctx->skip = 0; + + dev_dbg(tctx->dd->dev, "init: digest size: %d\n", + crypto_ahash_digestsize(tfm)); + + switch (crypto_ahash_digestsize(tfm)) { + case MD5_DIGEST_SIZE: + ctx->engine = SSS_HASH_ENGINE_MD5; + ctx->nregs = HASH_MD5_MAX_REG; + break; + case SHA1_DIGEST_SIZE: + ctx->engine = SSS_HASH_ENGINE_SHA1; + ctx->nregs = HASH_SHA1_MAX_REG; + break; + case SHA256_DIGEST_SIZE: + ctx->engine = SSS_HASH_ENGINE_SHA256; + ctx->nregs = HASH_SHA256_MAX_REG; + break; + default: + ctx->error = true; + return -EINVAL; + } + + return 0; +} + +/** + * s5p_hash_digest - calculate digest from req->src + * @req: AHASH request + * + * Return values: see s5p_hash_final above. + */ +static int s5p_hash_digest(struct ahash_request *req) +{ + return s5p_hash_init(req) ?: s5p_hash_finup(req); +} + +/** + * s5p_hash_cra_init_alg - init crypto alg transformation + * @tfm: crypto transformation + */ +static int s5p_hash_cra_init_alg(struct crypto_tfm *tfm) +{ + struct s5p_hash_ctx *tctx = crypto_tfm_ctx(tfm); + const char *alg_name = crypto_tfm_alg_name(tfm); + + tctx->dd = s5p_dev; + /* Allocate a fallback and abort if it failed. */ + tctx->fallback = crypto_alloc_shash(alg_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(tctx->fallback)) { + pr_err("fallback alloc fails for '%s'\n", alg_name); + return PTR_ERR(tctx->fallback); + } + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct s5p_hash_reqctx) + BUFLEN); + + return 0; +} + +/** + * s5p_hash_cra_init - init crypto tfm + * @tfm: crypto transformation + */ +static int s5p_hash_cra_init(struct crypto_tfm *tfm) +{ + return s5p_hash_cra_init_alg(tfm); +} + +/** + * s5p_hash_cra_exit - exit crypto tfm + * @tfm: crypto transformation + * + * free allocated fallback + */ +static void s5p_hash_cra_exit(struct crypto_tfm *tfm) +{ + struct s5p_hash_ctx *tctx = crypto_tfm_ctx(tfm); + + crypto_free_shash(tctx->fallback); + tctx->fallback = NULL; +} + +/** + * s5p_hash_export - export hash state + * @req: AHASH request + * @out: buffer for exported state + */ +static int s5p_hash_export(struct ahash_request *req, void *out) +{ + const struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + + memcpy(out, ctx, sizeof(*ctx) + ctx->bufcnt); + + return 0; +} + +/** + * s5p_hash_import - import hash state + * @req: AHASH request + * @in: buffer with state to be imported from + */ +static int s5p_hash_import(struct ahash_request *req, const void *in) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct s5p_hash_ctx *tctx = crypto_ahash_ctx(tfm); + const struct s5p_hash_reqctx *ctx_in = in; + + memcpy(ctx, in, sizeof(*ctx) + BUFLEN); + if (ctx_in->bufcnt > BUFLEN) { + ctx->error = true; + return -EINVAL; + } + + ctx->dd = tctx->dd; + ctx->error = false; + + return 0; +} + +static struct ahash_alg algs_sha1_md5_sha256[] = { +{ + .init = s5p_hash_init, + .update = s5p_hash_update, + .final = s5p_hash_final, + .finup = s5p_hash_finup, + .digest = s5p_hash_digest, + .export = s5p_hash_export, + .import = s5p_hash_import, + .halg.statesize = sizeof(struct s5p_hash_reqctx) + BUFLEN, + .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.base = { + .cra_name = "sha1", + .cra_driver_name = "exynos-sha1", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = HASH_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct s5p_hash_ctx), + .cra_module = THIS_MODULE, + .cra_init = s5p_hash_cra_init, + .cra_exit = s5p_hash_cra_exit, + } +}, +{ + .init = s5p_hash_init, + .update = s5p_hash_update, + .final = s5p_hash_final, + .finup = s5p_hash_finup, + .digest = s5p_hash_digest, + .export = s5p_hash_export, + .import = s5p_hash_import, + .halg.statesize = sizeof(struct s5p_hash_reqctx) + BUFLEN, + .halg.digestsize = MD5_DIGEST_SIZE, + .halg.base = { + .cra_name = "md5", + .cra_driver_name = "exynos-md5", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = HASH_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct s5p_hash_ctx), + .cra_module = THIS_MODULE, + .cra_init = s5p_hash_cra_init, + .cra_exit = s5p_hash_cra_exit, + } +}, +{ + .init = s5p_hash_init, + .update = s5p_hash_update, + .final = s5p_hash_final, + .finup = s5p_hash_finup, + .digest = s5p_hash_digest, + .export = s5p_hash_export, + .import = s5p_hash_import, + .halg.statesize = sizeof(struct s5p_hash_reqctx) + BUFLEN, + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.base = { + .cra_name = "sha256", + .cra_driver_name = "exynos-sha256", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = HASH_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct s5p_hash_ctx), + .cra_module = THIS_MODULE, + .cra_init = s5p_hash_cra_init, + .cra_exit = s5p_hash_cra_exit, + } +} + +}; + +static void s5p_set_aes(struct s5p_aes_dev *dev, + const u8 *key, const u8 *iv, const u8 *ctr, + unsigned int keylen) +{ + void __iomem *keystart; + + if (iv) + memcpy_toio(dev->aes_ioaddr + SSS_REG_AES_IV_DATA(0), iv, + AES_BLOCK_SIZE); + + if (ctr) + memcpy_toio(dev->aes_ioaddr + SSS_REG_AES_CNT_DATA(0), ctr, + AES_BLOCK_SIZE); + + if (keylen == AES_KEYSIZE_256) + keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(0); + else if (keylen == AES_KEYSIZE_192) + keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(2); + else + keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(4); + + memcpy_toio(keystart, key, keylen); +} + +static bool s5p_is_sg_aligned(struct scatterlist *sg) +{ + while (sg) { + if (!IS_ALIGNED(sg->length, AES_BLOCK_SIZE)) + return false; + sg = sg_next(sg); + } + + return true; +} + +static int s5p_set_indata_start(struct s5p_aes_dev *dev, + struct skcipher_request *req) +{ + struct scatterlist *sg; + int err; + + dev->sg_src_cpy = NULL; + sg = req->src; + if (!s5p_is_sg_aligned(sg)) { + dev_dbg(dev->dev, + "At least one unaligned source scatter list, making a copy\n"); + err = s5p_make_sg_cpy(dev, sg, &dev->sg_src_cpy); + if (err) + return err; + + sg = dev->sg_src_cpy; + } + + err = s5p_set_indata(dev, sg); + if (err) { + s5p_free_sg_cpy(dev, &dev->sg_src_cpy); + return err; + } + + return 0; +} + +static int s5p_set_outdata_start(struct s5p_aes_dev *dev, + struct skcipher_request *req) +{ + struct scatterlist *sg; + int err; + + dev->sg_dst_cpy = NULL; + sg = req->dst; + if (!s5p_is_sg_aligned(sg)) { + dev_dbg(dev->dev, + "At least one unaligned dest scatter list, making a copy\n"); + err = s5p_make_sg_cpy(dev, sg, &dev->sg_dst_cpy); + if (err) + return err; + + sg = dev->sg_dst_cpy; + } + + err = s5p_set_outdata(dev, sg); + if (err) { + s5p_free_sg_cpy(dev, &dev->sg_dst_cpy); + return err; + } + + return 0; +} + +static void s5p_aes_crypt_start(struct s5p_aes_dev *dev, unsigned long mode) +{ + struct skcipher_request *req = dev->req; + u32 aes_control; + unsigned long flags; + int err; + u8 *iv, *ctr; + + /* This sets bit [13:12] to 00, which selects 128-bit counter */ + aes_control = SSS_AES_KEY_CHANGE_MODE; + if (mode & FLAGS_AES_DECRYPT) + aes_control |= SSS_AES_MODE_DECRYPT; + + if ((mode & FLAGS_AES_MODE_MASK) == FLAGS_AES_CBC) { + aes_control |= SSS_AES_CHAIN_MODE_CBC; + iv = req->iv; + ctr = NULL; + } else if ((mode & FLAGS_AES_MODE_MASK) == FLAGS_AES_CTR) { + aes_control |= SSS_AES_CHAIN_MODE_CTR; + iv = NULL; + ctr = req->iv; + } else { + iv = NULL; /* AES_ECB */ + ctr = NULL; + } + + if (dev->ctx->keylen == AES_KEYSIZE_192) + aes_control |= SSS_AES_KEY_SIZE_192; + else if (dev->ctx->keylen == AES_KEYSIZE_256) + aes_control |= SSS_AES_KEY_SIZE_256; + + aes_control |= SSS_AES_FIFO_MODE; + + /* as a variant it is possible to use byte swapping on DMA side */ + aes_control |= SSS_AES_BYTESWAP_DI + | SSS_AES_BYTESWAP_DO + | SSS_AES_BYTESWAP_IV + | SSS_AES_BYTESWAP_KEY + | SSS_AES_BYTESWAP_CNT; + + spin_lock_irqsave(&dev->lock, flags); + + SSS_WRITE(dev, FCINTENCLR, + SSS_FCINTENCLR_BTDMAINTENCLR | SSS_FCINTENCLR_BRDMAINTENCLR); + SSS_WRITE(dev, FCFIFOCTRL, 0x00); + + err = s5p_set_indata_start(dev, req); + if (err) + goto indata_error; + + err = s5p_set_outdata_start(dev, req); + if (err) + goto outdata_error; + + SSS_AES_WRITE(dev, AES_CONTROL, aes_control); + s5p_set_aes(dev, dev->ctx->aes_key, iv, ctr, dev->ctx->keylen); + + s5p_set_dma_indata(dev, dev->sg_src); + s5p_set_dma_outdata(dev, dev->sg_dst); + + SSS_WRITE(dev, FCINTENSET, + SSS_FCINTENSET_BTDMAINTENSET | SSS_FCINTENSET_BRDMAINTENSET); + + spin_unlock_irqrestore(&dev->lock, flags); + + return; + +outdata_error: + s5p_unset_indata(dev); + +indata_error: + s5p_sg_done(dev); + dev->busy = false; + spin_unlock_irqrestore(&dev->lock, flags); + s5p_aes_complete(req, err); +} + +static void s5p_tasklet_cb(unsigned long data) +{ + struct s5p_aes_dev *dev = (struct s5p_aes_dev *)data; + struct crypto_async_request *async_req, *backlog; + struct s5p_aes_reqctx *reqctx; + unsigned long flags; + + spin_lock_irqsave(&dev->lock, flags); + backlog = crypto_get_backlog(&dev->queue); + async_req = crypto_dequeue_request(&dev->queue); + + if (!async_req) { + dev->busy = false; + spin_unlock_irqrestore(&dev->lock, flags); + return; + } + spin_unlock_irqrestore(&dev->lock, flags); + + if (backlog) + crypto_request_complete(backlog, -EINPROGRESS); + + dev->req = skcipher_request_cast(async_req); + dev->ctx = crypto_tfm_ctx(dev->req->base.tfm); + reqctx = skcipher_request_ctx(dev->req); + + s5p_aes_crypt_start(dev, reqctx->mode); +} + +static int s5p_aes_handle_req(struct s5p_aes_dev *dev, + struct skcipher_request *req) +{ + unsigned long flags; + int err; + + spin_lock_irqsave(&dev->lock, flags); + err = crypto_enqueue_request(&dev->queue, &req->base); + if (dev->busy) { + spin_unlock_irqrestore(&dev->lock, flags); + return err; + } + dev->busy = true; + + spin_unlock_irqrestore(&dev->lock, flags); + + tasklet_schedule(&dev->tasklet); + + return err; +} + +static int s5p_aes_crypt(struct skcipher_request *req, unsigned long mode) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct s5p_aes_reqctx *reqctx = skcipher_request_ctx(req); + struct s5p_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + struct s5p_aes_dev *dev = ctx->dev; + + if (!req->cryptlen) + return 0; + + if (!IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE) && + ((mode & FLAGS_AES_MODE_MASK) != FLAGS_AES_CTR)) { + dev_dbg(dev->dev, "request size is not exact amount of AES blocks\n"); + return -EINVAL; + } + + reqctx->mode = mode; + + return s5p_aes_handle_req(dev, req); +} + +static int s5p_aes_setkey(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); + struct s5p_aes_ctx *ctx = crypto_tfm_ctx(tfm); + + if (keylen != AES_KEYSIZE_128 && + keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) + return -EINVAL; + + memcpy(ctx->aes_key, key, keylen); + ctx->keylen = keylen; + + return 0; +} + +static int s5p_aes_ecb_encrypt(struct skcipher_request *req) +{ + return s5p_aes_crypt(req, 0); +} + +static int s5p_aes_ecb_decrypt(struct skcipher_request *req) +{ + return s5p_aes_crypt(req, FLAGS_AES_DECRYPT); +} + +static int s5p_aes_cbc_encrypt(struct skcipher_request *req) +{ + return s5p_aes_crypt(req, FLAGS_AES_CBC); +} + +static int s5p_aes_cbc_decrypt(struct skcipher_request *req) +{ + return s5p_aes_crypt(req, FLAGS_AES_DECRYPT | FLAGS_AES_CBC); +} + +static int s5p_aes_ctr_crypt(struct skcipher_request *req) +{ + return s5p_aes_crypt(req, FLAGS_AES_CTR); +} + +static int s5p_aes_init_tfm(struct crypto_skcipher *tfm) +{ + struct s5p_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + + ctx->dev = s5p_dev; + crypto_skcipher_set_reqsize(tfm, sizeof(struct s5p_aes_reqctx)); + + return 0; +} + +static struct skcipher_alg algs[] = { + { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-s5p", + .base.cra_priority = 100, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct s5p_aes_ctx), + .base.cra_alignmask = 0x0f, + .base.cra_module = THIS_MODULE, + + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = s5p_aes_setkey, + .encrypt = s5p_aes_ecb_encrypt, + .decrypt = s5p_aes_ecb_decrypt, + .init = s5p_aes_init_tfm, + }, + { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-s5p", + .base.cra_priority = 100, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct s5p_aes_ctx), + .base.cra_alignmask = 0x0f, + .base.cra_module = THIS_MODULE, + + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = s5p_aes_setkey, + .encrypt = s5p_aes_cbc_encrypt, + .decrypt = s5p_aes_cbc_decrypt, + .init = s5p_aes_init_tfm, + }, + { + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "ctr-aes-s5p", + .base.cra_priority = 100, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct s5p_aes_ctx), + .base.cra_alignmask = 0x0f, + .base.cra_module = THIS_MODULE, + + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = s5p_aes_setkey, + .encrypt = s5p_aes_ctr_crypt, + .decrypt = s5p_aes_ctr_crypt, + .init = s5p_aes_init_tfm, + }, +}; + +static int s5p_aes_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + int i, err; + const struct samsung_aes_variant *variant; + struct s5p_aes_dev *pdata; + struct resource *res; + unsigned int hash_i; + + if (s5p_dev) + return -EEXIST; + + pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL); + if (!pdata) + return -ENOMEM; + + variant = find_s5p_sss_version(pdev); + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) + return -EINVAL; + + /* + * Note: HASH and PRNG uses the same registers in secss, avoid + * overwrite each other. This will drop HASH when CONFIG_EXYNOS_RNG + * is enabled in config. We need larger size for HASH registers in + * secss, current describe only AES/DES + */ + if (IS_ENABLED(CONFIG_CRYPTO_DEV_EXYNOS_HASH)) { + if (variant == &exynos_aes_data) { + res->end += 0x300; + pdata->use_hash = true; + } + } + + pdata->res = res; + pdata->ioaddr = devm_ioremap_resource(dev, res); + if (IS_ERR(pdata->ioaddr)) { + if (!pdata->use_hash) + return PTR_ERR(pdata->ioaddr); + /* try AES without HASH */ + res->end -= 0x300; + pdata->use_hash = false; + pdata->ioaddr = devm_ioremap_resource(dev, res); + if (IS_ERR(pdata->ioaddr)) + return PTR_ERR(pdata->ioaddr); + } + + pdata->clk = devm_clk_get(dev, variant->clk_names[0]); + if (IS_ERR(pdata->clk)) + return dev_err_probe(dev, PTR_ERR(pdata->clk), + "failed to find secss clock %s\n", + variant->clk_names[0]); + + err = clk_prepare_enable(pdata->clk); + if (err < 0) { + dev_err(dev, "Enabling clock %s failed, err %d\n", + variant->clk_names[0], err); + return err; + } + + if (variant->clk_names[1]) { + pdata->pclk = devm_clk_get(dev, variant->clk_names[1]); + if (IS_ERR(pdata->pclk)) { + err = dev_err_probe(dev, PTR_ERR(pdata->pclk), + "failed to find clock %s\n", + variant->clk_names[1]); + goto err_clk; + } + + err = clk_prepare_enable(pdata->pclk); + if (err < 0) { + dev_err(dev, "Enabling clock %s failed, err %d\n", + variant->clk_names[0], err); + goto err_clk; + } + } else { + pdata->pclk = NULL; + } + + spin_lock_init(&pdata->lock); + spin_lock_init(&pdata->hash_lock); + + pdata->aes_ioaddr = pdata->ioaddr + variant->aes_offset; + pdata->io_hash_base = pdata->ioaddr + variant->hash_offset; + + pdata->irq_fc = platform_get_irq(pdev, 0); + if (pdata->irq_fc < 0) { + err = pdata->irq_fc; + dev_warn(dev, "feed control interrupt is not available.\n"); + goto err_irq; + } + err = devm_request_threaded_irq(dev, pdata->irq_fc, NULL, + s5p_aes_interrupt, IRQF_ONESHOT, + pdev->name, pdev); + if (err < 0) { + dev_warn(dev, "feed control interrupt is not available.\n"); + goto err_irq; + } + + pdata->busy = false; + pdata->dev = dev; + platform_set_drvdata(pdev, pdata); + s5p_dev = pdata; + + tasklet_init(&pdata->tasklet, s5p_tasklet_cb, (unsigned long)pdata); + crypto_init_queue(&pdata->queue, CRYPTO_QUEUE_LEN); + + for (i = 0; i < ARRAY_SIZE(algs); i++) { + err = crypto_register_skcipher(&algs[i]); + if (err) { + dev_err(dev, "can't register '%s': %d\n", + algs[i].base.cra_name, err); + goto err_algs; + } + } + + if (pdata->use_hash) { + tasklet_init(&pdata->hash_tasklet, s5p_hash_tasklet_cb, + (unsigned long)pdata); + crypto_init_queue(&pdata->hash_queue, SSS_HASH_QUEUE_LENGTH); + + for (hash_i = 0; hash_i < ARRAY_SIZE(algs_sha1_md5_sha256); + hash_i++) { + struct ahash_alg *alg; + + alg = &algs_sha1_md5_sha256[hash_i]; + err = crypto_register_ahash(alg); + if (err) { + dev_err(dev, "can't register '%s': %d\n", + alg->halg.base.cra_driver_name, err); + goto err_hash; + } + } + } + + dev_info(dev, "s5p-sss driver registered\n"); + + return 0; + +err_hash: + crypto_unregister_ahashes(algs_sha1_md5_sha256, hash_i); + tasklet_kill(&pdata->hash_tasklet); + res->end -= 0x300; + +err_algs: + crypto_unregister_skciphers(algs, i); + tasklet_kill(&pdata->tasklet); + +err_irq: + clk_disable_unprepare(pdata->pclk); + +err_clk: + clk_disable_unprepare(pdata->clk); + s5p_dev = NULL; + + return err; +} + +static void s5p_aes_remove(struct platform_device *pdev) +{ + struct s5p_aes_dev *pdata = platform_get_drvdata(pdev); + + crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); + + tasklet_kill(&pdata->tasklet); + if (pdata->use_hash) { + crypto_unregister_ahashes(algs_sha1_md5_sha256, + ARRAY_SIZE(algs_sha1_md5_sha256)); + + pdata->res->end -= 0x300; + tasklet_kill(&pdata->hash_tasklet); + pdata->use_hash = false; + } + + clk_disable_unprepare(pdata->pclk); + + clk_disable_unprepare(pdata->clk); + s5p_dev = NULL; +} + +static struct platform_driver s5p_aes_crypto = { + .probe = s5p_aes_probe, + .remove = s5p_aes_remove, + .driver = { + .name = "s5p-secss", + .of_match_table = s5p_sss_dt_match, + }, +}; + +module_platform_driver(s5p_aes_crypto); + +MODULE_DESCRIPTION("S5PV210 AES hw acceleration support."); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Vladimir Zapolskiy "); +MODULE_AUTHOR("Kamil Konieczny "); diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c new file mode 100644 index 000000000..9846cbeb3 --- /dev/null +++ b/drivers/crypto/sa2ul.c @@ -0,0 +1,2474 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * K3 SA2UL crypto accelerator driver + * + * Copyright (C) 2018-2020 Texas Instruments Incorporated - http://www.ti.com + * + * Authors: Keerthy + * Vitaly Andrianov + * Tero Kristo + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sa2ul.h" + +/* Byte offset for key in encryption security context */ +#define SC_ENC_KEY_OFFSET (1 + 27 + 4) +/* Byte offset for Aux-1 in encryption security context */ +#define SC_ENC_AUX1_OFFSET (1 + 27 + 4 + 32) + +#define SA_CMDL_UPD_ENC 0x0001 +#define SA_CMDL_UPD_AUTH 0x0002 +#define SA_CMDL_UPD_ENC_IV 0x0004 +#define SA_CMDL_UPD_AUTH_IV 0x0008 +#define SA_CMDL_UPD_AUX_KEY 0x0010 + +#define SA_AUTH_SUBKEY_LEN 16 +#define SA_CMDL_PAYLOAD_LENGTH_MASK 0xFFFF +#define SA_CMDL_SOP_BYPASS_LEN_MASK 0xFF000000 + +#define MODE_CONTROL_BYTES 27 +#define SA_HASH_PROCESSING 0 +#define SA_CRYPTO_PROCESSING 0 +#define SA_UPLOAD_HASH_TO_TLR BIT(6) + +#define SA_SW0_FLAGS_MASK 0xF0000 +#define SA_SW0_CMDL_INFO_MASK 0x1F00000 +#define SA_SW0_CMDL_PRESENT BIT(4) +#define SA_SW0_ENG_ID_MASK 0x3E000000 +#define SA_SW0_DEST_INFO_PRESENT BIT(30) +#define SA_SW2_EGRESS_LENGTH 0xFF000000 +#define SA_BASIC_HASH 0x10 + +#define SHA256_DIGEST_WORDS 8 +/* Make 32-bit word from 4 bytes */ +#define SA_MK_U32(b0, b1, b2, b3) (((b0) << 24) | ((b1) << 16) | \ + ((b2) << 8) | (b3)) + +/* size of SCCTL structure in bytes */ +#define SA_SCCTL_SZ 16 + +/* Max Authentication tag size */ +#define SA_MAX_AUTH_TAG_SZ 64 + +enum sa_algo_id { + SA_ALG_CBC_AES = 0, + SA_ALG_EBC_AES, + SA_ALG_CBC_DES3, + SA_ALG_ECB_DES3, + SA_ALG_SHA1, + SA_ALG_SHA256, + SA_ALG_SHA512, + SA_ALG_AUTHENC_SHA1_AES, + SA_ALG_AUTHENC_SHA256_AES, +}; + +struct sa_match_data { + u8 priv; + u8 priv_id; + u32 supported_algos; +}; + +static struct device *sa_k3_dev; + +/** + * struct sa_cmdl_cfg - Command label configuration descriptor + * @aalg: authentication algorithm ID + * @enc_eng_id: Encryption Engine ID supported by the SA hardware + * @auth_eng_id: Authentication Engine ID + * @iv_size: Initialization Vector size + * @akey: Authentication key + * @akey_len: Authentication key length + * @enc: True, if this is an encode request + */ +struct sa_cmdl_cfg { + int aalg; + u8 enc_eng_id; + u8 auth_eng_id; + u8 iv_size; + const u8 *akey; + u16 akey_len; + bool enc; +}; + +/** + * struct algo_data - Crypto algorithm specific data + * @enc_eng: Encryption engine info structure + * @auth_eng: Authentication engine info structure + * @auth_ctrl: Authentication control word + * @hash_size: Size of digest + * @iv_idx: iv index in psdata + * @iv_out_size: iv out size + * @ealg_id: Encryption Algorithm ID + * @aalg_id: Authentication algorithm ID + * @mci_enc: Mode Control Instruction for Encryption algorithm + * @mci_dec: Mode Control Instruction for Decryption + * @inv_key: Whether the encryption algorithm demands key inversion + * @ctx: Pointer to the algorithm context + * @keyed_mac: Whether the authentication algorithm has key + * @prep_iopad: Function pointer to generate intermediate ipad/opad + */ +struct algo_data { + struct sa_eng_info enc_eng; + struct sa_eng_info auth_eng; + u8 auth_ctrl; + u8 hash_size; + u8 iv_idx; + u8 iv_out_size; + u8 ealg_id; + u8 aalg_id; + u8 *mci_enc; + u8 *mci_dec; + bool inv_key; + struct sa_tfm_ctx *ctx; + bool keyed_mac; + void (*prep_iopad)(struct algo_data *algo, const u8 *key, + u16 key_sz, __be32 *ipad, __be32 *opad); +}; + +/** + * struct sa_alg_tmpl: A generic template encompassing crypto/aead algorithms + * @type: Type of the crypto algorithm. + * @alg: Union of crypto algorithm definitions. + * @registered: Flag indicating if the crypto algorithm is already registered + */ +struct sa_alg_tmpl { + u32 type; /* CRYPTO_ALG_TYPE from */ + union { + struct skcipher_alg skcipher; + struct ahash_alg ahash; + struct aead_alg aead; + } alg; + bool registered; +}; + +/** + * struct sa_mapped_sg: scatterlist information for tx and rx + * @mapped: Set to true if the @sgt is mapped + * @dir: mapping direction used for @sgt + * @split_sg: Set if the sg is split and needs to be freed up + * @static_sg: Static scatterlist entry for overriding data + * @sgt: scatterlist table for DMA API use + */ +struct sa_mapped_sg { + bool mapped; + enum dma_data_direction dir; + struct scatterlist static_sg; + struct scatterlist *split_sg; + struct sg_table sgt; +}; +/** + * struct sa_rx_data: RX Packet miscellaneous data place holder + * @req: crypto request data pointer + * @ddev: pointer to the DMA device + * @tx_in: dma_async_tx_descriptor pointer for rx channel + * @mapped_sg: Information on tx (0) and rx (1) scatterlist DMA mapping + * @enc: Flag indicating either encryption or decryption + * @enc_iv_size: Initialisation vector size + * @iv_idx: Initialisation vector index + */ +struct sa_rx_data { + void *req; + struct device *ddev; + struct dma_async_tx_descriptor *tx_in; + struct sa_mapped_sg mapped_sg[2]; + u8 enc; + u8 enc_iv_size; + u8 iv_idx; +}; + +/** + * struct sa_req: SA request definition + * @dev: device for the request + * @size: total data to the xmitted via DMA + * @enc_offset: offset of cipher data + * @enc_size: data to be passed to cipher engine + * @enc_iv: cipher IV + * @auth_offset: offset of the authentication data + * @auth_size: size of the authentication data + * @auth_iv: authentication IV + * @type: algorithm type for the request + * @cmdl: command label pointer + * @base: pointer to the base request + * @ctx: pointer to the algorithm context data + * @enc: true if this is an encode request + * @src: source data + * @dst: destination data + * @callback: DMA callback for the request + * @mdata_size: metadata size passed to DMA + */ +struct sa_req { + struct device *dev; + u16 size; + u8 enc_offset; + u16 enc_size; + u8 *enc_iv; + u8 auth_offset; + u16 auth_size; + u8 *auth_iv; + u32 type; + u32 *cmdl; + struct crypto_async_request *base; + struct sa_tfm_ctx *ctx; + bool enc; + struct scatterlist *src; + struct scatterlist *dst; + dma_async_tx_callback callback; + u16 mdata_size; +}; + +/* + * Mode Control Instructions for various Key lengths 128, 192, 256 + * For CBC (Cipher Block Chaining) mode for encryption + */ +static u8 mci_cbc_enc_array[3][MODE_CONTROL_BYTES] = { + { 0x61, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x61, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x61, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, +}; + +/* + * Mode Control Instructions for various Key lengths 128, 192, 256 + * For CBC (Cipher Block Chaining) mode for decryption + */ +static u8 mci_cbc_dec_array[3][MODE_CONTROL_BYTES] = { + { 0x71, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x71, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x71, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, +}; + +/* + * Mode Control Instructions for various Key lengths 128, 192, 256 + * For CBC (Cipher Block Chaining) mode for encryption + */ +static u8 mci_cbc_enc_no_iv_array[3][MODE_CONTROL_BYTES] = { + { 0x21, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x21, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x21, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, +}; + +/* + * Mode Control Instructions for various Key lengths 128, 192, 256 + * For CBC (Cipher Block Chaining) mode for decryption + */ +static u8 mci_cbc_dec_no_iv_array[3][MODE_CONTROL_BYTES] = { + { 0x31, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x31, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x31, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, +}; + +/* + * Mode Control Instructions for various Key lengths 128, 192, 256 + * For ECB (Electronic Code Book) mode for encryption + */ +static u8 mci_ecb_enc_array[3][27] = { + { 0x21, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x21, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x21, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, +}; + +/* + * Mode Control Instructions for various Key lengths 128, 192, 256 + * For ECB (Electronic Code Book) mode for decryption + */ +static u8 mci_ecb_dec_array[3][27] = { + { 0x31, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x31, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x31, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, +}; + +/* + * Mode Control Instructions for DES algorithm + * For CBC (Cipher Block Chaining) mode and ECB mode + * encryption and for decryption respectively + */ +static u8 mci_cbc_3des_enc_array[MODE_CONTROL_BYTES] = { + 0x60, 0x00, 0x00, 0x18, 0x88, 0x52, 0xaa, 0x4b, 0x7e, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, +}; + +static u8 mci_cbc_3des_dec_array[MODE_CONTROL_BYTES] = { + 0x70, 0x00, 0x00, 0x85, 0x0a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, +}; + +static u8 mci_ecb_3des_enc_array[MODE_CONTROL_BYTES] = { + 0x20, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, +}; + +static u8 mci_ecb_3des_dec_array[MODE_CONTROL_BYTES] = { + 0x30, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, +}; + +/* + * Perform 16 byte or 128 bit swizzling + * The SA2UL Expects the security context to + * be in little Endian and the bus width is 128 bits or 16 bytes + * Hence swap 16 bytes at a time from higher to lower address + */ +static void sa_swiz_128(u8 *in, u16 len) +{ + u8 data[16]; + int i, j; + + for (i = 0; i < len; i += 16) { + memcpy(data, &in[i], 16); + for (j = 0; j < 16; j++) + in[i + j] = data[15 - j]; + } +} + +/* Prepare the ipad and opad from key as per SHA algorithm step 1*/ +static void prepare_kipad(u8 *k_ipad, const u8 *key, u16 key_sz) +{ + int i; + + for (i = 0; i < key_sz; i++) + k_ipad[i] = key[i] ^ 0x36; + + /* Instead of XOR with 0 */ + for (; i < SHA1_BLOCK_SIZE; i++) + k_ipad[i] = 0x36; +} + +static void prepare_kopad(u8 *k_opad, const u8 *key, u16 key_sz) +{ + int i; + + for (i = 0; i < key_sz; i++) + k_opad[i] = key[i] ^ 0x5c; + + /* Instead of XOR with 0 */ + for (; i < SHA1_BLOCK_SIZE; i++) + k_opad[i] = 0x5c; +} + +static void sa_export_shash(void *state, struct shash_desc *hash, + int digest_size, __be32 *out) +{ + struct sha1_state *sha1; + struct sha256_state *sha256; + u32 *result; + + switch (digest_size) { + case SHA1_DIGEST_SIZE: + sha1 = state; + result = sha1->state; + break; + case SHA256_DIGEST_SIZE: + sha256 = state; + result = sha256->state; + break; + default: + dev_err(sa_k3_dev, "%s: bad digest_size=%d\n", __func__, + digest_size); + return; + } + + crypto_shash_export(hash, state); + + cpu_to_be32_array(out, result, digest_size / 4); +} + +static void sa_prepare_iopads(struct algo_data *data, const u8 *key, + u16 key_sz, __be32 *ipad, __be32 *opad) +{ + SHASH_DESC_ON_STACK(shash, data->ctx->shash); + int block_size = crypto_shash_blocksize(data->ctx->shash); + int digest_size = crypto_shash_digestsize(data->ctx->shash); + union { + struct sha1_state sha1; + struct sha256_state sha256; + u8 k_pad[SHA1_BLOCK_SIZE]; + } sha; + + shash->tfm = data->ctx->shash; + + prepare_kipad(sha.k_pad, key, key_sz); + + crypto_shash_init(shash); + crypto_shash_update(shash, sha.k_pad, block_size); + sa_export_shash(&sha, shash, digest_size, ipad); + + prepare_kopad(sha.k_pad, key, key_sz); + + crypto_shash_init(shash); + crypto_shash_update(shash, sha.k_pad, block_size); + + sa_export_shash(&sha, shash, digest_size, opad); + + memzero_explicit(&sha, sizeof(sha)); +} + +/* Derive the inverse key used in AES-CBC decryption operation */ +static inline int sa_aes_inv_key(u8 *inv_key, const u8 *key, u16 key_sz) +{ + struct crypto_aes_ctx ctx; + int key_pos; + + if (aes_expandkey(&ctx, key, key_sz)) { + dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz); + return -EINVAL; + } + + /* work around to get the right inverse for AES_KEYSIZE_192 size keys */ + if (key_sz == AES_KEYSIZE_192) { + ctx.key_enc[52] = ctx.key_enc[51] ^ ctx.key_enc[46]; + ctx.key_enc[53] = ctx.key_enc[52] ^ ctx.key_enc[47]; + } + + /* Based crypto_aes_expand_key logic */ + switch (key_sz) { + case AES_KEYSIZE_128: + case AES_KEYSIZE_192: + key_pos = key_sz + 24; + break; + + case AES_KEYSIZE_256: + key_pos = key_sz + 24 - 4; + break; + + default: + dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz); + return -EINVAL; + } + + memcpy(inv_key, &ctx.key_enc[key_pos], key_sz); + return 0; +} + +/* Set Security context for the encryption engine */ +static int sa_set_sc_enc(struct algo_data *ad, const u8 *key, u16 key_sz, + u8 enc, u8 *sc_buf) +{ + const u8 *mci = NULL; + + /* Set Encryption mode selector to crypto processing */ + sc_buf[0] = SA_CRYPTO_PROCESSING; + + if (enc) + mci = ad->mci_enc; + else + mci = ad->mci_dec; + /* Set the mode control instructions in security context */ + if (mci) + memcpy(&sc_buf[1], mci, MODE_CONTROL_BYTES); + + /* For AES-CBC decryption get the inverse key */ + if (ad->inv_key && !enc) { + if (sa_aes_inv_key(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz)) + return -EINVAL; + /* For all other cases: key is used */ + } else { + memcpy(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz); + } + + return 0; +} + +/* Set Security context for the authentication engine */ +static void sa_set_sc_auth(struct algo_data *ad, const u8 *key, u16 key_sz, + u8 *sc_buf) +{ + __be32 *ipad = (void *)(sc_buf + 32); + __be32 *opad = (void *)(sc_buf + 64); + + /* Set Authentication mode selector to hash processing */ + sc_buf[0] = SA_HASH_PROCESSING; + /* Auth SW ctrl word: bit[6]=1 (upload computed hash to TLR section) */ + sc_buf[1] = SA_UPLOAD_HASH_TO_TLR; + sc_buf[1] |= ad->auth_ctrl; + + /* Copy the keys or ipad/opad */ + if (ad->keyed_mac) + ad->prep_iopad(ad, key, key_sz, ipad, opad); + else { + /* basic hash */ + sc_buf[1] |= SA_BASIC_HASH; + } +} + +static inline void sa_copy_iv(__be32 *out, const u8 *iv, bool size16) +{ + int j; + + for (j = 0; j < ((size16) ? 4 : 2); j++) { + *out = cpu_to_be32(*((u32 *)iv)); + iv += 4; + out++; + } +} + +/* Format general command label */ +static int sa_format_cmdl_gen(struct sa_cmdl_cfg *cfg, u8 *cmdl, + struct sa_cmdl_upd_info *upd_info) +{ + u8 enc_offset = 0, auth_offset = 0, total = 0; + u8 enc_next_eng = SA_ENG_ID_OUTPORT2; + u8 auth_next_eng = SA_ENG_ID_OUTPORT2; + u32 *word_ptr = (u32 *)cmdl; + int i; + + /* Clear the command label */ + memzero_explicit(cmdl, (SA_MAX_CMDL_WORDS * sizeof(u32))); + + /* Initialize the command update structure */ + memzero_explicit(upd_info, sizeof(*upd_info)); + + if (cfg->enc_eng_id && cfg->auth_eng_id) { + if (cfg->enc) { + auth_offset = SA_CMDL_HEADER_SIZE_BYTES; + enc_next_eng = cfg->auth_eng_id; + + if (cfg->iv_size) + auth_offset += cfg->iv_size; + } else { + enc_offset = SA_CMDL_HEADER_SIZE_BYTES; + auth_next_eng = cfg->enc_eng_id; + } + } + + if (cfg->enc_eng_id) { + upd_info->flags |= SA_CMDL_UPD_ENC; + upd_info->enc_size.index = enc_offset >> 2; + upd_info->enc_offset.index = upd_info->enc_size.index + 1; + /* Encryption command label */ + cmdl[enc_offset + SA_CMDL_OFFSET_NESC] = enc_next_eng; + + /* Encryption modes requiring IV */ + if (cfg->iv_size) { + upd_info->flags |= SA_CMDL_UPD_ENC_IV; + upd_info->enc_iv.index = + (enc_offset + SA_CMDL_HEADER_SIZE_BYTES) >> 2; + upd_info->enc_iv.size = cfg->iv_size; + + cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] = + SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size; + + cmdl[enc_offset + SA_CMDL_OFFSET_OPTION_CTRL1] = + (SA_CTX_ENC_AUX2_OFFSET | (cfg->iv_size >> 3)); + total += SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size; + } else { + cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] = + SA_CMDL_HEADER_SIZE_BYTES; + total += SA_CMDL_HEADER_SIZE_BYTES; + } + } + + if (cfg->auth_eng_id) { + upd_info->flags |= SA_CMDL_UPD_AUTH; + upd_info->auth_size.index = auth_offset >> 2; + upd_info->auth_offset.index = upd_info->auth_size.index + 1; + cmdl[auth_offset + SA_CMDL_OFFSET_NESC] = auth_next_eng; + cmdl[auth_offset + SA_CMDL_OFFSET_LABEL_LEN] = + SA_CMDL_HEADER_SIZE_BYTES; + total += SA_CMDL_HEADER_SIZE_BYTES; + } + + total = roundup(total, 8); + + for (i = 0; i < total / 4; i++) + word_ptr[i] = swab32(word_ptr[i]); + + return total; +} + +/* Update Command label */ +static inline void sa_update_cmdl(struct sa_req *req, u32 *cmdl, + struct sa_cmdl_upd_info *upd_info) +{ + int i = 0, j; + + if (likely(upd_info->flags & SA_CMDL_UPD_ENC)) { + cmdl[upd_info->enc_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK; + cmdl[upd_info->enc_size.index] |= req->enc_size; + cmdl[upd_info->enc_offset.index] &= + ~SA_CMDL_SOP_BYPASS_LEN_MASK; + cmdl[upd_info->enc_offset.index] |= + FIELD_PREP(SA_CMDL_SOP_BYPASS_LEN_MASK, + req->enc_offset); + + if (likely(upd_info->flags & SA_CMDL_UPD_ENC_IV)) { + __be32 *data = (__be32 *)&cmdl[upd_info->enc_iv.index]; + u32 *enc_iv = (u32 *)req->enc_iv; + + for (j = 0; i < upd_info->enc_iv.size; i += 4, j++) { + data[j] = cpu_to_be32(*enc_iv); + enc_iv++; + } + } + } + + if (likely(upd_info->flags & SA_CMDL_UPD_AUTH)) { + cmdl[upd_info->auth_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK; + cmdl[upd_info->auth_size.index] |= req->auth_size; + cmdl[upd_info->auth_offset.index] &= + ~SA_CMDL_SOP_BYPASS_LEN_MASK; + cmdl[upd_info->auth_offset.index] |= + FIELD_PREP(SA_CMDL_SOP_BYPASS_LEN_MASK, + req->auth_offset); + if (upd_info->flags & SA_CMDL_UPD_AUTH_IV) { + sa_copy_iv((void *)&cmdl[upd_info->auth_iv.index], + req->auth_iv, + (upd_info->auth_iv.size > 8)); + } + if (upd_info->flags & SA_CMDL_UPD_AUX_KEY) { + int offset = (req->auth_size & 0xF) ? 4 : 0; + + memcpy(&cmdl[upd_info->aux_key_info.index], + &upd_info->aux_key[offset], 16); + } + } +} + +/* Format SWINFO words to be sent to SA */ +static +void sa_set_swinfo(u8 eng_id, u16 sc_id, dma_addr_t sc_phys, + u8 cmdl_present, u8 cmdl_offset, u8 flags, + u8 hash_size, u32 *swinfo) +{ + swinfo[0] = sc_id; + swinfo[0] |= FIELD_PREP(SA_SW0_FLAGS_MASK, flags); + if (likely(cmdl_present)) + swinfo[0] |= FIELD_PREP(SA_SW0_CMDL_INFO_MASK, + cmdl_offset | SA_SW0_CMDL_PRESENT); + swinfo[0] |= FIELD_PREP(SA_SW0_ENG_ID_MASK, eng_id); + + swinfo[0] |= SA_SW0_DEST_INFO_PRESENT; + swinfo[1] = (u32)(sc_phys & 0xFFFFFFFFULL); + swinfo[2] = (u32)((sc_phys & 0xFFFFFFFF00000000ULL) >> 32); + swinfo[2] |= FIELD_PREP(SA_SW2_EGRESS_LENGTH, hash_size); +} + +/* Dump the security context */ +static void sa_dump_sc(u8 *buf, dma_addr_t dma_addr) +{ +#ifdef DEBUG + dev_info(sa_k3_dev, "Security context dump:: 0x%pad\n", &dma_addr); + print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET, + 16, 1, buf, SA_CTX_MAX_SZ, false); +#endif +} + +static +int sa_init_sc(struct sa_ctx_info *ctx, const struct sa_match_data *match_data, + const u8 *enc_key, u16 enc_key_sz, + const u8 *auth_key, u16 auth_key_sz, + struct algo_data *ad, u8 enc, u32 *swinfo) +{ + int enc_sc_offset = 0; + int auth_sc_offset = 0; + u8 *sc_buf = ctx->sc; + u16 sc_id = ctx->sc_id; + u8 first_engine = 0; + + memzero_explicit(sc_buf, SA_CTX_MAX_SZ); + + if (ad->auth_eng.eng_id) { + if (enc) + first_engine = ad->enc_eng.eng_id; + else + first_engine = ad->auth_eng.eng_id; + + enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ; + auth_sc_offset = enc_sc_offset + ad->enc_eng.sc_size; + sc_buf[1] = SA_SCCTL_FE_AUTH_ENC; + if (!ad->hash_size) + return -EINVAL; + ad->hash_size = roundup(ad->hash_size, 8); + + } else if (ad->enc_eng.eng_id && !ad->auth_eng.eng_id) { + enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ; + first_engine = ad->enc_eng.eng_id; + sc_buf[1] = SA_SCCTL_FE_ENC; + ad->hash_size = ad->iv_out_size; + } + + /* SCCTL Owner info: 0=host, 1=CP_ACE */ + sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0; + memcpy(&sc_buf[2], &sc_id, 2); + sc_buf[4] = 0x0; + sc_buf[5] = match_data->priv_id; + sc_buf[6] = match_data->priv; + sc_buf[7] = 0x0; + + /* Prepare context for encryption engine */ + if (ad->enc_eng.sc_size) { + if (sa_set_sc_enc(ad, enc_key, enc_key_sz, enc, + &sc_buf[enc_sc_offset])) + return -EINVAL; + } + + /* Prepare context for authentication engine */ + if (ad->auth_eng.sc_size) + sa_set_sc_auth(ad, auth_key, auth_key_sz, + &sc_buf[auth_sc_offset]); + + /* Set the ownership of context to CP_ACE */ + sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0x80; + + /* swizzle the security context */ + sa_swiz_128(sc_buf, SA_CTX_MAX_SZ); + + sa_set_swinfo(first_engine, ctx->sc_id, ctx->sc_phys, 1, 0, + SA_SW_INFO_FLAG_EVICT, ad->hash_size, swinfo); + + sa_dump_sc(sc_buf, ctx->sc_phys); + + return 0; +} + +/* Free the per direction context memory */ +static void sa_free_ctx_info(struct sa_ctx_info *ctx, + struct sa_crypto_data *data) +{ + unsigned long bn; + + bn = ctx->sc_id - data->sc_id_start; + spin_lock(&data->scid_lock); + __clear_bit(bn, data->ctx_bm); + data->sc_id--; + spin_unlock(&data->scid_lock); + + if (ctx->sc) { + dma_pool_free(data->sc_pool, ctx->sc, ctx->sc_phys); + ctx->sc = NULL; + } +} + +static int sa_init_ctx_info(struct sa_ctx_info *ctx, + struct sa_crypto_data *data) +{ + unsigned long bn; + int err; + + spin_lock(&data->scid_lock); + bn = find_first_zero_bit(data->ctx_bm, SA_MAX_NUM_CTX); + __set_bit(bn, data->ctx_bm); + data->sc_id++; + spin_unlock(&data->scid_lock); + + ctx->sc_id = (u16)(data->sc_id_start + bn); + + ctx->sc = dma_pool_alloc(data->sc_pool, GFP_KERNEL, &ctx->sc_phys); + if (!ctx->sc) { + dev_err(&data->pdev->dev, "Failed to allocate SC memory\n"); + err = -ENOMEM; + goto scid_rollback; + } + + return 0; + +scid_rollback: + spin_lock(&data->scid_lock); + __clear_bit(bn, data->ctx_bm); + data->sc_id--; + spin_unlock(&data->scid_lock); + + return err; +} + +static void sa_cipher_cra_exit(struct crypto_skcipher *tfm) +{ + struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); + + dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", + __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, + ctx->dec.sc_id, &ctx->dec.sc_phys); + + sa_free_ctx_info(&ctx->enc, data); + sa_free_ctx_info(&ctx->dec, data); + + crypto_free_skcipher(ctx->fallback.skcipher); +} + +static int sa_cipher_cra_init(struct crypto_skcipher *tfm) +{ + struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); + const char *name = crypto_tfm_alg_name(&tfm->base); + struct crypto_skcipher *child; + int ret; + + memzero_explicit(ctx, sizeof(*ctx)); + ctx->dev_data = data; + + ret = sa_init_ctx_info(&ctx->enc, data); + if (ret) + return ret; + ret = sa_init_ctx_info(&ctx->dec, data); + if (ret) { + sa_free_ctx_info(&ctx->enc, data); + return ret; + } + + child = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); + + if (IS_ERR(child)) { + dev_err(sa_k3_dev, "Error allocating fallback algo %s\n", name); + return PTR_ERR(child); + } + + ctx->fallback.skcipher = child; + crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(child) + + sizeof(struct skcipher_request)); + + dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", + __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, + ctx->dec.sc_id, &ctx->dec.sc_phys); + return 0; +} + +static int sa_cipher_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen, struct algo_data *ad) +{ + struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + struct crypto_skcipher *child = ctx->fallback.skcipher; + int cmdl_len; + struct sa_cmdl_cfg cfg; + int ret; + + if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) + return -EINVAL; + + ad->enc_eng.eng_id = SA_ENG_ID_EM1; + ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ; + + memzero_explicit(&cfg, sizeof(cfg)); + cfg.enc_eng_id = ad->enc_eng.eng_id; + cfg.iv_size = crypto_skcipher_ivsize(tfm); + + crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(child, tfm->base.crt_flags & + CRYPTO_TFM_REQ_MASK); + ret = crypto_skcipher_setkey(child, key, keylen); + if (ret) + return ret; + + /* Setup Encryption Security Context & Command label template */ + if (sa_init_sc(&ctx->enc, ctx->dev_data->match_data, key, keylen, NULL, 0, + ad, 1, &ctx->enc.epib[1])) + goto badkey; + + cmdl_len = sa_format_cmdl_gen(&cfg, + (u8 *)ctx->enc.cmdl, + &ctx->enc.cmdl_upd_info); + if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) + goto badkey; + + ctx->enc.cmdl_size = cmdl_len; + + /* Setup Decryption Security Context & Command label template */ + if (sa_init_sc(&ctx->dec, ctx->dev_data->match_data, key, keylen, NULL, 0, + ad, 0, &ctx->dec.epib[1])) + goto badkey; + + cfg.enc_eng_id = ad->enc_eng.eng_id; + cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl, + &ctx->dec.cmdl_upd_info); + + if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) + goto badkey; + + ctx->dec.cmdl_size = cmdl_len; + ctx->iv_idx = ad->iv_idx; + + return 0; + +badkey: + dev_err(sa_k3_dev, "%s: badkey\n", __func__); + return -EINVAL; +} + +static int sa_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct algo_data ad = { 0 }; + /* Convert the key size (16/24/32) to the key size index (0/1/2) */ + int key_idx = (keylen >> 3) - 2; + + if (key_idx >= 3) + return -EINVAL; + + ad.mci_enc = mci_cbc_enc_array[key_idx]; + ad.mci_dec = mci_cbc_dec_array[key_idx]; + ad.inv_key = true; + ad.ealg_id = SA_EALG_ID_AES_CBC; + ad.iv_idx = 4; + ad.iv_out_size = 16; + + return sa_cipher_setkey(tfm, key, keylen, &ad); +} + +static int sa_aes_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct algo_data ad = { 0 }; + /* Convert the key size (16/24/32) to the key size index (0/1/2) */ + int key_idx = (keylen >> 3) - 2; + + if (key_idx >= 3) + return -EINVAL; + + ad.mci_enc = mci_ecb_enc_array[key_idx]; + ad.mci_dec = mci_ecb_dec_array[key_idx]; + ad.inv_key = true; + ad.ealg_id = SA_EALG_ID_AES_ECB; + + return sa_cipher_setkey(tfm, key, keylen, &ad); +} + +static int sa_3des_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct algo_data ad = { 0 }; + + ad.mci_enc = mci_cbc_3des_enc_array; + ad.mci_dec = mci_cbc_3des_dec_array; + ad.ealg_id = SA_EALG_ID_3DES_CBC; + ad.iv_idx = 6; + ad.iv_out_size = 8; + + return sa_cipher_setkey(tfm, key, keylen, &ad); +} + +static int sa_3des_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct algo_data ad = { 0 }; + + ad.mci_enc = mci_ecb_3des_enc_array; + ad.mci_dec = mci_ecb_3des_dec_array; + + return sa_cipher_setkey(tfm, key, keylen, &ad); +} + +static void sa_sync_from_device(struct sa_rx_data *rxd) +{ + struct sg_table *sgt; + + if (rxd->mapped_sg[0].dir == DMA_BIDIRECTIONAL) + sgt = &rxd->mapped_sg[0].sgt; + else + sgt = &rxd->mapped_sg[1].sgt; + + dma_sync_sgtable_for_cpu(rxd->ddev, sgt, DMA_FROM_DEVICE); +} + +static void sa_free_sa_rx_data(struct sa_rx_data *rxd) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(rxd->mapped_sg); i++) { + struct sa_mapped_sg *mapped_sg = &rxd->mapped_sg[i]; + + if (mapped_sg->mapped) { + dma_unmap_sgtable(rxd->ddev, &mapped_sg->sgt, + mapped_sg->dir, 0); + kfree(mapped_sg->split_sg); + } + } + + kfree(rxd); +} + +static void sa_aes_dma_in_callback(void *data) +{ + struct sa_rx_data *rxd = data; + struct skcipher_request *req; + u32 *result; + __be32 *mdptr; + size_t ml, pl; + int i; + + sa_sync_from_device(rxd); + req = container_of(rxd->req, struct skcipher_request, base); + + if (req->iv) { + mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, + &ml); + result = (u32 *)req->iv; + + for (i = 0; i < (rxd->enc_iv_size / 4); i++) + result[i] = be32_to_cpu(mdptr[i + rxd->iv_idx]); + } + + sa_free_sa_rx_data(rxd); + + skcipher_request_complete(req, 0); +} + +static void +sa_prepare_tx_desc(u32 *mdptr, u32 pslen, u32 *psdata, u32 epiblen, u32 *epib) +{ + u32 *out, *in; + int i; + + for (out = mdptr, in = epib, i = 0; i < epiblen / sizeof(u32); i++) + *out++ = *in++; + + mdptr[4] = (0xFFFF << 16); + for (out = &mdptr[5], in = psdata, i = 0; + i < pslen / sizeof(u32); i++) + *out++ = *in++; +} + +static int sa_run(struct sa_req *req) +{ + struct sa_rx_data *rxd; + gfp_t gfp_flags; + u32 cmdl[SA_MAX_CMDL_WORDS]; + struct sa_crypto_data *pdata = dev_get_drvdata(sa_k3_dev); + struct device *ddev; + struct dma_chan *dma_rx; + int sg_nents, src_nents, dst_nents; + struct scatterlist *src, *dst; + size_t pl, ml, split_size; + struct sa_ctx_info *sa_ctx = req->enc ? &req->ctx->enc : &req->ctx->dec; + int ret; + struct dma_async_tx_descriptor *tx_out; + u32 *mdptr; + bool diff_dst; + enum dma_data_direction dir_src; + struct sa_mapped_sg *mapped_sg; + + gfp_flags = req->base->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + rxd = kzalloc_obj(*rxd, gfp_flags); + if (!rxd) + return -ENOMEM; + + if (req->src != req->dst) { + diff_dst = true; + dir_src = DMA_TO_DEVICE; + } else { + diff_dst = false; + dir_src = DMA_BIDIRECTIONAL; + } + + /* + * SA2UL has an interesting feature where the receive DMA channel + * is selected based on the data passed to the engine. Within the + * transition range, there is also a space where it is impossible + * to determine where the data will end up, and this should be + * avoided. This will be handled by the SW fallback mechanism by + * the individual algorithm implementations. + */ + if (req->size >= 256) + dma_rx = pdata->dma_rx2; + else + dma_rx = pdata->dma_rx1; + + ddev = dmaengine_get_dma_device(pdata->dma_tx); + rxd->ddev = ddev; + + memcpy(cmdl, sa_ctx->cmdl, sa_ctx->cmdl_size); + + sa_update_cmdl(req, cmdl, &sa_ctx->cmdl_upd_info); + + if (req->type != CRYPTO_ALG_TYPE_AHASH) { + if (req->enc) + req->type |= + (SA_REQ_SUBTYPE_ENC << SA_REQ_SUBTYPE_SHIFT); + else + req->type |= + (SA_REQ_SUBTYPE_DEC << SA_REQ_SUBTYPE_SHIFT); + } + + cmdl[sa_ctx->cmdl_size / sizeof(u32)] = req->type; + + /* + * Map the packets, first we check if the data fits into a single + * sg entry and use that if possible. If it does not fit, we check + * if we need to do sg_split to align the scatterlist data on the + * actual data size being processed by the crypto engine. + */ + src = req->src; + sg_nents = sg_nents_for_len(src, req->size); + + split_size = req->size; + + mapped_sg = &rxd->mapped_sg[0]; + if (sg_nents == 1 && split_size <= req->src->length) { + src = &mapped_sg->static_sg; + src_nents = 1; + sg_init_table(src, 1); + sg_set_page(src, sg_page(req->src), split_size, + req->src->offset); + + mapped_sg->sgt.sgl = src; + mapped_sg->sgt.orig_nents = src_nents; + ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0); + if (ret) { + kfree(rxd); + return ret; + } + + mapped_sg->dir = dir_src; + mapped_sg->mapped = true; + } else { + mapped_sg->sgt.sgl = req->src; + mapped_sg->sgt.orig_nents = sg_nents; + ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0); + if (ret) { + kfree(rxd); + return ret; + } + + mapped_sg->dir = dir_src; + mapped_sg->mapped = true; + + ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents, 0, 1, + &split_size, &src, &src_nents, gfp_flags); + if (ret) { + src_nents = mapped_sg->sgt.nents; + src = mapped_sg->sgt.sgl; + } else { + mapped_sg->split_sg = src; + } + } + + dma_sync_sgtable_for_device(ddev, &mapped_sg->sgt, DMA_TO_DEVICE); + + if (!diff_dst) { + dst_nents = src_nents; + dst = src; + } else { + dst_nents = sg_nents_for_len(req->dst, req->size); + mapped_sg = &rxd->mapped_sg[1]; + + if (dst_nents == 1 && split_size <= req->dst->length) { + dst = &mapped_sg->static_sg; + dst_nents = 1; + sg_init_table(dst, 1); + sg_set_page(dst, sg_page(req->dst), split_size, + req->dst->offset); + + mapped_sg->sgt.sgl = dst; + mapped_sg->sgt.orig_nents = dst_nents; + ret = dma_map_sgtable(ddev, &mapped_sg->sgt, + DMA_FROM_DEVICE, 0); + if (ret) + goto err_cleanup; + + mapped_sg->dir = DMA_FROM_DEVICE; + mapped_sg->mapped = true; + } else { + mapped_sg->sgt.sgl = req->dst; + mapped_sg->sgt.orig_nents = dst_nents; + ret = dma_map_sgtable(ddev, &mapped_sg->sgt, + DMA_FROM_DEVICE, 0); + if (ret) + goto err_cleanup; + + mapped_sg->dir = DMA_FROM_DEVICE; + mapped_sg->mapped = true; + + ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents, + 0, 1, &split_size, &dst, &dst_nents, + gfp_flags); + if (ret) { + dst_nents = mapped_sg->sgt.nents; + dst = mapped_sg->sgt.sgl; + } else { + mapped_sg->split_sg = dst; + } + } + } + + rxd->tx_in = dmaengine_prep_slave_sg(dma_rx, dst, dst_nents, + DMA_DEV_TO_MEM, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!rxd->tx_in) { + dev_err(pdata->dev, "IN prep_slave_sg() failed\n"); + ret = -EINVAL; + goto err_cleanup; + } + + rxd->req = (void *)req->base; + rxd->enc = req->enc; + rxd->iv_idx = req->ctx->iv_idx; + rxd->enc_iv_size = sa_ctx->cmdl_upd_info.enc_iv.size; + rxd->tx_in->callback = req->callback; + rxd->tx_in->callback_param = rxd; + + tx_out = dmaengine_prep_slave_sg(pdata->dma_tx, src, + src_nents, DMA_MEM_TO_DEV, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + + if (!tx_out) { + dev_err(pdata->dev, "OUT prep_slave_sg() failed\n"); + ret = -EINVAL; + goto err_cleanup; + } + + /* + * Prepare metadata for DMA engine. This essentially describes the + * crypto algorithm to be used, data sizes, different keys etc. + */ + mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(tx_out, &pl, &ml); + + sa_prepare_tx_desc(mdptr, (sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS * + sizeof(u32))), cmdl, sizeof(sa_ctx->epib), + sa_ctx->epib); + + ml = sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS * sizeof(u32)); + dmaengine_desc_set_metadata_len(tx_out, req->mdata_size); + + dmaengine_submit(tx_out); + dmaengine_submit(rxd->tx_in); + + dma_async_issue_pending(dma_rx); + dma_async_issue_pending(pdata->dma_tx); + + return -EINPROGRESS; + +err_cleanup: + sa_free_sa_rx_data(rxd); + + return ret; +} + +static int sa_cipher_run(struct skcipher_request *req, u8 *iv, int enc) +{ + struct sa_tfm_ctx *ctx = + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + struct crypto_alg *alg = req->base.tfm->__crt_alg; + struct sa_req sa_req = { 0 }; + + if (!req->cryptlen) + return 0; + + if (req->cryptlen % alg->cra_blocksize) + return -EINVAL; + + /* Use SW fallback if the data size is not supported */ + if (req->cryptlen > SA_MAX_DATA_SZ || + (req->cryptlen >= SA_UNSAFE_DATA_SZ_MIN && + req->cryptlen <= SA_UNSAFE_DATA_SZ_MAX)) { + struct skcipher_request *subreq = skcipher_request_ctx(req); + + skcipher_request_set_tfm(subreq, ctx->fallback.skcipher); + skcipher_request_set_callback(subreq, req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + if (enc) + return crypto_skcipher_encrypt(subreq); + else + return crypto_skcipher_decrypt(subreq); + } + + sa_req.size = req->cryptlen; + sa_req.enc_size = req->cryptlen; + sa_req.src = req->src; + sa_req.dst = req->dst; + sa_req.enc_iv = iv; + sa_req.type = CRYPTO_ALG_TYPE_SKCIPHER; + sa_req.enc = enc; + sa_req.callback = sa_aes_dma_in_callback; + sa_req.mdata_size = 44; + sa_req.base = &req->base; + sa_req.ctx = ctx; + + return sa_run(&sa_req); +} + +static int sa_encrypt(struct skcipher_request *req) +{ + return sa_cipher_run(req, req->iv, 1); +} + +static int sa_decrypt(struct skcipher_request *req) +{ + return sa_cipher_run(req, req->iv, 0); +} + +static void sa_sha_dma_in_callback(void *data) +{ + struct sa_rx_data *rxd = data; + struct ahash_request *req; + struct crypto_ahash *tfm; + unsigned int authsize; + int i; + size_t ml, pl; + u32 *result; + __be32 *mdptr; + + sa_sync_from_device(rxd); + req = container_of(rxd->req, struct ahash_request, base); + tfm = crypto_ahash_reqtfm(req); + authsize = crypto_ahash_digestsize(tfm); + + mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml); + result = (u32 *)req->result; + + for (i = 0; i < (authsize / 4); i++) + result[i] = be32_to_cpu(mdptr[i + 4]); + + sa_free_sa_rx_data(rxd); + + ahash_request_complete(req, 0); +} + +static int zero_message_process(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + int sa_digest_size = crypto_ahash_digestsize(tfm); + + switch (sa_digest_size) { + case SHA1_DIGEST_SIZE: + memcpy(req->result, sha1_zero_message_hash, sa_digest_size); + break; + case SHA256_DIGEST_SIZE: + memcpy(req->result, sha256_zero_message_hash, sa_digest_size); + break; + case SHA512_DIGEST_SIZE: + memcpy(req->result, sha512_zero_message_hash, sa_digest_size); + break; + default: + return -EINVAL; + } + + return 0; +} + +static int sa_sha_run(struct ahash_request *req) +{ + struct sa_tfm_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); + struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); + struct sa_req sa_req = { 0 }; + size_t auth_len; + + auth_len = req->nbytes; + + if (!auth_len) + return zero_message_process(req); + + if (auth_len > SA_MAX_DATA_SZ || + (auth_len >= SA_UNSAFE_DATA_SZ_MIN && + auth_len <= SA_UNSAFE_DATA_SZ_MAX)) { + struct ahash_request *subreq = &rctx->fallback_req; + int ret; + + ahash_request_set_tfm(subreq, ctx->fallback.ahash); + ahash_request_set_callback(subreq, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + ahash_request_set_crypt(subreq, req->src, req->result, auth_len); + + ret = crypto_ahash_digest(subreq); + + return ret; + } + + sa_req.size = auth_len; + sa_req.auth_size = auth_len; + sa_req.src = req->src; + sa_req.dst = req->src; + sa_req.enc = true; + sa_req.type = CRYPTO_ALG_TYPE_AHASH; + sa_req.callback = sa_sha_dma_in_callback; + sa_req.mdata_size = 28; + sa_req.ctx = ctx; + sa_req.base = &req->base; + + return sa_run(&sa_req); +} + +static int sa_sha_setup(struct sa_tfm_ctx *ctx, struct algo_data *ad) +{ + int bs = crypto_shash_blocksize(ctx->shash); + int cmdl_len; + struct sa_cmdl_cfg cfg; + + ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ; + ad->auth_eng.eng_id = SA_ENG_ID_AM1; + ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ; + + memset(ctx->authkey, 0, bs); + memset(&cfg, 0, sizeof(cfg)); + cfg.aalg = ad->aalg_id; + cfg.enc_eng_id = ad->enc_eng.eng_id; + cfg.auth_eng_id = ad->auth_eng.eng_id; + cfg.iv_size = 0; + cfg.akey = NULL; + cfg.akey_len = 0; + + ctx->dev_data = dev_get_drvdata(sa_k3_dev); + /* Setup Encryption Security Context & Command label template */ + if (sa_init_sc(&ctx->enc, ctx->dev_data->match_data, NULL, 0, NULL, 0, + ad, 0, &ctx->enc.epib[1])) + goto badkey; + + cmdl_len = sa_format_cmdl_gen(&cfg, + (u8 *)ctx->enc.cmdl, + &ctx->enc.cmdl_upd_info); + if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) + goto badkey; + + ctx->enc.cmdl_size = cmdl_len; + + return 0; + +badkey: + dev_err(sa_k3_dev, "%s: badkey\n", __func__); + return -EINVAL; +} + +static int sa_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) +{ + struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); + struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); + int ret; + + memset(ctx, 0, sizeof(*ctx)); + ctx->dev_data = data; + ret = sa_init_ctx_info(&ctx->enc, data); + if (ret) + return ret; + + if (alg_base) { + ctx->shash = crypto_alloc_shash(alg_base, 0, 0); + if (IS_ERR(ctx->shash)) { + dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n", + alg_base); + return PTR_ERR(ctx->shash); + } + /* for fallback */ + ctx->fallback.ahash = + crypto_alloc_ahash(alg_base, 0, CRYPTO_ALG_ASYNC); + if (IS_ERR(ctx->fallback.ahash)) { + dev_err(ctx->dev_data->dev, + "Could not load fallback driver\n"); + return PTR_ERR(ctx->fallback.ahash); + } + } + + dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", + __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, + ctx->dec.sc_id, &ctx->dec.sc_phys); + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct sa_sha_req_ctx) + + crypto_ahash_reqsize(ctx->fallback.ahash)); + + return 0; +} + +static int sa_sha_digest(struct ahash_request *req) +{ + return sa_sha_run(req); +} + +static int sa_sha_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); + struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); + + dev_dbg(sa_k3_dev, "init: digest size: %u, rctx=%p\n", + crypto_ahash_digestsize(tfm), rctx); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); + ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + ahash_request_set_crypt(&rctx->fallback_req, NULL, NULL, 0); + + return crypto_ahash_init(&rctx->fallback_req); +} + +static int sa_sha_update(struct ahash_request *req) +{ + struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); + + ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + ahash_request_set_crypt(&rctx->fallback_req, req->src, NULL, req->nbytes); + + return crypto_ahash_update(&rctx->fallback_req); +} + +static int sa_sha_final(struct ahash_request *req) +{ + struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); + + ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + ahash_request_set_crypt(&rctx->fallback_req, NULL, req->result, 0); + + return crypto_ahash_final(&rctx->fallback_req); +} + +static int sa_sha_finup(struct ahash_request *req) +{ + struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); + + ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result, req->nbytes); + + return crypto_ahash_finup(&rctx->fallback_req); +} + +static int sa_sha_import(struct ahash_request *req, const void *in) +{ + struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); + ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + + return crypto_ahash_import(&rctx->fallback_req, in); +} + +static int sa_sha_export(struct ahash_request *req, void *out) +{ + struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); + struct ahash_request *subreq = &rctx->fallback_req; + + ahash_request_set_callback(subreq, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + + return crypto_ahash_export(subreq, out); +} + +static int sa_sha1_cra_init(struct crypto_tfm *tfm) +{ + struct algo_data ad = { 0 }; + struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); + + sa_sha_cra_init_alg(tfm, "sha1"); + + ad.aalg_id = SA_AALG_ID_SHA1; + ad.hash_size = SHA1_DIGEST_SIZE; + ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1; + + sa_sha_setup(ctx, &ad); + + return 0; +} + +static int sa_sha256_cra_init(struct crypto_tfm *tfm) +{ + struct algo_data ad = { 0 }; + struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); + + sa_sha_cra_init_alg(tfm, "sha256"); + + ad.aalg_id = SA_AALG_ID_SHA2_256; + ad.hash_size = SHA256_DIGEST_SIZE; + ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256; + + sa_sha_setup(ctx, &ad); + + return 0; +} + +static int sa_sha512_cra_init(struct crypto_tfm *tfm) +{ + struct algo_data ad = { 0 }; + struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); + + sa_sha_cra_init_alg(tfm, "sha512"); + + ad.aalg_id = SA_AALG_ID_SHA2_512; + ad.hash_size = SHA512_DIGEST_SIZE; + ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA512; + + sa_sha_setup(ctx, &ad); + + return 0; +} + +static void sa_sha_cra_exit(struct crypto_tfm *tfm) +{ + struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); + struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); + + dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", + __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, + ctx->dec.sc_id, &ctx->dec.sc_phys); + + if (crypto_tfm_alg_type(tfm) == CRYPTO_ALG_TYPE_AHASH) + sa_free_ctx_info(&ctx->enc, data); + + crypto_free_shash(ctx->shash); + crypto_free_ahash(ctx->fallback.ahash); +} + +static void sa_aead_dma_in_callback(void *data) +{ + struct sa_rx_data *rxd = data; + struct aead_request *req; + struct crypto_aead *tfm; + unsigned int start; + unsigned int authsize; + u8 auth_tag[SA_MAX_AUTH_TAG_SZ]; + size_t pl, ml; + int i; + int err = 0; + u32 *mdptr; + + sa_sync_from_device(rxd); + req = container_of(rxd->req, struct aead_request, base); + tfm = crypto_aead_reqtfm(req); + start = req->assoclen + req->cryptlen; + authsize = crypto_aead_authsize(tfm); + + mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml); + for (i = 0; i < (authsize / 4); i++) + mdptr[i + 4] = swab32(mdptr[i + 4]); + + if (rxd->enc) { + scatterwalk_map_and_copy(&mdptr[4], req->dst, start, authsize, + 1); + } else { + start -= authsize; + scatterwalk_map_and_copy(auth_tag, req->src, start, authsize, + 0); + + err = crypto_memneq(&mdptr[4], auth_tag, authsize) ? -EBADMSG : 0; + } + + sa_free_sa_rx_data(rxd); + + aead_request_complete(req, err); +} + +static int sa_cra_init_aead(struct crypto_aead *tfm, const char *hash, + const char *fallback) +{ + struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm); + struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); + int ret; + + memzero_explicit(ctx, sizeof(*ctx)); + ctx->dev_data = data; + + ctx->shash = crypto_alloc_shash(hash, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->shash)) { + dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n", hash); + return PTR_ERR(ctx->shash); + } + + ctx->fallback.aead = crypto_alloc_aead(fallback, 0, + CRYPTO_ALG_NEED_FALLBACK); + + if (IS_ERR(ctx->fallback.aead)) { + dev_err(sa_k3_dev, "fallback driver %s couldn't be loaded\n", + fallback); + return PTR_ERR(ctx->fallback.aead); + } + + crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) + + crypto_aead_reqsize(ctx->fallback.aead)); + + ret = sa_init_ctx_info(&ctx->enc, data); + if (ret) + return ret; + + ret = sa_init_ctx_info(&ctx->dec, data); + if (ret) { + sa_free_ctx_info(&ctx->enc, data); + return ret; + } + + dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", + __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, + ctx->dec.sc_id, &ctx->dec.sc_phys); + + return ret; +} + +static int sa_cra_init_aead_sha1(struct crypto_aead *tfm) +{ + return sa_cra_init_aead(tfm, "sha1", + "authenc(hmac(sha1),cbc(aes))"); +} + +static int sa_cra_init_aead_sha256(struct crypto_aead *tfm) +{ + return sa_cra_init_aead(tfm, "sha256", + "authenc(hmac(sha256),cbc(aes))"); +} + +static void sa_exit_tfm_aead(struct crypto_aead *tfm) +{ + struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm); + struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); + + crypto_free_shash(ctx->shash); + crypto_free_aead(ctx->fallback.aead); + + sa_free_ctx_info(&ctx->enc, data); + sa_free_ctx_info(&ctx->dec, data); +} + +/* AEAD algorithm configuration interface function */ +static int sa_aead_setkey(struct crypto_aead *authenc, + const u8 *key, unsigned int keylen, + struct algo_data *ad) +{ + struct sa_tfm_ctx *ctx = crypto_aead_ctx(authenc); + struct crypto_authenc_keys keys; + int cmdl_len; + struct sa_cmdl_cfg cfg; + int key_idx; + + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) + return -EINVAL; + + /* Convert the key size (16/24/32) to the key size index (0/1/2) */ + key_idx = (keys.enckeylen >> 3) - 2; + if (key_idx >= 3) + return -EINVAL; + + ad->ctx = ctx; + ad->enc_eng.eng_id = SA_ENG_ID_EM1; + ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ; + ad->auth_eng.eng_id = SA_ENG_ID_AM1; + ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ; + ad->mci_enc = mci_cbc_enc_no_iv_array[key_idx]; + ad->mci_dec = mci_cbc_dec_no_iv_array[key_idx]; + ad->inv_key = true; + ad->keyed_mac = true; + ad->ealg_id = SA_EALG_ID_AES_CBC; + ad->prep_iopad = sa_prepare_iopads; + + memset(&cfg, 0, sizeof(cfg)); + cfg.enc = true; + cfg.aalg = ad->aalg_id; + cfg.enc_eng_id = ad->enc_eng.eng_id; + cfg.auth_eng_id = ad->auth_eng.eng_id; + cfg.iv_size = crypto_aead_ivsize(authenc); + cfg.akey = keys.authkey; + cfg.akey_len = keys.authkeylen; + + /* Setup Encryption Security Context & Command label template */ + if (sa_init_sc(&ctx->enc, ctx->dev_data->match_data, keys.enckey, + keys.enckeylen, keys.authkey, keys.authkeylen, + ad, 1, &ctx->enc.epib[1])) + return -EINVAL; + + cmdl_len = sa_format_cmdl_gen(&cfg, + (u8 *)ctx->enc.cmdl, + &ctx->enc.cmdl_upd_info); + if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) + return -EINVAL; + + ctx->enc.cmdl_size = cmdl_len; + + /* Setup Decryption Security Context & Command label template */ + if (sa_init_sc(&ctx->dec, ctx->dev_data->match_data, keys.enckey, + keys.enckeylen, keys.authkey, keys.authkeylen, + ad, 0, &ctx->dec.epib[1])) + return -EINVAL; + + cfg.enc = false; + cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl, + &ctx->dec.cmdl_upd_info); + + if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) + return -EINVAL; + + ctx->dec.cmdl_size = cmdl_len; + + crypto_aead_clear_flags(ctx->fallback.aead, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(ctx->fallback.aead, + crypto_aead_get_flags(authenc) & + CRYPTO_TFM_REQ_MASK); + + return crypto_aead_setkey(ctx->fallback.aead, key, keylen); +} + +static int sa_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) +{ + struct sa_tfm_ctx *ctx = crypto_tfm_ctx(crypto_aead_tfm(tfm)); + + return crypto_aead_setauthsize(ctx->fallback.aead, authsize); +} + +static int sa_aead_cbc_sha1_setkey(struct crypto_aead *authenc, + const u8 *key, unsigned int keylen) +{ + struct algo_data ad = { 0 }; + + ad.ealg_id = SA_EALG_ID_AES_CBC; + ad.aalg_id = SA_AALG_ID_HMAC_SHA1; + ad.hash_size = SHA1_DIGEST_SIZE; + ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1; + + return sa_aead_setkey(authenc, key, keylen, &ad); +} + +static int sa_aead_cbc_sha256_setkey(struct crypto_aead *authenc, + const u8 *key, unsigned int keylen) +{ + struct algo_data ad = { 0 }; + + ad.ealg_id = SA_EALG_ID_AES_CBC; + ad.aalg_id = SA_AALG_ID_HMAC_SHA2_256; + ad.hash_size = SHA256_DIGEST_SIZE; + ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256; + + return sa_aead_setkey(authenc, key, keylen, &ad); +} + +static int sa_aead_run(struct aead_request *req, u8 *iv, int enc) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm); + struct sa_req sa_req = { 0 }; + size_t auth_size, enc_size; + + enc_size = req->cryptlen; + auth_size = req->assoclen + req->cryptlen; + + if (!enc) { + enc_size -= crypto_aead_authsize(tfm); + auth_size -= crypto_aead_authsize(tfm); + } + + if (auth_size > SA_MAX_DATA_SZ || + (auth_size >= SA_UNSAFE_DATA_SZ_MIN && + auth_size <= SA_UNSAFE_DATA_SZ_MAX)) { + struct aead_request *subreq = aead_request_ctx(req); + int ret; + + aead_request_set_tfm(subreq, ctx->fallback.aead); + aead_request_set_callback(subreq, req->base.flags, + req->base.complete, req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + aead_request_set_ad(subreq, req->assoclen); + + ret = enc ? crypto_aead_encrypt(subreq) : + crypto_aead_decrypt(subreq); + return ret; + } + + sa_req.enc_offset = req->assoclen; + sa_req.enc_size = enc_size; + sa_req.auth_size = auth_size; + sa_req.size = auth_size; + sa_req.enc_iv = iv; + sa_req.type = CRYPTO_ALG_TYPE_AEAD; + sa_req.enc = enc; + sa_req.callback = sa_aead_dma_in_callback; + sa_req.mdata_size = 52; + sa_req.base = &req->base; + sa_req.ctx = ctx; + sa_req.src = req->src; + sa_req.dst = req->dst; + + return sa_run(&sa_req); +} + +/* AEAD algorithm encrypt interface function */ +static int sa_aead_encrypt(struct aead_request *req) +{ + return sa_aead_run(req, req->iv, 1); +} + +/* AEAD algorithm decrypt interface function */ +static int sa_aead_decrypt(struct aead_request *req) +{ + return sa_aead_run(req, req->iv, 0); +} + +static struct sa_alg_tmpl sa_algs[] = { + [SA_ALG_CBC_AES] = { + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-sa2ul", + .base.cra_priority = 30000, + .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct sa_tfm_ctx), + .base.cra_module = THIS_MODULE, + .init = sa_cipher_cra_init, + .exit = sa_cipher_cra_exit, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = sa_aes_cbc_setkey, + .encrypt = sa_encrypt, + .decrypt = sa_decrypt, + } + }, + [SA_ALG_EBC_AES] = { + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-sa2ul", + .base.cra_priority = 30000, + .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct sa_tfm_ctx), + .base.cra_module = THIS_MODULE, + .init = sa_cipher_cra_init, + .exit = sa_cipher_cra_exit, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = sa_aes_ecb_setkey, + .encrypt = sa_encrypt, + .decrypt = sa_decrypt, + } + }, + [SA_ALG_CBC_DES3] = { + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .base.cra_name = "cbc(des3_ede)", + .base.cra_driver_name = "cbc-des3-sa2ul", + .base.cra_priority = 30000, + .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct sa_tfm_ctx), + .base.cra_module = THIS_MODULE, + .init = sa_cipher_cra_init, + .exit = sa_cipher_cra_exit, + .min_keysize = 3 * DES_KEY_SIZE, + .max_keysize = 3 * DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = sa_3des_cbc_setkey, + .encrypt = sa_encrypt, + .decrypt = sa_decrypt, + } + }, + [SA_ALG_ECB_DES3] = { + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .base.cra_name = "ecb(des3_ede)", + .base.cra_driver_name = "ecb-des3-sa2ul", + .base.cra_priority = 30000, + .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct sa_tfm_ctx), + .base.cra_module = THIS_MODULE, + .init = sa_cipher_cra_init, + .exit = sa_cipher_cra_exit, + .min_keysize = 3 * DES_KEY_SIZE, + .max_keysize = 3 * DES_KEY_SIZE, + .setkey = sa_3des_ecb_setkey, + .encrypt = sa_encrypt, + .decrypt = sa_decrypt, + } + }, + [SA_ALG_SHA1] = { + .type = CRYPTO_ALG_TYPE_AHASH, + .alg.ahash = { + .halg.base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-sa2ul", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sa_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sa_sha1_cra_init, + .cra_exit = sa_sha_cra_exit, + }, + .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.statesize = sizeof(struct sa_sha_req_ctx) + + sizeof(struct sha1_state), + .init = sa_sha_init, + .update = sa_sha_update, + .final = sa_sha_final, + .finup = sa_sha_finup, + .digest = sa_sha_digest, + .export = sa_sha_export, + .import = sa_sha_import, + }, + }, + [SA_ALG_SHA256] = { + .type = CRYPTO_ALG_TYPE_AHASH, + .alg.ahash = { + .halg.base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-sa2ul", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sa_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sa_sha256_cra_init, + .cra_exit = sa_sha_cra_exit, + }, + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.statesize = sizeof(struct sa_sha_req_ctx) + + sizeof(struct sha256_state), + .init = sa_sha_init, + .update = sa_sha_update, + .final = sa_sha_final, + .finup = sa_sha_finup, + .digest = sa_sha_digest, + .export = sa_sha_export, + .import = sa_sha_import, + }, + }, + [SA_ALG_SHA512] = { + .type = CRYPTO_ALG_TYPE_AHASH, + .alg.ahash = { + .halg.base = { + .cra_name = "sha512", + .cra_driver_name = "sha512-sa2ul", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sa_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_init = sa_sha512_cra_init, + .cra_exit = sa_sha_cra_exit, + }, + .halg.digestsize = SHA512_DIGEST_SIZE, + .halg.statesize = sizeof(struct sa_sha_req_ctx) + + sizeof(struct sha512_state), + .init = sa_sha_init, + .update = sa_sha_update, + .final = sa_sha_final, + .finup = sa_sha_finup, + .digest = sa_sha_digest, + .export = sa_sha_export, + .import = sa_sha_import, + }, + }, + [SA_ALG_AUTHENC_SHA1_AES] = { + .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = + "authenc(hmac(sha1),cbc(aes))-sa2ul", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_AEAD | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct sa_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_priority = 3000, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + + .init = sa_cra_init_aead_sha1, + .exit = sa_exit_tfm_aead, + .setkey = sa_aead_cbc_sha1_setkey, + .setauthsize = sa_aead_setauthsize, + .encrypt = sa_aead_encrypt, + .decrypt = sa_aead_decrypt, + }, + }, + [SA_ALG_AUTHENC_SHA256_AES] = { + .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = + "authenc(hmac(sha256),cbc(aes))-sa2ul", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_AEAD | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct sa_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 0, + .cra_priority = 3000, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + + .init = sa_cra_init_aead_sha256, + .exit = sa_exit_tfm_aead, + .setkey = sa_aead_cbc_sha256_setkey, + .setauthsize = sa_aead_setauthsize, + .encrypt = sa_aead_encrypt, + .decrypt = sa_aead_decrypt, + }, + }, +}; + +/* Register the algorithms in crypto framework */ +static void sa_register_algos(struct sa_crypto_data *dev_data) +{ + const struct sa_match_data *match_data = dev_data->match_data; + struct device *dev = dev_data->dev; + char *alg_name; + u32 type; + int i, err; + + for (i = 0; i < ARRAY_SIZE(sa_algs); i++) { + /* Skip unsupported algos */ + if (!(match_data->supported_algos & BIT(i))) + continue; + + type = sa_algs[i].type; + if (type == CRYPTO_ALG_TYPE_SKCIPHER) { + alg_name = sa_algs[i].alg.skcipher.base.cra_name; + err = crypto_register_skcipher(&sa_algs[i].alg.skcipher); + } else if (type == CRYPTO_ALG_TYPE_AHASH) { + alg_name = sa_algs[i].alg.ahash.halg.base.cra_name; + err = crypto_register_ahash(&sa_algs[i].alg.ahash); + } else if (type == CRYPTO_ALG_TYPE_AEAD) { + alg_name = sa_algs[i].alg.aead.base.cra_name; + err = crypto_register_aead(&sa_algs[i].alg.aead); + } else { + dev_err(dev, + "un-supported crypto algorithm (%d)", + sa_algs[i].type); + continue; + } + + if (err) + dev_err(dev, "Failed to register '%s'\n", alg_name); + else + sa_algs[i].registered = true; + } +} + +/* Unregister the algorithms in crypto framework */ +static void sa_unregister_algos(const struct device *dev) +{ + u32 type; + int i; + + for (i = 0; i < ARRAY_SIZE(sa_algs); i++) { + type = sa_algs[i].type; + if (!sa_algs[i].registered) + continue; + if (type == CRYPTO_ALG_TYPE_SKCIPHER) + crypto_unregister_skcipher(&sa_algs[i].alg.skcipher); + else if (type == CRYPTO_ALG_TYPE_AHASH) + crypto_unregister_ahash(&sa_algs[i].alg.ahash); + else if (type == CRYPTO_ALG_TYPE_AEAD) + crypto_unregister_aead(&sa_algs[i].alg.aead); + + sa_algs[i].registered = false; + } +} + +static int sa_init_mem(struct sa_crypto_data *dev_data) +{ + struct device *dev = &dev_data->pdev->dev; + /* Setup dma pool for security context buffers */ + dev_data->sc_pool = dma_pool_create("keystone-sc", dev, + SA_CTX_MAX_SZ, 64, 0); + if (!dev_data->sc_pool) { + dev_err(dev, "Failed to create dma pool"); + return -ENOMEM; + } + + return 0; +} + +static int sa_dma_init(struct sa_crypto_data *dd) +{ + int ret; + struct dma_slave_config cfg; + + dd->dma_rx1 = NULL; + dd->dma_tx = NULL; + dd->dma_rx2 = NULL; + + ret = dma_coerce_mask_and_coherent(dd->dev, DMA_BIT_MASK(48)); + if (ret) + return ret; + + dd->dma_rx1 = dma_request_chan(dd->dev, "rx1"); + if (IS_ERR(dd->dma_rx1)) + return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx1), + "Unable to request rx1 DMA channel\n"); + + dd->dma_rx2 = dma_request_chan(dd->dev, "rx2"); + if (IS_ERR(dd->dma_rx2)) { + ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx2), + "Unable to request rx2 DMA channel\n"); + goto err_dma_rx2; + } + + dd->dma_tx = dma_request_chan(dd->dev, "tx"); + if (IS_ERR(dd->dma_tx)) { + ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_tx), + "Unable to request tx DMA channel\n"); + goto err_dma_tx; + } + + memzero_explicit(&cfg, sizeof(cfg)); + + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cfg.src_maxburst = 4; + cfg.dst_maxburst = 4; + + ret = dmaengine_slave_config(dd->dma_rx1, &cfg); + if (ret) { + dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n", + ret); + goto err_dma_config; + } + + ret = dmaengine_slave_config(dd->dma_rx2, &cfg); + if (ret) { + dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n", + ret); + goto err_dma_config; + } + + ret = dmaengine_slave_config(dd->dma_tx, &cfg); + if (ret) { + dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n", + ret); + goto err_dma_config; + } + + return 0; + +err_dma_config: + dma_release_channel(dd->dma_tx); +err_dma_tx: + dma_release_channel(dd->dma_rx2); +err_dma_rx2: + dma_release_channel(dd->dma_rx1); + + return ret; +} + +static int sa_link_child(struct device *dev, void *data) +{ + struct device *parent = data; + + device_link_add(dev, parent, DL_FLAG_AUTOPROBE_CONSUMER); + + return 0; +} + +static struct sa_match_data am654_match_data = { + .priv = 1, + .priv_id = 1, + .supported_algos = BIT(SA_ALG_CBC_AES) | + BIT(SA_ALG_EBC_AES) | + BIT(SA_ALG_CBC_DES3) | + BIT(SA_ALG_ECB_DES3) | + BIT(SA_ALG_SHA1) | + BIT(SA_ALG_SHA256) | + BIT(SA_ALG_SHA512) | + BIT(SA_ALG_AUTHENC_SHA1_AES) | + BIT(SA_ALG_AUTHENC_SHA256_AES), +}; + +static struct sa_match_data am64_match_data = { + .priv = 0, + .priv_id = 0, + .supported_algos = BIT(SA_ALG_CBC_AES) | + BIT(SA_ALG_EBC_AES) | + BIT(SA_ALG_SHA256) | + BIT(SA_ALG_SHA512) | + BIT(SA_ALG_AUTHENC_SHA256_AES), +}; + +static const struct of_device_id of_match[] = { + { .compatible = "ti,j721e-sa2ul", .data = &am654_match_data, }, + { .compatible = "ti,am654-sa2ul", .data = &am654_match_data, }, + { .compatible = "ti,am64-sa2ul", .data = &am64_match_data, }, + { .compatible = "ti,am62-sa3ul", .data = &am64_match_data, }, + {}, +}; +MODULE_DEVICE_TABLE(of, of_match); + +static int sa_ul_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *node = dev->of_node; + static void __iomem *saul_base; + struct sa_crypto_data *dev_data; + u32 status, val; + int ret; + + dev_data = devm_kzalloc(dev, sizeof(*dev_data), GFP_KERNEL); + if (!dev_data) + return -ENOMEM; + + dev_data->match_data = of_device_get_match_data(dev); + if (!dev_data->match_data) + return -ENODEV; + + saul_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(saul_base)) + return PTR_ERR(saul_base); + + sa_k3_dev = dev; + dev_data->dev = dev; + dev_data->pdev = pdev; + dev_data->base = saul_base; + platform_set_drvdata(pdev, dev_data); + dev_set_drvdata(sa_k3_dev, dev_data); + + pm_runtime_enable(dev); + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) { + dev_err(dev, "%s: failed to get sync: %d\n", __func__, ret); + pm_runtime_disable(dev); + return ret; + } + + ret = sa_init_mem(dev_data); + if (ret) + goto disable_pm; + + ret = sa_dma_init(dev_data); + if (ret) + goto destroy_dma_pool; + + spin_lock_init(&dev_data->scid_lock); + + val = SA_EEC_ENCSS_EN | SA_EEC_AUTHSS_EN | SA_EEC_CTXCACH_EN | + SA_EEC_CPPI_PORT_IN_EN | SA_EEC_CPPI_PORT_OUT_EN | + SA_EEC_TRNG_EN; + status = readl_relaxed(saul_base + SA_ENGINE_STATUS); + /* Only enable engines if all are not already enabled */ + if (val & ~status) + writel_relaxed(val, saul_base + SA_ENGINE_ENABLE_CONTROL); + + sa_register_algos(dev_data); + + ret = of_platform_populate(node, NULL, NULL, dev); + if (ret) + goto release_dma; + + device_for_each_child(dev, dev, sa_link_child); + + return 0; + +release_dma: + sa_unregister_algos(dev); + + dma_release_channel(dev_data->dma_rx2); + dma_release_channel(dev_data->dma_rx1); + dma_release_channel(dev_data->dma_tx); + +destroy_dma_pool: + dma_pool_destroy(dev_data->sc_pool); + +disable_pm: + pm_runtime_put_sync(dev); + pm_runtime_disable(dev); + + return ret; +} + +static void sa_ul_remove(struct platform_device *pdev) +{ + struct sa_crypto_data *dev_data = platform_get_drvdata(pdev); + + of_platform_depopulate(&pdev->dev); + + sa_unregister_algos(&pdev->dev); + + dma_release_channel(dev_data->dma_rx2); + dma_release_channel(dev_data->dma_rx1); + dma_release_channel(dev_data->dma_tx); + + dma_pool_destroy(dev_data->sc_pool); + + platform_set_drvdata(pdev, NULL); + + pm_runtime_put_sync(&pdev->dev); + pm_runtime_disable(&pdev->dev); +} + +static struct platform_driver sa_ul_driver = { + .probe = sa_ul_probe, + .remove = sa_ul_remove, + .driver = { + .name = "saul-crypto", + .of_match_table = of_match, + }, +}; +module_platform_driver(sa_ul_driver); +MODULE_DESCRIPTION("K3 SA2UL crypto accelerator driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/crypto/sa2ul.h b/drivers/crypto/sa2ul.h new file mode 100644 index 000000000..12c17a68d --- /dev/null +++ b/drivers/crypto/sa2ul.h @@ -0,0 +1,407 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * K3 SA2UL crypto accelerator driver + * + * Copyright (C) 2018-2020 Texas Instruments Incorporated - http://www.ti.com + * + * Authors: Keerthy + * Vitaly Andrianov + * Tero Kristo + */ + +#ifndef _K3_SA2UL_ +#define _K3_SA2UL_ + +#include +#include +#include + +#define SA_ENGINE_STATUS 0x0008 +#define SA_ENGINE_ENABLE_CONTROL 0x1000 + +struct sa_tfm_ctx; +/* + * SA_ENGINE_ENABLE_CONTROL register bits + */ +#define SA_EEC_ENCSS_EN 0x00000001 +#define SA_EEC_AUTHSS_EN 0x00000002 +#define SA_EEC_TRNG_EN 0x00000008 +#define SA_EEC_PKA_EN 0x00000010 +#define SA_EEC_CTXCACH_EN 0x00000080 +#define SA_EEC_CPPI_PORT_IN_EN 0x00000200 +#define SA_EEC_CPPI_PORT_OUT_EN 0x00000800 + +/* + * Encoding used to identify the typo of crypto operation + * performed on the packet when the packet is returned + * by SA + */ +#define SA_REQ_SUBTYPE_ENC 0x0001 +#define SA_REQ_SUBTYPE_DEC 0x0002 +#define SA_REQ_SUBTYPE_SHIFT 16 +#define SA_REQ_SUBTYPE_MASK 0xffff + +/* Number of 32 bit words in EPIB */ +#define SA_DMA_NUM_EPIB_WORDS 4 + +/* Number of 32 bit words in PS data */ +#define SA_DMA_NUM_PS_WORDS 16 +#define NKEY_SZ 3 +#define MCI_SZ 27 + +/* + * Maximum number of simultaeneous security contexts + * supported by the driver + */ +#define SA_MAX_NUM_CTX 512 + +/* + * Assumption: CTX size is multiple of 32 + */ +#define SA_CTX_SIZE_TO_DMA_SIZE(ctx_sz) \ + ((ctx_sz) ? ((ctx_sz) / 32 - 1) : 0) + +#define SA_CTX_ENC_KEY_OFFSET 32 +#define SA_CTX_ENC_AUX1_OFFSET 64 +#define SA_CTX_ENC_AUX2_OFFSET 96 +#define SA_CTX_ENC_AUX3_OFFSET 112 +#define SA_CTX_ENC_AUX4_OFFSET 128 + +/* Next Engine Select code in CP_ACE */ +#define SA_ENG_ID_EM1 2 /* Enc/Dec engine with AES/DEC core */ +#define SA_ENG_ID_EM2 3 /* Encryption/Decryption enginefor pass 2 */ +#define SA_ENG_ID_AM1 4 /* Auth. engine with SHA1/MD5/SHA2 core */ +#define SA_ENG_ID_AM2 5 /* Authentication engine for pass 2 */ +#define SA_ENG_ID_OUTPORT2 20 /* Egress module 2 */ + +/* + * Command Label Definitions + */ +#define SA_CMDL_OFFSET_NESC 0 /* Next Engine Select Code */ +#define SA_CMDL_OFFSET_LABEL_LEN 1 /* Engine Command Label Length */ +/* 16-bit Length of Data to be processed */ +#define SA_CMDL_OFFSET_DATA_LEN 2 +#define SA_CMDL_OFFSET_DATA_OFFSET 4 /* Stat Data Offset */ +#define SA_CMDL_OFFSET_OPTION_CTRL1 5 /* Option Control Byte 1 */ +#define SA_CMDL_OFFSET_OPTION_CTRL2 6 /* Option Control Byte 2 */ +#define SA_CMDL_OFFSET_OPTION_CTRL3 7 /* Option Control Byte 3 */ +#define SA_CMDL_OFFSET_OPTION_BYTE 8 + +#define SA_CMDL_HEADER_SIZE_BYTES 8 + +#define SA_CMDL_OPTION_BYTES_MAX_SIZE 72 +#define SA_CMDL_MAX_SIZE_BYTES (SA_CMDL_HEADER_SIZE_BYTES + \ + SA_CMDL_OPTION_BYTES_MAX_SIZE) + +/* SWINFO word-0 flags */ +#define SA_SW_INFO_FLAG_EVICT 0x0001 +#define SA_SW_INFO_FLAG_TEAR 0x0002 +#define SA_SW_INFO_FLAG_NOPD 0x0004 + +/* + * This type represents the various packet types to be processed + * by the PHP engine in SA. + * It is used to identify the corresponding PHP processing function. + */ +#define SA_CTX_PE_PKT_TYPE_3GPP_AIR 0 /* 3GPP Air Cipher */ +#define SA_CTX_PE_PKT_TYPE_SRTP 1 /* SRTP */ +#define SA_CTX_PE_PKT_TYPE_IPSEC_AH 2 /* IPSec Authentication Header */ +/* IPSec Encapsulating Security Payload */ +#define SA_CTX_PE_PKT_TYPE_IPSEC_ESP 3 +/* Indicates that it is in data mode, It may not be used by PHP */ +#define SA_CTX_PE_PKT_TYPE_NONE 4 +#define SA_CTX_ENC_TYPE1_SZ 64 /* Encryption SC with Key only */ +#define SA_CTX_ENC_TYPE2_SZ 96 /* Encryption SC with Key and Aux1 */ + +#define SA_CTX_AUTH_TYPE1_SZ 64 /* Auth SC with Key only */ +#define SA_CTX_AUTH_TYPE2_SZ 96 /* Auth SC with Key and Aux1 */ +/* Size of security context for PHP engine */ +#define SA_CTX_PHP_PE_CTX_SZ 64 + +#define SA_CTX_MAX_SZ (64 + SA_CTX_ENC_TYPE2_SZ + SA_CTX_AUTH_TYPE2_SZ) + +/* + * Encoding of F/E control in SCCTL + * Bit 0-1: Fetch PHP Bytes + * Bit 2-3: Fetch Encryption/Air Ciphering Bytes + * Bit 4-5: Fetch Authentication Bytes or Encr pass 2 + * Bit 6-7: Evict PHP Bytes + * + * where 00 = 0 bytes + * 01 = 64 bytes + * 10 = 96 bytes + * 11 = 128 bytes + */ +#define SA_CTX_DMA_SIZE_0 0 +#define SA_CTX_DMA_SIZE_64 1 +#define SA_CTX_DMA_SIZE_96 2 +#define SA_CTX_DMA_SIZE_128 3 + +/* + * Byte offset of the owner word in SCCTL + * in the security context + */ +#define SA_CTX_SCCTL_OWNER_OFFSET 0 + +#define SA_CTX_ENC_KEY_OFFSET 32 +#define SA_CTX_ENC_AUX1_OFFSET 64 +#define SA_CTX_ENC_AUX2_OFFSET 96 +#define SA_CTX_ENC_AUX3_OFFSET 112 +#define SA_CTX_ENC_AUX4_OFFSET 128 + +#define SA_SCCTL_FE_AUTH_ENC 0x65 +#define SA_SCCTL_FE_ENC 0x8D + +#define SA_ALIGN_MASK (sizeof(u32) - 1) +#define SA_ALIGNED __aligned(32) + +#define SA_AUTH_SW_CTRL_MD5 1 +#define SA_AUTH_SW_CTRL_SHA1 2 +#define SA_AUTH_SW_CTRL_SHA224 3 +#define SA_AUTH_SW_CTRL_SHA256 4 +#define SA_AUTH_SW_CTRL_SHA384 5 +#define SA_AUTH_SW_CTRL_SHA512 6 + +/* SA2UL can only handle maximum data size of 64KB */ +#define SA_MAX_DATA_SZ U16_MAX + +/* + * SA2UL can provide unpredictable results with packet sizes that fall + * the following range, so avoid using it. + */ +#define SA_UNSAFE_DATA_SZ_MIN 240 +#define SA_UNSAFE_DATA_SZ_MAX 255 + +struct sa_match_data; + +/** + * struct sa_crypto_data - Crypto driver instance data + * @base: Base address of the register space + * @soc_data: Pointer to SoC specific data + * @pdev: Platform device pointer + * @sc_pool: security context pool + * @dev: Device pointer + * @scid_lock: secure context ID lock + * @sc_id_start: starting index for SC ID + * @sc_id_end: Ending index for SC ID + * @sc_id: Security Context ID + * @ctx_bm: Bitmap to keep track of Security context ID's + * @ctx: SA tfm context pointer + * @dma_rx1: Pointer to DMA rx channel for sizes < 256 Bytes + * @dma_rx2: Pointer to DMA rx channel for sizes > 256 Bytes + * @dma_tx: Pointer to DMA TX channel + */ +struct sa_crypto_data { + void __iomem *base; + const struct sa_match_data *match_data; + struct platform_device *pdev; + struct dma_pool *sc_pool; + struct device *dev; + spinlock_t scid_lock; /* lock for SC-ID allocation */ + /* Security context data */ + u16 sc_id_start; + u16 sc_id_end; + u16 sc_id; + unsigned long ctx_bm[DIV_ROUND_UP(SA_MAX_NUM_CTX, + BITS_PER_LONG)]; + struct sa_tfm_ctx *ctx; + struct dma_chan *dma_rx1; + struct dma_chan *dma_rx2; + struct dma_chan *dma_tx; +}; + +/** + * struct sa_cmdl_param_info: Command label parameters info + * @index: Index of the parameter in the command label format + * @offset: the offset of the parameter + * @size: Size of the parameter + */ +struct sa_cmdl_param_info { + u16 index; + u16 offset; + u16 size; +}; + +/* Maximum length of Auxiliary data in 32bit words */ +#define SA_MAX_AUX_DATA_WORDS 8 + +/** + * struct sa_cmdl_upd_info: Command label updation info + * @flags: flags in command label + * @submode: Encryption submodes + * @enc_size: Size of first pass encryption size + * @enc_size2: Size of second pass encryption size + * @enc_offset: Encryption payload offset in the packet + * @enc_iv: Encryption initialization vector for pass2 + * @enc_iv2: Encryption initialization vector for pass2 + * @aad: Associated data + * @payload: Payload info + * @auth_size: Authentication size for pass 1 + * @auth_size2: Authentication size for pass 2 + * @auth_offset: Authentication payload offset + * @auth_iv: Authentication initialization vector + * @aux_key_info: Authentication aux key information + * @aux_key: Aux key for authentication + */ +struct sa_cmdl_upd_info { + u16 flags; + u16 submode; + struct sa_cmdl_param_info enc_size; + struct sa_cmdl_param_info enc_size2; + struct sa_cmdl_param_info enc_offset; + struct sa_cmdl_param_info enc_iv; + struct sa_cmdl_param_info enc_iv2; + struct sa_cmdl_param_info aad; + struct sa_cmdl_param_info payload; + struct sa_cmdl_param_info auth_size; + struct sa_cmdl_param_info auth_size2; + struct sa_cmdl_param_info auth_offset; + struct sa_cmdl_param_info auth_iv; + struct sa_cmdl_param_info aux_key_info; + u32 aux_key[SA_MAX_AUX_DATA_WORDS]; +}; + +/* + * Number of 32bit words appended after the command label + * in PSDATA to identify the crypto request context. + * word-0: Request type + * word-1: pointer to request + */ +#define SA_PSDATA_CTX_WORDS 4 + +/* Maximum size of Command label in 32 words */ +#define SA_MAX_CMDL_WORDS (SA_DMA_NUM_PS_WORDS - SA_PSDATA_CTX_WORDS) + +/** + * struct sa_ctx_info: SA context information + * @sc: Pointer to security context + * @sc_phys: Security context physical address that is passed on to SA2UL + * @sc_id: Security context ID + * @cmdl_size: Command label size + * @cmdl: Command label for a particular iteration + * @cmdl_upd_info: structure holding command label updation info + * @epib: Extended protocol information block words + */ +struct sa_ctx_info { + u8 *sc; + dma_addr_t sc_phys; + u16 sc_id; + u16 cmdl_size; + u32 cmdl[SA_MAX_CMDL_WORDS]; + struct sa_cmdl_upd_info cmdl_upd_info; + /* Store Auxiliary data such as K2/K3 subkeys in AES-XCBC */ + u32 epib[SA_DMA_NUM_EPIB_WORDS]; +}; + +/** + * struct sa_tfm_ctx: TFM context structure + * @dev_data: struct sa_crypto_data pointer + * @enc: struct sa_ctx_info for encryption + * @dec: struct sa_ctx_info for decryption + * @keylen: encrption/decryption keylength + * @iv_idx: Initialization vector index + * @key: encryption key + * @fallback: SW fallback algorithm + */ +struct sa_tfm_ctx { + struct sa_crypto_data *dev_data; + struct sa_ctx_info enc; + struct sa_ctx_info dec; + struct sa_ctx_info auth; + int keylen; + int iv_idx; + u32 key[AES_KEYSIZE_256 / sizeof(u32)]; + u8 authkey[SHA512_BLOCK_SIZE]; + struct crypto_shash *shash; + /* for fallback */ + union { + struct crypto_skcipher *skcipher; + struct crypto_ahash *ahash; + struct crypto_aead *aead; + } fallback; +}; + +/** + * struct sa_sha_req_ctx: Structure used for sha request + * @dev_data: struct sa_crypto_data pointer + * @cmdl: Complete command label with psdata and epib included + * @fallback_req: SW fallback request container + */ +struct sa_sha_req_ctx { + struct sa_crypto_data *dev_data; + u32 cmdl[SA_MAX_CMDL_WORDS + SA_PSDATA_CTX_WORDS]; + struct ahash_request fallback_req; +}; + +enum sa_submode { + SA_MODE_GEN = 0, + SA_MODE_CCM, + SA_MODE_GCM, + SA_MODE_GMAC +}; + +/* Encryption algorithms */ +enum sa_ealg_id { + SA_EALG_ID_NONE = 0, /* No encryption */ + SA_EALG_ID_NULL, /* NULL encryption */ + SA_EALG_ID_AES_CTR, /* AES Counter mode */ + SA_EALG_ID_AES_F8, /* AES F8 mode */ + SA_EALG_ID_AES_CBC, /* AES CBC mode */ + SA_EALG_ID_DES_CBC, /* DES CBC mode */ + SA_EALG_ID_3DES_CBC, /* 3DES CBC mode */ + SA_EALG_ID_CCM, /* Counter with CBC-MAC mode */ + SA_EALG_ID_GCM, /* Galois Counter mode */ + SA_EALG_ID_AES_ECB, + SA_EALG_ID_LAST +}; + +/* Authentication algorithms */ +enum sa_aalg_id { + SA_AALG_ID_NONE = 0, /* No Authentication */ + SA_AALG_ID_NULL = SA_EALG_ID_LAST, /* NULL Authentication */ + SA_AALG_ID_MD5, /* MD5 mode */ + SA_AALG_ID_SHA1, /* SHA1 mode */ + SA_AALG_ID_SHA2_224, /* 224-bit SHA2 mode */ + SA_AALG_ID_SHA2_256, /* 256-bit SHA2 mode */ + SA_AALG_ID_SHA2_512, /* 512-bit SHA2 mode */ + SA_AALG_ID_HMAC_MD5, /* HMAC with MD5 mode */ + SA_AALG_ID_HMAC_SHA1, /* HMAC with SHA1 mode */ + SA_AALG_ID_HMAC_SHA2_224, /* HMAC with 224-bit SHA2 mode */ + SA_AALG_ID_HMAC_SHA2_256, /* HMAC with 256-bit SHA2 mode */ + SA_AALG_ID_GMAC, /* Galois Message Auth. Code mode */ + SA_AALG_ID_CMAC, /* Cipher-based Mes. Auth. Code mode */ + SA_AALG_ID_CBC_MAC, /* Cipher Block Chaining */ + SA_AALG_ID_AES_XCBC /* AES Extended Cipher Block Chaining */ +}; + +/* + * Mode control engine algorithms used to index the + * mode control instruction tables + */ +enum sa_eng_algo_id { + SA_ENG_ALGO_ECB = 0, + SA_ENG_ALGO_CBC, + SA_ENG_ALGO_CFB, + SA_ENG_ALGO_OFB, + SA_ENG_ALGO_CTR, + SA_ENG_ALGO_F8, + SA_ENG_ALGO_F8F9, + SA_ENG_ALGO_GCM, + SA_ENG_ALGO_GMAC, + SA_ENG_ALGO_CCM, + SA_ENG_ALGO_CMAC, + SA_ENG_ALGO_CBCMAC, + SA_NUM_ENG_ALGOS +}; + +/** + * struct sa_eng_info: Security accelerator engine info + * @eng_id: Engine ID + * @sc_size: security context size + */ +struct sa_eng_info { + u8 eng_id; + u16 sc_size; +}; + +#endif /* _K3_SA2UL_ */ diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c new file mode 100644 index 000000000..d586f1315 --- /dev/null +++ b/drivers/crypto/sahara.c @@ -0,0 +1,1435 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Cryptographic API. + * + * Support for SAHARA cryptographic accelerator. + * + * Copyright (c) 2014 Steffen Trumtrar + * Copyright (c) 2013 Vista Silicon S.L. + * Author: Javier Martin + * + * Based on omap-aes.c and tegra-aes.c + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SHA_BUFFER_LEN PAGE_SIZE +#define SAHARA_MAX_SHA_BLOCK_SIZE SHA256_BLOCK_SIZE + +#define SAHARA_NAME "sahara" +#define SAHARA_VERSION_3 3 +#define SAHARA_VERSION_4 4 +#define SAHARA_TIMEOUT_MS 1000 +#define SAHARA_MAX_HW_DESC 2 +#define SAHARA_MAX_HW_LINK 20 + +#define FLAGS_MODE_MASK 0x000f +#define FLAGS_ENCRYPT BIT(0) +#define FLAGS_CBC BIT(1) + +#define SAHARA_HDR_BASE 0x00800000 +#define SAHARA_HDR_SKHA_ALG_AES 0 +#define SAHARA_HDR_SKHA_MODE_ECB 0 +#define SAHARA_HDR_SKHA_OP_ENC BIT(2) +#define SAHARA_HDR_SKHA_MODE_CBC BIT(3) +#define SAHARA_HDR_FORM_DATA (5 << 16) +#define SAHARA_HDR_FORM_KEY BIT(19) +#define SAHARA_HDR_LLO BIT(24) +#define SAHARA_HDR_CHA_SKHA BIT(28) +#define SAHARA_HDR_CHA_MDHA BIT(29) +#define SAHARA_HDR_PARITY_BIT BIT(31) + +#define SAHARA_HDR_MDHA_SET_MODE_MD_KEY 0x20880000 +#define SAHARA_HDR_MDHA_SET_MODE_HASH 0x208D0000 +#define SAHARA_HDR_MDHA_HASH 0xA0850000 +#define SAHARA_HDR_MDHA_STORE_DIGEST 0x20820000 +#define SAHARA_HDR_MDHA_ALG_SHA1 0 +#define SAHARA_HDR_MDHA_ALG_MD5 1 +#define SAHARA_HDR_MDHA_ALG_SHA256 2 +#define SAHARA_HDR_MDHA_ALG_SHA224 3 +#define SAHARA_HDR_MDHA_PDATA BIT(2) +#define SAHARA_HDR_MDHA_HMAC BIT(3) +#define SAHARA_HDR_MDHA_INIT BIT(5) +#define SAHARA_HDR_MDHA_IPAD BIT(6) +#define SAHARA_HDR_MDHA_OPAD BIT(7) +#define SAHARA_HDR_MDHA_SWAP BIT(8) +#define SAHARA_HDR_MDHA_MAC_FULL BIT(9) +#define SAHARA_HDR_MDHA_SSL BIT(10) + +#define SAHARA_REG_VERSION 0x00 +#define SAHARA_REG_DAR 0x04 +#define SAHARA_REG_CONTROL 0x08 +#define SAHARA_CONTROL_SET_THROTTLE(x) (((x) & 0xff) << 24) +#define SAHARA_CONTROL_SET_MAXBURST(x) (((x) & 0xff) << 16) +#define SAHARA_CONTROL_RNG_AUTORSD BIT(7) +#define SAHARA_CONTROL_ENABLE_INT BIT(4) +#define SAHARA_REG_CMD 0x0C +#define SAHARA_CMD_RESET BIT(0) +#define SAHARA_CMD_CLEAR_INT BIT(8) +#define SAHARA_CMD_CLEAR_ERR BIT(9) +#define SAHARA_CMD_SINGLE_STEP BIT(10) +#define SAHARA_CMD_MODE_BATCH BIT(16) +#define SAHARA_CMD_MODE_DEBUG BIT(18) +#define SAHARA_REG_STATUS 0x10 +#define SAHARA_STATUS_GET_STATE(x) ((x) & 0x7) +#define SAHARA_STATE_IDLE 0 +#define SAHARA_STATE_BUSY 1 +#define SAHARA_STATE_ERR 2 +#define SAHARA_STATE_FAULT 3 +#define SAHARA_STATE_COMPLETE 4 +#define SAHARA_STATE_COMP_FLAG BIT(2) +#define SAHARA_STATUS_DAR_FULL BIT(3) +#define SAHARA_STATUS_ERROR BIT(4) +#define SAHARA_STATUS_SECURE BIT(5) +#define SAHARA_STATUS_FAIL BIT(6) +#define SAHARA_STATUS_INIT BIT(7) +#define SAHARA_STATUS_RNG_RESEED BIT(8) +#define SAHARA_STATUS_ACTIVE_RNG BIT(9) +#define SAHARA_STATUS_ACTIVE_MDHA BIT(10) +#define SAHARA_STATUS_ACTIVE_SKHA BIT(11) +#define SAHARA_STATUS_MODE_BATCH BIT(16) +#define SAHARA_STATUS_MODE_DEDICATED BIT(17) +#define SAHARA_STATUS_MODE_DEBUG BIT(18) +#define SAHARA_STATUS_GET_ISTATE(x) (((x) >> 24) & 0xff) +#define SAHARA_REG_ERRSTATUS 0x14 +#define SAHARA_ERRSTATUS_GET_SOURCE(x) ((x) & 0xf) +#define SAHARA_ERRSOURCE_CHA 14 +#define SAHARA_ERRSOURCE_DMA 15 +#define SAHARA_ERRSTATUS_DMA_DIR BIT(8) +#define SAHARA_ERRSTATUS_GET_DMASZ(x) (((x) >> 9) & 0x3) +#define SAHARA_ERRSTATUS_GET_DMASRC(x) (((x) >> 13) & 0x7) +#define SAHARA_ERRSTATUS_GET_CHASRC(x) (((x) >> 16) & 0xfff) +#define SAHARA_ERRSTATUS_GET_CHAERR(x) (((x) >> 28) & 0x3) +#define SAHARA_REG_FADDR 0x18 +#define SAHARA_REG_CDAR 0x1C +#define SAHARA_REG_IDAR 0x20 + +struct sahara_hw_desc { + u32 hdr; + u32 len1; + u32 p1; + u32 len2; + u32 p2; + u32 next; +}; + +struct sahara_hw_link { + u32 len; + u32 p; + u32 next; +}; + +struct sahara_ctx { + /* AES-specific context */ + int keylen; + u8 key[AES_KEYSIZE_128]; + struct crypto_skcipher *fallback; +}; + +struct sahara_aes_reqctx { + unsigned long mode; + u8 iv_out[AES_BLOCK_SIZE]; + struct skcipher_request fallback_req; // keep at the end +}; + +/* + * struct sahara_sha_reqctx - private data per request + * @buf: holds data for requests smaller than block_size + * @rembuf: used to prepare one block_size-aligned request + * @context: hw-specific context for request. Digest is extracted from this + * @mode: specifies what type of hw-descriptor needs to be built + * @digest_size: length of digest for this request + * @context_size: length of hw-context for this request. + * Always digest_size + 4 + * @buf_cnt: number of bytes saved in buf + * @sg_in_idx: number of hw links + * @in_sg: scatterlist for input data + * @in_sg_chain: scatterlists for chained input data + * @total: total number of bytes for transfer + * @last: is this the last block + * @first: is this the first block + */ +struct sahara_sha_reqctx { + u8 buf[SAHARA_MAX_SHA_BLOCK_SIZE]; + u8 rembuf[SAHARA_MAX_SHA_BLOCK_SIZE]; + u8 context[SHA256_DIGEST_SIZE + 4]; + unsigned int mode; + unsigned int digest_size; + unsigned int context_size; + unsigned int buf_cnt; + unsigned int sg_in_idx; + struct scatterlist *in_sg; + struct scatterlist in_sg_chain[2]; + size_t total; + unsigned int last; + unsigned int first; +}; + +struct sahara_dev { + struct device *device; + unsigned int version; + void __iomem *regs_base; + struct clk *clk_ipg; + struct clk *clk_ahb; + struct completion dma_completion; + + struct sahara_ctx *ctx; + unsigned long flags; + + struct sahara_hw_desc *hw_desc[SAHARA_MAX_HW_DESC]; + dma_addr_t hw_phys_desc[SAHARA_MAX_HW_DESC]; + + u8 *key_base; + dma_addr_t key_phys_base; + + u8 *iv_base; + dma_addr_t iv_phys_base; + + u8 *context_base; + dma_addr_t context_phys_base; + + struct sahara_hw_link *hw_link[SAHARA_MAX_HW_LINK]; + dma_addr_t hw_phys_link[SAHARA_MAX_HW_LINK]; + + size_t total; + struct scatterlist *in_sg; + int nb_in_sg; + struct scatterlist *out_sg; + int nb_out_sg; + + struct crypto_engine *engine; +}; + +static struct sahara_dev *dev_ptr; + +static inline void sahara_write(struct sahara_dev *dev, u32 data, u32 reg) +{ + writel(data, dev->regs_base + reg); +} + +static inline unsigned int sahara_read(struct sahara_dev *dev, u32 reg) +{ + return readl(dev->regs_base + reg); +} + +static u32 sahara_aes_key_hdr(struct sahara_dev *dev) +{ + u32 hdr = SAHARA_HDR_BASE | SAHARA_HDR_SKHA_ALG_AES | + SAHARA_HDR_FORM_KEY | SAHARA_HDR_LLO | + SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT; + + if (dev->flags & FLAGS_CBC) { + hdr |= SAHARA_HDR_SKHA_MODE_CBC; + hdr ^= SAHARA_HDR_PARITY_BIT; + } + + if (dev->flags & FLAGS_ENCRYPT) { + hdr |= SAHARA_HDR_SKHA_OP_ENC; + hdr ^= SAHARA_HDR_PARITY_BIT; + } + + return hdr; +} + +static u32 sahara_aes_data_link_hdr(struct sahara_dev *dev) +{ + return SAHARA_HDR_BASE | SAHARA_HDR_FORM_DATA | + SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT; +} + +static const char *sahara_err_src[16] = { + "No error", + "Header error", + "Descriptor length error", + "Descriptor length or pointer error", + "Link length error", + "Link pointer error", + "Input buffer error", + "Output buffer error", + "Output buffer starvation", + "Internal state fault", + "General descriptor problem", + "Reserved", + "Descriptor address error", + "Link address error", + "CHA error", + "DMA error" +}; + +static const char *sahara_err_dmasize[4] = { + "Byte transfer", + "Half-word transfer", + "Word transfer", + "Reserved" +}; + +static const char *sahara_err_dmasrc[8] = { + "No error", + "AHB bus error", + "Internal IP bus error", + "Parity error", + "DMA crosses 256 byte boundary", + "DMA is busy", + "Reserved", + "DMA HW error" +}; + +static const char *sahara_cha_errsrc[12] = { + "Input buffer non-empty", + "Illegal address", + "Illegal mode", + "Illegal data size", + "Illegal key size", + "Write during processing", + "CTX read during processing", + "HW error", + "Input buffer disabled/underflow", + "Output buffer disabled/overflow", + "DES key parity error", + "Reserved" +}; + +static const char *sahara_cha_err[4] = { "No error", "SKHA", "MDHA", "RNG" }; + +static void sahara_decode_error(struct sahara_dev *dev, unsigned int error) +{ + u8 source = SAHARA_ERRSTATUS_GET_SOURCE(error); + u16 chasrc = ffs(SAHARA_ERRSTATUS_GET_CHASRC(error)); + + dev_err(dev->device, "%s: Error Register = 0x%08x\n", __func__, error); + + dev_err(dev->device, " - %s.\n", sahara_err_src[source]); + + if (source == SAHARA_ERRSOURCE_DMA) { + if (error & SAHARA_ERRSTATUS_DMA_DIR) + dev_err(dev->device, " * DMA read.\n"); + else + dev_err(dev->device, " * DMA write.\n"); + + dev_err(dev->device, " * %s.\n", + sahara_err_dmasize[SAHARA_ERRSTATUS_GET_DMASZ(error)]); + dev_err(dev->device, " * %s.\n", + sahara_err_dmasrc[SAHARA_ERRSTATUS_GET_DMASRC(error)]); + } else if (source == SAHARA_ERRSOURCE_CHA) { + dev_err(dev->device, " * %s.\n", + sahara_cha_errsrc[chasrc]); + dev_err(dev->device, " * %s.\n", + sahara_cha_err[SAHARA_ERRSTATUS_GET_CHAERR(error)]); + } + dev_err(dev->device, "\n"); +} + +static const char *sahara_state[4] = { "Idle", "Busy", "Error", "HW Fault" }; + +static void sahara_decode_status(struct sahara_dev *dev, unsigned int status) +{ + u8 state; + + if (!__is_defined(DEBUG)) + return; + + state = SAHARA_STATUS_GET_STATE(status); + + dev_dbg(dev->device, "%s: Status Register = 0x%08x\n", + __func__, status); + + dev_dbg(dev->device, " - State = %d:\n", state); + if (state & SAHARA_STATE_COMP_FLAG) + dev_dbg(dev->device, " * Descriptor completed. IRQ pending.\n"); + + dev_dbg(dev->device, " * %s.\n", + sahara_state[state & ~SAHARA_STATE_COMP_FLAG]); + + if (status & SAHARA_STATUS_DAR_FULL) + dev_dbg(dev->device, " - DAR Full.\n"); + if (status & SAHARA_STATUS_ERROR) + dev_dbg(dev->device, " - Error.\n"); + if (status & SAHARA_STATUS_SECURE) + dev_dbg(dev->device, " - Secure.\n"); + if (status & SAHARA_STATUS_FAIL) + dev_dbg(dev->device, " - Fail.\n"); + if (status & SAHARA_STATUS_RNG_RESEED) + dev_dbg(dev->device, " - RNG Reseed Request.\n"); + if (status & SAHARA_STATUS_ACTIVE_RNG) + dev_dbg(dev->device, " - RNG Active.\n"); + if (status & SAHARA_STATUS_ACTIVE_MDHA) + dev_dbg(dev->device, " - MDHA Active.\n"); + if (status & SAHARA_STATUS_ACTIVE_SKHA) + dev_dbg(dev->device, " - SKHA Active.\n"); + + if (status & SAHARA_STATUS_MODE_BATCH) + dev_dbg(dev->device, " - Batch Mode.\n"); + else if (status & SAHARA_STATUS_MODE_DEDICATED) + dev_dbg(dev->device, " - Dedicated Mode.\n"); + else if (status & SAHARA_STATUS_MODE_DEBUG) + dev_dbg(dev->device, " - Debug Mode.\n"); + + dev_dbg(dev->device, " - Internal state = 0x%02x\n", + SAHARA_STATUS_GET_ISTATE(status)); + + dev_dbg(dev->device, "Current DAR: 0x%08x\n", + sahara_read(dev, SAHARA_REG_CDAR)); + dev_dbg(dev->device, "Initial DAR: 0x%08x\n\n", + sahara_read(dev, SAHARA_REG_IDAR)); +} + +static void sahara_dump_descriptors(struct sahara_dev *dev) +{ + int i; + + if (!__is_defined(DEBUG)) + return; + + for (i = 0; i < SAHARA_MAX_HW_DESC; i++) { + dev_dbg(dev->device, "Descriptor (%d) (%pad):\n", + i, &dev->hw_phys_desc[i]); + dev_dbg(dev->device, "\thdr = 0x%08x\n", dev->hw_desc[i]->hdr); + dev_dbg(dev->device, "\tlen1 = %u\n", dev->hw_desc[i]->len1); + dev_dbg(dev->device, "\tp1 = 0x%08x\n", dev->hw_desc[i]->p1); + dev_dbg(dev->device, "\tlen2 = %u\n", dev->hw_desc[i]->len2); + dev_dbg(dev->device, "\tp2 = 0x%08x\n", dev->hw_desc[i]->p2); + dev_dbg(dev->device, "\tnext = 0x%08x\n", + dev->hw_desc[i]->next); + } + dev_dbg(dev->device, "\n"); +} + +static void sahara_dump_links(struct sahara_dev *dev) +{ + int i; + + if (!__is_defined(DEBUG)) + return; + + for (i = 0; i < SAHARA_MAX_HW_LINK; i++) { + dev_dbg(dev->device, "Link (%d) (%pad):\n", + i, &dev->hw_phys_link[i]); + dev_dbg(dev->device, "\tlen = %u\n", dev->hw_link[i]->len); + dev_dbg(dev->device, "\tp = 0x%08x\n", dev->hw_link[i]->p); + dev_dbg(dev->device, "\tnext = 0x%08x\n", + dev->hw_link[i]->next); + } + dev_dbg(dev->device, "\n"); +} + +static int sahara_hw_descriptor_create(struct sahara_dev *dev) +{ + struct sahara_ctx *ctx = dev->ctx; + struct scatterlist *sg; + int ret; + int i, j; + int idx = 0; + u32 len; + + memcpy(dev->key_base, ctx->key, ctx->keylen); + + if (dev->flags & FLAGS_CBC) { + dev->hw_desc[idx]->len1 = AES_BLOCK_SIZE; + dev->hw_desc[idx]->p1 = dev->iv_phys_base; + } else { + dev->hw_desc[idx]->len1 = 0; + dev->hw_desc[idx]->p1 = 0; + } + dev->hw_desc[idx]->len2 = ctx->keylen; + dev->hw_desc[idx]->p2 = dev->key_phys_base; + dev->hw_desc[idx]->next = dev->hw_phys_desc[1]; + dev->hw_desc[idx]->hdr = sahara_aes_key_hdr(dev); + + idx++; + + + dev->nb_in_sg = sg_nents_for_len(dev->in_sg, dev->total); + if (dev->nb_in_sg < 0) { + dev_err(dev->device, "Invalid numbers of src SG.\n"); + return dev->nb_in_sg; + } + dev->nb_out_sg = sg_nents_for_len(dev->out_sg, dev->total); + if (dev->nb_out_sg < 0) { + dev_err(dev->device, "Invalid numbers of dst SG.\n"); + return dev->nb_out_sg; + } + if ((dev->nb_in_sg + dev->nb_out_sg) > SAHARA_MAX_HW_LINK) { + dev_err(dev->device, "not enough hw links (%d)\n", + dev->nb_in_sg + dev->nb_out_sg); + return -EINVAL; + } + + ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg, + DMA_TO_DEVICE); + if (!ret) { + dev_err(dev->device, "couldn't map in sg\n"); + return -EINVAL; + } + + ret = dma_map_sg(dev->device, dev->out_sg, dev->nb_out_sg, + DMA_FROM_DEVICE); + if (!ret) { + dev_err(dev->device, "couldn't map out sg\n"); + goto unmap_in; + } + + /* Create input links */ + dev->hw_desc[idx]->p1 = dev->hw_phys_link[0]; + sg = dev->in_sg; + len = dev->total; + for (i = 0; i < dev->nb_in_sg; i++) { + dev->hw_link[i]->len = min(len, sg->length); + dev->hw_link[i]->p = sg->dma_address; + if (i == (dev->nb_in_sg - 1)) { + dev->hw_link[i]->next = 0; + } else { + len -= min(len, sg->length); + dev->hw_link[i]->next = dev->hw_phys_link[i + 1]; + sg = sg_next(sg); + } + } + + /* Create output links */ + dev->hw_desc[idx]->p2 = dev->hw_phys_link[i]; + sg = dev->out_sg; + len = dev->total; + for (j = i; j < dev->nb_out_sg + i; j++) { + dev->hw_link[j]->len = min(len, sg->length); + dev->hw_link[j]->p = sg->dma_address; + if (j == (dev->nb_out_sg + i - 1)) { + dev->hw_link[j]->next = 0; + } else { + len -= min(len, sg->length); + dev->hw_link[j]->next = dev->hw_phys_link[j + 1]; + sg = sg_next(sg); + } + } + + /* Fill remaining fields of hw_desc[1] */ + dev->hw_desc[idx]->hdr = sahara_aes_data_link_hdr(dev); + dev->hw_desc[idx]->len1 = dev->total; + dev->hw_desc[idx]->len2 = dev->total; + dev->hw_desc[idx]->next = 0; + + sahara_dump_descriptors(dev); + sahara_dump_links(dev); + + sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR); + + return 0; + +unmap_in: + dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, + DMA_TO_DEVICE); + + return -EINVAL; +} + +static void sahara_aes_cbc_update_iv(struct skcipher_request *req) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req); + unsigned int ivsize = crypto_skcipher_ivsize(skcipher); + + /* Update IV buffer to contain the last ciphertext block */ + if (rctx->mode & FLAGS_ENCRYPT) { + sg_pcopy_to_buffer(req->dst, sg_nents(req->dst), req->iv, + ivsize, req->cryptlen - ivsize); + } else { + memcpy(req->iv, rctx->iv_out, ivsize); + } +} + +static int sahara_aes_process(struct skcipher_request *req) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + struct sahara_dev *dev = dev_ptr; + struct sahara_ctx *ctx; + struct sahara_aes_reqctx *rctx; + int ret; + unsigned long time_left; + + /* Request is ready to be dispatched by the device */ + dev_dbg(dev->device, + "dispatch request (nbytes=%d, src=%p, dst=%p)\n", + req->cryptlen, req->src, req->dst); + + /* assign new request to device */ + dev->total = req->cryptlen; + dev->in_sg = req->src; + dev->out_sg = req->dst; + + rctx = skcipher_request_ctx(req); + ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + rctx->mode &= FLAGS_MODE_MASK; + dev->flags = (dev->flags & ~FLAGS_MODE_MASK) | rctx->mode; + + if ((dev->flags & FLAGS_CBC) && req->iv) { + unsigned int ivsize = crypto_skcipher_ivsize(skcipher); + + memcpy(dev->iv_base, req->iv, ivsize); + + if (!(dev->flags & FLAGS_ENCRYPT)) { + sg_pcopy_to_buffer(req->src, sg_nents(req->src), + rctx->iv_out, ivsize, + req->cryptlen - ivsize); + } + } + + /* assign new context to device */ + dev->ctx = ctx; + + reinit_completion(&dev->dma_completion); + + ret = sahara_hw_descriptor_create(dev); + if (ret) + return -EINVAL; + + time_left = wait_for_completion_timeout(&dev->dma_completion, + msecs_to_jiffies(SAHARA_TIMEOUT_MS)); + + dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg, + DMA_FROM_DEVICE); + dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, + DMA_TO_DEVICE); + + if (!time_left) { + dev_err(dev->device, "AES timeout\n"); + return -ETIMEDOUT; + } + + if ((dev->flags & FLAGS_CBC) && req->iv) + sahara_aes_cbc_update_iv(req); + + return 0; +} + +static int sahara_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct sahara_ctx *ctx = crypto_skcipher_ctx(tfm); + + ctx->keylen = keylen; + + /* SAHARA only supports 128bit keys */ + if (keylen == AES_KEYSIZE_128) { + memcpy(ctx->key, key, keylen); + return 0; + } + + if (keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256) + return -EINVAL; + + /* + * The requested key size is not supported by HW, do a fallback. + */ + crypto_skcipher_clear_flags(ctx->fallback, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(ctx->fallback, tfm->base.crt_flags & + CRYPTO_TFM_REQ_MASK); + return crypto_skcipher_setkey(ctx->fallback, key, keylen); +} + +static int sahara_aes_fallback(struct skcipher_request *req, unsigned long mode) +{ + struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req); + struct sahara_ctx *ctx = crypto_skcipher_ctx( + crypto_skcipher_reqtfm(req)); + + skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback); + skcipher_request_set_callback(&rctx->fallback_req, + req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(&rctx->fallback_req, req->src, + req->dst, req->cryptlen, req->iv); + + if (mode & FLAGS_ENCRYPT) + return crypto_skcipher_encrypt(&rctx->fallback_req); + + return crypto_skcipher_decrypt(&rctx->fallback_req); +} + +static int sahara_aes_crypt(struct skcipher_request *req, unsigned long mode) +{ + struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req); + struct sahara_ctx *ctx = crypto_skcipher_ctx( + crypto_skcipher_reqtfm(req)); + struct sahara_dev *dev = dev_ptr; + + if (!req->cryptlen) + return 0; + + if (unlikely(ctx->keylen != AES_KEYSIZE_128)) + return sahara_aes_fallback(req, mode); + + dev_dbg(dev->device, "nbytes: %d, enc: %d, cbc: %d\n", + req->cryptlen, !!(mode & FLAGS_ENCRYPT), !!(mode & FLAGS_CBC)); + + if (!IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE)) + return -EINVAL; + + rctx->mode = mode; + + return crypto_transfer_skcipher_request_to_engine(dev->engine, req); +} + +static int sahara_aes_ecb_encrypt(struct skcipher_request *req) +{ + return sahara_aes_crypt(req, FLAGS_ENCRYPT); +} + +static int sahara_aes_ecb_decrypt(struct skcipher_request *req) +{ + return sahara_aes_crypt(req, 0); +} + +static int sahara_aes_cbc_encrypt(struct skcipher_request *req) +{ + return sahara_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC); +} + +static int sahara_aes_cbc_decrypt(struct skcipher_request *req) +{ + return sahara_aes_crypt(req, FLAGS_CBC); +} + +static int sahara_aes_init_tfm(struct crypto_skcipher *tfm) +{ + const char *name = crypto_tfm_alg_name(&tfm->base); + struct sahara_ctx *ctx = crypto_skcipher_ctx(tfm); + + ctx->fallback = crypto_alloc_skcipher(name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fallback)) { + pr_err("Error allocating fallback algo %s\n", name); + return PTR_ERR(ctx->fallback); + } + + crypto_skcipher_set_reqsize(tfm, sizeof(struct sahara_aes_reqctx) + + crypto_skcipher_reqsize(ctx->fallback)); + + return 0; +} + +static void sahara_aes_exit_tfm(struct crypto_skcipher *tfm) +{ + struct sahara_ctx *ctx = crypto_skcipher_ctx(tfm); + + crypto_free_skcipher(ctx->fallback); +} + +static u32 sahara_sha_init_hdr(struct sahara_dev *dev, + struct sahara_sha_reqctx *rctx) +{ + u32 hdr = 0; + + hdr = rctx->mode; + + if (rctx->first) { + hdr |= SAHARA_HDR_MDHA_SET_MODE_HASH; + hdr |= SAHARA_HDR_MDHA_INIT; + } else { + hdr |= SAHARA_HDR_MDHA_SET_MODE_MD_KEY; + } + + if (rctx->last) + hdr |= SAHARA_HDR_MDHA_PDATA; + + if (hweight_long(hdr) % 2 == 0) + hdr |= SAHARA_HDR_PARITY_BIT; + + return hdr; +} + +static int sahara_sha_hw_links_create(struct sahara_dev *dev, + struct sahara_sha_reqctx *rctx, + int start) +{ + struct scatterlist *sg; + unsigned int len; + unsigned int i; + int ret; + + dev->in_sg = rctx->in_sg; + + dev->nb_in_sg = sg_nents_for_len(dev->in_sg, rctx->total); + if (dev->nb_in_sg < 0) { + dev_err(dev->device, "Invalid numbers of src SG.\n"); + return dev->nb_in_sg; + } + if ((dev->nb_in_sg) > SAHARA_MAX_HW_LINK) { + dev_err(dev->device, "not enough hw links (%d)\n", + dev->nb_in_sg + dev->nb_out_sg); + return -EINVAL; + } + + sg = dev->in_sg; + ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg, DMA_TO_DEVICE); + if (!ret) + return -EFAULT; + + len = rctx->total; + for (i = start; i < dev->nb_in_sg + start; i++) { + dev->hw_link[i]->len = min(len, sg->length); + dev->hw_link[i]->p = sg->dma_address; + if (i == (dev->nb_in_sg + start - 1)) { + dev->hw_link[i]->next = 0; + } else { + len -= min(len, sg->length); + dev->hw_link[i]->next = dev->hw_phys_link[i + 1]; + sg = sg_next(sg); + } + } + + return i; +} + +static int sahara_sha_hw_data_descriptor_create(struct sahara_dev *dev, + struct sahara_sha_reqctx *rctx, + struct ahash_request *req, + int index) +{ + unsigned result_len; + int i = index; + + if (rctx->first) + /* Create initial descriptor: #8*/ + dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx); + else + /* Create hash descriptor: #10. Must follow #6. */ + dev->hw_desc[index]->hdr = SAHARA_HDR_MDHA_HASH; + + dev->hw_desc[index]->len1 = rctx->total; + if (dev->hw_desc[index]->len1 == 0) { + /* if len1 is 0, p1 must be 0, too */ + dev->hw_desc[index]->p1 = 0; + rctx->sg_in_idx = 0; + } else { + /* Create input links */ + dev->hw_desc[index]->p1 = dev->hw_phys_link[index]; + i = sahara_sha_hw_links_create(dev, rctx, index); + + rctx->sg_in_idx = index; + if (i < 0) + return i; + } + + dev->hw_desc[index]->p2 = dev->hw_phys_link[i]; + + /* Save the context for the next operation */ + result_len = rctx->context_size; + dev->hw_link[i]->p = dev->context_phys_base; + + dev->hw_link[i]->len = result_len; + dev->hw_desc[index]->len2 = result_len; + + dev->hw_link[i]->next = 0; + + return 0; +} + +/* + * Load descriptor aka #6 + * + * To load a previously saved context back to the MDHA unit + * + * p1: Saved Context + * p2: NULL + * + */ +static int sahara_sha_hw_context_descriptor_create(struct sahara_dev *dev, + struct sahara_sha_reqctx *rctx, + struct ahash_request *req, + int index) +{ + dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx); + + dev->hw_desc[index]->len1 = rctx->context_size; + dev->hw_desc[index]->p1 = dev->hw_phys_link[index]; + dev->hw_desc[index]->len2 = 0; + dev->hw_desc[index]->p2 = 0; + + dev->hw_link[index]->len = rctx->context_size; + dev->hw_link[index]->p = dev->context_phys_base; + dev->hw_link[index]->next = 0; + + return 0; +} + +static int sahara_sha_prepare_request(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); + unsigned int hash_later; + unsigned int block_size; + unsigned int len; + + block_size = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + + /* append bytes from previous operation */ + len = rctx->buf_cnt + req->nbytes; + + /* only the last transfer can be padded in hardware */ + if (!rctx->last && (len < block_size)) { + /* to few data, save for next operation */ + scatterwalk_map_and_copy(rctx->buf + rctx->buf_cnt, req->src, + 0, req->nbytes, 0); + rctx->buf_cnt += req->nbytes; + + return 0; + } + + /* add data from previous operation first */ + if (rctx->buf_cnt) + memcpy(rctx->rembuf, rctx->buf, rctx->buf_cnt); + + /* data must always be a multiple of block_size */ + hash_later = rctx->last ? 0 : len & (block_size - 1); + if (hash_later) { + unsigned int offset = req->nbytes - hash_later; + /* Save remaining bytes for later use */ + scatterwalk_map_and_copy(rctx->buf, req->src, offset, + hash_later, 0); + } + + rctx->total = len - hash_later; + /* have data from previous operation and current */ + if (rctx->buf_cnt && req->nbytes) { + sg_init_table(rctx->in_sg_chain, 2); + sg_set_buf(rctx->in_sg_chain, rctx->rembuf, rctx->buf_cnt); + sg_chain(rctx->in_sg_chain, 2, req->src); + rctx->in_sg = rctx->in_sg_chain; + /* only data from previous operation */ + } else if (rctx->buf_cnt) { + rctx->in_sg = rctx->in_sg_chain; + sg_init_one(rctx->in_sg, rctx->rembuf, rctx->buf_cnt); + /* no data from previous operation */ + } else { + rctx->in_sg = req->src; + } + + /* on next call, we only have the remaining data in the buffer */ + rctx->buf_cnt = hash_later; + + return -EINPROGRESS; +} + +static int sahara_sha_process(struct ahash_request *req) +{ + struct sahara_dev *dev = dev_ptr; + struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); + int ret; + unsigned long time_left; + + ret = sahara_sha_prepare_request(req); + if (!ret) + return ret; + + if (rctx->first) { + ret = sahara_sha_hw_data_descriptor_create(dev, rctx, req, 0); + if (ret) + return ret; + + dev->hw_desc[0]->next = 0; + rctx->first = 0; + } else { + memcpy(dev->context_base, rctx->context, rctx->context_size); + + sahara_sha_hw_context_descriptor_create(dev, rctx, req, 0); + dev->hw_desc[0]->next = dev->hw_phys_desc[1]; + ret = sahara_sha_hw_data_descriptor_create(dev, rctx, req, 1); + if (ret) + return ret; + + dev->hw_desc[1]->next = 0; + } + + sahara_dump_descriptors(dev); + sahara_dump_links(dev); + + reinit_completion(&dev->dma_completion); + + sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR); + + time_left = wait_for_completion_timeout(&dev->dma_completion, + msecs_to_jiffies(SAHARA_TIMEOUT_MS)); + + if (rctx->sg_in_idx) + dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, + DMA_TO_DEVICE); + + if (!time_left) { + dev_err(dev->device, "SHA timeout\n"); + return -ETIMEDOUT; + } + + memcpy(rctx->context, dev->context_base, rctx->context_size); + + if (req->result && rctx->last) + memcpy(req->result, rctx->context, rctx->digest_size); + + return 0; +} + +static int sahara_do_one_request(struct crypto_engine *engine, void *areq) +{ + struct crypto_async_request *async_req = areq; + int err; + + if (crypto_tfm_alg_type(async_req->tfm) == CRYPTO_ALG_TYPE_AHASH) { + struct ahash_request *req = ahash_request_cast(async_req); + + err = sahara_sha_process(req); + local_bh_disable(); + crypto_finalize_hash_request(engine, req, err); + local_bh_enable(); + } else { + struct skcipher_request *req = skcipher_request_cast(async_req); + + err = sahara_aes_process(skcipher_request_cast(async_req)); + local_bh_disable(); + crypto_finalize_skcipher_request(engine, req, err); + local_bh_enable(); + } + + return 0; +} + +static int sahara_sha_enqueue(struct ahash_request *req, int last) +{ + struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); + struct sahara_dev *dev = dev_ptr; + + if (!req->nbytes && !last) + return 0; + + rctx->last = last; + + return crypto_transfer_hash_request_to_engine(dev->engine, req); +} + +static int sahara_sha_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); + + memset(rctx, 0, sizeof(*rctx)); + + switch (crypto_ahash_digestsize(tfm)) { + case SHA1_DIGEST_SIZE: + rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA1; + rctx->digest_size = SHA1_DIGEST_SIZE; + break; + case SHA256_DIGEST_SIZE: + rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA256; + rctx->digest_size = SHA256_DIGEST_SIZE; + break; + default: + return -EINVAL; + } + + rctx->context_size = rctx->digest_size + 4; + rctx->first = 1; + + return 0; +} + +static int sahara_sha_update(struct ahash_request *req) +{ + return sahara_sha_enqueue(req, 0); +} + +static int sahara_sha_final(struct ahash_request *req) +{ + req->nbytes = 0; + return sahara_sha_enqueue(req, 1); +} + +static int sahara_sha_finup(struct ahash_request *req) +{ + return sahara_sha_enqueue(req, 1); +} + +static int sahara_sha_digest(struct ahash_request *req) +{ + sahara_sha_init(req); + + return sahara_sha_finup(req); +} + +static int sahara_sha_export(struct ahash_request *req, void *out) +{ + struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); + + memcpy(out, rctx, sizeof(struct sahara_sha_reqctx)); + + return 0; +} + +static int sahara_sha_import(struct ahash_request *req, const void *in) +{ + struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); + + memcpy(rctx, in, sizeof(struct sahara_sha_reqctx)); + + return 0; +} + +static int sahara_sha_cra_init(struct crypto_tfm *tfm) +{ + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct sahara_sha_reqctx)); + + return 0; +} + +static struct skcipher_engine_alg aes_algs[] = { +{ + .base = { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "sahara-ecb-aes", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct sahara_ctx), + .base.cra_alignmask = 0x0, + .base.cra_module = THIS_MODULE, + + .init = sahara_aes_init_tfm, + .exit = sahara_aes_exit_tfm, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = sahara_aes_setkey, + .encrypt = sahara_aes_ecb_encrypt, + .decrypt = sahara_aes_ecb_decrypt, + }, + .op = { + .do_one_request = sahara_do_one_request, + }, +}, { + .base = { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "sahara-cbc-aes", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct sahara_ctx), + .base.cra_alignmask = 0x0, + .base.cra_module = THIS_MODULE, + + .init = sahara_aes_init_tfm, + .exit = sahara_aes_exit_tfm, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = sahara_aes_setkey, + .encrypt = sahara_aes_cbc_encrypt, + .decrypt = sahara_aes_cbc_decrypt, + }, + .op = { + .do_one_request = sahara_do_one_request, + }, +} +}; + +static struct ahash_engine_alg sha_v3_algs[] = { +{ + .base = { + .init = sahara_sha_init, + .update = sahara_sha_update, + .final = sahara_sha_final, + .finup = sahara_sha_finup, + .digest = sahara_sha_digest, + .export = sahara_sha_export, + .import = sahara_sha_import, + .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.statesize = sizeof(struct sahara_sha_reqctx), + .halg.base = { + .cra_name = "sha1", + .cra_driver_name = "sahara-sha1", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sahara_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = sahara_sha_cra_init, + } + }, + .op = { + .do_one_request = sahara_do_one_request, + }, +}, +}; + +static struct ahash_engine_alg sha_v4_algs[] = { +{ + .base = { + .init = sahara_sha_init, + .update = sahara_sha_update, + .final = sahara_sha_final, + .finup = sahara_sha_finup, + .digest = sahara_sha_digest, + .export = sahara_sha_export, + .import = sahara_sha_import, + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.statesize = sizeof(struct sahara_sha_reqctx), + .halg.base = { + .cra_name = "sha256", + .cra_driver_name = "sahara-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sahara_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = sahara_sha_cra_init, + } + }, + .op = { + .do_one_request = sahara_do_one_request, + }, +}, +}; + +static irqreturn_t sahara_irq_handler(int irq, void *data) +{ + struct sahara_dev *dev = data; + unsigned int stat = sahara_read(dev, SAHARA_REG_STATUS); + unsigned int err = sahara_read(dev, SAHARA_REG_ERRSTATUS); + + sahara_write(dev, SAHARA_CMD_CLEAR_INT | SAHARA_CMD_CLEAR_ERR, + SAHARA_REG_CMD); + + sahara_decode_status(dev, stat); + + if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_BUSY) + return IRQ_NONE; + + if (SAHARA_STATUS_GET_STATE(stat) != SAHARA_STATE_COMPLETE) + sahara_decode_error(dev, err); + + complete(&dev->dma_completion); + + return IRQ_HANDLED; +} + + +static int sahara_register_algs(struct sahara_dev *dev) +{ + int err; + + err = crypto_engine_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); + if (err) + return err; + + err = crypto_engine_register_ahashes(sha_v3_algs, + ARRAY_SIZE(sha_v3_algs)); + if (err) + goto err_aes_algs; + + if (dev->version > SAHARA_VERSION_3) { + err = crypto_engine_register_ahashes(sha_v4_algs, + ARRAY_SIZE(sha_v4_algs)); + if (err) + goto err_sha_v3_algs; + } + + return 0; + +err_sha_v3_algs: + crypto_engine_unregister_ahashes(sha_v3_algs, ARRAY_SIZE(sha_v3_algs)); + +err_aes_algs: + crypto_engine_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); + + return err; +} + +static void sahara_unregister_algs(struct sahara_dev *dev) +{ + crypto_engine_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); + crypto_engine_unregister_ahashes(sha_v3_algs, ARRAY_SIZE(sha_v3_algs)); + + if (dev->version > SAHARA_VERSION_3) + crypto_engine_unregister_ahashes(sha_v4_algs, + ARRAY_SIZE(sha_v4_algs)); +} + +static const struct of_device_id sahara_dt_ids[] = { + { .compatible = "fsl,imx53-sahara" }, + { .compatible = "fsl,imx27-sahara" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, sahara_dt_ids); + +static int sahara_probe(struct platform_device *pdev) +{ + struct sahara_dev *dev; + u32 version; + int irq; + int err; + int i; + + dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL); + if (!dev) + return -ENOMEM; + + dev->device = &pdev->dev; + platform_set_drvdata(pdev, dev); + + /* Get the base address */ + dev->regs_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(dev->regs_base)) + return PTR_ERR(dev->regs_base); + + /* Get the IRQ */ + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + err = devm_request_irq(&pdev->dev, irq, sahara_irq_handler, + 0, dev_name(&pdev->dev), dev); + if (err) + return err; + + /* clocks */ + dev->clk_ipg = devm_clk_get_enabled(&pdev->dev, "ipg"); + if (IS_ERR(dev->clk_ipg)) + return dev_err_probe(&pdev->dev, PTR_ERR(dev->clk_ipg), + "Could not get ipg clock\n"); + + dev->clk_ahb = devm_clk_get_enabled(&pdev->dev, "ahb"); + if (IS_ERR(dev->clk_ahb)) + return dev_err_probe(&pdev->dev, PTR_ERR(dev->clk_ahb), + "Could not get ahb clock\n"); + + /* Allocate HW descriptors */ + dev->hw_desc[0] = dmam_alloc_coherent(&pdev->dev, + SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc), + &dev->hw_phys_desc[0], GFP_KERNEL); + if (!dev->hw_desc[0]) + return -ENOMEM; + dev->hw_desc[1] = dev->hw_desc[0] + 1; + dev->hw_phys_desc[1] = dev->hw_phys_desc[0] + + sizeof(struct sahara_hw_desc); + + /* Allocate space for iv and key */ + dev->key_base = dmam_alloc_coherent(&pdev->dev, 2 * AES_KEYSIZE_128, + &dev->key_phys_base, GFP_KERNEL); + if (!dev->key_base) + return -ENOMEM; + dev->iv_base = dev->key_base + AES_KEYSIZE_128; + dev->iv_phys_base = dev->key_phys_base + AES_KEYSIZE_128; + + /* Allocate space for context: largest digest + message length field */ + dev->context_base = dmam_alloc_coherent(&pdev->dev, + SHA256_DIGEST_SIZE + 4, + &dev->context_phys_base, GFP_KERNEL); + if (!dev->context_base) + return -ENOMEM; + + /* Allocate space for HW links */ + dev->hw_link[0] = dmam_alloc_coherent(&pdev->dev, + SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link), + &dev->hw_phys_link[0], GFP_KERNEL); + if (!dev->hw_link[0]) + return -ENOMEM; + for (i = 1; i < SAHARA_MAX_HW_LINK; i++) { + dev->hw_phys_link[i] = dev->hw_phys_link[i - 1] + + sizeof(struct sahara_hw_link); + dev->hw_link[i] = dev->hw_link[i - 1] + 1; + } + + dev_ptr = dev; + + dev->engine = crypto_engine_alloc_init(&pdev->dev, true); + if (!dev->engine) + return -ENOMEM; + + err = crypto_engine_start(dev->engine); + if (err) { + crypto_engine_exit(dev->engine); + return dev_err_probe(&pdev->dev, err, + "Could not start crypto engine\n"); + } + + init_completion(&dev->dma_completion); + + version = sahara_read(dev, SAHARA_REG_VERSION); + if (of_device_is_compatible(pdev->dev.of_node, "fsl,imx27-sahara")) { + if (version != SAHARA_VERSION_3) + err = -ENODEV; + } else if (of_device_is_compatible(pdev->dev.of_node, + "fsl,imx53-sahara")) { + if (((version >> 8) & 0xff) != SAHARA_VERSION_4) + err = -ENODEV; + version = (version >> 8) & 0xff; + } + if (err == -ENODEV) { + dev_err_probe(&pdev->dev, err, + "SAHARA version %d not supported\n", version); + goto err_algs; + } + + dev->version = version; + + sahara_write(dev, SAHARA_CMD_RESET | SAHARA_CMD_MODE_BATCH, + SAHARA_REG_CMD); + sahara_write(dev, SAHARA_CONTROL_SET_THROTTLE(0) | + SAHARA_CONTROL_SET_MAXBURST(8) | + SAHARA_CONTROL_RNG_AUTORSD | + SAHARA_CONTROL_ENABLE_INT, + SAHARA_REG_CONTROL); + + err = sahara_register_algs(dev); + if (err) + goto err_algs; + + dev_info(&pdev->dev, "SAHARA version %d initialized\n", version); + + return 0; + +err_algs: + crypto_engine_exit(dev->engine); + + return err; +} + +static void sahara_remove(struct platform_device *pdev) +{ + struct sahara_dev *dev = platform_get_drvdata(pdev); + + crypto_engine_exit(dev->engine); + sahara_unregister_algs(dev); +} + +static struct platform_driver sahara_driver = { + .probe = sahara_probe, + .remove = sahara_remove, + .driver = { + .name = SAHARA_NAME, + .of_match_table = sahara_dt_ids, + }, +}; + +module_platform_driver(sahara_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Javier Martin "); +MODULE_AUTHOR("Steffen Trumtrar "); +MODULE_DESCRIPTION("SAHARA2 HW crypto accelerator"); diff --git a/drivers/crypto/starfive/Kconfig b/drivers/crypto/starfive/Kconfig new file mode 100644 index 000000000..11518ca3e --- /dev/null +++ b/drivers/crypto/starfive/Kconfig @@ -0,0 +1,26 @@ +# +# StarFive crypto drivers configuration +# + +config CRYPTO_DEV_JH7110 + tristate "StarFive JH7110 cryptographic engine driver" + depends on (SOC_STARFIVE && AMBA_PL08X) || COMPILE_TEST + depends on HAS_DMA + select CRYPTO_ENGINE + select CRYPTO_HMAC + select CRYPTO_SHA256 + select CRYPTO_SHA512 + select CRYPTO_SM3 + select CRYPTO_RSA + select CRYPTO_AES + select CRYPTO_CCM + select CRYPTO_GCM + select CRYPTO_ECB + select CRYPTO_CBC + select CRYPTO_CTR + help + Support for StarFive JH7110 crypto hardware acceleration engine. + This module provides acceleration for public key algo, + skciphers, AEAD and hash functions. + + If you choose 'M' here, this module will be called jh7110-crypto. diff --git a/drivers/crypto/starfive/Makefile b/drivers/crypto/starfive/Makefile new file mode 100644 index 000000000..8c137afe5 --- /dev/null +++ b/drivers/crypto/starfive/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_CRYPTO_DEV_JH7110) += jh7110-crypto.o +jh7110-crypto-objs := jh7110-cryp.o jh7110-hash.o jh7110-rsa.o jh7110-aes.o diff --git a/drivers/crypto/starfive/jh7110-aes.c b/drivers/crypto/starfive/jh7110-aes.c new file mode 100644 index 000000000..f59adb2a5 --- /dev/null +++ b/drivers/crypto/starfive/jh7110-aes.c @@ -0,0 +1,1155 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * StarFive AES acceleration driver + * + * Copyright (c) 2022 StarFive Technology + */ + +#include +#include +#include +#include +#include +#include "jh7110-cryp.h" +#include +#include +#include +#include +#include + +#define STARFIVE_AES_REGS_OFFSET 0x100 +#define STARFIVE_AES_AESDIO0R (STARFIVE_AES_REGS_OFFSET + 0x0) +#define STARFIVE_AES_KEY0 (STARFIVE_AES_REGS_OFFSET + 0x4) +#define STARFIVE_AES_KEY1 (STARFIVE_AES_REGS_OFFSET + 0x8) +#define STARFIVE_AES_KEY2 (STARFIVE_AES_REGS_OFFSET + 0xC) +#define STARFIVE_AES_KEY3 (STARFIVE_AES_REGS_OFFSET + 0x10) +#define STARFIVE_AES_KEY4 (STARFIVE_AES_REGS_OFFSET + 0x14) +#define STARFIVE_AES_KEY5 (STARFIVE_AES_REGS_OFFSET + 0x18) +#define STARFIVE_AES_KEY6 (STARFIVE_AES_REGS_OFFSET + 0x1C) +#define STARFIVE_AES_KEY7 (STARFIVE_AES_REGS_OFFSET + 0x20) +#define STARFIVE_AES_CSR (STARFIVE_AES_REGS_OFFSET + 0x24) +#define STARFIVE_AES_IV0 (STARFIVE_AES_REGS_OFFSET + 0x28) +#define STARFIVE_AES_IV1 (STARFIVE_AES_REGS_OFFSET + 0x2C) +#define STARFIVE_AES_IV2 (STARFIVE_AES_REGS_OFFSET + 0x30) +#define STARFIVE_AES_IV3 (STARFIVE_AES_REGS_OFFSET + 0x34) +#define STARFIVE_AES_NONCE0 (STARFIVE_AES_REGS_OFFSET + 0x3C) +#define STARFIVE_AES_NONCE1 (STARFIVE_AES_REGS_OFFSET + 0x40) +#define STARFIVE_AES_NONCE2 (STARFIVE_AES_REGS_OFFSET + 0x44) +#define STARFIVE_AES_NONCE3 (STARFIVE_AES_REGS_OFFSET + 0x48) +#define STARFIVE_AES_ALEN0 (STARFIVE_AES_REGS_OFFSET + 0x4C) +#define STARFIVE_AES_ALEN1 (STARFIVE_AES_REGS_OFFSET + 0x50) +#define STARFIVE_AES_MLEN0 (STARFIVE_AES_REGS_OFFSET + 0x54) +#define STARFIVE_AES_MLEN1 (STARFIVE_AES_REGS_OFFSET + 0x58) +#define STARFIVE_AES_IVLEN (STARFIVE_AES_REGS_OFFSET + 0x5C) + +#define FLG_MODE_MASK GENMASK(2, 0) +#define FLG_ENCRYPT BIT(4) + +/* Misc */ +#define CCM_B0_ADATA 0x40 +#define AES_BLOCK_32 (AES_BLOCK_SIZE / sizeof(u32)) + +static inline int starfive_aes_wait_busy(struct starfive_cryp_dev *cryp) +{ + u32 status; + + return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status, + !(status & STARFIVE_AES_BUSY), 10, 100000); +} + +static inline int starfive_aes_wait_keydone(struct starfive_cryp_dev *cryp) +{ + u32 status; + + return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status, + (status & STARFIVE_AES_KEY_DONE), 10, 100000); +} + +static inline int starfive_aes_wait_gcmdone(struct starfive_cryp_dev *cryp) +{ + u32 status; + + return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status, + (status & STARFIVE_AES_GCM_DONE), 10, 100000); +} + +static inline int is_gcm(struct starfive_cryp_dev *cryp) +{ + return (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_GCM; +} + +static inline bool is_encrypt(struct starfive_cryp_dev *cryp) +{ + return cryp->flags & FLG_ENCRYPT; +} + +static void starfive_aes_aead_hw_start(struct starfive_cryp_ctx *ctx, u32 hw_mode) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + unsigned int value; + + switch (hw_mode) { + case STARFIVE_AES_MODE_GCM: + value = readl(ctx->cryp->base + STARFIVE_AES_CSR); + value |= STARFIVE_AES_GCM_START; + writel(value, cryp->base + STARFIVE_AES_CSR); + starfive_aes_wait_gcmdone(cryp); + break; + case STARFIVE_AES_MODE_CCM: + value = readl(ctx->cryp->base + STARFIVE_AES_CSR); + value |= STARFIVE_AES_CCM_START; + writel(value, cryp->base + STARFIVE_AES_CSR); + break; + } +} + +static inline void starfive_aes_set_alen(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + + writel(upper_32_bits(cryp->assoclen), cryp->base + STARFIVE_AES_ALEN0); + writel(lower_32_bits(cryp->assoclen), cryp->base + STARFIVE_AES_ALEN1); +} + +static inline void starfive_aes_set_mlen(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + + writel(upper_32_bits(cryp->total_in), cryp->base + STARFIVE_AES_MLEN0); + writel(lower_32_bits(cryp->total_in), cryp->base + STARFIVE_AES_MLEN1); +} + +static inline int starfive_aes_ccm_check_iv(const u8 *iv) +{ + /* 2 <= L <= 8, so 1 <= L' <= 7. */ + if (iv[0] < 1 || iv[0] > 7) + return -EINVAL; + + return 0; +} + +static int starfive_aes_write_iv(struct starfive_cryp_ctx *ctx, u32 *iv) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + + writel(iv[0], cryp->base + STARFIVE_AES_IV0); + writel(iv[1], cryp->base + STARFIVE_AES_IV1); + writel(iv[2], cryp->base + STARFIVE_AES_IV2); + + if (is_gcm(cryp)) { + if (starfive_aes_wait_gcmdone(cryp)) + return -ETIMEDOUT; + + return 0; + } + + writel(iv[3], cryp->base + STARFIVE_AES_IV3); + + return 0; +} + +static inline void starfive_aes_get_iv(struct starfive_cryp_dev *cryp, u32 *iv) +{ + iv[0] = readl(cryp->base + STARFIVE_AES_IV0); + iv[1] = readl(cryp->base + STARFIVE_AES_IV1); + iv[2] = readl(cryp->base + STARFIVE_AES_IV2); + iv[3] = readl(cryp->base + STARFIVE_AES_IV3); +} + +static inline void starfive_aes_write_nonce(struct starfive_cryp_ctx *ctx, u32 *nonce) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + + writel(nonce[0], cryp->base + STARFIVE_AES_NONCE0); + writel(nonce[1], cryp->base + STARFIVE_AES_NONCE1); + writel(nonce[2], cryp->base + STARFIVE_AES_NONCE2); + writel(nonce[3], cryp->base + STARFIVE_AES_NONCE3); +} + +static int starfive_aes_write_key(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + u32 *key = (u32 *)ctx->key; + + if (ctx->keylen >= AES_KEYSIZE_128) { + writel(key[0], cryp->base + STARFIVE_AES_KEY0); + writel(key[1], cryp->base + STARFIVE_AES_KEY1); + writel(key[2], cryp->base + STARFIVE_AES_KEY2); + writel(key[3], cryp->base + STARFIVE_AES_KEY3); + } + + if (ctx->keylen >= AES_KEYSIZE_192) { + writel(key[4], cryp->base + STARFIVE_AES_KEY4); + writel(key[5], cryp->base + STARFIVE_AES_KEY5); + } + + if (ctx->keylen >= AES_KEYSIZE_256) { + writel(key[6], cryp->base + STARFIVE_AES_KEY6); + writel(key[7], cryp->base + STARFIVE_AES_KEY7); + } + + if (starfive_aes_wait_keydone(cryp)) + return -ETIMEDOUT; + + return 0; +} + +static int starfive_aes_ccm_init(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + u8 iv[AES_BLOCK_SIZE], b0[AES_BLOCK_SIZE]; + unsigned int textlen; + + memcpy(iv, cryp->req.areq->iv, AES_BLOCK_SIZE); + memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1); + + /* Build B0 */ + memcpy(b0, iv, AES_BLOCK_SIZE); + + b0[0] |= (8 * ((cryp->authsize - 2) / 2)); + + if (cryp->assoclen) + b0[0] |= CCM_B0_ADATA; + + textlen = cryp->total_in; + + b0[AES_BLOCK_SIZE - 2] = textlen >> 8; + b0[AES_BLOCK_SIZE - 1] = textlen & 0xFF; + + starfive_aes_write_nonce(ctx, (u32 *)b0); + + return 0; +} + +static int starfive_aes_hw_init(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_request_ctx *rctx = ctx->rctx; + struct starfive_cryp_dev *cryp = ctx->cryp; + u32 hw_mode; + + /* reset */ + rctx->csr.aes.v = 0; + rctx->csr.aes.aesrst = 1; + writel(rctx->csr.aes.v, cryp->base + STARFIVE_AES_CSR); + + /* csr setup */ + hw_mode = cryp->flags & FLG_MODE_MASK; + + rctx->csr.aes.v = 0; + + switch (ctx->keylen) { + case AES_KEYSIZE_128: + rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_128; + break; + case AES_KEYSIZE_192: + rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_192; + break; + case AES_KEYSIZE_256: + rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_256; + break; + } + + rctx->csr.aes.mode = hw_mode; + rctx->csr.aes.cmode = !is_encrypt(cryp); + rctx->csr.aes.stmode = STARFIVE_AES_MODE_XFB_1; + + if (cryp->side_chan) { + rctx->csr.aes.delay_aes = 1; + rctx->csr.aes.vaes_start = 1; + } + + writel(rctx->csr.aes.v, cryp->base + STARFIVE_AES_CSR); + + cryp->err = starfive_aes_write_key(ctx); + if (cryp->err) + return cryp->err; + + switch (hw_mode) { + case STARFIVE_AES_MODE_GCM: + starfive_aes_set_alen(ctx); + starfive_aes_set_mlen(ctx); + writel(GCM_AES_IV_SIZE, cryp->base + STARFIVE_AES_IVLEN); + starfive_aes_aead_hw_start(ctx, hw_mode); + starfive_aes_write_iv(ctx, (void *)cryp->req.areq->iv); + break; + case STARFIVE_AES_MODE_CCM: + starfive_aes_set_alen(ctx); + starfive_aes_set_mlen(ctx); + starfive_aes_ccm_init(ctx); + starfive_aes_aead_hw_start(ctx, hw_mode); + break; + case STARFIVE_AES_MODE_CBC: + case STARFIVE_AES_MODE_CTR: + starfive_aes_write_iv(ctx, (void *)cryp->req.sreq->iv); + break; + default: + break; + } + + return cryp->err; +} + +static int starfive_aes_read_authtag(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + struct starfive_cryp_request_ctx *rctx = ctx->rctx; + int i; + + if (starfive_aes_wait_busy(cryp)) + return dev_err_probe(cryp->dev, -ETIMEDOUT, + "Timeout waiting for tag generation."); + + if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_GCM) { + cryp->tag_out[0] = readl(cryp->base + STARFIVE_AES_NONCE0); + cryp->tag_out[1] = readl(cryp->base + STARFIVE_AES_NONCE1); + cryp->tag_out[2] = readl(cryp->base + STARFIVE_AES_NONCE2); + cryp->tag_out[3] = readl(cryp->base + STARFIVE_AES_NONCE3); + } else { + for (i = 0; i < AES_BLOCK_32; i++) + cryp->tag_out[i] = readl(cryp->base + STARFIVE_AES_AESDIO0R); + } + + if (is_encrypt(cryp)) { + scatterwalk_map_and_copy(cryp->tag_out, rctx->out_sg, + cryp->total_in, cryp->authsize, 1); + } else { + if (crypto_memneq(cryp->tag_in, cryp->tag_out, cryp->authsize)) + return -EBADMSG; + } + + return 0; +} + +static void starfive_aes_finish_req(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + int err = cryp->err; + + if (!err && cryp->authsize) + err = starfive_aes_read_authtag(ctx); + + if (!err && ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC || + (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CTR)) + starfive_aes_get_iv(cryp, (void *)cryp->req.sreq->iv); + + if (cryp->authsize) + crypto_finalize_aead_request(cryp->engine, cryp->req.areq, err); + else + crypto_finalize_skcipher_request(cryp->engine, cryp->req.sreq, + err); +} + +static int starfive_aes_gcm_write_adata(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + struct starfive_cryp_request_ctx *rctx = ctx->rctx; + u32 *buffer; + int total_len, loop; + + total_len = ALIGN(cryp->assoclen, AES_BLOCK_SIZE) / sizeof(unsigned int); + buffer = (u32 *)rctx->adata; + + for (loop = 0; loop < total_len; loop += 4) { + writel(*buffer, cryp->base + STARFIVE_AES_NONCE0); + buffer++; + writel(*buffer, cryp->base + STARFIVE_AES_NONCE1); + buffer++; + writel(*buffer, cryp->base + STARFIVE_AES_NONCE2); + buffer++; + writel(*buffer, cryp->base + STARFIVE_AES_NONCE3); + buffer++; + } + + if (starfive_aes_wait_gcmdone(cryp)) + return dev_err_probe(cryp->dev, -ETIMEDOUT, + "Timeout processing gcm aad block"); + + return 0; +} + +static int starfive_aes_ccm_write_adata(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + struct starfive_cryp_request_ctx *rctx = ctx->rctx; + u32 *buffer; + u8 *ci; + int total_len, loop; + + total_len = cryp->assoclen; + + ci = rctx->adata; + writeb(*ci, cryp->base + STARFIVE_AES_AESDIO0R); + ci++; + writeb(*ci, cryp->base + STARFIVE_AES_AESDIO0R); + ci++; + total_len -= 2; + buffer = (u32 *)ci; + + for (loop = 0; loop < 3; loop++, buffer++) + writel(*buffer, cryp->base + STARFIVE_AES_AESDIO0R); + + total_len -= 12; + + while (total_len > 0) { + for (loop = 0; loop < AES_BLOCK_32; loop++, buffer++) + writel(*buffer, cryp->base + STARFIVE_AES_AESDIO0R); + + total_len -= AES_BLOCK_SIZE; + } + + if (starfive_aes_wait_busy(cryp)) + return dev_err_probe(cryp->dev, -ETIMEDOUT, + "Timeout processing ccm aad block"); + + return 0; +} + +static void starfive_aes_dma_done(void *param) +{ + struct starfive_cryp_dev *cryp = param; + + complete(&cryp->dma_done); +} + +static void starfive_aes_dma_init(struct starfive_cryp_dev *cryp) +{ + cryp->cfg_in.direction = DMA_MEM_TO_DEV; + cryp->cfg_in.src_addr_width = DMA_SLAVE_BUSWIDTH_16_BYTES; + cryp->cfg_in.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cryp->cfg_in.src_maxburst = cryp->dma_maxburst; + cryp->cfg_in.dst_maxburst = cryp->dma_maxburst; + cryp->cfg_in.dst_addr = cryp->phys_base + STARFIVE_ALG_FIFO_OFFSET; + + dmaengine_slave_config(cryp->tx, &cryp->cfg_in); + + cryp->cfg_out.direction = DMA_DEV_TO_MEM; + cryp->cfg_out.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cryp->cfg_out.dst_addr_width = DMA_SLAVE_BUSWIDTH_16_BYTES; + cryp->cfg_out.src_maxburst = 4; + cryp->cfg_out.dst_maxburst = 4; + cryp->cfg_out.src_addr = cryp->phys_base + STARFIVE_ALG_FIFO_OFFSET; + + dmaengine_slave_config(cryp->rx, &cryp->cfg_out); + + init_completion(&cryp->dma_done); +} + +static int starfive_aes_dma_xfer(struct starfive_cryp_dev *cryp, + struct scatterlist *src, + struct scatterlist *dst, + int len) +{ + struct dma_async_tx_descriptor *in_desc, *out_desc; + union starfive_alg_cr alg_cr; + int ret = 0, in_save, out_save; + + alg_cr.v = 0; + alg_cr.start = 1; + alg_cr.aes_dma_en = 1; + writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET); + + in_save = sg_dma_len(src); + out_save = sg_dma_len(dst); + + writel(ALIGN(len, AES_BLOCK_SIZE), cryp->base + STARFIVE_DMA_IN_LEN_OFFSET); + writel(ALIGN(len, AES_BLOCK_SIZE), cryp->base + STARFIVE_DMA_OUT_LEN_OFFSET); + + sg_dma_len(src) = ALIGN(len, AES_BLOCK_SIZE); + sg_dma_len(dst) = ALIGN(len, AES_BLOCK_SIZE); + + out_desc = dmaengine_prep_slave_sg(cryp->rx, dst, 1, DMA_DEV_TO_MEM, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!out_desc) { + ret = -EINVAL; + goto dma_err; + } + + out_desc->callback = starfive_aes_dma_done; + out_desc->callback_param = cryp; + + reinit_completion(&cryp->dma_done); + dmaengine_submit(out_desc); + dma_async_issue_pending(cryp->rx); + + in_desc = dmaengine_prep_slave_sg(cryp->tx, src, 1, DMA_MEM_TO_DEV, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!in_desc) { + ret = -EINVAL; + goto dma_err; + } + + dmaengine_submit(in_desc); + dma_async_issue_pending(cryp->tx); + + if (!wait_for_completion_timeout(&cryp->dma_done, + msecs_to_jiffies(1000))) + ret = -ETIMEDOUT; + +dma_err: + sg_dma_len(src) = in_save; + sg_dma_len(dst) = out_save; + + alg_cr.v = 0; + alg_cr.clear = 1; + writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET); + + return ret; +} + +static int starfive_aes_map_sg(struct starfive_cryp_dev *cryp, + struct scatterlist *src, + struct scatterlist *dst) +{ + struct scatterlist *stsg, *dtsg; + struct scatterlist _src[2], _dst[2]; + unsigned int remain = cryp->total_in; + unsigned int len, src_nents, dst_nents; + int ret; + + if (src == dst) { + for (stsg = src, dtsg = dst; remain > 0; + stsg = sg_next(stsg), dtsg = sg_next(dtsg)) { + src_nents = dma_map_sg(cryp->dev, stsg, 1, DMA_BIDIRECTIONAL); + if (src_nents == 0) + return -ENOMEM; + + dst_nents = src_nents; + len = min(sg_dma_len(stsg), remain); + + ret = starfive_aes_dma_xfer(cryp, stsg, dtsg, len); + dma_unmap_sg(cryp->dev, stsg, 1, DMA_BIDIRECTIONAL); + if (ret) + return ret; + + remain -= len; + } + } else { + for (stsg = src, dtsg = dst;;) { + src_nents = dma_map_sg(cryp->dev, stsg, 1, DMA_TO_DEVICE); + if (src_nents == 0) + return -ENOMEM; + + dst_nents = dma_map_sg(cryp->dev, dtsg, 1, DMA_FROM_DEVICE); + if (dst_nents == 0) + return -ENOMEM; + + len = min(sg_dma_len(stsg), sg_dma_len(dtsg)); + len = min(len, remain); + + ret = starfive_aes_dma_xfer(cryp, stsg, dtsg, len); + dma_unmap_sg(cryp->dev, stsg, 1, DMA_TO_DEVICE); + dma_unmap_sg(cryp->dev, dtsg, 1, DMA_FROM_DEVICE); + if (ret) + return ret; + + remain -= len; + if (remain == 0) + break; + + if (sg_dma_len(stsg) - len) { + stsg = scatterwalk_ffwd(_src, stsg, len); + dtsg = sg_next(dtsg); + } else if (sg_dma_len(dtsg) - len) { + dtsg = scatterwalk_ffwd(_dst, dtsg, len); + stsg = sg_next(stsg); + } else { + stsg = sg_next(stsg); + dtsg = sg_next(dtsg); + } + } + } + + return 0; +} + +static int starfive_aes_do_one_req(struct crypto_engine *engine, void *areq) +{ + struct skcipher_request *req = + container_of(areq, struct skcipher_request, base); + struct starfive_cryp_ctx *ctx = + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + struct starfive_cryp_request_ctx *rctx = skcipher_request_ctx(req); + struct starfive_cryp_dev *cryp = ctx->cryp; + int ret; + + cryp->req.sreq = req; + cryp->total_in = req->cryptlen; + cryp->total_out = req->cryptlen; + cryp->assoclen = 0; + cryp->authsize = 0; + + rctx->in_sg = req->src; + rctx->out_sg = req->dst; + + ctx->rctx = rctx; + + ret = starfive_aes_hw_init(ctx); + if (ret) + return ret; + + if (!cryp->total_in) + goto finish_req; + + starfive_aes_dma_init(cryp); + + ret = starfive_aes_map_sg(cryp, rctx->in_sg, rctx->out_sg); + if (ret) + return ret; + +finish_req: + starfive_aes_finish_req(ctx); + + return 0; +} + +static int starfive_aes_init_tfm(struct crypto_skcipher *tfm, + const char *alg_name) +{ + struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm); + + ctx->cryp = starfive_cryp_find_dev(ctx); + if (!ctx->cryp) + return -ENODEV; + + ctx->skcipher_fbk = crypto_alloc_skcipher(alg_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->skcipher_fbk)) + return dev_err_probe(ctx->cryp->dev, PTR_ERR(ctx->skcipher_fbk), + "%s() failed to allocate fallback for %s\n", + __func__, alg_name); + + crypto_skcipher_set_reqsize(tfm, sizeof(struct starfive_cryp_request_ctx) + + crypto_skcipher_reqsize(ctx->skcipher_fbk)); + + return 0; +} + +static void starfive_aes_exit_tfm(struct crypto_skcipher *tfm) +{ + struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm); + + crypto_free_skcipher(ctx->skcipher_fbk); +} + +static int starfive_aes_aead_do_one_req(struct crypto_engine *engine, void *areq) +{ + struct aead_request *req = + container_of(areq, struct aead_request, base); + struct starfive_cryp_ctx *ctx = + crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct starfive_cryp_dev *cryp = ctx->cryp; + struct starfive_cryp_request_ctx *rctx = aead_request_ctx(req); + struct scatterlist _src[2], _dst[2]; + int ret; + + cryp->req.areq = req; + cryp->assoclen = req->assoclen; + cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(req)); + + rctx->in_sg = scatterwalk_ffwd(_src, req->src, cryp->assoclen); + if (req->src == req->dst) + rctx->out_sg = rctx->in_sg; + else + rctx->out_sg = scatterwalk_ffwd(_dst, req->dst, cryp->assoclen); + + if (is_encrypt(cryp)) { + cryp->total_in = req->cryptlen; + cryp->total_out = req->cryptlen; + } else { + cryp->total_in = req->cryptlen - cryp->authsize; + cryp->total_out = cryp->total_in; + scatterwalk_map_and_copy(cryp->tag_in, req->src, + cryp->total_in + cryp->assoclen, + cryp->authsize, 0); + } + + if (cryp->assoclen) { + rctx->adata = kzalloc(cryp->assoclen + AES_BLOCK_SIZE, GFP_KERNEL); + if (!rctx->adata) + return -ENOMEM; + + if (sg_copy_to_buffer(req->src, sg_nents_for_len(req->src, cryp->assoclen), + rctx->adata, cryp->assoclen) != cryp->assoclen) { + kfree(rctx->adata); + return -EINVAL; + } + } + + if (cryp->total_in) + sg_zero_buffer(rctx->in_sg, sg_nents(rctx->in_sg), + rctx->in_sg->length - cryp->total_in, + cryp->total_in); + + ctx->rctx = rctx; + + ret = starfive_aes_hw_init(ctx); + if (ret) { + if (cryp->assoclen) + kfree(rctx->adata); + return ret; + } + + if (!cryp->assoclen) + goto write_text; + + if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CCM) + ret = starfive_aes_ccm_write_adata(ctx); + else + ret = starfive_aes_gcm_write_adata(ctx); + + kfree(rctx->adata); + + if (ret) + return ret; + +write_text: + if (!cryp->total_in) + goto finish_req; + + starfive_aes_dma_init(cryp); + + ret = starfive_aes_map_sg(cryp, rctx->in_sg, rctx->out_sg); + if (ret) + return ret; + +finish_req: + starfive_aes_finish_req(ctx); + return 0; +} + +static int starfive_aes_aead_init_tfm(struct crypto_aead *tfm, + const char *alg_name) +{ + struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm); + + ctx->cryp = starfive_cryp_find_dev(ctx); + if (!ctx->cryp) + return -ENODEV; + + ctx->aead_fbk = crypto_alloc_aead(alg_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->aead_fbk)) + return dev_err_probe(ctx->cryp->dev, PTR_ERR(ctx->aead_fbk), + "%s() failed to allocate fallback for %s\n", + __func__, alg_name); + + crypto_aead_set_reqsize(tfm, sizeof(struct starfive_cryp_request_ctx) + + crypto_aead_reqsize(ctx->aead_fbk)); + + return 0; +} + +static void starfive_aes_aead_exit_tfm(struct crypto_aead *tfm) +{ + struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm); + + crypto_free_aead(ctx->aead_fbk); +} + +static bool starfive_aes_check_unaligned(struct starfive_cryp_dev *cryp, + struct scatterlist *src, + struct scatterlist *dst) +{ + struct scatterlist *tsg; + int i; + + for_each_sg(src, tsg, sg_nents(src), i) + if (!IS_ALIGNED(tsg->offset, sizeof(u32)) || + (!IS_ALIGNED(tsg->length, AES_BLOCK_SIZE) && + !sg_is_last(tsg))) + return true; + + if (src != dst) + for_each_sg(dst, tsg, sg_nents(dst), i) + if (!IS_ALIGNED(tsg->offset, sizeof(u32)) || + (!IS_ALIGNED(tsg->length, AES_BLOCK_SIZE) && + !sg_is_last(tsg))) + return true; + + return false; +} + +static int starfive_aes_do_fallback(struct skcipher_request *req, bool enc) +{ + struct starfive_cryp_ctx *ctx = + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + struct skcipher_request *subreq = skcipher_request_ctx(req); + + skcipher_request_set_tfm(subreq, ctx->skcipher_fbk); + skcipher_request_set_callback(subreq, req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + + return enc ? crypto_skcipher_encrypt(subreq) : + crypto_skcipher_decrypt(subreq); +} + +static int starfive_aes_crypt(struct skcipher_request *req, unsigned long flags) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm); + struct starfive_cryp_dev *cryp = ctx->cryp; + unsigned int blocksize_align = crypto_skcipher_blocksize(tfm) - 1; + + cryp->flags = flags; + + if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_ECB || + (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC) + if (req->cryptlen & blocksize_align) + return -EINVAL; + + if (starfive_aes_check_unaligned(cryp, req->src, req->dst)) + return starfive_aes_do_fallback(req, is_encrypt(cryp)); + + return crypto_transfer_skcipher_request_to_engine(cryp->engine, req); +} + +static int starfive_aes_aead_do_fallback(struct aead_request *req, bool enc) +{ + struct starfive_cryp_ctx *ctx = + crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct aead_request *subreq = aead_request_ctx(req); + + aead_request_set_tfm(subreq, ctx->aead_fbk); + aead_request_set_callback(subreq, req->base.flags, + req->base.complete, + req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + aead_request_set_ad(subreq, req->assoclen); + + return enc ? crypto_aead_encrypt(subreq) : + crypto_aead_decrypt(subreq); +} + +static int starfive_aes_aead_crypt(struct aead_request *req, unsigned long flags) +{ + struct starfive_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct starfive_cryp_dev *cryp = ctx->cryp; + struct scatterlist *src, *dst, _src[2], _dst[2]; + + cryp->flags = flags; + + /* aes-ccm does not support tag verification for non-aligned text, + * use fallback for ccm decryption instead. + */ + if (((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CCM) && + !is_encrypt(cryp)) + return starfive_aes_aead_do_fallback(req, 0); + + src = scatterwalk_ffwd(_src, req->src, req->assoclen); + + if (req->src == req->dst) + dst = src; + else + dst = scatterwalk_ffwd(_dst, req->dst, req->assoclen); + + if (starfive_aes_check_unaligned(cryp, src, dst)) + return starfive_aes_aead_do_fallback(req, is_encrypt(cryp)); + + return crypto_transfer_aead_request_to_engine(cryp->engine, req); +} + +static int starfive_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (!key || !keylen) + return -EINVAL; + + if (keylen != AES_KEYSIZE_128 && + keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) + return -EINVAL; + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + return crypto_skcipher_setkey(ctx->skcipher_fbk, key, keylen); +} + +static int starfive_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm); + + if (!key || !keylen) + return -EINVAL; + + if (keylen != AES_KEYSIZE_128 && + keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) + return -EINVAL; + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + return crypto_aead_setkey(ctx->aead_fbk, key, keylen); +} + +static int starfive_aes_gcm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm); + int ret; + + ret = crypto_gcm_check_authsize(authsize); + if (ret) + return ret; + + return crypto_aead_setauthsize(ctx->aead_fbk, authsize); +} + +static int starfive_aes_ccm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm); + + switch (authsize) { + case 4: + case 6: + case 8: + case 10: + case 12: + case 14: + case 16: + break; + default: + return -EINVAL; + } + + return crypto_aead_setauthsize(ctx->aead_fbk, authsize); +} + +static int starfive_aes_ecb_encrypt(struct skcipher_request *req) +{ + return starfive_aes_crypt(req, STARFIVE_AES_MODE_ECB | FLG_ENCRYPT); +} + +static int starfive_aes_ecb_decrypt(struct skcipher_request *req) +{ + return starfive_aes_crypt(req, STARFIVE_AES_MODE_ECB); +} + +static int starfive_aes_cbc_encrypt(struct skcipher_request *req) +{ + return starfive_aes_crypt(req, STARFIVE_AES_MODE_CBC | FLG_ENCRYPT); +} + +static int starfive_aes_cbc_decrypt(struct skcipher_request *req) +{ + return starfive_aes_crypt(req, STARFIVE_AES_MODE_CBC); +} + +static int starfive_aes_ctr_encrypt(struct skcipher_request *req) +{ + return starfive_aes_crypt(req, STARFIVE_AES_MODE_CTR | FLG_ENCRYPT); +} + +static int starfive_aes_ctr_decrypt(struct skcipher_request *req) +{ + return starfive_aes_crypt(req, STARFIVE_AES_MODE_CTR); +} + +static int starfive_aes_gcm_encrypt(struct aead_request *req) +{ + return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_GCM | FLG_ENCRYPT); +} + +static int starfive_aes_gcm_decrypt(struct aead_request *req) +{ + return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_GCM); +} + +static int starfive_aes_ccm_encrypt(struct aead_request *req) +{ + int ret; + + ret = starfive_aes_ccm_check_iv(req->iv); + if (ret) + return ret; + + return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_CCM | FLG_ENCRYPT); +} + +static int starfive_aes_ccm_decrypt(struct aead_request *req) +{ + int ret; + + ret = starfive_aes_ccm_check_iv(req->iv); + if (ret) + return ret; + + return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_CCM); +} + +static int starfive_aes_ecb_init_tfm(struct crypto_skcipher *tfm) +{ + return starfive_aes_init_tfm(tfm, "ecb(aes-lib)"); +} + +static int starfive_aes_cbc_init_tfm(struct crypto_skcipher *tfm) +{ + return starfive_aes_init_tfm(tfm, "cbc(aes-lib)"); +} + +static int starfive_aes_ctr_init_tfm(struct crypto_skcipher *tfm) +{ + return starfive_aes_init_tfm(tfm, "ctr(aes-lib)"); +} + +static int starfive_aes_ccm_init_tfm(struct crypto_aead *tfm) +{ + return starfive_aes_aead_init_tfm(tfm, "ccm_base(ctr(aes-lib),cbcmac-aes-lib)"); +} + +static int starfive_aes_gcm_init_tfm(struct crypto_aead *tfm) +{ + return starfive_aes_aead_init_tfm(tfm, "gcm_base(ctr(aes-lib),ghash-lib)"); +} + +static struct skcipher_engine_alg skcipher_algs[] = { +{ + .base.init = starfive_aes_ecb_init_tfm, + .base.exit = starfive_aes_exit_tfm, + .base.setkey = starfive_aes_setkey, + .base.encrypt = starfive_aes_ecb_encrypt, + .base.decrypt = starfive_aes_ecb_decrypt, + .base.min_keysize = AES_MIN_KEY_SIZE, + .base.max_keysize = AES_MAX_KEY_SIZE, + .base.base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "starfive-ecb-aes", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + .op = { + .do_one_request = starfive_aes_do_one_req, + }, +}, { + .base.init = starfive_aes_cbc_init_tfm, + .base.exit = starfive_aes_exit_tfm, + .base.setkey = starfive_aes_setkey, + .base.encrypt = starfive_aes_cbc_encrypt, + .base.decrypt = starfive_aes_cbc_decrypt, + .base.min_keysize = AES_MIN_KEY_SIZE, + .base.max_keysize = AES_MAX_KEY_SIZE, + .base.ivsize = AES_BLOCK_SIZE, + .base.base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "starfive-cbc-aes", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + .op = { + .do_one_request = starfive_aes_do_one_req, + }, +}, { + .base.init = starfive_aes_ctr_init_tfm, + .base.exit = starfive_aes_exit_tfm, + .base.setkey = starfive_aes_setkey, + .base.encrypt = starfive_aes_ctr_encrypt, + .base.decrypt = starfive_aes_ctr_decrypt, + .base.min_keysize = AES_MIN_KEY_SIZE, + .base.max_keysize = AES_MAX_KEY_SIZE, + .base.ivsize = AES_BLOCK_SIZE, + .base.base = { + .cra_name = "ctr(aes)", + .cra_driver_name = "starfive-ctr-aes", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + .op = { + .do_one_request = starfive_aes_do_one_req, + }, +}, +}; + +static struct aead_engine_alg aead_algs[] = { +{ + .base.setkey = starfive_aes_aead_setkey, + .base.setauthsize = starfive_aes_gcm_setauthsize, + .base.encrypt = starfive_aes_gcm_encrypt, + .base.decrypt = starfive_aes_gcm_decrypt, + .base.init = starfive_aes_gcm_init_tfm, + .base.exit = starfive_aes_aead_exit_tfm, + .base.ivsize = GCM_AES_IV_SIZE, + .base.maxauthsize = AES_BLOCK_SIZE, + .base.base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "starfive-gcm-aes", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + .op = { + .do_one_request = starfive_aes_aead_do_one_req, + }, +}, { + .base.setkey = starfive_aes_aead_setkey, + .base.setauthsize = starfive_aes_ccm_setauthsize, + .base.encrypt = starfive_aes_ccm_encrypt, + .base.decrypt = starfive_aes_ccm_decrypt, + .base.init = starfive_aes_ccm_init_tfm, + .base.exit = starfive_aes_aead_exit_tfm, + .base.ivsize = AES_BLOCK_SIZE, + .base.maxauthsize = AES_BLOCK_SIZE, + .base.base = { + .cra_name = "ccm(aes)", + .cra_driver_name = "starfive-ccm-aes", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + .op = { + .do_one_request = starfive_aes_aead_do_one_req, + }, +}, +}; + +int starfive_aes_register_algs(void) +{ + int ret; + + ret = crypto_engine_register_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs)); + if (ret) + return ret; + + ret = crypto_engine_register_aeads(aead_algs, ARRAY_SIZE(aead_algs)); + if (ret) + crypto_engine_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs)); + + return ret; +} + +void starfive_aes_unregister_algs(void) +{ + crypto_engine_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs)); + crypto_engine_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs)); +} diff --git a/drivers/crypto/starfive/jh7110-cryp.c b/drivers/crypto/starfive/jh7110-cryp.c new file mode 100644 index 000000000..842f76b0f --- /dev/null +++ b/drivers/crypto/starfive/jh7110-cryp.c @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Cryptographic API. + * + * Support for StarFive hardware cryptographic engine. + * Copyright (c) 2022 StarFive Technology + * + */ + +#include +#include "jh7110-cryp.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DRIVER_NAME "jh7110-crypto" + +struct starfive_dev_list { + struct list_head dev_list; + spinlock_t lock; /* protect dev_list */ +}; + +static struct starfive_dev_list dev_list = { + .dev_list = LIST_HEAD_INIT(dev_list.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(dev_list.lock), +}; + +struct starfive_cryp_dev *starfive_cryp_find_dev(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_dev *cryp; + + spin_lock_bh(&dev_list.lock); + if (!ctx->cryp) + ctx->cryp = list_first_entry_or_null(&dev_list.dev_list, + struct starfive_cryp_dev, + list); + cryp = ctx->cryp; + spin_unlock_bh(&dev_list.lock); + + return cryp; +} + +static u16 side_chan; +module_param(side_chan, ushort, 0); +MODULE_PARM_DESC(side_chan, "Enable side channel mitigation for AES module.\n" + "Enabling this feature will reduce speed performance.\n" + " 0 - Disabled\n" + " other - Enabled"); + +static int starfive_dma_init(struct starfive_cryp_dev *cryp) +{ + dma_cap_mask_t mask; + + dma_cap_zero(mask); + dma_cap_set(DMA_SLAVE, mask); + + cryp->tx = dma_request_chan(cryp->dev, "tx"); + if (IS_ERR(cryp->tx)) + return dev_err_probe(cryp->dev, PTR_ERR(cryp->tx), + "Error requesting tx dma channel.\n"); + + cryp->rx = dma_request_chan(cryp->dev, "rx"); + if (IS_ERR(cryp->rx)) { + dma_release_channel(cryp->tx); + return dev_err_probe(cryp->dev, PTR_ERR(cryp->rx), + "Error requesting rx dma channel.\n"); + } + + return 0; +} + +static void starfive_dma_cleanup(struct starfive_cryp_dev *cryp) +{ + dma_release_channel(cryp->tx); + dma_release_channel(cryp->rx); +} + +static int starfive_cryp_probe(struct platform_device *pdev) +{ + struct starfive_cryp_dev *cryp; + struct resource *res; + int ret; + + cryp = devm_kzalloc(&pdev->dev, sizeof(*cryp), GFP_KERNEL); + if (!cryp) + return -ENOMEM; + + platform_set_drvdata(pdev, cryp); + cryp->dev = &pdev->dev; + + cryp->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); + if (IS_ERR(cryp->base)) + return dev_err_probe(&pdev->dev, PTR_ERR(cryp->base), + "Error remapping memory for platform device\n"); + + cryp->phys_base = res->start; + cryp->dma_maxburst = 32; + cryp->side_chan = side_chan; + + cryp->hclk = devm_clk_get(&pdev->dev, "hclk"); + if (IS_ERR(cryp->hclk)) + return dev_err_probe(&pdev->dev, PTR_ERR(cryp->hclk), + "Error getting hardware reference clock\n"); + + cryp->ahb = devm_clk_get(&pdev->dev, "ahb"); + if (IS_ERR(cryp->ahb)) + return dev_err_probe(&pdev->dev, PTR_ERR(cryp->ahb), + "Error getting ahb reference clock\n"); + + cryp->rst = devm_reset_control_get_shared(cryp->dev, NULL); + if (IS_ERR(cryp->rst)) + return dev_err_probe(&pdev->dev, PTR_ERR(cryp->rst), + "Error getting hardware reset line\n"); + + clk_prepare_enable(cryp->hclk); + clk_prepare_enable(cryp->ahb); + reset_control_deassert(cryp->rst); + + spin_lock(&dev_list.lock); + list_add(&cryp->list, &dev_list.dev_list); + spin_unlock(&dev_list.lock); + + ret = starfive_dma_init(cryp); + if (ret) + goto err_dma_init; + + /* Initialize crypto engine */ + cryp->engine = crypto_engine_alloc_init(&pdev->dev, 1); + if (!cryp->engine) { + ret = -ENOMEM; + goto err_engine; + } + + ret = crypto_engine_start(cryp->engine); + if (ret) + goto err_engine_start; + + ret = starfive_aes_register_algs(); + if (ret) + goto err_engine_start; + + ret = starfive_hash_register_algs(); + if (ret) + goto err_algs_hash; + + ret = starfive_rsa_register_algs(); + if (ret) + goto err_algs_rsa; + + return 0; + +err_algs_rsa: + starfive_hash_unregister_algs(); +err_algs_hash: + starfive_aes_unregister_algs(); +err_engine_start: + crypto_engine_exit(cryp->engine); +err_engine: + starfive_dma_cleanup(cryp); +err_dma_init: + spin_lock(&dev_list.lock); + list_del(&cryp->list); + spin_unlock(&dev_list.lock); + + clk_disable_unprepare(cryp->hclk); + clk_disable_unprepare(cryp->ahb); + reset_control_assert(cryp->rst); + + return ret; +} + +static void starfive_cryp_remove(struct platform_device *pdev) +{ + struct starfive_cryp_dev *cryp = platform_get_drvdata(pdev); + + starfive_aes_unregister_algs(); + starfive_hash_unregister_algs(); + starfive_rsa_unregister_algs(); + + crypto_engine_exit(cryp->engine); + + starfive_dma_cleanup(cryp); + + spin_lock(&dev_list.lock); + list_del(&cryp->list); + spin_unlock(&dev_list.lock); + + clk_disable_unprepare(cryp->hclk); + clk_disable_unprepare(cryp->ahb); + reset_control_assert(cryp->rst); +} + +static const struct of_device_id starfive_dt_ids[] __maybe_unused = { + { .compatible = "starfive,jh7110-crypto", .data = NULL}, + {}, +}; +MODULE_DEVICE_TABLE(of, starfive_dt_ids); + +static struct platform_driver starfive_cryp_driver = { + .probe = starfive_cryp_probe, + .remove = starfive_cryp_remove, + .driver = { + .name = DRIVER_NAME, + .of_match_table = starfive_dt_ids, + }, +}; + +module_platform_driver(starfive_cryp_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("StarFive JH7110 Cryptographic Module"); diff --git a/drivers/crypto/starfive/jh7110-cryp.h b/drivers/crypto/starfive/jh7110-cryp.h new file mode 100644 index 000000000..f85d6fb81 --- /dev/null +++ b/drivers/crypto/starfive/jh7110-cryp.h @@ -0,0 +1,240 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __STARFIVE_STR_H__ +#define __STARFIVE_STR_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define STARFIVE_ALG_CR_OFFSET 0x0 +#define STARFIVE_ALG_FIFO_OFFSET 0x4 +#define STARFIVE_IE_MASK_OFFSET 0x8 +#define STARFIVE_IE_FLAG_OFFSET 0xc +#define STARFIVE_DMA_IN_LEN_OFFSET 0x10 +#define STARFIVE_DMA_OUT_LEN_OFFSET 0x14 + +#define STARFIVE_IE_MASK_AES_DONE 0x1 +#define STARFIVE_IE_MASK_HASH_DONE 0x4 +#define STARFIVE_IE_MASK_PKA_DONE 0x8 +#define STARFIVE_IE_FLAG_AES_DONE 0x1 +#define STARFIVE_IE_FLAG_HASH_DONE 0x4 +#define STARFIVE_IE_FLAG_PKA_DONE 0x8 + +#define STARFIVE_MSG_BUFFER_SIZE SZ_16K +#define MAX_KEY_SIZE SHA512_BLOCK_SIZE +#define STARFIVE_AES_IV_LEN AES_BLOCK_SIZE +#define STARFIVE_AES_CTR_LEN AES_BLOCK_SIZE +#define STARFIVE_RSA_MAX_KEYSZ 256 + +union starfive_aes_csr { + u32 v; + struct { + u32 cmode :1; +#define STARFIVE_AES_KEYMODE_128 0x0 +#define STARFIVE_AES_KEYMODE_192 0x1 +#define STARFIVE_AES_KEYMODE_256 0x2 + u32 keymode :2; +#define STARFIVE_AES_BUSY BIT(3) + u32 busy :1; + u32 done :1; +#define STARFIVE_AES_KEY_DONE BIT(5) + u32 krdy :1; + u32 aesrst :1; + u32 ie :1; +#define STARFIVE_AES_CCM_START BIT(8) + u32 ccm_start :1; +#define STARFIVE_AES_MODE_ECB 0x0 +#define STARFIVE_AES_MODE_CBC 0x1 +#define STARFIVE_AES_MODE_CTR 0x4 +#define STARFIVE_AES_MODE_CCM 0x5 +#define STARFIVE_AES_MODE_GCM 0x6 + u32 mode :3; +#define STARFIVE_AES_GCM_START BIT(12) + u32 gcm_start :1; +#define STARFIVE_AES_GCM_DONE BIT(13) + u32 gcm_done :1; + u32 delay_aes :1; + u32 vaes_start :1; + u32 rsvd_0 :8; +#define STARFIVE_AES_MODE_XFB_1 0x0 +#define STARFIVE_AES_MODE_XFB_128 0x5 + u32 stmode :3; + u32 rsvd_1 :5; + }; +}; + +union starfive_hash_csr { + u32 v; + struct { + u32 start :1; + u32 reset :1; + u32 ie :1; + u32 firstb :1; +#define STARFIVE_HASH_SM3 0x0 +#define STARFIVE_HASH_SHA224 0x3 +#define STARFIVE_HASH_SHA256 0x4 +#define STARFIVE_HASH_SHA384 0x5 +#define STARFIVE_HASH_SHA512 0x6 +#define STARFIVE_HASH_MODE_MASK 0x7 + u32 mode :3; + u32 rsvd_1 :1; + u32 final :1; + u32 rsvd_2 :2; +#define STARFIVE_HASH_HMAC_FLAGS 0x800 + u32 hmac :1; + u32 rsvd_3 :1; +#define STARFIVE_HASH_KEY_DONE BIT(13) + u32 key_done :1; + u32 key_flag :1; +#define STARFIVE_HASH_HMAC_DONE BIT(15) + u32 hmac_done :1; +#define STARFIVE_HASH_BUSY BIT(16) + u32 busy :1; + u32 hashdone :1; + u32 rsvd_4 :14; + }; +}; + +union starfive_pka_cacr { + u32 v; + struct { + u32 start :1; + u32 reset :1; + u32 ie :1; + u32 rsvd_0 :1; + u32 fifo_mode :1; + u32 not_r2 :1; + u32 ecc_sub :1; + u32 pre_expf :1; + u32 cmd :4; + u32 rsvd_1 :1; + u32 ctrl_dummy :1; + u32 ctrl_false :1; + u32 cln_done :1; + u32 opsize :6; + u32 rsvd_2 :2; + u32 exposize :6; + u32 rsvd_3 :1; + u32 bigendian :1; + }; +}; + +union starfive_pka_casr { + u32 v; + struct { +#define STARFIVE_PKA_DONE BIT(0) + u32 done :1; + u32 rsvd_0 :31; + }; +}; + +struct starfive_rsa_key { + u8 *n; + u8 *e; + u8 *d; + int e_bitlen; + int d_bitlen; + int bitlen; + size_t key_sz; +}; + +union starfive_alg_cr { + u32 v; + struct { + u32 start :1; + u32 aes_dma_en :1; + u32 rsvd_0 :1; + u32 hash_dma_en :1; + u32 alg_done :1; + u32 rsvd_1 :3; + u32 clear :1; + u32 rsvd_2 :23; + }; +}; + +struct starfive_cryp_ctx { + struct starfive_cryp_dev *cryp; + struct starfive_cryp_request_ctx *rctx; + + unsigned int hash_mode; + u8 key[MAX_KEY_SIZE]; + int keylen; + bool is_hmac; + struct starfive_rsa_key rsa_key; + struct crypto_akcipher *akcipher_fbk; + struct crypto_ahash *ahash_fbk; + struct crypto_aead *aead_fbk; + struct crypto_skcipher *skcipher_fbk; +}; + +struct starfive_cryp_dev { + struct list_head list; + struct device *dev; + struct clk *hclk; + struct clk *ahb; + struct reset_control *rst; + + void __iomem *base; + phys_addr_t phys_base; + + u32 dma_maxburst; + struct dma_chan *tx; + struct dma_chan *rx; + struct dma_slave_config cfg_in; + struct dma_slave_config cfg_out; + struct crypto_engine *engine; + struct completion dma_done; + size_t assoclen; + size_t total_in; + size_t total_out; + u32 tag_in[4]; + u32 tag_out[4]; + unsigned int authsize; + unsigned long flags; + int err; + bool side_chan; + union starfive_alg_cr alg_cr; + union { + struct ahash_request *hreq; + struct aead_request *areq; + struct skcipher_request *sreq; + } req; +}; + +struct starfive_cryp_request_ctx { + union { + union starfive_hash_csr hash; + union starfive_pka_cacr pka; + union starfive_aes_csr aes; + } csr; + + struct scatterlist *in_sg; + struct scatterlist *out_sg; + size_t total; + unsigned int blksize; + unsigned int digsize; + unsigned long in_sg_len; + unsigned char *adata; + u8 rsa_data[STARFIVE_RSA_MAX_KEYSZ] __aligned(sizeof(u32)); + + /* Must be last as it ends in a flexible-array member. */ + struct ahash_request ahash_fbk_req; +}; + +struct starfive_cryp_dev *starfive_cryp_find_dev(struct starfive_cryp_ctx *ctx); + +int starfive_hash_register_algs(void); +void starfive_hash_unregister_algs(void); + +int starfive_rsa_register_algs(void); +void starfive_rsa_unregister_algs(void); + +int starfive_aes_register_algs(void); +void starfive_aes_unregister_algs(void); +#endif diff --git a/drivers/crypto/starfive/jh7110-hash.c b/drivers/crypto/starfive/jh7110-hash.c new file mode 100644 index 000000000..008a47baa --- /dev/null +++ b/drivers/crypto/starfive/jh7110-hash.c @@ -0,0 +1,854 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Hash function and HMAC support for StarFive driver + * + * Copyright (c) 2022 StarFive Technology + * + */ + +#include +#include +#include +#include "jh7110-cryp.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define STARFIVE_HASH_REGS_OFFSET 0x300 +#define STARFIVE_HASH_SHACSR (STARFIVE_HASH_REGS_OFFSET + 0x0) +#define STARFIVE_HASH_SHAWDR (STARFIVE_HASH_REGS_OFFSET + 0x4) +#define STARFIVE_HASH_SHARDR (STARFIVE_HASH_REGS_OFFSET + 0x8) +#define STARFIVE_HASH_SHAWSR (STARFIVE_HASH_REGS_OFFSET + 0xC) +#define STARFIVE_HASH_SHAWLEN3 (STARFIVE_HASH_REGS_OFFSET + 0x10) +#define STARFIVE_HASH_SHAWLEN2 (STARFIVE_HASH_REGS_OFFSET + 0x14) +#define STARFIVE_HASH_SHAWLEN1 (STARFIVE_HASH_REGS_OFFSET + 0x18) +#define STARFIVE_HASH_SHAWLEN0 (STARFIVE_HASH_REGS_OFFSET + 0x1C) +#define STARFIVE_HASH_SHAWKR (STARFIVE_HASH_REGS_OFFSET + 0x20) +#define STARFIVE_HASH_SHAWKLEN (STARFIVE_HASH_REGS_OFFSET + 0x24) + +#define STARFIVE_HASH_BUFLEN SHA512_BLOCK_SIZE +#define STARFIVE_HASH_RESET 0x2 + +static inline int starfive_hash_wait_busy(struct starfive_cryp_dev *cryp) +{ + u32 status; + + return readl_relaxed_poll_timeout(cryp->base + STARFIVE_HASH_SHACSR, status, + !(status & STARFIVE_HASH_BUSY), 10, 100000); +} + +static inline int starfive_hash_wait_hmac_done(struct starfive_cryp_dev *cryp) +{ + u32 status; + + return readl_relaxed_poll_timeout(cryp->base + STARFIVE_HASH_SHACSR, status, + (status & STARFIVE_HASH_HMAC_DONE), 10, 100000); +} + +static inline int starfive_hash_wait_key_done(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + u32 status; + + return readl_relaxed_poll_timeout(cryp->base + STARFIVE_HASH_SHACSR, status, + (status & STARFIVE_HASH_KEY_DONE), 10, 100000); +} + +static int starfive_hash_hmac_key(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_request_ctx *rctx = ctx->rctx; + struct starfive_cryp_dev *cryp = ctx->cryp; + int klen = ctx->keylen, loop; + unsigned int *key = (unsigned int *)ctx->key; + unsigned char *cl; + + writel(ctx->keylen, cryp->base + STARFIVE_HASH_SHAWKLEN); + + rctx->csr.hash.hmac = 1; + rctx->csr.hash.key_flag = 1; + + writel(rctx->csr.hash.v, cryp->base + STARFIVE_HASH_SHACSR); + + for (loop = 0; loop < klen / sizeof(unsigned int); loop++, key++) + writel(*key, cryp->base + STARFIVE_HASH_SHAWKR); + + if (klen & 0x3) { + cl = (unsigned char *)key; + for (loop = 0; loop < (klen & 0x3); loop++, cl++) + writeb(*cl, cryp->base + STARFIVE_HASH_SHAWKR); + } + + if (starfive_hash_wait_key_done(ctx)) + return dev_err_probe(cryp->dev, -ETIMEDOUT, "starfive_hash_wait_key_done error\n"); + + return 0; +} + +static void starfive_hash_start(struct starfive_cryp_dev *cryp) +{ + union starfive_hash_csr csr; + + csr.v = readl(cryp->base + STARFIVE_HASH_SHACSR); + csr.firstb = 0; + csr.final = 1; + writel(csr.v, cryp->base + STARFIVE_HASH_SHACSR); +} + +static void starfive_hash_dma_callback(void *param) +{ + struct starfive_cryp_dev *cryp = param; + + complete(&cryp->dma_done); +} + +static void starfive_hash_dma_init(struct starfive_cryp_dev *cryp) +{ + cryp->cfg_in.src_addr_width = DMA_SLAVE_BUSWIDTH_16_BYTES; + cryp->cfg_in.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cryp->cfg_in.src_maxburst = cryp->dma_maxburst; + cryp->cfg_in.dst_maxburst = cryp->dma_maxburst; + cryp->cfg_in.dst_addr = cryp->phys_base + STARFIVE_ALG_FIFO_OFFSET; + + dmaengine_slave_config(cryp->tx, &cryp->cfg_in); + + init_completion(&cryp->dma_done); +} + +static int starfive_hash_dma_xfer(struct starfive_cryp_dev *cryp, + struct scatterlist *sg) +{ + struct dma_async_tx_descriptor *in_desc; + union starfive_alg_cr alg_cr; + int ret = 0; + + alg_cr.v = 0; + alg_cr.start = 1; + alg_cr.hash_dma_en = 1; + writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET); + + writel(sg_dma_len(sg), cryp->base + STARFIVE_DMA_IN_LEN_OFFSET); + sg_dma_len(sg) = ALIGN(sg_dma_len(sg), sizeof(u32)); + + in_desc = dmaengine_prep_slave_sg(cryp->tx, sg, 1, DMA_MEM_TO_DEV, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!in_desc) { + ret = -EINVAL; + goto end; + } + + reinit_completion(&cryp->dma_done); + in_desc->callback = starfive_hash_dma_callback; + in_desc->callback_param = cryp; + + dmaengine_submit(in_desc); + dma_async_issue_pending(cryp->tx); + + if (!wait_for_completion_timeout(&cryp->dma_done, + msecs_to_jiffies(1000))) + ret = -ETIMEDOUT; + +end: + alg_cr.v = 0; + alg_cr.clear = 1; + writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET); + + return ret; +} + +static int starfive_hash_copy_hash(struct ahash_request *req) +{ + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); + int count, *data; + int mlen; + + if (!req->result) + return 0; + + mlen = rctx->digsize / sizeof(u32); + data = (u32 *)req->result; + + for (count = 0; count < mlen; count++) + put_unaligned(readl(ctx->cryp->base + STARFIVE_HASH_SHARDR), + &data[count]); + + return 0; +} + +static void starfive_hash_done_task(struct starfive_cryp_dev *cryp) +{ + int err = cryp->err; + + if (!err) + err = starfive_hash_copy_hash(cryp->req.hreq); + + crypto_finalize_hash_request(cryp->engine, cryp->req.hreq, err); +} + +static int starfive_hash_one_request(struct crypto_engine *engine, void *areq) +{ + struct ahash_request *req = container_of(areq, struct ahash_request, + base); + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); + struct starfive_cryp_request_ctx *rctx = ctx->rctx; + struct starfive_cryp_dev *cryp = ctx->cryp; + struct scatterlist *tsg; + int ret, src_nents, i; + + writel(STARFIVE_HASH_RESET, cryp->base + STARFIVE_HASH_SHACSR); + + if (starfive_hash_wait_busy(cryp)) + return dev_err_probe(cryp->dev, -ETIMEDOUT, "Error resetting hardware\n"); + + rctx->csr.hash.v = 0; + rctx->csr.hash.mode = ctx->hash_mode; + + if (ctx->is_hmac) { + ret = starfive_hash_hmac_key(ctx); + if (ret) + return ret; + } else { + rctx->csr.hash.start = 1; + rctx->csr.hash.firstb = 1; + writel(rctx->csr.hash.v, cryp->base + STARFIVE_HASH_SHACSR); + } + + /* No input message, get digest and end. */ + if (!rctx->total) + goto hash_start; + + starfive_hash_dma_init(cryp); + + for_each_sg(rctx->in_sg, tsg, rctx->in_sg_len, i) { + src_nents = dma_map_sg(cryp->dev, tsg, 1, DMA_TO_DEVICE); + if (src_nents == 0) + return -ENOMEM; + + ret = starfive_hash_dma_xfer(cryp, tsg); + dma_unmap_sg(cryp->dev, tsg, 1, DMA_TO_DEVICE); + if (ret) + return ret; + } + +hash_start: + starfive_hash_start(cryp); + + if (starfive_hash_wait_busy(cryp)) + return dev_err_probe(cryp->dev, -ETIMEDOUT, "Error generating digest\n"); + + if (ctx->is_hmac) + cryp->err = starfive_hash_wait_hmac_done(cryp); + + starfive_hash_done_task(cryp); + + return 0; +} + +static int starfive_hash_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk); + ahash_request_set_callback(&rctx->ahash_fbk_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + + ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src, + req->result, req->nbytes); + + return crypto_ahash_init(&rctx->ahash_fbk_req); +} + +static int starfive_hash_update(struct ahash_request *req) +{ + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk); + ahash_request_set_callback(&rctx->ahash_fbk_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + + ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src, + req->result, req->nbytes); + + return crypto_ahash_update(&rctx->ahash_fbk_req); +} + +static int starfive_hash_final(struct ahash_request *req) +{ + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk); + ahash_request_set_callback(&rctx->ahash_fbk_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + + ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src, + req->result, req->nbytes); + + return crypto_ahash_final(&rctx->ahash_fbk_req); +} + +static int starfive_hash_finup(struct ahash_request *req) +{ + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk); + ahash_request_set_callback(&rctx->ahash_fbk_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + + ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src, + req->result, req->nbytes); + + return crypto_ahash_finup(&rctx->ahash_fbk_req); +} + +static int starfive_hash_digest(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); + struct starfive_cryp_dev *cryp = ctx->cryp; + int sg_len; + + memset(rctx, 0, sizeof(struct starfive_cryp_request_ctx)); + + cryp->req.hreq = req; + rctx->total = req->nbytes; + rctx->in_sg = req->src; + rctx->blksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + rctx->digsize = crypto_ahash_digestsize(tfm); + sg_len = sg_nents_for_len(rctx->in_sg, rctx->total); + if (sg_len < 0) + return sg_len; + rctx->in_sg_len = sg_len; + ctx->rctx = rctx; + + return crypto_transfer_hash_request_to_engine(cryp->engine, req); +} + +static int starfive_hash_export(struct ahash_request *req, void *out) +{ + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk); + ahash_request_set_callback(&rctx->ahash_fbk_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + + return crypto_ahash_export(&rctx->ahash_fbk_req, out); +} + +static int starfive_hash_import(struct ahash_request *req, const void *in) +{ + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk); + ahash_request_set_callback(&rctx->ahash_fbk_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + + return crypto_ahash_import(&rctx->ahash_fbk_req, in); +} + +static int starfive_hash_init_tfm(struct crypto_ahash *hash, + const char *alg_name, + unsigned int mode, + bool is_hmac) +{ + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); + + ctx->cryp = starfive_cryp_find_dev(ctx); + + if (!ctx->cryp) + return -ENODEV; + + ctx->ahash_fbk = crypto_alloc_ahash(alg_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + + if (IS_ERR(ctx->ahash_fbk)) + return dev_err_probe(ctx->cryp->dev, PTR_ERR(ctx->ahash_fbk), + "starfive_hash: Could not load fallback driver.\n"); + + crypto_ahash_set_statesize(hash, crypto_ahash_statesize(ctx->ahash_fbk)); + crypto_ahash_set_reqsize(hash, sizeof(struct starfive_cryp_request_ctx) + + crypto_ahash_reqsize(ctx->ahash_fbk)); + + ctx->is_hmac = is_hmac; + ctx->hash_mode = mode; + + return 0; +} + +static void starfive_hash_exit_tfm(struct crypto_ahash *hash) +{ + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); + + crypto_free_ahash(ctx->ahash_fbk); +} + +static int starfive_hash_long_setkey(struct starfive_cryp_ctx *ctx, + const u8 *key, unsigned int keylen, + const char *alg_name) +{ + struct crypto_wait wait; + struct ahash_request *req; + struct scatterlist sg; + struct crypto_ahash *ahash_tfm; + u8 *buf; + int ret; + + ahash_tfm = crypto_alloc_ahash(alg_name, 0, 0); + if (IS_ERR(ahash_tfm)) + return PTR_ERR(ahash_tfm); + + req = ahash_request_alloc(ahash_tfm, GFP_KERNEL); + if (!req) { + ret = -ENOMEM; + goto err_free_ahash; + } + + crypto_init_wait(&wait); + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + crypto_ahash_clear_flags(ahash_tfm, ~0); + + buf = kzalloc(keylen + STARFIVE_HASH_BUFLEN, GFP_KERNEL); + if (!buf) { + ret = -ENOMEM; + goto err_free_req; + } + + memcpy(buf, key, keylen); + sg_init_one(&sg, buf, keylen); + ahash_request_set_crypt(req, &sg, ctx->key, keylen); + + ret = crypto_wait_req(crypto_ahash_digest(req), &wait); + + kfree(buf); +err_free_req: + ahash_request_free(req); +err_free_ahash: + crypto_free_ahash(ahash_tfm); + return ret; +} + +static int starfive_hash_setkey(struct crypto_ahash *hash, + const u8 *key, unsigned int keylen) +{ + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); + unsigned int digestsize = crypto_ahash_digestsize(hash); + unsigned int blocksize = crypto_ahash_blocksize(hash); + const char *alg_name; + + crypto_ahash_setkey(ctx->ahash_fbk, key, keylen); + + if (keylen <= blocksize) { + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + return 0; + } + + ctx->keylen = digestsize; + + switch (digestsize) { + case SHA224_DIGEST_SIZE: + alg_name = "sha224-starfive"; + break; + case SHA256_DIGEST_SIZE: + if (ctx->hash_mode == STARFIVE_HASH_SM3) + alg_name = "sm3-starfive"; + else + alg_name = "sha256-starfive"; + break; + case SHA384_DIGEST_SIZE: + alg_name = "sha384-starfive"; + break; + case SHA512_DIGEST_SIZE: + alg_name = "sha512-starfive"; + break; + default: + return -EINVAL; + } + + return starfive_hash_long_setkey(ctx, key, keylen, alg_name); +} + +static int starfive_sha224_init_tfm(struct crypto_ahash *hash) +{ + return starfive_hash_init_tfm(hash, "sha224-lib", + STARFIVE_HASH_SHA224, 0); +} + +static int starfive_sha256_init_tfm(struct crypto_ahash *hash) +{ + return starfive_hash_init_tfm(hash, "sha256-lib", + STARFIVE_HASH_SHA256, 0); +} + +static int starfive_sha384_init_tfm(struct crypto_ahash *hash) +{ + return starfive_hash_init_tfm(hash, "sha384-lib", + STARFIVE_HASH_SHA384, 0); +} + +static int starfive_sha512_init_tfm(struct crypto_ahash *hash) +{ + return starfive_hash_init_tfm(hash, "sha512-lib", + STARFIVE_HASH_SHA512, 0); +} + +static int starfive_sm3_init_tfm(struct crypto_ahash *hash) +{ + return starfive_hash_init_tfm(hash, "sm3-lib", + STARFIVE_HASH_SM3, 0); +} + +static int starfive_hmac_sha224_init_tfm(struct crypto_ahash *hash) +{ + return starfive_hash_init_tfm(hash, "hmac-sha224-lib", + STARFIVE_HASH_SHA224, 1); +} + +static int starfive_hmac_sha256_init_tfm(struct crypto_ahash *hash) +{ + return starfive_hash_init_tfm(hash, "hmac-sha256-lib", + STARFIVE_HASH_SHA256, 1); +} + +static int starfive_hmac_sha384_init_tfm(struct crypto_ahash *hash) +{ + return starfive_hash_init_tfm(hash, "hmac-sha384-lib", + STARFIVE_HASH_SHA384, 1); +} + +static int starfive_hmac_sha512_init_tfm(struct crypto_ahash *hash) +{ + return starfive_hash_init_tfm(hash, "hmac-sha512-lib", + STARFIVE_HASH_SHA512, 1); +} + +static int starfive_hmac_sm3_init_tfm(struct crypto_ahash *hash) +{ + return starfive_hash_init_tfm(hash, "hmac(sm3-lib)", + STARFIVE_HASH_SM3, 1); +} + +static struct ahash_engine_alg algs_sha2_sm3[] = { +{ + .base.init = starfive_hash_init, + .base.update = starfive_hash_update, + .base.final = starfive_hash_final, + .base.finup = starfive_hash_finup, + .base.digest = starfive_hash_digest, + .base.export = starfive_hash_export, + .base.import = starfive_hash_import, + .base.init_tfm = starfive_sha224_init_tfm, + .base.exit_tfm = starfive_hash_exit_tfm, + .base.halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha224", + .cra_driver_name = "sha224-starfive", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = starfive_hash_one_request, + }, +}, { + .base.init = starfive_hash_init, + .base.update = starfive_hash_update, + .base.final = starfive_hash_final, + .base.finup = starfive_hash_finup, + .base.digest = starfive_hash_digest, + .base.export = starfive_hash_export, + .base.import = starfive_hash_import, + .base.init_tfm = starfive_hmac_sha224_init_tfm, + .base.exit_tfm = starfive_hash_exit_tfm, + .base.setkey = starfive_hash_setkey, + .base.halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "hmac(sha224)", + .cra_driver_name = "sha224-hmac-starfive", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = starfive_hash_one_request, + }, +}, { + .base.init = starfive_hash_init, + .base.update = starfive_hash_update, + .base.final = starfive_hash_final, + .base.finup = starfive_hash_finup, + .base.digest = starfive_hash_digest, + .base.export = starfive_hash_export, + .base.import = starfive_hash_import, + .base.init_tfm = starfive_sha256_init_tfm, + .base.exit_tfm = starfive_hash_exit_tfm, + .base.halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-starfive", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = starfive_hash_one_request, + }, +}, { + .base.init = starfive_hash_init, + .base.update = starfive_hash_update, + .base.final = starfive_hash_final, + .base.finup = starfive_hash_finup, + .base.digest = starfive_hash_digest, + .base.export = starfive_hash_export, + .base.import = starfive_hash_import, + .base.init_tfm = starfive_hmac_sha256_init_tfm, + .base.exit_tfm = starfive_hash_exit_tfm, + .base.setkey = starfive_hash_setkey, + .base.halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "sha256-hmac-starfive", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = starfive_hash_one_request, + }, +}, { + .base.init = starfive_hash_init, + .base.update = starfive_hash_update, + .base.final = starfive_hash_final, + .base.finup = starfive_hash_finup, + .base.digest = starfive_hash_digest, + .base.export = starfive_hash_export, + .base.import = starfive_hash_import, + .base.init_tfm = starfive_sha384_init_tfm, + .base.exit_tfm = starfive_hash_exit_tfm, + .base.halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct sha512_state), + .base = { + .cra_name = "sha384", + .cra_driver_name = "sha384-starfive", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = starfive_hash_one_request, + }, +}, { + .base.init = starfive_hash_init, + .base.update = starfive_hash_update, + .base.final = starfive_hash_final, + .base.finup = starfive_hash_finup, + .base.digest = starfive_hash_digest, + .base.export = starfive_hash_export, + .base.import = starfive_hash_import, + .base.init_tfm = starfive_hmac_sha384_init_tfm, + .base.exit_tfm = starfive_hash_exit_tfm, + .base.setkey = starfive_hash_setkey, + .base.halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct sha512_state), + .base = { + .cra_name = "hmac(sha384)", + .cra_driver_name = "sha384-hmac-starfive", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = starfive_hash_one_request, + }, +}, { + .base.init = starfive_hash_init, + .base.update = starfive_hash_update, + .base.final = starfive_hash_final, + .base.finup = starfive_hash_finup, + .base.digest = starfive_hash_digest, + .base.export = starfive_hash_export, + .base.import = starfive_hash_import, + .base.init_tfm = starfive_sha512_init_tfm, + .base.exit_tfm = starfive_hash_exit_tfm, + .base.halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct sha512_state), + .base = { + .cra_name = "sha512", + .cra_driver_name = "sha512-starfive", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = starfive_hash_one_request, + }, +}, { + .base.init = starfive_hash_init, + .base.update = starfive_hash_update, + .base.final = starfive_hash_final, + .base.finup = starfive_hash_finup, + .base.digest = starfive_hash_digest, + .base.export = starfive_hash_export, + .base.import = starfive_hash_import, + .base.init_tfm = starfive_hmac_sha512_init_tfm, + .base.exit_tfm = starfive_hash_exit_tfm, + .base.setkey = starfive_hash_setkey, + .base.halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct sha512_state), + .base = { + .cra_name = "hmac(sha512)", + .cra_driver_name = "sha512-hmac-starfive", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = starfive_hash_one_request, + }, +}, { + .base.init = starfive_hash_init, + .base.update = starfive_hash_update, + .base.final = starfive_hash_final, + .base.finup = starfive_hash_finup, + .base.digest = starfive_hash_digest, + .base.export = starfive_hash_export, + .base.import = starfive_hash_import, + .base.init_tfm = starfive_sm3_init_tfm, + .base.exit_tfm = starfive_hash_exit_tfm, + .base.halg = { + .digestsize = SM3_DIGEST_SIZE, + .statesize = sizeof(struct sm3_ctx), + .base = { + .cra_name = "sm3", + .cra_driver_name = "sm3-starfive", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SM3_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = starfive_hash_one_request, + }, +}, { + .base.init = starfive_hash_init, + .base.update = starfive_hash_update, + .base.final = starfive_hash_final, + .base.finup = starfive_hash_finup, + .base.digest = starfive_hash_digest, + .base.export = starfive_hash_export, + .base.import = starfive_hash_import, + .base.init_tfm = starfive_hmac_sm3_init_tfm, + .base.exit_tfm = starfive_hash_exit_tfm, + .base.setkey = starfive_hash_setkey, + .base.halg = { + .digestsize = SM3_DIGEST_SIZE, + .statesize = sizeof(struct sm3_ctx), + .base = { + .cra_name = "hmac(sm3)", + .cra_driver_name = "sm3-hmac-starfive", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SM3_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = starfive_hash_one_request, + }, +}, +}; + +int starfive_hash_register_algs(void) +{ + return crypto_engine_register_ahashes(algs_sha2_sm3, ARRAY_SIZE(algs_sha2_sm3)); +} + +void starfive_hash_unregister_algs(void) +{ + crypto_engine_unregister_ahashes(algs_sha2_sm3, ARRAY_SIZE(algs_sha2_sm3)); +} diff --git a/drivers/crypto/starfive/jh7110-rsa.c b/drivers/crypto/starfive/jh7110-rsa.c new file mode 100644 index 000000000..d109c743f --- /dev/null +++ b/drivers/crypto/starfive/jh7110-rsa.c @@ -0,0 +1,592 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * StarFive Public Key Algo acceleration driver + * + * Copyright (c) 2022 StarFive Technology + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "jh7110-cryp.h" + +#define STARFIVE_PKA_REGS_OFFSET 0x400 +#define STARFIVE_PKA_CACR_OFFSET (STARFIVE_PKA_REGS_OFFSET + 0x0) +#define STARFIVE_PKA_CASR_OFFSET (STARFIVE_PKA_REGS_OFFSET + 0x4) +#define STARFIVE_PKA_CAAR_OFFSET (STARFIVE_PKA_REGS_OFFSET + 0x8) +#define STARFIVE_PKA_CAER_OFFSET (STARFIVE_PKA_REGS_OFFSET + 0x108) +#define STARFIVE_PKA_CANR_OFFSET (STARFIVE_PKA_REGS_OFFSET + 0x208) + +/* R ^ 2 mod N and N0' */ +#define CRYPTO_CMD_PRE 0x0 +/* A * R mod N ==> A */ +#define CRYPTO_CMD_ARN 0x5 +/* A * E * R mod N ==> A */ +#define CRYPTO_CMD_AERN 0x6 +/* A * A * R mod N ==> A */ +#define CRYPTO_CMD_AARN 0x7 + +#define STARFIVE_RSA_RESET 0x2 + +static inline int starfive_pka_wait_done(struct starfive_cryp_ctx *ctx) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + u32 status; + + return readl_relaxed_poll_timeout(cryp->base + STARFIVE_PKA_CASR_OFFSET, status, + status & STARFIVE_PKA_DONE, 10, 100000); +} + +static void starfive_rsa_free_key(struct starfive_rsa_key *key) +{ + if (!key->key_sz) + return; + + kfree_sensitive(key->d); + kfree_sensitive(key->e); + kfree_sensitive(key->n); + memset(key, 0, sizeof(*key)); +} + +static unsigned int starfive_rsa_get_nbit(u8 *pa, u32 snum, int key_sz) +{ + u32 i; + u8 value; + + i = snum >> 3; + + value = pa[key_sz - i - 1]; + value >>= snum & 0x7; + value &= 0x1; + + return value; +} + +static int starfive_rsa_montgomery_form(struct starfive_cryp_ctx *ctx, + u32 *out, u32 *in, u8 mont, + u32 *mod, int bit_len) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + struct starfive_cryp_request_ctx *rctx = ctx->rctx; + int count = (ALIGN(rctx->total, 4) / 4) - 1; + int loop; + u32 temp; + u8 opsize; + + opsize = (bit_len - 1) >> 5; + rctx->csr.pka.v = 0; + + writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); + + for (loop = 0; loop <= opsize; loop++) + writel(mod[opsize - loop], cryp->base + STARFIVE_PKA_CANR_OFFSET + loop * 4); + + if (mont) { + rctx->csr.pka.v = 0; + rctx->csr.pka.cln_done = 1; + rctx->csr.pka.opsize = opsize; + rctx->csr.pka.exposize = opsize; + rctx->csr.pka.cmd = CRYPTO_CMD_PRE; + rctx->csr.pka.start = 1; + rctx->csr.pka.not_r2 = 1; + rctx->csr.pka.ie = 1; + + writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); + + if (starfive_pka_wait_done(ctx)) + return -ETIMEDOUT; + + for (loop = 0; loop <= opsize; loop++) + writel(in[opsize - loop], cryp->base + STARFIVE_PKA_CAAR_OFFSET + loop * 4); + + writel(0x1000000, cryp->base + STARFIVE_PKA_CAER_OFFSET); + + for (loop = 1; loop <= opsize; loop++) + writel(0, cryp->base + STARFIVE_PKA_CAER_OFFSET + loop * 4); + + rctx->csr.pka.v = 0; + rctx->csr.pka.cln_done = 1; + rctx->csr.pka.opsize = opsize; + rctx->csr.pka.exposize = opsize; + rctx->csr.pka.cmd = CRYPTO_CMD_AERN; + rctx->csr.pka.start = 1; + rctx->csr.pka.ie = 1; + + writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); + + if (starfive_pka_wait_done(ctx)) + return -ETIMEDOUT; + } else { + rctx->csr.pka.v = 0; + rctx->csr.pka.cln_done = 1; + rctx->csr.pka.opsize = opsize; + rctx->csr.pka.exposize = opsize; + rctx->csr.pka.cmd = CRYPTO_CMD_PRE; + rctx->csr.pka.start = 1; + rctx->csr.pka.pre_expf = 1; + rctx->csr.pka.ie = 1; + + writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); + + if (starfive_pka_wait_done(ctx)) + return -ETIMEDOUT; + + for (loop = 0; loop <= count; loop++) + writel(in[count - loop], cryp->base + STARFIVE_PKA_CAER_OFFSET + loop * 4); + + /*pad with 0 up to opsize*/ + for (loop = count + 1; loop <= opsize; loop++) + writel(0, cryp->base + STARFIVE_PKA_CAER_OFFSET + loop * 4); + + rctx->csr.pka.v = 0; + rctx->csr.pka.cln_done = 1; + rctx->csr.pka.opsize = opsize; + rctx->csr.pka.exposize = opsize; + rctx->csr.pka.cmd = CRYPTO_CMD_ARN; + rctx->csr.pka.start = 1; + rctx->csr.pka.ie = 1; + + writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); + + if (starfive_pka_wait_done(ctx)) + return -ETIMEDOUT; + } + + for (loop = 0; loop <= opsize; loop++) { + temp = readl(cryp->base + STARFIVE_PKA_CAAR_OFFSET + 0x4 * loop); + out[opsize - loop] = temp; + } + + return 0; +} + +static int starfive_rsa_cpu_start(struct starfive_cryp_ctx *ctx, u32 *result, + u8 *de, u32 *n, int key_sz) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + struct starfive_cryp_request_ctx *rctx = ctx->rctx; + struct starfive_rsa_key *key = &ctx->rsa_key; + u32 temp; + int ret = 0; + int opsize, mlen, loop; + unsigned int *mta; + + opsize = (key_sz - 1) >> 2; + + mta = kmalloc(key_sz, GFP_KERNEL); + if (!mta) + return -ENOMEM; + + ret = starfive_rsa_montgomery_form(ctx, mta, (u32 *)rctx->rsa_data, + 0, n, key_sz << 3); + if (ret) { + dev_err_probe(cryp->dev, ret, "Conversion to Montgomery failed"); + goto rsa_err; + } + + for (loop = 0; loop <= opsize; loop++) + writel(mta[opsize - loop], + cryp->base + STARFIVE_PKA_CAER_OFFSET + loop * 4); + + for (loop = key->bitlen - 1; loop > 0; loop--) { + mlen = starfive_rsa_get_nbit(de, loop - 1, key_sz); + + rctx->csr.pka.v = 0; + rctx->csr.pka.cln_done = 1; + rctx->csr.pka.opsize = opsize; + rctx->csr.pka.exposize = opsize; + rctx->csr.pka.cmd = CRYPTO_CMD_AARN; + rctx->csr.pka.start = 1; + rctx->csr.pka.ie = 1; + + writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); + + ret = -ETIMEDOUT; + if (starfive_pka_wait_done(ctx)) + goto rsa_err; + + if (mlen) { + rctx->csr.pka.v = 0; + rctx->csr.pka.cln_done = 1; + rctx->csr.pka.opsize = opsize; + rctx->csr.pka.exposize = opsize; + rctx->csr.pka.cmd = CRYPTO_CMD_AERN; + rctx->csr.pka.start = 1; + rctx->csr.pka.ie = 1; + + writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); + + if (starfive_pka_wait_done(ctx)) + goto rsa_err; + } + } + + for (loop = 0; loop <= opsize; loop++) { + temp = readl(cryp->base + STARFIVE_PKA_CAAR_OFFSET + 0x4 * loop); + result[opsize - loop] = temp; + } + + ret = starfive_rsa_montgomery_form(ctx, result, result, 1, n, key_sz << 3); + if (ret) + dev_err_probe(cryp->dev, ret, "Conversion from Montgomery failed"); +rsa_err: + kfree(mta); + return ret; +} + +static int starfive_rsa_start(struct starfive_cryp_ctx *ctx, u8 *result, + u8 *de, u8 *n, int key_sz) +{ + return starfive_rsa_cpu_start(ctx, (u32 *)result, de, (u32 *)n, key_sz); +} + +static int starfive_rsa_enc_core(struct starfive_cryp_ctx *ctx, int enc) +{ + struct starfive_cryp_dev *cryp = ctx->cryp; + struct starfive_cryp_request_ctx *rctx = ctx->rctx; + struct starfive_rsa_key *key = &ctx->rsa_key; + int ret = 0, shift = 0; + + writel(STARFIVE_RSA_RESET, cryp->base + STARFIVE_PKA_CACR_OFFSET); + + if (!IS_ALIGNED(rctx->total, sizeof(u32))) { + shift = sizeof(u32) - (rctx->total & 0x3); + memset(rctx->rsa_data, 0, shift); + } + + rctx->total = sg_copy_to_buffer(rctx->in_sg, sg_nents(rctx->in_sg), + rctx->rsa_data + shift, rctx->total); + + if (enc) { + key->bitlen = key->e_bitlen; + ret = starfive_rsa_start(ctx, rctx->rsa_data, key->e, + key->n, key->key_sz); + } else { + key->bitlen = key->d_bitlen; + ret = starfive_rsa_start(ctx, rctx->rsa_data, key->d, + key->n, key->key_sz); + } + + if (ret) + goto err_rsa_crypt; + + sg_copy_buffer(rctx->out_sg, sg_nents(rctx->out_sg), + rctx->rsa_data, key->key_sz, 0, 0); + +err_rsa_crypt: + writel(STARFIVE_RSA_RESET, cryp->base + STARFIVE_PKA_CACR_OFFSET); + return ret; +} + +static int starfive_rsa_enc(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); + struct starfive_cryp_dev *cryp = ctx->cryp; + struct starfive_rsa_key *key = &ctx->rsa_key; + struct starfive_cryp_request_ctx *rctx = akcipher_request_ctx(req); + int ret; + + if (!key->key_sz) { + akcipher_request_set_tfm(req, ctx->akcipher_fbk); + ret = crypto_akcipher_encrypt(req); + akcipher_request_set_tfm(req, tfm); + return ret; + } + + if (unlikely(!key->n || !key->e)) + return -EINVAL; + + if (req->dst_len < key->key_sz) + return dev_err_probe(cryp->dev, -EOVERFLOW, + "Output buffer length less than parameter n\n"); + + rctx->in_sg = req->src; + rctx->out_sg = req->dst; + rctx->total = req->src_len; + ctx->rctx = rctx; + + return starfive_rsa_enc_core(ctx, 1); +} + +static int starfive_rsa_dec(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); + struct starfive_cryp_dev *cryp = ctx->cryp; + struct starfive_rsa_key *key = &ctx->rsa_key; + struct starfive_cryp_request_ctx *rctx = akcipher_request_ctx(req); + int ret; + + if (!key->key_sz) { + akcipher_request_set_tfm(req, ctx->akcipher_fbk); + ret = crypto_akcipher_decrypt(req); + akcipher_request_set_tfm(req, tfm); + return ret; + } + + if (unlikely(!key->n || !key->d)) + return -EINVAL; + + if (req->dst_len < key->key_sz) + return dev_err_probe(cryp->dev, -EOVERFLOW, + "Output buffer length less than parameter n\n"); + + rctx->in_sg = req->src; + rctx->out_sg = req->dst; + ctx->rctx = rctx; + rctx->total = req->src_len; + + return starfive_rsa_enc_core(ctx, 0); +} + +static int starfive_rsa_set_n(struct starfive_rsa_key *rsa_key, + const char *value, size_t vlen) +{ + const char *ptr = value; + unsigned int bitslen; + int ret; + + while (!*ptr && vlen) { + ptr++; + vlen--; + } + rsa_key->key_sz = vlen; + bitslen = rsa_key->key_sz << 3; + + /* check valid key size */ + if (bitslen & 0x1f) + return -EINVAL; + + ret = -ENOMEM; + rsa_key->n = kmemdup(ptr, rsa_key->key_sz, GFP_KERNEL); + if (!rsa_key->n) + goto err; + + return 0; + err: + rsa_key->key_sz = 0; + rsa_key->n = NULL; + starfive_rsa_free_key(rsa_key); + return ret; +} + +static int starfive_rsa_set_e(struct starfive_rsa_key *rsa_key, + const char *value, size_t vlen) +{ + const char *ptr = value; + unsigned char pt; + int loop; + + while (!*ptr && vlen) { + ptr++; + vlen--; + } + pt = *ptr; + + if (!rsa_key->key_sz || !vlen || vlen > rsa_key->key_sz) { + rsa_key->e = NULL; + return -EINVAL; + } + + rsa_key->e = kzalloc(rsa_key->key_sz, GFP_KERNEL); + if (!rsa_key->e) + return -ENOMEM; + + for (loop = 8; loop > 0; loop--) { + if (pt >> (loop - 1)) + break; + } + + rsa_key->e_bitlen = (vlen - 1) * 8 + loop; + + memcpy(rsa_key->e + (rsa_key->key_sz - vlen), ptr, vlen); + + return 0; +} + +static int starfive_rsa_set_d(struct starfive_rsa_key *rsa_key, + const char *value, size_t vlen) +{ + const char *ptr = value; + unsigned char pt; + int loop; + int ret; + + while (!*ptr && vlen) { + ptr++; + vlen--; + } + pt = *ptr; + + ret = -EINVAL; + if (!rsa_key->key_sz || !vlen || vlen > rsa_key->key_sz) + goto err; + + ret = -ENOMEM; + rsa_key->d = kzalloc(rsa_key->key_sz, GFP_KERNEL); + if (!rsa_key->d) + goto err; + + for (loop = 8; loop > 0; loop--) { + if (pt >> (loop - 1)) + break; + } + + rsa_key->d_bitlen = (vlen - 1) * 8 + loop; + + memcpy(rsa_key->d + (rsa_key->key_sz - vlen), ptr, vlen); + + return 0; + err: + rsa_key->d = NULL; + return ret; +} + +static int starfive_rsa_setkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen, bool private) +{ + struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); + struct rsa_key raw_key = {NULL}; + struct starfive_rsa_key *rsa_key = &ctx->rsa_key; + int ret; + + if (private) + ret = rsa_parse_priv_key(&raw_key, key, keylen); + else + ret = rsa_parse_pub_key(&raw_key, key, keylen); + if (ret < 0) + goto err; + + starfive_rsa_free_key(rsa_key); + + /* Use fallback for mod > 256 + 1 byte prefix */ + if (raw_key.n_sz > STARFIVE_RSA_MAX_KEYSZ + 1) + return 0; + + ret = starfive_rsa_set_n(rsa_key, raw_key.n, raw_key.n_sz); + if (ret) + return ret; + + ret = starfive_rsa_set_e(rsa_key, raw_key.e, raw_key.e_sz); + if (ret) + goto err; + + if (private) { + ret = starfive_rsa_set_d(rsa_key, raw_key.d, raw_key.d_sz); + if (ret) + goto err; + } + + if (!rsa_key->n || !rsa_key->e) { + ret = -EINVAL; + goto err; + } + + if (private && !rsa_key->d) { + ret = -EINVAL; + goto err; + } + + return 0; + err: + starfive_rsa_free_key(rsa_key); + return ret; +} + +static int starfive_rsa_set_pub_key(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); + int ret; + + ret = crypto_akcipher_set_pub_key(ctx->akcipher_fbk, key, keylen); + if (ret) + return ret; + + return starfive_rsa_setkey(tfm, key, keylen, false); +} + +static int starfive_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); + int ret; + + ret = crypto_akcipher_set_priv_key(ctx->akcipher_fbk, key, keylen); + if (ret) + return ret; + + return starfive_rsa_setkey(tfm, key, keylen, true); +} + +static unsigned int starfive_rsa_max_size(struct crypto_akcipher *tfm) +{ + struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); + + if (ctx->rsa_key.key_sz) + return ctx->rsa_key.key_sz; + + return crypto_akcipher_maxsize(ctx->akcipher_fbk); +} + +static int starfive_rsa_init_tfm(struct crypto_akcipher *tfm) +{ + struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); + + ctx->cryp = starfive_cryp_find_dev(ctx); + if (!ctx->cryp) + return -ENODEV; + + ctx->akcipher_fbk = crypto_alloc_akcipher("rsa-generic", 0, 0); + if (IS_ERR(ctx->akcipher_fbk)) + return PTR_ERR(ctx->akcipher_fbk); + + akcipher_set_reqsize(tfm, sizeof(struct starfive_cryp_request_ctx) + + sizeof(struct crypto_akcipher) + 32); + + return 0; +} + +static void starfive_rsa_exit_tfm(struct crypto_akcipher *tfm) +{ + struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); + struct starfive_rsa_key *key = (struct starfive_rsa_key *)&ctx->rsa_key; + + crypto_free_akcipher(ctx->akcipher_fbk); + starfive_rsa_free_key(key); +} + +static struct akcipher_alg starfive_rsa = { + .encrypt = starfive_rsa_enc, + .decrypt = starfive_rsa_dec, + .set_pub_key = starfive_rsa_set_pub_key, + .set_priv_key = starfive_rsa_set_priv_key, + .max_size = starfive_rsa_max_size, + .init = starfive_rsa_init_tfm, + .exit = starfive_rsa_exit_tfm, + .base = { + .cra_name = "rsa", + .cra_driver_name = "starfive-rsa", + .cra_flags = CRYPTO_ALG_TYPE_AKCIPHER | + CRYPTO_ALG_NEED_FALLBACK, + .cra_priority = 3000, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), + }, +}; + +int starfive_rsa_register_algs(void) +{ + return crypto_register_akcipher(&starfive_rsa); +} + +void starfive_rsa_unregister_algs(void) +{ + crypto_unregister_akcipher(&starfive_rsa); +} diff --git a/drivers/crypto/stm32/Kconfig b/drivers/crypto/stm32/Kconfig new file mode 100644 index 000000000..d6dc848c8 --- /dev/null +++ b/drivers/crypto/stm32/Kconfig @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: GPL-2.0-only +config CRYPTO_DEV_STM32_HASH + tristate "Support for STM32 hash accelerators" + depends on ARCH_STM32 || ARCH_U8500 + depends on HAS_DMA + select CRYPTO_HASH + select CRYPTO_MD5 + select CRYPTO_SHA1 + select CRYPTO_SHA256 + select CRYPTO_SHA512 + select CRYPTO_SHA3 + select CRYPTO_ENGINE + help + This enables support for the HASH hw accelerator which can be found + on STMicroelectronics STM32 SOC. + +config CRYPTO_DEV_STM32_CRYP + tristate "Support for STM32 cryp accelerators" + depends on ARCH_STM32 || ARCH_U8500 + select CRYPTO_HASH + select CRYPTO_ENGINE + select CRYPTO_LIB_DES + help + This enables support for the CRYP (AES/DES/TDES) hw accelerator which + can be found on STMicroelectronics STM32 SOC. diff --git a/drivers/crypto/stm32/Makefile b/drivers/crypto/stm32/Makefile new file mode 100644 index 000000000..c63004026 --- /dev/null +++ b/drivers/crypto/stm32/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_STM32_HASH) += stm32-hash.o +obj-$(CONFIG_CRYPTO_DEV_STM32_CRYP) += stm32-cryp.o diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c new file mode 100644 index 000000000..3fc2db290 --- /dev/null +++ b/drivers/crypto/stm32/stm32-cryp.c @@ -0,0 +1,2776 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) STMicroelectronics SA 2017 + * Author: Fabien Dessenne + * Ux500 support taken from snippets in the old Ux500 cryp driver + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DRIVER_NAME "stm32-cryp" + +/* Bit [0] encrypt / decrypt */ +#define FLG_ENCRYPT BIT(0) +/* Bit [8..1] algo & operation mode */ +#define FLG_AES BIT(1) +#define FLG_DES BIT(2) +#define FLG_TDES BIT(3) +#define FLG_ECB BIT(4) +#define FLG_CBC BIT(5) +#define FLG_CTR BIT(6) +#define FLG_GCM BIT(7) +#define FLG_CCM BIT(8) +/* Mode mask = bits [15..0] */ +#define FLG_MODE_MASK GENMASK(15, 0) +/* Bit [31..16] status */ +#define FLG_IN_OUT_DMA BIT(16) +#define FLG_HEADER_DMA BIT(17) + +/* Registers */ +#define CRYP_CR 0x00000000 +#define CRYP_SR 0x00000004 +#define CRYP_DIN 0x00000008 +#define CRYP_DOUT 0x0000000C +#define CRYP_DMACR 0x00000010 +#define CRYP_IMSCR 0x00000014 +#define CRYP_RISR 0x00000018 +#define CRYP_MISR 0x0000001C +#define CRYP_K0LR 0x00000020 +#define CRYP_K0RR 0x00000024 +#define CRYP_K1LR 0x00000028 +#define CRYP_K1RR 0x0000002C +#define CRYP_K2LR 0x00000030 +#define CRYP_K2RR 0x00000034 +#define CRYP_K3LR 0x00000038 +#define CRYP_K3RR 0x0000003C +#define CRYP_IV0LR 0x00000040 +#define CRYP_IV0RR 0x00000044 +#define CRYP_IV1LR 0x00000048 +#define CRYP_IV1RR 0x0000004C +#define CRYP_CSGCMCCM0R 0x00000050 +#define CRYP_CSGCM0R 0x00000070 + +#define UX500_CRYP_CR 0x00000000 +#define UX500_CRYP_SR 0x00000004 +#define UX500_CRYP_DIN 0x00000008 +#define UX500_CRYP_DINSIZE 0x0000000C +#define UX500_CRYP_DOUT 0x00000010 +#define UX500_CRYP_DOUSIZE 0x00000014 +#define UX500_CRYP_DMACR 0x00000018 +#define UX500_CRYP_IMSC 0x0000001C +#define UX500_CRYP_RIS 0x00000020 +#define UX500_CRYP_MIS 0x00000024 +#define UX500_CRYP_K1L 0x00000028 +#define UX500_CRYP_K1R 0x0000002C +#define UX500_CRYP_K2L 0x00000030 +#define UX500_CRYP_K2R 0x00000034 +#define UX500_CRYP_K3L 0x00000038 +#define UX500_CRYP_K3R 0x0000003C +#define UX500_CRYP_K4L 0x00000040 +#define UX500_CRYP_K4R 0x00000044 +#define UX500_CRYP_IV0L 0x00000048 +#define UX500_CRYP_IV0R 0x0000004C +#define UX500_CRYP_IV1L 0x00000050 +#define UX500_CRYP_IV1R 0x00000054 + +/* Registers values */ +#define CR_DEC_NOT_ENC 0x00000004 +#define CR_TDES_ECB 0x00000000 +#define CR_TDES_CBC 0x00000008 +#define CR_DES_ECB 0x00000010 +#define CR_DES_CBC 0x00000018 +#define CR_AES_ECB 0x00000020 +#define CR_AES_CBC 0x00000028 +#define CR_AES_CTR 0x00000030 +#define CR_AES_KP 0x00000038 /* Not on Ux500 */ +#define CR_AES_XTS 0x00000038 /* Only on Ux500 */ +#define CR_AES_GCM 0x00080000 +#define CR_AES_CCM 0x00080008 +#define CR_AES_UNKNOWN 0xFFFFFFFF +#define CR_ALGO_MASK 0x00080038 +#define CR_DATA32 0x00000000 +#define CR_DATA16 0x00000040 +#define CR_DATA8 0x00000080 +#define CR_DATA1 0x000000C0 +#define CR_KEY128 0x00000000 +#define CR_KEY192 0x00000100 +#define CR_KEY256 0x00000200 +#define CR_KEYRDEN 0x00000400 /* Only on Ux500 */ +#define CR_KSE 0x00000800 /* Only on Ux500 */ +#define CR_FFLUSH 0x00004000 +#define CR_CRYPEN 0x00008000 +#define CR_PH_INIT 0x00000000 +#define CR_PH_HEADER 0x00010000 +#define CR_PH_PAYLOAD 0x00020000 +#define CR_PH_FINAL 0x00030000 +#define CR_PH_MASK 0x00030000 +#define CR_NBPBL_SHIFT 20 + +#define SR_IFNF BIT(1) +#define SR_OFNE BIT(2) +#define SR_BUSY BIT(8) + +#define DMACR_DIEN BIT(0) +#define DMACR_DOEN BIT(1) + +#define IMSCR_IN BIT(0) +#define IMSCR_OUT BIT(1) + +#define MISR_IN BIT(0) +#define MISR_OUT BIT(1) + +/* Misc */ +#define AES_BLOCK_32 (AES_BLOCK_SIZE / sizeof(u32)) +#define GCM_CTR_INIT 2 +#define CRYP_AUTOSUSPEND_DELAY 50 + +#define CRYP_DMA_BURST_REG 4 + +enum stm32_dma_mode { + NO_DMA, + DMA_PLAIN_SG, + DMA_NEED_SG_TRUNC +}; + +struct stm32_cryp_caps { + bool aeads_support; + bool linear_aes_key; + bool kp_mode; + bool iv_protection; + bool swap_final; + bool padding_wa; + u32 cr; + u32 sr; + u32 din; + u32 dout; + u32 dmacr; + u32 imsc; + u32 mis; + u32 k1l; + u32 k1r; + u32 k3r; + u32 iv0l; + u32 iv0r; + u32 iv1l; + u32 iv1r; +}; + +struct stm32_cryp_ctx { + struct stm32_cryp *cryp; + int keylen; + __be32 key[AES_KEYSIZE_256 / sizeof(u32)]; + unsigned long flags; +}; + +struct stm32_cryp_reqctx { + unsigned long mode; +}; + +struct stm32_cryp { + struct list_head list; + struct device *dev; + void __iomem *regs; + phys_addr_t phys_base; + struct clk *clk; + unsigned long flags; + u32 irq_status; + const struct stm32_cryp_caps *caps; + struct stm32_cryp_ctx *ctx; + + struct crypto_engine *engine; + + struct skcipher_request *req; + struct aead_request *areq; + + size_t authsize; + size_t hw_blocksize; + + size_t payload_in; + size_t header_in; + size_t payload_out; + + /* DMA process fields */ + struct scatterlist *in_sg; + struct scatterlist *header_sg; + struct scatterlist *out_sg; + size_t in_sg_len; + size_t header_sg_len; + size_t out_sg_len; + struct completion dma_completion; + + struct dma_chan *dma_lch_in; + struct dma_chan *dma_lch_out; + enum stm32_dma_mode dma_mode; + + /* IT process fields */ + struct scatter_walk in_walk; + struct scatter_walk out_walk; + + __be32 last_ctr[4]; + u32 gcm_ctr; +}; + +struct stm32_cryp_list { + struct list_head dev_list; + spinlock_t lock; /* protect dev_list */ +}; + +static struct stm32_cryp_list cryp_list = { + .dev_list = LIST_HEAD_INIT(cryp_list.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(cryp_list.lock), +}; + +static inline bool is_aes(struct stm32_cryp *cryp) +{ + return cryp->flags & FLG_AES; +} + +static inline bool is_des(struct stm32_cryp *cryp) +{ + return cryp->flags & FLG_DES; +} + +static inline bool is_tdes(struct stm32_cryp *cryp) +{ + return cryp->flags & FLG_TDES; +} + +static inline bool is_ecb(struct stm32_cryp *cryp) +{ + return cryp->flags & FLG_ECB; +} + +static inline bool is_cbc(struct stm32_cryp *cryp) +{ + return cryp->flags & FLG_CBC; +} + +static inline bool is_ctr(struct stm32_cryp *cryp) +{ + return cryp->flags & FLG_CTR; +} + +static inline bool is_gcm(struct stm32_cryp *cryp) +{ + return cryp->flags & FLG_GCM; +} + +static inline bool is_ccm(struct stm32_cryp *cryp) +{ + return cryp->flags & FLG_CCM; +} + +static inline bool is_encrypt(struct stm32_cryp *cryp) +{ + return cryp->flags & FLG_ENCRYPT; +} + +static inline bool is_decrypt(struct stm32_cryp *cryp) +{ + return !is_encrypt(cryp); +} + +static inline u32 stm32_cryp_read(struct stm32_cryp *cryp, u32 ofst) +{ + return readl_relaxed(cryp->regs + ofst); +} + +static inline void stm32_cryp_write(struct stm32_cryp *cryp, u32 ofst, u32 val) +{ + writel_relaxed(val, cryp->regs + ofst); +} + +static inline int stm32_cryp_wait_busy(struct stm32_cryp *cryp) +{ + u32 status; + + return readl_relaxed_poll_timeout(cryp->regs + cryp->caps->sr, status, + !(status & SR_BUSY), 10, 100000); +} + +static inline void stm32_cryp_enable(struct stm32_cryp *cryp) +{ + writel_relaxed(readl_relaxed(cryp->regs + cryp->caps->cr) | CR_CRYPEN, + cryp->regs + cryp->caps->cr); +} + +static inline int stm32_cryp_wait_enable(struct stm32_cryp *cryp) +{ + u32 status; + + return readl_relaxed_poll_timeout(cryp->regs + cryp->caps->cr, status, + !(status & CR_CRYPEN), 10, 100000); +} + +static inline int stm32_cryp_wait_input(struct stm32_cryp *cryp) +{ + u32 status; + + return readl_relaxed_poll_timeout_atomic(cryp->regs + cryp->caps->sr, status, + status & SR_IFNF, 1, 10); +} + +static inline int stm32_cryp_wait_output(struct stm32_cryp *cryp) +{ + u32 status; + + return readl_relaxed_poll_timeout_atomic(cryp->regs + cryp->caps->sr, status, + status & SR_OFNE, 1, 10); +} + +static inline void stm32_cryp_key_read_enable(struct stm32_cryp *cryp) +{ + writel_relaxed(readl_relaxed(cryp->regs + cryp->caps->cr) | CR_KEYRDEN, + cryp->regs + cryp->caps->cr); +} + +static inline void stm32_cryp_key_read_disable(struct stm32_cryp *cryp) +{ + writel_relaxed(readl_relaxed(cryp->regs + cryp->caps->cr) & ~CR_KEYRDEN, + cryp->regs + cryp->caps->cr); +} + +static void stm32_cryp_irq_read_data(struct stm32_cryp *cryp); +static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp); +static void stm32_cryp_irq_write_gcmccm_header(struct stm32_cryp *cryp); +static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp); +static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err); +static int stm32_cryp_dma_start(struct stm32_cryp *cryp); +static int stm32_cryp_it_start(struct stm32_cryp *cryp); + +static struct stm32_cryp *stm32_cryp_find_dev(struct stm32_cryp_ctx *ctx) +{ + struct stm32_cryp *cryp; + + spin_lock_bh(&cryp_list.lock); + if (!ctx->cryp) + ctx->cryp = list_first_entry_or_null(&cryp_list.dev_list, + struct stm32_cryp, list); + cryp = ctx->cryp; + spin_unlock_bh(&cryp_list.lock); + + return cryp; +} + +static void stm32_cryp_hw_write_iv(struct stm32_cryp *cryp, __be32 *iv) +{ + if (!iv) + return; + + stm32_cryp_write(cryp, cryp->caps->iv0l, be32_to_cpu(*iv++)); + stm32_cryp_write(cryp, cryp->caps->iv0r, be32_to_cpu(*iv++)); + + if (is_aes(cryp)) { + stm32_cryp_write(cryp, cryp->caps->iv1l, be32_to_cpu(*iv++)); + stm32_cryp_write(cryp, cryp->caps->iv1r, be32_to_cpu(*iv++)); + } +} + +static void stm32_cryp_get_iv(struct stm32_cryp *cryp) +{ + struct skcipher_request *req = cryp->req; + __be32 *tmp = (void *)req->iv; + + if (!tmp) + return; + + if (cryp->caps->iv_protection) + stm32_cryp_key_read_enable(cryp); + + *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv0l)); + *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv0r)); + + if (is_aes(cryp)) { + *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv1l)); + *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv1r)); + } + + if (cryp->caps->iv_protection) + stm32_cryp_key_read_disable(cryp); +} + +/** + * ux500_swap_bits_in_byte() - mirror the bits in a byte + * @b: the byte to be mirrored + * + * The bits are swapped the following way: + * Byte b include bits 0-7, nibble 1 (n1) include bits 0-3 and + * nibble 2 (n2) bits 4-7. + * + * Nibble 1 (n1): + * (The "old" (moved) bit is replaced with a zero) + * 1. Move bit 6 and 7, 4 positions to the left. + * 2. Move bit 3 and 5, 2 positions to the left. + * 3. Move bit 1-4, 1 position to the left. + * + * Nibble 2 (n2): + * 1. Move bit 0 and 1, 4 positions to the right. + * 2. Move bit 2 and 4, 2 positions to the right. + * 3. Move bit 3-6, 1 position to the right. + * + * Combine the two nibbles to a complete and swapped byte. + */ +static inline u8 ux500_swap_bits_in_byte(u8 b) +{ +#define R_SHIFT_4_MASK 0xc0 /* Bits 6 and 7, right shift 4 */ +#define R_SHIFT_2_MASK 0x28 /* (After right shift 4) Bits 3 and 5, + right shift 2 */ +#define R_SHIFT_1_MASK 0x1e /* (After right shift 2) Bits 1-4, + right shift 1 */ +#define L_SHIFT_4_MASK 0x03 /* Bits 0 and 1, left shift 4 */ +#define L_SHIFT_2_MASK 0x14 /* (After left shift 4) Bits 2 and 4, + left shift 2 */ +#define L_SHIFT_1_MASK 0x78 /* (After left shift 1) Bits 3-6, + left shift 1 */ + + u8 n1; + u8 n2; + + /* Swap most significant nibble */ + /* Right shift 4, bits 6 and 7 */ + n1 = ((b & R_SHIFT_4_MASK) >> 4) | (b & ~(R_SHIFT_4_MASK >> 4)); + /* Right shift 2, bits 3 and 5 */ + n1 = ((n1 & R_SHIFT_2_MASK) >> 2) | (n1 & ~(R_SHIFT_2_MASK >> 2)); + /* Right shift 1, bits 1-4 */ + n1 = (n1 & R_SHIFT_1_MASK) >> 1; + + /* Swap least significant nibble */ + /* Left shift 4, bits 0 and 1 */ + n2 = ((b & L_SHIFT_4_MASK) << 4) | (b & ~(L_SHIFT_4_MASK << 4)); + /* Left shift 2, bits 2 and 4 */ + n2 = ((n2 & L_SHIFT_2_MASK) << 2) | (n2 & ~(L_SHIFT_2_MASK << 2)); + /* Left shift 1, bits 3-6 */ + n2 = (n2 & L_SHIFT_1_MASK) << 1; + + return n1 | n2; +} + +/** + * ux500_swizzle_key() - Shuffle around words and bits in the AES key + * @in: key to swizzle + * @out: swizzled key + * @len: length of key, in bytes + * + * This "key swizzling procedure" is described in the examples in the + * DB8500 design specification. There is no real description of why + * the bits have been arranged like this in the hardware. + */ +static inline void ux500_swizzle_key(const u8 *in, u8 *out, u32 len) +{ + int i = 0; + int bpw = sizeof(u32); + int j; + int index = 0; + + j = len - bpw; + while (j >= 0) { + for (i = 0; i < bpw; i++) { + index = len - j - bpw + i; + out[j + i] = + ux500_swap_bits_in_byte(in[index]); + } + j -= bpw; + } +} + +static void stm32_cryp_hw_write_key(struct stm32_cryp *c) +{ + unsigned int i; + int r_id; + + if (is_des(c)) { + stm32_cryp_write(c, c->caps->k1l, be32_to_cpu(c->ctx->key[0])); + stm32_cryp_write(c, c->caps->k1r, be32_to_cpu(c->ctx->key[1])); + return; + } + + /* + * On the Ux500 the AES key is considered as a single bit sequence + * of 128, 192 or 256 bits length. It is written linearly into the + * registers from K1L and down, and need to be processed to become + * a proper big-endian bit sequence. + */ + if (is_aes(c) && c->caps->linear_aes_key) { + u32 tmpkey[8]; + + ux500_swizzle_key((u8 *)c->ctx->key, + (u8 *)tmpkey, c->ctx->keylen); + + r_id = c->caps->k1l; + for (i = 0; i < c->ctx->keylen / sizeof(u32); i++, r_id += 4) + stm32_cryp_write(c, r_id, tmpkey[i]); + + return; + } + + r_id = c->caps->k3r; + for (i = c->ctx->keylen / sizeof(u32); i > 0; i--, r_id -= 4) + stm32_cryp_write(c, r_id, be32_to_cpu(c->ctx->key[i - 1])); +} + +static u32 stm32_cryp_get_hw_mode(struct stm32_cryp *cryp) +{ + if (is_aes(cryp) && is_ecb(cryp)) + return CR_AES_ECB; + + if (is_aes(cryp) && is_cbc(cryp)) + return CR_AES_CBC; + + if (is_aes(cryp) && is_ctr(cryp)) + return CR_AES_CTR; + + if (is_aes(cryp) && is_gcm(cryp)) + return CR_AES_GCM; + + if (is_aes(cryp) && is_ccm(cryp)) + return CR_AES_CCM; + + if (is_des(cryp) && is_ecb(cryp)) + return CR_DES_ECB; + + if (is_des(cryp) && is_cbc(cryp)) + return CR_DES_CBC; + + if (is_tdes(cryp) && is_ecb(cryp)) + return CR_TDES_ECB; + + if (is_tdes(cryp) && is_cbc(cryp)) + return CR_TDES_CBC; + + dev_err(cryp->dev, "Unknown mode\n"); + return CR_AES_UNKNOWN; +} + +static unsigned int stm32_cryp_get_input_text_len(struct stm32_cryp *cryp) +{ + return is_encrypt(cryp) ? cryp->areq->cryptlen : + cryp->areq->cryptlen - cryp->authsize; +} + +static int stm32_cryp_gcm_init(struct stm32_cryp *cryp, u32 cfg) +{ + int ret; + __be32 iv[4]; + + /* Phase 1 : init */ + memcpy(iv, cryp->areq->iv, 12); + iv[3] = cpu_to_be32(GCM_CTR_INIT); + cryp->gcm_ctr = GCM_CTR_INIT; + stm32_cryp_hw_write_iv(cryp, iv); + + stm32_cryp_write(cryp, cryp->caps->cr, cfg | CR_PH_INIT | CR_CRYPEN); + + /* Wait for end of processing */ + ret = stm32_cryp_wait_enable(cryp); + if (ret) { + dev_err(cryp->dev, "Timeout (gcm init)\n"); + return ret; + } + + /* Prepare next phase */ + if (cryp->areq->assoclen) { + cfg |= CR_PH_HEADER; + stm32_cryp_write(cryp, cryp->caps->cr, cfg); + } else if (stm32_cryp_get_input_text_len(cryp)) { + cfg |= CR_PH_PAYLOAD; + stm32_cryp_write(cryp, cryp->caps->cr, cfg); + } + + return 0; +} + +static void stm32_crypt_gcmccm_end_header(struct stm32_cryp *cryp) +{ + u32 cfg; + int err; + + /* Check if whole header written */ + if (!cryp->header_in) { + /* Wait for completion */ + err = stm32_cryp_wait_busy(cryp); + if (err) { + dev_err(cryp->dev, "Timeout (gcm/ccm header)\n"); + stm32_cryp_write(cryp, cryp->caps->imsc, 0); + stm32_cryp_finish_req(cryp, err); + return; + } + + if (stm32_cryp_get_input_text_len(cryp)) { + /* Phase 3 : payload */ + cfg = stm32_cryp_read(cryp, cryp->caps->cr); + cfg &= ~CR_CRYPEN; + stm32_cryp_write(cryp, cryp->caps->cr, cfg); + + cfg &= ~CR_PH_MASK; + cfg |= CR_PH_PAYLOAD | CR_CRYPEN; + stm32_cryp_write(cryp, cryp->caps->cr, cfg); + } else { + /* + * Phase 4 : tag. + * Nothing to read, nothing to write, caller have to + * end request + */ + } + } +} + +static void stm32_cryp_write_ccm_first_header(struct stm32_cryp *cryp) +{ + size_t written; + size_t len; + u32 alen = cryp->areq->assoclen; + u32 block[AES_BLOCK_32] = {0}; + u8 *b8 = (u8 *)block; + + if (alen <= 65280) { + /* Write first u32 of B1 */ + b8[0] = (alen >> 8) & 0xFF; + b8[1] = alen & 0xFF; + len = 2; + } else { + /* Build the two first u32 of B1 */ + b8[0] = 0xFF; + b8[1] = 0xFE; + b8[2] = (alen & 0xFF000000) >> 24; + b8[3] = (alen & 0x00FF0000) >> 16; + b8[4] = (alen & 0x0000FF00) >> 8; + b8[5] = alen & 0x000000FF; + len = 6; + } + + written = min_t(size_t, AES_BLOCK_SIZE - len, alen); + + memcpy_from_scatterwalk((char *)block + len, &cryp->in_walk, written); + + writesl(cryp->regs + cryp->caps->din, block, AES_BLOCK_32); + + cryp->header_in -= written; + + stm32_crypt_gcmccm_end_header(cryp); +} + +static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg) +{ + int ret; + u32 iv_32[AES_BLOCK_32], b0_32[AES_BLOCK_32]; + u8 *iv = (u8 *)iv_32, *b0 = (u8 *)b0_32; + __be32 *bd; + u32 *d; + unsigned int i, textlen; + + /* Phase 1 : init. Firstly set the CTR value to 1 (not 0) */ + memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE); + memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1); + iv[AES_BLOCK_SIZE - 1] = 1; + stm32_cryp_hw_write_iv(cryp, (__be32 *)iv); + + /* Build B0 */ + memcpy(b0, iv, AES_BLOCK_SIZE); + + b0[0] |= (8 * ((cryp->authsize - 2) / 2)); + + if (cryp->areq->assoclen) + b0[0] |= 0x40; + + textlen = stm32_cryp_get_input_text_len(cryp); + + b0[AES_BLOCK_SIZE - 2] = textlen >> 8; + b0[AES_BLOCK_SIZE - 1] = textlen & 0xFF; + + /* Enable HW */ + stm32_cryp_write(cryp, cryp->caps->cr, cfg | CR_PH_INIT | CR_CRYPEN); + + /* Write B0 */ + d = (u32 *)b0; + bd = (__be32 *)b0; + + for (i = 0; i < AES_BLOCK_32; i++) { + u32 xd = d[i]; + + if (!cryp->caps->padding_wa) + xd = be32_to_cpu(bd[i]); + stm32_cryp_write(cryp, cryp->caps->din, xd); + } + + /* Wait for end of processing */ + ret = stm32_cryp_wait_enable(cryp); + if (ret) { + dev_err(cryp->dev, "Timeout (ccm init)\n"); + return ret; + } + + /* Prepare next phase */ + if (cryp->areq->assoclen) { + cfg |= CR_PH_HEADER | CR_CRYPEN; + stm32_cryp_write(cryp, cryp->caps->cr, cfg); + + /* Write first (special) block (may move to next phase [payload]) */ + stm32_cryp_write_ccm_first_header(cryp); + } else if (stm32_cryp_get_input_text_len(cryp)) { + cfg |= CR_PH_PAYLOAD; + stm32_cryp_write(cryp, cryp->caps->cr, cfg); + } + + return 0; +} + +static int stm32_cryp_hw_init(struct stm32_cryp *cryp) +{ + int ret; + u32 cfg, hw_mode; + + pm_runtime_get_sync(cryp->dev); + + /* Disable interrupt */ + stm32_cryp_write(cryp, cryp->caps->imsc, 0); + + /* Set configuration */ + cfg = CR_DATA8 | CR_FFLUSH; + + switch (cryp->ctx->keylen) { + case AES_KEYSIZE_128: + cfg |= CR_KEY128; + break; + + case AES_KEYSIZE_192: + cfg |= CR_KEY192; + break; + + default: + case AES_KEYSIZE_256: + cfg |= CR_KEY256; + break; + } + + hw_mode = stm32_cryp_get_hw_mode(cryp); + if (hw_mode == CR_AES_UNKNOWN) + return -EINVAL; + + /* AES ECB/CBC decrypt: run key preparation first */ + if (is_decrypt(cryp) && + ((hw_mode == CR_AES_ECB) || (hw_mode == CR_AES_CBC))) { + /* Configure in key preparation mode */ + if (cryp->caps->kp_mode) + stm32_cryp_write(cryp, cryp->caps->cr, + cfg | CR_AES_KP); + else + stm32_cryp_write(cryp, + cryp->caps->cr, cfg | CR_AES_ECB | CR_KSE); + + /* Set key only after full configuration done */ + stm32_cryp_hw_write_key(cryp); + + /* Start prepare key */ + stm32_cryp_enable(cryp); + /* Wait for end of processing */ + ret = stm32_cryp_wait_busy(cryp); + if (ret) { + dev_err(cryp->dev, "Timeout (key preparation)\n"); + return ret; + } + + cfg |= hw_mode | CR_DEC_NOT_ENC; + + /* Apply updated config (Decrypt + algo) and flush */ + stm32_cryp_write(cryp, cryp->caps->cr, cfg); + } else { + cfg |= hw_mode; + if (is_decrypt(cryp)) + cfg |= CR_DEC_NOT_ENC; + + /* Apply config and flush */ + stm32_cryp_write(cryp, cryp->caps->cr, cfg); + + /* Set key only after configuration done */ + stm32_cryp_hw_write_key(cryp); + } + + switch (hw_mode) { + case CR_AES_GCM: + case CR_AES_CCM: + /* Phase 1 : init */ + if (hw_mode == CR_AES_CCM) + ret = stm32_cryp_ccm_init(cryp, cfg); + else + ret = stm32_cryp_gcm_init(cryp, cfg); + + if (ret) + return ret; + + break; + + case CR_DES_CBC: + case CR_TDES_CBC: + case CR_AES_CBC: + case CR_AES_CTR: + stm32_cryp_hw_write_iv(cryp, (__be32 *)cryp->req->iv); + break; + + default: + break; + } + + /* Enable now */ + stm32_cryp_enable(cryp); + + return 0; +} + +static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err) +{ + if (!err && (is_gcm(cryp) || is_ccm(cryp))) + /* Phase 4 : output tag */ + err = stm32_cryp_read_auth_tag(cryp); + + if (!err && (!(is_gcm(cryp) || is_ccm(cryp) || is_ecb(cryp)))) + stm32_cryp_get_iv(cryp); + + pm_runtime_put_autosuspend(cryp->dev); + + if (is_gcm(cryp) || is_ccm(cryp)) + crypto_finalize_aead_request(cryp->engine, cryp->areq, err); + else + crypto_finalize_skcipher_request(cryp->engine, cryp->req, err); +} + +static void stm32_cryp_header_dma_callback(void *param) +{ + struct stm32_cryp *cryp = (struct stm32_cryp *)param; + int ret; + u32 reg; + + dma_unmap_sg(cryp->dev, cryp->header_sg, cryp->header_sg_len, DMA_TO_DEVICE); + + reg = stm32_cryp_read(cryp, cryp->caps->dmacr); + stm32_cryp_write(cryp, cryp->caps->dmacr, reg & ~(DMACR_DOEN | DMACR_DIEN)); + + kfree(cryp->header_sg); + + reg = stm32_cryp_read(cryp, cryp->caps->cr); + + if (cryp->header_in) { + stm32_cryp_write(cryp, cryp->caps->cr, reg | CR_CRYPEN); + + ret = stm32_cryp_wait_input(cryp); + if (ret) { + dev_err(cryp->dev, "input header ready timeout after dma\n"); + stm32_cryp_finish_req(cryp, ret); + return; + } + stm32_cryp_irq_write_gcmccm_header(cryp); + WARN_ON(cryp->header_in); + } + + if (stm32_cryp_get_input_text_len(cryp)) { + /* Phase 3 : payload */ + reg = stm32_cryp_read(cryp, cryp->caps->cr); + stm32_cryp_write(cryp, cryp->caps->cr, reg & ~CR_CRYPEN); + + reg &= ~CR_PH_MASK; + reg |= CR_PH_PAYLOAD | CR_CRYPEN; + stm32_cryp_write(cryp, cryp->caps->cr, reg); + + if (cryp->flags & FLG_IN_OUT_DMA) { + ret = stm32_cryp_dma_start(cryp); + if (ret) + stm32_cryp_finish_req(cryp, ret); + } else { + stm32_cryp_it_start(cryp); + } + } else { + /* + * Phase 4 : tag. + * Nothing to read, nothing to write => end request + */ + stm32_cryp_finish_req(cryp, 0); + } +} + +static void stm32_cryp_dma_callback(void *param) +{ + struct stm32_cryp *cryp = (struct stm32_cryp *)param; + int ret; + u32 reg; + + complete(&cryp->dma_completion); /* completion to indicate no timeout */ + + dma_sync_sg_for_device(cryp->dev, cryp->out_sg, cryp->out_sg_len, DMA_FROM_DEVICE); + + if (cryp->in_sg != cryp->out_sg) + dma_unmap_sg(cryp->dev, cryp->in_sg, cryp->in_sg_len, DMA_TO_DEVICE); + + dma_unmap_sg(cryp->dev, cryp->out_sg, cryp->out_sg_len, DMA_FROM_DEVICE); + + reg = stm32_cryp_read(cryp, cryp->caps->dmacr); + stm32_cryp_write(cryp, cryp->caps->dmacr, reg & ~(DMACR_DOEN | DMACR_DIEN)); + + reg = stm32_cryp_read(cryp, cryp->caps->cr); + + if (is_gcm(cryp) || is_ccm(cryp)) { + kfree(cryp->in_sg); + kfree(cryp->out_sg); + } else { + if (cryp->in_sg != cryp->req->src) + kfree(cryp->in_sg); + if (cryp->out_sg != cryp->req->dst) + kfree(cryp->out_sg); + } + + if (cryp->payload_in) { + stm32_cryp_write(cryp, cryp->caps->cr, reg | CR_CRYPEN); + + ret = stm32_cryp_wait_input(cryp); + if (ret) { + dev_err(cryp->dev, "input ready timeout after dma\n"); + stm32_cryp_finish_req(cryp, ret); + return; + } + stm32_cryp_irq_write_data(cryp); + + ret = stm32_cryp_wait_output(cryp); + if (ret) { + dev_err(cryp->dev, "output ready timeout after dma\n"); + stm32_cryp_finish_req(cryp, ret); + return; + } + stm32_cryp_irq_read_data(cryp); + } + + stm32_cryp_finish_req(cryp, 0); +} + +static int stm32_cryp_header_dma_start(struct stm32_cryp *cryp) +{ + int ret; + struct dma_async_tx_descriptor *tx_in; + u32 reg; + size_t align_size; + + ret = dma_map_sg(cryp->dev, cryp->header_sg, cryp->header_sg_len, DMA_TO_DEVICE); + if (!ret) { + dev_err(cryp->dev, "dma_map_sg() error\n"); + return -ENOMEM; + } + + dma_sync_sg_for_device(cryp->dev, cryp->header_sg, cryp->header_sg_len, DMA_TO_DEVICE); + + tx_in = dmaengine_prep_slave_sg(cryp->dma_lch_in, cryp->header_sg, cryp->header_sg_len, + DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!tx_in) { + dev_err(cryp->dev, "IN prep_slave_sg() failed\n"); + return -EINVAL; + } + + tx_in->callback_param = cryp; + tx_in->callback = stm32_cryp_header_dma_callback; + + /* Advance scatterwalk to not DMA'ed data */ + align_size = ALIGN_DOWN(cryp->header_in, cryp->hw_blocksize); + scatterwalk_skip(&cryp->in_walk, align_size); + cryp->header_in -= align_size; + + ret = dma_submit_error(dmaengine_submit(tx_in)); + if (ret < 0) { + dev_err(cryp->dev, "DMA in submit failed\n"); + return ret; + } + dma_async_issue_pending(cryp->dma_lch_in); + + reg = stm32_cryp_read(cryp, cryp->caps->dmacr); + stm32_cryp_write(cryp, cryp->caps->dmacr, reg | DMACR_DIEN); + + return 0; +} + +static int stm32_cryp_dma_start(struct stm32_cryp *cryp) +{ + int ret; + size_t align_size; + struct dma_async_tx_descriptor *tx_in, *tx_out; + u32 reg; + + if (cryp->in_sg != cryp->out_sg) { + ret = dma_map_sg(cryp->dev, cryp->in_sg, cryp->in_sg_len, DMA_TO_DEVICE); + if (!ret) { + dev_err(cryp->dev, "dma_map_sg() error\n"); + return -ENOMEM; + } + } + + ret = dma_map_sg(cryp->dev, cryp->out_sg, cryp->out_sg_len, DMA_FROM_DEVICE); + if (!ret) { + dev_err(cryp->dev, "dma_map_sg() error\n"); + return -ENOMEM; + } + + dma_sync_sg_for_device(cryp->dev, cryp->in_sg, cryp->in_sg_len, DMA_TO_DEVICE); + + tx_in = dmaengine_prep_slave_sg(cryp->dma_lch_in, cryp->in_sg, cryp->in_sg_len, + DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!tx_in) { + dev_err(cryp->dev, "IN prep_slave_sg() failed\n"); + return -EINVAL; + } + + /* No callback necessary */ + tx_in->callback_param = cryp; + tx_in->callback = NULL; + + tx_out = dmaengine_prep_slave_sg(cryp->dma_lch_out, cryp->out_sg, cryp->out_sg_len, + DMA_DEV_TO_MEM, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!tx_out) { + dev_err(cryp->dev, "OUT prep_slave_sg() failed\n"); + return -EINVAL; + } + + reinit_completion(&cryp->dma_completion); + tx_out->callback = stm32_cryp_dma_callback; + tx_out->callback_param = cryp; + + /* Advance scatterwalk to not DMA'ed data */ + align_size = ALIGN_DOWN(cryp->payload_in, cryp->hw_blocksize); + scatterwalk_skip(&cryp->in_walk, align_size); + cryp->payload_in -= align_size; + + ret = dma_submit_error(dmaengine_submit(tx_in)); + if (ret < 0) { + dev_err(cryp->dev, "DMA in submit failed\n"); + return ret; + } + dma_async_issue_pending(cryp->dma_lch_in); + + /* Advance scatterwalk to not DMA'ed data */ + scatterwalk_skip(&cryp->out_walk, align_size); + cryp->payload_out -= align_size; + ret = dma_submit_error(dmaengine_submit(tx_out)); + if (ret < 0) { + dev_err(cryp->dev, "DMA out submit failed\n"); + return ret; + } + dma_async_issue_pending(cryp->dma_lch_out); + + reg = stm32_cryp_read(cryp, cryp->caps->dmacr); + stm32_cryp_write(cryp, cryp->caps->dmacr, reg | DMACR_DOEN | DMACR_DIEN); + + if (!wait_for_completion_timeout(&cryp->dma_completion, msecs_to_jiffies(1000))) { + dev_err(cryp->dev, "DMA out timed out\n"); + dmaengine_terminate_sync(cryp->dma_lch_out); + return -ETIMEDOUT; + } + + return 0; +} + +static int stm32_cryp_it_start(struct stm32_cryp *cryp) +{ + /* Enable interrupt and let the IRQ handler do everything */ + stm32_cryp_write(cryp, cryp->caps->imsc, IMSCR_IN | IMSCR_OUT); + + return 0; +} + +static int stm32_cryp_cipher_one_req(struct crypto_engine *engine, void *areq); + +static int stm32_cryp_init_tfm(struct crypto_skcipher *tfm) +{ + crypto_skcipher_set_reqsize(tfm, sizeof(struct stm32_cryp_reqctx)); + + return 0; +} + +static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq); + +static int stm32_cryp_aes_aead_init(struct crypto_aead *tfm) +{ + crypto_aead_set_reqsize(tfm, sizeof(struct stm32_cryp_reqctx)); + + return 0; +} + +static int stm32_cryp_crypt(struct skcipher_request *req, unsigned long mode) +{ + struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx( + crypto_skcipher_reqtfm(req)); + struct stm32_cryp_reqctx *rctx = skcipher_request_ctx(req); + struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx); + + if (!cryp) + return -ENODEV; + + rctx->mode = mode; + + return crypto_transfer_skcipher_request_to_engine(cryp->engine, req); +} + +static int stm32_cryp_aead_crypt(struct aead_request *req, unsigned long mode) +{ + struct stm32_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct stm32_cryp_reqctx *rctx = aead_request_ctx(req); + struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx); + + if (!cryp) + return -ENODEV; + + rctx->mode = mode; + + return crypto_transfer_aead_request_to_engine(cryp->engine, req); +} + +static int stm32_cryp_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx(tfm); + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + return 0; +} + +static int stm32_cryp_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) + return -EINVAL; + else + return stm32_cryp_setkey(tfm, key, keylen); +} + +static int stm32_cryp_des_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + return verify_skcipher_des_key(tfm, key) ?: + stm32_cryp_setkey(tfm, key, keylen); +} + +static int stm32_cryp_tdes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + return verify_skcipher_des3_key(tfm, key) ?: + stm32_cryp_setkey(tfm, key, keylen); +} + +static int stm32_cryp_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct stm32_cryp_ctx *ctx = crypto_aead_ctx(tfm); + + if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) + return -EINVAL; + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + return 0; +} + +static int stm32_cryp_aes_gcm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + switch (authsize) { + case 4: + case 8: + case 12: + case 13: + case 14: + case 15: + case 16: + break; + default: + return -EINVAL; + } + + return 0; +} + +static int stm32_cryp_aes_ccm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + switch (authsize) { + case 4: + case 6: + case 8: + case 10: + case 12: + case 14: + case 16: + break; + default: + return -EINVAL; + } + + return 0; +} + +static int stm32_cryp_aes_ecb_encrypt(struct skcipher_request *req) +{ + if (req->cryptlen % AES_BLOCK_SIZE) + return -EINVAL; + + if (req->cryptlen == 0) + return 0; + + return stm32_cryp_crypt(req, FLG_AES | FLG_ECB | FLG_ENCRYPT); +} + +static int stm32_cryp_aes_ecb_decrypt(struct skcipher_request *req) +{ + if (req->cryptlen % AES_BLOCK_SIZE) + return -EINVAL; + + if (req->cryptlen == 0) + return 0; + + return stm32_cryp_crypt(req, FLG_AES | FLG_ECB); +} + +static int stm32_cryp_aes_cbc_encrypt(struct skcipher_request *req) +{ + if (req->cryptlen % AES_BLOCK_SIZE) + return -EINVAL; + + if (req->cryptlen == 0) + return 0; + + return stm32_cryp_crypt(req, FLG_AES | FLG_CBC | FLG_ENCRYPT); +} + +static int stm32_cryp_aes_cbc_decrypt(struct skcipher_request *req) +{ + if (req->cryptlen % AES_BLOCK_SIZE) + return -EINVAL; + + if (req->cryptlen == 0) + return 0; + + return stm32_cryp_crypt(req, FLG_AES | FLG_CBC); +} + +static int stm32_cryp_aes_ctr_encrypt(struct skcipher_request *req) +{ + if (req->cryptlen == 0) + return 0; + + return stm32_cryp_crypt(req, FLG_AES | FLG_CTR | FLG_ENCRYPT); +} + +static int stm32_cryp_aes_ctr_decrypt(struct skcipher_request *req) +{ + if (req->cryptlen == 0) + return 0; + + return stm32_cryp_crypt(req, FLG_AES | FLG_CTR); +} + +static int stm32_cryp_aes_gcm_encrypt(struct aead_request *req) +{ + return stm32_cryp_aead_crypt(req, FLG_AES | FLG_GCM | FLG_ENCRYPT); +} + +static int stm32_cryp_aes_gcm_decrypt(struct aead_request *req) +{ + return stm32_cryp_aead_crypt(req, FLG_AES | FLG_GCM); +} + +static inline int crypto_ccm_check_iv(const u8 *iv) +{ + /* 2 <= L <= 8, so 1 <= L' <= 7. */ + if (iv[0] < 1 || iv[0] > 7) + return -EINVAL; + + return 0; +} + +static int stm32_cryp_aes_ccm_encrypt(struct aead_request *req) +{ + int err; + + err = crypto_ccm_check_iv(req->iv); + if (err) + return err; + + return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM | FLG_ENCRYPT); +} + +static int stm32_cryp_aes_ccm_decrypt(struct aead_request *req) +{ + int err; + + err = crypto_ccm_check_iv(req->iv); + if (err) + return err; + + return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM); +} + +static int stm32_cryp_des_ecb_encrypt(struct skcipher_request *req) +{ + if (req->cryptlen % DES_BLOCK_SIZE) + return -EINVAL; + + if (req->cryptlen == 0) + return 0; + + return stm32_cryp_crypt(req, FLG_DES | FLG_ECB | FLG_ENCRYPT); +} + +static int stm32_cryp_des_ecb_decrypt(struct skcipher_request *req) +{ + if (req->cryptlen % DES_BLOCK_SIZE) + return -EINVAL; + + if (req->cryptlen == 0) + return 0; + + return stm32_cryp_crypt(req, FLG_DES | FLG_ECB); +} + +static int stm32_cryp_des_cbc_encrypt(struct skcipher_request *req) +{ + if (req->cryptlen % DES_BLOCK_SIZE) + return -EINVAL; + + if (req->cryptlen == 0) + return 0; + + return stm32_cryp_crypt(req, FLG_DES | FLG_CBC | FLG_ENCRYPT); +} + +static int stm32_cryp_des_cbc_decrypt(struct skcipher_request *req) +{ + if (req->cryptlen % DES_BLOCK_SIZE) + return -EINVAL; + + if (req->cryptlen == 0) + return 0; + + return stm32_cryp_crypt(req, FLG_DES | FLG_CBC); +} + +static int stm32_cryp_tdes_ecb_encrypt(struct skcipher_request *req) +{ + if (req->cryptlen % DES_BLOCK_SIZE) + return -EINVAL; + + if (req->cryptlen == 0) + return 0; + + return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB | FLG_ENCRYPT); +} + +static int stm32_cryp_tdes_ecb_decrypt(struct skcipher_request *req) +{ + if (req->cryptlen % DES_BLOCK_SIZE) + return -EINVAL; + + if (req->cryptlen == 0) + return 0; + + return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB); +} + +static int stm32_cryp_tdes_cbc_encrypt(struct skcipher_request *req) +{ + if (req->cryptlen % DES_BLOCK_SIZE) + return -EINVAL; + + if (req->cryptlen == 0) + return 0; + + return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC | FLG_ENCRYPT); +} + +static int stm32_cryp_tdes_cbc_decrypt(struct skcipher_request *req) +{ + if (req->cryptlen % DES_BLOCK_SIZE) + return -EINVAL; + + if (req->cryptlen == 0) + return 0; + + return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC); +} + +static enum stm32_dma_mode stm32_cryp_dma_check_sg(struct scatterlist *test_sg, size_t len, + size_t block_size) +{ + struct scatterlist *sg; + int i; + + if (len <= 16) + return NO_DMA; /* Faster */ + + for_each_sg(test_sg, sg, sg_nents(test_sg), i) { + if (!IS_ALIGNED(sg->length, block_size) && !sg_is_last(sg)) + return NO_DMA; + + if (sg->offset % sizeof(u32)) + return NO_DMA; + + if (sg_is_last(sg) && !IS_ALIGNED(sg->length, AES_BLOCK_SIZE)) + return DMA_NEED_SG_TRUNC; + } + + return DMA_PLAIN_SG; +} + +static enum stm32_dma_mode stm32_cryp_dma_check(struct stm32_cryp *cryp, struct scatterlist *in_sg, + struct scatterlist *out_sg) +{ + enum stm32_dma_mode ret = DMA_PLAIN_SG; + + if (!is_aes(cryp)) + return NO_DMA; + + if (!cryp->dma_lch_in || !cryp->dma_lch_out) + return NO_DMA; + + ret = stm32_cryp_dma_check_sg(in_sg, cryp->payload_in, AES_BLOCK_SIZE); + if (ret == NO_DMA) + return ret; + + ret = stm32_cryp_dma_check_sg(out_sg, cryp->payload_out, AES_BLOCK_SIZE); + if (ret == NO_DMA) + return ret; + + /* Check CTR counter overflow */ + if (is_aes(cryp) && is_ctr(cryp)) { + u32 c; + __be32 iv3; + + memcpy(&iv3, &cryp->req->iv[3 * sizeof(u32)], sizeof(iv3)); + c = be32_to_cpu(iv3); + if ((c + cryp->payload_in) < cryp->payload_in) + return NO_DMA; + } + + /* Workaround */ + if (is_aes(cryp) && is_ctr(cryp) && ret == DMA_NEED_SG_TRUNC) + return NO_DMA; + + return ret; +} + +static int stm32_cryp_truncate_sg(struct scatterlist **new_sg, size_t *new_sg_len, + struct scatterlist *sg, off_t skip, size_t size) +{ + struct scatterlist *cur; + int alloc_sg_len; + + *new_sg_len = 0; + + if (!sg || !size) { + *new_sg = NULL; + return 0; + } + + alloc_sg_len = sg_nents_for_len(sg, skip + size); + if (alloc_sg_len < 0) + return alloc_sg_len; + + /* We allocate to much sg entry, but it is easier */ + *new_sg = kmalloc_objs(struct scatterlist, (size_t)alloc_sg_len); + if (!*new_sg) + return -ENOMEM; + + sg_init_table(*new_sg, (unsigned int)alloc_sg_len); + + cur = *new_sg; + while (sg && size) { + unsigned int len = sg->length; + unsigned int offset = sg->offset; + + if (skip > len) { + skip -= len; + sg = sg_next(sg); + continue; + } + + if (skip) { + len -= skip; + offset += skip; + skip = 0; + } + + if (size < len) + len = size; + + if (len > 0) { + (*new_sg_len)++; + size -= len; + sg_set_page(cur, sg_page(sg), len, offset); + if (size == 0) + sg_mark_end(cur); + cur = sg_next(cur); + } + + sg = sg_next(sg); + } + + return 0; +} + +static int stm32_cryp_cipher_prepare(struct stm32_cryp *cryp, struct scatterlist *in_sg, + struct scatterlist *out_sg) +{ + size_t align_size; + int ret; + + cryp->dma_mode = stm32_cryp_dma_check(cryp, in_sg, out_sg); + + scatterwalk_start(&cryp->in_walk, in_sg); + scatterwalk_start(&cryp->out_walk, out_sg); + + if (cryp->dma_mode == NO_DMA) { + cryp->flags &= ~FLG_IN_OUT_DMA; + + if (is_ctr(cryp)) + memset(cryp->last_ctr, 0, sizeof(cryp->last_ctr)); + + } else if (cryp->dma_mode == DMA_NEED_SG_TRUNC) { + + cryp->flags |= FLG_IN_OUT_DMA; + + align_size = ALIGN_DOWN(cryp->payload_in, cryp->hw_blocksize); + ret = stm32_cryp_truncate_sg(&cryp->in_sg, &cryp->in_sg_len, in_sg, 0, align_size); + if (ret) + return ret; + + ret = stm32_cryp_truncate_sg(&cryp->out_sg, &cryp->out_sg_len, out_sg, 0, + align_size); + if (ret) { + kfree(cryp->in_sg); + return ret; + } + } else { + cryp->flags |= FLG_IN_OUT_DMA; + + cryp->in_sg = in_sg; + cryp->out_sg = out_sg; + + ret = sg_nents_for_len(cryp->in_sg, cryp->payload_in); + if (ret < 0) + return ret; + cryp->in_sg_len = (size_t)ret; + + ret = sg_nents_for_len(out_sg, cryp->payload_out); + if (ret < 0) + return ret; + cryp->out_sg_len = (size_t)ret; + } + + return 0; +} + +static int stm32_cryp_aead_prepare(struct stm32_cryp *cryp, struct scatterlist *in_sg, + struct scatterlist *out_sg) +{ + size_t align_size; + off_t skip; + int ret, ret2; + + cryp->header_sg = NULL; + cryp->in_sg = NULL; + cryp->out_sg = NULL; + + if (!cryp->dma_lch_in || !cryp->dma_lch_out) { + cryp->dma_mode = NO_DMA; + cryp->flags &= ~(FLG_IN_OUT_DMA | FLG_HEADER_DMA); + + return 0; + } + + /* CCM hw_init may have advanced in header */ + skip = cryp->areq->assoclen - cryp->header_in; + + align_size = ALIGN_DOWN(cryp->header_in, cryp->hw_blocksize); + ret = stm32_cryp_truncate_sg(&cryp->header_sg, &cryp->header_sg_len, in_sg, skip, + align_size); + if (ret) + return ret; + + ret = stm32_cryp_dma_check_sg(cryp->header_sg, align_size, AES_BLOCK_SIZE); + if (ret == NO_DMA) { + /* We cannot DMA the header */ + kfree(cryp->header_sg); + cryp->header_sg = NULL; + + cryp->flags &= ~FLG_HEADER_DMA; + } else { + cryp->flags |= FLG_HEADER_DMA; + } + + /* Now skip all header to be at payload start */ + skip = cryp->areq->assoclen; + align_size = ALIGN_DOWN(cryp->payload_in, cryp->hw_blocksize); + ret = stm32_cryp_truncate_sg(&cryp->in_sg, &cryp->in_sg_len, in_sg, skip, align_size); + if (ret) { + kfree(cryp->header_sg); + return ret; + } + + /* For out buffer align_size is same as in buffer */ + ret = stm32_cryp_truncate_sg(&cryp->out_sg, &cryp->out_sg_len, out_sg, skip, align_size); + if (ret) { + kfree(cryp->header_sg); + kfree(cryp->in_sg); + return ret; + } + + ret = stm32_cryp_dma_check_sg(cryp->in_sg, align_size, AES_BLOCK_SIZE); + ret2 = stm32_cryp_dma_check_sg(cryp->out_sg, align_size, AES_BLOCK_SIZE); + if (ret == NO_DMA || ret2 == NO_DMA) { + kfree(cryp->in_sg); + cryp->in_sg = NULL; + + kfree(cryp->out_sg); + cryp->out_sg = NULL; + + cryp->flags &= ~FLG_IN_OUT_DMA; + } else { + cryp->flags |= FLG_IN_OUT_DMA; + } + + return 0; +} + +static int stm32_cryp_prepare_req(struct skcipher_request *req, + struct aead_request *areq) +{ + struct stm32_cryp_ctx *ctx; + struct stm32_cryp *cryp; + struct stm32_cryp_reqctx *rctx; + struct scatterlist *in_sg, *out_sg; + int ret; + + if (!req && !areq) + return -EINVAL; + + ctx = req ? crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)) : + crypto_aead_ctx(crypto_aead_reqtfm(areq)); + + cryp = ctx->cryp; + + rctx = req ? skcipher_request_ctx(req) : aead_request_ctx(areq); + rctx->mode &= FLG_MODE_MASK; + + cryp->flags = (cryp->flags & ~FLG_MODE_MASK) | rctx->mode; + cryp->hw_blocksize = is_aes(cryp) ? AES_BLOCK_SIZE : DES_BLOCK_SIZE; + cryp->ctx = ctx; + + if (req) { + cryp->req = req; + cryp->areq = NULL; + cryp->header_in = 0; + cryp->payload_in = req->cryptlen; + cryp->payload_out = req->cryptlen; + cryp->authsize = 0; + + in_sg = req->src; + out_sg = req->dst; + + ret = stm32_cryp_cipher_prepare(cryp, in_sg, out_sg); + if (ret) + return ret; + + ret = stm32_cryp_hw_init(cryp); + } else { + /* + * Length of input and output data: + * Encryption case: + * INPUT = AssocData || PlainText + * <- assoclen -> <- cryptlen -> + * + * OUTPUT = AssocData || CipherText || AuthTag + * <- assoclen -> <-- cryptlen --> <- authsize -> + * + * Decryption case: + * INPUT = AssocData || CipherTex || AuthTag + * <- assoclen ---> <---------- cryptlen ----------> + * + * OUTPUT = AssocData || PlainText + * <- assoclen -> <- cryptlen - authsize -> + */ + cryp->areq = areq; + cryp->req = NULL; + cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq)); + if (is_encrypt(cryp)) { + cryp->payload_in = areq->cryptlen; + cryp->header_in = areq->assoclen; + cryp->payload_out = areq->cryptlen; + } else { + cryp->payload_in = areq->cryptlen - cryp->authsize; + cryp->header_in = areq->assoclen; + cryp->payload_out = cryp->payload_in; + } + + in_sg = areq->src; + out_sg = areq->dst; + + scatterwalk_start(&cryp->in_walk, in_sg); + /* In output, jump after assoc data */ + scatterwalk_start_at_pos(&cryp->out_walk, out_sg, + areq->assoclen); + + ret = stm32_cryp_hw_init(cryp); + if (ret) + return ret; + + ret = stm32_cryp_aead_prepare(cryp, in_sg, out_sg); + } + + return ret; +} + +static int stm32_cryp_cipher_one_req(struct crypto_engine *engine, void *areq) +{ + struct skcipher_request *req = container_of(areq, + struct skcipher_request, + base); + struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx( + crypto_skcipher_reqtfm(req)); + struct stm32_cryp *cryp = ctx->cryp; + int ret; + + if (!cryp) + return -ENODEV; + + ret = stm32_cryp_prepare_req(req, NULL); + if (ret) + return ret; + + if (cryp->flags & FLG_IN_OUT_DMA) + ret = stm32_cryp_dma_start(cryp); + else + ret = stm32_cryp_it_start(cryp); + + if (ret == -ETIMEDOUT) + stm32_cryp_finish_req(cryp, ret); + + return ret; +} + +static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq) +{ + struct aead_request *req = container_of(areq, struct aead_request, + base); + struct stm32_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct stm32_cryp *cryp = ctx->cryp; + int err; + + if (!cryp) + return -ENODEV; + + err = stm32_cryp_prepare_req(NULL, req); + if (err) + return err; + + if (!stm32_cryp_get_input_text_len(cryp) && !cryp->header_in && + !(cryp->flags & FLG_HEADER_DMA)) { + /* No input data to process: get tag and finish */ + stm32_cryp_finish_req(cryp, 0); + return 0; + } + + if (cryp->flags & FLG_HEADER_DMA) + return stm32_cryp_header_dma_start(cryp); + + if (!cryp->header_in && cryp->flags & FLG_IN_OUT_DMA) + return stm32_cryp_dma_start(cryp); + + return stm32_cryp_it_start(cryp); +} + +static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp) +{ + u32 cfg, size_bit; + unsigned int i; + int ret = 0; + + /* Update Config */ + cfg = stm32_cryp_read(cryp, cryp->caps->cr); + + cfg &= ~CR_PH_MASK; + cfg |= CR_PH_FINAL; + cfg &= ~CR_DEC_NOT_ENC; + cfg |= CR_CRYPEN; + + stm32_cryp_write(cryp, cryp->caps->cr, cfg); + + if (is_gcm(cryp)) { + /* GCM: write aad and payload size (in bits) */ + size_bit = cryp->areq->assoclen * 8; + if (cryp->caps->swap_final) + size_bit = (__force u32)cpu_to_be32(size_bit); + + stm32_cryp_write(cryp, cryp->caps->din, 0); + stm32_cryp_write(cryp, cryp->caps->din, size_bit); + + size_bit = is_encrypt(cryp) ? cryp->areq->cryptlen : + cryp->areq->cryptlen - cryp->authsize; + size_bit *= 8; + if (cryp->caps->swap_final) + size_bit = (__force u32)cpu_to_be32(size_bit); + + stm32_cryp_write(cryp, cryp->caps->din, 0); + stm32_cryp_write(cryp, cryp->caps->din, size_bit); + } else { + /* CCM: write CTR0 */ + u32 iv32[AES_BLOCK_32]; + u8 *iv = (u8 *)iv32; + __be32 *biv = (__be32 *)iv32; + + memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE); + memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1); + + for (i = 0; i < AES_BLOCK_32; i++) { + u32 xiv = iv32[i]; + + if (!cryp->caps->padding_wa) + xiv = be32_to_cpu(biv[i]); + stm32_cryp_write(cryp, cryp->caps->din, xiv); + } + } + + /* Wait for output data */ + ret = stm32_cryp_wait_output(cryp); + if (ret) { + dev_err(cryp->dev, "Timeout (read tag)\n"); + return ret; + } + + if (is_encrypt(cryp)) { + u32 out_tag[AES_BLOCK_32]; + + /* Get and write tag */ + readsl(cryp->regs + cryp->caps->dout, out_tag, AES_BLOCK_32); + memcpy_to_scatterwalk(&cryp->out_walk, out_tag, cryp->authsize); + } else { + /* Get and check tag */ + u32 in_tag[AES_BLOCK_32], out_tag[AES_BLOCK_32]; + + memcpy_from_scatterwalk(in_tag, &cryp->in_walk, cryp->authsize); + readsl(cryp->regs + cryp->caps->dout, out_tag, AES_BLOCK_32); + + if (crypto_memneq(in_tag, out_tag, cryp->authsize)) + ret = -EBADMSG; + } + + /* Disable cryp */ + cfg &= ~CR_CRYPEN; + stm32_cryp_write(cryp, cryp->caps->cr, cfg); + + return ret; +} + +static void stm32_cryp_check_ctr_counter(struct stm32_cryp *cryp) +{ + u32 cr; + + if (unlikely(cryp->last_ctr[3] == cpu_to_be32(0xFFFFFFFF))) { + /* + * In this case, we need to increment manually the ctr counter, + * as HW doesn't handle the U32 carry. + */ + crypto_inc((u8 *)cryp->last_ctr, sizeof(cryp->last_ctr)); + + cr = stm32_cryp_read(cryp, cryp->caps->cr); + stm32_cryp_write(cryp, cryp->caps->cr, cr & ~CR_CRYPEN); + + stm32_cryp_hw_write_iv(cryp, cryp->last_ctr); + + stm32_cryp_write(cryp, cryp->caps->cr, cr); + } + + /* The IV registers are BE */ + cryp->last_ctr[0] = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv0l)); + cryp->last_ctr[1] = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv0r)); + cryp->last_ctr[2] = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv1l)); + cryp->last_ctr[3] = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv1r)); +} + +static void stm32_cryp_irq_read_data(struct stm32_cryp *cryp) +{ + u32 block[AES_BLOCK_32]; + + readsl(cryp->regs + cryp->caps->dout, block, cryp->hw_blocksize / sizeof(u32)); + memcpy_to_scatterwalk(&cryp->out_walk, block, min(cryp->hw_blocksize, + cryp->payload_out)); + cryp->payload_out -= min(cryp->hw_blocksize, cryp->payload_out); +} + +static void stm32_cryp_irq_write_block(struct stm32_cryp *cryp) +{ + u32 block[AES_BLOCK_32] = {0}; + + memcpy_from_scatterwalk(block, &cryp->in_walk, min(cryp->hw_blocksize, + cryp->payload_in)); + writesl(cryp->regs + cryp->caps->din, block, cryp->hw_blocksize / sizeof(u32)); + cryp->payload_in -= min(cryp->hw_blocksize, cryp->payload_in); +} + +static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp) +{ + int err; + u32 cfg, block[AES_BLOCK_32] = {0}; + unsigned int i; + + /* 'Special workaround' procedure described in the datasheet */ + + /* a) disable ip */ + stm32_cryp_write(cryp, cryp->caps->imsc, 0); + cfg = stm32_cryp_read(cryp, cryp->caps->cr); + cfg &= ~CR_CRYPEN; + stm32_cryp_write(cryp, cryp->caps->cr, cfg); + + /* b) Update IV1R */ + stm32_cryp_write(cryp, cryp->caps->iv1r, cryp->gcm_ctr - 2); + + /* c) change mode to CTR */ + cfg &= ~CR_ALGO_MASK; + cfg |= CR_AES_CTR; + stm32_cryp_write(cryp, cryp->caps->cr, cfg); + + /* a) enable IP */ + cfg |= CR_CRYPEN; + stm32_cryp_write(cryp, cryp->caps->cr, cfg); + + /* b) pad and write the last block */ + stm32_cryp_irq_write_block(cryp); + /* wait end of process */ + err = stm32_cryp_wait_output(cryp); + if (err) { + dev_err(cryp->dev, "Timeout (write gcm last data)\n"); + return stm32_cryp_finish_req(cryp, err); + } + + /* c) get and store encrypted data */ + /* + * Same code as stm32_cryp_irq_read_data(), but we want to store + * block value + */ + readsl(cryp->regs + cryp->caps->dout, block, cryp->hw_blocksize / sizeof(u32)); + + memcpy_to_scatterwalk(&cryp->out_walk, block, min(cryp->hw_blocksize, + cryp->payload_out)); + cryp->payload_out -= min(cryp->hw_blocksize, cryp->payload_out); + + /* d) change mode back to AES GCM */ + cfg &= ~CR_ALGO_MASK; + cfg |= CR_AES_GCM; + stm32_cryp_write(cryp, cryp->caps->cr, cfg); + + /* e) change phase to Final */ + cfg &= ~CR_PH_MASK; + cfg |= CR_PH_FINAL; + stm32_cryp_write(cryp, cryp->caps->cr, cfg); + + /* f) write padded data */ + writesl(cryp->regs + cryp->caps->din, block, AES_BLOCK_32); + + /* g) Empty fifo out */ + err = stm32_cryp_wait_output(cryp); + if (err) { + dev_err(cryp->dev, "Timeout (write gcm padded data)\n"); + return stm32_cryp_finish_req(cryp, err); + } + + for (i = 0; i < AES_BLOCK_32; i++) + stm32_cryp_read(cryp, cryp->caps->dout); + + /* h) run the he normal Final phase */ + stm32_cryp_finish_req(cryp, 0); +} + +static void stm32_cryp_irq_set_npblb(struct stm32_cryp *cryp) +{ + u32 cfg; + + /* disable ip, set NPBLB and reneable ip */ + cfg = stm32_cryp_read(cryp, cryp->caps->cr); + cfg &= ~CR_CRYPEN; + stm32_cryp_write(cryp, cryp->caps->cr, cfg); + + cfg |= (cryp->hw_blocksize - cryp->payload_in) << CR_NBPBL_SHIFT; + cfg |= CR_CRYPEN; + stm32_cryp_write(cryp, cryp->caps->cr, cfg); +} + +static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp) +{ + int err = 0; + u32 cfg, iv1tmp; + u32 cstmp1[AES_BLOCK_32], cstmp2[AES_BLOCK_32]; + u32 block[AES_BLOCK_32] = {0}; + unsigned int i; + + /* 'Special workaround' procedure described in the datasheet */ + + /* a) disable ip */ + stm32_cryp_write(cryp, cryp->caps->imsc, 0); + + cfg = stm32_cryp_read(cryp, cryp->caps->cr); + cfg &= ~CR_CRYPEN; + stm32_cryp_write(cryp, cryp->caps->cr, cfg); + + /* b) get IV1 from CRYP_CSGCMCCM7 */ + iv1tmp = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + 7 * 4); + + /* c) Load CRYP_CSGCMCCMxR */ + for (i = 0; i < ARRAY_SIZE(cstmp1); i++) + cstmp1[i] = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + i * 4); + + /* d) Write IV1R */ + stm32_cryp_write(cryp, cryp->caps->iv1r, iv1tmp); + + /* e) change mode to CTR */ + cfg &= ~CR_ALGO_MASK; + cfg |= CR_AES_CTR; + stm32_cryp_write(cryp, cryp->caps->cr, cfg); + + /* a) enable IP */ + cfg |= CR_CRYPEN; + stm32_cryp_write(cryp, cryp->caps->cr, cfg); + + /* b) pad and write the last block */ + stm32_cryp_irq_write_block(cryp); + /* wait end of process */ + err = stm32_cryp_wait_output(cryp); + if (err) { + dev_err(cryp->dev, "Timeout (write ccm padded data)\n"); + return stm32_cryp_finish_req(cryp, err); + } + + /* c) get and store decrypted data */ + /* + * Same code as stm32_cryp_irq_read_data(), but we want to store + * block value + */ + readsl(cryp->regs + cryp->caps->dout, block, cryp->hw_blocksize / sizeof(u32)); + + memcpy_to_scatterwalk(&cryp->out_walk, block, min(cryp->hw_blocksize, + cryp->payload_out)); + cryp->payload_out -= min(cryp->hw_blocksize, cryp->payload_out); + + /* d) Load again CRYP_CSGCMCCMxR */ + for (i = 0; i < ARRAY_SIZE(cstmp2); i++) + cstmp2[i] = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + i * 4); + + /* e) change mode back to AES CCM */ + cfg &= ~CR_ALGO_MASK; + cfg |= CR_AES_CCM; + stm32_cryp_write(cryp, cryp->caps->cr, cfg); + + /* f) change phase to header */ + cfg &= ~CR_PH_MASK; + cfg |= CR_PH_HEADER; + stm32_cryp_write(cryp, cryp->caps->cr, cfg); + + /* g) XOR and write padded data */ + for (i = 0; i < ARRAY_SIZE(block); i++) { + block[i] ^= cstmp1[i]; + block[i] ^= cstmp2[i]; + stm32_cryp_write(cryp, cryp->caps->din, block[i]); + } + + /* h) wait for completion */ + err = stm32_cryp_wait_busy(cryp); + if (err) + dev_err(cryp->dev, "Timeout (write ccm padded data)\n"); + + /* i) run the he normal Final phase */ + stm32_cryp_finish_req(cryp, err); +} + +static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp) +{ + if (unlikely(!cryp->payload_in)) { + dev_warn(cryp->dev, "No more data to process\n"); + return; + } + + if (unlikely(cryp->payload_in < AES_BLOCK_SIZE && + (stm32_cryp_get_hw_mode(cryp) == CR_AES_GCM) && + is_encrypt(cryp))) { + /* Padding for AES GCM encryption */ + if (cryp->caps->padding_wa) { + /* Special case 1 */ + stm32_cryp_irq_write_gcm_padded_data(cryp); + return; + } + + /* Setting padding bytes (NBBLB) */ + stm32_cryp_irq_set_npblb(cryp); + } + + if (unlikely((cryp->payload_in < AES_BLOCK_SIZE) && + (stm32_cryp_get_hw_mode(cryp) == CR_AES_CCM) && + is_decrypt(cryp))) { + /* Padding for AES CCM decryption */ + if (cryp->caps->padding_wa) { + /* Special case 2 */ + stm32_cryp_irq_write_ccm_padded_data(cryp); + return; + } + + /* Setting padding bytes (NBBLB) */ + stm32_cryp_irq_set_npblb(cryp); + } + + if (is_aes(cryp) && is_ctr(cryp)) + stm32_cryp_check_ctr_counter(cryp); + + stm32_cryp_irq_write_block(cryp); +} + +static void stm32_cryp_irq_write_gcmccm_header(struct stm32_cryp *cryp) +{ + u32 block[AES_BLOCK_32] = {0}; + size_t written; + + written = min(AES_BLOCK_SIZE, cryp->header_in); + + memcpy_from_scatterwalk(block, &cryp->in_walk, written); + + writesl(cryp->regs + cryp->caps->din, block, AES_BLOCK_32); + + cryp->header_in -= written; + + stm32_crypt_gcmccm_end_header(cryp); +} + +static irqreturn_t stm32_cryp_irq_thread(int irq, void *arg) +{ + struct stm32_cryp *cryp = arg; + u32 ph; + u32 it_mask = stm32_cryp_read(cryp, cryp->caps->imsc); + + if (cryp->irq_status & MISR_OUT) + /* Output FIFO IRQ: read data */ + stm32_cryp_irq_read_data(cryp); + + if (cryp->irq_status & MISR_IN) { + if (is_gcm(cryp) || is_ccm(cryp)) { + ph = stm32_cryp_read(cryp, cryp->caps->cr) & CR_PH_MASK; + if (unlikely(ph == CR_PH_HEADER)) + /* Write Header */ + stm32_cryp_irq_write_gcmccm_header(cryp); + else + /* Input FIFO IRQ: write data */ + stm32_cryp_irq_write_data(cryp); + if (is_gcm(cryp)) + cryp->gcm_ctr++; + } else { + /* Input FIFO IRQ: write data */ + stm32_cryp_irq_write_data(cryp); + } + } + + /* Mask useless interrupts */ + if (!cryp->payload_in && !cryp->header_in) + it_mask &= ~IMSCR_IN; + if (!cryp->payload_out) + it_mask &= ~IMSCR_OUT; + stm32_cryp_write(cryp, cryp->caps->imsc, it_mask); + + if (!cryp->payload_in && !cryp->header_in && !cryp->payload_out) { + local_bh_disable(); + stm32_cryp_finish_req(cryp, 0); + local_bh_enable(); + } + + return IRQ_HANDLED; +} + +static irqreturn_t stm32_cryp_irq(int irq, void *arg) +{ + struct stm32_cryp *cryp = arg; + + cryp->irq_status = stm32_cryp_read(cryp, cryp->caps->mis); + + return IRQ_WAKE_THREAD; +} + +static int stm32_cryp_dma_init(struct stm32_cryp *cryp) +{ + struct dma_slave_config dma_conf; + struct dma_chan *chan; + int ret; + + memset(&dma_conf, 0, sizeof(dma_conf)); + + dma_conf.direction = DMA_MEM_TO_DEV; + dma_conf.dst_addr = cryp->phys_base + cryp->caps->din; + dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + dma_conf.dst_maxburst = CRYP_DMA_BURST_REG; + dma_conf.device_fc = false; + + chan = dma_request_chan(cryp->dev, "in"); + if (IS_ERR(chan)) + return PTR_ERR(chan); + + cryp->dma_lch_in = chan; + ret = dmaengine_slave_config(cryp->dma_lch_in, &dma_conf); + if (ret) { + dma_release_channel(cryp->dma_lch_in); + cryp->dma_lch_in = NULL; + dev_err(cryp->dev, "Couldn't configure DMA in slave.\n"); + return ret; + } + + memset(&dma_conf, 0, sizeof(dma_conf)); + + dma_conf.direction = DMA_DEV_TO_MEM; + dma_conf.src_addr = cryp->phys_base + cryp->caps->dout; + dma_conf.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + dma_conf.src_maxburst = CRYP_DMA_BURST_REG; + dma_conf.device_fc = false; + + chan = dma_request_chan(cryp->dev, "out"); + if (IS_ERR(chan)) { + dma_release_channel(cryp->dma_lch_in); + cryp->dma_lch_in = NULL; + return PTR_ERR(chan); + } + + cryp->dma_lch_out = chan; + + ret = dmaengine_slave_config(cryp->dma_lch_out, &dma_conf); + if (ret) { + dma_release_channel(cryp->dma_lch_out); + cryp->dma_lch_out = NULL; + dev_err(cryp->dev, "Couldn't configure DMA out slave.\n"); + dma_release_channel(cryp->dma_lch_in); + cryp->dma_lch_in = NULL; + return ret; + } + + init_completion(&cryp->dma_completion); + + return 0; +} + +static struct skcipher_engine_alg crypto_algs[] = { +{ + .base = { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "stm32-ecb-aes", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), + .base.cra_alignmask = 0, + .base.cra_module = THIS_MODULE, + + .init = stm32_cryp_init_tfm, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = stm32_cryp_aes_setkey, + .encrypt = stm32_cryp_aes_ecb_encrypt, + .decrypt = stm32_cryp_aes_ecb_decrypt, + }, + .op = { + .do_one_request = stm32_cryp_cipher_one_req, + }, +}, +{ + .base = { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "stm32-cbc-aes", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), + .base.cra_alignmask = 0, + .base.cra_module = THIS_MODULE, + + .init = stm32_cryp_init_tfm, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = stm32_cryp_aes_setkey, + .encrypt = stm32_cryp_aes_cbc_encrypt, + .decrypt = stm32_cryp_aes_cbc_decrypt, + }, + .op = { + .do_one_request = stm32_cryp_cipher_one_req, + }, +}, +{ + .base = { + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "stm32-ctr-aes", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), + .base.cra_alignmask = 0, + .base.cra_module = THIS_MODULE, + + .init = stm32_cryp_init_tfm, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = stm32_cryp_aes_setkey, + .encrypt = stm32_cryp_aes_ctr_encrypt, + .decrypt = stm32_cryp_aes_ctr_decrypt, + }, + .op = { + .do_one_request = stm32_cryp_cipher_one_req, + }, +}, +{ + .base = { + .base.cra_name = "ecb(des)", + .base.cra_driver_name = "stm32-ecb-des", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), + .base.cra_alignmask = 0, + .base.cra_module = THIS_MODULE, + + .init = stm32_cryp_init_tfm, + .min_keysize = DES_BLOCK_SIZE, + .max_keysize = DES_BLOCK_SIZE, + .setkey = stm32_cryp_des_setkey, + .encrypt = stm32_cryp_des_ecb_encrypt, + .decrypt = stm32_cryp_des_ecb_decrypt, + }, + .op = { + .do_one_request = stm32_cryp_cipher_one_req, + }, +}, +{ + .base = { + .base.cra_name = "cbc(des)", + .base.cra_driver_name = "stm32-cbc-des", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), + .base.cra_alignmask = 0, + .base.cra_module = THIS_MODULE, + + .init = stm32_cryp_init_tfm, + .min_keysize = DES_BLOCK_SIZE, + .max_keysize = DES_BLOCK_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = stm32_cryp_des_setkey, + .encrypt = stm32_cryp_des_cbc_encrypt, + .decrypt = stm32_cryp_des_cbc_decrypt, + }, + .op = { + .do_one_request = stm32_cryp_cipher_one_req, + }, +}, +{ + .base = { + .base.cra_name = "ecb(des3_ede)", + .base.cra_driver_name = "stm32-ecb-des3", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), + .base.cra_alignmask = 0, + .base.cra_module = THIS_MODULE, + + .init = stm32_cryp_init_tfm, + .min_keysize = 3 * DES_BLOCK_SIZE, + .max_keysize = 3 * DES_BLOCK_SIZE, + .setkey = stm32_cryp_tdes_setkey, + .encrypt = stm32_cryp_tdes_ecb_encrypt, + .decrypt = stm32_cryp_tdes_ecb_decrypt, + }, + .op = { + .do_one_request = stm32_cryp_cipher_one_req, + }, +}, +{ + .base = { + .base.cra_name = "cbc(des3_ede)", + .base.cra_driver_name = "stm32-cbc-des3", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), + .base.cra_alignmask = 0, + .base.cra_module = THIS_MODULE, + + .init = stm32_cryp_init_tfm, + .min_keysize = 3 * DES_BLOCK_SIZE, + .max_keysize = 3 * DES_BLOCK_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = stm32_cryp_tdes_setkey, + .encrypt = stm32_cryp_tdes_cbc_encrypt, + .decrypt = stm32_cryp_tdes_cbc_decrypt, + }, + .op = { + .do_one_request = stm32_cryp_cipher_one_req, + }, +}, +}; + +static struct aead_engine_alg aead_algs[] = { +{ + .base.setkey = stm32_cryp_aes_aead_setkey, + .base.setauthsize = stm32_cryp_aes_gcm_setauthsize, + .base.encrypt = stm32_cryp_aes_gcm_encrypt, + .base.decrypt = stm32_cryp_aes_gcm_decrypt, + .base.init = stm32_cryp_aes_aead_init, + .base.ivsize = 12, + .base.maxauthsize = AES_BLOCK_SIZE, + + .base.base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "stm32-gcm-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct stm32_cryp_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .op = { + .do_one_request = stm32_cryp_aead_one_req, + }, +}, +{ + .base.setkey = stm32_cryp_aes_aead_setkey, + .base.setauthsize = stm32_cryp_aes_ccm_setauthsize, + .base.encrypt = stm32_cryp_aes_ccm_encrypt, + .base.decrypt = stm32_cryp_aes_ccm_decrypt, + .base.init = stm32_cryp_aes_aead_init, + .base.ivsize = AES_BLOCK_SIZE, + .base.maxauthsize = AES_BLOCK_SIZE, + + .base.base = { + .cra_name = "ccm(aes)", + .cra_driver_name = "stm32-ccm-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct stm32_cryp_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .op = { + .do_one_request = stm32_cryp_aead_one_req, + }, +}, +}; + +static const struct stm32_cryp_caps ux500_data = { + .aeads_support = false, + .linear_aes_key = true, + .kp_mode = false, + .iv_protection = true, + .swap_final = true, + .padding_wa = true, + .cr = UX500_CRYP_CR, + .sr = UX500_CRYP_SR, + .din = UX500_CRYP_DIN, + .dout = UX500_CRYP_DOUT, + .dmacr = UX500_CRYP_DMACR, + .imsc = UX500_CRYP_IMSC, + .mis = UX500_CRYP_MIS, + .k1l = UX500_CRYP_K1L, + .k1r = UX500_CRYP_K1R, + .k3r = UX500_CRYP_K3R, + .iv0l = UX500_CRYP_IV0L, + .iv0r = UX500_CRYP_IV0R, + .iv1l = UX500_CRYP_IV1L, + .iv1r = UX500_CRYP_IV1R, +}; + +static const struct stm32_cryp_caps f7_data = { + .aeads_support = true, + .linear_aes_key = false, + .kp_mode = true, + .iv_protection = false, + .swap_final = true, + .padding_wa = true, + .cr = CRYP_CR, + .sr = CRYP_SR, + .din = CRYP_DIN, + .dout = CRYP_DOUT, + .dmacr = CRYP_DMACR, + .imsc = CRYP_IMSCR, + .mis = CRYP_MISR, + .k1l = CRYP_K1LR, + .k1r = CRYP_K1RR, + .k3r = CRYP_K3RR, + .iv0l = CRYP_IV0LR, + .iv0r = CRYP_IV0RR, + .iv1l = CRYP_IV1LR, + .iv1r = CRYP_IV1RR, +}; + +static const struct stm32_cryp_caps mp1_data = { + .aeads_support = true, + .linear_aes_key = false, + .kp_mode = true, + .iv_protection = false, + .swap_final = false, + .padding_wa = false, + .cr = CRYP_CR, + .sr = CRYP_SR, + .din = CRYP_DIN, + .dout = CRYP_DOUT, + .dmacr = CRYP_DMACR, + .imsc = CRYP_IMSCR, + .mis = CRYP_MISR, + .k1l = CRYP_K1LR, + .k1r = CRYP_K1RR, + .k3r = CRYP_K3RR, + .iv0l = CRYP_IV0LR, + .iv0r = CRYP_IV0RR, + .iv1l = CRYP_IV1LR, + .iv1r = CRYP_IV1RR, +}; + +static const struct of_device_id stm32_dt_ids[] = { + { .compatible = "stericsson,ux500-cryp", .data = &ux500_data}, + { .compatible = "st,stm32f756-cryp", .data = &f7_data}, + { .compatible = "st,stm32mp1-cryp", .data = &mp1_data}, + {}, +}; +MODULE_DEVICE_TABLE(of, stm32_dt_ids); + +static int stm32_cryp_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct stm32_cryp *cryp; + struct reset_control *rst; + int irq, ret; + + cryp = devm_kzalloc(dev, sizeof(*cryp), GFP_KERNEL); + if (!cryp) + return -ENOMEM; + + cryp->caps = of_device_get_match_data(dev); + if (!cryp->caps) + return -ENODEV; + + cryp->dev = dev; + + cryp->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(cryp->regs)) + return PTR_ERR(cryp->regs); + + cryp->phys_base = platform_get_resource(pdev, IORESOURCE_MEM, 0)->start; + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + ret = devm_request_threaded_irq(dev, irq, stm32_cryp_irq, + stm32_cryp_irq_thread, IRQF_ONESHOT, + dev_name(dev), cryp); + if (ret) + return ret; + + cryp->clk = devm_clk_get(dev, NULL); + if (IS_ERR(cryp->clk)) { + dev_err_probe(dev, PTR_ERR(cryp->clk), "Could not get clock\n"); + + return PTR_ERR(cryp->clk); + } + + ret = clk_prepare_enable(cryp->clk); + if (ret) { + dev_err(cryp->dev, "Failed to enable clock\n"); + return ret; + } + + pm_runtime_set_autosuspend_delay(dev, CRYP_AUTOSUSPEND_DELAY); + pm_runtime_use_autosuspend(dev); + + pm_runtime_get_noresume(dev); + pm_runtime_set_active(dev); + pm_runtime_enable(dev); + + rst = devm_reset_control_get(dev, NULL); + if (IS_ERR(rst)) { + ret = PTR_ERR(rst); + if (ret == -EPROBE_DEFER) + goto err_rst; + } else { + reset_control_assert(rst); + udelay(2); + reset_control_deassert(rst); + } + + platform_set_drvdata(pdev, cryp); + + ret = stm32_cryp_dma_init(cryp); + switch (ret) { + case 0: + break; + case -ENODEV: + dev_dbg(dev, "DMA mode not available\n"); + break; + default: + goto err_dma; + } + + spin_lock(&cryp_list.lock); + list_add(&cryp->list, &cryp_list.dev_list); + spin_unlock(&cryp_list.lock); + + /* Initialize crypto engine */ + cryp->engine = crypto_engine_alloc_init(dev, 1); + if (!cryp->engine) { + dev_err(dev, "Could not init crypto engine\n"); + ret = -ENOMEM; + goto err_engine1; + } + + ret = crypto_engine_start(cryp->engine); + if (ret) { + dev_err(dev, "Could not start crypto engine\n"); + goto err_engine2; + } + + ret = crypto_engine_register_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs)); + if (ret) { + dev_err(dev, "Could not register algs\n"); + goto err_algs; + } + + if (cryp->caps->aeads_support) { + ret = crypto_engine_register_aeads(aead_algs, ARRAY_SIZE(aead_algs)); + if (ret) + goto err_aead_algs; + } + + dev_info(dev, "Initialized\n"); + + pm_runtime_put_sync(dev); + + return 0; + +err_aead_algs: + crypto_engine_unregister_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs)); +err_algs: +err_engine2: + crypto_engine_exit(cryp->engine); +err_engine1: + spin_lock(&cryp_list.lock); + list_del(&cryp->list); + spin_unlock(&cryp_list.lock); + + if (cryp->dma_lch_in) + dma_release_channel(cryp->dma_lch_in); + if (cryp->dma_lch_out) + dma_release_channel(cryp->dma_lch_out); +err_dma: +err_rst: + pm_runtime_disable(dev); + pm_runtime_put_noidle(dev); + + clk_disable_unprepare(cryp->clk); + + return ret; +} + +static void stm32_cryp_remove(struct platform_device *pdev) +{ + struct stm32_cryp *cryp = platform_get_drvdata(pdev); + int ret; + + ret = pm_runtime_get_sync(cryp->dev); + + if (cryp->caps->aeads_support) + crypto_engine_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs)); + crypto_engine_unregister_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs)); + + crypto_engine_exit(cryp->engine); + + spin_lock(&cryp_list.lock); + list_del(&cryp->list); + spin_unlock(&cryp_list.lock); + + if (cryp->dma_lch_in) + dma_release_channel(cryp->dma_lch_in); + + if (cryp->dma_lch_out) + dma_release_channel(cryp->dma_lch_out); + + pm_runtime_disable(cryp->dev); + pm_runtime_put_noidle(cryp->dev); + + if (ret >= 0) + clk_disable_unprepare(cryp->clk); +} + +#ifdef CONFIG_PM +static int stm32_cryp_runtime_suspend(struct device *dev) +{ + struct stm32_cryp *cryp = dev_get_drvdata(dev); + + clk_disable_unprepare(cryp->clk); + + return 0; +} + +static int stm32_cryp_runtime_resume(struct device *dev) +{ + struct stm32_cryp *cryp = dev_get_drvdata(dev); + int ret; + + ret = clk_prepare_enable(cryp->clk); + if (ret) { + dev_err(cryp->dev, "Failed to prepare_enable clock\n"); + return ret; + } + + return 0; +} +#endif + +static const struct dev_pm_ops stm32_cryp_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, + pm_runtime_force_resume) + SET_RUNTIME_PM_OPS(stm32_cryp_runtime_suspend, + stm32_cryp_runtime_resume, NULL) +}; + +static struct platform_driver stm32_cryp_driver = { + .probe = stm32_cryp_probe, + .remove = stm32_cryp_remove, + .driver = { + .name = DRIVER_NAME, + .pm = &stm32_cryp_pm_ops, + .of_match_table = stm32_dt_ids, + }, +}; + +module_platform_driver(stm32_cryp_driver); + +MODULE_AUTHOR("Fabien Dessenne "); +MODULE_DESCRIPTION("STMicroelectronics STM32 CRYP hardware driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c new file mode 100644 index 000000000..e6a1e9b9f --- /dev/null +++ b/drivers/crypto/stm32/stm32-hash.c @@ -0,0 +1,2536 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of STM32 Crypto driver for Linux. + * + * Copyright (C) 2017, STMicroelectronics - All Rights Reserved + * Author(s): Lionel DEBIEVE for STMicroelectronics. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define HASH_CR 0x00 +#define HASH_DIN 0x04 +#define HASH_STR 0x08 +#define HASH_UX500_HREG(x) (0x0c + ((x) * 0x04)) +#define HASH_IMR 0x20 +#define HASH_SR 0x24 +#define HASH_CSR(x) (0x0F8 + ((x) * 0x04)) +#define HASH_HREG(x) (0x310 + ((x) * 0x04)) +#define HASH_HWCFGR 0x3F0 +#define HASH_VER 0x3F4 +#define HASH_ID 0x3F8 + +/* Control Register */ +#define HASH_CR_INIT BIT(2) +#define HASH_CR_DMAE BIT(3) +#define HASH_CR_DATATYPE_POS 4 +#define HASH_CR_MODE BIT(6) +#define HASH_CR_ALGO_POS 7 +#define HASH_CR_MDMAT BIT(13) +#define HASH_CR_DMAA BIT(14) +#define HASH_CR_LKEY BIT(16) + +/* Interrupt */ +#define HASH_DINIE BIT(0) +#define HASH_DCIE BIT(1) + +/* Interrupt Mask */ +#define HASH_MASK_CALC_COMPLETION BIT(0) +#define HASH_MASK_DATA_INPUT BIT(1) + +/* Status Flags */ +#define HASH_SR_DATA_INPUT_READY BIT(0) +#define HASH_SR_OUTPUT_READY BIT(1) +#define HASH_SR_DMA_ACTIVE BIT(2) +#define HASH_SR_BUSY BIT(3) + +/* STR Register */ +#define HASH_STR_NBLW_MASK GENMASK(4, 0) +#define HASH_STR_DCAL BIT(8) + +/* HWCFGR Register */ +#define HASH_HWCFG_DMA_MASK GENMASK(3, 0) + +/* Context swap register */ +#define HASH_CSR_NB_SHA256_HMAC 54 +#define HASH_CSR_NB_SHA256 38 +#define HASH_CSR_NB_SHA512_HMAC 103 +#define HASH_CSR_NB_SHA512 91 +#define HASH_CSR_NB_SHA3_HMAC 88 +#define HASH_CSR_NB_SHA3 72 +#define HASH_CSR_NB_MAX HASH_CSR_NB_SHA512_HMAC + +#define HASH_FLAGS_INIT BIT(0) +#define HASH_FLAGS_OUTPUT_READY BIT(1) +#define HASH_FLAGS_CPU BIT(2) +#define HASH_FLAGS_DMA_ACTIVE BIT(3) +#define HASH_FLAGS_HMAC_INIT BIT(4) +#define HASH_FLAGS_HMAC_FINAL BIT(5) +#define HASH_FLAGS_HMAC_KEY BIT(6) +#define HASH_FLAGS_SHA3_MODE BIT(7) +#define HASH_FLAGS_FINAL BIT(15) +#define HASH_FLAGS_FINUP BIT(16) +#define HASH_FLAGS_ALGO_MASK GENMASK(20, 17) +#define HASH_FLAGS_ALGO_SHIFT 17 +#define HASH_FLAGS_ERRORS BIT(21) +#define HASH_FLAGS_EMPTY BIT(22) +#define HASH_FLAGS_HMAC BIT(23) +#define HASH_FLAGS_SGS_COPIED BIT(24) + +#define HASH_OP_UPDATE 1 +#define HASH_OP_FINAL 2 + +#define HASH_BURST_LEVEL 4 + +enum stm32_hash_data_format { + HASH_DATA_32_BITS = 0x0, + HASH_DATA_16_BITS = 0x1, + HASH_DATA_8_BITS = 0x2, + HASH_DATA_1_BIT = 0x3 +}; + +#define HASH_BUFLEN (SHA3_224_BLOCK_SIZE + 4) +#define HASH_MAX_KEY_SIZE (SHA512_BLOCK_SIZE * 8) + +enum stm32_hash_algo { + HASH_SHA1 = 0, + HASH_MD5 = 1, + HASH_SHA224 = 2, + HASH_SHA256 = 3, + HASH_SHA3_224 = 4, + HASH_SHA3_256 = 5, + HASH_SHA3_384 = 6, + HASH_SHA3_512 = 7, + HASH_SHA384 = 12, + HASH_SHA512 = 15, +}; + +enum ux500_hash_algo { + HASH_SHA256_UX500 = 0, + HASH_SHA1_UX500 = 1, +}; + +#define HASH_AUTOSUSPEND_DELAY 50 + +struct stm32_hash_ctx { + struct stm32_hash_dev *hdev; + struct crypto_shash *xtfm; + unsigned long flags; + + u8 key[HASH_MAX_KEY_SIZE]; + int keylen; +}; + +struct stm32_hash_state { + u32 flags; + + u16 bufcnt; + u16 blocklen; + + u8 buffer[HASH_BUFLEN] __aligned(sizeof(u32)); + + /* hash state */ + u32 hw_context[3 + HASH_CSR_NB_MAX]; +}; + +struct stm32_hash_request_ctx { + struct stm32_hash_dev *hdev; + unsigned long op; + + u8 digest[SHA512_DIGEST_SIZE] __aligned(sizeof(u32)); + size_t digcnt; + + struct scatterlist *sg; + struct scatterlist sgl[2]; /* scatterlist used to realize alignment */ + unsigned int offset; + unsigned int total; + struct scatterlist sg_key; + + dma_addr_t dma_addr; + size_t dma_ct; + int nents; + + u8 data_type; + + struct stm32_hash_state state; +}; + +struct stm32_hash_algs_info { + struct ahash_engine_alg *algs_list; + size_t size; +}; + +struct stm32_hash_pdata { + const int alg_shift; + const struct stm32_hash_algs_info *algs_info; + size_t algs_info_size; + bool has_sr; + bool has_mdmat; + bool context_secured; + bool broken_emptymsg; + bool ux500; +}; + +struct stm32_hash_dev { + struct list_head list; + struct device *dev; + struct clk *clk; + struct reset_control *rst; + void __iomem *io_base; + phys_addr_t phys_base; + u8 xmit_buf[HASH_BUFLEN] __aligned(sizeof(u32)); + u32 dma_mode; + bool polled; + + struct ahash_request *req; + struct crypto_engine *engine; + + unsigned long flags; + + struct dma_chan *dma_lch; + struct completion dma_completion; + + const struct stm32_hash_pdata *pdata; +}; + +struct stm32_hash_drv { + struct list_head dev_list; + spinlock_t lock; /* List protection access */ +}; + +static struct stm32_hash_drv stm32_hash = { + .dev_list = LIST_HEAD_INIT(stm32_hash.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(stm32_hash.lock), +}; + +static void stm32_hash_dma_callback(void *param); +static int stm32_hash_prepare_request(struct ahash_request *req); +static void stm32_hash_unprepare_request(struct ahash_request *req); + +static inline u32 stm32_hash_read(struct stm32_hash_dev *hdev, u32 offset) +{ + return readl_relaxed(hdev->io_base + offset); +} + +static inline void stm32_hash_write(struct stm32_hash_dev *hdev, + u32 offset, u32 value) +{ + writel_relaxed(value, hdev->io_base + offset); +} + +/** + * stm32_hash_wait_busy - wait until hash processor is available. It return an + * error if the hash core is processing a block of data for more than 10 ms. + * @hdev: the stm32_hash_dev device. + */ +static inline int stm32_hash_wait_busy(struct stm32_hash_dev *hdev) +{ + u32 status; + + /* The Ux500 lacks the special status register, we poll the DCAL bit instead */ + if (!hdev->pdata->has_sr) + return readl_relaxed_poll_timeout(hdev->io_base + HASH_STR, status, + !(status & HASH_STR_DCAL), 10, 10000); + + return readl_relaxed_poll_timeout(hdev->io_base + HASH_SR, status, + !(status & HASH_SR_BUSY), 10, 10000); +} + +/** + * stm32_hash_set_nblw - set the number of valid bytes in the last word. + * @hdev: the stm32_hash_dev device. + * @length: the length of the final word. + */ +static void stm32_hash_set_nblw(struct stm32_hash_dev *hdev, int length) +{ + u32 reg; + + reg = stm32_hash_read(hdev, HASH_STR); + reg &= ~(HASH_STR_NBLW_MASK); + reg |= (8U * ((length) % 4U)); + stm32_hash_write(hdev, HASH_STR, reg); +} + +static int stm32_hash_write_key(struct stm32_hash_dev *hdev) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(hdev->req); + struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm); + u32 reg; + int keylen = ctx->keylen; + void *key = ctx->key; + + if (keylen) { + stm32_hash_set_nblw(hdev, keylen); + + while (keylen > 0) { + stm32_hash_write(hdev, HASH_DIN, *(u32 *)key); + keylen -= 4; + key += 4; + } + + reg = stm32_hash_read(hdev, HASH_STR); + reg |= HASH_STR_DCAL; + stm32_hash_write(hdev, HASH_STR, reg); + + return -EINPROGRESS; + } + + return 0; +} + +/** + * stm32_hash_write_ctrl - Initialize the hash processor, only if + * HASH_FLAGS_INIT is set. + * @hdev: the stm32_hash_dev device + */ +static void stm32_hash_write_ctrl(struct stm32_hash_dev *hdev) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(hdev->req); + struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm); + struct stm32_hash_state *state = &rctx->state; + u32 alg = (state->flags & HASH_FLAGS_ALGO_MASK) >> HASH_FLAGS_ALGO_SHIFT; + + u32 reg = HASH_CR_INIT; + + if (!(hdev->flags & HASH_FLAGS_INIT)) { + if (hdev->pdata->ux500) { + reg |= ((alg & BIT(0)) << HASH_CR_ALGO_POS); + } else { + if (hdev->pdata->alg_shift == HASH_CR_ALGO_POS) + reg |= ((alg & BIT(1)) << 17) | + ((alg & BIT(0)) << HASH_CR_ALGO_POS); + else + reg |= alg << hdev->pdata->alg_shift; + } + + reg |= (rctx->data_type << HASH_CR_DATATYPE_POS); + + if (state->flags & HASH_FLAGS_HMAC) { + hdev->flags |= HASH_FLAGS_HMAC; + reg |= HASH_CR_MODE; + if (ctx->keylen > crypto_ahash_blocksize(tfm)) + reg |= HASH_CR_LKEY; + } + + if (!hdev->polled) + stm32_hash_write(hdev, HASH_IMR, HASH_DCIE); + + stm32_hash_write(hdev, HASH_CR, reg); + + hdev->flags |= HASH_FLAGS_INIT; + + /* + * After first block + 1 words are fill up, + * we only need to fill 1 block to start partial computation + */ + rctx->state.blocklen -= sizeof(u32); + + dev_dbg(hdev->dev, "Write Control %x\n", reg); + } +} + +static void stm32_hash_append_sg(struct stm32_hash_request_ctx *rctx) +{ + struct stm32_hash_state *state = &rctx->state; + size_t count; + + while ((state->bufcnt < state->blocklen) && rctx->total) { + count = min(rctx->sg->length - rctx->offset, rctx->total); + count = min_t(size_t, count, state->blocklen - state->bufcnt); + + if (count <= 0) { + if ((rctx->sg->length == 0) && !sg_is_last(rctx->sg)) { + rctx->sg = sg_next(rctx->sg); + continue; + } else { + break; + } + } + + scatterwalk_map_and_copy(state->buffer + state->bufcnt, + rctx->sg, rctx->offset, count, 0); + + state->bufcnt += count; + rctx->offset += count; + rctx->total -= count; + + if (rctx->offset == rctx->sg->length) { + rctx->sg = sg_next(rctx->sg); + if (rctx->sg) + rctx->offset = 0; + else + rctx->total = 0; + } + } +} + +static int stm32_hash_xmit_cpu(struct stm32_hash_dev *hdev, + const u8 *buf, size_t length, int final) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req); + struct stm32_hash_state *state = &rctx->state; + unsigned int count, len32; + const u32 *buffer = (const u32 *)buf; + u32 reg; + + if (final) { + hdev->flags |= HASH_FLAGS_FINAL; + + /* Do not process empty messages if hw is buggy. */ + if (!(hdev->flags & HASH_FLAGS_INIT) && !length && + hdev->pdata->broken_emptymsg) { + state->flags |= HASH_FLAGS_EMPTY; + return 0; + } + } + + len32 = DIV_ROUND_UP(length, sizeof(u32)); + + dev_dbg(hdev->dev, "%s: length: %zd, final: %x len32 %i\n", + __func__, length, final, len32); + + hdev->flags |= HASH_FLAGS_CPU; + + stm32_hash_write_ctrl(hdev); + + if (stm32_hash_wait_busy(hdev)) + return -ETIMEDOUT; + + if ((hdev->flags & HASH_FLAGS_HMAC) && + (!(hdev->flags & HASH_FLAGS_HMAC_KEY))) { + hdev->flags |= HASH_FLAGS_HMAC_KEY; + stm32_hash_write_key(hdev); + if (stm32_hash_wait_busy(hdev)) + return -ETIMEDOUT; + } + + for (count = 0; count < len32; count++) + stm32_hash_write(hdev, HASH_DIN, buffer[count]); + + if (final) { + if (stm32_hash_wait_busy(hdev)) + return -ETIMEDOUT; + + stm32_hash_set_nblw(hdev, length); + reg = stm32_hash_read(hdev, HASH_STR); + reg |= HASH_STR_DCAL; + stm32_hash_write(hdev, HASH_STR, reg); + if (hdev->flags & HASH_FLAGS_HMAC) { + if (stm32_hash_wait_busy(hdev)) + return -ETIMEDOUT; + stm32_hash_write_key(hdev); + } + return -EINPROGRESS; + } + + return 0; +} + +static int hash_swap_reg(struct stm32_hash_request_ctx *rctx) +{ + struct stm32_hash_state *state = &rctx->state; + + switch ((state->flags & HASH_FLAGS_ALGO_MASK) >> + HASH_FLAGS_ALGO_SHIFT) { + case HASH_MD5: + case HASH_SHA1: + case HASH_SHA224: + case HASH_SHA256: + if (state->flags & HASH_FLAGS_HMAC) + return HASH_CSR_NB_SHA256_HMAC; + else + return HASH_CSR_NB_SHA256; + break; + + case HASH_SHA384: + case HASH_SHA512: + if (state->flags & HASH_FLAGS_HMAC) + return HASH_CSR_NB_SHA512_HMAC; + else + return HASH_CSR_NB_SHA512; + break; + + case HASH_SHA3_224: + case HASH_SHA3_256: + case HASH_SHA3_384: + case HASH_SHA3_512: + if (state->flags & HASH_FLAGS_HMAC) + return HASH_CSR_NB_SHA3_HMAC; + else + return HASH_CSR_NB_SHA3; + break; + + default: + return -EINVAL; + } +} + +static int stm32_hash_update_cpu(struct stm32_hash_dev *hdev) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req); + struct stm32_hash_state *state = &rctx->state; + int bufcnt, err = 0, final; + + dev_dbg(hdev->dev, "%s flags %x\n", __func__, state->flags); + + final = state->flags & HASH_FLAGS_FINAL; + + while ((rctx->total >= state->blocklen) || + (state->bufcnt + rctx->total >= state->blocklen)) { + stm32_hash_append_sg(rctx); + bufcnt = state->bufcnt; + state->bufcnt = 0; + err = stm32_hash_xmit_cpu(hdev, state->buffer, bufcnt, 0); + if (err) + return err; + } + + stm32_hash_append_sg(rctx); + + if (final) { + bufcnt = state->bufcnt; + state->bufcnt = 0; + return stm32_hash_xmit_cpu(hdev, state->buffer, bufcnt, 1); + } + + return err; +} + +static int stm32_hash_xmit_dma(struct stm32_hash_dev *hdev, + struct scatterlist *sg, int length, int mdmat) +{ + struct dma_async_tx_descriptor *in_desc; + dma_cookie_t cookie; + u32 reg; + int err; + + dev_dbg(hdev->dev, "%s mdmat: %x length: %d\n", __func__, mdmat, length); + + /* do not use dma if there is no data to send */ + if (length <= 0) + return 0; + + in_desc = dmaengine_prep_slave_sg(hdev->dma_lch, sg, 1, + DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | + DMA_CTRL_ACK); + if (!in_desc) { + dev_err(hdev->dev, "dmaengine_prep_slave error\n"); + return -ENOMEM; + } + + reinit_completion(&hdev->dma_completion); + in_desc->callback = stm32_hash_dma_callback; + in_desc->callback_param = hdev; + + hdev->flags |= HASH_FLAGS_DMA_ACTIVE; + + reg = stm32_hash_read(hdev, HASH_CR); + + if (hdev->pdata->has_mdmat) { + if (mdmat) + reg |= HASH_CR_MDMAT; + else + reg &= ~HASH_CR_MDMAT; + } + reg |= HASH_CR_DMAE; + + stm32_hash_write(hdev, HASH_CR, reg); + + + cookie = dmaengine_submit(in_desc); + err = dma_submit_error(cookie); + if (err) + return -ENOMEM; + + dma_async_issue_pending(hdev->dma_lch); + + if (!wait_for_completion_timeout(&hdev->dma_completion, + msecs_to_jiffies(100))) + err = -ETIMEDOUT; + + if (dma_async_is_tx_complete(hdev->dma_lch, cookie, + NULL, NULL) != DMA_COMPLETE) + err = -ETIMEDOUT; + + if (err) { + dev_err(hdev->dev, "DMA Error %i\n", err); + dmaengine_terminate_all(hdev->dma_lch); + return err; + } + + return -EINPROGRESS; +} + +static void stm32_hash_dma_callback(void *param) +{ + struct stm32_hash_dev *hdev = param; + + complete(&hdev->dma_completion); +} + +static int stm32_hash_hmac_dma_send(struct stm32_hash_dev *hdev) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(hdev->req); + struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm); + int err; + + if (ctx->keylen < rctx->state.blocklen || hdev->dma_mode > 0) { + err = stm32_hash_write_key(hdev); + if (stm32_hash_wait_busy(hdev)) + return -ETIMEDOUT; + } else { + if (!(hdev->flags & HASH_FLAGS_HMAC_KEY)) + sg_init_one(&rctx->sg_key, ctx->key, + ALIGN(ctx->keylen, sizeof(u32))); + + rctx->dma_ct = dma_map_sg(hdev->dev, &rctx->sg_key, 1, + DMA_TO_DEVICE); + if (rctx->dma_ct == 0) { + dev_err(hdev->dev, "dma_map_sg error\n"); + return -ENOMEM; + } + + err = stm32_hash_xmit_dma(hdev, &rctx->sg_key, ctx->keylen, 0); + + dma_unmap_sg(hdev->dev, &rctx->sg_key, 1, DMA_TO_DEVICE); + } + + return err; +} + +static int stm32_hash_dma_init(struct stm32_hash_dev *hdev) +{ + struct dma_slave_config dma_conf; + struct dma_chan *chan; + int err; + + memset(&dma_conf, 0, sizeof(dma_conf)); + + dma_conf.direction = DMA_MEM_TO_DEV; + dma_conf.dst_addr = hdev->phys_base + HASH_DIN; + dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + dma_conf.src_maxburst = HASH_BURST_LEVEL; + dma_conf.dst_maxburst = HASH_BURST_LEVEL; + dma_conf.device_fc = false; + + chan = dma_request_chan(hdev->dev, "in"); + if (IS_ERR(chan)) + return PTR_ERR(chan); + + hdev->dma_lch = chan; + + err = dmaengine_slave_config(hdev->dma_lch, &dma_conf); + if (err) { + dma_release_channel(hdev->dma_lch); + hdev->dma_lch = NULL; + dev_err(hdev->dev, "Couldn't configure DMA slave.\n"); + return err; + } + + init_completion(&hdev->dma_completion); + + return 0; +} + +static int stm32_hash_dma_send(struct stm32_hash_dev *hdev) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req); + u32 *buffer = (void *)rctx->state.buffer; + struct scatterlist sg[1], *tsg; + int err = 0, reg, ncp = 0; + unsigned int i, len = 0, bufcnt = 0; + bool final = hdev->flags & HASH_FLAGS_FINAL; + bool is_last = false; + u32 last_word; + + dev_dbg(hdev->dev, "%s total: %d bufcnt: %d final: %d\n", + __func__, rctx->total, rctx->state.bufcnt, final); + + if (rctx->nents < 0) + return -EINVAL; + + stm32_hash_write_ctrl(hdev); + + if (hdev->flags & HASH_FLAGS_HMAC && (!(hdev->flags & HASH_FLAGS_HMAC_KEY))) { + hdev->flags |= HASH_FLAGS_HMAC_KEY; + err = stm32_hash_hmac_dma_send(hdev); + if (err != -EINPROGRESS) + return err; + } + + for_each_sg(rctx->sg, tsg, rctx->nents, i) { + sg[0] = *tsg; + len = sg->length; + + if (sg_is_last(sg) || (bufcnt + sg[0].length) >= rctx->total) { + if (!final) { + /* Always manually put the last word of a non-final transfer. */ + len -= sizeof(u32); + sg_pcopy_to_buffer(rctx->sg, rctx->nents, &last_word, 4, len); + sg->length -= sizeof(u32); + } else { + /* + * In Multiple DMA mode, DMA must be aborted before the final + * transfer. + */ + sg->length = rctx->total - bufcnt; + if (hdev->dma_mode > 0) { + len = (ALIGN(sg->length, 16) - 16); + + ncp = sg_pcopy_to_buffer(rctx->sg, rctx->nents, + rctx->state.buffer, + sg->length - len, + rctx->total - sg->length + len); + + if (!len) + break; + + sg->length = len; + } else { + is_last = true; + if (!(IS_ALIGNED(sg->length, sizeof(u32)))) { + len = sg->length; + sg->length = ALIGN(sg->length, + sizeof(u32)); + } + } + } + } + + rctx->dma_ct = dma_map_sg(hdev->dev, sg, 1, + DMA_TO_DEVICE); + if (rctx->dma_ct == 0) { + dev_err(hdev->dev, "dma_map_sg error\n"); + return -ENOMEM; + } + + err = stm32_hash_xmit_dma(hdev, sg, len, !is_last); + + /* The last word of a non final transfer is sent manually. */ + if (!final) { + stm32_hash_write(hdev, HASH_DIN, last_word); + len += sizeof(u32); + } + + rctx->total -= len; + + bufcnt += sg[0].length; + dma_unmap_sg(hdev->dev, sg, 1, DMA_TO_DEVICE); + + if (err == -ENOMEM || err == -ETIMEDOUT) + return err; + if (is_last) + break; + } + + /* + * When the second last block transfer of 4 words is performed by the DMA, + * the software must set the DMA Abort bit (DMAA) to 1 before completing the + * last transfer of 4 words or less. + */ + if (final) { + if (hdev->dma_mode > 0) { + if (stm32_hash_wait_busy(hdev)) + return -ETIMEDOUT; + reg = stm32_hash_read(hdev, HASH_CR); + reg &= ~HASH_CR_DMAE; + reg |= HASH_CR_DMAA; + stm32_hash_write(hdev, HASH_CR, reg); + + if (ncp) { + memset(buffer + ncp, 0, 4 - DIV_ROUND_UP(ncp, sizeof(u32))); + writesl(hdev->io_base + HASH_DIN, buffer, + DIV_ROUND_UP(ncp, sizeof(u32))); + } + + stm32_hash_set_nblw(hdev, ncp); + reg = stm32_hash_read(hdev, HASH_STR); + reg |= HASH_STR_DCAL; + stm32_hash_write(hdev, HASH_STR, reg); + err = -EINPROGRESS; + } + + /* + * The hash processor needs the key to be loaded a second time in order + * to process the HMAC. + */ + if (hdev->flags & HASH_FLAGS_HMAC) { + if (stm32_hash_wait_busy(hdev)) + return -ETIMEDOUT; + err = stm32_hash_hmac_dma_send(hdev); + } + + return err; + } + + if (err != -EINPROGRESS) + return err; + + return 0; +} + +static struct stm32_hash_dev *stm32_hash_find_dev(struct stm32_hash_ctx *ctx) +{ + struct stm32_hash_dev *hdev; + + spin_lock_bh(&stm32_hash.lock); + if (!ctx->hdev) + ctx->hdev = list_first_entry_or_null(&stm32_hash.dev_list, + struct stm32_hash_dev, list); + hdev = ctx->hdev; + spin_unlock_bh(&stm32_hash.lock); + + return hdev; +} + +static int stm32_hash_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm); + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx); + struct stm32_hash_state *state = &rctx->state; + bool sha3_mode = ctx->flags & HASH_FLAGS_SHA3_MODE; + + rctx->hdev = hdev; + state->flags = 0; + + if (!(hdev->dma_lch && hdev->pdata->has_mdmat)) + state->flags |= HASH_FLAGS_CPU; + + if (sha3_mode) + state->flags |= HASH_FLAGS_SHA3_MODE; + + rctx->digcnt = crypto_ahash_digestsize(tfm); + switch (rctx->digcnt) { + case MD5_DIGEST_SIZE: + state->flags |= HASH_MD5 << HASH_FLAGS_ALGO_SHIFT; + break; + case SHA1_DIGEST_SIZE: + if (hdev->pdata->ux500) + state->flags |= HASH_SHA1_UX500 << HASH_FLAGS_ALGO_SHIFT; + else + state->flags |= HASH_SHA1 << HASH_FLAGS_ALGO_SHIFT; + break; + case SHA224_DIGEST_SIZE: + if (sha3_mode) + state->flags |= HASH_SHA3_224 << HASH_FLAGS_ALGO_SHIFT; + else + state->flags |= HASH_SHA224 << HASH_FLAGS_ALGO_SHIFT; + break; + case SHA256_DIGEST_SIZE: + if (sha3_mode) { + state->flags |= HASH_SHA3_256 << HASH_FLAGS_ALGO_SHIFT; + } else { + if (hdev->pdata->ux500) + state->flags |= HASH_SHA256_UX500 << HASH_FLAGS_ALGO_SHIFT; + else + state->flags |= HASH_SHA256 << HASH_FLAGS_ALGO_SHIFT; + } + break; + case SHA384_DIGEST_SIZE: + if (sha3_mode) + state->flags |= HASH_SHA3_384 << HASH_FLAGS_ALGO_SHIFT; + else + state->flags |= HASH_SHA384 << HASH_FLAGS_ALGO_SHIFT; + break; + case SHA512_DIGEST_SIZE: + if (sha3_mode) + state->flags |= HASH_SHA3_512 << HASH_FLAGS_ALGO_SHIFT; + else + state->flags |= HASH_SHA512 << HASH_FLAGS_ALGO_SHIFT; + break; + default: + return -EINVAL; + } + + rctx->state.bufcnt = 0; + rctx->state.blocklen = crypto_ahash_blocksize(tfm) + sizeof(u32); + if (rctx->state.blocklen > HASH_BUFLEN) { + dev_err(hdev->dev, "Error, block too large"); + return -EINVAL; + } + rctx->nents = 0; + rctx->total = 0; + rctx->offset = 0; + rctx->data_type = HASH_DATA_8_BITS; + + if (ctx->flags & HASH_FLAGS_HMAC) + state->flags |= HASH_FLAGS_HMAC; + + dev_dbg(hdev->dev, "%s Flags %x\n", __func__, state->flags); + + return 0; +} + +static int stm32_hash_update_req(struct stm32_hash_dev *hdev) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req); + struct stm32_hash_state *state = &rctx->state; + + dev_dbg(hdev->dev, "update_req: total: %u, digcnt: %zd, final: 0", + rctx->total, rctx->digcnt); + + if (!(state->flags & HASH_FLAGS_CPU)) + return stm32_hash_dma_send(hdev); + + return stm32_hash_update_cpu(hdev); +} + +static int stm32_hash_final_req(struct stm32_hash_dev *hdev) +{ + struct ahash_request *req = hdev->req; + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + struct stm32_hash_state *state = &rctx->state; + int buflen = state->bufcnt; + + if (!(state->flags & HASH_FLAGS_CPU)) { + hdev->flags |= HASH_FLAGS_FINAL; + return stm32_hash_dma_send(hdev); + } + + if (state->flags & HASH_FLAGS_FINUP) + return stm32_hash_update_req(hdev); + + state->bufcnt = 0; + + return stm32_hash_xmit_cpu(hdev, state->buffer, buflen, 1); +} + +static void stm32_hash_emptymsg_fallback(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct stm32_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + struct stm32_hash_dev *hdev = rctx->hdev; + int ret; + + dev_dbg(hdev->dev, "use fallback message size 0 key size %d\n", + ctx->keylen); + + if (!ctx->xtfm) { + dev_err(hdev->dev, "no fallback engine\n"); + return; + } + + if (ctx->keylen) { + ret = crypto_shash_setkey(ctx->xtfm, ctx->key, ctx->keylen); + if (ret) { + dev_err(hdev->dev, "failed to set key ret=%d\n", ret); + return; + } + } + + ret = crypto_shash_tfm_digest(ctx->xtfm, NULL, 0, rctx->digest); + if (ret) + dev_err(hdev->dev, "shash digest error\n"); +} + +static void stm32_hash_copy_hash(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + struct stm32_hash_state *state = &rctx->state; + struct stm32_hash_dev *hdev = rctx->hdev; + __be32 *hash = (void *)rctx->digest; + unsigned int i, hashsize; + + if (hdev->pdata->broken_emptymsg && (state->flags & HASH_FLAGS_EMPTY)) + return stm32_hash_emptymsg_fallback(req); + + hashsize = crypto_ahash_digestsize(tfm); + + for (i = 0; i < hashsize / sizeof(u32); i++) { + if (hdev->pdata->ux500) + hash[i] = cpu_to_be32(stm32_hash_read(hdev, + HASH_UX500_HREG(i))); + else + hash[i] = cpu_to_be32(stm32_hash_read(hdev, + HASH_HREG(i))); + } +} + +static int stm32_hash_finish(struct ahash_request *req) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + u32 reg; + + reg = stm32_hash_read(rctx->hdev, HASH_SR); + reg &= ~HASH_SR_OUTPUT_READY; + stm32_hash_write(rctx->hdev, HASH_SR, reg); + + if (!req->result) + return -EINVAL; + + memcpy(req->result, rctx->digest, rctx->digcnt); + + return 0; +} + +static void stm32_hash_finish_req(struct ahash_request *req, int err) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + struct stm32_hash_state *state = &rctx->state; + struct stm32_hash_dev *hdev = rctx->hdev; + + if (hdev->flags & HASH_FLAGS_DMA_ACTIVE) + state->flags |= HASH_FLAGS_DMA_ACTIVE; + else + state->flags &= ~HASH_FLAGS_DMA_ACTIVE; + + if (!err && (HASH_FLAGS_FINAL & hdev->flags)) { + stm32_hash_copy_hash(req); + err = stm32_hash_finish(req); + } + + /* Finalized request mist be unprepared here */ + stm32_hash_unprepare_request(req); + + crypto_finalize_hash_request(hdev->engine, req, err); +} + +static int stm32_hash_handle_queue(struct stm32_hash_dev *hdev, + struct ahash_request *req) +{ + return crypto_transfer_hash_request_to_engine(hdev->engine, req); +} + +static int stm32_hash_one_request(struct crypto_engine *engine, void *areq) +{ + struct ahash_request *req = container_of(areq, struct ahash_request, + base); + struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx); + struct stm32_hash_state *state = &rctx->state; + int swap_reg; + int err = 0; + + if (!hdev) + return -ENODEV; + + dev_dbg(hdev->dev, "processing new req, op: %lu, nbytes %d\n", + rctx->op, req->nbytes); + + pm_runtime_get_sync(hdev->dev); + + err = stm32_hash_prepare_request(req); + if (err) + return err; + + hdev->req = req; + hdev->flags = 0; + swap_reg = hash_swap_reg(rctx); + + if (state->flags & HASH_FLAGS_INIT) { + u32 *preg = rctx->state.hw_context; + u32 reg; + int i; + + if (!hdev->pdata->ux500) + stm32_hash_write(hdev, HASH_IMR, *preg++); + stm32_hash_write(hdev, HASH_STR, *preg++); + stm32_hash_write(hdev, HASH_CR, *preg); + reg = *preg++ | HASH_CR_INIT; + stm32_hash_write(hdev, HASH_CR, reg); + + for (i = 0; i < swap_reg; i++) + stm32_hash_write(hdev, HASH_CSR(i), *preg++); + + hdev->flags |= HASH_FLAGS_INIT; + + if (state->flags & HASH_FLAGS_HMAC) + hdev->flags |= HASH_FLAGS_HMAC | + HASH_FLAGS_HMAC_KEY; + + if (state->flags & HASH_FLAGS_CPU) + hdev->flags |= HASH_FLAGS_CPU; + + if (state->flags & HASH_FLAGS_DMA_ACTIVE) + hdev->flags |= HASH_FLAGS_DMA_ACTIVE; + } + + if (rctx->op == HASH_OP_UPDATE) + err = stm32_hash_update_req(hdev); + else if (rctx->op == HASH_OP_FINAL) + err = stm32_hash_final_req(hdev); + + /* If we have an IRQ, wait for that, else poll for completion */ + if (err == -EINPROGRESS && hdev->polled) { + if (stm32_hash_wait_busy(hdev)) + err = -ETIMEDOUT; + else { + hdev->flags |= HASH_FLAGS_OUTPUT_READY; + err = 0; + } + } + + if (err != -EINPROGRESS) + /* done task will not finish it, so do it here */ + stm32_hash_finish_req(req, err); + + return 0; +} + +static int stm32_hash_copy_sgs(struct stm32_hash_request_ctx *rctx, + struct scatterlist *sg, int bs, + unsigned int new_len) +{ + struct stm32_hash_state *state = &rctx->state; + int pages; + void *buf; + + pages = get_order(new_len); + + buf = (void *)__get_free_pages(GFP_ATOMIC, pages); + if (!buf) { + pr_err("Couldn't allocate pages for unaligned cases.\n"); + return -ENOMEM; + } + + memcpy(buf, rctx->hdev->xmit_buf, state->bufcnt); + + scatterwalk_map_and_copy(buf + state->bufcnt, sg, rctx->offset, + min(new_len, rctx->total) - state->bufcnt, 0); + sg_init_table(rctx->sgl, 1); + sg_set_buf(rctx->sgl, buf, new_len); + rctx->sg = rctx->sgl; + state->flags |= HASH_FLAGS_SGS_COPIED; + rctx->nents = 1; + rctx->offset += new_len - state->bufcnt; + state->bufcnt = 0; + rctx->total = new_len; + + return 0; +} + +static int stm32_hash_align_sgs(struct scatterlist *sg, + int nbytes, int bs, bool init, bool final, + struct stm32_hash_request_ctx *rctx) +{ + struct stm32_hash_state *state = &rctx->state; + struct stm32_hash_dev *hdev = rctx->hdev; + struct scatterlist *sg_tmp = sg; + int offset = rctx->offset; + int new_len; + int n = 0; + int bufcnt = state->bufcnt; + bool secure_ctx = hdev->pdata->context_secured; + bool aligned = true; + + if (!sg || !sg->length || !nbytes) { + if (bufcnt) { + bufcnt = DIV_ROUND_UP(bufcnt, bs) * bs; + sg_init_table(rctx->sgl, 1); + sg_set_buf(rctx->sgl, rctx->hdev->xmit_buf, bufcnt); + rctx->sg = rctx->sgl; + rctx->nents = 1; + } + + return 0; + } + + new_len = nbytes; + + if (offset) + aligned = false; + + if (final) { + new_len = DIV_ROUND_UP(new_len, bs) * bs; + } else { + new_len = (new_len - 1) / bs * bs; // return n block - 1 block + + /* + * Context save in some version of HASH IP can only be done when the + * FIFO is ready to get a new block. This implies to send n block plus a + * 32 bit word in the first DMA send. + */ + if (init && secure_ctx) { + new_len += sizeof(u32); + if (unlikely(new_len > nbytes)) + new_len -= bs; + } + } + + if (!new_len) + return 0; + + if (nbytes != new_len) + aligned = false; + + while (nbytes > 0 && sg_tmp) { + n++; + + if (bufcnt) { + if (!IS_ALIGNED(bufcnt, bs)) { + aligned = false; + break; + } + nbytes -= bufcnt; + bufcnt = 0; + if (!nbytes) + aligned = false; + + continue; + } + + if (offset < sg_tmp->length) { + if (!IS_ALIGNED(offset + sg_tmp->offset, 4)) { + aligned = false; + break; + } + + if (!IS_ALIGNED(sg_tmp->length - offset, bs)) { + aligned = false; + break; + } + } + + if (offset) { + offset -= sg_tmp->length; + if (offset < 0) { + nbytes += offset; + offset = 0; + } + } else { + nbytes -= sg_tmp->length; + } + + sg_tmp = sg_next(sg_tmp); + + if (nbytes < 0) { + aligned = false; + break; + } + } + + if (!aligned) + return stm32_hash_copy_sgs(rctx, sg, bs, new_len); + + rctx->total = new_len; + rctx->offset += new_len; + rctx->nents = n; + if (state->bufcnt) { + sg_init_table(rctx->sgl, 2); + sg_set_buf(rctx->sgl, rctx->hdev->xmit_buf, state->bufcnt); + sg_chain(rctx->sgl, 2, sg); + rctx->sg = rctx->sgl; + } else { + rctx->sg = sg; + } + + return 0; +} + +static int stm32_hash_prepare_request(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm); + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx); + struct stm32_hash_state *state = &rctx->state; + unsigned int nbytes; + int ret, hash_later, bs; + bool update = rctx->op & HASH_OP_UPDATE; + bool init = !(state->flags & HASH_FLAGS_INIT); + bool finup = state->flags & HASH_FLAGS_FINUP; + bool final = state->flags & HASH_FLAGS_FINAL; + + if (!hdev->dma_lch || state->flags & HASH_FLAGS_CPU) + return 0; + + bs = crypto_ahash_blocksize(tfm); + + nbytes = state->bufcnt; + + /* + * In case of update request nbytes must correspond to the content of the + * buffer + the offset minus the content of the request already in the + * buffer. + */ + if (update || finup) + nbytes += req->nbytes - rctx->offset; + + dev_dbg(hdev->dev, + "%s: nbytes=%d, bs=%d, total=%d, offset=%d, bufcnt=%d\n", + __func__, nbytes, bs, rctx->total, rctx->offset, state->bufcnt); + + if (!nbytes) + return 0; + + rctx->total = nbytes; + + if (update && req->nbytes && (!IS_ALIGNED(state->bufcnt, bs))) { + int len = bs - state->bufcnt % bs; + + if (len > req->nbytes) + len = req->nbytes; + scatterwalk_map_and_copy(state->buffer + state->bufcnt, req->src, + 0, len, 0); + state->bufcnt += len; + rctx->offset = len; + } + + /* copy buffer in a temporary one that is used for sg alignment */ + memcpy(hdev->xmit_buf, state->buffer, state->bufcnt); + + ret = stm32_hash_align_sgs(req->src, nbytes, bs, init, final, rctx); + if (ret) + return ret; + + hash_later = nbytes - rctx->total; + if (hash_later < 0) + hash_later = 0; + + if (hash_later && hash_later <= state->blocklen) { + scatterwalk_map_and_copy(state->buffer, + req->src, + req->nbytes - hash_later, + hash_later, 0); + + state->bufcnt = hash_later; + } else { + state->bufcnt = 0; + } + + if (hash_later > state->blocklen) { + /* FIXME: add support of this case */ + pr_err("Buffer contains more than one block.\n"); + return -ENOMEM; + } + + rctx->total = min(nbytes, rctx->total); + + return 0; +} + +static void stm32_hash_unprepare_request(struct ahash_request *req) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + struct stm32_hash_state *state = &rctx->state; + struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); + struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx); + u32 *preg = state->hw_context; + int swap_reg, i; + + if (hdev->dma_lch) + dmaengine_terminate_sync(hdev->dma_lch); + + if (state->flags & HASH_FLAGS_SGS_COPIED) + free_pages((unsigned long)sg_virt(rctx->sg), get_order(rctx->sg->length)); + + rctx->sg = NULL; + rctx->offset = 0; + + state->flags &= ~(HASH_FLAGS_SGS_COPIED); + + if (!(hdev->flags & HASH_FLAGS_INIT)) + goto pm_runtime; + + state->flags |= HASH_FLAGS_INIT; + + if (stm32_hash_wait_busy(hdev)) { + dev_warn(hdev->dev, "Wait busy failed."); + return; + } + + swap_reg = hash_swap_reg(rctx); + + if (!hdev->pdata->ux500) + *preg++ = stm32_hash_read(hdev, HASH_IMR); + *preg++ = stm32_hash_read(hdev, HASH_STR); + *preg++ = stm32_hash_read(hdev, HASH_CR); + for (i = 0; i < swap_reg; i++) + *preg++ = stm32_hash_read(hdev, HASH_CSR(i)); + +pm_runtime: + pm_runtime_put_autosuspend(hdev->dev); +} + +static int stm32_hash_enqueue(struct ahash_request *req, unsigned int op) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + struct stm32_hash_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct stm32_hash_dev *hdev = ctx->hdev; + + rctx->op = op; + + return stm32_hash_handle_queue(hdev, req); +} + +static int stm32_hash_update(struct ahash_request *req) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + struct stm32_hash_state *state = &rctx->state; + + if (!req->nbytes) + return 0; + + + if (state->flags & HASH_FLAGS_CPU) { + rctx->total = req->nbytes; + rctx->sg = req->src; + rctx->offset = 0; + + if ((state->bufcnt + rctx->total < state->blocklen)) { + stm32_hash_append_sg(rctx); + return 0; + } + } else { /* DMA mode */ + if (state->bufcnt + req->nbytes <= state->blocklen) { + scatterwalk_map_and_copy(state->buffer + state->bufcnt, req->src, + 0, req->nbytes, 0); + state->bufcnt += req->nbytes; + return 0; + } + } + + return stm32_hash_enqueue(req, HASH_OP_UPDATE); +} + +static int stm32_hash_final(struct ahash_request *req) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + struct stm32_hash_state *state = &rctx->state; + + state->flags |= HASH_FLAGS_FINAL; + + return stm32_hash_enqueue(req, HASH_OP_FINAL); +} + +static int stm32_hash_finup(struct ahash_request *req) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + struct stm32_hash_state *state = &rctx->state; + + if (!req->nbytes) + goto out; + + state->flags |= HASH_FLAGS_FINUP; + + if ((state->flags & HASH_FLAGS_CPU)) { + rctx->total = req->nbytes; + rctx->sg = req->src; + rctx->offset = 0; + } + +out: + return stm32_hash_final(req); +} + +static int stm32_hash_digest(struct ahash_request *req) +{ + return stm32_hash_init(req) ?: stm32_hash_finup(req); +} + +static int stm32_hash_export(struct ahash_request *req, void *out) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + + memcpy(out, &rctx->state, sizeof(rctx->state)); + + return 0; +} + +static int stm32_hash_import(struct ahash_request *req, const void *in) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + + stm32_hash_init(req); + memcpy(&rctx->state, in, sizeof(rctx->state)); + + return 0; +} + +static int stm32_hash_setkey(struct crypto_ahash *tfm, + const u8 *key, unsigned int keylen) +{ + struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm); + + if (keylen <= HASH_MAX_KEY_SIZE) { + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + } else { + return -ENOMEM; + } + + return 0; +} + +static int stm32_hash_init_fallback(struct crypto_tfm *tfm) +{ + struct stm32_hash_ctx *ctx = crypto_tfm_ctx(tfm); + struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx); + const char *name = crypto_tfm_alg_name(tfm); + struct crypto_shash *xtfm; + + /* The fallback is only needed on Ux500 */ + if (!hdev->pdata->ux500) + return 0; + + xtfm = crypto_alloc_shash(name, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(xtfm)) { + dev_err(hdev->dev, "failed to allocate %s fallback\n", + name); + return PTR_ERR(xtfm); + } + dev_info(hdev->dev, "allocated %s fallback\n", name); + ctx->xtfm = xtfm; + + return 0; +} + +static int stm32_hash_cra_init_algs(struct crypto_tfm *tfm, u32 algs_flags) +{ + struct stm32_hash_ctx *ctx = crypto_tfm_ctx(tfm); + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct stm32_hash_request_ctx)); + + ctx->keylen = 0; + + if (algs_flags) + ctx->flags |= algs_flags; + + return stm32_hash_init_fallback(tfm); +} + +static int stm32_hash_cra_init(struct crypto_tfm *tfm) +{ + return stm32_hash_cra_init_algs(tfm, 0); +} + +static int stm32_hash_cra_hmac_init(struct crypto_tfm *tfm) +{ + return stm32_hash_cra_init_algs(tfm, HASH_FLAGS_HMAC); +} + +static int stm32_hash_cra_sha3_init(struct crypto_tfm *tfm) +{ + return stm32_hash_cra_init_algs(tfm, HASH_FLAGS_SHA3_MODE); +} + +static int stm32_hash_cra_sha3_hmac_init(struct crypto_tfm *tfm) +{ + return stm32_hash_cra_init_algs(tfm, HASH_FLAGS_SHA3_MODE | + HASH_FLAGS_HMAC); +} + +static void stm32_hash_cra_exit(struct crypto_tfm *tfm) +{ + struct stm32_hash_ctx *ctx = crypto_tfm_ctx(tfm); + + if (ctx->xtfm) + crypto_free_shash(ctx->xtfm); +} + +static irqreturn_t stm32_hash_irq_thread(int irq, void *dev_id) +{ + struct stm32_hash_dev *hdev = dev_id; + + if (HASH_FLAGS_OUTPUT_READY & hdev->flags) { + hdev->flags &= ~HASH_FLAGS_OUTPUT_READY; + goto finish; + } + + return IRQ_HANDLED; + +finish: + /* Finish current request */ + stm32_hash_finish_req(hdev->req, 0); + + return IRQ_HANDLED; +} + +static irqreturn_t stm32_hash_irq_handler(int irq, void *dev_id) +{ + struct stm32_hash_dev *hdev = dev_id; + u32 reg; + + reg = stm32_hash_read(hdev, HASH_SR); + if (reg & HASH_SR_OUTPUT_READY) { + hdev->flags |= HASH_FLAGS_OUTPUT_READY; + /* Disable IT*/ + stm32_hash_write(hdev, HASH_IMR, 0); + return IRQ_WAKE_THREAD; + } + + return IRQ_NONE; +} + +static struct ahash_engine_alg algs_md5[] = { + { + .base.init = stm32_hash_init, + .base.update = stm32_hash_update, + .base.final = stm32_hash_final, + .base.finup = stm32_hash_finup, + .base.digest = stm32_hash_digest, + .base.export = stm32_hash_export, + .base.import = stm32_hash_import, + .base.halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_state), + .base = { + .cra_name = "md5", + .cra_driver_name = "stm32-md5", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_init = stm32_hash_cra_init, + .cra_exit = stm32_hash_cra_exit, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = stm32_hash_one_request, + }, + }, + { + .base.init = stm32_hash_init, + .base.update = stm32_hash_update, + .base.final = stm32_hash_final, + .base.finup = stm32_hash_finup, + .base.digest = stm32_hash_digest, + .base.export = stm32_hash_export, + .base.import = stm32_hash_import, + .base.setkey = stm32_hash_setkey, + .base.halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_state), + .base = { + .cra_name = "hmac(md5)", + .cra_driver_name = "stm32-hmac-md5", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_init = stm32_hash_cra_hmac_init, + .cra_exit = stm32_hash_cra_exit, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = stm32_hash_one_request, + }, + } +}; + +static struct ahash_engine_alg algs_sha1[] = { + { + .base.init = stm32_hash_init, + .base.update = stm32_hash_update, + .base.final = stm32_hash_final, + .base.finup = stm32_hash_finup, + .base.digest = stm32_hash_digest, + .base.export = stm32_hash_export, + .base.import = stm32_hash_import, + .base.halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_state), + .base = { + .cra_name = "sha1", + .cra_driver_name = "stm32-sha1", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_init = stm32_hash_cra_init, + .cra_exit = stm32_hash_cra_exit, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = stm32_hash_one_request, + }, + }, + { + .base.init = stm32_hash_init, + .base.update = stm32_hash_update, + .base.final = stm32_hash_final, + .base.finup = stm32_hash_finup, + .base.digest = stm32_hash_digest, + .base.export = stm32_hash_export, + .base.import = stm32_hash_import, + .base.setkey = stm32_hash_setkey, + .base.halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_state), + .base = { + .cra_name = "hmac(sha1)", + .cra_driver_name = "stm32-hmac-sha1", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_init = stm32_hash_cra_hmac_init, + .cra_exit = stm32_hash_cra_exit, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = stm32_hash_one_request, + }, + }, +}; + +static struct ahash_engine_alg algs_sha224[] = { + { + .base.init = stm32_hash_init, + .base.update = stm32_hash_update, + .base.final = stm32_hash_final, + .base.finup = stm32_hash_finup, + .base.digest = stm32_hash_digest, + .base.export = stm32_hash_export, + .base.import = stm32_hash_import, + .base.halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_state), + .base = { + .cra_name = "sha224", + .cra_driver_name = "stm32-sha224", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_init = stm32_hash_cra_init, + .cra_exit = stm32_hash_cra_exit, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = stm32_hash_one_request, + }, + }, + { + .base.init = stm32_hash_init, + .base.update = stm32_hash_update, + .base.final = stm32_hash_final, + .base.finup = stm32_hash_finup, + .base.digest = stm32_hash_digest, + .base.setkey = stm32_hash_setkey, + .base.export = stm32_hash_export, + .base.import = stm32_hash_import, + .base.halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_state), + .base = { + .cra_name = "hmac(sha224)", + .cra_driver_name = "stm32-hmac-sha224", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_init = stm32_hash_cra_hmac_init, + .cra_exit = stm32_hash_cra_exit, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = stm32_hash_one_request, + }, + }, +}; + +static struct ahash_engine_alg algs_sha256[] = { + { + .base.init = stm32_hash_init, + .base.update = stm32_hash_update, + .base.final = stm32_hash_final, + .base.finup = stm32_hash_finup, + .base.digest = stm32_hash_digest, + .base.export = stm32_hash_export, + .base.import = stm32_hash_import, + .base.halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_state), + .base = { + .cra_name = "sha256", + .cra_driver_name = "stm32-sha256", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_init = stm32_hash_cra_init, + .cra_exit = stm32_hash_cra_exit, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = stm32_hash_one_request, + }, + }, + { + .base.init = stm32_hash_init, + .base.update = stm32_hash_update, + .base.final = stm32_hash_final, + .base.finup = stm32_hash_finup, + .base.digest = stm32_hash_digest, + .base.export = stm32_hash_export, + .base.import = stm32_hash_import, + .base.setkey = stm32_hash_setkey, + .base.halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_state), + .base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "stm32-hmac-sha256", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_init = stm32_hash_cra_hmac_init, + .cra_exit = stm32_hash_cra_exit, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = stm32_hash_one_request, + }, + }, +}; + +static struct ahash_engine_alg algs_sha384_sha512[] = { + { + .base.init = stm32_hash_init, + .base.update = stm32_hash_update, + .base.final = stm32_hash_final, + .base.finup = stm32_hash_finup, + .base.digest = stm32_hash_digest, + .base.export = stm32_hash_export, + .base.import = stm32_hash_import, + .base.halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_state), + .base = { + .cra_name = "sha384", + .cra_driver_name = "stm32-sha384", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_init = stm32_hash_cra_init, + .cra_exit = stm32_hash_cra_exit, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = stm32_hash_one_request, + }, + }, + { + .base.init = stm32_hash_init, + .base.update = stm32_hash_update, + .base.final = stm32_hash_final, + .base.finup = stm32_hash_finup, + .base.digest = stm32_hash_digest, + .base.setkey = stm32_hash_setkey, + .base.export = stm32_hash_export, + .base.import = stm32_hash_import, + .base.halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_state), + .base = { + .cra_name = "hmac(sha384)", + .cra_driver_name = "stm32-hmac-sha384", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_init = stm32_hash_cra_hmac_init, + .cra_exit = stm32_hash_cra_exit, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = stm32_hash_one_request, + }, + }, + { + .base.init = stm32_hash_init, + .base.update = stm32_hash_update, + .base.final = stm32_hash_final, + .base.finup = stm32_hash_finup, + .base.digest = stm32_hash_digest, + .base.export = stm32_hash_export, + .base.import = stm32_hash_import, + .base.halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_state), + .base = { + .cra_name = "sha512", + .cra_driver_name = "stm32-sha512", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_init = stm32_hash_cra_init, + .cra_exit = stm32_hash_cra_exit, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = stm32_hash_one_request, + }, + }, + { + .base.init = stm32_hash_init, + .base.update = stm32_hash_update, + .base.final = stm32_hash_final, + .base.finup = stm32_hash_finup, + .base.digest = stm32_hash_digest, + .base.export = stm32_hash_export, + .base.import = stm32_hash_import, + .base.setkey = stm32_hash_setkey, + .base.halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_state), + .base = { + .cra_name = "hmac(sha512)", + .cra_driver_name = "stm32-hmac-sha512", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_init = stm32_hash_cra_hmac_init, + .cra_exit = stm32_hash_cra_exit, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = stm32_hash_one_request, + }, + }, +}; + +static struct ahash_engine_alg algs_sha3[] = { + { + .base.init = stm32_hash_init, + .base.update = stm32_hash_update, + .base.final = stm32_hash_final, + .base.finup = stm32_hash_finup, + .base.digest = stm32_hash_digest, + .base.export = stm32_hash_export, + .base.import = stm32_hash_import, + .base.halg = { + .digestsize = SHA3_224_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_state), + .base = { + .cra_name = "sha3-224", + .cra_driver_name = "stm32-sha3-224", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA3_224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_init = stm32_hash_cra_sha3_init, + .cra_exit = stm32_hash_cra_exit, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = stm32_hash_one_request, + }, + }, + { + .base.init = stm32_hash_init, + .base.update = stm32_hash_update, + .base.final = stm32_hash_final, + .base.finup = stm32_hash_finup, + .base.digest = stm32_hash_digest, + .base.export = stm32_hash_export, + .base.import = stm32_hash_import, + .base.setkey = stm32_hash_setkey, + .base.halg = { + .digestsize = SHA3_224_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_state), + .base = { + .cra_name = "hmac(sha3-224)", + .cra_driver_name = "stm32-hmac-sha3-224", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA3_224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_init = stm32_hash_cra_sha3_hmac_init, + .cra_exit = stm32_hash_cra_exit, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = stm32_hash_one_request, + }, + }, + { + .base.init = stm32_hash_init, + .base.update = stm32_hash_update, + .base.final = stm32_hash_final, + .base.finup = stm32_hash_finup, + .base.digest = stm32_hash_digest, + .base.export = stm32_hash_export, + .base.import = stm32_hash_import, + .base.halg = { + .digestsize = SHA3_256_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_state), + .base = { + .cra_name = "sha3-256", + .cra_driver_name = "stm32-sha3-256", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA3_256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_init = stm32_hash_cra_sha3_init, + .cra_exit = stm32_hash_cra_exit, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = stm32_hash_one_request, + }, + }, + { + .base.init = stm32_hash_init, + .base.update = stm32_hash_update, + .base.final = stm32_hash_final, + .base.finup = stm32_hash_finup, + .base.digest = stm32_hash_digest, + .base.export = stm32_hash_export, + .base.import = stm32_hash_import, + .base.setkey = stm32_hash_setkey, + .base.halg = { + .digestsize = SHA3_256_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_state), + .base = { + .cra_name = "hmac(sha3-256)", + .cra_driver_name = "stm32-hmac-sha3-256", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA3_256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_init = stm32_hash_cra_sha3_hmac_init, + .cra_exit = stm32_hash_cra_exit, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = stm32_hash_one_request, + }, + }, + { + .base.init = stm32_hash_init, + .base.update = stm32_hash_update, + .base.final = stm32_hash_final, + .base.finup = stm32_hash_finup, + .base.digest = stm32_hash_digest, + .base.export = stm32_hash_export, + .base.import = stm32_hash_import, + .base.halg = { + .digestsize = SHA3_384_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_state), + .base = { + .cra_name = "sha3-384", + .cra_driver_name = "stm32-sha3-384", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA3_384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_init = stm32_hash_cra_sha3_init, + .cra_exit = stm32_hash_cra_exit, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = stm32_hash_one_request, + }, + }, + { + .base.init = stm32_hash_init, + .base.update = stm32_hash_update, + .base.final = stm32_hash_final, + .base.finup = stm32_hash_finup, + .base.digest = stm32_hash_digest, + .base.export = stm32_hash_export, + .base.import = stm32_hash_import, + .base.setkey = stm32_hash_setkey, + .base.halg = { + .digestsize = SHA3_384_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_state), + .base = { + .cra_name = "hmac(sha3-384)", + .cra_driver_name = "stm32-hmac-sha3-384", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA3_384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_init = stm32_hash_cra_sha3_hmac_init, + .cra_exit = stm32_hash_cra_exit, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = stm32_hash_one_request, + }, + }, + { + .base.init = stm32_hash_init, + .base.update = stm32_hash_update, + .base.final = stm32_hash_final, + .base.finup = stm32_hash_finup, + .base.digest = stm32_hash_digest, + .base.export = stm32_hash_export, + .base.import = stm32_hash_import, + .base.halg = { + .digestsize = SHA3_512_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_state), + .base = { + .cra_name = "sha3-512", + .cra_driver_name = "stm32-sha3-512", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA3_512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_init = stm32_hash_cra_sha3_init, + .cra_exit = stm32_hash_cra_exit, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = stm32_hash_one_request, + }, + }, + { + .base.init = stm32_hash_init, + .base.update = stm32_hash_update, + .base.final = stm32_hash_final, + .base.finup = stm32_hash_finup, + .base.digest = stm32_hash_digest, + .base.export = stm32_hash_export, + .base.import = stm32_hash_import, + .base.setkey = stm32_hash_setkey, + .base.halg = { + .digestsize = SHA3_512_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_state), + .base = { + .cra_name = "hmac(sha3-512)", + .cra_driver_name = "stm32-hmac-sha3-512", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA3_512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_init = stm32_hash_cra_sha3_hmac_init, + .cra_exit = stm32_hash_cra_exit, + .cra_module = THIS_MODULE, + } + }, + .op = { + .do_one_request = stm32_hash_one_request, + }, + } +}; + +static int stm32_hash_register_algs(struct stm32_hash_dev *hdev) +{ + unsigned int i, j; + int err; + + for (i = 0; i < hdev->pdata->algs_info_size; i++) { + for (j = 0; j < hdev->pdata->algs_info[i].size; j++) { + err = crypto_engine_register_ahash( + &hdev->pdata->algs_info[i].algs_list[j]); + if (err) + goto err_algs; + } + } + + return 0; +err_algs: + dev_err(hdev->dev, "Algo %d : %d failed\n", i, j); + for (; i--; ) { + for (; j--;) + crypto_engine_unregister_ahash( + &hdev->pdata->algs_info[i].algs_list[j]); + } + + return err; +} + +static int stm32_hash_unregister_algs(struct stm32_hash_dev *hdev) +{ + unsigned int i, j; + + for (i = 0; i < hdev->pdata->algs_info_size; i++) { + for (j = 0; j < hdev->pdata->algs_info[i].size; j++) + crypto_engine_unregister_ahash( + &hdev->pdata->algs_info[i].algs_list[j]); + } + + return 0; +} + +static struct stm32_hash_algs_info stm32_hash_algs_info_ux500[] = { + { + .algs_list = algs_sha1, + .size = ARRAY_SIZE(algs_sha1), + }, + { + .algs_list = algs_sha256, + .size = ARRAY_SIZE(algs_sha256), + }, +}; + +static const struct stm32_hash_pdata stm32_hash_pdata_ux500 = { + .alg_shift = 7, + .algs_info = stm32_hash_algs_info_ux500, + .algs_info_size = ARRAY_SIZE(stm32_hash_algs_info_ux500), + .broken_emptymsg = true, + .ux500 = true, +}; + +static struct stm32_hash_algs_info stm32_hash_algs_info_stm32f4[] = { + { + .algs_list = algs_md5, + .size = ARRAY_SIZE(algs_md5), + }, + { + .algs_list = algs_sha1, + .size = ARRAY_SIZE(algs_sha1), + }, +}; + +static const struct stm32_hash_pdata stm32_hash_pdata_stm32f4 = { + .alg_shift = 7, + .algs_info = stm32_hash_algs_info_stm32f4, + .algs_info_size = ARRAY_SIZE(stm32_hash_algs_info_stm32f4), + .has_sr = true, + .has_mdmat = true, +}; + +static struct stm32_hash_algs_info stm32_hash_algs_info_stm32f7[] = { + { + .algs_list = algs_md5, + .size = ARRAY_SIZE(algs_md5), + }, + { + .algs_list = algs_sha1, + .size = ARRAY_SIZE(algs_sha1), + }, + { + .algs_list = algs_sha224, + .size = ARRAY_SIZE(algs_sha224), + }, + { + .algs_list = algs_sha256, + .size = ARRAY_SIZE(algs_sha256), + }, +}; + +static const struct stm32_hash_pdata stm32_hash_pdata_stm32f7 = { + .alg_shift = 7, + .algs_info = stm32_hash_algs_info_stm32f7, + .algs_info_size = ARRAY_SIZE(stm32_hash_algs_info_stm32f7), + .has_sr = true, + .has_mdmat = true, +}; + +static struct stm32_hash_algs_info stm32_hash_algs_info_stm32mp13[] = { + { + .algs_list = algs_sha1, + .size = ARRAY_SIZE(algs_sha1), + }, + { + .algs_list = algs_sha224, + .size = ARRAY_SIZE(algs_sha224), + }, + { + .algs_list = algs_sha256, + .size = ARRAY_SIZE(algs_sha256), + }, + { + .algs_list = algs_sha384_sha512, + .size = ARRAY_SIZE(algs_sha384_sha512), + }, + { + .algs_list = algs_sha3, + .size = ARRAY_SIZE(algs_sha3), + }, +}; + +static const struct stm32_hash_pdata stm32_hash_pdata_stm32mp13 = { + .alg_shift = 17, + .algs_info = stm32_hash_algs_info_stm32mp13, + .algs_info_size = ARRAY_SIZE(stm32_hash_algs_info_stm32mp13), + .has_sr = true, + .has_mdmat = true, + .context_secured = true, +}; + +static const struct of_device_id stm32_hash_of_match[] = { + { .compatible = "stericsson,ux500-hash", .data = &stm32_hash_pdata_ux500 }, + { .compatible = "st,stm32f456-hash", .data = &stm32_hash_pdata_stm32f4 }, + { .compatible = "st,stm32f756-hash", .data = &stm32_hash_pdata_stm32f7 }, + { .compatible = "st,stm32mp13-hash", .data = &stm32_hash_pdata_stm32mp13 }, + {}, +}; + +MODULE_DEVICE_TABLE(of, stm32_hash_of_match); + +static int stm32_hash_get_of_match(struct stm32_hash_dev *hdev, + struct device *dev) +{ + hdev->pdata = of_device_get_match_data(dev); + if (!hdev->pdata) { + dev_err(dev, "no compatible OF match\n"); + return -EINVAL; + } + + return 0; +} + +static int stm32_hash_probe(struct platform_device *pdev) +{ + struct stm32_hash_dev *hdev; + struct device *dev = &pdev->dev; + struct resource *res; + int ret, irq; + + hdev = devm_kzalloc(dev, sizeof(*hdev), GFP_KERNEL); + if (!hdev) + return -ENOMEM; + + hdev->io_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); + if (IS_ERR(hdev->io_base)) + return PTR_ERR(hdev->io_base); + + hdev->phys_base = res->start; + + ret = stm32_hash_get_of_match(hdev, dev); + if (ret) + return ret; + + irq = platform_get_irq_optional(pdev, 0); + if (irq < 0 && irq != -ENXIO) + return irq; + + if (irq > 0) { + ret = devm_request_threaded_irq(dev, irq, + stm32_hash_irq_handler, + stm32_hash_irq_thread, + IRQF_ONESHOT, + dev_name(dev), hdev); + if (ret) + return ret; + } else { + dev_info(dev, "No IRQ, use polling mode\n"); + hdev->polled = true; + } + + hdev->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(hdev->clk)) + return dev_err_probe(dev, PTR_ERR(hdev->clk), + "failed to get clock for hash\n"); + + ret = clk_prepare_enable(hdev->clk); + if (ret) { + dev_err(dev, "failed to enable hash clock (%d)\n", ret); + return ret; + } + + pm_runtime_set_autosuspend_delay(dev, HASH_AUTOSUSPEND_DELAY); + pm_runtime_use_autosuspend(dev); + + pm_runtime_get_noresume(dev); + pm_runtime_set_active(dev); + pm_runtime_enable(dev); + + hdev->rst = devm_reset_control_get(&pdev->dev, NULL); + if (IS_ERR(hdev->rst)) { + if (PTR_ERR(hdev->rst) == -EPROBE_DEFER) { + ret = -EPROBE_DEFER; + goto err_reset; + } + } else { + reset_control_assert(hdev->rst); + udelay(2); + reset_control_deassert(hdev->rst); + } + + hdev->dev = dev; + + platform_set_drvdata(pdev, hdev); + + ret = stm32_hash_dma_init(hdev); + switch (ret) { + case 0: + break; + case -ENOENT: + case -ENODEV: + dev_info(dev, "DMA mode not available\n"); + break; + default: + dev_err(dev, "DMA init error %d\n", ret); + goto err_dma; + } + + spin_lock(&stm32_hash.lock); + list_add_tail(&hdev->list, &stm32_hash.dev_list); + spin_unlock(&stm32_hash.lock); + + /* Initialize crypto engine */ + hdev->engine = crypto_engine_alloc_init(dev, 1); + if (!hdev->engine) { + ret = -ENOMEM; + goto err_engine; + } + + ret = crypto_engine_start(hdev->engine); + if (ret) + goto err_engine_start; + + if (hdev->pdata->ux500) + /* FIXME: implement DMA mode for Ux500 */ + hdev->dma_mode = 0; + else + hdev->dma_mode = stm32_hash_read(hdev, HASH_HWCFGR) & HASH_HWCFG_DMA_MASK; + + /* Register algos */ + ret = stm32_hash_register_algs(hdev); + if (ret) + goto err_algs; + + dev_info(dev, "Init HASH done HW ver %x DMA mode %u\n", + stm32_hash_read(hdev, HASH_VER), hdev->dma_mode); + + pm_runtime_put_sync(dev); + + return 0; + +err_algs: +err_engine_start: + crypto_engine_exit(hdev->engine); +err_engine: + spin_lock(&stm32_hash.lock); + list_del(&hdev->list); + spin_unlock(&stm32_hash.lock); +err_dma: + if (hdev->dma_lch) + dma_release_channel(hdev->dma_lch); +err_reset: + pm_runtime_disable(dev); + pm_runtime_put_noidle(dev); + + clk_disable_unprepare(hdev->clk); + + return ret; +} + +static void stm32_hash_remove(struct platform_device *pdev) +{ + struct stm32_hash_dev *hdev = platform_get_drvdata(pdev); + int ret; + + ret = pm_runtime_get_sync(hdev->dev); + + stm32_hash_unregister_algs(hdev); + + crypto_engine_exit(hdev->engine); + + spin_lock(&stm32_hash.lock); + list_del(&hdev->list); + spin_unlock(&stm32_hash.lock); + + if (hdev->dma_lch) + dma_release_channel(hdev->dma_lch); + + pm_runtime_disable(hdev->dev); + pm_runtime_put_noidle(hdev->dev); + + if (ret >= 0) + clk_disable_unprepare(hdev->clk); +} + +#ifdef CONFIG_PM +static int stm32_hash_runtime_suspend(struct device *dev) +{ + struct stm32_hash_dev *hdev = dev_get_drvdata(dev); + + clk_disable_unprepare(hdev->clk); + + return 0; +} + +static int stm32_hash_runtime_resume(struct device *dev) +{ + struct stm32_hash_dev *hdev = dev_get_drvdata(dev); + int ret; + + ret = clk_prepare_enable(hdev->clk); + if (ret) { + dev_err(hdev->dev, "Failed to prepare_enable clock\n"); + return ret; + } + + return 0; +} +#endif + +static const struct dev_pm_ops stm32_hash_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, + pm_runtime_force_resume) + SET_RUNTIME_PM_OPS(stm32_hash_runtime_suspend, + stm32_hash_runtime_resume, NULL) +}; + +static struct platform_driver stm32_hash_driver = { + .probe = stm32_hash_probe, + .remove = stm32_hash_remove, + .driver = { + .name = "stm32-hash", + .pm = &stm32_hash_pm_ops, + .of_match_table = stm32_hash_of_match, + } +}; + +module_platform_driver(stm32_hash_driver); + +MODULE_DESCRIPTION("STM32 SHA1/SHA2/SHA3 & MD5 (HMAC) hw accelerator driver"); +MODULE_AUTHOR("Lionel Debieve "); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c new file mode 100644 index 000000000..fa96b6bf1 --- /dev/null +++ b/drivers/crypto/talitos.c @@ -0,0 +1,3635 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * talitos - Freescale Integrated Security Engine (SEC) device driver + * + * Copyright (c) 2008-2011 Freescale Semiconductor, Inc. + * + * Scatterlist Crypto API glue code copied from files with the following: + * Copyright (c) 2006-2007 Herbert Xu + * + * Crypto algorithm registration code copied from hifn driver: + * 2007+ Copyright (c) Evgeniy Polyakov + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "talitos.h" + +static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr, + unsigned int len, bool is_sec1) +{ + ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr)); + if (is_sec1) { + ptr->len1 = cpu_to_be16(len); + } else { + ptr->len = cpu_to_be16(len); + ptr->eptr = upper_32_bits(dma_addr); + } +} + +static void copy_talitos_ptr(struct talitos_ptr *dst_ptr, + struct talitos_ptr *src_ptr, bool is_sec1) +{ + dst_ptr->ptr = src_ptr->ptr; + if (is_sec1) { + dst_ptr->len1 = src_ptr->len1; + } else { + dst_ptr->len = src_ptr->len; + dst_ptr->eptr = src_ptr->eptr; + } +} + +static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr, + bool is_sec1) +{ + if (is_sec1) + return be16_to_cpu(ptr->len1); + else + return be16_to_cpu(ptr->len); +} + +static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val, + bool is_sec1) +{ + if (!is_sec1) + ptr->j_extent = val; +} + +static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1) +{ + if (!is_sec1) + ptr->j_extent |= val; +} + +/* + * map virtual single (contiguous) pointer to h/w descriptor pointer + */ +static void __map_single_talitos_ptr(struct device *dev, + struct talitos_ptr *ptr, + unsigned int len, void *data, + enum dma_data_direction dir, + unsigned long attrs) +{ + dma_addr_t dma_addr = dma_map_single_attrs(dev, data, len, dir, attrs); + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); + + to_talitos_ptr(ptr, dma_addr, len, is_sec1); +} + +static void map_single_talitos_ptr(struct device *dev, + struct talitos_ptr *ptr, + unsigned int len, void *data, + enum dma_data_direction dir) +{ + __map_single_talitos_ptr(dev, ptr, len, data, dir, 0); +} + +static void map_single_talitos_ptr_nosync(struct device *dev, + struct talitos_ptr *ptr, + unsigned int len, void *data, + enum dma_data_direction dir) +{ + __map_single_talitos_ptr(dev, ptr, len, data, dir, + DMA_ATTR_SKIP_CPU_SYNC); +} + +/* + * unmap bus single (contiguous) h/w descriptor pointer + */ +static void unmap_single_talitos_ptr(struct device *dev, + struct talitos_ptr *ptr, + enum dma_data_direction dir) +{ + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); + + dma_unmap_single(dev, be32_to_cpu(ptr->ptr), + from_talitos_ptr_len(ptr, is_sec1), dir); +} + +static int reset_channel(struct device *dev, int ch) +{ + struct talitos_private *priv = dev_get_drvdata(dev); + unsigned int timeout = TALITOS_TIMEOUT; + bool is_sec1 = has_ftr_sec1(priv); + + if (is_sec1) { + setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, + TALITOS1_CCCR_LO_RESET); + + while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) & + TALITOS1_CCCR_LO_RESET) && --timeout) + cpu_relax(); + } else { + setbits32(priv->chan[ch].reg + TALITOS_CCCR, + TALITOS2_CCCR_RESET); + + while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & + TALITOS2_CCCR_RESET) && --timeout) + cpu_relax(); + } + + if (timeout == 0) { + dev_err(dev, "failed to reset channel %d\n", ch); + return -EIO; + } + + /* set 36-bit addressing, done writeback enable and done IRQ enable */ + setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE | + TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE); + /* enable chaining descriptors */ + if (is_sec1) + setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, + TALITOS_CCCR_LO_NE); + + /* and ICCR writeback, if available */ + if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) + setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, + TALITOS_CCCR_LO_IWSE); + + return 0; +} + +static int reset_device(struct device *dev) +{ + struct talitos_private *priv = dev_get_drvdata(dev); + unsigned int timeout = TALITOS_TIMEOUT; + bool is_sec1 = has_ftr_sec1(priv); + u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR; + + setbits32(priv->reg + TALITOS_MCR, mcr); + + while ((in_be32(priv->reg + TALITOS_MCR) & mcr) + && --timeout) + cpu_relax(); + + if (priv->irq[1] > 0) { + mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3; + setbits32(priv->reg + TALITOS_MCR, mcr); + } + + if (timeout == 0) { + dev_err(dev, "failed to reset device\n"); + return -EIO; + } + + return 0; +} + +/* + * Reset and initialize the device + */ +static int init_device(struct device *dev) +{ + struct talitos_private *priv = dev_get_drvdata(dev); + int ch, err; + bool is_sec1 = has_ftr_sec1(priv); + + /* + * Master reset + * errata documentation: warning: certain SEC interrupts + * are not fully cleared by writing the MCR:SWR bit, + * set bit twice to completely reset + */ + err = reset_device(dev); + if (err) + return err; + + err = reset_device(dev); + if (err) + return err; + + /* reset channels */ + for (ch = 0; ch < priv->num_channels; ch++) { + err = reset_channel(dev, ch); + if (err) + return err; + } + + /* enable channel done and error interrupts */ + if (is_sec1) { + clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT); + clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); + /* disable parity error check in DEU (erroneous? test vect.) */ + setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE); + } else { + setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT); + setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); + } + + /* disable integrity check error interrupts (use writeback instead) */ + if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) + setbits32(priv->reg_mdeu + TALITOS_EUICR_LO, + TALITOS_MDEUICR_LO_ICE); + + return 0; +} + +static void dma_map_request(struct device *dev, struct talitos_request *request, + struct talitos_desc *desc, bool is_sec1) +{ + struct talitos_edesc *edesc = + container_of(desc, struct talitos_edesc, desc); + dma_addr_t dma_desc, prev_dma_desc; + struct talitos_edesc *prev_edesc = NULL; + + if (is_sec1) { + while (edesc) { + edesc->desc.hdr1 = edesc->desc.hdr; + + dma_desc = dma_map_single(dev, &edesc->desc.hdr1, + TALITOS_DESC_SIZE, + DMA_BIDIRECTIONAL); + + if (!prev_edesc) { + request->dma_desc = dma_desc; + goto next; + } + + /* Chain in any previous descriptors. */ + + prev_edesc->desc.next_desc = cpu_to_be32(dma_desc); + + dma_sync_single_for_device(dev, prev_dma_desc, + TALITOS_DESC_SIZE, + DMA_TO_DEVICE); + +next: + prev_edesc = edesc; + prev_dma_desc = dma_desc; + edesc = edesc->next_desc; + } + } else { + request->dma_desc = dma_map_single(dev, desc, TALITOS_DESC_SIZE, + DMA_BIDIRECTIONAL); + } +} + +/** + * talitos_submit - submits a descriptor to the device for processing + * @dev: the SEC device to be used + * @ch: the SEC device channel to be used + * @desc: the descriptor to be processed by the device + * @callback: whom to call when processing is complete + * @context: a handle for use by caller (optional) + * + * desc must contain valid dma-mapped (bus physical) address pointers. + * callback must check err and feedback in descriptor header + * for device processing status. + */ +static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, + void (*callback)(struct device *dev, + struct talitos_desc *desc, + void *context, int error), + void *context) +{ + struct talitos_private *priv = dev_get_drvdata(dev); + struct talitos_request *request; + unsigned long flags; + int head; + bool is_sec1 = has_ftr_sec1(priv); + + spin_lock_irqsave(&priv->chan[ch].head_lock, flags); + + if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) { + /* h/w fifo is full */ + spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags); + return -EAGAIN; + } + + head = priv->chan[ch].head; + request = &priv->chan[ch].fifo[head]; + + /* map descriptor and save caller data */ + dma_map_request(dev, request, desc, is_sec1); + request->callback = callback; + request->context = context; + + /* increment fifo head */ + priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1); + + smp_wmb(); + request->desc = desc; + + /* GO! */ + wmb(); + out_be32(priv->chan[ch].reg + TALITOS_FF, + upper_32_bits(request->dma_desc)); + out_be32(priv->chan[ch].reg + TALITOS_FF_LO, + lower_32_bits(request->dma_desc)); + + spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags); + + return -EINPROGRESS; +} + +static __be32 get_request_hdr(struct device *dev, + struct talitos_request *request, bool is_sec1) +{ + struct talitos_edesc *edesc; + dma_addr_t dma_desc; + + if (!is_sec1) { + dma_sync_single_for_cpu(dev, request->dma_desc, + TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); + + return request->desc->hdr; + } + + edesc = container_of(request->desc, struct talitos_edesc, desc); + dma_desc = request->dma_desc; + while (edesc->next_desc) { + dma_desc = be32_to_cpu(edesc->desc.next_desc); + edesc = edesc->next_desc; + } + + dma_sync_single_for_cpu(dev, dma_desc, TALITOS_DESC_SIZE, + DMA_BIDIRECTIONAL); + + return edesc->desc.hdr1; +} + +static void dma_unmap_request(struct device *dev, + struct talitos_request *request, bool is_sec1) +{ + struct talitos_edesc *edesc; + + if (is_sec1) { + dma_unmap_single(dev, request->dma_desc, TALITOS_DESC_SIZE, + DMA_BIDIRECTIONAL); + edesc = container_of(request->desc, struct talitos_edesc, desc); + while (edesc->next_desc) { + dma_unmap_single(dev, + be32_to_cpu(edesc->desc.next_desc), + TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); + edesc = edesc->next_desc; + } + } else { + dma_unmap_single(dev, request->dma_desc, TALITOS_DESC_SIZE, + DMA_BIDIRECTIONAL); + } +} + +/* + * process what was done, notify callback of error if not + */ +static void flush_channel(struct device *dev, int ch, int error, int reset_ch) +{ + struct talitos_private *priv = dev_get_drvdata(dev); + struct talitos_request *request, saved_req; + unsigned long flags; + int tail, status; + bool is_sec1 = has_ftr_sec1(priv); + + spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); + + tail = priv->chan[ch].tail; + while (priv->chan[ch].fifo[tail].desc) { + __be32 hdr; + + request = &priv->chan[ch].fifo[tail]; + + /* descriptors with their done bits set don't get the error */ + rmb(); + hdr = get_request_hdr(dev, request, is_sec1); + + if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE) + status = 0; + else + if (!error) + break; + else + status = error; + + dma_unmap_request(dev, request, is_sec1); + + /* copy entries so we can call callback outside lock */ + saved_req.desc = request->desc; + saved_req.callback = request->callback; + saved_req.context = request->context; + + /* release request entry in fifo */ + smp_wmb(); + request->desc = NULL; + + /* increment fifo tail */ + priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1); + + spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags); + + atomic_dec(&priv->chan[ch].submit_count); + + saved_req.callback(dev, saved_req.desc, saved_req.context, + status); + /* channel may resume processing in single desc error case */ + if (error && !reset_ch && status == error) + return; + spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); + tail = priv->chan[ch].tail; + } + + spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags); +} + +/* + * process completed requests for channels that have done status + */ +#define DEF_TALITOS1_DONE(name, ch_done_mask) \ +static void talitos1_done_##name(unsigned long data) \ +{ \ + struct device *dev = (struct device *)data; \ + struct talitos_private *priv = dev_get_drvdata(dev); \ + unsigned long flags; \ + \ + if (ch_done_mask & 0x10000000) \ + flush_channel(dev, 0, 0, 0); \ + if (ch_done_mask & 0x40000000) \ + flush_channel(dev, 1, 0, 0); \ + if (ch_done_mask & 0x00010000) \ + flush_channel(dev, 2, 0, 0); \ + if (ch_done_mask & 0x00040000) \ + flush_channel(dev, 3, 0, 0); \ + \ + /* At this point, all completed channels have been processed */ \ + /* Unmask done interrupts for channels completed later on. */ \ + spin_lock_irqsave(&priv->reg_lock, flags); \ + clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ + clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); \ + spin_unlock_irqrestore(&priv->reg_lock, flags); \ +} + +DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE) +DEF_TALITOS1_DONE(ch0, TALITOS1_ISR_CH_0_DONE) + +#define DEF_TALITOS2_DONE(name, ch_done_mask) \ +static void talitos2_done_##name(unsigned long data) \ +{ \ + struct device *dev = (struct device *)data; \ + struct talitos_private *priv = dev_get_drvdata(dev); \ + unsigned long flags; \ + \ + if (ch_done_mask & 1) \ + flush_channel(dev, 0, 0, 0); \ + if (ch_done_mask & (1 << 2)) \ + flush_channel(dev, 1, 0, 0); \ + if (ch_done_mask & (1 << 4)) \ + flush_channel(dev, 2, 0, 0); \ + if (ch_done_mask & (1 << 6)) \ + flush_channel(dev, 3, 0, 0); \ + \ + /* At this point, all completed channels have been processed */ \ + /* Unmask done interrupts for channels completed later on. */ \ + spin_lock_irqsave(&priv->reg_lock, flags); \ + setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ + setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); \ + spin_unlock_irqrestore(&priv->reg_lock, flags); \ +} + +DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE) +DEF_TALITOS2_DONE(ch0, TALITOS2_ISR_CH_0_DONE) +DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE) +DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE) + +static __be32 search_desc_hdr_in_request(struct talitos_request *request, + dma_addr_t cur_desc, bool is_sec1) +{ + struct talitos_edesc *edesc; + + if (request->dma_desc == cur_desc) { + return request->desc->hdr; + } else if (is_sec1) { + edesc = container_of(request->desc, struct talitos_edesc, desc); + while (edesc->next_desc) { + if (edesc->desc.next_desc == cpu_to_be32(cur_desc)) + return edesc->next_desc->desc.hdr1; + edesc = edesc->next_desc; + } + } + return 0; +} + +/* + * locate current (offending) descriptor + */ +static __be32 current_desc_hdr(struct device *dev, int ch) +{ + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); + struct talitos_request *request; + int tail, iter; + dma_addr_t cur_desc; + __be32 hdr = 0; + + cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32; + cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO); + + if (!cur_desc) { + dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n"); + return 0; + } + + tail = priv->chan[ch].tail; + iter = tail; + do { + request = &priv->chan[ch].fifo[iter]; + + hdr = search_desc_hdr_in_request(request, cur_desc, is_sec1); + if (hdr) + break; + + iter = (iter + 1) & (priv->fifo_len - 1); + } while (iter != tail); + + if (!hdr) + dev_err(dev, "couldn't locate current descriptor\n"); + + return hdr; +} + +/* + * user diagnostics; report root cause of error based on execution unit status + */ +static void report_eu_error(struct device *dev, int ch, __be32 desc_hdr) +{ + struct talitos_private *priv = dev_get_drvdata(dev); + int i; + + if (!desc_hdr) + desc_hdr = cpu_to_be32(in_be32(priv->chan[ch].reg + TALITOS_DESCBUF)); + + switch (desc_hdr & DESC_HDR_SEL0_MASK) { + case DESC_HDR_SEL0_AFEU: + dev_err(dev, "AFEUISR 0x%08x_%08x\n", + in_be32(priv->reg_afeu + TALITOS_EUISR), + in_be32(priv->reg_afeu + TALITOS_EUISR_LO)); + break; + case DESC_HDR_SEL0_DEU: + dev_err(dev, "DEUISR 0x%08x_%08x\n", + in_be32(priv->reg_deu + TALITOS_EUISR), + in_be32(priv->reg_deu + TALITOS_EUISR_LO)); + break; + case DESC_HDR_SEL0_MDEUA: + case DESC_HDR_SEL0_MDEUB: + dev_err(dev, "MDEUISR 0x%08x_%08x\n", + in_be32(priv->reg_mdeu + TALITOS_EUISR), + in_be32(priv->reg_mdeu + TALITOS_EUISR_LO)); + break; + case DESC_HDR_SEL0_RNG: + dev_err(dev, "RNGUISR 0x%08x_%08x\n", + in_be32(priv->reg_rngu + TALITOS_ISR), + in_be32(priv->reg_rngu + TALITOS_ISR_LO)); + break; + case DESC_HDR_SEL0_PKEU: + dev_err(dev, "PKEUISR 0x%08x_%08x\n", + in_be32(priv->reg_pkeu + TALITOS_EUISR), + in_be32(priv->reg_pkeu + TALITOS_EUISR_LO)); + break; + case DESC_HDR_SEL0_AESU: + dev_err(dev, "AESUISR 0x%08x_%08x\n", + in_be32(priv->reg_aesu + TALITOS_EUISR), + in_be32(priv->reg_aesu + TALITOS_EUISR_LO)); + break; + case DESC_HDR_SEL0_CRCU: + dev_err(dev, "CRCUISR 0x%08x_%08x\n", + in_be32(priv->reg_crcu + TALITOS_EUISR), + in_be32(priv->reg_crcu + TALITOS_EUISR_LO)); + break; + case DESC_HDR_SEL0_KEU: + dev_err(dev, "KEUISR 0x%08x_%08x\n", + in_be32(priv->reg_pkeu + TALITOS_EUISR), + in_be32(priv->reg_pkeu + TALITOS_EUISR_LO)); + break; + } + + switch (desc_hdr & DESC_HDR_SEL1_MASK) { + case DESC_HDR_SEL1_MDEUA: + case DESC_HDR_SEL1_MDEUB: + dev_err(dev, "MDEUISR 0x%08x_%08x\n", + in_be32(priv->reg_mdeu + TALITOS_EUISR), + in_be32(priv->reg_mdeu + TALITOS_EUISR_LO)); + break; + case DESC_HDR_SEL1_CRCU: + dev_err(dev, "CRCUISR 0x%08x_%08x\n", + in_be32(priv->reg_crcu + TALITOS_EUISR), + in_be32(priv->reg_crcu + TALITOS_EUISR_LO)); + break; + } + + for (i = 0; i < 8; i++) + dev_err(dev, "DESCBUF 0x%08x_%08x\n", + in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i), + in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i)); +} + +/* + * recover from error interrupts + */ +static void talitos_error(struct device *dev, u32 isr, u32 isr_lo) +{ + struct talitos_private *priv = dev_get_drvdata(dev); + unsigned int timeout = TALITOS_TIMEOUT; + int ch, error, reset_dev = 0; + u32 v_lo; + bool is_sec1 = has_ftr_sec1(priv); + int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */ + + for (ch = 0; ch < priv->num_channels; ch++) { + /* skip channels without errors */ + if (is_sec1) { + /* bits 29, 31, 17, 19 */ + if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6)))) + continue; + } else { + if (!(isr & (1 << (ch * 2 + 1)))) + continue; + } + + error = -EINVAL; + + v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO); + + if (v_lo & TALITOS_CCPSR_LO_DOF) { + dev_err(dev, "double fetch fifo overflow error\n"); + error = -EAGAIN; + reset_ch = 1; + } + if (v_lo & TALITOS_CCPSR_LO_SOF) { + /* h/w dropped descriptor */ + dev_err(dev, "single fetch fifo overflow error\n"); + error = -EAGAIN; + } + if (v_lo & TALITOS_CCPSR_LO_MDTE) + dev_err(dev, "master data transfer error\n"); + if (v_lo & TALITOS_CCPSR_LO_SGDLZ) + dev_err(dev, is_sec1 ? "pointer not complete error\n" + : "s/g data length zero error\n"); + if (v_lo & TALITOS_CCPSR_LO_FPZ) + dev_err(dev, is_sec1 ? "parity error\n" + : "fetch pointer zero error\n"); + if (v_lo & TALITOS_CCPSR_LO_IDH) + dev_err(dev, "illegal descriptor header error\n"); + if (v_lo & TALITOS_CCPSR_LO_IEU) + dev_err(dev, is_sec1 ? "static assignment error\n" + : "invalid exec unit error\n"); + if (v_lo & TALITOS_CCPSR_LO_EU) + report_eu_error(dev, ch, current_desc_hdr(dev, ch)); + if (!is_sec1) { + if (v_lo & TALITOS_CCPSR_LO_GB) + dev_err(dev, "gather boundary error\n"); + if (v_lo & TALITOS_CCPSR_LO_GRL) + dev_err(dev, "gather return/length error\n"); + if (v_lo & TALITOS_CCPSR_LO_SB) + dev_err(dev, "scatter boundary error\n"); + if (v_lo & TALITOS_CCPSR_LO_SRL) + dev_err(dev, "scatter return/length error\n"); + } + + flush_channel(dev, ch, error, reset_ch); + + if (reset_ch) { + reset_channel(dev, ch); + } else { + setbits32(priv->chan[ch].reg + TALITOS_CCCR, + TALITOS2_CCCR_CONT); + setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0); + while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & + TALITOS2_CCCR_CONT) && --timeout) + cpu_relax(); + if (timeout == 0) { + dev_err(dev, "failed to restart channel %d\n", + ch); + reset_dev = 1; + } + } + } + if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) || + (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) { + if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR)) + dev_err(dev, "TEA error: ISR 0x%08x_%08x\n", + isr, isr_lo); + else + dev_err(dev, "done overflow, internal time out, or " + "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo); + + /* purge request queues */ + for (ch = 0; ch < priv->num_channels; ch++) + flush_channel(dev, ch, -EIO, 1); + + /* reset and reinitialize the device */ + init_device(dev); + } +} + +#define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \ +static irqreturn_t talitos1_interrupt_##name(int irq, void *data) \ +{ \ + struct device *dev = data; \ + struct talitos_private *priv = dev_get_drvdata(dev); \ + u32 isr, isr_lo; \ + unsigned long flags; \ + \ + spin_lock_irqsave(&priv->reg_lock, flags); \ + isr = in_be32(priv->reg + TALITOS_ISR); \ + isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \ + /* Acknowledge interrupt */ \ + out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \ + out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \ + \ + if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) { \ + spin_unlock_irqrestore(&priv->reg_lock, flags); \ + talitos_error(dev, isr & ch_err_mask, isr_lo); \ + } \ + else { \ + if (likely(isr & ch_done_mask)) { \ + /* mask further done interrupts. */ \ + setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ + /* done_task will unmask done interrupts at exit */ \ + tasklet_schedule(&priv->done_task[tlet]); \ + } \ + spin_unlock_irqrestore(&priv->reg_lock, flags); \ + } \ + \ + return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \ + IRQ_NONE; \ +} + +DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0) + +#define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \ +static irqreturn_t talitos2_interrupt_##name(int irq, void *data) \ +{ \ + struct device *dev = data; \ + struct talitos_private *priv = dev_get_drvdata(dev); \ + u32 isr, isr_lo; \ + unsigned long flags; \ + \ + spin_lock_irqsave(&priv->reg_lock, flags); \ + isr = in_be32(priv->reg + TALITOS_ISR); \ + isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \ + /* Acknowledge interrupt */ \ + out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \ + out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \ + \ + if (unlikely(isr & ch_err_mask || isr_lo)) { \ + spin_unlock_irqrestore(&priv->reg_lock, flags); \ + talitos_error(dev, isr & ch_err_mask, isr_lo); \ + } \ + else { \ + if (likely(isr & ch_done_mask)) { \ + /* mask further done interrupts. */ \ + clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ + /* done_task will unmask done interrupts at exit */ \ + tasklet_schedule(&priv->done_task[tlet]); \ + } \ + spin_unlock_irqrestore(&priv->reg_lock, flags); \ + } \ + \ + return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \ + IRQ_NONE; \ +} + +DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0) +DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR, + 0) +DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR, + 1) + +/* + * hwrng + */ +static int talitos_rng_data_present(struct hwrng *rng, int wait) +{ + struct device *dev = (struct device *)rng->priv; + struct talitos_private *priv = dev_get_drvdata(dev); + u32 ofl; + int i; + + for (i = 0; i < 20; i++) { + ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) & + TALITOS_RNGUSR_LO_OFL; + if (ofl || !wait) + break; + udelay(10); + } + + return !!ofl; +} + +static int talitos_rng_data_read(struct hwrng *rng, u32 *data) +{ + struct device *dev = (struct device *)rng->priv; + struct talitos_private *priv = dev_get_drvdata(dev); + + /* rng fifo requires 64-bit accesses */ + *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO); + *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO); + + return sizeof(u32); +} + +static int talitos_rng_init(struct hwrng *rng) +{ + struct device *dev = (struct device *)rng->priv; + struct talitos_private *priv = dev_get_drvdata(dev); + unsigned int timeout = TALITOS_TIMEOUT; + + setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR); + while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO) + & TALITOS_RNGUSR_LO_RD) + && --timeout) + cpu_relax(); + if (timeout == 0) { + dev_err(dev, "failed to reset rng hw\n"); + return -ENODEV; + } + + /* start generating */ + setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0); + + return 0; +} + +static int talitos_register_rng(struct device *dev) +{ + struct talitos_private *priv = dev_get_drvdata(dev); + int err; + + priv->rng.name = dev_driver_string(dev); + priv->rng.init = talitos_rng_init; + priv->rng.data_present = talitos_rng_data_present; + priv->rng.data_read = talitos_rng_data_read; + priv->rng.priv = (unsigned long)dev; + + err = hwrng_register(&priv->rng); + if (!err) + priv->rng_registered = true; + + return err; +} + +static void talitos_unregister_rng(struct device *dev) +{ + struct talitos_private *priv = dev_get_drvdata(dev); + + if (!priv->rng_registered) + return; + + hwrng_unregister(&priv->rng); + priv->rng_registered = false; +} + +/* + * crypto alg + */ +#define TALITOS_CRA_PRIORITY 3000 +/* + * Defines a priority for doing AEAD with descriptors type + * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP + */ +#define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1) +#ifdef CONFIG_CRYPTO_DEV_TALITOS2 +#define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE) +#else +#define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA256_BLOCK_SIZE) +#endif +#define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ + +struct talitos_ctx { + struct device *dev; + int ch; + __be32 desc_hdr_template; + u8 key[TALITOS_MAX_KEY_SIZE]; + u8 iv[TALITOS_MAX_IV_LENGTH]; + dma_addr_t dma_key; + unsigned int keylen; + unsigned int enckeylen; + unsigned int authkeylen; +}; + +#define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE +#define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512 + +struct talitos_ahash_req_ctx { + u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)]; + unsigned int hw_context_size; + u8 buf[2][HASH_MAX_BLOCK_SIZE]; + int buf_idx; + unsigned int swinit; + unsigned int first_request; + unsigned int last_request; + unsigned int to_hash_later; + unsigned int nbuf; + struct scatterlist bufsl[2]; + struct scatterlist *psrc; +}; + +struct talitos_export_state { + u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)]; + u8 buf[HASH_MAX_BLOCK_SIZE]; + unsigned int swinit; + unsigned int first_request; + unsigned int last_request; + unsigned int to_hash_later; + unsigned int nbuf; +}; + +static int aead_setkey(struct crypto_aead *authenc, + const u8 *key, unsigned int keylen) +{ + struct talitos_ctx *ctx = crypto_aead_ctx(authenc); + struct device *dev = ctx->dev; + struct crypto_authenc_keys keys; + + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) + goto badkey; + + if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE) + goto badkey; + + if (ctx->keylen) + dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); + + memcpy(ctx->key, keys.authkey, keys.authkeylen); + memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen); + + ctx->keylen = keys.authkeylen + keys.enckeylen; + ctx->enckeylen = keys.enckeylen; + ctx->authkeylen = keys.authkeylen; + ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen, + DMA_TO_DEVICE); + + memzero_explicit(&keys, sizeof(keys)); + return 0; + +badkey: + memzero_explicit(&keys, sizeof(keys)); + return -EINVAL; +} + +static int aead_des3_setkey(struct crypto_aead *authenc, + const u8 *key, unsigned int keylen) +{ + struct talitos_ctx *ctx = crypto_aead_ctx(authenc); + struct device *dev = ctx->dev; + struct crypto_authenc_keys keys; + int err; + + err = crypto_authenc_extractkeys(&keys, key, keylen); + if (unlikely(err)) + goto out; + + err = -EINVAL; + if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE) + goto out; + + err = verify_aead_des3_key(authenc, keys.enckey, keys.enckeylen); + if (err) + goto out; + + if (ctx->keylen) + dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); + + memcpy(ctx->key, keys.authkey, keys.authkeylen); + memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen); + + ctx->keylen = keys.authkeylen + keys.enckeylen; + ctx->enckeylen = keys.enckeylen; + ctx->authkeylen = keys.authkeylen; + ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen, + DMA_TO_DEVICE); + +out: + memzero_explicit(&keys, sizeof(keys)); + return err; +} + +static void talitos_sg_unmap(struct device *dev, + struct talitos_edesc *edesc, + struct scatterlist *src, + struct scatterlist *dst, + unsigned int len, unsigned int offset) +{ + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); + unsigned int src_nents = edesc->src_nents ? : 1; + unsigned int dst_nents = edesc->dst_nents ? : 1; + + if (is_sec1 && dst && dst_nents > 1) { + dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset, + len, DMA_FROM_DEVICE); + sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len, + offset); + } + if (src != dst) { + if (src_nents == 1 || !is_sec1) + dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); + + if (dst && (dst_nents == 1 || !is_sec1)) + dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); + } else if (src_nents == 1 || !is_sec1) { + dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); + } +} + +static void ipsec_esp_unmap(struct device *dev, + struct talitos_edesc *edesc, + struct aead_request *areq, bool encrypt) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct talitos_ctx *ctx = crypto_aead_ctx(aead); + unsigned int ivsize = crypto_aead_ivsize(aead); + unsigned int authsize = crypto_aead_authsize(aead); + unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); + bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP; + struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3]; + + if (is_ipsec_esp) + unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], + DMA_FROM_DEVICE); + unmap_single_talitos_ptr(dev, civ_ptr, DMA_TO_DEVICE); + + talitos_sg_unmap(dev, edesc, areq->src, areq->dst, + cryptlen + authsize, areq->assoclen); + + if (edesc->dma_len) + dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, + DMA_BIDIRECTIONAL); + + if (!is_ipsec_esp) { + unsigned int dst_nents = edesc->dst_nents ? : 1; + + sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize, + areq->assoclen + cryptlen - ivsize); + } +} + +/* + * ipsec_esp descriptor callbacks + */ +static void ipsec_esp_encrypt_done(struct device *dev, + struct talitos_desc *desc, void *context, + int err) +{ + struct aead_request *areq = context; + struct crypto_aead *authenc = crypto_aead_reqtfm(areq); + unsigned int ivsize = crypto_aead_ivsize(authenc); + struct talitos_edesc *edesc; + + edesc = container_of(desc, struct talitos_edesc, desc); + + ipsec_esp_unmap(dev, edesc, areq, true); + + dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE); + + kfree(edesc); + + aead_request_complete(areq, err); +} + +static void ipsec_esp_decrypt_swauth_done(struct device *dev, + struct talitos_desc *desc, + void *context, int err) +{ + struct aead_request *req = context; + struct crypto_aead *authenc = crypto_aead_reqtfm(req); + unsigned int authsize = crypto_aead_authsize(authenc); + struct talitos_edesc *edesc; + char *oicv, *icv; + + edesc = container_of(desc, struct talitos_edesc, desc); + + ipsec_esp_unmap(dev, edesc, req, false); + + if (!err) { + /* auth check */ + oicv = edesc->buf + edesc->dma_len; + icv = oicv - authsize; + + err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0; + } + + kfree(edesc); + + aead_request_complete(req, err); +} + +static void ipsec_esp_decrypt_hwauth_done(struct device *dev, + struct talitos_desc *desc, + void *context, int err) +{ + struct aead_request *req = context; + struct talitos_edesc *edesc; + + edesc = container_of(desc, struct talitos_edesc, desc); + + ipsec_esp_unmap(dev, edesc, req, false); + + /* check ICV auth status */ + if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) != + DESC_HDR_LO_ICCR1_PASS)) + err = -EBADMSG; + + kfree(edesc); + + aead_request_complete(req, err); +} + +/* + * convert scatterlist to SEC h/w link table format + * stop at cryptlen bytes + */ +static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count, + unsigned int offset, int datalen, int elen, + struct talitos_ptr *link_tbl_ptr, int align) +{ + int n_sg = elen ? sg_count + 1 : sg_count; + int count = 0; + int cryptlen = datalen + elen; + int padding = ALIGN(cryptlen, align) - cryptlen; + + while (cryptlen && sg && n_sg--) { + unsigned int len = sg_dma_len(sg); + + if (offset >= len) { + offset -= len; + goto next; + } + + len -= offset; + + if (len > cryptlen) + len = cryptlen; + + if (datalen > 0 && len > datalen) { + to_talitos_ptr(link_tbl_ptr + count, + sg_dma_address(sg) + offset, datalen, 0); + to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0); + count++; + len -= datalen; + offset += datalen; + } + to_talitos_ptr(link_tbl_ptr + count, + sg_dma_address(sg) + offset, sg_next(sg) ? len : len + padding, 0); + to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0); + count++; + cryptlen -= len; + datalen -= len; + offset = 0; + +next: + sg = sg_next(sg); + } + + /* tag end of link table */ + if (count > 0) + to_talitos_ptr_ext_set(link_tbl_ptr + count - 1, + DESC_PTR_LNKTBL_RET, 0); + + return count; +} + +static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src, + unsigned int len, struct talitos_edesc *edesc, + struct talitos_ptr *ptr, int sg_count, + unsigned int offset, int tbl_off, int elen, + bool force, int align) +{ + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); + int aligned_len = ALIGN(len, align); + + if (!src) { + to_talitos_ptr(ptr, 0, 0, is_sec1); + return 1; + } + to_talitos_ptr_ext_set(ptr, elen, is_sec1); + if (sg_count == 1 && !force) { + to_talitos_ptr(ptr, sg_dma_address(src) + offset, aligned_len, is_sec1); + return sg_count; + } + if (is_sec1) { + to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, aligned_len, is_sec1); + return sg_count; + } + sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len, elen, + &edesc->link_tbl[tbl_off], align); + if (sg_count == 1 && !force) { + /* Only one segment now, so no link tbl needed*/ + copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1); + return sg_count; + } + to_talitos_ptr(ptr, edesc->dma_link_tbl + + tbl_off * sizeof(struct talitos_ptr), aligned_len, is_sec1); + to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1); + + return sg_count; +} + +static int talitos_sg_map(struct device *dev, struct scatterlist *src, + unsigned int len, struct talitos_edesc *edesc, + struct talitos_ptr *ptr, int sg_count, + unsigned int offset, int tbl_off) +{ + return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset, + tbl_off, 0, false, 1); +} + +/* + * fill in and submit ipsec_esp descriptor + */ +static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, + bool encrypt, + void (*callback)(struct device *dev, + struct talitos_desc *desc, + void *context, int error)) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(areq); + unsigned int authsize = crypto_aead_authsize(aead); + struct talitos_ctx *ctx = crypto_aead_ctx(aead); + struct device *dev = ctx->dev; + struct talitos_desc *desc = &edesc->desc; + unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); + unsigned int ivsize = crypto_aead_ivsize(aead); + int tbl_off = 0; + int sg_count, ret; + int elen = 0; + bool sync_needed = false; + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); + bool is_ipsec_esp = desc->hdr & DESC_HDR_TYPE_IPSEC_ESP; + struct talitos_ptr *civ_ptr = &desc->ptr[is_ipsec_esp ? 2 : 3]; + struct talitos_ptr *ckey_ptr = &desc->ptr[is_ipsec_esp ? 3 : 2]; + dma_addr_t dma_icv = edesc->dma_link_tbl + edesc->dma_len - authsize; + + /* hmac key */ + to_talitos_ptr(&desc->ptr[0], ctx->dma_key, ctx->authkeylen, is_sec1); + + sg_count = edesc->src_nents ?: 1; + if (is_sec1 && sg_count > 1) + sg_copy_to_buffer(areq->src, sg_count, edesc->buf, + areq->assoclen + cryptlen); + else + sg_count = dma_map_sg(dev, areq->src, sg_count, + (areq->src == areq->dst) ? + DMA_BIDIRECTIONAL : DMA_TO_DEVICE); + + /* hmac data */ + ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc, + &desc->ptr[1], sg_count, 0, tbl_off); + + if (ret > 1) { + tbl_off += ret; + sync_needed = true; + } + + /* cipher iv */ + to_talitos_ptr(civ_ptr, edesc->iv_dma, ivsize, is_sec1); + + /* cipher key */ + to_talitos_ptr(ckey_ptr, ctx->dma_key + ctx->authkeylen, + ctx->enckeylen, is_sec1); + + /* + * cipher in + * map and adjust cipher len to aead request cryptlen. + * extent is bytes of HMAC postpended to ciphertext, + * typically 12 for ipsec + */ + if (is_ipsec_esp && (desc->hdr & DESC_HDR_MODE1_MDEU_CICV)) + elen = authsize; + + ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4], + sg_count, areq->assoclen, tbl_off, elen, + false, 1); + + if (ret > 1) { + tbl_off += ret; + sync_needed = true; + } + + /* cipher out */ + if (areq->src != areq->dst) { + sg_count = edesc->dst_nents ? : 1; + if (!is_sec1 || sg_count == 1) + dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE); + } + + if (is_ipsec_esp && encrypt) + elen = authsize; + else + elen = 0; + ret = talitos_sg_map_ext(dev, areq->dst, cryptlen, edesc, &desc->ptr[5], + sg_count, areq->assoclen, tbl_off, elen, + is_ipsec_esp && !encrypt, 1); + tbl_off += ret; + + if (!encrypt && is_ipsec_esp) { + struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; + + /* Add an entry to the link table for ICV data */ + to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1); + to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RET, is_sec1); + + /* icv data follows link tables */ + to_talitos_ptr(tbl_ptr, dma_icv, authsize, is_sec1); + to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1); + sync_needed = true; + } else if (!encrypt) { + to_talitos_ptr(&desc->ptr[6], dma_icv, authsize, is_sec1); + sync_needed = true; + } else if (!is_ipsec_esp) { + talitos_sg_map(dev, areq->dst, authsize, edesc, &desc->ptr[6], + sg_count, areq->assoclen + cryptlen, tbl_off); + } + + /* iv out */ + if (is_ipsec_esp) + map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, + DMA_FROM_DEVICE); + + if (sync_needed) + dma_sync_single_for_device(dev, edesc->dma_link_tbl, + edesc->dma_len, + DMA_BIDIRECTIONAL); + + ret = talitos_submit(dev, ctx->ch, desc, callback, areq); + if (ret != -EINPROGRESS) { + ipsec_esp_unmap(dev, edesc, areq, encrypt); + kfree(edesc); + } + return ret; +} + +/* + * allocate and map the extended descriptor + */ +static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, + struct scatterlist *src, + struct scatterlist *dst, + u8 *iv, + unsigned int assoclen, + unsigned int cryptlen, + unsigned int authsize, + unsigned int ivsize, + int icv_stashing, + u32 cryptoflags, + bool encrypt) +{ + struct talitos_edesc *edesc; + int src_nents, dst_nents, alloc_len, dma_len, src_len, dst_len; + dma_addr_t iv_dma = 0; + gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : + GFP_ATOMIC; + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); + int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN; + + if (cryptlen + authsize > max_len) { + dev_err(dev, "length exceeds h/w max limit\n"); + return ERR_PTR(-EINVAL); + } + + if (!dst || dst == src) { + src_len = assoclen + cryptlen + authsize; + src_nents = sg_nents_for_len(src, src_len); + if (src_nents < 0) { + dev_err(dev, "Invalid number of src SG.\n"); + return ERR_PTR(-EINVAL); + } + src_nents = (src_nents == 1) ? 0 : src_nents; + dst_nents = dst ? src_nents : 0; + dst_len = 0; + } else { /* dst && dst != src*/ + src_len = assoclen + cryptlen + (encrypt ? 0 : authsize); + src_nents = sg_nents_for_len(src, src_len); + if (src_nents < 0) { + dev_err(dev, "Invalid number of src SG.\n"); + return ERR_PTR(-EINVAL); + } + src_nents = (src_nents == 1) ? 0 : src_nents; + dst_len = assoclen + cryptlen + (encrypt ? authsize : 0); + dst_nents = sg_nents_for_len(dst, dst_len); + if (dst_nents < 0) { + dev_err(dev, "Invalid number of dst SG.\n"); + return ERR_PTR(-EINVAL); + } + dst_nents = (dst_nents == 1) ? 0 : dst_nents; + } + + /* + * allocate space for base edesc plus the link tables, + * allowing for two separate entries for AD and generated ICV (+ 2), + * and space for two sets of ICVs (stashed and generated) + */ + alloc_len = sizeof(struct talitos_edesc); + if (src_nents || dst_nents || !encrypt) { + if (is_sec1) + dma_len = (src_nents ? src_len : 0) + + (dst_nents ? dst_len : 0) + authsize; + else + dma_len = (src_nents + dst_nents + 2) * + sizeof(struct talitos_ptr) + authsize; + alloc_len += dma_len; + } else { + dma_len = 0; + } + alloc_len += icv_stashing ? authsize : 0; + alloc_len += ivsize; + + edesc = kmalloc(ALIGN(alloc_len, dma_get_cache_alignment()), flags); + if (!edesc) + return ERR_PTR(-ENOMEM); + if (ivsize) { + iv = memcpy(((u8 *)edesc) + alloc_len - ivsize, iv, ivsize); + iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); + } + memset(&edesc->desc, 0, sizeof(edesc->desc)); + + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + edesc->iv_dma = iv_dma; + edesc->dma_len = dma_len; + edesc->next_desc = NULL; + if (dma_len) + edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0], + edesc->dma_len, + DMA_BIDIRECTIONAL); + + return edesc; +} + +static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv, + int icv_stashing, bool encrypt) +{ + struct crypto_aead *authenc = crypto_aead_reqtfm(areq); + unsigned int authsize = crypto_aead_authsize(authenc); + struct talitos_ctx *ctx = crypto_aead_ctx(authenc); + unsigned int ivsize = crypto_aead_ivsize(authenc); + unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); + + return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, + iv, areq->assoclen, cryptlen, + authsize, ivsize, icv_stashing, + areq->base.flags, encrypt); +} + +static int aead_encrypt(struct aead_request *req) +{ + struct crypto_aead *authenc = crypto_aead_reqtfm(req); + struct talitos_ctx *ctx = crypto_aead_ctx(authenc); + struct talitos_edesc *edesc; + + /* allocate extended descriptor */ + edesc = aead_edesc_alloc(req, req->iv, 0, true); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + /* set encrypt */ + edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; + + return ipsec_esp(edesc, req, true, ipsec_esp_encrypt_done); +} + +static int aead_decrypt(struct aead_request *req) +{ + struct crypto_aead *authenc = crypto_aead_reqtfm(req); + unsigned int authsize = crypto_aead_authsize(authenc); + struct talitos_ctx *ctx = crypto_aead_ctx(authenc); + struct talitos_private *priv = dev_get_drvdata(ctx->dev); + struct talitos_edesc *edesc; + void *icvdata; + + /* allocate extended descriptor */ + edesc = aead_edesc_alloc(req, req->iv, 1, false); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + if ((edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) && + (priv->features & TALITOS_FTR_HW_AUTH_CHECK) && + ((!edesc->src_nents && !edesc->dst_nents) || + priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) { + + /* decrypt and check the ICV */ + edesc->desc.hdr = ctx->desc_hdr_template | + DESC_HDR_DIR_INBOUND | + DESC_HDR_MODE1_MDEU_CICV; + + /* reset integrity check result bits */ + + return ipsec_esp(edesc, req, false, + ipsec_esp_decrypt_hwauth_done); + } + + /* Have to check the ICV with software */ + edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; + + /* stash incoming ICV for later cmp with ICV generated by the h/w */ + icvdata = edesc->buf + edesc->dma_len; + + sg_pcopy_to_buffer(req->src, edesc->src_nents ? : 1, icvdata, authsize, + req->assoclen + req->cryptlen - authsize); + + return ipsec_esp(edesc, req, false, ipsec_esp_decrypt_swauth_done); +} + +static int skcipher_setkey(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); + struct device *dev = ctx->dev; + + if (ctx->keylen) + dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); + + memcpy(&ctx->key, key, keylen); + ctx->keylen = keylen; + + ctx->dma_key = dma_map_single(dev, ctx->key, keylen, DMA_TO_DEVICE); + + return 0; +} + +static int skcipher_des_setkey(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + return verify_skcipher_des_key(cipher, key) ?: + skcipher_setkey(cipher, key, keylen); +} + +static int skcipher_des3_setkey(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + return verify_skcipher_des3_key(cipher, key) ?: + skcipher_setkey(cipher, key, keylen); +} + +static int skcipher_aes_setkey(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + if (keylen == AES_KEYSIZE_128 || keylen == AES_KEYSIZE_192 || + keylen == AES_KEYSIZE_256) + return skcipher_setkey(cipher, key, keylen); + + return -EINVAL; +} + +static void common_nonsnoop_unmap(struct device *dev, + struct talitos_edesc *edesc, + struct skcipher_request *areq) +{ + unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); + + talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen, 0); + unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE); + + if (edesc->dma_len) + dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, + DMA_BIDIRECTIONAL); +} + +static void skcipher_done(struct device *dev, + struct talitos_desc *desc, void *context, + int err) +{ + struct skcipher_request *areq = context; + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); + struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); + unsigned int ivsize = crypto_skcipher_ivsize(cipher); + struct talitos_edesc *edesc; + + edesc = container_of(desc, struct talitos_edesc, desc); + + common_nonsnoop_unmap(dev, edesc, areq); + memcpy(areq->iv, ctx->iv, ivsize); + + kfree(edesc); + + skcipher_request_complete(areq, err); +} + +static int common_nonsnoop(struct talitos_edesc *edesc, + struct skcipher_request *areq, + void (*callback) (struct device *dev, + struct talitos_desc *desc, + void *context, int error)) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); + struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); + struct device *dev = ctx->dev; + struct talitos_desc *desc = &edesc->desc; + unsigned int cryptlen = areq->cryptlen; + unsigned int ivsize = crypto_skcipher_ivsize(cipher); + int sg_count, ret; + bool sync_needed = false; + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); + bool is_ctr = (desc->hdr & DESC_HDR_SEL0_MASK) == DESC_HDR_SEL0_AESU && + (desc->hdr & DESC_HDR_MODE0_AESU_MASK) == DESC_HDR_MODE0_AESU_CTR; + + /* first DWORD empty */ + + /* cipher iv */ + to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, ivsize, is_sec1); + + /* cipher key */ + to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, is_sec1); + + sg_count = edesc->src_nents ?: 1; + if (is_sec1 && sg_count > 1) + sg_copy_to_buffer(areq->src, sg_count, edesc->buf, + cryptlen); + else + sg_count = dma_map_sg(dev, areq->src, sg_count, + (areq->src == areq->dst) ? + DMA_BIDIRECTIONAL : DMA_TO_DEVICE); + /* + * cipher in + */ + sg_count = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[3], + sg_count, 0, 0, 0, false, is_ctr ? 16 : 1); + if (sg_count > 1) + sync_needed = true; + + /* cipher out */ + if (areq->src != areq->dst) { + sg_count = edesc->dst_nents ? : 1; + if (!is_sec1 || sg_count == 1) + dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE); + } + + ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4], + sg_count, 0, (edesc->src_nents + 1)); + if (ret > 1) + sync_needed = true; + + /* iv out */ + map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv, + DMA_FROM_DEVICE); + + /* last DWORD empty */ + + if (sync_needed) + dma_sync_single_for_device(dev, edesc->dma_link_tbl, + edesc->dma_len, DMA_BIDIRECTIONAL); + + ret = talitos_submit(dev, ctx->ch, desc, callback, areq); + if (ret != -EINPROGRESS) { + common_nonsnoop_unmap(dev, edesc, areq); + kfree(edesc); + } + return ret; +} + +static struct talitos_edesc *skcipher_edesc_alloc(struct skcipher_request * + areq, bool encrypt) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); + struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); + unsigned int ivsize = crypto_skcipher_ivsize(cipher); + + return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, + areq->iv, 0, areq->cryptlen, 0, ivsize, 0, + areq->base.flags, encrypt); +} + +static int skcipher_encrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); + struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); + struct talitos_edesc *edesc; + unsigned int blocksize = + crypto_tfm_alg_blocksize(crypto_skcipher_tfm(cipher)); + + if (!areq->cryptlen) + return 0; + + if (areq->cryptlen % blocksize) + return -EINVAL; + + /* allocate extended descriptor */ + edesc = skcipher_edesc_alloc(areq, true); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + /* set encrypt */ + edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; + + return common_nonsnoop(edesc, areq, skcipher_done); +} + +static int skcipher_decrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); + struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher); + struct talitos_edesc *edesc; + unsigned int blocksize = + crypto_tfm_alg_blocksize(crypto_skcipher_tfm(cipher)); + + if (!areq->cryptlen) + return 0; + + if (areq->cryptlen % blocksize) + return -EINVAL; + + /* allocate extended descriptor */ + edesc = skcipher_edesc_alloc(areq, false); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; + + return common_nonsnoop(edesc, areq, skcipher_done); +} + +static void common_nonsnoop_hash_unmap(struct device *dev, + struct talitos_edesc *edesc, + struct ahash_request *areq) +{ + struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); + struct talitos_desc *desc = &edesc->desc; + + unmap_single_talitos_ptr(dev, &desc->ptr[5], DMA_FROM_DEVICE); + + if (edesc->last && req_ctx->last_request) + memcpy(areq->result, req_ctx->hw_context, + crypto_ahash_digestsize(tfm)); + + if (edesc->src) + talitos_sg_unmap(dev, edesc, edesc->src, NULL, 0, 0); + + /* When using hashctx-in, must unmap it. */ + if (from_talitos_ptr_len(&desc->ptr[1], is_sec1)) + unmap_single_talitos_ptr(dev, &desc->ptr[1], + DMA_TO_DEVICE); + + if (edesc->dma_len) + dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, + DMA_BIDIRECTIONAL); +} + +static void free_edesc_list_from(struct ahash_request *areq, struct talitos_edesc *edesc) +{ + struct talitos_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct talitos_edesc *next; + + while (edesc) { + next = edesc->next_desc; + common_nonsnoop_hash_unmap(ctx->dev, edesc, areq); + kfree(edesc); + edesc = next; + } +} + +static void ahash_done(struct device *dev, + struct talitos_desc *desc, void *context, + int err) +{ + struct ahash_request *areq = context; + struct talitos_edesc *edesc = + container_of(desc, struct talitos_edesc, desc); + struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + bool is_sec1 = has_ftr_sec1(dev_get_drvdata(dev)); + struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); + struct talitos_edesc *next; + + if (is_sec1) { + if (!req_ctx->last_request && req_ctx->to_hash_later) { + /* Position any partial block for next update/final/finup */ + req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1; + req_ctx->nbuf = req_ctx->to_hash_later; + } + + free_edesc_list_from(areq, edesc); + ahash_request_complete(areq, err); + } else { + next = edesc->next_desc; + + common_nonsnoop_hash_unmap(dev, edesc, areq); + kfree(edesc); + + if (err) + goto out; + + if (next) { + err = talitos_submit(dev, ctx->ch, &next->desc, + ahash_done, areq); + if (err != -EINPROGRESS) + goto out; + return; + } +out: + if (!req_ctx->last_request && req_ctx->to_hash_later) { + /* Position any partial block for next update/final/finup */ + req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1; + req_ctx->nbuf = req_ctx->to_hash_later; + } + if (err && next) + free_edesc_list_from(areq, next); + ahash_request_complete(areq, err); + } +} + +/* + * SEC1 doesn't like hashing of 0 sized message, so we do the padding + * ourself and submit a padded block + */ +static void talitos_handle_buggy_hash(struct talitos_ctx *ctx, + struct talitos_edesc *edesc, + struct talitos_ptr *ptr) +{ + static u8 padded_hash[64] = { + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + }; + + pr_err_once("Bug in SEC1, padding ourself\n"); + edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD; + map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash), + (char *)padded_hash, DMA_TO_DEVICE); +} + +static void common_nonsnoop_hash(struct talitos_edesc *edesc, + struct ahash_request *areq, + unsigned int length) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); + struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + struct device *dev = ctx->dev; + struct talitos_desc *desc = &edesc->desc; + bool sync_needed = false; + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); + int sg_count; + + /* first DWORD empty */ + + /* hash context in */ + if (!edesc->first || !req_ctx->first_request || req_ctx->swinit) { + map_single_talitos_ptr_nosync(dev, &desc->ptr[1], + req_ctx->hw_context_size, + req_ctx->hw_context, + DMA_TO_DEVICE); + req_ctx->swinit = 0; + } + /* Indicate next op is not the first. */ + req_ctx->first_request = 0; + + /* HMAC key */ + if (ctx->keylen) + to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, + is_sec1); + + sg_count = edesc->src_nents ?: 1; + if (is_sec1 && sg_count > 1) + sg_copy_to_buffer(edesc->src, sg_count, edesc->buf, length); + else if (length) + sg_count = dma_map_sg(dev, edesc->src, sg_count, DMA_TO_DEVICE); + + /* + * data in + */ + sg_count = talitos_sg_map(dev, edesc->src, length, edesc, &desc->ptr[3], + sg_count, 0, 0); + if (sg_count > 1) + sync_needed = true; + + /* fifth DWORD empty */ + + /* hash/HMAC out -or- hash context out */ + if (edesc->last && req_ctx->last_request) + map_single_talitos_ptr(dev, &desc->ptr[5], + crypto_ahash_digestsize(tfm), + req_ctx->hw_context, DMA_FROM_DEVICE); + else + map_single_talitos_ptr_nosync(dev, &desc->ptr[5], + req_ctx->hw_context_size, + req_ctx->hw_context, + DMA_FROM_DEVICE); + + /* last DWORD empty */ + + if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0) + talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]); + + if (sync_needed) + dma_sync_single_for_device(dev, edesc->dma_link_tbl, + edesc->dma_len, DMA_BIDIRECTIONAL); +} + +static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq, + struct scatterlist *src, + unsigned int nbytes) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); + + return talitos_edesc_alloc(ctx->dev, src, NULL, NULL, 0, + nbytes, 0, 0, 0, areq->base.flags, false); +} + +static struct talitos_edesc * +ahash_process_req_prepare(struct ahash_request *areq, unsigned int nbytes, + unsigned int blocksize, bool is_sec1) +{ + struct talitos_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + struct talitos_edesc *first = NULL, *prev_edesc = NULL, *edesc; + size_t desc_max = is_sec1 ? TALITOS1_MAX_DATA_LEN : + TALITOS2_MAX_DATA_LEN; + struct scatterlist tmp[2]; + size_t to_hash_this_desc; + struct scatterlist *src; + size_t offset = 0; + + do { + src = scatterwalk_ffwd(tmp, req_ctx->psrc, offset); + + to_hash_this_desc = + min(nbytes, ALIGN_DOWN(desc_max, blocksize)); + + /* Allocate extended descriptor */ + edesc = ahash_edesc_alloc(areq, src, to_hash_this_desc); + if (IS_ERR(edesc)) { + if (first) + free_edesc_list_from(areq, first); + return edesc; + } + + edesc->src = + scatterwalk_ffwd(edesc->bufsl, req_ctx->psrc, offset); + edesc->desc.hdr = ctx->desc_hdr_template; + edesc->first = offset == 0; + edesc->last = nbytes - to_hash_this_desc == 0; + + /* On last one, request SEC to pad; otherwise continue */ + if (req_ctx->last_request && edesc->last) + edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD; + else + edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT; + + /* request SEC to INIT hash. */ + if (req_ctx->first_request && edesc->first && !req_ctx->swinit) + edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT; + + /* + * When the tfm context has a keylen, it's an HMAC. + * A first or last (ie. not middle) descriptor must request HMAC. + */ + if (ctx->keylen && ((req_ctx->first_request && edesc->first) || + (req_ctx->last_request && edesc->last))) + edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC; + + /* clear the DN bit */ + if (is_sec1 && !edesc->last) + edesc->desc.hdr &= ~DESC_HDR_DONE_NOTIFY; + + common_nonsnoop_hash(edesc, areq, to_hash_this_desc); + + offset += to_hash_this_desc; + nbytes -= to_hash_this_desc; + + if (!prev_edesc) + first = edesc; + else + prev_edesc->next_desc = edesc; + prev_edesc = edesc; + } while (nbytes); + + return first; +} + +static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); + struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + struct talitos_edesc *edesc; + unsigned int blocksize = + crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + bool is_sec1 = has_ftr_sec1(dev_get_drvdata(ctx->dev)); + unsigned int nbytes_to_hash; + unsigned int to_hash_later; + unsigned int nsg; + int nents; + struct device *dev = ctx->dev; + u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx]; + int ret; + + if (!req_ctx->last_request && (nbytes + req_ctx->nbuf <= blocksize)) { + /* Buffer up to one whole block */ + nents = sg_nents_for_len(areq->src, nbytes); + if (nents < 0) { + dev_err(dev, "Invalid number of src SG.\n"); + return nents; + } + sg_copy_to_buffer(areq->src, nents, + ctx_buf + req_ctx->nbuf, nbytes); + req_ctx->nbuf += nbytes; + return 0; + } + + /* At least (blocksize + 1) bytes are available to hash */ + nbytes_to_hash = nbytes + req_ctx->nbuf; + to_hash_later = nbytes_to_hash & (blocksize - 1); + + if (req_ctx->last_request) + to_hash_later = 0; + else if (to_hash_later) + /* There is a partial block. Hash the full block(s) now */ + nbytes_to_hash -= to_hash_later; + else { + /* Keep one block buffered */ + nbytes_to_hash -= blocksize; + to_hash_later = blocksize; + } + + /* Chain in any previously buffered data */ + if (req_ctx->nbuf) { + nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1; + sg_init_table(req_ctx->bufsl, nsg); + sg_set_buf(req_ctx->bufsl, ctx_buf, req_ctx->nbuf); + if (nsg > 1) + sg_chain(req_ctx->bufsl, 2, areq->src); + req_ctx->psrc = req_ctx->bufsl; + } else + req_ctx->psrc = areq->src; + + if (to_hash_later) { + nents = sg_nents_for_len(areq->src, nbytes); + if (nents < 0) { + dev_err(dev, "Invalid number of src SG.\n"); + return nents; + } + sg_pcopy_to_buffer(areq->src, nents, + req_ctx->buf[(req_ctx->buf_idx + 1) & 1], + to_hash_later, + nbytes - to_hash_later); + } + req_ctx->to_hash_later = to_hash_later; + + edesc = ahash_process_req_prepare(areq, nbytes_to_hash, blocksize, + is_sec1); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + ret = talitos_submit(dev, ctx->ch, &edesc->desc, ahash_done, areq); + if (ret != -EINPROGRESS) + free_edesc_list_from(areq, edesc); + + return ret; +} + +static int ahash_init(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); + struct device *dev = ctx->dev; + struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + unsigned int size; + dma_addr_t dma; + + /* Initialize the context */ + req_ctx->buf_idx = 0; + req_ctx->nbuf = 0; + req_ctx->first_request = 1; + req_ctx->swinit = 0; /* assume h/w init of context */ + size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) + ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 + : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; + req_ctx->hw_context_size = size; + req_ctx->last_request = 0; + + dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, + DMA_TO_DEVICE); + dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE); + + return 0; +} + +/* + * on h/w without explicit sha224 support, we initialize h/w context + * manually with sha224 constants, and tell it to run sha256. + */ +static int ahash_init_sha224_swinit(struct ahash_request *areq) +{ + struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + + req_ctx->hw_context[0] = SHA224_H0; + req_ctx->hw_context[1] = SHA224_H1; + req_ctx->hw_context[2] = SHA224_H2; + req_ctx->hw_context[3] = SHA224_H3; + req_ctx->hw_context[4] = SHA224_H4; + req_ctx->hw_context[5] = SHA224_H5; + req_ctx->hw_context[6] = SHA224_H6; + req_ctx->hw_context[7] = SHA224_H7; + + /* init 64-bit count */ + req_ctx->hw_context[8] = 0; + req_ctx->hw_context[9] = 0; + + ahash_init(areq); + req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/ + + return 0; +} + +static int ahash_update(struct ahash_request *areq) +{ + struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + + req_ctx->last_request = 0; + + return ahash_process_req(areq, areq->nbytes); +} + +static int ahash_final(struct ahash_request *areq) +{ + struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + + req_ctx->last_request = 1; + + return ahash_process_req(areq, 0); +} + +static int ahash_finup(struct ahash_request *areq) +{ + struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + + req_ctx->last_request = 1; + + return ahash_process_req(areq, areq->nbytes); +} + +static int ahash_digest(struct ahash_request *areq) +{ + ahash_init(areq); + return ahash_finup(areq); +} + +static int ahash_digest_sha224_swinit(struct ahash_request *areq) +{ + ahash_init_sha224_swinit(areq); + return ahash_finup(areq); +} + +static int ahash_export(struct ahash_request *areq, void *out) +{ + struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + struct talitos_export_state *export = out; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); + struct device *dev = ctx->dev; + dma_addr_t dma; + + dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, + DMA_FROM_DEVICE); + dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_FROM_DEVICE); + + memcpy(export->hw_context, req_ctx->hw_context, + req_ctx->hw_context_size); + memcpy(export->buf, req_ctx->buf[req_ctx->buf_idx], req_ctx->nbuf); + export->swinit = req_ctx->swinit; + export->first_request = req_ctx->first_request; + export->last_request = req_ctx->last_request; + export->to_hash_later = req_ctx->to_hash_later; + export->nbuf = req_ctx->nbuf; + + return 0; +} + +static int ahash_import(struct ahash_request *areq, const void *in) +{ + struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); + struct device *dev = ctx->dev; + const struct talitos_export_state *export = in; + unsigned int size; + dma_addr_t dma; + + memset(req_ctx, 0, sizeof(*req_ctx)); + size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) + ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 + : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; + req_ctx->hw_context_size = size; + memcpy(req_ctx->hw_context, export->hw_context, size); + memcpy(req_ctx->buf[0], export->buf, export->nbuf); + req_ctx->swinit = export->swinit; + req_ctx->first_request = export->first_request; + req_ctx->last_request = export->last_request; + req_ctx->to_hash_later = export->to_hash_later; + req_ctx->nbuf = export->nbuf; + + dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, + DMA_TO_DEVICE); + dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE); + + return 0; +} + +static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen, + u8 *hash) +{ + struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); + + struct scatterlist sg[1]; + struct ahash_request *req; + struct crypto_wait wait; + int ret; + + crypto_init_wait(&wait); + + req = ahash_request_alloc(tfm, GFP_KERNEL); + if (!req) + return -ENOMEM; + + /* Keep tfm keylen == 0 during hash of the long key */ + ctx->keylen = 0; + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + + sg_init_one(&sg[0], key, keylen); + + ahash_request_set_crypt(req, sg, hash, keylen); + ret = crypto_wait_req(crypto_ahash_digest(req), &wait); + + ahash_request_free(req); + + return ret; +} + +static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); + struct device *dev = ctx->dev; + unsigned int blocksize = + crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + unsigned int digestsize = crypto_ahash_digestsize(tfm); + unsigned int keysize = keylen; + u8 hash[SHA512_DIGEST_SIZE]; + int ret; + + if (keylen <= blocksize) + memcpy(ctx->key, key, keysize); + else { + /* Must get the hash of the long key */ + ret = keyhash(tfm, key, keylen, hash); + + if (ret) + return -EINVAL; + + keysize = digestsize; + memcpy(ctx->key, hash, digestsize); + } + + if (ctx->keylen) + dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); + + ctx->keylen = keysize; + ctx->dma_key = dma_map_single(dev, ctx->key, keysize, DMA_TO_DEVICE); + + return 0; +} + + +struct talitos_alg_template { + u32 type; + u32 priority; + union { + struct skcipher_alg skcipher; + struct ahash_alg hash; + struct aead_alg aead; + } alg; + __be32 desc_hdr_template; +}; + +static struct talitos_alg_template driver_algs[] = { + /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */ + { .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CBC | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA1_HMAC, + }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-aes-talitos-hsna", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CBC | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA1_HMAC, + }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha1)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + .setkey = aead_des3_setkey, + }, + .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_CBC | + DESC_HDR_MODE0_DEU_3DES | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA1_HMAC, + }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha1)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-3des-talitos-hsna", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + .setkey = aead_des3_setkey, + }, + .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_CBC | + DESC_HDR_MODE0_DEU_3DES | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA1_HMAC, + }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CBC | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA224_HMAC, + }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-aes-talitos-hsna", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CBC | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA224_HMAC, + }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha224)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + .setkey = aead_des3_setkey, + }, + .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_CBC | + DESC_HDR_MODE0_DEU_3DES | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA224_HMAC, + }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha224)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-3des-talitos-hsna", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + .setkey = aead_des3_setkey, + }, + .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_CBC | + DESC_HDR_MODE0_DEU_3DES | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA224_HMAC, + }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CBC | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA256_HMAC, + }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-aes-talitos-hsna", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CBC | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA256_HMAC, + }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha256)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + .setkey = aead_des3_setkey, + }, + .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_CBC | + DESC_HDR_MODE0_DEU_3DES | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA256_HMAC, + }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha256)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-3des-talitos-hsna", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + .setkey = aead_des3_setkey, + }, + .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_CBC | + DESC_HDR_MODE0_DEU_3DES | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA256_HMAC, + }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CBC | + DESC_HDR_SEL1_MDEUB | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEUB_SHA384_HMAC, + }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha384)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + .setkey = aead_des3_setkey, + }, + .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_CBC | + DESC_HDR_MODE0_DEU_3DES | + DESC_HDR_SEL1_MDEUB | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEUB_SHA384_HMAC, + }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CBC | + DESC_HDR_SEL1_MDEUB | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEUB_SHA512_HMAC, + }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha512)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + .setkey = aead_des3_setkey, + }, + .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_CBC | + DESC_HDR_MODE0_DEU_3DES | + DESC_HDR_SEL1_MDEUB | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEUB_SHA512_HMAC, + }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(aes))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CBC | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_MD5_HMAC, + }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(aes))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-aes-talitos-hsna", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CBC | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_MD5_HMAC, + }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + .setkey = aead_des3_setkey, + }, + .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_CBC | + DESC_HDR_MODE0_DEU_3DES | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_MD5_HMAC, + }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-3des-talitos-hsna", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + .setkey = aead_des3_setkey, + }, + .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_CBC | + DESC_HDR_MODE0_DEU_3DES | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_MD5_HMAC, + }, + /* SKCIPHER algorithms. */ + { .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-talitos", + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = skcipher_aes_setkey, + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_AESU, + }, + { .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-talitos", + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = skcipher_aes_setkey, + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CBC, + }, + { .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "ctr-aes-talitos", + .base.cra_blocksize = 1, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = skcipher_aes_setkey, + }, + .desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CTR, + }, + { .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "ctr-aes-talitos", + .base.cra_blocksize = 1, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = skcipher_aes_setkey, + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CTR, + }, + { .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .base.cra_name = "ecb(des)", + .base.cra_driver_name = "ecb-des-talitos", + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = skcipher_des_setkey, + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_DEU, + }, + { .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .base.cra_name = "cbc(des)", + .base.cra_driver_name = "cbc-des-talitos", + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = skcipher_des_setkey, + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_CBC, + }, + { .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .base.cra_name = "ecb(des3_ede)", + .base.cra_driver_name = "ecb-3des-talitos", + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = skcipher_des3_setkey, + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_3DES, + }, + { .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .base.cra_name = "cbc(des3_ede)", + .base.cra_driver_name = "cbc-3des-talitos", + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setkey = skcipher_des3_setkey, + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_CBC | + DESC_HDR_MODE0_DEU_3DES, + }, + /* AHASH algorithms. */ + { .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = MD5_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), + .halg.base = { + .cra_name = "md5", + .cra_driver_name = "md5-talitos", + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + } + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_MDEUA | + DESC_HDR_MODE0_MDEU_MD5, + }, + { .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), + .halg.base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-talitos", + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + } + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_MDEUA | + DESC_HDR_MODE0_MDEU_SHA1, + }, + { .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA224_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), + .halg.base = { + .cra_name = "sha224", + .cra_driver_name = "sha224-talitos", + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + } + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_MDEUA | + DESC_HDR_MODE0_MDEU_SHA224, + }, + { .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), + .halg.base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-talitos", + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + } + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_MDEUA | + DESC_HDR_MODE0_MDEU_SHA256, + }, + { .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA384_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), + .halg.base = { + .cra_name = "sha384", + .cra_driver_name = "sha384-talitos", + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + } + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_MDEUB | + DESC_HDR_MODE0_MDEUB_SHA384, + }, + { .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA512_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), + .halg.base = { + .cra_name = "sha512", + .cra_driver_name = "sha512-talitos", + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + } + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_MDEUB | + DESC_HDR_MODE0_MDEUB_SHA512, + }, + { .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = MD5_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), + .halg.base = { + .cra_name = "hmac(md5)", + .cra_driver_name = "hmac-md5-talitos", + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + } + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_MDEUA | + DESC_HDR_MODE0_MDEU_MD5, + }, + { .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), + .halg.base = { + .cra_name = "hmac(sha1)", + .cra_driver_name = "hmac-sha1-talitos", + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + } + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_MDEUA | + DESC_HDR_MODE0_MDEU_SHA1, + }, + { .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA224_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), + .halg.base = { + .cra_name = "hmac(sha224)", + .cra_driver_name = "hmac-sha224-talitos", + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + } + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_MDEUA | + DESC_HDR_MODE0_MDEU_SHA224, + }, + { .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), + .halg.base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "hmac-sha256-talitos", + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + } + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_MDEUA | + DESC_HDR_MODE0_MDEU_SHA256, + }, + { .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA384_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), + .halg.base = { + .cra_name = "hmac(sha384)", + .cra_driver_name = "hmac-sha384-talitos", + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + } + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_MDEUB | + DESC_HDR_MODE0_MDEUB_SHA384, + }, + { .type = CRYPTO_ALG_TYPE_AHASH, + .alg.hash = { + .halg.digestsize = SHA512_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), + .halg.base = { + .cra_name = "hmac(sha512)", + .cra_driver_name = "hmac-sha512-talitos", + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + } + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_MDEUB | + DESC_HDR_MODE0_MDEUB_SHA512, + } +}; + +struct talitos_crypto_alg { + struct list_head entry; + struct device *dev; + struct talitos_alg_template algt; +}; + +static int talitos_init_common(struct talitos_ctx *ctx, + struct talitos_crypto_alg *talitos_alg) +{ + struct talitos_private *priv; + + /* update context with ptr to dev */ + ctx->dev = talitos_alg->dev; + + /* assign SEC channel to tfm in round-robin fashion */ + priv = dev_get_drvdata(ctx->dev); + ctx->ch = atomic_inc_return(&priv->last_chan) & + (priv->num_channels - 1); + + /* copy descriptor header template value */ + ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template; + + /* select done notification */ + ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY; + + return 0; +} + +static int talitos_cra_init_aead(struct crypto_aead *tfm) +{ + struct aead_alg *alg = crypto_aead_alg(tfm); + struct talitos_crypto_alg *talitos_alg; + struct talitos_ctx *ctx = crypto_aead_ctx(tfm); + + talitos_alg = container_of(alg, struct talitos_crypto_alg, + algt.alg.aead); + + return talitos_init_common(ctx, talitos_alg); +} + +static int talitos_cra_init_skcipher(struct crypto_skcipher *tfm) +{ + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct talitos_crypto_alg *talitos_alg; + struct talitos_ctx *ctx = crypto_skcipher_ctx(tfm); + + talitos_alg = container_of(alg, struct talitos_crypto_alg, + algt.alg.skcipher); + + return talitos_init_common(ctx, talitos_alg); +} + +static int talitos_cra_init_ahash(struct crypto_tfm *tfm) +{ + struct crypto_alg *alg = tfm->__crt_alg; + struct talitos_crypto_alg *talitos_alg; + struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); + + talitos_alg = container_of(__crypto_ahash_alg(alg), + struct talitos_crypto_alg, + algt.alg.hash); + + ctx->keylen = 0; + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct talitos_ahash_req_ctx)); + + return talitos_init_common(ctx, talitos_alg); +} + +static void talitos_cra_exit(struct crypto_tfm *tfm) +{ + struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); + struct device *dev = ctx->dev; + + if (ctx->keylen) + dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); +} + +/* + * given the alg's descriptor header template, determine whether descriptor + * type and primary/secondary execution units required match the hw + * capabilities description provided in the device tree node. + */ +static int hw_supports(struct device *dev, __be32 desc_hdr_template) +{ + struct talitos_private *priv = dev_get_drvdata(dev); + int ret; + + ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) && + (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units); + + if (SECONDARY_EU(desc_hdr_template)) + ret = ret && (1 << SECONDARY_EU(desc_hdr_template) + & priv->exec_units); + + return ret; +} + +static void talitos_remove(struct platform_device *ofdev) +{ + struct device *dev = &ofdev->dev; + struct talitos_private *priv = dev_get_drvdata(dev); + struct talitos_crypto_alg *t_alg, *n; + int i; + + list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) { + switch (t_alg->algt.type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + crypto_unregister_skcipher(&t_alg->algt.alg.skcipher); + break; + case CRYPTO_ALG_TYPE_AEAD: + crypto_unregister_aead(&t_alg->algt.alg.aead); + break; + case CRYPTO_ALG_TYPE_AHASH: + crypto_unregister_ahash(&t_alg->algt.alg.hash); + break; + } + list_del(&t_alg->entry); + } + + if (hw_supports(dev, DESC_HDR_SEL0_RNG)) + talitos_unregister_rng(dev); + + for (i = 0; i < 2; i++) + if (priv->irq[i] > 0) + free_irq(priv->irq[i], dev); + + tasklet_kill(&priv->done_task[0]); + if (priv->irq[1] > 0) + tasklet_kill(&priv->done_task[1]); +} + +static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, + struct talitos_alg_template + *template) +{ + struct talitos_private *priv = dev_get_drvdata(dev); + struct talitos_crypto_alg *t_alg; + struct crypto_alg *alg; + + t_alg = devm_kzalloc(dev, sizeof(struct talitos_crypto_alg), + GFP_KERNEL); + if (!t_alg) + return ERR_PTR(-ENOMEM); + + t_alg->algt = *template; + + switch (t_alg->algt.type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + alg = &t_alg->algt.alg.skcipher.base; + alg->cra_exit = talitos_cra_exit; + t_alg->algt.alg.skcipher.init = talitos_cra_init_skcipher; + t_alg->algt.alg.skcipher.setkey = + t_alg->algt.alg.skcipher.setkey ?: skcipher_setkey; + t_alg->algt.alg.skcipher.encrypt = skcipher_encrypt; + t_alg->algt.alg.skcipher.decrypt = skcipher_decrypt; + if (!strcmp(alg->cra_name, "ctr(aes)") && !has_ftr_sec1(priv) && + DESC_TYPE(t_alg->algt.desc_hdr_template) != + DESC_TYPE(DESC_HDR_TYPE_AESU_CTR_NONSNOOP)) { + devm_kfree(dev, t_alg); + return ERR_PTR(-ENOTSUPP); + } + break; + case CRYPTO_ALG_TYPE_AEAD: + alg = &t_alg->algt.alg.aead.base; + alg->cra_exit = talitos_cra_exit; + t_alg->algt.alg.aead.init = talitos_cra_init_aead; + t_alg->algt.alg.aead.setkey = t_alg->algt.alg.aead.setkey ?: + aead_setkey; + t_alg->algt.alg.aead.encrypt = aead_encrypt; + t_alg->algt.alg.aead.decrypt = aead_decrypt; + if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) && + !strncmp(alg->cra_name, "authenc(hmac(sha224)", 20)) { + devm_kfree(dev, t_alg); + return ERR_PTR(-ENOTSUPP); + } + break; + case CRYPTO_ALG_TYPE_AHASH: + alg = &t_alg->algt.alg.hash.halg.base; + alg->cra_init = talitos_cra_init_ahash; + alg->cra_exit = talitos_cra_exit; + t_alg->algt.alg.hash.init = ahash_init; + t_alg->algt.alg.hash.update = ahash_update; + t_alg->algt.alg.hash.final = ahash_final; + t_alg->algt.alg.hash.finup = ahash_finup; + t_alg->algt.alg.hash.digest = ahash_digest; + if (!strncmp(alg->cra_name, "hmac", 4)) + t_alg->algt.alg.hash.setkey = ahash_setkey; + t_alg->algt.alg.hash.import = ahash_import; + t_alg->algt.alg.hash.export = ahash_export; + + if (!(priv->features & TALITOS_FTR_HMAC_OK) && + !strncmp(alg->cra_name, "hmac", 4)) { + devm_kfree(dev, t_alg); + return ERR_PTR(-ENOTSUPP); + } + if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) && + (!strcmp(alg->cra_name, "sha224") || + !strcmp(alg->cra_name, "hmac(sha224)"))) { + t_alg->algt.alg.hash.init = ahash_init_sha224_swinit; + t_alg->algt.alg.hash.digest = + ahash_digest_sha224_swinit; + t_alg->algt.desc_hdr_template = + DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_MDEUA | + DESC_HDR_MODE0_MDEU_SHA256; + } + break; + default: + dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type); + devm_kfree(dev, t_alg); + return ERR_PTR(-EINVAL); + } + + alg->cra_module = THIS_MODULE; + if (t_alg->algt.priority) + alg->cra_priority = t_alg->algt.priority; + else + alg->cra_priority = TALITOS_CRA_PRIORITY; + if (has_ftr_sec1(priv) && t_alg->algt.type != CRYPTO_ALG_TYPE_AHASH) + alg->cra_alignmask = 3; + else + alg->cra_alignmask = 0; + alg->cra_ctxsize = sizeof(struct talitos_ctx); + alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY; + + t_alg->dev = dev; + + return t_alg; +} + +static int talitos_probe_irq(struct platform_device *ofdev) +{ + struct device *dev = &ofdev->dev; + struct talitos_private *priv = dev_get_drvdata(dev); + int err; + bool is_sec1 = has_ftr_sec1(priv); + + priv->irq[0] = platform_get_irq(ofdev, 0); + if (priv->irq[0] < 0) + return priv->irq[0]; + + if (is_sec1) { + err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0, + dev_driver_string(dev), dev); + goto primary_out; + } + + priv->irq[1] = platform_get_irq_optional(ofdev, 1); + if (priv->irq[1] == -EPROBE_DEFER) + return priv->irq[1]; + + /* get the primary irq line */ + if (priv->irq[1] < 0) { + err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0, + dev_driver_string(dev), dev); + goto primary_out; + } + + err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0, + dev_driver_string(dev), dev); + if (err) + goto primary_out; + + /* get the secondary irq line */ + err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0, + dev_driver_string(dev), dev); + if (err) { + dev_err(dev, "failed to request secondary irq\n"); + priv->irq[1] = 0; + } + + return err; + +primary_out: + if (err) { + dev_err(dev, "failed to request primary irq\n"); + priv->irq[0] = 0; + } + + return err; +} + +static int talitos_probe(struct platform_device *ofdev) +{ + struct device *dev = &ofdev->dev; + struct device_node *np = ofdev->dev.of_node; + struct talitos_private *priv; + unsigned int num_channels; + int i, err; + int stride; + + if (of_property_read_u32(np, "fsl,num-channels", &num_channels)) + return -EINVAL; + + priv = devm_kzalloc(dev, struct_size(priv, chan, num_channels), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->num_channels = num_channels; + + INIT_LIST_HEAD(&priv->alg_list); + + dev_set_drvdata(dev, priv); + + priv->ofdev = ofdev; + + spin_lock_init(&priv->reg_lock); + + priv->reg = devm_platform_ioremap_resource(ofdev, 0); + if (IS_ERR(priv->reg)) { + dev_err(dev, "failed to of_iomap\n"); + err = PTR_ERR(priv->reg); + goto err_out; + } + + /* get SEC version capabilities from device tree */ + of_property_read_u32(np, "fsl,channel-fifo-len", &priv->chfifo_len); + of_property_read_u32(np, "fsl,exec-units-mask", &priv->exec_units); + of_property_read_u32(np, "fsl,descriptor-types-mask", + &priv->desc_types); + + if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len || + !priv->exec_units || !priv->desc_types) { + dev_err(dev, "invalid property data in device tree node\n"); + err = -EINVAL; + goto err_out; + } + + if (of_device_is_compatible(np, "fsl,sec3.0")) + priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT; + + if (of_device_is_compatible(np, "fsl,sec2.1")) + priv->features |= TALITOS_FTR_HW_AUTH_CHECK | + TALITOS_FTR_SHA224_HWINIT | + TALITOS_FTR_HMAC_OK; + + if (of_device_is_compatible(np, "fsl,sec1.0")) + priv->features |= TALITOS_FTR_SEC1; + + if (of_device_is_compatible(np, "fsl,sec1.2")) { + priv->reg_deu = priv->reg + TALITOS12_DEU; + priv->reg_aesu = priv->reg + TALITOS12_AESU; + priv->reg_mdeu = priv->reg + TALITOS12_MDEU; + stride = TALITOS1_CH_STRIDE; + } else if (of_device_is_compatible(np, "fsl,sec1.0")) { + priv->reg_deu = priv->reg + TALITOS10_DEU; + priv->reg_aesu = priv->reg + TALITOS10_AESU; + priv->reg_mdeu = priv->reg + TALITOS10_MDEU; + priv->reg_afeu = priv->reg + TALITOS10_AFEU; + priv->reg_rngu = priv->reg + TALITOS10_RNGU; + priv->reg_pkeu = priv->reg + TALITOS10_PKEU; + stride = TALITOS1_CH_STRIDE; + } else { + priv->reg_deu = priv->reg + TALITOS2_DEU; + priv->reg_aesu = priv->reg + TALITOS2_AESU; + priv->reg_mdeu = priv->reg + TALITOS2_MDEU; + priv->reg_afeu = priv->reg + TALITOS2_AFEU; + priv->reg_rngu = priv->reg + TALITOS2_RNGU; + priv->reg_pkeu = priv->reg + TALITOS2_PKEU; + priv->reg_keu = priv->reg + TALITOS2_KEU; + priv->reg_crcu = priv->reg + TALITOS2_CRCU; + stride = TALITOS2_CH_STRIDE; + } + + err = talitos_probe_irq(ofdev); + if (err) + goto err_out; + + if (has_ftr_sec1(priv)) { + if (priv->num_channels == 1) + tasklet_init(&priv->done_task[0], talitos1_done_ch0, + (unsigned long)dev); + else + tasklet_init(&priv->done_task[0], talitos1_done_4ch, + (unsigned long)dev); + } else { + if (priv->irq[1] > 0) { + tasklet_init(&priv->done_task[0], talitos2_done_ch0_2, + (unsigned long)dev); + tasklet_init(&priv->done_task[1], talitos2_done_ch1_3, + (unsigned long)dev); + } else if (priv->num_channels == 1) { + tasklet_init(&priv->done_task[0], talitos2_done_ch0, + (unsigned long)dev); + } else { + tasklet_init(&priv->done_task[0], talitos2_done_4ch, + (unsigned long)dev); + } + } + + priv->fifo_len = roundup_pow_of_two(priv->chfifo_len); + + for (i = 0; i < priv->num_channels; i++) { + priv->chan[i].reg = priv->reg + stride * (i + 1); + if (priv->irq[1] < 0 || !(i & 1)) + priv->chan[i].reg += TALITOS_CH_BASE_OFFSET; + + spin_lock_init(&priv->chan[i].head_lock); + spin_lock_init(&priv->chan[i].tail_lock); + + priv->chan[i].fifo = devm_kcalloc(dev, + priv->fifo_len, + sizeof(struct talitos_request), + GFP_KERNEL); + if (!priv->chan[i].fifo) { + dev_err(dev, "failed to allocate request fifo %d\n", i); + err = -ENOMEM; + goto err_out; + } + + atomic_set(&priv->chan[i].submit_count, + -(priv->chfifo_len - 1)); + } + + dma_set_mask(dev, DMA_BIT_MASK(36)); + + /* reset and initialize the h/w */ + err = init_device(dev); + if (err) { + dev_err(dev, "failed to initialize device\n"); + goto err_out; + } + + /* register the RNG, if available */ + if (hw_supports(dev, DESC_HDR_SEL0_RNG)) { + err = talitos_register_rng(dev); + if (err) { + dev_err(dev, "failed to register hwrng: %d\n", err); + goto err_out; + } else + dev_info(dev, "hwrng\n"); + } + + /* register crypto algorithms the device supports */ + for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { + if (hw_supports(dev, driver_algs[i].desc_hdr_template)) { + struct talitos_crypto_alg *t_alg; + struct crypto_alg *alg = NULL; + + t_alg = talitos_alg_alloc(dev, &driver_algs[i]); + if (IS_ERR(t_alg)) { + err = PTR_ERR(t_alg); + if (err == -ENOTSUPP) + continue; + goto err_out; + } + + switch (t_alg->algt.type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + err = crypto_register_skcipher( + &t_alg->algt.alg.skcipher); + alg = &t_alg->algt.alg.skcipher.base; + break; + + case CRYPTO_ALG_TYPE_AEAD: + err = crypto_register_aead( + &t_alg->algt.alg.aead); + alg = &t_alg->algt.alg.aead.base; + break; + + case CRYPTO_ALG_TYPE_AHASH: + err = crypto_register_ahash( + &t_alg->algt.alg.hash); + alg = &t_alg->algt.alg.hash.halg.base; + break; + } + if (err) { + dev_err(dev, "%s alg registration failed\n", + alg->cra_driver_name); + devm_kfree(dev, t_alg); + } else + list_add_tail(&t_alg->entry, &priv->alg_list); + } + } + if (!list_empty(&priv->alg_list)) + dev_info(dev, "%s algorithms registered in /proc/crypto\n", + (char *)of_get_property(np, "compatible", NULL)); + + return 0; + +err_out: + talitos_remove(ofdev); + + return err; +} + +static const struct of_device_id talitos_match[] = { +#ifdef CONFIG_CRYPTO_DEV_TALITOS1 + { + .compatible = "fsl,sec1.0", + }, +#endif +#ifdef CONFIG_CRYPTO_DEV_TALITOS2 + { + .compatible = "fsl,sec2.0", + }, +#endif + {}, +}; +MODULE_DEVICE_TABLE(of, talitos_match); + +static struct platform_driver talitos_driver = { + .driver = { + .name = "talitos", + .of_match_table = talitos_match, + }, + .probe = talitos_probe, + .remove = talitos_remove, +}; + +module_platform_driver(talitos_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Kim Phillips "); +MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver"); diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h new file mode 100644 index 000000000..d4ff8d589 --- /dev/null +++ b/drivers/crypto/talitos.h @@ -0,0 +1,427 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Freescale SEC (talitos) device register and descriptor header defines + * + * Copyright (c) 2006-2011 Freescale Semiconductor, Inc. + */ + +#define TALITOS_TIMEOUT 100000 +#define TALITOS1_MAX_DATA_LEN 32768 +#define TALITOS2_MAX_DATA_LEN 65535 + +#define DESC_TYPE(desc_hdr) ((be32_to_cpu(desc_hdr) >> 3) & 0x1f) +#define PRIMARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 28) & 0xf) +#define SECONDARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 16) & 0xf) + +/* descriptor pointer entry */ +struct talitos_ptr { + union { + struct { /* SEC2 format */ + __be16 len; /* length */ + u8 j_extent; /* jump to sg link table and/or extent*/ + u8 eptr; /* extended address */ + }; + struct { /* SEC1 format */ + __be16 res; + __be16 len1; /* length */ + }; + }; + __be32 ptr; /* address */ +}; + +/* descriptor */ +struct talitos_desc { + __be32 hdr; /* header high bits */ + union { + __be32 hdr_lo; /* header low bits */ + __be32 hdr1; /* header for SEC1 */ + }; + struct talitos_ptr ptr[7]; /* ptr/len pair array */ + __be32 next_desc; /* next descriptor (SEC1) */ +}; + +#define TALITOS_DESC_SIZE (sizeof(struct talitos_desc) - sizeof(__be32)) + +/* + * talitos_edesc - s/w-extended descriptor + * @bufsl: scatterlist buffer + * @src: pointer to input scatterlist + * @first: first descriptor of a chain + * @last: last descriptor of a chain + * + * @src_nents: number of segments in input scatterlist + * @dst_nents: number of segments in output scatterlist + * @iv_dma: dma address of iv for checking continuity and link table + * @dma_len: length of dma mapped link_tbl space + * @dma_link_tbl: bus physical address of link_tbl/buf + * @next_desc: next descriptor + * @desc: h/w descriptor + * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2) + * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1) + * + * if decrypting (with authcheck), or either one of src_nents or dst_nents + * is greater than 1, an integrity check value is concatenated to the end + * of link_tbl data + */ +struct talitos_edesc { + struct scatterlist bufsl[2]; + struct scatterlist *src; + int first; + int last; + + int src_nents; + int dst_nents; + dma_addr_t iv_dma; + int dma_len; + dma_addr_t dma_link_tbl; + struct talitos_edesc *next_desc; + struct talitos_desc desc; + union { + DECLARE_FLEX_ARRAY(struct talitos_ptr, link_tbl); + DECLARE_FLEX_ARRAY(u8, buf); + }; +}; + +/** + * talitos_request - descriptor submission request + * @desc: descriptor pointer (kernel virtual) + * @dma_desc: descriptor's physical bus address + * @callback: whom to call when descriptor processing is done + * @context: caller context (optional) + */ +struct talitos_request { + struct talitos_desc *desc; + dma_addr_t dma_desc; + void (*callback) (struct device *dev, struct talitos_desc *desc, + void *context, int error); + void *context; +}; + +/* per-channel fifo management */ +struct talitos_channel { + void __iomem *reg; + + /* request fifo */ + struct talitos_request *fifo; + + /* number of requests pending in channel h/w fifo */ + atomic_t submit_count ____cacheline_aligned; + + /* request submission (head) lock */ + spinlock_t head_lock ____cacheline_aligned; + /* index to next free descriptor request */ + int head; + + /* request release (tail) lock */ + spinlock_t tail_lock ____cacheline_aligned; + /* index to next in-progress/done descriptor request */ + int tail; +}; + +struct talitos_private { + struct device *dev; + struct platform_device *ofdev; + void __iomem *reg; + void __iomem *reg_deu; + void __iomem *reg_aesu; + void __iomem *reg_mdeu; + void __iomem *reg_afeu; + void __iomem *reg_rngu; + void __iomem *reg_pkeu; + void __iomem *reg_keu; + void __iomem *reg_crcu; + int irq[2]; + + /* SEC global registers lock */ + spinlock_t reg_lock ____cacheline_aligned; + + /* SEC version geometry (from device tree node) */ + unsigned int num_channels; + unsigned int chfifo_len; + unsigned int exec_units; + unsigned int desc_types; + + /* SEC Compatibility info */ + unsigned long features; + + /* + * length of the request fifo + * fifo_len is chfifo_len rounded up to next power of 2 + * so we can use bitwise ops to wrap + */ + unsigned int fifo_len; + + /* next channel to be assigned next incoming descriptor */ + atomic_t last_chan ____cacheline_aligned; + + /* request callback tasklet */ + struct tasklet_struct done_task[2]; + + /* list of registered algorithms */ + struct list_head alg_list; + + /* hwrng device */ + struct hwrng rng; + bool rng_registered; + + struct talitos_channel chan[] __counted_by(num_channels); + +}; + +/* .features flag */ +#define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001 +#define TALITOS_FTR_HW_AUTH_CHECK 0x00000002 +#define TALITOS_FTR_SHA224_HWINIT 0x00000004 +#define TALITOS_FTR_HMAC_OK 0x00000008 +#define TALITOS_FTR_SEC1 0x00000010 + +/* + * If both CONFIG_CRYPTO_DEV_TALITOS1 and CONFIG_CRYPTO_DEV_TALITOS2 are + * defined, we check the features which are set according to the device tree. + * Otherwise, we answer true or false directly + */ +static inline bool has_ftr_sec1(struct talitos_private *priv) +{ + if (IS_ENABLED(CONFIG_CRYPTO_DEV_TALITOS1) && + IS_ENABLED(CONFIG_CRYPTO_DEV_TALITOS2)) + return priv->features & TALITOS_FTR_SEC1; + + return IS_ENABLED(CONFIG_CRYPTO_DEV_TALITOS1); +} + +/* + * TALITOS_xxx_LO addresses point to the low data bits (32-63) of the register + */ + +#define ISR1_FORMAT(x) (((x) << 28) | ((x) << 16)) +#define ISR2_FORMAT(x) (((x) << 4) | (x)) + +/* global register offset addresses */ +#define TALITOS_MCR 0x1030 /* master control register */ +#define TALITOS_MCR_RCA0 (1 << 15) /* remap channel 0 */ +#define TALITOS_MCR_RCA1 (1 << 14) /* remap channel 1 */ +#define TALITOS_MCR_RCA2 (1 << 13) /* remap channel 2 */ +#define TALITOS_MCR_RCA3 (1 << 12) /* remap channel 3 */ +#define TALITOS1_MCR_SWR 0x1000000 /* s/w reset */ +#define TALITOS2_MCR_SWR 0x1 /* s/w reset */ +#define TALITOS_MCR_LO 0x1034 +#define TALITOS_IMR 0x1008 /* interrupt mask register */ +/* enable channel IRQs */ +#define TALITOS1_IMR_INIT ISR1_FORMAT(0xf) +#define TALITOS1_IMR_DONE ISR1_FORMAT(0x5) /* done IRQs */ +/* enable channel IRQs */ +#define TALITOS2_IMR_INIT (ISR2_FORMAT(0xf) | 0x10000) +#define TALITOS2_IMR_DONE ISR1_FORMAT(0x5) /* done IRQs */ +#define TALITOS_IMR_LO 0x100C +#define TALITOS1_IMR_LO_INIT 0x2000000 /* allow RNGU error IRQs */ +#define TALITOS2_IMR_LO_INIT 0x20000 /* allow RNGU error IRQs */ +#define TALITOS_ISR 0x1010 /* interrupt status register */ +#define TALITOS1_ISR_4CHERR ISR1_FORMAT(0xa) /* 4 ch errors mask */ +#define TALITOS1_ISR_4CHDONE ISR1_FORMAT(0x5) /* 4 ch done mask */ +#define TALITOS1_ISR_CH_0_ERR (2 << 28) /* ch 0 errors mask */ +#define TALITOS1_ISR_CH_0_DONE (1 << 28) /* ch 0 done mask */ +#define TALITOS1_ISR_TEA_ERR 0x00000040 +#define TALITOS2_ISR_4CHERR ISR2_FORMAT(0xa) /* 4 ch errors mask */ +#define TALITOS2_ISR_4CHDONE ISR2_FORMAT(0x5) /* 4 ch done mask */ +#define TALITOS2_ISR_CH_0_ERR 2 /* ch 0 errors mask */ +#define TALITOS2_ISR_CH_0_DONE 1 /* ch 0 done mask */ +#define TALITOS2_ISR_CH_0_2_ERR ISR2_FORMAT(0x2) /* ch 0, 2 err mask */ +#define TALITOS2_ISR_CH_0_2_DONE ISR2_FORMAT(0x1) /* ch 0, 2 done mask */ +#define TALITOS2_ISR_CH_1_3_ERR ISR2_FORMAT(0x8) /* ch 1, 3 err mask */ +#define TALITOS2_ISR_CH_1_3_DONE ISR2_FORMAT(0x4) /* ch 1, 3 done mask */ +#define TALITOS_ISR_LO 0x1014 +#define TALITOS_ICR 0x1018 /* interrupt clear register */ +#define TALITOS_ICR_LO 0x101C + +/* channel register address stride */ +#define TALITOS_CH_BASE_OFFSET 0x1000 /* default channel map base */ +#define TALITOS1_CH_STRIDE 0x1000 +#define TALITOS2_CH_STRIDE 0x100 + +/* channel configuration register */ +#define TALITOS_CCCR 0x8 +#define TALITOS2_CCCR_CONT 0x2 /* channel continue on SEC2 */ +#define TALITOS2_CCCR_RESET 0x1 /* channel reset on SEC2 */ +#define TALITOS_CCCR_LO 0xc +#define TALITOS_CCCR_LO_IWSE 0x80 /* chan. ICCR writeback enab. */ +#define TALITOS_CCCR_LO_EAE 0x20 /* extended address enable */ +#define TALITOS_CCCR_LO_CDWE 0x10 /* chan. done writeback enab. */ +#define TALITOS_CCCR_LO_NE 0x8 /* fetch next descriptor enab. */ +#define TALITOS_CCCR_LO_NT 0x4 /* notification type */ +#define TALITOS_CCCR_LO_CDIE 0x2 /* channel done IRQ enable */ +#define TALITOS1_CCCR_LO_RESET 0x1 /* channel reset on SEC1 */ + +/* CCPSR: channel pointer status register */ +#define TALITOS_CCPSR 0x10 +#define TALITOS_CCPSR_LO 0x14 +#define TALITOS_CCPSR_LO_DOF 0x8000 /* double FF write oflow error */ +#define TALITOS_CCPSR_LO_SOF 0x4000 /* single FF write oflow error */ +#define TALITOS_CCPSR_LO_MDTE 0x2000 /* master data transfer error */ +#define TALITOS_CCPSR_LO_SGDLZ 0x1000 /* s/g data len zero error */ +#define TALITOS_CCPSR_LO_FPZ 0x0800 /* fetch ptr zero error */ +#define TALITOS_CCPSR_LO_IDH 0x0400 /* illegal desc hdr error */ +#define TALITOS_CCPSR_LO_IEU 0x0200 /* invalid EU error */ +#define TALITOS_CCPSR_LO_EU 0x0100 /* EU error detected */ +#define TALITOS_CCPSR_LO_GB 0x0080 /* gather boundary error */ +#define TALITOS_CCPSR_LO_GRL 0x0040 /* gather return/length error */ +#define TALITOS_CCPSR_LO_SB 0x0020 /* scatter boundary error */ +#define TALITOS_CCPSR_LO_SRL 0x0010 /* scatter return/length error */ + +/* channel fetch fifo register */ +#define TALITOS_FF 0x48 +#define TALITOS_FF_LO 0x4c + +/* current descriptor pointer register */ +#define TALITOS_CDPR 0x40 +#define TALITOS_CDPR_LO 0x44 + +/* descriptor buffer register */ +#define TALITOS_DESCBUF 0x80 +#define TALITOS_DESCBUF_LO 0x84 + +/* gather link table */ +#define TALITOS_GATHER 0xc0 +#define TALITOS_GATHER_LO 0xc4 + +/* scatter link table */ +#define TALITOS_SCATTER 0xe0 +#define TALITOS_SCATTER_LO 0xe4 + +/* execution unit registers base */ +#define TALITOS2_DEU 0x2000 +#define TALITOS2_AESU 0x4000 +#define TALITOS2_MDEU 0x6000 +#define TALITOS2_AFEU 0x8000 +#define TALITOS2_RNGU 0xa000 +#define TALITOS2_PKEU 0xc000 +#define TALITOS2_KEU 0xe000 +#define TALITOS2_CRCU 0xf000 + +#define TALITOS12_AESU 0x4000 +#define TALITOS12_DEU 0x5000 +#define TALITOS12_MDEU 0x6000 + +#define TALITOS10_AFEU 0x8000 +#define TALITOS10_DEU 0xa000 +#define TALITOS10_MDEU 0xc000 +#define TALITOS10_RNGU 0xe000 +#define TALITOS10_PKEU 0x10000 +#define TALITOS10_AESU 0x12000 + +/* execution unit interrupt status registers */ +#define TALITOS_EUDSR 0x10 /* data size */ +#define TALITOS_EUDSR_LO 0x14 +#define TALITOS_EURCR 0x18 /* reset control*/ +#define TALITOS_EURCR_LO 0x1c +#define TALITOS_EUSR 0x28 /* rng status */ +#define TALITOS_EUSR_LO 0x2c +#define TALITOS_EUISR 0x30 +#define TALITOS_EUISR_LO 0x34 +#define TALITOS_EUICR 0x38 /* int. control */ +#define TALITOS_EUICR_LO 0x3c +#define TALITOS_EU_FIFO 0x800 /* output FIFO */ +#define TALITOS_EU_FIFO_LO 0x804 /* output FIFO */ +/* DES unit */ +#define TALITOS1_DEUICR_KPE 0x00200000 /* Key Parity Error */ +/* message digest unit */ +#define TALITOS_MDEUICR_LO_ICE 0x4000 /* integrity check IRQ enable */ +/* random number unit */ +#define TALITOS_RNGUSR_LO_RD 0x1 /* reset done */ +#define TALITOS_RNGUSR_LO_OFL 0xff0000/* output FIFO length */ +#define TALITOS_RNGURCR_LO_SR 0x1 /* software reset */ + +#define TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 0x28 +#define TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512 0x48 + +/* + * talitos descriptor header (hdr) bits + */ + +/* written back when done */ +#define DESC_HDR_DONE cpu_to_be32(0xff000000) +#define DESC_HDR_LO_ICCR1_MASK cpu_to_be32(0x00180000) +#define DESC_HDR_LO_ICCR1_PASS cpu_to_be32(0x00080000) +#define DESC_HDR_LO_ICCR1_FAIL cpu_to_be32(0x00100000) + +/* primary execution unit select */ +#define DESC_HDR_SEL0_MASK cpu_to_be32(0xf0000000) +#define DESC_HDR_SEL0_AFEU cpu_to_be32(0x10000000) +#define DESC_HDR_SEL0_DEU cpu_to_be32(0x20000000) +#define DESC_HDR_SEL0_MDEUA cpu_to_be32(0x30000000) +#define DESC_HDR_SEL0_MDEUB cpu_to_be32(0xb0000000) +#define DESC_HDR_SEL0_RNG cpu_to_be32(0x40000000) +#define DESC_HDR_SEL0_PKEU cpu_to_be32(0x50000000) +#define DESC_HDR_SEL0_AESU cpu_to_be32(0x60000000) +#define DESC_HDR_SEL0_KEU cpu_to_be32(0x70000000) +#define DESC_HDR_SEL0_CRCU cpu_to_be32(0x80000000) + +/* primary execution unit mode (MODE0) and derivatives */ +#define DESC_HDR_MODE0_ENCRYPT cpu_to_be32(0x00100000) +#define DESC_HDR_MODE0_AESU_MASK cpu_to_be32(0x00600000) +#define DESC_HDR_MODE0_AESU_CBC cpu_to_be32(0x00200000) +#define DESC_HDR_MODE0_AESU_CTR cpu_to_be32(0x00600000) +#define DESC_HDR_MODE0_DEU_CBC cpu_to_be32(0x00400000) +#define DESC_HDR_MODE0_DEU_3DES cpu_to_be32(0x00200000) +#define DESC_HDR_MODE0_MDEU_CONT cpu_to_be32(0x08000000) +#define DESC_HDR_MODE0_MDEU_INIT cpu_to_be32(0x01000000) +#define DESC_HDR_MODE0_MDEU_HMAC cpu_to_be32(0x00800000) +#define DESC_HDR_MODE0_MDEU_PAD cpu_to_be32(0x00400000) +#define DESC_HDR_MODE0_MDEU_SHA224 cpu_to_be32(0x00300000) +#define DESC_HDR_MODE0_MDEU_MD5 cpu_to_be32(0x00200000) +#define DESC_HDR_MODE0_MDEU_SHA256 cpu_to_be32(0x00100000) +#define DESC_HDR_MODE0_MDEU_SHA1 cpu_to_be32(0x00000000) +#define DESC_HDR_MODE0_MDEUB_SHA384 cpu_to_be32(0x00000000) +#define DESC_HDR_MODE0_MDEUB_SHA512 cpu_to_be32(0x00200000) +#define DESC_HDR_MODE0_MDEU_MD5_HMAC (DESC_HDR_MODE0_MDEU_MD5 | \ + DESC_HDR_MODE0_MDEU_HMAC) +#define DESC_HDR_MODE0_MDEU_SHA256_HMAC (DESC_HDR_MODE0_MDEU_SHA256 | \ + DESC_HDR_MODE0_MDEU_HMAC) +#define DESC_HDR_MODE0_MDEU_SHA1_HMAC (DESC_HDR_MODE0_MDEU_SHA1 | \ + DESC_HDR_MODE0_MDEU_HMAC) + +/* secondary execution unit select (SEL1) */ +#define DESC_HDR_SEL1_MASK cpu_to_be32(0x000f0000) +#define DESC_HDR_SEL1_MDEUA cpu_to_be32(0x00030000) +#define DESC_HDR_SEL1_MDEUB cpu_to_be32(0x000b0000) +#define DESC_HDR_SEL1_CRCU cpu_to_be32(0x00080000) + +/* secondary execution unit mode (MODE1) and derivatives */ +#define DESC_HDR_MODE1_MDEU_CICV cpu_to_be32(0x00004000) +#define DESC_HDR_MODE1_MDEU_INIT cpu_to_be32(0x00001000) +#define DESC_HDR_MODE1_MDEU_HMAC cpu_to_be32(0x00000800) +#define DESC_HDR_MODE1_MDEU_PAD cpu_to_be32(0x00000400) +#define DESC_HDR_MODE1_MDEU_SHA224 cpu_to_be32(0x00000300) +#define DESC_HDR_MODE1_MDEU_MD5 cpu_to_be32(0x00000200) +#define DESC_HDR_MODE1_MDEU_SHA256 cpu_to_be32(0x00000100) +#define DESC_HDR_MODE1_MDEU_SHA1 cpu_to_be32(0x00000000) +#define DESC_HDR_MODE1_MDEUB_SHA384 cpu_to_be32(0x00000000) +#define DESC_HDR_MODE1_MDEUB_SHA512 cpu_to_be32(0x00000200) +#define DESC_HDR_MODE1_MDEU_MD5_HMAC (DESC_HDR_MODE1_MDEU_MD5 | \ + DESC_HDR_MODE1_MDEU_HMAC) +#define DESC_HDR_MODE1_MDEU_SHA256_HMAC (DESC_HDR_MODE1_MDEU_SHA256 | \ + DESC_HDR_MODE1_MDEU_HMAC) +#define DESC_HDR_MODE1_MDEU_SHA1_HMAC (DESC_HDR_MODE1_MDEU_SHA1 | \ + DESC_HDR_MODE1_MDEU_HMAC) +#define DESC_HDR_MODE1_MDEU_SHA224_HMAC (DESC_HDR_MODE1_MDEU_SHA224 | \ + DESC_HDR_MODE1_MDEU_HMAC) +#define DESC_HDR_MODE1_MDEUB_SHA384_HMAC (DESC_HDR_MODE1_MDEUB_SHA384 | \ + DESC_HDR_MODE1_MDEU_HMAC) +#define DESC_HDR_MODE1_MDEUB_SHA512_HMAC (DESC_HDR_MODE1_MDEUB_SHA512 | \ + DESC_HDR_MODE1_MDEU_HMAC) + +/* direction of overall data flow (DIR) */ +#define DESC_HDR_DIR_INBOUND cpu_to_be32(0x00000002) + +/* request done notification (DN) */ +#define DESC_HDR_DONE_NOTIFY cpu_to_be32(0x00000001) + +/* descriptor types */ +#define DESC_HDR_TYPE_AESU_CTR_NONSNOOP cpu_to_be32(0 << 3) +#define DESC_HDR_TYPE_IPSEC_ESP cpu_to_be32(1 << 3) +#define DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU cpu_to_be32(2 << 3) +#define DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU cpu_to_be32(4 << 3) + +/* link table extent field bits */ +#define DESC_PTR_LNKTBL_JUMP 0x80 +#define DESC_PTR_LNKTBL_RET 0x02 +#define DESC_PTR_LNKTBL_NEXT 0x01 diff --git a/drivers/crypto/tegra/Makefile b/drivers/crypto/tegra/Makefile new file mode 100644 index 000000000..a32001e58 --- /dev/null +++ b/drivers/crypto/tegra/Makefile @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +tegra-se-objs := tegra-se-key.o tegra-se-main.o + +tegra-se-y += tegra-se-aes.o +tegra-se-y += tegra-se-hash.o + +obj-$(CONFIG_CRYPTO_DEV_TEGRA) += tegra-se.o diff --git a/drivers/crypto/tegra/tegra-se-aes.c b/drivers/crypto/tegra/tegra-se-aes.c new file mode 100644 index 000000000..0fd1d7035 --- /dev/null +++ b/drivers/crypto/tegra/tegra-se-aes.c @@ -0,0 +1,2061 @@ +// SPDX-License-Identifier: GPL-2.0-only +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +/* + * Crypto driver to handle block cipher algorithms using NVIDIA Security Engine. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tegra-se.h" + +struct tegra_aes_ctx { + struct tegra_se *se; + u32 alg; + u32 ivsize; + u32 key1_id; + u32 key2_id; + u32 keylen; + u8 key1[AES_MAX_KEY_SIZE]; + u8 key2[AES_MAX_KEY_SIZE]; +}; + +struct tegra_aes_reqctx { + struct tegra_se_datbuf datbuf; + bool encrypt; + u32 config; + u32 crypto_config; + u32 len; + u32 *iv; +}; + +struct tegra_aead_ctx { + struct tegra_se *se; + u32 alg; + u32 key_id; + u32 keylen; + u8 key[AES_MAX_KEY_SIZE]; +}; + +struct tegra_aead_reqctx { + struct tegra_se_datbuf inbuf; + struct tegra_se_datbuf outbuf; + struct scatterlist *src_sg; + struct scatterlist *dst_sg; + unsigned int assoclen; + unsigned int cryptlen; + unsigned int authsize; + bool encrypt; + u32 crypto_config; + u32 config; + u32 key_id; + u32 iv[4]; + u8 authdata[16]; +}; + +struct tegra_cmac_ctx { + struct tegra_se *se; + unsigned int alg; + u32 key_id; + u32 keylen; + u8 key[AES_MAX_KEY_SIZE]; + struct crypto_shash *fallback_tfm; +}; + +struct tegra_cmac_reqctx { + struct scatterlist *src_sg; + struct tegra_se_datbuf datbuf; + struct tegra_se_datbuf residue; + unsigned int total_len; + unsigned int blk_size; + unsigned int task; + u32 crypto_config; + u32 config; + u32 key_id; + u32 *iv; + u32 result[CMAC_RESULT_REG_COUNT]; +}; + +/* increment counter (128-bit int) */ +static void ctr_iv_inc(__u8 *counter, __u8 bits, __u32 nums) +{ + do { + --bits; + nums += counter[bits]; + counter[bits] = nums & 0xff; + nums >>= 8; + } while (bits && nums); +} + +static void tegra_cbc_iv_copyback(struct skcipher_request *req, struct tegra_aes_ctx *ctx) +{ + struct tegra_aes_reqctx *rctx = skcipher_request_ctx(req); + unsigned int offset; + + offset = req->cryptlen - ctx->ivsize; + + if (rctx->encrypt) + memcpy(req->iv, rctx->datbuf.buf + offset, ctx->ivsize); + else + scatterwalk_map_and_copy(req->iv, req->src, offset, ctx->ivsize, 0); +} + +static void tegra_aes_update_iv(struct skcipher_request *req, struct tegra_aes_ctx *ctx) +{ + int num; + + if (ctx->alg == SE_ALG_CBC) { + tegra_cbc_iv_copyback(req, ctx); + } else if (ctx->alg == SE_ALG_CTR) { + num = req->cryptlen / ctx->ivsize; + if (req->cryptlen % ctx->ivsize) + num++; + + ctr_iv_inc(req->iv, ctx->ivsize, num); + } +} + +static int tegra234_aes_crypto_cfg(u32 alg, bool encrypt) +{ + switch (alg) { + case SE_ALG_CMAC: + case SE_ALG_GMAC: + case SE_ALG_GCM: + case SE_ALG_GCM_FINAL: + return 0; + case SE_ALG_CBC: + if (encrypt) + return SE_CRYPTO_CFG_CBC_ENCRYPT; + else + return SE_CRYPTO_CFG_CBC_DECRYPT; + case SE_ALG_ECB: + if (encrypt) + return SE_CRYPTO_CFG_ECB_ENCRYPT; + else + return SE_CRYPTO_CFG_ECB_DECRYPT; + case SE_ALG_XTS: + if (encrypt) + return SE_CRYPTO_CFG_XTS_ENCRYPT; + else + return SE_CRYPTO_CFG_XTS_DECRYPT; + + case SE_ALG_CTR: + return SE_CRYPTO_CFG_CTR; + case SE_ALG_CBC_MAC: + return SE_CRYPTO_CFG_CBC_MAC; + + default: + break; + } + + return -EINVAL; +} + +static int tegra234_aes_cfg(u32 alg, bool encrypt) +{ + switch (alg) { + case SE_ALG_CBC: + case SE_ALG_ECB: + case SE_ALG_XTS: + case SE_ALG_CTR: + if (encrypt) + return SE_CFG_AES_ENCRYPT; + else + return SE_CFG_AES_DECRYPT; + + case SE_ALG_GMAC: + if (encrypt) + return SE_CFG_GMAC_ENCRYPT; + else + return SE_CFG_GMAC_DECRYPT; + + case SE_ALG_GCM: + if (encrypt) + return SE_CFG_GCM_ENCRYPT; + else + return SE_CFG_GCM_DECRYPT; + + case SE_ALG_GCM_FINAL: + if (encrypt) + return SE_CFG_GCM_FINAL_ENCRYPT; + else + return SE_CFG_GCM_FINAL_DECRYPT; + + case SE_ALG_CMAC: + return SE_CFG_CMAC; + + case SE_ALG_CBC_MAC: + return SE_AES_ENC_ALG_AES_ENC | + SE_AES_DST_HASH_REG; + } + return -EINVAL; +} + +static unsigned int tegra_aes_prep_cmd(struct tegra_aes_ctx *ctx, + struct tegra_aes_reqctx *rctx) +{ + unsigned int data_count, res_bits, i = 0, j; + struct tegra_se *se = ctx->se; + u32 *cpuvaddr = se->cmdbuf->addr; + dma_addr_t addr = rctx->datbuf.addr; + + data_count = rctx->len / AES_BLOCK_SIZE; + res_bits = (rctx->len % AES_BLOCK_SIZE) * 8; + + /* + * Hardware processes data_count + 1 blocks. + * Reduce 1 block if there is no residue + */ + if (!res_bits) + data_count--; + + if (rctx->iv) { + cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT); + cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr); + for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++) + cpuvaddr[i++] = rctx->iv[j]; + } + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1); + cpuvaddr[i++] = SE_LAST_BLOCK_VAL(data_count) | + SE_LAST_BLOCK_RES_BITS(res_bits); + + cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6); + cpuvaddr[i++] = rctx->config; + cpuvaddr[i++] = rctx->crypto_config; + + /* Source address setting */ + cpuvaddr[i++] = lower_32_bits(addr); + cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(addr)) | SE_ADDR_HI_SZ(rctx->len); + + /* Destination address setting */ + cpuvaddr[i++] = lower_32_bits(addr); + cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(addr)) | + SE_ADDR_HI_SZ(rctx->len); + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1); + cpuvaddr[i++] = SE_AES_OP_WRSTALL | SE_AES_OP_LASTBUF | + SE_AES_OP_START; + + cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); + cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | + host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); + + dev_dbg(se->dev, "cfg %#x crypto cfg %#x\n", rctx->config, rctx->crypto_config); + + return i; +} + +static int tegra_aes_do_one_req(struct crypto_engine *engine, void *areq) +{ + struct skcipher_request *req = container_of(areq, struct skcipher_request, base); + struct tegra_aes_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + struct tegra_aes_reqctx *rctx = skcipher_request_ctx(req); + struct tegra_se *se = ctx->se; + unsigned int cmdlen, key1_id, key2_id; + int ret; + + rctx->iv = (ctx->alg == SE_ALG_ECB) ? NULL : (u32 *)req->iv; + rctx->len = req->cryptlen; + key1_id = ctx->key1_id; + key2_id = ctx->key2_id; + + /* Pad input to AES Block size */ + if (ctx->alg != SE_ALG_XTS) { + if (rctx->len % AES_BLOCK_SIZE) + rctx->len += AES_BLOCK_SIZE - (rctx->len % AES_BLOCK_SIZE); + } + + rctx->datbuf.size = rctx->len; + rctx->datbuf.buf = dma_alloc_coherent(se->dev, rctx->datbuf.size, + &rctx->datbuf.addr, GFP_KERNEL); + if (!rctx->datbuf.buf) { + ret = -ENOMEM; + goto out_finalize; + } + + scatterwalk_map_and_copy(rctx->datbuf.buf, req->src, 0, req->cryptlen, 0); + + rctx->config = tegra234_aes_cfg(ctx->alg, rctx->encrypt); + rctx->crypto_config = tegra234_aes_crypto_cfg(ctx->alg, rctx->encrypt); + + if (!key1_id) { + ret = tegra_key_submit_reserved_aes(ctx->se, ctx->key1, + ctx->keylen, ctx->alg, &key1_id); + if (ret) + goto out; + } + + rctx->crypto_config |= SE_AES_KEY_INDEX(key1_id); + + if (ctx->alg == SE_ALG_XTS) { + if (!key2_id) { + ret = tegra_key_submit_reserved_xts(ctx->se, ctx->key2, + ctx->keylen, ctx->alg, &key2_id); + if (ret) + goto out; + } + + rctx->crypto_config |= SE_AES_KEY2_INDEX(key2_id); + } + + /* Prepare the command and submit for execution */ + cmdlen = tegra_aes_prep_cmd(ctx, rctx); + ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); + + /* Copy the result */ + tegra_aes_update_iv(req, ctx); + scatterwalk_map_and_copy(rctx->datbuf.buf, req->dst, 0, req->cryptlen, 1); + +out: + /* Free the buffer */ + dma_free_coherent(ctx->se->dev, rctx->datbuf.size, + rctx->datbuf.buf, rctx->datbuf.addr); + + if (tegra_key_is_reserved(key1_id)) + tegra_key_invalidate_reserved(ctx->se, key1_id, ctx->alg); + + if (tegra_key_is_reserved(key2_id)) + tegra_key_invalidate_reserved(ctx->se, key2_id, ctx->alg); + +out_finalize: + local_bh_disable(); + crypto_finalize_skcipher_request(se->engine, req, ret); + local_bh_enable(); + + return 0; +} + +static int tegra_aes_cra_init(struct crypto_skcipher *tfm) +{ + struct tegra_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct tegra_se_alg *se_alg; + const char *algname; + int ret; + + se_alg = container_of(alg, struct tegra_se_alg, alg.skcipher.base); + + crypto_skcipher_set_reqsize(tfm, sizeof(struct tegra_aes_reqctx)); + + ctx->ivsize = crypto_skcipher_ivsize(tfm); + ctx->se = se_alg->se_dev; + ctx->key1_id = 0; + ctx->key2_id = 0; + ctx->keylen = 0; + + algname = crypto_tfm_alg_name(&tfm->base); + ret = se_algname_to_algid(algname); + if (ret < 0) { + dev_err(ctx->se->dev, "invalid algorithm\n"); + return ret; + } + + ctx->alg = ret; + + return 0; +} + +static void tegra_aes_cra_exit(struct crypto_skcipher *tfm) +{ + struct tegra_aes_ctx *ctx = crypto_tfm_ctx(&tfm->base); + + if (ctx->key1_id) + tegra_key_invalidate(ctx->se, ctx->key1_id, ctx->alg); + + if (ctx->key2_id) + tegra_key_invalidate(ctx->se, ctx->key2_id, ctx->alg); +} + +static int tegra_aes_setkey(struct crypto_skcipher *tfm, + const u8 *key, u32 keylen) +{ + struct tegra_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + int ret; + + if (aes_check_keylen(keylen)) { + dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen); + return -EINVAL; + } + + ret = tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key1_id); + if (ret) { + ctx->keylen = keylen; + memcpy(ctx->key1, key, keylen); + } + + return 0; +} + +static int tegra_xts_setkey(struct crypto_skcipher *tfm, + const u8 *key, u32 keylen) +{ + struct tegra_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + u32 len = keylen / 2; + int ret; + + ret = xts_verify_key(tfm, key, keylen); + if (ret || aes_check_keylen(len)) { + dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen); + return -EINVAL; + } + + ret = tegra_key_submit(ctx->se, key, len, + ctx->alg, &ctx->key1_id); + if (ret) { + ctx->keylen = len; + memcpy(ctx->key1, key, len); + } + + ret = tegra_key_submit(ctx->se, key + len, len, + ctx->alg, &ctx->key2_id); + if (ret) { + ctx->keylen = len; + memcpy(ctx->key2, key + len, len); + } + + return 0; +} + +static int tegra_aes_kac_manifest(u32 user, u32 alg, u32 keylen) +{ + int manifest; + + manifest = SE_KAC_USER_NS; + + switch (alg) { + case SE_ALG_CBC: + case SE_ALG_ECB: + case SE_ALG_CTR: + manifest |= SE_KAC_ENC; + break; + case SE_ALG_XTS: + manifest |= SE_KAC_XTS; + break; + case SE_ALG_GCM: + manifest |= SE_KAC_GCM; + break; + case SE_ALG_CMAC: + manifest |= SE_KAC_CMAC; + break; + case SE_ALG_CBC_MAC: + manifest |= SE_KAC_ENC; + break; + default: + return -EINVAL; + } + + switch (keylen) { + case AES_KEYSIZE_128: + manifest |= SE_KAC_SIZE_128; + break; + case AES_KEYSIZE_192: + manifest |= SE_KAC_SIZE_192; + break; + case AES_KEYSIZE_256: + manifest |= SE_KAC_SIZE_256; + break; + default: + return -EINVAL; + } + + return manifest; +} + +static int tegra_aes_crypt(struct skcipher_request *req, bool encrypt) + +{ + struct crypto_skcipher *tfm; + struct tegra_aes_ctx *ctx; + struct tegra_aes_reqctx *rctx; + + tfm = crypto_skcipher_reqtfm(req); + ctx = crypto_skcipher_ctx(tfm); + rctx = skcipher_request_ctx(req); + + if (ctx->alg != SE_ALG_XTS) { + if (!IS_ALIGNED(req->cryptlen, crypto_skcipher_blocksize(tfm))) { + dev_dbg(ctx->se->dev, "invalid length (%d)", req->cryptlen); + return -EINVAL; + } + } else if (req->cryptlen < XTS_BLOCK_SIZE) { + dev_dbg(ctx->se->dev, "invalid length (%d)", req->cryptlen); + return -EINVAL; + } + + if (!req->cryptlen) + return 0; + + rctx->encrypt = encrypt; + + return crypto_transfer_skcipher_request_to_engine(ctx->se->engine, req); +} + +static int tegra_aes_encrypt(struct skcipher_request *req) +{ + return tegra_aes_crypt(req, true); +} + +static int tegra_aes_decrypt(struct skcipher_request *req) +{ + return tegra_aes_crypt(req, false); +} + +static struct tegra_se_alg tegra_aes_algs[] = { + { + .alg.skcipher.op.do_one_request = tegra_aes_do_one_req, + .alg.skcipher.base = { + .init = tegra_aes_cra_init, + .exit = tegra_aes_cra_exit, + .setkey = tegra_aes_setkey, + .encrypt = tegra_aes_encrypt, + .decrypt = tegra_aes_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-tegra", + .cra_priority = 500, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_aes_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + } + }, { + .alg.skcipher.op.do_one_request = tegra_aes_do_one_req, + .alg.skcipher.base = { + .init = tegra_aes_cra_init, + .exit = tegra_aes_cra_exit, + .setkey = tegra_aes_setkey, + .encrypt = tegra_aes_encrypt, + .decrypt = tegra_aes_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-tegra", + .cra_priority = 500, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_aes_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + } + }, { + .alg.skcipher.op.do_one_request = tegra_aes_do_one_req, + .alg.skcipher.base = { + .init = tegra_aes_cra_init, + .exit = tegra_aes_cra_exit, + .setkey = tegra_aes_setkey, + .encrypt = tegra_aes_encrypt, + .decrypt = tegra_aes_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .base = { + .cra_name = "ctr(aes)", + .cra_driver_name = "ctr-aes-tegra", + .cra_priority = 500, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct tegra_aes_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + } + }, { + .alg.skcipher.op.do_one_request = tegra_aes_do_one_req, + .alg.skcipher.base = { + .init = tegra_aes_cra_init, + .exit = tegra_aes_cra_exit, + .setkey = tegra_xts_setkey, + .encrypt = tegra_aes_encrypt, + .decrypt = tegra_aes_decrypt, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .base = { + .cra_name = "xts(aes)", + .cra_driver_name = "xts-aes-tegra", + .cra_priority = 500, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_aes_ctx), + .cra_alignmask = (__alignof__(u64) - 1), + .cra_module = THIS_MODULE, + }, + } + }, +}; + +static unsigned int tegra_gmac_prep_cmd(struct tegra_aead_ctx *ctx, + struct tegra_aead_reqctx *rctx) +{ + unsigned int data_count, res_bits, i = 0; + struct tegra_se *se = ctx->se; + u32 *cpuvaddr = se->cmdbuf->addr; + + data_count = (rctx->assoclen / AES_BLOCK_SIZE); + res_bits = (rctx->assoclen % AES_BLOCK_SIZE) * 8; + + /* + * Hardware processes data_count + 1 blocks. + * Reduce 1 block if there is no residue + */ + if (!res_bits) + data_count--; + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1); + cpuvaddr[i++] = SE_LAST_BLOCK_VAL(data_count) | + SE_LAST_BLOCK_RES_BITS(res_bits); + + cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 4); + cpuvaddr[i++] = rctx->config; + cpuvaddr[i++] = rctx->crypto_config; + cpuvaddr[i++] = lower_32_bits(rctx->inbuf.addr); + cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->inbuf.addr)) | + SE_ADDR_HI_SZ(rctx->assoclen); + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1); + cpuvaddr[i++] = SE_AES_OP_WRSTALL | SE_AES_OP_FINAL | + SE_AES_OP_INIT | SE_AES_OP_LASTBUF | + SE_AES_OP_START; + + cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); + cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | + host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); + + return i; +} + +static unsigned int tegra_gcm_crypt_prep_cmd(struct tegra_aead_ctx *ctx, + struct tegra_aead_reqctx *rctx) +{ + unsigned int data_count, res_bits, i = 0, j; + struct tegra_se *se = ctx->se; + u32 *cpuvaddr = se->cmdbuf->addr, op; + + data_count = (rctx->cryptlen / AES_BLOCK_SIZE); + res_bits = (rctx->cryptlen % AES_BLOCK_SIZE) * 8; + op = SE_AES_OP_WRSTALL | SE_AES_OP_FINAL | + SE_AES_OP_LASTBUF | SE_AES_OP_START; + + /* + * If there is no assoc data, + * this will be the init command + */ + if (!rctx->assoclen) + op |= SE_AES_OP_INIT; + + /* + * Hardware processes data_count + 1 blocks. + * Reduce 1 block if there is no residue + */ + if (!res_bits) + data_count--; + + cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT); + cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr); + for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++) + cpuvaddr[i++] = rctx->iv[j]; + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1); + cpuvaddr[i++] = SE_LAST_BLOCK_VAL(data_count) | + SE_LAST_BLOCK_RES_BITS(res_bits); + + cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6); + cpuvaddr[i++] = rctx->config; + cpuvaddr[i++] = rctx->crypto_config; + + /* Source Address */ + cpuvaddr[i++] = lower_32_bits(rctx->inbuf.addr); + cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->inbuf.addr)) | + SE_ADDR_HI_SZ(rctx->cryptlen); + + /* Destination Address */ + cpuvaddr[i++] = lower_32_bits(rctx->outbuf.addr); + cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->outbuf.addr)) | + SE_ADDR_HI_SZ(rctx->cryptlen); + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1); + cpuvaddr[i++] = op; + + cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); + cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | + host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); + + dev_dbg(se->dev, "cfg %#x crypto cfg %#x\n", rctx->config, rctx->crypto_config); + return i; +} + +static int tegra_gcm_prep_final_cmd(struct tegra_se *se, u32 *cpuvaddr, + struct tegra_aead_reqctx *rctx) +{ + unsigned int i = 0, j; + u32 op; + + op = SE_AES_OP_WRSTALL | SE_AES_OP_FINAL | + SE_AES_OP_LASTBUF | SE_AES_OP_START; + + /* + * Set init for zero sized vector + */ + if (!rctx->assoclen && !rctx->cryptlen) + op |= SE_AES_OP_INIT; + + cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->aad_len, 2); + cpuvaddr[i++] = rctx->assoclen * 8; + cpuvaddr[i++] = 0; + + cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->cryp_msg_len, 2); + cpuvaddr[i++] = rctx->cryptlen * 8; + cpuvaddr[i++] = 0; + + cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT); + cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr); + for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++) + cpuvaddr[i++] = rctx->iv[j]; + + cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6); + cpuvaddr[i++] = rctx->config; + cpuvaddr[i++] = rctx->crypto_config; + cpuvaddr[i++] = 0; + cpuvaddr[i++] = 0; + + /* Destination Address */ + cpuvaddr[i++] = lower_32_bits(rctx->outbuf.addr); + cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->outbuf.addr)) | + SE_ADDR_HI_SZ(0x10); /* HW always generates 128-bit tag */ + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1); + cpuvaddr[i++] = op; + + cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); + cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | + host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); + + dev_dbg(se->dev, "cfg %#x crypto cfg %#x\n", rctx->config, rctx->crypto_config); + + return i; +} + +static int tegra_gcm_do_gmac(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx) +{ + struct tegra_se *se = ctx->se; + unsigned int cmdlen; + + scatterwalk_map_and_copy(rctx->inbuf.buf, + rctx->src_sg, 0, rctx->assoclen, 0); + + rctx->config = tegra234_aes_cfg(SE_ALG_GMAC, rctx->encrypt); + rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_GMAC, rctx->encrypt) | + SE_AES_KEY_INDEX(rctx->key_id); + + cmdlen = tegra_gmac_prep_cmd(ctx, rctx); + + return tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); +} + +static int tegra_gcm_do_crypt(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx) +{ + struct tegra_se *se = ctx->se; + int cmdlen, ret; + + scatterwalk_map_and_copy(rctx->inbuf.buf, rctx->src_sg, + rctx->assoclen, rctx->cryptlen, 0); + + rctx->config = tegra234_aes_cfg(SE_ALG_GCM, rctx->encrypt); + rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_GCM, rctx->encrypt) | + SE_AES_KEY_INDEX(rctx->key_id); + + /* Prepare command and submit */ + cmdlen = tegra_gcm_crypt_prep_cmd(ctx, rctx); + ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); + if (ret) + return ret; + + /* Copy the result */ + scatterwalk_map_and_copy(rctx->outbuf.buf, rctx->dst_sg, + rctx->assoclen, rctx->cryptlen, 1); + + return 0; +} + +static int tegra_gcm_do_final(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx) +{ + struct tegra_se *se = ctx->se; + u32 *cpuvaddr = se->cmdbuf->addr; + int cmdlen, ret, offset; + + rctx->config = tegra234_aes_cfg(SE_ALG_GCM_FINAL, rctx->encrypt); + rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_GCM_FINAL, rctx->encrypt) | + SE_AES_KEY_INDEX(rctx->key_id); + + /* Prepare command and submit */ + cmdlen = tegra_gcm_prep_final_cmd(se, cpuvaddr, rctx); + ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); + if (ret) + return ret; + + if (rctx->encrypt) { + /* Copy the result */ + offset = rctx->assoclen + rctx->cryptlen; + scatterwalk_map_and_copy(rctx->outbuf.buf, rctx->dst_sg, + offset, rctx->authsize, 1); + } + + return 0; +} + +static int tegra_gcm_do_verify(struct tegra_se *se, struct tegra_aead_reqctx *rctx) +{ + unsigned int offset; + u8 mac[16]; + + offset = rctx->assoclen + rctx->cryptlen; + scatterwalk_map_and_copy(mac, rctx->src_sg, offset, rctx->authsize, 0); + + if (crypto_memneq(rctx->outbuf.buf, mac, rctx->authsize)) + return -EBADMSG; + + return 0; +} + +static inline int tegra_ccm_check_iv(const u8 *iv) +{ + /* iv[0] gives value of q-1 + * 2 <= q <= 8 as per NIST 800-38C notation + * 2 <= L <= 8, so 1 <= L' <= 7. as per rfc 3610 notation + */ + if (iv[0] < 1 || iv[0] > 7) { + pr_debug("ccm_check_iv failed %d\n", iv[0]); + return -EINVAL; + } + + return 0; +} + +static unsigned int tegra_cbcmac_prep_cmd(struct tegra_aead_ctx *ctx, + struct tegra_aead_reqctx *rctx) +{ + unsigned int data_count, i = 0; + struct tegra_se *se = ctx->se; + u32 *cpuvaddr = se->cmdbuf->addr; + + data_count = (rctx->inbuf.size / AES_BLOCK_SIZE) - 1; + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1); + cpuvaddr[i++] = SE_LAST_BLOCK_VAL(data_count); + + cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6); + cpuvaddr[i++] = rctx->config; + cpuvaddr[i++] = rctx->crypto_config; + + cpuvaddr[i++] = lower_32_bits(rctx->inbuf.addr); + cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->inbuf.addr)) | + SE_ADDR_HI_SZ(rctx->inbuf.size); + + cpuvaddr[i++] = lower_32_bits(rctx->outbuf.addr); + cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->outbuf.addr)) | + SE_ADDR_HI_SZ(0x10); /* HW always generates 128 bit tag */ + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1); + cpuvaddr[i++] = SE_AES_OP_WRSTALL | + SE_AES_OP_LASTBUF | SE_AES_OP_START; + + cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); + cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | + host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); + + return i; +} + +static unsigned int tegra_ctr_prep_cmd(struct tegra_aead_ctx *ctx, + struct tegra_aead_reqctx *rctx) +{ + unsigned int i = 0, j; + struct tegra_se *se = ctx->se; + u32 *cpuvaddr = se->cmdbuf->addr; + + cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT); + cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr); + for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++) + cpuvaddr[i++] = rctx->iv[j]; + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1); + cpuvaddr[i++] = (rctx->inbuf.size / AES_BLOCK_SIZE) - 1; + cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6); + cpuvaddr[i++] = rctx->config; + cpuvaddr[i++] = rctx->crypto_config; + + /* Source address setting */ + cpuvaddr[i++] = lower_32_bits(rctx->inbuf.addr); + cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->inbuf.addr)) | + SE_ADDR_HI_SZ(rctx->inbuf.size); + + /* Destination address setting */ + cpuvaddr[i++] = lower_32_bits(rctx->outbuf.addr); + cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->outbuf.addr)) | + SE_ADDR_HI_SZ(rctx->inbuf.size); + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1); + cpuvaddr[i++] = SE_AES_OP_WRSTALL | SE_AES_OP_LASTBUF | + SE_AES_OP_START; + + cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); + cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | + host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); + + dev_dbg(se->dev, "cfg %#x crypto cfg %#x\n", + rctx->config, rctx->crypto_config); + + return i; +} + +static int tegra_ccm_do_cbcmac(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx) +{ + struct tegra_se *se = ctx->se; + int cmdlen; + + rctx->config = tegra234_aes_cfg(SE_ALG_CBC_MAC, rctx->encrypt); + rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_CBC_MAC, + rctx->encrypt) | + SE_AES_KEY_INDEX(rctx->key_id); + + /* Prepare command and submit */ + cmdlen = tegra_cbcmac_prep_cmd(ctx, rctx); + + return tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); +} + +static int tegra_ccm_set_msg_len(u8 *block, unsigned int msglen, int csize) +{ + __be32 data; + + memset(block, 0, csize); + block += csize; + + if (csize >= 4) + csize = 4; + else if (msglen > (1 << (8 * csize))) + return -EOVERFLOW; + + data = cpu_to_be32(msglen); + memcpy(block - csize, (u8 *)&data + 4 - csize, csize); + + return 0; +} + +static int tegra_ccm_format_nonce(struct tegra_aead_reqctx *rctx, u8 *nonce) +{ + unsigned int q, t; + u8 *q_ptr, *iv = (u8 *)rctx->iv; + + memcpy(nonce, rctx->iv, 16); + + /*** 1. Prepare Flags Octet ***/ + + /* Encode t (mac length) */ + t = rctx->authsize; + nonce[0] |= (((t - 2) / 2) << 3); + + /* Adata */ + if (rctx->assoclen) + nonce[0] |= (1 << 6); + + /*** Encode Q - message length ***/ + q = iv[0] + 1; + q_ptr = nonce + 16 - q; + + return tegra_ccm_set_msg_len(q_ptr, rctx->cryptlen, q); +} + +static int tegra_ccm_format_adata(u8 *adata, unsigned int a) +{ + int len = 0; + + /* add control info for associated data + * RFC 3610 and NIST Special Publication 800-38C + */ + if (a < 65280) { + *(__be16 *)adata = cpu_to_be16(a); + len = 2; + } else { + *(__be16 *)adata = cpu_to_be16(0xfffe); + *(__be32 *)&adata[2] = cpu_to_be32(a); + len = 6; + } + + return len; +} + +static int tegra_ccm_add_padding(u8 *buf, unsigned int len) +{ + unsigned int padlen = 16 - (len % 16); + u8 padding[16] = {0}; + + if (padlen == 16) + return 0; + + memcpy(buf, padding, padlen); + + return padlen; +} + +static int tegra_ccm_format_blocks(struct tegra_aead_reqctx *rctx) +{ + unsigned int alen = 0, offset = 0; + u8 nonce[16], adata[16]; + int ret; + + ret = tegra_ccm_format_nonce(rctx, nonce); + if (ret) + return ret; + + memcpy(rctx->inbuf.buf, nonce, 16); + offset = 16; + + if (rctx->assoclen) { + alen = tegra_ccm_format_adata(adata, rctx->assoclen); + memcpy(rctx->inbuf.buf + offset, adata, alen); + offset += alen; + + scatterwalk_map_and_copy(rctx->inbuf.buf + offset, + rctx->src_sg, 0, rctx->assoclen, 0); + + offset += rctx->assoclen; + offset += tegra_ccm_add_padding(rctx->inbuf.buf + offset, + rctx->assoclen + alen); + } + + return offset; +} + +static int tegra_ccm_mac_result(struct tegra_se *se, struct tegra_aead_reqctx *rctx) +{ + u32 result[16]; + int i, ret; + + /* Read and clear Result */ + for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) + result[i] = readl(se->base + se->hw->regs->result + (i * 4)); + + for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) + writel(0, se->base + se->hw->regs->result + (i * 4)); + + if (rctx->encrypt) { + memcpy(rctx->authdata, result, rctx->authsize); + } else { + ret = crypto_memneq(rctx->authdata, result, rctx->authsize); + if (ret) + return -EBADMSG; + } + + return 0; +} + +static int tegra_ccm_ctr_result(struct tegra_se *se, struct tegra_aead_reqctx *rctx) +{ + /* Copy result */ + scatterwalk_map_and_copy(rctx->outbuf.buf + 16, rctx->dst_sg, + rctx->assoclen, rctx->cryptlen, 1); + + if (rctx->encrypt) + scatterwalk_map_and_copy(rctx->outbuf.buf, rctx->dst_sg, + rctx->assoclen + rctx->cryptlen, + rctx->authsize, 1); + else + memcpy(rctx->authdata, rctx->outbuf.buf, rctx->authsize); + + return 0; +} + +static int tegra_ccm_compute_auth(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx) +{ + struct tegra_se *se = ctx->se; + struct scatterlist *sg; + int offset, ret; + + offset = tegra_ccm_format_blocks(rctx); + if (offset < 0) + return -EINVAL; + + /* Copy plain text to the buffer */ + sg = rctx->encrypt ? rctx->src_sg : rctx->dst_sg; + + scatterwalk_map_and_copy(rctx->inbuf.buf + offset, + sg, rctx->assoclen, + rctx->cryptlen, 0); + offset += rctx->cryptlen; + offset += tegra_ccm_add_padding(rctx->inbuf.buf + offset, rctx->cryptlen); + + rctx->inbuf.size = offset; + + ret = tegra_ccm_do_cbcmac(ctx, rctx); + if (ret) + return ret; + + return tegra_ccm_mac_result(se, rctx); +} + +static int tegra_ccm_do_ctr(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx) +{ + struct tegra_se *se = ctx->se; + unsigned int cmdlen, offset = 0; + struct scatterlist *sg = rctx->src_sg; + int ret; + + rctx->config = tegra234_aes_cfg(SE_ALG_CTR, rctx->encrypt); + rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_CTR, rctx->encrypt) | + SE_AES_KEY_INDEX(rctx->key_id); + + /* Copy authdata in the top of buffer for encryption/decryption */ + if (rctx->encrypt) + memcpy(rctx->inbuf.buf, rctx->authdata, rctx->authsize); + else + scatterwalk_map_and_copy(rctx->inbuf.buf, sg, + rctx->assoclen + rctx->cryptlen, + rctx->authsize, 0); + + offset += rctx->authsize; + offset += tegra_ccm_add_padding(rctx->inbuf.buf + offset, rctx->authsize); + + /* If there is no cryptlen, proceed to submit the task */ + if (rctx->cryptlen) { + scatterwalk_map_and_copy(rctx->inbuf.buf + offset, sg, + rctx->assoclen, rctx->cryptlen, 0); + offset += rctx->cryptlen; + offset += tegra_ccm_add_padding(rctx->inbuf.buf + offset, rctx->cryptlen); + } + + rctx->inbuf.size = offset; + + /* Prepare command and submit */ + cmdlen = tegra_ctr_prep_cmd(ctx, rctx); + ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); + if (ret) + return ret; + + return tegra_ccm_ctr_result(se, rctx); +} + +static int tegra_ccm_crypt_init(struct aead_request *req, struct tegra_se *se, + struct tegra_aead_reqctx *rctx) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + u8 *iv = (u8 *)rctx->iv; + int ret, i; + + rctx->src_sg = req->src; + rctx->dst_sg = req->dst; + rctx->assoclen = req->assoclen; + rctx->authsize = crypto_aead_authsize(tfm); + + if (rctx->encrypt) + rctx->cryptlen = req->cryptlen; + else + rctx->cryptlen = req->cryptlen - rctx->authsize; + + memcpy(iv, req->iv, 16); + + ret = tegra_ccm_check_iv(iv); + if (ret) + return ret; + + /* Note: rfc 3610 and NIST 800-38C require counter (ctr_0) of + * zero to encrypt auth tag. + * req->iv has the formatted ctr_0 (i.e. Flags || N || 0). + */ + memset(iv + 15 - iv[0], 0, iv[0] + 1); + + /* Clear any previous result */ + for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) + writel(0, se->base + se->hw->regs->result + (i * 4)); + + return 0; +} + +static int tegra_ccm_do_one_req(struct crypto_engine *engine, void *areq) +{ + struct aead_request *req = container_of(areq, struct aead_request, base); + struct tegra_aead_reqctx *rctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct tegra_se *se = ctx->se; + unsigned int bufsize; + int ret; + + ret = tegra_ccm_crypt_init(req, se, rctx); + if (ret) + goto out_finalize; + + rctx->key_id = ctx->key_id; + + /* Allocate buffers required */ + bufsize = rctx->assoclen + rctx->authsize + rctx->cryptlen + 100; + rctx->inbuf.size = bufsize; + rctx->inbuf.buf = dma_alloc_coherent(ctx->se->dev, bufsize, + &rctx->inbuf.addr, GFP_KERNEL); + ret = -ENOMEM; + if (!rctx->inbuf.buf) + goto out_finalize; + + rctx->outbuf.size = bufsize; + rctx->outbuf.buf = dma_alloc_coherent(ctx->se->dev, bufsize, + &rctx->outbuf.addr, GFP_KERNEL); + if (!rctx->outbuf.buf) + goto out_free_inbuf; + + if (!ctx->key_id) { + ret = tegra_key_submit_reserved_aes(ctx->se, ctx->key, + ctx->keylen, ctx->alg, &rctx->key_id); + if (ret) + goto out; + } + + if (rctx->encrypt) { + /* CBC MAC Operation */ + ret = tegra_ccm_compute_auth(ctx, rctx); + if (ret) + goto out; + + /* CTR operation */ + ret = tegra_ccm_do_ctr(ctx, rctx); + if (ret) + goto out; + } else { + /* CTR operation */ + ret = tegra_ccm_do_ctr(ctx, rctx); + if (ret) + goto out; + + /* CBC MAC Operation */ + ret = tegra_ccm_compute_auth(ctx, rctx); + if (ret) + goto out; + } + +out: + dma_free_coherent(ctx->se->dev, bufsize, + rctx->outbuf.buf, rctx->outbuf.addr); + +out_free_inbuf: + dma_free_coherent(ctx->se->dev, bufsize, + rctx->inbuf.buf, rctx->inbuf.addr); + + if (tegra_key_is_reserved(rctx->key_id)) + tegra_key_invalidate_reserved(ctx->se, rctx->key_id, ctx->alg); + +out_finalize: + local_bh_disable(); + crypto_finalize_aead_request(ctx->se->engine, req, ret); + local_bh_enable(); + + return 0; +} + +static int tegra_gcm_do_one_req(struct crypto_engine *engine, void *areq) +{ + struct aead_request *req = container_of(areq, struct aead_request, base); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct tegra_aead_reqctx *rctx = aead_request_ctx(req); + unsigned int bufsize; + int ret; + + rctx->src_sg = req->src; + rctx->dst_sg = req->dst; + rctx->assoclen = req->assoclen; + rctx->authsize = crypto_aead_authsize(tfm); + + if (rctx->encrypt) + rctx->cryptlen = req->cryptlen; + else + rctx->cryptlen = req->cryptlen - rctx->authsize; + + memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); + rctx->iv[3] = (1 << 24); + + rctx->key_id = ctx->key_id; + + /* Allocate buffers required */ + bufsize = rctx->assoclen + rctx->authsize + rctx->cryptlen; + rctx->inbuf.size = bufsize; + rctx->inbuf.buf = dma_alloc_coherent(ctx->se->dev, bufsize, + &rctx->inbuf.addr, GFP_KERNEL); + if (!rctx->inbuf.buf) { + ret = -ENOMEM; + goto out_finalize; + } + + rctx->outbuf.size = bufsize; + rctx->outbuf.buf = dma_alloc_coherent(ctx->se->dev, bufsize, + &rctx->outbuf.addr, GFP_KERNEL); + if (!rctx->outbuf.buf) { + ret = -ENOMEM; + goto out_free_inbuf; + } + + if (!ctx->key_id) { + ret = tegra_key_submit_reserved_aes(ctx->se, ctx->key, + ctx->keylen, ctx->alg, &rctx->key_id); + if (ret) + goto out; + } + + /* If there is associated data perform GMAC operation */ + if (rctx->assoclen) { + ret = tegra_gcm_do_gmac(ctx, rctx); + if (ret) + goto out; + } + + /* GCM Encryption/Decryption operation */ + if (rctx->cryptlen) { + ret = tegra_gcm_do_crypt(ctx, rctx); + if (ret) + goto out; + } + + /* GCM_FINAL operation */ + ret = tegra_gcm_do_final(ctx, rctx); + if (ret) + goto out; + + if (!rctx->encrypt) + ret = tegra_gcm_do_verify(ctx->se, rctx); + +out: + dma_free_coherent(ctx->se->dev, bufsize, + rctx->outbuf.buf, rctx->outbuf.addr); + +out_free_inbuf: + dma_free_coherent(ctx->se->dev, bufsize, + rctx->inbuf.buf, rctx->inbuf.addr); + + if (tegra_key_is_reserved(rctx->key_id)) + tegra_key_invalidate_reserved(ctx->se, rctx->key_id, ctx->alg); + +out_finalize: + local_bh_disable(); + crypto_finalize_aead_request(ctx->se->engine, req, ret); + local_bh_enable(); + + return 0; +} + +static int tegra_aead_cra_init(struct crypto_aead *tfm) +{ + struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct aead_alg *alg = crypto_aead_alg(tfm); + struct tegra_se_alg *se_alg; + const char *algname; + int ret; + + algname = crypto_tfm_alg_name(&tfm->base); + + se_alg = container_of(alg, struct tegra_se_alg, alg.aead.base); + + crypto_aead_set_reqsize(tfm, sizeof(struct tegra_aead_reqctx)); + + ctx->se = se_alg->se_dev; + ctx->key_id = 0; + ctx->keylen = 0; + + ret = se_algname_to_algid(algname); + if (ret < 0) { + dev_err(ctx->se->dev, "invalid algorithm\n"); + return ret; + } + + ctx->alg = ret; + + return 0; +} + +static int tegra_ccm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) +{ + switch (authsize) { + case 4: + case 6: + case 8: + case 10: + case 12: + case 14: + case 16: + return 0; + default: + return -EINVAL; + } +} + +static int tegra_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) +{ + return crypto_gcm_check_authsize(authsize); +} + +static void tegra_aead_cra_exit(struct crypto_aead *tfm) +{ + struct tegra_aead_ctx *ctx = crypto_tfm_ctx(&tfm->base); + + if (ctx->key_id) + tegra_key_invalidate(ctx->se, ctx->key_id, ctx->alg); +} + +static int tegra_aead_crypt(struct aead_request *req, bool encrypt) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct tegra_aead_reqctx *rctx = aead_request_ctx(req); + + rctx->encrypt = encrypt; + + return crypto_transfer_aead_request_to_engine(ctx->se->engine, req); +} + +static int tegra_aead_encrypt(struct aead_request *req) +{ + return tegra_aead_crypt(req, true); +} + +static int tegra_aead_decrypt(struct aead_request *req) +{ + return tegra_aead_crypt(req, false); +} + +static int tegra_aead_setkey(struct crypto_aead *tfm, + const u8 *key, u32 keylen) +{ + struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); + int ret; + + if (aes_check_keylen(keylen)) { + dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen); + return -EINVAL; + } + + ret = tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key_id); + if (ret) { + ctx->keylen = keylen; + memcpy(ctx->key, key, keylen); + } + + return 0; +} + +static unsigned int tegra_cmac_prep_cmd(struct tegra_cmac_ctx *ctx, + struct tegra_cmac_reqctx *rctx) +{ + unsigned int data_count, res_bits = 0, i = 0, j; + struct tegra_se *se = ctx->se; + u32 *cpuvaddr = se->cmdbuf->addr, op; + + data_count = (rctx->datbuf.size / AES_BLOCK_SIZE); + + op = SE_AES_OP_WRSTALL | SE_AES_OP_START | SE_AES_OP_LASTBUF; + + if (!(rctx->task & SHA_UPDATE)) { + op |= SE_AES_OP_FINAL; + res_bits = (rctx->datbuf.size % AES_BLOCK_SIZE) * 8; + } + + if (!res_bits && data_count) + data_count--; + + if (rctx->task & SHA_FIRST) { + rctx->task &= ~SHA_FIRST; + + cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT); + cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr); + /* Load 0 IV */ + for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++) + cpuvaddr[i++] = 0; + } + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1); + cpuvaddr[i++] = SE_LAST_BLOCK_VAL(data_count) | + SE_LAST_BLOCK_RES_BITS(res_bits); + + cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6); + cpuvaddr[i++] = rctx->config; + cpuvaddr[i++] = rctx->crypto_config; + + /* Source Address */ + cpuvaddr[i++] = lower_32_bits(rctx->datbuf.addr); + cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->datbuf.addr)) | + SE_ADDR_HI_SZ(rctx->datbuf.size); + cpuvaddr[i++] = 0; + cpuvaddr[i++] = SE_ADDR_HI_SZ(AES_BLOCK_SIZE); + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1); + cpuvaddr[i++] = op; + + cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); + cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | + host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); + + return i; +} + +static void tegra_cmac_copy_result(struct tegra_se *se, struct tegra_cmac_reqctx *rctx) +{ + int i; + + for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) + rctx->result[i] = readl(se->base + se->hw->regs->result + (i * 4)); +} + +static void tegra_cmac_paste_result(struct tegra_se *se, struct tegra_cmac_reqctx *rctx) +{ + int i; + + for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) + writel(rctx->result[i], + se->base + se->hw->regs->result + (i * 4)); +} + +static int tegra_cmac_do_init(struct ahash_request *req) +{ + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_se *se = ctx->se; + int i; + + rctx->total_len = 0; + rctx->datbuf.size = 0; + rctx->residue.size = 0; + rctx->key_id = ctx->key_id; + rctx->task |= SHA_FIRST; + rctx->blk_size = crypto_ahash_blocksize(tfm); + + rctx->residue.buf = dma_alloc_coherent(se->dev, rctx->blk_size * 2, + &rctx->residue.addr, GFP_KERNEL); + if (!rctx->residue.buf) + return -ENOMEM; + + rctx->residue.size = 0; + + /* Clear any previous result */ + for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) + writel(0, se->base + se->hw->regs->result + (i * 4)); + + return 0; +} + +static int tegra_cmac_do_update(struct ahash_request *req) +{ + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_se *se = ctx->se; + unsigned int nblks, nresidue, cmdlen; + int ret; + + if (!req->nbytes) + return 0; + + nresidue = (req->nbytes + rctx->residue.size) % rctx->blk_size; + nblks = (req->nbytes + rctx->residue.size) / rctx->blk_size; + + /* + * Reserve the last block as residue during final() to process. + */ + if (!nresidue && nblks) { + nresidue += rctx->blk_size; + nblks--; + } + + rctx->src_sg = req->src; + rctx->datbuf.size = (req->nbytes + rctx->residue.size) - nresidue; + rctx->total_len += rctx->datbuf.size; + rctx->config = tegra234_aes_cfg(SE_ALG_CMAC, 0); + rctx->crypto_config = SE_AES_KEY_INDEX(rctx->key_id); + + /* + * Keep one block and residue bytes in residue and + * return. The bytes will be processed in final() + */ + if (nblks < 1) { + scatterwalk_map_and_copy(rctx->residue.buf + rctx->residue.size, + rctx->src_sg, 0, req->nbytes, 0); + + rctx->residue.size += req->nbytes; + return 0; + } + + rctx->datbuf.buf = dma_alloc_coherent(se->dev, rctx->datbuf.size, + &rctx->datbuf.addr, GFP_KERNEL); + if (!rctx->datbuf.buf) + return -ENOMEM; + + /* Copy the previous residue first */ + if (rctx->residue.size) + memcpy(rctx->datbuf.buf, rctx->residue.buf, rctx->residue.size); + + scatterwalk_map_and_copy(rctx->datbuf.buf + rctx->residue.size, + rctx->src_sg, 0, req->nbytes - nresidue, 0); + + scatterwalk_map_and_copy(rctx->residue.buf, rctx->src_sg, + req->nbytes - nresidue, nresidue, 0); + + /* Update residue value with the residue after current block */ + rctx->residue.size = nresidue; + + /* + * If this is not the first task, paste the previous copied + * intermediate results to the registers so that it gets picked up. + */ + if (!(rctx->task & SHA_FIRST)) + tegra_cmac_paste_result(ctx->se, rctx); + + cmdlen = tegra_cmac_prep_cmd(ctx, rctx); + ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); + + tegra_cmac_copy_result(ctx->se, rctx); + + dma_free_coherent(ctx->se->dev, rctx->datbuf.size, + rctx->datbuf.buf, rctx->datbuf.addr); + + return ret; +} + +static int tegra_cmac_do_final(struct ahash_request *req) +{ + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_se *se = ctx->se; + u32 *result = (u32 *)req->result; + int ret = 0, i, cmdlen; + + if (!req->nbytes && !rctx->total_len && ctx->fallback_tfm) { + return crypto_shash_tfm_digest(ctx->fallback_tfm, + NULL, 0, req->result); + } + + if (rctx->residue.size) { + rctx->datbuf.buf = dma_alloc_coherent(se->dev, rctx->residue.size, + &rctx->datbuf.addr, GFP_KERNEL); + if (!rctx->datbuf.buf) { + ret = -ENOMEM; + goto out_free; + } + + memcpy(rctx->datbuf.buf, rctx->residue.buf, rctx->residue.size); + } + + rctx->datbuf.size = rctx->residue.size; + rctx->total_len += rctx->residue.size; + rctx->config = tegra234_aes_cfg(SE_ALG_CMAC, 0); + + /* + * If this is not the first task, paste the previous copied + * intermediate results to the registers so that it gets picked up. + */ + if (!(rctx->task & SHA_FIRST)) + tegra_cmac_paste_result(ctx->se, rctx); + + /* Prepare command and submit */ + cmdlen = tegra_cmac_prep_cmd(ctx, rctx); + ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); + if (ret) + goto out; + + /* Read and clear Result register */ + for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) + result[i] = readl(se->base + se->hw->regs->result + (i * 4)); + + for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) + writel(0, se->base + se->hw->regs->result + (i * 4)); + +out: + if (rctx->residue.size) + dma_free_coherent(se->dev, rctx->datbuf.size, + rctx->datbuf.buf, rctx->datbuf.addr); +out_free: + dma_free_coherent(se->dev, crypto_ahash_blocksize(tfm) * 2, + rctx->residue.buf, rctx->residue.addr); + return ret; +} + +static int tegra_cmac_do_one_req(struct crypto_engine *engine, void *areq) +{ + struct ahash_request *req = ahash_request_cast(areq); + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_se *se = ctx->se; + int ret = 0; + + if (rctx->task & SHA_INIT) { + ret = tegra_cmac_do_init(req); + if (ret) + goto out; + + rctx->task &= ~SHA_INIT; + } + + if (!ctx->key_id) { + ret = tegra_key_submit_reserved_aes(ctx->se, ctx->key, + ctx->keylen, ctx->alg, &rctx->key_id); + if (ret) + goto out; + } + + if (rctx->task & SHA_UPDATE) { + ret = tegra_cmac_do_update(req); + if (ret) + goto out; + + rctx->task &= ~SHA_UPDATE; + } + + if (rctx->task & SHA_FINAL) { + ret = tegra_cmac_do_final(req); + if (ret) + goto out; + + rctx->task &= ~SHA_FINAL; + } +out: + if (tegra_key_is_reserved(rctx->key_id)) + tegra_key_invalidate_reserved(ctx->se, rctx->key_id, ctx->alg); + + local_bh_disable(); + crypto_finalize_hash_request(se->engine, req, ret); + local_bh_enable(); + + return 0; +} + +static void tegra_cmac_init_fallback(struct crypto_ahash *tfm, struct tegra_cmac_ctx *ctx, + const char *algname) +{ + unsigned int statesize; + + ctx->fallback_tfm = crypto_alloc_shash(algname, 0, CRYPTO_ALG_NEED_FALLBACK); + + if (IS_ERR(ctx->fallback_tfm)) { + dev_warn(ctx->se->dev, "failed to allocate fallback for %s\n", algname); + ctx->fallback_tfm = NULL; + return; + } + + statesize = crypto_shash_statesize(ctx->fallback_tfm); + + if (statesize > sizeof(struct tegra_cmac_reqctx)) + crypto_ahash_set_statesize(tfm, statesize); +} + +static int tegra_cmac_cra_init(struct crypto_tfm *tfm) +{ + struct tegra_cmac_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_ahash *ahash_tfm = __crypto_ahash_cast(tfm); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg); + struct tegra_se_alg *se_alg; + const char *algname; + int ret; + + algname = crypto_tfm_alg_name(tfm); + se_alg = container_of(alg, struct tegra_se_alg, alg.ahash.base); + + crypto_ahash_set_reqsize(ahash_tfm, sizeof(struct tegra_cmac_reqctx)); + + ctx->se = se_alg->se_dev; + ctx->key_id = 0; + ctx->keylen = 0; + + ret = se_algname_to_algid(algname); + if (ret < 0) { + dev_err(ctx->se->dev, "invalid algorithm\n"); + return ret; + } + + ctx->alg = ret; + + tegra_cmac_init_fallback(ahash_tfm, ctx, algname); + + return 0; +} + +static void tegra_cmac_cra_exit(struct crypto_tfm *tfm) +{ + struct tegra_cmac_ctx *ctx = crypto_tfm_ctx(tfm); + + if (ctx->fallback_tfm) + crypto_free_shash(ctx->fallback_tfm); + + tegra_key_invalidate(ctx->se, ctx->key_id, ctx->alg); +} + +static int tegra_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + int ret; + + if (aes_check_keylen(keylen)) { + dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen); + return -EINVAL; + } + + if (ctx->fallback_tfm) + crypto_shash_setkey(ctx->fallback_tfm, key, keylen); + + ret = tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key_id); + if (ret) { + ctx->keylen = keylen; + memcpy(ctx->key, key, keylen); + } + + return 0; +} + +static int tegra_cmac_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + + rctx->task = SHA_INIT; + + return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); +} + +static int tegra_cmac_update(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + + rctx->task |= SHA_UPDATE; + + return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); +} + +static int tegra_cmac_final(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + + rctx->task |= SHA_FINAL; + + return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); +} + +static int tegra_cmac_finup(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + + rctx->task |= SHA_UPDATE | SHA_FINAL; + + return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); +} + +static int tegra_cmac_digest(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + + rctx->task |= SHA_INIT | SHA_UPDATE | SHA_FINAL; + + return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); +} + +static int tegra_cmac_export(struct ahash_request *req, void *out) +{ + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + + memcpy(out, rctx, sizeof(*rctx)); + + return 0; +} + +static int tegra_cmac_import(struct ahash_request *req, const void *in) +{ + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + + memcpy(rctx, in, sizeof(*rctx)); + + return 0; +} + +static struct tegra_se_alg tegra_aead_algs[] = { + { + .alg.aead.op.do_one_request = tegra_gcm_do_one_req, + .alg.aead.base = { + .init = tegra_aead_cra_init, + .exit = tegra_aead_cra_exit, + .setkey = tegra_aead_setkey, + .setauthsize = tegra_gcm_setauthsize, + .encrypt = tegra_aead_encrypt, + .decrypt = tegra_aead_decrypt, + .maxauthsize = AES_BLOCK_SIZE, + .ivsize = GCM_AES_IV_SIZE, + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "gcm-aes-tegra", + .cra_priority = 500, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct tegra_aead_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + } + }, { + .alg.aead.op.do_one_request = tegra_ccm_do_one_req, + .alg.aead.base = { + .init = tegra_aead_cra_init, + .exit = tegra_aead_cra_exit, + .setkey = tegra_aead_setkey, + .setauthsize = tegra_ccm_setauthsize, + .encrypt = tegra_aead_encrypt, + .decrypt = tegra_aead_decrypt, + .maxauthsize = AES_BLOCK_SIZE, + .ivsize = AES_BLOCK_SIZE, + .chunksize = AES_BLOCK_SIZE, + .base = { + .cra_name = "ccm(aes)", + .cra_driver_name = "ccm-aes-tegra", + .cra_priority = 500, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct tegra_aead_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + } + } +}; + +static struct tegra_se_alg tegra_cmac_algs[] = { + { + .alg.ahash.op.do_one_request = tegra_cmac_do_one_req, + .alg.ahash.base = { + .init = tegra_cmac_init, + .setkey = tegra_cmac_setkey, + .update = tegra_cmac_update, + .final = tegra_cmac_final, + .finup = tegra_cmac_finup, + .digest = tegra_cmac_digest, + .export = tegra_cmac_export, + .import = tegra_cmac_import, + .halg.digestsize = AES_BLOCK_SIZE, + .halg.statesize = sizeof(struct tegra_cmac_reqctx), + .halg.base = { + .cra_name = "cmac(aes)", + .cra_driver_name = "tegra-se-cmac", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_cmac_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_cmac_cra_init, + .cra_exit = tegra_cmac_cra_exit, + } + } + } +}; + +int tegra_init_aes(struct tegra_se *se) +{ + struct aead_engine_alg *aead_alg; + struct ahash_engine_alg *ahash_alg; + struct skcipher_engine_alg *sk_alg; + int i, ret; + + se->manifest = tegra_aes_kac_manifest; + + for (i = 0; i < ARRAY_SIZE(tegra_aes_algs); i++) { + sk_alg = &tegra_aes_algs[i].alg.skcipher; + tegra_aes_algs[i].se_dev = se; + + ret = crypto_engine_register_skcipher(sk_alg); + if (ret) { + dev_err(se->dev, "failed to register %s\n", + sk_alg->base.base.cra_name); + goto err_aes; + } + } + + for (i = 0; i < ARRAY_SIZE(tegra_aead_algs); i++) { + aead_alg = &tegra_aead_algs[i].alg.aead; + tegra_aead_algs[i].se_dev = se; + + ret = crypto_engine_register_aead(aead_alg); + if (ret) { + dev_err(se->dev, "failed to register %s\n", + aead_alg->base.base.cra_name); + goto err_aead; + } + } + + for (i = 0; i < ARRAY_SIZE(tegra_cmac_algs); i++) { + ahash_alg = &tegra_cmac_algs[i].alg.ahash; + tegra_cmac_algs[i].se_dev = se; + + ret = crypto_engine_register_ahash(ahash_alg); + if (ret) { + dev_err(se->dev, "failed to register %s\n", + ahash_alg->base.halg.base.cra_name); + goto err_cmac; + } + } + + return 0; + +err_cmac: + while (i--) + crypto_engine_unregister_ahash(&tegra_cmac_algs[i].alg.ahash); + + i = ARRAY_SIZE(tegra_aead_algs); +err_aead: + while (i--) + crypto_engine_unregister_aead(&tegra_aead_algs[i].alg.aead); + + i = ARRAY_SIZE(tegra_aes_algs); +err_aes: + while (i--) + crypto_engine_unregister_skcipher(&tegra_aes_algs[i].alg.skcipher); + + return ret; +} + +void tegra_deinit_aes(struct tegra_se *se) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(tegra_aes_algs); i++) + crypto_engine_unregister_skcipher(&tegra_aes_algs[i].alg.skcipher); + + for (i = 0; i < ARRAY_SIZE(tegra_aead_algs); i++) + crypto_engine_unregister_aead(&tegra_aead_algs[i].alg.aead); + + for (i = 0; i < ARRAY_SIZE(tegra_cmac_algs); i++) + crypto_engine_unregister_ahash(&tegra_cmac_algs[i].alg.ahash); +} diff --git a/drivers/crypto/tegra/tegra-se-hash.c b/drivers/crypto/tegra/tegra-se-hash.c new file mode 100644 index 000000000..23d549801 --- /dev/null +++ b/drivers/crypto/tegra/tegra-se-hash.c @@ -0,0 +1,1158 @@ +// SPDX-License-Identifier: GPL-2.0-only +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +/* + * Crypto driver to handle HASH algorithms using NVIDIA Security Engine. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tegra-se.h" + +struct tegra_sha_ctx { + struct tegra_se *se; + unsigned int alg; + bool fallback; + u32 key_id; + struct crypto_ahash *fallback_tfm; +}; + +struct tegra_sha_reqctx { + struct scatterlist *src_sg; + struct tegra_se_datbuf datbuf; + struct tegra_se_datbuf residue; + struct tegra_se_datbuf digest; + struct tegra_se_datbuf intr_res; + unsigned int alg; + unsigned int config; + unsigned int total_len; + unsigned int blk_size; + unsigned int task; + u32 key_id; + u32 result[HASH_RESULT_REG_COUNT]; + struct ahash_request fallback_req; +}; + +static int tegra_sha_get_config(u32 alg) +{ + int cfg = 0; + + switch (alg) { + case SE_ALG_SHA1: + cfg |= SE_SHA_ENC_ALG_SHA; + cfg |= SE_SHA_ENC_MODE_SHA1; + break; + + case SE_ALG_HMAC_SHA224: + cfg |= SE_SHA_ENC_ALG_HMAC; + fallthrough; + case SE_ALG_SHA224: + cfg |= SE_SHA_ENC_ALG_SHA; + cfg |= SE_SHA_ENC_MODE_SHA224; + break; + + case SE_ALG_HMAC_SHA256: + cfg |= SE_SHA_ENC_ALG_HMAC; + fallthrough; + case SE_ALG_SHA256: + cfg |= SE_SHA_ENC_ALG_SHA; + cfg |= SE_SHA_ENC_MODE_SHA256; + break; + + case SE_ALG_HMAC_SHA384: + cfg |= SE_SHA_ENC_ALG_HMAC; + fallthrough; + case SE_ALG_SHA384: + cfg |= SE_SHA_ENC_ALG_SHA; + cfg |= SE_SHA_ENC_MODE_SHA384; + break; + + case SE_ALG_HMAC_SHA512: + cfg |= SE_SHA_ENC_ALG_HMAC; + fallthrough; + case SE_ALG_SHA512: + cfg |= SE_SHA_ENC_ALG_SHA; + cfg |= SE_SHA_ENC_MODE_SHA512; + break; + + case SE_ALG_SHA3_224: + cfg |= SE_SHA_ENC_ALG_SHA; + cfg |= SE_SHA_ENC_MODE_SHA3_224; + break; + case SE_ALG_SHA3_256: + cfg |= SE_SHA_ENC_ALG_SHA; + cfg |= SE_SHA_ENC_MODE_SHA3_256; + break; + case SE_ALG_SHA3_384: + cfg |= SE_SHA_ENC_ALG_SHA; + cfg |= SE_SHA_ENC_MODE_SHA3_384; + break; + case SE_ALG_SHA3_512: + cfg |= SE_SHA_ENC_ALG_SHA; + cfg |= SE_SHA_ENC_MODE_SHA3_512; + break; + default: + return -EINVAL; + } + + return cfg; +} + +static int tegra_sha_fallback_init(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + + return crypto_ahash_init(&rctx->fallback_req); +} + +static int tegra_sha_fallback_update(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, NULL, req->nbytes); + + return crypto_ahash_update(&rctx->fallback_req); +} + +static int tegra_sha_fallback_final(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, NULL, req->result, 0); + + return crypto_ahash_final(&rctx->fallback_req); +} + +static int tegra_sha_fallback_finup(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result, + req->nbytes); + + return crypto_ahash_finup(&rctx->fallback_req); +} + +static int tegra_sha_fallback_digest(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result, + req->nbytes); + + return crypto_ahash_digest(&rctx->fallback_req); +} + +static int tegra_sha_fallback_import(struct ahash_request *req, const void *in) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + + return crypto_ahash_import(&rctx->fallback_req, in); +} + +static int tegra_sha_fallback_export(struct ahash_request *req, void *out) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + ahash_request_set_callback(&rctx->fallback_req, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, + req->base.complete, req->base.data); + + return crypto_ahash_export(&rctx->fallback_req, out); +} + +static int tegra_se_insert_hash_result(struct tegra_sha_ctx *ctx, u32 *cpuvaddr, + struct tegra_sha_reqctx *rctx) +{ + __be32 *res_be = (__be32 *)rctx->intr_res.buf; + u32 *res = (u32 *)rctx->intr_res.buf; + int i = 0, j; + + cpuvaddr[i++] = 0; + cpuvaddr[i++] = host1x_opcode_setpayload(HASH_RESULT_REG_COUNT); + cpuvaddr[i++] = se_host1x_opcode_incr_w(SE_SHA_HASH_RESULT); + + for (j = 0; j < HASH_RESULT_REG_COUNT; j++) { + int idx = j; + + /* + * The initial, intermediate and final hash value of SHA-384, SHA-512 + * in SHA_HASH_RESULT registers follow the below layout of bytes. + * + * +---------------+------------+ + * | HASH_RESULT_0 | B4...B7 | + * +---------------+------------+ + * | HASH_RESULT_1 | B0...B3 | + * +---------------+------------+ + * | HASH_RESULT_2 | B12...B15 | + * +---------------+------------+ + * | HASH_RESULT_3 | B8...B11 | + * +---------------+------------+ + * | ...... | + * +---------------+------------+ + * | HASH_RESULT_14| B60...B63 | + * +---------------+------------+ + * | HASH_RESULT_15| B56...B59 | + * +---------------+------------+ + * + */ + if (ctx->alg == SE_ALG_SHA384 || ctx->alg == SE_ALG_SHA512) + idx = (j % 2) ? j - 1 : j + 1; + + /* For SHA-1, SHA-224, SHA-256, SHA-384, SHA-512 the initial + * intermediate and final hash value when stored in + * SHA_HASH_RESULT registers, the byte order is NOT in + * little-endian. + */ + if (ctx->alg <= SE_ALG_SHA512) + cpuvaddr[i++] = be32_to_cpu(res_be[idx]); + else + cpuvaddr[i++] = res[idx]; + } + + return i; +} + +static int tegra_sha_prep_cmd(struct tegra_sha_ctx *ctx, u32 *cpuvaddr, + struct tegra_sha_reqctx *rctx) +{ + struct tegra_se *se = ctx->se; + u64 msg_len, msg_left; + int i = 0; + + msg_len = rctx->total_len * 8; + msg_left = rctx->datbuf.size * 8; + + /* + * If IN_ADDR_HI_0.SZ > SHA_MSG_LEFT_[0-3] to the HASH engine, + * HW treats it as the last buffer and process the data. + * Therefore, add an extra byte to msg_left if it is not the + * last buffer. + */ + if (rctx->task & SHA_UPDATE) { + msg_left += 8; + msg_len += 8; + } + + cpuvaddr[i++] = host1x_opcode_setpayload(8); + cpuvaddr[i++] = se_host1x_opcode_incr_w(SE_SHA_MSG_LENGTH); + cpuvaddr[i++] = lower_32_bits(msg_len); + cpuvaddr[i++] = upper_32_bits(msg_len); + cpuvaddr[i++] = 0; + cpuvaddr[i++] = 0; + cpuvaddr[i++] = lower_32_bits(msg_left); + cpuvaddr[i++] = upper_32_bits(msg_left); + cpuvaddr[i++] = 0; + cpuvaddr[i++] = 0; + cpuvaddr[i++] = host1x_opcode_setpayload(2); + cpuvaddr[i++] = se_host1x_opcode_incr_w(SE_SHA_CFG); + cpuvaddr[i++] = rctx->config; + + if (rctx->task & SHA_FIRST) { + cpuvaddr[i++] = SE_SHA_TASK_HASH_INIT; + rctx->task &= ~SHA_FIRST; + } else { + /* + * If it isn't the first task, program the HASH_RESULT register + * with the intermediate result from the previous task + */ + i += tegra_se_insert_hash_result(ctx, cpuvaddr + i, rctx); + } + + cpuvaddr[i++] = host1x_opcode_setpayload(4); + cpuvaddr[i++] = se_host1x_opcode_incr_w(SE_SHA_IN_ADDR); + cpuvaddr[i++] = rctx->datbuf.addr; + cpuvaddr[i++] = (u32)(SE_ADDR_HI_MSB(upper_32_bits(rctx->datbuf.addr)) | + SE_ADDR_HI_SZ(rctx->datbuf.size)); + + if (rctx->task & SHA_UPDATE) { + cpuvaddr[i++] = rctx->intr_res.addr; + cpuvaddr[i++] = (u32)(SE_ADDR_HI_MSB(upper_32_bits(rctx->intr_res.addr)) | + SE_ADDR_HI_SZ(rctx->intr_res.size)); + } else { + cpuvaddr[i++] = rctx->digest.addr; + cpuvaddr[i++] = (u32)(SE_ADDR_HI_MSB(upper_32_bits(rctx->digest.addr)) | + SE_ADDR_HI_SZ(rctx->digest.size)); + } + + if (rctx->key_id) { + cpuvaddr[i++] = host1x_opcode_setpayload(1); + cpuvaddr[i++] = se_host1x_opcode_nonincr_w(SE_SHA_CRYPTO_CFG); + cpuvaddr[i++] = SE_AES_KEY_INDEX(rctx->key_id); + } + + cpuvaddr[i++] = host1x_opcode_setpayload(1); + cpuvaddr[i++] = se_host1x_opcode_nonincr_w(SE_SHA_OPERATION); + cpuvaddr[i++] = SE_SHA_OP_WRSTALL | SE_SHA_OP_START | + SE_SHA_OP_LASTBUF; + cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); + cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | + host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); + + dev_dbg(se->dev, "msg len %llu msg left %llu sz %zd cfg %#x", + msg_len, msg_left, rctx->datbuf.size, rctx->config); + + return i; +} + +static int tegra_sha_do_init(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_se *se = ctx->se; + + if (ctx->fallback) + return tegra_sha_fallback_init(req); + + rctx->total_len = 0; + rctx->datbuf.size = 0; + rctx->residue.size = 0; + rctx->key_id = ctx->key_id; + rctx->task |= SHA_FIRST; + rctx->alg = ctx->alg; + rctx->blk_size = crypto_ahash_blocksize(tfm); + rctx->digest.size = crypto_ahash_digestsize(tfm); + + rctx->digest.buf = dma_alloc_coherent(se->dev, rctx->digest.size, + &rctx->digest.addr, GFP_KERNEL); + if (!rctx->digest.buf) + goto digbuf_fail; + + rctx->residue.buf = dma_alloc_coherent(se->dev, rctx->blk_size, + &rctx->residue.addr, GFP_KERNEL); + if (!rctx->residue.buf) + goto resbuf_fail; + + rctx->intr_res.size = HASH_RESULT_REG_COUNT * 4; + rctx->intr_res.buf = dma_alloc_coherent(se->dev, rctx->intr_res.size, + &rctx->intr_res.addr, GFP_KERNEL); + if (!rctx->intr_res.buf) + goto intr_res_fail; + + return 0; + +intr_res_fail: + dma_free_coherent(se->dev, rctx->residue.size, rctx->residue.buf, + rctx->residue.addr); +resbuf_fail: + dma_free_coherent(se->dev, rctx->digest.size, rctx->digest.buf, + rctx->digest.addr); +digbuf_fail: + return -ENOMEM; +} + +static int tegra_sha_do_update(struct ahash_request *req) +{ + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct tegra_se *se = ctx->se; + unsigned int nblks, nresidue, size; + u32 *cpuvaddr = se->cmdbuf->addr; + int ret; + + nresidue = (req->nbytes + rctx->residue.size) % rctx->blk_size; + nblks = (req->nbytes + rctx->residue.size) / rctx->blk_size; + + /* + * If nbytes is a multiple of block size and there is no residue, + * then reserve the last block as residue during final() to process. + */ + if (!nresidue && nblks) { + nresidue = rctx->blk_size; + nblks--; + } + + rctx->src_sg = req->src; + rctx->datbuf.size = (req->nbytes + rctx->residue.size) - nresidue; + + /* + * If nbytes are less than a block size, copy it residue and + * return. The bytes will be processed in final() + */ + if (nblks < 1) { + scatterwalk_map_and_copy(rctx->residue.buf + rctx->residue.size, + rctx->src_sg, 0, req->nbytes, 0); + rctx->residue.size += req->nbytes; + + return 0; + } + + rctx->datbuf.buf = dma_alloc_coherent(se->dev, rctx->datbuf.size, + &rctx->datbuf.addr, GFP_KERNEL); + if (!rctx->datbuf.buf) + return -ENOMEM; + + /* Copy the previous residue first */ + if (rctx->residue.size) + memcpy(rctx->datbuf.buf, rctx->residue.buf, rctx->residue.size); + + scatterwalk_map_and_copy(rctx->datbuf.buf + rctx->residue.size, + rctx->src_sg, 0, req->nbytes - nresidue, 0); + + scatterwalk_map_and_copy(rctx->residue.buf, rctx->src_sg, + req->nbytes - nresidue, nresidue, 0); + + /* Update residue value with the residue after current block */ + rctx->residue.size = nresidue; + rctx->total_len += rctx->datbuf.size; + + rctx->config = tegra_sha_get_config(rctx->alg) | + SE_SHA_DST_MEMORY; + + size = tegra_sha_prep_cmd(ctx, cpuvaddr, rctx); + ret = tegra_se_host1x_submit(se, se->cmdbuf, size); + + dma_free_coherent(se->dev, rctx->datbuf.size, + rctx->datbuf.buf, rctx->datbuf.addr); + + return ret; +} + +static int tegra_sha_do_final(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_se *se = ctx->se; + u32 *cpuvaddr = se->cmdbuf->addr; + int size, ret = 0; + + if (rctx->residue.size) { + rctx->datbuf.buf = dma_alloc_coherent(se->dev, rctx->residue.size, + &rctx->datbuf.addr, GFP_KERNEL); + if (!rctx->datbuf.buf) { + ret = -ENOMEM; + goto out_free; + } + + memcpy(rctx->datbuf.buf, rctx->residue.buf, rctx->residue.size); + } + + rctx->datbuf.size = rctx->residue.size; + rctx->total_len += rctx->residue.size; + + rctx->config = tegra_sha_get_config(rctx->alg) | + SE_SHA_DST_MEMORY; + + size = tegra_sha_prep_cmd(ctx, cpuvaddr, rctx); + ret = tegra_se_host1x_submit(se, se->cmdbuf, size); + if (ret) + goto out; + + /* Copy result */ + memcpy(req->result, rctx->digest.buf, rctx->digest.size); + +out: + if (rctx->residue.size) + dma_free_coherent(se->dev, rctx->datbuf.size, + rctx->datbuf.buf, rctx->datbuf.addr); +out_free: + dma_free_coherent(se->dev, crypto_ahash_blocksize(tfm), + rctx->residue.buf, rctx->residue.addr); + dma_free_coherent(se->dev, rctx->digest.size, rctx->digest.buf, + rctx->digest.addr); + + dma_free_coherent(se->dev, rctx->intr_res.size, rctx->intr_res.buf, + rctx->intr_res.addr); + + return ret; +} + +static int tegra_sha_do_one_req(struct crypto_engine *engine, void *areq) +{ + struct ahash_request *req = ahash_request_cast(areq); + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_se *se = ctx->se; + int ret = 0; + + if (rctx->task & SHA_INIT) { + ret = tegra_sha_do_init(req); + if (ret) + goto out; + + rctx->task &= ~SHA_INIT; + } + + if (rctx->task & SHA_UPDATE) { + ret = tegra_sha_do_update(req); + if (ret) + goto out; + + rctx->task &= ~SHA_UPDATE; + } + + if (rctx->task & SHA_FINAL) { + ret = tegra_sha_do_final(req); + if (ret) + goto out; + + rctx->task &= ~SHA_FINAL; + } + +out: + local_bh_disable(); + crypto_finalize_hash_request(se->engine, req, ret); + local_bh_enable(); + + return 0; +} + +static void tegra_sha_init_fallback(struct crypto_ahash *tfm, struct tegra_sha_ctx *ctx, + const char *algname) +{ + unsigned int statesize; + + ctx->fallback_tfm = crypto_alloc_ahash(algname, 0, CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + + if (IS_ERR(ctx->fallback_tfm)) { + dev_warn(ctx->se->dev, + "failed to allocate fallback for %s\n", algname); + ctx->fallback_tfm = NULL; + return; + } + + statesize = crypto_ahash_statesize(ctx->fallback_tfm); + + if (statesize > sizeof(struct tegra_sha_reqctx)) + crypto_ahash_set_statesize(tfm, statesize); + + /* Update reqsize if fallback is added */ + crypto_ahash_set_reqsize(tfm, + sizeof(struct tegra_sha_reqctx) + + crypto_ahash_reqsize(ctx->fallback_tfm)); +} + +static int tegra_sha_cra_init(struct crypto_tfm *tfm) +{ + struct tegra_sha_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_ahash *ahash_tfm = __crypto_ahash_cast(tfm); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg); + struct tegra_se_alg *se_alg; + const char *algname; + int ret; + + algname = crypto_tfm_alg_name(tfm); + se_alg = container_of(alg, struct tegra_se_alg, alg.ahash.base); + + crypto_ahash_set_reqsize(ahash_tfm, sizeof(struct tegra_sha_reqctx)); + + ctx->se = se_alg->se_dev; + ctx->fallback = false; + ctx->key_id = 0; + + ret = se_algname_to_algid(algname); + if (ret < 0) { + dev_err(ctx->se->dev, "invalid algorithm\n"); + return ret; + } + + if (se_alg->alg_base) + tegra_sha_init_fallback(ahash_tfm, ctx, algname); + + ctx->alg = ret; + + return 0; +} + +static void tegra_sha_cra_exit(struct crypto_tfm *tfm) +{ + struct tegra_sha_ctx *ctx = crypto_tfm_ctx(tfm); + + if (ctx->fallback_tfm) + crypto_free_ahash(ctx->fallback_tfm); + + tegra_key_invalidate(ctx->se, ctx->key_id, ctx->alg); +} + +static int tegra_hmac_fallback_setkey(struct tegra_sha_ctx *ctx, const u8 *key, + unsigned int keylen) +{ + if (!ctx->fallback_tfm) { + dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen); + return -EINVAL; + } + + ctx->fallback = true; + return crypto_ahash_setkey(ctx->fallback_tfm, key, keylen); +} + +static int tegra_hmac_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + int ret; + + if (aes_check_keylen(keylen)) + return tegra_hmac_fallback_setkey(ctx, key, keylen); + + ret = tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key_id); + if (ret) + return tegra_hmac_fallback_setkey(ctx, key, keylen); + + ctx->fallback = false; + + return 0; +} + +static int tegra_sha_init(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + rctx->task = SHA_INIT; + + return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); +} + +static int tegra_sha_update(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + if (ctx->fallback) + return tegra_sha_fallback_update(req); + + rctx->task |= SHA_UPDATE; + + return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); +} + +static int tegra_sha_final(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + if (ctx->fallback) + return tegra_sha_fallback_final(req); + + rctx->task |= SHA_FINAL; + + return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); +} + +static int tegra_sha_finup(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + if (ctx->fallback) + return tegra_sha_fallback_finup(req); + + rctx->task |= SHA_UPDATE | SHA_FINAL; + + return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); +} + +static int tegra_sha_digest(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + if (ctx->fallback) + return tegra_sha_fallback_digest(req); + + rctx->task |= SHA_INIT | SHA_UPDATE | SHA_FINAL; + + return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); +} + +static int tegra_sha_export(struct ahash_request *req, void *out) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + if (ctx->fallback) + return tegra_sha_fallback_export(req, out); + + memcpy(out, rctx, sizeof(*rctx)); + + return 0; +} + +static int tegra_sha_import(struct ahash_request *req, const void *in) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + if (ctx->fallback) + return tegra_sha_fallback_import(req, in); + + memcpy(rctx, in, sizeof(*rctx)); + + return 0; +} + +static struct tegra_se_alg tegra_hash_algs[] = { + { + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "sha1", + .cra_driver_name = "tegra-se-sha1", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + }, { + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .halg.digestsize = SHA224_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "sha224", + .cra_driver_name = "tegra-se-sha224", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + }, { + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "sha256", + .cra_driver_name = "tegra-se-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + }, { + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .halg.digestsize = SHA384_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "sha384", + .cra_driver_name = "tegra-se-sha384", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + }, { + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .halg.digestsize = SHA512_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "sha512", + .cra_driver_name = "tegra-se-sha512", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + }, { + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .halg.digestsize = SHA3_224_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "sha3-224", + .cra_driver_name = "tegra-se-sha3-224", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA3_224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + }, { + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .halg.digestsize = SHA3_256_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "sha3-256", + .cra_driver_name = "tegra-se-sha3-256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA3_256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + }, { + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .halg.digestsize = SHA3_384_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "sha3-384", + .cra_driver_name = "tegra-se-sha3-384", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA3_384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + }, { + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .halg.digestsize = SHA3_512_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "sha3-512", + .cra_driver_name = "tegra-se-sha3-512", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA3_512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + }, { + .alg_base = "sha224", + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .setkey = tegra_hmac_setkey, + .halg.digestsize = SHA224_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "hmac(sha224)", + .cra_driver_name = "tegra-se-hmac-sha224", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + }, { + .alg_base = "sha256", + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .setkey = tegra_hmac_setkey, + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "tegra-se-hmac-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + }, { + .alg_base = "sha384", + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .setkey = tegra_hmac_setkey, + .halg.digestsize = SHA384_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "hmac(sha384)", + .cra_driver_name = "tegra-se-hmac-sha384", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + }, { + .alg_base = "sha512", + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .setkey = tegra_hmac_setkey, + .halg.digestsize = SHA512_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "hmac(sha512)", + .cra_driver_name = "tegra-se-hmac-sha512", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + } +}; + +static int tegra_hash_kac_manifest(u32 user, u32 alg, u32 keylen) +{ + int manifest; + + manifest = SE_KAC_USER_NS; + + switch (alg) { + case SE_ALG_HMAC_SHA224: + case SE_ALG_HMAC_SHA256: + case SE_ALG_HMAC_SHA384: + case SE_ALG_HMAC_SHA512: + manifest |= SE_KAC_HMAC; + break; + default: + return -EINVAL; + } + + switch (keylen) { + case AES_KEYSIZE_128: + manifest |= SE_KAC_SIZE_128; + break; + case AES_KEYSIZE_192: + manifest |= SE_KAC_SIZE_192; + break; + case AES_KEYSIZE_256: + default: + manifest |= SE_KAC_SIZE_256; + break; + } + + return manifest; +} + +int tegra_init_hash(struct tegra_se *se) +{ + struct ahash_engine_alg *alg; + int i, ret; + + se->manifest = tegra_hash_kac_manifest; + + for (i = 0; i < ARRAY_SIZE(tegra_hash_algs); i++) { + tegra_hash_algs[i].se_dev = se; + alg = &tegra_hash_algs[i].alg.ahash; + + ret = crypto_engine_register_ahash(alg); + if (ret) { + dev_err(se->dev, "failed to register %s\n", + alg->base.halg.base.cra_name); + goto sha_err; + } + } + + return 0; + +sha_err: + while (i--) + crypto_engine_unregister_ahash(&tegra_hash_algs[i].alg.ahash); + + return ret; +} + +void tegra_deinit_hash(struct tegra_se *se) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(tegra_hash_algs); i++) + crypto_engine_unregister_ahash(&tegra_hash_algs[i].alg.ahash); +} diff --git a/drivers/crypto/tegra/tegra-se-key.c b/drivers/crypto/tegra/tegra-se-key.c new file mode 100644 index 000000000..956fa9b4e --- /dev/null +++ b/drivers/crypto/tegra/tegra-se-key.c @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0-only +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +/* + * Crypto driver file to manage keys of NVIDIA Security Engine. + */ + +#include +#include +#include + +#include "tegra-se.h" + +#define SE_KEY_FULL_MASK GENMASK(SE_MAX_KEYSLOT, 0) + +/* Reserve keyslot 0, 14, 15 */ +#define SE_KEY_RSVD_MASK (BIT(0) | BIT(14) | BIT(15)) +#define SE_KEY_VALID_MASK (SE_KEY_FULL_MASK & ~SE_KEY_RSVD_MASK) + +/* Mutex lock to guard keyslots */ +static DEFINE_MUTEX(kslt_lock); + +/* Keyslot bitmask (0 = available, 1 = in use/not available) */ +static u16 tegra_se_keyslots = SE_KEY_RSVD_MASK; + +static u16 tegra_keyslot_alloc(void) +{ + u16 keyid; + + mutex_lock(&kslt_lock); + /* Check if all key slots are full */ + if (tegra_se_keyslots == GENMASK(SE_MAX_KEYSLOT, 0)) { + mutex_unlock(&kslt_lock); + return 0; + } + + keyid = ffz(tegra_se_keyslots); + tegra_se_keyslots |= BIT(keyid); + + mutex_unlock(&kslt_lock); + + return keyid; +} + +static void tegra_keyslot_free(u16 slot) +{ + mutex_lock(&kslt_lock); + tegra_se_keyslots &= ~(BIT(slot)); + mutex_unlock(&kslt_lock); +} + +static unsigned int tegra_key_prep_ins_cmd(struct tegra_se *se, u32 *cpuvaddr, + const u32 *key, u32 keylen, u16 slot, u32 alg) +{ + int i = 0, j; + + cpuvaddr[i++] = host1x_opcode_setpayload(1); + cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->op); + cpuvaddr[i++] = SE_AES_OP_WRSTALL | SE_AES_OP_DUMMY; + + cpuvaddr[i++] = host1x_opcode_setpayload(1); + cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->manifest); + cpuvaddr[i++] = se->manifest(se->owner, alg, keylen); + cpuvaddr[i++] = host1x_opcode_setpayload(1); + cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->key_dst); + + cpuvaddr[i++] = SE_AES_KEY_DST_INDEX(slot); + + for (j = 0; j < keylen / 4; j++) { + /* Set key address */ + cpuvaddr[i++] = host1x_opcode_setpayload(1); + cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->key_addr); + cpuvaddr[i++] = j; + + /* Set key data */ + cpuvaddr[i++] = host1x_opcode_setpayload(1); + cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->key_data); + cpuvaddr[i++] = key[j]; + } + + cpuvaddr[i++] = host1x_opcode_setpayload(1); + cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->config); + cpuvaddr[i++] = SE_CFG_INS; + + cpuvaddr[i++] = host1x_opcode_setpayload(1); + cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->op); + cpuvaddr[i++] = SE_AES_OP_WRSTALL | SE_AES_OP_START | + SE_AES_OP_LASTBUF; + + cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); + cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | + host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); + + dev_dbg(se->dev, "key-slot %u key-manifest %#x\n", + slot, se->manifest(se->owner, alg, keylen)); + + return i; +} + +static bool tegra_key_in_kslt(u32 keyid) +{ + bool ret; + + if (keyid > SE_MAX_KEYSLOT) + return false; + + mutex_lock(&kslt_lock); + ret = ((BIT(keyid) & SE_KEY_VALID_MASK) && + (BIT(keyid) & tegra_se_keyslots)); + mutex_unlock(&kslt_lock); + + return ret; +} + +static int tegra_key_insert(struct tegra_se *se, const u8 *key, + u32 keylen, u16 slot, u32 alg) +{ + const u32 *keyval = (u32 *)key; + u32 *addr = se->keybuf->addr, size; + int ret; + + mutex_lock(&kslt_lock); + + size = tegra_key_prep_ins_cmd(se, addr, keyval, keylen, slot, alg); + ret = tegra_se_host1x_submit(se, se->keybuf, size); + + mutex_unlock(&kslt_lock); + + return ret; +} + +void tegra_key_invalidate(struct tegra_se *se, u32 keyid, u32 alg) +{ + u8 zkey[AES_MAX_KEY_SIZE] = {0}; + + if (!keyid) + return; + + /* Overwrite the key with 0s */ + tegra_key_insert(se, zkey, AES_MAX_KEY_SIZE, keyid, alg); + + tegra_keyslot_free(keyid); +} + +void tegra_key_invalidate_reserved(struct tegra_se *se, u32 keyid, u32 alg) +{ + u8 zkey[AES_MAX_KEY_SIZE] = {0}; + + if (!keyid) + return; + + /* Overwrite the key with 0s */ + tegra_key_insert(se, zkey, AES_MAX_KEY_SIZE, keyid, alg); +} + +inline int tegra_key_submit_reserved(struct tegra_se *se, const u8 *key, + u32 keylen, u32 alg, u32 *keyid) +{ + return tegra_key_insert(se, key, keylen, *keyid, alg); +} + +int tegra_key_submit(struct tegra_se *se, const u8 *key, u32 keylen, u32 alg, u32 *keyid) +{ + int ret; + + /* Use the existing slot if it is already allocated */ + if (!tegra_key_in_kslt(*keyid)) { + *keyid = tegra_keyslot_alloc(); + if (!(*keyid)) { + dev_dbg(se->dev, "failed to allocate key slot\n"); + return -ENOMEM; + } + } + + ret = tegra_key_insert(se, key, keylen, *keyid, alg); + if (ret) + return ret; + + return 0; +} diff --git a/drivers/crypto/tegra/tegra-se-main.c b/drivers/crypto/tegra/tegra-se-main.c new file mode 100644 index 000000000..497ff2704 --- /dev/null +++ b/drivers/crypto/tegra/tegra-se-main.c @@ -0,0 +1,438 @@ +// SPDX-License-Identifier: GPL-2.0-only +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +/* + * Crypto driver for NVIDIA Security Engine in Tegra Chips + */ + +#include +#include +#include +#include + +#include + +#include "tegra-se.h" + +static struct host1x_bo *tegra_se_cmdbuf_get(struct host1x_bo *host_bo) +{ + struct tegra_se_cmdbuf *cmdbuf = container_of(host_bo, struct tegra_se_cmdbuf, bo); + + kref_get(&cmdbuf->ref); + + return host_bo; +} + +static void tegra_se_cmdbuf_release(struct kref *ref) +{ + struct tegra_se_cmdbuf *cmdbuf = container_of(ref, struct tegra_se_cmdbuf, ref); + + dma_free_attrs(cmdbuf->dev, cmdbuf->size, cmdbuf->addr, + cmdbuf->iova, 0); + + kfree(cmdbuf); +} + +static void tegra_se_cmdbuf_put(struct host1x_bo *host_bo) +{ + struct tegra_se_cmdbuf *cmdbuf = container_of(host_bo, struct tegra_se_cmdbuf, bo); + + kref_put(&cmdbuf->ref, tegra_se_cmdbuf_release); +} + +static struct host1x_bo_mapping * +tegra_se_cmdbuf_pin(struct device *dev, struct host1x_bo *bo, enum dma_data_direction direction) +{ + struct tegra_se_cmdbuf *cmdbuf = container_of(bo, struct tegra_se_cmdbuf, bo); + struct host1x_bo_mapping *map; + int err; + + map = kzalloc_obj(*map); + if (!map) + return ERR_PTR(-ENOMEM); + + kref_init(&map->ref); + map->bo = bo; + map->direction = direction; + map->dev = dev; + + map->sgt = kzalloc_obj(*map->sgt); + if (!map->sgt) { + err = -ENOMEM; + goto free; + } + + err = dma_get_sgtable(dev, map->sgt, cmdbuf->addr, + cmdbuf->iova, cmdbuf->words * 4); + if (err) + goto free_sgt; + + err = dma_map_sgtable(dev, map->sgt, direction, 0); + if (err) + goto free_sgt; + + map->phys = sg_dma_address(map->sgt->sgl); + map->size = cmdbuf->words * 4; + map->chunks = err; + + return map; + +free_sgt: + sg_free_table(map->sgt); + kfree(map->sgt); +free: + kfree(map); + return ERR_PTR(err); +} + +static void tegra_se_cmdbuf_unpin(struct host1x_bo_mapping *map) +{ + if (!map) + return; + + dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0); + sg_free_table(map->sgt); + kfree(map->sgt); + + kfree(map); +} + +static void *tegra_se_cmdbuf_mmap(struct host1x_bo *host_bo) +{ + struct tegra_se_cmdbuf *cmdbuf = container_of(host_bo, struct tegra_se_cmdbuf, bo); + + return cmdbuf->addr; +} + +static void tegra_se_cmdbuf_munmap(struct host1x_bo *host_bo, void *addr) +{ +} + +static const struct host1x_bo_ops tegra_se_cmdbuf_ops = { + .get = tegra_se_cmdbuf_get, + .put = tegra_se_cmdbuf_put, + .pin = tegra_se_cmdbuf_pin, + .unpin = tegra_se_cmdbuf_unpin, + .mmap = tegra_se_cmdbuf_mmap, + .munmap = tegra_se_cmdbuf_munmap, +}; + +static struct tegra_se_cmdbuf *tegra_se_host1x_bo_alloc(struct tegra_se *se, ssize_t size) +{ + struct tegra_se_cmdbuf *cmdbuf; + struct device *dev = se->dev->parent; + + cmdbuf = kzalloc_obj(*cmdbuf); + if (!cmdbuf) + return NULL; + + cmdbuf->addr = dma_alloc_attrs(dev, size, &cmdbuf->iova, + GFP_KERNEL, 0); + if (!cmdbuf->addr) + return NULL; + + cmdbuf->size = size; + cmdbuf->dev = dev; + + host1x_bo_init(&cmdbuf->bo, &tegra_se_cmdbuf_ops); + kref_init(&cmdbuf->ref); + + return cmdbuf; +} + +int tegra_se_host1x_submit(struct tegra_se *se, struct tegra_se_cmdbuf *cmdbuf, u32 size) +{ + struct host1x_job *job; + int ret; + + job = host1x_job_alloc(se->channel, 1, 0, true); + if (!job) { + dev_err(se->dev, "failed to allocate host1x job\n"); + return -ENOMEM; + } + + job->syncpt = host1x_syncpt_get(se->syncpt); + job->syncpt_incrs = 1; + job->client = &se->client; + job->class = se->client.class; + job->serialize = true; + job->engine_fallback_streamid = se->stream_id; + job->engine_streamid_offset = SE_STREAM_ID; + + cmdbuf->words = size; + + host1x_job_add_gather(job, &cmdbuf->bo, size, 0); + + ret = host1x_job_pin(job, se->dev); + if (ret) { + dev_err(se->dev, "failed to pin host1x job\n"); + goto job_put; + } + + ret = host1x_job_submit(job); + if (ret) { + dev_err(se->dev, "failed to submit host1x job\n"); + goto job_unpin; + } + + ret = host1x_syncpt_wait(job->syncpt, job->syncpt_end, + MAX_SCHEDULE_TIMEOUT, NULL); + if (ret) { + dev_err(se->dev, "host1x job timed out\n"); + goto job_put; + } + + host1x_job_put(job); + return 0; + +job_unpin: + host1x_job_unpin(job); +job_put: + host1x_job_put(job); + + return ret; +} + +static int tegra_se_client_init(struct host1x_client *client) +{ + struct tegra_se *se = container_of(client, struct tegra_se, client); + int ret; + + se->channel = host1x_channel_request(&se->client); + if (!se->channel) { + dev_err(se->dev, "host1x channel map failed\n"); + return -ENODEV; + } + + se->syncpt = host1x_syncpt_request(&se->client, 0); + if (!se->syncpt) { + dev_err(se->dev, "host1x syncpt allocation failed\n"); + ret = -EINVAL; + goto channel_put; + } + + se->syncpt_id = host1x_syncpt_id(se->syncpt); + + se->cmdbuf = tegra_se_host1x_bo_alloc(se, SZ_4K); + if (!se->cmdbuf) { + ret = -ENOMEM; + goto syncpt_put; + } + + se->keybuf = tegra_se_host1x_bo_alloc(se, SZ_4K); + if (!se->keybuf) { + ret = -ENOMEM; + goto cmdbuf_put; + } + + ret = se->hw->init_alg(se); + if (ret) { + dev_err(se->dev, "failed to register algorithms\n"); + goto keybuf_put; + } + + return 0; + +keybuf_put: + tegra_se_cmdbuf_put(&se->keybuf->bo); +cmdbuf_put: + tegra_se_cmdbuf_put(&se->cmdbuf->bo); +syncpt_put: + host1x_syncpt_put(se->syncpt); +channel_put: + host1x_channel_put(se->channel); + + return ret; +} + +static int tegra_se_client_deinit(struct host1x_client *client) +{ + struct tegra_se *se = container_of(client, struct tegra_se, client); + + se->hw->deinit_alg(se); + tegra_se_cmdbuf_put(&se->cmdbuf->bo); + host1x_syncpt_put(se->syncpt); + host1x_channel_put(se->channel); + + return 0; +} + +static const struct host1x_client_ops tegra_se_client_ops = { + .init = tegra_se_client_init, + .exit = tegra_se_client_deinit, +}; + +static int tegra_se_host1x_register(struct tegra_se *se) +{ + INIT_LIST_HEAD(&se->client.list); + se->client.dev = se->dev; + se->client.ops = &tegra_se_client_ops; + se->client.class = se->hw->host1x_class; + se->client.num_syncpts = 1; + + host1x_client_register(&se->client); + + return 0; +} + +static int tegra_se_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct tegra_se *se; + int ret; + + se = devm_kzalloc(dev, sizeof(*se), GFP_KERNEL); + if (!se) + return -ENOMEM; + + se->dev = dev; + se->owner = TEGRA_GPSE_ID; + se->hw = device_get_match_data(&pdev->dev); + + se->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(se->base)) + return PTR_ERR(se->base); + + dma_set_mask_and_coherent(dev, DMA_BIT_MASK(39)); + platform_set_drvdata(pdev, se); + + se->clk = devm_clk_get_enabled(se->dev, NULL); + if (IS_ERR(se->clk)) + return dev_err_probe(dev, PTR_ERR(se->clk), + "failed to enable clocks\n"); + + if (!tegra_dev_iommu_get_stream_id(dev, &se->stream_id)) + return dev_err_probe(dev, -ENODEV, + "failed to get IOMMU stream ID\n"); + + writel(se->stream_id, se->base + SE_STREAM_ID); + + se->engine = crypto_engine_alloc_init(dev, 0); + if (!se->engine) + return -ENOMEM; + + ret = crypto_engine_start(se->engine); + if (ret) { + crypto_engine_exit(se->engine); + return dev_err_probe(dev, ret, "failed to start crypto engine\n"); + } + + ret = tegra_se_host1x_register(se); + if (ret) { + crypto_engine_exit(se->engine); + return dev_err_probe(dev, ret, "failed to init host1x params\n"); + } + + return 0; +} + +static void tegra_se_remove(struct platform_device *pdev) +{ + struct tegra_se *se = platform_get_drvdata(pdev); + + crypto_engine_exit(se->engine); + host1x_client_unregister(&se->client); +} + +static const struct tegra_se_regs tegra234_aes1_regs = { + .config = SE_AES1_CFG, + .op = SE_AES1_OPERATION, + .last_blk = SE_AES1_LAST_BLOCK, + .linear_ctr = SE_AES1_LINEAR_CTR, + .aad_len = SE_AES1_AAD_LEN, + .cryp_msg_len = SE_AES1_CRYPTO_MSG_LEN, + .manifest = SE_AES1_KEYMANIFEST, + .key_addr = SE_AES1_KEY_ADDR, + .key_data = SE_AES1_KEY_DATA, + .key_dst = SE_AES1_KEY_DST, + .result = SE_AES1_CMAC_RESULT, +}; + +static const struct tegra_se_regs tegra234_hash_regs = { + .config = SE_SHA_CFG, + .op = SE_SHA_OPERATION, + .manifest = SE_SHA_KEYMANIFEST, + .key_addr = SE_SHA_KEY_ADDR, + .key_data = SE_SHA_KEY_DATA, + .key_dst = SE_SHA_KEY_DST, + .result = SE_SHA_HASH_RESULT, +}; + +static const struct tegra_se_hw tegra234_aes_hw = { + .regs = &tegra234_aes1_regs, + .kac_ver = 1, + .host1x_class = 0x3b, + .init_alg = tegra_init_aes, + .deinit_alg = tegra_deinit_aes, +}; + +static const struct tegra_se_hw tegra234_hash_hw = { + .regs = &tegra234_hash_regs, + .kac_ver = 1, + .host1x_class = 0x3d, + .init_alg = tegra_init_hash, + .deinit_alg = tegra_deinit_hash, +}; + +static const struct of_device_id tegra_se_of_match[] = { + { + .compatible = "nvidia,tegra234-se-aes", + .data = &tegra234_aes_hw + }, { + .compatible = "nvidia,tegra234-se-hash", + .data = &tegra234_hash_hw, + }, + { }, +}; +MODULE_DEVICE_TABLE(of, tegra_se_of_match); + +static struct platform_driver tegra_se_driver = { + .driver = { + .name = "tegra-se", + .of_match_table = tegra_se_of_match, + }, + .probe = tegra_se_probe, + .remove = tegra_se_remove, +}; + +static int tegra_se_host1x_probe(struct host1x_device *dev) +{ + return host1x_device_init(dev); +} + +static void tegra_se_host1x_remove(struct host1x_device *dev) +{ + host1x_device_exit(dev); +} + +static struct host1x_driver tegra_se_host1x_driver = { + .driver = { + .name = "tegra-se-host1x", + }, + .probe = tegra_se_host1x_probe, + .remove = tegra_se_host1x_remove, + .subdevs = tegra_se_of_match, +}; + +static int __init tegra_se_module_init(void) +{ + int ret; + + ret = host1x_driver_register(&tegra_se_host1x_driver); + if (ret) + return ret; + + return platform_driver_register(&tegra_se_driver); +} + +static void __exit tegra_se_module_exit(void) +{ + host1x_driver_unregister(&tegra_se_host1x_driver); + platform_driver_unregister(&tegra_se_driver); +} + +module_init(tegra_se_module_init); +module_exit(tegra_se_module_exit); + +MODULE_DESCRIPTION("NVIDIA Tegra Security Engine Driver"); +MODULE_AUTHOR("Akhil R "); +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/tegra/tegra-se.h b/drivers/crypto/tegra/tegra-se.h new file mode 100644 index 000000000..b6cac9384 --- /dev/null +++ b/drivers/crypto/tegra/tegra-se.h @@ -0,0 +1,589 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * Header file for NVIDIA Security Engine driver. + */ + +#ifndef _TEGRA_SE_H +#define _TEGRA_SE_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SE_OWNERSHIP 0x14 +#define SE_OWNERSHIP_UID(x) FIELD_GET(GENMASK(7, 0), x) +#define TEGRA_GPSE_ID 3 + +#define SE_STREAM_ID 0x90 + +#define SE_SHA_CFG 0x4004 +#define SE_SHA_IN_ADDR 0x400c +#define SE_SHA_KEY_ADDR 0x4094 +#define SE_SHA_KEY_DATA 0x4098 +#define SE_SHA_KEYMANIFEST 0x409c +#define SE_SHA_CRYPTO_CFG 0x40a4 +#define SE_SHA_KEY_DST 0x40a8 +#define SE_SHA_SRC_KSLT 0x4180 +#define SE_SHA_TGT_KSLT 0x4184 +#define SE_SHA_MSG_LENGTH 0x401c +#define SE_SHA_OPERATION 0x407c +#define SE_SHA_HASH_RESULT 0x40b0 + +#define SE_SHA_ENC_MODE(x) FIELD_PREP(GENMASK(31, 24), x) +#define SE_SHA_ENC_MODE_SHA1 SE_SHA_ENC_MODE(0) +#define SE_SHA_ENC_MODE_SHA224 SE_SHA_ENC_MODE(4) +#define SE_SHA_ENC_MODE_SHA256 SE_SHA_ENC_MODE(5) +#define SE_SHA_ENC_MODE_SHA384 SE_SHA_ENC_MODE(6) +#define SE_SHA_ENC_MODE_SHA512 SE_SHA_ENC_MODE(7) +#define SE_SHA_ENC_MODE_SHA_CTX_INTEGRITY SE_SHA_ENC_MODE(8) +#define SE_SHA_ENC_MODE_SHA3_224 SE_SHA_ENC_MODE(9) +#define SE_SHA_ENC_MODE_SHA3_256 SE_SHA_ENC_MODE(10) +#define SE_SHA_ENC_MODE_SHA3_384 SE_SHA_ENC_MODE(11) +#define SE_SHA_ENC_MODE_SHA3_512 SE_SHA_ENC_MODE(12) +#define SE_SHA_ENC_MODE_SHAKE128 SE_SHA_ENC_MODE(13) +#define SE_SHA_ENC_MODE_SHAKE256 SE_SHA_ENC_MODE(14) +#define SE_SHA_ENC_MODE_HMAC_SHA256_1KEY SE_SHA_ENC_MODE(0) +#define SE_SHA_ENC_MODE_HMAC_SHA256_2KEY SE_SHA_ENC_MODE(1) +#define SE_SHA_ENC_MODE_SM3_256 SE_SHA_ENC_MODE(0) + +#define SE_SHA_CFG_ENC_ALG(x) FIELD_PREP(GENMASK(15, 12), x) +#define SE_SHA_ENC_ALG_NOP SE_SHA_CFG_ENC_ALG(0) +#define SE_SHA_ENC_ALG_SHA_ENC SE_SHA_CFG_ENC_ALG(1) +#define SE_SHA_ENC_ALG_RNG SE_SHA_CFG_ENC_ALG(2) +#define SE_SHA_ENC_ALG_SHA SE_SHA_CFG_ENC_ALG(3) +#define SE_SHA_ENC_ALG_SM3 SE_SHA_CFG_ENC_ALG(4) +#define SE_SHA_ENC_ALG_HMAC SE_SHA_CFG_ENC_ALG(7) +#define SE_SHA_ENC_ALG_KDF SE_SHA_CFG_ENC_ALG(8) +#define SE_SHA_ENC_ALG_KEY_INVLD SE_SHA_CFG_ENC_ALG(10) +#define SE_SHA_ENC_ALG_KEY_INQUIRE SE_SHA_CFG_ENC_ALG(12) +#define SE_SHA_ENC_ALG_INS SE_SHA_CFG_ENC_ALG(13) + +#define SE_SHA_OP_LASTBUF FIELD_PREP(BIT(16), 1) +#define SE_SHA_OP_WRSTALL FIELD_PREP(BIT(15), 1) + +#define SE_SHA_OP_OP(x) FIELD_PREP(GENMASK(2, 0), x) +#define SE_SHA_OP_START SE_SHA_OP_OP(1) +#define SE_SHA_OP_RESTART_OUT SE_SHA_OP_OP(2) +#define SE_SHA_OP_RESTART_IN SE_SHA_OP_OP(4) +#define SE_SHA_OP_RESTART_INOUT SE_SHA_OP_OP(5) +#define SE_SHA_OP_DUMMY SE_SHA_OP_OP(6) + +#define SE_SHA_CFG_DEC_ALG(x) FIELD_PREP(GENMASK(11, 8), x) +#define SE_SHA_DEC_ALG_NOP SE_SHA_CFG_DEC_ALG(0) +#define SE_SHA_DEC_ALG_AES_DEC SE_SHA_CFG_DEC_ALG(1) +#define SE_SHA_DEC_ALG_HMAC SE_SHA_CFG_DEC_ALG(7) +#define SE_SHA_DEC_ALG_HMAC_VERIFY SE_SHA_CFG_DEC_ALG(9) + +#define SE_SHA_CFG_DST(x) FIELD_PREP(GENMASK(4, 2), x) +#define SE_SHA_DST_MEMORY SE_SHA_CFG_DST(0) +#define SE_SHA_DST_HASH_REG SE_SHA_CFG_DST(1) +#define SE_SHA_DST_KEYTABLE SE_SHA_CFG_DST(2) +#define SE_SHA_DST_SRK SE_SHA_CFG_DST(3) + +#define SE_SHA_TASK_HASH_INIT BIT(0) + +/* AES Configuration */ +#define SE_AES0_CFG 0x1004 +#define SE_AES0_CRYPTO_CONFIG 0x1008 +#define SE_AES0_KEY_DST 0x1030 +#define SE_AES0_OPERATION 0x1038 +#define SE_AES0_LINEAR_CTR 0x101c +#define SE_AES0_LAST_BLOCK 0x102c +#define SE_AES0_KEY_ADDR 0x10bc +#define SE_AES0_KEY_DATA 0x10c0 +#define SE_AES0_CMAC_RESULT 0x10c4 +#define SE_AES0_SRC_KSLT 0x1100 +#define SE_AES0_TGT_KSLT 0x1104 +#define SE_AES0_KEYMANIFEST 0x1114 +#define SE_AES0_AAD_LEN 0x112c +#define SE_AES0_CRYPTO_MSG_LEN 0x1134 + +#define SE_AES1_CFG 0x2004 +#define SE_AES1_CRYPTO_CONFIG 0x2008 +#define SE_AES1_KEY_DST 0x2030 +#define SE_AES1_OPERATION 0x2038 +#define SE_AES1_LINEAR_CTR 0x201c +#define SE_AES1_LAST_BLOCK 0x202c +#define SE_AES1_KEY_ADDR 0x20bc +#define SE_AES1_KEY_DATA 0x20c0 +#define SE_AES1_CMAC_RESULT 0x20c4 +#define SE_AES1_SRC_KSLT 0x2100 +#define SE_AES1_TGT_KSLT 0x2104 +#define SE_AES1_KEYMANIFEST 0x2114 +#define SE_AES1_AAD_LEN 0x212c +#define SE_AES1_CRYPTO_MSG_LEN 0x2134 + +#define SE_AES_CFG_ENC_MODE(x) FIELD_PREP(GENMASK(31, 24), x) +#define SE_AES_ENC_MODE_GMAC SE_AES_CFG_ENC_MODE(3) +#define SE_AES_ENC_MODE_GCM SE_AES_CFG_ENC_MODE(4) +#define SE_AES_ENC_MODE_GCM_FINAL SE_AES_CFG_ENC_MODE(5) +#define SE_AES_ENC_MODE_CMAC SE_AES_CFG_ENC_MODE(7) +#define SE_AES_ENC_MODE_CBC_MAC SE_AES_CFG_ENC_MODE(12) + +#define SE_AES_CFG_DEC_MODE(x) FIELD_PREP(GENMASK(23, 16), x) +#define SE_AES_DEC_MODE_GMAC SE_AES_CFG_DEC_MODE(3) +#define SE_AES_DEC_MODE_GCM SE_AES_CFG_DEC_MODE(4) +#define SE_AES_DEC_MODE_GCM_FINAL SE_AES_CFG_DEC_MODE(5) +#define SE_AES_DEC_MODE_CBC_MAC SE_AES_CFG_DEC_MODE(12) + +#define SE_AES_CFG_ENC_ALG(x) FIELD_PREP(GENMASK(15, 12), x) +#define SE_AES_ENC_ALG_NOP SE_AES_CFG_ENC_ALG(0) +#define SE_AES_ENC_ALG_AES_ENC SE_AES_CFG_ENC_ALG(1) +#define SE_AES_ENC_ALG_RNG SE_AES_CFG_ENC_ALG(2) +#define SE_AES_ENC_ALG_SHA SE_AES_CFG_ENC_ALG(3) +#define SE_AES_ENC_ALG_HMAC SE_AES_CFG_ENC_ALG(7) +#define SE_AES_ENC_ALG_KDF SE_AES_CFG_ENC_ALG(8) +#define SE_AES_ENC_ALG_INS SE_AES_CFG_ENC_ALG(13) + +#define SE_AES_CFG_DEC_ALG(x) FIELD_PREP(GENMASK(11, 8), x) +#define SE_AES_DEC_ALG_NOP SE_AES_CFG_DEC_ALG(0) +#define SE_AES_DEC_ALG_AES_DEC SE_AES_CFG_DEC_ALG(1) + +#define SE_AES_CFG_DST(x) FIELD_PREP(GENMASK(4, 2), x) +#define SE_AES_DST_MEMORY SE_AES_CFG_DST(0) +#define SE_AES_DST_HASH_REG SE_AES_CFG_DST(1) +#define SE_AES_DST_KEYTABLE SE_AES_CFG_DST(2) +#define SE_AES_DST_SRK SE_AES_CFG_DST(3) + +/* AES Crypto Configuration */ +#define SE_AES_KEY2_INDEX(x) FIELD_PREP(GENMASK(31, 28), x) +#define SE_AES_KEY_INDEX(x) FIELD_PREP(GENMASK(27, 24), x) + +#define SE_AES_CRYPTO_CFG_SCC_DIS FIELD_PREP(BIT(20), 1) + +#define SE_AES_CRYPTO_CFG_CTR_CNTN(x) FIELD_PREP(GENMASK(18, 11), x) + +#define SE_AES_CRYPTO_CFG_IV_MODE(x) FIELD_PREP(BIT(10), x) +#define SE_AES_IV_MODE_SWIV SE_AES_CRYPTO_CFG_IV_MODE(0) +#define SE_AES_IV_MODE_HWIV SE_AES_CRYPTO_CFG_IV_MODE(1) + +#define SE_AES_CRYPTO_CFG_CORE_SEL(x) FIELD_PREP(BIT(9), x) +#define SE_AES_CORE_SEL_DECRYPT SE_AES_CRYPTO_CFG_CORE_SEL(0) +#define SE_AES_CORE_SEL_ENCRYPT SE_AES_CRYPTO_CFG_CORE_SEL(1) + +#define SE_AES_CRYPTO_CFG_IV_SEL(x) FIELD_PREP(GENMASK(8, 7), x) +#define SE_AES_IV_SEL_UPDATED SE_AES_CRYPTO_CFG_IV_SEL(1) +#define SE_AES_IV_SEL_REG SE_AES_CRYPTO_CFG_IV_SEL(2) +#define SE_AES_IV_SEL_RANDOM SE_AES_CRYPTO_CFG_IV_SEL(3) + +#define SE_AES_CRYPTO_CFG_VCTRAM_SEL(x) FIELD_PREP(GENMASK(6, 5), x) +#define SE_AES_VCTRAM_SEL_MEMORY SE_AES_CRYPTO_CFG_VCTRAM_SEL(0) +#define SE_AES_VCTRAM_SEL_TWEAK SE_AES_CRYPTO_CFG_VCTRAM_SEL(1) +#define SE_AES_VCTRAM_SEL_AESOUT SE_AES_CRYPTO_CFG_VCTRAM_SEL(2) +#define SE_AES_VCTRAM_SEL_PREV_MEM SE_AES_CRYPTO_CFG_VCTRAM_SEL(3) + +#define SE_AES_CRYPTO_CFG_INPUT_SEL(x) FIELD_PREP(GENMASK(4, 3), x) +#define SE_AES_INPUT_SEL_MEMORY SE_AES_CRYPTO_CFG_INPUT_SEL(0) +#define SE_AES_INPUT_SEL_RANDOM SE_AES_CRYPTO_CFG_INPUT_SEL(1) +#define SE_AES_INPUT_SEL_AESOUT SE_AES_CRYPTO_CFG_INPUT_SEL(2) +#define SE_AES_INPUT_SEL_LINEAR_CTR SE_AES_CRYPTO_CFG_INPUT_SEL(3) +#define SE_AES_INPUT_SEL_REG SE_AES_CRYPTO_CFG_INPUT_SEL(1) + +#define SE_AES_CRYPTO_CFG_XOR_POS(x) FIELD_PREP(GENMASK(2, 1), x) +#define SE_AES_XOR_POS_BYPASS SE_AES_CRYPTO_CFG_XOR_POS(0) +#define SE_AES_XOR_POS_BOTH SE_AES_CRYPTO_CFG_XOR_POS(1) +#define SE_AES_XOR_POS_TOP SE_AES_CRYPTO_CFG_XOR_POS(2) +#define SE_AES_XOR_POS_BOTTOM SE_AES_CRYPTO_CFG_XOR_POS(3) + +#define SE_AES_CRYPTO_CFG_HASH_EN(x) FIELD_PREP(BIT(0), x) +#define SE_AES_HASH_DISABLE SE_AES_CRYPTO_CFG_HASH_EN(0) +#define SE_AES_HASH_ENABLE SE_AES_CRYPTO_CFG_HASH_EN(1) + +#define SE_LAST_BLOCK_VAL(x) FIELD_PREP(GENMASK(19, 0), x) +#define SE_LAST_BLOCK_RES_BITS(x) FIELD_PREP(GENMASK(26, 20), x) + +#define SE_AES_OP_LASTBUF FIELD_PREP(BIT(16), 1) +#define SE_AES_OP_WRSTALL FIELD_PREP(BIT(15), 1) +#define SE_AES_OP_FINAL FIELD_PREP(BIT(5), 1) +#define SE_AES_OP_INIT FIELD_PREP(BIT(4), 1) + +#define SE_AES_OP_OP(x) FIELD_PREP(GENMASK(2, 0), x) +#define SE_AES_OP_START SE_AES_OP_OP(1) +#define SE_AES_OP_RESTART_OUT SE_AES_OP_OP(2) +#define SE_AES_OP_RESTART_IN SE_AES_OP_OP(4) +#define SE_AES_OP_RESTART_INOUT SE_AES_OP_OP(5) +#define SE_AES_OP_DUMMY SE_AES_OP_OP(6) + +#define SE_KAC_SIZE(x) FIELD_PREP(GENMASK(15, 14), x) +#define SE_KAC_SIZE_128 SE_KAC_SIZE(0) +#define SE_KAC_SIZE_192 SE_KAC_SIZE(1) +#define SE_KAC_SIZE_256 SE_KAC_SIZE(2) + +#define SE_KAC_EXPORTABLE FIELD_PREP(BIT(12), 1) + +#define SE_KAC_PURPOSE(x) FIELD_PREP(GENMASK(11, 8), x) +#define SE_KAC_ENC SE_KAC_PURPOSE(0) +#define SE_KAC_CMAC SE_KAC_PURPOSE(1) +#define SE_KAC_HMAC SE_KAC_PURPOSE(2) +#define SE_KAC_GCM_KW SE_KAC_PURPOSE(3) +#define SE_KAC_HMAC_KDK SE_KAC_PURPOSE(6) +#define SE_KAC_HMAC_KDD SE_KAC_PURPOSE(7) +#define SE_KAC_HMAC_KDD_KUW SE_KAC_PURPOSE(8) +#define SE_KAC_XTS SE_KAC_PURPOSE(9) +#define SE_KAC_GCM SE_KAC_PURPOSE(10) + +#define SE_KAC_USER_NS FIELD_PREP(GENMASK(6, 4), 3) + +#define SE_AES_KEY_DST_INDEX(x) FIELD_PREP(GENMASK(11, 8), x) +#define SE_ADDR_HI_MSB(x) FIELD_PREP(GENMASK(31, 24), x) +#define SE_ADDR_HI_SZ(x) FIELD_PREP(GENMASK(23, 0), x) + +#define SE_CFG_AES_ENCRYPT (SE_AES_ENC_ALG_AES_ENC | \ + SE_AES_DEC_ALG_NOP | \ + SE_AES_DST_MEMORY) + +#define SE_CFG_AES_DECRYPT (SE_AES_ENC_ALG_NOP | \ + SE_AES_DEC_ALG_AES_DEC | \ + SE_AES_DST_MEMORY) + +#define SE_CFG_GMAC_ENCRYPT (SE_AES_ENC_ALG_AES_ENC | \ + SE_AES_DEC_ALG_NOP | \ + SE_AES_ENC_MODE_GMAC | \ + SE_AES_DST_MEMORY) + +#define SE_CFG_GMAC_DECRYPT (SE_AES_ENC_ALG_NOP | \ + SE_AES_DEC_ALG_AES_DEC | \ + SE_AES_DEC_MODE_GMAC | \ + SE_AES_DST_MEMORY) + +#define SE_CFG_GCM_ENCRYPT (SE_AES_ENC_ALG_AES_ENC | \ + SE_AES_DEC_ALG_NOP | \ + SE_AES_ENC_MODE_GCM | \ + SE_AES_DST_MEMORY) + +#define SE_CFG_GCM_DECRYPT (SE_AES_ENC_ALG_NOP | \ + SE_AES_DEC_ALG_AES_DEC | \ + SE_AES_DEC_MODE_GCM | \ + SE_AES_DST_MEMORY) + +#define SE_CFG_GCM_FINAL_ENCRYPT (SE_AES_ENC_ALG_AES_ENC | \ + SE_AES_DEC_ALG_NOP | \ + SE_AES_ENC_MODE_GCM_FINAL | \ + SE_AES_DST_MEMORY) + +#define SE_CFG_GCM_FINAL_DECRYPT (SE_AES_ENC_ALG_NOP | \ + SE_AES_DEC_ALG_AES_DEC | \ + SE_AES_DEC_MODE_GCM_FINAL | \ + SE_AES_DST_MEMORY) + +#define SE_CFG_CMAC (SE_AES_ENC_ALG_AES_ENC | \ + SE_AES_ENC_MODE_CMAC | \ + SE_AES_DST_HASH_REG) + +#define SE_CFG_CBC_MAC (SE_AES_ENC_ALG_AES_ENC | \ + SE_AES_ENC_MODE_CBC_MAC) + +#define SE_CFG_INS (SE_AES_ENC_ALG_INS | \ + SE_AES_DEC_ALG_NOP) + +#define SE_CRYPTO_CFG_ECB_ENCRYPT (SE_AES_INPUT_SEL_MEMORY | \ + SE_AES_XOR_POS_BYPASS | \ + SE_AES_CORE_SEL_ENCRYPT) + +#define SE_CRYPTO_CFG_ECB_DECRYPT (SE_AES_INPUT_SEL_MEMORY | \ + SE_AES_XOR_POS_BYPASS | \ + SE_AES_CORE_SEL_DECRYPT) + +#define SE_CRYPTO_CFG_CBC_ENCRYPT (SE_AES_INPUT_SEL_MEMORY | \ + SE_AES_VCTRAM_SEL_AESOUT | \ + SE_AES_XOR_POS_TOP | \ + SE_AES_CORE_SEL_ENCRYPT | \ + SE_AES_IV_SEL_REG) + +#define SE_CRYPTO_CFG_CBC_DECRYPT (SE_AES_INPUT_SEL_MEMORY | \ + SE_AES_VCTRAM_SEL_PREV_MEM | \ + SE_AES_XOR_POS_BOTTOM | \ + SE_AES_CORE_SEL_DECRYPT | \ + SE_AES_IV_SEL_REG) + +#define SE_CRYPTO_CFG_CTR (SE_AES_INPUT_SEL_LINEAR_CTR | \ + SE_AES_VCTRAM_SEL_MEMORY | \ + SE_AES_XOR_POS_BOTTOM | \ + SE_AES_CORE_SEL_ENCRYPT | \ + SE_AES_CRYPTO_CFG_CTR_CNTN(1) | \ + SE_AES_IV_SEL_REG) + +#define SE_CRYPTO_CFG_XTS_ENCRYPT (SE_AES_INPUT_SEL_MEMORY | \ + SE_AES_VCTRAM_SEL_TWEAK | \ + SE_AES_XOR_POS_BOTH | \ + SE_AES_CORE_SEL_ENCRYPT | \ + SE_AES_IV_SEL_REG) + +#define SE_CRYPTO_CFG_XTS_DECRYPT (SE_AES_INPUT_SEL_MEMORY | \ + SE_AES_VCTRAM_SEL_TWEAK | \ + SE_AES_XOR_POS_BOTH | \ + SE_AES_CORE_SEL_DECRYPT | \ + SE_AES_IV_SEL_REG) + +#define SE_CRYPTO_CFG_XTS_DECRYPT (SE_AES_INPUT_SEL_MEMORY | \ + SE_AES_VCTRAM_SEL_TWEAK | \ + SE_AES_XOR_POS_BOTH | \ + SE_AES_CORE_SEL_DECRYPT | \ + SE_AES_IV_SEL_REG) + +#define SE_CRYPTO_CFG_CBC_MAC (SE_AES_INPUT_SEL_MEMORY | \ + SE_AES_VCTRAM_SEL_AESOUT | \ + SE_AES_XOR_POS_TOP | \ + SE_AES_CORE_SEL_ENCRYPT | \ + SE_AES_HASH_ENABLE | \ + SE_AES_IV_SEL_REG) + +#define HASH_RESULT_REG_COUNT 50 +#define CMAC_RESULT_REG_COUNT 4 + +#define SE_CRYPTO_CTR_REG_COUNT 4 +#define SE_MAX_KEYSLOT 15 +#define SE_MAX_MEM_ALLOC SZ_4M + +#define TEGRA_AES_RESERVED_KSLT 14 +#define TEGRA_XTS_RESERVED_KSLT 15 + +#define SHA_FIRST BIT(0) +#define SHA_INIT BIT(1) +#define SHA_UPDATE BIT(2) +#define SHA_FINAL BIT(3) + +/* Security Engine operation modes */ +enum se_aes_alg { + SE_ALG_CBC, /* Cipher Block Chaining (CBC) mode */ + SE_ALG_ECB, /* Electronic Codebook (ECB) mode */ + SE_ALG_CTR, /* Counter (CTR) mode */ + SE_ALG_XTS, /* XTS mode */ + SE_ALG_GMAC, /* GMAC mode */ + SE_ALG_GCM, /* GCM mode */ + SE_ALG_GCM_FINAL, /* GCM FINAL mode */ + SE_ALG_CMAC, /* Cipher-based MAC (CMAC) mode */ + SE_ALG_CBC_MAC, /* CBC MAC mode */ +}; + +enum se_hash_alg { + SE_ALG_RNG_DRBG, /* Deterministic Random Bit Generator */ + SE_ALG_SHA1, /* Secure Hash Algorithm-1 (SHA1) mode */ + SE_ALG_SHA224, /* Secure Hash Algorithm-224 (SHA224) mode */ + SE_ALG_SHA256, /* Secure Hash Algorithm-256 (SHA256) mode */ + SE_ALG_SHA384, /* Secure Hash Algorithm-384 (SHA384) mode */ + SE_ALG_SHA512, /* Secure Hash Algorithm-512 (SHA512) mode */ + SE_ALG_SHA3_224, /* Secure Hash Algorithm3-224 (SHA3-224) mode */ + SE_ALG_SHA3_256, /* Secure Hash Algorithm3-256 (SHA3-256) mode */ + SE_ALG_SHA3_384, /* Secure Hash Algorithm3-384 (SHA3-384) mode */ + SE_ALG_SHA3_512, /* Secure Hash Algorithm3-512 (SHA3-512) mode */ + SE_ALG_SHAKE128, /* Secure Hash Algorithm3 (SHAKE128) mode */ + SE_ALG_SHAKE256, /* Secure Hash Algorithm3 (SHAKE256) mode */ + SE_ALG_HMAC_SHA224, /* Hash based MAC (HMAC) - 224 */ + SE_ALG_HMAC_SHA256, /* Hash based MAC (HMAC) - 256 */ + SE_ALG_HMAC_SHA384, /* Hash based MAC (HMAC) - 384 */ + SE_ALG_HMAC_SHA512, /* Hash based MAC (HMAC) - 512 */ +}; + +struct tegra_se_alg { + struct tegra_se *se_dev; + const char *alg_base; + + union { + struct skcipher_engine_alg skcipher; + struct aead_engine_alg aead; + struct ahash_engine_alg ahash; + } alg; +}; + +struct tegra_se_regs { + u32 op; + u32 config; + u32 last_blk; + u32 linear_ctr; + u32 out_addr; + u32 aad_len; + u32 cryp_msg_len; + u32 manifest; + u32 key_addr; + u32 key_data; + u32 key_dst; + u32 result; +}; + +struct tegra_se_hw { + const struct tegra_se_regs *regs; + int (*init_alg)(struct tegra_se *se); + void (*deinit_alg)(struct tegra_se *se); + bool support_sm_alg; + u32 host1x_class; + u32 kac_ver; +}; + +struct tegra_se { + int (*manifest)(u32 user, u32 alg, u32 keylen); + const struct tegra_se_hw *hw; + struct host1x_client client; + struct host1x_channel *channel; + struct tegra_se_cmdbuf *cmdbuf; + struct tegra_se_cmdbuf *keybuf; + struct crypto_engine *engine; + struct host1x_syncpt *syncpt; + struct device *dev; + struct clk *clk; + unsigned int opcode_addr; + unsigned int stream_id; + unsigned int syncpt_id; + void __iomem *base; + u32 owner; +}; + +struct tegra_se_cmdbuf { + dma_addr_t iova; + u32 *addr; + struct device *dev; + struct kref ref; + struct host1x_bo bo; + ssize_t size; + u32 words; +}; + +struct tegra_se_datbuf { + u8 *buf; + dma_addr_t addr; + ssize_t size; +}; + +static inline int se_algname_to_algid(const char *name) +{ + if (!strcmp(name, "cbc(aes)")) + return SE_ALG_CBC; + else if (!strcmp(name, "ecb(aes)")) + return SE_ALG_ECB; + else if (!strcmp(name, "ctr(aes)")) + return SE_ALG_CTR; + else if (!strcmp(name, "xts(aes)")) + return SE_ALG_XTS; + else if (!strcmp(name, "cmac(aes)")) + return SE_ALG_CMAC; + else if (!strcmp(name, "gcm(aes)")) + return SE_ALG_GCM; + else if (!strcmp(name, "ccm(aes)")) + return SE_ALG_CBC_MAC; + + else if (!strcmp(name, "sha1")) + return SE_ALG_SHA1; + else if (!strcmp(name, "sha224")) + return SE_ALG_SHA224; + else if (!strcmp(name, "sha256")) + return SE_ALG_SHA256; + else if (!strcmp(name, "sha384")) + return SE_ALG_SHA384; + else if (!strcmp(name, "sha512")) + return SE_ALG_SHA512; + else if (!strcmp(name, "sha3-224")) + return SE_ALG_SHA3_224; + else if (!strcmp(name, "sha3-256")) + return SE_ALG_SHA3_256; + else if (!strcmp(name, "sha3-384")) + return SE_ALG_SHA3_384; + else if (!strcmp(name, "sha3-512")) + return SE_ALG_SHA3_512; + else if (!strcmp(name, "hmac(sha224)")) + return SE_ALG_HMAC_SHA224; + else if (!strcmp(name, "hmac(sha256)")) + return SE_ALG_HMAC_SHA256; + else if (!strcmp(name, "hmac(sha384)")) + return SE_ALG_HMAC_SHA384; + else if (!strcmp(name, "hmac(sha512)")) + return SE_ALG_HMAC_SHA512; + else + return -EINVAL; +} + +/* Functions */ +int tegra_init_aes(struct tegra_se *se); +int tegra_init_hash(struct tegra_se *se); +void tegra_deinit_aes(struct tegra_se *se); +void tegra_deinit_hash(struct tegra_se *se); +int tegra_key_submit(struct tegra_se *se, const u8 *key, + u32 keylen, u32 alg, u32 *keyid); + +int tegra_key_submit_reserved(struct tegra_se *se, const u8 *key, + u32 keylen, u32 alg, u32 *keyid); + +void tegra_key_invalidate(struct tegra_se *se, u32 keyid, u32 alg); +void tegra_key_invalidate_reserved(struct tegra_se *se, u32 keyid, u32 alg); +int tegra_se_host1x_submit(struct tegra_se *se, struct tegra_se_cmdbuf *cmdbuf, u32 size); + +static inline int tegra_key_submit_reserved_aes(struct tegra_se *se, const u8 *key, + u32 keylen, u32 alg, u32 *keyid) +{ + *keyid = TEGRA_AES_RESERVED_KSLT; + return tegra_key_submit_reserved(se, key, keylen, alg, keyid); +} + +static inline int tegra_key_submit_reserved_xts(struct tegra_se *se, const u8 *key, + u32 keylen, u32 alg, u32 *keyid) +{ + *keyid = TEGRA_XTS_RESERVED_KSLT; + return tegra_key_submit_reserved(se, key, keylen, alg, keyid); +} + +static inline bool tegra_key_is_reserved(u32 keyid) +{ + return ((keyid == TEGRA_AES_RESERVED_KSLT) || + (keyid == TEGRA_XTS_RESERVED_KSLT)); +} + +/* HOST1x OPCODES */ +static inline u32 host1x_opcode_setpayload(unsigned int payload) +{ + return (9 << 28) | payload; +} + +static inline u32 host1x_opcode_incr_w(unsigned int offset) +{ + /* 22-bit offset supported */ + return (10 << 28) | offset; +} + +static inline u32 host1x_opcode_nonincr_w(unsigned int offset) +{ + /* 22-bit offset supported */ + return (11 << 28) | offset; +} + +static inline u32 host1x_opcode_incr(unsigned int offset, unsigned int count) +{ + return (1 << 28) | (offset << 16) | count; +} + +static inline u32 host1x_opcode_nonincr(unsigned int offset, unsigned int count) +{ + return (2 << 28) | (offset << 16) | count; +} + +static inline u32 host1x_uclass_incr_syncpt_cond_f(u32 v) +{ + return (v & 0xff) << 10; +} + +static inline u32 host1x_uclass_incr_syncpt_indx_f(u32 v) +{ + return (v & 0x3ff) << 0; +} + +static inline u32 host1x_uclass_wait_syncpt_r(void) +{ + return 0x8; +} + +static inline u32 host1x_uclass_incr_syncpt_r(void) +{ + return 0x0; +} + +#define se_host1x_opcode_incr_w(x) host1x_opcode_incr_w((x) / 4) +#define se_host1x_opcode_nonincr_w(x) host1x_opcode_nonincr_w((x) / 4) +#define se_host1x_opcode_incr(x, y) host1x_opcode_incr((x) / 4, y) +#define se_host1x_opcode_nonincr(x, y) host1x_opcode_nonincr((x) / 4, y) + +#endif /*_TEGRA_SE_H*/ diff --git a/drivers/crypto/ti/Kconfig b/drivers/crypto/ti/Kconfig new file mode 100644 index 000000000..1a3a571ac --- /dev/null +++ b/drivers/crypto/ti/Kconfig @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0-only +config CRYPTO_DEV_TI_DTHEV2 + tristate "Support for TI DTHE V2 cryptography engine" + depends on ARCH_K3 || COMPILE_TEST + select CRYPTO_ENGINE + select CRYPTO_SKCIPHER + select CRYPTO_ECB + select CRYPTO_CBC + select CRYPTO_CTR + select CRYPTO_XTS + select CRYPTO_GCM + select CRYPTO_CCM + select SG_SPLIT + help + This enables support for the TI DTHE V2 hw cryptography engine + which can be found on TI K3 SOCs. Selecting this enables use + of hardware offloading for cryptographic algorithms on + these devices, providing enhanced resistance against side-channel + attacks. diff --git a/drivers/crypto/ti/Makefile b/drivers/crypto/ti/Makefile new file mode 100644 index 000000000..b883078f2 --- /dev/null +++ b/drivers/crypto/ti/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_TI_DTHEV2) += dthev2.o +dthev2-objs := dthev2-common.o dthev2-aes.o diff --git a/drivers/crypto/ti/dthev2-aes.c b/drivers/crypto/ti/dthev2-aes.c new file mode 100644 index 000000000..a3c069ca2 --- /dev/null +++ b/drivers/crypto/ti/dthev2-aes.c @@ -0,0 +1,1375 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * K3 DTHE V2 crypto accelerator driver + * + * Copyright (C) Texas Instruments 2025 - https://www.ti.com + * Author: T Pratham + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "dthev2-common.h" + +#include +#include +#include +#include +#include +#include +#include + +/* Registers */ + +// AES Engine +#define DTHE_P_AES_BASE 0x7000 + +#define DTHE_P_AES_KEY1_0 0x0038 +#define DTHE_P_AES_KEY1_1 0x003C +#define DTHE_P_AES_KEY1_2 0x0030 +#define DTHE_P_AES_KEY1_3 0x0034 +#define DTHE_P_AES_KEY1_4 0x0028 +#define DTHE_P_AES_KEY1_5 0x002C +#define DTHE_P_AES_KEY1_6 0x0020 +#define DTHE_P_AES_KEY1_7 0x0024 + +#define DTHE_P_AES_KEY2_0 0x0018 +#define DTHE_P_AES_KEY2_1 0x001C +#define DTHE_P_AES_KEY2_2 0x0010 +#define DTHE_P_AES_KEY2_3 0x0014 +#define DTHE_P_AES_KEY2_4 0x0008 +#define DTHE_P_AES_KEY2_5 0x000C +#define DTHE_P_AES_KEY2_6 0x0000 +#define DTHE_P_AES_KEY2_7 0x0004 + +#define DTHE_P_AES_IV_IN_0 0x0040 +#define DTHE_P_AES_IV_IN_1 0x0044 +#define DTHE_P_AES_IV_IN_2 0x0048 +#define DTHE_P_AES_IV_IN_3 0x004C +#define DTHE_P_AES_CTRL 0x0050 +#define DTHE_P_AES_C_LENGTH_0 0x0054 +#define DTHE_P_AES_C_LENGTH_1 0x0058 +#define DTHE_P_AES_AUTH_LENGTH 0x005C +#define DTHE_P_AES_DATA_IN_OUT 0x0060 +#define DTHE_P_AES_TAG_OUT 0x0070 + +#define DTHE_P_AES_SYSCONFIG 0x0084 +#define DTHE_P_AES_IRQSTATUS 0x008C +#define DTHE_P_AES_IRQENABLE 0x0090 + +/* Register write values and macros */ + +enum aes_ctrl_mode_masks { + AES_CTRL_ECB_MASK = 0x00, + AES_CTRL_CBC_MASK = BIT(5), + AES_CTRL_CTR_MASK = BIT(6), + AES_CTRL_XTS_MASK = BIT(12) | BIT(11), + AES_CTRL_GCM_MASK = BIT(17) | BIT(16) | BIT(6), + AES_CTRL_CCM_MASK = BIT(18) | BIT(6), +}; + +#define DTHE_AES_CTRL_MODE_CLEAR_MASK ~GENMASK(28, 5) + +#define DTHE_AES_CTRL_DIR_ENC BIT(2) + +#define DTHE_AES_CTRL_KEYSIZE_16B BIT(3) +#define DTHE_AES_CTRL_KEYSIZE_24B BIT(4) +#define DTHE_AES_CTRL_KEYSIZE_32B (BIT(3) | BIT(4)) + +#define DTHE_AES_CTRL_CTR_WIDTH_128B (BIT(7) | BIT(8)) + +#define DTHE_AES_CCM_L_FROM_IV_MASK GENMASK(2, 0) +#define DTHE_AES_CCM_M_BITS GENMASK(2, 0) +#define DTHE_AES_CTRL_CCM_L_FIELD_MASK GENMASK(21, 19) +#define DTHE_AES_CTRL_CCM_M_FIELD_MASK GENMASK(24, 22) + +#define DTHE_AES_CTRL_SAVE_CTX_SET BIT(29) + +#define DTHE_AES_CTRL_OUTPUT_READY BIT_MASK(0) +#define DTHE_AES_CTRL_INPUT_READY BIT_MASK(1) +#define DTHE_AES_CTRL_SAVED_CTX_READY BIT_MASK(30) +#define DTHE_AES_CTRL_CTX_READY BIT_MASK(31) + +#define DTHE_AES_SYSCONFIG_DMA_DATA_IN_OUT_EN GENMASK(6, 5) +#define DTHE_AES_IRQENABLE_EN_ALL GENMASK(3, 0) + +/* Misc */ +#define AES_IV_SIZE AES_BLOCK_SIZE +#define AES_BLOCK_WORDS (AES_BLOCK_SIZE / sizeof(u32)) +#define AES_IV_WORDS AES_BLOCK_WORDS +#define DTHE_AES_GCM_AAD_MAXLEN (BIT_ULL(32) - 1) +#define DTHE_AES_CCM_AAD_MAXLEN (BIT(16) - BIT(8)) +#define DTHE_AES_CCM_CRYPT_MAXLEN (BIT_ULL(61) - 1) +#define POLL_TIMEOUT_INTERVAL HZ + +static int dthe_cipher_init_tfm(struct crypto_skcipher *tfm) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + struct dthe_data *dev_data = dthe_get_dev(ctx); + + ctx->dev_data = dev_data; + ctx->keylen = 0; + + return 0; +} + +static int dthe_cipher_init_tfm_fallback(struct crypto_skcipher *tfm) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + struct dthe_data *dev_data = dthe_get_dev(ctx); + const char *alg_name = crypto_tfm_alg_name(crypto_skcipher_tfm(tfm)); + + ctx->dev_data = dev_data; + ctx->keylen = 0; + + ctx->skcipher_fb = crypto_alloc_sync_skcipher(alg_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->skcipher_fb)) { + dev_err(dev_data->dev, "fallback driver %s couldn't be loaded\n", + alg_name); + return PTR_ERR(ctx->skcipher_fb); + } + + return 0; +} + +static void dthe_cipher_exit_tfm(struct crypto_skcipher *tfm) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + + crypto_free_sync_skcipher(ctx->skcipher_fb); +} + +static int dthe_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256) + return -EINVAL; + + ctx->keylen = keylen; + memcpy(ctx->key, key, keylen); + + return 0; +} + +static int dthe_aes_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + + ctx->aes_mode = DTHE_AES_ECB; + + return dthe_aes_setkey(tfm, key, keylen); +} + +static int dthe_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + + ctx->aes_mode = DTHE_AES_CBC; + + return dthe_aes_setkey(tfm, key, keylen); +} + +static int dthe_aes_ctr_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + int ret = dthe_aes_setkey(tfm, key, keylen); + + if (ret) + return ret; + + ctx->aes_mode = DTHE_AES_CTR; + + crypto_sync_skcipher_clear_flags(ctx->skcipher_fb, CRYPTO_TFM_REQ_MASK); + crypto_sync_skcipher_set_flags(ctx->skcipher_fb, + crypto_skcipher_get_flags(tfm) & + CRYPTO_TFM_REQ_MASK); + + return crypto_sync_skcipher_setkey(ctx->skcipher_fb, key, keylen); +} + +static int dthe_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (keylen != 2 * AES_KEYSIZE_128 && + keylen != 2 * AES_KEYSIZE_192 && + keylen != 2 * AES_KEYSIZE_256) + return -EINVAL; + + ctx->aes_mode = DTHE_AES_XTS; + ctx->keylen = keylen / 2; + memcpy(ctx->key, key, keylen); + + crypto_sync_skcipher_clear_flags(ctx->skcipher_fb, CRYPTO_TFM_REQ_MASK); + crypto_sync_skcipher_set_flags(ctx->skcipher_fb, + crypto_skcipher_get_flags(tfm) & + CRYPTO_TFM_REQ_MASK); + + return crypto_sync_skcipher_setkey(ctx->skcipher_fb, key, keylen); +} + +static void dthe_aes_set_ctrl_key(struct dthe_tfm_ctx *ctx, + struct dthe_aes_req_ctx *rctx, + u32 *iv_in) +{ + struct dthe_data *dev_data = dthe_get_dev(ctx); + void __iomem *aes_base_reg = dev_data->regs + DTHE_P_AES_BASE; + u32 ctrl_val = 0; + + writel_relaxed(ctx->key[0], aes_base_reg + DTHE_P_AES_KEY1_0); + writel_relaxed(ctx->key[1], aes_base_reg + DTHE_P_AES_KEY1_1); + writel_relaxed(ctx->key[2], aes_base_reg + DTHE_P_AES_KEY1_2); + writel_relaxed(ctx->key[3], aes_base_reg + DTHE_P_AES_KEY1_3); + + if (ctx->keylen > AES_KEYSIZE_128) { + writel_relaxed(ctx->key[4], aes_base_reg + DTHE_P_AES_KEY1_4); + writel_relaxed(ctx->key[5], aes_base_reg + DTHE_P_AES_KEY1_5); + } + if (ctx->keylen == AES_KEYSIZE_256) { + writel_relaxed(ctx->key[6], aes_base_reg + DTHE_P_AES_KEY1_6); + writel_relaxed(ctx->key[7], aes_base_reg + DTHE_P_AES_KEY1_7); + } + + if (ctx->aes_mode == DTHE_AES_XTS) { + size_t key2_offset = ctx->keylen / sizeof(u32); + + writel_relaxed(ctx->key[key2_offset + 0], aes_base_reg + DTHE_P_AES_KEY2_0); + writel_relaxed(ctx->key[key2_offset + 1], aes_base_reg + DTHE_P_AES_KEY2_1); + writel_relaxed(ctx->key[key2_offset + 2], aes_base_reg + DTHE_P_AES_KEY2_2); + writel_relaxed(ctx->key[key2_offset + 3], aes_base_reg + DTHE_P_AES_KEY2_3); + + if (ctx->keylen > AES_KEYSIZE_128) { + writel_relaxed(ctx->key[key2_offset + 4], aes_base_reg + DTHE_P_AES_KEY2_4); + writel_relaxed(ctx->key[key2_offset + 5], aes_base_reg + DTHE_P_AES_KEY2_5); + } + if (ctx->keylen == AES_KEYSIZE_256) { + writel_relaxed(ctx->key[key2_offset + 6], aes_base_reg + DTHE_P_AES_KEY2_6); + writel_relaxed(ctx->key[key2_offset + 7], aes_base_reg + DTHE_P_AES_KEY2_7); + } + } + + if (rctx->enc) + ctrl_val |= DTHE_AES_CTRL_DIR_ENC; + + if (ctx->keylen == AES_KEYSIZE_128) + ctrl_val |= DTHE_AES_CTRL_KEYSIZE_16B; + else if (ctx->keylen == AES_KEYSIZE_192) + ctrl_val |= DTHE_AES_CTRL_KEYSIZE_24B; + else + ctrl_val |= DTHE_AES_CTRL_KEYSIZE_32B; + + // Write AES mode + ctrl_val &= DTHE_AES_CTRL_MODE_CLEAR_MASK; + switch (ctx->aes_mode) { + case DTHE_AES_ECB: + ctrl_val |= AES_CTRL_ECB_MASK; + break; + case DTHE_AES_CBC: + ctrl_val |= AES_CTRL_CBC_MASK; + break; + case DTHE_AES_CTR: + ctrl_val |= AES_CTRL_CTR_MASK; + ctrl_val |= DTHE_AES_CTRL_CTR_WIDTH_128B; + break; + case DTHE_AES_XTS: + ctrl_val |= AES_CTRL_XTS_MASK; + break; + case DTHE_AES_GCM: + ctrl_val |= AES_CTRL_GCM_MASK; + break; + case DTHE_AES_CCM: + ctrl_val |= AES_CTRL_CCM_MASK; + ctrl_val |= FIELD_PREP(DTHE_AES_CTRL_CCM_L_FIELD_MASK, + (iv_in[0] & DTHE_AES_CCM_L_FROM_IV_MASK)); + ctrl_val |= FIELD_PREP(DTHE_AES_CTRL_CCM_M_FIELD_MASK, + ((ctx->authsize - 2) >> 1) & DTHE_AES_CCM_M_BITS); + break; + } + + if (iv_in) { + ctrl_val |= DTHE_AES_CTRL_SAVE_CTX_SET; + for (int i = 0; i < AES_IV_WORDS; ++i) + writel_relaxed(iv_in[i], + aes_base_reg + DTHE_P_AES_IV_IN_0 + (DTHE_REG_SIZE * i)); + } + + writel_relaxed(ctrl_val, aes_base_reg + DTHE_P_AES_CTRL); +} + +static int dthe_aes_do_fallback(struct skcipher_request *req) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + struct dthe_aes_req_ctx *rctx = skcipher_request_ctx(req); + + SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->skcipher_fb); + + skcipher_request_set_callback(subreq, skcipher_request_flags(req), + req->base.complete, req->base.data); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + + return rctx->enc ? crypto_skcipher_encrypt(subreq) : + crypto_skcipher_decrypt(subreq); +} + +static void dthe_aes_dma_in_callback(void *data) +{ + struct skcipher_request *req = (struct skcipher_request *)data; + struct dthe_aes_req_ctx *rctx = skcipher_request_ctx(req); + + complete(&rctx->aes_compl); +} + +static int dthe_aes_run(struct crypto_engine *engine, void *areq) +{ + struct skcipher_request *req = container_of(areq, struct skcipher_request, base); + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + struct dthe_data *dev_data = dthe_get_dev(ctx); + struct dthe_aes_req_ctx *rctx = skcipher_request_ctx(req); + + unsigned int len = req->cryptlen; + struct scatterlist *src = req->src; + struct scatterlist *dst = req->dst; + + int src_nents = sg_nents_for_len(src, len); + int dst_nents = sg_nents_for_len(dst, len); + + int src_mapped_nents; + int dst_mapped_nents; + + bool diff_dst; + enum dma_data_direction src_dir, dst_dir; + + struct device *tx_dev, *rx_dev; + struct dma_async_tx_descriptor *desc_in, *desc_out; + + int ret; + + void __iomem *aes_base_reg = dev_data->regs + DTHE_P_AES_BASE; + + u32 aes_irqenable_val = readl_relaxed(aes_base_reg + DTHE_P_AES_IRQENABLE); + u32 aes_sysconfig_val = readl_relaxed(aes_base_reg + DTHE_P_AES_SYSCONFIG); + + aes_sysconfig_val |= DTHE_AES_SYSCONFIG_DMA_DATA_IN_OUT_EN; + writel_relaxed(aes_sysconfig_val, aes_base_reg + DTHE_P_AES_SYSCONFIG); + + aes_irqenable_val |= DTHE_AES_IRQENABLE_EN_ALL; + writel_relaxed(aes_irqenable_val, aes_base_reg + DTHE_P_AES_IRQENABLE); + + if (src == dst) { + diff_dst = false; + src_dir = DMA_BIDIRECTIONAL; + dst_dir = DMA_BIDIRECTIONAL; + } else { + diff_dst = true; + src_dir = DMA_TO_DEVICE; + dst_dir = DMA_FROM_DEVICE; + } + + /* + * CTR mode can operate on any input length, but the hardware + * requires input length to be a multiple of the block size. + * We need to handle the padding in the driver. + */ + if (ctx->aes_mode == DTHE_AES_CTR && req->cryptlen % AES_BLOCK_SIZE) { + unsigned int pad_size = AES_BLOCK_SIZE - (req->cryptlen % AES_BLOCK_SIZE); + u8 *pad_buf = rctx->padding; + struct scatterlist *sg; + + len += pad_size; + src_nents++; + dst_nents++; + + src = kmalloc_objs(*src, src_nents, GFP_ATOMIC); + if (!src) { + ret = -ENOMEM; + goto aes_ctr_src_alloc_err; + } + + sg_init_table(src, src_nents); + sg = dthe_copy_sg(src, req->src, req->cryptlen); + memzero_explicit(pad_buf, AES_BLOCK_SIZE); + sg_set_buf(sg, pad_buf, pad_size); + + if (diff_dst) { + dst = kmalloc_objs(*dst, dst_nents, GFP_ATOMIC); + if (!dst) { + ret = -ENOMEM; + goto aes_ctr_dst_alloc_err; + } + + sg_init_table(dst, dst_nents); + sg = dthe_copy_sg(dst, req->dst, req->cryptlen); + sg_set_buf(sg, pad_buf, pad_size); + } else { + dst = src; + } + } + + tx_dev = dmaengine_get_dma_device(dev_data->dma_aes_tx); + rx_dev = dmaengine_get_dma_device(dev_data->dma_aes_rx); + + src_mapped_nents = dma_map_sg(tx_dev, src, src_nents, src_dir); + if (src_mapped_nents == 0) { + ret = -EINVAL; + goto aes_map_src_err; + } + + if (!diff_dst) { + dst_mapped_nents = src_mapped_nents; + } else { + dst_mapped_nents = dma_map_sg(rx_dev, dst, dst_nents, dst_dir); + if (dst_mapped_nents == 0) { + ret = -EINVAL; + goto aes_map_dst_err; + } + } + + desc_in = dmaengine_prep_slave_sg(dev_data->dma_aes_rx, dst, dst_mapped_nents, + DMA_DEV_TO_MEM, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!desc_in) { + dev_err(dev_data->dev, "IN prep_slave_sg() failed\n"); + ret = -EINVAL; + goto aes_prep_err; + } + + desc_out = dmaengine_prep_slave_sg(dev_data->dma_aes_tx, src, src_mapped_nents, + DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!desc_out) { + dev_err(dev_data->dev, "OUT prep_slave_sg() failed\n"); + ret = -EINVAL; + goto aes_prep_err; + } + + desc_in->callback = dthe_aes_dma_in_callback; + desc_in->callback_param = req; + + init_completion(&rctx->aes_compl); + + if (ctx->aes_mode == DTHE_AES_ECB) + dthe_aes_set_ctrl_key(ctx, rctx, NULL); + else + dthe_aes_set_ctrl_key(ctx, rctx, (u32 *)req->iv); + + writel_relaxed(lower_32_bits(len), aes_base_reg + DTHE_P_AES_C_LENGTH_0); + writel_relaxed(upper_32_bits(len), aes_base_reg + DTHE_P_AES_C_LENGTH_1); + + dmaengine_submit(desc_in); + dmaengine_submit(desc_out); + + dma_async_issue_pending(dev_data->dma_aes_rx); + dma_async_issue_pending(dev_data->dma_aes_tx); + + // Need to do a timeout to ensure finalise gets called if DMA callback fails for any reason + ret = wait_for_completion_timeout(&rctx->aes_compl, msecs_to_jiffies(DTHE_DMA_TIMEOUT_MS)); + if (!ret) { + ret = -ETIMEDOUT; + dmaengine_terminate_sync(dev_data->dma_aes_rx); + dmaengine_terminate_sync(dev_data->dma_aes_tx); + + for (int i = 0; i < AES_BLOCK_WORDS; ++i) + readl_relaxed(aes_base_reg + DTHE_P_AES_DATA_IN_OUT + (DTHE_REG_SIZE * i)); + } else { + ret = 0; + } + + // For modes other than ECB, read IV_OUT + if (ctx->aes_mode != DTHE_AES_ECB) { + u32 *iv_out = (u32 *)req->iv; + + for (int i = 0; i < AES_IV_WORDS; ++i) + iv_out[i] = readl_relaxed(aes_base_reg + + DTHE_P_AES_IV_IN_0 + + (DTHE_REG_SIZE * i)); + } + +aes_prep_err: + if (dst_dir != DMA_BIDIRECTIONAL) + dma_unmap_sg(rx_dev, dst, dst_nents, dst_dir); +aes_map_dst_err: + dma_unmap_sg(tx_dev, src, src_nents, src_dir); + +aes_map_src_err: + if (ctx->aes_mode == DTHE_AES_CTR && req->cryptlen % AES_BLOCK_SIZE) { + memzero_explicit(rctx->padding, AES_BLOCK_SIZE); + if (diff_dst) + kfree(dst); +aes_ctr_dst_alloc_err: + kfree(src); +aes_ctr_src_alloc_err: + /* + * Fallback to software if ENOMEM + */ + if (ret == -ENOMEM) + ret = dthe_aes_do_fallback(req); + } + + local_bh_disable(); + crypto_finalize_skcipher_request(dev_data->engine, req, ret); + local_bh_enable(); + return 0; +} + +static int dthe_aes_crypt(struct skcipher_request *req) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + struct dthe_data *dev_data = dthe_get_dev(ctx); + struct crypto_engine *engine; + + /* + * If data is not a multiple of AES_BLOCK_SIZE: + * - need to return -EINVAL for ECB, CBC as they are block ciphers + * - need to fallback to software as H/W doesn't support Ciphertext Stealing for XTS + * - do nothing for CTR + */ + if (req->cryptlen % AES_BLOCK_SIZE) { + if (ctx->aes_mode == DTHE_AES_XTS) + return dthe_aes_do_fallback(req); + + if (ctx->aes_mode != DTHE_AES_CTR) + return -EINVAL; + } + + /* + * If data length input is zero, no need to do any operation. + * Except for XTS mode, where data length should be non-zero. + */ + if (req->cryptlen == 0) { + if (ctx->aes_mode == DTHE_AES_XTS) + return -EINVAL; + return 0; + } + + engine = dev_data->engine; + return crypto_transfer_skcipher_request_to_engine(engine, req); +} + +static int dthe_aes_encrypt(struct skcipher_request *req) +{ + struct dthe_aes_req_ctx *rctx = skcipher_request_ctx(req); + + rctx->enc = 1; + return dthe_aes_crypt(req); +} + +static int dthe_aes_decrypt(struct skcipher_request *req) +{ + struct dthe_aes_req_ctx *rctx = skcipher_request_ctx(req); + + rctx->enc = 0; + return dthe_aes_crypt(req); +} + +static int dthe_aead_init_tfm(struct crypto_aead *tfm) +{ + struct dthe_tfm_ctx *ctx = crypto_aead_ctx(tfm); + struct dthe_data *dev_data = dthe_get_dev(ctx); + + ctx->dev_data = dev_data; + + const char *alg_name = crypto_tfm_alg_name(crypto_aead_tfm(tfm)); + + ctx->aead_fb = crypto_alloc_sync_aead(alg_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->aead_fb)) { + dev_err(dev_data->dev, "fallback driver %s couldn't be loaded\n", + alg_name); + return PTR_ERR(ctx->aead_fb); + } + + return 0; +} + +static void dthe_aead_exit_tfm(struct crypto_aead *tfm) +{ + struct dthe_tfm_ctx *ctx = crypto_aead_ctx(tfm); + + crypto_free_sync_aead(ctx->aead_fb); +} + +/** + * dthe_aead_prep_aad - Prepare AAD scatterlist from input request + * @sg: Input scatterlist containing AAD + * @assoclen: Length of AAD + * @pad_buf: Buffer to hold AAD padding if needed + * + * Description: + * Creates a scatterlist containing only the AAD portion with padding + * to align to AES_BLOCK_SIZE. This simplifies DMA handling by allowing + * AAD to be sent separately via TX-only DMA. + * + * Return: + * Pointer to the AAD scatterlist, or ERR_PTR(error) on failure. + * The calling function needs to free the returned scatterlist when done. + **/ +static struct scatterlist *dthe_aead_prep_aad(struct scatterlist *sg, + unsigned int assoclen, + u8 *pad_buf) +{ + struct scatterlist *aad_sg; + struct scatterlist *to_sg; + int aad_nents; + + if (assoclen == 0) + return NULL; + + aad_nents = sg_nents_for_len(sg, assoclen); + if (assoclen % AES_BLOCK_SIZE) + aad_nents++; + + aad_sg = kmalloc_objs(struct scatterlist, aad_nents, GFP_ATOMIC); + if (!aad_sg) + return ERR_PTR(-ENOMEM); + + sg_init_table(aad_sg, aad_nents); + to_sg = dthe_copy_sg(aad_sg, sg, assoclen); + if (assoclen % AES_BLOCK_SIZE) { + unsigned int pad_len = AES_BLOCK_SIZE - (assoclen % AES_BLOCK_SIZE); + + memset(pad_buf, 0, pad_len); + sg_set_buf(to_sg, pad_buf, pad_len); + } + + return aad_sg; +} + +/** + * dthe_aead_prep_crypt - Prepare crypt scatterlist from req->src/req->dst + * @sg: Input req->src/req->dst scatterlist + * @assoclen: Length of AAD (to skip) + * @cryptlen: Length of ciphertext/plaintext (minus the size of TAG in decryption) + * @pad_buf: Zeroed buffer to hold crypt padding if needed + * + * Description: + * Creates a scatterlist containing only the ciphertext/plaintext portion + * (skipping AAD) with padding to align to AES_BLOCK_SIZE. + * + * Return: + * Pointer to the ciphertext scatterlist, or ERR_PTR(error) on failure. + * The calling function needs to free the returned scatterlist when done. + **/ +static struct scatterlist *dthe_aead_prep_crypt(struct scatterlist *sg, + unsigned int assoclen, + unsigned int cryptlen, + u8 *pad_buf) +{ + struct scatterlist *out_sg[1]; + struct scatterlist *crypt_sg; + struct scatterlist *to_sg; + size_t split_sizes[1] = {cryptlen}; + int out_mapped_nents[1]; + int crypt_nents; + int err; + + if (cryptlen == 0) + return NULL; + + /* Skip AAD, extract ciphertext portion */ + err = sg_split(sg, 0, assoclen, 1, split_sizes, out_sg, out_mapped_nents, GFP_ATOMIC); + if (err) + goto dthe_aead_prep_crypt_split_err; + + crypt_nents = sg_nents_for_len(out_sg[0], cryptlen); + if (cryptlen % AES_BLOCK_SIZE) + crypt_nents++; + + crypt_sg = kmalloc_objs(struct scatterlist, crypt_nents, GFP_ATOMIC); + if (!crypt_sg) { + err = -ENOMEM; + goto dthe_aead_prep_crypt_mem_err; + } + + sg_init_table(crypt_sg, crypt_nents); + to_sg = dthe_copy_sg(crypt_sg, out_sg[0], cryptlen); + if (cryptlen % AES_BLOCK_SIZE) { + unsigned int pad_len = AES_BLOCK_SIZE - (cryptlen % AES_BLOCK_SIZE); + + sg_set_buf(to_sg, pad_buf, pad_len); + } + +dthe_aead_prep_crypt_mem_err: + kfree(out_sg[0]); + +dthe_aead_prep_crypt_split_err: + if (err) + return ERR_PTR(err); + return crypt_sg; +} + +static int dthe_aead_read_tag(struct dthe_tfm_ctx *ctx, u32 *tag) +{ + struct dthe_data *dev_data = dthe_get_dev(ctx); + void __iomem *aes_base_reg = dev_data->regs + DTHE_P_AES_BASE; + u32 val; + int ret; + + ret = readl_relaxed_poll_timeout(aes_base_reg + DTHE_P_AES_CTRL, val, + (val & DTHE_AES_CTRL_SAVED_CTX_READY), + 0, POLL_TIMEOUT_INTERVAL); + if (ret) + return ret; + + for (int i = 0; i < AES_BLOCK_WORDS; ++i) + tag[i] = readl_relaxed(aes_base_reg + + DTHE_P_AES_TAG_OUT + + DTHE_REG_SIZE * i); + return 0; +} + +static int dthe_aead_enc_get_tag(struct aead_request *req) +{ + struct dthe_tfm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + u32 tag[AES_BLOCK_WORDS]; + int nents; + int ret; + + ret = dthe_aead_read_tag(ctx, tag); + if (ret) + return ret; + + nents = sg_nents_for_len(req->dst, req->cryptlen + req->assoclen + ctx->authsize); + + sg_pcopy_from_buffer(req->dst, nents, tag, ctx->authsize, + req->assoclen + req->cryptlen); + + return 0; +} + +static int dthe_aead_dec_verify_tag(struct aead_request *req) +{ + struct dthe_tfm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + u32 tag_out[AES_BLOCK_WORDS]; + u32 tag_in[AES_BLOCK_WORDS]; + int nents; + int ret; + + ret = dthe_aead_read_tag(ctx, tag_out); + if (ret) + return ret; + + nents = sg_nents_for_len(req->src, req->assoclen + req->cryptlen); + + sg_pcopy_to_buffer(req->src, nents, tag_in, ctx->authsize, + req->assoclen + req->cryptlen - ctx->authsize); + + if (crypto_memneq(tag_in, tag_out, ctx->authsize)) + return -EBADMSG; + else + return 0; +} + +static int dthe_aead_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int keylen) +{ + struct dthe_tfm_ctx *ctx = crypto_aead_ctx(tfm); + + if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256) + return -EINVAL; + + crypto_sync_aead_clear_flags(ctx->aead_fb, CRYPTO_TFM_REQ_MASK); + crypto_sync_aead_set_flags(ctx->aead_fb, + crypto_aead_get_flags(tfm) & + CRYPTO_TFM_REQ_MASK); + + return crypto_sync_aead_setkey(ctx->aead_fb, key, keylen); +} + +static int dthe_gcm_aes_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int keylen) +{ + struct dthe_tfm_ctx *ctx = crypto_aead_ctx(tfm); + int ret; + + ret = dthe_aead_setkey(tfm, key, keylen); + if (ret) + return ret; + + ctx->aes_mode = DTHE_AES_GCM; + ctx->keylen = keylen; + memcpy(ctx->key, key, keylen); + + return ret; +} + +static int dthe_ccm_aes_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int keylen) +{ + struct dthe_tfm_ctx *ctx = crypto_aead_ctx(tfm); + int ret; + + ret = dthe_aead_setkey(tfm, key, keylen); + if (ret) + return ret; + + ctx->aes_mode = DTHE_AES_CCM; + ctx->keylen = keylen; + memcpy(ctx->key, key, keylen); + + return ret; +} + +static int dthe_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) +{ + struct dthe_tfm_ctx *ctx = crypto_aead_ctx(tfm); + + /* Invalid auth size will be handled by crypto_aead_setauthsize() */ + ctx->authsize = authsize; + + return crypto_sync_aead_setauthsize(ctx->aead_fb, authsize); +} + +static int dthe_aead_do_fallback(struct aead_request *req) +{ + struct dthe_tfm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct dthe_aes_req_ctx *rctx = aead_request_ctx(req); + + SYNC_AEAD_REQUEST_ON_STACK(subreq, ctx->aead_fb); + + aead_request_set_callback(subreq, req->base.flags, + req->base.complete, req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, req->iv); + aead_request_set_ad(subreq, req->assoclen); + + return rctx->enc ? crypto_aead_encrypt(subreq) : + crypto_aead_decrypt(subreq); +} + +static void dthe_aead_dma_in_callback(void *data) +{ + struct aead_request *req = (struct aead_request *)data; + struct dthe_aes_req_ctx *rctx = aead_request_ctx(req); + + complete(&rctx->aes_compl); +} + +static int dthe_aead_run(struct crypto_engine *engine, void *areq) +{ + struct aead_request *req = container_of(areq, struct aead_request, base); + struct dthe_tfm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct dthe_aes_req_ctx *rctx = aead_request_ctx(req); + struct dthe_data *dev_data = dthe_get_dev(ctx); + + unsigned int cryptlen = req->cryptlen; + unsigned int assoclen = req->assoclen; + unsigned int authsize = ctx->authsize; + unsigned int unpadded_cryptlen; + struct scatterlist *src = NULL; + struct scatterlist *dst = NULL; + struct scatterlist *aad_sg = NULL; + u32 iv_in[AES_IV_WORDS]; + + int aad_nents = 0; + int src_nents = 0; + int dst_nents = 0; + int aad_mapped_nents = 0; + int src_mapped_nents = 0; + int dst_mapped_nents = 0; + + u8 *src_assoc_padbuf = rctx->padding; + u8 *src_crypt_padbuf = rctx->padding + AES_BLOCK_SIZE; + u8 *dst_crypt_padbuf = rctx->padding + AES_BLOCK_SIZE; + + bool diff_dst; + enum dma_data_direction aad_dir, src_dir, dst_dir; + + struct device *tx_dev, *rx_dev; + struct dma_async_tx_descriptor *desc_in, *desc_out, *desc_aad_out; + + int ret; + int err; + + void __iomem *aes_base_reg = dev_data->regs + DTHE_P_AES_BASE; + + u32 aes_irqenable_val = readl_relaxed(aes_base_reg + DTHE_P_AES_IRQENABLE); + u32 aes_sysconfig_val = readl_relaxed(aes_base_reg + DTHE_P_AES_SYSCONFIG); + + aes_sysconfig_val |= DTHE_AES_SYSCONFIG_DMA_DATA_IN_OUT_EN; + writel_relaxed(aes_sysconfig_val, aes_base_reg + DTHE_P_AES_SYSCONFIG); + + aes_irqenable_val |= DTHE_AES_IRQENABLE_EN_ALL; + writel_relaxed(aes_irqenable_val, aes_base_reg + DTHE_P_AES_IRQENABLE); + + /* In decryption, the last authsize bytes are the TAG */ + if (!rctx->enc) + cryptlen -= authsize; + unpadded_cryptlen = cryptlen; + + memset(src_assoc_padbuf, 0, AES_BLOCK_SIZE); + memset(src_crypt_padbuf, 0, AES_BLOCK_SIZE); + memset(dst_crypt_padbuf, 0, AES_BLOCK_SIZE); + + tx_dev = dmaengine_get_dma_device(dev_data->dma_aes_tx); + rx_dev = dmaengine_get_dma_device(dev_data->dma_aes_rx); + + if (req->src == req->dst) { + diff_dst = false; + src_dir = DMA_BIDIRECTIONAL; + dst_dir = DMA_BIDIRECTIONAL; + } else { + diff_dst = true; + src_dir = DMA_TO_DEVICE; + dst_dir = DMA_FROM_DEVICE; + } + aad_dir = DMA_TO_DEVICE; + + /* Prep AAD scatterlist (always from req->src) */ + aad_sg = dthe_aead_prep_aad(req->src, req->assoclen, src_assoc_padbuf); + if (IS_ERR(aad_sg)) { + ret = PTR_ERR(aad_sg); + goto aead_prep_aad_err; + } + + /* Prep ciphertext src scatterlist */ + src = dthe_aead_prep_crypt(req->src, req->assoclen, cryptlen, src_crypt_padbuf); + if (IS_ERR(src)) { + ret = PTR_ERR(src); + goto aead_prep_src_err; + } + + /* Prep ciphertext dst scatterlist (only if separate dst) */ + if (diff_dst) { + dst = dthe_aead_prep_crypt(req->dst, req->assoclen, unpadded_cryptlen, + dst_crypt_padbuf); + if (IS_ERR(dst)) { + ret = PTR_ERR(dst); + goto aead_prep_dst_err; + } + } else { + dst = src; + } + + /* Calculate padded lengths for nents calculations */ + if (req->assoclen % AES_BLOCK_SIZE) + assoclen += AES_BLOCK_SIZE - (req->assoclen % AES_BLOCK_SIZE); + if (cryptlen % AES_BLOCK_SIZE) + cryptlen += AES_BLOCK_SIZE - (cryptlen % AES_BLOCK_SIZE); + + if (assoclen != 0) { + /* Map AAD for TX only */ + aad_nents = sg_nents_for_len(aad_sg, assoclen); + aad_mapped_nents = dma_map_sg(tx_dev, aad_sg, aad_nents, aad_dir); + if (aad_mapped_nents == 0) { + dev_err(dev_data->dev, "Failed to map AAD for TX\n"); + ret = -EINVAL; + goto aead_dma_map_aad_err; + } + + /* Prepare DMA descriptors for AAD TX */ + desc_aad_out = dmaengine_prep_slave_sg(dev_data->dma_aes_tx, aad_sg, + aad_mapped_nents, DMA_MEM_TO_DEV, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!desc_aad_out) { + dev_err(dev_data->dev, "AAD TX prep_slave_sg() failed\n"); + ret = -EINVAL; + goto aead_dma_prep_aad_err; + } + } + + if (cryptlen != 0) { + /* Map ciphertext src for TX (BIDIRECTIONAL if in-place) */ + src_nents = sg_nents_for_len(src, cryptlen); + src_mapped_nents = dma_map_sg(tx_dev, src, src_nents, src_dir); + if (src_mapped_nents == 0) { + dev_err(dev_data->dev, "Failed to map ciphertext src for TX\n"); + ret = -EINVAL; + goto aead_dma_prep_aad_err; + } + + /* Prepare DMA descriptors for ciphertext TX */ + desc_out = dmaengine_prep_slave_sg(dev_data->dma_aes_tx, src, + src_mapped_nents, DMA_MEM_TO_DEV, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!desc_out) { + dev_err(dev_data->dev, "Ciphertext TX prep_slave_sg() failed\n"); + ret = -EINVAL; + goto aead_dma_prep_src_err; + } + + /* Map ciphertext dst for RX (only if separate dst) */ + if (diff_dst) { + dst_nents = sg_nents_for_len(dst, cryptlen); + dst_mapped_nents = dma_map_sg(rx_dev, dst, dst_nents, dst_dir); + if (dst_mapped_nents == 0) { + dev_err(dev_data->dev, "Failed to map ciphertext dst for RX\n"); + ret = -EINVAL; + goto aead_dma_prep_src_err; + } + } else { + dst_nents = src_nents; + dst_mapped_nents = src_mapped_nents; + } + + /* Prepare DMA descriptor for ciphertext RX */ + desc_in = dmaengine_prep_slave_sg(dev_data->dma_aes_rx, dst, + dst_mapped_nents, DMA_DEV_TO_MEM, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!desc_in) { + dev_err(dev_data->dev, "Ciphertext RX prep_slave_sg() failed\n"); + ret = -EINVAL; + goto aead_dma_prep_dst_err; + } + + desc_in->callback = dthe_aead_dma_in_callback; + desc_in->callback_param = req; + } else if (assoclen != 0) { + /* AAD-only operation */ + desc_aad_out->callback = dthe_aead_dma_in_callback; + desc_aad_out->callback_param = req; + } + + init_completion(&rctx->aes_compl); + + /* + * HACK: There is an unknown hw issue where if the previous operation had alen = 0 and + * plen != 0, the current operation's tag calculation is incorrect in the case where + * plen = 0 and alen != 0 currently. This is a workaround for now which somehow works; + * by resetting the context by writing a 1 to the C_LENGTH_0 and AUTH_LENGTH registers. + */ + if (cryptlen == 0) { + writel_relaxed(1, aes_base_reg + DTHE_P_AES_C_LENGTH_0); + writel_relaxed(1, aes_base_reg + DTHE_P_AES_AUTH_LENGTH); + } + + if (ctx->aes_mode == DTHE_AES_GCM) { + if (req->iv) { + memcpy(iv_in, req->iv, GCM_AES_IV_SIZE); + } else { + iv_in[0] = 0; + iv_in[1] = 0; + iv_in[2] = 0; + } + iv_in[3] = 0x01000000; + } else { + memcpy(iv_in, req->iv, AES_IV_SIZE); + } + + /* Clear key2 to reset previous GHASH intermediate data */ + for (int i = 0; i < AES_KEYSIZE_256 / sizeof(u32); ++i) + writel_relaxed(0, aes_base_reg + DTHE_P_AES_KEY2_6 + DTHE_REG_SIZE * i); + + dthe_aes_set_ctrl_key(ctx, rctx, iv_in); + + writel_relaxed(lower_32_bits(unpadded_cryptlen), aes_base_reg + DTHE_P_AES_C_LENGTH_0); + writel_relaxed(upper_32_bits(unpadded_cryptlen), aes_base_reg + DTHE_P_AES_C_LENGTH_1); + writel_relaxed(req->assoclen, aes_base_reg + DTHE_P_AES_AUTH_LENGTH); + + /* Submit DMA descriptors: AAD TX, ciphertext TX, ciphertext RX */ + if (assoclen != 0) + dmaengine_submit(desc_aad_out); + if (cryptlen != 0) { + dmaengine_submit(desc_out); + dmaengine_submit(desc_in); + } + + if (cryptlen != 0) + dma_async_issue_pending(dev_data->dma_aes_rx); + dma_async_issue_pending(dev_data->dma_aes_tx); + + /* Need to do timeout to ensure finalise gets called if DMA callback fails for any reason */ + ret = wait_for_completion_timeout(&rctx->aes_compl, msecs_to_jiffies(DTHE_DMA_TIMEOUT_MS)); + if (!ret) { + ret = -ETIMEDOUT; + if (cryptlen != 0) + dmaengine_terminate_sync(dev_data->dma_aes_rx); + dmaengine_terminate_sync(dev_data->dma_aes_tx); + + for (int i = 0; i < AES_BLOCK_WORDS; ++i) + readl_relaxed(aes_base_reg + DTHE_P_AES_DATA_IN_OUT + DTHE_REG_SIZE * i); + } else { + ret = 0; + } + + if (cryptlen != 0) + dma_sync_sg_for_cpu(rx_dev, dst, dst_nents, dst_dir); + + if (rctx->enc) + err = dthe_aead_enc_get_tag(req); + else + err = dthe_aead_dec_verify_tag(req); + + ret = (ret) ? ret : err; + +aead_dma_prep_dst_err: + if (diff_dst && cryptlen != 0) + dma_unmap_sg(rx_dev, dst, dst_nents, dst_dir); +aead_dma_prep_src_err: + if (cryptlen != 0) + dma_unmap_sg(tx_dev, src, src_nents, src_dir); +aead_dma_prep_aad_err: + if (assoclen != 0) + dma_unmap_sg(tx_dev, aad_sg, aad_nents, aad_dir); + +aead_dma_map_aad_err: + if (diff_dst && cryptlen != 0) + kfree(dst); +aead_prep_dst_err: + if (cryptlen != 0) + kfree(src); +aead_prep_src_err: + if (assoclen != 0) + kfree(aad_sg); + +aead_prep_aad_err: + memzero_explicit(rctx->padding, 2 * AES_BLOCK_SIZE); + + if (ret) + ret = dthe_aead_do_fallback(req); + + local_bh_disable(); + crypto_finalize_aead_request(engine, req, ret); + local_bh_enable(); + return 0; +} + +static int dthe_aead_crypt(struct aead_request *req) +{ + struct dthe_tfm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct dthe_aes_req_ctx *rctx = aead_request_ctx(req); + struct dthe_data *dev_data = dthe_get_dev(ctx); + struct crypto_engine *engine; + unsigned int cryptlen = req->cryptlen; + bool is_zero_ctr = true; + + /* In decryption, last authsize bytes are the TAG */ + if (!rctx->enc) + cryptlen -= ctx->authsize; + + if (ctx->aes_mode == DTHE_AES_CCM) { + /* + * For CCM Mode, the 128-bit IV contains the following: + * | 0 .. 2 | 3 .. 7 | 8 .. (127-8*L) | (128-8*L) .. 127 | + * | L-1 | Zero | Nonce | Counter | + * L needs to be between 2-8 (inclusive), i.e. 1 <= (L-1) <= 7 + * and the next 5 bits need to be zeroes. Else return -EINVAL + */ + u8 *iv = req->iv; + u8 L = iv[0]; + + /* variable L stores L-1 here */ + if (L < 1 || L > 7) + return -EINVAL; + /* + * DTHEv2 HW can only work with zero initial counter in CCM mode. + * Check if the initial counter value is zero or not + */ + for (int i = 0; i < L + 1; ++i) { + if (iv[AES_IV_SIZE - 1 - i] != 0) { + is_zero_ctr = false; + break; + } + } + } + + /* + * Need to fallback to software in the following cases due to HW restrictions: + * - Both AAD and plaintext/ciphertext are zero length + * - For AES-GCM, AAD length is more than 2^32 - 1 bytes + * - For AES-CCM, AAD length is more than 2^16 - 2^8 bytes + * - For AES-CCM, plaintext/ciphertext length is more than 2^61 - 1 bytes + * - For AES-CCM, AAD length is non-zero but plaintext/ciphertext length is zero + * - For AES-CCM, the initial counter (last L+1 bytes of IV) is not all zeroes + * + * PS: req->cryptlen is currently unsigned int type, which causes the second and fourth + * cases above tautologically false. If req->cryptlen is to be changed to a 64-bit + * type, the check for these would also need to be added below. + */ + if ((req->assoclen == 0 && cryptlen == 0) || + (ctx->aes_mode == DTHE_AES_CCM && req->assoclen > DTHE_AES_CCM_AAD_MAXLEN) || + (ctx->aes_mode == DTHE_AES_CCM && cryptlen == 0) || + (ctx->aes_mode == DTHE_AES_CCM && !is_zero_ctr)) + return dthe_aead_do_fallback(req); + + engine = dev_data->engine; + return crypto_transfer_aead_request_to_engine(engine, req); +} + +static int dthe_aead_encrypt(struct aead_request *req) +{ + struct dthe_aes_req_ctx *rctx = aead_request_ctx(req); + + rctx->enc = 1; + return dthe_aead_crypt(req); +} + +static int dthe_aead_decrypt(struct aead_request *req) +{ + struct dthe_aes_req_ctx *rctx = aead_request_ctx(req); + + rctx->enc = 0; + return dthe_aead_crypt(req); +} + +static struct skcipher_engine_alg cipher_algs[] = { + { + .base.init = dthe_cipher_init_tfm, + .base.setkey = dthe_aes_ecb_setkey, + .base.encrypt = dthe_aes_encrypt, + .base.decrypt = dthe_aes_decrypt, + .base.min_keysize = AES_MIN_KEY_SIZE, + .base.max_keysize = AES_MAX_KEY_SIZE, + .base.base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-dthev2", + .cra_priority = 299, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_alignmask = AES_BLOCK_SIZE - 1, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct dthe_tfm_ctx), + .cra_reqsize = sizeof(struct dthe_aes_req_ctx), + .cra_module = THIS_MODULE, + }, + .op.do_one_request = dthe_aes_run, + }, /* ECB AES */ + { + .base.init = dthe_cipher_init_tfm, + .base.setkey = dthe_aes_cbc_setkey, + .base.encrypt = dthe_aes_encrypt, + .base.decrypt = dthe_aes_decrypt, + .base.min_keysize = AES_MIN_KEY_SIZE, + .base.max_keysize = AES_MAX_KEY_SIZE, + .base.ivsize = AES_IV_SIZE, + .base.base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-dthev2", + .cra_priority = 299, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_alignmask = AES_BLOCK_SIZE - 1, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct dthe_tfm_ctx), + .cra_reqsize = sizeof(struct dthe_aes_req_ctx), + .cra_module = THIS_MODULE, + }, + .op.do_one_request = dthe_aes_run, + }, /* CBC AES */ + { + .base.init = dthe_cipher_init_tfm_fallback, + .base.exit = dthe_cipher_exit_tfm, + .base.setkey = dthe_aes_ctr_setkey, + .base.encrypt = dthe_aes_encrypt, + .base.decrypt = dthe_aes_decrypt, + .base.min_keysize = AES_MIN_KEY_SIZE, + .base.max_keysize = AES_MAX_KEY_SIZE, + .base.ivsize = AES_IV_SIZE, + .base.chunksize = AES_BLOCK_SIZE, + .base.base = { + .cra_name = "ctr(aes)", + .cra_driver_name = "ctr-aes-dthev2", + .cra_priority = 299, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct dthe_tfm_ctx), + .cra_reqsize = sizeof(struct dthe_aes_req_ctx), + .cra_module = THIS_MODULE, + }, + .op.do_one_request = dthe_aes_run, + }, /* CTR AES */ + { + .base.init = dthe_cipher_init_tfm_fallback, + .base.exit = dthe_cipher_exit_tfm, + .base.setkey = dthe_aes_xts_setkey, + .base.encrypt = dthe_aes_encrypt, + .base.decrypt = dthe_aes_decrypt, + .base.min_keysize = AES_MIN_KEY_SIZE * 2, + .base.max_keysize = AES_MAX_KEY_SIZE * 2, + .base.ivsize = AES_IV_SIZE, + .base.base = { + .cra_name = "xts(aes)", + .cra_driver_name = "xts-aes-dthev2", + .cra_priority = 299, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_alignmask = AES_BLOCK_SIZE - 1, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct dthe_tfm_ctx), + .cra_reqsize = sizeof(struct dthe_aes_req_ctx), + .cra_module = THIS_MODULE, + }, + .op.do_one_request = dthe_aes_run, + }, /* XTS AES */ +}; + +static struct aead_engine_alg aead_algs[] = { + { + .base.init = dthe_aead_init_tfm, + .base.exit = dthe_aead_exit_tfm, + .base.setkey = dthe_gcm_aes_setkey, + .base.setauthsize = dthe_aead_setauthsize, + .base.maxauthsize = AES_BLOCK_SIZE, + .base.encrypt = dthe_aead_encrypt, + .base.decrypt = dthe_aead_decrypt, + .base.chunksize = AES_BLOCK_SIZE, + .base.ivsize = GCM_AES_IV_SIZE, + .base.base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "gcm-aes-dthev2", + .cra_priority = 299, + .cra_flags = CRYPTO_ALG_TYPE_AEAD | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct dthe_tfm_ctx), + .cra_reqsize = sizeof(struct dthe_aes_req_ctx), + .cra_module = THIS_MODULE, + }, + .op.do_one_request = dthe_aead_run, + }, /* GCM AES */ + { + .base.init = dthe_aead_init_tfm, + .base.exit = dthe_aead_exit_tfm, + .base.setkey = dthe_ccm_aes_setkey, + .base.setauthsize = dthe_aead_setauthsize, + .base.maxauthsize = AES_BLOCK_SIZE, + .base.encrypt = dthe_aead_encrypt, + .base.decrypt = dthe_aead_decrypt, + .base.chunksize = AES_BLOCK_SIZE, + .base.ivsize = AES_IV_SIZE, + .base.base = { + .cra_name = "ccm(aes)", + .cra_driver_name = "ccm-aes-dthev2", + .cra_priority = 299, + .cra_flags = CRYPTO_ALG_TYPE_AEAD | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct dthe_tfm_ctx), + .cra_reqsize = sizeof(struct dthe_aes_req_ctx), + .cra_module = THIS_MODULE, + }, + .op.do_one_request = dthe_aead_run, + }, /* CCM AES */ +}; + +int dthe_register_aes_algs(void) +{ + int ret = 0; + + ret = crypto_engine_register_skciphers(cipher_algs, ARRAY_SIZE(cipher_algs)); + if (ret) + return ret; + ret = crypto_engine_register_aeads(aead_algs, ARRAY_SIZE(aead_algs)); + if (ret) + crypto_engine_unregister_skciphers(cipher_algs, ARRAY_SIZE(cipher_algs)); + + return ret; +} + +void dthe_unregister_aes_algs(void) +{ + crypto_engine_unregister_skciphers(cipher_algs, ARRAY_SIZE(cipher_algs)); + crypto_engine_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs)); +} diff --git a/drivers/crypto/ti/dthev2-common.c b/drivers/crypto/ti/dthev2-common.c new file mode 100644 index 000000000..ba6cde75d --- /dev/null +++ b/drivers/crypto/ti/dthev2-common.c @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * K3 DTHE V2 crypto accelerator driver + * + * Copyright (C) Texas Instruments 2025 - https://www.ti.com + * Author: T Pratham + */ + +#include +#include +#include +#include +#include + +#include "dthev2-common.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DRIVER_NAME "dthev2" + +static struct dthe_list dthe_dev_list = { + .dev_list = LIST_HEAD_INIT(dthe_dev_list.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(dthe_dev_list.lock), +}; + +struct dthe_data *dthe_get_dev(struct dthe_tfm_ctx *ctx) +{ + struct dthe_data *dev_data; + + if (ctx->dev_data) + return ctx->dev_data; + + spin_lock_bh(&dthe_dev_list.lock); + dev_data = list_first_entry(&dthe_dev_list.dev_list, struct dthe_data, list); + if (dev_data) + list_move_tail(&dev_data->list, &dthe_dev_list.dev_list); + spin_unlock_bh(&dthe_dev_list.lock); + + return dev_data; +} + +struct scatterlist *dthe_copy_sg(struct scatterlist *dst, + struct scatterlist *src, + int buflen) +{ + struct scatterlist *from_sg, *to_sg; + int sglen; + + for (to_sg = dst, from_sg = src; buflen && from_sg; buflen -= sglen) { + sglen = from_sg->length; + if (sglen > buflen) + sglen = buflen; + sg_set_buf(to_sg, sg_virt(from_sg), sglen); + from_sg = sg_next(from_sg); + to_sg = sg_next(to_sg); + } + + return to_sg; +} + +static int dthe_dma_init(struct dthe_data *dev_data) +{ + int ret; + struct dma_slave_config cfg; + + dev_data->dma_aes_rx = NULL; + dev_data->dma_aes_tx = NULL; + dev_data->dma_sha_tx = NULL; + + dev_data->dma_aes_rx = dma_request_chan(dev_data->dev, "rx"); + if (IS_ERR(dev_data->dma_aes_rx)) { + return dev_err_probe(dev_data->dev, PTR_ERR(dev_data->dma_aes_rx), + "Unable to request rx DMA channel\n"); + } + + dev_data->dma_aes_tx = dma_request_chan(dev_data->dev, "tx1"); + if (IS_ERR(dev_data->dma_aes_tx)) { + ret = dev_err_probe(dev_data->dev, PTR_ERR(dev_data->dma_aes_tx), + "Unable to request tx1 DMA channel\n"); + goto err_dma_aes_tx; + } + + dev_data->dma_sha_tx = dma_request_chan(dev_data->dev, "tx2"); + if (IS_ERR(dev_data->dma_sha_tx)) { + ret = dev_err_probe(dev_data->dev, PTR_ERR(dev_data->dma_sha_tx), + "Unable to request tx2 DMA channel\n"); + goto err_dma_sha_tx; + } + + memzero_explicit(&cfg, sizeof(cfg)); + + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cfg.src_maxburst = 4; + + ret = dmaengine_slave_config(dev_data->dma_aes_rx, &cfg); + if (ret) { + dev_err(dev_data->dev, "Can't configure IN dmaengine slave: %d\n", ret); + goto err_dma_config; + } + + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cfg.dst_maxburst = 4; + + ret = dmaengine_slave_config(dev_data->dma_aes_tx, &cfg); + if (ret) { + dev_err(dev_data->dev, "Can't configure OUT dmaengine slave: %d\n", ret); + goto err_dma_config; + } + + return 0; + +err_dma_config: + dma_release_channel(dev_data->dma_sha_tx); +err_dma_sha_tx: + dma_release_channel(dev_data->dma_aes_tx); +err_dma_aes_tx: + dma_release_channel(dev_data->dma_aes_rx); + + return ret; +} + +static int dthe_register_algs(void) +{ + return dthe_register_aes_algs(); +} + +static void dthe_unregister_algs(void) +{ + dthe_unregister_aes_algs(); +} + +static int dthe_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dthe_data *dev_data; + int ret; + + dev_data = devm_kzalloc(dev, sizeof(*dev_data), GFP_KERNEL); + if (!dev_data) + return -ENOMEM; + + dev_data->dev = dev; + dev_data->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(dev_data->regs)) + return PTR_ERR(dev_data->regs); + + platform_set_drvdata(pdev, dev_data); + + spin_lock(&dthe_dev_list.lock); + list_add(&dev_data->list, &dthe_dev_list.dev_list); + spin_unlock(&dthe_dev_list.lock); + + ret = dthe_dma_init(dev_data); + if (ret) + goto probe_dma_err; + + dev_data->engine = crypto_engine_alloc_init(dev, 1); + if (!dev_data->engine) { + ret = -ENOMEM; + goto probe_engine_err; + } + + ret = crypto_engine_start(dev_data->engine); + if (ret) { + dev_err(dev, "Failed to start crypto engine\n"); + goto probe_engine_start_err; + } + + ret = dthe_register_algs(); + if (ret) { + dev_err(dev, "Failed to register algs\n"); + goto probe_engine_start_err; + } + + return 0; + +probe_engine_start_err: + crypto_engine_exit(dev_data->engine); +probe_engine_err: + dma_release_channel(dev_data->dma_aes_rx); + dma_release_channel(dev_data->dma_aes_tx); + dma_release_channel(dev_data->dma_sha_tx); +probe_dma_err: + spin_lock(&dthe_dev_list.lock); + list_del(&dev_data->list); + spin_unlock(&dthe_dev_list.lock); + + return ret; +} + +static void dthe_remove(struct platform_device *pdev) +{ + struct dthe_data *dev_data = platform_get_drvdata(pdev); + + spin_lock(&dthe_dev_list.lock); + list_del(&dev_data->list); + spin_unlock(&dthe_dev_list.lock); + + dthe_unregister_algs(); + + crypto_engine_exit(dev_data->engine); + + dma_release_channel(dev_data->dma_aes_rx); + dma_release_channel(dev_data->dma_aes_tx); + dma_release_channel(dev_data->dma_sha_tx); +} + +static const struct of_device_id dthe_of_match[] = { + { .compatible = "ti,am62l-dthev2", }, + {}, +}; +MODULE_DEVICE_TABLE(of, dthe_of_match); + +static struct platform_driver dthe_driver = { + .probe = dthe_probe, + .remove = dthe_remove, + .driver = { + .name = DRIVER_NAME, + .of_match_table = dthe_of_match, + }, +}; + +module_platform_driver(dthe_driver); + +MODULE_AUTHOR("T Pratham "); +MODULE_DESCRIPTION("Texas Instruments DTHE V2 driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/ti/dthev2-common.h b/drivers/crypto/ti/dthev2-common.h new file mode 100644 index 000000000..d4a3b9c18 --- /dev/null +++ b/drivers/crypto/ti/dthev2-common.h @@ -0,0 +1,134 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * K3 DTHE V2 crypto accelerator driver + * + * Copyright (C) Texas Instruments 2025 - https://www.ti.com + * Author: T Pratham + */ + +#ifndef __TI_DTHEV2_H__ +#define __TI_DTHEV2_H__ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#define DTHE_REG_SIZE 4 +#define DTHE_DMA_TIMEOUT_MS 2000 +/* + * Size of largest possible key (of all algorithms) to be stored in dthe_tfm_ctx + * This is currently the keysize of XTS-AES-256 which is 512 bits (64 bytes) + */ +#define DTHE_MAX_KEYSIZE (AES_MAX_KEY_SIZE * 2) + +enum dthe_aes_mode { + DTHE_AES_ECB = 0, + DTHE_AES_CBC, + DTHE_AES_CTR, + DTHE_AES_XTS, + DTHE_AES_GCM, + DTHE_AES_CCM, +}; + +/* Driver specific struct definitions */ + +/** + * struct dthe_data - DTHE_V2 driver instance data + * @dev: Device pointer + * @regs: Base address of the register space + * @list: list node for dev + * @engine: Crypto engine instance + * @dma_aes_rx: AES Rx DMA Channel + * @dma_aes_tx: AES Tx DMA Channel + * @dma_sha_tx: SHA Tx DMA Channel + */ +struct dthe_data { + struct device *dev; + void __iomem *regs; + struct list_head list; + struct crypto_engine *engine; + + struct dma_chan *dma_aes_rx; + struct dma_chan *dma_aes_tx; + + struct dma_chan *dma_sha_tx; +}; + +/** + * struct dthe_list - device data list head + * @dev_list: linked list head + * @lock: Spinlock protecting accesses to the list + */ +struct dthe_list { + struct list_head dev_list; + spinlock_t lock; +}; + +/** + * struct dthe_tfm_ctx - Transform ctx struct containing ctx for all sub-components of DTHE V2 + * @dev_data: Device data struct pointer + * @keylen: AES key length + * @authsize: Authentication size for modes with authentication + * @key: AES key + * @aes_mode: AES mode + * @aead_fb: Fallback crypto aead handle + * @skcipher_fb: Fallback crypto skcipher handle for AES-XTS mode + */ +struct dthe_tfm_ctx { + struct dthe_data *dev_data; + unsigned int keylen; + unsigned int authsize; + u32 key[DTHE_MAX_KEYSIZE / sizeof(u32)]; + enum dthe_aes_mode aes_mode; + union { + struct crypto_sync_aead *aead_fb; + struct crypto_sync_skcipher *skcipher_fb; + }; +}; + +/** + * struct dthe_aes_req_ctx - AES engine req ctx struct + * @enc: flag indicating encryption or decryption operation + * @padding: padding buffer for handling unaligned data + * @aes_compl: Completion variable for use in manual completion in case of DMA callback failure + */ +struct dthe_aes_req_ctx { + int enc; + u8 padding[2 * AES_BLOCK_SIZE]; + struct completion aes_compl; +}; + +/* Struct definitions end */ + +struct dthe_data *dthe_get_dev(struct dthe_tfm_ctx *ctx); + +/** + * dthe_copy_sg - Copy sg entries from src to dst + * @dst: Destination sg to be filled + * @src: Source sg to be copied from + * @buflen: Number of bytes to be copied + * + * Description: + * Copy buflen bytes of data from src to dst. + * + **/ +struct scatterlist *dthe_copy_sg(struct scatterlist *dst, + struct scatterlist *src, + int buflen); + +int dthe_register_aes_algs(void); +void dthe_unregister_aes_algs(void); + +#endif diff --git a/drivers/crypto/virtio/Kconfig b/drivers/crypto/virtio/Kconfig new file mode 100644 index 000000000..5f8915f4a --- /dev/null +++ b/drivers/crypto/virtio/Kconfig @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0-only +config CRYPTO_DEV_VIRTIO + tristate "VirtIO crypto driver" + depends on VIRTIO + select CRYPTO_AEAD + select CRYPTO_AKCIPHER2 + select CRYPTO_SKCIPHER + select CRYPTO_ENGINE + select CRYPTO_RSA + select MPILIB + help + This driver provides support for virtio crypto device. If you + choose 'M' here, this module will be called virtio_crypto. diff --git a/drivers/crypto/virtio/Makefile b/drivers/crypto/virtio/Makefile new file mode 100644 index 000000000..bfa6cbae3 --- /dev/null +++ b/drivers/crypto/virtio/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_CRYPTO_DEV_VIRTIO) += virtio_crypto.o +virtio_crypto-objs := \ + virtio_crypto_skcipher_algs.o \ + virtio_crypto_akcipher_algs.o \ + virtio_crypto_mgr.o \ + virtio_crypto_core.o diff --git a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c new file mode 100644 index 000000000..9078f2297 --- /dev/null +++ b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c @@ -0,0 +1,569 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + /* Asymmetric algorithms supported by virtio crypto device + * + * Authors: zhenwei pi + * lei he + * + * Copyright 2022 Bytedance CO., LTD. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "virtio_crypto_common.h" + +struct virtio_crypto_rsa_ctx { + unsigned int key_size; +}; + +struct virtio_crypto_akcipher_ctx { + struct virtio_crypto *vcrypto; + bool session_valid; + __u64 session_id; + union { + struct virtio_crypto_rsa_ctx rsa_ctx; + }; +}; + +struct virtio_crypto_akcipher_request { + struct virtio_crypto_request base; + void *src_buf; + void *dst_buf; + uint32_t opcode; +}; + +struct virtio_crypto_akcipher_algo { + uint32_t algonum; + uint32_t service; + unsigned int active_devs; + struct akcipher_engine_alg algo; +}; + +static DEFINE_MUTEX(algs_lock); + +static void virtio_crypto_akcipher_finalize_req( + struct virtio_crypto_akcipher_request *vc_akcipher_req, + struct akcipher_request *req, int err) +{ + kfree(vc_akcipher_req->src_buf); + kfree(vc_akcipher_req->dst_buf); + vc_akcipher_req->src_buf = NULL; + vc_akcipher_req->dst_buf = NULL; + virtcrypto_clear_request(&vc_akcipher_req->base); + + crypto_finalize_akcipher_request(vc_akcipher_req->base.dataq->engine, req, err); +} + +static void virtio_crypto_dataq_akcipher_callback(struct virtio_crypto_request *vc_req, int len) +{ + struct virtio_crypto_akcipher_request *vc_akcipher_req = + container_of(vc_req, struct virtio_crypto_akcipher_request, base); + struct akcipher_request *akcipher_req = + container_of((void *)vc_akcipher_req, struct akcipher_request, + __ctx); + int error; + + switch (vc_req->status) { + case VIRTIO_CRYPTO_OK: + error = 0; + break; + case VIRTIO_CRYPTO_INVSESS: + case VIRTIO_CRYPTO_ERR: + error = -EINVAL; + break; + case VIRTIO_CRYPTO_BADMSG: + error = -EBADMSG; + break; + default: + error = -EIO; + break; + } + + /* actual length may be less than dst buffer */ + akcipher_req->dst_len = min_t(unsigned int, len - sizeof(vc_req->status), + akcipher_req->dst_len); + sg_copy_from_buffer(akcipher_req->dst, sg_nents(akcipher_req->dst), + vc_akcipher_req->dst_buf, akcipher_req->dst_len); + virtio_crypto_akcipher_finalize_req(vc_akcipher_req, akcipher_req, error); +} + +static int virtio_crypto_alg_akcipher_init_session(struct virtio_crypto_akcipher_ctx *ctx, + struct virtio_crypto_ctrl_header *header, + struct virtio_crypto_akcipher_session_para *para, + const uint8_t *key, unsigned int keylen) +{ + struct scatterlist outhdr_sg, key_sg, inhdr_sg, *sgs[3]; + struct virtio_crypto *vcrypto = ctx->vcrypto; + uint8_t *pkey; + int err; + unsigned int num_out = 0, num_in = 0; + struct virtio_crypto_op_ctrl_req *ctrl; + struct virtio_crypto_session_input *input; + struct virtio_crypto_ctrl_request *vc_ctrl_req; + + pkey = kmemdup(key, keylen, GFP_KERNEL); + if (!pkey) + return -ENOMEM; + + vc_ctrl_req = kzalloc_obj(*vc_ctrl_req); + if (!vc_ctrl_req) { + err = -ENOMEM; + goto out; + } + + ctrl = &vc_ctrl_req->ctrl; + memcpy(&ctrl->header, header, sizeof(ctrl->header)); + memcpy(&ctrl->u.akcipher_create_session.para, para, sizeof(*para)); + input = &vc_ctrl_req->input; + input->status = cpu_to_le32(VIRTIO_CRYPTO_ERR); + + sg_init_one(&outhdr_sg, ctrl, sizeof(*ctrl)); + sgs[num_out++] = &outhdr_sg; + + sg_init_one(&key_sg, pkey, keylen); + sgs[num_out++] = &key_sg; + + sg_init_one(&inhdr_sg, input, sizeof(*input)); + sgs[num_out + num_in++] = &inhdr_sg; + + err = virtio_crypto_ctrl_vq_request(vcrypto, sgs, num_out, num_in, vc_ctrl_req); + if (err < 0) + goto out; + + if (le32_to_cpu(input->status) != VIRTIO_CRYPTO_OK) { + pr_err("virtio_crypto: Create session failed status: %u\n", + le32_to_cpu(input->status)); + err = -EINVAL; + goto out; + } + + ctx->session_id = le64_to_cpu(input->session_id); + ctx->session_valid = true; + err = 0; + +out: + kfree(vc_ctrl_req); + kfree_sensitive(pkey); + + return err; +} + +static int virtio_crypto_alg_akcipher_close_session(struct virtio_crypto_akcipher_ctx *ctx) +{ + struct scatterlist outhdr_sg, inhdr_sg, *sgs[2]; + struct virtio_crypto_destroy_session_req *destroy_session; + struct virtio_crypto *vcrypto = ctx->vcrypto; + unsigned int num_out = 0, num_in = 0; + int err; + struct virtio_crypto_op_ctrl_req *ctrl; + struct virtio_crypto_inhdr *ctrl_status; + struct virtio_crypto_ctrl_request *vc_ctrl_req; + + if (!ctx->session_valid) + return 0; + + vc_ctrl_req = kzalloc_obj(*vc_ctrl_req); + if (!vc_ctrl_req) + return -ENOMEM; + + ctrl_status = &vc_ctrl_req->ctrl_status; + ctrl_status->status = VIRTIO_CRYPTO_ERR; + ctrl = &vc_ctrl_req->ctrl; + ctrl->header.opcode = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_DESTROY_SESSION); + ctrl->header.queue_id = 0; + + destroy_session = &ctrl->u.destroy_session; + destroy_session->session_id = cpu_to_le64(ctx->session_id); + + sg_init_one(&outhdr_sg, ctrl, sizeof(*ctrl)); + sgs[num_out++] = &outhdr_sg; + + sg_init_one(&inhdr_sg, &ctrl_status->status, sizeof(ctrl_status->status)); + sgs[num_out + num_in++] = &inhdr_sg; + + err = virtio_crypto_ctrl_vq_request(vcrypto, sgs, num_out, num_in, vc_ctrl_req); + if (err < 0) + goto out; + + if (ctrl_status->status != VIRTIO_CRYPTO_OK) { + pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n", + ctrl_status->status, + le64_to_cpu(destroy_session->session_id)); + err = -EINVAL; + goto out; + } + + err = 0; + ctx->session_valid = false; + +out: + kfree(vc_ctrl_req); + + return err; +} + +static int __virtio_crypto_akcipher_do_req(struct virtio_crypto_akcipher_request *vc_akcipher_req, + struct akcipher_request *req, struct data_queue *data_vq) +{ + struct crypto_akcipher *atfm = crypto_akcipher_reqtfm(req); + struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(atfm); + struct virtio_crypto_request *vc_req = &vc_akcipher_req->base; + struct virtio_crypto *vcrypto = ctx->vcrypto; + struct virtio_crypto_op_data_req *req_data = vc_req->req_data; + struct scatterlist *sgs[4], outhdr_sg, inhdr_sg, srcdata_sg, dstdata_sg; + void *src_buf, *dst_buf = NULL; + unsigned int num_out = 0, num_in = 0; + int node = dev_to_node(&vcrypto->vdev->dev); + unsigned long flags; + int ret; + + /* out header */ + sg_init_one(&outhdr_sg, req_data, sizeof(*req_data)); + sgs[num_out++] = &outhdr_sg; + + /* src data */ + src_buf = kcalloc_node(req->src_len, 1, GFP_KERNEL, node); + if (!src_buf) + return -ENOMEM; + + sg_copy_to_buffer(req->src, sg_nents(req->src), src_buf, req->src_len); + sg_init_one(&srcdata_sg, src_buf, req->src_len); + sgs[num_out++] = &srcdata_sg; + + /* dst data */ + dst_buf = kcalloc_node(req->dst_len, 1, GFP_KERNEL, node); + if (!dst_buf) + goto free_src; + + sg_init_one(&dstdata_sg, dst_buf, req->dst_len); + sgs[num_out + num_in++] = &dstdata_sg; + + vc_akcipher_req->src_buf = src_buf; + vc_akcipher_req->dst_buf = dst_buf; + + /* in header */ + sg_init_one(&inhdr_sg, &vc_req->status, sizeof(vc_req->status)); + sgs[num_out + num_in++] = &inhdr_sg; + + spin_lock_irqsave(&data_vq->lock, flags); + ret = virtqueue_add_sgs(data_vq->vq, sgs, num_out, num_in, vc_req, GFP_ATOMIC); + virtqueue_kick(data_vq->vq); + spin_unlock_irqrestore(&data_vq->lock, flags); + if (ret) + goto err; + + return 0; + +err: + kfree(dst_buf); +free_src: + kfree(src_buf); + return -ENOMEM; +} + +static int virtio_crypto_rsa_do_req(struct crypto_engine *engine, void *vreq) +{ + struct akcipher_request *req = container_of(vreq, struct akcipher_request, base); + struct virtio_crypto_akcipher_request *vc_akcipher_req = akcipher_request_ctx(req); + struct virtio_crypto_request *vc_req = &vc_akcipher_req->base; + struct crypto_akcipher *atfm = crypto_akcipher_reqtfm(req); + struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(atfm); + struct virtio_crypto *vcrypto = ctx->vcrypto; + struct data_queue *data_vq = vc_req->dataq; + struct virtio_crypto_op_header *header; + struct virtio_crypto_akcipher_data_req *akcipher_req; + int ret; + + vc_req->sgs = NULL; + vc_req->req_data = kzalloc_node(sizeof(*vc_req->req_data), + GFP_KERNEL, dev_to_node(&vcrypto->vdev->dev)); + if (!vc_req->req_data) + return -ENOMEM; + + /* build request header */ + header = &vc_req->req_data->header; + header->opcode = cpu_to_le32(vc_akcipher_req->opcode); + header->algo = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_RSA); + header->session_id = cpu_to_le64(ctx->session_id); + + /* build request akcipher data */ + akcipher_req = &vc_req->req_data->u.akcipher_req; + akcipher_req->para.src_data_len = cpu_to_le32(req->src_len); + akcipher_req->para.dst_data_len = cpu_to_le32(req->dst_len); + + ret = __virtio_crypto_akcipher_do_req(vc_akcipher_req, req, data_vq); + if (ret < 0) { + kfree_sensitive(vc_req->req_data); + vc_req->req_data = NULL; + return ret; + } + + return 0; +} + +static int virtio_crypto_rsa_req(struct akcipher_request *req, uint32_t opcode) +{ + struct crypto_akcipher *atfm = crypto_akcipher_reqtfm(req); + struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(atfm); + struct virtio_crypto_akcipher_request *vc_akcipher_req = akcipher_request_ctx(req); + struct virtio_crypto_request *vc_req = &vc_akcipher_req->base; + struct virtio_crypto *vcrypto = ctx->vcrypto; + /* Use the first data virtqueue as default */ + struct data_queue *data_vq = &vcrypto->data_vq[0]; + + vc_req->dataq = data_vq; + vc_req->alg_cb = virtio_crypto_dataq_akcipher_callback; + vc_akcipher_req->opcode = opcode; + + return crypto_transfer_akcipher_request_to_engine(data_vq->engine, req); +} + +static int virtio_crypto_rsa_encrypt(struct akcipher_request *req) +{ + return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_ENCRYPT); +} + +static int virtio_crypto_rsa_decrypt(struct akcipher_request *req) +{ + return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_DECRYPT); +} + +static int virtio_crypto_rsa_set_key(struct crypto_akcipher *tfm, + const void *key, + unsigned int keylen, + bool private, + int padding_algo, + int hash_algo) +{ + struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm); + struct virtio_crypto_rsa_ctx *rsa_ctx = &ctx->rsa_ctx; + struct virtio_crypto *vcrypto; + struct virtio_crypto_ctrl_header header; + struct virtio_crypto_akcipher_session_para para; + struct rsa_key rsa_key = {0}; + int node = virtio_crypto_get_current_node(); + uint32_t keytype; + int ret; + MPI n; + + if (private) { + keytype = VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PRIVATE; + ret = rsa_parse_priv_key(&rsa_key, key, keylen); + } else { + keytype = VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PUBLIC; + ret = rsa_parse_pub_key(&rsa_key, key, keylen); + } + + if (ret) + return ret; + + n = mpi_read_raw_data(rsa_key.n, rsa_key.n_sz); + if (!n) + return -ENOMEM; + + rsa_ctx->key_size = mpi_get_size(n); + mpi_free(n); + + if (!ctx->vcrypto) { + vcrypto = virtcrypto_get_dev_node(node, VIRTIO_CRYPTO_SERVICE_AKCIPHER, + VIRTIO_CRYPTO_AKCIPHER_RSA); + if (!vcrypto) { + pr_err("virtio_crypto: Could not find a virtio device in the system or unsupported algo\n"); + return -ENODEV; + } + + ctx->vcrypto = vcrypto; + } else { + virtio_crypto_alg_akcipher_close_session(ctx); + } + + /* set ctrl header */ + header.opcode = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_CREATE_SESSION); + header.algo = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_RSA); + header.queue_id = 0; + + /* set RSA para */ + para.algo = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_RSA); + para.keytype = cpu_to_le32(keytype); + para.keylen = cpu_to_le32(keylen); + para.u.rsa.padding_algo = cpu_to_le32(padding_algo); + para.u.rsa.hash_algo = cpu_to_le32(hash_algo); + + return virtio_crypto_alg_akcipher_init_session(ctx, &header, ¶, key, keylen); +} + +static int virtio_crypto_rsa_raw_set_priv_key(struct crypto_akcipher *tfm, + const void *key, + unsigned int keylen) +{ + return virtio_crypto_rsa_set_key(tfm, key, keylen, 1, + VIRTIO_CRYPTO_RSA_RAW_PADDING, + VIRTIO_CRYPTO_RSA_NO_HASH); +} + + +static int virtio_crypto_p1pad_rsa_sha1_set_priv_key(struct crypto_akcipher *tfm, + const void *key, + unsigned int keylen) +{ + return virtio_crypto_rsa_set_key(tfm, key, keylen, 1, + VIRTIO_CRYPTO_RSA_PKCS1_PADDING, + VIRTIO_CRYPTO_RSA_SHA1); +} + +static int virtio_crypto_rsa_raw_set_pub_key(struct crypto_akcipher *tfm, + const void *key, + unsigned int keylen) +{ + return virtio_crypto_rsa_set_key(tfm, key, keylen, 0, + VIRTIO_CRYPTO_RSA_RAW_PADDING, + VIRTIO_CRYPTO_RSA_NO_HASH); +} + +static int virtio_crypto_p1pad_rsa_sha1_set_pub_key(struct crypto_akcipher *tfm, + const void *key, + unsigned int keylen) +{ + return virtio_crypto_rsa_set_key(tfm, key, keylen, 0, + VIRTIO_CRYPTO_RSA_PKCS1_PADDING, + VIRTIO_CRYPTO_RSA_SHA1); +} + +static unsigned int virtio_crypto_rsa_max_size(struct crypto_akcipher *tfm) +{ + struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm); + struct virtio_crypto_rsa_ctx *rsa_ctx = &ctx->rsa_ctx; + + return rsa_ctx->key_size; +} + +static int virtio_crypto_rsa_init_tfm(struct crypto_akcipher *tfm) +{ + akcipher_set_reqsize(tfm, + sizeof(struct virtio_crypto_akcipher_request)); + + return 0; +} + +static void virtio_crypto_rsa_exit_tfm(struct crypto_akcipher *tfm) +{ + struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm); + + virtio_crypto_alg_akcipher_close_session(ctx); + virtcrypto_dev_put(ctx->vcrypto); +} + +static struct virtio_crypto_akcipher_algo virtio_crypto_akcipher_algs[] = { + { + .algonum = VIRTIO_CRYPTO_AKCIPHER_RSA, + .service = VIRTIO_CRYPTO_SERVICE_AKCIPHER, + .algo.base = { + .encrypt = virtio_crypto_rsa_encrypt, + .decrypt = virtio_crypto_rsa_decrypt, + .set_pub_key = virtio_crypto_rsa_raw_set_pub_key, + .set_priv_key = virtio_crypto_rsa_raw_set_priv_key, + .max_size = virtio_crypto_rsa_max_size, + .init = virtio_crypto_rsa_init_tfm, + .exit = virtio_crypto_rsa_exit_tfm, + .base = { + .cra_name = "rsa", + .cra_driver_name = "virtio-crypto-rsa", + .cra_priority = 150, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct virtio_crypto_akcipher_ctx), + }, + }, + .algo.op = { + .do_one_request = virtio_crypto_rsa_do_req, + }, + }, + { + .algonum = VIRTIO_CRYPTO_AKCIPHER_RSA, + .service = VIRTIO_CRYPTO_SERVICE_AKCIPHER, + .algo.base = { + .encrypt = virtio_crypto_rsa_encrypt, + .decrypt = virtio_crypto_rsa_decrypt, + /* + * Must specify an arbitrary hash algorithm upon + * set_{pub,priv}_key (even though it's not used + * by encrypt/decrypt) because qemu checks for it. + */ + .set_pub_key = virtio_crypto_p1pad_rsa_sha1_set_pub_key, + .set_priv_key = virtio_crypto_p1pad_rsa_sha1_set_priv_key, + .max_size = virtio_crypto_rsa_max_size, + .init = virtio_crypto_rsa_init_tfm, + .exit = virtio_crypto_rsa_exit_tfm, + .base = { + .cra_name = "pkcs1pad(rsa)", + .cra_driver_name = "virtio-pkcs1-rsa", + .cra_priority = 150, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct virtio_crypto_akcipher_ctx), + }, + }, + .algo.op = { + .do_one_request = virtio_crypto_rsa_do_req, + }, + }, +}; + +int virtio_crypto_akcipher_algs_register(struct virtio_crypto *vcrypto) +{ + int ret = 0; + int i = 0; + + mutex_lock(&algs_lock); + + for (i = 0; i < ARRAY_SIZE(virtio_crypto_akcipher_algs); i++) { + uint32_t service = virtio_crypto_akcipher_algs[i].service; + uint32_t algonum = virtio_crypto_akcipher_algs[i].algonum; + + if (!virtcrypto_algo_is_supported(vcrypto, service, algonum)) + continue; + + if (virtio_crypto_akcipher_algs[i].active_devs == 0) { + ret = crypto_engine_register_akcipher(&virtio_crypto_akcipher_algs[i].algo); + if (ret) + goto unlock; + } + + virtio_crypto_akcipher_algs[i].active_devs++; + dev_info(&vcrypto->vdev->dev, "Registered akcipher algo %s\n", + virtio_crypto_akcipher_algs[i].algo.base.base.cra_name); + } + +unlock: + mutex_unlock(&algs_lock); + return ret; +} + +void virtio_crypto_akcipher_algs_unregister(struct virtio_crypto *vcrypto) +{ + int i = 0; + + mutex_lock(&algs_lock); + + for (i = 0; i < ARRAY_SIZE(virtio_crypto_akcipher_algs); i++) { + uint32_t service = virtio_crypto_akcipher_algs[i].service; + uint32_t algonum = virtio_crypto_akcipher_algs[i].algonum; + + if (virtio_crypto_akcipher_algs[i].active_devs == 0 || + !virtcrypto_algo_is_supported(vcrypto, service, algonum)) + continue; + + if (virtio_crypto_akcipher_algs[i].active_devs == 1) + crypto_engine_unregister_akcipher(&virtio_crypto_akcipher_algs[i].algo); + + virtio_crypto_akcipher_algs[i].active_devs--; + } + + mutex_unlock(&algs_lock); +} diff --git a/drivers/crypto/virtio/virtio_crypto_common.h b/drivers/crypto/virtio/virtio_crypto_common.h new file mode 100644 index 000000000..0c2efdc83 --- /dev/null +++ b/drivers/crypto/virtio/virtio_crypto_common.h @@ -0,0 +1,153 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Common header for Virtio crypto device. + * + * Copyright 2016 HUAWEI TECHNOLOGIES CO., LTD. + */ + +#ifndef _VIRTIO_CRYPTO_COMMON_H +#define _VIRTIO_CRYPTO_COMMON_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +/* Internal representation of a data virtqueue */ +struct data_queue { + /* Virtqueue associated with this send _queue */ + struct virtqueue *vq; + + /* To protect the vq operations for the dataq */ + spinlock_t lock; + + /* Name of the tx queue: dataq.$index */ + char name[32]; + + struct crypto_engine *engine; + struct work_struct done_work; +}; + +struct virtio_crypto { + struct virtio_device *vdev; + struct virtqueue *ctrl_vq; + struct data_queue *data_vq; + + /* Work struct for config space updates */ + struct work_struct config_work; + + /* To protect the vq operations for the controlq */ + spinlock_t ctrl_lock; + + /* Maximum of data queues supported by the device */ + u32 max_data_queues; + + /* Number of queue currently used by the driver */ + u32 curr_queue; + + /* + * Specifies the services mask which the device support, + * see VIRTIO_CRYPTO_SERVICE_* + */ + u32 crypto_services; + + /* Detailed algorithms mask */ + u32 cipher_algo_l; + u32 cipher_algo_h; + u32 hash_algo; + u32 mac_algo_l; + u32 mac_algo_h; + u32 aead_algo; + u32 akcipher_algo; + + /* Maximum length of cipher key */ + u32 max_cipher_key_len; + /* Maximum length of authenticated key */ + u32 max_auth_key_len; + /* Maximum size of per request */ + u64 max_size; + + unsigned long status; + atomic_t ref_count; + struct list_head list; + struct module *owner; + uint8_t dev_id; + + /* Does the affinity hint is set for virtqueues? */ + bool affinity_hint_set; +}; + +struct virtio_crypto_sym_session_info { + /* Backend session id, which come from the host side */ + __u64 session_id; +}; + +/* + * Note: there are padding fields in request, clear them to zero before + * sending to host to avoid to divulge any information. + * Ex, virtio_crypto_ctrl_request::ctrl::u::destroy_session::padding[48] + */ +struct virtio_crypto_ctrl_request { + struct virtio_crypto_op_ctrl_req ctrl; + struct virtio_crypto_session_input input; + struct virtio_crypto_inhdr ctrl_status; + struct completion compl; +}; + +struct virtio_crypto_request; +typedef void (*virtio_crypto_data_callback) + (struct virtio_crypto_request *vc_req, int len); + +struct virtio_crypto_request { + uint8_t status; + struct virtio_crypto_op_data_req *req_data; + struct scatterlist **sgs; + struct data_queue *dataq; + virtio_crypto_data_callback alg_cb; +}; + +int virtcrypto_devmgr_add_dev(struct virtio_crypto *vcrypto_dev); +struct list_head *virtcrypto_devmgr_get_head(void); +void virtcrypto_devmgr_rm_dev(struct virtio_crypto *vcrypto_dev); +int virtcrypto_dev_get(struct virtio_crypto *vcrypto_dev); +void virtcrypto_dev_put(struct virtio_crypto *vcrypto_dev); +int virtcrypto_dev_started(struct virtio_crypto *vcrypto_dev); +bool virtcrypto_algo_is_supported(struct virtio_crypto *vcrypto_dev, + uint32_t service, + uint32_t algo); +struct virtio_crypto *virtcrypto_get_dev_node(int node, + uint32_t service, + uint32_t algo); +int virtcrypto_dev_start(struct virtio_crypto *vcrypto); +void virtcrypto_dev_stop(struct virtio_crypto *vcrypto); +int virtio_crypto_skcipher_crypt_req( + struct crypto_engine *engine, void *vreq); + +void +virtcrypto_clear_request(struct virtio_crypto_request *vc_req); + +static inline int virtio_crypto_get_current_node(void) +{ + int cpu, node; + + cpu = get_cpu(); + node = cpu_to_node(cpu); + put_cpu(); + + return node; +} + +int virtio_crypto_skcipher_algs_register(struct virtio_crypto *vcrypto); +void virtio_crypto_skcipher_algs_unregister(struct virtio_crypto *vcrypto); +int virtio_crypto_akcipher_algs_register(struct virtio_crypto *vcrypto); +void virtio_crypto_akcipher_algs_unregister(struct virtio_crypto *vcrypto); +int virtio_crypto_ctrl_vq_request(struct virtio_crypto *vcrypto, struct scatterlist *sgs[], + unsigned int out_sgs, unsigned int in_sgs, + struct virtio_crypto_ctrl_request *vc_ctrl_req); + +#endif /* _VIRTIO_CRYPTO_COMMON_H */ diff --git a/drivers/crypto/virtio/virtio_crypto_core.c b/drivers/crypto/virtio/virtio_crypto_core.c new file mode 100644 index 000000000..ee83bf656 --- /dev/null +++ b/drivers/crypto/virtio/virtio_crypto_core.c @@ -0,0 +1,595 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + /* Driver for Virtio crypto device. + * + * Copyright 2016 HUAWEI TECHNOLOGIES CO., LTD. + */ + +#include +#include +#include +#include + +#include +#include "virtio_crypto_common.h" + + +void +virtcrypto_clear_request(struct virtio_crypto_request *vc_req) +{ + if (vc_req) { + kfree_sensitive(vc_req->req_data); + kfree(vc_req->sgs); + } +} + +static void virtio_crypto_ctrlq_callback(struct virtio_crypto_ctrl_request *vc_ctrl_req) +{ + complete(&vc_ctrl_req->compl); +} + +static void virtcrypto_ctrlq_callback(struct virtqueue *vq) +{ + struct virtio_crypto *vcrypto = vq->vdev->priv; + struct virtio_crypto_ctrl_request *vc_ctrl_req; + unsigned long flags; + unsigned int len; + + spin_lock_irqsave(&vcrypto->ctrl_lock, flags); + do { + virtqueue_disable_cb(vq); + while ((vc_ctrl_req = virtqueue_get_buf(vq, &len)) != NULL) { + spin_unlock_irqrestore(&vcrypto->ctrl_lock, flags); + virtio_crypto_ctrlq_callback(vc_ctrl_req); + spin_lock_irqsave(&vcrypto->ctrl_lock, flags); + } + } while (!virtqueue_enable_cb(vq)); + spin_unlock_irqrestore(&vcrypto->ctrl_lock, flags); +} + +int virtio_crypto_ctrl_vq_request(struct virtio_crypto *vcrypto, struct scatterlist *sgs[], + unsigned int out_sgs, unsigned int in_sgs, + struct virtio_crypto_ctrl_request *vc_ctrl_req) +{ + int err; + unsigned long flags; + + init_completion(&vc_ctrl_req->compl); + + spin_lock_irqsave(&vcrypto->ctrl_lock, flags); + err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, out_sgs, in_sgs, vc_ctrl_req, GFP_ATOMIC); + if (err < 0) { + spin_unlock_irqrestore(&vcrypto->ctrl_lock, flags); + return err; + } + + virtqueue_kick(vcrypto->ctrl_vq); + spin_unlock_irqrestore(&vcrypto->ctrl_lock, flags); + + wait_for_completion(&vc_ctrl_req->compl); + + return 0; +} + +static void virtcrypto_done_work(struct work_struct *work) +{ + struct data_queue *data_vq = from_work(data_vq, work, done_work); + struct virtqueue *vq = data_vq->vq; + struct virtio_crypto_request *vc_req; + unsigned long flags; + unsigned int len; + + spin_lock_irqsave(&data_vq->lock, flags); + do { + virtqueue_disable_cb(vq); + while ((vc_req = virtqueue_get_buf(vq, &len)) != NULL) { + spin_unlock_irqrestore(&data_vq->lock, flags); + if (vc_req->alg_cb) + vc_req->alg_cb(vc_req, len); + spin_lock_irqsave(&data_vq->lock, flags); + } + } while (!virtqueue_enable_cb(vq)); + spin_unlock_irqrestore(&data_vq->lock, flags); +} + +static void virtcrypto_dataq_callback(struct virtqueue *vq) +{ + struct virtio_crypto *vcrypto = vq->vdev->priv; + struct data_queue *dq = &vcrypto->data_vq[vq->index]; + + queue_work(system_bh_wq, &dq->done_work); +} + +static int virtcrypto_find_vqs(struct virtio_crypto *vi) +{ + struct virtqueue_info *vqs_info; + struct virtqueue **vqs; + int ret = -ENOMEM; + int i, total_vqs; + struct device *dev = &vi->vdev->dev; + + /* + * We expect 1 data virtqueue, followed by + * possible N-1 data queues used in multiqueue mode, + * followed by control vq. + */ + total_vqs = vi->max_data_queues + 1; + + /* Allocate space for find_vqs parameters */ + vqs = kzalloc_objs(*vqs, total_vqs); + if (!vqs) + goto err_vq; + vqs_info = kzalloc_objs(*vqs_info, total_vqs); + if (!vqs_info) + goto err_vqs_info; + + /* Parameters for control virtqueue */ + vqs_info[total_vqs - 1].callback = virtcrypto_ctrlq_callback; + vqs_info[total_vqs - 1].name = "controlq"; + + /* Allocate/initialize parameters for data virtqueues */ + for (i = 0; i < vi->max_data_queues; i++) { + vqs_info[i].callback = virtcrypto_dataq_callback; + snprintf(vi->data_vq[i].name, sizeof(vi->data_vq[i].name), + "dataq.%d", i); + vqs_info[i].name = vi->data_vq[i].name; + } + + ret = virtio_find_vqs(vi->vdev, total_vqs, vqs, vqs_info, NULL); + if (ret) + goto err_find; + + vi->ctrl_vq = vqs[total_vqs - 1]; + + for (i = 0; i < vi->max_data_queues; i++) { + spin_lock_init(&vi->data_vq[i].lock); + vi->data_vq[i].vq = vqs[i]; + /* Initialize crypto engine */ + vi->data_vq[i].engine = crypto_engine_alloc_init_and_set(dev, true, true, + virtqueue_get_vring_size(vqs[i])); + if (!vi->data_vq[i].engine) { + ret = -ENOMEM; + goto err_engine; + } + INIT_WORK(&vi->data_vq[i].done_work, virtcrypto_done_work); + } + + kfree(vqs_info); + kfree(vqs); + + return 0; + +err_engine: +err_find: + kfree(vqs_info); +err_vqs_info: + kfree(vqs); +err_vq: + return ret; +} + +static int virtcrypto_alloc_queues(struct virtio_crypto *vi) +{ + vi->data_vq = kzalloc_objs(*vi->data_vq, vi->max_data_queues); + if (!vi->data_vq) + return -ENOMEM; + + return 0; +} + +static void virtcrypto_clean_affinity(struct virtio_crypto *vi, long hcpu) +{ + int i; + + if (vi->affinity_hint_set) { + for (i = 0; i < vi->max_data_queues; i++) + virtqueue_set_affinity(vi->data_vq[i].vq, NULL); + + vi->affinity_hint_set = false; + } +} + +static void virtcrypto_set_affinity(struct virtio_crypto *vcrypto) +{ + int i = 0; + int cpu; + + /* + * In single queue mode, we don't set the cpu affinity. + */ + if (vcrypto->curr_queue == 1 || vcrypto->max_data_queues == 1) { + virtcrypto_clean_affinity(vcrypto, -1); + return; + } + + /* + * In multiqueue mode, we let the queue to be private to one cpu + * by setting the affinity hint to eliminate the contention. + * + * TODO: adds cpu hotplug support by register cpu notifier. + * + */ + for_each_online_cpu(cpu) { + virtqueue_set_affinity(vcrypto->data_vq[i].vq, cpumask_of(cpu)); + if (++i >= vcrypto->max_data_queues) + break; + } + + vcrypto->affinity_hint_set = true; +} + +static void virtcrypto_free_queues(struct virtio_crypto *vi) +{ + kfree(vi->data_vq); +} + +static int virtcrypto_init_vqs(struct virtio_crypto *vi) +{ + int ret; + + /* Allocate send & receive queues */ + ret = virtcrypto_alloc_queues(vi); + if (ret) + goto err; + + ret = virtcrypto_find_vqs(vi); + if (ret) + goto err_free; + + cpus_read_lock(); + virtcrypto_set_affinity(vi); + cpus_read_unlock(); + + return 0; + +err_free: + virtcrypto_free_queues(vi); +err: + return ret; +} + +static int virtcrypto_update_status(struct virtio_crypto *vcrypto) +{ + u32 status; + int err; + + virtio_cread_le(vcrypto->vdev, + struct virtio_crypto_config, status, &status); + + /* + * Unknown status bits would be a host error and the driver + * should consider the device to be broken. + */ + if (status & (~VIRTIO_CRYPTO_S_HW_READY)) { + dev_warn(&vcrypto->vdev->dev, + "Unknown status bits: 0x%x\n", status); + + virtio_break_device(vcrypto->vdev); + return -EPERM; + } + + if (vcrypto->status == status) + return 0; + + vcrypto->status = status; + + if (vcrypto->status & VIRTIO_CRYPTO_S_HW_READY) { + err = virtcrypto_dev_start(vcrypto); + if (err) { + dev_err(&vcrypto->vdev->dev, + "Failed to start virtio crypto device.\n"); + + return -EPERM; + } + dev_info(&vcrypto->vdev->dev, "Accelerator device is ready\n"); + } else { + virtcrypto_dev_stop(vcrypto); + dev_info(&vcrypto->vdev->dev, "Accelerator is not ready\n"); + } + + return 0; +} + +static int virtcrypto_start_crypto_engines(struct virtio_crypto *vcrypto) +{ + int32_t i; + int ret; + + for (i = 0; i < vcrypto->max_data_queues; i++) { + if (vcrypto->data_vq[i].engine) { + ret = crypto_engine_start(vcrypto->data_vq[i].engine); + if (ret) + goto err; + } + } + + return 0; + +err: + while (--i >= 0) + if (vcrypto->data_vq[i].engine) + crypto_engine_exit(vcrypto->data_vq[i].engine); + + return ret; +} + +static void virtcrypto_clear_crypto_engines(struct virtio_crypto *vcrypto) +{ + u32 i; + + for (i = 0; i < vcrypto->max_data_queues; i++) + if (vcrypto->data_vq[i].engine) + crypto_engine_exit(vcrypto->data_vq[i].engine); +} + +static void virtcrypto_del_vqs(struct virtio_crypto *vcrypto) +{ + struct virtio_device *vdev = vcrypto->vdev; + + virtcrypto_clean_affinity(vcrypto, -1); + + vdev->config->del_vqs(vdev); + + virtcrypto_free_queues(vcrypto); +} + +static void vcrypto_config_changed_work(struct work_struct *work) +{ + struct virtio_crypto *vcrypto = + container_of(work, struct virtio_crypto, config_work); + + virtcrypto_update_status(vcrypto); +} + +static int virtcrypto_probe(struct virtio_device *vdev) +{ + int err = -EFAULT; + struct virtio_crypto *vcrypto; + u32 max_data_queues = 0, max_cipher_key_len = 0; + u32 max_auth_key_len = 0; + u64 max_size = 0; + u32 cipher_algo_l = 0; + u32 cipher_algo_h = 0; + u32 hash_algo = 0; + u32 mac_algo_l = 0; + u32 mac_algo_h = 0; + u32 aead_algo = 0; + u32 akcipher_algo = 0; + u32 crypto_services = 0; + + if (!virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) + return -ENODEV; + + if (!vdev->config->get) { + dev_err(&vdev->dev, "%s failure: config access disabled\n", + __func__); + return -EINVAL; + } + + if (num_possible_nodes() > 1 && dev_to_node(&vdev->dev) < 0) { + /* + * If the accelerator is connected to a node with no memory + * there is no point in using the accelerator since the remote + * memory transaction will be very slow. + */ + dev_err(&vdev->dev, "Invalid NUMA configuration.\n"); + return -EINVAL; + } + + vcrypto = kzalloc_node(sizeof(*vcrypto), GFP_KERNEL, + dev_to_node(&vdev->dev)); + if (!vcrypto) + return -ENOMEM; + + virtio_cread_le(vdev, struct virtio_crypto_config, + max_dataqueues, &max_data_queues); + if (max_data_queues < 1) + max_data_queues = 1; + + virtio_cread_le(vdev, struct virtio_crypto_config, + max_cipher_key_len, &max_cipher_key_len); + virtio_cread_le(vdev, struct virtio_crypto_config, + max_auth_key_len, &max_auth_key_len); + virtio_cread_le(vdev, struct virtio_crypto_config, + max_size, &max_size); + virtio_cread_le(vdev, struct virtio_crypto_config, + crypto_services, &crypto_services); + virtio_cread_le(vdev, struct virtio_crypto_config, + cipher_algo_l, &cipher_algo_l); + virtio_cread_le(vdev, struct virtio_crypto_config, + cipher_algo_h, &cipher_algo_h); + virtio_cread_le(vdev, struct virtio_crypto_config, + hash_algo, &hash_algo); + virtio_cread_le(vdev, struct virtio_crypto_config, + mac_algo_l, &mac_algo_l); + virtio_cread_le(vdev, struct virtio_crypto_config, + mac_algo_h, &mac_algo_h); + virtio_cread_le(vdev, struct virtio_crypto_config, + aead_algo, &aead_algo); + if (crypto_services & (1 << VIRTIO_CRYPTO_SERVICE_AKCIPHER)) + virtio_cread_le(vdev, struct virtio_crypto_config, + akcipher_algo, &akcipher_algo); + + /* Add virtio crypto device to global table */ + err = virtcrypto_devmgr_add_dev(vcrypto); + if (err) { + dev_err(&vdev->dev, "Failed to add new virtio crypto device.\n"); + goto free; + } + vcrypto->owner = THIS_MODULE; + vcrypto = vdev->priv = vcrypto; + vcrypto->vdev = vdev; + + spin_lock_init(&vcrypto->ctrl_lock); + + /* Use single data queue as default */ + vcrypto->curr_queue = 1; + vcrypto->max_data_queues = max_data_queues; + vcrypto->max_cipher_key_len = max_cipher_key_len; + vcrypto->max_auth_key_len = max_auth_key_len; + vcrypto->max_size = max_size; + vcrypto->crypto_services = crypto_services; + vcrypto->cipher_algo_l = cipher_algo_l; + vcrypto->cipher_algo_h = cipher_algo_h; + vcrypto->mac_algo_l = mac_algo_l; + vcrypto->mac_algo_h = mac_algo_h; + vcrypto->hash_algo = hash_algo; + vcrypto->aead_algo = aead_algo; + vcrypto->akcipher_algo = akcipher_algo; + + dev_info(&vdev->dev, + "max_queues: %u, max_cipher_key_len: %u, max_auth_key_len: %u, max_size 0x%llx\n", + vcrypto->max_data_queues, + vcrypto->max_cipher_key_len, + vcrypto->max_auth_key_len, + vcrypto->max_size); + + err = virtcrypto_init_vqs(vcrypto); + if (err) { + dev_err(&vdev->dev, "Failed to initialize vqs.\n"); + goto free_dev; + } + + err = virtcrypto_start_crypto_engines(vcrypto); + if (err) + goto free_vqs; + + virtio_device_ready(vdev); + + err = virtcrypto_update_status(vcrypto); + if (err) + goto free_engines; + + INIT_WORK(&vcrypto->config_work, vcrypto_config_changed_work); + + return 0; + +free_engines: + virtcrypto_clear_crypto_engines(vcrypto); +free_vqs: + virtio_reset_device(vdev); + virtcrypto_del_vqs(vcrypto); +free_dev: + virtcrypto_devmgr_rm_dev(vcrypto); +free: + kfree(vcrypto); + return err; +} + +static void virtcrypto_free_unused_reqs(struct virtio_crypto *vcrypto) +{ + struct virtio_crypto_request *vc_req; + int i; + struct virtqueue *vq; + + for (i = 0; i < vcrypto->max_data_queues; i++) { + vq = vcrypto->data_vq[i].vq; + while ((vc_req = virtqueue_detach_unused_buf(vq)) != NULL) + virtcrypto_clear_request(vc_req); + cond_resched(); + } +} + +static void virtcrypto_remove(struct virtio_device *vdev) +{ + struct virtio_crypto *vcrypto = vdev->priv; + int i; + + dev_info(&vdev->dev, "Start virtcrypto_remove.\n"); + + flush_work(&vcrypto->config_work); + if (virtcrypto_dev_started(vcrypto)) + virtcrypto_dev_stop(vcrypto); + for (i = 0; i < vcrypto->max_data_queues; i++) + cancel_work_sync(&vcrypto->data_vq[i].done_work); + virtio_reset_device(vdev); + virtcrypto_free_unused_reqs(vcrypto); + virtcrypto_clear_crypto_engines(vcrypto); + virtcrypto_del_vqs(vcrypto); + virtcrypto_devmgr_rm_dev(vcrypto); + kfree(vcrypto); +} + +static void virtcrypto_config_changed(struct virtio_device *vdev) +{ + struct virtio_crypto *vcrypto = vdev->priv; + + schedule_work(&vcrypto->config_work); +} + +#ifdef CONFIG_PM_SLEEP +static int virtcrypto_freeze(struct virtio_device *vdev) +{ + struct virtio_crypto *vcrypto = vdev->priv; + + flush_work(&vcrypto->config_work); + virtio_reset_device(vdev); + virtcrypto_free_unused_reqs(vcrypto); + if (virtcrypto_dev_started(vcrypto)) + virtcrypto_dev_stop(vcrypto); + + virtcrypto_clear_crypto_engines(vcrypto); + virtcrypto_del_vqs(vcrypto); + return 0; +} + +static int virtcrypto_restore(struct virtio_device *vdev) +{ + struct virtio_crypto *vcrypto = vdev->priv; + int err; + + err = virtcrypto_init_vqs(vcrypto); + if (err) + return err; + + err = virtcrypto_start_crypto_engines(vcrypto); + if (err) + goto free_vqs; + + virtio_device_ready(vdev); + + err = virtcrypto_dev_start(vcrypto); + if (err) { + dev_err(&vdev->dev, "Failed to start virtio crypto device.\n"); + goto free_engines; + } + + return 0; + +free_engines: + virtcrypto_clear_crypto_engines(vcrypto); +free_vqs: + virtio_reset_device(vdev); + virtcrypto_del_vqs(vcrypto); + return err; +} +#endif + +static const unsigned int features[] = { + /* none */ +}; + +static const struct virtio_device_id id_table[] = { + { VIRTIO_ID_CRYPTO, VIRTIO_DEV_ANY_ID }, + { 0 }, +}; + +static struct virtio_driver virtio_crypto_driver = { + .driver.name = KBUILD_MODNAME, + .feature_table = features, + .feature_table_size = ARRAY_SIZE(features), + .id_table = id_table, + .probe = virtcrypto_probe, + .remove = virtcrypto_remove, + .config_changed = virtcrypto_config_changed, +#ifdef CONFIG_PM_SLEEP + .freeze = virtcrypto_freeze, + .restore = virtcrypto_restore, +#endif +}; + +module_virtio_driver(virtio_crypto_driver); + +MODULE_DEVICE_TABLE(virtio, id_table); +MODULE_DESCRIPTION("virtio crypto device driver"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Gonglei "); diff --git a/drivers/crypto/virtio/virtio_crypto_mgr.c b/drivers/crypto/virtio/virtio_crypto_mgr.c new file mode 100644 index 000000000..06c74fa13 --- /dev/null +++ b/drivers/crypto/virtio/virtio_crypto_mgr.c @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + /* Management for virtio crypto devices (refer to adf_dev_mgr.c) + * + * Copyright 2016 HUAWEI TECHNOLOGIES CO., LTD. + */ + +#include +#include +#include + +#include +#include "virtio_crypto_common.h" + +static LIST_HEAD(virtio_crypto_table); +static uint32_t num_devices; + +/* The table_lock protects the above global list and num_devices */ +static DEFINE_MUTEX(table_lock); + +#define VIRTIO_CRYPTO_MAX_DEVICES 32 + + +/* + * virtcrypto_devmgr_add_dev() - Add vcrypto_dev to the acceleration + * framework. + * @vcrypto_dev: Pointer to virtio crypto device. + * + * Function adds virtio crypto device to the global list. + * To be used by virtio crypto device specific drivers. + * + * Return: 0 on success, error code othewise. + */ +int virtcrypto_devmgr_add_dev(struct virtio_crypto *vcrypto_dev) +{ + struct list_head *itr; + + mutex_lock(&table_lock); + if (num_devices == VIRTIO_CRYPTO_MAX_DEVICES) { + pr_info("virtio_crypto: only support up to %d devices\n", + VIRTIO_CRYPTO_MAX_DEVICES); + mutex_unlock(&table_lock); + return -EFAULT; + } + + list_for_each(itr, &virtio_crypto_table) { + struct virtio_crypto *ptr = + list_entry(itr, struct virtio_crypto, list); + + if (ptr == vcrypto_dev) { + mutex_unlock(&table_lock); + return -EEXIST; + } + } + atomic_set(&vcrypto_dev->ref_count, 0); + list_add_tail(&vcrypto_dev->list, &virtio_crypto_table); + vcrypto_dev->dev_id = num_devices++; + mutex_unlock(&table_lock); + return 0; +} + +struct list_head *virtcrypto_devmgr_get_head(void) +{ + return &virtio_crypto_table; +} + +/* + * virtcrypto_devmgr_rm_dev() - Remove vcrypto_dev from the acceleration + * framework. + * @vcrypto_dev: Pointer to virtio crypto device. + * + * Function removes virtio crypto device from the acceleration framework. + * To be used by virtio crypto device specific drivers. + * + * Return: void + */ +void virtcrypto_devmgr_rm_dev(struct virtio_crypto *vcrypto_dev) +{ + mutex_lock(&table_lock); + list_del(&vcrypto_dev->list); + num_devices--; + mutex_unlock(&table_lock); +} + +/* + * virtcrypto_dev_get() - Increment vcrypto_dev reference count + * @vcrypto_dev: Pointer to virtio crypto device. + * + * Increment the vcrypto_dev refcount and if this is the first time + * incrementing it during this period the vcrypto_dev is in use, + * increment the module refcount too. + * To be used by virtio crypto device specific drivers. + * + * Return: 0 when successful, EFAULT when fail to bump module refcount + */ +int virtcrypto_dev_get(struct virtio_crypto *vcrypto_dev) +{ + if (atomic_add_return(1, &vcrypto_dev->ref_count) == 1) + if (!try_module_get(vcrypto_dev->owner)) + return -EFAULT; + return 0; +} + +/* + * virtcrypto_dev_put() - Decrement vcrypto_dev reference count + * @vcrypto_dev: Pointer to virtio crypto device. + * + * Decrement the vcrypto_dev refcount and if this is the last time + * decrementing it during this period the vcrypto_dev is in use, + * decrement the module refcount too. + * To be used by virtio crypto device specific drivers. + * + * Return: void + */ +void virtcrypto_dev_put(struct virtio_crypto *vcrypto_dev) +{ + if (atomic_sub_return(1, &vcrypto_dev->ref_count) == 0) + module_put(vcrypto_dev->owner); +} + +/* + * virtcrypto_dev_started() - Check whether device has started + * @vcrypto_dev: Pointer to virtio crypto device. + * + * To be used by virtio crypto device specific drivers. + * + * Return: 1 when the device has started, 0 otherwise + */ +int virtcrypto_dev_started(struct virtio_crypto *vcrypto_dev) +{ + return (vcrypto_dev->status & VIRTIO_CRYPTO_S_HW_READY); +} + +/* + * virtcrypto_get_dev_node() - Get vcrypto_dev on the node. + * @node: Node id the driver works. + * @service: Crypto service that needs to be supported by the + * dev + * @algo: The algorithm number that needs to be supported by the + * dev + * + * Function returns the virtio crypto device used fewest on the node, + * and supports the given crypto service and algorithm. + * + * To be used by virtio crypto device specific drivers. + * + * Return: pointer to vcrypto_dev or NULL if not found. + */ +struct virtio_crypto *virtcrypto_get_dev_node(int node, uint32_t service, + uint32_t algo) +{ + struct virtio_crypto *vcrypto_dev = NULL, *tmp_dev; + unsigned long best = ~0; + unsigned long ctr; + + mutex_lock(&table_lock); + list_for_each_entry(tmp_dev, virtcrypto_devmgr_get_head(), list) { + + if ((node == dev_to_node(&tmp_dev->vdev->dev) || + dev_to_node(&tmp_dev->vdev->dev) < 0) && + virtcrypto_dev_started(tmp_dev) && + virtcrypto_algo_is_supported(tmp_dev, service, algo)) { + ctr = atomic_read(&tmp_dev->ref_count); + if (best > ctr) { + vcrypto_dev = tmp_dev; + best = ctr; + } + } + } + + if (!vcrypto_dev) { + pr_info("virtio_crypto: Could not find a device on node %d\n", + node); + /* Get any started device */ + list_for_each_entry(tmp_dev, + virtcrypto_devmgr_get_head(), list) { + if (virtcrypto_dev_started(tmp_dev) && + virtcrypto_algo_is_supported(tmp_dev, + service, algo)) { + vcrypto_dev = tmp_dev; + break; + } + } + } + mutex_unlock(&table_lock); + if (!vcrypto_dev) + return NULL; + + virtcrypto_dev_get(vcrypto_dev); + return vcrypto_dev; +} + +/* + * virtcrypto_dev_start() - Start virtio crypto device + * @vcrypto: Pointer to virtio crypto device. + * + * Function notifies all the registered services that the virtio crypto device + * is ready to be used. + * To be used by virtio crypto device specific drivers. + * + * Return: 0 on success, EFAULT when fail to register algorithms + */ +int virtcrypto_dev_start(struct virtio_crypto *vcrypto) +{ + if (virtio_crypto_skcipher_algs_register(vcrypto)) { + pr_err("virtio_crypto: Failed to register crypto skcipher algs\n"); + return -EFAULT; + } + + if (virtio_crypto_akcipher_algs_register(vcrypto)) { + pr_err("virtio_crypto: Failed to register crypto akcipher algs\n"); + virtio_crypto_skcipher_algs_unregister(vcrypto); + return -EFAULT; + } + + return 0; +} + +/* + * virtcrypto_dev_stop() - Stop virtio crypto device + * @vcrypto: Pointer to virtio crypto device. + * + * Function notifies all the registered services that the virtio crypto device + * shall no longer be used. + * To be used by virtio crypto device specific drivers. + * + * Return: void + */ +void virtcrypto_dev_stop(struct virtio_crypto *vcrypto) +{ + virtio_crypto_skcipher_algs_unregister(vcrypto); + virtio_crypto_akcipher_algs_unregister(vcrypto); +} + +/* + * vcrypto_algo_is_supported() + * @vcrypto: Pointer to virtio crypto device. + * @service: The bit number for service validate. + * See VIRTIO_CRYPTO_SERVICE_* + * @algo : The bit number for the algorithm to validate. + * + * + * Validate if the virtio crypto device supports a service and + * algo. + * + * Return true if device supports a service and algo. + */ + +bool virtcrypto_algo_is_supported(struct virtio_crypto *vcrypto, + uint32_t service, + uint32_t algo) +{ + uint32_t service_mask = 1u << service; + uint32_t algo_mask = 0; + bool low = true; + + if (algo > 31) { + algo -= 32; + low = false; + } + + if (!(vcrypto->crypto_services & service_mask)) + return false; + + switch (service) { + case VIRTIO_CRYPTO_SERVICE_CIPHER: + if (low) + algo_mask = vcrypto->cipher_algo_l; + else + algo_mask = vcrypto->cipher_algo_h; + break; + + case VIRTIO_CRYPTO_SERVICE_HASH: + algo_mask = vcrypto->hash_algo; + break; + + case VIRTIO_CRYPTO_SERVICE_MAC: + if (low) + algo_mask = vcrypto->mac_algo_l; + else + algo_mask = vcrypto->mac_algo_h; + break; + + case VIRTIO_CRYPTO_SERVICE_AEAD: + algo_mask = vcrypto->aead_algo; + break; + + case VIRTIO_CRYPTO_SERVICE_AKCIPHER: + algo_mask = vcrypto->akcipher_algo; + break; + } + + if (!(algo_mask & (1u << algo))) + return false; + + return true; +} diff --git a/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c b/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c new file mode 100644 index 000000000..3ca441ae2 --- /dev/null +++ b/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c @@ -0,0 +1,643 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + /* Algorithms supported by virtio crypto device + * + * Authors: Gonglei + * + * Copyright 2016 HUAWEI TECHNOLOGIES CO., LTD. + */ + +#include +#include +#include +#include +#include +#include +#include "virtio_crypto_common.h" + + +struct virtio_crypto_skcipher_ctx { + struct virtio_crypto *vcrypto; + + struct virtio_crypto_sym_session_info enc_sess_info; + struct virtio_crypto_sym_session_info dec_sess_info; +}; + +struct virtio_crypto_sym_request { + struct virtio_crypto_request base; + + /* Cipher or aead */ + uint32_t type; + uint8_t *iv; + /* Encryption? */ + bool encrypt; +}; + +struct virtio_crypto_algo { + uint32_t algonum; + uint32_t service; + unsigned int active_devs; + struct skcipher_engine_alg algo; +}; + +/* + * The algs_lock protects the below global virtio_crypto_active_devs + * and crypto algorithms registion. + */ +static DEFINE_MUTEX(algs_lock); +static void virtio_crypto_skcipher_finalize_req( + struct virtio_crypto_sym_request *vc_sym_req, + struct skcipher_request *req, + int err); + +static void virtio_crypto_dataq_sym_callback + (struct virtio_crypto_request *vc_req, int len) +{ + struct virtio_crypto_sym_request *vc_sym_req = + container_of(vc_req, struct virtio_crypto_sym_request, base); + struct skcipher_request *ablk_req = + container_of((void *)vc_sym_req, struct skcipher_request, + __ctx); + int error; + + /* Finish the encrypt or decrypt process */ + if (vc_sym_req->type == VIRTIO_CRYPTO_SYM_OP_CIPHER) { + switch (vc_req->status) { + case VIRTIO_CRYPTO_OK: + error = 0; + break; + case VIRTIO_CRYPTO_INVSESS: + case VIRTIO_CRYPTO_ERR: + error = -EINVAL; + break; + case VIRTIO_CRYPTO_BADMSG: + error = -EBADMSG; + break; + default: + error = -EIO; + break; + } + virtio_crypto_skcipher_finalize_req(vc_sym_req, + ablk_req, error); + } +} + +static u64 virtio_crypto_alg_sg_nents_length(struct scatterlist *sg) +{ + u64 total = 0; + + for (total = 0; sg; sg = sg_next(sg)) + total += sg->length; + + return total; +} + +static int +virtio_crypto_alg_validate_key(int key_len, uint32_t *alg) +{ + switch (key_len) { + case AES_KEYSIZE_128: + case AES_KEYSIZE_192: + case AES_KEYSIZE_256: + *alg = VIRTIO_CRYPTO_CIPHER_AES_CBC; + break; + default: + return -EINVAL; + } + return 0; +} + +static int virtio_crypto_alg_skcipher_init_session( + struct virtio_crypto_skcipher_ctx *ctx, + uint32_t alg, const uint8_t *key, + unsigned int keylen, + int encrypt) +{ + struct scatterlist outhdr, key_sg, inhdr, *sgs[3]; + struct virtio_crypto *vcrypto = ctx->vcrypto; + int op = encrypt ? VIRTIO_CRYPTO_OP_ENCRYPT : VIRTIO_CRYPTO_OP_DECRYPT; + int err; + unsigned int num_out = 0, num_in = 0; + struct virtio_crypto_op_ctrl_req *ctrl; + struct virtio_crypto_session_input *input; + struct virtio_crypto_sym_create_session_req *sym_create_session; + struct virtio_crypto_ctrl_request *vc_ctrl_req; + + /* + * Avoid to do DMA from the stack, switch to using + * dynamically-allocated for the key + */ + uint8_t *cipher_key = kmemdup(key, keylen, GFP_ATOMIC); + + if (!cipher_key) + return -ENOMEM; + + vc_ctrl_req = kzalloc_obj(*vc_ctrl_req); + if (!vc_ctrl_req) { + err = -ENOMEM; + goto out; + } + + /* Pad ctrl header */ + ctrl = &vc_ctrl_req->ctrl; + ctrl->header.opcode = cpu_to_le32(VIRTIO_CRYPTO_CIPHER_CREATE_SESSION); + ctrl->header.algo = cpu_to_le32(alg); + /* Set the default dataqueue id to 0 */ + ctrl->header.queue_id = 0; + + input = &vc_ctrl_req->input; + input->status = cpu_to_le32(VIRTIO_CRYPTO_ERR); + /* Pad cipher's parameters */ + sym_create_session = &ctrl->u.sym_create_session; + sym_create_session->op_type = cpu_to_le32(VIRTIO_CRYPTO_SYM_OP_CIPHER); + sym_create_session->u.cipher.para.algo = ctrl->header.algo; + sym_create_session->u.cipher.para.keylen = cpu_to_le32(keylen); + sym_create_session->u.cipher.para.op = cpu_to_le32(op); + + sg_init_one(&outhdr, ctrl, sizeof(*ctrl)); + sgs[num_out++] = &outhdr; + + /* Set key */ + sg_init_one(&key_sg, cipher_key, keylen); + sgs[num_out++] = &key_sg; + + /* Return status and session id back */ + sg_init_one(&inhdr, input, sizeof(*input)); + sgs[num_out + num_in++] = &inhdr; + + err = virtio_crypto_ctrl_vq_request(vcrypto, sgs, num_out, num_in, vc_ctrl_req); + if (err < 0) + goto out; + + if (le32_to_cpu(input->status) != VIRTIO_CRYPTO_OK) { + pr_err("virtio_crypto: Create session failed status: %u\n", + le32_to_cpu(input->status)); + err = -EINVAL; + goto out; + } + + if (encrypt) + ctx->enc_sess_info.session_id = le64_to_cpu(input->session_id); + else + ctx->dec_sess_info.session_id = le64_to_cpu(input->session_id); + + err = 0; +out: + kfree(vc_ctrl_req); + kfree_sensitive(cipher_key); + return err; +} + +static int virtio_crypto_alg_skcipher_close_session( + struct virtio_crypto_skcipher_ctx *ctx, + int encrypt) +{ + struct scatterlist outhdr, status_sg, *sgs[2]; + struct virtio_crypto_destroy_session_req *destroy_session; + struct virtio_crypto *vcrypto = ctx->vcrypto; + int err; + unsigned int num_out = 0, num_in = 0; + struct virtio_crypto_op_ctrl_req *ctrl; + struct virtio_crypto_inhdr *ctrl_status; + struct virtio_crypto_ctrl_request *vc_ctrl_req; + + vc_ctrl_req = kzalloc_obj(*vc_ctrl_req); + if (!vc_ctrl_req) + return -ENOMEM; + + ctrl_status = &vc_ctrl_req->ctrl_status; + ctrl_status->status = VIRTIO_CRYPTO_ERR; + /* Pad ctrl header */ + ctrl = &vc_ctrl_req->ctrl; + ctrl->header.opcode = cpu_to_le32(VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION); + /* Set the default virtqueue id to 0 */ + ctrl->header.queue_id = 0; + + destroy_session = &ctrl->u.destroy_session; + + if (encrypt) + destroy_session->session_id = cpu_to_le64(ctx->enc_sess_info.session_id); + else + destroy_session->session_id = cpu_to_le64(ctx->dec_sess_info.session_id); + + sg_init_one(&outhdr, ctrl, sizeof(*ctrl)); + sgs[num_out++] = &outhdr; + + /* Return status and session id back */ + sg_init_one(&status_sg, &ctrl_status->status, sizeof(ctrl_status->status)); + sgs[num_out + num_in++] = &status_sg; + + err = virtio_crypto_ctrl_vq_request(vcrypto, sgs, num_out, num_in, vc_ctrl_req); + if (err < 0) + goto out; + + if (ctrl_status->status != VIRTIO_CRYPTO_OK) { + pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n", + ctrl_status->status, + le64_to_cpu(destroy_session->session_id)); + + err = -EINVAL; + goto out; + } + + err = 0; +out: + kfree(vc_ctrl_req); + return err; +} + +static int virtio_crypto_alg_skcipher_init_sessions( + struct virtio_crypto_skcipher_ctx *ctx, + const uint8_t *key, unsigned int keylen) +{ + uint32_t alg; + int ret; + struct virtio_crypto *vcrypto = ctx->vcrypto; + + if (keylen > vcrypto->max_cipher_key_len) { + pr_err("virtio_crypto: the key is too long\n"); + return -EINVAL; + } + + if (virtio_crypto_alg_validate_key(keylen, &alg)) + return -EINVAL; + + /* Create encryption session */ + ret = virtio_crypto_alg_skcipher_init_session(ctx, + alg, key, keylen, 1); + if (ret) + return ret; + /* Create decryption session */ + ret = virtio_crypto_alg_skcipher_init_session(ctx, + alg, key, keylen, 0); + if (ret) { + virtio_crypto_alg_skcipher_close_session(ctx, 1); + return ret; + } + return 0; +} + +/* Note: kernel crypto API realization */ +static int virtio_crypto_skcipher_setkey(struct crypto_skcipher *tfm, + const uint8_t *key, + unsigned int keylen) +{ + struct virtio_crypto_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); + uint32_t alg; + int ret; + + ret = virtio_crypto_alg_validate_key(keylen, &alg); + if (ret) + return ret; + + if (!ctx->vcrypto) { + /* New key */ + int node = virtio_crypto_get_current_node(); + struct virtio_crypto *vcrypto = + virtcrypto_get_dev_node(node, + VIRTIO_CRYPTO_SERVICE_CIPHER, alg); + if (!vcrypto) { + pr_err("virtio_crypto: Could not find a virtio device in the system or unsupported algo\n"); + return -ENODEV; + } + + ctx->vcrypto = vcrypto; + } else { + /* Rekeying, we should close the created sessions previously */ + virtio_crypto_alg_skcipher_close_session(ctx, 1); + virtio_crypto_alg_skcipher_close_session(ctx, 0); + } + + ret = virtio_crypto_alg_skcipher_init_sessions(ctx, key, keylen); + if (ret) { + virtcrypto_dev_put(ctx->vcrypto); + ctx->vcrypto = NULL; + + return ret; + } + + return 0; +} + +static int +__virtio_crypto_skcipher_do_req(struct virtio_crypto_sym_request *vc_sym_req, + struct skcipher_request *req, + struct data_queue *data_vq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct virtio_crypto_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); + struct virtio_crypto_request *vc_req = &vc_sym_req->base; + unsigned int ivsize = crypto_skcipher_ivsize(tfm); + struct virtio_crypto *vcrypto = ctx->vcrypto; + struct virtio_crypto_op_data_req *req_data; + int src_nents, dst_nents; + int err; + unsigned long flags; + struct scatterlist outhdr, iv_sg, status_sg, **sgs; + u64 dst_len; + unsigned int num_out = 0, num_in = 0; + int sg_total; + uint8_t *iv; + struct scatterlist *sg; + + src_nents = sg_nents_for_len(req->src, req->cryptlen); + if (src_nents < 0) { + pr_err("Invalid number of src SG.\n"); + return src_nents; + } + + dst_nents = sg_nents(req->dst); + + pr_debug("virtio_crypto: Number of sgs (src_nents: %d, dst_nents: %d)\n", + src_nents, dst_nents); + + /* Why 3? outhdr + iv + inhdr */ + sg_total = src_nents + dst_nents + 3; + sgs = kcalloc_node(sg_total, sizeof(*sgs), GFP_KERNEL, + dev_to_node(&vcrypto->vdev->dev)); + if (!sgs) + return -ENOMEM; + + req_data = kzalloc_node(sizeof(*req_data), GFP_KERNEL, + dev_to_node(&vcrypto->vdev->dev)); + if (!req_data) { + kfree(sgs); + return -ENOMEM; + } + + vc_req->req_data = req_data; + vc_sym_req->type = VIRTIO_CRYPTO_SYM_OP_CIPHER; + /* Head of operation */ + if (vc_sym_req->encrypt) { + req_data->header.session_id = + cpu_to_le64(ctx->enc_sess_info.session_id); + req_data->header.opcode = + cpu_to_le32(VIRTIO_CRYPTO_CIPHER_ENCRYPT); + } else { + req_data->header.session_id = + cpu_to_le64(ctx->dec_sess_info.session_id); + req_data->header.opcode = + cpu_to_le32(VIRTIO_CRYPTO_CIPHER_DECRYPT); + } + req_data->u.sym_req.op_type = cpu_to_le32(VIRTIO_CRYPTO_SYM_OP_CIPHER); + req_data->u.sym_req.u.cipher.para.iv_len = cpu_to_le32(ivsize); + req_data->u.sym_req.u.cipher.para.src_data_len = + cpu_to_le32(req->cryptlen); + + dst_len = virtio_crypto_alg_sg_nents_length(req->dst); + if (unlikely(dst_len > U32_MAX)) { + pr_err("virtio_crypto: The dst_len is beyond U32_MAX\n"); + err = -EINVAL; + goto free; + } + + dst_len = min_t(unsigned int, req->cryptlen, dst_len); + pr_debug("virtio_crypto: src_len: %u, dst_len: %llu\n", + req->cryptlen, dst_len); + + if (unlikely(req->cryptlen + dst_len + ivsize + + sizeof(vc_req->status) > vcrypto->max_size)) { + pr_err("virtio_crypto: The length is too big\n"); + err = -EINVAL; + goto free; + } + + req_data->u.sym_req.u.cipher.para.dst_data_len = + cpu_to_le32((uint32_t)dst_len); + + /* Outhdr */ + sg_init_one(&outhdr, req_data, sizeof(*req_data)); + sgs[num_out++] = &outhdr; + + /* IV */ + + /* + * Avoid to do DMA from the stack, switch to using + * dynamically-allocated for the IV + */ + iv = kzalloc_node(ivsize, GFP_ATOMIC, + dev_to_node(&vcrypto->vdev->dev)); + if (!iv) { + err = -ENOMEM; + goto free; + } + memcpy(iv, req->iv, ivsize); + if (!vc_sym_req->encrypt) + scatterwalk_map_and_copy(req->iv, req->src, + req->cryptlen - AES_BLOCK_SIZE, + AES_BLOCK_SIZE, 0); + + sg_init_one(&iv_sg, iv, ivsize); + sgs[num_out++] = &iv_sg; + vc_sym_req->iv = iv; + + /* Source data */ + for (sg = req->src; src_nents; sg = sg_next(sg), src_nents--) + sgs[num_out++] = sg; + + /* Destination data */ + for (sg = req->dst; sg; sg = sg_next(sg)) + sgs[num_out + num_in++] = sg; + + /* Status */ + sg_init_one(&status_sg, &vc_req->status, sizeof(vc_req->status)); + sgs[num_out + num_in++] = &status_sg; + + vc_req->sgs = sgs; + + spin_lock_irqsave(&data_vq->lock, flags); + err = virtqueue_add_sgs(data_vq->vq, sgs, num_out, + num_in, vc_req, GFP_ATOMIC); + virtqueue_kick(data_vq->vq); + spin_unlock_irqrestore(&data_vq->lock, flags); + if (unlikely(err < 0)) + goto free_iv; + + return 0; + +free_iv: + kfree_sensitive(iv); +free: + kfree_sensitive(req_data); + kfree(sgs); + return err; +} + +static int virtio_crypto_skcipher_encrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *atfm = crypto_skcipher_reqtfm(req); + struct virtio_crypto_skcipher_ctx *ctx = crypto_skcipher_ctx(atfm); + struct virtio_crypto_sym_request *vc_sym_req = + skcipher_request_ctx(req); + struct virtio_crypto_request *vc_req = &vc_sym_req->base; + struct virtio_crypto *vcrypto = ctx->vcrypto; + /* Use the first data virtqueue as default */ + struct data_queue *data_vq = &vcrypto->data_vq[0]; + + if (!req->cryptlen) + return 0; + if (req->cryptlen % AES_BLOCK_SIZE) + return -EINVAL; + + vc_req->dataq = data_vq; + vc_req->alg_cb = virtio_crypto_dataq_sym_callback; + vc_sym_req->encrypt = true; + + return crypto_transfer_skcipher_request_to_engine(data_vq->engine, req); +} + +static int virtio_crypto_skcipher_decrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *atfm = crypto_skcipher_reqtfm(req); + struct virtio_crypto_skcipher_ctx *ctx = crypto_skcipher_ctx(atfm); + struct virtio_crypto_sym_request *vc_sym_req = + skcipher_request_ctx(req); + struct virtio_crypto_request *vc_req = &vc_sym_req->base; + struct virtio_crypto *vcrypto = ctx->vcrypto; + /* Use the first data virtqueue as default */ + struct data_queue *data_vq = &vcrypto->data_vq[0]; + + if (!req->cryptlen) + return 0; + if (req->cryptlen % AES_BLOCK_SIZE) + return -EINVAL; + + vc_req->dataq = data_vq; + vc_req->alg_cb = virtio_crypto_dataq_sym_callback; + vc_sym_req->encrypt = false; + + return crypto_transfer_skcipher_request_to_engine(data_vq->engine, req); +} + +static int virtio_crypto_skcipher_init(struct crypto_skcipher *tfm) +{ + crypto_skcipher_set_reqsize(tfm, sizeof(struct virtio_crypto_sym_request)); + + return 0; +} + +static void virtio_crypto_skcipher_exit(struct crypto_skcipher *tfm) +{ + struct virtio_crypto_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (!ctx->vcrypto) + return; + + virtio_crypto_alg_skcipher_close_session(ctx, 1); + virtio_crypto_alg_skcipher_close_session(ctx, 0); + virtcrypto_dev_put(ctx->vcrypto); + ctx->vcrypto = NULL; +} + +int virtio_crypto_skcipher_crypt_req( + struct crypto_engine *engine, void *vreq) +{ + struct skcipher_request *req = container_of(vreq, struct skcipher_request, base); + struct virtio_crypto_sym_request *vc_sym_req = + skcipher_request_ctx(req); + struct virtio_crypto_request *vc_req = &vc_sym_req->base; + struct data_queue *data_vq = vc_req->dataq; + int ret; + + ret = __virtio_crypto_skcipher_do_req(vc_sym_req, req, data_vq); + if (ret < 0) + return ret; + + return 0; +} + +static void virtio_crypto_skcipher_finalize_req( + struct virtio_crypto_sym_request *vc_sym_req, + struct skcipher_request *req, + int err) +{ + if (vc_sym_req->encrypt) + scatterwalk_map_and_copy(req->iv, req->dst, + req->cryptlen - AES_BLOCK_SIZE, + AES_BLOCK_SIZE, 0); + kfree_sensitive(vc_sym_req->iv); + virtcrypto_clear_request(&vc_sym_req->base); + + crypto_finalize_skcipher_request(vc_sym_req->base.dataq->engine, + req, err); +} + +static struct virtio_crypto_algo virtio_crypto_algs[] = { { + .algonum = VIRTIO_CRYPTO_CIPHER_AES_CBC, + .service = VIRTIO_CRYPTO_SERVICE_CIPHER, + .algo.base = { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "virtio_crypto_aes_cbc", + .base.cra_priority = 150, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct virtio_crypto_skcipher_ctx), + .base.cra_module = THIS_MODULE, + .init = virtio_crypto_skcipher_init, + .exit = virtio_crypto_skcipher_exit, + .setkey = virtio_crypto_skcipher_setkey, + .decrypt = virtio_crypto_skcipher_decrypt, + .encrypt = virtio_crypto_skcipher_encrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .algo.op = { + .do_one_request = virtio_crypto_skcipher_crypt_req, + }, +} }; + +int virtio_crypto_skcipher_algs_register(struct virtio_crypto *vcrypto) +{ + int ret = 0; + int i = 0; + + mutex_lock(&algs_lock); + + for (i = 0; i < ARRAY_SIZE(virtio_crypto_algs); i++) { + + uint32_t service = virtio_crypto_algs[i].service; + uint32_t algonum = virtio_crypto_algs[i].algonum; + + if (!virtcrypto_algo_is_supported(vcrypto, service, algonum)) + continue; + + if (virtio_crypto_algs[i].active_devs == 0) { + ret = crypto_engine_register_skcipher(&virtio_crypto_algs[i].algo); + if (ret) + goto unlock; + } + + virtio_crypto_algs[i].active_devs++; + dev_info(&vcrypto->vdev->dev, "Registered algo %s\n", + virtio_crypto_algs[i].algo.base.base.cra_name); + } + +unlock: + mutex_unlock(&algs_lock); + return ret; +} + +void virtio_crypto_skcipher_algs_unregister(struct virtio_crypto *vcrypto) +{ + int i = 0; + + mutex_lock(&algs_lock); + + for (i = 0; i < ARRAY_SIZE(virtio_crypto_algs); i++) { + + uint32_t service = virtio_crypto_algs[i].service; + uint32_t algonum = virtio_crypto_algs[i].algonum; + + if (virtio_crypto_algs[i].active_devs == 0 || + !virtcrypto_algo_is_supported(vcrypto, service, algonum)) + continue; + + if (virtio_crypto_algs[i].active_devs == 1) + crypto_engine_unregister_skcipher(&virtio_crypto_algs[i].algo); + + virtio_crypto_algs[i].active_devs--; + } + + mutex_unlock(&algs_lock); +} diff --git a/drivers/crypto/xilinx/Makefile b/drivers/crypto/xilinx/Makefile new file mode 100644 index 000000000..730feff5b --- /dev/null +++ b/drivers/crypto/xilinx/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_ZYNQMP_AES) += zynqmp-aes-gcm.o +obj-$(CONFIG_CRYPTO_DEV_ZYNQMP_SHA3) += zynqmp-sha.o diff --git a/drivers/crypto/xilinx/zynqmp-aes-gcm.c b/drivers/crypto/xilinx/zynqmp-aes-gcm.c new file mode 100644 index 000000000..d54c268df --- /dev/null +++ b/drivers/crypto/xilinx/zynqmp-aes-gcm.c @@ -0,0 +1,1026 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Xilinx ZynqMP AES Driver. + * Copyright (C) 2020-2022 Xilinx Inc. + * Copyright (C) 2022-2025 Advanced Micro Devices, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define ZYNQMP_DMA_BIT_MASK 32U +#define VERSAL_DMA_BIT_MASK 64U +#define XILINX_AES_AUTH_SIZE 16U +#define XILINX_AES_BLK_SIZE 1U +#define ZYNQMP_AES_MIN_INPUT_BLK_SIZE 4U +#define ZYNQMP_AES_WORD_LEN 4U + +#define VERSAL_AES_QWORD_LEN 16U +#define ZYNQMP_AES_GCM_TAG_MISMATCH_ERR 0x01 +#define ZYNQMP_AES_WRONG_KEY_SRC_ERR 0x13 +#define ZYNQMP_AES_PUF_NOT_PROGRAMMED 0xE300 +#define XILINX_KEY_MAGIC 0x3EA0 + +enum xilinx_aead_op { + XILINX_AES_DECRYPT = 0, + XILINX_AES_ENCRYPT +}; + +enum zynqmp_aead_keysrc { + ZYNQMP_AES_KUP_KEY = 0, + ZYNQMP_AES_DEV_KEY, + ZYNQMP_AES_PUF_KEY +}; + +struct xilinx_aead_dev { + struct device *dev; + struct crypto_engine *engine; + struct xilinx_aead_alg *aead_algs; +}; + +struct xilinx_aead_alg { + struct xilinx_aead_dev *aead_dev; + struct aead_engine_alg aead; + int (*aes_aead_cipher)(struct aead_request *areq); + u8 dma_bit_mask; +}; + +struct xilinx_hwkey_info { + u16 magic; + u16 type; +} __packed; + +struct zynqmp_aead_hw_req { + u64 src; + u64 iv; + u64 key; + u64 dst; + u64 size; + u64 op; + u64 keysrc; +}; + +struct xilinx_aead_tfm_ctx { + struct device *dev; + dma_addr_t key_dma_addr; + u8 *key; + u32 keylen; + u32 authsize; + u8 keysrc; + struct crypto_aead *fbk_cipher; +}; + +struct xilinx_aead_req_ctx { + enum xilinx_aead_op op; +}; + +static struct xilinx_aead_dev *aead_dev; + +enum versal_aead_keysrc { + VERSAL_AES_BBRAM_KEY = 0, + VERSAL_AES_BBRAM_RED_KEY, + VERSAL_AES_BH_KEY, + VERSAL_AES_BH_RED_KEY, + VERSAL_AES_EFUSE_KEY, + VERSAL_AES_EFUSE_RED_KEY, + VERSAL_AES_EFUSE_USER_KEY_0, + VERSAL_AES_EFUSE_USER_KEY_1, + VERSAL_AES_EFUSE_USER_RED_KEY_0, + VERSAL_AES_EFUSE_USER_RED_KEY_1, + VERSAL_AES_KUP_KEY, + VERSAL_AES_PUF_KEY, + VERSAL_AES_USER_KEY_0, + VERSAL_AES_USER_KEY_1, + VERSAL_AES_USER_KEY_2, + VERSAL_AES_USER_KEY_3, + VERSAL_AES_USER_KEY_4, + VERSAL_AES_USER_KEY_5, + VERSAL_AES_USER_KEY_6, + VERSAL_AES_USER_KEY_7, + VERSAL_AES_EXPANDED_KEYS, + VERSAL_AES_ALL_KEYS, +}; + +enum versal_aead_op { + VERSAL_AES_ENCRYPT = 0, + VERSAL_AES_DECRYPT +}; + +enum versal_aes_keysize { + HW_AES_KEY_SIZE_128 = 0, + HW_AES_KEY_SIZE_256 = 2, +}; + +struct versal_init_ops { + u64 iv; + u32 op; + u32 keysrc; + u32 size; +}; + +struct versal_in_params { + u64 in_data_addr; + u32 size; + u32 is_last; +}; + +static int zynqmp_aes_aead_cipher(struct aead_request *req) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead); + struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req); + dma_addr_t dma_addr_data, dma_addr_hw_req; + struct device *dev = tfm_ctx->dev; + struct zynqmp_aead_hw_req *hwreq; + unsigned int data_size; + unsigned int status; + int ret; + size_t dma_size; + void *dmabuf; + char *kbuf; + + dma_size = req->cryptlen + XILINX_AES_AUTH_SIZE; + kbuf = kmalloc(dma_size, GFP_KERNEL); + if (!kbuf) + return -ENOMEM; + + dmabuf = kmalloc(sizeof(*hwreq) + GCM_AES_IV_SIZE, GFP_KERNEL); + if (!dmabuf) { + kfree(kbuf); + return -ENOMEM; + } + hwreq = dmabuf; + data_size = req->cryptlen; + scatterwalk_map_and_copy(kbuf, req->src, 0, req->cryptlen, 0); + memcpy(dmabuf + sizeof(struct zynqmp_aead_hw_req), req->iv, GCM_AES_IV_SIZE); + dma_addr_data = dma_map_single(dev, kbuf, dma_size, DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(dev, dma_addr_data))) { + ret = -ENOMEM; + goto freemem; + } + + hwreq->src = dma_addr_data; + hwreq->dst = dma_addr_data; + hwreq->keysrc = tfm_ctx->keysrc; + hwreq->op = rq_ctx->op; + + if (hwreq->op == XILINX_AES_ENCRYPT) + hwreq->size = data_size; + else + hwreq->size = data_size - XILINX_AES_AUTH_SIZE; + + if (hwreq->keysrc == ZYNQMP_AES_KUP_KEY) + hwreq->key = tfm_ctx->key_dma_addr; + else + hwreq->key = 0; + + dma_addr_hw_req = dma_map_single(dev, dmabuf, sizeof(struct zynqmp_aead_hw_req) + + GCM_AES_IV_SIZE, + DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, dma_addr_hw_req))) { + ret = -ENOMEM; + dma_unmap_single(dev, dma_addr_data, dma_size, DMA_BIDIRECTIONAL); + goto freemem; + } + hwreq->iv = dma_addr_hw_req + sizeof(struct zynqmp_aead_hw_req); + dma_sync_single_for_device(dev, dma_addr_hw_req, sizeof(struct zynqmp_aead_hw_req) + + GCM_AES_IV_SIZE, DMA_TO_DEVICE); + ret = zynqmp_pm_aes_engine(dma_addr_hw_req, &status); + dma_unmap_single(dev, dma_addr_hw_req, sizeof(struct zynqmp_aead_hw_req) + GCM_AES_IV_SIZE, + DMA_TO_DEVICE); + dma_unmap_single(dev, dma_addr_data, dma_size, DMA_BIDIRECTIONAL); + if (ret) { + dev_err(dev, "ERROR: AES PM API failed\n"); + } else if (status) { + switch (status) { + case ZYNQMP_AES_GCM_TAG_MISMATCH_ERR: + ret = -EBADMSG; + break; + case ZYNQMP_AES_WRONG_KEY_SRC_ERR: + ret = -EINVAL; + dev_err(dev, "ERROR: Wrong KeySrc, enable secure mode\n"); + break; + case ZYNQMP_AES_PUF_NOT_PROGRAMMED: + ret = -EINVAL; + dev_err(dev, "ERROR: PUF is not registered\n"); + break; + default: + ret = -EINVAL; + break; + } + } else { + if (hwreq->op == XILINX_AES_ENCRYPT) + data_size = data_size + crypto_aead_authsize(aead); + else + data_size = data_size - XILINX_AES_AUTH_SIZE; + + sg_copy_from_buffer(req->dst, sg_nents(req->dst), + kbuf, data_size); + ret = 0; + } + +freemem: + memzero_explicit(kbuf, dma_size); + kfree(kbuf); + memzero_explicit(dmabuf, sizeof(struct zynqmp_aead_hw_req) + GCM_AES_IV_SIZE); + kfree(dmabuf); + + return ret; +} + +static int versal_aes_aead_cipher(struct aead_request *req) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead); + struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req); + dma_addr_t dma_addr_data, dma_addr_hw_req, dma_addr_in; + u32 total_len = req->assoclen + req->cryptlen; + struct device *dev = tfm_ctx->dev; + struct versal_init_ops *hwreq; + struct versal_in_params *in; + u32 gcm_offset, out_len; + size_t dmabuf_size; + size_t kbuf_size; + void *dmabuf; + char *kbuf; + int ret; + + kbuf_size = total_len + XILINX_AES_AUTH_SIZE; + kbuf = kmalloc(kbuf_size, GFP_KERNEL); + if (unlikely(!kbuf)) { + ret = -ENOMEM; + goto err; + } + dmabuf_size = sizeof(struct versal_init_ops) + + sizeof(struct versal_in_params) + + GCM_AES_IV_SIZE; + dmabuf = kmalloc(dmabuf_size, GFP_KERNEL); + if (unlikely(!dmabuf)) { + ret = -ENOMEM; + goto buf1_free; + } + + dma_addr_hw_req = dma_map_single(dev, dmabuf, dmabuf_size, DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(dev, dma_addr_hw_req))) { + ret = -ENOMEM; + goto buf2_free; + } + scatterwalk_map_and_copy(kbuf, req->src, 0, total_len, 0); + dma_addr_data = dma_map_single(dev, kbuf, kbuf_size, DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(dev, dma_addr_data))) { + dma_unmap_single(dev, dma_addr_hw_req, dmabuf_size, DMA_BIDIRECTIONAL); + ret = -ENOMEM; + goto buf2_free; + } + hwreq = dmabuf; + in = dmabuf + sizeof(struct versal_init_ops); + memcpy(dmabuf + sizeof(struct versal_init_ops) + + sizeof(struct versal_in_params), req->iv, GCM_AES_IV_SIZE); + hwreq->iv = dma_addr_hw_req + sizeof(struct versal_init_ops) + + sizeof(struct versal_in_params); + hwreq->keysrc = tfm_ctx->keysrc; + dma_addr_in = dma_addr_hw_req + sizeof(struct versal_init_ops); + if (rq_ctx->op == XILINX_AES_ENCRYPT) { + hwreq->op = VERSAL_AES_ENCRYPT; + out_len = total_len + crypto_aead_authsize(aead); + in->size = req->cryptlen; + } else { + hwreq->op = VERSAL_AES_DECRYPT; + out_len = total_len - XILINX_AES_AUTH_SIZE; + in->size = req->cryptlen - XILINX_AES_AUTH_SIZE; + } + + if (tfm_ctx->keylen == AES_KEYSIZE_128) + hwreq->size = HW_AES_KEY_SIZE_128; + else + hwreq->size = HW_AES_KEY_SIZE_256; + + /* Request aes key write for volatile user keys */ + if (hwreq->keysrc >= VERSAL_AES_USER_KEY_0 && hwreq->keysrc <= VERSAL_AES_USER_KEY_7) { + ret = versal_pm_aes_key_write(hwreq->size, hwreq->keysrc, + tfm_ctx->key_dma_addr); + if (ret) + goto unmap; + } + + in->in_data_addr = dma_addr_data + req->assoclen; + in->is_last = 1; + gcm_offset = req->assoclen + in->size; + dma_sync_single_for_device(dev, dma_addr_hw_req, dmabuf_size, DMA_BIDIRECTIONAL); + ret = versal_pm_aes_op_init(dma_addr_hw_req); + if (ret) + goto clearkey; + + if (req->assoclen > 0) { + /* Currently GMAC is OFF by default */ + ret = versal_pm_aes_update_aad(dma_addr_data, req->assoclen); + if (ret) + goto clearkey; + } + if (rq_ctx->op == XILINX_AES_ENCRYPT) { + ret = versal_pm_aes_enc_update(dma_addr_in, + dma_addr_data + req->assoclen); + if (ret) + goto clearkey; + + ret = versal_pm_aes_enc_final(dma_addr_data + gcm_offset); + if (ret) + goto clearkey; + } else { + ret = versal_pm_aes_dec_update(dma_addr_in, + dma_addr_data + req->assoclen); + if (ret) + goto clearkey; + + ret = versal_pm_aes_dec_final(dma_addr_data + gcm_offset); + if (ret) { + ret = -EBADMSG; + goto clearkey; + } + } + dma_unmap_single(dev, dma_addr_data, kbuf_size, DMA_BIDIRECTIONAL); + dma_unmap_single(dev, dma_addr_hw_req, dmabuf_size, DMA_BIDIRECTIONAL); + sg_copy_from_buffer(req->dst, sg_nents(req->dst), + kbuf, out_len); + dma_addr_data = 0; + dma_addr_hw_req = 0; + +clearkey: + if (hwreq->keysrc >= VERSAL_AES_USER_KEY_0 && hwreq->keysrc <= VERSAL_AES_USER_KEY_7) + versal_pm_aes_key_zero(hwreq->keysrc); +unmap: + if (unlikely(dma_addr_data)) + dma_unmap_single(dev, dma_addr_data, kbuf_size, DMA_BIDIRECTIONAL); + if (unlikely(dma_addr_hw_req)) + dma_unmap_single(dev, dma_addr_hw_req, dmabuf_size, DMA_BIDIRECTIONAL); +buf2_free: + memzero_explicit(dmabuf, dmabuf_size); + kfree(dmabuf); +buf1_free: + memzero_explicit(kbuf, kbuf_size); + kfree(kbuf); +err: + return ret; +} + +static int zynqmp_fallback_check(struct xilinx_aead_tfm_ctx *tfm_ctx, + struct aead_request *req) +{ + struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req); + + if (tfm_ctx->authsize != XILINX_AES_AUTH_SIZE && rq_ctx->op == XILINX_AES_DECRYPT) + return 1; + + if (req->assoclen != 0 || + req->cryptlen < ZYNQMP_AES_MIN_INPUT_BLK_SIZE) + return 1; + if (tfm_ctx->keylen == AES_KEYSIZE_128 || + tfm_ctx->keylen == AES_KEYSIZE_192) + return 1; + + if ((req->cryptlen % ZYNQMP_AES_WORD_LEN) != 0) + return 1; + + if (rq_ctx->op == XILINX_AES_DECRYPT && + req->cryptlen <= XILINX_AES_AUTH_SIZE) + return 1; + + return 0; +} + +static int versal_fallback_check(struct xilinx_aead_tfm_ctx *tfm_ctx, + struct aead_request *req) +{ + struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req); + + if (tfm_ctx->authsize != XILINX_AES_AUTH_SIZE && rq_ctx->op == XILINX_AES_DECRYPT) + return 1; + + if (tfm_ctx->keylen == AES_KEYSIZE_192) + return 1; + + if (req->cryptlen < ZYNQMP_AES_MIN_INPUT_BLK_SIZE || + req->cryptlen % ZYNQMP_AES_WORD_LEN || + req->assoclen % VERSAL_AES_QWORD_LEN) + return 1; + + if (rq_ctx->op == XILINX_AES_DECRYPT && + req->cryptlen <= XILINX_AES_AUTH_SIZE) + return 1; + + return 0; +} + +static int xilinx_handle_aes_req(struct crypto_engine *engine, void *req) +{ + struct aead_request *areq = + container_of(req, struct aead_request, base); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct aead_alg *alg = crypto_aead_alg(aead); + struct xilinx_aead_alg *drv_ctx; + int err; + + drv_ctx = container_of(alg, struct xilinx_aead_alg, aead.base); + err = drv_ctx->aes_aead_cipher(areq); + local_bh_disable(); + crypto_finalize_aead_request(engine, areq, err); + local_bh_enable(); + + return 0; +} + +static int zynqmp_aes_aead_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(aead); + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); + int err; + + if (keylen == AES_KEYSIZE_256) { + memcpy(tfm_ctx->key, key, keylen); + dma_sync_single_for_device(tfm_ctx->dev, tfm_ctx->key_dma_addr, + AES_KEYSIZE_256, + DMA_TO_DEVICE); + } + + tfm_ctx->fbk_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; + tfm_ctx->fbk_cipher->base.crt_flags |= (aead->base.crt_flags & + CRYPTO_TFM_REQ_MASK); + + err = crypto_aead_setkey(tfm_ctx->fbk_cipher, key, keylen); + if (err) + goto err; + tfm_ctx->keylen = keylen; + tfm_ctx->keysrc = ZYNQMP_AES_KUP_KEY; +err: + return err; +} + +static int zynqmp_paes_aead_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(aead); + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); + struct xilinx_hwkey_info hwkey; + unsigned char keysrc; + int err = -EINVAL; + + if (keylen != sizeof(struct xilinx_hwkey_info)) + return -EINVAL; + memcpy(&hwkey, key, sizeof(struct xilinx_hwkey_info)); + if (hwkey.magic != XILINX_KEY_MAGIC) + return -EINVAL; + keysrc = hwkey.type; + if (keysrc == ZYNQMP_AES_DEV_KEY || + keysrc == ZYNQMP_AES_PUF_KEY) { + tfm_ctx->keysrc = keysrc; + tfm_ctx->keylen = sizeof(struct xilinx_hwkey_info); + err = 0; + } + + return err; +} + +static int versal_aes_aead_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(aead); + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); + struct xilinx_hwkey_info hwkey; + unsigned char keysrc; + int err; + + tfm_ctx->keysrc = VERSAL_AES_USER_KEY_0; + if (keylen == sizeof(struct xilinx_hwkey_info)) { + memcpy(&hwkey, key, sizeof(struct xilinx_hwkey_info)); + if (hwkey.magic != XILINX_KEY_MAGIC) + return -EINVAL; + + keysrc = hwkey.type; + if (keysrc >= VERSAL_AES_USER_KEY_1 && + keysrc <= VERSAL_AES_USER_KEY_7) { + tfm_ctx->keysrc = keysrc; + tfm_ctx->keylen = sizeof(struct xilinx_hwkey_info); + return 0; + } + return -EINVAL; + } + + if (keylen == AES_KEYSIZE_256 || keylen == AES_KEYSIZE_128) { + tfm_ctx->keylen = keylen; + memcpy(tfm_ctx->key, key, keylen); + dma_sync_single_for_device(tfm_ctx->dev, tfm_ctx->key_dma_addr, + AES_KEYSIZE_256, + DMA_TO_DEVICE); + } + + tfm_ctx->fbk_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; + tfm_ctx->fbk_cipher->base.crt_flags |= (aead->base.crt_flags & + CRYPTO_TFM_REQ_MASK); + err = crypto_aead_setkey(tfm_ctx->fbk_cipher, key, keylen); + if (!err) + tfm_ctx->keylen = keylen; + + return err; +} + +static int versal_paes_aead_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(aead); + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); + struct xilinx_hwkey_info hwkey; + unsigned char keysrc; + int err = 0; + + if (keylen != sizeof(struct xilinx_hwkey_info)) + return -EINVAL; + + memcpy(&hwkey, key, sizeof(struct xilinx_hwkey_info)); + if (hwkey.magic != XILINX_KEY_MAGIC) + return -EINVAL; + + keysrc = hwkey.type; + + switch (keysrc) { + case VERSAL_AES_EFUSE_USER_KEY_0: + case VERSAL_AES_EFUSE_USER_KEY_1: + case VERSAL_AES_EFUSE_USER_RED_KEY_0: + case VERSAL_AES_EFUSE_USER_RED_KEY_1: + case VERSAL_AES_PUF_KEY: + tfm_ctx->keysrc = keysrc; + tfm_ctx->keylen = sizeof(struct xilinx_hwkey_info); + break; + default: + err = -EINVAL; + break; + } + + return err; +} + +static int xilinx_aes_aead_setauthsize(struct crypto_aead *aead, + unsigned int authsize) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(aead); + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); + + tfm_ctx->authsize = authsize; + return tfm_ctx->fbk_cipher ? crypto_aead_setauthsize(tfm_ctx->fbk_cipher, authsize) : 0; +} + +static int xilinx_aes_fallback_crypt(struct aead_request *req, bool encrypt) +{ + struct aead_request *subreq = aead_request_ctx(req); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead); + + aead_request_set_tfm(subreq, tfm_ctx->fbk_cipher); + aead_request_set_callback(subreq, req->base.flags, NULL, NULL); + aead_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + aead_request_set_ad(subreq, req->assoclen); + + return encrypt ? crypto_aead_encrypt(subreq) : crypto_aead_decrypt(subreq); +} + +static int zynqmp_aes_aead_encrypt(struct aead_request *req) +{ + struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead); + struct aead_alg *alg = crypto_aead_alg(aead); + struct xilinx_aead_alg *drv_ctx; + int err; + + drv_ctx = container_of(alg, struct xilinx_aead_alg, aead.base); + if (tfm_ctx->keysrc == ZYNQMP_AES_KUP_KEY && + tfm_ctx->keylen == sizeof(struct xilinx_hwkey_info)) + return -EINVAL; + + rq_ctx->op = XILINX_AES_ENCRYPT; + err = zynqmp_fallback_check(tfm_ctx, req); + if (err && tfm_ctx->keysrc != ZYNQMP_AES_KUP_KEY) + return -EOPNOTSUPP; + + if (err) + return xilinx_aes_fallback_crypt(req, true); + + return crypto_transfer_aead_request_to_engine(drv_ctx->aead_dev->engine, req); +} + +static int versal_aes_aead_encrypt(struct aead_request *req) +{ + struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead); + struct aead_alg *alg = crypto_aead_alg(aead); + struct xilinx_aead_alg *drv_ctx; + int err; + + drv_ctx = container_of(alg, struct xilinx_aead_alg, aead.base); + rq_ctx->op = XILINX_AES_ENCRYPT; + if (tfm_ctx->keysrc >= VERSAL_AES_USER_KEY_0 && + tfm_ctx->keysrc <= VERSAL_AES_USER_KEY_7 && + tfm_ctx->keylen == sizeof(struct xilinx_hwkey_info)) + return -EINVAL; + err = versal_fallback_check(tfm_ctx, req); + if (err && (tfm_ctx->keysrc < VERSAL_AES_USER_KEY_0 || + tfm_ctx->keysrc > VERSAL_AES_USER_KEY_7)) + return -EOPNOTSUPP; + if (err) + return xilinx_aes_fallback_crypt(req, true); + + return crypto_transfer_aead_request_to_engine(drv_ctx->aead_dev->engine, req); +} + +static int zynqmp_aes_aead_decrypt(struct aead_request *req) +{ + struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead); + struct aead_alg *alg = crypto_aead_alg(aead); + struct xilinx_aead_alg *drv_ctx; + int err; + + rq_ctx->op = XILINX_AES_DECRYPT; + drv_ctx = container_of(alg, struct xilinx_aead_alg, aead.base); + if (tfm_ctx->keysrc == ZYNQMP_AES_KUP_KEY && + tfm_ctx->keylen == sizeof(struct xilinx_hwkey_info)) + return -EINVAL; + err = zynqmp_fallback_check(tfm_ctx, req); + if (err && tfm_ctx->keysrc != ZYNQMP_AES_KUP_KEY) + return -EOPNOTSUPP; + if (err) + return xilinx_aes_fallback_crypt(req, false); + + return crypto_transfer_aead_request_to_engine(drv_ctx->aead_dev->engine, req); +} + +static int xilinx_paes_aead_init(struct crypto_aead *aead) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(aead); + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); + struct xilinx_aead_alg *drv_alg; + struct aead_alg *alg = crypto_aead_alg(aead); + + drv_alg = container_of(alg, struct xilinx_aead_alg, aead.base); + tfm_ctx->dev = drv_alg->aead_dev->dev; + tfm_ctx->keylen = 0; + tfm_ctx->key = NULL; + tfm_ctx->fbk_cipher = NULL; + crypto_aead_set_reqsize(aead, sizeof(struct xilinx_aead_req_ctx)); + + return 0; +} + +static int versal_aes_aead_decrypt(struct aead_request *req) +{ + struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead); + struct aead_alg *alg = crypto_aead_alg(aead); + struct xilinx_aead_alg *drv_ctx; + int err; + + drv_ctx = container_of(alg, struct xilinx_aead_alg, aead.base); + rq_ctx->op = XILINX_AES_DECRYPT; + if (tfm_ctx->keysrc >= VERSAL_AES_USER_KEY_0 && + tfm_ctx->keysrc <= VERSAL_AES_USER_KEY_7 && + tfm_ctx->keylen == sizeof(struct xilinx_hwkey_info)) + return -EINVAL; + + err = versal_fallback_check(tfm_ctx, req); + if (err && + (tfm_ctx->keysrc < VERSAL_AES_USER_KEY_0 || + tfm_ctx->keysrc > VERSAL_AES_USER_KEY_7)) + return -EOPNOTSUPP; + if (err) + return xilinx_aes_fallback_crypt(req, false); + + return crypto_transfer_aead_request_to_engine(drv_ctx->aead_dev->engine, req); +} + +static int xilinx_aes_aead_init(struct crypto_aead *aead) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(aead); + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); + struct xilinx_aead_alg *drv_ctx; + struct aead_alg *alg = crypto_aead_alg(aead); + + drv_ctx = container_of(alg, struct xilinx_aead_alg, aead.base); + tfm_ctx->dev = drv_ctx->aead_dev->dev; + tfm_ctx->keylen = 0; + + tfm_ctx->fbk_cipher = crypto_alloc_aead(drv_ctx->aead.base.base.cra_name, + 0, + CRYPTO_ALG_NEED_FALLBACK); + + if (IS_ERR(tfm_ctx->fbk_cipher)) { + dev_err(tfm_ctx->dev, "failed to allocate fallback for %s\n", + drv_ctx->aead.base.base.cra_name); + return PTR_ERR(tfm_ctx->fbk_cipher); + } + tfm_ctx->key = kmalloc(AES_KEYSIZE_256, GFP_KERNEL); + if (!tfm_ctx->key) { + crypto_free_aead(tfm_ctx->fbk_cipher); + return -ENOMEM; + } + tfm_ctx->key_dma_addr = dma_map_single(tfm_ctx->dev, tfm_ctx->key, + AES_KEYSIZE_256, + DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(tfm_ctx->dev, tfm_ctx->key_dma_addr))) { + kfree(tfm_ctx->key); + crypto_free_aead(tfm_ctx->fbk_cipher); + tfm_ctx->fbk_cipher = NULL; + return -ENOMEM; + } + crypto_aead_set_reqsize(aead, + max(sizeof(struct xilinx_aead_req_ctx), + sizeof(struct aead_request) + + crypto_aead_reqsize(tfm_ctx->fbk_cipher))); + return 0; +} + +static void xilinx_paes_aead_exit(struct crypto_aead *aead) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(aead); + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); + + memzero_explicit(tfm_ctx, sizeof(struct xilinx_aead_tfm_ctx)); +} + +static void xilinx_aes_aead_exit(struct crypto_aead *aead) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(aead); + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); + + dma_unmap_single(tfm_ctx->dev, tfm_ctx->key_dma_addr, AES_KEYSIZE_256, DMA_TO_DEVICE); + kfree(tfm_ctx->key); + if (tfm_ctx->fbk_cipher) { + crypto_free_aead(tfm_ctx->fbk_cipher); + tfm_ctx->fbk_cipher = NULL; + } + memzero_explicit(tfm_ctx, sizeof(struct xilinx_aead_tfm_ctx)); +} + +static struct xilinx_aead_alg zynqmp_aes_algs[] = { + { + .aes_aead_cipher = zynqmp_aes_aead_cipher, + .aead.base = { + .setkey = zynqmp_aes_aead_setkey, + .setauthsize = xilinx_aes_aead_setauthsize, + .encrypt = zynqmp_aes_aead_encrypt, + .decrypt = zynqmp_aes_aead_decrypt, + .init = xilinx_aes_aead_init, + .exit = xilinx_aes_aead_exit, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = XILINX_AES_AUTH_SIZE, + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "xilinx-zynqmp-aes-gcm", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_AEAD | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = XILINX_AES_BLK_SIZE, + .cra_ctxsize = sizeof(struct xilinx_aead_tfm_ctx), + .cra_module = THIS_MODULE, + } + }, + .aead.op = { + .do_one_request = xilinx_handle_aes_req, + }, + .dma_bit_mask = ZYNQMP_DMA_BIT_MASK, + }, + { + .aes_aead_cipher = zynqmp_aes_aead_cipher, + .aead.base = { + .setkey = zynqmp_paes_aead_setkey, + .setauthsize = xilinx_aes_aead_setauthsize, + .encrypt = zynqmp_aes_aead_encrypt, + .decrypt = zynqmp_aes_aead_decrypt, + .init = xilinx_paes_aead_init, + .exit = xilinx_paes_aead_exit, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = XILINX_AES_AUTH_SIZE, + .base = { + .cra_name = "gcm(paes)", + .cra_driver_name = "xilinx-zynqmp-paes-gcm", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_AEAD | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = XILINX_AES_BLK_SIZE, + .cra_ctxsize = sizeof(struct xilinx_aead_tfm_ctx), + .cra_module = THIS_MODULE, + } + }, + .aead.op = { + .do_one_request = xilinx_handle_aes_req, + }, + .dma_bit_mask = ZYNQMP_DMA_BIT_MASK, + }, + { /* sentinel */ } +}; + +static struct xilinx_aead_alg versal_aes_algs[] = { + { + .aes_aead_cipher = versal_aes_aead_cipher, + .aead.base = { + .setkey = versal_aes_aead_setkey, + .setauthsize = xilinx_aes_aead_setauthsize, + .encrypt = versal_aes_aead_encrypt, + .decrypt = versal_aes_aead_decrypt, + .init = xilinx_aes_aead_init, + .exit = xilinx_aes_aead_exit, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = XILINX_AES_AUTH_SIZE, + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "versal-aes-gcm", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AEAD | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = XILINX_AES_BLK_SIZE, + .cra_ctxsize = sizeof(struct xilinx_aead_tfm_ctx), + .cra_module = THIS_MODULE, + } + }, + .aead.op = { + .do_one_request = xilinx_handle_aes_req, + }, + .dma_bit_mask = VERSAL_DMA_BIT_MASK, + }, + { + .aes_aead_cipher = versal_aes_aead_cipher, + .aead.base = { + .setkey = versal_paes_aead_setkey, + .setauthsize = xilinx_aes_aead_setauthsize, + .encrypt = versal_aes_aead_encrypt, + .decrypt = versal_aes_aead_decrypt, + .init = xilinx_paes_aead_init, + .exit = xilinx_paes_aead_exit, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = XILINX_AES_AUTH_SIZE, + .base = { + .cra_name = "gcm(paes)", + .cra_driver_name = "versal-paes-gcm", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AEAD | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = XILINX_AES_BLK_SIZE, + .cra_ctxsize = sizeof(struct xilinx_aead_tfm_ctx), + .cra_module = THIS_MODULE, + } + }, + .aead.op = { + .do_one_request = xilinx_handle_aes_req, + }, + .dma_bit_mask = VERSAL_DMA_BIT_MASK, + }, + { /* sentinel */ } +}; + +static struct xlnx_feature aes_feature_map[] = { + { + .family = PM_ZYNQMP_FAMILY_CODE, + .feature_id = PM_SECURE_AES, + .data = zynqmp_aes_algs, + }, + { + .family = PM_VERSAL_FAMILY_CODE, + .feature_id = XSECURE_API_AES_OP_INIT, + .data = versal_aes_algs, + }, + { /* sentinel */ } +}; + +static int xilinx_aes_aead_probe(struct platform_device *pdev) +{ + struct xilinx_aead_alg *aead_algs; + struct device *dev = &pdev->dev; + int err; + int i; + + /* Verify the hardware is present */ + aead_algs = xlnx_get_crypto_dev_data(aes_feature_map); + if (IS_ERR(aead_algs)) { + dev_err(dev, "AES is not supported on the platform\n"); + return PTR_ERR(aead_algs); + } + + /* ZynqMP AES driver supports only one instance */ + if (aead_dev) + return -ENODEV; + + aead_dev = devm_kzalloc(dev, sizeof(*aead_dev), GFP_KERNEL); + if (!aead_dev) + return -ENOMEM; + aead_dev->dev = dev; + aead_dev->aead_algs = aead_algs; + platform_set_drvdata(pdev, aead_dev); + err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(aead_algs[0].dma_bit_mask)); + if (err < 0) { + dev_err(dev, "No usable DMA configuration\n"); + return err; + } + + aead_dev->engine = crypto_engine_alloc_init(dev, 1); + if (!aead_dev->engine) { + dev_err(dev, "Cannot alloc AES engine\n"); + return -ENOMEM; + } + + err = crypto_engine_start(aead_dev->engine); + if (err) { + dev_err(dev, "Cannot start AES engine\n"); + goto err_engine_start; + } + + for (i = 0; aead_dev->aead_algs[i].dma_bit_mask; i++) { + aead_dev->aead_algs[i].aead_dev = aead_dev; + err = crypto_engine_register_aead(&aead_dev->aead_algs[i].aead); + if (err < 0) { + dev_err(dev, "Failed to register AEAD alg %d.\n", i); + goto err_alg_register; + } + } + + return 0; + +err_alg_register: + while (i > 0) + crypto_engine_unregister_aead(&aead_dev->aead_algs[--i].aead); +err_engine_start: + crypto_engine_exit(aead_dev->engine); + + return err; +} + +static void xilinx_aes_aead_remove(struct platform_device *pdev) +{ + aead_dev = platform_get_drvdata(pdev); + crypto_engine_exit(aead_dev->engine); + for (int i = 0; aead_dev->aead_algs[i].dma_bit_mask; i++) + crypto_engine_unregister_aead(&aead_dev->aead_algs[i].aead); + + aead_dev = NULL; +} + +static struct platform_driver xilinx_aes_driver = { + .probe = xilinx_aes_aead_probe, + .remove = xilinx_aes_aead_remove, + .driver = { + .name = "zynqmp-aes", + }, +}; + +static struct platform_device *platform_dev; + +static int __init aes_driver_init(void) +{ + int ret; + + ret = platform_driver_register(&xilinx_aes_driver); + if (ret) + return ret; + + platform_dev = platform_device_register_simple(xilinx_aes_driver.driver.name, + 0, NULL, 0); + if (IS_ERR(platform_dev)) { + ret = PTR_ERR(platform_dev); + platform_driver_unregister(&xilinx_aes_driver); + } + + return ret; +} + +static void __exit aes_driver_exit(void) +{ + platform_device_unregister(platform_dev); + platform_driver_unregister(&xilinx_aes_driver); +} + +module_init(aes_driver_init); +module_exit(aes_driver_exit); +MODULE_DESCRIPTION("zynqmp aes-gcm hardware acceleration support."); +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/xilinx/zynqmp-sha.c b/drivers/crypto/xilinx/zynqmp-sha.c new file mode 100644 index 000000000..5813017b6 --- /dev/null +++ b/drivers/crypto/xilinx/zynqmp-sha.c @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Xilinx ZynqMP SHA Driver. + * Copyright (c) 2022 Xilinx Inc. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define ZYNQMP_DMA_BIT_MASK 32U +#define ZYNQMP_DMA_ALLOC_FIXED_SIZE 0x1000U + +enum zynqmp_sha_op { + ZYNQMP_SHA3_INIT = 1, + ZYNQMP_SHA3_UPDATE = 2, + ZYNQMP_SHA3_FINAL = 4, +}; + +struct zynqmp_sha_drv_ctx { + struct shash_alg sha3_384; + struct device *dev; +}; + +struct zynqmp_sha_tfm_ctx { + struct device *dev; + struct crypto_shash *fbk_tfm; +}; + +static dma_addr_t update_dma_addr, final_dma_addr; +static char *ubuf, *fbuf; + +static DEFINE_SPINLOCK(zynqmp_sha_lock); + +static int zynqmp_sha_init_tfm(struct crypto_shash *hash) +{ + const char *fallback_driver_name = crypto_shash_alg_name(hash); + struct zynqmp_sha_tfm_ctx *tfm_ctx = crypto_shash_ctx(hash); + struct shash_alg *alg = crypto_shash_alg(hash); + struct crypto_shash *fallback_tfm; + struct zynqmp_sha_drv_ctx *drv_ctx; + + drv_ctx = container_of(alg, struct zynqmp_sha_drv_ctx, sha3_384); + tfm_ctx->dev = drv_ctx->dev; + + /* Allocate a fallback and abort if it failed. */ + fallback_tfm = crypto_alloc_shash(fallback_driver_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(fallback_tfm)) + return PTR_ERR(fallback_tfm); + + if (crypto_shash_descsize(hash) < + crypto_shash_statesize(tfm_ctx->fbk_tfm)) { + crypto_free_shash(fallback_tfm); + return -EINVAL; + } + + tfm_ctx->fbk_tfm = fallback_tfm; + + return 0; +} + +static void zynqmp_sha_exit_tfm(struct crypto_shash *hash) +{ + struct zynqmp_sha_tfm_ctx *tfm_ctx = crypto_shash_ctx(hash); + + crypto_free_shash(tfm_ctx->fbk_tfm); +} + +static int zynqmp_sha_continue(struct shash_desc *desc, + struct shash_desc *fbdesc, int err) +{ + err = err ?: crypto_shash_export(fbdesc, shash_desc_ctx(desc)); + shash_desc_zero(fbdesc); + return err; +} + +static int zynqmp_sha_init(struct shash_desc *desc) +{ + struct zynqmp_sha_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); + struct crypto_shash *fbtfm = tctx->fbk_tfm; + SHASH_DESC_ON_STACK(fbdesc, fbtfm); + int err; + + fbdesc->tfm = fbtfm; + err = crypto_shash_init(fbdesc); + return zynqmp_sha_continue(desc, fbdesc, err); +} + +static int zynqmp_sha_update(struct shash_desc *desc, const u8 *data, unsigned int length) +{ + struct zynqmp_sha_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); + struct crypto_shash *fbtfm = tctx->fbk_tfm; + SHASH_DESC_ON_STACK(fbdesc, fbtfm); + int err; + + fbdesc->tfm = fbtfm; + err = crypto_shash_import(fbdesc, shash_desc_ctx(desc)) ?: + crypto_shash_update(fbdesc, data, length); + return zynqmp_sha_continue(desc, fbdesc, err); +} + +static int zynqmp_sha_finup(struct shash_desc *desc, const u8 *data, unsigned int length, u8 *out) +{ + struct zynqmp_sha_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); + struct crypto_shash *fbtfm = tctx->fbk_tfm; + SHASH_DESC_ON_STACK(fbdesc, fbtfm); + + fbdesc->tfm = fbtfm; + return crypto_shash_import(fbdesc, shash_desc_ctx(desc)) ?: + crypto_shash_finup(fbdesc, data, length, out); +} + +static int __zynqmp_sha_digest(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) +{ + unsigned int remaining_len = len; + int update_size; + int ret; + + ret = zynqmp_pm_sha_hash(0, 0, ZYNQMP_SHA3_INIT); + if (ret) + return ret; + + while (remaining_len != 0) { + memzero_explicit(ubuf, ZYNQMP_DMA_ALLOC_FIXED_SIZE); + if (remaining_len >= ZYNQMP_DMA_ALLOC_FIXED_SIZE) { + update_size = ZYNQMP_DMA_ALLOC_FIXED_SIZE; + remaining_len -= ZYNQMP_DMA_ALLOC_FIXED_SIZE; + } else { + update_size = remaining_len; + remaining_len = 0; + } + memcpy(ubuf, data, update_size); + flush_icache_range((unsigned long)ubuf, (unsigned long)ubuf + update_size); + ret = zynqmp_pm_sha_hash(update_dma_addr, update_size, ZYNQMP_SHA3_UPDATE); + if (ret) + return ret; + + data += update_size; + } + + ret = zynqmp_pm_sha_hash(final_dma_addr, SHA3_384_DIGEST_SIZE, ZYNQMP_SHA3_FINAL); + memcpy(out, fbuf, SHA3_384_DIGEST_SIZE); + memzero_explicit(fbuf, SHA3_384_DIGEST_SIZE); + + return ret; +} + +static int zynqmp_sha_digest(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) +{ + scoped_guard(spinlock_bh, &zynqmp_sha_lock) + return __zynqmp_sha_digest(desc, data, len, out); +} + +static struct zynqmp_sha_drv_ctx sha3_drv_ctx = { + .sha3_384 = { + .init = zynqmp_sha_init, + .update = zynqmp_sha_update, + .finup = zynqmp_sha_finup, + .digest = zynqmp_sha_digest, + .init_tfm = zynqmp_sha_init_tfm, + .exit_tfm = zynqmp_sha_exit_tfm, + .descsize = SHA3_384_EXPORT_SIZE, + .digestsize = SHA3_384_DIGEST_SIZE, + .base = { + .cra_name = "sha3-384", + .cra_driver_name = "zynqmp-sha3-384", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA3_384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct zynqmp_sha_tfm_ctx), + .cra_module = THIS_MODULE, + } + } +}; + +static int zynqmp_sha_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + int err; + u32 v; + + /* Verify the hardware is present */ + err = zynqmp_pm_get_api_version(&v); + if (err) + return err; + + + err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(ZYNQMP_DMA_BIT_MASK)); + if (err < 0) { + dev_err(dev, "No usable DMA configuration\n"); + return err; + } + + err = crypto_register_shash(&sha3_drv_ctx.sha3_384); + if (err < 0) { + dev_err(dev, "Failed to register shash alg.\n"); + return err; + } + + sha3_drv_ctx.dev = dev; + platform_set_drvdata(pdev, &sha3_drv_ctx); + + ubuf = dma_alloc_coherent(dev, ZYNQMP_DMA_ALLOC_FIXED_SIZE, &update_dma_addr, GFP_KERNEL); + if (!ubuf) { + err = -ENOMEM; + goto err_shash; + } + + fbuf = dma_alloc_coherent(dev, SHA3_384_DIGEST_SIZE, &final_dma_addr, GFP_KERNEL); + if (!fbuf) { + err = -ENOMEM; + goto err_mem; + } + + return 0; + +err_mem: + dma_free_coherent(sha3_drv_ctx.dev, ZYNQMP_DMA_ALLOC_FIXED_SIZE, ubuf, update_dma_addr); + +err_shash: + crypto_unregister_shash(&sha3_drv_ctx.sha3_384); + + return err; +} + +static void zynqmp_sha_remove(struct platform_device *pdev) +{ + sha3_drv_ctx.dev = platform_get_drvdata(pdev); + + dma_free_coherent(sha3_drv_ctx.dev, ZYNQMP_DMA_ALLOC_FIXED_SIZE, ubuf, update_dma_addr); + dma_free_coherent(sha3_drv_ctx.dev, SHA3_384_DIGEST_SIZE, fbuf, final_dma_addr); + crypto_unregister_shash(&sha3_drv_ctx.sha3_384); +} + +static struct platform_driver zynqmp_sha_driver = { + .probe = zynqmp_sha_probe, + .remove = zynqmp_sha_remove, + .driver = { + .name = "zynqmp-sha3-384", + }, +}; + +module_platform_driver(zynqmp_sha_driver); +MODULE_DESCRIPTION("ZynqMP SHA3 hardware acceleration support."); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Harsha "); -- cgit v1.3.1