Merge tag 'io_uring-7.3-20261002' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linuxgrafted

Pull io_uring fixes from Jens Axboe:

 - Fix a task_work add use-after-free with SQPOLL.

   The sqpoll thread could pop and complete the last request while
   io_req_normal_work_add() was still looking at them after the mpscq
   push.

   Use the same approach as DEFER_TASKRUN to protect from that, holding
   an RCU read lock across the add, and have exit wait for an RCU grace
   period for SQPOLL rings as well.

 - CQE32 ring fixes: correct the free entry check for 32b CQEs, zero the
   big_cqe for aux CQEs, and only post the dummy skip CQE on CQE_MIXED
   rings

 - Mark the source filter table as COW when cloning bpf filters, so
   registering another filter on the source doesn't modify the shared
   table in place

 - Initialize the task context before running the BPF loop

 - Requeue zcrx multishot receives stopped by a local resource

 - End a TX_TIMESTAMP multishot cmd when the CQ is full (lollipopkit)

* tag 'io_uring-7.3-20261002' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux:
  io_uring: fix task_work add use-after-free with SQPOLL
  io_uring/cmd_net: end TX_TIMESTAMP multishot when the CQ is full
  io_uring/zcrx: requeue multishot receives stopped by a local resource
  io_uring: initialize task context before running the BPF loop
  io_uring: zero big_cqe for aux CQEs on CQE32 rings
  io_uring: fix free entry check for 32b CQEs on CQE32 rings
  io_uring: only post the dummy skip CQE on CQE_MIXED rings
  io_uring/bpf_filter: mark source as COW when cloning filters

35 files changed, 22347 insertions, 0 deletions

diff --git a/drivers/s390/crypto/Makefile b/drivers/s390/crypto/Makefile
new file mode 100644
index 000000000..e83c6603c
--- /dev/null
+++ b/drivers/s390/crypto/Makefile
@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# S/390 crypto devices
+#
+
+ap-objs := ap_bus.o ap_card.o ap_queue.o
+obj-$(CONFIG_AP) += ap.o
+# zcrypt_api.o and zcrypt_msgtype*.o depend on ap.o
+zcrypt-objs := zcrypt_api.o zcrypt_card.o zcrypt_queue.o
+zcrypt-objs += zcrypt_msgtype6.o zcrypt_msgtype50.o
+zcrypt-objs += zcrypt_ccamisc.o zcrypt_ep11misc.o
+obj-$(CONFIG_ZCRYPT) += zcrypt.o
+# adapter drivers depend on ap.o and zcrypt.o
+obj-$(CONFIG_ZCRYPT) += zcrypt_cex4.o
+
+# pkey base and api module
+pkey-objs := pkey_base.o pkey_api.o pkey_sysfs.o
+obj-$(CONFIG_PKEY) += pkey.o
+
+# pkey cca handler module
+pkey-cca-objs := pkey_cca.o
+obj-$(CONFIG_PKEY_CCA) += pkey-cca.o
+
+# pkey ep11 handler module
+pkey-ep11-objs := pkey_ep11.o
+obj-$(CONFIG_PKEY_EP11) += pkey-ep11.o
+
+# pkey pckmo handler module
+pkey-pckmo-objs := pkey_pckmo.o
+obj-$(CONFIG_PKEY_PCKMO) += pkey-pckmo.o
+
+# pkey uv handler module
+pkey-uv-objs := pkey_uv.o
+obj-$(CONFIG_PKEY_UV) += pkey-uv.o
+
+# adjunct processor matrix
+vfio_ap-objs := vfio_ap_drv.o vfio_ap_ops.o
+obj-$(CONFIG_VFIO_AP) += vfio_ap.o
diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c
new file mode 100644
index 000000000..d82df5b4e
--- /dev/null
+++ b/drivers/s390/crypto/ap_bus.c
@@ -0,0 +1,2681 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright IBM Corp. 2006, 2023
+ * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ *	      Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *	      Ralph Wuerthner <rwuerthn@de.ibm.com>
+ *	      Felix Beck <felix.beck@de.ibm.com>
+ *	      Holger Dengler <hd@linux.vnet.ibm.com>
+ *	      Harald Freudenberger <freude@linux.ibm.com>
+ *
+ * Adjunct processor bus.
+ */
+
+#define pr_fmt(fmt) "ap: " fmt
+
+#include <linux/kernel_stat.h>
+#include <linux/moduleparam.h>
+#include <linux/export.h>
+#include <linux/hex.h>
+#include <linux/init.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/freezer.h>
+#include <linux/interrupt.h>
+#include <linux/workqueue.h>
+#include <linux/slab.h>
+#include <linux/notifier.h>
+#include <linux/kthread.h>
+#include <linux/mutex.h>
+#include <asm/machine.h>
+#include <asm/airq.h>
+#include <asm/tpi.h>
+#include <linux/atomic.h>
+#include <asm/isc.h>
+#include <linux/hrtimer.h>
+#include <linux/ktime.h>
+#include <asm/facility.h>
+#include <linux/crypto.h>
+#include <linux/device-id/ap.h>
+#include <linux/debugfs.h>
+#include <linux/ctype.h>
+#include <linux/module.h>
+#include <asm/uv.h>
+#include <asm/chsc.h>
+#include <linux/mempool.h>
+
+#include "ap_bus.h"
+#include "ap_debug.h"
+
+MODULE_AUTHOR("IBM Corporation");
+MODULE_DESCRIPTION("Adjunct Processor Bus driver");
+MODULE_LICENSE("GPL");
+
+int ap_domain_index = -1;	/* Adjunct Processor Domain Index */
+static DEFINE_SPINLOCK(ap_domain_lock);
+module_param_named(domain, ap_domain_index, int, 0444);
+MODULE_PARM_DESC(domain, "domain index for ap devices");
+EXPORT_SYMBOL(ap_domain_index);
+
+static int ap_thread_flag;
+module_param_named(poll_thread, ap_thread_flag, int, 0444);
+MODULE_PARM_DESC(poll_thread, "Turn on/off poll thread, default is 0 (off).");
+
+static char *apm_str;
+module_param_named(apmask, apm_str, charp, 0444);
+MODULE_PARM_DESC(apmask, "AP bus adapter mask.");
+
+static char *aqm_str;
+module_param_named(aqmask, aqm_str, charp, 0444);
+MODULE_PARM_DESC(aqmask, "AP bus domain mask.");
+
+static int ap_useirq = 1;
+module_param_named(useirq, ap_useirq, int, 0444);
+MODULE_PARM_DESC(useirq, "Use interrupt if available, default is 1 (on).");
+
+atomic_t ap_max_msg_size = ATOMIC_INIT(AP_DEFAULT_MAX_MSG_SIZE);
+EXPORT_SYMBOL(ap_max_msg_size);
+
+static struct device *ap_root_device;
+
+/* Hashtable of all queue devices on the AP bus */
+DEFINE_HASHTABLE(ap_queues, 8);
+/* lock used for the ap_queues hashtable */
+DEFINE_SPINLOCK(ap_queues_lock);
+
+/* Default permissions (ioctl, card and domain masking) */
+struct ap_perms ap_perms;
+EXPORT_SYMBOL(ap_perms);
+/* true if apmask and/or aqmask are NOT default */
+bool ap_apmask_aqmask_in_use;
+/* counter for how many driver_overrides are currently active */
+int ap_driver_override_ctr;
+/*
+ * Mutex for consistent read and write of the ap_perms struct,
+ * ap_apmask_aqmask_in_use, ap_driver_override_ctr
+ * and the ap bus sysfs attributes apmask and aqmask.
+ */
+DEFINE_MUTEX(ap_attr_mutex);
+EXPORT_SYMBOL(ap_attr_mutex);
+
+/* # of bindings complete since init */
+static atomic64_t ap_bindings_complete_count = ATOMIC64_INIT(0);
+
+/* completion for APQN bindings complete */
+static DECLARE_COMPLETION(ap_apqn_bindings_complete);
+
+static struct ap_config_info qci[2];
+static struct ap_config_info *const ap_qci_info = &qci[0];
+static struct ap_config_info *const ap_qci_info_old = &qci[1];
+
+/*
+ * AP bus related debug feature things.
+ */
+debug_info_t *ap_dbf_info;
+
+/*
+ * There is a need for a do-not-allocate-memory path through the AP bus
+ * layer. The pkey layer may be triggered via the in-kernel interface from
+ * a protected key crypto algorithm (namely PAES) to convert a secure key
+ * into a protected key. This happens in a workqueue context, so sleeping
+ * is allowed but memory allocations causing IO operations are not permitted.
+ * To accomplish this, an AP message memory pool with pre-allocated space
+ * is established. When ap_init_apmsg() with use_mempool set to true is
+ * called, instead of kmalloc() the ap message buffer is allocated from
+ * the ap_msg_pool. This pool only holds a limited amount of buffers:
+ * ap_msg_pool_min_items with the item size AP_DEFAULT_MAX_MSG_SIZE and
+ * exactly one of these items (if available) is returned if ap_init_apmsg()
+ * with the use_mempool arg set to true is called. When this pool is exhausted
+ * and use_mempool is set true, ap_init_apmsg() returns -ENOMEM without
+ * any attempt to allocate memory and the caller has to deal with that.
+ */
+static mempool_t *ap_msg_pool;
+static unsigned int ap_msg_pool_min_items = 8;
+module_param_named(msgpool_min_items, ap_msg_pool_min_items, uint, 0400);
+MODULE_PARM_DESC(msgpool_min_items, "AP message pool minimal items");
+
+/*
+ * AP bus rescan related things.
+ */
+static bool ap_scan_bus(void);
+static bool ap_scan_bus_result; /* result of last ap_scan_bus() */
+static DEFINE_MUTEX(ap_scan_bus_mutex); /* mutex ap_scan_bus() invocations */
+static struct task_struct *ap_scan_bus_task; /* thread holding the scan mutex */
+static atomic64_t ap_scan_bus_count; /* counter ap_scan_bus() invocations */
+static int ap_scan_bus_time = AP_CONFIG_TIME;
+static struct timer_list ap_scan_bus_timer;
+static void ap_scan_bus_wq_callback(struct work_struct *);
+static DECLARE_WORK(ap_scan_bus_work, ap_scan_bus_wq_callback);
+
+/*
+ * Tasklet & timer for AP request polling and interrupts
+ */
+static void ap_tasklet_fn(unsigned long);
+static DECLARE_TASKLET_OLD(ap_tasklet, ap_tasklet_fn);
+static DECLARE_WAIT_QUEUE_HEAD(ap_poll_wait);
+static struct task_struct *ap_poll_kthread;
+static DEFINE_MUTEX(ap_poll_thread_mutex);
+static DEFINE_SPINLOCK(ap_poll_timer_lock);
+static struct hrtimer ap_poll_timer;
+/*
+ * In LPAR poll with 4kHz frequency. Poll every 250000 nanoseconds.
+ * If z/VM change to 1500000 nanoseconds to adjust to z/VM polling.
+ */
+static unsigned long poll_high_timeout = 250000UL;
+
+/*
+ * Some state machine states only require a low frequency polling.
+ * We use 25 Hz frequency for these.
+ */
+static unsigned long poll_low_timeout = 40000000UL;
+
+/* Maximum domain id, if not given via qci */
+static int ap_max_domain_id = 15;
+/* Maximum adapter id, if not given via qci */
+static int ap_max_adapter_id = 63;
+
+static const struct bus_type ap_bus_type;
+
+/* Adapter interrupt definitions */
+static void ap_interrupt_handler(struct airq_struct *airq,
+				 struct tpi_info *tpi_info);
+
+static bool ap_irq_flag;
+
+static struct airq_struct ap_airq = {
+	.handler = ap_interrupt_handler,
+	.isc = AP_ISC,
+};
+
+/**
+ * ap_airq_ptr() - Get the address of the adapter interrupt indicator
+ *
+ * Returns the address of the local-summary-indicator of the adapter
+ * interrupt handler for AP, or NULL if adapter interrupts are not
+ * available.
+ */
+void *ap_airq_ptr(void)
+{
+	if (ap_irq_flag)
+		return ap_airq.lsi_ptr;
+	return NULL;
+}
+
+/**
+ * ap_interrupts_available(): Test if AP interrupts are available.
+ *
+ * Returns 1 if AP interrupts are available.
+ */
+static int ap_interrupts_available(void)
+{
+	return test_facility(65);
+}
+
+/**
+ * ap_qci_available(): Test if AP configuration
+ * information can be queried via QCI subfunction.
+ *
+ * Returns 1 if subfunction PQAP(QCI) is available.
+ */
+static int ap_qci_available(void)
+{
+	return test_facility(12);
+}
+
+/**
+ * ap_apft_available(): Test if AP facilities test (APFT)
+ * facility is available.
+ *
+ * Returns 1 if APFT is available.
+ */
+static int ap_apft_available(void)
+{
+	return test_facility(15);
+}
+
+/*
+ * ap_qact_available(): Test if the PQAP(QACT) subfunction is available.
+ *
+ * Returns 1 if the QACT subfunction is available.
+ */
+static inline int ap_qact_available(void)
+{
+	return ap_qci_info->qact;
+}
+
+/*
+ * ap_sb_available(): Test if the AP secure binding facility is available.
+ *
+ * Returns 1 if secure binding facility is available.
+ */
+int ap_sb_available(void)
+{
+	return ap_qci_info->apsb;
+}
+
+/*
+ * ap_is_se_guest(): Check for SE guest with AP pass-through support.
+ */
+bool ap_is_se_guest(void)
+{
+	return is_prot_virt_guest() && ap_sb_available();
+}
+EXPORT_SYMBOL(ap_is_se_guest);
+
+/**
+ * ap_init_qci_info(): Allocate and query qci config info.
+ * Does also update the static variables ap_max_domain_id
+ * and ap_max_adapter_id if this info is available.
+ */
+static void __init ap_init_qci_info(void)
+{
+	if (!ap_qci_available() ||
+	    ap_qci(ap_qci_info)) {
+		AP_DBF_INFO("%s QCI not supported\n", __func__);
+		return;
+	}
+	memcpy(ap_qci_info_old, ap_qci_info, sizeof(*ap_qci_info));
+	AP_DBF_INFO("%s successful fetched initial qci info\n", __func__);
+
+	if (ap_qci_info->apxa) {
+		if (ap_qci_info->na) {
+			ap_max_adapter_id = ap_qci_info->na;
+			AP_DBF_INFO("%s new ap_max_adapter_id is %d\n",
+				    __func__, ap_max_adapter_id);
+		}
+		if (ap_qci_info->nd) {
+			ap_max_domain_id = ap_qci_info->nd;
+			AP_DBF_INFO("%s new ap_max_domain_id is %d\n",
+				    __func__, ap_max_domain_id);
+		}
+	}
+}
+
+/*
+ * ap_test_config(): helper function to extract the nrth bit
+ *		     within the unsigned int array field.
+ */
+static inline int ap_test_config(unsigned int *field, unsigned int nr)
+{
+	return ap_test_bit((field + (nr >> 5)), (nr & 0x1f));
+}
+
+/*
+ * ap_test_config_card_id(): Test, whether an AP card ID is configured.
+ *
+ * Returns 0 if the card is not configured
+ *	   1 if the card is configured or
+ *	     if the configuration information is not available
+ */
+static inline int ap_test_config_card_id(unsigned int id)
+{
+	if (id > ap_max_adapter_id)
+		return 0;
+	if (ap_qci_info->flags)
+		return ap_test_config(ap_qci_info->apm, id);
+	return 1;
+}
+
+/*
+ * ap_test_config_usage_domain(): Test, whether an AP usage domain
+ * is configured.
+ *
+ * Returns 0 if the usage domain is not configured
+ *	   1 if the usage domain is configured or
+ *	     if the configuration information is not available
+ */
+int ap_test_config_usage_domain(unsigned int domain)
+{
+	if (domain > ap_max_domain_id)
+		return 0;
+	if (ap_qci_info->flags)
+		return ap_test_config(ap_qci_info->aqm, domain);
+	return 1;
+}
+EXPORT_SYMBOL(ap_test_config_usage_domain);
+
+/*
+ * ap_test_config_ctrl_domain(): Test, whether an AP control domain
+ * is configured.
+ * @domain AP control domain ID
+ *
+ * Returns 1 if the control domain is configured
+ *	   0 in all other cases
+ */
+int ap_test_config_ctrl_domain(unsigned int domain)
+{
+	if (!ap_qci_info || domain > ap_max_domain_id)
+		return 0;
+	return ap_test_config(ap_qci_info->adm, domain);
+}
+EXPORT_SYMBOL(ap_test_config_ctrl_domain);
+
+/*
+ * ap_queue_info(): Check and get AP queue info.
+ * Returns: 1 if APQN exists and info is filled,
+ *	    0 if APQN seems to exist but there is no info
+ *	      available (eg. caused by an asynch pending error)
+ *	   -1 invalid APQN, TAPQ error or AP queue status which
+ *	      indicates there is no APQN.
+ */
+static int ap_queue_info(ap_qid_t qid, struct ap_tapq_hwinfo *hwinfo,
+			 bool *decfg, bool *cstop)
+{
+	struct ap_queue_status status;
+
+	hwinfo->value = 0;
+
+	/* make sure we don't run into a specifiation exception */
+	if (AP_QID_CARD(qid) > ap_max_adapter_id ||
+	    AP_QID_QUEUE(qid) > ap_max_domain_id)
+		return -1;
+
+	/* call TAPQ on this APQN */
+	status = ap_test_queue(qid, ap_apft_available(), hwinfo);
+
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+	case AP_RESPONSE_RESET_IN_PROGRESS:
+	case AP_RESPONSE_DECONFIGURED:
+	case AP_RESPONSE_CHECKSTOPPED:
+	case AP_RESPONSE_BUSY:
+		/* For all these RCs the tapq info should be available */
+		break;
+	default:
+		/* On a pending async error the info should be available */
+		if (!status.async)
+			return -1;
+		break;
+	}
+
+	/* There should be at least one of the mode bits set */
+	if (WARN_ON_ONCE(!hwinfo->value))
+		return 0;
+
+	*decfg = status.response_code == AP_RESPONSE_DECONFIGURED;
+	*cstop = status.response_code == AP_RESPONSE_CHECKSTOPPED;
+
+	return 1;
+}
+
+void ap_wait(enum ap_sm_wait wait)
+{
+	ktime_t hr_time;
+
+	switch (wait) {
+	case AP_SM_WAIT_AGAIN:
+	case AP_SM_WAIT_INTERRUPT:
+		if (ap_irq_flag)
+			break;
+		if (ap_poll_kthread) {
+			wake_up(&ap_poll_wait);
+			break;
+		}
+		fallthrough;
+	case AP_SM_WAIT_LOW_TIMEOUT:
+	case AP_SM_WAIT_HIGH_TIMEOUT:
+		spin_lock_bh(&ap_poll_timer_lock);
+		if (!hrtimer_is_queued(&ap_poll_timer)) {
+			hr_time =
+				wait == AP_SM_WAIT_LOW_TIMEOUT ?
+				poll_low_timeout : poll_high_timeout;
+			hrtimer_forward_now(&ap_poll_timer, hr_time);
+			hrtimer_restart(&ap_poll_timer);
+		}
+		spin_unlock_bh(&ap_poll_timer_lock);
+		break;
+	case AP_SM_WAIT_NONE:
+	default:
+		break;
+	}
+}
+
+/**
+ * ap_request_timeout(): Handling of request timeouts
+ * @t: timer making this callback
+ *
+ * Handles request timeouts.
+ */
+void ap_request_timeout(struct timer_list *t)
+{
+	struct ap_queue *aq = timer_container_of(aq, t, timeout);
+
+	spin_lock_bh(&aq->lock);
+	ap_wait(ap_sm_event(aq, AP_SM_EVENT_TIMEOUT));
+	spin_unlock_bh(&aq->lock);
+}
+
+/**
+ * ap_poll_timeout(): AP receive polling for finished AP requests.
+ * @unused: Unused pointer.
+ *
+ * Schedules the AP tasklet using a high resolution timer.
+ */
+static enum hrtimer_restart ap_poll_timeout(struct hrtimer *unused)
+{
+	tasklet_schedule(&ap_tasklet);
+	return HRTIMER_NORESTART;
+}
+
+/**
+ * ap_interrupt_handler() - Schedule ap_tasklet on interrupt
+ * @airq: pointer to adapter interrupt descriptor
+ * @tpi_info: ignored
+ */
+static void ap_interrupt_handler(struct airq_struct *airq,
+				 struct tpi_info *tpi_info)
+{
+	inc_irq_stat(IRQIO_APB);
+	tasklet_schedule(&ap_tasklet);
+}
+
+/**
+ * ap_tasklet_fn(): Tasklet to poll all AP devices.
+ * @dummy: Unused variable
+ *
+ * Poll all AP devices on the bus.
+ */
+static void ap_tasklet_fn(unsigned long dummy)
+{
+	int bkt;
+	struct ap_queue *aq;
+	enum ap_sm_wait wait = AP_SM_WAIT_NONE;
+
+	/* Reset the indicator if interrupts are used. Thus new interrupts can
+	 * be received. Doing it in the beginning of the tasklet is therefore
+	 * important that no requests on any AP get lost.
+	 */
+	if (ap_irq_flag)
+		WRITE_ONCE(*ap_airq.lsi_ptr, 0);
+
+	spin_lock_bh(&ap_queues_lock);
+	hash_for_each(ap_queues, bkt, aq, hnode) {
+		spin_lock_bh(&aq->lock);
+		wait = min(wait, ap_sm_event_loop(aq, AP_SM_EVENT_POLL));
+		spin_unlock_bh(&aq->lock);
+	}
+	spin_unlock_bh(&ap_queues_lock);
+
+	ap_wait(wait);
+}
+
+static int ap_pending_requests(void)
+{
+	int bkt;
+	struct ap_queue *aq;
+
+	spin_lock_bh(&ap_queues_lock);
+	hash_for_each(ap_queues, bkt, aq, hnode) {
+		if (aq->queue_count == 0)
+			continue;
+		spin_unlock_bh(&ap_queues_lock);
+		return 1;
+	}
+	spin_unlock_bh(&ap_queues_lock);
+	return 0;
+}
+
+/**
+ * ap_poll_thread(): Thread that polls for finished requests.
+ * @data: Unused pointer
+ *
+ * AP bus poll thread. The purpose of this thread is to poll for
+ * finished requests in a loop if there is a "free" cpu - that is
+ * a cpu that doesn't have anything better to do. The polling stops
+ * as soon as there is another task or if all messages have been
+ * delivered.
+ */
+static int ap_poll_thread(void *data)
+{
+	DECLARE_WAITQUEUE(wait, current);
+
+	set_user_nice(current, MAX_NICE);
+	set_freezable();
+	while (!kthread_should_stop()) {
+		add_wait_queue(&ap_poll_wait, &wait);
+		set_current_state(TASK_INTERRUPTIBLE);
+		if (!ap_pending_requests()) {
+			schedule();
+			try_to_freeze();
+		}
+		set_current_state(TASK_RUNNING);
+		remove_wait_queue(&ap_poll_wait, &wait);
+		if (need_resched()) {
+			schedule();
+			try_to_freeze();
+			continue;
+		}
+		ap_tasklet_fn(0);
+	}
+
+	return 0;
+}
+
+static int ap_poll_thread_start(void)
+{
+	int rc;
+
+	if (ap_irq_flag || ap_poll_kthread)
+		return 0;
+	mutex_lock(&ap_poll_thread_mutex);
+	ap_poll_kthread = kthread_run(ap_poll_thread, NULL, "appoll");
+	rc = PTR_ERR_OR_ZERO(ap_poll_kthread);
+	if (rc)
+		ap_poll_kthread = NULL;
+	mutex_unlock(&ap_poll_thread_mutex);
+	return rc;
+}
+
+static void ap_poll_thread_stop(void)
+{
+	if (!ap_poll_kthread)
+		return;
+	mutex_lock(&ap_poll_thread_mutex);
+	kthread_stop(ap_poll_kthread);
+	ap_poll_kthread = NULL;
+	mutex_unlock(&ap_poll_thread_mutex);
+}
+
+#define is_card_dev(x) ((x)->parent == ap_root_device)
+#define is_queue_dev(x) ((x)->parent != ap_root_device)
+
+/*
+ * ap_init_apmsg() - Initialize ap_message.
+ */
+int ap_init_apmsg(struct ap_message *ap_msg, u32 flags)
+{
+	unsigned int maxmsgsize;
+
+	memset(ap_msg, 0, sizeof(*ap_msg));
+	ap_msg->flags = flags;
+
+	if (flags & AP_MSG_FLAG_MEMPOOL) {
+		ap_msg->msg = mempool_alloc_preallocated(ap_msg_pool);
+		if (!ap_msg->msg)
+			return -ENOMEM;
+		ap_msg->bufsize = AP_DEFAULT_MAX_MSG_SIZE;
+		return 0;
+	}
+
+	maxmsgsize = atomic_read(&ap_max_msg_size);
+	ap_msg->msg = kmalloc(maxmsgsize, GFP_KERNEL);
+	if (!ap_msg->msg)
+		return -ENOMEM;
+	ap_msg->bufsize = maxmsgsize;
+
+	return 0;
+}
+EXPORT_SYMBOL(ap_init_apmsg);
+
+/*
+ * ap_release_apmsg() - Release ap_message.
+ */
+void ap_release_apmsg(struct ap_message *ap_msg)
+{
+	if (ap_msg->flags & AP_MSG_FLAG_MEMPOOL) {
+		memzero_explicit(ap_msg->msg, ap_msg->bufsize);
+		mempool_free(ap_msg->msg, ap_msg_pool);
+	} else {
+		kfree_sensitive(ap_msg->msg);
+	}
+}
+EXPORT_SYMBOL(ap_release_apmsg);
+
+/**
+ * ap_bus_match()
+ * @dev: Pointer to device
+ * @drv: Pointer to device_driver
+ *
+ * AP bus driver registration/unregistration.
+ */
+static int ap_bus_match(struct device *dev, const struct device_driver *drv)
+{
+	const struct ap_driver *ap_drv = to_ap_drv(drv);
+	struct ap_device_id *id;
+
+	/*
+	 * Compare device type of the device with the list of
+	 * supported types of the device_driver.
+	 */
+	for (id = ap_drv->ids; id->match_flags; id++) {
+		if (is_card_dev(dev) &&
+		    id->match_flags & AP_DEVICE_ID_MATCH_CARD_TYPE &&
+		    id->dev_type == to_ap_dev(dev)->device_type)
+			return 1;
+		if (is_queue_dev(dev) &&
+		    id->match_flags & AP_DEVICE_ID_MATCH_QUEUE_TYPE &&
+		    id->dev_type == to_ap_dev(dev)->device_type)
+			return 1;
+	}
+	return 0;
+}
+
+/**
+ * ap_uevent(): Uevent function for AP devices.
+ * @dev: Pointer to device
+ * @env: Pointer to kobj_uevent_env
+ *
+ * It sets up a single environment variable DEV_TYPE which contains the
+ * hardware device type.
+ */
+static int ap_uevent(const struct device *dev, struct kobj_uevent_env *env)
+{
+	int rc = 0;
+	const struct ap_device *ap_dev = to_ap_dev(dev);
+
+	/* Uevents from ap bus core don't need extensions to the env */
+	if (dev == ap_root_device)
+		return 0;
+
+	if (is_card_dev(dev)) {
+		struct ap_card *ac = to_ap_card(&ap_dev->device);
+
+		/* Set up DEV_TYPE environment variable. */
+		rc = add_uevent_var(env, "DEV_TYPE=%04X", ap_dev->device_type);
+		if (rc)
+			return rc;
+		/* Add MODALIAS= */
+		rc = add_uevent_var(env, "MODALIAS=ap:t%02X", ap_dev->device_type);
+		if (rc)
+			return rc;
+
+		/* Add MODE=<accel|cca|ep11> */
+		if (ac->hwinfo.accel)
+			rc = add_uevent_var(env, "MODE=accel");
+		else if (ac->hwinfo.cca)
+			rc = add_uevent_var(env, "MODE=cca");
+		else if (ac->hwinfo.ep11)
+			rc = add_uevent_var(env, "MODE=ep11");
+		if (rc)
+			return rc;
+	} else {
+		struct ap_queue *aq = to_ap_queue(&ap_dev->device);
+
+		/* Add MODE=<accel|cca|ep11> */
+		if (aq->card->hwinfo.accel)
+			rc = add_uevent_var(env, "MODE=accel");
+		else if (aq->card->hwinfo.cca)
+			rc = add_uevent_var(env, "MODE=cca");
+		else if (aq->card->hwinfo.ep11)
+			rc = add_uevent_var(env, "MODE=ep11");
+		if (rc)
+			return rc;
+	}
+
+	return 0;
+}
+
+static void ap_send_init_scan_done_uevent(void)
+{
+	char *envp[] = { "INITSCAN=done", NULL };
+
+	kobject_uevent_env(&ap_root_device->kobj, KOBJ_CHANGE, envp);
+}
+
+static void ap_send_bindings_complete_uevent(void)
+{
+	char buf[32];
+	char *envp[] = { "BINDINGS=complete", buf, NULL };
+
+	snprintf(buf, sizeof(buf), "COMPLETECOUNT=%llu",
+		 atomic64_inc_return(&ap_bindings_complete_count));
+	kobject_uevent_env(&ap_root_device->kobj, KOBJ_CHANGE, envp);
+}
+
+void ap_send_config_uevent(struct ap_device *ap_dev, bool cfg)
+{
+	char buf[16];
+	char *envp[] = { buf, NULL };
+
+	snprintf(buf, sizeof(buf), "CONFIG=%d", cfg ? 1 : 0);
+
+	kobject_uevent_env(&ap_dev->device.kobj, KOBJ_CHANGE, envp);
+}
+EXPORT_SYMBOL(ap_send_config_uevent);
+
+void ap_send_online_uevent(struct ap_device *ap_dev, int online)
+{
+	char buf[16];
+	char *envp[] = { buf, NULL };
+
+	snprintf(buf, sizeof(buf), "ONLINE=%d", online ? 1 : 0);
+
+	kobject_uevent_env(&ap_dev->device.kobj, KOBJ_CHANGE, envp);
+}
+EXPORT_SYMBOL(ap_send_online_uevent);
+
+void ap_send_se_bind_uevent(struct ap_device *ap_dev)
+{
+	char *envp[] = { "SE_BIND=1", NULL };
+
+	kobject_uevent_env(&ap_dev->device.kobj, KOBJ_CHANGE, envp);
+}
+
+void ap_send_se_assoc_uevent(struct ap_device *ap_dev, unsigned int assoc_idx)
+{
+	char buf[32];
+	char *envp[] = { buf, NULL };
+
+	snprintf(buf, sizeof(buf), "SE_ASSOC=%u", assoc_idx);
+
+	kobject_uevent_env(&ap_dev->device.kobj, KOBJ_CHANGE, envp);
+}
+
+static void ap_send_mask_changed_uevent(unsigned long *newapm,
+					unsigned long *newaqm)
+{
+	char buf[100];
+	char *envp[] = { buf, NULL };
+
+	if (newapm)
+		snprintf(buf, sizeof(buf),
+			 "APMASK=0x%016lx%016lx%016lx%016lx\n",
+			 newapm[0], newapm[1], newapm[2], newapm[3]);
+	else
+		snprintf(buf, sizeof(buf),
+			 "AQMASK=0x%016lx%016lx%016lx%016lx\n",
+			 newaqm[0], newaqm[1], newaqm[2], newaqm[3]);
+
+	kobject_uevent_env(&ap_root_device->kobj, KOBJ_CHANGE, envp);
+}
+
+/*
+ * calc # of bound APQNs
+ */
+
+struct __ap_calc_ctrs {
+	unsigned int apqns;
+	unsigned int bound;
+};
+
+static int __ap_calc_helper(struct device *dev, void *arg)
+{
+	struct __ap_calc_ctrs *pctrs = (struct __ap_calc_ctrs *)arg;
+
+	if (is_queue_dev(dev)) {
+		pctrs->apqns++;
+		if (dev->driver)
+			pctrs->bound++;
+	}
+
+	return 0;
+}
+
+static void ap_calc_bound_apqns(unsigned int *apqns, unsigned int *bound)
+{
+	struct __ap_calc_ctrs ctrs;
+
+	memset(&ctrs, 0, sizeof(ctrs));
+	bus_for_each_dev(&ap_bus_type, NULL, (void *)&ctrs, __ap_calc_helper);
+
+	*apqns = ctrs.apqns;
+	*bound = ctrs.bound;
+}
+
+/*
+ * After ap bus scan do check if all existing APQNs are
+ * bound to device drivers.
+ */
+static void ap_check_bindings_complete(void)
+{
+	unsigned int apqns, bound;
+
+	if (atomic64_read(&ap_scan_bus_count) >= 1) {
+		ap_calc_bound_apqns(&apqns, &bound);
+		if (bound == apqns) {
+			if (!completion_done(&ap_apqn_bindings_complete)) {
+				complete_all(&ap_apqn_bindings_complete);
+				ap_send_bindings_complete_uevent();
+				pr_debug("all apqn bindings complete\n");
+			}
+		}
+	}
+}
+
+/*
+ * Interface to wait for the AP bus to have done one initial ap bus
+ * scan and all detected APQNs have been bound to device drivers.
+ * If these both conditions are not fulfilled, this function blocks
+ * on a condition with wait_for_completion_interruptible_timeout().
+ * If these both conditions are fulfilled (before the timeout hits)
+ * the return value is 0. If the timeout (in jiffies) hits instead
+ * -ETIME is returned. On failures negative return values are
+ * returned to the caller.
+ */
+int ap_wait_apqn_bindings_complete(unsigned long timeout)
+{
+	int rc = 0;
+	long l;
+
+	if (completion_done(&ap_apqn_bindings_complete))
+		return 0;
+
+	if (timeout)
+		l = wait_for_completion_interruptible_timeout(
+			&ap_apqn_bindings_complete, timeout);
+	else
+		l = wait_for_completion_interruptible(
+			&ap_apqn_bindings_complete);
+	if (l < 0)
+		rc = l == -ERESTARTSYS ? -EINTR : l;
+	else if (l == 0 && timeout)
+		rc = -ETIME;
+
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+EXPORT_SYMBOL(ap_wait_apqn_bindings_complete);
+
+static int __ap_queue_devices_with_id_unregister(struct device *dev, void *data)
+{
+	if (is_queue_dev(dev) &&
+	    AP_QID_CARD(to_ap_queue(dev)->qid) == (int)(long)data)
+		device_unregister(dev);
+	return 0;
+}
+
+static int __ap_revise_reserved(struct device *dev, void *dummy)
+{
+	int rc, card, queue, devres, drvres, ovrd;
+
+	if (is_queue_dev(dev)) {
+		struct ap_driver *ap_drv = to_ap_drv(dev->driver);
+		struct ap_queue *aq = to_ap_queue(dev);
+
+		card = AP_QID_CARD(aq->qid);
+		queue = AP_QID_QUEUE(aq->qid);
+
+		ovrd = device_match_driver_override(dev, &ap_drv->driver);
+		if (ovrd > 0) {
+			/* override set and matches, nothing to do */
+		} else if (ovrd == 0) {
+			pr_debug("reprobing queue=%02x.%04x\n", card, queue);
+			rc = device_reprobe(dev);
+			if (rc) {
+				AP_DBF_WARN("%s reprobing queue=%02x.%04x failed\n",
+					    __func__, card, queue);
+			}
+		} else {
+			mutex_lock(&ap_attr_mutex);
+			devres = test_bit_inv(card, ap_perms.apm) &&
+				test_bit_inv(queue, ap_perms.aqm);
+			mutex_unlock(&ap_attr_mutex);
+			drvres = to_ap_drv(dev->driver)->flags
+				& AP_DRIVER_FLAG_DEFAULT;
+			if (!!devres != !!drvres) {
+				pr_debug("reprobing queue=%02x.%04x\n", card, queue);
+				rc = device_reprobe(dev);
+				if (rc) {
+					AP_DBF_WARN("%s reprobing queue=%02x.%04x failed\n",
+						    __func__, card, queue);
+				}
+			}
+		}
+	}
+
+	return 0;
+}
+
+static void ap_bus_revise_bindings(void)
+{
+	bus_for_each_dev(&ap_bus_type, NULL, NULL, __ap_revise_reserved);
+}
+
+/**
+ * ap_owned_by_def_drv: indicates whether an AP adapter is reserved for the
+ *			default host driver or not.
+ * @card: the APID of the adapter card to check
+ * @queue: the APQI of the queue to check
+ *
+ * Note: the ap_attr_mutex must be locked by the caller of this function.
+ *
+ * Return: an int specifying whether the AP adapter is reserved for the host (1)
+ *	   or not (0).
+ */
+int ap_owned_by_def_drv(int card, int queue)
+{
+	struct ap_queue *aq;
+	int rc = 0;
+
+	if (card < 0 || card >= AP_DEVICES || queue < 0 || queue >= AP_DOMAINS)
+		return -EINVAL;
+
+	aq = ap_get_qdev(AP_MKQID(card, queue));
+	if (aq) {
+		const struct device_driver *drv = aq->ap_dev.device.driver;
+		const struct ap_driver *ap_drv = to_ap_drv(drv);
+		bool override = device_has_driver_override(&aq->ap_dev.device);
+
+		if (override && drv && ap_drv->flags & AP_DRIVER_FLAG_DEFAULT)
+			rc = 1;
+		put_device(&aq->ap_dev.device);
+		if (override)
+			goto out;
+	}
+
+	if (test_bit_inv(card, ap_perms.apm) &&
+	    test_bit_inv(queue, ap_perms.aqm))
+		rc = 1;
+
+out:
+	return rc;
+}
+EXPORT_SYMBOL(ap_owned_by_def_drv);
+
+/**
+ * ap_apqn_in_matrix_owned_by_def_drv: indicates whether every APQN contained in
+ *				       a set is reserved for the host drivers
+ *				       or not.
+ * @apm: a bitmap specifying a set of APIDs comprising the APQNs to check
+ * @aqm: a bitmap specifying a set of APQIs comprising the APQNs to check
+ *
+ * Note: the ap_attr_mutex must be locked by the caller of this function.
+ *
+ * Return: an int specifying whether each APQN is reserved for the host (1) or
+ *	   not (0)
+ */
+int ap_apqn_in_matrix_owned_by_def_drv(unsigned long *apm,
+				       unsigned long *aqm)
+{
+	int card, queue, rc = 0;
+
+	for (card = 0; !rc && card < AP_DEVICES; card++)
+		if (test_bit_inv(card, apm))
+			for (queue = 0; !rc && queue < AP_DOMAINS; queue++)
+				if (test_bit_inv(queue, aqm))
+					rc = ap_owned_by_def_drv(card, queue);
+
+	return rc;
+}
+EXPORT_SYMBOL(ap_apqn_in_matrix_owned_by_def_drv);
+
+static int ap_device_probe(struct device *dev)
+{
+	struct ap_device *ap_dev = to_ap_dev(dev);
+	struct ap_driver *ap_drv = to_ap_drv(dev->driver);
+	int card, queue, devres, drvres, rc = -ENODEV, ovrd;
+
+	if (!get_device(dev))
+		return rc;
+
+	if (is_queue_dev(dev)) {
+		/*
+		 * If the apqn is marked as reserved/used by ap bus and
+		 * default drivers, only probe with drivers with the default
+		 * flag set. If it is not marked, only probe with drivers
+		 * with the default flag not set.
+		 */
+		card = AP_QID_CARD(to_ap_queue(dev)->qid);
+		queue = AP_QID_QUEUE(to_ap_queue(dev)->qid);
+		ovrd = device_match_driver_override(dev, &ap_drv->driver);
+		if (ovrd > 0) {
+			/* override set and matches, nothing to do */
+		} else if (ovrd == 0) {
+			goto out;
+		} else {
+			mutex_lock(&ap_attr_mutex);
+			devres = test_bit_inv(card, ap_perms.apm) &&
+				test_bit_inv(queue, ap_perms.aqm);
+			mutex_unlock(&ap_attr_mutex);
+			drvres = ap_drv->flags & AP_DRIVER_FLAG_DEFAULT;
+			if (!!devres != !!drvres)
+				goto out;
+		}
+	}
+
+	/*
+	 * Rearm the bindings complete completion to trigger
+	 * bindings complete when all devices are bound again
+	 */
+	reinit_completion(&ap_apqn_bindings_complete);
+
+	/* Add queue/card to list of active queues/cards */
+	spin_lock_bh(&ap_queues_lock);
+	if (is_queue_dev(dev))
+		hash_add(ap_queues, &to_ap_queue(dev)->hnode,
+			 to_ap_queue(dev)->qid);
+	spin_unlock_bh(&ap_queues_lock);
+
+	rc = ap_drv->probe ? ap_drv->probe(ap_dev) : -ENODEV;
+
+	if (rc) {
+		spin_lock_bh(&ap_queues_lock);
+		if (is_queue_dev(dev))
+			hash_del(&to_ap_queue(dev)->hnode);
+		spin_unlock_bh(&ap_queues_lock);
+	}
+
+out:
+	if (rc) {
+		put_device(dev);
+	} else {
+		if (is_queue_dev(dev)) {
+			pr_debug("queue=%02x.%04x new driver=%s\n",
+				 card, queue, ap_drv->driver.name);
+		} else {
+			pr_debug("card=%02x new driver=%s\n",
+				 to_ap_card(dev)->id, ap_drv->driver.name);
+		}
+	}
+	return rc;
+}
+
+static void ap_device_remove(struct device *dev)
+{
+	struct ap_device *ap_dev = to_ap_dev(dev);
+	struct ap_driver *ap_drv = to_ap_drv(dev->driver);
+
+	/* prepare ap queue device removal */
+	if (is_queue_dev(dev))
+		ap_queue_prepare_remove(to_ap_queue(dev));
+
+	/* driver's chance to clean up gracefully */
+	if (ap_drv->remove)
+		ap_drv->remove(ap_dev);
+
+	/* now do the ap queue device remove */
+	if (is_queue_dev(dev))
+		ap_queue_remove(to_ap_queue(dev));
+
+	/* Remove queue/card from list of active queues/cards */
+	spin_lock_bh(&ap_queues_lock);
+	if (is_queue_dev(dev))
+		hash_del(&to_ap_queue(dev)->hnode);
+	spin_unlock_bh(&ap_queues_lock);
+
+	put_device(dev);
+}
+
+struct ap_queue *ap_get_qdev(ap_qid_t qid)
+{
+	int bkt;
+	struct ap_queue *aq;
+
+	spin_lock_bh(&ap_queues_lock);
+	hash_for_each(ap_queues, bkt, aq, hnode) {
+		if (aq->qid == qid) {
+			get_device(&aq->ap_dev.device);
+			spin_unlock_bh(&ap_queues_lock);
+			return aq;
+		}
+	}
+	spin_unlock_bh(&ap_queues_lock);
+
+	return NULL;
+}
+EXPORT_SYMBOL(ap_get_qdev);
+
+int ap_driver_register(struct ap_driver *ap_drv, struct module *owner,
+		       char *name)
+{
+	struct device_driver *drv = &ap_drv->driver;
+	int rc;
+
+	drv->bus = &ap_bus_type;
+	drv->owner = owner;
+	drv->name = name;
+	rc = driver_register(drv);
+
+	ap_check_bindings_complete();
+
+	return rc;
+}
+EXPORT_SYMBOL(ap_driver_register);
+
+void ap_driver_unregister(struct ap_driver *ap_drv)
+{
+	driver_unregister(&ap_drv->driver);
+}
+EXPORT_SYMBOL(ap_driver_unregister);
+
+/*
+ * Enforce a synchronous AP bus rescan.
+ * Returns true if the bus scan finds a change in the AP configuration
+ * and AP devices have been added or deleted when this function returns.
+ */
+bool ap_bus_force_rescan(void)
+{
+	unsigned long scan_counter = atomic64_read(&ap_scan_bus_count);
+	bool rc = false;
+
+	pr_debug("> scan counter=%lu\n", scan_counter);
+
+	/* Only trigger AP bus scans after the initial scan is done */
+	if (scan_counter <= 0)
+		goto out;
+
+	/*
+	 * There is one unlikely but nevertheless valid scenario where the
+	 * thread holding the mutex may try to send some crypto load but
+	 * all cards are offline so a rescan is triggered which causes
+	 * a recursive call of ap_bus_force_rescan(). A simple return if
+	 * the mutex is already locked by this thread solves this.
+	 */
+	if (mutex_is_locked(&ap_scan_bus_mutex)) {
+		if (ap_scan_bus_task == current)
+			goto out;
+	}
+
+	/* Try to acquire the AP scan bus mutex */
+	if (mutex_trylock(&ap_scan_bus_mutex)) {
+		/* mutex acquired, run the AP bus scan */
+		ap_scan_bus_task = current;
+		ap_scan_bus_result = ap_scan_bus();
+		rc = ap_scan_bus_result;
+		ap_scan_bus_task = NULL;
+		mutex_unlock(&ap_scan_bus_mutex);
+		goto out;
+	}
+
+	/*
+	 * Mutex acquire failed. So there is currently another task
+	 * already running the AP bus scan. Then let's simple wait
+	 * for the lock which means the other task has finished and
+	 * stored the result in ap_scan_bus_result.
+	 */
+	if (mutex_lock_killable(&ap_scan_bus_mutex)) {
+		/* fatal signal received, go out */
+		goto out;
+	}
+	rc = ap_scan_bus_result;
+	mutex_unlock(&ap_scan_bus_mutex);
+
+out:
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+EXPORT_SYMBOL(ap_bus_force_rescan);
+
+/*
+ * A config change has happened, force an ap bus rescan.
+ */
+static int ap_bus_cfg_chg(struct notifier_block *nb,
+			  unsigned long action, void *data)
+{
+	if (action != CHSC_NOTIFY_AP_CFG)
+		return NOTIFY_DONE;
+
+	pr_debug("config change, forcing bus rescan\n");
+
+	ap_bus_force_rescan();
+
+	return NOTIFY_OK;
+}
+
+static struct notifier_block ap_bus_nb = {
+	.notifier_call = ap_bus_cfg_chg,
+};
+
+int ap_hex2bitmap(const char *str, unsigned long *bitmap, int bits)
+{
+	int i, n, b;
+
+	/* bits needs to be a multiple of 8 */
+	if (bits & 0x07)
+		return -EINVAL;
+
+	if (str[0] == '0' && str[1] == 'x')
+		str++;
+	if (*str == 'x')
+		str++;
+
+	for (i = 0; isxdigit(*str) && i < bits; str++) {
+		b = hex_to_bin(*str);
+		for (n = 0; n < 4; n++)
+			if (b & (0x08 >> n))
+				set_bit_inv(i + n, bitmap);
+		i += 4;
+	}
+
+	if (*str == '\n')
+		str++;
+	if (*str)
+		return -EINVAL;
+	return 0;
+}
+EXPORT_SYMBOL(ap_hex2bitmap);
+
+/*
+ * modify_bitmap() - parse bitmask argument and modify an existing
+ * bit mask accordingly. A concatenation (done with ',') of these
+ * terms is recognized:
+ *   +<bitnr>[-<bitnr>] or -<bitnr>[-<bitnr>]
+ * <bitnr> may be any valid number (hex, decimal or octal) in the range
+ * 0...bits-1; the leading + or - is required. Here are some examples:
+ *   +0-15,+32,-128,-0xFF
+ *   -0-255,+1-16,+0x128
+ *   +1,+2,+3,+4,-5,-7-10
+ * Returns the new bitmap after all changes have been applied. Every
+ * positive value in the string will set a bit and every negative value
+ * in the string will clear a bit. As a bit may be touched more than once,
+ * the last 'operation' wins:
+ * +0-255,-128 = first bits 0-255 will be set, then bit 128 will be
+ * cleared again. All other bits are unmodified.
+ */
+static int modify_bitmap(const char *str, unsigned long *bitmap, int bits)
+{
+	unsigned long a, i, z;
+	char *np, sign;
+
+	/* bits needs to be a multiple of 8 */
+	if (bits & 0x07)
+		return -EINVAL;
+
+	while (*str) {
+		sign = *str++;
+		if (sign != '+' && sign != '-')
+			return -EINVAL;
+		a = z = simple_strtoul(str, &np, 0);
+		if (str == np || a >= bits)
+			return -EINVAL;
+		str = np;
+		if (*str == '-') {
+			z = simple_strtoul(++str, &np, 0);
+			if (str == np || a > z || z >= bits)
+				return -EINVAL;
+			str = np;
+		}
+		for (i = a; i <= z; i++)
+			if (sign == '+')
+				set_bit_inv(i, bitmap);
+			else
+				clear_bit_inv(i, bitmap);
+		while (*str == ',' || *str == '\n')
+			str++;
+	}
+
+	return 0;
+}
+
+static int ap_parse_bitmap_str(const char *str, unsigned long *bitmap, int bits,
+			       unsigned long *newmap)
+{
+	unsigned long size;
+	int rc;
+
+	size = BITS_TO_LONGS(bits) * sizeof(unsigned long);
+	if (*str == '+' || *str == '-') {
+		memcpy(newmap, bitmap, size);
+		rc = modify_bitmap(str, newmap, bits);
+	} else {
+		memset(newmap, 0, size);
+		rc = ap_hex2bitmap(str, newmap, bits);
+	}
+	return rc;
+}
+
+int ap_parse_mask_str(const char *str,
+		      unsigned long *bitmap, int bits,
+		      struct mutex *lock)
+{
+	unsigned long *newmap, size;
+	int rc;
+
+	/* bits needs to be a multiple of 8 */
+	if (bits & 0x07)
+		return -EINVAL;
+
+	size = BITS_TO_LONGS(bits) * sizeof(unsigned long);
+	newmap = kmalloc(size, GFP_KERNEL);
+	if (!newmap)
+		return -ENOMEM;
+	if (mutex_lock_interruptible(lock)) {
+		kfree(newmap);
+		return -ERESTARTSYS;
+	}
+	rc = ap_parse_bitmap_str(str, bitmap, bits, newmap);
+	if (rc == 0)
+		memcpy(bitmap, newmap, size);
+	mutex_unlock(lock);
+	kfree(newmap);
+	return rc;
+}
+EXPORT_SYMBOL(ap_parse_mask_str);
+
+/*
+ * AP bus attributes.
+ */
+
+static ssize_t ap_domain_show(const struct bus_type *bus, char *buf)
+{
+	return sysfs_emit(buf, "%d\n", ap_domain_index);
+}
+
+static ssize_t ap_domain_store(const struct bus_type *bus,
+			       const char *buf, size_t count)
+{
+	int domain;
+
+	if (sscanf(buf, "%i\n", &domain) != 1 ||
+	    domain < 0 || domain > ap_max_domain_id ||
+	    !test_bit_inv(domain, ap_perms.aqm))
+		return -EINVAL;
+
+	spin_lock_bh(&ap_domain_lock);
+	ap_domain_index = domain;
+	spin_unlock_bh(&ap_domain_lock);
+
+	AP_DBF_INFO("%s stored new default domain=%d\n",
+		    __func__, domain);
+
+	return count;
+}
+
+static BUS_ATTR_RW(ap_domain);
+
+static ssize_t ap_control_domain_mask_show(const struct bus_type *bus, char *buf)
+{
+	if (!ap_qci_info->flags)	/* QCI not supported */
+		return sysfs_emit(buf, "not supported\n");
+
+	return sysfs_emit(buf, "0x%08x%08x%08x%08x%08x%08x%08x%08x\n",
+			  ap_qci_info->adm[0], ap_qci_info->adm[1],
+			  ap_qci_info->adm[2], ap_qci_info->adm[3],
+			  ap_qci_info->adm[4], ap_qci_info->adm[5],
+			  ap_qci_info->adm[6], ap_qci_info->adm[7]);
+}
+
+static BUS_ATTR_RO(ap_control_domain_mask);
+
+static ssize_t ap_usage_domain_mask_show(const struct bus_type *bus, char *buf)
+{
+	if (!ap_qci_info->flags)	/* QCI not supported */
+		return sysfs_emit(buf, "not supported\n");
+
+	return sysfs_emit(buf, "0x%08x%08x%08x%08x%08x%08x%08x%08x\n",
+			  ap_qci_info->aqm[0], ap_qci_info->aqm[1],
+			  ap_qci_info->aqm[2], ap_qci_info->aqm[3],
+			  ap_qci_info->aqm[4], ap_qci_info->aqm[5],
+			  ap_qci_info->aqm[6], ap_qci_info->aqm[7]);
+}
+
+static BUS_ATTR_RO(ap_usage_domain_mask);
+
+static ssize_t ap_adapter_mask_show(const struct bus_type *bus, char *buf)
+{
+	if (!ap_qci_info->flags)	/* QCI not supported */
+		return sysfs_emit(buf, "not supported\n");
+
+	return sysfs_emit(buf, "0x%08x%08x%08x%08x%08x%08x%08x%08x\n",
+			  ap_qci_info->apm[0], ap_qci_info->apm[1],
+			  ap_qci_info->apm[2], ap_qci_info->apm[3],
+			  ap_qci_info->apm[4], ap_qci_info->apm[5],
+			  ap_qci_info->apm[6], ap_qci_info->apm[7]);
+}
+
+static BUS_ATTR_RO(ap_adapter_mask);
+
+static ssize_t ap_interrupts_show(const struct bus_type *bus, char *buf)
+{
+	return sysfs_emit(buf, "%d\n", ap_irq_flag ? 1 : 0);
+}
+
+static BUS_ATTR_RO(ap_interrupts);
+
+static ssize_t config_time_show(const struct bus_type *bus, char *buf)
+{
+	return sysfs_emit(buf, "%d\n", ap_scan_bus_time);
+}
+
+static ssize_t config_time_store(const struct bus_type *bus,
+				 const char *buf, size_t count)
+{
+	int time;
+
+	if (sscanf(buf, "%d\n", &time) != 1 || time < 5 || time > 120)
+		return -EINVAL;
+	ap_scan_bus_time = time;
+	mod_timer(&ap_scan_bus_timer, jiffies + ap_scan_bus_time * HZ);
+	return count;
+}
+
+static BUS_ATTR_RW(config_time);
+
+static ssize_t poll_thread_show(const struct bus_type *bus, char *buf)
+{
+	return sysfs_emit(buf, "%d\n", ap_poll_kthread ? 1 : 0);
+}
+
+static ssize_t poll_thread_store(const struct bus_type *bus,
+				 const char *buf, size_t count)
+{
+	bool value;
+	int rc;
+
+	rc = kstrtobool(buf, &value);
+	if (rc)
+		return rc;
+
+	if (value) {
+		rc = ap_poll_thread_start();
+		if (rc)
+			count = rc;
+	} else {
+		ap_poll_thread_stop();
+	}
+	return count;
+}
+
+static BUS_ATTR_RW(poll_thread);
+
+static ssize_t poll_timeout_show(const struct bus_type *bus, char *buf)
+{
+	return sysfs_emit(buf, "%lu\n", poll_high_timeout);
+}
+
+static ssize_t poll_timeout_store(const struct bus_type *bus, const char *buf,
+				  size_t count)
+{
+	unsigned long value;
+	ktime_t hr_time;
+	int rc;
+
+	rc = kstrtoul(buf, 0, &value);
+	if (rc)
+		return rc;
+
+	/* 120 seconds = maximum poll interval */
+	if (value > 120000000000UL)
+		return -EINVAL;
+	poll_high_timeout = value;
+	hr_time = poll_high_timeout;
+
+	spin_lock_bh(&ap_poll_timer_lock);
+	hrtimer_cancel(&ap_poll_timer);
+	hrtimer_set_expires(&ap_poll_timer, hr_time);
+	hrtimer_start_expires(&ap_poll_timer, HRTIMER_MODE_ABS);
+	spin_unlock_bh(&ap_poll_timer_lock);
+
+	return count;
+}
+
+static BUS_ATTR_RW(poll_timeout);
+
+static ssize_t ap_max_domain_id_show(const struct bus_type *bus, char *buf)
+{
+	return sysfs_emit(buf, "%d\n", ap_max_domain_id);
+}
+
+static BUS_ATTR_RO(ap_max_domain_id);
+
+static ssize_t ap_max_adapter_id_show(const struct bus_type *bus, char *buf)
+{
+	return sysfs_emit(buf, "%d\n", ap_max_adapter_id);
+}
+
+static BUS_ATTR_RO(ap_max_adapter_id);
+
+static ssize_t apmask_show(const struct bus_type *bus, char *buf)
+{
+	int rc;
+
+	if (mutex_lock_interruptible(&ap_attr_mutex))
+		return -ERESTARTSYS;
+	rc = sysfs_emit(buf, "0x%016lx%016lx%016lx%016lx\n",
+			ap_perms.apm[0], ap_perms.apm[1],
+			ap_perms.apm[2], ap_perms.apm[3]);
+	mutex_unlock(&ap_attr_mutex);
+
+	return rc;
+}
+
+static int __verify_card_reservations(struct device_driver *drv, void *data)
+{
+	int rc = 0;
+	struct ap_driver *ap_drv = to_ap_drv(drv);
+	unsigned long *newapm = (unsigned long *)data;
+	unsigned long aqm_any[BITS_TO_LONGS(AP_DOMAINS)];
+
+	/*
+	 * increase the driver's module refcounter to be sure it is not
+	 * going away when we invoke the callback function.
+	 */
+	if (!try_module_get(drv->owner))
+		return 0;
+
+	if (ap_drv->in_use) {
+		bitmap_fill(aqm_any, AP_DOMAINS);
+		rc = ap_drv->in_use(newapm, aqm_any);
+		if (rc)
+			rc = -EBUSY;
+	}
+
+	/* release the driver's module */
+	module_put(drv->owner);
+
+	return rc;
+}
+
+static int apmask_commit(unsigned long *newapm)
+{
+	int rc;
+	unsigned long reserved[BITS_TO_LONGS(AP_DEVICES)];
+
+	/*
+	 * Check if any bits in the apmask have been set which will
+	 * result in queues being removed from non-default drivers
+	 */
+	if (bitmap_andnot(reserved, newapm, ap_perms.apm, AP_DEVICES)) {
+		rc = bus_for_each_drv(&ap_bus_type, NULL, reserved,
+				      __verify_card_reservations);
+		if (rc)
+			return rc;
+	}
+
+	memcpy(ap_perms.apm, newapm, APMASKSIZE);
+
+	/*
+	 * Update ap_apmask_aqmask_in_use. Note that the
+	 * ap_attr_mutex has to be obtained here.
+	 */
+	ap_apmask_aqmask_in_use =
+		bitmap_full(ap_perms.apm, AP_DEVICES) &&
+		bitmap_full(ap_perms.aqm, AP_DOMAINS) ?
+		false : true;
+
+	return 0;
+}
+
+static ssize_t apmask_store(const struct bus_type *bus, const char *buf,
+			    size_t count)
+{
+	DECLARE_BITMAP(newapm, AP_DEVICES);
+	int rc = -EINVAL, changes = 0;
+
+	if (mutex_lock_interruptible(&ap_attr_mutex))
+		return -ERESTARTSYS;
+
+	/* Do not allow apmask/aqmask if driver override is active */
+	if (ap_driver_override_ctr)
+		goto done;
+
+	rc = ap_parse_bitmap_str(buf, ap_perms.apm, AP_DEVICES, newapm);
+	if (rc)
+		goto done;
+
+	changes = memcmp(ap_perms.apm, newapm, APMASKSIZE);
+	if (changes)
+		rc = apmask_commit(newapm);
+
+done:
+	mutex_unlock(&ap_attr_mutex);
+	if (rc)
+		return rc;
+
+	if (changes) {
+		ap_bus_revise_bindings();
+		ap_send_mask_changed_uevent(newapm, NULL);
+	}
+
+	return count;
+}
+
+static BUS_ATTR_RW(apmask);
+
+static ssize_t aqmask_show(const struct bus_type *bus, char *buf)
+{
+	int rc;
+
+	if (mutex_lock_interruptible(&ap_attr_mutex))
+		return -ERESTARTSYS;
+	rc = sysfs_emit(buf, "0x%016lx%016lx%016lx%016lx\n",
+			ap_perms.aqm[0], ap_perms.aqm[1],
+			ap_perms.aqm[2], ap_perms.aqm[3]);
+	mutex_unlock(&ap_attr_mutex);
+
+	return rc;
+}
+
+static int __verify_queue_reservations(struct device_driver *drv, void *data)
+{
+	int rc = 0;
+	struct ap_driver *ap_drv = to_ap_drv(drv);
+	unsigned long *newaqm = (unsigned long *)data;
+	unsigned long apm_any[BITS_TO_LONGS(AP_DEVICES)];
+
+	/*
+	 * increase the driver's module refcounter to be sure it is not
+	 * going away when we invoke the callback function.
+	 */
+	if (!try_module_get(drv->owner))
+		return 0;
+
+	if (ap_drv->in_use) {
+		bitmap_fill(apm_any, AP_DEVICES);
+		rc = ap_drv->in_use(apm_any, newaqm);
+		if (rc)
+			rc = -EBUSY;
+	}
+
+	/* release the driver's module */
+	module_put(drv->owner);
+
+	return rc;
+}
+
+static int aqmask_commit(unsigned long *newaqm)
+{
+	int rc;
+	unsigned long reserved[BITS_TO_LONGS(AP_DOMAINS)];
+
+	/*
+	 * Check if any bits in the aqmask have been set which will
+	 * result in queues being removed from non-default drivers
+	 */
+	if (bitmap_andnot(reserved, newaqm, ap_perms.aqm, AP_DOMAINS)) {
+		rc = bus_for_each_drv(&ap_bus_type, NULL, reserved,
+				      __verify_queue_reservations);
+		if (rc)
+			return rc;
+	}
+
+	memcpy(ap_perms.aqm, newaqm, AQMASKSIZE);
+
+	/*
+	 * Update ap_apmask_aqmask_in_use. Note that the
+	 * ap_attr_mutex has to be obtained here.
+	 */
+	ap_apmask_aqmask_in_use =
+		bitmap_full(ap_perms.apm, AP_DEVICES) &&
+		bitmap_full(ap_perms.aqm, AP_DOMAINS) ?
+		false : true;
+
+	return 0;
+}
+
+static ssize_t aqmask_store(const struct bus_type *bus, const char *buf,
+			    size_t count)
+{
+	DECLARE_BITMAP(newaqm, AP_DOMAINS);
+	int rc = -EINVAL, changes = 0;
+
+	if (mutex_lock_interruptible(&ap_attr_mutex))
+		return -ERESTARTSYS;
+
+	/* Do not allow apmask/aqmask if driver override is active */
+	if (ap_driver_override_ctr)
+		goto done;
+
+	rc = ap_parse_bitmap_str(buf, ap_perms.aqm, AP_DOMAINS, newaqm);
+	if (rc)
+		goto done;
+
+	changes = memcmp(ap_perms.aqm, newaqm, APMASKSIZE);
+	if (changes)
+		rc = aqmask_commit(newaqm);
+
+done:
+	mutex_unlock(&ap_attr_mutex);
+	if (rc)
+		return rc;
+
+	if (changes) {
+		ap_bus_revise_bindings();
+		ap_send_mask_changed_uevent(NULL, newaqm);
+	}
+
+	return count;
+}
+
+static BUS_ATTR_RW(aqmask);
+
+static ssize_t scans_show(const struct bus_type *bus, char *buf)
+{
+	return sysfs_emit(buf, "%llu\n", atomic64_read(&ap_scan_bus_count));
+}
+
+static ssize_t scans_store(const struct bus_type *bus, const char *buf,
+			   size_t count)
+{
+	AP_DBF_INFO("%s force AP bus rescan\n", __func__);
+
+	ap_bus_force_rescan();
+
+	return count;
+}
+
+static BUS_ATTR_RW(scans);
+
+static ssize_t bindings_show(const struct bus_type *bus, char *buf)
+{
+	int rc;
+	unsigned int apqns, n;
+
+	ap_calc_bound_apqns(&apqns, &n);
+	if (atomic64_read(&ap_scan_bus_count) >= 1 && n == apqns)
+		rc = sysfs_emit(buf, "%u/%u (complete)\n", n, apqns);
+	else
+		rc = sysfs_emit(buf, "%u/%u\n", n, apqns);
+
+	return rc;
+}
+
+static BUS_ATTR_RO(bindings);
+
+static ssize_t bindings_complete_count_show(const struct bus_type *bus,
+					    char *buf)
+{
+	return sysfs_emit(buf, "%llu\n",
+			  atomic64_read(&ap_bindings_complete_count));
+}
+
+static BUS_ATTR_RO(bindings_complete_count);
+
+static ssize_t features_show(const struct bus_type *bus, char *buf)
+{
+	int n = 0;
+
+	if (!ap_qci_info->flags)	/* QCI not supported */
+		return sysfs_emit(buf, "-\n");
+
+	if (ap_qci_info->apsc)
+		n += sysfs_emit_at(buf, n, "APSC ");
+	if (ap_qci_info->apxa)
+		n += sysfs_emit_at(buf, n, "APXA ");
+	if (ap_qci_info->qact)
+		n += sysfs_emit_at(buf, n, "QACT ");
+	if (ap_qci_info->rc8a)
+		n += sysfs_emit_at(buf, n, "RC8A ");
+	if (ap_qci_info->apsb)
+		n += sysfs_emit_at(buf, n, "APSB ");
+
+	sysfs_emit_at(buf, n == 0 ? 0 : n - 1, "\n");
+
+	return n;
+}
+
+static BUS_ATTR_RO(features);
+
+static struct attribute *ap_bus_attrs[] = {
+	&bus_attr_ap_domain.attr,
+	&bus_attr_ap_control_domain_mask.attr,
+	&bus_attr_ap_usage_domain_mask.attr,
+	&bus_attr_ap_adapter_mask.attr,
+	&bus_attr_config_time.attr,
+	&bus_attr_poll_thread.attr,
+	&bus_attr_ap_interrupts.attr,
+	&bus_attr_poll_timeout.attr,
+	&bus_attr_ap_max_domain_id.attr,
+	&bus_attr_ap_max_adapter_id.attr,
+	&bus_attr_apmask.attr,
+	&bus_attr_aqmask.attr,
+	&bus_attr_scans.attr,
+	&bus_attr_bindings.attr,
+	&bus_attr_bindings_complete_count.attr,
+	&bus_attr_features.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(ap_bus);
+
+static const struct bus_type ap_bus_type = {
+	.name = "ap",
+	.bus_groups = ap_bus_groups,
+	.match = &ap_bus_match,
+	.uevent = &ap_uevent,
+	.probe = ap_device_probe,
+	.remove = ap_device_remove,
+};
+
+/**
+ * ap_select_domain(): Select an AP domain if possible and we haven't
+ * already done so before.
+ */
+static void ap_select_domain(void)
+{
+	struct ap_queue_status status;
+	int card, dom;
+
+	/*
+	 * Choose the default domain. Either the one specified with
+	 * the "domain=" parameter or the first domain with at least
+	 * one valid APQN.
+	 */
+	spin_lock_bh(&ap_domain_lock);
+	if (ap_domain_index >= 0) {
+		/* Domain has already been selected. */
+		goto out;
+	}
+	for (dom = 0; dom <= ap_max_domain_id; dom++) {
+		if (!ap_test_config_usage_domain(dom) ||
+		    !test_bit_inv(dom, ap_perms.aqm))
+			continue;
+		for (card = 0; card <= ap_max_adapter_id; card++) {
+			if (!ap_test_config_card_id(card) ||
+			    !test_bit_inv(card, ap_perms.apm))
+				continue;
+			status = ap_test_queue(AP_MKQID(card, dom),
+					       ap_apft_available(),
+					       NULL);
+			if (status.response_code == AP_RESPONSE_NORMAL)
+				break;
+		}
+		if (card <= ap_max_adapter_id)
+			break;
+	}
+	if (dom <= ap_max_domain_id) {
+		ap_domain_index = dom;
+		AP_DBF_INFO("%s new default domain is %d\n",
+			    __func__, ap_domain_index);
+	}
+out:
+	spin_unlock_bh(&ap_domain_lock);
+}
+
+/*
+ * This function checks the type and returns either 0 for not
+ * supported or the highest compatible type value (which may
+ * include the input type value).
+ */
+static int ap_get_compatible_type(ap_qid_t qid, int rawtype, unsigned int func)
+{
+	int comp_type = 0;
+
+	/* < CEX4 is not supported */
+	if (rawtype < AP_DEVICE_TYPE_CEX4) {
+		AP_DBF_WARN("%s queue=%02x.%04x unsupported type %d\n",
+			    __func__, AP_QID_CARD(qid),
+			    AP_QID_QUEUE(qid), rawtype);
+		return 0;
+	}
+	/* up to CEX8 known and fully supported */
+	if (rawtype <= AP_DEVICE_TYPE_CEX8)
+		return rawtype;
+	/*
+	 * unknown new type > CEX8, check for compatibility
+	 * to the highest known and supported type which is
+	 * currently CEX8 with the help of the QACT function.
+	 */
+	if (ap_qact_available()) {
+		struct ap_queue_status status;
+		union ap_qact_ap_info apinfo = {0};
+
+		apinfo.mode = (func >> 26) & 0x07;
+		apinfo.cat = AP_DEVICE_TYPE_CEX8;
+		status = ap_qact(qid, 0, &apinfo);
+		if (status.response_code == AP_RESPONSE_NORMAL &&
+		    apinfo.cat >= AP_DEVICE_TYPE_CEX4 &&
+		    apinfo.cat <= AP_DEVICE_TYPE_CEX8)
+			comp_type = apinfo.cat;
+	}
+	if (!comp_type)
+		AP_DBF_WARN("%s queue=%02x.%04x unable to map type %d\n",
+			    __func__, AP_QID_CARD(qid),
+			    AP_QID_QUEUE(qid), rawtype);
+	else if (comp_type != rawtype)
+		AP_DBF_INFO("%s queue=%02x.%04x map type %d to %d\n",
+			    __func__, AP_QID_CARD(qid), AP_QID_QUEUE(qid),
+			    rawtype, comp_type);
+	return comp_type;
+}
+
+/*
+ * Helper function to be used with bus_find_dev
+ * matches for the card device with the given id
+ */
+static int __match_card_device_with_id(struct device *dev, const void *data)
+{
+	return is_card_dev(dev) && to_ap_card(dev)->id == (int)(long)(void *)data;
+}
+
+/*
+ * Helper function to be used with bus_find_dev
+ * matches for the queue device with a given qid
+ */
+static int __match_queue_device_with_qid(struct device *dev, const void *data)
+{
+	return is_queue_dev(dev) && to_ap_queue(dev)->qid == (int)(long)data;
+}
+
+/*
+ * Helper function to be used with bus_find_dev
+ * matches any queue device with given queue id
+ */
+static int __match_queue_device_with_queue_id(struct device *dev, const void *data)
+{
+	return is_queue_dev(dev) &&
+		AP_QID_QUEUE(to_ap_queue(dev)->qid) == (int)(long)data;
+}
+
+/* Helper function for notify_config_changed */
+static int __drv_notify_config_changed(struct device_driver *drv, void *data)
+{
+	struct ap_driver *ap_drv = to_ap_drv(drv);
+
+	if (try_module_get(drv->owner)) {
+		if (ap_drv->on_config_changed)
+			ap_drv->on_config_changed(ap_qci_info, ap_qci_info_old);
+		module_put(drv->owner);
+	}
+
+	return 0;
+}
+
+/* Notify all drivers about an qci config change */
+static inline void notify_config_changed(void)
+{
+	bus_for_each_drv(&ap_bus_type, NULL, NULL,
+			 __drv_notify_config_changed);
+}
+
+/* Helper function for notify_scan_complete */
+static int __drv_notify_scan_complete(struct device_driver *drv, void *data)
+{
+	struct ap_driver *ap_drv = to_ap_drv(drv);
+
+	if (try_module_get(drv->owner)) {
+		if (ap_drv->on_scan_complete)
+			ap_drv->on_scan_complete(ap_qci_info,
+						 ap_qci_info_old);
+		module_put(drv->owner);
+	}
+
+	return 0;
+}
+
+/* Notify all drivers about bus scan complete */
+static inline void notify_scan_complete(void)
+{
+	bus_for_each_drv(&ap_bus_type, NULL, NULL,
+			 __drv_notify_scan_complete);
+}
+
+/*
+ * Helper function for ap_scan_bus().
+ * Remove card device and associated queue devices.
+ */
+static inline void ap_scan_rm_card_dev_and_queue_devs(struct ap_card *ac)
+{
+	bus_for_each_dev(&ap_bus_type, NULL,
+			 (void *)(long)ac->id,
+			 __ap_queue_devices_with_id_unregister);
+	device_unregister(&ac->ap_dev.device);
+}
+
+/*
+ * Helper function for ap_scan_bus().
+ * Does the scan bus job for all the domains within
+ * a valid adapter given by an ap_card ptr.
+ */
+static inline void ap_scan_domains(struct ap_card *ac)
+{
+	struct ap_tapq_hwinfo hwinfo;
+	bool decfg, chkstop;
+	struct ap_queue *aq;
+	struct device *dev;
+	ap_qid_t qid;
+	int rc, dom;
+
+	/*
+	 * Go through the configuration for the domains and compare them
+	 * to the existing queue devices. Also take care of the config
+	 * and error state for the queue devices.
+	 */
+
+	for (dom = 0; dom <= ap_max_domain_id; dom++) {
+		qid = AP_MKQID(ac->id, dom);
+		dev = bus_find_device(&ap_bus_type, NULL,
+				      (void *)(long)qid,
+				      __match_queue_device_with_qid);
+		aq = dev ? to_ap_queue(dev) : NULL;
+		if (!ap_test_config_usage_domain(dom)) {
+			if (dev) {
+				AP_DBF_INFO("%s(%d,%d) not in config anymore, rm queue dev\n",
+					    __func__, ac->id, dom);
+				device_unregister(dev);
+			}
+			goto put_dev_and_continue;
+		}
+		/* domain is valid, get info from this APQN */
+		rc = ap_queue_info(qid, &hwinfo, &decfg, &chkstop);
+		switch (rc) {
+		case -1:
+			if (dev) {
+				AP_DBF_INFO("%s(%d,%d) queue_info() failed, rm queue dev\n",
+					    __func__, ac->id, dom);
+				device_unregister(dev);
+			}
+			fallthrough;
+		case 0:
+			goto put_dev_and_continue;
+		default:
+			break;
+		}
+		/* if no queue device exists, create a new one */
+		if (!aq) {
+			aq = ap_queue_create(qid, ac);
+			if (!aq) {
+				AP_DBF_WARN("%s(%d,%d) ap_queue_create() failed\n",
+					    __func__, ac->id, dom);
+				continue;
+			}
+			aq->config = !decfg;
+			aq->chkstop = chkstop;
+			aq->se_bstate = hwinfo.bs;
+			dev = &aq->ap_dev.device;
+			dev->bus = &ap_bus_type;
+			dev->parent = &ac->ap_dev.device;
+			dev_set_name(dev, "%02x.%04x", ac->id, dom);
+			/* register queue device */
+			rc = device_register(dev);
+			if (rc) {
+				AP_DBF_WARN("%s(%d,%d) device_register() failed\n",
+					    __func__, ac->id, dom);
+				goto put_dev_and_continue;
+			}
+			/* get it and thus adjust reference counter */
+			get_device(dev);
+			if (decfg) {
+				AP_DBF_INFO("%s(%d,%d) new (decfg) queue dev created\n",
+					    __func__, ac->id, dom);
+			} else if (chkstop) {
+				AP_DBF_INFO("%s(%d,%d) new (chkstop) queue dev created\n",
+					    __func__, ac->id, dom);
+			} else {
+				/* nudge the queue's state machine */
+				ap_queue_init_state(aq);
+				AP_DBF_INFO("%s(%d,%d) new queue dev created\n",
+					    __func__, ac->id, dom);
+			}
+			goto put_dev_and_continue;
+		}
+		/* handle state changes on already existing queue device */
+		spin_lock_bh(&aq->lock);
+		/* SE bind state */
+		aq->se_bstate = hwinfo.bs;
+		/* checkstop state */
+		if (chkstop && !aq->chkstop) {
+			/* checkstop on */
+			aq->chkstop = true;
+			if (aq->dev_state > AP_DEV_STATE_UNINITIATED) {
+				aq->dev_state = AP_DEV_STATE_ERROR;
+				aq->last_err_rc = AP_RESPONSE_CHECKSTOPPED;
+			}
+			spin_unlock_bh(&aq->lock);
+			pr_debug("(%d,%d) queue dev checkstop on\n",
+				 ac->id, dom);
+			/* 'receive' pending messages with -EAGAIN */
+			ap_flush_queue(aq);
+			goto put_dev_and_continue;
+		} else if (!chkstop && aq->chkstop) {
+			/* checkstop off */
+			aq->chkstop = false;
+			if (aq->dev_state > AP_DEV_STATE_UNINITIATED)
+				_ap_queue_init_state(aq);
+			spin_unlock_bh(&aq->lock);
+			pr_debug("(%d,%d) queue dev checkstop off\n",
+				 ac->id, dom);
+			goto put_dev_and_continue;
+		}
+		/* config state change */
+		if (decfg && aq->config) {
+			/* config off this queue device */
+			aq->config = false;
+			if (aq->dev_state > AP_DEV_STATE_UNINITIATED) {
+				aq->dev_state = AP_DEV_STATE_ERROR;
+				aq->last_err_rc = AP_RESPONSE_DECONFIGURED;
+			}
+			spin_unlock_bh(&aq->lock);
+			pr_debug("(%d,%d) queue dev config off\n",
+				 ac->id, dom);
+			ap_send_config_uevent(&aq->ap_dev, aq->config);
+			/* 'receive' pending messages with -EAGAIN */
+			ap_flush_queue(aq);
+			goto put_dev_and_continue;
+		} else if (!decfg && !aq->config) {
+			/* config on this queue device */
+			aq->config = true;
+			if (aq->dev_state > AP_DEV_STATE_UNINITIATED)
+				_ap_queue_init_state(aq);
+			spin_unlock_bh(&aq->lock);
+			pr_debug("(%d,%d) queue dev config on\n",
+				 ac->id, dom);
+			ap_send_config_uevent(&aq->ap_dev, aq->config);
+			goto put_dev_and_continue;
+		}
+		/* handle other error states */
+		if (!decfg && aq->dev_state == AP_DEV_STATE_ERROR) {
+			spin_unlock_bh(&aq->lock);
+			/* 'receive' pending messages with -EAGAIN */
+			ap_flush_queue(aq);
+			/* re-init (with reset) the queue device */
+			ap_queue_init_state(aq);
+			AP_DBF_INFO("%s(%d,%d) queue dev reinit enforced\n",
+				    __func__, ac->id, dom);
+			goto put_dev_and_continue;
+		}
+		spin_unlock_bh(&aq->lock);
+put_dev_and_continue:
+		put_device(dev);
+	}
+}
+
+/*
+ * Helper function for ap_scan_bus().
+ * Does the scan bus job for the given adapter id.
+ */
+static inline void ap_scan_adapter(int ap)
+{
+	struct ap_tapq_hwinfo hwinfo;
+	int rc, dom, comp_type;
+	bool decfg, chkstop;
+	struct ap_card *ac;
+	struct device *dev;
+	ap_qid_t qid;
+
+	/* Is there currently a card device for this adapter ? */
+	dev = bus_find_device(&ap_bus_type, NULL,
+			      (void *)(long)ap,
+			      __match_card_device_with_id);
+	ac = dev ? to_ap_card(dev) : NULL;
+
+	/* Adapter not in configuration ? */
+	if (!ap_test_config_card_id(ap)) {
+		if (ac) {
+			AP_DBF_INFO("%s(%d) ap not in config any more, rm card and queue devs\n",
+				    __func__, ap);
+			ap_scan_rm_card_dev_and_queue_devs(ac);
+			put_device(dev);
+		}
+		return;
+	}
+
+	/*
+	 * Adapter ap is valid in the current configuration. So do some checks:
+	 * If no card device exists, build one. If a card device exists, check
+	 * for type and functions changed. For all this we need to find a valid
+	 * APQN first.
+	 */
+
+	for (dom = 0; dom <= ap_max_domain_id; dom++)
+		if (ap_test_config_usage_domain(dom)) {
+			qid = AP_MKQID(ap, dom);
+			if (ap_queue_info(qid, &hwinfo, &decfg, &chkstop) > 0)
+				break;
+		}
+	if (dom > ap_max_domain_id) {
+		/* Could not find one valid APQN for this adapter */
+		if (ac) {
+			AP_DBF_INFO("%s(%d) no type info (no APQN found), rm card and queue devs\n",
+				    __func__, ap);
+			ap_scan_rm_card_dev_and_queue_devs(ac);
+			put_device(dev);
+		} else {
+			pr_debug("(%d) no type info (no APQN found), ignored\n",
+				 ap);
+		}
+		return;
+	}
+	if (!hwinfo.at) {
+		/* No apdater type info available, an unusable adapter */
+		if (ac) {
+			AP_DBF_INFO("%s(%d) no valid type (0) info, rm card and queue devs\n",
+				    __func__, ap);
+			ap_scan_rm_card_dev_and_queue_devs(ac);
+			put_device(dev);
+		} else {
+			pr_debug("(%d) no valid type (0) info, ignored\n", ap);
+		}
+		return;
+	}
+	hwinfo.value &= TAPQ_CARD_HWINFO_MASK; /* filter card specific hwinfo */
+	if (ac) {
+		/* Check APQN against existing card device for changes */
+		if (ac->hwinfo.at != hwinfo.at) {
+			AP_DBF_INFO("%s(%d) hwtype %d changed, rm card and queue devs\n",
+				    __func__, ap, hwinfo.at);
+			ap_scan_rm_card_dev_and_queue_devs(ac);
+			put_device(dev);
+			ac = NULL;
+		} else if (ac->hwinfo.fac != hwinfo.fac) {
+			AP_DBF_INFO("%s(%d) functions 0x%08x changed, rm card and queue devs\n",
+				    __func__, ap, hwinfo.fac);
+			ap_scan_rm_card_dev_and_queue_devs(ac);
+			put_device(dev);
+			ac = NULL;
+		} else {
+			/* handle checkstop state change */
+			if (chkstop && !ac->chkstop) {
+				/* checkstop on */
+				ac->chkstop = true;
+				AP_DBF_INFO("%s(%d) card dev checkstop on\n",
+					    __func__, ap);
+			} else if (!chkstop && ac->chkstop) {
+				/* checkstop off */
+				ac->chkstop = false;
+				AP_DBF_INFO("%s(%d) card dev checkstop off\n",
+					    __func__, ap);
+			}
+			/* handle config state change */
+			if (decfg && ac->config) {
+				ac->config = false;
+				AP_DBF_INFO("%s(%d) card dev config off\n",
+					    __func__, ap);
+				ap_send_config_uevent(&ac->ap_dev, ac->config);
+			} else if (!decfg && !ac->config) {
+				ac->config = true;
+				AP_DBF_INFO("%s(%d) card dev config on\n",
+					    __func__, ap);
+				ap_send_config_uevent(&ac->ap_dev, ac->config);
+			}
+		}
+	}
+
+	if (!ac) {
+		/* Build a new card device */
+		comp_type = ap_get_compatible_type(qid, hwinfo.at, hwinfo.fac);
+		if (!comp_type) {
+			AP_DBF_WARN("%s(%d) type %d, can't get compatibility type\n",
+				    __func__, ap, hwinfo.at);
+			return;
+		}
+		ac = ap_card_create(ap, hwinfo, comp_type);
+		if (!ac) {
+			AP_DBF_WARN("%s(%d) ap_card_create() failed\n",
+				    __func__, ap);
+			return;
+		}
+		ac->config = !decfg;
+		ac->chkstop = chkstop;
+		dev = &ac->ap_dev.device;
+		dev->bus = &ap_bus_type;
+		dev->parent = ap_root_device;
+		dev_set_name(dev, "card%02x", ap);
+		/* maybe enlarge ap_max_msg_size to support this card */
+		if (ac->maxmsgsize > atomic_read(&ap_max_msg_size)) {
+			atomic_set(&ap_max_msg_size, ac->maxmsgsize);
+			AP_DBF_INFO("%s(%d) ap_max_msg_size update to %d byte\n",
+				    __func__, ap,
+				    atomic_read(&ap_max_msg_size));
+		}
+		/* Register the new card device with AP bus */
+		rc = device_register(dev);
+		if (rc) {
+			AP_DBF_WARN("%s(%d) device_register() failed\n",
+				    __func__, ap);
+			put_device(dev);
+			return;
+		}
+		/* get it and thus adjust reference counter */
+		get_device(dev);
+		if (decfg)
+			AP_DBF_INFO("%s(%d) new (decfg) card dev type=%d func=0x%08x created\n",
+				    __func__, ap, hwinfo.at, hwinfo.fac);
+		else if (chkstop)
+			AP_DBF_INFO("%s(%d) new (chkstop) card dev type=%d func=0x%08x created\n",
+				    __func__, ap, hwinfo.at, hwinfo.fac);
+		else
+			AP_DBF_INFO("%s(%d) new card dev type=%d func=0x%08x created\n",
+				    __func__, ap, hwinfo.at, hwinfo.fac);
+	}
+
+	/* Verify the domains and the queue devices for this card */
+	ap_scan_domains(ac);
+
+	/* release the card device */
+	put_device(&ac->ap_dev.device);
+}
+
+/**
+ * ap_get_configuration - get the host AP configuration
+ *
+ * Stores the host AP configuration information returned from the previous call
+ * to Query Configuration Information (QCI), then retrieves and stores the
+ * current AP configuration returned from QCI.
+ *
+ * Return: true if the host AP configuration changed between calls to QCI;
+ * otherwise, return false.
+ */
+static bool ap_get_configuration(void)
+{
+	if (!ap_qci_info->flags)	/* QCI not supported */
+		return false;
+
+	memcpy(ap_qci_info_old, ap_qci_info, sizeof(*ap_qci_info));
+	ap_qci(ap_qci_info);
+
+	return memcmp(ap_qci_info, ap_qci_info_old,
+		      sizeof(struct ap_config_info)) != 0;
+}
+
+/*
+ * ap_config_has_new_aps - Check current against old qci info if
+ * new adapters have appeared. Returns true if at least one new
+ * adapter in the apm mask is showing up. Existing adapters or
+ * receding adapters are not counted.
+ */
+static bool ap_config_has_new_aps(void)
+{
+
+	unsigned long m[BITS_TO_LONGS(AP_DEVICES)];
+
+	if (!ap_qci_info->flags)
+		return false;
+
+	bitmap_andnot(m, (unsigned long *)ap_qci_info->apm,
+		      (unsigned long *)ap_qci_info_old->apm, AP_DEVICES);
+	if (!bitmap_empty(m, AP_DEVICES))
+		return true;
+
+	return false;
+}
+
+/*
+ * ap_config_has_new_doms - Check current against old qci info if
+ * new (usage) domains have appeared. Returns true if at least one
+ * new domain in the aqm mask is showing up. Existing domains or
+ * receding domains are not counted.
+ */
+static bool ap_config_has_new_doms(void)
+{
+	unsigned long m[BITS_TO_LONGS(AP_DOMAINS)];
+
+	if (!ap_qci_info->flags)
+		return false;
+
+	bitmap_andnot(m, (unsigned long *)ap_qci_info->aqm,
+		      (unsigned long *)ap_qci_info_old->aqm, AP_DOMAINS);
+	if (!bitmap_empty(m, AP_DOMAINS))
+		return true;
+
+	return false;
+}
+
+/**
+ * ap_scan_bus(): Scan the AP bus for new devices
+ * Always run under mutex ap_scan_bus_mutex protection
+ * which needs to get locked/unlocked by the caller!
+ * Returns true if any config change has been detected
+ * during the scan, otherwise false.
+ */
+static bool ap_scan_bus(void)
+{
+	bool config_changed;
+	int ap;
+
+	pr_debug(">\n");
+
+	/* (re-)fetch configuration via QCI */
+	config_changed = ap_get_configuration();
+	if (config_changed) {
+		if (ap_config_has_new_aps() || ap_config_has_new_doms()) {
+			/*
+			 * Appearance of new adapters and/or domains need to
+			 * build new ap devices which need to get bound to an
+			 * device driver. Thus reset the APQN bindings complete
+			 * completion.
+			 */
+			reinit_completion(&ap_apqn_bindings_complete);
+		}
+		/* post a config change notify */
+		notify_config_changed();
+	}
+	ap_select_domain();
+
+	/* loop over all possible adapters */
+	for (ap = 0; ap <= ap_max_adapter_id; ap++)
+		ap_scan_adapter(ap);
+
+	/* scan complete notify */
+	if (config_changed)
+		notify_scan_complete();
+
+	/* check if there is at least one queue available with default domain */
+	if (ap_domain_index >= 0) {
+		struct device *dev =
+			bus_find_device(&ap_bus_type, NULL,
+					(void *)(long)ap_domain_index,
+					__match_queue_device_with_queue_id);
+		if (dev)
+			put_device(dev);
+		else
+			AP_DBF_INFO("%s no queue device with default domain %d available\n",
+				    __func__, ap_domain_index);
+	}
+
+	if (atomic64_inc_return(&ap_scan_bus_count) == 1) {
+		pr_debug("init scan complete\n");
+		ap_send_init_scan_done_uevent();
+	}
+
+	ap_check_bindings_complete();
+
+	mod_timer(&ap_scan_bus_timer, jiffies + ap_scan_bus_time * HZ);
+
+	pr_debug("< config_changed=%d\n", config_changed);
+
+	return config_changed;
+}
+
+/*
+ * Callback for the ap_scan_bus_timer
+ * Runs periodically, workqueue timer (ap_scan_bus_time)
+ */
+static void ap_scan_bus_timer_callback(struct timer_list *unused)
+{
+	/*
+	 * schedule work into the system long wq which when
+	 * the work is finally executed, calls the AP bus scan.
+	 */
+	queue_work(system_long_wq, &ap_scan_bus_work);
+}
+
+/*
+ * Callback for the ap_scan_bus_work
+ */
+static void ap_scan_bus_wq_callback(struct work_struct *unused)
+{
+	/*
+	 * Try to invoke an ap_scan_bus(). If the mutex acquisition
+	 * fails there is currently another task already running the
+	 * AP scan bus and there is no need to wait and re-trigger the
+	 * scan again. Please note at the end of the scan bus function
+	 * the AP scan bus timer is re-armed which triggers then the
+	 * ap_scan_bus_timer_callback which enqueues a work into the
+	 * system_long_wq which invokes this function here again.
+	 */
+	if (mutex_trylock(&ap_scan_bus_mutex)) {
+		ap_scan_bus_task = current;
+		ap_scan_bus_result = ap_scan_bus();
+		ap_scan_bus_task = NULL;
+		mutex_unlock(&ap_scan_bus_mutex);
+	}
+}
+
+static inline void __exit ap_async_exit(void)
+{
+	if (ap_thread_flag)
+		ap_poll_thread_stop();
+	chsc_notifier_unregister(&ap_bus_nb);
+	cancel_work(&ap_scan_bus_work);
+	hrtimer_cancel(&ap_poll_timer);
+	timer_delete(&ap_scan_bus_timer);
+}
+
+static inline int __init ap_async_init(void)
+{
+	int rc;
+
+	/* Setup the AP bus rescan timer. */
+	timer_setup(&ap_scan_bus_timer, ap_scan_bus_timer_callback, 0);
+
+	/*
+	 * Setup the high resolution poll timer.
+	 * If we are running under z/VM adjust polling to z/VM polling rate.
+	 */
+	if (machine_is_vm())
+		poll_high_timeout = 1500000;
+	hrtimer_setup(&ap_poll_timer, ap_poll_timeout, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
+
+	queue_work(system_long_wq, &ap_scan_bus_work);
+
+	rc = chsc_notifier_register(&ap_bus_nb);
+	if (rc)
+		goto out;
+
+	/* Start the low priority AP bus poll thread. */
+	if (!ap_thread_flag)
+		return 0;
+
+	rc = ap_poll_thread_start();
+	if (rc)
+		goto out_notifier;
+
+	return 0;
+
+out_notifier:
+	chsc_notifier_unregister(&ap_bus_nb);
+out:
+	cancel_work(&ap_scan_bus_work);
+	hrtimer_cancel(&ap_poll_timer);
+	timer_delete(&ap_scan_bus_timer);
+	return rc;
+}
+
+static inline void ap_irq_exit(void)
+{
+	if (ap_irq_flag)
+		unregister_adapter_interrupt(&ap_airq);
+}
+
+static inline int __init ap_irq_init(void)
+{
+	int rc;
+
+	if (!ap_interrupts_available() || !ap_useirq)
+		return 0;
+
+	rc = register_adapter_interrupt(&ap_airq);
+	ap_irq_flag = (rc == 0);
+
+	return rc;
+}
+
+static inline void ap_debug_exit(void)
+{
+	debug_unregister(ap_dbf_info);
+}
+
+static inline int __init ap_debug_init(void)
+{
+	ap_dbf_info = debug_register("ap", 2, 1,
+				     AP_DBF_MAX_SPRINTF_ARGS * sizeof(long));
+	debug_register_view(ap_dbf_info, &debug_sprintf_view);
+	debug_set_level(ap_dbf_info, DBF_ERR);
+
+	return 0;
+}
+
+static void __init ap_perms_init(void)
+{
+	/* all resources usable if no kernel parameter string given */
+	memset(&ap_perms.ioctlm, 0xFF, sizeof(ap_perms.ioctlm));
+	memset(&ap_perms.apm, 0xFF, sizeof(ap_perms.apm));
+	memset(&ap_perms.aqm, 0xFF, sizeof(ap_perms.aqm));
+
+	/* apm kernel parameter string */
+	if (apm_str) {
+		memset(&ap_perms.apm, 0, sizeof(ap_perms.apm));
+		ap_parse_mask_str(apm_str, ap_perms.apm, AP_DEVICES,
+				  &ap_attr_mutex);
+	}
+
+	/* aqm kernel parameter string */
+	if (aqm_str) {
+		memset(&ap_perms.aqm, 0, sizeof(ap_perms.aqm));
+		ap_parse_mask_str(aqm_str, ap_perms.aqm, AP_DOMAINS,
+				  &ap_attr_mutex);
+	}
+}
+
+/**
+ * ap_module_init(): The module initialization code.
+ *
+ * Initializes the module.
+ */
+static int __init ap_module_init(void)
+{
+	int rc;
+
+	if (!ap_instructions_available()) {
+		pr_warn("The hardware system does not support AP instructions\n");
+		return -ENODEV;
+	}
+
+	rc = ap_debug_init();
+	if (rc)
+		return rc;
+
+	/* init ap_queue hashtable */
+	hash_init(ap_queues);
+
+	/* create ap msg buffer memory pool */
+	ap_msg_pool = mempool_create_kmalloc_pool(ap_msg_pool_min_items,
+						  AP_DEFAULT_MAX_MSG_SIZE);
+	if (!ap_msg_pool) {
+		rc = -ENOMEM;
+		goto out;
+	}
+
+	/* set up the AP permissions (ioctls, ap and aq masks) */
+	ap_perms_init();
+
+	/* Get AP configuration data if available */
+	ap_init_qci_info();
+
+	/* check default domain setting */
+	if (ap_domain_index < -1 || ap_domain_index > ap_max_domain_id ||
+	    (ap_domain_index >= 0 &&
+	     !test_bit_inv(ap_domain_index, ap_perms.aqm))) {
+		pr_warn("%d is not a valid cryptographic domain\n",
+			ap_domain_index);
+		ap_domain_index = -1;
+	}
+
+	/* Create /sys/bus/ap. */
+	rc = bus_register(&ap_bus_type);
+	if (rc)
+		goto out;
+
+	/* Create /sys/devices/ap. */
+	ap_root_device = root_device_register("ap");
+	rc = PTR_ERR_OR_ZERO(ap_root_device);
+	if (rc)
+		goto out_bus;
+	ap_root_device->bus = &ap_bus_type;
+
+	/* enable interrupts if available */
+	rc = ap_irq_init();
+	if (rc)
+		goto out_device;
+
+	/* Setup asynchronous work (timers, workqueue, etc). */
+	rc = ap_async_init();
+	if (rc)
+		goto out_irq;
+
+	return 0;
+
+out_irq:
+	ap_irq_exit();
+out_device:
+	root_device_unregister(ap_root_device);
+out_bus:
+	bus_unregister(&ap_bus_type);
+out:
+	mempool_destroy(ap_msg_pool);
+	ap_debug_exit();
+	return rc;
+}
+
+static void __exit ap_module_exit(void)
+{
+	ap_async_exit();
+	ap_irq_exit();
+	root_device_unregister(ap_root_device);
+	bus_unregister(&ap_bus_type);
+	mempool_destroy(ap_msg_pool);
+	ap_debug_exit();
+}
+
+module_init(ap_module_init);
+module_exit(ap_module_exit);
diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h
new file mode 100644
index 000000000..fb4d67833
--- /dev/null
+++ b/drivers/s390/crypto/ap_bus.h
@@ -0,0 +1,380 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright IBM Corp. 2006, 2023
+ * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ *	      Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *	      Ralph Wuerthner <rwuerthn@de.ibm.com>
+ *	      Felix Beck <felix.beck@de.ibm.com>
+ *	      Holger Dengler <hd@linux.vnet.ibm.com>
+ *
+ * Adjunct processor bus header file.
+ */
+
+#ifndef _AP_BUS_H_
+#define _AP_BUS_H_
+
+#include <linux/device.h>
+#include <linux/types.h>
+#include <linux/hashtable.h>
+#include <asm/isc.h>
+#include <asm/ap.h>
+
+#define AP_DEVICES 256		/* Number of AP devices. */
+#define AP_DOMAINS 256		/* Number of AP domains. */
+#define AP_IOCTLS  256		/* Number of ioctls. */
+#define AP_RESET_TIMEOUT (HZ*0.7)	/* Time in ticks for reset timeouts. */
+#define AP_CONFIG_TIME 30	/* Time in seconds between AP bus rescans. */
+#define AP_POLL_TIME 1		/* Time in ticks between receive polls. */
+#define AP_DEFAULT_MAX_MSG_SIZE (12 * 1024)
+#define AP_TAPQ_ML_FIELD_CHUNK_SIZE (4096)
+
+extern int ap_domain_index;
+extern atomic_t ap_max_msg_size;
+
+extern DECLARE_HASHTABLE(ap_queues, 8);
+extern spinlock_t ap_queues_lock;
+
+static inline int ap_test_bit(unsigned int *ptr, unsigned int nr)
+{
+	return (*ptr & (0x80000000u >> nr)) != 0;
+}
+
+#define AP_RESPONSE_NORMAL		     0x00
+#define AP_RESPONSE_Q_NOT_AVAIL		     0x01
+#define AP_RESPONSE_RESET_IN_PROGRESS	     0x02
+#define AP_RESPONSE_DECONFIGURED	     0x03
+#define AP_RESPONSE_CHECKSTOPPED	     0x04
+#define AP_RESPONSE_BUSY		     0x05
+#define AP_RESPONSE_INVALID_ADDRESS	     0x06
+#define AP_RESPONSE_OTHERWISE_CHANGED	     0x07
+#define AP_RESPONSE_INVALID_GISA	     0x08
+#define AP_RESPONSE_Q_BOUND_TO_ANOTHER	     0x09
+#define AP_RESPONSE_STATE_CHANGE_IN_PROGRESS 0x0A
+#define AP_RESPONSE_Q_NOT_BOUND		     0x0B
+#define AP_RESPONSE_Q_FULL		     0x10
+#define AP_RESPONSE_NO_PENDING_REPLY	     0x10
+#define AP_RESPONSE_INDEX_TOO_BIG	     0x11
+#define AP_RESPONSE_NO_FIRST_PART	     0x13
+#define AP_RESPONSE_MESSAGE_TOO_BIG	     0x15
+#define AP_RESPONSE_REQ_FAC_NOT_INST	     0x16
+#define AP_RESPONSE_Q_BIND_ERROR	     0x30
+#define AP_RESPONSE_Q_NOT_AVAIL_FOR_ASSOC    0x31
+#define AP_RESPONSE_Q_NOT_EMPTY		     0x32
+#define AP_RESPONSE_BIND_LIMIT_EXCEEDED	     0x33
+#define AP_RESPONSE_INVALID_ASSOC_SECRET     0x34
+#define AP_RESPONSE_ASSOC_SECRET_NOT_UNIQUE  0x35
+#define AP_RESPONSE_ASSOC_FAILED	     0x36
+#define AP_RESPONSE_INVALID_DOMAIN	     0x42
+
+/*
+ * Supported AP device types
+ */
+#define AP_DEVICE_TYPE_CEX4	10
+#define AP_DEVICE_TYPE_CEX5	11
+#define AP_DEVICE_TYPE_CEX6	12
+#define AP_DEVICE_TYPE_CEX7	13
+#define AP_DEVICE_TYPE_CEX8	14
+
+/*
+ * AP queue state machine states
+ */
+enum ap_sm_state {
+	AP_SM_STATE_RESET_START = 0,
+	AP_SM_STATE_RESET_WAIT,
+	AP_SM_STATE_SETIRQ_WAIT,
+	AP_SM_STATE_IDLE,
+	AP_SM_STATE_WORKING,
+	AP_SM_STATE_QUEUE_FULL,
+	AP_SM_STATE_ASSOC_WAIT,
+	NR_AP_SM_STATES
+};
+
+/*
+ * AP queue state machine events
+ */
+enum ap_sm_event {
+	AP_SM_EVENT_POLL,
+	AP_SM_EVENT_TIMEOUT,
+	NR_AP_SM_EVENTS
+};
+
+/*
+ * AP queue state wait behaviour
+ */
+enum ap_sm_wait {
+	AP_SM_WAIT_AGAIN = 0,	 /* retry immediately */
+	AP_SM_WAIT_HIGH_TIMEOUT, /* poll high freq, wait for timeout */
+	AP_SM_WAIT_LOW_TIMEOUT,	 /* poll low freq, wait for timeout */
+	AP_SM_WAIT_INTERRUPT,	 /* wait for thin interrupt (if available) */
+	AP_SM_WAIT_NONE,	 /* no wait */
+	NR_AP_SM_WAIT
+};
+
+/*
+ * AP queue device states
+ */
+enum ap_dev_state {
+	AP_DEV_STATE_UNINITIATED = 0,	/* fresh and virgin, not touched */
+	AP_DEV_STATE_OPERATING,		/* queue dev is working normal */
+	AP_DEV_STATE_SHUTDOWN,		/* remove/unbind/shutdown in progress */
+	AP_DEV_STATE_ERROR,		/* device is in error state */
+	NR_AP_DEV_STATES
+};
+
+struct ap_device;
+struct ap_message;
+
+/*
+ * The ap driver struct includes a flags field which holds some info for
+ * the ap bus about the driver. Currently only one flag is supported and
+ * used: The DEFAULT flag marks an ap driver as a default driver which is
+ * used together with the apmask and aqmask whitelisting of the ap bus.
+ */
+#define AP_DRIVER_FLAG_DEFAULT 0x0001
+
+struct ap_driver {
+	struct device_driver driver;
+
+	int (*probe)(struct ap_device *);
+	void (*remove)(struct ap_device *);
+	int (*in_use)(unsigned long *apm, unsigned long *aqm);
+	/*
+	 * Called at the start of the ap bus scan function when
+	 * the crypto config information (qci) has changed.
+	 * This callback is not invoked if there is no AP
+	 * QCI support available.
+	 */
+	void (*on_config_changed)(struct ap_config_info *new_config_info,
+				  struct ap_config_info *old_config_info);
+	/*
+	 * Called at the end of the ap bus scan function when
+	 * the crypto config information (qci) has changed.
+	 * This callback is not invoked if there is no AP
+	 * QCI support available.
+	 */
+	void (*on_scan_complete)(struct ap_config_info *new_config_info,
+				 struct ap_config_info *old_config_info);
+
+	struct ap_device_id *ids;
+	unsigned int flags;
+};
+
+#define to_ap_drv(x) container_of_const((x), struct ap_driver, driver)
+
+int ap_driver_register(struct ap_driver *, struct module *, char *);
+void ap_driver_unregister(struct ap_driver *);
+
+struct ap_device {
+	struct device device;
+	int device_type;		/* AP device type. */
+};
+
+#define to_ap_dev(x) container_of((x), struct ap_device, device)
+
+struct ap_card {
+	struct ap_device ap_dev;
+	struct ap_tapq_hwinfo hwinfo;	/* TAPQ GR2 content */
+	atomic64_t total_request_count;	/* # requests ever for this AP device.*/
+	unsigned int maxmsgsize;	/* AP msg limit for this card */
+	int id;				/* AP card number. */
+	bool config;			/* configured state */
+	bool chkstop;			/* checkstop state */
+};
+
+#define TAPQ_CARD_HWINFO_MASK 0xFFFF0000FFFF0F1FUL
+#define ASSOC_IDX_INVALID 0x10000
+
+#define to_ap_card(x) container_of((x), struct ap_card, ap_dev.device)
+
+struct ap_queue {
+	struct ap_device ap_dev;
+	struct hlist_node hnode;	/* Node for the ap_queues hashtable */
+	struct ap_card *card;		/* Ptr to assoc. AP card. */
+	spinlock_t lock;		/* Per device lock. */
+	u64 total_request_count;	/* # requests ever for this AP device.*/
+	enum ap_dev_state dev_state;	/* queue device state */
+	ap_qid_t qid;			/* AP queue id. */
+	unsigned int se_bstate;		/* SE bind state (BS) */
+	unsigned int assoc_idx;		/* SE association index */
+	int queue_count;		/* # messages currently on AP queue. */
+	int pendingq_count;		/* # requests on pendingq list. */
+	int requestq_count;		/* # requests on requestq list. */
+	int request_timeout;		/* Request timeout in jiffies. */
+	struct timer_list timeout;	/* Timer for request timeouts. */
+	struct list_head pendingq;	/* List of message sent to AP queue. */
+	struct list_head requestq;	/* List of message yet to be sent. */
+	struct ap_message *reply;	/* Per device reply message. */
+	enum ap_sm_state sm_state;	/* ap queue state machine state */
+	int rapq_fbit;			/* fbit arg for next rapq invocation */
+	int last_err_rc;		/* last error state response code */
+	bool config;			/* configured state */
+	bool chkstop;			/* checkstop state */
+};
+
+#define to_ap_queue(x) container_of((x), struct ap_queue, ap_dev.device)
+
+typedef enum ap_sm_wait (ap_func_t)(struct ap_queue *queue);
+
+struct ap_response_type {
+	struct completion work;
+	int type;
+};
+
+struct ap_message {
+	struct list_head list;		/* Request queueing. */
+	unsigned long psmid;		/* Message id. */
+	void *msg;			/* Pointer to message buffer. */
+	size_t len;			/* actual msg len in msg buffer */
+	size_t bufsize;			/* allocated msg buffer size */
+	/* receive is called from tasklet context */
+	void (*receive)(struct ap_queue *, struct ap_message *,
+			struct ap_message *);
+	struct ap_response_type response;
+	int rc;				/* Return code for this message */
+	u16 flags;			/* Flags, see AP_MSG_FLAG_xxx */
+};
+
+#define AP_MSG_FLAG_SPECIAL  0x0001	/* flag msg as 'special' with NQAP */
+#define AP_MSG_FLAG_USAGE    0x0002	/* CCA, EP11: usage (no admin) msg */
+#define AP_MSG_FLAG_ADMIN    0x0004	/* CCA, EP11: admin (=control) msg */
+#define AP_MSG_FLAG_MEMPOOL  0x0008 /* ap msg buffer allocated via mempool */
+
+int ap_init_apmsg(struct ap_message *ap_msg, u32 flags);
+void ap_release_apmsg(struct ap_message *ap_msg);
+
+enum ap_sm_wait ap_sm_event(struct ap_queue *aq, enum ap_sm_event event);
+enum ap_sm_wait ap_sm_event_loop(struct ap_queue *aq, enum ap_sm_event event);
+
+int ap_queue_message(struct ap_queue *aq, struct ap_message *ap_msg);
+void ap_cancel_message(struct ap_queue *aq, struct ap_message *ap_msg);
+void ap_flush_queue(struct ap_queue *aq);
+bool ap_queue_usable(struct ap_queue *aq);
+
+void *ap_airq_ptr(void);
+int ap_sb_available(void);
+bool ap_is_se_guest(void);
+void ap_wait(enum ap_sm_wait wait);
+void ap_request_timeout(struct timer_list *t);
+bool ap_bus_force_rescan(void);
+
+int ap_test_config_usage_domain(unsigned int domain);
+int ap_test_config_ctrl_domain(unsigned int domain);
+
+void ap_queue_init_reply(struct ap_queue *aq, struct ap_message *ap_msg);
+struct ap_queue *ap_queue_create(ap_qid_t qid, struct ap_card *ac);
+void ap_queue_prepare_remove(struct ap_queue *aq);
+void ap_queue_remove(struct ap_queue *aq);
+void ap_queue_init_state(struct ap_queue *aq);
+void _ap_queue_init_state(struct ap_queue *aq);
+
+struct ap_card *ap_card_create(int id, struct ap_tapq_hwinfo info,
+			       int comp_type);
+
+#define APMASKSIZE (BITS_TO_LONGS(AP_DEVICES) * sizeof(unsigned long))
+#define AQMASKSIZE (BITS_TO_LONGS(AP_DOMAINS) * sizeof(unsigned long))
+
+struct ap_perms {
+	unsigned long ioctlm[BITS_TO_LONGS(AP_IOCTLS)];
+	unsigned long apm[BITS_TO_LONGS(AP_DEVICES)];
+	unsigned long aqm[BITS_TO_LONGS(AP_DOMAINS)];
+	unsigned long adm[BITS_TO_LONGS(AP_DOMAINS)];
+};
+
+extern struct ap_perms ap_perms;
+extern bool ap_apmask_aqmask_in_use;
+extern int ap_driver_override_ctr;
+extern struct mutex ap_attr_mutex;
+
+/*
+ * Get ap_queue device for this qid.
+ * Returns ptr to the struct ap_queue device or NULL if there
+ * was no ap_queue device with this qid found. When something is
+ * found, the reference count of the embedded device is increased.
+ * So the caller has to decrease the reference count after use
+ * with a call to put_device(&aq->ap_dev.device).
+ */
+struct ap_queue *ap_get_qdev(ap_qid_t qid);
+
+/*
+ * check APQN for owned/reserved by ap bus and default driver(s).
+ * Checks if this APQN is or will be in use by the ap bus
+ * and the default set of drivers.
+ * If yes, returns 1, if not returns 0. On error a negative
+ * errno value is returned.
+ */
+int ap_owned_by_def_drv(int card, int queue);
+
+/*
+ * check 'matrix' of APQNs for owned/reserved by ap bus and
+ * default driver(s).
+ * Checks if there is at least one APQN in the given 'matrix'
+ * marked as owned/reserved by the ap bus and default driver(s).
+ * If such an APQN is found the return value is 1, otherwise
+ * 0 is returned. On error a negative errno value is returned.
+ * The parameter apm is a bitmask which should be declared
+ * as DECLARE_BITMAP(apm, AP_DEVICES), the aqm parameter is
+ * similar, should be declared as DECLARE_BITMAP(aqm, AP_DOMAINS).
+ */
+int ap_apqn_in_matrix_owned_by_def_drv(unsigned long *apm,
+				       unsigned long *aqm);
+
+/*
+ * ap_parse_mask_str() - helper function to parse a bitmap string
+ * and clear/set the bits in the bitmap accordingly. The string may be
+ * given as absolute value, a hex string like 0x1F2E3D4C5B6A" simple
+ * overwriting the current content of the bitmap. Or as relative string
+ * like "+1-16,-32,-0x40,+128" where only single bits or ranges of
+ * bits are cleared or set. Distinction is done based on the very
+ * first character which may be '+' or '-' for the relative string
+ * and otherwise assume to be an absolute value string. If parsing fails
+ * a negative errno value is returned. All arguments and bitmaps are
+ * big endian order.
+ */
+int ap_parse_mask_str(const char *str,
+		      unsigned long *bitmap, int bits,
+		      struct mutex *lock);
+
+/*
+ * ap_hex2bitmap() - Convert a string containing a hexadecimal number (str)
+ * into a bitmap (bitmap) with bits set that correspond to the bits represented
+ * by the hex string. Input and output data is in big endian order.
+ *
+ * str - Input hex string of format "0x1234abcd". The leading "0x" is optional.
+ * At least one digit is required. Must be large enough to hold the number of
+ * bits represented by the bits parameter.
+ *
+ * bitmap - Pointer to a bitmap. Upon successful completion of this function,
+ * this bitmap will have bits set to match the value of str. If bitmap is longer
+ * than str, then the rightmost bits of bitmap are padded with zeros. Must be
+ * large enough to hold the number of bits represented by the bits parameter.
+ *
+ * bits - Length, in bits, of the bitmap represented by str. Must be a multiple
+ * of 8.
+ *
+ * Returns: 0		On success
+ *	    -EINVAL	If str format is invalid or bits is not a multiple of 8.
+ */
+int ap_hex2bitmap(const char *str, unsigned long *bitmap, int bits);
+
+/*
+ * Interface to wait for the AP bus to have done one initial ap bus
+ * scan and all detected APQNs have been bound to device drivers.
+ * If these both conditions are not fulfilled, this function blocks
+ * on a condition with wait_for_completion_killable_timeout().
+ * If these both conditions are fulfilled (before the timeout hits)
+ * the return value is 0. If the timeout (in jiffies) hits instead
+ * -ETIME is returned. On failures negative return values are
+ * returned to the caller.
+ * It may be that the AP bus scan finds new devices. Then the
+ * condition that all APQNs are bound to their device drivers
+ * is reset to false and this call again blocks until either all
+ * APQNs are bound to a device driver or the timeout hits again.
+ */
+int ap_wait_apqn_bindings_complete(unsigned long timeout);
+
+void ap_send_config_uevent(struct ap_device *ap_dev, bool cfg);
+void ap_send_online_uevent(struct ap_device *ap_dev, int online);
+void ap_send_se_bind_uevent(struct ap_device *ap_dev);
+void ap_send_se_assoc_uevent(struct ap_device *ap_dev, unsigned int assoc_idx);
+
+#endif /* _AP_BUS_H_ */
diff --git a/drivers/s390/crypto/ap_card.c b/drivers/s390/crypto/ap_card.c
new file mode 100644
index 000000000..c86397f4d
--- /dev/null
+++ b/drivers/s390/crypto/ap_card.c
@@ -0,0 +1,248 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright IBM Corp. 2016
+ * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *
+ * Adjunct processor bus, card related code.
+ */
+
+#define pr_fmt(fmt) "ap: " fmt
+
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <asm/facility.h>
+#include <asm/sclp.h>
+
+#include "ap_bus.h"
+
+/*
+ * AP card related attributes.
+ */
+static ssize_t hwtype_show(struct device *dev,
+			   struct device_attribute *attr, char *buf)
+{
+	struct ap_card *ac = to_ap_card(dev);
+
+	return sysfs_emit(buf, "%d\n", ac->ap_dev.device_type);
+}
+
+static DEVICE_ATTR_RO(hwtype);
+
+static ssize_t raw_hwtype_show(struct device *dev,
+			       struct device_attribute *attr, char *buf)
+{
+	struct ap_card *ac = to_ap_card(dev);
+
+	return sysfs_emit(buf, "%d\n", ac->hwinfo.at);
+}
+
+static DEVICE_ATTR_RO(raw_hwtype);
+
+static ssize_t depth_show(struct device *dev, struct device_attribute *attr,
+			  char *buf)
+{
+	struct ap_card *ac = to_ap_card(dev);
+
+	return sysfs_emit(buf, "%d\n", ac->hwinfo.qd + 1);
+}
+
+static DEVICE_ATTR_RO(depth);
+
+static ssize_t ap_functions_show(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	struct ap_card *ac = to_ap_card(dev);
+
+	return sysfs_emit(buf, "0x%08X\n", ac->hwinfo.fac);
+}
+
+static DEVICE_ATTR_RO(ap_functions);
+
+static ssize_t request_count_show(struct device *dev,
+				  struct device_attribute *attr,
+				  char *buf)
+{
+	struct ap_card *ac = to_ap_card(dev);
+	u64 req_cnt;
+
+	req_cnt = 0;
+	spin_lock_bh(&ap_queues_lock);
+	req_cnt = atomic64_read(&ac->total_request_count);
+	spin_unlock_bh(&ap_queues_lock);
+	return sysfs_emit(buf, "%llu\n", req_cnt);
+}
+
+static ssize_t request_count_store(struct device *dev,
+				   struct device_attribute *attr,
+				   const char *buf, size_t count)
+{
+	int bkt;
+	struct ap_queue *aq;
+	struct ap_card *ac = to_ap_card(dev);
+
+	spin_lock_bh(&ap_queues_lock);
+	hash_for_each(ap_queues, bkt, aq, hnode)
+		if (ac == aq->card)
+			aq->total_request_count = 0;
+	spin_unlock_bh(&ap_queues_lock);
+	atomic64_set(&ac->total_request_count, 0);
+
+	return count;
+}
+
+static DEVICE_ATTR_RW(request_count);
+
+static ssize_t requestq_count_show(struct device *dev,
+				   struct device_attribute *attr, char *buf)
+{
+	int bkt;
+	struct ap_queue *aq;
+	unsigned int reqq_cnt;
+	struct ap_card *ac = to_ap_card(dev);
+
+	reqq_cnt = 0;
+	spin_lock_bh(&ap_queues_lock);
+	hash_for_each(ap_queues, bkt, aq, hnode)
+		if (ac == aq->card)
+			reqq_cnt += aq->requestq_count;
+	spin_unlock_bh(&ap_queues_lock);
+	return sysfs_emit(buf, "%d\n", reqq_cnt);
+}
+
+static DEVICE_ATTR_RO(requestq_count);
+
+static ssize_t pendingq_count_show(struct device *dev,
+				   struct device_attribute *attr, char *buf)
+{
+	int bkt;
+	struct ap_queue *aq;
+	unsigned int penq_cnt;
+	struct ap_card *ac = to_ap_card(dev);
+
+	penq_cnt = 0;
+	spin_lock_bh(&ap_queues_lock);
+	hash_for_each(ap_queues, bkt, aq, hnode)
+		if (ac == aq->card)
+			penq_cnt += aq->pendingq_count;
+	spin_unlock_bh(&ap_queues_lock);
+	return sysfs_emit(buf, "%d\n", penq_cnt);
+}
+
+static DEVICE_ATTR_RO(pendingq_count);
+
+static ssize_t modalias_show(struct device *dev,
+			     struct device_attribute *attr, char *buf)
+{
+	return sysfs_emit(buf, "ap:t%02X\n", to_ap_dev(dev)->device_type);
+}
+
+static DEVICE_ATTR_RO(modalias);
+
+static ssize_t config_show(struct device *dev,
+			   struct device_attribute *attr, char *buf)
+{
+	struct ap_card *ac = to_ap_card(dev);
+
+	return sysfs_emit(buf, "%d\n", ac->config ? 1 : 0);
+}
+
+static ssize_t config_store(struct device *dev,
+			    struct device_attribute *attr,
+			    const char *buf, size_t count)
+{
+	int rc = 0, cfg;
+	struct ap_card *ac = to_ap_card(dev);
+
+	if (sscanf(buf, "%d\n", &cfg) != 1 || cfg < 0 || cfg > 1)
+		return -EINVAL;
+
+	if (cfg && !ac->config)
+		rc = sclp_ap_configure(ac->id);
+	else if (!cfg && ac->config)
+		rc = sclp_ap_deconfigure(ac->id);
+	if (rc)
+		return rc;
+
+	ac->config = cfg ? true : false;
+
+	ap_send_config_uevent(&ac->ap_dev, ac->config);
+
+	return count;
+}
+
+static DEVICE_ATTR_RW(config);
+
+static ssize_t chkstop_show(struct device *dev,
+			    struct device_attribute *attr, char *buf)
+{
+	struct ap_card *ac = to_ap_card(dev);
+
+	return sysfs_emit(buf, "%d\n", ac->chkstop ? 1 : 0);
+}
+
+static DEVICE_ATTR_RO(chkstop);
+
+static ssize_t max_msg_size_show(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	struct ap_card *ac = to_ap_card(dev);
+
+	return sysfs_emit(buf, "%u\n", ac->maxmsgsize);
+}
+
+static DEVICE_ATTR_RO(max_msg_size);
+
+static struct attribute *ap_card_dev_attrs[] = {
+	&dev_attr_hwtype.attr,
+	&dev_attr_raw_hwtype.attr,
+	&dev_attr_depth.attr,
+	&dev_attr_ap_functions.attr,
+	&dev_attr_request_count.attr,
+	&dev_attr_requestq_count.attr,
+	&dev_attr_pendingq_count.attr,
+	&dev_attr_modalias.attr,
+	&dev_attr_config.attr,
+	&dev_attr_chkstop.attr,
+	&dev_attr_max_msg_size.attr,
+	NULL
+};
+
+static struct attribute_group ap_card_dev_attr_group = {
+	.attrs = ap_card_dev_attrs
+};
+
+static const struct attribute_group *ap_card_dev_attr_groups[] = {
+	&ap_card_dev_attr_group,
+	NULL
+};
+
+static struct device_type ap_card_type = {
+	.name = "ap_card",
+	.groups = ap_card_dev_attr_groups,
+};
+
+static void ap_card_device_release(struct device *dev)
+{
+	struct ap_card *ac = to_ap_card(dev);
+
+	kfree(ac);
+}
+
+struct ap_card *ap_card_create(int id, struct ap_tapq_hwinfo hwinfo,
+			       int comp_type)
+{
+	struct ap_card *ac;
+
+	ac = kzalloc_obj(*ac);
+	if (!ac)
+		return NULL;
+	ac->ap_dev.device.release = ap_card_device_release;
+	ac->ap_dev.device.type = &ap_card_type;
+	ac->ap_dev.device_type = comp_type;
+	ac->hwinfo = hwinfo;
+	ac->id = id;
+	ac->maxmsgsize = hwinfo.ml > 3 ?
+		hwinfo.ml * AP_TAPQ_ML_FIELD_CHUNK_SIZE : AP_DEFAULT_MAX_MSG_SIZE;
+
+	return ac;
+}
diff --git a/drivers/s390/crypto/ap_debug.h b/drivers/s390/crypto/ap_debug.h
new file mode 100644
index 000000000..2f66271b8
--- /dev/null
+++ b/drivers/s390/crypto/ap_debug.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ *  Copyright IBM Corp. 2016
+ *  Author(s): Harald Freudenberger <freude@de.ibm.com>
+ */
+#ifndef AP_DEBUG_H
+#define AP_DEBUG_H
+
+#include <asm/debug.h>
+
+#define DBF_ERR		3	/* error conditions   */
+#define DBF_WARN	4	/* warning conditions */
+#define DBF_INFO	5	/* informational      */
+#define DBF_DEBUG	6	/* for debugging only */
+
+#define RC2ERR(rc) ((rc) ? DBF_ERR : DBF_INFO)
+#define RC2WARN(rc) ((rc) ? DBF_WARN : DBF_INFO)
+
+#define AP_DBF_MAX_SPRINTF_ARGS 6
+
+#define AP_DBF(...)					\
+	debug_sprintf_event(ap_dbf_info, ##__VA_ARGS__)
+#define AP_DBF_ERR(...)					\
+	debug_sprintf_event(ap_dbf_info, DBF_ERR, ##__VA_ARGS__)
+#define AP_DBF_WARN(...)					\
+	debug_sprintf_event(ap_dbf_info, DBF_WARN, ##__VA_ARGS__)
+#define AP_DBF_INFO(...)					\
+	debug_sprintf_event(ap_dbf_info, DBF_INFO, ##__VA_ARGS__)
+
+extern debug_info_t *ap_dbf_info;
+
+#endif /* AP_DEBUG_H */
diff --git a/drivers/s390/crypto/ap_queue.c b/drivers/s390/crypto/ap_queue.c
new file mode 100644
index 000000000..bc03b2101
--- /dev/null
+++ b/drivers/s390/crypto/ap_queue.c
@@ -0,0 +1,1392 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright IBM Corp. 2016, 2023
+ * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *
+ * Adjunct processor bus, queue related code.
+ */
+
+#define pr_fmt(fmt) "ap: " fmt
+
+#include <linux/export.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <asm/facility.h>
+
+#define CREATE_TRACE_POINTS
+#include <asm/trace/ap.h>
+
+#include "ap_bus.h"
+#include "ap_debug.h"
+
+EXPORT_TRACEPOINT_SYMBOL(s390_ap_nqap);
+EXPORT_TRACEPOINT_SYMBOL(s390_ap_dqap);
+
+static void __ap_flush_queue(struct ap_queue *aq);
+
+/*
+ * some AP queue helper functions
+ */
+
+static inline bool ap_q_supported_in_se(struct ap_queue *aq)
+{
+	return aq->card->hwinfo.ep11 || aq->card->hwinfo.accel;
+}
+
+static inline bool ap_q_supports_bind(struct ap_queue *aq)
+{
+	return aq->card->hwinfo.ep11 || aq->card->hwinfo.accel;
+}
+
+static inline bool ap_q_supports_assoc(struct ap_queue *aq)
+{
+	return aq->card->hwinfo.ep11;
+}
+
+static inline bool ap_q_needs_bind(struct ap_queue *aq)
+{
+	return ap_q_supports_bind(aq) && ap_sb_available();
+}
+
+/**
+ * ap_queue_enable_irq(): Enable interrupt support on this AP queue.
+ * @aq: The AP queue
+ * @ind: the notification indicator byte
+ *
+ * Enables interruption on AP queue via ap_aqic(). Based on the return
+ * value it waits a while and tests the AP queue if interrupts
+ * have been switched on using ap_test_queue().
+ */
+static int ap_queue_enable_irq(struct ap_queue *aq, void *ind)
+{
+	union ap_qirq_ctrl qirqctrl = { .value = 0 };
+	struct ap_queue_status status;
+
+	qirqctrl.ir = 1;
+	qirqctrl.isc = AP_ISC;
+	status = ap_aqic(aq->qid, qirqctrl, virt_to_phys(ind));
+	if (status.async)
+		return -EPERM;
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+	case AP_RESPONSE_OTHERWISE_CHANGED:
+		return 0;
+	case AP_RESPONSE_Q_NOT_AVAIL:
+	case AP_RESPONSE_DECONFIGURED:
+	case AP_RESPONSE_CHECKSTOPPED:
+	case AP_RESPONSE_INVALID_ADDRESS:
+		pr_err("Registering adapter interrupts for AP device %02x.%04x failed\n",
+		       AP_QID_CARD(aq->qid),
+		       AP_QID_QUEUE(aq->qid));
+		return -EOPNOTSUPP;
+	case AP_RESPONSE_RESET_IN_PROGRESS:
+	case AP_RESPONSE_BUSY:
+	default:
+		return -EBUSY;
+	}
+}
+
+/**
+ * __ap_send(): Send message to adjunct processor queue.
+ * @qid: The AP queue number
+ * @psmid: The program supplied message identifier
+ * @msg: The message text
+ * @msglen: The message length
+ * @special: Special Bit
+ *
+ * Returns AP queue status structure.
+ * Condition code 1 on NQAP can't happen because the L bit is 1.
+ * Condition code 2 on NQAP also means the send is incomplete,
+ * because a segment boundary was reached. The NQAP is repeated.
+ */
+static inline struct ap_queue_status
+__ap_send(ap_qid_t qid, unsigned long psmid, void *msg, size_t msglen,
+	  int special)
+{
+	struct ap_queue_status status;
+
+	if (special)
+		qid |= 0x400000UL;
+
+	status = ap_nqap(qid, psmid, msg, msglen);
+
+	trace_s390_ap_nqap(AP_QID_CARD(qid), AP_QID_QUEUE(qid),
+			   status.value, psmid);
+
+	return status;
+}
+
+/* State machine definitions and helpers */
+
+static enum ap_sm_wait ap_sm_nop(struct ap_queue *aq)
+{
+	return AP_SM_WAIT_NONE;
+}
+
+/**
+ * ap_sm_recv(): Receive pending reply messages from an AP queue but do
+ *	not change the state of the device.
+ * @aq: pointer to the AP queue
+ *
+ * Returns AP_SM_WAIT_NONE, AP_SM_WAIT_AGAIN, or AP_SM_WAIT_INTERRUPT
+ */
+static struct ap_queue_status ap_sm_recv(struct ap_queue *aq)
+{
+	struct ap_queue_status status;
+	struct ap_message *ap_msg;
+	bool found = false;
+	size_t reslen;
+	unsigned long resgr0 = 0;
+	int parts = 0;
+
+	/*
+	 * DQAP loop until response code and resgr0 indicate that
+	 * the msg is totally received. As we use the very same buffer
+	 * the msg is overwritten with each invocation. That's intended
+	 * and the receiver of the msg is informed with a msg rc code
+	 * of EMSGSIZE in such a case.
+	 */
+	do {
+		status = ap_dqap(aq->qid, &aq->reply->psmid,
+				 aq->reply->msg, aq->reply->bufsize,
+				 &aq->reply->len, &reslen, &resgr0);
+		parts++;
+	} while (status.response_code == 0xFF && resgr0 != 0);
+
+	trace_s390_ap_dqap(AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid),
+			   status.value, aq->reply->psmid);
+
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+		print_hex_dump_debug("aprpl: ", DUMP_PREFIX_ADDRESS, 16, 1,
+				     aq->reply->msg, aq->reply->len, false);
+		aq->queue_count = max_t(int, 0, aq->queue_count - 1);
+		if (!status.queue_empty && !aq->queue_count)
+			aq->queue_count++;
+		if (aq->queue_count > 0)
+			mod_timer(&aq->timeout,
+				  jiffies + aq->request_timeout);
+		list_for_each_entry(ap_msg, &aq->pendingq, list) {
+			if (ap_msg->psmid != aq->reply->psmid)
+				continue;
+			list_del_init(&ap_msg->list);
+			aq->pendingq_count--;
+			if (parts > 1) {
+				ap_msg->rc = -EMSGSIZE;
+				ap_msg->receive(aq, ap_msg, NULL);
+			} else {
+				ap_msg->receive(aq, ap_msg, aq->reply);
+			}
+			found = true;
+			break;
+		}
+		if (!found) {
+			AP_DBF_WARN("%s unassociated reply psmid=0x%016lx on 0x%02x.%04x\n",
+				    __func__, aq->reply->psmid,
+				    AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
+		}
+		fallthrough;
+	case AP_RESPONSE_NO_PENDING_REPLY:
+		if (!status.queue_empty || aq->queue_count <= 0)
+			break;
+		/* The card shouldn't forget requests but who knows. */
+		aq->queue_count = 0;
+		list_splice_init(&aq->pendingq, &aq->requestq);
+		aq->requestq_count += aq->pendingq_count;
+		pr_debug("queue 0x%02x.%04x rescheduled %d reqs (new req %d)\n",
+			 AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid),
+			 aq->pendingq_count, aq->requestq_count);
+		aq->pendingq_count = 0;
+		break;
+	default:
+		break;
+	}
+	return status;
+}
+
+/**
+ * ap_sm_read(): Receive pending reply messages from an AP queue.
+ * @aq: pointer to the AP queue
+ *
+ * Returns AP_SM_WAIT_NONE, AP_SM_WAIT_AGAIN, or AP_SM_WAIT_INTERRUPT
+ */
+static enum ap_sm_wait ap_sm_read(struct ap_queue *aq)
+{
+	struct ap_queue_status status;
+
+	if (!aq->reply)
+		return AP_SM_WAIT_NONE;
+	status = ap_sm_recv(aq);
+	if (status.async)
+		return AP_SM_WAIT_NONE;
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+		if (aq->queue_count > 0) {
+			aq->sm_state = AP_SM_STATE_WORKING;
+			return AP_SM_WAIT_AGAIN;
+		}
+		aq->sm_state = AP_SM_STATE_IDLE;
+		break;
+	case AP_RESPONSE_NO_PENDING_REPLY:
+		if (aq->queue_count > 0)
+			return status.irq_enabled ?
+				AP_SM_WAIT_INTERRUPT : AP_SM_WAIT_HIGH_TIMEOUT;
+		aq->sm_state = AP_SM_STATE_IDLE;
+		break;
+	default:
+		aq->dev_state = AP_DEV_STATE_ERROR;
+		aq->last_err_rc = status.response_code;
+		AP_DBF_WARN("%s RC 0x%02x on 0x%02x.%04x -> AP_DEV_STATE_ERROR\n",
+			    __func__, status.response_code,
+			    AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
+		return AP_SM_WAIT_NONE;
+	}
+	/* Check and maybe enable irq support (again) on this queue */
+	if (!status.irq_enabled && status.queue_empty) {
+		void *lsi_ptr = ap_airq_ptr();
+
+		if (lsi_ptr && ap_queue_enable_irq(aq, lsi_ptr) == 0) {
+			aq->sm_state = AP_SM_STATE_SETIRQ_WAIT;
+			return AP_SM_WAIT_AGAIN;
+		}
+	}
+	return AP_SM_WAIT_NONE;
+}
+
+/**
+ * ap_sm_write(): Send messages from the request queue to an AP queue.
+ * @aq: pointer to the AP queue
+ *
+ * Returns AP_SM_WAIT_NONE, AP_SM_WAIT_AGAIN, or AP_SM_WAIT_INTERRUPT
+ */
+static enum ap_sm_wait ap_sm_write(struct ap_queue *aq)
+{
+	struct ap_queue_status status;
+	struct ap_message *ap_msg;
+	ap_qid_t qid = aq->qid;
+
+	if (aq->requestq_count <= 0)
+		return AP_SM_WAIT_NONE;
+
+	/* Start the next request on the queue. */
+	ap_msg = list_entry(aq->requestq.next, struct ap_message, list);
+	print_hex_dump_debug("apreq: ", DUMP_PREFIX_ADDRESS, 16, 1,
+			     ap_msg->msg, ap_msg->len, false);
+	status = __ap_send(qid, ap_msg->psmid,
+			   ap_msg->msg, ap_msg->len,
+			   ap_msg->flags & AP_MSG_FLAG_SPECIAL);
+	if (status.async)
+		return AP_SM_WAIT_NONE;
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+		aq->queue_count = max_t(int, 1, aq->queue_count + 1);
+		if (aq->queue_count == 1)
+			mod_timer(&aq->timeout, jiffies + aq->request_timeout);
+		list_move_tail(&ap_msg->list, &aq->pendingq);
+		aq->requestq_count--;
+		aq->pendingq_count++;
+		if (aq->queue_count < aq->card->hwinfo.qd + 1) {
+			aq->sm_state = AP_SM_STATE_WORKING;
+			return AP_SM_WAIT_AGAIN;
+		}
+		fallthrough;
+	case AP_RESPONSE_Q_FULL:
+		aq->sm_state = AP_SM_STATE_QUEUE_FULL;
+		return status.irq_enabled ?
+			AP_SM_WAIT_INTERRUPT : AP_SM_WAIT_HIGH_TIMEOUT;
+	case AP_RESPONSE_RESET_IN_PROGRESS:
+		aq->sm_state = AP_SM_STATE_RESET_WAIT;
+		return AP_SM_WAIT_LOW_TIMEOUT;
+	case AP_RESPONSE_INVALID_DOMAIN:
+		AP_DBF_WARN("%s RESPONSE_INVALID_DOMAIN on NQAP\n", __func__);
+		fallthrough;
+	case AP_RESPONSE_MESSAGE_TOO_BIG:
+	case AP_RESPONSE_REQ_FAC_NOT_INST:
+		list_del_init(&ap_msg->list);
+		aq->requestq_count--;
+		ap_msg->rc = -EINVAL;
+		ap_msg->receive(aq, ap_msg, NULL);
+		return AP_SM_WAIT_AGAIN;
+	default:
+		aq->dev_state = AP_DEV_STATE_ERROR;
+		aq->last_err_rc = status.response_code;
+		AP_DBF_WARN("%s RC 0x%02x on 0x%02x.%04x -> AP_DEV_STATE_ERROR\n",
+			    __func__, status.response_code,
+			    AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
+		return AP_SM_WAIT_NONE;
+	}
+}
+
+/**
+ * ap_sm_read_write(): Send and receive messages to/from an AP queue.
+ * @aq: pointer to the AP queue
+ *
+ * Returns AP_SM_WAIT_NONE, AP_SM_WAIT_AGAIN, or AP_SM_WAIT_INTERRUPT
+ */
+static enum ap_sm_wait ap_sm_read_write(struct ap_queue *aq)
+{
+	return min(ap_sm_read(aq), ap_sm_write(aq));
+}
+
+/**
+ * ap_sm_reset(): Reset an AP queue.
+ * @aq: The AP queue
+ *
+ * Submit the Reset command to an AP queue.
+ */
+static enum ap_sm_wait ap_sm_reset(struct ap_queue *aq)
+{
+	struct ap_queue_status status;
+
+	status = ap_rapq(aq->qid, aq->rapq_fbit);
+	if (status.async)
+		return AP_SM_WAIT_NONE;
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+	case AP_RESPONSE_RESET_IN_PROGRESS:
+		aq->sm_state = AP_SM_STATE_RESET_WAIT;
+		aq->rapq_fbit = 0;
+		return AP_SM_WAIT_LOW_TIMEOUT;
+	default:
+		aq->dev_state = AP_DEV_STATE_ERROR;
+		aq->last_err_rc = status.response_code;
+		AP_DBF_WARN("%s RC 0x%02x on 0x%02x.%04x -> AP_DEV_STATE_ERROR\n",
+			    __func__, status.response_code,
+			    AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
+		return AP_SM_WAIT_NONE;
+	}
+}
+
+/**
+ * ap_sm_reset_wait(): Test queue for completion of the reset operation
+ * @aq: pointer to the AP queue
+ *
+ * Returns AP_POLL_IMMEDIATELY, AP_POLL_AFTER_TIMEROUT or 0.
+ */
+static enum ap_sm_wait ap_sm_reset_wait(struct ap_queue *aq)
+{
+	struct ap_queue_status status;
+	struct ap_tapq_hwinfo hwinfo;
+	void *lsi_ptr;
+
+	/* Get the status with TAPQ */
+	status = ap_test_queue(aq->qid, 1, &hwinfo);
+
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+		aq->se_bstate = hwinfo.bs;
+		lsi_ptr = ap_airq_ptr();
+		if (lsi_ptr && ap_queue_enable_irq(aq, lsi_ptr) == 0)
+			aq->sm_state = AP_SM_STATE_SETIRQ_WAIT;
+		else
+			aq->sm_state = (aq->queue_count > 0) ?
+				AP_SM_STATE_WORKING : AP_SM_STATE_IDLE;
+		return AP_SM_WAIT_AGAIN;
+	case AP_RESPONSE_BUSY:
+	case AP_RESPONSE_RESET_IN_PROGRESS:
+		return AP_SM_WAIT_LOW_TIMEOUT;
+	case AP_RESPONSE_Q_NOT_AVAIL:
+	case AP_RESPONSE_DECONFIGURED:
+	case AP_RESPONSE_CHECKSTOPPED:
+	default:
+		aq->dev_state = AP_DEV_STATE_ERROR;
+		aq->last_err_rc = status.response_code;
+		AP_DBF_WARN("%s RC 0x%02x on 0x%02x.%04x -> AP_DEV_STATE_ERROR\n",
+			    __func__, status.response_code,
+			    AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
+		return AP_SM_WAIT_NONE;
+	}
+}
+
+/**
+ * ap_sm_setirq_wait(): Test queue for completion of the irq enablement
+ * @aq: pointer to the AP queue
+ *
+ * Returns AP_POLL_IMMEDIATELY, AP_POLL_AFTER_TIMEROUT or 0.
+ */
+static enum ap_sm_wait ap_sm_setirq_wait(struct ap_queue *aq)
+{
+	struct ap_queue_status status;
+
+	if (aq->queue_count > 0 && aq->reply)
+		/* Try to read a completed message and get the status */
+		status = ap_sm_recv(aq);
+	else
+		/* Get the status with TAPQ */
+		status = ap_tapq(aq->qid, NULL);
+
+	if (status.irq_enabled == 1) {
+		/* Irqs are now enabled */
+		aq->sm_state = (aq->queue_count > 0) ?
+			AP_SM_STATE_WORKING : AP_SM_STATE_IDLE;
+	}
+
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+		if (aq->queue_count > 0)
+			return AP_SM_WAIT_AGAIN;
+		fallthrough;
+	case AP_RESPONSE_NO_PENDING_REPLY:
+		return AP_SM_WAIT_LOW_TIMEOUT;
+	default:
+		aq->dev_state = AP_DEV_STATE_ERROR;
+		aq->last_err_rc = status.response_code;
+		AP_DBF_WARN("%s RC 0x%02x on 0x%02x.%04x -> AP_DEV_STATE_ERROR\n",
+			    __func__, status.response_code,
+			    AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
+		return AP_SM_WAIT_NONE;
+	}
+}
+
+/**
+ * ap_sm_assoc_wait(): Test queue for completion of a pending
+ *		       association request.
+ * @aq: pointer to the AP queue
+ */
+static enum ap_sm_wait ap_sm_assoc_wait(struct ap_queue *aq)
+{
+	struct ap_queue_status status;
+	struct ap_tapq_hwinfo hwinfo;
+
+	status = ap_test_queue(aq->qid, 1, &hwinfo);
+	/* handle asynchronous error on this queue */
+	if (status.async && status.response_code) {
+		aq->dev_state = AP_DEV_STATE_ERROR;
+		aq->last_err_rc = status.response_code;
+		AP_DBF_WARN("%s asynch RC 0x%02x on 0x%02x.%04x -> AP_DEV_STATE_ERROR\n",
+			    __func__, status.response_code,
+			    AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
+		return AP_SM_WAIT_NONE;
+	}
+	if (status.response_code > AP_RESPONSE_BUSY) {
+		aq->dev_state = AP_DEV_STATE_ERROR;
+		aq->last_err_rc = status.response_code;
+		AP_DBF_WARN("%s RC 0x%02x on 0x%02x.%04x -> AP_DEV_STATE_ERROR\n",
+			    __func__, status.response_code,
+			    AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
+		return AP_SM_WAIT_NONE;
+	}
+
+	/* update queue's SE bind state */
+	aq->se_bstate = hwinfo.bs;
+
+	/* check bs bits */
+	switch (hwinfo.bs) {
+	case AP_BS_Q_USABLE:
+		/* association is through */
+		aq->sm_state = AP_SM_STATE_IDLE;
+		pr_debug("queue 0x%02x.%04x associated with %u\n",
+			 AP_QID_CARD(aq->qid),
+			 AP_QID_QUEUE(aq->qid), aq->assoc_idx);
+		ap_send_se_assoc_uevent(&aq->ap_dev, aq->assoc_idx);
+		return AP_SM_WAIT_NONE;
+	case AP_BS_Q_USABLE_NO_SECURE_KEY:
+		/* association still pending */
+		return AP_SM_WAIT_LOW_TIMEOUT;
+	default:
+		/* reset from 'outside' happened or no idea at all */
+		aq->assoc_idx = ASSOC_IDX_INVALID;
+		aq->dev_state = AP_DEV_STATE_ERROR;
+		aq->last_err_rc = status.response_code;
+		AP_DBF_WARN("%s bs 0x%02x on 0x%02x.%04x -> AP_DEV_STATE_ERROR\n",
+			    __func__, hwinfo.bs,
+			    AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
+		return AP_SM_WAIT_NONE;
+	}
+}
+
+/*
+ * AP state machine jump table
+ */
+static ap_func_t *ap_jumptable[NR_AP_SM_STATES][NR_AP_SM_EVENTS] = {
+	[AP_SM_STATE_RESET_START] = {
+		[AP_SM_EVENT_POLL] = ap_sm_reset,
+		[AP_SM_EVENT_TIMEOUT] = ap_sm_nop,
+	},
+	[AP_SM_STATE_RESET_WAIT] = {
+		[AP_SM_EVENT_POLL] = ap_sm_reset_wait,
+		[AP_SM_EVENT_TIMEOUT] = ap_sm_nop,
+	},
+	[AP_SM_STATE_SETIRQ_WAIT] = {
+		[AP_SM_EVENT_POLL] = ap_sm_setirq_wait,
+		[AP_SM_EVENT_TIMEOUT] = ap_sm_nop,
+	},
+	[AP_SM_STATE_IDLE] = {
+		[AP_SM_EVENT_POLL] = ap_sm_write,
+		[AP_SM_EVENT_TIMEOUT] = ap_sm_nop,
+	},
+	[AP_SM_STATE_WORKING] = {
+		[AP_SM_EVENT_POLL] = ap_sm_read_write,
+		[AP_SM_EVENT_TIMEOUT] = ap_sm_reset,
+	},
+	[AP_SM_STATE_QUEUE_FULL] = {
+		[AP_SM_EVENT_POLL] = ap_sm_read,
+		[AP_SM_EVENT_TIMEOUT] = ap_sm_reset,
+	},
+	[AP_SM_STATE_ASSOC_WAIT] = {
+		[AP_SM_EVENT_POLL] = ap_sm_assoc_wait,
+		[AP_SM_EVENT_TIMEOUT] = ap_sm_reset,
+	},
+};
+
+enum ap_sm_wait ap_sm_event(struct ap_queue *aq, enum ap_sm_event event)
+{
+	if (aq->config && !aq->chkstop &&
+	    aq->dev_state > AP_DEV_STATE_UNINITIATED)
+		return ap_jumptable[aq->sm_state][event](aq);
+	else
+		return AP_SM_WAIT_NONE;
+}
+
+enum ap_sm_wait ap_sm_event_loop(struct ap_queue *aq, enum ap_sm_event event)
+{
+	enum ap_sm_wait wait;
+
+	while ((wait = ap_sm_event(aq, event)) == AP_SM_WAIT_AGAIN)
+		;
+	return wait;
+}
+
+/*
+ * AP queue related attributes.
+ */
+static ssize_t request_count_show(struct device *dev,
+				  struct device_attribute *attr,
+				  char *buf)
+{
+	struct ap_queue *aq = to_ap_queue(dev);
+	bool valid = false;
+	u64 req_cnt;
+
+	spin_lock_bh(&aq->lock);
+	if (aq->dev_state > AP_DEV_STATE_UNINITIATED) {
+		req_cnt = aq->total_request_count;
+		valid = true;
+	}
+	spin_unlock_bh(&aq->lock);
+
+	if (valid)
+		return sysfs_emit(buf, "%llu\n", req_cnt);
+	else
+		return sysfs_emit(buf, "-\n");
+}
+
+static ssize_t request_count_store(struct device *dev,
+				   struct device_attribute *attr,
+				   const char *buf, size_t count)
+{
+	struct ap_queue *aq = to_ap_queue(dev);
+
+	spin_lock_bh(&aq->lock);
+	aq->total_request_count = 0;
+	spin_unlock_bh(&aq->lock);
+
+	return count;
+}
+
+static DEVICE_ATTR_RW(request_count);
+
+static ssize_t requestq_count_show(struct device *dev,
+				   struct device_attribute *attr, char *buf)
+{
+	struct ap_queue *aq = to_ap_queue(dev);
+	unsigned int reqq_cnt = 0;
+
+	spin_lock_bh(&aq->lock);
+	if (aq->dev_state > AP_DEV_STATE_UNINITIATED)
+		reqq_cnt = aq->requestq_count;
+	spin_unlock_bh(&aq->lock);
+	return sysfs_emit(buf, "%d\n", reqq_cnt);
+}
+
+static DEVICE_ATTR_RO(requestq_count);
+
+static ssize_t pendingq_count_show(struct device *dev,
+				   struct device_attribute *attr, char *buf)
+{
+	struct ap_queue *aq = to_ap_queue(dev);
+	unsigned int penq_cnt = 0;
+
+	spin_lock_bh(&aq->lock);
+	if (aq->dev_state > AP_DEV_STATE_UNINITIATED)
+		penq_cnt = aq->pendingq_count;
+	spin_unlock_bh(&aq->lock);
+	return sysfs_emit(buf, "%d\n", penq_cnt);
+}
+
+static DEVICE_ATTR_RO(pendingq_count);
+
+static ssize_t reset_show(struct device *dev,
+			  struct device_attribute *attr, char *buf)
+{
+	struct ap_queue *aq = to_ap_queue(dev);
+	int rc = 0;
+
+	spin_lock_bh(&aq->lock);
+	switch (aq->sm_state) {
+	case AP_SM_STATE_RESET_START:
+	case AP_SM_STATE_RESET_WAIT:
+		rc = sysfs_emit(buf, "Reset in progress.\n");
+		break;
+	case AP_SM_STATE_WORKING:
+	case AP_SM_STATE_QUEUE_FULL:
+		rc = sysfs_emit(buf, "Reset Timer armed.\n");
+		break;
+	default:
+		rc = sysfs_emit(buf, "No Reset Timer set.\n");
+	}
+	spin_unlock_bh(&aq->lock);
+	return rc;
+}
+
+static ssize_t reset_store(struct device *dev,
+			   struct device_attribute *attr,
+			   const char *buf, size_t count)
+{
+	struct ap_queue *aq = to_ap_queue(dev);
+
+	spin_lock_bh(&aq->lock);
+	__ap_flush_queue(aq);
+	aq->sm_state = AP_SM_STATE_RESET_START;
+	ap_wait(ap_sm_event(aq, AP_SM_EVENT_POLL));
+	spin_unlock_bh(&aq->lock);
+
+	AP_DBF_INFO("%s reset queue=%02x.%04x triggered by user\n",
+		    __func__, AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
+
+	return count;
+}
+
+static DEVICE_ATTR_RW(reset);
+
+static ssize_t interrupt_show(struct device *dev,
+			      struct device_attribute *attr, char *buf)
+{
+	struct ap_queue *aq = to_ap_queue(dev);
+	struct ap_queue_status status;
+	int rc = 0;
+
+	spin_lock_bh(&aq->lock);
+	if (aq->sm_state == AP_SM_STATE_SETIRQ_WAIT) {
+		rc = sysfs_emit(buf, "Enable Interrupt pending.\n");
+	} else {
+		status = ap_tapq(aq->qid, NULL);
+		if (status.irq_enabled)
+			rc = sysfs_emit(buf, "Interrupts enabled.\n");
+		else
+			rc = sysfs_emit(buf, "Interrupts disabled.\n");
+	}
+	spin_unlock_bh(&aq->lock);
+
+	return rc;
+}
+
+static DEVICE_ATTR_RO(interrupt);
+
+static ssize_t config_show(struct device *dev,
+			   struct device_attribute *attr, char *buf)
+{
+	struct ap_queue *aq = to_ap_queue(dev);
+	int rc;
+
+	spin_lock_bh(&aq->lock);
+	rc = sysfs_emit(buf, "%d\n", aq->config ? 1 : 0);
+	spin_unlock_bh(&aq->lock);
+	return rc;
+}
+
+static DEVICE_ATTR_RO(config);
+
+static ssize_t chkstop_show(struct device *dev,
+			    struct device_attribute *attr, char *buf)
+{
+	struct ap_queue *aq = to_ap_queue(dev);
+	int rc;
+
+	spin_lock_bh(&aq->lock);
+	rc = sysfs_emit(buf, "%d\n", aq->chkstop ? 1 : 0);
+	spin_unlock_bh(&aq->lock);
+	return rc;
+}
+
+static DEVICE_ATTR_RO(chkstop);
+
+static ssize_t ap_functions_show(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	struct ap_queue *aq = to_ap_queue(dev);
+	struct ap_queue_status status;
+	struct ap_tapq_hwinfo hwinfo;
+
+	status = ap_test_queue(aq->qid, 1, &hwinfo);
+	if (status.response_code > AP_RESPONSE_BUSY) {
+		pr_debug("RC 0x%02x on tapq(0x%02x.%04x)\n",
+			 status.response_code,
+			 AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
+		return -EIO;
+	}
+
+	return sysfs_emit(buf, "0x%08X\n", hwinfo.fac);
+}
+
+static DEVICE_ATTR_RO(ap_functions);
+
+static ssize_t driver_override_show(struct device *dev,
+				    struct device_attribute *attr,
+				    char *buf)
+{
+	guard(spinlock)(&dev->driver_override.lock);
+	return sysfs_emit(buf, "%s\n", dev->driver_override.name ?: "");
+}
+
+static ssize_t driver_override_store(struct device *dev,
+				     struct device_attribute *attr,
+				     const char *buf, size_t count)
+{
+	int rc = -EINVAL;
+	bool old_value;
+
+	if (mutex_lock_interruptible(&ap_attr_mutex))
+		return -ERESTARTSYS;
+
+	/* Do not allow driver override if apmask/aqmask is in use */
+	if (ap_apmask_aqmask_in_use)
+		goto out;
+
+	old_value = device_has_driver_override(dev);
+	rc = __device_set_driver_override(dev, buf, count);
+	if (rc)
+		goto out;
+	if (old_value && !device_has_driver_override(dev))
+		--ap_driver_override_ctr;
+	else if (!old_value && device_has_driver_override(dev))
+		++ap_driver_override_ctr;
+
+	rc = count;
+
+out:
+	mutex_unlock(&ap_attr_mutex);
+	return rc;
+}
+
+static DEVICE_ATTR_RW(driver_override);
+
+#ifdef CONFIG_AP_DEBUG
+static ssize_t states_show(struct device *dev,
+			   struct device_attribute *attr, char *buf)
+{
+	struct ap_queue *aq = to_ap_queue(dev);
+	int rc = 0;
+
+	spin_lock_bh(&aq->lock);
+	/* queue device state */
+	switch (aq->dev_state) {
+	case AP_DEV_STATE_UNINITIATED:
+		rc = sysfs_emit(buf, "UNINITIATED\n");
+		break;
+	case AP_DEV_STATE_OPERATING:
+		rc = sysfs_emit(buf, "OPERATING");
+		break;
+	case AP_DEV_STATE_SHUTDOWN:
+		rc = sysfs_emit(buf, "SHUTDOWN");
+		break;
+	case AP_DEV_STATE_ERROR:
+		rc = sysfs_emit(buf, "ERROR");
+		break;
+	default:
+		rc = sysfs_emit(buf, "UNKNOWN");
+	}
+	/* state machine state */
+	if (aq->dev_state) {
+		switch (aq->sm_state) {
+		case AP_SM_STATE_RESET_START:
+			rc += sysfs_emit_at(buf, rc, " [RESET_START]\n");
+			break;
+		case AP_SM_STATE_RESET_WAIT:
+			rc += sysfs_emit_at(buf, rc, " [RESET_WAIT]\n");
+			break;
+		case AP_SM_STATE_SETIRQ_WAIT:
+			rc += sysfs_emit_at(buf, rc, " [SETIRQ_WAIT]\n");
+			break;
+		case AP_SM_STATE_IDLE:
+			rc += sysfs_emit_at(buf, rc, " [IDLE]\n");
+			break;
+		case AP_SM_STATE_WORKING:
+			rc += sysfs_emit_at(buf, rc, " [WORKING]\n");
+			break;
+		case AP_SM_STATE_QUEUE_FULL:
+			rc += sysfs_emit_at(buf, rc, " [FULL]\n");
+			break;
+		case AP_SM_STATE_ASSOC_WAIT:
+			rc += sysfs_emit_at(buf, rc, " [ASSOC_WAIT]\n");
+			break;
+		default:
+			rc += sysfs_emit_at(buf, rc, " [UNKNOWN]\n");
+		}
+	}
+	spin_unlock_bh(&aq->lock);
+
+	return rc;
+}
+static DEVICE_ATTR_RO(states);
+
+static ssize_t last_err_rc_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	struct ap_queue *aq = to_ap_queue(dev);
+	int rc;
+
+	spin_lock_bh(&aq->lock);
+	rc = aq->last_err_rc;
+	spin_unlock_bh(&aq->lock);
+
+	switch (rc) {
+	case AP_RESPONSE_NORMAL:
+		return sysfs_emit(buf, "NORMAL\n");
+	case AP_RESPONSE_Q_NOT_AVAIL:
+		return sysfs_emit(buf, "Q_NOT_AVAIL\n");
+	case AP_RESPONSE_RESET_IN_PROGRESS:
+		return sysfs_emit(buf, "RESET_IN_PROGRESS\n");
+	case AP_RESPONSE_DECONFIGURED:
+		return sysfs_emit(buf, "DECONFIGURED\n");
+	case AP_RESPONSE_CHECKSTOPPED:
+		return sysfs_emit(buf, "CHECKSTOPPED\n");
+	case AP_RESPONSE_BUSY:
+		return sysfs_emit(buf, "BUSY\n");
+	case AP_RESPONSE_INVALID_ADDRESS:
+		return sysfs_emit(buf, "INVALID_ADDRESS\n");
+	case AP_RESPONSE_OTHERWISE_CHANGED:
+		return sysfs_emit(buf, "OTHERWISE_CHANGED\n");
+	case AP_RESPONSE_Q_FULL:
+		return sysfs_emit(buf, "Q_FULL/NO_PENDING_REPLY\n");
+	case AP_RESPONSE_INDEX_TOO_BIG:
+		return sysfs_emit(buf, "INDEX_TOO_BIG\n");
+	case AP_RESPONSE_NO_FIRST_PART:
+		return sysfs_emit(buf, "NO_FIRST_PART\n");
+	case AP_RESPONSE_MESSAGE_TOO_BIG:
+		return sysfs_emit(buf, "MESSAGE_TOO_BIG\n");
+	case AP_RESPONSE_REQ_FAC_NOT_INST:
+		return sysfs_emit(buf, "REQ_FAC_NOT_INST\n");
+	default:
+		return sysfs_emit(buf, "response code %d\n", rc);
+	}
+}
+static DEVICE_ATTR_RO(last_err_rc);
+#endif
+
+static struct attribute *ap_queue_dev_attrs[] = {
+	&dev_attr_request_count.attr,
+	&dev_attr_requestq_count.attr,
+	&dev_attr_pendingq_count.attr,
+	&dev_attr_reset.attr,
+	&dev_attr_interrupt.attr,
+	&dev_attr_config.attr,
+	&dev_attr_chkstop.attr,
+	&dev_attr_ap_functions.attr,
+	&dev_attr_driver_override.attr,
+#ifdef CONFIG_AP_DEBUG
+	&dev_attr_states.attr,
+	&dev_attr_last_err_rc.attr,
+#endif
+	NULL
+};
+
+static struct attribute_group ap_queue_dev_attr_group = {
+	.attrs = ap_queue_dev_attrs
+};
+
+static const struct attribute_group *ap_queue_dev_attr_groups[] = {
+	&ap_queue_dev_attr_group,
+	NULL
+};
+
+static struct device_type ap_queue_type = {
+	.name = "ap_queue",
+	.groups = ap_queue_dev_attr_groups,
+};
+
+static ssize_t se_bind_show(struct device *dev,
+			    struct device_attribute *attr, char *buf)
+{
+	struct ap_queue *aq = to_ap_queue(dev);
+	struct ap_queue_status status;
+	struct ap_tapq_hwinfo hwinfo;
+
+	if (!ap_q_supports_bind(aq))
+		return sysfs_emit(buf, "-\n");
+
+	status = ap_test_queue(aq->qid, 1, &hwinfo);
+	if (status.response_code > AP_RESPONSE_BUSY) {
+		pr_debug("RC 0x%02x on tapq(0x%02x.%04x)\n",
+			 status.response_code,
+			 AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
+		return -EIO;
+	}
+
+	/* update queue's SE bind state */
+	spin_lock_bh(&aq->lock);
+	aq->se_bstate = hwinfo.bs;
+	spin_unlock_bh(&aq->lock);
+
+	switch (hwinfo.bs) {
+	case AP_BS_Q_USABLE:
+	case AP_BS_Q_USABLE_NO_SECURE_KEY:
+		return sysfs_emit(buf, "bound\n");
+	default:
+		return sysfs_emit(buf, "unbound\n");
+	}
+}
+
+static ssize_t se_bind_store(struct device *dev,
+			     struct device_attribute *attr,
+			     const char *buf, size_t count)
+{
+	struct ap_queue *aq = to_ap_queue(dev);
+	struct ap_queue_status status;
+	struct ap_tapq_hwinfo hwinfo;
+	bool value;
+	int rc;
+
+	if (!ap_q_supports_bind(aq))
+		return -EINVAL;
+
+	/* only 0 (unbind) and 1 (bind) allowed */
+	rc = kstrtobool(buf, &value);
+	if (rc)
+		return rc;
+
+	if (!value) {
+		/* Unbind. Set F bit arg and trigger RAPQ */
+		spin_lock_bh(&aq->lock);
+		__ap_flush_queue(aq);
+		aq->rapq_fbit = 1;
+		_ap_queue_init_state(aq);
+		spin_unlock_bh(&aq->lock);
+		return count;
+	}
+
+	/* lock this ap to have fetch and update in an atomic way */
+	spin_lock_bh(&aq->lock);
+
+	/* Bind. Check current SE bind state */
+	status = ap_test_queue(aq->qid, 1, &hwinfo);
+	if (status.response_code) {
+		AP_DBF_WARN("%s RC 0x%02x on tapq(0x%02x.%04x)\n",
+			    __func__, status.response_code,
+			    AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
+		rc = -EIO;
+		goto error;
+	}
+
+	/* Update BS state */
+	aq->se_bstate = hwinfo.bs;
+	if (hwinfo.bs != AP_BS_Q_AVAIL_FOR_BINDING) {
+		AP_DBF_WARN("%s bind attempt with bs %d on queue 0x%02x.%04x\n",
+			    __func__, hwinfo.bs,
+			    AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
+		rc = -EINVAL;
+		goto error;
+	}
+
+	/* Check SM state */
+	if (aq->sm_state < AP_SM_STATE_IDLE) {
+		rc = -EBUSY;
+		goto error;
+	}
+
+	/* invoke BAPQ */
+	status = ap_bapq(aq->qid);
+	if (status.response_code) {
+		AP_DBF_WARN("%s RC 0x%02x on bapq(0x%02x.%04x)\n",
+			    __func__, status.response_code,
+			    AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
+		rc = -EIO;
+		goto error;
+	}
+	aq->assoc_idx = ASSOC_IDX_INVALID;
+
+	/* verify SE bind state */
+	status = ap_test_queue(aq->qid, 1, &hwinfo);
+	if (status.response_code) {
+		AP_DBF_WARN("%s RC 0x%02x on tapq(0x%02x.%04x)\n",
+			    __func__, status.response_code,
+			    AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
+		rc = -EIO;
+		goto error;
+	}
+	aq->se_bstate = hwinfo.bs;
+	if (!(hwinfo.bs == AP_BS_Q_USABLE ||
+	      hwinfo.bs == AP_BS_Q_USABLE_NO_SECURE_KEY)) {
+		AP_DBF_WARN("%s BAPQ success, but bs shows %d on queue 0x%02x.%04x\n",
+			    __func__, hwinfo.bs,
+			    AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
+		rc = -EIO;
+		goto error;
+	}
+
+	/* SE bind was successful */
+	spin_unlock_bh(&aq->lock);
+
+	AP_DBF_INFO("%s bapq(0x%02x.%04x) success\n", __func__,
+		    AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
+	ap_send_se_bind_uevent(&aq->ap_dev);
+
+	return count;
+
+error:
+	spin_unlock_bh(&aq->lock);
+	return rc;
+}
+
+static DEVICE_ATTR_RW(se_bind);
+
+static ssize_t se_associate_show(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	struct ap_queue *aq = to_ap_queue(dev);
+	struct ap_queue_status status;
+	struct ap_tapq_hwinfo hwinfo;
+
+	if (!ap_q_supports_assoc(aq))
+		return sysfs_emit(buf, "-\n");
+
+	status = ap_test_queue(aq->qid, 1, &hwinfo);
+	if (status.response_code > AP_RESPONSE_BUSY) {
+		pr_debug("RC 0x%02x on tapq(0x%02x.%04x)\n",
+			 status.response_code,
+			 AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
+		return -EIO;
+	}
+
+	/* update queue's SE bind state */
+	spin_lock_bh(&aq->lock);
+	aq->se_bstate = hwinfo.bs;
+	spin_unlock_bh(&aq->lock);
+
+	switch (hwinfo.bs) {
+	case AP_BS_Q_USABLE:
+		if (aq->assoc_idx == ASSOC_IDX_INVALID) {
+			AP_DBF_WARN("%s AP_BS_Q_USABLE but invalid assoc_idx\n", __func__);
+			return -EIO;
+		}
+		return sysfs_emit(buf, "associated %u\n", aq->assoc_idx);
+	case AP_BS_Q_USABLE_NO_SECURE_KEY:
+		if (aq->assoc_idx != ASSOC_IDX_INVALID)
+			return sysfs_emit(buf, "association pending\n");
+		fallthrough;
+	default:
+		return sysfs_emit(buf, "unassociated\n");
+	}
+}
+
+static ssize_t se_associate_store(struct device *dev,
+				  struct device_attribute *attr,
+				  const char *buf, size_t count)
+{
+	struct ap_queue *aq = to_ap_queue(dev);
+	struct ap_queue_status status;
+	struct ap_tapq_hwinfo hwinfo;
+	unsigned int value;
+	int rc;
+
+	if (!ap_q_supports_assoc(aq))
+		return -EINVAL;
+
+	/* association index needs to be >= 0 */
+	rc = kstrtouint(buf, 0, &value);
+	if (rc)
+		return rc;
+	if (value >= ASSOC_IDX_INVALID)
+		return -EINVAL;
+
+	/* lock this ap to have fetch and update in an atomic way */
+	spin_lock_bh(&aq->lock);
+
+	/* check current SE bind state */
+	status = ap_test_queue(aq->qid, 1, &hwinfo);
+	if (status.response_code) {
+		AP_DBF_WARN("%s RC 0x%02x on tapq(0x%02x.%04x)\n",
+			    __func__, status.response_code,
+			    AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
+		rc = -EIO;
+		goto out;
+	}
+	aq->se_bstate = hwinfo.bs;
+	if (hwinfo.bs != AP_BS_Q_USABLE_NO_SECURE_KEY) {
+		AP_DBF_WARN("%s association attempt with bs %d on queue 0x%02x.%04x\n",
+			    __func__, hwinfo.bs,
+			    AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
+		rc = -EINVAL;
+		goto out;
+	}
+
+	/* check SM state */
+	if (aq->sm_state != AP_SM_STATE_IDLE) {
+		rc = -EBUSY;
+		goto out;
+	}
+
+	/* trigger the asynchronous association request */
+	status = ap_aapq(aq->qid, value);
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+	case AP_RESPONSE_STATE_CHANGE_IN_PROGRESS:
+		aq->sm_state = AP_SM_STATE_ASSOC_WAIT;
+		aq->assoc_idx = value;
+		ap_wait(ap_sm_event(aq, AP_SM_EVENT_POLL));
+		rc = count;
+		break;
+	default:
+		AP_DBF_WARN("%s RC 0x%02x on aapq(0x%02x.%04x)\n",
+			    __func__, status.response_code,
+			    AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
+		rc = -EIO;
+		break;
+	}
+
+out:
+	spin_unlock_bh(&aq->lock);
+	return rc;
+}
+
+static DEVICE_ATTR_RW(se_associate);
+
+static struct attribute *ap_queue_dev_sb_attrs[] = {
+	&dev_attr_se_bind.attr,
+	&dev_attr_se_associate.attr,
+	NULL
+};
+
+static struct attribute_group ap_queue_dev_sb_attr_group = {
+	.attrs = ap_queue_dev_sb_attrs
+};
+
+static const struct attribute_group *ap_queue_dev_sb_attr_groups[] = {
+	&ap_queue_dev_sb_attr_group,
+	NULL
+};
+
+static void ap_queue_device_release(struct device *dev)
+{
+	struct ap_queue *aq = to_ap_queue(dev);
+
+	spin_lock_bh(&ap_queues_lock);
+	hash_del(&aq->hnode);
+	spin_unlock_bh(&ap_queues_lock);
+
+	kfree(aq);
+}
+
+struct ap_queue *ap_queue_create(ap_qid_t qid, struct ap_card *ac)
+{
+	struct ap_queue *aq;
+
+	aq = kzalloc_obj(*aq);
+	if (!aq)
+		return NULL;
+	aq->card = ac;
+	aq->ap_dev.device.release = ap_queue_device_release;
+	aq->ap_dev.device.type = &ap_queue_type;
+	aq->ap_dev.device_type = ac->ap_dev.device_type;
+	/* in SE environment add bind/associate attributes group */
+	if (ap_is_se_guest() && ap_q_supported_in_se(aq))
+		aq->ap_dev.device.groups = ap_queue_dev_sb_attr_groups;
+	aq->qid = qid;
+	spin_lock_init(&aq->lock);
+	INIT_LIST_HEAD(&aq->pendingq);
+	INIT_LIST_HEAD(&aq->requestq);
+	timer_setup(&aq->timeout, ap_request_timeout, 0);
+
+	return aq;
+}
+
+void ap_queue_init_reply(struct ap_queue *aq, struct ap_message *reply)
+{
+	aq->reply = reply;
+
+	spin_lock_bh(&aq->lock);
+	ap_wait(ap_sm_event(aq, AP_SM_EVENT_POLL));
+	spin_unlock_bh(&aq->lock);
+}
+EXPORT_SYMBOL(ap_queue_init_reply);
+
+/**
+ * ap_queue_message(): Queue a request to an AP device.
+ * @aq: The AP device to queue the message to
+ * @ap_msg: The message that is to be added
+ */
+int ap_queue_message(struct ap_queue *aq, struct ap_message *ap_msg)
+{
+	int rc = 0;
+
+	/* msg needs to have a valid receive-callback */
+	BUG_ON(!ap_msg->receive);
+
+	spin_lock_bh(&aq->lock);
+
+	/* only allow to queue new messages if device state is ok */
+	if (aq->dev_state == AP_DEV_STATE_OPERATING) {
+		list_add_tail(&ap_msg->list, &aq->requestq);
+		aq->requestq_count++;
+		aq->total_request_count++;
+		atomic64_inc(&aq->card->total_request_count);
+	} else {
+		rc = -ENODEV;
+	}
+
+	/* Send/receive as many request from the queue as possible. */
+	ap_wait(ap_sm_event_loop(aq, AP_SM_EVENT_POLL));
+
+	spin_unlock_bh(&aq->lock);
+
+	return rc;
+}
+EXPORT_SYMBOL(ap_queue_message);
+
+/**
+ * ap_queue_usable(): Check if queue is usable just now.
+ * @aq: The AP queue device to test for usability.
+ * This function is intended for the scheduler to query if it makes
+ * sense to enqueue a message into this AP queue device by calling
+ * ap_queue_message(). The perspective is very short-term as the
+ * state machine and device state(s) may change at any time.
+ */
+bool ap_queue_usable(struct ap_queue *aq)
+{
+	bool rc = true;
+
+	spin_lock_bh(&aq->lock);
+
+	/* check for not configured or checkstopped */
+	if (!aq->config || aq->chkstop) {
+		rc = false;
+		goto unlock_and_out;
+	}
+
+	/* device state needs to be ok */
+	if (aq->dev_state != AP_DEV_STATE_OPERATING) {
+		rc = false;
+		goto unlock_and_out;
+	}
+
+	/* SE guest's queues additionally need to be bound */
+	if (ap_is_se_guest()) {
+		if (!ap_q_supported_in_se(aq)) {
+			rc = false;
+			goto unlock_and_out;
+		}
+		if (ap_q_needs_bind(aq) &&
+		    !(aq->se_bstate == AP_BS_Q_USABLE ||
+		      aq->se_bstate == AP_BS_Q_USABLE_NO_SECURE_KEY))
+			rc = false;
+	}
+
+unlock_and_out:
+	spin_unlock_bh(&aq->lock);
+	return rc;
+}
+EXPORT_SYMBOL(ap_queue_usable);
+
+/**
+ * ap_cancel_message(): Cancel a crypto request.
+ * @aq: The AP device that has the message queued
+ * @ap_msg: The message that is to be removed
+ *
+ * Cancel a crypto request. This is done by removing the request
+ * from the device pending or request queue. Note that the
+ * request stays on the AP queue. When it finishes the message
+ * reply will be discarded because the psmid can't be found.
+ */
+void ap_cancel_message(struct ap_queue *aq, struct ap_message *ap_msg)
+{
+	struct ap_message *tmp;
+
+	spin_lock_bh(&aq->lock);
+	if (!list_empty(&ap_msg->list)) {
+		list_for_each_entry(tmp, &aq->pendingq, list)
+			if (tmp->psmid == ap_msg->psmid) {
+				aq->pendingq_count--;
+				goto found;
+			}
+		aq->requestq_count--;
+found:
+		list_del_init(&ap_msg->list);
+	}
+	spin_unlock_bh(&aq->lock);
+}
+EXPORT_SYMBOL(ap_cancel_message);
+
+/**
+ * __ap_flush_queue(): Flush requests.
+ * @aq: Pointer to the AP queue
+ *
+ * Flush all requests from the request/pending queue of an AP device.
+ */
+static void __ap_flush_queue(struct ap_queue *aq)
+{
+	struct ap_message *ap_msg, *next;
+
+	list_for_each_entry_safe(ap_msg, next, &aq->pendingq, list) {
+		list_del_init(&ap_msg->list);
+		aq->pendingq_count--;
+		ap_msg->rc = -EAGAIN;
+		ap_msg->receive(aq, ap_msg, NULL);
+	}
+	list_for_each_entry_safe(ap_msg, next, &aq->requestq, list) {
+		list_del_init(&ap_msg->list);
+		aq->requestq_count--;
+		ap_msg->rc = -EAGAIN;
+		ap_msg->receive(aq, ap_msg, NULL);
+	}
+	aq->queue_count = 0;
+}
+
+void ap_flush_queue(struct ap_queue *aq)
+{
+	spin_lock_bh(&aq->lock);
+	__ap_flush_queue(aq);
+	spin_unlock_bh(&aq->lock);
+}
+EXPORT_SYMBOL(ap_flush_queue);
+
+void ap_queue_prepare_remove(struct ap_queue *aq)
+{
+	spin_lock_bh(&aq->lock);
+	/* flush queue */
+	__ap_flush_queue(aq);
+	/* move queue device state to SHUTDOWN in progress */
+	aq->dev_state = AP_DEV_STATE_SHUTDOWN;
+	spin_unlock_bh(&aq->lock);
+	timer_delete_sync(&aq->timeout);
+}
+
+void ap_queue_remove(struct ap_queue *aq)
+{
+	/*
+	 * all messages have been flushed and the device state
+	 * is SHUTDOWN. Now reset with zero which also clears
+	 * the irq registration and move the device state
+	 * to the initial value AP_DEV_STATE_UNINITIATED.
+	 */
+	spin_lock_bh(&aq->lock);
+	ap_zapq(aq->qid, 0);
+	aq->dev_state = AP_DEV_STATE_UNINITIATED;
+	spin_unlock_bh(&aq->lock);
+}
+
+void _ap_queue_init_state(struct ap_queue *aq)
+{
+	aq->dev_state = AP_DEV_STATE_OPERATING;
+	aq->sm_state = AP_SM_STATE_RESET_START;
+	aq->last_err_rc = 0;
+	aq->assoc_idx = ASSOC_IDX_INVALID;
+	ap_wait(ap_sm_event(aq, AP_SM_EVENT_POLL));
+}
+
+void ap_queue_init_state(struct ap_queue *aq)
+{
+	spin_lock_bh(&aq->lock);
+	_ap_queue_init_state(aq);
+	spin_unlock_bh(&aq->lock);
+}
+EXPORT_SYMBOL(ap_queue_init_state);
diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c
new file mode 100644
index 000000000..0530b1ce2
--- /dev/null
+++ b/drivers/s390/crypto/pkey_api.c
@@ -0,0 +1,821 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  pkey device driver
+ *
+ *  Copyright IBM Corp. 2017, 2023
+ *
+ *  Author(s): Harald Freudenberger
+ */
+
+#define pr_fmt(fmt) "pkey: " fmt
+
+#include <linux/init.h>
+#include <linux/miscdevice.h>
+#include <linux/export.h>
+#include <linux/slab.h>
+
+#include "zcrypt_api.h"
+#include "zcrypt_ccamisc.h"
+
+#include "pkey_base.h"
+
+/*
+ * Helper functions
+ */
+static int key2protkey(const struct pkey_apqn *apqns, size_t nr_apqns,
+		       const u8 *key, size_t keylen,
+		       u8 *protkey, u32 *protkeylen, u32 *protkeytype,
+		       u32 xflags)
+{
+	int rc;
+
+	/* try the direct way */
+	rc = pkey_handler_key_to_protkey(apqns, nr_apqns,
+					 key, keylen,
+					 protkey, protkeylen,
+					 protkeytype, xflags);
+
+	/* if this did not work, try the slowpath way */
+	if (rc == -ENODEV) {
+		rc = pkey_handler_slowpath_key_to_protkey(apqns, nr_apqns,
+							  key, keylen,
+							  protkey, protkeylen,
+							  protkeytype, xflags);
+		if (rc)
+			rc = -ENODEV;
+	}
+
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+
+/*
+ * In-Kernel function: Transform a key blob (of any type) into a protected key
+ */
+int pkey_key2protkey(const u8 *key, u32 keylen,
+		     u8 *protkey, u32 *protkeylen, u32 *protkeytype, u32 xflags)
+{
+	int rc;
+
+	rc = key2protkey(NULL, 0, key, keylen,
+			 protkey, protkeylen, protkeytype, xflags);
+	if (rc == -ENODEV) {
+		pkey_handler_request_modules();
+		rc = key2protkey(NULL, 0, key, keylen,
+				 protkey, protkeylen, protkeytype, xflags);
+	}
+
+	return rc;
+}
+EXPORT_SYMBOL(pkey_key2protkey);
+
+/*
+ * Ioctl functions
+ */
+
+static void *_copy_key_from_user(void __user *ukey, size_t keylen)
+{
+	if (!ukey || keylen < MINKEYBLOBBUFSIZE || keylen > KEYBLOBBUFSIZE)
+		return ERR_PTR(-EINVAL);
+
+	return memdup_user(ukey, keylen);
+}
+
+static void *_copy_apqns_from_user(void __user *uapqns, size_t nr_apqns)
+{
+	if (!uapqns || nr_apqns == 0)
+		return NULL;
+
+	return memdup_array_user(uapqns, nr_apqns, sizeof(struct pkey_apqn));
+}
+
+static int pkey_ioctl_genseck(struct pkey_genseck __user *ugs)
+{
+	struct pkey_genseck kgs;
+	struct pkey_apqn apqn;
+	u32 keybuflen;
+	int rc;
+
+	if (copy_from_user(&kgs, ugs, sizeof(kgs)))
+		return -EFAULT;
+
+	apqn.card = kgs.cardnr;
+	apqn.domain = kgs.domain;
+	keybuflen = sizeof(kgs.seckey.seckey);
+	rc = pkey_handler_gen_key(&apqn, 1,
+				  kgs.keytype, PKEY_TYPE_CCA_DATA, 0, 0,
+				  kgs.seckey.seckey, &keybuflen, NULL, 0);
+	pr_debug("gen_key()=%d\n", rc);
+	if (!rc && copy_to_user(ugs, &kgs, sizeof(kgs)))
+		rc = -EFAULT;
+	memzero_explicit(&kgs, sizeof(kgs));
+
+	return rc;
+}
+
+static int pkey_ioctl_clr2seck(struct pkey_clr2seck __user *ucs)
+{
+	struct pkey_clr2seck kcs;
+	struct pkey_apqn apqn;
+	u32 keybuflen;
+	int rc;
+
+	if (copy_from_user(&kcs, ucs, sizeof(kcs)))
+		return -EFAULT;
+
+	apqn.card = kcs.cardnr;
+	apqn.domain = kcs.domain;
+	keybuflen = sizeof(kcs.seckey.seckey);
+	rc = pkey_handler_clr_to_key(&apqn, 1,
+				     kcs.keytype, PKEY_TYPE_CCA_DATA, 0, 0,
+				     kcs.clrkey.clrkey,
+				     pkey_keytype_aes_to_size(kcs.keytype),
+				     kcs.seckey.seckey, &keybuflen, NULL, 0);
+	pr_debug("clr_to_key()=%d\n", rc);
+	if (!rc && copy_to_user(ucs, &kcs, sizeof(kcs)))
+		rc = -EFAULT;
+	memzero_explicit(&kcs, sizeof(kcs));
+
+	return rc;
+}
+
+static int pkey_ioctl_sec2protk(struct pkey_sec2protk __user *usp)
+{
+	struct pkey_sec2protk ksp;
+	struct pkey_apqn apqn;
+	int rc;
+
+	if (copy_from_user(&ksp, usp, sizeof(ksp)))
+		return -EFAULT;
+
+	apqn.card = ksp.cardnr;
+	apqn.domain = ksp.domain;
+	ksp.protkey.len = sizeof(ksp.protkey.protkey);
+	rc = pkey_handler_key_to_protkey(&apqn, 1,
+					 ksp.seckey.seckey,
+					 sizeof(ksp.seckey.seckey),
+					 ksp.protkey.protkey,
+					 &ksp.protkey.len, &ksp.protkey.type,
+					 0);
+	pr_debug("key_to_protkey()=%d\n", rc);
+	if (!rc && copy_to_user(usp, &ksp, sizeof(ksp)))
+		rc = -EFAULT;
+	memzero_explicit(&ksp, sizeof(ksp));
+
+	return rc;
+}
+
+static int pkey_ioctl_clr2protk(struct pkey_clr2protk __user *ucp)
+{
+	struct pkey_clr2protk kcp;
+	struct clearkeytoken *t;
+	u8 *tmpbuf = NULL;
+	u32 keylen;
+	int rc;
+
+	if (copy_from_user(&kcp, ucp, sizeof(kcp)))
+		return -EFAULT;
+
+	/* build a 'clear key token' from the clear key value */
+	keylen = pkey_keytype_aes_to_size(kcp.keytype);
+	if (!keylen) {
+		PKEY_DBF_ERR("%s unknown/unsupported keytype %u\n",
+			     __func__, kcp.keytype);
+		rc = -EINVAL;
+		goto out;
+	}
+	tmpbuf = kzalloc(sizeof(*t) + keylen, GFP_KERNEL);
+	if (!tmpbuf) {
+		rc = -ENOMEM;
+		goto out;
+	}
+	t = (struct clearkeytoken *)tmpbuf;
+	t->type = TOKTYPE_NON_CCA;
+	t->version = TOKVER_CLEAR_KEY;
+	t->keytype = (keylen - 8) >> 3;
+	t->len = keylen;
+	memcpy(t->clearkey, kcp.clrkey.clrkey, keylen);
+	kcp.protkey.len = sizeof(kcp.protkey.protkey);
+
+	rc = key2protkey(NULL, 0,
+			 tmpbuf, sizeof(*t) + keylen,
+			 kcp.protkey.protkey,
+			 &kcp.protkey.len, &kcp.protkey.type, 0);
+	pr_debug("key2protkey()=%d\n", rc);
+	if (rc)
+		goto out;
+
+	if (copy_to_user(ucp, &kcp, sizeof(kcp)))
+		rc = -EFAULT;
+
+out:
+	memzero_explicit(&kcp, sizeof(kcp));
+	kfree_sensitive(tmpbuf);
+	return rc;
+}
+
+static int pkey_ioctl_findcard(struct pkey_findcard __user *ufc)
+{
+	struct pkey_apqn *apqns = NULL;
+	struct pkey_findcard kfc;
+	size_t nr_apqns;
+	int rc;
+
+	if (copy_from_user(&kfc, ufc, sizeof(kfc)))
+		return -EFAULT;
+
+	nr_apqns = MAXAPQNSINLIST;
+	apqns = kmalloc_objs(struct pkey_apqn, nr_apqns);
+	if (!apqns) {
+		rc = -ENOMEM;
+		goto out;
+	}
+
+	rc = pkey_handler_apqns_for_key(kfc.seckey.seckey,
+					sizeof(kfc.seckey.seckey),
+					PKEY_FLAGS_MATCH_CUR_MKVP,
+					apqns, &nr_apqns, 0);
+	if (rc == -ENODEV)
+		rc = pkey_handler_apqns_for_key(kfc.seckey.seckey,
+						sizeof(kfc.seckey.seckey),
+						PKEY_FLAGS_MATCH_ALT_MKVP,
+						apqns, &nr_apqns, 0);
+	pr_debug("apqns_for_key()=%d\n", rc);
+	if (rc)
+		goto out;
+
+	kfc.cardnr = apqns[0].card;
+	kfc.domain = apqns[0].domain;
+
+	if (copy_to_user(ufc, &kfc, sizeof(kfc)))
+		rc = -EFAULT;
+
+out:
+	kfree(apqns);
+	return rc;
+}
+
+static int pkey_ioctl_skey2pkey(struct pkey_skey2pkey __user *usp)
+{
+	struct pkey_skey2pkey ksp;
+	int rc;
+
+	if (copy_from_user(&ksp, usp, sizeof(ksp)))
+		return -EFAULT;
+
+	ksp.protkey.len = sizeof(ksp.protkey.protkey);
+	rc = pkey_handler_key_to_protkey(NULL, 0,
+					 ksp.seckey.seckey,
+					 sizeof(ksp.seckey.seckey),
+					 ksp.protkey.protkey,
+					 &ksp.protkey.len,
+					 &ksp.protkey.type, 0);
+	pr_debug("key_to_protkey()=%d\n", rc);
+	if (!rc && copy_to_user(usp, &ksp, sizeof(ksp)))
+		rc = -EFAULT;
+	memzero_explicit(&ksp, sizeof(ksp));
+
+	return rc;
+}
+
+static int pkey_ioctl_verifykey(struct pkey_verifykey __user *uvk)
+{
+	u32 keytype, keybitsize, flags;
+	struct pkey_verifykey kvk;
+	int rc;
+
+	if (copy_from_user(&kvk, uvk, sizeof(kvk)))
+		return -EFAULT;
+
+	kvk.cardnr = 0xFFFF;
+	kvk.domain = 0xFFFF;
+	rc = pkey_handler_verify_key(kvk.seckey.seckey,
+				     sizeof(kvk.seckey.seckey),
+				     &kvk.cardnr, &kvk.domain,
+				     &keytype, &keybitsize, &flags, 0);
+	pr_debug("verify_key()=%d\n", rc);
+	if (!rc && keytype != PKEY_TYPE_CCA_DATA)
+		rc = -EINVAL;
+	kvk.attributes = PKEY_VERIFY_ATTR_AES;
+	kvk.keysize = (u16)keybitsize;
+	if (flags & PKEY_FLAGS_MATCH_ALT_MKVP)
+		kvk.attributes |= PKEY_VERIFY_ATTR_OLD_MKVP;
+	if (!rc && copy_to_user(uvk, &kvk, sizeof(kvk)))
+		rc = -EFAULT;
+	memzero_explicit(&kvk, sizeof(kvk));
+
+	return rc;
+}
+
+static int pkey_ioctl_genprotk(struct pkey_genprotk __user *ugp)
+{
+	struct pkey_genprotk kgp;
+	int rc;
+
+	if (copy_from_user(&kgp, ugp, sizeof(kgp)))
+		return -EFAULT;
+
+	kgp.protkey.len = sizeof(kgp.protkey.protkey);
+	rc = pkey_handler_gen_key(NULL, 0, kgp.keytype,
+				  PKEY_TYPE_PROTKEY, 0, 0,
+				  kgp.protkey.protkey, &kgp.protkey.len,
+				  &kgp.protkey.type, 0);
+	pr_debug("gen_key()=%d\n", rc);
+	if (!rc && copy_to_user(ugp, &kgp, sizeof(kgp)))
+		rc = -EFAULT;
+	memzero_explicit(&kgp, sizeof(kgp));
+
+	return rc;
+}
+
+static int pkey_ioctl_verifyprotk(struct pkey_verifyprotk __user *uvp)
+{
+	struct pkey_verifyprotk kvp;
+	struct protaeskeytoken *t;
+	u8 *tmpbuf = NULL;
+	int rc;
+
+	if (copy_from_user(&kvp, uvp, sizeof(kvp)))
+		return -EFAULT;
+
+	if (kvp.protkey.len > sizeof(kvp.protkey.protkey)) {
+		PKEY_DBF_ERR("%s protkey length %u exceeds protkey buffer size\n",
+			     __func__, kvp.protkey.len);
+		rc = -EINVAL;
+		goto out;
+	}
+
+	/* build a 'protected key token' from the raw protected key */
+	tmpbuf = kzalloc(sizeof(*t), GFP_KERNEL);
+	if (!tmpbuf) {
+		rc = -ENOMEM;
+		goto out;
+	}
+	t = (struct protaeskeytoken *)tmpbuf;
+	t->type = TOKTYPE_NON_CCA;
+	t->version = TOKVER_PROTECTED_KEY;
+	t->keytype = kvp.protkey.type;
+	t->len = kvp.protkey.len;
+	memcpy(t->protkey, kvp.protkey.protkey, kvp.protkey.len);
+
+	rc = pkey_handler_verify_key(tmpbuf, sizeof(*t),
+				     NULL, NULL, NULL, NULL, NULL, 0);
+	pr_debug("verify_key()=%d\n", rc);
+
+out:
+	memzero_explicit(&kvp, sizeof(kvp));
+	kfree_sensitive(tmpbuf);
+	return rc;
+}
+
+static int pkey_ioctl_kblob2protk(struct pkey_kblob2pkey __user *utp)
+{
+	struct pkey_kblob2pkey ktp;
+	u8 *kkey = NULL;
+	int rc;
+
+	if (copy_from_user(&ktp, utp, sizeof(ktp)))
+		return -EFAULT;
+	kkey = _copy_key_from_user(ktp.key, ktp.keylen);
+	if (IS_ERR(kkey)) {
+		rc = PTR_ERR(kkey);
+		kkey = NULL;
+		goto out;
+	}
+	ktp.protkey.len = sizeof(ktp.protkey.protkey);
+	rc = key2protkey(NULL, 0, kkey, ktp.keylen,
+			 ktp.protkey.protkey, &ktp.protkey.len,
+			 &ktp.protkey.type, 0);
+	pr_debug("key2protkey()=%d\n", rc);
+	if (rc)
+		goto out;
+	if (copy_to_user(utp, &ktp, sizeof(ktp)))
+		rc = -EFAULT;
+
+out:
+	memzero_explicit(&ktp, sizeof(ktp));
+	kfree_sensitive(kkey);
+	return rc;
+}
+
+static int pkey_ioctl_genseck2(struct pkey_genseck2 __user *ugs)
+{
+	struct pkey_apqn *apqns = NULL;
+	u32 klen = KEYBLOBBUFSIZE;
+	struct pkey_genseck2 kgs;
+	u8 *kkey = NULL;
+	int rc;
+	u32 u;
+
+	if (copy_from_user(&kgs, ugs, sizeof(kgs)))
+		return -EFAULT;
+	u = pkey_aes_bitsize_to_keytype(kgs.size);
+	if (!u) {
+		PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n",
+			     __func__, kgs.size);
+		return -EINVAL;
+	}
+	apqns = _copy_apqns_from_user(kgs.apqns, kgs.apqn_entries);
+	if (IS_ERR(apqns)) {
+		rc = PTR_ERR(apqns);
+		apqns = NULL;
+		goto out;
+	}
+	kkey = kzalloc(klen, GFP_KERNEL);
+	if (!kkey) {
+		rc = -ENOMEM;
+		goto out;
+	}
+	rc = pkey_handler_gen_key(apqns, kgs.apqn_entries,
+				  u, kgs.type, kgs.size, kgs.keygenflags,
+				  kkey, &klen, NULL, 0);
+	pr_debug("gen_key()=%d\n", rc);
+	if (rc)
+		goto out;
+	if (kgs.key) {
+		if (kgs.keylen < klen) {
+			rc = -EINVAL;
+			goto out;
+		}
+		if (copy_to_user(kgs.key, kkey, klen)) {
+			rc = -EFAULT;
+			goto out;
+		}
+	}
+	kgs.keylen = klen;
+	if (copy_to_user(ugs, &kgs, sizeof(kgs)))
+		rc = -EFAULT;
+
+out:
+	kfree_sensitive(kkey);
+	kfree(apqns);
+	return rc;
+}
+
+static int pkey_ioctl_clr2seck2(struct pkey_clr2seck2 __user *ucs)
+{
+	struct pkey_apqn *apqns = NULL;
+	u32 klen = KEYBLOBBUFSIZE;
+	struct pkey_clr2seck2 kcs;
+	u8 *kkey = NULL;
+	int rc;
+	u32 u;
+
+	if (copy_from_user(&kcs, ucs, sizeof(kcs)))
+		return -EFAULT;
+	u = pkey_aes_bitsize_to_keytype(kcs.size);
+	if (!u) {
+		PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n",
+			     __func__, kcs.size);
+		rc = -EINVAL;
+		goto out;
+	}
+	apqns = _copy_apqns_from_user(kcs.apqns, kcs.apqn_entries);
+	if (IS_ERR(apqns)) {
+		rc = PTR_ERR(apqns);
+		apqns = NULL;
+		goto out;
+	}
+	kkey = kzalloc(klen, GFP_KERNEL);
+	if (!kkey) {
+		rc = -ENOMEM;
+		goto out;
+	}
+	rc = pkey_handler_clr_to_key(apqns, kcs.apqn_entries,
+				     u, kcs.type, kcs.size, kcs.keygenflags,
+				     kcs.clrkey.clrkey, kcs.size / 8,
+				     kkey, &klen, NULL, 0);
+	pr_debug("clr_to_key()=%d\n", rc);
+	if (rc)
+		goto out;
+	if (kcs.key) {
+		if (kcs.keylen < klen) {
+			rc = -EINVAL;
+			goto out;
+		}
+		if (copy_to_user(kcs.key, kkey, klen)) {
+			rc = -EFAULT;
+			goto out;
+		}
+	}
+	kcs.keylen = klen;
+	if (copy_to_user(ucs, &kcs, sizeof(kcs)))
+		rc = -EFAULT;
+
+out:
+	memzero_explicit(&kcs, sizeof(kcs));
+	kfree_sensitive(kkey);
+	kfree(apqns);
+	return rc;
+}
+
+static int pkey_ioctl_verifykey2(struct pkey_verifykey2 __user *uvk)
+{
+	struct pkey_verifykey2 kvk;
+	u8 *kkey;
+	int rc;
+
+	if (copy_from_user(&kvk, uvk, sizeof(kvk)))
+		return -EFAULT;
+	kkey = _copy_key_from_user(kvk.key, kvk.keylen);
+	if (IS_ERR(kkey)) {
+		rc = PTR_ERR(kkey);
+		kkey = NULL;
+		goto out;
+	}
+
+	rc = pkey_handler_verify_key(kkey, kvk.keylen,
+				     &kvk.cardnr, &kvk.domain,
+				     &kvk.type, &kvk.size, &kvk.flags, 0);
+	pr_debug("verify_key()=%d\n", rc);
+	if (rc)
+		goto out;
+	if (copy_to_user(uvk, &kvk, sizeof(kvk)))
+		rc = -EFAULT;
+
+out:
+	kfree_sensitive(kkey);
+	return rc;
+}
+
+static int pkey_ioctl_kblob2protk2(struct pkey_kblob2pkey2 __user *utp)
+{
+	struct pkey_apqn *apqns = NULL;
+	struct pkey_kblob2pkey2 ktp;
+	u8 *kkey = NULL;
+	int rc;
+
+	if (copy_from_user(&ktp, utp, sizeof(ktp)))
+		return -EFAULT;
+	apqns = _copy_apqns_from_user(ktp.apqns, ktp.apqn_entries);
+	if (IS_ERR(apqns)) {
+		rc = PTR_ERR(apqns);
+		apqns = NULL;
+		goto out;
+	}
+	kkey = _copy_key_from_user(ktp.key, ktp.keylen);
+	if (IS_ERR(kkey)) {
+		rc = PTR_ERR(kkey);
+		kkey = NULL;
+		goto out;
+	}
+	ktp.protkey.len = sizeof(ktp.protkey.protkey);
+	rc = key2protkey(apqns, ktp.apqn_entries, kkey, ktp.keylen,
+			 ktp.protkey.protkey, &ktp.protkey.len,
+			 &ktp.protkey.type, 0);
+	pr_debug("key2protkey()=%d\n", rc);
+	if (rc)
+		goto out;
+	if (copy_to_user(utp, &ktp, sizeof(ktp)))
+		rc = -EFAULT;
+
+out:
+	memzero_explicit(&ktp, sizeof(ktp));
+	kfree_sensitive(kkey);
+	kfree(apqns);
+	return rc;
+}
+
+static int pkey_ioctl_apqns4k(struct pkey_apqns4key __user *uak)
+{
+	struct pkey_apqn *apqns = NULL;
+	struct pkey_apqns4key kak;
+	size_t nr_apqns, len;
+	u8 *kkey = NULL;
+	int rc;
+
+	if (copy_from_user(&kak, uak, sizeof(kak)))
+		return -EFAULT;
+	nr_apqns = kak.apqn_entries;
+	if (nr_apqns) {
+		apqns = kmalloc_objs(struct pkey_apqn, nr_apqns);
+		if (!apqns) {
+			rc = -ENOMEM;
+			goto out;
+		}
+	}
+	kkey = _copy_key_from_user(kak.key, kak.keylen);
+	if (IS_ERR(kkey)) {
+		rc = PTR_ERR(kkey);
+		kkey = NULL;
+		goto out;
+	}
+	rc = pkey_handler_apqns_for_key(kkey, kak.keylen, kak.flags,
+					apqns, &nr_apqns, 0);
+	pr_debug("apqns_for_key()=%d\n", rc);
+	if (rc && rc != -ENOSPC)
+		goto out;
+	if (!rc && kak.apqns) {
+		if (nr_apqns > kak.apqn_entries) {
+			rc = -EINVAL;
+			goto out;
+		}
+		len = nr_apqns * sizeof(struct pkey_apqn);
+		if (len) {
+			if (copy_to_user(kak.apqns, apqns, len)) {
+				rc = -EFAULT;
+				goto out;
+			}
+		}
+	}
+	kak.apqn_entries = nr_apqns;
+	if (copy_to_user(uak, &kak, sizeof(kak)))
+		rc = -EFAULT;
+
+out:
+	kfree_sensitive(kkey);
+	kfree(apqns);
+	return rc;
+}
+
+static int pkey_ioctl_apqns4kt(struct pkey_apqns4keytype __user *uat)
+{
+	struct pkey_apqn *apqns = NULL;
+	struct pkey_apqns4keytype kat;
+	size_t nr_apqns, len;
+	int rc;
+
+	if (copy_from_user(&kat, uat, sizeof(kat)))
+		return -EFAULT;
+	nr_apqns = kat.apqn_entries;
+	if (nr_apqns) {
+		apqns = kmalloc_objs(struct pkey_apqn, nr_apqns);
+		if (!apqns) {
+			rc = -ENOMEM;
+			goto out;
+		}
+	}
+	rc = pkey_handler_apqns_for_keytype(kat.type,
+					    kat.cur_mkvp, kat.alt_mkvp,
+					    kat.flags, apqns, &nr_apqns, 0);
+	pr_debug("apqns_for_keytype()=%d\n", rc);
+	if (rc && rc != -ENOSPC)
+		goto out;
+	if (!rc && kat.apqns) {
+		if (nr_apqns > kat.apqn_entries) {
+			rc = -EINVAL;
+			goto out;
+		}
+		len = nr_apqns * sizeof(struct pkey_apqn);
+		if (len) {
+			if (copy_to_user(kat.apqns, apqns, len)) {
+				rc = -EFAULT;
+				goto out;
+			}
+		}
+	}
+	kat.apqn_entries = nr_apqns;
+	if (copy_to_user(uat, &kat, sizeof(kat)))
+		rc = -EFAULT;
+
+out:
+	kfree(apqns);
+	return rc;
+}
+
+static int pkey_ioctl_kblob2protk3(struct pkey_kblob2pkey3 __user *utp)
+{
+	u32 protkeylen = PROTKEYBLOBBUFSIZE;
+	u8 *kkey = NULL, *protkey = NULL;
+	struct pkey_apqn *apqns = NULL;
+	struct pkey_kblob2pkey3 ktp;
+	int rc;
+
+	if (copy_from_user(&ktp, utp, sizeof(ktp)))
+		return -EFAULT;
+	apqns = _copy_apqns_from_user(ktp.apqns, ktp.apqn_entries);
+	if (IS_ERR(apqns)) {
+		rc = PTR_ERR(apqns);
+		apqns = NULL;
+		goto out;
+	}
+	kkey = _copy_key_from_user(ktp.key, ktp.keylen);
+	if (IS_ERR(kkey)) {
+		rc = PTR_ERR(kkey);
+		kkey = NULL;
+		goto out;
+	}
+	protkey = kmalloc(protkeylen, GFP_KERNEL);
+	if (!protkey) {
+		rc = -ENOMEM;
+		goto out;
+	}
+	rc = key2protkey(apqns, ktp.apqn_entries, kkey, ktp.keylen,
+			 protkey, &protkeylen, &ktp.pkeytype, 0);
+	pr_debug("key2protkey()=%d\n", rc);
+	if (rc)
+		goto out;
+	if (ktp.pkey && ktp.pkeylen) {
+		if (protkeylen > ktp.pkeylen) {
+			rc = -EINVAL;
+			goto out;
+		}
+		if (copy_to_user(ktp.pkey, protkey, protkeylen)) {
+			rc = -EFAULT;
+			goto out;
+		}
+	}
+	ktp.pkeylen = protkeylen;
+	if (copy_to_user(utp, &ktp, sizeof(ktp)))
+		rc = -EFAULT;
+
+out:
+	kfree_sensitive(protkey);
+	kfree_sensitive(kkey);
+	kfree(apqns);
+	return rc;
+}
+
+static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd,
+				unsigned long arg)
+{
+	int rc;
+
+	switch (cmd) {
+	case PKEY_GENSECK:
+		rc = pkey_ioctl_genseck((struct pkey_genseck __user *)arg);
+		break;
+	case PKEY_CLR2SECK:
+		rc = pkey_ioctl_clr2seck((struct pkey_clr2seck __user *)arg);
+		break;
+	case PKEY_SEC2PROTK:
+		rc = pkey_ioctl_sec2protk((struct pkey_sec2protk __user *)arg);
+		break;
+	case PKEY_CLR2PROTK:
+		rc = pkey_ioctl_clr2protk((struct pkey_clr2protk __user *)arg);
+		break;
+	case PKEY_FINDCARD:
+		rc = pkey_ioctl_findcard((struct pkey_findcard __user *)arg);
+		break;
+	case PKEY_SKEY2PKEY:
+		rc = pkey_ioctl_skey2pkey((struct pkey_skey2pkey __user *)arg);
+		break;
+	case PKEY_VERIFYKEY:
+		rc = pkey_ioctl_verifykey((struct pkey_verifykey __user *)arg);
+		break;
+	case PKEY_GENPROTK:
+		rc = pkey_ioctl_genprotk((struct pkey_genprotk __user *)arg);
+		break;
+	case PKEY_VERIFYPROTK:
+		rc = pkey_ioctl_verifyprotk((struct pkey_verifyprotk __user *)arg);
+		break;
+	case PKEY_KBLOB2PROTK:
+		rc = pkey_ioctl_kblob2protk((struct pkey_kblob2pkey __user *)arg);
+		break;
+	case PKEY_GENSECK2:
+		rc = pkey_ioctl_genseck2((struct pkey_genseck2 __user *)arg);
+		break;
+	case PKEY_CLR2SECK2:
+		rc = pkey_ioctl_clr2seck2((struct pkey_clr2seck2 __user *)arg);
+		break;
+	case PKEY_VERIFYKEY2:
+		rc = pkey_ioctl_verifykey2((struct pkey_verifykey2 __user *)arg);
+		break;
+	case PKEY_KBLOB2PROTK2:
+		rc = pkey_ioctl_kblob2protk2((struct pkey_kblob2pkey2 __user *)arg);
+		break;
+	case PKEY_APQNS4K:
+		rc = pkey_ioctl_apqns4k((struct pkey_apqns4key __user *)arg);
+		break;
+	case PKEY_APQNS4KT:
+		rc = pkey_ioctl_apqns4kt((struct pkey_apqns4keytype __user *)arg);
+		break;
+	case PKEY_KBLOB2PROTK3:
+		rc = pkey_ioctl_kblob2protk3((struct pkey_kblob2pkey3 __user *)arg);
+		break;
+	default:
+		/* unknown/unsupported ioctl cmd */
+		return -ENOTTY;
+	}
+
+	return rc;
+}
+
+/*
+ * File io operations
+ */
+
+static const struct file_operations pkey_fops = {
+	.owner		= THIS_MODULE,
+	.open		= nonseekable_open,
+	.unlocked_ioctl = pkey_unlocked_ioctl,
+};
+
+static struct miscdevice pkey_dev = {
+	.name	= "pkey",
+	.minor	= MISC_DYNAMIC_MINOR,
+	.mode	= 0666,
+	.fops	= &pkey_fops,
+	.groups = pkey_attr_groups,
+};
+
+int __init pkey_api_init(void)
+{
+	/* register as a misc device */
+	return misc_register(&pkey_dev);
+}
+
+void __exit pkey_api_exit(void)
+{
+	misc_deregister(&pkey_dev);
+}
diff --git a/drivers/s390/crypto/pkey_base.c b/drivers/s390/crypto/pkey_base.c
new file mode 100644
index 000000000..c5677e1e1
--- /dev/null
+++ b/drivers/s390/crypto/pkey_base.c
@@ -0,0 +1,387 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  pkey base: debug feature, pkey handler registry
+ *
+ *  Copyright IBM Corp. 2024
+ */
+
+#define pr_fmt(fmt) "pkey: " fmt
+
+#include <linux/cpufeature.h>
+#include <linux/export.h>
+#include <linux/init.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/rculist.h>
+
+#include "pkey_base.h"
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("IBM Corporation");
+MODULE_DESCRIPTION("s390 protected key base and api");
+
+/*
+ * pkey debug feature
+ */
+debug_info_t *pkey_dbf_info;
+EXPORT_SYMBOL(pkey_dbf_info);
+
+/*
+ * pkey handler registry
+ */
+
+static DEFINE_SPINLOCK(handler_list_write_lock);
+static LIST_HEAD(handler_list);
+
+int pkey_handler_register(struct pkey_handler *handler)
+{
+	const struct pkey_handler *h;
+
+	if (!handler ||
+	    !handler->is_supported_key ||
+	    !handler->is_supported_keytype)
+		return -EINVAL;
+
+	if (!try_module_get(handler->module))
+		return -ENXIO;
+
+	spin_lock(&handler_list_write_lock);
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(h, &handler_list, list) {
+		if (h == handler) {
+			rcu_read_unlock();
+			spin_unlock(&handler_list_write_lock);
+			module_put(handler->module);
+			return -EEXIST;
+		}
+	}
+	rcu_read_unlock();
+
+	list_add_rcu(&handler->list, &handler_list);
+	spin_unlock(&handler_list_write_lock);
+	/*
+	 * Fast path to push the info about the updated list to the other
+	 * CPUs. If removed, the other CPUs may get the updated list when the
+	 * RCU context is synched. As this code is in general not performance
+	 * critical and the list update mostly only occurs at the early time in
+	 * system startup the focus is on concurrency versus performance.
+	 */
+	synchronize_rcu();
+
+	module_put(handler->module);
+
+	PKEY_DBF_INFO("%s pkey handler '%s' registered\n", __func__,
+		      handler->name ?: "<no name>");
+
+	return 0;
+}
+EXPORT_SYMBOL(pkey_handler_register);
+
+int pkey_handler_unregister(struct pkey_handler *handler)
+{
+	spin_lock(&handler_list_write_lock);
+	list_del_rcu(&handler->list);
+	INIT_LIST_HEAD_RCU(&handler->list);
+	spin_unlock(&handler_list_write_lock);
+	synchronize_rcu();
+
+	PKEY_DBF_INFO("%s pkey handler '%s' unregistered\n", __func__,
+		      handler->name ?: "<no name>");
+
+	return 0;
+}
+EXPORT_SYMBOL(pkey_handler_unregister);
+
+/*
+ * Handler invocation functions.
+ */
+
+const struct pkey_handler *pkey_handler_get_keybased(const u8 *key, u32 keylen)
+{
+	const struct pkey_handler *h;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(h, &handler_list, list) {
+		if (!try_module_get(h->module))
+			continue;
+		if (h->is_supported_key(key, keylen)) {
+			rcu_read_unlock();
+			return h;
+		}
+		module_put(h->module);
+	}
+	rcu_read_unlock();
+
+	return NULL;
+}
+EXPORT_SYMBOL(pkey_handler_get_keybased);
+
+const struct pkey_handler *pkey_handler_get_keytypebased(enum pkey_key_type kt)
+{
+	const struct pkey_handler *h;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(h, &handler_list, list) {
+		if (!try_module_get(h->module))
+			continue;
+		if (h->is_supported_keytype(kt)) {
+			rcu_read_unlock();
+			return h;
+		}
+		module_put(h->module);
+	}
+	rcu_read_unlock();
+
+	return NULL;
+}
+EXPORT_SYMBOL(pkey_handler_get_keytypebased);
+
+void pkey_handler_put(const struct pkey_handler *handler)
+{
+	const struct pkey_handler *h;
+
+	if (!handler)
+		return;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(h, &handler_list, list) {
+		if (h == handler) {
+			module_put(h->module);
+			break;
+		}
+	}
+	rcu_read_unlock();
+}
+EXPORT_SYMBOL(pkey_handler_put);
+
+int pkey_handler_key_to_protkey(const struct pkey_apqn *apqns, size_t nr_apqns,
+				const u8 *key, u32 keylen,
+				u8 *protkey, u32 *protkeylen, u32 *protkeytype,
+				u32 xflags)
+{
+	const struct pkey_handler *h;
+	int rc = -ENODEV;
+
+	h = pkey_handler_get_keybased(key, keylen);
+	if (h && h->key_to_protkey) {
+		rc = h->key_to_protkey(apqns, nr_apqns, key, keylen,
+				       protkey, protkeylen,
+				       protkeytype, xflags);
+	}
+	pkey_handler_put(h);
+
+	return rc;
+}
+EXPORT_SYMBOL(pkey_handler_key_to_protkey);
+
+/*
+ * This handler invocation is special as there may be more than
+ * one handler providing support for the very same key (type).
+ * And the handler may not respond true on is_supported_key(),
+ * so simple try and check return value here.
+ */
+int pkey_handler_slowpath_key_to_protkey(const struct pkey_apqn *apqns,
+					 size_t nr_apqns,
+					 const u8 *key, u32 keylen,
+					 u8 *protkey, u32 *protkeylen,
+					 u32 *protkeytype, u32 xflags)
+{
+	const struct pkey_handler *h, *htmp[10];
+	int i, n = 0, rc = -ENODEV;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(h, &handler_list, list) {
+		if (!try_module_get(h->module))
+			continue;
+		if (h->slowpath_key_to_protkey && n < ARRAY_SIZE(htmp))
+			htmp[n++] = h;
+		else
+			module_put(h->module);
+	}
+	rcu_read_unlock();
+
+	for (i = 0; i < n; i++) {
+		h = htmp[i];
+		if (rc)
+			rc = h->slowpath_key_to_protkey(apqns, nr_apqns,
+							key, keylen,
+							protkey, protkeylen,
+							protkeytype, xflags);
+		module_put(h->module);
+	}
+
+	return rc;
+}
+EXPORT_SYMBOL(pkey_handler_slowpath_key_to_protkey);
+
+int pkey_handler_gen_key(const struct pkey_apqn *apqns, size_t nr_apqns,
+			 u32 keytype, u32 keysubtype,
+			 u32 keybitsize, u32 flags,
+			 u8 *keybuf, u32 *keybuflen, u32 *keyinfo, u32 xflags)
+{
+	const struct pkey_handler *h;
+	int rc = -ENODEV;
+
+	h = pkey_handler_get_keytypebased(keysubtype);
+	if (h && h->gen_key) {
+		rc = h->gen_key(apqns, nr_apqns, keytype, keysubtype,
+				keybitsize, flags,
+				keybuf, keybuflen, keyinfo, xflags);
+	}
+	pkey_handler_put(h);
+
+	return rc;
+}
+EXPORT_SYMBOL(pkey_handler_gen_key);
+
+int pkey_handler_clr_to_key(const struct pkey_apqn *apqns, size_t nr_apqns,
+			    u32 keytype, u32 keysubtype,
+			    u32 keybitsize, u32 flags,
+			    const u8 *clrkey, u32 clrkeylen,
+			    u8 *keybuf, u32 *keybuflen, u32 *keyinfo,
+			    u32 xflags)
+{
+	const struct pkey_handler *h;
+	int rc = -ENODEV;
+
+	h = pkey_handler_get_keytypebased(keysubtype);
+	if (h && h->clr_to_key) {
+		rc = h->clr_to_key(apqns, nr_apqns, keytype, keysubtype,
+				   keybitsize, flags, clrkey, clrkeylen,
+				   keybuf, keybuflen, keyinfo, xflags);
+	}
+	pkey_handler_put(h);
+
+	return rc;
+}
+EXPORT_SYMBOL(pkey_handler_clr_to_key);
+
+int pkey_handler_verify_key(const u8 *key, u32 keylen,
+			    u16 *card, u16 *dom,
+			    u32 *keytype, u32 *keybitsize, u32 *flags,
+			    u32 xflags)
+{
+	const struct pkey_handler *h;
+	int rc = -ENODEV;
+
+	h = pkey_handler_get_keybased(key, keylen);
+	if (h && h->verify_key) {
+		rc = h->verify_key(key, keylen, card, dom,
+				   keytype, keybitsize, flags, xflags);
+	}
+	pkey_handler_put(h);
+
+	return rc;
+}
+EXPORT_SYMBOL(pkey_handler_verify_key);
+
+int pkey_handler_apqns_for_key(const u8 *key, u32 keylen, u32 flags,
+			       struct pkey_apqn *apqns, size_t *nr_apqns,
+			       u32 xflags)
+{
+	const struct pkey_handler *h;
+	int rc = -ENODEV;
+
+	h = pkey_handler_get_keybased(key, keylen);
+	if (h && h->apqns_for_key)
+		rc = h->apqns_for_key(key, keylen, flags, apqns, nr_apqns,
+				      xflags);
+	pkey_handler_put(h);
+
+	return rc;
+}
+EXPORT_SYMBOL(pkey_handler_apqns_for_key);
+
+int pkey_handler_apqns_for_keytype(enum pkey_key_type keysubtype,
+				   u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags,
+				   struct pkey_apqn *apqns, size_t *nr_apqns,
+				   u32 xflags)
+{
+	const struct pkey_handler *h;
+	int rc = -ENODEV;
+
+	h = pkey_handler_get_keytypebased(keysubtype);
+	if (h && h->apqns_for_keytype) {
+		rc = h->apqns_for_keytype(keysubtype,
+					  cur_mkvp, alt_mkvp, flags,
+					  apqns, nr_apqns, xflags);
+	}
+	pkey_handler_put(h);
+
+	return rc;
+}
+EXPORT_SYMBOL(pkey_handler_apqns_for_keytype);
+
+void pkey_handler_request_modules(void)
+{
+#ifdef CONFIG_MODULES
+	static const char * const pkey_handler_modules[] = {
+#if IS_MODULE(CONFIG_PKEY_CCA)
+		"pkey_cca",
+#endif
+#if IS_MODULE(CONFIG_PKEY_EP11)
+		"pkey_ep11",
+#endif
+#if IS_MODULE(CONFIG_PKEY_PCKMO)
+		"pkey_pckmo",
+#endif
+#if IS_MODULE(CONFIG_PKEY_UV)
+		"pkey_uv",
+#endif
+	};
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(pkey_handler_modules); i++) {
+		const struct pkey_handler *h;
+		bool found = false;
+
+		rcu_read_lock();
+		list_for_each_entry_rcu(h, &handler_list, list) {
+			if (h->module &&
+			    !strcmp(h->module->name, pkey_handler_modules[i])) {
+				found = true;
+				break;
+			}
+		}
+		rcu_read_unlock();
+		if (!found) {
+			pr_debug("request_module(%s)\n", pkey_handler_modules[i]);
+			request_module(pkey_handler_modules[i]);
+		}
+	}
+#endif
+}
+EXPORT_SYMBOL(pkey_handler_request_modules);
+
+/*
+ * Module init
+ */
+static int __init pkey_init(void)
+{
+	int rc;
+
+	/* init debug feature */
+	pkey_dbf_info = debug_register("pkey", 1, 1, 5 * sizeof(long));
+	debug_register_view(pkey_dbf_info, &debug_sprintf_view);
+	debug_set_level(pkey_dbf_info, 4);
+
+	/* the handler registry does not need any init */
+
+	rc = pkey_api_init();
+	if (rc)
+		debug_unregister(pkey_dbf_info);
+
+	return rc;
+}
+
+/*
+ * Module exit
+ */
+static void __exit pkey_exit(void)
+{
+	pkey_api_exit();
+}
+
+module_cpu_feature_match(S390_CPU_FEATURE_MSA, pkey_init);
+module_exit(pkey_exit);
diff --git a/drivers/s390/crypto/pkey_base.h b/drivers/s390/crypto/pkey_base.h
new file mode 100644
index 000000000..9cdb3e744
--- /dev/null
+++ b/drivers/s390/crypto/pkey_base.h
@@ -0,0 +1,240 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright IBM Corp. 2024
+ *
+ * Pkey base: debug feature, defines and structs
+ * common to all pkey code.
+ */
+
+#ifndef _PKEY_BASE_H_
+#define _PKEY_BASE_H_
+
+#include <linux/types.h>
+#include <asm/debug.h>
+#include <asm/pkey.h>
+
+/*
+ * pkey debug feature
+ */
+
+extern debug_info_t *pkey_dbf_info;
+
+#define PKEY_DBF_INFO(...) debug_sprintf_event(pkey_dbf_info, 5, ##__VA_ARGS__)
+#define PKEY_DBF_WARN(...) debug_sprintf_event(pkey_dbf_info, 4, ##__VA_ARGS__)
+#define PKEY_DBF_ERR(...)  debug_sprintf_event(pkey_dbf_info, 3, ##__VA_ARGS__)
+
+/*
+ * common defines and common structs
+ */
+
+#define KEYBLOBBUFSIZE 8192	/* key buffer size used for internal processing */
+#define MINKEYBLOBBUFSIZE (sizeof(struct keytoken_header))
+#define PROTKEYBLOBBUFSIZE 256	/* protected key buffer size used internal */
+#define MAXAPQNSINLIST 64	/* max 64 apqns within a apqn list */
+#define AES_WK_VP_SIZE 32	/* Size of WK VP block appended to a prot key */
+
+/* inside view of a generic protected key token */
+struct protkeytoken {
+	u8  type;     /* 0x00 for PAES specific key tokens */
+	u8  res0[3];
+	u8  version;  /* should be 0x01 for protected key token */
+	u8  res1[3];
+	u32 keytype;  /* key type, one of the PKEY_KEYTYPE values */
+	u32 len;      /* bytes actually stored in protkey[] */
+	u8  protkey[]; /* the protected key blob */
+} __packed;
+
+/* inside view of a protected AES key token */
+struct protaeskeytoken {
+	u8  type;     /* 0x00 for PAES specific key tokens */
+	u8  res0[3];
+	u8  version;  /* should be 0x01 for protected key token */
+	u8  res1[3];
+	u32 keytype;  /* key type, one of the PKEY_KEYTYPE values */
+	u32 len;      /* bytes actually stored in protkey[] */
+	u8  protkey[MAXPROTKEYSIZE]; /* the protected key blob */
+} __packed;
+
+/* inside view of a clear key token (type 0x00 version 0x02) */
+struct clearkeytoken {
+	u8  type;	/* 0x00 for PAES specific key tokens */
+	u8  res0[3];
+	u8  version;	/* 0x02 for clear key token */
+	u8  res1[3];
+	u32 keytype;	/* key type, one of the PKEY_KEYTYPE_* values */
+	u32 len;	/* bytes actually stored in clearkey[] */
+	u8  clearkey[]; /* clear key value */
+} __packed;
+
+/* helper function which translates the PKEY_KEYTYPE_AES_* to their keysize */
+static inline u32 pkey_keytype_aes_to_size(u32 keytype)
+{
+	switch (keytype) {
+	case PKEY_KEYTYPE_AES_128:
+		return 16;
+	case PKEY_KEYTYPE_AES_192:
+		return 24;
+	case PKEY_KEYTYPE_AES_256:
+		return 32;
+	default:
+		return 0;
+	}
+}
+
+/* helper function which translates AES key bit size into PKEY_KEYTYPE_AES_* */
+static inline u32 pkey_aes_bitsize_to_keytype(u32 keybitsize)
+{
+	switch (keybitsize) {
+	case 128:
+		return PKEY_KEYTYPE_AES_128;
+	case 192:
+		return PKEY_KEYTYPE_AES_192;
+	case 256:
+		return PKEY_KEYTYPE_AES_256;
+	default:
+		return 0;
+	}
+}
+
+/*
+ * helper function which translates the PKEY_KEYTYPE_*
+ * to the protected key size minus the WK VP length
+ */
+static inline u32 pkey_keytype_to_size(u32 keytype)
+{
+	switch (keytype) {
+	case PKEY_KEYTYPE_AES_128:
+		return 16;
+	case PKEY_KEYTYPE_AES_192:
+		return 24;
+	case PKEY_KEYTYPE_AES_256:
+		return 32;
+	case PKEY_KEYTYPE_ECC_P256:
+		return 32;
+	case PKEY_KEYTYPE_ECC_P384:
+		return 48;
+	case PKEY_KEYTYPE_ECC_P521:
+		return 80;
+	case PKEY_KEYTYPE_ECC_ED25519:
+		return 32;
+	case PKEY_KEYTYPE_ECC_ED448:
+		return 54;
+	case PKEY_KEYTYPE_AES_XTS_128:
+		return 32;
+	case PKEY_KEYTYPE_AES_XTS_256:
+		return 64;
+	case PKEY_KEYTYPE_HMAC_512:
+		return 64;
+	case PKEY_KEYTYPE_HMAC_1024:
+		return 128;
+	default:
+		return 0;
+	}
+}
+
+/*
+ * pkey_api.c:
+ */
+int __init pkey_api_init(void);
+void __exit pkey_api_exit(void);
+
+/*
+ * pkey_sysfs.c:
+ */
+
+extern const struct attribute_group *pkey_attr_groups[];
+
+/*
+ * pkey handler registry
+ */
+
+struct pkey_handler {
+	struct module *module;
+	const char *name;
+	/*
+	 * is_supported_key() and is_supported_keytype() are called
+	 * within an rcu_read_lock() scope and thus must not sleep!
+	 */
+	bool (*is_supported_key)(const u8 *key, u32 keylen);
+	bool (*is_supported_keytype)(enum pkey_key_type);
+	int (*key_to_protkey)(const struct pkey_apqn *apqns, size_t nr_apqns,
+			      const u8 *key, u32 keylen,
+			      u8 *protkey, u32 *protkeylen, u32 *protkeytype,
+			      u32 xflags);
+	int (*slowpath_key_to_protkey)(const struct pkey_apqn *apqns,
+				       size_t nr_apqns,
+				       const u8 *key, u32 keylen,
+				       u8 *protkey, u32 *protkeylen,
+				       u32 *protkeytype, u32 xflags);
+	int (*gen_key)(const struct pkey_apqn *apqns, size_t nr_apqns,
+		       u32 keytype, u32 keysubtype,
+		       u32 keybitsize, u32 flags,
+		       u8 *keybuf, u32 *keybuflen, u32 *keyinfo, u32 xflags);
+	int (*clr_to_key)(const struct pkey_apqn *apqns, size_t nr_apqns,
+			  u32 keytype, u32 keysubtype,
+			  u32 keybitsize, u32 flags,
+			  const u8 *clrkey, u32 clrkeylen,
+			  u8 *keybuf, u32 *keybuflen, u32 *keyinfo, u32 xflags);
+	int (*verify_key)(const u8 *key, u32 keylen,
+			  u16 *card, u16 *dom,
+			  u32 *keytype, u32 *keybitsize, u32 *flags,
+			  u32 xflags);
+	int (*apqns_for_key)(const u8 *key, u32 keylen, u32 flags,
+			     struct pkey_apqn *apqns, size_t *nr_apqns,
+			     u32 xflags);
+	int (*apqns_for_keytype)(enum pkey_key_type ktype,
+				 u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags,
+				 struct pkey_apqn *apqns, size_t *nr_apqns,
+				 u32 xflags);
+	/* used internal by pkey base */
+	struct list_head list;
+};
+
+int pkey_handler_register(struct pkey_handler *handler);
+int pkey_handler_unregister(struct pkey_handler *handler);
+
+/*
+ * invocation function for the registered pkey handlers
+ */
+
+const struct pkey_handler *pkey_handler_get_keybased(const u8 *key, u32 keylen);
+const struct pkey_handler *pkey_handler_get_keytypebased(enum pkey_key_type kt);
+void pkey_handler_put(const struct pkey_handler *handler);
+
+int pkey_handler_key_to_protkey(const struct pkey_apqn *apqns, size_t nr_apqns,
+				const u8 *key, u32 keylen,
+				u8 *protkey, u32 *protkeylen, u32 *protkeytype,
+				u32 xflags);
+int pkey_handler_slowpath_key_to_protkey(const struct pkey_apqn *apqns,
+					 size_t nr_apqns,
+					 const u8 *key, u32 keylen,
+					 u8 *protkey, u32 *protkeylen,
+					 u32 *protkeytype, u32 xflags);
+int pkey_handler_gen_key(const struct pkey_apqn *apqns, size_t nr_apqns,
+			 u32 keytype, u32 keysubtype,
+			 u32 keybitsize, u32 flags,
+			 u8 *keybuf, u32 *keybuflen, u32 *keyinfo, u32 xflags);
+int pkey_handler_clr_to_key(const struct pkey_apqn *apqns, size_t nr_apqns,
+			    u32 keytype, u32 keysubtype,
+			    u32 keybitsize, u32 flags,
+			    const u8 *clrkey, u32 clrkeylen,
+			    u8 *keybuf, u32 *keybuflen, u32 *keyinfo,
+			    u32 xflags);
+int pkey_handler_verify_key(const u8 *key, u32 keylen,
+			    u16 *card, u16 *dom,
+			    u32 *keytype, u32 *keybitsize, u32 *flags,
+			    u32 xflags);
+int pkey_handler_apqns_for_key(const u8 *key, u32 keylen, u32 flags,
+			       struct pkey_apqn *apqns, size_t *nr_apqns,
+			       u32 xflags);
+int pkey_handler_apqns_for_keytype(enum pkey_key_type ktype,
+				   u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags,
+				   struct pkey_apqn *apqns, size_t *nr_apqns,
+				   u32 xflags);
+
+/*
+ * Unconditional try to load all handler modules
+ */
+void pkey_handler_request_modules(void);
+
+#endif /* _PKEY_BASE_H_ */
diff --git a/drivers/s390/crypto/pkey_cca.c b/drivers/s390/crypto/pkey_cca.c
new file mode 100644
index 000000000..e8dc2f77c
--- /dev/null
+++ b/drivers/s390/crypto/pkey_cca.c
@@ -0,0 +1,629 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  pkey cca specific code
+ *
+ *  Copyright IBM Corp. 2024
+ */
+
+#define pr_fmt(fmt) "pkey: " fmt
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/cpufeature.h>
+
+#include "zcrypt_ccamisc.h"
+#include "pkey_base.h"
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("IBM Corporation");
+MODULE_DESCRIPTION("s390 protected key CCA handler");
+
+#if IS_MODULE(CONFIG_PKEY_CCA)
+static struct ap_device_id pkey_cca_card_ids[] = {
+	{ .dev_type = AP_DEVICE_TYPE_CEX4 },
+	{ .dev_type = AP_DEVICE_TYPE_CEX5 },
+	{ .dev_type = AP_DEVICE_TYPE_CEX6 },
+	{ .dev_type = AP_DEVICE_TYPE_CEX7 },
+	{ .dev_type = AP_DEVICE_TYPE_CEX8 },
+	{ /* end of list */ },
+};
+MODULE_DEVICE_TABLE(ap, pkey_cca_card_ids);
+#endif
+
+/*
+ * Check key blob for known and supported CCA key.
+ */
+static bool is_cca_key(const u8 *key, u32 keylen)
+{
+	struct keytoken_header *hdr = (struct keytoken_header *)key;
+
+	if (keylen < sizeof(*hdr))
+		return false;
+
+	switch (hdr->type) {
+	case TOKTYPE_CCA_INTERNAL:
+		switch (hdr->version) {
+		case TOKVER_CCA_AES:
+		case TOKVER_CCA_VLSC:
+			return true;
+		default:
+			return false;
+		}
+	case TOKTYPE_CCA_INTERNAL_PKA:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool is_cca_keytype(enum pkey_key_type key_type)
+{
+	switch (key_type) {
+	case PKEY_TYPE_CCA_DATA:
+	case PKEY_TYPE_CCA_CIPHER:
+	case PKEY_TYPE_CCA_ECC:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static int cca_apqns4key(const u8 *key, u32 keylen, u32 flags,
+			 struct pkey_apqn *apqns, size_t *nr_apqns, u32 pflags)
+{
+	struct keytoken_header *hdr = (struct keytoken_header *)key;
+	u32 _apqns[MAXAPQNSINLIST], _nr_apqns = ARRAY_SIZE(_apqns);
+	u32 xflags;
+	int rc;
+
+	xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0;
+
+	if (!flags)
+		flags = PKEY_FLAGS_MATCH_CUR_MKVP | PKEY_FLAGS_MATCH_ALT_MKVP;
+
+	if (keylen < sizeof(struct keytoken_header))
+		return -EINVAL;
+
+	zcrypt_wait_api_operational();
+
+	if (hdr->type == TOKTYPE_CCA_INTERNAL) {
+		const u8 *ptr_cur_mkvp = NULL;
+		const u8 *ptr_old_mkvp = NULL;
+		int minhwtype = ZCRYPT_CEX3C;
+
+		if (hdr->version == TOKVER_CCA_AES) {
+			struct secaeskeytoken *t = (struct secaeskeytoken *)key;
+
+			if (flags & PKEY_FLAGS_MATCH_CUR_MKVP)
+				ptr_cur_mkvp = t->mkvp;
+			if (flags & PKEY_FLAGS_MATCH_ALT_MKVP)
+				ptr_old_mkvp = t->mkvp;
+		} else if (hdr->version == TOKVER_CCA_VLSC) {
+			struct cipherkeytoken *t = (struct cipherkeytoken *)key;
+
+			minhwtype = ZCRYPT_CEX6;
+			if (flags & PKEY_FLAGS_MATCH_CUR_MKVP)
+				ptr_cur_mkvp = t->mkvp0;
+			if (flags & PKEY_FLAGS_MATCH_ALT_MKVP)
+				ptr_old_mkvp = t->mkvp0;
+		} else {
+			/* unknown CCA internal token type */
+			return -EINVAL;
+		}
+		rc = cca_findcard2(_apqns, &_nr_apqns, 0xFFFF, 0xFFFF,
+				   minhwtype, AES_MK_SET,
+				   ptr_cur_mkvp, ptr_old_mkvp, xflags);
+		if (rc)
+			goto out;
+
+	} else if (hdr->type == TOKTYPE_CCA_INTERNAL_PKA) {
+		struct eccprivkeytoken *t = (struct eccprivkeytoken *)key;
+		const u8 *ptr_cur_mkvp = NULL;
+		const u8 *ptr_old_mkvp = NULL;
+
+		if (t->secid == 0x20) {
+			if (flags & PKEY_FLAGS_MATCH_CUR_MKVP)
+				ptr_cur_mkvp = t->mkvp;
+			if (flags & PKEY_FLAGS_MATCH_ALT_MKVP)
+				ptr_old_mkvp = t->mkvp;
+		} else {
+			/* unknown CCA internal 2 token type */
+			return -EINVAL;
+		}
+		rc = cca_findcard2(_apqns, &_nr_apqns, 0xFFFF, 0xFFFF,
+				   ZCRYPT_CEX7, APKA_MK_SET,
+				   ptr_cur_mkvp, ptr_old_mkvp, xflags);
+		if (rc)
+			goto out;
+
+	} else {
+		PKEY_DBF_ERR("%s unknown/unsupported blob type %d version %d\n",
+			     __func__, hdr->type, hdr->version);
+		return -EINVAL;
+	}
+
+	if (apqns) {
+		if (*nr_apqns < _nr_apqns)
+			rc = -ENOSPC;
+		else
+			memcpy(apqns, _apqns, _nr_apqns * sizeof(u32));
+	}
+	*nr_apqns = _nr_apqns;
+
+out:
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+
+static int cca_apqns4type(enum pkey_key_type ktype,
+			  u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags,
+			  struct pkey_apqn *apqns, size_t *nr_apqns,
+			  u32 pflags)
+{
+	u32 _apqns[MAXAPQNSINLIST], _nr_apqns = ARRAY_SIZE(_apqns);
+	u32 xflags;
+	int rc;
+
+	xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0;
+
+	zcrypt_wait_api_operational();
+
+	if (ktype == PKEY_TYPE_CCA_DATA || ktype == PKEY_TYPE_CCA_CIPHER) {
+		const u8 *ptr_cur_mkvp = NULL;
+		const u8 *ptr_old_mkvp = NULL;
+		int minhwtype = ZCRYPT_CEX3C;
+
+		if (flags & PKEY_FLAGS_MATCH_CUR_MKVP)
+			ptr_cur_mkvp = cur_mkvp;
+		if (flags & PKEY_FLAGS_MATCH_ALT_MKVP)
+			ptr_old_mkvp = alt_mkvp;
+		if (ktype == PKEY_TYPE_CCA_CIPHER)
+			minhwtype = ZCRYPT_CEX6;
+		rc = cca_findcard2(_apqns, &_nr_apqns, 0xFFFF, 0xFFFF,
+				   minhwtype, AES_MK_SET,
+				   ptr_cur_mkvp, ptr_old_mkvp, xflags);
+		if (rc)
+			goto out;
+
+	} else if (ktype == PKEY_TYPE_CCA_ECC) {
+		const u8 *ptr_cur_mkvp = NULL;
+		const u8 *ptr_old_mkvp = NULL;
+
+		if (flags & PKEY_FLAGS_MATCH_CUR_MKVP)
+			ptr_cur_mkvp = cur_mkvp;
+		if (flags & PKEY_FLAGS_MATCH_ALT_MKVP)
+			ptr_old_mkvp = alt_mkvp;
+		rc = cca_findcard2(_apqns, &_nr_apqns, 0xFFFF, 0xFFFF,
+				   ZCRYPT_CEX7, APKA_MK_SET,
+				   ptr_cur_mkvp, ptr_old_mkvp, xflags);
+		if (rc)
+			goto out;
+
+	} else {
+		PKEY_DBF_ERR("%s unknown/unsupported key type %d",
+			     __func__, (int)ktype);
+		return -EINVAL;
+	}
+
+	if (apqns) {
+		if (*nr_apqns < _nr_apqns)
+			rc = -ENOSPC;
+		else
+			memcpy(apqns, _apqns, _nr_apqns * sizeof(u32));
+	}
+	*nr_apqns = _nr_apqns;
+
+out:
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+
+static int cca_key2protkey(const struct pkey_apqn *apqns, size_t nr_apqns,
+			   const u8 *key, u32 keylen,
+			   u8 *protkey, u32 *protkeylen, u32 *protkeytype,
+			   u32 pflags)
+{
+	struct keytoken_header *hdr = (struct keytoken_header *)key;
+	struct pkey_apqn _apqns[MAXAPQNSINLIST];
+	u32 xflags;
+	int i, rc;
+
+	xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0;
+
+	if (keylen < sizeof(*hdr))
+		return -EINVAL;
+
+	if (hdr->type == TOKTYPE_CCA_INTERNAL &&
+	    hdr->version == TOKVER_CCA_AES) {
+		/* CCA AES data key */
+		if (cca_check_secaeskeytoken(pkey_dbf_info, 3, key, keylen, 0))
+			return -EINVAL;
+	} else if (hdr->type == TOKTYPE_CCA_INTERNAL &&
+		   hdr->version == TOKVER_CCA_VLSC) {
+		/* CCA AES cipher key */
+		if (cca_check_secaescipherkey(pkey_dbf_info,
+					      3, key, keylen, 0, 1))
+			return -EINVAL;
+	} else if (hdr->type == TOKTYPE_CCA_INTERNAL_PKA) {
+		/* CCA ECC (private) key */
+		if (cca_check_sececckeytoken(pkey_dbf_info, 3, key, keylen, 1))
+			return -EINVAL;
+	} else {
+		PKEY_DBF_ERR("%s unknown/unsupported blob type %d version %d\n",
+			     __func__, hdr->type, hdr->version);
+		return -EINVAL;
+	}
+
+	zcrypt_wait_api_operational();
+
+	if (!apqns || (nr_apqns == 1 &&
+		       apqns[0].card == 0xFFFF && apqns[0].domain == 0xFFFF)) {
+		nr_apqns = MAXAPQNSINLIST;
+		rc = cca_apqns4key(key, keylen, 0, _apqns, &nr_apqns, pflags);
+		if (rc)
+			goto out;
+		apqns = _apqns;
+	}
+
+	for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) {
+		if (hdr->type == TOKTYPE_CCA_INTERNAL &&
+		    hdr->version == TOKVER_CCA_AES) {
+			rc = cca_sec2protkey(apqns[i].card, apqns[i].domain,
+					     key, protkey,
+					     protkeylen, protkeytype, xflags);
+		} else if (hdr->type == TOKTYPE_CCA_INTERNAL &&
+			   hdr->version == TOKVER_CCA_VLSC) {
+			rc = cca_cipher2protkey(apqns[i].card, apqns[i].domain,
+						key, protkey,
+						protkeylen, protkeytype, xflags);
+		} else if (hdr->type == TOKTYPE_CCA_INTERNAL_PKA) {
+			rc = cca_ecc2protkey(apqns[i].card, apqns[i].domain,
+					     key, protkey,
+					     protkeylen, protkeytype, xflags);
+		} else {
+			rc = -EINVAL;
+			break;
+		}
+	}
+
+out:
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+
+/*
+ * Generate CCA secure key.
+ * As of now only CCA AES Data or Cipher secure keys are
+ * supported.
+ * keytype is one of the PKEY_KEYTYPE_* constants,
+ * subtype may be 0 or PKEY_TYPE_CCA_DATA or PKEY_TYPE_CCA_CIPHER,
+ * keybitsize is the bit size of the key (may be 0 for
+ * keytype PKEY_KEYTYPE_AES_*).
+ */
+static int cca_gen_key(const struct pkey_apqn *apqns, size_t nr_apqns,
+		       u32 keytype, u32 subtype,
+		       u32 keybitsize, u32 flags,
+		       u8 *keybuf, u32 *keybuflen, u32 *_keyinfo, u32 pflags)
+{
+	struct pkey_apqn _apqns[MAXAPQNSINLIST];
+	int i, len, rc;
+	u32 xflags;
+
+	xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0;
+
+	/* check keytype, subtype, keybitsize */
+	switch (keytype) {
+	case PKEY_KEYTYPE_AES_128:
+	case PKEY_KEYTYPE_AES_192:
+	case PKEY_KEYTYPE_AES_256:
+		len = pkey_keytype_aes_to_size(keytype);
+		if (keybitsize && keybitsize != 8 * len) {
+			PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n",
+				     __func__, keybitsize);
+			return -EINVAL;
+		}
+		keybitsize = 8 * len;
+		switch (subtype) {
+		case PKEY_TYPE_CCA_DATA:
+		case PKEY_TYPE_CCA_CIPHER:
+			break;
+		default:
+			PKEY_DBF_ERR("%s unknown/unsupported subtype %d\n",
+				     __func__, subtype);
+			return -EINVAL;
+		}
+		break;
+	default:
+		PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n",
+			     __func__, keytype);
+		return -EINVAL;
+	}
+
+	zcrypt_wait_api_operational();
+
+	if (!apqns || (nr_apqns == 1 &&
+		       apqns[0].card == 0xFFFF && apqns[0].domain == 0xFFFF)) {
+		nr_apqns = MAXAPQNSINLIST;
+		rc = cca_apqns4type(subtype, NULL, NULL, 0,
+				    _apqns, &nr_apqns, pflags);
+		if (rc)
+			goto out;
+		apqns = _apqns;
+	}
+
+	for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) {
+		if (subtype == PKEY_TYPE_CCA_CIPHER) {
+			rc = cca_gencipherkey(apqns[i].card, apqns[i].domain,
+					      keybitsize, flags,
+					      keybuf, keybuflen, xflags);
+		} else {
+			/* PKEY_TYPE_CCA_DATA */
+			rc = cca_genseckey(apqns[i].card, apqns[i].domain,
+					   keybitsize, keybuf, xflags);
+			*keybuflen = (rc ? 0 : SECKEYBLOBSIZE);
+		}
+	}
+
+out:
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+
+/*
+ * Generate CCA secure key with given clear key value.
+ * As of now only CCA AES Data or Cipher secure keys are
+ * supported.
+ * keytype is one of the PKEY_KEYTYPE_* constants,
+ * subtype may be 0 or PKEY_TYPE_CCA_DATA or PKEY_TYPE_CCA_CIPHER,
+ * keybitsize is the bit size of the key (may be 0 for
+ * keytype PKEY_KEYTYPE_AES_*).
+ */
+static int cca_clr2key(const struct pkey_apqn *apqns, size_t nr_apqns,
+		       u32 keytype, u32 subtype,
+		       u32 keybitsize, u32 flags,
+		       const u8 *clrkey, u32 clrkeylen,
+		       u8 *keybuf, u32 *keybuflen, u32 *_keyinfo, u32 pflags)
+{
+	struct pkey_apqn _apqns[MAXAPQNSINLIST];
+	int i, len, rc;
+	u32 xflags;
+
+	if (pflags & PKEY_XFLAG_NOCLEARKEY) {
+		PKEY_DBF_ERR("%s clear key but xflag NOCLEARKEY\n", __func__);
+		return -EINVAL;
+	}
+
+	xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0;
+
+	/* check keytype, subtype, clrkeylen, keybitsize */
+	switch (keytype) {
+	case PKEY_KEYTYPE_AES_128:
+	case PKEY_KEYTYPE_AES_192:
+	case PKEY_KEYTYPE_AES_256:
+		len = pkey_keytype_aes_to_size(keytype);
+		if (keybitsize && keybitsize != 8 * len) {
+			PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n",
+				     __func__, keybitsize);
+			return -EINVAL;
+		}
+		keybitsize = 8 * len;
+		if (clrkeylen != len) {
+			PKEY_DBF_ERR("%s invalid clear key len %d != %d\n",
+				     __func__, clrkeylen, len);
+			return -EINVAL;
+		}
+		switch (subtype) {
+		case PKEY_TYPE_CCA_DATA:
+		case PKEY_TYPE_CCA_CIPHER:
+			break;
+		default:
+			PKEY_DBF_ERR("%s unknown/unsupported subtype %d\n",
+				     __func__, subtype);
+			return -EINVAL;
+		}
+		break;
+	default:
+		PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n",
+			     __func__, keytype);
+		return -EINVAL;
+	}
+
+	zcrypt_wait_api_operational();
+
+	if (!apqns || (nr_apqns == 1 &&
+		       apqns[0].card == 0xFFFF && apqns[0].domain == 0xFFFF)) {
+		nr_apqns = MAXAPQNSINLIST;
+		rc = cca_apqns4type(subtype, NULL, NULL, 0,
+				    _apqns, &nr_apqns, pflags);
+		if (rc)
+			goto out;
+		apqns = _apqns;
+	}
+
+	for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) {
+		if (subtype == PKEY_TYPE_CCA_CIPHER) {
+			rc = cca_clr2cipherkey(apqns[i].card, apqns[i].domain,
+					       keybitsize, flags, clrkey,
+					       keybuf, keybuflen, xflags);
+		} else {
+			/* PKEY_TYPE_CCA_DATA */
+			rc = cca_clr2seckey(apqns[i].card, apqns[i].domain,
+					    keybitsize, clrkey, keybuf, xflags);
+			*keybuflen = (rc ? 0 : SECKEYBLOBSIZE);
+		}
+	}
+
+out:
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+
+static int cca_verifykey(const u8 *key, u32 keylen,
+			 u16 *card, u16 *dom,
+			 u32 *keytype, u32 *keybitsize, u32 *flags, u32 pflags)
+{
+	struct keytoken_header *hdr = (struct keytoken_header *)key;
+	u32 apqns[MAXAPQNSINLIST], nr_apqns = ARRAY_SIZE(apqns);
+	u32 xflags;
+	int rc;
+
+	xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0;
+
+	if (keylen < sizeof(*hdr))
+		return -EINVAL;
+
+	zcrypt_wait_api_operational();
+
+	if (hdr->type == TOKTYPE_CCA_INTERNAL &&
+	    hdr->version == TOKVER_CCA_AES) {
+		struct secaeskeytoken *t = (struct secaeskeytoken *)key;
+
+		rc = cca_check_secaeskeytoken(pkey_dbf_info, 3, key, keylen, 0);
+		if (rc)
+			goto out;
+		*keytype = PKEY_TYPE_CCA_DATA;
+		*keybitsize = t->bitsize;
+		rc = cca_findcard2(apqns, &nr_apqns, *card, *dom,
+				   ZCRYPT_CEX3C, AES_MK_SET,
+				   t->mkvp, NULL, xflags);
+		if (!rc)
+			*flags = PKEY_FLAGS_MATCH_CUR_MKVP;
+		if (rc == -ENODEV) {
+			nr_apqns = ARRAY_SIZE(apqns);
+			rc = cca_findcard2(apqns, &nr_apqns, *card, *dom,
+					   ZCRYPT_CEX3C, AES_MK_SET,
+					   NULL, t->mkvp, xflags);
+			if (!rc)
+				*flags = PKEY_FLAGS_MATCH_ALT_MKVP;
+		}
+		if (rc)
+			goto out;
+
+		*card = ((struct pkey_apqn *)apqns)->card;
+		*dom = ((struct pkey_apqn *)apqns)->domain;
+
+	} else if (hdr->type == TOKTYPE_CCA_INTERNAL &&
+		   hdr->version == TOKVER_CCA_VLSC) {
+		struct cipherkeytoken *t = (struct cipherkeytoken *)key;
+
+		rc = cca_check_secaescipherkey(pkey_dbf_info, 3,
+					       key, keylen, 0, 1);
+		if (rc)
+			goto out;
+		*keytype = PKEY_TYPE_CCA_CIPHER;
+		*keybitsize = PKEY_SIZE_UNKNOWN;
+		if (!t->plfver && t->wpllen == 512)
+			*keybitsize = PKEY_SIZE_AES_128;
+		else if (!t->plfver && t->wpllen == 576)
+			*keybitsize = PKEY_SIZE_AES_192;
+		else if (!t->plfver && t->wpllen == 640)
+			*keybitsize = PKEY_SIZE_AES_256;
+		rc = cca_findcard2(apqns, &nr_apqns, *card, *dom,
+				   ZCRYPT_CEX6, AES_MK_SET,
+				   t->mkvp0, NULL, xflags);
+		if (!rc)
+			*flags = PKEY_FLAGS_MATCH_CUR_MKVP;
+		if (rc == -ENODEV) {
+			nr_apqns = ARRAY_SIZE(apqns);
+			rc = cca_findcard2(apqns, &nr_apqns, *card, *dom,
+					   ZCRYPT_CEX6, AES_MK_SET,
+					   NULL, t->mkvp0, xflags);
+			if (!rc)
+				*flags = PKEY_FLAGS_MATCH_ALT_MKVP;
+		}
+		if (rc)
+			goto out;
+
+		*card = ((struct pkey_apqn *)apqns)->card;
+		*dom = ((struct pkey_apqn *)apqns)->domain;
+
+	} else {
+		/* unknown/unsupported key blob */
+		rc = -EINVAL;
+	}
+
+out:
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+
+/*
+ * This function provides an alternate but usually slow way
+ * to convert a 'clear key token' with AES key material into
+ * a protected key. This is done via an intermediate step
+ * which creates a CCA AES DATA secure key first and then
+ * derives the protected key from this secure key.
+ */
+static int cca_slowpath_key2protkey(const struct pkey_apqn *apqns,
+				    size_t nr_apqns,
+				    const u8 *key, u32 keylen,
+				    u8 *protkey, u32 *protkeylen,
+				    u32 *protkeytype, u32 pflags)
+{
+	const struct keytoken_header *hdr = (const struct keytoken_header *)key;
+	const struct clearkeytoken *t = (const struct clearkeytoken *)key;
+	u8 tmpbuf[SECKEYBLOBSIZE]; /* 64 bytes */
+	u32 tmplen, keysize = 0;
+	int i, rc;
+
+	if (keylen < sizeof(*hdr))
+		return -EINVAL;
+
+	if (hdr->type == TOKTYPE_NON_CCA &&
+	    hdr->version == TOKVER_CLEAR_KEY)
+		keysize = pkey_keytype_aes_to_size(t->keytype);
+	if (!keysize || t->len != keysize)
+		return -EINVAL;
+
+	/* try two times in case of failure */
+	for (i = 0, rc = -ENODEV; i < 2 && rc; i++) {
+		tmplen = SECKEYBLOBSIZE;
+		rc = cca_clr2key(NULL, 0, t->keytype, PKEY_TYPE_CCA_DATA,
+				 8 * keysize, 0, t->clearkey, t->len,
+				 tmpbuf, &tmplen, NULL, pflags);
+		pr_debug("cca_clr2key()=%d\n", rc);
+		if (rc)
+			continue;
+		rc = cca_key2protkey(NULL, 0, tmpbuf, tmplen,
+				     protkey, protkeylen, protkeytype, pflags);
+		pr_debug("cca_key2protkey()=%d\n", rc);
+	}
+
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+
+static struct pkey_handler cca_handler = {
+	.module			 = THIS_MODULE,
+	.name			 = "PKEY CCA handler",
+	.is_supported_key	 = is_cca_key,
+	.is_supported_keytype	 = is_cca_keytype,
+	.key_to_protkey		 = cca_key2protkey,
+	.slowpath_key_to_protkey = cca_slowpath_key2protkey,
+	.gen_key		 = cca_gen_key,
+	.clr_to_key		 = cca_clr2key,
+	.verify_key		 = cca_verifykey,
+	.apqns_for_key		 = cca_apqns4key,
+	.apqns_for_keytype	 = cca_apqns4type,
+};
+
+/*
+ * Module init
+ */
+static int __init pkey_cca_init(void)
+{
+	/* register this module as pkey handler for all the cca stuff */
+	return pkey_handler_register(&cca_handler);
+}
+
+/*
+ * Module exit
+ */
+static void __exit pkey_cca_exit(void)
+{
+	/* unregister this module as pkey handler */
+	pkey_handler_unregister(&cca_handler);
+}
+
+module_init(pkey_cca_init);
+module_exit(pkey_cca_exit);
diff --git a/drivers/s390/crypto/pkey_ep11.c b/drivers/s390/crypto/pkey_ep11.c
new file mode 100644
index 000000000..f99c19323
--- /dev/null
+++ b/drivers/s390/crypto/pkey_ep11.c
@@ -0,0 +1,576 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  pkey ep11 specific code
+ *
+ *  Copyright IBM Corp. 2024
+ */
+
+#define pr_fmt(fmt) "pkey: " fmt
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/cpufeature.h>
+
+#include "zcrypt_ccamisc.h"
+#include "zcrypt_ep11misc.h"
+#include "pkey_base.h"
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("IBM Corporation");
+MODULE_DESCRIPTION("s390 protected key EP11 handler");
+
+#if IS_MODULE(CONFIG_PKEY_EP11)
+static struct ap_device_id pkey_ep11_card_ids[] = {
+	{ .dev_type = AP_DEVICE_TYPE_CEX4 },
+	{ .dev_type = AP_DEVICE_TYPE_CEX5 },
+	{ .dev_type = AP_DEVICE_TYPE_CEX6 },
+	{ .dev_type = AP_DEVICE_TYPE_CEX7 },
+	{ .dev_type = AP_DEVICE_TYPE_CEX8 },
+	{ /* end of list */ },
+};
+MODULE_DEVICE_TABLE(ap, pkey_ep11_card_ids);
+#endif
+
+/*
+ * Check key blob for known and supported EP11 key.
+ */
+static bool is_ep11_key(const u8 *key, u32 keylen)
+{
+	struct keytoken_header *hdr = (struct keytoken_header *)key;
+
+	if (keylen < sizeof(*hdr))
+		return false;
+
+	switch (hdr->type) {
+	case TOKTYPE_NON_CCA:
+		switch (hdr->version) {
+		case TOKVER_EP11_AES:
+		case TOKVER_EP11_AES_WITH_HEADER:
+		case TOKVER_EP11_ECC_WITH_HEADER:
+			return true;
+		default:
+			return false;
+		}
+	default:
+		return false;
+	}
+}
+
+static bool is_ep11_keytype(enum pkey_key_type key_type)
+{
+	switch (key_type) {
+	case PKEY_TYPE_EP11:
+	case PKEY_TYPE_EP11_AES:
+	case PKEY_TYPE_EP11_ECC:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static int ep11_apqns4key(const u8 *key, u32 keylen, u32 flags,
+			  struct pkey_apqn *apqns, size_t *nr_apqns, u32 pflags)
+{
+	struct keytoken_header *hdr = (struct keytoken_header *)key;
+	u32 _apqns[MAXAPQNSINLIST], _nr_apqns = ARRAY_SIZE(_apqns);
+	u32 xflags;
+	int rc;
+
+	xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0;
+
+	if (!flags)
+		flags = PKEY_FLAGS_MATCH_CUR_MKVP;
+
+	if (keylen < sizeof(struct keytoken_header) || flags == 0)
+		return -EINVAL;
+
+	zcrypt_wait_api_operational();
+
+	if (hdr->type == TOKTYPE_NON_CCA &&
+	    (hdr->version == TOKVER_EP11_AES_WITH_HEADER ||
+	     hdr->version == TOKVER_EP11_ECC_WITH_HEADER) &&
+	    is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) {
+		struct ep11keyblob *kb = (struct ep11keyblob *)
+			(key + sizeof(struct ep11kblob_header));
+		int minhwtype = 0, api = 0;
+
+		if (flags != PKEY_FLAGS_MATCH_CUR_MKVP)
+			return -EINVAL;
+		if (kb->attr & EP11_BLOB_PKEY_EXTRACTABLE) {
+			minhwtype = ZCRYPT_CEX7;
+			api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4;
+		}
+		rc = ep11_findcard2(_apqns, &_nr_apqns, 0xFFFF, 0xFFFF,
+				    minhwtype, api, kb->wkvp, xflags);
+		if (rc)
+			goto out;
+
+	} else if (hdr->type == TOKTYPE_NON_CCA &&
+		   hdr->version == TOKVER_EP11_AES &&
+		   is_ep11_keyblob(key)) {
+		struct ep11keyblob *kb = (struct ep11keyblob *)key;
+		int minhwtype = 0, api = 0;
+
+		if (flags != PKEY_FLAGS_MATCH_CUR_MKVP)
+			return -EINVAL;
+		if (kb->attr & EP11_BLOB_PKEY_EXTRACTABLE) {
+			minhwtype = ZCRYPT_CEX7;
+			api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4;
+		}
+		rc = ep11_findcard2(_apqns, &_nr_apqns, 0xFFFF, 0xFFFF,
+				    minhwtype, api, kb->wkvp, xflags);
+		if (rc)
+			goto out;
+
+	} else {
+		PKEY_DBF_ERR("%s unknown/unsupported blob type %d version %d\n",
+			     __func__, hdr->type, hdr->version);
+		return -EINVAL;
+	}
+
+	if (apqns) {
+		if (*nr_apqns < _nr_apqns)
+			rc = -ENOSPC;
+		else
+			memcpy(apqns, _apqns, _nr_apqns * sizeof(u32));
+	}
+	*nr_apqns = _nr_apqns;
+
+out:
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+
+static int ep11_apqns4type(enum pkey_key_type ktype,
+			   u8 cur_mkvp[32], u8 alt_mkvp[32], u32 flags,
+			   struct pkey_apqn *apqns, size_t *nr_apqns, u32 pflags)
+{
+	u32 _apqns[MAXAPQNSINLIST], _nr_apqns = ARRAY_SIZE(_apqns);
+	u32 xflags;
+	int rc;
+
+	xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0;
+
+	zcrypt_wait_api_operational();
+
+	if (ktype == PKEY_TYPE_EP11 ||
+	    ktype == PKEY_TYPE_EP11_AES ||
+	    ktype == PKEY_TYPE_EP11_ECC) {
+		u8 *wkvp = NULL;
+		int api;
+
+		if (flags & PKEY_FLAGS_MATCH_CUR_MKVP)
+			wkvp = cur_mkvp;
+		api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4;
+		rc = ep11_findcard2(_apqns, &_nr_apqns, 0xFFFF, 0xFFFF,
+				    ZCRYPT_CEX7, api, wkvp, xflags);
+		if (rc)
+			goto out;
+
+	} else {
+		PKEY_DBF_ERR("%s unknown/unsupported key type %d\n",
+			     __func__, (int)ktype);
+		return -EINVAL;
+	}
+
+	if (apqns) {
+		if (*nr_apqns < _nr_apqns)
+			rc = -ENOSPC;
+		else
+			memcpy(apqns, _apqns, _nr_apqns * sizeof(u32));
+	}
+	*nr_apqns = _nr_apqns;
+
+out:
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+
+static int ep11_key2protkey(const struct pkey_apqn *apqns, size_t nr_apqns,
+			    const u8 *key, u32 keylen,
+			    u8 *protkey, u32 *protkeylen, u32 *protkeytype,
+			    u32 pflags)
+{
+	struct keytoken_header *hdr = (struct keytoken_header *)key;
+	struct pkey_apqn _apqns[MAXAPQNSINLIST];
+	u32 xflags;
+	int i, rc;
+
+	xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0;
+
+	if (keylen < sizeof(*hdr))
+		return -EINVAL;
+
+	if (hdr->type == TOKTYPE_NON_CCA &&
+	    hdr->version == TOKVER_EP11_AES_WITH_HEADER &&
+	    is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) {
+		/* EP11 AES key blob with header */
+		if (ep11_check_aes_key_with_hdr(pkey_dbf_info,
+						3, key, keylen, 1))
+			return -EINVAL;
+	} else if (hdr->type == TOKTYPE_NON_CCA &&
+		   hdr->version == TOKVER_EP11_ECC_WITH_HEADER &&
+		   is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) {
+		/* EP11 ECC key blob with header */
+		if (ep11_check_ecc_key_with_hdr(pkey_dbf_info,
+						3, key, keylen, 1))
+			return -EINVAL;
+	} else if (hdr->type == TOKTYPE_NON_CCA &&
+		   hdr->version == TOKVER_EP11_AES &&
+		   is_ep11_keyblob(key)) {
+		/* EP11 AES key blob with header in session field */
+		if (ep11_check_aes_key(pkey_dbf_info, 3, key, keylen, 1))
+			return -EINVAL;
+	} else {
+		PKEY_DBF_ERR("%s unknown/unsupported blob type %d version %d\n",
+			     __func__, hdr->type, hdr->version);
+		return -EINVAL;
+	}
+
+	zcrypt_wait_api_operational();
+
+	if (!apqns || (nr_apqns == 1 &&
+		       apqns[0].card == 0xFFFF && apqns[0].domain == 0xFFFF)) {
+		nr_apqns = MAXAPQNSINLIST;
+		rc = ep11_apqns4key(key, keylen, 0, _apqns, &nr_apqns, pflags);
+		if (rc)
+			goto out;
+		apqns = _apqns;
+	}
+
+	for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) {
+		if (hdr->type == TOKTYPE_NON_CCA &&
+		    hdr->version == TOKVER_EP11_AES_WITH_HEADER &&
+		    is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) {
+			rc = ep11_kblob2protkey(apqns[i].card, apqns[i].domain,
+						key, hdr->len, protkey,
+						protkeylen, protkeytype, xflags);
+		} else if (hdr->type == TOKTYPE_NON_CCA &&
+			   hdr->version == TOKVER_EP11_ECC_WITH_HEADER &&
+			   is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) {
+			rc = ep11_kblob2protkey(apqns[i].card, apqns[i].domain,
+						key, hdr->len, protkey,
+						protkeylen, protkeytype, xflags);
+		} else if (hdr->type == TOKTYPE_NON_CCA &&
+			   hdr->version == TOKVER_EP11_AES &&
+			   is_ep11_keyblob(key)) {
+			rc = ep11_kblob2protkey(apqns[i].card, apqns[i].domain,
+						key, hdr->len, protkey,
+						protkeylen, protkeytype, xflags);
+		} else {
+			rc = -EINVAL;
+			break;
+		}
+	}
+
+out:
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+
+/*
+ * Generate EP11 secure key.
+ * As of now only EP11 AES secure keys are supported.
+ * keytype is one of the PKEY_KEYTYPE_* constants,
+ * subtype may be PKEY_TYPE_EP11 or PKEY_TYPE_EP11_AES
+ * or 0 (results in subtype PKEY_TYPE_EP11_AES),
+ * keybitsize is the bit size of the key (may be 0 for
+ * keytype PKEY_KEYTYPE_AES_*).
+ */
+static int ep11_gen_key(const struct pkey_apqn *apqns, size_t nr_apqns,
+			u32 keytype, u32 subtype,
+			u32 keybitsize, u32 flags,
+			u8 *keybuf, u32 *keybuflen, u32 *_keyinfo, u32 pflags)
+{
+	struct pkey_apqn _apqns[MAXAPQNSINLIST];
+	int i, len, rc;
+	u32 xflags;
+
+	xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0;
+
+	/* check keytype, subtype, keybitsize */
+	switch (keytype) {
+	case PKEY_KEYTYPE_AES_128:
+	case PKEY_KEYTYPE_AES_192:
+	case PKEY_KEYTYPE_AES_256:
+		len = pkey_keytype_aes_to_size(keytype);
+		if (keybitsize && keybitsize != 8 * len) {
+			PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n",
+				     __func__, keybitsize);
+			return -EINVAL;
+		}
+		keybitsize = 8 * len;
+		switch (subtype) {
+		case PKEY_TYPE_EP11:
+		case PKEY_TYPE_EP11_AES:
+			break;
+		default:
+			PKEY_DBF_ERR("%s unknown/unsupported subtype %d\n",
+				     __func__, subtype);
+			return -EINVAL;
+		}
+		break;
+	default:
+		PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n",
+			     __func__, keytype);
+		return -EINVAL;
+	}
+
+	zcrypt_wait_api_operational();
+
+	if (!apqns || (nr_apqns == 1 &&
+		       apqns[0].card == 0xFFFF && apqns[0].domain == 0xFFFF)) {
+		nr_apqns = MAXAPQNSINLIST;
+		rc = ep11_apqns4type(subtype, NULL, NULL, 0,
+				     _apqns, &nr_apqns, pflags);
+		if (rc)
+			goto out;
+		apqns = _apqns;
+	}
+
+	for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) {
+		rc = ep11_genaeskey(apqns[i].card, apqns[i].domain,
+				    keybitsize, flags,
+				    keybuf, keybuflen, subtype, xflags);
+	}
+
+out:
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+
+/*
+ * Generate EP11 secure key with given clear key value.
+ * As of now only EP11 AES secure keys are supported.
+ * keytype is one of the PKEY_KEYTYPE_* constants,
+ * subtype may be PKEY_TYPE_EP11 or PKEY_TYPE_EP11_AES
+ * or 0 (assumes PKEY_TYPE_EP11_AES then).
+ * keybitsize is the bit size of the key (may be 0 for
+ * keytype PKEY_KEYTYPE_AES_*).
+ */
+static int ep11_clr2key(const struct pkey_apqn *apqns, size_t nr_apqns,
+			u32 keytype, u32 subtype,
+			u32 keybitsize, u32 flags,
+			const u8 *clrkey, u32 clrkeylen,
+			u8 *keybuf, u32 *keybuflen, u32 *_keyinfo, u32 pflags)
+{
+	struct pkey_apqn _apqns[MAXAPQNSINLIST];
+	int i, len, rc;
+	u32 xflags;
+
+	if (pflags & PKEY_XFLAG_NOCLEARKEY) {
+		PKEY_DBF_ERR("%s clear key but xflag NOCLEARKEY\n", __func__);
+		return -EINVAL;
+	}
+
+	xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0;
+
+	/* check keytype, subtype, clrkeylen, keybitsize */
+	switch (keytype) {
+	case PKEY_KEYTYPE_AES_128:
+	case PKEY_KEYTYPE_AES_192:
+	case PKEY_KEYTYPE_AES_256:
+		len = pkey_keytype_aes_to_size(keytype);
+		if (keybitsize && keybitsize != 8 * len) {
+			PKEY_DBF_ERR("%s unknown/unsupported keybitsize %d\n",
+				     __func__, keybitsize);
+			return -EINVAL;
+		}
+		keybitsize = 8 * len;
+		if (clrkeylen != len) {
+			PKEY_DBF_ERR("%s invalid clear key len %d != %d\n",
+				     __func__, clrkeylen, len);
+			return -EINVAL;
+		}
+		switch (subtype) {
+		case PKEY_TYPE_EP11:
+		case PKEY_TYPE_EP11_AES:
+			break;
+		default:
+			PKEY_DBF_ERR("%s unknown/unsupported subtype %d\n",
+				     __func__, subtype);
+			return -EINVAL;
+		}
+		break;
+	default:
+		PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n",
+			     __func__, keytype);
+		return -EINVAL;
+	}
+
+	zcrypt_wait_api_operational();
+
+	if (!apqns || (nr_apqns == 1 &&
+		       apqns[0].card == 0xFFFF && apqns[0].domain == 0xFFFF)) {
+		nr_apqns = MAXAPQNSINLIST;
+		rc = ep11_apqns4type(subtype, NULL, NULL, 0,
+				     _apqns, &nr_apqns, pflags);
+		if (rc)
+			goto out;
+		apqns = _apqns;
+	}
+
+	for (rc = -ENODEV, i = 0; rc && i < nr_apqns; i++) {
+		rc = ep11_clr2keyblob(apqns[i].card, apqns[i].domain,
+				      keybitsize, flags, clrkey,
+				      keybuf, keybuflen, subtype, xflags);
+	}
+
+out:
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+
+static int ep11_verifykey(const u8 *key, u32 keylen,
+			  u16 *card, u16 *dom,
+			  u32 *keytype, u32 *keybitsize, u32 *flags, u32 pflags)
+{
+	struct keytoken_header *hdr = (struct keytoken_header *)key;
+	u32 apqns[MAXAPQNSINLIST], nr_apqns = ARRAY_SIZE(apqns);
+	u32 xflags;
+	int rc;
+
+	xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0;
+
+	if (keylen < sizeof(*hdr))
+		return -EINVAL;
+
+	zcrypt_wait_api_operational();
+
+	if (hdr->type == TOKTYPE_NON_CCA &&
+	    hdr->version == TOKVER_EP11_AES) {
+		struct ep11keyblob *kb = (struct ep11keyblob *)key;
+		int api;
+
+		rc = ep11_check_aes_key(pkey_dbf_info, 3, key, keylen, 1);
+		if (rc)
+			goto out;
+		*keytype = PKEY_TYPE_EP11;
+		*keybitsize = kb->head.bitlen;
+
+		api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4;
+		rc = ep11_findcard2(apqns, &nr_apqns, *card, *dom,
+				    ZCRYPT_CEX7, api,
+				    ep11_kb_wkvp(key, keylen), xflags);
+		if (rc)
+			goto out;
+
+		*flags = PKEY_FLAGS_MATCH_CUR_MKVP;
+
+		*card = ((struct pkey_apqn *)apqns)->card;
+		*dom = ((struct pkey_apqn *)apqns)->domain;
+
+	} else if (hdr->type == TOKTYPE_NON_CCA &&
+		   hdr->version == TOKVER_EP11_AES_WITH_HEADER) {
+		struct ep11kblob_header *kh = (struct ep11kblob_header *)key;
+		int api;
+
+		rc = ep11_check_aes_key_with_hdr(pkey_dbf_info,
+						 3, key, keylen, 1);
+		if (rc)
+			goto out;
+		*keytype = PKEY_TYPE_EP11_AES;
+		*keybitsize = kh->bitlen;
+
+		api = ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4;
+		rc = ep11_findcard2(apqns, &nr_apqns, *card, *dom,
+				    ZCRYPT_CEX7, api,
+				    ep11_kb_wkvp(key, keylen), xflags);
+		if (rc)
+			goto out;
+
+		*flags = PKEY_FLAGS_MATCH_CUR_MKVP;
+
+		*card = ((struct pkey_apqn *)apqns)->card;
+		*dom = ((struct pkey_apqn *)apqns)->domain;
+
+	} else {
+		/* unknown/unsupported key blob */
+		rc = -EINVAL;
+	}
+
+out:
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+
+/*
+ * This function provides an alternate but usually slow way
+ * to convert a 'clear key token' with AES key material into
+ * a protected key. That is done via an intermediate step
+ * which creates an EP11 AES secure key first and then derives
+ * the protected key from this secure key.
+ */
+static int ep11_slowpath_key2protkey(const struct pkey_apqn *apqns,
+				     size_t nr_apqns,
+				     const u8 *key, u32 keylen,
+				     u8 *protkey, u32 *protkeylen,
+				     u32 *protkeytype, u32 pflags)
+{
+	const struct keytoken_header *hdr = (const struct keytoken_header *)key;
+	const struct clearkeytoken *t = (const struct clearkeytoken *)key;
+	u8 tmpbuf[MAXEP11AESKEYBLOBSIZE]; /* 336 bytes */
+	u32 tmplen, keysize = 0;
+	int i, rc;
+
+	if (keylen < sizeof(*hdr))
+		return -EINVAL;
+
+	if (hdr->type == TOKTYPE_NON_CCA &&
+	    hdr->version == TOKVER_CLEAR_KEY)
+		keysize = pkey_keytype_aes_to_size(t->keytype);
+	if (!keysize || t->len != keysize)
+		return -EINVAL;
+
+	/* try two times in case of failure */
+	for (i = 0, rc = -ENODEV; i < 2 && rc; i++) {
+		tmplen = MAXEP11AESKEYBLOBSIZE;
+		rc = ep11_clr2key(NULL, 0, t->keytype, PKEY_TYPE_EP11,
+				  8 * keysize, 0, t->clearkey, t->len,
+				  tmpbuf, &tmplen, NULL, pflags);
+		pr_debug("ep11_clr2key()=%d\n", rc);
+		if (rc)
+			continue;
+		rc = ep11_key2protkey(NULL, 0, tmpbuf, tmplen,
+				      protkey, protkeylen, protkeytype, pflags);
+		pr_debug("ep11_key2protkey()=%d\n", rc);
+	}
+
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+
+static struct pkey_handler ep11_handler = {
+	.module			 = THIS_MODULE,
+	.name			 = "PKEY EP11 handler",
+	.is_supported_key	 = is_ep11_key,
+	.is_supported_keytype	 = is_ep11_keytype,
+	.key_to_protkey		 = ep11_key2protkey,
+	.slowpath_key_to_protkey = ep11_slowpath_key2protkey,
+	.gen_key		 = ep11_gen_key,
+	.clr_to_key		 = ep11_clr2key,
+	.verify_key		 = ep11_verifykey,
+	.apqns_for_key		 = ep11_apqns4key,
+	.apqns_for_keytype	 = ep11_apqns4type,
+};
+
+/*
+ * Module init
+ */
+static int __init pkey_ep11_init(void)
+{
+	/* register this module as pkey handler for all the ep11 stuff */
+	return pkey_handler_register(&ep11_handler);
+}
+
+/*
+ * Module exit
+ */
+static void __exit pkey_ep11_exit(void)
+{
+	/* unregister this module as pkey handler */
+	pkey_handler_unregister(&ep11_handler);
+}
+
+module_init(pkey_ep11_init);
+module_exit(pkey_ep11_exit);
diff --git a/drivers/s390/crypto/pkey_pckmo.c b/drivers/s390/crypto/pkey_pckmo.c
new file mode 100644
index 000000000..0cc015cc9
--- /dev/null
+++ b/drivers/s390/crypto/pkey_pckmo.c
@@ -0,0 +1,483 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  pkey pckmo specific code
+ *
+ *  Copyright IBM Corp. 2024
+ */
+
+#define pr_fmt(fmt) "pkey: " fmt
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/cpufeature.h>
+#include <asm/cpacf.h>
+#include <crypto/aes.h>
+#include <linux/random.h>
+
+#include "zcrypt_ccamisc.h"
+#include "pkey_base.h"
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("IBM Corporation");
+MODULE_DESCRIPTION("s390 protected key PCKMO handler");
+
+/*
+ * Check key blob for known and supported here.
+ */
+static bool is_pckmo_key(const u8 *key, u32 keylen)
+{
+	struct keytoken_header *hdr = (struct keytoken_header *)key;
+	struct clearkeytoken *t = (struct clearkeytoken *)key;
+
+	if (keylen < sizeof(*hdr))
+		return false;
+
+	switch (hdr->type) {
+	case TOKTYPE_NON_CCA:
+		switch (hdr->version) {
+		case TOKVER_CLEAR_KEY:
+			if (pkey_keytype_to_size(t->keytype))
+				return true;
+			return false;
+		case TOKVER_PROTECTED_KEY:
+			return true;
+		default:
+			return false;
+		}
+	default:
+		return false;
+	}
+}
+
+static bool is_pckmo_keytype(enum pkey_key_type keytype)
+{
+	switch (keytype) {
+	case PKEY_TYPE_PROTKEY:
+		return true;
+	default:
+		return false;
+	}
+}
+
+/*
+ * Create a protected key from a clear key value via PCKMO instruction.
+ */
+static int pckmo_clr2protkey(u32 keytype, const u8 *clrkey, u32 clrkeylen,
+			     u8 *protkey, u32 *protkeylen, u32 *protkeytype)
+{
+	/* mask of available pckmo subfunctions */
+	static cpacf_mask_t pckmo_functions;
+
+	int keysize, rc = -EINVAL;
+	u8 paramblock[160];
+	u32 pkeytype = 0;
+	unsigned int fc;
+
+	switch (keytype) {
+	case PKEY_KEYTYPE_AES_128:
+		fc = CPACF_PCKMO_ENC_AES_128_KEY;
+		break;
+	case PKEY_KEYTYPE_AES_192:
+		fc = CPACF_PCKMO_ENC_AES_192_KEY;
+		break;
+	case PKEY_KEYTYPE_AES_256:
+		fc = CPACF_PCKMO_ENC_AES_256_KEY;
+		break;
+	case PKEY_KEYTYPE_ECC_P256:
+		pkeytype = PKEY_KEYTYPE_ECC;
+		fc = CPACF_PCKMO_ENC_ECC_P256_KEY;
+		break;
+	case PKEY_KEYTYPE_ECC_P384:
+		pkeytype = PKEY_KEYTYPE_ECC;
+		fc = CPACF_PCKMO_ENC_ECC_P384_KEY;
+		break;
+	case PKEY_KEYTYPE_ECC_P521:
+		pkeytype = PKEY_KEYTYPE_ECC;
+		fc = CPACF_PCKMO_ENC_ECC_P521_KEY;
+		break;
+	case PKEY_KEYTYPE_ECC_ED25519:
+		pkeytype = PKEY_KEYTYPE_ECC;
+		fc = CPACF_PCKMO_ENC_ECC_ED25519_KEY;
+		break;
+	case PKEY_KEYTYPE_ECC_ED448:
+		pkeytype = PKEY_KEYTYPE_ECC;
+		fc = CPACF_PCKMO_ENC_ECC_ED448_KEY;
+		break;
+	case PKEY_KEYTYPE_AES_XTS_128:
+		fc = CPACF_PCKMO_ENC_AES_XTS_128_DOUBLE_KEY;
+		break;
+	case PKEY_KEYTYPE_AES_XTS_256:
+		fc = CPACF_PCKMO_ENC_AES_XTS_256_DOUBLE_KEY;
+		break;
+	case PKEY_KEYTYPE_HMAC_512:
+		fc = CPACF_PCKMO_ENC_HMAC_512_KEY;
+		break;
+	case PKEY_KEYTYPE_HMAC_1024:
+		fc = CPACF_PCKMO_ENC_HMAC_1024_KEY;
+		break;
+	default:
+		PKEY_DBF_ERR("%s unknown/unsupported keytype %u\n",
+			     __func__, keytype);
+		goto out;
+	}
+
+	keysize = pkey_keytype_to_size(keytype);
+	pkeytype = pkeytype ?: keytype;
+
+	if (clrkeylen && clrkeylen < keysize) {
+		PKEY_DBF_ERR("%s clear key size too small: %u < %d\n",
+			     __func__, clrkeylen, keysize);
+		goto out;
+	}
+	if (*protkeylen < keysize + AES_WK_VP_SIZE) {
+		PKEY_DBF_ERR("%s prot key buffer size too small: %u < %d\n",
+			     __func__, *protkeylen, keysize + AES_WK_VP_SIZE);
+		goto out;
+	}
+
+	/* Did we already check for PCKMO ? */
+	if (!pckmo_functions.bytes[0]) {
+		/* no, so check now */
+		if (!cpacf_query(CPACF_PCKMO, &pckmo_functions)) {
+			PKEY_DBF_ERR("%s cpacf_query() failed\n", __func__);
+			rc = -ENODEV;
+			goto out;
+		}
+	}
+	/* check for the pckmo subfunction we need now */
+	if (!cpacf_test_func(&pckmo_functions, fc)) {
+		PKEY_DBF_ERR("%s pckmo fc 0x%02x not available\n",
+			     __func__, fc);
+		rc = -ENODEV;
+		goto out;
+	}
+
+	/* prepare param block */
+	memset(paramblock, 0, sizeof(paramblock));
+	memcpy(paramblock, clrkey, keysize);
+
+	/* call the pckmo instruction */
+	cpacf_pckmo(fc, paramblock);
+
+	/* copy created protected key to key buffer including the wkvp block */
+	*protkeylen = keysize + AES_WK_VP_SIZE;
+	memcpy(protkey, paramblock, *protkeylen);
+	*protkeytype = pkeytype;
+
+	rc = 0;
+
+out:
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+
+/*
+ * Verify a raw protected key blob.
+ */
+static int pckmo_verify_protkey(const u8 *protkey, u32 protkeylen,
+				u32 protkeytype)
+{
+	u8 clrkey[16] = { 0 }, tmpkeybuf[16 + AES_WK_VP_SIZE];
+	u32 tmpkeybuflen, tmpkeytype;
+	int keysize, rc = -EINVAL;
+	u8 *wkvp;
+
+	/* check protkey type and size */
+	keysize = pkey_keytype_to_size(protkeytype);
+	if (!keysize) {
+		PKEY_DBF_ERR("%s unknown/unsupported keytype %u\n", __func__,
+			     protkeytype);
+		goto out;
+	}
+	if (protkeylen < keysize + AES_WK_VP_SIZE)
+		goto out;
+
+	/* generate a dummy AES 128 protected key */
+	tmpkeybuflen = sizeof(tmpkeybuf);
+	rc = pckmo_clr2protkey(PKEY_KEYTYPE_AES_128,
+			       clrkey, sizeof(clrkey),
+			       tmpkeybuf, &tmpkeybuflen, &tmpkeytype);
+	if (rc)
+		goto out;
+	memzero_explicit(tmpkeybuf, 16);
+	wkvp = tmpkeybuf + 16;
+
+	/* compare WK VP from the temp key with that of the given prot key */
+	if (memcmp(wkvp, protkey + keysize, AES_WK_VP_SIZE)) {
+		PKEY_DBF_ERR("%s protected key WK VP mismatch\n", __func__);
+		rc = -EKEYREJECTED;
+		goto out;
+	}
+
+out:
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+
+static int pckmo_key2protkey(const u8 *key, u32 keylen,
+			     u8 *protkey, u32 *protkeylen, u32 *protkeytype,
+			     u32 xflags)
+{
+	struct keytoken_header *hdr = (struct keytoken_header *)key;
+	int rc = -EINVAL;
+
+	if (keylen < sizeof(*hdr))
+		return -EINVAL;
+	if (hdr->type != TOKTYPE_NON_CCA)
+		return -EINVAL;
+
+	switch (hdr->version) {
+	case TOKVER_PROTECTED_KEY: {
+		struct protkeytoken *t = (struct protkeytoken *)key;
+		u32 keysize;
+
+		if (keylen < sizeof(*t))
+			goto out;
+		keysize = pkey_keytype_to_size(t->keytype);
+		if (!keysize) {
+			PKEY_DBF_ERR("%s protected key token: unknown keytype %u\n",
+				     __func__, t->keytype);
+			goto out;
+		}
+		switch (t->keytype) {
+		case PKEY_KEYTYPE_AES_128:
+		case PKEY_KEYTYPE_AES_192:
+		case PKEY_KEYTYPE_AES_256:
+			if (t->len != keysize + AES_WK_VP_SIZE ||
+			    keylen < sizeof(struct protaeskeytoken))
+				goto out;
+			rc = pckmo_verify_protkey(t->protkey, t->len,
+						  t->keytype);
+			if (rc)
+				goto out;
+			break;
+		default:
+			if (t->len != keysize + AES_WK_VP_SIZE ||
+			    keylen < sizeof(*t) + keysize + AES_WK_VP_SIZE)
+				goto out;
+			break;
+		}
+		if (t->len > *protkeylen) {
+			rc = -EINVAL;
+			goto out;
+		}
+		memcpy(protkey, t->protkey, t->len);
+		*protkeylen = t->len;
+		*protkeytype = t->keytype;
+		rc = 0;
+		break;
+	}
+	case TOKVER_CLEAR_KEY: {
+		struct clearkeytoken *t = (struct clearkeytoken *)key;
+		u32 keysize;
+
+		if (xflags & PKEY_XFLAG_NOCLEARKEY) {
+			PKEY_DBF_ERR("%s clear key token but xflag NOCLEARKEY\n",
+				     __func__);
+			goto out;
+		}
+		if (keylen < sizeof(*t) ||
+		    keylen < sizeof(*t) + t->len)
+			goto out;
+		keysize = pkey_keytype_to_size(t->keytype);
+		if (!keysize) {
+			PKEY_DBF_ERR("%s clear key token: unknown keytype %u\n",
+				     __func__, t->keytype);
+			goto out;
+		}
+		if (t->len != keysize) {
+			PKEY_DBF_ERR("%s clear key token: invalid key len %u\n",
+				     __func__, t->len);
+			goto out;
+		}
+		rc = pckmo_clr2protkey(t->keytype, t->clearkey, t->len,
+				       protkey, protkeylen, protkeytype);
+		break;
+	}
+	default:
+		PKEY_DBF_ERR("%s unknown non-CCA token version %d\n",
+			     __func__, hdr->version);
+		break;
+	}
+
+out:
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+
+/*
+ * Generate a random protected key.
+ */
+static int pckmo_gen_protkey(u32 keytype, u32 subtype,
+			     u8 *protkey, u32 *protkeylen, u32 *protkeytype)
+{
+	u8 clrkey[128];
+	int keysize;
+	int rc;
+
+	keysize = pkey_keytype_to_size(keytype);
+	if (!keysize) {
+		PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n",
+			     __func__, keytype);
+		return -EINVAL;
+	}
+	if (subtype != PKEY_TYPE_PROTKEY) {
+		PKEY_DBF_ERR("%s unknown/unsupported subtype %d\n",
+			     __func__, subtype);
+		return -EINVAL;
+	}
+
+	switch (keytype) {
+	case PKEY_KEYTYPE_AES_128:
+	case PKEY_KEYTYPE_AES_192:
+	case PKEY_KEYTYPE_AES_256:
+	case PKEY_KEYTYPE_AES_XTS_128:
+	case PKEY_KEYTYPE_AES_XTS_256:
+	case PKEY_KEYTYPE_HMAC_512:
+	case PKEY_KEYTYPE_HMAC_1024:
+		break;
+	default:
+		PKEY_DBF_ERR("%s unsupported keytype %d\n",
+			     __func__, keytype);
+		return -EINVAL;
+	}
+
+	/* generate a dummy random clear key */
+	get_random_bytes(clrkey, keysize);
+
+	/* convert it to a dummy protected key */
+	rc = pckmo_clr2protkey(keytype, clrkey, keysize,
+			       protkey, protkeylen, protkeytype);
+	if (rc)
+		goto out;
+
+	/* replace the key part of the protected key with random bytes */
+	get_random_bytes(protkey, keysize);
+
+out:
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+
+/*
+ * Verify a protected key token blob.
+ */
+static int pckmo_verify_key(const u8 *key, u32 keylen)
+{
+	struct keytoken_header *hdr = (struct keytoken_header *)key;
+	int rc = -EINVAL;
+
+	if (keylen < sizeof(*hdr))
+		return -EINVAL;
+	if (hdr->type != TOKTYPE_NON_CCA)
+		return -EINVAL;
+
+	switch (hdr->version) {
+	case TOKVER_PROTECTED_KEY: {
+		struct protkeytoken *t = (struct protkeytoken *)key;
+		u32 keysize;
+
+		if (keylen < sizeof(*t))
+			goto out;
+		keysize = pkey_keytype_to_size(t->keytype);
+		if (!keysize || t->len != keysize + AES_WK_VP_SIZE)
+			goto out;
+		switch (t->keytype) {
+		case PKEY_KEYTYPE_AES_128:
+		case PKEY_KEYTYPE_AES_192:
+		case PKEY_KEYTYPE_AES_256:
+			if (keylen < sizeof(struct protaeskeytoken))
+				goto out;
+			break;
+		default:
+			if (keylen < sizeof(*t) + keysize + AES_WK_VP_SIZE)
+				goto out;
+			break;
+		}
+		rc = pckmo_verify_protkey(t->protkey, t->len, t->keytype);
+		break;
+	}
+	default:
+		PKEY_DBF_ERR("%s unknown non-CCA token version %d\n",
+			     __func__, hdr->version);
+		break;
+	}
+
+out:
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+
+/*
+ * Wrapper functions used for the pkey handler struct
+ */
+
+static int pkey_pckmo_key2protkey(const struct pkey_apqn *_apqns,
+				  size_t _nr_apqns,
+				  const u8 *key, u32 keylen,
+				  u8 *protkey, u32 *protkeylen, u32 *keyinfo,
+				  u32 xflags)
+{
+	return pckmo_key2protkey(key, keylen,
+				 protkey, protkeylen, keyinfo, xflags);
+}
+
+static int pkey_pckmo_gen_key(const struct pkey_apqn *_apqns, size_t _nr_apqns,
+			      u32 keytype, u32 keysubtype,
+			      u32 _keybitsize, u32 _flags,
+			      u8 *keybuf, u32 *keybuflen, u32 *keyinfo,
+			      u32 _xflags __always_unused)
+{
+	return pckmo_gen_protkey(keytype, keysubtype,
+				 keybuf, keybuflen, keyinfo);
+}
+
+static int pkey_pckmo_verifykey(const u8 *key, u32 keylen,
+				u16 *_card, u16 *_dom,
+				u32 *_keytype, u32 *_keybitsize,
+				u32 *_flags, u32 _xflags __always_unused)
+{
+	return pckmo_verify_key(key, keylen);
+}
+
+static struct pkey_handler pckmo_handler = {
+	.module		      = THIS_MODULE,
+	.name		      = "PKEY PCKMO handler",
+	.is_supported_key     = is_pckmo_key,
+	.is_supported_keytype = is_pckmo_keytype,
+	.key_to_protkey	      = pkey_pckmo_key2protkey,
+	.gen_key	      = pkey_pckmo_gen_key,
+	.verify_key	      = pkey_pckmo_verifykey,
+};
+
+/*
+ * Module init
+ */
+static int __init pkey_pckmo_init(void)
+{
+	cpacf_mask_t func_mask;
+
+	/*
+	 * The pckmo instruction should be available - even if we don't
+	 * actually invoke it. This instruction comes with MSA 3 which
+	 * is also the minimum level for the kmc instructions which
+	 * are able to work with protected keys.
+	 */
+	if (!cpacf_query(CPACF_PCKMO, &func_mask))
+		return -ENODEV;
+
+	/* register this module as pkey handler for all the pckmo stuff */
+	return pkey_handler_register(&pckmo_handler);
+}
+
+/*
+ * Module exit
+ */
+static void __exit pkey_pckmo_exit(void)
+{
+	/* unregister this module as pkey handler */
+	pkey_handler_unregister(&pckmo_handler);
+}
+
+module_cpu_feature_match(S390_CPU_FEATURE_MSA, pkey_pckmo_init);
+module_exit(pkey_pckmo_exit);
diff --git a/drivers/s390/crypto/pkey_sysfs.c b/drivers/s390/crypto/pkey_sysfs.c
new file mode 100644
index 000000000..b6b0a46cb
--- /dev/null
+++ b/drivers/s390/crypto/pkey_sysfs.c
@@ -0,0 +1,646 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  pkey module sysfs related functions
+ *
+ *  Copyright IBM Corp. 2024
+ */
+
+#define pr_fmt(fmt) "pkey: " fmt
+
+#include <linux/sysfs.h>
+
+#include "zcrypt_ccamisc.h"
+#include "zcrypt_ep11misc.h"
+
+#include "pkey_base.h"
+
+/*
+ * Wrapper around pkey_handler_gen_key() which deals with the
+ * ENODEV return code and then tries to enforce a pkey handler
+ * module load.
+ */
+static int sys_pkey_handler_gen_key(u32 keytype, u32 keysubtype,
+				    u32 keybitsize, u32 flags,
+				    u8 *keybuf, u32 *keybuflen, u32 *keyinfo)
+{
+	int rc;
+
+	rc = pkey_handler_gen_key(NULL, 0,
+				  keytype, keysubtype,
+				  keybitsize, flags,
+				  keybuf, keybuflen, keyinfo, 0);
+	if (rc == -ENODEV) {
+		pkey_handler_request_modules();
+		rc = pkey_handler_gen_key(NULL, 0,
+					  keytype, keysubtype,
+					  keybitsize, flags,
+					  keybuf, keybuflen, keyinfo, 0);
+	}
+
+	return rc;
+}
+
+/*
+ * Sysfs attribute read function for all protected key binary attributes.
+ * The implementation can not deal with partial reads, because a new random
+ * protected key blob is generated with each read. In case of partial reads
+ * (i.e. off != 0 or count < key blob size) -EINVAL is returned.
+ */
+static ssize_t pkey_protkey_aes_attr_read(u32 keytype, bool is_xts, char *buf,
+					  loff_t off, size_t count)
+{
+	struct protaeskeytoken protkeytoken;
+	struct pkey_protkey protkey;
+	int rc;
+
+	if (off != 0 || count < sizeof(protkeytoken))
+		return -EINVAL;
+	if (is_xts)
+		if (count < 2 * sizeof(protkeytoken))
+			return -EINVAL;
+
+	memset(&protkeytoken, 0, sizeof(protkeytoken));
+	protkeytoken.type = TOKTYPE_NON_CCA;
+	protkeytoken.version = TOKVER_PROTECTED_KEY;
+	protkeytoken.keytype = keytype;
+
+	protkey.len = sizeof(protkey.protkey);
+	rc = sys_pkey_handler_gen_key(keytype, PKEY_TYPE_PROTKEY, 0, 0,
+				      protkey.protkey, &protkey.len,
+				      &protkey.type);
+	if (rc)
+		return rc;
+
+	protkeytoken.len = protkey.len;
+	memcpy(&protkeytoken.protkey, &protkey.protkey, protkey.len);
+
+	memcpy(buf, &protkeytoken, sizeof(protkeytoken));
+
+	if (is_xts) {
+		/* xts needs a second protected key, reuse protkey struct */
+		protkey.len = sizeof(protkey.protkey);
+		rc = sys_pkey_handler_gen_key(keytype, PKEY_TYPE_PROTKEY, 0, 0,
+					      protkey.protkey, &protkey.len,
+					      &protkey.type);
+		if (rc)
+			return rc;
+
+		protkeytoken.len = protkey.len;
+		memcpy(&protkeytoken.protkey, &protkey.protkey, protkey.len);
+
+		memcpy(buf + sizeof(protkeytoken), &protkeytoken,
+		       sizeof(protkeytoken));
+
+		return 2 * sizeof(protkeytoken);
+	}
+
+	return sizeof(protkeytoken);
+}
+
+/*
+ * Sysfs attribute read function for the AES XTS prot key binary attributes.
+ * The implementation can not deal with partial reads, because a new random
+ * protected key blob is generated with each read. In case of partial reads
+ * (i.e. off != 0 or count < key blob size) -EINVAL is returned.
+ */
+static ssize_t pkey_protkey_aes_xts_attr_read(u32 keytype, char *buf,
+					      loff_t off, size_t count)
+{
+	struct protkeytoken *t = (struct protkeytoken *)buf;
+	u32 protlen, prottype;
+	int rc;
+
+	switch (keytype) {
+	case PKEY_KEYTYPE_AES_XTS_128:
+		protlen = 64;
+		break;
+	case PKEY_KEYTYPE_AES_XTS_256:
+		protlen = 96;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (off != 0 || count < sizeof(*t) + protlen)
+		return -EINVAL;
+
+	memset(t, 0, sizeof(*t) + protlen);
+	t->type = TOKTYPE_NON_CCA;
+	t->version = TOKVER_PROTECTED_KEY;
+	t->keytype = keytype;
+
+	rc = sys_pkey_handler_gen_key(keytype, PKEY_TYPE_PROTKEY, 0, 0,
+				      t->protkey, &protlen, &prottype);
+	if (rc)
+		return rc;
+
+	t->len = protlen;
+
+	return sizeof(*t) + protlen;
+}
+
+/*
+ * Sysfs attribute read function for the HMAC prot key binary attributes.
+ * The implementation can not deal with partial reads, because a new random
+ * protected key blob is generated with each read. In case of partial reads
+ * (i.e. off != 0 or count < key blob size) -EINVAL is returned.
+ */
+static ssize_t pkey_protkey_hmac_attr_read(u32 keytype, char *buf,
+					   loff_t off, size_t count)
+{
+	struct protkeytoken *t = (struct protkeytoken *)buf;
+	u32 protlen, prottype;
+	int rc;
+
+	switch (keytype) {
+	case PKEY_KEYTYPE_HMAC_512:
+		protlen = 96;
+		break;
+	case PKEY_KEYTYPE_HMAC_1024:
+		protlen = 160;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (off != 0 || count < sizeof(*t) + protlen)
+		return -EINVAL;
+
+	memset(t, 0, sizeof(*t) + protlen);
+	t->type = TOKTYPE_NON_CCA;
+	t->version = TOKVER_PROTECTED_KEY;
+	t->keytype = keytype;
+
+	rc = sys_pkey_handler_gen_key(keytype, PKEY_TYPE_PROTKEY, 0, 0,
+				      t->protkey, &protlen, &prottype);
+	if (rc)
+		return rc;
+
+	t->len = protlen;
+
+	return sizeof(*t) + protlen;
+}
+
+static ssize_t protkey_aes_128_read(struct file *filp,
+				    struct kobject *kobj,
+				    const struct bin_attribute *attr,
+				    char *buf, loff_t off,
+				    size_t count)
+{
+	return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_128, false, buf,
+					  off, count);
+}
+
+static ssize_t protkey_aes_192_read(struct file *filp,
+				    struct kobject *kobj,
+				    const struct bin_attribute *attr,
+				    char *buf, loff_t off,
+				    size_t count)
+{
+	return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_192, false, buf,
+					  off, count);
+}
+
+static ssize_t protkey_aes_256_read(struct file *filp,
+				    struct kobject *kobj,
+				    const struct bin_attribute *attr,
+				    char *buf, loff_t off,
+				    size_t count)
+{
+	return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_256, false, buf,
+					  off, count);
+}
+
+static ssize_t protkey_aes_128_xts_read(struct file *filp,
+					struct kobject *kobj,
+					const struct bin_attribute *attr,
+					char *buf, loff_t off,
+					size_t count)
+{
+	return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_128, true, buf,
+					  off, count);
+}
+
+static ssize_t protkey_aes_256_xts_read(struct file *filp,
+					struct kobject *kobj,
+					const struct bin_attribute *attr,
+					char *buf, loff_t off,
+					size_t count)
+{
+	return pkey_protkey_aes_attr_read(PKEY_KEYTYPE_AES_256, true, buf,
+					  off, count);
+}
+
+static ssize_t protkey_aes_xts_128_read(struct file *filp,
+					struct kobject *kobj,
+					const struct bin_attribute *attr,
+					char *buf, loff_t off,
+					size_t count)
+{
+	return pkey_protkey_aes_xts_attr_read(PKEY_KEYTYPE_AES_XTS_128,
+					      buf, off, count);
+}
+
+static ssize_t protkey_aes_xts_256_read(struct file *filp,
+					struct kobject *kobj,
+					const struct bin_attribute *attr,
+					char *buf, loff_t off,
+					size_t count)
+{
+	return pkey_protkey_aes_xts_attr_read(PKEY_KEYTYPE_AES_XTS_256,
+					      buf, off, count);
+}
+
+static ssize_t protkey_hmac_512_read(struct file *filp,
+				     struct kobject *kobj,
+				     const struct bin_attribute *attr,
+				     char *buf, loff_t off,
+				     size_t count)
+{
+	return pkey_protkey_hmac_attr_read(PKEY_KEYTYPE_HMAC_512,
+					   buf, off, count);
+}
+
+static ssize_t protkey_hmac_1024_read(struct file *filp,
+				      struct kobject *kobj,
+				      const struct bin_attribute *attr,
+				      char *buf, loff_t off,
+				      size_t count)
+{
+	return pkey_protkey_hmac_attr_read(PKEY_KEYTYPE_HMAC_1024,
+					   buf, off, count);
+}
+
+static const BIN_ATTR_RO(protkey_aes_128, sizeof(struct protaeskeytoken));
+static const BIN_ATTR_RO(protkey_aes_192, sizeof(struct protaeskeytoken));
+static const BIN_ATTR_RO(protkey_aes_256, sizeof(struct protaeskeytoken));
+static const BIN_ATTR_RO(protkey_aes_128_xts, 2 * sizeof(struct protaeskeytoken));
+static const BIN_ATTR_RO(protkey_aes_256_xts, 2 * sizeof(struct protaeskeytoken));
+static const BIN_ATTR_RO(protkey_aes_xts_128, sizeof(struct protkeytoken) + 64);
+static const BIN_ATTR_RO(protkey_aes_xts_256, sizeof(struct protkeytoken) + 96);
+static const BIN_ATTR_RO(protkey_hmac_512, sizeof(struct protkeytoken) + 96);
+static const BIN_ATTR_RO(protkey_hmac_1024, sizeof(struct protkeytoken) + 160);
+
+static const struct bin_attribute *const protkey_attrs[] = {
+	&bin_attr_protkey_aes_128,
+	&bin_attr_protkey_aes_192,
+	&bin_attr_protkey_aes_256,
+	&bin_attr_protkey_aes_128_xts,
+	&bin_attr_protkey_aes_256_xts,
+	&bin_attr_protkey_aes_xts_128,
+	&bin_attr_protkey_aes_xts_256,
+	&bin_attr_protkey_hmac_512,
+	&bin_attr_protkey_hmac_1024,
+	NULL
+};
+
+static const struct attribute_group protkey_attr_group = {
+	.name	       = "protkey",
+	.bin_attrs = protkey_attrs,
+};
+
+/*
+ * Sysfs attribute read function for all secure key ccadata binary attributes.
+ * The implementation can not deal with partial reads, because a new random
+ * protected key blob is generated with each read. In case of partial reads
+ * (i.e. off != 0 or count < key blob size) -EINVAL is returned.
+ */
+static ssize_t pkey_ccadata_aes_attr_read(u32 keytype, bool is_xts, char *buf,
+					  loff_t off, size_t count)
+{
+	struct pkey_seckey *seckey = (struct pkey_seckey *)buf;
+	u32 buflen;
+	int rc;
+
+	if (off != 0 || count < sizeof(struct secaeskeytoken))
+		return -EINVAL;
+	if (is_xts)
+		if (count < 2 * sizeof(struct secaeskeytoken))
+			return -EINVAL;
+
+	buflen = sizeof(seckey->seckey);
+	rc = sys_pkey_handler_gen_key(keytype, PKEY_TYPE_CCA_DATA, 0, 0,
+				      seckey->seckey, &buflen, NULL);
+	if (rc)
+		return rc;
+
+	if (is_xts) {
+		seckey++;
+		buflen = sizeof(seckey->seckey);
+		rc = sys_pkey_handler_gen_key(keytype, PKEY_TYPE_CCA_DATA, 0, 0,
+					      seckey->seckey, &buflen, NULL);
+		if (rc)
+			return rc;
+
+		return 2 * sizeof(struct secaeskeytoken);
+	}
+
+	return sizeof(struct secaeskeytoken);
+}
+
+static ssize_t ccadata_aes_128_read(struct file *filp,
+				    struct kobject *kobj,
+				    const struct bin_attribute *attr,
+				    char *buf, loff_t off,
+				    size_t count)
+{
+	return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_128, false, buf,
+					  off, count);
+}
+
+static ssize_t ccadata_aes_192_read(struct file *filp,
+				    struct kobject *kobj,
+				    const struct bin_attribute *attr,
+				    char *buf, loff_t off,
+				    size_t count)
+{
+	return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_192, false, buf,
+					  off, count);
+}
+
+static ssize_t ccadata_aes_256_read(struct file *filp,
+				    struct kobject *kobj,
+				    const struct bin_attribute *attr,
+				    char *buf, loff_t off,
+				    size_t count)
+{
+	return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_256, false, buf,
+					  off, count);
+}
+
+static ssize_t ccadata_aes_128_xts_read(struct file *filp,
+					struct kobject *kobj,
+					const struct bin_attribute *attr,
+					char *buf, loff_t off,
+					size_t count)
+{
+	return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_128, true, buf,
+					  off, count);
+}
+
+static ssize_t ccadata_aes_256_xts_read(struct file *filp,
+					struct kobject *kobj,
+					const struct bin_attribute *attr,
+					char *buf, loff_t off,
+					size_t count)
+{
+	return pkey_ccadata_aes_attr_read(PKEY_KEYTYPE_AES_256, true, buf,
+					  off, count);
+}
+
+static const BIN_ATTR_RO(ccadata_aes_128, sizeof(struct secaeskeytoken));
+static const BIN_ATTR_RO(ccadata_aes_192, sizeof(struct secaeskeytoken));
+static const BIN_ATTR_RO(ccadata_aes_256, sizeof(struct secaeskeytoken));
+static const BIN_ATTR_RO(ccadata_aes_128_xts, 2 * sizeof(struct secaeskeytoken));
+static const BIN_ATTR_RO(ccadata_aes_256_xts, 2 * sizeof(struct secaeskeytoken));
+
+static const struct bin_attribute *const ccadata_attrs[] = {
+	&bin_attr_ccadata_aes_128,
+	&bin_attr_ccadata_aes_192,
+	&bin_attr_ccadata_aes_256,
+	&bin_attr_ccadata_aes_128_xts,
+	&bin_attr_ccadata_aes_256_xts,
+	NULL
+};
+
+static const struct attribute_group ccadata_attr_group = {
+	.name	       = "ccadata",
+	.bin_attrs = ccadata_attrs,
+};
+
+#define CCACIPHERTOKENSIZE	(sizeof(struct cipherkeytoken) + 80)
+
+/*
+ * Sysfs attribute read function for all secure key ccacipher binary attributes.
+ * The implementation can not deal with partial reads, because a new random
+ * secure key blob is generated with each read. In case of partial reads
+ * (i.e. off != 0 or count < key blob size) -EINVAL is returned.
+ */
+static ssize_t pkey_ccacipher_aes_attr_read(enum pkey_key_size keybits,
+					    bool is_xts, char *buf, loff_t off,
+					    size_t count)
+{
+	u32 keysize = CCACIPHERTOKENSIZE;
+	int rc;
+
+	if (off != 0 || count < CCACIPHERTOKENSIZE)
+		return -EINVAL;
+	if (is_xts)
+		if (count < 2 * CCACIPHERTOKENSIZE)
+			return -EINVAL;
+
+	memset(buf, 0, is_xts ? 2 * keysize : keysize);
+
+	rc = sys_pkey_handler_gen_key(pkey_aes_bitsize_to_keytype(keybits),
+				      PKEY_TYPE_CCA_CIPHER, keybits, 0,
+				      buf, &keysize, NULL);
+	if (rc)
+		return rc;
+
+	if (is_xts) {
+		keysize = CCACIPHERTOKENSIZE;
+		buf += CCACIPHERTOKENSIZE;
+		rc = sys_pkey_handler_gen_key(
+			pkey_aes_bitsize_to_keytype(keybits),
+			PKEY_TYPE_CCA_CIPHER, keybits, 0,
+			buf, &keysize, NULL);
+		if (rc)
+			return rc;
+		return 2 * CCACIPHERTOKENSIZE;
+	}
+
+	return CCACIPHERTOKENSIZE;
+}
+
+static ssize_t ccacipher_aes_128_read(struct file *filp,
+				      struct kobject *kobj,
+				      const struct bin_attribute *attr,
+				      char *buf, loff_t off,
+				      size_t count)
+{
+	return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_128, false, buf,
+					    off, count);
+}
+
+static ssize_t ccacipher_aes_192_read(struct file *filp,
+				      struct kobject *kobj,
+				      const struct bin_attribute *attr,
+				      char *buf, loff_t off,
+				      size_t count)
+{
+	return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_192, false, buf,
+					    off, count);
+}
+
+static ssize_t ccacipher_aes_256_read(struct file *filp,
+				      struct kobject *kobj,
+				      const struct bin_attribute *attr,
+				      char *buf, loff_t off,
+				      size_t count)
+{
+	return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_256, false, buf,
+					    off, count);
+}
+
+static ssize_t ccacipher_aes_128_xts_read(struct file *filp,
+					  struct kobject *kobj,
+					  const struct bin_attribute *attr,
+					  char *buf, loff_t off,
+					  size_t count)
+{
+	return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_128, true, buf,
+					    off, count);
+}
+
+static ssize_t ccacipher_aes_256_xts_read(struct file *filp,
+					  struct kobject *kobj,
+					  const struct bin_attribute *attr,
+					  char *buf, loff_t off,
+					  size_t count)
+{
+	return pkey_ccacipher_aes_attr_read(PKEY_SIZE_AES_256, true, buf,
+					    off, count);
+}
+
+static const BIN_ATTR_RO(ccacipher_aes_128, CCACIPHERTOKENSIZE);
+static const BIN_ATTR_RO(ccacipher_aes_192, CCACIPHERTOKENSIZE);
+static const BIN_ATTR_RO(ccacipher_aes_256, CCACIPHERTOKENSIZE);
+static const BIN_ATTR_RO(ccacipher_aes_128_xts, 2 * CCACIPHERTOKENSIZE);
+static const BIN_ATTR_RO(ccacipher_aes_256_xts, 2 * CCACIPHERTOKENSIZE);
+
+static const struct bin_attribute *const ccacipher_attrs[] = {
+	&bin_attr_ccacipher_aes_128,
+	&bin_attr_ccacipher_aes_192,
+	&bin_attr_ccacipher_aes_256,
+	&bin_attr_ccacipher_aes_128_xts,
+	&bin_attr_ccacipher_aes_256_xts,
+	NULL
+};
+
+static const struct attribute_group ccacipher_attr_group = {
+	.name	       = "ccacipher",
+	.bin_attrs = ccacipher_attrs,
+};
+
+/*
+ * Sysfs attribute read function for all ep11 aes key binary attributes.
+ * The implementation can not deal with partial reads, because a new random
+ * secure key blob is generated with each read. In case of partial reads
+ * (i.e. off != 0 or count < key blob size) -EINVAL is returned.
+ * This function and the sysfs attributes using it provide EP11 key blobs
+ * padded to the upper limit of MAXEP11AESKEYBLOBSIZE which is currently
+ * 336 bytes.
+ */
+static ssize_t pkey_ep11_aes_attr_read(enum pkey_key_size keybits,
+				       bool is_xts, char *buf, loff_t off,
+				       size_t count)
+{
+	u32 keysize = MAXEP11AESKEYBLOBSIZE;
+	int rc;
+
+	if (off != 0 || count < MAXEP11AESKEYBLOBSIZE)
+		return -EINVAL;
+	if (is_xts)
+		if (count < 2 * MAXEP11AESKEYBLOBSIZE)
+			return -EINVAL;
+
+	memset(buf, 0, is_xts ? 2 * keysize : keysize);
+
+	rc = sys_pkey_handler_gen_key(pkey_aes_bitsize_to_keytype(keybits),
+				      PKEY_TYPE_EP11_AES, keybits, 0,
+				      buf, &keysize, NULL);
+	if (rc)
+		return rc;
+
+	if (is_xts) {
+		keysize = MAXEP11AESKEYBLOBSIZE;
+		buf += MAXEP11AESKEYBLOBSIZE;
+		rc = sys_pkey_handler_gen_key(
+			pkey_aes_bitsize_to_keytype(keybits),
+			PKEY_TYPE_EP11_AES, keybits, 0,
+			buf, &keysize, NULL);
+		if (rc)
+			return rc;
+		return 2 * MAXEP11AESKEYBLOBSIZE;
+	}
+
+	return MAXEP11AESKEYBLOBSIZE;
+}
+
+static ssize_t ep11_aes_128_read(struct file *filp,
+				 struct kobject *kobj,
+				 const struct bin_attribute *attr,
+				 char *buf, loff_t off,
+				 size_t count)
+{
+	return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_128, false, buf,
+				       off, count);
+}
+
+static ssize_t ep11_aes_192_read(struct file *filp,
+				 struct kobject *kobj,
+				 const struct bin_attribute *attr,
+				 char *buf, loff_t off,
+				 size_t count)
+{
+	return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_192, false, buf,
+				       off, count);
+}
+
+static ssize_t ep11_aes_256_read(struct file *filp,
+				 struct kobject *kobj,
+				 const struct bin_attribute *attr,
+				 char *buf, loff_t off,
+				 size_t count)
+{
+	return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_256, false, buf,
+				       off, count);
+}
+
+static ssize_t ep11_aes_128_xts_read(struct file *filp,
+				     struct kobject *kobj,
+				     const struct bin_attribute *attr,
+				     char *buf, loff_t off,
+				     size_t count)
+{
+	return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_128, true, buf,
+				       off, count);
+}
+
+static ssize_t ep11_aes_256_xts_read(struct file *filp,
+				     struct kobject *kobj,
+				     const struct bin_attribute *attr,
+				     char *buf, loff_t off,
+				     size_t count)
+{
+	return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_256, true, buf,
+				       off, count);
+}
+
+static const BIN_ATTR_RO(ep11_aes_128, MAXEP11AESKEYBLOBSIZE);
+static const BIN_ATTR_RO(ep11_aes_192, MAXEP11AESKEYBLOBSIZE);
+static const BIN_ATTR_RO(ep11_aes_256, MAXEP11AESKEYBLOBSIZE);
+static const BIN_ATTR_RO(ep11_aes_128_xts, 2 * MAXEP11AESKEYBLOBSIZE);
+static const BIN_ATTR_RO(ep11_aes_256_xts, 2 * MAXEP11AESKEYBLOBSIZE);
+
+static const struct bin_attribute *const ep11_attrs[] = {
+	&bin_attr_ep11_aes_128,
+	&bin_attr_ep11_aes_192,
+	&bin_attr_ep11_aes_256,
+	&bin_attr_ep11_aes_128_xts,
+	&bin_attr_ep11_aes_256_xts,
+	NULL
+};
+
+static const struct attribute_group ep11_attr_group = {
+	.name	   = "ep11",
+	.bin_attrs = ep11_attrs,
+};
+
+const struct attribute_group *pkey_attr_groups[] = {
+	&protkey_attr_group,
+	&ccadata_attr_group,
+	&ccacipher_attr_group,
+	&ep11_attr_group,
+	NULL,
+};
diff --git a/drivers/s390/crypto/pkey_uv.c b/drivers/s390/crypto/pkey_uv.c
new file mode 100644
index 000000000..17a914d11
--- /dev/null
+++ b/drivers/s390/crypto/pkey_uv.c
@@ -0,0 +1,317 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  pkey uv specific code
+ *
+ *  Copyright IBM Corp. 2024
+ */
+
+#define pr_fmt(fmt) "pkey: " fmt
+
+#include <linux/cpufeature.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <asm/uv.h>
+
+#include "zcrypt_ccamisc.h"
+#include "pkey_base.h"
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("IBM Corporation");
+MODULE_DESCRIPTION("s390 protected key UV handler");
+
+/*
+ * One pre-allocated uv_secret_list for use with uv_find_secret()
+ */
+static struct uv_secret_list *uv_list;
+static DEFINE_MUTEX(uv_list_mutex);
+
+/*
+ * UV secret token struct and defines.
+ */
+
+#define TOKVER_UV_SECRET 0x09
+
+struct uvsecrettoken {
+	u8  type;		/* 0x00 = TOKTYPE_NON_CCA */
+	u8  res0[3];
+	u8  version;		/* 0x09 = TOKVER_UV_SECRET */
+	u8  res1[3];
+	u16 secret_type;	/* one of enum uv_secret_types from uv.h */
+	u16 secret_len;		/* length in bytes of the secret */
+	u8  secret_id[UV_SECRET_ID_LEN]; /* the secret id for this secret */
+} __packed;
+
+/*
+ * Check key blob for known and supported UV key.
+ */
+static bool is_uv_key(const u8 *key, u32 keylen)
+{
+	struct uvsecrettoken *t = (struct uvsecrettoken *)key;
+
+	if (keylen < sizeof(*t))
+		return false;
+
+	switch (t->type) {
+	case TOKTYPE_NON_CCA:
+		switch (t->version) {
+		case TOKVER_UV_SECRET:
+			switch (t->secret_type) {
+			case UV_SECRET_AES_128:
+			case UV_SECRET_AES_192:
+			case UV_SECRET_AES_256:
+			case UV_SECRET_AES_XTS_128:
+			case UV_SECRET_AES_XTS_256:
+			case UV_SECRET_HMAC_SHA_256:
+			case UV_SECRET_HMAC_SHA_512:
+			case UV_SECRET_ECDSA_P256:
+			case UV_SECRET_ECDSA_P384:
+			case UV_SECRET_ECDSA_P521:
+			case UV_SECRET_ECDSA_ED25519:
+			case UV_SECRET_ECDSA_ED448:
+				return true;
+			default:
+				return false;
+			}
+		default:
+			return false;
+		}
+	default:
+		return false;
+	}
+}
+
+static bool is_uv_keytype(enum pkey_key_type keytype)
+{
+	switch (keytype) {
+	case PKEY_TYPE_UVSECRET:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static int get_secret_metadata(const u8 secret_id[UV_SECRET_ID_LEN],
+			       struct uv_secret_list_item_hdr *secret)
+{
+	int rc;
+
+	mutex_lock(&uv_list_mutex);
+	memset(uv_list, 0, sizeof(*uv_list));
+	rc = uv_find_secret(secret_id, uv_list, secret);
+	mutex_unlock(&uv_list_mutex);
+
+	return rc;
+}
+
+static int retrieve_secret(const u8 secret_id[UV_SECRET_ID_LEN],
+			   u16 *secret_type, u8 *buf, u32 *buflen)
+{
+	struct uv_secret_list_item_hdr secret_meta_data;
+	int rc;
+
+	rc = get_secret_metadata(secret_id, &secret_meta_data);
+	if (rc)
+		return rc;
+
+	if (*buflen < secret_meta_data.length)
+		return -EINVAL;
+
+	rc = uv_retrieve_secret(secret_meta_data.index,
+				buf, secret_meta_data.length);
+	if (rc)
+		return rc;
+
+	*secret_type = secret_meta_data.type;
+	*buflen = secret_meta_data.length;
+
+	return 0;
+}
+
+static int uv_get_size_and_type(u16 secret_type, u32 *pkeysize, u32 *pkeytype)
+{
+	int rc = 0;
+
+	switch (secret_type) {
+	case UV_SECRET_AES_128:
+		*pkeysize = 16 + AES_WK_VP_SIZE;
+		*pkeytype = PKEY_KEYTYPE_AES_128;
+		break;
+	case UV_SECRET_AES_192:
+		*pkeysize = 24 + AES_WK_VP_SIZE;
+		*pkeytype = PKEY_KEYTYPE_AES_192;
+		break;
+	case UV_SECRET_AES_256:
+		*pkeysize = 32 + AES_WK_VP_SIZE;
+		*pkeytype = PKEY_KEYTYPE_AES_256;
+		break;
+	case UV_SECRET_AES_XTS_128:
+		*pkeysize = 16 + 16 + AES_WK_VP_SIZE;
+		*pkeytype = PKEY_KEYTYPE_AES_XTS_128;
+		break;
+	case UV_SECRET_AES_XTS_256:
+		*pkeysize = 32 + 32 + AES_WK_VP_SIZE;
+		*pkeytype = PKEY_KEYTYPE_AES_XTS_256;
+		break;
+	case UV_SECRET_HMAC_SHA_256:
+		*pkeysize = 64 + AES_WK_VP_SIZE;
+		*pkeytype = PKEY_KEYTYPE_HMAC_512;
+		break;
+	case UV_SECRET_HMAC_SHA_512:
+		*pkeysize = 128 + AES_WK_VP_SIZE;
+		*pkeytype = PKEY_KEYTYPE_HMAC_1024;
+		break;
+	case UV_SECRET_ECDSA_P256:
+		*pkeysize = 32 + AES_WK_VP_SIZE;
+		*pkeytype = PKEY_KEYTYPE_ECC_P256;
+		break;
+	case UV_SECRET_ECDSA_P384:
+		*pkeysize = 48 + AES_WK_VP_SIZE;
+		*pkeytype = PKEY_KEYTYPE_ECC_P384;
+		break;
+	case UV_SECRET_ECDSA_P521:
+		*pkeysize = 80 + AES_WK_VP_SIZE;
+		*pkeytype = PKEY_KEYTYPE_ECC_P521;
+		break;
+	case UV_SECRET_ECDSA_ED25519:
+		*pkeysize = 32 + AES_WK_VP_SIZE;
+		*pkeytype = PKEY_KEYTYPE_ECC_ED25519;
+		break;
+	case UV_SECRET_ECDSA_ED448:
+		*pkeysize = 64 + AES_WK_VP_SIZE;
+		*pkeytype = PKEY_KEYTYPE_ECC_ED448;
+		break;
+	default:
+		rc = -EINVAL;
+	}
+
+	return rc;
+}
+
+static int uv_key2protkey(const struct pkey_apqn *_apqns __always_unused,
+			  size_t _nr_apqns __always_unused,
+			  const u8 *key, u32 keylen,
+			  u8 *protkey, u32 *protkeylen, u32 *keyinfo,
+			  u32 _xflags __always_unused)
+{
+	struct uvsecrettoken *t = (struct uvsecrettoken *)key;
+	u32 pkeysize, pkeytype;
+	u16 secret_type;
+	int rc;
+
+	rc = uv_get_size_and_type(t->secret_type, &pkeysize, &pkeytype);
+	if (rc)
+		goto out;
+
+	if (*protkeylen < pkeysize) {
+		PKEY_DBF_ERR("%s prot key buffer size too small: %u < %u\n",
+			     __func__, *protkeylen, pkeysize);
+		rc = -EINVAL;
+		goto out;
+	}
+
+	rc = retrieve_secret(t->secret_id, &secret_type, protkey, protkeylen);
+	if (rc) {
+		PKEY_DBF_ERR("%s retrieve_secret() failed with %d\n",
+			     __func__, rc);
+		goto out;
+	}
+	if (secret_type != t->secret_type) {
+		PKEY_DBF_ERR("%s retrieved secret type %u != expected type %u\n",
+			     __func__, secret_type, t->secret_type);
+		rc = -EINVAL;
+		goto out;
+	}
+
+	if (keyinfo)
+		*keyinfo = pkeytype;
+
+out:
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+
+static int uv_verifykey(const u8 *key, u32 keylen,
+			u16 *_card __always_unused,
+			u16 *_dom __always_unused,
+			u32 *keytype, u32 *keybitsize, u32 *flags,
+			u32 xflags __always_unused)
+{
+	struct uvsecrettoken *t = (struct uvsecrettoken *)key;
+	struct uv_secret_list_item_hdr secret_meta_data;
+	u32 pkeysize, pkeytype, bitsize;
+	int rc;
+
+	rc = uv_get_size_and_type(t->secret_type, &pkeysize, &pkeytype);
+	if (rc)
+		goto out;
+
+	rc = get_secret_metadata(t->secret_id, &secret_meta_data);
+	if (rc)
+		goto out;
+
+	if (secret_meta_data.type != t->secret_type) {
+		rc = -EINVAL;
+		goto out;
+	}
+
+	/* set keytype; keybitsize and flags are not supported */
+	if (keytype)
+		*keytype = PKEY_TYPE_UVSECRET;
+	if (keybitsize) {
+		bitsize = 8 * pkey_keytype_to_size(pkeytype);
+		*keybitsize = bitsize ?: PKEY_SIZE_UNKNOWN;
+	}
+	if (flags)
+		*flags = pkeytype;
+
+out:
+	pr_debug("rc=%d\n", rc);
+	return rc;
+}
+
+static struct pkey_handler uv_handler = {
+	.module			 = THIS_MODULE,
+	.name			 = "PKEY UV handler",
+	.is_supported_key	 = is_uv_key,
+	.is_supported_keytype	 = is_uv_keytype,
+	.key_to_protkey		 = uv_key2protkey,
+	.verify_key		 = uv_verifykey,
+};
+
+/*
+ * Module init
+ */
+static int __init pkey_uv_init(void)
+{
+	int rc;
+
+	if (!is_prot_virt_guest())
+		return -ENODEV;
+
+	if (!test_bit_inv(BIT_UVC_CMD_RETR_SECRET, uv_info.inst_calls_list))
+		return -ENODEV;
+
+	uv_list = kmalloc_obj(*uv_list);
+	if (!uv_list)
+		return -ENOMEM;
+
+	rc = pkey_handler_register(&uv_handler);
+	if (rc)
+		kfree(uv_list);
+
+	return rc;
+}
+
+/*
+ * Module exit
+ */
+static void __exit pkey_uv_exit(void)
+{
+	pkey_handler_unregister(&uv_handler);
+	mutex_lock(&uv_list_mutex);
+	kvfree(uv_list);
+	mutex_unlock(&uv_list_mutex);
+}
+
+module_cpu_feature_match(S390_CPU_FEATURE_UV, pkey_uv_init);
+module_exit(pkey_uv_exit);
diff --git a/drivers/s390/crypto/vfio_ap_debug.h b/drivers/s390/crypto/vfio_ap_debug.h
new file mode 100644
index 000000000..180156121
--- /dev/null
+++ b/drivers/s390/crypto/vfio_ap_debug.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ *  Copyright IBM Corp. 2022
+ *
+ * Author(s): Tony Krowiak <akrowiak@linux.ibm.com>
+ */
+#ifndef VFIO_AP_DEBUG_H
+#define VFIO_AP_DEBUG_H
+
+#include <asm/debug.h>
+
+#define DBF_ERR		3	/* error conditions   */
+#define DBF_WARN	4	/* warning conditions */
+#define DBF_INFO	5	/* informational      */
+#define DBF_DEBUG	6	/* for debugging only */
+
+#define DBF_MAX_SPRINTF_ARGS 10
+
+#define VFIO_AP_DBF(...)					\
+	debug_sprintf_event(vfio_ap_dbf_info, ##__VA_ARGS__)
+#define VFIO_AP_DBF_ERR(...)					\
+	debug_sprintf_event(vfio_ap_dbf_info, DBF_ERR, ##__VA_ARGS__)
+#define VFIO_AP_DBF_WARN(...)					\
+	debug_sprintf_event(vfio_ap_dbf_info, DBF_WARN, ##__VA_ARGS__)
+#define VFIO_AP_DBF_INFO(...)					\
+	debug_sprintf_event(vfio_ap_dbf_info, DBF_INFO, ##__VA_ARGS__)
+#define VFIO_AP_DBF_DBG(...)					\
+	debug_sprintf_event(vfio_ap_dbf_info, DBF_DEBUG, ##__VA_ARGS__)
+
+extern debug_info_t *vfio_ap_dbf_info;
+
+#endif /* VFIO_AP_DEBUG_H */
diff --git a/drivers/s390/crypto/vfio_ap_drv.c b/drivers/s390/crypto/vfio_ap_drv.c
new file mode 100644
index 000000000..8e69ed286
--- /dev/null
+++ b/drivers/s390/crypto/vfio_ap_drv.c
@@ -0,0 +1,209 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * VFIO based AP device driver
+ *
+ * Copyright IBM Corp. 2018
+ *
+ * Author(s): Tony Krowiak <akrowiak@linux.ibm.com>
+ *	      Pierre Morel <pmorel@linux.ibm.com>
+ */
+
+#include <linux/module.h>
+#include <linux/device-id/ap.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <asm/facility.h>
+#include "vfio_ap_private.h"
+#include "vfio_ap_debug.h"
+
+#define VFIO_AP_ROOT_NAME "vfio_ap"
+#define VFIO_AP_DEV_NAME "matrix"
+
+MODULE_AUTHOR("IBM Corporation");
+MODULE_DESCRIPTION("VFIO AP device driver, Copyright IBM Corp. 2018");
+MODULE_LICENSE("GPL v2");
+
+struct ap_matrix_dev *matrix_dev;
+debug_info_t *vfio_ap_dbf_info;
+
+static ssize_t features_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	return sysfs_emit(buf, "guest_matrix hotplug ap_config\n");
+}
+static DEVICE_ATTR_RO(features);
+
+static struct attribute *matrix_dev_attrs[] = {
+	&dev_attr_features.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(matrix_dev);
+
+/* Only type 10 adapters (CEX4 and later) are supported
+ * by the AP matrix device driver
+ */
+static struct ap_device_id ap_queue_ids[] = {
+	{ .dev_type = AP_DEVICE_TYPE_CEX4,
+	  .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
+	{ .dev_type = AP_DEVICE_TYPE_CEX5,
+	  .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
+	{ .dev_type = AP_DEVICE_TYPE_CEX6,
+	  .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
+	{ .dev_type = AP_DEVICE_TYPE_CEX7,
+	  .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
+	{ .dev_type = AP_DEVICE_TYPE_CEX8,
+	  .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
+	{ /* end of sibling */ },
+};
+
+static struct ap_driver vfio_ap_drv = {
+	.probe = vfio_ap_mdev_probe_queue,
+	.remove = vfio_ap_mdev_remove_queue,
+	.in_use = vfio_ap_mdev_resource_in_use,
+	.on_config_changed = vfio_ap_on_cfg_changed,
+	.on_scan_complete = vfio_ap_on_scan_complete,
+	.ids = ap_queue_ids,
+};
+
+static void vfio_ap_matrix_dev_release(struct device *dev)
+{
+	struct ap_matrix_dev *matrix_dev;
+
+	matrix_dev = container_of(dev, struct ap_matrix_dev, device);
+	kfree(matrix_dev);
+}
+
+static const struct bus_type matrix_bus = {
+	.name = "matrix",
+};
+
+static struct device_driver matrix_driver = {
+	.name = "vfio_ap",
+	.bus = &matrix_bus,
+	.suppress_bind_attrs = true,
+	.dev_groups = matrix_dev_groups,
+};
+
+static int vfio_ap_matrix_dev_create(void)
+{
+	int ret;
+	struct device *root_device;
+
+	root_device = root_device_register(VFIO_AP_ROOT_NAME);
+	if (IS_ERR(root_device))
+		return PTR_ERR(root_device);
+
+	ret = bus_register(&matrix_bus);
+	if (ret)
+		goto bus_register_err;
+
+	matrix_dev = kzalloc_obj(*matrix_dev);
+	if (!matrix_dev) {
+		ret = -ENOMEM;
+		goto matrix_alloc_err;
+	}
+
+	/* Fill in config info via PQAP(QCI), if available */
+	if (test_facility(12)) {
+		ret = ap_qci(&matrix_dev->info);
+		if (ret)
+			goto matrix_alloc_err;
+	}
+
+	mutex_init(&matrix_dev->mdevs_lock);
+	INIT_LIST_HEAD(&matrix_dev->mdev_list);
+	mutex_init(&matrix_dev->guests_lock);
+
+	dev_set_name(&matrix_dev->device, "%s", VFIO_AP_DEV_NAME);
+	matrix_dev->device.parent = root_device;
+	matrix_dev->device.bus = &matrix_bus;
+	matrix_dev->device.release = vfio_ap_matrix_dev_release;
+	matrix_dev->vfio_ap_drv = &vfio_ap_drv;
+
+	ret = device_register(&matrix_dev->device);
+	if (ret)
+		goto matrix_reg_err;
+
+	ret = driver_register(&matrix_driver);
+	if (ret)
+		goto matrix_drv_err;
+
+	return 0;
+
+matrix_drv_err:
+	device_del(&matrix_dev->device);
+matrix_reg_err:
+	put_device(&matrix_dev->device);
+matrix_alloc_err:
+	bus_unregister(&matrix_bus);
+bus_register_err:
+	root_device_unregister(root_device);
+	return ret;
+}
+
+static void vfio_ap_matrix_dev_destroy(void)
+{
+	struct device *root_device = matrix_dev->device.parent;
+
+	driver_unregister(&matrix_driver);
+	device_unregister(&matrix_dev->device);
+	bus_unregister(&matrix_bus);
+	root_device_unregister(root_device);
+}
+
+static int __init vfio_ap_dbf_info_init(void)
+{
+	vfio_ap_dbf_info = debug_register("vfio_ap", 1, 1,
+					  DBF_MAX_SPRINTF_ARGS * sizeof(long));
+
+	if (!vfio_ap_dbf_info)
+		return -ENOENT;
+
+	debug_register_view(vfio_ap_dbf_info, &debug_sprintf_view);
+	debug_set_level(vfio_ap_dbf_info, DBF_WARN);
+
+	return 0;
+}
+
+static int __init vfio_ap_init(void)
+{
+	int ret;
+
+	ret = vfio_ap_dbf_info_init();
+	if (ret)
+		return ret;
+
+	/* If there are no AP instructions, there is nothing to pass through. */
+	if (!ap_instructions_available())
+		return -ENODEV;
+
+	ret = vfio_ap_matrix_dev_create();
+	if (ret)
+		return ret;
+
+	ret = ap_driver_register(&vfio_ap_drv, THIS_MODULE, VFIO_AP_DRV_NAME);
+	if (ret) {
+		vfio_ap_matrix_dev_destroy();
+		return ret;
+	}
+
+	ret = vfio_ap_mdev_register();
+	if (ret) {
+		ap_driver_unregister(&vfio_ap_drv);
+		vfio_ap_matrix_dev_destroy();
+
+		return ret;
+	}
+
+	return 0;
+}
+
+static void __exit vfio_ap_exit(void)
+{
+	vfio_ap_mdev_unregister();
+	ap_driver_unregister(&vfio_ap_drv);
+	vfio_ap_matrix_dev_destroy();
+	debug_unregister(vfio_ap_dbf_info);
+}
+
+module_init(vfio_ap_init);
+module_exit(vfio_ap_exit);
diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
new file mode 100644
index 000000000..4db878c18
--- /dev/null
+++ b/drivers/s390/crypto/vfio_ap_ops.c
@@ -0,0 +1,2960 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Adjunct processor matrix VFIO device driver callbacks.
+ *
+ * Copyright IBM Corp. 2018
+ *
+ * Author(s): Tony Krowiak <akrowiak@linux.ibm.com>
+ *	      Halil Pasic <pasic@linux.ibm.com>
+ *	      Pierre Morel <pmorel@linux.ibm.com>
+ */
+#include <linux/string.h>
+#include <linux/vfio.h>
+#include <linux/device.h>
+#include <linux/list.h>
+#include <linux/ctype.h>
+#include <linux/bitops.h>
+#include <linux/kvm_host.h>
+#include <linux/module.h>
+#include <linux/uuid.h>
+#include <asm/kvm.h>
+#include <asm/zcrypt.h>
+
+#include "vfio_ap_private.h"
+#include "vfio_ap_debug.h"
+
+#define VFIO_AP_MDEV_TYPE_HWVIRT "passthrough"
+#define VFIO_AP_MDEV_NAME_HWVIRT "VFIO AP Passthrough Device"
+
+#define AP_QUEUE_ASSIGNED "assigned"
+#define AP_QUEUE_UNASSIGNED "unassigned"
+#define AP_QUEUE_IN_USE "in use"
+
+#define AP_RESET_INTERVAL		20	/* Reset sleep interval (20ms)		*/
+
+static int vfio_ap_mdev_reset_queues(struct ap_matrix_mdev *matrix_mdev);
+static int vfio_ap_mdev_reset_qlist(struct list_head *qlist);
+static struct vfio_ap_queue *vfio_ap_find_queue(int apqn);
+static const struct vfio_device_ops vfio_ap_matrix_dev_ops;
+static void vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q);
+
+/**
+ * get_update_locks_for_kvm: Acquire the locks required to dynamically update a
+ *			     KVM guest's APCB in the proper order.
+ *
+ * @kvm: a pointer to a struct kvm object containing the KVM guest's APCB.
+ *
+ * The proper locking order is:
+ * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM
+ *			       guest's APCB.
+ * 2. kvm->lock:	       required to update a guest's APCB
+ * 3. kvm->arch.crypto.pqap_hook_rwsem: required to update pqap_hook and
+ *					serialize against PQAP intercepts
+ * 4. matrix_dev->mdevs_lock:  required to access data stored in a matrix_mdev
+ *
+ * Note: If @kvm is NULL, the KVM lock and pqap_hook_rwsem will not be taken.
+ */
+static inline void get_update_locks_for_kvm(struct kvm *kvm)
+{
+	mutex_lock(&matrix_dev->guests_lock);
+	if (kvm) {
+		mutex_lock(&kvm->lock);
+		down_write(&kvm->arch.crypto.pqap_hook_rwsem);
+	}
+	mutex_lock(&matrix_dev->mdevs_lock);
+}
+
+/**
+ * release_update_locks_for_kvm: Release the locks used to dynamically update a
+ *				 KVM guest's APCB in the proper order.
+ *
+ * @kvm: a pointer to a struct kvm object containing the KVM guest's APCB.
+ *
+ * The proper unlocking order is:
+ * 1. matrix_dev->mdevs_lock
+ * 2. kvm->arch.crypto.pqap_hook_rwsem
+ * 3. kvm->lock
+ * 4. matrix_dev->guests_lock
+ *
+ * Note: If @kvm is NULL, the KVM lock and pqap_hook_rwsem will not be released.
+ */
+static inline void release_update_locks_for_kvm(struct kvm *kvm)
+{
+	mutex_unlock(&matrix_dev->mdevs_lock);
+	if (kvm) {
+		up_write(&kvm->arch.crypto.pqap_hook_rwsem);
+		mutex_unlock(&kvm->lock);
+	}
+	mutex_unlock(&matrix_dev->guests_lock);
+}
+
+/**
+ * get_update_locks_for_mdev: Acquire the locks required to dynamically update a
+ *			      KVM guest's APCB in the proper order.
+ *
+ * @matrix_mdev: a pointer to a struct ap_matrix_mdev object containing the AP
+ *		 configuration data to use to update a KVM guest's APCB.
+ *
+ * The proper locking order is:
+ * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM
+ *			       guest's APCB.
+ * 2. matrix_mdev->kvm->lock:  required to update a guest's APCB
+ * 3. matrix_dev->mdevs_lock:  required to access data stored in a matrix_mdev
+ *
+ * Note: If @matrix_mdev is NULL or is not attached to a KVM guest, the KVM
+ *	 lock will not be taken.
+ */
+static inline void get_update_locks_for_mdev(struct ap_matrix_mdev *matrix_mdev)
+{
+	mutex_lock(&matrix_dev->guests_lock);
+	if (matrix_mdev && matrix_mdev->kvm)
+		mutex_lock(&matrix_mdev->kvm->lock);
+	mutex_lock(&matrix_dev->mdevs_lock);
+}
+
+/**
+ * release_update_locks_for_mdev: Release the locks used to dynamically update a
+ *				  KVM guest's APCB in the proper order.
+ *
+ * @matrix_mdev: a pointer to a struct ap_matrix_mdev object containing the AP
+ *		 configuration data to use to update a KVM guest's APCB.
+ *
+ * The proper unlocking order is:
+ * 1. matrix_dev->mdevs_lock
+ * 2. matrix_mdev->kvm->lock
+ * 3. matrix_dev->guests_lock
+ *
+ * Note: If @matrix_mdev is NULL or is not attached to a KVM guest, the KVM
+ *	 lock will not be released.
+ */
+static inline void release_update_locks_for_mdev(struct ap_matrix_mdev *matrix_mdev)
+{
+	mutex_unlock(&matrix_dev->mdevs_lock);
+	if (matrix_mdev && matrix_mdev->kvm)
+		mutex_unlock(&matrix_mdev->kvm->lock);
+	mutex_unlock(&matrix_dev->guests_lock);
+}
+
+/**
+ * get_update_locks_by_apqn: Find the mdev to which an APQN is assigned and
+ *			     acquire the locks required to update the APCB of
+ *			     the KVM guest to which the mdev is attached.
+ *
+ * @apqn: the APQN of a queue device.
+ *
+ * The proper locking order is:
+ * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM
+ *			       guest's APCB.
+ * 2. matrix_mdev->kvm->lock:  required to update a guest's APCB
+ * 3. matrix_dev->mdevs_lock:  required to access data stored in a matrix_mdev
+ *
+ * Note: If @apqn is not assigned to a matrix_mdev, the matrix_mdev->kvm->lock
+ *	 will not be taken.
+ *
+ * Return: the ap_matrix_mdev object to which @apqn is assigned or NULL if @apqn
+ *	   is not assigned to an ap_matrix_mdev.
+ */
+static struct ap_matrix_mdev *get_update_locks_by_apqn(int apqn)
+{
+	struct ap_matrix_mdev *matrix_mdev;
+
+	mutex_lock(&matrix_dev->guests_lock);
+
+	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
+		if (test_bit_inv(AP_QID_CARD(apqn), matrix_mdev->matrix.apm) &&
+		    test_bit_inv(AP_QID_QUEUE(apqn), matrix_mdev->matrix.aqm)) {
+			if (matrix_mdev->kvm)
+				mutex_lock(&matrix_mdev->kvm->lock);
+
+			mutex_lock(&matrix_dev->mdevs_lock);
+
+			return matrix_mdev;
+		}
+	}
+
+	mutex_lock(&matrix_dev->mdevs_lock);
+
+	return NULL;
+}
+
+/**
+ * get_update_locks_for_queue: get the locks required to update the APCB of the
+ *			       KVM guest to which the matrix mdev linked to a
+ *			       vfio_ap_queue object is attached.
+ *
+ * @q: a pointer to a vfio_ap_queue object.
+ *
+ * The proper locking order is:
+ * 1. q->matrix_dev->guests_lock: required to use the KVM pointer to update a
+ *				  KVM guest's APCB.
+ * 2. q->matrix_mdev->kvm->lock:  required to update a guest's APCB
+ * 3. matrix_dev->mdevs_lock:	  required to access data stored in matrix_mdev
+ *
+ * Note: if @queue is not linked to an ap_matrix_mdev object, the KVM lock
+ *	  will not be taken.
+ */
+static inline void get_update_locks_for_queue(struct vfio_ap_queue *q)
+{
+	mutex_lock(&matrix_dev->guests_lock);
+	if (q->matrix_mdev && q->matrix_mdev->kvm)
+		mutex_lock(&q->matrix_mdev->kvm->lock);
+	mutex_lock(&matrix_dev->mdevs_lock);
+}
+
+/**
+ * vfio_ap_mdev_get_queue - retrieve a queue with a specific APQN from a
+ *			    hash table of queues assigned to a matrix mdev
+ * @matrix_mdev: the matrix mdev
+ * @apqn: The APQN of a queue device
+ *
+ * Return: the pointer to the vfio_ap_queue struct representing the queue or
+ *	   NULL if the queue is not assigned to @matrix_mdev
+ */
+static struct vfio_ap_queue *vfio_ap_mdev_get_queue(
+					struct ap_matrix_mdev *matrix_mdev,
+					int apqn)
+{
+	struct vfio_ap_queue *q;
+
+	hash_for_each_possible(matrix_mdev->qtable.queues, q, mdev_qnode,
+			       apqn) {
+		if (q && q->apqn == apqn)
+			return q;
+	}
+
+	return NULL;
+}
+
+/**
+ * vfio_ap_wait_for_irqclear - clears the IR bit or gives up after 5 tries
+ * @apqn: The AP Queue number
+ *
+ * Checks the IRQ bit for the status of this APQN using ap_tapq.
+ * Returns if the ap_tapq function succeeded and the bit is clear.
+ * Returns if ap_tapq function failed with invalid, deconfigured or
+ * checkstopped AP.
+ * Otherwise retries up to 5 times after waiting 20ms.
+ */
+static void vfio_ap_wait_for_irqclear(int apqn)
+{
+	struct ap_queue_status status;
+	int retry = 5;
+
+	do {
+		status = ap_tapq(apqn, NULL);
+		switch (status.response_code) {
+		case AP_RESPONSE_NORMAL:
+		case AP_RESPONSE_RESET_IN_PROGRESS:
+			if (!status.irq_enabled)
+				return;
+			fallthrough;
+		case AP_RESPONSE_BUSY:
+			msleep(20);
+			break;
+		case AP_RESPONSE_Q_NOT_AVAIL:
+		case AP_RESPONSE_DECONFIGURED:
+		case AP_RESPONSE_CHECKSTOPPED:
+		default:
+			WARN_ONCE(1, "%s: tapq rc %02x: %04x\n", __func__,
+				  status.response_code, apqn);
+			return;
+		}
+	} while (--retry);
+
+	WARN_ONCE(1, "%s: tapq rc %02x: %04x could not clear IR bit\n",
+		  __func__, status.response_code, apqn);
+}
+
+/**
+ * vfio_ap_free_aqic_resources - free vfio_ap_queue resources
+ * @q: The vfio_ap_queue
+ *
+ * Unregisters the ISC in the GIB when the saved ISC not invalid.
+ * Unpins the guest's page holding the NIB when it exists.
+ * Resets the saved_iova and saved_isc to invalid values.
+ */
+static void vfio_ap_free_aqic_resources(struct vfio_ap_queue *q)
+{
+	if (!q)
+		return;
+	if (q->saved_isc != VFIO_AP_ISC_INVALID) {
+		if (!WARN_ON(!q->matrix_mdev) && q->matrix_mdev->kvm)
+			kvm_s390_gisc_unregister(q->matrix_mdev->kvm, q->saved_isc);
+		q->saved_isc = VFIO_AP_ISC_INVALID;
+	}
+	if (q->saved_iova && !WARN_ON(!q->matrix_mdev)) {
+		vfio_unpin_pages(&q->matrix_mdev->vdev, q->saved_iova, 1);
+		q->saved_iova = 0;
+	}
+}
+
+/**
+ * vfio_ap_irq_disable - disables and clears an ap_queue interrupt
+ * @q: The vfio_ap_queue
+ *
+ * Uses ap_aqic to disable the interruption and in case of success, reset
+ * in progress or IRQ disable command already proceeded: calls
+ * vfio_ap_wait_for_irqclear() to check for the IRQ bit to be clear
+ * and calls vfio_ap_free_aqic_resources() to free the resources associated
+ * with the AP interrupt handling.
+ *
+ * In the case the AP is busy, or a reset is in progress,
+ * retries after 20ms, up to 5 times.
+ *
+ * Returns if ap_aqic function failed with invalid, deconfigured or
+ * checkstopped AP.
+ *
+ * Return: &struct ap_queue_status
+ */
+static struct ap_queue_status vfio_ap_irq_disable(struct vfio_ap_queue *q)
+{
+	union ap_qirq_ctrl aqic_gisa = { .value = 0 };
+	struct ap_queue_status status;
+	int retries = 5;
+
+	do {
+		status = ap_aqic(q->apqn, aqic_gisa, 0);
+		switch (status.response_code) {
+		case AP_RESPONSE_OTHERWISE_CHANGED:
+		case AP_RESPONSE_NORMAL:
+			vfio_ap_wait_for_irqclear(q->apqn);
+			goto end_free;
+		case AP_RESPONSE_RESET_IN_PROGRESS:
+		case AP_RESPONSE_BUSY:
+			msleep(20);
+			break;
+		case AP_RESPONSE_Q_NOT_AVAIL:
+		case AP_RESPONSE_DECONFIGURED:
+		case AP_RESPONSE_CHECKSTOPPED:
+		case AP_RESPONSE_INVALID_ADDRESS:
+		default:
+			/* All cases in default means AP not operational */
+			WARN_ONCE(1, "%s: ap_aqic status %d\n", __func__,
+				  status.response_code);
+			goto end_free;
+		}
+	} while (retries--);
+
+	WARN_ONCE(1, "%s: ap_aqic status %d\n", __func__,
+		  status.response_code);
+end_free:
+	vfio_ap_free_aqic_resources(q);
+	return status;
+}
+
+/**
+ * vfio_ap_validate_nib - validate a notification indicator byte (nib) address.
+ *
+ * @vcpu: the object representing the vcpu executing the PQAP(AQIC) instruction.
+ * @nib: the location for storing the nib address.
+ *
+ * When the PQAP(AQIC) instruction is executed, general register 2 contains the
+ * address of the notification indicator byte (nib) used for IRQ notification.
+ * This function parses and validates the nib from gr2.
+ *
+ * Return: returns zero if the nib address is a valid; otherwise, returns
+ *	   -EINVAL.
+ */
+static int vfio_ap_validate_nib(struct kvm_vcpu *vcpu, dma_addr_t *nib)
+{
+	*nib = vcpu->run->s.regs.gprs[2];
+
+	if (!*nib)
+		return -EINVAL;
+	if (!kvm_s390_is_gpa_in_memslot(vcpu->kvm, *nib))
+		return -EINVAL;
+
+	return 0;
+}
+
+/**
+ * ensure_nib_shared() - Ensure the address of the NIB is secure and shared
+ * @addr: the physical (absolute) address of the NIB
+ *
+ * This function checks whether the NIB page, which has been pinned with
+ * vfio_pin_pages(), is a shared page belonging to a secure guest.
+ *
+ * It will call uv_pin_shared() on it; if the page was already pinned shared
+ * (i.e. if the NIB belongs to a secure guest and is shared), then 0
+ * (success) is returned. If the NIB was not shared, vfio_pin_pages() had
+ * exported it and now it does not belong to the secure guest anymore. In
+ * that case, an error is returned.
+ *
+ * Context: the NIB (at physical address @addr) has to be pinned with
+ *	    vfio_pin_pages() before calling this function.
+ *
+ * Return: 0 in case of success, otherwise an error < 0.
+ */
+static int ensure_nib_shared(unsigned long addr)
+{
+	/*
+	 * The nib has to be located in shared storage since guest and
+	 * host access it. vfio_pin_pages() will do a pin shared and
+	 * if that fails (possibly because it's not a shared page) it
+	 * calls export. We try to do a second pin shared here so that
+	 * the UV gives us an error code if we try to pin a non-shared
+	 * page.
+	 *
+	 * If the page is already pinned shared the UV will return a success.
+	 */
+	return uv_pin_shared(addr);
+}
+
+/**
+ * vfio_ap_irq_enable - Enable Interruption for a APQN
+ *
+ * @q:	 the vfio_ap_queue holding AQIC parameters
+ * @isc: the guest ISC to register with the GIB interface
+ * @vcpu: the vcpu object containing the registers specifying the parameters
+ *	  passed to the PQAP(AQIC) instruction.
+ *
+ * Pin the NIB saved in *q
+ * Register the guest ISC to GIB interface and retrieve the
+ * host ISC to issue the host side PQAP/AQIC
+ *
+ * status.response_code may be set to AP_RESPONSE_INVALID_ADDRESS in case the
+ * vfio_pin_pages or kvm_s390_gisc_register failed.
+ *
+ * Otherwise return the ap_queue_status returned by the ap_aqic(),
+ * all retry handling will be done by the guest.
+ *
+ * Return: &struct ap_queue_status
+ */
+static struct ap_queue_status vfio_ap_irq_enable(struct vfio_ap_queue *q,
+						 int isc,
+						 struct kvm_vcpu *vcpu)
+{
+	union ap_qirq_ctrl aqic_gisa = { .value = 0 };
+	struct ap_queue_status status = {};
+	struct kvm_s390_gisa *gisa;
+	struct page *h_page;
+	int nisc;
+	struct kvm *kvm;
+	phys_addr_t h_nib;
+	dma_addr_t nib;
+	int ret;
+
+	/* Verify that the notification indicator byte address is valid */
+	if (vfio_ap_validate_nib(vcpu, &nib)) {
+		VFIO_AP_DBF_WARN("%s: invalid NIB address: nib=%pad, apqn=%#04x\n",
+				 __func__, &nib, q->apqn);
+
+		status.response_code = AP_RESPONSE_INVALID_ADDRESS;
+		return status;
+	}
+
+	/* The pin will probably be successful even if the NIB was not shared */
+	ret = vfio_pin_pages(&q->matrix_mdev->vdev, nib, 1,
+			     IOMMU_READ | IOMMU_WRITE, &h_page);
+	switch (ret) {
+	case 1:
+		break;
+	default:
+		VFIO_AP_DBF_WARN("%s: vfio_pin_pages failed: rc=%d,"
+				 "nib=%pad, apqn=%#04x\n",
+				 __func__, ret, &nib, q->apqn);
+
+		status.response_code = AP_RESPONSE_INVALID_ADDRESS;
+		return status;
+	}
+
+	kvm = q->matrix_mdev->kvm;
+	gisa = kvm->arch.gisa_int.origin;
+
+	h_nib = page_to_phys(h_page) | (nib & ~PAGE_MASK);
+	aqic_gisa.gisc = isc;
+
+	/* NIB in non-shared storage is a rc 6 for PV guests */
+	if (kvm_s390_pv_cpu_is_protected(vcpu) &&
+	    ensure_nib_shared(h_nib & PAGE_MASK)) {
+		vfio_unpin_pages(&q->matrix_mdev->vdev, nib, 1);
+		status.response_code = AP_RESPONSE_INVALID_ADDRESS;
+		return status;
+	}
+
+	nisc = kvm_s390_gisc_register(kvm, isc);
+	if (nisc < 0) {
+		VFIO_AP_DBF_WARN("%s: gisc registration failed: nisc=%d, isc=%d, apqn=%#04x\n",
+				 __func__, nisc, isc, q->apqn);
+
+		vfio_unpin_pages(&q->matrix_mdev->vdev, nib, 1);
+		status.response_code = AP_RESPONSE_INVALID_ADDRESS;
+		return status;
+	}
+
+	aqic_gisa.isc = nisc;
+	aqic_gisa.ir = 1;
+	aqic_gisa.gisa = virt_to_phys(gisa) >> 4;
+
+	status = ap_aqic(q->apqn, aqic_gisa, h_nib);
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+		/* See if we did clear older IRQ configuration */
+		vfio_ap_free_aqic_resources(q);
+		q->saved_iova = nib;
+		q->saved_isc = isc;
+		break;
+	case AP_RESPONSE_OTHERWISE_CHANGED:
+		/* We could not modify IRQ settings: clear new configuration */
+		ret = kvm_s390_gisc_unregister(kvm, isc);
+		if (ret)
+			VFIO_AP_DBF_WARN("%s: kvm_s390_gisc_unregister: rc=%d isc=%d, apqn=%#04x\n",
+					 __func__, ret, isc, q->apqn);
+		vfio_unpin_pages(&q->matrix_mdev->vdev, nib, 1);
+		break;
+	default:
+		pr_warn("%s: apqn %04x: response: %02x\n", __func__, q->apqn,
+			status.response_code);
+		vfio_ap_irq_disable(q);
+		break;
+	}
+
+	if (status.response_code != AP_RESPONSE_NORMAL) {
+		VFIO_AP_DBF_WARN("%s: PQAP(AQIC) failed with status=%#02x: "
+				 "zone=%#x, ir=%#x, gisc=%#x, f=%#x,"
+				 "gisa=%#x, isc=%#x, apqn=%#04x\n",
+				 __func__, status.response_code,
+				 aqic_gisa.zone, aqic_gisa.ir, aqic_gisa.gisc,
+				 aqic_gisa.gf, aqic_gisa.gisa, aqic_gisa.isc,
+				 q->apqn);
+	}
+
+	return status;
+}
+
+/**
+ * vfio_ap_le_guid_to_be_uuid - convert a little endian guid array into an array
+ *				of big endian elements that can be passed by
+ *				value to an s390dbf sprintf event function to
+ *				format a UUID string.
+ *
+ * @guid: the object containing the little endian guid
+ * @uuid: a six-element array of long values that can be passed by value as
+ *	  arguments for a formatting string specifying a UUID.
+ *
+ * The S390 Debug Feature (s390dbf) allows the use of "%s" in the sprintf
+ * event functions if the memory for the passed string is available as long as
+ * the debug feature exists. Since a mediated device can be removed at any
+ * time, it's name can not be used because %s passes the reference to the string
+ * in memory and the reference will go stale once the device is removed .
+ *
+ * The s390dbf string formatting function allows a maximum of 9 arguments for a
+ * message to be displayed in the 'sprintf' view. In order to use the bytes
+ * comprising the mediated device's UUID to display the mediated device name,
+ * they will have to be converted into an array whose elements can be passed by
+ * value to sprintf. For example:
+ *
+ * guid array: { 83, 78, 17, 62, bb, f1, f0, 47, 91, 4d, 32, a2, 2e, 3a, 88, 04 }
+ * mdev name: 62177883-f1bb-47f0-914d-32a22e3a8804
+ * array returned: { 62177883, f1bb, 47f0, 914d, 32a2, 2e3a8804 }
+ * formatting string: "%08lx-%04lx-%04lx-%04lx-%02lx%04lx"
+ */
+static void vfio_ap_le_guid_to_be_uuid(guid_t *guid, unsigned long *uuid)
+{
+	/*
+	 * The input guid is ordered in little endian, so it needs to be
+	 * reordered for displaying a UUID as a string. This specifies the
+	 * guid indices in proper order.
+	 */
+	uuid[0] = le32_to_cpup((__le32 *)guid);
+	uuid[1] = le16_to_cpup((__le16 *)&guid->b[4]);
+	uuid[2] = le16_to_cpup((__le16 *)&guid->b[6]);
+	uuid[3] = *((__u16 *)&guid->b[8]);
+	uuid[4] = *((__u16 *)&guid->b[10]);
+	uuid[5] = *((__u32 *)&guid->b[12]);
+}
+
+/**
+ * handle_pqap - PQAP instruction callback
+ *
+ * @vcpu: The vcpu on which we received the PQAP instruction
+ *
+ * Get the general register contents to initialize internal variables.
+ * REG[0]: APQN
+ * REG[1]: IR and ISC
+ * REG[2]: NIB
+ *
+ * Response.status may be set to following Response Code:
+ * - AP_RESPONSE_Q_NOT_AVAIL: if the queue is not available
+ * - AP_RESPONSE_DECONFIGURED: if the queue is not configured
+ * - AP_RESPONSE_NORMAL (0) : in case of success
+ *   Check vfio_ap_setirq() and vfio_ap_clrirq() for other possible RC.
+ * We take the matrix_dev lock to ensure serialization on queues and
+ * mediated device access.
+ *
+ * Return: 0 if we could handle the request inside KVM.
+ * Otherwise, returns -EOPNOTSUPP to let QEMU handle the fault.
+ */
+static int handle_pqap(struct kvm_vcpu *vcpu)
+{
+	uint64_t status;
+	uint16_t apqn;
+	unsigned long uuid[6];
+	struct vfio_ap_queue *q;
+	struct ap_queue_status qstatus = {
+			       .response_code = AP_RESPONSE_Q_NOT_AVAIL, };
+	struct ap_matrix_mdev *matrix_mdev;
+
+	apqn = vcpu->run->s.regs.gprs[0] & 0xffff;
+
+	/* If we do not use the AIV facility just go to userland */
+	if (!(vcpu->arch.sie_block->eca & ECA_AIV)) {
+		VFIO_AP_DBF_WARN("%s: AIV facility not installed: apqn=0x%04x, eca=0x%04x\n",
+				 __func__, apqn, vcpu->arch.sie_block->eca);
+
+		return -EOPNOTSUPP;
+	}
+
+	mutex_lock(&matrix_dev->mdevs_lock);
+
+	if (!vcpu->kvm->arch.crypto.pqap_hook) {
+		VFIO_AP_DBF_WARN("%s: PQAP(AQIC) hook not registered with the vfio_ap driver: apqn=0x%04x\n",
+				 __func__, apqn);
+
+		goto out_unlock;
+	}
+
+	matrix_mdev = container_of(vcpu->kvm->arch.crypto.pqap_hook,
+				   struct ap_matrix_mdev, pqap_hook);
+
+	/* If the there is no guest using the mdev, there is nothing to do */
+	if (!matrix_mdev->kvm) {
+		vfio_ap_le_guid_to_be_uuid(&matrix_mdev->mdev->uuid, uuid);
+		VFIO_AP_DBF_WARN("%s: mdev %08lx-%04lx-%04lx-%04lx-%04lx%08lx not in use: apqn=0x%04x\n",
+				 __func__, uuid[0],  uuid[1], uuid[2],
+				 uuid[3], uuid[4], uuid[5], apqn);
+		goto out_unlock;
+	}
+
+	q = vfio_ap_mdev_get_queue(matrix_mdev, apqn);
+	if (!q) {
+		VFIO_AP_DBF_WARN("%s: Queue %02x.%04x not bound to the vfio_ap driver\n",
+				 __func__, AP_QID_CARD(apqn),
+				 AP_QID_QUEUE(apqn));
+		goto out_unlock;
+	}
+
+	status = vcpu->run->s.regs.gprs[1];
+
+	/* If IR bit(16) is set we enable the interrupt */
+	if ((status >> (63 - 16)) & 0x01)
+		qstatus = vfio_ap_irq_enable(q, status & 0x07, vcpu);
+	else
+		qstatus = vfio_ap_irq_disable(q);
+
+out_unlock:
+	memcpy(&vcpu->run->s.regs.gprs[1], &qstatus, sizeof(qstatus));
+	vcpu->run->s.regs.gprs[1] >>= 32;
+	mutex_unlock(&matrix_dev->mdevs_lock);
+	return 0;
+}
+
+static void vfio_ap_matrix_init(struct ap_config_info *info,
+				struct ap_matrix *matrix)
+{
+	matrix->apm_max = info->apxa ? info->na : 63;
+	matrix->aqm_max = info->apxa ? info->nd : 15;
+	matrix->adm_max = info->apxa ? info->nd : 15;
+}
+
+static void signal_guest_ap_cfg_changed(struct ap_matrix_mdev *matrix_mdev)
+{
+	if (matrix_mdev->cfg_chg_trigger)
+		eventfd_signal(matrix_mdev->cfg_chg_trigger);
+}
+
+static void vfio_ap_mdev_update_guest_apcb(struct ap_matrix_mdev *matrix_mdev)
+{
+	if (matrix_mdev->kvm) {
+		kvm_arch_crypto_set_masks(matrix_mdev->kvm,
+					  matrix_mdev->shadow_apcb.apm,
+					  matrix_mdev->shadow_apcb.aqm,
+					  matrix_mdev->shadow_apcb.adm);
+
+		signal_guest_ap_cfg_changed(matrix_mdev);
+	}
+}
+
+static bool vfio_ap_mdev_filter_cdoms(struct ap_matrix_mdev *matrix_mdev)
+{
+	DECLARE_BITMAP(prev_shadow_adm, AP_DOMAINS);
+
+	bitmap_copy(prev_shadow_adm, matrix_mdev->shadow_apcb.adm, AP_DOMAINS);
+	bitmap_and(matrix_mdev->shadow_apcb.adm, matrix_mdev->matrix.adm,
+		   (unsigned long *)matrix_dev->info.adm, AP_DOMAINS);
+
+	return !bitmap_equal(prev_shadow_adm, matrix_mdev->shadow_apcb.adm,
+			     AP_DOMAINS);
+}
+
+static bool _queue_passable(struct vfio_ap_queue *q)
+{
+	if (!q)
+		return false;
+
+	switch (q->reset_status.response_code) {
+	case AP_RESPONSE_NORMAL:
+	case AP_RESPONSE_DECONFIGURED:
+	case AP_RESPONSE_CHECKSTOPPED:
+		return true;
+	default:
+		return false;
+	}
+}
+
+/*
+ * vfio_ap_mdev_filter_matrix - filter the APQNs assigned to the matrix mdev
+ *				to ensure no queue devices are passed through to
+ *				the guest that are not bound to the vfio_ap
+ *				device driver.
+ *
+ * @matrix_mdev: the matrix mdev whose matrix is to be filtered.
+ * @apm_filtered: a 256-bit bitmap for storing the APIDs filtered from the
+ *		  guest's AP configuration that are still in the host's AP
+ *		  configuration.
+ *
+ * Note: If an APQN referencing a queue device that is not bound to the vfio_ap
+ *	 driver, its APID will be filtered from the guest's APCB. The matrix
+ *	 structure precludes filtering an individual APQN, so its APID will be
+ *	 filtered. Consequently, all queues associated with the adapter that
+ *	 are in the host's AP configuration must be reset. If queues are
+ *	 subsequently made available again to the guest, they should re-appear
+ *	 in a reset state
+ *
+ * Return: a boolean value indicating whether the KVM guest's APCB was changed
+ *	   by the filtering or not.
+ */
+static bool vfio_ap_mdev_filter_matrix(struct ap_matrix_mdev *matrix_mdev,
+				       unsigned long *apm_filtered)
+{
+	unsigned long apid, apqi, apqn;
+	DECLARE_BITMAP(prev_shadow_apm, AP_DEVICES);
+	DECLARE_BITMAP(prev_shadow_aqm, AP_DOMAINS);
+
+	bitmap_copy(prev_shadow_apm, matrix_mdev->shadow_apcb.apm, AP_DEVICES);
+	bitmap_copy(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm, AP_DOMAINS);
+	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb);
+	bitmap_clear(apm_filtered, 0, AP_DEVICES);
+
+	/*
+	 * Copy the adapters, domains and control domains to the shadow_apcb
+	 * from the matrix mdev, but only those that are assigned to the host's
+	 * AP configuration.
+	 */
+	bitmap_and(matrix_mdev->shadow_apcb.apm, matrix_mdev->matrix.apm,
+		   (unsigned long *)matrix_dev->info.apm, AP_DEVICES);
+	bitmap_and(matrix_mdev->shadow_apcb.aqm, matrix_mdev->matrix.aqm,
+		   (unsigned long *)matrix_dev->info.aqm, AP_DOMAINS);
+
+	for_each_set_bit_inv(apid, matrix_mdev->shadow_apcb.apm, AP_DEVICES) {
+		for_each_set_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm,
+				     AP_DOMAINS) {
+			/*
+			 * If the APQN is not bound to the vfio_ap device
+			 * driver, then we can't assign it to the guest's
+			 * AP configuration. The AP architecture won't
+			 * allow filtering of a single APQN, so let's filter
+			 * the APID since an adapter represents a physical
+			 * hardware device.
+			 */
+			apqn = AP_MKQID(apid, apqi);
+			if (!_queue_passable(vfio_ap_mdev_get_queue(matrix_mdev, apqn))) {
+				clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm);
+
+				/*
+				 * If the adapter was previously plugged into
+				 * the guest, let's let the caller know that
+				 * the APID was filtered.
+				 */
+				if (test_bit_inv(apid, prev_shadow_apm))
+					set_bit_inv(apid, apm_filtered);
+
+				break;
+			}
+		}
+	}
+
+	return !bitmap_equal(prev_shadow_apm, matrix_mdev->shadow_apcb.apm,
+			     AP_DEVICES) ||
+	       !bitmap_equal(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm,
+			     AP_DOMAINS);
+}
+
+static int vfio_ap_mdev_init_dev(struct vfio_device *vdev)
+{
+	struct ap_matrix_mdev *matrix_mdev =
+		container_of(vdev, struct ap_matrix_mdev, vdev);
+
+	matrix_mdev->mdev = to_mdev_device(vdev->dev);
+	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix);
+	matrix_mdev->pqap_hook = handle_pqap;
+	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb);
+	hash_init(matrix_mdev->qtable.queues);
+
+	return 0;
+}
+
+static int vfio_ap_mdev_probe(struct mdev_device *mdev)
+{
+	struct ap_matrix_mdev *matrix_mdev;
+	int ret;
+
+	matrix_mdev = vfio_alloc_device(ap_matrix_mdev, vdev, &mdev->dev,
+					&vfio_ap_matrix_dev_ops);
+	if (IS_ERR(matrix_mdev))
+		return PTR_ERR(matrix_mdev);
+
+	ret = vfio_register_emulated_iommu_dev(&matrix_mdev->vdev);
+	if (ret)
+		goto err_put_vdev;
+
+	/*
+	 * Take the matrix_dev->guests_lock mutex before adding the matrix_mdev
+	 * to the mdev_list. All functions that traverse the list must also hold
+	 * this lock to guard against additions to or removals from the list
+	 * while it is being traversed.
+	 */
+	mutex_lock(&matrix_dev->guests_lock);
+	dev_set_drvdata(&mdev->dev, matrix_mdev);
+	list_add(&matrix_mdev->node, &matrix_dev->mdev_list);
+	mutex_unlock(&matrix_dev->guests_lock);
+	return 0;
+
+err_put_vdev:
+	vfio_put_device(&matrix_mdev->vdev);
+	return ret;
+}
+
+static void vfio_ap_mdev_link_queue(struct ap_matrix_mdev *matrix_mdev,
+				    struct vfio_ap_queue *q)
+{
+	if (!q || vfio_ap_mdev_get_queue(matrix_mdev, q->apqn))
+		return;
+
+	q->matrix_mdev = matrix_mdev;
+	hash_add(matrix_mdev->qtable.queues, &q->mdev_qnode, q->apqn);
+}
+
+static void vfio_ap_mdev_link_apqn(struct ap_matrix_mdev *matrix_mdev, int apqn)
+{
+	struct vfio_ap_queue *q;
+
+	q = vfio_ap_find_queue(apqn);
+	vfio_ap_mdev_link_queue(matrix_mdev, q);
+}
+
+static void vfio_ap_unlink_queue_fr_mdev(struct vfio_ap_queue *q)
+{
+	hash_del(&q->mdev_qnode);
+}
+
+static void vfio_ap_unlink_mdev_fr_queue(struct vfio_ap_queue *q)
+{
+	q->matrix_mdev = NULL;
+}
+
+static void vfio_ap_mdev_unlink_fr_queues(struct ap_matrix_mdev *matrix_mdev)
+{
+	struct vfio_ap_queue *q;
+	unsigned long apid, apqi;
+
+	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {
+		for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm,
+				     AP_DOMAINS) {
+			q = vfio_ap_mdev_get_queue(matrix_mdev,
+						   AP_MKQID(apid, apqi));
+			if (q)
+				q->matrix_mdev = NULL;
+		}
+	}
+}
+
+static void vfio_ap_mdev_remove(struct mdev_device *mdev)
+{
+	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(&mdev->dev);
+
+	vfio_unregister_group_dev(&matrix_mdev->vdev);
+
+	mutex_lock(&matrix_dev->guests_lock);
+	mutex_lock(&matrix_dev->mdevs_lock);
+	vfio_ap_mdev_reset_queues(matrix_mdev);
+	vfio_ap_mdev_unlink_fr_queues(matrix_mdev);
+	list_del(&matrix_mdev->node);
+	mutex_unlock(&matrix_dev->mdevs_lock);
+	mutex_unlock(&matrix_dev->guests_lock);
+	vfio_put_device(&matrix_mdev->vdev);
+}
+
+#define MDEV_SHARING_ERR "Userspace may not assign queue %02lx.%04lx to mdev: already assigned to %s"
+
+#define MDEV_IN_USE_ERR "Can not reserve queue %02lx.%04lx for host driver: in use by mdev"
+
+static void vfio_ap_mdev_log_sharing_err(struct ap_matrix_mdev *assignee,
+					 struct ap_matrix_mdev *assigned_to,
+					 unsigned long *apm, unsigned long *aqm)
+{
+	unsigned long apid, apqi;
+
+	for_each_set_bit_inv(apid, apm, AP_DEVICES) {
+		for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) {
+			dev_warn(mdev_dev(assignee->mdev), MDEV_SHARING_ERR,
+				 apid, apqi, dev_name(mdev_dev(assigned_to->mdev)));
+		}
+	}
+}
+
+static void vfio_ap_mdev_log_in_use_err(struct ap_matrix_mdev *assignee,
+					unsigned long *apm, unsigned long *aqm)
+{
+	unsigned long apid, apqi;
+
+	for_each_set_bit_inv(apid, apm, AP_DEVICES) {
+		for_each_set_bit_inv(apqi, aqm, AP_DOMAINS)
+			dev_warn(mdev_dev(assignee->mdev), MDEV_IN_USE_ERR, apid, apqi);
+	}
+}
+
+/**
+ * vfio_ap_mdev_verify_no_sharing - verify APQNs are not shared by matrix mdevs
+ *
+ * @assignee: the matrix mdev to which @mdev_apm and @mdev_aqm are being
+ *	      assigned; or, NULL if this function was called by the AP bus
+ *	      driver in_use callback to verify none of the APQNs being reserved
+ *	      for the host device driver are in use by a vfio_ap mediated device
+ * @mdev_apm: mask indicating the APIDs of the APQNs to be verified
+ * @mdev_aqm: mask indicating the APQIs of the APQNs to be verified
+ *
+ * Verifies that each APQN derived from the Cartesian product of APIDs
+ * represented by the bits set in @mdev_apm and the APQIs of the bits set in
+ * @mdev_aqm is not assigned to a mediated device other than the mdev to which
+ * the APQN is being assigned (@assignee). AP queue sharing is not allowed.
+ *
+ * Return: 0 if the APQNs are not shared; otherwise return -EADDRINUSE.
+ */
+static int vfio_ap_mdev_verify_no_sharing(struct ap_matrix_mdev *assignee,
+					  unsigned long *mdev_apm,
+					  unsigned long *mdev_aqm)
+{
+	struct ap_matrix_mdev *assigned_to;
+	DECLARE_BITMAP(apm, AP_DEVICES);
+	DECLARE_BITMAP(aqm, AP_DOMAINS);
+
+	list_for_each_entry(assigned_to, &matrix_dev->mdev_list, node) {
+		/*
+		 * If the mdev to which the mdev_apm and mdev_aqm is being
+		 * assigned is the same as the mdev being verified
+		 */
+		if (assignee == assigned_to)
+			continue;
+
+		memset(apm, 0, sizeof(apm));
+		memset(aqm, 0, sizeof(aqm));
+
+		/*
+		 * We work on full longs, as we can only exclude the leftover
+		 * bits in non-inverse order. The leftover is all zeros.
+		 */
+		if (!bitmap_and(apm, mdev_apm, assigned_to->matrix.apm,	AP_DEVICES))
+			continue;
+
+		if (!bitmap_and(aqm, mdev_aqm, assigned_to->matrix.aqm,	AP_DOMAINS))
+			continue;
+
+		if (assignee)
+			vfio_ap_mdev_log_sharing_err(assignee, assigned_to, apm, aqm);
+		else
+			vfio_ap_mdev_log_in_use_err(assigned_to, apm, aqm);
+
+		return -EADDRINUSE;
+	}
+
+	return 0;
+}
+
+/**
+ * vfio_ap_mdev_validate_masks - verify that the APQNs assigned to the mdev are
+ *				 not reserved for the default zcrypt driver and
+ *				 are not assigned to another mdev.
+ *
+ * @matrix_mdev: the mdev to which the APQNs being validated are assigned.
+ *
+ * Return: One of the following values:
+ * o the error returned from the ap_apqn_in_matrix_owned_by_def_drv() function,
+ *   most likely -EBUSY indicating the ap_attr_mutex lock is already held.
+ * o EADDRNOTAVAIL if an APQN assigned to @matrix_mdev is reserved for the
+ *		   zcrypt default driver.
+ * o EADDRINUSE if an APQN assigned to @matrix_mdev is assigned to another mdev
+ * o A zero indicating validation succeeded.
+ */
+static int vfio_ap_mdev_validate_masks(struct ap_matrix_mdev *matrix_mdev)
+{
+	if (ap_apqn_in_matrix_owned_by_def_drv(matrix_mdev->matrix.apm,
+					       matrix_mdev->matrix.aqm))
+		return -EADDRNOTAVAIL;
+
+	return vfio_ap_mdev_verify_no_sharing(matrix_mdev,
+					      matrix_mdev->matrix.apm,
+					      matrix_mdev->matrix.aqm);
+}
+
+static void vfio_ap_mdev_link_adapter(struct ap_matrix_mdev *matrix_mdev,
+				      unsigned long apid)
+{
+	unsigned long apqi;
+
+	for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS)
+		vfio_ap_mdev_link_apqn(matrix_mdev,
+				       AP_MKQID(apid, apqi));
+}
+
+static void collect_queues_to_reset(struct ap_matrix_mdev *matrix_mdev,
+				    unsigned long apid,
+				    struct list_head *qlist)
+{
+	struct vfio_ap_queue *q;
+	unsigned long  apqi;
+
+	for_each_set_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm, AP_DOMAINS) {
+		q = vfio_ap_mdev_get_queue(matrix_mdev, AP_MKQID(apid, apqi));
+		if (q)
+			list_add_tail(&q->reset_qnode, qlist);
+	}
+}
+
+static void reset_queues_for_apid(struct ap_matrix_mdev *matrix_mdev,
+				  unsigned long apid)
+{
+	struct list_head qlist;
+
+	INIT_LIST_HEAD(&qlist);
+	collect_queues_to_reset(matrix_mdev, apid, &qlist);
+	vfio_ap_mdev_reset_qlist(&qlist);
+}
+
+static int reset_queues_for_apids(struct ap_matrix_mdev *matrix_mdev,
+				  unsigned long *apm_reset)
+{
+	struct list_head qlist;
+	unsigned long apid;
+
+	if (bitmap_empty(apm_reset, AP_DEVICES))
+		return 0;
+
+	INIT_LIST_HEAD(&qlist);
+
+	for_each_set_bit_inv(apid, apm_reset, AP_DEVICES)
+		collect_queues_to_reset(matrix_mdev, apid, &qlist);
+
+	return vfio_ap_mdev_reset_qlist(&qlist);
+}
+
+/**
+ * assign_adapter_store - parses the APID from @buf and sets the
+ * corresponding bit in the mediated matrix device's APM
+ *
+ * @dev:	the matrix device
+ * @attr:	the mediated matrix device's assign_adapter attribute
+ * @buf:	a buffer containing the AP adapter number (APID) to
+ *		be assigned
+ * @count:	the number of bytes in @buf
+ *
+ * Return: the number of bytes processed if the APID is valid; otherwise,
+ * returns one of the following errors:
+ *
+ *	1. -EINVAL
+ *	   The APID is not a valid number
+ *
+ *	2. -ENODEV
+ *	   The APID exceeds the maximum value configured for the system
+ *
+ *	3. -EADDRNOTAVAIL
+ *	   An APQN derived from the cross product of the APID being assigned
+ *	   and the APQIs previously assigned is not bound to the vfio_ap device
+ *	   driver; or, if no APQIs have yet been assigned, the APID is not
+ *	   contained in an APQN bound to the vfio_ap device driver.
+ *
+ *	4. -EADDRINUSE
+ *	   An APQN derived from the cross product of the APID being assigned
+ *	   and the APQIs previously assigned is being used by another mediated
+ *	   matrix device
+ *
+ *	5. -EAGAIN
+ *	   A lock required to validate the mdev's AP configuration could not
+ *	   be obtained.
+ */
+static ssize_t assign_adapter_store(struct device *dev,
+				    struct device_attribute *attr,
+				    const char *buf, size_t count)
+{
+	int ret;
+	unsigned long apid;
+	DECLARE_BITMAP(apm_filtered, AP_DEVICES);
+	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
+
+	mutex_lock(&ap_attr_mutex);
+	get_update_locks_for_mdev(matrix_mdev);
+
+	ret = kstrtoul(buf, 0, &apid);
+	if (ret)
+		goto done;
+
+	if (apid > matrix_mdev->matrix.apm_max) {
+		ret = -ENODEV;
+		goto done;
+	}
+
+	if (test_bit_inv(apid, matrix_mdev->matrix.apm)) {
+		ret = count;
+		goto done;
+	}
+
+	set_bit_inv(apid, matrix_mdev->matrix.apm);
+
+	ret = vfio_ap_mdev_validate_masks(matrix_mdev);
+	if (ret) {
+		clear_bit_inv(apid, matrix_mdev->matrix.apm);
+		goto done;
+	}
+
+	vfio_ap_mdev_link_adapter(matrix_mdev, apid);
+
+	if (vfio_ap_mdev_filter_matrix(matrix_mdev, apm_filtered)) {
+		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
+		reset_queues_for_apids(matrix_mdev, apm_filtered);
+	}
+
+	ret = count;
+done:
+	release_update_locks_for_mdev(matrix_mdev);
+	mutex_unlock(&ap_attr_mutex);
+
+	return ret;
+}
+static DEVICE_ATTR_WO(assign_adapter);
+
+static struct vfio_ap_queue
+*vfio_ap_unlink_apqn_fr_mdev(struct ap_matrix_mdev *matrix_mdev,
+			     unsigned long apid, unsigned long apqi)
+{
+	struct vfio_ap_queue *q = NULL;
+
+	q = vfio_ap_mdev_get_queue(matrix_mdev, AP_MKQID(apid, apqi));
+	/* If the queue is assigned to the matrix mdev, unlink it. */
+	if (q)
+		vfio_ap_unlink_queue_fr_mdev(q);
+
+	return q;
+}
+
+/**
+ * vfio_ap_mdev_unlink_adapter - unlink all queues associated with unassigned
+ *				 adapter from the matrix mdev to which the
+ *				 adapter was assigned.
+ * @matrix_mdev: the matrix mediated device to which the adapter was assigned.
+ * @apid: the APID of the unassigned adapter.
+ * @qlist: list for storing queues associated with unassigned adapter that
+ *	   need to be reset.
+ */
+static void vfio_ap_mdev_unlink_adapter(struct ap_matrix_mdev *matrix_mdev,
+					unsigned long apid,
+					struct list_head *qlist)
+{
+	unsigned long apqi;
+	struct vfio_ap_queue *q;
+
+	for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS) {
+		q = vfio_ap_unlink_apqn_fr_mdev(matrix_mdev, apid, apqi);
+
+		if (q && qlist) {
+			if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) &&
+			    test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm))
+				list_add_tail(&q->reset_qnode, qlist);
+		}
+	}
+}
+
+static void vfio_ap_mdev_hot_unplug_adapters(struct ap_matrix_mdev *matrix_mdev,
+					     unsigned long *apids)
+{
+	struct vfio_ap_queue *q, *tmpq;
+	struct list_head qlist;
+	unsigned long apid;
+	bool apcb_update = false;
+
+	INIT_LIST_HEAD(&qlist);
+
+	for_each_set_bit_inv(apid, apids, AP_DEVICES) {
+		vfio_ap_mdev_unlink_adapter(matrix_mdev, apid, &qlist);
+
+		if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm)) {
+			clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm);
+			apcb_update = true;
+		}
+	}
+
+	/* Only update apcb if needed to avoid impacting guest */
+	if (apcb_update)
+		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
+
+	vfio_ap_mdev_reset_qlist(&qlist);
+
+	list_for_each_entry_safe(q, tmpq, &qlist, reset_qnode) {
+		vfio_ap_unlink_mdev_fr_queue(q);
+		list_del(&q->reset_qnode);
+	}
+}
+
+static void vfio_ap_mdev_hot_unplug_adapter(struct ap_matrix_mdev *matrix_mdev,
+					    unsigned long apid)
+{
+	DECLARE_BITMAP(apids, AP_DEVICES);
+
+	bitmap_zero(apids, AP_DEVICES);
+	set_bit_inv(apid, apids);
+	vfio_ap_mdev_hot_unplug_adapters(matrix_mdev, apids);
+}
+
+/**
+ * unassign_adapter_store - parses the APID from @buf and clears the
+ * corresponding bit in the mediated matrix device's APM
+ *
+ * @dev:	the matrix device
+ * @attr:	the mediated matrix device's unassign_adapter attribute
+ * @buf:	a buffer containing the adapter number (APID) to be unassigned
+ * @count:	the number of bytes in @buf
+ *
+ * Return: the number of bytes processed if the APID is valid; otherwise,
+ * returns one of the following errors:
+ *	-EINVAL if the APID is not a number
+ *	-ENODEV if the APID it exceeds the maximum value configured for the
+ *		system
+ */
+static ssize_t unassign_adapter_store(struct device *dev,
+				      struct device_attribute *attr,
+				      const char *buf, size_t count)
+{
+	int ret;
+	unsigned long apid;
+	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
+
+	get_update_locks_for_mdev(matrix_mdev);
+
+	ret = kstrtoul(buf, 0, &apid);
+	if (ret)
+		goto done;
+
+	if (apid > matrix_mdev->matrix.apm_max) {
+		ret = -ENODEV;
+		goto done;
+	}
+
+	if (!test_bit_inv(apid, matrix_mdev->matrix.apm)) {
+		ret = count;
+		goto done;
+	}
+
+	clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm);
+	vfio_ap_mdev_hot_unplug_adapter(matrix_mdev, apid);
+	ret = count;
+done:
+	release_update_locks_for_mdev(matrix_mdev);
+	return ret;
+}
+static DEVICE_ATTR_WO(unassign_adapter);
+
+static void vfio_ap_mdev_link_domain(struct ap_matrix_mdev *matrix_mdev,
+				     unsigned long apqi)
+{
+	unsigned long apid;
+
+	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES)
+		vfio_ap_mdev_link_apqn(matrix_mdev,
+				       AP_MKQID(apid, apqi));
+}
+
+/**
+ * assign_domain_store - parses the APQI from @buf and sets the
+ * corresponding bit in the mediated matrix device's AQM
+ *
+ * @dev:	the matrix device
+ * @attr:	the mediated matrix device's assign_domain attribute
+ * @buf:	a buffer containing the AP queue index (APQI) of the domain to
+ *		be assigned
+ * @count:	the number of bytes in @buf
+ *
+ * Return: the number of bytes processed if the APQI is valid; otherwise returns
+ * one of the following errors:
+ *
+ *	1. -EINVAL
+ *	   The APQI is not a valid number
+ *
+ *	2. -ENODEV
+ *	   The APQI exceeds the maximum value configured for the system
+ *
+ *	3. -EADDRNOTAVAIL
+ *	   An APQN derived from the cross product of the APQI being assigned
+ *	   and the APIDs previously assigned is not bound to the vfio_ap device
+ *	   driver; or, if no APIDs have yet been assigned, the APQI is not
+ *	   contained in an APQN bound to the vfio_ap device driver.
+ *
+ *	4. -EADDRINUSE
+ *	   An APQN derived from the cross product of the APQI being assigned
+ *	   and the APIDs previously assigned is being used by another mediated
+ *	   matrix device
+ *
+ *	5. -EAGAIN
+ *	   The lock required to validate the mdev's AP configuration could not
+ *	   be obtained.
+ */
+static ssize_t assign_domain_store(struct device *dev,
+				   struct device_attribute *attr,
+				   const char *buf, size_t count)
+{
+	int ret;
+	unsigned long apqi;
+	DECLARE_BITMAP(apm_filtered, AP_DEVICES);
+	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
+
+	mutex_lock(&ap_attr_mutex);
+	get_update_locks_for_mdev(matrix_mdev);
+
+	ret = kstrtoul(buf, 0, &apqi);
+	if (ret)
+		goto done;
+
+	if (apqi > matrix_mdev->matrix.aqm_max) {
+		ret = -ENODEV;
+		goto done;
+	}
+
+	if (test_bit_inv(apqi, matrix_mdev->matrix.aqm)) {
+		ret = count;
+		goto done;
+	}
+
+	set_bit_inv(apqi, matrix_mdev->matrix.aqm);
+
+	ret = vfio_ap_mdev_validate_masks(matrix_mdev);
+	if (ret) {
+		clear_bit_inv(apqi, matrix_mdev->matrix.aqm);
+		goto done;
+	}
+
+	vfio_ap_mdev_link_domain(matrix_mdev, apqi);
+
+	if (vfio_ap_mdev_filter_matrix(matrix_mdev, apm_filtered)) {
+		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
+		reset_queues_for_apids(matrix_mdev, apm_filtered);
+	}
+
+	ret = count;
+done:
+	release_update_locks_for_mdev(matrix_mdev);
+	mutex_unlock(&ap_attr_mutex);
+
+	return ret;
+}
+static DEVICE_ATTR_WO(assign_domain);
+
+static void vfio_ap_mdev_unlink_domain(struct ap_matrix_mdev *matrix_mdev,
+				       unsigned long apqi,
+				       struct list_head *qlist)
+{
+	unsigned long apid;
+	struct vfio_ap_queue *q;
+
+	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {
+		q = vfio_ap_unlink_apqn_fr_mdev(matrix_mdev, apid, apqi);
+
+		if (q && qlist) {
+			if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) &&
+			    test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm))
+				list_add_tail(&q->reset_qnode, qlist);
+		}
+	}
+}
+
+static void vfio_ap_mdev_hot_unplug_domains(struct ap_matrix_mdev *matrix_mdev,
+					    unsigned long *apqis)
+{
+	struct vfio_ap_queue *q, *tmpq;
+	struct list_head qlist;
+	unsigned long apqi;
+	bool apcb_update = false;
+
+	INIT_LIST_HEAD(&qlist);
+
+	for_each_set_bit_inv(apqi, apqis, AP_DOMAINS) {
+		vfio_ap_mdev_unlink_domain(matrix_mdev, apqi, &qlist);
+
+		if (test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) {
+			clear_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm);
+			apcb_update = true;
+		}
+	}
+
+	/* Only update apcb if needed to avoid impacting guest */
+	if (apcb_update)
+		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
+
+	vfio_ap_mdev_reset_qlist(&qlist);
+
+	list_for_each_entry_safe(q, tmpq, &qlist, reset_qnode) {
+		vfio_ap_unlink_mdev_fr_queue(q);
+		list_del(&q->reset_qnode);
+	}
+}
+
+static void vfio_ap_mdev_hot_unplug_domain(struct ap_matrix_mdev *matrix_mdev,
+					   unsigned long apqi)
+{
+	DECLARE_BITMAP(apqis, AP_DOMAINS);
+
+	bitmap_zero(apqis, AP_DEVICES);
+	set_bit_inv(apqi, apqis);
+	vfio_ap_mdev_hot_unplug_domains(matrix_mdev, apqis);
+}
+
+/**
+ * unassign_domain_store - parses the APQI from @buf and clears the
+ * corresponding bit in the mediated matrix device's AQM
+ *
+ * @dev:	the matrix device
+ * @attr:	the mediated matrix device's unassign_domain attribute
+ * @buf:	a buffer containing the AP queue index (APQI) of the domain to
+ *		be unassigned
+ * @count:	the number of bytes in @buf
+ *
+ * Return: the number of bytes processed if the APQI is valid; otherwise,
+ * returns one of the following errors:
+ *	-EINVAL if the APQI is not a number
+ *	-ENODEV if the APQI exceeds the maximum value configured for the system
+ */
+static ssize_t unassign_domain_store(struct device *dev,
+				     struct device_attribute *attr,
+				     const char *buf, size_t count)
+{
+	int ret;
+	unsigned long apqi;
+	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
+
+	get_update_locks_for_mdev(matrix_mdev);
+
+	ret = kstrtoul(buf, 0, &apqi);
+	if (ret)
+		goto done;
+
+	if (apqi > matrix_mdev->matrix.aqm_max) {
+		ret = -ENODEV;
+		goto done;
+	}
+
+	if (!test_bit_inv(apqi, matrix_mdev->matrix.aqm)) {
+		ret = count;
+		goto done;
+	}
+
+	clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm);
+	vfio_ap_mdev_hot_unplug_domain(matrix_mdev, apqi);
+	ret = count;
+
+done:
+	release_update_locks_for_mdev(matrix_mdev);
+	return ret;
+}
+static DEVICE_ATTR_WO(unassign_domain);
+
+/**
+ * assign_control_domain_store - parses the domain ID from @buf and sets
+ * the corresponding bit in the mediated matrix device's ADM
+ *
+ * @dev:	the matrix device
+ * @attr:	the mediated matrix device's assign_control_domain attribute
+ * @buf:	a buffer containing the domain ID to be assigned
+ * @count:	the number of bytes in @buf
+ *
+ * Return: the number of bytes processed if the domain ID is valid; otherwise,
+ * returns one of the following errors:
+ *	-EINVAL if the ID is not a number
+ *	-ENODEV if the ID exceeds the maximum value configured for the system
+ */
+static ssize_t assign_control_domain_store(struct device *dev,
+					   struct device_attribute *attr,
+					   const char *buf, size_t count)
+{
+	int ret;
+	unsigned long id;
+	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
+
+	get_update_locks_for_mdev(matrix_mdev);
+
+	ret = kstrtoul(buf, 0, &id);
+	if (ret)
+		goto done;
+
+	if (id > matrix_mdev->matrix.adm_max) {
+		ret = -ENODEV;
+		goto done;
+	}
+
+	if (test_bit_inv(id, matrix_mdev->matrix.adm)) {
+		ret = count;
+		goto done;
+	}
+
+	/* Set the bit in the ADM (bitmask) corresponding to the AP control
+	 * domain number (id). The bits in the mask, from most significant to
+	 * least significant, correspond to IDs 0 up to the one less than the
+	 * number of control domains that can be assigned.
+	 */
+	set_bit_inv(id, matrix_mdev->matrix.adm);
+	if (vfio_ap_mdev_filter_cdoms(matrix_mdev))
+		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
+
+	ret = count;
+done:
+	release_update_locks_for_mdev(matrix_mdev);
+	return ret;
+}
+static DEVICE_ATTR_WO(assign_control_domain);
+
+/**
+ * unassign_control_domain_store - parses the domain ID from @buf and
+ * clears the corresponding bit in the mediated matrix device's ADM
+ *
+ * @dev:	the matrix device
+ * @attr:	the mediated matrix device's unassign_control_domain attribute
+ * @buf:	a buffer containing the domain ID to be unassigned
+ * @count:	the number of bytes in @buf
+ *
+ * Return: the number of bytes processed if the domain ID is valid; otherwise,
+ * returns one of the following errors:
+ *	-EINVAL if the ID is not a number
+ *	-ENODEV if the ID exceeds the maximum value configured for the system
+ */
+static ssize_t unassign_control_domain_store(struct device *dev,
+					     struct device_attribute *attr,
+					     const char *buf, size_t count)
+{
+	int ret;
+	unsigned long domid;
+	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
+
+	get_update_locks_for_mdev(matrix_mdev);
+
+	ret = kstrtoul(buf, 0, &domid);
+	if (ret)
+		goto done;
+
+	if (domid > matrix_mdev->matrix.adm_max) {
+		ret = -ENODEV;
+		goto done;
+	}
+
+	if (!test_bit_inv(domid, matrix_mdev->matrix.adm)) {
+		ret = count;
+		goto done;
+	}
+
+	clear_bit_inv(domid, matrix_mdev->matrix.adm);
+
+	if (test_bit_inv(domid, matrix_mdev->shadow_apcb.adm)) {
+		clear_bit_inv(domid, matrix_mdev->shadow_apcb.adm);
+		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
+	}
+
+	ret = count;
+done:
+	release_update_locks_for_mdev(matrix_mdev);
+	return ret;
+}
+static DEVICE_ATTR_WO(unassign_control_domain);
+
+static ssize_t control_domains_show(struct device *dev,
+				    struct device_attribute *dev_attr,
+				    char *buf)
+{
+	unsigned long id;
+	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
+	unsigned long max_domid = matrix_mdev->matrix.adm_max;
+	int nchars = 0;
+
+	mutex_lock(&matrix_dev->mdevs_lock);
+	for_each_set_bit_inv(id, matrix_mdev->matrix.adm, max_domid + 1)
+		nchars += sysfs_emit_at(buf, nchars, "%04lx\n", id);
+	mutex_unlock(&matrix_dev->mdevs_lock);
+
+	return nchars;
+}
+static DEVICE_ATTR_RO(control_domains);
+
+static ssize_t vfio_ap_mdev_matrix_show(struct ap_matrix *matrix, char *buf)
+{
+	unsigned long apid;
+	unsigned long apqi;
+	unsigned long apid1;
+	unsigned long apqi1;
+	unsigned long napm_bits = matrix->apm_max + 1;
+	unsigned long naqm_bits = matrix->aqm_max + 1;
+	int nchars = 0;
+
+	apid1 = find_first_bit_inv(matrix->apm, napm_bits);
+	apqi1 = find_first_bit_inv(matrix->aqm, naqm_bits);
+
+	if ((apid1 < napm_bits) && (apqi1 < naqm_bits)) {
+		for_each_set_bit_inv(apid, matrix->apm, napm_bits) {
+			for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits)
+				nchars += sysfs_emit_at(buf, nchars, "%02lx.%04lx\n", apid, apqi);
+		}
+	} else if (apid1 < napm_bits) {
+		for_each_set_bit_inv(apid, matrix->apm, napm_bits)
+			nchars += sysfs_emit_at(buf, nchars, "%02lx.\n", apid);
+	} else if (apqi1 < naqm_bits) {
+		for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits)
+			nchars += sysfs_emit_at(buf, nchars, ".%04lx\n", apqi);
+	}
+
+	return nchars;
+}
+
+static ssize_t matrix_show(struct device *dev, struct device_attribute *attr,
+			   char *buf)
+{
+	ssize_t nchars;
+	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
+
+	mutex_lock(&matrix_dev->mdevs_lock);
+	nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->matrix, buf);
+	mutex_unlock(&matrix_dev->mdevs_lock);
+
+	return nchars;
+}
+static DEVICE_ATTR_RO(matrix);
+
+static ssize_t guest_matrix_show(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	ssize_t nchars;
+	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
+
+	mutex_lock(&matrix_dev->mdevs_lock);
+	nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->shadow_apcb, buf);
+	mutex_unlock(&matrix_dev->mdevs_lock);
+
+	return nchars;
+}
+static DEVICE_ATTR_RO(guest_matrix);
+
+static ssize_t write_ap_bitmap(unsigned long *bitmap, char *buf, int offset, char sep)
+{
+	return sysfs_emit_at(buf, offset, "0x%016lx%016lx%016lx%016lx%c",
+			 bitmap[0], bitmap[1], bitmap[2], bitmap[3], sep);
+}
+
+static ssize_t ap_config_show(struct device *dev, struct device_attribute *attr,
+			      char *buf)
+{
+	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
+	int idx = 0;
+
+	idx += write_ap_bitmap(matrix_mdev->matrix.apm, buf, idx, ',');
+	idx += write_ap_bitmap(matrix_mdev->matrix.aqm, buf, idx, ',');
+	idx += write_ap_bitmap(matrix_mdev->matrix.adm, buf, idx, '\n');
+
+	return idx;
+}
+
+/* Number of characters needed for a complete hex mask representing the bits in ..  */
+#define AP_DEVICES_STRLEN	(AP_DEVICES / 4 + 3)
+#define AP_DOMAINS_STRLEN	(AP_DOMAINS / 4 + 3)
+#define AP_CONFIG_STRLEN	(AP_DEVICES_STRLEN + 2 * AP_DOMAINS_STRLEN)
+
+static int parse_bitmap(char **strbufptr, unsigned long *bitmap, int nbits)
+{
+	char *curmask;
+
+	curmask = strsep(strbufptr, ",\n");
+	if (!curmask)
+		return -EINVAL;
+
+	bitmap_clear(bitmap, 0, nbits);
+	return ap_hex2bitmap(curmask, bitmap, nbits);
+}
+
+static int ap_matrix_overflow_check(struct ap_matrix_mdev *matrix_mdev)
+{
+	unsigned long bit;
+
+	for_each_set_bit_inv(bit, matrix_mdev->matrix.apm, AP_DEVICES) {
+		if (bit > matrix_mdev->matrix.apm_max)
+			return -ENODEV;
+	}
+
+	for_each_set_bit_inv(bit, matrix_mdev->matrix.aqm, AP_DOMAINS) {
+		if (bit > matrix_mdev->matrix.aqm_max)
+			return -ENODEV;
+	}
+
+	for_each_set_bit_inv(bit, matrix_mdev->matrix.adm, AP_DOMAINS) {
+		if (bit > matrix_mdev->matrix.adm_max)
+			return -ENODEV;
+	}
+
+	return 0;
+}
+
+static void ap_matrix_copy(struct ap_matrix *dst, struct ap_matrix *src)
+{
+	/* This check works around false positive gcc -Wstringop-overread */
+	if (!src)
+		return;
+
+	bitmap_copy(dst->apm, src->apm, AP_DEVICES);
+	bitmap_copy(dst->aqm, src->aqm, AP_DOMAINS);
+	bitmap_copy(dst->adm, src->adm, AP_DOMAINS);
+}
+
+static ssize_t ap_config_store(struct device *dev, struct device_attribute *attr,
+			       const char *buf, size_t count)
+{
+	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
+	struct ap_matrix m_new, m_old, m_added, m_removed;
+	DECLARE_BITMAP(apm_filtered, AP_DEVICES);
+	unsigned long newbit;
+	char *newbuf, *rest;
+	int rc = count;
+	bool do_update;
+
+	newbuf = kstrndup(buf, AP_CONFIG_STRLEN, GFP_KERNEL);
+	if (!newbuf)
+		return -ENOMEM;
+	rest = newbuf;
+
+	mutex_lock(&ap_attr_mutex);
+	get_update_locks_for_mdev(matrix_mdev);
+
+	/* Save old state */
+	ap_matrix_copy(&m_old, &matrix_mdev->matrix);
+	if (parse_bitmap(&rest, m_new.apm, AP_DEVICES) ||
+	    parse_bitmap(&rest, m_new.aqm, AP_DOMAINS) ||
+	    parse_bitmap(&rest, m_new.adm, AP_DOMAINS)) {
+		rc = -EINVAL;
+		goto out;
+	}
+
+	bitmap_andnot(m_removed.apm, m_old.apm, m_new.apm, AP_DEVICES);
+	bitmap_andnot(m_removed.aqm, m_old.aqm, m_new.aqm, AP_DOMAINS);
+	bitmap_andnot(m_added.apm, m_new.apm, m_old.apm, AP_DEVICES);
+	bitmap_andnot(m_added.aqm, m_new.aqm, m_old.aqm, AP_DOMAINS);
+
+	/* Need new bitmaps in matrix_mdev for validation */
+	ap_matrix_copy(&matrix_mdev->matrix, &m_new);
+
+	/* Ensure new state is valid, else undo new state */
+	rc = vfio_ap_mdev_validate_masks(matrix_mdev);
+	if (rc) {
+		ap_matrix_copy(&matrix_mdev->matrix, &m_old);
+		goto out;
+	}
+	rc = ap_matrix_overflow_check(matrix_mdev);
+	if (rc) {
+		ap_matrix_copy(&matrix_mdev->matrix, &m_old);
+		goto out;
+	}
+	rc = count;
+
+	/* Need old bitmaps in matrix_mdev for unplug/unlink */
+	ap_matrix_copy(&matrix_mdev->matrix, &m_old);
+
+	/* Unlink removed adapters/domains */
+	vfio_ap_mdev_hot_unplug_adapters(matrix_mdev, m_removed.apm);
+	vfio_ap_mdev_hot_unplug_domains(matrix_mdev, m_removed.aqm);
+
+	/* Need new bitmaps in matrix_mdev for linking new adapters/domains */
+	ap_matrix_copy(&matrix_mdev->matrix, &m_new);
+
+	/* Link newly added adapters */
+	for_each_set_bit_inv(newbit, m_added.apm, AP_DEVICES)
+		vfio_ap_mdev_link_adapter(matrix_mdev, newbit);
+
+	for_each_set_bit_inv(newbit, m_added.aqm, AP_DOMAINS)
+		vfio_ap_mdev_link_domain(matrix_mdev, newbit);
+
+	/* filter resources not bound to vfio-ap */
+	do_update = vfio_ap_mdev_filter_matrix(matrix_mdev, apm_filtered);
+	do_update |= vfio_ap_mdev_filter_cdoms(matrix_mdev);
+
+	/* Apply changes to shadow apbc if things changed */
+	if (do_update) {
+		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
+		reset_queues_for_apids(matrix_mdev, apm_filtered);
+	}
+out:
+	release_update_locks_for_mdev(matrix_mdev);
+	mutex_unlock(&ap_attr_mutex);
+	kfree(newbuf);
+	return rc;
+}
+static DEVICE_ATTR_RW(ap_config);
+
+static struct attribute *vfio_ap_mdev_attrs[] = {
+	&dev_attr_assign_adapter.attr,
+	&dev_attr_unassign_adapter.attr,
+	&dev_attr_assign_domain.attr,
+	&dev_attr_unassign_domain.attr,
+	&dev_attr_assign_control_domain.attr,
+	&dev_attr_unassign_control_domain.attr,
+	&dev_attr_ap_config.attr,
+	&dev_attr_control_domains.attr,
+	&dev_attr_matrix.attr,
+	&dev_attr_guest_matrix.attr,
+	NULL,
+};
+
+static struct attribute_group vfio_ap_mdev_attr_group = {
+	.attrs = vfio_ap_mdev_attrs
+};
+
+static const struct attribute_group *vfio_ap_mdev_attr_groups[] = {
+	&vfio_ap_mdev_attr_group,
+	NULL
+};
+
+/**
+ * vfio_ap_mdev_set_kvm - sets all data for @matrix_mdev that are needed
+ * to manage AP resources for the guest whose state is represented by @kvm
+ *
+ * @matrix_mdev: a mediated matrix device
+ * @kvm: reference to KVM instance
+ *
+ * Return: 0 if no other mediated matrix device has a reference to @kvm;
+ * otherwise, returns an -EPERM.
+ */
+static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev,
+				struct kvm *kvm)
+{
+	if (kvm->arch.crypto.crycbd) {
+		get_update_locks_for_kvm(kvm);
+		if (kvm->arch.crypto.pqap_hook) {
+			release_update_locks_for_kvm(kvm);
+			return -EPERM;
+		}
+		kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook;
+
+		kvm_get_kvm(kvm);
+		matrix_mdev->kvm = kvm;
+		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
+		release_update_locks_for_kvm(kvm);
+	}
+
+	return 0;
+}
+
+static void unmap_iova(struct ap_matrix_mdev *matrix_mdev, u64 iova, u64 length)
+{
+	struct ap_queue_table *qtable = &matrix_mdev->qtable;
+	struct vfio_ap_queue *q;
+	int loop_cursor;
+
+	hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) {
+		if (q->saved_iova >= iova && q->saved_iova < iova + length)
+			vfio_ap_irq_disable(q);
+	}
+}
+
+static void vfio_ap_mdev_dma_unmap(struct vfio_device *vdev, u64 iova,
+				   u64 length)
+{
+	struct ap_matrix_mdev *matrix_mdev =
+		container_of(vdev, struct ap_matrix_mdev, vdev);
+
+	mutex_lock(&matrix_dev->mdevs_lock);
+
+	unmap_iova(matrix_mdev, iova, length);
+
+	mutex_unlock(&matrix_dev->mdevs_lock);
+}
+
+/**
+ * vfio_ap_mdev_unset_kvm - performs clean-up of resources no longer needed
+ * by @matrix_mdev.
+ *
+ * @matrix_mdev: a matrix mediated device
+ */
+static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev)
+{
+	struct kvm *kvm = matrix_mdev->kvm;
+
+	if (kvm && kvm->arch.crypto.crycbd) {
+		get_update_locks_for_kvm(kvm);
+		kvm->arch.crypto.pqap_hook = NULL;
+
+		kvm_arch_crypto_clear_masks(kvm);
+		vfio_ap_mdev_reset_queues(matrix_mdev);
+		matrix_mdev->kvm = NULL;
+
+		release_update_locks_for_kvm(kvm);
+		kvm_put_kvm(kvm);
+	}
+}
+
+static struct vfio_ap_queue *vfio_ap_find_queue(int apqn)
+{
+	struct ap_queue *queue;
+	struct vfio_ap_queue *q = NULL;
+
+	queue = ap_get_qdev(apqn);
+	if (!queue)
+		return NULL;
+
+	if (queue->ap_dev.device.driver == &matrix_dev->vfio_ap_drv->driver)
+		q = dev_get_drvdata(&queue->ap_dev.device);
+
+	put_device(&queue->ap_dev.device);
+
+	return q;
+}
+
+static int apq_status_check(int apqn, struct ap_queue_status *status)
+{
+	switch (status->response_code) {
+	case AP_RESPONSE_NORMAL:
+	case AP_RESPONSE_DECONFIGURED:
+	case AP_RESPONSE_CHECKSTOPPED:
+		return 0;
+	case AP_RESPONSE_RESET_IN_PROGRESS:
+	case AP_RESPONSE_BUSY:
+		return -EBUSY;
+	case AP_RESPONSE_ASSOC_SECRET_NOT_UNIQUE:
+	case AP_RESPONSE_ASSOC_FAILED:
+		/*
+		 * These asynchronous response codes indicate a PQAP(AAPQ)
+		 * instruction to associate a secret with the guest failed. All
+		 * subsequent AP instructions will end with the asynchronous
+		 * response code until the AP queue is reset; so, let's return
+		 * a value indicating a reset needs to be performed again.
+		 */
+		return -EAGAIN;
+	case AP_RESPONSE_Q_NOT_AVAIL:
+		return -ENODEV;
+	default:
+		WARN(true,
+		     "failed to verify reset of queue %02x.%04x: TAPQ rc=%u\n",
+		     AP_QID_CARD(apqn), AP_QID_QUEUE(apqn),
+		     status->response_code);
+		return -EIO;
+	}
+}
+
+#define WAIT_MSG "Waited %dms for reset of queue %02x.%04x (%u, %u, %u)"
+
+static void apq_reset_check(struct work_struct *reset_work)
+{
+	int ret = -EBUSY, elapsed = 0;
+	struct ap_queue_status status;
+	struct vfio_ap_queue *q;
+
+	q = container_of(reset_work, struct vfio_ap_queue, reset_work);
+	memcpy(&status, &q->reset_status, sizeof(status));
+	while (true) {
+		msleep(AP_RESET_INTERVAL);
+		elapsed += AP_RESET_INTERVAL;
+		status = ap_tapq(q->apqn, NULL);
+		ret = apq_status_check(q->apqn, &status);
+		if (ret == -EIO)
+			return;
+		if (ret == -ENODEV) {
+			vfio_ap_free_aqic_resources(q);
+			return;
+		}
+		if (ret == -EBUSY) {
+			pr_notice_ratelimited(WAIT_MSG, elapsed,
+					      AP_QID_CARD(q->apqn),
+					      AP_QID_QUEUE(q->apqn),
+					      status.response_code,
+					      status.queue_empty,
+					      status.irq_enabled);
+		} else {
+			if (q->reset_status.response_code == AP_RESPONSE_RESET_IN_PROGRESS ||
+			    q->reset_status.response_code == AP_RESPONSE_BUSY ||
+			    q->reset_status.response_code == AP_RESPONSE_STATE_CHANGE_IN_PROGRESS ||
+			    ret == -EAGAIN) {
+				status = ap_zapq(q->apqn, 0);
+				memcpy(&q->reset_status, &status, sizeof(status));
+				continue;
+			}
+			if (q->saved_isc != VFIO_AP_ISC_INVALID)
+				vfio_ap_free_aqic_resources(q);
+			break;
+		}
+	}
+}
+
+static void vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q)
+{
+	struct ap_queue_status status;
+
+	if (!q)
+		return;
+	status = ap_zapq(q->apqn, 0);
+	memcpy(&q->reset_status, &status, sizeof(status));
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+	case AP_RESPONSE_RESET_IN_PROGRESS:
+	case AP_RESPONSE_BUSY:
+	case AP_RESPONSE_STATE_CHANGE_IN_PROGRESS:
+		/*
+		 * Let's verify whether the ZAPQ completed successfully on a work queue.
+		 */
+		queue_work(system_long_wq, &q->reset_work);
+		break;
+	case AP_RESPONSE_DECONFIGURED:
+	case AP_RESPONSE_CHECKSTOPPED:
+	case AP_RESPONSE_Q_NOT_AVAIL:
+		vfio_ap_free_aqic_resources(q);
+		break;
+	default:
+		WARN(true,
+		     "PQAP/ZAPQ for %02x.%04x failed with invalid rc=%u\n",
+		     AP_QID_CARD(q->apqn), AP_QID_QUEUE(q->apqn),
+		     status.response_code);
+	}
+}
+
+static int vfio_ap_mdev_reset_queues(struct ap_matrix_mdev *matrix_mdev)
+{
+	int ret = 0, loop_cursor;
+	struct vfio_ap_queue *q;
+
+	hash_for_each(matrix_mdev->qtable.queues, loop_cursor, q, mdev_qnode)
+		vfio_ap_mdev_reset_queue(q);
+
+	hash_for_each(matrix_mdev->qtable.queues, loop_cursor, q, mdev_qnode) {
+		flush_work(&q->reset_work);
+
+		if (q->reset_status.response_code)
+			ret = -EIO;
+	}
+
+	return ret;
+}
+
+static int vfio_ap_mdev_reset_qlist(struct list_head *qlist)
+{
+	int ret = 0;
+	struct vfio_ap_queue *q;
+
+	list_for_each_entry(q, qlist, reset_qnode)
+		vfio_ap_mdev_reset_queue(q);
+
+	list_for_each_entry(q, qlist, reset_qnode) {
+		flush_work(&q->reset_work);
+
+		if (q->reset_status.response_code)
+			ret = -EIO;
+	}
+
+	return ret;
+}
+
+static int vfio_ap_mdev_open_device(struct vfio_device *vdev)
+{
+	struct ap_matrix_mdev *matrix_mdev =
+		container_of(vdev, struct ap_matrix_mdev, vdev);
+
+	if (!vdev->kvm)
+		return -EINVAL;
+
+	return vfio_ap_mdev_set_kvm(matrix_mdev, vdev->kvm);
+}
+
+static void vfio_ap_mdev_close_device(struct vfio_device *vdev)
+{
+	struct ap_matrix_mdev *matrix_mdev =
+		container_of(vdev, struct ap_matrix_mdev, vdev);
+
+	vfio_ap_mdev_unset_kvm(matrix_mdev);
+}
+
+static void vfio_ap_mdev_request(struct vfio_device *vdev, unsigned int count)
+{
+	struct device *dev = vdev->dev;
+	struct ap_matrix_mdev *matrix_mdev;
+
+	matrix_mdev = container_of(vdev, struct ap_matrix_mdev, vdev);
+
+	get_update_locks_for_mdev(matrix_mdev);
+
+	if (matrix_mdev->kvm) {
+		kvm_arch_crypto_clear_masks(matrix_mdev->kvm);
+		signal_guest_ap_cfg_changed(matrix_mdev);
+	}
+
+	if (matrix_mdev->req_trigger) {
+		if (!(count % 10))
+			dev_notice_ratelimited(dev,
+					       "Relaying device request to user (#%u)\n",
+					       count);
+
+		eventfd_signal(matrix_mdev->req_trigger);
+	} else if (count == 0) {
+		dev_notice(dev,
+			   "No device request registered, blocked until released by user\n");
+	}
+
+	release_update_locks_for_mdev(matrix_mdev);
+}
+
+static int vfio_ap_mdev_get_device_info(unsigned long arg)
+{
+	unsigned long minsz;
+	struct vfio_device_info info;
+
+	minsz = offsetofend(struct vfio_device_info, num_irqs);
+
+	if (copy_from_user(&info, (void __user *)arg, minsz))
+		return -EFAULT;
+
+	if (info.argsz < minsz)
+		return -EINVAL;
+
+	info.flags = VFIO_DEVICE_FLAGS_AP | VFIO_DEVICE_FLAGS_RESET;
+	info.num_regions = 0;
+	info.num_irqs = VFIO_AP_NUM_IRQS;
+
+	return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+}
+
+static ssize_t vfio_ap_get_irq_info(unsigned long arg)
+{
+	unsigned long minsz;
+	struct vfio_irq_info info;
+
+	minsz = offsetofend(struct vfio_irq_info, count);
+
+	if (copy_from_user(&info, (void __user *)arg, minsz))
+		return -EFAULT;
+
+	if (info.argsz < minsz || info.index >= VFIO_AP_NUM_IRQS)
+		return -EINVAL;
+
+	switch (info.index) {
+	case VFIO_AP_REQ_IRQ_INDEX:
+		info.count = 1;
+		info.flags = VFIO_IRQ_INFO_EVENTFD;
+		break;
+	case VFIO_AP_CFG_CHG_IRQ_INDEX:
+		info.count = 1;
+		info.flags = VFIO_IRQ_INFO_EVENTFD;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+}
+
+static int vfio_ap_irq_set_init(struct vfio_irq_set *irq_set, unsigned long arg)
+{
+	int ret;
+	size_t data_size;
+	unsigned long minsz;
+
+	minsz = offsetofend(struct vfio_irq_set, count);
+
+	if (copy_from_user(irq_set, (void __user *)arg, minsz))
+		return -EFAULT;
+
+	ret = vfio_set_irqs_validate_and_prepare(irq_set, 1, VFIO_AP_NUM_IRQS,
+						 &data_size);
+	if (ret)
+		return ret;
+
+	if (!(irq_set->flags & VFIO_IRQ_SET_ACTION_TRIGGER))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int vfio_ap_set_request_irq(struct ap_matrix_mdev *matrix_mdev,
+				   unsigned long arg)
+{
+	s32 fd;
+	void __user *data;
+	unsigned long minsz;
+	struct eventfd_ctx *req_trigger;
+
+	minsz = offsetofend(struct vfio_irq_set, count);
+	data = (void __user *)(arg + minsz);
+
+	if (get_user(fd, (s32 __user *)data))
+		return -EFAULT;
+
+	if (fd == -1) {
+		if (matrix_mdev->req_trigger)
+			eventfd_ctx_put(matrix_mdev->req_trigger);
+		matrix_mdev->req_trigger = NULL;
+	} else if (fd >= 0) {
+		req_trigger = eventfd_ctx_fdget(fd);
+		if (IS_ERR(req_trigger))
+			return PTR_ERR(req_trigger);
+
+		if (matrix_mdev->req_trigger)
+			eventfd_ctx_put(matrix_mdev->req_trigger);
+
+		matrix_mdev->req_trigger = req_trigger;
+	} else {
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int vfio_ap_set_cfg_change_irq(struct ap_matrix_mdev *matrix_mdev, unsigned long arg)
+{
+	s32 fd;
+	void __user *data;
+	unsigned long minsz;
+	struct eventfd_ctx *cfg_chg_trigger;
+
+	minsz = offsetofend(struct vfio_irq_set, count);
+	data = (void __user *)(arg + minsz);
+
+	if (get_user(fd, (s32 __user *)data))
+		return -EFAULT;
+
+	if (fd == -1) {
+		if (matrix_mdev->cfg_chg_trigger)
+			eventfd_ctx_put(matrix_mdev->cfg_chg_trigger);
+		matrix_mdev->cfg_chg_trigger = NULL;
+	} else if (fd >= 0) {
+		cfg_chg_trigger = eventfd_ctx_fdget(fd);
+		if (IS_ERR(cfg_chg_trigger))
+			return PTR_ERR(cfg_chg_trigger);
+
+		if (matrix_mdev->cfg_chg_trigger)
+			eventfd_ctx_put(matrix_mdev->cfg_chg_trigger);
+
+		matrix_mdev->cfg_chg_trigger = cfg_chg_trigger;
+	} else {
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int vfio_ap_set_irqs(struct ap_matrix_mdev *matrix_mdev,
+			    unsigned long arg)
+{
+	int ret;
+	struct vfio_irq_set irq_set;
+
+	ret = vfio_ap_irq_set_init(&irq_set, arg);
+	if (ret)
+		return ret;
+
+	switch (irq_set.flags & VFIO_IRQ_SET_DATA_TYPE_MASK) {
+	case VFIO_IRQ_SET_DATA_EVENTFD:
+		switch (irq_set.index) {
+		case VFIO_AP_REQ_IRQ_INDEX:
+			return vfio_ap_set_request_irq(matrix_mdev, arg);
+		case VFIO_AP_CFG_CHG_IRQ_INDEX:
+			return vfio_ap_set_cfg_change_irq(matrix_mdev, arg);
+		default:
+			return -EINVAL;
+		}
+	default:
+		return -EINVAL;
+	}
+}
+
+static ssize_t vfio_ap_mdev_ioctl(struct vfio_device *vdev,
+				    unsigned int cmd, unsigned long arg)
+{
+	struct ap_matrix_mdev *matrix_mdev =
+		container_of(vdev, struct ap_matrix_mdev, vdev);
+	int ret;
+
+	mutex_lock(&matrix_dev->mdevs_lock);
+	switch (cmd) {
+	case VFIO_DEVICE_GET_INFO:
+		ret = vfio_ap_mdev_get_device_info(arg);
+		break;
+	case VFIO_DEVICE_RESET:
+		ret = vfio_ap_mdev_reset_queues(matrix_mdev);
+		break;
+	case VFIO_DEVICE_GET_IRQ_INFO:
+		ret = vfio_ap_get_irq_info(arg);
+		break;
+	case VFIO_DEVICE_SET_IRQS:
+		ret = vfio_ap_set_irqs(matrix_mdev, arg);
+		break;
+	default:
+		ret = -EOPNOTSUPP;
+		break;
+	}
+	mutex_unlock(&matrix_dev->mdevs_lock);
+
+	return ret;
+}
+
+static struct ap_matrix_mdev *vfio_ap_mdev_for_queue(struct vfio_ap_queue *q)
+{
+	struct ap_matrix_mdev *matrix_mdev;
+	unsigned long apid = AP_QID_CARD(q->apqn);
+	unsigned long apqi = AP_QID_QUEUE(q->apqn);
+
+	lockdep_assert_held(&matrix_dev->guests_lock);
+
+	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
+		if (test_bit_inv(apid, matrix_mdev->matrix.apm) &&
+		    test_bit_inv(apqi, matrix_mdev->matrix.aqm))
+			return matrix_mdev;
+	}
+
+	return NULL;
+}
+
+static ssize_t status_show(struct device *dev,
+			   struct device_attribute *attr,
+			   char *buf)
+{
+	ssize_t nchars = 0;
+	struct vfio_ap_queue *q;
+	unsigned long apid, apqi;
+	struct ap_matrix_mdev *matrix_mdev;
+	struct ap_device *apdev = to_ap_dev(dev);
+
+	mutex_lock(&matrix_dev->guests_lock);
+	mutex_lock(&matrix_dev->mdevs_lock);
+	q = dev_get_drvdata(&apdev->device);
+
+	/*
+	 * Make sure the drvdata has been set before proceeding. There is a
+	 * possibility that the drvdata was not set if the vfio_ap_queue object
+	 * could not be allocated when the queue device was probed. In that case,
+	 * the locks used in vfio_ap_mdev_probe_queue() are released prior to
+	 * removing the sysfs status attribute to avoid a lockdep
+	 * splat. That opens a very small window where the status attribute is
+	 * still available without the vfio_ap_queue object having been
+	 * stored in the device drvdata. In that case, indicate the queue is not
+	 * assigned.
+	 */
+	if (!q) {
+		nchars = sysfs_emit(buf, "%s\n", AP_QUEUE_UNASSIGNED);
+		goto done;
+	}
+
+	matrix_mdev = vfio_ap_mdev_for_queue(q);
+
+	/* If the queue is assigned to the matrix mediated device, then
+	 * determine whether it is passed through to a guest; otherwise,
+	 * indicate that it is unassigned.
+	 */
+	if (matrix_mdev) {
+		apid = AP_QID_CARD(q->apqn);
+		apqi = AP_QID_QUEUE(q->apqn);
+		/*
+		 * If the queue is passed through to the guest, then indicate
+		 * that it is in use; otherwise, indicate that it is
+		 * merely assigned to a matrix mediated device.
+		 */
+		if (matrix_mdev->kvm &&
+		    test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) &&
+		    test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm))
+			nchars = sysfs_emit(buf, "%s\n", AP_QUEUE_IN_USE);
+		else
+			nchars = sysfs_emit(buf, "%s\n", AP_QUEUE_ASSIGNED);
+	} else {
+		nchars = sysfs_emit(buf, "%s\n", AP_QUEUE_UNASSIGNED);
+	}
+
+done:
+	mutex_unlock(&matrix_dev->mdevs_lock);
+	mutex_unlock(&matrix_dev->guests_lock);
+
+	return nchars;
+}
+
+static DEVICE_ATTR_RO(status);
+
+static struct attribute *vfio_queue_attrs[] = {
+	&dev_attr_status.attr,
+	NULL,
+};
+
+static const struct attribute_group vfio_queue_attr_group = {
+	.attrs = vfio_queue_attrs,
+};
+
+static const struct vfio_device_ops vfio_ap_matrix_dev_ops = {
+	.init = vfio_ap_mdev_init_dev,
+	.open_device = vfio_ap_mdev_open_device,
+	.close_device = vfio_ap_mdev_close_device,
+	.ioctl = vfio_ap_mdev_ioctl,
+	.dma_unmap = vfio_ap_mdev_dma_unmap,
+	.bind_iommufd = vfio_iommufd_emulated_bind,
+	.unbind_iommufd = vfio_iommufd_emulated_unbind,
+	.attach_ioas = vfio_iommufd_emulated_attach_ioas,
+	.detach_ioas = vfio_iommufd_emulated_detach_ioas,
+	.request = vfio_ap_mdev_request
+};
+
+static struct mdev_driver vfio_ap_matrix_driver = {
+	.device_api = VFIO_DEVICE_API_AP_STRING,
+	.max_instances = MAX_ZDEV_ENTRIES_EXT,
+	.driver = {
+		.name = "vfio_ap_mdev",
+		.owner = THIS_MODULE,
+		.mod_name = KBUILD_MODNAME,
+		.dev_groups = vfio_ap_mdev_attr_groups,
+	},
+	.probe = vfio_ap_mdev_probe,
+	.remove = vfio_ap_mdev_remove,
+};
+
+int vfio_ap_mdev_register(void)
+{
+	int ret;
+
+	ret = mdev_register_driver(&vfio_ap_matrix_driver);
+	if (ret)
+		return ret;
+
+	matrix_dev->mdev_type.sysfs_name = VFIO_AP_MDEV_TYPE_HWVIRT;
+	matrix_dev->mdev_type.pretty_name = VFIO_AP_MDEV_NAME_HWVIRT;
+	matrix_dev->mdev_types = &matrix_dev->mdev_type;
+	ret = mdev_register_parent(&matrix_dev->parent, &matrix_dev->device,
+				   &vfio_ap_matrix_driver,
+				   &matrix_dev->mdev_types, 1);
+	if (ret)
+		goto err_driver;
+	return 0;
+
+err_driver:
+	mdev_unregister_driver(&vfio_ap_matrix_driver);
+	return ret;
+}
+
+void vfio_ap_mdev_unregister(void)
+{
+	mdev_unregister_parent(&matrix_dev->parent);
+	mdev_unregister_driver(&vfio_ap_matrix_driver);
+}
+
+int vfio_ap_mdev_probe_queue(struct ap_device *apdev)
+{
+	int ret, apqn;
+	struct vfio_ap_queue *q;
+	DECLARE_BITMAP(apm_filtered, AP_DEVICES);
+	struct ap_matrix_mdev *matrix_mdev;
+
+	apqn = to_ap_queue(&apdev->device)->qid;
+	matrix_mdev = get_update_locks_by_apqn(apqn);
+
+	ret = sysfs_create_group(&apdev->device.kobj, &vfio_queue_attr_group);
+	if (ret)
+		goto err_release_locks;
+
+	q = kzalloc_obj(*q);
+	if (!q) {
+		ret = -ENOMEM;
+		goto err_remove_group;
+	}
+
+	q->apqn = apqn;
+	q->saved_isc = VFIO_AP_ISC_INVALID;
+	memset(&q->reset_status, 0, sizeof(q->reset_status));
+	INIT_WORK(&q->reset_work, apq_reset_check);
+
+	if (matrix_mdev) {
+		vfio_ap_mdev_link_queue(matrix_mdev, q);
+
+		/*
+		 * If we're in the process of handling the adding of adapters or
+		 * domains to the host's AP configuration, then let the
+		 * vfio_ap device driver's on_scan_complete callback filter the
+		 * matrix and update the guest's AP configuration after all of
+		 * the new queue devices are probed.
+		 */
+		if (!bitmap_empty(matrix_mdev->apm_add, AP_DEVICES) ||
+		    !bitmap_empty(matrix_mdev->aqm_add, AP_DOMAINS))
+			goto done;
+
+		if (vfio_ap_mdev_filter_matrix(matrix_mdev, apm_filtered)) {
+			vfio_ap_mdev_update_guest_apcb(matrix_mdev);
+			reset_queues_for_apids(matrix_mdev, apm_filtered);
+		}
+	}
+
+done:
+	dev_set_drvdata(&apdev->device, q);
+	release_update_locks_for_mdev(matrix_mdev);
+
+	return ret;
+
+err_remove_group:
+	release_update_locks_for_mdev(matrix_mdev);
+	sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group);
+	return ret;
+
+err_release_locks:
+	release_update_locks_for_mdev(matrix_mdev);
+	return ret;
+}
+
+void vfio_ap_mdev_remove_queue(struct ap_device *apdev)
+{
+	unsigned long apid, apqi;
+	struct vfio_ap_queue *q;
+	struct ap_matrix_mdev *matrix_mdev;
+
+	sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group);
+	q = dev_get_drvdata(&apdev->device);
+	get_update_locks_for_queue(q);
+	matrix_mdev = q->matrix_mdev;
+	apid = AP_QID_CARD(q->apqn);
+	apqi = AP_QID_QUEUE(q->apqn);
+
+	if (matrix_mdev) {
+		/* If the queue is assigned to the guest's AP configuration */
+		if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) &&
+		    test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) {
+			/*
+			 * Since the queues are defined via a matrix of adapters
+			 * and domains, it is not possible to hot unplug a
+			 * single queue; so, let's unplug the adapter.
+			 */
+			clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm);
+			vfio_ap_mdev_update_guest_apcb(matrix_mdev);
+			reset_queues_for_apid(matrix_mdev, apid);
+			goto done;
+		}
+	}
+
+	/*
+	 * If the queue is not in the host's AP configuration, then resetting
+	 * it will fail with response code 01, (APQN not valid); so, let's make
+	 * sure it is in the host's config. If it is not, free the KVM GISC
+	 * resources.
+	 */
+	if (test_bit_inv(apid, (unsigned long *)matrix_dev->info.apm) &&
+	    test_bit_inv(apqi, (unsigned long *)matrix_dev->info.aqm)) {
+		vfio_ap_mdev_reset_queue(q);
+		flush_work(&q->reset_work);
+	} else {
+		vfio_ap_free_aqic_resources(q);
+	}
+
+done:
+	if (matrix_mdev)
+		vfio_ap_unlink_queue_fr_mdev(q);
+
+	dev_set_drvdata(&apdev->device, NULL);
+	kfree(q);
+	release_update_locks_for_mdev(matrix_mdev);
+}
+
+/**
+ * vfio_ap_mdev_resource_in_use: check whether any of a set of APQNs is
+ *				 assigned to a mediated device under the control
+ *				 of the vfio_ap device driver.
+ *
+ * @apm: a bitmap specifying a set of APIDs comprising the APQNs to check.
+ * @aqm: a bitmap specifying a set of APQIs comprising the APQNs to check.
+ *
+ * Return:
+ *	* -EADDRINUSE if one or more of the APQNs specified via @apm/@aqm are
+ *	  assigned to a mediated device under the control of the vfio_ap
+ *	  device driver.
+ *	* Otherwise, return 0.
+ */
+int vfio_ap_mdev_resource_in_use(unsigned long *apm, unsigned long *aqm)
+{
+	int ret;
+
+	mutex_lock(&matrix_dev->guests_lock);
+	mutex_lock(&matrix_dev->mdevs_lock);
+	ret = vfio_ap_mdev_verify_no_sharing(NULL, apm, aqm);
+	mutex_unlock(&matrix_dev->mdevs_lock);
+	mutex_unlock(&matrix_dev->guests_lock);
+
+	return ret;
+}
+
+/**
+ * vfio_ap_mdev_hot_unplug_cfg - hot unplug the adapters, domains and control
+ *				 domains that have been removed from the host's
+ *				 AP configuration from a guest.
+ *
+ * @matrix_mdev: an ap_matrix_mdev object attached to a KVM guest.
+ * @aprem: the adapters that have been removed from the host's AP configuration
+ * @aqrem: the domains that have been removed from the host's AP configuration
+ * @cdrem: the control domains that have been removed from the host's AP
+ *	   configuration.
+ */
+static void vfio_ap_mdev_hot_unplug_cfg(struct ap_matrix_mdev *matrix_mdev,
+					unsigned long *aprem,
+					unsigned long *aqrem,
+					unsigned long *cdrem)
+{
+	bool do_hotplug = false;
+
+	if (bitmap_intersects(matrix_mdev->shadow_apcb.apm, aprem, AP_DEVICES)) {
+		bitmap_andnot(matrix_mdev->shadow_apcb.apm,
+			      matrix_mdev->shadow_apcb.apm,
+			      aprem, AP_DEVICES);
+		do_hotplug = true;
+	}
+
+	if (bitmap_intersects(matrix_mdev->shadow_apcb.aqm, aqrem, AP_DOMAINS)) {
+		bitmap_andnot(matrix_mdev->shadow_apcb.aqm,
+			      matrix_mdev->shadow_apcb.aqm,
+			      aqrem, AP_DOMAINS);
+		do_hotplug = true;
+	}
+
+	if (bitmap_intersects(matrix_mdev->shadow_apcb.adm, cdrem, AP_DOMAINS)) {
+		bitmap_andnot(matrix_mdev->shadow_apcb.adm,
+			      matrix_mdev->shadow_apcb.adm,
+			      cdrem, AP_DOMAINS);
+		do_hotplug = true;
+	}
+
+	if (do_hotplug)
+		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
+}
+
+/**
+ * vfio_ap_mdev_cfg_remove - determines which guests are using the adapters,
+ *			     domains and control domains that have been removed
+ *			     from the host AP configuration and unplugs them
+ *			     from those guests.
+ *
+ * @ap_remove:	bitmap specifying which adapters have been removed from the host
+ *		config.
+ * @aq_remove:	bitmap specifying which domains have been removed from the host
+ *		config.
+ * @cd_remove:	bitmap specifying which control domains have been removed from
+ *		the host config.
+ */
+static void vfio_ap_mdev_cfg_remove(unsigned long *ap_remove,
+				    unsigned long *aq_remove,
+				    unsigned long *cd_remove)
+{
+	struct ap_matrix_mdev *matrix_mdev;
+	DECLARE_BITMAP(aprem, AP_DEVICES);
+	DECLARE_BITMAP(aqrem, AP_DOMAINS);
+	DECLARE_BITMAP(cdrem, AP_DOMAINS);
+	int do_remove;
+
+	/*
+	 * It is safe to traverse this list here because the
+	 * required guard - matrix_dev->guests_lock - is taken in the
+	 * vfio_ap_on_cfg_changed function prior to this function getting
+	 * called.
+	 */
+	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
+		/*
+		 * The mdevs_lock must be held to access fields within matrix_mdev,
+		 * and kvm->lock must be taken before mdevs_lock to satisfy the lock
+		 * ordering requirement and prevent a lockdep splat.
+		 */
+		if (matrix_mdev->kvm)
+			mutex_lock(&matrix_mdev->kvm->lock);
+		mutex_lock(&matrix_dev->mdevs_lock);
+
+		do_remove = bitmap_and(aprem, ap_remove,
+				       matrix_mdev->matrix.apm,
+				       AP_DEVICES);
+		do_remove |= bitmap_and(aqrem, aq_remove,
+					  matrix_mdev->matrix.aqm,
+					  AP_DOMAINS);
+		do_remove |= bitmap_and(cdrem, cd_remove,
+					matrix_mdev->matrix.adm,
+					AP_DOMAINS);
+
+		if (do_remove)
+			vfio_ap_mdev_hot_unplug_cfg(matrix_mdev, aprem, aqrem,
+						    cdrem);
+
+		mutex_unlock(&matrix_dev->mdevs_lock);
+		if (matrix_mdev->kvm)
+			mutex_unlock(&matrix_mdev->kvm->lock);
+	}
+}
+
+/**
+ * vfio_ap_mdev_on_cfg_remove - responds to the removal of adapters, domains and
+ *				control domains from the host AP configuration
+ *				by unplugging them from the guests that are
+ *				using them.
+ * @cur_config_info: the current host AP configuration information
+ * @prev_config_info: the previous host AP configuration information
+ */
+static void vfio_ap_mdev_on_cfg_remove(struct ap_config_info *cur_config_info,
+				       struct ap_config_info *prev_config_info)
+{
+	int do_remove;
+	DECLARE_BITMAP(aprem, AP_DEVICES);
+	DECLARE_BITMAP(aqrem, AP_DOMAINS);
+	DECLARE_BITMAP(cdrem, AP_DOMAINS);
+
+	do_remove = bitmap_andnot(aprem,
+				  (unsigned long *)prev_config_info->apm,
+				  (unsigned long *)cur_config_info->apm,
+				  AP_DEVICES);
+	do_remove |= bitmap_andnot(aqrem,
+				   (unsigned long *)prev_config_info->aqm,
+				   (unsigned long *)cur_config_info->aqm,
+				   AP_DEVICES);
+	do_remove |= bitmap_andnot(cdrem,
+				   (unsigned long *)prev_config_info->adm,
+				   (unsigned long *)cur_config_info->adm,
+				   AP_DEVICES);
+
+	if (do_remove)
+		vfio_ap_mdev_cfg_remove(aprem, aqrem, cdrem);
+}
+
+/**
+ * vfio_ap_filter_apid_by_qtype: filter APIDs from an AP mask for adapters that
+ *				 are older than AP type 10 (CEX4).
+ * @apm: a bitmap of the APIDs to examine
+ * @aqm: a bitmap of the APQIs of the queues to query for the AP type.
+ */
+static void vfio_ap_filter_apid_by_qtype(unsigned long *apm, unsigned long *aqm)
+{
+	bool apid_cleared;
+	struct ap_queue_status status;
+	unsigned long apid, apqi;
+	struct ap_tapq_hwinfo info;
+
+	for_each_set_bit_inv(apid, apm, AP_DEVICES) {
+		apid_cleared = false;
+
+		for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) {
+			status = ap_test_queue(AP_MKQID(apid, apqi), 1, &info);
+			switch (status.response_code) {
+			/*
+			 * According to the architecture in each case
+			 * below, the queue's info should be filled.
+			 */
+			case AP_RESPONSE_NORMAL:
+			case AP_RESPONSE_RESET_IN_PROGRESS:
+			case AP_RESPONSE_DECONFIGURED:
+			case AP_RESPONSE_CHECKSTOPPED:
+			case AP_RESPONSE_BUSY:
+				/*
+				 * The vfio_ap device driver only
+				 * supports CEX4 and newer adapters, so
+				 * remove the APID if the adapter is
+				 * older than a CEX4.
+				 */
+				if (info.at < AP_DEVICE_TYPE_CEX4) {
+					clear_bit_inv(apid, apm);
+					apid_cleared = true;
+				}
+
+				break;
+
+			default:
+				/*
+				 * If we don't know the adapter type,
+				 * clear its APID since it can't be
+				 * determined whether the vfio_ap
+				 * device driver supports it.
+				 */
+				clear_bit_inv(apid, apm);
+				apid_cleared = true;
+				break;
+			}
+
+			/*
+			 * If we've already cleared the APID from the apm, there
+			 * is no need to continue examining the remainin AP
+			 * queues to determine the type of the adapter.
+			 */
+			if (apid_cleared)
+				continue;
+		}
+	}
+}
+
+/**
+ * vfio_ap_mdev_cfg_add - store bitmaps specifying the adapters, domains and
+ *			  control domains that have been added to the host's
+ *			  AP configuration for each matrix mdev to which they
+ *			  are assigned.
+ *
+ * @apm_add: a bitmap specifying the adapters that have been added to the AP
+ *	     configuration.
+ * @aqm_add: a bitmap specifying the domains that have been added to the AP
+ *	     configuration.
+ * @adm_add: a bitmap specifying the control domains that have been added to the
+ *	     AP configuration.
+ */
+static void vfio_ap_mdev_cfg_add(unsigned long *apm_add, unsigned long *aqm_add,
+				 unsigned long *adm_add)
+{
+	struct ap_matrix_mdev *matrix_mdev;
+
+	if (list_empty(&matrix_dev->mdev_list))
+		return;
+
+	vfio_ap_filter_apid_by_qtype(apm_add, aqm_add);
+
+	/*
+	 * It is safe to traverse this list here because the
+	 * required guard - matrix_dev->guests_lock - is taken in the
+	 * vfio_ap_on_cfg_changed function prior to this function getting
+	 * called.
+	 */
+	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
+		/*
+		 * The mdevs_lock must be held in order to access fields
+		 * within matrix_mdev
+		 */
+		mutex_lock(&matrix_dev->mdevs_lock);
+
+		bitmap_and(matrix_mdev->apm_add,
+			   matrix_mdev->matrix.apm, apm_add, AP_DEVICES);
+		bitmap_and(matrix_mdev->aqm_add,
+			   matrix_mdev->matrix.aqm, aqm_add, AP_DOMAINS);
+		bitmap_and(matrix_mdev->adm_add,
+			   matrix_mdev->matrix.adm, adm_add, AP_DEVICES);
+
+		mutex_unlock(&matrix_dev->mdevs_lock);
+	}
+}
+
+/**
+ * vfio_ap_mdev_on_cfg_add - responds to the addition of adapters, domains and
+ *			     control domains to the host AP configuration
+ *			     by updating the bitmaps that specify what adapters,
+ *			     domains and control domains have been added so they
+ *			     can be hot plugged into the guest when the AP bus
+ *			     scan completes (see vfio_ap_on_scan_complete
+ *			     function).
+ * @cur_config_info: the current AP configuration information
+ * @prev_config_info: the previous AP configuration information
+ */
+static void vfio_ap_mdev_on_cfg_add(struct ap_config_info *cur_config_info,
+				    struct ap_config_info *prev_config_info)
+{
+	bool do_add;
+	DECLARE_BITMAP(apm_add, AP_DEVICES);
+	DECLARE_BITMAP(aqm_add, AP_DOMAINS);
+	DECLARE_BITMAP(adm_add, AP_DOMAINS);
+
+	do_add = bitmap_andnot(apm_add,
+			       (unsigned long *)cur_config_info->apm,
+			       (unsigned long *)prev_config_info->apm,
+			       AP_DEVICES);
+	do_add |= bitmap_andnot(aqm_add,
+				(unsigned long *)cur_config_info->aqm,
+				(unsigned long *)prev_config_info->aqm,
+				AP_DOMAINS);
+	do_add |= bitmap_andnot(adm_add,
+				(unsigned long *)cur_config_info->adm,
+				(unsigned long *)prev_config_info->adm,
+				AP_DOMAINS);
+
+	if (do_add)
+		vfio_ap_mdev_cfg_add(apm_add, aqm_add, adm_add);
+}
+
+/**
+ * vfio_ap_on_cfg_changed - handles notification of changes to the host AP
+ *			    configuration.
+ *
+ * @cur_cfg_info: the current host AP configuration
+ * @prev_cfg_info: the previous host AP configuration
+ */
+void vfio_ap_on_cfg_changed(struct ap_config_info *cur_cfg_info,
+			    struct ap_config_info *prev_cfg_info)
+{
+	if (!cur_cfg_info || !prev_cfg_info)
+		return;
+
+	/*
+	 * Take the guests_lock mutex here to guard access to the
+	 * matrix_dev->mdev_list in the two functions called below.
+	 */
+	mutex_lock(&matrix_dev->guests_lock);
+
+	vfio_ap_mdev_on_cfg_remove(cur_cfg_info, prev_cfg_info);
+	vfio_ap_mdev_on_cfg_add(cur_cfg_info, prev_cfg_info);
+	memcpy(&matrix_dev->info, cur_cfg_info, sizeof(*cur_cfg_info));
+
+	mutex_unlock(&matrix_dev->guests_lock);
+}
+
+static void vfio_ap_mdev_hot_plug_cfg(struct ap_matrix_mdev *matrix_mdev)
+{
+	DECLARE_BITMAP(apm_filtered, AP_DEVICES);
+	bool filter_domains, filter_adapters, filter_cdoms, do_hotplug = false;
+
+	/*
+	 * Zero out the apm_filtered bitmap in case there are no adapters or
+	 * domains to be added, but only control domains. In that case,
+	 * vfio_ap_mdev_filter_matrix() - which initializes apm_filtered - will
+	 * not get called and the reset_queues_for_apids will crash because it
+	 * will access an uninitialized bitmap.
+	 */
+	bitmap_zero(apm_filtered, AP_DEVICES);
+
+	filter_adapters = bitmap_intersects(matrix_mdev->matrix.apm,
+					    matrix_mdev->apm_add, AP_DEVICES);
+	filter_domains = bitmap_intersects(matrix_mdev->matrix.aqm,
+					   matrix_mdev->aqm_add, AP_DOMAINS);
+	filter_cdoms = bitmap_intersects(matrix_mdev->matrix.adm,
+					 matrix_mdev->adm_add, AP_DOMAINS);
+
+	if (filter_adapters || filter_domains)
+		do_hotplug = vfio_ap_mdev_filter_matrix(matrix_mdev, apm_filtered);
+
+	if (filter_cdoms)
+		do_hotplug |= vfio_ap_mdev_filter_cdoms(matrix_mdev);
+
+	if (do_hotplug)
+		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
+
+	reset_queues_for_apids(matrix_mdev, apm_filtered);
+}
+
+void vfio_ap_on_scan_complete(struct ap_config_info *new_config_info,
+			      struct ap_config_info *old_config_info)
+{
+	struct ap_matrix_mdev *matrix_mdev;
+
+	mutex_lock(&matrix_dev->guests_lock);
+
+	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
+		/*
+		 * The mdevs_lock must be held to access fields within matrix_mdev,
+		 * and kvm->lock must be taken before mdevs_lock to satisfy the lock
+		 * ordering requirement and prevent a lockdep splat.
+		 */
+		if (matrix_mdev->kvm)
+			mutex_lock(&matrix_mdev->kvm->lock);
+		mutex_lock(&matrix_dev->mdevs_lock);
+
+		if (bitmap_empty(matrix_mdev->apm_add, AP_DEVICES) &&
+		    bitmap_empty(matrix_mdev->aqm_add, AP_DOMAINS) &&
+		    bitmap_empty(matrix_mdev->adm_add, AP_DOMAINS))
+			goto do_unlock;
+
+		vfio_ap_mdev_hot_plug_cfg(matrix_mdev);
+		bitmap_clear(matrix_mdev->apm_add, 0, AP_DEVICES);
+		bitmap_clear(matrix_mdev->aqm_add, 0, AP_DOMAINS);
+		bitmap_clear(matrix_mdev->adm_add, 0, AP_DOMAINS);
+
+do_unlock:
+		mutex_unlock(&matrix_dev->mdevs_lock);
+		if (matrix_mdev->kvm)
+			mutex_unlock(&matrix_mdev->kvm->lock);
+	}
+
+	mutex_unlock(&matrix_dev->guests_lock);
+}
diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h
new file mode 100644
index 000000000..9bff666b0
--- /dev/null
+++ b/drivers/s390/crypto/vfio_ap_private.h
@@ -0,0 +1,168 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Private data and functions for adjunct processor VFIO matrix driver.
+ *
+ * Author(s): Tony Krowiak <akrowiak@linux.ibm.com>
+ *	      Halil Pasic <pasic@linux.ibm.com>
+ *	      Pierre Morel <pmorel@linux.ibm.com>
+ *
+ * Copyright IBM Corp. 2018
+ */
+
+#ifndef _VFIO_AP_PRIVATE_H_
+#define _VFIO_AP_PRIVATE_H_
+
+#include <linux/types.h>
+#include <linux/mdev.h>
+#include <linux/delay.h>
+#include <linux/eventfd.h>
+#include <linux/mutex.h>
+#include <linux/kvm_host.h>
+#include <linux/vfio.h>
+#include <linux/hashtable.h>
+
+#include "ap_bus.h"
+
+#define VFIO_AP_MODULE_NAME "vfio_ap"
+#define VFIO_AP_DRV_NAME "vfio_ap"
+
+/**
+ * struct ap_matrix_dev - Contains the data for the matrix device.
+ *
+ * @device:	generic device structure associated with the AP matrix device
+ * @info:	the struct containing the output from the PQAP(QCI) instruction
+ * @mdev_list:	the list of mediated matrix devices created
+ * @mdevs_lock: mutex for locking the AP matrix device. This lock will be
+ *		taken every time we fiddle with state managed by the vfio_ap
+ *		driver, be it using @mdev_list or writing the state of a
+ *		single ap_matrix_mdev device. It's quite coarse but we don't
+ *		expect much contention.
+ * @vfio_ap_drv: the vfio_ap device driver
+ * @guests_lock: mutex for controlling access to a guest that is using AP
+ *		 devices passed through by the vfio_ap device driver. This lock
+ *		 will be taken when the AP devices are plugged into or unplugged
+ *		 from a guest, and when an ap_matrix_mdev device is added to or
+ *		 removed from @mdev_list or the list is iterated.
+ */
+struct ap_matrix_dev {
+	struct device device;
+	struct ap_config_info info;
+	struct list_head mdev_list;
+	struct mutex mdevs_lock; /* serializes access to each ap_matrix_mdev */
+	struct ap_driver  *vfio_ap_drv;
+	struct mutex guests_lock; /* serializes access to each KVM guest */
+	struct mdev_parent parent;
+	struct mdev_type mdev_type;
+	struct mdev_type *mdev_types;
+};
+
+extern struct ap_matrix_dev *matrix_dev;
+
+/**
+ * struct ap_matrix - matrix of adapters, domains and control domains
+ *
+ * @apm_max: max adapter number in @apm
+ * @apm: identifies the AP adapters in the matrix
+ * @aqm_max: max domain number in @aqm
+ * @aqm: identifies the AP queues (domains) in the matrix
+ * @adm_max: max domain number in @adm
+ * @adm: identifies the AP control domains in the matrix
+ *
+ * The AP matrix is comprised of three bit masks identifying the adapters,
+ * queues (domains) and control domains that belong to an AP matrix. The bits in
+ * each mask, from left to right, correspond to IDs 0 to 255. When a bit is set
+ * the corresponding ID belongs to the matrix.
+ */
+struct ap_matrix {
+	unsigned long apm_max;
+	DECLARE_BITMAP(apm, AP_DEVICES);
+	unsigned long aqm_max;
+	DECLARE_BITMAP(aqm, AP_DOMAINS);
+	unsigned long adm_max;
+	DECLARE_BITMAP(adm, AP_DOMAINS);
+};
+
+/**
+ * struct ap_queue_table - a table of queue objects.
+ *
+ * @queues: a hashtable of queues (struct vfio_ap_queue).
+ */
+struct ap_queue_table {
+	DECLARE_HASHTABLE(queues, 8);
+};
+
+/**
+ * struct ap_matrix_mdev - Contains the data associated with a matrix mediated
+ *			   device.
+ * @vdev:	the vfio device
+ * @node:	allows the ap_matrix_mdev struct to be added to a list
+ * @matrix:	the adapters, usage domains and control domains assigned to the
+ *		mediated matrix device.
+ * @shadow_apcb:    the shadow copy of the APCB field of the KVM guest's CRYCB
+ * @kvm:	the struct holding guest's state
+ * @pqap_hook:	the function pointer to the interception handler for the
+ *		PQAP(AQIC) instruction.
+ * @mdev:	the mediated device
+ * @qtable:	table of queues (struct vfio_ap_queue) assigned to the mdev
+ * @req_trigger eventfd ctx for signaling userspace to return a device
+ * @cfg_chg_trigger eventfd ctx to signal AP config changed to userspace
+ * @apm_add:	bitmap of APIDs added to the host's AP configuration
+ * @aqm_add:	bitmap of APQIs added to the host's AP configuration
+ * @adm_add:	bitmap of control domain numbers added to the host's AP
+ *		configuration
+ */
+struct ap_matrix_mdev {
+	struct vfio_device vdev;
+	struct list_head node;
+	struct ap_matrix matrix;
+	struct ap_matrix shadow_apcb;
+	struct kvm *kvm;
+	crypto_hook pqap_hook;
+	struct mdev_device *mdev;
+	struct ap_queue_table qtable;
+	struct eventfd_ctx *req_trigger;
+	struct eventfd_ctx *cfg_chg_trigger;
+	DECLARE_BITMAP(apm_add, AP_DEVICES);
+	DECLARE_BITMAP(aqm_add, AP_DOMAINS);
+	DECLARE_BITMAP(adm_add, AP_DOMAINS);
+};
+
+/**
+ * struct vfio_ap_queue - contains the data associated with a queue bound to the
+ *			  vfio_ap device driver
+ * @matrix_mdev: the matrix mediated device
+ * @saved_iova: the notification indicator byte (nib) address
+ * @apqn: the APQN of the AP queue device
+ * @saved_isc: the guest ISC registered with the GIB interface
+ * @mdev_qnode: allows the vfio_ap_queue struct to be added to a hashtable
+ * @reset_qnode: allows the vfio_ap_queue struct to be added to a list of queues
+ *		 that need to be reset
+ * @reset_status: the status from the last reset of the queue
+ * @reset_work: work to wait for queue reset to complete
+ */
+struct vfio_ap_queue {
+	struct ap_matrix_mdev *matrix_mdev;
+	dma_addr_t saved_iova;
+	int	apqn;
+#define VFIO_AP_ISC_INVALID 0xff
+	unsigned char saved_isc;
+	struct hlist_node mdev_qnode;
+	struct list_head reset_qnode;
+	struct ap_queue_status reset_status;
+	struct work_struct reset_work;
+};
+
+int vfio_ap_mdev_register(void);
+void vfio_ap_mdev_unregister(void);
+
+int vfio_ap_mdev_probe_queue(struct ap_device *queue);
+void vfio_ap_mdev_remove_queue(struct ap_device *queue);
+
+int vfio_ap_mdev_resource_in_use(unsigned long *apm, unsigned long *aqm);
+
+void vfio_ap_on_cfg_changed(struct ap_config_info *new_config_info,
+			    struct ap_config_info *old_config_info);
+void vfio_ap_on_scan_complete(struct ap_config_info *new_config_info,
+			      struct ap_config_info *old_config_info);
+
+#endif /* _VFIO_AP_PRIVATE_H_ */
diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c
new file mode 100644
index 000000000..ec6a4c2f9
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_api.c
@@ -0,0 +1,2023 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ *  Copyright IBM Corp. 2001, 2018
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *	       Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *				  Ralph Wuerthner <rwuerthn@de.ibm.com>
+ *  MSGTYPE restruct:		  Holger Dengler <hd@linux.vnet.ibm.com>
+ *  Multiple device nodes: Harald Freudenberger <freude@linux.ibm.com>
+ */
+
+#define pr_fmt(fmt) "zcrypt: " fmt
+
+#include <linux/export.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/miscdevice.h>
+#include <linux/fs.h>
+#include <linux/slab.h>
+#include <linux/atomic.h>
+#include <linux/uaccess.h>
+#include <linux/hw_random.h>
+#include <linux/debugfs.h>
+#include <linux/cdev.h>
+#include <linux/ctype.h>
+#include <linux/capability.h>
+#include <asm/debug.h>
+
+#define CREATE_TRACE_POINTS
+#include <asm/trace/zcrypt.h>
+
+#include "zcrypt_api.h"
+#include "zcrypt_debug.h"
+
+#include "zcrypt_msgtype6.h"
+#include "zcrypt_msgtype50.h"
+#include "zcrypt_ccamisc.h"
+#include "zcrypt_ep11misc.h"
+
+/*
+ * Module description.
+ */
+MODULE_AUTHOR("IBM Corporation");
+MODULE_DESCRIPTION("Cryptographic Coprocessor interface, " \
+		   "Copyright IBM Corp. 2001, 2012");
+MODULE_LICENSE("GPL");
+
+unsigned int zcrypt_mempool_threshold = 5;
+module_param_named(mempool_threshold, zcrypt_mempool_threshold, uint, 0400);
+MODULE_PARM_DESC(mempool_threshold, "CCA and EP11 request/reply mempool minimal items (min: 1)");
+
+/*
+ * zcrypt tracepoint functions
+ */
+EXPORT_TRACEPOINT_SYMBOL(s390_zcrypt_req);
+EXPORT_TRACEPOINT_SYMBOL(s390_zcrypt_rep);
+
+DEFINE_SPINLOCK(zcrypt_list_lock);
+LIST_HEAD(zcrypt_card_list);
+
+static atomic_t zcrypt_open_count = ATOMIC_INIT(0);
+
+static LIST_HEAD(zcrypt_ops_list);
+
+/* Zcrypt related debug feature stuff. */
+debug_info_t *zcrypt_dbf_info;
+
+/*
+ * Process a rescan of the transport layer.
+ * Runs a synchronous AP bus rescan.
+ * Returns true if something has changed (for example the
+ * bus scan has found and build up new devices) and it is
+ * worth to do a retry. Otherwise false is returned meaning
+ * no changes on the AP bus level.
+ */
+static inline bool zcrypt_process_rescan(void)
+{
+	return ap_bus_force_rescan();
+}
+
+void zcrypt_msgtype_register(struct zcrypt_ops *zops)
+{
+	list_add_tail(&zops->list, &zcrypt_ops_list);
+}
+
+void zcrypt_msgtype_unregister(struct zcrypt_ops *zops)
+{
+	list_del_init(&zops->list);
+}
+
+struct zcrypt_ops *zcrypt_msgtype(unsigned char *name, int variant)
+{
+	struct zcrypt_ops *zops;
+
+	list_for_each_entry(zops, &zcrypt_ops_list, list)
+		if (zops->variant == variant &&
+		    (!strncmp(zops->name, name, sizeof(zops->name))))
+			return zops;
+	return NULL;
+}
+EXPORT_SYMBOL(zcrypt_msgtype);
+
+/*
+ * Multi device nodes extension functions.
+ */
+
+struct zcdn_device;
+
+static void zcdn_device_release(struct device *dev);
+static const struct class zcrypt_class = {
+	.name = ZCRYPT_NAME,
+	.dev_release = zcdn_device_release,
+};
+static dev_t zcrypt_devt;
+static struct cdev zcrypt_cdev;
+
+struct zcdn_device {
+	struct device device;
+	struct ap_perms perms;
+};
+
+#define to_zcdn_dev(x) container_of((x), struct zcdn_device, device)
+
+#define ZCDN_MAX_NAME 32
+
+static int zcdn_create(const char *name);
+static int zcdn_destroy(const char *name);
+
+/*
+ * Find zcdn device by name.
+ * Returns reference to the zcdn device which needs to be released
+ * with put_device() after use.
+ */
+static inline struct zcdn_device *find_zcdndev_by_name(const char *name)
+{
+	struct device *dev = class_find_device_by_name(&zcrypt_class, name);
+
+	return dev ? to_zcdn_dev(dev) : NULL;
+}
+
+/*
+ * Find zcdn device by devt value.
+ * Returns reference to the zcdn device which needs to be released
+ * with put_device() after use.
+ */
+static inline struct zcdn_device *find_zcdndev_by_devt(dev_t devt)
+{
+	struct device *dev = class_find_device_by_devt(&zcrypt_class, devt);
+
+	return dev ? to_zcdn_dev(dev) : NULL;
+}
+
+static ssize_t ioctlmask_show(struct device *dev,
+			      struct device_attribute *attr,
+			      char *buf)
+{
+	struct zcdn_device *zcdndev = to_zcdn_dev(dev);
+	int i, n;
+
+	if (mutex_lock_interruptible(&ap_attr_mutex))
+		return -ERESTARTSYS;
+
+	n = sysfs_emit(buf, "0x");
+	for (i = 0; i < sizeof(zcdndev->perms.ioctlm) / sizeof(long); i++)
+		n += sysfs_emit_at(buf, n, "%016lx", zcdndev->perms.ioctlm[i]);
+	n += sysfs_emit_at(buf, n, "\n");
+
+	mutex_unlock(&ap_attr_mutex);
+
+	return n;
+}
+
+static ssize_t ioctlmask_store(struct device *dev,
+			       struct device_attribute *attr,
+			       const char *buf, size_t count)
+{
+	int rc;
+	struct zcdn_device *zcdndev = to_zcdn_dev(dev);
+
+	rc = ap_parse_mask_str(buf, zcdndev->perms.ioctlm,
+			       AP_IOCTLS, &ap_attr_mutex);
+	if (rc)
+		return rc;
+
+	return count;
+}
+
+static DEVICE_ATTR_RW(ioctlmask);
+
+static ssize_t apmask_show(struct device *dev,
+			   struct device_attribute *attr,
+			   char *buf)
+{
+	struct zcdn_device *zcdndev = to_zcdn_dev(dev);
+	int i, n;
+
+	if (mutex_lock_interruptible(&ap_attr_mutex))
+		return -ERESTARTSYS;
+
+	n = sysfs_emit(buf, "0x");
+	for (i = 0; i < sizeof(zcdndev->perms.apm) / sizeof(long); i++)
+		n += sysfs_emit_at(buf, n, "%016lx", zcdndev->perms.apm[i]);
+	n += sysfs_emit_at(buf, n, "\n");
+
+	mutex_unlock(&ap_attr_mutex);
+
+	return n;
+}
+
+static ssize_t apmask_store(struct device *dev,
+			    struct device_attribute *attr,
+			    const char *buf, size_t count)
+{
+	int rc;
+	struct zcdn_device *zcdndev = to_zcdn_dev(dev);
+
+	rc = ap_parse_mask_str(buf, zcdndev->perms.apm,
+			       AP_DEVICES, &ap_attr_mutex);
+	if (rc)
+		return rc;
+
+	return count;
+}
+
+static DEVICE_ATTR_RW(apmask);
+
+static ssize_t aqmask_show(struct device *dev,
+			   struct device_attribute *attr,
+			   char *buf)
+{
+	struct zcdn_device *zcdndev = to_zcdn_dev(dev);
+	int i, n;
+
+	if (mutex_lock_interruptible(&ap_attr_mutex))
+		return -ERESTARTSYS;
+
+	n = sysfs_emit(buf, "0x");
+	for (i = 0; i < sizeof(zcdndev->perms.aqm) / sizeof(long); i++)
+		n += sysfs_emit_at(buf, n, "%016lx", zcdndev->perms.aqm[i]);
+	n += sysfs_emit_at(buf, n, "\n");
+
+	mutex_unlock(&ap_attr_mutex);
+
+	return n;
+}
+
+static ssize_t aqmask_store(struct device *dev,
+			    struct device_attribute *attr,
+			    const char *buf, size_t count)
+{
+	int rc;
+	struct zcdn_device *zcdndev = to_zcdn_dev(dev);
+
+	rc = ap_parse_mask_str(buf, zcdndev->perms.aqm,
+			       AP_DOMAINS, &ap_attr_mutex);
+	if (rc)
+		return rc;
+
+	return count;
+}
+
+static DEVICE_ATTR_RW(aqmask);
+
+static ssize_t admask_show(struct device *dev,
+			   struct device_attribute *attr,
+			   char *buf)
+{
+	struct zcdn_device *zcdndev = to_zcdn_dev(dev);
+	int i, n;
+
+	if (mutex_lock_interruptible(&ap_attr_mutex))
+		return -ERESTARTSYS;
+
+	n = sysfs_emit(buf, "0x");
+	for (i = 0; i < sizeof(zcdndev->perms.adm) / sizeof(long); i++)
+		n += sysfs_emit_at(buf, n, "%016lx", zcdndev->perms.adm[i]);
+	n += sysfs_emit_at(buf, n, "\n");
+
+	mutex_unlock(&ap_attr_mutex);
+
+	return n;
+}
+
+static ssize_t admask_store(struct device *dev,
+			    struct device_attribute *attr,
+			    const char *buf, size_t count)
+{
+	int rc;
+	struct zcdn_device *zcdndev = to_zcdn_dev(dev);
+
+	rc = ap_parse_mask_str(buf, zcdndev->perms.adm,
+			       AP_DOMAINS, &ap_attr_mutex);
+	if (rc)
+		return rc;
+
+	return count;
+}
+
+static DEVICE_ATTR_RW(admask);
+
+static struct attribute *zcdn_dev_attrs[] = {
+	&dev_attr_ioctlmask.attr,
+	&dev_attr_apmask.attr,
+	&dev_attr_aqmask.attr,
+	&dev_attr_admask.attr,
+	NULL
+};
+
+static struct attribute_group zcdn_dev_attr_group = {
+	.attrs = zcdn_dev_attrs
+};
+
+static const struct attribute_group *zcdn_dev_attr_groups[] = {
+	&zcdn_dev_attr_group,
+	NULL
+};
+
+static ssize_t zcdn_create_store(const struct class *class,
+				 const struct class_attribute *attr,
+				 const char *buf, size_t count)
+{
+	int rc;
+	char name[ZCDN_MAX_NAME];
+
+	strscpy(name, skip_spaces(buf), sizeof(name));
+
+	rc = zcdn_create(strim(name));
+
+	return rc ? rc : count;
+}
+
+static const struct class_attribute class_attr_zcdn_create =
+	__ATTR(create, 0600, NULL, zcdn_create_store);
+
+static ssize_t zcdn_destroy_store(const struct class *class,
+				  const struct class_attribute *attr,
+				  const char *buf, size_t count)
+{
+	int rc;
+	char name[ZCDN_MAX_NAME];
+
+	strscpy(name, skip_spaces(buf), sizeof(name));
+
+	rc = zcdn_destroy(strim(name));
+
+	return rc ? rc : count;
+}
+
+static const struct class_attribute class_attr_zcdn_destroy =
+	__ATTR(destroy, 0600, NULL, zcdn_destroy_store);
+
+static void zcdn_device_release(struct device *dev)
+{
+	struct zcdn_device *zcdndev = to_zcdn_dev(dev);
+
+	ZCRYPT_DBF_INFO("%s releasing zcdn device %d:%d\n",
+			__func__, MAJOR(dev->devt), MINOR(dev->devt));
+
+	kfree(zcdndev);
+}
+
+static int zcdn_create(const char *name)
+{
+	dev_t devt;
+	int i, rc = 0;
+	struct zcdn_device *zcdndev;
+
+	if (mutex_lock_interruptible(&ap_attr_mutex))
+		return -ERESTARTSYS;
+
+	/* check if device node with this name already exists */
+	if (name[0]) {
+		zcdndev = find_zcdndev_by_name(name);
+		if (zcdndev) {
+			put_device(&zcdndev->device);
+			rc = -EEXIST;
+			goto unlockout;
+		}
+	}
+
+	/* find an unused minor number */
+	for (i = 0; i < ZCRYPT_MAX_MINOR_NODES; i++) {
+		devt = MKDEV(MAJOR(zcrypt_devt), MINOR(zcrypt_devt) + i);
+		zcdndev = find_zcdndev_by_devt(devt);
+		if (zcdndev)
+			put_device(&zcdndev->device);
+		else
+			break;
+	}
+	if (i == ZCRYPT_MAX_MINOR_NODES) {
+		rc = -ENOSPC;
+		goto unlockout;
+	}
+
+	/* alloc and prepare a new zcdn device */
+	zcdndev = kzalloc_obj(*zcdndev);
+	if (!zcdndev) {
+		rc = -ENOMEM;
+		goto unlockout;
+	}
+	zcdndev->device.release = zcdn_device_release;
+	zcdndev->device.class = &zcrypt_class;
+	zcdndev->device.devt = devt;
+	zcdndev->device.groups = zcdn_dev_attr_groups;
+	if (name[0])
+		rc = dev_set_name(&zcdndev->device, "%s", name);
+	else
+		rc = dev_set_name(&zcdndev->device, ZCRYPT_NAME "_%d", (int)MINOR(devt));
+	if (rc) {
+		kfree(zcdndev);
+		goto unlockout;
+	}
+	rc = device_register(&zcdndev->device);
+	if (rc) {
+		put_device(&zcdndev->device);
+		goto unlockout;
+	}
+
+	ZCRYPT_DBF_INFO("%s created zcdn device %d:%d\n",
+			__func__, MAJOR(devt), MINOR(devt));
+
+unlockout:
+	mutex_unlock(&ap_attr_mutex);
+	return rc;
+}
+
+static int zcdn_destroy(const char *name)
+{
+	int rc = 0;
+	struct zcdn_device *zcdndev;
+
+	if (mutex_lock_interruptible(&ap_attr_mutex))
+		return -ERESTARTSYS;
+
+	/* try to find this zcdn device */
+	zcdndev = find_zcdndev_by_name(name);
+	if (!zcdndev) {
+		rc = -ENOENT;
+		goto unlockout;
+	}
+
+	/*
+	 * The zcdn device is not hard destroyed. It is subject to
+	 * reference counting and thus just needs to be unregistered.
+	 */
+	put_device(&zcdndev->device);
+	device_unregister(&zcdndev->device);
+
+unlockout:
+	mutex_unlock(&ap_attr_mutex);
+	return rc;
+}
+
+static void zcdn_destroy_all(void)
+{
+	int i;
+	dev_t devt;
+	struct zcdn_device *zcdndev;
+
+	mutex_lock(&ap_attr_mutex);
+	for (i = 0; i < ZCRYPT_MAX_MINOR_NODES; i++) {
+		devt = MKDEV(MAJOR(zcrypt_devt), MINOR(zcrypt_devt) + i);
+		zcdndev = find_zcdndev_by_devt(devt);
+		if (zcdndev) {
+			put_device(&zcdndev->device);
+			device_unregister(&zcdndev->device);
+		}
+	}
+	mutex_unlock(&ap_attr_mutex);
+}
+
+/*
+ * zcrypt_read (): Not supported beyond zcrypt 1.3.1.
+ *
+ * This function is not supported beyond zcrypt 1.3.1.
+ */
+static ssize_t zcrypt_read(struct file *filp, char __user *buf,
+			   size_t count, loff_t *f_pos)
+{
+	return -EPERM;
+}
+
+/*
+ * zcrypt_write(): Not allowed.
+ *
+ * Write is not allowed
+ */
+static ssize_t zcrypt_write(struct file *filp, const char __user *buf,
+			    size_t count, loff_t *f_pos)
+{
+	return -EPERM;
+}
+
+/*
+ * zcrypt_open(): Count number of users.
+ *
+ * Device open function to count number of users.
+ */
+static int zcrypt_open(struct inode *inode, struct file *filp)
+{
+	struct ap_perms *perms = &ap_perms;
+
+	if (filp->f_inode->i_cdev == &zcrypt_cdev) {
+		struct zcdn_device *zcdndev;
+
+		if (mutex_lock_interruptible(&ap_attr_mutex))
+			return -ERESTARTSYS;
+		zcdndev = find_zcdndev_by_devt(filp->f_inode->i_rdev);
+		/* find returns a reference, no get_device() needed */
+		mutex_unlock(&ap_attr_mutex);
+		if (zcdndev)
+			perms = &zcdndev->perms;
+	}
+	filp->private_data = (void *)perms;
+
+	atomic_inc(&zcrypt_open_count);
+	return stream_open(inode, filp);
+}
+
+/*
+ * zcrypt_release(): Count number of users.
+ *
+ * Device close function to count number of users.
+ */
+static int zcrypt_release(struct inode *inode, struct file *filp)
+{
+	if (filp->f_inode->i_cdev == &zcrypt_cdev) {
+		struct zcdn_device *zcdndev;
+
+		mutex_lock(&ap_attr_mutex);
+		zcdndev = find_zcdndev_by_devt(filp->f_inode->i_rdev);
+		mutex_unlock(&ap_attr_mutex);
+		if (zcdndev) {
+			/* 2 puts here: one for find, one for open */
+			put_device(&zcdndev->device);
+			put_device(&zcdndev->device);
+		}
+	}
+
+	atomic_dec(&zcrypt_open_count);
+	return 0;
+}
+
+static inline int zcrypt_check_ioctl(struct ap_perms *perms,
+				     unsigned int cmd)
+{
+	int rc = -EPERM;
+	int ioctlnr = (cmd & _IOC_NRMASK) >> _IOC_NRSHIFT;
+
+	if (ioctlnr > 0 && ioctlnr < AP_IOCTLS) {
+		if (test_bit_inv(ioctlnr, perms->ioctlm))
+			rc = 0;
+	}
+
+	if (rc)
+		ZCRYPT_DBF_WARN("%s ioctl check failed: ioctlnr=0x%04x rc=%d\n",
+				__func__, ioctlnr, rc);
+
+	return rc;
+}
+
+static inline bool zcrypt_check_card(struct ap_perms *perms, int card)
+{
+	return test_bit_inv(card, perms->apm) ? true : false;
+}
+
+static inline bool zcrypt_check_queue(struct ap_perms *perms, int queue)
+{
+	return test_bit_inv(queue, perms->aqm) ? true : false;
+}
+
+static inline struct zcrypt_queue *zcrypt_pick_queue(struct zcrypt_card *zc,
+						     struct zcrypt_queue *zq,
+						     struct module **pmod,
+						     unsigned int weight)
+{
+	if (!zq || !try_module_get(zq->queue->ap_dev.device.driver->owner))
+		return NULL;
+	zcrypt_card_get(zc);
+	zcrypt_queue_get(zq);
+	get_device(&zq->queue->ap_dev.device);
+	atomic_add(weight, &zc->load);
+	atomic_add(weight, &zq->load);
+	zq->request_count++;
+	*pmod = zq->queue->ap_dev.device.driver->owner;
+	return zq;
+}
+
+static inline void zcrypt_drop_queue(struct zcrypt_card *zc,
+				     struct zcrypt_queue *zq,
+				     struct module *mod,
+				     unsigned int weight)
+{
+	zq->request_count--;
+	atomic_sub(weight, &zc->load);
+	atomic_sub(weight, &zq->load);
+	put_device(&zq->queue->ap_dev.device);
+	zcrypt_queue_put(zq);
+	zcrypt_card_put(zc);
+	module_put(mod);
+}
+
+static inline bool zcrypt_card_compare(struct zcrypt_card *zc,
+				       struct zcrypt_card *pref_zc,
+				       unsigned int weight,
+				       unsigned int pref_weight)
+{
+	if (!pref_zc)
+		return true;
+	weight += atomic_read(&zc->load);
+	pref_weight += atomic_read(&pref_zc->load);
+	if (weight == pref_weight)
+		return atomic64_read(&zc->card->total_request_count) <
+			atomic64_read(&pref_zc->card->total_request_count);
+	return weight < pref_weight;
+}
+
+static inline bool zcrypt_queue_compare(struct zcrypt_queue *zq,
+					struct zcrypt_queue *pref_zq,
+					unsigned int weight,
+					unsigned int pref_weight)
+{
+	if (!pref_zq)
+		return true;
+	weight += atomic_read(&zq->load);
+	pref_weight += atomic_read(&pref_zq->load);
+	if (weight == pref_weight)
+		return zq->queue->total_request_count <
+			pref_zq->queue->total_request_count;
+	return weight < pref_weight;
+}
+
+/*
+ * zcrypt ioctls.
+ */
+static long zcrypt_rsa_modexpo(struct ap_perms *perms,
+			       struct zcrypt_track *tr,
+			       struct ica_rsa_modexpo *mex)
+{
+	struct zcrypt_card *zc, *pref_zc;
+	struct zcrypt_queue *zq, *pref_zq;
+	struct ap_message ap_msg;
+	unsigned int wgt = 0, pref_wgt = 0;
+	unsigned int func_code = 0;
+	int cpen, qpen, qid = 0, rc;
+	struct module *mod;
+
+	trace_s390_zcrypt_req(mex, TP_ICARSAMODEXPO);
+
+	rc = ap_init_apmsg(&ap_msg, 0);
+	if (rc)
+		goto out;
+
+	if (mex->outputdatalength < mex->inputdatalength) {
+		rc = -EINVAL;
+		goto out;
+	}
+
+	/*
+	 * As long as outputdatalength is big enough, we can set the
+	 * outputdatalength equal to the inputdatalength, since that is the
+	 * number of bytes we will copy in any case
+	 */
+	mex->outputdatalength = mex->inputdatalength;
+
+	rc = get_rsa_modex_fc(mex, &func_code);
+	if (rc)
+		goto out;
+
+	pref_zc = NULL;
+	pref_zq = NULL;
+	spin_lock(&zcrypt_list_lock);
+	for_each_zcrypt_card(zc) {
+		/* Check for usable accelerator or CCA card */
+		if (!zc->online || !zc->card->config || zc->card->chkstop ||
+		    !(zc->card->hwinfo.accel || zc->card->hwinfo.cca))
+			continue;
+		/* Check for size limits */
+		if (zc->min_mod_size > mex->inputdatalength ||
+		    zc->max_mod_size < mex->inputdatalength)
+			continue;
+		/* check if device node has admission for this card */
+		if (!zcrypt_check_card(perms, zc->card->id))
+			continue;
+		/* get weight index of the card device	*/
+		wgt = zc->speed_rating[func_code];
+		/* penalty if this msg was previously sent via this card */
+		cpen = (tr && tr->again_counter && tr->last_qid &&
+			AP_QID_CARD(tr->last_qid) == zc->card->id) ?
+			TRACK_AGAIN_CARD_WEIGHT_PENALTY : 0;
+		if (!zcrypt_card_compare(zc, pref_zc, wgt + cpen, pref_wgt))
+			continue;
+		for_each_zcrypt_queue(zq, zc) {
+			/* check if device is usable and eligible */
+			if (!zq->online || !zq->ops->rsa_modexpo ||
+			    !ap_queue_usable(zq->queue))
+				continue;
+			/* check if device node has admission for this queue */
+			if (!zcrypt_check_queue(perms,
+						AP_QID_QUEUE(zq->queue->qid)))
+				continue;
+			/* penalty if the msg was previously sent at this qid */
+			qpen = (tr && tr->again_counter && tr->last_qid &&
+				tr->last_qid == zq->queue->qid) ?
+				TRACK_AGAIN_QUEUE_WEIGHT_PENALTY : 0;
+			if (!zcrypt_queue_compare(zq, pref_zq,
+						  wgt + cpen + qpen, pref_wgt))
+				continue;
+			pref_zc = zc;
+			pref_zq = zq;
+			pref_wgt = wgt + cpen + qpen;
+		}
+	}
+	pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, &mod, wgt);
+	spin_unlock(&zcrypt_list_lock);
+
+	if (!pref_zq) {
+		pr_debug("no matching queue found => ENODEV\n");
+		rc = -ENODEV;
+		goto out;
+	}
+
+	qid = pref_zq->queue->qid;
+	rc = pref_zq->ops->rsa_modexpo(pref_zq, mex, &ap_msg);
+
+	spin_lock(&zcrypt_list_lock);
+	zcrypt_drop_queue(pref_zc, pref_zq, mod, wgt);
+	spin_unlock(&zcrypt_list_lock);
+
+out:
+	ap_release_apmsg(&ap_msg);
+	if (tr) {
+		tr->last_rc = rc;
+		tr->last_qid = qid;
+	}
+	trace_s390_zcrypt_rep(mex, func_code, rc,
+			      AP_QID_CARD(qid), AP_QID_QUEUE(qid),
+			      ap_msg.psmid);
+	return rc;
+}
+
+static long zcrypt_rsa_crt(struct ap_perms *perms,
+			   struct zcrypt_track *tr,
+			   struct ica_rsa_modexpo_crt *crt)
+{
+	struct zcrypt_card *zc, *pref_zc;
+	struct zcrypt_queue *zq, *pref_zq;
+	struct ap_message ap_msg;
+	unsigned int wgt = 0, pref_wgt = 0;
+	unsigned int func_code = 0;
+	int cpen, qpen, qid = 0, rc;
+	struct module *mod;
+
+	trace_s390_zcrypt_req(crt, TP_ICARSACRT);
+
+	rc = ap_init_apmsg(&ap_msg, 0);
+	if (rc)
+		goto out;
+
+	if (crt->outputdatalength < crt->inputdatalength) {
+		rc = -EINVAL;
+		goto out;
+	}
+
+	/*
+	 * As long as outputdatalength is big enough, we can set the
+	 * outputdatalength equal to the inputdatalength, since that is the
+	 * number of bytes we will copy in any case
+	 */
+	crt->outputdatalength = crt->inputdatalength;
+
+	rc = get_rsa_crt_fc(crt, &func_code);
+	if (rc)
+		goto out;
+
+	pref_zc = NULL;
+	pref_zq = NULL;
+	spin_lock(&zcrypt_list_lock);
+	for_each_zcrypt_card(zc) {
+		/* Check for usable accelerator or CCA card */
+		if (!zc->online || !zc->card->config || zc->card->chkstop ||
+		    !(zc->card->hwinfo.accel || zc->card->hwinfo.cca))
+			continue;
+		/* Check for size limits */
+		if (zc->min_mod_size > crt->inputdatalength ||
+		    zc->max_mod_size < crt->inputdatalength)
+			continue;
+		/* check if device node has admission for this card */
+		if (!zcrypt_check_card(perms, zc->card->id))
+			continue;
+		/* get weight index of the card device	*/
+		wgt = zc->speed_rating[func_code];
+		/* penalty if this msg was previously sent via this card */
+		cpen = (tr && tr->again_counter && tr->last_qid &&
+			AP_QID_CARD(tr->last_qid) == zc->card->id) ?
+			TRACK_AGAIN_CARD_WEIGHT_PENALTY : 0;
+		if (!zcrypt_card_compare(zc, pref_zc, wgt + cpen, pref_wgt))
+			continue;
+		for_each_zcrypt_queue(zq, zc) {
+			/* check if device is usable and eligible */
+			if (!zq->online || !zq->ops->rsa_modexpo_crt ||
+			    !ap_queue_usable(zq->queue))
+				continue;
+			/* check if device node has admission for this queue */
+			if (!zcrypt_check_queue(perms,
+						AP_QID_QUEUE(zq->queue->qid)))
+				continue;
+			/* penalty if the msg was previously sent at this qid */
+			qpen = (tr && tr->again_counter && tr->last_qid &&
+				tr->last_qid == zq->queue->qid) ?
+				TRACK_AGAIN_QUEUE_WEIGHT_PENALTY : 0;
+			if (!zcrypt_queue_compare(zq, pref_zq,
+						  wgt + cpen + qpen, pref_wgt))
+				continue;
+			pref_zc = zc;
+			pref_zq = zq;
+			pref_wgt = wgt + cpen + qpen;
+		}
+	}
+	pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, &mod, wgt);
+	spin_unlock(&zcrypt_list_lock);
+
+	if (!pref_zq) {
+		pr_debug("no matching queue found => ENODEV\n");
+		rc = -ENODEV;
+		goto out;
+	}
+
+	qid = pref_zq->queue->qid;
+	rc = pref_zq->ops->rsa_modexpo_crt(pref_zq, crt, &ap_msg);
+
+	spin_lock(&zcrypt_list_lock);
+	zcrypt_drop_queue(pref_zc, pref_zq, mod, wgt);
+	spin_unlock(&zcrypt_list_lock);
+
+out:
+	ap_release_apmsg(&ap_msg);
+	if (tr) {
+		tr->last_rc = rc;
+		tr->last_qid = qid;
+	}
+	trace_s390_zcrypt_rep(crt, func_code, rc,
+			      AP_QID_CARD(qid), AP_QID_QUEUE(qid),
+			      ap_msg.psmid);
+	return rc;
+}
+
+static long _zcrypt_send_cprb(u32 xflags, struct ap_perms *perms,
+			      struct zcrypt_track *tr,
+			      struct ica_xcRB *xcrb)
+{
+	bool userspace = xflags & ZCRYPT_XFLAG_USERSPACE;
+	unsigned int card, domain, func_code = 0;
+	unsigned int wgt = 0, pref_wgt = 0;
+	struct zcrypt_queue *zq, *pref_zq;
+	struct zcrypt_card *zc, *pref_zc;
+	int cpen, qpen, qid = 0, rc;
+	struct ap_message ap_msg;
+	struct module *mod;
+
+	trace_s390_zcrypt_req(xcrb, TB_ZSECSENDCPRB);
+
+	xcrb->status = 0;
+
+	rc = ap_init_apmsg(&ap_msg, xflags & ZCRYPT_XFLAG_NOMEMALLOC ?
+			   AP_MSG_FLAG_MEMPOOL : 0);
+	if (rc)
+		goto out;
+
+	rc = prep_cca_ap_msg(userspace, xcrb, &ap_msg, &func_code, &domain);
+	if (rc)
+		goto out;
+	print_hex_dump_debug("ccareq: ", DUMP_PREFIX_ADDRESS, 16, 1,
+			     ap_msg.msg, ap_msg.len, false);
+
+	if (perms != &ap_perms && domain < AP_DOMAINS) {
+		if (ap_msg.flags & AP_MSG_FLAG_ADMIN) {
+			domain = array_index_nospec(domain, AP_DOMAINS);
+			if (!test_bit_inv(domain, perms->adm)) {
+				rc = -ENODEV;
+				goto out;
+			}
+		} else if ((ap_msg.flags & AP_MSG_FLAG_USAGE) == 0) {
+			rc = -EOPNOTSUPP;
+			goto out;
+		}
+	}
+	/*
+	 * If a valid target domain is set and this domain is NOT a usage
+	 * domain but a control only domain, autoselect target domain.
+	 */
+	if (domain < AP_DOMAINS &&
+	    !ap_test_config_usage_domain(domain) &&
+	    ap_test_config_ctrl_domain(domain))
+		domain = AUTOSEL_DOM;
+
+	pref_zc = NULL;
+	pref_zq = NULL;
+	card = xcrb->user_defined;
+	spin_lock(&zcrypt_list_lock);
+	for_each_zcrypt_card(zc) {
+		/* Check for usable CCA card */
+		if (!zc->online || !zc->card->config || zc->card->chkstop ||
+		    !zc->card->hwinfo.cca)
+			continue;
+		/* Check for user selected CCA card */
+		if (card != AUTOSELECT && card != zc->card->id)
+			continue;
+		/* check if request size exceeds card max msg size */
+		if (ap_msg.len > zc->card->maxmsgsize)
+			continue;
+		/* check if device node has admission for this card */
+		if (!zcrypt_check_card(perms, zc->card->id))
+			continue;
+		/* get weight index of the card device	*/
+		wgt = speed_idx_cca(func_code) * zc->speed_rating[SECKEY];
+		/* penalty if this msg was previously sent via this card */
+		cpen = (tr && tr->again_counter && tr->last_qid &&
+			AP_QID_CARD(tr->last_qid) == zc->card->id) ?
+			TRACK_AGAIN_CARD_WEIGHT_PENALTY : 0;
+		if (!zcrypt_card_compare(zc, pref_zc, wgt + cpen, pref_wgt))
+			continue;
+		for_each_zcrypt_queue(zq, zc) {
+			/* check for device usable and eligible */
+			if (!zq->online || !zq->ops->send_cprb ||
+			    !ap_queue_usable(zq->queue) ||
+			    (domain != AUTOSEL_DOM &&
+			     domain != AP_QID_QUEUE(zq->queue->qid)))
+				continue;
+			/* check if device node has admission for this queue */
+			if (!zcrypt_check_queue(perms,
+						AP_QID_QUEUE(zq->queue->qid)))
+				continue;
+			/* penalty if the msg was previously sent at this qid */
+			qpen = (tr && tr->again_counter && tr->last_qid &&
+				tr->last_qid == zq->queue->qid) ?
+				TRACK_AGAIN_QUEUE_WEIGHT_PENALTY : 0;
+			if (!zcrypt_queue_compare(zq, pref_zq,
+						  wgt + cpen + qpen, pref_wgt))
+				continue;
+			pref_zc = zc;
+			pref_zq = zq;
+			pref_wgt = wgt + cpen + qpen;
+		}
+	}
+	pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, &mod, wgt);
+	spin_unlock(&zcrypt_list_lock);
+
+	if (!pref_zq) {
+		pr_debug("no match for address %02x.%04x => ENODEV\n",
+			 card, domain);
+		rc = -ENODEV;
+		goto out;
+	}
+
+	rc = pref_zq->ops->send_cprb(userspace, pref_zq, xcrb, &ap_msg);
+	if (!rc) {
+		print_hex_dump_debug("ccarpl: ", DUMP_PREFIX_ADDRESS, 16, 1,
+				     ap_msg.msg, ap_msg.len, false);
+	}
+
+	spin_lock(&zcrypt_list_lock);
+	zcrypt_drop_queue(pref_zc, pref_zq, mod, wgt);
+	spin_unlock(&zcrypt_list_lock);
+
+out:
+	ap_release_apmsg(&ap_msg);
+	if (tr) {
+		tr->last_rc = rc;
+		tr->last_qid = qid;
+	}
+	trace_s390_zcrypt_rep(xcrb, func_code, rc,
+			      AP_QID_CARD(qid), AP_QID_QUEUE(qid),
+			      ap_msg.psmid);
+	return rc;
+}
+
+long zcrypt_send_cprb(struct ica_xcRB *xcrb, u32 xflags)
+{
+	struct zcrypt_track tr;
+	int rc;
+
+	memset(&tr, 0, sizeof(tr));
+
+	do {
+		rc = _zcrypt_send_cprb(xflags, &ap_perms, &tr, xcrb);
+	} while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
+
+	/* on ENODEV failure: retry once again after a requested rescan */
+	if (rc == -ENODEV && zcrypt_process_rescan())
+		do {
+			rc = _zcrypt_send_cprb(xflags, &ap_perms, &tr, xcrb);
+		} while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
+	if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX)
+		rc = -EIO;
+	if (rc)
+		pr_debug("rc=%d\n", rc);
+
+	return rc;
+}
+EXPORT_SYMBOL(zcrypt_send_cprb);
+
+static bool is_desired_ep11_card(unsigned int dev_id,
+				 unsigned short target_num,
+				 struct ep11_target_dev *targets)
+{
+	while (target_num-- > 0) {
+		if (targets->ap_id == dev_id || targets->ap_id == AUTOSEL_AP)
+			return true;
+		targets++;
+	}
+	return false;
+}
+
+static bool is_desired_ep11_queue(unsigned int dev_qid,
+				  unsigned short target_num,
+				  struct ep11_target_dev *targets)
+{
+	int card = AP_QID_CARD(dev_qid), dom = AP_QID_QUEUE(dev_qid);
+
+	while (target_num-- > 0) {
+		if ((targets->ap_id == card || targets->ap_id == AUTOSEL_AP) &&
+		    (targets->dom_id == dom || targets->dom_id == AUTOSEL_DOM))
+			return true;
+		targets++;
+	}
+	return false;
+}
+
+static long _zcrypt_send_ep11_cprb(u32 xflags, struct ap_perms *perms,
+				   struct zcrypt_track *tr,
+				   struct ep11_urb *xcrb)
+{
+	bool userspace = xflags & ZCRYPT_XFLAG_USERSPACE;
+	struct zcrypt_card *zc, *pref_zc;
+	struct zcrypt_queue *zq, *pref_zq;
+	struct ep11_target_dev *targets = NULL;
+	unsigned short target_num;
+	unsigned int wgt = 0, pref_wgt = 0;
+	unsigned int func_code = 0, domain;
+	struct ap_message ap_msg;
+	int cpen, qpen, qid = 0, rc;
+	struct module *mod;
+
+	trace_s390_zcrypt_req(xcrb, TP_ZSENDEP11CPRB);
+
+	rc = ap_init_apmsg(&ap_msg, xflags & ZCRYPT_XFLAG_NOMEMALLOC ?
+			   AP_MSG_FLAG_MEMPOOL : 0);
+	if (rc)
+		goto out;
+
+	target_num = (unsigned short)xcrb->targets_num;
+
+	/* empty list indicates autoselect (all available targets) */
+	rc = -ENOMEM;
+	if (target_num != 0) {
+		if (userspace) {
+			targets = kzalloc_objs(*targets, target_num);
+			if (!targets)
+				goto out;
+			if (copy_from_user(targets, xcrb->targets,
+					   target_num * sizeof(*targets))) {
+				rc = -EFAULT;
+				goto out;
+			}
+		} else {
+			targets = (struct ep11_target_dev __force __kernel *)xcrb->targets;
+		}
+	}
+
+	rc = prep_ep11_ap_msg(userspace, xcrb, &ap_msg, &func_code, &domain);
+	if (rc)
+		goto out;
+	print_hex_dump_debug("ep11req: ", DUMP_PREFIX_ADDRESS, 16, 1,
+			     ap_msg.msg, ap_msg.len, false);
+
+	if (perms != &ap_perms && domain < AP_DOMAINS) {
+		if (ap_msg.flags & AP_MSG_FLAG_ADMIN) {
+			domain = array_index_nospec(domain, AP_DOMAINS);
+			if (!test_bit_inv(domain, perms->adm)) {
+				rc = -ENODEV;
+				goto out;
+			}
+		} else if ((ap_msg.flags & AP_MSG_FLAG_USAGE) == 0) {
+			rc = -EOPNOTSUPP;
+			goto out;
+		}
+	}
+
+	pref_zc = NULL;
+	pref_zq = NULL;
+	spin_lock(&zcrypt_list_lock);
+	for_each_zcrypt_card(zc) {
+		/* Check for usable EP11 card */
+		if (!zc->online || !zc->card->config || zc->card->chkstop ||
+		    !zc->card->hwinfo.ep11)
+			continue;
+		/* Check for user selected EP11 card */
+		if (targets &&
+		    !is_desired_ep11_card(zc->card->id, target_num, targets))
+			continue;
+		/* check if request size exceeds card max msg size */
+		if (ap_msg.len > zc->card->maxmsgsize)
+			continue;
+		/* check if device node has admission for this card */
+		if (!zcrypt_check_card(perms, zc->card->id))
+			continue;
+		/* get weight index of the card device	*/
+		wgt = speed_idx_ep11(func_code) * zc->speed_rating[SECKEY];
+		/* penalty if this msg was previously sent via this card */
+		cpen = (tr && tr->again_counter && tr->last_qid &&
+			AP_QID_CARD(tr->last_qid) == zc->card->id) ?
+			TRACK_AGAIN_CARD_WEIGHT_PENALTY : 0;
+		if (!zcrypt_card_compare(zc, pref_zc, wgt + cpen, pref_wgt))
+			continue;
+		for_each_zcrypt_queue(zq, zc) {
+			/* check if device is usable and eligible */
+			if (!zq->online || !zq->ops->send_ep11_cprb ||
+			    !ap_queue_usable(zq->queue) ||
+			    (targets &&
+			     !is_desired_ep11_queue(zq->queue->qid,
+						    target_num, targets)))
+				continue;
+			/* check if device node has admission for this queue */
+			if (!zcrypt_check_queue(perms,
+						AP_QID_QUEUE(zq->queue->qid)))
+				continue;
+			/* penalty if the msg was previously sent at this qid */
+			qpen = (tr && tr->again_counter && tr->last_qid &&
+				tr->last_qid == zq->queue->qid) ?
+				TRACK_AGAIN_QUEUE_WEIGHT_PENALTY : 0;
+			if (!zcrypt_queue_compare(zq, pref_zq,
+						  wgt + cpen + qpen, pref_wgt))
+				continue;
+			pref_zc = zc;
+			pref_zq = zq;
+			pref_wgt = wgt + cpen + qpen;
+		}
+	}
+	pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, &mod, wgt);
+	spin_unlock(&zcrypt_list_lock);
+
+	if (!pref_zq) {
+		if (targets && target_num == 1) {
+			pr_debug("no match for address %02x.%04x => ENODEV\n",
+				 (int)targets->ap_id, (int)targets->dom_id);
+		} else if (targets) {
+			pr_debug("no match for %d target addrs => ENODEV\n",
+				 (int)target_num);
+		} else {
+			pr_debug("no match for address ff.ffff => ENODEV\n");
+		}
+		rc = -ENODEV;
+		goto out;
+	}
+
+	qid = pref_zq->queue->qid;
+	rc = pref_zq->ops->send_ep11_cprb(userspace, pref_zq, xcrb, &ap_msg);
+	if (!rc) {
+		print_hex_dump_debug("ep11rpl: ", DUMP_PREFIX_ADDRESS, 16, 1,
+				     ap_msg.msg, ap_msg.len, false);
+	}
+
+	spin_lock(&zcrypt_list_lock);
+	zcrypt_drop_queue(pref_zc, pref_zq, mod, wgt);
+	spin_unlock(&zcrypt_list_lock);
+
+out:
+	if (userspace)
+		kfree(targets);
+	ap_release_apmsg(&ap_msg);
+	if (tr) {
+		tr->last_rc = rc;
+		tr->last_qid = qid;
+	}
+	trace_s390_zcrypt_rep(xcrb, func_code, rc,
+			      AP_QID_CARD(qid), AP_QID_QUEUE(qid),
+			      ap_msg.psmid);
+	return rc;
+}
+
+long zcrypt_send_ep11_cprb(struct ep11_urb *xcrb, u32 xflags)
+{
+	struct zcrypt_track tr;
+	int rc;
+
+	memset(&tr, 0, sizeof(tr));
+
+	do {
+		rc = _zcrypt_send_ep11_cprb(xflags, &ap_perms, &tr, xcrb);
+	} while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
+
+	/* on ENODEV failure: retry once again after a requested rescan */
+	if (rc == -ENODEV && zcrypt_process_rescan())
+		do {
+			rc = _zcrypt_send_ep11_cprb(xflags, &ap_perms, &tr, xcrb);
+		} while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
+	if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX)
+		rc = -EIO;
+	if (rc)
+		pr_debug("rc=%d\n", rc);
+
+	return rc;
+}
+EXPORT_SYMBOL(zcrypt_send_ep11_cprb);
+
+static long zcrypt_rng(char *buffer)
+{
+	struct zcrypt_card *zc, *pref_zc;
+	struct zcrypt_queue *zq, *pref_zq;
+	unsigned int wgt = 0, pref_wgt = 0;
+	unsigned int func_code = 0;
+	struct ap_message ap_msg;
+	int qid = 0, rc = -ENODEV;
+	struct module *mod;
+
+	trace_s390_zcrypt_req(buffer, TP_HWRNGCPRB);
+
+	rc = ap_init_apmsg(&ap_msg, 0);
+	if (rc)
+		goto out;
+	rc = prep_rng_ap_msg(&ap_msg, &func_code, NULL);
+	if (rc)
+		goto out;
+
+	pref_zc = NULL;
+	pref_zq = NULL;
+	spin_lock(&zcrypt_list_lock);
+	for_each_zcrypt_card(zc) {
+		/* Check for usable CCA card */
+		if (!zc->online || !zc->card->config || zc->card->chkstop ||
+		    !zc->card->hwinfo.cca)
+			continue;
+		/* get weight index of the card device	*/
+		wgt = zc->speed_rating[func_code];
+		if (!zcrypt_card_compare(zc, pref_zc, wgt, pref_wgt))
+			continue;
+		for_each_zcrypt_queue(zq, zc) {
+			/* check if device is usable and eligible */
+			if (!zq->online || !zq->ops->rng ||
+			    !ap_queue_usable(zq->queue))
+				continue;
+			if (!zcrypt_queue_compare(zq, pref_zq, wgt, pref_wgt))
+				continue;
+			pref_zc = zc;
+			pref_zq = zq;
+			pref_wgt = wgt;
+		}
+	}
+	pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, &mod, wgt);
+	spin_unlock(&zcrypt_list_lock);
+
+	if (!pref_zq) {
+		pr_debug("no matching queue found => ENODEV\n");
+		rc = -ENODEV;
+		goto out;
+	}
+
+	qid = pref_zq->queue->qid;
+	rc = pref_zq->ops->rng(pref_zq, buffer, &ap_msg);
+
+	spin_lock(&zcrypt_list_lock);
+	zcrypt_drop_queue(pref_zc, pref_zq, mod, wgt);
+	spin_unlock(&zcrypt_list_lock);
+
+out:
+	ap_release_apmsg(&ap_msg);
+	trace_s390_zcrypt_rep(buffer, func_code, rc,
+			      AP_QID_CARD(qid), AP_QID_QUEUE(qid),
+			      ap_msg.psmid);
+	return rc;
+}
+
+static void zcrypt_device_status_mask(struct zcrypt_device_status *devstatus)
+{
+	struct zcrypt_card *zc;
+	struct zcrypt_queue *zq;
+	struct zcrypt_device_status *stat;
+	int card, queue;
+
+	memset(devstatus, 0, MAX_ZDEV_ENTRIES
+	       * sizeof(struct zcrypt_device_status));
+
+	spin_lock(&zcrypt_list_lock);
+	for_each_zcrypt_card(zc) {
+		for_each_zcrypt_queue(zq, zc) {
+			card = AP_QID_CARD(zq->queue->qid);
+			if (card >= MAX_ZDEV_CARDIDS)
+				continue;
+			queue = AP_QID_QUEUE(zq->queue->qid);
+			stat = &devstatus[card * AP_DOMAINS + queue];
+			stat->hwtype = zc->card->ap_dev.device_type;
+			stat->functions = zc->card->hwinfo.fac >> 26;
+			stat->qid = zq->queue->qid;
+			stat->online = zq->online ? 0x01 : 0x00;
+		}
+	}
+	spin_unlock(&zcrypt_list_lock);
+}
+
+void zcrypt_device_status_mask_ext(struct zcrypt_device_status_ext *devstatus,
+				   int maxcard, int maxqueue)
+{
+	struct zcrypt_card *zc;
+	struct zcrypt_queue *zq;
+	struct zcrypt_device_status_ext *stat;
+	int card, queue;
+
+	maxcard = min_t(int, maxcard, MAX_ZDEV_CARDIDS_EXT);
+	maxqueue = min_t(int, maxqueue, MAX_ZDEV_DOMAINS_EXT);
+
+	spin_lock(&zcrypt_list_lock);
+	for_each_zcrypt_card(zc) {
+		for_each_zcrypt_queue(zq, zc) {
+			card = AP_QID_CARD(zq->queue->qid);
+			queue = AP_QID_QUEUE(zq->queue->qid);
+			if (card >= maxcard || queue >= maxqueue)
+				continue;
+			stat = &devstatus[card * maxqueue + queue];
+			stat->hwtype = zc->card->ap_dev.device_type;
+			stat->functions = zc->card->hwinfo.fac >> 26;
+			stat->qid = zq->queue->qid;
+			stat->online = zq->online ? 0x01 : 0x00;
+		}
+	}
+	spin_unlock(&zcrypt_list_lock);
+}
+EXPORT_SYMBOL(zcrypt_device_status_mask_ext);
+
+int zcrypt_device_status_ext(int card, int queue,
+			     struct zcrypt_device_status_ext *devstat)
+{
+	struct zcrypt_card *zc;
+	struct zcrypt_queue *zq;
+
+	memset(devstat, 0, sizeof(*devstat));
+
+	spin_lock(&zcrypt_list_lock);
+	for_each_zcrypt_card(zc) {
+		for_each_zcrypt_queue(zq, zc) {
+			if (card == AP_QID_CARD(zq->queue->qid) &&
+			    queue == AP_QID_QUEUE(zq->queue->qid)) {
+				devstat->hwtype = zc->card->ap_dev.device_type;
+				devstat->functions = zc->card->hwinfo.fac >> 26;
+				devstat->qid = zq->queue->qid;
+				devstat->online = zq->online ? 0x01 : 0x00;
+				spin_unlock(&zcrypt_list_lock);
+				return 0;
+			}
+		}
+	}
+	spin_unlock(&zcrypt_list_lock);
+
+	return -ENODEV;
+}
+EXPORT_SYMBOL(zcrypt_device_status_ext);
+
+static void zcrypt_status_mask(char status[], size_t max_adapters)
+{
+	struct zcrypt_card *zc;
+	struct zcrypt_queue *zq;
+	int card;
+
+	memset(status, 0, max_adapters);
+	spin_lock(&zcrypt_list_lock);
+	for_each_zcrypt_card(zc) {
+		for_each_zcrypt_queue(zq, zc) {
+			card = AP_QID_CARD(zq->queue->qid);
+			if (AP_QID_QUEUE(zq->queue->qid) != ap_domain_index ||
+			    card >= max_adapters)
+				continue;
+			status[card] = zc->online ? zc->user_space_type : 0x0d;
+		}
+	}
+	spin_unlock(&zcrypt_list_lock);
+}
+
+static void zcrypt_qdepth_mask(char qdepth[], size_t max_adapters)
+{
+	struct zcrypt_card *zc;
+	struct zcrypt_queue *zq;
+	int card;
+
+	memset(qdepth, 0, max_adapters);
+	spin_lock(&zcrypt_list_lock);
+	local_bh_disable();
+	for_each_zcrypt_card(zc) {
+		for_each_zcrypt_queue(zq, zc) {
+			card = AP_QID_CARD(zq->queue->qid);
+			if (AP_QID_QUEUE(zq->queue->qid) != ap_domain_index ||
+			    card >= max_adapters)
+				continue;
+			spin_lock(&zq->queue->lock);
+			qdepth[card] =
+				zq->queue->pendingq_count +
+				zq->queue->requestq_count;
+			spin_unlock(&zq->queue->lock);
+		}
+	}
+	local_bh_enable();
+	spin_unlock(&zcrypt_list_lock);
+}
+
+static void zcrypt_perdev_reqcnt(u32 reqcnt[], size_t max_adapters)
+{
+	struct zcrypt_card *zc;
+	struct zcrypt_queue *zq;
+	int card;
+	u64 cnt;
+
+	memset(reqcnt, 0, sizeof(int) * max_adapters);
+	spin_lock(&zcrypt_list_lock);
+	local_bh_disable();
+	for_each_zcrypt_card(zc) {
+		for_each_zcrypt_queue(zq, zc) {
+			card = AP_QID_CARD(zq->queue->qid);
+			if (AP_QID_QUEUE(zq->queue->qid) != ap_domain_index ||
+			    card >= max_adapters)
+				continue;
+			spin_lock(&zq->queue->lock);
+			cnt = zq->queue->total_request_count;
+			spin_unlock(&zq->queue->lock);
+			reqcnt[card] = (cnt < UINT_MAX) ? (u32)cnt : UINT_MAX;
+		}
+	}
+	local_bh_enable();
+	spin_unlock(&zcrypt_list_lock);
+}
+
+static int zcrypt_pendingq_count(void)
+{
+	struct zcrypt_card *zc;
+	struct zcrypt_queue *zq;
+	int pendingq_count;
+
+	pendingq_count = 0;
+	spin_lock(&zcrypt_list_lock);
+	local_bh_disable();
+	for_each_zcrypt_card(zc) {
+		for_each_zcrypt_queue(zq, zc) {
+			if (AP_QID_QUEUE(zq->queue->qid) != ap_domain_index)
+				continue;
+			spin_lock(&zq->queue->lock);
+			pendingq_count += zq->queue->pendingq_count;
+			spin_unlock(&zq->queue->lock);
+		}
+	}
+	local_bh_enable();
+	spin_unlock(&zcrypt_list_lock);
+	return pendingq_count;
+}
+
+static int zcrypt_requestq_count(void)
+{
+	struct zcrypt_card *zc;
+	struct zcrypt_queue *zq;
+	int requestq_count;
+
+	requestq_count = 0;
+	spin_lock(&zcrypt_list_lock);
+	local_bh_disable();
+	for_each_zcrypt_card(zc) {
+		for_each_zcrypt_queue(zq, zc) {
+			if (AP_QID_QUEUE(zq->queue->qid) != ap_domain_index)
+				continue;
+			spin_lock(&zq->queue->lock);
+			requestq_count += zq->queue->requestq_count;
+			spin_unlock(&zq->queue->lock);
+		}
+	}
+	local_bh_enable();
+	spin_unlock(&zcrypt_list_lock);
+	return requestq_count;
+}
+
+static int icarsamodexpo_ioctl(struct ap_perms *perms, unsigned long arg)
+{
+	int rc;
+	struct zcrypt_track tr;
+	struct ica_rsa_modexpo mex;
+	struct ica_rsa_modexpo __user *umex = (void __user *)arg;
+
+	memset(&tr, 0, sizeof(tr));
+	if (copy_from_user(&mex, umex, sizeof(mex)))
+		return -EFAULT;
+
+	do {
+		rc = zcrypt_rsa_modexpo(perms, &tr, &mex);
+	} while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
+
+	/* on ENODEV failure: retry once again after a requested rescan */
+	if (rc == -ENODEV && zcrypt_process_rescan())
+		do {
+			rc = zcrypt_rsa_modexpo(perms, &tr, &mex);
+		} while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
+	if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX)
+		rc = -EIO;
+	if (rc) {
+		pr_debug("ioctl ICARSAMODEXPO rc=%d\n", rc);
+		return rc;
+	}
+	return put_user(mex.outputdatalength, &umex->outputdatalength);
+}
+
+static int icarsacrt_ioctl(struct ap_perms *perms, unsigned long arg)
+{
+	int rc;
+	struct zcrypt_track tr;
+	struct ica_rsa_modexpo_crt crt;
+	struct ica_rsa_modexpo_crt __user *ucrt = (void __user *)arg;
+
+	memset(&tr, 0, sizeof(tr));
+	if (copy_from_user(&crt, ucrt, sizeof(crt)))
+		return -EFAULT;
+
+	do {
+		rc = zcrypt_rsa_crt(perms, &tr, &crt);
+	} while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
+
+	/* on ENODEV failure: retry once again after a requested rescan */
+	if (rc == -ENODEV && zcrypt_process_rescan())
+		do {
+			rc = zcrypt_rsa_crt(perms, &tr, &crt);
+		} while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
+	if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX)
+		rc = -EIO;
+	if (rc) {
+		pr_debug("ioctl ICARSACRT rc=%d\n", rc);
+		return rc;
+	}
+	return put_user(crt.outputdatalength, &ucrt->outputdatalength);
+}
+
+static int zsecsendcprb_ioctl(struct ap_perms *perms, unsigned long arg)
+{
+	int rc;
+	struct ica_xcRB xcrb;
+	struct zcrypt_track tr;
+	u32 xflags = ZCRYPT_XFLAG_USERSPACE;
+	struct ica_xcRB __user *uxcrb = (void __user *)arg;
+
+	memset(&tr, 0, sizeof(tr));
+	if (copy_from_user(&xcrb, uxcrb, sizeof(xcrb)))
+		return -EFAULT;
+
+	do {
+		rc = _zcrypt_send_cprb(xflags, perms, &tr, &xcrb);
+	} while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
+
+	/* on ENODEV failure: retry once again after a requested rescan */
+	if (rc == -ENODEV && zcrypt_process_rescan())
+		do {
+			rc = _zcrypt_send_cprb(xflags, perms, &tr, &xcrb);
+		} while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
+	if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX)
+		rc = -EIO;
+	if (rc)
+		pr_debug("ioctl ZSENDCPRB rc=%d status=0x%x\n",
+			 rc, xcrb.status);
+	if (copy_to_user(uxcrb, &xcrb, sizeof(xcrb)))
+		return -EFAULT;
+	return rc;
+}
+
+static int zsendep11cprb_ioctl(struct ap_perms *perms, unsigned long arg)
+{
+	int rc;
+	struct ep11_urb xcrb;
+	struct zcrypt_track tr;
+	u32 xflags = ZCRYPT_XFLAG_USERSPACE;
+	struct ep11_urb __user *uxcrb = (void __user *)arg;
+
+	memset(&tr, 0, sizeof(tr));
+	if (copy_from_user(&xcrb, uxcrb, sizeof(xcrb)))
+		return -EFAULT;
+
+	do {
+		rc = _zcrypt_send_ep11_cprb(xflags, perms, &tr, &xcrb);
+	} while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
+
+	/* on ENODEV failure: retry once again after a requested rescan */
+	if (rc == -ENODEV && zcrypt_process_rescan())
+		do {
+			rc = _zcrypt_send_ep11_cprb(xflags, perms, &tr, &xcrb);
+		} while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
+	if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX)
+		rc = -EIO;
+	if (rc)
+		pr_debug("ioctl ZSENDEP11CPRB rc=%d\n", rc);
+	if (copy_to_user(uxcrb, &xcrb, sizeof(xcrb)))
+		return -EFAULT;
+	return rc;
+}
+
+static long zcrypt_unlocked_ioctl(struct file *filp, unsigned int cmd,
+				  unsigned long arg)
+{
+	int rc;
+	struct ap_perms *perms =
+		(struct ap_perms *)filp->private_data;
+
+	rc = zcrypt_check_ioctl(perms, cmd);
+	if (rc)
+		return rc;
+
+	switch (cmd) {
+	case ICARSAMODEXPO:
+		return icarsamodexpo_ioctl(perms, arg);
+	case ICARSACRT:
+		return icarsacrt_ioctl(perms, arg);
+	case ZSECSENDCPRB:
+		return zsecsendcprb_ioctl(perms, arg);
+	case ZSENDEP11CPRB:
+		return zsendep11cprb_ioctl(perms, arg);
+	case ZCRYPT_DEVICE_STATUS: {
+		struct zcrypt_device_status_ext *device_status;
+		size_t total_size = MAX_ZDEV_ENTRIES_EXT
+			* sizeof(struct zcrypt_device_status_ext);
+
+		device_status = kvzalloc_objs(struct zcrypt_device_status_ext,
+					      MAX_ZDEV_ENTRIES_EXT);
+		if (!device_status)
+			return -ENOMEM;
+		zcrypt_device_status_mask_ext(device_status,
+					      MAX_ZDEV_CARDIDS_EXT,
+					      MAX_ZDEV_DOMAINS_EXT);
+		if (copy_to_user((char __user *)arg, device_status,
+				 total_size))
+			rc = -EFAULT;
+		kvfree(device_status);
+		return rc;
+	}
+	case ZCRYPT_STATUS_MASK: {
+		char status[AP_DEVICES];
+
+		zcrypt_status_mask(status, AP_DEVICES);
+		if (copy_to_user((char __user *)arg, status, sizeof(status)))
+			return -EFAULT;
+		return 0;
+	}
+	case ZCRYPT_QDEPTH_MASK: {
+		char qdepth[AP_DEVICES];
+
+		zcrypt_qdepth_mask(qdepth, AP_DEVICES);
+		if (copy_to_user((char __user *)arg, qdepth, sizeof(qdepth)))
+			return -EFAULT;
+		return 0;
+	}
+	case ZCRYPT_PERDEV_REQCNT: {
+		u32 *reqcnt;
+
+		reqcnt = kcalloc(AP_DEVICES, sizeof(u32), GFP_KERNEL);
+		if (!reqcnt)
+			return -ENOMEM;
+		zcrypt_perdev_reqcnt(reqcnt, AP_DEVICES);
+		if (copy_to_user((int __user *)arg, reqcnt,
+				 sizeof(u32) * AP_DEVICES))
+			rc = -EFAULT;
+		kfree(reqcnt);
+		return rc;
+	}
+	case Z90STAT_REQUESTQ_COUNT:
+		return put_user(zcrypt_requestq_count(), (int __user *)arg);
+	case Z90STAT_PENDINGQ_COUNT:
+		return put_user(zcrypt_pendingq_count(), (int __user *)arg);
+	case Z90STAT_TOTALOPEN_COUNT:
+		return put_user(atomic_read(&zcrypt_open_count),
+				(int __user *)arg);
+	case Z90STAT_DOMAIN_INDEX:
+		return put_user(ap_domain_index, (int __user *)arg);
+	/*
+	 * Deprecated ioctls
+	 */
+	case ZDEVICESTATUS: {
+		/* the old ioctl supports only 64 adapters */
+		struct zcrypt_device_status *device_status;
+		size_t total_size = MAX_ZDEV_ENTRIES
+			* sizeof(struct zcrypt_device_status);
+
+		device_status = kzalloc(total_size, GFP_KERNEL);
+		if (!device_status)
+			return -ENOMEM;
+		zcrypt_device_status_mask(device_status);
+		if (copy_to_user((char __user *)arg, device_status,
+				 total_size))
+			rc = -EFAULT;
+		kfree(device_status);
+		return rc;
+	}
+	case Z90STAT_STATUS_MASK: {
+		/* the old ioctl supports only 64 adapters */
+		char status[MAX_ZDEV_CARDIDS];
+
+		zcrypt_status_mask(status, MAX_ZDEV_CARDIDS);
+		if (copy_to_user((char __user *)arg, status, sizeof(status)))
+			return -EFAULT;
+		return 0;
+	}
+	case Z90STAT_QDEPTH_MASK: {
+		/* the old ioctl supports only 64 adapters */
+		char qdepth[MAX_ZDEV_CARDIDS];
+
+		zcrypt_qdepth_mask(qdepth, MAX_ZDEV_CARDIDS);
+		if (copy_to_user((char __user *)arg, qdepth, sizeof(qdepth)))
+			return -EFAULT;
+		return 0;
+	}
+	case Z90STAT_PERDEV_REQCNT: {
+		/* the old ioctl supports only 64 adapters */
+		u32 reqcnt[MAX_ZDEV_CARDIDS];
+
+		zcrypt_perdev_reqcnt(reqcnt, MAX_ZDEV_CARDIDS);
+		if (copy_to_user((int __user *)arg, reqcnt, sizeof(reqcnt)))
+			return -EFAULT;
+		return 0;
+	}
+	/* unknown ioctl number */
+	default:
+		pr_debug("unknown ioctl 0x%08x\n", cmd);
+		return -ENOIOCTLCMD;
+	}
+}
+
+/*
+ * Misc device file operations.
+ */
+static const struct file_operations zcrypt_fops = {
+	.owner		= THIS_MODULE,
+	.read		= zcrypt_read,
+	.write		= zcrypt_write,
+	.unlocked_ioctl	= zcrypt_unlocked_ioctl,
+	.open		= zcrypt_open,
+	.release	= zcrypt_release,
+};
+
+/*
+ * Misc device.
+ */
+static struct miscdevice zcrypt_misc_device = {
+	.minor	    = MISC_DYNAMIC_MINOR,
+	.name	    = "z90crypt",
+	.fops	    = &zcrypt_fops,
+};
+
+static int zcrypt_rng_device_count;
+static u32 *zcrypt_rng_buffer;
+static int zcrypt_rng_buffer_index;
+static DEFINE_MUTEX(zcrypt_rng_mutex);
+
+static int zcrypt_rng_data_read(struct hwrng *rng, u32 *data)
+{
+	int rc;
+
+	/*
+	 * We don't need locking here because the RNG API guarantees serialized
+	 * read method calls.
+	 */
+	if (zcrypt_rng_buffer_index == 0) {
+		rc = zcrypt_rng((char *)zcrypt_rng_buffer);
+		/* on ENODEV failure: retry once again after an AP bus rescan */
+		if (rc == -ENODEV && zcrypt_process_rescan())
+			rc = zcrypt_rng((char *)zcrypt_rng_buffer);
+		if (rc < 0)
+			return -EIO;
+		zcrypt_rng_buffer_index = rc / sizeof(*data);
+	}
+	*data = zcrypt_rng_buffer[--zcrypt_rng_buffer_index];
+	return sizeof(*data);
+}
+
+static struct hwrng zcrypt_rng_dev = {
+	.name		= "zcrypt",
+	.data_read	= zcrypt_rng_data_read,
+	.quality	= 990,
+};
+
+int zcrypt_rng_device_add(void)
+{
+	int rc = 0;
+
+	mutex_lock(&zcrypt_rng_mutex);
+	if (zcrypt_rng_device_count == 0) {
+		zcrypt_rng_buffer = kzalloc(PAGE_SIZE, GFP_KERNEL);
+		if (!zcrypt_rng_buffer) {
+			rc = -ENOMEM;
+			goto out;
+		}
+		zcrypt_rng_buffer_index = 0;
+		rc = hwrng_register(&zcrypt_rng_dev);
+		if (rc)
+			goto out_free;
+		zcrypt_rng_device_count = 1;
+	} else {
+		zcrypt_rng_device_count++;
+	}
+	mutex_unlock(&zcrypt_rng_mutex);
+	return 0;
+
+out_free:
+	kfree(zcrypt_rng_buffer);
+out:
+	mutex_unlock(&zcrypt_rng_mutex);
+	return rc;
+}
+
+void zcrypt_rng_device_remove(void)
+{
+	mutex_lock(&zcrypt_rng_mutex);
+	zcrypt_rng_device_count--;
+	if (zcrypt_rng_device_count == 0) {
+		hwrng_unregister(&zcrypt_rng_dev);
+		kfree(zcrypt_rng_buffer);
+	}
+	mutex_unlock(&zcrypt_rng_mutex);
+}
+
+/*
+ * Wait until the zcrypt api is operational.
+ * The AP bus scan and the binding of ap devices to device drivers is
+ * an asynchronous job. This function waits until these initial jobs
+ * are done and so the zcrypt api should be ready to serve crypto
+ * requests - if there are resources available. The function uses an
+ * internal timeout of 30s. The very first caller will either wait for
+ * ap bus bindings complete or the timeout happens. This state will be
+ * remembered for further callers which will only be blocked until a
+ * decision is made (timeout or bindings complete).
+ * On timeout -ETIME is returned, on success the return value is 0.
+ */
+int zcrypt_wait_api_operational(void)
+{
+	static DEFINE_MUTEX(zcrypt_wait_api_lock);
+	static int zcrypt_wait_api_state;
+	int rc;
+
+	rc = mutex_lock_interruptible(&zcrypt_wait_api_lock);
+	if (rc)
+		return rc;
+
+	switch (zcrypt_wait_api_state) {
+	case 0:
+		/* initial state, invoke wait for the ap bus complete */
+		rc = ap_wait_apqn_bindings_complete(
+			msecs_to_jiffies(ZCRYPT_WAIT_BINDINGS_COMPLETE_MS));
+		switch (rc) {
+		case 0:
+			/* ap bus bindings are complete */
+			zcrypt_wait_api_state = 1;
+			break;
+		case -EINTR:
+			/* interrupted, go back to caller */
+			break;
+		case -ETIME:
+			/* timeout */
+			ZCRYPT_DBF_WARN("%s ap_wait_init_apqn_bindings_complete()=ETIME\n",
+					__func__);
+			zcrypt_wait_api_state = -ETIME;
+			break;
+		default:
+			/* other failure */
+			pr_debug("ap_wait_init_apqn_bindings_complete()=%d\n", rc);
+			break;
+		}
+		break;
+	case 1:
+		/* a previous caller already found ap bus bindings complete */
+		rc = 0;
+		break;
+	default:
+		/* a previous caller had timeout or other failure */
+		rc = zcrypt_wait_api_state;
+		break;
+	}
+
+	mutex_unlock(&zcrypt_wait_api_lock);
+
+	return rc;
+}
+EXPORT_SYMBOL(zcrypt_wait_api_operational);
+
+int __init zcrypt_debug_init(void)
+{
+	zcrypt_dbf_info = debug_register("zcrypt", 2, 1,
+					 ZCRYPT_DBF_MAX_SPRINTF_ARGS * sizeof(long));
+	debug_register_view(zcrypt_dbf_info, &debug_sprintf_view);
+	debug_set_level(zcrypt_dbf_info, DBF_ERR);
+
+	return 0;
+}
+
+void zcrypt_debug_exit(void)
+{
+	debug_unregister(zcrypt_dbf_info);
+}
+
+static int __init zcdn_init(void)
+{
+	int rc;
+
+	/* create a new class 'zcrypt' */
+	rc = class_register(&zcrypt_class);
+	if (rc)
+		goto out_class_register_failed;
+
+	/* alloc device minor range */
+	rc = alloc_chrdev_region(&zcrypt_devt,
+				 0, ZCRYPT_MAX_MINOR_NODES,
+				 ZCRYPT_NAME);
+	if (rc)
+		goto out_alloc_chrdev_failed;
+
+	cdev_init(&zcrypt_cdev, &zcrypt_fops);
+	zcrypt_cdev.owner = THIS_MODULE;
+	rc = cdev_add(&zcrypt_cdev, zcrypt_devt, ZCRYPT_MAX_MINOR_NODES);
+	if (rc)
+		goto out_cdev_add_failed;
+
+	/* need some class specific sysfs attributes */
+	rc = class_create_file(&zcrypt_class, &class_attr_zcdn_create);
+	if (rc)
+		goto out_class_create_file_1_failed;
+	rc = class_create_file(&zcrypt_class, &class_attr_zcdn_destroy);
+	if (rc)
+		goto out_class_create_file_2_failed;
+
+	return 0;
+
+out_class_create_file_2_failed:
+	class_remove_file(&zcrypt_class, &class_attr_zcdn_create);
+out_class_create_file_1_failed:
+	cdev_del(&zcrypt_cdev);
+out_cdev_add_failed:
+	unregister_chrdev_region(zcrypt_devt, ZCRYPT_MAX_MINOR_NODES);
+out_alloc_chrdev_failed:
+	class_unregister(&zcrypt_class);
+out_class_register_failed:
+	return rc;
+}
+
+static void zcdn_exit(void)
+{
+	class_remove_file(&zcrypt_class, &class_attr_zcdn_create);
+	class_remove_file(&zcrypt_class, &class_attr_zcdn_destroy);
+	zcdn_destroy_all();
+	cdev_del(&zcrypt_cdev);
+	unregister_chrdev_region(zcrypt_devt, ZCRYPT_MAX_MINOR_NODES);
+	class_unregister(&zcrypt_class);
+}
+
+/*
+ * zcrypt_api_init(): Module initialization.
+ *
+ * The module initialization code.
+ */
+int __init zcrypt_api_init(void)
+{
+	int rc;
+
+	/* make sure the mempool threshold is >= 1 */
+	if (zcrypt_mempool_threshold < 1) {
+		rc = -EINVAL;
+		goto out;
+	}
+
+	rc = zcrypt_debug_init();
+	if (rc)
+		goto out;
+
+	rc = zcdn_init();
+	if (rc)
+		goto out_zcdn_init_failed;
+
+	rc = zcrypt_ccamisc_init();
+	if (rc)
+		goto out_ccamisc_init_failed;
+
+	rc = zcrypt_ep11misc_init();
+	if (rc)
+		goto out_ep11misc_init_failed;
+
+	/* Register the request sprayer. */
+	rc = misc_register(&zcrypt_misc_device);
+	if (rc < 0)
+		goto out_misc_register_failed;
+
+	zcrypt_msgtype6_init();
+	zcrypt_msgtype50_init();
+
+	return 0;
+
+out_misc_register_failed:
+	zcrypt_ep11misc_exit();
+out_ep11misc_init_failed:
+	zcrypt_ccamisc_exit();
+out_ccamisc_init_failed:
+	zcdn_exit();
+out_zcdn_init_failed:
+	zcrypt_debug_exit();
+out:
+	return rc;
+}
+
+/*
+ * zcrypt_api_exit(): Module termination.
+ *
+ * The module termination code.
+ */
+void __exit zcrypt_api_exit(void)
+{
+	zcdn_exit();
+	misc_deregister(&zcrypt_misc_device);
+	zcrypt_msgtype6_exit();
+	zcrypt_msgtype50_exit();
+	zcrypt_ccamisc_exit();
+	zcrypt_ep11misc_exit();
+	zcrypt_debug_exit();
+}
+
+module_init(zcrypt_api_init);
+module_exit(zcrypt_api_exit);
diff --git a/drivers/s390/crypto/zcrypt_api.h b/drivers/s390/crypto/zcrypt_api.h
new file mode 100644
index 000000000..9f8df809b
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_api.h
@@ -0,0 +1,199 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ *  Copyright IBM Corp. 2001, 2019
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *	       Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *				  Ralph Wuerthner <rwuerthn@de.ibm.com>
+ *  MSGTYPE restruct:		  Holger Dengler <hd@linux.vnet.ibm.com>
+ */
+
+#ifndef _ZCRYPT_API_H_
+#define _ZCRYPT_API_H_
+
+#include <linux/atomic.h>
+#include <asm/debug.h>
+#include <asm/zcrypt.h>
+#include "ap_bus.h"
+
+/**
+ * Supported device types
+ */
+#define ZCRYPT_CEX2C		5
+#define ZCRYPT_CEX2A		6
+#define ZCRYPT_CEX3C		7
+#define ZCRYPT_CEX3A		8
+#define ZCRYPT_CEX4	       10
+#define ZCRYPT_CEX5	       11
+#define ZCRYPT_CEX6	       12
+#define ZCRYPT_CEX7	       13
+
+/**
+ * Large random numbers are pulled in 4096 byte chunks from the crypto cards
+ * and stored in a page. Be careful when increasing this buffer due to size
+ * limitations for AP requests.
+ */
+#define ZCRYPT_RNG_BUFFER_SIZE	4096
+
+/**
+ * The zcrypt_wait_api_operational() function waits this
+ * amount in milliseconds for ap_wait_aqpn_bindings_complete().
+ * Also on a cprb send failure with ENODEV the send functions
+ * trigger an ap bus rescan and wait this time in milliseconds
+ * for ap_wait_aqpn_bindings_complete() before resending.
+ */
+#define ZCRYPT_WAIT_BINDINGS_COMPLETE_MS 30000
+
+/*
+ * Identifier for Crypto Request Performance Index
+ */
+enum crypto_ops {
+	MEX_1K,
+	MEX_2K,
+	MEX_4K,
+	CRT_1K,
+	CRT_2K,
+	CRT_4K,
+	HWRNG,
+	SECKEY,
+	NUM_OPS
+};
+
+struct zcrypt_queue;
+
+/* struct to hold tracking information for a userspace request/response */
+struct zcrypt_track {
+	int again_counter;		/* retry attempts counter */
+	int last_qid;			/* last qid used */
+	int last_rc;			/* last return code */
+};
+
+/* defines related to message tracking */
+#define TRACK_AGAIN_MAX 10
+#define TRACK_AGAIN_CARD_WEIGHT_PENALTY  1000
+#define TRACK_AGAIN_QUEUE_WEIGHT_PENALTY 10000
+
+/*
+ * xflags - to be used with zcrypt_send_cprb() and
+ * zcrypt_send_ep11_cprb() for the xflags parameter.
+ */
+#define ZCRYPT_XFLAG_USERSPACE	0x0001	/* data ptrs address userspace */
+#define ZCRYPT_XFLAG_NOMEMALLOC 0x0002	/* do not allocate memory via kmalloc */
+
+struct zcrypt_ops {
+	long (*rsa_modexpo)(struct zcrypt_queue *, struct ica_rsa_modexpo *,
+			    struct ap_message *);
+	long (*rsa_modexpo_crt)(struct zcrypt_queue *,
+				struct ica_rsa_modexpo_crt *,
+				struct ap_message *);
+	long (*send_cprb)(bool userspace, struct zcrypt_queue *, struct ica_xcRB *,
+			  struct ap_message *);
+	long (*send_ep11_cprb)(bool userspace, struct zcrypt_queue *, struct ep11_urb *,
+			       struct ap_message *);
+	long (*rng)(struct zcrypt_queue *, char *, struct ap_message *);
+	struct list_head list;		/* zcrypt ops list. */
+	struct module *owner;
+	int variant;
+	char name[128];
+};
+
+struct zcrypt_card {
+	struct list_head list;		/* Device list. */
+	struct list_head zqueues;	/* List of zcrypt queues */
+	struct kref refcount;		/* device refcounting */
+	int online;			/* User online/offline */
+	struct ap_card *card;		/* The "real" ap card device. */
+	char *type_string;		/* User space device name. */
+	int user_space_type;		/* User space device id. */
+	int min_mod_size;		/* Min number of bits. */
+	int max_mod_size;		/* Max number of bits. */
+	int max_exp_bit_length;
+	const int *speed_rating;	/* Speed idx of crypto ops. */
+	atomic_t load;			/* Utilization of the crypto device */
+	int request_count;		/* # current requests. */
+};
+
+struct zcrypt_queue {
+	struct list_head list;		/* Device list. */
+	struct kref refcount;		/* device refcounting */
+	int online;			/* User online/offline */
+	struct zcrypt_card *zcard;
+	struct zcrypt_ops *ops;		/* Crypto operations. */
+	struct ap_queue *queue;		/* The "real" ap queue device. */
+	struct ap_message reply;	/* Per-device reply structure. */
+	atomic_t load;			/* Utilization of the crypto device */
+	int request_count;		/* # current requests. */
+};
+
+/* transport layer rescanning */
+extern atomic_t zcrypt_rescan_req;
+
+extern spinlock_t zcrypt_list_lock;
+extern struct list_head zcrypt_card_list;
+
+extern unsigned int zcrypt_mempool_threshold;
+
+#define for_each_zcrypt_card(_zc) \
+	list_for_each_entry(_zc, &zcrypt_card_list, list)
+
+#define for_each_zcrypt_queue(_zq, _zc) \
+	list_for_each_entry(_zq, &(_zc)->zqueues, list)
+
+struct zcrypt_card *zcrypt_card_alloc(void);
+void zcrypt_card_free(struct zcrypt_card *);
+void zcrypt_card_get(struct zcrypt_card *);
+int zcrypt_card_put(struct zcrypt_card *);
+int zcrypt_card_register(struct zcrypt_card *);
+void zcrypt_card_unregister(struct zcrypt_card *);
+
+struct zcrypt_queue *zcrypt_queue_alloc(size_t);
+void zcrypt_queue_free(struct zcrypt_queue *);
+void zcrypt_queue_get(struct zcrypt_queue *);
+int zcrypt_queue_put(struct zcrypt_queue *);
+int zcrypt_queue_register(struct zcrypt_queue *);
+void zcrypt_queue_unregister(struct zcrypt_queue *);
+bool zcrypt_queue_force_online(struct zcrypt_queue *zq, int online);
+
+int zcrypt_rng_device_add(void);
+void zcrypt_rng_device_remove(void);
+
+void zcrypt_msgtype_register(struct zcrypt_ops *);
+void zcrypt_msgtype_unregister(struct zcrypt_ops *);
+struct zcrypt_ops *zcrypt_msgtype(unsigned char *, int);
+int zcrypt_api_init(void);
+void zcrypt_api_exit(void);
+long zcrypt_send_cprb(struct ica_xcRB *xcRB, u32 xflags);
+long zcrypt_send_ep11_cprb(struct ep11_urb *urb, u32 xflags);
+void zcrypt_device_status_mask_ext(struct zcrypt_device_status_ext *devstatus,
+				   int maxcard, int maxqueue);
+int zcrypt_device_status_ext(int card, int queue,
+			     struct zcrypt_device_status_ext *devstatus);
+
+int zcrypt_wait_api_operational(void);
+
+static inline unsigned long z_copy_from_user(bool userspace,
+					     void *to,
+					     const void __user *from,
+					     unsigned long n)
+{
+	if (likely(userspace))
+		return copy_from_user(to, from, n);
+	memcpy(to, (void __force *)from, n);
+	return 0;
+}
+
+static inline unsigned long z_copy_to_user(bool userspace,
+					   void __user *to,
+					   const void *from,
+					   unsigned long n)
+{
+	if (likely(userspace))
+		return copy_to_user(to, from, n);
+	memcpy((void __force *)to, from, n);
+	return 0;
+}
+
+#endif /* _ZCRYPT_API_H_ */
diff --git a/drivers/s390/crypto/zcrypt_card.c b/drivers/s390/crypto/zcrypt_card.c
new file mode 100644
index 000000000..16f04e650
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_card.c
@@ -0,0 +1,225 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ *  Copyright IBM Corp. 2001, 2012
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *	       Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *				  Ralph Wuerthner <rwuerthn@de.ibm.com>
+ *  MSGTYPE restruct:		  Holger Dengler <hd@linux.vnet.ibm.com>
+ */
+
+#include <linux/export.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/miscdevice.h>
+#include <linux/fs.h>
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#include <linux/slab.h>
+#include <linux/atomic.h>
+#include <linux/uaccess.h>
+#include <linux/hw_random.h>
+#include <linux/debugfs.h>
+#include <asm/debug.h>
+
+#include "zcrypt_debug.h"
+#include "zcrypt_api.h"
+
+#include "zcrypt_msgtype6.h"
+#include "zcrypt_msgtype50.h"
+
+/*
+ * Device attributes common for all crypto card devices.
+ */
+
+static ssize_t type_show(struct device *dev,
+			 struct device_attribute *attr, char *buf)
+{
+	struct zcrypt_card *zc = dev_get_drvdata(dev);
+
+	return sysfs_emit(buf, "%s\n", zc->type_string);
+}
+
+static DEVICE_ATTR_RO(type);
+
+static ssize_t online_show(struct device *dev,
+			   struct device_attribute *attr,
+			   char *buf)
+{
+	struct zcrypt_card *zc = dev_get_drvdata(dev);
+	struct ap_card *ac = to_ap_card(dev);
+	int online = ac->config && !ac->chkstop && zc->online ? 1 : 0;
+
+	return sysfs_emit(buf, "%d\n", online);
+}
+
+static ssize_t online_store(struct device *dev,
+			    struct device_attribute *attr,
+			    const char *buf, size_t count)
+{
+	struct zcrypt_card *zc = dev_get_drvdata(dev);
+	struct ap_card *ac = to_ap_card(dev);
+	struct zcrypt_queue *zq;
+	int online, id, i = 0, maxzqs = 0;
+	struct zcrypt_queue **zq_uelist = NULL;
+
+	if (sscanf(buf, "%d\n", &online) != 1 || online < 0 || online > 1)
+		return -EINVAL;
+
+	if (online && (!ac->config || ac->chkstop))
+		return -ENODEV;
+
+	zc->online = online;
+	id = zc->card->id;
+
+	ZCRYPT_DBF_INFO("%s card=%02x online=%d\n", __func__, id, online);
+
+	ap_send_online_uevent(&ac->ap_dev, online);
+
+	spin_lock(&zcrypt_list_lock);
+	/*
+	 * As we are in atomic context here, directly sending uevents
+	 * does not work. So collect the zqueues in a dynamic array
+	 * and process them after zcrypt_list_lock release. As we get/put
+	 * the zqueue objects, we make sure they exist after lock release.
+	 */
+	list_for_each_entry(zq, &zc->zqueues, list)
+		maxzqs++;
+	if (maxzqs > 0)
+		zq_uelist = kzalloc_objs(*zq_uelist, maxzqs + 1, GFP_ATOMIC);
+	list_for_each_entry(zq, &zc->zqueues, list)
+		if (zcrypt_queue_force_online(zq, online))
+			if (zq_uelist) {
+				zcrypt_queue_get(zq);
+				zq_uelist[i++] = zq;
+			}
+	spin_unlock(&zcrypt_list_lock);
+	if (zq_uelist) {
+		for (i = 0; zq_uelist[i]; i++) {
+			zq = zq_uelist[i];
+			ap_send_online_uevent(&zq->queue->ap_dev, online);
+			zcrypt_queue_put(zq);
+		}
+		kfree(zq_uelist);
+	}
+
+	return count;
+}
+
+static DEVICE_ATTR_RW(online);
+
+static ssize_t load_show(struct device *dev,
+			 struct device_attribute *attr,
+			 char *buf)
+{
+	struct zcrypt_card *zc = dev_get_drvdata(dev);
+
+	return sysfs_emit(buf, "%d\n", atomic_read(&zc->load));
+}
+
+static DEVICE_ATTR_RO(load);
+
+static struct attribute *zcrypt_card_attrs[] = {
+	&dev_attr_type.attr,
+	&dev_attr_online.attr,
+	&dev_attr_load.attr,
+	NULL,
+};
+
+static const struct attribute_group zcrypt_card_attr_group = {
+	.attrs = zcrypt_card_attrs,
+};
+
+struct zcrypt_card *zcrypt_card_alloc(void)
+{
+	struct zcrypt_card *zc;
+
+	zc = kzalloc_obj(*zc);
+	if (!zc)
+		return NULL;
+	INIT_LIST_HEAD(&zc->list);
+	INIT_LIST_HEAD(&zc->zqueues);
+	kref_init(&zc->refcount);
+	return zc;
+}
+EXPORT_SYMBOL(zcrypt_card_alloc);
+
+void zcrypt_card_free(struct zcrypt_card *zc)
+{
+	kfree(zc);
+}
+EXPORT_SYMBOL(zcrypt_card_free);
+
+static void zcrypt_card_release(struct kref *kref)
+{
+	struct zcrypt_card *zdev =
+		container_of(kref, struct zcrypt_card, refcount);
+	zcrypt_card_free(zdev);
+}
+
+void zcrypt_card_get(struct zcrypt_card *zc)
+{
+	kref_get(&zc->refcount);
+}
+EXPORT_SYMBOL(zcrypt_card_get);
+
+int zcrypt_card_put(struct zcrypt_card *zc)
+{
+	return kref_put(&zc->refcount, zcrypt_card_release);
+}
+EXPORT_SYMBOL(zcrypt_card_put);
+
+/**
+ * zcrypt_card_register() - Register a crypto card device.
+ * @zc: Pointer to a crypto card device
+ *
+ * Register a crypto card device. Returns 0 if successful.
+ */
+int zcrypt_card_register(struct zcrypt_card *zc)
+{
+	int rc;
+
+	spin_lock(&zcrypt_list_lock);
+	list_add_tail(&zc->list, &zcrypt_card_list);
+	spin_unlock(&zcrypt_list_lock);
+
+	zc->online = 1;
+
+	ZCRYPT_DBF_INFO("%s card=%02x register online=1\n",
+			__func__, zc->card->id);
+
+	rc = sysfs_create_group(&zc->card->ap_dev.device.kobj,
+				&zcrypt_card_attr_group);
+	if (rc) {
+		spin_lock(&zcrypt_list_lock);
+		list_del_init(&zc->list);
+		spin_unlock(&zcrypt_list_lock);
+	}
+
+	return rc;
+}
+EXPORT_SYMBOL(zcrypt_card_register);
+
+/**
+ * zcrypt_card_unregister(): Unregister a crypto card device.
+ * @zc: Pointer to crypto card device
+ *
+ * Unregister a crypto card device.
+ */
+void zcrypt_card_unregister(struct zcrypt_card *zc)
+{
+	ZCRYPT_DBF_INFO("%s card=%02x unregister\n",
+			__func__, zc->card->id);
+
+	spin_lock(&zcrypt_list_lock);
+	list_del_init(&zc->list);
+	spin_unlock(&zcrypt_list_lock);
+	sysfs_remove_group(&zc->card->ap_dev.device.kobj,
+			   &zcrypt_card_attr_group);
+	zcrypt_card_put(zc);
+}
+EXPORT_SYMBOL(zcrypt_card_unregister);
diff --git a/drivers/s390/crypto/zcrypt_cca_key.h b/drivers/s390/crypto/zcrypt_cca_key.h
new file mode 100644
index 000000000..8a69eed75
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_cca_key.h
@@ -0,0 +1,238 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ *  Copyright IBM Corp. 2001, 2006
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ */
+
+#ifndef _ZCRYPT_CCA_KEY_H_
+#define _ZCRYPT_CCA_KEY_H_
+
+struct t6_keyblock_hdr {
+	unsigned short blen;
+	unsigned short ulen;
+	unsigned short flags;
+};
+
+/**
+ * mapping for the cca private ME key token.
+ * Three parts of interest here: the header, the private section and
+ * the public section.
+ *
+ * mapping for the cca key token header
+ */
+struct cca_token_hdr {
+	unsigned char  token_identifier;
+	unsigned char  version;
+	unsigned short token_length;
+	unsigned char  reserved[4];
+} __packed;
+
+#define CCA_TKN_HDR_ID_EXT 0x1E
+
+#define CCA_PVT_USAGE_ALL 0x80
+
+/**
+ * mapping for the cca public section
+ * In a private key, the modulus doesn't appear in the public
+ * section. So, an arbitrary public exponent of 0x010001 will be
+ * used, for a section length of 0x0F always.
+ */
+struct cca_public_sec {
+	unsigned char  section_identifier;
+	unsigned char  version;
+	unsigned short section_length;
+	unsigned char  reserved[2];
+	unsigned short exponent_len;
+	unsigned short modulus_bit_len;
+	unsigned short modulus_byte_len;    /* In a private key, this is 0 */
+} __packed;
+
+/**
+ * mapping for the cca private CRT key 'token'
+ * The first three parts (the only parts considered in this release)
+ * are: the header, the private section and the public section.
+ * The header and public section are the same as for the
+ * struct cca_private_ext_ME
+ *
+ * Following the structure are the quantities p, q, dp, dq, u, pad,
+ * and modulus, in that order, where pad_len is the modulo 8
+ * complement of the residue modulo 8 of the sum of
+ * (p_len + q_len + dp_len + dq_len + u_len).
+ */
+struct cca_pvt_ext_crt_sec {
+	unsigned char  section_identifier;
+	unsigned char  version;
+	unsigned short section_length;
+	unsigned char  private_key_hash[20];
+	unsigned char  reserved1[4];
+	unsigned char  key_format;
+	unsigned char  reserved2;
+	unsigned char  key_name_hash[20];
+	unsigned char  key_use_flags[4];
+	unsigned short p_len;
+	unsigned short q_len;
+	unsigned short dp_len;
+	unsigned short dq_len;
+	unsigned short u_len;
+	unsigned short mod_len;
+	unsigned char  reserved3[4];
+	unsigned short pad_len;
+	unsigned char  reserved4[52];
+	unsigned char  confounder[8];
+} __packed;
+
+#define CCA_PVT_EXT_CRT_SEC_ID_PVT 0x08
+#define CCA_PVT_EXT_CRT_SEC_FMT_CL 0x40
+
+/**
+ * Set up private key fields of a type6 MEX message.
+ *
+ * @mex: pointer to user input data
+ * @p: pointer to memory area for the key
+ *
+ * Returns the size of the key area or negative errno value.
+ */
+static inline int zcrypt_type6_mex_key_en(struct ica_rsa_modexpo *mex, void *p)
+{
+	static struct cca_token_hdr static_pub_hdr = {
+		.token_identifier	=  0x1E,
+	};
+	static struct cca_public_sec static_pub_sec = {
+		.section_identifier	=  0x04,
+	};
+	struct {
+		struct t6_keyblock_hdr t6_hdr;
+		struct cca_token_hdr pubhdr;
+		struct cca_public_sec pubsec;
+		char exponent[];
+	} __packed *key = p;
+	unsigned char *ptr;
+
+	/*
+	 * The inputdatalength was a selection criteria in the dispatching
+	 * function zcrypt_rsa_modexpo(). However, do a plausibility check
+	 * here to make sure the following copy_from_user() can't be utilized
+	 * to compromise the system.
+	 */
+	if (WARN_ON_ONCE(mex->inputdatalength > 512))
+		return -EINVAL;
+
+	memset(key, 0, sizeof(*key));
+
+	key->pubhdr = static_pub_hdr;
+	key->pubsec = static_pub_sec;
+
+	/* key parameter block */
+	ptr = key->exponent;
+	if (copy_from_user(ptr, mex->b_key, mex->inputdatalength))
+		return -EFAULT;
+	ptr += mex->inputdatalength;
+	/* modulus */
+	if (copy_from_user(ptr, mex->n_modulus, mex->inputdatalength))
+		return -EFAULT;
+
+	key->pubsec.modulus_bit_len = 8 * mex->inputdatalength;
+	key->pubsec.modulus_byte_len = mex->inputdatalength;
+	key->pubsec.exponent_len = mex->inputdatalength;
+	key->pubsec.section_length = sizeof(key->pubsec) +
+					2 * mex->inputdatalength;
+	key->pubhdr.token_length =
+		key->pubsec.section_length + sizeof(key->pubhdr);
+	key->t6_hdr.ulen = key->pubhdr.token_length + 4;
+	key->t6_hdr.blen = key->pubhdr.token_length + 6;
+
+	return sizeof(*key) + 2 * mex->inputdatalength;
+}
+
+/**
+ * Set up private key fields of a type6 CRT message.
+ *
+ * @mex: pointer to user input data
+ * @p: pointer to memory area for the key
+ *
+ * Returns the size of the key area or -EFAULT
+ */
+static inline int zcrypt_type6_crt_key(struct ica_rsa_modexpo_crt *crt, void *p)
+{
+	static struct cca_public_sec static_cca_pub_sec = {
+		.section_identifier = 4,
+		.section_length = 0x000f,
+		.exponent_len = 0x0003,
+	};
+	static char pk_exponent[3] = { 0x01, 0x00, 0x01 };
+	struct {
+		struct t6_keyblock_hdr t6_hdr;
+		struct cca_token_hdr token;
+		struct cca_pvt_ext_crt_sec pvt;
+		char key_parts[];
+	} __packed *key = p;
+	struct cca_public_sec *pub;
+	int short_len, long_len, pad_len, key_len, size;
+
+	/*
+	 * The inputdatalength was a selection criteria in the dispatching
+	 * function zcrypt_rsa_crt(). However, do a plausibility check
+	 * here to make sure the following copy_from_user() can't be utilized
+	 * to compromise the system.
+	 */
+	if (WARN_ON_ONCE(crt->inputdatalength > 512))
+		return -EINVAL;
+
+	memset(key, 0, sizeof(*key));
+
+	short_len = (crt->inputdatalength + 1) / 2;
+	long_len = short_len + 8;
+	pad_len = -(3 * long_len + 2 * short_len) & 7;
+	key_len = 3 * long_len + 2 * short_len + pad_len + crt->inputdatalength;
+	size = sizeof(*key) + key_len + sizeof(*pub) + 3;
+
+	/* parameter block.key block */
+	key->t6_hdr.blen = size;
+	key->t6_hdr.ulen = size - 2;
+
+	/* key token header */
+	key->token.token_identifier = CCA_TKN_HDR_ID_EXT;
+	key->token.token_length = size - 6;
+
+	/* private section */
+	key->pvt.section_identifier = CCA_PVT_EXT_CRT_SEC_ID_PVT;
+	key->pvt.section_length = sizeof(key->pvt) + key_len;
+	key->pvt.key_format = CCA_PVT_EXT_CRT_SEC_FMT_CL;
+	key->pvt.key_use_flags[0] = CCA_PVT_USAGE_ALL;
+	key->pvt.p_len = key->pvt.dp_len = key->pvt.u_len = long_len;
+	key->pvt.q_len = key->pvt.dq_len = short_len;
+	key->pvt.mod_len = crt->inputdatalength;
+	key->pvt.pad_len = pad_len;
+
+	/* key parts */
+	if (copy_from_user(key->key_parts, crt->np_prime, long_len) ||
+	    copy_from_user(key->key_parts + long_len,
+			   crt->nq_prime, short_len) ||
+	    copy_from_user(key->key_parts + long_len + short_len,
+			   crt->bp_key, long_len) ||
+	    copy_from_user(key->key_parts + 2 * long_len + short_len,
+			   crt->bq_key, short_len) ||
+	    copy_from_user(key->key_parts + 2 * long_len + 2 * short_len,
+			   crt->u_mult_inv, long_len))
+		return -EFAULT;
+	memset(key->key_parts + 3 * long_len + 2 * short_len, 0, pad_len);
+	memset(key->key_parts + 3 * long_len + 2 * short_len + pad_len,
+	       0xff, crt->inputdatalength);
+	pub = (struct cca_public_sec *)(key->key_parts + key_len);
+	*pub = static_cca_pub_sec;
+	pub->modulus_bit_len = 8 * crt->inputdatalength;
+	/*
+	 * In a private key, the modulus doesn't appear in the public
+	 * section. So, an arbitrary public exponent of 0x010001 will be
+	 * used.
+	 */
+	memcpy((char *)(pub + 1), pk_exponent, 3);
+
+	return size;
+}
+
+#endif /* _ZCRYPT_CCA_KEY_H_ */
diff --git a/drivers/s390/crypto/zcrypt_ccamisc.c b/drivers/s390/crypto/zcrypt_ccamisc.c
new file mode 100644
index 000000000..19909bf43
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_ccamisc.c
@@ -0,0 +1,1901 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ *  Copyright IBM Corp. 2019
+ *  Author(s): Harald Freudenberger <freude@linux.ibm.com>
+ *	       Ingo Franzki <ifranzki@linux.ibm.com>
+ *
+ *  Collection of CCA misc functions used by zcrypt and pkey
+ */
+
+#define pr_fmt(fmt) "zcrypt: " fmt
+
+#include <linux/export.h>
+#include <linux/init.h>
+#include <linux/mempool.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/random.h>
+#include <linux/align.h>
+#include <asm/zcrypt.h>
+#include <asm/pkey.h>
+
+#include "ap_bus.h"
+#include "zcrypt_api.h"
+#include "zcrypt_debug.h"
+#include "zcrypt_msgtype6.h"
+#include "zcrypt_ccamisc.h"
+
+/* Size of parameter block used for all cca requests/replies */
+#define PARMBSIZE 512
+
+/* Size of vardata block used for some of the cca requests/replies */
+#define VARDATASIZE 4096
+
+/*
+ * Cprb memory pool held for urgent cases where no memory
+ * can be allocated via kmalloc. This pool is only used
+ * when alloc_and_prep_cprbmem() is called with the xflag
+ * ZCRYPT_XFLAG_NOMEMALLOC. The cprb memory needs to hold
+ * space for request AND reply!
+ */
+#define CPRB_MEMPOOL_ITEM_SIZE (16 * 1024)
+static mempool_t *cprb_mempool;
+
+/*
+ * This is a pre-allocated memory for the device status array
+ * used within the findcard() functions. It is currently
+ * 128 * 128 * 4 bytes = 64 KB big. Usage of this memory is
+ * controlled via dev_status_mem_mutex. Needs adaption if more
+ * than 128 cards or domains to be are supported.
+ */
+#define ZCRYPT_DEV_STATUS_CARD_MAX 128
+#define ZCRYPT_DEV_STATUS_QUEUE_MAX 128
+#define ZCRYPT_DEV_STATUS_ENTRIES (ZCRYPT_DEV_STATUS_CARD_MAX * \
+				   ZCRYPT_DEV_STATUS_QUEUE_MAX)
+#define ZCRYPT_DEV_STATUS_EXT_SIZE (ZCRYPT_DEV_STATUS_ENTRIES * \
+		sizeof(struct zcrypt_device_status_ext))
+static void *dev_status_mem;
+static DEFINE_MUTEX(dev_status_mem_mutex);
+
+/*
+ * Simple check if the token is a valid CCA secure AES data key
+ * token. If keybitsize is given, the bitsize of the key is
+ * also checked. Returns 0 on success or errno value on failure.
+ */
+int cca_check_secaeskeytoken(debug_info_t *dbg, int dbflvl,
+			     const u8 *token, u32 keysize, int keybitsize)
+{
+	struct secaeskeytoken *t = (struct secaeskeytoken *)token;
+
+#define DBF(...) debug_sprintf_event(dbg, dbflvl, ##__VA_ARGS__)
+
+	if (keysize < sizeof(*t)) {
+		if (dbg)
+			DBF("%s keysize %u < min token size %zu\n",
+			    __func__, keysize, sizeof(*t));
+		return -EINVAL;
+	}
+	if (t->type != TOKTYPE_CCA_INTERNAL) {
+		if (dbg)
+			DBF("%s token check failed, type 0x%02x != 0x%02x\n",
+			    __func__, (int)t->type, TOKTYPE_CCA_INTERNAL);
+		return -EINVAL;
+	}
+	if (t->version != TOKVER_CCA_AES) {
+		if (dbg)
+			DBF("%s token check failed, version 0x%02x != 0x%02x\n",
+			    __func__, (int)t->version, TOKVER_CCA_AES);
+		return -EINVAL;
+	}
+	if (keybitsize > 0 && t->bitsize != keybitsize) {
+		if (dbg)
+			DBF("%s token check failed, bitsize %d != %d\n",
+			    __func__, (int)t->bitsize, keybitsize);
+		return -EINVAL;
+	}
+
+#undef DBF
+
+	return 0;
+}
+EXPORT_SYMBOL(cca_check_secaeskeytoken);
+
+/*
+ * Simple check if the token is a valid CCA secure AES cipher key
+ * token. If keybitsize is given, the bitsize of the key is
+ * also checked. If checkcpacfexport is enabled, the key is also
+ * checked for the export flag to allow CPACF export.
+ * Returns 0 on success or errno value on failure.
+ */
+int cca_check_secaescipherkey(debug_info_t *dbg, int dbflvl,
+			      const u8 *token, u32 keysize,
+			      int keybitsize, int checkcpacfexport)
+{
+	struct cipherkeytoken *t = (struct cipherkeytoken *)token;
+	bool keybitsizeok = true;
+
+#define DBF(...) debug_sprintf_event(dbg, dbflvl, ##__VA_ARGS__)
+
+	if (keysize < sizeof(*t)) {
+		if (dbg)
+			DBF("%s keysize %u < min token size %zu\n",
+			    __func__, keysize, sizeof(*t));
+		return -EINVAL;
+	}
+	if (t->type != TOKTYPE_CCA_INTERNAL) {
+		if (dbg)
+			DBF("%s token check failed, type 0x%02x != 0x%02x\n",
+			    __func__, (int)t->type, TOKTYPE_CCA_INTERNAL);
+		return -EINVAL;
+	}
+	if (t->version != TOKVER_CCA_VLSC) {
+		if (dbg)
+			DBF("%s token check failed, version 0x%02x != 0x%02x\n",
+			    __func__, (int)t->version, TOKVER_CCA_VLSC);
+		return -EINVAL;
+	}
+	if (t->len > keysize) {
+		if (dbg)
+			DBF("%s token check failed, len %d > keysize %u\n",
+			    __func__, (int)t->len, keysize);
+		return -EINVAL;
+	}
+	if (t->len < sizeof(*t)) {
+		if (dbg)
+			DBF("%s token check failed, len %d < min token size %zu\n",
+			    __func__, (int)t->len, sizeof(*t));
+		return -EINVAL;
+	}
+	if (t->algtype != 0x02) {
+		if (dbg)
+			DBF("%s token check failed, algtype 0x%02x != 0x02\n",
+			    __func__, (int)t->algtype);
+		return -EINVAL;
+	}
+	if (t->keytype != 0x0001) {
+		if (dbg)
+			DBF("%s token check failed, keytype 0x%04x != 0x0001\n",
+			    __func__, (int)t->keytype);
+		return -EINVAL;
+	}
+	if (t->plfver != 0x00 && t->plfver != 0x01) {
+		if (dbg)
+			DBF("%s token check failed, unknown plfver 0x%02x\n",
+			    __func__, (int)t->plfver);
+		return -EINVAL;
+	}
+	if (t->wpllen != 512 && t->wpllen != 576 && t->wpllen != 640) {
+		if (dbg)
+			DBF("%s token check failed, unknown wpllen %d\n",
+			    __func__, (int)t->wpllen);
+		return -EINVAL;
+	}
+	if (keybitsize > 0) {
+		switch (keybitsize) {
+		case 128:
+			if (t->wpllen != (t->plfver ? 640 : 512))
+				keybitsizeok = false;
+			break;
+		case 192:
+			if (t->wpllen != (t->plfver ? 640 : 576))
+				keybitsizeok = false;
+			break;
+		case 256:
+			if (t->wpllen != 640)
+				keybitsizeok = false;
+			break;
+		default:
+			keybitsizeok = false;
+			break;
+		}
+		if (!keybitsizeok) {
+			if (dbg)
+				DBF("%s token check failed, bitsize %d\n",
+				    __func__, keybitsize);
+			return -EINVAL;
+		}
+	}
+	if (checkcpacfexport && !(t->kmf1 & KMF1_XPRT_CPAC)) {
+		if (dbg)
+			DBF("%s token check failed, XPRT_CPAC bit is 0\n",
+			    __func__);
+		return -EINVAL;
+	}
+
+#undef DBF
+
+	return 0;
+}
+EXPORT_SYMBOL(cca_check_secaescipherkey);
+
+/*
+ * Simple check if the token is a valid CCA secure ECC private
+ * key token. Returns 0 on success or errno value on failure.
+ */
+int cca_check_sececckeytoken(debug_info_t *dbg, int dbflvl,
+			     const u8 *token, u32 keysize,
+			     int checkcpacfexport)
+{
+	struct eccprivkeytoken *t = (struct eccprivkeytoken *)token;
+
+#define DBF(...) debug_sprintf_event(dbg, dbflvl, ##__VA_ARGS__)
+
+	if (keysize < sizeof(*t)) {
+		if (dbg)
+			DBF("%s keysize %u < min token size %zu\n",
+			    __func__, keysize, sizeof(*t));
+		return -EINVAL;
+	}
+	if (t->type != TOKTYPE_CCA_INTERNAL_PKA) {
+		if (dbg)
+			DBF("%s token check failed, type 0x%02x != 0x%02x\n",
+			    __func__, (int)t->type, TOKTYPE_CCA_INTERNAL_PKA);
+		return -EINVAL;
+	}
+	if (t->len > keysize) {
+		if (dbg)
+			DBF("%s token check failed, len %d > keysize %u\n",
+			    __func__, (int)t->len, keysize);
+		return -EINVAL;
+	}
+	if (t->len < sizeof(*t)) {
+		if (dbg)
+			DBF("%s token check failed, len %d < min token size %zu\n",
+			    __func__, (int)t->len, sizeof(*t));
+		return -EINVAL;
+	}
+	if (t->secid != 0x20) {
+		if (dbg)
+			DBF("%s token check failed, secid 0x%02x != 0x20\n",
+			    __func__, (int)t->secid);
+		return -EINVAL;
+	}
+	if (checkcpacfexport && !(t->kutc & 0x01)) {
+		if (dbg)
+			DBF("%s token check failed, XPRTCPAC bit is 0\n",
+			    __func__);
+		return -EINVAL;
+	}
+
+#undef DBF
+
+	return 0;
+}
+EXPORT_SYMBOL(cca_check_sececckeytoken);
+
+/*
+ * Allocate consecutive memory for request CPRB, request param
+ * block, reply CPRB and reply param block and fill in values
+ * for the common fields. Returns 0 on success or errno value
+ * on failure.
+ * It is guaranteed that request and a possible param block
+ * are aligned to a 4 byte boundary. Furthermore if a param
+ * block is used, the memory allocated for this is rounded up to
+ * the next multiple of 4 bytes.
+ */
+static int alloc_and_prep_cprbmem(size_t paramblen,
+				  u8 **p_cprb_mem,
+				  struct CPRBX **p_req_cprb,
+				  struct CPRBX **p_rep_cprb,
+				  u32 xflags)
+{
+	u8 *cprbmem = NULL;
+	size_t cprbplusparamblen =
+		ALIGN(sizeof(struct CPRBX), 4) + ALIGN(paramblen, 4);
+	size_t len = 2 * cprbplusparamblen;
+	struct CPRBX *preqcblk, *prepcblk;
+
+	/*
+	 * allocate consecutive memory for request CPRB, request param
+	 * block, reply CPRB and reply param block
+	 */
+	if (xflags & ZCRYPT_XFLAG_NOMEMALLOC) {
+		if (len <= CPRB_MEMPOOL_ITEM_SIZE)
+			cprbmem = mempool_alloc_preallocated(cprb_mempool);
+	} else {
+		cprbmem = kmalloc(len, GFP_KERNEL);
+	}
+	if (!cprbmem)
+		return -ENOMEM;
+	memset(cprbmem, 0, len);
+
+	preqcblk = (struct CPRBX *)cprbmem;
+	prepcblk = (struct CPRBX *)(cprbmem + cprbplusparamblen);
+
+	/* fill request cprb struct */
+	preqcblk->cprb_len = sizeof(struct CPRBX);
+	preqcblk->cprb_ver_id = 0x02;
+	memcpy(preqcblk->func_id, "T2", 2);
+	preqcblk->rpl_msgbl = cprbplusparamblen;
+	if (paramblen) {
+		preqcblk->req_parmb = ((u8 __user *)preqcblk) +
+			ALIGN(sizeof(struct CPRBX), 4);
+		preqcblk->rpl_parmb = ((u8 __user *)prepcblk) +
+			ALIGN(sizeof(struct CPRBX), 4);
+	}
+
+	*p_cprb_mem = cprbmem;
+	*p_req_cprb = preqcblk;
+	*p_rep_cprb = prepcblk;
+
+	return 0;
+}
+
+/*
+ * Free the cprb memory allocated with the function above.
+ * If the scrub value is not zero, the memory is filled
+ * with zeros before freeing (useful if there was some
+ * clear key material in there).
+ */
+static void free_cprbmem(void *mem, size_t paramblen, bool scrub, u32 xflags)
+{
+	size_t cprblen = ALIGN(sizeof(struct CPRBX), 4) + ALIGN(paramblen, 4);
+
+	if (mem && scrub)
+		memzero_explicit(mem, 2 * cprblen);
+
+	if (xflags & ZCRYPT_XFLAG_NOMEMALLOC)
+		mempool_free(mem, cprb_mempool);
+	else
+		kfree(mem);
+}
+
+/*
+ * Helper function to prepare the xcrb struct
+ */
+static inline void prep_xcrb(struct ica_xcRB *pxcrb,
+			     u16 cardnr,
+			     struct CPRBX *preqcblk,
+			     struct CPRBX *prepcblk)
+{
+	memset(pxcrb, 0, sizeof(*pxcrb));
+	memcpy(&pxcrb->agent_ID, "CA", 2);
+	pxcrb->user_defined = (cardnr == 0xFFFF ? AUTOSELECT : cardnr);
+	pxcrb->request_control_blk_length =
+		preqcblk->cprb_len + preqcblk->req_parml;
+	pxcrb->request_control_blk_addr = (void __user *)preqcblk;
+	pxcrb->reply_control_blk_length = preqcblk->rpl_msgbl;
+	pxcrb->reply_control_blk_addr = (void __user *)prepcblk;
+}
+
+/*
+ * Generate (random) CCA AES DATA secure key.
+ */
+int cca_genseckey(u16 cardnr, u16 domain,
+		  u32 keybitsize, u8 *seckey, u32 xflags)
+{
+	int i, rc, keysize;
+	int seckeysize;
+	u8 *mem, *ptr;
+	struct CPRBX *preqcblk, *prepcblk;
+	struct ica_xcRB xcrb;
+	struct kgreqparm {
+		u8  subfunc_code[2];
+		u16 rule_array_len;
+		struct lv1 {
+			u16 len;
+			char  key_form[8];
+			char  key_length[8];
+			char  key_type1[8];
+			char  key_type2[8];
+		} lv1;
+		struct lv2 {
+			u16 len;
+			struct keyid {
+				u16 len;
+				u16 attr;
+				u8  data[SECKEYBLOBSIZE];
+			} keyid[6];
+		} lv2;
+	} __packed * preqparm;
+	struct kgrepparm {
+		u8  subfunc_code[2];
+		u16 rule_array_len;
+		struct lv3 {
+			u16 len;
+			u16 keyblocklen;
+			struct {
+				u16 toklen;
+				u16 tokattr;
+				u8  tok[];
+				/* ... some more data ... */
+			} keyblock;
+		} lv3;
+	} __packed * prepparm;
+
+	/* get already prepared memory for 2 cprbs with param block each */
+	rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem,
+				    &preqcblk, &prepcblk, xflags);
+	if (rc)
+		return rc;
+
+	/* fill request cprb struct */
+	preqcblk->domain = domain;
+
+	/* fill request cprb param block with KG request */
+	preqparm = (struct kgreqparm __force *)preqcblk->req_parmb;
+	memcpy(preqparm->subfunc_code, "KG", 2);
+	preqparm->rule_array_len = sizeof(preqparm->rule_array_len);
+	preqparm->lv1.len = sizeof(struct lv1);
+	memcpy(preqparm->lv1.key_form,	 "OP      ", 8);
+	switch (keybitsize) {
+	case PKEY_SIZE_AES_128:
+	case PKEY_KEYTYPE_AES_128: /* older ioctls used this */
+		keysize = 16;
+		memcpy(preqparm->lv1.key_length, "KEYLN16 ", 8);
+		break;
+	case PKEY_SIZE_AES_192:
+	case PKEY_KEYTYPE_AES_192: /* older ioctls used this */
+		keysize = 24;
+		memcpy(preqparm->lv1.key_length, "KEYLN24 ", 8);
+		break;
+	case PKEY_SIZE_AES_256:
+	case PKEY_KEYTYPE_AES_256: /* older ioctls used this */
+		keysize = 32;
+		memcpy(preqparm->lv1.key_length, "KEYLN32 ", 8);
+		break;
+	default:
+		ZCRYPT_DBF_ERR("%s unknown/unsupported keybitsize %d\n",
+			       __func__, keybitsize);
+		rc = -EINVAL;
+		goto out;
+	}
+	memcpy(preqparm->lv1.key_type1,  "AESDATA ", 8);
+	preqparm->lv2.len = sizeof(struct lv2);
+	for (i = 0; i < 6; i++) {
+		preqparm->lv2.keyid[i].len = sizeof(struct keyid);
+		preqparm->lv2.keyid[i].attr = (i == 2 ? 0x30 : 0x10);
+	}
+	preqcblk->req_parml = sizeof(struct kgreqparm);
+
+	/* fill xcrb struct */
+	prep_xcrb(&xcrb, cardnr, preqcblk, prepcblk);
+
+	/* forward xcrb with request CPRB and reply CPRB to zcrypt dd */
+	rc = zcrypt_send_cprb(&xcrb, xflags);
+	if (rc) {
+		ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, errno %d\n",
+			       __func__, (int)cardnr, (int)domain, rc);
+		goto out;
+	}
+
+	/* check response returncode and reasoncode */
+	if (prepcblk->ccp_rtcode != 0) {
+		ZCRYPT_DBF_ERR("%s secure key generate failure, card response %d/%d\n",
+			       __func__,
+			  (int)prepcblk->ccp_rtcode,
+			  (int)prepcblk->ccp_rscode);
+		rc = -EIO;
+		goto out;
+	}
+
+	/* process response cprb param block */
+	ptr =  ((u8 *)prepcblk) + sizeof(struct CPRBX);
+	prepcblk->rpl_parmb = (u8 __user *)ptr;
+	prepparm = (struct kgrepparm *)ptr;
+
+	/* check length of the returned secure key token */
+	seckeysize = prepparm->lv3.keyblock.toklen
+		- sizeof(prepparm->lv3.keyblock.toklen)
+		- sizeof(prepparm->lv3.keyblock.tokattr);
+	if (seckeysize != SECKEYBLOBSIZE) {
+		ZCRYPT_DBF_ERR("%s secure token size mismatch %d != %d bytes\n",
+			       __func__, seckeysize, SECKEYBLOBSIZE);
+		rc = -EIO;
+		goto out;
+	}
+
+	/* check secure key token */
+	rc = cca_check_secaeskeytoken(zcrypt_dbf_info, DBF_ERR,
+				      prepparm->lv3.keyblock.tok,
+				      seckeysize, 8 * keysize);
+	if (rc) {
+		rc = -EIO;
+		goto out;
+	}
+
+	/* copy the generated secure key token */
+	memcpy(seckey, prepparm->lv3.keyblock.tok, SECKEYBLOBSIZE);
+
+out:
+	free_cprbmem(mem, PARMBSIZE, false, xflags);
+	return rc;
+}
+EXPORT_SYMBOL(cca_genseckey);
+
+/*
+ * Generate an CCA AES DATA secure key with given key value.
+ */
+int cca_clr2seckey(u16 cardnr, u16 domain, u32 keybitsize,
+		   const u8 *clrkey, u8 *seckey, u32 xflags)
+{
+	int rc, keysize, seckeysize;
+	u8 *mem, *ptr;
+	struct CPRBX *preqcblk, *prepcblk;
+	struct ica_xcRB xcrb;
+	struct cmreqparm {
+		u8  subfunc_code[2];
+		u16 rule_array_len;
+		char  rule_array[8];
+		struct lv1 {
+			u16 len;
+			u8  clrkey[];
+		} lv1;
+		/* followed by struct lv2 */
+	} __packed * preqparm;
+	struct lv2 {
+		u16 len;
+		struct keyid {
+			u16 len;
+			u16 attr;
+			u8  data[SECKEYBLOBSIZE];
+		} keyid;
+	} __packed * plv2;
+	struct cmrepparm {
+		u8  subfunc_code[2];
+		u16 rule_array_len;
+		struct lv3 {
+			u16 len;
+			u16 keyblocklen;
+			struct {
+				u16 toklen;
+				u16 tokattr;
+				u8  tok[];
+				/* ... some more data ... */
+			} keyblock;
+		} lv3;
+	} __packed * prepparm;
+
+	/* get already prepared memory for 2 cprbs with param block each */
+	rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem,
+				    &preqcblk, &prepcblk, xflags);
+	if (rc)
+		return rc;
+
+	/* fill request cprb struct */
+	preqcblk->domain = domain;
+
+	/* fill request cprb param block with CM request */
+	preqparm = (struct cmreqparm __force *)preqcblk->req_parmb;
+	memcpy(preqparm->subfunc_code, "CM", 2);
+	memcpy(preqparm->rule_array, "AES     ", 8);
+	preqparm->rule_array_len =
+		sizeof(preqparm->rule_array_len) + sizeof(preqparm->rule_array);
+	switch (keybitsize) {
+	case PKEY_SIZE_AES_128:
+	case PKEY_KEYTYPE_AES_128: /* older ioctls used this */
+		keysize = 16;
+		break;
+	case PKEY_SIZE_AES_192:
+	case PKEY_KEYTYPE_AES_192: /* older ioctls used this */
+		keysize = 24;
+		break;
+	case PKEY_SIZE_AES_256:
+	case PKEY_KEYTYPE_AES_256: /* older ioctls used this */
+		keysize = 32;
+		break;
+	default:
+		ZCRYPT_DBF_ERR("%s unknown/unsupported keybitsize %d\n",
+			       __func__, keybitsize);
+		rc = -EINVAL;
+		goto out;
+	}
+	preqparm->lv1.len = sizeof(struct lv1) + keysize;
+	memcpy(preqparm->lv1.clrkey, clrkey, keysize);
+	plv2 = (struct lv2 *)(((u8 *)preqparm) + sizeof(*preqparm) + keysize);
+	plv2->len = sizeof(struct lv2);
+	plv2->keyid.len = sizeof(struct keyid);
+	plv2->keyid.attr = 0x30;
+	preqcblk->req_parml = sizeof(*preqparm) + keysize + sizeof(*plv2);
+
+	/* fill xcrb struct */
+	prep_xcrb(&xcrb, cardnr, preqcblk, prepcblk);
+
+	/* forward xcrb with request CPRB and reply CPRB to zcrypt dd */
+	rc = zcrypt_send_cprb(&xcrb, xflags);
+	if (rc) {
+		ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n",
+			       __func__, (int)cardnr, (int)domain, rc);
+		goto out;
+	}
+
+	/* check response returncode and reasoncode */
+	if (prepcblk->ccp_rtcode != 0) {
+		ZCRYPT_DBF_ERR("%s clear key import failure, card response %d/%d\n",
+			       __func__,
+			       (int)prepcblk->ccp_rtcode,
+			       (int)prepcblk->ccp_rscode);
+		rc = -EIO;
+		goto out;
+	}
+
+	/* process response cprb param block */
+	ptr = ((u8 *)prepcblk) + sizeof(struct CPRBX);
+	prepcblk->rpl_parmb = (u8 __user *)ptr;
+	prepparm = (struct cmrepparm *)ptr;
+
+	/* check length of the returned secure key token */
+	seckeysize = prepparm->lv3.keyblock.toklen
+		- sizeof(prepparm->lv3.keyblock.toklen)
+		- sizeof(prepparm->lv3.keyblock.tokattr);
+	if (seckeysize != SECKEYBLOBSIZE) {
+		ZCRYPT_DBF_ERR("%s secure token size mismatch %d != %d bytes\n",
+			       __func__, seckeysize, SECKEYBLOBSIZE);
+		rc = -EIO;
+		goto out;
+	}
+
+	/* check secure key token */
+	rc = cca_check_secaeskeytoken(zcrypt_dbf_info, DBF_ERR,
+				      prepparm->lv3.keyblock.tok,
+				      seckeysize, 8 * keysize);
+	if (rc) {
+		rc = -EIO;
+		goto out;
+	}
+
+	/* copy the generated secure key token */
+	if (seckey)
+		memcpy(seckey, prepparm->lv3.keyblock.tok, SECKEYBLOBSIZE);
+
+out:
+	free_cprbmem(mem, PARMBSIZE, true, xflags);
+	return rc;
+}
+EXPORT_SYMBOL(cca_clr2seckey);
+
+/*
+ * Derive proteced key from an CCA AES DATA secure key.
+ */
+int cca_sec2protkey(u16 cardnr, u16 domain,
+		    const u8 *seckey, u8 *protkey, u32 *protkeylen,
+		    u32 *protkeytype, u32 xflags)
+{
+	int rc;
+	u8 *mem, *ptr;
+	struct CPRBX *preqcblk, *prepcblk;
+	struct ica_xcRB xcrb;
+	struct uskreqparm {
+		u8  subfunc_code[2];
+		u16 rule_array_len;
+		struct lv1 {
+			u16 len;
+			u16 attr_len;
+			u16 attr_flags;
+		} lv1;
+		struct lv2 {
+			u16 len;
+			u16 attr_len;
+			u16 attr_flags;
+			u8  token[];	      /* cca secure key token */
+		} lv2;
+	} __packed * preqparm;
+	struct uskrepparm {
+		u8  subfunc_code[2];
+		u16 rule_array_len;
+		struct lv3 {
+			u16 len;
+			u16 attr_len;
+			u16 attr_flags;
+			struct cpacfkeyblock {
+				u8  version;  /* version of this struct */
+				u8  flags[2];
+				u8  algo;
+				u8  form;
+				u8  pad1[3];
+				u16 len;
+				u8  key[64];  /* the key (len bytes) */
+				u16 keyattrlen;
+				u8  keyattr[32];
+				u8  pad2[1];
+				u8  vptype;
+				u8  vp[32];  /* verification pattern */
+			} ckb;
+		} lv3;
+	} __packed * prepparm;
+
+	/* get already prepared memory for 2 cprbs with param block each */
+	rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem,
+				    &preqcblk, &prepcblk, xflags);
+	if (rc)
+		return rc;
+
+	/* fill request cprb struct */
+	preqcblk->domain = domain;
+
+	/* fill request cprb param block with USK request */
+	preqparm = (struct uskreqparm __force *)preqcblk->req_parmb;
+	memcpy(preqparm->subfunc_code, "US", 2);
+	preqparm->rule_array_len = sizeof(preqparm->rule_array_len);
+	preqparm->lv1.len = sizeof(struct lv1);
+	preqparm->lv1.attr_len = sizeof(struct lv1) - sizeof(preqparm->lv1.len);
+	preqparm->lv1.attr_flags = 0x0001;
+	preqparm->lv2.len = sizeof(struct lv2) + SECKEYBLOBSIZE;
+	preqparm->lv2.attr_len = sizeof(struct lv2)
+		- sizeof(preqparm->lv2.len) + SECKEYBLOBSIZE;
+	preqparm->lv2.attr_flags = 0x0000;
+	memcpy(preqparm->lv2.token, seckey, SECKEYBLOBSIZE);
+	preqcblk->req_parml = sizeof(struct uskreqparm) + SECKEYBLOBSIZE;
+
+	/* fill xcrb struct */
+	prep_xcrb(&xcrb, cardnr, preqcblk, prepcblk);
+
+	/* forward xcrb with request CPRB and reply CPRB to zcrypt dd */
+	rc = zcrypt_send_cprb(&xcrb, xflags);
+	if (rc) {
+		ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n",
+			       __func__, (int)cardnr, (int)domain, rc);
+		goto out;
+	}
+
+	/* check response returncode and reasoncode */
+	if (prepcblk->ccp_rtcode != 0) {
+		ZCRYPT_DBF_ERR("%s unwrap secure key failure, card response %d/%d\n",
+			       __func__,
+			       (int)prepcblk->ccp_rtcode,
+			       (int)prepcblk->ccp_rscode);
+		if (prepcblk->ccp_rtcode == 8 && prepcblk->ccp_rscode == 2290)
+			rc = -EBUSY;
+		else
+			rc = -EIO;
+		goto out;
+	}
+	if (prepcblk->ccp_rscode != 0) {
+		ZCRYPT_DBF_WARN("%s unwrap secure key warning, card response %d/%d\n",
+				__func__,
+				(int)prepcblk->ccp_rtcode,
+				(int)prepcblk->ccp_rscode);
+	}
+
+	/* process response cprb param block */
+	ptr = ((u8 *)prepcblk) + sizeof(struct CPRBX);
+	prepcblk->rpl_parmb = (u8 __user *)ptr;
+	prepparm = (struct uskrepparm *)ptr;
+
+	/* check the returned keyblock */
+	if (prepparm->lv3.ckb.version != 0x01 &&
+	    prepparm->lv3.ckb.version != 0x02) {
+		ZCRYPT_DBF_ERR("%s reply param keyblock version mismatch 0x%02x\n",
+			       __func__, (int)prepparm->lv3.ckb.version);
+		rc = -EIO;
+		goto out;
+	}
+
+	/* copy the translated protected key */
+	switch (prepparm->lv3.ckb.len) {
+	case 16 + 32:
+		/* AES 128 protected key */
+		if (protkeytype)
+			*protkeytype = PKEY_KEYTYPE_AES_128;
+		break;
+	case 24 + 32:
+		/* AES 192 protected key */
+		if (protkeytype)
+			*protkeytype = PKEY_KEYTYPE_AES_192;
+		break;
+	case 32 + 32:
+		/* AES 256 protected key */
+		if (protkeytype)
+			*protkeytype = PKEY_KEYTYPE_AES_256;
+		break;
+	default:
+		ZCRYPT_DBF_ERR("%s unknown/unsupported keylen %d\n",
+			       __func__, prepparm->lv3.ckb.len);
+		rc = -EIO;
+		goto out;
+	}
+	memcpy(protkey, prepparm->lv3.ckb.key, prepparm->lv3.ckb.len);
+	if (protkeylen)
+		*protkeylen = prepparm->lv3.ckb.len;
+
+out:
+	free_cprbmem(mem, PARMBSIZE, true, xflags);
+	return rc;
+}
+EXPORT_SYMBOL(cca_sec2protkey);
+
+/*
+ * AES cipher key skeleton created with CSNBKTB2 with these flags:
+ * INTERNAL, NO-KEY, AES, CIPHER, ANY-MODE, NOEX-SYM, NOEXAASY,
+ * NOEXUASY, XPRTCPAC, NOEX-RAW, NOEX-DES, NOEX-AES, NOEX-RSA
+ * used by cca_gencipherkey() and cca_clr2cipherkey().
+ */
+static const u8 aes_cipher_key_skeleton[] = {
+	0x01, 0x00, 0x00, 0x38, 0x05, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
+	0x00, 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x02, 0x00, 0x01, 0x02, 0xc0, 0x00, 0xff,
+	0x00, 0x03, 0x08, 0xc8, 0x00, 0x00, 0x00, 0x00 };
+#define SIZEOF_SKELETON (sizeof(aes_cipher_key_skeleton))
+
+/*
+ * Generate (random) CCA AES CIPHER secure key.
+ */
+int cca_gencipherkey(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags,
+		     u8 *keybuf, u32 *keybufsize, u32 xflags)
+{
+	int rc;
+	u8 *mem, *ptr;
+	struct CPRBX *preqcblk, *prepcblk;
+	struct ica_xcRB xcrb;
+	struct gkreqparm {
+		u8  subfunc_code[2];
+		u16 rule_array_len;
+		char rule_array[2 * 8];
+		struct {
+			u16 len;
+			u8  key_type_1[8];
+			u8  key_type_2[8];
+			u16 clear_key_bit_len;
+			u16 key_name_1_len;
+			u16 key_name_2_len;
+			u16 user_data_1_len;
+			u16 user_data_2_len;
+			/* u8  key_name_1[]; */
+			/* u8  key_name_2[]; */
+			/* u8  user_data_1[]; */
+			/* u8  user_data_2[]; */
+		} vud;
+		struct {
+			u16 len;
+			struct {
+				u16 len;
+				u16 flag;
+				/* u8  kek_id_1[]; */
+			} tlv1;
+			struct {
+				u16 len;
+				u16 flag;
+				/* u8  kek_id_2[]; */
+			} tlv2;
+			struct {
+				u16 len;
+				u16 flag;
+				u8  gen_key_id_1[SIZEOF_SKELETON];
+			} tlv3;
+			struct {
+				u16 len;
+				u16 flag;
+				/* u8  gen_key_id_1_label[]; */
+			} tlv4;
+			struct {
+				u16 len;
+				u16 flag;
+				/* u8  gen_key_id_2[]; */
+			} tlv5;
+			struct {
+				u16 len;
+				u16 flag;
+				/* u8  gen_key_id_2_label[]; */
+			} tlv6;
+		} kb;
+	} __packed * preqparm;
+	struct gkrepparm {
+		u8  subfunc_code[2];
+		u16 rule_array_len;
+		struct {
+			u16 len;
+		} vud;
+		struct {
+			u16 len;
+			struct {
+				u16 len;
+				u16 flag;
+				u8  gen_key[]; /* 120-136 bytes */
+			} tlv1;
+		} kb;
+	} __packed * prepparm;
+	struct cipherkeytoken *t;
+	u32 keybuflen;
+
+	/* get already prepared memory for 2 cprbs with param block each */
+	rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem,
+				    &preqcblk, &prepcblk, xflags);
+	if (rc)
+		return rc;
+
+	/* fill request cprb struct */
+	preqcblk->domain = domain;
+	preqcblk->req_parml = sizeof(struct gkreqparm);
+
+	/* prepare request param block with GK request */
+	preqparm = (struct gkreqparm __force *)preqcblk->req_parmb;
+	memcpy(preqparm->subfunc_code, "GK", 2);
+	preqparm->rule_array_len =  sizeof(uint16_t) + 2 * 8;
+	memcpy(preqparm->rule_array, "AES     OP      ", 2 * 8);
+
+	/* prepare vud block */
+	preqparm->vud.len = sizeof(preqparm->vud);
+	switch (keybitsize) {
+	case 128:
+	case 192:
+	case 256:
+		break;
+	default:
+		ZCRYPT_DBF_ERR("%s unknown/unsupported keybitsize %d\n",
+			       __func__, keybitsize);
+		rc = -EINVAL;
+		goto out;
+	}
+	preqparm->vud.clear_key_bit_len = keybitsize;
+	memcpy(preqparm->vud.key_type_1, "TOKEN   ", 8);
+	memset(preqparm->vud.key_type_2, ' ', sizeof(preqparm->vud.key_type_2));
+
+	/* prepare kb block */
+	preqparm->kb.len = sizeof(preqparm->kb);
+	preqparm->kb.tlv1.len = sizeof(preqparm->kb.tlv1);
+	preqparm->kb.tlv1.flag = 0x0030;
+	preqparm->kb.tlv2.len = sizeof(preqparm->kb.tlv2);
+	preqparm->kb.tlv2.flag = 0x0030;
+	preqparm->kb.tlv3.len = sizeof(preqparm->kb.tlv3);
+	preqparm->kb.tlv3.flag = 0x0030;
+	memcpy(preqparm->kb.tlv3.gen_key_id_1,
+	       aes_cipher_key_skeleton, SIZEOF_SKELETON);
+	preqparm->kb.tlv4.len = sizeof(preqparm->kb.tlv4);
+	preqparm->kb.tlv4.flag = 0x0030;
+	preqparm->kb.tlv5.len = sizeof(preqparm->kb.tlv5);
+	preqparm->kb.tlv5.flag = 0x0030;
+	preqparm->kb.tlv6.len = sizeof(preqparm->kb.tlv6);
+	preqparm->kb.tlv6.flag = 0x0030;
+
+	/* patch the skeleton key token export flags inside the kb block */
+	if (keygenflags) {
+		t = (struct cipherkeytoken *)preqparm->kb.tlv3.gen_key_id_1;
+		t->kmf1 |= (u16)(keygenflags & 0x0000FF00);
+		t->kmf1 &= (u16)~(keygenflags & 0x000000FF);
+	}
+
+	/* prepare xcrb struct */
+	prep_xcrb(&xcrb, cardnr, preqcblk, prepcblk);
+
+	/* forward xcrb with request CPRB and reply CPRB to zcrypt dd */
+	rc = zcrypt_send_cprb(&xcrb, xflags);
+	if (rc) {
+		ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n",
+			       __func__, (int)cardnr, (int)domain, rc);
+		goto out;
+	}
+
+	/* check response returncode and reasoncode */
+	if (prepcblk->ccp_rtcode != 0) {
+		ZCRYPT_DBF_ERR("%s cipher key generate failure, card response %d/%d\n",
+			       __func__,
+			       (int)prepcblk->ccp_rtcode,
+			       (int)prepcblk->ccp_rscode);
+		rc = -EIO;
+		goto out;
+	}
+
+	/* process response cprb param block */
+	ptr = ((u8 *)prepcblk) + sizeof(struct CPRBX);
+	prepcblk->rpl_parmb = (u8 __user *)ptr;
+	prepparm = (struct gkrepparm *)ptr;
+
+	/* do some plausibility checks on the key block */
+	if (prepparm->kb.len < 120 + 5 * sizeof(uint16_t) ||
+	    prepparm->kb.len > 136 + 5 * sizeof(uint16_t)) {
+		ZCRYPT_DBF_ERR("%s reply with invalid or unknown key block\n",
+			       __func__);
+		rc = -EIO;
+		goto out;
+	}
+
+	/* and some checks on the generated key */
+	t = (struct cipherkeytoken *)prepparm->kb.tlv1.gen_key;
+	if (prepparm->kb.tlv1.len < 2 * sizeof(uint16_t) + sizeof(*t)) {
+		rc = -EIO;
+		goto out;
+	}
+	keybuflen = prepparm->kb.tlv1.len - 2 * sizeof(uint16_t);
+	rc = cca_check_secaescipherkey(zcrypt_dbf_info, DBF_ERR,
+				       prepparm->kb.tlv1.gen_key,
+				       keybuflen, keybitsize, 1);
+	if (rc) {
+		rc = -EIO;
+		goto out;
+	}
+
+	/* copy the generated vlsc key token */
+	if (keybuf) {
+		if (*keybufsize >= keybuflen)
+			memcpy(keybuf, t, keybuflen);
+		else
+			rc = -EINVAL;
+	}
+	*keybufsize = keybuflen;
+
+out:
+	free_cprbmem(mem, PARMBSIZE, false, xflags);
+	return rc;
+}
+EXPORT_SYMBOL(cca_gencipherkey);
+
+/*
+ * Helper function, does a the CSNBKPI2 CPRB.
+ */
+static int _ip_cprb_helper(u16 cardnr, u16 domain,
+			   const char *rule_array_1,
+			   const char *rule_array_2,
+			   const char *rule_array_3,
+			   const u8 *clr_key_value,
+			   int clr_key_bit_size,
+			   u8 *key_token,
+			   int *key_token_size,
+			   u32 xflags,
+			   bool scrub)
+{
+	int rc, n;
+	u8 *mem, *ptr;
+	struct CPRBX *preqcblk, *prepcblk;
+	struct ica_xcRB xcrb;
+	struct rule_array_block {
+		u8  subfunc_code[2];
+		u16 rule_array_len;
+		char rule_array[];
+	} __packed * preq_ra_block;
+	struct vud_block {
+		u16 len;
+		struct {
+			u16 len;
+			u16 flag;	     /* 0x0064 */
+			u16 clr_key_bit_len;
+		} tlv1;
+		struct {
+			u16 len;
+			u16 flag;	/* 0x0063 */
+			u8  clr_key[];	/* clear key value bytes */
+		} tlv2;
+	} __packed * preq_vud_block;
+	struct key_block {
+		u16 len;
+		struct {
+			u16 len;
+			u16 flag;	  /* 0x0030 */
+			u8  key_token[];  /* key skeleton */
+		} tlv1;
+	} __packed * preq_key_block;
+	struct iprepparm {
+		u8  subfunc_code[2];
+		u16 rule_array_len;
+		struct {
+			u16 len;
+		} vud;
+		struct {
+			u16 len;
+			struct {
+				u16 len;
+				u16 flag;	  /* 0x0030 */
+				u8  key_token[];  /* key token */
+			} tlv1;
+		} kb;
+	} __packed * prepparm;
+	struct cipherkeytoken *t;
+	int complete = strncmp(rule_array_2, "COMPLETE", 8) ? 0 : 1;
+
+	/* get already prepared memory for 2 cprbs with param block each */
+	rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem,
+				    &preqcblk, &prepcblk, xflags);
+	if (rc)
+		return rc;
+
+	/* fill request cprb struct */
+	preqcblk->domain = domain;
+	preqcblk->req_parml = 0;
+
+	/* prepare request param block with IP request */
+	preq_ra_block = (struct rule_array_block __force *)preqcblk->req_parmb;
+	memcpy(preq_ra_block->subfunc_code, "IP", 2);
+	preq_ra_block->rule_array_len =  sizeof(uint16_t) + 2 * 8;
+	memcpy(preq_ra_block->rule_array, rule_array_1, 8);
+	memcpy(preq_ra_block->rule_array + 8, rule_array_2, 8);
+	preqcblk->req_parml = sizeof(struct rule_array_block) + 2 * 8;
+	if (rule_array_3) {
+		preq_ra_block->rule_array_len += 8;
+		memcpy(preq_ra_block->rule_array + 16, rule_array_3, 8);
+		preqcblk->req_parml += 8;
+	}
+
+	/* prepare vud block */
+	preq_vud_block = (struct vud_block __force *)
+		(preqcblk->req_parmb + preqcblk->req_parml);
+	n = complete ? 0 : (clr_key_bit_size + 7) / 8;
+	preq_vud_block->len = sizeof(struct vud_block) + n;
+	preq_vud_block->tlv1.len = sizeof(preq_vud_block->tlv1);
+	preq_vud_block->tlv1.flag = 0x0064;
+	preq_vud_block->tlv1.clr_key_bit_len = complete ? 0 : clr_key_bit_size;
+	preq_vud_block->tlv2.len = sizeof(preq_vud_block->tlv2) + n;
+	preq_vud_block->tlv2.flag = 0x0063;
+	if (!complete)
+		memcpy(preq_vud_block->tlv2.clr_key, clr_key_value, n);
+	preqcblk->req_parml += preq_vud_block->len;
+
+	/* prepare key block */
+	preq_key_block = (struct key_block __force *)
+		(preqcblk->req_parmb + preqcblk->req_parml);
+	n = *key_token_size;
+	preq_key_block->len = sizeof(struct key_block) + n;
+	preq_key_block->tlv1.len = sizeof(preq_key_block->tlv1) + n;
+	preq_key_block->tlv1.flag = 0x0030;
+	memcpy(preq_key_block->tlv1.key_token, key_token, *key_token_size);
+	preqcblk->req_parml += preq_key_block->len;
+
+	/* prepare xcrb struct */
+	prep_xcrb(&xcrb, cardnr, preqcblk, prepcblk);
+
+	/* forward xcrb with request CPRB and reply CPRB to zcrypt dd */
+	rc = zcrypt_send_cprb(&xcrb, xflags);
+	if (rc) {
+		ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n",
+			       __func__, (int)cardnr, (int)domain, rc);
+		goto out;
+	}
+
+	/* check response returncode and reasoncode */
+	if (prepcblk->ccp_rtcode != 0) {
+		ZCRYPT_DBF_ERR("%s CSNBKPI2 failure, card response %d/%d\n",
+			       __func__,
+			       (int)prepcblk->ccp_rtcode,
+			       (int)prepcblk->ccp_rscode);
+		rc = -EIO;
+		goto out;
+	}
+
+	/* process response cprb param block */
+	ptr = ((u8 *)prepcblk) + sizeof(struct CPRBX);
+	prepcblk->rpl_parmb = (u8 __user *)ptr;
+	prepparm = (struct iprepparm *)ptr;
+
+	/* do some plausibility checks on the key block */
+	if (prepparm->kb.len < 120 + 3 * sizeof(uint16_t) ||
+	    prepparm->kb.len > 136 + 3 * sizeof(uint16_t)) {
+		ZCRYPT_DBF_ERR("%s reply with invalid or unknown key block\n",
+			       __func__);
+		rc = -EIO;
+		goto out;
+	}
+
+	/* do not check the key here, it may be incomplete */
+
+	/*
+	 * Copy the vlsc key token back.
+	 * The available space in the destination (key_token) and the source
+	 * (t) buffer is always larger as the valid range of prepparm->kb.len.
+	 * Validate t->len by comparing it with the length information in the
+	 * param block of the request (prepparm->kb.len)
+	 * The value range of prepparm->kb.len has been checked above.
+	 */
+	t = (struct cipherkeytoken *)prepparm->kb.tlv1.key_token;
+	if (t->len != prepparm->kb.len - 3 * sizeof(uint16_t)) {
+		ZCRYPT_DBF_ERR("%s reply with invalid key_token length %u\n",
+			       __func__, t->len);
+		rc = -EIO;
+		goto out;
+	}
+	memcpy(key_token, t, t->len);
+	*key_token_size = t->len;
+
+out:
+	free_cprbmem(mem, PARMBSIZE, scrub, xflags);
+	return rc;
+}
+
+/*
+ * Build CCA AES CIPHER secure key with a given clear key value.
+ */
+int cca_clr2cipherkey(u16 card, u16 dom, u32 keybitsize, u32 keygenflags,
+		      const u8 *clrkey, u8 *keybuf, u32 *keybufsize, u32 xflags)
+{
+	int rc;
+	void *mem;
+	int tokensize;
+	u8 *token, exorbuf[32];
+	struct cipherkeytoken *t;
+
+	/* fill exorbuf with random data */
+	get_random_bytes(exorbuf, sizeof(exorbuf));
+
+	/*
+	 * Allocate space for the key token to build.
+	 * Also we only need up to MAXCCAVLSCTOKENSIZE bytes for this
+	 * we use the already existing cprb mempool to solve this
+	 * short term memory requirement.
+	 */
+	mem = (xflags & ZCRYPT_XFLAG_NOMEMALLOC) ?
+		mempool_alloc_preallocated(cprb_mempool) :
+		mempool_alloc(cprb_mempool, GFP_KERNEL);
+	if (!mem)
+		return -ENOMEM;
+
+	/* prepare the token with the key skeleton */
+	token = (u8 *)mem;
+	tokensize = SIZEOF_SKELETON;
+	memcpy(token, aes_cipher_key_skeleton, tokensize);
+
+	/* patch the skeleton key token export flags */
+	if (keygenflags) {
+		t = (struct cipherkeytoken *)token;
+		t->kmf1 |= (u16)(keygenflags & 0x0000FF00);
+		t->kmf1 &= (u16)~(keygenflags & 0x000000FF);
+	}
+
+	/*
+	 * Do the key import with the clear key value in 4 steps:
+	 * 1/4 FIRST import with only random data
+	 * 2/4 EXOR the clear key
+	 * 3/4 EXOR the very same random data again
+	 * 4/4 COMPLETE the secure cipher key import
+	 */
+	rc = _ip_cprb_helper(card, dom, "AES     ", "FIRST   ", "MIN3PART",
+			     exorbuf, keybitsize, token, &tokensize,
+			     xflags, true);
+	if (rc) {
+		ZCRYPT_DBF_ERR("%s clear key import 1/4 with CSNBKPI2 failed, rc=%d\n",
+			       __func__, rc);
+		goto out;
+	}
+	rc = _ip_cprb_helper(card, dom, "AES     ", "ADD-PART", NULL,
+			     clrkey, keybitsize, token, &tokensize,
+			     xflags, true);
+	if (rc) {
+		ZCRYPT_DBF_ERR("%s clear key import 2/4 with CSNBKPI2 failed, rc=%d\n",
+			       __func__, rc);
+		goto out;
+	}
+	rc = _ip_cprb_helper(card, dom, "AES     ", "ADD-PART", NULL,
+			     exorbuf, keybitsize, token, &tokensize,
+			     xflags, true);
+	if (rc) {
+		ZCRYPT_DBF_ERR("%s clear key import 3/4 with CSNBKPI2 failed, rc=%d\n",
+			       __func__, rc);
+		goto out;
+	}
+	rc = _ip_cprb_helper(card, dom, "AES     ", "COMPLETE", NULL,
+			     NULL, keybitsize, token, &tokensize,
+			     xflags, true);
+	if (rc) {
+		ZCRYPT_DBF_ERR("%s clear key import 4/4 with CSNBKPI2 failed, rc=%d\n",
+			       __func__, rc);
+		goto out;
+	}
+
+	/* copy the generated key token */
+	if (keybuf) {
+		if (tokensize > *keybufsize)
+			rc = -EINVAL;
+		else
+			memcpy(keybuf, token, tokensize);
+	}
+	*keybufsize = tokensize;
+
+out:
+	memzero_explicit(exorbuf, sizeof(exorbuf));
+	memzero_explicit(mem, CPRB_MEMPOOL_ITEM_SIZE);
+	mempool_free(mem, cprb_mempool);
+	return rc;
+}
+EXPORT_SYMBOL(cca_clr2cipherkey);
+
+/*
+ * Derive proteced key from CCA AES cipher secure key.
+ */
+int cca_cipher2protkey(u16 cardnr, u16 domain, const u8 *ckey,
+		       u8 *protkey, u32 *protkeylen, u32 *protkeytype,
+		       u32 xflags)
+{
+	int rc;
+	u8 *mem, *ptr;
+	struct CPRBX *preqcblk, *prepcblk;
+	struct ica_xcRB xcrb;
+	struct aureqparm {
+		u8  subfunc_code[2];
+		u16 rule_array_len;
+		u8  rule_array[8];
+		struct {
+			u16 len;
+			u16 tk_blob_len;
+			u16 tk_blob_tag;
+			u8  tk_blob[66];
+		} vud;
+		struct {
+			u16 len;
+			u16 cca_key_token_len;
+			u16 cca_key_token_flags;
+			u8  cca_key_token[]; /* 64 or more */
+		} kb;
+	} __packed * preqparm;
+	struct aurepparm {
+		u8  subfunc_code[2];
+		u16 rule_array_len;
+		struct {
+			u16 len;
+			u16 sublen;
+			u16 tag;
+			struct cpacfkeyblock {
+				u8  version;  /* version of this struct */
+				u8  flags[2];
+				u8  algo;
+				u8  form;
+				u8  pad1[3];
+				u16 keylen;
+				u8  key[64];  /* the key (keylen bytes) */
+				u16 keyattrlen;
+				u8  keyattr[32];
+				u8  pad2[1];
+				u8  vptype;
+				u8  vp[32];  /* verification pattern */
+			} ckb;
+		} vud;
+		struct {
+			u16 len;
+		} kb;
+	} __packed * prepparm;
+	int keytoklen = ((struct cipherkeytoken *)ckey)->len;
+
+	if (keytoklen > PARMBSIZE - sizeof(struct aureqparm))
+		return -EINVAL;
+
+	/* get already prepared memory for 2 cprbs with param block each */
+	rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem,
+				    &preqcblk, &prepcblk, xflags);
+	if (rc)
+		return rc;
+
+	/* fill request cprb struct */
+	preqcblk->domain = domain;
+
+	/* fill request cprb param block with AU request */
+	preqparm = (struct aureqparm __force *)preqcblk->req_parmb;
+	memcpy(preqparm->subfunc_code, "AU", 2);
+	preqparm->rule_array_len =
+		sizeof(preqparm->rule_array_len)
+		+ sizeof(preqparm->rule_array);
+	memcpy(preqparm->rule_array, "EXPT-SK ", 8);
+	/* vud, tk blob */
+	preqparm->vud.len = sizeof(preqparm->vud);
+	preqparm->vud.tk_blob_len = sizeof(preqparm->vud.tk_blob)
+		+ 2 * sizeof(uint16_t);
+	preqparm->vud.tk_blob_tag = 0x00C2;
+	/* kb, cca token */
+	preqparm->kb.len = keytoklen + 3 * sizeof(uint16_t);
+	preqparm->kb.cca_key_token_len = keytoklen + 2 * sizeof(uint16_t);
+	memcpy(preqparm->kb.cca_key_token, ckey, keytoklen);
+	/* now fill length of param block into cprb */
+	preqcblk->req_parml = sizeof(struct aureqparm) + keytoklen;
+
+	/* fill xcrb struct */
+	prep_xcrb(&xcrb, cardnr, preqcblk, prepcblk);
+
+	/* forward xcrb with request CPRB and reply CPRB to zcrypt dd */
+	rc = zcrypt_send_cprb(&xcrb, xflags);
+	if (rc) {
+		ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n",
+			       __func__, (int)cardnr, (int)domain, rc);
+		goto out;
+	}
+
+	/* check response returncode and reasoncode */
+	if (prepcblk->ccp_rtcode != 0) {
+		ZCRYPT_DBF_ERR("%s unwrap secure key failure, card response %d/%d\n",
+			       __func__,
+			       (int)prepcblk->ccp_rtcode,
+			       (int)prepcblk->ccp_rscode);
+		if (prepcblk->ccp_rtcode == 8 && prepcblk->ccp_rscode == 2290)
+			rc = -EBUSY;
+		else
+			rc = -EIO;
+		goto out;
+	}
+	if (prepcblk->ccp_rscode != 0) {
+		ZCRYPT_DBF_WARN("%s unwrap secure key warning, card response %d/%d\n",
+				__func__,
+				(int)prepcblk->ccp_rtcode,
+				(int)prepcblk->ccp_rscode);
+	}
+
+	/* process response cprb param block */
+	ptr = ((u8 *)prepcblk) + sizeof(struct CPRBX);
+	prepcblk->rpl_parmb = (u8 __user *)ptr;
+	prepparm = (struct aurepparm *)ptr;
+
+	/* check the returned keyblock */
+	if (prepparm->vud.ckb.version != 0x01 &&
+	    prepparm->vud.ckb.version != 0x02) {
+		ZCRYPT_DBF_ERR("%s reply param keyblock version mismatch 0x%02x\n",
+			       __func__, (int)prepparm->vud.ckb.version);
+		rc = -EIO;
+		goto out;
+	}
+	if (prepparm->vud.ckb.algo != 0x02) {
+		ZCRYPT_DBF_ERR("%s reply param keyblock algo mismatch 0x%02x != 0x02\n",
+			       __func__, (int)prepparm->vud.ckb.algo);
+		rc = -EIO;
+		goto out;
+	}
+
+	/* copy the translated protected key */
+	switch (prepparm->vud.ckb.keylen) {
+	case 16 + 32:
+		/* AES 128 protected key */
+		if (protkeytype)
+			*protkeytype = PKEY_KEYTYPE_AES_128;
+		break;
+	case 24 + 32:
+		/* AES 192 protected key */
+		if (protkeytype)
+			*protkeytype = PKEY_KEYTYPE_AES_192;
+		break;
+	case 32 + 32:
+		/* AES 256 protected key */
+		if (protkeytype)
+			*protkeytype = PKEY_KEYTYPE_AES_256;
+		break;
+	default:
+		ZCRYPT_DBF_ERR("%s unknown/unsupported keylen %d\n",
+			       __func__, prepparm->vud.ckb.keylen);
+		rc = -EIO;
+		goto out;
+	}
+	memcpy(protkey, prepparm->vud.ckb.key, prepparm->vud.ckb.keylen);
+	if (protkeylen)
+		*protkeylen = prepparm->vud.ckb.keylen;
+
+out:
+	free_cprbmem(mem, PARMBSIZE, true, xflags);
+	return rc;
+}
+EXPORT_SYMBOL(cca_cipher2protkey);
+
+/*
+ * Derive protected key from CCA ECC secure private key.
+ */
+int cca_ecc2protkey(u16 cardnr, u16 domain, const u8 *key,
+		    u8 *protkey, u32 *protkeylen, u32 *protkeytype, u32 xflags)
+{
+	int rc;
+	u8 *mem, *ptr;
+	struct CPRBX *preqcblk, *prepcblk;
+	struct ica_xcRB xcrb;
+	struct aureqparm {
+		u8  subfunc_code[2];
+		u16 rule_array_len;
+		u8  rule_array[8];
+		struct {
+			u16 len;
+			u16 tk_blob_len;
+			u16 tk_blob_tag;
+			u8  tk_blob[66];
+		} vud;
+		struct {
+			u16 len;
+			u16 cca_key_token_len;
+			u16 cca_key_token_flags;
+			u8  cca_key_token[];
+		} kb;
+	} __packed * preqparm;
+	struct aurepparm {
+		u8  subfunc_code[2];
+		u16 rule_array_len;
+		struct {
+			u16 len;
+			u16 sublen;
+			u16 tag;
+			struct cpacfkeyblock {
+				u8  version;  /* version of this struct */
+				u8  flags[2];
+				u8  algo;
+				u8  form;
+				u8  pad1[3];
+				u16 keylen;
+				u8  key[];  /* the key (keylen bytes) */
+				/* u16 keyattrlen; */
+				/* u8  keyattr[32]; */
+				/* u8  pad2[1]; */
+				/* u8  vptype; */
+				/* u8  vp[32];	verification pattern */
+			} ckb;
+		} vud;
+		/* followed by a key block */
+	} __packed * prepparm;
+	int keylen = ((struct eccprivkeytoken *)key)->len;
+
+	if (keylen > PARMBSIZE - sizeof(struct aureqparm))
+		return -EINVAL;
+
+	/* get already prepared memory for 2 cprbs with param block each */
+	rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem,
+				    &preqcblk, &prepcblk, xflags);
+	if (rc)
+		return rc;
+
+	/* fill request cprb struct */
+	preqcblk->domain = domain;
+
+	/* fill request cprb param block with AU request */
+	preqparm = (struct aureqparm __force *)preqcblk->req_parmb;
+	memcpy(preqparm->subfunc_code, "AU", 2);
+	preqparm->rule_array_len =
+		sizeof(preqparm->rule_array_len)
+		+ sizeof(preqparm->rule_array);
+	memcpy(preqparm->rule_array, "EXPT-SK ", 8);
+	/* vud, tk blob */
+	preqparm->vud.len = sizeof(preqparm->vud);
+	preqparm->vud.tk_blob_len = sizeof(preqparm->vud.tk_blob)
+		+ 2 * sizeof(uint16_t);
+	preqparm->vud.tk_blob_tag = 0x00C2;
+	/* kb, cca token */
+	preqparm->kb.len = keylen + 3 * sizeof(uint16_t);
+	preqparm->kb.cca_key_token_len = keylen + 2 * sizeof(uint16_t);
+	memcpy(preqparm->kb.cca_key_token, key, keylen);
+	/* now fill length of param block into cprb */
+	preqcblk->req_parml = sizeof(struct aureqparm) + keylen;
+
+	/* fill xcrb struct */
+	prep_xcrb(&xcrb, cardnr, preqcblk, prepcblk);
+
+	/* forward xcrb with request CPRB and reply CPRB to zcrypt dd */
+	rc = zcrypt_send_cprb(&xcrb, xflags);
+	if (rc) {
+		ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n",
+			       __func__, (int)cardnr, (int)domain, rc);
+		goto out;
+	}
+
+	/* check response returncode and reasoncode */
+	if (prepcblk->ccp_rtcode != 0) {
+		ZCRYPT_DBF_ERR("%s unwrap secure key failure, card response %d/%d\n",
+			       __func__,
+			       (int)prepcblk->ccp_rtcode,
+			       (int)prepcblk->ccp_rscode);
+		if (prepcblk->ccp_rtcode == 8 && prepcblk->ccp_rscode == 2290)
+			rc = -EBUSY;
+		else
+			rc = -EIO;
+		goto out;
+	}
+	if (prepcblk->ccp_rscode != 0) {
+		ZCRYPT_DBF_WARN("%s unwrap secure key warning, card response %d/%d\n",
+				__func__,
+				(int)prepcblk->ccp_rtcode,
+				(int)prepcblk->ccp_rscode);
+	}
+
+	/* process response cprb param block */
+	ptr = ((u8 *)prepcblk) + sizeof(struct CPRBX);
+	prepcblk->rpl_parmb = (u8 __user *)ptr;
+	prepparm = (struct aurepparm *)ptr;
+
+	/* check the returned keyblock */
+	if (prepparm->vud.ckb.version != 0x02) {
+		ZCRYPT_DBF_ERR("%s reply param keyblock version mismatch 0x%02x != 0x02\n",
+			       __func__, (int)prepparm->vud.ckb.version);
+		rc = -EIO;
+		goto out;
+	}
+	if (prepparm->vud.ckb.algo != 0x81) {
+		ZCRYPT_DBF_ERR("%s reply param keyblock algo mismatch 0x%02x != 0x81\n",
+			       __func__, (int)prepparm->vud.ckb.algo);
+		rc = -EIO;
+		goto out;
+	}
+
+	/* copy the translated protected key */
+	if (prepparm->vud.ckb.keylen > *protkeylen) {
+		ZCRYPT_DBF_ERR("%s prot keylen mismatch %d > buffersize %u\n",
+			       __func__, prepparm->vud.ckb.keylen, *protkeylen);
+		rc = -EIO;
+		goto out;
+	}
+	memcpy(protkey, prepparm->vud.ckb.key, prepparm->vud.ckb.keylen);
+	*protkeylen = prepparm->vud.ckb.keylen;
+	if (protkeytype)
+		*protkeytype = PKEY_KEYTYPE_ECC;
+
+out:
+	free_cprbmem(mem, PARMBSIZE, true, xflags);
+	return rc;
+}
+EXPORT_SYMBOL(cca_ecc2protkey);
+
+/*
+ * query cryptographic facility from CCA adapter
+ */
+int cca_query_crypto_facility(u16 cardnr, u16 domain,
+			      const char *keyword,
+			      u8 *rarray, size_t *rarraylen,
+			      u8 *varray, size_t *varraylen,
+			      u32 xflags)
+{
+	int rc;
+	u16 len;
+	u8 *mem, *ptr;
+	struct CPRBX *preqcblk, *prepcblk;
+	struct ica_xcRB xcrb;
+	struct fqreqparm {
+		u8  subfunc_code[2];
+		u16 rule_array_len;
+		char  rule_array[8];
+		struct lv1 {
+			u16 len;
+			u8  data[VARDATASIZE];
+		} lv1;
+		u16 dummylen;
+	} __packed * preqparm;
+	size_t parmbsize = sizeof(struct fqreqparm);
+	struct fqrepparm {
+		u8  subfunc_code[2];
+		u8  lvdata[];
+	} __packed * prepparm;
+
+	/* get already prepared memory for 2 cprbs with param block each */
+	rc = alloc_and_prep_cprbmem(parmbsize, &mem,
+				    &preqcblk, &prepcblk, xflags);
+	if (rc)
+		return rc;
+
+	/* fill request cprb struct */
+	preqcblk->domain = domain;
+
+	/* fill request cprb param block with FQ request */
+	preqparm = (struct fqreqparm __force *)preqcblk->req_parmb;
+	memcpy(preqparm->subfunc_code, "FQ", 2);
+	memcpy(preqparm->rule_array, keyword, sizeof(preqparm->rule_array));
+	preqparm->rule_array_len =
+		sizeof(preqparm->rule_array_len) + sizeof(preqparm->rule_array);
+	preqparm->lv1.len = sizeof(preqparm->lv1);
+	preqparm->dummylen = sizeof(preqparm->dummylen);
+	preqcblk->req_parml = parmbsize;
+
+	/* fill xcrb struct */
+	prep_xcrb(&xcrb, cardnr, preqcblk, prepcblk);
+
+	/* forward xcrb with request CPRB and reply CPRB to zcrypt dd */
+	rc = zcrypt_send_cprb(&xcrb, xflags);
+	if (rc) {
+		ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n",
+			       __func__, (int)cardnr, (int)domain, rc);
+		goto out;
+	}
+
+	/* check response returncode and reasoncode */
+	if (prepcblk->ccp_rtcode != 0) {
+		ZCRYPT_DBF_ERR("%s unwrap secure key failure, card response %d/%d\n",
+			       __func__,
+			       (int)prepcblk->ccp_rtcode,
+			       (int)prepcblk->ccp_rscode);
+		rc = -EIO;
+		goto out;
+	}
+
+	/* process response cprb param block */
+	ptr = ((u8 *)prepcblk) + sizeof(struct CPRBX);
+	prepcblk->rpl_parmb = (u8 __user *)ptr;
+	prepparm = (struct fqrepparm *)ptr;
+	ptr = prepparm->lvdata;
+
+	/* check and possibly copy reply rule array */
+	len = *((u16 *)ptr);
+	if (len > sizeof(u16)) {
+		ptr += sizeof(u16);
+		len -= sizeof(u16);
+		if (rarray && rarraylen && *rarraylen > 0) {
+			*rarraylen = (len > *rarraylen ? *rarraylen : len);
+			memcpy(rarray, ptr, *rarraylen);
+		}
+		ptr += len;
+	}
+	/* check and possible copy reply var array */
+	len = *((u16 *)ptr);
+	if (len > sizeof(u16)) {
+		ptr += sizeof(u16);
+		len -= sizeof(u16);
+		if (varray && varraylen && *varraylen > 0) {
+			*varraylen = (len > *varraylen ? *varraylen : len);
+			memcpy(varray, ptr, *varraylen);
+		}
+		ptr += len;
+	}
+
+out:
+	free_cprbmem(mem, parmbsize, false, xflags);
+	return rc;
+}
+EXPORT_SYMBOL(cca_query_crypto_facility);
+
+/*
+ * Fetch cca_info values about a CCA queue via
+ * query_crypto_facility from adapter.
+ */
+int cca_get_info(u16 cardnr, u16 domain, struct cca_info *ci, u32 xflags)
+{
+	void *mem;
+	int rc, found = 0;
+	size_t rlen, vlen;
+	u8 *rarray, *varray;
+	struct zcrypt_device_status_ext devstat;
+
+	memset(ci, 0, sizeof(*ci));
+
+	/* if specific domain given, fetch status and hw info for this apqn */
+	if (domain != AUTOSEL_DOM) {
+		rc = zcrypt_device_status_ext(cardnr, domain, &devstat);
+		if (rc)
+			return rc;
+		ci->hwtype = devstat.hwtype;
+	}
+
+	/*
+	 * Prep memory for rule array and var array use.
+	 * Use the cprb mempool for this.
+	 */
+	mem = (xflags & ZCRYPT_XFLAG_NOMEMALLOC) ?
+		mempool_alloc_preallocated(cprb_mempool) :
+		mempool_alloc(cprb_mempool, GFP_KERNEL);
+	if (!mem)
+		return -ENOMEM;
+	rarray = (u8 *)mem;
+	varray = (u8 *)mem + PAGE_SIZE / 2;
+	rlen = vlen = PAGE_SIZE / 2;
+
+	/* QF for this card/domain */
+	rc = cca_query_crypto_facility(cardnr, domain, "STATICSA",
+				       rarray, &rlen, varray, &vlen, xflags);
+	if (rc == 0 && rlen >= 10 * 8 && vlen >= 204) {
+		memcpy(ci->serial, rarray, 8);
+		ci->new_asym_mk_state = (char)rarray[4 * 8];
+		ci->cur_asym_mk_state = (char)rarray[5 * 8];
+		ci->old_asym_mk_state = (char)rarray[6 * 8];
+		if (ci->old_asym_mk_state == '2')
+			memcpy(ci->old_asym_mkvp, varray + 64, 16);
+		if (ci->cur_asym_mk_state == '2')
+			memcpy(ci->cur_asym_mkvp, varray + 84, 16);
+		if (ci->new_asym_mk_state == '3')
+			memcpy(ci->new_asym_mkvp, varray + 104, 16);
+		ci->new_aes_mk_state = (char)rarray[7 * 8];
+		ci->cur_aes_mk_state = (char)rarray[8 * 8];
+		ci->old_aes_mk_state = (char)rarray[9 * 8];
+		if (ci->old_aes_mk_state == '2')
+			memcpy(&ci->old_aes_mkvp, varray + 172, 8);
+		if (ci->cur_aes_mk_state == '2')
+			memcpy(&ci->cur_aes_mkvp, varray + 184, 8);
+		if (ci->new_aes_mk_state == '3')
+			memcpy(&ci->new_aes_mkvp, varray + 196, 8);
+		found++;
+	}
+	if (!found)
+		goto out;
+	rlen = vlen = PAGE_SIZE / 2;
+	rc = cca_query_crypto_facility(cardnr, domain, "STATICSB",
+				       rarray, &rlen, varray, &vlen, xflags);
+	if (rc == 0 && rlen >= 13 * 8 && vlen >= 240) {
+		ci->new_apka_mk_state = (char)rarray[10 * 8];
+		ci->cur_apka_mk_state = (char)rarray[11 * 8];
+		ci->old_apka_mk_state = (char)rarray[12 * 8];
+		if (ci->old_apka_mk_state == '2')
+			memcpy(&ci->old_apka_mkvp, varray + 208, 8);
+		if (ci->cur_apka_mk_state == '2')
+			memcpy(&ci->cur_apka_mkvp, varray + 220, 8);
+		if (ci->new_apka_mk_state == '3')
+			memcpy(&ci->new_apka_mkvp, varray + 232, 8);
+		found++;
+	}
+
+out:
+	mempool_free(mem, cprb_mempool);
+	return found == 2 ? 0 : -ENOENT;
+}
+EXPORT_SYMBOL(cca_get_info);
+
+int cca_findcard2(u32 *apqns, u32 *nr_apqns, u16 cardnr, u16 domain,
+		  int minhwtype, int mktype,
+		  const u8 *ptr_cur_mkvp, const u8 *ptr_old_mkvp, u32 xflags)
+{
+	struct zcrypt_device_status_ext *device_status;
+	int i, card, dom, curmatch, oldmatch;
+	struct cca_info ci;
+	u32 _nr_apqns = 0;
+
+	/* occupy the device status memory */
+	mutex_lock(&dev_status_mem_mutex);
+	memset(dev_status_mem, 0, ZCRYPT_DEV_STATUS_EXT_SIZE);
+	device_status = (struct zcrypt_device_status_ext *)dev_status_mem;
+
+	/* fetch crypto device status into this struct */
+	zcrypt_device_status_mask_ext(device_status,
+				      ZCRYPT_DEV_STATUS_CARD_MAX,
+				      ZCRYPT_DEV_STATUS_QUEUE_MAX);
+
+	/* walk through all the crypto apqnss */
+	for (i = 0; i < ZCRYPT_DEV_STATUS_ENTRIES; i++) {
+		card = AP_QID_CARD(device_status[i].qid);
+		dom = AP_QID_QUEUE(device_status[i].qid);
+		/* check online state */
+		if (!device_status[i].online)
+			continue;
+		/* check for cca functions */
+		if (!(device_status[i].functions & 0x04))
+			continue;
+		/* check cardnr */
+		if (cardnr != 0xFFFF && card != cardnr)
+			continue;
+		/* check domain */
+		if (domain != 0xFFFF && dom != domain)
+			continue;
+		/* get cca info on this apqn */
+		if (cca_get_info(card, dom, &ci, xflags))
+			continue;
+		/* current master key needs to be valid */
+		if (mktype == AES_MK_SET && ci.cur_aes_mk_state != '2')
+			continue;
+		if (mktype == APKA_MK_SET && ci.cur_apka_mk_state != '2')
+			continue;
+		/* check min hardware type */
+		if (minhwtype > 0 && minhwtype > ci.hwtype)
+			continue;
+		if (ptr_cur_mkvp || ptr_old_mkvp) {
+			/* check mkvps */
+			curmatch = oldmatch = 0;
+			if (mktype == AES_MK_SET) {
+				if (ptr_cur_mkvp &&
+				    !memcmp(ptr_cur_mkvp, ci.cur_aes_mkvp,
+					    sizeof(ci.cur_aes_mkvp)))
+					curmatch = 1;
+				if (ptr_old_mkvp &&
+				    ci.old_aes_mk_state == '2' &&
+				    !memcmp(ptr_old_mkvp, ci.old_aes_mkvp,
+					    sizeof(ci.old_aes_mkvp)))
+					oldmatch = 1;
+			} else {
+				if (ptr_cur_mkvp &&
+				    !memcmp(ptr_cur_mkvp, ci.cur_apka_mkvp,
+					    sizeof(ci.cur_apka_mkvp)))
+					curmatch = 1;
+				if (ptr_old_mkvp &&
+				    ci.old_apka_mk_state == '2' &&
+				    !memcmp(ptr_old_mkvp, ci.old_apka_mkvp,
+					    sizeof(ci.old_apka_mkvp)))
+					oldmatch = 1;
+			}
+			if (curmatch + oldmatch < 1)
+				continue;
+		}
+		/* apqn passed all filtering criterons, add to the array */
+		if (_nr_apqns < *nr_apqns)
+			apqns[_nr_apqns++] = (((u16)card) << 16) | ((u16)dom);
+	}
+
+	*nr_apqns = _nr_apqns;
+
+	/* release the device status memory */
+	mutex_unlock(&dev_status_mem_mutex);
+
+	return _nr_apqns ? 0 : -ENODEV;
+}
+EXPORT_SYMBOL(cca_findcard2);
+
+int __init zcrypt_ccamisc_init(void)
+{
+	/* Pre-allocate a small memory pool for cca cprbs. */
+	cprb_mempool = mempool_create_kmalloc_pool(zcrypt_mempool_threshold,
+						   CPRB_MEMPOOL_ITEM_SIZE);
+	if (!cprb_mempool)
+		return -ENOMEM;
+
+	/* Pre-allocate one crypto status card struct used in findcard() */
+	dev_status_mem = kvmalloc(ZCRYPT_DEV_STATUS_EXT_SIZE, GFP_KERNEL);
+	if (!dev_status_mem) {
+		mempool_destroy(cprb_mempool);
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+void zcrypt_ccamisc_exit(void)
+{
+	mutex_lock(&dev_status_mem_mutex);
+	kvfree(dev_status_mem);
+	mutex_unlock(&dev_status_mem_mutex);
+	mempool_destroy(cprb_mempool);
+}
diff --git a/drivers/s390/crypto/zcrypt_ccamisc.h b/drivers/s390/crypto/zcrypt_ccamisc.h
new file mode 100644
index 000000000..a4534f8b2
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_ccamisc.h
@@ -0,0 +1,268 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ *  Copyright IBM Corp. 2019
+ *  Author(s): Harald Freudenberger <freude@linux.ibm.com>
+ *	       Ingo Franzki <ifranzki@linux.ibm.com>
+ *
+ *  Collection of CCA misc functions used by zcrypt and pkey
+ */
+
+#ifndef _ZCRYPT_CCAMISC_H_
+#define _ZCRYPT_CCAMISC_H_
+
+#include <asm/zcrypt.h>
+#include <asm/pkey.h>
+#include "zcrypt_api.h"
+
+/* Key token types */
+#define TOKTYPE_NON_CCA		 0x00 /* Non-CCA key token */
+#define TOKTYPE_CCA_INTERNAL	 0x01 /* CCA internal sym key token */
+#define TOKTYPE_CCA_INTERNAL_PKA 0x1f /* CCA internal asym key token */
+
+/* For TOKTYPE_NON_CCA: */
+#define TOKVER_PROTECTED_KEY	0x01 /* Protected key token */
+#define TOKVER_CLEAR_KEY	0x02 /* Clear key token */
+
+/* For TOKTYPE_CCA_INTERNAL: */
+#define TOKVER_CCA_AES		0x04 /* CCA AES key token */
+#define TOKVER_CCA_VLSC		0x05 /* var length sym cipher key token */
+
+/* Max size of a cca variable length cipher key token */
+#define MAXCCAVLSCTOKENSIZE 725
+
+/* header part of a CCA key token */
+struct keytoken_header {
+	u8  type;     /* one of the TOKTYPE values */
+	u8  res0[1];
+	u16 len;      /* vlsc token: total length in bytes */
+	u8  version;  /* one of the TOKVER values */
+	u8  res1[3];
+} __packed;
+
+/* inside view of a CCA secure key token (only type 0x01 version 0x04) */
+struct secaeskeytoken {
+	u8  type;     /* 0x01 for internal key token */
+	u8  res0[3];
+	u8  version;  /* should be 0x04 */
+	u8  res1[1];
+	u8  flag;     /* key flags */
+	u8  res2[1];
+	u8  mkvp[8];  /* master key verification pattern */
+	u8  key[32];  /* key value (encrypted) */
+	u8  cv[8];    /* control vector */
+	u16 bitsize;  /* key bit size */
+	u16 keysize;  /* key byte size */
+	u8  tvv[4];   /* token validation value */
+} __packed;
+
+/* inside view of a variable length symmetric cipher AES key token */
+struct cipherkeytoken {
+	u8  type;     /* 0x01 for internal key token */
+	u8  res0[1];
+	u16 len;      /* total key token length in bytes */
+	u8  version;  /* should be 0x05 */
+	u8  res1[3];
+	u8  kms;      /* key material state, 0x03 means wrapped with MK */
+	u8  kvpt;     /* key verification pattern type, should be 0x01 */
+	u8  mkvp0[8]; /* master key verification pattern, lo part */
+	u8  mkvp1[8]; /* master key verification pattern, hi part (unused) */
+	u8  eskwm;    /* encrypted section key wrapping method */
+	u8  hashalg;  /* hash algorithmus used for wrapping key */
+	u8  plfver;   /* pay load format version */
+	u8  res2[1];
+	u8  adsver;   /* associated data section version */
+	u8  res3[1];
+	u16 adslen;   /* associated data section length */
+	u8  kllen;    /* optional key label length */
+	u8  ieaslen;  /* optional extended associated data length */
+	u8  uadlen;   /* optional user definable associated data length */
+	u8  res4[1];
+	u16 wpllen;   /* wrapped payload length in bits: */
+		      /*   plfver  0x00 0x01		 */
+		      /*   AES-128  512  640		 */
+		      /*   AES-192  576  640		 */
+		      /*   AES-256  640  640		 */
+	u8  res5[1];
+	u8  algtype;  /* 0x02 for AES cipher */
+	u16 keytype;  /* 0x0001 for 'cipher' */
+	u8  kufc;     /* key usage field count */
+	u16 kuf1;     /* key usage field 1 */
+	u16 kuf2;     /* key usage field 2 */
+	u8  kmfc;     /* key management field count */
+	u16 kmf1;     /* key management field 1 */
+	u16 kmf2;     /* key management field 2 */
+	u16 kmf3;     /* key management field 3 */
+	u8  vdata[]; /* variable part data follows */
+} __packed;
+
+/* inside view of an CCA secure ECC private key */
+struct eccprivkeytoken {
+	u8  type;     /* 0x1f for internal asym key token */
+	u8  version;  /* should be 0x00 */
+	u16 len;      /* total key token length in bytes */
+	u8  res1[4];
+	u8  secid;    /* 0x20 for ECC priv key section marker */
+	u8  secver;   /* section version */
+	u16 seclen;   /* section length */
+	u8  wtype;    /* wrapping method, 0x00 clear, 0x01 AES */
+	u8  htype;    /* hash method, 0x02 for SHA-256 */
+	u8  res2[2];
+	u8  kutc;     /* key usage and translation control */
+	u8  ctype;    /* curve type */
+	u8  kfs;      /* key format and security */
+	u8  ksrc;     /* key source */
+	u16 pbitlen;  /* length of prime p in bits */
+	u16 ibmadlen; /* IBM associated data length in bytes */
+	u8  mkvp[8];  /* master key verification pattern */
+	u8  opk[48];  /* encrypted object protection key data */
+	u16 adatalen; /* associated data length in bytes */
+	u16 fseclen;  /* formatted section length in bytes */
+	u8  more_data[]; /* more data follows */
+} __packed;
+
+/* Some defines for the CCA AES cipherkeytoken kmf1 field */
+#define KMF1_XPRT_SYM  0x8000
+#define KMF1_XPRT_UASY 0x4000
+#define KMF1_XPRT_AASY 0x2000
+#define KMF1_XPRT_RAW  0x1000
+#define KMF1_XPRT_CPAC 0x0800
+#define KMF1_XPRT_DES  0x0080
+#define KMF1_XPRT_AES  0x0040
+#define KMF1_XPRT_RSA  0x0008
+
+/*
+ * Simple check if the token is a valid CCA secure AES data key
+ * token. If keybitsize is given, the bitsize of the key is
+ * also checked. Returns 0 on success or errno value on failure.
+ */
+int cca_check_secaeskeytoken(debug_info_t *dbg, int dbflvl,
+			     const u8 *token, u32 keysize, int keybitsize);
+
+/*
+ * Simple check if the token is a valid CCA secure AES cipher key
+ * token. If keybitsize is given, the bitsize of the key is
+ * also checked. If checkcpacfexport is enabled, the key is also
+ * checked for the export flag to allow CPACF export.
+ * Returns 0 on success or errno value on failure.
+ */
+int cca_check_secaescipherkey(debug_info_t *dbg, int dbflvl,
+			      const u8 *token, u32 keysize,
+			      int keybitsize, int checkcpacfexport);
+
+/*
+ * Simple check if the token is a valid CCA secure ECC private
+ * key token. Returns 0 on success or errno value on failure.
+ */
+int cca_check_sececckeytoken(debug_info_t *dbg, int dbflvl,
+			     const u8 *token, u32 keysize,
+			     int checkcpacfexport);
+
+/*
+ * Generate (random) CCA AES DATA secure key.
+ */
+int cca_genseckey(u16 cardnr, u16 domain, u32 keybitsize, u8 *seckey,
+		  u32 xflags);
+
+/*
+ * Generate CCA AES DATA secure key with given clear key value.
+ */
+int cca_clr2seckey(u16 cardnr, u16 domain, u32 keybitsize,
+		   const u8 *clrkey, u8 *seckey, u32 xflags);
+
+/*
+ * Derive proteced key from an CCA AES DATA secure key.
+ */
+int cca_sec2protkey(u16 cardnr, u16 domain,
+		    const u8 *seckey, u8 *protkey, u32 *protkeylen,
+		    u32 *protkeytype, u32 xflags);
+
+/*
+ * Generate (random) CCA AES CIPHER secure key.
+ */
+int cca_gencipherkey(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags,
+		     u8 *keybuf, u32 *keybufsize, u32 xflags);
+
+/*
+ * Derive proteced key from CCA AES cipher secure key.
+ */
+int cca_cipher2protkey(u16 cardnr, u16 domain, const u8 *ckey,
+		       u8 *protkey, u32 *protkeylen, u32 *protkeytype,
+		       u32 xflags);
+
+/*
+ * Build CCA AES CIPHER secure key with a given clear key value.
+ */
+int cca_clr2cipherkey(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags,
+		      const u8 *clrkey, u8 *keybuf, u32 *keybufsize,
+		      u32 xflags);
+
+/*
+ * Derive proteced key from CCA ECC secure private key.
+ */
+int cca_ecc2protkey(u16 cardnr, u16 domain, const u8 *key,
+		    u8 *protkey, u32 *protkeylen, u32 *protkeytype, u32 xflags);
+
+/*
+ * Query cryptographic facility from CCA adapter
+ */
+int cca_query_crypto_facility(u16 cardnr, u16 domain,
+			      const char *keyword,
+			      u8 *rarray, size_t *rarraylen,
+			      u8 *varray, size_t *varraylen,
+			      u32 xflags);
+
+/*
+ * Build a list of cca apqns meeting the following constrains:
+ * - apqn is online and is in fact a CCA apqn
+ * - if cardnr is not FFFF only apqns with this cardnr
+ * - if domain is not FFFF only apqns with this domainnr
+ * - if minhwtype > 0 only apqns with hwtype >= minhwtype
+ * - if cur_mkvp != 0 only apqns where cur_mkvp == mkvp
+ * - if old_mkvp != 0 only apqns where old_mkvp == mkvp
+ * The mktype determines which set of master keys to use:
+ *   0 = AES_MK_SET - AES MK set, 1 = APKA MK_SET - APKA MK set
+ * The caller should set *nr_apqns to the nr of elements available in *apqns.
+ * On return *nr_apqns is then updated with the nr of apqns filled into *apqns.
+ * The return value is either 0 for success or a negative errno value.
+ * If no apqn meeting the criteria is found, -ENODEV is returned.
+ */
+int cca_findcard2(u32 *apqns, u32 *nr_apqns, u16 cardnr, u16 domain,
+		  int minhwtype, int mktype,
+		  const u8 *cur_mkvp, const u8 *old_mkvp, u32 xflags);
+
+#define AES_MK_SET  0
+#define APKA_MK_SET 1
+
+/* struct to hold info for each CCA queue */
+struct cca_info {
+	u8   new_asym_mkvp[16];	/* verify pattern of new asym master key */
+	u8   cur_asym_mkvp[16];	/* verify pattern of current asym master key */
+	u8   old_asym_mkvp[16];	/* verify pattern of old asym master key */
+	u8   new_aes_mkvp[8];	/* truncated sha256 of new aes master key */
+	u8   cur_aes_mkvp[8];	/* truncated sha256 of current aes master key */
+	u8   old_aes_mkvp[8];	/* truncated sha256 of old aes master key */
+	u8   new_apka_mkvp[8];	/* truncated sha256 of new apka master key */
+	u8   cur_apka_mkvp[8];	/* truncated sha256 of current apka mk */
+	u8   old_apka_mkvp[8];	/* truncated sha256 of old apka mk */
+	char serial[9];		/* serial number (8 ascii numbers + 0x00) */
+	char new_aes_mk_state;	/* '1' empty, '2' partially full, '3' full */
+	char cur_aes_mk_state;	/* '1' invalid, '2' valid */
+	char old_aes_mk_state;	/* '1' invalid, '2' valid */
+	char new_apka_mk_state; /* '1' empty, '2' partially full, '3' full */
+	char cur_apka_mk_state; /* '1' invalid, '2' valid */
+	char old_apka_mk_state; /* '1' invalid, '2' valid */
+	char new_asym_mk_state;	/* '1' empty, '2' partially full, '3' full */
+	char cur_asym_mk_state;	/* '1' invalid, '2' valid */
+	char old_asym_mk_state;	/* '1' invalid, '2' valid */
+	int  hwtype;		/* one of the defined AP_DEVICE_TYPE_* */
+};
+
+/*
+ * Fetch cca information about an CCA queue.
+ */
+int cca_get_info(u16 card, u16 dom, struct cca_info *ci, u32 xflags);
+
+int zcrypt_ccamisc_init(void);
+void zcrypt_ccamisc_exit(void);
+
+#endif /* _ZCRYPT_CCAMISC_H_ */
diff --git a/drivers/s390/crypto/zcrypt_cex4.c b/drivers/s390/crypto/zcrypt_cex4.c
new file mode 100644
index 000000000..43871cd89
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_cex4.c
@@ -0,0 +1,779 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  Copyright IBM Corp. 2012, 2022
+ *  Author(s): Holger Dengler <hd@linux.vnet.ibm.com>
+ */
+
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/hex.h>
+#include <linux/init.h>
+#include <linux/err.h>
+#include <linux/atomic.h>
+#include <linux/uaccess.h>
+#include <linux/device-id/ap.h>
+
+#include "ap_bus.h"
+#include "zcrypt_api.h"
+#include "zcrypt_msgtype6.h"
+#include "zcrypt_msgtype50.h"
+#include "zcrypt_error.h"
+#include "zcrypt_cex4.h"
+#include "zcrypt_ccamisc.h"
+#include "zcrypt_ep11misc.h"
+
+#define CEX4A_MIN_MOD_SIZE	  1	/*    8 bits	*/
+#define CEX4A_MAX_MOD_SIZE_2K	256	/* 2048 bits	*/
+#define CEX4A_MAX_MOD_SIZE_4K	512	/* 4096 bits	*/
+
+#define CEX4C_MIN_MOD_SIZE	 16	/*  256 bits	*/
+#define CEX4C_MAX_MOD_SIZE	512	/* 4096 bits	*/
+
+/* Waiting time for requests to be processed.
+ * Currently there are some types of request which are not deterministic.
+ * But the maximum time limit managed by the stomper code is set to 60sec.
+ * Hence we have to wait at least that time period.
+ */
+#define CEX4_CLEANUP_TIME	(900 * HZ)
+
+MODULE_AUTHOR("IBM Corporation");
+MODULE_DESCRIPTION("CEX[45678] Cryptographic Card device driver, " \
+		   "Copyright IBM Corp. 2022");
+MODULE_LICENSE("GPL");
+
+static struct ap_device_id zcrypt_cex4_card_ids[] = {
+	{ .dev_type = AP_DEVICE_TYPE_CEX4,
+	  .match_flags = AP_DEVICE_ID_MATCH_CARD_TYPE },
+	{ .dev_type = AP_DEVICE_TYPE_CEX5,
+	  .match_flags = AP_DEVICE_ID_MATCH_CARD_TYPE },
+	{ .dev_type = AP_DEVICE_TYPE_CEX6,
+	  .match_flags = AP_DEVICE_ID_MATCH_CARD_TYPE },
+	{ .dev_type = AP_DEVICE_TYPE_CEX7,
+	  .match_flags = AP_DEVICE_ID_MATCH_CARD_TYPE },
+	{ .dev_type = AP_DEVICE_TYPE_CEX8,
+	  .match_flags = AP_DEVICE_ID_MATCH_CARD_TYPE },
+	{ /* end of list */ },
+};
+
+MODULE_DEVICE_TABLE(ap, zcrypt_cex4_card_ids);
+
+static struct ap_device_id zcrypt_cex4_queue_ids[] = {
+	{ .dev_type = AP_DEVICE_TYPE_CEX4,
+	  .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
+	{ .dev_type = AP_DEVICE_TYPE_CEX5,
+	  .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
+	{ .dev_type = AP_DEVICE_TYPE_CEX6,
+	  .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
+	{ .dev_type = AP_DEVICE_TYPE_CEX7,
+	  .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
+	{ .dev_type = AP_DEVICE_TYPE_CEX8,
+	  .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
+	{ /* end of list */ },
+};
+
+MODULE_DEVICE_TABLE(ap, zcrypt_cex4_queue_ids);
+
+/*
+ * CCA card additional device attributes
+ */
+static ssize_t cca_serialnr_show(struct device *dev,
+				 struct device_attribute *attr,
+				 char *buf)
+{
+	struct ap_card *ac = to_ap_card(dev);
+	struct cca_info ci;
+
+	memset(&ci, 0, sizeof(ci));
+
+	cca_get_info(ac->id, AUTOSEL_DOM, &ci, 0);
+
+	return sysfs_emit(buf, "%s\n", ci.serial);
+}
+
+static struct device_attribute dev_attr_cca_serialnr =
+	__ATTR(serialnr, 0444, cca_serialnr_show, NULL);
+
+static struct attribute *cca_card_attrs[] = {
+	&dev_attr_cca_serialnr.attr,
+	NULL,
+};
+
+static const struct attribute_group cca_card_attr_grp = {
+	.attrs = cca_card_attrs,
+};
+
+/*
+ * Simple helper macro to format raw mkvp byte array into hex
+ */
+#define MKVP_TO_HEXBUF(mkvp, buf) \
+	do { \
+		BUILD_BUG_ON(sizeof(buf) <= 2 * sizeof(mkvp)); \
+		bin2hex(buf, mkvp, sizeof(mkvp)); \
+		buf[2 * sizeof(mkvp)] = '\0'; \
+	} while (0)
+
+/*
+ * CCA queue additional device attributes
+ */
+static ssize_t cca_mkvps_show(struct device *dev,
+			      struct device_attribute *attr,
+			      char *buf)
+{
+	static const char * const new_state[] = { "empty", "partial", "full" };
+	static const char * const cao_state[] = { "invalid", "valid" };
+	struct zcrypt_queue *zq = dev_get_drvdata(dev);
+	struct cca_info ci;
+	char hexbuf[2 * 16 + 1];
+	int n = 0;
+
+	memset(&ci, 0, sizeof(ci));
+
+	cca_get_info(AP_QID_CARD(zq->queue->qid),
+		     AP_QID_QUEUE(zq->queue->qid),
+		     &ci, 0);
+
+	if (ci.new_aes_mk_state >= '1' && ci.new_aes_mk_state <= '3') {
+		MKVP_TO_HEXBUF(ci.new_aes_mkvp, hexbuf);
+		n += sysfs_emit_at(buf, n, "AES NEW: %s 0x%s\n",
+				   new_state[ci.new_aes_mk_state - '1'],
+				   hexbuf);
+	} else {
+		n += sysfs_emit_at(buf, n, "AES NEW: - -\n");
+	}
+
+	if (ci.cur_aes_mk_state >= '1' && ci.cur_aes_mk_state <= '2') {
+		MKVP_TO_HEXBUF(ci.cur_aes_mkvp, hexbuf);
+		n += sysfs_emit_at(buf, n, "AES CUR: %s 0x%s\n",
+				   cao_state[ci.cur_aes_mk_state - '1'],
+				   hexbuf);
+	} else {
+		n += sysfs_emit_at(buf, n, "AES CUR: - -\n");
+	}
+
+	if (ci.old_aes_mk_state >= '1' && ci.old_aes_mk_state <= '2') {
+		MKVP_TO_HEXBUF(ci.old_aes_mkvp, hexbuf);
+		n += sysfs_emit_at(buf, n, "AES OLD: %s 0x%s\n",
+				   cao_state[ci.old_aes_mk_state - '1'],
+				   hexbuf);
+	} else {
+		n += sysfs_emit_at(buf, n, "AES OLD: - -\n");
+	}
+
+	if (ci.new_apka_mk_state >= '1' && ci.new_apka_mk_state <= '3') {
+		MKVP_TO_HEXBUF(ci.new_apka_mkvp, hexbuf);
+		n += sysfs_emit_at(buf, n, "APKA NEW: %s 0x%s\n",
+				   new_state[ci.new_apka_mk_state - '1'],
+				   hexbuf);
+	} else {
+		n += sysfs_emit_at(buf, n, "APKA NEW: - -\n");
+	}
+
+	if (ci.cur_apka_mk_state >= '1' && ci.cur_apka_mk_state <= '2') {
+		MKVP_TO_HEXBUF(ci.cur_apka_mkvp, hexbuf);
+		n += sysfs_emit_at(buf, n, "APKA CUR: %s 0x%s\n",
+				   cao_state[ci.cur_apka_mk_state - '1'],
+				   hexbuf);
+	} else {
+		n += sysfs_emit_at(buf, n, "APKA CUR: - -\n");
+	}
+
+	if (ci.old_apka_mk_state >= '1' && ci.old_apka_mk_state <= '2') {
+		MKVP_TO_HEXBUF(ci.old_apka_mkvp, hexbuf);
+		n += sysfs_emit_at(buf, n, "APKA OLD: %s 0x%s\n",
+				   cao_state[ci.old_apka_mk_state - '1'],
+				   hexbuf);
+	} else {
+		n += sysfs_emit_at(buf, n, "APKA OLD: - -\n");
+	}
+
+	if (ci.new_asym_mk_state >= '1' && ci.new_asym_mk_state <= '3') {
+		MKVP_TO_HEXBUF(ci.new_asym_mkvp, hexbuf);
+		n += sysfs_emit_at(buf, n, "ASYM NEW: %s 0x%s\n",
+				   new_state[ci.new_asym_mk_state - '1'],
+				   hexbuf);
+	} else {
+		n += sysfs_emit_at(buf, n, "ASYM NEW: - -\n");
+	}
+
+	if (ci.cur_asym_mk_state >= '1' && ci.cur_asym_mk_state <= '2') {
+		MKVP_TO_HEXBUF(ci.cur_asym_mkvp, hexbuf);
+		n += sysfs_emit_at(buf, n, "ASYM CUR: %s 0x%s\n",
+				   cao_state[ci.cur_asym_mk_state - '1'],
+				   hexbuf);
+	} else {
+		n += sysfs_emit_at(buf, n, "ASYM CUR: - -\n");
+	}
+
+	if (ci.old_asym_mk_state >= '1' && ci.old_asym_mk_state <= '2') {
+		MKVP_TO_HEXBUF(ci.old_asym_mkvp, hexbuf);
+		n += sysfs_emit_at(buf, n, "ASYM OLD: %s 0x%s\n",
+				   cao_state[ci.old_asym_mk_state - '1'],
+				   hexbuf);
+	} else {
+		n += sysfs_emit_at(buf, n, "ASYM OLD: - -\n");
+	}
+
+	return n;
+}
+
+static struct device_attribute dev_attr_cca_mkvps =
+	__ATTR(mkvps, 0444, cca_mkvps_show, NULL);
+
+static struct attribute *cca_queue_attrs[] = {
+	&dev_attr_cca_mkvps.attr,
+	NULL,
+};
+
+static const struct attribute_group cca_queue_attr_grp = {
+	.attrs = cca_queue_attrs,
+};
+
+/*
+ * EP11 card additional device attributes
+ */
+static ssize_t ep11_api_ordinalnr_show(struct device *dev,
+				       struct device_attribute *attr,
+				       char *buf)
+{
+	struct ap_card *ac = to_ap_card(dev);
+	struct ep11_card_info ci;
+
+	memset(&ci, 0, sizeof(ci));
+
+	ep11_get_card_info(ac->id, &ci, 0);
+
+	if (ci.API_ord_nr > 0)
+		return sysfs_emit(buf, "%u\n", ci.API_ord_nr);
+	else
+		return sysfs_emit(buf, "\n");
+}
+
+static struct device_attribute dev_attr_ep11_api_ordinalnr =
+	__ATTR(API_ordinalnr, 0444, ep11_api_ordinalnr_show, NULL);
+
+static ssize_t ep11_fw_version_show(struct device *dev,
+				    struct device_attribute *attr,
+				    char *buf)
+{
+	struct ap_card *ac = to_ap_card(dev);
+	struct ep11_card_info ci;
+
+	memset(&ci, 0, sizeof(ci));
+
+	ep11_get_card_info(ac->id, &ci, 0);
+
+	if (ci.FW_version > 0)
+		return sysfs_emit(buf, "%d.%d\n",
+				  (int)(ci.FW_version >> 8),
+				  (int)(ci.FW_version & 0xFF));
+	else
+		return sysfs_emit(buf, "\n");
+}
+
+static struct device_attribute dev_attr_ep11_fw_version =
+	__ATTR(FW_version, 0444, ep11_fw_version_show, NULL);
+
+static ssize_t ep11_serialnr_show(struct device *dev,
+				  struct device_attribute *attr,
+				  char *buf)
+{
+	struct ap_card *ac = to_ap_card(dev);
+	struct ep11_card_info ci;
+
+	memset(&ci, 0, sizeof(ci));
+
+	ep11_get_card_info(ac->id, &ci, 0);
+
+	if (ci.serial[0])
+		return sysfs_emit(buf, "%16.16s\n", ci.serial);
+	else
+		return sysfs_emit(buf, "\n");
+}
+
+static struct device_attribute dev_attr_ep11_serialnr =
+	__ATTR(serialnr, 0444, ep11_serialnr_show, NULL);
+
+static const struct {
+	int	    mode_bit;
+	const char *mode_txt;
+} ep11_op_modes[] = {
+	{ 0, "FIPS2009" },
+	{ 1, "BSI2009" },
+	{ 2, "FIPS2011" },
+	{ 3, "BSI2011" },
+	{ 4, "SIGG-IMPORT" },
+	{ 5, "SIGG" },
+	{ 6, "BSICC2017" },
+	{ 7, "FIPS2021" },
+	{ 8, "FIPS2024" },
+	{ 0, NULL }
+};
+
+static ssize_t ep11_card_op_modes_show(struct device *dev,
+				       struct device_attribute *attr,
+				       char *buf)
+{
+	struct ap_card *ac = to_ap_card(dev);
+	struct ep11_card_info ci;
+	int i, n = 0;
+
+	memset(&ci, 0, sizeof(ci));
+
+	ep11_get_card_info(ac->id, &ci, 0);
+
+	for (i = 0; ep11_op_modes[i].mode_txt; i++) {
+		if (ci.op_mode & (1ULL << ep11_op_modes[i].mode_bit)) {
+			if (n > 0)
+				buf[n++] = ' ';
+			n += sysfs_emit_at(buf, n, "%s",
+					   ep11_op_modes[i].mode_txt);
+		}
+	}
+	n += sysfs_emit_at(buf, n, "\n");
+
+	return n;
+}
+
+static struct device_attribute dev_attr_ep11_card_op_modes =
+	__ATTR(op_modes, 0444, ep11_card_op_modes_show, NULL);
+
+static struct attribute *ep11_card_attrs[] = {
+	&dev_attr_ep11_api_ordinalnr.attr,
+	&dev_attr_ep11_fw_version.attr,
+	&dev_attr_ep11_serialnr.attr,
+	&dev_attr_ep11_card_op_modes.attr,
+	NULL,
+};
+
+static const struct attribute_group ep11_card_attr_grp = {
+	.attrs = ep11_card_attrs,
+};
+
+/*
+ * EP11 queue additional device attributes
+ */
+
+static ssize_t ep11_mkvps_show(struct device *dev,
+			       struct device_attribute *attr,
+			       char *buf)
+{
+	struct zcrypt_queue *zq = dev_get_drvdata(dev);
+	int n = 0;
+	struct ep11_domain_info di;
+	static const char * const cwk_state[] = { "invalid", "valid" };
+	static const char * const nwk_state[] = { "empty", "uncommitted",
+						  "committed" };
+
+	memset(&di, 0, sizeof(di));
+
+	if (zq->online)
+		ep11_get_domain_info(AP_QID_CARD(zq->queue->qid),
+				     AP_QID_QUEUE(zq->queue->qid),
+				     &di, 0);
+
+	if (di.cur_wk_state == '0') {
+		n = sysfs_emit(buf, "WK CUR: %s -\n",
+			       cwk_state[di.cur_wk_state - '0']);
+	} else if (di.cur_wk_state == '1') {
+		n = sysfs_emit(buf, "WK CUR: %s 0x",
+			       cwk_state[di.cur_wk_state - '0']);
+		bin2hex(buf + n, di.cur_wkvp, sizeof(di.cur_wkvp));
+		n += 2 * sizeof(di.cur_wkvp);
+		n += sysfs_emit_at(buf, n, "\n");
+	} else {
+		n = sysfs_emit(buf, "WK CUR: - -\n");
+	}
+
+	if (di.new_wk_state == '0') {
+		n += sysfs_emit_at(buf, n, "WK NEW: %s -\n",
+				   nwk_state[di.new_wk_state - '0']);
+	} else if (di.new_wk_state >= '1' && di.new_wk_state <= '2') {
+		n += sysfs_emit_at(buf, n, "WK NEW: %s 0x",
+				   nwk_state[di.new_wk_state - '0']);
+		bin2hex(buf + n, di.new_wkvp, sizeof(di.new_wkvp));
+		n += 2 * sizeof(di.new_wkvp);
+		n += sysfs_emit_at(buf, n, "\n");
+	} else {
+		n += sysfs_emit_at(buf, n, "WK NEW: - -\n");
+	}
+
+	return n;
+}
+
+static struct device_attribute dev_attr_ep11_mkvps =
+	__ATTR(mkvps, 0444, ep11_mkvps_show, NULL);
+
+static ssize_t ep11_queue_op_modes_show(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
+{
+	struct zcrypt_queue *zq = dev_get_drvdata(dev);
+	int i, n = 0;
+	struct ep11_domain_info di;
+
+	memset(&di, 0, sizeof(di));
+
+	if (zq->online)
+		ep11_get_domain_info(AP_QID_CARD(zq->queue->qid),
+				     AP_QID_QUEUE(zq->queue->qid),
+				     &di, 0);
+
+	for (i = 0; ep11_op_modes[i].mode_txt; i++) {
+		if (di.op_mode & (1ULL << ep11_op_modes[i].mode_bit)) {
+			if (n > 0)
+				buf[n++] = ' ';
+			n += sysfs_emit_at(buf, n, "%s",
+					   ep11_op_modes[i].mode_txt);
+		}
+	}
+	n += sysfs_emit_at(buf, n, "\n");
+
+	return n;
+}
+
+static struct device_attribute dev_attr_ep11_queue_op_modes =
+	__ATTR(op_modes, 0444, ep11_queue_op_modes_show, NULL);
+
+static struct attribute *ep11_queue_attrs[] = {
+	&dev_attr_ep11_mkvps.attr,
+	&dev_attr_ep11_queue_op_modes.attr,
+	NULL,
+};
+
+static const struct attribute_group ep11_queue_attr_grp = {
+	.attrs = ep11_queue_attrs,
+};
+
+/*
+ * Probe function for CEX[45678] card device. It always
+ * accepts the AP device since the bus_match already checked
+ * the hardware type.
+ * @ap_dev: pointer to the AP device.
+ */
+static int zcrypt_cex4_card_probe(struct ap_device *ap_dev)
+{
+	/*
+	 * Normalized speed ratings per crypto adapter
+	 * MEX_1k, MEX_2k, MEX_4k, CRT_1k, CRT_2k, CRT_4k, RNG, SECKEY
+	 */
+	static const int CEX4A_SPEED_IDX[NUM_OPS] = {
+		 14,  19, 249, 42, 228, 1458, 0, 0};
+	static const int CEX5A_SPEED_IDX[NUM_OPS] = {
+		  8,   9,  20, 18,  66,	 458, 0, 0};
+	static const int CEX6A_SPEED_IDX[NUM_OPS] = {
+		  6,   9,  20, 17,  65,	 438, 0, 0};
+	static const int CEX7A_SPEED_IDX[NUM_OPS] = {
+		  6,   8,  17, 15,  54,	 362, 0, 0};
+	static const int CEX8A_SPEED_IDX[NUM_OPS] = {
+		  6,   8,  17, 15,  54,	 362, 0, 0};
+
+	static const int CEX4C_SPEED_IDX[NUM_OPS] = {
+		 59,  69, 308, 83, 278, 2204, 209, 40};
+	static const int CEX5C_SPEED_IDX[] = {
+		 24,  31,  50, 37,  90,	 479,  27, 10};
+	static const int CEX6C_SPEED_IDX[NUM_OPS] = {
+		 16,  20,  32, 27,  77,	 455,  24,  9};
+	static const int CEX7C_SPEED_IDX[NUM_OPS] = {
+		 14,  16,  26, 23,  64,	 376,  23,  8};
+	static const int CEX8C_SPEED_IDX[NUM_OPS] = {
+		 14,  16,  26, 23,  64,	 376,  23,  8};
+
+	static const int CEX4P_SPEED_IDX[NUM_OPS] = {
+		  0,   0,   0,	 0,   0,   0,	0,  50};
+	static const int CEX5P_SPEED_IDX[NUM_OPS] = {
+		  0,   0,   0,	 0,   0,   0,	0,  10};
+	static const int CEX6P_SPEED_IDX[NUM_OPS] = {
+		  0,   0,   0,	 0,   0,   0,	0,   9};
+	static const int CEX7P_SPEED_IDX[NUM_OPS] = {
+		  0,   0,   0,	 0,   0,   0,	0,   8};
+	static const int CEX8P_SPEED_IDX[NUM_OPS] = {
+		  0,   0,   0,	 0,   0,   0,	0,   8};
+
+	struct ap_card *ac = to_ap_card(&ap_dev->device);
+	struct zcrypt_card *zc;
+	int rc = 0;
+
+	zc = zcrypt_card_alloc();
+	if (!zc)
+		return -ENOMEM;
+	zc->card = ac;
+	dev_set_drvdata(&ap_dev->device, zc);
+	if (ac->hwinfo.accel) {
+		if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX4) {
+			zc->type_string = "CEX4A";
+			zc->user_space_type = ZCRYPT_CEX4;
+			zc->speed_rating = CEX4A_SPEED_IDX;
+		} else if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX5) {
+			zc->type_string = "CEX5A";
+			zc->user_space_type = ZCRYPT_CEX5;
+			zc->speed_rating = CEX5A_SPEED_IDX;
+		} else if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX6) {
+			zc->type_string = "CEX6A";
+			zc->user_space_type = ZCRYPT_CEX6;
+			zc->speed_rating = CEX6A_SPEED_IDX;
+		} else if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX7) {
+			zc->type_string = "CEX7A";
+			zc->speed_rating = CEX7A_SPEED_IDX;
+			/* wrong user space type, just for compatibility
+			 * with the ZCRYPT_STATUS_MASK ioctl.
+			 */
+			zc->user_space_type = ZCRYPT_CEX6;
+		} else {
+			zc->type_string = "CEX8A";
+			zc->speed_rating = CEX8A_SPEED_IDX;
+			/* wrong user space type, just for compatibility
+			 * with the ZCRYPT_STATUS_MASK ioctl.
+			 */
+			zc->user_space_type = ZCRYPT_CEX6;
+		}
+		zc->min_mod_size = CEX4A_MIN_MOD_SIZE;
+		if (ac->hwinfo.mex4k && ac->hwinfo.crt4k) {
+			zc->max_mod_size = CEX4A_MAX_MOD_SIZE_4K;
+			zc->max_exp_bit_length =
+				CEX4A_MAX_MOD_SIZE_4K;
+		} else {
+			zc->max_mod_size = CEX4A_MAX_MOD_SIZE_2K;
+			zc->max_exp_bit_length =
+				CEX4A_MAX_MOD_SIZE_2K;
+		}
+	} else if (ac->hwinfo.cca) {
+		if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX4) {
+			zc->type_string = "CEX4C";
+			zc->speed_rating = CEX4C_SPEED_IDX;
+			/* wrong user space type, must be CEX3C
+			 * just keep it for cca compatibility
+			 */
+			zc->user_space_type = ZCRYPT_CEX3C;
+		} else if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX5) {
+			zc->type_string = "CEX5C";
+			zc->speed_rating = CEX5C_SPEED_IDX;
+			/* wrong user space type, must be CEX3C
+			 * just keep it for cca compatibility
+			 */
+			zc->user_space_type = ZCRYPT_CEX3C;
+		} else if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX6) {
+			zc->type_string = "CEX6C";
+			zc->speed_rating = CEX6C_SPEED_IDX;
+			/* wrong user space type, must be CEX3C
+			 * just keep it for cca compatibility
+			 */
+			zc->user_space_type = ZCRYPT_CEX3C;
+		} else if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX7) {
+			zc->type_string = "CEX7C";
+			zc->speed_rating = CEX7C_SPEED_IDX;
+			/* wrong user space type, must be CEX3C
+			 * just keep it for cca compatibility
+			 */
+			zc->user_space_type = ZCRYPT_CEX3C;
+		} else {
+			zc->type_string = "CEX8C";
+			zc->speed_rating = CEX8C_SPEED_IDX;
+			/* wrong user space type, must be CEX3C
+			 * just keep it for cca compatibility
+			 */
+			zc->user_space_type = ZCRYPT_CEX3C;
+		}
+		zc->min_mod_size = CEX4C_MIN_MOD_SIZE;
+		zc->max_mod_size = CEX4C_MAX_MOD_SIZE;
+		zc->max_exp_bit_length = CEX4C_MAX_MOD_SIZE;
+	} else if (ac->hwinfo.ep11) {
+		if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX4) {
+			zc->type_string = "CEX4P";
+			zc->user_space_type = ZCRYPT_CEX4;
+			zc->speed_rating = CEX4P_SPEED_IDX;
+		} else if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX5) {
+			zc->type_string = "CEX5P";
+			zc->user_space_type = ZCRYPT_CEX5;
+			zc->speed_rating = CEX5P_SPEED_IDX;
+		} else if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX6) {
+			zc->type_string = "CEX6P";
+			zc->user_space_type = ZCRYPT_CEX6;
+			zc->speed_rating = CEX6P_SPEED_IDX;
+		} else if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX7) {
+			zc->type_string = "CEX7P";
+			zc->speed_rating = CEX7P_SPEED_IDX;
+			/* wrong user space type, just for compatibility
+			 * with the ZCRYPT_STATUS_MASK ioctl.
+			 */
+			zc->user_space_type = ZCRYPT_CEX6;
+		} else {
+			zc->type_string = "CEX8P";
+			zc->speed_rating = CEX8P_SPEED_IDX;
+			/* wrong user space type, just for compatibility
+			 * with the ZCRYPT_STATUS_MASK ioctl.
+			 */
+			zc->user_space_type = ZCRYPT_CEX6;
+		}
+		zc->min_mod_size = CEX4C_MIN_MOD_SIZE;
+		zc->max_mod_size = CEX4C_MAX_MOD_SIZE;
+		zc->max_exp_bit_length = CEX4C_MAX_MOD_SIZE;
+	} else {
+		zcrypt_card_free(zc);
+		return -ENODEV;
+	}
+	zc->online = 1;
+
+	rc = zcrypt_card_register(zc);
+	if (rc) {
+		zcrypt_card_free(zc);
+		return rc;
+	}
+
+	if (ac->hwinfo.cca) {
+		rc = sysfs_create_group(&ap_dev->device.kobj,
+					&cca_card_attr_grp);
+		if (rc) {
+			zcrypt_card_unregister(zc);
+			zcrypt_card_free(zc);
+		}
+	} else if (ac->hwinfo.ep11) {
+		rc = sysfs_create_group(&ap_dev->device.kobj,
+					&ep11_card_attr_grp);
+		if (rc) {
+			zcrypt_card_unregister(zc);
+			zcrypt_card_free(zc);
+		}
+	}
+
+	return rc;
+}
+
+/*
+ * This is called to remove the CEX[45678] card driver
+ * information if an AP card device is removed.
+ */
+static void zcrypt_cex4_card_remove(struct ap_device *ap_dev)
+{
+	struct zcrypt_card *zc = dev_get_drvdata(&ap_dev->device);
+	struct ap_card *ac = to_ap_card(&ap_dev->device);
+
+	if (ac->hwinfo.cca)
+		sysfs_remove_group(&ap_dev->device.kobj, &cca_card_attr_grp);
+	else if (ac->hwinfo.ep11)
+		sysfs_remove_group(&ap_dev->device.kobj, &ep11_card_attr_grp);
+
+	zcrypt_card_unregister(zc);
+}
+
+static struct ap_driver zcrypt_cex4_card_driver = {
+	.probe = zcrypt_cex4_card_probe,
+	.remove = zcrypt_cex4_card_remove,
+	.ids = zcrypt_cex4_card_ids,
+	.flags = AP_DRIVER_FLAG_DEFAULT,
+};
+
+/*
+ * Probe function for CEX[45678] queue device. It always
+ * accepts the AP device since the bus_match already checked
+ * the hardware type.
+ * @ap_dev: pointer to the AP device.
+ */
+static int zcrypt_cex4_queue_probe(struct ap_device *ap_dev)
+{
+	struct ap_queue *aq = to_ap_queue(&ap_dev->device);
+	struct zcrypt_queue *zq;
+	int rc;
+
+	if (aq->card->hwinfo.accel) {
+		zq = zcrypt_queue_alloc(aq->card->maxmsgsize);
+		if (!zq)
+			return -ENOMEM;
+		zq->ops = zcrypt_msgtype(MSGTYPE50_NAME,
+					 MSGTYPE50_VARIANT_DEFAULT);
+	} else if (aq->card->hwinfo.cca) {
+		zq = zcrypt_queue_alloc(aq->card->maxmsgsize);
+		if (!zq)
+			return -ENOMEM;
+		zq->ops = zcrypt_msgtype(MSGTYPE06_NAME,
+					 MSGTYPE06_VARIANT_DEFAULT);
+	} else if (aq->card->hwinfo.ep11) {
+		zq = zcrypt_queue_alloc(aq->card->maxmsgsize);
+		if (!zq)
+			return -ENOMEM;
+		zq->ops = zcrypt_msgtype(MSGTYPE06_NAME,
+					 MSGTYPE06_VARIANT_EP11);
+	} else {
+		return -ENODEV;
+	}
+
+	zq->queue = aq;
+	zq->online = 1;
+	atomic_set(&zq->load, 0);
+	ap_queue_init_state(aq);
+	ap_queue_init_reply(aq, &zq->reply);
+	aq->request_timeout = CEX4_CLEANUP_TIME;
+	dev_set_drvdata(&ap_dev->device, zq);
+	rc = zcrypt_queue_register(zq);
+	if (rc) {
+		zcrypt_queue_free(zq);
+		return rc;
+	}
+
+	if (aq->card->hwinfo.cca) {
+		rc = sysfs_create_group(&ap_dev->device.kobj,
+					&cca_queue_attr_grp);
+		if (rc) {
+			zcrypt_queue_unregister(zq);
+			zcrypt_queue_free(zq);
+		}
+	} else if (aq->card->hwinfo.ep11) {
+		rc = sysfs_create_group(&ap_dev->device.kobj,
+					&ep11_queue_attr_grp);
+		if (rc) {
+			zcrypt_queue_unregister(zq);
+			zcrypt_queue_free(zq);
+		}
+	}
+
+	return rc;
+}
+
+/*
+ * This is called to remove the CEX[45678] queue driver
+ * information if an AP queue device is removed.
+ */
+static void zcrypt_cex4_queue_remove(struct ap_device *ap_dev)
+{
+	struct zcrypt_queue *zq = dev_get_drvdata(&ap_dev->device);
+	struct ap_queue *aq = to_ap_queue(&ap_dev->device);
+
+	if (aq->card->hwinfo.cca)
+		sysfs_remove_group(&ap_dev->device.kobj, &cca_queue_attr_grp);
+	else if (aq->card->hwinfo.ep11)
+		sysfs_remove_group(&ap_dev->device.kobj, &ep11_queue_attr_grp);
+
+	zcrypt_queue_unregister(zq);
+}
+
+static struct ap_driver zcrypt_cex4_queue_driver = {
+	.probe = zcrypt_cex4_queue_probe,
+	.remove = zcrypt_cex4_queue_remove,
+	.ids = zcrypt_cex4_queue_ids,
+	.flags = AP_DRIVER_FLAG_DEFAULT,
+};
+
+int __init zcrypt_cex4_init(void)
+{
+	int rc;
+
+	rc = ap_driver_register(&zcrypt_cex4_card_driver,
+				THIS_MODULE, "cex4card");
+	if (rc)
+		return rc;
+
+	rc = ap_driver_register(&zcrypt_cex4_queue_driver,
+				THIS_MODULE, "cex4queue");
+	if (rc)
+		ap_driver_unregister(&zcrypt_cex4_card_driver);
+
+	return rc;
+}
+
+void __exit zcrypt_cex4_exit(void)
+{
+	ap_driver_unregister(&zcrypt_cex4_queue_driver);
+	ap_driver_unregister(&zcrypt_cex4_card_driver);
+}
+
+module_init(zcrypt_cex4_init);
+module_exit(zcrypt_cex4_exit);
diff --git a/drivers/s390/crypto/zcrypt_cex4.h b/drivers/s390/crypto/zcrypt_cex4.h
new file mode 100644
index 000000000..748390a37
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_cex4.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ *  Copyright IBM Corp. 2012
+ *  Author(s): Holger Dengler <hd@linux.vnet.ibm.com>
+ */
+
+#ifndef _ZCRYPT_CEX4_H_
+#define _ZCRYPT_CEX4_H_
+
+int zcrypt_cex4_init(void);
+void zcrypt_cex4_exit(void);
+
+#endif /* _ZCRYPT_CEX4_H_ */
diff --git a/drivers/s390/crypto/zcrypt_debug.h b/drivers/s390/crypto/zcrypt_debug.h
new file mode 100644
index 000000000..9a208dc4c
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_debug.h
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ *  Copyright IBM Corp. 2016
+ *  Author(s): Holger Dengler (hd@linux.vnet.ibm.com)
+ *	       Harald Freudenberger <freude@de.ibm.com>
+ */
+#ifndef ZCRYPT_DEBUG_H
+#define ZCRYPT_DEBUG_H
+
+#include <asm/debug.h>
+
+#define DBF_ERR		3	/* error conditions   */
+#define DBF_WARN	4	/* warning conditions */
+#define DBF_INFO	5	/* informational      */
+#define DBF_DEBUG	6	/* for debugging only */
+
+#define RC2ERR(rc) ((rc) ? DBF_ERR : DBF_INFO)
+#define RC2WARN(rc) ((rc) ? DBF_WARN : DBF_INFO)
+
+#define ZCRYPT_DBF_MAX_SPRINTF_ARGS 6
+
+#define ZCRYPT_DBF(...)					\
+	debug_sprintf_event(zcrypt_dbf_info, ##__VA_ARGS__)
+#define ZCRYPT_DBF_ERR(...)					\
+	debug_sprintf_event(zcrypt_dbf_info, DBF_ERR, ##__VA_ARGS__)
+#define ZCRYPT_DBF_WARN(...)					\
+	debug_sprintf_event(zcrypt_dbf_info, DBF_WARN, ##__VA_ARGS__)
+#define ZCRYPT_DBF_INFO(...)					\
+	debug_sprintf_event(zcrypt_dbf_info, DBF_INFO, ##__VA_ARGS__)
+
+extern debug_info_t *zcrypt_dbf_info;
+
+int zcrypt_debug_init(void);
+void zcrypt_debug_exit(void);
+
+#endif /* ZCRYPT_DEBUG_H */
diff --git a/drivers/s390/crypto/zcrypt_ep11misc.c b/drivers/s390/crypto/zcrypt_ep11misc.c
new file mode 100644
index 000000000..2d900ffc5
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_ep11misc.c
@@ -0,0 +1,1642 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ *  Copyright IBM Corp. 2019
+ *  Author(s): Harald Freudenberger <freude@linux.ibm.com>
+ *
+ *  Collection of EP11 misc functions used by zcrypt and pkey
+ */
+
+#define pr_fmt(fmt) "zcrypt: " fmt
+
+#include <linux/export.h>
+#include <linux/init.h>
+#include <linux/mempool.h>
+#include <linux/module.h>
+#include <linux/random.h>
+#include <linux/slab.h>
+#include <linux/align.h>
+#include <asm/zcrypt.h>
+#include <asm/pkey.h>
+#include <crypto/aes.h>
+
+#include "ap_bus.h"
+#include "zcrypt_api.h"
+#include "zcrypt_debug.h"
+#include "zcrypt_msgtype6.h"
+#include "zcrypt_ep11misc.h"
+#include "zcrypt_ccamisc.h"
+
+#define EP11_PINBLOB_V1_BYTES 56
+
+/* default iv used here */
+static const u8 def_iv[16] = { 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
+			       0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff };
+
+/*
+ * Cprb memory pool held for urgent cases where no memory
+ * can be allocated via kmalloc. This pool is only used when
+ * alloc_cprbmem() is called with the xflag ZCRYPT_XFLAG_NOMEMALLOC.
+ */
+#define CPRB_MEMPOOL_ITEM_SIZE (8 * 1024)
+static mempool_t *cprb_mempool;
+
+/*
+ * This is a pre-allocated memory for the device status array
+ * used within the ep11_findcard2() function. It is currently
+ * 128 * 128 * 4 bytes = 64 KB big. Usage of this memory is
+ * controlled via dev_status_mem_mutex. Needs adaption if more
+ * than 128 cards or domains to be are supported.
+ */
+#define ZCRYPT_DEV_STATUS_CARD_MAX 128
+#define ZCRYPT_DEV_STATUS_QUEUE_MAX 128
+#define ZCRYPT_DEV_STATUS_ENTRIES (ZCRYPT_DEV_STATUS_CARD_MAX * \
+				   ZCRYPT_DEV_STATUS_QUEUE_MAX)
+#define ZCRYPT_DEV_STATUS_EXT_SIZE (ZCRYPT_DEV_STATUS_ENTRIES * \
+		sizeof(struct zcrypt_device_status_ext))
+static void *dev_status_mem;
+static DEFINE_MUTEX(dev_status_mem_mutex);
+
+static int ep11_kb_split(const u8 *kb, size_t kblen, u32 kbver,
+			 struct ep11kblob_header **kbhdr, size_t *kbhdrsize,
+			 u8 **kbpl, size_t *kbplsize)
+{
+	struct ep11kblob_header *hdr = NULL;
+	size_t hdrsize, plsize = 0;
+	int rc = -EINVAL;
+	u8 *pl = NULL;
+
+	if (kblen < sizeof(struct ep11kblob_header))
+		goto out;
+	hdr = (struct ep11kblob_header *)kb;
+
+	switch (kbver) {
+	case TOKVER_EP11_AES:
+		/* header overlays the payload */
+		hdrsize = 0;
+		break;
+	case TOKVER_EP11_ECC_WITH_HEADER:
+	case TOKVER_EP11_AES_WITH_HEADER:
+		/* payload starts after the header */
+		hdrsize = sizeof(struct ep11kblob_header);
+		break;
+	default:
+		goto out;
+	}
+
+	plsize = kblen - hdrsize;
+	pl = (u8 *)kb + hdrsize;
+
+	if (kbhdr)
+		*kbhdr = hdr;
+	if (kbhdrsize)
+		*kbhdrsize = hdrsize;
+	if (kbpl)
+		*kbpl = pl;
+	if (kbplsize)
+		*kbplsize = plsize;
+
+	rc = 0;
+out:
+	return rc;
+}
+
+static int ep11_kb_decode(const u8 *kb, size_t kblen,
+			  struct ep11kblob_header **kbhdr, size_t *kbhdrsize,
+			  struct ep11keyblob **kbpl, size_t *kbplsize)
+{
+	struct ep11kblob_header *tmph, *hdr = NULL;
+	size_t hdrsize = 0, plsize = 0;
+	struct ep11keyblob *pl = NULL;
+	int rc = -EINVAL;
+	u8 *tmpp;
+
+	if (kblen < sizeof(struct ep11kblob_header))
+		goto out;
+	tmph = (struct ep11kblob_header *)kb;
+
+	if (tmph->type != TOKTYPE_NON_CCA &&
+	    tmph->len > kblen)
+		goto out;
+
+	if (ep11_kb_split(kb, kblen, tmph->version,
+			  &hdr, &hdrsize, &tmpp, &plsize))
+		goto out;
+
+	if (plsize < sizeof(struct ep11keyblob))
+		goto out;
+
+	if (!is_ep11_keyblob(tmpp))
+		goto out;
+
+	pl = (struct ep11keyblob *)tmpp;
+	plsize = hdr->len - hdrsize;
+
+	if (kbhdr)
+		*kbhdr = hdr;
+	if (kbhdrsize)
+		*kbhdrsize = hdrsize;
+	if (kbpl)
+		*kbpl = pl;
+	if (kbplsize)
+		*kbplsize = plsize;
+
+	rc = 0;
+out:
+	return rc;
+}
+
+/*
+ * For valid ep11 keyblobs, returns a reference to the wrappingkey verification
+ * pattern. Otherwise NULL.
+ */
+const u8 *ep11_kb_wkvp(const u8 *keyblob, u32 keybloblen)
+{
+	struct ep11keyblob *kb;
+
+	if (ep11_kb_decode(keyblob, keybloblen, NULL, NULL, &kb, NULL))
+		return NULL;
+	return kb->wkvp;
+}
+EXPORT_SYMBOL(ep11_kb_wkvp);
+
+/*
+ * Simple check if the key blob is a valid EP11 AES key blob with header.
+ */
+int ep11_check_aes_key_with_hdr(debug_info_t *dbg, int dbflvl,
+				const u8 *key, u32 keylen, int checkcpacfexp)
+{
+	struct ep11kblob_header *hdr = (struct ep11kblob_header *)key;
+	struct ep11keyblob *kb = (struct ep11keyblob *)(key + sizeof(*hdr));
+
+#define DBF(...) debug_sprintf_event(dbg, dbflvl, ##__VA_ARGS__)
+
+	if (keylen < sizeof(*hdr) + sizeof(*kb)) {
+		DBF("%s key check failed, keylen %u < %zu\n",
+		    __func__, keylen, sizeof(*hdr) + sizeof(*kb));
+		return -EINVAL;
+	}
+
+	if (hdr->type != TOKTYPE_NON_CCA) {
+		if (dbg)
+			DBF("%s key check failed, type 0x%02x != 0x%02x\n",
+			    __func__, (int)hdr->type, TOKTYPE_NON_CCA);
+		return -EINVAL;
+	}
+	if (hdr->hver != 0x00) {
+		if (dbg)
+			DBF("%s key check failed, header version 0x%02x != 0x00\n",
+			    __func__, (int)hdr->hver);
+		return -EINVAL;
+	}
+	if (hdr->version != TOKVER_EP11_AES_WITH_HEADER) {
+		if (dbg)
+			DBF("%s key check failed, version 0x%02x != 0x%02x\n",
+			    __func__, (int)hdr->version, TOKVER_EP11_AES_WITH_HEADER);
+		return -EINVAL;
+	}
+	if (hdr->len > keylen) {
+		if (dbg)
+			DBF("%s key check failed, header len %d keylen %u mismatch\n",
+			    __func__, (int)hdr->len, keylen);
+		return -EINVAL;
+	}
+	if (hdr->len < sizeof(*hdr) + sizeof(*kb)) {
+		if (dbg)
+			DBF("%s key check failed, header len %d < %zu\n",
+			    __func__, (int)hdr->len, sizeof(*hdr) + sizeof(*kb));
+		return -EINVAL;
+	}
+
+	if (kb->version != EP11_STRUCT_MAGIC) {
+		if (dbg)
+			DBF("%s key check failed, blob magic 0x%04x != 0x%04x\n",
+			    __func__, (int)kb->version, EP11_STRUCT_MAGIC);
+		return -EINVAL;
+	}
+	if (checkcpacfexp && !(kb->attr & EP11_BLOB_PKEY_EXTRACTABLE)) {
+		if (dbg)
+			DBF("%s key check failed, PKEY_EXTRACTABLE is off\n",
+			    __func__);
+		return -EINVAL;
+	}
+
+#undef DBF
+
+	return 0;
+}
+EXPORT_SYMBOL(ep11_check_aes_key_with_hdr);
+
+/*
+ * Simple check if the key blob is a valid EP11 ECC key blob with header.
+ */
+int ep11_check_ecc_key_with_hdr(debug_info_t *dbg, int dbflvl,
+				const u8 *key, u32 keylen, int checkcpacfexp)
+{
+	struct ep11kblob_header *hdr = (struct ep11kblob_header *)key;
+	struct ep11keyblob *kb = (struct ep11keyblob *)(key + sizeof(*hdr));
+
+#define DBF(...) debug_sprintf_event(dbg, dbflvl, ##__VA_ARGS__)
+
+	if (keylen < sizeof(*hdr) + sizeof(*kb)) {
+		DBF("%s key check failed, keylen %u < %zu\n",
+		    __func__, keylen, sizeof(*hdr) + sizeof(*kb));
+		return -EINVAL;
+	}
+
+	if (hdr->type != TOKTYPE_NON_CCA) {
+		if (dbg)
+			DBF("%s key check failed, type 0x%02x != 0x%02x\n",
+			    __func__, (int)hdr->type, TOKTYPE_NON_CCA);
+		return -EINVAL;
+	}
+	if (hdr->hver != 0x00) {
+		if (dbg)
+			DBF("%s key check failed, header version 0x%02x != 0x00\n",
+			    __func__, (int)hdr->hver);
+		return -EINVAL;
+	}
+	if (hdr->version != TOKVER_EP11_ECC_WITH_HEADER) {
+		if (dbg)
+			DBF("%s key check failed, version 0x%02x != 0x%02x\n",
+			    __func__, (int)hdr->version, TOKVER_EP11_ECC_WITH_HEADER);
+		return -EINVAL;
+	}
+	if (hdr->len > keylen) {
+		if (dbg)
+			DBF("%s key check failed, header len %d keylen %u mismatch\n",
+			    __func__, (int)hdr->len, keylen);
+		return -EINVAL;
+	}
+	if (hdr->len < sizeof(*hdr) + sizeof(*kb)) {
+		if (dbg)
+			DBF("%s key check failed, header len %d < %zu\n",
+			    __func__, (int)hdr->len, sizeof(*hdr) + sizeof(*kb));
+		return -EINVAL;
+	}
+
+	if (kb->version != EP11_STRUCT_MAGIC) {
+		if (dbg)
+			DBF("%s key check failed, blob magic 0x%04x != 0x%04x\n",
+			    __func__, (int)kb->version, EP11_STRUCT_MAGIC);
+		return -EINVAL;
+	}
+	if (checkcpacfexp && !(kb->attr & EP11_BLOB_PKEY_EXTRACTABLE)) {
+		if (dbg)
+			DBF("%s key check failed, PKEY_EXTRACTABLE is off\n",
+			    __func__);
+		return -EINVAL;
+	}
+
+#undef DBF
+
+	return 0;
+}
+EXPORT_SYMBOL(ep11_check_ecc_key_with_hdr);
+
+/*
+ * Simple check if the key blob is a valid EP11 AES key blob with
+ * the header in the session field (old style EP11 AES key).
+ */
+int ep11_check_aes_key(debug_info_t *dbg, int dbflvl,
+		       const u8 *key, u32 keylen, int checkcpacfexp)
+{
+	struct ep11keyblob *kb = (struct ep11keyblob *)key;
+
+#define DBF(...) debug_sprintf_event(dbg, dbflvl, ##__VA_ARGS__)
+
+	if (keylen < sizeof(*kb)) {
+		DBF("%s key check failed, keylen %u < %zu\n",
+		    __func__, keylen, sizeof(*kb));
+		return -EINVAL;
+	}
+
+	if (kb->head.type != TOKTYPE_NON_CCA) {
+		if (dbg)
+			DBF("%s key check failed, type 0x%02x != 0x%02x\n",
+			    __func__, (int)kb->head.type, TOKTYPE_NON_CCA);
+		return -EINVAL;
+	}
+	if (kb->head.version != TOKVER_EP11_AES) {
+		if (dbg)
+			DBF("%s key check failed, version 0x%02x != 0x%02x\n",
+			    __func__, (int)kb->head.version, TOKVER_EP11_AES);
+		return -EINVAL;
+	}
+	if (kb->head.len > keylen) {
+		if (dbg)
+			DBF("%s key check failed, header len %d keylen %u mismatch\n",
+			    __func__, (int)kb->head.len, keylen);
+		return -EINVAL;
+	}
+	if (kb->head.len < sizeof(*kb)) {
+		if (dbg)
+			DBF("%s key check failed, header len %d < %zu\n",
+			    __func__, (int)kb->head.len, sizeof(*kb));
+		return -EINVAL;
+	}
+
+	if (kb->version != EP11_STRUCT_MAGIC) {
+		if (dbg)
+			DBF("%s key check failed, blob magic 0x%04x != 0x%04x\n",
+			    __func__, (int)kb->version, EP11_STRUCT_MAGIC);
+		return -EINVAL;
+	}
+	if (checkcpacfexp && !(kb->attr & EP11_BLOB_PKEY_EXTRACTABLE)) {
+		if (dbg)
+			DBF("%s key check failed, PKEY_EXTRACTABLE is off\n",
+			    __func__);
+		return -EINVAL;
+	}
+
+#undef DBF
+
+	return 0;
+}
+EXPORT_SYMBOL(ep11_check_aes_key);
+
+/*
+ * Allocate and prepare ep11 cprb plus additional payload.
+ * It is guaranteed that the memory is aligned to a 4 byte boundary.
+ * Furthermore the memory allocation is rounded up to the next
+ * multiple of 4 bytes (with taking the payload_len into account).
+ */
+static void *alloc_cprbmem(size_t payload_len, u32 xflags)
+{
+	size_t memlen = ALIGN(sizeof(struct ep11_cprb) + payload_len, 4);
+	struct ep11_cprb *cprb = NULL;
+
+	if (xflags & ZCRYPT_XFLAG_NOMEMALLOC) {
+		if (memlen <= CPRB_MEMPOOL_ITEM_SIZE)
+			cprb = mempool_alloc_preallocated(cprb_mempool);
+	} else {
+		cprb = kmalloc(memlen, GFP_KERNEL);
+	}
+	if (!cprb)
+		return NULL;
+	memset(cprb, 0, memlen);
+
+	cprb->cprb_len = sizeof(struct ep11_cprb);
+	cprb->cprb_ver_id = 0x04;
+	memcpy(cprb->func_id, "T4", 2);
+	cprb->ret_code = 0xFFFFFFFF;
+	cprb->payload_len = payload_len;
+
+	return cprb;
+}
+
+/*
+ * Free ep11 cprb buffer space.
+ */
+static void free_cprbmem(void *mem, size_t payload_len, bool scrub, u32 xflags)
+{
+	size_t memlen = ALIGN(sizeof(struct ep11_cprb) + payload_len, 4);
+
+	if (mem && scrub)
+		memzero_explicit(mem, memlen);
+
+	if (xflags & ZCRYPT_XFLAG_NOMEMALLOC)
+		mempool_free(mem, cprb_mempool);
+	else
+		kfree(mem);
+}
+
+/*
+ * Some helper functions related to ASN1 encoding.
+ * Limited to length info <= 2 byte.
+ */
+
+#define ASN1TAGLEN(x) (2 + (x) + ((x) > 127 ? 1 : 0) + ((x) > 255 ? 1 : 0))
+
+static int asn1tag_write(u8 *ptr, u8 tag, const u8 *pvalue, u16 valuelen)
+{
+	ptr[0] = tag;
+	if (valuelen > 255) {
+		ptr[1] = 0x82;
+		*((u16 *)(ptr + 2)) = valuelen;
+		memcpy(ptr + 4, pvalue, valuelen);
+		return 4 + valuelen;
+	}
+	if (valuelen > 127) {
+		ptr[1] = 0x81;
+		ptr[2] = (u8)valuelen;
+		memcpy(ptr + 3, pvalue, valuelen);
+		return 3 + valuelen;
+	}
+	ptr[1] = (u8)valuelen;
+	memcpy(ptr + 2, pvalue, valuelen);
+	return 2 + valuelen;
+}
+
+/* EP11 payload > 127 bytes starts with this struct */
+struct pl_head {
+	u8  tag;
+	u8  lenfmt;
+	u16 len;
+	u8  func_tag;
+	u8  func_len;
+	u32 func;
+	u8  dom_tag;
+	u8  dom_len;
+	u32 dom;
+} __packed;
+
+/* prep ep11 payload head helper function */
+static inline void prep_head(struct pl_head *h,
+			     size_t pl_size, int api, int func)
+{
+	h->tag = 0x30;
+	h->lenfmt = 0x82;
+	h->len = pl_size - 4;
+	h->func_tag = 0x04;
+	h->func_len = sizeof(u32);
+	h->func = (api << 16) + func;
+	h->dom_tag = 0x04;
+	h->dom_len = sizeof(u32);
+}
+
+/* prep urb helper function */
+static inline void prep_urb(struct ep11_urb *u,
+			    struct ep11_target_dev *t, int nt,
+			    struct ep11_cprb *req, size_t req_len,
+			    struct ep11_cprb *rep, size_t rep_len)
+{
+	memset(u, 0, sizeof(*u));
+	u->targets = (u8 __user *)t;
+	u->targets_num = nt;
+	u->req = (u8 __user *)req;
+	u->req_len = req_len;
+	u->resp = (u8 __user *)rep;
+	u->resp_len = rep_len;
+}
+
+/* Check ep11 reply payload, return 0 or suggested errno value. */
+static int check_reply_pl(const u8 *pl, const char *func)
+{
+	int len;
+	u32 ret;
+
+	/* start tag */
+	if (*pl++ != 0x30) {
+		ZCRYPT_DBF_ERR("%s reply start tag mismatch\n", func);
+		return -EIO;
+	}
+
+	/* payload length format */
+	if (*pl < 127) {
+		len = *pl;
+		pl++;
+	} else if (*pl == 0x81) {
+		pl++;
+		len = *pl;
+		pl++;
+	} else if (*pl == 0x82) {
+		pl++;
+		len = *((u16 *)pl);
+		pl += 2;
+	} else {
+		ZCRYPT_DBF_ERR("%s reply start tag lenfmt mismatch 0x%02hhx\n",
+			       func, *pl);
+		return -EIO;
+	}
+
+	/* len should cover at least 3 fields with 32 bit value each */
+	if (len < 3 * 6) {
+		ZCRYPT_DBF_ERR("%s reply length %d too small\n", func, len);
+		return -EIO;
+	}
+
+	/* function tag, length and value */
+	if (pl[0] != 0x04 || pl[1] != 0x04) {
+		ZCRYPT_DBF_ERR("%s function tag or length mismatch\n", func);
+		return -EIO;
+	}
+	pl += 6;
+
+	/* dom tag, length and value */
+	if (pl[0] != 0x04 || pl[1] != 0x04) {
+		ZCRYPT_DBF_ERR("%s dom tag or length mismatch\n", func);
+		return -EIO;
+	}
+	pl += 6;
+
+	/* return value tag, length and value */
+	if (pl[0] != 0x04 || pl[1] != 0x04) {
+		ZCRYPT_DBF_ERR("%s return value tag or length mismatch\n",
+			       func);
+		return -EIO;
+	}
+	pl += 2;
+	ret = *((u32 *)pl);
+	if (ret != 0) {
+		ZCRYPT_DBF_ERR("%s return value 0x%08x != 0\n", func, ret);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+/* Check ep11 reply cprb, return 0 or suggested errno value. */
+static int check_reply_cprb(const struct ep11_cprb *rep, const char *func)
+{
+	/* check ep11 reply return code field */
+	if (rep->ret_code) {
+		ZCRYPT_DBF_ERR("%s ep11 reply ret_code=0x%08x\n", __func__,
+			       rep->ret_code);
+		if (rep->ret_code == 0x000c0003)
+			return -EBUSY;
+		else
+			return -EIO;
+	}
+
+	return 0;
+}
+
+/*
+ * Helper function which does an ep11 query with given query type.
+ */
+static int ep11_query_info(u16 cardnr, u16 domain, u32 query_type,
+			   size_t buflen, u8 *buf, u32 xflags)
+{
+	struct ep11_info_req_pl {
+		struct pl_head head;
+		u8  query_type_tag;
+		u8  query_type_len;
+		u32 query_type;
+		u8  query_subtype_tag;
+		u8  query_subtype_len;
+		u32 query_subtype;
+	} __packed * req_pl;
+	struct ep11_info_rep_pl {
+		struct pl_head head;
+		u8  rc_tag;
+		u8  rc_len;
+		u32 rc;
+		u8  data_tag;
+		u8  data_lenfmt;
+		u16 data_len;
+	} __packed * rep_pl;
+	struct ep11_cprb *req = NULL, *rep = NULL;
+	struct ep11_target_dev target;
+	struct ep11_urb urb;
+	int api = EP11_API_V1, rc = -ENOMEM;
+
+	/* request cprb and payload */
+	req = alloc_cprbmem(sizeof(struct ep11_info_req_pl), xflags);
+	if (!req)
+		goto out;
+	req_pl = (struct ep11_info_req_pl *)(((u8 *)req) + sizeof(*req));
+	prep_head(&req_pl->head, sizeof(*req_pl), api, 38); /* get xcp info */
+	req_pl->query_type_tag = 0x04;
+	req_pl->query_type_len = sizeof(u32);
+	req_pl->query_type = query_type;
+	req_pl->query_subtype_tag = 0x04;
+	req_pl->query_subtype_len = sizeof(u32);
+
+	/* reply cprb and payload */
+	rep = alloc_cprbmem(sizeof(struct ep11_info_rep_pl) + buflen, xflags);
+	if (!rep)
+		goto out;
+	rep_pl = (struct ep11_info_rep_pl *)(((u8 *)rep) + sizeof(*rep));
+
+	/* urb and target */
+	target.ap_id = cardnr;
+	target.dom_id = domain;
+	prep_urb(&urb, &target, 1,
+		 req, sizeof(*req) + sizeof(*req_pl),
+		 rep, sizeof(*rep) + sizeof(*rep_pl) + buflen);
+
+	rc = zcrypt_send_ep11_cprb(&urb, xflags);
+	if (rc) {
+		ZCRYPT_DBF_ERR("%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n",
+			       __func__, (int)cardnr, (int)domain, rc);
+		goto out;
+	}
+
+	/* check ep11 reply cprb */
+	rc = check_reply_cprb(rep, __func__);
+	if (rc)
+		goto out;
+
+	/* check payload */
+	rc = check_reply_pl((u8 *)rep_pl, __func__);
+	if (rc)
+		goto out;
+	if (rep_pl->data_tag != 0x04 || rep_pl->data_lenfmt != 0x82) {
+		ZCRYPT_DBF_ERR("%s unknown reply data format\n", __func__);
+		rc = -EIO;
+		goto out;
+	}
+	if (rep_pl->data_len > buflen) {
+		ZCRYPT_DBF_ERR("%s mismatch between reply data len and buffer len\n",
+			       __func__);
+		rc = -ENOSPC;
+		goto out;
+	}
+
+	memcpy(buf, ((u8 *)rep_pl) + sizeof(*rep_pl), rep_pl->data_len);
+
+out:
+	free_cprbmem(req, 0, false, xflags);
+	free_cprbmem(rep, 0, false, xflags);
+	return rc;
+}
+
+/*
+ * Provide information about an EP11 card.
+ */
+int ep11_get_card_info(u16 card, struct ep11_card_info *info, u32 xflags)
+{
+	int rc;
+	struct ep11_module_query_info {
+		u32 API_ord_nr;
+		u32 firmware_id;
+		u8  FW_major_vers;
+		u8  FW_minor_vers;
+		u8  CSP_major_vers;
+		u8  CSP_minor_vers;
+		u8  fwid[32];
+		u8  xcp_config_hash[32];
+		u8  CSP_config_hash[32];
+		u8  serial[16];
+		u8  module_date_time[16];
+		u64 op_mode;
+		u32 PKCS11_flags;
+		u32 ext_flags;
+		u32 domains;
+		u32 sym_state_bytes;
+		u32 digest_state_bytes;
+		u32 pin_blob_bytes;
+		u32 SPKI_bytes;
+		u32 priv_key_blob_bytes;
+		u32 sym_blob_bytes;
+		u32 max_payload_bytes;
+		u32 CP_profile_bytes;
+		u32 max_CP_index;
+	} __packed * pmqi = NULL;
+
+	/* use the cprb mempool to satisfy this short term mem alloc */
+	pmqi = (xflags & ZCRYPT_XFLAG_NOMEMALLOC) ?
+		mempool_alloc_preallocated(cprb_mempool) :
+		mempool_alloc(cprb_mempool, GFP_KERNEL);
+	if (!pmqi)
+		return -ENOMEM;
+	rc = ep11_query_info(card, AUTOSEL_DOM,
+			     0x01 /* module info query */,
+			     sizeof(*pmqi), (u8 *)pmqi, xflags);
+	if (rc)
+		goto out;
+
+	memset(info, 0, sizeof(*info));
+	info->API_ord_nr = pmqi->API_ord_nr;
+	info->FW_version = (pmqi->FW_major_vers << 8) + pmqi->FW_minor_vers;
+	memcpy(info->serial, pmqi->serial, sizeof(info->serial));
+	info->op_mode = pmqi->op_mode;
+
+out:
+	mempool_free(pmqi, cprb_mempool);
+	return rc;
+}
+EXPORT_SYMBOL(ep11_get_card_info);
+
+/*
+ * Provide information about a domain within an EP11 card.
+ */
+int ep11_get_domain_info(u16 card, u16 domain,
+			 struct ep11_domain_info *info, u32 xflags)
+{
+	int rc;
+	struct ep11_domain_query_info {
+		u32 dom_index;
+		u8  cur_WK_VP[32];
+		u8  new_WK_VP[32];
+		u32 dom_flags;
+		u64 op_mode;
+	} __packed dom_query_info;
+
+	rc = ep11_query_info(card, domain, 0x03 /* domain info query */,
+			     sizeof(dom_query_info), (u8 *)&dom_query_info,
+			     xflags);
+	if (rc)
+		goto out;
+
+	memset(info, 0, sizeof(*info));
+	info->cur_wk_state = '0';
+	info->new_wk_state = '0';
+	if (dom_query_info.dom_flags & 0x10 /* left imprint mode */) {
+		if (dom_query_info.dom_flags & 0x02 /* cur wk valid */) {
+			info->cur_wk_state = '1';
+			memcpy(info->cur_wkvp, dom_query_info.cur_WK_VP, 32);
+		}
+		if (dom_query_info.dom_flags & 0x04 || /* new wk present */
+		    dom_query_info.dom_flags & 0x08 /* new wk committed */) {
+			info->new_wk_state =
+				dom_query_info.dom_flags & 0x08 ? '2' : '1';
+			memcpy(info->new_wkvp, dom_query_info.new_WK_VP, 32);
+		}
+	}
+	info->op_mode = dom_query_info.op_mode;
+
+out:
+	return rc;
+}
+EXPORT_SYMBOL(ep11_get_domain_info);
+
+/*
+ * Default EP11 AES key generate attributes, used when no keygenflags given:
+ * XCP_BLOB_ENCRYPT | XCP_BLOB_DECRYPT | XCP_BLOB_PROTKEY_EXTRACTABLE
+ */
+#define KEY_ATTR_DEFAULTS 0x00200c00
+
+static int _ep11_genaeskey(u16 card, u16 domain,
+			   u32 keybitsize, u32 keygenflags,
+			   u8 *keybuf, size_t *keybufsize, u32 xflags)
+{
+	struct keygen_req_pl {
+		struct pl_head head;
+		u8  var_tag;
+		u8  var_len;
+		u32 var;
+		u8  keybytes_tag;
+		u8  keybytes_len;
+		u32 keybytes;
+		u8  mech_tag;
+		u8  mech_len;
+		u32 mech;
+		u8  attr_tag;
+		u8  attr_len;
+		u32 attr_header;
+		u32 attr_bool_mask;
+		u32 attr_bool_bits;
+		u32 attr_val_len_type;
+		u32 attr_val_len_value;
+		/* followed by empty pin tag or empty pinblob tag */
+	} __packed * req_pl;
+	struct keygen_rep_pl {
+		struct pl_head head;
+		u8  rc_tag;
+		u8  rc_len;
+		u32 rc;
+		u8  data_tag;
+		u8  data_lenfmt;
+		u16 data_len;
+		u8  data[512];
+	} __packed * rep_pl;
+	struct ep11_cprb *req = NULL, *rep = NULL;
+	size_t req_pl_size, pinblob_size = 0;
+	struct ep11_target_dev target;
+	struct ep11_urb urb;
+	int api, rc = -ENOMEM;
+	u8 *p;
+
+	switch (keybitsize) {
+	case 128:
+	case 192:
+	case 256:
+		break;
+	default:
+		ZCRYPT_DBF_ERR("%s unknown/unsupported keybitsize %d\n",
+			       __func__, keybitsize);
+		rc = -EINVAL;
+		goto out;
+	}
+
+	/* request cprb and payload */
+	api = (!keygenflags || keygenflags & 0x00200000) ?
+		EP11_API_V4 : EP11_API_V1;
+	if (ap_is_se_guest()) {
+		/*
+		 * genkey within SE environment requires API ordinal 6
+		 * with empty pinblob
+		 */
+		api = EP11_API_V6;
+		pinblob_size = EP11_PINBLOB_V1_BYTES;
+	}
+	req_pl_size = sizeof(struct keygen_req_pl) + ASN1TAGLEN(pinblob_size);
+	req = alloc_cprbmem(req_pl_size, xflags);
+	if (!req)
+		goto out;
+	req_pl = (struct keygen_req_pl *)(((u8 *)req) + sizeof(*req));
+	prep_head(&req_pl->head, req_pl_size, api, 21); /* GenerateKey */
+	req_pl->var_tag = 0x04;
+	req_pl->var_len = sizeof(u32);
+	req_pl->keybytes_tag = 0x04;
+	req_pl->keybytes_len = sizeof(u32);
+	req_pl->keybytes = keybitsize / 8;
+	req_pl->mech_tag = 0x04;
+	req_pl->mech_len = sizeof(u32);
+	req_pl->mech = 0x00001080; /* CKM_AES_KEY_GEN */
+	req_pl->attr_tag = 0x04;
+	req_pl->attr_len = 5 * sizeof(u32);
+	req_pl->attr_header = 0x10010000;
+	req_pl->attr_bool_mask = keygenflags ? keygenflags : KEY_ATTR_DEFAULTS;
+	req_pl->attr_bool_bits = keygenflags ? keygenflags : KEY_ATTR_DEFAULTS;
+	req_pl->attr_val_len_type = 0x00000161; /* CKA_VALUE_LEN */
+	req_pl->attr_val_len_value = keybitsize / 8;
+	p = ((u8 *)req_pl) + sizeof(*req_pl);
+	/* pin tag */
+	*p++ = 0x04;
+	*p++ = pinblob_size;
+
+	/* reply cprb and payload */
+	rep = alloc_cprbmem(sizeof(struct keygen_rep_pl), xflags);
+	if (!rep)
+		goto out;
+	rep_pl = (struct keygen_rep_pl *)(((u8 *)rep) + sizeof(*rep));
+
+	/* urb and target */
+	target.ap_id = card;
+	target.dom_id = domain;
+	prep_urb(&urb, &target, 1,
+		 req, sizeof(*req) + req_pl_size,
+		 rep, sizeof(*rep) + sizeof(*rep_pl));
+
+	rc = zcrypt_send_ep11_cprb(&urb, xflags);
+	if (rc) {
+		ZCRYPT_DBF_ERR("%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n",
+			       __func__, (int)card, (int)domain, rc);
+		goto out;
+	}
+
+	/* check ep11 reply cprb */
+	rc = check_reply_cprb(rep, __func__);
+	if (rc)
+		goto out;
+
+	/* check payload */
+	rc = check_reply_pl((u8 *)rep_pl, __func__);
+	if (rc)
+		goto out;
+	if (rep_pl->data_tag != 0x04 || rep_pl->data_lenfmt != 0x82) {
+		ZCRYPT_DBF_ERR("%s unknown reply data format\n", __func__);
+		rc = -EIO;
+		goto out;
+	}
+	if (rep_pl->data_len > *keybufsize) {
+		ZCRYPT_DBF_ERR("%s mismatch reply data len / key buffer len\n",
+			       __func__);
+		rc = -ENOSPC;
+		goto out;
+	}
+
+	/* copy key blob */
+	memcpy(keybuf, rep_pl->data, rep_pl->data_len);
+	*keybufsize = rep_pl->data_len;
+
+out:
+	free_cprbmem(req, 0, false, xflags);
+	free_cprbmem(rep, sizeof(struct keygen_rep_pl), true, xflags);
+	return rc;
+}
+
+int ep11_genaeskey(u16 card, u16 domain, u32 keybitsize, u32 keygenflags,
+		   u8 *keybuf, u32 *keybufsize, u32 keybufver, u32 xflags)
+{
+	struct ep11kblob_header *hdr;
+	size_t hdr_size, pl_size;
+	u8 *pl;
+	int rc;
+
+	switch (keybufver) {
+	case TOKVER_EP11_AES:
+	case TOKVER_EP11_AES_WITH_HEADER:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	rc = ep11_kb_split(keybuf, *keybufsize, keybufver,
+			   &hdr, &hdr_size, &pl, &pl_size);
+	if (rc)
+		return rc;
+
+	rc = _ep11_genaeskey(card, domain, keybitsize, keygenflags,
+			     pl, &pl_size, xflags);
+	if (rc)
+		return rc;
+
+	*keybufsize = hdr_size + pl_size;
+
+	/* update header information */
+	hdr->type = TOKTYPE_NON_CCA;
+	hdr->len = *keybufsize;
+	hdr->version = keybufver;
+	hdr->bitlen = keybitsize;
+
+	return 0;
+}
+EXPORT_SYMBOL(ep11_genaeskey);
+
+static int ep11_cryptsingle(u16 card, u16 domain,
+			    u16 mode, u32 mech, const u8 *iv,
+			    const u8 *key, size_t keysize,
+			    const u8 *inbuf, size_t inbufsize,
+			    u8 *outbuf, size_t *outbufsize,
+			    u32 xflags)
+{
+	struct crypt_req_pl {
+		struct pl_head head;
+		u8  var_tag;
+		u8  var_len;
+		u32 var;
+		u8  mech_tag;
+		u8  mech_len;
+		u32 mech;
+		/*
+		 * maybe followed by iv data
+		 * followed by key tag + key blob
+		 * followed by plaintext tag + plaintext
+		 */
+	} __packed * req_pl;
+	struct crypt_rep_pl {
+		struct pl_head head;
+		u8  rc_tag;
+		u8  rc_len;
+		u32 rc;
+		u8  data_tag;
+		u8  data_lenfmt;
+		/* data follows */
+	} __packed * rep_pl;
+	struct ep11_cprb *req = NULL, *rep = NULL;
+	struct ep11_target_dev target;
+	struct ep11_urb urb;
+	size_t req_pl_size, rep_pl_size = 0;
+	int n, api = EP11_API_V1, rc = -ENOMEM;
+	u8 *p;
+
+	/* the simple asn1 coding used has length limits */
+	if (keysize > 0xFFFF || inbufsize > 0xFFFF)
+		return -EINVAL;
+
+	/* request cprb and payload */
+	req_pl_size = sizeof(struct crypt_req_pl) + (iv ? 16 : 0)
+		+ ASN1TAGLEN(keysize) + ASN1TAGLEN(inbufsize);
+	req = alloc_cprbmem(req_pl_size, xflags);
+	if (!req)
+		goto out;
+	req_pl = (struct crypt_req_pl *)(((u8 *)req) + sizeof(*req));
+	prep_head(&req_pl->head, req_pl_size, api, (mode ? 20 : 19));
+	req_pl->var_tag = 0x04;
+	req_pl->var_len = sizeof(u32);
+	/* mech is mech + mech params (iv here) */
+	req_pl->mech_tag = 0x04;
+	req_pl->mech_len = sizeof(u32) + (iv ? 16 : 0);
+	req_pl->mech = (mech ? mech : 0x00001085); /* CKM_AES_CBC_PAD */
+	p = ((u8 *)req_pl) + sizeof(*req_pl);
+	if (iv) {
+		memcpy(p, iv, 16);
+		p += 16;
+	}
+	/* key and input data */
+	p += asn1tag_write(p, 0x04, key, keysize);
+	p += asn1tag_write(p, 0x04, inbuf, inbufsize);
+
+	/* reply cprb and payload, assume out data size <= in data size + 32 */
+	rep_pl_size = sizeof(struct crypt_rep_pl) + ASN1TAGLEN(inbufsize + 32);
+	rep = alloc_cprbmem(rep_pl_size, xflags);
+	if (!rep)
+		goto out;
+	rep_pl = (struct crypt_rep_pl *)(((u8 *)rep) + sizeof(*rep));
+
+	/* urb and target */
+	target.ap_id = card;
+	target.dom_id = domain;
+	prep_urb(&urb, &target, 1,
+		 req, sizeof(*req) + req_pl_size,
+		 rep, sizeof(*rep) + rep_pl_size);
+
+	rc = zcrypt_send_ep11_cprb(&urb, xflags);
+	if (rc) {
+		ZCRYPT_DBF_ERR("%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n",
+			       __func__, (int)card, (int)domain, rc);
+		goto out;
+	}
+
+	/* check ep11 reply cprb */
+	rc = check_reply_cprb(rep, __func__);
+	if (rc)
+		goto out;
+
+	/* check payload */
+	rc = check_reply_pl((u8 *)rep_pl, __func__);
+	if (rc)
+		goto out;
+	if (rep_pl->data_tag != 0x04) {
+		ZCRYPT_DBF_ERR("%s unknown reply data format\n", __func__);
+		rc = -EIO;
+		goto out;
+	}
+	p = ((u8 *)rep_pl) + sizeof(*rep_pl);
+	if (rep_pl->data_lenfmt <= 127) {
+		n = rep_pl->data_lenfmt;
+	} else if (rep_pl->data_lenfmt == 0x81) {
+		n = *p++;
+	} else if (rep_pl->data_lenfmt == 0x82) {
+		n = *((u16 *)p);
+		p += 2;
+	} else {
+		ZCRYPT_DBF_ERR("%s unknown reply data length format 0x%02hhx\n",
+			       __func__, rep_pl->data_lenfmt);
+		rc = -EIO;
+		goto out;
+	}
+	if (n > *outbufsize) {
+		ZCRYPT_DBF_ERR("%s mismatch reply data len %d / output buffer %zu\n",
+			       __func__, n, *outbufsize);
+		rc = -ENOSPC;
+		goto out;
+	}
+
+	memcpy(outbuf, p, n);
+	*outbufsize = n;
+
+out:
+	free_cprbmem(req, req_pl_size, true, xflags);
+	free_cprbmem(rep, rep_pl_size, true, xflags);
+	return rc;
+}
+
+static int _ep11_unwrapkey(u16 card, u16 domain,
+			   const u8 *kek, size_t keksize,
+			   const u8 *enckey, size_t enckeysize,
+			   u32 mech, const u8 *iv,
+			   u32 keybitsize, u32 keygenflags,
+			   u8 *keybuf, size_t *keybufsize, u32 xflags)
+{
+	struct uw_req_pl {
+		struct pl_head head;
+		u8  attr_tag;
+		u8  attr_len;
+		u32 attr_header;
+		u32 attr_bool_mask;
+		u32 attr_bool_bits;
+		u32 attr_key_type;
+		u32 attr_key_type_value;
+		u32 attr_val_len;
+		u32 attr_val_len_value;
+		u8  mech_tag;
+		u8  mech_len;
+		u32 mech;
+		/*
+		 * maybe followed by iv data
+		 * followed by kek tag + kek blob
+		 * followed by empty mac tag
+		 * followed by empty pin tag or empty pinblob tag
+		 * followed by encryted key tag + bytes
+		 */
+	} __packed * req_pl;
+	struct uw_rep_pl {
+		struct pl_head head;
+		u8  rc_tag;
+		u8  rc_len;
+		u32 rc;
+		u8  data_tag;
+		u8  data_lenfmt;
+		u16 data_len;
+		u8  data[512];
+	} __packed * rep_pl;
+	struct ep11_cprb *req = NULL, *rep = NULL;
+	size_t req_pl_size, pinblob_size = 0;
+	struct ep11_target_dev target;
+	struct ep11_urb urb;
+	int api, rc = -ENOMEM;
+	u8 *p;
+
+	/* request cprb and payload */
+	api = (!keygenflags || keygenflags & 0x00200000) ?
+		EP11_API_V4 : EP11_API_V1;
+	if (ap_is_se_guest()) {
+		/*
+		 * unwrap within SE environment requires API ordinal 6
+		 * with empty pinblob
+		 */
+		api = EP11_API_V6;
+		pinblob_size = EP11_PINBLOB_V1_BYTES;
+	}
+	req_pl_size = sizeof(struct uw_req_pl) + (iv ? 16 : 0)
+		+ ASN1TAGLEN(keksize) + ASN1TAGLEN(0)
+		+ ASN1TAGLEN(pinblob_size) + ASN1TAGLEN(enckeysize);
+	req = alloc_cprbmem(req_pl_size, xflags);
+	if (!req)
+		goto out;
+	req_pl = (struct uw_req_pl *)(((u8 *)req) + sizeof(*req));
+	prep_head(&req_pl->head, req_pl_size, api, 34); /* UnwrapKey */
+	req_pl->attr_tag = 0x04;
+	req_pl->attr_len = 7 * sizeof(u32);
+	req_pl->attr_header = 0x10020000;
+	req_pl->attr_bool_mask = keygenflags ? keygenflags : KEY_ATTR_DEFAULTS;
+	req_pl->attr_bool_bits = keygenflags ? keygenflags : KEY_ATTR_DEFAULTS;
+	req_pl->attr_key_type = 0x00000100; /* CKA_KEY_TYPE */
+	req_pl->attr_key_type_value = 0x0000001f; /* CKK_AES */
+	req_pl->attr_val_len = 0x00000161; /* CKA_VALUE_LEN */
+	req_pl->attr_val_len_value = keybitsize / 8;
+	/* mech is mech + mech params (iv here) */
+	req_pl->mech_tag = 0x04;
+	req_pl->mech_len = sizeof(u32) + (iv ? 16 : 0);
+	req_pl->mech = (mech ? mech : 0x00001085); /* CKM_AES_CBC_PAD */
+	p = ((u8 *)req_pl) + sizeof(*req_pl);
+	if (iv) {
+		memcpy(p, iv, 16);
+		p += 16;
+	}
+	/* kek */
+	p += asn1tag_write(p, 0x04, kek, keksize);
+	/* empty mac key tag */
+	*p++ = 0x04;
+	*p++ = 0;
+	/* pin tag */
+	*p++ = 0x04;
+	*p++ = pinblob_size;
+	p += pinblob_size;
+	/* encrypted key value tag and bytes */
+	p += asn1tag_write(p, 0x04, enckey, enckeysize);
+
+	/* reply cprb and payload */
+	rep = alloc_cprbmem(sizeof(struct uw_rep_pl), xflags);
+	if (!rep)
+		goto out;
+	rep_pl = (struct uw_rep_pl *)(((u8 *)rep) + sizeof(*rep));
+
+	/* urb and target */
+	target.ap_id = card;
+	target.dom_id = domain;
+	prep_urb(&urb, &target, 1,
+		 req, sizeof(*req) + req_pl_size,
+		 rep, sizeof(*rep) + sizeof(*rep_pl));
+
+	rc = zcrypt_send_ep11_cprb(&urb, xflags);
+	if (rc) {
+		ZCRYPT_DBF_ERR("%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n",
+			       __func__, (int)card, (int)domain, rc);
+		goto out;
+	}
+
+	/* check ep11 reply cprb */
+	rc = check_reply_cprb(rep, __func__);
+	if (rc)
+		goto out;
+
+	/* check payload */
+	rc = check_reply_pl((u8 *)rep_pl, __func__);
+	if (rc)
+		goto out;
+	if (rep_pl->data_tag != 0x04 || rep_pl->data_lenfmt != 0x82) {
+		ZCRYPT_DBF_ERR("%s unknown reply data format\n", __func__);
+		rc = -EIO;
+		goto out;
+	}
+	if (rep_pl->data_len > *keybufsize) {
+		ZCRYPT_DBF_ERR("%s mismatch reply data len / key buffer len\n",
+			       __func__);
+		rc = -ENOSPC;
+		goto out;
+	}
+
+	/* copy key blob */
+	memcpy(keybuf, rep_pl->data, rep_pl->data_len);
+	*keybufsize = rep_pl->data_len;
+
+out:
+	free_cprbmem(req, req_pl_size, true, xflags);
+	free_cprbmem(rep, sizeof(struct uw_rep_pl), true, xflags);
+	return rc;
+}
+
+static int ep11_unwrapkey(u16 card, u16 domain,
+			  const u8 *kek, size_t keksize,
+			  const u8 *enckey, size_t enckeysize,
+			  u32 mech, const u8 *iv,
+			  u32 keybitsize, u32 keygenflags,
+			  u8 *keybuf, u32 *keybufsize,
+			  u8 keybufver, u32 xflags)
+{
+	struct ep11kblob_header *hdr;
+	size_t hdr_size, pl_size;
+	u8 *pl;
+	int rc;
+
+	rc = ep11_kb_split(keybuf, *keybufsize, keybufver,
+			   &hdr, &hdr_size, &pl, &pl_size);
+	if (rc)
+		return rc;
+
+	rc = _ep11_unwrapkey(card, domain, kek, keksize, enckey, enckeysize,
+			     mech, iv, keybitsize, keygenflags,
+			     pl, &pl_size, xflags);
+	if (rc)
+		return rc;
+
+	*keybufsize = hdr_size + pl_size;
+
+	/* update header information */
+	hdr = (struct ep11kblob_header *)keybuf;
+	hdr->type = TOKTYPE_NON_CCA;
+	hdr->len = *keybufsize;
+	hdr->version = keybufver;
+	hdr->bitlen = keybitsize;
+
+	return 0;
+}
+
+static int _ep11_wrapkey(u16 card, u16 domain,
+			 const u8 *key, size_t keysize,
+			 u32 mech, const u8 *iv,
+			 u8 *databuf, size_t *datasize, u32 xflags)
+{
+	struct wk_req_pl {
+		struct pl_head head;
+		u8  var_tag;
+		u8  var_len;
+		u32 var;
+		u8  mech_tag;
+		u8  mech_len;
+		u32 mech;
+		/*
+		 * followed by iv data
+		 * followed by key tag + key blob
+		 * followed by dummy kek param
+		 * followed by dummy mac param
+		 */
+	} __packed * req_pl;
+	struct wk_rep_pl {
+		struct pl_head head;
+		u8  rc_tag;
+		u8  rc_len;
+		u32 rc;
+		u8  data_tag;
+		u8  data_lenfmt;
+		u16 data_len;
+		u8  data[1024];
+	} __packed * rep_pl;
+	struct ep11_cprb *req = NULL, *rep = NULL;
+	struct ep11_target_dev target;
+	struct ep11_urb urb;
+	size_t req_pl_size;
+	int api, rc = -ENOMEM;
+	u8 *p;
+
+	/* request cprb and payload */
+	req_pl_size = sizeof(struct wk_req_pl) + (iv ? 16 : 0)
+		+ ASN1TAGLEN(keysize) + 4;
+	req = alloc_cprbmem(req_pl_size, xflags);
+	if (!req)
+		goto out;
+	if (!mech || mech == 0x80060001)
+		req->flags |= 0x20; /* CPACF_WRAP needs special bit */
+	req_pl = (struct wk_req_pl *)(((u8 *)req) + sizeof(*req));
+	api = (!mech || mech == 0x80060001) ? /* CKM_IBM_CPACF_WRAP */
+		EP11_API_V4 : EP11_API_V1;
+	prep_head(&req_pl->head, req_pl_size, api, 33); /* WrapKey */
+	req_pl->var_tag = 0x04;
+	req_pl->var_len = sizeof(u32);
+	/* mech is mech + mech params (iv here) */
+	req_pl->mech_tag = 0x04;
+	req_pl->mech_len = sizeof(u32) + (iv ? 16 : 0);
+	req_pl->mech = (mech ? mech : 0x80060001); /* CKM_IBM_CPACF_WRAP */
+	p = ((u8 *)req_pl) + sizeof(*req_pl);
+	if (iv) {
+		memcpy(p, iv, 16);
+		p += 16;
+	}
+	/* key blob */
+	p += asn1tag_write(p, 0x04, key, keysize);
+	/* empty kek tag */
+	*p++ = 0x04;
+	*p++ = 0;
+	/* empty mac tag */
+	*p++ = 0x04;
+	*p++ = 0;
+
+	/* reply cprb and payload */
+	rep = alloc_cprbmem(sizeof(struct wk_rep_pl), xflags);
+	if (!rep)
+		goto out;
+	rep_pl = (struct wk_rep_pl *)(((u8 *)rep) + sizeof(*rep));
+
+	/* urb and target */
+	target.ap_id = card;
+	target.dom_id = domain;
+	prep_urb(&urb, &target, 1,
+		 req, sizeof(*req) + req_pl_size,
+		 rep, sizeof(*rep) + sizeof(*rep_pl));
+
+	rc = zcrypt_send_ep11_cprb(&urb, xflags);
+	if (rc) {
+		ZCRYPT_DBF_ERR("%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n",
+			       __func__, (int)card, (int)domain, rc);
+		goto out;
+	}
+
+	/* check ep11 reply cprb */
+	rc = check_reply_cprb(rep, __func__);
+	if (rc)
+		goto out;
+
+	/* check payload */
+	rc = check_reply_pl((u8 *)rep_pl, __func__);
+	if (rc)
+		goto out;
+	if (rep_pl->data_tag != 0x04 || rep_pl->data_lenfmt != 0x82) {
+		ZCRYPT_DBF_ERR("%s unknown reply data format\n", __func__);
+		rc = -EIO;
+		goto out;
+	}
+	if (rep_pl->data_len > *datasize) {
+		ZCRYPT_DBF_ERR("%s mismatch reply data len / data buffer len\n",
+			       __func__);
+		rc = -ENOSPC;
+		goto out;
+	}
+
+	/* copy the data from the cprb to the data buffer */
+	memcpy(databuf, rep_pl->data, rep_pl->data_len);
+	*datasize = rep_pl->data_len;
+
+out:
+	free_cprbmem(req, req_pl_size, true, xflags);
+	free_cprbmem(rep, sizeof(struct wk_rep_pl), true, xflags);
+	return rc;
+}
+
+int ep11_clr2keyblob(u16 card, u16 domain, u32 keybitsize, u32 keygenflags,
+		     const u8 *clrkey, u8 *keybuf, u32 *keybufsize,
+		     u32 keytype, u32 xflags)
+{
+	int rc;
+	void *mem;
+	u8 encbuf[64], *kek;
+	size_t clrkeylen, keklen, encbuflen = sizeof(encbuf);
+
+	if (keybitsize == 128 || keybitsize == 192 || keybitsize == 256) {
+		clrkeylen = keybitsize / 8;
+	} else {
+		ZCRYPT_DBF_ERR("%s unknown/unsupported keybitsize %d\n",
+			       __func__, keybitsize);
+		return -EINVAL;
+	}
+
+	/*
+	 * Allocate space for the temp kek.
+	 * Also we only need up to MAXEP11AESKEYBLOBSIZE bytes for this
+	 * we use the already existing cprb mempool to solve this
+	 * short term memory requirement.
+	 */
+	mem = (xflags & ZCRYPT_XFLAG_NOMEMALLOC) ?
+		mempool_alloc_preallocated(cprb_mempool) :
+		mempool_alloc(cprb_mempool, GFP_KERNEL);
+	if (!mem)
+		return -ENOMEM;
+	kek = (u8 *)mem;
+	keklen = MAXEP11AESKEYBLOBSIZE;
+
+	/* Step 1: generate AES 256 bit random kek key */
+	rc = _ep11_genaeskey(card, domain, 256,
+			     0x00006c00, /* EN/DECRYPT, WRAP/UNWRAP */
+			     kek, &keklen, xflags);
+	if (rc) {
+		ZCRYPT_DBF_ERR("%s generate kek key failed, rc=%d\n",
+			       __func__, rc);
+		goto out;
+	}
+
+	/* Step 2: encrypt clear key value with the kek key */
+	rc = ep11_cryptsingle(card, domain, 0, 0, def_iv, kek, keklen,
+			      clrkey, clrkeylen, encbuf, &encbuflen, xflags);
+	if (rc) {
+		ZCRYPT_DBF_ERR("%s encrypting key value with kek key failed, rc=%d\n",
+			       __func__, rc);
+		goto out;
+	}
+
+	/* Step 3: import the encrypted key value as a new key */
+	rc = ep11_unwrapkey(card, domain, kek, keklen,
+			    encbuf, encbuflen, 0, def_iv,
+			    keybitsize, keygenflags,
+			    keybuf, keybufsize,
+			    keytype, xflags);
+	if (rc) {
+		ZCRYPT_DBF_ERR("%s importing key value as new key failed, rc=%d\n",
+			       __func__, rc);
+		goto out;
+	}
+
+out:
+	mempool_free(mem, cprb_mempool);
+	return rc;
+}
+EXPORT_SYMBOL(ep11_clr2keyblob);
+
+int ep11_kblob2protkey(u16 card, u16 dom,
+		       const u8 *keyblob, u32 keybloblen,
+		       u8 *protkey, u32 *protkeylen, u32 *protkeytype,
+		       u32 xflags)
+{
+	struct ep11kblob_header *hdr;
+	struct ep11keyblob *key;
+	size_t wkbuflen, keylen;
+	struct wk_info {
+		u16 version;
+		u8  res1[16];
+		u32 pkeytype;
+		u32 pkeybitsize;
+		u64 pkeysize;
+		u8  res2[8];
+		u8  pkey[];
+	} __packed * wki;
+	u8 *wkbuf = NULL;
+	int rc = -EIO;
+
+	if (ep11_kb_decode((u8 *)keyblob, keybloblen, &hdr, NULL, &key, &keylen))
+		return -EINVAL;
+
+	if (hdr->version == TOKVER_EP11_AES) {
+		/* wipe overlayed header */
+		memset(hdr, 0, sizeof(*hdr));
+	}
+	/* !!! hdr is no longer a valid header !!! */
+
+	/* need a temp working buffer */
+	wkbuflen = (keylen + AES_BLOCK_SIZE) & (~(AES_BLOCK_SIZE - 1));
+	if (wkbuflen > CPRB_MEMPOOL_ITEM_SIZE) {
+		/* this should never happen */
+		rc = -ENOMEM;
+		ZCRYPT_DBF_WARN("%s wkbuflen %d > cprb mempool item size %d, rc=%d\n",
+				__func__, (int)wkbuflen, CPRB_MEMPOOL_ITEM_SIZE, rc);
+		return rc;
+	}
+	/* use the cprb mempool to satisfy this short term mem allocation */
+	wkbuf = (xflags & ZCRYPT_XFLAG_NOMEMALLOC) ?
+		mempool_alloc_preallocated(cprb_mempool) :
+		mempool_alloc(cprb_mempool, GFP_ATOMIC);
+	if (!wkbuf) {
+		rc = -ENOMEM;
+		ZCRYPT_DBF_WARN("%s allocating tmp buffer via cprb mempool failed, rc=%d\n",
+				__func__, rc);
+		return rc;
+	}
+
+	/* ep11 secure key -> protected key + info */
+	rc = _ep11_wrapkey(card, dom, (u8 *)key, keylen,
+			   0, def_iv, wkbuf, &wkbuflen, xflags);
+	if (rc) {
+		ZCRYPT_DBF_ERR("%s rewrapping ep11 key to pkey failed, rc=%d\n",
+			       __func__, rc);
+		goto out;
+	}
+	wki = (struct wk_info *)wkbuf;
+
+	/* check struct version and pkey type */
+	if (wki->version != 1 || wki->pkeytype < 1 || wki->pkeytype > 5) {
+		ZCRYPT_DBF_ERR("%s wk info version %d or pkeytype %d mismatch.\n",
+			       __func__, (int)wki->version, (int)wki->pkeytype);
+		rc = -EIO;
+		goto out;
+	}
+
+	/* check protected key type field */
+	switch (wki->pkeytype) {
+	case 1: /* AES */
+		switch (wki->pkeysize) {
+		case 16 + 32:
+			/* AES 128 protected key */
+			if (protkeytype)
+				*protkeytype = PKEY_KEYTYPE_AES_128;
+			break;
+		case 24 + 32:
+			/* AES 192 protected key */
+			if (protkeytype)
+				*protkeytype = PKEY_KEYTYPE_AES_192;
+			break;
+		case 32 + 32:
+			/* AES 256 protected key */
+			if (protkeytype)
+				*protkeytype = PKEY_KEYTYPE_AES_256;
+			break;
+		default:
+			ZCRYPT_DBF_ERR("%s unknown/unsupported AES pkeysize %d\n",
+				       __func__, (int)wki->pkeysize);
+			rc = -EIO;
+			goto out;
+		}
+		break;
+	case 3: /* EC-P */
+	case 4: /* EC-ED */
+	case 5: /* EC-BP */
+		if (protkeytype)
+			*protkeytype = PKEY_KEYTYPE_ECC;
+		break;
+	case 2: /* TDES */
+	default:
+		ZCRYPT_DBF_ERR("%s unknown/unsupported key type %d\n",
+			       __func__, (int)wki->pkeytype);
+		rc = -EIO;
+		goto out;
+	}
+
+	/* copy the translated protected key */
+	if (wki->pkeysize > *protkeylen) {
+		ZCRYPT_DBF_ERR("%s wk info pkeysize %llu > protkeysize %u\n",
+			       __func__, wki->pkeysize, *protkeylen);
+		rc = -EINVAL;
+		goto out;
+	}
+	memcpy(protkey, wki->pkey, wki->pkeysize);
+	*protkeylen = wki->pkeysize;
+
+out:
+	mempool_free(wkbuf, cprb_mempool);
+	return rc;
+}
+EXPORT_SYMBOL(ep11_kblob2protkey);
+
+int ep11_findcard2(u32 *apqns, u32 *nr_apqns, u16 cardnr, u16 domain,
+		   int minhwtype, int minapi, const u8 *wkvp, u32 xflags)
+{
+	struct zcrypt_device_status_ext *device_status;
+	struct ep11_domain_info edi;
+	struct ep11_card_info eci;
+	u32 _nr_apqns = 0;
+	int i, card, dom;
+
+	/* occupy the device status memory */
+	mutex_lock(&dev_status_mem_mutex);
+	memset(dev_status_mem, 0, ZCRYPT_DEV_STATUS_EXT_SIZE);
+	device_status = (struct zcrypt_device_status_ext *)dev_status_mem;
+
+	/* fetch crypto device status into this struct */
+	zcrypt_device_status_mask_ext(device_status,
+				      ZCRYPT_DEV_STATUS_CARD_MAX,
+				      ZCRYPT_DEV_STATUS_QUEUE_MAX);
+
+	/* walk through all the crypto apqnss */
+	for (i = 0; i < ZCRYPT_DEV_STATUS_ENTRIES; i++) {
+		card = AP_QID_CARD(device_status[i].qid);
+		dom = AP_QID_QUEUE(device_status[i].qid);
+		/* check online state */
+		if (!device_status[i].online)
+			continue;
+		/* check for ep11 functions */
+		if (!(device_status[i].functions & 0x01))
+			continue;
+		/* check cardnr */
+		if (cardnr != 0xFFFF && card != cardnr)
+			continue;
+		/* check domain */
+		if (domain != 0xFFFF && dom != domain)
+			continue;
+		/* check min hardware type */
+		if (minhwtype && device_status[i].hwtype < minhwtype)
+			continue;
+		/* check min api version if given */
+		if (minapi > 0) {
+			if (ep11_get_card_info(card, &eci, xflags))
+				continue;
+			if (minapi > eci.API_ord_nr)
+				continue;
+		}
+		/* check wkvp if given */
+		if (wkvp) {
+			if (ep11_get_domain_info(card, dom, &edi, xflags))
+				continue;
+			if (edi.cur_wk_state != '1')
+				continue;
+			if (memcmp(wkvp, edi.cur_wkvp, 16))
+				continue;
+		}
+		/* apqn passed all filtering criterons, add to the array */
+		if (_nr_apqns < *nr_apqns)
+			apqns[_nr_apqns++] = (((u16)card) << 16) | ((u16)dom);
+	}
+
+	*nr_apqns = _nr_apqns;
+
+	mutex_unlock(&dev_status_mem_mutex);
+
+	return _nr_apqns ? 0 : -ENODEV;
+}
+EXPORT_SYMBOL(ep11_findcard2);
+
+int __init zcrypt_ep11misc_init(void)
+{
+	/* Pre-allocate a small memory pool for ep11 cprbs. */
+	cprb_mempool = mempool_create_kmalloc_pool(2 * zcrypt_mempool_threshold,
+						   CPRB_MEMPOOL_ITEM_SIZE);
+	if (!cprb_mempool)
+		return -ENOMEM;
+
+	/* Pre-allocate one crypto status card struct used in ep11_findcard2() */
+	dev_status_mem = kvmalloc(ZCRYPT_DEV_STATUS_EXT_SIZE, GFP_KERNEL);
+	if (!dev_status_mem) {
+		mempool_destroy(cprb_mempool);
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+void zcrypt_ep11misc_exit(void)
+{
+	mutex_lock(&dev_status_mem_mutex);
+	kvfree(dev_status_mem);
+	mutex_unlock(&dev_status_mem_mutex);
+	mempool_destroy(cprb_mempool);
+}
diff --git a/drivers/s390/crypto/zcrypt_ep11misc.h b/drivers/s390/crypto/zcrypt_ep11misc.h
new file mode 100644
index 000000000..05006817b
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_ep11misc.h
@@ -0,0 +1,158 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ *  Copyright IBM Corp. 2019
+ *  Author(s): Harald Freudenberger <freude@linux.ibm.com>
+ *
+ *  Collection of EP11 misc functions used by zcrypt and pkey
+ */
+
+#ifndef _ZCRYPT_EP11MISC_H_
+#define _ZCRYPT_EP11MISC_H_
+
+#include <asm/zcrypt.h>
+#include <asm/pkey.h>
+
+#define EP11_API_V1 1  /* min EP11 API, default if no higher api required */
+#define EP11_API_V4 4  /* supported EP11 API for the ep11misc cprbs */
+#define EP11_API_V6 6  /* min EP11 API for some cprbs in SE environment */
+#define EP11_STRUCT_MAGIC 0x1234
+#define EP11_BLOB_PKEY_EXTRACTABLE 0x00200000
+
+/*
+ * Internal used values for the version field of the key header.
+ * Should match to the enum pkey_key_type in pkey.h.
+ */
+#define TOKVER_EP11_AES  0x03  /* EP11 AES key blob (old style) */
+#define TOKVER_EP11_AES_WITH_HEADER 0x06 /* EP11 AES key blob with header */
+#define TOKVER_EP11_ECC_WITH_HEADER 0x07 /* EP11 ECC key blob with header */
+
+/* inside view of an EP11 secure key blob */
+struct ep11keyblob {
+	union {
+		u8 session[32];
+		/* only used for PKEY_TYPE_EP11: */
+		struct ep11kblob_header head;
+	};
+	u8  wkvp[16];  /* wrapping key verification pattern */
+	u64 attr;      /* boolean key attributes */
+	u64 mode;      /* mode bits */
+	u16 version;   /* 0x1234, EP11_STRUCT_MAGIC */
+	u8  iv[14];
+	u8  encrypted_key_data[144];
+	u8  mac[32];
+} __packed;
+
+/* check ep11 key magic to find out if this is an ep11 key blob */
+static inline bool is_ep11_keyblob(const u8 *key)
+{
+	struct ep11keyblob *kb = (struct ep11keyblob *)key;
+
+	return (kb->version == EP11_STRUCT_MAGIC);
+}
+
+/*
+ * For valid ep11 keyblobs, returns a reference to the wrappingkey verification
+ * pattern. Otherwise NULL.
+ */
+const u8 *ep11_kb_wkvp(const u8 *kblob, u32 kbloblen);
+
+/*
+ * Simple check if the key blob is a valid EP11 AES key blob with header.
+ * If checkcpacfexport is enabled, the key is also checked for the
+ * attributes needed to export this key for CPACF use.
+ * Returns 0 on success or errno value on failure.
+ */
+int ep11_check_aes_key_with_hdr(debug_info_t *dbg, int dbflvl,
+				const u8 *key, u32 keylen, int checkcpacfexp);
+
+/*
+ * Simple check if the key blob is a valid EP11 ECC key blob with header.
+ * If checkcpacfexport is enabled, the key is also checked for the
+ * attributes needed to export this key for CPACF use.
+ * Returns 0 on success or errno value on failure.
+ */
+int ep11_check_ecc_key_with_hdr(debug_info_t *dbg, int dbflvl,
+				const u8 *key, u32 keylen, int checkcpacfexp);
+
+/*
+ * Simple check if the key blob is a valid EP11 AES key blob with
+ * the header in the session field (old style EP11 AES key).
+ * If checkcpacfexport is enabled, the key is also checked for the
+ * attributes needed to export this key for CPACF use.
+ * Returns 0 on success or errno value on failure.
+ */
+int ep11_check_aes_key(debug_info_t *dbg, int dbflvl,
+		       const u8 *key, u32 keylen, int checkcpacfexp);
+
+/* EP11 card info struct */
+struct ep11_card_info {
+	u64  op_mode;	    /* card operational mode(s) */
+	char serial[16];    /* serial number string (16 ascii, no 0x00 !) */
+	u32  API_ord_nr;    /* API ordinal number */
+	u16  FW_version;    /* Firmware major and minor version */
+};
+
+/* EP11 domain info struct */
+struct ep11_domain_info {
+	u64  op_mode;	    /* domain operational mode(s) */
+	u8   cur_wkvp[32];  /* current wrapping key verification pattern */
+	u8   new_wkvp[32];  /* new wrapping key verification pattern */
+	char cur_wk_state;  /* '0' invalid, '1' valid */
+	char new_wk_state;  /* '0' empty, '1' uncommitted, '2' committed */
+};
+
+/*
+ * Provide information about an EP11 card.
+ */
+int ep11_get_card_info(u16 card, struct ep11_card_info *info, u32 xflags);
+
+/*
+ * Provide information about a domain within an EP11 card.
+ */
+int ep11_get_domain_info(u16 card, u16 domain,
+			 struct ep11_domain_info *info, u32 xflags);
+
+/*
+ * Generate (random) EP11 AES secure key.
+ */
+int ep11_genaeskey(u16 card, u16 domain, u32 keybitsize, u32 keygenflags,
+		   u8 *keybuf, u32 *keybufsize, u32 keybufver, u32 xflags);
+
+/*
+ * Generate EP11 AES secure key with given clear key value.
+ */
+int ep11_clr2keyblob(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags,
+		     const u8 *clrkey, u8 *keybuf, u32 *keybufsize,
+		     u32 keytype, u32 xflags);
+
+/*
+ * Build a list of ep11 apqns meeting the following constrains:
+ * - apqn is online and is in fact an EP11 apqn
+ * - if cardnr is not FFFF only apqns with this cardnr
+ * - if domain is not FFFF only apqns with this domainnr
+ * - if minhwtype > 0 only apqns with hwtype >= minhwtype
+ * - if minapi > 0 only apqns with API_ord_nr >= minapi
+ * - if wkvp != NULL only apqns where the wkvp (EP11_WKVPLEN bytes) matches
+ *   to the first EP11_WKVPLEN bytes of the wkvp of the current wrapping
+ *   key for this domain. When a wkvp is given there will always be a re-fetch
+ *   of the domain info for the potential apqn - so this triggers an request
+ *   reply to each apqn eligible.
+ * The caller should set *nr_apqns to the nr of elements available in *apqns.
+ * On return *nr_apqns is then updated with the nr of apqns filled into *apqns.
+ * The return value is either 0 for success or a negative errno value.
+ * If no apqn meeting the criteria is found, -ENODEV is returned.
+ */
+int ep11_findcard2(u32 *apqns, u32 *nr_apqns, u16 cardnr, u16 domain,
+		   int minhwtype, int minapi, const u8 *wkvp, u32 xflags);
+
+/*
+ * Derive proteced key from EP11 key blob (AES and ECC keys).
+ */
+int ep11_kblob2protkey(u16 card, u16 dom, const u8 *key, u32 keylen,
+		       u8 *protkey, u32 *protkeylen, u32 *protkeytype,
+		       u32 xflags);
+
+int zcrypt_ep11misc_init(void);
+void zcrypt_ep11misc_exit(void);
+
+#endif /* _ZCRYPT_EP11MISC_H_ */
diff --git a/drivers/s390/crypto/zcrypt_error.h b/drivers/s390/crypto/zcrypt_error.h
new file mode 100644
index 000000000..fd10302e6
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_error.h
@@ -0,0 +1,143 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ *  Copyright IBM Corp. 2001, 2006
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ */
+
+#ifndef _ZCRYPT_ERROR_H_
+#define _ZCRYPT_ERROR_H_
+
+#include <linux/atomic.h>
+#include "zcrypt_debug.h"
+#include "zcrypt_api.h"
+#include "zcrypt_msgtype6.h"
+
+/**
+ * Reply Messages
+ *
+ * Error reply messages are of two types:
+ *    82:  Error (see below)
+ *    88:  Error (see below)
+ * Both type 82 and type 88 have the same structure in the header.
+ *
+ * Request reply messages are of three known types:
+ *    80:  Reply from a Type 50 Request (see CEX2A-RELATED STRUCTS)
+ *    84:  Reply from a Type 4 Request (see PCICA-RELATED STRUCTS)
+ *    86:  Reply from a Type 6 Request (see PCICC/PCIXCC/CEX2C-RELATED STRUCTS)
+ *
+ */
+struct error_hdr {
+	unsigned char reserved1;	/* 0x00			*/
+	unsigned char type;		/* 0x82 or 0x88		*/
+	unsigned char reserved2[2];	/* 0x0000		*/
+	unsigned char reply_code;	/* reply code		*/
+	unsigned char reserved3[3];	/* 0x000000		*/
+};
+
+#define TYPE82_RSP_CODE 0x82
+#define TYPE88_RSP_CODE 0x88
+
+#define REP82_ERROR_MACHINE_FAILURE	    0x10
+#define REP82_ERROR_PREEMPT_FAILURE	    0x12
+#define REP82_ERROR_CHECKPT_FAILURE	    0x14
+#define REP82_ERROR_MESSAGE_TYPE	    0x20
+#define REP82_ERROR_INVALID_COMM_CD	    0x21 /* Type 84	*/
+#define REP82_ERROR_INVALID_MSG_LEN	    0x23
+#define REP82_ERROR_RESERVD_FIELD	    0x24 /* was 0x50	*/
+#define REP82_ERROR_FORMAT_FIELD	    0x29
+#define REP82_ERROR_INVALID_COMMAND	    0x30
+#define REP82_ERROR_MALFORMED_MSG	    0x40
+#define REP82_ERROR_INVALID_SPECIAL_CMD	    0x41
+#define REP82_ERROR_RESERVED_FIELDO	    0x50 /* old value	*/
+#define REP82_ERROR_WORD_ALIGNMENT	    0x60
+#define REP82_ERROR_MESSAGE_LENGTH	    0x80
+#define REP82_ERROR_OPERAND_INVALID	    0x82
+#define REP82_ERROR_OPERAND_SIZE	    0x84
+#define REP82_ERROR_EVEN_MOD_IN_OPND	    0x85
+#define REP82_ERROR_RESERVED_FIELD	    0x88
+#define REP82_ERROR_INVALID_DOMAIN_PENDING  0x8A
+#define REP82_ERROR_FILTERED_BY_HYPERVISOR  0x8B
+#define REP82_ERROR_TRANSPORT_FAIL	    0x90
+#define REP82_ERROR_PACKET_TRUNCATED	    0xA0
+#define REP82_ERROR_ZERO_BUFFER_LEN	    0xB0
+
+#define REP88_ERROR_MODULE_FAILURE	    0x10
+#define REP88_ERROR_MESSAGE_TYPE	    0x20
+#define REP88_ERROR_MESSAGE_MALFORMD	    0x22
+#define REP88_ERROR_MESSAGE_LENGTH	    0x23
+#define REP88_ERROR_RESERVED_FIELD	    0x24
+#define REP88_ERROR_KEY_TYPE		    0x34
+#define REP88_ERROR_INVALID_KEY	    0x82 /* CEX2A	*/
+#define REP88_ERROR_OPERAND		    0x84 /* CEX2A	*/
+#define REP88_ERROR_OPERAND_EVEN_MOD	    0x85 /* CEX2A	*/
+
+static inline int convert_error(struct zcrypt_queue *zq,
+				struct ap_message *reply)
+{
+	int queue = AP_QID_QUEUE(zq->queue->qid);
+	int card = AP_QID_CARD(zq->queue->qid);
+	struct error_hdr *ehdr = reply->msg;
+	struct {
+		struct type86_hdr hdr;
+		struct type86_fmt2_ext fmt2;
+	} __packed * t86hdr = reply->msg;
+
+	switch (ehdr->reply_code) {
+	case REP82_ERROR_INVALID_MSG_LEN:	 /* 0x23 */
+	case REP82_ERROR_RESERVD_FIELD:		 /* 0x24 */
+	case REP82_ERROR_FORMAT_FIELD:		 /* 0x29 */
+	case REP82_ERROR_MALFORMED_MSG:		 /* 0x40 */
+	case REP82_ERROR_INVALID_SPECIAL_CMD:	 /* 0x41 */
+	case REP82_ERROR_MESSAGE_LENGTH:	 /* 0x80 */
+	case REP82_ERROR_OPERAND_INVALID:	 /* 0x82 */
+	case REP82_ERROR_OPERAND_SIZE:		 /* 0x84 */
+	case REP82_ERROR_EVEN_MOD_IN_OPND:	 /* 0x85 */
+	case REP82_ERROR_INVALID_DOMAIN_PENDING: /* 0x8A */
+	case REP82_ERROR_FILTERED_BY_HYPERVISOR: /* 0x8B */
+	case REP82_ERROR_PACKET_TRUNCATED:	 /* 0xA0 */
+	case REP88_ERROR_MESSAGE_MALFORMD:	 /* 0x22 */
+	case REP88_ERROR_KEY_TYPE:		 /* 0x34 */
+		/* RY indicates malformed request */
+		if (ehdr->reply_code == REP82_ERROR_FILTERED_BY_HYPERVISOR &&
+		    ehdr->type == TYPE86_RSP_CODE) {
+			ZCRYPT_DBF_WARN("%s dev=%02x.%04x RY=0x%02x apfs=0x%x => rc=EINVAL\n",
+					__func__, card, queue,
+					ehdr->reply_code, t86hdr->fmt2.apfs);
+		} else {
+			ZCRYPT_DBF_WARN("%s dev=%02x.%04x RY=0x%02x => rc=EINVAL\n",
+					__func__, card, queue, ehdr->reply_code);
+		}
+		return -EINVAL;
+	case REP82_ERROR_MACHINE_FAILURE:	 /* 0x10 */
+	case REP82_ERROR_MESSAGE_TYPE:		 /* 0x20 */
+	case REP82_ERROR_TRANSPORT_FAIL:	 /* 0x90 */
+		/*
+		 * Msg to wrong type or card/infrastructure failure. Return
+		 * EAGAIN, the upper layer may do a retry on the request.
+		 */
+		/* For type 86 response show the apfs value (failure reason) */
+		if (ehdr->reply_code == REP82_ERROR_TRANSPORT_FAIL &&
+		    ehdr->type == TYPE86_RSP_CODE) {
+			ZCRYPT_DBF_WARN(
+				"%s dev=%02x.%04x RY=0x%02x apfs=0x%x => bus rescan, rc=EAGAIN\n",
+				__func__, card, queue, ehdr->reply_code,
+				t86hdr->fmt2.apfs);
+		} else {
+			ZCRYPT_DBF_WARN("%s dev=%02x.%04x RY=0x%02x => bus rescan, rc=EAGAIN\n",
+					__func__, card, queue,
+					ehdr->reply_code);
+		}
+		return -EAGAIN;
+	default:
+		/* Assume request is valid and a retry will be worth it */
+		ZCRYPT_DBF_WARN("%s dev=%02x.%04x RY=0x%02x => rc=EAGAIN\n",
+				__func__, card, queue, ehdr->reply_code);
+		return -EAGAIN;
+	}
+}
+
+#endif /* _ZCRYPT_ERROR_H_ */
diff --git a/drivers/s390/crypto/zcrypt_msgtype50.c b/drivers/s390/crypto/zcrypt_msgtype50.c
new file mode 100644
index 000000000..d6fc2d8e7
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_msgtype50.c
@@ -0,0 +1,562 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ *  Copyright IBM Corp. 2001, 2023
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *				  Ralph Wuerthner <rwuerthn@de.ibm.com>
+ *  MSGTYPE restruct:		  Holger Dengler <hd@linux.vnet.ibm.com>
+ */
+
+#define pr_fmt(fmt) "zcrypt: " fmt
+
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/init.h>
+#include <linux/err.h>
+#include <linux/atomic.h>
+#include <linux/uaccess.h>
+
+#include "ap_bus.h"
+#include "zcrypt_api.h"
+#include "zcrypt_error.h"
+#include "zcrypt_msgtype50.h"
+
+/* >= CEX3A: 4096 bits */
+#define CEX3A_MAX_MOD_SIZE 512
+
+/* >= CEX3A: 512 bit modulus, (max outputdatalength) + type80_hdr */
+#define CEX3A_MAX_RESPONSE_SIZE 0x210
+
+MODULE_AUTHOR("IBM Corporation");
+MODULE_DESCRIPTION("Cryptographic Accelerator (message type 50), " \
+		   "Copyright IBM Corp. 2001, 2023");
+MODULE_LICENSE("GPL");
+
+/*
+ * The type 50 message family is associated with a CEXxA cards.
+ *
+ * The four members of the family are described below.
+ *
+ * Note that all unsigned char arrays are right-justified and left-padded
+ * with zeroes.
+ *
+ * Note that all reserved fields must be zeroes.
+ */
+struct type50_hdr {
+	unsigned char	reserved1;
+	unsigned char	msg_type_code;	/* 0x50 */
+	unsigned short	msg_len;
+	unsigned char	reserved2;
+	unsigned char	ignored;
+	unsigned short	reserved3;
+} __packed;
+
+#define TYPE50_TYPE_CODE	0x50
+
+#define TYPE50_MEB1_FMT		0x0001
+#define TYPE50_MEB2_FMT		0x0002
+#define TYPE50_MEB3_FMT		0x0003
+#define TYPE50_CRB1_FMT		0x0011
+#define TYPE50_CRB2_FMT		0x0012
+#define TYPE50_CRB3_FMT		0x0013
+
+/* Mod-Exp, with a small modulus */
+struct type50_meb1_msg {
+	struct type50_hdr header;
+	unsigned short	keyblock_type;	/* 0x0001 */
+	unsigned char	reserved[6];
+	unsigned char	exponent[128];
+	unsigned char	modulus[128];
+	unsigned char	message[128];
+} __packed;
+
+/* Mod-Exp, with a large modulus */
+struct type50_meb2_msg {
+	struct type50_hdr header;
+	unsigned short	keyblock_type;	/* 0x0002 */
+	unsigned char	reserved[6];
+	unsigned char	exponent[256];
+	unsigned char	modulus[256];
+	unsigned char	message[256];
+} __packed;
+
+/* Mod-Exp, with a larger modulus */
+struct type50_meb3_msg {
+	struct type50_hdr header;
+	unsigned short	keyblock_type;	/* 0x0003 */
+	unsigned char	reserved[6];
+	unsigned char	exponent[512];
+	unsigned char	modulus[512];
+	unsigned char	message[512];
+} __packed;
+
+/* CRT, with a small modulus */
+struct type50_crb1_msg {
+	struct type50_hdr header;
+	unsigned short	keyblock_type;	/* 0x0011 */
+	unsigned char	reserved[6];
+	unsigned char	p[64];
+	unsigned char	q[64];
+	unsigned char	dp[64];
+	unsigned char	dq[64];
+	unsigned char	u[64];
+	unsigned char	message[128];
+} __packed;
+
+/* CRT, with a large modulus */
+struct type50_crb2_msg {
+	struct type50_hdr header;
+	unsigned short	keyblock_type;	/* 0x0012 */
+	unsigned char	reserved[6];
+	unsigned char	p[128];
+	unsigned char	q[128];
+	unsigned char	dp[128];
+	unsigned char	dq[128];
+	unsigned char	u[128];
+	unsigned char	message[256];
+} __packed;
+
+/* CRT, with a larger modulus */
+struct type50_crb3_msg {
+	struct type50_hdr header;
+	unsigned short	keyblock_type;	/* 0x0013 */
+	unsigned char	reserved[6];
+	unsigned char	p[256];
+	unsigned char	q[256];
+	unsigned char	dp[256];
+	unsigned char	dq[256];
+	unsigned char	u[256];
+	unsigned char	message[512];
+} __packed;
+
+/*
+ * The type 80 response family is associated with a CEXxA cards.
+ *
+ * Note that all unsigned char arrays are right-justified and left-padded
+ * with zeroes.
+ *
+ * Note that all reserved fields must be zeroes.
+ */
+
+#define TYPE80_RSP_CODE 0x80
+
+struct type80_hdr {
+	unsigned char	reserved1;
+	unsigned char	type;		/* 0x80 */
+	unsigned short	len;
+	unsigned char	code;		/* 0x00 */
+	unsigned char	reserved2[3];
+	unsigned char	reserved3[8];
+} __packed;
+
+int get_rsa_modex_fc(struct ica_rsa_modexpo *mex, int *fcode)
+{
+	if (!mex->inputdatalength)
+		return -EINVAL;
+
+	if (mex->inputdatalength <= 128)	/* 1024 bit */
+		*fcode = MEX_1K;
+	else if (mex->inputdatalength <= 256)	/* 2048 bit */
+		*fcode = MEX_2K;
+	else					/* 4096 bit */
+		*fcode = MEX_4K;
+
+	return 0;
+}
+
+int get_rsa_crt_fc(struct ica_rsa_modexpo_crt *crt, int *fcode)
+{
+	if (!crt->inputdatalength)
+		return -EINVAL;
+
+	if (crt->inputdatalength <= 128)	/* 1024 bit */
+		*fcode = CRT_1K;
+	else if (crt->inputdatalength <= 256)	/* 2048 bit */
+		*fcode = CRT_2K;
+	else					/* 4096 bit */
+		*fcode = CRT_4K;
+
+	return 0;
+}
+
+/*
+ * Convert a ICAMEX message to a type50 MEX message.
+ *
+ * @zq: crypto queue pointer
+ * @ap_msg: crypto request pointer
+ * @mex: pointer to user input data
+ *
+ * Returns 0 on success or -EFAULT.
+ */
+static int ICAMEX_msg_to_type50MEX_msg(struct zcrypt_queue *zq,
+				       struct ap_message *ap_msg,
+				       struct ica_rsa_modexpo *mex)
+{
+	unsigned char *mod, *exp, *inp;
+	int mod_len;
+
+	mod_len = mex->inputdatalength;
+
+	if (mod_len <= 128) {
+		struct type50_meb1_msg *meb1 = ap_msg->msg;
+
+		memset(meb1, 0, sizeof(*meb1));
+		ap_msg->len = sizeof(*meb1);
+		meb1->header.msg_type_code = TYPE50_TYPE_CODE;
+		meb1->header.msg_len = sizeof(*meb1);
+		meb1->keyblock_type = TYPE50_MEB1_FMT;
+		mod = meb1->modulus + sizeof(meb1->modulus) - mod_len;
+		exp = meb1->exponent + sizeof(meb1->exponent) - mod_len;
+		inp = meb1->message + sizeof(meb1->message) - mod_len;
+	} else if (mod_len <= 256) {
+		struct type50_meb2_msg *meb2 = ap_msg->msg;
+
+		memset(meb2, 0, sizeof(*meb2));
+		ap_msg->len = sizeof(*meb2);
+		meb2->header.msg_type_code = TYPE50_TYPE_CODE;
+		meb2->header.msg_len = sizeof(*meb2);
+		meb2->keyblock_type = TYPE50_MEB2_FMT;
+		mod = meb2->modulus + sizeof(meb2->modulus) - mod_len;
+		exp = meb2->exponent + sizeof(meb2->exponent) - mod_len;
+		inp = meb2->message + sizeof(meb2->message) - mod_len;
+	} else if (mod_len <= 512) {
+		struct type50_meb3_msg *meb3 = ap_msg->msg;
+
+		memset(meb3, 0, sizeof(*meb3));
+		ap_msg->len = sizeof(*meb3);
+		meb3->header.msg_type_code = TYPE50_TYPE_CODE;
+		meb3->header.msg_len = sizeof(*meb3);
+		meb3->keyblock_type = TYPE50_MEB3_FMT;
+		mod = meb3->modulus + sizeof(meb3->modulus) - mod_len;
+		exp = meb3->exponent + sizeof(meb3->exponent) - mod_len;
+		inp = meb3->message + sizeof(meb3->message) - mod_len;
+	} else {
+		return -EINVAL;
+	}
+
+	if (copy_from_user(mod, mex->n_modulus, mod_len) ||
+	    copy_from_user(exp, mex->b_key, mod_len) ||
+	    copy_from_user(inp, mex->inputdata, mod_len))
+		return -EFAULT;
+
+	return 0;
+}
+
+/*
+ * Convert a ICACRT message to a type50 CRT message.
+ *
+ * @zq: crypto queue pointer
+ * @ap_msg: crypto request pointer
+ * @crt: pointer to user input data
+ *
+ * Returns 0 on success or -EFAULT.
+ */
+static int ICACRT_msg_to_type50CRT_msg(struct zcrypt_queue *zq,
+				       struct ap_message *ap_msg,
+				       struct ica_rsa_modexpo_crt *crt)
+{
+	int mod_len, short_len;
+	unsigned char *p, *q, *dp, *dq, *u, *inp;
+
+	mod_len = crt->inputdatalength;
+	short_len = (mod_len + 1) / 2;
+
+	/*
+	 * CEX2A and CEX3A w/o FW update can handle requests up to
+	 * 256 byte modulus (2k keys).
+	 * CEX3A with FW update and newer CEXxA cards are able to handle
+	 * 512 byte modulus (4k keys).
+	 */
+	if (mod_len <= 128) {		/* up to 1024 bit key size */
+		struct type50_crb1_msg *crb1 = ap_msg->msg;
+
+		memset(crb1, 0, sizeof(*crb1));
+		ap_msg->len = sizeof(*crb1);
+		crb1->header.msg_type_code = TYPE50_TYPE_CODE;
+		crb1->header.msg_len = sizeof(*crb1);
+		crb1->keyblock_type = TYPE50_CRB1_FMT;
+		p = crb1->p + sizeof(crb1->p) - short_len;
+		q = crb1->q + sizeof(crb1->q) - short_len;
+		dp = crb1->dp + sizeof(crb1->dp) - short_len;
+		dq = crb1->dq + sizeof(crb1->dq) - short_len;
+		u = crb1->u + sizeof(crb1->u) - short_len;
+		inp = crb1->message + sizeof(crb1->message) - mod_len;
+	} else if (mod_len <= 256) {	/* up to 2048 bit key size */
+		struct type50_crb2_msg *crb2 = ap_msg->msg;
+
+		memset(crb2, 0, sizeof(*crb2));
+		ap_msg->len = sizeof(*crb2);
+		crb2->header.msg_type_code = TYPE50_TYPE_CODE;
+		crb2->header.msg_len = sizeof(*crb2);
+		crb2->keyblock_type = TYPE50_CRB2_FMT;
+		p = crb2->p + sizeof(crb2->p) - short_len;
+		q = crb2->q + sizeof(crb2->q) - short_len;
+		dp = crb2->dp + sizeof(crb2->dp) - short_len;
+		dq = crb2->dq + sizeof(crb2->dq) - short_len;
+		u = crb2->u + sizeof(crb2->u) - short_len;
+		inp = crb2->message + sizeof(crb2->message) - mod_len;
+	} else if ((mod_len <= 512) &&	/* up to 4096 bit key size */
+		   (zq->zcard->max_mod_size == CEX3A_MAX_MOD_SIZE)) {
+		struct type50_crb3_msg *crb3 = ap_msg->msg;
+
+		memset(crb3, 0, sizeof(*crb3));
+		ap_msg->len = sizeof(*crb3);
+		crb3->header.msg_type_code = TYPE50_TYPE_CODE;
+		crb3->header.msg_len = sizeof(*crb3);
+		crb3->keyblock_type = TYPE50_CRB3_FMT;
+		p = crb3->p + sizeof(crb3->p) - short_len;
+		q = crb3->q + sizeof(crb3->q) - short_len;
+		dp = crb3->dp + sizeof(crb3->dp) - short_len;
+		dq = crb3->dq + sizeof(crb3->dq) - short_len;
+		u = crb3->u + sizeof(crb3->u) - short_len;
+		inp = crb3->message + sizeof(crb3->message) - mod_len;
+	} else {
+		return -EINVAL;
+	}
+
+	/*
+	 * correct the offset of p, bp and mult_inv according zcrypt.h
+	 * block size right aligned (skip the first byte)
+	 */
+	if (copy_from_user(p, crt->np_prime + MSGTYPE_ADJUSTMENT, short_len) ||
+	    copy_from_user(q, crt->nq_prime, short_len) ||
+	    copy_from_user(dp, crt->bp_key + MSGTYPE_ADJUSTMENT, short_len) ||
+	    copy_from_user(dq, crt->bq_key, short_len) ||
+	    copy_from_user(u, crt->u_mult_inv + MSGTYPE_ADJUSTMENT, short_len) ||
+	    copy_from_user(inp, crt->inputdata, mod_len))
+		return -EFAULT;
+
+	return 0;
+}
+
+/*
+ * Copy results from a type 80 reply message back to user space.
+ *
+ * @zq: crypto device pointer
+ * @reply: reply AP message.
+ * @data: pointer to user output data
+ * @length: size of user output data
+ *
+ * Returns 0 on success or -EFAULT.
+ */
+static int convert_type80(struct zcrypt_queue *zq,
+			  struct ap_message *reply,
+			  char __user *outputdata,
+			  unsigned int outputdatalength)
+{
+	struct type80_hdr *t80h = reply->msg;
+	unsigned char *data;
+
+	if (t80h->len < sizeof(*t80h) + outputdatalength) {
+		/* The result is too short, the CEXxA card may not do that.. */
+		zq->online = 0;
+		pr_err("Crypto dev=%02x.%04x code=0x%02x => online=0 rc=EAGAIN\n",
+		       AP_QID_CARD(zq->queue->qid),
+		       AP_QID_QUEUE(zq->queue->qid), t80h->code);
+		ZCRYPT_DBF_ERR("%s dev=%02x.%04x code=0x%02x => online=0 rc=EAGAIN\n",
+			       __func__, AP_QID_CARD(zq->queue->qid),
+			       AP_QID_QUEUE(zq->queue->qid), t80h->code);
+		ap_send_online_uevent(&zq->queue->ap_dev, zq->online);
+		return -EAGAIN;
+	}
+	BUG_ON(t80h->len > CEX3A_MAX_RESPONSE_SIZE);
+	data = reply->msg + t80h->len - outputdatalength;
+	if (copy_to_user(outputdata, data, outputdatalength))
+		return -EFAULT;
+	return 0;
+}
+
+static int convert_response(struct zcrypt_queue *zq,
+			    struct ap_message *reply,
+			    char __user *outputdata,
+			    unsigned int outputdatalength)
+{
+	/* Response type byte is the second byte in the response. */
+	unsigned char rtype = ((unsigned char *)reply->msg)[1];
+
+	switch (rtype) {
+	case TYPE82_RSP_CODE:
+	case TYPE88_RSP_CODE:
+		return convert_error(zq, reply);
+	case TYPE80_RSP_CODE:
+		return convert_type80(zq, reply,
+				      outputdata, outputdatalength);
+	default: /* Unknown response type, this should NEVER EVER happen */
+		zq->online = 0;
+		pr_err("Crypto dev=%02x.%04x unknown response type 0x%02x => online=0 rc=EAGAIN\n",
+		       AP_QID_CARD(zq->queue->qid),
+		       AP_QID_QUEUE(zq->queue->qid),
+		       (int)rtype);
+		ZCRYPT_DBF_ERR(
+			"%s dev=%02x.%04x unknown response type 0x%02x => online=0 rc=EAGAIN\n",
+			__func__, AP_QID_CARD(zq->queue->qid),
+			AP_QID_QUEUE(zq->queue->qid), (int)rtype);
+		ap_send_online_uevent(&zq->queue->ap_dev, zq->online);
+		return -EAGAIN;
+	}
+}
+
+/*
+ * This function is called from the AP bus code after a crypto request
+ * "msg" has finished with the reply message "reply".
+ * It is called from tasklet context.
+ * @aq: pointer to the AP device
+ * @msg: pointer to the AP message
+ * @reply: pointer to the AP reply message
+ */
+static void zcrypt_msgtype50_receive(struct ap_queue *aq,
+				     struct ap_message *msg,
+				     struct ap_message *reply)
+{
+	static struct error_hdr error_reply = {
+		.type = TYPE82_RSP_CODE,
+		.reply_code = REP82_ERROR_MACHINE_FAILURE,
+	};
+	struct type80_hdr *t80h;
+	int len;
+
+	/* Copy the reply message to the request message buffer. */
+	if (!reply)
+		goto out;	/* ap_msg->rc indicates the error */
+	t80h = reply->msg;
+	if (t80h->type == TYPE80_RSP_CODE) {
+		len = t80h->len;
+		if (len > reply->bufsize || len > msg->bufsize ||
+		    len != reply->len) {
+			pr_debug("len mismatch => EMSGSIZE\n");
+			msg->rc = -EMSGSIZE;
+			goto out;
+		}
+		memcpy(msg->msg, reply->msg, len);
+		msg->len = len;
+	} else {
+		memcpy(msg->msg, reply->msg, sizeof(error_reply));
+		msg->len = sizeof(error_reply);
+	}
+out:
+	complete(&msg->response.work);
+}
+
+static atomic_t zcrypt_step = ATOMIC_INIT(0);
+
+/*
+ * The request distributor calls this function if it picked the CEXxA
+ * device to handle a modexpo request.
+ * @zq: pointer to zcrypt_queue structure that identifies the
+ *	CEXxA device to the request distributor
+ * @mex: pointer to the modexpo request buffer
+ * This function assumes that ap_msg has been initialized with
+ * ap_init_apmsg() and thus a valid buffer with the size of
+ * ap_msg->bufsize is available within ap_msg. Also the caller has
+ * to make sure ap_release_apmsg() is always called even on failure.
+ */
+static long zcrypt_msgtype50_modexpo(struct zcrypt_queue *zq,
+				     struct ica_rsa_modexpo *mex,
+				     struct ap_message *ap_msg)
+{
+	int rc;
+
+	if (ap_msg->bufsize < MSGTYPE50_CRB3_MAX_MSG_SIZE)
+		return -EMSGSIZE;
+	ap_msg->receive = zcrypt_msgtype50_receive;
+	ap_msg->psmid = (((unsigned long)current->pid) << 32) +
+		atomic_inc_return(&zcrypt_step);
+	rc = ICAMEX_msg_to_type50MEX_msg(zq, ap_msg, mex);
+	if (rc)
+		goto out;
+	init_completion(&ap_msg->response.work);
+	rc = ap_queue_message(zq->queue, ap_msg);
+	if (rc)
+		goto out;
+	rc = wait_for_completion_interruptible(&ap_msg->response.work);
+	if (rc == 0) {
+		rc = ap_msg->rc;
+		if (rc == 0)
+			rc = convert_response(zq, ap_msg,
+					      mex->outputdata,
+					      mex->outputdatalength);
+	} else {
+		/* Signal pending. */
+		ap_cancel_message(zq->queue, ap_msg);
+	}
+
+out:
+	if (rc)
+		pr_debug("send me cprb at dev=%02x.%04x rc=%d\n",
+			 AP_QID_CARD(zq->queue->qid),
+			 AP_QID_QUEUE(zq->queue->qid), rc);
+	return rc;
+}
+
+/*
+ * The request distributor calls this function if it picked the CEXxA
+ * device to handle a modexpo_crt request.
+ * @zq: pointer to zcrypt_queue structure that identifies the
+ *	CEXxA device to the request distributor
+ * @crt: pointer to the modexpoc_crt request buffer
+ * This function assumes that ap_msg has been initialized with
+ * ap_init_apmsg() and thus a valid buffer with the size of
+ * ap_msg->bufsize is available within ap_msg. Also the caller has
+ * to make sure ap_release_apmsg() is always called even on failure.
+ */
+static long zcrypt_msgtype50_modexpo_crt(struct zcrypt_queue *zq,
+					 struct ica_rsa_modexpo_crt *crt,
+					 struct ap_message *ap_msg)
+{
+	int rc;
+
+	if (ap_msg->bufsize < MSGTYPE50_CRB3_MAX_MSG_SIZE)
+		return -EMSGSIZE;
+	ap_msg->receive = zcrypt_msgtype50_receive;
+	ap_msg->psmid = (((unsigned long)current->pid) << 32) +
+		atomic_inc_return(&zcrypt_step);
+	rc = ICACRT_msg_to_type50CRT_msg(zq, ap_msg, crt);
+	if (rc)
+		goto out;
+	init_completion(&ap_msg->response.work);
+	rc = ap_queue_message(zq->queue, ap_msg);
+	if (rc)
+		goto out;
+	rc = wait_for_completion_interruptible(&ap_msg->response.work);
+	if (rc == 0) {
+		rc = ap_msg->rc;
+		if (rc == 0)
+			rc = convert_response(zq, ap_msg,
+					      crt->outputdata,
+					      crt->outputdatalength);
+	} else {
+		/* Signal pending. */
+		ap_cancel_message(zq->queue, ap_msg);
+	}
+
+out:
+	if (rc)
+		pr_debug("send crt cprb at dev=%02x.%04x rc=%d\n",
+			 AP_QID_CARD(zq->queue->qid),
+			 AP_QID_QUEUE(zq->queue->qid), rc);
+	return rc;
+}
+
+/*
+ * The crypto operations for message type 50.
+ */
+static struct zcrypt_ops zcrypt_msgtype50_ops = {
+	.rsa_modexpo = zcrypt_msgtype50_modexpo,
+	.rsa_modexpo_crt = zcrypt_msgtype50_modexpo_crt,
+	.owner = THIS_MODULE,
+	.name = MSGTYPE50_NAME,
+	.variant = MSGTYPE50_VARIANT_DEFAULT,
+};
+
+void __init zcrypt_msgtype50_init(void)
+{
+	zcrypt_msgtype_register(&zcrypt_msgtype50_ops);
+}
+
+void __exit zcrypt_msgtype50_exit(void)
+{
+	zcrypt_msgtype_unregister(&zcrypt_msgtype50_ops);
+}
diff --git a/drivers/s390/crypto/zcrypt_msgtype50.h b/drivers/s390/crypto/zcrypt_msgtype50.h
new file mode 100644
index 000000000..323e93b90
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_msgtype50.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ *  Copyright IBM Corp. 2001, 2023
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *  MSGTYPE restruct:		  Holger Dengler <hd@linux.vnet.ibm.com>
+ */
+
+#ifndef _ZCRYPT_MSGTYPE50_H_
+#define _ZCRYPT_MSGTYPE50_H_
+
+#define MSGTYPE50_NAME			"zcrypt_msgtype50"
+#define MSGTYPE50_VARIANT_DEFAULT	0
+
+#define MSGTYPE50_CRB3_MAX_MSG_SIZE 0x710 /* sizeof(struct type50_crb3_msg) */
+
+#define MSGTYPE_ADJUSTMENT 0x08  /* type04 extension (not needed in type50) */
+
+int get_rsa_modex_fc(struct ica_rsa_modexpo *mex, int *fc);
+int get_rsa_crt_fc(struct ica_rsa_modexpo_crt *crt, int *fc);
+
+void zcrypt_msgtype50_init(void);
+void zcrypt_msgtype50_exit(void);
+
+#endif /* _ZCRYPT_MSGTYPE50_H_ */
diff --git a/drivers/s390/crypto/zcrypt_msgtype6.c b/drivers/s390/crypto/zcrypt_msgtype6.c
new file mode 100644
index 000000000..3df1d676d
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_msgtype6.c
@@ -0,0 +1,1543 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ *  Copyright IBM Corp. 2001, 2023
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *				  Ralph Wuerthner <rwuerthn@de.ibm.com>
+ *  MSGTYPE restruct:		  Holger Dengler <hd@linux.vnet.ibm.com>
+ */
+
+#define pr_fmt(fmt) "zcrypt: " fmt
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/err.h>
+#include <linux/delay.h>
+#include <linux/slab.h>
+#include <linux/atomic.h>
+#include <linux/uaccess.h>
+#include <linux/unaligned.h>
+
+#include "ap_bus.h"
+#include "zcrypt_api.h"
+#include "zcrypt_error.h"
+#include "zcrypt_msgtype6.h"
+#include "zcrypt_cca_key.h"
+
+#define CEXXC_MAX_ICA_RESPONSE_SIZE 0x77c /* max size type86 v2 reply	    */
+
+#define CEIL4(x) ((((x) + 3) / 4) * 4)
+
+#define CEXXC_RESPONSE_TYPE_ICA  0
+#define CEXXC_RESPONSE_TYPE_XCRB 1
+#define CEXXC_RESPONSE_TYPE_EP11 2
+
+/* smallest possible EP11 payload size */
+#define MIN_EP11_PAYLOAD_SIZE 5
+
+MODULE_AUTHOR("IBM Corporation");
+MODULE_DESCRIPTION("Cryptographic Coprocessor (message type 6), " \
+		   "Copyright IBM Corp. 2001, 2023");
+MODULE_LICENSE("GPL");
+
+struct function_and_rules_block {
+	unsigned char function_code[2];
+	unsigned short ulen;
+	unsigned char only_rule[8];
+} __packed;
+
+/*
+ * The following is used to initialize the CPRBX passed to the CEXxC/CEXxP
+ * card in a type6 message. The 3 fields that must be filled in at execution
+ * time are  req_parml, rpl_parml and usage_domain.
+ * Everything about this interface is ascii/big-endian, since the
+ * device does *not* have 'Intel inside'.
+ *
+ * The CPRBX is followed immediately by the parm block.
+ * The parm block contains:
+ * - function code ('PD' 0x5044 or 'PK' 0x504B)
+ * - rule block (one of:)
+ *   + 0x000A 'PKCS-1.2' (MCL2 'PD')
+ *   + 0x000A 'ZERO-PAD' (MCL2 'PK')
+ *   + 0x000A 'ZERO-PAD' (MCL3 'PD' or CEX2C 'PD')
+ *   + 0x000A 'MRP     ' (MCL3 'PK' or CEX2C 'PK')
+ * - VUD block
+ */
+static const struct CPRBX static_cprbx = {
+	.cprb_len	=  0x00DC,
+	.cprb_ver_id	=  0x02,
+	.func_id	= {'T', '2'},
+};
+
+int speed_idx_cca(int req_type)
+{
+	switch (req_type) {
+	case 0x4142:
+	case 0x4149:
+	case 0x414D:
+	case 0x4341:
+	case 0x4344:
+	case 0x4354:
+	case 0x4358:
+	case 0x444B:
+	case 0x4558:
+	case 0x4643:
+	case 0x4651:
+	case 0x4C47:
+	case 0x4C4B:
+	case 0x4C51:
+	case 0x4F48:
+	case 0x504F:
+	case 0x5053:
+	case 0x5058:
+	case 0x5343:
+	case 0x5344:
+	case 0x5345:
+	case 0x5350:
+		return LOW;
+	case 0x414B:
+	case 0x4345:
+	case 0x4349:
+	case 0x434D:
+	case 0x4847:
+	case 0x4849:
+	case 0x484D:
+	case 0x4850:
+	case 0x4851:
+	case 0x4954:
+	case 0x4958:
+	case 0x4B43:
+	case 0x4B44:
+	case 0x4B45:
+	case 0x4B47:
+	case 0x4B48:
+	case 0x4B49:
+	case 0x4B4E:
+	case 0x4B50:
+	case 0x4B52:
+	case 0x4B54:
+	case 0x4B58:
+	case 0x4D50:
+	case 0x4D53:
+	case 0x4D56:
+	case 0x4D58:
+	case 0x5044:
+	case 0x5045:
+	case 0x5046:
+	case 0x5047:
+	case 0x5049:
+	case 0x504B:
+	case 0x504D:
+	case 0x5254:
+	case 0x5347:
+	case 0x5349:
+	case 0x534B:
+	case 0x534D:
+	case 0x5356:
+	case 0x5358:
+	case 0x5443:
+	case 0x544B:
+	case 0x5647:
+		return HIGH;
+	default:
+		return MEDIUM;
+	}
+}
+
+int speed_idx_ep11(int req_type)
+{
+	switch (req_type) {
+	case  1:
+	case  2:
+	case 36:
+	case 37:
+	case 38:
+	case 39:
+	case 40:
+		return LOW;
+	case 17:
+	case 18:
+	case 19:
+	case 20:
+	case 21:
+	case 22:
+	case 26:
+	case 30:
+	case 31:
+	case 32:
+	case 33:
+	case 34:
+	case 35:
+		return HIGH;
+	default:
+		return MEDIUM;
+	}
+}
+
+/*
+ * Convert a ICAMEX message to a type6 MEX message.
+ *
+ * @zq: crypto device pointer
+ * @ap_msg: pointer to AP message
+ * @mex: pointer to user input data
+ *
+ * Returns 0 on success or negative errno value.
+ */
+static int icamex_msg_to_type6mex_msgx(struct zcrypt_queue *zq,
+				       struct ap_message *ap_msg,
+				       struct ica_rsa_modexpo *mex)
+{
+	static struct type6_hdr static_type6_hdrX = {
+		.type		=  0x06,
+		.offset1	=  0x00000058,
+		.agent_id	= {'C', 'A',},
+		.function_code	= {'P', 'K'},
+	};
+	static struct function_and_rules_block static_pke_fnr = {
+		.function_code	= {'P', 'K'},
+		.ulen		= 10,
+		.only_rule	= {'M', 'R', 'P', ' ', ' ', ' ', ' ', ' '}
+	};
+	struct {
+		struct type6_hdr hdr;
+		struct CPRBX cprbx;
+		struct function_and_rules_block fr;
+		unsigned short length;
+		char text[];
+	} __packed * msg = ap_msg->msg;
+	int size;
+
+	/*
+	 * The inputdatalength was a selection criteria in the dispatching
+	 * function zcrypt_rsa_modexpo(). However, make sure the following
+	 * copy_from_user() never exceeds the allocated buffer space.
+	 */
+	if (WARN_ON_ONCE(mex->inputdatalength > PAGE_SIZE))
+		return -EINVAL;
+
+	/* VUD.ciphertext */
+	msg->length = mex->inputdatalength + 2;
+	if (copy_from_user(msg->text, mex->inputdata, mex->inputdatalength))
+		return -EFAULT;
+
+	/* Set up key which is located after the variable length text. */
+	size = zcrypt_type6_mex_key_en(mex, msg->text + mex->inputdatalength);
+	if (size < 0)
+		return size;
+	size += sizeof(*msg) + mex->inputdatalength;
+
+	/* message header, cprbx and f&r */
+	msg->hdr = static_type6_hdrX;
+	msg->hdr.tocardlen1 = size - sizeof(msg->hdr);
+	msg->hdr.fromcardlen1 = CEXXC_MAX_ICA_RESPONSE_SIZE - sizeof(msg->hdr);
+
+	msg->cprbx = static_cprbx;
+	msg->cprbx.domain = AP_QID_QUEUE(zq->queue->qid);
+	msg->cprbx.rpl_msgbl = msg->hdr.fromcardlen1;
+
+	msg->fr = static_pke_fnr;
+
+	msg->cprbx.req_parml = size - sizeof(msg->hdr) - sizeof(msg->cprbx);
+
+	ap_msg->len = size;
+	return 0;
+}
+
+/*
+ * Convert a ICACRT message to a type6 CRT message.
+ *
+ * @zq: crypto device pointer
+ * @ap_msg: pointer to AP message
+ * @crt: pointer to user input data
+ *
+ * Returns 0 on success or negative errno value.
+ */
+static int icacrt_msg_to_type6crt_msgx(struct zcrypt_queue *zq,
+				       struct ap_message *ap_msg,
+				       struct ica_rsa_modexpo_crt *crt)
+{
+	static struct type6_hdr static_type6_hdrX = {
+		.type		=  0x06,
+		.offset1	=  0x00000058,
+		.agent_id	= {'C', 'A',},
+		.function_code	= {'P', 'D'},
+	};
+	static struct function_and_rules_block static_pkd_fnr = {
+		.function_code	= {'P', 'D'},
+		.ulen		= 10,
+		.only_rule	= {'Z', 'E', 'R', 'O', '-', 'P', 'A', 'D'}
+	};
+
+	struct {
+		struct type6_hdr hdr;
+		struct CPRBX cprbx;
+		struct function_and_rules_block fr;
+		unsigned short length;
+		char text[];
+	} __packed * msg = ap_msg->msg;
+	int size;
+
+	/*
+	 * The inputdatalength was a selection criteria in the dispatching
+	 * function zcrypt_rsa_crt(). However, make sure the following
+	 * copy_from_user() never exceeds the allocated buffer space.
+	 */
+	if (WARN_ON_ONCE(crt->inputdatalength > PAGE_SIZE))
+		return -EINVAL;
+
+	/* VUD.ciphertext */
+	msg->length = crt->inputdatalength + 2;
+	if (copy_from_user(msg->text, crt->inputdata, crt->inputdatalength))
+		return -EFAULT;
+
+	/* Set up key which is located after the variable length text. */
+	size = zcrypt_type6_crt_key(crt, msg->text + crt->inputdatalength);
+	if (size < 0)
+		return size;
+	size += sizeof(*msg) + crt->inputdatalength;	/* total size of msg */
+
+	/* message header, cprbx and f&r */
+	msg->hdr = static_type6_hdrX;
+	msg->hdr.tocardlen1 = size -  sizeof(msg->hdr);
+	msg->hdr.fromcardlen1 = CEXXC_MAX_ICA_RESPONSE_SIZE - sizeof(msg->hdr);
+
+	msg->cprbx = static_cprbx;
+	msg->cprbx.domain = AP_QID_QUEUE(zq->queue->qid);
+	msg->cprbx.req_parml = msg->cprbx.rpl_msgbl =
+		size - sizeof(msg->hdr) - sizeof(msg->cprbx);
+
+	msg->fr = static_pkd_fnr;
+
+	ap_msg->len = size;
+	return 0;
+}
+
+/*
+ * Convert a XCRB message to a type6 CPRB message.
+ *
+ * @zq: crypto device pointer
+ * @ap_msg: pointer to AP message
+ * @xcRB: pointer to user input data
+ *
+ * Returns 0 on success or -EFAULT, -EINVAL.
+ */
+struct type86_fmt2_msg {
+	struct type86_hdr hdr;
+	struct type86_fmt2_ext fmt2;
+} __packed;
+
+static int xcrb_msg_to_type6cprb_msgx(bool userspace, struct ap_message *ap_msg,
+				      struct ica_xcRB *xcrb,
+				      unsigned int *fcode,
+				      unsigned int *domain)
+{
+	static struct type6_hdr static_type6_hdrX = {
+		.type		=  0x06,
+		.offset1	=  0x00000058,
+	};
+	struct {
+		struct type6_hdr hdr;
+		union {
+			struct CPRBX cprbx;
+			DECLARE_FLEX_ARRAY(u8, userdata);
+		};
+	} __packed * msg = ap_msg->msg;
+
+	size_t req_cblen, rep_cblen, req_sumlen, rep_sumlen;
+	char *function_code, *req_data;
+
+	/* request length and overflow checks */
+	if (xcrb->request_control_blk_length < sizeof(struct CPRBX))
+		return -EINVAL;
+	req_cblen = CEIL4((size_t)xcrb->request_control_blk_length);
+	if (req_cblen > U32_MAX)
+		return -EINVAL;
+	req_sumlen = req_cblen + xcrb->request_data_length;
+	if (req_sumlen > U32_MAX)
+		return -EINVAL;
+	ap_msg->len = sizeof(struct type6_hdr) + req_sumlen;
+	if (ap_msg->len > ap_msg->bufsize)
+		return -EINVAL;
+	if (req_cblen <= xcrb->request_data_length ?
+	    req_sumlen < xcrb->request_data_length :
+	    req_sumlen < req_cblen) {
+		return -EINVAL;
+	}
+
+	/* reply length and overflow checks */
+	if (xcrb->reply_control_blk_length < sizeof(struct CPRBX))
+		return -EINVAL;
+	rep_cblen = CEIL4((size_t)xcrb->reply_control_blk_length);
+	if (rep_cblen > U32_MAX)
+		return -EINVAL;
+	rep_sumlen = rep_cblen + xcrb->reply_data_length;
+	if (rep_sumlen > U32_MAX)
+		return -EINVAL;
+	if (rep_cblen <= xcrb->reply_data_length ?
+	    rep_sumlen < xcrb->reply_data_length :
+	    rep_sumlen < rep_cblen) {
+		return -EINVAL;
+	}
+
+	/* prepare type6 header */
+	msg->hdr = static_type6_hdrX;
+	memcpy(msg->hdr.agent_id, &xcrb->agent_ID, sizeof(xcrb->agent_ID));
+	msg->hdr.tocardlen1 = xcrb->request_control_blk_length;
+	if (xcrb->request_data_length) {
+		msg->hdr.offset2 = msg->hdr.offset1 + req_cblen;
+		msg->hdr.tocardlen2 = xcrb->request_data_length;
+	}
+	msg->hdr.fromcardlen1 = xcrb->reply_control_blk_length;
+	msg->hdr.fromcardlen2 = xcrb->reply_data_length;
+
+	/* prepare CPRB */
+	if (z_copy_from_user(userspace, msg->userdata,
+			     xcrb->request_control_blk_addr,
+			     xcrb->request_control_blk_length))
+		return -EFAULT;
+	/* pad tail with 0 up to req_cblen */
+	if (xcrb->request_control_blk_length < req_cblen)
+		memset(msg->userdata + xcrb->request_control_blk_length,
+		       0, req_cblen - xcrb->request_control_blk_length);
+	/* copy subfunction code into AP msg type 6 function code field */
+	if (msg->cprbx.cprb_len > req_cblen - sizeof(msg->hdr.function_code))
+		return -EINVAL;
+	function_code = ((unsigned char *)&msg->cprbx) + msg->cprbx.cprb_len;
+	memcpy(msg->hdr.function_code, function_code,
+	       sizeof(msg->hdr.function_code));
+
+	*fcode = (msg->hdr.function_code[0] << 8) | msg->hdr.function_code[1];
+	if (domain)
+		*domain = msg->cprbx.domain;
+
+	/* check subfunction, US and AU need special flag with NQAP */
+	if (memcmp(function_code, "US", 2) == 0 ||
+	    memcmp(function_code, "AU", 2) == 0)
+		ap_msg->flags |= AP_MSG_FLAG_SPECIAL;
+
+	/* check CPRB minor version, set info bits in ap_message flag field */
+	switch (*(unsigned short *)(&msg->cprbx.func_id[0])) {
+	case 0x5432: /* "T2" */
+		ap_msg->flags |= AP_MSG_FLAG_USAGE;
+		break;
+	case 0x5433: /* "T3" */
+	case 0x5435: /* "T5" */
+	case 0x5436: /* "T6" */
+	case 0x5437: /* "T7" */
+		ap_msg->flags |= AP_MSG_FLAG_ADMIN;
+		break;
+	default:
+		pr_debug("unknown CPRB minor version '%c%c'\n",
+			 msg->cprbx.func_id[0], msg->cprbx.func_id[1]);
+	}
+
+	/* copy data block */
+	if (xcrb->request_data_length) {
+		req_data = ap_msg->msg + sizeof(struct type6_hdr) + req_cblen;
+		if (z_copy_from_user(userspace, req_data,
+				     xcrb->request_data_address,
+				     xcrb->request_data_length))
+			return -EFAULT;
+	}
+
+	return 0;
+}
+
+/*
+ * Simple asn1 int reader/decoder helper function
+ * Returns number of bytes processed or < 0 on failure
+ * Only accepts int length values of 1, 2 or 4.
+ */
+static inline int asn1_int_decode(const u8 *buf, size_t intlen, u32 *u)
+{
+	switch (intlen) {
+	case 1:
+		*u = (u32)(*buf);
+		return 1;
+	case 2:
+		*u = (u32)get_unaligned_be16(buf);
+		return 2;
+	case 4:
+		*u = (u32)get_unaligned_be32(buf);
+		return 4;
+	default:
+		return -EINVAL;
+	}
+}
+
+/*
+ * Simple asn1 length parse helper function
+ * Returns number of bytes processed or < 0 on failure
+ * Only accepts length encoded within the length octet
+ * or for long form 1, 2 or 4 octet length bytes.
+ */
+static inline int asn1_length_decode(const u8 *buf, size_t buflen, u32 *u)
+{
+	int i;
+
+	if (buflen < 1)
+		return -EINVAL;
+
+	if (*buf < 128) {
+		*u = (u32)(*buf & 0x7F);
+		return 1;
+	}
+
+	i = *buf & 0x7F;
+	if (--buflen < i)
+		return -EINVAL;
+	i = asn1_int_decode(++buf, i, u);
+
+	return i < 0 ? i : i + 1;
+}
+
+static int xcrb_msg_to_type6_ep11cprb_msgx(bool userspace, struct ap_message *ap_msg,
+					   struct ep11_urb *xcrb,
+					   unsigned int *fcode,
+					   unsigned int *domain)
+{
+	static struct type6_hdr static_type6_ep11_hdr = {
+		.type		=  0x06,
+		.rqid		= {0x00, 0x01},
+		.function_code	= {0x00, 0x00},
+		.agent_id	= {'X', 'C'},
+		.offset1	=  0x00000058,
+	};
+
+	struct {
+		struct type6_hdr hdr;
+		union {
+			struct ep11_cprb cprbx;
+			DECLARE_FLEX_ARRAY(u8, userdata);
+		};
+	} __packed * msg = ap_msg->msg;
+
+	size_t req_len, rep_len, pld_len;
+	unsigned char *pld;
+	int offs = 0, i;
+	unsigned int u;
+
+	/* request length and overflow checks */
+	if (xcrb->req_len < sizeof(struct ep11_cprb) + MIN_EP11_PAYLOAD_SIZE)
+		return -EINVAL;
+	req_len = CEIL4(xcrb->req_len);
+	if (req_len < xcrb->req_len || req_len > U32_MAX)
+		return -EINVAL;
+	ap_msg->len = sizeof(struct type6_hdr) + req_len;
+	if (ap_msg->len > ap_msg->bufsize)
+		return -EINVAL;
+
+	/* reply length and overflow checks */
+	if (xcrb->resp_len < sizeof(struct ep11_cprb))
+		return -EINVAL;
+	rep_len = CEIL4(xcrb->resp_len);
+	if (rep_len < xcrb->resp_len || rep_len > U32_MAX)
+		return -EINVAL;
+
+	/* prepare type6 header */
+	msg->hdr = static_type6_ep11_hdr;
+	msg->hdr.tocardlen1   = xcrb->req_len;
+	msg->hdr.fromcardlen1 = xcrb->resp_len;
+
+	/* Import CPRB data from the ioctl input parameter */
+	if (z_copy_from_user(userspace, msg->userdata,
+			     (char __force __user *)xcrb->req, xcrb->req_len))
+		return -EFAULT;
+	/* pad tail with 0 up to req_len */
+	if (xcrb->req_len < req_len)
+		memset(msg->userdata + xcrb->req_len, 0,
+		       req_len - xcrb->req_len);
+
+	pld = msg->userdata + sizeof(struct ep11_cprb);
+	pld_len = msg->cprbx.payload_len;
+	if (pld_len != xcrb->req_len - sizeof(struct ep11_cprb))
+		return -EINVAL;
+	/*
+	 * At this point pld_len is always >= MIN_EP11_PAYLOAD_SIZE
+	 * and the smallest supported asn1 payload is:
+	 *   payload tag (1 octet)
+	 *   payload length (1-5 octets)
+	 *   function tag (1 octet)
+	 *   function length (1-5 octets)
+	 *   function value (1-4 octets)
+	 */
+
+	/* payload tag */
+	if (pld[offs++] != 0x30)
+		return -EINVAL;
+	/* payload length field */
+	i = asn1_length_decode(pld + offs, pld_len - offs, &u);
+	if (i < 0)
+		return -EINVAL;
+	offs += i;
+	if (offs >= pld_len || u > pld_len - offs)
+		return -EINVAL;
+	/* function tag */
+	if (pld[offs++] != 0x04)
+		return -EINVAL;
+	/* function length */
+	if (offs >= pld_len)
+		return -EINVAL;
+	i = asn1_length_decode(pld + offs, pld_len - offs, &u);
+	if (i < 0)
+		return -EINVAL;
+	offs += i;
+	if (offs >= pld_len || u > pld_len - offs)
+		return -EINVAL;
+	/* function value */
+	i = asn1_int_decode(pld + offs, u, &u);
+	if (i < 0)
+		return -EINVAL;
+	offs += i;
+	*fcode = 0xFFFF & u;
+
+	/* enable special processing based on the cprbs flags special bit */
+	if (msg->cprbx.flags & 0x20)
+		ap_msg->flags |= AP_MSG_FLAG_SPECIAL;
+
+	/* set info bits in ap_message flag field */
+	if (msg->cprbx.flags & 0x80)
+		ap_msg->flags |= AP_MSG_FLAG_ADMIN;
+	else
+		ap_msg->flags |= AP_MSG_FLAG_USAGE;
+
+	if (domain)
+		*domain = msg->cprbx.target_id;
+
+	return 0;
+}
+
+/*
+ * Copy results from a type 86 ICA reply message back to user space.
+ *
+ * @zq: crypto device pointer
+ * @reply: reply AP message.
+ * @data: pointer to user output data
+ * @length: size of user output data
+ *
+ * Returns 0 on success or -EINVAL, -EFAULT, -EAGAIN in case of an error.
+ */
+struct type86x_reply {
+	struct type86_hdr hdr;
+	struct type86_fmt2_ext fmt2;
+	struct CPRBX cprbx;
+	unsigned char pad[4];	/* 4 byte function code/rules block ? */
+	unsigned short length;	/* length of data including length field size */
+	char data[];
+} __packed;
+
+struct type86_ep11_reply {
+	struct type86_hdr hdr;
+	struct type86_fmt2_ext fmt2;
+	struct ep11_cprb cprbx;
+} __packed;
+
+static int convert_type86_ica(struct zcrypt_queue *zq,
+			      struct ap_message *reply,
+			      char __user *outputdata,
+			      unsigned int outputdatalength)
+{
+	struct type86x_reply *msg = reply->msg;
+	unsigned short service_rc, service_rs;
+	unsigned int data_len;
+
+	service_rc = msg->cprbx.ccp_rtcode;
+	if (unlikely(service_rc != 0)) {
+		service_rs = msg->cprbx.ccp_rscode;
+		if ((service_rc == 8 && service_rs == 66) ||
+		    (service_rc == 8 && service_rs == 65) ||
+		    (service_rc == 8 && service_rs == 72) ||
+		    (service_rc == 8 && service_rs == 770) ||
+		    (service_rc == 12 && service_rs == 769)) {
+			ZCRYPT_DBF_WARN("%s dev=%02x.%04x rc/rs=%d/%d => rc=EINVAL\n",
+					__func__, AP_QID_CARD(zq->queue->qid),
+					AP_QID_QUEUE(zq->queue->qid),
+					(int)service_rc, (int)service_rs);
+			return -EINVAL;
+		}
+		zq->online = 0;
+		pr_err("Crypto dev=%02x.%04x rc/rs=%d/%d online=0 rc=EAGAIN\n",
+		       AP_QID_CARD(zq->queue->qid),
+		       AP_QID_QUEUE(zq->queue->qid),
+		       (int)service_rc, (int)service_rs);
+		ZCRYPT_DBF_ERR("%s dev=%02x.%04x rc/rs=%d/%d => online=0 rc=EAGAIN\n",
+			       __func__, AP_QID_CARD(zq->queue->qid),
+			       AP_QID_QUEUE(zq->queue->qid),
+			       (int)service_rc, (int)service_rs);
+		ap_send_online_uevent(&zq->queue->ap_dev, zq->online);
+		return -EAGAIN;
+	}
+	data_len = msg->length - sizeof(msg->length);
+	if (data_len > outputdatalength)
+		return -EMSGSIZE;
+
+	/* Copy the crypto response to user space. */
+	if (copy_to_user(outputdata, msg->data, data_len))
+		return -EFAULT;
+	return 0;
+}
+
+/*
+ * Copy results from a type 86 XCRB reply message back to user space.
+ *
+ * @zq: crypto device pointer
+ * @reply: reply AP message.
+ * @xcrb: pointer to XCRB
+ *
+ * Returns 0 on success or -EINVAL, -EFAULT, -EAGAIN in case of an error.
+ */
+static int convert_type86_xcrb(bool userspace, struct zcrypt_queue *zq,
+			       struct ap_message *reply,
+			       struct ica_xcRB *xcrb)
+{
+	struct type86_fmt2_msg *msg = reply->msg;
+	char *data = reply->msg;
+
+	/* Copy CPRB to user */
+	if (xcrb->reply_control_blk_length < msg->fmt2.count1) {
+		pr_debug("reply_control_blk_length %u < required %u => EMSGSIZE\n",
+			 xcrb->reply_control_blk_length, msg->fmt2.count1);
+		return -EMSGSIZE;
+	}
+	if (z_copy_to_user(userspace, xcrb->reply_control_blk_addr,
+			   data + msg->fmt2.offset1, msg->fmt2.count1))
+		return -EFAULT;
+	xcrb->reply_control_blk_length = msg->fmt2.count1;
+
+	/* Copy data buffer to user */
+	if (msg->fmt2.count2) {
+		if (xcrb->reply_data_length < msg->fmt2.count2) {
+			pr_debug("reply_data_length %u < required %u => EMSGSIZE\n",
+				 xcrb->reply_data_length, msg->fmt2.count2);
+			return -EMSGSIZE;
+		}
+		if (z_copy_to_user(userspace, xcrb->reply_data_addr,
+				   data + msg->fmt2.offset2, msg->fmt2.count2))
+			return -EFAULT;
+	}
+	xcrb->reply_data_length = msg->fmt2.count2;
+
+	return 0;
+}
+
+/*
+ * Copy results from a type 86 EP11 XCRB reply message back to user space.
+ *
+ * @zq: crypto device pointer
+ * @reply: reply AP message.
+ * @xcrb: pointer to EP11 user request block
+ *
+ * Returns 0 on success or -EINVAL, -EFAULT, -EAGAIN in case of an error.
+ */
+static int convert_type86_ep11_xcrb(bool userspace, struct zcrypt_queue *zq,
+				    struct ap_message *reply,
+				    struct ep11_urb *xcrb)
+{
+	struct type86_fmt2_msg *msg = reply->msg;
+	char *data = reply->msg;
+
+	if (xcrb->resp_len < msg->fmt2.count1) {
+		pr_debug("resp_len %u < required %u => EMSGSIZE\n",
+			 (unsigned int)xcrb->resp_len, msg->fmt2.count1);
+		return -EMSGSIZE;
+	}
+
+	/* Copy response CPRB to user */
+	if (z_copy_to_user(userspace, (char __force __user *)xcrb->resp,
+			   data + msg->fmt2.offset1, msg->fmt2.count1))
+		return -EFAULT;
+	xcrb->resp_len = msg->fmt2.count1;
+	return 0;
+}
+
+static int convert_type86_rng(struct zcrypt_queue *zq,
+			      struct ap_message *reply,
+			      char *buffer)
+{
+	struct {
+		struct type86_hdr hdr;
+		struct type86_fmt2_ext fmt2;
+		struct CPRBX cprbx;
+	} __packed * msg = reply->msg;
+	char *data = reply->msg;
+
+	if (msg->cprbx.ccp_rtcode != 0 || msg->cprbx.ccp_rscode != 0)
+		return -EINVAL;
+	memcpy(buffer, data + msg->fmt2.offset2, msg->fmt2.count2);
+	return msg->fmt2.count2;
+}
+
+static int convert_response_ica(struct zcrypt_queue *zq,
+				struct ap_message *reply,
+				char __user *outputdata,
+				unsigned int outputdatalength)
+{
+	struct type86x_reply *msg = reply->msg;
+
+	switch (msg->hdr.type) {
+	case TYPE82_RSP_CODE:
+	case TYPE88_RSP_CODE:
+		return convert_error(zq, reply);
+	case TYPE86_RSP_CODE:
+		if (msg->cprbx.ccp_rtcode &&
+		    msg->cprbx.ccp_rscode == 0x14f &&
+		    outputdatalength > 256) {
+			if (zq->zcard->max_exp_bit_length <= 17) {
+				zq->zcard->max_exp_bit_length = 17;
+				return -EAGAIN;
+			} else {
+				return -EINVAL;
+			}
+		}
+		if (msg->hdr.reply_code)
+			return convert_error(zq, reply);
+		if (msg->cprbx.cprb_ver_id == 0x02)
+			return convert_type86_ica(zq, reply,
+						  outputdata, outputdatalength);
+		fallthrough;	/* wrong cprb version is an unknown response */
+	default:
+		/* Unknown response type, this should NEVER EVER happen */
+		zq->online = 0;
+		pr_err("Crypto dev=%02x.%04x unknown response type 0x%02x => online=0 rc=EAGAIN\n",
+		       AP_QID_CARD(zq->queue->qid),
+		       AP_QID_QUEUE(zq->queue->qid),
+		       (int)msg->hdr.type);
+		ZCRYPT_DBF_ERR(
+			"%s dev=%02x.%04x unknown response type 0x%02x => online=0 rc=EAGAIN\n",
+			__func__, AP_QID_CARD(zq->queue->qid),
+			AP_QID_QUEUE(zq->queue->qid), (int)msg->hdr.type);
+		ap_send_online_uevent(&zq->queue->ap_dev, zq->online);
+		return -EAGAIN;
+	}
+}
+
+static int convert_response_xcrb(bool userspace, struct zcrypt_queue *zq,
+				 struct ap_message *reply,
+				 struct ica_xcRB *xcrb)
+{
+	struct type86x_reply *msg = reply->msg;
+
+	switch (msg->hdr.type) {
+	case TYPE82_RSP_CODE:
+	case TYPE88_RSP_CODE:
+		xcrb->status = 0x0008044DL; /* HDD_InvalidParm */
+		return convert_error(zq, reply);
+	case TYPE86_RSP_CODE:
+		if (msg->hdr.reply_code) {
+			xcrb->status = msg->fmt2.apfs;
+			return convert_error(zq, reply);
+		}
+		if (msg->cprbx.cprb_ver_id == 0x02)
+			return convert_type86_xcrb(userspace, zq, reply, xcrb);
+		fallthrough;	/* wrong cprb version is an unknown response */
+	default: /* Unknown response type, this should NEVER EVER happen */
+		xcrb->status = 0x0008044DL; /* HDD_InvalidParm */
+		zq->online = 0;
+		pr_err("Crypto dev=%02x.%04x unknown response type 0x%02x => online=0 rc=EAGAIN\n",
+		       AP_QID_CARD(zq->queue->qid),
+		       AP_QID_QUEUE(zq->queue->qid),
+		       (int)msg->hdr.type);
+		ZCRYPT_DBF_ERR(
+			"%s dev=%02x.%04x unknown response type 0x%02x => online=0 rc=EAGAIN\n",
+			__func__, AP_QID_CARD(zq->queue->qid),
+			AP_QID_QUEUE(zq->queue->qid), (int)msg->hdr.type);
+		ap_send_online_uevent(&zq->queue->ap_dev, zq->online);
+		return -EAGAIN;
+	}
+}
+
+static int convert_response_ep11_xcrb(bool userspace, struct zcrypt_queue *zq,
+				      struct ap_message *reply, struct ep11_urb *xcrb)
+{
+	struct type86_ep11_reply *msg = reply->msg;
+
+	switch (msg->hdr.type) {
+	case TYPE82_RSP_CODE:
+	case TYPE87_RSP_CODE:
+		return convert_error(zq, reply);
+	case TYPE86_RSP_CODE:
+		if (msg->hdr.reply_code)
+			return convert_error(zq, reply);
+		if (msg->cprbx.cprb_ver_id == 0x04)
+			return convert_type86_ep11_xcrb(userspace, zq, reply, xcrb);
+		fallthrough;	/* wrong cprb version is an unknown resp */
+	default: /* Unknown response type, this should NEVER EVER happen */
+		zq->online = 0;
+		pr_err("Crypto dev=%02x.%04x unknown response type 0x%02x => online=0 rc=EAGAIN\n",
+		       AP_QID_CARD(zq->queue->qid),
+		       AP_QID_QUEUE(zq->queue->qid),
+		       (int)msg->hdr.type);
+		ZCRYPT_DBF_ERR(
+			"%s dev=%02x.%04x unknown response type 0x%02x => online=0 rc=EAGAIN\n",
+			__func__, AP_QID_CARD(zq->queue->qid),
+			AP_QID_QUEUE(zq->queue->qid), (int)msg->hdr.type);
+		ap_send_online_uevent(&zq->queue->ap_dev, zq->online);
+		return -EAGAIN;
+	}
+}
+
+static int convert_response_rng(struct zcrypt_queue *zq,
+				struct ap_message *reply,
+				char *data)
+{
+	struct type86x_reply *msg = reply->msg;
+
+	switch (msg->hdr.type) {
+	case TYPE82_RSP_CODE:
+	case TYPE88_RSP_CODE:
+		return -EINVAL;
+	case TYPE86_RSP_CODE:
+		if (msg->hdr.reply_code)
+			return -EINVAL;
+		if (msg->cprbx.cprb_ver_id == 0x02)
+			return convert_type86_rng(zq, reply, data);
+		fallthrough;	/* wrong cprb version is an unknown response */
+	default: /* Unknown response type, this should NEVER EVER happen */
+		zq->online = 0;
+		pr_err("Crypto dev=%02x.%04x unknown response type 0x%02x => online=0 rc=EAGAIN\n",
+		       AP_QID_CARD(zq->queue->qid),
+		       AP_QID_QUEUE(zq->queue->qid),
+		       (int)msg->hdr.type);
+		ZCRYPT_DBF_ERR(
+			"%s dev=%02x.%04x unknown response type 0x%02x => online=0 rc=EAGAIN\n",
+			__func__, AP_QID_CARD(zq->queue->qid),
+			AP_QID_QUEUE(zq->queue->qid), (int)msg->hdr.type);
+		ap_send_online_uevent(&zq->queue->ap_dev, zq->online);
+		return -EAGAIN;
+	}
+}
+
+/*
+ * This function is called from the AP bus code after a crypto request
+ * "msg" has finished with the reply message "reply".
+ * It is called from tasklet context.
+ * @aq: pointer to the AP queue
+ * @msg: pointer to the AP message
+ * @reply: pointer to the AP reply message
+ */
+static void zcrypt_msgtype6_receive(struct ap_queue *aq,
+				    struct ap_message *msg,
+				    struct ap_message *reply)
+{
+	static struct error_hdr error_reply = {
+		.type = TYPE82_RSP_CODE,
+		.reply_code = REP82_ERROR_MACHINE_FAILURE,
+	};
+	struct ap_response_type *resp_type = &msg->response;
+	struct type86x_reply *t86r;
+	int len;
+
+	/* Copy the reply message to the request message buffer. */
+	if (!reply)
+		goto out;	/* ap_msg->rc indicates the error */
+	t86r = reply->msg;
+	if (t86r->hdr.type == TYPE86_RSP_CODE &&
+	    t86r->cprbx.cprb_ver_id == 0x02) {
+		switch (resp_type->type) {
+		case CEXXC_RESPONSE_TYPE_ICA:
+			len = sizeof(struct type86x_reply) + t86r->length;
+			if (len > reply->bufsize || len > msg->bufsize ||
+			    len != reply->len) {
+				pr_debug("len mismatch => EMSGSIZE\n");
+				msg->rc = -EMSGSIZE;
+				goto out;
+			}
+			memcpy(msg->msg, reply->msg, len);
+			msg->len = len;
+			break;
+		case CEXXC_RESPONSE_TYPE_XCRB:
+			if (t86r->fmt2.count2)
+				len = t86r->fmt2.offset2 + t86r->fmt2.count2;
+			else
+				len = t86r->fmt2.offset1 + t86r->fmt2.count1;
+			if (len > reply->bufsize || len > msg->bufsize ||
+			    len != reply->len) {
+				pr_debug("len mismatch => EMSGSIZE\n");
+				msg->rc = -EMSGSIZE;
+				goto out;
+			}
+			memcpy(msg->msg, reply->msg, len);
+			msg->len = len;
+			break;
+		default:
+			memcpy(msg->msg, &error_reply, sizeof(error_reply));
+			msg->len = sizeof(error_reply);
+		}
+	} else {
+		memcpy(msg->msg, reply->msg, sizeof(error_reply));
+		msg->len = sizeof(error_reply);
+	}
+out:
+	complete(&resp_type->work);
+}
+
+/*
+ * This function is called from the AP bus code after a crypto request
+ * "msg" has finished with the reply message "reply".
+ * It is called from tasklet context.
+ * @aq: pointer to the AP queue
+ * @msg: pointer to the AP message
+ * @reply: pointer to the AP reply message
+ */
+static void zcrypt_msgtype6_receive_ep11(struct ap_queue *aq,
+					 struct ap_message *msg,
+					 struct ap_message *reply)
+{
+	static struct error_hdr error_reply = {
+		.type = TYPE82_RSP_CODE,
+		.reply_code = REP82_ERROR_MACHINE_FAILURE,
+	};
+	struct ap_response_type *resp_type = &msg->response;
+	struct type86_ep11_reply *t86r;
+	int len;
+
+	/* Copy the reply message to the request message buffer. */
+	if (!reply)
+		goto out;	/* ap_msg->rc indicates the error */
+	t86r = reply->msg;
+	if (t86r->hdr.type == TYPE86_RSP_CODE &&
+	    t86r->cprbx.cprb_ver_id == 0x04) {
+		switch (resp_type->type) {
+		case CEXXC_RESPONSE_TYPE_EP11:
+			len = t86r->fmt2.offset1 + t86r->fmt2.count1;
+			if (len > reply->bufsize || len > msg->bufsize ||
+			    len != reply->len) {
+				pr_debug("len mismatch => EMSGSIZE\n");
+				msg->rc = -EMSGSIZE;
+				goto out;
+			}
+			memcpy(msg->msg, reply->msg, len);
+			msg->len = len;
+			break;
+		default:
+			memcpy(msg->msg, &error_reply, sizeof(error_reply));
+			msg->len = sizeof(error_reply);
+		}
+	} else {
+		memcpy(msg->msg, reply->msg, sizeof(error_reply));
+		msg->len = sizeof(error_reply);
+	}
+out:
+	complete(&resp_type->work);
+}
+
+static atomic_t zcrypt_step = ATOMIC_INIT(0);
+
+/*
+ * The request distributor calls this function if it picked the CEXxC
+ * device to handle a modexpo request.
+ * This function assumes that ap_msg has been initialized with
+ * ap_init_apmsg() and thus a valid buffer with the size of
+ * ap_msg->bufsize is available within ap_msg. Also the caller has
+ * to make sure ap_release_apmsg() is always called even on failure.
+ * @zq: pointer to zcrypt_queue structure that identifies the
+ *	CEXxC device to the request distributor
+ * @mex: pointer to the modexpo request buffer
+ */
+static long zcrypt_msgtype6_modexpo(struct zcrypt_queue *zq,
+				    struct ica_rsa_modexpo *mex,
+				    struct ap_message *ap_msg)
+{
+	struct ap_response_type *resp_type = &ap_msg->response;
+	int rc;
+
+	ap_msg->receive = zcrypt_msgtype6_receive;
+	ap_msg->psmid = (((unsigned long)current->pid) << 32) +
+		atomic_inc_return(&zcrypt_step);
+	rc = icamex_msg_to_type6mex_msgx(zq, ap_msg, mex);
+	if (rc)
+		goto out;
+	resp_type->type = CEXXC_RESPONSE_TYPE_ICA;
+	init_completion(&resp_type->work);
+	rc = ap_queue_message(zq->queue, ap_msg);
+	if (rc)
+		goto out;
+	rc = wait_for_completion_interruptible(&resp_type->work);
+	if (rc == 0) {
+		rc = ap_msg->rc;
+		if (rc == 0)
+			rc = convert_response_ica(zq, ap_msg,
+						  mex->outputdata,
+						  mex->outputdatalength);
+	} else {
+		/* Signal pending. */
+		ap_cancel_message(zq->queue, ap_msg);
+	}
+
+out:
+	return rc;
+}
+
+/*
+ * The request distributor calls this function if it picked the CEXxC
+ * device to handle a modexpo_crt request.
+ * This function assumes that ap_msg has been initialized with
+ * ap_init_apmsg() and thus a valid buffer with the size of
+ * ap_msg->bufsize is available within ap_msg. Also the caller has
+ * to make sure ap_release_apmsg() is always called even on failure.
+ * @zq: pointer to zcrypt_queue structure that identifies the
+ *	CEXxC device to the request distributor
+ * @crt: pointer to the modexpoc_crt request buffer
+ */
+static long zcrypt_msgtype6_modexpo_crt(struct zcrypt_queue *zq,
+					struct ica_rsa_modexpo_crt *crt,
+					struct ap_message *ap_msg)
+{
+	struct ap_response_type *resp_type = &ap_msg->response;
+	int rc;
+
+	ap_msg->receive = zcrypt_msgtype6_receive;
+	ap_msg->psmid = (((unsigned long)current->pid) << 32) +
+		atomic_inc_return(&zcrypt_step);
+	rc = icacrt_msg_to_type6crt_msgx(zq, ap_msg, crt);
+	if (rc)
+		goto out;
+	resp_type->type = CEXXC_RESPONSE_TYPE_ICA;
+	init_completion(&resp_type->work);
+	rc = ap_queue_message(zq->queue, ap_msg);
+	if (rc)
+		goto out;
+	rc = wait_for_completion_interruptible(&resp_type->work);
+	if (rc == 0) {
+		rc = ap_msg->rc;
+		if (rc == 0)
+			rc = convert_response_ica(zq, ap_msg,
+						  crt->outputdata,
+						  crt->outputdatalength);
+	} else {
+		/* Signal pending. */
+		ap_cancel_message(zq->queue, ap_msg);
+	}
+
+out:
+	return rc;
+}
+
+/*
+ * Prepare a CCA AP msg request.
+ * Prepare a CCA AP msg: fetch the required data from userspace,
+ * prepare the AP msg, fill some info into the ap_message struct,
+ * extract some data from the CPRB and give back to the caller.
+ * This function assumes that ap_msg has been initialized with
+ * ap_init_apmsg() and thus a valid buffer with the size of
+ * ap_msg->bufsize is available within ap_msg. Also the caller has
+ * to make sure ap_release_apmsg() is always called even on failure.
+ */
+int prep_cca_ap_msg(bool userspace, struct ica_xcRB *xcrb,
+		    struct ap_message *ap_msg,
+		    unsigned int *func_code, unsigned int *domain)
+{
+	struct ap_response_type *resp_type = &ap_msg->response;
+
+	ap_msg->receive = zcrypt_msgtype6_receive;
+	ap_msg->psmid = (((unsigned long)current->pid) << 32) +
+				atomic_inc_return(&zcrypt_step);
+	resp_type->type = CEXXC_RESPONSE_TYPE_XCRB;
+	return xcrb_msg_to_type6cprb_msgx(userspace, ap_msg,
+					  xcrb, func_code, domain);
+}
+
+/*
+ * The request distributor calls this function if it picked the CEXxC
+ * device to handle a send_cprb request.
+ * @zq: pointer to zcrypt_queue structure that identifies the
+ *	CEXxC device to the request distributor
+ * @xcrb: pointer to the send_cprb request buffer
+ */
+static long zcrypt_msgtype6_send_cprb(bool userspace, struct zcrypt_queue *zq,
+				      struct ica_xcRB *xcrb,
+				      struct ap_message *ap_msg)
+{
+	struct ap_response_type *resp_type = &ap_msg->response;
+	struct {
+		struct type6_hdr hdr;
+		struct CPRBX cprbx;
+		/* ... more data blocks ... */
+	} __packed * msg = ap_msg->msg;
+	unsigned int max_payload_size;
+	int rc, delta;
+
+	/* calculate maximum payload for this card and msg type */
+	max_payload_size = zq->reply.bufsize - sizeof(struct type86_fmt2_msg);
+
+	/* limit each of the two from fields to the maximum payload size */
+	msg->hdr.fromcardlen1 = min(msg->hdr.fromcardlen1, max_payload_size);
+	msg->hdr.fromcardlen2 = min(msg->hdr.fromcardlen2, max_payload_size);
+
+	/* calculate delta if the sum of both exceeds max payload size */
+	delta = msg->hdr.fromcardlen1 + msg->hdr.fromcardlen2
+		- max_payload_size;
+	if (delta > 0) {
+		/*
+		 * Sum exceeds maximum payload size, prune fromcardlen1
+		 * (always trust fromcardlen2)
+		 */
+		if (delta > msg->hdr.fromcardlen1) {
+			rc = -EINVAL;
+			goto out;
+		}
+		msg->hdr.fromcardlen1 -= delta;
+	}
+
+	/* update domain field within the CPRB struct */
+	msg->cprbx.domain = AP_QID_QUEUE(zq->queue->qid);
+
+	init_completion(&resp_type->work);
+	rc = ap_queue_message(zq->queue, ap_msg);
+	if (rc)
+		goto out;
+	rc = wait_for_completion_interruptible(&resp_type->work);
+	if (rc == 0) {
+		rc = ap_msg->rc;
+		if (rc == 0)
+			rc = convert_response_xcrb(userspace, zq, ap_msg, xcrb);
+	} else {
+		/* Signal pending. */
+		ap_cancel_message(zq->queue, ap_msg);
+	}
+
+	if (rc == -EAGAIN && ap_msg->flags & AP_MSG_FLAG_ADMIN)
+		rc = -EIO; /* do not retry administrative requests */
+
+out:
+	if (rc)
+		pr_debug("send cprb at dev=%02x.%04x rc=%d\n",
+			 AP_QID_CARD(zq->queue->qid),
+			 AP_QID_QUEUE(zq->queue->qid), rc);
+	return rc;
+}
+
+/*
+ * Prepare an EP11 AP msg request.
+ * Prepare an EP11 AP msg: fetch the required data from userspace,
+ * prepare the AP msg, fill some info into the ap_message struct,
+ * extract some data from the CPRB and give back to the caller.
+ * This function assumes that ap_msg has been initialized with
+ * ap_init_apmsg() and thus a valid buffer with the size of
+ * ap_msg->bufsize is available within ap_msg. Also the caller has
+ * to make sure ap_release_apmsg() is always called even on failure.
+ */
+int prep_ep11_ap_msg(bool userspace, struct ep11_urb *xcrb,
+		     struct ap_message *ap_msg,
+		     unsigned int *func_code, unsigned int *domain)
+{
+	struct ap_response_type *resp_type = &ap_msg->response;
+
+	ap_msg->receive = zcrypt_msgtype6_receive_ep11;
+	ap_msg->psmid = (((unsigned long)current->pid) << 32) +
+				atomic_inc_return(&zcrypt_step);
+	resp_type->type = CEXXC_RESPONSE_TYPE_EP11;
+	return xcrb_msg_to_type6_ep11cprb_msgx(userspace, ap_msg, xcrb,
+					       func_code, domain);
+}
+
+/*
+ * Simple asn1 int writer/encoder helper function
+ * Returns number of bytes processed or < 0 on failure
+ * Only accepts int length values of 1, 2 or 4.
+ */
+static inline int asn1_int_encode(u8 *buf, size_t intlen, u32 u)
+{
+	switch (intlen) {
+	case 1:
+		*buf = (u8)u;
+		return 1;
+	case 2:
+		put_unaligned_be16((u16)u, buf);
+		return 2;
+	case 4:
+		put_unaligned_be32((u32)u, buf);
+		return 4;
+	default:
+		return -EINVAL;
+	}
+}
+
+/*
+ * The request distributor calls this function if it picked the CEX4P
+ * device to handle a send_ep11_cprb request.
+ * @zq: pointer to zcrypt_queue structure that identifies the
+ *	  CEX4P device to the request distributor
+ * @xcrb: pointer to the ep11 user request block
+ */
+static long zcrypt_msgtype6_send_ep11_cprb(bool userspace, struct zcrypt_queue *zq,
+					   struct ep11_urb *xcrb,
+					   struct ap_message *ap_msg)
+{
+	int rc;
+	struct ap_response_type *resp_type = &ap_msg->response;
+	struct {
+		struct type6_hdr hdr;
+		struct ep11_cprb cprbx;
+	} __packed * msg = ap_msg->msg;
+
+	/*
+	 * The target domain field within the cprb body/payload block will be
+	 * replaced by the usage domain for non-management commands only.
+	 * Therefore we check the first bit of the 'flags' parameter for
+	 * management command indication.
+	 */
+	if (!(msg->cprbx.flags & 0x80)) {
+		int i, offs = 0;
+		size_t pld_len;
+		u8 *pld;
+		u32 u;
+
+		/* update target field in ep11_cprb */
+		msg->cprbx.target_id = (u32)AP_QID_QUEUE(zq->queue->qid);
+
+		/* ptr and length to payload */
+		pld = ap_msg->msg +
+			sizeof(struct type6_hdr) + sizeof(struct ep11_cprb);
+		pld_len = msg->cprbx.payload_len;
+		if (pld_len < MIN_EP11_PAYLOAD_SIZE)
+			return -EINVAL;
+
+		/*
+		 * Parse the asn1 payload, at least we have
+		 *   pld tag (1 octet)
+		 *   payload length (1-5 octets)
+		 *   function tag (1 octet)
+		 *   function length (1-5 octets)
+		 *   function value (1-4 octets)
+		 *   ----- optional fields -----
+		 *   domain tag (1 octet)
+		 *   domain length (1-5 octets)
+		 *   domain value (1-4 octets)
+		 *   ... maybe much more data ...
+		 */
+
+		/* payload tag */
+		if (pld[offs++] != 0x30)
+			return -EINVAL;
+		/* payload length field */
+		i = asn1_length_decode(pld + offs, pld_len - offs, &u);
+		if (i < 0)
+			return -EINVAL;
+		offs += i;
+		if (offs >= pld_len || u > pld_len - offs)
+			return -EINVAL;
+		/* function tag */
+		if (pld[offs++] != 0x04)
+			return -EINVAL;
+		/* function length */
+		if (offs >= pld_len)
+			return -EINVAL;
+		i = asn1_length_decode(pld + offs, pld_len - offs, &u);
+		if (i < 0)
+			return -EINVAL;
+		offs += i;
+		if (u > pld_len - offs)
+			return -EINVAL;
+		/* skip over the function value */
+		offs += u;
+		/* is there some payload left which could hold a domain value ? */
+		if (offs < pld_len && pld_len - offs >= 3) {
+			/* domain tag */
+			if (pld[offs++] != 0x04)
+				return -EINVAL;
+			/* domain length */
+			i = asn1_length_decode(pld + offs, pld_len - offs, &u);
+			if (i < 0)
+				return -EINVAL;
+			offs += i;
+			if (offs >= pld_len || u > pld_len - offs)
+				return -EINVAL;
+			/*
+			 * pld[offs] is now at the start of the domain value
+			 * with the value sprawled in u octets.
+			 */
+			i = asn1_int_encode(pld + offs, u,
+					    AP_QID_QUEUE(zq->queue->qid));
+			if (i < 0)
+				return -EINVAL;
+			offs += i;
+		}
+	}
+
+	/*
+	 * Set the queue's reply buffer length minus the two prepend headers
+	 * as reply limit for the card firmware.
+	 */
+	msg->hdr.fromcardlen1 = zq->reply.bufsize -
+		sizeof(struct type86_hdr) - sizeof(struct type86_fmt2_ext);
+
+	init_completion(&resp_type->work);
+	rc = ap_queue_message(zq->queue, ap_msg);
+	if (rc)
+		goto out;
+	rc = wait_for_completion_interruptible(&resp_type->work);
+	if (rc == 0) {
+		rc = ap_msg->rc;
+		if (rc == 0)
+			rc = convert_response_ep11_xcrb(userspace, zq, ap_msg, xcrb);
+	} else {
+		/* Signal pending. */
+		ap_cancel_message(zq->queue, ap_msg);
+	}
+
+	if (rc == -EAGAIN && ap_msg->flags & AP_MSG_FLAG_ADMIN)
+		rc = -EIO; /* do not retry administrative requests */
+
+out:
+	if (rc)
+		pr_debug("send cprb at dev=%02x.%04x rc=%d\n",
+			 AP_QID_CARD(zq->queue->qid),
+			 AP_QID_QUEUE(zq->queue->qid), rc);
+	return rc;
+}
+
+/*
+ * Prepare a type6 CPRB message for random number generation
+ *
+ * @ap_dev: AP device pointer
+ * @ap_msg: pointer to AP message
+ */
+static inline void rng_type6cprb_msgx(struct ap_message *ap_msg,
+				      unsigned int random_number_length,
+				      unsigned int *domain)
+{
+	struct {
+		struct type6_hdr hdr;
+		struct CPRBX cprbx;
+		char function_code[2];
+		short int rule_length;
+		char rule[8];
+		short int verb_length;
+		short int key_length;
+	} __packed * msg = ap_msg->msg;
+	static struct type6_hdr static_type6_hdrX = {
+		.type		= 0x06,
+		.offset1	= 0x00000058,
+		.agent_id	= {'C', 'A'},
+		.function_code	= {'R', 'L'},
+		.tocardlen1	= sizeof(*msg) - sizeof(msg->hdr),
+		.fromcardlen1	= sizeof(*msg) - sizeof(msg->hdr),
+	};
+	static struct CPRBX local_cprbx = {
+		.cprb_len	= 0x00dc,
+		.cprb_ver_id	= 0x02,
+		.func_id	= {'T', '2'},
+		.req_parml	= sizeof(*msg) - sizeof(msg->hdr) -
+				  sizeof(msg->cprbx),
+		.rpl_msgbl	= sizeof(*msg) - sizeof(msg->hdr),
+	};
+
+	msg->hdr = static_type6_hdrX;
+	msg->hdr.fromcardlen2 = random_number_length;
+	msg->cprbx = local_cprbx;
+	msg->cprbx.rpl_datal = random_number_length;
+	memcpy(msg->function_code, msg->hdr.function_code, 0x02);
+	msg->rule_length = 0x0a;
+	memcpy(msg->rule, "RANDOM  ", 8);
+	msg->verb_length = 0x02;
+	msg->key_length = 0x02;
+	ap_msg->len = sizeof(*msg);
+	if (domain)
+		*domain = msg->cprbx.domain;
+}
+
+/*
+ * Prepare a CEXXC get random request ap message.
+ * This function assumes that ap_msg has been initialized with
+ * ap_init_apmsg() and thus a valid buffer with the size of
+ * ap_max_msg_size is available within ap_msg. Also the caller has
+ * to make sure ap_release_apmsg() is always called even on failure.
+ */
+int prep_rng_ap_msg(struct ap_message *ap_msg, int *func_code,
+		    unsigned int *domain)
+{
+	struct ap_response_type *resp_type = &ap_msg->response;
+
+	if (ap_msg->bufsize < AP_DEFAULT_MAX_MSG_SIZE)
+		return -EMSGSIZE;
+	ap_msg->receive = zcrypt_msgtype6_receive;
+	ap_msg->psmid = (((unsigned long)current->pid) << 32) +
+				atomic_inc_return(&zcrypt_step);
+
+	resp_type->type = CEXXC_RESPONSE_TYPE_XCRB;
+
+	rng_type6cprb_msgx(ap_msg, ZCRYPT_RNG_BUFFER_SIZE, domain);
+
+	*func_code = HWRNG;
+	return 0;
+}
+
+/*
+ * The request distributor calls this function if it picked the CEXxC
+ * device to generate random data.
+ * @zq: pointer to zcrypt_queue structure that identifies the
+ *	CEXxC device to the request distributor
+ * @buffer: pointer to a memory page to return random data
+ */
+static long zcrypt_msgtype6_rng(struct zcrypt_queue *zq,
+				char *buffer, struct ap_message *ap_msg)
+{
+	struct {
+		struct type6_hdr hdr;
+		struct CPRBX cprbx;
+		char function_code[2];
+		short int rule_length;
+		char rule[8];
+		short int verb_length;
+		short int key_length;
+	} __packed * msg = ap_msg->msg;
+	struct ap_response_type *resp_type = &ap_msg->response;
+	int rc;
+
+	msg->cprbx.domain = AP_QID_QUEUE(zq->queue->qid);
+
+	init_completion(&resp_type->work);
+	rc = ap_queue_message(zq->queue, ap_msg);
+	if (rc)
+		goto out;
+	rc = wait_for_completion_interruptible(&resp_type->work);
+	if (rc == 0) {
+		rc = ap_msg->rc;
+		if (rc == 0)
+			rc = convert_response_rng(zq, ap_msg, buffer);
+	} else {
+		/* Signal pending. */
+		ap_cancel_message(zq->queue, ap_msg);
+	}
+out:
+	return rc;
+}
+
+/*
+ * The crypto operations for a CEXxC card.
+ */
+
+static struct zcrypt_ops zcrypt_msgtype6_ops = {
+	.owner = THIS_MODULE,
+	.name = MSGTYPE06_NAME,
+	.variant = MSGTYPE06_VARIANT_DEFAULT,
+	.rsa_modexpo = zcrypt_msgtype6_modexpo,
+	.rsa_modexpo_crt = zcrypt_msgtype6_modexpo_crt,
+	.send_cprb = zcrypt_msgtype6_send_cprb,
+	.rng = zcrypt_msgtype6_rng,
+};
+
+static struct zcrypt_ops zcrypt_msgtype6_ep11_ops = {
+	.owner = THIS_MODULE,
+	.name = MSGTYPE06_NAME,
+	.variant = MSGTYPE06_VARIANT_EP11,
+	.rsa_modexpo = NULL,
+	.rsa_modexpo_crt = NULL,
+	.send_ep11_cprb = zcrypt_msgtype6_send_ep11_cprb,
+};
+
+void __init zcrypt_msgtype6_init(void)
+{
+	zcrypt_msgtype_register(&zcrypt_msgtype6_ops);
+	zcrypt_msgtype_register(&zcrypt_msgtype6_ep11_ops);
+}
+
+void __exit zcrypt_msgtype6_exit(void)
+{
+	zcrypt_msgtype_unregister(&zcrypt_msgtype6_ops);
+	zcrypt_msgtype_unregister(&zcrypt_msgtype6_ep11_ops);
+}
diff --git a/drivers/s390/crypto/zcrypt_msgtype6.h b/drivers/s390/crypto/zcrypt_msgtype6.h
new file mode 100644
index 000000000..8cc227df5
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_msgtype6.h
@@ -0,0 +1,116 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ *  Copyright IBM Corp. 2001, 2012
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *  MSGTYPE restruct:		  Holger Dengler <hd@linux.vnet.ibm.com>
+ */
+
+#ifndef _ZCRYPT_MSGTYPE6_H_
+#define _ZCRYPT_MSGTYPE6_H_
+
+#include <asm/zcrypt.h>
+
+#define MSGTYPE06_NAME			"zcrypt_msgtype6"
+#define MSGTYPE06_VARIANT_DEFAULT	0
+#define MSGTYPE06_VARIANT_NORNG		1
+#define MSGTYPE06_VARIANT_EP11		2
+
+/**
+ * The type 6 message family is associated with CEXxC/CEXxP cards.
+ *
+ * It contains a message header followed by a CPRB, both of which
+ * are described below.
+ *
+ * Note that all reserved fields must be zeroes.
+ */
+struct type6_hdr {
+	unsigned char reserved1;	/* 0x00				*/
+	unsigned char type;		/* 0x06				*/
+	unsigned char reserved2[2];	/* 0x0000			*/
+	unsigned char right[4];		/* 0x00000000			*/
+	unsigned char reserved3[2];	/* 0x0000			*/
+	unsigned char reserved4[2];	/* 0x0000			*/
+	unsigned int  apfs;		/* 0x00000000			*/
+	unsigned int  offset1;		/* 0x00000058 (offset to CPRB)	*/
+	unsigned int  offset2;		/* 0x00000000			*/
+	unsigned int  offset3;		/* 0x00000000			*/
+	unsigned int  offset4;		/* 0x00000000			*/
+	unsigned char agent_id[16];	/* 0x4341000000000000		*/
+					/* 0x0000000000000000		*/
+	unsigned char rqid[2];		/* rqid.  internal to 603	*/
+	unsigned char reserved5[2];	/* 0x0000			*/
+	unsigned char function_code[2];	/* for PKD, 0x5044 (ascii 'PD')	*/
+	unsigned char reserved6[2];	/* 0x0000			*/
+	unsigned int  tocardlen1;	/* (request CPRB len + 3) & -4	*/
+	unsigned int  tocardlen2;	/* db len 0x00000000 for PKD	*/
+	unsigned int  tocardlen3;	/* 0x00000000			*/
+	unsigned int  tocardlen4;	/* 0x00000000			*/
+	unsigned int  fromcardlen1;	/* response buffer length	*/
+	unsigned int  fromcardlen2;	/* db len 0x00000000 for PKD	*/
+	unsigned int  fromcardlen3;	/* 0x00000000			*/
+	unsigned int  fromcardlen4;	/* 0x00000000			*/
+} __packed;
+
+/**
+ * The type 86 message family is associated with CEXxC/CEXxP cards.
+ *
+ * It contains a message header followed by a CPRB.  The CPRB is
+ * the same as the request CPRB, which is described above.
+ *
+ * If format is 1, an error condition exists and no data beyond
+ * the 8-byte message header is of interest.
+ *
+ * The non-error message is shown below.
+ *
+ * Note that all reserved fields must be zeroes.
+ */
+struct type86_hdr {
+	unsigned char reserved1;	/* 0x00				*/
+	unsigned char type;		/* 0x86				*/
+	unsigned char format;		/* 0x01 (error) or 0x02 (ok)	*/
+	unsigned char reserved2;	/* 0x00				*/
+	unsigned char reply_code;	/* reply code (see above)	*/
+	unsigned char reserved3[3];	/* 0x000000			*/
+} __packed;
+
+#define TYPE86_RSP_CODE 0x86
+#define TYPE87_RSP_CODE 0x87
+#define TYPE86_FMT2	0x02
+
+struct type86_fmt2_ext {
+	unsigned char	  reserved[4];	/* 0x00000000			*/
+	unsigned int	  apfs;		/* final status			*/
+	unsigned int	  count1;	/* length of CPRB + parameters	*/
+	unsigned int	  offset1;	/* offset to CPRB		*/
+	unsigned int	  count2;	/* 0x00000000			*/
+	unsigned int	  offset2;	/* db offset 0x00000000 for PKD	*/
+	unsigned int	  count3;	/* 0x00000000			*/
+	unsigned int	  offset3;	/* 0x00000000			*/
+	unsigned int	  count4;	/* 0x00000000			*/
+	unsigned int	  offset4;	/* 0x00000000			*/
+} __packed;
+
+int prep_cca_ap_msg(bool userspace, struct ica_xcRB *xcrb,
+		    struct ap_message *ap_msg,
+		    unsigned int *fc, unsigned int *dom);
+int prep_ep11_ap_msg(bool userspace, struct ep11_urb *xcrb,
+		     struct ap_message *ap_msg,
+		     unsigned int *fc, unsigned int *dom);
+int prep_rng_ap_msg(struct ap_message *ap_msg,
+		    int *fc, unsigned int *dom);
+
+#define LOW	10
+#define MEDIUM	100
+#define HIGH	500
+
+int speed_idx_cca(int);
+int speed_idx_ep11(int);
+
+void zcrypt_msgtype6_init(void);
+void zcrypt_msgtype6_exit(void);
+
+#endif /* _ZCRYPT_MSGTYPE6_H_ */
diff --git a/drivers/s390/crypto/zcrypt_queue.c b/drivers/s390/crypto/zcrypt_queue.c
new file mode 100644
index 000000000..87457cde2
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_queue.c
@@ -0,0 +1,234 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ *  Copyright IBM Corp. 2001, 2012
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *	       Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *				  Ralph Wuerthner <rwuerthn@de.ibm.com>
+ *  MSGTYPE restruct:		  Holger Dengler <hd@linux.vnet.ibm.com>
+ */
+
+#include <linux/export.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/miscdevice.h>
+#include <linux/fs.h>
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#include <linux/slab.h>
+#include <linux/atomic.h>
+#include <linux/uaccess.h>
+#include <linux/hw_random.h>
+#include <linux/debugfs.h>
+#include <asm/debug.h>
+
+#include "zcrypt_debug.h"
+#include "zcrypt_api.h"
+
+#include "zcrypt_msgtype6.h"
+#include "zcrypt_msgtype50.h"
+
+/*
+ * Device attributes common for all crypto queue devices.
+ */
+
+static ssize_t online_show(struct device *dev,
+			   struct device_attribute *attr,
+			   char *buf)
+{
+	struct zcrypt_queue *zq = dev_get_drvdata(dev);
+	struct ap_queue *aq = to_ap_queue(dev);
+	int online = aq->config && !aq->chkstop && zq->online ? 1 : 0;
+
+	return sysfs_emit(buf, "%d\n", online);
+}
+
+static ssize_t online_store(struct device *dev,
+			    struct device_attribute *attr,
+			    const char *buf, size_t count)
+{
+	struct zcrypt_queue *zq = dev_get_drvdata(dev);
+	struct ap_queue *aq = to_ap_queue(dev);
+	struct zcrypt_card *zc = zq->zcard;
+	int online;
+
+	if (sscanf(buf, "%d\n", &online) != 1 || online < 0 || online > 1)
+		return -EINVAL;
+
+	if (online && (!aq->config || !aq->card->config ||
+		       aq->chkstop || aq->card->chkstop))
+		return -ENODEV;
+	if (online && !zc->online)
+		return -EINVAL;
+	zq->online = online;
+
+	ZCRYPT_DBF_INFO("%s queue=%02x.%04x online=%d\n",
+			__func__, AP_QID_CARD(zq->queue->qid),
+			AP_QID_QUEUE(zq->queue->qid), online);
+
+	ap_send_online_uevent(&aq->ap_dev, online);
+
+	if (!online)
+		ap_flush_queue(zq->queue);
+	return count;
+}
+
+static DEVICE_ATTR_RW(online);
+
+static ssize_t load_show(struct device *dev,
+			 struct device_attribute *attr,
+			 char *buf)
+{
+	struct zcrypt_queue *zq = dev_get_drvdata(dev);
+
+	return sysfs_emit(buf, "%d\n", atomic_read(&zq->load));
+}
+
+static DEVICE_ATTR_RO(load);
+
+static struct attribute *zcrypt_queue_attrs[] = {
+	&dev_attr_online.attr,
+	&dev_attr_load.attr,
+	NULL,
+};
+
+static const struct attribute_group zcrypt_queue_attr_group = {
+	.attrs = zcrypt_queue_attrs,
+};
+
+bool zcrypt_queue_force_online(struct zcrypt_queue *zq, int online)
+{
+	if (!!zq->online != !!online) {
+		zq->online = online;
+		if (!online)
+			ap_flush_queue(zq->queue);
+		return true;
+	}
+	return false;
+}
+
+struct zcrypt_queue *zcrypt_queue_alloc(size_t reply_buf_size)
+{
+	struct zcrypt_queue *zq;
+
+	zq = kzalloc_obj(*zq);
+	if (!zq)
+		return NULL;
+	zq->reply.msg = kmalloc(reply_buf_size, GFP_KERNEL);
+	if (!zq->reply.msg)
+		goto out_free;
+	zq->reply.bufsize = reply_buf_size;
+	INIT_LIST_HEAD(&zq->list);
+	kref_init(&zq->refcount);
+	return zq;
+
+out_free:
+	kfree(zq);
+	return NULL;
+}
+EXPORT_SYMBOL(zcrypt_queue_alloc);
+
+void zcrypt_queue_free(struct zcrypt_queue *zq)
+{
+	kfree(zq->reply.msg);
+	kfree(zq);
+}
+EXPORT_SYMBOL(zcrypt_queue_free);
+
+static void zcrypt_queue_release(struct kref *kref)
+{
+	struct zcrypt_queue *zq =
+		container_of(kref, struct zcrypt_queue, refcount);
+	zcrypt_queue_free(zq);
+}
+
+void zcrypt_queue_get(struct zcrypt_queue *zq)
+{
+	kref_get(&zq->refcount);
+}
+EXPORT_SYMBOL(zcrypt_queue_get);
+
+int zcrypt_queue_put(struct zcrypt_queue *zq)
+{
+	return kref_put(&zq->refcount, zcrypt_queue_release);
+}
+EXPORT_SYMBOL(zcrypt_queue_put);
+
+/**
+ * zcrypt_queue_register() - Register a crypto queue device.
+ * @zq: Pointer to a crypto queue device
+ *
+ * Register a crypto queue device. Returns 0 if successful.
+ */
+int zcrypt_queue_register(struct zcrypt_queue *zq)
+{
+	struct zcrypt_card *zc;
+	int rc;
+
+	spin_lock(&zcrypt_list_lock);
+	zc = dev_get_drvdata(&zq->queue->card->ap_dev.device);
+	zcrypt_card_get(zc);
+	zq->zcard = zc;
+	zq->online = 1;	/* New devices are online by default. */
+
+	ZCRYPT_DBF_INFO("%s queue=%02x.%04x register online=1\n",
+			__func__, AP_QID_CARD(zq->queue->qid),
+			AP_QID_QUEUE(zq->queue->qid));
+
+	list_add_tail(&zq->list, &zc->zqueues);
+	spin_unlock(&zcrypt_list_lock);
+
+	rc = sysfs_create_group(&zq->queue->ap_dev.device.kobj,
+				&zcrypt_queue_attr_group);
+	if (rc)
+		goto out;
+
+	if (zq->ops->rng) {
+		rc = zcrypt_rng_device_add();
+		if (rc)
+			goto out_unregister;
+	}
+	return 0;
+
+out_unregister:
+	sysfs_remove_group(&zq->queue->ap_dev.device.kobj,
+			   &zcrypt_queue_attr_group);
+out:
+	spin_lock(&zcrypt_list_lock);
+	list_del_init(&zq->list);
+	spin_unlock(&zcrypt_list_lock);
+	zcrypt_card_put(zc);
+	return rc;
+}
+EXPORT_SYMBOL(zcrypt_queue_register);
+
+/**
+ * zcrypt_queue_unregister(): Unregister a crypto queue device.
+ * @zq: Pointer to crypto queue device
+ *
+ * Unregister a crypto queue device.
+ */
+void zcrypt_queue_unregister(struct zcrypt_queue *zq)
+{
+	struct zcrypt_card *zc;
+
+	ZCRYPT_DBF_INFO("%s queue=%02x.%04x unregister\n",
+			__func__, AP_QID_CARD(zq->queue->qid),
+			AP_QID_QUEUE(zq->queue->qid));
+
+	zc = zq->zcard;
+	spin_lock(&zcrypt_list_lock);
+	list_del_init(&zq->list);
+	spin_unlock(&zcrypt_list_lock);
+	if (zq->ops->rng)
+		zcrypt_rng_device_remove();
+	sysfs_remove_group(&zq->queue->ap_dev.device.kobj,
+			   &zcrypt_queue_attr_group);
+	zcrypt_card_put(zc);
+	zcrypt_queue_put(zq);
+}
+EXPORT_SYMBOL(zcrypt_queue_unregister);