device.c

// SPDX-License-Identifier: GPL-2.0

/*
 * Copyright 2016-2019 HabanaLabs, Ltd.
 * All Rights Reserved.
 */

#define pr_fmt(fmt)			"habanalabs: " fmt

#include "habanalabs.h"

#include <linux/pci.h>
#include <linux/hwmon.h>
#include <uapi/misc/habanalabs.h>

enum hl_device_status hl_device_status(struct hl_device *hdev)
{
	enum hl_device_status status;

	if (atomic_read(&hdev->in_reset))
		status = HL_DEVICE_STATUS_IN_RESET;
	else if (hdev->needs_reset)
		status = HL_DEVICE_STATUS_NEEDS_RESET;
	else if (hdev->disabled)
		status = HL_DEVICE_STATUS_MALFUNCTION;
	else
		status = HL_DEVICE_STATUS_OPERATIONAL;

	return status;
}

bool hl_device_operational(struct hl_device *hdev,
		enum hl_device_status *status)
{
	enum hl_device_status current_status;

	current_status = hl_device_status(hdev);
	if (status)
		*status = current_status;

	switch (current_status) {
	case HL_DEVICE_STATUS_IN_RESET:
	case HL_DEVICE_STATUS_MALFUNCTION:
	case HL_DEVICE_STATUS_NEEDS_RESET:
		return false;
	case HL_DEVICE_STATUS_OPERATIONAL:
	default:
		return true;
	}
}

static void hpriv_release(struct kref *ref)
{
	struct hl_fpriv *hpriv;
	struct hl_device *hdev;

	hpriv = container_of(ref, struct hl_fpriv, refcount);

	hdev = hpriv->hdev;

	put_pid(hpriv->taskpid);

	hl_debugfs_remove_file(hpriv);

	mutex_destroy(&hpriv->restore_phase_mutex);

	mutex_lock(&hdev->fpriv_list_lock);
	list_del(&hpriv->dev_node);
	hdev->compute_ctx = NULL;
	mutex_unlock(&hdev->fpriv_list_lock);

	kfree(hpriv);

	if (hdev->reset_upon_device_release)
		hl_device_reset(hdev, 0);
}

void hl_hpriv_get(struct hl_fpriv *hpriv)
{
	kref_get(&hpriv->refcount);
}

int hl_hpriv_put(struct hl_fpriv *hpriv)
{
	return kref_put(&hpriv->refcount, hpriv_release);
}

/*
 * hl_device_release - release function for habanalabs device
 *
 * @inode: pointer to inode structure
 * @filp: pointer to file structure
 *
 * Called when process closes an habanalabs device
 */
static int hl_device_release(struct inode *inode, struct file *filp)
{
	struct hl_fpriv *hpriv = filp->private_data;
	struct hl_device *hdev = hpriv->hdev;

	filp->private_data = NULL;

	if (!hdev) {
		pr_crit("Closing FD after device was removed. Memory leak will occur and it is advised to reboot.\n");
		put_pid(hpriv->taskpid);
		return 0;
	}

	/* Each pending user interrupt holds the user's context, hence we
	 * must release them all before calling hl_ctx_mgr_fini().
	 */
	hl_release_pending_user_interrupts(hpriv->hdev);

	hl_cb_mgr_fini(hdev, &hpriv->cb_mgr);
	hl_ctx_mgr_fini(hdev, &hpriv->ctx_mgr);

	if (!hl_hpriv_put(hpriv))
		dev_warn(hdev->dev,
			"Device is still in use because there are live CS and/or memory mappings\n");

	return 0;
}

static int hl_device_release_ctrl(struct inode *inode, struct file *filp)
{
	struct hl_fpriv *hpriv = filp->private_data;
	struct hl_device *hdev = hpriv->hdev;

	filp->private_data = NULL;

	if (!hdev) {
		pr_err("Closing FD after device was removed\n");
		goto out;
	}

	mutex_lock(&hdev->fpriv_list_lock);
	list_del(&hpriv->dev_node);
	mutex_unlock(&hdev->fpriv_list_lock);
out:
	put_pid(hpriv->taskpid);

	kfree(hpriv);

	return 0;
}

/*
 * hl_mmap - mmap function for habanalabs device
 *
 * @*filp: pointer to file structure
 * @*vma: pointer to vm_area_struct of the process
 *
 * Called when process does an mmap on habanalabs device. Call the device's mmap
 * function at the end of the common code.
 */
static int hl_mmap(struct file *filp, struct vm_area_struct *vma)
{
	struct hl_fpriv *hpriv = filp->private_data;
	struct hl_device *hdev = hpriv->hdev;
	unsigned long vm_pgoff;

	if (!hdev) {
		pr_err_ratelimited("Trying to mmap after device was removed! Please close FD\n");
		return -ENODEV;
	}

	vm_pgoff = vma->vm_pgoff;
	vma->vm_pgoff = HL_MMAP_OFFSET_VALUE_GET(vm_pgoff);

	switch (vm_pgoff & HL_MMAP_TYPE_MASK) {
	case HL_MMAP_TYPE_CB:
		return hl_cb_mmap(hpriv, vma);

	case HL_MMAP_TYPE_BLOCK:
		return hl_hw_block_mmap(hpriv, vma);
	}

	return -EINVAL;
}

static const struct file_operations hl_ops = {
	.owner = THIS_MODULE,
	.open = hl_device_open,
	.release = hl_device_release,
	.mmap = hl_mmap,
	.unlocked_ioctl = hl_ioctl,
	.compat_ioctl = hl_ioctl
};

static const struct file_operations hl_ctrl_ops = {
	.owner = THIS_MODULE,
	.open = hl_device_open_ctrl,
	.release = hl_device_release_ctrl,
	.unlocked_ioctl = hl_ioctl_control,
	.compat_ioctl = hl_ioctl_control
};

static void device_release_func(struct device *dev)
{
	kfree(dev);
}

/*
 * device_init_cdev - Initialize cdev and device for habanalabs device
 *
 * @hdev: pointer to habanalabs device structure
 * @hclass: pointer to the class object of the device
 * @minor: minor number of the specific device
 * @fpos: file operations to install for this device
 * @name: name of the device as it will appear in the filesystem
 * @cdev: pointer to the char device object that will be initialized
 * @dev: pointer to the device object that will be initialized
 *
 * Initialize a cdev and a Linux device for habanalabs's device.
 */
static int device_init_cdev(struct hl_device *hdev, struct class *hclass,
				int minor, const struct file_operations *fops,
				char *name, struct cdev *cdev,
				struct device **dev)
{
	cdev_init(cdev, fops);
	cdev->owner = THIS_MODULE;

	*dev = kzalloc(sizeof(**dev), GFP_KERNEL);
	if (!*dev)
		return -ENOMEM;

	device_initialize(*dev);
	(*dev)->devt = MKDEV(hdev->major, minor);
	(*dev)->class = hclass;
	(*dev)->release = device_release_func;
	dev_set_drvdata(*dev, hdev);
	dev_set_name(*dev, "%s", name);

	return 0;
}

static int device_cdev_sysfs_add(struct hl_device *hdev)
{
	int rc;

	rc = cdev_device_add(&hdev->cdev, hdev->dev);
	if (rc) {
		dev_err(hdev->dev,
			"failed to add a char device to the system\n");
		return rc;
	}

	rc = cdev_device_add(&hdev->cdev_ctrl, hdev->dev_ctrl);
	if (rc) {
		dev_err(hdev->dev,
			"failed to add a control char device to the system\n");
		goto delete_cdev_device;
	}

	/* hl_sysfs_init() must be done after adding the device to the system */
	rc = hl_sysfs_init(hdev);
	if (rc) {
		dev_err(hdev->dev, "failed to initialize sysfs\n");
		goto delete_ctrl_cdev_device;
	}

	hdev->cdev_sysfs_created = true;

	return 0;

delete_ctrl_cdev_device:
	cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
delete_cdev_device:
	cdev_device_del(&hdev->cdev, hdev->dev);
	return rc;
}

static void device_cdev_sysfs_del(struct hl_device *hdev)
{
	if (!hdev->cdev_sysfs_created)
		goto put_devices;

	hl_sysfs_fini(hdev);
	cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
	cdev_device_del(&hdev->cdev, hdev->dev);

put_devices:
	put_device(hdev->dev);
	put_device(hdev->dev_ctrl);
}

static void device_hard_reset_pending(struct work_struct *work)
{
	struct hl_device_reset_work *device_reset_work =
		container_of(work, struct hl_device_reset_work,
				reset_work.work);
	struct hl_device *hdev = device_reset_work->hdev;
	int rc;

	rc = hl_device_reset(hdev, HL_RESET_HARD | HL_RESET_FROM_RESET_THREAD);
	if ((rc == -EBUSY) && !hdev->device_fini_pending) {
		dev_info(hdev->dev,
			"Could not reset device. will try again in %u seconds",
			HL_PENDING_RESET_PER_SEC);

		queue_delayed_work(device_reset_work->wq,
			&device_reset_work->reset_work,
			msecs_to_jiffies(HL_PENDING_RESET_PER_SEC * 1000));
	}
}

/*
 * device_early_init - do some early initialization for the habanalabs device
 *
 * @hdev: pointer to habanalabs device structure
 *
 * Install the relevant function pointers and call the early_init function,
 * if such a function exists
 */
static int device_early_init(struct hl_device *hdev)
{
	int i, rc;
	char workq_name[32];

	switch (hdev->asic_type) {
	case ASIC_GOYA:
		goya_set_asic_funcs(hdev);
		strscpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name));
		break;
	case ASIC_GAUDI:
		gaudi_set_asic_funcs(hdev);
		strscpy(hdev->asic_name, "GAUDI", sizeof(hdev->asic_name));
		break;
	case ASIC_GAUDI_SEC:
		gaudi_set_asic_funcs(hdev);
		strscpy(hdev->asic_name, "GAUDI SEC", sizeof(hdev->asic_name));
		break;
	default:
		dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
			hdev->asic_type);
		return -EINVAL;
	}

	rc = hdev->asic_funcs->early_init(hdev);
	if (rc)
		return rc;

	rc = hl_asid_init(hdev);
	if (rc)
		goto early_fini;

	if (hdev->asic_prop.completion_queues_count) {
		hdev->cq_wq = kcalloc(hdev->asic_prop.completion_queues_count,
				sizeof(*hdev->cq_wq),
				GFP_KERNEL);
		if (!hdev->cq_wq) {
			rc = -ENOMEM;
			goto asid_fini;
		}
	}

	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) {
		snprintf(workq_name, 32, "hl-free-jobs-%u", (u32) i);
		hdev->cq_wq[i] = create_singlethread_workqueue(workq_name);
		if (hdev->cq_wq[i] == NULL) {
			dev_err(hdev->dev, "Failed to allocate CQ workqueue\n");
			rc = -ENOMEM;
			goto free_cq_wq;
		}
	}

	hdev->eq_wq = alloc_workqueue("hl-events", WQ_UNBOUND, 0);
	if (hdev->eq_wq == NULL) {
		dev_err(hdev->dev, "Failed to allocate EQ workqueue\n");
		rc = -ENOMEM;
		goto free_cq_wq;
	}

	hdev->sob_reset_wq = alloc_workqueue("hl-sob-reset", WQ_UNBOUND, 0);
	if (!hdev->sob_reset_wq) {
		dev_err(hdev->dev,
			"Failed to allocate SOB reset workqueue\n");
		rc = -ENOMEM;
		goto free_eq_wq;
	}

	hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info),
					GFP_KERNEL);
	if (!hdev->hl_chip_info) {
		rc = -ENOMEM;
		goto free_sob_reset_wq;
	}

	rc = hl_mmu_if_set_funcs(hdev);
	if (rc)
		goto free_chip_info;

	hl_cb_mgr_init(&hdev->kernel_cb_mgr);

	hdev->device_reset_work.wq =
			create_singlethread_workqueue("hl_device_reset");
	if (!hdev->device_reset_work.wq) {
		rc = -ENOMEM;
		dev_err(hdev->dev, "Failed to create device reset WQ\n");
		goto free_cb_mgr;
	}

	INIT_DELAYED_WORK(&hdev->device_reset_work.reset_work,
			device_hard_reset_pending);
	hdev->device_reset_work.hdev = hdev;
	hdev->device_fini_pending = 0;

	mutex_init(&hdev->send_cpu_message_lock);
	mutex_init(&hdev->debug_lock);
	INIT_LIST_HEAD(&hdev->cs_mirror_list);
	spin_lock_init(&hdev->cs_mirror_lock);
	INIT_LIST_HEAD(&hdev->fpriv_list);
	mutex_init(&hdev->fpriv_list_lock);
	atomic_set(&hdev->in_reset, 0);

	return 0;

free_cb_mgr:
	hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr);
free_chip_info:
	kfree(hdev->hl_chip_info);
free_sob_reset_wq:
	destroy_workqueue(hdev->sob_reset_wq);
free_eq_wq:
	destroy_workqueue(hdev->eq_wq);
free_cq_wq:
	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
		if (hdev->cq_wq[i])
			destroy_workqueue(hdev->cq_wq[i]);
	kfree(hdev->cq_wq);
asid_fini:
	hl_asid_fini(hdev);
early_fini:
	if (hdev->asic_funcs->early_fini)
		hdev->asic_funcs->early_fini(hdev);

	return rc;
}

/*
 * device_early_fini - finalize all that was done in device_early_init
 *
 * @hdev: pointer to habanalabs device structure
 *
 */
static void device_early_fini(struct hl_device *hdev)
{
	int i;

	mutex_destroy(&hdev->debug_lock);
	mutex_destroy(&hdev->send_cpu_message_lock);

	mutex_destroy(&hdev->fpriv_list_lock);

	hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr);

	kfree(hdev->hl_chip_info);

	destroy_workqueue(hdev->sob_reset_wq);
	destroy_workqueue(hdev->eq_wq);
	destroy_workqueue(hdev->device_reset_work.wq);

	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
		destroy_workqueue(hdev->cq_wq[i]);
	kfree(hdev->cq_wq);

	hl_asid_fini(hdev);

	if (hdev->asic_funcs->early_fini)
		hdev->asic_funcs->early_fini(hdev);
}

static void set_freq_to_low_job(struct work_struct *work)
{
	struct hl_device *hdev = container_of(work, struct hl_device,
						work_freq.work);

	mutex_lock(&hdev->fpriv_list_lock);

	if (!hdev->compute_ctx)
		hl_device_set_frequency(hdev, PLL_LOW);

	mutex_unlock(&hdev->fpriv_list_lock);

	schedule_delayed_work(&hdev->work_freq,
			usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC));
}

static void hl_device_heartbeat(struct work_struct *work)
{
	struct hl_device *hdev = container_of(work, struct hl_device,
						work_heartbeat.work);

	if (!hl_device_operational(hdev, NULL))
		goto reschedule;

	if (!hdev->asic_funcs->send_heartbeat(hdev))
		goto reschedule;

	dev_err(hdev->dev, "Device heartbeat failed!\n");
	hl_device_reset(hdev, HL_RESET_HARD | HL_RESET_HEARTBEAT);

	return;

reschedule:
	schedule_delayed_work(&hdev->work_heartbeat,
			usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
}

/*
 * device_late_init - do late stuff initialization for the habanalabs device
 *
 * @hdev: pointer to habanalabs device structure
 *
 * Do stuff that either needs the device H/W queues to be active or needs
 * to happen after all the rest of the initialization is finished
 */
static int device_late_init(struct hl_device *hdev)
{
	int rc;

	if (hdev->asic_funcs->late_init) {
		rc = hdev->asic_funcs->late_init(hdev);
		if (rc) {
			dev_err(hdev->dev,
				"failed late initialization for the H/W\n");
			return rc;
		}
	}

	hdev->high_pll = hdev->asic_prop.high_pll;

	/* force setting to low frequency */
	hdev->curr_pll_profile = PLL_LOW;

	if (hdev->pm_mng_profile == PM_AUTO)
		hdev->asic_funcs->set_pll_profile(hdev, PLL_LOW);
	else
		hdev->asic_funcs->set_pll_profile(hdev, PLL_LAST);

	INIT_DELAYED_WORK(&hdev->work_freq, set_freq_to_low_job);
	schedule_delayed_work(&hdev->work_freq,
	usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC));

	if (hdev->heartbeat) {
		INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat);
		schedule_delayed_work(&hdev->work_heartbeat,
				usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
	}

	hdev->late_init_done = true;

	return 0;
}

/*
 * device_late_fini - finalize all that was done in device_late_init
 *
 * @hdev: pointer to habanalabs device structure
 *
 */
static void device_late_fini(struct hl_device *hdev)
{
	if (!hdev->late_init_done)
		return;

	cancel_delayed_work_sync(&hdev->work_freq);
	if (hdev->heartbeat)
		cancel_delayed_work_sync(&hdev->work_heartbeat);

	if (hdev->asic_funcs->late_fini)
		hdev->asic_funcs->late_fini(hdev);

	hdev->late_init_done = false;
}

int hl_device_utilization(struct hl_device *hdev, u32 *utilization)
{
	u64 max_power, curr_power, dc_power, dividend;
	int rc;

	max_power = hdev->asic_prop.max_power_default;
	dc_power = hdev->asic_prop.dc_power_default;
	rc = hl_fw_cpucp_power_get(hdev, &curr_power);

	if (rc)
		return rc;

	curr_power = clamp(curr_power, dc_power, max_power);

	dividend = (curr_power - dc_power) * 100;
	*utilization = (u32) div_u64(dividend, (max_power - dc_power));

	return 0;
}

/*
 * hl_device_set_frequency - set the frequency of the device
 *
 * @hdev: pointer to habanalabs device structure
 * @freq: the new frequency value
 *
 * Change the frequency if needed. This function has no protection against
 * concurrency, therefore it is assumed that the calling function has protected
 * itself against the case of calling this function from multiple threads with
 * different values
 *
 * Returns 0 if no change was done, otherwise returns 1
 */
int hl_device_set_frequency(struct hl_device *hdev, enum hl_pll_frequency freq)
{
	if ((hdev->pm_mng_profile == PM_MANUAL) ||
			(hdev->curr_pll_profile == freq))
		return 0;

	dev_dbg(hdev->dev, "Changing device frequency to %s\n",
		freq == PLL_HIGH ? "high" : "low");

	hdev->asic_funcs->set_pll_profile(hdev, freq);

	hdev->curr_pll_profile = freq;

	return 1;
}

int hl_device_set_debug_mode(struct hl_device *hdev, bool enable)
{
	int rc = 0;

	mutex_lock(&hdev->debug_lock);

	if (!enable) {
		if (!hdev->in_debug) {
			dev_err(hdev->dev,
				"Failed to disable debug mode because device was not in debug mode\n");
			rc = -EFAULT;
			goto out;
		}

		if (!hdev->hard_reset_pending)
			hdev->asic_funcs->halt_coresight(hdev);

		hdev->in_debug = 0;

		if (!hdev->hard_reset_pending)
			hdev->asic_funcs->set_clock_gating(hdev);

		goto out;
	}

	if (hdev->in_debug) {
		dev_err(hdev->dev,
			"Failed to enable debug mode because device is already in debug mode\n");
		rc = -EFAULT;
		goto out;
	}

	hdev->asic_funcs->disable_clock_gating(hdev);
	hdev->in_debug = 1;

out:
	mutex_unlock(&hdev->debug_lock);

	return rc;
}

/*
 * hl_device_suspend - initiate device suspend
 *
 * @hdev: pointer to habanalabs device structure
 *
 * Puts the hw in the suspend state (all asics).
 * Returns 0 for success or an error on failure.
 * Called at driver suspend.
 */
int hl_device_suspend(struct hl_device *hdev)
{
	int rc;

	pci_save_state(hdev->pdev);

	/* Block future CS/VM/JOB completion operations */
	rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
	if (rc) {
		dev_err(hdev->dev, "Can't suspend while in reset\n");
		return -EIO;
	}

	/* This blocks all other stuff that is not blocked by in_reset */
	hdev->disabled = true;

	/*
	 * Flush anyone that is inside the critical section of enqueue
	 * jobs to the H/W
	 */
	hdev->asic_funcs->hw_queues_lock(hdev);
	hdev->asic_funcs->hw_queues_unlock(hdev);

	/* Flush processes that are sending message to CPU */
	mutex_lock(&hdev->send_cpu_message_lock);
	mutex_unlock(&hdev->send_cpu_message_lock);

	rc = hdev->asic_funcs->suspend(hdev);
	if (rc)
		dev_err(hdev->dev,
			"Failed to disable PCI access of device CPU\n");

	/* Shut down the device */
	pci_disable_device(hdev->pdev);
	pci_set_power_state(hdev->pdev, PCI_D3hot);

	return 0;
}

/*
 * hl_device_resume - initiate device resume
 *
 * @hdev: pointer to habanalabs device structure
 *
 * Bring the hw back to operating state (all asics).
 * Returns 0 for success or an error on failure.
 * Called at driver resume.
 */
int hl_device_resume(struct hl_device *hdev)
{
	int rc;

	pci_set_power_state(hdev->pdev, PCI_D0);
	pci_restore_state(hdev->pdev);
	rc = pci_enable_device_mem(hdev->pdev);
	if (rc) {
		dev_err(hdev->dev,
			"Failed to enable PCI device in resume\n");
		return rc;
	}

	pci_set_master(hdev->pdev);

	rc = hdev->asic_funcs->resume(hdev);
	if (rc) {
		dev_err(hdev->dev, "Failed to resume device after suspend\n");
		goto disable_device;
	}


	hdev->disabled = false;
	atomic_set(&hdev->in_reset, 0);

	rc = hl_device_reset(hdev, HL_RESET_HARD);
	if (rc) {
		dev_err(hdev->dev, "Failed to reset device during resume\n");
		goto disable_device;
	}

	return 0;

disable_device:
	pci_clear_master(hdev->pdev);
	pci_disable_device(hdev->pdev);

	return rc;
}

static int device_kill_open_processes(struct hl_device *hdev, u32 timeout)
{
	struct hl_fpriv	*hpriv;
	struct task_struct *task = NULL;
	u32 pending_cnt;


	/* Giving time for user to close FD, and for processes that are inside
	 * hl_device_open to finish
	 */
	if (!list_empty(&hdev->fpriv_list))
		ssleep(1);

	if (timeout) {
		pending_cnt = timeout;
	} else {
		if (hdev->process_kill_trial_cnt) {
			/* Processes have been already killed */
			pending_cnt = 1;
			goto wait_for_processes;
		} else {
			/* Wait a small period after process kill */
			pending_cnt = HL_PENDING_RESET_PER_SEC;
		}
	}

	mutex_lock(&hdev->fpriv_list_lock);

	/* This section must be protected because we are dereferencing
	 * pointers that are freed if the process exits
	 */
	list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node) {
		task = get_pid_task(hpriv->taskpid, PIDTYPE_PID);
		if (task) {
			dev_info(hdev->dev, "Killing user process pid=%d\n",
				task_pid_nr(task));
			send_sig(SIGKILL, task, 1);
			usleep_range(1000, 10000);

			put_task_struct(task);
		}
	}

	mutex_unlock(&hdev->fpriv_list_lock);

	/*
	 * We killed the open users, but that doesn't mean they are closed.
	 * It could be that they are running a long cleanup phase in the driver
	 * e.g. MMU unmappings, or running other long teardown flow even before
	 * our cleanup.
	 * Therefore we need to wait again to make sure they are closed before
	 * continuing with the reset.
	 */

wait_for_processes:
	while ((!list_empty(&hdev->fpriv_list)) && (pending_cnt)) {
		dev_dbg(hdev->dev,
			"Waiting for all unmap operations to finish before hard reset\n");

		pending_cnt--;

		ssleep(1);
	}

	/* All processes exited successfully */
	if (list_empty(&hdev->fpriv_list))
		return 0;

	/* Give up waiting for processes to exit */
	if (hdev->process_kill_trial_cnt == HL_PENDING_RESET_MAX_TRIALS)
		return -ETIME;

	hdev->process_kill_trial_cnt++;

	return -EBUSY;
}

static void device_disable_open_processes(struct hl_device *hdev)
{
	struct hl_fpriv *hpriv;

	mutex_lock(&hdev->fpriv_list_lock);
	list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node)
		hpriv->hdev = NULL;
	mutex_unlock(&hdev->fpriv_list_lock);
}

/*
 * hl_device_reset - reset the device
 *
 * @hdev: pointer to habanalabs device structure
 * @flags: reset flags.
 *
 * Block future CS and wait for pending CS to be enqueued
 * Call ASIC H/W fini
 * Flush all completions
 * Re-initialize all internal data structures
 * Call ASIC H/W init, late_init
 * Test queues
 * Enable device
 *
 * Returns 0 for success or an error on failure.
 */
int hl_device_reset(struct hl_device *hdev, u32 flags)
{
	u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0};
	bool hard_reset, from_hard_reset_thread;
	int i, rc;

	if (!hdev->init_done) {
		dev_err(hdev->dev,
			"Can't reset before initialization is done\n");
		return 0;
	}

	hard_reset = (flags & HL_RESET_HARD) != 0;
	from_hard_reset_thread = (flags & HL_RESET_FROM_RESET_THREAD) != 0;

	if ((!hard_reset) && (!hdev->supports_soft_reset)) {
		dev_dbg(hdev->dev, "Doing hard-reset instead of soft-reset\n");
		hard_reset = true;
	}

	/* Re-entry of reset thread */
	if (from_hard_reset_thread && hdev->process_kill_trial_cnt)
		goto kill_processes;

	/*
	 * Prevent concurrency in this function - only one reset should be
	 * done at any given time. Only need to perform this if we didn't
	 * get from the dedicated hard reset thread
	 */
	if (!from_hard_reset_thread) {
		/* Block future CS/VM/JOB completion operations */
		rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
		if (rc)
			return 0;

		/*
		 * if reset is due to heartbeat, device CPU is no responsive in
		 * which case no point sending PCI disable message to it
		 */
		if (hard_reset && !(flags & HL_RESET_HEARTBEAT)) {
			/* Disable PCI access from device F/W so he won't send
			 * us additional interrupts. We disable MSI/MSI-X at
			 * the halt_engines function and we can't have the F/W
			 * sending us interrupts after that. We need to disable
			 * the access here because if the device is marked
			 * disable, the message won't be send. Also, in case
			 * of heartbeat, the device CPU is marked as disable
			 * so this message won't be sent
			 */
			if (hl_fw_send_pci_access_msg(hdev,
					CPUCP_PACKET_DISABLE_PCI_ACCESS))
				dev_warn(hdev->dev,
					"Failed to disable PCI access by F/W\n");
		}

		/* This also blocks future CS/VM/JOB completion operations */
		hdev->disabled = true;

		/* Flush anyone that is inside the critical section of enqueue
		 * jobs to the H/W
		 */
		hdev->asic_funcs->hw_queues_lock(hdev);
		hdev->asic_funcs->hw_queues_unlock(hdev);

		/* Flush anyone that is inside device open */
		mutex_lock(&hdev->fpriv_list_lock);
		mutex_unlock(&hdev->fpriv_list_lock);

		dev_err(hdev->dev, "Going to RESET device!\n");
	}

again:
	if ((hard_reset) && (!from_hard_reset_thread)) {
		hdev->hard_reset_pending = true;

		hdev->process_kill_trial_cnt = 0;

		/*
		 * Because the reset function can't run from interrupt or
		 * from heartbeat work, we need to call the reset function
		 * from a dedicated work
		 */
		queue_delayed_work(hdev->device_reset_work.wq,
			&hdev->device_reset_work.reset_work, 0);

		return 0;
	}

	if (hard_reset) {
		device_late_fini(hdev);

		/*
		 * Now that the heartbeat thread is closed, flush processes
		 * which are sending messages to CPU
		 */
		mutex_lock(&hdev->send_cpu_message_lock);
		mutex_unlock(&hdev->send_cpu_message_lock);
	}

	/*
	 * Halt the engines and disable interrupts so we won't get any more
	 * completions from H/W and we won't have any accesses from the
	 * H/W to the host machine
	 */
	hdev->asic_funcs->halt_engines(hdev, hard_reset);

	/* Go over all the queues, release all CS and their jobs */
	hl_cs_rollback_all(hdev);

	/* Release all pending user interrupts, each pending user interrupt
	 * holds a reference to user context
	 */
	hl_release_pending_user_interrupts(hdev);

kill_processes:
	if (hard_reset) {
		/* Kill processes here after CS rollback. This is because the
		 * process can't really exit until all its CSs are done, which
		 * is what we do in cs rollback
		 */
		rc = device_kill_open_processes(hdev, 0);

		if (rc == -EBUSY) {
			if (hdev->device_fini_pending) {
				dev_crit(hdev->dev,
					"Failed to kill all open processes, stopping hard reset\n");
				goto out_err;
			}

			/* signal reset thread to reschedule */
			return rc;
		}

		if (rc) {
			dev_crit(hdev->dev,
				"Failed to kill all open processes, stopping hard reset\n");
			goto out_err;
		}

		/* Flush the Event queue workers to make sure no other thread is
		 * reading or writing to registers during the reset
		 */
		flush_workqueue(hdev->eq_wq);
	}

	/* Reset the H/W. It will be in idle state after this returns */
	hdev->asic_funcs->hw_fini(hdev, hard_reset);

	if (hard_reset) {
		/* Release kernel context */
		if (hdev->kernel_ctx && hl_ctx_put(hdev->kernel_ctx) == 1)
			hdev->kernel_ctx = NULL;
		hl_vm_fini(hdev);
		hl_mmu_fini(hdev);
		hl_eq_reset(hdev, &hdev->event_queue);
	}

	/* Re-initialize PI,CI to 0 in all queues (hw queue, cq) */
	hl_hw_queue_reset(hdev, hard_reset);
	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
		hl_cq_reset(hdev, &hdev->completion_queue[i]);

	mutex_lock(&hdev->fpriv_list_lock);

	/* Make sure the context switch phase will run again */
	if (hdev->compute_ctx) {
		atomic_set(&hdev->compute_ctx->thread_ctx_switch_token, 1);
		hdev->compute_ctx->thread_ctx_switch_wait_token = 0;
	}

	mutex_unlock(&hdev->fpriv_list_lock);

	/* Finished tear-down, starting to re-initialize */

	if (hard_reset) {
		hdev->device_cpu_disabled = false;
		hdev->hard_reset_pending = false;

		if (hdev->kernel_ctx) {
			dev_crit(hdev->dev,
				"kernel ctx was alive during hard reset, something is terribly wrong\n");
			rc = -EBUSY;
			goto out_err;
		}

		rc = hl_mmu_init(hdev);
		if (rc) {
			dev_err(hdev->dev,
				"Failed to initialize MMU S/W after hard reset\n");
			goto out_err;
		}

		/* Allocate the kernel context */
		hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx),
						GFP_KERNEL);
		if (!hdev->kernel_ctx) {
			rc = -ENOMEM;
			hl_mmu_fini(hdev);
			goto out_err;
		}

		hdev->compute_ctx = NULL;

		rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
		if (rc) {
			dev_err(hdev->dev,
				"failed to init kernel ctx in hard reset\n");
			kfree(hdev->kernel_ctx);
			hdev->kernel_ctx = NULL;
			hl_mmu_fini(hdev);
			goto out_err;
		}
	}

	/* Device is now enabled as part of the initialization requires
	 * communication with the device firmware to get information that
	 * is required for the initialization itself
	 */
	hdev->disabled = false;

	rc = hdev->asic_funcs->hw_init(hdev);
	if (rc) {
		dev_err(hdev->dev,
			"failed to initialize the H/W after reset\n");
		goto out_err;
	}

	/* If device is not idle fail the reset process */
	if (!hdev->asic_funcs->is_device_idle(hdev, idle_mask,
			HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL)) {
		dev_err(hdev->dev,
			"device is not idle (mask %#llx %#llx) after reset\n",
			idle_mask[0], idle_mask[1]);
		rc = -EIO;
		goto out_err;
	}

	/* Check that the communication with the device is working */
	rc = hdev->asic_funcs->test_queues(hdev);
	if (rc) {
		dev_err(hdev->dev,
			"Failed to detect if device is alive after reset\n");
		goto out_err;
	}

	if (hard_reset) {
		rc = device_late_init(hdev);
		if (rc) {
			dev_err(hdev->dev,
				"Failed late init after hard reset\n");
			goto out_err;
		}

		rc = hl_vm_init(hdev);
		if (rc) {
			dev_err(hdev->dev,
				"Failed to init memory module after hard reset\n");
			goto out_err;
		}

		hl_set_max_power(hdev);
	} else {
		rc = hdev->asic_funcs->soft_reset_late_init(hdev);
		if (rc) {
			dev_err(hdev->dev,
				"Failed late init after soft reset\n");
			goto out_err;
		}
	}

	atomic_set(&hdev->in_reset, 0);
	hdev->needs_reset = false;

	dev_notice(hdev->dev, "Successfully finished resetting the device\n");

	if (hard_reset) {
		hdev->hard_reset_cnt++;

		/* After reset is done, we are ready to receive events from
		 * the F/W. We can't do it before because we will ignore events
		 * and if those events are fatal, we won't know about it and
		 * the device will be operational although it shouldn't be
		 */
		hdev->asic_funcs->enable_events_from_fw(hdev);
	} else {
		hdev->soft_reset_cnt++;
	}

	return 0;

out_err:
	hdev->disabled = true;

	if (hard_reset) {
		dev_err(hdev->dev,
			"Failed to reset! Device is NOT usable\n");
		hdev->hard_reset_cnt++;
	} else {
		dev_err(hdev->dev,
			"Failed to do soft-reset, trying hard reset\n");
		hdev->soft_reset_cnt++;
		hard_reset = true;
		goto again;
	}

	atomic_set(&hdev->in_reset, 0);

	return rc;
}

/*
 * hl_device_init - main initialization function for habanalabs device
 *
 * @hdev: pointer to habanalabs device structure
 *
 * Allocate an id for the device, do early initialization and then call the
 * ASIC specific initialization functions. Finally, create the cdev and the
 * Linux device to expose it to the user
 */
int hl_device_init(struct hl_device *hdev, struct class *hclass)
{
	int i, rc, cq_cnt, user_interrupt_cnt, cq_ready_cnt;
	char *name;
	bool add_cdev_sysfs_on_err = false;

	name = kasprintf(GFP_KERNEL, "hl%d", hdev->id / 2);
	if (!name) {
		rc = -ENOMEM;
		goto out_disabled;
	}

	/* Initialize cdev and device structures */
	rc = device_init_cdev(hdev, hclass, hdev->id, &hl_ops, name,
				&hdev->cdev, &hdev->dev);

	kfree(name);

	if (rc)
		goto out_disabled;

	name = kasprintf(GFP_KERNEL, "hl_controlD%d", hdev->id / 2);
	if (!name) {
		rc = -ENOMEM;
		goto free_dev;
	}

	/* Initialize cdev and device structures for control device */
	rc = device_init_cdev(hdev, hclass, hdev->id_control, &hl_ctrl_ops,
				name, &hdev->cdev_ctrl, &hdev->dev_ctrl);

	kfree(name);

	if (rc)
		goto free_dev;

	/* Initialize ASIC function pointers and perform early init */
	rc = device_early_init(hdev);
	if (rc)
		goto free_dev_ctrl;

	user_interrupt_cnt = hdev->asic_prop.user_interrupt_count;

	if (user_interrupt_cnt) {
		hdev->user_interrupt = kcalloc(user_interrupt_cnt,
				sizeof(*hdev->user_interrupt),
				GFP_KERNEL);

		if (!hdev->user_interrupt) {
			rc = -ENOMEM;
			goto early_fini;
		}
	}

	/*
	 * Start calling ASIC initialization. First S/W then H/W and finally
	 * late init
	 */
	rc = hdev->asic_funcs->sw_init(hdev);
	if (rc)
		goto user_interrupts_fini;

	/*
	 * Initialize the H/W queues. Must be done before hw_init, because
	 * there the addresses of the kernel queue are being written to the
	 * registers of the device
	 */
	rc = hl_hw_queues_create(hdev);
	if (rc) {
		dev_err(hdev->dev, "failed to initialize kernel queues\n");
		goto sw_fini;
	}

	cq_cnt = hdev->asic_prop.completion_queues_count;

	/*
	 * Initialize the completion queues. Must be done before hw_init,
	 * because there the addresses of the completion queues are being
	 * passed as arguments to request_irq
	 */
	if (cq_cnt) {
		hdev->completion_queue = kcalloc(cq_cnt,
				sizeof(*hdev->completion_queue),
				GFP_KERNEL);

		if (!hdev->completion_queue) {
			dev_err(hdev->dev,
				"failed to allocate completion queues\n");
			rc = -ENOMEM;
			goto hw_queues_destroy;
		}
	}

	for (i = 0, cq_ready_cnt = 0 ; i < cq_cnt ; i++, cq_ready_cnt++) {
		rc = hl_cq_init(hdev, &hdev->completion_queue[i],
				hdev->asic_funcs->get_queue_id_for_cq(hdev, i));
		if (rc) {
			dev_err(hdev->dev,
				"failed to initialize completion queue\n");
			goto cq_fini;
		}
		hdev->completion_queue[i].cq_idx = i;
	}

	/*
	 * Initialize the event queue. Must be done before hw_init,
	 * because there the address of the event queue is being
	 * passed as argument to request_irq
	 */
	rc = hl_eq_init(hdev, &hdev->event_queue);
	if (rc) {
		dev_err(hdev->dev, "failed to initialize event queue\n");
		goto cq_fini;
	}

	/* MMU S/W must be initialized before kernel context is created */
	rc = hl_mmu_init(hdev);
	if (rc) {
		dev_err(hdev->dev, "Failed to initialize MMU S/W structures\n");
		goto eq_fini;
	}

	/* Allocate the kernel context */
	hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL);
	if (!hdev->kernel_ctx) {
		rc = -ENOMEM;
		goto mmu_fini;
	}

	hdev->compute_ctx = NULL;

	hl_debugfs_add_device(hdev);

	/* debugfs nodes are created in hl_ctx_init so it must be called after
	 * hl_debugfs_add_device.
	 */
	rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
	if (rc) {
		dev_err(hdev->dev, "failed to initialize kernel context\n");
		kfree(hdev->kernel_ctx);
		goto remove_device_from_debugfs;
	}

	rc = hl_cb_pool_init(hdev);
	if (rc) {
		dev_err(hdev->dev, "failed to initialize CB pool\n");
		goto release_ctx;
	}

	/*
	 * From this point, override rc (=0) in case of an error to allow
	 * debugging (by adding char devices and create sysfs nodes as part of
	 * the error flow).
	 */
	add_cdev_sysfs_on_err = true;

	/* Device is now enabled as part of the initialization requires
	 * communication with the device firmware to get information that
	 * is required for the initialization itself
	 */
	hdev->disabled = false;

	rc = hdev->asic_funcs->hw_init(hdev);
	if (rc) {
		dev_err(hdev->dev, "failed to initialize the H/W\n");
		rc = 0;
		goto out_disabled;
	}

	/* Check that the communication with the device is working */
	rc = hdev->asic_funcs->test_queues(hdev);
	if (rc) {
		dev_err(hdev->dev, "Failed to detect if device is alive\n");
		rc = 0;
		goto out_disabled;
	}

	rc = device_late_init(hdev);
	if (rc) {
		dev_err(hdev->dev, "Failed late initialization\n");
		rc = 0;
		goto out_disabled;
	}

	dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n",
		hdev->asic_name,
		hdev->asic_prop.dram_size / 1024 / 1024 / 1024);

	rc = hl_vm_init(hdev);
	if (rc) {
		dev_err(hdev->dev, "Failed to initialize memory module\n");
		rc = 0;
		goto out_disabled;
	}

	/*
	 * Expose devices and sysfs nodes to user.
	 * From here there is no need to add char devices and create sysfs nodes
	 * in case of an error.
	 */
	add_cdev_sysfs_on_err = false;
	rc = device_cdev_sysfs_add(hdev);
	if (rc) {
		dev_err(hdev->dev,
			"Failed to add char devices and sysfs nodes\n");
		rc = 0;
		goto out_disabled;
	}

	/* Need to call this again because the max power might change,
	 * depending on card type for certain ASICs
	 */
	hl_set_max_power(hdev);

	/*
	 * hl_hwmon_init() must be called after device_late_init(), because only
	 * there we get the information from the device about which
	 * hwmon-related sensors the device supports.
	 * Furthermore, it must be done after adding the device to the system.
	 */
	rc = hl_hwmon_init(hdev);
	if (rc) {
		dev_err(hdev->dev, "Failed to initialize hwmon\n");
		rc = 0;
		goto out_disabled;
	}

	dev_notice(hdev->dev,
		"Successfully added device to habanalabs driver\n");

	hdev->init_done = true;

	/* After initialization is done, we are ready to receive events from
	 * the F/W. We can't do it before because we will ignore events and if
	 * those events are fatal, we won't know about it and the device will
	 * be operational although it shouldn't be
	 */
	hdev->asic_funcs->enable_events_from_fw(hdev);

	return 0;

release_ctx:
	if (hl_ctx_put(hdev->kernel_ctx) != 1)
		dev_err(hdev->dev,
			"kernel ctx is still alive on initialization failure\n");
remove_device_from_debugfs:
	hl_debugfs_remove_device(hdev);
mmu_fini:
	hl_mmu_fini(hdev);
eq_fini:
	hl_eq_fini(hdev, &hdev->event_queue);
cq_fini:
	for (i = 0 ; i < cq_ready_cnt ; i++)
		hl_cq_fini(hdev, &hdev->completion_queue[i]);
	kfree(hdev->completion_queue);
hw_queues_destroy:
	hl_hw_queues_destroy(hdev);
sw_fini:
	hdev->asic_funcs->sw_fini(hdev);
user_interrupts_fini:
	kfree(hdev->user_interrupt);
early_fini:
	device_early_fini(hdev);
free_dev_ctrl:
	put_device(hdev->dev_ctrl);
free_dev:
	put_device(hdev->dev);
out_disabled:
	hdev->disabled = true;
	if (add_cdev_sysfs_on_err)
		device_cdev_sysfs_add(hdev);
	if (hdev->pdev)
		dev_err(&hdev->pdev->dev,
			"Failed to initialize hl%d. Device is NOT usable !\n",
			hdev->id / 2);
	else
		pr_err("Failed to initialize hl%d. Device is NOT usable !\n",
			hdev->id / 2);

	return rc;
}

/*
 * hl_device_fini - main tear-down function for habanalabs device
 *
 * @hdev: pointer to habanalabs device structure
 *
 * Destroy the device, call ASIC fini functions and release the id
 */
void hl_device_fini(struct hl_device *hdev)
{
	ktime_t timeout;
	int i, rc;

	dev_info(hdev->dev, "Removing device\n");

	hdev->device_fini_pending = 1;
	flush_delayed_work(&hdev->device_reset_work.reset_work);

	/*
	 * This function is competing with the reset function, so try to
	 * take the reset atomic and if we are already in middle of reset,
	 * wait until reset function is finished. Reset function is designed
	 * to always finish. However, in Gaudi, because of all the network
	 * ports, the hard reset could take between 10-30 seconds
	 */

	timeout = ktime_add_us(ktime_get(),
				HL_HARD_RESET_MAX_TIMEOUT * 1000 * 1000);
	rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
	while (rc) {
		usleep_range(50, 200);
		rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
		if (ktime_compare(ktime_get(), timeout) > 0) {
			dev_crit(hdev->dev,
				"Failed to remove device because reset function did not finish\n");
			return;
		}
	}

	/* Disable PCI access from device F/W so it won't send us additional
	 * interrupts. We disable MSI/MSI-X at the halt_engines function and we
	 * can't have the F/W sending us interrupts after that. We need to
	 * disable the access here because if the device is marked disable, the
	 * message won't be send. Also, in case of heartbeat, the device CPU is
	 * marked as disable so this message won't be sent
	 */
	hl_fw_send_pci_access_msg(hdev,	CPUCP_PACKET_DISABLE_PCI_ACCESS);

	/* Mark device as disabled */
	hdev->disabled = true;

	/* Flush anyone that is inside the critical section of enqueue
	 * jobs to the H/W
	 */
	hdev->asic_funcs->hw_queues_lock(hdev);
	hdev->asic_funcs->hw_queues_unlock(hdev);

	/* Flush anyone that is inside device open */
	mutex_lock(&hdev->fpriv_list_lock);
	mutex_unlock(&hdev->fpriv_list_lock);

	hdev->hard_reset_pending = true;

	hl_hwmon_fini(hdev);

	device_late_fini(hdev);

	/*
	 * Halt the engines and disable interrupts so we won't get any more
	 * completions from H/W and we won't have any accesses from the
	 * H/W to the host machine
	 */
	hdev->asic_funcs->halt_engines(hdev, true);

	/* Go over all the queues, release all CS and their jobs */
	hl_cs_rollback_all(hdev);

	/* Kill processes here after CS rollback. This is because the process
	 * can't really exit until all its CSs are done, which is what we
	 * do in cs rollback
	 */
	dev_info(hdev->dev,
		"Waiting for all processes to exit (timeout of %u seconds)",
		HL_PENDING_RESET_LONG_SEC);

	rc = device_kill_open_processes(hdev, HL_PENDING_RESET_LONG_SEC);
	if (rc) {
		dev_crit(hdev->dev, "Failed to kill all open processes\n");
		device_disable_open_processes(hdev);
	}

	hl_cb_pool_fini(hdev);

	/* Reset the H/W. It will be in idle state after this returns */
	hdev->asic_funcs->hw_fini(hdev, true);

	/* Release kernel context */
	if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1))
		dev_err(hdev->dev, "kernel ctx is still alive\n");

	hl_debugfs_remove_device(hdev);

	hl_vm_fini(hdev);

	hl_mmu_fini(hdev);

	hl_eq_fini(hdev, &hdev->event_queue);

	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
		hl_cq_fini(hdev, &hdev->completion_queue[i]);
	kfree(hdev->completion_queue);
	kfree(hdev->user_interrupt);

	hl_hw_queues_destroy(hdev);

	/* Call ASIC S/W finalize function */
	hdev->asic_funcs->sw_fini(hdev);

	device_early_fini(hdev);

	/* Hide devices and sysfs nodes from user */
	device_cdev_sysfs_del(hdev);

	pr_info("removed device successfully\n");
}

/*
 * MMIO register access helper functions.
 */

/*
 * hl_rreg - Read an MMIO register
 *
 * @hdev: pointer to habanalabs device structure
 * @reg: MMIO register offset (in bytes)
 *
 * Returns the value of the MMIO register we are asked to read
 *
 */
inline u32 hl_rreg(struct hl_device *hdev, u32 reg)
{
	return readl(hdev->rmmio + reg);
}

/*
 * hl_wreg - Write to an MMIO register
 *
 * @hdev: pointer to habanalabs device structure
 * @reg: MMIO register offset (in bytes)
 * @val: 32-bit value
 *
 * Writes the 32-bit value into the MMIO register
 *
 */
inline void hl_wreg(struct hl_device *hdev, u32 reg, u32 val)
{
	writel(val, hdev->rmmio + reg);
}