/usr/src/file_protector-1.1-1497/transport/ Current File: //usr/src/file_protector-1.1-1497/transport/transport.c
/**
@file
@brief    Message transport between kernel and userspace
@details  Copyright (c) 2017-2021 Acronis International GmbH
@author   Mikhail Krivtsov (mikhail.krivtsov@acronis.com)
@since    $Id: $
*/

#include "transport.h"

#include "debug.h"
#include "device.h"
#include "memory.h"
#include "message.h"
#include "ring.h"
#include "set.h"
#include "syscall_common.h"
#include "task_info_map.h"
#include "tracepoints.h"

#include <linux/list.h>
#include <linux/jiffies.h>	// msecs_to_jiffies()
#include <linux/mutex.h>
#include <linux/sched.h>
#include <linux/spinlock.h>
#include <linux/uaccess.h>	// copy_from_user(), copy_to_user()
#include <linux/wait.h>		// wait_event*(), wake_up*()

#define TRANSPORT_MSG_SIZE_MAX (1<<10)
#define TRANSPORT_QUEUE_CAPACITY (0x1000 / sizeof(msg_t *))
#define TRANSPORT_WAIT_REPLY_TIMEOUT_MSECS (60*1000)

// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

typedef struct {
	struct list_head transport_list_node;

	pid_t control_tgid;

	// FIXME: Use 'msg_wait_queue.lock' instead of 'msg_spinlock'
	// #define msg_spinlock msg_wait_queue.lock
	spinlock_t msg_spinlock;
	wait_queue_head_t msg_wait_queue;
	bool shutdown;
	ring_t msg_ring;
	// sent messages waiting for 'reply'
	set_t sent_msgs_set;
} transport_t;

typedef struct {
	struct mutex transport_mutex;
	unsigned transport_count;

	spinlock_t transport_spinlock;
	struct list_head transport_list;
	msg_id_t msg_id_sequence;
} transport_global_t;

static transport_global_t transport_global;

static void transport_global_init(void)
{
	mutex_init(&transport_global.transport_mutex);
	transport_global.transport_count = 0;
	spin_lock_init(&transport_global.transport_spinlock);
	INIT_LIST_HEAD(&transport_global.transport_list);
	transport_global.msg_id_sequence = 0;
}

static void transport_global_register(transport_t *transport)
{
	spin_lock(&transport_global.transport_spinlock);
	list_add_tail(&transport->transport_list_node, &transport_global.transport_list);
	spin_unlock(&transport_global.transport_spinlock);
}

static void transport_global_unregister(transport_t *transport)
{
	spin_lock(&transport_global.transport_spinlock);
	if (!list_empty(&transport->transport_list_node)) {
		list_del_init(&transport->transport_list_node);
	}
	spin_unlock(&transport_global.transport_spinlock);
}

static bool transport_is_control_tgid_impl(pid_t tgid)
{
	transport_t *transport;
	bool is_control_tgid = false;
	list_for_each_entry(transport, &transport_global.transport_list, transport_list_node) {
		if (tgid == transport->control_tgid) {
			is_control_tgid = true;
			break;
		}
	}
	return is_control_tgid;
}

static bool transport_is_control_tgid(pid_t tgid)
{
	bool is_control_tgid;
	spin_lock(&transport_global.transport_spinlock);
	is_control_tgid = transport_is_control_tgid_impl(tgid);
	spin_unlock(&transport_global.transport_spinlock);
	return is_control_tgid;
}

static msg_id_t transport_global_sequence_next_impl(void)
{
	msg_id_t msg_id = ++transport_global.msg_id_sequence;
	if (!msg_id) {
		msg_id = ++transport_global.msg_id_sequence;
	}
	DPRINTF("msg_id=%llX", msg_id);
	return msg_id;
}

static msg_id_t transport_global_sequence_next(void)
{
	msg_id_t msg_id;
	spin_lock(&transport_global.transport_spinlock);
	msg_id = transport_global_sequence_next_impl();
	spin_unlock(&transport_global.transport_spinlock);
	return msg_id;
}

static void drop_msgs_impl(ring_t *ring)
{
	while (!ring_is_empty(ring)) {
		msg_t *msg = *(msg_t **) ring_consumer_ptr(ring);
		msg_unref(msg);
		ring_consumer_index_move_one(ring);
	}
}

/*
    'msg ref/unref' for messages stored in 'sent_msgs_set' are invoked in
    'msg_reply_wait_count inc/dec'.
    There is no need for separate 'msg ref/unref' calls.
*/
static void drop_sent_msgs_impl(set_t *set)
{
	void *item_ptr = set_begin_ptr(set);
	void *end_ptr = set_end_ptr(set);
	while (item_ptr < end_ptr) {
		msg_t *msg = *(msg_t **) item_ptr;
		msg_reply_wait_count_dec(msg);
		item_ptr = set_ptr_next(set, item_ptr);
	}
	set->count = 0;
}

static void transport_shutdown(transport_t *transport)
{
	DPRINTF("transport=%p", transport);
	spin_lock(&transport->msg_spinlock);
	{
		transport->shutdown = true;

		// Discard undelivered messages
		drop_msgs_impl(&transport->msg_ring);

		// Discard messages waiting for 'reply'
		drop_sent_msgs_impl(&transport->sent_msgs_set);
	}
	spin_unlock(&transport->msg_spinlock);

	// wakeup all userspace 'read' waiters
	wake_up_all(&transport->msg_wait_queue);
}

// identify and shutdown transport failed to reply
static void transport_shutdown_msg(transport_t *transport, msg_t *unreplied_msg)
{
	bool found = false;

	DPRINTF("transport=%p unreplied_msg=%p", transport, unreplied_msg);

	spin_lock(&transport->msg_spinlock);
	{
		void *item_ptr = set_begin_ptr(&transport->sent_msgs_set);
		void *end_ptr = set_end_ptr(&transport->sent_msgs_set);
		while (item_ptr < end_ptr) {
			if (unreplied_msg == *(msg_t **) item_ptr) {
				found = true;
				break;
			}
			item_ptr = set_ptr_next(&transport->sent_msgs_set, item_ptr);
		}
	}
	spin_unlock(&transport->msg_spinlock);

	if (found) {
		WPRINTF("deativating transport on reply wait timeout");
		transport_shutdown(transport);
	}
}

// identify and shutdown transport failed to reply
static void transport_global_shutdown_msg(msg_t *unreplied_msg)
{
	DPRINTF("unreplied_msg=%p", unreplied_msg);
	spin_lock(&transport_global.transport_spinlock);
	{
		transport_t *transport;
		transport_t *next_transport;
		list_for_each_entry_safe(transport, next_transport,
				&transport_global.transport_list,
				transport_list_node) {
			transport_shutdown_msg(transport, unreplied_msg);
		}
	}
	spin_unlock(&transport_global.transport_spinlock);
}

static void transport_free(transport_t *transport)
{
	DPRINTF("transport=%p", transport);
	transport_global_unregister(transport);
	transport_shutdown(transport);

	IPRINTF("message queue items_count_max=%u capacity=%u",
			ring_items_count_max(&transport->msg_ring),
			ring_capacity(&transport->msg_ring));
	IPRINTF("sent_msgs_set items_count_max=%u capacity=%u",
			set_items_count_max(&transport->sent_msgs_set),
			set_capacity(&transport->sent_msgs_set));

	mem_free(ring_buffer(&transport->msg_ring));
	mem_free(set_buffer(&transport->sent_msgs_set));

	mem_free(transport);
}

static bool transport_ring_init(ring_t *ring)
{
	size_t buffer_size = TRANSPORT_QUEUE_CAPACITY * sizeof(msg_t *);
	msg_t **msgs;
	bool success;
	if (!buffer_size) {
		msgs = NULL;
		success = true;
	} else {
		msgs = mem_alloc0(buffer_size);
		success = (bool) msgs;
	}
	ring_init(ring, msgs, buffer_size, sizeof(msg_t *));
	return success;
}

static bool transport_set_init(set_t *set)
{
	size_t buffer_size = TRANSPORT_QUEUE_CAPACITY * sizeof(msg_t *);
	msg_t **msgs;
	bool success;
	if (!buffer_size) {
		msgs = NULL;
		success = true;
	} else {
		msgs = mem_alloc0(buffer_size);
		success = (bool) msgs;
	}
	set_init(set, msgs, buffer_size, sizeof(msg_t *));
	return success;
}

static bool transport_send_msg_nowait(transport_t *transport, msg_t *msg)
{
	spin_lock(&transport->msg_spinlock);
	{
		if (transport->shutdown) {
			spin_unlock(&transport->msg_spinlock);
			return false;
		}

		if (msg_img_is_reply_required(MSG_IMG(msg))) {
			unsigned item_index;
			if (set_is_full(&transport->sent_msgs_set)) {
				WPRINTF("'sent_msgs_set' overflow (capacity=%u)", set_capacity(&transport->sent_msgs_set));
				spin_unlock(&transport->msg_spinlock);
				transport_shutdown(transport);
				return false;
			}
			item_index = set_items_count(&transport->sent_msgs_set);
			/*
			    'msg ref/unref' for messages stored in 'sent_msgs_set' are invoked in
			    'msg_reply_wait_count inc/dec'.
			    There is no need for separate 'msg ref/unref' calls.
			*/
			*(msg_t **) set_item_ptr(&transport->sent_msgs_set, item_index) = msg_reply_wait_count_inc(msg);
			set_items_count_set(&transport->sent_msgs_set, item_index + 1);
		}

		if (ring_is_full(&transport->msg_ring)) {
			WPRINTF("message queue overflow (capacity=%u)", ring_capacity(&transport->msg_ring));
			spin_unlock(&transport->msg_spinlock);
			transport_shutdown(transport);
			return false;
		}

		*(msg_t **) ring_producer_ptr(&transport->msg_ring) = msg_ref(msg);
		ring_producer_index_move_one(&transport->msg_ring);
	}
	spin_unlock(&transport->msg_spinlock);

	// wakeup userspace reader
	wake_up_interruptible_sync(&transport->msg_wait_queue);

	return true;
}

static bool transport_send_hello_nowait(transport_t *transport)
{
	msg_t *msg = hello_msg_new();
	bool success;
	if (!msg) {
		success = false;
	} else {
		success = transport_send_msg_nowait(transport, msg);
		msg_unref(msg);
	}
	return success;
}

static bool send_msg_nowait(msg_t *msg)
{
	bool sent = false;
	spin_lock(&transport_global.transport_spinlock);
	if (!transport_is_control_tgid_impl(current->tgid)) {
		transport_t *transport;
		transport_t *next_transport;
		list_for_each_entry_safe(transport, next_transport,
				&transport_global.transport_list,
				transport_list_node) {
			sent |= transport_send_msg_nowait(transport, msg);
		}
	}
	spin_unlock(&transport_global.transport_spinlock);
	return sent;
}

static transport_t *transport_new(void)
{
	transport_t *transport = mem_alloc0(sizeof(transport_t));
	if (transport) {
		INIT_LIST_HEAD(&transport->transport_list_node);

		// remember client's process doing 'open' to auto-ignore it
		transport->control_tgid = current->tgid;

		spin_lock_init(&transport->msg_spinlock);
		init_waitqueue_head(&transport->msg_wait_queue);
		transport->shutdown = false;

		if (transport_ring_init(&transport->msg_ring)
		&& transport_set_init(&transport->sent_msgs_set)
		&& 0 == task_info_status_set(current->tgid, TS_IGNORE)
		&& transport_send_hello_nowait(transport)) {
			transport_global_register(transport);
		} else {
			transport_free(transport);
			transport = NULL;
		}
	}
	DPRINTF("transport=%p", transport);
	return transport;
}

// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

int __init transport_mod_init(void)
{
	int ret;

	transport_global_init();

	ret = device_mod_init();
	if (ret) {
		EPRINTF("'device_mod_init()' failure %i", ret);
	}
	return ret;
}

void transport_mod_down(void)
{
	DPRINTF("");
	device_mod_down();
	DPRINTF("");
}

// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

static long transport_ioctl_pid_info(msg_t **reply_msg, msg_t *query_msg)
{
	long ret;

	if (query_msg->img_size < (sizeof(msg_img_t) + sizeof(get_pid_info_img_t))) {
		EPRINTF("'%s' message is too short", msg_type_to_string(MSG_TYPE(query_msg)));
		ret = -EINVAL;
	} else {
		msg_img_t *msg_img = MSG_IMG(query_msg);
		get_pid_info_img_t *img = IMG_PAYLOAD(msg_img);
		pid_t pid = img->pid;
		ret = pid_info_msg_new(reply_msg, pid);
	}
	DPRINTF("ret=%li", ret);
	return ret;
}

static long transport_ioctl_fs_root(msg_t **reply_msg, msg_t *query_msg)
{
	long ret;

	if (query_msg->img_size < (sizeof(msg_img_t) + sizeof(get_fs_root_img_t))) {
		EPRINTF("'%s' message is too short", msg_type_to_string(MSG_TYPE(query_msg)));
		ret = -EINVAL;
	} else {
		msg_img_t *msg_img = MSG_IMG(query_msg);
		get_fs_root_img_t *img = IMG_PAYLOAD(msg_img);
		pid_t pid = img->pid;
		ret = fs_root_msg_new(reply_msg, pid);
	}
	DPRINTF("ret=%li", ret);
	return ret;
}

static long transport_ioctl_write_read_msg(msg_t **reply_msg, msg_t *query_msg)
{
	long ret;
	msg_type_t msg_type;

	if (MSG_REPLY(query_msg)) {
		EPRINTF("'reply' ioctl is not supported");
		ret = -EINVAL;
		goto out;
	}

	msg_type = MSG_TYPE(query_msg);
	switch (msg_type) {
	case MT_GET_PID_INFO:
		ret = transport_ioctl_pid_info(reply_msg, query_msg);
		break;

	case MT_GET_FS_ROOT:
		ret = transport_ioctl_fs_root(reply_msg, query_msg);
		break;

	default:
		EPRINTF("Unexpected '%s' message", msg_type_to_string(msg_type));
		HEX_DUMP("query_msg: ", MSG_IMG(query_msg), MSG_SIZE(query_msg));
		ret = -EINVAL;
		break;
	}
out:
	DPRINTF("ret=%li", ret);
	return ret;
}

static long transport_ioctl_copy_from_user(ioctl_hdr_t *ioctl_hdr,
		msg_t **query_msg, void __user *user_data)
{
	long ret;

	size_t msg_size;
	msg_t *msg;
	msg_img_t *msg_img;
	void *payload;

	if (copy_from_user(ioctl_hdr, user_data, sizeof(ioctl_hdr_t))) {
		EPRINTF("'copy_from_user()' failure");
		ret = -EFAULT;
		goto out;
	}
	msg_size = ioctl_hdr->size;
	if (msg_size < sizeof(msg_img_t)) {
		EPRINTF("message image is too small");
		ret = -EINVAL;
		goto out;
	}
	if (msg_size > TRANSPORT_MSG_SIZE_MAX) {
		EPRINTF("size > TRANSPORT_MSG_SIZE_MAX");
		ret = -E2BIG;
		goto out;
	}
	msg = msg_new(msg_size);
	if (!msg) {
		ret = -ENOMEM;
		goto out;
	}
	msg_img = MSG_IMG(msg);
	payload = (uint8_t *)user_data + sizeof(ioctl_hdr_t);
	if (copy_from_user(msg_img, payload, msg_size)) {
		msg_unref(msg);
		EPRINTF("'copy_from_user()' failure");
		ret = -EFAULT;
		goto out;
	}
	*query_msg = msg;
	ret = 0;
out:
	DPRINTF("ret=%li", ret);
	return ret;
}

static long transport_ioctl_copy_to_user(ioctl_hdr_t *ioctl_hdr,
		msg_t *reply_msg, void __user *user_data)
{
	long ret;

	size_t msg_size = MSG_SIZE(reply_msg);
	size_t capacity;
	void *payload;
	msg_img_t *msg_img;

	ioctl_hdr->size = msg_size;
	if (copy_to_user(user_data, ioctl_hdr, sizeof(ioctl_hdr_t))) {
		EPRINTF("'copy_to_user()' failure");
		ret = -EFAULT;
		goto out;
	}
	capacity = ioctl_hdr->capacity;
	if (capacity < msg_size) {
		WPRINTF("capacity=%zu < msg_size=%zu", capacity, msg_size);
		ret = -ENOSPC;
		goto out;
	}
	payload = (uint8_t *)user_data + sizeof(ioctl_hdr_t);
	msg_img = MSG_IMG(reply_msg);
	if (copy_to_user(payload, msg_img, msg_size)) {
		EPRINTF("'copy_to_user()' failure");
		ret = -EFAULT;
		goto out;
	}
	ret = 0;
out:
	DPRINTF("ret=%li", ret);
	return ret;
}

long transport_device_ioctl(struct file *filp, unsigned int cmd,
		unsigned long arg)
{
	transport_t *transport = filp->private_data;
	long ret;
	if (transport->shutdown) {
		ret = -EIO;
		goto out;
	}
	switch (cmd) {
	case IOCTL_WRITE_AND_READ_MSG:
	{
		ioctl_hdr_t ioctl_hdr;
		void *user_data = (void *)arg;
		msg_t *query_msg;
		ret = transport_ioctl_copy_from_user(&ioctl_hdr, &query_msg, user_data);
		if (!ret) {
			msg_t *reply_msg;
			ret = transport_ioctl_write_read_msg(&reply_msg, query_msg);
			if (!ret) {
				ret = transport_ioctl_copy_to_user(&ioctl_hdr, reply_msg, user_data);
				msg_unref(reply_msg);
			}
			msg_unref(query_msg);
		}
		break;
	}
	default:
		EPRINTF("Unexpected IOCTL cmd=%u", cmd);
		ret = -ENOIOCTLCMD;
	}
out:
	if (-EINVAL == ret) {
		transport_shutdown(transport);
	}
	DPRINTF("ret=%li", ret);
	return ret;
}

// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

ssize_t transport_device_read(struct file *filp, char __user *user_data,
			      size_t size, loff_t *offset)
{
	msg_t *msg;
	transport_t *transport = filp->private_data;

	size_t img_size;
	ssize_t ret;

	if (filp->f_flags & O_NONBLOCK) {
		EPRINTF("'non-blocking' mode is not supported yet");
		ret = -EINVAL;
		transport_shutdown(transport);
		goto out;
	}

	if (!size) {
		EPRINTF("'empty read' is not supported");
		ret = -EINVAL;
		transport_shutdown(transport);
		goto out;
	}

	task_info_map_delete_exited();

retry_wait:
	// We may start with 'wait*()' because it itself starts
	// with 'condition' check.
	if (wait_event_interruptible_exclusive(transport->msg_wait_queue,
			transport->shutdown
			|| !ring_is_empty(&transport->msg_ring))) {
		ret = -EINTR;
		goto out;
	}

	// Lock the state and check if processing is actually possible.
	spin_lock(&transport->msg_spinlock);
	{
		if (transport->shutdown) {
			ret = -EIO;
			spin_unlock(&transport->msg_spinlock);
			goto out;
		}

		if (ring_is_empty(&transport->msg_ring)) {
			WPRINTF("wakeup without messages");
			spin_unlock(&transport->msg_spinlock);
			goto retry_wait;
		}
		msg = *(msg_t **) ring_consumer_ptr(&transport->msg_ring);
		img_size = msg->img_size;
		DPRINTF("size=%zu img_size=%zu", size, img_size);
		if (size < img_size) {
			ret = -ENOSPC;
			spin_unlock(&transport->msg_spinlock);
			goto out;
		}
		ring_consumer_index_move_one(&transport->msg_ring);
	}
	spin_unlock(&transport->msg_spinlock);

	// 'copy_to_user' MAY sleep (for example in page fault handler)
	if (copy_to_user(user_data, &msg->img, img_size)) {
		WPRINTF("'copy_to_user()' failure");
		ret = -EFAULT;
		transport_shutdown(transport);
	} else {
		ret = img_size;
	}
	msg_unref(msg);
out:
	DPRINTF("ret=%zi", ret);
	return ret;
}

// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

static void msg_wait_reply(msg_t *msg)
{
	long ret;

	// Do not block 'control' process
	if (atomic_read(&msg->reply_wait_count)
	&& transport_is_control_tgid(current->tgid)) {
		WPRINTF_RATELIMITED("avoiding blocking wait in control process");
		return;
	}

	// We may start with 'wait*()' because it itself starts
	// with 'condition' check.
	DPRINTF("waiting for userspace reply...");
	ret = wait_event_interruptible_timeout(msg->wait_queue,
		!atomic_read(&msg->reply_wait_count),
		msecs_to_jiffies(TRANSPORT_WAIT_REPLY_TIMEOUT_MSECS));
	if (!ret) {
		// Timeout here means unexpected issue with userspace.
		FPRINTF("timeout waiting for userspace reply (msg_type=%i/%s)",
				MSG_TYPE(msg),
				msg_type_to_string(MSG_TYPE(msg)));
		HEX_DUMP("msg: ", MSG_IMG(msg), MSG_SIZE(msg));
		dump_stack();
		// identify and shutdown transport failed to reply
		transport_global_shutdown_msg(msg);
	} else if (ret < 0) {
		// Calling process has been interrupted.
		// Currently nothing is done here.
		// FIXME: What actually shall be done here?
		DPRINTF("interrupted waiting for userspace reply");
		msg->interrupted = true;
	} else {
		// Userspace reply has been received (msg->reply_msg) or
		// waiting has been explicitly aborted (msg->aborted) for
		// example on userspace disconnect.
		DPRINTF("wait finished (msg->block=%i)", msg->block);
	}
}

static void msg_mark_async(msg_t *msg)
{
	MSG_ID(msg) = 0;
	MSG_REPLY(msg) = false;
}

void send_msg_async(msg_t *msg)
{
	DPRINTF("msg=%p", msg);
	msg_mark_async(msg);
	send_msg_nowait(msg);
	DPRINTF("");
}

static void msg_mark_sync(msg_t *msg)
{
	MSG_ID(msg) = transport_global_sequence_next();
	MSG_REPLY(msg) = false;
}

bool send_msg_sync_nowait(msg_t *msg)
{
	bool sent;
	DPRINTF("msg=%p", msg);
	msg_mark_sync(msg);
	sent = send_msg_nowait(msg);
	DPRINTF("msg=%p sent=%i", msg, sent);
	return sent;
}

void send_msg_sync(msg_t *msg)
{
	DPRINTF("msg=%p", msg);
	if (send_msg_sync_nowait(msg)) {
		msg_wait_reply(msg);
	}
	DPRINTF("");
}

void send_msg_sync_unref(msg_t *msg)
{
	if (msg) {
		send_msg_sync(msg);
		msg_unref(msg);
	}
}

static void msg_mark_reply(msg_t *reply, msg_id_t msg_id)
{
	MSG_ID(reply) = msg_id;
	MSG_REPLY(reply) = true;
}

static void transport_send_reply(transport_t *transport, msg_t *query,
				 msg_t *reply)
{
	DPRINTF("reply=%p", reply);
	msg_mark_reply(reply, MSG_ID(query));
	transport_send_msg_nowait(transport, reply);
	DPRINTF("");
}

// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

static int transport_handle_ping_msg(transport_t *transport, msg_t *ping)
{
	int ret;
	msg_t *pong;

	if (ping->img_size < (sizeof(msg_img_t) + sizeof(ping_img_t))) {
		DPRINTF("'ping' message is too short. ignoring it.");
		ret = -EINVAL;
		goto out;
	}

	if (MSG_ID(ping)) {
		// 'reply' is necessary
		msg_t *reply = ping_reply_msg_new(ping);
		if (!reply) {
			ret = -ENOMEM;
			goto out;
		}
		transport_send_reply(transport, ping, reply);
		msg_unref(reply);
	}

	// 'pong' is always necessary
	pong = pong_msg_new(ping);
	if (!pong) {
		ret = -ENOMEM;
		goto out;
	}
	// reflect ping's 'reply' policy
	if (MSG_ID(ping)) {
		msg_mark_sync(pong);
	} else {
		msg_mark_async(pong);
	}
	transport_send_msg_nowait(transport, pong);
	msg_unref(pong);
	ret = 0;
out:
	return ret;
}

static int transport_handle_set_pid_status_msg(msg_t *msg)
{
	msg_img_t *msg_img;
	pid_set_st_img_t *img;
	pid_t pid;
	task_status_t status;
	int ret;

	if (msg->img_size < (sizeof(msg_img_t) + sizeof(pid_set_st_img_t))) {
		DPRINTF("'pid' message is too short. ignoring it.");
		ret = -EINVAL;
		goto out;
	}

	msg_img = MSG_IMG(msg);
	img = IMG_PAYLOAD(msg_img);
	pid = img->pid;
	status = img->status;

	ret = task_info_status_set(pid, status);
out:
	DPRINTF("ret=%i", ret);
	return ret;
}

static int transport_handle_del_pid_msg(msg_t *msg)
{
	msg_img_t *msg_img;
	pid_del_img_t *img;
	pid_t pid;
	int ret;

	if (msg->img_size < (sizeof(msg_img_t) + sizeof(pid_del_img_t))) {
		DPRINTF("'pid' message is too short. ignoring it.");
		ret = -EINVAL;
		goto out;
	}

	msg_img = MSG_IMG(msg);
	img = IMG_PAYLOAD(msg_img);
	pid = img->pid;

	ret = task_info_map_del(pid);
out:
	DPRINTF("ret=%i", ret);
	return ret;
}

// TODO: Use 'ioctl' for MT_GET_PID_INFO to make it synchronous
static int transport_handle_get_pid_info_msg(transport_t *transport,
		msg_t *query_msg)
{
	int ret;
	msg_t *pid_info_msg;

	WPRINTF("TODO: Use 'ioctl' for MT_GET_PID_INFO");
	ret = transport_ioctl_pid_info(&pid_info_msg, query_msg);
	if (!ret) {
		msg_mark_async(pid_info_msg);
		transport_send_msg_nowait(transport, pid_info_msg);
		msg_unref(pid_info_msg);
	}
	DPRINTF("ret=%i", ret);
	return ret;
}

// FIXME: do something with 'reply'. For example merge several replies
// into one; link replies into list; extract 'responces' and merge them.
static void handle_reply(msg_t *query_msg, msg_t *reply_msg)
{
	// handle 'long' 'reply'
	size_t headers_size = sizeof(msg_img_t) + sizeof(reply_img_t);
	// Note: for compatibility with legacy short 'reply_img_t' default 'reply_type' is RT_ALLOW
	if (MSG_SIZE(reply_msg) >= headers_size) {
		msg_img_t *reply_msg_img = MSG_IMG(reply_msg);
		reply_img_t *reply_img = IMG_PAYLOAD(reply_msg_img);
		reply_type_t reply_type = reply_img->type;
		DPRINTF("MSG_SIZE(reply_msg)=%zu - headers_size=%zu = %zu reply_type=%u",
				MSG_SIZE(reply_msg), headers_size,
				MSG_SIZE(reply_msg) - headers_size, reply_type);
		if (RT_BLOCK == reply_type) {
			query_msg->block = true;
		}
	}
}

static int transport_handle_reply(transport_t *transport, msg_t *reply)
{
	int ret;
	msg_id_t reply_id = MSG_ID(reply);
	msg_type_t reply_type = MSG_TYPE(reply);

	DPRINTF("");
	// find 'query' matching this 'reply'
	spin_lock(&transport->msg_spinlock);
	{
		void *item_ptr = set_begin_ptr(&transport->sent_msgs_set);
		void *end_ptr = set_end_ptr(&transport->sent_msgs_set);
		while (item_ptr < end_ptr) {
			msg_t *query = *(msg_t **) item_ptr;
			if (MSG_ID(query) == reply_id
			&& MSG_TYPE(query) == reply_type) {
				// remove 'query' from 'set'
				*(msg_t **) item_ptr = *(msg_t **) set_item_ptr(
						&transport->sent_msgs_set,
						set_items_count_dec(&transport->sent_msgs_set));

				handle_reply(query, reply);

				msg_reply_wait_count_dec(query);
				ret = 0;
				goto unlock;
			}
			item_ptr = set_ptr_next(&transport->sent_msgs_set, item_ptr);
		}
		WPRINTF("Unexpected 'reply' with type=%i id=%llX", reply_type, reply_id);
		ret = -EINVAL;
	}
unlock:
	spin_unlock(&transport->msg_spinlock);
	DPRINTF("ret=%i", ret);
	return ret;
}

static int transport_handle_msg(transport_t *transport, msg_t *msg)
{
	int ret;

	if (msg->img_size < sizeof(msg_img_t)) {
		DPRINTF("message image is too small");
		ret = -EINVAL;
		goto out;
	}

	if (MSG_REPLY(msg)) {
		ret = transport_handle_reply(transport, msg);
	} else { // !reply
		msg_type_t type = MSG_TYPE(msg);
		switch (type) {
		case MT_PING:
			ret = transport_handle_ping_msg(transport, msg);
			break;

		case MT_PID_SET_ST:
			ret = transport_handle_set_pid_status_msg(msg);
			break;

		case MT_PID_DEL:
			ret = transport_handle_del_pid_msg(msg);
			break;

		case MT_GET_PID_INFO:
			ret = transport_handle_get_pid_info_msg(transport, msg);
			break;

		case MT_HELLO:
		case MT_PONG:
		default:
			WPRINTF("Unexpected message type=%i/%s", type, msg_type_to_string(type));
			ret = -EINVAL;
		}
	}

out:
	DPRINTF("ret=%i", ret);
	return ret;
}

ssize_t transport_device_write(struct file *filp, const char __user *user_data,
			       size_t size, loff_t *offset)
{
	transport_t *transport = filp->private_data;
	msg_t *msg;
	msg_img_t *msg_img;
	ssize_t ret;

	if (transport->shutdown) {
		ret = -EIO;
		goto out;
	}

	if (filp->f_flags & O_NONBLOCK) {
		EPRINTF("'non-blocking' mode is not supported yet");
		ret = -EINVAL;
		transport_shutdown(transport);
		goto out;
	}

	if (!size) {
		WPRINTF("'zero write' is not supported");
		ret = -EINVAL;
		transport_shutdown(transport);
		goto out;
	}
	if (size > TRANSPORT_MSG_SIZE_MAX) {
		WPRINTF("size > TRANSPORT_MSG_SIZE_MAX");
		ret = -E2BIG;
		goto out;
	}
	msg = msg_new(size);
	if (!msg) {
		ret = -ENOMEM;
		goto out;
	}
	msg_img = MSG_IMG(msg);
	if (copy_from_user(msg_img, user_data, size)) {
		EPRINTF("'copy_from_user()' failure");
		ret = -EFAULT;
		transport_shutdown(transport);
		goto free_msg;
	}
	ret = transport_handle_msg(transport, msg);
	if (ret) {
		// make sure error code is negative
		if (ret > 0) {
			EPRINTF("error code must be negative");
			ret = -ret;
		}
		goto free_msg;
	}
	ret = size;
free_msg:
	msg_unref(msg);
out:
	DPRINTF("ret=%zi", ret);
	return ret;
}

// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

/*
    Warning: 'transport_open()' and 'transport_release()' may be
    simultaneously invoked by several threads or processes.

    Note: We can match different 'transport' instances using 'device'
    'major'/'minor' from 'inode->i_rdev'. Pointer to selected 'trasport'
    can be stored in 'filp->private_data' for later use in '*_read()',
    '*_write()', etc.

    Note: We may create 'transport' on 'first open' and destroy it on
    'last close'.
*/
/*
    There is possibility of 'deadlock' between our 'kernel' and
    'userspace' code while processing events generated by our userspace
    process until registration of our userspace process in 'ignore' list.
*/
int transport_device_open(struct inode *inode, struct file *filp)
{
	transport_t *transport;
	int ret;

	DPRINTF("inode->i_rdev: major=%u minor=%u", imajor(inode), iminor(inode));
	DPRINTF("filp->f_flags=%X", filp->f_flags);
	if (filp->f_flags & O_NONBLOCK) {
		EPRINTF("'non-blocking' mode is not supported yet");
		ret = -EINVAL;
		goto out;
	}

	mutex_lock(&transport_global.transport_mutex);
	{
		DPRINTF("transport_count=%u", transport_global.transport_count);

		transport = transport_new();
		if (!transport) {
			WPRINTF("'%s()' failure", "transport_new");
			ret = -ENOMEM;
			goto unlock_open_close_mutex;
		}
		filp->private_data = transport;

		if (!transport_global.transport_count) {
			// FIXME: 'attach' may fail
			IPRINTF("attaching interceptors");
			ret = syscall_hooks_attach();
			if (ret) {
				EPRINTF("'%s()' failure %i", "syscall_hooks_attach", ret);
				goto unlock_open_close_mutex;
			}
			ret = tracepoints_attach();
			if (ret) {
				EPRINTF("'%s()' failure %i", "tracepoints_attach", ret);
				syscall_hooks_detach();
				goto unlock_open_close_mutex;
			}
			IPRINTF("interceptors attached");
		}
		++transport_global.transport_count;
		ret = 0;
	}
unlock_open_close_mutex:
	mutex_unlock(&transport_global.transport_mutex);
out:
	DPRINTF("ret=%i", ret);
	return ret;
}

// 'release()' means 'close()'
int transport_device_release(struct inode *inode, struct file *filp)
{
	mutex_lock(&transport_global.transport_mutex);
	{
		transport_t *transport = filp->private_data;
		transport_shutdown(transport);
		transport_free(transport);

		DPRINTF("transport_count=%u", transport_global.transport_count);
		if (!--transport_global.transport_count) {
			IPRINTF("detaching interceptors");
			tracepoints_detach();
			// FIXME: 'syscall_hooks_detach()' may fail
			syscall_hooks_detach();
			task_info_map_clear();
			IPRINTF("interceptors detached");
		}
	}
	mutex_unlock(&transport_global.transport_mutex);
	return 0;
}