ANDROID: mm/memfd-ashmem-shim: Introduce shim layer

Certain applications treat any shared memory buffer that they obtain
as an ashmem buffer, meaning that they will attempt to invoke ashmem
ioctl commands on that buffer.

Android is transitioning to replacing ashmem with memfd, and memfd
currently does not support ashmem ioctl commands. So, when an
application attempts to invoke an ashmem ioctl command on a memfd,
the invocation will fail and report an error back to the app.

In order to preserve compatibility between these apps and memfds,
add a shim layer which will handle ashmem ioctl commands for memfds.

This also folds in the following commits from the android14-6.1 branch:

1. ANDROID: mm/memfd-ashmem-shim: Fix variable length array usage
2. ANDROID: mm/memfd-ashmem-shim: Simplify buffer name retrieval

Bug: 111903542
Bug: 415769373
Change-Id: I268a29ee2805739550d79fd2c21d3cfb5a852642
[isaacmanjarres: resolved trivial merge conflicts in mm/Kconfig
and folded in fixes/simplifications that were merged into the
android14-6.1 branch after the initial commit landed.]
Signed-off-by: Isaac J. Manjarres <isaacmanjarres@google.com>

mm/Kconfig

@@ -1316,6 +1316,17 @@ config LOCK_MM_AND_FIND_VMA
 	bool
 	depends on !STACK_GROWSUP
 
+config MEMFD_ASHMEM_SHIM
+	bool "Memfd ashmem ioctl compatibility support"
+	depends on MEMFD_CREATE
+	help
+	  This provides compatibility support for ashmem ioctl commands against
+	  memfd file descriptors. This is useful for compatibility on Android
+	  for older applications that may use ashmem's ioctl commands on the
+	  now memfds passed to them.
+
+	  Unless you are running Android, say N.
+
 source "mm/damon/Kconfig"
 
 endmenu

mm/Makefile

@@ -138,6 +138,7 @@ obj-$(CONFIG_PERCPU_STATS) += percpu-stats.o
 obj-$(CONFIG_ZONE_DEVICE) += memremap.o
 obj-$(CONFIG_HMM_MIRROR) += hmm.o
 obj-$(CONFIG_MEMFD_CREATE) += memfd.o
+obj-$(CONFIG_MEMFD_ASHMEM_SHIM) += memfd-ashmem-shim.o
 obj-$(CONFIG_MAPPING_DIRTY_HELPERS) += mapping_dirty_helpers.o
 obj-$(CONFIG_PTDUMP_CORE) += ptdump.o
 obj-$(CONFIG_PAGE_REPORTING) += page_reporting.o

mm/memfd-ashmem-shim-internal.h

@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Ashmem compatability for memfd
+ *
+ * Copyright (c) 2025, Google LLC.
+ * Author: Isaac J. Manjarres <isaacmanjarres@google.com>
+ */
+
+#ifndef _MM_MEMFD_ASHMEM_SHIM_INTERNAL_H
+#define _MM_MEMFD_ASHMEM_SHIM_INTERNAL_H
+
+#include <linux/compat.h>
+#include <linux/ioctl.h>
+#include <linux/types.h>
+
+#define ASHMEM_NAME_LEN		256
+
+/* Return values from ASHMEM_PIN: Was the mapping purged while unpinned? */
+#define ASHMEM_NOT_PURGED	0
+#define ASHMEM_WAS_PURGED	1
+
+/* Return values from ASHMEM_GET_PIN_STATUS: Is the mapping pinned? */
+#define ASHMEM_IS_UNPINNED	0
+#define ASHMEM_IS_PINNED	1
+
+struct ashmem_pin {
+	__u32 offset;	/* offset into region, in bytes, page-aligned */
+	__u32 len;	/* length forward from offset, in bytes, page-aligned */
+};
+
+#define __ASHMEMIOC		0x77
+
+#define ASHMEM_SET_NAME		_IOW(__ASHMEMIOC, 1, char[ASHMEM_NAME_LEN])
+#define ASHMEM_GET_NAME		_IOR(__ASHMEMIOC, 2, char[ASHMEM_NAME_LEN])
+#define ASHMEM_SET_SIZE		_IOW(__ASHMEMIOC, 3, size_t)
+#define ASHMEM_GET_SIZE		_IO(__ASHMEMIOC, 4)
+#define ASHMEM_SET_PROT_MASK	_IOW(__ASHMEMIOC, 5, unsigned long)
+#define ASHMEM_GET_PROT_MASK	_IO(__ASHMEMIOC, 6)
+#define ASHMEM_PIN		_IOW(__ASHMEMIOC, 7, struct ashmem_pin)
+#define ASHMEM_UNPIN		_IOW(__ASHMEMIOC, 8, struct ashmem_pin)
+#define ASHMEM_GET_PIN_STATUS	_IO(__ASHMEMIOC, 9)
+#define ASHMEM_PURGE_ALL_CACHES	_IO(__ASHMEMIOC, 10)
+#define ASHMEM_GET_FILE_ID		_IOR(__ASHMEMIOC, 11, unsigned long)
+
+/* support of 32bit userspace on 64bit platforms */
+#ifdef CONFIG_COMPAT
+#define COMPAT_ASHMEM_SET_SIZE		_IOW(__ASHMEMIOC, 3, compat_size_t)
+#define COMPAT_ASHMEM_SET_PROT_MASK	_IOW(__ASHMEMIOC, 5, unsigned int)
+#endif
+
+#endif /* _MM_MEMFD_ASHMEM_SHIM_INTERNAL_H */

mm/memfd-ashmem-shim.c

@@ -0,0 +1,213 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Ashmem compatability for memfd
+ *
+ * Copyright (c) 2025, Google LLC.
+ * Author: Isaac J. Manjarres <isaacmanjarres@google.com>
+ */
+
+#include <asm-generic/mman-common.h>
+#include <linux/capability.h>
+#include <linux/fs.h>
+#include <linux/memfd.h>
+#include <linux/uaccess.h>
+
+#include "memfd-ashmem-shim.h"
+#include "memfd-ashmem-shim-internal.h"
+
+/* memfd file names all start with memfd: */
+#define MEMFD_PREFIX "memfd:"
+#define MEMFD_PREFIX_LEN (sizeof(MEMFD_PREFIX) - 1)
+
+static const char *get_memfd_name(struct file *file)
+{
+	/* This pointer is always valid, so no need to check if it's NULL. */
+	const char *file_name = file->f_path.dentry->d_name.name;
+
+	if (file_name != strstr(file_name, MEMFD_PREFIX))
+		return NULL;
+
+	return file_name;
+}
+
+static long get_name(struct file *file, void __user *name)
+{
+	const char *file_name = get_memfd_name(file);
+	size_t len;
+
+	if (!file_name)
+		return -EINVAL;
+
+	/* Strip MEMFD_PREFIX to retain compatibility with ashmem driver. */
+	file_name = &file_name[MEMFD_PREFIX_LEN];
+
+	/*
+	 * The expectation is that the user provided buffer is ASHMEM_NAME_LEN in size, which is
+	 * larger than the maximum size of a name for a memfd buffer, so the name should always fit
+	 * within the given buffer.
+	 *
+	 * However, we should ensure that the string will indeed fit in the user provided buffer.
+	 *
+	 * Add 1 to the copy size to account for the NUL terminator
+	 */
+	len = strlen(file_name) + 1;
+	if (len > ASHMEM_NAME_LEN)
+		return -EINVAL;
+
+	return copy_to_user(name, file_name, len) ? -EFAULT : 0;
+}
+
+static long get_prot_mask(struct file *file)
+{
+	long prot_mask = PROT_READ | PROT_EXEC;
+	long seals = memfd_fcntl(file, F_GET_SEALS, 0);
+
+	if (seals < 0)
+		return seals;
+
+	/* memfds are readable and executable by default. Only writability can be changed. */
+	if (!(seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)))
+		prot_mask |= PROT_WRITE;
+
+	return prot_mask;
+}
+
+static long set_prot_mask(struct file *file, unsigned long prot)
+{
+	long curr_prot = get_prot_mask(file);
+	long ret = 0;
+
+	if (curr_prot < 0)
+		return curr_prot;
+
+	/*
+	 * memfds are always readable and executable; there is no way to remove either mapping
+	 * permission, nor is there a known usecase that requires it.
+	 *
+	 * Attempting to remove either of these mapping permissions will return successfully, but
+	 * will be a nop, as the buffer will still be mappable with these permissions.
+	 */
+	prot |= PROT_READ | PROT_EXEC;
+
+	/* Only allow permissions to be removed. */
+	if ((curr_prot & prot) != prot)
+		return -EINVAL;
+
+	/*
+	 * Removing PROT_WRITE:
+	 *
+	 * We could prevent any other mappings from having write permissions by adding the
+	 * F_SEAL_WRITE mapping. However, that would conflict with known usecases where it is
+	 * desirable to maintain an existing writable mapping, but forbid future writable mappings.
+	 *
+	 * To support those usecases, we use F_SEAL_FUTURE_WRITE.
+	 */
+	if (!(prot & PROT_WRITE))
+		ret = memfd_fcntl(file, F_ADD_SEALS, F_SEAL_FUTURE_WRITE);
+
+	return ret;
+}
+
+/*
+ * memfd_ashmem_shim_ioctl - ioctl handler for ashmem commands
+ * @file: The shmem file.
+ * @cmd: The ioctl command.
+ * @arg: The argument for the ioctl command.
+ *
+ * The purpose of this handler is to allow old applications to continue working
+ * on newer kernels by allowing them to invoke ashmem ioctl commands on memfds.
+ *
+ * The ioctl handler attempts to retain as much compatibility with the ashmem
+ * driver as possible.
+ */
+long memfd_ashmem_shim_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	long ret = -ENOTTY;
+	unsigned long inode_nr;
+
+	switch (cmd) {
+	/*
+	 * Older applications won't create memfds and try to use ASHMEM_SET_NAME/ASHMEM_SET_SIZE on
+	 * them intentionally.
+	 *
+	 * Instead, we can end up in this scenario if an old application receives a memfd that was
+	 * created by another process.
+	 *
+	 * However, the current process shouldn't expect to be able to reliably [re]name/size a
+	 * buffer that was shared with it, since the process that shared that buffer with it, or
+	 * any other process that references the buffer could have already mapped it.
+	 *
+	 * Additionally in the case of ASHMEM_SET_SIZE, when processes create memfds that are going
+	 * to be shared with other processes in Android, they also specify the size of the memory
+	 * region and seal the file against any size changes. Therefore, ASHMEM_SET_SIZE should not
+	 * be supported anyway.
+	 *
+	 * Therefore, it is reasonable to return -EINVAL here, as if the buffer was already mapped.
+	 */
+	case ASHMEM_SET_NAME:
+	case ASHMEM_SET_SIZE:
+		ret = -EINVAL;
+		break;
+	case ASHMEM_GET_NAME:
+		ret = get_name(file, (void __user *)arg);
+		break;
+	case ASHMEM_GET_SIZE:
+		ret = i_size_read(file_inode(file));
+		break;
+	case ASHMEM_SET_PROT_MASK:
+		ret = set_prot_mask(file, arg);
+		break;
+	case ASHMEM_GET_PROT_MASK:
+		ret = get_prot_mask(file);
+		break;
+	/*
+	 * Unpinning ashmem buffers was deprecated with the release of Android 10,
+	 * as it did not yield any remarkable benefits. Therefore, ignore pinning
+	 * related requests.
+	 *
+	 * This makes it so that memory is always "pinned" or never entirely freed
+	 * until all references to the ashmem buffer are dropped. The memory occupied
+	 * by the buffer is still subject to being reclaimed (swapped out) under memory
+	 * pressure, but that is not the same as being freed.
+	 *
+	 * This makes it so that:
+	 *
+	 * 1. Memory is always pinned and therefore never purged.
+	 * 2. Requests to unpin memory (make it a candidate for being freed) are ignored.
+	 */
+	case ASHMEM_PIN:
+		ret = ASHMEM_NOT_PURGED;
+		break;
+	case ASHMEM_UNPIN:
+		ret = 0;
+		break;
+	case ASHMEM_GET_PIN_STATUS:
+		ret = ASHMEM_IS_PINNED;
+		break;
+	case ASHMEM_PURGE_ALL_CACHES:
+		ret = capable(CAP_SYS_ADMIN) ? 0 : -EPERM;
+		break;
+	case ASHMEM_GET_FILE_ID:
+		inode_nr = file_inode(file)->i_ino;
+		if (copy_to_user((void __user *)arg, &inode_nr, sizeof(inode_nr)))
+			ret = -EFAULT;
+		else
+			ret = 0;
+		break;
+	}
+
+	return ret;
+}
+
+#ifdef CONFIG_COMPAT
+long memfd_ashmem_shim_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	if (cmd == COMPAT_ASHMEM_SET_SIZE)
+		cmd = ASHMEM_SET_SIZE;
+	else if (cmd == COMPAT_ASHMEM_SET_PROT_MASK)
+		cmd = ASHMEM_SET_PROT_MASK;
+
+	return memfd_ashmem_shim_ioctl(file, cmd, arg);
+}
+#endif

mm/memfd-ashmem-shim.h

@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __MM_MEMFD_ASHMEM_SHIM_H
+#define __MM_MEMFD_ASHMEM_SHIM_H
+
+/*
+ * mm/memfd-ashmem-shim.h
+ *
+ * Ashmem compatability for memfd
+ *
+ * Copyright (c) 2025, Google LLC.
+ * Author: Isaac J. Manjarres <isaacmanjarres@google.com>
+ *
+ */
+
+#include <linux/fs.h>
+
+long memfd_ashmem_shim_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
+#ifdef CONFIG_COMPAT
+long memfd_ashmem_shim_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
+#endif
+#endif /* __MM_MEMFD_ASHMEM_SHIM_H */