zfs/module/os/linux/zfs/zpl_file.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or https://opensource.org/licenses/CDDL-1.0.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2011, Lawrence Livermore National Security, LLC.
 * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
 */


#ifdef CONFIG_COMPAT
#include <linux/compat.h>
#endif
#include <linux/fs.h>
#include <sys/file.h>
#include <sys/dmu_objset.h>
#include <sys/zfs_znode.h>
#include <sys/zfs_vfsops.h>
#include <sys/zfs_vnops.h>
#include <sys/zfs_project.h>
#if defined(HAVE_VFS_SET_PAGE_DIRTY_NOBUFFERS) || \
    defined(HAVE_VFS_FILEMAP_DIRTY_FOLIO)
#include <linux/pagemap.h>
#endif
#ifdef HAVE_FILE_FADVISE
#include <linux/fadvise.h>
#endif
#ifdef HAVE_VFS_FILEMAP_DIRTY_FOLIO
#include <linux/writeback.h>
#endif

/*
 * When using fallocate(2) to preallocate space, inflate the requested
 * capacity check by 10% to account for the required metadata blocks.
 */
static unsigned int zfs_fallocate_reserve_percent = 110;

static int
zpl_open(struct inode *ip, struct file *filp)
{
	cred_t *cr = CRED();
	int error;
	fstrans_cookie_t cookie;

	error = generic_file_open(ip, filp);
	if (error)
		return (error);

	crhold(cr);
	cookie = spl_fstrans_mark();
	error = -zfs_open(ip, filp->f_mode, filp->f_flags, cr);
	spl_fstrans_unmark(cookie);
	crfree(cr);
	ASSERT3S(error, <=, 0);

	return (error);
}

static int
zpl_release(struct inode *ip, struct file *filp)
{
	cred_t *cr = CRED();
	int error;
	fstrans_cookie_t cookie;

	cookie = spl_fstrans_mark();
	if (ITOZ(ip)->z_atime_dirty)
		zfs_mark_inode_dirty(ip);

	crhold(cr);
	error = -zfs_close(ip, filp->f_flags, cr);
	spl_fstrans_unmark(cookie);
	crfree(cr);
	ASSERT3S(error, <=, 0);

	return (error);
}

static int
zpl_iterate(struct file *filp, zpl_dir_context_t *ctx)
{
	cred_t *cr = CRED();
	int error;
	fstrans_cookie_t cookie;

	crhold(cr);
	cookie = spl_fstrans_mark();
	error = -zfs_readdir(file_inode(filp), ctx, cr);
	spl_fstrans_unmark(cookie);
	crfree(cr);
	ASSERT3S(error, <=, 0);

	return (error);
}

#if !defined(HAVE_VFS_ITERATE) && !defined(HAVE_VFS_ITERATE_SHARED)
static int
zpl_readdir(struct file *filp, void *dirent, filldir_t filldir)
{
	zpl_dir_context_t ctx =
	    ZPL_DIR_CONTEXT_INIT(dirent, filldir, filp->f_pos);
	int error;

	error = zpl_iterate(filp, &ctx);
	filp->f_pos = ctx.pos;

	return (error);
}
#endif /* !HAVE_VFS_ITERATE && !HAVE_VFS_ITERATE_SHARED */

#if defined(HAVE_FSYNC_WITHOUT_DENTRY)
/*
 * Linux 2.6.35 - 3.0 API,
 * As of 2.6.35 the dentry argument to the fops->fsync() hook was deemed
 * redundant.  The dentry is still accessible via filp->f_path.dentry,
 * and we are guaranteed that filp will never be NULL.
 */
static int
zpl_fsync(struct file *filp, int datasync)
{
	struct inode *inode = filp->f_mapping->host;
	cred_t *cr = CRED();
	int error;
	fstrans_cookie_t cookie;

	crhold(cr);
	cookie = spl_fstrans_mark();
	error = -zfs_fsync(ITOZ(inode), datasync, cr);
	spl_fstrans_unmark(cookie);
	crfree(cr);
	ASSERT3S(error, <=, 0);

	return (error);
}

#ifdef HAVE_FILE_AIO_FSYNC
static int
zpl_aio_fsync(struct kiocb *kiocb, int datasync)
{
	return (zpl_fsync(kiocb->ki_filp, datasync));
}
#endif

#elif defined(HAVE_FSYNC_RANGE)
/*
 * Linux 3.1 API,
 * As of 3.1 the responsibility to call filemap_write_and_wait_range() has
 * been pushed down in to the .fsync() vfs hook.  Additionally, the i_mutex
 * lock is no longer held by the caller, for zfs we don't require the lock
 * to be held so we don't acquire it.
 */
static int
zpl_fsync(struct file *filp, loff_t start, loff_t end, int datasync)
{
	struct inode *inode = filp->f_mapping->host;
	znode_t *zp = ITOZ(inode);
	zfsvfs_t *zfsvfs = ITOZSB(inode);
	cred_t *cr = CRED();
	int error;
	fstrans_cookie_t cookie;

	/*
	 * The variables z_sync_writes_cnt and z_async_writes_cnt work in
	 * tandem so that sync writes can detect if there are any non-sync
	 * writes going on and vice-versa. The "vice-versa" part to this logic
	 * is located in zfs_putpage() where non-sync writes check if there are
	 * any ongoing sync writes. If any sync and non-sync writes overlap,
	 * we do a commit to complete the non-sync writes since the latter can
	 * potentially take several seconds to complete and thus block sync
	 * writes in the upcoming call to filemap_write_and_wait_range().
	 */
	atomic_inc_32(&zp->z_sync_writes_cnt);
	/*
	 * If the following check does not detect an overlapping non-sync write
	 * (say because it's just about to start), then it is guaranteed that
	 * the non-sync write will detect this sync write. This is because we
	 * always increment z_sync_writes_cnt / z_async_writes_cnt before doing
	 * the check on z_async_writes_cnt / z_sync_writes_cnt here and in
	 * zfs_putpage() respectively.
	 */
	if (atomic_load_32(&zp->z_async_writes_cnt) > 0) {
		if ((error = zpl_enter(zfsvfs, FTAG)) != 0) {
			atomic_dec_32(&zp->z_sync_writes_cnt);
			return (error);
		}
		zil_commit(zfsvfs->z_log, zp->z_id);
		zpl_exit(zfsvfs, FTAG);
	}

	error = filemap_write_and_wait_range(inode->i_mapping, start, end);

	/*
	 * The sync write is not complete yet but we decrement
	 * z_sync_writes_cnt since zfs_fsync() increments and decrements
	 * it internally. If a non-sync write starts just after the decrement
	 * operation but before we call zfs_fsync(), it may not detect this
	 * overlapping sync write but it does not matter since we have already
	 * gone past filemap_write_and_wait_range() and we won't block due to
	 * the non-sync write.
	 */
	atomic_dec_32(&zp->z_sync_writes_cnt);

	if (error)
		return (error);

	crhold(cr);
	cookie = spl_fstrans_mark();
	error = -zfs_fsync(zp, datasync, cr);
	spl_fstrans_unmark(cookie);
	crfree(cr);
	ASSERT3S(error, <=, 0);

	return (error);
}

#ifdef HAVE_FILE_AIO_FSYNC
static int
zpl_aio_fsync(struct kiocb *kiocb, int datasync)
{
	return (zpl_fsync(kiocb->ki_filp, kiocb->ki_pos, -1, datasync));
}
#endif

#else
#error "Unsupported fops->fsync() implementation"
#endif

static inline int
zfs_io_flags(struct kiocb *kiocb)
{
	int flags = 0;

#if defined(IOCB_DSYNC)
	if (kiocb->ki_flags & IOCB_DSYNC)
		flags |= O_DSYNC;
#endif
#if defined(IOCB_SYNC)
	if (kiocb->ki_flags & IOCB_SYNC)
		flags |= O_SYNC;
#endif
#if defined(IOCB_APPEND)
	if (kiocb->ki_flags & IOCB_APPEND)
		flags |= O_APPEND;
#endif
#if defined(IOCB_DIRECT)
	if (kiocb->ki_flags & IOCB_DIRECT)
		flags |= O_DIRECT;
#endif
	return (flags);
}

/*
 * If relatime is enabled, call file_accessed() if zfs_relatime_need_update()
 * is true.  This is needed since datasets with inherited "relatime" property
 * aren't necessarily mounted with the MNT_RELATIME flag (e.g. after
 * `zfs set relatime=...`), which is what relatime test in VFS by
 * relatime_need_update() is based on.
 */
static inline void
zpl_file_accessed(struct file *filp)
{
	struct inode *ip = filp->f_mapping->host;

	if (!IS_NOATIME(ip) && ITOZSB(ip)->z_relatime) {
		if (zfs_relatime_need_update(ip))
			file_accessed(filp);
	} else {
		file_accessed(filp);
	}
}

#if defined(HAVE_VFS_RW_ITERATE)

/*
 * When HAVE_VFS_IOV_ITER is defined the iov_iter structure supports
 * iovecs, kvevs, bvecs and pipes, plus all the required interfaces to
 * manipulate the iov_iter are available.  In which case the full iov_iter
 * can be attached to the uio and correctly handled in the lower layers.
 * Otherwise, for older kernels extract the iovec and pass it instead.
 */
static void
zpl_uio_init(zfs_uio_t *uio, struct kiocb *kiocb, struct iov_iter *to,
    loff_t pos, ssize_t count, size_t skip)
{
#if defined(HAVE_VFS_IOV_ITER)
	zfs_uio_iov_iter_init(uio, to, pos, count, skip);
#else
	zfs_uio_iovec_init(uio, zfs_uio_iter_iov(to), to->nr_segs, pos,
	    zfs_uio_iov_iter_type(to) & ITER_KVEC ?
	    UIO_SYSSPACE : UIO_USERSPACE,
	    count, skip);
#endif
}

static ssize_t
zpl_iter_read(struct kiocb *kiocb, struct iov_iter *to)
{
	cred_t *cr = CRED();
	fstrans_cookie_t cookie;
	struct file *filp = kiocb->ki_filp;
	ssize_t count = iov_iter_count(to);
	zfs_uio_t uio;

	zpl_uio_init(&uio, kiocb, to, kiocb->ki_pos, count, 0);

	crhold(cr);
	cookie = spl_fstrans_mark();

	int error = -zfs_read(ITOZ(filp->f_mapping->host), &uio,
	    filp->f_flags | zfs_io_flags(kiocb), cr);

	spl_fstrans_unmark(cookie);
	crfree(cr);

	if (error < 0)
		return (error);

	ssize_t read = count - uio.uio_resid;
	kiocb->ki_pos += read;

	zpl_file_accessed(filp);

	return (read);
}

static inline ssize_t
zpl_generic_write_checks(struct kiocb *kiocb, struct iov_iter *from,
    size_t *countp)
{
#ifdef HAVE_GENERIC_WRITE_CHECKS_KIOCB
	ssize_t ret = generic_write_checks(kiocb, from);
	if (ret <= 0)
		return (ret);

	*countp = ret;
#else
	struct file *file = kiocb->ki_filp;
	struct address_space *mapping = file->f_mapping;
	struct inode *ip = mapping->host;
	int isblk = S_ISBLK(ip->i_mode);

	*countp = iov_iter_count(from);
	ssize_t ret = generic_write_checks(file, &kiocb->ki_pos, countp, isblk);
	if (ret)
		return (ret);
#endif

	return (0);
}

static ssize_t
zpl_iter_write(struct kiocb *kiocb, struct iov_iter *from)
{
	cred_t *cr = CRED();
	fstrans_cookie_t cookie;
	struct file *filp = kiocb->ki_filp;
	struct inode *ip = filp->f_mapping->host;
	zfs_uio_t uio;
	size_t count = 0;
	ssize_t ret;

	ret = zpl_generic_write_checks(kiocb, from, &count);
	if (ret)
		return (ret);

	zpl_uio_init(&uio, kiocb, from, kiocb->ki_pos, count, from->iov_offset);

	crhold(cr);
	cookie = spl_fstrans_mark();

	int error = -zfs_write(ITOZ(ip), &uio,
	    filp->f_flags | zfs_io_flags(kiocb), cr);

	spl_fstrans_unmark(cookie);
	crfree(cr);

	if (error < 0)
		return (error);

	ssize_t wrote = count - uio.uio_resid;
	kiocb->ki_pos += wrote;

	return (wrote);
}

#else /* !HAVE_VFS_RW_ITERATE */

static ssize_t
zpl_aio_read(struct kiocb *kiocb, const struct iovec *iov,
    unsigned long nr_segs, loff_t pos)
{
	cred_t *cr = CRED();
	fstrans_cookie_t cookie;
	struct file *filp = kiocb->ki_filp;
	size_t count;
	ssize_t ret;

	ret = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
	if (ret)
		return (ret);

	zfs_uio_t uio;
	zfs_uio_iovec_init(&uio, iov, nr_segs, kiocb->ki_pos, UIO_USERSPACE,
	    count, 0);

	crhold(cr);
	cookie = spl_fstrans_mark();

	int error = -zfs_read(ITOZ(filp->f_mapping->host), &uio,
	    filp->f_flags | zfs_io_flags(kiocb), cr);

	spl_fstrans_unmark(cookie);
	crfree(cr);

	if (error < 0)
		return (error);

	ssize_t read = count - uio.uio_resid;
	kiocb->ki_pos += read;

	zpl_file_accessed(filp);

	return (read);
}

static ssize_t
zpl_aio_write(struct kiocb *kiocb, const struct iovec *iov,
    unsigned long nr_segs, loff_t pos)
{
	cred_t *cr = CRED();
	fstrans_cookie_t cookie;
	struct file *filp = kiocb->ki_filp;
	struct inode *ip = filp->f_mapping->host;
	size_t count;
	ssize_t ret;

	ret = generic_segment_checks(iov, &nr_segs, &count, VERIFY_READ);
	if (ret)
		return (ret);

	ret = generic_write_checks(filp, &pos, &count, S_ISBLK(ip->i_mode));
	if (ret)
		return (ret);

	kiocb->ki_pos = pos;

	zfs_uio_t uio;
	zfs_uio_iovec_init(&uio, iov, nr_segs, kiocb->ki_pos, UIO_USERSPACE,
	    count, 0);

	crhold(cr);
	cookie = spl_fstrans_mark();

	int error = -zfs_write(ITOZ(ip), &uio,
	    filp->f_flags | zfs_io_flags(kiocb), cr);

	spl_fstrans_unmark(cookie);
	crfree(cr);

	if (error < 0)
		return (error);

	ssize_t wrote = count - uio.uio_resid;
	kiocb->ki_pos += wrote;

	return (wrote);
}
#endif /* HAVE_VFS_RW_ITERATE */

#if defined(HAVE_VFS_RW_ITERATE)
static ssize_t
zpl_direct_IO_impl(int rw, struct kiocb *kiocb, struct iov_iter *iter)
{
	if (rw == WRITE)
		return (zpl_iter_write(kiocb, iter));
	else
		return (zpl_iter_read(kiocb, iter));
}
#if defined(HAVE_VFS_DIRECT_IO_ITER)
static ssize_t
zpl_direct_IO(struct kiocb *kiocb, struct iov_iter *iter)
{
	return (zpl_direct_IO_impl(iov_iter_rw(iter), kiocb, iter));
}
#elif defined(HAVE_VFS_DIRECT_IO_ITER_OFFSET)
static ssize_t
zpl_direct_IO(struct kiocb *kiocb, struct iov_iter *iter, loff_t pos)
{
	ASSERT3S(pos, ==, kiocb->ki_pos);
	return (zpl_direct_IO_impl(iov_iter_rw(iter), kiocb, iter));
}
#elif defined(HAVE_VFS_DIRECT_IO_ITER_RW_OFFSET)
static ssize_t
zpl_direct_IO(int rw, struct kiocb *kiocb, struct iov_iter *iter, loff_t pos)
{
	ASSERT3S(pos, ==, kiocb->ki_pos);
	return (zpl_direct_IO_impl(rw, kiocb, iter));
}
#else
#error "Unknown direct IO interface"
#endif

#else /* HAVE_VFS_RW_ITERATE */

#if defined(HAVE_VFS_DIRECT_IO_IOVEC)
static ssize_t
zpl_direct_IO(int rw, struct kiocb *kiocb, const struct iovec *iov,
    loff_t pos, unsigned long nr_segs)
{
	if (rw == WRITE)
		return (zpl_aio_write(kiocb, iov, nr_segs, pos));
	else
		return (zpl_aio_read(kiocb, iov, nr_segs, pos));
}
#elif defined(HAVE_VFS_DIRECT_IO_ITER_RW_OFFSET)
static ssize_t
zpl_direct_IO(int rw, struct kiocb *kiocb, struct iov_iter *iter, loff_t pos)
{
	const struct iovec *iovp = iov_iter_iovec(iter);
	unsigned long nr_segs = iter->nr_segs;

	ASSERT3S(pos, ==, kiocb->ki_pos);
	if (rw == WRITE)
		return (zpl_aio_write(kiocb, iovp, nr_segs, pos));
	else
		return (zpl_aio_read(kiocb, iovp, nr_segs, pos));
}
#else
#error "Unknown direct IO interface"
#endif

#endif /* HAVE_VFS_RW_ITERATE */

static loff_t
zpl_llseek(struct file *filp, loff_t offset, int whence)
{
#if defined(SEEK_HOLE) && defined(SEEK_DATA)
	fstrans_cookie_t cookie;

	if (whence == SEEK_DATA || whence == SEEK_HOLE) {
		struct inode *ip = filp->f_mapping->host;
		loff_t maxbytes = ip->i_sb->s_maxbytes;
		loff_t error;

		spl_inode_lock_shared(ip);
		cookie = spl_fstrans_mark();
		error = -zfs_holey(ITOZ(ip), whence, &offset);
		spl_fstrans_unmark(cookie);
		if (error == 0)
			error = lseek_execute(filp, ip, offset, maxbytes);
		spl_inode_unlock_shared(ip);

		return (error);
	}
#endif /* SEEK_HOLE && SEEK_DATA */

	return (generic_file_llseek(filp, offset, whence));
}

/*
 * It's worth taking a moment to describe how mmap is implemented
 * for zfs because it differs considerably from other Linux filesystems.
 * However, this issue is handled the same way under OpenSolaris.
 *
 * The issue is that by design zfs bypasses the Linux page cache and
 * leaves all caching up to the ARC.  This has been shown to work
 * well for the common read(2)/write(2) case.  However, mmap(2)
 * is problem because it relies on being tightly integrated with the
 * page cache.  To handle this we cache mmap'ed files twice, once in
 * the ARC and a second time in the page cache.  The code is careful
 * to keep both copies synchronized.
 *
 * When a file with an mmap'ed region is written to using write(2)
 * both the data in the ARC and existing pages in the page cache
 * are updated.  For a read(2) data will be read first from the page
 * cache then the ARC if needed.  Neither a write(2) or read(2) will
 * will ever result in new pages being added to the page cache.
 *
 * New pages are added to the page cache only via .readpage() which
 * is called when the vfs needs to read a page off disk to back the
 * virtual memory region.  These pages may be modified without
 * notifying the ARC and will be written out periodically via
 * .writepage().  This will occur due to either a sync or the usual
 * page aging behavior.  Note because a read(2) of a mmap'ed file
 * will always check the page cache first even when the ARC is out
 * of date correct data will still be returned.
 *
 * While this implementation ensures correct behavior it does have
 * have some drawbacks.  The most obvious of which is that it
 * increases the required memory footprint when access mmap'ed
 * files.  It also adds additional complexity to the code keeping
 * both caches synchronized.
 *
 * Longer term it may be possible to cleanly resolve this wart by
 * mapping page cache pages directly on to the ARC buffers.  The
 * Linux address space operations are flexible enough to allow
 * selection of which pages back a particular index.  The trick
 * would be working out the details of which subsystem is in
 * charge, the ARC, the page cache, or both.  It may also prove
 * helpful to move the ARC buffers to a scatter-gather lists
 * rather than a vmalloc'ed region.
 */
static int
zpl_mmap(struct file *filp, struct vm_area_struct *vma)
{
	struct inode *ip = filp->f_mapping->host;
	int error;
	fstrans_cookie_t cookie;

	cookie = spl_fstrans_mark();
	error = -zfs_map(ip, vma->vm_pgoff, (caddr_t *)vma->vm_start,
	    (size_t)(vma->vm_end - vma->vm_start), vma->vm_flags);
	spl_fstrans_unmark(cookie);
	if (error)
		return (error);

	error = generic_file_mmap(filp, vma);
	if (error)
		return (error);

#if !defined(HAVE_FILEMAP_RANGE_HAS_PAGE)
	znode_t *zp = ITOZ(ip);
	mutex_enter(&zp->z_lock);
	zp->z_is_mapped = B_TRUE;
	mutex_exit(&zp->z_lock);
#endif

	return (error);
}

/*
 * Populate a page with data for the Linux page cache.  This function is
 * only used to support mmap(2).  There will be an identical copy of the
 * data in the ARC which is kept up to date via .write() and .writepage().
 */
static inline int
zpl_readpage_common(struct page *pp)
{
	fstrans_cookie_t cookie;

	ASSERT(PageLocked(pp));

	cookie = spl_fstrans_mark();
	int error = -zfs_getpage(pp->mapping->host, pp);
	spl_fstrans_unmark(cookie);

	unlock_page(pp);

	return (error);
}

#ifdef HAVE_VFS_READ_FOLIO
static int
zpl_read_folio(struct file *filp, struct folio *folio)
{
	return (zpl_readpage_common(&folio->page));
}
#else
static int
zpl_readpage(struct file *filp, struct page *pp)
{
	return (zpl_readpage_common(pp));
}
#endif

static int
zpl_readpage_filler(void *data, struct page *pp)
{
	return (zpl_readpage_common(pp));
}

/*
 * Populate a set of pages with data for the Linux page cache.  This
 * function will only be called for read ahead and never for demand
 * paging.  For simplicity, the code relies on read_cache_pages() to
 * correctly lock each page for IO and call zpl_readpage().
 */
#ifdef HAVE_VFS_READPAGES
static int
zpl_readpages(struct file *filp, struct address_space *mapping,
    struct list_head *pages, unsigned nr_pages)
{
	return (read_cache_pages(mapping, pages, zpl_readpage_filler, NULL));
}
#else
static void
zpl_readahead(struct readahead_control *ractl)
{
	struct page *page;

	while ((page = readahead_page(ractl)) != NULL) {
		int ret;

		ret = zpl_readpage_filler(NULL, page);
		put_page(page);
		if (ret)
			break;
	}
}
#endif

static int
zpl_putpage(struct page *pp, struct writeback_control *wbc, void *data)
{
	boolean_t *for_sync = data;
	fstrans_cookie_t cookie;

	ASSERT(PageLocked(pp));
	ASSERT(!PageWriteback(pp));

	cookie = spl_fstrans_mark();
	(void) zfs_putpage(pp->mapping->host, pp, wbc, *for_sync);
	spl_fstrans_unmark(cookie);

	return (0);
}

#ifdef HAVE_WRITEPAGE_T_FOLIO
static int
zpl_putfolio(struct folio *pp, struct writeback_control *wbc, void *data)
{
	(void) zpl_putpage(&pp->page, wbc, data);
	return (0);
}
#endif

static inline int
zpl_write_cache_pages(struct address_space *mapping,
    struct writeback_control *wbc, void *data)
{
	int result;

#ifdef HAVE_WRITEPAGE_T_FOLIO
	result = write_cache_pages(mapping, wbc, zpl_putfolio, data);
#else
	result = write_cache_pages(mapping, wbc, zpl_putpage, data);
#endif
	return (result);
}

static int
zpl_writepages(struct address_space *mapping, struct writeback_control *wbc)
{
	znode_t		*zp = ITOZ(mapping->host);
	zfsvfs_t	*zfsvfs = ITOZSB(mapping->host);
	enum writeback_sync_modes sync_mode;
	int result;

	if ((result = zpl_enter(zfsvfs, FTAG)) != 0)
		return (result);
	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
		wbc->sync_mode = WB_SYNC_ALL;
	zpl_exit(zfsvfs, FTAG);
	sync_mode = wbc->sync_mode;

	/*
	 * We don't want to run write_cache_pages() in SYNC mode here, because
	 * that would make putpage() wait for a single page to be committed to
	 * disk every single time, resulting in atrocious performance. Instead
	 * we run it once in non-SYNC mode so that the ZIL gets all the data,
	 * and then we commit it all in one go.
	 */
	boolean_t for_sync = (sync_mode == WB_SYNC_ALL);
	wbc->sync_mode = WB_SYNC_NONE;
	result = zpl_write_cache_pages(mapping, wbc, &for_sync);
	if (sync_mode != wbc->sync_mode) {
		if ((result = zpl_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
			return (result);
		if (zfsvfs->z_log != NULL)
			zil_commit(zfsvfs->z_log, zp->z_id);
		zpl_exit(zfsvfs, FTAG);

		/*
		 * We need to call write_cache_pages() again (we can't just
		 * return after the commit) because the previous call in
		 * non-SYNC mode does not guarantee that we got all the dirty
		 * pages (see the implementation of write_cache_pages() for
		 * details). That being said, this is a no-op in most cases.
		 */
		wbc->sync_mode = sync_mode;
		result = zpl_write_cache_pages(mapping, wbc, &for_sync);
	}
	return (result);
}

/*
 * Write out dirty pages to the ARC, this function is only required to
 * support mmap(2).  Mapped pages may be dirtied by memory operations
 * which never call .write().  These dirty pages are kept in sync with
 * the ARC buffers via this hook.
 */
static int
zpl_writepage(struct page *pp, struct writeback_control *wbc)
{
	if (ITOZSB(pp->mapping->host)->z_os->os_sync == ZFS_SYNC_ALWAYS)
		wbc->sync_mode = WB_SYNC_ALL;

	boolean_t for_sync = (wbc->sync_mode == WB_SYNC_ALL);

	return (zpl_putpage(pp, wbc, &for_sync));
}

/*
 * The flag combination which matches the behavior of zfs_space() is
 * FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE.  The FALLOC_FL_PUNCH_HOLE
 * flag was introduced in the 2.6.38 kernel.
 *
 * The original mode=0 (allocate space) behavior can be reasonably emulated
 * by checking if enough space exists and creating a sparse file, as real
 * persistent space reservation is not possible due to COW, snapshots, etc.
 */
static long
zpl_fallocate_common(struct inode *ip, int mode, loff_t offset, loff_t len)
{
	cred_t *cr = CRED();
	loff_t olen;
	fstrans_cookie_t cookie;
	int error = 0;

	int test_mode = FALLOC_FL_PUNCH_HOLE;
#ifdef HAVE_FALLOC_FL_ZERO_RANGE
	test_mode |= FALLOC_FL_ZERO_RANGE;
#endif

	if ((mode & ~(FALLOC_FL_KEEP_SIZE | test_mode)) != 0)
		return (-EOPNOTSUPP);

	if (offset < 0 || len <= 0)
		return (-EINVAL);

	spl_inode_lock(ip);
	olen = i_size_read(ip);

	crhold(cr);
	cookie = spl_fstrans_mark();
	if (mode & (test_mode)) {
		flock64_t bf;

		if (mode & FALLOC_FL_KEEP_SIZE) {
			if (offset > olen)
				goto out_unmark;

			if (offset + len > olen)
				len = olen - offset;
		}
		bf.l_type = F_WRLCK;
		bf.l_whence = SEEK_SET;
		bf.l_start = offset;
		bf.l_len = len;
		bf.l_pid = 0;

		error = -zfs_space(ITOZ(ip), F_FREESP, &bf, O_RDWR, offset, cr);
	} else if ((mode & ~FALLOC_FL_KEEP_SIZE) == 0) {
		unsigned int percent = zfs_fallocate_reserve_percent;
		struct kstatfs statfs;

		/* Legacy mode, disable fallocate compatibility. */
		if (percent == 0) {
			error = -EOPNOTSUPP;
			goto out_unmark;
		}

		/*
		 * Use zfs_statvfs() instead of dmu_objset_space() since it
		 * also checks project quota limits, which are relevant here.
		 */
		error = zfs_statvfs(ip, &statfs);
		if (error)
			goto out_unmark;

		/*
		 * Shrink available space a bit to account for overhead/races.
		 * We know the product previously fit into availbytes from
		 * dmu_objset_space(), so the smaller product will also fit.
		 */
		if (len > statfs.f_bavail * (statfs.f_bsize * 100 / percent)) {
			error = -ENOSPC;
			goto out_unmark;
		}
		if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > olen)
			error = zfs_freesp(ITOZ(ip), offset + len, 0, 0, FALSE);
	}
out_unmark:
	spl_fstrans_unmark(cookie);
	spl_inode_unlock(ip);

	crfree(cr);

	return (error);
}

static long
zpl_fallocate(struct file *filp, int mode, loff_t offset, loff_t len)
{
	return zpl_fallocate_common(file_inode(filp),
	    mode, offset, len);
}

static int
zpl_ioctl_getversion(struct file *filp, void __user *arg)
{
	uint32_t generation = file_inode(filp)->i_generation;

	return (copy_to_user(arg, &generation, sizeof (generation)));
}

#ifdef HAVE_FILE_FADVISE
static int
zpl_fadvise(struct file *filp, loff_t offset, loff_t len, int advice)
{
	struct inode *ip = file_inode(filp);
	znode_t *zp = ITOZ(ip);
	zfsvfs_t *zfsvfs = ITOZSB(ip);
	objset_t *os = zfsvfs->z_os;
	int error = 0;

	if (S_ISFIFO(ip->i_mode))
		return (-ESPIPE);

	if (offset < 0 || len < 0)
		return (-EINVAL);

	if ((error = zpl_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
		return (error);

	switch (advice) {
	case POSIX_FADV_SEQUENTIAL:
	case POSIX_FADV_WILLNEED:
#ifdef HAVE_GENERIC_FADVISE
		if (zn_has_cached_data(zp, offset, offset + len - 1))
			error = generic_fadvise(filp, offset, len, advice);
#endif
		/*
		 * Pass on the caller's size directly, but note that
		 * dmu_prefetch_max will effectively cap it.  If there
		 * really is a larger sequential access pattern, perhaps
		 * dmu_zfetch will detect it.
		 */
		if (len == 0)
			len = i_size_read(ip) - offset;

		dmu_prefetch(os, zp->z_id, 0, offset, len,
		    ZIO_PRIORITY_ASYNC_READ);
		break;
	case POSIX_FADV_NORMAL:
	case POSIX_FADV_RANDOM:
	case POSIX_FADV_DONTNEED:
	case POSIX_FADV_NOREUSE:
		/* ignored for now */
		break;
	default:
		error = -EINVAL;
		break;
	}

	zfs_exit(zfsvfs, FTAG);

	return (error);
}
#endif /* HAVE_FILE_FADVISE */

#define	ZFS_FL_USER_VISIBLE	(FS_FL_USER_VISIBLE | ZFS_PROJINHERIT_FL)
#define	ZFS_FL_USER_MODIFIABLE	(FS_FL_USER_MODIFIABLE | ZFS_PROJINHERIT_FL)

static uint32_t
__zpl_ioctl_getflags(struct inode *ip)
{
	uint64_t zfs_flags = ITOZ(ip)->z_pflags;
	uint32_t ioctl_flags = 0;

	if (zfs_flags & ZFS_IMMUTABLE)
		ioctl_flags |= FS_IMMUTABLE_FL;

	if (zfs_flags & ZFS_APPENDONLY)
		ioctl_flags |= FS_APPEND_FL;

	if (zfs_flags & ZFS_NODUMP)
		ioctl_flags |= FS_NODUMP_FL;

	if (zfs_flags & ZFS_PROJINHERIT)
		ioctl_flags |= ZFS_PROJINHERIT_FL;

	return (ioctl_flags & ZFS_FL_USER_VISIBLE);
}

/*
 * Map zfs file z_pflags (xvattr_t) to linux file attributes. Only file
 * attributes common to both Linux and Solaris are mapped.
 */
static int
zpl_ioctl_getflags(struct file *filp, void __user *arg)
{
	uint32_t flags;
	int err;

	flags = __zpl_ioctl_getflags(file_inode(filp));
	err = copy_to_user(arg, &flags, sizeof (flags));

	return (err);
}

/*
 * fchange() is a helper macro to detect if we have been asked to change a
 * flag. This is ugly, but the requirement that we do this is a consequence of
 * how the Linux file attribute interface was designed. Another consequence is
 * that concurrent modification of files suffers from a TOCTOU race. Neither
 * are things we can fix without modifying the kernel-userland interface, which
 * is outside of our jurisdiction.
 */

#define	fchange(f0, f1, b0, b1) (!((f0) & (b0)) != !((f1) & (b1)))

static int
__zpl_ioctl_setflags(struct inode *ip, uint32_t ioctl_flags, xvattr_t *xva)
{
	uint64_t zfs_flags = ITOZ(ip)->z_pflags;
	xoptattr_t *xoap;

	if (ioctl_flags & ~(FS_IMMUTABLE_FL | FS_APPEND_FL | FS_NODUMP_FL |
	    ZFS_PROJINHERIT_FL))
		return (-EOPNOTSUPP);

	if (ioctl_flags & ~ZFS_FL_USER_MODIFIABLE)
		return (-EACCES);

	if ((fchange(ioctl_flags, zfs_flags, FS_IMMUTABLE_FL, ZFS_IMMUTABLE) ||
	    fchange(ioctl_flags, zfs_flags, FS_APPEND_FL, ZFS_APPENDONLY)) &&
	    !capable(CAP_LINUX_IMMUTABLE))
		return (-EPERM);

	if (!zpl_inode_owner_or_capable(zfs_init_idmap, ip))
		return (-EACCES);

	xva_init(xva);
	xoap = xva_getxoptattr(xva);

#define	FLAG_CHANGE(iflag, zflag, xflag, xfield)	do {	\
	if (((ioctl_flags & (iflag)) && !(zfs_flags & (zflag))) ||	\
	    ((zfs_flags & (zflag)) && !(ioctl_flags & (iflag)))) {	\
		XVA_SET_REQ(xva, (xflag));	\
		(xfield) = ((ioctl_flags & (iflag)) != 0);	\
	}	\
} while (0)

	FLAG_CHANGE(FS_IMMUTABLE_FL, ZFS_IMMUTABLE, XAT_IMMUTABLE,
	    xoap->xoa_immutable);
	FLAG_CHANGE(FS_APPEND_FL, ZFS_APPENDONLY, XAT_APPENDONLY,
	    xoap->xoa_appendonly);
	FLAG_CHANGE(FS_NODUMP_FL, ZFS_NODUMP, XAT_NODUMP,
	    xoap->xoa_nodump);
	FLAG_CHANGE(ZFS_PROJINHERIT_FL, ZFS_PROJINHERIT, XAT_PROJINHERIT,
	    xoap->xoa_projinherit);

#undef	FLAG_CHANGE

	return (0);
}

static int
zpl_ioctl_setflags(struct file *filp, void __user *arg)
{
	struct inode *ip = file_inode(filp);
	uint32_t flags;
	cred_t *cr = CRED();
	xvattr_t xva;
	int err;
	fstrans_cookie_t cookie;

	if (copy_from_user(&flags, arg, sizeof (flags)))
		return (-EFAULT);

	err = __zpl_ioctl_setflags(ip, flags, &xva);
	if (err)
		return (err);

	crhold(cr);
	cookie = spl_fstrans_mark();
	err = -zfs_setattr(ITOZ(ip), (vattr_t *)&xva, 0, cr, zfs_init_idmap);
	spl_fstrans_unmark(cookie);
	crfree(cr);

	return (err);
}

static int
zpl_ioctl_getxattr(struct file *filp, void __user *arg)
{
	zfsxattr_t fsx = { 0 };
	struct inode *ip = file_inode(filp);
	int err;

	fsx.fsx_xflags = __zpl_ioctl_getflags(ip);
	fsx.fsx_projid = ITOZ(ip)->z_projid;
	err = copy_to_user(arg, &fsx, sizeof (fsx));

	return (err);
}

static int
zpl_ioctl_setxattr(struct file *filp, void __user *arg)
{
	struct inode *ip = file_inode(filp);
	zfsxattr_t fsx;
	cred_t *cr = CRED();
	xvattr_t xva;
	xoptattr_t *xoap;
	int err;
	fstrans_cookie_t cookie;

	if (copy_from_user(&fsx, arg, sizeof (fsx)))
		return (-EFAULT);

	if (!zpl_is_valid_projid(fsx.fsx_projid))
		return (-EINVAL);

	err = __zpl_ioctl_setflags(ip, fsx.fsx_xflags, &xva);
	if (err)
		return (err);

	xoap = xva_getxoptattr(&xva);
	XVA_SET_REQ(&xva, XAT_PROJID);
	xoap->xoa_projid = fsx.fsx_projid;

	crhold(cr);
	cookie = spl_fstrans_mark();
	err = -zfs_setattr(ITOZ(ip), (vattr_t *)&xva, 0, cr, zfs_init_idmap);
	spl_fstrans_unmark(cookie);
	crfree(cr);

	return (err);
}

/*
 * Expose Additional File Level Attributes of ZFS.
 */
static int
zpl_ioctl_getdosflags(struct file *filp, void __user *arg)
{
	struct inode *ip = file_inode(filp);
	uint64_t dosflags = ITOZ(ip)->z_pflags;
	dosflags &= ZFS_DOS_FL_USER_VISIBLE;
	int err = copy_to_user(arg, &dosflags, sizeof (dosflags));

	return (err);
}

static int
__zpl_ioctl_setdosflags(struct inode *ip, uint64_t ioctl_flags, xvattr_t *xva)
{
	uint64_t zfs_flags = ITOZ(ip)->z_pflags;
	xoptattr_t *xoap;

	if (ioctl_flags & (~ZFS_DOS_FL_USER_VISIBLE))
		return (-EOPNOTSUPP);

	if ((fchange(ioctl_flags, zfs_flags, ZFS_IMMUTABLE, ZFS_IMMUTABLE) ||
	    fchange(ioctl_flags, zfs_flags, ZFS_APPENDONLY, ZFS_APPENDONLY)) &&
	    !capable(CAP_LINUX_IMMUTABLE))
		return (-EPERM);

	if (!zpl_inode_owner_or_capable(zfs_init_idmap, ip))
		return (-EACCES);

	xva_init(xva);
	xoap = xva_getxoptattr(xva);

#define	FLAG_CHANGE(iflag, xflag, xfield)	do {	\
	if (((ioctl_flags & (iflag)) && !(zfs_flags & (iflag))) ||	\
	    ((zfs_flags & (iflag)) && !(ioctl_flags & (iflag)))) {	\
		XVA_SET_REQ(xva, (xflag));	\
		(xfield) = ((ioctl_flags & (iflag)) != 0);	\
	}	\
} while (0)

	FLAG_CHANGE(ZFS_IMMUTABLE, XAT_IMMUTABLE, xoap->xoa_immutable);
	FLAG_CHANGE(ZFS_APPENDONLY, XAT_APPENDONLY, xoap->xoa_appendonly);
	FLAG_CHANGE(ZFS_NODUMP, XAT_NODUMP, xoap->xoa_nodump);
	FLAG_CHANGE(ZFS_READONLY, XAT_READONLY, xoap->xoa_readonly);
	FLAG_CHANGE(ZFS_HIDDEN, XAT_HIDDEN, xoap->xoa_hidden);
	FLAG_CHANGE(ZFS_SYSTEM, XAT_SYSTEM, xoap->xoa_system);
	FLAG_CHANGE(ZFS_ARCHIVE, XAT_ARCHIVE, xoap->xoa_archive);
	FLAG_CHANGE(ZFS_NOUNLINK, XAT_NOUNLINK, xoap->xoa_nounlink);
	FLAG_CHANGE(ZFS_REPARSE, XAT_REPARSE, xoap->xoa_reparse);
	FLAG_CHANGE(ZFS_OFFLINE, XAT_OFFLINE, xoap->xoa_offline);
	FLAG_CHANGE(ZFS_SPARSE, XAT_SPARSE, xoap->xoa_sparse);

#undef	FLAG_CHANGE

	return (0);
}

/*
 * Set Additional File Level Attributes of ZFS.
 */
static int
zpl_ioctl_setdosflags(struct file *filp, void __user *arg)
{
	struct inode *ip = file_inode(filp);
	uint64_t dosflags;
	cred_t *cr = CRED();
	xvattr_t xva;
	int err;
	fstrans_cookie_t cookie;

	if (copy_from_user(&dosflags, arg, sizeof (dosflags)))
		return (-EFAULT);

	err = __zpl_ioctl_setdosflags(ip, dosflags, &xva);
	if (err)
		return (err);

	crhold(cr);
	cookie = spl_fstrans_mark();
	err = -zfs_setattr(ITOZ(ip), (vattr_t *)&xva, 0, cr, zfs_init_idmap);
	spl_fstrans_unmark(cookie);
	crfree(cr);

	return (err);
}

static long
zpl_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
{
	switch (cmd) {
	case FS_IOC_GETVERSION:
		return (zpl_ioctl_getversion(filp, (void *)arg));
	case FS_IOC_GETFLAGS:
		return (zpl_ioctl_getflags(filp, (void *)arg));
	case FS_IOC_SETFLAGS:
		return (zpl_ioctl_setflags(filp, (void *)arg));
	case ZFS_IOC_FSGETXATTR:
		return (zpl_ioctl_getxattr(filp, (void *)arg));
	case ZFS_IOC_FSSETXATTR:
		return (zpl_ioctl_setxattr(filp, (void *)arg));
	case ZFS_IOC_GETDOSFLAGS:
		return (zpl_ioctl_getdosflags(filp, (void *)arg));
	case ZFS_IOC_SETDOSFLAGS:
		return (zpl_ioctl_setdosflags(filp, (void *)arg));
	case ZFS_IOC_COMPAT_FICLONE:
		return (zpl_ioctl_ficlone(filp, (void *)arg));
	case ZFS_IOC_COMPAT_FICLONERANGE:
		return (zpl_ioctl_ficlonerange(filp, (void *)arg));
	case ZFS_IOC_COMPAT_FIDEDUPERANGE:
		return (zpl_ioctl_fideduperange(filp, (void *)arg));
	default:
		return (-ENOTTY);
	}
}

#ifdef CONFIG_COMPAT
static long
zpl_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
{
	switch (cmd) {
	case FS_IOC32_GETVERSION:
		cmd = FS_IOC_GETVERSION;
		break;
	case FS_IOC32_GETFLAGS:
		cmd = FS_IOC_GETFLAGS;
		break;
	case FS_IOC32_SETFLAGS:
		cmd = FS_IOC_SETFLAGS;
		break;
	default:
		return (-ENOTTY);
	}
	return (zpl_ioctl(filp, cmd, (unsigned long)compat_ptr(arg)));
}
#endif /* CONFIG_COMPAT */

const struct address_space_operations zpl_address_space_operations = {
#ifdef HAVE_VFS_READPAGES
	.readpages	= zpl_readpages,
#else
	.readahead	= zpl_readahead,
#endif
#ifdef HAVE_VFS_READ_FOLIO
	.read_folio	= zpl_read_folio,
#else
	.readpage	= zpl_readpage,
#endif
	.writepage	= zpl_writepage,
	.writepages	= zpl_writepages,
	.direct_IO	= zpl_direct_IO,
#ifdef HAVE_VFS_SET_PAGE_DIRTY_NOBUFFERS
	.set_page_dirty = __set_page_dirty_nobuffers,
#endif
#ifdef HAVE_VFS_FILEMAP_DIRTY_FOLIO
	.dirty_folio	= filemap_dirty_folio,
#endif
};

#ifdef HAVE_VFS_FILE_OPERATIONS_EXTEND
const struct file_operations_extend zpl_file_operations = {
	.kabi_fops = {
#else
const struct file_operations zpl_file_operations = {
#endif
	.open		= zpl_open,
	.release	= zpl_release,
	.llseek		= zpl_llseek,
#ifdef HAVE_VFS_RW_ITERATE
#ifdef HAVE_NEW_SYNC_READ
	.read		= new_sync_read,
	.write		= new_sync_write,
#endif
	.read_iter	= zpl_iter_read,
	.write_iter	= zpl_iter_write,
#ifdef HAVE_VFS_IOV_ITER
#ifdef HAVE_COPY_SPLICE_READ
	.splice_read	= copy_splice_read,
#else
	.splice_read	= generic_file_splice_read,
#endif
	.splice_write	= iter_file_splice_write,
#endif
#else
	.read		= do_sync_read,
	.write		= do_sync_write,
	.aio_read	= zpl_aio_read,
	.aio_write	= zpl_aio_write,
#endif
	.mmap		= zpl_mmap,
	.fsync		= zpl_fsync,
#ifdef HAVE_FILE_AIO_FSYNC
	.aio_fsync	= zpl_aio_fsync,
#endif
	.fallocate	= zpl_fallocate,
#ifdef HAVE_VFS_COPY_FILE_RANGE
	.copy_file_range	= zpl_copy_file_range,
#endif
#ifdef HAVE_VFS_CLONE_FILE_RANGE
	.clone_file_range	= zpl_clone_file_range,
#endif
#ifdef HAVE_VFS_REMAP_FILE_RANGE
	.remap_file_range	= zpl_remap_file_range,
#endif
#ifdef HAVE_VFS_DEDUPE_FILE_RANGE
	.dedupe_file_range	= zpl_dedupe_file_range,
#endif
#ifdef HAVE_FILE_FADVISE
	.fadvise	= zpl_fadvise,
#endif
	.unlocked_ioctl	= zpl_ioctl,
#ifdef CONFIG_COMPAT
	.compat_ioctl	= zpl_compat_ioctl,
#endif
#ifdef HAVE_VFS_FILE_OPERATIONS_EXTEND
	}, /* kabi_fops */
	.copy_file_range	= zpl_copy_file_range,
	.clone_file_range	= zpl_clone_file_range,
#endif
};

const struct file_operations zpl_dir_file_operations = {
	.llseek		= generic_file_llseek,
	.read		= generic_read_dir,
#if defined(HAVE_VFS_ITERATE_SHARED)
	.iterate_shared	= zpl_iterate,
#elif defined(HAVE_VFS_ITERATE)
	.iterate	= zpl_iterate,
#else
	.readdir	= zpl_readdir,
#endif
	.fsync		= zpl_fsync,
	.unlocked_ioctl = zpl_ioctl,
#ifdef CONFIG_COMPAT
	.compat_ioctl   = zpl_compat_ioctl,
#endif
};

/* CSTYLED */
module_param(zfs_fallocate_reserve_percent, uint, 0644);
MODULE_PARM_DESC(zfs_fallocate_reserve_percent,
	"Percentage of length to use for the available capacity check");