Hello all,

The current version of the VFS locking patch adds a new semaphore to
fs/super.c.  This is used to make sure a filesystem does not get mounted
on a logical volume while a snapshot is being taken.  It also results in
all mounts on the system being serialized, and isn't in line with the
VFS locking scheme in general.

I've been meaning to fix it forever, here's an updated version that adds
a super with s->s_dev set to the source volume if nothing is currently
mounted on the source volume.  This allows me to use the s_umount
semaphore in the super block to keep things safe, which is cleaner
overall.  

The other benefit over the existing patch is this one has zero footprint
outside the lockfs calls.  You're only running new code if you take a
snapshot.

I've done some testing here, but wanted to let LVM people review it
before going further.  Patch is below against 2.4.21-rc6.  

This provides zero new functionality over the existing VFS locking
patch, and is experimental.  Do not apply this on production servers,
and do not apply unless you want to help test.

-chris

===== drivers/md/lvm.c 1.19 vs edited =====
--- 1.19/drivers/md/lvm.c	Wed Feb  5 21:53:36 2003
+++ edited/drivers/md/lvm.c	Mon Jun  2 14:19:16 2003
@@ -229,9 +229,6 @@
 #define DEVICE_OFF(device)
 #define LOCAL_END_REQUEST
 
-/* lvm_do_lv_create calls fsync_dev_lockfs()/unlockfs() */
-/* #define	LVM_VFS_ENHANCEMENT */
-
 #include <linux/config.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
@@ -2171,12 +2168,8 @@
 	if (lv_ptr->lv_access & LV_SNAPSHOT) {
 		lv_t *org = lv_ptr->lv_snapshot_org, *last;
 
-		/* sync the original logical volume */
-		fsync_dev(org->lv_dev);
-#ifdef	LVM_VFS_ENHANCEMENT
 		/* VFS function call to sync and lock the filesystem */
 		fsync_dev_lockfs(org->lv_dev);
-#endif
 
 		down_write(&org->lv_lock);
 		org->lv_access |= LV_SNAPSHOT_ORG;
@@ -2201,11 +2194,9 @@
 	else
 		set_device_ro(lv_ptr->lv_dev, 1);
 
-#ifdef	LVM_VFS_ENHANCEMENT
 /* VFS function call to unlock the filesystem */
 	if (lv_ptr->lv_access & LV_SNAPSHOT)
 		unlockfs(lv_ptr->lv_snapshot_org->lv_dev);
-#endif
 
 	lvm_gendisk.part[MINOR(lv_ptr->lv_dev)].de =
 	    lvm_fs_create_lv(vg_ptr, lv_ptr);
===== fs/buffer.c 1.83 vs edited =====
--- 1.83/fs/buffer.c	Wed May 14 12:51:00 2003
+++ edited/fs/buffer.c	Mon Jun  2 14:19:16 2003
@@ -363,6 +363,34 @@
 	fsync_dev(dev);
 }
 
+int fsync_dev_lockfs(kdev_t dev)
+{
+	/* you are not allowed to try locking all the filesystems
+	** on the system, your chances of getting through without
+	** total deadlock are slim to none.
+	*/
+	if (!dev)
+		return fsync_dev(dev) ;
+
+	sync_buffers(dev, 0);
+
+	lock_kernel();
+	/* note, the FS might need to start transactions to 
+	** sync the inodes, or the quota, no locking until
+	** after these are done
+	*/
+	sync_inodes(dev);
+	DQUOT_SYNC(dev);
+	/* if inodes or quotas could be dirtied during the
+	** sync_supers_lockfs call, the FS is responsible for getting
+	** them on disk, without deadlocking against the lock
+	*/
+	sync_supers_lockfs(dev) ;
+	unlock_kernel();
+
+	return sync_buffers(dev, 1) ;
+}
+
 asmlinkage long sys_sync(void)
 {
 	fsync_dev(0);
===== fs/super.c 1.49 vs edited =====
--- 1.49/fs/super.c	Wed Dec 18 21:34:24 2002
+++ edited/fs/super.c	Tue Jun  3 14:54:57 2003
@@ -38,6 +38,12 @@
 spinlock_t sb_lock = SPIN_LOCK_UNLOCKED;
 
 /*
+ * stub of a filesystem used to make sure an FS isn't mounted
+ * in the middle of a lockfs call
+ */
+static DECLARE_FSTYPE_DEV(lockfs_fs_type, "lockfs", NULL);
+
+/*
  * Handling of filesystem drivers list.
  * Rules:
  *	Inclusion to/removals from/scanning of list are protected by spinlock.
@@ -433,6 +439,25 @@
 	put_super(sb);
 }
 
+static void write_super_lockfs(struct super_block *sb)
+{
+	lock_super(sb);
+	if (sb->s_root && sb->s_op) {
+		if (sb->s_dirt && sb->s_op->write_super)
+			sb->s_op->write_super(sb);
+		if (sb->s_op->write_super_lockfs)
+			sb->s_op->write_super_lockfs(sb);
+	}
+	unlock_super(sb);
+
+	/* 
+	 * if no lockfs call is provided, use the sync_fs call instead.
+	 * this must be done without the super lock held
+	 */
+	if (!sb->s_op->write_super_lockfs && sb->s_op->sync_fs)
+		sb->s_op->sync_fs(sb);
+}
+
 static inline void write_super(struct super_block *sb)
 {
 	lock_super(sb);
@@ -480,6 +505,119 @@
 	spin_unlock(&sb_lock);
 }
 
+static struct super_block *find_super_for_lockfs(kdev_t dev)
+{
+	struct super_block *lockfs_sb = alloc_super();
+	struct super_block * s;
+
+	if (!dev)
+		return NULL;
+restart:
+	spin_lock(&sb_lock);
+	s = find_super(dev);
+	if (s) {
+		spin_unlock(&sb_lock);
+		down_read(&s->s_umount);
+		if (s->s_root) {
+			destroy_super(lockfs_sb);
+			return s;
+		}
+		drop_super(s);
+		goto restart;
+	}
+	/* if (s) we either return or goto, so we know s == NULL here.
+	 * At this point, there are no mounted filesystems on this device,
+	 * so we pretend to mount one.
+	 */
+	if (!lockfs_sb) {
+		spin_unlock(&sb_lock);
+		return NULL;
+	}
+	s = lockfs_sb;
+	s->s_dev = dev;
+	if (lockfs_fs_type.fs_supers.prev == NULL)
+		INIT_LIST_HEAD(&lockfs_fs_type.fs_supers);
+	insert_super(s, &lockfs_fs_type);
+	s->s_root = (struct dentry *)1;
+	/* alloc_super gives us a write lock on s_umount, this
+	 * way we know there are no concurrent lockfs holders for this dev.  
+	 * It allows us to remove the temp super from the list of supers 
+	 * immediately when unlockfs is called
+	 */
+	return s;
+}
+/*
+ * Note: don't check the dirty flag before waiting, we want the lock
+ * to happen every time this is called.  dev must be non-zero
+ */
+void sync_supers_lockfs(kdev_t dev)
+{
+	struct super_block *sb;
+	sb = find_super_for_lockfs(dev);
+	if (sb) {
+		write_super_lockfs(sb);
+		/* the drop_super is done by unlockfs */
+	}
+}
+
+static void drop_super_lockfs(struct super_block *s)
+{
+	if (s->s_type == &lockfs_fs_type) {
+		struct file_system_type *fs = s->s_type;
+
+		/* 
+		 * nobody else is allowed to grab_super() on our temp
+		 */
+		if (!deactivate_super(s))
+			BUG();
+
+		spin_lock(&sb_lock);
+		s->s_root = NULL;
+		list_del(&s->s_list);
+		list_del(&s->s_instances);
+		spin_unlock(&sb_lock);
+
+		up_write(&s->s_umount);
+		put_super(s);
+		put_filesystem(fs);
+	} else
+		drop_super(s);
+}
+
+void unlockfs(kdev_t dev)
+{
+	struct super_block *s;
+	if (!dev)
+		return;
+
+	spin_lock(&sb_lock);
+	s = find_super(dev);
+	if (s) {
+		/* 
+		 * find_super and the original lockfs call both incremented
+		 * the reference count.  drop one of them
+		 */
+		s->s_count--;
+		spin_unlock(&sb_lock);
+		if (s->s_root) {
+			if (s->s_op->unlockfs)
+				s->s_op->unlockfs(s);
+			drop_super_lockfs(s);
+			goto out;
+		} else {
+			printk("unlockfs: no s_root, dev %s\n", kdevname(dev));
+			BUG();
+		}
+	} else {
+		printk("unlockfs: no super found, dev %s\n", kdevname(dev));
+		BUG();
+	}
+
+	spin_unlock(&sb_lock);
+out:
+	return;
+}
+
 /**
  *	get_super	-	get the superblock of a device
  *	@dev: device to get the superblock for
===== fs/reiserfs/super.c 1.33 vs edited =====
--- 1.33/fs/reiserfs/super.c	Sat May  3 07:47:21 2003
+++ edited/fs/reiserfs/super.c	Tue Jun  3 14:30:01 2003
@@ -45,7 +45,7 @@
     reiserfs_prepare_for_journal(s, SB_BUFFER_WITH_SB(s), 1);
     journal_mark_dirty(&th, s, SB_BUFFER_WITH_SB (s));
     reiserfs_block_writes(&th) ;
-    journal_end(&th, s, 1) ;
+    journal_end_sync(&th, s, 1) ;
   }
   s->s_dirt = dirty;
   unlock_kernel() ;
===== include/linux/fs.h 1.74 vs edited =====
--- 1.74/include/linux/fs.h	Fri Jan  3 22:09:16 2003
+++ edited/include/linux/fs.h	Tue Jun  3 08:11:27 2003
@@ -1255,6 +1255,7 @@
 extern int sync_buffers(kdev_t, int);
 extern void sync_dev(kdev_t);
 extern int fsync_dev(kdev_t);
+extern int fsync_dev_lockfs(kdev_t);
 extern int fsync_super(struct super_block *);
 extern int fsync_no_super(kdev_t);
 extern void sync_inodes_sb(struct super_block *);
@@ -1271,6 +1272,8 @@
 extern int filemap_fdatasync(struct address_space *);
 extern int filemap_fdatawait(struct address_space *);
 extern void sync_supers(kdev_t dev, int wait);
+extern void sync_supers_lockfs(kdev_t);
+extern void unlockfs(kdev_t);
 extern int bmap(struct inode *, int);
 extern int notify_change(struct dentry *, struct iattr *);
 extern int permission(struct inode *, int);
===== kernel/ksyms.c 1.68 vs edited =====
--- 1.68/kernel/ksyms.c	Fri May 23 17:40:47 2003
+++ edited/kernel/ksyms.c	Mon Jun  2 14:19:16 2003
@@ -186,6 +186,8 @@
 EXPORT_SYMBOL(invalidate_inode_pages);
 EXPORT_SYMBOL(truncate_inode_pages);
 EXPORT_SYMBOL(fsync_dev);
+EXPORT_SYMBOL(fsync_dev_lockfs);
+EXPORT_SYMBOL(unlockfs);
 EXPORT_SYMBOL(fsync_no_super);
 EXPORT_SYMBOL(permission);
 EXPORT_SYMBOL(vfs_permission);
diff -ruN linux-2.4.21/drivers/md/dm-snapshot.c linux/drivers/md/dm-snapshot.c
--- linux-2.4.21/drivers/md/dm-snapshot.c	Wed Jun 18 22:07:15 2003
+++ linux/drivers/md/dm-snapshot.c	Wed Jun 18 22:06:13 2003
@@ -525,7 +525,7 @@
 	}
 
 	/* Flush IO to the origin device */
-	fsync_dev(s->origin->dev);
+	fsync_dev_lockfs(s->origin->dev);
 
 	/* Add snapshot to the list of snapshots for this origin */
 	if (register_snapshot(s)) {
@@ -533,11 +533,13 @@
 		ti->error = "Cannot register snapshot origin";
 		goto bad6;
 	}
+	unlockfs(s->origin->dev);
 
 	ti->private = s;
 	return 0;
 
  bad6:
+	unlockfs(s->origin->dev);
 	kcopyd_client_destroy(s->kcopyd_client);
 
  bad5:
