Initial LTFS order awarness functionality

Makefile.in

@@ -44,7 +44,7 @@ LIBOBJ=lib/wildmatch.o lib/compat.o lib/snprintf.o lib/mdfour.o lib/md5.o \
 zlib_OBJS=zlib/deflate.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o \
 	zlib/trees.o zlib/zutil.o zlib/adler32.o zlib/compress.o zlib/crc32.o
 OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \
-	util1.o util2.o main.o checksum.o match.o syscall.o android.o log.o backup.o delete.o
+	util1.o util2.o main.o checksum.o match.o syscall.o android.o log.o backup.o delete.o ltfs.o
 OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \
 	usage.o fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
 OBJS3=progress.o pipe.o @MD5_ASM@ @ROLL_SIMD@ @ROLL_ASM@

compat.c

@@ -45,6 +45,7 @@ extern int preserve_uid;
 extern int preserve_gid;
 extern int preserve_atimes;
 extern int preserve_crtimes;
+extern int ltfs_mode;
 extern int preserve_acls;
 extern int preserve_xattrs;
 extern int xfer_flags_as_varint;
@@ -87,7 +88,7 @@ struct name_num_item *xattr_sum_nni;
 int xattr_sum_len = 0;
 
 /* These index values are for the file-list's extra-attribute array. */
-int pathname_ndx, depth_ndx, atimes_ndx, crtimes_ndx, uid_ndx, gid_ndx, acls_ndx, xattrs_ndx, unsort_ndx;
+int pathname_ndx, depth_ndx, atimes_ndx, crtimes_ndx, startblock_ndx, uid_ndx, gid_ndx, acls_ndx, xattrs_ndx, unsort_ndx;
 
 int receiver_symlink_times = 0; /* receiver can set the time on a symlink */
 int sender_symlink_iconv = 0;	/* sender should convert symlink content */
@@ -581,6 +582,10 @@ void setup_protocol(int f_out,int f_in)
 		atimes_ndx = (file_extra_cnt += EXTRA64_CNT);
 	if (preserve_crtimes)
 		crtimes_ndx = (file_extra_cnt += EXTRA64_CNT);
+#ifdef SUPPORT_XATTRS
+	if (ltfs_mode)
+		startblock_ndx = (file_extra_cnt += EXTRA64_CNT);
+#endif
 	if (am_sender) /* This is most likely in the file_extras64 union as well. */
 		pathname_ndx = (file_extra_cnt += PTR_EXTRA_CNT);
 	else

flist.c

@@ -57,6 +57,7 @@ extern int missing_args;
 extern int eol_nulls;
 extern int atimes_ndx;
 extern int crtimes_ndx;
+extern int startblock_ndx;
 extern int relative_paths;
 extern int implied_dirs;
 extern int ignore_perishable;
@@ -602,6 +603,8 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file,
 	if (crtimes_ndx && !(xflags & XMIT_CRTIME_EQ_MTIME))
 		write_varlong(f, crtime, 4);
 #endif
+	if (startblock_ndx)
+		write_varlong(f, F_STARTBLOCK(file), 3);
 	if (!(xflags & XMIT_SAME_MODE))
 		write_int(f, to_wire_mode(mode));
 	if (atimes_ndx && !S_ISDIR(mode) && !(xflags & XMIT_SAME_ATIME))
@@ -697,6 +700,7 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x
 #ifdef SUPPORT_CRTIMES
 	static time_t crtime;
 #endif
+	static int64 startblock;
 	static mode_t mode;
 #ifdef SUPPORT_HARD_LINKS
 	static int64 dev;
@@ -873,6 +877,8 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x
 #endif
 	}
 #endif
+	if (startblock_ndx)
+		startblock = read_varlong(f, 3);
 	if (!(xflags & XMIT_SAME_MODE)) {
 		mode = from_wire_mode(read_int(f));
 		/* Reject modes whose type bits are not one of the standard
@@ -1097,6 +1103,8 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x
 	if (crtimes_ndx)
 		F_CRTIME(file) = crtime;
 #endif
+	if (startblock_ndx)
+		F_STARTBLOCK(file) = startblock;
 	if (unsort_ndx)
 		F_NDX(file) = flist->used + flist->ndx_start;
 
@@ -1518,6 +1526,12 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 	if (crtimes_ndx)
 		F_CRTIME(file) = get_create_time(fname, &st);
 #endif
+	if (startblock_ndx) {
+		int64 blk = am_sender && S_ISREG(file->mode) ? ltfs_startblock(fname) : -1;
+		/* Unknown sorts first; keep it non-negative so write_varlong
+		 * stays compact (a real LTFS data block is well past block 0). */
+		F_STARTBLOCK(file) = blk < 0 ? 0 : blk;
+	}
 
 	if (basename != thisname)
 		file->dirname = lastdir;

generator.c

@@ -32,6 +32,7 @@ extern int am_root;
 extern int am_server;
 extern int am_daemon;
 extern int inc_recurse;
+extern int ltfs_mode;
 extern int relative_paths;
 extern int implied_dirs;
 extern int keep_dirlinks;
@@ -2243,6 +2244,7 @@ void check_for_finished_files(int itemizing, enum logcode code, int check_redo)
 void generate_files(int f_out, const char *local_name)
 {
 	int i, ndx, next_loopchk = 0;
+	int *ltfs_order = NULL;
 	char fbuf[MAXPATHLEN];
 	int itemizing;
 	enum logcode code;
@@ -2326,16 +2328,23 @@ void generate_files(int f_out, const char *local_name)
 					change_local_filter_dir(fbuf, strlen(fbuf), F_DEPTH(fp));
 			}
 		}
+		/* For an LTFS source, read the files in physical tape order
+		 * (by start block) rather than name order, so the drive makes
+		 * one forward streaming pass instead of seeking back and forth.
+		 * ltfs_order maps the natural sweep position to a sorted[] index;
+		 * a NULL result falls back to the natural low..high order. */
+		ltfs_order = ltfs_mode ? ltfs_build_order(cur_flist) : NULL;
 		for (i = cur_flist->low; i <= cur_flist->high; i++) {
-			struct file_struct *file = cur_flist->sorted[i];
+			int si = ltfs_order ? ltfs_order[i - cur_flist->low] : i;
+			struct file_struct *file = cur_flist->sorted[si];
 
 			if (!F_IS_ACTIVE(file))
 				continue;
 
 			if (unsort_ndx)
 				ndx = F_NDX(file);
 			else
-				ndx = i + cur_flist->ndx_start;
+				ndx = si + cur_flist->ndx_start;
 
 			if (solo_file)
 				strlcpy(fbuf, solo_file, sizeof fbuf);
@@ -2354,6 +2363,9 @@ void generate_files(int f_out, const char *local_name)
 			}
 		}
 
+		if (ltfs_order)
+			free(ltfs_order);
+
 		if (!inc_recurse) {
 			write_ndx(f_out, NDX_DONE);
 			break;

ltfs.c

@@ -0,0 +1,136 @@
+/*
+ * LTFS (Linear Tape File System) awareness for rsync.
+ *
+ * Copyright (C) 2026 Wayne Davison & contributors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, visit the http://fsf.org website.
+ *
+ * ---------------------------------------------------------------------------
+ *
+ * On an LTFS-mounted tape every file's metadata -- name, size, mtime, and the
+ * physical block where its data begins -- lives in the tape index, which is
+ * served from memory once the cartridge is mounted.  Reading that metadata is
+ * therefore cheap, so building rsync's file list from an LTFS source is fast.
+ *
+ * Reading file *content*, on the other hand, requires physically positioning
+ * the tape.  rsync's normal name-sorted traversal bears no relation to the
+ * order in which bytes are laid down on the medium, so a restore that opens
+ * files in name order makes the drive seek back and forth ("shoe-shining"),
+ * turning a single forward streaming pass into hours of repositioning.
+ *
+ * This module lets the generator drive the data-read phase in physical
+ * (start-block) order instead.  LTFS exposes each file's starting block as a
+ * virtual extended attribute; we read it during generation and hand the
+ * generator a permutation of the file list sorted by that block, so the tape
+ * streams forward in one pass.  Entries whose start block is unknown
+ * (directories, symlinks, files not on tape) sort first, in their original
+ * order, which conveniently front-loads directory creation before the bulk
+ * data read begins.
+ */
+
+#include "rsync.h"
+#ifdef SUPPORT_XATTRS
+#include "lib/sysxattrs.h"
+#endif
+
+extern int ltfs_mode;
+extern int startblock_ndx;
+
+/* Return the LTFS starting block of fname, or -1 if it cannot be determined.
+ * The attribute value is an ASCII decimal block number. */
+int64 ltfs_startblock(const char *fname)
+{
+#ifdef SUPPORT_XATTRS
+	/* The virtual xattr names under which LTFS publishes a file's starting
+	 * data block.  The bare "ltfs.*" name is what an LTFS FUSE mount
+	 * presents; the "user.ltfs.*" alias lets the feature be exercised on an
+	 * ordinary filesystem (e.g. by the test suite) where only the "user."
+	 * namespace is writable. */
+	static const char *startblock_attrs[] = {
+		"ltfs.startblock",
+		"user.ltfs.startblock",
+	};
+	char buf[32];
+	unsigned int i;
+
+	for (i = 0; i < sizeof startblock_attrs / sizeof startblock_attrs[0]; i++) {
+		ssize_t len = sys_lgetxattr(fname, startblock_attrs[i], buf, sizeof buf - 1);
+		if (len > 0) {
+			char *end;
+			int64 blk;
+			buf[len] = '\0';
+			blk = (int64)strtoll(buf, &end, 10);
+			if (end != buf && blk >= 0)
+				return blk;
+		}
+	}
+#else
+	(void)fname;
+#endif
+	return -1;
+}
+
+struct ltfs_ent {
+	int64 startblock;
+	int idx;		/* index into flist->sorted[] */
+};
+
+static int ltfs_ent_cmp(const void *a, const void *b)
+{
+	const struct ltfs_ent *ea = a, *eb = b;
+
+	if (ea->startblock != eb->startblock)
+		return ea->startblock < eb->startblock ? -1 : 1;
+	/* Stable tie-break so unknown-block entries (all -1, e.g. directories)
+	 * keep their original parent-before-child name ordering. */
+	return ea->idx < eb->idx ? -1 : ea->idx > eb->idx ? 1 : 0;
+}
+
+/* Build a tape-physical read order for the active range of flist.  Returns a
+ * malloc'd array of (flist->high - flist->low + 1) entries, each an index into
+ * flist->sorted[], ordered by ascending LTFS start block.  The caller iterates
+ * the returned array in place of the natural low..high sweep.  Returns NULL
+ * (caller falls back to natural order) if ltfs_mode is off, no start-block
+ * metadata was negotiated, or the range is empty. */
+int *ltfs_build_order(struct file_list *flist)
+{
+	struct ltfs_ent *ents;
+	int *order;
+	int n, j, count;
+
+	if (!ltfs_mode || !startblock_ndx || flist->high < flist->low)
+		return NULL;
+
+	n = flist->high - flist->low + 1;
+	ents = new_array(struct ltfs_ent, n);
+	order = new_array(int, n);
+
+	for (j = 0, count = 0; j < n; j++) {
+		struct file_struct *file = flist->sorted[flist->low + j];
+		ents[count].idx = flist->low + j;
+		if (F_IS_ACTIVE(file) && S_ISREG(file->mode))
+			ents[count].startblock = F_STARTBLOCK(file);
+		else
+			ents[count].startblock = -1;
+		count++;
+	}
+
+	qsort(ents, count, sizeof ents[0], ltfs_ent_cmp);
+
+	for (j = 0; j < count; j++)
+		order[j] = ents[j].idx;
+
+	free(ents);
+	return order;
+}

options.c

@@ -112,6 +112,7 @@ int human_readable = 1;
 int recurse = 0;
 int mkpath_dest_arg = 0;
 int allow_inc_recurse = 1;
+int ltfs_mode = 0;
 int xfer_dirs = -1;
 int am_daemon = 0;
 /* Set after a successful per-module chroot ("use chroot = yes") in
@@ -594,7 +595,7 @@ enum {OPT_SERVER = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
       OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_INFO, OPT_DEBUG, OPT_BLOCK_SIZE,
       OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT, OPT_STDERR,
       OPT_OLD_COMPRESS, OPT_NEW_COMPRESS, OPT_NO_COMPRESS, OPT_OLD_ARGS,
-      OPT_STOP_AFTER, OPT_STOP_AT,
+      OPT_STOP_AFTER, OPT_STOP_AT, OPT_LTFS,
       OPT_REFUSED_BASE = 9000};
 
 static struct poptOption long_options[] = {
@@ -623,6 +624,8 @@ static struct poptOption long_options[] = {
   {"no-r",             0,  POPT_ARG_VAL,    &recurse, 0, 0, 0 },
   {"inc-recursive",    0,  POPT_ARG_VAL,    &allow_inc_recurse, 1, 0, 0 },
   {"no-inc-recursive", 0,  POPT_ARG_VAL,    &allow_inc_recurse, 0, 0, 0 },
+  {"ltfs",             0,  POPT_ARG_NONE,   0, OPT_LTFS, 0, 0 },
+  {"no-ltfs",          0,  POPT_ARG_VAL,    &ltfs_mode, 0, 0, 0 },
   {"i-r",              0,  POPT_ARG_VAL,    &allow_inc_recurse, 1, 0, 0 },
   {"no-i-r",           0,  POPT_ARG_VAL,    &allow_inc_recurse, 0, 0, 0 },
   {"dirs",            'd', POPT_ARG_VAL,    &xfer_dirs, 2, 0, 0 },
@@ -1028,6 +1031,11 @@ static void set_refuse_options(void)
 #ifndef SUPPORT_CRTIMES
 	parse_one_refuse_match(0, "crtimes", list_end);
 #endif
+#ifndef SUPPORT_XATTRS
+	/* --ltfs orders the read by each file's ltfs.startblock xattr, so it is
+	 * meaningless (and would silently no-op) without xattr support. */
+	parse_one_refuse_match(0, "ltfs", list_end);
+#endif
 
 	/* Now we use the descrip values to actually mark the options for refusal. */
 	for (op = long_options; op != list_end; op++) {
@@ -1569,6 +1577,14 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 			preserve_devices = preserve_specials = 0;
 			break;
 
+		case OPT_LTFS:
+			ltfs_mode = 1;
+			/* Imply -t (like --archive does) so the index quick-check can
+			 * skip unchanged files on a later run.  Processed here in option
+			 * order so a subsequent --no-times can still override it. */
+			preserve_mtimes = 1;
+			break;
+
 		case 'h':
 			human_readable++;
 			break;
@@ -2397,6 +2413,30 @@ int parse_arguments(int *argc_p, const char ***argv_p)
 			bwlimit_writemax = 512;
 	}
 
+	if (ltfs_mode) {
+		/* A delta read would only re-read the source file we must
+		 * stream off the tape anyway, so force whole-file. */
+		if (whole_file < 0)
+			whole_file = 1;
+		/* We need the complete file list before we can order the read
+		 * by physical block, so incremental recursion is incompatible. */
+		allow_inc_recurse = 0;
+		/* --checksum would read every byte of every file off the tape
+		 * just to decide what to transfer, defeating the whole point. */
+		if (always_checksum) {
+			snprintf(err_buf, sizeof err_buf,
+				 "--checksum cannot be used with --ltfs (it would read the entire tape)\n");
+			goto cleanup;
+		}
+		/* --ltfs implies -t (see OPT_LTFS); a 0 here means the user added an
+		 * explicit --no-times afterward.  We honor it, but warn: without
+		 * preserved mtimes the index quick-check can't skip unchanged files,
+		 * so every run re-reads the whole tape. */
+		if (!preserve_mtimes && !am_server)
+			rprintf(FWARNING,
+				"--ltfs with --no-times: unchanged files cannot be skipped by mtime, so every run re-reads the tape.\n");
+	}
+
 	if (append_mode) {
 		if (whole_file > 0) {
 			snprintf(err_buf, sizeof err_buf,
@@ -2765,6 +2805,11 @@ void server_options(char **args, int *argc_p)
 	} else if (preserve_specials)
 		args[ac++] = "--specials";
 
+	/* The sender reads the start-block metadata and both sides must agree
+	 * on the file-list extra layout, so tell the server side about --ltfs. */
+	if (ltfs_mode)
+		args[ac++] = "--ltfs";
+
 	/* The server side doesn't use our log-format, but in certain
 	 * circumstances they need to know a little about the option. */
 	if (stdout_format && am_sender) {