main: refuse to write the output into the input file

Makefile.am

@@ -309,6 +309,7 @@ EXTRA_DIST=								\
 	tests/08-passphrase-file.sh					\
 	tests/09-explicit-params.sh					\
 	tests/10-check-params.sh					\
+	tests/12-same-file.sh						\
 	tests/shared_test_functions.sh					\
 	tests/shared_valgrind_functions.sh				\
 	tests/test_scrypt.sh						\

main.c

@@ -25,12 +25,17 @@
  */
 #include "platform.h"
 
+#include <sys/types.h>
+#include <sys/stat.h>
+
 #include <errno.h>
+#include <fcntl.h>
 #include <math.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <unistd.h>
 
 #include "getopt.h"
 #include "humansize.h"
@@ -96,6 +101,101 @@ scrypt_mode_info(const char * infilename)
 	return (-1);
 }
 
+/**
+ * same_destructive_object(sb_in, sb_out):
+ * Return non-zero if the two stat results identify storage which scrypt must
+ * not read and write at the same time.  Regular files are identified by
+ * device/inode; block-device aliases are identified by the underlying device.
+ * Character devices such as /dev/null remain permitted.
+ */
+static int
+same_destructive_object(const struct stat * sb_in, const struct stat * sb_out)
+{
+
+	if (S_ISREG(sb_in->st_mode) && S_ISREG(sb_out->st_mode))
+		return ((sb_in->st_dev == sb_out->st_dev) &&
+		    (sb_in->st_ino == sb_out->st_ino));
+
+	if (S_ISBLK(sb_in->st_mode) && S_ISBLK(sb_out->st_mode))
+		return (sb_in->st_rdev == sb_out->st_rdev);
+
+	return (0);
+}
+
+/**
+ * same_file(infile, outfilename):
+ * Return non-zero if the already-open ${infile} and the path ${outfilename}
+ * identify the same destructive storage object.  If ${outfilename} is NULL,
+ * compare against standard output instead.  If we cannot tell -- most
+ * importantly if ${outfilename} does not exist yet -- return zero.
+ */
+static int
+same_file(FILE * infile, const char * outfilename)
+{
+	struct stat sb_in;
+	struct stat sb_out;
+
+	/* If we can't stat either file, assume that they're different. */
+	if (fstat(fileno(infile), &sb_in))
+		return (0);
+	if (outfilename != NULL) {
+		if (stat(outfilename, &sb_out))
+			return (0);
+	} else if (fstat(fileno(stdout), &sb_out)) {
+		return (0);
+	}
+
+	return (same_destructive_object(&sb_in, &sb_out));
+}
+
+/**
+ * open_output(infile, outfilename, outfile):
+ * Open ${outfilename} for writing without truncating it until the descriptor
+ * we actually opened has been proved to be different from ${infile}.  Return
+ * 1 if both descriptors identify the same destructive object, -1 on error,
+ * and 0
+ * on success with ${outfile} set.
+ */
+static int
+open_output(FILE * infile, const char * outfilename, FILE ** outfile)
+{
+	struct stat sb_in;
+	struct stat sb_out;
+	int fd;
+	int saved_errno;
+
+	/* Do not truncate until we have compared the opened descriptor. */
+	if ((fd = open(outfilename, O_WRONLY | O_CREAT, 0666)) == -1)
+		return (-1);
+
+	/* Compare the actual objects behind both open descriptors. */
+	if (fstat(fileno(infile), &sb_in))
+		goto err0;
+	if (fstat(fd, &sb_out))
+		goto err0;
+	if (same_destructive_object(&sb_in, &sb_out)) {
+		(void)close(fd);
+		return (1);
+	}
+
+	/* Match fopen(..., "wb") truncation for regular output files. */
+	if (S_ISREG(sb_out.st_mode) && ftruncate(fd, 0))
+		goto err0;
+
+	/* fdopen does not truncate an already-open descriptor. */
+	if ((*outfile = fdopen(fd, "wb")) == NULL)
+		goto err0;
+
+	/* Success! */
+	return (0);
+
+err0:
+	saved_errno = errno;
+	(void)close(fd);
+	errno = saved_errno;
+	return (-1);
+}
+
 /**
  * scrypt_mode_enc_dec(params, passphrase_entry, passphrase_arg, dec, verbose,
  *     force_resources, infilename, outfilename):
@@ -117,6 +217,7 @@ scrypt_mode_enc_dec(struct scryptenc_params params,
 	FILE * infile;
 	FILE * outfile = stdout;
 	char * passwd;
+	int openrc;
 	int rc;
 
 	/* If the input isn't stdin, open the file. */
@@ -129,6 +230,17 @@ scrypt_mode_enc_dec(struct scryptenc_params params,
 		infile = stdin;
 	}
 
+	/*
+	 * Refuse obvious same-file aliases before prompting.  open_output()
+	 * repeats this check on the descriptor it actually opens, closing the
+	 * path race before any regular output file is truncated.
+	 */
+	if (same_file(infile, outfilename)) {
+		warn0("Input and output files are the same: %s",
+		    outfilename != NULL ? outfilename : "standard output");
+		goto err1;
+	}
+
 	/* Get the password. */
 	if (passphrase_entry_readpass(&passwd, passphrase_entry,
 	    passphrase_arg, "Please enter passphrase",
@@ -154,10 +266,14 @@ scrypt_mode_enc_dec(struct scryptenc_params params,
 		}
 	}
 
-	/* If we have an output filename, open it. */
+	/* Bind the destructive same-file check to the output we actually opened. */
 	if (outfilename != NULL) {
-		if ((outfile = fopen(outfilename, "wb")) == NULL) {
-			warnp("Cannot open output file: %s", outfilename);
+		if ((openrc = open_output(infile, outfilename, &outfile)) != 0) {
+			if (openrc > 0)
+				warn0("Input and output files are the same: %s",
+				    outfilename);
+			else
+				warnp("Cannot open output file: %s", outfilename);
 			goto err2;
 		}
 	}

scrypt.1

@@ -63,6 +63,21 @@ if specified, or the standard output otherwise.
 The user will be prompted to enter the passphrase used at
 encryption time to generate the derived encryption key.
 .Pp
+.Ar outfile
+cannot be the same file as
+.Ar infile :
+.Nm
+truncates
+.Ar outfile
+before it reads
+.Ar infile ,
+so it exits with an error instead of destroying the input.
+The same restriction applies when standard output refers to the input file.
+This check cannot restore an input which the shell has already truncated
+before
+.Nm
+starts.
+.Pp
 .Nm Cm info
 provides information about the encryption parameters used for
 .Ar infile .