From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Patrik Kopkan <pkopkan@redhat.com>
Date: Tue, 28 Jul 2020 18:38:07 +0200
Subject: [PATCH] 00335: Backport pathfix change
Tools/scripts/pathfix.py backports
Add -k and -a command line options to preserve and add shebang flags
In upstream since 3.8: https://bugs.python.org/issue37064
Assume all .py files are Python scripts when working recursively:
In upstream since 3.8: https://bugs.python.org/issue38347
Co-authored-by: Victor Stinner <vstinner@redhat.com>
---
Lib/test/test_tools/test_pathfix.py | 129 ++++++++++++++++++
.../2019-05-27-15-26-12.bpo-37064.k_SPW2.rst | 2 +
Tools/scripts/pathfix.py | 60 +++++++-
3 files changed, 186 insertions(+), 5 deletions(-)
create mode 100644 Lib/test/test_tools/test_pathfix.py
create mode 100644 Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst
diff --git a/Lib/test/test_tools/test_pathfix.py b/Lib/test/test_tools/test_pathfix.py
new file mode 100644
index 0000000000..ec361178e6
--- /dev/null
+++ b/Lib/test/test_tools/test_pathfix.py
@@ -0,0 +1,129 @@
+import os
+import subprocess
+import sys
+import unittest
+from test import support
+from test.test_tools import import_tool, scriptsdir, skip_if_missing
+
+
+# need Tools/script/ directory: skip if run on Python installed on the system
+skip_if_missing()
+
+
+class TestPathfixFunctional(unittest.TestCase):
+ script = os.path.join(scriptsdir, 'pathfix.py')
+
+ def setUp(self):
+ self.addCleanup(support.unlink, support.TESTFN)
+
+ def pathfix(self, shebang, pathfix_flags, exitcode=0, stdout='', stderr='',
+ directory=''):
+ if directory:
+ # bpo-38347: Test filename should contain lowercase, uppercase,
+ # "-", "_" and digits.
+ filename = os.path.join(directory, 'script-A_1.py')
+ pathfix_arg = directory
+ else:
+ filename = support.TESTFN
+ pathfix_arg = filename
+
+ with open(filename, 'w', encoding='utf8') as f:
+ f.write(f'{shebang}\n' + 'print("Hello world")\n')
+
+ proc = subprocess.run(
+ [sys.executable, self.script,
+ *pathfix_flags, '-n', pathfix_arg],
+ capture_output=True, text=1)
+
+ if stdout == '' and proc.returncode == 0:
+ stdout = f'{filename}: updating\n'
+ self.assertEqual(proc.returncode, exitcode, proc)
+ self.assertEqual(proc.stdout, stdout, proc)
+ self.assertEqual(proc.stderr, stderr, proc)
+
+ with open(filename, 'r', encoding='utf8') as f:
+ output = f.read()
+
+ lines = output.split('\n')
+ self.assertEqual(lines[1:], ['print("Hello world")', ''])
+ new_shebang = lines[0]
+
+ if proc.returncode != 0:
+ self.assertEqual(shebang, new_shebang)
+
+ return new_shebang
+
+ def test_recursive(self):
+ tmpdir = support.TESTFN + '.d'
+ self.addCleanup(support.rmtree, tmpdir)
+ os.mkdir(tmpdir)
+ expected_stderr = f"recursedown('{os.path.basename(tmpdir)}')\n"
+ self.assertEqual(
+ self.pathfix(
+ '#! /usr/bin/env python',
+ ['-i', '/usr/bin/python3'],
+ directory=tmpdir,
+ stderr=expected_stderr),
+ '#! /usr/bin/python3')
+
+ def test_pathfix(self):
+ self.assertEqual(
+ self.pathfix(
+ '#! /usr/bin/env python',
+ ['-i', '/usr/bin/python3']),
+ '#! /usr/bin/python3')
+ self.assertEqual(
+ self.pathfix(
+ '#! /usr/bin/env python -R',
+ ['-i', '/usr/bin/python3']),
+ '#! /usr/bin/python3')
+
+ def test_pathfix_keeping_flags(self):
+ self.assertEqual(
+ self.pathfix(
+ '#! /usr/bin/env python -R',
+ ['-i', '/usr/bin/python3', '-k']),
+ '#! /usr/bin/python3 -R')
+ self.assertEqual(
+ self.pathfix(
+ '#! /usr/bin/env python',
+ ['-i', '/usr/bin/python3', '-k']),
+ '#! /usr/bin/python3')
+
+ def test_pathfix_adding_flag(self):
+ self.assertEqual(
+ self.pathfix(
+ '#! /usr/bin/env python',
+ ['-i', '/usr/bin/python3', '-a', 's']),
+ '#! /usr/bin/python3 -s')
+ self.assertEqual(
+ self.pathfix(
+ '#! /usr/bin/env python -S',
+ ['-i', '/usr/bin/python3', '-a', 's']),
+ '#! /usr/bin/python3 -s')
+ self.assertEqual(
+ self.pathfix(
+ '#! /usr/bin/env python -V',
+ ['-i', '/usr/bin/python3', '-a', 'v', '-k']),
+ '#! /usr/bin/python3 -vV')
+ self.assertEqual(
+ self.pathfix(
+ '#! /usr/bin/env python',
+ ['-i', '/usr/bin/python3', '-a', 'Rs']),
+ '#! /usr/bin/python3 -Rs')
+ self.assertEqual(
+ self.pathfix(
+ '#! /usr/bin/env python -W default',
+ ['-i', '/usr/bin/python3', '-a', 's', '-k']),
+ '#! /usr/bin/python3 -sW default')
+
+ def test_pathfix_adding_errors(self):
+ self.pathfix(
+ '#! /usr/bin/env python -E',
+ ['-i', '/usr/bin/python3', '-a', 'W default', '-k'],
+ exitcode=2,
+ stderr="-a option doesn't support whitespaces")
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst b/Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst
new file mode 100644
index 0000000000..d1210e2953
--- /dev/null
+++ b/Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst
@@ -0,0 +1,2 @@
+Add option -k to pathscript.py script: preserve shebang flags.
+Add option -a to pathscript.py script: add flags.
diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py
index 28ee428a3a..127c2fe41e 100755
--- a/Tools/scripts/pathfix.py
+++ b/Tools/scripts/pathfix.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
-# Change the #! line occurring in Python scripts. The new interpreter
+# Change the #! line (shebang) occurring in Python scripts. The new interpreter
# pathname must be given with a -i option.
#
# Command line arguments are files or directories to be processed.
@@ -10,7 +10,13 @@
# arguments).
# The original file is kept as a back-up (with a "~" attached to its name),
# -n flag can be used to disable this.
-#
+
+# Sometimes you may find shebangs with flags such as `#! /usr/bin/env python -si`.
+# Normally, pathfix overwrites the entire line, including the flags.
+# To change interpreter and keep flags from the original shebang line, use -k.
+# If you want to keep flags and add to them one single literal flag, use option -a.
+
+
# Undoubtedly you can do this using find and sed or perl, but this is
# a nice example of Python code that recurses down a directory tree
# and uses regular expressions. Also note several subtleties like
@@ -33,16 +39,21 @@ rep = sys.stdout.write
new_interpreter = None
preserve_timestamps = False
create_backup = True
+keep_flags = False
+add_flags = b''
def main():
global new_interpreter
global preserve_timestamps
global create_backup
- usage = ('usage: %s -i /interpreter -p -n file-or-directory ...\n' %
+ global keep_flags
+ global add_flags
+
+ usage = ('usage: %s -i /interpreter -p -n -k -a file-or-directory ...\n' %
sys.argv[0])
try:
- opts, args = getopt.getopt(sys.argv[1:], 'i:pn')
+ opts, args = getopt.getopt(sys.argv[1:], 'i:a:kpn')
except getopt.error as msg:
err(str(msg) + '\n')
err(usage)
@@ -54,6 +65,13 @@ def main():
preserve_timestamps = True
if o == '-n':
create_backup = False
+ if o == '-k':
+ keep_flags = True
+ if o == '-a':
+ add_flags = a.encode()
+ if b' ' in add_flags:
+ err("-a option doesn't support whitespaces")
+ sys.exit(2)
if not new_interpreter or not new_interpreter.startswith(b'/') or \
not args:
err('-i option or file-or-directory missing\n')
@@ -96,6 +114,7 @@ def recursedown(dirname):
if recursedown(fullname): bad = 1
return bad
+
def fix(filename):
## dbg('fix(%r)\n' % (filename,))
try:
@@ -166,12 +185,43 @@ def fix(filename):
# Return success
return 0
+
+def parse_shebang(shebangline):
+ shebangline = shebangline.rstrip(b'\n')
+ start = shebangline.find(b' -')
+ if start == -1:
+ return b''
+ return shebangline[start:]
+
+
+def populate_flags(shebangline):
+ old_flags = b''
+ if keep_flags:
+ old_flags = parse_shebang(shebangline)
+ if old_flags:
+ old_flags = old_flags[2:]
+ if not (old_flags or add_flags):
+ return b''
+ # On Linux, the entire string following the interpreter name
+ # is passed as a single argument to the interpreter.
+ # e.g. "#! /usr/bin/python3 -W Error -s" runs "/usr/bin/python3 "-W Error -s"
+ # so shebang should have single '-' where flags are given and
+ # flag might need argument for that reasons adding new flags is
+ # between '-' and original flags
+ # e.g. #! /usr/bin/python3 -sW Error
+ return b' -' + add_flags + old_flags
+
+
def fixline(line):
if not line.startswith(b'#!'):
return line
+
if b"python" not in line:
return line
- return b'#! ' + new_interpreter + b'\n'
+
+ flags = populate_flags(line)
+ return b'#! ' + new_interpreter + flags + b'\n'
+
if __name__ == '__main__':
main()