Mini Shell
# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
# For details: https://github.com/pylint-dev/pylint/blob/main/LICENSE
# Copyright (c) https://github.com/pylint-dev/pylint/blob/main/CONTRIBUTORS.txt
"""Unicode and some other ASCII characters can be used to create programs that run
much different compared to what a human reader would expect from them.
PEP 672 lists some examples.
See: https://www.python.org/dev/peps/pep-0672/
The following checkers are intended to make users are aware of these issues.
"""
from __future__ import annotations
import codecs
import contextlib
import io
import re
from collections import OrderedDict
from collections.abc import Iterable
from functools import lru_cache
from tokenize import detect_encoding
from typing import NamedTuple, TypeVar
from astroid import nodes
import pylint.interfaces
import pylint.lint
from pylint import checkers
_StrLike = TypeVar("_StrLike", str, bytes)
# Based on:
# https://golangexample.com/go-linter-which-checks-for-dangerous-unicode-character-sequences/
# We use '\u' because it doesn't require a map lookup and is therefore faster
BIDI_UNICODE = [
"\u202A", # \N{LEFT-TO-RIGHT EMBEDDING}
"\u202B", # \N{RIGHT-TO-LEFT EMBEDDING}
"\u202C", # \N{POP DIRECTIONAL FORMATTING}
"\u202D", # \N{LEFT-TO-RIGHT OVERRIDE}
"\u202E", # \N{RIGHT-TO-LEFT OVERRIDE}
"\u2066", # \N{LEFT-TO-RIGHT ISOLATE}
"\u2067", # \N{RIGHT-TO-LEFT ISOLATE}
"\u2068", # \N{FIRST STRONG ISOLATE}
"\u2069", # \N{POP DIRECTIONAL ISOLATE}
# The following was part of PEP 672:
# https://www.python.org/dev/peps/pep-0672/
# so the list above might not be complete
"\u200F", # \n{RIGHT-TO-LEFT MARK}
# We don't use
# "\u200E" # \n{LEFT-TO-RIGHT MARK}
# as this is the default for latin files and can't be used
# to hide code
]
class _BadChar(NamedTuple):
"""Representation of an ASCII char considered bad."""
name: str
unescaped: str
escaped: str
code: str
help_text: str
def description(self) -> str:
"""Used for the detailed error message description."""
return (
f"Invalid unescaped character {self.name}, "
f'use "{self.escaped}" instead.'
)
def human_code(self) -> str:
"""Used to generate the human readable error message."""
return f"invalid-character-{self.name}"
# Based on https://www.python.org/dev/peps/pep-0672/
BAD_CHARS = [
_BadChar(
"backspace",
"\b",
"\\b",
"E2510",
(
"Moves the cursor back, so the character after it will overwrite the "
"character before."
),
),
_BadChar(
"carriage-return",
"\r",
"\\r",
"E2511",
(
"Moves the cursor to the start of line, subsequent characters overwrite "
"the start of the line."
),
),
_BadChar(
"sub",
"\x1A",
"\\x1A",
"E2512",
(
'Ctrl+Z "End of text" on Windows. Some programs (such as type) ignore '
"the rest of the file after it."
),
),
_BadChar(
"esc",
"\x1B",
"\\x1B",
"E2513",
(
"Commonly initiates escape codes which allow arbitrary control "
"of the terminal."
),
),
_BadChar(
"nul",
"\0",
"\\0",
"E2514",
"Mostly end of input for python.",
),
_BadChar(
# Zero Width with Space. At the time of writing not accepted by Python.
# But used in Trojan Source Examples, so still included and tested for.
"zero-width-space",
"\u200B", # \n{ZERO WIDTH SPACE}
"\\u200B",
"E2515",
"Invisible space character could hide real code execution.",
),
]
BAD_ASCII_SEARCH_DICT = {char.unescaped: char for char in BAD_CHARS}
def _line_length(line: _StrLike, codec: str) -> int:
"""Get the length of a string like line as displayed in an editor."""
if isinstance(line, bytes):
decoded = _remove_bom(line, codec).decode(codec, "replace")
else:
decoded = line
stripped = decoded.rstrip("\n")
if stripped != decoded:
stripped = stripped.rstrip("\r")
return len(stripped)
def _map_positions_to_result(
line: _StrLike,
search_dict: dict[_StrLike, _BadChar],
new_line: _StrLike,
byte_str_length: int = 1,
) -> dict[int, _BadChar]:
"""Get all occurrences of search dict keys within line.
Ignores Windows end of line and can handle bytes as well as string.
Also takes care of encodings for which the length of an encoded code point does not
default to 8 Bit.
"""
result: dict[int, _BadChar] = {}
for search_for, char in search_dict.items():
if search_for not in line:
continue
# Special Handling for Windows '\r\n'
if char.unescaped == "\r" and line.endswith(new_line):
ignore_pos = len(line) - 2 * byte_str_length
else:
ignore_pos = None
start = 0
pos = line.find(search_for, start)
while pos > 0:
if pos != ignore_pos:
# Calculate the column
col = int(pos / byte_str_length)
result[col] = char
start = pos + 1
pos = line.find(search_for, start)
return result
UNICODE_BOMS = {
"utf-8": codecs.BOM_UTF8,
"utf-16": codecs.BOM_UTF16,
"utf-32": codecs.BOM_UTF32,
"utf-16le": codecs.BOM_UTF16_LE,
"utf-16be": codecs.BOM_UTF16_BE,
"utf-32le": codecs.BOM_UTF32_LE,
"utf-32be": codecs.BOM_UTF32_BE,
}
BOM_SORTED_TO_CODEC = OrderedDict(
# Sorted by length of BOM of each codec
(UNICODE_BOMS[codec], codec)
for codec in ("utf-32le", "utf-32be", "utf-8", "utf-16le", "utf-16be")
)
UTF_NAME_REGEX_COMPILED = re.compile(
"utf[ -]?(8|16|32)[ -]?(le|be|)?(sig)?", flags=re.IGNORECASE
)
def _normalize_codec_name(codec: str) -> str:
"""Make sure the codec name is always given as defined in the BOM dict."""
return UTF_NAME_REGEX_COMPILED.sub(r"utf-\1\2", codec).lower()
def _remove_bom(encoded: bytes, encoding: str) -> bytes:
"""Remove the bom if given from a line."""
if encoding not in UNICODE_BOMS:
return encoded
bom = UNICODE_BOMS[encoding]
if encoded.startswith(bom):
return encoded[len(bom) :]
return encoded
def _encode_without_bom(string: str, encoding: str) -> bytes:
"""Encode a string but remove the BOM."""
return _remove_bom(string.encode(encoding), encoding)
def _byte_to_str_length(codec: str) -> int:
"""Return how many byte are usually(!) a character point."""
if codec.startswith("utf-32"):
return 4
if codec.startswith("utf-16"):
return 2
return 1
@lru_cache(maxsize=1000)
def _cached_encode_search(string: str, encoding: str) -> bytes:
"""A cached version of encode used for search pattern."""
return _encode_without_bom(string, encoding)
def _fix_utf16_32_line_stream(steam: Iterable[bytes], codec: str) -> Iterable[bytes]:
r"""Handle line ending for UTF16 and UTF32 correctly.
Currently, Python simply strips the required zeros after \n after the
line ending. Leading to lines that can't be decoded properly
"""
if not codec.startswith("utf-16") and not codec.startswith("utf-32"):
yield from steam
else:
# First we get all the bytes in memory
content = b"".join(line for line in steam)
new_line = _cached_encode_search("\n", codec)
# Now we split the line by the real new line in the correct encoding
# we can't use split as it would strip the \n that we need
start = 0
while True:
pos = content.find(new_line, start)
if pos >= 0:
yield content[start : pos + len(new_line)]
else:
# Yield the rest and finish
if content[start:]:
yield content[start:]
break
start = pos + len(new_line)
def extract_codec_from_bom(first_line: bytes) -> str:
"""Try to extract the codec (unicode only) by checking for the BOM.
For details about BOM see https://unicode.org/faq/utf_bom.html#BOM
Args:
first_line: the first line of a file
Returns:
a codec name
Raises:
ValueError: if no codec was found
"""
for bom, codec in BOM_SORTED_TO_CODEC.items():
if first_line.startswith(bom):
return codec
raise ValueError("No BOM found. Could not detect Unicode codec.")
class UnicodeChecker(checkers.BaseRawFileChecker):
"""Check characters that could be used to hide bad code to humans.
This includes:
- Bidirectional Unicode (see https://trojansource.codes/)
- Bad ASCII characters (see PEP672)
If a programmer requires to use such a character they should use the escaped
version, that is also much easier to read and does not depend on the editor used.
The Checker also includes a check that UTF-16 and UTF-32 are not used to encode
Python files.
At the time of writing Python supported only UTF-8. See
https://stackoverflow.com/questions/69897842/ and https://bugs.python.org/issue1503789
for background.
"""
name = "unicode_checker"
msgs = {
"E2501": (
# This error will be only displayed to users once Python Supports
# UTF-16/UTF-32 (if at all)
"UTF-16 and UTF-32 aren't backward compatible. Use UTF-8 instead",
"invalid-unicode-codec",
(
"For compatibility use UTF-8 instead of UTF-16/UTF-32. "
"See also https://bugs.python.org/issue1503789 for a history "
"of this issue. And "
"https://softwareengineering.stackexchange.com/questions/102205/ "
"for some possible problems when using UTF-16 for instance."
),
),
"E2502": (
(
"Contains control characters that can permit obfuscated code "
"executed differently than displayed"
),
"bidirectional-unicode",
(
"bidirectional unicode are typically not displayed characters required "
"to display right-to-left (RTL) script "
"(i.e. Chinese, Japanese, Arabic, Hebrew, ...) correctly. "
"So can you trust this code? "
"Are you sure it displayed correctly in all editors? "
"If you did not write it or your language is not RTL,"
" remove the special characters, as they could be used to trick you into "
"executing code, "
"that does something else than what it looks like.\n"
"More Information:\n"
"https://en.wikipedia.org/wiki/Bidirectional_text\n"
"https://trojansource.codes/"
),
),
"C2503": (
"PEP8 recommends UTF-8 as encoding for Python files",
"bad-file-encoding",
(
"PEP8 recommends UTF-8 default encoding for Python files. See "
"https://peps.python.org/pep-0008/#source-file-encoding"
),
),
**{
bad_char.code: (
bad_char.description(),
bad_char.human_code(),
bad_char.help_text,
)
for bad_char in BAD_CHARS
},
}
@staticmethod
def _is_invalid_codec(codec: str) -> bool:
return codec.startswith(("utf-16", "utf-32"))
@staticmethod
def _is_unicode(codec: str) -> bool:
return codec.startswith("utf")
@classmethod
def _find_line_matches(cls, line: bytes, codec: str) -> dict[int, _BadChar]:
"""Find all matches of BAD_CHARS within line.
Args:
line: the input
codec: that will be used to convert line/or search string into
Return:
A dictionary with the column offset and the BadASCIIChar
"""
# We try to decode in Unicode to get the correct column offset
# if we would use bytes, it could be off because UTF-8 has no fixed length
try:
line_search = line.decode(codec, errors="strict")
search_dict = BAD_ASCII_SEARCH_DICT
return _map_positions_to_result(line_search, search_dict, "\n")
except UnicodeDecodeError:
# If we can't decode properly, we simply use bytes, even so the column offsets
# might be wrong a bit, but it is still better then nothing
line_search_byte = line
search_dict_byte: dict[bytes, _BadChar] = {}
for char in BAD_CHARS:
# Some characters might not exist in all encodings
with contextlib.suppress(UnicodeDecodeError):
search_dict_byte[_cached_encode_search(char.unescaped, codec)] = (
char
)
return _map_positions_to_result(
line_search_byte,
search_dict_byte,
_cached_encode_search("\n", codec),
byte_str_length=_byte_to_str_length(codec),
)
@staticmethod
def _determine_codec(stream: io.BytesIO) -> tuple[str, int]:
"""Determine the codec from the given stream.
first tries https://www.python.org/dev/peps/pep-0263/
and if this fails also checks for BOMs of UTF-16 and UTF-32
to be future-proof.
Args:
stream: The byte stream to analyse
Returns: A tuple consisting of:
- normalized codec name
- the line in which the codec was found
Raises:
SyntaxError: if failing to detect codec
"""
try:
# First try to detect encoding with PEP 263
# Doesn't work with UTF-16/32 at the time of writing
# see https://bugs.python.org/issue1503789
codec, lines = detect_encoding(stream.readline)
# lines are empty if UTF-8 BOM is found
codec_definition_line = len(lines) or 1
except SyntaxError as e:
# Codec could not be detected by Python, we try manually to check for
# UTF 16/32 BOMs, which aren't supported by Python at the time of writing.
# This is only included to be future save and handle these codecs as well
stream.seek(0)
try:
codec = extract_codec_from_bom(stream.readline())
codec_definition_line = 1
except ValueError as ve:
# Failed to detect codec, so the syntax error originated not from
# UTF16/32 codec usage. So simply raise the error again.
raise e from ve
return _normalize_codec_name(codec), codec_definition_line
def _check_codec(self, codec: str, codec_definition_line: int) -> None:
"""Check validity of the codec."""
if codec != "utf-8":
msg = "bad-file-encoding"
if self._is_invalid_codec(codec):
msg = "invalid-unicode-codec"
self.add_message(
msg,
# Currently Nodes will lead to crashes of pylint
# node=node,
line=codec_definition_line,
end_lineno=codec_definition_line,
confidence=pylint.interfaces.HIGH,
col_offset=None,
end_col_offset=None,
)
def _check_invalid_chars(self, line: bytes, lineno: int, codec: str) -> None:
"""Look for chars considered bad."""
matches = self._find_line_matches(line, codec)
for col, char in matches.items():
self.add_message(
char.human_code(),
# Currently Nodes will lead to crashes of pylint
# node=node,
line=lineno,
end_lineno=lineno,
confidence=pylint.interfaces.HIGH,
col_offset=col + 1,
end_col_offset=col + len(char.unescaped) + 1,
)
def _check_bidi_chars(self, line: bytes, lineno: int, codec: str) -> None:
"""Look for Bidirectional Unicode, if we use unicode."""
if not self._is_unicode(codec):
return
for dangerous in BIDI_UNICODE:
if _cached_encode_search(dangerous, codec) in line:
# Note that we don't add a col_offset on purpose:
# Using these unicode characters it depends on the editor
# how it displays the location of characters in the line.
# So we mark the complete line.
self.add_message(
"bidirectional-unicode",
# Currently Nodes will lead to crashes of pylint
# node=node,
line=lineno,
end_lineno=lineno,
# We mark the complete line, as bidi controls make it hard
# to determine the correct cursor position within an editor
col_offset=0,
end_col_offset=_line_length(line, codec),
confidence=pylint.interfaces.HIGH,
)
# We look for bidirectional unicode only once per line
# as we mark the complete line anyway
break
def process_module(self, node: nodes.Module) -> None:
"""Perform the actual check by checking module stream."""
with node.stream() as stream:
codec, codec_line = self._determine_codec(stream)
self._check_codec(codec, codec_line)
stream.seek(0)
# Check for invalid content (controls/chars)
for lineno, line in enumerate(
_fix_utf16_32_line_stream(stream, codec), start=1
):
if lineno == 1:
line = _remove_bom(line, codec)
self._check_bidi_chars(line, lineno, codec)
self._check_invalid_chars(line, lineno, codec)
def register(linter: pylint.lint.PyLinter) -> None:
linter.register_checker(UnicodeChecker(linter))
Zerion Mini Shell 1.0