From b773b8ba12431b7d15f9812e29861de327d0cd8f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= <zbyszek@in.waw.pl>
Date: Wed, 16 Aug 2023 17:24:36 +0200
Subject: [PATCH 6/7] Revert "Add a `validate(...)` function that can be used
to sansitize the expression and variables before starting the virtual
machine."
This reverts commit 21ff376a0853aff6aea63343c6010d6434917319.
---
numexpr/necompiler.py | 194 +++++++++++-------------------------------
1 file changed, 51 insertions(+), 143 deletions(-)
diff --git a/numexpr/necompiler.py b/numexpr/necompiler.py
index f846c1d23d..9291d44244 100644
--- a/numexpr/necompiler.py
+++ b/numexpr/necompiler.py
@@ -8,7 +8,6 @@
# rights to use.
####################################################################
-from typing import Optional, Dict
import __future__
import sys
import numpy
@@ -527,7 +526,7 @@ context_info = [
]
-def getContext(kwargs, _frame_depth=1):
+def getContext(kwargs, frame_depth=1):
d = kwargs.copy()
context = {}
for name, allowed, default in context_info:
@@ -536,11 +535,11 @@ def getContext(kwargs, _frame_depth=1):
context[name] = value
else:
raise ValueError("'%s' must be one of %s" % (name, allowed))
-
+
if d:
raise ValueError("Unknown keyword argument '%s'" % d.popitem()[0])
if context['truediv'] == 'auto':
- caller_globals = sys._getframe(_frame_depth + 1).f_globals
+ caller_globals = sys._getframe(frame_depth + 1).f_globals
context['truediv'] = caller_globals.get('division', None) == __future__.division
return context
@@ -614,11 +613,11 @@ def NumExpr(ex, signature=(), **kwargs):
# NumExpr can be called either directly by the end-user, in which case
# kwargs need to be sanitized by getContext, or by evaluate,
# in which case kwargs are in already sanitized.
- # In that case _frame_depth is wrong (it should be 2) but it doesn't matter
+ # In that case frame_depth is wrong (it should be 2) but it doesn't matter
# since it will not be used (because truediv='auto' has already been
# translated to either True or False).
- _frame_depth = 1
- context = getContext(kwargs, _frame_depth=_frame_depth)
+
+ context = getContext(kwargs, frame_depth=1)
threeAddrProgram, inputsig, tempsig, constants, input_names = precompile(ex, signature, context)
program = compileThreeAddrForm(threeAddrProgram)
return interpreter.NumExpr(inputsig.encode('ascii'),
@@ -718,11 +717,11 @@ def getExprNames(text, context):
return [a.value for a in input_order], ex_uses_vml
-def getArguments(names, local_dict=None, global_dict=None, _frame_depth: int=2):
+def getArguments(names, local_dict=None, global_dict=None):
"""
Get the arguments based on the names.
"""
- call_frame = sys._getframe(_frame_depth)
+ call_frame = sys._getframe(2)
clear_local_dict = False
if local_dict is None:
@@ -761,40 +760,32 @@ _numexpr_last = {}
evaluate_lock = threading.Lock()
-# MAYBE: decorate this function to add attributes instead of having the
-# _numexpr_last dictionary?
-def validate(ex: str,
- local_dict: Optional[Dict] = None,
- global_dict: Optional[Dict] = None,
- out: numpy.ndarray = None,
- order: str = 'K',
- casting: str = 'safe',
- _frame_depth: int = 2,
- **kwargs) -> Optional[Exception]:
+def evaluate(ex, local_dict=None, global_dict=None,
+ out=None, order='K', casting='safe', **kwargs):
"""
- Returns
+ Evaluate a simple array expression element-wise, using the new iterator.
+
+ ex is a string forming an expression, like "2*a+3*b". The values for "a"
+ and "b" will by default be taken from the calling function's frame
+ (through use of sys._getframe()). Alternatively, they can be specifed
+ using the 'local_dict' or 'global_dict' arguments.
Parameters
----------
- ex: str
- a string forming an expression, like "2*a+3*b". The values for "a"
- and "b" will by default be taken from the calling function's frame
- (through use of sys._getframe()). Alternatively, they can be specified
- using the 'local_dict' or 'global_dict' arguments.
- local_dict: dictionary, optional
+ local_dict : dictionary, optional
A dictionary that replaces the local operands in current frame.
- global_dict: dictionary, optional
+ global_dict : dictionary, optional
A dictionary that replaces the global operands in current frame.
- out: NumPy array, optional
+ out : NumPy array, optional
An existing array where the outcome is going to be stored. Care is
required so that this array has the same shape and type than the
actual outcome of the computation. Useful for avoiding unnecessary
new array allocations.
- order: {'C', 'F', 'A', or 'K'}, optional
+ order : {'C', 'F', 'A', or 'K'}, optional
Controls the iteration order for operands. 'C' means C order, 'F'
means Fortran order, 'A' means 'F' order if all the arrays are
Fortran contiguous, 'C' order otherwise, and 'K' means as close to
@@ -802,7 +793,7 @@ def validate(ex: str,
efficient computations, typically 'K'eep order (the default) is
desired.
- casting: {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional
+ casting : {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional
Controls what kind of data casting may occur when making a copy or
buffering. Setting this to 'unsafe' is not recommended, as it can
adversely affect accumulations.
@@ -813,117 +804,37 @@ def validate(ex: str,
* 'same_kind' means only safe casts or casts within a kind,
like float64 to float32, are allowed.
* 'unsafe' means any data conversions may be done.
-
- _frame_depth: int
- The calling frame depth. Unless you are a NumExpr developer you should
- not set this value.
"""
global _numexpr_last
-
- try:
-
- if not isinstance(ex, str):
- raise ValueError("must specify expression as a string")
-
- # Get the names for this expression
- context = getContext(kwargs)
- expr_key = (ex, tuple(sorted(context.items())))
- if expr_key not in _names_cache:
- _names_cache[expr_key] = getExprNames(ex, context)
- names, ex_uses_vml = _names_cache[expr_key]
- arguments = getArguments(names, local_dict, global_dict, _frame_depth=_frame_depth)
-
- # Create a signature
- signature = [(name, getType(arg)) for (name, arg) in
- zip(names, arguments)]
-
- # Look up numexpr if possible.
- numexpr_key = expr_key + (tuple(signature),)
- try:
- compiled_ex = _numexpr_cache[numexpr_key]
- except KeyError:
- compiled_ex = _numexpr_cache[numexpr_key] = NumExpr(ex, signature, **context)
- kwargs = {'out': out, 'order': order, 'casting': casting,
- 'ex_uses_vml': ex_uses_vml}
- _numexpr_last = dict(ex=compiled_ex, argnames=names, kwargs=kwargs)
- # with evaluate_lock:
- # return compiled_ex(*arguments, **kwargs)
- except Exception as e:
- return e
- return None
-
-def evaluate(ex: str,
- local_dict: Optional[Dict] = None,
- global_dict: Optional[Dict] = None,
- out: numpy.ndarray = None,
- order: str = 'K',
- casting: str = 'safe',
- _frame_depth: int=3,
- **kwargs) -> numpy.ndarray:
- """
- Evaluate a simple array expression element-wise using the virtual machine.
-
- Parameters
- ----------
- ex: str
- a string forming an expression, like "2*a+3*b". The values for "a"
- and "b" will by default be taken from the calling function's frame
- (through use of sys._getframe()). Alternatively, they can be specified
- using the 'local_dict' or 'global_dict' arguments.
-
- local_dict: dictionary, optional
- A dictionary that replaces the local operands in current frame.
-
- global_dict: dictionary, optional
- A dictionary that replaces the global operands in current frame.
-
- out: NumPy array, optional
- An existing array where the outcome is going to be stored. Care is
- required so that this array has the same shape and type than the
- actual outcome of the computation. Useful for avoiding unnecessary
- new array allocations.
-
- order: {'C', 'F', 'A', or 'K'}, optional
- Controls the iteration order for operands. 'C' means C order, 'F'
- means Fortran order, 'A' means 'F' order if all the arrays are
- Fortran contiguous, 'C' order otherwise, and 'K' means as close to
- the order the array elements appear in memory as possible. For
- efficient computations, typically 'K'eep order (the default) is
- desired.
-
- casting: {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional
- Controls what kind of data casting may occur when making a copy or
- buffering. Setting this to 'unsafe' is not recommended, as it can
- adversely affect accumulations.
-
- * 'no' means the data types should not be cast at all.
- * 'equiv' means only byte-order changes are allowed.
- * 'safe' means only casts which can preserve values are allowed.
- * 'same_kind' means only safe casts or casts within a kind,
- like float64 to float32, are allowed.
- * 'unsafe' means any data conversions may be done.
-
- _frame_depth: int
- The calling frame depth. Unless you are a NumExpr developer you should
- not set this value.
- """
- # We could avoid code duplication if we called validate and then re_evaluate
- # here, but they we have difficulties with the `sys.getframe(2)` call in
- # `getArguments`
- e = validate(ex, local_dict=local_dict, global_dict=global_dict,
- out=out, order=order, casting=casting,
- _frame_depth=_frame_depth, **kwargs)
- if e is None:
- return re_evaluate(local_dict=local_dict, _frame_depth=_frame_depth)
- else:
- raise e
+ if not isinstance(ex, str):
+ raise ValueError("must specify expression as a string")
+ # Get the names for this expression
+ context = getContext(kwargs, frame_depth=1)
+ expr_key = (ex, tuple(sorted(context.items())))
+ if expr_key not in _names_cache:
+ _names_cache[expr_key] = getExprNames(ex, context)
+ names, ex_uses_vml = _names_cache[expr_key]
+ arguments = getArguments(names, local_dict, global_dict)
-
+ # Create a signature
+ signature = [(name, getType(arg)) for (name, arg) in
+ zip(names, arguments)]
+ # Look up numexpr if possible.
+ numexpr_key = expr_key + (tuple(signature),)
+ try:
+ compiled_ex = _numexpr_cache[numexpr_key]
+ except KeyError:
+ compiled_ex = _numexpr_cache[numexpr_key] = NumExpr(ex, signature, **context)
+ kwargs = {'out': out, 'order': order, 'casting': casting,
+ 'ex_uses_vml': ex_uses_vml}
+ _numexpr_last = dict(ex=compiled_ex, argnames=names, kwargs=kwargs)
+ with evaluate_lock:
+ return compiled_ex(*arguments, **kwargs)
-def re_evaluate(local_dict: Optional[Dict] = None,
- _frame_depth: int=2) -> numpy.ndarray:
+
+def re_evaluate(local_dict=None):
"""
Re-evaluate the previous executed array expression without any check.
@@ -933,20 +844,17 @@ def re_evaluate(local_dict: Optional[Dict] = None,
Parameters
----------
- local_dict: dictionary, optional
+
+ local_dict : dictionary, optional
A dictionary that replaces the local operands in current frame.
- _frame_depth: int
- The calling frame depth. Unless you are a NumExpr developer you should
- not set this value.
+
"""
- global _numexpr_last
-
try:
compiled_ex = _numexpr_last['ex']
except KeyError:
- raise RuntimeError("A previous evaluate() execution was not found, please call `validate` or `evaluate` once before `re_evaluate`")
+ raise RuntimeError("not a previous evaluate() execution found")
argnames = _numexpr_last['argnames']
- args = getArguments(argnames, local_dict, _frame_depth=_frame_depth)
+ args = getArguments(argnames, local_dict)
kwargs = _numexpr_last['kwargs']
with evaluate_lock:
return compiled_ex(*args, **kwargs)