From 20fd16aff18188002c9efdd7f885fc2752df7b43 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Fri, 3 Apr 2026 12:52:08 -0700 Subject: [PATCH] Add signature generation functions for types and function overloads PiperOrigin-RevId: 894210091 --- MODULE.bazel | 2 +- README.md | 112 +++++-- cel_expr_python/BUILD | 26 +- cel_expr_python/cel_env_test.py | 274 ++++++++++++++++-- cel_expr_python/cel_extension.h | 30 +- cel_expr_python/ext/BUILD | 29 +- cel_expr_python/ext/ext_bindings.cc | 12 +- cel_expr_python/ext/ext_encoders.cc | 11 +- cel_expr_python/ext/ext_math.cc | 28 +- cel_expr_python/ext/ext_optional.cc | 22 +- cel_expr_python/ext/ext_proto.cc | 9 +- .../ext/{ext_string.cc => ext_strings.cc} | 29 +- cel_expr_python/py_cel_env.cc | 34 ++- cel_expr_python/py_cel_env.h | 8 +- cel_expr_python/py_cel_env_internal.cc | 182 ++++++++++-- cel_expr_python/py_cel_env_internal.h | 21 +- cel_expr_python/py_cel_function_decl.cc | 12 + cel_expr_python/py_cel_function_decl.h | 3 + cel_expr_python/py_cel_overload.cc | 20 +- cel_expr_python/py_cel_overload.h | 3 + cel_expr_python/py_cel_python_extension.cc | 33 ++- cel_expr_python/py_cel_python_extension.h | 5 +- conformance/BUILD | 2 +- conformance/conformance_test.py | 4 +- custom_ext/BUILD | 1 - custom_ext/sample_cel_ext.cc | 38 +-- 26 files changed, 731 insertions(+), 219 deletions(-) rename cel_expr_python/ext/{ext_string.cc => ext_strings.cc} (54%) diff --git a/MODULE.bazel b/MODULE.bazel index 51589db..b6c2057 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -14,7 +14,7 @@ bazel_dep(name = "bazel_skylib", version = "1.8.2") # https://registry.bazel.build/modules/cel-cpp bazel_dep(name = "cel-cpp", version = "0.14.0", repo_name = "com_google_cel_cpp") -# 03/30/2026 +# 04/01/2026 _CEL_CPP_COMMIT = "5a3463337cf2a9b90b53833af2bbc1f35da90d64" _CEL_CPP_SHA256 = "299be398d1495340eb92da31f9a0667e1351479752e8a567ac31c385e4aea73c" diff --git a/README.md b/README.md index 813c472..eed8a1d 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,76 @@ Resulting message type: Resulting message value: 123 ``` +### Custom functions + +When configuring `cel.Env` you can supply custom functions. For each function +there needs to be a declaration and an implementation. The implementation, which +is a Python function, can be provided either as part of the declaration +or separately. + +Let's say we want to be able to invoke this function from CEL expressions: +```python + def good_time_of_day(ampm, arg): + if ampm == 'am': + time_of_day = 'morning' + else: + time_of_day = 'afternoon' + return f"Good {time_of_day}, {arg}" +``` + +The implementation can be supplied along with the declaration: +```python + cel_env = cel.NewEnv(functions=[ + cel.FunctionDecl( + "hello", + [ + cel.Overload( + "hello(string,string)", + return_type=cel.Type.STRING, + parameters=[ + cel.Type.STRING, + cel.Type.STRING, + ], + impl=good_time_of_day, + ) + ], + ) + ]) +``` + +It can also be provided separately in a dictionary that maps overload IDs to +their respective implementations: + +```python + cel_env = cel.NewEnv(functions=[ + cel.FunctionDecl( + "hello", + [ + cel.Overload( + "hello(string,string)", + return_type=cel.Type.STRING, + parameters=[ + cel.Type.STRING, + cel.Type.STRING, + ], + ) + ], + ) + ], + function_impls={ + "hello(string,string)": good_time_of_day, + }) +``` + +Now that the function implementation is bound to the CEL environment, +we can invoke it from CEL like this: +```python + result = env.compile("hello('am', 'breakfast is ready!')").eval() + print(result.value()) # Good morning, breakfast is ready! + result = env.compile("hello('pm', 'tea is served.')").eval() + print(result.value()) # Good afternoon, tea is served. +``` + ### Extensions #### Standard extensions @@ -234,36 +304,34 @@ expr = cel_env.compile("my_func(1)") To define a custom extension in C++, define a class extending `cel_python::CelExtension`. There are two methods you will need to implement: -`ConfigureCompiler` and `ConfigureRuntime`. The implementations of these methods -use the same API as extensions written for the C++ CEL runtime. In fact, +`GetCompilerLibrary` and `ConfigureRuntime`. The implementations of these +methods use the same API as extensions written for the C++ CEL runtime. In fact, extensions written for the C++ runtime can be used unchanged with cel-expr-python - you would just need to write a trivial wrapper class invoking the registration functions defined by the C++ extension. -```cpp - absl::Status ConfigureCompiler( - cel::CompilerBuilder& compiler_builder, - const proto2::DescriptorPool& descriptor_pool); -``` This method adds extension function definitions to the provided `CompilerBuilder`, for example: ```cpp -absl::Status ConfigureCompiler( - cel::CompilerBuilder& compiler_builder, - const proto2::DescriptorPool& descriptor_pool) { - CEL_PYTHON_ASSIGN_OR_RETURN( - auto func_translate, - cel::MakeFunctionDecl("translate", - cel::MakeMemberOverloadDecl("translate_inst", - /*return_type=*/cel::StringType(), - /*target=*/cel::StringType(), - /*from_lang=*/cel::StringType(), - /*to_lang=*/cel::StringType()))); - CEL_PYTHON_RETURN_IF_ERROR( - compiler_builder.GetCheckerBuilder().AddFunction(func_translate)); - return absl::OkStatus(); -} + cel::CompilerLibrary GetCompilerLibrary() { + return cel::CompilerLibrary( + "translate-ext", + [](cel::TypeCheckerBuilder& checker_builder) -> absl::Status { + CEL_PYTHON_ASSIGN_OR_RETURN( + auto func_translate, + cel::MakeFunctionDecl( + "translate", + cel::MakeMemberOverloadDecl("translate_inst", + /*return_type=*/cel::StringType(), + /*target=*/cel::StringType(), + /*from_lang=*/cel::StringType(), + /*to_lang=*/cel::StringType()))); + CEL_PYTHON_RETURN_IF_ERROR( + checker_builder.AddFunction(func_translate)); + return absl::OkStatus(); + }); + } ``` The other method registers the actual implementation diff --git a/cel_expr_python/BUILD b/cel_expr_python/BUILD index 35624d6..2abc600 100644 --- a/cel_expr_python/BUILD +++ b/cel_expr_python/BUILD @@ -63,6 +63,7 @@ pybind_extension( "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", "@com_google_cel_cpp//checker:type_checker_builder", + "@com_google_cel_cpp//checker:type_checker_builder_factory", "@com_google_cel_cpp//checker:validation_result", "@com_google_cel_cpp//common:ast", "@com_google_cel_cpp//common:ast_proto", @@ -78,8 +79,13 @@ pybind_extension( "@com_google_cel_cpp//compiler", "@com_google_cel_cpp//env", "@com_google_cel_cpp//env:config", + "@com_google_cel_cpp//env:env_runtime", + "@com_google_cel_cpp//env:env_std_extensions", "@com_google_cel_cpp//env:env_yaml", + "@com_google_cel_cpp//env:runtime_std_extensions", "@com_google_cel_cpp//extensions/protobuf:runtime_adapter", + "@com_google_cel_cpp//parser", + "@com_google_cel_cpp//parser:options", "@com_google_cel_cpp//parser:parser_interface", "@com_google_cel_cpp//runtime", "@com_google_cel_cpp//runtime:activation", @@ -88,7 +94,6 @@ pybind_extension( "@com_google_cel_cpp//runtime:reference_resolver", "@com_google_cel_cpp//runtime:runtime_builder", "@com_google_cel_cpp//runtime:runtime_options", - "@com_google_cel_cpp//runtime:standard_runtime_builder_factory", "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_protobuf//:protobuf", @@ -99,14 +104,25 @@ pybind_extension( # For pybind11-based CEL extensions. pybind_library( name = "cel_extension", - hdrs = ["cel_extension.h"], + srcs = [ + "py_error_status.cc", + ], + hdrs = [ + "cel_extension.h", + "py_error_status.h", + ], visibility = ["//visibility:public"], deps = [ + ":status_macros", + "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", "@com_google_cel_cpp//compiler", "@com_google_cel_cpp//runtime:runtime_builder", "@com_google_cel_cpp//runtime:runtime_options", - "@com_google_protobuf//:protobuf", ], ) @@ -135,6 +151,10 @@ py_test( srcs = ["cel_env_test.py"], deps = [ ":cel", + "//cel_expr_python/ext:ext_bindings", + "//cel_expr_python/ext:ext_math", + "//cel_expr_python/ext:ext_optional", + "//cel_expr_python/ext:ext_strings", "//testing:proto2_test_all_types_py_pb2", "@com_google_absl_py//absl/testing:absltest", ], diff --git a/cel_expr_python/cel_env_test.py b/cel_expr_python/cel_env_test.py index 5b49875..c7b1f00 100644 --- a/cel_expr_python/cel_env_test.py +++ b/cel_expr_python/cel_env_test.py @@ -18,8 +18,14 @@ ability to be created from and serialized to YAML format. """ +import textwrap + from absl.testing import absltest from cel_expr_python import cel +from cel_expr_python.ext import ext_bindings +from cel_expr_python.ext import ext_math +from cel_expr_python.ext import ext_optional +from cel_expr_python.ext import ext_strings from cel.expr.conformance.proto2 import test_all_types_pb2 as test_all_types_pb @@ -91,6 +97,16 @@ def test_invalid_yaml(self): str(e.exception), ) + def test_config_export_container(self): + env = cel.NewEnv(container="test.container") + yaml = env.config().to_yaml() + self.assertEqual( + normalize_yaml(yaml), + normalize_yaml(""" + container: "test.container" + """), + ) + def test_config_export_variables(self): config = cel.NewEnv( variables={ @@ -236,27 +252,247 @@ def test_config_variable_types(self): self.assertEqual(res.type(), cel.Type.INT) self.assertEqual(res.value(), 42) + def test_config_export_extension_version(self): + env = cel.NewEnv( + extensions=[ + ext_math.ExtMath(0), + ext_optional.ExtOptional(1), + ext_strings.ExtStrings(2), + ext_bindings.ExtBindings(), + ], + ) + yaml = env.config().to_yaml() + self.assertEqual( + normalize_yaml(yaml), + normalize_yaml(""" + extensions: + - name: "bindings" + - name: "math" + version: 0 + - name: "optional" + version: 1 + - name: "strings" + version: 2 + """), + ) + + def test_config_extension_version_out_of_range(self): + cases = [ + [ + lambda: ext_math.ExtMath(42), + r"'math' extension version: 42 not in range \[0, \d+\]", + ], + [ + lambda: ext_optional.ExtOptional(6), + r"'optional' extension version: 6 not in range \[0, \d+\]", + ], + [ + lambda: ext_strings.ExtStrings(18), + r"'strings' extension version: 18 not in range \[0, \d+\]", + ], + ] + for test_case in cases: + with self.assertRaises(Exception) as e: + cel.NewEnv( + extensions=[test_case[0]()], + ) + self.assertRegex(str(e.exception), test_case[1]) + + def test_config_extensions(self): + config = cel.NewEnvConfigFromYaml(""" + extensions: + - name: math + - name: strings + """) + env = cel.NewEnv( + config=config, + extensions=[TestCelExtension()], + ) + yaml = env.config().to_yaml() + self.assertEqual( + normalize_yaml(yaml), + normalize_yaml(""" + extensions: + - name: "math" + - name: "strings" + - name: "test_cel_extension" + """), + ) + res = env.compile("'%.4f'.format([math.sqrt(2)])").eval() + self.assertEqual(res.value(), "1.4142") + res = env.compile("hello('World')").eval() + self.assertEqual(res.value(), "Hello, World!") + + def test_config_extension_override_same_version(self): + config = cel.NewEnvConfigFromYaml(""" + extensions: + - name: cel.lib.ext.math + version: 1 + - name: strings + version: 2 + """) + env = cel.NewEnv( + config=config, + extensions=[ext_math.ExtMath(1), ext_strings.ExtStrings(2)], + ) + res = env.compile("'%.3f'.format([math.floor(3.14)])").eval() + self.assertEqual(res.value(), "3.000") + + def test_config_extension_override_different_version(self): + config = cel.NewEnvConfigFromYaml(""" + extensions: + - name: math + version: 0 + - name: cel.lib.ext.strings + version: 2 + """) + with self.assertRaises(Exception) as e: + cel.NewEnv( + config=config, + extensions=[ext_math.ExtMath()], + ) + self.assertIn( + "Extension 'math' version 0 is already included. Cannot" + " also include version 2", + str(e.exception), + ) + with self.assertRaises(Exception) as e: + cel.NewEnv( + config=config, + extensions=[ext_strings.ExtStrings(1)], + ) + self.assertIn( + "Extension 'cel.lib.ext.strings' version 2 is already included. Cannot" + " also include version 1", + str(e.exception), + ) + + def test_config_functions(self): + config = cel.NewEnvConfigFromYaml(""" + functions: + - name: is_ok + overloads: + - id: "is_ok_string" + target: + type_name: string + return: + type_name: bool + """) + env = cel.NewEnv( + config=config, + functions=[ + cel.FunctionDecl( + "hello", + [ + cel.Overload( + "good_time_of_day", + return_type=cel.Type.STRING, + parameters=[ + cel.Type.STRING, + cel.Type.STRING, + ], + impl=lambda ampm, arg: ( + "Good" + f" {'morning' if ampm == 'am' else 'afternoon'}," + f" {arg}!" + ), + ) + ], + ) + ], + function_impls={ + "is_ok_string": lambda arg: arg in ["excellent", "good", "fair"], + }, + ) + yaml = env.config().to_yaml() + self.assertEqual( + normalize_yaml(yaml), + normalize_yaml(""" + functions: + - name: "hello" + overloads: + - id: "good_time_of_day" + args: + - type_name: "string" + - type_name: "string" + return: + type_name: "string" + - name: "is_ok" + overloads: + - id: "is_ok_string" + target: + type_name: "string" + return: + type_name: "bool" + """), + ) + res = env.compile("hello('am', 'Sunshine')").eval() + self.assertEqual(res.value(), "Good morning, Sunshine!") + res = env.compile("hello('pm', 'tea is served')").eval() + self.assertEqual(res.value(), "Good afternoon, tea is served!") + res = env.compile("'good'.is_ok()").eval() + self.assertTrue(res.value()) + res = env.compile("'bad'.is_ok()").eval() + self.assertFalse(res.value()) + + def test_config_function_override(self): + config = cel.NewEnvConfigFromYaml(""" + functions: + - name: foo + overloads: + - id: "unique_id" + """) + with self.assertRaises(Exception) as e: + cel.NewEnv( + config=config, + functions=[ + cel.FunctionDecl( + "bar", + [ + cel.Overload( + "unique_id", + impl=lambda: "hello", + ) + ], + ) + ], + function_impls={ + "unique_id": lambda: "goodbye", + }, + ) + self.assertIn( + "An implementation for function overload id 'unique_id' already" + " exists.", + str(e.exception), + ) + + +class TestCelExtension(cel.CelExtension): + """An example CEL extension for testing.""" + + def __init__(self): + super().__init__( + "test_cel_extension", + functions=[ + cel.FunctionDecl( + "hello", + [ + cel.Overload( + "hello(string)", + return_type=cel.Type.STRING, + parameters=[ + cel.Type.STRING, + ], + impl=lambda arg: f"Hello, {arg}!", + ) + ], + ), + ], + ) + def normalize_yaml(yaml: str) -> str: - lines = yaml.split("\n") - indent = -1 - unindented_lines = [] - for line in lines: - pos = -1 - for i, char in enumerate(line): - if char != " " and char != "\t": - pos = i - break - if pos == -1: - # Skip blank lines. - continue - if indent == -1: - indent = pos - if pos >= indent: - unindented_lines.append(line[indent:]) - else: - unindented_lines.append(line) - return "\n".join(unindented_lines) + return textwrap.dedent(yaml).strip() if __name__ == "__main__": diff --git a/cel_expr_python/cel_extension.h b/cel_expr_python/cel_extension.h index 96a1d88..48a7151 100644 --- a/cel_expr_python/cel_extension.h +++ b/cel_expr_python/cel_extension.h @@ -24,7 +24,6 @@ #include "compiler/compiler.h" #include "runtime/runtime_builder.h" #include "runtime/runtime_options.h" -#include "google/protobuf/descriptor.h" #if defined(__cpp_exceptions) || defined(__EXCEPTIONS) // This include causes many issues if included in C++ environments that don't @@ -39,13 +38,13 @@ namespace cel_python { // Python. class CelExtension { public: - explicit CelExtension(std::string name) : name_(std::move(name)) {}; + explicit CelExtension(std::string name, std::string alias = "", + int version = -1) + : name_(std::move(name)), alias_(std::move(alias)), version_(version) {} virtual ~CelExtension() = default; - virtual absl::Status ConfigureCompiler( - cel::CompilerBuilder& compiler_builder, - const google::protobuf::DescriptorPool& descriptor_pool) { - return absl::OkStatus(); + virtual cel::CompilerLibrary GetCompilerLibrary() { + return cel::CompilerLibrary(name_, nullptr, nullptr); } virtual absl::Status ConfigureRuntime(cel::RuntimeBuilder& runtime_builder, @@ -54,9 +53,13 @@ class CelExtension { } std::string name() const { return name_; } + std::string alias() const { return alias_; } + int version() const { return version_; } private: std::string name_; + std::string alias_; + int version_; }; #define CEL_MODULE_NAME "cel_expr_python.cel" @@ -76,13 +79,20 @@ class CelExtension { // // CEL_EXTENSION_MODULE(sample_cel_ext, SampleCelExtension); // -#define CEL_EXTENSION_MODULE(module_name, class_name) \ - PYBIND11_MODULE(module_name, m) { \ - pybind11::module_::import(CEL_MODULE_NAME); \ +#define CEL_EXTENSION_MODULE(module_name, class_name) \ + PYBIND11_MODULE(module_name, m) { \ + pybind11::module_::import(CEL_MODULE_NAME); \ pybind11::class_(m, #class_name) \ - .def(pybind11::init<>()); \ + .def(pybind11::init<>()); \ } +#define CEL_VERSIONED_EXTENSION_MODULE(module_name, class_name) \ + PYBIND11_MODULE(module_name, m) { \ + pybind11::module_::import(CEL_MODULE_NAME); \ + pybind11::class_(m, #class_name) \ + .def(pybind11::init<>()) \ + .def(pybind11::init(), pybind11::arg("version")); \ + } } // namespace cel_python #endif // THIRD_PARTY_CEL_PYTHON_CEL_EXTENSION_H_ diff --git a/cel_expr_python/ext/BUILD b/cel_expr_python/ext/BUILD index cfd3f14..cdb038e 100644 --- a/cel_expr_python/ext/BUILD +++ b/cel_expr_python/ext/BUILD @@ -13,10 +13,8 @@ pybind_extension( visibility = ["//visibility:public"], deps = [ "//cel_expr_python:cel_extension", - "@com_google_absl//absl/status", "@com_google_cel_cpp//compiler", "@com_google_cel_cpp//extensions:bindings_ext", - "@com_google_protobuf//:protobuf", ], ) @@ -32,12 +30,10 @@ pybind_extension( deps = [ "//cel_expr_python:cel_extension", "@com_google_absl//absl/status", - "@com_google_cel_cpp//checker:type_checker_builder", "@com_google_cel_cpp//compiler", "@com_google_cel_cpp//extensions:encoders", "@com_google_cel_cpp//runtime:runtime_builder", "@com_google_cel_cpp//runtime:runtime_options", - "@com_google_protobuf//:protobuf", ], ) @@ -52,25 +48,13 @@ pybind_extension( visibility = ["//visibility:public"], deps = [ "//cel_expr_python:cel_extension", - "//cel_expr_python:status_macros", - "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/types:span", - "@com_google_cel_cpp//checker:type_checker_builder", - "@com_google_cel_cpp//common:decl", - "@com_google_cel_cpp//common:function_descriptor", - "@com_google_cel_cpp//common:kind", - "@com_google_cel_cpp//common:type", - "@com_google_cel_cpp//common:value", + "@com_google_absl//absl/strings", "@com_google_cel_cpp//compiler", "@com_google_cel_cpp//extensions:math_ext", "@com_google_cel_cpp//extensions:math_ext_decls", - "@com_google_cel_cpp//runtime:function", "@com_google_cel_cpp//runtime:runtime_builder", "@com_google_cel_cpp//runtime:runtime_options", - "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", - "@com_google_protobuf//:protobuf", ], ) @@ -86,12 +70,13 @@ pybind_extension( deps = [ "//cel_expr_python:cel_extension", "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_cel_cpp//checker:optional", "@com_google_cel_cpp//compiler", "@com_google_cel_cpp//compiler:optional", "@com_google_cel_cpp//runtime:optional_types", "@com_google_cel_cpp//runtime:runtime_builder", "@com_google_cel_cpp//runtime:runtime_options", - "@com_google_protobuf//:protobuf", ], ) @@ -106,17 +91,15 @@ pybind_extension( visibility = ["//visibility:public"], deps = [ "//cel_expr_python:cel_extension", - "@com_google_absl//absl/status", "@com_google_cel_cpp//compiler", "@com_google_cel_cpp//extensions:proto_ext", - "@com_google_protobuf//:protobuf", ], ) pybind_extension( - name = "ext_string", + name = "ext_strings", srcs = [ - "ext_string.cc", + "ext_strings.cc", ], data = [ "//cel_expr_python:cel", @@ -125,10 +108,10 @@ pybind_extension( deps = [ "//cel_expr_python:cel_extension", "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", "@com_google_cel_cpp//compiler", "@com_google_cel_cpp//extensions:strings", "@com_google_cel_cpp//runtime:runtime_builder", "@com_google_cel_cpp//runtime:runtime_options", - "@com_google_protobuf//:protobuf", ], ) diff --git a/cel_expr_python/ext/ext_bindings.cc b/cel_expr_python/ext/ext_bindings.cc index 1441782..49d9731 100644 --- a/cel_expr_python/ext/ext_bindings.cc +++ b/cel_expr_python/ext/ext_bindings.cc @@ -12,23 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "absl/status/status.h" #include "compiler/compiler.h" #include "extensions/bindings_ext.h" #include "cel_expr_python/cel_extension.h" -#include "google/protobuf/descriptor.h" namespace cel_python { class ExtBindings : public CelExtension { public: - explicit ExtBindings() : CelExtension("cel.lib.ext.cel.bindings") {} + explicit ExtBindings() + : CelExtension("cel.lib.ext.cel.bindings", "bindings") {} - absl::Status ConfigureCompiler( - cel::CompilerBuilder& compiler_builder, - const google::protobuf::DescriptorPool& descriptor_pool) override { - return compiler_builder.AddLibrary( - cel::extensions::BindingsCompilerLibrary()); + cel::CompilerLibrary GetCompilerLibrary() override { + return cel::extensions::BindingsCompilerLibrary(); } }; diff --git a/cel_expr_python/ext/ext_encoders.cc b/cel_expr_python/ext/ext_encoders.cc index 6981bf6..59c67ae 100644 --- a/cel_expr_python/ext/ext_encoders.cc +++ b/cel_expr_python/ext/ext_encoders.cc @@ -13,25 +13,20 @@ // limitations under the License. #include "absl/status/status.h" -#include "checker/type_checker_builder.h" #include "compiler/compiler.h" #include "extensions/encoders.h" #include "runtime/runtime_builder.h" #include "runtime/runtime_options.h" #include "cel_expr_python/cel_extension.h" -#include "google/protobuf/descriptor.h" namespace cel_python { class ExtEncoders : public CelExtension { public: - explicit ExtEncoders() : CelExtension("cel.lib.ext.encoders") {} + explicit ExtEncoders() : CelExtension("cel.lib.ext.encoders", "encoders") {} - absl::Status ConfigureCompiler( - cel::CompilerBuilder& compiler_builder, - const google::protobuf::DescriptorPool& descriptor_pool) override { - return compiler_builder.GetCheckerBuilder().AddLibrary( - cel::extensions::EncodersCheckerLibrary()); + cel::CompilerLibrary GetCompilerLibrary() override { + return cel::extensions::EncodersCompilerLibrary(); } absl::Status ConfigureRuntime(cel::RuntimeBuilder& runtime_builder, diff --git a/cel_expr_python/ext/ext_math.cc b/cel_expr_python/ext/ext_math.cc index 1519d8f..4ed18ec 100644 --- a/cel_expr_python/ext/ext_math.cc +++ b/cel_expr_python/ext/ext_math.cc @@ -13,35 +13,41 @@ // limitations under the License. #include "absl/status/status.h" +#include "absl/strings/str_cat.h" #include "compiler/compiler.h" #include "extensions/math_ext.h" #include "extensions/math_ext_decls.h" #include "runtime/runtime_builder.h" #include "runtime/runtime_options.h" #include "cel_expr_python/cel_extension.h" -#include "cel_expr_python/status_macros.h" -#include "google/protobuf/descriptor.h" +#include "cel_expr_python/py_error_status.h" namespace cel_python { class ExtMath : public CelExtension { public: - explicit ExtMath() : CelExtension("cel.lib.ext.math") {} + explicit ExtMath(int version) + : CelExtension("cel.lib.ext.math", "math", version) { + if (version < 0 || version > cel::extensions::kMathExtensionLatestVersion) { + throw StatusToException(absl::InvalidArgumentError(absl::StrCat( + "'math' extension version: ", version, " not in range [0, ", + cel::extensions::kMathExtensionLatestVersion, "]"))); + } + } + + ExtMath() : ExtMath(cel::extensions::kMathExtensionLatestVersion) {} - absl::Status ConfigureCompiler( - cel::CompilerBuilder& compiler_builder, - const google::protobuf::DescriptorPool& descriptor_pool) override { - return compiler_builder.AddLibrary(cel::extensions::MathCompilerLibrary()); + cel::CompilerLibrary GetCompilerLibrary() override { + return cel::extensions::MathCompilerLibrary(version()); } absl::Status ConfigureRuntime(cel::RuntimeBuilder& runtime_builder, const cel::RuntimeOptions& opts) override { - CEL_PYTHON_RETURN_IF_ERROR(cel::extensions::RegisterMathExtensionFunctions( - runtime_builder.function_registry(), opts)); - return absl::OkStatus(); + return cel::extensions::RegisterMathExtensionFunctions( + runtime_builder.function_registry(), opts, version()); } }; -CEL_EXTENSION_MODULE(ext_math, ExtMath); +CEL_VERSIONED_EXTENSION_MODULE(ext_math, ExtMath); } // namespace cel_python diff --git a/cel_expr_python/ext/ext_optional.cc b/cel_expr_python/ext/ext_optional.cc index 80a01e2..a54242d 100644 --- a/cel_expr_python/ext/ext_optional.cc +++ b/cel_expr_python/ext/ext_optional.cc @@ -13,24 +13,32 @@ // limitations under the License. #include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "checker/optional.h" #include "compiler/compiler.h" #include "compiler/optional.h" #include "runtime/optional_types.h" #include "runtime/runtime_builder.h" #include "runtime/runtime_options.h" #include "cel_expr_python/cel_extension.h" -#include "google/protobuf/descriptor.h" +#include "cel_expr_python/py_error_status.h" namespace cel_python { class ExtOptional : public CelExtension { public: - explicit ExtOptional() : CelExtension("cel.lib.optional") {} + explicit ExtOptional(int version) : CelExtension("optional", "", version) { + if (version < 0 || version > cel::kOptionalExtensionLatestVersion) { + throw StatusToException(absl::InvalidArgumentError(absl::StrCat( + "'optional' extension version: ", version, " not in range [0, ", + cel::kOptionalExtensionLatestVersion, "]"))); + } + } + + ExtOptional() : ExtOptional(cel::kOptionalExtensionLatestVersion) {} - absl::Status ConfigureCompiler( - cel::CompilerBuilder& compiler_builder, - const google::protobuf::DescriptorPool& descriptor_pool) override { - return compiler_builder.AddLibrary(cel::OptionalCompilerLibrary()); + cel::CompilerLibrary GetCompilerLibrary() override { + return cel::OptionalCompilerLibrary(version()); } absl::Status ConfigureRuntime(cel::RuntimeBuilder& runtime_builder, @@ -43,6 +51,6 @@ class ExtOptional : public CelExtension { } }; -CEL_EXTENSION_MODULE(ext_optional, ExtOptional); +CEL_VERSIONED_EXTENSION_MODULE(ext_optional, ExtOptional); } // namespace cel_python diff --git a/cel_expr_python/ext/ext_proto.cc b/cel_expr_python/ext/ext_proto.cc index edd1675..a63cbec 100644 --- a/cel_expr_python/ext/ext_proto.cc +++ b/cel_expr_python/ext/ext_proto.cc @@ -12,11 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "absl/status/status.h" #include "compiler/compiler.h" #include "extensions/proto_ext.h" #include "cel_expr_python/cel_extension.h" -#include "google/protobuf/descriptor.h" namespace cel_python { @@ -24,11 +22,8 @@ class ExtProto : public CelExtension { public: explicit ExtProto() : CelExtension("cel.lib.ext.proto") {} - absl::Status ConfigureCompiler( - cel::CompilerBuilder& compiler_builder, - const google::protobuf::DescriptorPool& descriptor_pool) override { - return compiler_builder.AddLibrary( - cel::extensions::ProtoExtCompilerLibrary()); + cel::CompilerLibrary GetCompilerLibrary() override { + return cel::extensions::ProtoExtCompilerLibrary(); } }; diff --git a/cel_expr_python/ext/ext_string.cc b/cel_expr_python/ext/ext_strings.cc similarity index 54% rename from cel_expr_python/ext/ext_string.cc rename to cel_expr_python/ext/ext_strings.cc index 49dd780..d1e6456 100644 --- a/cel_expr_python/ext/ext_string.cc +++ b/cel_expr_python/ext/ext_strings.cc @@ -13,33 +13,42 @@ // limitations under the License. #include "absl/status/status.h" +#include "absl/strings/str_cat.h" #include "compiler/compiler.h" #include "extensions/strings.h" #include "runtime/runtime_builder.h" #include "runtime/runtime_options.h" #include "cel_expr_python/cel_extension.h" -#include "google/protobuf/descriptor.h" +#include "cel_expr_python/py_error_status.h" namespace cel_python { -class ExtString : public CelExtension { +class ExtStrings : public CelExtension { public: - explicit ExtString() : CelExtension("cel.lib.ext.string") {} + explicit ExtStrings(int version) + : CelExtension("cel.lib.ext.strings", "strings", version) { + if (version < 0 || + version > cel::extensions::kStringsExtensionLatestVersion) { + throw StatusToException(absl::InvalidArgumentError(absl::StrCat( + "'strings' extension version: ", version, " not in range [0, ", + cel::extensions::kStringsExtensionLatestVersion, "]"))); + } + } + + ExtStrings() : ExtStrings(cel::extensions::kStringsExtensionLatestVersion) {} - absl::Status ConfigureCompiler( - cel::CompilerBuilder& compiler_builder, - const google::protobuf::DescriptorPool& descriptor_pool) override { - return compiler_builder.AddLibrary( - cel::extensions::StringsCompilerLibrary()); + cel::CompilerLibrary GetCompilerLibrary() override { + return cel::extensions::StringsCompilerLibrary(version()); } absl::Status ConfigureRuntime(cel::RuntimeBuilder& runtime_builder, const cel::RuntimeOptions& opts) override { return cel::extensions::RegisterStringsFunctions( - runtime_builder.function_registry(), opts); + runtime_builder.function_registry(), opts, + cel::extensions::StringsExtensionOptions{.version = version()}); } }; -CEL_EXTENSION_MODULE(ext_string, ExtString); +CEL_VERSIONED_EXTENSION_MODULE(ext_strings, ExtStrings); } // namespace cel_python diff --git a/cel_expr_python/py_cel_env.cc b/cel_expr_python/py_cel_env.cc index d201ae6..dcfa77d 100644 --- a/cel_expr_python/py_cel_env.cc +++ b/cel_expr_python/py_cel_env.cc @@ -31,6 +31,7 @@ #include "cel_expr_python/py_cel_env_config.h" #include "cel_expr_python/py_cel_env_internal.h" #include "cel_expr_python/py_cel_expression.h" +#include "cel_expr_python/py_cel_function_decl.h" #include "cel_expr_python/py_cel_type.h" #include "cel_expr_python/py_error_status.h" #include @@ -44,10 +45,14 @@ void PyCelEnv::DefinePythonBindings(pybind11::module& m) { py::class_> cel_class(m, "Env"); m.def( "NewEnv", - [](py::object descriptor_pool, std::optional config, - std::optional> variables, - std::optional> extensions, - const std::optional& container) { + [](py::object descriptor_pool, std::optional& config, + std::optional>& variables, + std::optional>& extensions, + const std::optional& container, + std::optional>>& + functions, + std::optional>& + function_impls) { PyObject* pool_ptr; if (descriptor_pool.is_none()) { // Replicates python's `descriptor_pool.Default()` @@ -75,11 +80,16 @@ void PyCelEnv::DefinePythonBindings(pybind11::module& m) { return PyCelEnv(config.value_or(PyCelEnvConfig()), pool_ptr, std::move(variables).value_or( std::unordered_map{}), - ext_ptrs, container.value_or("")); + ext_ptrs, container.value_or(""), + functions.value_or( + std::vector>{}), + function_impls.value_or( + std::unordered_map{})); }, py::arg("descriptor_pool") = py::none(), py::arg("config") = py::none(), py::arg("variables") = py::none(), py::arg("extensions") = py::none(), - py::arg("container") = py::none()); + py::arg("container") = py::none(), py::arg("functions") = py::none(), + py::arg("function_impls") = py::none()); cel_class .def("config", [](PyCelEnv& self) { return self.GetEnv()->GetEnvConfig(); }) @@ -112,13 +122,15 @@ void PyCelEnv::DefinePythonBindings(pybind11::module& m) { py::arg("arena") = nullptr); } -PyCelEnv::PyCelEnv(const PyCelEnvConfig& config, PyObject* descriptor_pool, - std::unordered_map variable_types, - const std::vector& extensions, - std::string container) { +PyCelEnv::PyCelEnv( + const PyCelEnvConfig& config, PyObject* descriptor_pool, + const std::unordered_map& variable_types, + const std::vector& extensions, const std::string& container, + const std::vector>& functions, + const std::unordered_map& function_impls) { env_ = ThrowIfError(PyCelEnvInternal::NewCelEnvInternal( config, descriptor_pool, std::move(variable_types), extensions, - std::move(container))); + std::move(container), std::move(functions), std::move(function_impls))); ABSL_CHECK(PyGILState_Check()); } diff --git a/cel_expr_python/py_cel_env.h b/cel_expr_python/py_cel_env.h index 328d0db..37c7cac 100644 --- a/cel_expr_python/py_cel_env.h +++ b/cel_expr_python/py_cel_env.h @@ -28,6 +28,7 @@ #include "cel_expr_python/py_cel_env_config.h" #include "cel_expr_python/py_cel_expression.h" #include "cel_expr_python/py_cel_function.h" +#include "cel_expr_python/py_cel_function_decl.h" #include "cel_expr_python/py_cel_type.h" #include @@ -67,8 +68,11 @@ class PyCelEnv { private: // Private constructor. Use `py_cel.NewEnv()` in python to obtain an instance. PyCelEnv(const PyCelEnvConfig& config, PyObject* descriptor_pool, - std::unordered_map variable_types, - const std::vector& extensions, std::string container); + const std::unordered_map& variable_types, + const std::vector& extensions, + const std::string& container, + const std::vector>& functions, + const std::unordered_map& function_impls); std::shared_ptr env_; }; diff --git a/cel_expr_python/py_cel_env_internal.cc b/cel_expr_python/py_cel_env_internal.cc index c92820b..4bc46e4 100644 --- a/cel_expr_python/py_cel_env_internal.cc +++ b/cel_expr_python/py_cel_env_internal.cc @@ -25,30 +25,47 @@ #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "checker/type_checker_builder.h" +#include "common/function_descriptor.h" +#include "common/kind.h" +#include "common/type.h" #include "compiler/compiler.h" #include "env/config.h" #include "env/env.h" +#include "env/env_std_extensions.h" +#include "env/runtime_std_extensions.h" #include "env/type_info.h" #include "runtime/reference_resolver.h" #include "runtime/runtime.h" #include "runtime/runtime_builder.h" #include "runtime/runtime_options.h" -#include "runtime/standard_runtime_builder_factory.h" #include "cel_expr_python/cel_extension.h" #include "cel_expr_python/py_cel_env_config.h" +#include "cel_expr_python/py_cel_function.h" +#include "cel_expr_python/py_cel_function_decl.h" +#include "cel_expr_python/py_cel_overload.h" #include "cel_expr_python/py_cel_python_extension.h" #include "cel_expr_python/py_cel_type.h" #include "cel_expr_python/py_descriptor_database.h" +#include "cel_expr_python/py_error_status.h" +#include "cel_expr_python/py_message_factory.h" #include "cel_expr_python/status_macros.h" #include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" #include namespace cel_python { -PyCelEnvInternal::PyCelEnvInternal(const PyCelEnvConfig& env_config, - PyObject* py_descriptor_pool, - const std::vector& extensions, - const std::string& container) +namespace { + +static const cel::FunctionDescriptorOptions kFunctionDescriptorOptions = { + .is_strict = true, .is_contextual = true}; + +} // namespace + +PyCelEnvInternal::PyCelEnvInternal( + const PyCelEnvConfig& env_config, PyObject* py_descriptor_pool, + std::vector>& extension_handles, + std::unordered_map& function_impls) : env_config_(env_config), py_descriptor_database_(py_descriptor_pool), descriptor_pool_( @@ -56,11 +73,30 @@ PyCelEnvInternal::PyCelEnvInternal(const PyCelEnvConfig& env_config, message_factory_(descriptor_pool_.get()), py_message_factory_( std::make_shared(py_descriptor_pool)), - container_(std::move(container)) { + extensions_(std::move(extension_handles)), + function_impls_(std::move(function_impls)) { cel_env_.SetDescriptorPool(descriptor_pool_); cel_env_.SetConfig(env_config_.GetConfig()); - for (PyObject* ext : extensions) { - extensions_.push_back(std::make_unique(ext)); + cel::RegisterStandardExtensions(cel_env_); + + cel_env_runtime_.SetDescriptorPool(descriptor_pool_); + cel_env_runtime_.SetConfig(env_config_.GetConfig()); + cel::RegisterStandardExtensions(cel_env_runtime_); + + for (std::unique_ptr& extension_handle : extensions_) { + // This should never fail because we have already called GetExtension() once + // before calling this constructor. + CelExtension* extension = ThrowIfError(extension_handle->GetExtension()); + cel_env_.RegisterCompilerLibrary( + extension->name(), extension->name(), 0, + [extension]() { return extension->GetCompilerLibrary(); }); + cel_env_runtime_.RegisterExtensionFunctions( + extension->name(), extension->name(), 0, + [extension]( + cel::RuntimeBuilder& runtime_builder, + const cel::RuntimeOptions& runtime_options) -> absl::Status { + return extension->ConfigureRuntime(runtime_builder, runtime_options); + }); } } @@ -68,7 +104,9 @@ absl::StatusOr> PyCelEnvInternal::NewCelEnvInternal( const PyCelEnvConfig& env_config, PyObject* py_descriptor_pool, const std::unordered_map& variable_types, - const std::vector& extensions, const std::string& container) { + const std::vector& extensions, const std::string& container, + const std::vector>& functions, + const std::unordered_map& function_impls) { cel::Config config = env_config.GetConfig(); for (const auto& [name, type] : variable_types) { @@ -79,8 +117,64 @@ PyCelEnvInternal::NewCelEnvInternal( })); } + std::vector> extension_handles; + for (PyObject* ext : extensions) { + auto extension_handle = std::make_unique(ext); + CEL_PYTHON_ASSIGN_OR_RETURN(CelExtension * extension, + extension_handle->GetExtension()); + + std::string name; + if (!extension->alias().empty() && + extension->alias() != extension->name()) { + // If the configuration lists the extension by name, use the name; + // otherwise, use the alias. + name = extension->alias(); + for (const cel::Config::ExtensionConfig& extension_config : + config.GetExtensionConfigs()) { + if (extension_config.name == extension->name()) { + name = extension_config.name; + break; + } + } + } else { + name = extension->name(); + } + CEL_PYTHON_RETURN_IF_ERROR(config.AddExtensionConfig( + name, extension->version() >= 0 + ? extension->version() + : cel::Config::ExtensionConfig::kLatest)); + extension_handles.push_back(std::move(extension_handle)); + } + + config.SetContainerConfig(cel::Config::ContainerConfig{.name = container}); + + std::unordered_map impls; + for (const std::shared_ptr& function : functions) { + CEL_PYTHON_RETURN_IF_ERROR( + config.AddFunctionConfig(function->ToFunctionConfig())); + + for (const PyCelOverload& overload : function->overloads()) { + if (overload.py_function().is_none()) { + continue; + } + if (!impls.insert({overload.overload_id(), overload.py_function()}) + .second) { + return absl::AlreadyExistsError( + absl::StrCat("An implementation for function overload id '", + overload.overload_id(), "' already exists.")); + } + } + } + + for (const auto& [overload_id, py_function] : function_impls) { + if (!impls.insert({overload_id, py_function}).second) { + return absl::AlreadyExistsError( + absl::StrCat("An implementation for function overload id '", + overload_id, "' already exists.")); + } + } return std::shared_ptr(new PyCelEnvInternal( - PyCelEnvConfig(config), py_descriptor_pool, extensions, container)); + PyCelEnvConfig(config), py_descriptor_pool, extension_handles, impls)); } absl::StatusOr PyCelEnvInternal::GetCompiler( @@ -91,22 +185,21 @@ absl::StatusOr PyCelEnvInternal::GetCompiler( return env->compiler_.get(); } + const cel::Config& config = env->env_config_.GetConfig(); + CEL_PYTHON_ASSIGN_OR_RETURN( std::unique_ptr compiler_builder, env->cel_env_.NewCompilerBuilder()); - compiler_builder->GetCheckerBuilder().set_container(env->container_); - for (std::unique_ptr& extension_handle : - env->extensions_) { - CEL_PYTHON_ASSIGN_OR_RETURN(CelExtension * extension, - extension_handle->GetExtension(env)); - CEL_PYTHON_RETURN_IF_ERROR(extension->ConfigureCompiler( - *compiler_builder, *(env->descriptor_pool_.get()))); - } + + cel::TypeCheckerBuilder& checker_builder = + compiler_builder->GetCheckerBuilder(); + + checker_builder.set_container(config.GetContainerConfig().name); // Convert variable types from cel::TypeInfo to PyCelType. - google::protobuf::Arena* arena = compiler_builder->GetCheckerBuilder().arena(); + google::protobuf::Arena* arena = checker_builder.arena(); for (const cel::Config::VariableConfig& variable_config : - env->env_config_.GetConfig().GetVariableConfigs()) { + config.GetVariableConfigs()) { CEL_PYTHON_ASSIGN_OR_RETURN( cel::Type cel_type, cel::TypeInfoToType(variable_config.type_info, @@ -125,8 +218,8 @@ absl::StatusOr PyCelEnvInternal::GetRuntime( return it->second.get(); } - cel::RuntimeOptions opts; - opts.container = env->container_; + cel::RuntimeOptions& opts = env->cel_env_runtime_.mutable_runtime_options(); + opts.container = env->GetEnvConfig().GetConfig().GetContainerConfig().name; opts.enable_empty_wrapper_null_unboxing = true; opts.enable_qualified_type_identifiers = true; opts.enable_timestamp_duration_overflow_errors = true; @@ -137,16 +230,42 @@ absl::StatusOr PyCelEnvInternal::GetRuntime( opts.fail_on_warnings = false; break; } - CEL_PYTHON_ASSIGN_OR_RETURN( - cel::RuntimeBuilder builder, - cel::CreateStandardRuntimeBuilder(env->descriptor_pool_.get(), opts)); + CEL_PYTHON_ASSIGN_OR_RETURN(cel::RuntimeBuilder builder, + env->cel_env_runtime_.CreateRuntimeBuilder()); CEL_PYTHON_RETURN_IF_ERROR(cel::EnableReferenceResolver( builder, cel::ReferenceResolverEnabled::kAlways)); - for (std::unique_ptr& extension_handle : - env->extensions_) { - CEL_PYTHON_ASSIGN_OR_RETURN(CelExtension * extension, - extension_handle->GetExtension(env)); - CEL_PYTHON_RETURN_IF_ERROR(extension->ConfigureRuntime(builder, opts)); + + for (const cel::Config::FunctionConfig& function_config : + env->GetEnvConfig().GetConfig().GetFunctionConfigs()) { + for (const cel::Config::FunctionOverloadConfig& overload_config : + function_config.overload_configs) { + auto it = env->function_impls_.find(overload_config.overload_id); + if (it == env->function_impls_.end()) { + continue; + } + py::object py_function = it->second; + std::vector param_kinds; + param_kinds.reserve(overload_config.parameters.size()); + for (const cel::Config::TypeInfo& parameter : + overload_config.parameters) { + CEL_PYTHON_ASSIGN_OR_RETURN( + cel::Type type, + cel::TypeInfoToType(parameter, env->descriptor_pool_.get(), + &env->arena_)); + param_kinds.push_back(static_cast(type.kind())); + } + cel::FunctionDescriptor descriptor( + function_config.name, overload_config.is_member_function, param_kinds, + kFunctionDescriptorOptions); + CEL_PYTHON_ASSIGN_OR_RETURN( + cel::Type return_type, + cel::TypeInfoToType(overload_config.return_type, + env->descriptor_pool_.get(), &env->arena_)); + CEL_PYTHON_RETURN_IF_ERROR(builder.function_registry().Register( + descriptor, std::make_unique( + function_config.name, + PyCelType::FromCelType(return_type), py_function))); + } } CEL_PYTHON_ASSIGN_OR_RETURN(std::unique_ptr runtime, std::move(builder).Build()); @@ -177,8 +296,7 @@ CelExtensionHandle::~CelExtensionHandle() { PyGILState_Release(gil_state); } -absl::StatusOr CelExtensionHandle::GetExtension( - const std::shared_ptr& env) { +absl::StatusOr CelExtensionHandle::GetExtension() { if (cel_extension_) { return cel_extension_; } diff --git a/cel_expr_python/py_cel_env_internal.h b/cel_expr_python/py_cel_env_internal.h index d6db52a..13f3d2b 100644 --- a/cel_expr_python/py_cel_env_internal.h +++ b/cel_expr_python/py_cel_env_internal.h @@ -20,21 +20,27 @@ #include #include #include +#include #include #include "absl/container/flat_hash_map.h" #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "common/function_descriptor.h" #include "compiler/compiler.h" #include "env/env.h" +#include "env/env_runtime.h" #include "runtime/runtime.h" #include "runtime/runtime_builder.h" #include "runtime/runtime_options.h" #include "cel_expr_python/cel_extension.h" #include "cel_expr_python/py_cel_env_config.h" +#include "cel_expr_python/py_cel_function.h" +#include "cel_expr_python/py_cel_function_decl.h" #include "cel_expr_python/py_cel_type.h" #include "cel_expr_python/py_descriptor_database.h" #include "cel_expr_python/py_message_factory.h" +#include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/dynamic_message.h" #include "google/protobuf/message.h" @@ -48,8 +54,7 @@ class CelExtensionHandle { explicit CelExtensionHandle(PyObject* extension); ~CelExtensionHandle(); - absl::StatusOr GetExtension( - const std::shared_ptr& env); + absl::StatusOr GetExtension(); private: // The Python object that was passed to the constructor and is retained for @@ -69,7 +74,9 @@ class PyCelEnvInternal { static absl::StatusOr> NewCelEnvInternal( const PyCelEnvConfig& env_config, PyObject* py_descriptor_pool, const std::unordered_map& variable_types, - const std::vector& extensions, const std::string& container); + const std::vector& extensions, const std::string& container, + const std::vector>& functions, + const std::unordered_map& function_impls); const PyCelEnvConfig& GetEnvConfig() const { return env_config_; } @@ -105,8 +112,8 @@ class PyCelEnvInternal { // Use NewCelEnvInternal() to create an instance. PyCelEnvInternal(const PyCelEnvConfig& env_config, PyObject* py_descriptor_pool, - const std::vector& extensions, - const std::string& container); + std::vector>& extensions, + std::unordered_map& function_impls); absl::Status ConfigureStandardExtension( cel::CompilerBuilder& compiler_builder, const std::string& extension); @@ -115,7 +122,9 @@ class PyCelEnvInternal { const std::string& extension, const cel::RuntimeOptions& opts); + google::protobuf::Arena arena_; cel::Env cel_env_; + cel::EnvRuntime cel_env_runtime_; PyCelEnvConfig env_config_; PyDescriptorDatabase py_descriptor_database_; std::shared_ptr descriptor_pool_; @@ -124,7 +133,7 @@ class PyCelEnvInternal { // Synchronized by the GIL. std::unordered_map variable_types_; std::vector> extensions_; - std::string container_; + std::unordered_map function_impls_; std::unique_ptr compiler_; absl::flat_hash_map> runtimes_; diff --git a/cel_expr_python/py_cel_function_decl.cc b/cel_expr_python/py_cel_function_decl.cc index 97aac54..b5f084b 100644 --- a/cel_expr_python/py_cel_function_decl.cc +++ b/cel_expr_python/py_cel_function_decl.cc @@ -18,6 +18,7 @@ #include #include +#include "env/config.h" #include "cel_expr_python/py_cel_overload.h" #include #include @@ -33,4 +34,15 @@ void PyCelFunctionDecl::DefinePythonBindings(pybind11::module& m) { py::arg("overloads")); } +cel::Config::FunctionConfig PyCelFunctionDecl::ToFunctionConfig() const { + cel::Config::FunctionConfig function_config; + function_config.name = name_; + function_config.overload_configs.reserve(overloads_.size()); + for (const PyCelOverload& overload : overloads_) { + function_config.overload_configs.push_back( + overload.ToFunctionOverloadConfig()); + } + return function_config; +} + } // namespace cel_python diff --git a/cel_expr_python/py_cel_function_decl.h b/cel_expr_python/py_cel_function_decl.h index ec71930..ae76bfb 100644 --- a/cel_expr_python/py_cel_function_decl.h +++ b/cel_expr_python/py_cel_function_decl.h @@ -19,6 +19,7 @@ #include #include +#include "env/config.h" #include "cel_expr_python/py_cel_overload.h" #include @@ -35,6 +36,8 @@ class PyCelFunctionDecl { std::string name() const { return name_; } const std::vector& overloads() const { return overloads_; } + cel::Config::FunctionConfig ToFunctionConfig() const; + private: std::string name_; std::vector overloads_; diff --git a/cel_expr_python/py_cel_overload.cc b/cel_expr_python/py_cel_overload.cc index c7555c1..5311f46 100644 --- a/cel_expr_python/py_cel_overload.cc +++ b/cel_expr_python/py_cel_overload.cc @@ -15,10 +15,12 @@ #include "cel_expr_python/py_cel_overload.h" #include +#include #include #include #include +#include "env/config.h" #include "cel_expr_python/py_cel_type.h" #include #include @@ -36,9 +38,9 @@ void PyCelOverload::DefinePythonBindings(py::module_& m) { return PyCelOverload(overload_id, return_type, parameters, is_member, std::move(impl)); }), - py::arg("overload_id"), py::arg("return_type"), - py::arg("parameters"), py::arg("is_member") = false, - py::arg("impl") = py::none()); + py::arg("overload_id"), py::arg("return_type") = PyCelType::Dyn(), + py::arg("parameters") = std::vector{}, + py::arg("is_member") = false, py::arg("impl") = py::none()); } PyCelOverload::PyCelOverload(std::string overload_id, @@ -51,4 +53,16 @@ PyCelOverload::PyCelOverload(std::string overload_id, is_member_(is_member), py_function_(std::move(py_function)) {} +cel::Config::FunctionOverloadConfig PyCelOverload::ToFunctionOverloadConfig() + const { + cel::Config::FunctionOverloadConfig overload_config; + overload_config.overload_id = overload_id_; + overload_config.return_type = PyCelType::ToTypeInfo(return_type_); + overload_config.parameters.reserve(parameters_.size()); + for (const PyCelType& parameter : parameters_) { + overload_config.parameters.push_back(PyCelType::ToTypeInfo(parameter)); + } + overload_config.is_member_function = is_member_; + return overload_config; +} } // namespace cel_python diff --git a/cel_expr_python/py_cel_overload.h b/cel_expr_python/py_cel_overload.h index fd5d2c9..05d4e16 100644 --- a/cel_expr_python/py_cel_overload.h +++ b/cel_expr_python/py_cel_overload.h @@ -20,6 +20,7 @@ #include #include +#include "env/config.h" #include "cel_expr_python/py_cel_type.h" #include @@ -44,6 +45,8 @@ class PyCelOverload { bool is_member() const { return is_member_; } py::object py_function() const { return py_function_; } + cel::Config::FunctionOverloadConfig ToFunctionOverloadConfig() const; + private: std::string overload_id_; PyCelType return_type_; diff --git a/cel_expr_python/py_cel_python_extension.cc b/cel_expr_python/py_cel_python_extension.cc index 84f1fe1..15e13e7 100644 --- a/cel_expr_python/py_cel_python_extension.cc +++ b/cel_expr_python/py_cel_python_extension.cc @@ -61,19 +61,22 @@ PyCelPythonExtension::PyCelPythonExtension( std::string name, std::vector functions) : CelExtension(std::move(name)), functions_(std::move(functions)) {} -absl::Status PyCelPythonExtension::ConfigureCompiler( - cel::CompilerBuilder& compiler_builder, - const google::protobuf::DescriptorPool& descriptor_pool) { - google::protobuf::Arena* arena = compiler_builder.GetCheckerBuilder().arena(); - for (const PyCelFunctionDecl& function : functions_) { +namespace { + +absl::Status ConfigureChecker(cel::TypeCheckerBuilder& checker_builder, + const std::vector& functions) { + google::protobuf::Arena* arena = checker_builder.arena(); + const google::protobuf::DescriptorPool* descriptor_pool = + checker_builder.descriptor_pool(); + for (const PyCelFunctionDecl& function : functions) { cel::FunctionDecl function_decl; function_decl.set_name(function.name()); for (const PyCelOverload& overload : function.overloads()) { cel::OverloadDecl overload_decl; overload_decl.set_id(overload.overload_id()); CEL_PYTHON_ASSIGN_OR_RETURN( - cel::Type cel_type, - PyCelType::ToCelType(overload.return_type(), arena, descriptor_pool)); + cel::Type cel_type, PyCelType::ToCelType(overload.return_type(), + arena, *descriptor_pool)); overload_decl.set_result(cel_type); overload_decl.set_member(overload.is_member()); auto& mutable_args = overload_decl.mutable_args(); @@ -81,18 +84,28 @@ absl::Status PyCelPythonExtension::ConfigureCompiler( for (const auto& arg : overload.parameters()) { CEL_PYTHON_ASSIGN_OR_RETURN( cel::Type cel_type, - PyCelType::ToCelType(arg, arena, descriptor_pool)); + PyCelType::ToCelType(arg, arena, *descriptor_pool)); mutable_args.push_back(cel_type); } CEL_PYTHON_RETURN_IF_ERROR( function_decl.AddOverload(std::move(overload_decl))); } - CEL_PYTHON_RETURN_IF_ERROR( - compiler_builder.GetCheckerBuilder().MergeFunction(function_decl)); + CEL_PYTHON_RETURN_IF_ERROR(checker_builder.MergeFunction(function_decl)); } return absl::OkStatus(); } +} // namespace + +cel::CompilerLibrary PyCelPythonExtension::GetCompilerLibrary() { + return cel::CompilerLibrary( + name(), + /*configure_parser=*/nullptr, + /*configure_checker=*/ + [this](cel::TypeCheckerBuilder& checker_builder) -> absl::Status { + return ConfigureChecker(checker_builder, functions_); + }); +} absl::Status PyCelPythonExtension::ConfigureRuntime( cel::RuntimeBuilder& runtime_builder, const cel::RuntimeOptions& opts) { diff --git a/cel_expr_python/py_cel_python_extension.h b/cel_expr_python/py_cel_python_extension.h index 0f49a39..7acc442 100644 --- a/cel_expr_python/py_cel_python_extension.h +++ b/cel_expr_python/py_cel_python_extension.h @@ -24,7 +24,6 @@ #include "runtime/runtime_options.h" #include "cel_expr_python/cel_extension.h" #include "cel_expr_python/py_cel_function_decl.h" -#include "google/protobuf/descriptor.h" #include namespace cel_python { @@ -36,9 +35,7 @@ class PyCelPythonExtension : public CelExtension { std::vector functions); protected: - absl::Status ConfigureCompiler( - cel::CompilerBuilder& compiler_builder, - const google::protobuf::DescriptorPool& descriptor_pool) override; + cel::CompilerLibrary GetCompilerLibrary() override; absl::Status ConfigureRuntime(cel::RuntimeBuilder& runtime_builder, const cel::RuntimeOptions& opts) override; diff --git a/conformance/BUILD b/conformance/BUILD index 3101953..bed1210 100644 --- a/conformance/BUILD +++ b/conformance/BUILD @@ -45,7 +45,7 @@ py_test( "//cel_expr_python/ext:ext_math", "//cel_expr_python/ext:ext_optional", "//cel_expr_python/ext:ext_proto", - "//cel_expr_python/ext:ext_string", + "//cel_expr_python/ext:ext_strings", "@com_google_absl_py//absl/testing:absltest", "//testing:proto2_test_all_types_py_pb2", "//testing:proto3_test_all_types_py_pb2", diff --git a/conformance/conformance_test.py b/conformance/conformance_test.py index 58a82b9..10329c6 100644 --- a/conformance/conformance_test.py +++ b/conformance/conformance_test.py @@ -33,7 +33,7 @@ from cel_expr_python.ext import ext_math from cel_expr_python.ext import ext_optional from cel_expr_python.ext import ext_proto -from cel_expr_python.ext import ext_string +from cel_expr_python.ext import ext_strings from cel.expr.conformance.proto2 import test_all_types_extensions_pb2 as test_all_types_extensions_proto2 # pylint: disable=unused-import from cel.expr.conformance.proto2 import test_all_types_pb2 as test_all_types_proto2 # pylint: disable=unused-import from cel.expr.conformance.proto3 import test_all_types_pb2 as test_all_types_proto3 # pylint: disable=unused-import @@ -138,7 +138,7 @@ class ConformanceTest(absltest.TestCase): "math_ext": [ext_math.ExtMath()], "optionals": [ext_optional.ExtOptional()], "proto2_ext": [ext_proto.ExtProto()], - "string_ext": [ext_string.ExtString()], + "string_ext": [ext_strings.ExtStrings()], "type_deduction": [ext_optional.ExtOptional()], } diff --git a/custom_ext/BUILD b/custom_ext/BUILD index 5a3ba43..ae0d9d5 100644 --- a/custom_ext/BUILD +++ b/custom_ext/BUILD @@ -11,7 +11,6 @@ pybind_extension( "//cel_expr_python:cel", ], deps = [ - "//cel_expr_python:cel", "//cel_expr_python:cel_extension", "//cel_expr_python:status_macros", "@com_google_absl//absl/base:nullability", diff --git a/custom_ext/sample_cel_ext.cc b/custom_ext/sample_cel_ext.cc index ab8c96f..3427473 100644 --- a/custom_ext/sample_cel_ext.cc +++ b/custom_ext/sample_cel_ext.cc @@ -68,29 +68,31 @@ class SampleCelExtension : public cel_python::CelExtension { explicit SampleCelExtension() : cel_python::CelExtension("sample.cel.cpp.ext") {} - absl::Status ConfigureCompiler( - cel::CompilerBuilder& compiler_builder, - const google::protobuf::DescriptorPool& descriptor_pool) override { - CEL_PYTHON_ASSIGN_OR_RETURN( - auto func_translate, - MakeFunctionDecl("translate", - MakeMemberOverloadDecl("translate_inst", + cel::CompilerLibrary GetCompilerLibrary() override { + return cel::CompilerLibrary( + "sample.cel.cpp.ext", + [](cel::TypeCheckerBuilder& checker_builder) -> absl::Status { + CEL_PYTHON_ASSIGN_OR_RETURN( + auto func_translate, + MakeFunctionDecl("translate", MakeMemberOverloadDecl( + "translate_inst", /*return_type=*/StringType(), /*target=*/StringType(), /*from_lang=*/StringType(), /*to_lang=*/StringType()))); - CEL_PYTHON_RETURN_IF_ERROR( - compiler_builder.GetCheckerBuilder().AddFunction(func_translate)); + CEL_PYTHON_RETURN_IF_ERROR( + checker_builder.AddFunction(func_translate)); - CEL_PYTHON_ASSIGN_OR_RETURN( - auto func_translate_late, - MakeFunctionDecl("translate_late", - MakeOverloadDecl("late_bound_translation", - /*return_type=*/StringType(), - /*text=*/StringType()))); - CEL_PYTHON_RETURN_IF_ERROR( - compiler_builder.GetCheckerBuilder().AddFunction(func_translate_late)); - return absl::OkStatus(); + CEL_PYTHON_ASSIGN_OR_RETURN( + auto func_translate_late, + MakeFunctionDecl("translate_late", + MakeOverloadDecl("late_bound_translation", + /*return_type=*/StringType(), + /*text=*/StringType()))); + CEL_PYTHON_RETURN_IF_ERROR( + checker_builder.AddFunction(func_translate_late)); + return absl::OkStatus(); + }); } absl::Status ConfigureRuntime(cel::RuntimeBuilder& runtime_builder,