diff --git a/packages/gapic-generator/gapic/generator/generator.py b/packages/gapic-generator/gapic/generator/generator.py index 9fe56aa9de1d..2ee073fb5319 100644 --- a/packages/gapic-generator/gapic/generator/generator.py +++ b/packages/gapic-generator/gapic/generator/generator.py @@ -37,6 +37,12 @@ from google.protobuf.compiler.plugin_pb2 import CodeGeneratorResponse +ALLOWED_PRIVATE_TEMPLATES = ( + "__init__.py.j2", + "_compat.py.j2", +) + + class Generator: """A protoc code generator for client libraries. @@ -118,9 +124,9 @@ def get_response(self, api_schema: api.API, opts: Options) -> CodeGeneratorRespo # and instead of iterating over it/them, we iterate over samples # and plug those into the template. for template_name in client_templates: - # Quick check: Skip "private" templates. + # Quick check: Skip "private" templates except explicitly allowed ones. filename = template_name.split("/")[-1] - if filename.startswith("_") and filename != "__init__.py.j2": + if filename.startswith("_") and filename not in ALLOWED_PRIVATE_TEMPLATES: continue # Append to the output files dictionary. diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/_compat.py.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/_compat.py.j2 new file mode 100644 index 000000000000..cb347f3414d8 --- /dev/null +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/_compat.py.j2 @@ -0,0 +1,279 @@ +# {% include '_license.j2' %} + +"""A compatibility module for older versions of google-api-core.""" + +import functools +import json +import operator +import os +import re +import uuid +from typing import Any, Callable, Dict, List, Optional, Tuple + +from google.auth.exceptions import MutualTLSChannelError + + +try: + from google.api_core.universe import ( # type: ignore + get_default_mtls_endpoint, + get_api_endpoint, + get_universe_domain, + ) +except ImportError: # pragma: NO COVER + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Universe domain support was introduced in google-api-core 2.18.0. + # Remove these fallback definitions when google-api-core >= 2.18.0 becomes the minimum required version in generated client setup dependencies. + def get_default_mtls_endpoint(api_endpoint: Optional[str]) -> Optional[str]: + """Converts api endpoint to mTLS endpoint.""" + if not api_endpoint: + return api_endpoint + + mtls_endpoint_re = re.compile( + r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" + ) + + m = mtls_endpoint_re.match(api_endpoint) + if m is None: + # Could not parse api_endpoint; return as-is. + return api_endpoint + + name, mtls, sandbox, googledomain = m.groups() + if mtls or not googledomain: + return api_endpoint + + if sandbox: + return api_endpoint.replace( + "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" + ) + + return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") + + def get_api_endpoint( # type: ignore[misc] + api_override: Optional[str], + client_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]], + universe_domain: str, + use_mtls_endpoint: str, + default_universe: str, + default_mtls_endpoint: Optional[str], + default_endpoint_template: str, + ) -> str: + """Return the API endpoint used by the client.""" + api_endpoint: Optional[str] = None + if api_override is not None: + api_endpoint = api_override + elif use_mtls_endpoint == "always" or (use_mtls_endpoint == "auto" and client_cert_source): + if universe_domain != default_universe: + raise MutualTLSChannelError( + f"mTLS is not supported in any universe other than {default_universe}." + ) + api_endpoint = default_mtls_endpoint + else: + api_endpoint = default_endpoint_template.format(UNIVERSE_DOMAIN=universe_domain) + return api_endpoint # type: ignore[return-value] + + def get_universe_domain( # type: ignore[misc] + *potential_universes: Optional[str], + default_universe: str = "googleapis.com", + ) -> str: + """Return the universe domain used by the client.""" + resolved = next( + (x.strip() for x in potential_universes if x is not None), + default_universe, + ) + if not resolved: + raise ValueError("Universe Domain cannot be an empty string.") + return resolved + + +try: + from google.api_core.gapic_v1.config import ( # type: ignore + use_client_cert_effective, + get_client_cert_source, + read_environment_variables, + ) +except ImportError: # pragma: NO COVER + from google.auth.transport import mtls # type: ignore + + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Remove these fallbacks when google-api-core >= 2.18.0 is the minimum required version. + + def use_client_cert_effective() -> bool: + """Returns whether client certificate should be used for mTLS.""" + if hasattr(mtls, "should_use_client_cert"): + return mtls.should_use_client_cert() + else: + use_client_cert_str = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false").lower() + if use_client_cert_str not in ("true", "false"): + raise ValueError( + "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be" + " either `true` or `false`" + ) + return use_client_cert_str == "true" + + def get_client_cert_source( + provided_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]], + use_cert_flag: bool, + ) -> Optional[Callable[[], Tuple[bytes, bytes]]]: + """Return the client cert source to be used by the client.""" + client_cert_source = None + if use_cert_flag: + if provided_cert_source: + client_cert_source = provided_cert_source + elif ( + hasattr(mtls, "has_default_client_cert_source") + and mtls.has_default_client_cert_source() + ): + client_cert_source = mtls.default_client_cert_source() + else: + raise ValueError( + "Client certificate is required for mTLS, but no client certificate source was provided or found." + ) + return client_cert_source + + def read_environment_variables() -> Tuple[bool, str, Optional[str]]: + """Returns the environment variables used by the client.""" + use_client_cert = use_client_cert_effective() + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower() + universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN") + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError( + "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` " + "must be `never`, `auto` or `always`" + ) + return use_client_cert, use_mtls_endpoint, universe_domain_env + + +try: + from google.api_core.gapic_v1.request import setup_request_id # type: ignore +except ImportError: # pragma: NO COVER + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Request ID setup helper was introduced in google-api-core 2.26.0. + # Remove this fallback definition when google-api-core >= 2.26.0 becomes the minimum required version in generated client setup dependencies. + def setup_request_id(request: Any, field_name: str, is_proto3_optional: bool) -> None: + """Populate a UUID4 field in the request if it is not already set. + + Args: + request (Union[google.protobuf.message.Message, dict]): The request object. + field_name (str): The name of the field to populate. + is_proto3_optional (bool): Whether the field is proto3 optional. + """ + if request is None: + return + + request_id_val = str(uuid.uuid4()) + + if isinstance(request, dict): + if is_proto3_optional: + if field_name not in request or request[field_name] is None: + request[field_name] = request_id_val + elif not request.get(field_name): + request[field_name] = request_id_val + return + + if is_proto3_optional: + try: + # Pure protobuf messages + if not request.HasField(field_name): + setattr(request, field_name, request_id_val) + except (AttributeError, ValueError): + # Proto-plus messages or other objects + if hasattr(request, "_pb"): + try: + if not request._pb.HasField(field_name): + setattr(request, field_name, request_id_val) + return + except (AttributeError, ValueError): + pass + if getattr(request, field_name, None) is None: + setattr(request, field_name, request_id_val) + else: + if not getattr(request, field_name, None): + setattr(request, field_name, request_id_val) + + +try: + from google.api_core.rest_helpers import ( # type: ignore + flatten_query_params, + transcode_request, + ) +except ImportError: # pragma: NO COVER + # TODO: Remove these fallbacks when google-api-core >= 2.18.0 is the minimum required version. + from google.protobuf import json_format # type: ignore + from google.api_core import path_template # type: ignore + + def flatten_query_params(obj: Any, strict: bool = False) -> List[Tuple[str, Any]]: # type: ignore[misc] + if obj is not None and not isinstance(obj, dict): + raise TypeError("flatten_query_params must be called with dict object") + return _flatten(obj, key_path=[], strict=strict) + + def _flatten(obj: Any, key_path: List[str], strict: bool = False) -> List[Tuple[str, Any]]: + if obj is None: + return [] + if isinstance(obj, dict): + return _flatten_dict(obj, key_path=key_path, strict=strict) + if isinstance(obj, list): + return _flatten_list(obj, key_path=key_path, strict=strict) + return _flatten_value(obj, key_path=key_path, strict=strict) + + def _is_primitive_value(obj: Any) -> bool: + if obj is None: + return False + if isinstance(obj, (list, dict)): + raise ValueError("query params may not contain repeated dicts or lists") + return True + + def _flatten_value(obj: Any, key_path: List[str], strict: bool = False) -> List[Tuple[str, Any]]: + return [(".".join(key_path), _canonicalize(obj, strict=strict))] + + def _flatten_dict(obj: Dict[str, Any], key_path: List[str], strict: bool = False) -> List[Tuple[str, Any]]: + items = ( + _flatten(value, key_path=key_path + [key], strict=strict) + for key, value in obj.items() + ) + return functools.reduce(operator.concat, items, []) # type: ignore[arg-type] + + def _flatten_list(elems: List[Any], key_path: List[str], strict: bool = False) -> List[Tuple[str, Any]]: + items = ( + _flatten_value(elem, key_path=key_path, strict=strict) + for elem in elems + if _is_primitive_value(elem) + ) + return functools.reduce(operator.concat, items, []) # type: ignore[arg-type] + + def _canonicalize(obj: Any, strict: bool = False) -> Any: + if strict: + value = str(obj) + if isinstance(obj, bool): + value = value.lower() + return value + return obj + + def transcode_request( # type: ignore[misc] + http_options: List[Dict[str, str]], + request: Any, + required_fields_default_values: Optional[Dict[str, Any]] = None, + rest_numeric_enums: bool = False, + ) -> Tuple[Dict[str, Any], Optional[str], Dict[str, Any]]: + pb_request = getattr(request, "_pb", request) + transcoded_request = path_template.transcode(http_options, pb_request) + + body_json = None + if transcoded_request.get("body") is not None: + body_json = json_format.MessageToJson( + transcoded_request["body"], + use_integers_for_enums=rest_numeric_enums, + ) + + query_params_json = {} + if transcoded_request.get("query_params") is not None: + query_params_json = json.loads(json_format.MessageToJson( + transcoded_request["query_params"], + use_integers_for_enums=rest_numeric_enums, + )) + + if required_fields_default_values: + for k, v in required_fields_default_values.items(): + if k not in query_params_json: + query_params_json[k] = v + + if rest_numeric_enums: + query_params_json["$alt"] = "json;enum-encoding=int" + + return transcoded_request, body_json, query_params_json diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/_shared_macros.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/_shared_macros.j2 index 755e4530e7ba..5f1b236f1780 100644 --- a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/_shared_macros.j2 +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/_shared_macros.j2 @@ -28,11 +28,7 @@ {% if method_settings is not none %} {% for auto_populated_field in method_settings.auto_populated_fields %} {% set is_proto3_optional = method.input.fields[auto_populated_field].proto3_optional %} - {% if is_async %} - self._client._setup_request_id(request, '{{ auto_populated_field }}', {{ is_proto3_optional }}) - {% else %} - self._setup_request_id(request, '{{ auto_populated_field }}', {{ is_proto3_optional }}) - {% endif %} + setup_request_id(request, '{{ auto_populated_field }}', {{ is_proto3_optional }}) {% endfor %} {% endif %}{# if method_settings is not none #} {% endwith %}{# method_settings #} diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/async_client.py.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/async_client.py.j2 index 5fe416902b86..7d9809bcb02c 100644 --- a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/async_client.py.j2 +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/async_client.py.j2 @@ -8,9 +8,6 @@ import logging as std_logging from collections import OrderedDict import re from typing import Dict, Callable, Mapping, MutableMapping, MutableSequence, Optional, {% if service.any_server_streaming %}AsyncIterable, Awaitable, {% endif %}{% if service.any_client_streaming %}AsyncIterator, {% endif %}Sequence, Tuple, Type, Union -{% if api.all_method_settings.values()|map(attribute="auto_populated_fields", default=[])|select|list %} -import uuid -{% endif %} {% if service.any_deprecated %} import warnings {% endif %} @@ -21,6 +18,16 @@ from {{package_path}} import gapic_version as package_version from google.api_core.client_options import ClientOptions from google.api_core import exceptions as core_exceptions from google.api_core import gapic_v1 +{% set has_auto_populated_fields = namespace(value=false) %} +{% for method in service.methods.values() %} + {% set method_settings = api.all_method_settings.get(method.meta.address.proto) %} + {% if method_settings and method_settings.auto_populated_fields %} + {% set has_auto_populated_fields.value = true %} + {% endif %} +{% endfor %} +{% if has_auto_populated_fields.value %} +from {{package_path}}._compat import setup_request_id +{% endif %} from google.api_core import retry_async as retries from google.auth import credentials as ga_credentials # type: ignore from google.oauth2 import service_account # type: ignore diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 index e2e3edb24967..451d28e9ea32 100644 --- a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 @@ -16,9 +16,6 @@ import logging as std_logging import os import re from typing import Dict, Callable, Mapping, MutableMapping, MutableSequence, Optional, {% if service.any_server_streaming %}Iterable, {% endif %}{% if service.any_client_streaming %}Iterator, {% endif %}Sequence, Tuple, Type, Union, cast -{% if api.all_method_settings.values()|map(attribute="auto_populated_fields", default=[])|select|list %} -import uuid -{% endif %} import warnings {% set package_path = api.naming.module_namespace|join('.') + "." + api.naming.versioned_module_name %} @@ -30,6 +27,16 @@ from google.api_core import exceptions as core_exceptions from google.api_core import extended_operation {% endif %} from google.api_core import gapic_v1 +{% set has_auto_populated_fields = namespace(value=false) %} +{% for method in service.methods.values() %} + {% set method_settings = api.all_method_settings.get(method.meta.address.proto) %} + {% if method_settings and method_settings.auto_populated_fields %} + {% set has_auto_populated_fields.value = true %} + {% endif %} +{% endfor %} +{% if has_auto_populated_fields.value %} +from {{package_path}}._compat import setup_request_id +{% endif %} from google.api_core import retry as retries from google.auth import credentials as ga_credentials # type: ignore from google.auth.transport import mtls # type: ignore @@ -453,37 +460,7 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta): # NOTE (b/349488459): universe validation is disabled until further notice. return True - {% if api.all_method_settings.values()|map(attribute="auto_populated_fields", default=[])|select|list %} - @staticmethod - def _setup_request_id(request, field_name: str, is_proto3_optional: bool): - """Populate a UUID4 field in the request if it is not already set. - - Args: - request (Union[google.protobuf.message.Message, dict]): The request object. - field_name (str): The name of the field to populate. - is_proto3_optional (bool): Whether the field is proto3 optional. - """ - if isinstance(request, dict): - if is_proto3_optional: - if field_name not in request: - request[field_name] = str(uuid.uuid4()) - elif not request.get(field_name): - request[field_name] = str(uuid.uuid4()) - return - if is_proto3_optional: - try: - # Pure protobuf messages - if not request.HasField(field_name): - setattr(request, field_name, str(uuid.uuid4())) - except (AttributeError, ValueError): - # Proto-plus messages or other objects - if field_name not in request: - setattr(request, field_name, str(uuid.uuid4())) - else: - if not getattr(request, field_name): - setattr(request, field_name, str(uuid.uuid4())) - {% endif %} def _add_cred_info_for_auth_errors( self, @@ -588,12 +565,13 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta): google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport creation failed for any reason. """ - self._client_options = client_options - if isinstance(self._client_options, dict): - self._client_options = client_options_lib.from_dict(self._client_options) - if self._client_options is None: - self._client_options = client_options_lib.ClientOptions() - self._client_options = cast(client_options_lib.ClientOptions, self._client_options) + if isinstance(client_options, dict): + client_options = client_options_lib.from_dict(client_options) + if client_options is None: + client_options = client_options_lib.ClientOptions() + self._client_options: client_options_lib.ClientOptions = cast( + client_options_lib.ClientOptions, client_options + ) universe_domain_opt = getattr(self._client_options, 'universe_domain', None) diff --git a/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 b/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 index c432b75621ea..2d19415208a4 100644 --- a/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 +++ b/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 @@ -6,9 +6,17 @@ {% import "tests/unit/gapic/%name_%version/%sub/test_macros.j2" as test_macros %} {% import "%namespace/%name_%version/%sub/services/%service/_shared_macros.j2" as shared_macros %} +{% set has_auto_populated_fields = namespace(value=false) %} +{% for method in service.methods.values() %} + {% set method_settings = api.all_method_settings.get(method.meta.address.proto) %} + {% if method_settings and method_settings.auto_populated_fields %} + {% set has_auto_populated_fields.value = true %} + {% endif %} +{% endfor %} + import os import asyncio -{% if api.all_method_settings.values()|map(attribute="auto_populated_fields", default=[])|select|list %} +{% if has_auto_populated_fields.value %} import re {% endif %} from unittest import mock @@ -107,7 +115,7 @@ CRED_INFO_JSON = { "principal": "service-account@example.com", } CRED_INFO_STRING = json.dumps(CRED_INFO_JSON) -{% if api.all_method_settings.values()|map(attribute="auto_populated_fields", default=[])|select|list %} +{% if has_auto_populated_fields.value %} _UUID4_RE = re.compile(r"{{ uuid4_re }}") {% endif %} @@ -434,84 +442,6 @@ def test__add_cred_info_for_auth_errors_no_get_cred_info(error_code): client._add_cred_info_for_auth_errors(error) assert error.details == [] -{% if api.all_method_settings.values()|map(attribute="auto_populated_fields", default=[])|select|list %} -def test__setup_request_id(): - class MockRequest: - def __init__(self, **kwargs): - for k, v in kwargs.items(): - setattr(self, k, v) - def __contains__(self, key): - return hasattr(self, key) - - class MockProtoRequest: - def __init__(self, **kwargs): - for k, v in kwargs.items(): - setattr(self, k, v) - def HasField(self, key): - return hasattr(self, key) - - # Test with proto3 optional field not in request - request = MockRequest() - {{ service.client_name }}._setup_request_id(request, "request_id", True) - assert re.match(r"{{ test_macros.get_uuid4_re() }}", request.request_id) - - # Test with proto3 optional field already in request - request = MockRequest(request_id="already_set") - {{ service.client_name }}._setup_request_id(request, "request_id", True) - assert request.request_id == "already_set" - - # Test with non-proto3 optional field empty - request = MockRequest(request_id="") - {{ service.client_name }}._setup_request_id(request, "request_id", False) - assert re.match(r"{{ test_macros.get_uuid4_re() }}", request.request_id) - - # Test with non-proto3 optional field already set - request = MockRequest(request_id="already_set") - {{ service.client_name }}._setup_request_id(request, "request_id", False) - assert request.request_id == "already_set" - - # Test with proto3 optional field not in request (MockProtoRequest) - request = MockProtoRequest() - {{ service.client_name }}._setup_request_id(request, "request_id", True) - assert re.match(r"{{ test_macros.get_uuid4_re() }}", request.request_id) - - # Test with proto3 optional field already in request (MockProtoRequest) - request = MockProtoRequest(request_id="already_set") - {{ service.client_name }}._setup_request_id(request, "request_id", True) - assert request.request_id == "already_set" - - # Test with ValueError - class MockValueErrorRequest: - def HasField(self, key): - raise ValueError("Mismatched field") - def __contains__(self, key): - return hasattr(self, key) - - request = MockValueErrorRequest() - {{ service.client_name }}._setup_request_id(request, "request_id", True) - assert re.match(r"{{ test_macros.get_uuid4_re() }}", request.request_id) - - # Test with dict and proto3 optional field not in request - request = {} - {{ service.client_name }}._setup_request_id(request, "request_id", True) - assert re.match(r"{{ test_macros.get_uuid4_re() }}", request["request_id"]) - - # Test with dict and proto3 optional field already in request - request = {"request_id": "already_set"} - {{ service.client_name }}._setup_request_id(request, "request_id", True) - assert request["request_id"] == "already_set" - - # Test with dict and non-proto3 optional field empty - request = {"request_id": ""} - {{ service.client_name }}._setup_request_id(request, "request_id", False) - assert re.match(r"{{ test_macros.get_uuid4_re() }}", request["request_id"]) - - # Test with dict and non-proto3 optional field already set - request = {"request_id": "already_set"} - {{ service.client_name }}._setup_request_id(request, "request_id", False) - assert request["request_id"] == "already_set" - -{% endif %} @pytest.mark.parametrize("client_class,transport_name", [ {% if 'grpc' in opts.transport %} ({{ service.client_name }}, "grpc"), diff --git a/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_compat.py.j2 b/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_compat.py.j2 new file mode 100644 index 000000000000..064c42017a2f --- /dev/null +++ b/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_compat.py.j2 @@ -0,0 +1,241 @@ +# {% include '_license.j2' %} + +import builtins +import importlib +import sys +from unittest import mock +import pytest +from google.auth.exceptions import MutualTLSChannelError +from google.protobuf import descriptor_pb2 + +{% set package_path = api.naming.module_namespace|join('.') + "." + api.naming.versioned_module_name %} +from {{package_path}} import _compat + + +def test_compat_normal_import(): + assert _compat.setup_request_id is not None + + +def test_compat_fallback_implementations(): + orig_import = builtins.__import__ + + def custom_import(name, globals=None, locals=None, fromlist=(), level=0): + if name in ( + "google.api_core.universe", + "google.api_core.gapic_v1.config", + "google.api_core.gapic_v1.request", + "google.api_core.rest_helpers", + ): + raise ImportError(f"Mocked ImportError for {name}") + return orig_import(name, globals, locals, fromlist, level) + + with mock.patch("builtins.__import__", side_effect=custom_import): + fallback = importlib.reload(_compat) + + # get_default_mtls_endpoint tests + assert fallback.get_default_mtls_endpoint(None) is None + assert fallback.get_default_mtls_endpoint("") == "" + assert fallback.get_default_mtls_endpoint("foo.googleapis.com") == "foo.mtls.googleapis.com" + assert fallback.get_default_mtls_endpoint("foo.sandbox.googleapis.com") == "foo.mtls.sandbox.googleapis.com" + assert fallback.get_default_mtls_endpoint("foo.mtls.googleapis.com") == "foo.mtls.googleapis.com" + assert fallback.get_default_mtls_endpoint("custom.domain.com") == "custom.domain.com" + assert fallback.get_default_mtls_endpoint(":::invalid-url:::") == ":::invalid-url:::" + + # get_api_endpoint tests + assert fallback.get_api_endpoint("https://override.com", None, "googleapis.com", "auto", "googleapis.com", "mtls.com", "https://{UNIVERSE_DOMAIN}") == "https://override.com" + assert fallback.get_api_endpoint(None, lambda: (b"", b""), "googleapis.com", "always", "googleapis.com", "mtls.com", "https://{UNIVERSE_DOMAIN}") == "mtls.com" + assert fallback.get_api_endpoint(None, lambda: (b"", b""), "googleapis.com", "auto", "googleapis.com", "mtls.com", "https://{UNIVERSE_DOMAIN}") == "mtls.com" + with pytest.raises(MutualTLSChannelError): + fallback.get_api_endpoint(None, lambda: (b"", b""), "otheruniverse.com", "always", "googleapis.com", "mtls.com", "https://{UNIVERSE_DOMAIN}") + assert fallback.get_api_endpoint(None, None, "googleapis.com", "never", "googleapis.com", "mtls.com", "https://{UNIVERSE_DOMAIN}") == "https://googleapis.com" + + # get_universe_domain tests + assert fallback.get_universe_domain("custom.com", None, "googleapis.com") == "custom.com" + assert fallback.get_universe_domain(None, "env.com", "googleapis.com") == "env.com" + assert fallback.get_universe_domain(None, None, "googleapis.com") == "googleapis.com" + with pytest.raises(ValueError): + fallback.get_universe_domain(" ", None, "googleapis.com") + + # use_client_cert_effective tests + with mock.patch("google.auth.transport.mtls.should_use_client_cert", return_value=True, create=True): + assert fallback.use_client_cert_effective() is True + + with mock.patch.object(fallback, "mtls", spec=object()): + with mock.patch.dict("os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + assert fallback.use_client_cert_effective() is True + with mock.patch.dict("os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"}): + assert fallback.use_client_cert_effective() is False + with mock.patch.dict("os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "invalid"}): + with pytest.raises(ValueError): + fallback.use_client_cert_effective() + + # get_client_cert_source tests + cert_fn = lambda: (b"cert", b"key") + assert fallback.get_client_cert_source(cert_fn, True) == cert_fn + assert fallback.get_client_cert_source(None, False) is None + + with mock.patch("google.auth.transport.mtls.has_default_client_cert_source", return_value=True, create=True): + with mock.patch("google.auth.transport.mtls.default_client_cert_source", return_value=cert_fn, create=True): + assert fallback.get_client_cert_source(None, True) == cert_fn + + with mock.patch.object(fallback, "mtls", spec=object()): + with pytest.raises(ValueError): + fallback.get_client_cert_source(None, True) + + # read_environment_variables tests + with mock.patch.dict("os.environ", {"GOOGLE_API_USE_MTLS_ENDPOINT": "always", "GOOGLE_CLOUD_UNIVERSE_DOMAIN": "myuniverse.com"}): + use_cert, use_mtls, universe = fallback.read_environment_variables() + assert use_mtls == "always" + assert universe == "myuniverse.com" + + with mock.patch.dict("os.environ", {"GOOGLE_API_USE_MTLS_ENDPOINT": "invalid"}): + with pytest.raises(MutualTLSChannelError): + fallback.read_environment_variables() + + # setup_request_id tests + fallback.setup_request_id(None, "request_id", False) + + d1 = {} + fallback.setup_request_id(d1, "request_id", is_proto3_optional=True) + assert "request_id" in d1 + + d1_existing = {"request_id": None} + fallback.setup_request_id(d1_existing, "request_id", is_proto3_optional=True) + assert d1_existing["request_id"] is not None + + d2 = {} + fallback.setup_request_id(d2, "request_id", is_proto3_optional=False) + assert "request_id" in d2 + + d3 = {"request_id": "existing"} + fallback.setup_request_id(d3, "request_id", is_proto3_optional=False) + assert d3["request_id"] == "existing" + + d4 = {"request_id": "val"} + fallback.setup_request_id(d4, "request_id", is_proto3_optional=True) + assert d4["request_id"] == "val" + + class DummyPopulated: + def __init__(self): + self.request_id = "val" + p_existing = DummyPopulated() + fallback.setup_request_id(p_existing, "request_id", is_proto3_optional=False) + assert p_existing.request_id == "val" + + class NonOptPlain: + def __init__(self): + self.request_id = "" + nop = NonOptPlain() + fallback.setup_request_id(nop, "request_id", is_proto3_optional=False) + assert nop.request_id != "" + + class DummyProto: + def __init__(self): + self.request_id = "" + self._has = False + def HasField(self, name): + if not self._has: + return False + return True + + p1 = DummyProto() + fallback.setup_request_id(p1, "request_id", is_proto3_optional=True) + assert p1.request_id != "" + + class DummyWrapper: + def __init__(self): + self._pb = DummyProto() + + w1 = DummyWrapper() + fallback.setup_request_id(w1, "request_id", is_proto3_optional=True) + assert getattr(w1, "request_id", None) is not None or getattr(w1._pb, "request_id", None) != "" + + class BadProto: + def HasField(self, name): + raise AttributeError() + class BadWrapper: + def __init__(self): + self._pb = BadProto() + self.request_id = None + bw = BadWrapper() + fallback.setup_request_id(bw, "request_id", is_proto3_optional=True) + assert bw.request_id is not None + + class SetProto: + def __init__(self): + self.request_id = "already_set" + def HasField(self, name): + return True + sp = SetProto() + fallback.setup_request_id(sp, "request_id", is_proto3_optional=True) + assert sp.request_id == "already_set" + + class SetProtoNonOpt: + def __init__(self): + self.request_id = "already_set" + sp_non_opt = SetProtoNonOpt() + fallback.setup_request_id(sp_non_opt, "request_id", is_proto3_optional=False) + assert sp_non_opt.request_id == "already_set" + + class DummyPlain: + def __init__(self): + self.request_id = None + + pl1 = DummyPlain() + fallback.setup_request_id(pl1, "request_id", is_proto3_optional=True) + assert pl1.request_id is not None + + pl2 = DummyPlain() + fallback.setup_request_id(pl2, "request_id", is_proto3_optional=False) + assert pl2.request_id is not None + + # flatten_query_params tests + assert fallback.flatten_query_params(None) == [] + res = fallback.flatten_query_params({"a": "val1", "b": [1, 2], "c": True}) + assert ("a", "val1") in res + assert ("b", 1) in res + assert ("b", 2) in res + assert ("c", True) in res + + res_strict = fallback.flatten_query_params({"c": True}, strict=True) + assert ("c", "true") in res_strict + + assert fallback._canonicalize(True, strict=True) == "true" + assert fallback._canonicalize(True, strict=False) is True + assert fallback._canonicalize(123, strict=True) == "123" + assert fallback._canonicalize("str", strict=True) == "str" + assert fallback._canonicalize("str", strict=False) == "str" + assert fallback._is_primitive_value(None) is False + + with pytest.raises(TypeError): + fallback.flatten_query_params("invalid") + + with pytest.raises(ValueError): + fallback.flatten_query_params({"a": [{"nested": "dict"}]}) + + with pytest.raises(ValueError): + fallback._is_primitive_value([1, 2]) + + # transcode_request tests + dummy_req = descriptor_pb2.DescriptorProto() + http_opts = [{"method": "get", "uri": "/v1/test"}] + transcoded, body, query = fallback.transcode_request( + http_opts, + dummy_req, + required_fields_default_values={"non_existent_key": "val"}, + rest_numeric_enums=True, + ) + assert transcoded is not None + assert query.get("non_existent_key") == "val" + assert query.get("$alt") == "json;enum-encoding=int" + + transcoded2, body2, query2 = fallback.transcode_request( + http_opts, + dummy_req, + required_fields_default_values={"name": "override_name"}, + rest_numeric_enums=False, + ) + assert body2 is None + assert "$alt" not in query2 + + importlib.reload(_compat) diff --git a/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_macros.j2 b/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_macros.j2 index bccc38afe2a1..982b2fb19b44 100644 --- a/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_macros.j2 +++ b/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_macros.j2 @@ -2215,10 +2215,6 @@ def test_initialize_client_w_{{transport_name}}(): {% endfor %}{# method in service.methods.values() #} {% endmacro %}{# empty_call_test #} -{% macro get_uuid4_re() -%} -{{ uuid4_re }} -{%- endmacro %}{# uuid_re #} - {% macro routing_parameter_test(service, api, transport, is_async) %} {% for method in service.methods.values() %}{# method #} {# See existing proposal b/330610501 to add support for explicit routing in BIDI/client side streaming #} diff --git a/packages/gapic-generator/noxfile.py b/packages/gapic-generator/noxfile.py index 9d3d9fefaa8d..e977224acfb8 100644 --- a/packages/gapic-generator/noxfile.py +++ b/packages/gapic-generator/noxfile.py @@ -392,6 +392,8 @@ def showcase_library( # Install the library without a constraints file. session.install("-e", tmp_dir) + session.install("-e", "../google-api-core", "--no-deps") + yield tmp_dir diff --git a/packages/gapic-generator/tests/integration/goldens/asset/google/cloud/asset_v1/_compat.py b/packages/gapic-generator/tests/integration/goldens/asset/google/cloud/asset_v1/_compat.py new file mode 100644 index 000000000000..54c22cd7d09e --- /dev/null +++ b/packages/gapic-generator/tests/integration/goldens/asset/google/cloud/asset_v1/_compat.py @@ -0,0 +1,310 @@ +# # Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""A compatibility module for older versions of google-api-core.""" + +import functools +import json +import operator +import os +import re +import uuid +from typing import Any, Callable, Dict, List, Optional, Tuple + +from google.auth.exceptions import MutualTLSChannelError + + +try: + from google.api_core.universe import ( # type: ignore + get_default_mtls_endpoint, + get_api_endpoint, + get_universe_domain, + ) +except ImportError: # pragma: NO COVER + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Universe domain support was introduced in google-api-core 2.18.0. + # Remove these fallback definitions when google-api-core >= 2.18.0 becomes the minimum required version in generated client setup dependencies. + def get_default_mtls_endpoint(api_endpoint: Optional[str]) -> Optional[str]: + """Converts api endpoint to mTLS endpoint.""" + if not api_endpoint: + return api_endpoint + + mtls_endpoint_re = re.compile( + r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" + ) + + m = mtls_endpoint_re.match(api_endpoint) + if m is None: + # Could not parse api_endpoint; return as-is. + return api_endpoint + + name, mtls, sandbox, googledomain = m.groups() + if mtls or not googledomain: + return api_endpoint + + if sandbox: + return api_endpoint.replace( + "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" + ) + + return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") + + def get_api_endpoint( # type: ignore[misc] + api_override: Optional[str], + client_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]], + universe_domain: str, + use_mtls_endpoint: str, + default_universe: str, + default_mtls_endpoint: Optional[str], + default_endpoint_template: str, + ) -> str: + """Return the API endpoint used by the client.""" + api_endpoint: Optional[str] = None + if api_override is not None: + api_endpoint = api_override + elif use_mtls_endpoint == "always" or ( + use_mtls_endpoint == "auto" and client_cert_source + ): + if universe_domain != default_universe: + raise MutualTLSChannelError( + f"mTLS is not supported in any universe other than {default_universe}." + ) + api_endpoint = default_mtls_endpoint + else: + api_endpoint = default_endpoint_template.format( + UNIVERSE_DOMAIN=universe_domain + ) + return api_endpoint # type: ignore[return-value] + + def get_universe_domain( # type: ignore[misc] + *potential_universes: Optional[str], + default_universe: str = "googleapis.com", + ) -> str: + """Return the universe domain used by the client.""" + resolved = next( + (x.strip() for x in potential_universes if x is not None), + default_universe, + ) + if not resolved: + raise ValueError("Universe Domain cannot be an empty string.") + return resolved + + +try: + from google.api_core.gapic_v1.config import ( # type: ignore + use_client_cert_effective, + get_client_cert_source, + read_environment_variables, + ) +except ImportError: # pragma: NO COVER + from google.auth.transport import mtls # type: ignore + + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Remove these fallbacks when google-api-core >= 2.18.0 is the minimum required version. + + def use_client_cert_effective() -> bool: + """Returns whether client certificate should be used for mTLS.""" + if hasattr(mtls, "should_use_client_cert"): + return mtls.should_use_client_cert() + else: + use_client_cert_str = os.getenv( + "GOOGLE_API_USE_CLIENT_CERTIFICATE", "false" + ).lower() + if use_client_cert_str not in ("true", "false"): + raise ValueError( + "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be" + " either `true` or `false`" + ) + return use_client_cert_str == "true" + + def get_client_cert_source( + provided_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]], + use_cert_flag: bool, + ) -> Optional[Callable[[], Tuple[bytes, bytes]]]: + """Return the client cert source to be used by the client.""" + client_cert_source = None + if use_cert_flag: + if provided_cert_source: + client_cert_source = provided_cert_source + elif ( + hasattr(mtls, "has_default_client_cert_source") + and mtls.has_default_client_cert_source() + ): + client_cert_source = mtls.default_client_cert_source() + else: + raise ValueError( + "Client certificate is required for mTLS, but no client certificate source was provided or found." + ) + return client_cert_source + + def read_environment_variables() -> Tuple[bool, str, Optional[str]]: + """Returns the environment variables used by the client.""" + use_client_cert = use_client_cert_effective() + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower() + universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN") + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError( + "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` " + "must be `never`, `auto` or `always`" + ) + return use_client_cert, use_mtls_endpoint, universe_domain_env + + +try: + from google.api_core.gapic_v1.request import setup_request_id # type: ignore +except ImportError: # pragma: NO COVER + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Request ID setup helper was introduced in google-api-core 2.26.0. + # Remove this fallback definition when google-api-core >= 2.26.0 becomes the minimum required version in generated client setup dependencies. + def setup_request_id( + request: Any, field_name: str, is_proto3_optional: bool + ) -> None: + """Populate a UUID4 field in the request if it is not already set. + + Args: + request (Union[google.protobuf.message.Message, dict]): The request object. + field_name (str): The name of the field to populate. + is_proto3_optional (bool): Whether the field is proto3 optional. + """ + if request is None: + return + + request_id_val = str(uuid.uuid4()) + + if isinstance(request, dict): + if is_proto3_optional: + if field_name not in request or request[field_name] is None: + request[field_name] = request_id_val + elif not request.get(field_name): + request[field_name] = request_id_val + return + + if is_proto3_optional: + try: + # Pure protobuf messages + if not request.HasField(field_name): + setattr(request, field_name, request_id_val) + except (AttributeError, ValueError): + # Proto-plus messages or other objects + if hasattr(request, "_pb"): + try: + if not request._pb.HasField(field_name): + setattr(request, field_name, request_id_val) + return + except (AttributeError, ValueError): + pass + if getattr(request, field_name, None) is None: + setattr(request, field_name, request_id_val) + else: + if not getattr(request, field_name, None): + setattr(request, field_name, request_id_val) + + +try: + from google.api_core.rest_helpers import ( # type: ignore + flatten_query_params, + transcode_request, + ) +except ImportError: # pragma: NO COVER + # TODO: Remove these fallbacks when google-api-core >= 2.18.0 is the minimum required version. + from google.protobuf import json_format # type: ignore + from google.api_core import path_template # type: ignore + + def flatten_query_params(obj: Any, strict: bool = False) -> List[Tuple[str, Any]]: # type: ignore[misc] + if obj is not None and not isinstance(obj, dict): + raise TypeError("flatten_query_params must be called with dict object") + return _flatten(obj, key_path=[], strict=strict) + + def _flatten( + obj: Any, key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + if obj is None: + return [] + if isinstance(obj, dict): + return _flatten_dict(obj, key_path=key_path, strict=strict) + if isinstance(obj, list): + return _flatten_list(obj, key_path=key_path, strict=strict) + return _flatten_value(obj, key_path=key_path, strict=strict) + + def _is_primitive_value(obj: Any) -> bool: + if obj is None: + return False + if isinstance(obj, (list, dict)): + raise ValueError("query params may not contain repeated dicts or lists") + return True + + def _flatten_value( + obj: Any, key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + return [(".".join(key_path), _canonicalize(obj, strict=strict))] + + def _flatten_dict( + obj: Dict[str, Any], key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + items = ( + _flatten(value, key_path=key_path + [key], strict=strict) + for key, value in obj.items() + ) + return functools.reduce(operator.concat, items, []) # type: ignore[arg-type] + + def _flatten_list( + elems: List[Any], key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + items = ( + _flatten_value(elem, key_path=key_path, strict=strict) + for elem in elems + if _is_primitive_value(elem) + ) + return functools.reduce(operator.concat, items, []) # type: ignore[arg-type] + + def _canonicalize(obj: Any, strict: bool = False) -> Any: + if strict: + value = str(obj) + if isinstance(obj, bool): + value = value.lower() + return value + return obj + + def transcode_request( # type: ignore[misc] + http_options: List[Dict[str, str]], + request: Any, + required_fields_default_values: Optional[Dict[str, Any]] = None, + rest_numeric_enums: bool = False, + ) -> Tuple[Dict[str, Any], Optional[str], Dict[str, Any]]: + pb_request = getattr(request, "_pb", request) + transcoded_request = path_template.transcode(http_options, pb_request) + + body_json = None + if transcoded_request.get("body") is not None: + body_json = json_format.MessageToJson( + transcoded_request["body"], + use_integers_for_enums=rest_numeric_enums, + ) + + query_params_json = {} + if transcoded_request.get("query_params") is not None: + query_params_json = json.loads( + json_format.MessageToJson( + transcoded_request["query_params"], + use_integers_for_enums=rest_numeric_enums, + ) + ) + + if required_fields_default_values: + for k, v in required_fields_default_values.items(): + if k not in query_params_json: + query_params_json[k] = v + + if rest_numeric_enums: + query_params_json["$alt"] = "json;enum-encoding=int" + + return transcoded_request, body_json, query_params_json diff --git a/packages/gapic-generator/tests/integration/goldens/asset/tests/unit/gapic/asset_v1/test_compat.py b/packages/gapic-generator/tests/integration/goldens/asset/tests/unit/gapic/asset_v1/test_compat.py new file mode 100644 index 000000000000..610c616b82b0 --- /dev/null +++ b/packages/gapic-generator/tests/integration/goldens/asset/tests/unit/gapic/asset_v1/test_compat.py @@ -0,0 +1,364 @@ +# # Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import builtins +import importlib +import sys +from unittest import mock +import pytest +from google.auth.exceptions import MutualTLSChannelError +from google.protobuf import descriptor_pb2 + + +from google.cloud.asset_v1 import _compat + + +def test_compat_normal_import(): + assert _compat.setup_request_id is not None + + +def test_compat_fallback_implementations(): + orig_import = builtins.__import__ + + def custom_import(name, globals=None, locals=None, fromlist=(), level=0): + if name in ( + "google.api_core.universe", + "google.api_core.gapic_v1.config", + "google.api_core.gapic_v1.request", + "google.api_core.rest_helpers", + ): + raise ImportError(f"Mocked ImportError for {name}") + return orig_import(name, globals, locals, fromlist, level) + + with mock.patch("builtins.__import__", side_effect=custom_import): + fallback = importlib.reload(_compat) + + # get_default_mtls_endpoint tests + assert fallback.get_default_mtls_endpoint(None) is None + assert fallback.get_default_mtls_endpoint("") == "" + assert ( + fallback.get_default_mtls_endpoint("foo.googleapis.com") + == "foo.mtls.googleapis.com" + ) + assert ( + fallback.get_default_mtls_endpoint("foo.sandbox.googleapis.com") + == "foo.mtls.sandbox.googleapis.com" + ) + assert ( + fallback.get_default_mtls_endpoint("foo.mtls.googleapis.com") + == "foo.mtls.googleapis.com" + ) + assert ( + fallback.get_default_mtls_endpoint("custom.domain.com") + == "custom.domain.com" + ) + assert ( + fallback.get_default_mtls_endpoint(":::invalid-url:::") + == ":::invalid-url:::" + ) + + # get_api_endpoint tests + assert ( + fallback.get_api_endpoint( + "https://override.com", + None, + "googleapis.com", + "auto", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "https://override.com" + ) + assert ( + fallback.get_api_endpoint( + None, + lambda: (b"", b""), + "googleapis.com", + "always", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "mtls.com" + ) + assert ( + fallback.get_api_endpoint( + None, + lambda: (b"", b""), + "googleapis.com", + "auto", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "mtls.com" + ) + with pytest.raises(MutualTLSChannelError): + fallback.get_api_endpoint( + None, + lambda: (b"", b""), + "otheruniverse.com", + "always", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + assert ( + fallback.get_api_endpoint( + None, + None, + "googleapis.com", + "never", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "https://googleapis.com" + ) + + # get_universe_domain tests + assert ( + fallback.get_universe_domain("custom.com", None, "googleapis.com") + == "custom.com" + ) + assert ( + fallback.get_universe_domain(None, "env.com", "googleapis.com") == "env.com" + ) + assert ( + fallback.get_universe_domain(None, None, "googleapis.com") + == "googleapis.com" + ) + with pytest.raises(ValueError): + fallback.get_universe_domain(" ", None, "googleapis.com") + + # use_client_cert_effective tests + with mock.patch( + "google.auth.transport.mtls.should_use_client_cert", + return_value=True, + create=True, + ): + assert fallback.use_client_cert_effective() is True + + with mock.patch.object(fallback, "mtls", spec=object()): + with mock.patch.dict( + "os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"} + ): + assert fallback.use_client_cert_effective() is True + with mock.patch.dict( + "os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"} + ): + assert fallback.use_client_cert_effective() is False + with mock.patch.dict( + "os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "invalid"} + ): + with pytest.raises(ValueError): + fallback.use_client_cert_effective() + + # get_client_cert_source tests + cert_fn = lambda: (b"cert", b"key") + assert fallback.get_client_cert_source(cert_fn, True) == cert_fn + assert fallback.get_client_cert_source(None, False) is None + + with mock.patch( + "google.auth.transport.mtls.has_default_client_cert_source", + return_value=True, + create=True, + ): + with mock.patch( + "google.auth.transport.mtls.default_client_cert_source", + return_value=cert_fn, + create=True, + ): + assert fallback.get_client_cert_source(None, True) == cert_fn + + with mock.patch.object(fallback, "mtls", spec=object()): + with pytest.raises(ValueError): + fallback.get_client_cert_source(None, True) + + # read_environment_variables tests + with mock.patch.dict( + "os.environ", + { + "GOOGLE_API_USE_MTLS_ENDPOINT": "always", + "GOOGLE_CLOUD_UNIVERSE_DOMAIN": "myuniverse.com", + }, + ): + use_cert, use_mtls, universe = fallback.read_environment_variables() + assert use_mtls == "always" + assert universe == "myuniverse.com" + + with mock.patch.dict("os.environ", {"GOOGLE_API_USE_MTLS_ENDPOINT": "invalid"}): + with pytest.raises(MutualTLSChannelError): + fallback.read_environment_variables() + + # setup_request_id tests + fallback.setup_request_id(None, "request_id", False) + + d1 = {} + fallback.setup_request_id(d1, "request_id", is_proto3_optional=True) + assert "request_id" in d1 + + d1_existing = {"request_id": None} + fallback.setup_request_id(d1_existing, "request_id", is_proto3_optional=True) + assert d1_existing["request_id"] is not None + + d2 = {} + fallback.setup_request_id(d2, "request_id", is_proto3_optional=False) + assert "request_id" in d2 + + d3 = {"request_id": "existing"} + fallback.setup_request_id(d3, "request_id", is_proto3_optional=False) + assert d3["request_id"] == "existing" + + d4 = {"request_id": "val"} + fallback.setup_request_id(d4, "request_id", is_proto3_optional=True) + assert d4["request_id"] == "val" + + class DummyPopulated: + def __init__(self): + self.request_id = "val" + + p_existing = DummyPopulated() + fallback.setup_request_id(p_existing, "request_id", is_proto3_optional=False) + assert p_existing.request_id == "val" + + class NonOptPlain: + def __init__(self): + self.request_id = "" + + nop = NonOptPlain() + fallback.setup_request_id(nop, "request_id", is_proto3_optional=False) + assert nop.request_id != "" + + class DummyProto: + def __init__(self): + self.request_id = "" + self._has = False + + def HasField(self, name): + if not self._has: + return False + return True + + p1 = DummyProto() + fallback.setup_request_id(p1, "request_id", is_proto3_optional=True) + assert p1.request_id != "" + + class DummyWrapper: + def __init__(self): + self._pb = DummyProto() + + w1 = DummyWrapper() + fallback.setup_request_id(w1, "request_id", is_proto3_optional=True) + assert ( + getattr(w1, "request_id", None) is not None + or getattr(w1._pb, "request_id", None) != "" + ) + + class BadProto: + def HasField(self, name): + raise AttributeError() + + class BadWrapper: + def __init__(self): + self._pb = BadProto() + self.request_id = None + + bw = BadWrapper() + fallback.setup_request_id(bw, "request_id", is_proto3_optional=True) + assert bw.request_id is not None + + class SetProto: + def __init__(self): + self.request_id = "already_set" + + def HasField(self, name): + return True + + sp = SetProto() + fallback.setup_request_id(sp, "request_id", is_proto3_optional=True) + assert sp.request_id == "already_set" + + class SetProtoNonOpt: + def __init__(self): + self.request_id = "already_set" + + sp_non_opt = SetProtoNonOpt() + fallback.setup_request_id(sp_non_opt, "request_id", is_proto3_optional=False) + assert sp_non_opt.request_id == "already_set" + + class DummyPlain: + def __init__(self): + self.request_id = None + + pl1 = DummyPlain() + fallback.setup_request_id(pl1, "request_id", is_proto3_optional=True) + assert pl1.request_id is not None + + pl2 = DummyPlain() + fallback.setup_request_id(pl2, "request_id", is_proto3_optional=False) + assert pl2.request_id is not None + + # flatten_query_params tests + assert fallback.flatten_query_params(None) == [] + res = fallback.flatten_query_params({"a": "val1", "b": [1, 2], "c": True}) + assert ("a", "val1") in res + assert ("b", 1) in res + assert ("b", 2) in res + assert ("c", True) in res + + res_strict = fallback.flatten_query_params({"c": True}, strict=True) + assert ("c", "true") in res_strict + + assert fallback._canonicalize(True, strict=True) == "true" + assert fallback._canonicalize(True, strict=False) is True + assert fallback._canonicalize(123, strict=True) == "123" + assert fallback._canonicalize("str", strict=True) == "str" + assert fallback._canonicalize("str", strict=False) == "str" + assert fallback._is_primitive_value(None) is False + + with pytest.raises(TypeError): + fallback.flatten_query_params("invalid") + + with pytest.raises(ValueError): + fallback.flatten_query_params({"a": [{"nested": "dict"}]}) + + with pytest.raises(ValueError): + fallback._is_primitive_value([1, 2]) + + # transcode_request tests + dummy_req = descriptor_pb2.DescriptorProto() + http_opts = [{"method": "get", "uri": "/v1/test"}] + transcoded, body, query = fallback.transcode_request( + http_opts, + dummy_req, + required_fields_default_values={"non_existent_key": "val"}, + rest_numeric_enums=True, + ) + assert transcoded is not None + assert query.get("non_existent_key") == "val" + assert query.get("$alt") == "json;enum-encoding=int" + + transcoded2, body2, query2 = fallback.transcode_request( + http_opts, + dummy_req, + required_fields_default_values={"name": "override_name"}, + rest_numeric_enums=False, + ) + assert body2 is None + assert "$alt" not in query2 + + importlib.reload(_compat) diff --git a/packages/gapic-generator/tests/integration/goldens/credentials/google/iam/credentials_v1/_compat.py b/packages/gapic-generator/tests/integration/goldens/credentials/google/iam/credentials_v1/_compat.py new file mode 100644 index 000000000000..54c22cd7d09e --- /dev/null +++ b/packages/gapic-generator/tests/integration/goldens/credentials/google/iam/credentials_v1/_compat.py @@ -0,0 +1,310 @@ +# # Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""A compatibility module for older versions of google-api-core.""" + +import functools +import json +import operator +import os +import re +import uuid +from typing import Any, Callable, Dict, List, Optional, Tuple + +from google.auth.exceptions import MutualTLSChannelError + + +try: + from google.api_core.universe import ( # type: ignore + get_default_mtls_endpoint, + get_api_endpoint, + get_universe_domain, + ) +except ImportError: # pragma: NO COVER + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Universe domain support was introduced in google-api-core 2.18.0. + # Remove these fallback definitions when google-api-core >= 2.18.0 becomes the minimum required version in generated client setup dependencies. + def get_default_mtls_endpoint(api_endpoint: Optional[str]) -> Optional[str]: + """Converts api endpoint to mTLS endpoint.""" + if not api_endpoint: + return api_endpoint + + mtls_endpoint_re = re.compile( + r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" + ) + + m = mtls_endpoint_re.match(api_endpoint) + if m is None: + # Could not parse api_endpoint; return as-is. + return api_endpoint + + name, mtls, sandbox, googledomain = m.groups() + if mtls or not googledomain: + return api_endpoint + + if sandbox: + return api_endpoint.replace( + "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" + ) + + return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") + + def get_api_endpoint( # type: ignore[misc] + api_override: Optional[str], + client_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]], + universe_domain: str, + use_mtls_endpoint: str, + default_universe: str, + default_mtls_endpoint: Optional[str], + default_endpoint_template: str, + ) -> str: + """Return the API endpoint used by the client.""" + api_endpoint: Optional[str] = None + if api_override is not None: + api_endpoint = api_override + elif use_mtls_endpoint == "always" or ( + use_mtls_endpoint == "auto" and client_cert_source + ): + if universe_domain != default_universe: + raise MutualTLSChannelError( + f"mTLS is not supported in any universe other than {default_universe}." + ) + api_endpoint = default_mtls_endpoint + else: + api_endpoint = default_endpoint_template.format( + UNIVERSE_DOMAIN=universe_domain + ) + return api_endpoint # type: ignore[return-value] + + def get_universe_domain( # type: ignore[misc] + *potential_universes: Optional[str], + default_universe: str = "googleapis.com", + ) -> str: + """Return the universe domain used by the client.""" + resolved = next( + (x.strip() for x in potential_universes if x is not None), + default_universe, + ) + if not resolved: + raise ValueError("Universe Domain cannot be an empty string.") + return resolved + + +try: + from google.api_core.gapic_v1.config import ( # type: ignore + use_client_cert_effective, + get_client_cert_source, + read_environment_variables, + ) +except ImportError: # pragma: NO COVER + from google.auth.transport import mtls # type: ignore + + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Remove these fallbacks when google-api-core >= 2.18.0 is the minimum required version. + + def use_client_cert_effective() -> bool: + """Returns whether client certificate should be used for mTLS.""" + if hasattr(mtls, "should_use_client_cert"): + return mtls.should_use_client_cert() + else: + use_client_cert_str = os.getenv( + "GOOGLE_API_USE_CLIENT_CERTIFICATE", "false" + ).lower() + if use_client_cert_str not in ("true", "false"): + raise ValueError( + "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be" + " either `true` or `false`" + ) + return use_client_cert_str == "true" + + def get_client_cert_source( + provided_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]], + use_cert_flag: bool, + ) -> Optional[Callable[[], Tuple[bytes, bytes]]]: + """Return the client cert source to be used by the client.""" + client_cert_source = None + if use_cert_flag: + if provided_cert_source: + client_cert_source = provided_cert_source + elif ( + hasattr(mtls, "has_default_client_cert_source") + and mtls.has_default_client_cert_source() + ): + client_cert_source = mtls.default_client_cert_source() + else: + raise ValueError( + "Client certificate is required for mTLS, but no client certificate source was provided or found." + ) + return client_cert_source + + def read_environment_variables() -> Tuple[bool, str, Optional[str]]: + """Returns the environment variables used by the client.""" + use_client_cert = use_client_cert_effective() + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower() + universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN") + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError( + "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` " + "must be `never`, `auto` or `always`" + ) + return use_client_cert, use_mtls_endpoint, universe_domain_env + + +try: + from google.api_core.gapic_v1.request import setup_request_id # type: ignore +except ImportError: # pragma: NO COVER + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Request ID setup helper was introduced in google-api-core 2.26.0. + # Remove this fallback definition when google-api-core >= 2.26.0 becomes the minimum required version in generated client setup dependencies. + def setup_request_id( + request: Any, field_name: str, is_proto3_optional: bool + ) -> None: + """Populate a UUID4 field in the request if it is not already set. + + Args: + request (Union[google.protobuf.message.Message, dict]): The request object. + field_name (str): The name of the field to populate. + is_proto3_optional (bool): Whether the field is proto3 optional. + """ + if request is None: + return + + request_id_val = str(uuid.uuid4()) + + if isinstance(request, dict): + if is_proto3_optional: + if field_name not in request or request[field_name] is None: + request[field_name] = request_id_val + elif not request.get(field_name): + request[field_name] = request_id_val + return + + if is_proto3_optional: + try: + # Pure protobuf messages + if not request.HasField(field_name): + setattr(request, field_name, request_id_val) + except (AttributeError, ValueError): + # Proto-plus messages or other objects + if hasattr(request, "_pb"): + try: + if not request._pb.HasField(field_name): + setattr(request, field_name, request_id_val) + return + except (AttributeError, ValueError): + pass + if getattr(request, field_name, None) is None: + setattr(request, field_name, request_id_val) + else: + if not getattr(request, field_name, None): + setattr(request, field_name, request_id_val) + + +try: + from google.api_core.rest_helpers import ( # type: ignore + flatten_query_params, + transcode_request, + ) +except ImportError: # pragma: NO COVER + # TODO: Remove these fallbacks when google-api-core >= 2.18.0 is the minimum required version. + from google.protobuf import json_format # type: ignore + from google.api_core import path_template # type: ignore + + def flatten_query_params(obj: Any, strict: bool = False) -> List[Tuple[str, Any]]: # type: ignore[misc] + if obj is not None and not isinstance(obj, dict): + raise TypeError("flatten_query_params must be called with dict object") + return _flatten(obj, key_path=[], strict=strict) + + def _flatten( + obj: Any, key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + if obj is None: + return [] + if isinstance(obj, dict): + return _flatten_dict(obj, key_path=key_path, strict=strict) + if isinstance(obj, list): + return _flatten_list(obj, key_path=key_path, strict=strict) + return _flatten_value(obj, key_path=key_path, strict=strict) + + def _is_primitive_value(obj: Any) -> bool: + if obj is None: + return False + if isinstance(obj, (list, dict)): + raise ValueError("query params may not contain repeated dicts or lists") + return True + + def _flatten_value( + obj: Any, key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + return [(".".join(key_path), _canonicalize(obj, strict=strict))] + + def _flatten_dict( + obj: Dict[str, Any], key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + items = ( + _flatten(value, key_path=key_path + [key], strict=strict) + for key, value in obj.items() + ) + return functools.reduce(operator.concat, items, []) # type: ignore[arg-type] + + def _flatten_list( + elems: List[Any], key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + items = ( + _flatten_value(elem, key_path=key_path, strict=strict) + for elem in elems + if _is_primitive_value(elem) + ) + return functools.reduce(operator.concat, items, []) # type: ignore[arg-type] + + def _canonicalize(obj: Any, strict: bool = False) -> Any: + if strict: + value = str(obj) + if isinstance(obj, bool): + value = value.lower() + return value + return obj + + def transcode_request( # type: ignore[misc] + http_options: List[Dict[str, str]], + request: Any, + required_fields_default_values: Optional[Dict[str, Any]] = None, + rest_numeric_enums: bool = False, + ) -> Tuple[Dict[str, Any], Optional[str], Dict[str, Any]]: + pb_request = getattr(request, "_pb", request) + transcoded_request = path_template.transcode(http_options, pb_request) + + body_json = None + if transcoded_request.get("body") is not None: + body_json = json_format.MessageToJson( + transcoded_request["body"], + use_integers_for_enums=rest_numeric_enums, + ) + + query_params_json = {} + if transcoded_request.get("query_params") is not None: + query_params_json = json.loads( + json_format.MessageToJson( + transcoded_request["query_params"], + use_integers_for_enums=rest_numeric_enums, + ) + ) + + if required_fields_default_values: + for k, v in required_fields_default_values.items(): + if k not in query_params_json: + query_params_json[k] = v + + if rest_numeric_enums: + query_params_json["$alt"] = "json;enum-encoding=int" + + return transcoded_request, body_json, query_params_json diff --git a/packages/gapic-generator/tests/integration/goldens/credentials/tests/unit/gapic/credentials_v1/test_compat.py b/packages/gapic-generator/tests/integration/goldens/credentials/tests/unit/gapic/credentials_v1/test_compat.py new file mode 100644 index 000000000000..73adde90dcb1 --- /dev/null +++ b/packages/gapic-generator/tests/integration/goldens/credentials/tests/unit/gapic/credentials_v1/test_compat.py @@ -0,0 +1,364 @@ +# # Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import builtins +import importlib +import sys +from unittest import mock +import pytest +from google.auth.exceptions import MutualTLSChannelError +from google.protobuf import descriptor_pb2 + + +from google.iam.credentials_v1 import _compat + + +def test_compat_normal_import(): + assert _compat.setup_request_id is not None + + +def test_compat_fallback_implementations(): + orig_import = builtins.__import__ + + def custom_import(name, globals=None, locals=None, fromlist=(), level=0): + if name in ( + "google.api_core.universe", + "google.api_core.gapic_v1.config", + "google.api_core.gapic_v1.request", + "google.api_core.rest_helpers", + ): + raise ImportError(f"Mocked ImportError for {name}") + return orig_import(name, globals, locals, fromlist, level) + + with mock.patch("builtins.__import__", side_effect=custom_import): + fallback = importlib.reload(_compat) + + # get_default_mtls_endpoint tests + assert fallback.get_default_mtls_endpoint(None) is None + assert fallback.get_default_mtls_endpoint("") == "" + assert ( + fallback.get_default_mtls_endpoint("foo.googleapis.com") + == "foo.mtls.googleapis.com" + ) + assert ( + fallback.get_default_mtls_endpoint("foo.sandbox.googleapis.com") + == "foo.mtls.sandbox.googleapis.com" + ) + assert ( + fallback.get_default_mtls_endpoint("foo.mtls.googleapis.com") + == "foo.mtls.googleapis.com" + ) + assert ( + fallback.get_default_mtls_endpoint("custom.domain.com") + == "custom.domain.com" + ) + assert ( + fallback.get_default_mtls_endpoint(":::invalid-url:::") + == ":::invalid-url:::" + ) + + # get_api_endpoint tests + assert ( + fallback.get_api_endpoint( + "https://override.com", + None, + "googleapis.com", + "auto", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "https://override.com" + ) + assert ( + fallback.get_api_endpoint( + None, + lambda: (b"", b""), + "googleapis.com", + "always", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "mtls.com" + ) + assert ( + fallback.get_api_endpoint( + None, + lambda: (b"", b""), + "googleapis.com", + "auto", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "mtls.com" + ) + with pytest.raises(MutualTLSChannelError): + fallback.get_api_endpoint( + None, + lambda: (b"", b""), + "otheruniverse.com", + "always", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + assert ( + fallback.get_api_endpoint( + None, + None, + "googleapis.com", + "never", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "https://googleapis.com" + ) + + # get_universe_domain tests + assert ( + fallback.get_universe_domain("custom.com", None, "googleapis.com") + == "custom.com" + ) + assert ( + fallback.get_universe_domain(None, "env.com", "googleapis.com") == "env.com" + ) + assert ( + fallback.get_universe_domain(None, None, "googleapis.com") + == "googleapis.com" + ) + with pytest.raises(ValueError): + fallback.get_universe_domain(" ", None, "googleapis.com") + + # use_client_cert_effective tests + with mock.patch( + "google.auth.transport.mtls.should_use_client_cert", + return_value=True, + create=True, + ): + assert fallback.use_client_cert_effective() is True + + with mock.patch.object(fallback, "mtls", spec=object()): + with mock.patch.dict( + "os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"} + ): + assert fallback.use_client_cert_effective() is True + with mock.patch.dict( + "os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"} + ): + assert fallback.use_client_cert_effective() is False + with mock.patch.dict( + "os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "invalid"} + ): + with pytest.raises(ValueError): + fallback.use_client_cert_effective() + + # get_client_cert_source tests + cert_fn = lambda: (b"cert", b"key") + assert fallback.get_client_cert_source(cert_fn, True) == cert_fn + assert fallback.get_client_cert_source(None, False) is None + + with mock.patch( + "google.auth.transport.mtls.has_default_client_cert_source", + return_value=True, + create=True, + ): + with mock.patch( + "google.auth.transport.mtls.default_client_cert_source", + return_value=cert_fn, + create=True, + ): + assert fallback.get_client_cert_source(None, True) == cert_fn + + with mock.patch.object(fallback, "mtls", spec=object()): + with pytest.raises(ValueError): + fallback.get_client_cert_source(None, True) + + # read_environment_variables tests + with mock.patch.dict( + "os.environ", + { + "GOOGLE_API_USE_MTLS_ENDPOINT": "always", + "GOOGLE_CLOUD_UNIVERSE_DOMAIN": "myuniverse.com", + }, + ): + use_cert, use_mtls, universe = fallback.read_environment_variables() + assert use_mtls == "always" + assert universe == "myuniverse.com" + + with mock.patch.dict("os.environ", {"GOOGLE_API_USE_MTLS_ENDPOINT": "invalid"}): + with pytest.raises(MutualTLSChannelError): + fallback.read_environment_variables() + + # setup_request_id tests + fallback.setup_request_id(None, "request_id", False) + + d1 = {} + fallback.setup_request_id(d1, "request_id", is_proto3_optional=True) + assert "request_id" in d1 + + d1_existing = {"request_id": None} + fallback.setup_request_id(d1_existing, "request_id", is_proto3_optional=True) + assert d1_existing["request_id"] is not None + + d2 = {} + fallback.setup_request_id(d2, "request_id", is_proto3_optional=False) + assert "request_id" in d2 + + d3 = {"request_id": "existing"} + fallback.setup_request_id(d3, "request_id", is_proto3_optional=False) + assert d3["request_id"] == "existing" + + d4 = {"request_id": "val"} + fallback.setup_request_id(d4, "request_id", is_proto3_optional=True) + assert d4["request_id"] == "val" + + class DummyPopulated: + def __init__(self): + self.request_id = "val" + + p_existing = DummyPopulated() + fallback.setup_request_id(p_existing, "request_id", is_proto3_optional=False) + assert p_existing.request_id == "val" + + class NonOptPlain: + def __init__(self): + self.request_id = "" + + nop = NonOptPlain() + fallback.setup_request_id(nop, "request_id", is_proto3_optional=False) + assert nop.request_id != "" + + class DummyProto: + def __init__(self): + self.request_id = "" + self._has = False + + def HasField(self, name): + if not self._has: + return False + return True + + p1 = DummyProto() + fallback.setup_request_id(p1, "request_id", is_proto3_optional=True) + assert p1.request_id != "" + + class DummyWrapper: + def __init__(self): + self._pb = DummyProto() + + w1 = DummyWrapper() + fallback.setup_request_id(w1, "request_id", is_proto3_optional=True) + assert ( + getattr(w1, "request_id", None) is not None + or getattr(w1._pb, "request_id", None) != "" + ) + + class BadProto: + def HasField(self, name): + raise AttributeError() + + class BadWrapper: + def __init__(self): + self._pb = BadProto() + self.request_id = None + + bw = BadWrapper() + fallback.setup_request_id(bw, "request_id", is_proto3_optional=True) + assert bw.request_id is not None + + class SetProto: + def __init__(self): + self.request_id = "already_set" + + def HasField(self, name): + return True + + sp = SetProto() + fallback.setup_request_id(sp, "request_id", is_proto3_optional=True) + assert sp.request_id == "already_set" + + class SetProtoNonOpt: + def __init__(self): + self.request_id = "already_set" + + sp_non_opt = SetProtoNonOpt() + fallback.setup_request_id(sp_non_opt, "request_id", is_proto3_optional=False) + assert sp_non_opt.request_id == "already_set" + + class DummyPlain: + def __init__(self): + self.request_id = None + + pl1 = DummyPlain() + fallback.setup_request_id(pl1, "request_id", is_proto3_optional=True) + assert pl1.request_id is not None + + pl2 = DummyPlain() + fallback.setup_request_id(pl2, "request_id", is_proto3_optional=False) + assert pl2.request_id is not None + + # flatten_query_params tests + assert fallback.flatten_query_params(None) == [] + res = fallback.flatten_query_params({"a": "val1", "b": [1, 2], "c": True}) + assert ("a", "val1") in res + assert ("b", 1) in res + assert ("b", 2) in res + assert ("c", True) in res + + res_strict = fallback.flatten_query_params({"c": True}, strict=True) + assert ("c", "true") in res_strict + + assert fallback._canonicalize(True, strict=True) == "true" + assert fallback._canonicalize(True, strict=False) is True + assert fallback._canonicalize(123, strict=True) == "123" + assert fallback._canonicalize("str", strict=True) == "str" + assert fallback._canonicalize("str", strict=False) == "str" + assert fallback._is_primitive_value(None) is False + + with pytest.raises(TypeError): + fallback.flatten_query_params("invalid") + + with pytest.raises(ValueError): + fallback.flatten_query_params({"a": [{"nested": "dict"}]}) + + with pytest.raises(ValueError): + fallback._is_primitive_value([1, 2]) + + # transcode_request tests + dummy_req = descriptor_pb2.DescriptorProto() + http_opts = [{"method": "get", "uri": "/v1/test"}] + transcoded, body, query = fallback.transcode_request( + http_opts, + dummy_req, + required_fields_default_values={"non_existent_key": "val"}, + rest_numeric_enums=True, + ) + assert transcoded is not None + assert query.get("non_existent_key") == "val" + assert query.get("$alt") == "json;enum-encoding=int" + + transcoded2, body2, query2 = fallback.transcode_request( + http_opts, + dummy_req, + required_fields_default_values={"name": "override_name"}, + rest_numeric_enums=False, + ) + assert body2 is None + assert "$alt" not in query2 + + importlib.reload(_compat) diff --git a/packages/gapic-generator/tests/integration/goldens/eventarc/google/cloud/eventarc_v1/_compat.py b/packages/gapic-generator/tests/integration/goldens/eventarc/google/cloud/eventarc_v1/_compat.py new file mode 100644 index 000000000000..54c22cd7d09e --- /dev/null +++ b/packages/gapic-generator/tests/integration/goldens/eventarc/google/cloud/eventarc_v1/_compat.py @@ -0,0 +1,310 @@ +# # Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""A compatibility module for older versions of google-api-core.""" + +import functools +import json +import operator +import os +import re +import uuid +from typing import Any, Callable, Dict, List, Optional, Tuple + +from google.auth.exceptions import MutualTLSChannelError + + +try: + from google.api_core.universe import ( # type: ignore + get_default_mtls_endpoint, + get_api_endpoint, + get_universe_domain, + ) +except ImportError: # pragma: NO COVER + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Universe domain support was introduced in google-api-core 2.18.0. + # Remove these fallback definitions when google-api-core >= 2.18.0 becomes the minimum required version in generated client setup dependencies. + def get_default_mtls_endpoint(api_endpoint: Optional[str]) -> Optional[str]: + """Converts api endpoint to mTLS endpoint.""" + if not api_endpoint: + return api_endpoint + + mtls_endpoint_re = re.compile( + r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" + ) + + m = mtls_endpoint_re.match(api_endpoint) + if m is None: + # Could not parse api_endpoint; return as-is. + return api_endpoint + + name, mtls, sandbox, googledomain = m.groups() + if mtls or not googledomain: + return api_endpoint + + if sandbox: + return api_endpoint.replace( + "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" + ) + + return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") + + def get_api_endpoint( # type: ignore[misc] + api_override: Optional[str], + client_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]], + universe_domain: str, + use_mtls_endpoint: str, + default_universe: str, + default_mtls_endpoint: Optional[str], + default_endpoint_template: str, + ) -> str: + """Return the API endpoint used by the client.""" + api_endpoint: Optional[str] = None + if api_override is not None: + api_endpoint = api_override + elif use_mtls_endpoint == "always" or ( + use_mtls_endpoint == "auto" and client_cert_source + ): + if universe_domain != default_universe: + raise MutualTLSChannelError( + f"mTLS is not supported in any universe other than {default_universe}." + ) + api_endpoint = default_mtls_endpoint + else: + api_endpoint = default_endpoint_template.format( + UNIVERSE_DOMAIN=universe_domain + ) + return api_endpoint # type: ignore[return-value] + + def get_universe_domain( # type: ignore[misc] + *potential_universes: Optional[str], + default_universe: str = "googleapis.com", + ) -> str: + """Return the universe domain used by the client.""" + resolved = next( + (x.strip() for x in potential_universes if x is not None), + default_universe, + ) + if not resolved: + raise ValueError("Universe Domain cannot be an empty string.") + return resolved + + +try: + from google.api_core.gapic_v1.config import ( # type: ignore + use_client_cert_effective, + get_client_cert_source, + read_environment_variables, + ) +except ImportError: # pragma: NO COVER + from google.auth.transport import mtls # type: ignore + + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Remove these fallbacks when google-api-core >= 2.18.0 is the minimum required version. + + def use_client_cert_effective() -> bool: + """Returns whether client certificate should be used for mTLS.""" + if hasattr(mtls, "should_use_client_cert"): + return mtls.should_use_client_cert() + else: + use_client_cert_str = os.getenv( + "GOOGLE_API_USE_CLIENT_CERTIFICATE", "false" + ).lower() + if use_client_cert_str not in ("true", "false"): + raise ValueError( + "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be" + " either `true` or `false`" + ) + return use_client_cert_str == "true" + + def get_client_cert_source( + provided_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]], + use_cert_flag: bool, + ) -> Optional[Callable[[], Tuple[bytes, bytes]]]: + """Return the client cert source to be used by the client.""" + client_cert_source = None + if use_cert_flag: + if provided_cert_source: + client_cert_source = provided_cert_source + elif ( + hasattr(mtls, "has_default_client_cert_source") + and mtls.has_default_client_cert_source() + ): + client_cert_source = mtls.default_client_cert_source() + else: + raise ValueError( + "Client certificate is required for mTLS, but no client certificate source was provided or found." + ) + return client_cert_source + + def read_environment_variables() -> Tuple[bool, str, Optional[str]]: + """Returns the environment variables used by the client.""" + use_client_cert = use_client_cert_effective() + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower() + universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN") + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError( + "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` " + "must be `never`, `auto` or `always`" + ) + return use_client_cert, use_mtls_endpoint, universe_domain_env + + +try: + from google.api_core.gapic_v1.request import setup_request_id # type: ignore +except ImportError: # pragma: NO COVER + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Request ID setup helper was introduced in google-api-core 2.26.0. + # Remove this fallback definition when google-api-core >= 2.26.0 becomes the minimum required version in generated client setup dependencies. + def setup_request_id( + request: Any, field_name: str, is_proto3_optional: bool + ) -> None: + """Populate a UUID4 field in the request if it is not already set. + + Args: + request (Union[google.protobuf.message.Message, dict]): The request object. + field_name (str): The name of the field to populate. + is_proto3_optional (bool): Whether the field is proto3 optional. + """ + if request is None: + return + + request_id_val = str(uuid.uuid4()) + + if isinstance(request, dict): + if is_proto3_optional: + if field_name not in request or request[field_name] is None: + request[field_name] = request_id_val + elif not request.get(field_name): + request[field_name] = request_id_val + return + + if is_proto3_optional: + try: + # Pure protobuf messages + if not request.HasField(field_name): + setattr(request, field_name, request_id_val) + except (AttributeError, ValueError): + # Proto-plus messages or other objects + if hasattr(request, "_pb"): + try: + if not request._pb.HasField(field_name): + setattr(request, field_name, request_id_val) + return + except (AttributeError, ValueError): + pass + if getattr(request, field_name, None) is None: + setattr(request, field_name, request_id_val) + else: + if not getattr(request, field_name, None): + setattr(request, field_name, request_id_val) + + +try: + from google.api_core.rest_helpers import ( # type: ignore + flatten_query_params, + transcode_request, + ) +except ImportError: # pragma: NO COVER + # TODO: Remove these fallbacks when google-api-core >= 2.18.0 is the minimum required version. + from google.protobuf import json_format # type: ignore + from google.api_core import path_template # type: ignore + + def flatten_query_params(obj: Any, strict: bool = False) -> List[Tuple[str, Any]]: # type: ignore[misc] + if obj is not None and not isinstance(obj, dict): + raise TypeError("flatten_query_params must be called with dict object") + return _flatten(obj, key_path=[], strict=strict) + + def _flatten( + obj: Any, key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + if obj is None: + return [] + if isinstance(obj, dict): + return _flatten_dict(obj, key_path=key_path, strict=strict) + if isinstance(obj, list): + return _flatten_list(obj, key_path=key_path, strict=strict) + return _flatten_value(obj, key_path=key_path, strict=strict) + + def _is_primitive_value(obj: Any) -> bool: + if obj is None: + return False + if isinstance(obj, (list, dict)): + raise ValueError("query params may not contain repeated dicts or lists") + return True + + def _flatten_value( + obj: Any, key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + return [(".".join(key_path), _canonicalize(obj, strict=strict))] + + def _flatten_dict( + obj: Dict[str, Any], key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + items = ( + _flatten(value, key_path=key_path + [key], strict=strict) + for key, value in obj.items() + ) + return functools.reduce(operator.concat, items, []) # type: ignore[arg-type] + + def _flatten_list( + elems: List[Any], key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + items = ( + _flatten_value(elem, key_path=key_path, strict=strict) + for elem in elems + if _is_primitive_value(elem) + ) + return functools.reduce(operator.concat, items, []) # type: ignore[arg-type] + + def _canonicalize(obj: Any, strict: bool = False) -> Any: + if strict: + value = str(obj) + if isinstance(obj, bool): + value = value.lower() + return value + return obj + + def transcode_request( # type: ignore[misc] + http_options: List[Dict[str, str]], + request: Any, + required_fields_default_values: Optional[Dict[str, Any]] = None, + rest_numeric_enums: bool = False, + ) -> Tuple[Dict[str, Any], Optional[str], Dict[str, Any]]: + pb_request = getattr(request, "_pb", request) + transcoded_request = path_template.transcode(http_options, pb_request) + + body_json = None + if transcoded_request.get("body") is not None: + body_json = json_format.MessageToJson( + transcoded_request["body"], + use_integers_for_enums=rest_numeric_enums, + ) + + query_params_json = {} + if transcoded_request.get("query_params") is not None: + query_params_json = json.loads( + json_format.MessageToJson( + transcoded_request["query_params"], + use_integers_for_enums=rest_numeric_enums, + ) + ) + + if required_fields_default_values: + for k, v in required_fields_default_values.items(): + if k not in query_params_json: + query_params_json[k] = v + + if rest_numeric_enums: + query_params_json["$alt"] = "json;enum-encoding=int" + + return transcoded_request, body_json, query_params_json diff --git a/packages/gapic-generator/tests/integration/goldens/eventarc/tests/unit/gapic/eventarc_v1/test_compat.py b/packages/gapic-generator/tests/integration/goldens/eventarc/tests/unit/gapic/eventarc_v1/test_compat.py new file mode 100644 index 000000000000..7b5ff1e41920 --- /dev/null +++ b/packages/gapic-generator/tests/integration/goldens/eventarc/tests/unit/gapic/eventarc_v1/test_compat.py @@ -0,0 +1,364 @@ +# # Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import builtins +import importlib +import sys +from unittest import mock +import pytest +from google.auth.exceptions import MutualTLSChannelError +from google.protobuf import descriptor_pb2 + + +from google.cloud.eventarc_v1 import _compat + + +def test_compat_normal_import(): + assert _compat.setup_request_id is not None + + +def test_compat_fallback_implementations(): + orig_import = builtins.__import__ + + def custom_import(name, globals=None, locals=None, fromlist=(), level=0): + if name in ( + "google.api_core.universe", + "google.api_core.gapic_v1.config", + "google.api_core.gapic_v1.request", + "google.api_core.rest_helpers", + ): + raise ImportError(f"Mocked ImportError for {name}") + return orig_import(name, globals, locals, fromlist, level) + + with mock.patch("builtins.__import__", side_effect=custom_import): + fallback = importlib.reload(_compat) + + # get_default_mtls_endpoint tests + assert fallback.get_default_mtls_endpoint(None) is None + assert fallback.get_default_mtls_endpoint("") == "" + assert ( + fallback.get_default_mtls_endpoint("foo.googleapis.com") + == "foo.mtls.googleapis.com" + ) + assert ( + fallback.get_default_mtls_endpoint("foo.sandbox.googleapis.com") + == "foo.mtls.sandbox.googleapis.com" + ) + assert ( + fallback.get_default_mtls_endpoint("foo.mtls.googleapis.com") + == "foo.mtls.googleapis.com" + ) + assert ( + fallback.get_default_mtls_endpoint("custom.domain.com") + == "custom.domain.com" + ) + assert ( + fallback.get_default_mtls_endpoint(":::invalid-url:::") + == ":::invalid-url:::" + ) + + # get_api_endpoint tests + assert ( + fallback.get_api_endpoint( + "https://override.com", + None, + "googleapis.com", + "auto", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "https://override.com" + ) + assert ( + fallback.get_api_endpoint( + None, + lambda: (b"", b""), + "googleapis.com", + "always", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "mtls.com" + ) + assert ( + fallback.get_api_endpoint( + None, + lambda: (b"", b""), + "googleapis.com", + "auto", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "mtls.com" + ) + with pytest.raises(MutualTLSChannelError): + fallback.get_api_endpoint( + None, + lambda: (b"", b""), + "otheruniverse.com", + "always", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + assert ( + fallback.get_api_endpoint( + None, + None, + "googleapis.com", + "never", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "https://googleapis.com" + ) + + # get_universe_domain tests + assert ( + fallback.get_universe_domain("custom.com", None, "googleapis.com") + == "custom.com" + ) + assert ( + fallback.get_universe_domain(None, "env.com", "googleapis.com") == "env.com" + ) + assert ( + fallback.get_universe_domain(None, None, "googleapis.com") + == "googleapis.com" + ) + with pytest.raises(ValueError): + fallback.get_universe_domain(" ", None, "googleapis.com") + + # use_client_cert_effective tests + with mock.patch( + "google.auth.transport.mtls.should_use_client_cert", + return_value=True, + create=True, + ): + assert fallback.use_client_cert_effective() is True + + with mock.patch.object(fallback, "mtls", spec=object()): + with mock.patch.dict( + "os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"} + ): + assert fallback.use_client_cert_effective() is True + with mock.patch.dict( + "os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"} + ): + assert fallback.use_client_cert_effective() is False + with mock.patch.dict( + "os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "invalid"} + ): + with pytest.raises(ValueError): + fallback.use_client_cert_effective() + + # get_client_cert_source tests + cert_fn = lambda: (b"cert", b"key") + assert fallback.get_client_cert_source(cert_fn, True) == cert_fn + assert fallback.get_client_cert_source(None, False) is None + + with mock.patch( + "google.auth.transport.mtls.has_default_client_cert_source", + return_value=True, + create=True, + ): + with mock.patch( + "google.auth.transport.mtls.default_client_cert_source", + return_value=cert_fn, + create=True, + ): + assert fallback.get_client_cert_source(None, True) == cert_fn + + with mock.patch.object(fallback, "mtls", spec=object()): + with pytest.raises(ValueError): + fallback.get_client_cert_source(None, True) + + # read_environment_variables tests + with mock.patch.dict( + "os.environ", + { + "GOOGLE_API_USE_MTLS_ENDPOINT": "always", + "GOOGLE_CLOUD_UNIVERSE_DOMAIN": "myuniverse.com", + }, + ): + use_cert, use_mtls, universe = fallback.read_environment_variables() + assert use_mtls == "always" + assert universe == "myuniverse.com" + + with mock.patch.dict("os.environ", {"GOOGLE_API_USE_MTLS_ENDPOINT": "invalid"}): + with pytest.raises(MutualTLSChannelError): + fallback.read_environment_variables() + + # setup_request_id tests + fallback.setup_request_id(None, "request_id", False) + + d1 = {} + fallback.setup_request_id(d1, "request_id", is_proto3_optional=True) + assert "request_id" in d1 + + d1_existing = {"request_id": None} + fallback.setup_request_id(d1_existing, "request_id", is_proto3_optional=True) + assert d1_existing["request_id"] is not None + + d2 = {} + fallback.setup_request_id(d2, "request_id", is_proto3_optional=False) + assert "request_id" in d2 + + d3 = {"request_id": "existing"} + fallback.setup_request_id(d3, "request_id", is_proto3_optional=False) + assert d3["request_id"] == "existing" + + d4 = {"request_id": "val"} + fallback.setup_request_id(d4, "request_id", is_proto3_optional=True) + assert d4["request_id"] == "val" + + class DummyPopulated: + def __init__(self): + self.request_id = "val" + + p_existing = DummyPopulated() + fallback.setup_request_id(p_existing, "request_id", is_proto3_optional=False) + assert p_existing.request_id == "val" + + class NonOptPlain: + def __init__(self): + self.request_id = "" + + nop = NonOptPlain() + fallback.setup_request_id(nop, "request_id", is_proto3_optional=False) + assert nop.request_id != "" + + class DummyProto: + def __init__(self): + self.request_id = "" + self._has = False + + def HasField(self, name): + if not self._has: + return False + return True + + p1 = DummyProto() + fallback.setup_request_id(p1, "request_id", is_proto3_optional=True) + assert p1.request_id != "" + + class DummyWrapper: + def __init__(self): + self._pb = DummyProto() + + w1 = DummyWrapper() + fallback.setup_request_id(w1, "request_id", is_proto3_optional=True) + assert ( + getattr(w1, "request_id", None) is not None + or getattr(w1._pb, "request_id", None) != "" + ) + + class BadProto: + def HasField(self, name): + raise AttributeError() + + class BadWrapper: + def __init__(self): + self._pb = BadProto() + self.request_id = None + + bw = BadWrapper() + fallback.setup_request_id(bw, "request_id", is_proto3_optional=True) + assert bw.request_id is not None + + class SetProto: + def __init__(self): + self.request_id = "already_set" + + def HasField(self, name): + return True + + sp = SetProto() + fallback.setup_request_id(sp, "request_id", is_proto3_optional=True) + assert sp.request_id == "already_set" + + class SetProtoNonOpt: + def __init__(self): + self.request_id = "already_set" + + sp_non_opt = SetProtoNonOpt() + fallback.setup_request_id(sp_non_opt, "request_id", is_proto3_optional=False) + assert sp_non_opt.request_id == "already_set" + + class DummyPlain: + def __init__(self): + self.request_id = None + + pl1 = DummyPlain() + fallback.setup_request_id(pl1, "request_id", is_proto3_optional=True) + assert pl1.request_id is not None + + pl2 = DummyPlain() + fallback.setup_request_id(pl2, "request_id", is_proto3_optional=False) + assert pl2.request_id is not None + + # flatten_query_params tests + assert fallback.flatten_query_params(None) == [] + res = fallback.flatten_query_params({"a": "val1", "b": [1, 2], "c": True}) + assert ("a", "val1") in res + assert ("b", 1) in res + assert ("b", 2) in res + assert ("c", True) in res + + res_strict = fallback.flatten_query_params({"c": True}, strict=True) + assert ("c", "true") in res_strict + + assert fallback._canonicalize(True, strict=True) == "true" + assert fallback._canonicalize(True, strict=False) is True + assert fallback._canonicalize(123, strict=True) == "123" + assert fallback._canonicalize("str", strict=True) == "str" + assert fallback._canonicalize("str", strict=False) == "str" + assert fallback._is_primitive_value(None) is False + + with pytest.raises(TypeError): + fallback.flatten_query_params("invalid") + + with pytest.raises(ValueError): + fallback.flatten_query_params({"a": [{"nested": "dict"}]}) + + with pytest.raises(ValueError): + fallback._is_primitive_value([1, 2]) + + # transcode_request tests + dummy_req = descriptor_pb2.DescriptorProto() + http_opts = [{"method": "get", "uri": "/v1/test"}] + transcoded, body, query = fallback.transcode_request( + http_opts, + dummy_req, + required_fields_default_values={"non_existent_key": "val"}, + rest_numeric_enums=True, + ) + assert transcoded is not None + assert query.get("non_existent_key") == "val" + assert query.get("$alt") == "json;enum-encoding=int" + + transcoded2, body2, query2 = fallback.transcode_request( + http_opts, + dummy_req, + required_fields_default_values={"name": "override_name"}, + rest_numeric_enums=False, + ) + assert body2 is None + assert "$alt" not in query2 + + importlib.reload(_compat) diff --git a/packages/gapic-generator/tests/integration/goldens/logging/google/cloud/logging_v2/_compat.py b/packages/gapic-generator/tests/integration/goldens/logging/google/cloud/logging_v2/_compat.py new file mode 100644 index 000000000000..54c22cd7d09e --- /dev/null +++ b/packages/gapic-generator/tests/integration/goldens/logging/google/cloud/logging_v2/_compat.py @@ -0,0 +1,310 @@ +# # Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""A compatibility module for older versions of google-api-core.""" + +import functools +import json +import operator +import os +import re +import uuid +from typing import Any, Callable, Dict, List, Optional, Tuple + +from google.auth.exceptions import MutualTLSChannelError + + +try: + from google.api_core.universe import ( # type: ignore + get_default_mtls_endpoint, + get_api_endpoint, + get_universe_domain, + ) +except ImportError: # pragma: NO COVER + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Universe domain support was introduced in google-api-core 2.18.0. + # Remove these fallback definitions when google-api-core >= 2.18.0 becomes the minimum required version in generated client setup dependencies. + def get_default_mtls_endpoint(api_endpoint: Optional[str]) -> Optional[str]: + """Converts api endpoint to mTLS endpoint.""" + if not api_endpoint: + return api_endpoint + + mtls_endpoint_re = re.compile( + r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" + ) + + m = mtls_endpoint_re.match(api_endpoint) + if m is None: + # Could not parse api_endpoint; return as-is. + return api_endpoint + + name, mtls, sandbox, googledomain = m.groups() + if mtls or not googledomain: + return api_endpoint + + if sandbox: + return api_endpoint.replace( + "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" + ) + + return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") + + def get_api_endpoint( # type: ignore[misc] + api_override: Optional[str], + client_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]], + universe_domain: str, + use_mtls_endpoint: str, + default_universe: str, + default_mtls_endpoint: Optional[str], + default_endpoint_template: str, + ) -> str: + """Return the API endpoint used by the client.""" + api_endpoint: Optional[str] = None + if api_override is not None: + api_endpoint = api_override + elif use_mtls_endpoint == "always" or ( + use_mtls_endpoint == "auto" and client_cert_source + ): + if universe_domain != default_universe: + raise MutualTLSChannelError( + f"mTLS is not supported in any universe other than {default_universe}." + ) + api_endpoint = default_mtls_endpoint + else: + api_endpoint = default_endpoint_template.format( + UNIVERSE_DOMAIN=universe_domain + ) + return api_endpoint # type: ignore[return-value] + + def get_universe_domain( # type: ignore[misc] + *potential_universes: Optional[str], + default_universe: str = "googleapis.com", + ) -> str: + """Return the universe domain used by the client.""" + resolved = next( + (x.strip() for x in potential_universes if x is not None), + default_universe, + ) + if not resolved: + raise ValueError("Universe Domain cannot be an empty string.") + return resolved + + +try: + from google.api_core.gapic_v1.config import ( # type: ignore + use_client_cert_effective, + get_client_cert_source, + read_environment_variables, + ) +except ImportError: # pragma: NO COVER + from google.auth.transport import mtls # type: ignore + + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Remove these fallbacks when google-api-core >= 2.18.0 is the minimum required version. + + def use_client_cert_effective() -> bool: + """Returns whether client certificate should be used for mTLS.""" + if hasattr(mtls, "should_use_client_cert"): + return mtls.should_use_client_cert() + else: + use_client_cert_str = os.getenv( + "GOOGLE_API_USE_CLIENT_CERTIFICATE", "false" + ).lower() + if use_client_cert_str not in ("true", "false"): + raise ValueError( + "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be" + " either `true` or `false`" + ) + return use_client_cert_str == "true" + + def get_client_cert_source( + provided_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]], + use_cert_flag: bool, + ) -> Optional[Callable[[], Tuple[bytes, bytes]]]: + """Return the client cert source to be used by the client.""" + client_cert_source = None + if use_cert_flag: + if provided_cert_source: + client_cert_source = provided_cert_source + elif ( + hasattr(mtls, "has_default_client_cert_source") + and mtls.has_default_client_cert_source() + ): + client_cert_source = mtls.default_client_cert_source() + else: + raise ValueError( + "Client certificate is required for mTLS, but no client certificate source was provided or found." + ) + return client_cert_source + + def read_environment_variables() -> Tuple[bool, str, Optional[str]]: + """Returns the environment variables used by the client.""" + use_client_cert = use_client_cert_effective() + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower() + universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN") + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError( + "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` " + "must be `never`, `auto` or `always`" + ) + return use_client_cert, use_mtls_endpoint, universe_domain_env + + +try: + from google.api_core.gapic_v1.request import setup_request_id # type: ignore +except ImportError: # pragma: NO COVER + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Request ID setup helper was introduced in google-api-core 2.26.0. + # Remove this fallback definition when google-api-core >= 2.26.0 becomes the minimum required version in generated client setup dependencies. + def setup_request_id( + request: Any, field_name: str, is_proto3_optional: bool + ) -> None: + """Populate a UUID4 field in the request if it is not already set. + + Args: + request (Union[google.protobuf.message.Message, dict]): The request object. + field_name (str): The name of the field to populate. + is_proto3_optional (bool): Whether the field is proto3 optional. + """ + if request is None: + return + + request_id_val = str(uuid.uuid4()) + + if isinstance(request, dict): + if is_proto3_optional: + if field_name not in request or request[field_name] is None: + request[field_name] = request_id_val + elif not request.get(field_name): + request[field_name] = request_id_val + return + + if is_proto3_optional: + try: + # Pure protobuf messages + if not request.HasField(field_name): + setattr(request, field_name, request_id_val) + except (AttributeError, ValueError): + # Proto-plus messages or other objects + if hasattr(request, "_pb"): + try: + if not request._pb.HasField(field_name): + setattr(request, field_name, request_id_val) + return + except (AttributeError, ValueError): + pass + if getattr(request, field_name, None) is None: + setattr(request, field_name, request_id_val) + else: + if not getattr(request, field_name, None): + setattr(request, field_name, request_id_val) + + +try: + from google.api_core.rest_helpers import ( # type: ignore + flatten_query_params, + transcode_request, + ) +except ImportError: # pragma: NO COVER + # TODO: Remove these fallbacks when google-api-core >= 2.18.0 is the minimum required version. + from google.protobuf import json_format # type: ignore + from google.api_core import path_template # type: ignore + + def flatten_query_params(obj: Any, strict: bool = False) -> List[Tuple[str, Any]]: # type: ignore[misc] + if obj is not None and not isinstance(obj, dict): + raise TypeError("flatten_query_params must be called with dict object") + return _flatten(obj, key_path=[], strict=strict) + + def _flatten( + obj: Any, key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + if obj is None: + return [] + if isinstance(obj, dict): + return _flatten_dict(obj, key_path=key_path, strict=strict) + if isinstance(obj, list): + return _flatten_list(obj, key_path=key_path, strict=strict) + return _flatten_value(obj, key_path=key_path, strict=strict) + + def _is_primitive_value(obj: Any) -> bool: + if obj is None: + return False + if isinstance(obj, (list, dict)): + raise ValueError("query params may not contain repeated dicts or lists") + return True + + def _flatten_value( + obj: Any, key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + return [(".".join(key_path), _canonicalize(obj, strict=strict))] + + def _flatten_dict( + obj: Dict[str, Any], key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + items = ( + _flatten(value, key_path=key_path + [key], strict=strict) + for key, value in obj.items() + ) + return functools.reduce(operator.concat, items, []) # type: ignore[arg-type] + + def _flatten_list( + elems: List[Any], key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + items = ( + _flatten_value(elem, key_path=key_path, strict=strict) + for elem in elems + if _is_primitive_value(elem) + ) + return functools.reduce(operator.concat, items, []) # type: ignore[arg-type] + + def _canonicalize(obj: Any, strict: bool = False) -> Any: + if strict: + value = str(obj) + if isinstance(obj, bool): + value = value.lower() + return value + return obj + + def transcode_request( # type: ignore[misc] + http_options: List[Dict[str, str]], + request: Any, + required_fields_default_values: Optional[Dict[str, Any]] = None, + rest_numeric_enums: bool = False, + ) -> Tuple[Dict[str, Any], Optional[str], Dict[str, Any]]: + pb_request = getattr(request, "_pb", request) + transcoded_request = path_template.transcode(http_options, pb_request) + + body_json = None + if transcoded_request.get("body") is not None: + body_json = json_format.MessageToJson( + transcoded_request["body"], + use_integers_for_enums=rest_numeric_enums, + ) + + query_params_json = {} + if transcoded_request.get("query_params") is not None: + query_params_json = json.loads( + json_format.MessageToJson( + transcoded_request["query_params"], + use_integers_for_enums=rest_numeric_enums, + ) + ) + + if required_fields_default_values: + for k, v in required_fields_default_values.items(): + if k not in query_params_json: + query_params_json[k] = v + + if rest_numeric_enums: + query_params_json["$alt"] = "json;enum-encoding=int" + + return transcoded_request, body_json, query_params_json diff --git a/packages/gapic-generator/tests/integration/goldens/logging/tests/unit/gapic/logging_v2/test_compat.py b/packages/gapic-generator/tests/integration/goldens/logging/tests/unit/gapic/logging_v2/test_compat.py new file mode 100644 index 000000000000..716e1ac126fd --- /dev/null +++ b/packages/gapic-generator/tests/integration/goldens/logging/tests/unit/gapic/logging_v2/test_compat.py @@ -0,0 +1,364 @@ +# # Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import builtins +import importlib +import sys +from unittest import mock +import pytest +from google.auth.exceptions import MutualTLSChannelError +from google.protobuf import descriptor_pb2 + + +from google.cloud.logging_v2 import _compat + + +def test_compat_normal_import(): + assert _compat.setup_request_id is not None + + +def test_compat_fallback_implementations(): + orig_import = builtins.__import__ + + def custom_import(name, globals=None, locals=None, fromlist=(), level=0): + if name in ( + "google.api_core.universe", + "google.api_core.gapic_v1.config", + "google.api_core.gapic_v1.request", + "google.api_core.rest_helpers", + ): + raise ImportError(f"Mocked ImportError for {name}") + return orig_import(name, globals, locals, fromlist, level) + + with mock.patch("builtins.__import__", side_effect=custom_import): + fallback = importlib.reload(_compat) + + # get_default_mtls_endpoint tests + assert fallback.get_default_mtls_endpoint(None) is None + assert fallback.get_default_mtls_endpoint("") == "" + assert ( + fallback.get_default_mtls_endpoint("foo.googleapis.com") + == "foo.mtls.googleapis.com" + ) + assert ( + fallback.get_default_mtls_endpoint("foo.sandbox.googleapis.com") + == "foo.mtls.sandbox.googleapis.com" + ) + assert ( + fallback.get_default_mtls_endpoint("foo.mtls.googleapis.com") + == "foo.mtls.googleapis.com" + ) + assert ( + fallback.get_default_mtls_endpoint("custom.domain.com") + == "custom.domain.com" + ) + assert ( + fallback.get_default_mtls_endpoint(":::invalid-url:::") + == ":::invalid-url:::" + ) + + # get_api_endpoint tests + assert ( + fallback.get_api_endpoint( + "https://override.com", + None, + "googleapis.com", + "auto", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "https://override.com" + ) + assert ( + fallback.get_api_endpoint( + None, + lambda: (b"", b""), + "googleapis.com", + "always", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "mtls.com" + ) + assert ( + fallback.get_api_endpoint( + None, + lambda: (b"", b""), + "googleapis.com", + "auto", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "mtls.com" + ) + with pytest.raises(MutualTLSChannelError): + fallback.get_api_endpoint( + None, + lambda: (b"", b""), + "otheruniverse.com", + "always", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + assert ( + fallback.get_api_endpoint( + None, + None, + "googleapis.com", + "never", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "https://googleapis.com" + ) + + # get_universe_domain tests + assert ( + fallback.get_universe_domain("custom.com", None, "googleapis.com") + == "custom.com" + ) + assert ( + fallback.get_universe_domain(None, "env.com", "googleapis.com") == "env.com" + ) + assert ( + fallback.get_universe_domain(None, None, "googleapis.com") + == "googleapis.com" + ) + with pytest.raises(ValueError): + fallback.get_universe_domain(" ", None, "googleapis.com") + + # use_client_cert_effective tests + with mock.patch( + "google.auth.transport.mtls.should_use_client_cert", + return_value=True, + create=True, + ): + assert fallback.use_client_cert_effective() is True + + with mock.patch.object(fallback, "mtls", spec=object()): + with mock.patch.dict( + "os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"} + ): + assert fallback.use_client_cert_effective() is True + with mock.patch.dict( + "os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"} + ): + assert fallback.use_client_cert_effective() is False + with mock.patch.dict( + "os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "invalid"} + ): + with pytest.raises(ValueError): + fallback.use_client_cert_effective() + + # get_client_cert_source tests + cert_fn = lambda: (b"cert", b"key") + assert fallback.get_client_cert_source(cert_fn, True) == cert_fn + assert fallback.get_client_cert_source(None, False) is None + + with mock.patch( + "google.auth.transport.mtls.has_default_client_cert_source", + return_value=True, + create=True, + ): + with mock.patch( + "google.auth.transport.mtls.default_client_cert_source", + return_value=cert_fn, + create=True, + ): + assert fallback.get_client_cert_source(None, True) == cert_fn + + with mock.patch.object(fallback, "mtls", spec=object()): + with pytest.raises(ValueError): + fallback.get_client_cert_source(None, True) + + # read_environment_variables tests + with mock.patch.dict( + "os.environ", + { + "GOOGLE_API_USE_MTLS_ENDPOINT": "always", + "GOOGLE_CLOUD_UNIVERSE_DOMAIN": "myuniverse.com", + }, + ): + use_cert, use_mtls, universe = fallback.read_environment_variables() + assert use_mtls == "always" + assert universe == "myuniverse.com" + + with mock.patch.dict("os.environ", {"GOOGLE_API_USE_MTLS_ENDPOINT": "invalid"}): + with pytest.raises(MutualTLSChannelError): + fallback.read_environment_variables() + + # setup_request_id tests + fallback.setup_request_id(None, "request_id", False) + + d1 = {} + fallback.setup_request_id(d1, "request_id", is_proto3_optional=True) + assert "request_id" in d1 + + d1_existing = {"request_id": None} + fallback.setup_request_id(d1_existing, "request_id", is_proto3_optional=True) + assert d1_existing["request_id"] is not None + + d2 = {} + fallback.setup_request_id(d2, "request_id", is_proto3_optional=False) + assert "request_id" in d2 + + d3 = {"request_id": "existing"} + fallback.setup_request_id(d3, "request_id", is_proto3_optional=False) + assert d3["request_id"] == "existing" + + d4 = {"request_id": "val"} + fallback.setup_request_id(d4, "request_id", is_proto3_optional=True) + assert d4["request_id"] == "val" + + class DummyPopulated: + def __init__(self): + self.request_id = "val" + + p_existing = DummyPopulated() + fallback.setup_request_id(p_existing, "request_id", is_proto3_optional=False) + assert p_existing.request_id == "val" + + class NonOptPlain: + def __init__(self): + self.request_id = "" + + nop = NonOptPlain() + fallback.setup_request_id(nop, "request_id", is_proto3_optional=False) + assert nop.request_id != "" + + class DummyProto: + def __init__(self): + self.request_id = "" + self._has = False + + def HasField(self, name): + if not self._has: + return False + return True + + p1 = DummyProto() + fallback.setup_request_id(p1, "request_id", is_proto3_optional=True) + assert p1.request_id != "" + + class DummyWrapper: + def __init__(self): + self._pb = DummyProto() + + w1 = DummyWrapper() + fallback.setup_request_id(w1, "request_id", is_proto3_optional=True) + assert ( + getattr(w1, "request_id", None) is not None + or getattr(w1._pb, "request_id", None) != "" + ) + + class BadProto: + def HasField(self, name): + raise AttributeError() + + class BadWrapper: + def __init__(self): + self._pb = BadProto() + self.request_id = None + + bw = BadWrapper() + fallback.setup_request_id(bw, "request_id", is_proto3_optional=True) + assert bw.request_id is not None + + class SetProto: + def __init__(self): + self.request_id = "already_set" + + def HasField(self, name): + return True + + sp = SetProto() + fallback.setup_request_id(sp, "request_id", is_proto3_optional=True) + assert sp.request_id == "already_set" + + class SetProtoNonOpt: + def __init__(self): + self.request_id = "already_set" + + sp_non_opt = SetProtoNonOpt() + fallback.setup_request_id(sp_non_opt, "request_id", is_proto3_optional=False) + assert sp_non_opt.request_id == "already_set" + + class DummyPlain: + def __init__(self): + self.request_id = None + + pl1 = DummyPlain() + fallback.setup_request_id(pl1, "request_id", is_proto3_optional=True) + assert pl1.request_id is not None + + pl2 = DummyPlain() + fallback.setup_request_id(pl2, "request_id", is_proto3_optional=False) + assert pl2.request_id is not None + + # flatten_query_params tests + assert fallback.flatten_query_params(None) == [] + res = fallback.flatten_query_params({"a": "val1", "b": [1, 2], "c": True}) + assert ("a", "val1") in res + assert ("b", 1) in res + assert ("b", 2) in res + assert ("c", True) in res + + res_strict = fallback.flatten_query_params({"c": True}, strict=True) + assert ("c", "true") in res_strict + + assert fallback._canonicalize(True, strict=True) == "true" + assert fallback._canonicalize(True, strict=False) is True + assert fallback._canonicalize(123, strict=True) == "123" + assert fallback._canonicalize("str", strict=True) == "str" + assert fallback._canonicalize("str", strict=False) == "str" + assert fallback._is_primitive_value(None) is False + + with pytest.raises(TypeError): + fallback.flatten_query_params("invalid") + + with pytest.raises(ValueError): + fallback.flatten_query_params({"a": [{"nested": "dict"}]}) + + with pytest.raises(ValueError): + fallback._is_primitive_value([1, 2]) + + # transcode_request tests + dummy_req = descriptor_pb2.DescriptorProto() + http_opts = [{"method": "get", "uri": "/v1/test"}] + transcoded, body, query = fallback.transcode_request( + http_opts, + dummy_req, + required_fields_default_values={"non_existent_key": "val"}, + rest_numeric_enums=True, + ) + assert transcoded is not None + assert query.get("non_existent_key") == "val" + assert query.get("$alt") == "json;enum-encoding=int" + + transcoded2, body2, query2 = fallback.transcode_request( + http_opts, + dummy_req, + required_fields_default_values={"name": "override_name"}, + rest_numeric_enums=False, + ) + assert body2 is None + assert "$alt" not in query2 + + importlib.reload(_compat) diff --git a/packages/gapic-generator/tests/integration/goldens/logging_internal/google/cloud/logging_v2/_compat.py b/packages/gapic-generator/tests/integration/goldens/logging_internal/google/cloud/logging_v2/_compat.py new file mode 100644 index 000000000000..54c22cd7d09e --- /dev/null +++ b/packages/gapic-generator/tests/integration/goldens/logging_internal/google/cloud/logging_v2/_compat.py @@ -0,0 +1,310 @@ +# # Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""A compatibility module for older versions of google-api-core.""" + +import functools +import json +import operator +import os +import re +import uuid +from typing import Any, Callable, Dict, List, Optional, Tuple + +from google.auth.exceptions import MutualTLSChannelError + + +try: + from google.api_core.universe import ( # type: ignore + get_default_mtls_endpoint, + get_api_endpoint, + get_universe_domain, + ) +except ImportError: # pragma: NO COVER + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Universe domain support was introduced in google-api-core 2.18.0. + # Remove these fallback definitions when google-api-core >= 2.18.0 becomes the minimum required version in generated client setup dependencies. + def get_default_mtls_endpoint(api_endpoint: Optional[str]) -> Optional[str]: + """Converts api endpoint to mTLS endpoint.""" + if not api_endpoint: + return api_endpoint + + mtls_endpoint_re = re.compile( + r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" + ) + + m = mtls_endpoint_re.match(api_endpoint) + if m is None: + # Could not parse api_endpoint; return as-is. + return api_endpoint + + name, mtls, sandbox, googledomain = m.groups() + if mtls or not googledomain: + return api_endpoint + + if sandbox: + return api_endpoint.replace( + "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" + ) + + return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") + + def get_api_endpoint( # type: ignore[misc] + api_override: Optional[str], + client_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]], + universe_domain: str, + use_mtls_endpoint: str, + default_universe: str, + default_mtls_endpoint: Optional[str], + default_endpoint_template: str, + ) -> str: + """Return the API endpoint used by the client.""" + api_endpoint: Optional[str] = None + if api_override is not None: + api_endpoint = api_override + elif use_mtls_endpoint == "always" or ( + use_mtls_endpoint == "auto" and client_cert_source + ): + if universe_domain != default_universe: + raise MutualTLSChannelError( + f"mTLS is not supported in any universe other than {default_universe}." + ) + api_endpoint = default_mtls_endpoint + else: + api_endpoint = default_endpoint_template.format( + UNIVERSE_DOMAIN=universe_domain + ) + return api_endpoint # type: ignore[return-value] + + def get_universe_domain( # type: ignore[misc] + *potential_universes: Optional[str], + default_universe: str = "googleapis.com", + ) -> str: + """Return the universe domain used by the client.""" + resolved = next( + (x.strip() for x in potential_universes if x is not None), + default_universe, + ) + if not resolved: + raise ValueError("Universe Domain cannot be an empty string.") + return resolved + + +try: + from google.api_core.gapic_v1.config import ( # type: ignore + use_client_cert_effective, + get_client_cert_source, + read_environment_variables, + ) +except ImportError: # pragma: NO COVER + from google.auth.transport import mtls # type: ignore + + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Remove these fallbacks when google-api-core >= 2.18.0 is the minimum required version. + + def use_client_cert_effective() -> bool: + """Returns whether client certificate should be used for mTLS.""" + if hasattr(mtls, "should_use_client_cert"): + return mtls.should_use_client_cert() + else: + use_client_cert_str = os.getenv( + "GOOGLE_API_USE_CLIENT_CERTIFICATE", "false" + ).lower() + if use_client_cert_str not in ("true", "false"): + raise ValueError( + "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be" + " either `true` or `false`" + ) + return use_client_cert_str == "true" + + def get_client_cert_source( + provided_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]], + use_cert_flag: bool, + ) -> Optional[Callable[[], Tuple[bytes, bytes]]]: + """Return the client cert source to be used by the client.""" + client_cert_source = None + if use_cert_flag: + if provided_cert_source: + client_cert_source = provided_cert_source + elif ( + hasattr(mtls, "has_default_client_cert_source") + and mtls.has_default_client_cert_source() + ): + client_cert_source = mtls.default_client_cert_source() + else: + raise ValueError( + "Client certificate is required for mTLS, but no client certificate source was provided or found." + ) + return client_cert_source + + def read_environment_variables() -> Tuple[bool, str, Optional[str]]: + """Returns the environment variables used by the client.""" + use_client_cert = use_client_cert_effective() + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower() + universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN") + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError( + "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` " + "must be `never`, `auto` or `always`" + ) + return use_client_cert, use_mtls_endpoint, universe_domain_env + + +try: + from google.api_core.gapic_v1.request import setup_request_id # type: ignore +except ImportError: # pragma: NO COVER + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Request ID setup helper was introduced in google-api-core 2.26.0. + # Remove this fallback definition when google-api-core >= 2.26.0 becomes the minimum required version in generated client setup dependencies. + def setup_request_id( + request: Any, field_name: str, is_proto3_optional: bool + ) -> None: + """Populate a UUID4 field in the request if it is not already set. + + Args: + request (Union[google.protobuf.message.Message, dict]): The request object. + field_name (str): The name of the field to populate. + is_proto3_optional (bool): Whether the field is proto3 optional. + """ + if request is None: + return + + request_id_val = str(uuid.uuid4()) + + if isinstance(request, dict): + if is_proto3_optional: + if field_name not in request or request[field_name] is None: + request[field_name] = request_id_val + elif not request.get(field_name): + request[field_name] = request_id_val + return + + if is_proto3_optional: + try: + # Pure protobuf messages + if not request.HasField(field_name): + setattr(request, field_name, request_id_val) + except (AttributeError, ValueError): + # Proto-plus messages or other objects + if hasattr(request, "_pb"): + try: + if not request._pb.HasField(field_name): + setattr(request, field_name, request_id_val) + return + except (AttributeError, ValueError): + pass + if getattr(request, field_name, None) is None: + setattr(request, field_name, request_id_val) + else: + if not getattr(request, field_name, None): + setattr(request, field_name, request_id_val) + + +try: + from google.api_core.rest_helpers import ( # type: ignore + flatten_query_params, + transcode_request, + ) +except ImportError: # pragma: NO COVER + # TODO: Remove these fallbacks when google-api-core >= 2.18.0 is the minimum required version. + from google.protobuf import json_format # type: ignore + from google.api_core import path_template # type: ignore + + def flatten_query_params(obj: Any, strict: bool = False) -> List[Tuple[str, Any]]: # type: ignore[misc] + if obj is not None and not isinstance(obj, dict): + raise TypeError("flatten_query_params must be called with dict object") + return _flatten(obj, key_path=[], strict=strict) + + def _flatten( + obj: Any, key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + if obj is None: + return [] + if isinstance(obj, dict): + return _flatten_dict(obj, key_path=key_path, strict=strict) + if isinstance(obj, list): + return _flatten_list(obj, key_path=key_path, strict=strict) + return _flatten_value(obj, key_path=key_path, strict=strict) + + def _is_primitive_value(obj: Any) -> bool: + if obj is None: + return False + if isinstance(obj, (list, dict)): + raise ValueError("query params may not contain repeated dicts or lists") + return True + + def _flatten_value( + obj: Any, key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + return [(".".join(key_path), _canonicalize(obj, strict=strict))] + + def _flatten_dict( + obj: Dict[str, Any], key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + items = ( + _flatten(value, key_path=key_path + [key], strict=strict) + for key, value in obj.items() + ) + return functools.reduce(operator.concat, items, []) # type: ignore[arg-type] + + def _flatten_list( + elems: List[Any], key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + items = ( + _flatten_value(elem, key_path=key_path, strict=strict) + for elem in elems + if _is_primitive_value(elem) + ) + return functools.reduce(operator.concat, items, []) # type: ignore[arg-type] + + def _canonicalize(obj: Any, strict: bool = False) -> Any: + if strict: + value = str(obj) + if isinstance(obj, bool): + value = value.lower() + return value + return obj + + def transcode_request( # type: ignore[misc] + http_options: List[Dict[str, str]], + request: Any, + required_fields_default_values: Optional[Dict[str, Any]] = None, + rest_numeric_enums: bool = False, + ) -> Tuple[Dict[str, Any], Optional[str], Dict[str, Any]]: + pb_request = getattr(request, "_pb", request) + transcoded_request = path_template.transcode(http_options, pb_request) + + body_json = None + if transcoded_request.get("body") is not None: + body_json = json_format.MessageToJson( + transcoded_request["body"], + use_integers_for_enums=rest_numeric_enums, + ) + + query_params_json = {} + if transcoded_request.get("query_params") is not None: + query_params_json = json.loads( + json_format.MessageToJson( + transcoded_request["query_params"], + use_integers_for_enums=rest_numeric_enums, + ) + ) + + if required_fields_default_values: + for k, v in required_fields_default_values.items(): + if k not in query_params_json: + query_params_json[k] = v + + if rest_numeric_enums: + query_params_json["$alt"] = "json;enum-encoding=int" + + return transcoded_request, body_json, query_params_json diff --git a/packages/gapic-generator/tests/integration/goldens/logging_internal/tests/unit/gapic/logging_v2/test_compat.py b/packages/gapic-generator/tests/integration/goldens/logging_internal/tests/unit/gapic/logging_v2/test_compat.py new file mode 100644 index 000000000000..716e1ac126fd --- /dev/null +++ b/packages/gapic-generator/tests/integration/goldens/logging_internal/tests/unit/gapic/logging_v2/test_compat.py @@ -0,0 +1,364 @@ +# # Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import builtins +import importlib +import sys +from unittest import mock +import pytest +from google.auth.exceptions import MutualTLSChannelError +from google.protobuf import descriptor_pb2 + + +from google.cloud.logging_v2 import _compat + + +def test_compat_normal_import(): + assert _compat.setup_request_id is not None + + +def test_compat_fallback_implementations(): + orig_import = builtins.__import__ + + def custom_import(name, globals=None, locals=None, fromlist=(), level=0): + if name in ( + "google.api_core.universe", + "google.api_core.gapic_v1.config", + "google.api_core.gapic_v1.request", + "google.api_core.rest_helpers", + ): + raise ImportError(f"Mocked ImportError for {name}") + return orig_import(name, globals, locals, fromlist, level) + + with mock.patch("builtins.__import__", side_effect=custom_import): + fallback = importlib.reload(_compat) + + # get_default_mtls_endpoint tests + assert fallback.get_default_mtls_endpoint(None) is None + assert fallback.get_default_mtls_endpoint("") == "" + assert ( + fallback.get_default_mtls_endpoint("foo.googleapis.com") + == "foo.mtls.googleapis.com" + ) + assert ( + fallback.get_default_mtls_endpoint("foo.sandbox.googleapis.com") + == "foo.mtls.sandbox.googleapis.com" + ) + assert ( + fallback.get_default_mtls_endpoint("foo.mtls.googleapis.com") + == "foo.mtls.googleapis.com" + ) + assert ( + fallback.get_default_mtls_endpoint("custom.domain.com") + == "custom.domain.com" + ) + assert ( + fallback.get_default_mtls_endpoint(":::invalid-url:::") + == ":::invalid-url:::" + ) + + # get_api_endpoint tests + assert ( + fallback.get_api_endpoint( + "https://override.com", + None, + "googleapis.com", + "auto", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "https://override.com" + ) + assert ( + fallback.get_api_endpoint( + None, + lambda: (b"", b""), + "googleapis.com", + "always", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "mtls.com" + ) + assert ( + fallback.get_api_endpoint( + None, + lambda: (b"", b""), + "googleapis.com", + "auto", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "mtls.com" + ) + with pytest.raises(MutualTLSChannelError): + fallback.get_api_endpoint( + None, + lambda: (b"", b""), + "otheruniverse.com", + "always", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + assert ( + fallback.get_api_endpoint( + None, + None, + "googleapis.com", + "never", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "https://googleapis.com" + ) + + # get_universe_domain tests + assert ( + fallback.get_universe_domain("custom.com", None, "googleapis.com") + == "custom.com" + ) + assert ( + fallback.get_universe_domain(None, "env.com", "googleapis.com") == "env.com" + ) + assert ( + fallback.get_universe_domain(None, None, "googleapis.com") + == "googleapis.com" + ) + with pytest.raises(ValueError): + fallback.get_universe_domain(" ", None, "googleapis.com") + + # use_client_cert_effective tests + with mock.patch( + "google.auth.transport.mtls.should_use_client_cert", + return_value=True, + create=True, + ): + assert fallback.use_client_cert_effective() is True + + with mock.patch.object(fallback, "mtls", spec=object()): + with mock.patch.dict( + "os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"} + ): + assert fallback.use_client_cert_effective() is True + with mock.patch.dict( + "os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"} + ): + assert fallback.use_client_cert_effective() is False + with mock.patch.dict( + "os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "invalid"} + ): + with pytest.raises(ValueError): + fallback.use_client_cert_effective() + + # get_client_cert_source tests + cert_fn = lambda: (b"cert", b"key") + assert fallback.get_client_cert_source(cert_fn, True) == cert_fn + assert fallback.get_client_cert_source(None, False) is None + + with mock.patch( + "google.auth.transport.mtls.has_default_client_cert_source", + return_value=True, + create=True, + ): + with mock.patch( + "google.auth.transport.mtls.default_client_cert_source", + return_value=cert_fn, + create=True, + ): + assert fallback.get_client_cert_source(None, True) == cert_fn + + with mock.patch.object(fallback, "mtls", spec=object()): + with pytest.raises(ValueError): + fallback.get_client_cert_source(None, True) + + # read_environment_variables tests + with mock.patch.dict( + "os.environ", + { + "GOOGLE_API_USE_MTLS_ENDPOINT": "always", + "GOOGLE_CLOUD_UNIVERSE_DOMAIN": "myuniverse.com", + }, + ): + use_cert, use_mtls, universe = fallback.read_environment_variables() + assert use_mtls == "always" + assert universe == "myuniverse.com" + + with mock.patch.dict("os.environ", {"GOOGLE_API_USE_MTLS_ENDPOINT": "invalid"}): + with pytest.raises(MutualTLSChannelError): + fallback.read_environment_variables() + + # setup_request_id tests + fallback.setup_request_id(None, "request_id", False) + + d1 = {} + fallback.setup_request_id(d1, "request_id", is_proto3_optional=True) + assert "request_id" in d1 + + d1_existing = {"request_id": None} + fallback.setup_request_id(d1_existing, "request_id", is_proto3_optional=True) + assert d1_existing["request_id"] is not None + + d2 = {} + fallback.setup_request_id(d2, "request_id", is_proto3_optional=False) + assert "request_id" in d2 + + d3 = {"request_id": "existing"} + fallback.setup_request_id(d3, "request_id", is_proto3_optional=False) + assert d3["request_id"] == "existing" + + d4 = {"request_id": "val"} + fallback.setup_request_id(d4, "request_id", is_proto3_optional=True) + assert d4["request_id"] == "val" + + class DummyPopulated: + def __init__(self): + self.request_id = "val" + + p_existing = DummyPopulated() + fallback.setup_request_id(p_existing, "request_id", is_proto3_optional=False) + assert p_existing.request_id == "val" + + class NonOptPlain: + def __init__(self): + self.request_id = "" + + nop = NonOptPlain() + fallback.setup_request_id(nop, "request_id", is_proto3_optional=False) + assert nop.request_id != "" + + class DummyProto: + def __init__(self): + self.request_id = "" + self._has = False + + def HasField(self, name): + if not self._has: + return False + return True + + p1 = DummyProto() + fallback.setup_request_id(p1, "request_id", is_proto3_optional=True) + assert p1.request_id != "" + + class DummyWrapper: + def __init__(self): + self._pb = DummyProto() + + w1 = DummyWrapper() + fallback.setup_request_id(w1, "request_id", is_proto3_optional=True) + assert ( + getattr(w1, "request_id", None) is not None + or getattr(w1._pb, "request_id", None) != "" + ) + + class BadProto: + def HasField(self, name): + raise AttributeError() + + class BadWrapper: + def __init__(self): + self._pb = BadProto() + self.request_id = None + + bw = BadWrapper() + fallback.setup_request_id(bw, "request_id", is_proto3_optional=True) + assert bw.request_id is not None + + class SetProto: + def __init__(self): + self.request_id = "already_set" + + def HasField(self, name): + return True + + sp = SetProto() + fallback.setup_request_id(sp, "request_id", is_proto3_optional=True) + assert sp.request_id == "already_set" + + class SetProtoNonOpt: + def __init__(self): + self.request_id = "already_set" + + sp_non_opt = SetProtoNonOpt() + fallback.setup_request_id(sp_non_opt, "request_id", is_proto3_optional=False) + assert sp_non_opt.request_id == "already_set" + + class DummyPlain: + def __init__(self): + self.request_id = None + + pl1 = DummyPlain() + fallback.setup_request_id(pl1, "request_id", is_proto3_optional=True) + assert pl1.request_id is not None + + pl2 = DummyPlain() + fallback.setup_request_id(pl2, "request_id", is_proto3_optional=False) + assert pl2.request_id is not None + + # flatten_query_params tests + assert fallback.flatten_query_params(None) == [] + res = fallback.flatten_query_params({"a": "val1", "b": [1, 2], "c": True}) + assert ("a", "val1") in res + assert ("b", 1) in res + assert ("b", 2) in res + assert ("c", True) in res + + res_strict = fallback.flatten_query_params({"c": True}, strict=True) + assert ("c", "true") in res_strict + + assert fallback._canonicalize(True, strict=True) == "true" + assert fallback._canonicalize(True, strict=False) is True + assert fallback._canonicalize(123, strict=True) == "123" + assert fallback._canonicalize("str", strict=True) == "str" + assert fallback._canonicalize("str", strict=False) == "str" + assert fallback._is_primitive_value(None) is False + + with pytest.raises(TypeError): + fallback.flatten_query_params("invalid") + + with pytest.raises(ValueError): + fallback.flatten_query_params({"a": [{"nested": "dict"}]}) + + with pytest.raises(ValueError): + fallback._is_primitive_value([1, 2]) + + # transcode_request tests + dummy_req = descriptor_pb2.DescriptorProto() + http_opts = [{"method": "get", "uri": "/v1/test"}] + transcoded, body, query = fallback.transcode_request( + http_opts, + dummy_req, + required_fields_default_values={"non_existent_key": "val"}, + rest_numeric_enums=True, + ) + assert transcoded is not None + assert query.get("non_existent_key") == "val" + assert query.get("$alt") == "json;enum-encoding=int" + + transcoded2, body2, query2 = fallback.transcode_request( + http_opts, + dummy_req, + required_fields_default_values={"name": "override_name"}, + rest_numeric_enums=False, + ) + assert body2 is None + assert "$alt" not in query2 + + importlib.reload(_compat) diff --git a/packages/gapic-generator/tests/integration/goldens/redis/google/cloud/redis_v1/_compat.py b/packages/gapic-generator/tests/integration/goldens/redis/google/cloud/redis_v1/_compat.py new file mode 100644 index 000000000000..54c22cd7d09e --- /dev/null +++ b/packages/gapic-generator/tests/integration/goldens/redis/google/cloud/redis_v1/_compat.py @@ -0,0 +1,310 @@ +# # Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""A compatibility module for older versions of google-api-core.""" + +import functools +import json +import operator +import os +import re +import uuid +from typing import Any, Callable, Dict, List, Optional, Tuple + +from google.auth.exceptions import MutualTLSChannelError + + +try: + from google.api_core.universe import ( # type: ignore + get_default_mtls_endpoint, + get_api_endpoint, + get_universe_domain, + ) +except ImportError: # pragma: NO COVER + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Universe domain support was introduced in google-api-core 2.18.0. + # Remove these fallback definitions when google-api-core >= 2.18.0 becomes the minimum required version in generated client setup dependencies. + def get_default_mtls_endpoint(api_endpoint: Optional[str]) -> Optional[str]: + """Converts api endpoint to mTLS endpoint.""" + if not api_endpoint: + return api_endpoint + + mtls_endpoint_re = re.compile( + r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" + ) + + m = mtls_endpoint_re.match(api_endpoint) + if m is None: + # Could not parse api_endpoint; return as-is. + return api_endpoint + + name, mtls, sandbox, googledomain = m.groups() + if mtls or not googledomain: + return api_endpoint + + if sandbox: + return api_endpoint.replace( + "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" + ) + + return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") + + def get_api_endpoint( # type: ignore[misc] + api_override: Optional[str], + client_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]], + universe_domain: str, + use_mtls_endpoint: str, + default_universe: str, + default_mtls_endpoint: Optional[str], + default_endpoint_template: str, + ) -> str: + """Return the API endpoint used by the client.""" + api_endpoint: Optional[str] = None + if api_override is not None: + api_endpoint = api_override + elif use_mtls_endpoint == "always" or ( + use_mtls_endpoint == "auto" and client_cert_source + ): + if universe_domain != default_universe: + raise MutualTLSChannelError( + f"mTLS is not supported in any universe other than {default_universe}." + ) + api_endpoint = default_mtls_endpoint + else: + api_endpoint = default_endpoint_template.format( + UNIVERSE_DOMAIN=universe_domain + ) + return api_endpoint # type: ignore[return-value] + + def get_universe_domain( # type: ignore[misc] + *potential_universes: Optional[str], + default_universe: str = "googleapis.com", + ) -> str: + """Return the universe domain used by the client.""" + resolved = next( + (x.strip() for x in potential_universes if x is not None), + default_universe, + ) + if not resolved: + raise ValueError("Universe Domain cannot be an empty string.") + return resolved + + +try: + from google.api_core.gapic_v1.config import ( # type: ignore + use_client_cert_effective, + get_client_cert_source, + read_environment_variables, + ) +except ImportError: # pragma: NO COVER + from google.auth.transport import mtls # type: ignore + + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Remove these fallbacks when google-api-core >= 2.18.0 is the minimum required version. + + def use_client_cert_effective() -> bool: + """Returns whether client certificate should be used for mTLS.""" + if hasattr(mtls, "should_use_client_cert"): + return mtls.should_use_client_cert() + else: + use_client_cert_str = os.getenv( + "GOOGLE_API_USE_CLIENT_CERTIFICATE", "false" + ).lower() + if use_client_cert_str not in ("true", "false"): + raise ValueError( + "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be" + " either `true` or `false`" + ) + return use_client_cert_str == "true" + + def get_client_cert_source( + provided_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]], + use_cert_flag: bool, + ) -> Optional[Callable[[], Tuple[bytes, bytes]]]: + """Return the client cert source to be used by the client.""" + client_cert_source = None + if use_cert_flag: + if provided_cert_source: + client_cert_source = provided_cert_source + elif ( + hasattr(mtls, "has_default_client_cert_source") + and mtls.has_default_client_cert_source() + ): + client_cert_source = mtls.default_client_cert_source() + else: + raise ValueError( + "Client certificate is required for mTLS, but no client certificate source was provided or found." + ) + return client_cert_source + + def read_environment_variables() -> Tuple[bool, str, Optional[str]]: + """Returns the environment variables used by the client.""" + use_client_cert = use_client_cert_effective() + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower() + universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN") + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError( + "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` " + "must be `never`, `auto` or `always`" + ) + return use_client_cert, use_mtls_endpoint, universe_domain_env + + +try: + from google.api_core.gapic_v1.request import setup_request_id # type: ignore +except ImportError: # pragma: NO COVER + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Request ID setup helper was introduced in google-api-core 2.26.0. + # Remove this fallback definition when google-api-core >= 2.26.0 becomes the minimum required version in generated client setup dependencies. + def setup_request_id( + request: Any, field_name: str, is_proto3_optional: bool + ) -> None: + """Populate a UUID4 field in the request if it is not already set. + + Args: + request (Union[google.protobuf.message.Message, dict]): The request object. + field_name (str): The name of the field to populate. + is_proto3_optional (bool): Whether the field is proto3 optional. + """ + if request is None: + return + + request_id_val = str(uuid.uuid4()) + + if isinstance(request, dict): + if is_proto3_optional: + if field_name not in request or request[field_name] is None: + request[field_name] = request_id_val + elif not request.get(field_name): + request[field_name] = request_id_val + return + + if is_proto3_optional: + try: + # Pure protobuf messages + if not request.HasField(field_name): + setattr(request, field_name, request_id_val) + except (AttributeError, ValueError): + # Proto-plus messages or other objects + if hasattr(request, "_pb"): + try: + if not request._pb.HasField(field_name): + setattr(request, field_name, request_id_val) + return + except (AttributeError, ValueError): + pass + if getattr(request, field_name, None) is None: + setattr(request, field_name, request_id_val) + else: + if not getattr(request, field_name, None): + setattr(request, field_name, request_id_val) + + +try: + from google.api_core.rest_helpers import ( # type: ignore + flatten_query_params, + transcode_request, + ) +except ImportError: # pragma: NO COVER + # TODO: Remove these fallbacks when google-api-core >= 2.18.0 is the minimum required version. + from google.protobuf import json_format # type: ignore + from google.api_core import path_template # type: ignore + + def flatten_query_params(obj: Any, strict: bool = False) -> List[Tuple[str, Any]]: # type: ignore[misc] + if obj is not None and not isinstance(obj, dict): + raise TypeError("flatten_query_params must be called with dict object") + return _flatten(obj, key_path=[], strict=strict) + + def _flatten( + obj: Any, key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + if obj is None: + return [] + if isinstance(obj, dict): + return _flatten_dict(obj, key_path=key_path, strict=strict) + if isinstance(obj, list): + return _flatten_list(obj, key_path=key_path, strict=strict) + return _flatten_value(obj, key_path=key_path, strict=strict) + + def _is_primitive_value(obj: Any) -> bool: + if obj is None: + return False + if isinstance(obj, (list, dict)): + raise ValueError("query params may not contain repeated dicts or lists") + return True + + def _flatten_value( + obj: Any, key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + return [(".".join(key_path), _canonicalize(obj, strict=strict))] + + def _flatten_dict( + obj: Dict[str, Any], key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + items = ( + _flatten(value, key_path=key_path + [key], strict=strict) + for key, value in obj.items() + ) + return functools.reduce(operator.concat, items, []) # type: ignore[arg-type] + + def _flatten_list( + elems: List[Any], key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + items = ( + _flatten_value(elem, key_path=key_path, strict=strict) + for elem in elems + if _is_primitive_value(elem) + ) + return functools.reduce(operator.concat, items, []) # type: ignore[arg-type] + + def _canonicalize(obj: Any, strict: bool = False) -> Any: + if strict: + value = str(obj) + if isinstance(obj, bool): + value = value.lower() + return value + return obj + + def transcode_request( # type: ignore[misc] + http_options: List[Dict[str, str]], + request: Any, + required_fields_default_values: Optional[Dict[str, Any]] = None, + rest_numeric_enums: bool = False, + ) -> Tuple[Dict[str, Any], Optional[str], Dict[str, Any]]: + pb_request = getattr(request, "_pb", request) + transcoded_request = path_template.transcode(http_options, pb_request) + + body_json = None + if transcoded_request.get("body") is not None: + body_json = json_format.MessageToJson( + transcoded_request["body"], + use_integers_for_enums=rest_numeric_enums, + ) + + query_params_json = {} + if transcoded_request.get("query_params") is not None: + query_params_json = json.loads( + json_format.MessageToJson( + transcoded_request["query_params"], + use_integers_for_enums=rest_numeric_enums, + ) + ) + + if required_fields_default_values: + for k, v in required_fields_default_values.items(): + if k not in query_params_json: + query_params_json[k] = v + + if rest_numeric_enums: + query_params_json["$alt"] = "json;enum-encoding=int" + + return transcoded_request, body_json, query_params_json diff --git a/packages/gapic-generator/tests/integration/goldens/redis/tests/unit/gapic/redis_v1/test_compat.py b/packages/gapic-generator/tests/integration/goldens/redis/tests/unit/gapic/redis_v1/test_compat.py new file mode 100644 index 000000000000..7713b20e1892 --- /dev/null +++ b/packages/gapic-generator/tests/integration/goldens/redis/tests/unit/gapic/redis_v1/test_compat.py @@ -0,0 +1,364 @@ +# # Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import builtins +import importlib +import sys +from unittest import mock +import pytest +from google.auth.exceptions import MutualTLSChannelError +from google.protobuf import descriptor_pb2 + + +from google.cloud.redis_v1 import _compat + + +def test_compat_normal_import(): + assert _compat.setup_request_id is not None + + +def test_compat_fallback_implementations(): + orig_import = builtins.__import__ + + def custom_import(name, globals=None, locals=None, fromlist=(), level=0): + if name in ( + "google.api_core.universe", + "google.api_core.gapic_v1.config", + "google.api_core.gapic_v1.request", + "google.api_core.rest_helpers", + ): + raise ImportError(f"Mocked ImportError for {name}") + return orig_import(name, globals, locals, fromlist, level) + + with mock.patch("builtins.__import__", side_effect=custom_import): + fallback = importlib.reload(_compat) + + # get_default_mtls_endpoint tests + assert fallback.get_default_mtls_endpoint(None) is None + assert fallback.get_default_mtls_endpoint("") == "" + assert ( + fallback.get_default_mtls_endpoint("foo.googleapis.com") + == "foo.mtls.googleapis.com" + ) + assert ( + fallback.get_default_mtls_endpoint("foo.sandbox.googleapis.com") + == "foo.mtls.sandbox.googleapis.com" + ) + assert ( + fallback.get_default_mtls_endpoint("foo.mtls.googleapis.com") + == "foo.mtls.googleapis.com" + ) + assert ( + fallback.get_default_mtls_endpoint("custom.domain.com") + == "custom.domain.com" + ) + assert ( + fallback.get_default_mtls_endpoint(":::invalid-url:::") + == ":::invalid-url:::" + ) + + # get_api_endpoint tests + assert ( + fallback.get_api_endpoint( + "https://override.com", + None, + "googleapis.com", + "auto", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "https://override.com" + ) + assert ( + fallback.get_api_endpoint( + None, + lambda: (b"", b""), + "googleapis.com", + "always", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "mtls.com" + ) + assert ( + fallback.get_api_endpoint( + None, + lambda: (b"", b""), + "googleapis.com", + "auto", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "mtls.com" + ) + with pytest.raises(MutualTLSChannelError): + fallback.get_api_endpoint( + None, + lambda: (b"", b""), + "otheruniverse.com", + "always", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + assert ( + fallback.get_api_endpoint( + None, + None, + "googleapis.com", + "never", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "https://googleapis.com" + ) + + # get_universe_domain tests + assert ( + fallback.get_universe_domain("custom.com", None, "googleapis.com") + == "custom.com" + ) + assert ( + fallback.get_universe_domain(None, "env.com", "googleapis.com") == "env.com" + ) + assert ( + fallback.get_universe_domain(None, None, "googleapis.com") + == "googleapis.com" + ) + with pytest.raises(ValueError): + fallback.get_universe_domain(" ", None, "googleapis.com") + + # use_client_cert_effective tests + with mock.patch( + "google.auth.transport.mtls.should_use_client_cert", + return_value=True, + create=True, + ): + assert fallback.use_client_cert_effective() is True + + with mock.patch.object(fallback, "mtls", spec=object()): + with mock.patch.dict( + "os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"} + ): + assert fallback.use_client_cert_effective() is True + with mock.patch.dict( + "os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"} + ): + assert fallback.use_client_cert_effective() is False + with mock.patch.dict( + "os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "invalid"} + ): + with pytest.raises(ValueError): + fallback.use_client_cert_effective() + + # get_client_cert_source tests + cert_fn = lambda: (b"cert", b"key") + assert fallback.get_client_cert_source(cert_fn, True) == cert_fn + assert fallback.get_client_cert_source(None, False) is None + + with mock.patch( + "google.auth.transport.mtls.has_default_client_cert_source", + return_value=True, + create=True, + ): + with mock.patch( + "google.auth.transport.mtls.default_client_cert_source", + return_value=cert_fn, + create=True, + ): + assert fallback.get_client_cert_source(None, True) == cert_fn + + with mock.patch.object(fallback, "mtls", spec=object()): + with pytest.raises(ValueError): + fallback.get_client_cert_source(None, True) + + # read_environment_variables tests + with mock.patch.dict( + "os.environ", + { + "GOOGLE_API_USE_MTLS_ENDPOINT": "always", + "GOOGLE_CLOUD_UNIVERSE_DOMAIN": "myuniverse.com", + }, + ): + use_cert, use_mtls, universe = fallback.read_environment_variables() + assert use_mtls == "always" + assert universe == "myuniverse.com" + + with mock.patch.dict("os.environ", {"GOOGLE_API_USE_MTLS_ENDPOINT": "invalid"}): + with pytest.raises(MutualTLSChannelError): + fallback.read_environment_variables() + + # setup_request_id tests + fallback.setup_request_id(None, "request_id", False) + + d1 = {} + fallback.setup_request_id(d1, "request_id", is_proto3_optional=True) + assert "request_id" in d1 + + d1_existing = {"request_id": None} + fallback.setup_request_id(d1_existing, "request_id", is_proto3_optional=True) + assert d1_existing["request_id"] is not None + + d2 = {} + fallback.setup_request_id(d2, "request_id", is_proto3_optional=False) + assert "request_id" in d2 + + d3 = {"request_id": "existing"} + fallback.setup_request_id(d3, "request_id", is_proto3_optional=False) + assert d3["request_id"] == "existing" + + d4 = {"request_id": "val"} + fallback.setup_request_id(d4, "request_id", is_proto3_optional=True) + assert d4["request_id"] == "val" + + class DummyPopulated: + def __init__(self): + self.request_id = "val" + + p_existing = DummyPopulated() + fallback.setup_request_id(p_existing, "request_id", is_proto3_optional=False) + assert p_existing.request_id == "val" + + class NonOptPlain: + def __init__(self): + self.request_id = "" + + nop = NonOptPlain() + fallback.setup_request_id(nop, "request_id", is_proto3_optional=False) + assert nop.request_id != "" + + class DummyProto: + def __init__(self): + self.request_id = "" + self._has = False + + def HasField(self, name): + if not self._has: + return False + return True + + p1 = DummyProto() + fallback.setup_request_id(p1, "request_id", is_proto3_optional=True) + assert p1.request_id != "" + + class DummyWrapper: + def __init__(self): + self._pb = DummyProto() + + w1 = DummyWrapper() + fallback.setup_request_id(w1, "request_id", is_proto3_optional=True) + assert ( + getattr(w1, "request_id", None) is not None + or getattr(w1._pb, "request_id", None) != "" + ) + + class BadProto: + def HasField(self, name): + raise AttributeError() + + class BadWrapper: + def __init__(self): + self._pb = BadProto() + self.request_id = None + + bw = BadWrapper() + fallback.setup_request_id(bw, "request_id", is_proto3_optional=True) + assert bw.request_id is not None + + class SetProto: + def __init__(self): + self.request_id = "already_set" + + def HasField(self, name): + return True + + sp = SetProto() + fallback.setup_request_id(sp, "request_id", is_proto3_optional=True) + assert sp.request_id == "already_set" + + class SetProtoNonOpt: + def __init__(self): + self.request_id = "already_set" + + sp_non_opt = SetProtoNonOpt() + fallback.setup_request_id(sp_non_opt, "request_id", is_proto3_optional=False) + assert sp_non_opt.request_id == "already_set" + + class DummyPlain: + def __init__(self): + self.request_id = None + + pl1 = DummyPlain() + fallback.setup_request_id(pl1, "request_id", is_proto3_optional=True) + assert pl1.request_id is not None + + pl2 = DummyPlain() + fallback.setup_request_id(pl2, "request_id", is_proto3_optional=False) + assert pl2.request_id is not None + + # flatten_query_params tests + assert fallback.flatten_query_params(None) == [] + res = fallback.flatten_query_params({"a": "val1", "b": [1, 2], "c": True}) + assert ("a", "val1") in res + assert ("b", 1) in res + assert ("b", 2) in res + assert ("c", True) in res + + res_strict = fallback.flatten_query_params({"c": True}, strict=True) + assert ("c", "true") in res_strict + + assert fallback._canonicalize(True, strict=True) == "true" + assert fallback._canonicalize(True, strict=False) is True + assert fallback._canonicalize(123, strict=True) == "123" + assert fallback._canonicalize("str", strict=True) == "str" + assert fallback._canonicalize("str", strict=False) == "str" + assert fallback._is_primitive_value(None) is False + + with pytest.raises(TypeError): + fallback.flatten_query_params("invalid") + + with pytest.raises(ValueError): + fallback.flatten_query_params({"a": [{"nested": "dict"}]}) + + with pytest.raises(ValueError): + fallback._is_primitive_value([1, 2]) + + # transcode_request tests + dummy_req = descriptor_pb2.DescriptorProto() + http_opts = [{"method": "get", "uri": "/v1/test"}] + transcoded, body, query = fallback.transcode_request( + http_opts, + dummy_req, + required_fields_default_values={"non_existent_key": "val"}, + rest_numeric_enums=True, + ) + assert transcoded is not None + assert query.get("non_existent_key") == "val" + assert query.get("$alt") == "json;enum-encoding=int" + + transcoded2, body2, query2 = fallback.transcode_request( + http_opts, + dummy_req, + required_fields_default_values={"name": "override_name"}, + rest_numeric_enums=False, + ) + assert body2 is None + assert "$alt" not in query2 + + importlib.reload(_compat) diff --git a/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/_compat.py b/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/_compat.py new file mode 100644 index 000000000000..54c22cd7d09e --- /dev/null +++ b/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/_compat.py @@ -0,0 +1,310 @@ +# # Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""A compatibility module for older versions of google-api-core.""" + +import functools +import json +import operator +import os +import re +import uuid +from typing import Any, Callable, Dict, List, Optional, Tuple + +from google.auth.exceptions import MutualTLSChannelError + + +try: + from google.api_core.universe import ( # type: ignore + get_default_mtls_endpoint, + get_api_endpoint, + get_universe_domain, + ) +except ImportError: # pragma: NO COVER + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Universe domain support was introduced in google-api-core 2.18.0. + # Remove these fallback definitions when google-api-core >= 2.18.0 becomes the minimum required version in generated client setup dependencies. + def get_default_mtls_endpoint(api_endpoint: Optional[str]) -> Optional[str]: + """Converts api endpoint to mTLS endpoint.""" + if not api_endpoint: + return api_endpoint + + mtls_endpoint_re = re.compile( + r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" + ) + + m = mtls_endpoint_re.match(api_endpoint) + if m is None: + # Could not parse api_endpoint; return as-is. + return api_endpoint + + name, mtls, sandbox, googledomain = m.groups() + if mtls or not googledomain: + return api_endpoint + + if sandbox: + return api_endpoint.replace( + "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" + ) + + return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") + + def get_api_endpoint( # type: ignore[misc] + api_override: Optional[str], + client_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]], + universe_domain: str, + use_mtls_endpoint: str, + default_universe: str, + default_mtls_endpoint: Optional[str], + default_endpoint_template: str, + ) -> str: + """Return the API endpoint used by the client.""" + api_endpoint: Optional[str] = None + if api_override is not None: + api_endpoint = api_override + elif use_mtls_endpoint == "always" or ( + use_mtls_endpoint == "auto" and client_cert_source + ): + if universe_domain != default_universe: + raise MutualTLSChannelError( + f"mTLS is not supported in any universe other than {default_universe}." + ) + api_endpoint = default_mtls_endpoint + else: + api_endpoint = default_endpoint_template.format( + UNIVERSE_DOMAIN=universe_domain + ) + return api_endpoint # type: ignore[return-value] + + def get_universe_domain( # type: ignore[misc] + *potential_universes: Optional[str], + default_universe: str = "googleapis.com", + ) -> str: + """Return the universe domain used by the client.""" + resolved = next( + (x.strip() for x in potential_universes if x is not None), + default_universe, + ) + if not resolved: + raise ValueError("Universe Domain cannot be an empty string.") + return resolved + + +try: + from google.api_core.gapic_v1.config import ( # type: ignore + use_client_cert_effective, + get_client_cert_source, + read_environment_variables, + ) +except ImportError: # pragma: NO COVER + from google.auth.transport import mtls # type: ignore + + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Remove these fallbacks when google-api-core >= 2.18.0 is the minimum required version. + + def use_client_cert_effective() -> bool: + """Returns whether client certificate should be used for mTLS.""" + if hasattr(mtls, "should_use_client_cert"): + return mtls.should_use_client_cert() + else: + use_client_cert_str = os.getenv( + "GOOGLE_API_USE_CLIENT_CERTIFICATE", "false" + ).lower() + if use_client_cert_str not in ("true", "false"): + raise ValueError( + "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be" + " either `true` or `false`" + ) + return use_client_cert_str == "true" + + def get_client_cert_source( + provided_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]], + use_cert_flag: bool, + ) -> Optional[Callable[[], Tuple[bytes, bytes]]]: + """Return the client cert source to be used by the client.""" + client_cert_source = None + if use_cert_flag: + if provided_cert_source: + client_cert_source = provided_cert_source + elif ( + hasattr(mtls, "has_default_client_cert_source") + and mtls.has_default_client_cert_source() + ): + client_cert_source = mtls.default_client_cert_source() + else: + raise ValueError( + "Client certificate is required for mTLS, but no client certificate source was provided or found." + ) + return client_cert_source + + def read_environment_variables() -> Tuple[bool, str, Optional[str]]: + """Returns the environment variables used by the client.""" + use_client_cert = use_client_cert_effective() + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower() + universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN") + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError( + "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` " + "must be `never`, `auto` or `always`" + ) + return use_client_cert, use_mtls_endpoint, universe_domain_env + + +try: + from google.api_core.gapic_v1.request import setup_request_id # type: ignore +except ImportError: # pragma: NO COVER + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Request ID setup helper was introduced in google-api-core 2.26.0. + # Remove this fallback definition when google-api-core >= 2.26.0 becomes the minimum required version in generated client setup dependencies. + def setup_request_id( + request: Any, field_name: str, is_proto3_optional: bool + ) -> None: + """Populate a UUID4 field in the request if it is not already set. + + Args: + request (Union[google.protobuf.message.Message, dict]): The request object. + field_name (str): The name of the field to populate. + is_proto3_optional (bool): Whether the field is proto3 optional. + """ + if request is None: + return + + request_id_val = str(uuid.uuid4()) + + if isinstance(request, dict): + if is_proto3_optional: + if field_name not in request or request[field_name] is None: + request[field_name] = request_id_val + elif not request.get(field_name): + request[field_name] = request_id_val + return + + if is_proto3_optional: + try: + # Pure protobuf messages + if not request.HasField(field_name): + setattr(request, field_name, request_id_val) + except (AttributeError, ValueError): + # Proto-plus messages or other objects + if hasattr(request, "_pb"): + try: + if not request._pb.HasField(field_name): + setattr(request, field_name, request_id_val) + return + except (AttributeError, ValueError): + pass + if getattr(request, field_name, None) is None: + setattr(request, field_name, request_id_val) + else: + if not getattr(request, field_name, None): + setattr(request, field_name, request_id_val) + + +try: + from google.api_core.rest_helpers import ( # type: ignore + flatten_query_params, + transcode_request, + ) +except ImportError: # pragma: NO COVER + # TODO: Remove these fallbacks when google-api-core >= 2.18.0 is the minimum required version. + from google.protobuf import json_format # type: ignore + from google.api_core import path_template # type: ignore + + def flatten_query_params(obj: Any, strict: bool = False) -> List[Tuple[str, Any]]: # type: ignore[misc] + if obj is not None and not isinstance(obj, dict): + raise TypeError("flatten_query_params must be called with dict object") + return _flatten(obj, key_path=[], strict=strict) + + def _flatten( + obj: Any, key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + if obj is None: + return [] + if isinstance(obj, dict): + return _flatten_dict(obj, key_path=key_path, strict=strict) + if isinstance(obj, list): + return _flatten_list(obj, key_path=key_path, strict=strict) + return _flatten_value(obj, key_path=key_path, strict=strict) + + def _is_primitive_value(obj: Any) -> bool: + if obj is None: + return False + if isinstance(obj, (list, dict)): + raise ValueError("query params may not contain repeated dicts or lists") + return True + + def _flatten_value( + obj: Any, key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + return [(".".join(key_path), _canonicalize(obj, strict=strict))] + + def _flatten_dict( + obj: Dict[str, Any], key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + items = ( + _flatten(value, key_path=key_path + [key], strict=strict) + for key, value in obj.items() + ) + return functools.reduce(operator.concat, items, []) # type: ignore[arg-type] + + def _flatten_list( + elems: List[Any], key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + items = ( + _flatten_value(elem, key_path=key_path, strict=strict) + for elem in elems + if _is_primitive_value(elem) + ) + return functools.reduce(operator.concat, items, []) # type: ignore[arg-type] + + def _canonicalize(obj: Any, strict: bool = False) -> Any: + if strict: + value = str(obj) + if isinstance(obj, bool): + value = value.lower() + return value + return obj + + def transcode_request( # type: ignore[misc] + http_options: List[Dict[str, str]], + request: Any, + required_fields_default_values: Optional[Dict[str, Any]] = None, + rest_numeric_enums: bool = False, + ) -> Tuple[Dict[str, Any], Optional[str], Dict[str, Any]]: + pb_request = getattr(request, "_pb", request) + transcoded_request = path_template.transcode(http_options, pb_request) + + body_json = None + if transcoded_request.get("body") is not None: + body_json = json_format.MessageToJson( + transcoded_request["body"], + use_integers_for_enums=rest_numeric_enums, + ) + + query_params_json = {} + if transcoded_request.get("query_params") is not None: + query_params_json = json.loads( + json_format.MessageToJson( + transcoded_request["query_params"], + use_integers_for_enums=rest_numeric_enums, + ) + ) + + if required_fields_default_values: + for k, v in required_fields_default_values.items(): + if k not in query_params_json: + query_params_json[k] = v + + if rest_numeric_enums: + query_params_json["$alt"] = "json;enum-encoding=int" + + return transcoded_request, body_json, query_params_json diff --git a/packages/gapic-generator/tests/integration/goldens/redis_selective/tests/unit/gapic/redis_v1/test_compat.py b/packages/gapic-generator/tests/integration/goldens/redis_selective/tests/unit/gapic/redis_v1/test_compat.py new file mode 100644 index 000000000000..7713b20e1892 --- /dev/null +++ b/packages/gapic-generator/tests/integration/goldens/redis_selective/tests/unit/gapic/redis_v1/test_compat.py @@ -0,0 +1,364 @@ +# # Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import builtins +import importlib +import sys +from unittest import mock +import pytest +from google.auth.exceptions import MutualTLSChannelError +from google.protobuf import descriptor_pb2 + + +from google.cloud.redis_v1 import _compat + + +def test_compat_normal_import(): + assert _compat.setup_request_id is not None + + +def test_compat_fallback_implementations(): + orig_import = builtins.__import__ + + def custom_import(name, globals=None, locals=None, fromlist=(), level=0): + if name in ( + "google.api_core.universe", + "google.api_core.gapic_v1.config", + "google.api_core.gapic_v1.request", + "google.api_core.rest_helpers", + ): + raise ImportError(f"Mocked ImportError for {name}") + return orig_import(name, globals, locals, fromlist, level) + + with mock.patch("builtins.__import__", side_effect=custom_import): + fallback = importlib.reload(_compat) + + # get_default_mtls_endpoint tests + assert fallback.get_default_mtls_endpoint(None) is None + assert fallback.get_default_mtls_endpoint("") == "" + assert ( + fallback.get_default_mtls_endpoint("foo.googleapis.com") + == "foo.mtls.googleapis.com" + ) + assert ( + fallback.get_default_mtls_endpoint("foo.sandbox.googleapis.com") + == "foo.mtls.sandbox.googleapis.com" + ) + assert ( + fallback.get_default_mtls_endpoint("foo.mtls.googleapis.com") + == "foo.mtls.googleapis.com" + ) + assert ( + fallback.get_default_mtls_endpoint("custom.domain.com") + == "custom.domain.com" + ) + assert ( + fallback.get_default_mtls_endpoint(":::invalid-url:::") + == ":::invalid-url:::" + ) + + # get_api_endpoint tests + assert ( + fallback.get_api_endpoint( + "https://override.com", + None, + "googleapis.com", + "auto", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "https://override.com" + ) + assert ( + fallback.get_api_endpoint( + None, + lambda: (b"", b""), + "googleapis.com", + "always", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "mtls.com" + ) + assert ( + fallback.get_api_endpoint( + None, + lambda: (b"", b""), + "googleapis.com", + "auto", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "mtls.com" + ) + with pytest.raises(MutualTLSChannelError): + fallback.get_api_endpoint( + None, + lambda: (b"", b""), + "otheruniverse.com", + "always", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + assert ( + fallback.get_api_endpoint( + None, + None, + "googleapis.com", + "never", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "https://googleapis.com" + ) + + # get_universe_domain tests + assert ( + fallback.get_universe_domain("custom.com", None, "googleapis.com") + == "custom.com" + ) + assert ( + fallback.get_universe_domain(None, "env.com", "googleapis.com") == "env.com" + ) + assert ( + fallback.get_universe_domain(None, None, "googleapis.com") + == "googleapis.com" + ) + with pytest.raises(ValueError): + fallback.get_universe_domain(" ", None, "googleapis.com") + + # use_client_cert_effective tests + with mock.patch( + "google.auth.transport.mtls.should_use_client_cert", + return_value=True, + create=True, + ): + assert fallback.use_client_cert_effective() is True + + with mock.patch.object(fallback, "mtls", spec=object()): + with mock.patch.dict( + "os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"} + ): + assert fallback.use_client_cert_effective() is True + with mock.patch.dict( + "os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"} + ): + assert fallback.use_client_cert_effective() is False + with mock.patch.dict( + "os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "invalid"} + ): + with pytest.raises(ValueError): + fallback.use_client_cert_effective() + + # get_client_cert_source tests + cert_fn = lambda: (b"cert", b"key") + assert fallback.get_client_cert_source(cert_fn, True) == cert_fn + assert fallback.get_client_cert_source(None, False) is None + + with mock.patch( + "google.auth.transport.mtls.has_default_client_cert_source", + return_value=True, + create=True, + ): + with mock.patch( + "google.auth.transport.mtls.default_client_cert_source", + return_value=cert_fn, + create=True, + ): + assert fallback.get_client_cert_source(None, True) == cert_fn + + with mock.patch.object(fallback, "mtls", spec=object()): + with pytest.raises(ValueError): + fallback.get_client_cert_source(None, True) + + # read_environment_variables tests + with mock.patch.dict( + "os.environ", + { + "GOOGLE_API_USE_MTLS_ENDPOINT": "always", + "GOOGLE_CLOUD_UNIVERSE_DOMAIN": "myuniverse.com", + }, + ): + use_cert, use_mtls, universe = fallback.read_environment_variables() + assert use_mtls == "always" + assert universe == "myuniverse.com" + + with mock.patch.dict("os.environ", {"GOOGLE_API_USE_MTLS_ENDPOINT": "invalid"}): + with pytest.raises(MutualTLSChannelError): + fallback.read_environment_variables() + + # setup_request_id tests + fallback.setup_request_id(None, "request_id", False) + + d1 = {} + fallback.setup_request_id(d1, "request_id", is_proto3_optional=True) + assert "request_id" in d1 + + d1_existing = {"request_id": None} + fallback.setup_request_id(d1_existing, "request_id", is_proto3_optional=True) + assert d1_existing["request_id"] is not None + + d2 = {} + fallback.setup_request_id(d2, "request_id", is_proto3_optional=False) + assert "request_id" in d2 + + d3 = {"request_id": "existing"} + fallback.setup_request_id(d3, "request_id", is_proto3_optional=False) + assert d3["request_id"] == "existing" + + d4 = {"request_id": "val"} + fallback.setup_request_id(d4, "request_id", is_proto3_optional=True) + assert d4["request_id"] == "val" + + class DummyPopulated: + def __init__(self): + self.request_id = "val" + + p_existing = DummyPopulated() + fallback.setup_request_id(p_existing, "request_id", is_proto3_optional=False) + assert p_existing.request_id == "val" + + class NonOptPlain: + def __init__(self): + self.request_id = "" + + nop = NonOptPlain() + fallback.setup_request_id(nop, "request_id", is_proto3_optional=False) + assert nop.request_id != "" + + class DummyProto: + def __init__(self): + self.request_id = "" + self._has = False + + def HasField(self, name): + if not self._has: + return False + return True + + p1 = DummyProto() + fallback.setup_request_id(p1, "request_id", is_proto3_optional=True) + assert p1.request_id != "" + + class DummyWrapper: + def __init__(self): + self._pb = DummyProto() + + w1 = DummyWrapper() + fallback.setup_request_id(w1, "request_id", is_proto3_optional=True) + assert ( + getattr(w1, "request_id", None) is not None + or getattr(w1._pb, "request_id", None) != "" + ) + + class BadProto: + def HasField(self, name): + raise AttributeError() + + class BadWrapper: + def __init__(self): + self._pb = BadProto() + self.request_id = None + + bw = BadWrapper() + fallback.setup_request_id(bw, "request_id", is_proto3_optional=True) + assert bw.request_id is not None + + class SetProto: + def __init__(self): + self.request_id = "already_set" + + def HasField(self, name): + return True + + sp = SetProto() + fallback.setup_request_id(sp, "request_id", is_proto3_optional=True) + assert sp.request_id == "already_set" + + class SetProtoNonOpt: + def __init__(self): + self.request_id = "already_set" + + sp_non_opt = SetProtoNonOpt() + fallback.setup_request_id(sp_non_opt, "request_id", is_proto3_optional=False) + assert sp_non_opt.request_id == "already_set" + + class DummyPlain: + def __init__(self): + self.request_id = None + + pl1 = DummyPlain() + fallback.setup_request_id(pl1, "request_id", is_proto3_optional=True) + assert pl1.request_id is not None + + pl2 = DummyPlain() + fallback.setup_request_id(pl2, "request_id", is_proto3_optional=False) + assert pl2.request_id is not None + + # flatten_query_params tests + assert fallback.flatten_query_params(None) == [] + res = fallback.flatten_query_params({"a": "val1", "b": [1, 2], "c": True}) + assert ("a", "val1") in res + assert ("b", 1) in res + assert ("b", 2) in res + assert ("c", True) in res + + res_strict = fallback.flatten_query_params({"c": True}, strict=True) + assert ("c", "true") in res_strict + + assert fallback._canonicalize(True, strict=True) == "true" + assert fallback._canonicalize(True, strict=False) is True + assert fallback._canonicalize(123, strict=True) == "123" + assert fallback._canonicalize("str", strict=True) == "str" + assert fallback._canonicalize("str", strict=False) == "str" + assert fallback._is_primitive_value(None) is False + + with pytest.raises(TypeError): + fallback.flatten_query_params("invalid") + + with pytest.raises(ValueError): + fallback.flatten_query_params({"a": [{"nested": "dict"}]}) + + with pytest.raises(ValueError): + fallback._is_primitive_value([1, 2]) + + # transcode_request tests + dummy_req = descriptor_pb2.DescriptorProto() + http_opts = [{"method": "get", "uri": "/v1/test"}] + transcoded, body, query = fallback.transcode_request( + http_opts, + dummy_req, + required_fields_default_values={"non_existent_key": "val"}, + rest_numeric_enums=True, + ) + assert transcoded is not None + assert query.get("non_existent_key") == "val" + assert query.get("$alt") == "json;enum-encoding=int" + + transcoded2, body2, query2 = fallback.transcode_request( + http_opts, + dummy_req, + required_fields_default_values={"name": "override_name"}, + rest_numeric_enums=False, + ) + assert body2 is None + assert "$alt" not in query2 + + importlib.reload(_compat) diff --git a/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/google/cloud/storagebatchoperations_v1/_compat.py b/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/google/cloud/storagebatchoperations_v1/_compat.py new file mode 100644 index 000000000000..54c22cd7d09e --- /dev/null +++ b/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/google/cloud/storagebatchoperations_v1/_compat.py @@ -0,0 +1,310 @@ +# # Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""A compatibility module for older versions of google-api-core.""" + +import functools +import json +import operator +import os +import re +import uuid +from typing import Any, Callable, Dict, List, Optional, Tuple + +from google.auth.exceptions import MutualTLSChannelError + + +try: + from google.api_core.universe import ( # type: ignore + get_default_mtls_endpoint, + get_api_endpoint, + get_universe_domain, + ) +except ImportError: # pragma: NO COVER + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Universe domain support was introduced in google-api-core 2.18.0. + # Remove these fallback definitions when google-api-core >= 2.18.0 becomes the minimum required version in generated client setup dependencies. + def get_default_mtls_endpoint(api_endpoint: Optional[str]) -> Optional[str]: + """Converts api endpoint to mTLS endpoint.""" + if not api_endpoint: + return api_endpoint + + mtls_endpoint_re = re.compile( + r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" + ) + + m = mtls_endpoint_re.match(api_endpoint) + if m is None: + # Could not parse api_endpoint; return as-is. + return api_endpoint + + name, mtls, sandbox, googledomain = m.groups() + if mtls or not googledomain: + return api_endpoint + + if sandbox: + return api_endpoint.replace( + "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" + ) + + return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") + + def get_api_endpoint( # type: ignore[misc] + api_override: Optional[str], + client_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]], + universe_domain: str, + use_mtls_endpoint: str, + default_universe: str, + default_mtls_endpoint: Optional[str], + default_endpoint_template: str, + ) -> str: + """Return the API endpoint used by the client.""" + api_endpoint: Optional[str] = None + if api_override is not None: + api_endpoint = api_override + elif use_mtls_endpoint == "always" or ( + use_mtls_endpoint == "auto" and client_cert_source + ): + if universe_domain != default_universe: + raise MutualTLSChannelError( + f"mTLS is not supported in any universe other than {default_universe}." + ) + api_endpoint = default_mtls_endpoint + else: + api_endpoint = default_endpoint_template.format( + UNIVERSE_DOMAIN=universe_domain + ) + return api_endpoint # type: ignore[return-value] + + def get_universe_domain( # type: ignore[misc] + *potential_universes: Optional[str], + default_universe: str = "googleapis.com", + ) -> str: + """Return the universe domain used by the client.""" + resolved = next( + (x.strip() for x in potential_universes if x is not None), + default_universe, + ) + if not resolved: + raise ValueError("Universe Domain cannot be an empty string.") + return resolved + + +try: + from google.api_core.gapic_v1.config import ( # type: ignore + use_client_cert_effective, + get_client_cert_source, + read_environment_variables, + ) +except ImportError: # pragma: NO COVER + from google.auth.transport import mtls # type: ignore + + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Remove these fallbacks when google-api-core >= 2.18.0 is the minimum required version. + + def use_client_cert_effective() -> bool: + """Returns whether client certificate should be used for mTLS.""" + if hasattr(mtls, "should_use_client_cert"): + return mtls.should_use_client_cert() + else: + use_client_cert_str = os.getenv( + "GOOGLE_API_USE_CLIENT_CERTIFICATE", "false" + ).lower() + if use_client_cert_str not in ("true", "false"): + raise ValueError( + "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be" + " either `true` or `false`" + ) + return use_client_cert_str == "true" + + def get_client_cert_source( + provided_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]], + use_cert_flag: bool, + ) -> Optional[Callable[[], Tuple[bytes, bytes]]]: + """Return the client cert source to be used by the client.""" + client_cert_source = None + if use_cert_flag: + if provided_cert_source: + client_cert_source = provided_cert_source + elif ( + hasattr(mtls, "has_default_client_cert_source") + and mtls.has_default_client_cert_source() + ): + client_cert_source = mtls.default_client_cert_source() + else: + raise ValueError( + "Client certificate is required for mTLS, but no client certificate source was provided or found." + ) + return client_cert_source + + def read_environment_variables() -> Tuple[bool, str, Optional[str]]: + """Returns the environment variables used by the client.""" + use_client_cert = use_client_cert_effective() + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower() + universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN") + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError( + "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` " + "must be `never`, `auto` or `always`" + ) + return use_client_cert, use_mtls_endpoint, universe_domain_env + + +try: + from google.api_core.gapic_v1.request import setup_request_id # type: ignore +except ImportError: # pragma: NO COVER + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Request ID setup helper was introduced in google-api-core 2.26.0. + # Remove this fallback definition when google-api-core >= 2.26.0 becomes the minimum required version in generated client setup dependencies. + def setup_request_id( + request: Any, field_name: str, is_proto3_optional: bool + ) -> None: + """Populate a UUID4 field in the request if it is not already set. + + Args: + request (Union[google.protobuf.message.Message, dict]): The request object. + field_name (str): The name of the field to populate. + is_proto3_optional (bool): Whether the field is proto3 optional. + """ + if request is None: + return + + request_id_val = str(uuid.uuid4()) + + if isinstance(request, dict): + if is_proto3_optional: + if field_name not in request or request[field_name] is None: + request[field_name] = request_id_val + elif not request.get(field_name): + request[field_name] = request_id_val + return + + if is_proto3_optional: + try: + # Pure protobuf messages + if not request.HasField(field_name): + setattr(request, field_name, request_id_val) + except (AttributeError, ValueError): + # Proto-plus messages or other objects + if hasattr(request, "_pb"): + try: + if not request._pb.HasField(field_name): + setattr(request, field_name, request_id_val) + return + except (AttributeError, ValueError): + pass + if getattr(request, field_name, None) is None: + setattr(request, field_name, request_id_val) + else: + if not getattr(request, field_name, None): + setattr(request, field_name, request_id_val) + + +try: + from google.api_core.rest_helpers import ( # type: ignore + flatten_query_params, + transcode_request, + ) +except ImportError: # pragma: NO COVER + # TODO: Remove these fallbacks when google-api-core >= 2.18.0 is the minimum required version. + from google.protobuf import json_format # type: ignore + from google.api_core import path_template # type: ignore + + def flatten_query_params(obj: Any, strict: bool = False) -> List[Tuple[str, Any]]: # type: ignore[misc] + if obj is not None and not isinstance(obj, dict): + raise TypeError("flatten_query_params must be called with dict object") + return _flatten(obj, key_path=[], strict=strict) + + def _flatten( + obj: Any, key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + if obj is None: + return [] + if isinstance(obj, dict): + return _flatten_dict(obj, key_path=key_path, strict=strict) + if isinstance(obj, list): + return _flatten_list(obj, key_path=key_path, strict=strict) + return _flatten_value(obj, key_path=key_path, strict=strict) + + def _is_primitive_value(obj: Any) -> bool: + if obj is None: + return False + if isinstance(obj, (list, dict)): + raise ValueError("query params may not contain repeated dicts or lists") + return True + + def _flatten_value( + obj: Any, key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + return [(".".join(key_path), _canonicalize(obj, strict=strict))] + + def _flatten_dict( + obj: Dict[str, Any], key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + items = ( + _flatten(value, key_path=key_path + [key], strict=strict) + for key, value in obj.items() + ) + return functools.reduce(operator.concat, items, []) # type: ignore[arg-type] + + def _flatten_list( + elems: List[Any], key_path: List[str], strict: bool = False + ) -> List[Tuple[str, Any]]: + items = ( + _flatten_value(elem, key_path=key_path, strict=strict) + for elem in elems + if _is_primitive_value(elem) + ) + return functools.reduce(operator.concat, items, []) # type: ignore[arg-type] + + def _canonicalize(obj: Any, strict: bool = False) -> Any: + if strict: + value = str(obj) + if isinstance(obj, bool): + value = value.lower() + return value + return obj + + def transcode_request( # type: ignore[misc] + http_options: List[Dict[str, str]], + request: Any, + required_fields_default_values: Optional[Dict[str, Any]] = None, + rest_numeric_enums: bool = False, + ) -> Tuple[Dict[str, Any], Optional[str], Dict[str, Any]]: + pb_request = getattr(request, "_pb", request) + transcoded_request = path_template.transcode(http_options, pb_request) + + body_json = None + if transcoded_request.get("body") is not None: + body_json = json_format.MessageToJson( + transcoded_request["body"], + use_integers_for_enums=rest_numeric_enums, + ) + + query_params_json = {} + if transcoded_request.get("query_params") is not None: + query_params_json = json.loads( + json_format.MessageToJson( + transcoded_request["query_params"], + use_integers_for_enums=rest_numeric_enums, + ) + ) + + if required_fields_default_values: + for k, v in required_fields_default_values.items(): + if k not in query_params_json: + query_params_json[k] = v + + if rest_numeric_enums: + query_params_json["$alt"] = "json;enum-encoding=int" + + return transcoded_request, body_json, query_params_json diff --git a/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/tests/unit/gapic/storagebatchoperations_v1/test_compat.py b/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/tests/unit/gapic/storagebatchoperations_v1/test_compat.py new file mode 100644 index 000000000000..37fcf59e5f71 --- /dev/null +++ b/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/tests/unit/gapic/storagebatchoperations_v1/test_compat.py @@ -0,0 +1,364 @@ +# # Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import builtins +import importlib +import sys +from unittest import mock +import pytest +from google.auth.exceptions import MutualTLSChannelError +from google.protobuf import descriptor_pb2 + + +from google.cloud.storagebatchoperations_v1 import _compat + + +def test_compat_normal_import(): + assert _compat.setup_request_id is not None + + +def test_compat_fallback_implementations(): + orig_import = builtins.__import__ + + def custom_import(name, globals=None, locals=None, fromlist=(), level=0): + if name in ( + "google.api_core.universe", + "google.api_core.gapic_v1.config", + "google.api_core.gapic_v1.request", + "google.api_core.rest_helpers", + ): + raise ImportError(f"Mocked ImportError for {name}") + return orig_import(name, globals, locals, fromlist, level) + + with mock.patch("builtins.__import__", side_effect=custom_import): + fallback = importlib.reload(_compat) + + # get_default_mtls_endpoint tests + assert fallback.get_default_mtls_endpoint(None) is None + assert fallback.get_default_mtls_endpoint("") == "" + assert ( + fallback.get_default_mtls_endpoint("foo.googleapis.com") + == "foo.mtls.googleapis.com" + ) + assert ( + fallback.get_default_mtls_endpoint("foo.sandbox.googleapis.com") + == "foo.mtls.sandbox.googleapis.com" + ) + assert ( + fallback.get_default_mtls_endpoint("foo.mtls.googleapis.com") + == "foo.mtls.googleapis.com" + ) + assert ( + fallback.get_default_mtls_endpoint("custom.domain.com") + == "custom.domain.com" + ) + assert ( + fallback.get_default_mtls_endpoint(":::invalid-url:::") + == ":::invalid-url:::" + ) + + # get_api_endpoint tests + assert ( + fallback.get_api_endpoint( + "https://override.com", + None, + "googleapis.com", + "auto", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "https://override.com" + ) + assert ( + fallback.get_api_endpoint( + None, + lambda: (b"", b""), + "googleapis.com", + "always", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "mtls.com" + ) + assert ( + fallback.get_api_endpoint( + None, + lambda: (b"", b""), + "googleapis.com", + "auto", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "mtls.com" + ) + with pytest.raises(MutualTLSChannelError): + fallback.get_api_endpoint( + None, + lambda: (b"", b""), + "otheruniverse.com", + "always", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + assert ( + fallback.get_api_endpoint( + None, + None, + "googleapis.com", + "never", + "googleapis.com", + "mtls.com", + "https://{UNIVERSE_DOMAIN}", + ) + == "https://googleapis.com" + ) + + # get_universe_domain tests + assert ( + fallback.get_universe_domain("custom.com", None, "googleapis.com") + == "custom.com" + ) + assert ( + fallback.get_universe_domain(None, "env.com", "googleapis.com") == "env.com" + ) + assert ( + fallback.get_universe_domain(None, None, "googleapis.com") + == "googleapis.com" + ) + with pytest.raises(ValueError): + fallback.get_universe_domain(" ", None, "googleapis.com") + + # use_client_cert_effective tests + with mock.patch( + "google.auth.transport.mtls.should_use_client_cert", + return_value=True, + create=True, + ): + assert fallback.use_client_cert_effective() is True + + with mock.patch.object(fallback, "mtls", spec=object()): + with mock.patch.dict( + "os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"} + ): + assert fallback.use_client_cert_effective() is True + with mock.patch.dict( + "os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"} + ): + assert fallback.use_client_cert_effective() is False + with mock.patch.dict( + "os.environ", {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "invalid"} + ): + with pytest.raises(ValueError): + fallback.use_client_cert_effective() + + # get_client_cert_source tests + cert_fn = lambda: (b"cert", b"key") + assert fallback.get_client_cert_source(cert_fn, True) == cert_fn + assert fallback.get_client_cert_source(None, False) is None + + with mock.patch( + "google.auth.transport.mtls.has_default_client_cert_source", + return_value=True, + create=True, + ): + with mock.patch( + "google.auth.transport.mtls.default_client_cert_source", + return_value=cert_fn, + create=True, + ): + assert fallback.get_client_cert_source(None, True) == cert_fn + + with mock.patch.object(fallback, "mtls", spec=object()): + with pytest.raises(ValueError): + fallback.get_client_cert_source(None, True) + + # read_environment_variables tests + with mock.patch.dict( + "os.environ", + { + "GOOGLE_API_USE_MTLS_ENDPOINT": "always", + "GOOGLE_CLOUD_UNIVERSE_DOMAIN": "myuniverse.com", + }, + ): + use_cert, use_mtls, universe = fallback.read_environment_variables() + assert use_mtls == "always" + assert universe == "myuniverse.com" + + with mock.patch.dict("os.environ", {"GOOGLE_API_USE_MTLS_ENDPOINT": "invalid"}): + with pytest.raises(MutualTLSChannelError): + fallback.read_environment_variables() + + # setup_request_id tests + fallback.setup_request_id(None, "request_id", False) + + d1 = {} + fallback.setup_request_id(d1, "request_id", is_proto3_optional=True) + assert "request_id" in d1 + + d1_existing = {"request_id": None} + fallback.setup_request_id(d1_existing, "request_id", is_proto3_optional=True) + assert d1_existing["request_id"] is not None + + d2 = {} + fallback.setup_request_id(d2, "request_id", is_proto3_optional=False) + assert "request_id" in d2 + + d3 = {"request_id": "existing"} + fallback.setup_request_id(d3, "request_id", is_proto3_optional=False) + assert d3["request_id"] == "existing" + + d4 = {"request_id": "val"} + fallback.setup_request_id(d4, "request_id", is_proto3_optional=True) + assert d4["request_id"] == "val" + + class DummyPopulated: + def __init__(self): + self.request_id = "val" + + p_existing = DummyPopulated() + fallback.setup_request_id(p_existing, "request_id", is_proto3_optional=False) + assert p_existing.request_id == "val" + + class NonOptPlain: + def __init__(self): + self.request_id = "" + + nop = NonOptPlain() + fallback.setup_request_id(nop, "request_id", is_proto3_optional=False) + assert nop.request_id != "" + + class DummyProto: + def __init__(self): + self.request_id = "" + self._has = False + + def HasField(self, name): + if not self._has: + return False + return True + + p1 = DummyProto() + fallback.setup_request_id(p1, "request_id", is_proto3_optional=True) + assert p1.request_id != "" + + class DummyWrapper: + def __init__(self): + self._pb = DummyProto() + + w1 = DummyWrapper() + fallback.setup_request_id(w1, "request_id", is_proto3_optional=True) + assert ( + getattr(w1, "request_id", None) is not None + or getattr(w1._pb, "request_id", None) != "" + ) + + class BadProto: + def HasField(self, name): + raise AttributeError() + + class BadWrapper: + def __init__(self): + self._pb = BadProto() + self.request_id = None + + bw = BadWrapper() + fallback.setup_request_id(bw, "request_id", is_proto3_optional=True) + assert bw.request_id is not None + + class SetProto: + def __init__(self): + self.request_id = "already_set" + + def HasField(self, name): + return True + + sp = SetProto() + fallback.setup_request_id(sp, "request_id", is_proto3_optional=True) + assert sp.request_id == "already_set" + + class SetProtoNonOpt: + def __init__(self): + self.request_id = "already_set" + + sp_non_opt = SetProtoNonOpt() + fallback.setup_request_id(sp_non_opt, "request_id", is_proto3_optional=False) + assert sp_non_opt.request_id == "already_set" + + class DummyPlain: + def __init__(self): + self.request_id = None + + pl1 = DummyPlain() + fallback.setup_request_id(pl1, "request_id", is_proto3_optional=True) + assert pl1.request_id is not None + + pl2 = DummyPlain() + fallback.setup_request_id(pl2, "request_id", is_proto3_optional=False) + assert pl2.request_id is not None + + # flatten_query_params tests + assert fallback.flatten_query_params(None) == [] + res = fallback.flatten_query_params({"a": "val1", "b": [1, 2], "c": True}) + assert ("a", "val1") in res + assert ("b", 1) in res + assert ("b", 2) in res + assert ("c", True) in res + + res_strict = fallback.flatten_query_params({"c": True}, strict=True) + assert ("c", "true") in res_strict + + assert fallback._canonicalize(True, strict=True) == "true" + assert fallback._canonicalize(True, strict=False) is True + assert fallback._canonicalize(123, strict=True) == "123" + assert fallback._canonicalize("str", strict=True) == "str" + assert fallback._canonicalize("str", strict=False) == "str" + assert fallback._is_primitive_value(None) is False + + with pytest.raises(TypeError): + fallback.flatten_query_params("invalid") + + with pytest.raises(ValueError): + fallback.flatten_query_params({"a": [{"nested": "dict"}]}) + + with pytest.raises(ValueError): + fallback._is_primitive_value([1, 2]) + + # transcode_request tests + dummy_req = descriptor_pb2.DescriptorProto() + http_opts = [{"method": "get", "uri": "/v1/test"}] + transcoded, body, query = fallback.transcode_request( + http_opts, + dummy_req, + required_fields_default_values={"non_existent_key": "val"}, + rest_numeric_enums=True, + ) + assert transcoded is not None + assert query.get("non_existent_key") == "val" + assert query.get("$alt") == "json;enum-encoding=int" + + transcoded2, body2, query2 = fallback.transcode_request( + http_opts, + dummy_req, + required_fields_default_values={"name": "override_name"}, + rest_numeric_enums=False, + ) + assert body2 is None + assert "$alt" not in query2 + + importlib.reload(_compat) diff --git a/packages/gapic-generator/tests/unit/generator/test_generator.py b/packages/gapic-generator/tests/unit/generator/test_generator.py index 9d8545c4192f..6b80a20c0d79 100644 --- a/packages/gapic-generator/tests/unit/generator/test_generator.py +++ b/packages/gapic-generator/tests/unit/generator/test_generator.py @@ -132,8 +132,30 @@ def test_get_response_ignores_private_files(): ] ) assert len(cgr.file) == 1 - assert cgr.file[0].name == "foo/bar/baz.py" - assert cgr.file[0].content == "I am a template result.\n" + + +def test_get_response_renders_allowed_private_templates(): + generator_obj = make_generator() + with mock.patch.object(jinja2.FileSystemLoader, "list_templates") as list_templates: + list_templates.return_value = [ + "foo/bar/__init__.py.j2", + "foo/bar/_compat.py.j2", + "foo/bar/_ignored.py.j2", + ] + with mock.patch.object(jinja2.Environment, "get_template") as get_template: + get_template.return_value = jinja2.Template("I am a template result.") + cgr = generator_obj.get_response( + api_schema=make_api(), opts=Options.build("") + ) + list_templates.assert_called_once() + get_template.assert_has_calls( + [ + mock.call("foo/bar/__init__.py.j2"), + mock.call("foo/bar/_compat.py.j2"), + ], + any_order=True, + ) + assert len(cgr.file) == 2 def test_get_response_fails_invalid_file_paths(): diff --git a/packages/google-api-core/pyproject.toml b/packages/google-api-core/pyproject.toml index 28c42be84295..69eddd7f05cf 100644 --- a/packages/google-api-core/pyproject.toml +++ b/packages/google-api-core/pyproject.toml @@ -77,6 +77,9 @@ version = { attr = "google.api_core.version.__version__" } # benchmarks, etc. include = ["google*"] +[tool.setuptools.package-data] +"*" = ["py.typed"] + [tool.mypy] python_version = "3.14" namespace_packages = true