diff --git a/google/cloud/datastore_admin_v1/__init__.py b/google/cloud/datastore_admin_v1/__init__.py index 70a79c07..d93e680e 100644 --- a/google/cloud/datastore_admin_v1/__init__.py +++ b/google/cloud/datastore_admin_v1/__init__.py @@ -19,6 +19,7 @@ from .types.datastore_admin import CommonMetadata from .types.datastore_admin import CreateIndexRequest +from .types.datastore_admin import DatastoreFirestoreMigrationMetadata from .types.datastore_admin import DeleteIndexRequest from .types.datastore_admin import EntityFilter from .types.datastore_admin import ExportEntitiesMetadata @@ -33,12 +34,17 @@ from .types.datastore_admin import Progress from .types.datastore_admin import OperationType from .types.index import Index +from .types.migration import MigrationProgressEvent +from .types.migration import MigrationStateEvent +from .types.migration import MigrationState +from .types.migration import MigrationStep __all__ = ( "DatastoreAdminAsyncClient", "CommonMetadata", "CreateIndexRequest", "DatastoreAdminClient", + "DatastoreFirestoreMigrationMetadata", "DeleteIndexRequest", "EntityFilter", "ExportEntitiesMetadata", @@ -51,6 +57,10 @@ "IndexOperationMetadata", "ListIndexesRequest", "ListIndexesResponse", + "MigrationProgressEvent", + "MigrationState", + "MigrationStateEvent", + "MigrationStep", "OperationType", "Progress", ) diff --git a/google/cloud/datastore_admin_v1/services/datastore_admin/async_client.py b/google/cloud/datastore_admin_v1/services/datastore_admin/async_client.py index c64a328c..63026a35 100644 --- a/google/cloud/datastore_admin_v1/services/datastore_admin/async_client.py +++ b/google/cloud/datastore_admin_v1/services/datastore_admin/async_client.py @@ -16,7 +16,7 @@ from collections import OrderedDict import functools import re -from typing import Dict, Sequence, Tuple, Type, Union +from typing import Dict, Optional, Sequence, Tuple, Type, Union import pkg_resources from google.api_core.client_options import ClientOptions @@ -31,12 +31,12 @@ except AttributeError: # pragma: NO COVER OptionalRetry = Union[retries.Retry, object] # type: ignore -from google.api_core import operation -from google.api_core import operation_async +from google.api_core import operation # type: ignore +from google.api_core import operation_async # type: ignore from google.cloud.datastore_admin_v1.services.datastore_admin import pagers from google.cloud.datastore_admin_v1.types import datastore_admin from google.cloud.datastore_admin_v1.types import index -from google.protobuf import empty_pb2 +from google.protobuf import empty_pb2 # type: ignore from .transports.base import DatastoreAdminTransport, DEFAULT_CLIENT_INFO from .transports.grpc_asyncio import DatastoreAdminGrpcAsyncIOTransport from .client import DatastoreAdminClient @@ -166,6 +166,42 @@ def from_service_account_file(cls, filename: str, *args, **kwargs): from_service_account_json = from_service_account_file + @classmethod + def get_mtls_endpoint_and_cert_source( + cls, client_options: Optional[ClientOptions] = None + ): + """Return the API endpoint and client cert source for mutual TLS. + + The client cert source is determined in the following order: + (1) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is not "true", the + client cert source is None. + (2) if `client_options.client_cert_source` is provided, use the provided one; if the + default client cert source exists, use the default one; otherwise the client cert + source is None. + + The API endpoint is determined in the following order: + (1) if `client_options.api_endpoint` if provided, use the provided one. + (2) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is "always", use the + default mTLS endpoint; if the environment variabel is "never", use the default API + endpoint; otherwise if client cert source exists, use the default mTLS endpoint, otherwise + use the default API endpoint. + + More details can be found at https://google.aip.dev/auth/4114. + + Args: + client_options (google.api_core.client_options.ClientOptions): Custom options for the + client. Only the `api_endpoint` and `client_cert_source` properties may be used + in this method. + + Returns: + Tuple[str, Callable[[], Tuple[bytes, bytes]]]: returns the API endpoint and the + client cert source to use. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If any errors happen. + """ + return DatastoreAdminClient.get_mtls_endpoint_and_cert_source(client_options) # type: ignore + @property def transport(self) -> DatastoreAdminTransport: """Returns the transport used by the client instance. diff --git a/google/cloud/datastore_admin_v1/services/datastore_admin/client.py b/google/cloud/datastore_admin_v1/services/datastore_admin/client.py index 695446c5..461757fd 100644 --- a/google/cloud/datastore_admin_v1/services/datastore_admin/client.py +++ b/google/cloud/datastore_admin_v1/services/datastore_admin/client.py @@ -34,12 +34,12 @@ except AttributeError: # pragma: NO COVER OptionalRetry = Union[retries.Retry, object] # type: ignore -from google.api_core import operation -from google.api_core import operation_async +from google.api_core import operation # type: ignore +from google.api_core import operation_async # type: ignore from google.cloud.datastore_admin_v1.services.datastore_admin import pagers from google.cloud.datastore_admin_v1.types import datastore_admin from google.cloud.datastore_admin_v1.types import index -from google.protobuf import empty_pb2 +from google.protobuf import empty_pb2 # type: ignore from .transports.base import DatastoreAdminTransport, DEFAULT_CLIENT_INFO from .transports.grpc import DatastoreAdminGrpcTransport from .transports.grpc_asyncio import DatastoreAdminGrpcAsyncIOTransport @@ -277,6 +277,73 @@ def parse_common_location_path(path: str) -> Dict[str, str]: m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)$", path) return m.groupdict() if m else {} + @classmethod + def get_mtls_endpoint_and_cert_source( + cls, client_options: Optional[client_options_lib.ClientOptions] = None + ): + """Return the API endpoint and client cert source for mutual TLS. + + The client cert source is determined in the following order: + (1) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is not "true", the + client cert source is None. + (2) if `client_options.client_cert_source` is provided, use the provided one; if the + default client cert source exists, use the default one; otherwise the client cert + source is None. + + The API endpoint is determined in the following order: + (1) if `client_options.api_endpoint` if provided, use the provided one. + (2) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is "always", use the + default mTLS endpoint; if the environment variabel is "never", use the default API + endpoint; otherwise if client cert source exists, use the default mTLS endpoint, otherwise + use the default API endpoint. + + More details can be found at https://google.aip.dev/auth/4114. + + Args: + client_options (google.api_core.client_options.ClientOptions): Custom options for the + client. Only the `api_endpoint` and `client_cert_source` properties may be used + in this method. + + Returns: + Tuple[str, Callable[[], Tuple[bytes, bytes]]]: returns the API endpoint and the + client cert source to use. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If any errors happen. + """ + if client_options is None: + client_options = client_options_lib.ClientOptions() + use_client_cert = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto") + if use_client_cert not in ("true", "false"): + raise ValueError( + "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + ) + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError( + "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + ) + + # Figure out the client cert source to use. + client_cert_source = None + if use_client_cert == "true": + if client_options.client_cert_source: + client_cert_source = client_options.client_cert_source + elif mtls.has_default_client_cert_source(): + client_cert_source = mtls.default_client_cert_source() + + # Figure out which api endpoint to use. + if client_options.api_endpoint is not None: + api_endpoint = client_options.api_endpoint + elif use_mtls_endpoint == "always" or ( + use_mtls_endpoint == "auto" and client_cert_source + ): + api_endpoint = cls.DEFAULT_MTLS_ENDPOINT + else: + api_endpoint = cls.DEFAULT_ENDPOINT + + return api_endpoint, client_cert_source + def __init__( self, *, @@ -327,57 +394,22 @@ def __init__( if client_options is None: client_options = client_options_lib.ClientOptions() - # Create SSL credentials for mutual TLS if needed. - if os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") not in ( - "true", - "false", - ): - raise ValueError( - "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) - use_client_cert = ( - os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true" + api_endpoint, client_cert_source_func = self.get_mtls_endpoint_and_cert_source( + client_options ) - client_cert_source_func = None - is_mtls = False - if use_client_cert: - if client_options.client_cert_source: - is_mtls = True - client_cert_source_func = client_options.client_cert_source - else: - is_mtls = mtls.has_default_client_cert_source() - if is_mtls: - client_cert_source_func = mtls.default_client_cert_source() - else: - client_cert_source_func = None - - # Figure out which api endpoint to use. - if client_options.api_endpoint is not None: - api_endpoint = client_options.api_endpoint - else: - use_mtls_env = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto") - if use_mtls_env == "never": - api_endpoint = self.DEFAULT_ENDPOINT - elif use_mtls_env == "always": - api_endpoint = self.DEFAULT_MTLS_ENDPOINT - elif use_mtls_env == "auto": - if is_mtls: - api_endpoint = self.DEFAULT_MTLS_ENDPOINT - else: - api_endpoint = self.DEFAULT_ENDPOINT - else: - raise MutualTLSChannelError( - "Unsupported GOOGLE_API_USE_MTLS_ENDPOINT value. Accepted " - "values: never, auto, always" - ) + api_key_value = getattr(client_options, "api_key", None) + if api_key_value and credentials: + raise ValueError( + "client_options.api_key and credentials are mutually exclusive" + ) # Save or instantiate the transport. # Ordinarily, we provide the transport, but allowing a custom transport # instance provides an extensibility point for unusual situations. if isinstance(transport, DatastoreAdminTransport): # transport is a DatastoreAdminTransport instance. - if credentials or client_options.credentials_file: + if credentials or client_options.credentials_file or api_key_value: raise ValueError( "When providing a transport instance, " "provide its credentials directly." @@ -389,6 +421,15 @@ def __init__( ) self._transport = transport else: + import google.auth._default # type: ignore + + if api_key_value and hasattr( + google.auth._default, "get_api_key_credentials" + ): + credentials = google.auth._default.get_api_key_credentials( + api_key_value + ) + Transport = type(self).get_transport_class(transport) self._transport = Transport( credentials=credentials, diff --git a/google/cloud/datastore_admin_v1/services/datastore_admin/transports/base.py b/google/cloud/datastore_admin_v1/services/datastore_admin/transports/base.py index 58358b0a..02c832ab 100644 --- a/google/cloud/datastore_admin_v1/services/datastore_admin/transports/base.py +++ b/google/cloud/datastore_admin_v1/services/datastore_admin/transports/base.py @@ -28,7 +28,7 @@ from google.cloud.datastore_admin_v1.types import datastore_admin from google.cloud.datastore_admin_v1.types import index -from google.longrunning import operations_pb2 +from google.longrunning import operations_pb2 # type: ignore try: DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( @@ -107,7 +107,6 @@ def __init__( credentials, _ = google.auth.load_credentials_from_file( credentials_file, **scopes_kwargs, quota_project_id=quota_project_id ) - elif credentials is None: credentials, _ = google.auth.default( **scopes_kwargs, quota_project_id=quota_project_id diff --git a/google/cloud/datastore_admin_v1/services/datastore_admin/transports/grpc.py b/google/cloud/datastore_admin_v1/services/datastore_admin/transports/grpc.py index d7018fcb..3d08e9db 100644 --- a/google/cloud/datastore_admin_v1/services/datastore_admin/transports/grpc.py +++ b/google/cloud/datastore_admin_v1/services/datastore_admin/transports/grpc.py @@ -23,11 +23,11 @@ from google.auth import credentials as ga_credentials # type: ignore from google.auth.transport.grpc import SslCredentials # type: ignore -import grpc +import grpc # type: ignore from google.cloud.datastore_admin_v1.types import datastore_admin from google.cloud.datastore_admin_v1.types import index -from google.longrunning import operations_pb2 +from google.longrunning import operations_pb2 # type: ignore from .base import DatastoreAdminTransport, DEFAULT_CLIENT_INFO diff --git a/google/cloud/datastore_admin_v1/services/datastore_admin/transports/grpc_asyncio.py b/google/cloud/datastore_admin_v1/services/datastore_admin/transports/grpc_asyncio.py index 84b5299b..3f9e5330 100644 --- a/google/cloud/datastore_admin_v1/services/datastore_admin/transports/grpc_asyncio.py +++ b/google/cloud/datastore_admin_v1/services/datastore_admin/transports/grpc_asyncio.py @@ -22,12 +22,12 @@ from google.auth import credentials as ga_credentials # type: ignore from google.auth.transport.grpc import SslCredentials # type: ignore -import grpc -from grpc.experimental import aio +import grpc # type: ignore +from grpc.experimental import aio # type: ignore from google.cloud.datastore_admin_v1.types import datastore_admin from google.cloud.datastore_admin_v1.types import index -from google.longrunning import operations_pb2 +from google.longrunning import operations_pb2 # type: ignore from .base import DatastoreAdminTransport, DEFAULT_CLIENT_INFO from .grpc import DatastoreAdminGrpcTransport diff --git a/google/cloud/datastore_admin_v1/types/__init__.py b/google/cloud/datastore_admin_v1/types/__init__.py index ac4ff905..ccda56e1 100644 --- a/google/cloud/datastore_admin_v1/types/__init__.py +++ b/google/cloud/datastore_admin_v1/types/__init__.py @@ -16,6 +16,7 @@ from .datastore_admin import ( CommonMetadata, CreateIndexRequest, + DatastoreFirestoreMigrationMetadata, DeleteIndexRequest, EntityFilter, ExportEntitiesMetadata, @@ -31,10 +32,17 @@ OperationType, ) from .index import Index +from .migration import ( + MigrationProgressEvent, + MigrationStateEvent, + MigrationState, + MigrationStep, +) __all__ = ( "CommonMetadata", "CreateIndexRequest", + "DatastoreFirestoreMigrationMetadata", "DeleteIndexRequest", "EntityFilter", "ExportEntitiesMetadata", @@ -49,4 +57,8 @@ "Progress", "OperationType", "Index", + "MigrationProgressEvent", + "MigrationStateEvent", + "MigrationState", + "MigrationStep", ) diff --git a/google/cloud/datastore_admin_v1/types/datastore_admin.py b/google/cloud/datastore_admin_v1/types/datastore_admin.py index 0d42d802..14a2871f 100644 --- a/google/cloud/datastore_admin_v1/types/datastore_admin.py +++ b/google/cloud/datastore_admin_v1/types/datastore_admin.py @@ -13,10 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import proto +import proto # type: ignore from google.cloud.datastore_admin_v1.types import index as gda_index -from google.protobuf import timestamp_pb2 +from google.cloud.datastore_admin_v1.types import migration +from google.protobuf import timestamp_pb2 # type: ignore __protobuf__ = proto.module( @@ -37,6 +38,7 @@ "ListIndexesRequest", "ListIndexesResponse", "IndexOperationMetadata", + "DatastoreFirestoreMigrationMetadata", }, ) @@ -410,4 +412,27 @@ class IndexOperationMetadata(proto.Message): index_id = proto.Field(proto.STRING, number=3,) +class DatastoreFirestoreMigrationMetadata(proto.Message): + r"""Metadata for Datastore to Firestore migration operations. + + The DatastoreFirestoreMigration operation is not started by the + end-user via an explicit "creation" method. This is an intentional + deviation from the LRO design pattern. + + This singleton resource can be accessed at: + ``projects/{project_id}/datastore-firestore-migration`` + + Attributes: + migration_state (google.cloud.datastore_admin_v1.types.MigrationState): + The current state of migration from Cloud + Datastore to Cloud Firestore in Datastore mode. + migration_step (google.cloud.datastore_admin_v1.types.MigrationStep): + The current step of migration from Cloud + Datastore to Cloud Firestore in Datastore mode. + """ + + migration_state = proto.Field(proto.ENUM, number=1, enum=migration.MigrationState,) + migration_step = proto.Field(proto.ENUM, number=2, enum=migration.MigrationStep,) + + __all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/google/cloud/datastore_admin_v1/types/index.py b/google/cloud/datastore_admin_v1/types/index.py index 7a57c945..b372cccf 100644 --- a/google/cloud/datastore_admin_v1/types/index.py +++ b/google/cloud/datastore_admin_v1/types/index.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import proto +import proto # type: ignore __protobuf__ = proto.module(package="google.datastore.admin.v1", manifest={"Index",},) diff --git a/google/cloud/datastore_admin_v1/types/migration.py b/google/cloud/datastore_admin_v1/types/migration.py new file mode 100644 index 00000000..92569f8f --- /dev/null +++ b/google/cloud/datastore_admin_v1/types/migration.py @@ -0,0 +1,135 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import proto # type: ignore + + +__protobuf__ = proto.module( + package="google.datastore.admin.v1", + manifest={ + "MigrationState", + "MigrationStep", + "MigrationStateEvent", + "MigrationProgressEvent", + }, +) + + +class MigrationState(proto.Enum): + r"""States for a migration.""" + MIGRATION_STATE_UNSPECIFIED = 0 + RUNNING = 1 + PAUSED = 2 + COMPLETE = 3 + + +class MigrationStep(proto.Enum): + r"""Steps in a migration.""" + MIGRATION_STEP_UNSPECIFIED = 0 + PREPARE = 6 + START = 1 + APPLY_WRITES_SYNCHRONOUSLY = 7 + COPY_AND_VERIFY = 2 + REDIRECT_EVENTUALLY_CONSISTENT_READS = 3 + REDIRECT_STRONGLY_CONSISTENT_READS = 4 + REDIRECT_WRITES = 5 + + +class MigrationStateEvent(proto.Message): + r"""An event signifying a change in state of a `migration from Cloud + Datastore to Cloud Firestore in Datastore + mode `__. + + Attributes: + state (google.cloud.datastore_admin_v1.types.MigrationState): + The new state of the migration. + """ + + state = proto.Field(proto.ENUM, number=1, enum="MigrationState",) + + +class MigrationProgressEvent(proto.Message): + r"""An event signifying the start of a new step in a `migration from + Cloud Datastore to Cloud Firestore in Datastore + mode `__. + + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + step (google.cloud.datastore_admin_v1.types.MigrationStep): + The step that is starting. + + An event with step set to ``START`` indicates that the + migration has been reverted back to the initial + pre-migration state. + prepare_step_details (google.cloud.datastore_admin_v1.types.MigrationProgressEvent.PrepareStepDetails): + Details for the ``PREPARE`` step. + + This field is a member of `oneof`_ ``step_details``. + redirect_writes_step_details (google.cloud.datastore_admin_v1.types.MigrationProgressEvent.RedirectWritesStepDetails): + Details for the ``REDIRECT_WRITES`` step. + + This field is a member of `oneof`_ ``step_details``. + """ + + class ConcurrencyMode(proto.Enum): + r"""Concurrency modes for transactions in Cloud Firestore.""" + CONCURRENCY_MODE_UNSPECIFIED = 0 + PESSIMISTIC = 1 + OPTIMISTIC = 2 + + class PrepareStepDetails(proto.Message): + r"""Details for the ``PREPARE`` step. + + Attributes: + concurrency_mode (google.cloud.datastore_admin_v1.types.MigrationProgressEvent.ConcurrencyMode): + The concurrency mode this database will use when it reaches + the ``REDIRECT_WRITES`` step. + """ + + concurrency_mode = proto.Field( + proto.ENUM, number=1, enum="MigrationProgressEvent.ConcurrencyMode", + ) + + class RedirectWritesStepDetails(proto.Message): + r"""Details for the ``REDIRECT_WRITES`` step. + + Attributes: + concurrency_mode (google.cloud.datastore_admin_v1.types.MigrationProgressEvent.ConcurrencyMode): + Ths concurrency mode for this database. + """ + + concurrency_mode = proto.Field( + proto.ENUM, number=1, enum="MigrationProgressEvent.ConcurrencyMode", + ) + + step = proto.Field(proto.ENUM, number=1, enum="MigrationStep",) + prepare_step_details = proto.Field( + proto.MESSAGE, number=2, oneof="step_details", message=PrepareStepDetails, + ) + redirect_writes_step_details = proto.Field( + proto.MESSAGE, + number=3, + oneof="step_details", + message=RedirectWritesStepDetails, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/google/cloud/datastore_v1/services/datastore/async_client.py b/google/cloud/datastore_v1/services/datastore/async_client.py index 2246300e..d313c5bd 100644 --- a/google/cloud/datastore_v1/services/datastore/async_client.py +++ b/google/cloud/datastore_v1/services/datastore/async_client.py @@ -16,7 +16,7 @@ from collections import OrderedDict import functools import re -from typing import Dict, Sequence, Tuple, Type, Union +from typing import Dict, Optional, Sequence, Tuple, Type, Union import pkg_resources from google.api_core.client_options import ClientOptions @@ -106,6 +106,42 @@ def from_service_account_file(cls, filename: str, *args, **kwargs): from_service_account_json = from_service_account_file + @classmethod + def get_mtls_endpoint_and_cert_source( + cls, client_options: Optional[ClientOptions] = None + ): + """Return the API endpoint and client cert source for mutual TLS. + + The client cert source is determined in the following order: + (1) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is not "true", the + client cert source is None. + (2) if `client_options.client_cert_source` is provided, use the provided one; if the + default client cert source exists, use the default one; otherwise the client cert + source is None. + + The API endpoint is determined in the following order: + (1) if `client_options.api_endpoint` if provided, use the provided one. + (2) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is "always", use the + default mTLS endpoint; if the environment variabel is "never", use the default API + endpoint; otherwise if client cert source exists, use the default mTLS endpoint, otherwise + use the default API endpoint. + + More details can be found at https://google.aip.dev/auth/4114. + + Args: + client_options (google.api_core.client_options.ClientOptions): Custom options for the + client. Only the `api_endpoint` and `client_cert_source` properties may be used + in this method. + + Returns: + Tuple[str, Callable[[], Tuple[bytes, bytes]]]: returns the API endpoint and the + client cert source to use. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If any errors happen. + """ + return DatastoreClient.get_mtls_endpoint_and_cert_source(client_options) # type: ignore + @property def transport(self) -> DatastoreTransport: """Returns the transport used by the client instance. diff --git a/google/cloud/datastore_v1/services/datastore/client.py b/google/cloud/datastore_v1/services/datastore/client.py index 5205b893..ff43f6d8 100644 --- a/google/cloud/datastore_v1/services/datastore/client.py +++ b/google/cloud/datastore_v1/services/datastore/client.py @@ -224,6 +224,73 @@ def parse_common_location_path(path: str) -> Dict[str, str]: m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)$", path) return m.groupdict() if m else {} + @classmethod + def get_mtls_endpoint_and_cert_source( + cls, client_options: Optional[client_options_lib.ClientOptions] = None + ): + """Return the API endpoint and client cert source for mutual TLS. + + The client cert source is determined in the following order: + (1) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is not "true", the + client cert source is None. + (2) if `client_options.client_cert_source` is provided, use the provided one; if the + default client cert source exists, use the default one; otherwise the client cert + source is None. + + The API endpoint is determined in the following order: + (1) if `client_options.api_endpoint` if provided, use the provided one. + (2) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is "always", use the + default mTLS endpoint; if the environment variabel is "never", use the default API + endpoint; otherwise if client cert source exists, use the default mTLS endpoint, otherwise + use the default API endpoint. + + More details can be found at https://google.aip.dev/auth/4114. + + Args: + client_options (google.api_core.client_options.ClientOptions): Custom options for the + client. Only the `api_endpoint` and `client_cert_source` properties may be used + in this method. + + Returns: + Tuple[str, Callable[[], Tuple[bytes, bytes]]]: returns the API endpoint and the + client cert source to use. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If any errors happen. + """ + if client_options is None: + client_options = client_options_lib.ClientOptions() + use_client_cert = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto") + if use_client_cert not in ("true", "false"): + raise ValueError( + "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + ) + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError( + "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + ) + + # Figure out the client cert source to use. + client_cert_source = None + if use_client_cert == "true": + if client_options.client_cert_source: + client_cert_source = client_options.client_cert_source + elif mtls.has_default_client_cert_source(): + client_cert_source = mtls.default_client_cert_source() + + # Figure out which api endpoint to use. + if client_options.api_endpoint is not None: + api_endpoint = client_options.api_endpoint + elif use_mtls_endpoint == "always" or ( + use_mtls_endpoint == "auto" and client_cert_source + ): + api_endpoint = cls.DEFAULT_MTLS_ENDPOINT + else: + api_endpoint = cls.DEFAULT_ENDPOINT + + return api_endpoint, client_cert_source + def __init__( self, *, @@ -274,57 +341,22 @@ def __init__( if client_options is None: client_options = client_options_lib.ClientOptions() - # Create SSL credentials for mutual TLS if needed. - if os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") not in ( - "true", - "false", - ): - raise ValueError( - "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) - use_client_cert = ( - os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true" + api_endpoint, client_cert_source_func = self.get_mtls_endpoint_and_cert_source( + client_options ) - client_cert_source_func = None - is_mtls = False - if use_client_cert: - if client_options.client_cert_source: - is_mtls = True - client_cert_source_func = client_options.client_cert_source - else: - is_mtls = mtls.has_default_client_cert_source() - if is_mtls: - client_cert_source_func = mtls.default_client_cert_source() - else: - client_cert_source_func = None - - # Figure out which api endpoint to use. - if client_options.api_endpoint is not None: - api_endpoint = client_options.api_endpoint - else: - use_mtls_env = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto") - if use_mtls_env == "never": - api_endpoint = self.DEFAULT_ENDPOINT - elif use_mtls_env == "always": - api_endpoint = self.DEFAULT_MTLS_ENDPOINT - elif use_mtls_env == "auto": - if is_mtls: - api_endpoint = self.DEFAULT_MTLS_ENDPOINT - else: - api_endpoint = self.DEFAULT_ENDPOINT - else: - raise MutualTLSChannelError( - "Unsupported GOOGLE_API_USE_MTLS_ENDPOINT value. Accepted " - "values: never, auto, always" - ) + api_key_value = getattr(client_options, "api_key", None) + if api_key_value and credentials: + raise ValueError( + "client_options.api_key and credentials are mutually exclusive" + ) # Save or instantiate the transport. # Ordinarily, we provide the transport, but allowing a custom transport # instance provides an extensibility point for unusual situations. if isinstance(transport, DatastoreTransport): # transport is a DatastoreTransport instance. - if credentials or client_options.credentials_file: + if credentials or client_options.credentials_file or api_key_value: raise ValueError( "When providing a transport instance, " "provide its credentials directly." @@ -336,6 +368,15 @@ def __init__( ) self._transport = transport else: + import google.auth._default # type: ignore + + if api_key_value and hasattr( + google.auth._default, "get_api_key_credentials" + ): + credentials = google.auth._default.get_api_key_credentials( + api_key_value + ) + Transport = type(self).get_transport_class(transport) self._transport = Transport( credentials=credentials, diff --git a/google/cloud/datastore_v1/services/datastore/transports/base.py b/google/cloud/datastore_v1/services/datastore/transports/base.py index e23e42a2..2d44b1ac 100644 --- a/google/cloud/datastore_v1/services/datastore/transports/base.py +++ b/google/cloud/datastore_v1/services/datastore/transports/base.py @@ -102,7 +102,6 @@ def __init__( credentials, _ = google.auth.load_credentials_from_file( credentials_file, **scopes_kwargs, quota_project_id=quota_project_id ) - elif credentials is None: credentials, _ = google.auth.default( **scopes_kwargs, quota_project_id=quota_project_id diff --git a/google/cloud/datastore_v1/services/datastore/transports/grpc.py b/google/cloud/datastore_v1/services/datastore/transports/grpc.py index 79071c80..55c93eb8 100644 --- a/google/cloud/datastore_v1/services/datastore/transports/grpc.py +++ b/google/cloud/datastore_v1/services/datastore/transports/grpc.py @@ -22,7 +22,7 @@ from google.auth import credentials as ga_credentials # type: ignore from google.auth.transport.grpc import SslCredentials # type: ignore -import grpc +import grpc # type: ignore from google.cloud.datastore_v1.types import datastore from .base import DatastoreTransport, DEFAULT_CLIENT_INFO diff --git a/google/cloud/datastore_v1/services/datastore/transports/grpc_asyncio.py b/google/cloud/datastore_v1/services/datastore/transports/grpc_asyncio.py index 7543acee..2e71d297 100644 --- a/google/cloud/datastore_v1/services/datastore/transports/grpc_asyncio.py +++ b/google/cloud/datastore_v1/services/datastore/transports/grpc_asyncio.py @@ -21,8 +21,8 @@ from google.auth import credentials as ga_credentials # type: ignore from google.auth.transport.grpc import SslCredentials # type: ignore -import grpc -from grpc.experimental import aio +import grpc # type: ignore +from grpc.experimental import aio # type: ignore from google.cloud.datastore_v1.types import datastore from .base import DatastoreTransport, DEFAULT_CLIENT_INFO diff --git a/google/cloud/datastore_v1/types/datastore.py b/google/cloud/datastore_v1/types/datastore.py index efd696aa..f3f87f6c 100644 --- a/google/cloud/datastore_v1/types/datastore.py +++ b/google/cloud/datastore_v1/types/datastore.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import proto +import proto # type: ignore from google.cloud.datastore_v1.types import entity from google.cloud.datastore_v1.types import query as gd_query diff --git a/google/cloud/datastore_v1/types/entity.py b/google/cloud/datastore_v1/types/entity.py index 8bf66889..828d43e5 100644 --- a/google/cloud/datastore_v1/types/entity.py +++ b/google/cloud/datastore_v1/types/entity.py @@ -13,11 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import proto +import proto # type: ignore -from google.protobuf import struct_pb2 -from google.protobuf import timestamp_pb2 -from google.type import latlng_pb2 +from google.protobuf import struct_pb2 # type: ignore +from google.protobuf import timestamp_pb2 # type: ignore +from google.type import latlng_pb2 # type: ignore __protobuf__ = proto.module( diff --git a/google/cloud/datastore_v1/types/query.py b/google/cloud/datastore_v1/types/query.py index 6ae31257..b42e7191 100644 --- a/google/cloud/datastore_v1/types/query.py +++ b/google/cloud/datastore_v1/types/query.py @@ -13,10 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import proto +import proto # type: ignore from google.cloud.datastore_v1.types import entity as gd_entity -from google.protobuf import wrappers_pb2 +from google.protobuf import wrappers_pb2 # type: ignore __protobuf__ = proto.module( diff --git a/tests/unit/gapic/datastore_admin_v1/test_datastore_admin.py b/tests/unit/gapic/datastore_admin_v1/test_datastore_admin.py index 293b1111..e05c072b 100644 --- a/tests/unit/gapic/datastore_admin_v1/test_datastore_admin.py +++ b/tests/unit/gapic/datastore_admin_v1/test_datastore_admin.py @@ -29,7 +29,7 @@ from google.api_core import gapic_v1 from google.api_core import grpc_helpers from google.api_core import grpc_helpers_async -from google.api_core import operation_async +from google.api_core import operation_async # type: ignore from google.api_core import operations_v1 from google.api_core import path_template from google.auth import credentials as ga_credentials @@ -255,20 +255,20 @@ def test_datastore_admin_client_client_options( # unsupported value. with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): with pytest.raises(MutualTLSChannelError): - client = client_class() + client = client_class(transport=transport_name) # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. with mock.patch.dict( os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} ): with pytest.raises(ValueError): - client = client_class() + client = client_class(transport=transport_name) # Check the case quota_project_id is provided options = client_options.ClientOptions(quota_project_id="octopus") with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None - client = client_class(transport=transport_name, client_options=options) + client = client_class(client_options=options, transport=transport_name) patched.assert_called_once_with( credentials=None, credentials_file=None, @@ -327,7 +327,7 @@ def test_datastore_admin_client_mtls_env_auto( ) with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None - client = client_class(transport=transport_name, client_options=options) + client = client_class(client_options=options, transport=transport_name) if use_client_cert_env == "false": expected_client_cert_source = None @@ -404,6 +404,87 @@ def test_datastore_admin_client_mtls_env_auto( ) +@pytest.mark.parametrize( + "client_class", [DatastoreAdminClient, DatastoreAdminAsyncClient] +) +@mock.patch.object( + DatastoreAdminClient, + "DEFAULT_ENDPOINT", + modify_default_endpoint(DatastoreAdminClient), +) +@mock.patch.object( + DatastoreAdminAsyncClient, + "DEFAULT_ENDPOINT", + modify_default_endpoint(DatastoreAdminAsyncClient), +) +def test_datastore_admin_client_get_mtls_endpoint_and_cert_source(client_class): + mock_client_cert_source = mock.Mock() + + # Test the case GOOGLE_API_USE_CLIENT_CERTIFICATE is "true". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + mock_api_endpoint = "foo" + options = client_options.ClientOptions( + client_cert_source=mock_client_cert_source, api_endpoint=mock_api_endpoint + ) + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source( + options + ) + assert api_endpoint == mock_api_endpoint + assert cert_source == mock_client_cert_source + + # Test the case GOOGLE_API_USE_CLIENT_CERTIFICATE is "false". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"}): + mock_client_cert_source = mock.Mock() + mock_api_endpoint = "foo" + options = client_options.ClientOptions( + client_cert_source=mock_client_cert_source, api_endpoint=mock_api_endpoint + ) + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source( + options + ) + assert api_endpoint == mock_api_endpoint + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "never". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_ENDPOINT + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "always". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "auto" and default cert doesn't exist. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + with mock.patch( + "google.auth.transport.mtls.has_default_client_cert_source", + return_value=False, + ): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_ENDPOINT + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "auto" and default cert exists. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + with mock.patch( + "google.auth.transport.mtls.has_default_client_cert_source", + return_value=True, + ): + with mock.patch( + "google.auth.transport.mtls.default_client_cert_source", + return_value=mock_client_cert_source, + ): + ( + api_endpoint, + cert_source, + ) = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT + assert cert_source == mock_client_cert_source + + @pytest.mark.parametrize( "client_class,transport_class,transport_name", [ @@ -422,7 +503,7 @@ def test_datastore_admin_client_client_options_scopes( options = client_options.ClientOptions(scopes=["1", "2"],) with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None - client = client_class(transport=transport_name, client_options=options) + client = client_class(client_options=options, transport=transport_name) patched.assert_called_once_with( credentials=None, credentials_file=None, @@ -453,7 +534,7 @@ def test_datastore_admin_client_client_options_credentials_file( options = client_options.ClientOptions(credentials_file="credentials.json") with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None - client = client_class(transport=transport_name, client_options=options) + client = client_class(client_options=options, transport=transport_name) patched.assert_called_once_with( credentials=None, credentials_file="credentials.json", @@ -486,9 +567,8 @@ def test_datastore_admin_client_client_options_from_dict(): ) -def test_export_entities( - transport: str = "grpc", request_type=datastore_admin.ExportEntitiesRequest -): +@pytest.mark.parametrize("request_type", [datastore_admin.ExportEntitiesRequest, dict,]) +def test_export_entities(request_type, transport: str = "grpc"): client = DatastoreAdminClient( credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) @@ -512,10 +592,6 @@ def test_export_entities( assert isinstance(response, future.Future) -def test_export_entities_from_dict(): - test_export_entities(request_type=dict) - - def test_export_entities_empty_call(): # This test is a coverage failsafe to make sure that totally empty calls, # i.e. request == None and no flattened fields passed, work. @@ -673,9 +749,8 @@ async def test_export_entities_flattened_error_async(): ) -def test_import_entities( - transport: str = "grpc", request_type=datastore_admin.ImportEntitiesRequest -): +@pytest.mark.parametrize("request_type", [datastore_admin.ImportEntitiesRequest, dict,]) +def test_import_entities(request_type, transport: str = "grpc"): client = DatastoreAdminClient( credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) @@ -699,10 +774,6 @@ def test_import_entities( assert isinstance(response, future.Future) -def test_import_entities_from_dict(): - test_import_entities(request_type=dict) - - def test_import_entities_empty_call(): # This test is a coverage failsafe to make sure that totally empty calls, # i.e. request == None and no flattened fields passed, work. @@ -860,9 +931,8 @@ async def test_import_entities_flattened_error_async(): ) -def test_create_index( - transport: str = "grpc", request_type=datastore_admin.CreateIndexRequest -): +@pytest.mark.parametrize("request_type", [datastore_admin.CreateIndexRequest, dict,]) +def test_create_index(request_type, transport: str = "grpc"): client = DatastoreAdminClient( credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) @@ -886,10 +956,6 @@ def test_create_index( assert isinstance(response, future.Future) -def test_create_index_from_dict(): - test_create_index(request_type=dict) - - def test_create_index_empty_call(): # This test is a coverage failsafe to make sure that totally empty calls, # i.e. request == None and no flattened fields passed, work. @@ -939,9 +1005,8 @@ async def test_create_index_async_from_dict(): await test_create_index_async(request_type=dict) -def test_delete_index( - transport: str = "grpc", request_type=datastore_admin.DeleteIndexRequest -): +@pytest.mark.parametrize("request_type", [datastore_admin.DeleteIndexRequest, dict,]) +def test_delete_index(request_type, transport: str = "grpc"): client = DatastoreAdminClient( credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) @@ -965,10 +1030,6 @@ def test_delete_index( assert isinstance(response, future.Future) -def test_delete_index_from_dict(): - test_delete_index(request_type=dict) - - def test_delete_index_empty_call(): # This test is a coverage failsafe to make sure that totally empty calls, # i.e. request == None and no flattened fields passed, work. @@ -1018,9 +1079,8 @@ async def test_delete_index_async_from_dict(): await test_delete_index_async(request_type=dict) -def test_get_index( - transport: str = "grpc", request_type=datastore_admin.GetIndexRequest -): +@pytest.mark.parametrize("request_type", [datastore_admin.GetIndexRequest, dict,]) +def test_get_index(request_type, transport: str = "grpc"): client = DatastoreAdminClient( credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) @@ -1055,10 +1115,6 @@ def test_get_index( assert response.state == index.Index.State.CREATING -def test_get_index_from_dict(): - test_get_index(request_type=dict) - - def test_get_index_empty_call(): # This test is a coverage failsafe to make sure that totally empty calls, # i.e. request == None and no flattened fields passed, work. @@ -1119,9 +1175,8 @@ async def test_get_index_async_from_dict(): await test_get_index_async(request_type=dict) -def test_list_indexes( - transport: str = "grpc", request_type=datastore_admin.ListIndexesRequest -): +@pytest.mark.parametrize("request_type", [datastore_admin.ListIndexesRequest, dict,]) +def test_list_indexes(request_type, transport: str = "grpc"): client = DatastoreAdminClient( credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) @@ -1148,10 +1203,6 @@ def test_list_indexes( assert response.next_page_token == "next_page_token_value" -def test_list_indexes_from_dict(): - test_list_indexes(request_type=dict) - - def test_list_indexes_empty_call(): # This test is a coverage failsafe to make sure that totally empty calls, # i.e. request == None and no flattened fields passed, work. @@ -1204,8 +1255,10 @@ async def test_list_indexes_async_from_dict(): await test_list_indexes_async(request_type=dict) -def test_list_indexes_pager(): - client = DatastoreAdminClient(credentials=ga_credentials.AnonymousCredentials,) +def test_list_indexes_pager(transport_name: str = "grpc"): + client = DatastoreAdminClient( + credentials=ga_credentials.AnonymousCredentials, transport=transport_name, + ) # Mock the actual call within the gRPC stub, and fake the request. with mock.patch.object(type(client.transport.list_indexes), "__call__") as call: @@ -1235,8 +1288,10 @@ def test_list_indexes_pager(): assert all(isinstance(i, index.Index) for i in results) -def test_list_indexes_pages(): - client = DatastoreAdminClient(credentials=ga_credentials.AnonymousCredentials,) +def test_list_indexes_pages(transport_name: str = "grpc"): + client = DatastoreAdminClient( + credentials=ga_credentials.AnonymousCredentials, transport=transport_name, + ) # Mock the actual call within the gRPC stub, and fake the request. with mock.patch.object(type(client.transport.list_indexes), "__call__") as call: @@ -1343,6 +1398,23 @@ def test_credentials_transport_error(): transport=transport, ) + # It is an error to provide an api_key and a transport instance. + transport = transports.DatastoreAdminGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + options = client_options.ClientOptions() + options.api_key = "api_key" + with pytest.raises(ValueError): + client = DatastoreAdminClient(client_options=options, transport=transport,) + + # It is an error to provide an api_key and a credential. + options = mock.Mock() + options.api_key = "api_key" + with pytest.raises(ValueError): + client = DatastoreAdminClient( + client_options=options, credentials=ga_credentials.AnonymousCredentials() + ) + # It is an error to provide scopes and a transport instance. transport = transports.DatastoreAdminGrpcTransport( credentials=ga_credentials.AnonymousCredentials(), @@ -1861,7 +1933,7 @@ def test_parse_common_location_path(): assert expected == actual -def test_client_withDEFAULT_CLIENT_INFO(): +def test_client_with_default_client_info(): client_info = gapic_v1.client_info.ClientInfo() with mock.patch.object( @@ -1926,3 +1998,33 @@ def test_client_ctx(): with client: pass close.assert_called() + + +@pytest.mark.parametrize( + "client_class,transport_class", + [ + (DatastoreAdminClient, transports.DatastoreAdminGrpcTransport), + (DatastoreAdminAsyncClient, transports.DatastoreAdminGrpcAsyncIOTransport), + ], +) +def test_api_key_credentials(client_class, transport_class): + with mock.patch.object( + google.auth._default, "get_api_key_credentials", create=True + ) as get_api_key_credentials: + mock_cred = mock.Mock() + get_api_key_credentials.return_value = mock_cred + options = client_options.ClientOptions() + options.api_key = "api_key" + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=mock_cred, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) diff --git a/tests/unit/gapic/datastore_v1/test_datastore.py b/tests/unit/gapic/datastore_v1/test_datastore.py index acfffd84..06f177ca 100644 --- a/tests/unit/gapic/datastore_v1/test_datastore.py +++ b/tests/unit/gapic/datastore_v1/test_datastore.py @@ -38,10 +38,10 @@ from google.cloud.datastore_v1.types import entity from google.cloud.datastore_v1.types import query from google.oauth2 import service_account -from google.protobuf import struct_pb2 -from google.protobuf import timestamp_pb2 -from google.protobuf import wrappers_pb2 -from google.type import latlng_pb2 +from google.protobuf import struct_pb2 # type: ignore +from google.protobuf import timestamp_pb2 # type: ignore +from google.protobuf import wrappers_pb2 # type: ignore +from google.type import latlng_pb2 # type: ignore import google.auth @@ -238,20 +238,20 @@ def test_datastore_client_client_options(client_class, transport_class, transpor # unsupported value. with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): with pytest.raises(MutualTLSChannelError): - client = client_class() + client = client_class(transport=transport_name) # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. with mock.patch.dict( os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} ): with pytest.raises(ValueError): - client = client_class() + client = client_class(transport=transport_name) # Check the case quota_project_id is provided options = client_options.ClientOptions(quota_project_id="octopus") with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None - client = client_class(transport=transport_name, client_options=options) + client = client_class(client_options=options, transport=transport_name) patched.assert_called_once_with( credentials=None, credentials_file=None, @@ -308,7 +308,7 @@ def test_datastore_client_mtls_env_auto( ) with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None - client = client_class(transport=transport_name, client_options=options) + client = client_class(client_options=options, transport=transport_name) if use_client_cert_env == "false": expected_client_cert_source = None @@ -385,6 +385,83 @@ def test_datastore_client_mtls_env_auto( ) +@pytest.mark.parametrize("client_class", [DatastoreClient, DatastoreAsyncClient]) +@mock.patch.object( + DatastoreClient, "DEFAULT_ENDPOINT", modify_default_endpoint(DatastoreClient) +) +@mock.patch.object( + DatastoreAsyncClient, + "DEFAULT_ENDPOINT", + modify_default_endpoint(DatastoreAsyncClient), +) +def test_datastore_client_get_mtls_endpoint_and_cert_source(client_class): + mock_client_cert_source = mock.Mock() + + # Test the case GOOGLE_API_USE_CLIENT_CERTIFICATE is "true". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + mock_api_endpoint = "foo" + options = client_options.ClientOptions( + client_cert_source=mock_client_cert_source, api_endpoint=mock_api_endpoint + ) + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source( + options + ) + assert api_endpoint == mock_api_endpoint + assert cert_source == mock_client_cert_source + + # Test the case GOOGLE_API_USE_CLIENT_CERTIFICATE is "false". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"}): + mock_client_cert_source = mock.Mock() + mock_api_endpoint = "foo" + options = client_options.ClientOptions( + client_cert_source=mock_client_cert_source, api_endpoint=mock_api_endpoint + ) + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source( + options + ) + assert api_endpoint == mock_api_endpoint + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "never". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_ENDPOINT + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "always". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "auto" and default cert doesn't exist. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + with mock.patch( + "google.auth.transport.mtls.has_default_client_cert_source", + return_value=False, + ): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_ENDPOINT + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "auto" and default cert exists. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + with mock.patch( + "google.auth.transport.mtls.has_default_client_cert_source", + return_value=True, + ): + with mock.patch( + "google.auth.transport.mtls.default_client_cert_source", + return_value=mock_client_cert_source, + ): + ( + api_endpoint, + cert_source, + ) = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT + assert cert_source == mock_client_cert_source + + @pytest.mark.parametrize( "client_class,transport_class,transport_name", [ @@ -403,7 +480,7 @@ def test_datastore_client_client_options_scopes( options = client_options.ClientOptions(scopes=["1", "2"],) with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None - client = client_class(transport=transport_name, client_options=options) + client = client_class(client_options=options, transport=transport_name) patched.assert_called_once_with( credentials=None, credentials_file=None, @@ -434,7 +511,7 @@ def test_datastore_client_client_options_credentials_file( options = client_options.ClientOptions(credentials_file="credentials.json") with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None - client = client_class(transport=transport_name, client_options=options) + client = client_class(client_options=options, transport=transport_name) patched.assert_called_once_with( credentials=None, credentials_file="credentials.json", @@ -465,7 +542,8 @@ def test_datastore_client_client_options_from_dict(): ) -def test_lookup(transport: str = "grpc", request_type=datastore.LookupRequest): +@pytest.mark.parametrize("request_type", [datastore.LookupRequest, dict,]) +def test_lookup(request_type, transport: str = "grpc"): client = DatastoreClient( credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) @@ -489,10 +567,6 @@ def test_lookup(transport: str = "grpc", request_type=datastore.LookupRequest): assert isinstance(response, datastore.LookupResponse) -def test_lookup_from_dict(): - test_lookup(request_type=dict) - - def test_lookup_empty_call(): # This test is a coverage failsafe to make sure that totally empty calls, # i.e. request == None and no flattened fields passed, work. @@ -668,7 +742,8 @@ async def test_lookup_flattened_error_async(): ) -def test_run_query(transport: str = "grpc", request_type=datastore.RunQueryRequest): +@pytest.mark.parametrize("request_type", [datastore.RunQueryRequest, dict,]) +def test_run_query(request_type, transport: str = "grpc"): client = DatastoreClient( credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) @@ -692,10 +767,6 @@ def test_run_query(transport: str = "grpc", request_type=datastore.RunQueryReque assert isinstance(response, datastore.RunQueryResponse) -def test_run_query_from_dict(): - test_run_query(request_type=dict) - - def test_run_query_empty_call(): # This test is a coverage failsafe to make sure that totally empty calls, # i.e. request == None and no flattened fields passed, work. @@ -745,9 +816,8 @@ async def test_run_query_async_from_dict(): await test_run_query_async(request_type=dict) -def test_begin_transaction( - transport: str = "grpc", request_type=datastore.BeginTransactionRequest -): +@pytest.mark.parametrize("request_type", [datastore.BeginTransactionRequest, dict,]) +def test_begin_transaction(request_type, transport: str = "grpc"): client = DatastoreClient( credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) @@ -776,10 +846,6 @@ def test_begin_transaction( assert response.transaction == b"transaction_blob" -def test_begin_transaction_from_dict(): - test_begin_transaction(request_type=dict) - - def test_begin_transaction_empty_call(): # This test is a coverage failsafe to make sure that totally empty calls, # i.e. request == None and no flattened fields passed, work. @@ -906,7 +972,8 @@ async def test_begin_transaction_flattened_error_async(): ) -def test_commit(transport: str = "grpc", request_type=datastore.CommitRequest): +@pytest.mark.parametrize("request_type", [datastore.CommitRequest, dict,]) +def test_commit(request_type, transport: str = "grpc"): client = DatastoreClient( credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) @@ -931,10 +998,6 @@ def test_commit(transport: str = "grpc", request_type=datastore.CommitRequest): assert response.index_updates == 1389 -def test_commit_from_dict(): - test_commit(request_type=dict) - - def test_commit_empty_call(): # This test is a coverage failsafe to make sure that totally empty calls, # i.e. request == None and no flattened fields passed, work. @@ -1141,7 +1204,8 @@ async def test_commit_flattened_error_async(): ) -def test_rollback(transport: str = "grpc", request_type=datastore.RollbackRequest): +@pytest.mark.parametrize("request_type", [datastore.RollbackRequest, dict,]) +def test_rollback(request_type, transport: str = "grpc"): client = DatastoreClient( credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) @@ -1165,10 +1229,6 @@ def test_rollback(transport: str = "grpc", request_type=datastore.RollbackReques assert isinstance(response, datastore.RollbackResponse) -def test_rollback_from_dict(): - test_rollback(request_type=dict) - - def test_rollback_empty_call(): # This test is a coverage failsafe to make sure that totally empty calls, # i.e. request == None and no flattened fields passed, work. @@ -1300,9 +1360,8 @@ async def test_rollback_flattened_error_async(): ) -def test_allocate_ids( - transport: str = "grpc", request_type=datastore.AllocateIdsRequest -): +@pytest.mark.parametrize("request_type", [datastore.AllocateIdsRequest, dict,]) +def test_allocate_ids(request_type, transport: str = "grpc"): client = DatastoreClient( credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) @@ -1326,10 +1385,6 @@ def test_allocate_ids( assert isinstance(response, datastore.AllocateIdsResponse) -def test_allocate_ids_from_dict(): - test_allocate_ids(request_type=dict) - - def test_allocate_ids_empty_call(): # This test is a coverage failsafe to make sure that totally empty calls, # i.e. request == None and no flattened fields passed, work. @@ -1483,7 +1538,8 @@ async def test_allocate_ids_flattened_error_async(): ) -def test_reserve_ids(transport: str = "grpc", request_type=datastore.ReserveIdsRequest): +@pytest.mark.parametrize("request_type", [datastore.ReserveIdsRequest, dict,]) +def test_reserve_ids(request_type, transport: str = "grpc"): client = DatastoreClient( credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) @@ -1507,10 +1563,6 @@ def test_reserve_ids(transport: str = "grpc", request_type=datastore.ReserveIdsR assert isinstance(response, datastore.ReserveIdsResponse) -def test_reserve_ids_from_dict(): - test_reserve_ids(request_type=dict) - - def test_reserve_ids_empty_call(): # This test is a coverage failsafe to make sure that totally empty calls, # i.e. request == None and no flattened fields passed, work. @@ -1684,6 +1736,23 @@ def test_credentials_transport_error(): transport=transport, ) + # It is an error to provide an api_key and a transport instance. + transport = transports.DatastoreGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + options = client_options.ClientOptions() + options.api_key = "api_key" + with pytest.raises(ValueError): + client = DatastoreClient(client_options=options, transport=transport,) + + # It is an error to provide an api_key and a credential. + options = mock.Mock() + options.api_key = "api_key" + with pytest.raises(ValueError): + client = DatastoreClient( + client_options=options, credentials=ga_credentials.AnonymousCredentials() + ) + # It is an error to provide scopes and a transport instance. transport = transports.DatastoreGrpcTransport( credentials=ga_credentials.AnonymousCredentials(), @@ -2155,7 +2224,7 @@ def test_parse_common_location_path(): assert expected == actual -def test_client_withDEFAULT_CLIENT_INFO(): +def test_client_with_default_client_info(): client_info = gapic_v1.client_info.ClientInfo() with mock.patch.object( @@ -2220,3 +2289,33 @@ def test_client_ctx(): with client: pass close.assert_called() + + +@pytest.mark.parametrize( + "client_class,transport_class", + [ + (DatastoreClient, transports.DatastoreGrpcTransport), + (DatastoreAsyncClient, transports.DatastoreGrpcAsyncIOTransport), + ], +) +def test_api_key_credentials(client_class, transport_class): + with mock.patch.object( + google.auth._default, "get_api_key_credentials", create=True + ) as get_api_key_credentials: + mock_cred = mock.Mock() + get_api_key_credentials.return_value = mock_cred + options = client_options.ClientOptions() + options.api_key = "api_key" + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=mock_cred, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + )