From 633193a14fbb0ca5abf171cfadeebc48fe9adce9 Mon Sep 17 00:00:00 2001 From: Bob Hogg Date: Thu, 22 Jun 2023 15:59:39 +0000 Subject: [PATCH 1/2] chore: Update to google-cloud-datastore 2.16.0 --- google/cloud/ndb/_datastore_api.py | 12 ++--- google/cloud/ndb/_datastore_query.py | 2 +- google/cloud/ndb/_gql.py | 8 +--- google/cloud/ndb/client.py | 3 +- google/cloud/ndb/key.py | 15 +++---- google/cloud/ndb/model.py | 8 ++-- google/cloud/ndb/query.py | 3 +- noxfile.py | 2 + setup.py | 2 +- testing/constraints-3.7.txt | 2 +- tests/conftest.py | 2 +- tests/system/conftest.py | 8 ++-- tests/unit/test_client.py | 4 +- tests/unit/test_key.py | 8 ++-- tests/unit/test_query.py | 66 ++++++++++------------------ 15 files changed, 59 insertions(+), 86 deletions(-) diff --git a/google/cloud/ndb/_datastore_api.py b/google/cloud/ndb/_datastore_api.py index 26651ac1..46a41a3c 100644 --- a/google/cloud/ndb/_datastore_api.py +++ b/google/cloud/ndb/_datastore_api.py @@ -307,7 +307,7 @@ def _datastore_lookup(keys, read_options, retries=None, timeout=None, metadata=( client = context_module.get_context().client request = datastore_pb2.LookupRequest( project_id=client.project, - database_id=client.database, + database_id=(client.database or ""), keys=[key for key in keys], read_options=read_options, ) @@ -881,7 +881,7 @@ def _datastore_commit(mutations, transaction, retries=None, timeout=None, metada client = context_module.get_context().client request = datastore_pb2.CommitRequest( project_id=client.project, - database_id=client.database, + database_id=client.database or "", mode=mode, mutations=mutations, transaction=transaction, @@ -1008,7 +1008,7 @@ def _datastore_allocate_ids(keys, retries=None, timeout=None, metadata=()): """ client = context_module.get_context().client request = datastore_pb2.AllocateIdsRequest( - project_id=client.project, database_id=client.database, keys=keys + project_id=client.project, database_id=(client.database or ""), keys=keys ) metadata = _add_routing_info(metadata, request) @@ -1070,7 +1070,7 @@ def _datastore_begin_transaction(read_only, retries=None, timeout=None, metadata request = datastore_pb2.BeginTransactionRequest( project_id=client.project, - database_id=client.database, + database_id=(client.database or ""), transaction_options=options, ) metadata = _add_routing_info(metadata, request) @@ -1121,7 +1121,9 @@ def _datastore_rollback(transaction, retries=None, timeout=None, metadata=()): """ client = context_module.get_context().client request = datastore_pb2.RollbackRequest( - project_id=client.project, database_id=client.database, transaction=transaction + project_id=client.project, + database_id=(client.database or ""), + transaction=transaction, ) metadata = _add_routing_info(metadata, request) diff --git a/google/cloud/ndb/_datastore_query.py b/google/cloud/ndb/_datastore_query.py index 9d12b973..90c32ba1 100644 --- a/google/cloud/ndb/_datastore_query.py +++ b/google/cloud/ndb/_datastore_query.py @@ -1011,8 +1011,8 @@ def _datastore_run_query(query): query_pb = _query_to_protobuf(query) partition_id = entity_pb2.PartitionId( project_id=query.project, - namespace_id=query.namespace, database_id=query.database, + namespace_id=query.namespace, ) read_options = _datastore_api.get_read_options(query) request = datastore_pb2.RunQueryRequest( diff --git a/google/cloud/ndb/_gql.py b/google/cloud/ndb/_gql.py index 8e35d7e4..d72abad2 100644 --- a/google/cloud/ndb/_gql.py +++ b/google/cloud/ndb/_gql.py @@ -94,13 +94,7 @@ class GQL(object): _hint = "" def __init__( - self, - query_string, - _app=None, - _auth_domain=None, - namespace=None, - *, - database: str = None + self, query_string, _app=None, _auth_domain=None, namespace=None, database=None ): """Parses the input query into the class as a pre-compiled query. diff --git a/google/cloud/ndb/client.py b/google/cloud/ndb/client.py index 97d75f89..c7959a92 100644 --- a/google/cloud/ndb/client.py +++ b/google/cloud/ndb/client.py @@ -104,8 +104,7 @@ def __init__( namespace=None, credentials=None, client_options=None, - *, - database: str = "" + database=None, ): self.namespace = namespace self.host = os.environ.get(environment_vars.GCD_HOST, DATASTORE_API_HOST) diff --git a/google/cloud/ndb/key.py b/google/cloud/ndb/key.py index 0f3b2922..b2919159 100644 --- a/google/cloud/ndb/key.py +++ b/google/cloud/ndb/key.py @@ -146,7 +146,7 @@ class Key(object): from google.cloud.ndb import context as context_module client = mock.Mock( project="testing", - database="", + database=None, namespace=None, stub=mock.Mock(spec=()), spec=("project", "database", "namespace", "stub"), @@ -378,7 +378,7 @@ def __hash__(self): def _tuple(self): """Helper to return an orderable tuple.""" - return (self.app(), self.namespace(), self.database(), self.pairs()) + return (self.app(), self.namespace(), self.database() or "", self.pairs()) def __eq__(self, other): """Equality comparison operation.""" @@ -464,7 +464,7 @@ def __setstate__(self, state): _clean_flat_path(flat) project = _project_from_app(kwargs["app"]) - database = "" + database = None if "database" in kwargs: database = kwargs["database"] @@ -599,8 +599,7 @@ def database(self): >>> key.database() 'mydb' """ - db = self._key.database or "" - return db + return self._key.database def id(self): """The string or integer ID in the last ``(kind, id)`` pair, if any. @@ -1449,12 +1448,12 @@ def _parse_from_args( # Offload verification of parent to ``google.cloud.datastore.Key()``. parent_ds_key = parent._key + if database == "": + database = None + if namespace == "": namespace = None - if database is None: - database = "" - return google.cloud.datastore.Key( *flat, parent=parent_ds_key, diff --git a/google/cloud/ndb/model.py b/google/cloud/ndb/model.py index f8c4d619..35da9c44 100644 --- a/google/cloud/ndb/model.py +++ b/google/cloud/ndb/model.py @@ -22,7 +22,7 @@ client = mock.Mock( project="testing", - database="", + database=None, namespace=None, stub=mock.Mock(spec=()), spec=("project", "namespace", "database", "stub"), @@ -4864,8 +4864,8 @@ def __init__(_self, **kwargs): id_ = self._get_arg(kwargs, "id") project = self._get_arg(kwargs, "project") app = self._get_arg(kwargs, "app") - namespace = self._get_arg(kwargs, "namespace", key_module.UNDEFINED) database = self._get_arg(kwargs, "database", key_module.UNDEFINED) + namespace = self._get_arg(kwargs, "namespace", key_module.UNDEFINED) parent = self._get_arg(kwargs, "parent") projection = self._get_arg(kwargs, "projection") @@ -5724,7 +5724,7 @@ def _get_by_id( max_memcache_items=None, force_writes=None, _options=None, - database: str = None, + database=None, ): """Get an instance of Model class by ID. @@ -5768,6 +5768,8 @@ def _get_by_id( ``global_cache_timeout``. max_memcache_items (int): No longer supported. force_writes (bool): No longer supported. + database (Optional[str]): Database for the entity to load. If not + passed, uses the client's value. Returns: Optional[Model]: The retrieved entity, if one is found. diff --git a/google/cloud/ndb/query.py b/google/cloud/ndb/query.py index 1e2a5752..815e949d 100644 --- a/google/cloud/ndb/query.py +++ b/google/cloud/ndb/query.py @@ -1327,8 +1327,7 @@ def __init__( offset=None, keys_only=None, default_options=None, - *, - database: str = None, + database=None, ): # Avoid circular import in Python 2.7 from google.cloud.ndb import model diff --git a/noxfile.py b/noxfile.py index 9b1897c7..eb2f4d82 100644 --- a/noxfile.py +++ b/noxfile.py @@ -175,6 +175,8 @@ def system(session, use_named_db): if use_named_db and os.environ.get("RUN_NAMED_DB_TESTS", "false") == "false": session.skip("RUN_NAMED_DB_TESTS is set to false, skipping") + os.environ["IS_NAMED_DB_TEST"] = str(use_named_db) + system_test_exists = os.path.exists(system_test_path) system_test_folder_exists = os.path.exists(system_test_folder_path) # Sanity check: only run tests if found. diff --git a/setup.py b/setup.py index 833fd452..1b8dbe73 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ def main(): readme = readme_file.read() dependencies = [ "google-api-core[grpc] >= 1.34.0, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*", - "google-cloud-datastore @ git+https://github.com/googleapis/python-datastore@multi-db", + "google-cloud-datastore >= 2.16.0, < 3.0.0dev", "protobuf >= 3.19.5, <5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5", "pymemcache >= 2.1.0, < 5.0.0dev", "redis >= 3.0.0, < 5.0.0dev", diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt index d2aa0389..ef05b87c 100644 --- a/testing/constraints-3.7.txt +++ b/testing/constraints-3.7.txt @@ -5,7 +5,7 @@ # # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", # Then this file should have foo==1.14.0 -google-cloud-datastore @ git+https://github.com/googleapis/python-datastore@multi-db +google-cloud-datastore==2.16.0 google-api-core==1.34.0 protobuf==3.19.5 pymemcache==2.1.0 diff --git a/tests/conftest.py b/tests/conftest.py index 05ff3176..c8d6b07d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -88,7 +88,7 @@ def context_factory(): def context(**kwargs): client = mock.Mock( project="testing", - database="", + database=None, namespace=None, spec=("project", "database", "namespace"), stub=mock.Mock(spec=()), diff --git a/tests/system/conftest.py b/tests/system/conftest.py index f4209015..1b8ae315 100644 --- a/tests/system/conftest.py +++ b/tests/system/conftest.py @@ -15,8 +15,8 @@ log = logging.getLogger(__name__) -_DATASTORE_DATABASE = "SYSTEM_TESTS_DATABASE" -TEST_DATABASE = os.getenv(_DATASTORE_DATABASE) +TEST_DATABASE = os.getenv("SYSTEM_TESTS_DATABASE") +IS_NAMED_DB_TEST = os.getenv("IS_NAMED_DB_TEST") @pytest.fixture(scope="session", autouse=True) @@ -132,8 +132,8 @@ def database(): def _get_database(): - db = "" - if TEST_DATABASE is not None: + db = None + if IS_NAMED_DB_TEST == "True": db = TEST_DATABASE return db diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 5d17f87f..0f7019fc 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -47,7 +47,7 @@ def test_constructor_no_args(): assert client.SCOPE == ("https://www.googleapis.com/auth/datastore",) assert client.host == _http.DATASTORE_API_HOST assert client.project == "testing" - assert client.database == "" + assert client.database is None assert client.namespace is None assert client.secure is True @@ -63,7 +63,7 @@ def test_constructor_no_args_emulator(): assert client.SCOPE == ("https://www.googleapis.com/auth/datastore",) assert client.host == "foo" assert client.project == "testing" - assert client.database == "" + assert client.database is None assert client.namespace is None assert client.secure is False diff --git a/tests/unit/test_key.py b/tests/unit/test_key.py index 431ee649..58dbed48 100644 --- a/tests/unit/test_key.py +++ b/tests/unit/test_key.py @@ -231,13 +231,11 @@ def test_constructor_with_project_and_app(): @staticmethod @pytest.mark.usefixtures("in_context") - def test_constructor_with_default_database(): + def test_constructor_with_default_database_as_empty_string(): key = key_module.Key("Kind", 1337, database="") - assert key._key == google.cloud.datastore.Key( - "Kind", 1337, project="testing", database="" - ) - assert key.database() == "" + assert key._key == google.cloud.datastore.Key("Kind", 1337, project="testing") + assert key.database() is None @staticmethod @pytest.mark.usefixtures("in_context") diff --git a/tests/unit/test_query.py b/tests/unit/test_query.py index 43023c20..b50045a6 100644 --- a/tests/unit/test_query.py +++ b/tests/unit/test_query.py @@ -1705,7 +1705,7 @@ def test_fetch_async_w_project_and_namespace_from_query(_datastore_query): response = _datastore_query.fetch.return_value assert query.fetch_async() is response _datastore_query.fetch.assert_called_once_with( - query_module.QueryOptions(project="foo", database="", namespace="bar") + query_module.QueryOptions(project="foo", namespace="bar") ) @staticmethod @@ -1716,9 +1716,7 @@ def test_fetch_async_with_keys_only(_datastore_query): response = _datastore_query.fetch.return_value assert query.fetch_async(keys_only=True) is response _datastore_query.fetch.assert_called_once_with( - query_module.QueryOptions( - project="testing", database="", projection=["__key__"] - ) + query_module.QueryOptions(project="testing", projection=["__key__"]) ) @staticmethod @@ -1730,7 +1728,7 @@ def test_fetch_async_with_keys_only_as_option(_datastore_query): response = _datastore_query.fetch.return_value assert query.fetch_async(options=options) is response _datastore_query.fetch.assert_called_once_with( - query_module.QueryOptions(project="testing", database="", keys_only=True) + query_module.QueryOptions(project="testing", keys_only=True) ) @staticmethod @@ -1748,9 +1746,7 @@ def test_fetch_async_with_projection(_datastore_query): response = _datastore_query.fetch.return_value assert query.fetch_async(projection=("foo", "bar")) is response _datastore_query.fetch.assert_called_once_with( - query_module.QueryOptions( - project="testing", database="", projection=["foo", "bar"] - ) + query_module.QueryOptions(project="testing", projection=["foo", "bar"]) ) @staticmethod @@ -1765,9 +1761,7 @@ def test_fetch_async_with_projection_with_properties(_datastore_query): bar._name = "bar" assert query.fetch_async(projection=(foo, bar)) is response _datastore_query.fetch.assert_called_once_with( - query_module.QueryOptions( - project="testing", database="", projection=["foo", "bar"] - ) + query_module.QueryOptions(project="testing", projection=["foo", "bar"]) ) @staticmethod @@ -1779,9 +1773,7 @@ def test_fetch_async_with_projection_from_query(_datastore_query): response = _datastore_query.fetch.return_value assert query.fetch_async(options=options) is response _datastore_query.fetch.assert_called_once_with( - query_module.QueryOptions( - project="testing", database="", projection=("foo", "bar") - ) + query_module.QueryOptions(project="testing", projection=("foo", "bar")) ) @staticmethod @@ -1799,7 +1791,7 @@ def test_fetch_async_with_offset(_datastore_query): response = _datastore_query.fetch.return_value assert query.fetch_async(offset=20) is response _datastore_query.fetch.assert_called_once_with( - query_module.QueryOptions(project="testing", database="", offset=20) + query_module.QueryOptions(project="testing", offset=20) ) @staticmethod @@ -1810,7 +1802,7 @@ def test_fetch_async_with_limit(_datastore_query): response = _datastore_query.fetch.return_value assert query.fetch_async(limit=20) is response _datastore_query.fetch.assert_called_once_with( - query_module.QueryOptions(project="testing", database="", limit=20) + query_module.QueryOptions(project="testing", limit=20) ) @staticmethod @@ -1821,7 +1813,7 @@ def test_fetch_async_with_limit_as_positional_arg(_datastore_query): response = _datastore_query.fetch.return_value assert query.fetch_async(20) is response _datastore_query.fetch.assert_called_once_with( - query_module.QueryOptions(project="testing", database="", limit=20) + query_module.QueryOptions(project="testing", limit=20) ) @staticmethod @@ -1853,7 +1845,7 @@ def test_fetch_async_with_produce_cursors(_datastore_query): response = _datastore_query.fetch.return_value assert query.fetch_async(produce_cursors=True) is response _datastore_query.fetch.assert_called_once_with( - query_module.QueryOptions(project="testing", database="") + query_module.QueryOptions(project="testing") ) @staticmethod @@ -1864,9 +1856,7 @@ def test_fetch_async_with_start_cursor(_datastore_query): response = _datastore_query.fetch.return_value assert query.fetch_async(start_cursor="cursor") is response _datastore_query.fetch.assert_called_once_with( - query_module.QueryOptions( - project="testing", database="", start_cursor="cursor" - ) + query_module.QueryOptions(project="testing", start_cursor="cursor") ) @staticmethod @@ -1877,9 +1867,7 @@ def test_fetch_async_with_end_cursor(_datastore_query): response = _datastore_query.fetch.return_value assert query.fetch_async(end_cursor="cursor") is response _datastore_query.fetch.assert_called_once_with( - query_module.QueryOptions( - project="testing", database="", end_cursor="cursor" - ) + query_module.QueryOptions(project="testing", end_cursor="cursor") ) @staticmethod @@ -1890,7 +1878,7 @@ def test_fetch_async_with_deadline(_datastore_query): response = _datastore_query.fetch.return_value assert query.fetch_async(deadline=20) is response _datastore_query.fetch.assert_called_once_with( - query_module.QueryOptions(project="testing", database="", timeout=20) + query_module.QueryOptions(project="testing", timeout=20) ) @staticmethod @@ -1901,7 +1889,7 @@ def test_fetch_async_with_timeout(_datastore_query): response = _datastore_query.fetch.return_value assert query.fetch_async(timeout=20) is response _datastore_query.fetch.assert_called_once_with( - query_module.QueryOptions(project="testing", database="", timeout=20) + query_module.QueryOptions(project="testing", timeout=20) ) @staticmethod @@ -1912,9 +1900,7 @@ def test_fetch_async_with_read_policy(_datastore_query): response = _datastore_query.fetch.return_value assert query.fetch_async(read_policy="foo") is response _datastore_query.fetch.assert_called_once_with( - query_module.QueryOptions( - project="testing", database="", read_consistency="foo" - ) + query_module.QueryOptions(project="testing", read_consistency="foo") ) @staticmethod @@ -1925,7 +1911,7 @@ def test_fetch_async_with_transaction(_datastore_query): response = _datastore_query.fetch.return_value assert query.fetch_async(transaction="foo") is response _datastore_query.fetch.assert_called_once_with( - query_module.QueryOptions(project="testing", database="", transaction="foo") + query_module.QueryOptions(project="testing", transaction="foo") ) @staticmethod @@ -1973,7 +1959,7 @@ def test_fetch_with_limit_as_positional_arg(_datastore_query): query = query_module.Query() assert query.fetch(20) == "foo" _datastore_query.fetch.assert_called_once_with( - query_module.QueryOptions(project="testing", database="", limit=20) + query_module.QueryOptions(project="testing", limit=20) ) @staticmethod @@ -2019,9 +2005,7 @@ def test_iter(): query = query_module.Query() iterator = query.iter() assert isinstance(iterator, _datastore_query.QueryIterator) - assert iterator._query == query_module.QueryOptions( - project="testing", database="" - ) + assert iterator._query == query_module.QueryOptions(project="testing") @staticmethod @pytest.mark.usefixtures("in_context") @@ -2032,7 +2016,7 @@ def test_iter_with_projection(): iterator = query.iter(projection=(foo,)) assert isinstance(iterator, _datastore_query.QueryIterator) assert iterator._query == query_module.QueryOptions( - project="testing", database="", projection=["foo"] + project="testing", projection=["foo"] ) @staticmethod @@ -2041,9 +2025,7 @@ def test___iter__(): query = query_module.Query() iterator = iter(query) assert isinstance(iterator, _datastore_query.QueryIterator) - assert iterator._query == query_module.QueryOptions( - project="testing", database="" - ) + assert iterator._query == query_module.QueryOptions(project="testing") @staticmethod @pytest.mark.usefixtures("in_context") @@ -2131,7 +2113,7 @@ def test_get(_datastore_query): _datastore_query.fetch.return_value = utils.future_result(["foo", "bar"]) assert query.get() == "foo" _datastore_query.fetch.assert_called_once_with( - query_module.QueryOptions(project="testing", database="", limit=1) + query_module.QueryOptions(project="testing", limit=1) ) @staticmethod @@ -2194,7 +2176,6 @@ def next(self): query_module.QueryOptions( filters=query.filters, project="testing", - database="", limit=5, ), raw=True, @@ -2233,7 +2214,6 @@ def next(self): _datastore_query.iterate.assert_called_once_with( query_module.QueryOptions( project="testing", - database="", limit=5, start_cursor="cursor000", ), @@ -2263,7 +2243,6 @@ def has_next_async(self): _datastore_query.iterate.assert_called_once_with( query_module.QueryOptions( project="testing", - database="", limit=5, start_cursor="cursor000", ), @@ -2298,7 +2277,6 @@ def has_next_async(self): query_module.QueryOptions( filters=query.filters, project="testing", - database="", limit=5, ), raw=True, @@ -2333,7 +2311,7 @@ def next(self): assert more _datastore_query.iterate.assert_called_once_with( - query_module.QueryOptions(project="testing", database="", limit=5), + query_module.QueryOptions(project="testing", limit=5), raw=True, ) From 372b9b10e6b1c0126422af3e928f538abde2f0d0 Mon Sep 17 00:00:00 2001 From: Bob Hogg Date: Mon, 26 Jun 2023 23:56:22 +0000 Subject: [PATCH 2/2] fix: consolidate with python-datastore --- google/cloud/ndb/_datastore_api.py | 13 ++++++------- google/cloud/ndb/_datastore_query.py | 2 +- google/cloud/ndb/_gql.py | 9 +-------- google/cloud/ndb/model.py | 4 ---- google/cloud/ndb/query.py | 20 ++++++++++---------- tests/unit/test__datastore_api.py | 15 +++++++-------- tests/unit/test__gql.py | 9 ++------- tests/unit/test_model.py | 3 +++ tests/unit/test_query.py | 16 +++++----------- 9 files changed, 35 insertions(+), 56 deletions(-) diff --git a/google/cloud/ndb/_datastore_api.py b/google/cloud/ndb/_datastore_api.py index 46a41a3c..7e163789 100644 --- a/google/cloud/ndb/_datastore_api.py +++ b/google/cloud/ndb/_datastore_api.py @@ -307,10 +307,10 @@ def _datastore_lookup(keys, read_options, retries=None, timeout=None, metadata=( client = context_module.get_context().client request = datastore_pb2.LookupRequest( project_id=client.project, - database_id=(client.database or ""), keys=[key for key in keys], read_options=read_options, ) + helpers.set_database_id_to_request(request, client.database) metadata = _add_routing_info(metadata, request) return make_call( @@ -881,11 +881,11 @@ def _datastore_commit(mutations, transaction, retries=None, timeout=None, metada client = context_module.get_context().client request = datastore_pb2.CommitRequest( project_id=client.project, - database_id=client.database or "", mode=mode, mutations=mutations, transaction=transaction, ) + helpers.set_database_id_to_request(request, client.database) metadata = _add_routing_info(metadata, request) return make_call( @@ -1007,9 +1007,8 @@ def _datastore_allocate_ids(keys, retries=None, timeout=None, metadata=()): :class:`google.cloud.datastore_v1.datastore_pb2.AllocateIdsResponse` """ client = context_module.get_context().client - request = datastore_pb2.AllocateIdsRequest( - project_id=client.project, database_id=(client.database or ""), keys=keys - ) + request = datastore_pb2.AllocateIdsRequest(project_id=client.project, keys=keys) + helpers.set_database_id_to_request(request, client.database) metadata = _add_routing_info(metadata, request) return make_call( @@ -1070,9 +1069,9 @@ def _datastore_begin_transaction(read_only, retries=None, timeout=None, metadata request = datastore_pb2.BeginTransactionRequest( project_id=client.project, - database_id=(client.database or ""), transaction_options=options, ) + helpers.set_database_id_to_request(request, client.database) metadata = _add_routing_info(metadata, request) return make_call( @@ -1122,9 +1121,9 @@ def _datastore_rollback(transaction, retries=None, timeout=None, metadata=()): client = context_module.get_context().client request = datastore_pb2.RollbackRequest( project_id=client.project, - database_id=(client.database or ""), transaction=transaction, ) + helpers.set_database_id_to_request(request, client.database) metadata = _add_routing_info(metadata, request) return make_call( diff --git a/google/cloud/ndb/_datastore_query.py b/google/cloud/ndb/_datastore_query.py index 90c32ba1..98204f52 100644 --- a/google/cloud/ndb/_datastore_query.py +++ b/google/cloud/ndb/_datastore_query.py @@ -1017,11 +1017,11 @@ def _datastore_run_query(query): read_options = _datastore_api.get_read_options(query) request = datastore_pb2.RunQueryRequest( project_id=query.project, - database_id=query.database, partition_id=partition_id, query=query_pb, read_options=read_options, ) + helpers.set_database_id_to_request(request, query.database) metadata = _datastore_api._add_routing_info((), request) response = yield _datastore_api.make_call( diff --git a/google/cloud/ndb/_gql.py b/google/cloud/ndb/_gql.py index d72abad2..2d0a2745 100644 --- a/google/cloud/ndb/_gql.py +++ b/google/cloud/ndb/_gql.py @@ -93,22 +93,17 @@ class GQL(object): _limit = -1 _hint = "" - def __init__( - self, query_string, _app=None, _auth_domain=None, namespace=None, database=None - ): + def __init__(self, query_string, _app=None, _auth_domain=None, namespace=None): """Parses the input query into the class as a pre-compiled query. Args: query_string (str): properly formatted GQL query string. namespace (str): The namespace to use for this query. Defaults to the client's value. - database (str): The database to use for this query. Defaults to the client's value. Raises: exceptions.BadQueryError: if the query is not parsable. """ self._app = _app - self._database = database - self._namespace = namespace self._auth_domain = _auth_domain @@ -716,7 +711,6 @@ def get_query(self): keys_only = None projection = self.projection() project = self._app - database = self._database namespace = self._namespace if self.is_distinct(): distinct_on = projection @@ -734,7 +728,6 @@ def get_query(self): filters=filters, order_by=order_by, project=project, - database=database, namespace=namespace, default_options=default_options, projection=projection, diff --git a/google/cloud/ndb/model.py b/google/cloud/ndb/model.py index 35da9c44..b780f6a5 100644 --- a/google/cloud/ndb/model.py +++ b/google/cloud/ndb/model.py @@ -5488,7 +5488,6 @@ def _prepare_for_put(self): distinct_on=None, group_by=None, default_options=None, - database=None, ) def _query(cls, *filters, **kwargs): """Generate a query for this class. @@ -5515,8 +5514,6 @@ def _query(cls, *filters, **kwargs): results. group_by (list[str]): Deprecated. Synonym for distinct_on. default_options (QueryOptions): QueryOptions object. - database (str): The database to perform the query against. - If not passed, uses the client's value. """ # Validating distinct if kwargs["distinct"]: @@ -5546,7 +5543,6 @@ def _query(cls, *filters, **kwargs): distinct_on=kwargs["distinct_on"], group_by=kwargs["group_by"], default_options=kwargs["default_options"], - database=kwargs["database"], ) query = query.filter(*cls._default_filters()) query = query.filter(*filters) diff --git a/google/cloud/ndb/query.py b/google/cloud/ndb/query.py index 815e949d..fdcdacd5 100644 --- a/google/cloud/ndb/query.py +++ b/google/cloud/ndb/query.py @@ -140,6 +140,7 @@ def ranked(cls, rank): import logging import six +from google.cloud.ndb import context as context_module from google.cloud.ndb import exceptions from google.cloud.ndb import _options from google.cloud.ndb import tasklets @@ -1267,8 +1268,8 @@ def __init__(self, config=None, context=None, **kwargs): if not self.project: self.project = context.client.project - if self.database is None: - self.database = context.client.database + # We always use the client's database, for consistency with python-datastore + self.database = context.client.database if self.namespace is None: if self.ancestor is None: @@ -1304,7 +1305,6 @@ class Query(object): results. group_by (list[str]): Deprecated. Synonym for distinct_on. default_options (QueryOptions): QueryOptions object. - database (str): The database to access. If not passed, uses the client's value. Raises: TypeError: If any of the arguments are invalid. @@ -1327,7 +1327,6 @@ def __init__( offset=None, keys_only=None, default_options=None, - database=None, ): # Avoid circular import in Python 2.7 from google.cloud.ndb import model @@ -1373,7 +1372,6 @@ def __init__( orders = self._option("orders", orders) project = self._option("project", project) app = self._option("app", app) - database = self._option("database", database) namespace = self._option("namespace", namespace) projection = self._option("projection", projection) distinct_on = self._option("distinct_on", distinct_on) @@ -1382,6 +1380,9 @@ def __init__( offset = self._option("offset", offset) keys_only = self._option("keys_only", keys_only) + # Except in the case of ancestor queries, we always use the client's database + database = context_module.get_context().client.database or None + if ancestor is not None: if isinstance(ancestor, ParameterizedThing): if isinstance(ancestor, ParameterizedFunction): @@ -1401,6 +1402,9 @@ def __init__( raise TypeError("ancestor/project id mismatch") else: project = ancestor.app() + + database = ancestor.database() + if namespace is not None: # if namespace is the empty string, that means default # namespace, but after a put, if the ancestor is using @@ -1412,6 +1416,7 @@ def __init__( raise TypeError("ancestor/namespace mismatch") else: namespace = ancestor.namespace() + if filters is not None: if not isinstance(filters, Node): raise TypeError( @@ -1482,8 +1487,6 @@ def __repr__(self): args = [] if self.project is not None: args.append("project=%r" % self.project) - if self.database is not None: - args.append("database=%r" % self.database) if self.namespace is not None: args.append("namespace=%r" % self.namespace) if self.kind is not None: @@ -1554,7 +1557,6 @@ def filter(self, *filters): filters=new_filters, order_by=self.order_by, project=self.project, - database=self.database, namespace=self.namespace, default_options=self.default_options, projection=self.projection, @@ -1588,7 +1590,6 @@ def order(self, *props): filters=self.filters, order_by=order_by, project=self.project, - database=self.database, namespace=self.namespace, default_options=self.default_options, projection=self.projection, @@ -1670,7 +1671,6 @@ def bind(self, *positional, **keyword): filters=filters, order_by=self.order_by, project=self.project, - database=self.database, namespace=self.namespace, default_options=self.default_options, projection=self.projection, diff --git a/tests/unit/test__datastore_api.py b/tests/unit/test__datastore_api.py index 11dc209c..cb3509f5 100644 --- a/tests/unit/test__datastore_api.py +++ b/tests/unit/test__datastore_api.py @@ -557,8 +557,9 @@ def key_pb(key): assert context.eventloop.add_idle.call_count == 1 +@mock.patch("google.cloud.datastore.helpers.set_database_id_to_request") @mock.patch("google.cloud.ndb._datastore_api.datastore_pb2") -def test__datastore_lookup(datastore_pb2, context): +def test__datastore_lookup(datastore_pb2, set_database_id_to_request, context): client = mock.Mock( project="theproject", database="testdb", @@ -576,10 +577,12 @@ def test__datastore_lookup(datastore_pb2, context): datastore_pb2.LookupRequest.assert_called_once_with( project_id="theproject", - database_id="testdb", keys=["foo", "bar"], read_options=None, ) + set_database_id_to_request.assert_called_once_with( + datastore_pb2.LookupRequest.return_value, "testdb" + ) client.stub.lookup.future.assert_called_once_with( datastore_pb2.LookupRequest.return_value, timeout=_api._DEFAULT_TIMEOUT, @@ -1246,7 +1249,6 @@ def test_wo_transaction(stub, datastore_pb2): datastore_pb2.CommitRequest.assert_called_once_with( project_id="testing", - database_id="", mode=datastore_pb2.CommitRequest.Mode.NON_TRANSACTIONAL, mutations=mutations, transaction=None, @@ -1269,7 +1271,6 @@ def test_w_transaction(stub, datastore_pb2): datastore_pb2.CommitRequest.assert_called_once_with( project_id="testing", - database_id="", mode=datastore_pb2.CommitRequest.Mode.TRANSACTIONAL, mutations=mutations, transaction=b"tx123", @@ -1361,7 +1362,7 @@ def test__datastore_allocate_ids(stub, datastore_pb2): assert _api._datastore_allocate_ids(keys).result() == "response" datastore_pb2.AllocateIdsRequest.assert_called_once_with( - project_id="testing", database_id="", keys=keys + project_id="testing", keys=keys ) request = datastore_pb2.AllocateIdsRequest.return_value @@ -1402,7 +1403,6 @@ def test_read_only(stub, datastore_pb2): transaction_options = datastore_pb2.TransactionOptions.return_value datastore_pb2.BeginTransactionRequest.assert_called_once_with( project_id="testing", - database_id="", transaction_options=transaction_options, ) @@ -1427,7 +1427,6 @@ def test_read_write(stub, datastore_pb2): transaction_options = datastore_pb2.TransactionOptions.return_value datastore_pb2.BeginTransactionRequest.assert_called_once_with( project_id="testing", - database_id="", transaction_options=transaction_options, ) @@ -1459,7 +1458,7 @@ def test__datastore_rollback(stub, datastore_pb2): assert _api._datastore_rollback(b"tx123").result() == "response" datastore_pb2.RollbackRequest.assert_called_once_with( - project_id="testing", database_id="", transaction=b"tx123" + project_id="testing", transaction=b"tx123" ) request = datastore_pb2.RollbackRequest.return_value diff --git a/tests/unit/test__gql.py b/tests/unit/test__gql.py index 46668a35..ee9371c8 100644 --- a/tests/unit/test__gql.py +++ b/tests/unit/test__gql.py @@ -65,11 +65,6 @@ def test_constructor_with_namespace(): gql = gql_module.GQL(GQL_QUERY, namespace="test-namespace") assert gql._namespace == "test-namespace" - @staticmethod - def test_constructor_with_database(): - gql = gql_module.GQL(GQL_QUERY, database="test-database") - assert gql._database == "test-database" - @staticmethod def test_constructor_bad_query(): with pytest.raises(exceptions.BadQueryError): @@ -288,13 +283,13 @@ class SomeKind(model.Model): prop4 = model.IntegerProperty() rep = ( - "Query(database='testdb', namespace='test-namespace', kind='SomeKind', filters=AND(FilterNode('prop2', '=', {}" + "Query(namespace='test-namespace', kind='SomeKind', filters=AND(FilterNode('prop2', '=', {}" "), FilterNode('prop3', '>', 5)), order_by=[PropertyOrder(name=" "'prop4', reverse=False), PropertyOrder(name='prop1', " "reverse=True)], limit=10, offset=5, " "projection=['prop1', 'prop2'])" ) - gql = gql_module.GQL(GQL_QUERY, database="testdb", namespace="test-namespace") + gql = gql_module.GQL(GQL_QUERY, namespace="test-namespace") query = gql.get_query() compat_rep = "'xxx'" assert repr(query) == rep.format(compat_rep) diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py index 8394aa5f..6cb0ac90 100644 --- a/tests/unit/test_model.py +++ b/tests/unit/test_model.py @@ -4690,6 +4690,7 @@ def test__check_properties_not_found(): model.Model._check_properties(properties) @staticmethod + @pytest.mark.usefixtures("in_context") def test_query(): class XModel(model.Model): x = model.IntegerProperty() @@ -4699,6 +4700,7 @@ class XModel(model.Model): assert query.filters == (XModel.x == 42) @staticmethod + @pytest.mark.usefixtures("in_context") def test_query_distinct(): class XModel(model.Model): x = model.IntegerProperty() @@ -4731,6 +4733,7 @@ class XModel(model.Model): XModel.query(distinct=True, group_by=("x",)) @staticmethod + @pytest.mark.usefixtures("in_context") def test_query_projection_of_unindexed_attribute(): class XModel(model.Model): x = model.IntegerProperty(indexed=False) diff --git a/tests/unit/test_query.py b/tests/unit/test_query.py index b50045a6..df7df55a 100644 --- a/tests/unit/test_query.py +++ b/tests/unit/test_query.py @@ -38,6 +38,7 @@ def test___all__(): class TestQueryOptions: @staticmethod + @pytest.mark.usefixtures("in_context") def test_constructor(): options = query_module.QueryOptions(kind="test", project="app") assert options.kind == "test" @@ -71,8 +72,8 @@ def test_constructor_with_bad_config(): @staticmethod @pytest.mark.usefixtures("in_context") def test___repr__(): - representation = "QueryOptions(kind='test', project='app', database='db')" - options = query_module.QueryOptions(kind="test", project="app", database="db") + representation = "QueryOptions(kind='test', project='app')" + options = query_module.QueryOptions(kind="test", project="app") assert options.__repr__() == representation @staticmethod @@ -94,13 +95,6 @@ def test_copy(): assert options.database == "bar" assert options.namespace == "foo" - @staticmethod - def test_specify_database(in_context): - with in_context.new().use() as context: - context.client.database = "newdb" - options = query_module.QueryOptions(context=context, database="actualdb") - assert options.database == "actualdb" - @staticmethod def test_explicitly_set_default_database(in_context): with in_context.new().use() as context: @@ -1229,6 +1223,7 @@ def test_OR(): class TestQuery: @staticmethod + @pytest.mark.usefixtures("in_context") def test_constructor(): query = query_module.Query(kind="Foo") assert query.kind == "Foo" @@ -1447,7 +1442,6 @@ def test___repr__(): kind="Foo", ancestor=key_module.Key("a", "b", app="app", namespace="space"), namespace="space", - database="base", app="app", group_by=["X"], projection=[model.Property(name="x")], @@ -1456,7 +1450,7 @@ def test___repr__(): order_by=[], ) rep = ( - "Query(project='app', database='base', namespace='space', kind='Foo', ancestor=" + "Query(project='app', namespace='space', kind='Foo', ancestor=" "Key('a', 'b', project='app', namespace='space'), filters=" "FilterNode('f', None, None), order_by=[], projection=['x'], " "distinct_on=['X'], default_options=QueryOptions(kind='Bar'))"