diff --git a/google-cloud-datastore-admin-v1/Rakefile b/google-cloud-datastore-admin-v1/Rakefile index 8d1267182135..6410873a1dbf 100644 --- a/google-cloud-datastore-admin-v1/Rakefile +++ b/google-cloud-datastore-admin-v1/Rakefile @@ -155,6 +155,32 @@ namespace :ci do end end +task :samples do + Rake::Task["samples:latest"].invoke +end + +namespace :samples do + task :latest do + Dir.chdir "samples" do + Bundler.with_clean_env do + ENV["GOOGLE_CLOUD_SAMPLES_TEST"] = "not_master" + sh "bundle update" + sh "bundle exec rake test" + end + end + end + + task :master do + Dir.chdir "samples" do + Bundler.with_clean_env do + ENV["GOOGLE_CLOUD_SAMPLES_TEST"] = "master" + sh "bundle update" + sh "bundle exec rake test" + end + end + end +end + task default: :test def header str, token = "#" diff --git a/google-cloud-datastore-admin-v1/samples/.rubocop.yml b/google-cloud-datastore-admin-v1/samples/.rubocop.yml new file mode 100644 index 000000000000..0427c0deda24 --- /dev/null +++ b/google-cloud-datastore-admin-v1/samples/.rubocop.yml @@ -0,0 +1,14 @@ +inherit_gem: + google-style: google-style.yml + + +Lint/UselessAssignment: + Exclude: + - "snippets.rb" +Metrics/BlockLength: + Exclude: + - "acceptance/*.rb" + - "Rakefile" +Style/GlobalVars: + Exclude: + - "Rakefile" diff --git a/google-cloud-datastore-admin-v1/samples/Gemfile b/google-cloud-datastore-admin-v1/samples/Gemfile new file mode 100644 index 000000000000..f455cbc0d62c --- /dev/null +++ b/google-cloud-datastore-admin-v1/samples/Gemfile @@ -0,0 +1,36 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START datastore_admin_dependencies] +source "https://rubygems.org" + +# [END datastore_admin_dependencies] + +if ENV["GOOGLE_CLOUD_SAMPLES_TEST"] == "master" + gem "google-cloud-datastore-admin-v1", path: "../../google-cloud-datastore-admin-v1" +else + # rubocop:disable Bundler/DuplicatedGem + # [START datastore_admin_dependencies] + gem "google-cloud-datastore-admin-v1" + # [END datastore_admin_dependencies] + # rubocop:enable Bundler/DuplicatedGem +end + +group :test do + gem "google-cloud-storage" + gem "google-style", "~> 1.24.0" + gem "minitest", "~> 5.14" + gem "minitest-focus", "~> 1.1" + gem "rake" +end diff --git a/google-cloud-datastore-admin-v1/samples/README.md b/google-cloud-datastore-admin-v1/samples/README.md new file mode 100644 index 000000000000..b4f18d01c28b --- /dev/null +++ b/google-cloud-datastore-admin-v1/samples/README.md @@ -0,0 +1,61 @@ +# Ruby Client for the Firestore in Datastore mode Admin V1 API Samples + +Firestore in Datastore mode is a NoSQL document database built for automatic scaling, high performance, and ease of application development. + +## Description + +These samples show how to use the [Firestore in Datastore mode Admin V1 API] +(https://cloud.google.com/datastore). + +## Setup + +Before you can run or test the sample, you will need to enable the Cloud Datastore API in the [Google Cloud Console](https://console.cloud.google.com/apis/library/datastore.googleapis.com). + +## Testing + +The tests for the sample are integration tests that run against the Datastore +service and require authentication. + +### Authenticating + +Set the following environment variable to your Google Cloud Platform +project ID: `GOOGLE_CLOUD_PROJECT` + +Set one of the following environment variables to the path to your Google Cloud +Platform keyfile: + +* `DATASTORE_KEYFILE` +* `GOOGLE_CLOUD_KEYFILE` +* `DATASTORE_KEYFILE_JSON` +* `GOOGLE_CLOUD_KEYFILE_JSON` + +For more information, see +[Authentication](https://googleapis.dev/ruby/google-cloud-datastore/latest/file.AUTHENTICATION.html). + +### Creating the Datastore indexes + +Install the [gcloud command-line +tool](https://cloud.google.com/sdk/gcloud) and use it to create the +indexes used in the tests. From the Datastore Admin V1 samples directory: + +``` sh +# Set the default project in your env +$ gcloud config set project PROJECT_ID + +# Authenticate the gcloud tool with your account +$ gcloud auth login + +# Create the indexes +$ gcloud datastore indexes create index.yaml +``` + +### Setting the Google Cloud Storage (GCS) export bucket + +Set the following environment variable to an existing Google Cloud Storage +bucket to be used in the export and import tests: `GOOGLE_CLOUD_STORAGE_BUCKET` + +### Running the tests + +```bash +$ bundle exec rake test +``` diff --git a/google-cloud-datastore-admin-v1/samples/Rakefile b/google-cloud-datastore-admin-v1/samples/Rakefile new file mode 100644 index 000000000000..e377590cbb7b --- /dev/null +++ b/google-cloud-datastore-admin-v1/samples/Rakefile @@ -0,0 +1,20 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "rake/testtask" + +Rake::TestTask.new "test" do |t| + t.test_files = FileList["**/*_test.rb"] + t.warning = false +end diff --git a/google-cloud-datastore-admin-v1/samples/acceptance/helper.rb b/google-cloud-datastore-admin-v1/samples/acceptance/helper.rb new file mode 100644 index 000000000000..95937fc4bed9 --- /dev/null +++ b/google-cloud-datastore-admin-v1/samples/acceptance/helper.rb @@ -0,0 +1,32 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "minitest/autorun" +require "minitest/focus" +require "securerandom" + +require "google/cloud/datastore/admin/v1" + +def storage_bucket_name + ENV["GOOGLE_CLOUD_STORAGE_BUCKET"] || "ruby-samples-test" +end + +def random_storage_file_prefix + "datastore-admin-v1-#{SecureRandom.hex 4}" +end + +# Returns URL to sample image in the fixtures storage bucket +def storage_url prefix: "datastore-admin-v1" + "gs://#{storage_bucket_name}/#{prefix}" +end diff --git a/google-cloud-datastore-admin-v1/samples/acceptance/snippets_test.rb b/google-cloud-datastore-admin-v1/samples/acceptance/snippets_test.rb new file mode 100644 index 000000000000..054d25f41ee4 --- /dev/null +++ b/google-cloud-datastore-admin-v1/samples/acceptance/snippets_test.rb @@ -0,0 +1,73 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative "helper" +require_relative "../snippets" + +describe "Firestore in Datastore mode Admin V1 samples" do + let(:project_id) { ENV["GOOGLE_CLOUD_PROJECT"] || raise("missing GOOGLE_CLOUD_PROJECT") } + let(:storage_file_prefix) { random_storage_file_prefix } + let(:output_url_prefix) { storage_url prefix: storage_file_prefix } + + it "client_create" do + client = client_create + assert_kind_of Google::Cloud::Datastore::Admin::V1::DatastoreAdmin::Client, client + end + + it "index_list, index_get" do + indexes = nil + out, _err = capture_io do + indexes = index_list project_id: project_id + end + assert indexes + refute_empty indexes + index = indexes.first + assert_kind_of Google::Cloud::Datastore::Admin::V1::Index, index + assert_includes out, "Got index: #{index.index_id}" + assert_includes out, "Got list of indexes" + + out, _err = capture_io do + index = index_get project_id: project_id, index_id: index.index_id + end + assert_kind_of Google::Cloud::Datastore::Admin::V1::Index, index + assert_includes out, "Got index: #{index.index_id}" + end + + it "entities_export, entities_import" do + begin + op = nil + out, _err = capture_io do + op = entities_export project_id: project_id, output_url_prefix: output_url_prefix + end + assert_includes out, "Entities were exported" + assert op + assert op.response + assert_equal "#{output_url_prefix}/#{storage_file_prefix}.overall_export_metadata", op.response.output_url + + out, _err = capture_io do + entities_import project_id: project_id, input_url: op.response.output_url + end + assert_includes out, "Entities were imported" + ensure + # cleanup: delete exported objects + require "google/cloud/storage" + storage = Google::Cloud::Storage.new + files = storage.bucket(storage_bucket_name).files prefix: storage_file_prefix + files.each do |f| + f.delete + puts "Deleted: #{f.name}" + end + end + end +end diff --git a/google-cloud-datastore-admin-v1/samples/index.yaml b/google-cloud-datastore-admin-v1/samples/index.yaml new file mode 100644 index 000000000000..c1abde9befa6 --- /dev/null +++ b/google-cloud-datastore-admin-v1/samples/index.yaml @@ -0,0 +1,23 @@ +indexes: + + - kind: Task + ancestor: no + properties: + - name: done + - name: priority + direction: desc + + - kind: Task + properties: + - name: collaborators + direction: asc + - name: created + direction: desc + + - kind: Task + ancestor: no + properties: + - name: priority + direction: asc + - name: created + direction: asc diff --git a/google-cloud-datastore-admin-v1/samples/snippets.rb b/google-cloud-datastore-admin-v1/samples/snippets.rb new file mode 100644 index 000000000000..4dff129a562c --- /dev/null +++ b/google-cloud-datastore-admin-v1/samples/snippets.rb @@ -0,0 +1,94 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License");x +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "google/cloud/datastore/admin/v1" + +def client_create + # [START datastore_admin_client_create] + # [START require_library] + # Import the client library + require "google/cloud/datastore/admin/v1" + # [END require_library] + + # Instantiate a client + client = Google::Cloud::Datastore::Admin::V1::DatastoreAdmin::Client.new + # [END datastore_admin_client_create] +end + +def entities_export project_id:, output_url_prefix: + client = Google::Cloud::Datastore::Admin::V1::DatastoreAdmin::Client.new + # [START datastore_admin_entities_export] + # project_id = "project-id" + # output_url_prefix = "gs://bucket-name" + op = client.export_entities project_id: project_id, output_url_prefix: output_url_prefix + + op.wait_until_done! + raise op.error.message if op.error? + + response = op.response + # Process the response. + + metadata = op.metadata + # Process the metadata. + + puts "Entities were exported" + # [END datastore_admin_entities_export] + op +end + +def entities_import project_id:, input_url: + client = Google::Cloud::Datastore::Admin::V1::DatastoreAdmin::Client.new + # [START datastore_admin_entities_import] + # project_id = "project-id" + # input_url = "gs://bucket-name/overall-export-metadata-file" + op = client.import_entities project_id: project_id, input_url: input_url + + op.wait_until_done! + raise op.error.message if op.error? + + response = op.response + # Process the response. + + metadata = op.metadata + # Process the metadata. + + puts "Entities were imported" + # [END datastore_admin_entities_import] + op +end + +def index_get project_id:, index_id: + client = Google::Cloud::Datastore::Admin::V1::DatastoreAdmin::Client.new + # [START datastore_admin_index_get] + # project_id = "project-id" + # index_id = "my-index" + index = client.get_index project_id: project_id, index_id: index_id + puts "Got index: #{index.index_id}" + # [END datastore_admin_index_get] + index +end + +def index_list project_id: + client = Google::Cloud::Datastore::Admin::V1::DatastoreAdmin::Client.new + # [START datastore_admin_index_list] + # project_id = "project-id" + indexes = client.list_indexes(project_id: project_id).map do |index| + puts "Got index: #{index.index_id}" + index + end + + puts "Got list of indexes" + # [END datastore_admin_index_list] + indexes +end diff --git a/google-cloud-datastore/Rakefile b/google-cloud-datastore/Rakefile index 38d525580273..a40a774b5daa 100644 --- a/google-cloud-datastore/Rakefile +++ b/google-cloud-datastore/Rakefile @@ -31,26 +31,7 @@ end # Acceptance tests desc "Run the datastore acceptance tests." task :acceptance, :project, :keyfile do |t, args| - project = args[:project] - project ||= ENV["DATASTORE_TEST_PROJECT"] || ENV["GCLOUD_TEST_PROJECT"] - keyfile = args[:keyfile] - keyfile ||= ENV["DATASTORE_TEST_KEYFILE"] || ENV["GCLOUD_TEST_KEYFILE"] - if keyfile - keyfile = File.read keyfile - else - keyfile ||= ENV["DATASTORE_TEST_KEYFILE_JSON"] || ENV["GCLOUD_TEST_KEYFILE_JSON"] - end - if project.nil? || keyfile.nil? - fail "You must provide a project and keyfile. e.g. rake acceptance[test123, /path/to/keyfile.json] or DATASTORE_TEST_PROJECT=test123 DATASTORE_TEST_KEYFILE=/path/to/keyfile.json rake acceptance" - end - # clear any env var already set - require "google/cloud/datastore/credentials" - Google::Cloud::Datastore::Credentials.env_vars.each do |path| - ENV[path] = nil - end - # always overwrite when running tests - ENV["DATASTORE_PROJECT"] = project - ENV["DATASTORE_KEYFILE_JSON"] = keyfile + set_env_vars project: args[:project], keyfile: args[:keyfile] Rake::Task["acceptance:run"].invoke end @@ -69,7 +50,26 @@ namespace :acceptance do end desc "Run acceptance cleanup." - task :cleanup do + task :cleanup, :project, :keyfile do |t, args| + set_env_vars project: args[:project], keyfile: args[:keyfile] + + $LOAD_PATH.unshift "lib" + require "google/cloud/datastore" + + kinds = %w(Character Company NeverUsedBeforeTask Person Post) + puts "cleanup: start" + datastore = Google::Cloud::Datastore.new + + kinds.each do |kind| + query = Google::Cloud::Datastore::Query.new.kind(kind).select("__key__") + loop do + keys = datastore.run(query).map(&:key) + puts "deleting #{kind} entities: #{keys.count}" + break if keys.empty? + datastore.delete keys + end + end + puts "cleanup: done" end Rake::TestTask.new :run do |t| @@ -169,3 +169,24 @@ def header str, token = "#" puts token * line_length puts "" end + +def set_env_vars project: nil, keyfile: nil + project ||= ENV["DATASTORE_TEST_PROJECT"] || ENV["GCLOUD_TEST_PROJECT"] + keyfile ||= ENV["DATASTORE_TEST_KEYFILE"] || ENV["GCLOUD_TEST_KEYFILE"] + if keyfile + keyfile = File.read keyfile + else + keyfile ||= ENV["DATASTORE_TEST_KEYFILE_JSON"] || ENV["GCLOUD_TEST_KEYFILE_JSON"] + end + if project.nil? || keyfile.nil? + fail "You must provide a project and keyfile. e.g. rake acceptance[test123, /path/to/keyfile.json] or DATASTORE_TEST_PROJECT=test123 DATASTORE_TEST_KEYFILE=/path/to/keyfile.json rake acceptance" + end + # clear any env var already set + require "google/cloud/datastore/credentials" + Google::Cloud::Datastore::Credentials.env_vars.each do |path| + ENV[path] = nil + end + # always overwrite when running tests + ENV["DATASTORE_PROJECT"] = project + ENV["DATASTORE_KEYFILE_JSON"] = keyfile +end diff --git a/google-cloud-datastore/samples/acceptance/sample_test.rb b/google-cloud-datastore/samples/acceptance/sample_test.rb index 1b6bcd7f6777..e2733cc63d91 100644 --- a/google-cloud-datastore/samples/acceptance/sample_test.rb +++ b/google-cloud-datastore/samples/acceptance/sample_test.rb @@ -219,7 +219,11 @@ tasks = datastore.run query refute tasks.empty? - tasks.each { |t| assert_basic_task t } + tasks.each do |t| + assert_equal "Task", t.key.kind + assert_equal false, t["done"] + assert_equal 4, t["priority"] + end end it "supports key_filter" do @@ -227,7 +231,9 @@ tasks = datastore.run query refute tasks.empty? - tasks.each { |t| assert_basic_task t } + tasks.each do |t| + assert_equal "Task", t.key.kind + end end it "supports ascending_sort" do