test: add integration test for builtin metrics

README.md

@@ -56,13 +56,13 @@ implementation 'com.google.cloud:google-cloud-bigtable'
 If you are using Gradle without BOM, add this to your dependencies:
 
 ```Groovy
-implementation 'com.google.cloud:google-cloud-bigtable:2.10.3'
+implementation 'com.google.cloud:google-cloud-bigtable:2.11.0'
 ```
 
 If you are using SBT, add this to your dependencies:
 
 ```Scala
-libraryDependencies += "com.google.cloud" % "google-cloud-bigtable" % "2.10.3"
+libraryDependencies += "com.google.cloud" % "google-cloud-bigtable" % "2.11.0"
 ```
 
 ## Authentication

google-cloud-bigtable-bom/pom.xml

@@ -9,6 +9,7 @@
         <groupId>com.google.cloud</groupId>
         <artifactId>google-cloud-shared-config</artifactId>
         <version>1.5.1</version>
+        <relativePath/>
     </parent>
 
     <name>Google Cloud Bigtable BOM</name>

google-cloud-bigtable-deps-bom/pom.xml

@@ -8,6 +8,7 @@
     <groupId>com.google.cloud</groupId>
     <artifactId>google-cloud-shared-config</artifactId>
     <version>1.5.1</version>
+    <relativePath/>
   </parent>
 
   <groupId>com.google.cloud</groupId>

google-cloud-bigtable-stats/pom.xml

@@ -59,7 +59,7 @@
         <dependency>
             <groupId>com.google.cloud</groupId>
             <artifactId>google-cloud-monitoring</artifactId>
-            <!-- Excluding http dependencies since they're not used and are causing conflict in google-cloud-bigtable -->
+            <!-- Excluding unused transitive dependencies to avoid conflict in google-cloud-bigtable -->
             <exclusions>
                 <exclusion>
                     <groupId>com.google.http-client</groupId>

google-cloud-bigtable/pom.xml

@@ -54,17 +54,25 @@
         <type>pom</type>
         <scope>import</scope>
       </dependency>
+      <dependency>
+        <groupId>com.google.cloud</groupId>
+        <artifactId>google-cloud-monitoring-bom</artifactId>
+        <version>3.4.1</version> <!-- needs to be in sync with google-cloud-bigtable-stats -->
+        <type>pom</type>
+        <scope>import</scope>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 
   <dependencies>
     <!-- NOTE: Dependencies are organized into two groups, production and test.
          Within a group, dependencies are sorted by (groupId, artifactId) -->
-
     <dependency>
       <groupId>com.google.cloud</groupId>
       <artifactId>google-cloud-bigtable-stats</artifactId>
-      <!-- Exclude all the shaded transitive dependencies -->
+      <!-- Exclude all dependencies that have been shaded. This is to workaround maven's immutable project structure:
+           after shading, the maven-shade-plugin tries to remove shaded dependencies, but it can't since the project
+           structure is immutable. So we have to manually exclude the shaded transitive dependencies manually. -->
       <exclusions>
         <exclusion>
           <groupId>io.opencensus</groupId>
@@ -254,6 +262,24 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <!-- cloud-monitoring client depends on grpc-xds, which has transitive dependency on opencensus-proto. We need to
+    use cloud-monitoring as a test dependency, which also marks the opencensus-proto as a test dependency. Promote
+    grpc-xds to a runtime dependency so that opencensus-proto will still be a runtime dependency in the flattened pom. -->
+    <dependency>
+      <groupId>io.grpc</groupId>
+      <artifactId>grpc-xds</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-monitoring</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.api.grpc</groupId>
+      <artifactId>proto-google-cloud-monitoring-v3</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>com.google.truth</groupId>
       <artifactId>truth</artifactId>

google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java

@@ -0,0 +1,137 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.bigtable.data.v2.it;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.TruthJUnit.assume;
+
+import com.google.api.client.util.Lists;
+import com.google.cloud.bigtable.data.v2.BigtableDataSettings;
+import com.google.cloud.bigtable.data.v2.models.Query;
+import com.google.cloud.bigtable.data.v2.models.Row;
+import com.google.cloud.bigtable.data.v2.models.RowMutation;
+import com.google.cloud.bigtable.test_helpers.env.EmulatorEnv;
+import com.google.cloud.bigtable.test_helpers.env.TestEnvRule;
+import com.google.cloud.monitoring.v3.MetricServiceClient;
+import com.google.monitoring.v3.ListTimeSeriesRequest;
+import com.google.monitoring.v3.ListTimeSeriesResponse;
+import com.google.monitoring.v3.ProjectName;
+import com.google.monitoring.v3.TimeInterval;
+import com.google.protobuf.util.Timestamps;
+import java.io.IOException;
+import java.time.Duration;
+import java.util.ArrayList;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class BuiltinMetricsIT {
+  @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule();
+  public static MetricServiceClient metricClient;
+
+  public static String[] VIEWS = {
+    "operation_latencies",
+    "retry_count",
+    "attempt_latencies",
+    "server_latencies",
+    "connectivity_error_count",
+    "application_latencies"
+  };
+
+  @BeforeClass
+  public static void setUpClass() throws IOException {
+    assume()
+        .withMessage("Builtin metrics integration test is not supported by emulator")
+        .that(testEnvRule.env())
+        .isNotInstanceOf(EmulatorEnv.class);
+
+    // Enable built in metrics
+    BigtableDataSettings.enableBuiltinMetrics();
+
+    // Create a cloud monitoring client
+    metricClient = MetricServiceClient.create();
+  }
+
+  @AfterClass
+  public static void tearDown() {
+    if (metricClient != null) {
+      metricClient.close();
+    }
+  }
+
+  @Test
+  public void testBuiltinMetrics() throws Exception {
+    // Send a MutateRow and ReadRows request
+    testEnvRule
+        .env()
+        .getDataClient()
+        .mutateRow(
+            RowMutation.create(testEnvRule.env().getTableId(), "a-new-key")
+                .setCell(testEnvRule.env().getFamilyId(), "q", "abc"));
+    ArrayList<Row> rows =
+        Lists.newArrayList(
+            testEnvRule
+                .env()
+                .getDataClient()
+                .readRows(Query.create(testEnvRule.env().getTableId()).limit(10)));
+
+    // Sleep 5 minutes so the metrics could be published and precomputation is done
+    Thread.sleep(Duration.ofMinutes(5).toMillis());
+
+    ProjectName name = ProjectName.of(testEnvRule.env().getProjectId());
+
+    // Restrict time to last 10 minutes
+    long startMillis = System.currentTimeMillis() - Duration.ofMinutes(10).toMillis();
+    TimeInterval interval =
+        TimeInterval.newBuilder()
+            .setStartTime(Timestamps.fromMillis(startMillis))
+            .setEndTime(Timestamps.fromMillis(System.currentTimeMillis()))
+            .build();
+
+    for (String view : VIEWS) {
+      // Filter on instance and method name
+      // Verify that metrics are published for MutateRow request
+      String metricFilter =
+          String.format(
+              "metric.type=\"bigtable.googleapis.com/client/%s\" "
+                  + "AND resource.labels.instance=\"%s\" AND metric.labels.method=\"Bigtable.MutateRow\"",
+              view, testEnvRule.env().getInstanceId());
+      ListTimeSeriesRequest.Builder requestBuilder =
+          ListTimeSeriesRequest.newBuilder()
+              .setName(name.toString())
+              .setFilter(metricFilter)
+              .setInterval(interval)
+              .setView(ListTimeSeriesRequest.TimeSeriesView.FULL);
+      ListTimeSeriesResponse response =
+          metricClient.listTimeSeriesCallable().call(requestBuilder.build());
+      assertThat(response.getTimeSeriesCount()).isGreaterThan(0);
+
+      // Verify that metrics are published for ReadRows request
+      metricFilter =
+          String.format(
+              "metric.type=\"bigtable.googleapis.com/client/operation_latencies\" "
+                  + "AND resource.labels.instance=\"%s\" AND metric.labels.method=\"Bigtable.ReadRows\"",
+              testEnvRule.env().getInstanceId());
+      requestBuilder.setFilter(metricFilter);
+      response = metricClient.listTimeSeriesCallable().call(requestBuilder.build());
+      assertThat(response.getTimeSeriesCount()).isGreaterThan(0);
+    }
+  }
+}

pom.xml

@@ -15,6 +15,7 @@
         <groupId>com.google.cloud</groupId>
         <artifactId>google-cloud-shared-config</artifactId>
         <version>1.5.3</version>
+        <relativePath/>
     </parent>
 
     <developers>
@@ -339,6 +340,22 @@
                 </plugins>
             </build>
         </profile>
+
+        <profile>
+            <!-- Shading workaround for IDEs: This project relocates some of its internal dependencies. Since IDEs can't
+             resolve the relocated symbols on their own, a workaround is added: first mvn install the shaded module,
+             then disable the with-shaded profile to force IDEs to resolve the relocated dependencies from the local
+             repository -->
+            <id>with-shaded</id>
+            <activation>
+                <property>
+                    <name>!skip-shaded</name>
+                </property>
+            </activation>
+            <modules>
+                <module>google-cloud-bigtable-stats</module>
+            </modules>
+        </profile>
     </profiles>
 
     <modules>
@@ -347,7 +364,6 @@
     <module>grpc-google-cloud-bigtable-v2</module>
     <module>proto-google-cloud-bigtable-admin-v2</module>
     <module>proto-google-cloud-bigtable-v2</module>
-    <module>google-cloud-bigtable-stats</module>
     <module>google-cloud-bigtable-emulator-core</module>
     <module>google-cloud-bigtable-emulator</module>
     <module>google-cloud-bigtable-bom</module>